diff --git a/.codex/skills/builder-interaction-formal-verification/SKILL.md b/.codex/skills/builder-interaction-formal-verification/SKILL.md index 2ef95d34e..714bd144b 100644 --- a/.codex/skills/builder-interaction-formal-verification/SKILL.md +++ b/.codex/skills/builder-interaction-formal-verification/SKILL.md @@ -427,7 +427,7 @@ This checkpoint exercises actual Arango transactions and public service behavior ## Draft save transaction protocol -Use `node scripts/verify-builder-formal-core.mjs --model draft-transaction --out /tmp/loom-draft-transaction-review --self-test` after changes to `internal/explorer/arango/drafts.go`. Read the result with pstack `/statechart`. The existing statechart frontend lowers checked TLC evidence through its IR into the Mermaid chart; the project adds reader labels, not a second TLA compiler. Use the module's `--witness FILE [SCRATCH_DRAFTS_SOURCE]` command to replay a saved counterexample against production SaveDraft. Pristine source must hold; the matching scratch source fault must reproduce it. +Use `node scripts/verify-builder-formal-core.mjs --model draft-transaction --out /tmp/loom-draft-transaction-review --self-test` after changes to `internal/backends/arango/internal/explorer/drafts.go`. Read the result with pstack `/statechart`. The existing statechart frontend lowers checked TLC evidence through its IR into the Mermaid chart; the project adds reader labels, not a second TLA compiler. Use the module's `--witness FILE [SCRATCH_DRAFTS_SOURCE]` command to replay a saved counterexample against production SaveDraft. Pristine source must hold; the matching scratch source fault must reproduce it. `DraftSaveTransaction` models one save of one existing draft, one immutable previous revision, and the three ordered query boundaries: read expected head, INSERT previous revision, guarded UPDATE head. Begin, each query callback, abort, commit and return are separate protocol actions. Finite outcomes include missing CAS rows, typed1200, other query errors, confirmed commit rejection and publication followed by a lost acknowledgement. Actual Go replay calls production Store.SaveDraft, observes the RowVisitor callbacks, checks literal expected query bindings and independently expected old/new workspace bytes, and checks final domain error classification with backend cause retained. diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c00cf72d5..6e1f6837f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -29,7 +29,6 @@ jobs: build-and-push: name: Build and push ${{ matrix.arch }} image needs: tests - # Never expose Quay credentials to code from a forked pull request. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'ubuntu-24.04-arm' }} strategy: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index bc1ba433f..e4042dbd5 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -18,6 +18,8 @@ jobs: - name: Install UI dependencies working-directory: ui run: npm ci --ignore-scripts --no-audit --no-fund + - name: No-code-comments gate + run: python3 scripts/check_no_code_comments.py --all-tracked - name: Check reviewed UI source checkpoint run: python3 scripts/verify_ui_contract_inventory.py --source-only - name: Check UI boundaries @@ -51,6 +53,34 @@ jobs: with: go-version: 1.26.5 cache: true + - name: Install ripgrep for the boundary check + run: | + if ! command -v rg >/dev/null 2>&1; then + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends ripgrep + fi + rg --version | head -1 + - name: Run Go package boundary check + run: bash scripts/check_dataframe_package_boundaries.sh + - name: Backend ownership guard named gates + run: | + set -o pipefail + go test -json -count=1 -timeout=600s ./internal/dataframe/compiler \ + -run '^(TestBackendOwnershipGuardContract|TestCompilerProductionFilesNeverImportBackendsOrDrivers)$' \ + | node scripts/lib/check-go-test-events.mjs \ + TestBackendOwnershipGuardContract \ + TestBackendOwnershipGuardContract/compiler_backends_and_drivers \ + TestBackendOwnershipGuardContract/neutral_domain_backend_import \ + TestBackendOwnershipGuardContract/renderer_runtime_import \ + TestBackendOwnershipGuardContract/legitimate_adapter_domain_import \ + TestBackendOwnershipGuardContract/actual_relocated_adapter_import \ + TestBackendOwnershipGuardContract/missing_or_malformed_input_is_error \ + TestBackendOwnershipGuardContract/facade_law_control_plane \ + TestCompilerProductionFilesNeverImportBackendsOrDrivers \ + TestCompilerProductionFilesNeverImportBackendsOrDrivers/non_vacuity_named_victims_bleed \ + TestCompilerProductionFilesNeverImportBackendsOrDrivers/non_vacuity_fires_on_a_real_postgres_adapter \ + TestCompilerProductionFilesNeverImportBackendsOrDrivers/non_vacuity_fires_on_a_real_arango_adapter \ + TestCompilerProductionFilesNeverImportBackendsOrDrivers/non_vacuity_malformed_input_is_verification_failure - name: Run all ordinary Go packages and catalog witnesses run: | package_list=$(go list . ./internal/... ./cmd/... ./generated/... ./conformance/...) @@ -217,6 +247,251 @@ jobs: ./internal/dataframe/published ./internal/store/clickhouse + pg-fixture: + name: PostgreSQL 18 opt-in fixture runner + runs-on: ubuntu-latest + timeout-minutes: 10 + services: + postgres18: + image: postgres:18 + env: + POSTGRES_DB: loom_test + POSTGRES_USER: loom_test + POSTGRES_PASSWORD: loom_ci_test_only + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -h 127.0.0.1 -U loom_test -d loom_test" + --health-interval 3s + --health-timeout 5s + --health-retries 40 + env: + LOOM_TEST_POSTGRES_URL: postgresql://loom_test:loom_ci_test_only@127.0.0.1:5432/loom_test?sslmode=disable + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.26.5 + cache: true + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 24.x + - name: Run fixture runner coverage without any service + run: timeout 180s node --test scripts/run-postgres-integration.test.mjs + - name: Run owned-namespace real PostgreSQL acceptance + run: timeout 540s node scripts/run-postgres-integration.mjs --out "$RUNNER_TEMP/pg-fixture" + - name: Run the post-window live reducer leg on PostgreSQL 18 + run: | + set -euo pipefail + go test -timeout=240s -count=1 -v \ + -run 'TestPostWindowLive' \ + ./internal/dataframe/compiler/render/postgres \ + | tee "$RUNNER_TEMP/pg-post-window-live.log" + - name: Assert the post-window live leg executed and never skipped + run: | + set -euo pipefail + log="$RUNNER_TEMP/pg-post-window-live.log" + grep -q -- '--- PASS: TestPostWindowLiveReducersExecuteWithPredicates' "$log" + if grep -q -- '--- SKIP: TestPostWindowLiveReducersExecuteWithPredicates' "$log"; then + echo '::error/post-window live reducer leg skipped despite the PostgreSQL 18 service being configured' + exit 1 + fi + - name: Preserve fixture evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: pg-fixture + path: ${{ runner.temp }}/pg-fixture + if-no-files-found: warn + + pg-repository-publish: + name: PostgreSQL repository-publish spine across PostgreSQL 18 and ClickHouse + runs-on: ubuntu-latest + timeout-minutes: 10 + services: + postgres18: + image: postgres:18 + env: + POSTGRES_DB: loom_test + POSTGRES_USER: loom_test + POSTGRES_PASSWORD: loom_ci_test_only + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -h 127.0.0.1 -U loom_test -d loom_test" + --health-interval 3s + --health-timeout 5s + --health-retries 40 + clickhouse: + image: clickhouse/clickhouse-server:25.3 + env: + CLICKHOUSE_DB: loom_test + CLICKHOUSE_USER: loom_ci + CLICKHOUSE_PASSWORD: loom_ci_password + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" + ports: + - 9000:9000 + - 8123:8123 + options: >- + --health-cmd "wget --no-verbose --tries=1 --spider http://127.0.0.1:8123/ping || exit 1" + --health-interval 5s + --health-timeout 3s + --health-retries 20 + env: + LOOM_TEST_POSTGRES_URL: postgresql://loom_test:loom_ci_test_only@127.0.0.1:5432/loom_test?sslmode=disable + LOOM_CLICKHOUSE_URL: clickhouse://127.0.0.1:9000 + LOOM_CLICKHOUSE_DATABASE: loom_test + LOOM_CLICKHOUSE_USERNAME: loom_ci + LOOM_CLICKHOUSE_PASSWORD: loom_ci_password + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.26.5 + cache: true + - name: Run the pgComp battery with the repository-publish spine leg + run: | + set -euo pipefail + go test -timeout=8m -count=1 -v \ + -run 'TestPostgresRuntime|TestPostgreSQL|TestHealthWiringPerCompositionArm|TestBackendCompositionContract|TestZZPgCompZeroLeftoverFixtures' \ + ./internal/server \ + | tee "$RUNNER_TEMP/pg-repository-publish.log" + - name: Assert the spine leg executed and never skipped + run: | + set -euo pipefail + log="$RUNNER_TEMP/pg-repository-publish.log" + grep -q -- '--- PASS: TestPostgresRuntimeRepositoryPublishRouteActivatesThroughFinalFlip' "$log" + if grep -q -- '--- SKIP: TestPostgresRuntimeRepositoryPublishRouteActivatesThroughFinalFlip' "$log"; then + echo '::error/repository-publish spine leg skipped despite both live services being configured' + exit 1 + fi + grep -q -- '--- PASS: TestPostgresRuntimeRepositoryPublishRetryAfterActivationFailureRevives' "$log" + if grep -q -- '--- SKIP: TestPostgresRuntimeRepositoryPublishRetryAfterActivationFailureRevives' "$log"; then + echo '::error/repository-publish retry-revival leg skipped despite both live services being configured' + exit 1 + fi + - name: Preserve spine evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: pg-repository-publish + path: ${{ runner.temp }}/pg-repository-publish.log + if-no-files-found: warn + + backend-conformance: + name: Backend architecture conformance across Arango and PostgreSQL + runs-on: ubuntu-latest + timeout-minutes: 10 + services: + arangodb: + image: arangodb:3.12 + env: + ARANGO_NO_AUTH: '1' + ports: + - 8529:8529 + options: >- + --health-cmd "wget --quiet --tries=1 --spider http://127.0.0.1:8529/_api/version" + --health-interval 3s + --health-timeout 5s + --health-retries 40 + postgres18: + image: postgres:18 + env: + POSTGRES_DB: loom_test + POSTGRES_USER: loom_test + POSTGRES_PASSWORD: loom_ci_test_only + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -h 127.0.0.1 -U loom_test -d loom_test" + --health-interval 3s + --health-timeout 5s + --health-retries 40 + env: + LOOM_TEST_ARANGO_URL: http://127.0.0.1:8529 + LOOM_TEST_POSTGRES_URL: postgresql://loom_test:loom_ci_test_only@127.0.0.1:5432/loom_test?sslmode=disable + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.26.5 + cache: true + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 24.x + - name: Verify the offline skip-set capture against the frozen allowlist + run: env -u LOOM_TEST_ARANGO_URL -u LOOM_TEST_POSTGRES_URL node scripts/backend-conformance-skipset-verify.mjs + - name: Run both conformance arms against the actual services + run: | + set -o pipefail + mkdir -p "$RUNNER_TEMP/backend-conformance" + status=0 + timeout 480s go test -json -count=1 -timeout=420s ./internal/server \ + -run '^TestBackendArchitectureConformance$' \ + > "$RUNNER_TEMP/backend-conformance/conformance.jsonl" || status=$? + node scripts/lib/check-go-test-events.mjs \ + TestBackendArchitectureConformance \ + TestBackendArchitectureConformance/arangodb \ + TestBackendArchitectureConformance/arangodb/scalarQueryServesLiteralScalars \ + TestBackendArchitectureConformance/arangodb/repeatedRelatedOccurrenceStaysDistinct \ + TestBackendArchitectureConformance/arangodb/nullAndMissingSemanticsPreserved \ + TestBackendArchitectureConformance/arangodb/explicitScopeServesAuthorizedAndExcludesUnauthorized \ + TestBackendArchitectureConformance/arangodb/restrictedEmptyScopeServesNothing \ + TestBackendArchitectureConformance/arangodb/unrestrictedSeesEverything \ + TestBackendArchitectureConformance/arangodb/generationIsolationServesActiveOnly \ + TestBackendArchitectureConformance/arangodb/stablePagingIsDeterministic \ + TestBackendArchitectureConformance/arangodb/explorerDraftCASOneWinnerExactPreimage \ + TestBackendArchitectureConformance/arangodb/publicationControlFenceRejectsStaleOwner \ + TestBackendArchitectureConformance/arangodb/cancellationTypedFailureNoFallback \ + TestBackendArchitectureConformance/arangodb/backendUnavailableAndWrongTagNoFallback \ + TestBackendArchitectureConformance/arangodb/upgradeFromPinnedPreRefactorReopensSafely \ + TestBackendArchitectureConformance/postgresql \ + TestBackendArchitectureConformance/postgresql/scalarQueryServesLiteralScalars \ + TestBackendArchitectureConformance/postgresql/repeatedRelatedOccurrenceStaysDistinct \ + TestBackendArchitectureConformance/postgresql/nullAndMissingSemanticsPreserved \ + TestBackendArchitectureConformance/postgresql/explicitScopeServesAuthorizedAndExcludesUnauthorized \ + TestBackendArchitectureConformance/postgresql/restrictedEmptyScopeServesNothing \ + TestBackendArchitectureConformance/postgresql/unrestrictedSeesEverything \ + TestBackendArchitectureConformance/postgresql/generationIsolationServesActiveOnly \ + TestBackendArchitectureConformance/postgresql/stablePagingIsDeterministic \ + TestBackendArchitectureConformance/postgresql/explorerDraftCASOneWinnerExactPreimage \ + TestBackendArchitectureConformance/postgresql/publicationControlFenceRejectsStaleOwner \ + TestBackendArchitectureConformance/postgresql/cancellationTypedFailureNoFallback \ + TestBackendArchitectureConformance/postgresql/backendUnavailableAndWrongTagNoFallback \ + TestBackendArchitectureConformance/postgresql/upgradeFromPinnedPreRefactorReopensSafely \ + TestBackendArchitectureConformance/crossBackendCandidateIdentityParity \ + < "$RUNNER_TEMP/backend-conformance/conformance.jsonl" || status=$? + # The gate fails closed on events alone; surface the failing tests' + # own output so a red lane is diagnosable from this step's log. + node -e ' + const fs = require("node:fs"); + const events = fs.readFileSync(process.argv[1], "utf8").split("\n").filter(Boolean).map(JSON.parse); + const outputs = new Map(); + const failed = []; + for (const e of events) { + if (e.Test && e.Action === "output") outputs.set(e.Test, [...(outputs.get(e.Test) ?? []), e.Output]); + if (e.Action === "fail" && e.Test) failed.push(e.Test); + } + for (const name of failed) for (const line of outputs.get(name) ?? []) process.stdout.write(line); + if (failed.length === 0 && events.some(e => e.Action === "fail")) console.error("package-level failure with no named failing test; see conformance.jsonl artifact"); + ' "$RUNNER_TEMP/backend-conformance/conformance.jsonl" || true + exit $status + - name: Preserve conformance evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: backend-conformance + path: ${{ runner.temp }}/backend-conformance + if-no-files-found: warn + node-unit: name: Node unit tests runs-on: ubuntu-latest @@ -239,9 +514,6 @@ jobs: working-directory: ui run: npm ci --ignore-scripts --no-audit --no-fund - name: Record executable routes and fail on a broken retained closure - # --out writes the document only after the full inventory succeeds, via an - # atomic rename, so a broken retained dependency closure fails without - # leaving a truncated or stale file for the always() upload to publish. run: node scripts/verify-builder-formal-core.mjs --phase inventory --out "$RUNNER_TEMP" - name: Preserve verification inventory if: always() @@ -281,8 +553,14 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 22.x + cache: npm + cache-dependency-path: ui/package-lock.json + - name: Install UI dependencies + working-directory: ui + run: npm ci --ignore-scripts --no-audit --no-fund - name: Run retained Python tests run: | + export LOOM_TYPESCRIPT_DIR="$GITHUB_WORKSPACE/ui/node_modules/typescript" python3 -m unittest discover -s scripts/tests python3 -m unittest scripts.test_validate_architecture_plan diff --git a/.gitignore b/.gitignore index 51f1c4ae2..4fa0113e2 100644 --- a/.gitignore +++ b/.gitignore @@ -29,10 +29,8 @@ META/ /ui/apps/*/dist/ /ui/packages/*/dist/ -# Explicit machine-local native CDA target configuration. /.codex/owned-cda-target.json -# Legacy locations of generated verification reports. Current outputs use .cache/. /verification/builder-interactions/test-obligations.json /verification/builder-interactions/test-obligations.md /verification/builder-interactions/route-action-proposals.json @@ -51,3 +49,5 @@ META/ /verification/builder-interactions/report.md /verification/builder-interactions/http-500-audit.json /docs/verification/playwright/source-conversion-manifest.md + +/loom-data diff --git a/.gitnexusignore b/.gitnexusignore index 02d5640c2..8cc0ca87b 100644 --- a/.gitnexusignore +++ b/.gitnexusignore @@ -1,4 +1,3 @@ -# Local datasets, analysis artifacts, and agent tooling are not application code. /CDA-FHIR/ /META_SMALL/ /.audit/ @@ -7,7 +6,6 @@ /loom /loom-DocumentReference.csv -# Keep generated API contracts, tests, fixtures, and CI in the architecture graph. !/generated/ !__tests__/ !__mocks__/ diff --git a/Dockerfile.dev b/Dockerfile.dev index 162ca4018..a37aa37bb 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,5 +1,3 @@ -# Development-only API image. Source is mounted by compose.dev.yaml so saves -# rebuild the binary in Air instead of rebuilding this image. FROM golang:1.26.5-bookworm RUN apt-get update \ diff --git a/Makefile b/Makefile index ec8160707..3a4df77fc 100644 --- a/Makefile +++ b/Makefile @@ -18,8 +18,6 @@ DATAFRAME_VARIABLES ?= examples/meta_gdc_case_matrix.variables.json DATAFRAME_PROFILE_VARIABLES ?= examples/meta_gdc_case_matrix.variables.json DATAFRAME_PROFILE_LIMIT ?= 1000 -# Make treats the positional version as a second goal. Reject every ambiguous -# form while parsing so no release recipe can start with the wrong version. ifneq ($(filter release-ui,$(MAKECMDGOALS)),) ifneq ($(firstword $(MAKECMDGOALS)),release-ui) $(error Usage: make release-ui X.Y.Z) @@ -93,14 +91,10 @@ compiler-bench: mkdir -p $(GOCACHE_DIR) GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=$(GO_TOOLCHAIN) $(GO) test $(GOFLAGS) ./conformance/compiler -run '^$$' -bench '^BenchmarkCompilerOracle$$' -benchmem -# Requires `arango-fhir-server --no-auth` and a loaded META project. Prints the -# actual GraphQL dataframe response and per-request wall-clock timings. dataframe-demo: mkdir -p $(GOCACHE_DIR) GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=$(GO_TOOLCHAIN) $(GO) run ./cmd/dataframe-query -url $(GRAPHQL_URL) -query $(DATAFRAME_QUERY) -variables $(DATAFRAME_VARIABLES) -repeat $(DATAFRAME_REPEAT) -limit $(DATAFRAME_LIMIT) -timeout $(DATAFRAME_TIMEOUT) -print-response=$(DATAFRAME_PRINT_RESPONSE) -# Requires a loaded META fixture database. Compiles the checked-in GDC fixture, -# writes exact rendered AQL, then runs Arango EXPLAIN and PROFILE 2. dataframe-profile: mkdir -p $(GOCACHE_DIR) GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=$(GO_TOOLCHAIN) $(GO) run ./cmd/dataframe-profile -variables $(DATAFRAME_PROFILE_VARIABLES) -limit $(DATAFRAME_PROFILE_LIMIT) @@ -122,9 +116,6 @@ docker-build: docker-run: docker run --rm -p 8080:8080 $(IMAGE) -# Full real-data acceptance path. Rebuild the canonical loom-demo deployment -# without replacing its data, then verify the locked fixture in an isolated -# Compose project that is removed after evidence capture. acceptance-real: ./scripts/acceptance-real.sh @@ -146,8 +137,6 @@ demo-browser-smoke: repository-up: ./scripts/loom-repo-up.sh --repository "$(if $(REPOSITORY),$(REPOSITORY),$(CURDIR))" -# Development-only Compose target. It has its own project, volumes, ports, and -# fixture. Use dev-rebuild after dependency or toolchain changes. dev: node scripts/loom-dev.mjs dev @@ -164,7 +153,6 @@ dev-down: clean: rm -rf bin -# Builder checkpoints. Full repository coverage remains in tests.yaml. VERIFY_FEATURE ?= ownership API_FEATURE ?= all @@ -185,7 +173,6 @@ verify-current: verify-fast verify-api verify-formal verify-full: verify-api verify-formal CI=true node scripts/verify-builder-formal-core.mjs --phase behavior -# Real Chromium component boundary, deliberately outside the fast default loop. BROWSER_FEATURE ?= all verify-browser: node scripts/verify-builder-formal-core.mjs --phase browser --feature $(BROWSER_FEATURE) diff --git a/README.md b/README.md index 04dbc93af..60264f6f4 100644 --- a/README.md +++ b/README.md @@ -337,7 +337,7 @@ without rebuilding the server image. | [`internal/dataframe/recipe`](internal/dataframe/recipe) | Recipe contract, validation, schema resolution, execution, and control services. | | [`internal/dataframe/publication`](internal/dataframe/publication) | Backend-neutral bounded streaming publication contract. | | [`internal/dataframe/published`](internal/dataframe/published) | Safe single-project published-data reads and aggregates. | -| [`internal/store/arango`](internal/store/arango) | ArangoDB boundary. | +| [`internal/backends/arango`](internal/backends/arango) | ArangoDB backend factory; adapters under its `internal/` tree. | | [`internal/store/clickhouse`](internal/store/clickhouse) | Typed ClickHouse driver boundary and DDL/DML. | | [`internal/api/graphql/graph`](internal/api/graphql/graph) | GraphQL HTTP transport and error presentation. | | [`internal/api/graphql/graph/query`](internal/api/graphql/graph/query) | Arango graph and FHIR dataframe API services. | diff --git a/cmd/arango-fhir-proto/main.go b/cmd/arango-fhir-proto/main.go index 388cbb55d..e9d10d1b7 100644 --- a/cmd/arango-fhir-proto/main.go +++ b/cmd/arango-fhir-proto/main.go @@ -16,15 +16,11 @@ import ( "strings" "syscall" - driver "github.com/arangodb/go-driver/v2/arangodb" - "github.com/arangodb/go-driver/v2/connection" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/catalog" - catalogarango "github.com/calypr/loom/internal/catalog/arango" publication "github.com/calypr/loom/internal/dataset" - publicationarango "github.com/calypr/loom/internal/dataset/arango" fhirschema "github.com/calypr/loom/internal/fhir/schema" "github.com/calypr/loom/internal/ingest" - arangostore "github.com/calypr/loom/internal/store/arango" ) const ( @@ -62,8 +58,6 @@ func main() { } } -// commandRoutes is the single dispatch table shared by main() and its tests so -// that command wiring stays observable without executing os.Exit. func commandRoutes() map[string]func(context.Context, []string) error { return map[string]func(context.Context, []string) error{ "load": runLoad, @@ -79,10 +73,6 @@ func commandRoutes() map[string]func(context.Context, []string) error { } } -// signalGuardedCommands lists the commands that mutate a dataset lifecycle -// manifest or an immutable generation. A cancelled run there can leave a -// half-mutated manifest, so main() runs them under a context that cancels on -// SIGINT/SIGTERM. Read and mutable-data commands stay unwrapped. func signalGuardedCommands() map[string]bool { return map[string]bool{ "load-generation": true, @@ -143,9 +133,6 @@ func parseLoadMutableCommand(args []string, errorHandling flag.ErrorHandling) (l if err := config.parseMemoryLimit(); err != nil { return loadCommandConfig{}, err } - // The mutable loader owns no dataset generation: it preserves the - // overwrite/upsert contract, so Dataset stays nil and ingest.Load selects - // loadResource (generation_orchestration.go:20-25). config.Options.Dataset = nil return config, nil } @@ -174,16 +161,11 @@ func parseLoadCommand(args []string, errorHandling flag.ErrorHandling) (loadComm return config, nil } -// configureLoadFlags registers every load flag, including the required -// --generation, for the immutable generation loaders. func configureLoadFlags(fs *flag.FlagSet, config *loadCommandConfig) { configureLoadFlagsCore(fs, config) fs.StringVar(&config.Generation, "generation", "", "Required opaque immutable dataset generation identifier") } -// configureLoadFlagsCore registers the load flags shared by the mutable and -// immutable loaders. --generation is deliberately excluded so the mutable -// `load` command cannot advertise a flag whose dataset reference it ignores. func configureLoadFlagsCore(fs *flag.FlagSet, config *loadCommandConfig) { fs.StringVar(&config.CPUProfile, "cpu-profile", "", "Write CPU profile to file") fs.StringVar(&config.MemProfile, "mem-profile", "", "Write heap profile to file at end of run") @@ -291,17 +273,8 @@ func parseMemoryBytes(raw string) (int64, error) { return int64(value * factor), nil } -// requireDatabaseExists refuses to operate against a database that has not been -// created. It mirrors the refuse-missing-database precedent in -// scripts/local-preview-index-migration.go:149-159 so an operator typo cannot -// have the CLI silently auto-create an empty database. The store's Open keeps -// auto-creating for the server/tests/harness; only the CLI adds this gate. func requireDatabaseExists(ctx context.Context, url, database string) error { - conn := connection.NewHttpConnection(connection.HttpConfiguration{ - Endpoint: connection.NewRoundRobinEndpoints([]string{url}), - }) - client := driver.NewClient(conn) - databaseExists, err := client.DatabaseExists(ctx, database) + databaseExists, err := arangofactory.DatabaseExists(ctx, url, database) if err != nil { return fmt.Errorf("check existing database: %w", err) } @@ -311,15 +284,15 @@ func requireDatabaseExists(ctx context.Context, url, database string) error { return nil } -// openClientRequiringDatabase gates the CLI on an existing database before it -// opens the shared store client. It never disables the store's auto-create; it -// simply declines to open when the database is absent, so no CLI path creates -// one implicitly. -func openClientRequiringDatabase(ctx context.Context, url, database string) (*arangostore.Client, error) { +func openClientRequiringDatabase(ctx context.Context, url, database string) (*arangofactory.Client, error) { if err := requireDatabaseExists(ctx, url, database); err != nil { return nil, err } - return arangostore.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) + if err != nil { + return nil, err + } + return arangoBackend.Client, nil } func runDiscoverPopulatedReferences(ctx context.Context, args []string) error { @@ -332,7 +305,7 @@ func runDiscoverPopulatedReferences(ctx context.Context, args []string) error { return err } defer client.Close(ctx) - adapter, err := catalogarango.New(client) + adapter, err := arangofactory.NewCatalogStore(client) if err != nil { return err } @@ -353,7 +326,7 @@ func runDiscoverPopulatedFields(ctx context.Context, args []string) error { return err } defer client.Close(ctx) - adapter, err := catalogarango.New(client) + adapter, err := arangofactory.NewCatalogStore(client) if err != nil { return err } @@ -367,7 +340,7 @@ func runDiscoverPopulatedFields(ctx context.Context, args []string) error { func runRebuildRelationshipCatalog(ctx context.Context, args []string) error { fs := flag.NewFlagSet("rebuild-relationship-catalog", flag.ExitOnError) opts := catalog.RelationshipRebuildOptions{} - connection := arangostore.ConnectionOptions{URL: defaultURL, Database: defaultDatabase} + connection := arangofactory.ConnectionOptions{URL: defaultURL, Database: defaultDatabase} fs.StringVar(&connection.URL, "url", defaultURL, "Backend base URL") fs.StringVar(&connection.Database, "database", defaultDatabase, "Arango database") fs.StringVar(&opts.Project, "project", defaultProject, "Project label") @@ -383,7 +356,7 @@ func runRebuildRelationshipCatalog(ctx context.Context, args []string) error { return err } defer client.Close(ctx) - adapter, err := catalogarango.New(client) + adapter, err := arangofactory.NewCatalogStore(client) if err != nil { return err } @@ -397,7 +370,7 @@ func runRebuildRelationshipCatalog(ctx context.Context, args []string) error { func runRebuildResourceInventory(ctx context.Context, args []string) error { fs := flag.NewFlagSet("rebuild-resource-inventory", flag.ExitOnError) opts := catalog.ResourceInventoryRebuildOptions{} - connection := arangostore.ConnectionOptions{URL: defaultURL, Database: defaultDatabase} + connection := arangofactory.ConnectionOptions{URL: defaultURL, Database: defaultDatabase} fs.StringVar(&connection.URL, "url", defaultURL, "Backend base URL") fs.StringVar(&connection.Database, "database", defaultDatabase, "Arango database") fs.StringVar(&opts.Project, "project", defaultProject, "Project label") @@ -417,7 +390,7 @@ func runRebuildResourceInventory(ctx context.Context, args []string) error { return err } defer client.Close(ctx) - lifecycleStore, err := publicationarango.New(client) + lifecycleStore, err := arangofactory.NewDatasetStore(client) if err != nil { return err } @@ -432,7 +405,7 @@ func runRebuildResourceInventory(ctx context.Context, args []string) error { if err != nil { return fmt.Errorf("staged generation %s/%s has invalid generated-resource coverage: %w", ref.Project, ref.Generation, err) } - adapter, err := catalogarango.New(client) + adapter, err := arangofactory.NewCatalogStore(client) if err != nil { return err } @@ -463,7 +436,7 @@ func canonicalManifestResourceTypes(resourceTypes []string) ([]string, error) { func runAuditRelationshipEdges(ctx context.Context, args []string) error { fs := flag.NewFlagSet("audit-relationship-edges", flag.ExitOnError) opts := catalog.RelationshipAuditOptions{} - connection := arangostore.ConnectionOptions{URL: defaultURL, Database: defaultDatabase} + connection := arangofactory.ConnectionOptions{URL: defaultURL, Database: defaultDatabase} fs.StringVar(&connection.URL, "url", defaultURL, "Backend base URL") fs.StringVar(&connection.Database, "database", defaultDatabase, "Arango database") fs.StringVar(&opts.Project, "project", defaultProject, "Project label") @@ -479,13 +452,13 @@ func runAuditRelationshipEdges(ctx context.Context, args []string) error { return printJSON(result) } -func auditRelationshipEdges(ctx context.Context, connection arangostore.ConnectionOptions, opts catalog.RelationshipAuditOptions) (catalog.RelationshipAuditSummary, error) { +func auditRelationshipEdges(ctx context.Context, connection arangofactory.ConnectionOptions, opts catalog.RelationshipAuditOptions) (catalog.RelationshipAuditSummary, error) { client, err := openClientRequiringDatabase(ctx, connection.URL, connection.Database) if err != nil { return catalog.RelationshipAuditSummary{}, err } defer client.Close(ctx) - adapter, err := catalogarango.New(client) + adapter, err := arangofactory.NewCatalogStore(client) if err != nil { return catalog.RelationshipAuditSummary{}, err } @@ -509,7 +482,7 @@ type repairGenerationReport struct { } type activateCommandConfig struct { - Connection arangostore.ConnectionOptions + Connection arangofactory.ConnectionOptions Project string Generation string } @@ -559,7 +532,7 @@ func runRepairGeneration(ctx context.Context, args []string) error { if err != nil { return err } - connection := arangostore.ConnectionOptions{URL: config.Load.Options.URL, Database: config.Load.Options.Database} + connection := arangofactory.ConnectionOptions{URL: config.Load.Options.URL, Database: config.Load.Options.Database} sourceAudit, err := auditRelationshipEdges(ctx, connection, catalog.RelationshipAuditOptions{Project: config.Load.Options.Project, DatasetGeneration: config.SourceGeneration}) if err != nil { return fmt.Errorf("audit source generation: %w", err) @@ -587,7 +560,7 @@ func runRepairGeneration(ctx context.Context, args []string) error { func parseActivateCommand(args []string, errorHandling flag.ErrorHandling) (activateCommandConfig, error) { fs := flag.NewFlagSet("activate-generation", errorHandling) - config := activateCommandConfig{Connection: arangostore.ConnectionOptions{URL: defaultURL, Database: defaultDatabase}} + config := activateCommandConfig{Connection: arangofactory.ConnectionOptions{URL: defaultURL, Database: defaultDatabase}} fs.StringVar(&config.Connection.URL, "url", defaultURL, "Backend base URL") fs.StringVar(&config.Connection.Database, "database", defaultDatabase, "Arango database") fs.StringVar(&config.Project, "project", defaultProject, "Project label") @@ -623,13 +596,13 @@ func runActivateGeneration(ctx context.Context, args []string) error { return printJSON(activateGenerationReport{Project: config.Project, Generation: config.Generation, TargetAudit: audit, Activated: true}) } -func activateStagedGeneration(ctx context.Context, connection arangostore.ConnectionOptions, ref publication.Ref) error { +func activateStagedGeneration(ctx context.Context, connection arangofactory.ConnectionOptions, ref publication.Ref) error { client, err := openClientRequiringDatabase(ctx, connection.URL, connection.Database) if err != nil { return err } defer client.Close(ctx) - lifecycle, err := publicationarango.New(client) + lifecycle, err := arangofactory.NewDatasetStore(client) if err != nil { return err } @@ -641,13 +614,13 @@ func activateStagedGeneration(ctx context.Context, connection arangostore.Connec } type semanticInventoryBackfillCommand struct { - Connection arangostore.ConnectionOptions + Connection arangofactory.ConnectionOptions Options ingest.SemanticInventoryBackfillOptions } func parseBackfillSemanticInventoryCommand(args []string, errorHandling flag.ErrorHandling) (semanticInventoryBackfillCommand, error) { fs := flag.NewFlagSet("backfill-semantic-inventory", errorHandling) - config := semanticInventoryBackfillCommand{Connection: arangostore.ConnectionOptions{URL: defaultURL, Database: defaultDatabase}} + config := semanticInventoryBackfillCommand{Connection: arangofactory.ConnectionOptions{URL: defaultURL, Database: defaultDatabase}} fs.StringVar(&config.Connection.URL, "url", defaultURL, "Backend base URL") fs.StringVar(&config.Connection.Database, "database", defaultDatabase, "Backend database") fs.StringVar(&config.Options.Project, "project", "", "Immutable generation project (required)") @@ -686,7 +659,7 @@ func runBackfillSemanticInventory(ctx context.Context, args []string) error { } defer func() { _ = client.Close(ctx) }() ref, _ := publication.NewRef(config.Options.Project, config.Options.DatasetGeneration) - lifecycle, err := publicationarango.New(client) + lifecycle, err := arangofactory.NewDatasetStore(client) if err != nil { return err } @@ -694,7 +667,7 @@ func runBackfillSemanticInventory(ctx context.Context, args []string) error { if err != nil { return fmt.Errorf("read backfill generation manifest: %w", err) } - store, err := catalogarango.New(client) + store, err := arangofactory.NewCatalogStore(client) if err != nil { return err } @@ -713,10 +686,10 @@ func runBackfillSemanticInventory(ctx context.Context, args []string) error { return printJSON(report) } -func parseDiscoverPopulatedReferenceOptions(args []string, errorHandling flag.ErrorHandling) (catalog.PopulatedReferenceOptions, arangostore.ConnectionOptions, error) { +func parseDiscoverPopulatedReferenceOptions(args []string, errorHandling flag.ErrorHandling) (catalog.PopulatedReferenceOptions, arangofactory.ConnectionOptions, error) { fs := flag.NewFlagSet("discover-populated-references", errorHandling) opts := catalog.PopulatedReferenceOptions{} - connection := arangostore.ConnectionOptions{URL: defaultURL, Database: defaultDatabase} + connection := arangofactory.ConnectionOptions{URL: defaultURL, Database: defaultDatabase} fs.StringVar(&connection.URL, "url", defaultURL, "Backend base URL") fs.StringVar(&connection.Database, "database", defaultDatabase, "Backend database") fs.StringVar(&opts.Project, "project", defaultProject, "Project label") @@ -726,18 +699,18 @@ func parseDiscoverPopulatedReferenceOptions(args []string, errorHandling flag.Er fs.StringVar(&opts.Mode, "mode", catalog.TraversalModeStorage, "Traversal discovery mode: storage or builder") fs.IntVar(&opts.CursorBatch, "cursor-batch-size", 1000, "Query cursor batch size") if err := fs.Parse(args); err != nil { - return catalog.PopulatedReferenceOptions{}, arangostore.ConnectionOptions{}, err + return catalog.PopulatedReferenceOptions{}, arangofactory.ConnectionOptions{}, err } if opts.Mode != catalog.TraversalModeStorage && opts.Mode != catalog.TraversalModeBuilder { - return catalog.PopulatedReferenceOptions{}, arangostore.ConnectionOptions{}, fmt.Errorf("invalid --mode %q: must be %s or %s", opts.Mode, catalog.TraversalModeStorage, catalog.TraversalModeBuilder) + return catalog.PopulatedReferenceOptions{}, arangofactory.ConnectionOptions{}, fmt.Errorf("invalid --mode %q: must be %s or %s", opts.Mode, catalog.TraversalModeStorage, catalog.TraversalModeBuilder) } return opts, connection, nil } -func parseDiscoverPopulatedFieldOptions(args []string, errorHandling flag.ErrorHandling) (catalog.PopulatedFieldOptions, arangostore.ConnectionOptions, error) { +func parseDiscoverPopulatedFieldOptions(args []string, errorHandling flag.ErrorHandling) (catalog.PopulatedFieldOptions, arangofactory.ConnectionOptions, error) { fs := flag.NewFlagSet("discover-populated-fields", errorHandling) opts := catalog.PopulatedFieldOptions{} - connection := arangostore.ConnectionOptions{URL: defaultURL, Database: defaultDatabase} + connection := arangofactory.ConnectionOptions{URL: defaultURL, Database: defaultDatabase} fs.StringVar(&connection.URL, "url", defaultURL, "Backend base URL") fs.StringVar(&connection.Database, "database", defaultDatabase, "Backend database") fs.StringVar(&opts.Project, "project", defaultProject, "Project label") @@ -746,7 +719,7 @@ func parseDiscoverPopulatedFieldOptions(args []string, errorHandling flag.ErrorH fs.BoolVar(&opts.PivotOnly, "pivot-only", false, "Return only pivot-candidate fields") fs.IntVar(&opts.CursorBatch, "cursor-batch-size", 1000, "Query cursor batch size") if err := fs.Parse(args); err != nil { - return catalog.PopulatedFieldOptions{}, arangostore.ConnectionOptions{}, err + return catalog.PopulatedFieldOptions{}, arangofactory.ConnectionOptions{}, err } return opts, connection, nil } diff --git a/cmd/arango-fhir-proto/main_test.go b/cmd/arango-fhir-proto/main_test.go index 9f4b59b7f..b7c0c6a69 100644 --- a/cmd/arango-fhir-proto/main_test.go +++ b/cmd/arango-fhir-proto/main_test.go @@ -276,8 +276,6 @@ func TestParseDiscoveryCommandsPassExplicitDatasetGeneration(t *testing.T) { } } -// U1 (F-CLI2+M2): the documented `load` command must be dispatched to a mutable -// loader rather than falling through to usage()+exit(2). func TestCommandRoutesDispatchesMutableLoad(t *testing.T) { routes := commandRoutes() handler, ok := routes["load"] @@ -311,15 +309,12 @@ func TestParseLoadMutableWiresFlagsWithoutGeneration(t *testing.T) { config.Options.AuthResourcePath != "EllrottLab-GDC_Data" { t.Fatalf("mutable load parsed options = %+v, want the QUICKSTART flags wired", config.Options) } - // --generation belongs to the immutable loader only. if _, err := parseLoadMutableCommand([]string{"--generation", "generation-a"}, flag.ContinueOnError); err == nil || !strings.Contains(err.Error(), "flag provided but not defined") { t.Fatalf("parseLoadMutableCommand accepted --generation; the mutable load must not expose it") } } -// U3 (F-CLI3(i)): manifest-mutating commands must run under shared signal -// cancellation; read/mutable-data commands must not. func TestManifestMutatingCommandsUseSignalCancellation(t *testing.T) { guarded := signalGuardedCommands() routes := commandRoutes() @@ -338,9 +333,6 @@ func TestManifestMutatingCommandsUseSignalCancellation(t *testing.T) { } } -// U4 (F-CLI1): the CLI must refuse to create a missing database instead of -// silently auto-creating it. Emulates the go-driver DatabaseExists request -// (GET /_db/{db}/_api/database/current) with httptest; no real Arango is used. func TestRequireDatabaseExistsRefusesMissingDatabase(t *testing.T) { var seen []string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -364,7 +356,6 @@ func TestRequireDatabaseExistsRefusesMissingDatabase(t *testing.T) { if !strings.Contains(err.Error(), "refusing to create missing database") || !strings.Contains(err.Error(), "ghost_db") { t.Fatalf("requireDatabaseExists error = %v, want a refusal naming the missing database", err) } - // Verify empirically which request the driver issued for DatabaseExists. if !containsSeen(seen, "/_db/ghost_db/_api/database/current") { t.Fatalf("DatabaseExists did not issue the expected current-database request; saw %v", seen) } diff --git a/cmd/dataframe-profile/main.go b/cmd/dataframe-profile/main.go index b603b7150..564c3552a 100644 --- a/cmd/dataframe-profile/main.go +++ b/cmd/dataframe-profile/main.go @@ -1,6 +1,3 @@ -// dataframe-profile compiles one checked-in dataframe fixture and profiles -// the exact rendered AQL directly against ArangoDB. It is intentionally a -// diagnostic command: normal GraphQL execution never enables PROFILE. package main import ( @@ -19,11 +16,11 @@ import ( compilerfixture "github.com/calypr/loom/conformance/compiler" "github.com/calypr/loom/generated/graphql/graph/model" queryapi "github.com/calypr/loom/internal/api/graphql/graph/query" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" - arangostore "github.com/calypr/loom/internal/store/arango" ) type profileReport struct { @@ -54,23 +51,23 @@ type expectedReport struct { } type explainReport struct { - Plans []arangostore.ExplainPlanEstimate `json:"plans"` - FullCollectionScans []arangostore.ExplainCollectionScan `json:"full_collection_scans"` - Indexes []arangostore.ExplainIndexSummary `json:"indexes"` - OptimizerRules []string `json:"optimizer_rules"` - Warnings []arangostore.ExplainWarning `json:"warnings"` + Plans []arangofactory.ExplainPlanEstimate `json:"plans"` + FullCollectionScans []arangofactory.ExplainCollectionScan `json:"full_collection_scans"` + Indexes []arangofactory.ExplainIndexSummary `json:"indexes"` + OptimizerRules []string `json:"optimizer_rules"` + Warnings []arangofactory.ExplainWarning `json:"warnings"` } type profileReportDetails struct { - RuntimeSeconds float64 `json:"runtime_seconds_sum_of_nodes"` - ScannedFull int `json:"scanned_full"` - ScannedIndex int `json:"scanned_index"` - PeakMemory uint64 `json:"peak_memory_bytes"` - Phases arangostore.ProfilePhases `json:"phases"` - ByType []arangostore.ProfileNodeGroup `json:"by_node_type"` - TraversalNodes []arangostore.ProfileNodeSummary `json:"traversal_nodes"` - EnumerateListNodes []arangostore.ProfileNodeSummary `json:"enumerate_list_nodes"` - TopNodes []arangostore.ProfileNodeSummary `json:"top_nodes"` + RuntimeSeconds float64 `json:"runtime_seconds_sum_of_nodes"` + ScannedFull int `json:"scanned_full"` + ScannedIndex int `json:"scanned_index"` + PeakMemory uint64 `json:"peak_memory_bytes"` + Phases arangofactory.ProfilePhases `json:"phases"` + ByType []arangofactory.ProfileNodeGroup `json:"by_node_type"` + TraversalNodes []arangofactory.ProfileNodeSummary `json:"traversal_nodes"` + EnumerateListNodes []arangofactory.ProfileNodeSummary `json:"enumerate_list_nodes"` + TopNodes []arangofactory.ProfileNodeSummary `json:"top_nodes"` } func main() { @@ -146,7 +143,7 @@ func main() { if err != nil { fatalf("resolve recipe plan %q: %v", label, err) } - compiledQueries, err := compiler.CompileResolvedRecipePlanWithPolicy(resolved, *limit, ir.DefaultPhysicalOptimizationPolicy()) + compiledQueries, err := compiler.CompileResolvedRecipePlan(compiler.ArangoTarget(), resolved, *limit, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { fatalf("compile GDC request %q: %v", label, err) } @@ -154,33 +151,38 @@ func main() { fatalf("compile GDC request %q produced %d outputs; profiling requires exactly one", label, len(compiledQueries)) } compiled := compiledQueries[0] - hash := sha256.Sum256([]byte(compiled.Query)) + aqlText, compiledBinds, dispatchErr := compiled.Command.AQLParameters() + if dispatchErr != nil { + fatalf("dispatch GDC request %q to AQL profiling: %v", label, dispatchErr) + } + hash := sha256.Sum256([]byte(aqlText)) aqlHash := hex.EncodeToString(hash[:]) if *aqlPath == "" { *aqlPath = filepath.Join("docs", "benchmarks", label+"-"+aqlHash[:16]+".aql") } - if err := os.WriteFile(*aqlPath, []byte(compiled.Query+"\n"), 0o644); err != nil { + if err := os.WriteFile(*aqlPath, []byte(aqlText+"\n"), 0o644); err != nil { fatalf("write AQL %q: %v", *aqlPath, err) } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() - client, err := arangostore.Open(ctx, *url, *database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: *url, Database: *database}) if err != nil { fatalf("open Arango: %v", err) } - defer client.Close(ctx) - explain, err := client.Explain(ctx, arangostore.ExplainRequest{Query: compiled.Query, BindVars: compiled.BindVars}) + defer arangoBackend.Close(ctx) + client := arangoBackend.Client + explain, err := client.Explain(ctx, arangofactory.ExplainRequest{Query: aqlText, BindVars: compiledBinds}) if err != nil { fatalf("EXPLAIN: %v", err) } started := time.Now() - profileRequest := arangostore.ProfileRequest{ - Query: compiled.Query, - BindVars: compiled.BindVars, + profileRequest := arangofactory.ProfileRequest{ + Query: aqlText, + BindVars: compiledBinds, BatchSize: *batchSize, Count: true, - Options: arangostore.ProfileOptions{Profile: *profile}, + Options: arangofactory.ProfileOptions{Profile: *profile}, } _, err = client.Profile(ctx, profileRequest) if err != nil { @@ -194,9 +196,9 @@ func main() { } warmSeconds := time.Since(warmStarted).Seconds() - assessment := arangostore.AssessExplainResult(explain) - summary := arangostore.SummarizeProfile(profileResult) - topNodes := append([]arangostore.ProfileNodeSummary(nil), summary.Nodes...) + assessment := arangofactory.AssessExplainResult(explain) + summary := arangofactory.SummarizeProfile(profileResult) + topNodes := append([]arangofactory.ProfileNodeSummary(nil), summary.Nodes...) if len(topNodes) > 20 { topNodes = topNodes[:20] } @@ -208,7 +210,7 @@ func main() { Fixture: label, AQLSHA256: aqlHash, ResultSHA256: canonicalResultHash(profileResult.Result), - BindVars: compiled.BindVars, + BindVars: compiledBinds, Rows: profileResult.Count, Explain: explainReport{ Plans: assessment.Plans, @@ -255,8 +257,8 @@ func main() { fmt.Println(string(encoded)) } -func filterProfileNodes(nodes []arangostore.ProfileNodeSummary, typ string) []arangostore.ProfileNodeSummary { - filtered := make([]arangostore.ProfileNodeSummary, 0) +func filterProfileNodes(nodes []arangofactory.ProfileNodeSummary, typ string) []arangofactory.ProfileNodeSummary { + filtered := make([]arangofactory.ProfileNodeSummary, 0) for _, node := range nodes { if node.Type == typ { filtered = append(filtered, node) diff --git a/cmd/dataframe-query/backend_neutral_diagnostics_test.go b/cmd/dataframe-query/backend_neutral_diagnostics_test.go new file mode 100644 index 000000000..e48c35027 --- /dev/null +++ b/cmd/dataframe-query/backend_neutral_diagnostics_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestBackendNeutralDiagnosticsAndPlanIdentity(t *testing.T) { + t.Run("agreed schema drives the neutral label", func(t *testing.T) { + paths := writeRequestFiles(t) + var stdout, stderr bytes.Buffer + body := `{"data":{"runFhirDataframe":{"rowCount":2,"rows":[{"id":"1"},{"id":"2"}],"diagnostics":{"inputResolutionMs":1.5,"requestPreparationMs":2.5,"compilationMs":3.5,"sourceQueryMs":42.5,"arangoQueryMs":0,"rowMaterializationMs":4.5,"resultAssemblyMs":0.5,"totalMs":60,"plan":{"traversalSets":1,"sharedTraversalCount":0,"requiredMatchReuseCount":0,"scopedSharingCandidateGroups":0,"scopedSharingCandidateSets":0,"potentialSharingOpportunityGroups":0,"potentialSharingOpportunitySets":0,"richSourceReuse":[]}}}}}` + if err := run(append(paths, "-print-response=false"), clientForResponses(200, body), &stdout, &stderr); err != nil { + t.Fatalf("run: %v (stderr=%s)", err, stderr.String()) + } + out := stdout.String() + if !strings.Contains(out, "source_query=42.500") { + t.Fatalf("stdout missing the neutral source_query reading of sourceQueryMs:\n%s", out) + } + if strings.Contains(out, "arango_query=") { + t.Fatalf("stdout still prints the AQL-specific stage label:\n%s", out) + } + }) + + t.Run("modern postgres zero does not mask the neutral field", func(t *testing.T) { + diagnostics := dataframeDiagnostics{SourceQueryMs: ptr(7.25), ArangoQueryMs: ptr(0)} + if got := diagnostics.sourceQueryMs(); got != 7.25 { + t.Fatalf("sourceQueryMs() = %v, want 7.25 from the neutral field", got) + } + }) + + t.Run("legacy server falls back to arangoQueryMs", func(t *testing.T) { + paths := writeRequestFiles(t) + var stdout, stderr bytes.Buffer + body := `{"data":{"runFhirDataframe":{"rowCount":1,"rows":[{"id":"1"}],"diagnostics":{"inputResolutionMs":1.5,"requestPreparationMs":2.5,"compilationMs":3.5,"arangoQueryMs":17.25,"rowMaterializationMs":4.5,"resultAssemblyMs":0.5,"totalMs":60,"plan":{"traversalSets":0,"sharedTraversalCount":0,"requiredMatchReuseCount":0,"scopedSharingCandidateGroups":0,"scopedSharingCandidateSets":0,"potentialSharingOpportunityGroups":0,"potentialSharingOpportunitySets":0,"richSourceReuse":[]}}}}}` + if err := run(append(paths, "-print-response=false"), clientForResponses(200, body), &stdout, &stderr); err != nil { + t.Fatalf("run: %v (stderr=%s)", err, stderr.String()) + } + out := stdout.String() + if !strings.Contains(out, "source_query=17.250") { + t.Fatalf("legacy server without sourceQueryMs must still report the cursor interval under the neutral label:\n%s", out) + } + }) + + t.Run("example queries select the neutral field", func(t *testing.T) { + for _, name := range []string{"meta_patient_dataframe.graphql", "meta_gdc_case_matrix.graphql"} { + raw, err := os.ReadFile(filepath.Join("..", "..", "examples", name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + selection := string(raw) + if !strings.Contains(selection, "sourceQueryMs") { + t.Fatalf("%s does not select the agreed sourceQueryMs field", name) + } + if !strings.Contains(selection, "arangoQueryMs") { + t.Fatalf("%s must keep selecting the required legacy arangoQueryMs field for compatibility", name) + } + } + }) +} + +func ptr(value float64) *float64 { return &value } diff --git a/cmd/dataframe-query/main.go b/cmd/dataframe-query/main.go index 1d4e8c5c5..3c14bedb1 100644 --- a/cmd/dataframe-query/main.go +++ b/cmd/dataframe-query/main.go @@ -1,10 +1,3 @@ -// dataframe-query executes a checked-in GraphQL dataframe request and prints -// the response with wall-clock timing. It is deliberately a small developer -// tool: benchmark loops belong in Go benchmarks, while this command makes one -// human-readable dataframe request easy to inspect end to end. -// -// A GraphQL operation error is a command failure even when the server answers -// with HTTP 200, so automation cannot mistake a failed query for an empty one. package main import ( @@ -38,15 +31,12 @@ func main() { code := 1 var usage usageError if errors.As(err, &usage) { - // Preserve the historical exit status for malformed flags. code = 2 } fmt.Fprintf(os.Stderr, "dataframe-query: %v\n", err) os.Exit(code) } -// usageError marks flag parsing failures so main can keep the exit status the -// standard flag package used while the rest of the command reports status 1. type usageError struct{ err error } func (e usageError) Error() string { return e.err.Error() } @@ -113,7 +103,7 @@ func run(args []string, clientFor func(time.Duration) *http.Client, stdout, stde fmt.Fprintf(stdout, "Response: rows=%d bytes=%d rows/sec=%.1f\n\n", responseMetrics.Rows, responseMetrics.Bytes, responseMetrics.RowsPerSecond) if responseMetrics.Diagnostics != nil { d := responseMetrics.Diagnostics - fmt.Fprintf(stdout, "Server stages (ms): input_resolution=%.3f request_preparation=%.3f compilation=%.3f arango_query=%.3f row_materialization=%.3f result_assembly=%.3f dataframe_service_total=%.3f\n", d.InputResolutionMs, d.RequestPreparationMs, d.CompilationMs, d.ArangoQueryMs, d.RowMaterializationMs, d.ResultAssemblyMs, d.TotalMs) + fmt.Fprintf(stdout, "Server stages (ms): input_resolution=%.3f request_preparation=%.3f compilation=%.3f source_query=%.3f row_materialization=%.3f result_assembly=%.3f dataframe_service_total=%.3f\n", d.InputResolutionMs, d.RequestPreparationMs, d.CompilationMs, d.sourceQueryMs(), d.RowMaterializationMs, d.ResultAssemblyMs, d.TotalMs) fmt.Fprintf(stdout, "Outside dataframe service (GraphQL serialization + HTTP): %.3f ms\n\n", durationMinusMillis(durations[len(durations)-1], d.TotalMs)) fmt.Fprintf(stdout, "Compiler plan: traversal_sets=%d shared_traversals=%d required_match_reuse=%d scope_safe_sharing_groups=%d scope_safe_sharing_sets=%d potential_sharing_groups=%d potential_sharing_sets=%d\n", d.Plan.TraversalSets, d.Plan.SharedTraversalCount, d.Plan.RequiredMatchReuseCount, d.Plan.ScopedSharingCandidateGroups, d.Plan.ScopedSharingCandidateSets, d.Plan.PotentialSharingOpportunityGroups, d.Plan.PotentialSharingOpportunitySets) for _, reuse := range d.Plan.RichSourceReuse { @@ -130,9 +120,6 @@ func run(args []string, clientFor func(time.Duration) *http.Client, stdout, stde return nil } -// requestSequence issues the repeat loop and inspects every response, not only -// the last one, so an operation error in any request fails the command with its -// request index. A failing request stops the loop. func requestSequence(client *http.Client, url string, payload []byte, repeat int, stderr io.Writer) ([]time.Duration, []byte, dataframeResponseMetrics, error) { durations := make([]time.Duration, 0, repeat) var responseBody []byte @@ -145,8 +132,6 @@ func requestSequence(client *http.Client, url string, payload []byte, repeat int return nil, nil, dataframeResponseMetrics{}, fmt.Errorf("request %d/%d: %w", run, repeat, err) } if err := classifyGraphQLEnvelope(body); err != nil { - // Keep the payload visible even with -print-response=false: a - // rejected envelope is exactly what the caller needs to see. printEnvelopeForDiagnosis(stderr, body) return nil, nil, dataframeResponseMetrics{}, fmt.Errorf("request %d/%d: %w", run, repeat, err) } @@ -157,8 +142,6 @@ func requestSequence(client *http.Client, url string, payload []byte, repeat int return durations, responseBody, responseMetrics, nil } -// printEnvelopeForDiagnosis writes the raw response payload for a rejected -// envelope, pretty-printing it when possible. func printEnvelopeForDiagnosis(writer io.Writer, body []byte) { fmt.Fprintf(writer, "dataframe-query: response preserved for diagnosis (%d bytes):\n", len(body)) if err := prettyPrint(writer, body); err != nil { @@ -166,10 +149,6 @@ func printEnvelopeForDiagnosis(writer io.Writer, body []byte) { } } -// dataframeResponseMetrics is intentionally derived from the GraphQL response -// rather than server internals. Servers that select the diagnostics field also -// return service-stage timings; older Loom servers remain compatible but omit -// that optional measurement block. type dataframeResponseMetrics struct { Rows int Bytes int @@ -181,13 +160,24 @@ type dataframeDiagnostics struct { InputResolutionMs float64 `json:"inputResolutionMs"` RequestPreparationMs float64 `json:"requestPreparationMs"` CompilationMs float64 `json:"compilationMs"` - ArangoQueryMs float64 `json:"arangoQueryMs"` + SourceQueryMs *float64 `json:"sourceQueryMs"` + ArangoQueryMs *float64 `json:"arangoQueryMs"` RowMaterializationMs float64 `json:"rowMaterializationMs"` ResultAssemblyMs float64 `json:"resultAssemblyMs"` TotalMs float64 `json:"totalMs"` Plan dataframeCompilerPlanDiagnostics `json:"plan"` } +func (d dataframeDiagnostics) sourceQueryMs() float64 { + if d.SourceQueryMs != nil { + return *d.SourceQueryMs + } + if d.ArangoQueryMs != nil { + return *d.ArangoQueryMs + } + return 0 +} + type dataframeCompilerPlanDiagnostics struct { TraversalSets int `json:"traversalSets"` SharedTraversalCount int `json:"sharedTraversalCount"` @@ -232,11 +222,6 @@ func inspectDataframeResponse(body []byte, duration time.Duration) dataframeResp return metrics } -// graphqlResponseError explains why a response that arrived with a successful -// HTTP status must not be reported as a complete dataframe result. GraphQL -// servers routinely answer operation failures with HTTP 200 plus an "errors" -// array, and a dataframe CLI that ignores them tells its caller "0 rows" when -// the query never ran. type graphqlResponseError struct { Kind string Reason string @@ -301,11 +286,6 @@ func graphqlPathString(path []any) string { return strings.Join(segments, ".") } -// classifyGraphQLEnvelope returns nil only for a GraphQL response the command -// may report as complete: no operation errors and a runFhirDataframe payload. -// Data mixed with errors is rejected rather than silently reported as rows. -// Each envelope field is decoded on its own so one malformed field cannot hide -// the others. func classifyGraphQLEnvelope(body []byte) error { var envelope map[string]json.RawMessage if err := json.Unmarshal(body, &envelope); err != nil { @@ -360,9 +340,6 @@ func graphQLErrors(raw json.RawMessage) ([]string, error) { return rendered, nil } -// graphqlDataframePayload reports whether "data" carries a runFhirDataframe -// object. A missing or null data, or a null field, means the operation did not -// produce a usable result. func graphqlDataframePayload(data json.RawMessage) bool { if len(data) == 0 || strings.TrimSpace(string(data)) == "null" { return false @@ -387,16 +364,10 @@ func durationMinusMillis(duration time.Duration, milliseconds float64) float64 { return remaining } -// encodeGraphQLRequest builds the GraphQL POST body from the raw operation and -// variables bytes so neither passes through a numeric decode. func encodeGraphQLRequest(query, variables []byte) ([]byte, error) { return json.Marshal(graphqlRequest{Query: string(query), Variables: variables}) } -// withLimit replaces or adds the top-level limit field, leaving every other -// byte of the variables document untouched. Fields are kept as RawMessage -// because decoding into any would route numbers through float64, which silently -// rewrites integers above 2^53 and reformats exponent and decimal literals. func withLimit(variables []byte, limit int) ([]byte, error) { if limit <= 0 { return variables, nil @@ -405,7 +376,6 @@ func withLimit(variables []byte, limit int) ([]byte, error) { if err := json.Unmarshal(variables, &fields); err != nil { return nil, err } - // json.Unmarshal accepts "null" into a nil map without an error. if fields == nil { return nil, fmt.Errorf("variables must be a JSON object") } diff --git a/cmd/dataframe-query/main_test.go b/cmd/dataframe-query/main_test.go index 61ac449de..fe97cc52a 100644 --- a/cmd/dataframe-query/main_test.go +++ b/cmd/dataframe-query/main_test.go @@ -78,8 +78,6 @@ func TestWithLimit(t *testing.T) { want: `{"input":{"project":"P1"},"limit":7}`, }, { - // Nested objects are byte-identical passthroughs: values, literals - // and key order all survive the top-level limit override. name: "preserves nested numbers exactly", variables: `{"limit":25,"input":{"filters":[{"value":9007199254740993}],"threshold":1.5e3,"score":0.30000000000000004,"ratio":-0.5E-2}}`, limit: 1000, @@ -171,10 +169,6 @@ func TestExecuteSendsExactOverriddenLimitBody(t *testing.T) { } } -// TestWithLimitPreservesLargeNumbers pins the second cli-boundary-probe -// finding: a float64 decode silently rewrote 9007199254740993 into -// 9007199254740992, so a variables file can never be re-encoded through -// float64. func TestWithLimitPreservesLargeNumbers(t *testing.T) { original := `{"input":{"project":"P1","window":9007199254740993}}` variables, err := withLimit([]byte(original), 7) @@ -321,7 +315,6 @@ func TestRunFailsForHTTPOkGraphQLErrorsWithoutPrintingResponse(t *testing.T) { if stdout.Len() != 0 { t.Fatalf("stdout should be empty when the request failed:\n%s", stdout.String()) } - // The payload is preserved for diagnosis even though printing is disabled. if !strings.Contains(stderr.String(), "project not found") { t.Fatalf("stderr missing preserved payload:\n%s", stderr.String()) } diff --git a/cmd/generate/extraction.go b/cmd/generate/extraction.go index e13882439..92516efce 100644 --- a/cmd/generate/extraction.go +++ b/cmd/generate/extraction.go @@ -26,7 +26,6 @@ func generateExtract(schema *Schema, path string) error { sb.WriteString("\tfhirschema \"github.com/calypr/loom/internal/fhir/schema\"\n") sb.WriteString(")\n\n") - // 1. Write EdgeDocument struct definition sb.WriteString(`// EdgeDocument represents an edge in ArangoDB. type EdgeDocument struct { Key string ` + "`" + `json:"_key"` + "`" + ` @@ -188,15 +187,11 @@ func buildEdgeRawJSON(key, from, to, label, projectJSON, fromType, toType string } sort.Strings(keys) - // Keep track of which resource types are supported (those having links metadata in the schema) supportedResources := []string{} for _, k := range keys { def := schema.Defs[k] - // Only concrete FHIR roots get graph loaders. Backbone definitions and - // abstract Resource remain available to field/selector generation but - // never become graph collections or row grains. if !isFHIRRootResourceDefinition(k, def) { continue } @@ -216,14 +211,12 @@ func buildEdgeRawJSON(key, from, to, label, projectJSON, fromType, toType string sb.WriteString("\tvar seen map[string]struct{}\n") sb.WriteString("\t_ = seen\n") - // Generate link extraction code statically! for _, link := range def.Links { ptr := link.TemplatePointers.ID if ptr == "" { continue } - // We need to generate code to traverse to templatePointers pathParts := strings.Split(ptr, "/")[1:] traversalCode, err := generateTraversalCode(schema, k, pathParts, "x", k, link, "\t") if err != nil { @@ -236,7 +229,6 @@ func buildEdgeRawJSON(key, from, to, label, projectJSON, fromType, toType string sb.WriteString("}\n\n") } - // 2. Write the unified ValidateAndExtract entrypoint sb.WriteString("// ValidateAndExtract parses, validates, and extracts edges for a FHIR resource.\n") sb.WriteString("func ValidateAndExtract(raw []byte, resourceType string, project string) (Result, error) {\n") sb.WriteString("\tswitch resourceType {\n") @@ -272,8 +264,6 @@ func buildEdgeRawJSON(key, from, to, label, projectJSON, fromType, toType string func generateTraversalCode(schema *Schema, structName string, path []string, varName string, currentType string, link Link, indent string) (string, error) { if len(path) == 0 { - // We have reached the string value (the reference). - // We emit the edge building code! var sb strings.Builder sb.WriteString(fmt.Sprintf("%srefVal := %s\n", indent, varName)) sb.WriteString(fmt.Sprintf("%srefType, targetID, ok := splitFHIRReference(refVal)\n", indent)) @@ -281,7 +271,6 @@ func generateTraversalCode(schema *Schema, structName string, path []string, var sb.WriteString(fmt.Sprintf("%s\trefType, ok = fhirschema.ConcreteResourceType(refType)\n", indent)) sb.WriteString(fmt.Sprintf("%s\tif ok {\n", indent)) - // Match prefix check allowAnyMatch := false matchPrefix := "" for _, rx := range link.TargetHints.RegexMatch { @@ -315,7 +304,6 @@ func generateTraversalCode(schema *Schema, structName string, path []string, var sb.WriteString(fmt.Sprintf("%s\t))\n", bodyIndent)) sb.WriteString(fmt.Sprintf("%s}\n", bodyIndent)) - // Check if backref exists if len(link.TargetHints.Backref) > 0 && link.TargetHints.Backref[0] != "" { backref := link.TargetHints.Backref[0] sb.WriteString(fmt.Sprintf("%sbackrefKey := getEdgeUUID(id, targetID, %q)\n", bodyIndent, backref)) @@ -349,17 +337,9 @@ func generateTraversalCode(schema *Schema, structName string, path []string, var part := path[0] if part == "-" { - // VarName must be a slice. Loop over the elements. - // Wait, what is the item type? - // We need to look up the parent property to see what its items are. - // E.g. x.PartOf which is a slice. Let's find what the element type is. - // We can pass currentType down, but since we are iterating, we need to know the type of items. - // Let's pass the element type in currentType. - // If part is "-", then the type of item is currentType. loopVar := fmt.Sprintf("item_%d_%s", len(path), strings.ReplaceAll(varName, ".", "_")) var sb strings.Builder sb.WriteString(fmt.Sprintf("%sfor _, %s := range %s {\n", indent, loopVar, varName)) - // Check for nil if item is a pointer sb.WriteString(fmt.Sprintf("%s\tif %s != nil {\n", indent+"\t", loopVar)) subCode, err := generateTraversalCode(schema, structName, path[1:], loopVar, currentType, link, indent+"\t\t") if err != nil { @@ -371,8 +351,6 @@ func generateTraversalCode(schema *Schema, structName string, path []string, var return sb.String(), nil } - // It's a field name! - // Look up in schema Defs[currentType] to find property named `part`. def, ok := schema.Defs[currentType] if !ok { return "", fmt.Errorf("definition %s not found in schema while resolving path part %s", currentType, part) @@ -380,8 +358,6 @@ func generateTraversalCode(schema *Schema, structName string, path []string, var prop, ok := def.Properties[part] if !ok { - // It could be that part is not a direct property, or it is in some other way. - // But in graph-fhir.json, they are always properties. return "", fmt.Errorf("property %s not found in definition %s", part, currentType) } @@ -398,14 +374,12 @@ func generateTraversalCode(schema *Schema, structName string, path []string, var } var sb strings.Builder - // If it's a pointer type, check for nil - goType := getGoType(prop, true) // treat as optional to see if it has pointer prefix + goType := getGoType(prop, true) isPointer := strings.HasPrefix(goType, "*") isSlice := strings.HasPrefix(goType, "[]") if isPointer { sb.WriteString(fmt.Sprintf("%sif %s != nil {\n", indent, nextVarName)) - // If it's a leaf string pointer (like *string), dereference it actualVar := nextVarName if nextType == "string" { actualVar = "*" + nextVarName diff --git a/cmd/generate/fhir_helpers.go.tmpl b/cmd/generate/fhir_helpers.go.tmpl index 479b092bf..99ad13375 100644 --- a/cmd/generate/fhir_helpers.go.tmpl +++ b/cmd/generate/fhir_helpers.go.tmpl @@ -12,10 +12,8 @@ import ( "github.com/google/uuid" ) -// FHIRComments handles comments that can be a single JSON string or a JSON array of strings. type FHIRComments []string -// UnmarshalJSON unmarshals FHIR comments from a string or array of strings. func (c *FHIRComments) UnmarshalJSON(data []byte) error { if len(data) == 0 { return nil @@ -36,7 +34,6 @@ func (c *FHIRComments) UnmarshalJSON(data []byte) error { return nil } -// MarshalJSON marshals FHIR comments to a single string (if length is 1) or an array of strings. func (c FHIRComments) MarshalJSON() ([]byte, error) { if len(c) == 1 { return json.Marshal(c[0]) @@ -44,7 +41,6 @@ func (c FHIRComments) MarshalJSON() ([]byte, error) { return json.Marshal([]string(c)) } -// ValidateFhirDateTime checks a string against FHIR date-time rules (YYYY-MM-DDThh:mm:ss[Z|(+|-)hh:mm] or partial date). func ValidateFhirDateTime(s string) error { if !strings.Contains(s, "T") { return validatePartialFhirDate(s, "FHIR date-time") @@ -57,15 +53,15 @@ func ValidateFhirDateTime(s string) error { func validatePartialFhirDate(s, typeName string) error { switch len(s) { - case 4: // YYYY + case 4: if _, err := time.Parse("2006", s); err != nil { return fmt.Errorf("value '%s' is not a valid partial %s (YYYY): %w", s, typeName, err) } - case 7: // YYYY-MM + case 7: if _, err := time.Parse("2006-01", s); err != nil { return fmt.Errorf("value '%s' is not a valid partial %s (YYYY-MM): %w", s, typeName, err) } - case 10: // YYYY-MM-DD + case 10: if _, err := time.Parse("2006-01-02", s); err != nil { return fmt.Errorf("value '%s' is not a valid partial %s (YYYY-MM-DD): %w", s, typeName, err) } @@ -75,12 +71,10 @@ func validatePartialFhirDate(s, typeName string) error { return nil } -// ValidateFhirDate checks a string against FHIR date rules (YYYY[-MM[-DD]]). func ValidateFhirDate(s string) error { return validatePartialFhirDate(s, "FHIR date") } -// ValidateFhirTime checks a string against FHIR time rules (HH:MM:SS[.SSSS]). func ValidateFhirTime(s string) error { if len(s) < 8 { return fmt.Errorf("value '%s' is too short for a FHIR time (minimum 8 characters)", s) @@ -112,7 +106,6 @@ func ValidateFhirTime(s string) error { return nil } -// ValidateFhirURI checks a string against FHIR URI rules (must be an absolute URI). func ValidateFhirURI(s string) error { u, err := url.ParseRequestURI(s) if err != nil { @@ -124,7 +117,6 @@ func ValidateFhirURI(s string) error { return nil } -// ValidateFhirUUID checks a string against FHIR UUID rules (RFC 4122). func ValidateFhirUUID(s string) error { if _, err := uuid.Parse(s); err != nil { return fmt.Errorf("value '%s' is not a valid UUID (RFC 4122): %w", s, err) @@ -132,7 +124,6 @@ func ValidateFhirUUID(s string) error { return nil } -// ValidateFhirBinary checks that a string is a valid base64-encoded value. func ValidateFhirBinary(s string) error { if _, err := base64.StdEncoding.DecodeString(s); err != nil { return fmt.Errorf("value is not a valid Base64 encoded string: %w", err) diff --git a/cmd/generate/graphql.go b/cmd/generate/graphql.go index 1dd409a7e..de9184e57 100644 --- a/cmd/generate/graphql.go +++ b/cmd/generate/graphql.go @@ -8,13 +8,6 @@ import ( "unicode" ) -// generateFHIRGraphQL emits the generated half of Loom's GraphQL schema. It -// deliberately contains only schema declarations: resolver behavior remains -// in the generic FHIR query service and the handwritten Query schema. -// -// The graph-fhir document has 136 definitions, including backbone and -// datatype definitions. Emitting all of them (rather than only concrete root -// resources) makes every JSON property selectable in a typed GraphQL query. func generateFHIRGraphQL(schema *Schema, path string) error { if schema == nil { return fmt.Errorf("nil FHIR schema") @@ -57,10 +50,6 @@ func generateFHIRGraphQL(schema *Schema, path string) error { b.WriteString("# Code generated by cmd/generate/main.go. DO NOT EDIT.\n") b.WriteString("# Generated from schemas/graph-fhir.json and loaded with internal/api/graphql/graph/schema/schema.graphqls.\n\n") b.WriteString("scalar Long\n\n") - // gqlgen normally derives resolver method names from GraphQL field names. - // The FHIR primitive-extension convention intentionally has both `code` and - // `_code`; @goField keeps those two legal Go identifiers distinct while - // leaving the public JSON/GraphQL names untouched. b.WriteString("directive @goField(forceResolver: Boolean, name: String, omittable: Boolean, type: String) on INPUT_FIELD_DEFINITION | FIELD_DEFINITION\n\n") b.WriteString("interface FHIRResource {\n") b.WriteString(" id: ID!\n") @@ -74,8 +63,6 @@ func generateFHIRGraphQL(schema *Schema, path string) error { } b.WriteString("}\n\n") - // Root operations use exact JSON Schema definition capitalization. These - // are extensions because Query itself is declared by the handwritten API. b.WriteString("extend type Query {\n") for _, root := range roots { b.WriteString(" ") @@ -180,8 +167,6 @@ func graphqlPropertyType(owner, name string, prop *Property, roots map[string]st return "JSON", nil } if name == "fhir_comments" { - // graph-fhir permits a string or an array. The runtime normalizes both - // forms to a string list before decoding. return "[String!]", nil } if prop.Ref != "" { @@ -199,9 +184,6 @@ func graphqlPropertyType(owner, name string, prop *Property, roots map[string]st if err != nil { return "", err } - // Arrays in the source schema are nullable; item schemas are concrete - // unless JSON Schema explicitly allows null (not present in this FHIR - // profile). return "[" + strings.TrimSuffix(item, "!") + "!]", nil } typeName, err := graphqlSchemaType(prop.Type) @@ -228,8 +210,6 @@ func graphqlPropertyType(owner, name string, prop *Property, roots map[string]st case "array": return "[JSON!]", nil case "": - // fhir_comments is handled above. Other typeless values are free-form - // objects in graph-fhir and intentionally use the existing JSON scalar. return "JSON", nil default: return "", fmt.Errorf("unsupported JSON Schema type %q on %q.%s", typeName, owner, name) diff --git a/cmd/generate/main.go b/cmd/generate/main.go index 7ee19e7a1..08fa1dc80 100644 --- a/cmd/generate/main.go +++ b/cmd/generate/main.go @@ -42,46 +42,36 @@ func main() { os.Exit(1) } - // 1. Generate package support used by the schema-derived files. if err := generateFHIRHelpers(filepath.Join(*structsDir, "helpers.go")); err != nil { fmt.Fprintf(os.Stderr, "Error generating helpers.go: %v\n", err) os.Exit(1) } - // 2. Generate model.go if err := generateModel(&schema, filepath.Join(*structsDir, "model.go")); err != nil { fmt.Fprintf(os.Stderr, "Error generating model.go: %v\n", err) os.Exit(1) } - // 3. Generate validate.go if err := generateValidate(&schema, filepath.Join(*structsDir, "validate.go")); err != nil { fmt.Fprintf(os.Stderr, "Error generating validate.go: %v\n", err) os.Exit(1) } - // 4. Generate extract.go if err := generateExtract(&schema, filepath.Join(*structsDir, "extract.go")); err != nil { fmt.Fprintf(os.Stderr, "Error generating extract.go: %v\n", err) os.Exit(1) } - // 5. Generate the concrete-resource registry and GraphQL markers. if err := generateFHIRResources(&schema, filepath.Join(*structsDir, "resources.go"), filepath.Join(*structsDir, "graphql.go")); err != nil { fmt.Fprintf(os.Stderr, "Error generating resource registry: %v\n", err) os.Exit(1) } - // 6. Generate internal compiler schema metadata if err := generateFHIRSchema(&schema, *metadataOut); err != nil { fmt.Fprintf(os.Stderr, "Error generating generated/fhirschema/generated.go: %v\n", err) os.Exit(1) } - // Keep the FHIR GraphQL surface generated from the same parsed schema as - // the compiler metadata and Go models. The SDL is intentionally emitted as - // a separate schema document so the handwritten dataframe API remains - // readable and stable. if strings.TrimSpace(*graphqlOut) != "" { if err := os.MkdirAll(filepath.Dir(*graphqlOut), 0755); err != nil { fmt.Fprintf(os.Stderr, "Error creating GraphQL schema directory: %v\n", err) diff --git a/cmd/generate/metadata.go b/cmd/generate/metadata.go index 9b8f8b492..2505c0273 100644 --- a/cmd/generate/metadata.go +++ b/cmd/generate/metadata.go @@ -71,10 +71,6 @@ func generateFHIRSchema(schema *Schema, path string) error { return os.WriteFile(path, []byte(sb.String()), 0644) } -// schemaConcreteLinkTargetType resolves a graph link target from schema -// semantics. Relationship labels are only a compatibility fallback for older -// schema entries; the fallback is accepted only when it resolves to a -// concrete root, so labels can never introduce a graph node by themselves. func schemaConcreteLinkTargetType(schema *Schema, link Link) (string, bool) { candidates := []string{refName(link.TargetSchema.Ref)} if candidates[0] == "" { @@ -98,16 +94,6 @@ func schemaConcreteLinkTargetType(schema *Schema, link Link) (string, bool) { return "", false } -// schemaFHIRRootResourceTypes discovers compiler-visible collections from the -// checked-in graph schema. It deliberately does not carry a copied list of -// FHIR resource names: a schema definition is a concrete root only when its -// own resourceType constant names that definition and it has the FHIR Resource -// root fields (a string id plus an explicit metadata object). -// -// The graph schema also publishes the abstract Resource base definition so -// generated field references can resolve. Its resourceType constant is the -// generic "Resource" placeholder rather than a concrete graph collection, so -// it is excluded by role rather than maintaining a list of concrete types. func schemaFHIRRootResourceTypes(schema *Schema) []string { if schema == nil { return nil diff --git a/cmd/generate/model.go b/cmd/generate/model.go index 02d7e43da..a33d15805 100644 --- a/cmd/generate/model.go +++ b/cmd/generate/model.go @@ -16,7 +16,6 @@ func getGoType(prop *Property, isField bool) string { return "*" + name } - // Check for fhir_comments which is anyOf [string, array of string] if prop.Type == nil && prop.Items == nil && prop.Ref == "" { return "FHIRComments" } @@ -93,10 +92,6 @@ func generateModel(schema *Schema, path string) error { jsonTag += ",omitempty" } - // gqlgen uses this explicit tag to preserve FHIR primitive-extension - // names such as _birthDate, whose Go field is XBirthDate. Keeping - // the tag on every field makes autobinding deterministic while the - // JSON tag remains the storage contract. sb.WriteString(fmt.Sprintf("\t%s %s `json:\"%s\" gqlgen:\"%s\"`\n", goName, goType, jsonTag, pk)) } sb.WriteString("}\n\n") diff --git a/cmd/generate/schema.go b/cmd/generate/schema.go index 26c053a66..eb08b671a 100644 --- a/cmd/generate/schema.go +++ b/cmd/generate/schema.go @@ -1,6 +1,5 @@ package main -// Schema is the JSON input contract consumed by all generators. type Schema struct { ID string `json:"$id"` Defs map[string]*Definition `json:"$defs"` diff --git a/cmd/generate/validation.go b/cmd/generate/validation.go index e7dbe6a52..c25b6c81c 100644 --- a/cmd/generate/validation.go +++ b/cmd/generate/validation.go @@ -24,7 +24,6 @@ func generateValidate(schema *Schema, path string) error { } sort.Strings(keys) - // Collect and write regexes sb.WriteString("// Pre-compiled regular expressions for pattern validation\n") sb.WriteString("var (\n") regexMap := make(map[string]string) @@ -43,7 +42,6 @@ func generateValidate(schema *Schema, path string) error { regexMap[varName] = prop.Pattern sb.WriteString(fmt.Sprintf("\t%s = regexp.MustCompile(%s)\n", varName, strconv.Quote(prop.Pattern))) } - // check array items pattern if prop.Items != nil && prop.Items.Pattern != "" && prop.Items.Pattern != "[ \\r\\n\\t\\S]+" && prop.Items.Pattern != "\\S*" { varName := fmt.Sprintf("rx_%s_%s_items", k, toGoName(pk)) regexMap[varName] = prop.Items.Pattern @@ -70,25 +68,18 @@ func generateValidate(schema *Schema, path string) error { required := isRequired(def, pk) goName := "x." + toGoName(pk) - // 1. Required field check if required { sb.WriteString(fmt.Sprintf("\tif %s == nil {\n", goName)) sb.WriteString(fmt.Sprintf("\t\treturn fmt.Errorf(\"required field '%s' is missing\")\n", pk)) sb.WriteString("\t}\n") } - // Under what condition do we validate constraints? - // For optional fields: if not nil. - // For required fields: since they are pointers (or strings/bools if required), we check if they are not nil. - // Actually, if we made a required field non-pointer (like string), we can check it directly. - // Let's determine if goType is pointer. goType := getGoType(prop, true) isPointer := strings.HasPrefix(goType, "*") isSlice := strings.HasPrefix(goType, "[]") isCustomComments := goType == "FHIRComments" if isCustomComments { - // No validation needed for FHIRComments continue } @@ -101,13 +92,11 @@ func generateValidate(schema *Schema, path string) error { indent = "\t\t" } - // Validate constraints on the field value valExpr := goName if isPointer { valExpr = "*" + goName } - // 2. Const check if prop.Const != nil { constStr, ok := prop.Const.(string) if ok { @@ -117,14 +106,12 @@ func generateValidate(schema *Schema, path string) error { } } - // 3. Pattern check if prop.Pattern != "" { if prop.Pattern == "[ \\r\\n\\t\\S]+" { sb.WriteString(fmt.Sprintf("%sif %s == \"\" {\n", indent, valExpr)) sb.WriteString(fmt.Sprintf("%s\treturn fmt.Errorf(\"field '%s' does not match pattern '%s'\")\n", indent, pk, escapePattern(prop.Pattern))) sb.WriteString(fmt.Sprintf("%s}\n", indent)) } else if prop.Pattern == "\\S*" { - // Skip unanchored \S* check since it always matches } else { varName := fmt.Sprintf("rx_%s_%s", k, toGoName(pk)) sb.WriteString(fmt.Sprintf("%sif !%s.MatchString(%s) {\n", indent, varName, valExpr)) @@ -133,35 +120,30 @@ func generateValidate(schema *Schema, path string) error { } } - // 4. MinLength check if prop.MinLength != nil { sb.WriteString(fmt.Sprintf("%sif utf8.RuneCountInString(%s) < %d {\n", indent, valExpr, *prop.MinLength)) sb.WriteString(fmt.Sprintf("%s\treturn fmt.Errorf(\"field '%s' is too short (min %d characters)\")\n", indent, pk, *prop.MinLength)) sb.WriteString(fmt.Sprintf("%s}\n", indent)) } - // 5. MaxLength check if prop.MaxLength != nil { sb.WriteString(fmt.Sprintf("%sif utf8.RuneCountInString(%s) > %d {\n", indent, valExpr, *prop.MaxLength)) sb.WriteString(fmt.Sprintf("%s\treturn fmt.Errorf(\"field '%s' is too long (max %d characters)\")\n", indent, pk, *prop.MaxLength)) sb.WriteString(fmt.Sprintf("%s}\n", indent)) } - // 6. Minimum check if prop.Minimum != nil { sb.WriteString(fmt.Sprintf("%sif %s < %f {\n", indent, valExpr, *prop.Minimum)) sb.WriteString(fmt.Sprintf("%s\treturn fmt.Errorf(\"field '%s' is below minimum %f\")\n", indent, pk, *prop.Minimum)) sb.WriteString(fmt.Sprintf("%s}\n", indent)) } - // 7. ExclusiveMinimum check if prop.ExclusiveMinimum != nil { sb.WriteString(fmt.Sprintf("%sif %s <= %f {\n", indent, valExpr, *prop.ExclusiveMinimum)) sb.WriteString(fmt.Sprintf("%s\treturn fmt.Errorf(\"field '%s' is below exclusive minimum %f\")\n", indent, pk, *prop.ExclusiveMinimum)) sb.WriteString(fmt.Sprintf("%s}\n", indent)) } - // 8. Format check if prop.Format != "" { switch prop.Format { case "date-time": @@ -191,14 +173,12 @@ func generateValidate(schema *Schema, path string) error { } } - // 9. Nested validation (Struct pointer) if prop.Ref != "" && refName(prop.Ref) != "links" { sb.WriteString(fmt.Sprintf("%sif err := %s.Validate(); err != nil {\n", indent, goName)) sb.WriteString(fmt.Sprintf("%s\treturn fmt.Errorf(\"field '%s' is invalid: %%w\", err)\n", indent, pk)) sb.WriteString(fmt.Sprintf("%s}\n", indent)) } - // 10. Slice item validation if isSlice && prop.Items != nil { hasItemValidation := (prop.Items.Pattern != "" && prop.Items.Pattern != "\\S*") || prop.Items.Format != "" || (prop.Items.Ref != "" && refName(prop.Items.Ref) != "links") if hasItemValidation { @@ -214,14 +194,12 @@ func generateValidate(schema *Schema, path string) error { itemValExpr = "*" + itemName } - // Check items constraints if prop.Items.Pattern != "" { if prop.Items.Pattern == "[ \\r\\n\\t\\S]+" { sb.WriteString(fmt.Sprintf("%sif %s == \"\" {\n", loopIndent, itemValExpr)) sb.WriteString(fmt.Sprintf("%s\treturn fmt.Errorf(\"field '%s[%%d]' does not match pattern '%s'\", i)\n", loopIndent, pk, escapePattern(prop.Items.Pattern))) sb.WriteString(fmt.Sprintf("%s}\n", loopIndent)) } else if prop.Items.Pattern == "\\S*" { - // Skip unanchored \S* check } else { varName := fmt.Sprintf("rx_%s_%s_items", k, toGoName(pk)) sb.WriteString(fmt.Sprintf("%sif !%s.MatchString(%s) {\n", loopIndent, varName, itemValExpr)) @@ -258,7 +236,6 @@ func generateValidate(schema *Schema, path string) error { } } - // Check nested object validation for slice item if prop.Items.Ref != "" && refName(prop.Items.Ref) != "links" { sb.WriteString(fmt.Sprintf("%sif err := %s.Validate(); err != nil {\n", loopIndent, itemName)) sb.WriteString(fmt.Sprintf("%s\treturn fmt.Errorf(\"field '%s[%%d]' is invalid: %%w\", i, err)\n", loopIndent, pk)) diff --git a/cmd/gqlgenfix/main.go b/cmd/gqlgenfix/main.go index ec046cecb..8d1d9b23a 100644 --- a/cmd/gqlgenfix/main.go +++ b/cmd/gqlgenfix/main.go @@ -1,5 +1,3 @@ -// Command gqlgenfix corrects two invalid pointer returns emitted by the pinned -// gqlgen version for Loom's generated GraphQL types. package main import ( diff --git a/cmd/loom-data/load.go b/cmd/loom-data/load.go new file mode 100644 index 000000000..f20b6a4bf --- /dev/null +++ b/cmd/loom-data/load.go @@ -0,0 +1,672 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "math" + "net/url" + "os" + "regexp" + "runtime" + "runtime/debug" + "runtime/pprof" + "runtime/trace" + "strconv" + "strings" + + arangofactory "github.com/calypr/loom/internal/backends/arango" + postgres "github.com/calypr/loom/internal/backends/postgres" + "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/catalog" + publication "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/ingest" +) + +type backendTag string + +const ( + backendArango backendTag = "arango" + backendPostgres backendTag = "postgres" +) + +const ( + defaultURL = "http://127.0.0.1:8529" + defaultDatabase = "fhir_proto" + defaultProject = "ARANGODB_PROTO" + defaultSchema = "schemas/graph-fhir.json" + defaultMetaDir = "META" +) + +const pgDSNEnv = "LOOM_DATA_PG_DSN" + +var ( + ErrBackendRequired = errors.New("loom-data: backend is required") + ErrBackendUnknown = errors.New("loom-data: backend must be arango or postgres") + ErrProjectRequired = errors.New("loom-data: --project is required for the postgres backend") + ErrPGDSNRequired = errors.New("loom-data: a PostgreSQL DSN is required: set --pg-dsn or LOOM_DATA_PG_DSN, never both") + ErrPGDSNConflict = errors.New("loom-data: --pg-dsn and LOOM_DATA_PG_DSN cannot both be set; exactly one source is required") + ErrPGSchemaRequired = errors.New("loom-data: --pg-schema is required for the postgres backend") + ErrPGManifestSchemaRequired = errors.New("loom-data: --pg-manifest-schema is required for the postgres backend (the dataset_release series lives on its own schema; see ingest.PGSourceOptions.ManifestSchema)") + ErrPGTruncateForbidden = errors.New("loom-data: --truncate is not permitted for a postgres generation load") + ErrPGApplicationNameInvalid = errors.New("loom-data: --pg-application-name must be 1-63 bytes of letters, digits, dot, underscore, or hyphen, starting with a letter or digit") + ErrPGMaxConnsInvalid = errors.New("loom-data: --pg-max-conns must be between 0 and 2147483647") +) + +var pgApplicationNameRE = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$`) + +func inspectLoadGenerationArgs(args []string) (backend string, backendSet bool, help bool) { + for i := 0; i < len(args); i++ { + arg := args[i] + switch { + case arg == "--": + return + case arg == "-h" || arg == "--help": + help = true + case strings.HasPrefix(arg, "--backend="): + backend, backendSet = arg[len("--backend="):], true + case strings.HasPrefix(arg, "-backend="): + backend, backendSet = arg[len("-backend="):], true + case arg == "--backend" || arg == "-backend": + backendSet = true + if i+1 < len(args) { + backend = args[i+1] + i++ + } + } + } + return +} + +func normalizeBackendTag(raw string, set bool) (backendTag, error) { + value := strings.ToLower(strings.TrimSpace(raw)) + if !set || value == "" { + return "", fmt.Errorf("%w: pass --backend arango or postgres; the backend is never inferred from connection settings", ErrBackendRequired) + } + switch value { + case string(backendArango): + return backendArango, nil + case string(backendPostgres): + return backendPostgres, nil + default: + return "", fmt.Errorf("%w: got %q", ErrBackendUnknown, value) + } +} + +type loadRunner struct { + loadArango func(context.Context, ingest.LoadOptions) (ingest.LoadSummary, error) + loadPostgres func(context.Context, ingest.LoadOptions, ingest.PGSourceOptions) (ingest.LoadSummary, error) + requireDatabase func(context.Context, string, string) error + openCatalogWriters func(ctx context.Context, dsn, schema, manifestSchema, application string) (ingest.GenerationCatalogWriters, func() error, error) + verifyStaged func(context.Context, publication.Ref, string, string, string) error + printSummary func(any) error + printNote func(string) +} + +func defaultRunner() loadRunner { + return loadRunner{ + loadArango: ingest.Load, + loadPostgres: ingest.LoadPostgresGeneration, + requireDatabase: requireDatabaseExists, + openCatalogWriters: openCatalogWriters, + verifyStaged: verifyStagedManifest, + printSummary: printJSON, + printNote: func(note string) { fmt.Fprintln(os.Stderr, note) }, + } +} + +func dispatchLoadGeneration(ctx context.Context, args []string, runner loadRunner) error { + rawBackend, backendSet, help := inspectLoadGenerationArgs(args) + tag, tagErr := normalizeBackendTag(rawBackend, backendSet) + if help && tagErr != nil { + usage() + return nil + } + if tagErr != nil { + return tagErr + } + switch tag { + case backendArango: + return runArangoLoadGeneration(ctx, args, runner) + case backendPostgres: + return runPostgresLoadGeneration(ctx, args, runner) + default: + return fmt.Errorf("%w: refusing to dispatch with unregistered backend %q", ErrBackendUnknown, string(tag)) + } +} + +type arangoLoadCommandConfig struct { + Options ingest.LoadOptions + + CPUProfile string + MemProfile string + TraceProfile string + BlockProfile string + MemoryLimit string + MemoryBytes int64 + + Generation string +} + +func configureLoadFlagsCore(fs *flag.FlagSet, options *ingest.LoadOptions, memoryLimit, cpuProfile, memProfile, traceProfile, blockProfile *string) { + fs.StringVar(cpuProfile, "cpu-profile", "", "Write CPU profile to file") + fs.StringVar(memProfile, "mem-profile", "", "Write heap profile to file at end of run") + fs.StringVar(traceProfile, "trace-profile", "", "Write runtime trace to file") + fs.StringVar(blockProfile, "block-profile", "", "Write block profile to file at end of run") + fs.StringVar(memoryLimit, "memory-limit", "4GiB", "Go memory limit for this load, for example 4GiB; 0 disables the soft limit") + fs.StringVar(&options.Schema, "schema", defaultSchema, "graph-fhir JSON schema") + fs.StringVar(&options.MetaDir, "meta-dir", defaultMetaDir, "Directory containing META/*.ndjson") + fs.StringVar(&options.AuthResourcePath, "auth-resource-path", "", "Optional auth resource path copied onto vertex data, for example EllrottLab-GDC_Data") + fs.IntVar(&options.BatchSize, "batch-size", 5000, "Bulk insert batch size") + fs.IntVar(&options.ProgressEvery, "progress-every", 50000, "Emit progress every N input rows") + fs.IntVar(&options.WriterCount, "writers", 2, "Concurrent writer goroutines") + fs.IntVar(&options.WorkerCount, "workers", 2, "Concurrent parse/profile workers") + fs.IntVar(&options.LineQueueSize, "line-queue-size", 1024, "Buffered input lines") + fs.IntVar(&options.WriteQueueSize, "write-queue-size", 8, "Buffered write batches") + profileLimits := catalog.DefaultProfileLimits() + fs.IntVar(&options.CatalogLimits.MaxFields, "max-catalog-fields", profileLimits.MaxFields, "Maximum retained catalog fields per resource") + fs.IntVar(&options.CatalogLimits.MaxDistinctValuesPerField, "max-distinct-values", profileLimits.MaxDistinctValuesPerField, "Maximum retained distinct values per field") + fs.IntVar(&options.CatalogLimits.MaxDistinctValueBytes, "max-distinct-value-bytes", profileLimits.MaxDistinctValueBytes, "Maximum retained bytes per catalog value") + fs.IntVar(&options.CatalogLimits.MaxPivotColumnsPerField, "max-pivot-columns", profileLimits.MaxPivotColumnsPerField, "Maximum retained pivot columns per field") + fs.IntVar(&options.CatalogLimits.MaxExtensionValuesPerField, "max-extension-values", profileLimits.MaxExtensionValuesPerField, "Maximum retained extension observations per field") + fs.IntVar(&options.CatalogLimits.MaxShapePlans, "max-shape-plans", profileLimits.MaxShapePlans, "Maximum cached payload shape plans") + fs.BoolVar(&options.FailFast, "fail-fast", false, "Stop on the first decode, validation, or edge conversion error") + fs.BoolVar(&options.UseGeneric, "use-generic", false, "Use the generic jsonschema + jsonschemagraph validator and extractor") +} + +func configureLoadFlags(fs *flag.FlagSet, config *arangoLoadCommandConfig) { + configureLoadFlagsCore(fs, &config.Options, &config.MemoryLimit, &config.CPUProfile, &config.MemProfile, &config.TraceProfile, &config.BlockProfile) + fs.StringVar(&config.Options.URL, "url", defaultURL, "Backend base URL") + fs.StringVar(&config.Options.Database, "database", defaultDatabase, "Backend database") + fs.StringVar(&config.Options.Project, "project", defaultProject, "Project label") + fs.StringVar(&config.Options.WriteAPI, "write-api", "import", "Bulk write API: import or document") + fs.StringVar(&config.Generation, "generation", "", "Required opaque immutable dataset generation identifier") + fs.StringVar(new(string), "backend", "", "Required backend tag: arango or postgres (validated before any flag below)") +} + +func parseArangoLoadCommand(args []string, errorHandling flag.ErrorHandling) (arangoLoadCommandConfig, error) { + fs := flag.NewFlagSet("load-generation", errorHandling) + fs.Usage = func() { + fmt.Fprintln(os.Stderr, "loom-data load-generation (arango): identical flag, output, and exit contract to arango-fhir-proto load-generation.") + fs.PrintDefaults() + } + config := arangoLoadCommandConfig{} + configureLoadFlags(fs, &config) + if err := fs.Parse(args); err != nil { + return arangoLoadCommandConfig{}, err + } + if err := config.parseMemoryLimit(); err != nil { + return arangoLoadCommandConfig{}, err + } + if config.Generation == "" { + return arangoLoadCommandConfig{}, fmt.Errorf("--generation is required for load-generation") + } + if config.Options.Truncate { + return arangoLoadCommandConfig{}, fmt.Errorf("--truncate=true is not permitted for load-generation") + } + ref, err := publication.NewRef(config.Options.Project, config.Generation) + if err != nil { + return arangoLoadCommandConfig{}, fmt.Errorf("invalid --generation for load-generation: %w", err) + } + config.Options.Dataset = &ref + return config, nil +} + +func (config *arangoLoadCommandConfig) parseMemoryLimit() error { + bytes, err := parseMemoryBytes(config.MemoryLimit) + if err != nil { + return fmt.Errorf("invalid --memory-limit: %w", err) + } + config.MemoryBytes = bytes + return nil +} + +func runArangoLoadGeneration(ctx context.Context, args []string, runner loadRunner) error { + config, err := parseArangoLoadCommand(args, flag.ExitOnError) + if err != nil { + return err + } + if err := runner.requireDatabase(ctx, config.Options.URL, config.Options.Database); err != nil { + return err + } + summary, err := runLoadWithProfiles(ctx, + profileSettings{CPUProfile: config.CPUProfile, MemProfile: config.MemProfile, TraceProfile: config.TraceProfile, BlockProfile: config.BlockProfile}, + config.MemoryBytes, + func(loadCtx context.Context) (ingest.LoadSummary, error) { + return runner.loadArango(loadCtx, config.Options) + }) + if err != nil { + return err + } + return runner.printSummary(summary) +} + +func requireDatabaseExists(ctx context.Context, url, database string) error { + databaseExists, err := arangofactory.DatabaseExists(ctx, url, database) + if err != nil { + return fmt.Errorf("check existing database: %w", err) + } + if !databaseExists { + return fmt.Errorf("refusing to create missing database %q", database) + } + return nil +} + +type postgresLoadCommandConfig struct { + Options ingest.LoadOptions + PG ingest.PGSourceOptions + + MemoryLimit string + MemoryBytes int64 + + CPUProfile string + MemProfile string + TraceProfile string + BlockProfile string + + Generation string + Truncate bool + DSNFromFlag string + ResolvedDSN string + pgMaxConnsInt int +} + +func configurePostgresLoadFlags(fs *flag.FlagSet, config *postgresLoadCommandConfig) { + configureLoadFlagsCore(fs, &config.Options, &config.MemoryLimit, &config.CPUProfile, &config.MemProfile, &config.TraceProfile, &config.BlockProfile) + fs.StringVar(&config.Options.Project, "project", "", "Project label (required)") + fs.StringVar(&config.Generation, "generation", "", "Required opaque immutable dataset generation identifier") + fs.BoolVar(&config.Truncate, "truncate", false, "Truncate before loading; always refused for a postgres generation load") + fs.StringVar(new(string), "backend", "", "Required backend tag: arango or postgres (validated before any flag below)") + fs.StringVar(&config.DSNFromFlag, "pg-dsn", "", "PostgreSQL connection string (mutually exclusive with LOOM_DATA_PG_DSN); never printed") + fs.StringVar(&config.PG.Schema, "pg-schema", "", "Required PostgreSQL schema the generation is loaded into (source series)") + fs.StringVar(&config.PG.ManifestSchema, "pg-manifest-schema", "", "Required PostgreSQL schema hosting the dataset_release series (generation manifests and the active-generation pointer); must differ from --pg-schema") + fs.IntVar(&config.pgMaxConnsInt, "pg-max-conns", 0, "PostgreSQL load pool maximum connections; zero selects the loader default (4)") + fs.DurationVar(&config.PG.AcquireTimeout, "pg-acquire-timeout", 0, "PostgreSQL pool acquire timeout; zero selects the loader default (30s)") + fs.StringVar(&config.PG.ApplicationName, "pg-application-name", "", "Optional application_name tag for load backends; never printed") + fs.IntVar(&config.PG.SemanticFlushThreshold, "semantic-flush-threshold", 0, "Semantic contribution buffer spill threshold; zero selects the loader default") +} + +func parsePostgresLoadCommand(args []string, errorHandling flag.ErrorHandling) (postgresLoadCommandConfig, error) { + fs := flag.NewFlagSet("load-generation", errorHandling) + fs.Usage = func() { + fmt.Fprintln(os.Stderr, "loom-data load-generation (postgres): one bounded load that leaves the generation STAGED (verified) and NOT activated; activation is a separate explicit operation. The DSN is never printed.") + fs.PrintDefaults() + } + config := postgresLoadCommandConfig{} + configurePostgresLoadFlags(fs, &config) + if err := fs.Parse(args); err != nil { + return postgresLoadCommandConfig{}, err + } + if config.Generation == "" { + return postgresLoadCommandConfig{}, fmt.Errorf("--generation is required for load-generation") + } + if config.Truncate { + return postgresLoadCommandConfig{}, ErrPGTruncateForbidden + } + bytes, err := parseMemoryBytes(config.MemoryLimit) + if err != nil { + return postgresLoadCommandConfig{}, fmt.Errorf("invalid --memory-limit: %w", err) + } + config.MemoryBytes = bytes + if strings.TrimSpace(config.Options.Project) == "" { + return postgresLoadCommandConfig{}, ErrProjectRequired + } + dsn, err := resolvePostgresDSN(fs, config.DSNFromFlag) + if err != nil { + return postgresLoadCommandConfig{}, err + } + config.ResolvedDSN = dsn + if strings.TrimSpace(config.PG.Schema) == "" { + return postgresLoadCommandConfig{}, ErrPGSchemaRequired + } + if strings.TrimSpace(config.PG.ManifestSchema) == "" { + return postgresLoadCommandConfig{}, ErrPGManifestSchemaRequired + } + name := strings.TrimSpace(config.PG.ApplicationName) + switch { + case name == "": + config.PG.ApplicationName = "" + case !pgApplicationNameRE.MatchString(name): + return postgresLoadCommandConfig{}, ErrPGApplicationNameInvalid + default: + config.PG.ApplicationName = name + } + if config.pgMaxConnsInt < 0 || config.pgMaxConnsInt > math.MaxInt32 { + return postgresLoadCommandConfig{}, ErrPGMaxConnsInvalid + } + config.PG.MaxConns = int32(config.pgMaxConnsInt) + ref, err := publication.NewRef(config.Options.Project, config.Generation) + if err != nil { + return postgresLoadCommandConfig{}, fmt.Errorf("invalid --generation for load-generation: %w", err) + } + config.Options.Dataset = &ref + config.Options.Truncate = false + return config, nil +} + +func resolvePostgresDSN(fs *flag.FlagSet, flagValue string) (string, error) { + explicit := false + fs.Visit(func(f *flag.Flag) { + if f.Name == "pg-dsn" { + explicit = true + } + }) + envValue := strings.TrimSpace(os.Getenv(pgDSNEnv)) + flagTrimmed := strings.TrimSpace(flagValue) + if explicit && envValue != "" { + return "", ErrPGDSNConflict + } + if explicit { + if flagTrimmed == "" { + return "", ErrPGDSNRequired + } + return flagValue, nil + } + if envValue == "" { + return "", ErrPGDSNRequired + } + return envValue, nil +} + +func runPostgresLoadGeneration(ctx context.Context, args []string, runner loadRunner) error { + config, err := parsePostgresLoadCommand(args, flag.ExitOnError) + if err != nil { + return err + } + dsn := config.ResolvedDSN + writers, closeWriters, err := runner.openCatalogWriters(ctx, dsn, config.PG.Schema, config.PG.ManifestSchema, config.PG.ApplicationName) + if err != nil { + return redactPGDiagnostic(err, dsn) + } + defer func() { _ = closeWriters() }() + config.PG.Catalog = writers + config.PG.DSN = dsn + + summary, err := runLoadWithProfiles(ctx, + profileSettings{CPUProfile: config.CPUProfile, MemProfile: config.MemProfile, TraceProfile: config.TraceProfile, BlockProfile: config.BlockProfile}, + config.MemoryBytes, + func(loadCtx context.Context) (ingest.LoadSummary, error) { + return runner.loadPostgres(loadCtx, config.Options, config.PG) + }) + if err != nil { + if summary.Dataset != nil { + runner.printNote("staged-manifest-inspect " + summaryJSON(summary)) + } + return redactPGDiagnostic(err, dsn) + } + if summary.Dataset == nil { + return errors.New("postgres generation load reported success without a dataset manifest; refusing to report success") + } + if err := runner.verifyStaged(ctx, *summary.Dataset, dsn, config.PG.Schema, config.PG.ManifestSchema); err != nil { + return redactPGDiagnostic(fmt.Errorf("verify postgres generation staged state: %w", err), dsn) + } + if err := runner.printSummary(summary); err != nil { + return err + } + runner.printNote(fmt.Sprintf("postgres generation %s/%s is STAGED (verified); not activated — activation is a separate explicit operation", + summary.Dataset.Project, summary.Dataset.Generation)) + return nil +} + +func openCatalogWriters(ctx context.Context, dsn, schema, manifestSchema, application string) (ingest.GenerationCatalogWriters, func() error, error) { + poolDSN, _ := joinApplicationName(dsn, application) + pg := ingest.PGSourceOptions{DSN: dsn, Schema: schema, ManifestSchema: manifestSchema} + runtime, err := postgres.Open(ctx, ingest.PGLoadConfig(pg, poolDSN, control.SetupNone)) + if err != nil { + return ingest.GenerationCatalogWriters{}, nil, fmt.Errorf("open postgres catalog pool: %w", err) + } + if runtime.FieldProfiles == nil || runtime.Inventory == nil || runtime.Relationships == nil || runtime.Contributions == nil { + _ = runtime.Close() + return ingest.GenerationCatalogWriters{}, nil, fmt.Errorf("postgres runtime assembled without catalog evidence ports") + } + return ingest.GenerationCatalogWriters{ + FieldProfiles: runtime.FieldProfiles, + Inventory: runtime.Inventory, + Relationships: runtime.Relationships, + Contributions: runtime.Contributions, + }, func() error { return runtime.Close() }, nil +} + +func verifyStagedManifest(ctx context.Context, ref publication.Ref, dsn, schema, manifestSchema string) error { + pg := ingest.PGSourceOptions{DSN: dsn, Schema: schema, ManifestSchema: manifestSchema} + runtime, err := postgres.Open(ctx, ingest.PGLoadConfig(pg, dsn, control.SetupNone)) + if err != nil { + return err + } + defer func() { _ = runtime.Close() }() + store := runtime.Manifests + if store == nil { + return fmt.Errorf("postgres runtime exposes no dataset manifest lifecycle port") + } + manifest, err := store.ReadManifest(ctx, ref) + if err != nil { + return err + } + if !manifest.IsStaged() { + return fmt.Errorf("dataset generation %s/%s has state %s, want STAGED", ref.Project, ref.Generation, manifest.State) + } + return nil +} + +var ( + pgURLCredentialsRE = regexp.MustCompile(`(?i)([a-z][a-z0-9+.-]*://)[^/\s:@]+:[^/\s@]+@`) + pgKeywordPasswordRE = regexp.MustCompile(`(?i)password\s*=\s*(?:'[^']*'|"[^"]*"|\S+)`) + pgKeywordPasswordValueRE = regexp.MustCompile(`(?i)password\s*=\s*(?:'([^']*)'|"([^"]*)"|(\S+))`) +) + +type redactedPGError struct { + original error + rendered string +} + +func (e *redactedPGError) Error() string { return e.rendered } +func (e *redactedPGError) Unwrap() error { return e.original } + +func redactPGDiagnostic(err error, dsn string) error { + if err == nil { + return nil + } + rendered := err.Error() + rendered = pgURLCredentialsRE.ReplaceAllString(rendered, "${1}[redacted]@") + rendered = pgKeywordPasswordRE.ReplaceAllString(rendered, "password=[redacted]") + for _, secret := range dsnSecrets(dsn) { + if secret != "" { + rendered = strings.ReplaceAll(rendered, secret, "[redacted]") + } + } + if rendered == err.Error() { + return err + } + return &redactedPGError{original: err, rendered: rendered} +} + +func dsnSecrets(dsn string) []string { + if strings.TrimSpace(dsn) == "" { + return nil + } + secrets := []string{dsn} + if parsed, err := url.Parse(dsn); err == nil && parsed.User != nil { + if info := parsed.User.String(); info != "" { + secrets = append(secrets, info) + } + if pw, ok := parsed.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + if match := pgKeywordPasswordValueRE.FindStringSubmatch(dsn); match != nil { + for _, group := range match[1:] { + if group != "" { + secrets = append(secrets, group) + break + } + } + } + return secrets +} + +func summaryJSON(summary ingest.LoadSummary) string { + data, err := json.Marshal(summary) + if err != nil { + if summary.Dataset != nil { + return fmt.Sprintf(`{"project":%q,"generation":%q}`, summary.Dataset.Project, summary.Dataset.Generation) + } + return "{}" + } + return string(data) +} + +type profileSettings struct { + CPUProfile string + MemProfile string + TraceProfile string + BlockProfile string +} + +func runLoadWithProfiles(ctx context.Context, profiles profileSettings, memoryBytes int64, load func(context.Context) (ingest.LoadSummary, error)) (summary ingest.LoadSummary, err error) { + previousMemoryLimit := int64(-1) + if memoryBytes > 0 { + previousMemoryLimit = debug.SetMemoryLimit(memoryBytes) + } + if previousMemoryLimit >= 0 { + defer debug.SetMemoryLimit(previousMemoryLimit) + } + stopProfiles, profileErr := startProfiles(profiles.CPUProfile, profiles.MemProfile, profiles.TraceProfile, profiles.BlockProfile) + if profileErr != nil { + return ingest.LoadSummary{}, profileErr + } + defer func() { + if stopErr := stopProfiles(); stopErr != nil && err == nil { + err = stopErr + } + }() + return load(ctx) +} + +func parseMemoryBytes(raw string) (int64, error) { + raw = strings.TrimSpace(raw) + if raw == "" || raw == "0" { + return 0, nil + } + lower := strings.ToLower(raw) + units := []struct { + suffix string + factor float64 + }{ + {suffix: "gib", factor: 1 << 30}, + {suffix: "gb", factor: 1e9}, + {suffix: "gi", factor: 1 << 30}, + {suffix: "g", factor: 1e9}, + {suffix: "mib", factor: 1 << 20}, + {suffix: "mb", factor: 1e6}, + {suffix: "mi", factor: 1 << 20}, + {suffix: "m", factor: 1e6}, + {suffix: "kib", factor: 1 << 10}, + {suffix: "kb", factor: 1e3}, + {suffix: "ki", factor: 1 << 10}, + {suffix: "k", factor: 1e3}, + {suffix: "b", factor: 1}, + } + number := lower + factor := float64(1) + for _, unit := range units { + if strings.HasSuffix(lower, unit.suffix) { + number = strings.TrimSpace(strings.TrimSuffix(lower, unit.suffix)) + factor = unit.factor + break + } + } + value, err := strconv.ParseFloat(number, 64) + if err != nil || value <= 0 || value*factor > math.MaxInt64 { + return 0, fmt.Errorf("expected a positive byte value such as 4GiB") + } + return int64(value * factor), nil +} + +func joinApplicationName(dsn, application string) (string, bool) { + if strings.TrimSpace(application) == "" { + return dsn, false + } + parsed, err := url.Parse(dsn) + if err != nil || (parsed.Scheme != "postgres" && parsed.Scheme != "postgresql") { + return dsn, false + } + query := parsed.Query() + if query.Get("application_name") != "" { + return dsn, true + } + query.Set("application_name", application) + parsed.RawQuery = query.Encode() + return parsed.String(), true +} + +func startProfiles(cpuPath, memPath, tracePath, blockPath string) (func() error, error) { + type closer func() error + var closers []closer + + if blockPath != "" { + runtime.SetBlockProfileRate(1) + closers = append(closers, func() error { + defer runtime.SetBlockProfileRate(0) + f, err := os.Create(blockPath) + if err != nil { + return err + } + defer f.Close() + return pprof.Lookup("block").WriteTo(f, 0) + }) + } + + if tracePath != "" { + f, err := os.Create(tracePath) + if err != nil { + return nil, err + } + if err := trace.Start(f); err != nil { + f.Close() + return nil, err + } + closers = append(closers, func() error { + trace.Stop() + return f.Close() + }) + } + + if cpuPath != "" { + f, err := os.Create(cpuPath) + if err != nil { + return nil, err + } + if err := pprof.StartCPUProfile(f); err != nil { + f.Close() + return nil, err + } + closers = append(closers, func() error { + pprof.StopCPUProfile() + return f.Close() + }) + } + + if memPath != "" { + closers = append(closers, func() error { + runtime.GC() + f, err := os.Create(memPath) + if err != nil { + return err + } + defer f.Close() + return pprof.WriteHeapProfile(f) + }) + } + + return func() error { + var firstErr error + for i := len(closers) - 1; i >= 0; i-- { + if err := closers[i](); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr + }, nil +} diff --git a/cmd/loom-data/load_test.go b/cmd/loom-data/load_test.go new file mode 100644 index 000000000..f005fb149 --- /dev/null +++ b/cmd/loom-data/load_test.go @@ -0,0 +1,1453 @@ +package main + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/json" + "errors" + "flag" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/catalog" + publication "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/ingest" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +type spyState struct { + mu sync.Mutex + + calls map[string]int + seenPGOptions *ingest.PGSourceOptions + seenPGLoadOpts *ingest.LoadOptions + seenArangoOpts *ingest.LoadOptions + seenRequireDB [2]string + seenCatalogArgs [3]string + seenVerify *publication.Ref + printed []any + notes []string + closes int + + pgErr error + arangoErr error + catalogErr error + verifyErr error + pgSummary ingest.LoadSummary + arangoSummary ingest.LoadSummary +} + +func newSpyState() *spyState { + return &spyState{calls: map[string]int{}} +} + +func (s *spyState) record(name string) { + s.mu.Lock() + defer s.mu.Unlock() + s.calls[name]++ +} + +func (s *spyState) count(name string) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.calls[name] +} + +func (s *spyState) totalCalls() int { + s.mu.Lock() + defer s.mu.Unlock() + total := 0 + for _, c := range s.calls { + total += c + } + return total +} + +func (s *spyState) runner() loadRunner { + return loadRunner{ + loadArango: func(_ context.Context, opts ingest.LoadOptions) (ingest.LoadSummary, error) { + s.record("loadArango") + s.mu.Lock() + s.seenArangoOpts = &opts + err := s.arangoErr + summary := s.arangoSummary + s.mu.Unlock() + return summary, err + }, + loadPostgres: func(_ context.Context, opts ingest.LoadOptions, pg ingest.PGSourceOptions) (ingest.LoadSummary, error) { + s.record("loadPostgres") + s.mu.Lock() + s.seenPGLoadOpts = &opts + s.seenPGOptions = &pg + err := s.pgErr + summary := s.pgSummary + s.mu.Unlock() + return summary, err + }, + requireDatabase: func(_ context.Context, url, database string) error { + s.record("requireDatabase") + s.mu.Lock() + s.seenRequireDB = [2]string{url, database} + s.mu.Unlock() + return nil + }, + openCatalogWriters: func(_ context.Context, dsn, schema, manifestSchema, application string) (ingest.GenerationCatalogWriters, func() error, error) { + s.record("openCatalogWriters") + s.mu.Lock() + s.seenCatalogArgs = [3]string{dsn, schema, application} + err := s.catalogErr + s.mu.Unlock() + if err != nil { + return ingest.GenerationCatalogWriters{}, nil, err + } + return ingest.GenerationCatalogWriters{ + FieldProfiles: stubFieldProfiles{}, + Inventory: stubInventory{}, + Relationships: stubRelationships{}, + Contributions: stubContributions{}, + }, func() error { + s.record("closeCatalogWriters") + return nil + }, nil + }, + verifyStaged: func(_ context.Context, ref publication.Ref, dsn, schema, manifestSchema string) error { + s.record("verifyStaged") + s.mu.Lock() + s.seenVerify = &ref + err := s.verifyErr + s.mu.Unlock() + return err + }, + printSummary: func(value any) error { + s.record("printSummary") + s.mu.Lock() + s.printed = append(s.printed, value) + s.mu.Unlock() + return nil + }, + printNote: func(note string) { + s.mu.Lock() + s.notes = append(s.notes, note) + s.mu.Unlock() + }, + } +} + +type stubFieldProfiles struct{} + +func (stubFieldProfiles) WriteFieldProfile(context.Context, control.FieldProfileScope, []catalog.FieldCatalogDocument, int) error { + return nil +} + +type stubInventory struct{} + +func (stubInventory) RebuildResourceInventory(context.Context, catalog.ResourceInventoryRebuildOptions) (catalog.ResourceInventoryRebuildSummary, error) { + return catalog.ResourceInventoryRebuildSummary{}, nil +} + +type stubRelationships struct{} + +func (stubRelationships) DiscoverRelationshipObservations(context.Context, catalog.RelationshipObservationOptions) (catalog.RelationshipObservationResult, error) { + return catalog.RelationshipObservationResult{}, nil +} + +type stubContributions struct{} + +func (stubContributions) WriteSemanticInventoryContributions(context.Context, []catalog.SemanticInventoryContribution, int) error { + return nil +} + +func captureOutput(t *testing.T, fn func()) (string, string) { + t.Helper() + origOut, origErr := os.Stdout, os.Stderr + outR, outW, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + errR, errW, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stdout, os.Stderr = outW, errW + var wg sync.WaitGroup + var outBuf, errBuf bytes.Buffer + wg.Add(2) + go func() { defer wg.Done(); _, _ = outBuf.ReadFrom(outR) }() + go func() { defer wg.Done(); _, _ = errBuf.ReadFrom(errR) }() + fn() + _ = outW.Close() + _ = errW.Close() + wg.Wait() + os.Stdout, os.Stderr = origOut, origErr + return outBuf.String(), errBuf.String() +} + +func mustRef(t *testing.T, project, generation string) publication.Ref { + t.Helper() + ref, err := publication.NewRef(project, generation) + if err != nil { + t.Fatalf("NewRef(%q,%q): %v", project, generation, err) + } + return ref +} + +func pgBaseArgs() []string { + return []string{ + "--backend=postgres", + "--project=prj-cli", + "--generation=gen-cli", + "--pg-dsn=postgresql://cliuser:SENTINELPW999@127.0.0.1:1/clidb?sslmode=disable", + "--pg-schema=verify_clidisp_s_unit", + "--pg-manifest-schema=verify_clidisp_sm_unit", + } +} + +func arangoBaseArgs() []string { + return []string{"--backend=arango", "--generation=gen-cli"} +} + +func TestInspectLoadGenerationArgs(t *testing.T) { + cases := []struct { + name string + args []string + wantTag string + wantSet bool + wantHelp bool + }{ + {"equals form", []string{"--backend=postgres"}, "postgres", true, false}, + {"single dash equals", []string{"-backend=arango"}, "arango", true, false}, + {"space form", []string{"--backend", "postgres"}, "postgres", true, false}, + {"flag after backend", []string{"--project=p", "--backend=postgres"}, "postgres", true, false}, + {"terminator stops scan", []string{"--", "--backend=postgres"}, "", false, false}, + {"missing", []string{"--project=p"}, "", false, false}, + {"help", []string{"-h"}, "", false, true}, + {"help with backend", []string{"--backend=postgres", "--help"}, "postgres", true, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tag, set, help := inspectLoadGenerationArgs(tc.args) + if tag != tc.wantTag || set != tc.wantSet || help != tc.wantHelp { + t.Fatalf("inspect(%v) = (%q,%v,%v), want (%q,%v,%v)", tc.args, tag, set, help, tc.wantTag, tc.wantSet, tc.wantHelp) + } + }) + } +} + +func TestNormalizeBackendTag(t *testing.T) { + if tag, err := normalizeBackendTag(" PostGRES ", true); err != nil || tag != backendPostgres { + t.Fatalf("postgres normalization = (%q,%v)", tag, err) + } + if tag, err := normalizeBackendTag("ARANGO", true); err != nil || tag != backendArango { + t.Fatalf("arango normalization = (%q,%v)", tag, err) + } + if _, err := normalizeBackendTag("", true); !errors.Is(err, ErrBackendRequired) { + t.Fatalf("empty tag error = %v, want ErrBackendRequired", err) + } + if _, err := normalizeBackendTag("postgres", false); !errors.Is(err, ErrBackendRequired) { + t.Fatalf("unset tag error = %v, want ErrBackendRequired", err) + } + _, err := normalizeBackendTag("cockroachdb", true) + if !errors.Is(err, ErrBackendUnknown) { + t.Fatalf("unknown tag error = %v, want ErrBackendUnknown", err) + } +} + +func TestBackendTagValidatedBeforeDialect(t *testing.T) { + t.Setenv(pgDSNEnv, "") + spy := newSpyState() + args := []string{ + "--backend=cockroachdb", + "--pg-schema=", + "--project=", + "--generation=gen-x", + } + _, _ = captureOutput(t, func() { + err := dispatchLoadGeneration(context.Background(), args, spy.runner()) + if !errors.Is(err, ErrBackendUnknown) { + t.Fatalf("error = %v, want ErrBackendUnknown ahead of every dialect guard", err) + } + if errors.Is(err, ErrPGSchemaRequired) || errors.Is(err, ErrProjectRequired) || errors.Is(err, ErrPGDSNRequired) { + t.Fatalf("dialect guard fired before the tag check: %v", err) + } + }) + if spy.totalCalls() != 0 { + t.Fatalf("misrouted backend touched %d seams, want 0", spy.totalCalls()) + } + + _, _ = captureOutput(t, func() { + control := append([]string{}, args...) + control[0] = "--backend=postgres" + err := dispatchLoadGeneration(context.Background(), control, spy.runner()) + if !errors.Is(err, ErrProjectRequired) { + t.Fatalf("control error = %v, want ErrProjectRequired", err) + } + }) +} + +func TestBackendRequiredRefusesEntireInvocation(t *testing.T) { + t.Setenv(pgDSNEnv, "postgresql://env-user/envpass@127.0.0.1:1/envdb") + spy := newSpyState() + _, _ = captureOutput(t, func() { + err := dispatchLoadGeneration(context.Background(), + []string{"--project=prj-a", "--generation=gen-a", "--pg-schema=s", "--pg-manifest-schema=m"}, spy.runner()) + if !errors.Is(err, ErrBackendRequired) { + t.Fatalf("error = %v, want ErrBackendRequired", err) + } + }) + if spy.totalCalls() != 0 { + t.Fatalf("backend-required refusal touched %d seams, want 0", spy.totalCalls()) + } +} + +func TestHelpWithoutBackendPrintsCombinedUsage(t *testing.T) { + spy := newSpyState() + _, stderr := captureOutput(t, func() { + if err := dispatchLoadGeneration(context.Background(), []string{"-h"}, spy.runner()); err != nil { + t.Fatalf("-h without a tag: %v", err) + } + }) + for _, want := range []string{"--backend arango", "--backend postgres", "STAGED"} { + if !strings.Contains(stderr, want) { + t.Fatalf("combined usage must mention %q, got %q", want, stderr) + } + } +} + +func TestDispatchPostgresUsesOnlyPGLoader(t *testing.T) { + t.Setenv(pgDSNEnv, "") + spy := newSpyState() + injected := errors.New("injected postgres loader failure") + spy.pgErr = injected + + _, stderr := captureOutput(t, func() { + err := dispatchLoadGeneration(context.Background(), pgBaseArgs(), spy.runner()) + if !errors.Is(err, injected) { + t.Fatalf("error = %v, want the injected PG error preserved by identity", err) + } + }) + if spy.count("loadPostgres") != 1 { + t.Fatalf("loadPostgres calls = %d, want 1", spy.count("loadPostgres")) + } + if spy.count("loadArango") != 0 || spy.count("requireDatabase") != 0 { + t.Fatalf("PG invocation touched Arango seams: %v", spy.calls) + } + if spy.count("openCatalogWriters") != 1 { + t.Fatalf("catalog wiring calls = %d, want 1", spy.count("openCatalogWriters")) + } + if spy.count("closeCatalogWriters") != 1 { + t.Fatalf("catalog pool must be closed after the failure") + } + if spy.count("verifyStaged") != 0 || spy.count("printSummary") != 0 { + t.Fatalf("failed load must not verify or print success") + } + if strings.Contains(stderr, "SENTINELPW999") { + t.Fatalf("stderr leaked the DSN password: %q", stderr) + } + if spy.seenPGOptions == nil || spy.seenPGOptions.DSN == "" || !spy.seenPGOptions.Catalog.Complete() { + t.Fatalf("loader received incomplete PGSourceOptions: %+v", spy.seenPGOptions) + } +} + +func TestDispatchArangoUsesExistingImplementation(t *testing.T) { + t.Setenv(pgDSNEnv, "") + spy := newSpyState() + ref := mustRef(t, "ARANGODB_PROTO", "gen-cli") + spy.arangoSummary = ingest.LoadSummary{Files: 1, VerticesInserted: 7, Dataset: &ref} + + _, _ = captureOutput(t, func() { + if err := dispatchLoadGeneration(context.Background(), arangoBaseArgs(), spy.runner()); err != nil { + t.Fatalf("arango dispatch: %v", err) + } + }) + if spy.count("loadArango") != 1 || spy.count("requireDatabase") != 1 { + t.Fatalf("arango route must use the existing implementation path: %v", spy.calls) + } + if spy.count("loadPostgres") != 0 || spy.count("openCatalogWriters") != 0 || spy.count("verifyStaged") != 0 { + t.Fatalf("arango invocation touched PG seams: %v", spy.calls) + } + if spy.seenRequireDB != [2]string{defaultURL, defaultDatabase} { + t.Fatalf("requireDatabase gate args = %v, want existing CLI defaults", spy.seenRequireDB) + } + if spy.seenArangoOpts == nil || spy.seenArangoOpts.Dataset == nil { + t.Fatalf("arango options lost the dataset reference") + } + if spy.seenArangoOpts.WriteAPI != "import" || spy.seenArangoOpts.BatchSize != 5000 || spy.seenArangoOpts.WorkerCount != 2 { + t.Fatalf("arango defaults drifted from the existing contract: %+v", spy.seenArangoOpts) + } + if spy.count("printSummary") != 1 { + t.Fatalf("success must print the summary once, got %v", spy.calls) + } +} + +func TestDispatchPostgresSuccessVerifiesStagedAndAnnouncesIt(t *testing.T) { + t.Setenv(pgDSNEnv, "") + spy := newSpyState() + ref := mustRef(t, "prj-cli", "gen-cli") + spy.pgSummary = ingest.LoadSummary{Files: 2, VerticesInserted: 5, EdgesInserted: 1, Dataset: &ref} + + _, _ = captureOutput(t, func() { + if err := dispatchLoadGeneration(context.Background(), pgBaseArgs(), spy.runner()); err != nil { + t.Fatalf("postgres dispatch: %v", err) + } + }) + if spy.count("verifyStaged") != 1 { + t.Fatalf("STAGED verification must run once: %v", spy.calls) + } + if spy.seenVerify == nil || *spy.seenVerify != ref { + t.Fatalf("verification read back the wrong manifest: %v", spy.seenVerify) + } + if spy.count("printSummary") != 1 { + t.Fatalf("summary must print once") + } + var announced bool + for _, note := range spy.notes { + if strings.Contains(note, "STAGED") && strings.Contains(note, "not activated") { + announced = true + } + } + if !announced { + t.Fatalf("success note must say STAGED and must not imply activation, got %q", spy.notes) + } + if spy.seenPGOptions == nil || spy.seenPGOptions.DSN != pgBaseArgs()[3][len("--pg-dsn="):] { + t.Fatalf("loader DSN not passed through: %+v", spy.seenPGOptions) + } +} + +func TestDispatchPostgresVerifyFailureSuppressesSuccess(t *testing.T) { + t.Setenv(pgDSNEnv, "") + ref := mustRef(t, "prj-cli", "gen-cli") + + spy := newSpyState() + spy.pgSummary = ingest.LoadSummary{Dataset: &ref} + spy.verifyErr = errors.New("manifest is LOADING") + errFailing := captureError(t, spy) + if errFailing == nil { + t.Fatal("a failed STAGED verification must exit nonzero") + } + if spy.count("printSummary") != 0 { + t.Fatal("a generation not verified STAGED must not print a success summary") + } + + spyOK := newSpyState() + spyOK.pgSummary = ingest.LoadSummary{Dataset: &ref} + if errOK := captureError(t, spyOK); errOK != nil { + t.Fatalf("control: identical run with STAGED verification = %v, want success", errOK) + } + if spyOK.count("printSummary") != 1 { + t.Fatal("control: success path must print the summary") + } +} + +func captureError(t *testing.T, spy *spyState) error { + t.Helper() + var result error + _, _ = captureOutput(t, func() { + result = dispatchLoadGeneration(context.Background(), pgBaseArgs(), spy.runner()) + }) + return result +} + +func TestDispatchPostgresNilManifestGuard(t *testing.T) { + t.Setenv(pgDSNEnv, "") + spy := newSpyState() + spy.pgSummary = ingest.LoadSummary{} + err := captureError(t, spy) + if err == nil || !strings.Contains(err.Error(), "dataset manifest") { + t.Fatalf("nil-manifest success must be refused, got %v", err) + } + if spy.count("printSummary") != 0 || spy.count("verifyStaged") != 0 { + t.Fatalf("refused success may neither verify nor print: %v", spy.calls) + } +} + +func TestDispatchFailureSurfacesInactiveManifestForInspection(t *testing.T) { + t.Setenv(pgDSNEnv, "") + ref := mustRef(t, "prj-cli", "gen-cli") + + spy := newSpyState() + spy.pgErr = errors.New("partial load") + spy.pgSummary = ingest.LoadSummary{Dataset: &ref, ValidationErrors: 3} + _ = captureError(t, spy) + var inspect bool + for _, note := range spy.notes { + if strings.Contains(note, "staged-manifest-inspect") && strings.Contains(note, "gen-cli") { + inspect = true + } + } + if !inspect { + t.Fatalf("failure with a known manifest must print the inspection note, got %q", spy.notes) + } + + spy2 := newSpyState() + spy2.pgErr = errors.New("too early to have a manifest") + _ = captureError(t, spy2) + for _, note := range spy2.notes { + if strings.Contains(note, "staged-manifest-inspect") { + t.Fatalf("inspection note printed without a manifest reference: %q", note) + } + } +} + +func TestCrossBackendFlagsAreRefusedWithoutEcho(t *testing.T) { + t.Setenv(pgDSNEnv, "") + _, _ = captureOutput(t, func() { + _, err := parsePostgresLoadCommand(append(pgBaseArgs(), "--url=http://arango-host-secret"), flag.ContinueOnError) + if err == nil || !strings.Contains(err.Error(), "-url") { + t.Fatalf("postgres route must refuse --url, got %v", err) + } + if strings.Contains(err.Error(), "arango-host-secret") { + t.Fatalf("parse refusal echoed the flag value: %v", err) + } + }) + _, _ = captureOutput(t, func() { + _, err := parseArangoLoadCommand(append(arangoBaseArgs(), "--pg-dsn=pgsecretvalue"), flag.ContinueOnError) + if err == nil || !strings.Contains(err.Error(), "-pg-dsn") { + t.Fatalf("arango route must refuse --pg-dsn, got %v", err) + } + if strings.Contains(err.Error(), "pgsecretvalue") { + t.Fatalf("parse refusal echoed the flag value: %v", err) + } + }) + _, _ = captureOutput(t, func() { + if _, err := parseArangoLoadCommand(append(arangoBaseArgs(), "--url=http://arango-host"), flag.ContinueOnError); err != nil { + t.Fatalf("control: arango must accept --url: %v", err) + } + }) +} + +func TestTypedRefusalsWithAliveControls(t *testing.T) { + t.Setenv(pgDSNEnv, "") + cases := []struct { + name string + args func() []string + sentinel error + envDSN string + controlOK func() []string + }{ + { + name: "project required", + args: func() []string { + return []string{"--backend=postgres", "--generation=gen-a", "--pg-dsn=postgresql://h/db", "--pg-schema=s", "--pg-manifest-schema=m"} + }, + sentinel: ErrProjectRequired, + controlOK: func() []string { + return pgBaseArgs() + }, + }, + { + name: "dsn required", + args: func() []string { + return []string{"--backend=postgres", "--project=p", "--generation=gen-a", "--pg-schema=s", "--pg-manifest-schema=m"} + }, + sentinel: ErrPGDSNRequired, + controlOK: func() []string { + a := []string{"--backend=postgres", "--project=p", "--generation=gen-a", "--pg-schema=s", "--pg-manifest-schema=m", "--pg-dsn=postgresql://h/db"} + return a + }, + }, + { + name: "dsn conflict", + args: func() []string { + return []string{"--backend=postgres", "--project=p", "--generation=gen-a", "--pg-schema=s", "--pg-manifest-schema=m", "--pg-dsn=postgresql://h/db"} + }, + sentinel: ErrPGDSNConflict, + envDSN: "postgresql://env-host/envdb", + controlOK: func() []string { + return []string{"--backend=postgres", "--project=p", "--generation=gen-a", "--pg-schema=s", "--pg-manifest-schema=m", "--pg-dsn=postgresql://h/db"} + }, + }, + { + name: "schema required", + args: func() []string { + return []string{"--backend=postgres", "--project=p", "--generation=gen-a", "--pg-dsn=postgresql://h/db"} + }, + sentinel: ErrPGSchemaRequired, + controlOK: func() []string { return pgBaseArgs() }, + }, + { + name: "manifest schema required", + args: func() []string { + return []string{"--backend=postgres", "--project=p", "--generation=gen-a", "--pg-dsn=postgresql://h/db", "--pg-schema=s"} + }, + sentinel: ErrPGManifestSchemaRequired, + controlOK: func() []string { return pgBaseArgs() }, + }, + { + name: "truncate forbidden", + args: func() []string { return append(pgBaseArgs(), "--truncate=true") }, + sentinel: ErrPGTruncateForbidden, + controlOK: func() []string { return append(pgBaseArgs(), "--truncate=false") }, + }, + { + name: "application name bounded", + args: func() []string { return append(pgBaseArgs(), "--pg-application-name=bad name;password=x") }, + sentinel: ErrPGApplicationNameInvalid, + controlOK: func() []string { return append(pgBaseArgs(), "--pg-application-name=verify_clidisp-ok.1") }, + }, + { + name: "max conns negative", + args: func() []string { return append(pgBaseArgs(), "--pg-max-conns=-1") }, + sentinel: ErrPGMaxConnsInvalid, + controlOK: func() []string { return append(pgBaseArgs(), "--pg-max-conns=4") }, + }, + { + name: "max conns int32 overflow", + args: func() []string { return append(pgBaseArgs(), "--pg-max-conns=5000000000") }, + sentinel: ErrPGMaxConnsInvalid, + controlOK: func() []string { return append(pgBaseArgs(), "--pg-max-conns=2147483647") }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + spy := newSpyState() + if tc.envDSN != "" { + t.Setenv(pgDSNEnv, tc.envDSN) + } else { + t.Setenv(pgDSNEnv, "") + } + err := captureErrorArgs(t, spy, tc.args()) + if !errors.Is(err, tc.sentinel) { + t.Fatalf("error = %v, want %v", err, tc.sentinel) + } + if spy.totalCalls() != 0 { + t.Fatalf("refusal touched %d seams, want 0: %v", spy.totalCalls(), spy.calls) + } + if tc.controlOK != nil { + t.Setenv(pgDSNEnv, "") + spyControl := newSpyState() + controlRef := mustRef(t, "prj-cli", "gen-cli") + spyControl.pgSummary = ingest.LoadSummary{Dataset: &controlRef} + if err := captureErrorArgs(t, spyControl, tc.controlOK()); err != nil { + t.Fatalf("control lost the guard's effect: %v", err) + } + if spyControl.count("loadPostgres") != 1 { + t.Fatalf("control did not reach the loader: %v", spyControl.calls) + } + } + }) + } +} + +func captureErrorArgs(t *testing.T, spy *spyState, args []string) error { + t.Helper() + var result error + _, _ = captureOutput(t, func() { + result = dispatchLoadGeneration(context.Background(), args, spy.runner()) + }) + return result +} + +func TestApplicationNamePassesThroughToLoader(t *testing.T) { + t.Setenv(pgDSNEnv, "") + spy := newSpyState() + ref := mustRef(t, "prj-cli", "gen-cli") + spy.pgSummary = ingest.LoadSummary{Dataset: &ref} + args := append(pgBaseArgs(), "--pg-application-name=verify_clidisp_app") + if err := captureErrorArgs(t, spy, args); err != nil { + t.Fatalf("dispatch: %v", err) + } + if spy.seenPGOptions == nil || spy.seenPGOptions.ApplicationName != "verify_clidisp_app" { + t.Fatalf("application name not passed to the loader: %+v", spy.seenPGOptions) + } + if spy.seenCatalogArgs[2] != "verify_clidisp_app" { + t.Fatalf("catalog pool did not receive the application name: %v", spy.seenCatalogArgs) + } +} + +func TestApplicationNamePaddedUsesTrimmedEverywhere(t *testing.T) { + t.Setenv(pgDSNEnv, "") + spy := newSpyState() + ref := mustRef(t, "prj-cli", "gen-cli") + spy.pgSummary = ingest.LoadSummary{Dataset: &ref} + args := append(pgBaseArgs(), "--pg-application-name= verify_clidisp_pad ") + if err := captureErrorArgs(t, spy, args); err != nil { + t.Fatalf("padded application name must validate: %v", err) + } + if spy.seenPGOptions == nil || spy.seenPGOptions.ApplicationName != "verify_clidisp_pad" { + t.Fatalf("loader saw a non-trimmed application name: %+v", spy.seenPGOptions) + } + if spy.seenCatalogArgs[2] != "verify_clidisp_pad" { + t.Fatalf("catalog pool saw a non-trimmed application name: %q", spy.seenCatalogArgs[2]) + } +} + +func TestArangoGenerationInvariantsPreserved(t *testing.T) { + t.Setenv(pgDSNEnv, "") + _, _ = captureOutput(t, func() { + if _, err := parseArangoLoadCommand([]string{"--backend=arango", "--project=prj-a"}, flag.ContinueOnError); err == nil || + !strings.Contains(err.Error(), "--generation is required for load-generation") { + t.Fatalf("missing --generation must refuse with the existing message, got %v", err) + } + if _, err := parseArangoLoadCommand([]string{"--backend=arango", "--project=prj-a", "--generation= generation-a"}, flag.ContinueOnError); err == nil || + !strings.Contains(err.Error(), "invalid --generation for load-generation") { + t.Fatalf("invalid --generation must refuse with the existing message, got %v", err) + } + config, err := parseArangoLoadCommand(arangoBaseArgs(), flag.ContinueOnError) + if err != nil { + t.Fatalf("baseline parse: %v", err) + } + if config.Options.Dataset == nil || config.Options.Dataset.Project != defaultProject { + t.Fatalf("dataset reference wiring drifted: %+v", config.Options.Dataset) + } + }) +} + +func TestJoinApplicationName(t *testing.T) { + joined, tagged := joinApplicationName("postgresql://user@host:5432/db?sslmode=disable", "loom-cli") + if !tagged || !strings.Contains(joined, "application_name=loom-cli") { + t.Fatalf("URL-form DSN not tagged: %q %v", joined, tagged) + } + if _, tagged := joinApplicationName("host=h dbname=d", "loom-cli"); tagged { + t.Fatal("keyword/value DSN must stay untagged rather than a guessed syntax") + } + existing, tagged := joinApplicationName("postgresql://user@host/db?application_name=keepme", "loom-cli") + if !tagged || !strings.Contains(existing, "application_name=keepme") { + t.Fatalf("explicit application_name must be preserved: %q", existing) + } +} + +const unitDSNPassword = "SENTINELPW999" +const unitDSNKeywordSecret = "KEYSECRET123" + +func TestRedactPGDiagnosticRemovesCredentials(t *testing.T) { + sentinel := errors.New("store sentinel") + + urlDSN := fmt.Sprintf("postgresql://cliuser:%s@bad host/clidb", unitDSNPassword) + raw := fmt.Errorf("%w: cannot parse `%s`", sentinel, urlDSN) + redacted := redactPGDiagnostic(raw, urlDSN) + if strings.Contains(redacted.Error(), unitDSNPassword) { + t.Fatalf("URL-form credential survived redaction: %q", redacted.Error()) + } + if !strings.Contains(raw.Error(), unitDSNPassword) { + t.Fatal("alive control broken: fixture no longer carries the credential, so the redaction test is vacuous") + } + if !errors.Is(redacted, sentinel) { + t.Fatalf("redaction must preserve errors.Is identity, got %v", redacted) + } + underlying, ok := redacted.(*redactedPGError) + if !ok || underlying.original != raw { + t.Fatalf("redacted diagnostic must wrap the original, got %T", redacted) + } + + keywordDSN := fmt.Sprintf("host=db.example user=u password=%s dbname=x", unitDSNKeywordSecret) + rawKW := fmt.Errorf("%w: failed to connect to `%s`", sentinel, keywordDSN) + redactedKW := redactPGDiagnostic(rawKW, keywordDSN) + if strings.Contains(redactedKW.Error(), unitDSNKeywordSecret) { + t.Fatalf("keyword/value credential survived redaction: %q", redactedKW.Error()) + } + if !strings.Contains(rawKW.Error(), unitDSNKeywordSecret) { + t.Fatal("alive control broken: keyword fixture no longer carries the credential") + } + if !errors.Is(redactedKW, sentinel) { + t.Fatal("keyword redaction must preserve classification") + } + + plain := errors.New("nothing secret here") + if got := redactPGDiagnostic(plain, urlDSN); got != plain { + t.Fatalf("credential-free error must pass through unchanged, got %v", got) + } +} + +var ( + buildOnce sync.Once + loomDataBin string + arangoProtoBin string + buildErr error +) + +func builtBinaries(t *testing.T) (string, string) { + t.Helper() + buildOnce.Do(func() { + dir, err := os.MkdirTemp("", "loom-data-cli-builds") + if err != nil { + buildErr = err + return + } + loomDataBin = filepath.Join(dir, "loom-data") + arangoProtoBin = filepath.Join(dir, "arango-fhir-proto") + for _, build := range []struct { + pkg, out string + }{ + {"github.com/calypr/loom/cmd/loom-data", loomDataBin}, + {"github.com/calypr/loom/cmd/arango-fhir-proto", arangoProtoBin}, + } { + cmd := exec.Command("go", "build", "-o", build.out, build.pkg) + if output, err := cmd.CombinedOutput(); err != nil { + buildErr = fmt.Errorf("build %s: %v: %s", build.pkg, err, output) + return + } + } + }) + if buildErr != nil { + t.Fatalf("binary build failed: %v", buildErr) + } + return loomDataBin, arangoProtoBin +} + +type cliResult struct { + stdout string + stderr string + code int +} + +func runCLI(t *testing.T, bin string, env []string, args ...string) cliResult { + t.Helper() + cmd := exec.Command(bin, args...) + cmd.Env = append(os.Environ(), env...) + var out, errb bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &errb + err := cmd.Run() + code := 0 + if exitErr, ok := err.(*exec.ExitError); ok { + code = exitErr.ExitCode() + } else if err != nil { + t.Fatalf("run %s: %v", bin, err) + } + return cliResult{stdout: out.String(), stderr: errb.String(), code: code} +} + +func noDSNEnv() []string { return []string{pgDSNEnv + "=", "LOOM_TEST_POSTGRES_URL="} } + +func TestCLIBinaryUsageAndExitCodes(t *testing.T) { + loomBin, _ := builtBinaries(t) + + if res := runCLI(t, loomBin, noDSNEnv()); res.code != 2 || !strings.Contains(res.stderr, "usage:") { + t.Fatalf("no args: code %d stderr %q, want usage + exit 2", res.code, res.stderr) + } + if res := runCLI(t, loomBin, noDSNEnv(), "not-a-command"); res.code != 2 || !strings.Contains(res.stderr, "usage:") { + t.Fatalf("unknown command: code %d, want exit 2 with usage", res.code) + } + res := runCLI(t, loomBin, noDSNEnv(), "load-generation", "--project=prj-a", "--generation=gen-a") + if res.code != 1 || !strings.Contains(res.stderr, "backend is required") { + t.Fatalf("missing backend: code %d stderr %q, want exit 1 naming the refusal", res.code, res.stderr) + } + if res.stdout != "" { + t.Fatalf("refusal must keep stdout machine-clean, got %q", res.stdout) + } + if res := runCLI(t, loomBin, noDSNEnv(), "load-generation", "--backend=postgres", "--zz-not-a-flag"); res.code != 2 { + t.Fatalf("unknown flag must exit 2 via flag.ExitOnError, got %d", res.code) + } + if res := runCLI(t, loomBin, noDSNEnv(), "load-generation", "--backend=postgres", "-h"); res.code != 0 || + !strings.Contains(res.stderr, "-pg-dsn") || !strings.Contains(res.stderr, "STAGED") || !strings.Contains(res.stderr, "NOT activated") { + t.Fatalf("route help: code %d stderr %q, want exit 0 naming pg flags and STAGED/NOT activated", res.code, res.stderr) + } +} + +var flagLineRE = regexp.MustCompile(`^ -([A-Za-z0-9][A-Za-z0-9_-]*)`) + +func parseFlagUsage(t *testing.T, stderr string) map[string]string { + t.Helper() + lines := strings.Split(stderr, "\n") + if len(lines) == 0 { + t.Fatal("empty usage stderr") + } + lines = lines[1:] + records := map[string]string{} + current := "" + var buf []string + for _, line := range lines { + if match := flagLineRE.FindStringSubmatch(line); match != nil { + if current != "" { + records[current] = strings.Join(buf, "\n") + } + current = match[1] + buf = []string{line} + continue + } + if current != "" { + buf = append(buf, line) + } + } + if current != "" { + records[current] = strings.Join(buf, "\n") + } + if len(records) == 0 { + t.Fatalf("no flag records parsed from %q", stderr) + } + return records +} + +func TestArangoUsageSurfaceIsDifferentialParity(t *testing.T) { + loomBin, arangoBin := builtBinaries(t) + + oldRes := runCLI(t, arangoBin, nil, "load-generation", "--zz-not-a-flag") + if oldRes.code != 2 || !strings.Contains(oldRes.stderr, "flag provided but not defined") { + t.Fatalf("old CLI parse-error probe must exit 2 with the flag error, got %d %q", oldRes.code, oldRes.stderr) + } + newRes := runCLI(t, loomBin, noDSNEnv(), "load-generation", "--backend=arango", "--zz-not-a-flag") + if newRes.code != 2 { + t.Fatalf("loom-data arango parse-error probe must exit 2, got %d", newRes.code) + } + + oldRecords := parseFlagUsage(t, oldRes.stderr) + newRecords := parseFlagUsage(t, newRes.stderr) + + backendRecord, hasBackend := newRecords["backend"] + if !hasBackend { + t.Fatal("loom-data load-generation must advertise --backend") + } + delete(newRecords, "backend") + if _, leaked := oldRecords["backend"]; leaked { + t.Fatal("the untouched Arango CLI must not have gained a --backend flag") + } + + if len(newRecords) != len(oldRecords) { + t.Fatalf("flag count drifted: loom-data %d vs arango-fhir-proto %d", len(newRecords), len(oldRecords)) + } + for name, oldLine := range oldRecords { + newLine, ok := newRecords[name] + if !ok { + t.Fatalf("loom-data lost flag --%s present in arango-fhir-proto", name) + } + if newLine != oldLine { + t.Fatalf("flag --%s surface drifted:\n old: %q\n new: %q", name, oldLine, newLine) + } + } + if !strings.Contains(backendRecord, "arango or postgres") { + t.Fatalf("--backend usage must name both tags, got %q", backendRecord) + } +} + +func TestPostgresRouteUsageSurface(t *testing.T) { + loomBin, arangoBin := builtBinaries(t) + + oldRecords := parseFlagUsage(t, runCLI(t, arangoBin, nil, "load-generation", "--zz-not-a-flag").stderr) + pgRes := runCLI(t, loomBin, noDSNEnv(), "load-generation", "--backend=postgres", "--zz-not-a-flag") + if pgRes.code != 2 { + t.Fatalf("pg parse-error probe must exit 2, got %d", pgRes.code) + } + pgRecords := parseFlagUsage(t, pgRes.stderr) + + for _, forbidden := range []string{"url", "database", "write-api"} { + if _, present := pgRecords[forbidden]; present { + t.Fatalf("postgres route advertises Arango-only flag --%s", forbidden) + } + } + for _, required := range []string{"pg-dsn", "pg-schema", "pg-max-conns", "pg-acquire-timeout", "pg-application-name", "semantic-flush-threshold", "truncate", "project", "generation", "backend"} { + if _, present := pgRecords[required]; !present { + t.Fatalf("postgres route is missing flag --%s", required) + } + } + arangoOnly := map[string]bool{"url": true, "database": true, "write-api": true, "project": true} + sharedChecked := 0 + for name, oldLine := range oldRecords { + if arangoOnly[name] { + continue + } + pgLine, ok := pgRecords[name] + if !ok { + t.Fatalf("postgres route lost shared flag --%s", name) + } + if pgLine != oldLine { + t.Fatalf("shared flag --%s drifted between routes:\n old: %q\n pg: %q", name, oldLine, pgLine) + } + sharedChecked++ + } + if sharedChecked < 20 { + t.Fatalf("shared-surface comparison degenerate: only %d shared flags checked", sharedChecked) + } +} + +func TestBadDSNErrsWithoutCredential(t *testing.T) { + loomBin, _ := builtBinaries(t) + + cases := []struct { + name string + dsn string + }{ + {"parse echo (invalid URL)", fmt.Sprintf("postgresql://cliuser:%s@bad host/clidb", unitDSNPassword)}, + {"connect refused (loopback)", fmt.Sprintf("postgresql://cliuser:%s@127.0.0.1:1/clidb?sslmode=disable", unitDSNPassword)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + res := runCLI(t, loomBin, noDSNEnv(), "load-generation", + "--backend=postgres", "--project=prj-cli", "--generation=gen-cli", + "--pg-schema=verify_clidisp_s_unit", + "--pg-manifest-schema=verify_clidisp_sm_unit", "--pg-dsn="+tc.dsn) + if res.code == 0 { + t.Fatal("a bad DSN must exit nonzero") + } + if strings.Contains(res.stderr, unitDSNPassword) || strings.Contains(res.stdout, unitDSNPassword) { + t.Fatalf("credential leaked into diagnostics: stdout=%q stderr=%q", res.stdout, res.stderr) + } + if !strings.Contains(res.stderr, "postgres") { + t.Fatalf("alive control: diagnostics must still say something useful, got %q", res.stderr) + } + }) + } + + res := runCLI(t, loomBin, []string{pgDSNEnv + "=postgresql://env-host/envdb"}, "load-generation", + "--backend=postgres", "--project=prj-cli", "--generation=gen-cli", + "--pg-schema=verify_clidisp_s_unit", + "--pg-manifest-schema=verify_clidisp_sm_unit", "--pg-dsn=postgresql://flag-host/flagdb") + if res.code != 1 || !strings.Contains(res.stderr, "cannot both be set") { + t.Fatalf("DSN conflict path: code %d stderr %q", res.code, res.stderr) + } +} + +const cliDispPrefix = "verify_clidisp_" + +var ( + cliDispDBNamePattern = regexp.MustCompile(`^verify_clidisp_[0-9]+_[0-9a-f]+$`) + cliDispSchemaNamePattern = regexp.MustCompile(`^verify_clidisp_s_[0-9]+_[0-9a-f]+$`) + cliDispManifestSchemaNamePattern = regexp.MustCompile(`^verify_clidisp_sm_[0-9]+_[0-9a-f]+$`) + cliDispAppPattern = regexp.MustCompile(`^verify-clidisp-[0-9a-f]+$`) +) + +var ( + cliDispOwnedMu sync.Mutex + cliDispOwned = map[string]bool{} + cliDispBaselineOnce sync.Once + cliDispBaseline int64 + cliDispBaselineErr error + cliDispUsed bool +) + +func cliDispDSNSecrets(raw string) []string { + secrets := []string{raw} + if parsed, err := url.Parse(raw); err == nil && parsed.User != nil { + if info := parsed.User.String(); info != "" { + secrets = append(secrets, info) + } + if pw, ok := parsed.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + return secrets +} + +func cliDispScrubber(raw string) func(any) string { + secrets := cliDispDSNSecrets(raw) + return func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } +} + +func cliDispCaptureBaseline(raw string) { + cliDispBaselineOnce.Do(func() { + scrub := cliDispScrubber(raw) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + cliDispBaselineErr = fmt.Errorf("connect for baseline: %s", scrub(err)) + return + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_clidisp\\_%' ESCAPE '\\'"). + Scan(&count); err != nil { + cliDispBaselineErr = fmt.Errorf("count baseline: %s", scrub(err)) + return + } + cliDispBaseline = count + }) +} + +func cliDispRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) +} + +type cliDispService struct { + t *testing.T + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + manifestSchema string + scrub func(any) string + appProbe string + witnesses *cliDispWitnessPoller +} + +func newCLIDispService(t *testing.T) *cliDispService { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the live PostgreSQL 18 CLI leg") + } + cliDispUsed = true + cliDispCaptureBaseline(raw) + if cliDispBaselineErr != nil { + t.Fatalf("baseline: %v", cliDispBaselineErr) + } + + parsed, err := url.Parse(raw) + if err != nil || (parsed.Scheme != "postgres" && parsed.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + secrets := cliDispDSNSecrets(raw) + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + + ctx, cancel := context.WithTimeout(context.Background(), 600*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for admin: %s", scrub(err)) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("%s%d_%s", cliDispPrefix, time.Now().UnixNano(), cliDispRandomHex(t)) + if !cliDispDBNamePattern.MatchString(dbName) { + t.Fatalf("owned database name %q violates the pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + t.Fatalf("create owned database: %s", scrub(err)) + } + cliDispOwnedMu.Lock() + cliDispOwned[dbName] = false + cliDispOwnedMu.Unlock() + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(context.Background()), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop owned database: %s", scrub(err)) + return + } + cliDispOwnedMu.Lock() + cliDispOwned[dbName] = true + cliDispOwnedMu.Unlock() + }) + + owned, err := url.Parse(raw) + if err != nil { + t.Fatalf("owned dsn parse: %s", scrub(err)) + } + if q := owned.Query(); q.Get("application_name") != "" || q.Get("dbname") != "" { + q.Del("application_name") + q.Del("dbname") + owned.RawQuery = q.Encode() + } + owned.Path = "/" + dbName + dsn := owned.String() + secrets = append(secrets, dsn) + + control, err := url.Parse(raw) + if err != nil { + t.Fatalf("control dsn parse: %s", scrub(err)) + } + q := control.Query() + q.Set("application_name", "verify_clidisp_admin_probe") + q.Del("dbname") + control.RawQuery = q.Encode() + control.Path = "/" + dbName + controlDSN := control.String() + secrets = append(secrets, controlDSN) + + iso, err := pgxpool.New(ctx, controlDSN) + if err != nil { + t.Fatalf("connect to owned database: %s", scrub(err)) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("%ss_%d_%s", cliDispPrefix, time.Now().UnixNano(), cliDispRandomHex(t)) + if !cliDispSchemaNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q violates the pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + t.Fatalf("create owned schema: %s", scrub(err)) + } + manifestSchema := fmt.Sprintf("%ssm_%d_%s", cliDispPrefix, time.Now().UnixNano(), cliDispRandomHex(t)) + if !cliDispManifestSchemaNamePattern.MatchString(manifestSchema) { + t.Fatalf("owned manifest schema name %q violates the pattern", manifestSchema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+manifestSchema+`"`); err != nil { + t.Fatalf("create owned manifest schema: %s", scrub(err)) + } + return &cliDispService{t: t, ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, manifestSchema: manifestSchema, scrub: scrub, appProbe: "verify_clidisp_admin_probe"} +} + +type cliDispWitnessPoller struct { + t *testing.T + svc *cliDispService + stop chan struct{} + done chan struct{} + stopCh sync.Once + mu sync.Mutex + seen map[string]int +} + +func startCLIDispWitness(t *testing.T, svc *cliDispService, apps ...string) *cliDispWitnessPoller { + t.Helper() + w := &cliDispWitnessPoller{t: t, svc: svc, stop: make(chan struct{}), done: make(chan struct{}), seen: map[string]int{}} + for _, app := range apps { + w.seen[app] = 0 + } + go func() { + defer close(w.done) + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-w.stop: + return + case <-ticker.C: + for app := range w.seen { + var count int + if err := svc.admin.QueryRow(svc.ctx, + "SELECT count(*) FROM pg_stat_activity WHERE application_name = $1 AND datname = $2", + app, svc.dbNameForWitness()).Scan(&count); err != nil { + continue + } + if count > 0 { + w.mu.Lock() + w.seen[app] = count + w.mu.Unlock() + } + } + } + } + }() + return w +} + +func (s *cliDispService) dbNameForWitness() string { + parsed, err := url.Parse(s.dsn) + if err != nil { + return "" + } + return strings.TrimPrefix(parsed.Path, "/") +} + +func (w *cliDispWitnessPoller) stopPolling() { + w.stopCh.Do(func() { close(w.stop) }) + <-w.done +} + +func (w *cliDispWitnessPoller) observed(app string) int { + w.mu.Lock() + defer w.mu.Unlock() + return w.seen[app] +} + +func cliDispWriteFixtures(t *testing.T, dir string) { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir fixtures: %v", err) + } + line := func(doc map[string]any) string { + encoded, err := json.Marshal(doc) + if err != nil { + t.Fatalf("marshal fixture line: %v", err) + } + return string(encoded) + } + patients := make([]string, 0, 3) + for _, id := range []string{"pat-0000", "pat-0001", "pat-0002"} { + patients = append(patients, line(map[string]any{"resourceType": "Patient", "id": id, "gender": "female"})) + } + if err := os.WriteFile(filepath.Join(dir, "Patient.ndjson"), []byte(strings.Join(patients, "\n")+"\n"), 0o600); err != nil { + t.Fatalf("write Patient fixture: %v", err) + } + observations := []string{ + line(map[string]any{ + "resourceType": "Observation", "id": "obs-a", "status": "final", + "subject": map[string]any{"reference": "Patient/pat-0000"}, + "code": map[string]any{"coding": []any{map[string]any{"system": "http://loinc.org", "code": "8867-4", "display": "Heart rate"}}}, + }), + line(map[string]any{ + "resourceType": "Observation", "id": "obs-b", "status": "final", + "subject": map[string]any{"reference": "Patient/pat-0001"}, + "code": map[string]any{"text": "plain-text code"}, + }), + } + if err := os.WriteFile(filepath.Join(dir, "Observation.ndjson"), []byte(strings.Join(observations, "\n")+"\n"), 0o600); err != nil { + t.Fatalf("write Observation fixture: %v", err) + } +} + +func TestCLIPostgresLoadGenerationLiveStagedLoad(t *testing.T) { + svc := newCLIDispService(t) + staging := t.TempDir() + cliDispWriteFixtures(t, staging) + + project := "prj-cli-disp" + generation := "gen-cli-live" + app := "verify-clidisp-" + cliDispRandomHex(t) + if !cliDispAppPattern.MatchString(app) { + t.Fatalf("witness application name %q violates the pattern", app) + } + + witness := startCLIDispWitness(t, svc, app, svc.appProbe, "verify-clidisp-no-such-app-xyz") + defer witness.stopPolling() + + args := []string{ + "--backend=postgres", + "--project=" + project, + "--generation=" + generation, + "--pg-dsn=" + svc.dsn, + "--pg-schema=" + svc.schema, + "--pg-manifest-schema=" + svc.manifestSchema, + "--meta-dir=" + staging, + "--schema=" + filepath.Join("..", "..", "schemas", "graph-fhir.json"), + "--pg-application-name=" + app, + "--use-generic=false", + } + var stdout, stderr string + var dispatchErr error + stdout, stderr = captureOutput(t, func() { + dispatchErr = dispatchLoadGeneration(svc.ctx, args, defaultRunner()) + }) + witness.stopPolling() + + if dispatchErr != nil { + t.Fatalf("live load: %s", svc.scrub(dispatchErr)) + } + if !strings.Contains(stderr, "STAGED") || !strings.Contains(stderr, "not activated") { + t.Fatalf("success must announce STAGED-and-not-activated, got %q", stderr) + } + + lines := strings.Split(strings.TrimSpace(stdout), "\n") + var summary ingest.LoadSummary + if err := json.Unmarshal([]byte(lines[len(lines)-1]), &summary); err != nil { + t.Fatalf("last stdout line is not the summary JSON: %v\n%q", err, lines[len(lines)-1]) + } + if summary.Files != 2 || summary.VerticesInserted != 5 || summary.EdgesInserted != 4 { + t.Fatalf("summary counts = files %d vertices %d edges %d, want literal 2/5/4", + summary.Files, summary.VerticesInserted, summary.EdgesInserted) + } + if summary.Dataset == nil || summary.Dataset.Project != project || summary.Dataset.Generation != generation { + t.Fatalf("summary dataset = %+v, want %s/%s", summary.Dataset, project, generation) + } + if strings.Contains(stdout, svc.dsn) || strings.Contains(stderr, svc.dsn) { + t.Fatal("raw DSN appeared in CLI output") + } + + manifestTable := pgx.Identifier{svc.manifestSchema, "dataset_generation_manifest"}.Sanitize() + activeTable := pgx.Identifier{svc.manifestSchema, "dataset_active_generation"}.Sanitize() + var state string + if err := svc.iso.QueryRow(svc.ctx, + "SELECT state FROM "+manifestTable+" WHERE project = $1 AND generation = $2", project, generation).Scan(&state); err != nil { + t.Fatalf("read manifest row: %s", svc.scrub(err)) + } + if state != "STAGED" { + t.Fatalf("manifest state = %q, want literal STAGED", state) + } + var activeCount int + if err := svc.iso.QueryRow(svc.ctx, + "SELECT count(*) FROM "+activeTable+" WHERE project = $1", project).Scan(&activeCount); err != nil { + t.Fatalf("count active pointers: %s", svc.scrub(err)) + } + if activeCount != 0 { + t.Fatalf("active-generation pointer count = %d, want 0: the CLI must never activate", activeCount) + } + + if witness.observed(app) == 0 { + t.Fatalf("load backends were never witnessed with application_name %q", app) + } + if witness.observed(svc.appProbe) == 0 { + t.Fatal("witness poller saw even the admin probe: poller is broken (alive control)") + } + if witness.observed("verify-clidisp-no-such-app-xyz") != 0 { + t.Fatal("witness predicate is not selective") + } + + virginSchema := fmt.Sprintf("%ss_%d_%s", cliDispPrefix, time.Now().UnixNano(), cliDispRandomHex(t)) + if _, err := svc.iso.Exec(svc.ctx, `CREATE SCHEMA "`+virginSchema+`"`); err != nil { + t.Fatalf("create virgin schema: %s", svc.scrub(err)) + } + virginManifest := fmt.Sprintf("%ssm_%d_%s", cliDispPrefix, time.Now().UnixNano(), cliDispRandomHex(t)) + if !cliDispManifestSchemaNamePattern.MatchString(virginManifest) { + t.Fatalf("virgin manifest schema name %q violates the pattern", virginManifest) + } + if _, err := svc.iso.Exec(svc.ctx, `CREATE SCHEMA "`+virginManifest+`"`); err != nil { + t.Fatalf("create virgin manifest schema: %s", svc.scrub(err)) + } + emptyDir := t.TempDir() + _, _ = captureOutput(t, func() { + err := dispatchLoadGeneration(svc.ctx, []string{ + "--backend=postgres", + "--project=" + project, + "--generation=gen-empty", + "--pg-dsn=" + svc.dsn, + "--pg-schema=" + virginSchema, + "--pg-manifest-schema=" + virginManifest, + "--meta-dir=" + emptyDir, + "--schema=" + filepath.Join("..", "..", "schemas", "graph-fhir.json"), + }, defaultRunner()) + if !errors.Is(err, ingest.ErrGenerationLoadRequiresFiles) { + t.Fatalf("empty staging dir error = %s, want ErrGenerationLoadRequiresFiles", svc.scrub(err)) + } + }) + var missing bool + if err := svc.iso.QueryRow(svc.ctx, + "SELECT to_regclass($1) IS NULL", virginManifest+".dataset_generation_manifest").Scan(&missing); err != nil { + t.Fatalf("probe virgin manifest schema: %s", svc.scrub(err)) + } + if !missing { + t.Fatal("the zero-files guard still migrated the virgin manifest schema") + } + var virginRelations int + if err := svc.iso.QueryRow(svc.ctx, + "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = $1", + virginSchema).Scan(&virginRelations); err != nil { + t.Fatalf("probe virgin data schema: %s", svc.scrub(err)) + } + if virginRelations != 0 { + t.Fatal("the zero-files guard still migrated the virgin data schema") + } +} + +func TestZZCLIDispLiveOwnershipAudit(t *testing.T) { + if !cliDispUsed { + t.Skip("live leg did not run in this process") + } + if cliDispBaselineErr != nil { + t.Fatalf("baseline was never captured: %v", cliDispBaselineErr) + } + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + scrub := cliDispScrubber(raw) + admin, err := pgxpool.New(context.Background(), raw) + if err != nil { + t.Fatalf("connect for audit: %s", scrub(err)) + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(context.Background(), + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_clidisp\\_%' ESCAPE '\\'").Scan(&count); err != nil { + t.Fatalf("audit count: %s", scrub(err)) + } + if count != cliDispBaseline { + t.Errorf("verify_clidisp_* database count = %d, want baseline %d (zero own-prefix survivors)", count, cliDispBaseline) + } + cliDispOwnedMu.Lock() + defer cliDispOwnedMu.Unlock() + for name, dropped := range cliDispOwned { + if !dropped { + t.Errorf("registered fixture %s was never dropped by its owner", name) + } + } +} diff --git a/cmd/loom-data/main.go b/cmd/loom-data/main.go new file mode 100644 index 000000000..0a2d46c09 --- /dev/null +++ b/cmd/loom-data/main.go @@ -0,0 +1,82 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/signal" + "syscall" +) + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + os.Exit(run(context.Background(), os.Args[1], os.Args[2:])) +} + +func run(ctx context.Context, command string, args []string) int { + handler, ok := commandRoutes()[command] + if !ok { + usage() + return 2 + } + var err error + if signalGuardedCommands()[command] { + err = withSignalCancellation(ctx, func(signalCtx context.Context) error { + return handler(signalCtx, args) + }) + } else { + err = handler(ctx, args) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + return 0 +} + +func commandRoutes() map[string]func(context.Context, []string) error { + return map[string]func(context.Context, []string) error{ + "load-generation": func(ctx context.Context, args []string) error { + return dispatchLoadGeneration(ctx, args, defaultRunner()) + }, + } +} + +func signalGuardedCommands() map[string]bool { + return map[string]bool{ + "load-generation": true, + } +} + +func withSignalCancellation(ctx context.Context, run func(context.Context) error) error { + signalCtx, cancel := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) + defer cancel() + return run(signalCtx) +} + +func usage() { + fmt.Fprintf(os.Stderr, `usage: + loom-data load-generation --backend arango --generation OPAQUE_ID [flags] + # immutable complete META directory load into ArangoDB, identical flags, + # output, and exit contract to arango-fhir-proto load-generation + loom-data load-generation --backend postgres --generation OPAQUE_ID --project PROJECT --pg-schema SCHEMA (--pg-dsn DSN | LOOM_DATA_PG_DSN) [flags] + # one bounded PostgreSQL generation load; on success the generation is + # STAGED (verified) and NOT activated — activation is a separate + # explicit operation, never performed by this command + --backend is required and must be arango or postgres; it is never inferred + from a DSN, and a load never falls back to another backend. +`) +} + +func printJSON(value any) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + fmt.Println(string(data)) + return nil +} diff --git a/cmd/loom-data/main_test.go b/cmd/loom-data/main_test.go new file mode 100644 index 000000000..7350f3cc7 --- /dev/null +++ b/cmd/loom-data/main_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" +) + +func TestCommandRoutesExposesOnlyLoadGeneration(t *testing.T) { + routes := commandRoutes() + if len(routes) != 1 { + t.Fatalf("routes = %v, want exactly load-generation", keysOf(routes)) + } + if _, ok := routes["load-generation"]; !ok { + t.Fatalf("routes missing load-generation: %v", keysOf(routes)) + } +} + +func TestLoadGenerationIsSignalGuarded(t *testing.T) { + if !signalGuardedCommands()["load-generation"] { + t.Fatal("load-generation must run under the SIGINT/SIGTERM-cancellable context: it mutates a lifecycle manifest") + } + if signalGuardedCommands()["no-such-command"] { + t.Fatal("signal guard membership test is vacuous") + } +} + +func TestRunUnknownCommandExitsWithUsageCode(t *testing.T) { + _, stderr := captureOutput(t, func() { + if code := run(context.Background(), "definitely-not-a-command", nil); code != 2 { + t.Fatalf("run() exit code = %d, want 2", code) + } + }) + if !strings.Contains(stderr, "usage:") { + t.Fatalf("unknown command must print usage, got %q", stderr) + } +} + +func TestRunTypedBackendRefusalExitsNonzero(t *testing.T) { + t.Setenv(pgDSNEnv, "") + stdout, stderr := captureOutput(t, func() { + code := run(context.Background(), "load-generation", []string{"--project=prj-a", "--generation=gen-a"}) + if code != 1 { + t.Fatalf("run() exit code = %d, want 1 for a typed refusal", code) + } + }) + if !strings.Contains(stderr, ErrBackendRequired.Error()) { + t.Fatalf("stderr must name the backend refusal, got %q", stderr) + } + if stdout != "" { + t.Fatalf("stdout must stay empty on refusal, got %q", stdout) + } +} + +func keysOf[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + return keys +} + +const dispatchSignalChildEnv = "LOOM_CLI_DISPATCH_SIGNAL_CHILD" +const dispatchSignalReadyEnv = "LOOM_CLI_DISPATCH_SIGNAL_READY" + +func TestLoadGenerationSignalCancelsProcessContext(t *testing.T) { + if os.Getenv(dispatchSignalChildEnv) == "1" { + readyPath := os.Getenv(dispatchSignalReadyEnv) + err := withSignalCancellation(context.Background(), func(ctx context.Context) error { + if err := os.WriteFile(readyPath, []byte("ready"), 0o600); err != nil { + return err + } + <-ctx.Done() + return ctx.Err() + }) + if !errors.Is(err, context.Canceled) { + os.Exit(2) + } + os.Exit(0) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + readyPath := filepath.Join(t.TempDir(), "ready") + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestLoadGenerationSignalCancelsProcessContext$") + cmd.Env = append(os.Environ(), dispatchSignalChildEnv+"=1", dispatchSignalReadyEnv+"="+readyPath) + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := os.Stat(readyPath); err == nil { + break + } + if time.Now().After(deadline) { + _ = cmd.Process.Kill() + _ = cmd.Wait() + t.Fatal("child did not enter the guarded load context") + } + time.Sleep(10 * time.Millisecond) + } + if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + t.Fatalf("send SIGTERM: %v", err) + } + if err := cmd.Wait(); err != nil { + t.Fatalf("guarded load did not cleanly observe SIGTERM: %v", err) + } +} diff --git a/cmd/loom-repository/main.go b/cmd/loom-repository/main.go index e4e101ff9..6b73efe17 100644 --- a/cmd/loom-repository/main.go +++ b/cmd/loom-repository/main.go @@ -355,7 +355,6 @@ func publish(ctx context.Context, client *http.Client, base string, manifest rep return value, nil } -// Generation storage uses legacy project IDs while Explorer APIs expose canonical IDs. func uploadURL(base string, manifest repositoryseed.Manifest) string { return strings.TrimRight(base, "/") + "/api/v1/datasets/" + url.PathEscape(projectid.Legacy(manifest.Project)) + "/generations/" + url.PathEscape(manifest.Generation) } diff --git a/cmd/preview-covering-index-artifact/main.go b/cmd/preview-covering-index-artifact/main.go index b9624070f..5b9a0b571 100644 --- a/cmd/preview-covering-index-artifact/main.go +++ b/cmd/preview-covering-index-artifact/main.go @@ -1,6 +1,3 @@ -// Command preview-covering-index-artifact recreates one captured Builder -// proposal with Loom's production authoring and dataframe compilers. It only -// writes the requested JSON artifact and never contacts Loom or ArangoDB. package main import ( @@ -381,7 +378,7 @@ func run(reportPath, queryPath, outPath, compiledQueryOutPath, sourceManifestPat if !ok || limitValue < 0 || limitValue != float64(int(limitValue)) { return errors.New("captured query limit bind is missing or invalid") } - queries, err := compiler.CompileResolvedRecipePlanWithPolicy(resolved, int(limitValue), ir.DefaultPhysicalOptimizationPolicy()) + queries, err := compiler.CompileResolvedRecipePlan(compiler.ArangoTarget(), resolved, int(limitValue), ir.DefaultPhysicalOptimizationPolicy()) if err != nil { return fmt.Errorf("compile production physical query: %w", err) } @@ -392,6 +389,10 @@ func run(reportPath, queryPath, outPath, compiledQueryOutPath, sourceManifestPat if actual.PreviewCoveringIndex == nil { return errors.New("current production recompile has no PreviewCoveringIndexSpec") } + actualQuery, actualBindVars, dispatchErr := actual.Command.AQLParameters() + if dispatchErr != nil { + return fmt.Errorf("dispatch current production recompile to AQL comparison: %w", dispatchErr) + } baseReceiptID := baseReceipt.ReceiptID if baseReceiptID == "" || baseReceipt.RecipeDigest == "" { return errors.New("captured reconcile response lacks saved base receipt identity") @@ -404,16 +405,16 @@ func run(reportPath, queryPath, outPath, compiledQueryOutPath, sourceManifestPat return fmt.Errorf("current compiler index spec: %w", err) } queryComparison := queryComparison{ - ExactAQLMatch: actual.Query == expected.Query, ExactBindVarsMatch: sameJSONValue(actual.BindVars, expected.BindVars), - CapturedAQLSHA256: digest([]byte(expected.Query)), RecompiledAQLSHA256: digest([]byte(actual.Query)), - CapturedBindsSHA256: digestJSON(expected.BindVars), RecompiledBindsSHA256: digestJSON(actual.BindVars), - FirstDifference: firstQueryDifference(expected.Query, actual.Query), - FirstBindDifference: bindDifference(expected.BindVars, actual.BindVars), + ExactAQLMatch: actualQuery == expected.Query, ExactBindVarsMatch: sameJSONValue(actualBindVars, expected.BindVars), + CapturedAQLSHA256: digest([]byte(expected.Query)), RecompiledAQLSHA256: digest([]byte(actualQuery)), + CapturedBindsSHA256: digestJSON(expected.BindVars), RecompiledBindsSHA256: digestJSON(actualBindVars), + FirstDifference: firstQueryDifference(expected.Query, actualQuery), + FirstBindDifference: bindDifference(expected.BindVars, actualBindVars), } var recompiledQueryBytes []byte recompiledQuerySHA256 := "" if compiledQueryOutPath != "" { - recompiledQueryBytes, err = encodeRecompiledQueryArtifact(actual.Query, actual.BindVars) + recompiledQueryBytes, err = encodeRecompiledQueryArtifact(actualQuery, actualBindVars) if err != nil { return fmt.Errorf("encode current recompiled query artifact: %w", err) } @@ -431,7 +432,7 @@ func run(reportPath, queryPath, outPath, compiledQueryOutPath, sourceManifestPat Algorithm: "sha256", Mode: "verified-go-run-source-closure-current-production-recompile-from-captured-base-and-candidate-request", CapturedQueryFile: queryPath, CapturedQueryFileSHA256: digest(queryBytes), CapturedAQLSHA256: digest([]byte(expected.Query)), CapturedBindVarsSHA256: digestJSON(expected.BindVars), - RecompiledAQLSHA256: digest([]byte(actual.Query)), RecompiledBindVarsSHA256: digestJSON(actual.BindVars), + RecompiledAQLSHA256: digest([]byte(actualQuery)), RecompiledBindVarsSHA256: digestJSON(actualBindVars), RecompiledQueryFile: compiledQueryOutPath, RecompiledQueryFileSHA256: recompiledQuerySHA256, CapturedBaseReceiptID: baseReceiptID, CapturedBaseRecipeDigest: baseReceipt.ResolvedRecipeDigest, CandidateRequestSHA256: digest(proposalBody), CandidateIsRequestOnly: true, @@ -577,9 +578,9 @@ func fingerprintProductionSources(moduleRoot, packagePattern string) (production if err != nil { return productionSourceFingerprint{}, fmt.Errorf("make module root absolute: %w", err) } - command := exec.Command("go", "list", "-deps", "-json", packagePattern) - command.Dir = root - listOutput, err := command.Output() + compiledCommand := exec.Command("go", "list", "-deps", "-json", packagePattern) + compiledCommand.Dir = root + listOutput, err := compiledCommand.Output() if err != nil { return productionSourceFingerprint{}, fmt.Errorf("go list production source closure: %w: %s", err, strings.TrimSpace(string(listOutput))) } @@ -946,11 +947,6 @@ func validateCapturedScope(report capturedReport, captured capturedQuery, reconc return nil } -// The captured catalog includes signed ConstructionChoice values whose Go -// domain representation is an interface. They are not needed to recreate a -// compile: CompileWorkspace issues fresh compiler choices from the mapped -// capability snapshot. Strip only that opaque field before decoding the -// public catalog facts. func withoutCapturedChoiceInterfaces(raw json.RawMessage) ([]byte, error) { var response map[string]json.RawMessage if err := json.Unmarshal(raw, &response); err != nil { @@ -1184,7 +1180,11 @@ func proveCandidateCorrespondence(base authoringv2.Workspace, proposal proposalR if len(finalSchema) != len(publicSchema)+1 { return candidateProof{}, fmt.Errorf("final output schema has %d columns including internal identity, want %d", len(finalSchema), len(publicSchema)+1) } - if strings.Count(compiled.Query, "FOR root IN @@root_collection") != 1 || strings.Contains(compiled.Query, "FOR related") { + compiledQueryText, _, compiledDispatchErr := compiled.Command.AQLParameters() + if compiledDispatchErr != nil { + return candidateProof{}, fmt.Errorf("dispatch current Pivot recompile to AQL shape check: %w", compiledDispatchErr) + } + if strings.Count(compiledQueryText, "FOR root IN @@root_collection") != 1 || strings.Contains(compiledQueryText, "FOR related") { return candidateProof{}, errors.New("current Pivot query is not a single root scan without related-row expansion") } return candidateProof{ diff --git a/compose.dev.yaml b/compose.dev.yaml index 0325338d4..729d30876 100644 --- a/compose.dev.yaml +++ b/compose.dev.yaml @@ -30,10 +30,7 @@ services: loom-api: environment: - # loom-dev.mjs injects a stable per-worktree key for this local no-auth - # stack; production server configuration remains fail-closed. LOOM_POPULATION_MAPPING_CURSOR_SECRET: ${LOOM_POPULATION_MAPPING_CURSOR_SECRET:-} - # Fault injectors are accepted only by the API's --no-auth dev mode. LOOM_DEV_RECIPE_QUALITY_MAX_ROWS: ${LOOM_DEV_RECIPE_QUALITY_MAX_ROWS:-} LOOM_DEV_RECIPE_QUALITY_MAX_DISTINCT_KEYS: ${LOOM_DEV_RECIPE_QUALITY_MAX_DISTINCT_KEYS:-} LOOM_DEV_ACTIVATION_CONFLICT_ONCE: ${LOOM_DEV_ACTIVATION_CONFLICT_ONCE:-} diff --git a/compose.yaml b/compose.yaml index aff047516..51f53142c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -31,8 +31,6 @@ services: loom-api: image: ${LOOM_API_IMAGE:-loom-demo-api:local} environment: - # demo-up.sh derives this for the local no-auth stack; production - # deployments must provide a stable server-configured secret. LOOM_POPULATION_MAPPING_CURSOR_SECRET: ${LOOM_POPULATION_MAPPING_CURSOR_SECRET:-} build: context: ${LOOM_API_BUILD_CONTEXT:-.} diff --git a/conformance/compiler/benchmark_test.go b/conformance/compiler/benchmark_test.go index b106d209b..b56c4fa31 100644 --- a/conformance/compiler/benchmark_test.go +++ b/conformance/compiler/benchmark_test.go @@ -25,10 +25,6 @@ var ( benchmarkErr error ) -// BenchmarkCompilerOracle measures pure lowering and AQL rendering for the -// checked-in oracle requests and every root advertised by generated metadata. -// Unsupported oracle cases are intentionally included: deterministic rejection -// is part of compiler cost and conformance behavior. func BenchmarkCompilerOracle(b *testing.B) { cases := loadCompilerBenchmarkCases(b) for _, testCase := range cases { @@ -117,8 +113,6 @@ func benchmarkCaseNames(cases []compilerBenchmarkCase) []string { func rootRecipe(resourceType string) recipe.Bundle { grain, ok := spec.InferRowGrain(resourceType) if !ok { - // Keep malformed/abstract roots structurally valid long enough for the - // schema-backed compiler to emit the intended root rejection. grain = spec.RowGrain("resource") } return recipe.Bundle{RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: "generated_" + resourceType, TranslationVersion: "benchmark", Outputs: []recipe.Output{{Name: "root", RootResourceType: resourceType, RowGrain: string(grain)}}} diff --git a/conformance/compiler/command_helpers_test.go b/conformance/compiler/command_helpers_test.go new file mode 100644 index 000000000..40eabb463 --- /dev/null +++ b/conformance/compiler/command_helpers_test.go @@ -0,0 +1,37 @@ +package compilerfixture + +import ( + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" +) + +func testAQLView(t *testing.T, command command.CompiledCommand) (string, map[string]any) { + t.Helper() + query, bindVars, err := command.AQLParameters() + if err != nil { + t.Fatalf("derive AQL parameters from %s command: %v", command.Backend(), err) + } + return query, bindVars +} + +func testAQLCommand(t *testing.T, query string, bindVars map[string]any) command.CompiledCommand { + t.Helper() + compiledCommand, err := command.NewAQLCommand(query, bindVars) + if err != nil { + t.Fatalf("build AQL command for test: %v", err) + } + return compiledCommand +} + +func testAQLText(t *testing.T, command command.CompiledCommand) string { + t.Helper() + query, _ := testAQLView(t, command) + return query +} + +func testAQLBinds(t *testing.T, command command.CompiledCommand) map[string]any { + t.Helper() + _, bindVars := testAQLView(t, command) + return bindVars +} diff --git a/conformance/compiler/corpus_coverage_test.go b/conformance/compiler/corpus_coverage_test.go index e97018299..48c8cbf19 100644 --- a/conformance/compiler/corpus_coverage_test.go +++ b/conformance/compiler/corpus_coverage_test.go @@ -5,10 +5,6 @@ import ( "testing" ) -// TestGenericFHIRCorpusCoverage keeps the optimization benchmark honest. The -// GDC request is intentionally only one mixed-shape case; each independent -// semantic feature must also have a small fixture that can be compiled and -// profiled in isolation. func TestGenericFHIRCorpusCoverage(t *testing.T) { fixtures, err := LoadDir(filepath.Join("fixtures")) if err != nil { diff --git a/conformance/compiler/gdc_fixture_test.go b/conformance/compiler/gdc_fixture_test.go index 751887f56..0e2f29812 100644 --- a/conformance/compiler/gdc_fixture_test.go +++ b/conformance/compiler/gdc_fixture_test.go @@ -9,10 +9,6 @@ import ( "github.com/calypr/loom/internal/dataframe/recipe" ) -// TestGDCFixtureCoversRichShape keeps the checked-in benchmark representative -// of the dataframe users actually want to build. It is intentionally a corpus -// test, not a compiler test: as physical feature families land, the same -// request remains the parity and Arango-cost benchmark target. func TestGDCFixtureCoversRichShape(t *testing.T) { fixtures, err := LoadDir(filepath.Join("fixtures")) if err != nil { diff --git a/conformance/compiler/generated_semantics_coverage_test.go b/conformance/compiler/generated_semantics_coverage_test.go index 19fd778f4..51203fda8 100644 --- a/conformance/compiler/generated_semantics_coverage_test.go +++ b/conformance/compiler/generated_semantics_coverage_test.go @@ -50,8 +50,8 @@ func TestPublicCompilerAcceptsEveryGeneratedBuilderTraversal(t *testing.T) { if compiled.PlanProfile != "generic_fhir_graph_recipe" { t.Fatalf("plan profile = %q, want generic_fhir_graph_recipe", compiled.PlanProfile) } - if !bindVarsContain(compiled.BindVars, traversal.ToType) { - t.Fatalf("target resource type %q is not represented as a bind value: %#v", traversal.ToType, compiled.BindVars) + if !bindVarsContain(testAQLBinds(t, compiled.Command), traversal.ToType) { + t.Fatalf("target resource type %q is not represented as a bind value: %#v", traversal.ToType, testAQLBinds(t, compiled.Command)) } }) } @@ -66,8 +66,8 @@ func TestPublicCompilerRejectsUnadvertisedRootBeforeRenderingAQL(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "not represented by the active generated FHIR schema") { t.Fatalf("error = %v, want generated-schema rejection", err) } - if compiled.Query != "" { - t.Fatalf("unsafe root produced AQL: %s", compiled.Query) + if compiled.Command.Backend() != "" { + t.Fatalf("unsafe root produced AQL: backend %s", compiled.Command.Backend()) } } @@ -82,15 +82,6 @@ type graphSchemaDocument struct { } `json:"$defs"` } -// loadGeneratedBuilderTraversals derives the same parent -> related-child -// orientation exposed by Loom's populated-reference catalog. A graph-schema -// link describes the stored forward FHIR reference (child -> parent), while -// the dataframe builder intentionally follows its generated reverse route -// (parent -> child) through INBOUND fhir_edge traversal. This coverage stays -// on that broad route family; the one separately proven forward contract -// (ResearchSubject --study--> ResearchStudy) has focused storage-route and -// extractor regressions. Other forward links remain unproven and must not be -// blessed by this generic coverage loop. func loadGeneratedBuilderTraversals(t *testing.T) []fhirschema.CompilerTraversal { t.Helper() path := filepath.Join("..", "..", "schemas", "graph-fhir.json") @@ -147,15 +138,15 @@ var collectionIteration = regexp.MustCompile(`(?m)^FOR\s+[A-Za-z_][A-Za-z0-9_]*\ func assertOnlyValidatedRootCollection(t *testing.T, compiled compiler.CompiledQuery, rootResourceType string) { t.Helper() - matches := collectionIteration.FindAllStringSubmatch(compiled.Query, -1) - if strings.Contains(compiled.Query, "FOR root IN @@root_collection") { - if len(matches) != 0 || compiled.BindVars["@root_collection"] != rootResourceType { - t.Fatalf("physical root collection = %#v / direct iterations %#v, want only validated root %q\nquery:\n%s", compiled.BindVars["@root_collection"], matches, rootResourceType, compiled.Query) + matches := collectionIteration.FindAllStringSubmatch(testAQLText(t, compiled.Command), -1) + if strings.Contains(testAQLText(t, compiled.Command), "FOR root IN @@root_collection") { + if len(matches) != 0 || testAQLBinds(t, compiled.Command)["@root_collection"] != rootResourceType { + t.Fatalf("physical root collection = %#v / direct iterations %#v, want only validated root %q\nquery:\n%s", testAQLBinds(t, compiled.Command)["@root_collection"], matches, rootResourceType, testAQLText(t, compiled.Command)) } return } if len(matches) != 1 || matches[0][1] != rootResourceType { - t.Fatalf("direct collection iterations = %#v, want only validated root %q\nquery:\n%s", matches, rootResourceType, compiled.Query) + t.Fatalf("direct collection iterations = %#v, want only validated root %q\nquery:\n%s", matches, rootResourceType, testAQLText(t, compiled.Command)) } } diff --git a/conformance/compiler/physical_plan_cost_test.go b/conformance/compiler/physical_plan_cost_test.go index 9f0dbf9cb..e59a11353 100644 --- a/conformance/compiler/physical_plan_cost_test.go +++ b/conformance/compiler/physical_plan_cost_test.go @@ -7,10 +7,6 @@ import ( "github.com/calypr/loom/internal/dataframe/compiler/ir" ) -// TestPhysicalPlanOptimizationCorpus is a structural regression gate. It does -// not pretend to replace live Arango PROFILE; it prevents future compiler -// changes from silently removing the generic rewrites that PROFILE is meant -// to measure. func TestPhysicalPlanOptimizationCorpus(t *testing.T) { fixtures, err := LoadDir(filepath.Join("fixtures")) if err != nil { diff --git a/conformance/compiler/policy_ablation_test.go b/conformance/compiler/policy_ablation_test.go index e01e36f87..7c38ca3a6 100644 --- a/conformance/compiler/policy_ablation_test.go +++ b/conformance/compiler/policy_ablation_test.go @@ -12,25 +12,25 @@ import ( "testing" "time" + "github.com/calypr/loom/internal/backends/arango" + "github.com/calypr/loom/internal/dataframe/compiler" "github.com/calypr/loom/internal/dataframe/compiler/ir" - dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" - "github.com/calypr/loom/internal/store/arango" ) -func explainCompiledQuery(ctx context.Context, opts arango.ConnectionOptions, compiled dataframeexecution.CompiledQuery) (arango.ExplainResult, error) { - client, err := arango.Open(ctx, opts.URL, opts.Database) +func explainCompiledQuery(ctx context.Context, opts arango.ConnectionOptions, compiled compiler.CompiledQuery) (arango.ExplainResult, error) { + backend, err := arango.Open(ctx, opts) if err != nil { return arango.ExplainResult{}, err } - defer client.Close(ctx) - return client.Explain(ctx, arango.ExplainRequest{Query: compiled.Query, BindVars: compiled.BindVars}) + defer backend.Close(ctx) + client := backend.Client + query, bindVars, err := compiled.Command.AQLParameters() + if err != nil { + return arango.ExplainResult{}, err + } + return client.Explain(ctx, arango.ExplainRequest{Query: query, BindVars: bindVars}) } -// TestGDCOptimizationPolicyAblationAgainstArango is opt-in because it reads -// the local development database and profiles four complete GDC executions. -// It proves that independent policy switches preserve the same dataframe -// result before a later work package uses the measurements to enable a new -// rewrite. func TestGDCOptimizationPolicyAblationAgainstArango(t *testing.T) { if os.Getenv("LOOM_COMPILER_ARANGO_INTEGRATION") == "" { t.Skip("set LOOM_COMPILER_ARANGO_INTEGRATION=1 to run policy ablation against Arango") @@ -75,10 +75,11 @@ func TestGDCOptimizationPolicyAblationAgainstArango(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - client, err := arango.Open(ctx, url, database) + arangoBackend, err := arango.Open(ctx, arango.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(ctx) var expectedHash string var expectedRows []map[string]any @@ -90,7 +91,7 @@ func TestGDCOptimizationPolicyAblationAgainstArango(t *testing.T) { t.Fatal(err) } rows := make([]map[string]any, 0, 1000) - err = client.QueryRows(ctx, compiled.Query, 1000, compiled.BindVars, func(row map[string]any) error { + err = client.QueryRows(ctx, testAQLText(t, compiled.Command), 1000, testAQLBinds(t, compiled.Command), func(row map[string]any) error { rows = append(rows, row) return nil }) @@ -111,8 +112,6 @@ func TestGDCOptimizationPolicyAblationAgainstArango(t *testing.T) { if index >= len(expectedRows) { break } - // JSON object encoding sorts map keys, making this a stable - // row-level diagnostic even though rows are decoded into maps. left, _ := json.Marshal(rows[index]) right, _ := json.Marshal(expectedRows[index]) if string(left) != string(right) { @@ -122,15 +121,11 @@ func TestGDCOptimizationPolicyAblationAgainstArango(t *testing.T) { } t.Fatalf("result hash = %s, want %s", resultHash, expectedHash) } - t.Logf("policy=%s rows=%d hash=%s aql_sha256=%x rule_states=%#v", candidate.name, len(rows), resultHash, sha256.Sum256([]byte(compiled.Query)), compiled.PlanDiagnostics.OptimizationPolicy.RuleStates) + t.Logf("policy=%s rows=%d hash=%s aql_sha256=%x rule_states=%#v", candidate.name, len(rows), resultHash, sha256.Sum256([]byte(testAQLText(t, compiled.Command))), compiled.PlanDiagnostics.OptimizationPolicy.RuleStates) }) } } -// TestTraversalSharingAblationAgainstArango profiles focused sibling and -// deep-traversal fixtures with sharing isolated from prepared selectors. The -// fixtures intentionally use an oracle project; this test rewrites only the -// project bind to the loaded local dataset and leaves route semantics intact. func TestTraversalSharingAblationAgainstArango(t *testing.T) { if os.Getenv("LOOM_COMPILER_ARANGO_INTEGRATION") == "" { t.Skip("set LOOM_COMPILER_ARANGO_INTEGRATION=1 to run traversal sharing profile") @@ -157,10 +152,11 @@ func TestTraversalSharingAblationAgainstArango(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - client, err := arango.Open(ctx, url, database) + arangoBackend, err := arango.Open(ctx, arango.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(ctx) for _, fixtureID := range []string{"patient-sibling-targets", "patient-deep-filter"} { fixture, ok := byID[fixtureID] @@ -187,7 +183,7 @@ func TestTraversalSharingAblationAgainstArango(t *testing.T) { for run := 0; run < 5; run++ { candidateRows := make([]map[string]any, 0, fixture.Limit) executeStart := time.Now() - err = client.QueryRows(ctx, compiled.Query, 1000, compiled.BindVars, func(row map[string]any) error { + err = client.QueryRows(ctx, testAQLText(t, compiled.Command), 1000, testAQLBinds(t, compiled.Command), func(row map[string]any) error { candidateRows = append(candidateRows, row) return nil }) @@ -225,17 +221,12 @@ func TestTraversalSharingAblationAgainstArango(t *testing.T) { } else if resultHash != expectedHash { t.Fatalf("%s result hash = %s, want %s", candidate.name, resultHash, expectedHash) } - t.Logf("policy=%s rows=%d hash=%s aql_sha256=%x warm_median=%0.6fs warm_min=%0.6fs runs=%#v shared=%d", candidate.name, len(rows), resultHash, sha256.Sum256([]byte(compiled.Query)), warmMedian, warm[0], executeSeconds, compiled.PlanDiagnostics.SharedTraversalCount) + t.Logf("policy=%s rows=%d hash=%s aql_sha256=%x warm_median=%0.6fs warm_min=%0.6fs runs=%#v shared=%d", candidate.name, len(rows), resultHash, sha256.Sum256([]byte(testAQLText(t, compiled.Command))), warmMedian, warm[0], executeSeconds, compiled.PlanDiagnostics.SharedTraversalCount) } }) } } -// TestPreparedSelectorAblationAgainstArango profiles the prepared selector -// payload independently from traversal sharing. It intentionally covers a -// child aggregate+slice, a child pivot, a nested child, and the full GDC -// dataframe so WP3 can make a generic payload decision instead of relying on -// one resource-specific query. func TestPreparedSelectorAblationAgainstArango(t *testing.T) { if os.Getenv("LOOM_COMPILER_ARANGO_INTEGRATION") == "" { t.Skip("set LOOM_COMPILER_ARANGO_INTEGRATION=1 to run prepared-selector profile") @@ -263,10 +254,11 @@ func TestPreparedSelectorAblationAgainstArango(t *testing.T) { opts := arango.ConnectionOptions{URL: url, Database: database} ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() - client, err := arango.Open(ctx, url, database) + arangoBackend, err := arango.Open(ctx, arango.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(ctx) for _, fixtureID := range []string{"specimen-aggregate-slice", "patient-observation-pivot", "patient-deep-filter", "gdc-case-matrix"} { fixture, ok := byID[fixtureID] @@ -294,7 +286,7 @@ func TestPreparedSelectorAblationAgainstArango(t *testing.T) { for run := 0; run < 5; run++ { candidateRows := make([]map[string]any, 0, fixture.Limit) executeStart := time.Now() - err = client.QueryRows(ctx, compiled.Query, 1000, compiled.BindVars, func(row map[string]any) error { + err = client.QueryRows(ctx, testAQLText(t, compiled.Command), 1000, testAQLBinds(t, compiled.Command), func(row map[string]any) error { candidateRows = append(candidateRows, row) return nil }) @@ -328,19 +320,13 @@ func TestPreparedSelectorAblationAgainstArango(t *testing.T) { t.Fatal(err) } assessment := arango.AssessExplainResult(explain) - preparedFields := strings.Count(compiled.Query, "__loom_prepared_") - t.Logf("policy=%s rows=%d response_bytes=%d hash=%s aql_sha256=%x prepared_tokens=%d warm_median=%0.6fs warm_min=%0.6fs runs=%#v indexes=%#v", candidate.name, len(rows), responseBytes, executionHash, sha256.Sum256([]byte(compiled.Query)), preparedFields, warm[len(warm)/2], warm[0], executeSeconds, assessment.Indexes) + preparedFields := strings.Count(testAQLText(t, compiled.Command), "__loom_prepared_") + t.Logf("policy=%s rows=%d response_bytes=%d hash=%s aql_sha256=%x prepared_tokens=%d warm_median=%0.6fs warm_min=%0.6fs runs=%#v indexes=%#v", candidate.name, len(rows), responseBytes, executionHash, sha256.Sum256([]byte(testAQLText(t, compiled.Command))), preparedFields, warm[len(warm)/2], warm[0], executeSeconds, assessment.Indexes) } }) } } -// TestRichConsumerReuseProfileAgainstArango records the baseline work for -// WP4's proposed rich-consumer fusion. It deliberately uses the production -// default (prepared selectors disabled after WP3) and reports the physical -// reuse groups plus the rendered loop count. There is no fused candidate in -// this test: fusion must be added only after a compatible group and a profile -// benefit are demonstrated. func TestRichConsumerReuseProfileAgainstArango(t *testing.T) { if os.Getenv("LOOM_COMPILER_ARANGO_INTEGRATION") == "" { t.Skip("set LOOM_COMPILER_ARANGO_INTEGRATION=1 to run rich-consumer profile") @@ -368,10 +354,11 @@ func TestRichConsumerReuseProfileAgainstArango(t *testing.T) { opts := arango.ConnectionOptions{URL: url, Database: database} ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() - client, err := arango.Open(ctx, url, database) + arangoBackend, err := arango.Open(ctx, arango.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(ctx) for _, fixtureID := range []string{"specimen-aggregate-slice", "gdc-case-matrix"} { fixture, ok := byID[fixtureID] @@ -391,7 +378,7 @@ func TestRichConsumerReuseProfileAgainstArango(t *testing.T) { for run := 0; run < 5; run++ { candidateRows := make([]map[string]any, 0, fixture.Limit) start := time.Now() - err = client.QueryRows(ctx, compiled.Query, 1000, compiled.BindVars, func(row map[string]any) error { + err = client.QueryRows(ctx, testAQLText(t, compiled.Command), 1000, testAQLBinds(t, compiled.Command), func(row map[string]any) error { candidateRows = append(candidateRows, row) return nil }) @@ -420,14 +407,11 @@ func TestRichConsumerReuseProfileAgainstArango(t *testing.T) { t.Fatal(err) } assessment := arango.AssessExplainResult(explain) - t.Logf("rows=%d response_bytes=%d hash=%s aql_sha256=%x warm_median=%0.6fs warm_min=%0.6fs runs=%#v rich_source_reuse=%#v rich_consumer_groups=%#v aql_for_loops=%d indexes=%#v", len(rows), responseBytes, resultHash, sha256.Sum256([]byte(compiled.Query)), warm[len(warm)/2], warm[0], executeSeconds, compiled.PlanDiagnostics.RichSourceReuse, compiled.PlanDiagnostics.RichConsumerGroups, strings.Count(compiled.Query, "FOR "), assessment.Indexes) + t.Logf("rows=%d response_bytes=%d hash=%s aql_sha256=%x warm_median=%0.6fs warm_min=%0.6fs runs=%#v rich_source_reuse=%#v rich_consumer_groups=%#v aql_for_loops=%d indexes=%#v", len(rows), responseBytes, resultHash, sha256.Sum256([]byte(testAQLText(t, compiled.Command))), warm[len(warm)/2], warm[0], executeSeconds, compiled.PlanDiagnostics.RichSourceReuse, compiled.PlanDiagnostics.RichConsumerGroups, strings.Count(testAQLText(t, compiled.Command), "FOR "), assessment.Indexes) }) } } -// TestCompactSetProjectionAblationAgainstArango compares full stored nodes -// with the typed identity-safe set projection while keeping traversal sharing -// and prepared selectors fixed. It is the WP5 evidence gate. func TestCompactSetProjectionAblationAgainstArango(t *testing.T) { if os.Getenv("LOOM_COMPILER_ARANGO_INTEGRATION") == "" { t.Skip("set LOOM_COMPILER_ARANGO_INTEGRATION=1 to run compact-set profile") @@ -455,10 +439,11 @@ func TestCompactSetProjectionAblationAgainstArango(t *testing.T) { opts := arango.ConnectionOptions{URL: url, Database: database} ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() - client, err := arango.Open(ctx, url, database) + arangoBackend, err := arango.Open(ctx, arango.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(ctx) for _, fixtureID := range []string{"specimen-aggregate-slice", "patient-specimen-file", "patient-deep-filter", "gdc-case-matrix"} { fixture, ok := byID[fixtureID] @@ -491,7 +476,7 @@ func TestCompactSetProjectionAblationAgainstArango(t *testing.T) { for run := 0; run < 5; run++ { candidateRows := make([]map[string]any, 0, limit) start := time.Now() - err = client.QueryRows(ctx, compiled.Query, 1000, compiled.BindVars, func(row map[string]any) error { + err = client.QueryRows(ctx, testAQLText(t, compiled.Command), 1000, testAQLBinds(t, compiled.Command), func(row map[string]any) error { candidateRows = append(candidateRows, row) return nil }) diff --git a/conformance/compiler/run_test.go b/conformance/compiler/run_test.go index d6d3dd3cb..6406f66a3 100644 --- a/conformance/compiler/run_test.go +++ b/conformance/compiler/run_test.go @@ -14,9 +14,6 @@ import ( "github.com/calypr/loom/internal/dataframe/spec" ) -// Result is the pure-compiler outcome for one oracle fixture. It intentionally -// stops before execution; result-row parity against META belongs to the Arango -// integration corpus. type Result struct { FixtureID string Compiled compiler.CompiledQuery @@ -38,7 +35,7 @@ func compileRecipe(bundle recipe.Bundle, project string, limit int, policy ir.Ph if err != nil { return compiler.CompiledQuery{}, err } - queries, err := compiler.CompileResolvedRecipePlanWithPolicy(resolved, limit, policy) + queries, err := compiler.CompileResolvedRecipePlan(compiler.ArangoTarget(), resolved, limit, policy) if err != nil { return compiler.CompiledQuery{}, err } @@ -75,18 +72,22 @@ func Verify(fixture Fixture) error { return fmt.Errorf("compiled optimizer rules %v do not contain %q", result.Compiled.OptimizationRules, want) } } + compiledText, compiledBinds, dispatchErr := result.Compiled.Command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } for _, fragment := range fixture.Expected.QueryContains { - if !strings.Contains(result.Compiled.Query, fragment) { + if !strings.Contains(compiledText, fragment) { return fmt.Errorf("compiled query does not contain %q", fragment) } } for _, fragment := range fixture.Expected.QueryNotContains { - if strings.Contains(result.Compiled.Query, fragment) { + if strings.Contains(compiledText, fragment) { return fmt.Errorf("compiled query unexpectedly contains %q", fragment) } } for key, want := range fixture.Expected.BindVars { - if got, ok := result.Compiled.BindVars[key]; !ok || fmt.Sprint(got) != fmt.Sprint(want) { + if got, ok := compiledBinds[key]; !ok || fmt.Sprint(got) != fmt.Sprint(want) { return fmt.Errorf("bind variable %q = %#v, want %#v", key, got, want) } } diff --git a/conformance/compiler/schema_roots_test.go b/conformance/compiler/schema_roots_test.go index 818f72c45..286be9b01 100644 --- a/conformance/compiler/schema_roots_test.go +++ b/conformance/compiler/schema_roots_test.go @@ -32,8 +32,8 @@ func TestPublicCompilerRejectsSchemaBackboneAndAbstractRoots(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "not represented by the active generated FHIR schema") { t.Fatalf("compile recipe root %q error = %v, want generated-schema rejection", resourceType, err) } - if compiled.Query != "" { - t.Fatalf("compile recipe root %q rendered AQL for a rejected root:\n%s", resourceType, compiled.Query) + if compiled.Command.Backend() != "" { + t.Fatalf("compile recipe root %q rendered AQL for a rejected root: backend %s", resourceType, compiled.Command.Backend()) } }) } diff --git a/conformance/compiler/types.go b/conformance/compiler/types.go index f0245d37f..183f3398e 100644 --- a/conformance/compiler/types.go +++ b/conformance/compiler/types.go @@ -1,4 +1,3 @@ -// Package compilerfixture loads the canonical recipe compiler oracle corpus. package compilerfixture import ( diff --git a/docs/DEVELOPER_ARCHITECTURE.md b/docs/DEVELOPER_ARCHITECTURE.md index a8d8c3e42..db28cef08 100644 --- a/docs/DEVELOPER_ARCHITECTURE.md +++ b/docs/DEVELOPER_ARCHITECTURE.md @@ -38,7 +38,9 @@ lifecycle or authoring types. Every HTTP method and path is defined in the canonical [`openapi/openapi.yaml`](../openapi/openapi.yaml) contract. Ownership map: `dataset` owns immutable FHIR generation lifecycle; `catalog` -owns persistence-neutral observed facts; `catalog/arango` owns catalog +owns persistence-neutral observed facts, the shared relationship-evidence +projection, and the cross-backend catalog read contracts (retained-source +keyset page, semantic coverage lookup); `catalog/arango` owns catalog persistence; `explorer` owns portable Explorer configs, editable drafts, immutable revisions, and publication state; `explorer/arango` owns its durable adapter; `dataframe/publication` owns dataframe publishing contracts and the @@ -83,6 +85,60 @@ column must not require GraphQL regeneration or a Loom restart. Publication and published-data reads are ClickHouse-only; Loom has no Elasticsearch publication or reader fallback. +## Backend composition and startup + +Server startup composes through one explicit seam. `run()` in +`internal/server/server.go` validates and normalizes the configuration and +then dispatches on `server.backend` only: + +- `arango` composes in `composeArangoRuntime` (`internal/server/compose.go`); +- `postgres` with the complete six-series placement ladder composes the full + runtime in `composePostgreSQLFullRuntime` (`internal/server/runtime.go`); +- every other `postgres` shape goes to the first-query route + (`composePostgreSQLFirstQueryRoute` in `server.go`), whose consumability + gate keeps the pinned startup refusal: a configuration that is not exactly + the `source` plus `dataset_release` pair receives `ErrPostgreSQLRuntimeNotWired` + before any dial and never falls through to the ArangoDB stores. + +Each arm opens its stores only through the three startup-only seams in +`compose.go` (`openArangoBackend`, `openPostgresBackend`, +`newClickHouseClient`). They are composition-private package variables read +once per startup; no request handler can reach them, so they are not a +service locator. `TestBackendCompositionContract` +(`internal/server/backend_composition_contract_test.go`) spies them and +pins the law that each selection opens exactly its own factories, once. + +Both ClickHouse-enabled arms build the identical publication plane through +`newPublicationPlane`, and both release-bearing arms share +`newReleaseHooks`, `buildLifecycleConfig` (the receipt-execution closures and +the typed lifecycle lane ports), and `wireExplorerRoutes`, which installs +routes through `registerRoutes` with typed `routeDependencies` and an +explicit `routeScopePolicy` that is validated before any route is installed +(a scoped policy without its resolver is a named refusal). The ClickHouse +plane is an independent lane: with `server.clickhouse.enabled` false no +ClickHouse client is constructed, and the first-query slice refuses the +combination outright. When the dataframe recipe engine is combined with the +publication plane, the `CombinePortsConfigured` guard runs immediately after +engine construction (ticket #146). + +Cleanup ownership is uniform. Each compose body keeps exactly one deferred +statement — the error-path sweep of a `cleanups` stack — and on success the +same stack is returned as the caller's closer. `serveRuntime` (the one +listen/shutdown loop in `compose.go`) owns that closer, so partial +construction, normal shutdown, and listen failure share one owner per +resource. `TestPostgresRuntimeComposeOwnsNoDeferredCleanup` witnesses both +production composes. + +The two backend row ports are tag-guarded at their own boundary +(`internal/server/row_sources.go`): the AQL lane serves AQL-tagged commands +only and the PostgreSQL lane serves PostgreSQL-tagged commands only, each +rejecting a cross-tagged command through the compiler's closed-dispatch +sentinel before any driver call. `TestRowCoreCrossTagGuardsRefusePreDispatch` +drives the guards directly. + +The shape is drawn in +[`architecture/integrated-runtime.mmd`](architecture/integrated-runtime.mmd). + ## Load and storage model `internal/ingest` reads an NDJSON directory, preflights it against the local @@ -105,7 +161,7 @@ as a compiler-selected physical optimization with write/read ownership, generation scope, freshness policy, and Explain coverage; it must not be added as an unused bootstrap collection. -Immutable loads use `internal/dataset` and `internal/dataset/arango`. Dataset +Immutable loads use `internal/dataset` and `internal/backends/arango/internal/dataset`. Dataset owns schema identity and generation manifests; ingest owns the Arango vertex and edge document shapes it writes. Their manifest records project, generation, and schema identity; the active pointer selects one READY generation per diff --git a/docs/EXPLORER_COMPILATION_ARCHITECTURE.md b/docs/EXPLORER_COMPILATION_ARCHITECTURE.md index 75c6df931..2537bc942 100644 --- a/docs/EXPLORER_COMPILATION_ARCHITECTURE.md +++ b/docs/EXPLORER_COMPILATION_ARCHITECTURE.md @@ -248,7 +248,7 @@ execution contract. | Compiler-proven immutable capability | `internal/explorer/capability` and `internal/server/explorer_capability.go` | | Native V2 translation | `internal/explorer/compilation` | | Versioned receipt domain and integrity | `internal/explorer/compilation_receipt.go` | -| Memory and Arango receipt persistence | `internal/explorer/memory_store.go`, `internal/explorer/arango/store.go` | +| Memory and Arango receipt persistence | `internal/explorer/memory_store.go`, `internal/backends/arango/internal/explorer/store.go` | | Explorer list/create/authoring/receipt/preview/publication workflows | `internal/explorer/lifecycle` | | Resolved-recipe execution boundary | `internal/dataframe/execution/engine.go` | | Generated HTTP request/response adapters | `internal/server/openapi_explorer_routes.go` | diff --git a/docs/GRAPHQL_API.md b/docs/GRAPHQL_API.md index a7aa3bab4..10b6483d5 100644 --- a/docs/GRAPHQL_API.md +++ b/docs/GRAPHQL_API.md @@ -53,11 +53,20 @@ mutation RunDataframe($input: FhirDataframeInput!, $limit: Int) { columns rows rowCount - diagnostics { compilationMs arangoQueryMs totalMs } + diagnostics { compilationMs sourceQueryMs arangoQueryMs totalMs } } } ``` +`sourceQueryMs` is the backend-neutral source cursor interval for whichever +backend serves the query. `arangoQueryMs` remains only as a legacy +compatibility projection: it equals `sourceQueryMs` for AQL-served requests +and is a literal `0` for PostgreSQL-served requests. Note the PostgreSQL +semantics change: before `sourceQueryMs` existed, PostgreSQL answers carried +the PostgreSQL cursor time in `arangoQueryMs`; consumers that read that field +on the PostgreSQL lane must switch to `sourceQueryMs`, which measures the +same cursor interval on both backends. + ```json { "limit": 100, diff --git a/docs/PACKAGE_AUDIT.csv b/docs/PACKAGE_AUDIT.csv index 23fbc84a1..f1172b9cb 100644 --- a/docs/PACKAGE_AUDIT.csv +++ b/docs/PACKAGE_AUDIT.csv @@ -1,76 +1,95 @@ location,import_path,package_name,kind,responsibility,production_files,test_files,other_files,generated_files,production_loc,test_loc,exported_symbol_count,exported_symbols,internal_dependencies,test_dependencies,production_importers,test_only_importers,external_dependencies,runtime_signals,compatibility_signals,decision_evidence,test_scope,audit_status .,github.com/calypr/loom,main,executable,Start the default Loom server process through the internal server composition root.,main.go,—,go.mod,—,7,0,0,—,internal/server,—,—,—,—,—,—,executable boundary; compare orchestration with sibling commands before combining,go test .,inventory complete; semantic disposition pending -cmd/arango-fhir-proto,github.com/calypr/loom/cmd/arango-fhir-proto,main,executable,"Provide operator commands for FHIR loading, catalog diagnostics, generation repair, and activation.",main.go,main_test.go; resource_inventory_test.go,—,—,846,470,0,—,internal/catalog; internal/catalog/arango; internal/dataset; internal/dataset/arango; internal/fhir/schema; internal/ingest; internal/store/arango,internal/catalog,—,—,github.com/arangodb/go-driver/v2/arangodb; github.com/arangodb/go-driver/v2/connection,—,legacy=4; migration=1,executable boundary; compare orchestration with sibling commands before combining,go test ./cmd/arango-fhir-proto,inventory complete; semantic disposition pending +cmd/arango-fhir-proto,github.com/calypr/loom/cmd/arango-fhir-proto,main,executable,"Provide operator commands for FHIR loading, catalog diagnostics, generation repair, and activation.",main.go,main_test.go; resource_inventory_test.go,—,—,842,470,0,—,internal/backends/arango; internal/catalog; internal/dataset; internal/fhir/schema; internal/ingest,internal/catalog,—,—,—,—,legacy=4; migration=1,executable boundary; compare orchestration with sibling commands before combining,go test ./cmd/arango-fhir-proto,inventory complete; semantic disposition pending cmd/arango-fhir-server,github.com/calypr/loom/cmd/arango-fhir-server,main,executable,Start the explicit Loom HTTP server binary.,main.go,—,—,—,7,0,0,—,internal/server,—,—,—,—,—,—,executable boundary; compare orchestration with sibling commands before combining,go test ./cmd/arango-fhir-server,inventory complete; semantic disposition pending -cmd/dataframe-profile,github.com/calypr/loom/cmd/dataframe-profile,main,executable,Run and compare developer-facing dataframe profiling workloads.,main.go,—,—,—,322,0,0,—,conformance/compiler; generated/graphql/graph/model; internal/api/graphql/graph/query; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/recipe; internal/dataframe/semantic; internal/store/arango,—,—,—,—,—,legacy=1,executable boundary; compare orchestration with sibling commands before combining,go test ./cmd/dataframe-profile,inventory complete; semantic disposition pending +cmd/dataframe-profile,github.com/calypr/loom/cmd/dataframe-profile,main,executable,Run and compare developer-facing dataframe profiling workloads.,main.go,—,—,—,331,0,0,—,conformance/compiler; generated/graphql/graph/model; internal/api/graphql/graph/query; internal/backends/arango; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/recipe; internal/dataframe/semantic,—,—,—,—,—,legacy=1,executable boundary; compare orchestration with sibling commands before combining,go test ./cmd/dataframe-profile,inventory complete; semantic disposition pending cmd/dataframe-query,github.com/calypr/loom/cmd/dataframe-query,main,executable,Execute a checked-in GraphQL dataframe request and report response timing and diagnostics.,main.go,main_test.go,—,—,459,432,3,Error; String; Unwrap,—,—,—,—,—,—,compatibility=1,executable boundary; compare orchestration with sibling commands before combining,go test ./cmd/dataframe-query,inventory complete; semantic disposition pending cmd/explorer-config-v2-convert,github.com/calypr/loom/cmd/explorer-config-v2-convert,main,executable,Convert legacy Explorer configuration input into the V2 authoring representation offline.,main.go,main_test.go,—,—,377,62,0,—,internal/dataframe/recipe; internal/explorer; internal/explorer/authoringv2,internal/explorer/authoringv2,—,—,—,—,legacy=3,executable boundary; compare orchestration with sibling commands before combining,go test ./cmd/explorer-config-v2-convert,inventory complete; semantic disposition pending cmd/generate,github.com/calypr/loom/cmd/generate,main,executable,"Generate FHIR models, schema metadata, validation, extraction, and GraphQL artifacts from source schemas.",extraction.go; graphql.go; helpers.go; main.go; metadata.go; model.go; names.go; resources.go; schema.go; validation.go,extraction_test.go; graphql_test.go; metadata_test.go; model_test.go; resources_test.go; root_test.go; schema_fixture_test.go,fhir_helpers.go.tmpl,—,1750,522,10,BindingStrength; Definition; EdgeDocument; Link; Property; Result; Schema; TargetHints; TargetSchema; TemplatePointers,—,—,—,—,—,go:embed,compatibility=1,executable boundary; compare orchestration with sibling commands before combining; runtime/build signals present; static importer counts are insufficient for deletion,go test ./cmd/generate,inventory complete; semantic disposition pending cmd/gqlgenfix,github.com/calypr/loom/cmd/gqlgenfix,main,executable,Post-process gqlgen output to enforce Loom-specific generated-code invariants.,main.go,main_test.go,—,—,173,67,1,MarshalJSON,—,—,—,—,—,—,—,executable boundary; compare orchestration with sibling commands before combining,go test ./cmd/gqlgenfix,inventory complete; semantic disposition pending cmd/loom-acceptance,github.com/calypr/loom/cmd/loom-acceptance,main,executable,Run the locked real-data acceptance scenario and emit machine-readable verification artifacts.,main.go,main_test.go,—,—,225,46,0,—,internal/acceptance; internal/explorer,internal/explorer,—,—,—,—,—,executable boundary; compare orchestration with sibling commands before combining,go test ./cmd/loom-acceptance,inventory complete; semantic disposition pending +cmd/loom-data,github.com/calypr/loom/cmd/loom-data,main,executable,"Load one immutable generation into the backend chosen only by an explicit arango or postgres tag, routing to the Arango loader or the PostgreSQL lifecycle loader without inference or fallback.",load.go; main.go,load_test.go; main_test.go,—,—,964,1703,2,Error; Unwrap,internal/backends/arango; internal/backends/postgres; internal/backends/postgres/control; internal/catalog; internal/dataset; internal/ingest,internal/backends/postgres/control; internal/catalog; internal/dataset; internal/ingest,—,—,—,—,—,executable boundary; compare orchestration with sibling commands before combining,go test ./cmd/loom-data,inventory complete; semantic disposition pending cmd/loom-repository,github.com/calypr/loom/cmd/loom-repository,main,executable,Preflight a repository checkout and launch Loom against its META and CONFIG data with an owned Compose project.,main.go,main_test.go,—,—,426,231,0,—,internal/projectid; internal/repositoryseed,internal/projectid; internal/repositoryseed,—,—,—,—,legacy=2,executable boundary; compare orchestration with sibling commands before combining,go test ./cmd/loom-repository,inventory complete; semantic disposition pending -cmd/preview-covering-index-artifact,github.com/calypr/loom/cmd/preview-covering-index-artifact,main,executable,Recreate a captured Builder proposal offline and emit the exact compiler artifact for index-coverage analysis.,main.go,main_test.go,—,—,1433,34,0,—,internal/authscope; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation,—,—,—,—,—,migration=1,executable boundary; compare orchestration with sibling commands before combining,go test ./cmd/preview-covering-index-artifact,inventory complete; semantic disposition pending -conformance/compiler,github.com/calypr/loom/conformance/compiler,compilerfixture,conformance,Load and exercise the canonical dataframe compiler oracle corpus for conformance and performance checks.,types.go,benchmark_test.go; corpus_coverage_test.go; gdc_fixture_test.go; generated_semantics_coverage_test.go; physical_plan_cost_test.go; policy_ablation_test.go; run_test.go; schema_roots_test.go; types_test.go,—,—,119,1226,4,Expected; Fixture; LoadDir; Validate,internal/dataframe/recipe,internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/execution; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec; internal/fhir/schema; internal/store/arango,cmd/dataframe-profile,—,—,—,—,non-runtime conformance boundary,go test ./conformance/compiler,inventory complete; semantic disposition pending +cmd/preview-covering-index-artifact,github.com/calypr/loom/cmd/preview-covering-index-artifact,main,executable,Recreate a captured Builder proposal offline and emit the exact compiler artifact for index-coverage analysis.,main.go,main_test.go,—,—,1449,34,0,—,internal/authscope; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation,—,—,—,—,—,migration=1,executable boundary; compare orchestration with sibling commands before combining,go test ./cmd/preview-covering-index-artifact,inventory complete; semantic disposition pending +conformance/compiler,github.com/calypr/loom/conformance/compiler,compilerfixture,conformance,Load and exercise the canonical dataframe compiler oracle corpus for conformance and performance checks.,types.go,benchmark_test.go; command_helpers_test.go; corpus_coverage_test.go; gdc_fixture_test.go; generated_semantics_coverage_test.go; physical_plan_cost_test.go; policy_ablation_test.go; run_test.go; schema_roots_test.go; types_test.go,—,—,119,1279,4,Expected; Fixture; LoadDir; Validate,internal/dataframe/recipe,internal/backends/arango; internal/dataframe/compiler; internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec; internal/fhir/schema,cmd/dataframe-profile,—,—,—,—,non-runtime conformance boundary,go test ./conformance/compiler,inventory complete; semantic disposition pending generated/fhir,github.com/calypr/loom/generated/fhir,fhir,generated,"Expose generated FHIR resource models and their extraction, validation, and GraphQL helpers.",extract.go; graphql.go; helpers.go; model.go; resources.go; validate.go,—,—,extract.go; graphql.go; helpers.go; model.go; resources.go; validate.go,72020,0,155,Address; Age; Annotation; Attachment; Availability; AvailabilityAvailableTime; AvailabilityNotAvailableTime; BodyStructure; BodyStructureIncludedStructure; BodyStructureIncludedStructureBodyLandmarkOrientation; BodyStructureIncludedStructureBodyLandmarkOrientationDistanceFromLandmark; CodeableConcept; CodeableReference; Coding; ConcreteResource; Condition; ConditionParticipant; ConditionStage; ContactDetail; ContactPoint; Count; DataRequirement; DataRequirementCodeFilter; DataRequirementDateFilter; DataRequirementSort; DataRequirementValueFilter; DiagnosticReport; DiagnosticReportMedia; DiagnosticReportSupportingInfo; Directory; Distance; DocumentReference; DocumentReferenceAttester; DocumentReferenceContent; DocumentReferenceContentProfile; DocumentReferenceRelatesTo; Dosage; DosageDoseAndRate; Duration; EdgeDocument; Expression; ExtendedContactDetail; Extension; ExtractEdges; FHIRComments; FHIRPrimitiveExtension; FamilyMemberHistory; FamilyMemberHistoryCondition; FamilyMemberHistoryParticipant; FamilyMemberHistoryProcedure; GetID; GetResourceType; Group; GroupCharacteristic; GroupMember; HumanName; Identifier; ImagingStudy; ImagingStudySeries; ImagingStudySeriesInstance; ImagingStudySeriesPerformer; IsFHIRResource; MarshalJSON; Medication; MedicationAdministration; MedicationAdministrationDosage; MedicationAdministrationPerformer; MedicationBatch; MedicationIngredient; MedicationRequest; MedicationRequestDispenseRequest; MedicationRequestDispenseRequestInitialFill; MedicationRequestSubstitution; MedicationStatement; MedicationStatementAdherence; Meta; Money; Narrative; NewConcreteResource; Observation; ObservationComponent; ObservationReferenceRange; ObservationTriggeredBy; Organization; OrganizationQualification; ParameterDefinition; Patient; PatientCommunication; PatientContact; PatientLink; Period; Practitioner; PractitionerCommunication; PractitionerQualification; PractitionerRole; Procedure; ProcedureFocalDevice; ProcedurePerformer; Quantity; Range; Ratio; RatioRange; Reference; RelatedArtifact; ResearchStudy; ResearchStudyAssociatedParty; ResearchStudyComparisonGroup; ResearchStudyLabel; ResearchStudyObjective; ResearchStudyOutcomeMeasure; ResearchStudyProgressStatus; ResearchStudyRecruitment; ResearchSubject; ResearchSubjectProgress; Resource; Result; SampledData; Signature; Specimen; SpecimenCollection; SpecimenContainer; SpecimenFeature; SpecimenProcessing; Substance; SubstanceDefinition; SubstanceDefinitionCharacterization; SubstanceDefinitionCode; SubstanceDefinitionMoiety; SubstanceDefinitionMolecularWeight; SubstanceDefinitionName; SubstanceDefinitionNameOfficial; SubstanceDefinitionProperty; SubstanceDefinitionRelationship; SubstanceDefinitionSourceMaterial; SubstanceDefinitionStructure; SubstanceDefinitionStructureRepresentation; SubstanceIngredient; Task; TaskInput; TaskOutput; TaskPerformer; TaskRestriction; Timing; TimingRepeat; TriggerDefinition; UnmarshalJSON; UsageContext; Validate; ValidateAndExtract; ValidateFhirBinary; ValidateFhirDate; ValidateFhirDateTime; ValidateFhirTime; ValidateFhirURI; ValidateFhirUUID,internal/fhir/schema,—,generated/graphql/graph/executor; internal/api/graphql/graph/resolver; internal/dataframe/recipe; internal/dataframe/spec; internal/ingest,—,github.com/bytedance/sonic; github.com/google/uuid,1 exported interface(s),—,"generated boundary; change its source/generator, not emitted code; runtime/build signals present; static importer counts are insufficient for deletion",go test ./generated/fhir,inventory complete; semantic disposition pending -generated/fhirschema,github.com/calypr/loom/generated/fhirschema,fhirschema,generated,Expose generated raw FHIR schema metadata tables.,generated.go,—,—,generated.go,55277,0,3,Definition; Property; Traversal,—,—,internal/fhir/schema,—,—,—,deprecated=1; compatibility=1,"generated boundary; change its source/generator, not emitted code",go test ./generated/fhirschema,inventory complete; semantic disposition pending +generated/fhirschema,github.com/calypr/loom/generated/fhirschema,fhirschema,generated,Expose generated raw FHIR schema metadata tables.,generated.go,—,—,generated.go,55277,0,3,Definition; Property; Traversal,—,—,internal/fhir/schema,internal/catalog,—,—,deprecated=1; compatibility=1,"generated boundary; change its source/generator, not emitted code",go test ./generated/fhirschema,inventory complete; semantic disposition pending generated/graphql/graph/executor,github.com/calypr/loom/generated/graphql/graph/executor,executor,generated,Provide gqlgen-generated GraphQL executable schema and execution plumbing.,fhir_schema.generated.go; prelude.generated.go; root_.generated.go; schema.generated.go,—,—,fhir_schema.generated.go; prelude.generated.go; root_.generated.go; schema.generated.go,155289,0,328,Address; AddressResolver; Age; AgeResolver; Annotation; AnnotationResolver; Attachment; AttachmentResolver; Availability; AvailabilityAvailableTime; AvailabilityAvailableTimeResolver; AvailabilityNotAvailableTime; AvailabilityNotAvailableTimeResolver; AvailabilityResolver; BodyStructure; BodyStructureIncludedStructure; BodyStructureIncludedStructureBodyLandmarkOrientation; BodyStructureIncludedStructureBodyLandmarkOrientationDistanceFromLandmark; BodyStructureIncludedStructureBodyLandmarkOrientationDistanceFromLandmarkResolver; BodyStructureIncludedStructureBodyLandmarkOrientationResolver; BodyStructureIncludedStructureResolver; BodyStructureResolver; CodeableConcept; CodeableConceptResolver; CodeableReference; CodeableReferenceResolver; Coding; CodingResolver; Complexity; ComplexityRoot; Condition; ConditionParticipant; ConditionParticipantResolver; ConditionResolver; ConditionStage; ConditionStageResolver; Config; ContactDetail; ContactDetailResolver; ContactPoint; ContactPointResolver; Count; CountResolver; DataRequirement; DataRequirementCodeFilter; DataRequirementCodeFilterResolver; DataRequirementDateFilter; DataRequirementDateFilterResolver; DataRequirementResolver; DataRequirementSort; DataRequirementSortResolver; DataRequirementValueFilter; DataRequirementValueFilterResolver; DataframeAggregateResult; DataframeAggregationsResult; DataframeColumn; DataframeCompilerPlanDiagnostics; DataframeMaterialization; DataframeOptimizationDecision; DataframeOptimizationPolicy; DataframePageInfo; DataframeQueryDiagnostics; DataframeRecipeArangoAssessment; DataframeRecipeColumn; DataframeRecipeExecution; DataframeRecipeExecutionOutput; DataframeRecipeExpansionExplanation; DataframeRecipeExplainCollectionScan; DataframeRecipeExplainIndexLocation; DataframeRecipeExplainIndexSummary; DataframeRecipeExplainPlanEstimate; DataframeRecipeExplainWarning; DataframeRecipeExplanation; DataframeRecipeExpressionExplanation; DataframeRecipeOptimizationDecision; DataframeRecipeOptimizationExplanation; DataframeRecipeOptimizationRuleState; DataframeRecipeOutputExplanation; DataframeRecipeOutputValidation; DataframeRecipePhysicalExplanation; DataframeRecipePhysicalOutputExplanation; DataframeRecipePreflight; DataframeRecipePreview; DataframeRecipePreviewOutput; DataframeRecipeResult; DataframeRecipeResultOutput; DataframeRecipeRevision; DataframeRecipeValidation; DataframeRichSourceReuse; DataframeRowConnection; DataframeSelector; DiagnosticReport; DiagnosticReportMedia; DiagnosticReportMediaResolver; DiagnosticReportResolver; DiagnosticReportSupportingInfo; DiagnosticReportSupportingInfoResolver; DirectiveRoot; Directory; Distance; DistanceResolver; DocumentReference; DocumentReferenceAttester; DocumentReferenceAttesterResolver; DocumentReferenceContent; DocumentReferenceContentProfile; DocumentReferenceContentProfileResolver; DocumentReferenceContentResolver; DocumentReferenceRelatesTo; DocumentReferenceRelatesToResolver; DocumentReferenceResolver; Dosage; DosageDoseAndRate; DosageDoseAndRateResolver; DosageResolver; Duration; DurationResolver; Exec; Expression; ExpressionResolver; ExtendedContactDetail; ExtendedContactDetailResolver; Extension; ExtensionResolver; FHIRPrimitiveExtension; FHIRPrimitiveExtensionResolver; FamilyMemberHistory; FamilyMemberHistoryCondition; FamilyMemberHistoryConditionResolver; FamilyMemberHistoryParticipant; FamilyMemberHistoryParticipantResolver; FamilyMemberHistoryProcedure; FamilyMemberHistoryProcedureResolver; FamilyMemberHistoryResolver; FhirDataframeResult; FhirGraphNode; FhirGraphPageInfo; FhirGraphPath; FhirGraphQueryExplanation; FhirGraphQueryResult; FhirGraphRelationship; Group; GroupCharacteristic; GroupCharacteristicResolver; GroupMember; GroupMemberResolver; GroupResolver; HumanName; HumanNameResolver; Identifier; IdentifierResolver; ImagingStudy; ImagingStudyResolver; ImagingStudySeries; ImagingStudySeriesInstance; ImagingStudySeriesInstanceResolver; ImagingStudySeriesPerformer; ImagingStudySeriesPerformerResolver; ImagingStudySeriesResolver; MarshalJSON; Medication; MedicationAdministration; MedicationAdministrationDosage; MedicationAdministrationDosageResolver; MedicationAdministrationPerformer; MedicationAdministrationPerformerResolver; MedicationAdministrationResolver; MedicationBatch; MedicationBatchResolver; MedicationIngredient; MedicationIngredientResolver; MedicationRequest; MedicationRequestDispenseRequest; MedicationRequestDispenseRequestInitialFill; MedicationRequestDispenseRequestInitialFillResolver; MedicationRequestDispenseRequestResolver; MedicationRequestResolver; MedicationRequestSubstitution; MedicationRequestSubstitutionResolver; MedicationResolver; MedicationStatement; MedicationStatementAdherence; MedicationStatementAdherenceResolver; MedicationStatementResolver; Meta; MetaResolver; Money; MoneyResolver; Mutation; MutationResolver; Narrative; NarrativeResolver; NewExecutableSchema; Observation; ObservationComponent; ObservationComponentResolver; ObservationReferenceRange; ObservationReferenceRangeResolver; ObservationResolver; ObservationTriggeredBy; ObservationTriggeredByResolver; Organization; OrganizationQualification; OrganizationQualificationResolver; OrganizationResolver; ParameterDefinition; ParameterDefinitionResolver; Patient; PatientCommunication; PatientCommunicationResolver; PatientContact; PatientContactResolver; PatientLink; PatientLinkResolver; PatientResolver; Period; PeriodResolver; Practitioner; PractitionerCommunication; PractitionerCommunicationResolver; PractitionerQualification; PractitionerQualificationResolver; PractitionerResolver; PractitionerRole; PractitionerRoleResolver; Procedure; ProcedureFocalDevice; ProcedureFocalDeviceResolver; ProcedurePerformer; ProcedurePerformerResolver; ProcedureResolver; Quantity; QuantityResolver; QueryResolver; Range; RangeResolver; Ratio; RatioRange; RatioRangeResolver; RatioResolver; Reference; ReferenceResolver; RelatedArtifact; RelatedArtifactResolver; ResearchStudy; ResearchStudyAssociatedParty; ResearchStudyAssociatedPartyResolver; ResearchStudyComparisonGroup; ResearchStudyComparisonGroupResolver; ResearchStudyLabel; ResearchStudyLabelResolver; ResearchStudyObjective; ResearchStudyObjectiveResolver; ResearchStudyOutcomeMeasure; ResearchStudyOutcomeMeasureResolver; ResearchStudyProgressStatus; ResearchStudyProgressStatusResolver; ResearchStudyRecruitment; ResearchStudyRecruitmentResolver; ResearchStudyResolver; ResearchSubject; ResearchSubjectProgress; ResearchSubjectProgressResolver; ResearchSubjectResolver; ResolverRoot; Resource; ResourceResolver; SampledData; SampledDataResolver; Schema; Signature; SignatureResolver; Specimen; SpecimenCollection; SpecimenCollectionResolver; SpecimenContainer; SpecimenContainerResolver; SpecimenFeature; SpecimenFeatureResolver; SpecimenProcessing; SpecimenProcessingResolver; SpecimenResolver; Substance; SubstanceDefinition; SubstanceDefinitionCharacterization; SubstanceDefinitionCharacterizationResolver; SubstanceDefinitionCode; SubstanceDefinitionCodeResolver; SubstanceDefinitionMoiety; SubstanceDefinitionMoietyResolver; SubstanceDefinitionMolecularWeight; SubstanceDefinitionMolecularWeightResolver; SubstanceDefinitionName; SubstanceDefinitionNameOfficial; SubstanceDefinitionNameOfficialResolver; SubstanceDefinitionNameResolver; SubstanceDefinitionProperty; SubstanceDefinitionPropertyResolver; SubstanceDefinitionRelationship; SubstanceDefinitionRelationshipResolver; SubstanceDefinitionResolver; SubstanceDefinitionSourceMaterial; SubstanceDefinitionSourceMaterialResolver; SubstanceDefinitionStructure; SubstanceDefinitionStructureRepresentation; SubstanceDefinitionStructureRepresentationResolver; SubstanceDefinitionStructureResolver; SubstanceIngredient; SubstanceIngredientResolver; SubstanceResolver; Task; TaskInput; TaskInputResolver; TaskOutput; TaskOutputResolver; TaskPerformer; TaskPerformerResolver; TaskResolver; TaskRestriction; TaskRestrictionResolver; Timing; TimingRepeat; TimingRepeatResolver; TimingResolver; TriggerDefinition; TriggerDefinitionResolver; UsageContext; UsageContextResolver,generated/fhir; generated/graphql/graph/model,—,internal/api/graphql/graph; internal/api/graphql/graph/resolver,—,github.com/99designs/gqlgen/graphql; github.com/99designs/gqlgen/graphql/introspection; github.com/vektah/gqlparser/v2; github.com/vektah/gqlparser/v2/ast,138 exported interface(s),—,"generated boundary; change its source/generator, not emitted code; runtime/build signals present; static importer counts are insufficient for deletion",go test ./generated/graphql/graph/executor,inventory complete; semantic disposition pending generated/graphql/graph/model,github.com/calypr/loom/generated/graphql/graph/model,model,generated,Provide gqlgen-generated GraphQL transport model types.,models.go,—,—,models.go,1333,0,95,DataframeAggregateInput; DataframeAggregateResult; DataframeAggregationSpecInput; DataframeAggregationsInput; DataframeAggregationsResult; DataframeColumn; DataframeCompilerPlanDiagnostics; DataframeDatasetInput; DataframeFilterInput; DataframeMaterialization; DataframeMaterializationState; DataframeOptimizationDecision; DataframeOptimizationPolicy; DataframePageInfo; DataframeQueryDiagnostics; DataframeRecipeArangoAssessment; DataframeRecipeBindingsInput; DataframeRecipeColumn; DataframeRecipeExecution; DataframeRecipeExecutionOutput; DataframeRecipeExecutionState; DataframeRecipeExpansionExplanation; DataframeRecipeExplainCollectionScan; DataframeRecipeExplainIndexLocation; DataframeRecipeExplainIndexSummary; DataframeRecipeExplainPlanEstimate; DataframeRecipeExplainWarning; DataframeRecipeExplanation; DataframeRecipeExpressionExplanation; DataframeRecipeOptimizationDecision; DataframeRecipeOptimizationExplanation; DataframeRecipeOptimizationRuleState; DataframeRecipeOutputExplanation; DataframeRecipeOutputValidation; DataframeRecipePhysicalExplanation; DataframeRecipePhysicalOutputExplanation; DataframeRecipePreflight; DataframeRecipePreview; DataframeRecipePreviewOutput; DataframeRecipeResult; DataframeRecipeResultOutput; DataframeRecipeRevision; DataframeRecipeValidation; DataframeRichSourceReuse; DataframeRowConnection; DataframeRowsInput; DataframeSelector; DataframeSelectorInput; DataframeSortInput; ExplainDataframeRecipeInput; FHIRResource; FHIRResourceType; FhirAggregateInput; FhirAggregateOperation; FhirCatalogProjectionInput; FhirCodeValueInput; FhirColumnNaming; FhirDataframeInput; FhirDataframeResult; FhirFieldPredicateInput; FhirFieldPredicateOperation; FhirFieldSelectInput; FhirFieldSelectorInput; FhirFilterInput; FhirFilterOperator; FhirFilterQuantifier; FhirFilterValueInput; FhirFilterValueKind; FhirGraphNode; FhirGraphPageInfo; FhirGraphPath; FhirGraphQueryExplanation; FhirGraphQueryInput; FhirGraphQueryResult; FhirGraphRelationship; FhirGraphTraversalInput; FhirPivotDiscoveryInput; FhirPivotInput; FhirRepresentativeSliceInput; FhirTraversalMatchMode; FhirTraversalStepInput; FhirValueMode; IsValid; MarshalGQL; MarshalJSON; Mutation; PreflightDataframeRecipeInput; PreviewDataframeRecipeInput; Query; RegisterDataframeRecipeRevisionInput; RunDataframeRecipeInput; String; UnmarshalGQL; UnmarshalJSON; ValidateDataframeRecipeInput,—,—,cmd/dataframe-profile; generated/graphql/graph/executor; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver,internal/server,—,1 exported interface(s),—,"generated boundary; change its source/generator, not emitted code; runtime/build signals present; static importer counts are insufficient for deletion",go test ./generated/graphql/graph/model,inventory complete; semantic disposition pending generated/loomapi,github.com/calypr/loom/generated/loomapi,loomapi,generated,"Provide OpenAPI-generated HTTP request, response, and strict-server contracts.",api.gen.go; strict_json.go,construction_json_test.go; correlated_json_test.go; request_boundary_test.go; selection_json_test.go; strict_json_test.go,—,api.gen.go,23408,512,1620,APIVersion; AcceptedConstruction; AcceptedConstructionReceipt; Action; ActionColumn; ActivateDatasetGeneration; ActivateDatasetGeneration200JSONResponse; ActivateDatasetGeneration400JSONResponse; ActivateDatasetGeneration401JSONResponse; ActivateDatasetGeneration403JSONResponse; ActivateDatasetGeneration409JSONResponse; ActivateDatasetGeneration500JSONResponse; ActivateDatasetGeneration503JSONResponse; ActivateDatasetGenerationParams; ActivateDatasetGenerationRequestObject; ActivateDatasetGenerationResponseObject; AggregateOperationCapability; AggregateOperationCapabilityOperation; AggregateOperationCapabilityResultCardinality; AggregateOperationCapabilityRowContext; AggregateSource; AggregateSourceOperation; AggregateTransformationCapabilities; ApplyCommandsRequest; ApplyCommandsResponse; ApplyExplorerBuilderCommands; ApplyExplorerBuilderCommands200JSONResponse; ApplyExplorerBuilderCommands400JSONResponse; ApplyExplorerBuilderCommands401JSONResponse; ApplyExplorerBuilderCommands403JSONResponse; ApplyExplorerBuilderCommands404JSONResponse; ApplyExplorerBuilderCommands409JSONResponse; ApplyExplorerBuilderCommands422JSONResponse; ApplyExplorerBuilderCommands500JSONResponse; ApplyExplorerBuilderCommands503JSONResponse; ApplyExplorerBuilderCommandsJSONRequestBody; ApplyExplorerBuilderCommandsParams; ApplyExplorerBuilderCommandsRequestObject; ApplyExplorerBuilderCommandsResponseObject; Artifact; ArtifactFormat; ArtifactMediaType; ArtifactPrepareRequest; ArtifactState; AsConfiguredColumnReadyResolution; AsConfiguredColumnUnavailableResolution; AsConstructionGroupCodingIdentitySourceOption; AsConstructionGroupFieldScalarSourceOption; AsConstructionGroupFieldSourceChoice; AsConstructionGroupFieldSourceCodingIdentityBinding; AsConstructionGroupFieldSourceCurrentStageInput; AsConstructionGroupFieldSourceInputBinding; AsConstructionGroupFieldSourceRelatedOccurrenceInput; AsConstructionGroupFieldSourceScalarBinding; AsConstructionGroupKeyFieldSource; AsConstructionGroupKeyInputColumn; AsConstructionGroupKeySemanticCategory; AsConstructionGroupSemanticCategoryChoice; AsConstructionRowValue0; AsConstructionRowValue1; AsContributorCodeValue; AsContributorStringValue; AsExactCategoryRecodeTransformation; AsExactSemanticInventoryItemIdentity; AsFamilySemanticInventoryItemIdentity; AsFieldChoiceSource; AsFieldConstructionChoiceSearchSource; AsFieldPresentationGroupUnderSemantic; AsFieldPresentationKeepRaw; AsFieldPresentationReady; AsFieldPresentationUnavailable; AsLegacyErrorBody; AsLegacyErrorResponseError0; AsRemoveColumnTransformation; AsRowDefinitionComparisonAvailable; AsRowDefinitionComparisonUnavailable; AsRowDefinitionExpandedSelection; AsRowDefinitionExplicitGroupSelection; AsRowDefinitionFieldGroupSelection; AsRowDefinitionRecordsSelection; AsSemanticBindingChoiceSource; AsSemanticConstructionChoiceSearchSource; AsSetColumnTransformation; AsTableShapeComparisonAvailable; AsTableShapeComparisonUnavailable; AssessExplorerRowChange; AssessExplorerRowChange200JSONResponse; AssessExplorerRowChange400JSONResponse; AssessExplorerRowChange401JSONResponse; AssessExplorerRowChange403JSONResponse; AssessExplorerRowChange404JSONResponse; AssessExplorerRowChange409JSONResponse; AssessExplorerRowChange422JSONResponse; AssessExplorerRowChange500JSONResponse; AssessExplorerRowChange503JSONResponse; AssessExplorerRowChangeJSONRequestBody; AssessExplorerRowChangeRequestObject; AssessExplorerRowChangeResponseObject; AssessRowChangeRequest; AssessRowChangeRequestMode; AuthResourcePath; AuthoringBadRequest; AuthoringBadRequestJSONResponse; AuthoringCapability; AuthoringCapabilityKind; AuthoringCapabilityOperations; AuthoringClientClosedRequest; AuthoringClientClosedRequestJSONResponse; AuthoringCommand; AuthoringCommandForm; AuthoringCommandInitialPresentation; AuthoringCommandMatchMode; AuthoringCommandRowValuePolicy; AuthoringCommandType; AuthoringConflict; AuthoringConflictJSONResponse; AuthoringFeatures; AuthoringForbidden; AuthoringForbiddenJSONResponse; AuthoringGatewayTimeout; AuthoringGatewayTimeoutJSONResponse; AuthoringInternalError; AuthoringInternalErrorJSONResponse; AuthoringNotFound; AuthoringNotFoundJSONResponse; AuthoringPayloadTooLarge; AuthoringPayloadTooLargeJSONResponse; AuthoringTooManyRequests; AuthoringTooManyRequestsJSONResponse; AuthoringUnavailable; AuthoringUnavailableJSONResponse; AuthoringUnprocessable; AuthoringUnprocessableJSONResponse; BrowseExplorerFrameSourceOptions; BrowseExplorerFrameSourceOptions200JSONResponse; BrowseExplorerFrameSourceOptions400JSONResponse; BrowseExplorerFrameSourceOptions401JSONResponse; BrowseExplorerFrameSourceOptions403JSONResponse; BrowseExplorerFrameSourceOptions404JSONResponse; BrowseExplorerFrameSourceOptions409JSONResponse; BrowseExplorerFrameSourceOptions422JSONResponse; BrowseExplorerFrameSourceOptions500JSONResponse; BrowseExplorerFrameSourceOptions503JSONResponse; BrowseExplorerFrameSourceOptionsJSONRequestBody; BrowseExplorerFrameSourceOptionsRequestObject; BrowseExplorerFrameSourceOptionsResponseObject; BrowseExplorerSemanticInventory; BrowseExplorerSemanticInventory200JSONResponse; BrowseExplorerSemanticInventory400JSONResponse; BrowseExplorerSemanticInventory401JSONResponse; BrowseExplorerSemanticInventory403JSONResponse; BrowseExplorerSemanticInventory404JSONResponse; BrowseExplorerSemanticInventory409JSONResponse; BrowseExplorerSemanticInventory422JSONResponse; BrowseExplorerSemanticInventory500JSONResponse; BrowseExplorerSemanticInventory503JSONResponse; BrowseExplorerSemanticInventoryJSONRequestBody; BrowseExplorerSemanticInventoryRequestObject; BrowseExplorerSemanticInventoryResponseObject; BuilderState; BuilderStateKind; BuilderStateLifecycleState; CandidateSearchRequest; CandidateSearchResponse; CandidateSearchResponseKind; Catalog; CatalogAvailability; CatalogAvailabilityReadiness; CatalogAvailabilityReadinessStatus; CatalogAvailabilitySourceReadiness; CatalogAvailabilitySourceReadinessStatus; CatalogCandidate; CatalogEdge; CatalogEdgeStorageDirection; CatalogNode; CatalogObservedField; CatalogObservedRelationship; CatalogObservedRelationshipBuilderDirection; CatalogObservedRelationshipStorageDirection; CatalogObservedResource; CellTraceBinding; CellTraceContribution; CellTraceFeature; CellTraceRequest; CellTraceResponse; CellTraceTrace; CellTraceTraceStatus; ChartPresentation; CheckExplorerPopulationMapping; CheckExplorerPopulationMapping200JSONResponse; CheckExplorerPopulationMapping400JSONResponse; CheckExplorerPopulationMapping401JSONResponse; CheckExplorerPopulationMapping403JSONResponse; CheckExplorerPopulationMapping404JSONResponse; CheckExplorerPopulationMapping409JSONResponse; CheckExplorerPopulationMapping422JSONResponse; CheckExplorerPopulationMapping500JSONResponse; CheckExplorerPopulationMapping503JSONResponse; CheckExplorerPopulationMapping504JSONResponse; CheckExplorerPopulationMappingJSONRequestBody; CheckExplorerPopulationMappingRequestObject; CheckExplorerPopulationMappingResponseObject; CodedValueKey; Column; ColumnSource; ColumnSourceKind; ColumnSourceRequest; ColumnSourceResponse; ColumnSourceRouteStep; ColumnSourceRouteStepMatchMode; ColumnSourceRouteStepStorageDirection; ColumnTransformationChange; ColumnValueTransformation; ColumnValueTransformationCapabilities; CommandResult; CommandResultType; CompileResponse; CompileResponseKind; ConceptCandidate; ConfiguredColumnContext; ConfiguredColumnContextRequest; ConfiguredColumnContextResponse; ConfiguredColumnReadyResolution; ConfiguredColumnReadyResolutionState; ConfiguredColumnResolution; ConfiguredColumnUnavailableResolution; ConfiguredColumnUnavailableResolutionState; Construction; ConstructionActiveRelatedRecordDescriptor; ConstructionCapabilitiesRequest; ConstructionCapabilitiesResponse; ConstructionCategoryDiscoveryRequest; ConstructionCategoryDiscoveryResponse; ConstructionCategoryDiscoveryResponseOutcome; ConstructionChoice; ConstructionChoiceFact; ConstructionChoiceOption; ConstructionChoiceOptionContributorPredicateOperators; ConstructionChoiceOptionDecision; ConstructionChoiceOptionForm; ConstructionChoiceOptionPreservation; ConstructionChoiceOptionRowEffect; ConstructionChoiceOptionShape; ConstructionChoiceOptionSupport; ConstructionChoicePresentation; ConstructionChoiceProposalRequest; ConstructionChoiceProposalResponse; ConstructionChoiceProposalResponsePreviewStatus; ConstructionChoiceProposalSelection; ConstructionChoiceProposalSelectionForm; ConstructionChoiceProposalSelectionRowValuePolicy; ConstructionChoiceSearchRequest; ConstructionChoiceSearchRequest_Source; ConstructionChoiceSearchResponse; ConstructionChoiceSelection; ConstructionChoiceSelectionForm; ConstructionChoiceSelectionRowValuePolicy; ConstructionChoice_Source; ConstructionCodeValue; ConstructionCodedGroup; ConstructionCodedGroupChoice; ConstructionCodedGroupMissingKeyPolicy; ConstructionCodedGroupSourceCardinality; ConstructionCodedGroupSourceFhirType; ConstructionCodedGroupSourceShape; ConstructionCodedPivot; ConstructionCodedPivotCategory; ConstructionCodedPivotDuplicatePolicy; ConstructionCodedPivotMissingCellPolicy; ConstructionCodedPivotSource; ConstructionCombine; ConstructionCombineJoinType; ConstructionCombineKey; ConstructionCombineKind; ConstructionCombineMembershipMode; ConstructionCombineProjection; ConstructionCombineRightMatchPolicy; ConstructionDependencyImpact; ConstructionDependencyIssue; ConstructionDerive; ConstructionDeriveDivisionByZeroPolicy; ConstructionDeriveMissingInputPolicy; ConstructionDeriveOperation; ConstructionDeriveSourceChoice; ConstructionDeriveSourceInput; ConstructionDeriveSourceSelection; ConstructionDiscoveredCategory; ConstructionExpand; ConstructionExpandEmptyPolicy; ConstructionExpandSourceChoice; ConstructionExpandSourceInput; ConstructionExpandSourceSelection; ConstructionExpandedFieldSource; ConstructionExpandedFieldSourceCardinality; ConstructionFilter; ConstructionFilterOperator; ConstructionFilterSourceChoice; ConstructionFilterSourceInput; ConstructionFilterSourceSelection; ConstructionFilterValue; ConstructionFilterValueKind; ConstructionGroup; ConstructionGroupAggregate; ConstructionGroupAggregateOperation; ConstructionGroupCodingIdentitySourceOption; ConstructionGroupCodingIdentitySourceOptionKind; ConstructionGroupCodingIdentitySourceOptionSourceRole; ConstructionGroupCodingIdentitySourceOptionValueKind; ConstructionGroupFieldChoice; ConstructionGroupFieldChoiceRelatedOccurrence; ConstructionGroupFieldCurrentStage; ConstructionGroupFieldCurrentStageSourceRole; ConstructionGroupFieldOwner; ConstructionGroupFieldOwnerRole; ConstructionGroupFieldRelatedOccurrence; ConstructionGroupFieldRelatedOccurrenceCodingEmptyPolicy; ConstructionGroupFieldRelatedOccurrenceEmptyPolicy; ConstructionGroupFieldRelatedOccurrenceMultiplicity; ConstructionGroupFieldRelatedOccurrenceOrdinalSemantics; ConstructionGroupFieldRelatedRouteStep; ConstructionGroupFieldRelatedRouteStepMatchMode; ConstructionGroupFieldRelatedRouteStepStorageDirection; ConstructionGroupFieldScalarSourceOption; ConstructionGroupFieldScalarSourceOptionKind; ConstructionGroupFieldScalarSourceOptionSourceRole; ConstructionGroupFieldScalarSourceOptionValueKind; ConstructionGroupFieldSource; ConstructionGroupFieldSourceAncestor; ConstructionGroupFieldSourceChoice; ConstructionGroupFieldSourceChoiceKind; ConstructionGroupFieldSourceCodingIdentityBinding; ConstructionGroupFieldSourceCodingIdentityBindingKind; ConstructionGroupFieldSourceCodingIdentityBindingSourceRole; ConstructionGroupFieldSourceCodingIdentityBindingValueKind; ConstructionGroupFieldSourceContext; ConstructionGroupFieldSourceCurrentStageInput; ConstructionGroupFieldSourceCurrentStageInputKind; ConstructionGroupFieldSourceExpansion; ConstructionGroupFieldSourceInput; ConstructionGroupFieldSourceInputBinding; ConstructionGroupFieldSourceInputBindingKind; ConstructionGroupFieldSourceInputBindingValueKind; ConstructionGroupFieldSourceRelatedOccurrenceInput; ConstructionGroupFieldSourceRelatedOccurrenceInputKind; ConstructionGroupFieldSourceRouteStep; ConstructionGroupFieldSourceRouteStepDirection; ConstructionGroupFieldSourceScalarBinding; ConstructionGroupFieldSourceScalarBindingKind; ConstructionGroupFieldSourceScalarBindingSourceRole; ConstructionGroupFieldSourceScalarBindingValueKind; ConstructionGroupKey; ConstructionGroupKeyFieldSource; ConstructionGroupKeyInputColumn; ConstructionGroupKeySemanticCategory; ConstructionGroupMissingKeyPolicy; ConstructionGroupSemanticCategory; ConstructionGroupSemanticCategoryChoice; ConstructionGroupSemanticCategoryChoiceKind; ConstructionGroupSemanticCategoryChoiceSemanticCategoryKind; ConstructionGroupSemanticCategoryKind; ConstructionGroupSemanticCategorySourceKind; ConstructionGroupSource; ConstructionInputColumn; ConstructionInputRef; ConstructionInputRefKind; ConstructionInputRevision; ConstructionInputRevisionKind; ConstructionInputsRequest; ConstructionInputsResponse; ConstructionLiteral; ConstructionLiteralKind; ConstructionOperand; ConstructionOperandKind; ConstructionOperation; ConstructionOperationCapability; ConstructionOperationCapabilityKind; ConstructionOperationKind; ConstructionOutputPresentation; ConstructionPivot; ConstructionPivotCategory; ConstructionPivotDuplicatePolicy; ConstructionPivotMissingCellPolicy; ConstructionPivotSourceSelection; ConstructionPivotUnlistedCategoryPolicy; ConstructionProposalRequest; ConstructionProposalResponse; ConstructionProposalResponsePreviewStatus; ConstructionRelatedEligibility; ConstructionRelatedEligibilityMatch; ConstructionRelatedEligibilityMatchKind; ConstructionRelatedExpand; ConstructionRelatedExpandAnchor; ConstructionRelatedExpandAnchorKind; ConstructionRelatedExpandEmptyPolicy; ConstructionRelatedExpandStageDescriptor; ConstructionRelatedExpandStageDescriptorAnchorKind; ConstructionRelatedField; ConstructionRelatedSource; ConstructionRelatedSourceForm; ConstructionRelatedSourceRowValuePolicy; ConstructionRouteStep; ConstructionRouteStepMatchMode; ConstructionRouteStepStorageDirection; ConstructionRowValue; ConstructionRowValue0; ConstructionRowValue1; ConstructionRowValueFieldSource; ConstructionRowValuePolicy; ConstructionSemanticGroupChoice; ConstructionSourceInput; ConstructionSourceInputChoice; ConstructionSourceProjection; ConstructionSourceProjectionAvailability; ConstructionStageColumn; ConstructionStageColumnDescriptor; ConstructionStageColumnDescriptorCardinality; ConstructionStageDescriptor; ConstructionStep; ConstructionTableScalar; ConstructionTableScalarKind; ConstructionUnpivot; ConstructionUnpivotInput; ConstructionUnpivotNullRowPolicy; ConstructionUnpivotSourceChoice; ConstructionUnpivotSourceInput; ConstructionUnpivotSourceSelection; ConstructionWorkspaceInput; ConstructionWorkspaceInputColumn; ConstructionWorkspaceInputColumnCardinality; ContractColumn; ContractColumnStructuralSuitability; ContributorCodeValue; ContributorCodeValueKind; ContributorPredicate; ContributorPredicateOperator; ContributorPredicateQuantifier; ContributorStringValue; ContributorStringValueKind; ContributorValue; ContributorWindowSource; ContributorWindowSourcePrecision; CorrelatedBinding; CorrelatedBindingSelectionScope; CorrelatedBindingValueScope; CreateDatasetGeneration; CreateDatasetGeneration200JSONResponse; CreateDatasetGeneration400JSONResponse; CreateDatasetGeneration401JSONResponse; CreateDatasetGeneration403JSONResponse; CreateDatasetGeneration409JSONResponse; CreateDatasetGeneration415JSONResponse; CreateDatasetGeneration422JSONResponse; CreateDatasetGeneration500JSONResponse; CreateDatasetGeneration503JSONResponse; CreateDatasetGenerationMultipartBody; CreateDatasetGenerationMultipartRequestBody; CreateDatasetGenerationRequestObject; CreateDatasetGenerationResponseObject; CreateExplorer; CreateExplorer201JSONResponse; CreateExplorer400JSONResponse; CreateExplorer401JSONResponse; CreateExplorer403JSONResponse; CreateExplorer409JSONResponse; CreateExplorer422JSONResponse; CreateExplorer500JSONResponse; CreateExplorerExplicitGroupRevision; CreateExplorerExplicitGroupRevision201JSONResponse; CreateExplorerExplicitGroupRevision400JSONResponse; CreateExplorerExplicitGroupRevision401JSONResponse; CreateExplorerExplicitGroupRevision403JSONResponse; CreateExplorerExplicitGroupRevision404JSONResponse; CreateExplorerExplicitGroupRevision409JSONResponse; CreateExplorerExplicitGroupRevision422JSONResponse; CreateExplorerExplicitGroupRevision500JSONResponse; CreateExplorerExplicitGroupRevision503JSONResponse; CreateExplorerExplicitGroupRevisionJSONRequestBody; CreateExplorerExplicitGroupRevisionParams; CreateExplorerExplicitGroupRevisionRequestObject; CreateExplorerExplicitGroupRevisionResponseObject; CreateExplorerJSONRequestBody; CreateExplorerParams; CreateExplorerRequestObject; CreateExplorerResponseObject; CreateExplorerSelection; CreateExplorerSelection201JSONResponse; CreateExplorerSelection400JSONResponse; CreateExplorerSelection401JSONResponse; CreateExplorerSelection403JSONResponse; CreateExplorerSelection404JSONResponse; CreateExplorerSelection409JSONResponse; CreateExplorerSelection422JSONResponse; CreateExplorerSelection500JSONResponse; CreateExplorerSelection503JSONResponse; CreateExplorerSelectionJSONRequestBody; CreateExplorerSelectionParams; CreateExplorerSelectionRequestObject; CreateExplorerSelectionResponseObject; CreateInterpretationRevision; CreateInterpretationRevision201JSONResponse; CreateInterpretationRevision400JSONResponse; CreateInterpretationRevision401JSONResponse; CreateInterpretationRevision403JSONResponse; CreateInterpretationRevision404JSONResponse; CreateInterpretationRevision409JSONResponse; CreateInterpretationRevision422JSONResponse; CreateInterpretationRevision500JSONResponse; CreateInterpretationRevision503JSONResponse; CreateInterpretationRevisionFromColumn; CreateInterpretationRevisionFromColumn201JSONResponse; CreateInterpretationRevisionFromColumn400JSONResponse; CreateInterpretationRevisionFromColumn401JSONResponse; CreateInterpretationRevisionFromColumn403JSONResponse; CreateInterpretationRevisionFromColumn404JSONResponse; CreateInterpretationRevisionFromColumn409JSONResponse; CreateInterpretationRevisionFromColumn422JSONResponse; CreateInterpretationRevisionFromColumn500JSONResponse; CreateInterpretationRevisionFromColumn503JSONResponse; CreateInterpretationRevisionFromColumnJSONRequestBody; CreateInterpretationRevisionFromColumnParams; CreateInterpretationRevisionFromColumnRequestObject; CreateInterpretationRevisionFromColumnResponseObject; CreateInterpretationRevisionJSONRequestBody; CreateInterpretationRevisionParams; CreateInterpretationRevisionRequestObject; CreateInterpretationRevisionResponseObject; Diagnostic; DiscoverExplorerConstructionCategories; DiscoverExplorerConstructionCategories200JSONResponse; DiscoverExplorerConstructionCategories400JSONResponse; DiscoverExplorerConstructionCategories401JSONResponse; DiscoverExplorerConstructionCategories403JSONResponse; DiscoverExplorerConstructionCategories404JSONResponse; DiscoverExplorerConstructionCategories409JSONResponse; DiscoverExplorerConstructionCategories422JSONResponse; DiscoverExplorerConstructionCategories500JSONResponse; DiscoverExplorerConstructionCategories503JSONResponse; DiscoverExplorerConstructionCategoriesJSONRequestBody; DiscoverExplorerConstructionCategoriesRequestObject; DiscoverExplorerConstructionCategoriesResponseObject; DiscoverExplorerTableShapeCategories; DiscoverExplorerTableShapeCategories200JSONResponse; DiscoverExplorerTableShapeCategories400JSONResponse; DiscoverExplorerTableShapeCategories401JSONResponse; DiscoverExplorerTableShapeCategories403JSONResponse; DiscoverExplorerTableShapeCategories404JSONResponse; DiscoverExplorerTableShapeCategories409JSONResponse; DiscoverExplorerTableShapeCategories422JSONResponse; DiscoverExplorerTableShapeCategories500JSONResponse; DiscoverExplorerTableShapeCategories503JSONResponse; DiscoverExplorerTableShapeCategoriesJSONRequestBody; DiscoverExplorerTableShapeCategoriesRequestObject; DiscoverExplorerTableShapeCategoriesResponseObject; Discriminator; Document; DocumentKind; DownloadExplorerArtifact; DownloadExplorerArtifact200ApplicationzipResponse; DownloadExplorerArtifact200ResponseHeaders; DownloadExplorerArtifact401JSONResponse; DownloadExplorerArtifact403JSONResponse; DownloadExplorerArtifact404JSONResponse; DownloadExplorerArtifact409JSONResponse; DownloadExplorerArtifact410JSONResponse; DownloadExplorerArtifact500JSONResponse; DownloadExplorerArtifact503JSONResponse; DownloadExplorerArtifactRequestObject; DownloadExplorerArtifactResponseObject; ErrorBody; ErrorResponse; ExactCategoryMapping; ExactCategoryRecodeTransformation; ExactCategoryRecodeTransformationExactCategoryRecodeUnknownPolicy; ExactCategoryRecodeTransformationKind; ExactSemanticInventoryItemIdentity; ExactSemanticInventoryItemIdentityKind; ExecuteDataframeGraphQL; ExecuteDataframeGraphQL200JSONResponse; ExecuteDataframeGraphQL400JSONResponse; ExecuteDataframeGraphQL401JSONResponse; ExecuteDataframeGraphQL403JSONResponse; ExecuteDataframeGraphQL404JSONResponse; ExecuteDataframeGraphQL409JSONResponse; ExecuteDataframeGraphQL413JSONResponse; ExecuteDataframeGraphQL422JSONResponse; ExecuteDataframeGraphQL429JSONResponse; ExecuteDataframeGraphQL499JSONResponse; ExecuteDataframeGraphQL500JSONResponse; ExecuteDataframeGraphQL503JSONResponse; ExecuteDataframeGraphQL504JSONResponse; ExecuteDataframeGraphQLJSONRequestBody; ExecuteDataframeGraphQLRequestObject; ExecuteDataframeGraphQLResponseObject; ExecuteGraphQL; ExecuteGraphQL200JSONResponse; ExecuteGraphQL400JSONResponse; ExecuteGraphQL401JSONResponse; ExecuteGraphQL403JSONResponse; ExecuteGraphQL404JSONResponse; ExecuteGraphQL409JSONResponse; ExecuteGraphQL413JSONResponse; ExecuteGraphQL422JSONResponse; ExecuteGraphQL429JSONResponse; ExecuteGraphQL499JSONResponse; ExecuteGraphQL500JSONResponse; ExecuteGraphQL503JSONResponse; ExecuteGraphQL504JSONResponse; ExecuteGraphQLJSONRequestBody; ExecuteGraphQLRequestObject; ExecuteGraphQLResponseObject; ExplicitGroupCreateRequest; ExplicitGroupInput; ExplicitGroupRevisionChoice; ExplicitGroupRevisionChoiceUnassignedMemberPolicies; ExplicitGroupRevisionSummary; ExplicitGroupSummary; ExplorerId; ExplorerMetadata; ExtensionBinding; FamilySemanticInventoryItemIdentity; FamilySemanticInventoryItemIdentityKind; FiberServerOptions; FieldChoiceSource; FieldChoiceSourceKind; FieldConstructionChoiceSearchSource; FieldConstructionChoiceSearchSourceKind; FieldPresentation; FieldPresentationContext; FieldPresentationDecision; FieldPresentationGroupUnderSemantic; FieldPresentationGroupUnderSemanticAccess; FieldPresentationGroupUnderSemanticKind; FieldPresentationKeepRaw; FieldPresentationKeepRawKind; FieldPresentationReady; FieldPresentationReadyKind; FieldPresentationReason; FieldPresentationRequest; FieldPresentationUnavailable; FieldPresentationUnavailableKind; FieldSource; FieldSourceProjectionMode; FileActions; FilterPresentation; FixedFilter; FrameDefinition; FrameDefinitionForm; FrameDefinitionManyPolicy; FrameDefinitionZeroPolicy; FrameSourceForm; FrameSourceFormDecision; FrameSourceFormForm; FrameSourceFormManyPolicy; FrameSourceFormZeroPolicy; FrameSourceOption; FrameSourceOptionDefaultForm; FrameSourceOptionsRequest; FrameSourceOptionsResponse; FromConfiguredColumnReadyResolution; FromConfiguredColumnUnavailableResolution; FromConstructionGroupCodingIdentitySourceOption; FromConstructionGroupFieldScalarSourceOption; FromConstructionGroupFieldSourceChoice; FromConstructionGroupFieldSourceCodingIdentityBinding; FromConstructionGroupFieldSourceCurrentStageInput; FromConstructionGroupFieldSourceInputBinding; FromConstructionGroupFieldSourceRelatedOccurrenceInput; FromConstructionGroupFieldSourceScalarBinding; FromConstructionGroupKeyFieldSource; FromConstructionGroupKeyInputColumn; FromConstructionGroupKeySemanticCategory; FromConstructionGroupSemanticCategoryChoice; FromConstructionRowValue0; FromConstructionRowValue1; FromContributorCodeValue; FromContributorStringValue; FromExactCategoryRecodeTransformation; FromExactSemanticInventoryItemIdentity; FromFamilySemanticInventoryItemIdentity; FromFieldChoiceSource; FromFieldConstructionChoiceSearchSource; FromFieldPresentationGroupUnderSemantic; FromFieldPresentationKeepRaw; FromFieldPresentationReady; FromFieldPresentationUnavailable; FromLegacyErrorBody; FromLegacyErrorResponseError0; FromRemoveColumnTransformation; FromRowDefinitionComparisonAvailable; FromRowDefinitionComparisonUnavailable; FromRowDefinitionExpandedSelection; FromRowDefinitionExplicitGroupSelection; FromRowDefinitionFieldGroupSelection; FromRowDefinitionRecordsSelection; FromSemanticBindingChoiceSource; FromSemanticConstructionChoiceSearchSource; FromSetColumnTransformation; FromTableShapeComparisonAvailable; FromTableShapeComparisonUnavailable; Generation; GenerationActivationResponse; GenerationStatusResponse; Get; GetApolloSandbox; GetApolloSandbox200TexthtmlResponse; GetApolloSandbox401JSONResponse; GetApolloSandbox404JSONResponse; GetApolloSandboxRequestObject; GetApolloSandboxResponseObject; GetDatasetGenerationStatus; GetDatasetGenerationStatus200JSONResponse; GetDatasetGenerationStatus400JSONResponse; GetDatasetGenerationStatus401JSONResponse; GetDatasetGenerationStatus403JSONResponse; GetDatasetGenerationStatus404JSONResponse; GetDatasetGenerationStatus500JSONResponse; GetDatasetGenerationStatus503JSONResponse; GetDatasetGenerationStatusParams; GetDatasetGenerationStatusRequestObject; GetDatasetGenerationStatusResponseObject; GetExplorer; GetExplorer200JSONResponse; GetExplorer401JSONResponse; GetExplorer403JSONResponse; GetExplorer404JSONResponse; GetExplorer500JSONResponse; GetExplorerAuthoringCapability; GetExplorerAuthoringCapability200JSONResponse; GetExplorerAuthoringCapability401JSONResponse; GetExplorerAuthoringCapability403JSONResponse; GetExplorerAuthoringCapability500JSONResponse; GetExplorerAuthoringCapabilityRequestObject; GetExplorerAuthoringCapabilityResponseObject; GetExplorerBuilder; GetExplorerBuilder200JSONResponse; GetExplorerBuilder401JSONResponse; GetExplorerBuilder403JSONResponse; GetExplorerBuilder404JSONResponse; GetExplorerBuilder409JSONResponse; GetExplorerBuilder500JSONResponse; GetExplorerBuilder503JSONResponse; GetExplorerBuilderRequestObject; GetExplorerBuilderResponseObject; GetExplorerColumnSource; GetExplorerColumnSource200JSONResponse; GetExplorerColumnSource400JSONResponse; GetExplorerColumnSource401JSONResponse; GetExplorerColumnSource403JSONResponse; GetExplorerColumnSource404JSONResponse; GetExplorerColumnSource409JSONResponse; GetExplorerColumnSource500JSONResponse; GetExplorerColumnSource503JSONResponse; GetExplorerColumnSourceJSONRequestBody; GetExplorerColumnSourceRequestObject; GetExplorerColumnSourceResponseObject; GetExplorerConstructionCapabilities; GetExplorerConstructionCapabilities200JSONResponse; GetExplorerConstructionCapabilities400JSONResponse; GetExplorerConstructionCapabilities401JSONResponse; GetExplorerConstructionCapabilities403JSONResponse; GetExplorerConstructionCapabilities404JSONResponse; GetExplorerConstructionCapabilities409JSONResponse; GetExplorerConstructionCapabilities422JSONResponse; GetExplorerConstructionCapabilities500JSONResponse; GetExplorerConstructionCapabilities503JSONResponse; GetExplorerConstructionCapabilitiesJSONRequestBody; GetExplorerConstructionCapabilitiesRequestObject; GetExplorerConstructionCapabilitiesResponseObject; GetExplorerConstructionInputs; GetExplorerConstructionInputs200JSONResponse; GetExplorerConstructionInputs400JSONResponse; GetExplorerConstructionInputs401JSONResponse; GetExplorerConstructionInputs403JSONResponse; GetExplorerConstructionInputs404JSONResponse; GetExplorerConstructionInputs409JSONResponse; GetExplorerConstructionInputs422JSONResponse; GetExplorerConstructionInputs500JSONResponse; GetExplorerConstructionInputs503JSONResponse; GetExplorerConstructionInputsJSONRequestBody; GetExplorerConstructionInputsRequestObject; GetExplorerConstructionInputsResponseObject; GetExplorerRequestObject; GetExplorerResponseObject; GetExplorerSelection; GetExplorerSelection200JSONResponse; GetExplorerSelection400JSONResponse; GetExplorerSelection401JSONResponse; GetExplorerSelection403JSONResponse; GetExplorerSelection404JSONResponse; GetExplorerSelection409JSONResponse; GetExplorerSelection422JSONResponse; GetExplorerSelection500JSONResponse; GetExplorerSelection503JSONResponse; GetExplorerSelectionParams; GetExplorerSelectionRequestObject; GetExplorerSelectionResponseObject; GetExplorerTableShapeCapabilities; GetExplorerTableShapeCapabilities200JSONResponse; GetExplorerTableShapeCapabilities400JSONResponse; GetExplorerTableShapeCapabilities401JSONResponse; GetExplorerTableShapeCapabilities403JSONResponse; GetExplorerTableShapeCapabilities404JSONResponse; GetExplorerTableShapeCapabilities409JSONResponse; GetExplorerTableShapeCapabilities422JSONResponse; GetExplorerTableShapeCapabilities500JSONResponse; GetExplorerTableShapeCapabilities503JSONResponse; GetExplorerTableShapeCapabilitiesJSONRequestBody; GetExplorerTableShapeCapabilitiesRequestObject; GetExplorerTableShapeCapabilitiesResponseObject; GetGraphQLPlayground; GetGraphQLPlayground200TexthtmlResponse; GetGraphQLPlayground401JSONResponse; GetGraphQLPlayground404JSONResponse; GetGraphQLPlaygroundRequestObject; GetGraphQLPlaygroundResponseObject; GetHealth; GetHealth200JSONResponse; GetHealth503JSONResponse; GetHealthRequestObject; GetHealthResponseObject; GetInterpretationRevision; GetInterpretationRevision200JSONResponse; GetInterpretationRevision401JSONResponse; GetInterpretationRevision403JSONResponse; GetInterpretationRevision404JSONResponse; GetInterpretationRevision500JSONResponse; GetInterpretationRevision503JSONResponse; GetInterpretationRevisionRequestObject; GetInterpretationRevisionResponseObject; GetLiveness; GetLiveness200JSONResponse; GetLivenessRequestObject; GetLivenessResponseObject; GetReadiness; GetReadiness200JSONResponse; GetReadiness503JSONResponse; GetReadinessRequestObject; GetReadinessResponseObject; GetRecipeExecution; GetRecipeExecution200JSONResponse; GetRecipeExecution401JSONResponse; GetRecipeExecution403JSONResponse; GetRecipeExecution404JSONResponse; GetRecipeExecution503JSONResponse; GetRecipeExecutionRequestObject; GetRecipeExecutionResponseObject; GetSpec; GetSpecJSON; GetSwagger; HTMLTexthtmlResponse; HandlerMiddlewareFunc; IdentifierBinding; InterpretationApplicability; InterpretationCandidatePayload; InterpretationFeatureDefinition; InterpretationLibrary; InterpretationLibraryListResponse; InterpretationLibrarySummary; InterpretationLibraryView; InterpretationPreviewCounts; InterpretationPreviewRequest; InterpretationPreviewResponse; InterpretationPreviewResponseCompleteness; InterpretationPreviewSample; InterpretationPreviewSampleState; InterpretationRevision; InterpretationRevisionCreateRequest; InterpretationRevisionFromColumnRequest; InterpretationRevisionSummary; InterpretationRule; InterpretationStructuralMatch; LegacyBadRequest; LegacyBadRequestJSONResponse; LegacyConflict; LegacyConflictJSONResponse; LegacyErrorBody; LegacyErrorResponse; LegacyErrorResponseError0; LegacyErrorResponse_Error; LegacyForbidden; LegacyForbiddenJSONResponse; LegacyInternalError; LegacyInternalErrorJSONResponse; LegacyNotFound; LegacyNotFoundJSONResponse; LegacyUnavailable; LegacyUnavailableJSONResponse; LegacyUnprocessable; LegacyUnprocessableJSONResponse; ListExplorerRowDefinitionChoices; ListExplorerRowDefinitionChoices200JSONResponse; ListExplorerRowDefinitionChoices400JSONResponse; ListExplorerRowDefinitionChoices401JSONResponse; ListExplorerRowDefinitionChoices403JSONResponse; ListExplorerRowDefinitionChoices404JSONResponse; ListExplorerRowDefinitionChoices409JSONResponse; ListExplorerRowDefinitionChoices422JSONResponse; ListExplorerRowDefinitionChoices500JSONResponse; ListExplorerRowDefinitionChoices503JSONResponse; ListExplorerRowDefinitionChoicesParams; ListExplorerRowDefinitionChoicesRequestObject; ListExplorerRowDefinitionChoicesResponseObject; ListExplorers; ListExplorers200JSONResponse; ListExplorers401JSONResponse; ListExplorers403JSONResponse; ListExplorers500JSONResponse; ListExplorersRequestObject; ListExplorersResponseObject; ListInterpretationLibraries; ListInterpretationLibraries200JSONResponse; ListInterpretationLibraries401JSONResponse; ListInterpretationLibraries403JSONResponse; ListInterpretationLibraries500JSONResponse; ListInterpretationLibraries503JSONResponse; ListInterpretationLibrariesRequestObject; ListInterpretationLibrariesResponseObject; LookupSource; LookupSourceProjectionMode; MarshalJSON; MergeConfiguredColumnReadyResolution; MergeConfiguredColumnUnavailableResolution; MergeConstructionGroupCodingIdentitySourceOption; MergeConstructionGroupFieldScalarSourceOption; MergeConstructionGroupFieldSourceChoice; MergeConstructionGroupFieldSourceCodingIdentityBinding; MergeConstructionGroupFieldSourceCurrentStageInput; MergeConstructionGroupFieldSourceInputBinding; MergeConstructionGroupFieldSourceRelatedOccurrenceInput; MergeConstructionGroupFieldSourceScalarBinding; MergeConstructionGroupKeyFieldSource; MergeConstructionGroupKeyInputColumn; MergeConstructionGroupKeySemanticCategory; MergeConstructionGroupSemanticCategoryChoice; MergeConstructionRowValue0; MergeConstructionRowValue1; MergeContributorCodeValue; MergeContributorStringValue; MergeExactCategoryRecodeTransformation; MergeExactSemanticInventoryItemIdentity; MergeFamilySemanticInventoryItemIdentity; MergeFieldChoiceSource; MergeFieldConstructionChoiceSearchSource; MergeFieldPresentationGroupUnderSemantic; MergeFieldPresentationKeepRaw; MergeFieldPresentationReady; MergeFieldPresentationUnavailable; MergeLegacyErrorBody; MergeLegacyErrorResponseError0; MergeRemoveColumnTransformation; MergeRowDefinitionComparisonAvailable; MergeRowDefinitionComparisonUnavailable; MergeRowDefinitionExpandedSelection; MergeRowDefinitionExplicitGroupSelection; MergeRowDefinitionFieldGroupSelection; MergeRowDefinitionRecordsSelection; MergeSemanticBindingChoiceSource; MergeSemanticConstructionChoiceSearchSource; MergeSetColumnTransformation; MergeTableShapeComparisonAvailable; MergeTableShapeComparisonUnavailable; MiddlewareFunc; NewStrictHandler; Output; OwnerRecordsSource; PathToRawSpec; Population; PopulationMappingBinding; PopulationMappingCounts; PopulationMappingRequest; PopulationMappingResponse; PopulationMappingResponseStatus; PopulationMemberRemovalProposalRequest; PopulationMemberRemovalProposalResponse; PopulationMemberRemovalProposalResponsePreviewStatus; PopulationRouteChoice; PopulationRouteStep; PopulationRouteStepStorageDirection; PopulationRoutesRequest; PopulationRoutesResponse; PrepareExplorerArtifact; PrepareExplorerArtifact200JSONResponse; PrepareExplorerArtifact400JSONResponse; PrepareExplorerArtifact401JSONResponse; PrepareExplorerArtifact403JSONResponse; PrepareExplorerArtifact404JSONResponse; PrepareExplorerArtifact409JSONResponse; PrepareExplorerArtifact413JSONResponse; PrepareExplorerArtifact422JSONResponse; PrepareExplorerArtifact500JSONResponse; PrepareExplorerArtifact503JSONResponse; PrepareExplorerArtifact504JSONResponse; PrepareExplorerArtifactJSONRequestBody; PrepareExplorerArtifactRequestObject; PrepareExplorerArtifactResponseObject; PreviewExplorer; PreviewExplorer200JSONResponse; PreviewExplorer400JSONResponse; PreviewExplorer401JSONResponse; PreviewExplorer403JSONResponse; PreviewExplorer404JSONResponse; PreviewExplorer409JSONResponse; PreviewExplorer413JSONResponse; PreviewExplorer422JSONResponse; PreviewExplorer429JSONResponse; PreviewExplorer499JSONResponse; PreviewExplorer500JSONResponse; PreviewExplorer503JSONResponse; PreviewExplorer504JSONResponse; PreviewExplorerJSONRequestBody; PreviewExplorerRequestObject; PreviewExplorerResponseObject; PreviewInterpretationCandidate; PreviewInterpretationCandidate200JSONResponse; PreviewInterpretationCandidate400JSONResponse; PreviewInterpretationCandidate401JSONResponse; PreviewInterpretationCandidate403JSONResponse; PreviewInterpretationCandidate404JSONResponse; PreviewInterpretationCandidate409JSONResponse; PreviewInterpretationCandidate413JSONResponse; PreviewInterpretationCandidate422JSONResponse; PreviewInterpretationCandidate429JSONResponse; PreviewInterpretationCandidate499JSONResponse; PreviewInterpretationCandidate500JSONResponse; PreviewInterpretationCandidate503JSONResponse; PreviewInterpretationCandidate504JSONResponse; PreviewInterpretationCandidateJSONRequestBody; PreviewInterpretationCandidateRequestObject; PreviewInterpretationCandidateResponseObject; PreviewRequest; PreviewResponse; PreviewResponseKind; PreviewRowSource; PreviewRowSourceKind; Project; ProjectionMode; ProposeExplorerConstruction; ProposeExplorerConstruction200JSONResponse; ProposeExplorerConstruction400JSONResponse; ProposeExplorerConstruction401JSONResponse; ProposeExplorerConstruction403JSONResponse; ProposeExplorerConstruction404JSONResponse; ProposeExplorerConstruction409JSONResponse; ProposeExplorerConstruction422JSONResponse; ProposeExplorerConstruction500JSONResponse; ProposeExplorerConstruction503JSONResponse; ProposeExplorerConstruction504JSONResponse; ProposeExplorerConstructionChoice; ProposeExplorerConstructionChoice200JSONResponse; ProposeExplorerConstructionChoice400JSONResponse; ProposeExplorerConstructionChoice401JSONResponse; ProposeExplorerConstructionChoice403JSONResponse; ProposeExplorerConstructionChoice404JSONResponse; ProposeExplorerConstructionChoice409JSONResponse; ProposeExplorerConstructionChoice422JSONResponse; ProposeExplorerConstructionChoice500JSONResponse; ProposeExplorerConstructionChoice503JSONResponse; ProposeExplorerConstructionChoice504JSONResponse; ProposeExplorerConstructionChoiceJSONRequestBody; ProposeExplorerConstructionChoiceRequestObject; ProposeExplorerConstructionChoiceResponseObject; ProposeExplorerConstructionJSONRequestBody; ProposeExplorerConstructionRequestObject; ProposeExplorerConstructionResponseObject; ProposeExplorerPopulationMemberRemoval; ProposeExplorerPopulationMemberRemoval200JSONResponse; ProposeExplorerPopulationMemberRemoval400JSONResponse; ProposeExplorerPopulationMemberRemoval401JSONResponse; ProposeExplorerPopulationMemberRemoval403JSONResponse; ProposeExplorerPopulationMemberRemoval404JSONResponse; ProposeExplorerPopulationMemberRemoval409JSONResponse; ProposeExplorerPopulationMemberRemoval422JSONResponse; ProposeExplorerPopulationMemberRemoval500JSONResponse; ProposeExplorerPopulationMemberRemoval503JSONResponse; ProposeExplorerPopulationMemberRemoval504JSONResponse; ProposeExplorerPopulationMemberRemovalJSONRequestBody; ProposeExplorerPopulationMemberRemovalRequestObject; ProposeExplorerPopulationMemberRemovalResponseObject; ProposeExplorerRowDefinition; ProposeExplorerRowDefinition200JSONResponse; ProposeExplorerRowDefinition400JSONResponse; ProposeExplorerRowDefinition401JSONResponse; ProposeExplorerRowDefinition403JSONResponse; ProposeExplorerRowDefinition404JSONResponse; ProposeExplorerRowDefinition409JSONResponse; ProposeExplorerRowDefinition422JSONResponse; ProposeExplorerRowDefinition500JSONResponse; ProposeExplorerRowDefinition503JSONResponse; ProposeExplorerRowDefinitionJSONRequestBody; ProposeExplorerRowDefinitionRequestObject; ProposeExplorerRowDefinitionResponseObject; ProposeExplorerTableShape; ProposeExplorerTableShape200JSONResponse; ProposeExplorerTableShape400JSONResponse; ProposeExplorerTableShape401JSONResponse; ProposeExplorerTableShape403JSONResponse; ProposeExplorerTableShape404JSONResponse; ProposeExplorerTableShape409JSONResponse; ProposeExplorerTableShape422JSONResponse; ProposeExplorerTableShape500JSONResponse; ProposeExplorerTableShape503JSONResponse; ProposeExplorerTableShapeJSONRequestBody; ProposeExplorerTableShapeRequestObject; ProposeExplorerTableShapeResponseObject; PublicationOutput; PublishExplorer; PublishExplorer200JSONResponse; PublishExplorer400JSONResponse; PublishExplorer401JSONResponse; PublishExplorer403JSONResponse; PublishExplorer404JSONResponse; PublishExplorer409JSONResponse; PublishExplorer422JSONResponse; PublishExplorer500JSONResponse; PublishExplorer503JSONResponse; PublishExplorerJSONRequestBody; PublishExplorerParams; PublishExplorerRequestObject; PublishExplorerResponseObject; PublishRepositoryExplorerConfig; PublishRepositoryExplorerConfig200JSONResponse; PublishRepositoryExplorerConfig400JSONResponse; PublishRepositoryExplorerConfig401JSONResponse; PublishRepositoryExplorerConfig403JSONResponse; PublishRepositoryExplorerConfig409JSONResponse; PublishRepositoryExplorerConfig422JSONResponse; PublishRepositoryExplorerConfig500JSONResponse; PublishRepositoryExplorerConfig503JSONResponse; PublishRepositoryExplorerConfigJSONRequestBody; PublishRepositoryExplorerConfigParams; PublishRepositoryExplorerConfigRequestObject; PublishRepositoryExplorerConfigResponseObject; PublishRequest; PublishResponse; PublishResponseKind; RawJSON; ReceiptOutput; ReceiptOutputRowMultiplication; ReceiptOutputStructuralSuitability; RecipeExecutionColumn; RecipeExecutionOutput; RecipeExecutionResponse; ReconcileExplorerBuilder; ReconcileExplorerBuilder200JSONResponse; ReconcileExplorerBuilder400JSONResponse; ReconcileExplorerBuilder401JSONResponse; ReconcileExplorerBuilder403JSONResponse; ReconcileExplorerBuilder404JSONResponse; ReconcileExplorerBuilder409JSONResponse; ReconcileExplorerBuilder422JSONResponse; ReconcileExplorerBuilder500JSONResponse; ReconcileExplorerBuilder503JSONResponse; ReconcileExplorerBuilderJSONRequestBody; ReconcileExplorerBuilderParams; ReconcileExplorerBuilderRequestObject; ReconcileExplorerBuilderResponseObject; ReconcileRequest; RegisterHandlers; RegisterHandlersWithOptions; RelatedExpandChoiceSearchRequest; RelatedExpandChoiceSearchResponse; RelatedExpandContributorChoice; RelatedExpandContributorChoiceOperators; RelatedExpandContributorChoiceSearchRequest; RelatedExpandContributorChoiceSearchResponse; RelatedExpandContributorChoiceSuggestionsSource; RelatedExpandContributorSource; RelatedExpandContributorSourceKind; RelatedExpandRouteChoice; RelatedExpandRouteChoiceKind; RelatedFieldChoice; RelatedFieldChoiceSearchRequest; RelatedFieldChoiceSearchResponse; RelatedSelection; RelatedSelectionKind; RelatedSourceContributorRule; RelatedSourceContributorRulePolicy; RelatedSourceField; RelatedSourceFieldCardinality; RelatedSourceFieldKind; RemoveColumnTransformation; RemoveColumnTransformationKind; RepeatedBoundary; RepeatedCoordinate; ResolveConfiguredColumnContext; ResolveConfiguredColumnContext200JSONResponse; ResolveConfiguredColumnContext400JSONResponse; ResolveConfiguredColumnContext401JSONResponse; ResolveConfiguredColumnContext403JSONResponse; ResolveConfiguredColumnContext404JSONResponse; ResolveConfiguredColumnContext409JSONResponse; ResolveConfiguredColumnContext422JSONResponse; ResolveConfiguredColumnContext500JSONResponse; ResolveConfiguredColumnContext503JSONResponse; ResolveConfiguredColumnContextJSONRequestBody; ResolveConfiguredColumnContextRequestObject; ResolveConfiguredColumnContextResponseObject; ResolveExplorerTableShape; ResolveExplorerTableShape200JSONResponse; ResolveExplorerTableShape400JSONResponse; ResolveExplorerTableShape401JSONResponse; ResolveExplorerTableShape403JSONResponse; ResolveExplorerTableShape404JSONResponse; ResolveExplorerTableShape409JSONResponse; ResolveExplorerTableShape422JSONResponse; ResolveExplorerTableShape500JSONResponse; ResolveExplorerTableShape503JSONResponse; ResolveExplorerTableShapeJSONRequestBody; ResolveExplorerTableShapeRequestObject; ResolveExplorerTableShapeResponseObject; RouteNode; RouteNodeMatchMode; RoutePolicy; RouteRebaseChoice; RowChangeAssessmentResponse; RowChangeAssessmentResponseMode; RowChangeAssessmentResponseStatus; RowChangeProposal; RowChangeProposalMode; RowChangeUnresolvedReference; RowChangeUnresolvedReferenceKind; RowDefinition; RowDefinitionChoice; RowDefinitionChoiceKind; RowDefinitionChoicePolicy; RowDefinitionChoicePolicyName; RowDefinitionChoiceValueType; RowDefinitionChoicesResponse; RowDefinitionComparison; RowDefinitionComparisonAvailable; RowDefinitionComparisonAvailableStatus; RowDefinitionComparisonExample; RowDefinitionComparisonUnavailable; RowDefinitionComparisonUnavailableStatus; RowDefinitionExpandedSelection; RowDefinitionExpandedSelectionExpandedEmptyCollectionPolicy; RowDefinitionExpandedSelectionKind; RowDefinitionExplicitGroupSelection; RowDefinitionExplicitGroupSelectionExplicitGroupUnassignedMemberPolicy; RowDefinitionExplicitGroupSelectionKind; RowDefinitionFieldGroupSelection; RowDefinitionFieldGroupSelectionFieldGroupMissingKeyPolicy; RowDefinitionFieldGroupSelectionKind; RowDefinitionPreviewSummary; RowDefinitionProposal; RowDefinitionProposalMode; RowDefinitionProposalRequest; RowDefinitionProposalRequest_Selection; RowDefinitionRecordsSelection; RowDefinitionRecordsSelectionKind; RowLineageCapability; RowLineageCapabilityStatus; RowLineageContributor; RowLineageRequest; RowLineageResponse; RowLineageResponseStatus; SchemaFieldOption; SchemaFieldOptionCardinality; SchemaFieldSearchRequest; SchemaFieldSearchResponse; SchemaFieldSearchResponseKind; SearchExplorerCandidates; SearchExplorerCandidates200JSONResponse; SearchExplorerCandidates400JSONResponse; SearchExplorerCandidates401JSONResponse; SearchExplorerCandidates403JSONResponse; SearchExplorerCandidates409JSONResponse; SearchExplorerCandidates500JSONResponse; SearchExplorerCandidates503JSONResponse; SearchExplorerCandidatesJSONRequestBody; SearchExplorerCandidatesRequestObject; SearchExplorerCandidatesResponseObject; SearchExplorerConstructionChoices; SearchExplorerConstructionChoices200JSONResponse; SearchExplorerConstructionChoices400JSONResponse; SearchExplorerConstructionChoices401JSONResponse; SearchExplorerConstructionChoices403JSONResponse; SearchExplorerConstructionChoices404JSONResponse; SearchExplorerConstructionChoices409JSONResponse; SearchExplorerConstructionChoices422JSONResponse; SearchExplorerConstructionChoices500JSONResponse; SearchExplorerConstructionChoices503JSONResponse; SearchExplorerConstructionChoicesJSONRequestBody; SearchExplorerConstructionChoicesRequestObject; SearchExplorerConstructionChoicesResponseObject; SearchExplorerPopulationRoutes; SearchExplorerPopulationRoutes200JSONResponse; SearchExplorerPopulationRoutes400JSONResponse; SearchExplorerPopulationRoutes401JSONResponse; SearchExplorerPopulationRoutes403JSONResponse; SearchExplorerPopulationRoutes404JSONResponse; SearchExplorerPopulationRoutes409JSONResponse; SearchExplorerPopulationRoutes422JSONResponse; SearchExplorerPopulationRoutes500JSONResponse; SearchExplorerPopulationRoutes503JSONResponse; SearchExplorerPopulationRoutesJSONRequestBody; SearchExplorerPopulationRoutesRequestObject; SearchExplorerPopulationRoutesResponseObject; SearchExplorerRelatedExpandChoices; SearchExplorerRelatedExpandChoices200JSONResponse; SearchExplorerRelatedExpandChoices400JSONResponse; SearchExplorerRelatedExpandChoices401JSONResponse; SearchExplorerRelatedExpandChoices403JSONResponse; SearchExplorerRelatedExpandChoices404JSONResponse; SearchExplorerRelatedExpandChoices409JSONResponse; SearchExplorerRelatedExpandChoices422JSONResponse; SearchExplorerRelatedExpandChoices500JSONResponse; SearchExplorerRelatedExpandChoices503JSONResponse; SearchExplorerRelatedExpandChoicesJSONRequestBody; SearchExplorerRelatedExpandChoicesRequestObject; SearchExplorerRelatedExpandChoicesResponseObject; SearchExplorerRelatedExpandContributors; SearchExplorerRelatedExpandContributors200JSONResponse; SearchExplorerRelatedExpandContributors400JSONResponse; SearchExplorerRelatedExpandContributors401JSONResponse; SearchExplorerRelatedExpandContributors403JSONResponse; SearchExplorerRelatedExpandContributors404JSONResponse; SearchExplorerRelatedExpandContributors409JSONResponse; SearchExplorerRelatedExpandContributors422JSONResponse; SearchExplorerRelatedExpandContributors500JSONResponse; SearchExplorerRelatedExpandContributors503JSONResponse; SearchExplorerRelatedExpandContributorsJSONRequestBody; SearchExplorerRelatedExpandContributorsRequestObject; SearchExplorerRelatedExpandContributorsResponseObject; SearchExplorerRelatedFieldChoices; SearchExplorerRelatedFieldChoices200JSONResponse; SearchExplorerRelatedFieldChoices400JSONResponse; SearchExplorerRelatedFieldChoices401JSONResponse; SearchExplorerRelatedFieldChoices403JSONResponse; SearchExplorerRelatedFieldChoices404JSONResponse; SearchExplorerRelatedFieldChoices409JSONResponse; SearchExplorerRelatedFieldChoices422JSONResponse; SearchExplorerRelatedFieldChoices500JSONResponse; SearchExplorerRelatedFieldChoices503JSONResponse; SearchExplorerRelatedFieldChoicesJSONRequestBody; SearchExplorerRelatedFieldChoicesRequestObject; SearchExplorerRelatedFieldChoicesResponseObject; SearchExplorerSchemaFields; SearchExplorerSchemaFields200JSONResponse; SearchExplorerSchemaFields400JSONResponse; SearchExplorerSchemaFields401JSONResponse; SearchExplorerSchemaFields403JSONResponse; SearchExplorerSchemaFields409JSONResponse; SearchExplorerSchemaFields422JSONResponse; SearchExplorerSchemaFields500JSONResponse; SearchExplorerSchemaFields503JSONResponse; SearchExplorerSchemaFieldsJSONRequestBody; SearchExplorerSchemaFieldsRequestObject; SearchExplorerSchemaFieldsResponseObject; SelectionBaseRevision; SelectionCreateRequest; SelectionExplicitResources; SelectionFilter; SelectionMember; SelectionPage; SelectionPublishedOutput; SelectionRequestSource; SelectionRequestSourceKind; SelectionResourceRef; SelectionRevision; SelectionRule; SelectionRuleKind; SelectionSource; SelectionSourceKind; SemanticBindingChoiceSource; SemanticBindingChoiceSourceKind; SemanticBindingChoiceSourceSourceKind; SemanticConstructionChoiceSearchSource; SemanticConstructionChoiceSearchSourceKind; SemanticConstructionChoiceSearchSourceSelectionMode; SemanticFrameFamily; SemanticInventoryBrowseRequest; SemanticInventoryBrowseRequestSelectionMode; SemanticInventoryBrowseResponse; SemanticInventoryBrowseResponseSourceAvailability; SemanticInventoryBrowseResponseState; SemanticInventoryItem; SemanticInventoryItemCompleteness; SemanticInventoryItemIdentity; SemanticInventoryItemSelectionMode; SemanticSelectionIntent; SemanticSelectionIntentProjectionMode; SemanticSelectionReadiness; SemanticSelectionReadinessStatus; SemanticSelectionResult; SemanticSelectionResultStatus; ServerInterface; ServerInterfaceWrapper; ServiceBadRequest; ServiceBadRequestJSONResponse; ServiceConflict; ServiceConflictJSONResponse; ServiceErrorBody; ServiceErrorResponse; ServiceForbidden; ServiceForbiddenJSONResponse; ServiceInternalError; ServiceInternalErrorJSONResponse; ServiceNotFound; ServiceNotFoundJSONResponse; ServiceUnauthorized; ServiceUnauthorizedJSONResponse; ServiceUnavailable; ServiceUnavailableJSONResponse; ServiceUnprocessable; ServiceUnprocessableJSONResponse; ServiceUnsupportedMediaType; ServiceUnsupportedMediaTypeJSONResponse; Set; SetColumnTransformation; SetColumnTransformationKind; SharedFilterBinding; SharedFilterBindingRef; StrictHandlerFunc; StrictMiddlewareFunc; StrictServerInterface; Tab; TablePresentation; TablePresentationCellRenderer; TableShapeBinaryOperatorChoice; TableShapeBinaryOperatorChoiceChoiceKind; TableShapeBinaryOperatorReference; TableShapeBinaryOperatorReferenceKind; TableShapeCapabilitiesRequest; TableShapeCapabilitiesResponse; TableShapeCategoryDiscoveryRequest; TableShapeCategoryDiscoveryResponse; TableShapeCategoryDiscoveryResponseKind; TableShapeCategoryValue; TableShapeCellContributor; TableShapeCellTrace; TableShapeCellTraceState; TableShapeCellValue; TableShapeChangedCell; TableShapeChangedRow; TableShapeChoiceAvailability; TableShapeChoiceAvailabilityKind; TableShapeColumnReference; TableShapeColumnReferenceKind; TableShapeComparison; TableShapeComparisonAvailable; TableShapeComparisonAvailableStatus; TableShapeComparisonUnavailable; TableShapeComparisonUnavailableStatus; TableShapeContributor; TableShapeDeclaredInformationLoss; TableShapeDeclaredInformationLossStatus; TableShapeDerivedColumnProposal; TableShapeDerivedOperandIntent; TableShapeDerivedOperandIntentKind; TableShapeDerivedOperandIntentRepresentation; TableShapeDerivedOperandReference; TableShapeDerivedOperandReferenceKind; TableShapeDerivedSelection; TableShapeDivisionByZeroPolicyReference; TableShapeDivisionByZeroPolicyReferenceKind; TableShapeDuplicatePolicyReference; TableShapeDuplicatePolicyReferenceKind; TableShapeEditorChoice; TableShapeEditorChoiceChoiceKind; TableShapeEvidenceLimitation; TableShapeExcludedRecord; TableShapeExclusionEvidence; TableShapeExclusionEvidenceStatus; TableShapeInformationLoss; TableShapeLogicalType; TableShapeMissingCellPolicyReference; TableShapeMissingCellPolicyReferenceKind; TableShapeMissingInputPolicyReference; TableShapeMissingInputPolicyReferenceKind; TableShapeOperandSelection; TableShapeOperandSelectionKind; TableShapeOutputDescriptorSupport; TableShapeOutputDescriptorSupportKind; TableShapeOutputName; TableShapeOutputSuggestion; TableShapeOutputSuggestionChoiceKind; TableShapePivotCategoryChoice; TableShapePivotCategoryChoiceChoiceKind; TableShapePivotCategoryDiscovery; TableShapePivotCategoryDiscoveryKind; TableShapePivotCategoryPair; TableShapePivotCategoryReference; TableShapePivotCategoryReferenceKind; TableShapePivotCategorySelection; TableShapePivotProposal; TableShapePivotSelection; TableShapePreviewSummary; TableShapeProposal; TableShapeProposalIntent; TableShapeProposalIntentKind; TableShapeProposalMode; TableShapeProposalRequest; TableShapeProposalRequestMode; TableShapeReshapeModeChoice; TableShapeReshapeModeChoiceChoiceKind; TableShapeReshapeModeChoiceMode; TableShapeReshapeModeReference; TableShapeReshapeModeReferenceKind; TableShapeResolutionRequest; TableShapeResolutionRequestKind; TableShapeResolutionResponse; TableShapeResolutionResponseKind; TableShapeResolvedCategory; TableShapeResolvedOutputDescriptor; TableShapeResolvedOutputDescriptorKind; TableShapeSavedPivotCategorySelection; TableShapeScalar; TableShapeScalarKind; TableShapeSourceIdentity; TableShapeTypeFact; TableShapeUnlistedCategoryPolicyReference; TableShapeUnlistedCategoryPolicyReferenceKind; TableShapeUnpivotNullRowPolicyReference; TableShapeUnpivotNullRowPolicyReferenceKind; TableShapeUnpivotProposal; TableShapeUnpivotSelection; TemporalFieldChoice; TemporalOrderingSource; TemporalOrderingSourceDirection; TemporalOrderingSourceTiePolicy; TemporalReductionCapabilities; TerminologyKey; TraceExplorerCell; TraceExplorerCell200JSONResponse; TraceExplorerCell400JSONResponse; TraceExplorerCell401JSONResponse; TraceExplorerCell403JSONResponse; TraceExplorerCell404JSONResponse; TraceExplorerCell409JSONResponse; TraceExplorerCell422JSONResponse; TraceExplorerCell500JSONResponse; TraceExplorerCell503JSONResponse; TraceExplorerCellJSONRequestBody; TraceExplorerCellRequestObject; TraceExplorerCellResponseObject; TraceExplorerRowLineage; TraceExplorerRowLineage200JSONResponse; TraceExplorerRowLineage400JSONResponse; TraceExplorerRowLineage401JSONResponse; TraceExplorerRowLineage403JSONResponse; TraceExplorerRowLineage404JSONResponse; TraceExplorerRowLineage409JSONResponse; TraceExplorerRowLineage422JSONResponse; TraceExplorerRowLineage500JSONResponse; TraceExplorerRowLineage503JSONResponse; TraceExplorerRowLineage504JSONResponse; TraceExplorerRowLineageJSONRequestBody; TraceExplorerRowLineageRequestObject; TraceExplorerRowLineageResponseObject; UnitIdentity; UnitNormalizationCapabilities; UnitNormalizationContract; UnitNormalizationPolicy; UnitNormalizationPresetCapability; UnitRuleReference; UnmarshalJSON; Valid; ValueByDiscriminator; ValueTransformationCapability; VisitActivateDatasetGenerationResponse; VisitApplyExplorerBuilderCommandsResponse; VisitAssessExplorerRowChangeResponse; VisitBrowseExplorerFrameSourceOptionsResponse; VisitBrowseExplorerSemanticInventoryResponse; VisitCheckExplorerPopulationMappingResponse; VisitCreateDatasetGenerationResponse; VisitCreateExplorerExplicitGroupRevisionResponse; VisitCreateExplorerResponse; VisitCreateExplorerSelectionResponse; VisitCreateInterpretationRevisionFromColumnResponse; VisitCreateInterpretationRevisionResponse; VisitDiscoverExplorerConstructionCategoriesResponse; VisitDiscoverExplorerTableShapeCategoriesResponse; VisitDownloadExplorerArtifactResponse; VisitExecuteDataframeGraphQLResponse; VisitExecuteGraphQLResponse; VisitGetApolloSandboxResponse; VisitGetDatasetGenerationStatusResponse; VisitGetExplorerAuthoringCapabilityResponse; VisitGetExplorerBuilderResponse; VisitGetExplorerColumnSourceResponse; VisitGetExplorerConstructionCapabilitiesResponse; VisitGetExplorerConstructionInputsResponse; VisitGetExplorerResponse; VisitGetExplorerSelectionResponse; VisitGetExplorerTableShapeCapabilitiesResponse; VisitGetGraphQLPlaygroundResponse; VisitGetHealthResponse; VisitGetInterpretationRevisionResponse; VisitGetLivenessResponse; VisitGetReadinessResponse; VisitGetRecipeExecutionResponse; VisitListExplorerRowDefinitionChoicesResponse; VisitListExplorersResponse; VisitListInterpretationLibrariesResponse; VisitPrepareExplorerArtifactResponse; VisitPreviewExplorerResponse; VisitPreviewInterpretationCandidateResponse; VisitProposeExplorerConstructionChoiceResponse; VisitProposeExplorerConstructionResponse; VisitProposeExplorerPopulationMemberRemovalResponse; VisitProposeExplorerRowDefinitionResponse; VisitProposeExplorerTableShapeResponse; VisitPublishExplorerResponse; VisitPublishRepositoryExplorerConfigResponse; VisitReconcileExplorerBuilderResponse; VisitResolveConfiguredColumnContextResponse; VisitResolveExplorerTableShapeResponse; VisitSearchExplorerCandidatesResponse; VisitSearchExplorerConstructionChoicesResponse; VisitSearchExplorerPopulationRoutesResponse; VisitSearchExplorerRelatedExpandChoicesResponse; VisitSearchExplorerRelatedExpandContributorsResponse; VisitSearchExplorerRelatedFieldChoicesResponse; VisitSearchExplorerSchemaFieldsResponse; VisitTraceExplorerCellResponse; VisitTraceExplorerRowLineageResponse; Workspace; WorkspaceOutputConsumer,internal/explorer/authoringv2,—,internal/server,internal/acceptance; internal/explorer/capability,github.com/getkin/kin-openapi/openapi3; github.com/gofiber/fiber/v3; github.com/oapi-codegen/runtime; github.com/oapi-codegen/runtime/types,60 exported interface(s),legacy=17; deprecated=1; compatibility=1,"generated boundary; change its source/generator, not emitted code; runtime/build signals present; static importer counts are insufficient for deletion",go test ./generated/loomapi,inventory complete; semantic disposition pending -internal/acceptance,github.com/calypr/loom/internal/acceptance,acceptance,domain/library,"Fetch and cache the locked FHIR fixture, drive the real-data acceptance scenario, and verify its outputs and performance evidence.",fixture.go; performance.go; runner.go; scenario.go,fixture_test.go; performance_test.go; runner_locking_test.go; runner_status_test.go; runner_zero_value_test.go; scenario_test.go,—,—,2286,1570,41,Acquire; Close; ComparePerformance; ComparePerformanceReportFiles; ComparePerformanceRuns; ComputedDigest; Config; Connections; DefaultLimits; Fetcher; FixtureDigest; FixtureLock; Lease; Limits; LoadLock; Namespace; New; Oracle; PerformanceComparisonReport; PerformanceFailure; PerformanceObservation; PerformanceStageSeconds; PerformanceSuspicion; RefreshLock; Report; ResourceLock; Run; RunID; RunScenario; RunSpec; Runner; ScenarioConfig; ScenarioResult; SelectionRule; StageReport; StaticTarget; Target; Validate; ValidatedFixture; WriteLock; WritePerformanceComparison,internal/dataframe/publication; internal/dataframe/publication/arango; internal/explorer/authoringv2; internal/store/arango; internal/store/clickhouse,generated/loomapi; internal/dataframe/publication; internal/explorer/authoringv2,cmd/loom-acceptance,—,—,2 exported interface(s),legacy=1; migration=18; compatibility=1,one production importer (cmd/loom-acceptance); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/acceptance,inventory complete; semantic disposition pending +internal/acceptance,github.com/calypr/loom/internal/acceptance,acceptance,domain/library,"Fetch and cache the locked FHIR fixture, drive the real-data acceptance scenario, and verify its outputs and performance evidence.",fixture.go; performance.go; runner.go; scenario.go,fixture_test.go; performance_test.go; runner_locking_test.go; runner_status_test.go; runner_zero_value_test.go; scenario_test.go,—,—,2283,1570,41,Acquire; Close; ComparePerformance; ComparePerformanceReportFiles; ComparePerformanceRuns; ComputedDigest; Config; Connections; DefaultLimits; Fetcher; FixtureDigest; FixtureLock; Lease; Limits; LoadLock; Namespace; New; Oracle; PerformanceComparisonReport; PerformanceFailure; PerformanceObservation; PerformanceStageSeconds; PerformanceSuspicion; RefreshLock; Report; ResourceLock; Run; RunID; RunScenario; RunSpec; Runner; ScenarioConfig; ScenarioResult; SelectionRule; StageReport; StaticTarget; Target; Validate; ValidatedFixture; WriteLock; WritePerformanceComparison,internal/backends/arango; internal/dataframe/publication; internal/explorer/authoringv2; internal/store/clickhouse,generated/loomapi; internal/dataframe/publication; internal/explorer/authoringv2,cmd/loom-acceptance,—,—,2 exported interface(s),legacy=1; migration=18; compatibility=1,one production importer (cmd/loom-acceptance); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/acceptance,inventory complete; semantic disposition pending internal/api/authpolicy,github.com/calypr/loom/internal/api/authpolicy,authpolicy,transport adapter,Centralize authorization-sentinel classification for transport operations while preserving operation-specific concealment.,policy.go,policy_test.go,—,—,56,44,2,Classify; Operation,internal/authscope; internal/dataframe/errors,internal/authscope; internal/dataframe/errors,internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/server,—,—,—,—,no automatic consolidation signal; perform semantic review,go test ./internal/api/authpolicy,inventory complete; semantic disposition pending internal/api/bulk/load,github.com/calypr/loom/internal/api/bulk/load,load,transport adapter,Adapt multipart HTTP generation uploads and activation requests to ingest and dataset services.,errors.go; generation_staging.go; handler.go; operations.go; service.go,cross_project_dataset_authorization_e2e_test.go; errors_test.go; fb22_ingress_control_test.go; form_spillover_test.go; operations_test.go; service_test.go,—,—,852,2499,23,ActivateDatasetGeneration; Cleanup; Config; CreateDatasetGeneration; DataframeReleaseStore; Error; FileCount; GenerationActivationResult; GenerationActivator; GenerationLoadRequest; GenerationLoadResult; GenerationLoader; GenerationStatusResult; GetDatasetGenerationStatus; Handler; IngestRunner; NewHandler; NewService; ParseStagingError; RunGeneration; Service; ServiceConfig; Unwrap,internal/authscope; internal/dataframe/errors; internal/dataframe/publication; internal/dataset; internal/ingest,internal/api/http; internal/authscope; internal/catalog; internal/dataframe/errors; internal/dataframe/publication; internal/dataset; internal/ingest,internal/server,—,github.com/gofiber/fiber/v3,2 exported interface(s),—,one production importer (internal/server); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/api/bulk/load,completed/keep internal/api/columncapabilities,github.com/calypr/loom/internal/api/columncapabilities,columncapabilities,transport adapter,"Derive transport-neutral filter, sort, and aggregate capabilities from published ClickHouse column types.",capabilities.go,capabilities_test.go,—,—,56,23,2,FromClickHouse; Tuple,—,—,internal/api/graphql/graph/dataframe; internal/server,—,—,—,—,no automatic consolidation signal; perform semantic review,go test ./internal/api/columncapabilities,inventory complete; semantic disposition pending -internal/api/graphql/graph,github.com/calypr/loom/internal/api/graphql/graph,graphqlapi,transport adapter,"Own GraphQL HTTP handler construction, error presentation, and generated executor binding.",doc.go; errors.go; handler.go,aggregate_http_integration_test.go; errors_test.go; handler_test.go; http_integration_test.go,—,—,511,931,8,Flush; Header; NewApolloSandboxHandler; NewHandler; NewPlaygroundHandler; RouteConfig; Write; WriteHeader,generated/graphql/graph/executor; internal/api/graphql/graph/resolver; internal/api/http; internal/dataframe/errors,internal/api/graphql/graph; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/api/http; internal/authscope; internal/catalog; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/publication; internal/dataframe/published; internal/dataset,internal/server,internal/api/graphql/graph,github.com/99designs/gqlgen/graphql; github.com/99designs/gqlgen/graphql/handler; github.com/99designs/gqlgen/graphql/playground; github.com/gofiber/fiber/v3/middleware/adaptor; github.com/vektah/gqlparser/v2/gqlerror,—,—,one production importer (internal/server); inspect ownership/inline potential,go test ./internal/api/graphql/graph,completed/keep +internal/api/graphql/graph,github.com/calypr/loom/internal/api/graphql/graph,graphqlapi,transport adapter,"Own GraphQL HTTP handler construction, error presentation, and generated executor binding.",doc.go; errors.go; handler.go,aggregate_http_integration_test.go; errors_test.go; handler_test.go; http_integration_test.go,—,—,511,944,8,Flush; Header; NewApolloSandboxHandler; NewHandler; NewPlaygroundHandler; RouteConfig; Write; WriteHeader,generated/graphql/graph/executor; internal/api/graphql/graph/resolver; internal/api/http; internal/dataframe/errors,internal/api/graphql/graph; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/api/http; internal/authscope; internal/catalog; internal/dataframe/compiler; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/publication; internal/dataframe/published; internal/dataset,internal/server,internal/api/graphql/graph,github.com/99designs/gqlgen/graphql; github.com/99designs/gqlgen/graphql/handler; github.com/99designs/gqlgen/graphql/playground; github.com/gofiber/fiber/v3/middleware/adaptor; github.com/vektah/gqlparser/v2/gqlerror,—,—,one production importer (internal/server); inspect ownership/inline potential,go test ./internal/api/graphql/graph,completed/keep internal/api/graphql/graph/dataframe,github.com/calypr/loom/internal/api/graphql/graph/dataframe,dataframe,transport adapter,"Adapt published ClickHouse dataframe discovery, row, aggregate, and export operations to GraphQL.",aggregates.go; export.go; model.go; rows.go; service.go,aggregates_test.go; export_test.go; fb22_principal_auth_path_control_test.go; filter_test.go; model_test.go; service_test.go,—,—,1072,506,17,AggregateInput; AggregateRowsResult; AggregationsInput; AggregationsJSON; ColumnFromPhysical; ColumnModel; Config; Dataset; ExportDataframe; Model; NewService; PersistedFailure; ProjectDatasets; Rows; Service; WithOperationContext; Write,generated/graphql/graph/model; internal/api/authpolicy; internal/api/columncapabilities; internal/api/http; internal/authscope; internal/dataframe/errors; internal/dataframe/publication; internal/dataframe/published; internal/projectid,generated/graphql/graph/model; internal/authscope; internal/dataframe/errors; internal/dataframe/published,internal/api/graphql/graph/resolver; internal/server,—,—,—,legacy=9,no automatic consolidation signal; perform semantic review,go test ./internal/api/graphql/graph/dataframe,inventory complete; semantic disposition pending -internal/api/graphql/graph/query,github.com/calypr/loom/internal/api/graphql/graph/query,queryapi,transport adapter,Resolve and authorize Arango-backed FHIR dataframe and graph query requests for GraphQL.,active_generation.go; doc.go; errors.go; fhir_service.go; fieldrefs.go; graph_service.go; helpers.go; input_resolution.go; introspection.go; recipe_mapping.go; recipe_resolve.go; service.go; types.go,active_generation_test.go; auth_scope_test.go; errors_test.go; fhir_service_test.go; fieldrefs_test.go; graph_service_test.go; introspection_test.go; project_identity_test.go; read_scope_ingress_control_chars_e2e_test.go; recipe_mapping_test.go; service_test.go,—,—,1882,1413,28,Config; ExplainFHIRGraph; FHIRGraphExplanation; FHIRGraphNode; FHIRGraphPageInfo; FHIRGraphPath; FHIRGraphQuery; FHIRGraphRelationship; FHIRGraphResult; FHIRGraphTraversal; FHIRListRequest; FHIRListResult; FHIRReadContext; FieldHint; FieldPredicate; FieldSelector; Introspect; IntrospectionRequest; IntrospectionResponse; ListFHIR; ListFHIRInContext; NewService; RecipeBundleFromInput; RelatedResourceHints; ResourceHints; Run; RunFHIRGraph; Service,generated/graphql/graph/model; internal/api/authpolicy; internal/api/recipe; internal/authscope; internal/catalog; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/recipe; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/dataframe/spec; internal/dataset; internal/fhir/schema; internal/projectid,generated/graphql/graph/model; internal/authscope; internal/catalog; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataset,cmd/dataframe-profile; internal/api/graphql/graph/resolver; internal/server,internal/api/graphql/graph,—,—,—,broad surface (13 production files); inspect cohesion,go test ./internal/api/graphql/graph/query,inventory complete; semantic disposition pending -internal/api/graphql/graph/resolver,github.com/calypr/loom/internal/api/graphql/graph/resolver,resolver,transport adapter,Implement gqlgen field resolvers and map GraphQL values to query and lifecycle services.,dataframe_diagnostics.go; fhir_reference_loader.go; fhir_resolver_helpers.go; fhir_schema.resolvers.go; graph_resolvers.go; graphql_output_helpers.go; operation_context.go; recipe_control_graphql.go; recipe_execution_reader.go; recipe_revision_model.go; resolver.go; schema.resolvers.go,fhir_reference_loader_test.go; output_mapping_test.go; recipe_control_test.go; recipe_execution_reader_authz_e2e_test.go; recipe_execution_twin_wire_parity_test.go; recipe_revision_register_test.go,—,fhir_schema.resolvers.go; schema.resolvers.go,5459,1756,362,AbatementDateTime; AbatementString; Active; Actual; ActualComparisonGroup; ActualNumber; Address; Age; AgeString; Alias; AllDay; AllowedBoolean; AmountString; Annotation; AsNeeded; AssignedComparisonGroup; Attachment; AuthorString; AuthoredOn; Availability; AvailabilityAvailableTime; AvailabilityNotAvailableTime; AvailableEndTime; AvailableStartTime; BirthDate; BodyStructure; BodyStructureIncludedStructure; BodyStructureIncludedStructureBodyLandmarkOrientation; BodyStructureIncludedStructureBodyLandmarkOrientationDistanceFromLandmark; BornDate; BornString; Citation; City; Code; CodeMap; CodeableConcept; CodeableReference; Coding; CollectedDateTime; Combined; Comment; Comparator; Conclusion; Condition; ConditionParticipant; ConditionStage; ContactDetail; ContactPoint; ContentType; ContributedToDeath; Count; CountMax; Country; Creation; Currency; Data; DataRequirement; DataRequirementCodeFilter; DataRequirementDateFilter; DataRequirementSort; DataRequirementValueFilter; DataframeAggregate; DataframeAggregations; DataframeDataset; DataframeRecipeExecution; DataframeRecipeRevision; DataframeRecipeRevisions; DataframeRows; Date; DateAsserted; DayOfWeek; DaysOfWeek; DeceasedBoolean; DeceasedDate; DeceasedDateTime; DeceasedString; Description; DescriptionSummary; DiagnosticReport; DiagnosticReportMedia; DiagnosticReportSupportingInfo; Dimensions; Direction; Display; Distance; District; Div; DoNotPerform; DocStatus; DocumentReference; DocumentReferenceAttester; DocumentReferenceContent; DocumentReferenceContentProfile; DocumentReferenceRelatesTo; Documentation; Dosage; DosageDoseAndRate; Duration; DurationMax; DurationUnit; EffectiveDateTime; EffectiveInstant; End; EndDate; EstimatedAge; Event; Exclude; ExpirationDate; Expiry; ExplainDataframeRecipe; ExplainFhirGraph; Expression; ExtendedContactDetail; Extension; FHIRPrimitiveExtension; Factor; Family; FamilyMemberHistory; FamilyMemberHistoryCondition; FamilyMemberHistoryParticipant; FamilyMemberHistoryProcedure; FhirComments; FhirGraph; Frames; Frequency; FrequencyMax; Gender; Given; Group; GroupCharacteristic; GroupMember; Hash; Height; HumanName; Identifier; ImagingStudy; ImagingStudySeries; ImagingStudySeriesInstance; ImagingStudySeriesPerformer; ImplicitRules; Inactive; Instance; InstantiatesCanonical; InstantiatesURI; Intent; Interval; IntervalUnit; IsActive; IsDefining; IsSubPotent; Issued; Label; Language; LastModified; LastUpdated; Limit; Line; LinkID; LotNumber; LowerLimit; Max; Medication; MedicationAdministration; MedicationAdministrationDosage; MedicationAdministrationPerformer; MedicationBatch; MedicationIngredient; MedicationRequest; MedicationRequestDispenseRequest; MedicationRequestDispenseRequestInitialFill; MedicationRequestSubstitution; MedicationStatement; MedicationStatementAdherence; Membership; Meta; Min; MolecularFormula; MolecularFormulaByMoiety; Money; MultipleBirthBoolean; MultipleBirthInteger; MustSupport; Mutation; Name; Narrative; NewAuthorizedRecipeExecutionReader; NewResolver; Number; NumberOfInstances; NumberOfRepeatsAllowed; NumberOfSeries; Observation; ObservationComponent; ObservationReferenceRange; ObservationTriggeredBy; OccurenceDateTime; OccurrenceDateTime; OccurrenceString; Offset; Offsets; OnsetDateTime; OnsetString; Organization; OrganizationQualification; Pages; ParameterDefinition; Path; Patient; PatientCommunication; PatientContact; PatientInstruction; PatientLink; PerformedDateTime; PerformedString; Period; PeriodMax; PeriodUnit; PostalCode; Practitioner; PractitionerCommunication; PractitionerQualification; PractitionerRole; Preferred; Prefix; PreflightDataframeRecipe; PreviewDataframeRecipe; Priority; Procedure; ProcedureFocalDevice; ProcedurePerformer; Profile; ProjectDataframeDatasets; PublicationDate; PublicationStatus; Quantity; Query; Range; Rank; Ratio; RatioRange; Reason; ReceivedTime; RecipeAuthorizer; RecipeControl; RecipeExecution; RecipeExecutionOutput; RecipeExecutionReader; RecipeExecutionStore; Recorded; RecordedDate; Reference; RegisterDataframeRecipeRevision; RelatedArtifact; RenderedDosageInstruction; Repetitions; Reported; ReportedBoolean; Representation; ResearchStudy; ResearchStudyAssociatedParty; ResearchStudyComparisonGroup; ResearchStudyLabel; ResearchStudyObjective; ResearchStudyOutcomeMeasure; ResearchStudyProgressStatus; ResearchStudyRecruitment; ResearchSubject; ResearchSubjectProgress; Resolver; ResolverConfig; Resource; RunDataframeRecipe; RunFhirDataframe; SampledData; SearchParam; Sequence; SigFormat; Signature; Size; Source; Specimen; SpecimenCollection; SpecimenContainer; SpecimenFeature; SpecimenProcessing; Start; StartDate; Started; State; Status; StatusChanged; StatusDate; SubscriptionTopic; Substance; SubstanceDefinition; SubstanceDefinitionCharacterization; SubstanceDefinitionCode; SubstanceDefinitionMoiety; SubstanceDefinitionMolecularWeight; SubstanceDefinitionName; SubstanceDefinitionNameOfficial; SubstanceDefinitionProperty; SubstanceDefinitionRelationship; SubstanceDefinitionSourceMaterial; SubstanceDefinitionStructure; SubstanceDefinitionStructureRepresentation; SubstanceIngredient; Suffix; System; TargetFormat; TargetNumber; Task; TaskInput; TaskOutput; TaskPerformer; TaskRestriction; Text; Time; TimeDateTime; TimeOfDay; Timing; TimingDate; TimingDateTime; TimingRepeat; Title; TriggerDefinition; Type; UID; URL; Unit; UpperLimit; UsageContext; Use; UserSelected; ValidateDataframeRecipe; Value; ValueBase64Binary; ValueBoolean; ValueCanonical; ValueCode; ValueDate; ValueDateTime; ValueDecimal; ValueID; ValueInstant; ValueInteger; ValueInteger64; ValueMarkdown; ValueOid; ValuePositiveInt; ValueSet; ValueString; ValueTime; ValueURI; ValueURL; ValueUUID; ValueUnsignedInt; Version; VersionID; When; Width; WithOperationContext,generated/fhir; generated/graphql/graph/executor; generated/graphql/graph/model; internal/api/authpolicy; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/authscope; internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/publication; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataset; internal/fhir/schema,generated/fhir; generated/graphql/graph/model; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/authscope; internal/catalog; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/publication; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataframe/spec,internal/api/graphql/graph; internal/server,—,github.com/99designs/gqlgen/graphql; github.com/vektah/gqlparser/v2/ast,3 exported interface(s),—,broad surface (12 production files); inspect cohesion; mixed handwritten/generated package (2 generated file(s)); preserve regeneration boundary; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/api/graphql/graph/resolver,inventory complete; semantic disposition pending -internal/api/http,github.com/calypr/loom/internal/api/http,httpapi,transport adapter,"Own shared Fiber server setup, middleware, request IDs, errors, health, and readiness behavior.",doc.go; errors.go; middleware.go; request_id.go; routes.go; server.go,errors_test.go; http_test.go; request_id_test.go,—,—,533,558,15,App; ContextWithRequestID; ErrorResponse; HTTPConfig; HTTPErrorBody; HTTPServer; Health; Liveness; Logger; MapDataframeError; MappedError; NewHTTPServer; Readiness; RequestIDFromContext; StatusForErrorCode,internal/authscope; internal/dataframe/errors,internal/authscope; internal/dataframe/errors,internal/api/graphql/graph; internal/api/graphql/graph/dataframe; internal/server,internal/api/bulk/load,github.com/gofiber/fiber/v3; github.com/google/uuid,—,—,no automatic consolidation signal; perform semantic review,go test ./internal/api/http,inventory complete; semantic disposition pending +internal/api/graphql/graph/query,github.com/calypr/loom/internal/api/graphql/graph/query,queryapi,transport adapter,Resolve and authorize Arango-backed FHIR dataframe and graph query requests for GraphQL.,active_generation.go; doc.go; errors.go; fhir_service.go; fieldrefs.go; graph_service.go; helpers.go; input_resolution.go; introspection.go; recipe_mapping.go; recipe_resolve.go; service.go; types.go,active_generation_test.go; auth_scope_test.go; errors_test.go; fhir_service_test.go; fieldrefs_test.go; graph_service_test.go; introspection_test.go; postgres_query_test.go; project_identity_test.go; read_scope_ingress_control_chars_e2e_test.go; recipe_mapping_test.go; service_test.go,—,—,1888,2361,28,Config; ExplainFHIRGraph; FHIRGraphExplanation; FHIRGraphNode; FHIRGraphPageInfo; FHIRGraphPath; FHIRGraphQuery; FHIRGraphRelationship; FHIRGraphResult; FHIRGraphTraversal; FHIRListRequest; FHIRListResult; FHIRReadContext; FieldHint; FieldPredicate; FieldSelector; Introspect; IntrospectionRequest; IntrospectionResponse; ListFHIR; ListFHIRInContext; NewService; RecipeBundleFromInput; RelatedResourceHints; ResourceHints; Run; RunFHIRGraph; Service,generated/graphql/graph/model; internal/api/authpolicy; internal/api/recipe; internal/authscope; internal/catalog; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/recipe; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/dataframe/spec; internal/dataset; internal/fhir/schema; internal/projectid,generated/graphql/graph/model; internal/authscope; internal/backends/postgres; internal/backends/postgres/control; internal/catalog; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataset; internal/ingest,cmd/dataframe-profile; internal/api/graphql/graph/resolver; internal/server,internal/api/graphql/graph,—,—,—,broad surface (13 production files); inspect cohesion,go test ./internal/api/graphql/graph/query,inventory complete; semantic disposition pending +internal/api/graphql/graph/resolver,github.com/calypr/loom/internal/api/graphql/graph/resolver,resolver,transport adapter,Implement gqlgen field resolvers and map GraphQL values to query and lifecycle services.,dataframe_diagnostics.go; fhir_reference_loader.go; fhir_resolver_helpers.go; fhir_schema.resolvers.go; graph_resolvers.go; graphql_output_helpers.go; operation_context.go; recipe_control_graphql.go; recipe_execution_reader.go; recipe_revision_model.go; resolver.go; schema.resolvers.go,fhir_reference_loader_test.go; output_mapping_test.go; recipe_control_test.go; recipe_execution_malformed_persisted_scope_test.go; recipe_execution_malformed_scope_oracle_probe_test.go; recipe_execution_reader_authz_e2e_test.go; recipe_execution_twin_wire_parity_test.go; recipe_revision_register_test.go,—,fhir_schema.resolvers.go; schema.resolvers.go,5459,2334,362,AbatementDateTime; AbatementString; Active; Actual; ActualComparisonGroup; ActualNumber; Address; Age; AgeString; Alias; AllDay; AllowedBoolean; AmountString; Annotation; AsNeeded; AssignedComparisonGroup; Attachment; AuthorString; AuthoredOn; Availability; AvailabilityAvailableTime; AvailabilityNotAvailableTime; AvailableEndTime; AvailableStartTime; BirthDate; BodyStructure; BodyStructureIncludedStructure; BodyStructureIncludedStructureBodyLandmarkOrientation; BodyStructureIncludedStructureBodyLandmarkOrientationDistanceFromLandmark; BornDate; BornString; Citation; City; Code; CodeMap; CodeableConcept; CodeableReference; Coding; CollectedDateTime; Combined; Comment; Comparator; Conclusion; Condition; ConditionParticipant; ConditionStage; ContactDetail; ContactPoint; ContentType; ContributedToDeath; Count; CountMax; Country; Creation; Currency; Data; DataRequirement; DataRequirementCodeFilter; DataRequirementDateFilter; DataRequirementSort; DataRequirementValueFilter; DataframeAggregate; DataframeAggregations; DataframeDataset; DataframeRecipeExecution; DataframeRecipeRevision; DataframeRecipeRevisions; DataframeRows; Date; DateAsserted; DayOfWeek; DaysOfWeek; DeceasedBoolean; DeceasedDate; DeceasedDateTime; DeceasedString; Description; DescriptionSummary; DiagnosticReport; DiagnosticReportMedia; DiagnosticReportSupportingInfo; Dimensions; Direction; Display; Distance; District; Div; DoNotPerform; DocStatus; DocumentReference; DocumentReferenceAttester; DocumentReferenceContent; DocumentReferenceContentProfile; DocumentReferenceRelatesTo; Documentation; Dosage; DosageDoseAndRate; Duration; DurationMax; DurationUnit; EffectiveDateTime; EffectiveInstant; End; EndDate; EstimatedAge; Event; Exclude; ExpirationDate; Expiry; ExplainDataframeRecipe; ExplainFhirGraph; Expression; ExtendedContactDetail; Extension; FHIRPrimitiveExtension; Factor; Family; FamilyMemberHistory; FamilyMemberHistoryCondition; FamilyMemberHistoryParticipant; FamilyMemberHistoryProcedure; FhirComments; FhirGraph; Frames; Frequency; FrequencyMax; Gender; Given; Group; GroupCharacteristic; GroupMember; Hash; Height; HumanName; Identifier; ImagingStudy; ImagingStudySeries; ImagingStudySeriesInstance; ImagingStudySeriesPerformer; ImplicitRules; Inactive; Instance; InstantiatesCanonical; InstantiatesURI; Intent; Interval; IntervalUnit; IsActive; IsDefining; IsSubPotent; Issued; Label; Language; LastModified; LastUpdated; Limit; Line; LinkID; LotNumber; LowerLimit; Max; Medication; MedicationAdministration; MedicationAdministrationDosage; MedicationAdministrationPerformer; MedicationBatch; MedicationIngredient; MedicationRequest; MedicationRequestDispenseRequest; MedicationRequestDispenseRequestInitialFill; MedicationRequestSubstitution; MedicationStatement; MedicationStatementAdherence; Membership; Meta; Min; MolecularFormula; MolecularFormulaByMoiety; Money; MultipleBirthBoolean; MultipleBirthInteger; MustSupport; Mutation; Name; Narrative; NewAuthorizedRecipeExecutionReader; NewResolver; Number; NumberOfInstances; NumberOfRepeatsAllowed; NumberOfSeries; Observation; ObservationComponent; ObservationReferenceRange; ObservationTriggeredBy; OccurenceDateTime; OccurrenceDateTime; OccurrenceString; Offset; Offsets; OnsetDateTime; OnsetString; Organization; OrganizationQualification; Pages; ParameterDefinition; Path; Patient; PatientCommunication; PatientContact; PatientInstruction; PatientLink; PerformedDateTime; PerformedString; Period; PeriodMax; PeriodUnit; PostalCode; Practitioner; PractitionerCommunication; PractitionerQualification; PractitionerRole; Preferred; Prefix; PreflightDataframeRecipe; PreviewDataframeRecipe; Priority; Procedure; ProcedureFocalDevice; ProcedurePerformer; Profile; ProjectDataframeDatasets; PublicationDate; PublicationStatus; Quantity; Query; Range; Rank; Ratio; RatioRange; Reason; ReceivedTime; RecipeAuthorizer; RecipeControl; RecipeExecution; RecipeExecutionOutput; RecipeExecutionReader; RecipeExecutionStore; Recorded; RecordedDate; Reference; RegisterDataframeRecipeRevision; RelatedArtifact; RenderedDosageInstruction; Repetitions; Reported; ReportedBoolean; Representation; ResearchStudy; ResearchStudyAssociatedParty; ResearchStudyComparisonGroup; ResearchStudyLabel; ResearchStudyObjective; ResearchStudyOutcomeMeasure; ResearchStudyProgressStatus; ResearchStudyRecruitment; ResearchSubject; ResearchSubjectProgress; Resolver; ResolverConfig; Resource; RunDataframeRecipe; RunFhirDataframe; SampledData; SearchParam; Sequence; SigFormat; Signature; Size; Source; Specimen; SpecimenCollection; SpecimenContainer; SpecimenFeature; SpecimenProcessing; Start; StartDate; Started; State; Status; StatusChanged; StatusDate; SubscriptionTopic; Substance; SubstanceDefinition; SubstanceDefinitionCharacterization; SubstanceDefinitionCode; SubstanceDefinitionMoiety; SubstanceDefinitionMolecularWeight; SubstanceDefinitionName; SubstanceDefinitionNameOfficial; SubstanceDefinitionProperty; SubstanceDefinitionRelationship; SubstanceDefinitionSourceMaterial; SubstanceDefinitionStructure; SubstanceDefinitionStructureRepresentation; SubstanceIngredient; Suffix; System; TargetFormat; TargetNumber; Task; TaskInput; TaskOutput; TaskPerformer; TaskRestriction; Text; Time; TimeDateTime; TimeOfDay; Timing; TimingDate; TimingDateTime; TimingRepeat; Title; TriggerDefinition; Type; UID; URL; Unit; UpperLimit; UsageContext; Use; UserSelected; ValidateDataframeRecipe; Value; ValueBase64Binary; ValueBoolean; ValueCanonical; ValueCode; ValueDate; ValueDateTime; ValueDecimal; ValueID; ValueInstant; ValueInteger; ValueInteger64; ValueMarkdown; ValueOid; ValuePositiveInt; ValueSet; ValueString; ValueTime; ValueURI; ValueURL; ValueUUID; ValueUnsignedInt; Version; VersionID; When; Width; WithOperationContext,generated/fhir; generated/graphql/graph/executor; generated/graphql/graph/model; internal/api/authpolicy; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/authscope; internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/publication; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataset; internal/fhir/schema,generated/fhir; generated/graphql/graph/model; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/authscope; internal/catalog; internal/dataframe/compiler; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/publication; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataframe/spec,internal/api/graphql/graph; internal/server,—,github.com/99designs/gqlgen/graphql; github.com/vektah/gqlparser/v2/ast,3 exported interface(s),—,broad surface (12 production files); inspect cohesion; mixed handwritten/generated package (2 generated file(s)); preserve regeneration boundary; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/api/graphql/graph/resolver,inventory complete; semantic disposition pending +internal/api/http,github.com/calypr/loom/internal/api/http,httpapi,transport adapter,"Own shared Fiber server setup, middleware, request IDs, errors, health, and readiness behavior.",doc.go; errors.go; middleware.go; request_id.go; routes.go; server.go,backend_error_boundary_contract_test.go; errors_test.go; health_readiness_test.go; http_test.go; request_id_test.go,—,—,642,1145,15,App; ContextWithRequestID; ErrorResponse; HTTPConfig; HTTPErrorBody; HTTPServer; Health; Liveness; Logger; MapDataframeError; MappedError; NewHTTPServer; Readiness; RequestIDFromContext; StatusForErrorCode,internal/authscope; internal/dataframe/errors,internal/authscope; internal/backends/arango; internal/backends/postgres/control; internal/dataframe/errors,internal/api/graphql/graph; internal/api/graphql/graph/dataframe; internal/server,internal/api/bulk/load; internal/backends/postgres/internal/store,github.com/gofiber/fiber/v3; github.com/google/uuid,—,—,no automatic consolidation signal; perform semantic review,go test ./internal/api/http,inventory complete; semantic disposition pending internal/api/recipe,github.com/calypr/loom/internal/api/recipe,recipeapi,transport adapter,Adapt scoped catalog field discovery to recipe schema resolution for one-shot and stored recipes.,discovery.go,discovery_auth_mode_test.go; discovery_test.go,—,—,110,126,3,FieldDiscovery; Fields; NewFieldDiscovery,internal/authscope; internal/catalog; internal/dataframe/errors; internal/dataframe/recipe/schema,internal/authscope; internal/catalog; internal/dataframe/errors; internal/dataframe/recipe/schema,internal/api/graphql/graph/query; internal/server,—,—,—,legacy=1; compatibility=1,no automatic consolidation signal; perform semantic review,go test ./internal/api/recipe,inventory complete; semantic disposition pending -internal/authscope,github.com/calypr/loom/internal/authscope,authscope,domain/library,"Represent request principals and resolve authorized project, generation, and resource scopes.",authentication.go; doc.go; fence_resource_access.go; principal.go; scope_resolver.go,all_generations_cache_key_test.go; auth_path_control_ingress_test.go; auth_resource_path_options_conflict_test.go; authentication_test.go; covers_execution_paths_control_test.go; scope_cache_key_namespace_test.go; scope_test.go,—,—,1142,1692,36,AllowAllAuthorizer; Authenticate; Authenticator; AuthorizeProject; AuthorizeReadProject; AuthorizeWrite; Authorizer; BasicAuthenticator; BearerTokenAuthenticator; CalyprAuthenticator; Clone; ContextWithPrincipal; CoversExecutionPaths; FenceUserAccessClient; GetAllowedResources; InvalidateGeneration; InvalidateProject; NewFenceUserAccessClientWithTrustedIssuer; NewScopeResolver; NormalizeAuthResourcePath; NormalizeAuthResourcePaths; Permission; Principal; PrincipalFromContext; ReadScope; ReadScopeMode; ResolveReadScope; ResolveReadScopeForGeneration; ResolveWriteScopeForGeneration; ResourceAccessClient; ScopeAuthorizer; ScopeResolver; ScopeResolverConfig; StaticAuthenticator; Unrestricted; ValidateTrustedFenceIssuer,internal/catalog; internal/projectid,internal/catalog; internal/catalog/arango; internal/store/arango,cmd/preview-covering-index-artifact; internal/api/authpolicy; internal/api/bulk/load; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/api/http; internal/api/recipe; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/execution; internal/dataframe/execution/chartifact; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer/arango; internal/explorer/lifecycle; internal/server,internal/api/graphql/graph; internal/catalog/arango; internal/dataframe/compiler; internal/dataframe/compiler/render/aql; internal/dataframe/execution/chartifact/arango,—,3 exported interface(s),legacy=5,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/authscope,inventory complete; semantic disposition pending -internal/catalog,github.com/calypr/loom/internal/catalog,catalog,domain/library,"Own persistence-neutral observed FHIR field, pivot, relationship, and capability facts.",auth_scope.go; cache.go; capability_evidence.go; generation.go; helpers.go; limits.go; rebuild.go; relationships.go; resource_inventory.go; semantic.go; semantic_coverage.go; semantic_inventory.go; semantic_schema.go; types.go; write_profiler.go,auth_scope_test.go; cache_test.go; capability_evidence_test.go; generation_test.go; relationships_test.go; semantic_coding_display_projection_test.go; semantic_coverage_diagnostics_test.go; semantic_coverage_test.go; semantic_inventory_buildid_cache_test.go; semantic_inventory_category_test.go; semantic_inventory_emitter_anchor_test.go; semantic_inventory_test.go; semantic_schema_identifier_type_test.go; semantic_schema_test.go; semantic_test.go; write_profiler_coverage_test.go; write_profiler_test.go,—,—,6215,3653,127,AuthResourcePathOptions; Cache; CacheOptions; CacheStats; CapabilityEvidence; CapabilityEvidenceOptions; CapabilityEvidenceReader; CensusSemanticCoverage; ConceptCandidate; DecodeSemanticInventoryCursor; DefaultProfileLimits; DiscoverFields; DiscoverReferences; DiscoverSemanticCodeableConceptTemplates; Documents; EffectiveAuthResourcePathsUnrestricted; EncodeSemanticInventoryCursor; EvidenceDiagnostic; EvidenceStatus; ExplicitAuthResourcePathsUnrestricted; ExtensionValueObservation; FieldCatalogDocument; FieldEnrichmentDigest; FieldEnrichmentObservation; FieldEnrichmentOptions; FieldEnrichmentResult; HasDatasetGeneration; InvalidRelationship; InvalidateAll; InvalidateProject; LegacySemanticBindingIDV1; Merge; MergeRelationshipCounts; NewCache; NewCacheWithOptions; NewProfilerForGenerationWithLimits; NewResourceInventoryBuild; NewSemanticInventoryBuild; NewSemanticInventoryEmitter; NewShapePlanCacheWithLimit; NewShapePlanCacheWithLimits; NormalizeDatasetGeneration; NormalizeProfileLimits; ObservePayload; ObservePayloadWithCoverage; ObservePayloadWithInventory; PopulatedField; PopulatedFieldOptions; PopulatedReference; PopulatedReferenceOptions; ProfileLimits; Profiler; ReadCapabilityEvidence; ReconcileSemanticCoverage; RelationshipAuditOptions; RelationshipAuditSummary; RelationshipCatalogDocument; RelationshipCatalogDocuments; RelationshipCountsFromRawEdges; RelationshipKey; RelationshipObservation; RelationshipObservationDigest; RelationshipObservationOptions; RelationshipObservationResult; RelationshipRebuildOptions; RelationshipRebuildSummary; RepeatedShapeDigest; ResourceInventoryBuild; ResourceInventoryBuildKey; ResourceInventoryCount; ResourceInventoryCoverageDigest; ResourceInventoryDigest; ResourceInventoryObservation; ResourceInventoryOptions; ResourceInventoryRebuildOptions; ResourceInventoryRebuildSummary; ResourceInventoryResult; RetentionBytes; SemanticCodeableConceptTemplate; SemanticCoverageCensus; SemanticCoverageCensusGap; SemanticCoverageCensusStatus; SemanticCoverageCounts; SemanticCoverageDiagnostic; SemanticCoverageDispositionKind; SemanticCoverageEmissionEvidence; SemanticCoverageEmissionSink; SemanticCoverageFacetAggregate; SemanticCoverageOccurrenceKind; SemanticCoverageOwnerLocator; SemanticCoveragePathAggregate; SemanticCoveragePresence; SemanticCoverageReason; SemanticCoverageReference; SemanticCoverageReferenceKind; SemanticCoverageRole; SemanticCoverageSourceContribution; SemanticCoverageSourceIdentity; SemanticCoverageSourceSink; SemanticInventoryBuild; SemanticInventoryBuildID; SemanticInventoryBuildKey; SemanticInventoryCheckpoint; SemanticInventoryContribution; SemanticInventoryEmitter; SemanticInventoryEntry; SemanticInventoryFamily; SemanticInventoryFamilyID; SemanticInventoryFamilyReference; SemanticInventoryObservationSink; SemanticInventoryPage; SemanticInventoryPageDigest; SemanticInventoryPageOptions; SemanticInventoryReference; SemanticInventoryResolveOptions; SemanticInventoryResolveResult; SemanticInventorySelectionMode; SemanticInventoryState; SemanticObservation; SemanticObservationCompleteness; SemanticObservationKey; SemanticObservationSource; SemanticObservationValue; ShapePlanCache; Stats; Truncated; ValidateSemanticInventoryBuild,internal/fhir/schema; internal/fhir/semantic,internal/fhir/schema; internal/fhir/semantic,cmd/arango-fhir-proto; internal/api/graphql/graph/query; internal/api/recipe; internal/authscope; internal/catalog/arango; internal/explorer/authoringv2; internal/explorer/lifecycle; internal/ingest; internal/server,internal/api/bulk/load; internal/api/graphql/graph; internal/api/graphql/graph/resolver; internal/dataframe/compiler; internal/explorer/capability; internal/explorer/compilation,github.com/bytedance/sonic,1 exported interface(s),legacy=12; migration=1; compatibility=1,broad surface (15 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/catalog,inventory complete; semantic disposition pending -internal/catalog/arango,github.com/calypr/loom/internal/catalog/arango,arango,storage adapter,Persist and retrieve catalog facts in ArangoDB.,codec.go; evidence.go; fields.go; relationships.go; resource_inventory.go; resources.go; semantic_coverage.go; semantic_coverage_envelope.go; semantic_inventory.go; store.go,all_generations_project_binding_integration_test.go; resource_inventory_integration_test.go; resource_inventory_unit_test.go; semantic_coded_category_integration_test.go; semantic_coverage_envelope_test.go; semantic_coverage_resource_integration_test.go; semantic_coverage_test.go; semantic_inventory_category_test.go; semantic_inventory_integration_test.go; semantic_inventory_selection_test.go; semantic_inventory_source_availability_test.go; store_test.go,—,—,5385,5346,62,AccumulateRelationshipCatalog; AddSource; AdvanceSemanticInventoryBackfill; AdvanceSemanticInventoryBuild; AuditRelationshipEdges; BeginSemanticCoverageBuild; BeginSemanticCoverageResource; BeginSemanticInventoryBuild; ClaimSemanticInventoryBackfill; CompleteRetainedSemanticCoverageResource; CompleteSemanticCoverageResource; CompleteSemanticInventoryBackfill; CompleteSemanticInventoryBuild; DiscoverExistingAuthResourcePaths; DiscoverFieldEnrichment; DiscoverFields; DiscoverReferences; DiscoverRelationshipObservations; DiscoverResourceInventory; EncodeSemanticCoverageSourceEnvelope; EnsureSemanticInventoryEntries; FailSemanticCoverageBuild; FailSemanticCoverageResource; FailSemanticInventoryBackfill; FailSemanticInventoryBuild; FinalizeSemanticCoverageBuild; LookupSemanticCoverage; MarkSemanticCoverageResourceMissing; New; NewSemanticCoverageSourceSetAccumulator; PageSemanticInventory; PrepareSemanticInventoryBackfill; ReadFieldEnrichment; ReadRelationshipObservations; ReadResourceInventory; ReadRetainedSemanticInventoryPage; RebuildRelationshipCatalog; RebuildResourceInventory; RefreshSemanticInventorySourceAvailability; ResolveSemanticInventorySelections; RetainedSemanticInventoryPage; RetainedSemanticInventoryRow; SemanticCoverageBuildIdentity; SemanticCoverageBuildState; SemanticCoverageCandidatePath; SemanticCoverageCandidateResult; SemanticCoverageCandidateStatus; SemanticCoverageCollectionSpec; SemanticCoverageLookupOptions; SemanticCoverageLookupResult; SemanticCoveragePage; SemanticCoverageResourceExpectation; SemanticCoverageSourcePartitionExpectation; SemanticCoverageSourceSetAccumulator; SemanticCoverageWriteStats; Snapshot; Store; ValidateAuthResourcePathOptions; WriteFieldCatalog; WriteRelationshipCatalog; WriteSemanticCoverageContributions; WriteSemanticInventoryContributions,internal/catalog; internal/fhir/schema; internal/store/arango,internal/authscope; internal/catalog; internal/store/arango,cmd/arango-fhir-proto; internal/explorer/lifecycle; internal/ingest; internal/server,internal/authscope,github.com/bytedance/sonic,—,—,storage adapter; preserve interface and persisted-data contracts,go test ./internal/catalog/arango,inventory complete; semantic disposition pending +internal/authscope,github.com/calypr/loom/internal/authscope,authscope,domain/library,"Represent request principals and resolve authorized project, generation, and resource scopes.",authentication.go; doc.go; fence_resource_access.go; principal.go; scope_resolver.go,all_generations_cache_key_test.go; auth_path_control_ingress_test.go; auth_resource_path_options_conflict_test.go; authentication_test.go; covers_execution_paths_control_test.go; persisted_scope_fails_closed_test.go; probe_double_normalize_test.go; scope_cache_key_namespace_test.go; scope_test.go,—,—,1175,2104,36,AllowAllAuthorizer; Authenticate; Authenticator; AuthorizeProject; AuthorizeReadProject; AuthorizeWrite; Authorizer; BasicAuthenticator; BearerTokenAuthenticator; CalyprAuthenticator; Clone; ContextWithPrincipal; CoversExecutionPaths; FenceUserAccessClient; GetAllowedResources; InvalidateGeneration; InvalidateProject; NewFenceUserAccessClientWithTrustedIssuer; NewScopeResolver; NormalizeAuthResourcePath; NormalizeAuthResourcePaths; Permission; Principal; PrincipalFromContext; ReadScope; ReadScopeMode; ResolveReadScope; ResolveReadScopeForGeneration; ResolveWriteScopeForGeneration; ResourceAccessClient; ScopeAuthorizer; ScopeResolver; ScopeResolverConfig; StaticAuthenticator; Unrestricted; ValidateTrustedFenceIssuer,internal/catalog; internal/projectid,internal/catalog,cmd/preview-covering-index-artifact; internal/api/authpolicy; internal/api/bulk/load; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/api/http; internal/api/recipe; internal/backends/arango/internal/explorer; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/execution; internal/dataframe/execution/chartifact; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer/lifecycle; internal/server,internal/api/graphql/graph; internal/backends/arango/internal/catalog; internal/backends/arango/internal/chartifact; internal/backends/postgres/internal/chartifact; internal/backends/postgres/internal/store; internal/dataframe/compiler; internal/dataframe/compiler/render/aql,—,3 exported interface(s),legacy=5,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/authscope,inventory complete; semantic disposition pending +internal/backends/arango,github.com/calypr/loom/internal/backends/arango,arango,storage adapter,"Expose the only public entry to the ArangoDB 3.12 backend: a single Open factory validates explicit config, owns exactly one lifecycle client with the required collection bootstrap policy, and returns the typed runtime carrying the catalog, dataset lifecycle, Explorer, recipe registry/revisions, and published-dataframe ports assembled from the private implementation tree, with alias re-exports for witness-side reassembly.",aliases.go; doc.go; open.go,ownership_contract_test.go,—,—,336,800,4,Close; Config; Open; Runtime,internal/backends/arango/internal/catalog; internal/backends/arango/internal/chartifact; internal/backends/arango/internal/dataset; internal/backends/arango/internal/explorer; internal/backends/arango/internal/publication; internal/backends/arango/internal/recipeexec; internal/backends/arango/internal/store,internal/backends/arango,cmd/arango-fhir-proto; cmd/dataframe-profile; cmd/loom-data; internal/acceptance; internal/ingest; internal/server,conformance/compiler; internal/api/http; internal/backends/arango; internal/dataframe/compiler; internal/dataframe/execution; internal/dataframe/publication/clickhouse; internal/dataset; internal/explorer/compilation,—,—,—,storage adapter; preserve interface and persisted-data contracts,go test ./internal/backends/arango,inventory complete; semantic disposition pending +internal/backends/arango/internal/catalog,github.com/calypr/loom/internal/backends/arango/internal/catalog,arango,domain/library,Persist and retrieve catalog facts in ArangoDB.,codec.go; evidence.go; fields.go; relationships.go; resource_inventory.go; resources.go; semantic_coverage.go; semantic_coverage_envelope.go; semantic_inventory.go; store.go,all_generations_auth_path_conflict_test.go; all_generations_project_binding_integration_test.go; resource_inventory_integration_test.go; resource_inventory_unit_test.go; semantic_coded_category_integration_test.go; semantic_coverage_envelope_test.go; semantic_coverage_resource_integration_test.go; semantic_coverage_test.go; semantic_inventory_category_test.go; semantic_inventory_integration_test.go; semantic_inventory_selection_test.go; semantic_inventory_source_availability_test.go; store_test.go,—,—,5267,5466,54,AccumulateRelationshipCatalog; AddSource; AdvanceSemanticInventoryBackfill; AdvanceSemanticInventoryBuild; AuditRelationshipEdges; BeginSemanticCoverageBuild; BeginSemanticCoverageResource; BeginSemanticInventoryBuild; ClaimSemanticInventoryBackfill; CompleteRetainedSemanticCoverageResource; CompleteSemanticCoverageResource; CompleteSemanticInventoryBackfill; CompleteSemanticInventoryBuild; DiscoverExistingAuthResourcePaths; DiscoverFieldEnrichment; DiscoverFields; DiscoverReferences; DiscoverRelationshipObservations; DiscoverResourceInventory; EncodeSemanticCoverageSourceEnvelope; EnsureSemanticInventoryEntries; FailSemanticCoverageBuild; FailSemanticCoverageResource; FailSemanticInventoryBackfill; FailSemanticInventoryBuild; FinalizeSemanticCoverageBuild; LookupSemanticCoverage; MarkSemanticCoverageResourceMissing; New; NewSemanticCoverageSourceSetAccumulator; PageSemanticInventory; PrepareSemanticInventoryBackfill; ReadFieldEnrichment; ReadRelationshipObservations; ReadResourceInventory; ReadRetainedSemanticInventoryPage; RebuildRelationshipCatalog; RebuildResourceInventory; RefreshSemanticInventorySourceAvailability; ResolveSemanticInventorySelections; SemanticCoverageBuildIdentity; SemanticCoverageCollectionSpec; SemanticCoveragePage; SemanticCoverageResourceExpectation; SemanticCoverageSourcePartitionExpectation; SemanticCoverageSourceSetAccumulator; SemanticCoverageWriteStats; Snapshot; Store; ValidateAuthResourcePathOptions; WriteFieldCatalog; WriteRelationshipCatalog; WriteSemanticCoverageContributions; WriteSemanticInventoryContributions,internal/backends/arango/internal/store; internal/catalog; internal/fhir/schema,internal/authscope; internal/backends/arango/internal/store; internal/catalog,internal/backends/arango,—,github.com/bytedance/sonic,—,—,one production importer (internal/backends/arango); inspect ownership/inline potential,go test ./internal/backends/arango/internal/catalog,inventory complete; semantic disposition pending +internal/backends/arango/internal/chartifact,github.com/calypr/loom/internal/backends/arango/internal/chartifact,arango,domain/library,Persist private ClickHouse artifact manifests and leases in a dedicated Arango collection.,catalog.go,catalog_arango_test.go; catalog_test.go,—,—,379,723,10,BootstrapSpec; Catalog; ClaimCleanup; Create; Delete; ListExpired; NewCatalog; ReleaseCleanup; Renew; Update,internal/backends/arango/internal/store; internal/dataframe/execution/chartifact,internal/authscope; internal/backends/arango/internal/store; internal/dataframe/execution/chartifact; internal/store/clickhouse,internal/backends/arango,—,—,—,—,one production importer (internal/backends/arango); inspect ownership/inline potential,go test ./internal/backends/arango/internal/chartifact,inventory complete; semantic disposition pending +internal/backends/arango/internal/dataset,github.com/calypr/loom/internal/backends/arango/internal/dataset,arango,domain/library,Persist immutable dataset generation lifecycle state in ArangoDB.,collections.go; doc.go; document_codec.go; manifest_lifecycle.go; release_store.go,generation_activation_cas_e2e_test.go; release_policy_change_e2e_test.go; revision_inflation_e2e_test.go; store_test.go,—,—,764,1652,15,Activate; ActiveDocumentKey; BootstrapSpec; CollectionSpecs; CompareAndSwapActivateRelease; CompareAndSwapPublishRelease; CreateManifest; ManifestDocumentKey; New; ReadActiveRelease; ReadManifest; ResolveActiveManifest; SaveRelease; Store; TransitionManifest,internal/backends/arango/internal/store; internal/dataset,internal/backends/arango/internal/store; internal/dataset,internal/backends/arango; internal/backends/arango/internal/explorer,internal/backends/arango/internal/publication,github.com/arangodb/go-driver/v2/arangodb/shared,—,compatibility=2,no automatic consolidation signal; perform semantic review,go test ./internal/backends/arango/internal/dataset,inventory complete; semantic disposition pending +internal/backends/arango/internal/explorer,github.com/calypr/loom/internal/backends/arango/internal/explorer,arango,domain/library,"Persist Explorer drafts, revisions, receipts, publications, capabilities, and repository state in ArangoDB.",activation.go; capability_snapshot_store.go; codec.go; collections.go; drafts.go; explicit_groups.go; explorers.go; interpretations.go; publication.go; receipts.go; repository.go; revisions.go; selection_references.go; selections.go; store.go; table_shape_capabilities.go,capability_snapshot_store_test.go; collections_test.go; cross_generation_publish_e2e_test.go; draft_conflict_test.go; draft_transaction_formal_replay_test.go; drafts_integration_test.go; drafts_test.go; explicit_group_storage_integration_test.go; explicit_groups_test.go; interpretations_test.go; receipt_store_test.go; repository_test.go; selection_references_test.go; selection_scale_integration_test.go; selection_storage_integration_test.go; selections_cleanup_e2e_test.go; store_test.go; table_shape_capabilities_test.go,—,—,3366,4552,56,AbortExplicitGroupRevision; AbortSelection; ActivateRepositoryGeneration; AppendExplicitGroupMemberships; AppendSelectionMembers; BeginExplicitGroupRevision; BeginSelection; BootstrapSpec; CapabilitySnapshotCollectionSpec; CapabilitySnapshotStore; CleanupExplicitGroupStaging; CleanupSelectionStaging; CollectionSpecs; CompleteExplicitGroupRevision; CompleteSelection; Create; CreateInterpretationRevision; DigestExplicitGroupRevision; DigestSelectionMembers; FailRevision; Get; GetByIdentity; GetByToken; GetCatalog; GetCatalogForLookup; GetCategoryScan; GetCompilationReceiptForExplorer; GetDraftRevision; GetExplicitGroupRevision; GetInterpretationRevision; GetResolution; GetRevision; GetSelection; InsertCompilationReceipt; InsertRevision; List; ListExplicitGroupDefinitions; ListExplicitGroupRevisions; ListInterpretationLibraries; MigrateLegacyRepositoryConfigs; New; NewCapabilitySnapshotStore; NewTableShapeCapabilityRepository; PublishAuthoring; Put; PutCatalog; PutCategoryScan; PutExplicitGroupDefinitions; PutResolution; SaveDraft; Store; TableShapeCapabilitiesCollectionSpec; TableShapeCapabilityRepository; ValidateSelectionReferences; VisitExplicitGroupMemberships; VisitSelectionMembers,internal/authscope; internal/backends/arango/internal/dataset; internal/backends/arango/internal/publication; internal/backends/arango/internal/store; internal/dataframe/publication; internal/dataset; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/tableshapecap; internal/projectid,internal/authscope; internal/backends/arango/internal/dataset; internal/backends/arango/internal/publication; internal/backends/arango/internal/store; internal/dataframe/publication; internal/dataframe/published; internal/dataframe/recipe; internal/dataset; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/lifecycle; internal/explorer/tableshapecap; internal/projectid,internal/backends/arango,—,github.com/arangodb/go-driver/v2/arangodb/shared,—,legacy=12; migration=4; compatibility=1,one production importer (internal/backends/arango); inspect ownership/inline potential; broad surface (16 production files); inspect cohesion,go test ./internal/backends/arango/internal/explorer,inventory complete; semantic disposition pending +internal/backends/arango/internal/publication,github.com/calypr/loom/internal/backends/arango/internal/publication,arango,domain/library,Persist dataframe publication registry and bundle metadata in ArangoDB.,bundle_registry.go; registry.go,bundle_registry_test.go; dangling_binding_e2e_test.go; execution_read_pin_integration_test.go; name_latest_binding_e2e_test.go; publish_atomicity_e2e_test.go; release_dangle_reconcile_e2e_test.go,—,—,533,2990,22,AcquireBundleLease; AcquireExecutionReadPin; BootstrapSpec; ClaimExecutionCleanup; Client; CompareAndSwapPointer; FindExecutionByKey; FindExecutionBySelector; GetExecution; GetPointer; ListExecutions; New; PublishExecution; Registry; ReleaseBundleLease; ReleaseExecutionCleanup; ReleaseExecutionReadPin; RenewBundleLease; RenewExecutionCleanup; RenewExecutionReadPin; SaveExecution; VisitExecutionPages,internal/backends/arango/internal/store; internal/dataframe/publication,internal/backends/arango/internal/dataset; internal/backends/arango/internal/publication; internal/backends/arango/internal/store; internal/dataframe/errors; internal/dataframe/publication; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/dataset; internal/store/clickhouse,internal/backends/arango; internal/backends/arango/internal/explorer,internal/backends/arango/internal/publication,github.com/arangodb/go-driver/v2/arangodb/shared,—,—,no automatic consolidation signal; perform semantic review,go test ./internal/backends/arango/internal/publication,inventory complete; semantic disposition pending +internal/backends/arango/internal/recipeexec,github.com/calypr/loom/internal/backends/arango/internal/recipeexec,arango,domain/library,Persist recipe versions and execution-use records in ArangoDB.,registry.go; revisions.go,registry_test.go; revisions_test.go; revisions_user_faults_test.go,—,—,294,555,15,BootstrapSpec; Client; Get; List; LoadRecipe; LoadRecipeVersion; New; NewRevisionRegistry; RecipeVersionUsed; Register; Registry; ReplaceRecipeVersion; RevisionBootstrapSpec; RevisionRegistry; SaveRecipe,internal/backends/arango/internal/store; internal/dataframe/errors; internal/dataframe/recipe; internal/dataframe/recipe/exec,internal/backends/arango/internal/store; internal/dataframe/errors; internal/dataframe/recipe; internal/dataframe/recipe/exec,internal/backends/arango,—,—,—,legacy=1,one production importer (internal/backends/arango); inspect ownership/inline potential,go test ./internal/backends/arango/internal/recipeexec,inventory complete; semantic disposition pending +internal/backends/arango/internal/store,github.com/calypr/loom/internal/backends/arango/internal/store,arango,domain/library,"Provide the low-level typed ArangoDB client, query streaming, profiling, and Explain assessment boundary.",client.go; errors.go; explain.go; explain_assessment.go; explain_client.go; options.go; preview_index.go; profile.go; profile_client.go; types.go,client_query_memory_limit_test.go; client_queryrows_test.go; client_revision_test.go; errors_test.go; explain_assessment_test.go; explain_client_test.go; explain_test.go; population_profile_integration_test.go; preview_index_test.go; profile_client_test.go; profile_integration_test.go; profile_test.go,—,—,1950,2210,70,AQLExecutor; AssessExplainResult; BatchInserter; Bootstrap; BootstrapSpec; Client; Close; CollectionCount; CollectionExists; CollectionRevision; CollectionSpec; ConnectionOptions; DatabaseExists; DiagnosticCause; EnsurePreviewCoveringIndex; EnsurePreviewCoveringIndexReplacing; EnsurePreviewCoveringIndexWithStoredValues; EnsurePreviewCoveringIndexWithStoredValuesReplacing; ExecuteAQL; Explain; ExplainAssessment; ExplainCollection; ExplainCollectionScan; ExplainIndex; ExplainIndexLocation; ExplainIndexSummary; ExplainIndexUse; ExplainIndexes; ExplainNode; ExplainOptions; ExplainPlan; ExplainPlanEstimate; ExplainRequest; ExplainResult; ExplainStats; ExplainWarning; InsertBatchRaw; IsArangoErrorResponse; IsArangoPermissionResponse; ListPreviewCoveringIndexes; Open; OptimizerOptions; ParseExplainResult; ParseProfileResult; PreviewCoveringIndex; Profile; ProfileExtra; ProfileNode; ProfileNodeGroup; ProfileNodeSummary; ProfileOptions; ProfilePhases; ProfileRequest; ProfileResult; ProfileStats; ProfileSummary; QueryMemoryLimitFromContext; QueryRows; QueryRowsWithMaxRuntime; RowBatchClient; RowQueryer; RowVisitor; SummarizeProfile; TransactionCollections; TransactionFunc; TransactionalRowClient; TranslateQueryError; UnmarshalJSON; WithQueryMemoryLimit; WithTransaction,internal/dataframe/errors,internal/dataframe/errors,internal/backends/arango; internal/backends/arango/internal/catalog; internal/backends/arango/internal/chartifact; internal/backends/arango/internal/dataset; internal/backends/arango/internal/explorer; internal/backends/arango/internal/publication; internal/backends/arango/internal/recipeexec,—,github.com/arangodb/go-driver/v2/arangodb; github.com/arangodb/go-driver/v2/arangodb/shared; github.com/arangodb/go-driver/v2/connection; github.com/arangodb/go-driver/v2/utils,5 exported interface(s),legacy=1,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/backends/arango/internal/store,inventory complete; semantic disposition pending +internal/backends/postgres,github.com/calypr/loom/internal/backends/postgres,postgres,storage adapter,"Own the PostgreSQL 18 factory: a single Open validates explicit config, opens exactly one readiness-gated pool with own-prefix connection tagging, composes the control runtime, and returns the typed runtime carrying the SourceRows and PlanJSON data-plane ports over tagged compiled commands through the typed Querier bridge; the sole legal importer of the private rows adapter, and no re-export surface - control-plane consumers import internal/backends/postgres/control directly (arch-backend-09 deleted aliases.go).",doc.go; open.go; plan_executor.go,engine_postgres_command_boundary_pg18_test.go; engine_postgres_pg18_test.go; explain_postgres_test.go; open_live_test.go; ownership_contract_test.go; plan_executor_test.go; sentinel_identity_test.go,—,—,178,4408,5,NewPlanJSONExecutor; NewSourceRows; Open; Querier; Runtime,internal/backends/postgres/control; internal/backends/postgres/internal/rows; internal/backends/postgres/internal/store; internal/dataframe/compiler/command; internal/dataframe/execution,internal/backends/postgres; internal/backends/postgres/control; internal/backends/postgres/internal/dataset; internal/backends/postgres/internal/rows; internal/backends/postgres/internal/store; internal/dataframe/compiler; internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataframe/spec; internal/server; internal/source,cmd/loom-data; internal/ingest; internal/server,internal/api/graphql/graph/query; internal/backends/postgres; internal/dataframe/execution,—,—,migration=1,storage adapter; preserve interface and persisted-data contracts,go test ./internal/backends/postgres,inventory complete; semantic disposition pending +internal/backends/postgres/control,github.com/calypr/loom/internal/backends/postgres/control,control,domain/library,"Own the cycle-free public control plane of the PostgreSQL backend as its single definition: configuration and series-assignment validation, the bare-pool client and store-owned batch vocabulary, the dataset manifest lifecycle and catalog evidence port assembly, and the structural query surface; production consumers and backend-adjacent witnesses import it directly for control-plane symbols, and it must never reach the factory root, the private rows adapter, or dataframe/execution (machine-enforced by scripts/packagepolicy).",catalog_read.go; config.go; errors.go; runtime.go; vocabulary.go; vocabulary_control.go,catalog_read_test.go,—,—,1386,284,65,AllSeries; Begin; CatalogStore; Close; Commit; Config; ConnSource; DiscoverExistingAuthResourcePaths; DiscoverFields; Error; Exec; FieldProfileScope; GenerationAdvisoryLockKey; Health; MigrateDatasetReleaseSchema; MigrateResult; MigrateSchema; NewBundleCatalog; NewCapabilitySnapshotStore; NewCatalogFieldProfileStore; NewCatalogRelationshipEvidenceRepository; NewCatalogRelationshipEvidenceStore; NewCatalogStore; NewChartifactCatalog; NewDatasetReleaseStore; NewEdgeRepository; NewEdgeWriter; NewExplicitGroupStore; NewExplorerStore; NewFieldProfileRepository; NewInterpretationStore; NewInventoryRepository; NewPublicationReconciler; NewPublishStore; NewRecipeExecRegistry; NewRecipeRevisionRegistry; NewResourceRepository; NewResourceWriter; NewSemanticContributionRepository; NewSemanticReadRepository; NewTableShapeCapabilityStore; Open; OpenPool; PoolClient; Query; QueryRow; Readiness; Rollback; Row; Rows; Runtime; Scan; SemanticContributionRepository; Series; SeriesSchemas; SetupPolicy; Unwrap; Validate; VerifyDatasetReleaseSchema; VerifyExplorerControlSchema; VerifyPublicationControlSchema; VerifyRecipeChainsSchema; VerifyRecipeExecRegistriesSchema; VerifySchema; WithAdvisoryLock,internal/backends/postgres/internal/catalog; internal/backends/postgres/internal/chartifact; internal/backends/postgres/internal/dataset; internal/backends/postgres/internal/explorer; internal/backends/postgres/internal/publication; internal/backends/postgres/internal/recipeexec; internal/backends/postgres/internal/recipes; internal/backends/postgres/internal/redact; internal/backends/postgres/internal/store; internal/catalog; internal/dataset,internal/backends/postgres/internal/catalog; internal/backends/postgres/internal/store; internal/catalog,cmd/loom-data; internal/backends/postgres; internal/ingest; internal/server,internal/api/graphql/graph/query; internal/api/http; internal/dataframe/compiler; internal/dataframe/compiler/render/postgres; internal/dataframe/execution,github.com/jackc/pgx/v5; github.com/jackc/pgx/v5/pgconn,2 exported interface(s),migration=21,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/backends/postgres/control,inventory complete; semantic disposition pending +internal/backends/postgres/internal/catalog,github.com/calypr/loom/internal/backends/postgres/internal/catalog,postgres,domain/library,"Persist and retrieve catalog facts in PostgreSQL 18: publish-last generation-scoped resource inventory builds, bounded field-profile records with completeness and truncation metadata, leased compare-and-swap-guarded semantic inventory build state, deterministic semantic-inventory contribution rows with content-addressed replay and conflict classification, byte-order-stable keyset pages over retained resources, and authorization-scoped semantic-inventory pages and selection resolution served directly from contribution aggregates.",field_discovery.go; field_profile.go; relationships.go; resource_inventory.go; semantic_build.go; semantic_contribution.go; semantic_page.go; store.go,field_discovery_test.go; field_profile_test.go; relationships_pg_test.go; relationships_test.go; resource_inventory_test.go; semantic_build_test.go; semantic_contribution_test.go; semantic_page_test.go,—,—,4340,11088,54,AdvanceSemanticInventoryBackfill; ClaimSemanticInventoryBackfill; CompleteSemanticInventoryBackfill; DiscoverExistingAuthResourcePaths; DiscoverFieldEnrichment; DiscoverFields; DiscoverRelationshipObservations; DiscoverResourceInventory; EnsureSemanticInventoryEntries; Error; FailSemanticInventoryBackfill; FieldProfileStore; FieldProfileUnavailableError; FieldProfileWriteError; InventoryStore; New; NewFieldProfileStore; NewRelationshipEvidenceRepository; NewRelationshipEvidenceStore; NewSemanticBuildRepository; NewSemanticContributionRepository; NewSemanticInventoryBackfillStore; NewSemanticReadRepository; PageSemanticInventory; PrepareSemanticInventoryBackfill; ReadFieldProfile; ReadRelationshipObservations; ReadResourceInventory; ReadRetainedSemanticInventoryPage; ReadSemanticInventoryBuild; RebuildResourceInventory; RelationshipEvidenceRepository; RelationshipOccurrenceGroup; RelationshipScope; RelationshipStore; ResolveSemanticInventorySelections; ResourceInventoryUnavailableError; ResourceInventoryWriteError; RetainedSemanticSourceMissingError; ScanRelationshipOccurrences; SemanticBuildRepository; SemanticContributionRepository; SemanticInventoryBackfillStore; SemanticInventoryClaimLostError; SemanticInventoryClaimedError; SemanticInventoryContributionConflictError; SemanticInventoryEntriesUnsupportedError; SemanticInventoryStorageMissingError; SemanticInventoryUnsupportedError; SemanticReadRepository; Store; Unwrap; WriteFieldProfile; WriteSemanticInventoryContributions,internal/backends/postgres/internal/store; internal/catalog; internal/fhir/schema,internal/backends/postgres/internal/catalog; internal/backends/postgres/internal/store; internal/catalog; internal/dataset; internal/ingest,internal/backends/postgres/control,internal/backends/postgres/internal/catalog,github.com/jackc/pgx/v5; github.com/jackc/pgx/v5/pgconn; github.com/jackc/pgx/v5/pgxpool,3 exported interface(s),legacy=9; migration=16,one production importer (internal/backends/postgres/control); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/backends/postgres/internal/catalog,inventory complete; semantic disposition pending +internal/backends/postgres/internal/chartifact,github.com/calypr/loom/internal/backends/postgres/internal/chartifact,postgres,domain/library,"Persist private ClickHouse artifact manifests and cleanup leases in the PostgreSQL 18 control plane with CAS progress writes, bounded expired scans, and one-winner cleanup claims while physical tables stay in ClickHouse.",catalog.go,catalog_manager_test.go; catalog_test.go,—,—,535,1605,10,Catalog; ClaimCleanup; Create; DB; Delete; ListExpired; NewCatalog; ReleaseCleanup; Renew; Update,internal/dataframe/execution/chartifact,internal/authscope; internal/backends/postgres/internal/publication; internal/backends/postgres/internal/store; internal/dataframe/execution/chartifact; internal/store/clickhouse,internal/backends/postgres/control,internal/backends/postgres/internal/store,github.com/jackc/pgx/v5; github.com/jackc/pgx/v5/pgconn,1 exported interface(s),migration=6,one production importer (internal/backends/postgres/control); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/backends/postgres/internal/chartifact,inventory complete; semantic disposition pending +internal/backends/postgres/internal/dataset,github.com/calypr/loom/internal/backends/postgres/internal/dataset,postgres,domain/library,"Persist dataset generation manifests, the active-generation compare-and-swap pointer, and immutable project release candidates with the active-release compare-and-swap pointer in PostgreSQL 18.",commands.go; manifest_lifecycle.go; migrate.go; release_store.go; store.go,commands_test.go; manifest_lifecycle_pg_test.go; release_store_pg_test.go,—,—,1725,2488,28,ActivateGeneration; Backend; Command; CommandBackend; CompareAndSwapActivateRelease; ConnSource; CreateManifest; EnsureGenerationPrerequisites; GenerationOwnerAdapter; MigrateSchema; New; NewPostgreSQLCommand; NewReleaseStore; OnReleaseActivated; OwnerMigrations; Parameter; ParameterType; PostgreSQL; ReadActiveRelease; ReadManifest; ReleaseActivation; ReleaseStore; ResolveActiveManifest; SaveRelease; Scan; Store; TransitionManifest; VerifySchema,internal/backends/postgres/internal/store; internal/dataset,internal/backends/postgres/internal/store; internal/dataset,internal/backends/postgres/control; internal/backends/postgres/internal/explorer,internal/backends/postgres; internal/backends/postgres/internal/store,github.com/jackc/pgx/v5; github.com/jackc/pgx/v5/pgconn; github.com/jackc/pgx/v5/pgxpool,go:embed; 2 exported interface(s),legacy=1; migration=28; compatibility=4,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/backends/postgres/internal/dataset,inventory complete; semantic disposition pending +internal/backends/postgres/internal/explorer,github.com/calypr/loom/internal/backends/postgres/internal/explorer,postgres,domain/library,"Persist Explorer owner records, draft saves with atomic version/digest CAS and exact preimage history, insert-once compilation receipts, immutable Explorer revisions, create-once immutable capability snapshots and table-shape capability receipts, project-scoped interpretation libraries advancing heads over immutable content-addressed revisions by transactional parent CAS, the staged selection lifecycle with project-scoped idempotency, an expiring writer owner, and insert-once membership staged in bounded idempotent batches with the digest/count finalized under the current writer fence and bounded cancellation-aware keyset membership paging, atomic publish authoring that commits a verified compilation receipt, the immutable Explorer revision, the active Explorer owner pointer, and the project release activation CAS in one PostgreSQL transaction, and staging explicit-group revision headers with ordered definition rows and insert-once group membership (one row per revision/group/reference so an overlap is several rows, bounded keyset paging, and a fenced digest/count-verified STAGING to COMPLETE transition) under guarded writer-lease, abort, and expired-cleanup lifecycle in PostgreSQL 18 with unique project/Explorer identity, typed columns, lossless JSONB payloads, DB-enforced row freezing and content-digest collision checks, guard-triggered immutability, and CAS-guarded revision lifecycle on the shared pg-store pool and migration runner.",capabilities.go; drafts.go; explicit_group_memberships.go; explicit_groups.go; explorers.go; interpretations.go; publication.go; receipts.go; revisions.go; selection_members.go; selections.go,capabilities_test.go; drafts_test.go; explicit_groups_readpath_test.go; explicit_groups_test.go; explorers_test.go; interpretations_test.go; pgstate02_fixture_test.go; pubauth_fixture_test.go; publication_golden_test.go; publication_guarddrop_test.go; publication_race_test.go; publication_retry_test.go; publication_test.go; publication_zz_test.go; receipts_test.go; revisions_test.go; selection_members_test.go; selections_test.go,—,—,7425,21659,60,AbortExplicitGroupRevision; AbortSelection; AppendExplicitGroupMemberships; AppendSelectionMembers; BeginExplicitGroupRevision; BeginSelection; CapabilitySnapshotStore; CleanupExplicitGroupStaging; CleanupSelectionStaging; CompleteExplicitGroupRevision; CompleteSelection; Create; CreateInterpretationRevision; DigestExplicitGroupRevision; DigestSelectionMembers; ExplicitGroupStore; FailRevision; FailRevisionFrom; Get; GetByIdentity; GetByToken; GetCatalog; GetCatalogForLookup; GetCategoryScan; GetCompilationReceiptForExplorer; GetDraftRevision; GetExplicitGroupRevision; GetInterpretationRevision; GetResolution; GetRevision; GetRevisionForExplorer; GetSelection; InsertCompilationReceipt; InsertRevision; InterpretationStore; List; ListExplicitGroupDefinitions; ListExplicitGroupRevisions; ListInterpretationLibraries; MigrateSchema; New; NewCapabilitySnapshotStore; NewExplicitGroupStore; NewInterpretationStore; NewPublishStore; NewTableShapeCapabilityStore; OwnerMigrations; PublishAuthoring; PublishStore; Put; PutCatalog; PutCategoryScan; PutExplicitGroupDefinitions; PutResolution; SaveDraft; Store; TableShapeCapabilityStore; VerifySchema; VisitExplicitGroupMemberships; VisitSelectionMembers,internal/backends/postgres/internal/dataset; internal/backends/postgres/internal/store; internal/dataframe/publication; internal/dataframe/recipe; internal/dataset; internal/explorer; internal/explorer/capability; internal/explorer/tableshapecap,internal/backends/postgres/internal/dataset; internal/backends/postgres/internal/store; internal/dataframe/publication; internal/dataframe/recipe; internal/dataset; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/tableshapecap,internal/backends/postgres/control,internal/backends/postgres/internal/store,github.com/jackc/pgx/v5; github.com/jackc/pgx/v5/pgconn; github.com/jackc/pgx/v5/pgxpool,go:embed,legacy=1; migration=44,one production importer (internal/backends/postgres/control); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/backends/postgres/internal/explorer,inventory complete; semantic disposition pending +internal/backends/postgres/internal/publication,github.com/calypr/loom/internal/backends/postgres/internal/publication,postgres,domain/library,"Persist dataframe publication execution metadata, exact logical pointers, fenced bundle leases, owner-conditional execution checkpoints, read retention pins arbitrated against cleanup claims on a shared retention anchor, and page-scoped stale execution reconciliation with owner-conditional cleanup claims, plus the single-transaction publish commit of PUBLISHED state with its reader pointer, in the PostgreSQL 18 control plane while ClickHouse remains the row store.",leases.go; migrate.go; publish.go; read_pins.go; reconcile.go; registry.go,leases_test.go; publish_atomicity_test.go; read_pins_test.go; reconcile_test.go; registry_test.go,—,—,2502,7214,44,AcquireBundleLease; AcquireExecutionReadPin; AcquireLease; BundleCatalog; Checkpoint; CheckpointRecord; ClaimExecutionCleanup; CompareAndSwapPointer; DB; ExpireLease; FencedLeases; FindExecutionByKey; FindExecutionBySelector; GetExecution; GetPointer; LatestCheckpoint; ListExecutions; MigrateSchema; New; NewBundleCatalog; NewLeaseKeeper; NewReadPins; NewReconciler; OwnerMigrations; PhysicalDropTable; Pins; PublishExecution; ReadPins; ReconcileDecision; ReconcileOutcome; Reconciler; Registry; ReleaseBundleLease; ReleaseExecutionCleanup; ReleaseExecutionReadPin; ReleaseLease; RenewBundleLease; RenewExecutionCleanup; RenewExecutionReadPin; RenewLease; SaveExecution; SweepStale; VerifySchema; VisitExecutionPages,internal/backends/postgres/internal/store; internal/dataframe/errors; internal/dataframe/publication; internal/dataset,internal/backends/postgres/internal/store; internal/dataframe/publication; internal/dataset,internal/backends/postgres/control,internal/backends/postgres/internal/chartifact; internal/backends/postgres/internal/store,github.com/google/uuid; github.com/jackc/pgx/v5; github.com/jackc/pgx/v5/pgconn,go:embed; 1 exported interface(s),legacy=4; migration=25; compatibility=2,one production importer (internal/backends/postgres/control); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/backends/postgres/internal/publication,inventory complete; semantic disposition pending +internal/backends/postgres/internal/recipeexec,github.com/calypr/loom/internal/backends/postgres/internal/recipeexec,postgres,domain/library,Persist immutable recipe registrations and fenced execution-occurrence usage markers in the PostgreSQL 18 control plane.,registry.go,registry_test.go,—,—,428,1087,12,DB; LoadRecipe; LoadRecipeVersion; MigrateSchema; New; RecipeVersionUsed; RecordRecipeExecutionOccurrence; Registry; RegistryMigrations; ReplaceRecipeVersion; SaveRecipe; VerifySchema,internal/backends/postgres/internal/store; internal/dataframe/recipe/exec,internal/backends/postgres/internal/store; internal/dataframe/recipe; internal/dataframe/recipe/exec,internal/backends/postgres/control,internal/backends/postgres/internal/store,github.com/jackc/pgx/v5; github.com/jackc/pgx/v5/pgconn,go:embed; 1 exported interface(s),legacy=1; migration=22,one production importer (internal/backends/postgres/control); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/backends/postgres/internal/recipeexec,inventory complete; semantic disposition pending +internal/backends/postgres/internal/recipes,github.com/calypr/loom/internal/backends/postgres/internal/recipes,postgres,domain/library,Persist project recipe revision chains with CAS head tracking in the PostgreSQL 18 control plane.,revisions.go,revisions_test.go,—,—,409,1072,9,ChainMigrations; DB; Get; List; MigrateSchema; New; Register; RevisionRegistry; VerifySchema,internal/backends/postgres/internal/store; internal/dataframe/errors; internal/dataframe/recipe,internal/backends/postgres/internal/store; internal/dataframe/errors; internal/dataframe/recipe,internal/backends/postgres/control,internal/backends/postgres/internal/store,github.com/jackc/pgx/v5; github.com/jackc/pgx/v5/pgconn,go:embed; 1 exported interface(s),migration=23,one production importer (internal/backends/postgres/control); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/backends/postgres/internal/recipes,inventory complete; semantic disposition pending +internal/backends/postgres/internal/redact,github.com/calypr/loom/internal/backends/postgres/internal/redact,redact,domain/library,Scrub DSN passwords and secret parameter values from pool-open diagnostics so credential material never reaches rendered error text while the cause chain stays intact.,redact.go,redact_internal_test.go,—,—,79,84,3,ExtractDSNPassword; RedactDSN; ScrubSecretText,—,—,internal/backends/postgres/control,—,—,—,—,one production importer (internal/backends/postgres/control); inspect ownership/inline potential,go test ./internal/backends/postgres/internal/redact,inventory complete; semantic disposition pending +internal/backends/postgres/internal/rows,github.com/calypr/loom/internal/backends/postgres/internal/rows,postgres,domain/library,"Stream typed rows from tagged compiled PostgreSQL commands over pgx connections, preserving bound-value integrity and three-state typed decoding, enforcing output column contracts, and classifying cancellation, visitor stops, contract violations, and driver failures with identifiable wrapped causes.",rows.go,rows_boundary_pg18_ab03_test.go; rows_postgres_test.go; rows_postquery_postgres_test.go; rows_test.go,—,—,483,3537,8,Error; Executor; NewExecutor; Querier; QueryError; RowVisitor; SourceRows; Unwrap,internal/backends/postgres/internal/store; internal/dataframe/compiler/command; internal/dataframe/execution,internal/backends/postgres/internal/store; internal/dataframe/compiler/command; internal/dataframe/errors; internal/dataframe/execution,internal/backends/postgres,—,github.com/jackc/pgx/v5; github.com/jackc/pgx/v5/pgconn; github.com/jackc/pgx/v5/pgtype,1 exported interface(s),migration=1,one production importer (internal/backends/postgres); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/backends/postgres/internal/rows,inventory complete; semantic disposition pending +internal/backends/postgres/internal/store,github.com/calypr/loom/internal/backends/postgres/internal/store,postgres,domain/library,"Provide the bounded pgx/v5-backed PostgreSQL 18 source store: readiness-pinged pool, advisory-lock serialized migration runner, scoped FHIR resource repository that accepts the logical internal/source contract on writes and answers reads with its store-owned physical row under the server-assigned keyset sequence, scoped typed edge-occurrence repository, publish-last inventory repository, scoped bounded field-profile record persistence, the ledgered idempotent COPY batch writer for resource and edge rows, and resource-only and edge-only batch facades over that writer.",advisory.go; batch_writer.go; client.go; commit_outcome.go; edge.go; edge_writer.go; errors.go; field_profile.go; inventory.go; migrate.go; migration_ownership.go; resource.go; resource_writer.go,batch_writer_test.go; client_test.go; edge_test.go; edge_writer_test.go; errors_test.go; field_profile_ddl_test.go; field_profile_filter_golden_test.go; inventory_test.go; migrate_test.go; ownership_contract_test.go; resource_test.go; resource_writer_test.go; verify_classify_test.go; verify_readonly_test.go,—,—,5898,13789,112,Acquire; AsUniqueConflict; Assignment; AssignmentConflict; Batch; BatchAccountingError; BatchEdgeConflictError; BatchIdentityConflictError; BatchResourceConflictError; BatchResult; BatchWriteError; BatchWriter; Cause; CheckServerVersion; Client; Close; CommitOutcomeUnknown; Config; ConnSource; ConnectError; DriverCondition; EdgeBatch; EdgeBatchResult; EdgeConflictError; EdgeOccurrenceRow; EdgeRepository; EdgeScope; EdgeSourceMissingError; EdgeWriter; EmbeddedMigrations; Error; FieldProfileCoverage; FieldProfileFilter; FieldProfileRepository; FieldProfileScope; FutureSchemaVersionError; GenerationAdvisoryLockKey; GetResource; HistoryEntry; HistoryManifest; InsertCounts; InsertEdge; InsertResource; InventoryCountConflictError; InventoryCountScope; InventoryPublishGuardError; InventoryRepository; InventorySourceGroup; IsAbortedTransaction; IsRefusedConnection; IsUniqueViolation; LedgerRecord; LoadMigrations; LookupManifest; Manifest; ManifestJSON; Migrate; MigrateResult; MigrateSchema; Migration; MigrationApplyError; MigrationChecksumMismatchError; MigrationRunner; MissingEmbeddedMigrationError; NewBatchWriter; NewEdgeRepository; NewEdgeWriter; NewFieldProfileRepository; NewInventoryRepository; NewMigrationRunner; NewResourceRepository; NewResourceWriter; Open; OutOfOrderMigrationError; Ping; PublishBuild; Query; QueryRow; ReadBuild; ReadCountSums; ReadEdgesFromSource; ReadEdgesToTarget; ReadFieldProfileRows; ReadFieldProfileRowsFiltered; ReadProfileCoverage; ReadResources; ResourceBatch; ResourceBatchResult; ResourceConflictError; ResourceNotFoundError; ResourceRepository; ResourceRow; ResourceWriter; RetryableTransactionConflict; Scan; ScanSourceCounts; SeriesID; TranslateCommitError; TranslateError; TranslatedError; UniqueConflictError; UnsupportedVersionError; Unwrap; UpsertFieldProfileBatch; Validate; ValidateAssignment; Verify; VerifySchema; WithAdvisoryLock; WriteBatch; WriteEdgeBatch; WriteResourceBatch,internal/catalog; internal/dataframe/errors; internal/source,internal/api/http; internal/authscope; internal/backends/postgres/internal/chartifact; internal/backends/postgres/internal/dataset; internal/backends/postgres/internal/explorer; internal/backends/postgres/internal/publication; internal/backends/postgres/internal/recipeexec; internal/backends/postgres/internal/recipes; internal/backends/postgres/internal/store; internal/catalog; internal/dataframe/errors; internal/dataframe/execution/chartifact; internal/dataframe/publication; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataset; internal/explorer; internal/source,internal/backends/postgres; internal/backends/postgres/control; internal/backends/postgres/internal/catalog; internal/backends/postgres/internal/dataset; internal/backends/postgres/internal/explorer; internal/backends/postgres/internal/publication; internal/backends/postgres/internal/recipeexec; internal/backends/postgres/internal/recipes; internal/backends/postgres/internal/rows,internal/backends/postgres/internal/chartifact; internal/backends/postgres/internal/store,github.com/jackc/pgx/v5; github.com/jackc/pgx/v5/pgconn; github.com/jackc/pgx/v5/pgxpool,go:embed; 1 exported interface(s),legacy=16; migration=143,broad surface (13 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/backends/postgres/internal/store,inventory complete; semantic disposition pending +internal/catalog,github.com/calypr/loom/internal/catalog,catalog,domain/library,"Own persistence-neutral observed FHIR field, pivot, relationship, and capability facts.",auth_scope.go; backend_contract.go; cache.go; capability_evidence.go; generation.go; helpers.go; limits.go; rebuild.go; relationship_evidence.go; relationships.go; resource_inventory.go; retained_semantic_page.go; semantic.go; semantic_coverage.go; semantic_coverage_contract.go; semantic_inventory.go; semantic_schema.go; types.go; write_profiler.go,auth_scope_test.go; cache_test.go; capability_evidence_test.go; generation_test.go; relationship_evidence_census_test.go; relationship_evidence_test.go; relationships_test.go; semantic_coding_display_projection_test.go; semantic_coverage_diagnostics_test.go; semantic_coverage_test.go; semantic_inventory_buildid_cache_test.go; semantic_inventory_category_test.go; semantic_inventory_emitter_anchor_test.go; semantic_inventory_test.go; semantic_schema_identifier_type_test.go; semantic_schema_test.go; semantic_test.go; write_profiler_coverage_test.go; write_profiler_test.go,—,—,6471,3946,146,AuthResourcePathOptions; Cache; CacheOptions; CacheStats; CapabilityEvidence; CapabilityEvidenceOptions; CapabilityEvidenceReader; CensusSemanticCoverage; ConceptCandidate; DecodeSemanticInventoryCursor; DefaultProfileLimits; DiscoverFields; DiscoverReferences; DiscoverSemanticCodeableConceptTemplates; Documents; EffectiveAuthResourcePathsUnrestricted; EncodeSemanticInventoryCursor; EvidenceDiagnostic; EvidenceStatus; EvidenceStatusFor; ExplicitAuthResourcePathsUnrestricted; ExtensionValueObservation; FieldCatalogDocument; FieldEnrichmentDigest; FieldEnrichmentObservation; FieldEnrichmentOptions; FieldEnrichmentResult; FieldProfileScope; FieldProfileWriter; HasDatasetGeneration; InvalidRelationship; InvalidateAll; InvalidateProject; InventoryRebuilder; LegacySemanticBindingIDV1; Merge; MergeRelationshipCounts; NewCache; NewCacheWithOptions; NewProfilerForGenerationWithLimits; NewResourceInventoryBuild; NewSemanticInventoryBuild; NewSemanticInventoryEmitter; NewShapePlanCacheWithLimit; NewShapePlanCacheWithLimits; NormalizeDatasetGeneration; NormalizeProfileLimits; ObservePayload; ObservePayloadWithCoverage; ObservePayloadWithInventory; PopulatedField; PopulatedFieldOptions; PopulatedReference; PopulatedReferenceOptions; ProfileLimits; Profiler; ProjectRelationshipObservation; ReadCapabilityEvidence; ReconcileSemanticCoverage; RelationshipAuditOptions; RelationshipAuditSummary; RelationshipCatalogDocument; RelationshipCatalogDocuments; RelationshipCountsFromRawEdges; RelationshipDiscoverer; RelationshipEvidenceKey; RelationshipEvidenceKeyFor; RelationshipEvidenceLess; RelationshipKey; RelationshipObservation; RelationshipObservationDigest; RelationshipObservationFromKey; RelationshipObservationOptions; RelationshipObservationResult; RelationshipRebuildOptions; RelationshipRebuildSummary; RepeatedShapeDigest; ResourceInventoryBuild; ResourceInventoryBuildKey; ResourceInventoryCount; ResourceInventoryCoverageDigest; ResourceInventoryDigest; ResourceInventoryObservation; ResourceInventoryOptions; ResourceInventoryRebuildOptions; ResourceInventoryRebuildSummary; ResourceInventoryResult; RetainedSemanticInventoryPage; RetainedSemanticInventoryRow; RetentionBytes; SemanticCodeableConceptTemplate; SemanticContributor; SemanticCoverageBuildState; SemanticCoverageCandidatePath; SemanticCoverageCandidateResult; SemanticCoverageCandidateStatus; SemanticCoverageCensus; SemanticCoverageCensusGap; SemanticCoverageCensusStatus; SemanticCoverageCounts; SemanticCoverageDiagnostic; SemanticCoverageDispositionKind; SemanticCoverageEmissionEvidence; SemanticCoverageEmissionSink; SemanticCoverageFacetAggregate; SemanticCoverageLookupOptions; SemanticCoverageLookupResult; SemanticCoverageOccurrenceKind; SemanticCoverageOwnerLocator; SemanticCoveragePathAggregate; SemanticCoveragePresence; SemanticCoverageReason; SemanticCoverageReference; SemanticCoverageReferenceKind; SemanticCoverageRole; SemanticCoverageSourceContribution; SemanticCoverageSourceIdentity; SemanticCoverageSourceSink; SemanticInventoryBuild; SemanticInventoryBuildID; SemanticInventoryBuildKey; SemanticInventoryCheckpoint; SemanticInventoryContribution; SemanticInventoryEmitter; SemanticInventoryEntry; SemanticInventoryFamily; SemanticInventoryFamilyID; SemanticInventoryFamilyReference; SemanticInventoryObservationSink; SemanticInventoryPage; SemanticInventoryPageDigest; SemanticInventoryPageOptions; SemanticInventoryReference; SemanticInventoryResolveOptions; SemanticInventoryResolveResult; SemanticInventorySelectionMode; SemanticInventoryState; SemanticObservation; SemanticObservationCompleteness; SemanticObservationKey; SemanticObservationSource; SemanticObservationValue; ShapePlanCache; Stats; Truncated; ValidateSemanticInventoryBuild,internal/fhir/schema; internal/fhir/semantic,generated/fhirschema; internal/fhir/schema; internal/fhir/semantic,cmd/arango-fhir-proto; cmd/loom-data; internal/api/graphql/graph/query; internal/api/recipe; internal/authscope; internal/backends/arango/internal/catalog; internal/backends/postgres/control; internal/backends/postgres/internal/catalog; internal/backends/postgres/internal/store; internal/explorer/authoringv2; internal/explorer/lifecycle; internal/ingest; internal/server,internal/api/bulk/load; internal/api/graphql/graph; internal/api/graphql/graph/resolver; internal/dataframe/compiler; internal/explorer/capability; internal/explorer/compilation,github.com/bytedance/sonic,5 exported interface(s),legacy=13; migration=1; compatibility=1,broad surface (19 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/catalog,inventory complete; semantic disposition pending internal/ci,github.com/calypr/loom/internal/ci,ci,domain/library,"Exercise repository-wide Go formatting and vet gates, including injected fault detection.",—,gofmt_gate_test.go; vet_gate_test.go,—,—,0,283,0,—,—,—,—,—,—,—,—,no in-repo production importer; verify runtime/tooling reachability,go test ./internal/ci,inventory complete; semantic disposition pending internal/dataframe/columntransform,github.com/calypr/loom/internal/dataframe/columntransform,columntransform,domain/library,"Define closed, backend-neutral transformations over one selected output column value.",types.go,types_test.go,—,—,79,61,7,CategoryMapping; Clone; ExactCategoryRecode; Kind; UnknownPolicy; Validate; ValueTransformation,—,—,internal/dataframe/compiler/lower; internal/dataframe/recipe; internal/explorer/authoringv2,internal/dataframe/compiler; internal/explorer/compilation,—,—,—,no automatic consolidation signal; perform semantic review,go test ./internal/dataframe/columntransform,inventory complete; semantic disposition pending -internal/dataframe/compiler,github.com/calypr/loom/internal/dataframe/compiler,compiler,domain/library,"Orchestrate semantic planning, physical lowering, optimization, rendering, and compiled-query assembly.",api.go; category_scan.go; category_scan_filter_pushdown.go; category_scan_related.go; cell_trace.go; clone_helpers.go; compiled_query.go; construction_preview_root_id_filter.go; graph_execution.go; ir_helpers.go; optimizer.go; output.go; population_mapping.go; preview_covering_index.go; recipe_execution.go; recipe_paging.go; row_lineage.go; semantic_group_projection_preview.go; table_shape_exclusions.go,category_scan_arango_test.go; category_scan_parent_presence_arango_test.go; category_scan_related_collection_identity_test.go; category_scan_related_presence_arango_test.go; category_scan_related_test.go; category_scan_test.go; cell_trace_test.go; cohort_drop_boundary_arango_test.go; construction_arango_test.go; construction_coded_group_arango_test.go; construction_coded_group_execution_test.go; construction_coded_pivot_test.go; construction_execution_test.go; construction_group_expand_arango_test.go; construction_group_preview_liveness_arango_test.go; construction_group_related_summary_arango_test.go; construction_group_source_streaming_test.go; construction_pivot_lineage_arango_test.go; construction_preview_root_id_filter_test.go; construction_related_count_arango_test.go; construction_related_expand_arango_test.go; construction_related_group_preview_arango_test.go; correlated_arango_test.go; correlated_legacy_compat_test.go; correlated_physical_test.go; correlated_selection_test.go; correlated_system_version_arango_test.go; explicit_cohort_values_arango_test.go; filter_semantics_test.go; filter_test.go; generic_physical_plan_test.go; grain_test.go; group_connected_field_source_arango_test.go; group_field_source_arango_test.go; nested_expansion_arango_test.go; operator_oracles_arango_test.go; physical_diagnostics_test.go; physical_lowering_test.go; physical_optimize_test.go; physical_plan_test.go; physical_projection_reserved_name_arango_test.go; physical_render_test.go; physical_required_match_test.go; physical_scope_test.go; pivot_presence_arango_audit_test.go; pivot_presence_audit_test.go; plan_test_helpers_test.go; population_frontier_arango_test.go; population_mapping_filter_exists_test.go; population_mapping_identity_guard_test.go; population_mapping_routed_group_arango_test.go; population_mapping_test.go; recipe_paging_related_filter_sequence_test.go; recipe_paging_routed_exists_arango_test.go; recipe_paging_test.go; root_page_related_seed_arango_test.go; row_expansion_test.go; row_item_projection_arango_test.go; row_lineage_cohort_filter_arango_test.go; row_lineage_composed_arango_test.go; row_lineage_group_pivot_arango_test.go; row_lineage_group_rows_arango_test.go; row_lineage_test.go; selection_semantics_test.go; selector_first_limit_test.go; semantic_group_arango_test.go; semantic_group_preview_arango_test.go; semantic_group_projection_factorization_arango_test.go; semantic_validation_test.go; source_group_preview_arango_test.go; streaming_missing_category_pivot_test.go; table_reshape_arango_test.go; table_reshape_execution_window_test.go; table_shape_exclusions_test.go; workspace_output_capture_test.go,—,—,7232,29200,37,CategoryOverflowWitness; CategoryScanProof; CategoryScanRefusal; CategoryScanRefusalCode; CategoryScanRefusalCodeOf; CompileCategoryScanOutputWithPolicy; CompileCategoryScanStageWithPolicy; CompileCellTraceOutputWithPolicy; CompilePopulationMappingOutputWithPolicy; CompileRecipeOutputPageWithPolicy; CompileRecipeOutputWithPolicy; CompileResolvedGraphQueryWithPolicy; CompileResolvedRecipePlanWithPolicy; CompileRowLineageOutput; CompileTableShapeExclusionsWithPolicy; CompiledCategoryScanQuery; CompiledCellTraceQuery; CompiledOutputPage; CompiledPopulationMappingQuery; CompiledQuery; CompiledRowLineageQuery; CompiledTableShapeExclusionQuery; Error; PreviewCoveringIndexReplacement; PreviewCoveringIndexSpec; PreviewGroupProjectionIndexSpec; PreviewGroupProjectionSpec; PreviewGroupScanSpec; PublicOutputColumns; RenderPreviewGroupProjectionAssessment; RenderPreviewGroupProjectionFactorization; RenderPreviewPhysicalPlanWithRootIndexHint; RowLineageCapability; RowLineageCapabilityForOutput; RowLineageUnsupportedError; TableShapeExclusionRefusal; TableShapeExclusionRefusalCode,internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/aql; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec,internal/authscope; internal/catalog; internal/dataframe/columntransform; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/aql; internal/dataframe/execution; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataframe/spec; internal/dataframe/unit; internal/explorer/capability; internal/fhir/schema; internal/projectid; internal/store/arango,cmd/dataframe-profile; cmd/preview-covering-index-artifact; internal/api/graphql/graph/query; internal/dataframe/execution; internal/explorer/lifecycle; internal/server,conformance/compiler; internal/dataframe/compiler; internal/dataframe/compiler/render/aql; internal/explorer/compilation,—,—,legacy=2; compatibility=1,broad surface (19 production files); inspect cohesion,go test ./internal/dataframe/compiler,inventory complete; semantic disposition pending +internal/dataframe/compiler,github.com/calypr/loom/internal/dataframe/compiler,compiler,domain/library,"Orchestrate semantic planning, physical lowering, optimization, rendering, and compiled-query assembly.",api.go; category_scan.go; category_scan_filter_pushdown.go; category_scan_related.go; cell_trace.go; clone_helpers.go; compiled_query.go; construction_preview_root_id_filter.go; graph_execution.go; ir_helpers.go; optimizer.go; output.go; population_mapping.go; postgres_target.go; preview_covering_index.go; recipe_execution.go; recipe_paging.go; row_lineage.go; semantic_group_projection_preview.go; table_shape_exclusions.go; target.go,backend_leaf_guard_test.go; backend_ownership_guard_contract_test.go; category_scan_arango_test.go; category_scan_parent_presence_arango_test.go; category_scan_related_collection_identity_test.go; category_scan_related_presence_arango_test.go; category_scan_related_test.go; category_scan_test.go; cell_trace_test.go; cohort_drop_boundary_arango_test.go; command_helpers_external_test.go; command_helpers_test.go; compiled_command_test.go; construction_arango_test.go; construction_coded_group_arango_test.go; construction_coded_group_execution_test.go; construction_coded_pivot_test.go; construction_execution_test.go; construction_group_expand_arango_test.go; construction_group_preview_liveness_arango_test.go; construction_group_related_summary_arango_test.go; construction_group_source_streaming_test.go; construction_pivot_lineage_arango_test.go; construction_preview_root_id_filter_test.go; construction_related_count_arango_test.go; construction_related_expand_arango_test.go; construction_related_group_preview_arango_test.go; correlated_arango_test.go; correlated_legacy_compat_test.go; correlated_physical_test.go; correlated_selection_test.go; correlated_system_version_arango_test.go; explicit_cohort_values_arango_test.go; filter_semantics_test.go; filter_test.go; generic_physical_plan_test.go; grain_test.go; group_connected_field_source_arango_test.go; group_field_source_arango_test.go; nested_expansion_arango_test.go; operator_oracles_arango_test.go; physical_diagnostics_test.go; physical_lowering_test.go; physical_optimize_test.go; physical_plan_test.go; physical_projection_reserved_name_arango_test.go; physical_render_test.go; physical_required_match_test.go; physical_scope_test.go; pivot_presence_arango_audit_test.go; pivot_presence_audit_test.go; plan_test_helpers_test.go; population_frontier_arango_test.go; population_mapping_filter_exists_test.go; population_mapping_identity_guard_test.go; population_mapping_routed_group_arango_test.go; population_mapping_test.go; postgres_coded_group_compile_test.go; postgres_parity_fixture_test.go; postgres_target_fingerprint_golden_test.go; public_columns_freeze_test.go; recipe_execution_pg18_test.go; recipe_execution_test.go; recipe_paging_related_filter_sequence_test.go; recipe_paging_routed_exists_arango_test.go; recipe_paging_test.go; root_page_related_seed_arango_test.go; row_expansion_test.go; row_item_projection_arango_test.go; row_lineage_cohort_filter_arango_test.go; row_lineage_composed_arango_test.go; row_lineage_group_pivot_arango_test.go; row_lineage_group_rows_arango_test.go; row_lineage_test.go; selected_source_contract_test.go; selection_semantics_test.go; selector_first_limit_test.go; semantic_group_arango_test.go; semantic_group_preview_arango_test.go; semantic_group_projection_factorization_arango_test.go; semantic_validation_test.go; source_group_preview_arango_test.go; streaming_missing_category_pivot_test.go; table_reshape_arango_test.go; table_reshape_execution_window_test.go; table_shape_exclusions_test.go; workspace_output_capture_test.go,—,—,8120,33173,44,ArangoTarget; Backend; CategoryOverflowWitness; CategoryScanProof; CategoryScanRefusal; CategoryScanRefusalCode; CategoryScanRefusalCodeOf; CompileCategoryScanOutput; CompileCategoryScanStage; CompileCellTraceOutput; CompileClickHousePrefix; CompilePopulationMappingOutput; CompileRecipeOutput; CompileRecipeOutputPage; CompileResolvedGraphQuery; CompileResolvedRecipePlan; CompileRowLineageOutput; CompileTableShapeExclusions; CompiledCategoryScanQuery; CompiledCellTraceQuery; CompiledClickHousePrefix; CompiledOutputPage; CompiledPopulationMappingQuery; CompiledQuery; CompiledRowLineageQuery; CompiledTableShapeExclusionQuery; Error; PostgreSQLTarget; PreviewCoveringIndexReplacement; PreviewCoveringIndexSpec; PreviewGroupProjectionIndexSpec; PreviewGroupProjectionSpec; PreviewGroupScanSpec; PublicOutputColumns; RenderPreviewGroupProjectionAssessment; RenderPreviewGroupProjectionFactorization; RenderPreviewPhysicalPlanWithRootIndexHint; RowLineageCapability; RowLineageCapabilityForOutput; RowLineageUnsupportedError; TableShapeExclusionRefusal; TableShapeExclusionRefusalCode; Target; Valid,internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/postgres; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec,internal/authscope; internal/backends/arango; internal/backends/postgres/control; internal/catalog; internal/dataframe/columntransform; internal/dataframe/compiler; internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/postgres; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataframe/spec; internal/dataframe/unit; internal/explorer/capability; internal/fhir/schema; internal/projectid; internal/source; scripts/packagepolicy,cmd/dataframe-profile; cmd/preview-covering-index-artifact; internal/api/graphql/graph/query; internal/dataframe/execution; internal/explorer/lifecycle; internal/server,conformance/compiler; internal/api/graphql/graph; internal/api/graphql/graph/resolver; internal/backends/postgres; internal/dataframe/compiler; internal/dataframe/compiler/command; internal/dataframe/compiler/render/aql; internal/explorer/compilation,—,—,legacy=2; compatibility=1,broad surface (21 production files); inspect cohesion,go test ./internal/dataframe/compiler,inventory complete; semantic disposition pending internal/dataframe/compiler/capability,github.com/calypr/loom/internal/dataframe/compiler/capability,capability,domain/library,Probe which dataframe language capabilities the compiler can prove and support.,probe.go; schema_fields.go,probe_test.go; schema_fields_test.go,—,—,803,313,22,CandidateCapability; CandidateRequest; Chart; Filter; Options; OwnerRecordsCapability; OwnerRecordsRequest; ProbeCandidate; ProbeOwnerRecords; ProbeProjection; ProbeRoot; ProbeTraversal; Rendered; Result; RootCapability; RootRequest; SchemaFieldDescriptor; SchemaFieldDescriptors; Scope; Traversal; TraversalCapability; TraversalRequest,internal/authscope; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/aql; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec; internal/fhir/schema,internal/authscope; internal/dataframe/recipe; internal/dataframe/spec; internal/fhir/schema,internal/explorer/lifecycle; internal/server,—,—,—,—,no automatic consolidation signal; perform semantic review,go test ./internal/dataframe/compiler/capability,inventory complete; semantic disposition pending -internal/dataframe/compiler/ir,github.com/calypr/loom/internal/dataframe/compiler/ir,ir,domain/library,"Define typed physical operations, values, scope proofs, policies, and validation.",clone.go; dataset_generation.go; diagnostics.go; doc.go; navigation.go; physical_combine.go; physical_combine_scalar.go; physical_operations.go; physical_stage_sequence.go; physical_stage_validation.go; physical_table_shape.go; physical_types.go; physical_validation.go; physical_validation_expr.go; physical_validation_refs.go; physical_validation_shapes.go; physical_values.go; policy.go; prefix.go; scope.go; selector_helpers.go; whole_relation_scope_evidence.go; workspace_combine_scope_operand.go,group_related_field_source_validation_test.go; physical_combine_scalar_test.go; physical_combine_test.go; physical_group_pivot_lineage_validation_test.go; physical_root_page_candidate_seed_test.go; physical_scope_cycle_test.go; physical_stage_validation_test.go; population_mapping_test.go; population_root_test.go; related_coding_group_lineage_validation_test.go; row_lineage_trace_validation_test.go; subplan_test.go; unit_validation_test.go; unnest_test.go; workspace_output_validation_test.go,—,—,11918,1691,196,AddDecision; CanonicalExecutionPhysicalPlan; ClickHouseArtifactIdentity; ClickHouseCombineScalarBaseType; ClonePhysicalExpression; ClonePhysicalOperation; ClonePhysicalOperations; ClonePhysicalPlan; ClonePhysicalPredicateExpression; ClonePhysicalSubplan; CompilerPlanDiagnostics; DatasetGenerationBindValue; DecomposePhysicalTraversalPrefixAt; DefaultPhysicalOptimizationPolicy; Digest; Error; EstimatePreparedSelectorWork; EstimateTraversalSharingWork; Matches; NewPhysicalOptimizationReport; NewPrivateAQLPrefixScopeOperand; NewPublishedWorkspaceScopeOperand; NewWholeRelationScopeEvidence; NewWorkspaceCombineScopeEvidence; NewWorkspaceOutputScopeOperand; NormalizeDatasetGeneration; PhysicalAggregate; PhysicalAggregateOperation; PhysicalCall; PhysicalCardinality; PhysicalCellTraceConstruction; PhysicalCellTraceConstructionInput; PhysicalCellTraceContribution; PhysicalCellTraceRelatedSource; PhysicalCellTraceReshape; PhysicalCellTraceReshapeSource; PhysicalCellTraceReshapeSourceKind; PhysicalCellTraceReturn; PhysicalClickHouseCombine; PhysicalClickHousePrefix; PhysicalCodedGroupPathSegment; PhysicalCohortRootSource; PhysicalCollectionScan; PhysicalCombineInputRef; PhysicalCombineKey; PhysicalCombineKind; PhysicalCombineOutputColumn; PhysicalCombineProjection; PhysicalConstructionStage; PhysicalContributorWindow; PhysicalCorrelation; PhysicalDerivedLet; PhysicalDocumentLookup; PhysicalEngine; PhysicalExpression; PhysicalExpressionKind; PhysicalExpressionLet; PhysicalExpressionProjection; PhysicalExtract; PhysicalFilter; PhysicalGraphReturn; PhysicalGroupMemberValue; PhysicalGroupRows; PhysicalGroupRowsCellTrace; PhysicalGroupedCodedPivotCategory; PhysicalGroupedPivot; PhysicalGroupedPivotCategory; PhysicalGroupedPivotKey; PhysicalKeySet; PhysicalKeySetLookup; PhysicalKeyedMap; PhysicalLimit; PhysicalLiteral; PhysicalMapReduction; PhysicalNullBehavior; PhysicalObject; PhysicalObjectKeys; PhysicalObjectLookup; PhysicalOperation; PhysicalOperationKind; PhysicalOptimizationDecision; PhysicalOptimizationPolicy; PhysicalOptimizationReport; PhysicalOptimizationRule; PhysicalOptimizationRuleState; PhysicalOwnerRecords; PhysicalPathExtend; PhysicalPathNode; PhysicalPathRelationship; PhysicalPathSeed; PhysicalPivotCategoryMatchKind; PhysicalPivotMap; PhysicalPlan; PhysicalPlanDiagnostics; PhysicalPopulationMappingIdentityPart; PhysicalPopulationMappingReturn; PhysicalPopulationRootSource; PhysicalPredicate; PhysicalPredicateExpression; PhysicalPredicateKind; PhysicalPreparedField; PhysicalPreparedReference; PhysicalPreparedSet; PhysicalProjection; PhysicalProjectionPresence; PhysicalRelatedField; PhysicalReturn; PhysicalRootPageCandidateSeed; PhysicalRootScan; PhysicalRowLineageReturn; PhysicalRowLineageStageMatch; PhysicalRowLineageTrace; PhysicalScopeWindowEnd; PhysicalSelectorExecutionMode; PhysicalSelectorFallback; PhysicalSelectorValueDemand; PhysicalSet; PhysicalSetKind; PhysicalSetOutput; PhysicalSetOutputField; PhysicalSetProjection; PhysicalSetProjectionField; PhysicalSetReduction; PhysicalSetReductionField; PhysicalSetReductionMode; PhysicalSlice; PhysicalSort; PhysicalSource; PhysicalStageCodedGroup; PhysicalStageCohortGroup; PhysicalStageColumn; PhysicalStageExpand; PhysicalStageGroup; PhysicalStageGroupAggregate; PhysicalStageGroupCodingKey; PhysicalStageGroupCurrentStageFieldSource; PhysicalStageGroupFieldSource; PhysicalStageGroupFieldSourceKind; PhysicalStageGroupKey; PhysicalStageGroupMissingKeyPolicy; PhysicalStageGroupRelatedOccurrenceAdapter; PhysicalStageGroupRelatedOccurrenceFieldSource; PhysicalStageGroupRelatedProjection; PhysicalStageGroupRelatedRouteStep; PhysicalStageGroupSemanticKey; PhysicalStageOperationKind; PhysicalStagePopulationMappingReturn; PhysicalStageRelatedExpand; PhysicalStageRelatedField; PhysicalStageRelatedRecordAnchor; PhysicalStageRelatedRouteStep; PhysicalStageRelatedSource; PhysicalStageRowValue; PhysicalStageSequence; PhysicalSubplan; PhysicalTableShapeExclusionReturn; PhysicalTemporalOrdering; PhysicalTraversal; PhysicalTraversalDecision; PhysicalTraversalDirection; PhysicalTraversalPrefix; PhysicalTraversalPrefixDecomposition; PhysicalTraversalPrefixError; PhysicalTraversalPrefixRejectionReason; PhysicalTraversalStrategy; PhysicalTraversalSubset; PhysicalUnitNormalization; PhysicalUnnest; PhysicalUnnestAncestor; PhysicalUnnestEmptyPolicy; PhysicalUnnestExactGroupShapeEqual; PhysicalUnnestOwner; PhysicalUnnestRouteStep; PhysicalUnnestSortKeys; PhysicalUnpivot; PhysicalUnpivotIdentityPart; PhysicalUnpivotInput; PhysicalUnpivotOutput; PhysicalValue; ResolvedClickHouseColumn; ResolvedClickHousePrivateArtifact; ResolvedClickHouseTable; RichConsumerGroup; RichSourceReuse; RuleEnabled; TotalConsumers; Validate; ValidateForWorkspaceCompilation; ValidateGenericPhysicalPlanScope; ValidateScope; ValidateWithPrivateStage; ValidateWithWorkspaceArtifacts; WholeRelationScopeEvidence; WholeRelationScopeIdentity; WithRule; WorkspaceCombineScopeOperand,internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema; internal/projectid,internal/dataframe/spec; internal/dataframe/unit,cmd/dataframe-profile; cmd/preview-covering-index-artifact; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/clickhouse; internal/dataframe/execution; internal/explorer/lifecycle; internal/server,conformance/compiler; internal/explorer/compilation; internal/store/clickhouse,—,—,legacy=9; compatibility=4,broad surface (23 production files); inspect cohesion,go test ./internal/dataframe/compiler/ir,inventory complete; semantic disposition pending -internal/dataframe/compiler/lower,github.com/calypr/loom/internal/dataframe/compiler/lower,lower,domain/library,Lower logical FHIR dataframe plans into storage-aware physical IR.,auth_scope.go; child_sets.go; column_naming.go; correlated_binding.go; doc.go; filter_literal.go; graph_plan.go; group_field_source_context.go; group_field_source_related.go; group_related_occurrence_lower.go; group_rows.go; helpers.go; plan_orchestration.go; population.go; recipe_column_transformations.go; recipe_compile.go; recipe_construction.go; recipe_construction_coded_group.go; recipe_construction_coded_pivot.go; recipe_construction_combine.go; recipe_construction_combine_metadata.go; recipe_construction_composite.go; recipe_construction_group_sources.go; recipe_construction_related_eligibility.go; recipe_construction_related_expand.go; recipe_construction_related_field.go; recipe_construction_reshape.go; recipe_derived_columns.go; recipe_dynamic_columns.go; recipe_expression.go; recipe_expression_lowering.go; recipe_schema.go; recipe_table_reshape.go; required_match.go; root_projection.go; row_expansion.go; storage_route.go; traversal.go; typed_predicate.go; whole_relation_scope.go; workspace_output_dependencies.go,bound_scalar_coding_group_test.go; child_set_selector_demand_test.go; construction_group_streaming_test.go; correlated_binding_test.go; empty_bound_group_source_test.go; graph_plan_test.go; group_field_source_context_test.go; group_related_occurrence_lower_test.go; group_rows_test.go; group_specimen_inbound_coding_test.go; owner_records_test.go; physical_nested_scope_test.go; population_test.go; recipe_construction_coded_group_test.go; recipe_construction_coded_pivot_display_test.go; recipe_construction_coded_pivot_test.go; recipe_construction_combine_test.go; recipe_construction_composite_test.go; recipe_construction_related_count_test.go; recipe_construction_reshape_test.go; recipe_construction_row_values_test.go; recipe_construction_test.go; recipe_derived_columns_test.go; recipe_expression_test.go; recipe_output_test.go; recipe_schema_cardinality_test.go; recipe_semantic_path_test.go; recipe_table_reshape_test.go; recipe_unpivot_capability_test.go; root_id_filter_test.go; row_expansion_test.go; traversal_test.go; whole_relation_scope_evidence_test.go; workspace_combine_scope_evidence_test.go; workspace_output_compile_test.go,—,—,12185,10741,40,BindGroupRelatedOccurrenceSource; BuildGenericPhysicalPlanWithPolicy; BuildGraphPhysicalPlan; BuildPhysicalTraversal; CloneCompiledOutputSchema; CompileResolvedGraphPlan; CompileResolvedRecipePlan; CompiledActiveRelatedRecordStage; CompiledGroupFieldOwner; CompiledGroupFieldSourceAncestor; CompiledGroupFieldSourceContext; CompiledGroupFieldSourceExpansion; CompiledGroupFieldSourceRouteStep; CompiledOutputColumn; CompiledRecipe; CompiledRecipeOutput; CompiledRelatedExpandAnchor; CompiledRelatedExpandStage; CompiledRelatedRecordAnchor; CompiledStageDescriptor; ComposeClickHouseCombine; ConstructionCombineColumnType; DescribeConstructionSourceStage; DynamicColumnMetadata; Error; LowerCorrelatedBinding; LowerCorrelatedBindingWithVersion; LowerCorrelatedPredicate; LowerCorrelatedPredicateWithIdentity; LowerExtensionBinding; LowerRecipeExpression; PivotReducerTypeError; RelatedEligibilityAnchorError; RelatedSourceAnchorError; ResolveGroupRelatedOccurrenceSource; StageOperationCapability; TableReshapeScalarKind; TraversalLoweringRequest; TraversalLoweringResult; WorkspaceOutputSource,internal/authscope; internal/dataframe/columntransform; internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/expression; internal/dataframe/lineage; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema; internal/projectid,internal/authscope; internal/dataframe/columntransform; internal/dataframe/compiler/ir; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/aql; internal/dataframe/errors; internal/dataframe/expression; internal/dataframe/lineage; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema,internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/execution; internal/dataframe/execution/chartifact; internal/explorer/compilation; internal/explorer/lifecycle; internal/server,internal/dataframe/compiler/render/aql,—,—,legacy=3; compatibility=3,broad surface (41 production files); inspect cohesion,go test ./internal/dataframe/compiler/lower,inventory complete; semantic disposition pending +internal/dataframe/compiler/command,github.com/calypr/loom/internal/dataframe/compiler/command,command,domain/library,"Own the closed dispatch-tagged executable command types, rendered PostgreSQL plan validation, and constructor guarantees shared by compilers and row adapters.",command.go; rendered_postgres.go,compiled_command_test.go; imports_guard_test.go; rendered_postgres_test.go,—,—,433,526,15,AQLCommand; AQLParameters; Backend; CompiledBackend; CompiledCommand; Dispatch; NewAQLCommand; NewPostgreSQLCommand; NewRenderedPostgresPhysicalPlan; PostgreSQLCommand; PostgreSQLParameter; PostgreSQLParameterType; PostgreSQLParameters; RenderedPostgresPhysicalPlan; WithAQLBinds,—,internal/dataframe/compiler; internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/compiler/render/aql; internal/dataframe/recipe; internal/dataframe/semantic,internal/backends/postgres; internal/backends/postgres/internal/rows; internal/dataframe/compiler; internal/dataframe/compiler/render/postgres; internal/dataframe/execution; internal/server,conformance/compiler; internal/dataframe/compiler/command; internal/dataframe/compiler/render/aql; internal/explorer/compilation,—,—,—,no automatic consolidation signal; perform semantic review,go test ./internal/dataframe/compiler/command,inventory complete; semantic disposition pending +internal/dataframe/compiler/ir,github.com/calypr/loom/internal/dataframe/compiler/ir,ir,domain/library,"Define typed physical operations, values, scope proofs, policies, and validation.",clone.go; dataset_generation.go; diagnostics.go; doc.go; navigation.go; physical_combine.go; physical_combine_scalar.go; physical_operations.go; physical_stage_sequence.go; physical_stage_validation.go; physical_table_shape.go; physical_types.go; physical_validation.go; physical_validation_expr.go; physical_validation_refs.go; physical_validation_shapes.go; physical_values.go; policy.go; postgres_storage.go; prefix.go; scope.go; selector_helpers.go; whole_relation_scope_evidence.go; workspace_combine_scope_operand.go,group_related_field_source_validation_test.go; physical_combine_scalar_test.go; physical_combine_test.go; physical_group_pivot_lineage_validation_test.go; physical_root_page_candidate_seed_test.go; physical_scope_cycle_test.go; physical_stage_validation_test.go; population_mapping_test.go; population_root_test.go; related_coding_group_lineage_validation_test.go; row_lineage_trace_validation_test.go; subplan_test.go; unit_validation_test.go; unnest_test.go; workspace_output_validation_test.go,—,—,12283,1804,215,AddDecision; CanonicalExecutionPhysicalPlan; ClickHouseArtifactIdentity; ClickHouseCombineScalarBaseType; ClonePhysicalExpression; ClonePhysicalOperation; ClonePhysicalOperations; ClonePhysicalPlan; ClonePhysicalPredicateExpression; ClonePhysicalSubplan; Column; CompilerPlanDiagnostics; DatasetGenerationBindValue; DecomposePhysicalTraversalPrefixAt; DefaultPhysicalOptimizationPolicy; Digest; Error; EstimatePreparedSelectorWork; EstimateTraversalSharingWork; Identifier; IsRegistered; Kind; Matches; MustColumn; Name; NewPhysicalOptimizationReport; NewPostgresStorageBinding; NewPostgresTableIdentifier; NewPrivateAQLPrefixScopeOperand; NewPublishedWorkspaceScopeOperand; NewWholeRelationScopeEvidence; NewWorkspaceCombineScopeEvidence; NewWorkspaceOutputScopeOperand; NormalizeDatasetGeneration; PhysicalAggregate; PhysicalAggregateOperation; PhysicalCall; PhysicalCardinality; PhysicalCellTraceConstruction; PhysicalCellTraceConstructionInput; PhysicalCellTraceContribution; PhysicalCellTraceRelatedSource; PhysicalCellTraceReshape; PhysicalCellTraceReshapeSource; PhysicalCellTraceReshapeSourceKind; PhysicalCellTraceReturn; PhysicalClickHouseCombine; PhysicalClickHousePrefix; PhysicalCodedGroupPathSegment; PhysicalCohortRootSource; PhysicalCollectionScan; PhysicalCombineInputRef; PhysicalCombineKey; PhysicalCombineKind; PhysicalCombineOutputColumn; PhysicalCombineProjection; PhysicalConstructionStage; PhysicalContributorWindow; PhysicalCorrelation; PhysicalDerivedLet; PhysicalDocumentLookup; PhysicalEngine; PhysicalExpression; PhysicalExpressionKind; PhysicalExpressionLet; PhysicalExpressionProjection; PhysicalExtract; PhysicalFilter; PhysicalGraphReturn; PhysicalGroupMemberValue; PhysicalGroupRows; PhysicalGroupRowsCellTrace; PhysicalGroupedCodedPivotCategory; PhysicalGroupedPivot; PhysicalGroupedPivotCategory; PhysicalGroupedPivotKey; PhysicalKeySet; PhysicalKeySetLookup; PhysicalKeyedMap; PhysicalLimit; PhysicalLiteral; PhysicalMapReduction; PhysicalNullBehavior; PhysicalObject; PhysicalObjectKeys; PhysicalObjectLookup; PhysicalOperation; PhysicalOperationKind; PhysicalOptimizationDecision; PhysicalOptimizationPolicy; PhysicalOptimizationReport; PhysicalOptimizationRule; PhysicalOptimizationRuleState; PhysicalOwnerRecords; PhysicalPathExtend; PhysicalPathNode; PhysicalPathRelationship; PhysicalPathSeed; PhysicalPivotCategoryMatchKind; PhysicalPivotMap; PhysicalPlan; PhysicalPlanDiagnostics; PhysicalPopulationMappingIdentityPart; PhysicalPopulationMappingReturn; PhysicalPopulationRootSource; PhysicalPredicate; PhysicalPredicateExpression; PhysicalPredicateKind; PhysicalPrefixSourceBackend; PhysicalPreparedField; PhysicalPreparedReference; PhysicalPreparedSet; PhysicalProjection; PhysicalProjectionPresence; PhysicalRelatedField; PhysicalReturn; PhysicalRootPageCandidateSeed; PhysicalRootScan; PhysicalRowLineageReturn; PhysicalRowLineageStageMatch; PhysicalRowLineageTrace; PhysicalScopeWindowEnd; PhysicalSelectorExecutionMode; PhysicalSelectorFallback; PhysicalSelectorValueDemand; PhysicalSet; PhysicalSetKind; PhysicalSetOutput; PhysicalSetOutputField; PhysicalSetProjection; PhysicalSetProjectionField; PhysicalSetReduction; PhysicalSetReductionField; PhysicalSetReductionMode; PhysicalSlice; PhysicalSort; PhysicalSource; PhysicalStageCodedGroup; PhysicalStageCohortGroup; PhysicalStageColumn; PhysicalStageExpand; PhysicalStageGroup; PhysicalStageGroupAggregate; PhysicalStageGroupCodingKey; PhysicalStageGroupCurrentStageFieldSource; PhysicalStageGroupFieldSource; PhysicalStageGroupFieldSourceKind; PhysicalStageGroupKey; PhysicalStageGroupMissingKeyPolicy; PhysicalStageGroupRelatedOccurrenceAdapter; PhysicalStageGroupRelatedOccurrenceFieldSource; PhysicalStageGroupRelatedProjection; PhysicalStageGroupRelatedRouteStep; PhysicalStageGroupSemanticKey; PhysicalStageOperationKind; PhysicalStagePopulationMappingReturn; PhysicalStageRelatedExpand; PhysicalStageRelatedField; PhysicalStageRelatedRecordAnchor; PhysicalStageRelatedRouteStep; PhysicalStageRelatedSource; PhysicalStageRowValue; PhysicalStageSequence; PhysicalSubplan; PhysicalTableShapeExclusionReturn; PhysicalTemporalOrdering; PhysicalTraversal; PhysicalTraversalDecision; PhysicalTraversalDirection; PhysicalTraversalPrefix; PhysicalTraversalPrefixDecomposition; PhysicalTraversalPrefixError; PhysicalTraversalPrefixRejectionReason; PhysicalTraversalStrategy; PhysicalTraversalSubset; PhysicalUnitNormalization; PhysicalUnnest; PhysicalUnnestAncestor; PhysicalUnnestEmptyPolicy; PhysicalUnnestExactGroupShapeEqual; PhysicalUnnestOwner; PhysicalUnnestRouteStep; PhysicalUnnestSortKeys; PhysicalUnpivot; PhysicalUnpivotIdentityPart; PhysicalUnpivotInput; PhysicalUnpivotOutput; PhysicalValue; PostgresColumnIdentifier; PostgresLogicalTable; PostgresLogicalTableKind; PostgresStorageBinding; PostgresTableIdentifier; RegisteredPostgresColumnNames; ResolvedClickHouseColumn; ResolvedClickHousePrivateArtifact; ResolvedClickHouseTable; RichConsumerGroup; RichSourceReuse; RuleEnabled; SQLText; Schema; Table; TableKind; TotalConsumers; Validate; ValidateForWorkspaceCompilation; ValidateGenericPhysicalPlanScope; ValidateScope; ValidateWithPrivateStage; ValidateWithWorkspaceArtifacts; WholeRelationScopeEvidence; WholeRelationScopeIdentity; WithRule; WorkspaceCombineScopeOperand,internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema; internal/projectid,internal/dataframe/spec; internal/dataframe/unit,cmd/dataframe-profile; cmd/preview-covering-index-artifact; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/clickhouse; internal/dataframe/compiler/render/postgres; internal/dataframe/execution; internal/explorer/lifecycle; internal/server,conformance/compiler; internal/backends/postgres; internal/dataframe/compiler/command; internal/explorer/compilation; internal/store/clickhouse,—,—,legacy=9; migration=7; compatibility=4,broad surface (24 production files); inspect cohesion,go test ./internal/dataframe/compiler/ir,inventory complete; semantic disposition pending +internal/dataframe/compiler/lower,github.com/calypr/loom/internal/dataframe/compiler/lower,lower,domain/library,Lower logical FHIR dataframe plans into storage-aware physical IR.,auth_scope.go; child_sets.go; column_naming.go; correlated_binding.go; doc.go; filter_literal.go; graph_plan.go; group_field_source_context.go; group_field_source_related.go; group_related_occurrence_lower.go; group_rows.go; helpers.go; plan_orchestration.go; population.go; recipe_column_transformations.go; recipe_compile.go; recipe_construction.go; recipe_construction_coded_group.go; recipe_construction_coded_pivot.go; recipe_construction_combine.go; recipe_construction_combine_metadata.go; recipe_construction_composite.go; recipe_construction_group_sources.go; recipe_construction_related_eligibility.go; recipe_construction_related_expand.go; recipe_construction_related_field.go; recipe_construction_reshape.go; recipe_derived_columns.go; recipe_dynamic_columns.go; recipe_expression.go; recipe_expression_lowering.go; recipe_schema.go; recipe_table_reshape.go; required_match.go; root_projection.go; row_expansion.go; storage_route.go; traversal.go; typed_predicate.go; whole_relation_scope.go; workspace_output_dependencies.go,bound_scalar_coding_group_test.go; child_set_selector_demand_test.go; construction_group_streaming_test.go; correlated_binding_test.go; empty_bound_group_source_test.go; graph_plan_test.go; group_field_source_context_test.go; group_related_occurrence_lower_test.go; group_rows_test.go; group_specimen_inbound_coding_test.go; owner_records_test.go; physical_nested_scope_test.go; population_test.go; recipe_construction_coded_group_test.go; recipe_construction_coded_pivot_display_test.go; recipe_construction_coded_pivot_test.go; recipe_construction_combine_test.go; recipe_construction_composite_test.go; recipe_construction_related_count_test.go; recipe_construction_reshape_test.go; recipe_construction_row_values_test.go; recipe_construction_test.go; recipe_derived_columns_test.go; recipe_expression_test.go; recipe_output_test.go; recipe_schema_cardinality_test.go; recipe_semantic_path_test.go; recipe_table_reshape_test.go; recipe_unpivot_capability_test.go; root_id_filter_test.go; row_expansion_test.go; traversal_test.go; whole_relation_scope_evidence_test.go; workspace_combine_scope_evidence_test.go; workspace_output_compile_test.go,—,—,12219,10802,40,BindGroupRelatedOccurrenceSource; BuildGenericPhysicalPlanWithPolicy; BuildGraphPhysicalPlan; BuildPhysicalTraversal; CloneCompiledOutputSchema; CompileResolvedGraphPlan; CompileResolvedRecipePlan; CompiledActiveRelatedRecordStage; CompiledGroupFieldOwner; CompiledGroupFieldSourceAncestor; CompiledGroupFieldSourceContext; CompiledGroupFieldSourceExpansion; CompiledGroupFieldSourceRouteStep; CompiledOutputColumn; CompiledRecipe; CompiledRecipeOutput; CompiledRelatedExpandAnchor; CompiledRelatedExpandStage; CompiledRelatedRecordAnchor; CompiledStageDescriptor; ComposeClickHouseCombine; ConstructionCombineColumnType; DescribeConstructionSourceStage; DynamicColumnMetadata; Error; LowerCorrelatedBinding; LowerCorrelatedBindingWithVersion; LowerCorrelatedPredicate; LowerCorrelatedPredicateWithIdentity; LowerExtensionBinding; LowerRecipeExpression; PivotReducerTypeError; RelatedEligibilityAnchorError; RelatedSourceAnchorError; ResolveGroupRelatedOccurrenceSource; StageOperationCapability; TableReshapeScalarKind; TraversalLoweringRequest; TraversalLoweringResult; WorkspaceOutputSource,internal/authscope; internal/dataframe/columntransform; internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/expression; internal/dataframe/lineage; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema; internal/projectid,internal/authscope; internal/dataframe/columntransform; internal/dataframe/compiler/ir; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/aql; internal/dataframe/errors; internal/dataframe/expression; internal/dataframe/lineage; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema,internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/execution; internal/dataframe/execution/chartifact; internal/explorer/compilation; internal/explorer/lifecycle; internal/server,internal/backends/postgres; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/postgres,—,—,legacy=3; compatibility=3,broad surface (41 production files); inspect cohesion,go test ./internal/dataframe/compiler/lower,inventory complete; semantic disposition pending internal/dataframe/compiler/optimize,github.com/calypr/loom/internal/dataframe/compiler/optimize,optimize,domain/library,Apply explainable semantics-preserving rewrites to validated physical IR.,doc.go; optimize.go,optimize_test.go,—,—,438,124,1,OptimizePhysicalPlanWithPolicy,internal/dataframe/compiler/ir,internal/dataframe/compiler/ir,internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/execution,internal/dataframe/compiler/lower,—,—,—,no automatic consolidation signal; perform semantic review,go test ./internal/dataframe/compiler/optimize,inventory complete; semantic disposition pending -internal/dataframe/compiler/render/aql,github.com/calypr/loom/internal/dataframe/compiler/render/aql,aql,domain/library,Render validated physical IR as deterministic parameterized AQL.,calls.go; cell_trace.go; collections.go; combine_prefix.go; construction_coded_group.go; construction_expand.go; construction_group.go; construction_related_expand.go; correlation_value_scope.go; doc.go; graph_render.go; group_row_value_preview.go; group_rows.go; navigation_layout.go; navigation_render.go; owner_records.go; population_mapping.go; predicates.go; related_count.go; related_group_preview.go; render.go; row_lineage.go; row_lineage_coded_group.go; row_lineage_composed.go; row_lineage_group_pivot.go; row_lineage_group_rows.go; row_lineage_pivot.go; row_lineage_related_coding.go; row_values.go; selector.go; selectors.go; source_group_preview.go; stage_sequence.go; table_reshape.go; table_reshape_dynamic.go; table_reshape_streaming.go; table_shape_exclusion.go; validation.go; values.go,call_test.go; construction_group_related_occurrence_test.go; construction_group_test.go; construction_related_expand_test.go; correlated_selection_test.go; graph_test.go; group_row_value_preview_test.go; navigation_layout_test.go; related_count_category_test.go; row_lineage_composed_test.go; selectors_test.go; source_group_preview_test.go; stage_sequence_test.go; table_reshape_dynamic_test.go; table_reshape_streaming_scalar_test.go; two_scan_preview_test.go; unnest_test.go,—,—,14514,3444,22,RenderClickHousePrefix; RenderPhysicalCohortGroupRowsLineage; RenderPhysicalGroupRowsLineage; RenderPhysicalPlan; RenderPhysicalPlanWithCategoryScanPresenceMarker; RenderPhysicalPlanWithDynamicCategoryPivotPreview; RenderPhysicalPlanWithRelatedExpandPreviewLimit; RenderPhysicalPlanWithRootIndexDisabled; RenderPhysicalPlanWithRootIndexHint; RenderPhysicalPlanWithRootPageKeys; RenderPhysicalPlanWithSemanticGroupProjectionAssessment; RenderPhysicalPlanWithSemanticGroupProjectionFactorization; RenderPhysicalPlanWithStreamingMissingCategoryPivotPreview; RenderPhysicalPlanWithStreamingMissingCategoryPivotPreviewAndRootIndexHint; RenderPhysicalPlanWithTerminalProjection; RenderPhysicalPlanWithTwoScanPivotPreview; RenderPhysicalPlanWithUnorderedTerminalProjection; RenderPhysicalPlanWithUnorderedTerminalProjectionPresenceMarker; RenderPhysicalRootScopeCount; RenderRelatedEligibilityCategoryOverflowWitness; RenderedPhysicalPlan; ValidateSemanticGroupProjectionPreviewPlan,internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema,internal/authscope; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/errors; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema,internal/dataframe/compiler; internal/dataframe/compiler/capability,internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/execution; internal/explorer/compilation,—,—,—,broad surface (39 production files); inspect cohesion,go test ./internal/dataframe/compiler/render/aql,inventory complete; semantic disposition pending -internal/dataframe/compiler/render/clickhouse,github.com/calypr/loom/internal/dataframe/compiler/render/clickhouse,clickhouse,storage adapter,Render typed Combine plans and resolved published or private table inputs as deterministic ClickHouse SQL.,combine.go; workspace_combine.go,combine_test.go; membership_nullable_integration_test.go; workspace_combine_test.go,—,—,930,1092,6,RenderCombine; RenderCombineWithLimit; RenderCompositeClickHouseCombine; RenderWorkspaceClickHouseCombine; RenderedCombine; WorkspaceArtifact,internal/dataframe/compiler/ir,internal/dataframe/compiler/ir; internal/store/clickhouse,internal/dataframe/execution,internal/store/clickhouse,—,—,—,one production importer (internal/dataframe/execution); inspect ownership/inline potential; storage adapter; preserve interface and persisted-data contracts,go test ./internal/dataframe/compiler/render/clickhouse,inventory complete; semantic disposition pending -internal/dataframe/errors,github.com/calypr/loom/internal/dataframe/errors,dataframeerrors,domain/library,Define and normalize stable structured dataframe errors for transport presentation.,errors.go,errors_test.go,—,—,488,172,21,AsUserError; Code; Details; Error; ErrorCode; ErrorOption; FieldPath; IsFeatureResolutionCode; IsRetryableCode; NewError; Normalize; PublicMessage; Retryable; SanitizePersistedFailure; Unwrap; UserError; WithCause; WithDetails; WithFieldPath; WithRetryable; Wrap,—,—,internal/api/authpolicy; internal/api/bulk/load; internal/api/graphql/graph; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/api/http; internal/api/recipe; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/execution; internal/dataframe/publication; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/dataframe/recipe/exec/arango; internal/dataframe/semantic; internal/dataframe/spec; internal/explorer/lifecycle; internal/server,internal/dataframe/publication/arango,—,1 exported interface(s),legacy=1,small shared semantic contract; compare direct callers before inlining; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/errors,inventory complete; semantic disposition pending -internal/dataframe/execution,github.com/calypr/loom/internal/dataframe/execution,execution,domain/library,"Prepare, authorize, execute, explain, and post-process compiled dataframe requests.",category_scan.go; cell_trace.go; clickhouse_artifact.go; control.go; control_explain_types.go; control_types.go; engine.go; execution_service.go; explain.go; group_preview_cache.go; group_preview_scan.go; helpers.go; named_uuid.go; population_mapping.go; postquery.go; related_preview_paging.go; row_lineage.go; service.go; table_shape_exclusions.go; types.go,auth_scope_test.go; category_scan_test.go; cell_trace_row_not_found_class_test.go; cell_trace_test.go; clickhouse_artifact_test.go; cohort_preview_paging_arango_test.go; construction_cell_trace_arango_test.go; control_validate_digest_test.go; engine_blank_project_review_test.go; engine_clickhouse_test.go; engine_run_output_class_test.go; engine_test.go; engine_workspace_artifact_test.go; execution_test.go; explain_test.go; group_preview_cache_test.go; group_preview_scan_test.go; helpers_test.go; identity_test.go; named_uuid_test.go; population_group_preview_test.go; population_mapping_test.go; postquery_test.go; row_lineage_test.go; table_shape_exclusions_test.go,—,—,5303,7038,92,AssessmentFromArango; CategoryScanRequest; CategoryScanResult; CategoryValue; CellTrace; CellTraceCompiled; CellTraceContribution; CellTraceRequest; CellTraceResult; CellTraceStatus; ClickHouseQueryRows; CompileResolvedBundle; CompiledQuery; CompiledQueryFingerprint; CompilerPlanDiagnostics; Config; Control; DynamicColumnCheck; DynamicDriftError; Engine; Error; Explain; ExplainAssessment; ExplainCollectionScan; ExplainIndexLocation; ExplainIndexSummary; ExplainPhysical; ExplainPlanEstimate; ExplainWarning; Materialize; MaterializeBundle; MaterializeClickHouseArtifact; MaterializeResolvedBundle; MaterializeVersion; New; NewService; OutputRows; OutputStream; PhysicalExplanation; PhysicalOutputExplanation; PopulationMapping; PopulationMappingCompiled; PopulationMappingDiagnostic; PopulationMappingRequest; PopulationMappingResult; PopulationMappingStatus; PopulationMemberReader; PopulationMemberReaderFunc; Preview; PreviewBundle; PreviewOutput; PreviewRequest; PreviewResolvedBundle; PreviewSummary; PrivateClickHouseArtifactInput; PrivateClickHouseArtifactSource; QueryDiagnostics; QueryRows; ResolutionError; Resolve; ResolveBundle; ResolveClickHouseInputs; ResolveVersion; Resolved; Result; RowIdentity; RowLineage; RowLineageContributor; RowLineageRequest; RowLineageResult; Run; RunCompiled; ScanCategories; ScanCategoriesCompiled; Service; ServiceConfig; Stream; StreamResult; Streams; TableShapeExclusion; TableShapeExclusionRequest; TableShapeExclusionResult; TableShapeExclusionStatus; TableShapeExclusions; TableShapeExclusionsCompiled; TableShapeSourceIdentity; Unwrap; Validate; Validation; VisitMembers; WithExecutionReadPins; WithPrivateClickHouseArtifacts,internal/authscope; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/clickhouse; internal/dataframe/errors; internal/dataframe/execution/chartifact; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataframe/spec; internal/projectid; internal/store/arango,internal/authscope; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/errors; internal/dataframe/execution/chartifact; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataframe/spec; internal/projectid; internal/store/arango; internal/store/clickhouse,internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/explorer/lifecycle; internal/server,conformance/compiler; internal/api/graphql/graph; internal/dataframe/compiler,github.com/google/uuid,1 exported interface(s),legacy=4; deprecated=1; compatibility=22,broad surface (20 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/execution,inventory complete; semantic disposition pending -internal/dataframe/execution/chartifact,github.com/calypr/loom/internal/dataframe/execution/chartifact,chartifact,domain/library,Manage private ClickHouse staging tables and leases used to cross execution boundaries without publishing outputs.,artifact.go,artifact_test.go,—,—,1038,490,24,Abort; Artifact; Begin; Catalog; CheckLease; ClickHouse; Column; ColumnsFromCompiledOutput; Config; Finalize; Identity; LeaseContext; Manager; Manifest; New; NormalizeSchema; Progress; Reconcile; Release; SchemaDigest; ScopeDigest; State; Write; Writer,internal/authscope; internal/dataframe/compiler/lower; internal/dataframe/expression; internal/store/clickhouse,internal/authscope; internal/dataframe/compiler/lower; internal/dataframe/expression; internal/store/clickhouse,internal/dataframe/execution; internal/dataframe/execution/chartifact/arango; internal/server,—,github.com/google/uuid,2 exported interface(s),legacy=1,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/execution/chartifact,inventory complete; semantic disposition pending -internal/dataframe/execution/chartifact/arango,github.com/calypr/loom/internal/dataframe/execution/chartifact/arango,arango,storage adapter,Persist private ClickHouse artifact manifests and leases in a dedicated Arango collection.,catalog.go,catalog_arango_test.go; catalog_test.go,—,—,379,723,10,BootstrapSpec; Catalog; ClaimCleanup; Create; Delete; ListExpired; NewCatalog; ReleaseCleanup; Renew; Update,internal/dataframe/execution/chartifact; internal/store/arango,internal/authscope; internal/dataframe/execution/chartifact; internal/store/arango; internal/store/clickhouse,internal/server,—,—,—,—,one production importer (internal/server); inspect ownership/inline potential; storage adapter; preserve interface and persisted-data contracts,go test ./internal/dataframe/execution/chartifact/arango,inventory complete; semantic disposition pending +internal/dataframe/compiler/render/aql,github.com/calypr/loom/internal/dataframe/compiler/render/aql,aql,domain/library,Render validated physical IR as deterministic parameterized AQL.,calls.go; cell_trace.go; collections.go; combine_prefix.go; construction_coded_group.go; construction_expand.go; construction_group.go; construction_related_expand.go; correlation_value_scope.go; doc.go; graph_render.go; group_row_value_preview.go; group_rows.go; navigation_layout.go; navigation_render.go; owner_records.go; population_mapping.go; predicates.go; related_count.go; related_group_preview.go; render.go; row_lineage.go; row_lineage_coded_group.go; row_lineage_composed.go; row_lineage_group_pivot.go; row_lineage_group_rows.go; row_lineage_pivot.go; row_lineage_related_coding.go; row_values.go; selector.go; selectors.go; source_group_preview.go; stage_sequence.go; table_reshape.go; table_reshape_dynamic.go; table_reshape_streaming.go; table_shape_exclusion.go; validation.go; values.go,call_test.go; command_helpers_ext_test.go; construction_group_related_occurrence_test.go; construction_group_test.go; construction_related_expand_test.go; correlated_selection_test.go; graph_test.go; group_row_value_preview_test.go; navigation_layout_test.go; recipe_unpivot_identity_test.go; related_count_category_test.go; row_lineage_composed_test.go; selectors_test.go; source_group_preview_test.go; stage_sequence_test.go; table_reshape_dynamic_test.go; table_reshape_streaming_scalar_test.go; two_scan_preview_test.go; unnest_test.go,—,—,14519,3608,22,RenderClickHousePrefix; RenderPhysicalCohortGroupRowsLineage; RenderPhysicalGroupRowsLineage; RenderPhysicalPlan; RenderPhysicalPlanWithCategoryScanPresenceMarker; RenderPhysicalPlanWithDynamicCategoryPivotPreview; RenderPhysicalPlanWithRelatedExpandPreviewLimit; RenderPhysicalPlanWithRootIndexDisabled; RenderPhysicalPlanWithRootIndexHint; RenderPhysicalPlanWithRootPageKeys; RenderPhysicalPlanWithSemanticGroupProjectionAssessment; RenderPhysicalPlanWithSemanticGroupProjectionFactorization; RenderPhysicalPlanWithStreamingMissingCategoryPivotPreview; RenderPhysicalPlanWithStreamingMissingCategoryPivotPreviewAndRootIndexHint; RenderPhysicalPlanWithTerminalProjection; RenderPhysicalPlanWithTwoScanPivotPreview; RenderPhysicalPlanWithUnorderedTerminalProjection; RenderPhysicalPlanWithUnorderedTerminalProjectionPresenceMarker; RenderPhysicalRootScopeCount; RenderRelatedEligibilityCategoryOverflowWitness; RenderedPhysicalPlan; ValidateSemanticGroupProjectionPreviewPlan,internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema,internal/authscope; internal/dataframe/compiler; internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/errors; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema,internal/dataframe/compiler; internal/dataframe/compiler/capability,internal/dataframe/compiler/command; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/postgres; internal/dataframe/execution; internal/explorer/compilation,—,—,—,broad surface (39 production files); inspect cohesion,go test ./internal/dataframe/compiler/render/aql,inventory complete; semantic disposition pending +internal/dataframe/compiler/render/clickhouse,github.com/calypr/loom/internal/dataframe/compiler/render/clickhouse,clickhouse,storage adapter,Render typed Combine plans and resolved published or private table inputs as deterministic ClickHouse SQL.,combine.go; workspace_combine.go,combine_test.go; membership_nullable_integration_test.go; workspace_combine_test.go,—,—,930,1093,6,RenderCombine; RenderCombineWithLimit; RenderCompositeClickHouseCombine; RenderWorkspaceClickHouseCombine; RenderedCombine; WorkspaceArtifact,internal/dataframe/compiler/ir,internal/dataframe/compiler/ir; internal/store/clickhouse,internal/dataframe/execution,internal/store/clickhouse,—,—,—,one production importer (internal/dataframe/execution); inspect ownership/inline potential; storage adapter; preserve interface and persisted-data contracts,go test ./internal/dataframe/compiler/render/clickhouse,inventory complete; semantic disposition pending +internal/dataframe/compiler/render/postgres,github.com/calypr/loom/internal/dataframe/compiler/render/postgres,postgres,storage adapter,"Render the supported physical IR read spine and typed selector extractions with two-valued JSONB path-presence evidence onto allowlisted PostgreSQL storage bindings as deterministic parameterized SQL with ordered typed arguments, failing closed on every unsupported plan shape.",calls.go; category_scan.go; cell_trace.go; coded_group.go; graph.go; group.go; null_semantics.go; objects.go; pivot.go; population.go; population_mapping.go; predicates.go; render.go; root_scan.go; row_lineage.go; scope.go; scope_ops.go; selectors.go; sets.go; stage_related_occurrence.go; stage_semantic.go; stage_sequence.go; table_shape_exclusions.go; traversal.go; unit_normalization.go; unnest.go; unpivot.go; window.go,calls_live_test.go; calls_test.go; category_scan_test.go; cell_trace_test.go; coded_group_test.go; coded_pivot_test.go; graph_test.go; group_test.go; main_test.go; null_semantics_test.go; obj11_production_test.go; objects_test.go; pivot14_noncoded_test.go; pivot_test.go; population_live_test.go; population_mapping_test.go; population_test.go; predicates_pg18_test.go; predicates_test.go; render_pg18_test.go; render_test.go; root_scan_test.go; row_lineage_test.go; scope_ops_witness_test.go; scope_test.go; selectors_test.go; sets_live_test.go; sets_test.go; stage_related_occurrence_p4_test.go; stage_related_occurrence_production_test.go; stage_related_occurrence_test.go; stage_semantic_production_test.go; stage_semantic_test.go; stage_sequence_production_test.go; stage_sequence_test.go; table_shape_exclusions_live_test.go; table_shape_exclusions_test.go; traversal_test.go; unit_normalization_production_test.go; unit_normalization_test.go; unit_normalization_tie_test.go; unnest_test.go; unpivot_test.go; window_test.go,—,—,23444,42018,18,NewPostgresPopulationRootBinding; PostgresPopulationRootBinding; RenderCategoryScan; RenderGraphPhysicalPlan; RenderGroupedCodedPivot; RenderGroupedPivot; RenderPhysicalPlan; RenderPopulationMappingPlan; RenderPopulationRootScan; RenderRootScan; RenderSelectorExtraction; RenderTableShapeExclusions; RenderUnnest; RenderUnpivot; RenderedCategoryScan; Validate; ValidatePopulationMappingSource; ValidatePopulationRootSource,internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema,internal/backends/postgres/control; internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec; internal/dataframe/unit; internal/explorer; internal/fhir/schema; internal/source,internal/dataframe/compiler; internal/dataframe/execution,—,—,—,legacy=13; migration=20; compatibility=2,broad surface (28 production files); inspect cohesion; storage adapter; preserve interface and persisted-data contracts,go test ./internal/dataframe/compiler/render/postgres,inventory complete; semantic disposition pending +internal/dataframe/errors,github.com/calypr/loom/internal/dataframe/errors,dataframeerrors,domain/library,Define and normalize stable structured dataframe errors for transport presentation.,errors.go,errors_test.go,—,—,488,172,21,AsUserError; Code; Details; Error; ErrorCode; ErrorOption; FieldPath; IsFeatureResolutionCode; IsRetryableCode; NewError; Normalize; PublicMessage; Retryable; SanitizePersistedFailure; Unwrap; UserError; WithCause; WithDetails; WithFieldPath; WithRetryable; Wrap,—,—,internal/api/authpolicy; internal/api/bulk/load; internal/api/graphql/graph; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/api/http; internal/api/recipe; internal/backends/arango/internal/recipeexec; internal/backends/arango/internal/store; internal/backends/postgres/internal/publication; internal/backends/postgres/internal/recipes; internal/backends/postgres/internal/store; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/execution; internal/dataframe/publication; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/dataframe/semantic; internal/dataframe/spec; internal/explorer/lifecycle; internal/server,internal/backends/arango/internal/publication; internal/backends/postgres; internal/backends/postgres/internal/rows; internal/dataframe/compiler,—,1 exported interface(s),legacy=1,small shared semantic contract; compare direct callers before inlining; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/errors,inventory complete; semantic disposition pending +internal/dataframe/execution,github.com/calypr/loom/internal/dataframe/execution,execution,domain/library,"Prepare, authorize, execute, explain, and post-process compiled dataframe requests.",category_scan.go; cell_trace.go; clickhouse_artifact.go; control.go; control_explain_types.go; control_types.go; engine.go; execution_service.go; explain.go; explain_postgres.go; group_preview_cache.go; group_preview_scan.go; helpers.go; named_uuid.go; population_mapping.go; postquery.go; preview_capabilities.go; query_budget.go; related_preview_paging.go; row_lineage.go; service.go; table_shape_exclusions.go; types.go,auth_scope_test.go; boundary_helpers_test.go; category_scan_postgres_test.go; category_scan_test.go; cell_trace_postgres_test.go; cell_trace_row_not_found_class_test.go; cell_trace_test.go; clickhouse_artifact_test.go; cohort_preview_paging_arango_test.go; command_boundary_contract_test.go; construction_cell_trace_arango_test.go; control_validate_digest_test.go; engine_blank_project_review_test.go; engine_clickhouse_test.go; engine_postgres_capture_pg18_test.go; engine_postgres_capture_test.go; engine_postgres_category_pg18_test.go; engine_postgres_cell_trace_pg18_test.go; engine_postgres_exclusions_pg18_test.go; engine_postgres_postquery_pg18_test.go; engine_postgres_preview_pg18_test.go; engine_postgres_test.go; engine_run_output_class_test.go; engine_test.go; engine_workspace_artifact_test.go; execution_test.go; explain_test.go; group_preview_cache_test.go; group_preview_scan_test.go; helpers_test.go; identity_test.go; named_uuid_test.go; population_group_preview_test.go; population_mapping_postgres_pg18_test.go; population_mapping_test.go; postgres_explain_parser_internal_test.go; postgres_target_fixture_test.go; postquery_test.go; preview_capability_boundary_test.go; recipe_paging_postgres_adapter_test.go; recipe_paging_postgres_test.go; recipe_preview_paging_postgres_internal_test.go; row_lineage_postgres_pg18_test.go; row_lineage_test.go; selected_source_contract_test.go; table_shape_exclusions_test.go; vendor_import_guard_test.go; verify_arch01_pins_test.go,—,—,7088,27056,109,BuildPostgreSQLExplainCommand; CategoryScanRequest; CategoryScanResult; CategoryValue; CellTrace; CellTraceCompiled; CellTraceContribution; CellTraceRequest; CellTraceResult; CellTraceStatus; ClickHouseQueryRows; CombinePortsConfigured; CompileResolvedBundle; CompiledQueryFingerprint; CompilerPlanDiagnostics; Config; Control; DynamicColumnCheck; DynamicDriftError; Engine; Error; Explain; ExplainAssessment; ExplainCollectionScan; ExplainIndexLocation; ExplainIndexSummary; ExplainPhysical; ExplainPlanEstimate; ExplainResolved; ExplainWarning; Materialize; MaterializeBundle; MaterializeClickHouseArtifact; MaterializeResolvedBundle; MaterializeVersion; New; NewService; OutputRows; OutputStream; PhysicalExplanation; PhysicalOutputExplanation; PopulationMapping; PopulationMappingCompiled; PopulationMappingDiagnostic; PopulationMappingRequest; PopulationMappingResult; PopulationMappingStatus; PopulationMemberReader; PopulationMemberReaderFunc; PostgreSQLExplainPlan; PostgreSQLPlanAssessment; PostgreSQLPlanEstimate; PostgreSQLPlanJSONExecutor; PostgreSQLPlanNode; PostgreSQLRootPageSteps; PostgreSQLScanEvidence; Preview; PreviewBundle; PreviewCollectionIndex; PreviewCollectionScanEvidence; PreviewCollectionScans; PreviewIndexCoverage; PreviewIndexNodeEvidence; PreviewOutput; PreviewRequest; PreviewResolvedBundle; PreviewSummary; PrivateClickHouseArtifactInput; PrivateClickHouseArtifactSource; QueryDiagnostics; QueryMemoryBudgetFromContext; ResolutionError; Resolve; ResolveBundle; ResolveClickHouseInputs; ResolveVersion; Resolved; Result; RowIdentity; RowLineage; RowLineageContributor; RowLineageRequest; RowLineageResult; RowVisitor; RowsRequest; Run; RunCompiled; ScanCategories; ScanCategoriesCompiled; Service; ServiceConfig; SourceRows; Stream; StreamResult; Streams; TableShapeExclusion; TableShapeExclusionRequest; TableShapeExclusionResult; TableShapeExclusionStatus; TableShapeExclusions; TableShapeExclusionsCompiled; TableShapeSourceIdentity; Unwrap; Validate; Validation; VisitMembers; WithExecutionReadPins; WithPrivateClickHouseArtifacts; WithQueryMemoryBudget,internal/authscope; internal/dataframe/compiler; internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/clickhouse; internal/dataframe/compiler/render/postgres; internal/dataframe/errors; internal/dataframe/execution/chartifact; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataframe/spec; internal/projectid,internal/authscope; internal/backends/arango; internal/backends/postgres; internal/backends/postgres/control; internal/dataframe/compiler; internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/postgres; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/execution/chartifact; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataframe/spec; internal/explorer; internal/explorer/capability; internal/explorer/lifecycle; internal/projectid; internal/source; internal/store/clickhouse,internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/backends/postgres; internal/backends/postgres/internal/rows; internal/explorer/lifecycle; internal/server,internal/api/graphql/graph; internal/dataframe/compiler; internal/dataframe/execution,github.com/google/uuid,1 exported interface(s),legacy=4; deprecated=1; compatibility=22,broad surface (23 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/execution,inventory complete; semantic disposition pending +internal/dataframe/execution/chartifact,github.com/calypr/loom/internal/dataframe/execution/chartifact,chartifact,domain/library,Manage private ClickHouse staging tables and leases used to cross execution boundaries without publishing outputs.,artifact.go,artifact_test.go,—,—,1038,490,24,Abort; Artifact; Begin; Catalog; CheckLease; ClickHouse; Column; ColumnsFromCompiledOutput; Config; Finalize; Identity; LeaseContext; Manager; Manifest; New; NormalizeSchema; Progress; Reconcile; Release; SchemaDigest; ScopeDigest; State; Write; Writer,internal/authscope; internal/dataframe/compiler/lower; internal/dataframe/expression; internal/store/clickhouse,internal/authscope; internal/dataframe/compiler/lower; internal/dataframe/expression; internal/store/clickhouse,internal/backends/arango/internal/chartifact; internal/backends/postgres/internal/chartifact; internal/dataframe/execution; internal/server,internal/backends/postgres/internal/store,github.com/google/uuid,2 exported interface(s),legacy=1,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/execution/chartifact,inventory complete; semantic disposition pending internal/dataframe/expression,github.com/calypr/loom/internal/dataframe/expression,expression,domain/library,Define and validate the typed backend-neutral expression AST used by recipes.,expression_checking.go; expression_types.go; recipe_adapter.go,expression_test.go; recipe_adapter_test.go,—,—,856,235,26,AllowsMany; Call; Cardinality; Check; CheckedExpression; Constant; Document; DocumentRef; Expression; FromRecipe; FromRecipeInContexts; Function; Literal; NodeKind; NullBehavior; NullType; Optional; Select; SelectorRef; SelectorResolver; String; Type; TypeContext; Valid; Validate; ValueKind,internal/dataframe/recipe,internal/dataframe/recipe,internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/execution/chartifact; internal/dataframe/semantic; internal/explorer; internal/explorer/lifecycle; internal/server,internal/dataframe/execution,—,—,compatibility=3,no automatic consolidation signal; perform semantic review,go test ./internal/dataframe/expression,inventory complete; semantic disposition pending internal/dataframe/lineage,github.com/calypr/loom/internal/dataframe/lineage,lineage,domain/library,Define stable identities and provenance for columns expanded from one authored source slot.,source_child.go,source_child_test.go,—,—,163,77,7,CloneSourceChild; Coordinate; IsSourceChildID; ParseSourceChildID; SourceChild; SourceChildKind; StableSourceChildID,—,—,internal/dataframe/compiler/lower; internal/dataframe/recipe; internal/explorer/authoringv2; internal/explorer/compilation,internal/server,—,—,—,no automatic consolidation signal; perform semantic review,go test ./internal/dataframe/lineage,inventory complete; semantic disposition pending -internal/dataframe/publication,github.com/calypr/loom/internal/dataframe/publication,publication,domain/library,"Own backend-neutral dataframe publication contracts, bundles, and publication orchestration.",bundle.go; flat_columns.go; quality.go; runner.go; types.go,bundle_test.go; flat_columns_test.go; pointer_name_encoding_test.go; row_smuggling_prevention_test.go,—,—,1353,385,55,BundleCatalog; BundleExecution; BundleExecutionPageFunc; BundleIdentity; BundleOutputRecord; BundlePointer; BundleState; Canonical; CanonicalizeLegacy; CloneQualityReports; ColumnProvenance; ColumnQuality; DataframeSelector; Error; ExactExecutionCatalog; ExecutionReadPinCatalog; FinalSchemaDigest; FlatColumnName; Key; KeyIntegrity; Limits; LogicalColumn; OutputSchema; OutputStream; PagedBundleCatalog; PhaseError; PhysicalColumn; PointerName; PublicationIdentity; Publish; PublishedOutput; QualifyFlatRow; QualityCompleteness; QualityFailedError; QualityIncompleteError; QualityIssues; QualityLimits; QualityOmission; QualityPolicy; QualityReport; QualityVerdict; Queryable; RelatedSourcePopulation; Result; Selector; SourceRowMetadata; SourceRowMetadataWriter; Successful; Target; Transaction; Unwrap; Valid; ValidateQualityReportBindings; ValidateQualityReports; WithPhase,internal/dataframe/errors; internal/dataset,—,internal/acceptance; internal/api/bulk/load; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/resolver; internal/dataframe/publication/arango; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/explorer; internal/explorer/arango; internal/explorer/lifecycle; internal/server,internal/api/graphql/graph; internal/dataframe/recipe,—,7 exported interface(s),legacy=1; migration=1; compatibility=2,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/publication,inventory complete; semantic disposition pending -internal/dataframe/publication/arango,github.com/calypr/loom/internal/dataframe/publication/arango,arango,storage adapter,Persist dataframe publication registry and bundle metadata in ArangoDB.,bundle_registry.go; registry.go,bundle_registry_test.go; dangling_binding_e2e_test.go; execution_read_pin_integration_test.go; name_latest_binding_e2e_test.go; publish_atomicity_e2e_test.go; release_dangle_reconcile_e2e_test.go,—,—,533,2990,22,AcquireBundleLease; AcquireExecutionReadPin; BootstrapSpec; ClaimExecutionCleanup; Client; CompareAndSwapPointer; FindExecutionByKey; FindExecutionBySelector; GetExecution; GetPointer; ListExecutions; New; PublishExecution; Registry; ReleaseBundleLease; ReleaseExecutionCleanup; ReleaseExecutionReadPin; RenewBundleLease; RenewExecutionCleanup; RenewExecutionReadPin; SaveExecution; VisitExecutionPages,internal/dataframe/publication; internal/store/arango,internal/dataframe/errors; internal/dataframe/publication; internal/dataframe/publication/arango; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/dataset; internal/dataset/arango; internal/store/arango; internal/store/clickhouse,internal/acceptance; internal/explorer/arango; internal/server,internal/dataframe/publication/arango; internal/dataframe/publication/clickhouse,github.com/arangodb/go-driver/v2/arangodb/shared,—,—,storage adapter; preserve interface and persisted-data contracts,go test ./internal/dataframe/publication/arango,inventory complete; semantic disposition pending -internal/dataframe/publication/clickhouse,github.com/calypr/loom/internal/dataframe/publication/clickhouse,clickhouse,storage adapter,Materialize and inspect published dataframe bundles in ClickHouse.,bundle_store.go,abort_ackloss_e2e_test.go; bundle_store_test.go; export_pin_crash_window_e2e_test.go; orphaned_published_growth_e2e_test.go; pinblock_reconcile_e2e_test.go; pointer_name_nul_ambiguity_e2e_test.go; publish_partial_pointer_e2e_test.go; reconcile_toctou_e2e_test.go; reconcile_toctou_test.go; release_dangle_reconcile_e2e_test.go; unpinned_page_read_e2e_test.go,—,—,1223,6524,21,Abort; ActiveReleaseReader; Begin; BundleClickHouseStore; ClickHouseBundleStore; Commit; CreateOutput; ExistingPublishedOutputs; ExistingQualityReports; FinalizeSchema; Idempotent; InsertRows; NewBundleStore; Reconcile; SetActiveReleaseReader; SetFinalSchemaDigest; SetOutputMetadata; SetQualityReports; SetSourceRowMetadata; SupportsObjectValues; WriteBatch,internal/dataframe/errors; internal/dataframe/publication; internal/dataset; internal/store/clickhouse,internal/dataframe/publication; internal/dataframe/publication/arango; internal/dataframe/published; internal/dataset; internal/dataset/arango; internal/store/arango; internal/store/clickhouse,internal/server,internal/dataframe/publication/arango,github.com/google/uuid,2 exported interface(s),legacy=1,one production importer (internal/server); inspect ownership/inline potential; storage adapter; preserve interface and persisted-data contracts; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/publication/clickhouse,inventory complete; semantic disposition pending -internal/dataframe/published,github.com/calypr/loom/internal/dataframe/published,published,domain/library,"Read, page, aggregate, and export authorized published ClickHouse dataframes.",aggregate_execute.go; aggregate_plan.go; aggregate_sql.go; aggregate_types.go; artifact.go; current.go; errors.go; export.go; page.go; query_helpers.go; read.go; read_pins.go; schema.go; selection.go; types.go,aggregate_clickhouse_integration_test.go; aggregate_execute_test.go; aggregate_plan_test.go; artifact_roundtrip_test.go; artifact_test.go; current_test.go; errors_test.go; export_test.go; nullsort_page_cursor_e2e_test.go; page_test.go; read_pins_test.go; read_test.go; selection_test.go,—,—,3674,2140,56,AggregateBatchRequest; AggregateBatchResult; AggregateJob; AggregateJobResult; AggregateResponseMode; AggregateResult; AggregationResult; AggregationSpec; AggregationsRequest; AggregationsResult; ArtifactColumn; ArtifactDescriptor; ArtifactFormat; ArtifactIdentity; ArtifactInterpretationIdentity; ArtifactMember; ArtifactRequest; ArtifactResult; ArtifactRowIdentity; ArtifactRowStream; ArtifactRowVisitor; ClickHouseQueryer; Column; CurrentProjectDataset; CurrentProjectDatasets; DataframeSelector; ExactExecutionMaterialization; ExactExportRequest; ExactExportResult; ExactExportVisitor; ExcludeSelfFilters; ExecuteAggregateBatch; ExportFormat; ExportRequest; Filter; Materialization; Normalize; NormalizeAggregationResults; Page; PageRequest; Reader; ResolveArtifactFormat; ResolveSelectionSource; SelectionSourceAdapter; Sort; SourceResourceRef; State; Stream; StreamExactExecution; StreamExactExport; StreamRequest; StreamSelectionSource; ValidateTypedFilters; WithExecutionReadPins; Write; WriteArtifact,internal/dataframe/errors; internal/dataframe/publication; internal/dataframe/unit; internal/dataset; internal/projectid,internal/dataframe/errors; internal/dataframe/publication; internal/dataset; internal/store/clickhouse,internal/api/graphql/graph/dataframe; internal/api/graphql/graph/resolver; internal/explorer; internal/explorer/artifactfs; internal/explorer/lifecycle; internal/server,internal/api/graphql/graph; internal/dataframe/publication/arango; internal/dataframe/publication/clickhouse; internal/explorer/arango; internal/store/clickhouse,github.com/google/uuid,1 exported interface(s),legacy=4; compatibility=2,broad surface (15 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/published,inventory complete; semantic disposition pending -internal/dataframe/recipe,github.com/calypr/loom/internal/dataframe/recipe,recipe,domain/library,"Define, canonicalize, expand, encode, and validate persistence-neutral recipe documents.",canonical.go; construction.go; construction_combine.go; construction_reshape.go; derived.go; derived_validation.go; document_types.go; document_validation.go; dynamic_validation.go; fragments.go; json_encoding.go; revisions.go; strict_json.go; table_reshape.go; validation_rich.go; workspace_output_dependencies.go,construction_combine_test.go; construction_group_policy_test.go; construction_test.go; correlated_selection_test.go; derived_validation_test.go; documentation_test.go; fragments_test.go; publication_identity_nul_http_test.go; table_reshape_test.go; types_test.go; workspace_output_dependencies_test.go,—,—,5611,1930,147,Aggregate; AggregateOperation; ArrayQuantifier; Bundle; CanonicalBundle; CanonicalJSON; CanonicalJSONWithWorkspaceOutputs; CatalogProjection; Clone; CodeValue; ColumnNaming; ColumnTransformation; Construction; ConstructionCodedGroup; ConstructionCodedGroupSource; ConstructionCodedPivot; ConstructionCodedPivotCategory; ConstructionCodedPivotSource; ConstructionCombine; ConstructionCombineJoinType; ConstructionCombineKey; ConstructionCombineKind; ConstructionCombineMembershipMode; ConstructionCombineProjection; ConstructionCombineRightMatchPolicy; ConstructionDerive; ConstructionExpand; ConstructionFilter; ConstructionGroup; ConstructionGroupAggregate; ConstructionGroupAggregateOp; ConstructionGroupCodingEmptyPolicy; ConstructionGroupCurrentStageFieldSource; ConstructionGroupFieldSource; ConstructionGroupFieldSourceInput; ConstructionGroupKey; ConstructionGroupMissingKeyPolicy; ConstructionGroupRelatedOccurrenceFieldSource; ConstructionGroupRelatedRouteStep; ConstructionGroupSemanticCategory; ConstructionInputKind; ConstructionInputRef; ConstructionOperand; ConstructionOperation; ConstructionOperationKind; ConstructionPivot; ConstructionPivotCategory; ConstructionRelatedEligibility; ConstructionRelatedExpand; ConstructionRelatedField; ConstructionRelatedFieldSource; ConstructionRelatedPredicate; ConstructionRelatedRepeatedBoundary; ConstructionRelatedRouteStep; ConstructionRelatedSource; ConstructionRowValue; ConstructionRowValuePolicy; ConstructionStep; ConstructionUnpivot; ConstructionUnpivotInput; ContributorWindow; DerivedColumn; DerivedLiteral; DerivedOperand; DerivedOperandKind; DerivedOperation; Digest; DigestWithWorkspaceOutputs; DivisionByZeroPolicy; DocumentRef; DynamicColumn; Error; ExpandBundle; ExpandExpression; ExpandFragments; Expansion; ExpansionEmptyPolicy; ExpansionIdentity; Explain; Explanation; Expression; ExtensionColumn; ExtensionColumnMapping; Field; Filter; FilterOperator; FilterValue; FilterValueKind; Fragment; FragmentLibrary; GroupRowValuePolicy; GroupRows; GroupedPivot; GroupedPivotCategory; Identity; IsValidConstructionLogicalType; MarshalJSON; MissingInputPolicy; Normalized; NormalizedPivotProjectionMode; NumericKind; Output; OutputDependencyOrder; OutputExplanation; OwnerRecordProjection; Parse; Pivot; PivotDiscovery; PivotDuplicatePolicy; PivotMissingCellPolicy; PivotUnlistedCategoryPolicy; PopulationConstraint; PopulationRouteStep; RecipeRevision; RepresentativeSlice; RevisionStore; RootColumnNaming; RuntimeBindings; SortedNames; StageColumn; TableReshape; TableReshapeKind; TableScalar; TableScalarKind; TemporalDirection; TemporalOrdering; TemporalPrecision; TemporalTiePolicy; TerminalCombineStep; Traversal; TraversalColumnNaming; TraversalMatchMode; UnitNormalizationPolicy; UnmarshalJSON; Unpivot; UnpivotInput; UnpivotNullRowPolicy; Valid; ValidPivotProjectionMode; Validate; ValidateConcreteValue; ValidatePivotCategoryKey; ValidateStructure; ValidateWithGroupRows; ValidateWithWorkspaceOutputs; ValidationError; ValueMode,generated/fhir; internal/authscope; internal/dataframe/columntransform; internal/dataframe/lineage; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema,internal/dataframe/lineage; internal/dataframe/publication; internal/fhir/schema,cmd/dataframe-profile; cmd/explorer-config-v2-convert; cmd/preview-covering-index-artifact; conformance/compiler; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/execution; internal/dataframe/expression; internal/dataframe/recipe/exec; internal/dataframe/recipe/exec/arango; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/explorer; internal/explorer/capability; internal/explorer/compilation; internal/explorer/lifecycle; internal/server,internal/dataframe/compiler/render/aql; internal/explorer/arango,—,1 exported interface(s),legacy=17; compatibility=6,broad surface (16 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/recipe,inventory complete; semantic disposition pending -internal/dataframe/recipe/exec,github.com/calypr/loom/internal/dataframe/recipe/exec,exec,domain/library,Define durable recipe-version storage contracts and immutable registration behavior.,persistent.go,persistent_test.go,—,—,84,102,5,Entry; PersistentRegistry; Reader; RegisterVersion; Store,internal/dataframe/recipe,internal/dataframe/recipe,internal/api/graphql/graph/resolver; internal/dataframe/execution; internal/dataframe/recipe/exec/arango; internal/server,internal/dataframe/compiler; internal/explorer/lifecycle,—,2 exported interface(s),—,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/recipe/exec,inventory complete; semantic disposition pending -internal/dataframe/recipe/exec/arango,github.com/calypr/loom/internal/dataframe/recipe/exec/arango,arango,storage adapter,Persist recipe versions and execution-use records in ArangoDB.,registry.go; revisions.go,registry_test.go; revisions_test.go; revisions_user_faults_test.go,—,—,294,555,15,BootstrapSpec; Client; Get; List; LoadRecipe; LoadRecipeVersion; New; NewRevisionRegistry; RecipeVersionUsed; Register; Registry; ReplaceRecipeVersion; RevisionBootstrapSpec; RevisionRegistry; SaveRecipe,internal/dataframe/errors; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/store/arango,internal/dataframe/errors; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/store/arango,internal/server,—,—,—,legacy=1,one production importer (internal/server); inspect ownership/inline potential; storage adapter; preserve interface and persisted-data contracts,go test ./internal/dataframe/recipe/exec/arango,inventory complete; semantic disposition pending +internal/dataframe/publication,github.com/calypr/loom/internal/dataframe/publication,publication,domain/library,"Own backend-neutral dataframe publication contracts, bundles, and publication orchestration.",bundle.go; flat_columns.go; quality.go; runner.go; types.go,bundle_test.go; flat_columns_test.go; pointer_name_encoding_test.go; row_smuggling_prevention_test.go,—,—,1353,385,55,BundleCatalog; BundleExecution; BundleExecutionPageFunc; BundleIdentity; BundleOutputRecord; BundlePointer; BundleState; Canonical; CanonicalizeLegacy; CloneQualityReports; ColumnProvenance; ColumnQuality; DataframeSelector; Error; ExactExecutionCatalog; ExecutionReadPinCatalog; FinalSchemaDigest; FlatColumnName; Key; KeyIntegrity; Limits; LogicalColumn; OutputSchema; OutputStream; PagedBundleCatalog; PhaseError; PhysicalColumn; PointerName; PublicationIdentity; Publish; PublishedOutput; QualifyFlatRow; QualityCompleteness; QualityFailedError; QualityIncompleteError; QualityIssues; QualityLimits; QualityOmission; QualityPolicy; QualityReport; QualityVerdict; Queryable; RelatedSourcePopulation; Result; Selector; SourceRowMetadata; SourceRowMetadataWriter; Successful; Target; Transaction; Unwrap; Valid; ValidateQualityReportBindings; ValidateQualityReports; WithPhase,internal/dataframe/errors; internal/dataset,—,internal/acceptance; internal/api/bulk/load; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/resolver; internal/backends/arango/internal/explorer; internal/backends/arango/internal/publication; internal/backends/postgres/internal/explorer; internal/backends/postgres/internal/publication; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/explorer; internal/explorer/lifecycle; internal/server,internal/api/graphql/graph; internal/backends/postgres/internal/store; internal/dataframe/recipe,—,7 exported interface(s),legacy=1; migration=1; compatibility=2,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/publication,inventory complete; semantic disposition pending +internal/dataframe/publication/clickhouse,github.com/calypr/loom/internal/dataframe/publication/clickhouse,clickhouse,storage adapter,Materialize and inspect published dataframe bundles in ClickHouse.,bundle_store.go,abort_ackloss_e2e_test.go; bundle_store_test.go; export_pin_crash_window_e2e_test.go; orphaned_published_growth_e2e_test.go; pinblock_reconcile_e2e_test.go; pointer_name_nul_ambiguity_e2e_test.go; publish_partial_pointer_e2e_test.go; reconcile_toctou_e2e_test.go; reconcile_toctou_test.go; release_dangle_reconcile_e2e_test.go; unpinned_page_read_e2e_test.go,—,—,1223,6528,21,Abort; ActiveReleaseReader; Begin; BundleClickHouseStore; ClickHouseBundleStore; Commit; CreateOutput; ExistingPublishedOutputs; ExistingQualityReports; FinalizeSchema; Idempotent; InsertRows; NewBundleStore; Reconcile; SetActiveReleaseReader; SetFinalSchemaDigest; SetOutputMetadata; SetQualityReports; SetSourceRowMetadata; SupportsObjectValues; WriteBatch,internal/dataframe/errors; internal/dataframe/publication; internal/dataset; internal/store/clickhouse,internal/backends/arango; internal/dataframe/publication; internal/dataframe/published; internal/dataset; internal/store/clickhouse,internal/server,internal/backends/arango/internal/publication,github.com/google/uuid,2 exported interface(s),legacy=1,one production importer (internal/server); inspect ownership/inline potential; storage adapter; preserve interface and persisted-data contracts; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/publication/clickhouse,inventory complete; semantic disposition pending +internal/dataframe/published,github.com/calypr/loom/internal/dataframe/published,published,domain/library,"Read, page, aggregate, and export authorized published ClickHouse dataframes.",aggregate_execute.go; aggregate_plan.go; aggregate_sql.go; aggregate_types.go; artifact.go; current.go; errors.go; export.go; page.go; query_helpers.go; read.go; read_pins.go; schema.go; selection.go; types.go,aggregate_clickhouse_integration_test.go; aggregate_execute_test.go; aggregate_plan_test.go; artifact_roundtrip_test.go; artifact_test.go; current_test.go; errors_test.go; export_test.go; nullsort_page_cursor_e2e_test.go; page_test.go; read_pins_test.go; read_test.go; selection_test.go,—,—,3674,2140,56,AggregateBatchRequest; AggregateBatchResult; AggregateJob; AggregateJobResult; AggregateResponseMode; AggregateResult; AggregationResult; AggregationSpec; AggregationsRequest; AggregationsResult; ArtifactColumn; ArtifactDescriptor; ArtifactFormat; ArtifactIdentity; ArtifactInterpretationIdentity; ArtifactMember; ArtifactRequest; ArtifactResult; ArtifactRowIdentity; ArtifactRowStream; ArtifactRowVisitor; ClickHouseQueryer; Column; CurrentProjectDataset; CurrentProjectDatasets; DataframeSelector; ExactExecutionMaterialization; ExactExportRequest; ExactExportResult; ExactExportVisitor; ExcludeSelfFilters; ExecuteAggregateBatch; ExportFormat; ExportRequest; Filter; Materialization; Normalize; NormalizeAggregationResults; Page; PageRequest; Reader; ResolveArtifactFormat; ResolveSelectionSource; SelectionSourceAdapter; Sort; SourceResourceRef; State; Stream; StreamExactExecution; StreamExactExport; StreamRequest; StreamSelectionSource; ValidateTypedFilters; WithExecutionReadPins; Write; WriteArtifact,internal/dataframe/errors; internal/dataframe/publication; internal/dataframe/unit; internal/dataset; internal/projectid,internal/dataframe/errors; internal/dataframe/publication; internal/dataset; internal/store/clickhouse,internal/api/graphql/graph/dataframe; internal/api/graphql/graph/resolver; internal/explorer; internal/explorer/artifactfs; internal/explorer/lifecycle; internal/server,internal/api/graphql/graph; internal/backends/arango/internal/explorer; internal/backends/arango/internal/publication; internal/dataframe/publication/clickhouse; internal/store/clickhouse,github.com/google/uuid,1 exported interface(s),legacy=4; compatibility=2,broad surface (15 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/published,inventory complete; semantic disposition pending +internal/dataframe/recipe,github.com/calypr/loom/internal/dataframe/recipe,recipe,domain/library,"Define, canonicalize, expand, encode, and validate persistence-neutral recipe documents.",canonical.go; construction.go; construction_combine.go; construction_reshape.go; derived.go; derived_validation.go; document_types.go; document_validation.go; dynamic_validation.go; fragments.go; json_encoding.go; revisions.go; strict_json.go; table_reshape.go; validation_rich.go; workspace_output_dependencies.go,construction_combine_test.go; construction_group_policy_test.go; construction_test.go; correlated_selection_test.go; derived_validation_test.go; documentation_test.go; fragments_test.go; publication_identity_nul_http_test.go; table_reshape_test.go; types_test.go; workspace_output_dependencies_test.go,—,—,5611,1930,147,Aggregate; AggregateOperation; ArrayQuantifier; Bundle; CanonicalBundle; CanonicalJSON; CanonicalJSONWithWorkspaceOutputs; CatalogProjection; Clone; CodeValue; ColumnNaming; ColumnTransformation; Construction; ConstructionCodedGroup; ConstructionCodedGroupSource; ConstructionCodedPivot; ConstructionCodedPivotCategory; ConstructionCodedPivotSource; ConstructionCombine; ConstructionCombineJoinType; ConstructionCombineKey; ConstructionCombineKind; ConstructionCombineMembershipMode; ConstructionCombineProjection; ConstructionCombineRightMatchPolicy; ConstructionDerive; ConstructionExpand; ConstructionFilter; ConstructionGroup; ConstructionGroupAggregate; ConstructionGroupAggregateOp; ConstructionGroupCodingEmptyPolicy; ConstructionGroupCurrentStageFieldSource; ConstructionGroupFieldSource; ConstructionGroupFieldSourceInput; ConstructionGroupKey; ConstructionGroupMissingKeyPolicy; ConstructionGroupRelatedOccurrenceFieldSource; ConstructionGroupRelatedRouteStep; ConstructionGroupSemanticCategory; ConstructionInputKind; ConstructionInputRef; ConstructionOperand; ConstructionOperation; ConstructionOperationKind; ConstructionPivot; ConstructionPivotCategory; ConstructionRelatedEligibility; ConstructionRelatedExpand; ConstructionRelatedField; ConstructionRelatedFieldSource; ConstructionRelatedPredicate; ConstructionRelatedRepeatedBoundary; ConstructionRelatedRouteStep; ConstructionRelatedSource; ConstructionRowValue; ConstructionRowValuePolicy; ConstructionStep; ConstructionUnpivot; ConstructionUnpivotInput; ContributorWindow; DerivedColumn; DerivedLiteral; DerivedOperand; DerivedOperandKind; DerivedOperation; Digest; DigestWithWorkspaceOutputs; DivisionByZeroPolicy; DocumentRef; DynamicColumn; Error; ExpandBundle; ExpandExpression; ExpandFragments; Expansion; ExpansionEmptyPolicy; ExpansionIdentity; Explain; Explanation; Expression; ExtensionColumn; ExtensionColumnMapping; Field; Filter; FilterOperator; FilterValue; FilterValueKind; Fragment; FragmentLibrary; GroupRowValuePolicy; GroupRows; GroupedPivot; GroupedPivotCategory; Identity; IsValidConstructionLogicalType; MarshalJSON; MissingInputPolicy; Normalized; NormalizedPivotProjectionMode; NumericKind; Output; OutputDependencyOrder; OutputExplanation; OwnerRecordProjection; Parse; Pivot; PivotDiscovery; PivotDuplicatePolicy; PivotMissingCellPolicy; PivotUnlistedCategoryPolicy; PopulationConstraint; PopulationRouteStep; RecipeRevision; RepresentativeSlice; RevisionStore; RootColumnNaming; RuntimeBindings; SortedNames; StageColumn; TableReshape; TableReshapeKind; TableScalar; TableScalarKind; TemporalDirection; TemporalOrdering; TemporalPrecision; TemporalTiePolicy; TerminalCombineStep; Traversal; TraversalColumnNaming; TraversalMatchMode; UnitNormalizationPolicy; UnmarshalJSON; Unpivot; UnpivotInput; UnpivotNullRowPolicy; Valid; ValidPivotProjectionMode; Validate; ValidateConcreteValue; ValidatePivotCategoryKey; ValidateStructure; ValidateWithGroupRows; ValidateWithWorkspaceOutputs; ValidationError; ValueMode,generated/fhir; internal/authscope; internal/dataframe/columntransform; internal/dataframe/lineage; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema,internal/dataframe/lineage; internal/dataframe/publication; internal/fhir/schema,cmd/dataframe-profile; cmd/explorer-config-v2-convert; cmd/preview-covering-index-artifact; conformance/compiler; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/backends/arango/internal/recipeexec; internal/backends/postgres/internal/explorer; internal/backends/postgres/internal/recipes; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/execution; internal/dataframe/expression; internal/dataframe/recipe/exec; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/explorer; internal/explorer/capability; internal/explorer/compilation; internal/explorer/lifecycle; internal/server,internal/backends/arango/internal/explorer; internal/backends/postgres; internal/backends/postgres/internal/recipeexec; internal/backends/postgres/internal/store; internal/dataframe/compiler/command; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/postgres,—,1 exported interface(s),legacy=17; compatibility=6,broad surface (16 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/recipe,inventory complete; semantic disposition pending +internal/dataframe/recipe/exec,github.com/calypr/loom/internal/dataframe/recipe/exec,exec,domain/library,Define durable recipe-version storage contracts and immutable registration behavior.,persistent.go,persistent_test.go,—,—,84,102,5,Entry; PersistentRegistry; Reader; RegisterVersion; Store,internal/dataframe/recipe,internal/dataframe/recipe,internal/api/graphql/graph/resolver; internal/backends/arango/internal/recipeexec; internal/backends/postgres/internal/recipeexec; internal/dataframe/execution; internal/server,internal/backends/postgres; internal/backends/postgres/internal/store; internal/dataframe/compiler; internal/explorer/lifecycle,—,2 exported interface(s),—,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/recipe/exec,inventory complete; semantic disposition pending internal/dataframe/recipe/schema,github.com/calypr/loom/internal/dataframe/recipe/schema,schema,domain/library,Resolve catalog-backed recipe declarations into concrete validated schemas.,discovery.go; extension_columns.go; resolve.go,resolve_test.go,—,—,882,293,8,Discovery; ExtensionValueObservation; FieldCandidate; Fields; Resolve; Resolved; Scope; SourceSnapshot,internal/dataframe/recipe; internal/fhir/schema,internal/dataframe/recipe; internal/dataframe/semantic,internal/api/graphql/graph/query; internal/api/recipe; internal/server,—,—,1 exported interface(s),legacy=2,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataframe/recipe/schema,inventory complete; semantic disposition pending -internal/dataframe/semantic,github.com/calypr/loom/internal/dataframe/semantic,semantic,domain/library,Convert validated dataframe requests and recipes into backend-neutral logical FHIR plans.,doc.go; dynamic_schema.go; execution_context.go; recipe_filter.go; recipe_plan_build.go; recipe_plan_output.go; recipe_plan_types.go; recipe_projection.go; recipe_rich_shaping.go; recipe_traversal.go; resolved_plan.go; selection_semantics.go; selector_spec.go; semantic_nodes.go; semantic_validation.go,dynamic_schema_test.go; recipe_plan_test.go; recipe_plan_user_fault_class_test.go; recipe_rich_shaping_test.go; resolved_plan_test.go,—,—,3022,931,47,BuildRecipePlan; Candidate; Column; DynamicSpec; ExecutionContext; ExpansionEmptyPolicy; ExpansionExplanation; Explain; ExpressionExplanation; FallbackSelectors; Freeze; FrozenSchema; LowerRecipeFilters; LowerRecipeFiltersForAlias; NormalizeRecipeMatchMode; OutputPlan; OutputPlanExplanation; PrimarySelector; RecipePlan; RecipePlanExplanation; ResolveRecipePlan; ResolveSemanticField; ResolvedColumn; ResolvedRecipePlan; SelectionSemanticSpec; SemanticAggregate; SemanticContributorWindow; SemanticDynamicMap; SemanticExpression; SemanticField; SemanticGroupRows; SemanticGroupedPivot; SemanticGroupedPivotCategory; SemanticNode; SemanticOccurrence; SemanticOwnerRecords; SemanticPivot; SemanticPopulation; SemanticPopulationRouteStep; SemanticRowExpansion; SemanticSlice; SemanticTableReshape; SemanticTemporalOrdering; SemanticUnpivot; SemanticUnpivotInput; Validate; ValidateSemanticGraph,internal/authscope; internal/dataframe/errors; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema,internal/dataframe/errors; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema,cmd/dataframe-profile; cmd/preview-covering-index-artifact; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/execution,conformance/compiler; internal/dataframe/compiler/render/aql; internal/dataframe/recipe/schema; internal/explorer/compilation; internal/explorer/lifecycle; internal/server,—,—,legacy=3; compatibility=1,broad surface (15 production files); inspect cohesion,go test ./internal/dataframe/semantic,inventory complete; semantic disposition pending -internal/dataframe/spec,github.com/calypr/loom/internal/dataframe/spec,spec,domain/library,"Define the backend-independent dataframe request, filter, grain, relationship, and selector language.",doc.go; filter.go; filter_semantics.go; grain.go; relationship_match.go; selector_validation.go; selectors.go,correlated_selection_test.go; grain_classification_test.go,—,—,745,71,31,AllowsMany; ArrayQuantifier; Cardinality; Clone; CodeValue; ContainsFilter; DefaultRowIdentity; ExpandsRows; FilterOperator; FilterValue; FilterValueKind; InferRowGrain; IsRequired; OperatorSupportsKind; ParseDirectScalarSelector; ParseSelector; ProjectionMode; Required; RootResourceForGrain; RowGrain; RowIdentity; Selector; SelectorCardinality; SelectorStep; TraversalMatchMode; TypedFilter; Valid; Validate; ValidateProjection; ValidateRootGrain; ValidateTypedFilterForResource,generated/fhir; internal/dataframe/errors; internal/fhir/schema,internal/dataframe/errors; internal/fhir/schema,internal/api/graphql/graph/query; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/execution; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer/compilation; internal/explorer/lifecycle; internal/server,conformance/compiler; internal/api/graphql/graph/resolver,—,—,legacy=1; compatibility=2,no automatic consolidation signal; perform semantic review,go test ./internal/dataframe/spec,inventory complete; semantic disposition pending -internal/dataframe/unit,github.com/calypr/loom/internal/dataframe/unit,unit,domain/library,"Define unit identity, dimensional compatibility, and approved conversion rules for quantity values.",arithmetic.go; unit.go,arithmetic_test.go; unit_test.go,—,—,442,203,27,ApprovedUnitPolicies; ApprovedUnitPolicy; ArithmeticOperand; ArithmeticOperation; ArithmeticUnitError; ArithmeticUnitErrorReason; ColumnArithmeticOperand; Equal; Error; LiteralArithmeticOperand; ResolveApprovedUnitPolicy; ResolveApprovedUnitRules; ResolveArithmeticUnit; String; SupportsSources; UnitConversionKind; UnitConversionRule; UnitDimension; UnitDimensionFor; UnitIdentity; UnitNormalization; UnitRuleReference; Valid; Validate; ValidateUnitPolicyReference; ValidateUnitRuleReference; Zero,—,—,internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer; internal/explorer/authoringv2; internal/explorer/compilation; internal/explorer/lifecycle; internal/server,internal/dataframe/compiler,—,—,—,no automatic consolidation signal; perform semantic review,go test ./internal/dataframe/unit,inventory complete; semantic disposition pending -internal/dataset,github.com/calypr/loom/internal/dataset,dataset,domain/library,"Define immutable dataset generation manifests, releases, active pointers, and lifecycle validation.",active_resolver.go; dataframe_selector.go; doc.go; manifest.go; ref.go; release.go; schema.go; validation.go,active_resolver_test.go; manifest_test.go; memory_store_test.go; ref_test.go; release_id_framing_test.go; release_requirements_e2e_test.go; release_test.go; release_verifier_fault_classification_test.go; schema_test.go,README.md,—,662,1599,33,Activate; ActivationRequest; Active; ActiveRelease; ActiveReleaseResolver; ActiveResolver; ContractVerification; Create; DataframeSelector; Error; IsStaged; Key; Manifest; ManifestReader; NewManifest; NewRef; NewSchemaSnapshot; ProjectRelease; PublicationVerification; PublicationVerifier; Ref; ReleasePublication; ReleaseRepository; ReleaseRequirementsError; ReleaseService; ResolveActive; SchemaSnapshot; State; Transition; Unwrap; Valid; Validate; ValidateGeneration,—,internal/dataset; internal/dataset/arango; internal/store/arango,cmd/arango-fhir-proto; internal/api/bulk/load; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/dataframe/publication; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/dataset/arango; internal/explorer; internal/explorer/arango; internal/explorer/lifecycle; internal/ingest; internal/server,internal/api/graphql/graph; internal/dataframe/publication/arango; internal/dataset,—,5 exported interface(s),legacy=1; compatibility=2,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataset,inventory complete; semantic disposition pending -internal/dataset/arango,github.com/calypr/loom/internal/dataset/arango,arango,storage adapter,Persist immutable dataset generation lifecycle state in ArangoDB.,collections.go; doc.go; document_codec.go; manifest_lifecycle.go; release_store.go,generation_activation_cas_e2e_test.go; revision_inflation_e2e_test.go; store_test.go,—,—,764,1240,15,Activate; ActiveDocumentKey; BootstrapSpec; CollectionSpecs; CompareAndSwapActivateRelease; CompareAndSwapPublishRelease; CreateManifest; ManifestDocumentKey; New; ReadActiveRelease; ReadManifest; ResolveActiveManifest; SaveRelease; Store; TransitionManifest,internal/dataset; internal/store/arango,internal/dataset; internal/store/arango,cmd/arango-fhir-proto; internal/explorer/arango; internal/ingest; internal/server,internal/dataframe/publication/arango; internal/dataframe/publication/clickhouse; internal/dataset,github.com/arangodb/go-driver/v2/arangodb/shared,—,compatibility=2,storage adapter; preserve interface and persisted-data contracts,go test ./internal/dataset/arango,inventory complete; semantic disposition pending -internal/explorer,github.com/calypr/loom/internal/explorer,explorer,domain/library,"Define Explorer aggregate identities, configs, revisions, receipts, output contracts, stores, and core service operations.",artifact.go; authoring_errors.go; canonical_json.go; compilation.go; compilation_receipt.go; config_v2.go; explicit_group_repository.go; explicit_groups.go; identity.go; interpretation.go; output_contract.go; selection.go; service.go; state_v1.go; store.go; types.go; viewer_projection.go,artifact_test.go; compilation_receipt_test.go; explicit_groups_test.go; interpretation_test.go; output_contract_test.go; repository_revision_id_test.go; selection_test.go; types_test.go; viewer_projection_order_test.go,—,—,4335,1587,165,AbortSelection; ActivateRepositoryGeneration; ActiveRevision; AppendSelectionMembers; ApplyWorkspaceCommands; ApplyWorkspaceCommandsChecked; ArtifactCompletion; ArtifactID; ArtifactRecord; ArtifactStage; ArtifactState; ArtifactStore; AuthoringDiagnostic; AuthoringError; BeginSelection; BuildViewerProjection; Canonical; CanonicalContent; CanonicalExplicitGroupDefinitions; CanonicalExplicitGroupMemberships; CanonicalInterpretationProject; CleanupSelectionStaging; CompilationArtifactDigest; CompilationKey; CompilationReceipt; CompilationReceiptForExplorer; CompilationWarning; CompleteSelection; ComputeContentDigest; ComputeRevisionID; ConfigAction; ConfigChart; ConfigColumn; ConfigExplorer; ConfigFilter; ConfigManagementForID; ConfigTable; ConfigV2; ConfigView; ConstructionProposalBinding; CreateInteractiveFrom; DatasetMetadata; DatasetOutput; DecodePublicOutputContracts; Diagnostic; DigestSelectionMembers; DraftRevision; DraftRevisionReader; EmittedColumn; Error; ExplicitGroupDefinition; ExplicitGroupDefinitionDigest; ExplicitGroupID; ExplicitGroupMembership; ExplicitGroupMembershipCursor; ExplicitGroupMembershipDigest; ExplicitGroupRepository; ExplicitGroupRevision; ExplicitGroupRevisionID; ExplicitGroupRevisionIDFor; ExplicitGroupRevisionState; Explorer; ExplorerRuntimeBindingV1; ExplorerRuntimeColumnV1; ExplorerRuntimeOutputV1; ExplorerRuntimeSchemaV1; ExplorerRuntimeTableColumnV1; ExplorerRuntimeTableV1; ExplorerRuntimeV1; ExplorerStateV1; ExplorerStateV1Active; ExplorerStateV1Draft; ExplorerStateV1Generated; ExplorerSummaryV1; FailRevision; FileActions; Get; GetDraftRevision; GetRevision; GetSelection; IdentityMapping; InterpretationApplicability; InterpretationContentDigest; InterpretationFeatureDefinition; InterpretationLibrary; InterpretationLibraryID; InterpretationPriority; InterpretationRepository; InterpretationRevision; InterpretationRevisionID; InterpretationRule; InterpretationRuleID; InterpretationStructuralCandidate; InterpretationStructuralMatch; Is; List; LoadExplorerState; ManagementMode; Matches; Materialization; MembershipDigest; MigrateLegacyConstructionSteps; NewInterpretationContentDigest; NewInterpretationLibraryID; NewInterpretationRevisionID; NewInterpretationRuleID; NewService; Output; PopulationMemberRemovalProposalBinding; PrepareInterpretationRevision; PublicOutputColumn; PublicOutputContract; PublicOutputContracts; PublicUnitNormalization; PublicUnitRuleIdentity; PublicationMetadata; PublishAuthoring; ReceiptCompiledOutputColumn; ReceiptConstructionActiveRelatedRecord; ReceiptConstructionCodedGroupChoice; ReceiptConstructionGroupFieldOwner; ReceiptConstructionGroupFieldSourceAncestor; ReceiptConstructionGroupFieldSourceContext; ReceiptConstructionGroupFieldSourceExpansion; ReceiptConstructionGroupFieldSourceRouteStep; ReceiptConstructionOperationChoice; ReceiptConstructionRelatedExpand; ReceiptConstructionRelatedExpandAnchor; ReceiptConstructionStage; ReceiptConstructionStageColumn; ReceiptContractSupportedForExecution; ReceiptID; RepositoryRevisionID; ResolvedInterpretation; ResourceRef; Revision; RevisionIdentityEnvelope; RevisionStatus; RowDefinitionProposalBinding; SelectRule; SelectionFilter; SelectionMember; SelectionRevision; SelectionRule; SelectionRuleDigest; SelectionRuleDigestWithMembers; SelectionSource; Service; SharedFilter; StableExplorerID; StartExplicitGroupMembershipDigest; Store; StoreCompilationReceipt; TableShapeProposalBinding; Unwrap; UpsertRepositoryV2; Validate; ValidateAgainst; ValidateID; ValidateSource; ValidateSourceSelection; ViewerProjectionIntegrityError; VisitSelectionMembers; WriteExplicitGroupMembershipFrame; WriteMembershipFrame,internal/dataframe/expression; internal/dataframe/publication; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/unit; internal/dataset; internal/explorer/authoringv2; internal/explorer/capability; internal/projectid,internal/dataframe/publication; internal/dataframe/recipe; internal/dataframe/unit; internal/explorer/authoringv2,cmd/explorer-config-v2-convert; cmd/loom-acceptance; internal/explorer/arango; internal/explorer/artifactfs; internal/explorer/compilation; internal/explorer/lifecycle; internal/server,—,—,6 exported interface(s),legacy=17; migration=2; compatibility=4,broad surface (17 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/explorer,inventory complete; semantic disposition pending -internal/explorer/arango,github.com/calypr/loom/internal/explorer/arango,arango,storage adapter,"Persist Explorer drafts, revisions, receipts, publications, capabilities, and repository state in ArangoDB.",activation.go; capability_snapshot_store.go; codec.go; collections.go; drafts.go; explicit_groups.go; explorers.go; interpretations.go; publication.go; receipts.go; repository.go; revisions.go; selection_references.go; selections.go; store.go; table_shape_capabilities.go,capability_snapshot_store_test.go; collections_test.go; cross_generation_publish_e2e_test.go; draft_conflict_test.go; draft_transaction_formal_replay_test.go; drafts_integration_test.go; drafts_test.go; explicit_group_storage_integration_test.go; explicit_groups_test.go; interpretations_test.go; receipt_store_test.go; repository_test.go; selection_references_test.go; selection_scale_integration_test.go; selection_storage_integration_test.go; selections_cleanup_e2e_test.go; store_test.go; table_shape_capabilities_test.go,—,—,3366,4552,56,AbortExplicitGroupRevision; AbortSelection; ActivateRepositoryGeneration; AppendExplicitGroupMemberships; AppendSelectionMembers; BeginExplicitGroupRevision; BeginSelection; BootstrapSpec; CapabilitySnapshotCollectionSpec; CapabilitySnapshotStore; CleanupExplicitGroupStaging; CleanupSelectionStaging; CollectionSpecs; CompleteExplicitGroupRevision; CompleteSelection; Create; CreateInterpretationRevision; DigestExplicitGroupRevision; DigestSelectionMembers; FailRevision; Get; GetByIdentity; GetByToken; GetCatalog; GetCatalogForLookup; GetCategoryScan; GetCompilationReceiptForExplorer; GetDraftRevision; GetExplicitGroupRevision; GetInterpretationRevision; GetResolution; GetRevision; GetSelection; InsertCompilationReceipt; InsertRevision; List; ListExplicitGroupDefinitions; ListExplicitGroupRevisions; ListInterpretationLibraries; MigrateLegacyRepositoryConfigs; New; NewCapabilitySnapshotStore; NewTableShapeCapabilityRepository; PublishAuthoring; Put; PutCatalog; PutCategoryScan; PutExplicitGroupDefinitions; PutResolution; SaveDraft; Store; TableShapeCapabilitiesCollectionSpec; TableShapeCapabilityRepository; ValidateSelectionReferences; VisitExplicitGroupMemberships; VisitSelectionMembers,internal/authscope; internal/dataframe/publication; internal/dataframe/publication/arango; internal/dataset; internal/dataset/arango; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/tableshapecap; internal/projectid; internal/store/arango,internal/authscope; internal/dataframe/publication; internal/dataframe/publication/arango; internal/dataframe/published; internal/dataframe/recipe; internal/dataset; internal/dataset/arango; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/lifecycle; internal/explorer/tableshapecap; internal/projectid; internal/store/arango,internal/server,—,github.com/arangodb/go-driver/v2/arangodb/shared,—,legacy=12; migration=4; compatibility=1,one production importer (internal/server); inspect ownership/inline potential; broad surface (16 production files); inspect cohesion; storage adapter; preserve interface and persisted-data contracts,go test ./internal/explorer/arango,inventory complete; semantic disposition pending +internal/dataframe/semantic,github.com/calypr/loom/internal/dataframe/semantic,semantic,domain/library,Convert validated dataframe requests and recipes into backend-neutral logical FHIR plans.,doc.go; dynamic_schema.go; execution_context.go; recipe_filter.go; recipe_plan_build.go; recipe_plan_output.go; recipe_plan_types.go; recipe_projection.go; recipe_rich_shaping.go; recipe_traversal.go; resolved_plan.go; selection_semantics.go; selector_spec.go; semantic_nodes.go; semantic_validation.go,dynamic_schema_test.go; recipe_plan_test.go; recipe_plan_user_fault_class_test.go; recipe_rich_shaping_test.go; resolved_plan_test.go,—,—,3022,931,47,BuildRecipePlan; Candidate; Column; DynamicSpec; ExecutionContext; ExpansionEmptyPolicy; ExpansionExplanation; Explain; ExpressionExplanation; FallbackSelectors; Freeze; FrozenSchema; LowerRecipeFilters; LowerRecipeFiltersForAlias; NormalizeRecipeMatchMode; OutputPlan; OutputPlanExplanation; PrimarySelector; RecipePlan; RecipePlanExplanation; ResolveRecipePlan; ResolveSemanticField; ResolvedColumn; ResolvedRecipePlan; SelectionSemanticSpec; SemanticAggregate; SemanticContributorWindow; SemanticDynamicMap; SemanticExpression; SemanticField; SemanticGroupRows; SemanticGroupedPivot; SemanticGroupedPivotCategory; SemanticNode; SemanticOccurrence; SemanticOwnerRecords; SemanticPivot; SemanticPopulation; SemanticPopulationRouteStep; SemanticRowExpansion; SemanticSlice; SemanticTableReshape; SemanticTemporalOrdering; SemanticUnpivot; SemanticUnpivotInput; Validate; ValidateSemanticGraph,internal/authscope; internal/dataframe/errors; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema,internal/dataframe/errors; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema,cmd/dataframe-profile; cmd/preview-covering-index-artifact; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/execution; internal/server,conformance/compiler; internal/backends/postgres; internal/dataframe/compiler/command; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/postgres; internal/dataframe/recipe/schema; internal/explorer/compilation; internal/explorer/lifecycle,—,—,legacy=3; compatibility=1,broad surface (15 production files); inspect cohesion,go test ./internal/dataframe/semantic,inventory complete; semantic disposition pending +internal/dataframe/spec,github.com/calypr/loom/internal/dataframe/spec,spec,domain/library,"Define the backend-independent dataframe request, filter, grain, relationship, and selector language.",doc.go; filter.go; filter_semantics.go; grain.go; relationship_match.go; selector_validation.go; selectors.go,correlated_selection_test.go; grain_classification_test.go,—,—,745,71,31,AllowsMany; ArrayQuantifier; Cardinality; Clone; CodeValue; ContainsFilter; DefaultRowIdentity; ExpandsRows; FilterOperator; FilterValue; FilterValueKind; InferRowGrain; IsRequired; OperatorSupportsKind; ParseDirectScalarSelector; ParseSelector; ProjectionMode; Required; RootResourceForGrain; RowGrain; RowIdentity; Selector; SelectorCardinality; SelectorStep; TraversalMatchMode; TypedFilter; Valid; Validate; ValidateProjection; ValidateRootGrain; ValidateTypedFilterForResource,generated/fhir; internal/dataframe/errors; internal/fhir/schema,internal/dataframe/errors; internal/fhir/schema,internal/api/graphql/graph/query; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/postgres; internal/dataframe/execution; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer/compilation; internal/explorer/lifecycle; internal/server,conformance/compiler; internal/api/graphql/graph/resolver; internal/backends/postgres,—,—,legacy=1; compatibility=2,no automatic consolidation signal; perform semantic review,go test ./internal/dataframe/spec,inventory complete; semantic disposition pending +internal/dataframe/unit,github.com/calypr/loom/internal/dataframe/unit,unit,domain/library,"Define unit identity, dimensional compatibility, and approved conversion rules for quantity values.",arithmetic.go; unit.go,arithmetic_test.go; unit_test.go,—,—,442,203,27,ApprovedUnitPolicies; ApprovedUnitPolicy; ArithmeticOperand; ArithmeticOperation; ArithmeticUnitError; ArithmeticUnitErrorReason; ColumnArithmeticOperand; Equal; Error; LiteralArithmeticOperand; ResolveApprovedUnitPolicy; ResolveApprovedUnitRules; ResolveArithmeticUnit; String; SupportsSources; UnitConversionKind; UnitConversionRule; UnitDimension; UnitDimensionFor; UnitIdentity; UnitNormalization; UnitRuleReference; Valid; Validate; ValidateUnitPolicyReference; ValidateUnitRuleReference; Zero,—,—,internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/postgres; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer; internal/explorer/authoringv2; internal/explorer/compilation; internal/explorer/lifecycle; internal/server,internal/dataframe/compiler,—,—,—,no automatic consolidation signal; perform semantic review,go test ./internal/dataframe/unit,inventory complete; semantic disposition pending +internal/dataset,github.com/calypr/loom/internal/dataset,dataset,domain/library,"Define immutable dataset generation manifests, releases, active pointers, and lifecycle validation.",active_resolver.go; dataframe_selector.go; doc.go; manifest.go; manifest_port.go; ref.go; release.go; schema.go; validation.go,active_resolver_test.go; manifest_test.go; memory_store_test.go; ref_test.go; release_id_framing_test.go; release_idempotency_test.go; release_requirements_e2e_test.go; release_test.go; release_verifier_fault_classification_test.go; schema_test.go,README.md,—,802,2253,34,Activate; ActivationRequest; Active; ActiveRelease; ActiveReleaseResolver; ActiveResolver; ContractVerification; Create; DataframeSelector; Error; IsStaged; Key; Manifest; ManifestLifecycle; ManifestReader; NewManifest; NewRef; NewSchemaSnapshot; ProjectRelease; PublicationVerification; PublicationVerifier; Ref; ReleasePublication; ReleaseRepository; ReleaseRequirementsError; ReleaseService; ResolveActive; SchemaSnapshot; State; Transition; Unwrap; Valid; Validate; ValidateGeneration,—,internal/backends/arango; internal/dataset,cmd/arango-fhir-proto; cmd/loom-data; internal/api/bulk/load; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/backends/arango/internal/dataset; internal/backends/arango/internal/explorer; internal/backends/postgres/control; internal/backends/postgres/internal/dataset; internal/backends/postgres/internal/explorer; internal/backends/postgres/internal/publication; internal/dataframe/publication; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/explorer; internal/explorer/lifecycle; internal/ingest; internal/server,internal/api/graphql/graph; internal/backends/arango/internal/publication; internal/backends/postgres/internal/catalog; internal/backends/postgres/internal/store; internal/dataset,—,6 exported interface(s),legacy=1; compatibility=2,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/dataset,inventory complete; semantic disposition pending +internal/explorer,github.com/calypr/loom/internal/explorer,explorer,domain/library,"Define Explorer aggregate identities, configs, revisions, receipts, output contracts, stores, and core service operations.",artifact.go; authoring_errors.go; canonical_json.go; compilation.go; compilation_receipt.go; config_v2.go; explicit_group_repository.go; explicit_groups.go; identity.go; interpretation.go; output_contract.go; selection.go; service.go; state_v1.go; store.go; types.go; viewer_projection.go,artifact_test.go; compilation_receipt_test.go; explicit_groups_test.go; interpretation_test.go; output_contract_test.go; repository_revision_id_test.go; selection_test.go; types_test.go; viewer_projection_order_test.go,—,—,4335,1587,165,AbortSelection; ActivateRepositoryGeneration; ActiveRevision; AppendSelectionMembers; ApplyWorkspaceCommands; ApplyWorkspaceCommandsChecked; ArtifactCompletion; ArtifactID; ArtifactRecord; ArtifactStage; ArtifactState; ArtifactStore; AuthoringDiagnostic; AuthoringError; BeginSelection; BuildViewerProjection; Canonical; CanonicalContent; CanonicalExplicitGroupDefinitions; CanonicalExplicitGroupMemberships; CanonicalInterpretationProject; CleanupSelectionStaging; CompilationArtifactDigest; CompilationKey; CompilationReceipt; CompilationReceiptForExplorer; CompilationWarning; CompleteSelection; ComputeContentDigest; ComputeRevisionID; ConfigAction; ConfigChart; ConfigColumn; ConfigExplorer; ConfigFilter; ConfigManagementForID; ConfigTable; ConfigV2; ConfigView; ConstructionProposalBinding; CreateInteractiveFrom; DatasetMetadata; DatasetOutput; DecodePublicOutputContracts; Diagnostic; DigestSelectionMembers; DraftRevision; DraftRevisionReader; EmittedColumn; Error; ExplicitGroupDefinition; ExplicitGroupDefinitionDigest; ExplicitGroupID; ExplicitGroupMembership; ExplicitGroupMembershipCursor; ExplicitGroupMembershipDigest; ExplicitGroupRepository; ExplicitGroupRevision; ExplicitGroupRevisionID; ExplicitGroupRevisionIDFor; ExplicitGroupRevisionState; Explorer; ExplorerRuntimeBindingV1; ExplorerRuntimeColumnV1; ExplorerRuntimeOutputV1; ExplorerRuntimeSchemaV1; ExplorerRuntimeTableColumnV1; ExplorerRuntimeTableV1; ExplorerRuntimeV1; ExplorerStateV1; ExplorerStateV1Active; ExplorerStateV1Draft; ExplorerStateV1Generated; ExplorerSummaryV1; FailRevision; FileActions; Get; GetDraftRevision; GetRevision; GetSelection; IdentityMapping; InterpretationApplicability; InterpretationContentDigest; InterpretationFeatureDefinition; InterpretationLibrary; InterpretationLibraryID; InterpretationPriority; InterpretationRepository; InterpretationRevision; InterpretationRevisionID; InterpretationRule; InterpretationRuleID; InterpretationStructuralCandidate; InterpretationStructuralMatch; Is; List; LoadExplorerState; ManagementMode; Matches; Materialization; MembershipDigest; MigrateLegacyConstructionSteps; NewInterpretationContentDigest; NewInterpretationLibraryID; NewInterpretationRevisionID; NewInterpretationRuleID; NewService; Output; PopulationMemberRemovalProposalBinding; PrepareInterpretationRevision; PublicOutputColumn; PublicOutputContract; PublicOutputContracts; PublicUnitNormalization; PublicUnitRuleIdentity; PublicationMetadata; PublishAuthoring; ReceiptCompiledOutputColumn; ReceiptConstructionActiveRelatedRecord; ReceiptConstructionCodedGroupChoice; ReceiptConstructionGroupFieldOwner; ReceiptConstructionGroupFieldSourceAncestor; ReceiptConstructionGroupFieldSourceContext; ReceiptConstructionGroupFieldSourceExpansion; ReceiptConstructionGroupFieldSourceRouteStep; ReceiptConstructionOperationChoice; ReceiptConstructionRelatedExpand; ReceiptConstructionRelatedExpandAnchor; ReceiptConstructionStage; ReceiptConstructionStageColumn; ReceiptContractSupportedForExecution; ReceiptID; RepositoryRevisionID; ResolvedInterpretation; ResourceRef; Revision; RevisionIdentityEnvelope; RevisionStatus; RowDefinitionProposalBinding; SelectRule; SelectionFilter; SelectionMember; SelectionRevision; SelectionRule; SelectionRuleDigest; SelectionRuleDigestWithMembers; SelectionSource; Service; SharedFilter; StableExplorerID; StartExplicitGroupMembershipDigest; Store; StoreCompilationReceipt; TableShapeProposalBinding; Unwrap; UpsertRepositoryV2; Validate; ValidateAgainst; ValidateID; ValidateSource; ValidateSourceSelection; ViewerProjectionIntegrityError; VisitSelectionMembers; WriteExplicitGroupMembershipFrame; WriteMembershipFrame,internal/dataframe/expression; internal/dataframe/publication; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/unit; internal/dataset; internal/explorer/authoringv2; internal/explorer/capability; internal/projectid,internal/dataframe/publication; internal/dataframe/recipe; internal/dataframe/unit; internal/explorer/authoringv2,cmd/explorer-config-v2-convert; cmd/loom-acceptance; internal/backends/arango/internal/explorer; internal/backends/postgres/internal/explorer; internal/explorer/artifactfs; internal/explorer/compilation; internal/explorer/lifecycle; internal/server,internal/backends/postgres/internal/store; internal/dataframe/compiler/render/postgres; internal/dataframe/execution,—,6 exported interface(s),legacy=17; migration=2; compatibility=4,broad surface (17 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/explorer,inventory complete; semantic disposition pending internal/explorer/artifactfs,github.com/calypr/loom/internal/explorer/artifactfs,artifactfs,domain/library,Persist and serve immutable Explorer training artifacts from a dedicated filesystem root.,store.go,reconcile_test.go; store_test.go,—,—,379,621,10,Abort; Begin; Close; Commit; Get; New; Open; Reconcile; Store; Write,internal/dataframe/published; internal/explorer,internal/dataframe/published; internal/explorer,internal/server,—,—,—,—,one production importer (internal/server); inspect ownership/inline potential,go test ./internal/explorer/artifactfs,inventory complete; semantic disposition pending -internal/explorer/authoringv2,github.com/calypr/loom/internal/explorer/authoringv2,authoringv2,domain/library,"Define, validate, canonicalize, and mutate the V2 Explorer Builder authoring workspace.",availability.go; canonical.go; commands.go; construction.go; construction_change.go; construction_choice.go; construction_migration.go; construction_row_values.go; construction_source_children.go; construction_validation.go; framing.go; migration.go; route_identity.go; row_change.go; row_definition.go; semantic_command.go; semantic_types.go; table_shape.go; transformations.go; types.go; validation.go; value_transformations.go; workspace_output_dependencies.go,commands_test.go; compile_preparation_test.go; construction_choice_test.go; construction_combine_test.go; construction_expand_test.go; construction_formal_replay_test.go; construction_group_coded_pivot_test.go; construction_group_field_source_test.go; construction_projection_replay_test.go; construction_proposal_test.go; construction_related_contributor_test.go; construction_row_values_test.go; construction_source_children_test.go; construction_test.go; correlated_test.go; framing_test.go; interpretation_test.go; migration_test.go; population_member_proposal_test.go; row_change_test.go; row_definition_test.go; row_proposal_test.go; semantic_codeable_concept_fallback_test.go; semantic_coded_category_test.go; semantic_command_test.go; table_shape_proposal_test.go; table_shape_test.go; transformations_test.go; types_test.go; value_transformations_test.go; workspace_output_dependencies_test.go; workspace_test.go,—,—,13260,8548,231,Action; ActionColumn; Addable; AggregateSource; AggregateTransformationCapabilities; AggregateTransformationCapabilitiesForCapability; AggregateTransformationCapabilitiesForCatalog; AnalyzeConstructionCandidate; AnalyzeStepEdit; ApplyCommands; ApplyCommandsRequest; ApplyCommandsResponse; ApplyInterpretationCandidate; ApplyRowChange; ApplyRowReplacement; ArithmeticOperand; AssessRowChange; AssessRowReplacement; BuilderState; CanPopulateConstructionRows; CanonicalJSON; CatalogAvailability; CatalogAvailabilityReadiness; CatalogAvailabilitySourceReadiness; CatalogCandidate; CatalogDiagnostic; CatalogEdge; CatalogNode; CatalogObservedField; CatalogObservedRelationship; CatalogObservedResource; CatalogSnapshot; ChartPresentation; CodeValue; Column; ColumnOutput; ColumnSource; ColumnTransformationChange; ColumnValueTransformationCapabilities; ColumnValueTransformationCapabilitiesForCatalog; Command; CommandResult; ConceptCandidate; Construction; ConstructionCanCarrySourceProjection; ConstructionChoiceSelection; ConstructionCodedGroup; ConstructionCodedGroupSource; ConstructionCodedPivot; ConstructionCodedPivotCategory; ConstructionCodedPivotSource; ConstructionCombine; ConstructionCombineJoinType; ConstructionCombineKey; ConstructionCombineKind; ConstructionCombineMembershipMode; ConstructionCombineProjection; ConstructionCombineRightMatchPolicy; ConstructionDependencyIssue; ConstructionDerive; ConstructionDerivedOperation; ConstructionDivisionByZeroPolicy; ConstructionExpand; ConstructionExpandEmptyPolicy; ConstructionExpandedFieldSource; ConstructionFilter; ConstructionFilterOperator; ConstructionFilterValueKind; ConstructionGroup; ConstructionGroupAggregate; ConstructionGroupAggregateOp; ConstructionGroupFieldCurrentStage; ConstructionGroupFieldRelatedOccurrence; ConstructionGroupFieldRelatedRouteStep; ConstructionGroupFieldSource; ConstructionGroupFieldSourceInput; ConstructionGroupKey; ConstructionGroupMissingKeyPolicy; ConstructionGroupSemanticCategory; ConstructionID; ConstructionImpact; ConstructionInputKind; ConstructionInputRef; ConstructionLiteral; ConstructionMissingInputPolicy; ConstructionNumericKind; ConstructionOperand; ConstructionOperandKind; ConstructionOperation; ConstructionOperationKind; ConstructionOutputPresentation; ConstructionPivot; ConstructionPivotCategory; ConstructionPivotDuplicatePolicy; ConstructionPivotMissingCellPolicy; ConstructionPivotUnlistedCategoryPolicy; ConstructionRelatedContributorRule; ConstructionRelatedEligibility; ConstructionRelatedEligibilityMatch; ConstructionRelatedExpand; ConstructionRelatedField; ConstructionRelatedFieldSource; ConstructionRelatedSource; ConstructionRowValue; ConstructionRowValuePolicy; ConstructionSourceProjection; ConstructionSourceProjectionName; ConstructionStep; ConstructionUnpivot; ConstructionUnpivotInput; ConstructionUnpivotNullPolicy; ContributorCode; ContributorOperator; ContributorPredicate; ContributorQuantifier; ContributorValue; ContributorValueKind; ContributorWindowSource; DecodeWorkspace; DerivedConstruction; Digest; Document; DraftSourceContext; EmptyCollectionPolicy; ExpandedRows; ExplicitGroupRowValue; ExplicitGroupSource; ExplorerMetadata; FeatureInterpretation; FeatureInterpretationKind; FieldGroupSource; FieldPath; FieldSource; FileActions; FilterPresentation; FilterValue; FixedFilter; FrameDefinition; FrameManyPolicy; FrameZeroPolicy; GroupSource; GroupSourceKind; GroupedRows; HasMissingInputs; LegacyNilConstructionStepsDigest; LookupMatch; LookupSource; MarshalJSON; MigrateAggregateTemporalPolicy; MigrateLegacyContributors; MigrateLosslessDefaults; MissingKeyPolicy; NewFrameDefinition; NormalizePresentationOrders; Normalized; Output; OutputDependencyOrder; OwnerRecordsSource; PinnedInterpretation; PivotCategory; PivotConstruction; Population; PopulationRouteStep; PrepareWorkspaceForCompilation; ProjectionMode; ProposeStepRemoval; RecordRows; RecordsRowDefinition; RelatedSelection; RepeatedBoundary; ResolveConstructionProposal; ResolveDraftRevision; ResolveDraftSourceContext; ResolveFrameSource; ResolvePopulationMemberProposal; ResolveRowDefinitionProposal; ResolveSemanticSelectionPlan; ResolveTableShapeProposal; ResolvedConstructionChoice; ResolvedDraftSourceContext; RouteMatchMode; RouteNode; RoutePolicy; RouteRebaseChoice; RowChangeAssessment; RowChangeProposal; RowChangeRequest; RowChangeUnresolvedReference; RowDefinition; RowDefinitionKind; SemanticReadinessStatus; SemanticSelection; SemanticSelectionPlan; SemanticSelectionReadiness; SemanticSelectionResult; SharedFilterBinding; SharedFilterBindingRef; SourceWhere; StageColumn; SupportsAnchor; SupportsTimestamp; Tab; TablePresentation; TableReshape; TableScalar; TableScalarKind; TableShape; TemporalFieldChoice; TemporalOrderingSource; TemporalReductionCapabilities; UnassignedMemberPolicy; UnitNormalizationCapabilities; UnitNormalizationPolicy; UnitNormalizationPresetCapability; UnmarshalJSON; UnpivotConstruction; UnpivotInput; UpgradeDocumentToConstruction; Valid; Validate; ValidateConcreteValue; ValidateConstructionExpandedFieldSource; ValidateContributorForCatalog; ValidateForPublication; ValidatePivotCategoryKey; ValidateStructure; ValidateTableShapeOutput; ValidateWithExplicitGroups; ValueTransformationCapability; Workspace; WorkspaceOutputConsumer,internal/catalog; internal/dataframe/columntransform; internal/dataframe/lineage; internal/dataframe/unit; internal/explorer/capability; internal/fhir/schema,internal/catalog; internal/dataframe/columntransform; internal/dataframe/lineage; internal/explorer/capability; internal/fhir/schema,cmd/explorer-config-v2-convert; cmd/preview-covering-index-artifact; generated/loomapi; internal/acceptance; internal/explorer; internal/explorer/arango; internal/explorer/compilation; internal/explorer/lifecycle; internal/repositoryseed; internal/server,internal/explorer/capability,—,—,legacy=50; migration=28; compatibility=3,broad surface (23 production files); inspect cohesion,go test ./internal/explorer/authoringv2,inventory complete; semantic disposition pending -internal/explorer/capability,github.com/calypr/loom/internal/explorer/capability,capability,domain/library,Build the immutable compiler-backed capability contract exposed to Explorer Builder.,builder.go; construction.go; construction_routes.go; domain.go; population_route_choice.go; repository.go; row_choice.go; value_operations.go,builder_test.go; construction_test.go; domain_test.go; row_choice_test.go; semantic_fallback_dto_test.go; value_operations_test.go,—,—,3950,1396,110,AggregateInput; AggregateOperation; AggregateOperationCapability; AggregateRowContext; Allows; Availability; AvailabilityReadiness; AvailabilitySourceReadiness; Build; Builder; Candidate; CandidateProof; CanonicalPayload; ChartAggregation; Clone; CodedGroupChoiceSource; CompilerCallbacks; ConceptCandidate; ConstructionChoice; ConstructionChoiceDecision; ConstructionChoiceFact; ConstructionChoiceForm; ConstructionChoiceIdentity; ConstructionChoiceOption; ConstructionChoicePredicateOperator; ConstructionChoicePresentation; ConstructionChoicePreservation; ConstructionChoiceRowEffect; ConstructionChoiceShape; ConstructionChoiceSource; ConstructionChoiceSourceKind; ConstructionChoiceSupport; ConstructionCodedGroupChoice; ConstructionRelatedFieldChoice; ConstructionRelatedResourceRouteChoice; ConstructionRoutePage; ConstructionRouteSearch; ConstructionRouteStep; DecodeConstructionChoiceID; DecodePopulationRouteChoiceID; DecodeRowChoiceID; DeriveAggregateOperationCapabilities; Diagnostic; Edge; EdgeProof; Evidence; FieldChoiceSource; FieldObservation; FilterOperator; GroupFieldChoiceInput; GroupFieldChoiceSource; GroupFieldCurrentStageInput; GroupFieldRelatedOccurrenceInput; GroupFieldRelatedRouteStep; IsRepeatedCardinality; IsSupportedConstructionSourceType; IsUsable; NewBuilder; NewConstructionCodedGroupChoice; NewConstructionRelatedFieldChoice; NewConstructionRelatedResourceRouteChoiceFromAnchor; NewFieldConstructionChoice; NewFieldConstructionChoiceForRoute; NewGroupCodingIdentityChoice; NewGroupFieldConstructionChoice; NewPopulationRouteChoiceID; NewRowChoice; NewSemanticConstructionChoice; NewSemanticConstructionChoiceForRoute; NewSemanticFrameConstructionChoice; NewSnapshot; Node; NodeProof; ObservedField; ObservedRelationship; ObservedResource; Operation; PlanConstructionRoutes; Policy; PopulationRouteChoiceIdentity; ProjectionMode; ProjectionPolicy; RelatedFieldChoiceSource; RelatedResourceChoiceSource; RelationshipObservation; RepeatedBoundary; RepeatedCoordinate; Repository; ResolveRowChoiceID; ResourceObservation; RoutePolicy; RowChoice; RowChoiceCardinality; RowChoiceFacts; RowChoiceIdentity; RowChoiceKind; RowChoiceOccurrence; RowChoicePathResolver; RowChoiceShape; SemanticBindingChoiceSource; SemanticFrameChoiceSource; SemanticFrameFamily; Snapshot; SnapshotIdentity; Status; UnmarshalJSON; Usable; ValidateConstructionSourceType; ValidateSnapshot; ValidateToken,internal/dataframe/recipe,generated/loomapi; internal/catalog; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/lifecycle,cmd/preview-covering-index-artifact; internal/explorer; internal/explorer/arango; internal/explorer/authoringv2; internal/explorer/compilation; internal/explorer/lifecycle; internal/server,internal/dataframe/compiler; internal/explorer/capability,—,2 exported interface(s),legacy=2,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/explorer/capability,inventory complete; semantic disposition pending -internal/explorer/compilation,github.com/calypr/loom/internal/explorer/compilation,compilation,domain/library,Translate validated Explorer authoring intent into native recipe and compilation artifacts.,compile.go; construction.go; derived_columns.go; indexed_projection.go; interpretation.go; row_expansion.go; semantic_compile.go; source_row.go; table_reshape.go,acceptance_observation_aql_diagnostic_test.go; construction_combine_test.go; construction_related_contributor_test.go; construction_test.go; correlated_test.go; derived_columns_test.go; indexed_projection_test.go; interpretation_test.go; row_expansion_test.go; semantic_compile_expand_source_test.go; source_row_test.go; table_reshape_test.go; translate_test.go; value_transformations_test.go; workspace_output_mapping_test.go; workspace_test.go,—,—,3622,4496,24,Canonical; Compile; CompileWorkspace; Error; InterpretationCandidateMatch; InterpretationCandidateResolution; InterpretationCandidateResolutionState; Match; PopulationFor; PresentationColumn; PresentationConfig; Reason; ResolveAddressableSourceRow; ResolveInterpretationCandidate; ResolveWorkspaceInterpretations; ResolvedInputs; ResolvedInputsDigest; ResolvedInterpretation; ResolvedPopulation; Result; SourceRowDescriptor; State; Unwrap; WorkspaceResult,internal/dataframe/compiler/lower; internal/dataframe/lineage; internal/dataframe/recipe; internal/dataframe/spec; internal/dataframe/unit; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/fhir/schema; internal/projectid,internal/catalog; internal/dataframe/columntransform; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/lineage; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/fhir/schema; internal/store/arango,cmd/preview-covering-index-artifact; internal/explorer/lifecycle; internal/server,—,—,1 exported interface(s),legacy=1,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/explorer/compilation,inventory complete; semantic disposition pending -internal/explorer/lifecycle,github.com/calypr/loom/internal/explorer/lifecycle,lifecycle,domain/library,"Coordinate transport-neutral Explorer authoring, preview, publication, query, and policy operations.",artifact.go; authoring.go; cell_trace.go; column_source.go; construction_category_discovery.go; construction_choice.go; construction_choice_proposal.go; construction_coded_pivot.go; construction_group_field_choices.go; construction_group_related_field_choices.go; construction_group_source.go; construction_pivot_source.go; construction_proposal.go; construction_routes.go; construction_semantic_group.go; derive_source.go; draft_restore.go; errors.go; expand_source.go; explicit_group_resolver.go; explicit_groups.go; filter_source.go; frame_source.go; interpretation.go; interpretation_apply.go; interpretation_context.go; interpretation_from_column.go; interpretation_library.go; interpretation_preview.go; policy.go; population.go; population_cursor.go; population_member_proposal.go; population_report.go; population_routes.go; preview.go; proposal_receipt.go; publish.go; query.go; repository_publish.go; row_change.go; row_choices.go; row_comparison.go; row_lineage.go; row_proposal.go; row_value_policy.go; schema_field_candidate.go; schema_fields.go; selection.go; semantic_catalog.go; service.go; table_shape_capabilities.go; table_shape_comparison.go; table_shape_proposal.go; table_shape_proposal_composition.go; table_shape_resolution.go; table_shape_saved.go; types.go; unpivot_source.go,artifact_test.go; builder_full_ui_replay_test.go; cell_trace_test.go; construction_category_discovery_errors_test.go; construction_category_discovery_test.go; construction_category_presence_test.go; construction_choice_category_search_test.go; construction_choice_proposal_test.go; construction_choice_test.go; construction_coded_pivot_saved_source_test.go; construction_group_field_choices_test.go; construction_pivot_source_test.go; construction_proposal_test.go; construction_proposal_trace_test.go; construction_related_expand_removal_test.go; construction_routes_test.go; construction_semantic_group_test.go; construction_v9_receipt_test.go; derive_source_test.go; expand_source_test.go; explicit_group_resolver_recovery_test.go; explicit_groups_test.go; filter_source_pivot_clear_test.go; filter_source_test.go; frame_source_test.go; fs1_reverse_order_collision_test.go; interpretation_context_test.go; interpretation_from_column_test.go; interpretation_library_test.go; interpretation_preview_test.go; interpretation_test.go; population_attachment_formal_replay_test.go; population_direction_test.go; population_formal_replay_test.go; population_member_proposal_test.go; population_removal_formal_replay_test.go; population_report_test.go; publish_release_requirements_classification_test.go; repository_publish_activation_conflict_test.go; repository_publish_generation_activation_classification_test.go; repository_publish_release_cas_conflict_test.go; row_change_test.go; row_choices_test.go; row_lineage_test.go; row_proposal_test.go; row_value_policy_test.go; schema_field_candidate_test.go; schema_fields_test.go; selection_test.go; semantic_authoring_test.go; semantic_catalog_test.go; semantic_category_test.go; semantic_coded_concept_test.go; semantic_field_presentation_test.go; semantic_frame_compatibility_test.go; service_test.go; table_shape_capabilities_test.go; table_shape_comparison_evidence_test.go; table_shape_proposal_test.go; unpivot_source_test.go,—,—,25547,31516,319,AcceptedConstruction; ApplyCommands; ApplyCommandsWithConstructionPreview; ApplyCommandsWithConstructionPreviewResponse; ArtifactOpenRequest; ArtifactPublishedReader; ArtifactRequest; ArtifactResult; AssessRowChange; AssessRowChangeRequest; AssessRowChangeResult; AuthorizedCapability; BrowseFrameSourceOptions; BrowseFrameSourceOptionsRequest; BrowseFrameSourceOptionsResponse; BrowseSemanticInventory; BrowseSemanticInventoryRequest; BrowseSemanticInventoryResponse; Builder; BuilderRequest; CandidatePreviewCompleteness; CandidatePreviewCounts; CandidatePreviewState; Capability; CapabilityResolver; CategoryScanner; CellTrace; CellTraceBinding; CellTraceExecutor; CellTraceFeature; CellTraceRequest; CellTraceResult; Clone; ColumnSource; ColumnSourceRequest; ColumnSourceResponse; ColumnSourceRouteStep; CompileReceiptRequest; Config; ConfiguredColumnContext; ConfiguredColumnContextRequest; ConfiguredColumnContextResult; ConfiguredColumnReady; ConfiguredColumnResolution; ConstructionCapabilitiesRequest; ConstructionCapabilitiesResponse; ConstructionCategoryDiscoveryRequest; ConstructionCategoryDiscoveryResponse; ConstructionChoiceProposalRequest; ConstructionChoiceProposalResponse; ConstructionChoiceProposalSelection; ConstructionChoiceSearchRequest; ConstructionChoiceSearchResponse; ConstructionChoiceSearchSource; ConstructionDependencyImpact; ConstructionDependencyIssue; ConstructionDeriveSourceCapability; ConstructionDeriveSourceChoice; ConstructionDeriveSourceFacts; ConstructionDeriveSourceSelection; ConstructionDiscoveredCategory; ConstructionExpandSourceCapability; ConstructionExpandSourceChoice; ConstructionExpandSourceFacts; ConstructionExpandSourceSelection; ConstructionFilterSourceCapability; ConstructionFilterSourceChoice; ConstructionFilterSourceFacts; ConstructionFilterSourceSelection; ConstructionGroupFieldChoice; ConstructionGroupRelatedOccurrenceChoice; ConstructionGroupSourceCapability; ConstructionGroupSourceChoice; ConstructionGroupSourceSelection; ConstructionPivotSourceCapability; ConstructionPivotSourceChoice; ConstructionPivotSourceSelection; ConstructionProposalRequest; ConstructionProposalResponse; ConstructionSemanticGroupChoice; ConstructionSourceProjectionAvailability; ConstructionSourceStageCompiler; ConstructionSourceStageRequest; ConstructionUnpivotSourceCapability; ConstructionUnpivotSourceChoice; ConstructionUnpivotSourceFacts; ConstructionUnpivotSourceSelection; ConstructionWorkspaceInput; ConstructionWorkspaceInputColumn; Create; CreateExplicitGroupRevision; CreateInterpretationRevision; CreateInterpretationRevisionFromColumn; CreateInterpretationRevisionFromColumnRequest; CreateInterpretationRevisionRequest; CreateRequest; CreateSelection; Decode; DiscoverConstructionCategories; DiscoverTableShapeCategories; Encode; Error; ErrorClass; ExactExecutionMaterialization; ExactSemanticInventoryItemIdentity; Execution; ExecutionOutput; ExpandedSelection; ExplicitGroupCreateRequest; ExplicitGroupCreateResult; ExplicitGroupInput; ExplicitGroupRevisionChoice; ExplicitGroupRevisionListRequest; ExplicitGroupRevisionProof; ExplicitGroupRevisionResolveRequest; ExplicitGroupRevisionResolver; ExplicitGroupSelection; ExplicitGroupSummary; FamilySemanticInventoryItemIdentity; FieldGroupSelection; FieldPresentationAccess; FieldPresentationContext; FieldPresentationDecisionKind; FieldPresentationGroupUnderSemantic; FieldPresentationKeepRaw; FieldPresentationReady; FieldPresentationReason; FieldPresentationRequest; FieldPresentationResultKind; FieldPresentationUnavailable; FrameSourceForm; FrameSourceOption; GenerationValidator; Get; GetConstructionCapabilities; GetInterpretationRevision; GetInterpretationRevisionRequest; GetTableShapeCatalog; HMACPopulationMappingCursorCodec; InterpretationCandidatePreviewSample; InterpretationLibrarySummary; InterpretationLibraryView; InterpretationRevisionSummary; List; ListExplicitGroupRevisions; ListInterpretationLibraries; ListInterpretationLibrariesResult; ListResult; ListRowChoices; ListRowDefinitionChoices; New; NewHMACPopulationMappingCursorCodec; NewRepositoryExplicitGroupRevisionResolver; NewSchemaRowChoiceResolver; OpenArtifact; PopulationMapping; PopulationMappingBinding; PopulationMappingCounts; PopulationMappingCursor; PopulationMappingCursorCodec; PopulationMappingDiagnostic; PopulationMappingExecutor; PopulationMappingReport; PopulationMappingRequest; PopulationMappingResult; PopulationMemberRemovalProposalRequest; PopulationMemberRemovalProposalResponse; PopulationRouteChoice; PopulationRoutesRequest; PopulationRoutesResponse; PrepareArtifact; Preview; PreviewInterpretationCandidate; PreviewInterpretationCandidateRequest; PreviewInterpretationCandidateResult; PreviewRequest; PreviewResult; PreviewSinkFactory; ProposeConstruction; ProposeConstructionChoice; ProposePopulationMemberRemoval; ProposeRowDefinition; ProposeTableShape; Publish; PublishRepository; PublishRequest; PublishResult; PublishedOutputIntent; ReadSelection; Ready; Reason; ReceiptCompiler; ReceiptMaterializer; ReceiptPreviewer; ReceiptReader; ReceiptStreamValidator; Reconcile; ReconcileRequest; ReconcileResult; ReconcileWithWorkspace; RelatedExpandChoiceSearchRequest; RelatedExpandChoiceSearchResponse; RelatedExpandContributorChoice; RelatedExpandContributorChoiceSearchRequest; RelatedExpandContributorChoiceSearchResponse; RelatedExpandRouteChoice; RelatedFieldChoice; RelatedFieldChoiceSearchRequest; RelatedFieldChoiceSearchResponse; ReleaseActivator; ReleasePreparer; RepositoryPublishRequest; RepositoryPublishResult; ResolveExplicitGroupRevision; ResolveRowChoiceID; ResolveTableShape; ResolvedPopulationRouteChoice; ResolvedRowChoice; RowChoiceKind; RowChoicePlanner; RowChoiceResolveRequest; RowChoiceResolver; RowDefinitionChoice; RowDefinitionChoicePolicy; RowDefinitionChoicesRequest; RowDefinitionChoicesResponse; RowDefinitionComparison; RowDefinitionComparisonExample; RowDefinitionComparisonStatus; RowDefinitionPreviewSummary; RowDefinitionProposal; RowDefinitionProposalRequest; RowDefinitionSelection; RowDefinitionSelectionKind; RowLineage; RowLineageExecutor; RowLineageRequest; RowLineageResult; SchemaFieldOption; SchemaFieldSearchResponse; SchemaRowChoiceResolver; SearchConstructionChoices; SearchPopulationRoutes; SearchRelatedExpandChoices; SearchRelatedExpandContributorChoices; SearchRelatedFieldChoices; SearchSchemaFields; SearchSchemaFieldsRequest; SelectionCreateResult; SelectionIntentCreateRequest; SelectionPublishedReader; SelectionReadIntentRequest; SelectionReadResult; SelectionReferenceValidator; SelectionSourceIntent; SelectionSourceResolver; SemanticInventoryItem; SemanticInventoryItemIdentityKind; ServerTiming; Service; State; StreamExactExecution; Suggestions; SuggestionsRequest; SuggestionsResult; TableShapeCatalogChoice; TableShapeCatalogColumn; TableShapeCatalogRequest; TableShapeCatalogResult; TableShapeCategoryDiscoveryRequest; TableShapeCategoryDiscoveryResult; TableShapeCategoryValue; TableShapeCellContributor; TableShapeCellTraceEvidence; TableShapeCellTraceState; TableShapeCellValue; TableShapeChangedCell; TableShapeChangedRow; TableShapeColumnChoice; TableShapeComparison; TableShapeComparisonStatus; TableShapeContributor; TableShapeDeclaredInformationLoss; TableShapeDerivedReference; TableShapeDerivedSelection; TableShapeDiscoveredCategory; TableShapeEvidenceLimitation; TableShapeExcludedRecord; TableShapeExclusionEvidence; TableShapeExclusionEvidenceStatus; TableShapeExclusionExecutor; TableShapeInformationLoss; TableShapeInformationLossStatus; TableShapeOperandSelection; TableShapePivotCategorySelection; TableShapePivotSelection; TableShapePostPivotOutput; TableShapePreviewSummary; TableShapeProposal; TableShapeProposalMode; TableShapeProposalRequest; TableShapeReshapeModeChoice; TableShapeResolutionRequest; TableShapeResolutionResult; TableShapeResolvedCategory; TableShapeSavedDerived; TableShapeSavedOperand; TableShapeSavedPivot; TableShapeSavedPivotCategory; TableShapeSavedProposalIntent; TableShapeSavedSupport; TableShapeSavedUnpivot; TableShapeSavedUnpivotInput; TableShapeSourceIdentity; TableShapeUnpivotSelection; UnmarshalJSON; Unwrap; Validate; ValidateCompilationReceipt,internal/authscope; internal/catalog; internal/catalog/arango; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/expression; internal/dataframe/publication; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/spec; internal/dataframe/unit; internal/dataset; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation; internal/explorer/tableshapecap; internal/fhir/schema; internal/projectid,internal/authscope; internal/catalog; internal/catalog/arango; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/expression; internal/dataframe/publication; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataframe/spec; internal/dataset; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation; internal/explorer/tableshapecap; internal/fhir/schema; internal/projectid,internal/server,internal/explorer/arango; internal/explorer/capability,github.com/google/uuid,8 exported interface(s),legacy=43; migration=2; compatibility=13,one production importer (internal/server); inspect ownership/inline potential; broad surface (59 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/explorer/lifecycle,inventory complete; semantic disposition pending -internal/explorer/tableshapecap,github.com/calypr/loom/internal/explorer/tableshapecap,tableshapecap,domain/library,Define immutable table-shape capability and resolution artifacts plus their create-once repository contract.,identity.go; repository.go; types.go,artifacts_test.go,—,—,1971,503,65,AvailabilityState; Binding; BooleanScalar; CanonicalContent; CatalogChoices; CatalogLookup; CatalogReceipt; CategoryProof; CategoryScanReceipt; CategoryValuesDigest; Choice; ChoiceRole; ColumnChoice; DecimalScalar; DecodeStrict; DerivedResolution; DiscoveredCategory; FindCategoryChoice; FindColumn; FindOperand; FindOperator; FindPolicy; FrozenCategory; IntegerScalar; IntentKind; LogicalType; Matches; MissingScalar; NamedOutput; NewCatalogReceipt; NewCategoryScanReceipt; NewPivotDerivedOperandChoiceID; NewResolutionReceipt; NullScalar; OperandChoice; OperandKind; OperandRef; OperatorChoice; PivotDerivedOperand; PivotResolution; PolicyChoice; ProposalIntent; PublicColumn; RemoveIntent; Repository; ResolutionKind; ResolutionReceipt; ResolvedOperand; ResolvedOperandKind; ResolvedUnpivotInput; RoleAvailability; SavedShapeSummary; Scalar; ScalarKind; SetIntent; StringScalar; TypeFact; UnmarshalJSON; UnpivotResolution; Valid; Validate; ValidateAgainstCatalog; ValidateCategoryScanAgainstCatalog; ValidateResolutionAgainstCatalog; ValidateResolutionAgainstCategoryScan,internal/projectid,—,internal/explorer/arango; internal/explorer/lifecycle; internal/server,—,—,1 exported interface(s),—,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/explorer/tableshapecap,inventory complete; semantic disposition pending -internal/fhir/schema,github.com/calypr/loom/internal/fhir/schema,schema,domain/library,"Add handwritten selector, traversal, pivot, and compiler semantics to generated FHIR metadata.",compiler_semantics.go; correlated.go; fields.go; generated_metadata.go; identifier.go; index.go; pivots.go; quantity.go; row_resolution.go; selectors.go; structure.go; terminal_metadata.go; traversals.go,compiler_semantics_test.go; correlated_selection_test.go; correlated_test.go; fields_test.go; identifier_test.go; index_test.go; pivots_test.go; root_resources_test.go; row_resolution_test.go; selectors_test.go; structure_test.go; terminal_metadata_test.go; traversals_test.go,—,—,2446,1341,86,ArrayElementType; ArrayElements; At; BindingStrength; CanonicalPath; CanonicalResourceType; CanonicalizePath; ChoiceValueSelectorOptions; CompilerTraversal; ConcreteResourceType; ContainsFilter; CorrelatedBinding; CorrelatedBindingSpec; CorrelatedKey; CorrelatedSelectionScope; CorrelatedValueScope; DefaultPivotSpec; Definition; DefinitionExists; DefinitionName; DefinitionStructure; DefinitionView; Definitions; Element; ElementListView; ElementView; Elements; ExtensionBinding; ExtensionBindingSpec; FieldKind; FieldPredicateSpec; FieldSelectorSpec; FieldSelectorSpecFromPath; FieldSemantics; FieldSpec; FieldsForResource; GeneratedIndex; HasRepeatedCodingPath; HasResource; IdentifierBinding; IdentifierBindingSpec; Index; ItemJSONType; JSONType; Len; LookupField; LookupTraversal; Name; NewIndex; ParseSelector; PivotSpec; PrimitiveKind; QuantityIdentityPaths; ReadDefinition; ReferencedType; RepeatedCodingPaths; RequiredElements; ResolveCompilerTraversal; ResolveFieldSemantics; ResolvePath; ResolveRowPath; ResolveTerminalScalarMetadata; ResolvedPath; ResolvesToCodeableConcept; ResourceSpec; ResourceType; ResourceTypes; RowCardinality; RowPathFacts; RowPathShape; Selector; SelectorExpression; SelectorFromField; SelectorStep; Snapshot; Structure; StructureMember; TerminalScalarMetadata; TraversalDirection; TraversalMultiplicity; TraversalSpec; ValidateCorrelatedBinding; ValidateCorrelatedKey; ValidateExtensionBinding; ValidateIdentifierBinding; ValidatePivotSelectors,generated/fhirschema,—,cmd/arango-fhir-proto; generated/fhir; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/catalog; internal/catalog/arango; internal/dataframe/compiler/capability; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/recipe; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/dataframe/spec; internal/explorer/authoringv2; internal/explorer/compilation; internal/explorer/lifecycle; internal/fhir/semantic; internal/ingest; internal/server; scripts,conformance/compiler; internal/dataframe/compiler,—,—,legacy=1; compatibility=2,broad surface (13 production files); inspect cohesion,go test ./internal/fhir/schema,inventory complete; semantic disposition pending +internal/explorer/authoringv2,github.com/calypr/loom/internal/explorer/authoringv2,authoringv2,domain/library,"Define, validate, canonicalize, and mutate the V2 Explorer Builder authoring workspace.",availability.go; canonical.go; commands.go; construction.go; construction_change.go; construction_choice.go; construction_migration.go; construction_row_values.go; construction_source_children.go; construction_validation.go; framing.go; migration.go; route_identity.go; row_change.go; row_definition.go; semantic_command.go; semantic_types.go; table_shape.go; transformations.go; types.go; validation.go; value_transformations.go; workspace_output_dependencies.go,commands_test.go; compile_preparation_test.go; construction_choice_test.go; construction_combine_test.go; construction_expand_test.go; construction_formal_replay_test.go; construction_group_coded_pivot_test.go; construction_group_field_source_test.go; construction_projection_replay_test.go; construction_proposal_test.go; construction_related_contributor_test.go; construction_row_values_test.go; construction_source_children_test.go; construction_test.go; correlated_test.go; framing_test.go; interpretation_test.go; migration_test.go; population_member_proposal_test.go; row_change_test.go; row_definition_test.go; row_proposal_test.go; semantic_codeable_concept_fallback_test.go; semantic_coded_category_test.go; semantic_command_test.go; table_shape_proposal_test.go; table_shape_test.go; transformations_test.go; types_test.go; value_transformations_test.go; workspace_output_dependencies_test.go; workspace_test.go,—,—,13260,8548,231,Action; ActionColumn; Addable; AggregateSource; AggregateTransformationCapabilities; AggregateTransformationCapabilitiesForCapability; AggregateTransformationCapabilitiesForCatalog; AnalyzeConstructionCandidate; AnalyzeStepEdit; ApplyCommands; ApplyCommandsRequest; ApplyCommandsResponse; ApplyInterpretationCandidate; ApplyRowChange; ApplyRowReplacement; ArithmeticOperand; AssessRowChange; AssessRowReplacement; BuilderState; CanPopulateConstructionRows; CanonicalJSON; CatalogAvailability; CatalogAvailabilityReadiness; CatalogAvailabilitySourceReadiness; CatalogCandidate; CatalogDiagnostic; CatalogEdge; CatalogNode; CatalogObservedField; CatalogObservedRelationship; CatalogObservedResource; CatalogSnapshot; ChartPresentation; CodeValue; Column; ColumnOutput; ColumnSource; ColumnTransformationChange; ColumnValueTransformationCapabilities; ColumnValueTransformationCapabilitiesForCatalog; Command; CommandResult; ConceptCandidate; Construction; ConstructionCanCarrySourceProjection; ConstructionChoiceSelection; ConstructionCodedGroup; ConstructionCodedGroupSource; ConstructionCodedPivot; ConstructionCodedPivotCategory; ConstructionCodedPivotSource; ConstructionCombine; ConstructionCombineJoinType; ConstructionCombineKey; ConstructionCombineKind; ConstructionCombineMembershipMode; ConstructionCombineProjection; ConstructionCombineRightMatchPolicy; ConstructionDependencyIssue; ConstructionDerive; ConstructionDerivedOperation; ConstructionDivisionByZeroPolicy; ConstructionExpand; ConstructionExpandEmptyPolicy; ConstructionExpandedFieldSource; ConstructionFilter; ConstructionFilterOperator; ConstructionFilterValueKind; ConstructionGroup; ConstructionGroupAggregate; ConstructionGroupAggregateOp; ConstructionGroupFieldCurrentStage; ConstructionGroupFieldRelatedOccurrence; ConstructionGroupFieldRelatedRouteStep; ConstructionGroupFieldSource; ConstructionGroupFieldSourceInput; ConstructionGroupKey; ConstructionGroupMissingKeyPolicy; ConstructionGroupSemanticCategory; ConstructionID; ConstructionImpact; ConstructionInputKind; ConstructionInputRef; ConstructionLiteral; ConstructionMissingInputPolicy; ConstructionNumericKind; ConstructionOperand; ConstructionOperandKind; ConstructionOperation; ConstructionOperationKind; ConstructionOutputPresentation; ConstructionPivot; ConstructionPivotCategory; ConstructionPivotDuplicatePolicy; ConstructionPivotMissingCellPolicy; ConstructionPivotUnlistedCategoryPolicy; ConstructionRelatedContributorRule; ConstructionRelatedEligibility; ConstructionRelatedEligibilityMatch; ConstructionRelatedExpand; ConstructionRelatedField; ConstructionRelatedFieldSource; ConstructionRelatedSource; ConstructionRowValue; ConstructionRowValuePolicy; ConstructionSourceProjection; ConstructionSourceProjectionName; ConstructionStep; ConstructionUnpivot; ConstructionUnpivotInput; ConstructionUnpivotNullPolicy; ContributorCode; ContributorOperator; ContributorPredicate; ContributorQuantifier; ContributorValue; ContributorValueKind; ContributorWindowSource; DecodeWorkspace; DerivedConstruction; Digest; Document; DraftSourceContext; EmptyCollectionPolicy; ExpandedRows; ExplicitGroupRowValue; ExplicitGroupSource; ExplorerMetadata; FeatureInterpretation; FeatureInterpretationKind; FieldGroupSource; FieldPath; FieldSource; FileActions; FilterPresentation; FilterValue; FixedFilter; FrameDefinition; FrameManyPolicy; FrameZeroPolicy; GroupSource; GroupSourceKind; GroupedRows; HasMissingInputs; LegacyNilConstructionStepsDigest; LookupMatch; LookupSource; MarshalJSON; MigrateAggregateTemporalPolicy; MigrateLegacyContributors; MigrateLosslessDefaults; MissingKeyPolicy; NewFrameDefinition; NormalizePresentationOrders; Normalized; Output; OutputDependencyOrder; OwnerRecordsSource; PinnedInterpretation; PivotCategory; PivotConstruction; Population; PopulationRouteStep; PrepareWorkspaceForCompilation; ProjectionMode; ProposeStepRemoval; RecordRows; RecordsRowDefinition; RelatedSelection; RepeatedBoundary; ResolveConstructionProposal; ResolveDraftRevision; ResolveDraftSourceContext; ResolveFrameSource; ResolvePopulationMemberProposal; ResolveRowDefinitionProposal; ResolveSemanticSelectionPlan; ResolveTableShapeProposal; ResolvedConstructionChoice; ResolvedDraftSourceContext; RouteMatchMode; RouteNode; RoutePolicy; RouteRebaseChoice; RowChangeAssessment; RowChangeProposal; RowChangeRequest; RowChangeUnresolvedReference; RowDefinition; RowDefinitionKind; SemanticReadinessStatus; SemanticSelection; SemanticSelectionPlan; SemanticSelectionReadiness; SemanticSelectionResult; SharedFilterBinding; SharedFilterBindingRef; SourceWhere; StageColumn; SupportsAnchor; SupportsTimestamp; Tab; TablePresentation; TableReshape; TableScalar; TableScalarKind; TableShape; TemporalFieldChoice; TemporalOrderingSource; TemporalReductionCapabilities; UnassignedMemberPolicy; UnitNormalizationCapabilities; UnitNormalizationPolicy; UnitNormalizationPresetCapability; UnmarshalJSON; UnpivotConstruction; UnpivotInput; UpgradeDocumentToConstruction; Valid; Validate; ValidateConcreteValue; ValidateConstructionExpandedFieldSource; ValidateContributorForCatalog; ValidateForPublication; ValidatePivotCategoryKey; ValidateStructure; ValidateTableShapeOutput; ValidateWithExplicitGroups; ValueTransformationCapability; Workspace; WorkspaceOutputConsumer,internal/catalog; internal/dataframe/columntransform; internal/dataframe/lineage; internal/dataframe/unit; internal/explorer/capability; internal/fhir/schema,internal/catalog; internal/dataframe/columntransform; internal/dataframe/lineage; internal/explorer/capability; internal/fhir/schema,cmd/explorer-config-v2-convert; cmd/preview-covering-index-artifact; generated/loomapi; internal/acceptance; internal/backends/arango/internal/explorer; internal/explorer; internal/explorer/compilation; internal/explorer/lifecycle; internal/repositoryseed; internal/server,internal/backends/postgres/internal/explorer; internal/explorer/capability,—,—,legacy=50; migration=28; compatibility=3,broad surface (23 production files); inspect cohesion,go test ./internal/explorer/authoringv2,inventory complete; semantic disposition pending +internal/explorer/capability,github.com/calypr/loom/internal/explorer/capability,capability,domain/library,Build the immutable compiler-backed capability contract exposed to Explorer Builder.,builder.go; construction.go; construction_routes.go; domain.go; population_route_choice.go; repository.go; row_choice.go; value_operations.go,builder_test.go; construction_test.go; domain_test.go; row_choice_test.go; semantic_fallback_dto_test.go; value_operations_test.go,—,—,3950,1396,110,AggregateInput; AggregateOperation; AggregateOperationCapability; AggregateRowContext; Allows; Availability; AvailabilityReadiness; AvailabilitySourceReadiness; Build; Builder; Candidate; CandidateProof; CanonicalPayload; ChartAggregation; Clone; CodedGroupChoiceSource; CompilerCallbacks; ConceptCandidate; ConstructionChoice; ConstructionChoiceDecision; ConstructionChoiceFact; ConstructionChoiceForm; ConstructionChoiceIdentity; ConstructionChoiceOption; ConstructionChoicePredicateOperator; ConstructionChoicePresentation; ConstructionChoicePreservation; ConstructionChoiceRowEffect; ConstructionChoiceShape; ConstructionChoiceSource; ConstructionChoiceSourceKind; ConstructionChoiceSupport; ConstructionCodedGroupChoice; ConstructionRelatedFieldChoice; ConstructionRelatedResourceRouteChoice; ConstructionRoutePage; ConstructionRouteSearch; ConstructionRouteStep; DecodeConstructionChoiceID; DecodePopulationRouteChoiceID; DecodeRowChoiceID; DeriveAggregateOperationCapabilities; Diagnostic; Edge; EdgeProof; Evidence; FieldChoiceSource; FieldObservation; FilterOperator; GroupFieldChoiceInput; GroupFieldChoiceSource; GroupFieldCurrentStageInput; GroupFieldRelatedOccurrenceInput; GroupFieldRelatedRouteStep; IsRepeatedCardinality; IsSupportedConstructionSourceType; IsUsable; NewBuilder; NewConstructionCodedGroupChoice; NewConstructionRelatedFieldChoice; NewConstructionRelatedResourceRouteChoiceFromAnchor; NewFieldConstructionChoice; NewFieldConstructionChoiceForRoute; NewGroupCodingIdentityChoice; NewGroupFieldConstructionChoice; NewPopulationRouteChoiceID; NewRowChoice; NewSemanticConstructionChoice; NewSemanticConstructionChoiceForRoute; NewSemanticFrameConstructionChoice; NewSnapshot; Node; NodeProof; ObservedField; ObservedRelationship; ObservedResource; Operation; PlanConstructionRoutes; Policy; PopulationRouteChoiceIdentity; ProjectionMode; ProjectionPolicy; RelatedFieldChoiceSource; RelatedResourceChoiceSource; RelationshipObservation; RepeatedBoundary; RepeatedCoordinate; Repository; ResolveRowChoiceID; ResourceObservation; RoutePolicy; RowChoice; RowChoiceCardinality; RowChoiceFacts; RowChoiceIdentity; RowChoiceKind; RowChoiceOccurrence; RowChoicePathResolver; RowChoiceShape; SemanticBindingChoiceSource; SemanticFrameChoiceSource; SemanticFrameFamily; Snapshot; SnapshotIdentity; Status; UnmarshalJSON; Usable; ValidateConstructionSourceType; ValidateSnapshot; ValidateToken,internal/dataframe/recipe,generated/loomapi; internal/catalog; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/lifecycle,cmd/preview-covering-index-artifact; internal/backends/arango/internal/explorer; internal/backends/postgres/internal/explorer; internal/explorer; internal/explorer/authoringv2; internal/explorer/compilation; internal/explorer/lifecycle; internal/server,internal/dataframe/compiler; internal/dataframe/execution; internal/explorer/capability,—,2 exported interface(s),legacy=2,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/explorer/capability,inventory complete; semantic disposition pending +internal/explorer/compilation,github.com/calypr/loom/internal/explorer/compilation,compilation,domain/library,Translate validated Explorer authoring intent into native recipe and compilation artifacts.,compile.go; construction.go; derived_columns.go; indexed_projection.go; interpretation.go; row_expansion.go; semantic_compile.go; source_row.go; table_reshape.go,acceptance_observation_aql_diagnostic_test.go; command_helpers_test.go; construction_combine_test.go; construction_related_contributor_test.go; construction_test.go; correlated_test.go; derived_columns_test.go; indexed_projection_test.go; interpretation_test.go; row_expansion_test.go; semantic_compile_expand_source_test.go; source_row_test.go; table_reshape_test.go; translate_test.go; value_transformations_test.go; workspace_output_mapping_test.go; workspace_test.go,—,—,3622,4536,24,Canonical; Compile; CompileWorkspace; Error; InterpretationCandidateMatch; InterpretationCandidateResolution; InterpretationCandidateResolutionState; Match; PopulationFor; PresentationColumn; PresentationConfig; Reason; ResolveAddressableSourceRow; ResolveInterpretationCandidate; ResolveWorkspaceInterpretations; ResolvedInputs; ResolvedInputsDigest; ResolvedInterpretation; ResolvedPopulation; Result; SourceRowDescriptor; State; Unwrap; WorkspaceResult,internal/dataframe/compiler/lower; internal/dataframe/lineage; internal/dataframe/recipe; internal/dataframe/spec; internal/dataframe/unit; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/fhir/schema; internal/projectid,internal/backends/arango; internal/catalog; internal/dataframe/columntransform; internal/dataframe/compiler; internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/lineage; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/fhir/schema,cmd/preview-covering-index-artifact; internal/explorer/lifecycle; internal/server,—,—,1 exported interface(s),legacy=1,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/explorer/compilation,inventory complete; semantic disposition pending +internal/explorer/lifecycle,github.com/calypr/loom/internal/explorer/lifecycle,lifecycle,domain/library,"Coordinate transport-neutral Explorer authoring, preview, publication, query, and policy operations.",artifact.go; authoring.go; cell_trace.go; column_source.go; construction_category_discovery.go; construction_choice.go; construction_choice_proposal.go; construction_coded_pivot.go; construction_group_field_choices.go; construction_group_related_field_choices.go; construction_group_source.go; construction_pivot_source.go; construction_proposal.go; construction_routes.go; construction_semantic_group.go; derive_source.go; draft_restore.go; errors.go; expand_source.go; explicit_group_resolver.go; explicit_groups.go; filter_source.go; frame_source.go; interpretation.go; interpretation_apply.go; interpretation_context.go; interpretation_from_column.go; interpretation_library.go; interpretation_preview.go; policy.go; population.go; population_cursor.go; population_member_proposal.go; population_report.go; population_routes.go; preview.go; proposal_receipt.go; publish.go; query.go; repository_publish.go; row_change.go; row_choices.go; row_comparison.go; row_lineage.go; row_proposal.go; row_value_policy.go; schema_field_candidate.go; schema_fields.go; selection.go; semantic_catalog.go; service.go; table_shape_capabilities.go; table_shape_comparison.go; table_shape_proposal.go; table_shape_proposal_composition.go; table_shape_resolution.go; table_shape_saved.go; types.go; unpivot_source.go,artifact_test.go; builder_full_ui_replay_test.go; cell_trace_test.go; construction_category_discovery_errors_test.go; construction_category_discovery_test.go; construction_category_presence_test.go; construction_choice_category_search_test.go; construction_choice_proposal_test.go; construction_choice_test.go; construction_coded_pivot_saved_source_test.go; construction_group_field_choices_test.go; construction_pivot_source_test.go; construction_proposal_test.go; construction_proposal_trace_test.go; construction_related_expand_removal_test.go; construction_routes_test.go; construction_semantic_group_test.go; construction_v9_receipt_test.go; derive_source_test.go; expand_source_test.go; explicit_group_resolver_recovery_test.go; explicit_groups_test.go; filter_source_pivot_clear_test.go; filter_source_test.go; frame_source_test.go; fs1_reverse_order_collision_test.go; interpretation_context_test.go; interpretation_from_column_test.go; interpretation_library_test.go; interpretation_preview_test.go; interpretation_test.go; population_attachment_formal_replay_test.go; population_direction_test.go; population_formal_replay_test.go; population_member_proposal_test.go; population_removal_formal_replay_test.go; population_report_test.go; publish_release_requirements_classification_test.go; repository_publish_activation_conflict_test.go; repository_publish_generation_activation_classification_test.go; repository_publish_release_cas_conflict_test.go; row_change_test.go; row_choices_test.go; row_lineage_test.go; row_proposal_test.go; row_value_policy_test.go; schema_field_candidate_test.go; schema_fields_test.go; selection_test.go; semantic_authoring_test.go; semantic_catalog_test.go; semantic_category_test.go; semantic_coded_concept_test.go; semantic_field_presentation_test.go; semantic_frame_compatibility_test.go; service_test.go; table_shape_capabilities_test.go; table_shape_comparison_evidence_test.go; table_shape_proposal_test.go; unpivot_source_test.go,—,—,25545,31520,319,AcceptedConstruction; ApplyCommands; ApplyCommandsWithConstructionPreview; ApplyCommandsWithConstructionPreviewResponse; ArtifactOpenRequest; ArtifactPublishedReader; ArtifactRequest; ArtifactResult; AssessRowChange; AssessRowChangeRequest; AssessRowChangeResult; AuthorizedCapability; BrowseFrameSourceOptions; BrowseFrameSourceOptionsRequest; BrowseFrameSourceOptionsResponse; BrowseSemanticInventory; BrowseSemanticInventoryRequest; BrowseSemanticInventoryResponse; Builder; BuilderRequest; CandidatePreviewCompleteness; CandidatePreviewCounts; CandidatePreviewState; Capability; CapabilityResolver; CategoryScanner; CellTrace; CellTraceBinding; CellTraceExecutor; CellTraceFeature; CellTraceRequest; CellTraceResult; Clone; ColumnSource; ColumnSourceRequest; ColumnSourceResponse; ColumnSourceRouteStep; CompileReceiptRequest; Config; ConfiguredColumnContext; ConfiguredColumnContextRequest; ConfiguredColumnContextResult; ConfiguredColumnReady; ConfiguredColumnResolution; ConstructionCapabilitiesRequest; ConstructionCapabilitiesResponse; ConstructionCategoryDiscoveryRequest; ConstructionCategoryDiscoveryResponse; ConstructionChoiceProposalRequest; ConstructionChoiceProposalResponse; ConstructionChoiceProposalSelection; ConstructionChoiceSearchRequest; ConstructionChoiceSearchResponse; ConstructionChoiceSearchSource; ConstructionDependencyImpact; ConstructionDependencyIssue; ConstructionDeriveSourceCapability; ConstructionDeriveSourceChoice; ConstructionDeriveSourceFacts; ConstructionDeriveSourceSelection; ConstructionDiscoveredCategory; ConstructionExpandSourceCapability; ConstructionExpandSourceChoice; ConstructionExpandSourceFacts; ConstructionExpandSourceSelection; ConstructionFilterSourceCapability; ConstructionFilterSourceChoice; ConstructionFilterSourceFacts; ConstructionFilterSourceSelection; ConstructionGroupFieldChoice; ConstructionGroupRelatedOccurrenceChoice; ConstructionGroupSourceCapability; ConstructionGroupSourceChoice; ConstructionGroupSourceSelection; ConstructionPivotSourceCapability; ConstructionPivotSourceChoice; ConstructionPivotSourceSelection; ConstructionProposalRequest; ConstructionProposalResponse; ConstructionSemanticGroupChoice; ConstructionSourceProjectionAvailability; ConstructionSourceStageCompiler; ConstructionSourceStageRequest; ConstructionUnpivotSourceCapability; ConstructionUnpivotSourceChoice; ConstructionUnpivotSourceFacts; ConstructionUnpivotSourceSelection; ConstructionWorkspaceInput; ConstructionWorkspaceInputColumn; Create; CreateExplicitGroupRevision; CreateInterpretationRevision; CreateInterpretationRevisionFromColumn; CreateInterpretationRevisionFromColumnRequest; CreateInterpretationRevisionRequest; CreateRequest; CreateSelection; Decode; DiscoverConstructionCategories; DiscoverTableShapeCategories; Encode; Error; ErrorClass; ExactExecutionMaterialization; ExactSemanticInventoryItemIdentity; Execution; ExecutionOutput; ExpandedSelection; ExplicitGroupCreateRequest; ExplicitGroupCreateResult; ExplicitGroupInput; ExplicitGroupRevisionChoice; ExplicitGroupRevisionListRequest; ExplicitGroupRevisionProof; ExplicitGroupRevisionResolveRequest; ExplicitGroupRevisionResolver; ExplicitGroupSelection; ExplicitGroupSummary; FamilySemanticInventoryItemIdentity; FieldGroupSelection; FieldPresentationAccess; FieldPresentationContext; FieldPresentationDecisionKind; FieldPresentationGroupUnderSemantic; FieldPresentationKeepRaw; FieldPresentationReady; FieldPresentationReason; FieldPresentationRequest; FieldPresentationResultKind; FieldPresentationUnavailable; FrameSourceForm; FrameSourceOption; GenerationValidator; Get; GetConstructionCapabilities; GetInterpretationRevision; GetInterpretationRevisionRequest; GetTableShapeCatalog; HMACPopulationMappingCursorCodec; InterpretationCandidatePreviewSample; InterpretationLibrarySummary; InterpretationLibraryView; InterpretationRevisionSummary; List; ListExplicitGroupRevisions; ListInterpretationLibraries; ListInterpretationLibrariesResult; ListResult; ListRowChoices; ListRowDefinitionChoices; New; NewHMACPopulationMappingCursorCodec; NewRepositoryExplicitGroupRevisionResolver; NewSchemaRowChoiceResolver; OpenArtifact; PopulationMapping; PopulationMappingBinding; PopulationMappingCounts; PopulationMappingCursor; PopulationMappingCursorCodec; PopulationMappingDiagnostic; PopulationMappingExecutor; PopulationMappingReport; PopulationMappingRequest; PopulationMappingResult; PopulationMemberRemovalProposalRequest; PopulationMemberRemovalProposalResponse; PopulationRouteChoice; PopulationRoutesRequest; PopulationRoutesResponse; PrepareArtifact; Preview; PreviewInterpretationCandidate; PreviewInterpretationCandidateRequest; PreviewInterpretationCandidateResult; PreviewRequest; PreviewResult; PreviewSinkFactory; ProposeConstruction; ProposeConstructionChoice; ProposePopulationMemberRemoval; ProposeRowDefinition; ProposeTableShape; Publish; PublishRepository; PublishRequest; PublishResult; PublishedOutputIntent; ReadSelection; Ready; Reason; ReceiptCompiler; ReceiptMaterializer; ReceiptPreviewer; ReceiptReader; ReceiptStreamValidator; Reconcile; ReconcileRequest; ReconcileResult; ReconcileWithWorkspace; RelatedExpandChoiceSearchRequest; RelatedExpandChoiceSearchResponse; RelatedExpandContributorChoice; RelatedExpandContributorChoiceSearchRequest; RelatedExpandContributorChoiceSearchResponse; RelatedExpandRouteChoice; RelatedFieldChoice; RelatedFieldChoiceSearchRequest; RelatedFieldChoiceSearchResponse; ReleaseActivator; ReleasePreparer; RepositoryPublishRequest; RepositoryPublishResult; ResolveExplicitGroupRevision; ResolveRowChoiceID; ResolveTableShape; ResolvedPopulationRouteChoice; ResolvedRowChoice; RowChoiceKind; RowChoicePlanner; RowChoiceResolveRequest; RowChoiceResolver; RowDefinitionChoice; RowDefinitionChoicePolicy; RowDefinitionChoicesRequest; RowDefinitionChoicesResponse; RowDefinitionComparison; RowDefinitionComparisonExample; RowDefinitionComparisonStatus; RowDefinitionPreviewSummary; RowDefinitionProposal; RowDefinitionProposalRequest; RowDefinitionSelection; RowDefinitionSelectionKind; RowLineage; RowLineageExecutor; RowLineageRequest; RowLineageResult; SchemaFieldOption; SchemaFieldSearchResponse; SchemaRowChoiceResolver; SearchConstructionChoices; SearchPopulationRoutes; SearchRelatedExpandChoices; SearchRelatedExpandContributorChoices; SearchRelatedFieldChoices; SearchSchemaFields; SearchSchemaFieldsRequest; SelectionCreateResult; SelectionIntentCreateRequest; SelectionPublishedReader; SelectionReadIntentRequest; SelectionReadResult; SelectionReferenceValidator; SelectionSourceIntent; SelectionSourceResolver; SemanticInventoryItem; SemanticInventoryItemIdentityKind; ServerTiming; Service; State; StreamExactExecution; Suggestions; SuggestionsRequest; SuggestionsResult; TableShapeCatalogChoice; TableShapeCatalogColumn; TableShapeCatalogRequest; TableShapeCatalogResult; TableShapeCategoryDiscoveryRequest; TableShapeCategoryDiscoveryResult; TableShapeCategoryValue; TableShapeCellContributor; TableShapeCellTraceEvidence; TableShapeCellTraceState; TableShapeCellValue; TableShapeChangedCell; TableShapeChangedRow; TableShapeColumnChoice; TableShapeComparison; TableShapeComparisonStatus; TableShapeContributor; TableShapeDeclaredInformationLoss; TableShapeDerivedReference; TableShapeDerivedSelection; TableShapeDiscoveredCategory; TableShapeEvidenceLimitation; TableShapeExcludedRecord; TableShapeExclusionEvidence; TableShapeExclusionEvidenceStatus; TableShapeExclusionExecutor; TableShapeInformationLoss; TableShapeInformationLossStatus; TableShapeOperandSelection; TableShapePivotCategorySelection; TableShapePivotSelection; TableShapePostPivotOutput; TableShapePreviewSummary; TableShapeProposal; TableShapeProposalMode; TableShapeProposalRequest; TableShapeReshapeModeChoice; TableShapeResolutionRequest; TableShapeResolutionResult; TableShapeResolvedCategory; TableShapeSavedDerived; TableShapeSavedOperand; TableShapeSavedPivot; TableShapeSavedPivotCategory; TableShapeSavedProposalIntent; TableShapeSavedSupport; TableShapeSavedUnpivot; TableShapeSavedUnpivotInput; TableShapeSourceIdentity; TableShapeUnpivotSelection; UnmarshalJSON; Unwrap; Validate; ValidateCompilationReceipt,internal/authscope; internal/catalog; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/expression; internal/dataframe/publication; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/spec; internal/dataframe/unit; internal/dataset; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation; internal/explorer/tableshapecap; internal/fhir/schema; internal/projectid,internal/authscope; internal/catalog; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/expression; internal/dataframe/publication; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataframe/spec; internal/dataset; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation; internal/explorer/tableshapecap; internal/fhir/schema; internal/projectid,internal/server,internal/backends/arango/internal/explorer; internal/dataframe/execution; internal/explorer/capability,github.com/google/uuid,8 exported interface(s),legacy=43; migration=2; compatibility=13,one production importer (internal/server); inspect ownership/inline potential; broad surface (59 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/explorer/lifecycle,inventory complete; semantic disposition pending +internal/explorer/tableshapecap,github.com/calypr/loom/internal/explorer/tableshapecap,tableshapecap,domain/library,Define immutable table-shape capability and resolution artifacts plus their create-once repository contract.,identity.go; repository.go; types.go,artifacts_test.go,—,—,1971,503,65,AvailabilityState; Binding; BooleanScalar; CanonicalContent; CatalogChoices; CatalogLookup; CatalogReceipt; CategoryProof; CategoryScanReceipt; CategoryValuesDigest; Choice; ChoiceRole; ColumnChoice; DecimalScalar; DecodeStrict; DerivedResolution; DiscoveredCategory; FindCategoryChoice; FindColumn; FindOperand; FindOperator; FindPolicy; FrozenCategory; IntegerScalar; IntentKind; LogicalType; Matches; MissingScalar; NamedOutput; NewCatalogReceipt; NewCategoryScanReceipt; NewPivotDerivedOperandChoiceID; NewResolutionReceipt; NullScalar; OperandChoice; OperandKind; OperandRef; OperatorChoice; PivotDerivedOperand; PivotResolution; PolicyChoice; ProposalIntent; PublicColumn; RemoveIntent; Repository; ResolutionKind; ResolutionReceipt; ResolvedOperand; ResolvedOperandKind; ResolvedUnpivotInput; RoleAvailability; SavedShapeSummary; Scalar; ScalarKind; SetIntent; StringScalar; TypeFact; UnmarshalJSON; UnpivotResolution; Valid; Validate; ValidateAgainstCatalog; ValidateCategoryScanAgainstCatalog; ValidateResolutionAgainstCatalog; ValidateResolutionAgainstCategoryScan,internal/projectid,—,internal/backends/arango/internal/explorer; internal/backends/postgres/internal/explorer; internal/explorer/lifecycle; internal/server,—,—,1 exported interface(s),—,runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/explorer/tableshapecap,inventory complete; semantic disposition pending +internal/fhir/schema,github.com/calypr/loom/internal/fhir/schema,schema,domain/library,"Add handwritten selector, traversal, pivot, and compiler semantics to generated FHIR metadata.",compiler_semantics.go; correlated.go; fields.go; generated_metadata.go; identifier.go; index.go; pivots.go; quantity.go; row_resolution.go; selectors.go; structure.go; terminal_metadata.go; traversals.go,compiler_semantics_test.go; correlated_selection_test.go; correlated_test.go; fields_test.go; identifier_test.go; index_test.go; pivots_test.go; root_resources_test.go; row_resolution_test.go; selectors_test.go; structure_test.go; terminal_metadata_test.go; traversals_test.go,—,—,2446,1341,86,ArrayElementType; ArrayElements; At; BindingStrength; CanonicalPath; CanonicalResourceType; CanonicalizePath; ChoiceValueSelectorOptions; CompilerTraversal; ConcreteResourceType; ContainsFilter; CorrelatedBinding; CorrelatedBindingSpec; CorrelatedKey; CorrelatedSelectionScope; CorrelatedValueScope; DefaultPivotSpec; Definition; DefinitionExists; DefinitionName; DefinitionStructure; DefinitionView; Definitions; Element; ElementListView; ElementView; Elements; ExtensionBinding; ExtensionBindingSpec; FieldKind; FieldPredicateSpec; FieldSelectorSpec; FieldSelectorSpecFromPath; FieldSemantics; FieldSpec; FieldsForResource; GeneratedIndex; HasRepeatedCodingPath; HasResource; IdentifierBinding; IdentifierBindingSpec; Index; ItemJSONType; JSONType; Len; LookupField; LookupTraversal; Name; NewIndex; ParseSelector; PivotSpec; PrimitiveKind; QuantityIdentityPaths; ReadDefinition; ReferencedType; RepeatedCodingPaths; RequiredElements; ResolveCompilerTraversal; ResolveFieldSemantics; ResolvePath; ResolveRowPath; ResolveTerminalScalarMetadata; ResolvedPath; ResolvesToCodeableConcept; ResourceSpec; ResourceType; ResourceTypes; RowCardinality; RowPathFacts; RowPathShape; Selector; SelectorExpression; SelectorFromField; SelectorStep; Snapshot; Structure; StructureMember; TerminalScalarMetadata; TraversalDirection; TraversalMultiplicity; TraversalSpec; ValidateCorrelatedBinding; ValidateCorrelatedKey; ValidateExtensionBinding; ValidateIdentifierBinding; ValidatePivotSelectors,generated/fhirschema,—,cmd/arango-fhir-proto; generated/fhir; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/backends/arango/internal/catalog; internal/backends/postgres/internal/catalog; internal/catalog; internal/dataframe/compiler/capability; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/postgres; internal/dataframe/recipe; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/dataframe/spec; internal/explorer/authoringv2; internal/explorer/compilation; internal/explorer/lifecycle; internal/fhir/semantic; internal/ingest; internal/server; scripts,conformance/compiler; internal/dataframe/compiler,—,—,legacy=1; compatibility=2,broad surface (13 production files); inspect cohesion,go test ./internal/fhir/schema,inventory complete; semantic disposition pending internal/fhir/semantic,github.com/calypr/loom/internal/fhir/semantic,semantic,domain/library,Walk generated FHIR definition indexes to enumerate typed referenced datatype occurrences.,member_walk.go; walk.go,member_walk_resource_dispatch_test.go; member_walk_test.go; walk_test.go,—,—,829,949,17,Error; Len; Less; MemberFact; Occurrence; Pop; Push; RepeatedBoundary; ResourceBoundary; Swap; Walk; WalkDiagnostic; WalkDiagnosticCode; WalkLimitError; WalkLimitKind; WalkMembers; WalkMembersWithResourceTypes,internal/fhir/schema,internal/fhir/schema,internal/catalog,—,—,—,—,one production importer (internal/catalog); inspect ownership/inline potential,go test ./internal/fhir/semantic,inventory complete; semantic disposition pending -internal/ingest,github.com/calypr/loom/internal/ingest,ingest,domain/library,"Preflight and load FHIR NDJSON into generation-scoped Arango vertices, edges, and catalog facts.",backend.go; doc.go; documents.go; files.go; generated_load.go; generation_identity.go; generation_orchestration.go; generation_workers.go; load_options.go; preflight.go; progress.go; resource_load.go; row_builder.go; semantic_inventory_backfill.go; source_schema.go,backend_test.go; basic_observation_fixture_validation_test.go; bench_test.go; documents_test.go; file_pipeline_test.go; files_test.go; generated_load_test.go; generation_cleanup_test.go; generation_failed_cleanup_e2e_test.go; generation_identity_test.go; generation_load_test.go; integration_test.go; keycollide_generated_e2e_test.go; load_options_test.go; preflight_test.go; related_medication_fixture_test.go; resource_load_test.go; row_builder_test.go; semantic_inventory_backfill_integration_test.go; semantic_inventory_backfill_test.go; semantic_inventory_test.go; source_schema_test.go; viewer_row_file_download_fixture_test.go,—,—,2944,3807,37,ActivationOutcomeError; BackfillSemanticInventory; Build; DiscoverNDJSON; EdgeDocument; EdgeFromGrip; Error; EventSink; GeneratedRowBuilder; GenerationLoadIncompleteError; GenericRowBuilder; IngestionMode; Load; LoadIncompleteError; LoadOptions; LoadSummary; NewGeneratedRowBuilder; NewGenericRowBuilder; OpenLineScanner; PreflightError; PreflightFiles; PreflightIssue; PreflightReport; PreflightResource; ResourceTypeFromPath; RowBuilder; RowErrorSample; SanitizeKey; SecondsSince; SemanticInventoryBackfillCollectionResult; SemanticInventoryBackfillOptions; SemanticInventoryBackfillProgress; SemanticInventoryBackfillReport; Unwrap; Valid; VertexDocument; VertexFromFHIRWithExtra,generated/fhir; internal/catalog; internal/catalog/arango; internal/dataset; internal/dataset/arango; internal/fhir/schema; internal/store/arango,generated/fhir; internal/catalog; internal/catalog/arango; internal/dataset; internal/dataset/arango; internal/fhir/schema; internal/store/arango,cmd/arango-fhir-proto; internal/api/bulk/load; internal/server,—,github.com/bmeg/jsonschema/v6; github.com/bmeg/jsonschemagraph/graph; github.com/bmeg/jsonschemagraph/model; github.com/bmeg/jsonschemagraph/util; github.com/bytedance/sonic,1 exported interface(s),legacy=1; migration=1; compatibility=1,broad surface (15 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/ingest,inventory complete; semantic disposition pending -internal/projectid,github.com/calypr/loom/internal/projectid,projectid,domain/library,Define canonical and legacy-compatible public project identity normalization.,projectid.go,projectid_test.go,—,—,51,18,2,Canonical; Legacy,—,—,cmd/loom-repository; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/authscope; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/execution; internal/dataframe/published; internal/explorer; internal/explorer/arango; internal/explorer/compilation; internal/explorer/lifecycle; internal/explorer/tableshapecap; internal/repositoryseed; internal/server,internal/dataframe/compiler,—,—,legacy=5; compatibility=1,small shared semantic contract; compare direct callers before inlining,go test ./internal/projectid,inventory complete; semantic disposition pending +internal/ingest,github.com/calypr/loom/internal/ingest,ingest,domain/library,"Preflight and load FHIR NDJSON into generation-scoped Arango vertices, edges, and catalog facts, and commit built resource-row and built-edge-occurrence batches to the PostgreSQL source store through the ledgered batch writer with scope verification, canonical payload encoding, and payload-backed path/ordinal recovery for repeated references.",backend.go; doc.go; documents.go; files.go; generated_load.go; generation_identity.go; generation_load.go; generation_orchestration.go; generation_workers.go; load_options.go; postgres_writer.go; preflight.go; progress.go; resource_load.go; row_builder.go; semantic_inventory_backfill.go; source_schema.go,backend_test.go; basic_observation_fixture_validation_test.go; bench_test.go; documents_test.go; file_pipeline_test.go; files_test.go; generated_load_test.go; generation_cleanup_test.go; generation_failed_cleanup_e2e_test.go; generation_identity_test.go; generation_load_postgres_durability_test.go; generation_load_postgres_test.go; generation_load_test.go; integration_test.go; keycollide_generated_e2e_test.go; load_options_test.go; postgres_writer_test.go; preflight_test.go; related_medication_fixture_test.go; resource_load_test.go; row_builder_test.go; semantic_inventory_backfill_integration_test.go; semantic_inventory_backfill_test.go; semantic_inventory_test.go; source_schema_test.go; viewer_row_file_download_fixture_test.go,—,—,5650,10567,63,ActivatePostgresGeneration; ActivationOutcomeError; BackfillSemanticInventory; Build; Complete; ConnectionOptions; DiscoverNDJSON; EdgeDocument; EdgeFromGrip; Error; EventSink; GeneratedRowBuilder; GenerationCatalogWriters; GenerationFieldProfileWriter; GenerationInventoryRebuilder; GenerationLoadIncompleteError; GenerationRelationshipDiscoverer; GenerationSemanticContributor; GenericRowBuilder; IngestionMode; Load; LoadIncompleteError; LoadOptions; LoadPostgresGeneration; LoadSummary; NewGeneratedRowBuilder; NewGenericRowBuilder; NewPostgresEdgeWriter; NewPostgresResourceWriter; OpenLineScanner; PGEdgeBatchScope; PGEdgeRowIdentityError; PGEdgeWriteResult; PGEdgeWriteScopeError; PGLoadConfig; PGResourceBatchScope; PGResourceRowIdentityError; PGResourceWriteResult; PGResourceWriteScopeError; PGSourceOptions; PostgresEdgeWriter; PostgresResourceWriter; PreflightError; PreflightFiles; PreflightIssue; PreflightReport; PreflightResource; ResourceTypeFromPath; RowBuilder; RowErrorSample; SanitizeKey; SecondsSince; SemanticInventoryBackfillCollectionResult; SemanticInventoryBackfillOptions; SemanticInventoryBackfillProgress; SemanticInventoryBackfillReport; SplitCollectionID; Unwrap; Valid; VertexDocument; VertexFromFHIRWithExtra; WriteEdgeBatch; WriteResourceBatch,generated/fhir; internal/backends/arango; internal/backends/postgres; internal/backends/postgres/control; internal/catalog; internal/dataset; internal/fhir/schema; internal/source,generated/fhir; internal/backends/arango; internal/backends/postgres/control; internal/catalog; internal/dataset; internal/fhir/schema; internal/source,cmd/arango-fhir-proto; cmd/loom-data; internal/api/bulk/load; internal/server,internal/api/graphql/graph/query; internal/backends/postgres/internal/catalog,github.com/bmeg/jsonschema/v6; github.com/bmeg/jsonschemagraph/graph; github.com/bmeg/jsonschemagraph/model; github.com/bmeg/jsonschemagraph/util; github.com/bytedance/sonic,5 exported interface(s),legacy=5; migration=9; compatibility=1,broad surface (17 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/ingest,inventory complete; semantic disposition pending +internal/projectid,github.com/calypr/loom/internal/projectid,projectid,domain/library,Define canonical and legacy-compatible public project identity normalization.,projectid.go,projectid_test.go,—,—,51,18,2,Canonical; Legacy,—,—,cmd/loom-repository; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/authscope; internal/backends/arango/internal/explorer; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/execution; internal/dataframe/published; internal/explorer; internal/explorer/compilation; internal/explorer/lifecycle; internal/explorer/tableshapecap; internal/repositoryseed; internal/server,internal/dataframe/compiler,—,—,legacy=5; compatibility=1,small shared semantic contract; compare direct callers before inlining,go test ./internal/projectid,inventory complete; semantic disposition pending internal/repositoryseed,github.com/calypr/loom/internal/repositoryseed,repositoryseed,domain/library,"Validate repository metadata and configuration, produce a source manifest, and refuse unmaterialized inputs before launch.",preflight.go,preflight_test.go,—,—,281,105,6,Error; File; Manifest; Options; Prepare; UnmaterializedError,internal/explorer/authoringv2; internal/projectid,—,cmd/loom-repository,—,—,—,—,one production importer (cmd/loom-repository); inspect ownership/inline potential,go test ./internal/repositoryseed,inventory complete; semantic disposition pending -internal/server,github.com/calypr/loom/internal/server,server,domain/library,"Compose storage, domain services, generated OpenAPI handlers, GraphQL adapters, and HTTP routes.",artifact_dev.go; auth_wiring.go; construction_inputs_catalog.go; explain_wiring.go; explorer_authoring_http.go; explorer_capability.go; explorer_combine_output_metadata.go; explorer_config_route.go; explorer_lifecycle_helpers.go; explorer_lifecycle_wire.go; explorer_output_reconcile.go; explorer_preview_response.go; explorer_preview_security.go; explorer_publication.go; explorer_receipt_contract.go; explorer_receipt_validation.go; local_workspace_writeback.go; openapi_data_routes.go; openapi_explorer_artifacts.go; openapi_explorer_authoring_direct.go; openapi_explorer_construction.go; openapi_explorer_construction_choice_proposal.go; openapi_explorer_construction_inputs.go; openapi_explorer_construction_proposal.go; openapi_explorer_conversion.go; openapi_explorer_direct.go; openapi_explorer_explicit_groups.go; openapi_explorer_population_member_proposal.go; openapi_explorer_routes.go; openapi_explorer_row_definition.go; openapi_explorer_selections.go; openapi_explorer_table_shape.go; openapi_frame_source_options.go; openapi_graphql_routes.go; openapi_recipe_execution.go; openapi_routes.go; openapi_semantic_inventory.go; options.go; private_artifact_reconciler.go; receipt_preview_diagnostic.go; recipe_authorization.go; recipe_combine_resolution.go; recipe_discovery.go; recipe_publication.go; release_verifier.go; routes.go; server.go,accepted_construction_contract_test.go; append_workspace_rows_integration_test.go; artifact_reconciler_wiring_test.go; auth_wiring_trusted_issuer_test.go; case008_row_definition_preview_route_test.go; config_test.go; construction_category_discovery_openapi_error_test.go; construction_inputs_catalog_test.go; construction_proposal_zero_column_test.go; core_authoring_protocol_arango_conformance_test.go; core_authoring_protocol_formal_replay_test.go; core_authoring_protocol_receipts_test.go; core_authoring_protocol_restore_test.go; core_authoring_protocol_rowschema_test.go; core_authoring_protocol_scoped_compile_test.go; core_authoring_protocol_test.go; dataframe_query_error_test.go; derived_policy_rows_integration_test.go; explorer_capability_test.go; explorer_coded_value_capability_test.go; explorer_concept_evidence_test.go; explorer_config_route_test.go; explorer_diagnostic_logging_test.go; explorer_output_reconcile_derive_source_test.go; explorer_output_reconcile_source_child_test.go; explorer_output_reconcile_unpivot_source_test.go; explorer_preview_response_test.go; explorer_preview_security_test.go; explorer_receipt_contract_error_test.go; explorer_receipt_contract_test.go; explorer_reconcile_scope_test.go; explorer_row_change_route_test.go; explorer_state_route_test.go; fs1_signed_slot_collision_test.go; http_test_helpers_test.go; local_workspace_writeback_test.go; openapi_column_id_contract_test.go; openapi_explorer_artifacts_download_test.go; openapi_explorer_authoring_write_fault_test.go; openapi_explorer_cell_trace_test.go; openapi_explorer_commands_notfound_test.go; openapi_explorer_construction_apply_preview_test.go; openapi_explorer_construction_derive_source_test.go; openapi_explorer_construction_expand_source_test.go; openapi_explorer_construction_filter_source_test.go; openapi_explorer_construction_inputs_test.go; openapi_explorer_construction_proposal_test.go; openapi_explorer_construction_unpivot_source_test.go; openapi_explorer_conversion_test.go; openapi_explorer_direct_test.go; openapi_explorer_draft_restore_test.go; openapi_explorer_legacy_details_control_test.go; openapi_explorer_population_mapping_construction_test.go; openapi_explorer_population_member_proposal_test.go; openapi_explorer_restricted_starting_collection_integration_test.go; openapi_explorer_route_roundtrip_test.go; openapi_explorer_route_rows_arango_test.go; openapi_explorer_row_definition_test.go; openapi_explorer_write_scope_test.go; openapi_field_presentation_fixture_test.go; openapi_graphql_status_test.go; openapi_graphql_transport_review_test.go; openapi_interpretation_routes_test.go; openapi_recipe_execution_test.go; openapi_route_inventory_test.go; openapi_routes_test.go; openapi_semantic_inventory_test.go; openapi_status_coverage_test.go; openapi_test_helpers_test.go; patient_one_construction_choice_route_test.go; population_member_receipt_test.go; private_artifact_reconciler_test.go; publication_rows_integration_test.go; publication_viewer_agreement_test.go; receipt_preview_diagnostic_test.go; recipe_authorization_test.go; recipe_blank_project_test.go; recipe_combine_resolution_test.go; recipe_discovery_test.go; recipe_execution_etl_failure_fields_e2e_test.go; recipe_execution_status_twin_e2e_test.go; recipe_execution_twin_authz_repair_test.go; recipe_project_keying_test.go; recipe_publication_test.go; release_verifier_fault_mapping_test.go; release_verifier_test.go; repository_activation_conflict_dev_test.go; routes_test.go; row_value_policy_stream_test.go; wiring_test.go,—,—,12091,29998,88,ActivateDatasetGeneration; ApplyExplorerBuilderCommands; AssessExplorerRowChange; AuthConfig; AuthorizeRead; AuthorizeWrite; BasicAuthConfig; BrowseExplorerFrameSourceOptions; BrowseExplorerSemanticInventory; CalyprAuthConfig; CheckExplorerPopulationMapping; ClickHouseConfig; Config; CreateDatasetGeneration; CreateExplorer; CreateExplorerExplicitGroupRevision; CreateExplorerSelection; CreateInterpretationRevision; CreateInterpretationRevisionFromColumn; DataframerConfig; DefaultConfig; DiscoverExplorerConstructionCategories; DiscoverExplorerTableShapeCategories; DownloadExplorerArtifact; Error; ExactExecutionMaterialization; ExecuteDataframeGraphQL; ExecuteGraphQL; Finish; GetApolloSandbox; GetDatasetGenerationStatus; GetExplorer; GetExplorerAuthoringCapability; GetExplorerBuilder; GetExplorerColumnSource; GetExplorerConstructionCapabilities; GetExplorerConstructionInputs; GetExplorerSelection; GetExplorerTableShapeCapabilities; GetGraphQLPlayground; GetHealth; GetInterpretationRevision; GetLiveness; GetReadiness; GetRecipeExecution; HTTPRoutes; ListExplorerRowDefinitionChoices; ListExplorers; ListInterpretationLibraries; LoadConfig; PrepareExplorerArtifact; PreviewExplorer; PreviewInterpretationCandidate; ProbeCandidate; ProbeEdge; ProbeNode; ProposeExplorerConstruction; ProposeExplorerConstructionChoice; ProposeExplorerPopulationMemberRemoval; ProposeExplorerRowDefinition; ProposeExplorerTableShape; PublishExplorer; PublishRepositoryExplorerConfig; ReconcileExplorerBuilder; Resolve; ResolveConfiguredColumnContext; ResolveExplorerTableShape; ResolveForCompilation; ResolveForExecution; ResolveToken; Run; SearchExplorerCandidates; SearchExplorerConstructionChoices; SearchExplorerPopulationRoutes; SearchExplorerRelatedExpandChoices; SearchExplorerRelatedExpandContributors; SearchExplorerRelatedFieldChoices; SearchExplorerSchemaFields; ServerConfig; StreamExactExport; TraceExplorerCell; TraceExplorerRowLineage; Unwrap; Validate; VerifyPublication; Visit; VisitProposeExplorerTableShapeResponse; Write,generated/loomapi; internal/api/authpolicy; internal/api/bulk/load; internal/api/columncapabilities; internal/api/graphql/graph; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/api/http; internal/api/recipe; internal/authscope; internal/catalog; internal/catalog/arango; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/execution/chartifact; internal/dataframe/execution/chartifact/arango; internal/dataframe/expression; internal/dataframe/publication; internal/dataframe/publication/arango; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/recipe/exec/arango; internal/dataframe/recipe/schema; internal/dataframe/spec; internal/dataframe/unit; internal/dataset; internal/dataset/arango; internal/explorer; internal/explorer/arango; internal/explorer/artifactfs; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation; internal/explorer/lifecycle; internal/explorer/tableshapecap; internal/fhir/schema; internal/ingest; internal/projectid; internal/store/arango; internal/store/clickhouse,generated/graphql/graph/model; generated/loomapi; internal/api/bulk/load; internal/api/graphql/graph; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/resolver; internal/api/http; internal/api/recipe; internal/authscope; internal/catalog; internal/catalog/arango; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/execution/chartifact; internal/dataframe/execution/chartifact/arango; internal/dataframe/expression; internal/dataframe/lineage; internal/dataframe/publication; internal/dataframe/publication/arango; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/dataframe/spec; internal/dataframe/unit; internal/dataset; internal/dataset/arango; internal/explorer; internal/explorer/arango; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation; internal/explorer/lifecycle; internal/fhir/schema; internal/ingest; internal/projectid; internal/store/arango; internal/store/clickhouse,.; cmd/arango-fhir-server,—,github.com/arangodb/go-driver/v2/arangodb/shared; github.com/gofiber/fiber/v3; go.yaml.in/yaml/v3; golang.org/x/sync/singleflight,—,legacy=38; migration=3; compatibility=3,broad surface (47 production files); inspect cohesion,go test ./internal/server,inventory complete; semantic disposition pending -internal/store/arango,github.com/calypr/loom/internal/store/arango,arango,storage adapter,"Provide the low-level typed ArangoDB client, query streaming, profiling, and Explain assessment boundary.",client.go; explain.go; explain_assessment.go; explain_client.go; options.go; preview_index.go; profile.go; profile_client.go; types.go,client_query_memory_limit_test.go; client_queryrows_test.go; client_revision_test.go; explain_assessment_test.go; explain_client_test.go; explain_test.go; population_profile_integration_test.go; preview_index_test.go; profile_client_test.go; profile_integration_test.go; profile_test.go,—,—,1666,1835,68,AQLExecutor; AssessExplainResult; BatchInserter; Bootstrap; BootstrapSpec; Client; Close; CollectionCount; CollectionExists; CollectionRevision; CollectionSpec; ConnectionOptions; EnsurePreviewCoveringIndex; EnsurePreviewCoveringIndexReplacing; EnsurePreviewCoveringIndexWithStoredValues; EnsurePreviewCoveringIndexWithStoredValuesReplacing; ExecuteAQL; Explain; ExplainAssessment; ExplainCollection; ExplainCollectionScan; ExplainIndex; ExplainIndexLocation; ExplainIndexSummary; ExplainIndexUse; ExplainIndexes; ExplainNode; ExplainOptions; ExplainPlan; ExplainPlanEstimate; ExplainRequest; ExplainResult; ExplainStats; ExplainWarning; InsertBatchRaw; IsQueryMemoryLimitExceeded; IsQueryOutOfMemory; IsQueryResourceLimitExceeded; IsQueryUserAssertion; ListPreviewCoveringIndexes; Open; OptimizerOptions; ParseExplainResult; ParseProfileResult; PreviewCoveringIndex; Profile; ProfileExtra; ProfileNode; ProfileNodeGroup; ProfileNodeSummary; ProfileOptions; ProfilePhases; ProfileRequest; ProfileResult; ProfileStats; ProfileSummary; QueryRows; QueryRowsWithMaxRuntime; RowBatchClient; RowQueryer; RowVisitor; SummarizeProfile; TransactionCollections; TransactionFunc; TransactionalRowClient; UnmarshalJSON; WithQueryMemoryLimit; WithTransaction,—,—,cmd/arango-fhir-proto; cmd/dataframe-profile; internal/acceptance; internal/catalog/arango; internal/dataframe/execution; internal/dataframe/execution/chartifact/arango; internal/dataframe/publication/arango; internal/dataframe/recipe/exec/arango; internal/dataset/arango; internal/explorer/arango; internal/ingest; internal/server,conformance/compiler; internal/authscope; internal/dataframe/compiler; internal/dataframe/publication/clickhouse; internal/dataset; internal/explorer/compilation,github.com/arangodb/go-driver/v2/arangodb; github.com/arangodb/go-driver/v2/arangodb/shared; github.com/arangodb/go-driver/v2/connection; github.com/arangodb/go-driver/v2/utils,5 exported interface(s),legacy=1,storage adapter; preserve interface and persisted-data contracts; runtime/build signals present; static importer counts are insufficient for deletion,go test ./internal/store/arango,inventory complete; semantic disposition pending -internal/store/clickhouse,github.com/calypr/loom/internal/store/clickhouse,clickhouse,storage adapter,Provide the low-level typed ClickHouse connection and database-management boundary.,client.go,client_test.go; integration_test.go,—,—,676,712,15,AddColumn; Client; Close; Column; CreateTable; DropColumns; DropTable; EnsureDatabase; InsertRows; New; Options; Ping; QueryRowsArgs; QueryRowsArgsVisit; VerifyOutput,—,internal/dataframe/compiler/ir; internal/dataframe/compiler/render/clickhouse; internal/dataframe/published,internal/acceptance; internal/dataframe/execution/chartifact; internal/dataframe/publication/clickhouse; internal/server,internal/dataframe/compiler/render/clickhouse; internal/dataframe/execution; internal/dataframe/execution/chartifact/arango; internal/dataframe/publication/arango; internal/dataframe/published,github.com/ClickHouse/clickhouse-go/v2; github.com/ClickHouse/clickhouse-go/v2/lib/driver,—,—,storage adapter; preserve interface and persisted-data contracts,go test ./internal/store/clickhouse,inventory complete; semantic disposition pending -scripts,github.com/calypr/loom/scripts,main,domain/library,"Hold repository developer, verification, migration, and release utilities that are not part of the Go module runtime packages.",check_fhir_semantic_resource_types.go; local-preview-index-migration.go,check_fhir_semantic_resource_types_test.go; local-preview-index-migration_test.go,README.md; package-lock.json; package.json,—,769,532,0,—,internal/fhir/schema,—,—,—,github.com/arangodb/go-driver/v2/arangodb; github.com/arangodb/go-driver/v2/arangodb/shared; github.com/arangodb/go-driver/v2/connection; github.com/arangodb/go-driver/v2/utils,build tags,legacy=8; migration=9,no in-repo production importer; verify runtime/tooling reachability; runtime/build signals present; static importer counts are insufficient for deletion,go test ./scripts,inventory complete; semantic disposition pending +internal/server,github.com/calypr/loom/internal/server,server,domain/library,"Compose storage, domain services, generated OpenAPI handlers, GraphQL adapters, and HTTP routes.",artifact_dev.go; auth_wiring.go; compose.go; construction_inputs_catalog.go; explain_wiring.go; explorer_authoring_http.go; explorer_capability.go; explorer_combine_output_metadata.go; explorer_config_route.go; explorer_lifecycle_helpers.go; explorer_lifecycle_wire.go; explorer_output_reconcile.go; explorer_preview_response.go; explorer_preview_security.go; explorer_publication.go; explorer_receipt_contract.go; explorer_receipt_validation.go; local_workspace_writeback.go; openapi_data_routes.go; openapi_explorer_artifacts.go; openapi_explorer_authoring_direct.go; openapi_explorer_construction.go; openapi_explorer_construction_choice_proposal.go; openapi_explorer_construction_inputs.go; openapi_explorer_construction_proposal.go; openapi_explorer_conversion.go; openapi_explorer_direct.go; openapi_explorer_explicit_groups.go; openapi_explorer_population_member_proposal.go; openapi_explorer_routes.go; openapi_explorer_row_definition.go; openapi_explorer_selections.go; openapi_explorer_table_shape.go; openapi_frame_source_options.go; openapi_graphql_routes.go; openapi_recipe_execution.go; openapi_routes.go; openapi_semantic_inventory.go; options.go; private_artifact_reconciler.go; receipt_preview_diagnostic.go; recipe_authorization.go; recipe_combine_resolution.go; recipe_discovery.go; recipe_publication.go; release_verifier.go; routes.go; row_sources.go; runtime.go; server.go,accepted_construction_contract_test.go; append_workspace_rows_integration_test.go; artifact_reconciler_wiring_test.go; auth_wiring_trusted_issuer_test.go; backend_composition_contract_test.go; backend_error_boundary_contract_test.go; case008_row_definition_preview_route_test.go; command_helpers_test.go; config_test.go; construction_category_discovery_openapi_error_test.go; construction_inputs_catalog_test.go; construction_proposal_zero_column_test.go; core_authoring_protocol_arango_conformance_test.go; core_authoring_protocol_formal_replay_test.go; core_authoring_protocol_receipts_test.go; core_authoring_protocol_restore_test.go; core_authoring_protocol_rowschema_test.go; core_authoring_protocol_scoped_compile_test.go; core_authoring_protocol_test.go; dataframe_query_error_test.go; derived_policy_rows_integration_test.go; explain_wiring_pg18_test.go; explain_wiring_preview_test.go; explorer_capability_test.go; explorer_coded_value_capability_test.go; explorer_concept_evidence_test.go; explorer_config_route_test.go; explorer_diagnostic_logging_test.go; explorer_output_reconcile_derive_source_test.go; explorer_output_reconcile_source_child_test.go; explorer_output_reconcile_unpivot_source_test.go; explorer_preview_response_test.go; explorer_preview_security_test.go; explorer_receipt_contract_error_test.go; explorer_receipt_contract_test.go; explorer_reconcile_scope_test.go; explorer_row_change_route_test.go; explorer_state_route_test.go; fs1_signed_slot_collision_test.go; health_readiness_wiring_test.go; http_test_helpers_test.go; local_workspace_writeback_test.go; openapi_column_id_contract_test.go; openapi_explorer_artifacts_download_test.go; openapi_explorer_authoring_write_fault_test.go; openapi_explorer_cell_trace_test.go; openapi_explorer_commands_notfound_test.go; openapi_explorer_construction_apply_preview_test.go; openapi_explorer_construction_derive_source_test.go; openapi_explorer_construction_expand_source_test.go; openapi_explorer_construction_filter_source_test.go; openapi_explorer_construction_inputs_test.go; openapi_explorer_construction_proposal_test.go; openapi_explorer_construction_unpivot_source_test.go; openapi_explorer_conversion_test.go; openapi_explorer_direct_test.go; openapi_explorer_draft_restore_test.go; openapi_explorer_legacy_details_control_test.go; openapi_explorer_population_mapping_construction_test.go; openapi_explorer_population_member_proposal_test.go; openapi_explorer_restricted_starting_collection_integration_test.go; openapi_explorer_route_roundtrip_test.go; openapi_explorer_route_rows_arango_test.go; openapi_explorer_row_definition_test.go; openapi_explorer_write_scope_test.go; openapi_field_presentation_fixture_test.go; openapi_graphql_status_test.go; openapi_graphql_transport_review_test.go; openapi_interpretation_routes_test.go; openapi_recipe_execution_test.go; openapi_route_inventory_test.go; openapi_routes_test.go; openapi_semantic_inventory_test.go; openapi_status_coverage_test.go; openapi_test_helpers_test.go; options_schemas_test.go; patient_one_construction_choice_route_test.go; population_member_receipt_test.go; postgres_query_test.go; postgres_refusal_contract_test.go; preview_postgres_wiring_pg18_test.go; private_artifact_reconciler_test.go; publication_rows_integration_test.go; publication_viewer_agreement_test.go; receipt_preview_diagnostic_test.go; recipe_authorization_test.go; recipe_blank_project_test.go; recipe_combine_resolution_test.go; recipe_discovery_test.go; recipe_execution_etl_failure_fields_e2e_test.go; recipe_execution_malformed_persisted_scope_test.go; recipe_execution_malformed_scope_header_probe_test.go; recipe_execution_status_twin_e2e_test.go; recipe_execution_twin_authz_repair_test.go; recipe_project_keying_test.go; recipe_publication_test.go; release_verifier_fault_mapping_test.go; release_verifier_test.go; repository_activation_conflict_dev_test.go; route_composition_contract_test.go; routes_test.go; row_tag_guard_contract_test.go; row_value_policy_stream_test.go; runtime_postgres_live_test.go; runtime_test.go; wiring_test.go; zz_readiness_contract_guard_test.go,—,—,14261,37094,103,Activate; ActivateDatasetGeneration; ActivateRepositoryGeneration; ApplyExplorerBuilderCommands; AssessExplorerRowChange; AuthConfig; AuthorizeRead; AuthorizeWrite; BasicAuthConfig; BrowseExplorerFrameSourceOptions; BrowseExplorerSemanticInventory; CalyprAuthConfig; CheckExplorerPopulationMapping; ClickHouseConfig; Config; CreateDatasetGeneration; CreateExplorer; CreateExplorerExplicitGroupRevision; CreateExplorerSelection; CreateInterpretationRevision; CreateInterpretationRevisionFromColumn; DataframerConfig; DefaultConfig; DiscoverExplorerConstructionCategories; DiscoverExplorerTableShapeCategories; DownloadExplorerArtifact; Error; ExactExecutionMaterialization; ExecuteDataframeGraphQL; ExecuteGraphQL; Explain; ExplainPhysical; Finish; GetApolloSandbox; GetDatasetGenerationStatus; GetExecution; GetExplorer; GetExplorerAuthoringCapability; GetExplorerBuilder; GetExplorerColumnSource; GetExplorerConstructionCapabilities; GetExplorerConstructionInputs; GetExplorerSelection; GetExplorerTableShapeCapabilities; GetGraphQLPlayground; GetHealth; GetInterpretationRevision; GetLiveness; GetPointer; GetReadiness; GetRecipeExecution; HTTPRoutes; ListExplorerRowDefinitionChoices; ListExplorers; ListInterpretationLibraries; LoadConfig; PostgreSQLConfig; PrepareExplorerArtifact; Preview; PreviewCollectionScanEvidenceFromExplain; PreviewExplorer; PreviewInterpretationCandidate; ProbeCandidate; ProbeEdge; ProbeNode; ProposeExplorerConstruction; ProposeExplorerConstructionChoice; ProposeExplorerPopulationMemberRemoval; ProposeExplorerRowDefinition; ProposeExplorerTableShape; PublishAuthoring; PublishExplorer; PublishRepositoryExplorerConfig; ReadManifest; ReconcileExplorerBuilder; Resolve; ResolveConfiguredColumnContext; ResolveExplorerTableShape; ResolveForCompilation; ResolveForExecution; ResolveToken; Run; RunGeneration; RunWithConfig; SearchExplorerCandidates; SearchExplorerConstructionChoices; SearchExplorerPopulationRoutes; SearchExplorerRelatedExpandChoices; SearchExplorerRelatedExpandContributors; SearchExplorerRelatedFieldChoices; SearchExplorerSchemaFields; ServerConfig; StreamExactExport; String; TraceExplorerCell; TraceExplorerRowLineage; Unwrap; Validate; ValidateSelectionReferences; VerifyPublication; Visit; VisitProposeExplorerTableShapeResponse; Write,generated/loomapi; internal/api/authpolicy; internal/api/bulk/load; internal/api/columncapabilities; internal/api/graphql/graph; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/api/http; internal/api/recipe; internal/authscope; internal/backends/arango; internal/backends/postgres; internal/backends/postgres/control; internal/catalog; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/execution/chartifact; internal/dataframe/expression; internal/dataframe/publication; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/dataframe/spec; internal/dataframe/unit; internal/dataset; internal/explorer; internal/explorer/artifactfs; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation; internal/explorer/lifecycle; internal/explorer/tableshapecap; internal/fhir/schema; internal/ingest; internal/projectid; internal/store/clickhouse,generated/graphql/graph/model; generated/loomapi; internal/api/bulk/load; internal/api/graphql/graph; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/resolver; internal/api/http; internal/api/recipe; internal/authscope; internal/backends/arango; internal/backends/postgres; internal/backends/postgres/control; internal/catalog; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/execution/chartifact; internal/dataframe/expression; internal/dataframe/lineage; internal/dataframe/publication; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/dataframe/spec; internal/dataframe/unit; internal/dataset; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation; internal/explorer/lifecycle; internal/fhir/schema; internal/ingest; internal/projectid; internal/source; internal/store/clickhouse,.; cmd/arango-fhir-server,internal/backends/postgres,github.com/gofiber/fiber/v3; go.yaml.in/yaml/v3; golang.org/x/sync/singleflight,—,legacy=41; migration=11; compatibility=3,broad surface (50 production files); inspect cohesion,go test ./internal/server,inventory complete; semantic disposition pending +internal/source,github.com/calypr/loom/internal/source,source,domain/library,"Define the backend-neutral logical source identity, payload, and authorization-scope contracts shared by ingestion and store read/write paths, free of any storage or driver detail.",doc.go; resource.go,ownership_contract_test.go,—,—,80,403,2,Resource; ResourceScope,—,—,internal/backends/postgres/internal/store; internal/ingest,internal/backends/postgres; internal/dataframe/compiler; internal/dataframe/compiler/render/postgres; internal/dataframe/execution; internal/server,—,—,legacy=4,small shared semantic contract; compare direct callers before inlining,go test ./internal/source,inventory complete; semantic disposition pending +internal/store/clickhouse,github.com/calypr/loom/internal/store/clickhouse,clickhouse,storage adapter,Provide the low-level typed ClickHouse connection and database-management boundary.,client.go,client_test.go; integration_test.go,—,—,676,712,15,AddColumn; Client; Close; Column; CreateTable; DropColumns; DropTable; EnsureDatabase; InsertRows; New; Options; Ping; QueryRowsArgs; QueryRowsArgsVisit; VerifyOutput,—,internal/dataframe/compiler/ir; internal/dataframe/compiler/render/clickhouse; internal/dataframe/published,internal/acceptance; internal/dataframe/execution/chartifact; internal/dataframe/publication/clickhouse; internal/server,internal/backends/arango/internal/chartifact; internal/backends/arango/internal/publication; internal/backends/postgres/internal/chartifact; internal/dataframe/compiler/render/clickhouse; internal/dataframe/execution; internal/dataframe/published,github.com/ClickHouse/clickhouse-go/v2; github.com/ClickHouse/clickhouse-go/v2/lib/driver,—,—,storage adapter; preserve interface and persisted-data contracts,go test ./internal/store/clickhouse,inventory complete; semantic disposition pending +scripts,github.com/calypr/loom/scripts,main,domain/library,"Hold repository developer, verification, migration, and release utilities that are not part of the Go module runtime packages.",check_fhir_semantic_resource_types.go; local-preview-index-migration.go,check_fhir_semantic_resource_types_test.go; local-preview-index-migration_test.go,README.md; package-lock.json; package.json; pg-exec-03-skipset-capture.json,—,769,532,0,—,internal/fhir/schema,—,—,—,github.com/arangodb/go-driver/v2/arangodb; github.com/arangodb/go-driver/v2/arangodb/shared; github.com/arangodb/go-driver/v2/connection; github.com/arangodb/go-driver/v2/utils,build tags,legacy=8; migration=9,no in-repo production importer; verify runtime/tooling reachability; runtime/build signals present; static importer counts are insufficient for deletion,go test ./scripts,inventory complete; semantic disposition pending +scripts/check-boundaries,github.com/calypr/loom/scripts/check-boundaries,main,domain/library,"Run the authoritative package-boundary policy over the repository and prove it can go RED: the --self-test negative fixtures plant forbidden imports, removed facades, re-exports, cycle leaks, and vacuous scans, each failing closed, with removal converging to green (arch-backend-09).",main.go,—,—,—,435,0,0,—,scripts/packagepolicy,—,—,—,—,—,—,no in-repo production importer; verify runtime/tooling reachability,go test ./scripts/check-boundaries,inventory complete; semantic disposition pending +scripts/packagepolicy,github.com/calypr/loom/scripts/packagepolicy,packagepolicy,domain/library,"Hold the single authoritative parsed-import ownership policy table, scanner, identifier-absence and no-re-export checks shared by the check-boundaries CLI, the shell entrypoint, and the compiler guard tests, separating import findings from verification failures and refusing vacuous scans.",policy.go; rules.go,policy_test.go,old_concrete_paths.txt,—,1055,318,24,AbsenceRule; ByLabel; CheckAbsences; CheckNoReExport; Clean; ErrorKind; FindRepoRoot; Finding; Forbidden; HasImportFindings; ImportFindings; NoReExportRule; OldConcreteImportPaths; RepoAbsences; RepoNoReExport; RepoRules; Result; Rule; ScanAll; ScanError; ScanRule; ScanTarget; Scope; String,—,—,scripts/check-boundaries,internal/dataframe/compiler,—,go:embed,compatibility=2,one production importer (scripts/check-boundaries); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion,go test ./scripts/packagepolicy,inventory complete; semantic disposition pending diff --git a/docs/PACKAGE_AUDIT.md b/docs/PACKAGE_AUDIT.md index a4e74027c..f62aa9fe1 100644 --- a/docs/PACKAGE_AUDIT.md +++ b/docs/PACKAGE_AUDIT.md @@ -6,14 +6,14 @@ A row marked as a consolidation candidate is **not** deletion authorization. Bef ## Snapshot -- Packages: 74 -- Production Go files: 577 -- Test Go files: 556 -- Production Go lines: 486,571 -- Test Go lines: 145,987 -- Package kinds: conformance=1, domain/library=38, executable=11, generated=5, storage adapter=10, transport adapter=9 -- Non-entrypoint handwritten packages with no production importer: scripts -- Non-entrypoint handwritten packages with one production importer: internal/acceptance; internal/api/bulk/load; internal/api/graphql/graph; internal/api/graphql/graph/dataframe; internal/dataframe/compiler/render/clickhouse; internal/dataframe/execution/chartifact/arango; internal/dataframe/publication/clickhouse; internal/dataframe/recipe/exec/arango; internal/explorer/arango; internal/explorer/artifactfs; internal/explorer/lifecycle; internal/fhir/semantic; internal/repositoryseed +- Packages: 94 +- Production Go files: 695 +- Test Go files: 844 +- Production Go lines: 553,642 +- Test Go lines: 347,644 +- Package kinds: conformance=1, domain/library=61, executable=12, generated=5, storage adapter=6, transport adapter=9 +- Non-entrypoint handwritten packages with no production importer: internal/ci; scripts; scripts/check-boundaries +- Non-entrypoint handwritten packages with one production importer: internal/acceptance; internal/api/bulk/load; internal/api/graphql/graph; internal/backends/arango/internal/catalog; internal/backends/arango/internal/chartifact; internal/backends/arango/internal/explorer; internal/backends/arango/internal/recipeexec; internal/backends/postgres/internal/catalog; internal/backends/postgres/internal/chartifact; internal/backends/postgres/internal/explorer; internal/backends/postgres/internal/publication; internal/backends/postgres/internal/recipeexec; internal/backends/postgres/internal/recipes; internal/backends/postgres/internal/redact; internal/backends/postgres/internal/rows; internal/dataframe/compiler/render/clickhouse; internal/dataframe/publication/clickhouse; internal/explorer/artifactfs; internal/explorer/lifecycle; internal/fhir/semantic; internal/repositoryseed; scripts/packagepolicy ## How to use the table @@ -50,79 +50,99 @@ These records preserve verified package-boundary decisions for both live package | Location | Kind | Responsibility | Contents | LOC (prod/test) | API | Depends on | Imported by (production) | Decision evidence | | --- | --- | --- | --- | ---: | --- | --- | --- | --- | | `.` | executable | Start the default Loom server process through the internal server composition root. | P: main.go; T: —; Other: go.mod | 7/0 | 0: — | internal/server | — | executable boundary; compare orchestration with sibling commands before combining | -| `cmd/arango-fhir-proto` | executable | Provide operator commands for FHIR loading, catalog diagnostics, generation repair, and activation. | P: main.go; T: main_test.go; resource_inventory_test.go; Other: — | 750/269 | 0: — | internal/catalog; internal/catalog/arango; internal/dataset; internal/dataset/arango; internal/fhir/schema; internal/ingest; internal/store/arango | — | executable boundary; compare orchestration with sibling commands before combining | +| `cmd/arango-fhir-proto` | executable | Provide operator commands for FHIR loading, catalog diagnostics, generation repair, and activation. | P: main.go; T: main_test.go; resource_inventory_test.go; Other: — | 842/470 | 0: — | internal/backends/arango; internal/catalog; internal/dataset; internal/fhir/schema; internal/ingest | — | executable boundary; compare orchestration with sibling commands before combining | | `cmd/arango-fhir-server` | executable | Start the explicit Loom HTTP server binary. | P: main.go; T: —; Other: — | 7/0 | 0: — | internal/server | — | executable boundary; compare orchestration with sibling commands before combining | -| `cmd/dataframe-profile` | executable | Run and compare developer-facing dataframe profiling workloads. | P: main.go; T: —; Other: — | 322/0 | 0: — | conformance/compiler; generated/graphql/graph/model; internal/api/graphql/graph/query; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/recipe; internal/dataframe/semantic; internal/store/arango | — | executable boundary; compare orchestration with sibling commands before combining | -| `cmd/dataframe-query` | executable | Execute a checked-in GraphQL dataframe request and report response timing and diagnostics. | P: main.go; T: main_test.go; Other: — | 241/81 | 0: — | — | — | executable boundary; compare orchestration with sibling commands before combining | +| `cmd/dataframe-profile` | executable | Run and compare developer-facing dataframe profiling workloads. | P: main.go; T: —; Other: — | 331/0 | 0: — | conformance/compiler; generated/graphql/graph/model; internal/api/graphql/graph/query; internal/backends/arango; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/recipe; internal/dataframe/semantic | — | executable boundary; compare orchestration with sibling commands before combining | +| `cmd/dataframe-query` | executable | Execute a checked-in GraphQL dataframe request and report response timing and diagnostics. | P: main.go; T: main_test.go; Other: — | 459/432 | 3: Error; String; Unwrap | — | — | executable boundary; compare orchestration with sibling commands before combining | | `cmd/explorer-config-v2-convert` | executable | Convert legacy Explorer configuration input into the V2 authoring representation offline. | P: main.go; T: main_test.go; Other: — | 377/62 | 0: — | internal/dataframe/recipe; internal/explorer; internal/explorer/authoringv2 | — | executable boundary; compare orchestration with sibling commands before combining | | `cmd/generate` | executable | Generate FHIR models, schema metadata, validation, extraction, and GraphQL artifacts from source schemas. | P: extraction.go; graphql.go; helpers.go; main.go; metadata.go; model.go; names.go; resources.go; schema.go; validation.go; T: extraction_test.go; graphql_test.go; metadata_test.go; model_test.go; resources_test.go; root_test.go; schema_fixture_test.go; Other: fhir_helpers.go.tmpl | 1,750/522 | 10: BindingStrength; Definition; EdgeDocument; Link; Property; Result; Schema; TargetHints; TargetSchema; TemplatePointers | — | — | executable boundary; compare orchestration with sibling commands before combining; runtime/build signals present; static importer counts are insufficient for deletion | -| `cmd/gqlgenfix` | executable | Post-process gqlgen output to enforce Loom-specific generated-code invariants. | P: main.go; T: main_test.go; Other: — | 168/67 | 1: MarshalJSON | — | — | executable boundary; compare orchestration with sibling commands before combining | +| `cmd/gqlgenfix` | executable | Post-process gqlgen output to enforce Loom-specific generated-code invariants. | P: main.go; T: main_test.go; Other: — | 173/67 | 1: MarshalJSON | — | — | executable boundary; compare orchestration with sibling commands before combining | | `cmd/loom-acceptance` | executable | Run the locked real-data acceptance scenario and emit machine-readable verification artifacts. | P: main.go; T: main_test.go; Other: — | 225/46 | 0: — | internal/acceptance; internal/explorer | — | executable boundary; compare orchestration with sibling commands before combining | +| `cmd/loom-data` | executable | Load one immutable generation into the backend chosen only by an explicit arango or postgres tag, routing to the Arango loader or the PostgreSQL lifecycle loader without inference or fallback. | P: load.go; main.go; T: load_test.go; main_test.go; Other: — | 964/1,703 | 2: Error; Unwrap | internal/backends/arango; internal/backends/postgres; internal/backends/postgres/control; internal/catalog; internal/dataset; internal/ingest | — | executable boundary; compare orchestration with sibling commands before combining | | `cmd/loom-repository` | executable | Preflight a repository checkout and launch Loom against its META and CONFIG data with an owned Compose project. | P: main.go; T: main_test.go; Other: — | 426/231 | 0: — | internal/projectid; internal/repositoryseed | — | executable boundary; compare orchestration with sibling commands before combining | -| `cmd/preview-covering-index-artifact` | executable | Recreate a captured Builder proposal offline and emit the exact compiler artifact for index-coverage analysis. | P: main.go; T: main_test.go; Other: — | 1,433/34 | 0: — | internal/authscope; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation | — | executable boundary; compare orchestration with sibling commands before combining | -| `conformance/compiler` | conformance | Load and exercise the canonical dataframe compiler oracle corpus for conformance and performance checks. | P: types.go; T: benchmark_test.go; corpus_coverage_test.go; gdc_fixture_test.go; generated_semantics_coverage_test.go; physical_plan_cost_test.go; policy_ablation_test.go; run_test.go; schema_roots_test.go; types_test.go; Other: — | 119/1,226 | 4: Expected; Fixture; LoadDir; Validate | internal/dataframe/recipe | cmd/dataframe-profile | non-runtime conformance boundary | +| `cmd/preview-covering-index-artifact` | executable | Recreate a captured Builder proposal offline and emit the exact compiler artifact for index-coverage analysis. | P: main.go; T: main_test.go; Other: — | 1,449/34 | 0: — | internal/authscope; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation | — | executable boundary; compare orchestration with sibling commands before combining | +| `conformance/compiler` | conformance | Load and exercise the canonical dataframe compiler oracle corpus for conformance and performance checks. | P: types.go; T: benchmark_test.go; command_helpers_test.go; corpus_coverage_test.go; gdc_fixture_test.go; generated_semantics_coverage_test.go; physical_plan_cost_test.go; policy_ablation_test.go; run_test.go; schema_roots_test.go; types_test.go; Other: — | 119/1,279 | 4: Expected; Fixture; LoadDir; Validate | internal/dataframe/recipe | cmd/dataframe-profile | non-runtime conformance boundary | | `generated/fhir` | generated | Expose generated FHIR resource models and their extraction, validation, and GraphQL helpers. | P: extract.go; graphql.go; helpers.go; model.go; resources.go; validate.go; T: —; Other: — | 72,020/0 | 155: Address; Age; Annotation; Attachment; Availability; AvailabilityAvailableTime; AvailabilityNotAvailableTime; BodyStructure; BodyStructureIncludedStructure; BodyStructureIncludedStructureBodyLandmarkOrientation; BodyStructureIncludedStructureBodyLandmarkOrientationDistanceFromLandmark; CodeableConcept; … (+143) | internal/fhir/schema | generated/graphql/graph/executor; internal/api/graphql/graph/resolver; internal/dataframe/recipe; internal/dataframe/spec; internal/ingest | generated boundary; change its source/generator, not emitted code; runtime/build signals present; static importer counts are insufficient for deletion | | `generated/fhirschema` | generated | Expose generated raw FHIR schema metadata tables. | P: generated.go; T: —; Other: — | 55,277/0 | 3: Definition; Property; Traversal | — | internal/fhir/schema | generated boundary; change its source/generator, not emitted code | -| `generated/graphql/graph/executor` | generated | Provide gqlgen-generated GraphQL executable schema and execution plumbing. | P: fhir_schema.generated.go; prelude.generated.go; root_.generated.go; schema.generated.go; T: —; Other: — | 155,290/0 | 328: Address; AddressResolver; Age; AgeResolver; Annotation; AnnotationResolver; Attachment; AttachmentResolver; Availability; AvailabilityAvailableTime; AvailabilityAvailableTimeResolver; AvailabilityNotAvailableTime; … (+316) | generated/fhir; generated/graphql/graph/model | internal/api/graphql/graph; internal/api/graphql/graph/resolver | generated boundary; change its source/generator, not emitted code; runtime/build signals present; static importer counts are insufficient for deletion | +| `generated/graphql/graph/executor` | generated | Provide gqlgen-generated GraphQL executable schema and execution plumbing. | P: fhir_schema.generated.go; prelude.generated.go; root_.generated.go; schema.generated.go; T: —; Other: — | 155,289/0 | 328: Address; AddressResolver; Age; AgeResolver; Annotation; AnnotationResolver; Attachment; AttachmentResolver; Availability; AvailabilityAvailableTime; AvailabilityAvailableTimeResolver; AvailabilityNotAvailableTime; … (+316) | generated/fhir; generated/graphql/graph/model | internal/api/graphql/graph; internal/api/graphql/graph/resolver | generated boundary; change its source/generator, not emitted code; runtime/build signals present; static importer counts are insufficient for deletion | | `generated/graphql/graph/model` | generated | Provide gqlgen-generated GraphQL transport model types. | P: models.go; T: —; Other: — | 1,333/0 | 95: DataframeAggregateInput; DataframeAggregateResult; DataframeAggregationSpecInput; DataframeAggregationsInput; DataframeAggregationsResult; DataframeColumn; DataframeCompilerPlanDiagnostics; DataframeDatasetInput; DataframeFilterInput; DataframeMaterialization; DataframeMaterializationState; DataframeOptimizationDecision; … (+83) | — | cmd/dataframe-profile; generated/graphql/graph/executor; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver | generated boundary; change its source/generator, not emitted code; runtime/build signals present; static importer counts are insufficient for deletion | -| `generated/loomapi` | generated | Provide OpenAPI-generated HTTP request, response, and strict-server contracts. | P: api.gen.go; strict_json.go; T: construction_json_test.go; correlated_json_test.go; selection_json_test.go; strict_json_test.go; Other: — | 22,807/365 | 1597: APIVersion; AcceptedConstruction; AcceptedConstructionReceipt; Action; ActionColumn; ActivateDatasetGeneration; ActivateDatasetGeneration200JSONResponse; ActivateDatasetGeneration400JSONResponse; ActivateDatasetGeneration401JSONResponse; ActivateDatasetGeneration403JSONResponse; ActivateDatasetGeneration409JSONResponse; ActivateDatasetGeneration500JSONResponse; … (+1585) | internal/explorer/authoringv2 | internal/server | generated boundary; change its source/generator, not emitted code; runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/acceptance` | domain/library | Fetch and cache the locked FHIR fixture, drive the real-data acceptance scenario, and verify its outputs and performance evidence. | P: fixture.go; performance.go; runner.go; scenario.go; T: fixture_test.go; performance_test.go; scenario_test.go; Other: — | 2,257/1,079 | 41: Acquire; Close; ComparePerformance; ComparePerformanceReportFiles; ComparePerformanceRuns; ComputedDigest; Config; Connections; DefaultLimits; Fetcher; FixtureDigest; FixtureLock; … (+29) | internal/dataframe/publication; internal/dataframe/publication/arango; internal/explorer/authoringv2; internal/store/arango; internal/store/clickhouse | cmd/loom-acceptance | one production importer (cmd/loom-acceptance); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion | +| `generated/loomapi` | generated | Provide OpenAPI-generated HTTP request, response, and strict-server contracts. | P: api.gen.go; strict_json.go; T: construction_json_test.go; correlated_json_test.go; request_boundary_test.go; selection_json_test.go; strict_json_test.go; Other: — | 23,408/512 | 1620: APIVersion; AcceptedConstruction; AcceptedConstructionReceipt; Action; ActionColumn; ActivateDatasetGeneration; ActivateDatasetGeneration200JSONResponse; ActivateDatasetGeneration400JSONResponse; ActivateDatasetGeneration401JSONResponse; ActivateDatasetGeneration403JSONResponse; ActivateDatasetGeneration409JSONResponse; ActivateDatasetGeneration500JSONResponse; … (+1608) | internal/explorer/authoringv2 | internal/server | generated boundary; change its source/generator, not emitted code; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/acceptance` | domain/library | Fetch and cache the locked FHIR fixture, drive the real-data acceptance scenario, and verify its outputs and performance evidence. | P: fixture.go; performance.go; runner.go; scenario.go; T: fixture_test.go; performance_test.go; runner_locking_test.go; runner_status_test.go; runner_zero_value_test.go; scenario_test.go; Other: — | 2,283/1,570 | 41: Acquire; Close; ComparePerformance; ComparePerformanceReportFiles; ComparePerformanceRuns; ComputedDigest; Config; Connections; DefaultLimits; Fetcher; FixtureDigest; FixtureLock; … (+29) | internal/backends/arango; internal/dataframe/publication; internal/explorer/authoringv2; internal/store/clickhouse | cmd/loom-acceptance | one production importer (cmd/loom-acceptance); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion | | `internal/api/authpolicy` | transport adapter | Centralize authorization-sentinel classification for transport operations while preserving operation-specific concealment. | P: policy.go; T: policy_test.go; Other: — | 56/44 | 2: Classify; Operation | internal/authscope; internal/dataframe/errors | internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/server | no automatic consolidation signal; perform semantic review | -| `internal/api/bulk/load` | transport adapter | Adapt multipart HTTP generation uploads and activation requests to ingest and dataset services. | P: errors.go; generation_staging.go; handler.go; operations.go; service.go; T: errors_test.go; operations_test.go; service_test.go; Other: — | 453/442 | 18: ActivateDatasetGeneration; Config; CreateDatasetGeneration; DataframeReleaseStore; GenerationActivationResult; GenerationActivator; GenerationLoadRequest; GenerationLoadResult; GenerationLoader; GenerationStatusResult; GetDatasetGenerationStatus; Handler; … (+6) | internal/authscope; internal/dataframe/errors; internal/dataframe/publication; internal/dataset; internal/ingest | internal/server | one production importer (internal/server); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/api/bulk/load` | transport adapter | Adapt multipart HTTP generation uploads and activation requests to ingest and dataset services. | P: errors.go; generation_staging.go; handler.go; operations.go; service.go; T: cross_project_dataset_authorization_e2e_test.go; errors_test.go; fb22_ingress_control_test.go; form_spillover_test.go; operations_test.go; service_test.go; Other: — | 852/2,499 | 23: ActivateDatasetGeneration; Cleanup; Config; CreateDatasetGeneration; DataframeReleaseStore; Error; FileCount; GenerationActivationResult; GenerationActivator; GenerationLoadRequest; GenerationLoadResult; GenerationLoader; … (+11) | internal/authscope; internal/dataframe/errors; internal/dataframe/publication; internal/dataset; internal/ingest | internal/server | one production importer (internal/server); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion | | `internal/api/columncapabilities` | transport adapter | Derive transport-neutral filter, sort, and aggregate capabilities from published ClickHouse column types. | P: capabilities.go; T: capabilities_test.go; Other: — | 56/23 | 2: FromClickHouse; Tuple | — | internal/api/graphql/graph/dataframe; internal/server | no automatic consolidation signal; perform semantic review | -| `internal/api/graphql/graph` | transport adapter | Own GraphQL HTTP handler construction, error presentation, and generated executor binding. | P: doc.go; errors.go; handler.go; T: aggregate_http_integration_test.go; errors_test.go; handler_test.go; http_integration_test.go; Other: — | 511/931 | 8: Flush; Header; NewApolloSandboxHandler; NewHandler; NewPlaygroundHandler; RouteConfig; Write; WriteHeader | generated/graphql/graph/executor; internal/api/graphql/graph/resolver; internal/api/http; internal/dataframe/errors | internal/server | one production importer (internal/server); inspect ownership/inline potential | -| `internal/api/graphql/graph/dataframe` | transport adapter | Adapt published ClickHouse dataframe discovery, row, aggregate, and export operations to GraphQL. | P: aggregates.go; export.go; model.go; rows.go; service.go; T: aggregates_test.go; export_test.go; filter_test.go; model_test.go; service_test.go; Other: — | 1,062/346 | 17: AggregateInput; AggregateRowsResult; AggregationsInput; AggregationsJSON; ColumnFromPhysical; ColumnModel; Config; Dataset; ExportDataframe; Model; NewService; PersistedFailure; … (+5) | generated/graphql/graph/model; internal/api/authpolicy; internal/api/columncapabilities; internal/api/http; internal/authscope; internal/dataframe/errors; internal/dataframe/publication; internal/dataframe/published; internal/projectid | internal/api/graphql/graph/resolver | one production importer (internal/api/graphql/graph/resolver); inspect ownership/inline potential | -| `internal/api/graphql/graph/query` | transport adapter | Resolve and authorize Arango-backed FHIR dataframe and graph query requests for GraphQL. | P: active_generation.go; doc.go; errors.go; fhir_service.go; fieldrefs.go; graph_service.go; helpers.go; input_resolution.go; introspection.go; recipe_mapping.go; recipe_resolve.go; service.go; types.go; T: active_generation_test.go; auth_scope_test.go; errors_test.go; fhir_service_test.go; fieldrefs_test.go; graph_service_test.go; introspection_test.go; project_identity_test.go; recipe_mapping_test.go; Other: — | 1,850/848 | 28: Config; ExplainFHIRGraph; FHIRGraphExplanation; FHIRGraphNode; FHIRGraphPageInfo; FHIRGraphPath; FHIRGraphQuery; FHIRGraphRelationship; FHIRGraphResult; FHIRGraphTraversal; FHIRListRequest; FHIRListResult; … (+16) | generated/graphql/graph/model; internal/api/authpolicy; internal/api/recipe; internal/authscope; internal/catalog; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/recipe; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/dataframe/spec; internal/dataset; internal/fhir/schema; internal/projectid | cmd/dataframe-profile; internal/api/graphql/graph/resolver; internal/server | broad surface (13 production files); inspect cohesion | -| `internal/api/graphql/graph/resolver` | transport adapter | Implement gqlgen field resolvers and map GraphQL values to query and lifecycle services. | P: dataframe_diagnostics.go; fhir_reference_loader.go; fhir_resolver_helpers.go; fhir_schema.resolvers.go; graph_resolvers.go; graphql_output_helpers.go; operation_context.go; recipe_control_graphql.go; recipe_execution_reader.go; recipe_revision_model.go; resolver.go; schema.resolvers.go; T: fhir_reference_loader_test.go; output_mapping_test.go; recipe_control_test.go; Other: — | 5,376/315 | 362: AbatementDateTime; AbatementString; Active; Actual; ActualComparisonGroup; ActualNumber; Address; Age; AgeString; Alias; AllDay; AllowedBoolean; … (+350) | generated/fhir; generated/graphql/graph/executor; generated/graphql/graph/model; internal/api/authpolicy; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/authscope; internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/publication; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataset; internal/fhir/schema | internal/api/graphql/graph; internal/server | broad surface (12 production files); inspect cohesion; mixed handwritten/generated package (2 generated file(s)); preserve regeneration boundary; runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/api/http` | transport adapter | Own shared Fiber server setup, middleware, request IDs, errors, health, and readiness behavior. | P: doc.go; errors.go; middleware.go; request_id.go; routes.go; server.go; T: errors_test.go; http_test.go; request_id_test.go; Other: — | 531/558 | 15: App; ContextWithRequestID; ErrorResponse; HTTPConfig; HTTPErrorBody; HTTPServer; Health; Liveness; Logger; MapDataframeError; MappedError; NewHTTPServer; … (+3) | internal/authscope; internal/dataframe/errors | internal/api/graphql/graph; internal/api/graphql/graph/dataframe; internal/server | no automatic consolidation signal; perform semantic review | -| `internal/api/recipe` | transport adapter | Adapt scoped catalog field discovery to recipe schema resolution for one-shot and stored recipes. | P: discovery.go; T: discovery_test.go; Other: — | 103/50 | 3: FieldDiscovery; Fields; NewFieldDiscovery | internal/authscope; internal/catalog; internal/dataframe/errors; internal/dataframe/recipe/schema | internal/api/graphql/graph/query; internal/server | no automatic consolidation signal; perform semantic review | -| `internal/authscope` | domain/library | Represent request principals and resolve authorized project, generation, and resource scopes. | P: authentication.go; doc.go; fence_resource_access.go; principal.go; scope_resolver.go; T: authentication_test.go; scope_test.go; Other: — | 914/476 | 34: AllowAllAuthorizer; Authenticate; Authenticator; AuthorizeProject; AuthorizeReadProject; AuthorizeWrite; Authorizer; BasicAuthenticator; BearerTokenAuthenticator; CalyprAuthenticator; Clone; ContextWithPrincipal; … (+22) | internal/catalog; internal/projectid | cmd/preview-covering-index-artifact; internal/api/authpolicy; internal/api/bulk/load; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/api/http; internal/api/recipe; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/execution; internal/dataframe/execution/chartifact; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer/arango; internal/explorer/lifecycle; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/catalog` | domain/library | Own persistence-neutral observed FHIR field, pivot, relationship, and capability facts. | P: auth_scope.go; cache.go; capability_evidence.go; generation.go; helpers.go; limits.go; rebuild.go; relationships.go; resource_inventory.go; semantic.go; semantic_coverage.go; semantic_inventory.go; semantic_schema.go; types.go; write_profiler.go; T: auth_scope_test.go; cache_test.go; capability_evidence_test.go; generation_test.go; relationships_test.go; semantic_coding_display_projection_test.go; semantic_coverage_diagnostics_test.go; semantic_coverage_test.go; semantic_inventory_buildid_cache_test.go; semantic_inventory_category_test.go; semantic_inventory_emitter_anchor_test.go; semantic_inventory_test.go; semantic_schema_identifier_type_test.go; semantic_schema_test.go; semantic_test.go; write_profiler_coverage_test.go; write_profiler_test.go; Other: — | 6,209/3,653 | 127: AuthResourcePathOptions; Cache; CacheOptions; CacheStats; CapabilityEvidence; CapabilityEvidenceOptions; CapabilityEvidenceReader; CensusSemanticCoverage; ConceptCandidate; DecodeSemanticInventoryCursor; DefaultProfileLimits; DiscoverFields; … (+115) | internal/fhir/schema; internal/fhir/semantic | cmd/arango-fhir-proto; internal/api/graphql/graph/query; internal/api/recipe; internal/authscope; internal/catalog/arango; internal/explorer/authoringv2; internal/explorer/lifecycle; internal/ingest; internal/server | broad surface (15 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/catalog/arango` | storage adapter | Persist and retrieve catalog facts in ArangoDB. | P: codec.go; evidence.go; fields.go; relationships.go; resource_inventory.go; resources.go; semantic_coverage.go; semantic_coverage_envelope.go; semantic_inventory.go; store.go; T: resource_inventory_integration_test.go; resource_inventory_unit_test.go; semantic_coded_category_integration_test.go; semantic_coverage_envelope_test.go; semantic_coverage_resource_integration_test.go; semantic_coverage_test.go; semantic_inventory_category_test.go; semantic_inventory_integration_test.go; semantic_inventory_selection_test.go; semantic_inventory_source_availability_test.go; store_test.go; Other: — | 5,331/5,128 | 61: AccumulateRelationshipCatalog; AddSource; AdvanceSemanticInventoryBackfill; AdvanceSemanticInventoryBuild; AuditRelationshipEdges; BeginSemanticCoverageBuild; BeginSemanticCoverageResource; BeginSemanticInventoryBuild; ClaimSemanticInventoryBackfill; CompleteRetainedSemanticCoverageResource; CompleteSemanticCoverageResource; CompleteSemanticInventoryBackfill; … (+49) | internal/catalog; internal/fhir/schema; internal/store/arango | cmd/arango-fhir-proto; internal/explorer/lifecycle; internal/ingest; internal/server | storage adapter; preserve interface and persisted-data contracts | +| `internal/api/graphql/graph` | transport adapter | Own GraphQL HTTP handler construction, error presentation, and generated executor binding. | P: doc.go; errors.go; handler.go; T: aggregate_http_integration_test.go; errors_test.go; handler_test.go; http_integration_test.go; Other: — | 511/944 | 8: Flush; Header; NewApolloSandboxHandler; NewHandler; NewPlaygroundHandler; RouteConfig; Write; WriteHeader | generated/graphql/graph/executor; internal/api/graphql/graph/resolver; internal/api/http; internal/dataframe/errors | internal/server | one production importer (internal/server); inspect ownership/inline potential | +| `internal/api/graphql/graph/dataframe` | transport adapter | Adapt published ClickHouse dataframe discovery, row, aggregate, and export operations to GraphQL. | P: aggregates.go; export.go; model.go; rows.go; service.go; T: aggregates_test.go; export_test.go; fb22_principal_auth_path_control_test.go; filter_test.go; model_test.go; service_test.go; Other: — | 1,072/506 | 17: AggregateInput; AggregateRowsResult; AggregationsInput; AggregationsJSON; ColumnFromPhysical; ColumnModel; Config; Dataset; ExportDataframe; Model; NewService; PersistedFailure; … (+5) | generated/graphql/graph/model; internal/api/authpolicy; internal/api/columncapabilities; internal/api/http; internal/authscope; internal/dataframe/errors; internal/dataframe/publication; internal/dataframe/published; internal/projectid | internal/api/graphql/graph/resolver; internal/server | no automatic consolidation signal; perform semantic review | +| `internal/api/graphql/graph/query` | transport adapter | Resolve and authorize Arango-backed FHIR dataframe and graph query requests for GraphQL. | P: active_generation.go; doc.go; errors.go; fhir_service.go; fieldrefs.go; graph_service.go; helpers.go; input_resolution.go; introspection.go; recipe_mapping.go; recipe_resolve.go; service.go; types.go; T: active_generation_test.go; auth_scope_test.go; errors_test.go; fhir_service_test.go; fieldrefs_test.go; graph_service_test.go; introspection_test.go; postgres_query_test.go; project_identity_test.go; read_scope_ingress_control_chars_e2e_test.go; recipe_mapping_test.go; service_test.go; Other: — | 1,888/2,361 | 28: Config; ExplainFHIRGraph; FHIRGraphExplanation; FHIRGraphNode; FHIRGraphPageInfo; FHIRGraphPath; FHIRGraphQuery; FHIRGraphRelationship; FHIRGraphResult; FHIRGraphTraversal; FHIRListRequest; FHIRListResult; … (+16) | generated/graphql/graph/model; internal/api/authpolicy; internal/api/recipe; internal/authscope; internal/catalog; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/recipe; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/dataframe/spec; internal/dataset; internal/fhir/schema; internal/projectid | cmd/dataframe-profile; internal/api/graphql/graph/resolver; internal/server | broad surface (13 production files); inspect cohesion | +| `internal/api/graphql/graph/resolver` | transport adapter | Implement gqlgen field resolvers and map GraphQL values to query and lifecycle services. | P: dataframe_diagnostics.go; fhir_reference_loader.go; fhir_resolver_helpers.go; fhir_schema.resolvers.go; graph_resolvers.go; graphql_output_helpers.go; operation_context.go; recipe_control_graphql.go; recipe_execution_reader.go; recipe_revision_model.go; resolver.go; schema.resolvers.go; T: fhir_reference_loader_test.go; output_mapping_test.go; recipe_control_test.go; recipe_execution_malformed_persisted_scope_test.go; recipe_execution_malformed_scope_oracle_probe_test.go; recipe_execution_reader_authz_e2e_test.go; recipe_execution_twin_wire_parity_test.go; recipe_revision_register_test.go; Other: — | 5,459/2,334 | 362: AbatementDateTime; AbatementString; Active; Actual; ActualComparisonGroup; ActualNumber; Address; Age; AgeString; Alias; AllDay; AllowedBoolean; … (+350) | generated/fhir; generated/graphql/graph/executor; generated/graphql/graph/model; internal/api/authpolicy; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/authscope; internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/publication; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataset; internal/fhir/schema | internal/api/graphql/graph; internal/server | broad surface (12 production files); inspect cohesion; mixed handwritten/generated package (2 generated file(s)); preserve regeneration boundary; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/api/http` | transport adapter | Own shared Fiber server setup, middleware, request IDs, errors, health, and readiness behavior. | P: doc.go; errors.go; middleware.go; request_id.go; routes.go; server.go; T: backend_error_boundary_contract_test.go; errors_test.go; health_readiness_test.go; http_test.go; request_id_test.go; Other: — | 642/1,145 | 15: App; ContextWithRequestID; ErrorResponse; HTTPConfig; HTTPErrorBody; HTTPServer; Health; Liveness; Logger; MapDataframeError; MappedError; NewHTTPServer; … (+3) | internal/authscope; internal/dataframe/errors | internal/api/graphql/graph; internal/api/graphql/graph/dataframe; internal/server | no automatic consolidation signal; perform semantic review | +| `internal/api/recipe` | transport adapter | Adapt scoped catalog field discovery to recipe schema resolution for one-shot and stored recipes. | P: discovery.go; T: discovery_auth_mode_test.go; discovery_test.go; Other: — | 110/126 | 3: FieldDiscovery; Fields; NewFieldDiscovery | internal/authscope; internal/catalog; internal/dataframe/errors; internal/dataframe/recipe/schema | internal/api/graphql/graph/query; internal/server | no automatic consolidation signal; perform semantic review | +| `internal/authscope` | domain/library | Represent request principals and resolve authorized project, generation, and resource scopes. | P: authentication.go; doc.go; fence_resource_access.go; principal.go; scope_resolver.go; T: all_generations_cache_key_test.go; auth_path_control_ingress_test.go; auth_resource_path_options_conflict_test.go; authentication_test.go; covers_execution_paths_control_test.go; persisted_scope_fails_closed_test.go; probe_double_normalize_test.go; scope_cache_key_namespace_test.go; scope_test.go; Other: — | 1,175/2,104 | 36: AllowAllAuthorizer; Authenticate; Authenticator; AuthorizeProject; AuthorizeReadProject; AuthorizeWrite; Authorizer; BasicAuthenticator; BearerTokenAuthenticator; CalyprAuthenticator; Clone; ContextWithPrincipal; … (+24) | internal/catalog; internal/projectid | cmd/preview-covering-index-artifact; internal/api/authpolicy; internal/api/bulk/load; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/api/http; internal/api/recipe; internal/backends/arango/internal/explorer; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/execution; internal/dataframe/execution/chartifact; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer/lifecycle; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/backends/arango` | storage adapter | Expose the only public entry to the ArangoDB 3.12 backend: a single Open factory validates explicit config, owns exactly one lifecycle client with the required collection bootstrap policy, and returns the typed runtime carrying the catalog, dataset lifecycle, Explorer, recipe registry/revisions, and published-dataframe ports assembled from the private implementation tree, with alias re-exports for witness-side reassembly. | P: aliases.go; doc.go; open.go; T: ownership_contract_test.go; Other: — | 336/800 | 4: Close; Config; Open; Runtime | internal/backends/arango/internal/catalog; internal/backends/arango/internal/chartifact; internal/backends/arango/internal/dataset; internal/backends/arango/internal/explorer; internal/backends/arango/internal/publication; internal/backends/arango/internal/recipeexec; internal/backends/arango/internal/store | cmd/arango-fhir-proto; cmd/dataframe-profile; cmd/loom-data; internal/acceptance; internal/ingest; internal/server | storage adapter; preserve interface and persisted-data contracts | +| `internal/backends/arango/internal/catalog` | domain/library | Persist and retrieve catalog facts in ArangoDB. | P: codec.go; evidence.go; fields.go; relationships.go; resource_inventory.go; resources.go; semantic_coverage.go; semantic_coverage_envelope.go; semantic_inventory.go; store.go; T: all_generations_auth_path_conflict_test.go; all_generations_project_binding_integration_test.go; resource_inventory_integration_test.go; resource_inventory_unit_test.go; semantic_coded_category_integration_test.go; semantic_coverage_envelope_test.go; semantic_coverage_resource_integration_test.go; semantic_coverage_test.go; semantic_inventory_category_test.go; semantic_inventory_integration_test.go; semantic_inventory_selection_test.go; semantic_inventory_source_availability_test.go; store_test.go; Other: — | 5,267/5,466 | 54: AccumulateRelationshipCatalog; AddSource; AdvanceSemanticInventoryBackfill; AdvanceSemanticInventoryBuild; AuditRelationshipEdges; BeginSemanticCoverageBuild; BeginSemanticCoverageResource; BeginSemanticInventoryBuild; ClaimSemanticInventoryBackfill; CompleteRetainedSemanticCoverageResource; CompleteSemanticCoverageResource; CompleteSemanticInventoryBackfill; … (+42) | internal/backends/arango/internal/store; internal/catalog; internal/fhir/schema | internal/backends/arango | one production importer (internal/backends/arango); inspect ownership/inline potential | +| `internal/backends/arango/internal/chartifact` | domain/library | Persist private ClickHouse artifact manifests and leases in a dedicated Arango collection. | P: catalog.go; T: catalog_arango_test.go; catalog_test.go; Other: — | 379/723 | 10: BootstrapSpec; Catalog; ClaimCleanup; Create; Delete; ListExpired; NewCatalog; ReleaseCleanup; Renew; Update | internal/backends/arango/internal/store; internal/dataframe/execution/chartifact | internal/backends/arango | one production importer (internal/backends/arango); inspect ownership/inline potential | +| `internal/backends/arango/internal/dataset` | domain/library | Persist immutable dataset generation lifecycle state in ArangoDB. | P: collections.go; doc.go; document_codec.go; manifest_lifecycle.go; release_store.go; T: generation_activation_cas_e2e_test.go; release_policy_change_e2e_test.go; revision_inflation_e2e_test.go; store_test.go; Other: — | 764/1,652 | 15: Activate; ActiveDocumentKey; BootstrapSpec; CollectionSpecs; CompareAndSwapActivateRelease; CompareAndSwapPublishRelease; CreateManifest; ManifestDocumentKey; New; ReadActiveRelease; ReadManifest; ResolveActiveManifest; … (+3) | internal/backends/arango/internal/store; internal/dataset | internal/backends/arango; internal/backends/arango/internal/explorer | no automatic consolidation signal; perform semantic review | +| `internal/backends/arango/internal/explorer` | domain/library | Persist Explorer drafts, revisions, receipts, publications, capabilities, and repository state in ArangoDB. | P: activation.go; capability_snapshot_store.go; codec.go; collections.go; drafts.go; explicit_groups.go; explorers.go; interpretations.go; publication.go; receipts.go; repository.go; revisions.go; selection_references.go; selections.go; store.go; table_shape_capabilities.go; T: capability_snapshot_store_test.go; collections_test.go; cross_generation_publish_e2e_test.go; draft_conflict_test.go; draft_transaction_formal_replay_test.go; drafts_integration_test.go; drafts_test.go; explicit_group_storage_integration_test.go; explicit_groups_test.go; interpretations_test.go; receipt_store_test.go; repository_test.go; selection_references_test.go; selection_scale_integration_test.go; selection_storage_integration_test.go; selections_cleanup_e2e_test.go; store_test.go; table_shape_capabilities_test.go; Other: — | 3,366/4,552 | 56: AbortExplicitGroupRevision; AbortSelection; ActivateRepositoryGeneration; AppendExplicitGroupMemberships; AppendSelectionMembers; BeginExplicitGroupRevision; BeginSelection; BootstrapSpec; CapabilitySnapshotCollectionSpec; CapabilitySnapshotStore; CleanupExplicitGroupStaging; CleanupSelectionStaging; … (+44) | internal/authscope; internal/backends/arango/internal/dataset; internal/backends/arango/internal/publication; internal/backends/arango/internal/store; internal/dataframe/publication; internal/dataset; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/tableshapecap; internal/projectid | internal/backends/arango | one production importer (internal/backends/arango); inspect ownership/inline potential; broad surface (16 production files); inspect cohesion | +| `internal/backends/arango/internal/publication` | domain/library | Persist dataframe publication registry and bundle metadata in ArangoDB. | P: bundle_registry.go; registry.go; T: bundle_registry_test.go; dangling_binding_e2e_test.go; execution_read_pin_integration_test.go; name_latest_binding_e2e_test.go; publish_atomicity_e2e_test.go; release_dangle_reconcile_e2e_test.go; Other: — | 533/2,990 | 22: AcquireBundleLease; AcquireExecutionReadPin; BootstrapSpec; ClaimExecutionCleanup; Client; CompareAndSwapPointer; FindExecutionByKey; FindExecutionBySelector; GetExecution; GetPointer; ListExecutions; New; … (+10) | internal/backends/arango/internal/store; internal/dataframe/publication | internal/backends/arango; internal/backends/arango/internal/explorer | no automatic consolidation signal; perform semantic review | +| `internal/backends/arango/internal/recipeexec` | domain/library | Persist recipe versions and execution-use records in ArangoDB. | P: registry.go; revisions.go; T: registry_test.go; revisions_test.go; revisions_user_faults_test.go; Other: — | 294/555 | 15: BootstrapSpec; Client; Get; List; LoadRecipe; LoadRecipeVersion; New; NewRevisionRegistry; RecipeVersionUsed; Register; Registry; ReplaceRecipeVersion; … (+3) | internal/backends/arango/internal/store; internal/dataframe/errors; internal/dataframe/recipe; internal/dataframe/recipe/exec | internal/backends/arango | one production importer (internal/backends/arango); inspect ownership/inline potential | +| `internal/backends/arango/internal/store` | domain/library | Provide the low-level typed ArangoDB client, query streaming, profiling, and Explain assessment boundary. | P: client.go; errors.go; explain.go; explain_assessment.go; explain_client.go; options.go; preview_index.go; profile.go; profile_client.go; types.go; T: client_query_memory_limit_test.go; client_queryrows_test.go; client_revision_test.go; errors_test.go; explain_assessment_test.go; explain_client_test.go; explain_test.go; population_profile_integration_test.go; preview_index_test.go; profile_client_test.go; profile_integration_test.go; profile_test.go; Other: — | 1,950/2,210 | 70: AQLExecutor; AssessExplainResult; BatchInserter; Bootstrap; BootstrapSpec; Client; Close; CollectionCount; CollectionExists; CollectionRevision; CollectionSpec; ConnectionOptions; … (+58) | internal/dataframe/errors | internal/backends/arango; internal/backends/arango/internal/catalog; internal/backends/arango/internal/chartifact; internal/backends/arango/internal/dataset; internal/backends/arango/internal/explorer; internal/backends/arango/internal/publication; internal/backends/arango/internal/recipeexec | runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/backends/postgres` | storage adapter | Own the PostgreSQL 18 factory: a single Open validates explicit config, opens exactly one readiness-gated pool with own-prefix connection tagging, composes the control runtime, and returns the typed runtime carrying the SourceRows and PlanJSON data-plane ports over tagged compiled commands through the typed Querier bridge; the sole legal importer of the private rows adapter, and no re-export surface - control-plane consumers import internal/backends/postgres/control directly (arch-backend-09 deleted aliases.go). | P: doc.go; open.go; plan_executor.go; T: engine_postgres_command_boundary_pg18_test.go; engine_postgres_pg18_test.go; explain_postgres_test.go; open_live_test.go; ownership_contract_test.go; plan_executor_test.go; sentinel_identity_test.go; Other: — | 178/4,408 | 5: NewPlanJSONExecutor; NewSourceRows; Open; Querier; Runtime | internal/backends/postgres/control; internal/backends/postgres/internal/rows; internal/backends/postgres/internal/store; internal/dataframe/compiler/command; internal/dataframe/execution | cmd/loom-data; internal/ingest; internal/server | storage adapter; preserve interface and persisted-data contracts | +| `internal/backends/postgres/control` | domain/library | Own the cycle-free public control plane of the PostgreSQL backend as its single definition: configuration and series-assignment validation, the bare-pool client and store-owned batch vocabulary, the dataset manifest lifecycle and catalog evidence port assembly, and the structural query surface; production consumers and backend-adjacent witnesses import it directly for control-plane symbols, and it must never reach the factory root, the private rows adapter, or dataframe/execution (machine-enforced by scripts/packagepolicy). | P: catalog_read.go; config.go; errors.go; runtime.go; vocabulary.go; vocabulary_control.go; T: catalog_read_test.go; Other: — | 1,386/284 | 65: AllSeries; Begin; CatalogStore; Close; Commit; Config; ConnSource; DiscoverExistingAuthResourcePaths; DiscoverFields; Error; Exec; FieldProfileScope; … (+53) | internal/backends/postgres/internal/catalog; internal/backends/postgres/internal/chartifact; internal/backends/postgres/internal/dataset; internal/backends/postgres/internal/explorer; internal/backends/postgres/internal/publication; internal/backends/postgres/internal/recipeexec; internal/backends/postgres/internal/recipes; internal/backends/postgres/internal/redact; internal/backends/postgres/internal/store; internal/catalog; internal/dataset | cmd/loom-data; internal/backends/postgres; internal/ingest; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/backends/postgres/internal/catalog` | domain/library | Persist and retrieve catalog facts in PostgreSQL 18: publish-last generation-scoped resource inventory builds, bounded field-profile records with completeness and truncation metadata, leased compare-and-swap-guarded semantic inventory build state, deterministic semantic-inventory contribution rows with content-addressed replay and conflict classification, byte-order-stable keyset pages over retained resources, and authorization-scoped semantic-inventory pages and selection resolution served directly from contribution aggregates. | P: field_discovery.go; field_profile.go; relationships.go; resource_inventory.go; semantic_build.go; semantic_contribution.go; semantic_page.go; store.go; T: field_discovery_test.go; field_profile_test.go; relationships_pg_test.go; relationships_test.go; resource_inventory_test.go; semantic_build_test.go; semantic_contribution_test.go; semantic_page_test.go; Other: — | 4,340/11,088 | 54: AdvanceSemanticInventoryBackfill; ClaimSemanticInventoryBackfill; CompleteSemanticInventoryBackfill; DiscoverExistingAuthResourcePaths; DiscoverFieldEnrichment; DiscoverFields; DiscoverRelationshipObservations; DiscoverResourceInventory; EnsureSemanticInventoryEntries; Error; FailSemanticInventoryBackfill; FieldProfileStore; … (+42) | internal/backends/postgres/internal/store; internal/catalog; internal/fhir/schema | internal/backends/postgres/control | one production importer (internal/backends/postgres/control); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/backends/postgres/internal/chartifact` | domain/library | Persist private ClickHouse artifact manifests and cleanup leases in the PostgreSQL 18 control plane with CAS progress writes, bounded expired scans, and one-winner cleanup claims while physical tables stay in ClickHouse. | P: catalog.go; T: catalog_manager_test.go; catalog_test.go; Other: — | 535/1,605 | 10: Catalog; ClaimCleanup; Create; DB; Delete; ListExpired; NewCatalog; ReleaseCleanup; Renew; Update | internal/dataframe/execution/chartifact | internal/backends/postgres/control | one production importer (internal/backends/postgres/control); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/backends/postgres/internal/dataset` | domain/library | Persist dataset generation manifests, the active-generation compare-and-swap pointer, and immutable project release candidates with the active-release compare-and-swap pointer in PostgreSQL 18. | P: commands.go; manifest_lifecycle.go; migrate.go; release_store.go; store.go; T: commands_test.go; manifest_lifecycle_pg_test.go; release_store_pg_test.go; Other: — | 1,725/2,488 | 28: ActivateGeneration; Backend; Command; CommandBackend; CompareAndSwapActivateRelease; ConnSource; CreateManifest; EnsureGenerationPrerequisites; GenerationOwnerAdapter; MigrateSchema; New; NewPostgreSQLCommand; … (+16) | internal/backends/postgres/internal/store; internal/dataset | internal/backends/postgres/control; internal/backends/postgres/internal/explorer | runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/backends/postgres/internal/explorer` | domain/library | Persist Explorer owner records, draft saves with atomic version/digest CAS and exact preimage history, insert-once compilation receipts, immutable Explorer revisions, create-once immutable capability snapshots and table-shape capability receipts, project-scoped interpretation libraries advancing heads over immutable content-addressed revisions by transactional parent CAS, the staged selection lifecycle with project-scoped idempotency, an expiring writer owner, and insert-once membership staged in bounded idempotent batches with the digest/count finalized under the current writer fence and bounded cancellation-aware keyset membership paging, atomic publish authoring that commits a verified compilation receipt, the immutable Explorer revision, the active Explorer owner pointer, and the project release activation CAS in one PostgreSQL transaction, and staging explicit-group revision headers with ordered definition rows and insert-once group membership (one row per revision/group/reference so an overlap is several rows, bounded keyset paging, and a fenced digest/count-verified STAGING to COMPLETE transition) under guarded writer-lease, abort, and expired-cleanup lifecycle in PostgreSQL 18 with unique project/Explorer identity, typed columns, lossless JSONB payloads, DB-enforced row freezing and content-digest collision checks, guard-triggered immutability, and CAS-guarded revision lifecycle on the shared pg-store pool and migration runner. | P: capabilities.go; drafts.go; explicit_group_memberships.go; explicit_groups.go; explorers.go; interpretations.go; publication.go; receipts.go; revisions.go; selection_members.go; selections.go; T: capabilities_test.go; drafts_test.go; explicit_groups_readpath_test.go; explicit_groups_test.go; explorers_test.go; interpretations_test.go; pgstate02_fixture_test.go; pubauth_fixture_test.go; publication_golden_test.go; publication_guarddrop_test.go; publication_race_test.go; publication_retry_test.go; publication_test.go; publication_zz_test.go; receipts_test.go; revisions_test.go; selection_members_test.go; selections_test.go; Other: — | 7,425/21,659 | 60: AbortExplicitGroupRevision; AbortSelection; AppendExplicitGroupMemberships; AppendSelectionMembers; BeginExplicitGroupRevision; BeginSelection; CapabilitySnapshotStore; CleanupExplicitGroupStaging; CleanupSelectionStaging; CompleteExplicitGroupRevision; CompleteSelection; Create; … (+48) | internal/backends/postgres/internal/dataset; internal/backends/postgres/internal/store; internal/dataframe/publication; internal/dataframe/recipe; internal/dataset; internal/explorer; internal/explorer/capability; internal/explorer/tableshapecap | internal/backends/postgres/control | one production importer (internal/backends/postgres/control); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/backends/postgres/internal/publication` | domain/library | Persist dataframe publication execution metadata, exact logical pointers, fenced bundle leases, owner-conditional execution checkpoints, read retention pins arbitrated against cleanup claims on a shared retention anchor, and page-scoped stale execution reconciliation with owner-conditional cleanup claims, plus the single-transaction publish commit of PUBLISHED state with its reader pointer, in the PostgreSQL 18 control plane while ClickHouse remains the row store. | P: leases.go; migrate.go; publish.go; read_pins.go; reconcile.go; registry.go; T: leases_test.go; publish_atomicity_test.go; read_pins_test.go; reconcile_test.go; registry_test.go; Other: — | 2,502/7,214 | 44: AcquireBundleLease; AcquireExecutionReadPin; AcquireLease; BundleCatalog; Checkpoint; CheckpointRecord; ClaimExecutionCleanup; CompareAndSwapPointer; DB; ExpireLease; FencedLeases; FindExecutionByKey; … (+32) | internal/backends/postgres/internal/store; internal/dataframe/errors; internal/dataframe/publication; internal/dataset | internal/backends/postgres/control | one production importer (internal/backends/postgres/control); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/backends/postgres/internal/recipeexec` | domain/library | Persist immutable recipe registrations and fenced execution-occurrence usage markers in the PostgreSQL 18 control plane. | P: registry.go; T: registry_test.go; Other: — | 428/1,087 | 12: DB; LoadRecipe; LoadRecipeVersion; MigrateSchema; New; RecipeVersionUsed; RecordRecipeExecutionOccurrence; Registry; RegistryMigrations; ReplaceRecipeVersion; SaveRecipe; VerifySchema | internal/backends/postgres/internal/store; internal/dataframe/recipe/exec | internal/backends/postgres/control | one production importer (internal/backends/postgres/control); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/backends/postgres/internal/recipes` | domain/library | Persist project recipe revision chains with CAS head tracking in the PostgreSQL 18 control plane. | P: revisions.go; T: revisions_test.go; Other: — | 409/1,072 | 9: ChainMigrations; DB; Get; List; MigrateSchema; New; Register; RevisionRegistry; VerifySchema | internal/backends/postgres/internal/store; internal/dataframe/errors; internal/dataframe/recipe | internal/backends/postgres/control | one production importer (internal/backends/postgres/control); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/backends/postgres/internal/redact` | domain/library | Scrub DSN passwords and secret parameter values from pool-open diagnostics so credential material never reaches rendered error text while the cause chain stays intact. | P: redact.go; T: redact_internal_test.go; Other: — | 79/84 | 3: ExtractDSNPassword; RedactDSN; ScrubSecretText | — | internal/backends/postgres/control | one production importer (internal/backends/postgres/control); inspect ownership/inline potential | +| `internal/backends/postgres/internal/rows` | domain/library | Stream typed rows from tagged compiled PostgreSQL commands over pgx connections, preserving bound-value integrity and three-state typed decoding, enforcing output column contracts, and classifying cancellation, visitor stops, contract violations, and driver failures with identifiable wrapped causes. | P: rows.go; T: rows_boundary_pg18_ab03_test.go; rows_postgres_test.go; rows_postquery_postgres_test.go; rows_test.go; Other: — | 483/3,537 | 8: Error; Executor; NewExecutor; Querier; QueryError; RowVisitor; SourceRows; Unwrap | internal/backends/postgres/internal/store; internal/dataframe/compiler/command; internal/dataframe/execution | internal/backends/postgres | one production importer (internal/backends/postgres); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/backends/postgres/internal/store` | domain/library | Provide the bounded pgx/v5-backed PostgreSQL 18 source store: readiness-pinged pool, advisory-lock serialized migration runner, scoped FHIR resource repository that accepts the logical internal/source contract on writes and answers reads with its store-owned physical row under the server-assigned keyset sequence, scoped typed edge-occurrence repository, publish-last inventory repository, scoped bounded field-profile record persistence, the ledgered idempotent COPY batch writer for resource and edge rows, and resource-only and edge-only batch facades over that writer. | P: advisory.go; batch_writer.go; client.go; commit_outcome.go; edge.go; edge_writer.go; errors.go; field_profile.go; inventory.go; migrate.go; migration_ownership.go; resource.go; resource_writer.go; T: batch_writer_test.go; client_test.go; edge_test.go; edge_writer_test.go; errors_test.go; field_profile_ddl_test.go; field_profile_filter_golden_test.go; inventory_test.go; migrate_test.go; ownership_contract_test.go; resource_test.go; resource_writer_test.go; verify_classify_test.go; verify_readonly_test.go; Other: — | 5,898/13,789 | 112: Acquire; AsUniqueConflict; Assignment; AssignmentConflict; Batch; BatchAccountingError; BatchEdgeConflictError; BatchIdentityConflictError; BatchResourceConflictError; BatchResult; BatchWriteError; BatchWriter; … (+100) | internal/catalog; internal/dataframe/errors; internal/source | internal/backends/postgres; internal/backends/postgres/control; internal/backends/postgres/internal/catalog; internal/backends/postgres/internal/dataset; internal/backends/postgres/internal/explorer; internal/backends/postgres/internal/publication; internal/backends/postgres/internal/recipeexec; internal/backends/postgres/internal/recipes; internal/backends/postgres/internal/rows | broad surface (13 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/catalog` | domain/library | Own persistence-neutral observed FHIR field, pivot, relationship, and capability facts. | P: auth_scope.go; backend_contract.go; cache.go; capability_evidence.go; generation.go; helpers.go; limits.go; rebuild.go; relationship_evidence.go; relationships.go; resource_inventory.go; retained_semantic_page.go; semantic.go; semantic_coverage.go; semantic_coverage_contract.go; semantic_inventory.go; semantic_schema.go; types.go; write_profiler.go; T: auth_scope_test.go; cache_test.go; capability_evidence_test.go; generation_test.go; relationship_evidence_census_test.go; relationship_evidence_test.go; relationships_test.go; semantic_coding_display_projection_test.go; semantic_coverage_diagnostics_test.go; semantic_coverage_test.go; semantic_inventory_buildid_cache_test.go; semantic_inventory_category_test.go; semantic_inventory_emitter_anchor_test.go; semantic_inventory_test.go; semantic_schema_identifier_type_test.go; semantic_schema_test.go; semantic_test.go; write_profiler_coverage_test.go; write_profiler_test.go; Other: — | 6,471/3,946 | 146: AuthResourcePathOptions; Cache; CacheOptions; CacheStats; CapabilityEvidence; CapabilityEvidenceOptions; CapabilityEvidenceReader; CensusSemanticCoverage; ConceptCandidate; DecodeSemanticInventoryCursor; DefaultProfileLimits; DiscoverFields; … (+134) | internal/fhir/schema; internal/fhir/semantic | cmd/arango-fhir-proto; cmd/loom-data; internal/api/graphql/graph/query; internal/api/recipe; internal/authscope; internal/backends/arango/internal/catalog; internal/backends/postgres/control; internal/backends/postgres/internal/catalog; internal/backends/postgres/internal/store; internal/explorer/authoringv2; internal/explorer/lifecycle; internal/ingest; internal/server | broad surface (19 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/ci` | domain/library | Exercise repository-wide Go formatting and vet gates, including injected fault detection. | P: —; T: gofmt_gate_test.go; vet_gate_test.go; Other: — | 0/283 | 0: — | — | — | no in-repo production importer; verify runtime/tooling reachability | | `internal/dataframe/columntransform` | domain/library | Define closed, backend-neutral transformations over one selected output column value. | P: types.go; T: types_test.go; Other: — | 79/61 | 7: CategoryMapping; Clone; ExactCategoryRecode; Kind; UnknownPolicy; Validate; ValueTransformation | — | internal/dataframe/compiler/lower; internal/dataframe/recipe; internal/explorer/authoringv2 | no automatic consolidation signal; perform semantic review | -| `internal/dataframe/compiler` | domain/library | Orchestrate semantic planning, physical lowering, optimization, rendering, and compiled-query assembly. | P: api.go; category_scan.go; category_scan_filter_pushdown.go; category_scan_related.go; cell_trace.go; clone_helpers.go; compiled_query.go; construction_preview_root_id_filter.go; graph_execution.go; ir_helpers.go; optimizer.go; output.go; population_mapping.go; preview_covering_index.go; recipe_execution.go; recipe_paging.go; row_lineage.go; semantic_group_projection_preview.go; table_shape_exclusions.go; T: category_scan_arango_test.go; category_scan_parent_presence_arango_test.go; category_scan_related_collection_identity_test.go; category_scan_related_presence_arango_test.go; category_scan_related_test.go; category_scan_test.go; cell_trace_test.go; cohort_drop_boundary_arango_test.go; construction_arango_test.go; construction_coded_group_arango_test.go; construction_coded_group_execution_test.go; construction_coded_pivot_test.go; construction_execution_test.go; construction_group_expand_arango_test.go; construction_group_preview_liveness_arango_test.go; construction_group_related_summary_arango_test.go; construction_group_source_streaming_test.go; construction_pivot_lineage_arango_test.go; construction_preview_root_id_filter_test.go; construction_related_count_arango_test.go; construction_related_expand_arango_test.go; construction_related_group_preview_arango_test.go; correlated_arango_test.go; correlated_legacy_compat_test.go; correlated_physical_test.go; correlated_selection_test.go; correlated_system_version_arango_test.go; explicit_cohort_values_arango_test.go; filter_semantics_test.go; filter_test.go; generic_physical_plan_test.go; grain_test.go; group_connected_field_source_arango_test.go; group_field_source_arango_test.go; nested_expansion_arango_test.go; operator_oracles_arango_test.go; physical_diagnostics_test.go; physical_lowering_test.go; physical_optimize_test.go; physical_plan_test.go; physical_projection_reserved_name_arango_test.go; physical_render_test.go; physical_required_match_test.go; physical_scope_test.go; pivot_presence_arango_audit_test.go; pivot_presence_audit_test.go; plan_test_helpers_test.go; population_frontier_arango_test.go; population_mapping_filter_exists_test.go; population_mapping_identity_guard_test.go; population_mapping_routed_group_arango_test.go; population_mapping_test.go; recipe_paging_related_filter_sequence_test.go; recipe_paging_routed_exists_arango_test.go; recipe_paging_test.go; root_page_related_seed_arango_test.go; row_expansion_test.go; row_item_projection_arango_test.go; row_lineage_cohort_filter_arango_test.go; row_lineage_composed_arango_test.go; row_lineage_group_pivot_arango_test.go; row_lineage_group_rows_arango_test.go; row_lineage_test.go; selection_semantics_test.go; selector_first_limit_test.go; semantic_group_arango_test.go; semantic_group_preview_arango_test.go; semantic_group_projection_factorization_arango_test.go; semantic_validation_test.go; source_group_preview_arango_test.go; streaming_missing_category_pivot_test.go; table_reshape_arango_test.go; table_reshape_execution_window_test.go; table_shape_exclusions_test.go; workspace_output_capture_test.go; Other: — | 7,232/29,200 | 37: CategoryOverflowWitness; CategoryScanProof; CategoryScanRefusal; CategoryScanRefusalCode; CategoryScanRefusalCodeOf; CompileCategoryScanOutputWithPolicy; CompileCategoryScanStageWithPolicy; CompileCellTraceOutputWithPolicy; CompilePopulationMappingOutputWithPolicy; CompileRecipeOutputPageWithPolicy; CompileRecipeOutputWithPolicy; CompileResolvedGraphQueryWithPolicy; … (+25) | internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/aql; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec | cmd/dataframe-profile; cmd/preview-covering-index-artifact; internal/api/graphql/graph/query; internal/dataframe/execution; internal/explorer/lifecycle; internal/server | broad surface (19 production files); inspect cohesion | +| `internal/dataframe/compiler` | domain/library | Orchestrate semantic planning, physical lowering, optimization, rendering, and compiled-query assembly. | P: api.go; category_scan.go; category_scan_filter_pushdown.go; category_scan_related.go; cell_trace.go; clone_helpers.go; compiled_query.go; construction_preview_root_id_filter.go; graph_execution.go; ir_helpers.go; optimizer.go; output.go; population_mapping.go; postgres_target.go; preview_covering_index.go; recipe_execution.go; recipe_paging.go; row_lineage.go; semantic_group_projection_preview.go; table_shape_exclusions.go; target.go; T: backend_leaf_guard_test.go; backend_ownership_guard_contract_test.go; category_scan_arango_test.go; category_scan_parent_presence_arango_test.go; category_scan_related_collection_identity_test.go; category_scan_related_presence_arango_test.go; category_scan_related_test.go; category_scan_test.go; cell_trace_test.go; cohort_drop_boundary_arango_test.go; command_helpers_external_test.go; command_helpers_test.go; compiled_command_test.go; construction_arango_test.go; construction_coded_group_arango_test.go; construction_coded_group_execution_test.go; construction_coded_pivot_test.go; construction_execution_test.go; construction_group_expand_arango_test.go; construction_group_preview_liveness_arango_test.go; construction_group_related_summary_arango_test.go; construction_group_source_streaming_test.go; construction_pivot_lineage_arango_test.go; construction_preview_root_id_filter_test.go; construction_related_count_arango_test.go; construction_related_expand_arango_test.go; construction_related_group_preview_arango_test.go; correlated_arango_test.go; correlated_legacy_compat_test.go; correlated_physical_test.go; correlated_selection_test.go; correlated_system_version_arango_test.go; explicit_cohort_values_arango_test.go; filter_semantics_test.go; filter_test.go; generic_physical_plan_test.go; grain_test.go; group_connected_field_source_arango_test.go; group_field_source_arango_test.go; nested_expansion_arango_test.go; operator_oracles_arango_test.go; physical_diagnostics_test.go; physical_lowering_test.go; physical_optimize_test.go; physical_plan_test.go; physical_projection_reserved_name_arango_test.go; physical_render_test.go; physical_required_match_test.go; physical_scope_test.go; pivot_presence_arango_audit_test.go; pivot_presence_audit_test.go; plan_test_helpers_test.go; population_frontier_arango_test.go; population_mapping_filter_exists_test.go; population_mapping_identity_guard_test.go; population_mapping_routed_group_arango_test.go; population_mapping_test.go; postgres_coded_group_compile_test.go; postgres_parity_fixture_test.go; postgres_target_fingerprint_golden_test.go; public_columns_freeze_test.go; recipe_execution_pg18_test.go; recipe_execution_test.go; recipe_paging_related_filter_sequence_test.go; recipe_paging_routed_exists_arango_test.go; recipe_paging_test.go; root_page_related_seed_arango_test.go; row_expansion_test.go; row_item_projection_arango_test.go; row_lineage_cohort_filter_arango_test.go; row_lineage_composed_arango_test.go; row_lineage_group_pivot_arango_test.go; row_lineage_group_rows_arango_test.go; row_lineage_test.go; selected_source_contract_test.go; selection_semantics_test.go; selector_first_limit_test.go; semantic_group_arango_test.go; semantic_group_preview_arango_test.go; semantic_group_projection_factorization_arango_test.go; semantic_validation_test.go; source_group_preview_arango_test.go; streaming_missing_category_pivot_test.go; table_reshape_arango_test.go; table_reshape_execution_window_test.go; table_shape_exclusions_test.go; workspace_output_capture_test.go; Other: — | 8,120/33,173 | 44: ArangoTarget; Backend; CategoryOverflowWitness; CategoryScanProof; CategoryScanRefusal; CategoryScanRefusalCode; CategoryScanRefusalCodeOf; CompileCategoryScanOutput; CompileCategoryScanStage; CompileCellTraceOutput; CompileClickHousePrefix; CompilePopulationMappingOutput; … (+32) | internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/postgres; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec | cmd/dataframe-profile; cmd/preview-covering-index-artifact; internal/api/graphql/graph/query; internal/dataframe/execution; internal/explorer/lifecycle; internal/server | broad surface (21 production files); inspect cohesion | | `internal/dataframe/compiler/capability` | domain/library | Probe which dataframe language capabilities the compiler can prove and support. | P: probe.go; schema_fields.go; T: probe_test.go; schema_fields_test.go; Other: — | 803/313 | 22: CandidateCapability; CandidateRequest; Chart; Filter; Options; OwnerRecordsCapability; OwnerRecordsRequest; ProbeCandidate; ProbeOwnerRecords; ProbeProjection; ProbeRoot; ProbeTraversal; … (+10) | internal/authscope; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/aql; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec; internal/fhir/schema | internal/explorer/lifecycle; internal/server | no automatic consolidation signal; perform semantic review | -| `internal/dataframe/compiler/ir` | domain/library | Define typed physical operations, values, scope proofs, policies, and validation. | P: clone.go; dataset_generation.go; diagnostics.go; doc.go; navigation.go; physical_combine.go; physical_combine_scalar.go; physical_operations.go; physical_stage_sequence.go; physical_stage_validation.go; physical_table_shape.go; physical_types.go; physical_validation.go; physical_validation_expr.go; physical_validation_refs.go; physical_validation_shapes.go; physical_values.go; policy.go; prefix.go; scope.go; selector_helpers.go; whole_relation_scope_evidence.go; workspace_combine_scope_operand.go; T: group_related_field_source_validation_test.go; physical_combine_scalar_test.go; physical_combine_test.go; physical_group_pivot_lineage_validation_test.go; physical_root_page_candidate_seed_test.go; physical_scope_cycle_test.go; physical_stage_validation_test.go; population_mapping_test.go; population_root_test.go; related_coding_group_lineage_validation_test.go; row_lineage_trace_validation_test.go; subplan_test.go; unit_validation_test.go; unnest_test.go; workspace_output_validation_test.go; Other: — | 11,918/1,691 | 196: AddDecision; CanonicalExecutionPhysicalPlan; ClickHouseArtifactIdentity; ClickHouseCombineScalarBaseType; ClonePhysicalExpression; ClonePhysicalOperation; ClonePhysicalOperations; ClonePhysicalPlan; ClonePhysicalPredicateExpression; ClonePhysicalSubplan; CompilerPlanDiagnostics; DatasetGenerationBindValue; … (+184) | internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema; internal/projectid | cmd/dataframe-profile; cmd/preview-covering-index-artifact; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/clickhouse; internal/dataframe/execution; internal/explorer/lifecycle; internal/server | broad surface (23 production files); inspect cohesion | -| `internal/dataframe/compiler/lower` | domain/library | Lower logical FHIR dataframe plans into storage-aware physical IR. | P: auth_scope.go; child_sets.go; column_naming.go; correlated_binding.go; doc.go; filter_literal.go; graph_plan.go; group_field_source_context.go; group_field_source_related.go; group_related_occurrence_lower.go; group_rows.go; helpers.go; plan_orchestration.go; population.go; recipe_column_transformations.go; recipe_compile.go; recipe_construction.go; recipe_construction_coded_group.go; recipe_construction_coded_pivot.go; recipe_construction_combine.go; recipe_construction_combine_metadata.go; recipe_construction_composite.go; recipe_construction_group_sources.go; recipe_construction_related_eligibility.go; recipe_construction_related_expand.go; recipe_construction_related_field.go; recipe_construction_reshape.go; recipe_derived_columns.go; recipe_dynamic_columns.go; recipe_expression.go; recipe_expression_lowering.go; recipe_schema.go; recipe_table_reshape.go; required_match.go; root_projection.go; row_expansion.go; storage_route.go; traversal.go; typed_predicate.go; whole_relation_scope.go; workspace_output_dependencies.go; T: bound_scalar_coding_group_test.go; child_set_selector_demand_test.go; construction_group_streaming_test.go; correlated_binding_test.go; empty_bound_group_source_test.go; graph_plan_test.go; group_field_source_context_test.go; group_related_occurrence_lower_test.go; group_rows_test.go; group_specimen_inbound_coding_test.go; owner_records_test.go; physical_nested_scope_test.go; population_test.go; recipe_construction_coded_group_test.go; recipe_construction_coded_pivot_display_test.go; recipe_construction_coded_pivot_test.go; recipe_construction_combine_test.go; recipe_construction_composite_test.go; recipe_construction_related_count_test.go; recipe_construction_reshape_test.go; recipe_construction_row_values_test.go; recipe_construction_test.go; recipe_derived_columns_test.go; recipe_expression_test.go; recipe_output_test.go; recipe_schema_cardinality_test.go; recipe_semantic_path_test.go; recipe_table_reshape_test.go; recipe_unpivot_capability_test.go; root_id_filter_test.go; row_expansion_test.go; traversal_test.go; whole_relation_scope_evidence_test.go; workspace_combine_scope_evidence_test.go; workspace_output_compile_test.go; Other: — | 12,149/10,496 | 39: BindGroupRelatedOccurrenceSource; BuildGenericPhysicalPlanWithPolicy; BuildGraphPhysicalPlan; BuildPhysicalTraversal; CloneCompiledOutputSchema; CompileResolvedGraphPlan; CompileResolvedRecipePlan; CompiledActiveRelatedRecordStage; CompiledGroupFieldOwner; CompiledGroupFieldSourceAncestor; CompiledGroupFieldSourceContext; CompiledGroupFieldSourceExpansion; … (+27) | internal/authscope; internal/dataframe/columntransform; internal/dataframe/compiler/ir; internal/dataframe/expression; internal/dataframe/lineage; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema; internal/projectid | internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/execution; internal/dataframe/execution/chartifact; internal/explorer/lifecycle; internal/server | broad surface (41 production files); inspect cohesion | +| `internal/dataframe/compiler/command` | domain/library | Own the closed dispatch-tagged executable command types, rendered PostgreSQL plan validation, and constructor guarantees shared by compilers and row adapters. | P: command.go; rendered_postgres.go; T: compiled_command_test.go; imports_guard_test.go; rendered_postgres_test.go; Other: — | 433/526 | 15: AQLCommand; AQLParameters; Backend; CompiledBackend; CompiledCommand; Dispatch; NewAQLCommand; NewPostgreSQLCommand; NewRenderedPostgresPhysicalPlan; PostgreSQLCommand; PostgreSQLParameter; PostgreSQLParameterType; … (+3) | — | internal/backends/postgres; internal/backends/postgres/internal/rows; internal/dataframe/compiler; internal/dataframe/compiler/render/postgres; internal/dataframe/execution; internal/server | no automatic consolidation signal; perform semantic review | +| `internal/dataframe/compiler/ir` | domain/library | Define typed physical operations, values, scope proofs, policies, and validation. | P: clone.go; dataset_generation.go; diagnostics.go; doc.go; navigation.go; physical_combine.go; physical_combine_scalar.go; physical_operations.go; physical_stage_sequence.go; physical_stage_validation.go; physical_table_shape.go; physical_types.go; physical_validation.go; physical_validation_expr.go; physical_validation_refs.go; physical_validation_shapes.go; physical_values.go; policy.go; postgres_storage.go; prefix.go; scope.go; selector_helpers.go; whole_relation_scope_evidence.go; workspace_combine_scope_operand.go; T: group_related_field_source_validation_test.go; physical_combine_scalar_test.go; physical_combine_test.go; physical_group_pivot_lineage_validation_test.go; physical_root_page_candidate_seed_test.go; physical_scope_cycle_test.go; physical_stage_validation_test.go; population_mapping_test.go; population_root_test.go; related_coding_group_lineage_validation_test.go; row_lineage_trace_validation_test.go; subplan_test.go; unit_validation_test.go; unnest_test.go; workspace_output_validation_test.go; Other: — | 12,283/1,804 | 215: AddDecision; CanonicalExecutionPhysicalPlan; ClickHouseArtifactIdentity; ClickHouseCombineScalarBaseType; ClonePhysicalExpression; ClonePhysicalOperation; ClonePhysicalOperations; ClonePhysicalPlan; ClonePhysicalPredicateExpression; ClonePhysicalSubplan; Column; CompilerPlanDiagnostics; … (+203) | internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema; internal/projectid | cmd/dataframe-profile; cmd/preview-covering-index-artifact; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/clickhouse; internal/dataframe/compiler/render/postgres; internal/dataframe/execution; internal/explorer/lifecycle; internal/server | broad surface (24 production files); inspect cohesion | +| `internal/dataframe/compiler/lower` | domain/library | Lower logical FHIR dataframe plans into storage-aware physical IR. | P: auth_scope.go; child_sets.go; column_naming.go; correlated_binding.go; doc.go; filter_literal.go; graph_plan.go; group_field_source_context.go; group_field_source_related.go; group_related_occurrence_lower.go; group_rows.go; helpers.go; plan_orchestration.go; population.go; recipe_column_transformations.go; recipe_compile.go; recipe_construction.go; recipe_construction_coded_group.go; recipe_construction_coded_pivot.go; recipe_construction_combine.go; recipe_construction_combine_metadata.go; recipe_construction_composite.go; recipe_construction_group_sources.go; recipe_construction_related_eligibility.go; recipe_construction_related_expand.go; recipe_construction_related_field.go; recipe_construction_reshape.go; recipe_derived_columns.go; recipe_dynamic_columns.go; recipe_expression.go; recipe_expression_lowering.go; recipe_schema.go; recipe_table_reshape.go; required_match.go; root_projection.go; row_expansion.go; storage_route.go; traversal.go; typed_predicate.go; whole_relation_scope.go; workspace_output_dependencies.go; T: bound_scalar_coding_group_test.go; child_set_selector_demand_test.go; construction_group_streaming_test.go; correlated_binding_test.go; empty_bound_group_source_test.go; graph_plan_test.go; group_field_source_context_test.go; group_related_occurrence_lower_test.go; group_rows_test.go; group_specimen_inbound_coding_test.go; owner_records_test.go; physical_nested_scope_test.go; population_test.go; recipe_construction_coded_group_test.go; recipe_construction_coded_pivot_display_test.go; recipe_construction_coded_pivot_test.go; recipe_construction_combine_test.go; recipe_construction_composite_test.go; recipe_construction_related_count_test.go; recipe_construction_reshape_test.go; recipe_construction_row_values_test.go; recipe_construction_test.go; recipe_derived_columns_test.go; recipe_expression_test.go; recipe_output_test.go; recipe_schema_cardinality_test.go; recipe_semantic_path_test.go; recipe_table_reshape_test.go; recipe_unpivot_capability_test.go; root_id_filter_test.go; row_expansion_test.go; traversal_test.go; whole_relation_scope_evidence_test.go; workspace_combine_scope_evidence_test.go; workspace_output_compile_test.go; Other: — | 12,219/10,802 | 40: BindGroupRelatedOccurrenceSource; BuildGenericPhysicalPlanWithPolicy; BuildGraphPhysicalPlan; BuildPhysicalTraversal; CloneCompiledOutputSchema; CompileResolvedGraphPlan; CompileResolvedRecipePlan; CompiledActiveRelatedRecordStage; CompiledGroupFieldOwner; CompiledGroupFieldSourceAncestor; CompiledGroupFieldSourceContext; CompiledGroupFieldSourceExpansion; … (+28) | internal/authscope; internal/dataframe/columntransform; internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/expression; internal/dataframe/lineage; internal/dataframe/recipe; internal/dataframe/semantic; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema; internal/projectid | internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/execution; internal/dataframe/execution/chartifact; internal/explorer/compilation; internal/explorer/lifecycle; internal/server | broad surface (41 production files); inspect cohesion | | `internal/dataframe/compiler/optimize` | domain/library | Apply explainable semantics-preserving rewrites to validated physical IR. | P: doc.go; optimize.go; T: optimize_test.go; Other: — | 438/124 | 1: OptimizePhysicalPlanWithPolicy | internal/dataframe/compiler/ir | internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/execution | no automatic consolidation signal; perform semantic review | -| `internal/dataframe/compiler/render/aql` | domain/library | Render validated physical IR as deterministic parameterized AQL. | P: calls.go; cell_trace.go; collections.go; combine_prefix.go; construction_coded_group.go; construction_expand.go; construction_group.go; construction_related_expand.go; correlation_value_scope.go; doc.go; graph_render.go; group_row_value_preview.go; group_rows.go; navigation_layout.go; navigation_render.go; owner_records.go; population_mapping.go; predicates.go; related_count.go; related_group_preview.go; render.go; row_lineage.go; row_lineage_coded_group.go; row_lineage_composed.go; row_lineage_group_pivot.go; row_lineage_group_rows.go; row_lineage_pivot.go; row_lineage_related_coding.go; row_values.go; selector.go; selectors.go; source_group_preview.go; stage_sequence.go; table_reshape.go; table_reshape_dynamic.go; table_reshape_streaming.go; table_shape_exclusion.go; validation.go; values.go; T: call_test.go; construction_group_related_occurrence_test.go; construction_group_test.go; construction_related_expand_test.go; correlated_selection_test.go; graph_test.go; group_row_value_preview_test.go; navigation_layout_test.go; related_count_category_test.go; row_lineage_composed_test.go; selectors_test.go; source_group_preview_test.go; stage_sequence_test.go; table_reshape_dynamic_test.go; table_reshape_streaming_scalar_test.go; two_scan_preview_test.go; unnest_test.go; Other: — | 14,514/3,444 | 22: RenderClickHousePrefix; RenderPhysicalCohortGroupRowsLineage; RenderPhysicalGroupRowsLineage; RenderPhysicalPlan; RenderPhysicalPlanWithCategoryScanPresenceMarker; RenderPhysicalPlanWithDynamicCategoryPivotPreview; RenderPhysicalPlanWithRelatedExpandPreviewLimit; RenderPhysicalPlanWithRootIndexDisabled; RenderPhysicalPlanWithRootIndexHint; RenderPhysicalPlanWithRootPageKeys; RenderPhysicalPlanWithSemanticGroupProjectionAssessment; RenderPhysicalPlanWithSemanticGroupProjectionFactorization; … (+10) | internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema | internal/dataframe/compiler; internal/dataframe/compiler/capability | broad surface (39 production files); inspect cohesion | -| `internal/dataframe/compiler/render/clickhouse` | storage adapter | Render typed Combine plans and resolved published or private table inputs as deterministic ClickHouse SQL. | P: combine.go; workspace_combine.go; T: combine_test.go; membership_nullable_integration_test.go; workspace_combine_test.go; Other: — | 930/1,092 | 6: RenderCombine; RenderCombineWithLimit; RenderCompositeClickHouseCombine; RenderWorkspaceClickHouseCombine; RenderedCombine; WorkspaceArtifact | internal/dataframe/compiler/ir | internal/dataframe/execution | one production importer (internal/dataframe/execution); inspect ownership/inline potential; storage adapter; preserve interface and persisted-data contracts | -| `internal/dataframe/errors` | domain/library | Define and normalize stable structured dataframe errors for transport presentation. | P: errors.go; T: errors_test.go; Other: — | 485/172 | 21: AsUserError; Code; Details; Error; ErrorCode; ErrorOption; FieldPath; IsFeatureResolutionCode; IsRetryableCode; NewError; Normalize; PublicMessage; … (+9) | — | internal/api/authpolicy; internal/api/bulk/load; internal/api/graphql/graph; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/api/http; internal/api/recipe; internal/dataframe/compiler/render/aql; internal/dataframe/execution; internal/dataframe/publication; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/explorer/lifecycle; internal/server | small shared semantic contract; compare direct callers before inlining; runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/dataframe/execution` | domain/library | Prepare, authorize, execute, explain, and post-process compiled dataframe requests. | P: category_scan.go; cell_trace.go; clickhouse_artifact.go; control.go; control_explain_types.go; control_types.go; engine.go; execution_service.go; explain.go; group_preview_cache.go; group_preview_scan.go; helpers.go; named_uuid.go; population_mapping.go; postquery.go; related_preview_paging.go; row_lineage.go; service.go; table_shape_exclusions.go; types.go; T: auth_scope_test.go; category_scan_test.go; cell_trace_test.go; clickhouse_artifact_test.go; cohort_preview_paging_arango_test.go; construction_cell_trace_arango_test.go; engine_clickhouse_test.go; engine_test.go; engine_workspace_artifact_test.go; execution_test.go; explain_test.go; group_preview_cache_test.go; group_preview_scan_test.go; helpers_test.go; identity_test.go; named_uuid_test.go; population_group_preview_test.go; population_mapping_test.go; postquery_test.go; row_lineage_test.go; table_shape_exclusions_test.go; Other: — | 5,276/6,315 | 92: AssessmentFromArango; CategoryScanRequest; CategoryScanResult; CategoryValue; CellTrace; CellTraceCompiled; CellTraceContribution; CellTraceRequest; CellTraceResult; CellTraceStatus; ClickHouseQueryRows; CompileResolvedBundle; … (+80) | internal/authscope; internal/dataframe/compiler; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/clickhouse; internal/dataframe/errors; internal/dataframe/execution/chartifact; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataframe/spec; internal/projectid; internal/store/arango | internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/explorer/lifecycle; internal/server | broad surface (20 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/dataframe/execution/chartifact` | domain/library | Manage private ClickHouse staging tables and leases used to cross execution boundaries without publishing outputs. | P: artifact.go; T: artifact_test.go; Other: — | 1,038/490 | 24: Abort; Artifact; Begin; Catalog; CheckLease; ClickHouse; Column; ColumnsFromCompiledOutput; Config; Finalize; Identity; LeaseContext; … (+12) | internal/authscope; internal/dataframe/compiler/lower; internal/dataframe/expression; internal/store/clickhouse | internal/dataframe/execution; internal/dataframe/execution/chartifact/arango; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/dataframe/execution/chartifact/arango` | storage adapter | Persist private ClickHouse artifact manifests and leases in a dedicated Arango collection. | P: catalog.go; T: catalog_arango_test.go; catalog_test.go; Other: — | 379/723 | 10: BootstrapSpec; Catalog; ClaimCleanup; Create; Delete; ListExpired; NewCatalog; ReleaseCleanup; Renew; Update | internal/dataframe/execution/chartifact; internal/store/arango | internal/server | one production importer (internal/server); inspect ownership/inline potential; storage adapter; preserve interface and persisted-data contracts | +| `internal/dataframe/compiler/render/aql` | domain/library | Render validated physical IR as deterministic parameterized AQL. | P: calls.go; cell_trace.go; collections.go; combine_prefix.go; construction_coded_group.go; construction_expand.go; construction_group.go; construction_related_expand.go; correlation_value_scope.go; doc.go; graph_render.go; group_row_value_preview.go; group_rows.go; navigation_layout.go; navigation_render.go; owner_records.go; population_mapping.go; predicates.go; related_count.go; related_group_preview.go; render.go; row_lineage.go; row_lineage_coded_group.go; row_lineage_composed.go; row_lineage_group_pivot.go; row_lineage_group_rows.go; row_lineage_pivot.go; row_lineage_related_coding.go; row_values.go; selector.go; selectors.go; source_group_preview.go; stage_sequence.go; table_reshape.go; table_reshape_dynamic.go; table_reshape_streaming.go; table_shape_exclusion.go; validation.go; values.go; T: call_test.go; command_helpers_ext_test.go; construction_group_related_occurrence_test.go; construction_group_test.go; construction_related_expand_test.go; correlated_selection_test.go; graph_test.go; group_row_value_preview_test.go; navigation_layout_test.go; recipe_unpivot_identity_test.go; related_count_category_test.go; row_lineage_composed_test.go; selectors_test.go; source_group_preview_test.go; stage_sequence_test.go; table_reshape_dynamic_test.go; table_reshape_streaming_scalar_test.go; two_scan_preview_test.go; unnest_test.go; Other: — | 14,519/3,608 | 22: RenderClickHousePrefix; RenderPhysicalCohortGroupRowsLineage; RenderPhysicalGroupRowsLineage; RenderPhysicalPlan; RenderPhysicalPlanWithCategoryScanPresenceMarker; RenderPhysicalPlanWithDynamicCategoryPivotPreview; RenderPhysicalPlanWithRelatedExpandPreviewLimit; RenderPhysicalPlanWithRootIndexDisabled; RenderPhysicalPlanWithRootIndexHint; RenderPhysicalPlanWithRootPageKeys; RenderPhysicalPlanWithSemanticGroupProjectionAssessment; RenderPhysicalPlanWithSemanticGroupProjectionFactorization; … (+10) | internal/dataframe/compiler/ir; internal/dataframe/errors; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema | internal/dataframe/compiler; internal/dataframe/compiler/capability | broad surface (39 production files); inspect cohesion | +| `internal/dataframe/compiler/render/clickhouse` | storage adapter | Render typed Combine plans and resolved published or private table inputs as deterministic ClickHouse SQL. | P: combine.go; workspace_combine.go; T: combine_test.go; membership_nullable_integration_test.go; workspace_combine_test.go; Other: — | 930/1,093 | 6: RenderCombine; RenderCombineWithLimit; RenderCompositeClickHouseCombine; RenderWorkspaceClickHouseCombine; RenderedCombine; WorkspaceArtifact | internal/dataframe/compiler/ir | internal/dataframe/execution | one production importer (internal/dataframe/execution); inspect ownership/inline potential; storage adapter; preserve interface and persisted-data contracts | +| `internal/dataframe/compiler/render/postgres` | storage adapter | Render the supported physical IR read spine and typed selector extractions with two-valued JSONB path-presence evidence onto allowlisted PostgreSQL storage bindings as deterministic parameterized SQL with ordered typed arguments, failing closed on every unsupported plan shape. | P: calls.go; category_scan.go; cell_trace.go; coded_group.go; graph.go; group.go; null_semantics.go; objects.go; pivot.go; population.go; population_mapping.go; predicates.go; render.go; root_scan.go; row_lineage.go; scope.go; scope_ops.go; selectors.go; sets.go; stage_related_occurrence.go; stage_semantic.go; stage_sequence.go; table_shape_exclusions.go; traversal.go; unit_normalization.go; unnest.go; unpivot.go; window.go; T: calls_live_test.go; calls_test.go; category_scan_test.go; cell_trace_test.go; coded_group_test.go; coded_pivot_test.go; graph_test.go; group_test.go; main_test.go; null_semantics_test.go; obj11_production_test.go; objects_test.go; pivot14_noncoded_test.go; pivot_test.go; population_live_test.go; population_mapping_test.go; population_test.go; predicates_pg18_test.go; predicates_test.go; render_pg18_test.go; render_test.go; root_scan_test.go; row_lineage_test.go; scope_ops_witness_test.go; scope_test.go; selectors_test.go; sets_live_test.go; sets_test.go; stage_related_occurrence_p4_test.go; stage_related_occurrence_production_test.go; stage_related_occurrence_test.go; stage_semantic_production_test.go; stage_semantic_test.go; stage_sequence_production_test.go; stage_sequence_test.go; table_shape_exclusions_live_test.go; table_shape_exclusions_test.go; traversal_test.go; unit_normalization_production_test.go; unit_normalization_test.go; unit_normalization_tie_test.go; unnest_test.go; unpivot_test.go; window_test.go; Other: — | 23,444/42,018 | 18: NewPostgresPopulationRootBinding; PostgresPopulationRootBinding; RenderCategoryScan; RenderGraphPhysicalPlan; RenderGroupedCodedPivot; RenderGroupedPivot; RenderPhysicalPlan; RenderPopulationMappingPlan; RenderPopulationRootScan; RenderRootScan; RenderSelectorExtraction; RenderTableShapeExclusions; … (+6) | internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema | internal/dataframe/compiler; internal/dataframe/execution | broad surface (28 production files); inspect cohesion; storage adapter; preserve interface and persisted-data contracts | +| `internal/dataframe/errors` | domain/library | Define and normalize stable structured dataframe errors for transport presentation. | P: errors.go; T: errors_test.go; Other: — | 488/172 | 21: AsUserError; Code; Details; Error; ErrorCode; ErrorOption; FieldPath; IsFeatureResolutionCode; IsRetryableCode; NewError; Normalize; PublicMessage; … (+9) | — | internal/api/authpolicy; internal/api/bulk/load; internal/api/graphql/graph; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/api/http; internal/api/recipe; internal/backends/arango/internal/recipeexec; internal/backends/arango/internal/store; internal/backends/postgres/internal/publication; internal/backends/postgres/internal/recipes; internal/backends/postgres/internal/store; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/execution; internal/dataframe/publication; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/dataframe/semantic; internal/dataframe/spec; internal/explorer/lifecycle; internal/server | small shared semantic contract; compare direct callers before inlining; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/dataframe/execution` | domain/library | Prepare, authorize, execute, explain, and post-process compiled dataframe requests. | P: category_scan.go; cell_trace.go; clickhouse_artifact.go; control.go; control_explain_types.go; control_types.go; engine.go; execution_service.go; explain.go; explain_postgres.go; group_preview_cache.go; group_preview_scan.go; helpers.go; named_uuid.go; population_mapping.go; postquery.go; preview_capabilities.go; query_budget.go; related_preview_paging.go; row_lineage.go; service.go; table_shape_exclusions.go; types.go; T: auth_scope_test.go; boundary_helpers_test.go; category_scan_postgres_test.go; category_scan_test.go; cell_trace_postgres_test.go; cell_trace_row_not_found_class_test.go; cell_trace_test.go; clickhouse_artifact_test.go; cohort_preview_paging_arango_test.go; command_boundary_contract_test.go; construction_cell_trace_arango_test.go; control_validate_digest_test.go; engine_blank_project_review_test.go; engine_clickhouse_test.go; engine_postgres_capture_pg18_test.go; engine_postgres_capture_test.go; engine_postgres_category_pg18_test.go; engine_postgres_cell_trace_pg18_test.go; engine_postgres_exclusions_pg18_test.go; engine_postgres_postquery_pg18_test.go; engine_postgres_preview_pg18_test.go; engine_postgres_test.go; engine_run_output_class_test.go; engine_test.go; engine_workspace_artifact_test.go; execution_test.go; explain_test.go; group_preview_cache_test.go; group_preview_scan_test.go; helpers_test.go; identity_test.go; named_uuid_test.go; population_group_preview_test.go; population_mapping_postgres_pg18_test.go; population_mapping_test.go; postgres_explain_parser_internal_test.go; postgres_target_fixture_test.go; postquery_test.go; preview_capability_boundary_test.go; recipe_paging_postgres_adapter_test.go; recipe_paging_postgres_test.go; recipe_preview_paging_postgres_internal_test.go; row_lineage_postgres_pg18_test.go; row_lineage_test.go; selected_source_contract_test.go; table_shape_exclusions_test.go; vendor_import_guard_test.go; verify_arch01_pins_test.go; Other: — | 7,088/27,056 | 109: BuildPostgreSQLExplainCommand; CategoryScanRequest; CategoryScanResult; CategoryValue; CellTrace; CellTraceCompiled; CellTraceContribution; CellTraceRequest; CellTraceResult; CellTraceStatus; ClickHouseQueryRows; CombinePortsConfigured; … (+97) | internal/authscope; internal/dataframe/compiler; internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/optimize; internal/dataframe/compiler/render/clickhouse; internal/dataframe/compiler/render/postgres; internal/dataframe/errors; internal/dataframe/execution/chartifact; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/semantic; internal/dataframe/spec; internal/projectid | internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/backends/postgres; internal/backends/postgres/internal/rows; internal/explorer/lifecycle; internal/server | broad surface (23 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/dataframe/execution/chartifact` | domain/library | Manage private ClickHouse staging tables and leases used to cross execution boundaries without publishing outputs. | P: artifact.go; T: artifact_test.go; Other: — | 1,038/490 | 24: Abort; Artifact; Begin; Catalog; CheckLease; ClickHouse; Column; ColumnsFromCompiledOutput; Config; Finalize; Identity; LeaseContext; … (+12) | internal/authscope; internal/dataframe/compiler/lower; internal/dataframe/expression; internal/store/clickhouse | internal/backends/arango/internal/chartifact; internal/backends/postgres/internal/chartifact; internal/dataframe/execution; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | | `internal/dataframe/expression` | domain/library | Define and validate the typed backend-neutral expression AST used by recipes. | P: expression_checking.go; expression_types.go; recipe_adapter.go; T: expression_test.go; recipe_adapter_test.go; Other: — | 856/235 | 26: AllowsMany; Call; Cardinality; Check; CheckedExpression; Constant; Document; DocumentRef; Expression; FromRecipe; FromRecipeInContexts; Function; … (+14) | internal/dataframe/recipe | internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/execution/chartifact; internal/dataframe/semantic; internal/explorer; internal/explorer/lifecycle; internal/server | no automatic consolidation signal; perform semantic review | | `internal/dataframe/lineage` | domain/library | Define stable identities and provenance for columns expanded from one authored source slot. | P: source_child.go; T: source_child_test.go; Other: — | 163/77 | 7: CloneSourceChild; Coordinate; IsSourceChildID; ParseSourceChildID; SourceChild; SourceChildKind; StableSourceChildID | — | internal/dataframe/compiler/lower; internal/dataframe/recipe; internal/explorer/authoringv2; internal/explorer/compilation | no automatic consolidation signal; perform semantic review | -| `internal/dataframe/publication` | domain/library | Own backend-neutral dataframe publication contracts, bundles, and publication orchestration. | P: bundle.go; flat_columns.go; quality.go; runner.go; types.go; T: bundle_test.go; flat_columns_test.go; Other: — | 1,335/91 | 55: BundleCatalog; BundleExecution; BundleExecutionPageFunc; BundleIdentity; BundleOutputRecord; BundlePointer; BundleState; Canonical; CanonicalizeLegacy; CloneQualityReports; ColumnProvenance; ColumnQuality; … (+43) | internal/dataframe/errors; internal/dataset | internal/acceptance; internal/api/bulk/load; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/resolver; internal/dataframe/publication/arango; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/explorer; internal/explorer/arango; internal/explorer/lifecycle; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/dataframe/publication/arango` | storage adapter | Persist dataframe publication registry and bundle metadata in ArangoDB. | P: bundle_registry.go; registry.go; T: bundle_registry_test.go; execution_read_pin_integration_test.go; Other: — | 520/617 | 22: AcquireBundleLease; AcquireExecutionReadPin; BootstrapSpec; ClaimExecutionCleanup; Client; CompareAndSwapPointer; FindExecutionByKey; FindExecutionBySelector; GetExecution; GetPointer; ListExecutions; New; … (+10) | internal/dataframe/publication; internal/store/arango | internal/acceptance; internal/explorer/arango; internal/server | storage adapter; preserve interface and persisted-data contracts | -| `internal/dataframe/publication/clickhouse` | storage adapter | Materialize and inspect published dataframe bundles in ClickHouse. | P: bundle_store.go; T: bundle_store_test.go; Other: — | 1,102/28 | 19: Abort; Begin; BundleClickHouseStore; ClickHouseBundleStore; Commit; CreateOutput; ExistingPublishedOutputs; ExistingQualityReports; FinalizeSchema; Idempotent; InsertRows; NewBundleStore; … (+7) | internal/dataframe/errors; internal/dataframe/publication; internal/store/clickhouse | internal/server | one production importer (internal/server); inspect ownership/inline potential; storage adapter; preserve interface and persisted-data contracts; runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/dataframe/published` | domain/library | Read, page, aggregate, and export authorized published ClickHouse dataframes. | P: aggregate_execute.go; aggregate_plan.go; aggregate_sql.go; aggregate_types.go; artifact.go; current.go; errors.go; export.go; page.go; query_helpers.go; read.go; read_pins.go; schema.go; selection.go; types.go; T: aggregate_clickhouse_integration_test.go; aggregate_execute_test.go; aggregate_plan_test.go; artifact_roundtrip_test.go; artifact_test.go; current_test.go; errors_test.go; export_test.go; page_test.go; read_pins_test.go; read_test.go; selection_test.go; Other: — | 3,642/1,763 | 56: AggregateBatchRequest; AggregateBatchResult; AggregateJob; AggregateJobResult; AggregateResponseMode; AggregateResult; AggregationResult; AggregationSpec; AggregationsRequest; AggregationsResult; ArtifactColumn; ArtifactDescriptor; … (+44) | internal/dataframe/errors; internal/dataframe/publication; internal/dataframe/unit; internal/dataset; internal/projectid | internal/api/graphql/graph/dataframe; internal/api/graphql/graph/resolver; internal/explorer; internal/explorer/artifactfs; internal/explorer/lifecycle; internal/server | broad surface (15 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/dataframe/recipe` | domain/library | Define, canonicalize, expand, encode, and validate persistence-neutral recipe documents. | P: canonical.go; construction.go; construction_combine.go; construction_reshape.go; derived.go; derived_validation.go; document_types.go; document_validation.go; dynamic_validation.go; fragments.go; json_encoding.go; revisions.go; strict_json.go; table_reshape.go; validation_rich.go; workspace_output_dependencies.go; T: construction_combine_test.go; construction_group_policy_test.go; construction_test.go; correlated_selection_test.go; derived_validation_test.go; documentation_test.go; fragments_test.go; table_reshape_test.go; types_test.go; workspace_output_dependencies_test.go; Other: — | 5,611/1,779 | 147: Aggregate; AggregateOperation; ArrayQuantifier; Bundle; CanonicalBundle; CanonicalJSON; CanonicalJSONWithWorkspaceOutputs; CatalogProjection; Clone; CodeValue; ColumnNaming; ColumnTransformation; … (+135) | generated/fhir; internal/authscope; internal/dataframe/columntransform; internal/dataframe/lineage; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema | cmd/dataframe-profile; cmd/explorer-config-v2-convert; cmd/preview-covering-index-artifact; conformance/compiler; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/execution; internal/dataframe/expression; internal/dataframe/recipe/exec; internal/dataframe/recipe/exec/arango; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/explorer; internal/explorer/capability; internal/explorer/compilation; internal/explorer/lifecycle; internal/server | broad surface (16 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/dataframe/recipe/exec` | domain/library | Define durable recipe-version storage contracts and immutable registration behavior. | P: persistent.go; T: persistent_test.go; Other: — | 84/102 | 5: Entry; PersistentRegistry; Reader; RegisterVersion; Store | internal/dataframe/recipe | internal/api/graphql/graph/resolver; internal/dataframe/execution; internal/dataframe/recipe/exec/arango; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/dataframe/recipe/exec/arango` | storage adapter | Persist recipe versions and execution-use records in ArangoDB. | P: registry.go; revisions.go; T: registry_test.go; revisions_test.go; Other: — | 293/350 | 15: BootstrapSpec; Client; Get; List; LoadRecipe; LoadRecipeVersion; New; NewRevisionRegistry; RecipeVersionUsed; Register; Registry; ReplaceRecipeVersion; … (+3) | internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/store/arango | internal/server | one production importer (internal/server); inspect ownership/inline potential; storage adapter; preserve interface and persisted-data contracts | +| `internal/dataframe/publication` | domain/library | Own backend-neutral dataframe publication contracts, bundles, and publication orchestration. | P: bundle.go; flat_columns.go; quality.go; runner.go; types.go; T: bundle_test.go; flat_columns_test.go; pointer_name_encoding_test.go; row_smuggling_prevention_test.go; Other: — | 1,353/385 | 55: BundleCatalog; BundleExecution; BundleExecutionPageFunc; BundleIdentity; BundleOutputRecord; BundlePointer; BundleState; Canonical; CanonicalizeLegacy; CloneQualityReports; ColumnProvenance; ColumnQuality; … (+43) | internal/dataframe/errors; internal/dataset | internal/acceptance; internal/api/bulk/load; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/resolver; internal/backends/arango/internal/explorer; internal/backends/arango/internal/publication; internal/backends/postgres/internal/explorer; internal/backends/postgres/internal/publication; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/explorer; internal/explorer/lifecycle; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/dataframe/publication/clickhouse` | storage adapter | Materialize and inspect published dataframe bundles in ClickHouse. | P: bundle_store.go; T: abort_ackloss_e2e_test.go; bundle_store_test.go; export_pin_crash_window_e2e_test.go; orphaned_published_growth_e2e_test.go; pinblock_reconcile_e2e_test.go; pointer_name_nul_ambiguity_e2e_test.go; publish_partial_pointer_e2e_test.go; reconcile_toctou_e2e_test.go; reconcile_toctou_test.go; release_dangle_reconcile_e2e_test.go; unpinned_page_read_e2e_test.go; Other: — | 1,223/6,528 | 21: Abort; ActiveReleaseReader; Begin; BundleClickHouseStore; ClickHouseBundleStore; Commit; CreateOutput; ExistingPublishedOutputs; ExistingQualityReports; FinalizeSchema; Idempotent; InsertRows; … (+9) | internal/dataframe/errors; internal/dataframe/publication; internal/dataset; internal/store/clickhouse | internal/server | one production importer (internal/server); inspect ownership/inline potential; storage adapter; preserve interface and persisted-data contracts; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/dataframe/published` | domain/library | Read, page, aggregate, and export authorized published ClickHouse dataframes. | P: aggregate_execute.go; aggregate_plan.go; aggregate_sql.go; aggregate_types.go; artifact.go; current.go; errors.go; export.go; page.go; query_helpers.go; read.go; read_pins.go; schema.go; selection.go; types.go; T: aggregate_clickhouse_integration_test.go; aggregate_execute_test.go; aggregate_plan_test.go; artifact_roundtrip_test.go; artifact_test.go; current_test.go; errors_test.go; export_test.go; nullsort_page_cursor_e2e_test.go; page_test.go; read_pins_test.go; read_test.go; selection_test.go; Other: — | 3,674/2,140 | 56: AggregateBatchRequest; AggregateBatchResult; AggregateJob; AggregateJobResult; AggregateResponseMode; AggregateResult; AggregationResult; AggregationSpec; AggregationsRequest; AggregationsResult; ArtifactColumn; ArtifactDescriptor; … (+44) | internal/dataframe/errors; internal/dataframe/publication; internal/dataframe/unit; internal/dataset; internal/projectid | internal/api/graphql/graph/dataframe; internal/api/graphql/graph/resolver; internal/explorer; internal/explorer/artifactfs; internal/explorer/lifecycle; internal/server | broad surface (15 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/dataframe/recipe` | domain/library | Define, canonicalize, expand, encode, and validate persistence-neutral recipe documents. | P: canonical.go; construction.go; construction_combine.go; construction_reshape.go; derived.go; derived_validation.go; document_types.go; document_validation.go; dynamic_validation.go; fragments.go; json_encoding.go; revisions.go; strict_json.go; table_reshape.go; validation_rich.go; workspace_output_dependencies.go; T: construction_combine_test.go; construction_group_policy_test.go; construction_test.go; correlated_selection_test.go; derived_validation_test.go; documentation_test.go; fragments_test.go; publication_identity_nul_http_test.go; table_reshape_test.go; types_test.go; workspace_output_dependencies_test.go; Other: — | 5,611/1,930 | 147: Aggregate; AggregateOperation; ArrayQuantifier; Bundle; CanonicalBundle; CanonicalJSON; CanonicalJSONWithWorkspaceOutputs; CatalogProjection; Clone; CodeValue; ColumnNaming; ColumnTransformation; … (+135) | generated/fhir; internal/authscope; internal/dataframe/columntransform; internal/dataframe/lineage; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema | cmd/dataframe-profile; cmd/explorer-config-v2-convert; cmd/preview-covering-index-artifact; conformance/compiler; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/backends/arango/internal/recipeexec; internal/backends/postgres/internal/explorer; internal/backends/postgres/internal/recipes; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/execution; internal/dataframe/expression; internal/dataframe/recipe/exec; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/explorer; internal/explorer/capability; internal/explorer/compilation; internal/explorer/lifecycle; internal/server | broad surface (16 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/dataframe/recipe/exec` | domain/library | Define durable recipe-version storage contracts and immutable registration behavior. | P: persistent.go; T: persistent_test.go; Other: — | 84/102 | 5: Entry; PersistentRegistry; Reader; RegisterVersion; Store | internal/dataframe/recipe | internal/api/graphql/graph/resolver; internal/backends/arango/internal/recipeexec; internal/backends/postgres/internal/recipeexec; internal/dataframe/execution; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | | `internal/dataframe/recipe/schema` | domain/library | Resolve catalog-backed recipe declarations into concrete validated schemas. | P: discovery.go; extension_columns.go; resolve.go; T: resolve_test.go; Other: — | 882/293 | 8: Discovery; ExtensionValueObservation; FieldCandidate; Fields; Resolve; Resolved; Scope; SourceSnapshot | internal/dataframe/recipe; internal/fhir/schema | internal/api/graphql/graph/query; internal/api/recipe; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/dataframe/semantic` | domain/library | Convert validated dataframe requests and recipes into backend-neutral logical FHIR plans. | P: doc.go; dynamic_schema.go; execution_context.go; recipe_filter.go; recipe_plan_build.go; recipe_plan_output.go; recipe_plan_types.go; recipe_projection.go; recipe_rich_shaping.go; recipe_traversal.go; resolved_plan.go; selection_semantics.go; selector_spec.go; semantic_nodes.go; semantic_validation.go; T: dynamic_schema_test.go; recipe_plan_test.go; recipe_rich_shaping_test.go; resolved_plan_test.go; Other: — | 3,006/763 | 47: BuildRecipePlan; Candidate; Column; DynamicSpec; ExecutionContext; ExpansionEmptyPolicy; ExpansionExplanation; Explain; ExpressionExplanation; FallbackSelectors; Freeze; FrozenSchema; … (+35) | internal/authscope; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema | cmd/dataframe-profile; cmd/preview-covering-index-artifact; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/execution | broad surface (15 production files); inspect cohesion | -| `internal/dataframe/spec` | domain/library | Define the backend-independent dataframe request, filter, grain, relationship, and selector language. | P: doc.go; filter.go; filter_semantics.go; grain.go; relationship_match.go; selector_validation.go; selectors.go; T: correlated_selection_test.go; Other: — | 744/35 | 31: AllowsMany; ArrayQuantifier; Cardinality; Clone; CodeValue; ContainsFilter; DefaultRowIdentity; ExpandsRows; FilterOperator; FilterValue; FilterValueKind; InferRowGrain; … (+19) | generated/fhir; internal/fhir/schema | internal/api/graphql/graph/query; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/execution; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer/compilation; internal/explorer/lifecycle; internal/server | no automatic consolidation signal; perform semantic review | -| `internal/dataframe/unit` | domain/library | Define unit identity, dimensional compatibility, and approved conversion rules for quantity values. | P: arithmetic.go; unit.go; T: arithmetic_test.go; unit_test.go; Other: — | 442/203 | 27: ApprovedUnitPolicies; ApprovedUnitPolicy; ArithmeticOperand; ArithmeticOperation; ArithmeticUnitError; ArithmeticUnitErrorReason; ColumnArithmeticOperand; Equal; Error; LiteralArithmeticOperand; ResolveApprovedUnitPolicy; ResolveApprovedUnitRules; … (+15) | — | internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer; internal/explorer/authoringv2; internal/explorer/compilation; internal/explorer/lifecycle; internal/server | no automatic consolidation signal; perform semantic review | -| `internal/dataset` | domain/library | Define immutable dataset generation manifests, releases, active pointers, and lifecycle validation. | P: active_resolver.go; dataframe_selector.go; doc.go; manifest.go; ref.go; release.go; schema.go; validation.go; T: active_resolver_test.go; manifest_test.go; memory_store_test.go; ref_test.go; release_test.go; schema_test.go; Other: README.md | 571/473 | 33: Activate; ActivationRequest; Active; ActiveRelease; ActiveReleaseResolver; ActiveResolver; ContractVerification; Create; DataframeSelector; Error; IsStaged; Key; … (+21) | — | cmd/arango-fhir-proto; internal/api/bulk/load; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/dataframe/publication; internal/dataframe/published; internal/dataset/arango; internal/explorer; internal/explorer/arango; internal/explorer/lifecycle; internal/ingest; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/dataset/arango` | storage adapter | Persist immutable dataset generation lifecycle state in ArangoDB. | P: collections.go; doc.go; document_codec.go; manifest_lifecycle.go; release_store.go; T: store_test.go; Other: — | 731/311 | 14: Activate; ActiveDocumentKey; BootstrapSpec; CollectionSpecs; CompareAndSwapActivateRelease; CreateManifest; ManifestDocumentKey; New; ReadActiveRelease; ReadManifest; ResolveActiveManifest; SaveRelease; … (+2) | internal/dataset; internal/store/arango | cmd/arango-fhir-proto; internal/explorer/arango; internal/ingest; internal/server | storage adapter; preserve interface and persisted-data contracts | -| `internal/explorer` | domain/library | Define Explorer aggregate identities, configs, revisions, receipts, output contracts, stores, and core service operations. | P: artifact.go; authoring_errors.go; canonical_json.go; compilation.go; compilation_receipt.go; config_v2.go; explicit_group_repository.go; explicit_groups.go; identity.go; interpretation.go; output_contract.go; selection.go; service.go; state_v1.go; store.go; types.go; viewer_projection.go; T: artifact_test.go; compilation_receipt_test.go; explicit_groups_test.go; interpretation_test.go; output_contract_test.go; repository_revision_id_test.go; selection_test.go; types_test.go; viewer_projection_order_test.go; Other: — | 4,335/1,587 | 165: AbortSelection; ActivateRepositoryGeneration; ActiveRevision; AppendSelectionMembers; ApplyWorkspaceCommands; ApplyWorkspaceCommandsChecked; ArtifactCompletion; ArtifactID; ArtifactRecord; ArtifactStage; ArtifactState; ArtifactStore; … (+153) | internal/dataframe/expression; internal/dataframe/publication; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/unit; internal/dataset; internal/explorer/authoringv2; internal/explorer/capability; internal/projectid | cmd/explorer-config-v2-convert; cmd/loom-acceptance; internal/explorer/arango; internal/explorer/artifactfs; internal/explorer/compilation; internal/explorer/lifecycle; internal/server | broad surface (17 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/explorer/arango` | storage adapter | Persist Explorer drafts, revisions, receipts, publications, capabilities, and repository state in ArangoDB. | P: activation.go; capability_snapshot_store.go; codec.go; collections.go; drafts.go; explicit_groups.go; explorers.go; interpretations.go; publication.go; receipts.go; repository.go; revisions.go; selection_references.go; selections.go; store.go; table_shape_capabilities.go; T: capability_snapshot_store_test.go; collections_test.go; draft_conflict_test.go; draft_transaction_formal_replay_test.go; drafts_integration_test.go; drafts_test.go; explicit_group_storage_integration_test.go; explicit_groups_test.go; interpretations_test.go; receipt_store_test.go; repository_test.go; selection_references_test.go; selection_scale_integration_test.go; selection_storage_integration_test.go; store_test.go; table_shape_capabilities_test.go; Other: — | 3,366/3,467 | 56: AbortExplicitGroupRevision; AbortSelection; ActivateRepositoryGeneration; AppendExplicitGroupMemberships; AppendSelectionMembers; BeginExplicitGroupRevision; BeginSelection; BootstrapSpec; CapabilitySnapshotCollectionSpec; CapabilitySnapshotStore; CleanupExplicitGroupStaging; CleanupSelectionStaging; … (+44) | internal/authscope; internal/dataframe/publication; internal/dataframe/publication/arango; internal/dataset; internal/dataset/arango; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/tableshapecap; internal/projectid; internal/store/arango | internal/server | one production importer (internal/server); inspect ownership/inline potential; broad surface (16 production files); inspect cohesion; storage adapter; preserve interface and persisted-data contracts | -| `internal/explorer/artifactfs` | domain/library | Persist and serve immutable Explorer training artifacts from a dedicated filesystem root. | P: store.go; T: store_test.go; Other: — | 259/140 | 9: Abort; Begin; Close; Commit; Get; New; Open; Store; Write | internal/dataframe/published; internal/explorer | internal/server | one production importer (internal/server); inspect ownership/inline potential | -| `internal/explorer/authoringv2` | domain/library | Define, validate, canonicalize, and mutate the V2 Explorer Builder authoring workspace. | P: availability.go; canonical.go; commands.go; construction.go; construction_change.go; construction_choice.go; construction_migration.go; construction_row_values.go; construction_source_children.go; construction_validation.go; framing.go; migration.go; route_identity.go; row_change.go; row_definition.go; semantic_command.go; semantic_types.go; table_shape.go; transformations.go; types.go; validation.go; value_transformations.go; workspace_output_dependencies.go; T: commands_test.go; compile_preparation_test.go; construction_choice_test.go; construction_combine_test.go; construction_formal_replay_test.go; construction_group_coded_pivot_test.go; construction_group_field_source_test.go; construction_proposal_test.go; construction_related_contributor_test.go; construction_row_values_test.go; construction_source_children_test.go; construction_test.go; correlated_test.go; framing_test.go; interpretation_test.go; migration_test.go; population_member_proposal_test.go; row_change_test.go; row_definition_test.go; row_proposal_test.go; semantic_codeable_concept_fallback_test.go; semantic_coded_category_test.go; semantic_command_test.go; table_shape_proposal_test.go; table_shape_test.go; transformations_test.go; types_test.go; value_transformations_test.go; workspace_output_dependencies_test.go; workspace_test.go; Other: — | 13,182/8,115 | 229: Action; ActionColumn; Addable; AggregateSource; AggregateTransformationCapabilities; AggregateTransformationCapabilitiesForCapability; AggregateTransformationCapabilitiesForCatalog; AnalyzeConstructionCandidate; AnalyzeStepEdit; ApplyCommands; ApplyCommandsRequest; ApplyCommandsResponse; … (+217) | internal/catalog; internal/dataframe/columntransform; internal/dataframe/lineage; internal/dataframe/unit; internal/explorer/capability; internal/fhir/schema | cmd/explorer-config-v2-convert; cmd/preview-covering-index-artifact; generated/loomapi; internal/acceptance; internal/explorer; internal/explorer/arango; internal/explorer/compilation; internal/explorer/lifecycle; internal/repositoryseed; internal/server | broad surface (23 production files); inspect cohesion | -| `internal/explorer/capability` | domain/library | Build the immutable compiler-backed capability contract exposed to Explorer Builder. | P: builder.go; construction.go; construction_routes.go; domain.go; population_route_choice.go; repository.go; row_choice.go; value_operations.go; T: builder_test.go; construction_test.go; domain_test.go; row_choice_test.go; semantic_fallback_dto_test.go; value_operations_test.go; Other: — | 3,950/1,396 | 110: AggregateInput; AggregateOperation; AggregateOperationCapability; AggregateRowContext; Allows; Availability; AvailabilityReadiness; AvailabilitySourceReadiness; Build; Builder; Candidate; CandidateProof; … (+98) | internal/dataframe/recipe | cmd/preview-covering-index-artifact; internal/explorer; internal/explorer/arango; internal/explorer/authoringv2; internal/explorer/compilation; internal/explorer/lifecycle; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/explorer/compilation` | domain/library | Translate validated Explorer authoring intent into native recipe and compilation artifacts. | P: compile.go; construction.go; derived_columns.go; indexed_projection.go; interpretation.go; row_expansion.go; semantic_compile.go; source_row.go; table_reshape.go; T: acceptance_observation_aql_diagnostic_test.go; construction_combine_test.go; construction_related_contributor_test.go; construction_test.go; correlated_test.go; derived_columns_test.go; indexed_projection_test.go; interpretation_test.go; row_expansion_test.go; source_row_test.go; table_reshape_test.go; translate_test.go; value_transformations_test.go; workspace_output_mapping_test.go; workspace_test.go; Other: — | 3,545/4,163 | 24: Canonical; Compile; CompileWorkspace; Error; InterpretationCandidateMatch; InterpretationCandidateResolution; InterpretationCandidateResolutionState; Match; PopulationFor; PresentationColumn; PresentationConfig; Reason; … (+12) | internal/dataframe/lineage; internal/dataframe/recipe; internal/dataframe/spec; internal/dataframe/unit; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/fhir/schema; internal/projectid | cmd/preview-covering-index-artifact; internal/explorer/lifecycle; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/explorer/lifecycle` | domain/library | Coordinate transport-neutral Explorer authoring, preview, publication, query, and policy operations. | P: artifact.go; authoring.go; cell_trace.go; column_source.go; construction_category_discovery.go; construction_choice.go; construction_choice_proposal.go; construction_coded_pivot.go; construction_group_field_choices.go; construction_group_related_field_choices.go; construction_group_source.go; construction_pivot_source.go; construction_proposal.go; construction_routes.go; construction_semantic_group.go; draft_restore.go; errors.go; explicit_group_resolver.go; explicit_groups.go; frame_source.go; interpretation.go; interpretation_apply.go; interpretation_context.go; interpretation_from_column.go; interpretation_library.go; interpretation_preview.go; policy.go; population.go; population_cursor.go; population_member_proposal.go; population_report.go; population_routes.go; preview.go; proposal_receipt.go; publish.go; query.go; repository_publish.go; row_change.go; row_choices.go; row_comparison.go; row_lineage.go; row_proposal.go; row_value_policy.go; schema_field_candidate.go; schema_fields.go; selection.go; semantic_catalog.go; service.go; table_shape_capabilities.go; table_shape_comparison.go; table_shape_proposal.go; table_shape_proposal_composition.go; table_shape_resolution.go; table_shape_saved.go; types.go; T: artifact_test.go; builder_full_ui_replay_test.go; cell_trace_test.go; construction_category_discovery_errors_test.go; construction_category_discovery_test.go; construction_category_presence_test.go; construction_choice_category_search_test.go; construction_choice_proposal_test.go; construction_choice_test.go; construction_coded_pivot_saved_source_test.go; construction_group_field_choices_test.go; construction_pivot_source_test.go; construction_proposal_test.go; construction_proposal_trace_test.go; construction_related_expand_removal_test.go; construction_routes_test.go; construction_semantic_group_test.go; construction_v9_receipt_test.go; explicit_group_resolver_recovery_test.go; explicit_groups_test.go; frame_source_test.go; interpretation_context_test.go; interpretation_from_column_test.go; interpretation_library_test.go; interpretation_preview_test.go; interpretation_test.go; population_attachment_formal_replay_test.go; population_direction_test.go; population_formal_replay_test.go; population_member_proposal_test.go; population_removal_formal_replay_test.go; population_report_test.go; row_change_test.go; row_choices_test.go; row_lineage_test.go; row_proposal_test.go; row_value_policy_test.go; schema_field_candidate_test.go; schema_fields_test.go; selection_test.go; semantic_authoring_test.go; semantic_catalog_test.go; semantic_category_test.go; semantic_coded_concept_test.go; semantic_field_presentation_test.go; semantic_frame_compatibility_test.go; service_test.go; table_shape_capabilities_test.go; table_shape_comparison_evidence_test.go; table_shape_proposal_test.go; Other: — | 22,476/23,247 | 303: AcceptedConstruction; ApplyCommands; ApplyCommandsWithConstructionPreview; ApplyCommandsWithConstructionPreviewResponse; ArtifactOpenRequest; ArtifactPublishedReader; ArtifactRequest; ArtifactResult; AssessRowChange; AssessRowChangeRequest; AssessRowChangeResult; AuthorizedCapability; … (+291) | internal/authscope; internal/catalog; internal/catalog/arango; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/expression; internal/dataframe/publication; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/spec; internal/dataframe/unit; internal/dataset; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation; internal/explorer/tableshapecap; internal/fhir/schema; internal/projectid | internal/server | one production importer (internal/server); inspect ownership/inline potential; broad surface (55 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/explorer/tableshapecap` | domain/library | Define immutable table-shape capability and resolution artifacts plus their create-once repository contract. | P: identity.go; repository.go; types.go; T: artifacts_test.go; Other: — | 1,971/503 | 65: AvailabilityState; Binding; BooleanScalar; CanonicalContent; CatalogChoices; CatalogLookup; CatalogReceipt; CategoryProof; CategoryScanReceipt; CategoryValuesDigest; Choice; ChoiceRole; … (+53) | internal/projectid | internal/explorer/arango; internal/explorer/lifecycle; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/fhir/schema` | domain/library | Add handwritten selector, traversal, pivot, and compiler semantics to generated FHIR metadata. | P: compiler_semantics.go; correlated.go; fields.go; generated_metadata.go; identifier.go; index.go; pivots.go; quantity.go; row_resolution.go; selectors.go; structure.go; terminal_metadata.go; traversals.go; T: compiler_semantics_test.go; correlated_selection_test.go; correlated_test.go; fields_test.go; identifier_test.go; index_test.go; pivots_test.go; root_resources_test.go; row_resolution_test.go; selectors_test.go; structure_test.go; terminal_metadata_test.go; traversals_test.go; Other: — | 2,446/1,341 | 86: ArrayElementType; ArrayElements; At; BindingStrength; CanonicalPath; CanonicalResourceType; CanonicalizePath; ChoiceValueSelectorOptions; CompilerTraversal; ConcreteResourceType; ContainsFilter; CorrelatedBinding; … (+74) | generated/fhirschema | cmd/arango-fhir-proto; generated/fhir; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/catalog; internal/catalog/arango; internal/dataframe/compiler/capability; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/recipe; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/dataframe/spec; internal/explorer/authoringv2; internal/explorer/compilation; internal/explorer/lifecycle; internal/fhir/semantic; internal/ingest; internal/server; scripts | broad surface (13 production files); inspect cohesion | +| `internal/dataframe/semantic` | domain/library | Convert validated dataframe requests and recipes into backend-neutral logical FHIR plans. | P: doc.go; dynamic_schema.go; execution_context.go; recipe_filter.go; recipe_plan_build.go; recipe_plan_output.go; recipe_plan_types.go; recipe_projection.go; recipe_rich_shaping.go; recipe_traversal.go; resolved_plan.go; selection_semantics.go; selector_spec.go; semantic_nodes.go; semantic_validation.go; T: dynamic_schema_test.go; recipe_plan_test.go; recipe_plan_user_fault_class_test.go; recipe_rich_shaping_test.go; resolved_plan_test.go; Other: — | 3,022/931 | 47: BuildRecipePlan; Candidate; Column; DynamicSpec; ExecutionContext; ExpansionEmptyPolicy; ExpansionExplanation; Explain; ExpressionExplanation; FallbackSelectors; Freeze; FrozenSchema; … (+35) | internal/authscope; internal/dataframe/errors; internal/dataframe/expression; internal/dataframe/recipe; internal/dataframe/spec; internal/dataframe/unit; internal/fhir/schema | cmd/dataframe-profile; cmd/preview-covering-index-artifact; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/lower; internal/dataframe/execution; internal/server | broad surface (15 production files); inspect cohesion | +| `internal/dataframe/spec` | domain/library | Define the backend-independent dataframe request, filter, grain, relationship, and selector language. | P: doc.go; filter.go; filter_semantics.go; grain.go; relationship_match.go; selector_validation.go; selectors.go; T: correlated_selection_test.go; grain_classification_test.go; Other: — | 745/71 | 31: AllowsMany; ArrayQuantifier; Cardinality; Clone; CodeValue; ContainsFilter; DefaultRowIdentity; ExpandsRows; FilterOperator; FilterValue; FilterValueKind; InferRowGrain; … (+19) | generated/fhir; internal/dataframe/errors; internal/fhir/schema | internal/api/graphql/graph/query; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/postgres; internal/dataframe/execution; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer/compilation; internal/explorer/lifecycle; internal/server | no automatic consolidation signal; perform semantic review | +| `internal/dataframe/unit` | domain/library | Define unit identity, dimensional compatibility, and approved conversion rules for quantity values. | P: arithmetic.go; unit.go; T: arithmetic_test.go; unit_test.go; Other: — | 442/203 | 27: ApprovedUnitPolicies; ApprovedUnitPolicy; ArithmeticOperand; ArithmeticOperation; ArithmeticUnitError; ArithmeticUnitErrorReason; ColumnArithmeticOperand; Equal; Error; LiteralArithmeticOperand; ResolveApprovedUnitPolicy; ResolveApprovedUnitRules; … (+15) | — | internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/postgres; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/semantic; internal/explorer; internal/explorer/authoringv2; internal/explorer/compilation; internal/explorer/lifecycle; internal/server | no automatic consolidation signal; perform semantic review | +| `internal/dataset` | domain/library | Define immutable dataset generation manifests, releases, active pointers, and lifecycle validation. | P: active_resolver.go; dataframe_selector.go; doc.go; manifest.go; manifest_port.go; ref.go; release.go; schema.go; validation.go; T: active_resolver_test.go; manifest_test.go; memory_store_test.go; ref_test.go; release_id_framing_test.go; release_idempotency_test.go; release_requirements_e2e_test.go; release_test.go; release_verifier_fault_classification_test.go; schema_test.go; Other: README.md | 802/2,253 | 34: Activate; ActivationRequest; Active; ActiveRelease; ActiveReleaseResolver; ActiveResolver; ContractVerification; Create; DataframeSelector; Error; IsStaged; Key; … (+22) | — | cmd/arango-fhir-proto; cmd/loom-data; internal/api/bulk/load; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/backends/arango/internal/dataset; internal/backends/arango/internal/explorer; internal/backends/postgres/control; internal/backends/postgres/internal/dataset; internal/backends/postgres/internal/explorer; internal/backends/postgres/internal/publication; internal/dataframe/publication; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/explorer; internal/explorer/lifecycle; internal/ingest; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/explorer` | domain/library | Define Explorer aggregate identities, configs, revisions, receipts, output contracts, stores, and core service operations. | P: artifact.go; authoring_errors.go; canonical_json.go; compilation.go; compilation_receipt.go; config_v2.go; explicit_group_repository.go; explicit_groups.go; identity.go; interpretation.go; output_contract.go; selection.go; service.go; state_v1.go; store.go; types.go; viewer_projection.go; T: artifact_test.go; compilation_receipt_test.go; explicit_groups_test.go; interpretation_test.go; output_contract_test.go; repository_revision_id_test.go; selection_test.go; types_test.go; viewer_projection_order_test.go; Other: — | 4,335/1,587 | 165: AbortSelection; ActivateRepositoryGeneration; ActiveRevision; AppendSelectionMembers; ApplyWorkspaceCommands; ApplyWorkspaceCommandsChecked; ArtifactCompletion; ArtifactID; ArtifactRecord; ArtifactStage; ArtifactState; ArtifactStore; … (+153) | internal/dataframe/expression; internal/dataframe/publication; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/unit; internal/dataset; internal/explorer/authoringv2; internal/explorer/capability; internal/projectid | cmd/explorer-config-v2-convert; cmd/loom-acceptance; internal/backends/arango/internal/explorer; internal/backends/postgres/internal/explorer; internal/explorer/artifactfs; internal/explorer/compilation; internal/explorer/lifecycle; internal/server | broad surface (17 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/explorer/artifactfs` | domain/library | Persist and serve immutable Explorer training artifacts from a dedicated filesystem root. | P: store.go; T: reconcile_test.go; store_test.go; Other: — | 379/621 | 10: Abort; Begin; Close; Commit; Get; New; Open; Reconcile; Store; Write | internal/dataframe/published; internal/explorer | internal/server | one production importer (internal/server); inspect ownership/inline potential | +| `internal/explorer/authoringv2` | domain/library | Define, validate, canonicalize, and mutate the V2 Explorer Builder authoring workspace. | P: availability.go; canonical.go; commands.go; construction.go; construction_change.go; construction_choice.go; construction_migration.go; construction_row_values.go; construction_source_children.go; construction_validation.go; framing.go; migration.go; route_identity.go; row_change.go; row_definition.go; semantic_command.go; semantic_types.go; table_shape.go; transformations.go; types.go; validation.go; value_transformations.go; workspace_output_dependencies.go; T: commands_test.go; compile_preparation_test.go; construction_choice_test.go; construction_combine_test.go; construction_expand_test.go; construction_formal_replay_test.go; construction_group_coded_pivot_test.go; construction_group_field_source_test.go; construction_projection_replay_test.go; construction_proposal_test.go; construction_related_contributor_test.go; construction_row_values_test.go; construction_source_children_test.go; construction_test.go; correlated_test.go; framing_test.go; interpretation_test.go; migration_test.go; population_member_proposal_test.go; row_change_test.go; row_definition_test.go; row_proposal_test.go; semantic_codeable_concept_fallback_test.go; semantic_coded_category_test.go; semantic_command_test.go; table_shape_proposal_test.go; table_shape_test.go; transformations_test.go; types_test.go; value_transformations_test.go; workspace_output_dependencies_test.go; workspace_test.go; Other: — | 13,260/8,548 | 231: Action; ActionColumn; Addable; AggregateSource; AggregateTransformationCapabilities; AggregateTransformationCapabilitiesForCapability; AggregateTransformationCapabilitiesForCatalog; AnalyzeConstructionCandidate; AnalyzeStepEdit; ApplyCommands; ApplyCommandsRequest; ApplyCommandsResponse; … (+219) | internal/catalog; internal/dataframe/columntransform; internal/dataframe/lineage; internal/dataframe/unit; internal/explorer/capability; internal/fhir/schema | cmd/explorer-config-v2-convert; cmd/preview-covering-index-artifact; generated/loomapi; internal/acceptance; internal/backends/arango/internal/explorer; internal/explorer; internal/explorer/compilation; internal/explorer/lifecycle; internal/repositoryseed; internal/server | broad surface (23 production files); inspect cohesion | +| `internal/explorer/capability` | domain/library | Build the immutable compiler-backed capability contract exposed to Explorer Builder. | P: builder.go; construction.go; construction_routes.go; domain.go; population_route_choice.go; repository.go; row_choice.go; value_operations.go; T: builder_test.go; construction_test.go; domain_test.go; row_choice_test.go; semantic_fallback_dto_test.go; value_operations_test.go; Other: — | 3,950/1,396 | 110: AggregateInput; AggregateOperation; AggregateOperationCapability; AggregateRowContext; Allows; Availability; AvailabilityReadiness; AvailabilitySourceReadiness; Build; Builder; Candidate; CandidateProof; … (+98) | internal/dataframe/recipe | cmd/preview-covering-index-artifact; internal/backends/arango/internal/explorer; internal/backends/postgres/internal/explorer; internal/explorer; internal/explorer/authoringv2; internal/explorer/compilation; internal/explorer/lifecycle; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/explorer/compilation` | domain/library | Translate validated Explorer authoring intent into native recipe and compilation artifacts. | P: compile.go; construction.go; derived_columns.go; indexed_projection.go; interpretation.go; row_expansion.go; semantic_compile.go; source_row.go; table_reshape.go; T: acceptance_observation_aql_diagnostic_test.go; command_helpers_test.go; construction_combine_test.go; construction_related_contributor_test.go; construction_test.go; correlated_test.go; derived_columns_test.go; indexed_projection_test.go; interpretation_test.go; row_expansion_test.go; semantic_compile_expand_source_test.go; source_row_test.go; table_reshape_test.go; translate_test.go; value_transformations_test.go; workspace_output_mapping_test.go; workspace_test.go; Other: — | 3,622/4,536 | 24: Canonical; Compile; CompileWorkspace; Error; InterpretationCandidateMatch; InterpretationCandidateResolution; InterpretationCandidateResolutionState; Match; PopulationFor; PresentationColumn; PresentationConfig; Reason; … (+12) | internal/dataframe/compiler/lower; internal/dataframe/lineage; internal/dataframe/recipe; internal/dataframe/spec; internal/dataframe/unit; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/fhir/schema; internal/projectid | cmd/preview-covering-index-artifact; internal/explorer/lifecycle; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/explorer/lifecycle` | domain/library | Coordinate transport-neutral Explorer authoring, preview, publication, query, and policy operations. | P: artifact.go; authoring.go; cell_trace.go; column_source.go; construction_category_discovery.go; construction_choice.go; construction_choice_proposal.go; construction_coded_pivot.go; construction_group_field_choices.go; construction_group_related_field_choices.go; construction_group_source.go; construction_pivot_source.go; construction_proposal.go; construction_routes.go; construction_semantic_group.go; derive_source.go; draft_restore.go; errors.go; expand_source.go; explicit_group_resolver.go; explicit_groups.go; filter_source.go; frame_source.go; interpretation.go; interpretation_apply.go; interpretation_context.go; interpretation_from_column.go; interpretation_library.go; interpretation_preview.go; policy.go; population.go; population_cursor.go; population_member_proposal.go; population_report.go; population_routes.go; preview.go; proposal_receipt.go; publish.go; query.go; repository_publish.go; row_change.go; row_choices.go; row_comparison.go; row_lineage.go; row_proposal.go; row_value_policy.go; schema_field_candidate.go; schema_fields.go; selection.go; semantic_catalog.go; service.go; table_shape_capabilities.go; table_shape_comparison.go; table_shape_proposal.go; table_shape_proposal_composition.go; table_shape_resolution.go; table_shape_saved.go; types.go; unpivot_source.go; T: artifact_test.go; builder_full_ui_replay_test.go; cell_trace_test.go; construction_category_discovery_errors_test.go; construction_category_discovery_test.go; construction_category_presence_test.go; construction_choice_category_search_test.go; construction_choice_proposal_test.go; construction_choice_test.go; construction_coded_pivot_saved_source_test.go; construction_group_field_choices_test.go; construction_pivot_source_test.go; construction_proposal_test.go; construction_proposal_trace_test.go; construction_related_expand_removal_test.go; construction_routes_test.go; construction_semantic_group_test.go; construction_v9_receipt_test.go; derive_source_test.go; expand_source_test.go; explicit_group_resolver_recovery_test.go; explicit_groups_test.go; filter_source_pivot_clear_test.go; filter_source_test.go; frame_source_test.go; fs1_reverse_order_collision_test.go; interpretation_context_test.go; interpretation_from_column_test.go; interpretation_library_test.go; interpretation_preview_test.go; interpretation_test.go; population_attachment_formal_replay_test.go; population_direction_test.go; population_formal_replay_test.go; population_member_proposal_test.go; population_removal_formal_replay_test.go; population_report_test.go; publish_release_requirements_classification_test.go; repository_publish_activation_conflict_test.go; repository_publish_generation_activation_classification_test.go; repository_publish_release_cas_conflict_test.go; row_change_test.go; row_choices_test.go; row_lineage_test.go; row_proposal_test.go; row_value_policy_test.go; schema_field_candidate_test.go; schema_fields_test.go; selection_test.go; semantic_authoring_test.go; semantic_catalog_test.go; semantic_category_test.go; semantic_coded_concept_test.go; semantic_field_presentation_test.go; semantic_frame_compatibility_test.go; service_test.go; table_shape_capabilities_test.go; table_shape_comparison_evidence_test.go; table_shape_proposal_test.go; unpivot_source_test.go; Other: — | 25,545/31,520 | 319: AcceptedConstruction; ApplyCommands; ApplyCommandsWithConstructionPreview; ApplyCommandsWithConstructionPreviewResponse; ArtifactOpenRequest; ArtifactPublishedReader; ArtifactRequest; ArtifactResult; AssessRowChange; AssessRowChangeRequest; AssessRowChangeResult; AuthorizedCapability; … (+307) | internal/authscope; internal/catalog; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/expression; internal/dataframe/publication; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/spec; internal/dataframe/unit; internal/dataset; internal/explorer; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation; internal/explorer/tableshapecap; internal/fhir/schema; internal/projectid | internal/server | one production importer (internal/server); inspect ownership/inline potential; broad surface (59 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/explorer/tableshapecap` | domain/library | Define immutable table-shape capability and resolution artifacts plus their create-once repository contract. | P: identity.go; repository.go; types.go; T: artifacts_test.go; Other: — | 1,971/503 | 65: AvailabilityState; Binding; BooleanScalar; CanonicalContent; CatalogChoices; CatalogLookup; CatalogReceipt; CategoryProof; CategoryScanReceipt; CategoryValuesDigest; Choice; ChoiceRole; … (+53) | internal/projectid | internal/backends/arango/internal/explorer; internal/backends/postgres/internal/explorer; internal/explorer/lifecycle; internal/server | runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/fhir/schema` | domain/library | Add handwritten selector, traversal, pivot, and compiler semantics to generated FHIR metadata. | P: compiler_semantics.go; correlated.go; fields.go; generated_metadata.go; identifier.go; index.go; pivots.go; quantity.go; row_resolution.go; selectors.go; structure.go; terminal_metadata.go; traversals.go; T: compiler_semantics_test.go; correlated_selection_test.go; correlated_test.go; fields_test.go; identifier_test.go; index_test.go; pivots_test.go; root_resources_test.go; row_resolution_test.go; selectors_test.go; structure_test.go; terminal_metadata_test.go; traversals_test.go; Other: — | 2,446/1,341 | 86: ArrayElementType; ArrayElements; At; BindingStrength; CanonicalPath; CanonicalResourceType; CanonicalizePath; ChoiceValueSelectorOptions; CompilerTraversal; ConcreteResourceType; ContainsFilter; CorrelatedBinding; … (+74) | generated/fhirschema | cmd/arango-fhir-proto; generated/fhir; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/backends/arango/internal/catalog; internal/backends/postgres/internal/catalog; internal/catalog; internal/dataframe/compiler/capability; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/compiler/render/aql; internal/dataframe/compiler/render/postgres; internal/dataframe/recipe; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/dataframe/spec; internal/explorer/authoringv2; internal/explorer/compilation; internal/explorer/lifecycle; internal/fhir/semantic; internal/ingest; internal/server; scripts | broad surface (13 production files); inspect cohesion | | `internal/fhir/semantic` | domain/library | Walk generated FHIR definition indexes to enumerate typed referenced datatype occurrences. | P: member_walk.go; walk.go; T: member_walk_resource_dispatch_test.go; member_walk_test.go; walk_test.go; Other: — | 829/949 | 17: Error; Len; Less; MemberFact; Occurrence; Pop; Push; RepeatedBoundary; ResourceBoundary; Swap; Walk; WalkDiagnostic; … (+5) | internal/fhir/schema | internal/catalog | one production importer (internal/catalog); inspect ownership/inline potential | -| `internal/ingest` | domain/library | Preflight and load FHIR NDJSON into generation-scoped Arango vertices, edges, and catalog facts. | P: backend.go; doc.go; documents.go; files.go; generated_load.go; generation_identity.go; generation_orchestration.go; generation_workers.go; load_options.go; preflight.go; progress.go; resource_load.go; row_builder.go; semantic_inventory_backfill.go; source_schema.go; T: backend_test.go; basic_observation_fixture_validation_test.go; bench_test.go; documents_test.go; file_pipeline_test.go; files_test.go; generated_load_test.go; generation_cleanup_test.go; generation_identity_test.go; generation_load_test.go; integration_test.go; load_options_test.go; preflight_test.go; related_medication_fixture_test.go; row_builder_test.go; semantic_inventory_backfill_integration_test.go; semantic_inventory_backfill_test.go; semantic_inventory_test.go; source_schema_test.go; viewer_row_file_download_fixture_test.go; Other: — | 2,832/2,879 | 36: ActivationOutcomeError; BackfillSemanticInventory; Build; DiscoverNDJSON; EdgeDocument; EdgeFromGrip; Error; EventSink; GeneratedRowBuilder; GenerationLoadIncompleteError; GenericRowBuilder; IngestionMode; … (+24) | generated/fhir; internal/catalog; internal/catalog/arango; internal/dataset; internal/dataset/arango; internal/fhir/schema; internal/store/arango | cmd/arango-fhir-proto; internal/api/bulk/load; internal/server | broad surface (15 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion | -| `internal/projectid` | domain/library | Define canonical and legacy-compatible public project identity normalization. | P: projectid.go; T: projectid_test.go; Other: — | 51/18 | 2: Canonical; Legacy | — | cmd/loom-repository; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/authscope; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/execution; internal/dataframe/published; internal/explorer; internal/explorer/arango; internal/explorer/compilation; internal/explorer/lifecycle; internal/explorer/tableshapecap; internal/repositoryseed; internal/server | small shared semantic contract; compare direct callers before inlining | +| `internal/ingest` | domain/library | Preflight and load FHIR NDJSON into generation-scoped Arango vertices, edges, and catalog facts, and commit built resource-row and built-edge-occurrence batches to the PostgreSQL source store through the ledgered batch writer with scope verification, canonical payload encoding, and payload-backed path/ordinal recovery for repeated references. | P: backend.go; doc.go; documents.go; files.go; generated_load.go; generation_identity.go; generation_load.go; generation_orchestration.go; generation_workers.go; load_options.go; postgres_writer.go; preflight.go; progress.go; resource_load.go; row_builder.go; semantic_inventory_backfill.go; source_schema.go; T: backend_test.go; basic_observation_fixture_validation_test.go; bench_test.go; documents_test.go; file_pipeline_test.go; files_test.go; generated_load_test.go; generation_cleanup_test.go; generation_failed_cleanup_e2e_test.go; generation_identity_test.go; generation_load_postgres_durability_test.go; generation_load_postgres_test.go; generation_load_test.go; integration_test.go; keycollide_generated_e2e_test.go; load_options_test.go; postgres_writer_test.go; preflight_test.go; related_medication_fixture_test.go; resource_load_test.go; row_builder_test.go; semantic_inventory_backfill_integration_test.go; semantic_inventory_backfill_test.go; semantic_inventory_test.go; source_schema_test.go; viewer_row_file_download_fixture_test.go; Other: — | 5,650/10,567 | 63: ActivatePostgresGeneration; ActivationOutcomeError; BackfillSemanticInventory; Build; Complete; ConnectionOptions; DiscoverNDJSON; EdgeDocument; EdgeFromGrip; Error; EventSink; GeneratedRowBuilder; … (+51) | generated/fhir; internal/backends/arango; internal/backends/postgres; internal/backends/postgres/control; internal/catalog; internal/dataset; internal/fhir/schema; internal/source | cmd/arango-fhir-proto; cmd/loom-data; internal/api/bulk/load; internal/server | broad surface (17 production files); inspect cohesion; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/projectid` | domain/library | Define canonical and legacy-compatible public project identity normalization. | P: projectid.go; T: projectid_test.go; Other: — | 51/18 | 2: Canonical; Legacy | — | cmd/loom-repository; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/authscope; internal/backends/arango/internal/explorer; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/execution; internal/dataframe/published; internal/explorer; internal/explorer/compilation; internal/explorer/lifecycle; internal/explorer/tableshapecap; internal/repositoryseed; internal/server | small shared semantic contract; compare direct callers before inlining | | `internal/repositoryseed` | domain/library | Validate repository metadata and configuration, produce a source manifest, and refuse unmaterialized inputs before launch. | P: preflight.go; T: preflight_test.go; Other: — | 281/105 | 6: Error; File; Manifest; Options; Prepare; UnmaterializedError | internal/explorer/authoringv2; internal/projectid | cmd/loom-repository | one production importer (cmd/loom-repository); inspect ownership/inline potential | -| `internal/server` | domain/library | Compose storage, domain services, generated OpenAPI handlers, GraphQL adapters, and HTTP routes. | P: artifact_dev.go; auth_wiring.go; construction_inputs_catalog.go; explain_wiring.go; explorer_authoring_http.go; explorer_capability.go; explorer_combine_output_metadata.go; explorer_config_route.go; explorer_lifecycle_helpers.go; explorer_lifecycle_wire.go; explorer_output_reconcile.go; explorer_preview_response.go; explorer_preview_security.go; explorer_publication.go; explorer_receipt_contract.go; explorer_receipt_validation.go; local_workspace_writeback.go; openapi_data_routes.go; openapi_explorer_artifacts.go; openapi_explorer_authoring_direct.go; openapi_explorer_construction.go; openapi_explorer_construction_choice_proposal.go; openapi_explorer_construction_inputs.go; openapi_explorer_construction_proposal.go; openapi_explorer_conversion.go; openapi_explorer_direct.go; openapi_explorer_explicit_groups.go; openapi_explorer_population_member_proposal.go; openapi_explorer_routes.go; openapi_explorer_row_definition.go; openapi_explorer_selections.go; openapi_explorer_table_shape.go; openapi_frame_source_options.go; openapi_graphql_routes.go; openapi_recipe_execution.go; openapi_routes.go; openapi_semantic_inventory.go; options.go; private_artifact_reconciler.go; receipt_preview_diagnostic.go; recipe_authorization.go; recipe_combine_resolution.go; recipe_discovery.go; recipe_publication.go; release_verifier.go; routes.go; server.go; T: accepted_construction_contract_test.go; append_workspace_rows_integration_test.go; auth_wiring_trusted_issuer_test.go; case008_row_definition_preview_route_test.go; config_test.go; construction_category_discovery_openapi_error_test.go; construction_inputs_catalog_test.go; construction_proposal_zero_column_test.go; dataframe_query_error_test.go; derived_policy_rows_integration_test.go; explorer_capability_test.go; explorer_coded_value_capability_test.go; explorer_concept_evidence_test.go; explorer_config_route_test.go; explorer_diagnostic_logging_test.go; explorer_output_reconcile_source_child_test.go; explorer_preview_response_test.go; explorer_preview_security_test.go; explorer_receipt_contract_error_test.go; explorer_receipt_contract_test.go; explorer_reconcile_scope_test.go; explorer_row_change_route_test.go; explorer_state_route_test.go; http_test_helpers_test.go; local_workspace_writeback_test.go; openapi_column_id_contract_test.go; openapi_explorer_cell_trace_test.go; openapi_explorer_construction_apply_preview_test.go; openapi_explorer_construction_inputs_test.go; openapi_explorer_construction_proposal_test.go; openapi_explorer_conversion_test.go; openapi_explorer_direct_test.go; openapi_explorer_draft_restore_test.go; openapi_explorer_population_mapping_construction_test.go; openapi_explorer_population_member_proposal_test.go; openapi_explorer_restricted_starting_collection_integration_test.go; openapi_explorer_route_roundtrip_test.go; openapi_explorer_route_rows_arango_test.go; openapi_explorer_row_definition_test.go; openapi_explorer_write_scope_test.go; openapi_field_presentation_fixture_test.go; openapi_interpretation_routes_test.go; openapi_recipe_execution_test.go; openapi_route_inventory_test.go; openapi_routes_test.go; openapi_semantic_inventory_test.go; openapi_status_coverage_test.go; openapi_test_helpers_test.go; patient_one_construction_choice_route_test.go; population_member_receipt_test.go; private_artifact_reconciler_test.go; publication_rows_integration_test.go; publication_viewer_agreement_test.go; receipt_preview_diagnostic_test.go; recipe_authorization_test.go; recipe_combine_resolution_test.go; recipe_discovery_test.go; recipe_publication_test.go; release_verifier_test.go; routes_test.go; row_value_policy_stream_test.go; wiring_test.go; Other: — | 11,461/16,961 | 88: ActivateDatasetGeneration; ApplyExplorerBuilderCommands; AssessExplorerRowChange; AuthConfig; AuthorizeRead; AuthorizeWrite; BasicAuthConfig; BrowseExplorerFrameSourceOptions; BrowseExplorerSemanticInventory; CalyprAuthConfig; CheckExplorerPopulationMapping; ClickHouseConfig; … (+76) | generated/loomapi; internal/api/authpolicy; internal/api/bulk/load; internal/api/columncapabilities; internal/api/graphql/graph; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/api/http; internal/api/recipe; internal/authscope; internal/catalog; internal/catalog/arango; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/execution/chartifact; internal/dataframe/execution/chartifact/arango; internal/dataframe/expression; internal/dataframe/publication; internal/dataframe/publication/arango; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/recipe/exec/arango; internal/dataframe/recipe/schema; internal/dataframe/spec; internal/dataframe/unit; internal/dataset; internal/dataset/arango; internal/explorer; internal/explorer/arango; internal/explorer/artifactfs; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation; internal/explorer/lifecycle; internal/explorer/tableshapecap; internal/fhir/schema; internal/ingest; internal/projectid; internal/store/arango; internal/store/clickhouse | .; cmd/arango-fhir-server | broad surface (47 production files); inspect cohesion | -| `internal/store/arango` | storage adapter | Provide the low-level typed ArangoDB client, query streaming, profiling, and Explain assessment boundary. | P: client.go; explain.go; explain_assessment.go; explain_client.go; options.go; preview_index.go; profile.go; profile_client.go; types.go; T: client_query_memory_limit_test.go; client_queryrows_test.go; client_revision_test.go; explain_assessment_test.go; explain_client_test.go; explain_test.go; population_profile_integration_test.go; preview_index_test.go; profile_client_test.go; profile_integration_test.go; profile_test.go; Other: — | 1,658/1,835 | 67: AQLExecutor; AssessExplainResult; BatchInserter; Bootstrap; BootstrapSpec; Client; Close; CollectionCount; CollectionExists; CollectionRevision; CollectionSpec; ConnectionOptions; … (+55) | — | cmd/arango-fhir-proto; cmd/dataframe-profile; internal/acceptance; internal/catalog/arango; internal/dataframe/execution; internal/dataframe/execution/chartifact/arango; internal/dataframe/publication/arango; internal/dataframe/recipe/exec/arango; internal/dataset/arango; internal/explorer/arango; internal/ingest; internal/server | storage adapter; preserve interface and persisted-data contracts; runtime/build signals present; static importer counts are insufficient for deletion | +| `internal/server` | domain/library | Compose storage, domain services, generated OpenAPI handlers, GraphQL adapters, and HTTP routes. | P: artifact_dev.go; auth_wiring.go; compose.go; construction_inputs_catalog.go; explain_wiring.go; explorer_authoring_http.go; explorer_capability.go; explorer_combine_output_metadata.go; explorer_config_route.go; explorer_lifecycle_helpers.go; explorer_lifecycle_wire.go; explorer_output_reconcile.go; explorer_preview_response.go; explorer_preview_security.go; explorer_publication.go; explorer_receipt_contract.go; explorer_receipt_validation.go; local_workspace_writeback.go; openapi_data_routes.go; openapi_explorer_artifacts.go; openapi_explorer_authoring_direct.go; openapi_explorer_construction.go; openapi_explorer_construction_choice_proposal.go; openapi_explorer_construction_inputs.go; openapi_explorer_construction_proposal.go; openapi_explorer_conversion.go; openapi_explorer_direct.go; openapi_explorer_explicit_groups.go; openapi_explorer_population_member_proposal.go; openapi_explorer_routes.go; openapi_explorer_row_definition.go; openapi_explorer_selections.go; openapi_explorer_table_shape.go; openapi_frame_source_options.go; openapi_graphql_routes.go; openapi_recipe_execution.go; openapi_routes.go; openapi_semantic_inventory.go; options.go; private_artifact_reconciler.go; receipt_preview_diagnostic.go; recipe_authorization.go; recipe_combine_resolution.go; recipe_discovery.go; recipe_publication.go; release_verifier.go; routes.go; row_sources.go; runtime.go; server.go; T: accepted_construction_contract_test.go; append_workspace_rows_integration_test.go; artifact_reconciler_wiring_test.go; auth_wiring_trusted_issuer_test.go; backend_composition_contract_test.go; backend_error_boundary_contract_test.go; case008_row_definition_preview_route_test.go; command_helpers_test.go; config_test.go; construction_category_discovery_openapi_error_test.go; construction_inputs_catalog_test.go; construction_proposal_zero_column_test.go; core_authoring_protocol_arango_conformance_test.go; core_authoring_protocol_formal_replay_test.go; core_authoring_protocol_receipts_test.go; core_authoring_protocol_restore_test.go; core_authoring_protocol_rowschema_test.go; core_authoring_protocol_scoped_compile_test.go; core_authoring_protocol_test.go; dataframe_query_error_test.go; derived_policy_rows_integration_test.go; explain_wiring_pg18_test.go; explain_wiring_preview_test.go; explorer_capability_test.go; explorer_coded_value_capability_test.go; explorer_concept_evidence_test.go; explorer_config_route_test.go; explorer_diagnostic_logging_test.go; explorer_output_reconcile_derive_source_test.go; explorer_output_reconcile_source_child_test.go; explorer_output_reconcile_unpivot_source_test.go; explorer_preview_response_test.go; explorer_preview_security_test.go; explorer_receipt_contract_error_test.go; explorer_receipt_contract_test.go; explorer_reconcile_scope_test.go; explorer_row_change_route_test.go; explorer_state_route_test.go; fs1_signed_slot_collision_test.go; health_readiness_wiring_test.go; http_test_helpers_test.go; local_workspace_writeback_test.go; openapi_column_id_contract_test.go; openapi_explorer_artifacts_download_test.go; openapi_explorer_authoring_write_fault_test.go; openapi_explorer_cell_trace_test.go; openapi_explorer_commands_notfound_test.go; openapi_explorer_construction_apply_preview_test.go; openapi_explorer_construction_derive_source_test.go; openapi_explorer_construction_expand_source_test.go; openapi_explorer_construction_filter_source_test.go; openapi_explorer_construction_inputs_test.go; openapi_explorer_construction_proposal_test.go; openapi_explorer_construction_unpivot_source_test.go; openapi_explorer_conversion_test.go; openapi_explorer_direct_test.go; openapi_explorer_draft_restore_test.go; openapi_explorer_legacy_details_control_test.go; openapi_explorer_population_mapping_construction_test.go; openapi_explorer_population_member_proposal_test.go; openapi_explorer_restricted_starting_collection_integration_test.go; openapi_explorer_route_roundtrip_test.go; openapi_explorer_route_rows_arango_test.go; openapi_explorer_row_definition_test.go; openapi_explorer_write_scope_test.go; openapi_field_presentation_fixture_test.go; openapi_graphql_status_test.go; openapi_graphql_transport_review_test.go; openapi_interpretation_routes_test.go; openapi_recipe_execution_test.go; openapi_route_inventory_test.go; openapi_routes_test.go; openapi_semantic_inventory_test.go; openapi_status_coverage_test.go; openapi_test_helpers_test.go; options_schemas_test.go; patient_one_construction_choice_route_test.go; population_member_receipt_test.go; postgres_query_test.go; postgres_refusal_contract_test.go; preview_postgres_wiring_pg18_test.go; private_artifact_reconciler_test.go; publication_rows_integration_test.go; publication_viewer_agreement_test.go; receipt_preview_diagnostic_test.go; recipe_authorization_test.go; recipe_blank_project_test.go; recipe_combine_resolution_test.go; recipe_discovery_test.go; recipe_execution_etl_failure_fields_e2e_test.go; recipe_execution_malformed_persisted_scope_test.go; recipe_execution_malformed_scope_header_probe_test.go; recipe_execution_status_twin_e2e_test.go; recipe_execution_twin_authz_repair_test.go; recipe_project_keying_test.go; recipe_publication_test.go; release_verifier_fault_mapping_test.go; release_verifier_test.go; repository_activation_conflict_dev_test.go; route_composition_contract_test.go; routes_test.go; row_tag_guard_contract_test.go; row_value_policy_stream_test.go; runtime_postgres_live_test.go; runtime_test.go; wiring_test.go; zz_readiness_contract_guard_test.go; Other: — | 14,261/37,094 | 103: Activate; ActivateDatasetGeneration; ActivateRepositoryGeneration; ApplyExplorerBuilderCommands; AssessExplorerRowChange; AuthConfig; AuthorizeRead; AuthorizeWrite; BasicAuthConfig; BrowseExplorerFrameSourceOptions; BrowseExplorerSemanticInventory; CalyprAuthConfig; … (+91) | generated/loomapi; internal/api/authpolicy; internal/api/bulk/load; internal/api/columncapabilities; internal/api/graphql/graph; internal/api/graphql/graph/dataframe; internal/api/graphql/graph/query; internal/api/graphql/graph/resolver; internal/api/http; internal/api/recipe; internal/authscope; internal/backends/arango; internal/backends/postgres; internal/backends/postgres/control; internal/catalog; internal/dataframe/compiler; internal/dataframe/compiler/capability; internal/dataframe/compiler/command; internal/dataframe/compiler/ir; internal/dataframe/compiler/lower; internal/dataframe/errors; internal/dataframe/execution; internal/dataframe/execution/chartifact; internal/dataframe/expression; internal/dataframe/publication; internal/dataframe/publication/clickhouse; internal/dataframe/published; internal/dataframe/recipe; internal/dataframe/recipe/exec; internal/dataframe/recipe/schema; internal/dataframe/semantic; internal/dataframe/spec; internal/dataframe/unit; internal/dataset; internal/explorer; internal/explorer/artifactfs; internal/explorer/authoringv2; internal/explorer/capability; internal/explorer/compilation; internal/explorer/lifecycle; internal/explorer/tableshapecap; internal/fhir/schema; internal/ingest; internal/projectid; internal/store/clickhouse | .; cmd/arango-fhir-server | broad surface (50 production files); inspect cohesion | +| `internal/source` | domain/library | Define the backend-neutral logical source identity, payload, and authorization-scope contracts shared by ingestion and store read/write paths, free of any storage or driver detail. | P: doc.go; resource.go; T: ownership_contract_test.go; Other: — | 80/403 | 2: Resource; ResourceScope | — | internal/backends/postgres/internal/store; internal/ingest | small shared semantic contract; compare direct callers before inlining | | `internal/store/clickhouse` | storage adapter | Provide the low-level typed ClickHouse connection and database-management boundary. | P: client.go; T: client_test.go; integration_test.go; Other: — | 676/712 | 15: AddColumn; Client; Close; Column; CreateTable; DropColumns; DropTable; EnsureDatabase; InsertRows; New; Options; Ping; … (+3) | — | internal/acceptance; internal/dataframe/execution/chartifact; internal/dataframe/publication/clickhouse; internal/server | storage adapter; preserve interface and persisted-data contracts | -| `scripts` | domain/library | Hold repository developer, verification, migration, and release utilities that are not part of the Go module runtime packages. | P: check_fhir_semantic_resource_types.go; local-preview-index-migration.go; T: check_fhir_semantic_resource_types_test.go; local-preview-index-migration_test.go; Other: README.md; package-lock.json; package.json | 769/532 | 0: — | internal/fhir/schema | — | no in-repo production importer; verify runtime/tooling reachability; runtime/build signals present; static importer counts are insufficient for deletion | +| `scripts` | domain/library | Hold repository developer, verification, migration, and release utilities that are not part of the Go module runtime packages. | P: check_fhir_semantic_resource_types.go; local-preview-index-migration.go; T: check_fhir_semantic_resource_types_test.go; local-preview-index-migration_test.go; Other: README.md; package-lock.json; package.json; pg-exec-03-skipset-capture.json | 769/532 | 0: — | internal/fhir/schema | — | no in-repo production importer; verify runtime/tooling reachability; runtime/build signals present; static importer counts are insufficient for deletion | +| `scripts/check-boundaries` | domain/library | Run the authoritative package-boundary policy over the repository and prove it can go RED: the --self-test negative fixtures plant forbidden imports, removed facades, re-exports, cycle leaks, and vacuous scans, each failing closed, with removal converging to green (arch-backend-09). | P: main.go; T: —; Other: — | 435/0 | 0: — | scripts/packagepolicy | — | no in-repo production importer; verify runtime/tooling reachability | +| `scripts/packagepolicy` | domain/library | Hold the single authoritative parsed-import ownership policy table, scanner, identifier-absence and no-re-export checks shared by the check-boundaries CLI, the shell entrypoint, and the compiler guard tests, separating import findings from verification failures and refusing vacuous scans. | P: policy.go; rules.go; T: policy_test.go; Other: old_concrete_paths.txt | 1,055/318 | 24: AbsenceRule; ByLabel; CheckAbsences; CheckNoReExport; Clean; ErrorKind; FindRepoRoot; Finding; Forbidden; HasImportFindings; ImportFindings; NoReExportRule; … (+12) | — | scripts/check-boundaries | one production importer (scripts/check-boundaries); inspect ownership/inline potential; runtime/build signals present; static importer counts are insufficient for deletion | ## Regeneration and validation diff --git a/docs/SEMANTIC_OPERATIONS_BACKEND_DESIGN.md b/docs/SEMANTIC_OPERATIONS_BACKEND_DESIGN.md index 9ba0dc0ba..9393afc3e 100644 --- a/docs/SEMANTIC_OPERATIONS_BACKEND_DESIGN.md +++ b/docs/SEMANTIC_OPERATIONS_BACKEND_DESIGN.md @@ -314,7 +314,7 @@ The low-cardinality full-population query and the proposed semantic relation hav ### Source anchors -- [Semantic observation contract](../internal/catalog/types.go), [inventory identity and storage types](../internal/catalog/semantic_inventory.go), and [entry materialization](../internal/catalog/arango/semantic_inventory.go). +- [Semantic observation contract](../internal/catalog/types.go), [inventory identity and storage types](../internal/catalog/semantic_inventory.go), and [entry materialization](../internal/backends/arango/internal/catalog/semantic_inventory.go). - [Generated semantic extraction](../internal/catalog/semantic_schema.go), [correlated binding validation](../internal/fhir/schema/correlated.go), and [Quantity identity paths](../internal/fhir/schema/quantity.go). - [Construction recipe and stable column references](../internal/dataframe/recipe/construction.go), [stage inference](../internal/dataframe/compiler/lower/recipe_construction.go), and [compilation receipt stages](../internal/explorer/compilation_receipt.go). - [Coded Group lowering](../internal/dataframe/compiler/lower/recipe_construction_coded_group.go), [Coded Pivot authorization](../internal/explorer/lifecycle/construction_coded_pivot.go), and [owner-correlated Pivot rendering](../internal/dataframe/compiler/render/aql/table_reshape.go). diff --git a/docs/architecture/integrated-runtime.mmd b/docs/architecture/integrated-runtime.mmd new file mode 100644 index 000000000..67b80bda9 --- /dev/null +++ b/docs/architecture/integrated-runtime.mmd @@ -0,0 +1,51 @@ +flowchart TD + subgraph Dispatch["run() -- backend selection only (server.go)"] + CFG[Config validated + normalized] --> SEL{server.backend} + SEL -->|arango| AR[runArangoRuntime] + SEL -->|postgres: six-series ladder| PGR[runPostgreSQLFullRuntime] + SEL -->|postgres: other shapes| PGF[runPostgreSQLFirstQueryRoute] + PGF -- "consumability gate pre-dial:
not exactly source+dataset_release pair" --> REF[ErrPostgreSQLRuntimeNotWired
zero opens, never falls through] + end + + subgraph Factories["startup-only open seams (compose.go, composition-private)"] + OA["openArangoBackend = arangofactory.Open"] + OP["openPostgresBackend = backendspostgres.Open"] + OC["newClickHouseClient = clickhousestore.New"] + end + + AR --> OA + PGR -- "assignment + validate pre-dial" --> OP + PGF -- "assignment + validate pre-dial" --> OP + + subgraph Plane["ClickHouse publication plane (independent lane)"] + NPP[newPublicationPlane
ClickHouse client + private artifacts +
materialization reader + bundle target] + end + PGR -- "server.clickhouse.enabled" --> NPP --> OC + AR -- "server.clickhouse.enabled" --> NPP + + subgraph Shared["shared builders (compose.go)"] + HOOKS[newReleaseHooks
publication verification + release CAS] + LIFE[buildLifecycleConfig
receipt-execution closures + typed lane ports] + ROUTES[wireExplorerRoutes
registerRoutes with typed routeDependencies
+ explicit routeScopePolicy] + HTTP[httpapi.NewHTTPServer] + end + AR --> HOOKS & LIFE & ROUTES & HTTP + PGR --> HOOKS & LIFE & ROUTES & HTTP + PGF --> HTTP + + subgraph RowPorts["row ports with closed-dispatch tag guards (row_sources.go)"] + AQLC["selectedAQLAdapter + withConfiguredAQLRows
serves AQL-tagged commands only"] + PGC["configuredPostgreSQLSourceRows
serves PostgreSQL-tagged commands only"] + end + OA --> AQLC + OP --> PGC + + subgraph Serve["one listen/shutdown loop (serveRuntime, compose.go)"] + SR[serveRuntime: Listen goroutine, ctx shutdown,
owns the runtime closer, logs close errors] + end + AR -- "cleanups stack -> closer" --> SR + PGR -- "cleanups stack -> closer" --> SR + PGF -- "cleanups stack -> closer" --> SR + + CONTRACT["TestBackendCompositionContract
(backend_composition_contract_test.go):
each selection opens exactly its own factories, once"] + CONTRACT -. spies .-> Factories diff --git a/docs/ui-contract-inventory/additional-mounted.fragment.json b/docs/ui-contract-inventory/additional-mounted.fragment.json index 6b6cf0882..caf670f12 100644 --- a/docs/ui-contract-inventory/additional-mounted.fragment.json +++ b/docs/ui-contract-inventory/additional-mounted.fragment.json @@ -20,15 +20,15 @@ "variant": "derive and save reusable revision from a configured column" }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4839", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4845", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4827", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4833", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/InterpretationPanel.tsx:101", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/InterpretationPanel.tsx:145", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/InterpretationPanel.tsx:153", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/InterpretationPanel.tsx:241", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/InterpretationPanel.tsx:245", - "ui/packages/loom-ui/src/api.ts:1766", - "ui/packages/loom-ui/src/api.ts:1768" + "ui/packages/loom-ui/src/api.ts:1755", + "ui/packages/loom-ui/src/api.ts:1757" ], "request": { "constructor": "InterpretationPanel.createRevision -> LoomClient.createInterpretationRevisionFromColumn", @@ -36,8 +36,8 @@ "apply": "The library revision is saved by this request; it does not mutate the configured feature. A later review/apply is a separate operation.", "endpointEvidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/InterpretationPanel.tsx:153", - "ui/packages/loom-ui/src/api.ts:1766", - "ui/packages/loom-ui/src/api.ts:1779" + "ui/packages/loom-ui/src/api.ts:1755", + "ui/packages/loom-ui/src/api.ts:1768" ] }, "gates": [ @@ -192,15 +192,15 @@ "variant": "preview an exact applicable revision for one column, then apply its candidate receipt" }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4844", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4857", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4832", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4845", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/InterpretationPanel.tsx:122", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/InterpretationPanel.tsx:126", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/InterpretationPanel.tsx:138", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/InterpretationPanel.tsx:177", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/InterpretationPanel.tsx:180", - "ui/packages/loom-ui/src/api.ts:1781", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/api.ts:1770", + "ui/packages/loom-ui/src/api.ts:1446" ], "request": { "previewConstructor": "InterpretationPanel.reviewRevision -> LoomClient.previewInterpretationCandidate", @@ -211,9 +211,9 @@ "endpointEvidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/InterpretationPanel.tsx:122", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/InterpretationPanel.tsx:126", - "ui/packages/loom-ui/src/api.ts:1781", + "ui/packages/loom-ui/src/api.ts:1770", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/InterpretationPanel.tsx:179", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1450" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1449" ] }, "gates": [ @@ -345,15 +345,15 @@ "variant": "attach a selection revision to the current output through one compiler-returned route choice, or replace its attached revision" }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4318", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4331", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4306", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4319", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationPanel.tsx:81", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationPanel.tsx:83", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationPanel.tsx:188", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationPanel.tsx:248", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationPanel.tsx:270", - "ui/packages/loom-ui/src/api.ts:1599", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/api.ts:1588", + "ui/packages/loom-ui/src/api.ts:1446" ], "request": { "routeSearchConstructor": "PopulationPanel routeQuery -> LoomClient.searchPopulationRoutes", @@ -363,9 +363,9 @@ "applyEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands", "endpointEvidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationPanel.tsx:83", - "ui/packages/loom-ui/src/api.ts:1599", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4331", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/api.ts:1588", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4319", + "ui/packages/loom-ui/src/api.ts:1446" ] }, "gates": [ @@ -499,16 +499,16 @@ }, "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationPanel.tsx:236", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4337", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4343", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4325", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4331", + "ui/packages/loom-ui/src/api.ts:1446" ], "request": { "constructor": "PopulationPanel.onClear -> BuilderWorkspace.applyCommands([CLEAR_TABLE_POPULATION])", "endpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands", "endpointEvidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4343", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4331", + "ui/packages/loom-ui/src/api.ts:1446" ] }, "gates": [ @@ -538,7 +538,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4344" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4332" ] } ], @@ -580,7 +580,7 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationPanel.tsx:167", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationPanel.tsx:173", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationPanel.tsx:176", - "ui/packages/loom-ui/src/api.ts:1699" + "ui/packages/loom-ui/src/api.ts:1688" ], "request": { "constructor": "PopulationPanel.checkCoverage -> usePopulationMappingMutation -> LoomClient.populationMapping", @@ -588,7 +588,7 @@ "requestLimit": 100, "endpointEvidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationPanel.tsx:167", - "ui/packages/loom-ui/src/api.ts:1699" + "ui/packages/loom-ui/src/api.ts:1688" ] }, "gates": [ @@ -672,13 +672,13 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationPanel.tsx:298", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationPanel.tsx:303", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationPanel.tsx:305", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1936", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1948", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1954", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1961", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1980", - "ui/packages/loom-ui/src/api.ts:1806", - "ui/packages/loom-ui/src/api.ts:1599" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1935", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1947", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1953", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1960", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1979", + "ui/packages/loom-ui/src/api.ts:1795", + "ui/packages/loom-ui/src/api.ts:1588" ], "request": { "selectionVariantConstructor": "BuilderWorkspace.excludePopulationMember -> LoomClient.createSelection", @@ -688,12 +688,12 @@ "applyConstructor": "BuilderWorkspace.excludePopulationMember -> applyCommands([SET_TABLE_POPULATION])", "applyEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands", "endpointEvidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1948", - "ui/packages/loom-ui/src/api.ts:1806", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1961", - "ui/packages/loom-ui/src/api.ts:1599", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1980", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1947", + "ui/packages/loom-ui/src/api.ts:1795", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1960", + "ui/packages/loom-ui/src/api.ts:1588", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1979", + "ui/packages/loom-ui/src/api.ts:1446" ] }, "gates": [ @@ -728,7 +728,7 @@ }, "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationPanel.tsx:303", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1937" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1936" ] }, { @@ -746,7 +746,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1954" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1953" ] } ], @@ -755,9 +755,9 @@ "status": "observed-ui", "statement": "After creating the exclusion variant, the client searches until it finds the same route; it reports failure rather than attaching if no matching route is returned.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1958", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1975", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1978" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1957", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1974", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1977" ] } ], @@ -766,9 +766,9 @@ "status": "observed-ui", "statement": "The source selection is immutable: exclusion creates a new revision from the attached selection with the chosen member excluded, then attaches that new revision.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1948", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1954", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1980" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1947", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1953", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1979" ] } ], @@ -806,8 +806,8 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationMemberRemoval.tsx:217", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationMemberRemoval.tsx:229", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationMemberRemoval.tsx:313", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4349", - "ui/packages/loom-ui/src/api.ts:1823" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4337", + "ui/packages/loom-ui/src/api.ts:1812" ], "request": { "memberConstructor": "PopulationMemberRemoval.memberQuery -> LoomClient.getSelection", @@ -821,9 +821,9 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationMemberRemoval.tsx:162", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationMemberRemoval.tsx:183", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PopulationMemberRemoval.tsx:193", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4349", - "ui/packages/loom-ui/src/api.ts:1823", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4337", + "ui/packages/loom-ui/src/api.ts:1812", + "ui/packages/loom-ui/src/api.ts:1446" ] }, "gates": [ @@ -952,11 +952,11 @@ ] }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3746", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3749", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3755", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3759", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3762", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3734", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3737", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3743", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3747", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3750", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:302", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:328", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:337", @@ -975,11 +975,11 @@ "applyEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands", "endpointEvidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:269", - "ui/packages/loom-ui/src/api.ts:1540", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3759", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3762", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3766", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/api.ts:1529", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3747", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3750", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3754", + "ui/packages/loom-ui/src/api.ts:1446" ] }, "gates": [ @@ -988,8 +988,8 @@ "when": "table has frames and Add Columns is in coded view", "effect": "The panel mounts to manage saved coded sources.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3746", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3747" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3734", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3735" ] }, { @@ -1020,7 +1020,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3753", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3741", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:394" ] }, @@ -1091,8 +1091,8 @@ "status": "observed-ui", "statement": "Creation of a new frame source is not reachable from this mounted instance: BuilderWorkspace passes allowNewSource=false, while this instance is mounted only when saved frames exist. The onSet callback is supplied but no current-panel control invokes it.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3746", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3755", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3734", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3743", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:386", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:302" ] @@ -1104,8 +1104,8 @@ "statement": "The active replacement route is bound to the currently edited frame ID; this Builder mount does not expose the new-source SET_FRAME_SOURCE path.", "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:328", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3755", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3759" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3743", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3747" ] } ], @@ -1137,14 +1137,14 @@ ] }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4464", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4733", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4452", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4721", "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:145", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:43", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:516", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:649", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:667", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:485", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:513", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:646", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:664", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:482", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:234", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:276", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:343", @@ -1154,10 +1154,10 @@ "constructor": "PreviewTable/ColumnSelector presentation callbacks -> BuilderWorkspace.applyPresentationChanges -> previewPresentationCommand", "endpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands", "endpointEvidence": [ + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2967", "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2969", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2971", "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:145", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/api.ts:1446" ] }, "gates": [ @@ -1190,8 +1190,8 @@ "trimBeforeSubmit": true }, "evidence": [ + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:668", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:671", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:674", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:98", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:100" ] @@ -1212,7 +1212,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:649", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:646", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:276" ] }, @@ -1254,8 +1254,8 @@ "maximum": null }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:485", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:500", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:482", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:497", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:739", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:754" ] @@ -1266,8 +1266,8 @@ "status": "observed-ui", "statement": "Column names are trimmed and empty names are rejected by restoring the prior label; table visibility/order applies to authored columns and constructed outputs through distinct UPDATE commands.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:671", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:675", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:668", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:672", "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:145", "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:157" ] @@ -1316,18 +1316,18 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/FeaturePolicyEditor.tsx:207", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/FeaturePolicyEditor.tsx:222", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/FeaturePolicyEditor.tsx:231", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4721", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4725", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4709", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4713", + "ui/packages/loom-ui/src/api.ts:1446" ], "request": { "constructor": "RelatedFeatureCreator.onAdd -> ColumnSelector.onAddSource -> BuilderWorkspace.applyCommands([ADD_COLUMN_SOURCE])", "endpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands", "endpointEvidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/FeaturePolicyEditor.tsx:223", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4721", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4725", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4709", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4713", + "ui/packages/loom-ui/src/api.ts:1446" ] }, "gates": [ @@ -1390,7 +1390,7 @@ "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/FeaturePolicyEditor.tsx:223", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/FeaturePolicyEditor.tsx:237", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4721" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4709" ] } ], @@ -1420,21 +1420,21 @@ "variant": "change output value cardinality policy for an explicit-group row value binding" }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:583", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:590", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:580", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:587", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:697", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:700", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:703", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:715", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4478", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4481" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:712", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4466", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4469" ], "request": { "constructor": "PreviewTable.onRowValuePolicyChange -> BuilderWorkspace.applyCommands([UPDATE_COLUMN_ROW_VALUE_POLICY])", "endpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands", "endpointEvidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4478", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4481", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4466", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4469", + "ui/packages/loom-ui/src/api.ts:1446" ] }, "gates": [ @@ -1443,10 +1443,10 @@ "when": "column has a columnId, is a base-occurrence field, supports the current transformation and is bound in GROUPS with EXPLICIT row source", "effect": "Only then does PreviewTable render the cohort member policy control.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:578", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:583", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:587", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:590" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:575", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:580", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:584", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:587" ] } ], @@ -1467,9 +1467,9 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:703", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:715", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:716" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:700", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:712", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:713" ] } ], @@ -1478,9 +1478,9 @@ "status": "observed-ui", "statement": "This output-side setting is separate from ADD_COLUMNS construction-choice rowValuePolicy; it edits one existing explicit-group row-value binding by authored physical column.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:583", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:590", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4481" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:580", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:587", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4469" ] } ], @@ -1515,10 +1515,10 @@ ] }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4767", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4771", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4778", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4795", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4755", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4759", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4766", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4783", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:363", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/FeaturePolicyEditor.tsx:1090", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/FeaturePolicyEditor.tsx:1117", @@ -1526,15 +1526,15 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/FeaturePolicyEditor.tsx:1304", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/FeaturePolicyEditor.tsx:1385", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/FeaturePolicyEditor.tsx:1414", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/api.ts:1446" ], "request": { "constructor": "ColumnSelector FeaturePolicyEditor callbacks -> BuilderWorkspace.applyCommands", "endpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands", "endpointEvidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4767", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4795", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4755", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4783", + "ui/packages/loom-ui/src/api.ts:1446" ] }, "gates": [ @@ -1767,8 +1767,8 @@ "status": "observed-ui", "statement": "Source and policy edits submit through explicit Builder commands and are scoped to the selected output and authored column; client capability filtering is not a backend conformance promise.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4767", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4795" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4755", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4783" ] } ], @@ -1799,9 +1799,9 @@ "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:268", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:588", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4749", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4750", - "ui/packages/loom-ui/src/api.ts:1549", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4737", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4738", + "ui/packages/loom-ui/src/api.ts:1538", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSourceInspector.tsx:25", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSourceInspector.tsx:46" ], @@ -1809,9 +1809,9 @@ "constructor": "ColumnSelector.inspectColumn -> BuilderWorkspace.onInspectSource -> LoomClient.inspectColumnSource", "endpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/column-source", "endpointEvidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4749", - "ui/packages/loom-ui/src/api.ts:1549", - "ui/packages/loom-ui/src/api.ts:1550" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4737", + "ui/packages/loom-ui/src/api.ts:1538", + "ui/packages/loom-ui/src/api.ts:1539" ] }, "gates": [ @@ -1843,7 +1843,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4755", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4743", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:593" ] } @@ -1855,8 +1855,8 @@ "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:595", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:596", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4760", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4762" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4748", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4750" ] } ], @@ -1889,27 +1889,27 @@ ] }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4489", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4490", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4497", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:889", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:895", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:903", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:934", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:939", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:1001", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:1027", - "ui/packages/loom-ui/src/api.ts:1710" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4477", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4478", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4485", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:886", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:892", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:900", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:931", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:936", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:998", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:1024", + "ui/packages/loom-ui/src/api.ts:1699" ], "request": { "constructor": "PreviewTable.loadRowLineage -> BuilderWorkspace.onRowLineage -> LoomClient.rowLineage", "endpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/row-lineage", "requestLimit": 25, "endpointEvidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4489", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4497", - "ui/packages/loom-ui/src/api.ts:1710", - "ui/packages/loom-ui/src/api.ts:1711" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4477", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4485", + "ui/packages/loom-ui/src/api.ts:1699", + "ui/packages/loom-ui/src/api.ts:1700" ] }, "gates": [ @@ -1918,9 +1918,9 @@ "when": "preview row has COMPOSITE source and preview.rowLineageCapability.status=AVAILABLE", "effect": "Opening its identity panel starts contributor lookup; later pages use returned nextOffset.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:903", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:1001", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:1027" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:900", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:998", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:1024" ] } ], @@ -1940,9 +1940,9 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:871", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:889", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:895" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:868", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:886", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:892" ] }, { @@ -1966,8 +1966,8 @@ "maximum": null }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:1027", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:1029" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:1024", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:1026" ] }, { @@ -1985,7 +1985,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4497" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4485" ] } ], @@ -1994,9 +1994,9 @@ "status": "observed-ui", "statement": "A row without a nonempty identity has no identity-inspection button; unsupported or incomplete lineage is reported as unavailable/incomplete rather than complete.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:889", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:911", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:1005" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:886", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:908", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:1002" ] } ], @@ -2005,11 +2005,11 @@ "status": "observed-ui", "statement": "Lineage requests are receipt/output/row bound and fetch 25 contributors per page; owner-record cell inspection reads the preview value locally without a separate lineage request.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4490", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4495", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4497", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:933", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:939" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4478", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4483", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4485", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:930", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:936" ] } ], @@ -2038,27 +2038,27 @@ "variant": "summarize saved row/column definitions, compile shapes, reductions, preview evidence and diagnostic issues; navigate to responsible setting or close" }, "evidence": [ + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4111", "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4123", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4135", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4136", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3151", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3155", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4124", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3149", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3153", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/DatasetReviewPanel.tsx:24", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/DatasetReviewPanel.tsx:334", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/DatasetReviewPanel.tsx:496", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/DatasetReviewPanel.tsx:557", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/DatasetReviewPanel.tsx:623", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1271" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1270" ], "request": { "panelRequest": "DatasetReviewPanel itself makes no API or authoring command request. Opening Review may call lifecycle-owned reconcileCurrent when a current receipt is missing or reconciliation is unresolved.", "navigationConstructor": "DatasetReviewPanel.onFocus -> BuilderWorkspace.focusDatasetReviewTarget", "navigationRequest": "None; local review visibility, feature mode, selected table/occurrence and focus state changes.", "endpointEvidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3151", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3155", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3149", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3153", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/DatasetReviewPanel.tsx:557", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1271" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1270" ] }, "gates": [ @@ -2067,9 +2067,9 @@ "when": "reviewOpen is true", "effect": "The panel is mounted. Receipt and preview are passed only when the receipt includes every current output ID.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4123", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4127", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4130" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4111", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4115", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4118" ] } ], @@ -2184,9 +2184,9 @@ "statement": "Focus actions route the user into the responsible table, row setup, or column editor without issuing an authoring write from the review panel itself.", "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/DatasetReviewPanel.tsx:553", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1271", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1279", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1282" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1270", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1278", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1281" ] } ], @@ -2219,17 +2219,17 @@ ] }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4591", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4579", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:85", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:395", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:481", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:485", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:496", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:500", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:529", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:537", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:405", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:440" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:393", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:479", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:483", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:494", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:498", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:527", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:535", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:403", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:438" ], "request": { "viewState": "showOrphans and isExpanded are local React state; ReactFlow node/edge inspection updates local inspection and may dispatch selectOccurrence. Route/root writes are mapped separately to existing canonical contracts.", @@ -2237,8 +2237,8 @@ "endpointEvidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:85", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:86", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:529", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:544" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:527", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:542" ] }, "gates": [ @@ -2247,10 +2247,10 @@ "when": "BuilderWorkspace selects GuidedGraphWorkspace and disabled is false", "effect": "Graph node/edge inspection is enabled; route-edit buttons are separately restricted by legal route helpers and lifecycle state.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4591", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4595", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:405", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:440" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4579", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4583", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:403", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:438" ] } ], @@ -2270,8 +2270,8 @@ }, "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:86", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:481", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:485" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:479", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:483" ] }, { @@ -2289,8 +2289,8 @@ }, "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:85", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:496", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:500" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:494", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:498" ] }, { @@ -2309,8 +2309,8 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:405", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:440" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:403", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:438" ] } ], @@ -2319,10 +2319,10 @@ "status": "observed-ui", "statement": "The graph is non-draggable and non-connectable; actual route edits are invoked only through explicit callbacks and legal edge filters.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:512", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:515", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:510", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:513", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:219", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:453" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:451" ] } ], @@ -2353,19 +2353,19 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:353", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:355", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:359", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:740", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:745", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4739", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4742", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:737", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:742", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4727", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4730", + "ui/packages/loom-ui/src/api.ts:1446" ], "request": { "constructor": "ColumnSelector.onRemove or PreviewTable.onRemoveColumn -> BuilderWorkspace.applyCommands([REMOVE_COLUMN])", "endpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands", "endpointEvidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4739", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4742", - "ui/packages/loom-ui/src/api.ts:1457" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4727", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4730", + "ui/packages/loom-ui/src/api.ts:1446" ] }, "gates": [ @@ -2398,7 +2398,7 @@ }, "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:355", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4745" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4733" ] } ], @@ -2409,7 +2409,7 @@ "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:353", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:359", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4739" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4727" ] } ], @@ -2519,8 +2519,8 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:437", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:457", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:832", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4693", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4707" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4681", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4695" ], "request": { "constructor": "ColumnSelector.onAdd/onAddAll -> BuilderWorkspace.applyCommands([ADD_COLUMN...])", @@ -2633,8 +2633,8 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:95", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:121", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:209", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3765", - "ui/packages/loom-ui/src/api.ts:1495" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3753", + "ui/packages/loom-ui/src/api.ts:1484" ], "request": { "browseConstructor": "CategoryPicker.categoryQuery -> LoomClient.browseSemanticInventory", @@ -2692,11 +2692,11 @@ "chooseReshapeRows('expand',[columnId])" ], "evidence": [ + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:549", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:552", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:555", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:726", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:729", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:732", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4467" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4455" ], "request": { "constructor": "PreviewTable.onExpandColumn -> BuilderWorkspace.chooseReshapeRows('expand',[columnId])", @@ -2718,8 +2718,8 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:552", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:555" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:549", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:552" ] } ], @@ -2734,14 +2734,14 @@ "open dataset review; reconcile if current receipt/reconciliation is unresolved" ], "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:760", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:768", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:772", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4461", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2831", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2849", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3151", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3155" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:757", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:765", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:769", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4449", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2830", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2848", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3149", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3153" ], "request": { "detail": "Preview limit updates the identity consumed by useAutomaticPreview; no independent submit button. Dataset review's conditional reconciliation is lifecycle-owned." @@ -2764,9 +2764,9 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:763", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:772", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:776" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:760", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:769", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:773" ] } ], @@ -2784,29 +2784,29 @@ "REMOVE_ROUTE" ], "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4601", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4615", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4621", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4630", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4641", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4652", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4666", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:604", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:622", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:650", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:686", - "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/routeActions.ts:9", - "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/routeActions.ts:35" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4589", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4603", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4609", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4618", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4629", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4640", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4654", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:602", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:620", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:648", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:684", + "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/routeActions.ts:4", + "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/routeActions.ts:30" ], "request": { "constructor": "GuidedGraphWorkspace callbacks -> BuilderWorkspace.applyCommands", "endpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands", "routeBounds": "Only edges returned by legalOutgoingEdges or replaceableIncomingEdges are selectable; parent routeActions source owns exact bounds.", "endpointEvidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4621", - "ui/packages/loom-ui/src/api.ts:1457", - "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/routeActions.ts:9", - "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/routeActions.ts:35" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4609", + "ui/packages/loom-ui/src/api.ts:1446", + "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/routeActions.ts:4", + "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/routeActions.ts:30" ] }, "dimensions": [ @@ -2826,9 +2826,9 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4641", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4645", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:652" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4629", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4633", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:650" ] } ], @@ -2842,9 +2842,9 @@ "select replacement root node and enter row-change assessment/preview path" ], "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:448", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:453", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4614" + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:446", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:451", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4602" ], "request": { "constructor": "GuidedGraphWorkspace.onChangeBase -> BuilderWorkspace.selectRootChoice({kind:'catalog-node',nodeId})", @@ -2867,7 +2867,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4614" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4602" ] } ], diff --git a/docs/ui-contract-inventory/expression.fragment.json b/docs/ui-contract-inventory/expression.fragment.json index 7e5fd931d..8037c9167 100644 --- a/docs/ui-contract-inventory/expression.fragment.json +++ b/docs/ui-contract-inventory/expression.fragment.json @@ -20,12 +20,12 @@ "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionWorkspace.tsx:40", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionWorkspace.tsx:106", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3777", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3765", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/FilterRowsEditor.tsx:41", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:784", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:843", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:214", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:396" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:777", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:836", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:209", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:389" ], "request": { "candidateConstructor": "ConstructionOperationEditor.buildFilterCandidate", @@ -36,10 +36,10 @@ "applyEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands", "endpointEvidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:233", - "ui/packages/loom-ui/src/api.ts:1685", - "ui/packages/loom-ui/src/api.ts:1697", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1800", - "ui/packages/loom-ui/src/api.ts:1450" + "ui/packages/loom-ui/src/api.ts:1674", + "ui/packages/loom-ui/src/api.ts:1686", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1799", + "ui/packages/loom-ui/src/api.ts:1439" ] }, "gates": [ @@ -48,8 +48,8 @@ "when": "selectedStage.capabilities[FILTER].supported", "effect": "Otherwise show the server reason and emit no candidate.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:129", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:784" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:126", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:777" ] }, { @@ -57,8 +57,8 @@ "when": "condition column id exists in selected predecessor stage", "effect": "Otherwise the candidate is incomplete or a saved condition is shown as unsupported.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:350", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:405" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:343", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:398" ] } ], @@ -77,7 +77,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:843" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:836" ], "description": "The condition column comes from the stage immediately before the filter step." }, @@ -130,7 +130,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:214" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:209" ], "description": "EXISTS/MISSING are always offered; EQUALS/NOT_EQUALS/IN require a mapped value kind; CONTAINS_TEXT requires STRING; comparisons require INTEGER, DECIMAL, DATE, or DATE_TIME." }, @@ -168,9 +168,9 @@ ] }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:245", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:265", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:934" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:238", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:258", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:927" ], "description": "IN has an Add another value control with no explicit UI maximum. INTEGER requires a JavaScript safe integer; DECIMAL must be finite; DATE must match YYYY-MM-DD; DATE_TIME must parse as a valid instant." } @@ -180,7 +180,7 @@ "status": "observed-ui", "statement": "The UI serializes typed FILTER values, retains predecessor-stage columns, and appends or replaces one FILTER step.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:396" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:389" ] }, { @@ -197,27 +197,27 @@ "status": "observed-ui", "statement": "Construction capabilities may include filterSourceInput with supported, stageId, optional refusal reasonCode/reason, and choices carrying choiceId, optional columnId, fieldPath, label, logicalType, and isPopulated. A missing or empty response columnId means no live materialization; selecting a signed field submits filterSources entries with non-empty choiceId and columnId through construction-proposals.", "evidence": [ - "ui/packages/loom-ui/src/types.ts:1995-2127", - "ui/packages/loom-ui/src/types.ts:2185-2204", - "ui/packages/loom-ui/src/api.ts:1694-1698" + "ui/packages/loom-ui/src/types.ts:1991-2116", + "ui/packages/loom-ui/src/types.ts:2174-2193", + "ui/packages/loom-ui/src/api.ts:1683-1687" ] }, { "status": "observed-ui", "statement": "Signed source pickers distinguish no listed fields, no populated eligible fields, and eligible populated fields already bound elsewhere. Own bindings count as selectable only when visible and solely bound in the lane; unavailable source-lane reasons stay separate. Existing live field carriers suppress duplicate signed choices only when authoritative field identity resolves unambiguously; stale or ambiguous carriers retain the signed-choice path, and saved bindings are not rewritten. Filter also accepts server-signed public field carriers and canonicalizes their root. prefix; it never matches labels.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:981-1003", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1097-1150", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/existingFieldReuse.ts:35-64", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/sourcePickerExplanations.ts:27-56", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/SourcePickerExplanation.tsx:9-47" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:967-989", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1083-1136", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/existingFieldReuse.ts:22-44", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/sourcePickerExplanations.ts:22-46", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/SourcePickerExplanation.tsx:5-43" ] }, { "status": "observed-ui", "statement": "Rows must match every saved row condition.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:806" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:799" ] } ], @@ -263,12 +263,12 @@ "previewEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/preview", "applyEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands via APPLY_CONSTRUCTION_PROPOSAL", "endpointEvidence": [ - "ui/packages/loom-ui/src/api.ts:1564", - "ui/packages/loom-ui/src/api.ts:1587", + "ui/packages/loom-ui/src/api.ts:1553", + "ui/packages/loom-ui/src/api.ts:1576", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:233", - "ui/packages/loom-ui/src/api.ts:1685", - "ui/packages/loom-ui/src/api.ts:1697", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1800" + "ui/packages/loom-ui/src/api.ts:1674", + "ui/packages/loom-ui/src/api.ts:1686", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1799" ], "pagination": { "routeChoicesDefaultPageSize": 10, @@ -471,10 +471,10 @@ "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionWorkspace.tsx:52", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionWorkspace.tsx:109", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4432", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3801", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:627", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1151" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4420", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3789", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:620", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1137" ], "request": { "capabilitiesEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/construction-capabilities", @@ -482,10 +482,10 @@ "previewEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/preview", "applyEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands via APPLY_CONSTRUCTION_PROPOSAL", "evidence": [ - "ui/packages/loom-ui/src/api.ts:1652", - "ui/packages/loom-ui/src/api.ts:1685", - "ui/packages/loom-ui/src/api.ts:1697", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1800" + "ui/packages/loom-ui/src/api.ts:1641", + "ui/packages/loom-ui/src/api.ts:1674", + "ui/packages/loom-ui/src/api.ts:1686", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1799" ] }, "gates": [ @@ -494,7 +494,7 @@ "when": "table has at least one construction step and append-stage DERIVE capability is supported", "effect": "Calculate appears in More actions only when true.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4432", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4420", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionWorkspace.tsx:109" ] }, @@ -503,8 +503,8 @@ "when": "COLUMN operands reference integer/decimal/number columns returned for the selected stage", "effect": "Other stage columns are not selectable as operands.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:124", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1003" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:121", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:989" ] } ], @@ -523,7 +523,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1151" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1137" ] }, { @@ -542,7 +542,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1193" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1179" ] }, { @@ -566,8 +566,8 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:994", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1008" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:980", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:994" ] }, { @@ -591,8 +591,8 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:994", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1008" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:980", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:994" ] }, { @@ -607,8 +607,8 @@ "maximum": null }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:580", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:601" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:573", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:594" ], "description": "Parser accepts exactly two numeric operands and one +, -, *, or / operator; column names match [A-Za-z_][A-Za-z0-9_]*; literals support decimal/exponent syntax, finite values, and safe integers for integer notation. No formula max length is declared." }, @@ -626,7 +626,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1240" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1226" ] }, { @@ -645,7 +645,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1257" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1243" ] }, { @@ -658,8 +658,8 @@ "maximum": null }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:638", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1295" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:631", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1281" ], "description": "Must match [A-Za-z_][A-Za-z0-9_]*, be unique case-sensitively against stage names, and has no declared maximum length." } @@ -669,16 +669,16 @@ "status": "observed-ui", "statement": "Formula view does not support functions, nested expressions, parentheses, or conditionals.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:610", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1174" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:603", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1160" ] }, { "status": "observed-ui", "statement": "A derive step retains predecessor columns and adds one output column; replacing a saved step preserves its output column id.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:652", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:676" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:645", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:669" ] } ], @@ -687,28 +687,28 @@ "status": "observed-ui", "statement": "Construction capabilities may include deriveSourceInput with supported, stageId, optional refusal reasonCode/reason, and choices carrying choiceId, optional columnId, fieldPath, label, logicalType, and isPopulated. A missing or empty response columnId means no live materialization; selecting a signed field submits deriveSources entries with non-empty choiceId and columnId through construction-proposals.", "evidence": [ - "ui/packages/loom-ui/src/types.ts:1995-2127", - "ui/packages/loom-ui/src/types.ts:2185-2204", - "ui/packages/loom-ui/src/api.ts:1694-1698" + "ui/packages/loom-ui/src/types.ts:1991-2116", + "ui/packages/loom-ui/src/types.ts:2174-2193", + "ui/packages/loom-ui/src/api.ts:1683-1687" ] }, { "status": "observed-ui", "statement": "Signed source pickers distinguish no listed fields, no populated eligible fields, and eligible populated fields already bound elsewhere. Own bindings count as selectable only when visible and solely bound in the lane; unavailable source-lane reasons stay separate. Existing live field carriers suppress duplicate signed choices only when authoritative field identity resolves unambiguously; stale or ambiguous carriers retain the signed-choice path, and saved bindings are not rewritten.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1248-1263", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1440-1475", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/existingFieldReuse.ts:35-64", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/sourcePickerExplanations.ts:27-56", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/SourcePickerExplanation.tsx:9-47" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1234-1249", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:1422-1454", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/existingFieldReuse.ts:22-44", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/sourcePickerExplanations.ts:22-46", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/SourcePickerExplanation.tsx:5-43" ] }, { "status": "observed-ui", "statement": "Operands are numeric stage columns or finite numeric literals; missing-input and division-by-zero policies are explicitly submitted.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:627", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:653" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:620", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx:646" ] } ], @@ -736,7 +736,7 @@ "variant": "derived numeric column attached to NONE, GROUPED_PIVOT, or UNPIVOT shape" }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4865", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4853", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/TableShapeSettingsPanel.tsx:343", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/TableShapeSettingsPanel.tsx:386", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/TableShapeEditor.tsx:132", @@ -750,10 +750,10 @@ "applyConstructor": "BuilderWorkspace applies APPLY_TABLE_SHAPE_PROPOSAL", "applyEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands", "evidence": [ - "ui/packages/loom-ui/src/api.ts:1621", - "ui/packages/loom-ui/src/api.ts:1640", - "ui/packages/loom-ui/src/api.ts:1646", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4883" + "ui/packages/loom-ui/src/api.ts:1610", + "ui/packages/loom-ui/src/api.ts:1629", + "ui/packages/loom-ui/src/api.ts:1635", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4871" ] }, "gates": [ @@ -762,7 +762,7 @@ "when": "table.document.construction is absent", "effect": "Table shape settings are mounted only for a non-construction table.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4865" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4853" ] }, { @@ -991,12 +991,12 @@ "variant": "local source scope selection for catalog browsing" }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1430", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1446", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1483", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1509", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1518", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1527", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1429", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1445", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1482", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1508", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1517", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1526", "ui/packages/loom-ui/src/features/ExplorerBuilder/catalogItems.ts:189", "ui/packages/loom-ui/src/features/ExplorerBuilder/catalogItems.ts:226", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1045" @@ -1009,8 +1009,8 @@ "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1049", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1509", - "ui/packages/loom-ui/src/api.ts:1495", - "ui/packages/loom-ui/src/api.ts:1555" + "ui/packages/loom-ui/src/api.ts:1484", + "ui/packages/loom-ui/src/api.ts:1544" ], "pagination": { "semanticInventoryPageSize": 50, @@ -1030,8 +1030,8 @@ "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/catalogItems.ts:194", "ui/packages/loom-ui/src/features/ExplorerBuilder/catalogItems.ts:200", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1430", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1443" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1429", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1442" ] }, { @@ -1039,8 +1039,8 @@ "when": "selected source kind is EXACT_RELATED", "effect": "Selecting it switches the catalog to Other fields; switching to Coded values returns source selection to ROOT.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1456", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1527" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1455", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1526" ] } ], @@ -1068,7 +1068,7 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/catalogItems.ts:189", "ui/packages/loom-ui/src/features/ExplorerBuilder/catalogItems.ts:194", "ui/packages/loom-ui/src/features/ExplorerBuilder/catalogItems.ts:226", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1430" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1429" ] }, { @@ -1094,7 +1094,7 @@ "status": "observed-ui", "statement": "Changing browse source changes catalog scope only; it does not itself add or mutate a table column.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1527", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1526", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1049" ] }, @@ -1104,7 +1104,7 @@ "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/catalogItems.ts:194", "ui/packages/loom-ui/src/features/ExplorerBuilder/catalogItems.ts:200", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1495" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1494" ] } ], @@ -1113,7 +1113,7 @@ "status": "observed-ui", "statement": "Catalog browse is bound to the current table, catalog snapshot, selected resource scope, and optional selected occurrence context.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1446", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1445", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1050" ] } @@ -1144,9 +1144,9 @@ "variant": "server-supported semantic construction choice" }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3637", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3667", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3715", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3625", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3655", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3703", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1045", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1505", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1735" @@ -1157,11 +1157,11 @@ "proposalEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/construction-choice-proposals", "applyEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands via APPLY_CONSTRUCTION_CHOICE", "evidence": [ - "ui/packages/loom-ui/src/api.ts:1495", - "ui/packages/loom-ui/src/api.ts:1555", - "ui/packages/loom-ui/src/api.ts:1691", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1876", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1906" + "ui/packages/loom-ui/src/api.ts:1484", + "ui/packages/loom-ui/src/api.ts:1544", + "ui/packages/loom-ui/src/api.ts:1680", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1875", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1905" ], "pagination": { "semanticInventoryPageSize": 50, @@ -1264,7 +1264,7 @@ "statement": "The catalog submits returned choice ids and selected supported forms; it does not infer a semantic form from the item's label.", "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1505", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1885" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1884" ] } ], @@ -1273,8 +1273,8 @@ "status": "observed-ui", "statement": "The proposal preview must return rows for the selected output before APPLY_CONSTRUCTION_CHOICE commands are sent.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1903", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1906" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1902", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1905" ] } ], @@ -1304,11 +1304,11 @@ "variant": "server-supported field construction choice" }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3684", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3715", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3672", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3703", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1163", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1505", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1876" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1875" ], "request": { "browseEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/semantic-inventory", @@ -1316,10 +1316,10 @@ "proposalEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/construction-choice-proposals", "applyEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands via APPLY_CONSTRUCTION_CHOICE", "evidence": [ - "ui/packages/loom-ui/src/api.ts:1495", - "ui/packages/loom-ui/src/api.ts:1555", - "ui/packages/loom-ui/src/api.ts:1691", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1906" + "ui/packages/loom-ui/src/api.ts:1484", + "ui/packages/loom-ui/src/api.ts:1544", + "ui/packages/loom-ui/src/api.ts:1680", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1905" ], "pagination": { "semanticInventoryPageSize": 50, @@ -1425,10 +1425,10 @@ "status": "observed-ui", "statement": "The Add selected flow sends server-returned choice ids/forms and validates preview output identity before applying the construction-choice command.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1885", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1893", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1903", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1906" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1884", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1892", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1902", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1905" ] } ], @@ -1438,7 +1438,7 @@ "statement": "Only selected fields are added; source projection remains gated by the capability response for the current stage.", "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:2027", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1876" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1875" ] } ], @@ -1470,12 +1470,12 @@ ] }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1544", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1564", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1580", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3698", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3901", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3934", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1543", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1563", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1579", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3686", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3889", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3922", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/RelatedFieldEditor.tsx:134", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/RelatedFieldEditor.tsx:207" ], @@ -1486,12 +1486,12 @@ "previewEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/preview", "applyEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands via APPLY_CONSTRUCTION_PROPOSAL", "evidence": [ - "ui/packages/loom-ui/src/api.ts:1555", - "ui/packages/loom-ui/src/api.ts:1576", + "ui/packages/loom-ui/src/api.ts:1544", + "ui/packages/loom-ui/src/api.ts:1565", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/RelatedFieldEditor.tsx:140", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1854", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1861", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1800" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1853", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1860", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1799" ], "pagination": { "activeRelatedFieldSearchDefaultPageSize": 20, @@ -1511,9 +1511,9 @@ "when": "RELATED_SOURCE capability supported, stage has rowIdentityColumn, editor is at current append stage, and returned source field is optional_one or required_one", "effect": "Unsupported and repeated source fields are not submitted; related catalog choices are suppressed when capability is false.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1557", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1564", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1580", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1556", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1563", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1579", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:313" ] }, @@ -1547,8 +1547,8 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1576", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1588", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1575", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1587", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:313" ] }, @@ -1570,8 +1570,8 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1572", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1580" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1571", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1579" ] }, { @@ -1618,9 +1618,9 @@ "status": "observed-ui", "statement": "Routed related source batching accepts only fields on the same exact route; its supported forms are ALL, COUNT, and PRESENCE; repeated source fields are rejected.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1842", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1847", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1580" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1841", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1846", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1579" ] }, { @@ -1630,7 +1630,7 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/RelatedFieldEditor.tsx:47", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/RelatedFieldEditor.tsx:51", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/RelatedFieldEditor.tsx:61", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3698" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3686" ] } ], @@ -1639,7 +1639,7 @@ "status": "observed-ui", "statement": "The routed RELATED_SOURCE operation uses the selected row identity anchor and exact returned route; related field edits are bound to the active related record at that stage.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1621", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1620", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/RelatedFieldEditor.tsx:47", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/RelatedFieldEditor.tsx:61" ] @@ -1668,20 +1668,20 @@ "variant": "rowValuePolicy on supported ALL construction choices" }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3433", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3436", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3652", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3655", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3428", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3431", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3640", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3643", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1634", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1885" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1884" ], "request": { "proposalEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/construction-choice-proposals", "applyEndpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands via APPLY_CONSTRUCTION_CHOICE", "evidence": [ - "ui/packages/loom-ui/src/api.ts:1691", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1885", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1906" + "ui/packages/loom-ui/src/api.ts:1680", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1884", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1905" ] }, "gates": [ @@ -1690,9 +1690,9 @@ "when": "final source stage reports ROW_VALUES supported", "effect": "Only then show the policy control and pass rowValuePolicy into eligible ALL construction choices.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3433", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3652", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1885" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3428", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3640", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1884" ] } ], @@ -1714,7 +1714,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3655", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3643", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1634" ], "description": "ALL means keep all distinct values; ONE means require one distinct value." @@ -1725,7 +1725,7 @@ "status": "observed-ui", "statement": "Missing values stay empty; ONE stops if contributing records disagree.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3662" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3650" ] } ], @@ -1735,7 +1735,7 @@ "statement": "The selected row-value policy is serialized only for ALL forms and only when the server capability is supported.", "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1634", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1885" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1884" ] } ], @@ -1763,12 +1763,12 @@ "variant": "configure a filter control for an output column" }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4678", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4666", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:295", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:305", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ColumnSelector.tsx:437", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4693", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4733" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4681", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4721" ], "request": { "existingColumnConstructor": "ColumnSelector configured column onChange \u2192 BuilderWorkspace.applyPresentationChanges \u2192 previewPresentationCommand emits UPDATE_COLUMN", @@ -1776,9 +1776,9 @@ "endpoint": "POST /api/v1/projects/{project}/explorers/{explorerId}/authoring/v2/commands", "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:145", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2969", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4693", - "ui/packages/loom-ui/src/api.ts:1450" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2967", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4681", + "ui/packages/loom-ui/src/api.ts:1439" ] }, "gates": [ diff --git a/docs/ui-contract-inventory/lifecycle.fragment.json b/docs/ui-contract-inventory/lifecycle.fragment.json index 2ba6f8827..b5a493a2b 100644 --- a/docs/ui-contract-inventory/lifecycle.fragment.json +++ b/docs/ui-contract-inventory/lifecycle.fragment.json @@ -55,13 +55,13 @@ "ui/apps/demo/src/main.tsx:8", "ui/apps/demo/src/main.tsx:37", "ui/apps/demo/src/main.tsx:74", - "ui/apps/demo/src/demo-route.mjs:11", + "ui/apps/demo/src/demo-route.mjs:2", "ui/packages/loom-ui/src/Builder.tsx:34", - "ui/packages/loom-ui/src/Builder.tsx:54", + "ui/packages/loom-ui/src/Builder.tsx:51", "ui/apps/demo/src/main.tsx:69", "ui/apps/demo/src/main.tsx:75", - "ui/packages/loom-ui/src/Builder.tsx:85", - "ui/packages/loom-ui/src/api.ts:1813" + "ui/packages/loom-ui/src/Builder.tsx:82", + "ui/packages/loom-ui/src/api.ts:1802" ], "constraints": [ { @@ -161,7 +161,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/selections/{selectionRevision}?limit=1[&auth_resource_path=...]", "actionId": "selectionRevisionId.present", "payloadShape": "no body; query {limit:1,auth_resource_path?}", - "constructor": "ui/packages/loom-ui/src/Builder.tsx:85 ownedClient.getSelection; ui/packages/loom-ui/src/api.ts:1813", + "constructor": "ui/packages/loom-ui/src/Builder.tsx:82 ownedClient.getSelection; ui/packages/loom-ui/src/api.ts:1802", "condition": "selectionRevisionId is present" } ] @@ -209,13 +209,13 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:426", "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:447", "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:453", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:722", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:739", - "ui/packages/loom-ui/src/api.ts:1367", - "ui/packages/loom-ui/src/api.ts:1390", - "ui/packages/loom-ui/src/api.ts:1410", - "ui/packages/loom-ui/src/api.ts:1418", - "ui/packages/loom-ui/src/Builder.tsx:119", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:721", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:738", + "ui/packages/loom-ui/src/api.ts:1356", + "ui/packages/loom-ui/src/api.ts:1379", + "ui/packages/loom-ui/src/api.ts:1399", + "ui/packages/loom-ui/src/api.ts:1407", + "ui/packages/loom-ui/src/Builder.tsx:116", "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:427", "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:455" ], @@ -285,7 +285,7 @@ } ], "entryPath": [ - "ui/packages/loom-ui/src/Builder.tsx:119", + "ui/packages/loom-ui/src/Builder.tsx:116", "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:398" ] }, @@ -326,7 +326,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers", "actionId": "workspace.mount", "payloadShape": "no body", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:427 explorers query; ui/packages/loom-ui/src/api.ts:1390 listExplorers", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:427 explorers query; ui/packages/loom-ui/src/api.ts:1379 listExplorers", "condition": "runs with initial workspace queries", "variant": "initial-hydration" }, @@ -338,7 +338,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/builder", "actionId": "workspace.mount_or_explorer_change", "payloadShape": "no body", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:447 useGetExplorerBuilderStateV2Query; ui/packages/loom-ui/src/api.ts:1410 getBuilder", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:447 useGetExplorerBuilderStateV2Query; ui/packages/loom-ui/src/api.ts:1399 getBuilder", "condition": "effectiveExplorerId is non-empty", "variant": "initial-hydration" }, @@ -350,7 +350,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/capability", "actionId": "workspace.mount_or_explorer_change", "payloadShape": "no body", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:455 useGetExplorerAuthoringCapabilityV2Query; ui/packages/loom-ui/src/api.ts:1418 getCapability", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:455 useGetExplorerAuthoringCapabilityV2Query; ui/packages/loom-ui/src/api.ts:1407 getCapability", "condition": "effectiveExplorerId is non-empty", "variant": "initial-hydration" } @@ -397,18 +397,18 @@ }, "availability": "capability_conditional", "source": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2443", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2461", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2475", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2482", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3128", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3136", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2442", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2460", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2474", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2481", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3126", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3134", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/BuilderToolbar.tsx:37", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/BuilderToolbar.tsx:349", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/BuilderToolbar.tsx:354", - "ui/packages/loom-ui/src/api.ts:1801", - "ui/packages/loom-ui/src/api.ts:1842", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3099" + "ui/packages/loom-ui/src/api.ts:1790", + "ui/packages/loom-ui/src/api.ts:1831", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3097" ], "constraints": [ { @@ -476,8 +476,8 @@ } ], "entryPath": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3099", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3128", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3097", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3126", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/BuilderToolbar.tsx:349" ] }, @@ -519,7 +519,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers[?auth_resource_path=...]", "actionId": "create_explorer", "payloadShape": "{name:string,title?:string,sourceExplorerId?:string}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2443 createCustomExplorer; ui/packages/loom-ui/src/api.ts:1801 createExplorer", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2442 createCustomExplorer; ui/packages/loom-ui/src/api.ts:1790 createExplorer", "condition": "trimmed name non-empty; busy=false", "variant": "create" }, @@ -531,7 +531,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}", "actionId": "delete_explorer", "payloadShape": "no body; optional X-Request-ID header", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2461 deleteCurrentExplorer; ui/packages/loom-ui/src/api.ts:1842 deleteExplorer", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2460 deleteCurrentExplorer; ui/packages/loom-ui/src/api.ts:1831 deleteExplorer", "condition": "delete capability true; fallback exists; user confirms", "variant": "delete" } @@ -588,35 +588,35 @@ }, "availability": "reachable", "source": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2729", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2745", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2773", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2808", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4077", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4084", - "ui/packages/loom-ui/src/api.ts:1450", - "ui/packages/loom-ui/src/api.ts:1486", - "ui/packages/loom-ui/src/api.ts:1697", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2728", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2744", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2772", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2807", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4065", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4072", + "ui/packages/loom-ui/src/api.ts:1439", + "ui/packages/loom-ui/src/api.ts:1475", + "ui/packages/loom-ui/src/api.ts:1686", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowRootPicker.tsx:14", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowRootPicker.tsx:17", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowRootPicker.tsx:51", "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/initialTablePlan.ts:15", "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/initialTablePlan.ts:32", "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/initialTablePlan.ts:66", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4046", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4034", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowRootPicker.tsx:4", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4079", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4067", "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/initialTablePlan.ts:43", "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/initialTablePlan.ts:59", "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/initialTablePlan.ts:84", "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/initialTablePlan.ts:93", "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:369", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2737", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2827", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4087", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2911", - "ui/packages/loom-ui/src/api.ts:1479", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2621" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2736", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2826", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4075", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2909", + "ui/packages/loom-ui/src/api.ts:1468", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2620" ], "constraints": [ { @@ -711,8 +711,8 @@ } ], "entryPath": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4046", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4077", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4034", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4065", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowRootPicker.tsx:4" ] }, @@ -760,7 +760,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/suggestions", "actionId": "choose_first_table_root", "payloadShape": "{snapshotToken:string,nodeId:string}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2745 getSuggestions; ui/packages/loom-ui/src/api.ts:1486 suggestions", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2744 getSuggestions; ui/packages/loom-ui/src/api.ts:1475 suggestions", "condition": "root and catalog snapshot exist; failure falls back to root-only", "variant": "first-table" }, @@ -772,7 +772,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/commands[?auth_resource_path=...]", "actionId": "create_first_table", "payloadShape": "{commandId:string,semanticsVersion:constant,snapshotToken:string,expectedDraftVersion:number,expectedDraftDigest?:string,commands:[{type:CREATE_TABLE,title:string,rootNodeId:string}]}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2773 applyCommandsWithResult; ui/packages/loom-ui/src/api.ts:1450 applyCommands", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2772 applyCommandsWithResult; ui/packages/loom-ui/src/api.ts:1439 applyCommands", "condition": "planInitialTable returns root-only or with-identity", "variant": "first-table" }, @@ -784,7 +784,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/commands[?auth_resource_path=...]", "actionId": "add_first_table_identity_column", "payloadShape": "{commandId:string,semanticsVersion:constant,snapshotToken:string,expectedDraftVersion:number,expectedDraftDigest?:string,commands:[{type:ADD_COLUMN,outputId:string,occurrenceId:string,candidateId:string,projectionMode:VALUE,initialPresentation:TABLE,title:string}]}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2808 applyCommandsWithResult; ui/packages/loom-ui/src/api.ts:1450 applyCommands", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2807 applyCommandsWithResult; ui/packages/loom-ui/src/api.ts:1439 applyCommands", "condition": "plan kind is with-identity and CREATE_TABLE returned outputId and occurrenceId", "variant": "first-table" }, @@ -796,7 +796,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/reconcile[?auth_resource_path=...]", "actionId": "automatic_preview_reconcile", "payloadShape": "{snapshotToken:string,draftVersion:number,draftDigest:string,outputId:string}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2911 reconcileCurrent; ui/packages/loom-ui/src/api.ts:1479 reconcile", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2909 reconcileCurrent; ui/packages/loom-ui/src/api.ts:1468 reconcile", "condition": "rooted table has a visible output and no valid reusable receipt", "variant": "automatic-preview" }, @@ -808,7 +808,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/preview", "actionId": "automatic_preview_load", "payloadShape": "{receiptId:string,outputId:string,limit:25|50|100|500|1000}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2621 executePreview; ui/packages/loom-ui/src/api.ts:1697 preview", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2620 executePreview; ui/packages/loom-ui/src/api.ts:1686 preview", "condition": "reconcile returns a matching receipt", "variant": "automatic-preview" } @@ -855,20 +855,20 @@ }, "availability": "reachable", "source": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:660", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4591", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4601", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4615", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4630", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4641", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4652", - "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/routeActions.ts:9", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:659", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4579", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4589", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4603", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4618", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4629", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4640", + "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/routeActions.ts:4", "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/reducer.ts:206", "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/reducer.ts:217", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:220", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2274", - "ui/packages/loom-ui/src/api.ts:1486", - "ui/packages/loom-ui/src/api.ts:1450" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2273", + "ui/packages/loom-ui/src/api.ts:1475", + "ui/packages/loom-ui/src/api.ts:1439" ], "constraints": [ { @@ -946,7 +946,7 @@ ], "entryPath": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx:220", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4591" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4579" ] }, "prose": { @@ -989,7 +989,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/suggestions", "actionId": "select_route_occurrence", "payloadShape": "{snapshotToken:string,nodeId:string}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2274 ensureSuggestions; ui/packages/loom-ui/src/api.ts:1486 suggestions", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2273 ensureSuggestions; ui/packages/loom-ui/src/api.ts:1475 suggestions", "condition": "current occurrence has no cached candidates", "variant": "candidate-load" }, @@ -1001,7 +1001,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/commands[?auth_resource_path=...]", "actionId": "edit_table_route", "payloadShape": "{commandId:string,semanticsVersion:constant,snapshotToken:string,expectedDraftVersion:number,expectedDraftDigest?:string,commands:[ADD_ROUTE|UPDATE_ROUTE_EDGE|SET_ROUTE_MATCH_MODE|REMOVE_ROUTE]}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4615 GuidedGraphWorkspace callbacks; ui/packages/loom-ui/src/api.ts:1450 applyCommands", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4603 GuidedGraphWorkspace callbacks; ui/packages/loom-ui/src/api.ts:1439 applyCommands", "condition": "edge is legal from selected occurrence; local table/occurrence selection itself sends no request", "variant": "each-route-edit" } @@ -1043,32 +1043,32 @@ }, "availability": "capability_conditional", "source": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1409", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1427", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1446", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1509", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3637", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3715", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3746", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1838", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1876", - "ui/packages/loom-ui/src/api.ts:1495", - "ui/packages/loom-ui/src/api.ts:1540", - "ui/packages/loom-ui/src/api.ts:1555", - "ui/packages/loom-ui/src/api.ts:1691", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1408", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1426", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1445", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1508", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3625", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3703", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3734", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1837", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1875", + "ui/packages/loom-ui/src/api.ts:1484", + "ui/packages/loom-ui/src/api.ts:1529", + "ui/packages/loom-ui/src/api.ts:1544", + "ui/packages/loom-ui/src/api.ts:1680", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1049", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:92", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:270", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3667", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3751", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3756", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3655", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3739", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3744", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1045", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1544", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1543", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:233", - "ui/packages/loom-ui/src/api.ts:1685", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1906", - "ui/packages/loom-ui/src/api.ts:1450", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1800" + "ui/packages/loom-ui/src/api.ts:1674", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1905", + "ui/packages/loom-ui/src/api.ts:1439", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1799" ], "constraints": [ { @@ -1154,9 +1154,9 @@ } ], "entryPath": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3637", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3667", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3746" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3625", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3655", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3734" ] }, "prose": { @@ -1199,7 +1199,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/semantic-inventory", "actionId": "browse_add_columns_catalog", "payloadShape": "{snapshotToken:string,rowRoot:string,selectionMode?,frameId?,sourceChoiceId?,outputId?,resourceType?,query?,cursor?,limit?,fieldPresentation?}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1045 calls browseSemanticInventory; ui/packages/loom-ui/src/api.ts:1495", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ConceptCatalog.tsx:1045 calls browseSemanticInventory; ui/packages/loom-ui/src/api.ts:1484", "condition": "catalog view is active", "variant": "catalog-browse" }, @@ -1211,7 +1211,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/frame-source-options", "actionId": "manage_saved_frame_source", "payloadShape": "{snapshotToken:string,outputId:string,resourceType?,query?,cursor?,limit?}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:270; ui/packages/loom-ui/src/api.ts:1540", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:270; ui/packages/loom-ui/src/api.ts:1529", "condition": "saved coded-value frame panel is open; allowNewSource=false", "variant": "saved-frame" }, @@ -1223,7 +1223,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/semantic-inventory", "actionId": "choose_coded_values_for_frame", "payloadShape": "{snapshotToken:string,outputId:string,rowRoot:string,frameId:string,query:string,cursor?,limit:50}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:92; ui/packages/loom-ui/src/api.ts:1495", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/FrameSourcePanel.tsx:92; ui/packages/loom-ui/src/api.ts:1484", "condition": "an existing frame is selected", "variant": "saved-frame" }, @@ -1235,7 +1235,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/construction-choice-proposals[?auth_resource_path=...]", "actionId": "add_selected_catalog_features", "payloadShape": "{commandId:string,snapshotToken:string,expectedDraftVersion:number,expectedDraftDigest:string,outputId:string,constructionChoices:array,limit:PreviewLimit}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1876; ui/packages/loom-ui/src/api.ts:1691", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1875; ui/packages/loom-ui/src/api.ts:1680", "condition": "ordinary selected choices are current and match selected output/draft/limit", "variant": "ordinary-choice" }, @@ -1247,7 +1247,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/construction-proposals[?auth_resource_path=...]", "actionId": "add_related_source_fields", "payloadShape": "{snapshotToken:string,expectedDraftVersion:number,expectedDraftDigest:string,outputId:string,candidateConstruction:Construction,changedStepId:string,limit:PreviewLimit}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1544 and :1861; ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:233; ui/packages/loom-ui/src/api.ts:1685", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1543 and :1861; ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:233; ui/packages/loom-ui/src/api.ts:1674", "condition": "all selected related fields share the same exact route; append-stage capability supports RELATED_SOURCE", "variant": "related-source" }, @@ -1259,7 +1259,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/commands[?auth_resource_path=...]", "actionId": "save_selected_catalog_features", "payloadShape": "{commandId:string,semanticsVersion:constant,snapshotToken:string,expectedDraftVersion:number,expectedDraftDigest?:string,commands:[{type:APPLY_CONSTRUCTION_CHOICE,...}]}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1906; ui/packages/loom-ui/src/api.ts:1450", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1905; ui/packages/loom-ui/src/api.ts:1439", "condition": "ordinary proposal preview matches current owner, snapshot, draft, output, and limit", "variant": "ordinary-choice" }, @@ -1271,7 +1271,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/commands[?auth_resource_path=...]", "actionId": "save_related_source_fields", "payloadShape": "{commandId:string,semanticsVersion:constant,snapshotToken:string,expectedDraftVersion:number,expectedDraftDigest?:string,commands:[{type:APPLY_CONSTRUCTION_PROPOSAL,outputId,proposalId}]}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1800; ui/packages/loom-ui/src/api.ts:1450", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1799; ui/packages/loom-ui/src/api.ts:1439", "condition": "proposal is ready and its preview matches current identity", "variant": "related-source" } @@ -1336,13 +1336,13 @@ "availability": "capability_conditional", "source": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/operationFamilies.ts:1", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1017", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1036", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3312", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3320", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3584", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4262", - "ui/packages/loom-ui/src/api.ts:1652", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1016", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1035", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3310", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3318", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3572", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4250", + "ui/packages/loom-ui/src/api.ts:1641", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:118", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:173" ], @@ -1412,9 +1412,9 @@ } ], "entryPath": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3312", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3584", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4262" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3310", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3572", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4250" ] }, "prose": { @@ -1456,7 +1456,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/construction-capabilities[?auth_resource_path=...]", "actionId": "select_operation_or_saved_step", "payloadShape": "{snapshotToken:string,expectedDraftVersion:number,expectedDraftDigest:string,outputId:string,stageId:string,groupStepId?:string}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1017 builds request; ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:173; ui/packages/loom-ui/src/api.ts:1652", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1016 builds request; ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:173; ui/packages/loom-ui/src/api.ts:1641", "condition": "rooted table, snapshot and draft identity ready; no unsupported saved source columns; response matches request", "variant": "capability-load" } @@ -1535,22 +1535,22 @@ }, "availability": "reachable", "source": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2032", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2050", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2102", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2145", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2248", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2258", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4247", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2031", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2049", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2101", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2144", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2247", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2257", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4235", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowChangePreviewPanel.tsx:5", "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/rootChangeSelection.ts:9", "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/rootChangeSelection.ts:81", - "ui/packages/loom-ui/src/api.ts:1468", - "ui/packages/loom-ui/src/api.ts:1697", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4284", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4298", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4396", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4406", + "ui/packages/loom-ui/src/api.ts:1457", + "ui/packages/loom-ui/src/api.ts:1686", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4272", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4286", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4384", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4394", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:616", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:630", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:655", @@ -1561,7 +1561,7 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionPanel.tsx:267", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionPanel.tsx:289", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionPanel.tsx:383", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4591", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4579", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:635", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:644", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:660", @@ -1569,7 +1569,7 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:670", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:1227", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:1231", - "ui/packages/loom-ui/src/api.ts:1450", + "ui/packages/loom-ui/src/api.ts:1439", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:649" ], "constraints": [ @@ -1665,9 +1665,9 @@ } ], "entryPath": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4284", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4396", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4591", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4272", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4384", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4579", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:616" ] }, @@ -1715,7 +1715,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/row-change", "actionId": "change_connected_row_root", "payloadShape": "{snapshotToken:string,draftVersion:number,draftDigest:string,outputId:string,rootNodeId:string,rootOccurrenceId?,routeRebase?}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2032; ui/packages/loom-ui/src/api.ts:1468", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2031; ui/packages/loom-ui/src/api.ts:1457", "condition": "connected/authored/catalog/compiler route choice; current identity matches", "variant": "connected-rebase" }, @@ -1727,7 +1727,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/preview", "actionId": "review_connected_row_root", "payloadShape": "{receiptId:string,outputId:string,limit:PreviewLimit}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2102; ui/packages/loom-ui/src/api.ts:1697", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2101; ui/packages/loom-ui/src/api.ts:1686", "condition": "assessment returns matching candidate receipt and proposal", "variant": "connected-rebase" }, @@ -1739,7 +1739,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/commands[?auth_resource_path=...]", "actionId": "apply_connected_row_root", "payloadShape": "{commandId:string,semanticsVersion:constant,snapshotToken:string,expectedDraftVersion:number,expectedDraftDigest?:string,commands:[{type:APPLY_TABLE_ROOT_REBASE,outputId,rowChange}]}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2258; ui/packages/loom-ui/src/api.ts:1450", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2257; ui/packages/loom-ui/src/api.ts:1439", "condition": "ready preview and unchanged root-change identity", "variant": "connected-rebase" }, @@ -1751,7 +1751,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/row-change", "actionId": "replace_table_root", "payloadShape": "{mode:REPLACE,snapshotToken:string,draftVersion:number,draftDigest:string,outputId:string,rootNodeId:string}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:616; ui/packages/loom-ui/src/api.ts:1468", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:616; ui/packages/loom-ui/src/api.ts:1457", "condition": "replacement-only root choice; client supports assessRowChange and preview", "variant": "replace" }, @@ -1763,7 +1763,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/preview", "actionId": "review_root_replacement", "payloadShape": "{receiptId:string,outputId:string}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:649; ui/packages/loom-ui/src/api.ts:1697", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:649; ui/packages/loom-ui/src/api.ts:1686", "condition": "assessment is READY, current and contains candidate receipt", "variant": "replace" }, @@ -1775,7 +1775,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/commands[?auth_resource_path=...]", "actionId": "apply_root_replacement", "payloadShape": "{commandId:string,semanticsVersion:constant,snapshotToken:string,expectedDraftVersion:number,expectedDraftDigest?:string,commands:[{type:APPLY_TABLE_ROOT_REBASE,outputId,rowChange}]}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:691; ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4406; ui/packages/loom-ui/src/api.ts:1450", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:691; ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4394; ui/packages/loom-ui/src/api.ts:1439", "condition": "replacement state ready and assessment still current", "variant": "replace" } @@ -1831,15 +1831,15 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:325", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:340", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:444", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1800", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1818", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3962", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1799", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1817", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3950", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionProposalPanel.tsx:11", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionProposalPanel.tsx:317", - "ui/packages/loom-ui/src/api.ts:1685", - "ui/packages/loom-ui/src/api.ts:1697", + "ui/packages/loom-ui/src/api.ts:1674", + "ui/packages/loom-ui/src/api.ts:1686", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:266", - "ui/packages/loom-ui/src/api.ts:1450" + "ui/packages/loom-ui/src/api.ts:1439" ], "constraints": [ { @@ -1925,7 +1925,7 @@ } ], "entryPath": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3962", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3950", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionProposalPanel.tsx:317" ] }, @@ -1969,7 +1969,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/construction-proposals[?auth_resource_path=...]", "actionId": "edit_or_remove_construction_step", "payloadShape": "{snapshotToken:string,expectedDraftVersion:number,expectedDraftDigest:string,outputId:string,candidateConstruction:Construction,changedStepId?,removeStepIds?,limit:PreviewLimit}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:233; ui/packages/loom-ui/src/api.ts:1685", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:233; ui/packages/loom-ui/src/api.ts:1674", "condition": "candidate changes and capability request is current; request waits 300 ms", "variant": "initial-or-retry" }, @@ -1981,7 +1981,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/preview", "actionId": "review_construction_change", "payloadShape": "{receiptId:proposalId,outputId,limit:PreviewLimit}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:266 and :271; ui/packages/loom-ui/src/api.ts:1697", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:266 and :271; ui/packages/loom-ui/src/api.ts:1686", "condition": "proposal response lacks a matching rows preview", "variant": "initial-or-retry" }, @@ -1993,7 +1993,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/commands[?auth_resource_path=...]", "actionId": "accept_construction_proposal", "payloadShape": "{commandId:string,semanticsVersion:constant,snapshotToken:string,expectedDraftVersion:number,expectedDraftDigest?:string,commands:[{type:APPLY_CONSTRUCTION_PROPOSAL,outputId,proposalId}]}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1800; ui/packages/loom-ui/src/api.ts:1450", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1799; ui/packages/loom-ui/src/api.ts:1439", "condition": "ready proposal, matching preview identity, no pending commands, no active publication, reconciliation not pending", "variant": "apply" }, @@ -2063,23 +2063,23 @@ }, "availability": "reachable", "source": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:778", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:793", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:800", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:812", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:845", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:862", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2500", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2522", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2561", - "ui/packages/loom-ui/src/api.ts:1450", - "ui/packages/loom-ui/src/api.ts:1479", - "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/model.ts:247", - "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/model.ts:319", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3151", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:777", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:792", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:799", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:811", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:844", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:861", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2499", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2521", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2560", + "ui/packages/loom-ui/src/api.ts:1439", + "ui/packages/loom-ui/src/api.ts:1468", + "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/model.ts:245", + "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/model.ts:317", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3149", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3153", "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3155", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3157", - "ui/packages/loom-ui/src/api.ts:1410" + "ui/packages/loom-ui/src/api.ts:1399" ], "constraints": [ { @@ -2165,8 +2165,8 @@ } ], "entryPath": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:778", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3151" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:777", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3149" ] }, "prose": { @@ -2207,7 +2207,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/commands[?auth_resource_path=...]", "actionId": "save_builder_draft_change", "payloadShape": "{commandId:string,semanticsVersion:constant,snapshotToken:string,expectedDraftVersion:number,expectedDraftDigest?:string,commands:ExplorerBuilderCommand[]}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:778; ui/packages/loom-ui/src/api.ts:1450", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:777; ui/packages/loom-ui/src/api.ts:1439", "condition": "current server draft identity; serialized command queue", "variant": "save" }, @@ -2219,7 +2219,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/reconcile[?auth_resource_path=...]", "actionId": "compile_current_builder_draft", "payloadShape": "{snapshotToken:string,draftVersion:number,draftDigest:string,outputId?:string}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2500; ui/packages/loom-ui/src/api.ts:1479", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2499; ui/packages/loom-ui/src/api.ts:1468", "condition": "catalog snapshot exists; all tables rooted; Review/preview needs receipt or unresolved state", "variant": "reconcile" }, @@ -2231,7 +2231,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/builder", "actionId": "refresh_after_stale_or_desynchronized_draft", "payloadShape": "no body", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2561 and :866; ui/packages/loom-ui/src/api.ts:1410", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2560 and :866; ui/packages/loom-ui/src/api.ts:1399", "condition": "command or reconcile reports stale snapshot or draft desynchronization", "variant": "refresh" } @@ -2287,32 +2287,32 @@ }, "availability": "reachable", "source": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:565", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2347", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2500", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2608", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2831", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2853", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2888", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2911", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4218", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4222", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4456", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:760", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:794", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:564", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2346", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2499", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2607", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2830", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2852", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2886", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2909", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4206", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4210", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4444", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:757", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:791", "ui/packages/loom-ui/src/features/ExplorerBuilder/useAutomaticPreview.ts:23", "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/previewRecovery.ts:3", "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/previewRecovery.ts:32", - "ui/packages/loom-ui/src/api.ts:1697", - "ui/packages/loom-ui/src/api.ts:1479", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2645", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2662", + "ui/packages/loom-ui/src/api.ts:1686", + "ui/packages/loom-ui/src/api.ts:1468", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2644", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2661", "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/previewRecovery.ts:10", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2668", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2675", - "ui/packages/loom-ui/src/api.ts:1410", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2684", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2689" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2667", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2674", + "ui/packages/loom-ui/src/api.ts:1399", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2683", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2688" ], "constraints": [ { @@ -2398,9 +2398,9 @@ } ], "entryPath": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2831", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4456", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:760" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2830", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4444", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:757" ] }, "prose": { @@ -2444,7 +2444,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/reconcile[?auth_resource_path=...]", "actionId": "preview_selected_output", "payloadShape": "{snapshotToken:string,draftVersion:number,draftDigest:string,outputId:string}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2911 and :2500; ui/packages/loom-ui/src/api.ts:1479", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2909 and :2500; ui/packages/loom-ui/src/api.ts:1468", "condition": "no reusable receipt matches output/snapshot; or explicit reload", "variant": "normal-or-recompile" }, @@ -2456,7 +2456,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/preview", "actionId": "preview_selected_output", "payloadShape": "{receiptId:string,outputId:string,limit:25|50|100|500|1000}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2608 and :2831; ui/packages/loom-ui/src/api.ts:1697", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2607 and :2831; ui/packages/loom-ui/src/api.ts:1686", "condition": "selected rooted table has visible output, valid draft identity, no blocking diagnostic/proposal/command", "variant": "normal" }, @@ -2478,7 +2478,7 @@ 1000 ] }, - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2645; ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/previewRecovery.ts:32", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2644; ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/previewRecovery.ts:32", "condition": "error.retryable and transientRetries is zero", "variant": "retryable-error" }, @@ -2490,7 +2490,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/reconcile[?auth_resource_path=...]", "actionId": "rebuild_stale_preview_receipt", "payloadShape": "{snapshotToken:string,draftVersion:number,draftDigest:string,outputId:string}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2662 and :2500; ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/previewRecovery.ts:10", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2661 and :2500; ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/previewRecovery.ts:10", "condition": "receipt-recompile error and receiptRefreshes is zero", "variant": "stale-receipt" }, @@ -2502,7 +2502,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/preview", "actionId": "refresh_preview_after_recompile", "payloadShape": "{receiptId:string,outputId:string,limit:PreviewLimit}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2668 and :2621", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2667 and :2621", "condition": "recompile returned current receipt", "variant": "stale-receipt" }, @@ -2514,7 +2514,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/builder", "actionId": "refresh_preview_catalog", "payloadShape": "no body", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2675; ui/packages/loom-ui/src/api.ts:1410", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2674; ui/packages/loom-ui/src/api.ts:1399", "condition": "catalog/receipt stale code and receiptRefreshes is zero", "variant": "stale-catalog" }, @@ -2526,7 +2526,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/reconcile[?auth_resource_path=...]", "actionId": "rebuild_preview_after_catalog_refresh", "payloadShape": "{snapshotToken:string,draftVersion:number,draftDigest:string,outputId:string}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2684 and :2500; ui/packages/loom-ui/src/api.ts:1479", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2683 and :2500; ui/packages/loom-ui/src/api.ts:1468", "condition": "Builder refresh returned data and current identity still matches", "variant": "stale-catalog" }, @@ -2538,7 +2538,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/preview", "actionId": "fetch_preview_after_catalog_refresh", "payloadShape": "{receiptId:string,outputId:string,limit:PreviewLimit}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2689 and :2621", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2688 and :2621", "condition": "reconciliation returned current receipt", "variant": "stale-catalog" } @@ -2574,17 +2574,17 @@ "availability": "reachable", "source": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:483", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3384", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4051", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4053", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3382", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4039", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4041", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionWorkspace.tsx:73", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionWorkspace.tsx:79", - "ui/packages/loom-ui/src/api.ts:1450", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3385", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3386", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3392", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3395", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4055", + "ui/packages/loom-ui/src/api.ts:1439", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3383", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3384", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3390", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3393", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4043", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionWorkspace.tsx:81" ], "constraints": [ @@ -2653,8 +2653,8 @@ } ], "entryPath": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4051", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4053" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4039", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4041" ] }, "prose": { @@ -2695,7 +2695,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/commands[?auth_resource_path=...]", "actionId": "undo_last_saved_draft_change", "payloadShape": "{commandId:string,semanticsVersion:constant,snapshotToken:string,expectedDraftVersion:number,expectedDraftDigest?:string,commands:[{type:RESTORE_DRAFT_REVISION,draftRevisionId:string}]}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3384; ui/packages/loom-ui/src/api.ts:1450", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3382; ui/packages/loom-ui/src/api.ts:1439", "condition": "tables empty and previousDraftRevisionId exists", "variant": "undo" } @@ -2731,8 +2731,8 @@ "unit": "visible Delete explorer control" }, "source": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3136", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3137", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3134", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3135", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/BuilderToolbar.tsx:354", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/BuilderToolbar.tsx:358", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/BuilderToolbar.tsx:361" @@ -2864,8 +2864,8 @@ "unit": "active row-definition proposal for selected table" }, "source": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4284", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4403", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4272", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4391", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:520", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:527", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:581", @@ -2875,12 +2875,12 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:737", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:916", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:1120", - "ui/packages/loom-ui/src/api.ts:1607", - "ui/packages/loom-ui/src/api.ts:1612", - "ui/packages/loom-ui/src/types.ts:721" + "ui/packages/loom-ui/src/api.ts:1596", + "ui/packages/loom-ui/src/api.ts:1601", + "ui/packages/loom-ui/src/types.ts:720" ], "entryPath": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4284", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4272", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:916" ], "constraints": [ @@ -2943,7 +2943,7 @@ "endpoint": "/api/v1/projects/{encodedProject}/explorers/{explorerId}/authoring/v2/row-definition-choices?outputId={outputId}&snapshotToken={snapshotToken}", "actionId": "open_row_definition_settings", "payloadShape": "no body; query {outputId,snapshotToken}", - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:520; ui/packages/loom-ui/src/api.ts:1607", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:520; ui/packages/loom-ui/src/api.ts:1596", "condition": "settings opened for selected table" }, { @@ -2961,7 +2961,7 @@ "selection": "RowDefinitionSelection", "limit?": "number" }, - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:581; ui/packages/loom-ui/src/api.ts:1612", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:581; ui/packages/loom-ui/src/api.ts:1601", "condition": "selection change waits 300 ms; current settings are loaded", "variant": "row-shape-change" }, @@ -2986,7 +2986,7 @@ } ] }, - "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:718; ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4403; ui/packages/loom-ui/src/api.ts:1450", + "constructor": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:718; ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4391; ui/packages/loom-ui/src/api.ts:1439", "condition": "proposal fresh/current, comparison AVAILABLE, proposalId present, actions enabled", "variant": "apply" } diff --git a/docs/ui-contract-inventory/relational.fragment.json b/docs/ui-contract-inventory/relational.fragment.json index 96195a6b5..16cf08f98 100644 --- a/docs/ui-contract-inventory/relational.fragment.json +++ b/docs/ui-contract-inventory/relational.fragment.json @@ -47,55 +47,55 @@ "basis": "backend-promise", "statement": "The UI describes one output row per unique combination of chosen group keys, with summary columns.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2283-2315" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2274-2306" ] }, { "basis": "observed-ui", "statement": "The client candidate builder does not enforce a non-empty key list, but requires at least one summary and valid unique output names.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:984-1004" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:979-999" ] }, { "basis": "observed-ui", "statement": "COUNT_ROWS needs no field input; other summary kinds use compatible stage columns.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3320-3401" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3311-3392" ] }, { "basis": "backend-promise", "statement": "The selected missing-key policy is presented as running before summaries: GROUP combines absent/null keys, EXCLUDE removes rows missing any key, ERROR stops.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3310-3317", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3417-3436" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3301-3308", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3408-3427" ] }, { "basis": "observed-ui", "statement": "Related field choices are disabled when selected fields conflict across routes; coding identity cannot combine with semantic/coded keys.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:993-997", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3132-3177" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:988-992", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3123-3168" ] } ], "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1017-1034", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3253-3311", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3481-3505", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1016-1033", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3251-3309", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3476-3500", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:872-897", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2279-2336", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:559-665", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:984-1027", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1967-1984", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3320-3401", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3413-3436", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2749-2791", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3851-3900", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2270-2327", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:554-660", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:979-1022", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1962-1979", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3311-3392", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3404-3427", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2740-2782", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3842-3891", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:233-252", - "ui/packages/loom-ui/src/api.ts:1685-1689" + "ui/packages/loom-ui/src/api.ts:1674-1678" ] }, { @@ -115,24 +115,24 @@ "basis": "observed-ui", "statement": "Every chosen source field becomes a group key; the default summary counts source records.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1087-1151" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1082-1146" ] }, { "basis": "observed-ui", "statement": "A source-field group can only be proposed from the source projection stage.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1132-1151", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1967-1976" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1127-1146", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1962-1971" ] } ], "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3481-3488", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2054-2071", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1087-1151", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1967-1976", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2317-2336" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3476-3483", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2049-2066", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1082-1146", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1962-1971", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2308-2327" ], "semantic_domains": [ { @@ -152,8 +152,8 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2054-2071", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1087-1124" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2049-2066", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1082-1119" ] }, { @@ -172,8 +172,8 @@ "maximum": null }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1107-1115", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1132-1150" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1102-1110", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1127-1145" ] }, { @@ -191,7 +191,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1116-1124" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1111-1119" ] } ] @@ -227,9 +227,9 @@ "maximum": null }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2979-2984", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3060-3081", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3132-3178" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2970-2975", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3051-3072", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3123-3169" ] }, { @@ -248,7 +248,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3055-3081" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3046-3072" ] }, { @@ -267,7 +267,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:993-995" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:988-990" ] }, { @@ -289,8 +289,8 @@ "maximum": null }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3320-3350", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:990-1003" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3311-3341", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:985-998" ] }, { @@ -310,7 +310,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3413-3436" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3404-3427" ] }, { @@ -333,8 +333,8 @@ "maximum": null }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2749-2791", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2820-2840" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2740-2782", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2811-2831" ] }, { @@ -356,8 +356,8 @@ "maximum": 50 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3644-3663", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3892-3899" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3635-3654", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3883-3890" ] }, { @@ -377,7 +377,7 @@ }, "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:255-261", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:993-1003" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:988-998" ] } ], @@ -386,33 +386,33 @@ "basis": "backend-promise", "statement": "The UI describes one output row per unique combination of chosen group keys, with summary columns; a zero-key summary is not rejected by the client candidate builder.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2283-2315", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:984-1004" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2274-2306", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:979-999" ] }, { "basis": "observed-ui", "statement": "The candidate builder requires at least one summary and validates unique physical output names; COUNT_ROWS has no field input, other summaries use compatible stage columns.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:990-1003", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3320-3401" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:985-998", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3311-3392" ] }, { "basis": "backend-promise", "statement": "The selected missing-key policy is presented as running before summaries: GROUP combines absent/null keys, EXCLUDE removes rows missing any key, ERROR stops.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3310-3317", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3417-3436" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3301-3308", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3408-3427" ] }, { "basis": "observed-ui", "statement": "Group field choices using related occurrences are disabled when saved or selected fields conflict across routes; a coding-identity key cannot be combined with semantic/coded keys.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:993-997", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3132-3177", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3055-3081" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:988-992", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3123-3168", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3046-3072" ] } ] @@ -453,11 +453,11 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3488-3496", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:456-460", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3962-3977", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3665-3677", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4191-4249" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3483-3491", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:451-455", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3953-3968", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3653-3665", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4182-4240" ] }, { @@ -477,8 +477,8 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4044-4067", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4075-4080" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4035-4058", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4066-4071" ] }, { @@ -499,8 +499,8 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4009-4020", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:918-930" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4000-4011", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:913-925" ] }, { @@ -522,8 +522,8 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4021-4043", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:688-697" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4012-4034", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:683-692" ] }, { @@ -542,7 +542,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:668-697" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:663-692" ] } ], @@ -551,72 +551,72 @@ "basis": "observed-ui", "statement": "Construction capabilities may include expandSourceInput with supported, stageId, optional refusal reasonCode/reason, and choices carrying choiceId, optional columnId, fieldPath, label, logicalType, and isPopulated. A missing or empty response columnId means no live materialization; selecting a signed field submits expandSources entries with non-empty choiceId and columnId through construction-proposals. Signed EXPAND choices represent direct-root primitive arrays and logicalType denotes the element type. Saved EXPAND operations may retain server-signed expandedField facts (path, cardinality=many, optional fhirType, logicalType); these facts are not authored by the picker.", "evidence": [ - "ui/packages/loom-ui/src/types.ts:1995-2127", - "ui/packages/loom-ui/src/types.ts:2185-2204", - "ui/packages/loom-ui/src/api.ts:1694-1698", - "ui/packages/loom-ui/src/types.ts:1747-1764" + "ui/packages/loom-ui/src/types.ts:1991-2116", + "ui/packages/loom-ui/src/types.ts:2174-2193", + "ui/packages/loom-ui/src/api.ts:1683-1687", + "ui/packages/loom-ui/src/types.ts:1746-1760" ] }, { "basis": "observed-ui", "statement": "The Rows > Reshape table > Expand repeated field entry is shown only for a current preview, append-stage capabilities, supported expandSourceInput, and at least one populated signed choice. The handler rechecks those capabilities and opens an empty EXPAND form; selecting a signed field emits expandSources with its choiceId and input columnId through the existing construction proposal flow.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3400-3455", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3665-3677", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3398-3450", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3653-3665", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:1011-1021", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2042-2044", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4191-4249" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2037-2039", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4182-4240" ] }, { "basis": "observed-ui", "statement": "The Split values into rows column action appears only for eligible public scalar-list columns when the current preview, append-stage capability identity, and EXPAND support are present; its click handler rechecks those conditions. The separate signed-field entry uses expandSourceInput support and populated choices.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3488-3496", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3253-3285", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:729-738", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3400-3455", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3665-3677" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3483-3491", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3251-3283", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:726-735", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3398-3450", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3653-3665" ] }, { "basis": "observed-ui", "statement": "The candidate requires exactly one selected list input and a selected empty-list policy; it requires valid unique output names.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:701-713", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1220-1232" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:696-708", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1215-1227" ] }, { "basis": "backend-promise", "statement": "The UI describes each list item as a row for its input record and repeats other fields. For empty or missing lists, ERROR stops, EXCLUDE drops the input row, or PRESERVE_PARENT keeps it with an empty item.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3995-4003", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3998-4007", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4044-4067" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3986-3994", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3989-3998", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4035-4058" ] }, { "basis": "backend-promise", "statement": "Optional item position output is a zero-based integer column; other current-stage columns remain as passthrough outputs.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:668-699", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4021-4043" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:663-694", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4012-4034" ] } ], "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3253-3285", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3488-3496", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4466-4468", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:549-560", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:729-738", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:668-713", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1220-1232", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3962-3977", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3995-4067", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3251-3283", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3483-3491", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4454-4456", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:546-557", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx:726-735", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:663-708", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1215-1227", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3953-3968", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3986-4058", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:233-252", - "ui/packages/loom-ui/src/api.ts:1685-1689" + "ui/packages/loom-ui/src/api.ts:1674-1678" ] }, { @@ -638,24 +638,24 @@ "basis": "backend-promise", "statement": "The result has one row per observed code and exposes system, version, code, and source-record count.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2723-2723", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1187-1211" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2714-2714", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1182-1206" ] }, { "basis": "observed-ui", "statement": "Applying the step replaces current columns with the coded-group outputs; incomplete coding-path selections cannot be proposed.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1183-1185", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1187-1211" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1178-1180", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1182-1206" ] } ], "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3481-3483", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2317-2336", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1154-1211", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1982-1984" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3476-3478", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2308-2327", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1149-1206", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1977-1979" ], "semantic_domains": [ { @@ -675,8 +675,8 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3481-3483", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1183-1185" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3476-3478", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1178-1180" ] }, { @@ -695,8 +695,8 @@ "maximum": 4 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1165-1175", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1183-1211" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1160-1170", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1178-1206" ] }, { @@ -714,8 +714,8 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1165-1175", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2724-2743" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1160-1170", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2715-2734" ] } ], @@ -724,15 +724,15 @@ "basis": "observed-ui", "statement": "The form only creates a candidate when the selected codingPath matches a server choice and exactly four output columns are present.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1183-1185" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1178-1180" ] }, { "basis": "backend-promise", "statement": "The UI describes the result as one row per observed code with system, version, code, and source-record count; applying replaces current columns with these outputs.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2723-2723", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1187-1211" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2714-2714", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1182-1206" ] } ], @@ -780,7 +780,7 @@ } ], "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4283-4311", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4271-4299", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:294-345", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:527-552", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:872-897", @@ -791,8 +791,8 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ExplicitGroupAuthoring.tsx:129-159", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ExplicitGroupAuthoring.tsx:166-208", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/ExplicitGroupAuthoring.tsx:209-239", - "ui/packages/loom-ui/src/api.ts:1607-1620", - "ui/packages/loom-ui/src/api.ts:1833-1840", + "ui/packages/loom-ui/src/api.ts:1596-1609", + "ui/packages/loom-ui/src/api.ts:1822-1829", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:800-830" ], "semantic_domains": [ @@ -926,8 +926,8 @@ "maximum": null }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4253-4259", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4332-4357" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4244-4250", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4323-4348" ] }, { @@ -945,7 +945,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4358-4364" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4349-4355" ] }, { @@ -963,7 +963,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4366-4372" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4357-4363" ] }, { @@ -986,8 +986,8 @@ }, "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:377-425", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4384-4400", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1245-1250" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4375-4391", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1240-1245" ] }, { @@ -1008,7 +1008,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4463-4477" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4454-4468" ] }, { @@ -1025,7 +1025,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4479-4484" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4470-4475" ] }, { @@ -1047,7 +1047,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4486-4495" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4477-4486" ] }, { @@ -1065,8 +1065,8 @@ "maximum": null }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4251-4259", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1967-1994" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4242-4250", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1962-1989" ] } ], @@ -1075,47 +1075,47 @@ "basis": "observed-ui", "statement": "The Pivot editor owns an accessible Finding categories progress state for the current request identity during debounce and while its discovery owner has not published a matching terminal state or adopted result. It hides prior-pair categories, rejects superseded callbacks by revision/context/owner/stage identity, hides progress while disabled, and retains unrelated form choices across discovery failure and retry.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2135-2165", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2224-2243", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2372-2480", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.unit.test.tsx:4477-4635" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2130-2160", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2219-2234", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2363-2471", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.unit.test.tsx:4475-4633" ] }, { "basis": "observed-ui", "statement": "At least one group key and included category are required; category/value roles differ; every discovered category must be selected to form a new pivot candidate.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1245-1250", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4390-4398", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1990-1994" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1240-1245", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4381-4389", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1985-1989" ] }, { "basis": "backend-promise", "statement": "Each selected category becomes an output column while group columns identify output rows; duplicate, missing-cell, and unlisted-category behavior is explicit.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:826-838", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4463-4495" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:821-833", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4454-4486" ] }, { "basis": "observed-ui", "statement": "Only 50 matching categories are rendered at once; no client selection-count maximum is imposed, and discovery may return a server limit.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4388-4400", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4379-4391", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:419-425" ] } ], "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3491-3510", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3486-3505", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:979-1000", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:826-838", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1233-1258", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1958-1999", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:821-833", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1228-1253", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1953-1994", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/useConstructionLifecycle.ts:377-425", - "ui/packages/loom-ui/src/api.ts:1663-1678", - "ui/packages/loom-ui/src/api.ts:1685-1689" + "ui/packages/loom-ui/src/api.ts:1652-1667", + "ui/packages/loom-ui/src/api.ts:1674-1678" ] }, { @@ -1248,9 +1248,9 @@ } ], "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2340-2352", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3488-3510", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1045-1077", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2331-2343", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3483-3505", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1040-1072", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/CodedPivotEditor.tsx:108-127", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/CodedPivotEditor.tsx:249-277" ], @@ -1288,8 +1288,8 @@ "maximum": null }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4570-4685", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1260-1282" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4561-4676", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1255-1277" ] }, { @@ -1313,8 +1313,8 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4660-4685", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1270-1275" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4651-4676", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1265-1270" ] }, { @@ -1333,8 +1333,8 @@ "maximum": 2 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:840-865", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1260-1282" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:835-860", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1255-1277" ] }, { @@ -1351,7 +1351,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4641-4644" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4632-4635" ] } ], @@ -1360,27 +1360,27 @@ "basis": "observed-ui", "statement": "Construction capabilities may include unpivotSourceInput with supported, stageId, optional refusal reasonCode/reason, and choices carrying choiceId, optional columnId, fieldPath, label, logicalType, and isPopulated. A missing or empty response columnId means no live materialization; selecting a signed field submits unpivotSources entries with non-empty choiceId and columnId through construction-proposals.", "evidence": [ - "ui/packages/loom-ui/src/types.ts:1995-2127", - "ui/packages/loom-ui/src/types.ts:2185-2204", - "ui/packages/loom-ui/src/api.ts:1694-1698" + "ui/packages/loom-ui/src/types.ts:1991-2116", + "ui/packages/loom-ui/src/types.ts:2174-2193", + "ui/packages/loom-ui/src/api.ts:1683-1687" ] }, { "basis": "observed-ui", "statement": "Signed source pickers distinguish no listed fields, no populated eligible fields, and eligible populated fields already bound elsewhere. Own bindings count as selectable only when visible and solely bound in the lane; unavailable source-lane reasons stay separate. Existing live field carriers suppress duplicate signed choices only when authoritative field identity resolves unambiguously; stale or ambiguous carriers retain the signed-choice path, and saved bindings are not rewritten. The source-level explanation is derived independently of input rows, including when no inputs have been selected.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4905-4920", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4967-4985", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/existingFieldReuse.ts:35-64", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/sourcePickerExplanations.ts:27-56", - "ui/packages/loom-ui/src/features/ExplorerBuilder/components/SourcePickerExplanation.tsx:9-47" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4896-4911", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:4958-4970", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/existingFieldReuse.ts:22-44", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/sourcePickerExplanations.ts:22-46", + "ui/packages/loom-ui/src/features/ExplorerBuilder/components/SourcePickerExplanation.tsx:5-43" ] }, { "basis": "observed-ui", "statement": "At least one scalar input is required; inputs are unique; NULL and MISSING cannot be key values; output names must be complete and unique.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1265-1275" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1260-1270" ] }, { @@ -1388,15 +1388,15 @@ "statement": "The UI describes one output row per selected input column, with source-name and value outputs; other columns repeat.", "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:1002-1013", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:840-865" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:835-860" ] } ], "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:979-1015", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2363-2373", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:857-865", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1260-1282" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2354-2364", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:852-860", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:1255-1277" ] }, { @@ -1483,7 +1483,7 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/SourceCollectionExpandEditor.tsx:115-145", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/SourceCollectionExpandEditor.tsx:159-180", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/SourceCollectionExpandEditor.tsx:269-287", - "ui/packages/loom-ui/src/api.ts:1607-1620" + "ui/packages/loom-ui/src/api.ts:1596-1609" ] }, { @@ -1704,13 +1704,13 @@ } ], "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3512-3526", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3242-3251", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2380-2390", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3507-3521", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3240-3249", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2371-2381", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/RelatedExpandEditor.tsx:79-151", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/RelatedExpandEditor.tsx:278-310", - "ui/packages/loom-ui/src/api.ts:1564-1575", - "ui/packages/loom-ui/src/api.ts:1587-1598" + "ui/packages/loom-ui/src/api.ts:1553-1564", + "ui/packages/loom-ui/src/api.ts:1576-1587" ] }, { @@ -1753,7 +1753,7 @@ "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionCombineEditor.tsx:267-278", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionCombineEditor.tsx:591-633", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1089-1099" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1088-1098" ] }, { @@ -1844,12 +1844,12 @@ "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionWorkspace.tsx:40-70", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionWorkspace.tsx:88-181", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1180-1188", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3172-3216", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4430-4440", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3860-3896", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1179-1187", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3170-3214", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4418-4428", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3848-3884", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionCombineEditor.tsx:357-389", - "ui/packages/loom-ui/src/api.ts:1685-1689" + "ui/packages/loom-ui/src/api.ts:1674-1678" ] }, { @@ -1964,7 +1964,7 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionCombineEditor.tsx:434-455", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionCombineEditor.tsx:634-673", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionCombineEditor.tsx:357-389", - "ui/packages/loom-ui/src/api.ts:1685-1689" + "ui/packages/loom-ui/src/api.ts:1674-1678" ] }, { @@ -2091,7 +2091,7 @@ "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionCombineEditor.tsx:392-400", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionCombineEditor.tsx:357-389", - "ui/packages/loom-ui/src/api.ts:1685-1689" + "ui/packages/loom-ui/src/api.ts:1674-1678" ] }, { @@ -2277,12 +2277,12 @@ } ], "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4865-4892", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4853-4880", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/TableShapeSettingsPanel.tsx:343-363", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/TableShapeEditor.tsx:163-205", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/tableShapeController.ts:286-340", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/tableShapeController.ts:372-383", - "ui/packages/loom-ui/src/api.ts:1621-1650" + "ui/packages/loom-ui/src/api.ts:1610-1639" ] }, { @@ -2437,12 +2437,12 @@ } ], "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4865-4892", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4853-4880", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/TableShapeEditor.tsx:163-205", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/UnpivotEditor.tsx:14-71", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/tableShapeModel.ts:648-678", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/tableShapeController.ts:319-332", - "ui/packages/loom-ui/src/api.ts:1621-1650" + "ui/packages/loom-ui/src/api.ts:1610-1639" ] }, { @@ -2486,9 +2486,9 @@ "maximum": null }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1410-1445", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1483-1503", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1509-1539", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1409-1444", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1482-1502", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1508-1538", "ui/packages/loom-ui/src/features/ExplorerBuilder/catalogItems.ts:159-187", "ui/packages/loom-ui/src/features/ExplorerBuilder/catalogItems.ts:189-254" ] @@ -2514,7 +2514,7 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1572-1591" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1571-1590" ] }, { @@ -2535,8 +2535,8 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1572-1591", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1637-1646" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1571-1590", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1636-1645" ] }, { @@ -2555,8 +2555,8 @@ "maximum": 1 }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1643-1646", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1885-1889" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1642-1645", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1884-1888" ] }, { @@ -2575,7 +2575,7 @@ "maximum": null }, "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1842-1864" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1841-1863" ] } ], @@ -2584,7 +2584,7 @@ "basis": "observed-ui", "statement": "Only optional-one or required-one source fields are accepted; repeated source cardinality is excluded from RELATED_SOURCE.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1581-1591", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1580-1590", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/RelatedSourceStepEditor.tsx:92-111" ] }, @@ -2592,8 +2592,8 @@ "basis": "observed-ui", "statement": "Candidate requires current append stage, row identity, unique output name, and same exact route across a selected batch.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1555-1570", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1842-1864" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1554-1569", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1841-1863" ] }, { @@ -2605,13 +2605,13 @@ } ], "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1417-1445", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1509-1539", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1544-1673", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1838-1864", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1416-1444", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1508-1538", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1543-1672", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1837-1863", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/RelatedSourceStepEditor.tsx:175-181", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/RelatedSourceStepEditor.tsx:197-275", - "ui/packages/loom-ui/src/api.ts:1555-1563" + "ui/packages/loom-ui/src/api.ts:1544-1552" ], "availability": "Capability- or selection-conditional; this record remains in the inventory when current response disables or hides the affordance.", "reachability_condition": "Requires supported RELATED_SOURCE, row identity, and supported related field choices." @@ -2714,12 +2714,12 @@ } ], "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1420-1426", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3698-3712", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3934-3959", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1419-1425", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3686-3700", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3922-3947", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/RelatedFieldEditor.tsx:34-80", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/RelatedFieldEditor.tsx:134-165", - "ui/packages/loom-ui/src/api.ts:1576-1586" + "ui/packages/loom-ui/src/api.ts:1565-1575" ], "availability": "Capability- or selection-conditional; this record remains in the inventory when current response disables or hides the affordance.", "reachability_condition": "Requires an active related record stage and supported RELATED_FIELD choices." @@ -2841,8 +2841,8 @@ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:294-355", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:527-552", "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:915-949", - "ui/packages/loom-ui/src/api.ts:1607-1610", - "ui/packages/loom-ui/src/api.ts:1612-1620" + "ui/packages/loom-ui/src/api.ts:1596-1599", + "ui/packages/loom-ui/src/api.ts:1601-1609" ] } ], @@ -2902,7 +2902,7 @@ "state": "Button remains visible and disabled when neither GROUP nor coded-pivot-backed grouping is supported; capability reason is shown.", "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:872-882", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3481-3505" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3476-3500" ], "basis": "observed-ui" }, @@ -2911,7 +2911,7 @@ "ui_path": "Group and summarize > Summaries", "state": "SUM and MEAN remain in the summary selector but are disabled when there are no numeric columns.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3326-3350" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:3317-3341" ], "basis": "observed-ui" }, @@ -2921,7 +2921,7 @@ "state": "Button remains visible and disabled when RELATED_EXPAND support or related anchors are absent.", "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionSettingsPanel.tsx:956-967", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3512-3526" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3507-3521" ], "basis": "observed-ui" }, @@ -2930,8 +2930,8 @@ "ui_path": "Reshape editor > Reshape table choices", "state": "Turn categories into columns and Turn columns into rows remain rendered disabled with capability reasons when unsupported; unsupported category/policy choices are also disabled in their lists.", "evidence": [ - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2338-2373", - "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2505-2520" + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2329-2364", + "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx:2496-2511" ], "basis": "observed-ui" }, @@ -2962,8 +2962,8 @@ "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionWorkspace.tsx:106-124", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionWorkspace.tsx:157-179", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1180-1188", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4430-4431" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:1179-1187", + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:4418-4419" ], "basis": "observed-ui" } diff --git a/docs/ui-contract-inventory/ui-supported-affordances.json b/docs/ui-contract-inventory/ui-supported-affordances.json index dbfbb31b1..18445449f 100644 --- a/docs/ui-contract-inventory/ui-supported-affordances.json +++ b/docs/ui-contract-inventory/ui-supported-affordances.json @@ -3,22 +3,22 @@ "fragments": [ { "path": "docs/ui-contract-inventory/expression.fragment.json", - "sha256": "bd16cd0e8a3aad589e77029d330f60bc3ef82b748db7e5fdea69d577a67f285c" + "sha256": "40eea829f49785bc8a3d661de84ea798e4b5ab22be802e4cc1dddad4063863ec" }, { "path": "docs/ui-contract-inventory/relational.fragment.json", - "sha256": "45fb7819815325f21c76547a5782164b9056c63af6e55d59a2dfdde2f81872f3" + "sha256": "f0d2e2edac2ab7d7751812f7bec189780f1a47054f711517ffaa9d14c84f9934" }, { "path": "docs/ui-contract-inventory/lifecycle.fragment.json", - "sha256": "632ce89ff832f4382fe17dd6040ada896552a6de8c4ca61fac107d92f31acefc" + "sha256": "a8c7fa93404d04c58312a17cf1e722e2c2f2c440e284dc8962ce4fa664c7899f" }, { "path": "docs/ui-contract-inventory/additional-mounted.fragment.json", - "sha256": "22a47c5f101a3c59621513f5b31489d3e651916149f01d547049ffeffafa7bc8" + "sha256": "f2d2920a2fefa4587d260fff09ae4a0af0c1f4b94d286c1c8a15b088d46421f1" } ], - "expandedSha256": "f137a248928c76f89ce7df0dd9d2e8b5fff2da28948ab0fe700b801c65b78500", + "expandedSha256": "3d1de6bf8e531684f7c1b5d62c82dd49daebbbad7be2f871e3249bb3882e50c2", "metadata": { "schemaVersion": 1, "status": "source-backed-provisional; model-and-adapter-coverage-pending", @@ -59,7 +59,7 @@ "evidence": [ "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/index.ts:1", "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationsPanel.tsx:268", - "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3637" + "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3625" ], "proof": { "search": "rg -n 'ConstructionOperationsPanel' ui/packages/loom-ui/src --glob '!**/*.test.*'", @@ -76,7 +76,7 @@ "watchedFiles": [ { "path": "ui/apps/demo/src/demo-route.mjs", - "sha256": "3119d91779bb6ee6cec5e08c2d81bc270e3f9875a8afff2065d6330cec5f9232" + "sha256": "76db6cc77901f7ff6281abfd63a4ae406c2d406b5a2b1c1c53a95023ef2d3a89" }, { "path": "ui/apps/demo/src/main.tsx", @@ -84,15 +84,15 @@ }, { "path": "ui/packages/loom-ui/src/Builder.tsx", - "sha256": "b1cdf7f51dae264a57ef4818467c19aeda7275912d15344bbd89a942af6e90f0" + "sha256": "a4e5408e3653cdf9c0d144325405fa0ac02dbde112584ab8c3590a1e05ce6a1a" }, { "path": "ui/packages/loom-ui/src/api.ts", - "sha256": "b1d02d0681652231cf4e6fa6f9b5cba1df5104b0a32e7429c8f1b5f932418335" + "sha256": "1fa5bc9de431e5c721bcad32a0b6fabc3f3a3a15c7129cb17be13ef508790625" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx", - "sha256": "fc7dd098c4f6cba40b4e24b5d77de756acfa27f8921a311287b5521a9f3d9808" + "sha256": "3e1691e77e5488970d70739764c9d17eb808d0859e1627df1c41b7bc391f0b40" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/builderLoadError.ts", @@ -112,7 +112,7 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/model.ts", - "sha256": "1a288981f3f63dcb256b36afe2d6b17e21500352326a760be356d106564a2aca" + "sha256": "fd87470cae6d156018730d1f674cca837b098fa6316b996009247edd6d702093" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/previewRecovery.ts", @@ -120,7 +120,7 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/reconcileAcceptance.ts", - "sha256": "679a1b1dcbb4c2d22137b2d69dc58c0e1f28acc8662da6f98bfc46d4ee6a40bb" + "sha256": "e617f1a2cc90db4a73286fb7598104e8380b6776aced78bf6082c751d02b48d8" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/reconcileRecovery.ts", @@ -136,7 +136,7 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/routeActions.ts", - "sha256": "e502bed10c9155540a25b12cde706b034e9b2d96919921bc48a00910f716eb2c" + "sha256": "ab98788207abc9d13e8e213ba439de182c61852b9c3e5601e4da0e36b3dfb066" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/serverDraftSync.ts", @@ -200,7 +200,7 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/GuidedGraphWorkspace.tsx", - "sha256": "6640fb5fb310301a4c6b9e4ac4baec8411ebb7708fe20fa0cec5e61ecc52a123" + "sha256": "49a0b690f9e88c92c2cec5f0ca97845a2a7a45f12e4bde62f66586c177cea391" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/InterpretationPanel.tsx", @@ -228,15 +228,15 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/PreviewTable.tsx", - "sha256": "b7c22dfceff422198b9278e5315565b0bcb55348cc5498e88609db2cdb871ebc" + "sha256": "bb48a271f3b5b47091a55ed27cb96dee9660d27bc03156fd973cd415dc5bd8dc" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RouteExtensionPanel.tsx", - "sha256": "fc7cbf9071ee5390a1cd0015ddb1e8993cf11b2b6e03dfe24c155eca96375a86" + "sha256": "0a3c853e2138df119dda66cbebeb3eeb2410ad8e5996959dfa271198994d733f" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowChangePreviewPanel.tsx", - "sha256": "2ef9c7654587042f71ef6c48ddaf2117cd3b7076286fef3f0934ea2eb3953cf0" + "sha256": "ada8601e4200b934a111f427c03ae34443ff058aeac887f9f809e05367f20cbc" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/RowDefinitionPanel.tsx", @@ -252,7 +252,7 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/SourcePickerExplanation.tsx", - "sha256": "a8115a57a659bf0a876d9204c3060a40e6cdfe9c7fe208569841d00fd3829be1" + "sha256": "1677861791e0d46fe42592a0497067b9bad40af3469c45d84a512528a507540c" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/components/TableShapeComparison.tsx", @@ -308,7 +308,7 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationEditor.tsx", - "sha256": "c3a505bc657ecfd3494532d6ba75ab64bed20e2d4c865e15f564b243608583ae" + "sha256": "eb93256726cac1d5a5f8b304e1ca7cd14e2158b5a6fe40c72b173a3b23454df0" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionOperationsPanel.tsx", @@ -316,7 +316,7 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/ConstructionReshapeEditor.tsx", - "sha256": "8f97c58a5ecbe5207d9258cb6ba1e288471ea4d06872b59e8ca4a9a3b6c9a3bf" + "sha256": "a3f66597ebe7cf9d5f81d04f537e7f3a291b359817f24e670750c2949449f006" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/FilterRowsEditor.tsx", @@ -352,7 +352,7 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/existingFieldReuse.ts", - "sha256": "07fdfed45e9c2e4cb8bc9d2818ebf489d76c291111c2488206f4e53f9c4583e6" + "sha256": "30b803a579a24de425870b45cf96d31e1e4f821c231994f2eb227ec8ebc0b8d7" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/index.ts", @@ -364,7 +364,7 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionOperations/sourcePickerExplanations.ts", - "sha256": "d976e4cdb18a3276ddb658a06c452ca371f31e597679c9b94d58ce8ee0a63cc8" + "sha256": "71ac7ac10e9eaea853212e9c3fb64589a5e93d4fd0cf2ebc6cde82023a235bf8" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionColumnSelection.tsx", @@ -376,7 +376,7 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionProposalPreview.tsx", - "sha256": "32c247f3bd482e6c3f0cedd5120d57c1ee28c0a6a2611793e62d222c10782af4" + "sha256": "3ad9aaca16a4410c0889150a025d0311c7a21d08a250e0bb81300b00fc290581" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/ConstructionWorkspace.tsx", @@ -404,7 +404,7 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/constructionApplyOwnership.ts", - "sha256": "13c34fa493dfe6990b07601d8ad650ae550ba8d3649b50cead12c3fda04aa942" + "sha256": "e03636a22ef578eebbd2421c585faa7d1574ce6f4740c13016984d49bc76686b" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/constructionHistory.ts", @@ -412,11 +412,11 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/constructionStages.ts", - "sha256": "cc02183b69e5a0f0973247b6dcc35fd0a26d23dae75fefe356495b1ec5f48abc" + "sha256": "c784b04cf2199d38e493fa66147f502fcbf754907dea5048622184b702860164" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/outputEligibility.ts", - "sha256": "a6de61b692b04447695075470886e7099f9c98fd6ecad718b5e0811e4ade8c60" + "sha256": "45e1dd8fd4c81491895629f7ce82d870b2f53da08261fee638438741f124950b" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/constructionWorkspace/relatedSourceOutputLabel.ts", @@ -444,7 +444,7 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/hooks/useDirtyBeforeUnload.ts", - "sha256": "89b48458a668a7b94def911c413822ee19656af62432ebeef07f044984f02292" + "sha256": "8dbe41674d5867d6d70047050d0fc65cc5bdb40a027e569673f727332bb380a4" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/hooks/useDismissibleLayer.ts", @@ -456,7 +456,7 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/hooks/useGraphLayout.ts", - "sha256": "cfc98f064b04129c0c247120724f943843d985536f9ca6ef2c5c864998725098" + "sha256": "5d68e4cd8b78f66c88970d06af42a6ac6b0141c8f4aa3f3c918e304accaaa93c" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/hooks/useGraphViewportFitter.ts", @@ -464,7 +464,7 @@ }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/hooks/usePortalHost.ts", - "sha256": "2c2efb6c7749289bbd77b8b412c7a2056ba52a3f023860a3f1db7458e1cdf9fe" + "sha256": "958ecee6cf5d91bfbd37395c53373f1adc8881cd1701f3c932a366d74ddd2884" }, { "path": "ui/packages/loom-ui/src/features/ExplorerBuilder/hooks/useTraversalOverflow.ts", @@ -484,7 +484,7 @@ }, { "path": "ui/packages/loom-ui/src/types.ts", - "sha256": "0917431df5cb8cfd52991296aa4dd9bf798143247cd21bae9e8327ce406a7165" + "sha256": "1eff5fab8d41af347dc2e92aeed721d15a5e9e1afecaf0750f05b42a1b4e7246" } ], "mountedOperationSources": [ diff --git a/docs/verification/CORE_PROTOCOL_VERIFICATION_PLAN.md b/docs/verification/CORE_PROTOCOL_VERIFICATION_PLAN.md index c1e3d0d4c..100779c71 100644 --- a/docs/verification/CORE_PROTOCOL_VERIFICATION_PLAN.md +++ b/docs/verification/CORE_PROTOCOL_VERIFICATION_PLAN.md @@ -133,7 +133,7 @@ The current eleven operation-history witnesses are supporting checks. They are n | Public HTTP adapters | `internal/server/openapi_explorer_routes.go` and `internal/server/explorer_authoring_http.go` | | Authoritative state and reconciliation | `internal/explorer/lifecycle/query.go`, `authoring.go` and `draft_restore.go` | | Compilation and receipt validation | `internal/server/explorer_receipt_contract.go`, `explorer_receipt_validation.go`, `internal/explorer/compilation_receipt.go` and the compiler callbacks they invoke | -| Persistence | `internal/explorer/arango/drafts.go` and existing transaction/database checks | +| Persistence | `internal/backends/arango/internal/explorer/drafts.go` and existing transaction/database checks | | Existing verification | `verification/builder-interactions/`, `scripts/verify-builder-formal-core.mjs`, `scripts/lib/formal-*.mjs`, mounted Workspace tests and public server tests | “Compilations API” is a product-area label here. Inventory the actual public routes and internal receipt operations; do not invent a standalone `/compilations` endpoint or model only the endpoint name. diff --git a/examples/meta_gdc_case_matrix.graphql b/examples/meta_gdc_case_matrix.graphql index 77d6ee729..324066cdf 100644 --- a/examples/meta_gdc_case_matrix.graphql +++ b/examples/meta_gdc_case_matrix.graphql @@ -2,7 +2,7 @@ mutation RunGDCDataframe($input: FhirDataframeInput!, $limit: Int) { runFhirDataframe(input: $input, limit: $limit) { columns rowCount - diagnostics { inputResolutionMs requestPreparationMs compilationMs arangoQueryMs rowMaterializationMs resultAssemblyMs totalMs plan { traversalSets sharedTraversalCount requiredMatchReuseCount scopedSharingCandidateGroups scopedSharingCandidateSets potentialSharingOpportunityGroups potentialSharingOpportunitySets optimizationPolicy { name enabled minimumSavings decisions { rule enabled candidateSets estimatedBaselineWork estimatedOptimizedWork estimatedSavings reason } } richSourceReuse { sourceSet aggregateConsumers pivotConsumers sliceConsumers totalConsumers } } } + diagnostics { inputResolutionMs requestPreparationMs compilationMs sourceQueryMs arangoQueryMs rowMaterializationMs resultAssemblyMs totalMs plan { traversalSets sharedTraversalCount requiredMatchReuseCount scopedSharingCandidateGroups scopedSharingCandidateSets potentialSharingOpportunityGroups potentialSharingOpportunitySets optimizationPolicy { name enabled minimumSavings decisions { rule enabled candidateSets estimatedBaselineWork estimatedOptimizedWork estimatedSavings reason } } richSourceReuse { sourceSet aggregateConsumers pivotConsumers sliceConsumers totalConsumers } } } rows } } diff --git a/examples/meta_patient_dataframe.graphql b/examples/meta_patient_dataframe.graphql index b9ad14884..88ee1d02a 100644 --- a/examples/meta_patient_dataframe.graphql +++ b/examples/meta_patient_dataframe.graphql @@ -2,7 +2,7 @@ mutation RunDataframe($input: FhirDataframeInput!, $limit: Int) { runFhirDataframe(input: $input, limit: $limit) { columns rowCount - diagnostics { inputResolutionMs requestPreparationMs compilationMs arangoQueryMs rowMaterializationMs resultAssemblyMs totalMs plan { traversalSets sharedTraversalCount requiredMatchReuseCount scopedSharingCandidateGroups scopedSharingCandidateSets potentialSharingOpportunityGroups potentialSharingOpportunitySets optimizationPolicy { name enabled minimumSavings decisions { rule enabled candidateSets estimatedBaselineWork estimatedOptimizedWork estimatedSavings reason } } richSourceReuse { sourceSet aggregateConsumers pivotConsumers sliceConsumers totalConsumers } } } + diagnostics { inputResolutionMs requestPreparationMs compilationMs sourceQueryMs arangoQueryMs rowMaterializationMs resultAssemblyMs totalMs plan { traversalSets sharedTraversalCount requiredMatchReuseCount scopedSharingCandidateGroups scopedSharingCandidateSets potentialSharingOpportunityGroups potentialSharingOpportunitySets optimizationPolicy { name enabled minimumSavings decisions { rule enabled candidateSets estimatedBaselineWork estimatedOptimizedWork estimatedSavings reason } } richSourceReuse { sourceSet aggregateConsumers pivotConsumers sliceConsumers totalConsumers } } } rows } } diff --git a/experimental/README.md b/experimental/README.md index 9bb9d5467..a094d83a7 100644 --- a/experimental/README.md +++ b/experimental/README.md @@ -7,7 +7,7 @@ used by the quickstart. Start it from the repository root: rtk docker compose -f experimental/docker-compose.yml up -d ``` -The runtime implementation lives under [`internal/store/arango/`](../internal/store/arango/) +The runtime implementation lives under [`internal/backends/arango/internal/store/`](../internal/backends/arango/internal/store/) and [`internal/ingest/`](../internal/ingest/). This is not a home for a second query engine or manually maintained AQL recipes. diff --git a/experimental/docker-compose.yml b/experimental/docker-compose.yml index e59d937bb..b0682e592 100644 --- a/experimental/docker-compose.yml +++ b/experimental/docker-compose.yml @@ -17,3 +17,24 @@ services: - "9000:9000" volumes: - ./clickhouse-data:/var/lib/clickhouse + + postgres18-test: + profiles: ["postgres-test"] + image: postgres:18 + container_name: loom-postgres18-test + environment: + POSTGRES_DB: loom_test + POSTGRES_USER: loom_test + POSTGRES_PASSWORD: loom-test-only + ports: + - "55432:5432" + volumes: + - loom-pg18-test-data:/var/lib/postgresql + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U loom_test -d loom_test"] + interval: 5s + timeout: 3s + retries: 20 + +volumes: + loom-pg18-test-data: diff --git a/generated/fhir/helpers.go b/generated/fhir/helpers.go index 479b092bf..99ad13375 100644 --- a/generated/fhir/helpers.go +++ b/generated/fhir/helpers.go @@ -12,10 +12,8 @@ import ( "github.com/google/uuid" ) -// FHIRComments handles comments that can be a single JSON string or a JSON array of strings. type FHIRComments []string -// UnmarshalJSON unmarshals FHIR comments from a string or array of strings. func (c *FHIRComments) UnmarshalJSON(data []byte) error { if len(data) == 0 { return nil @@ -36,7 +34,6 @@ func (c *FHIRComments) UnmarshalJSON(data []byte) error { return nil } -// MarshalJSON marshals FHIR comments to a single string (if length is 1) or an array of strings. func (c FHIRComments) MarshalJSON() ([]byte, error) { if len(c) == 1 { return json.Marshal(c[0]) @@ -44,7 +41,6 @@ func (c FHIRComments) MarshalJSON() ([]byte, error) { return json.Marshal([]string(c)) } -// ValidateFhirDateTime checks a string against FHIR date-time rules (YYYY-MM-DDThh:mm:ss[Z|(+|-)hh:mm] or partial date). func ValidateFhirDateTime(s string) error { if !strings.Contains(s, "T") { return validatePartialFhirDate(s, "FHIR date-time") @@ -57,15 +53,15 @@ func ValidateFhirDateTime(s string) error { func validatePartialFhirDate(s, typeName string) error { switch len(s) { - case 4: // YYYY + case 4: if _, err := time.Parse("2006", s); err != nil { return fmt.Errorf("value '%s' is not a valid partial %s (YYYY): %w", s, typeName, err) } - case 7: // YYYY-MM + case 7: if _, err := time.Parse("2006-01", s); err != nil { return fmt.Errorf("value '%s' is not a valid partial %s (YYYY-MM): %w", s, typeName, err) } - case 10: // YYYY-MM-DD + case 10: if _, err := time.Parse("2006-01-02", s); err != nil { return fmt.Errorf("value '%s' is not a valid partial %s (YYYY-MM-DD): %w", s, typeName, err) } @@ -75,12 +71,10 @@ func validatePartialFhirDate(s, typeName string) error { return nil } -// ValidateFhirDate checks a string against FHIR date rules (YYYY[-MM[-DD]]). func ValidateFhirDate(s string) error { return validatePartialFhirDate(s, "FHIR date") } -// ValidateFhirTime checks a string against FHIR time rules (HH:MM:SS[.SSSS]). func ValidateFhirTime(s string) error { if len(s) < 8 { return fmt.Errorf("value '%s' is too short for a FHIR time (minimum 8 characters)", s) @@ -112,7 +106,6 @@ func ValidateFhirTime(s string) error { return nil } -// ValidateFhirURI checks a string against FHIR URI rules (must be an absolute URI). func ValidateFhirURI(s string) error { u, err := url.ParseRequestURI(s) if err != nil { @@ -124,7 +117,6 @@ func ValidateFhirURI(s string) error { return nil } -// ValidateFhirUUID checks a string against FHIR UUID rules (RFC 4122). func ValidateFhirUUID(s string) error { if _, err := uuid.Parse(s); err != nil { return fmt.Errorf("value '%s' is not a valid UUID (RFC 4122): %w", s, err) @@ -132,7 +124,6 @@ func ValidateFhirUUID(s string) error { return nil } -// ValidateFhirBinary checks that a string is a valid base64-encoded value. func ValidateFhirBinary(s string) error { if _, err := base64.StdEncoding.DecodeString(s); err != nil { return fmt.Errorf("value is not a valid Base64 encoded string: %w", err) diff --git a/generated/graphql/graph/executor/root_.generated.go b/generated/graphql/graph/executor/root_.generated.go index 53f3ba0f9..60c18f6a9 100644 --- a/generated/graphql/graph/executor/root_.generated.go +++ b/generated/graphql/graph/executor/root_.generated.go @@ -653,6 +653,7 @@ type ComplexityRoot struct { RequestPreparationMs func(childComplexity int) int ResultAssemblyMs func(childComplexity int) int RowMaterializationMs func(childComplexity int) int + SourceQueryMs func(childComplexity int) int TotalMs func(childComplexity int) int } @@ -5730,6 +5731,12 @@ func (e *executableSchema) Complexity(ctx context.Context, typeName, field strin } return e.ComplexityRoot.DataframeQueryDiagnostics.RowMaterializationMs(childComplexity), true + case "DataframeQueryDiagnostics.sourceQueryMs": + if e.ComplexityRoot.DataframeQueryDiagnostics.SourceQueryMs == nil { + break + } + + return e.ComplexityRoot.DataframeQueryDiagnostics.SourceQueryMs(childComplexity), true case "DataframeQueryDiagnostics.totalMs": if e.ComplexityRoot.DataframeQueryDiagnostics.TotalMs == nil { break @@ -20166,10 +20173,6 @@ enum FhirValueMode { DISTINCT } -# Typed, storage-independent dataframe predicates. ` + "`" + `select` + "`" + ` is a FHIR path -# (or a resolved catalog selector), while ` + "`" + `fieldRef` + "`" + ` is optional frontend -# provenance used during scoped field resolution. Values are a strict tagged -# union; exactly one member matching ` + "`" + `kind` + "`" + ` is accepted by recipe validation. enum FhirFilterOperator { EQUALS NOT_EQUALS @@ -20220,8 +20223,6 @@ input FhirFilterInput { select: String! fieldRef: String operator: FhirFilterOperator! - # Scalar selectors do not accept an array quantifier. Repeated selectors - # must opt into ANY, ALL, or NONE explicitly after schema resolution. quantifier: FhirFilterQuantifier values: [FhirFilterValueInput!]! = [] } @@ -20319,9 +20320,6 @@ input FhirDataframeInput { cursor: String } -# Explicit, bounded FHIR graph traversal. Graph mode defaults each edge to -# REQUIRED (unlike the dataframe builder's OPTIONAL default) and returns -# complete node paths rather than flattened rows. input FhirGraphQueryInput { project: String! authResourcePaths: [String!] @@ -20389,13 +20387,11 @@ type FhirDataframeResult { diagnostics: DataframeQueryDiagnostics! } -# Server-side timings for one dataframe request. ` + "`" + `arangoQueryMs` + "`" + ` excludes -# Loom's per-row flattening/delivery time; together those two fields identify -# whether the database plan or result materialization is dominant. type DataframeQueryDiagnostics { inputResolutionMs: Float! requestPreparationMs: Float! compilationMs: Float! + sourceQueryMs: Float! arangoQueryMs: Float! rowMaterializationMs: Float! resultAssemblyMs: Float! @@ -20403,8 +20399,6 @@ type DataframeQueryDiagnostics { plan: DataframeCompilerPlanDiagnostics! } -# Compiler facts for deciding which AQL rewrite to implement next. These are -# not Arango estimates; pair them with a live PROFILE for operator-level cost. type DataframeCompilerPlanDiagnostics { traversalSets: Int! sharedTraversalCount: Int! @@ -24253,6 +24247,8 @@ func (ec *executionContext) childFields_DataframeQueryDiagnostics(ctx context.Co return ec.fieldContext_DataframeQueryDiagnostics_requestPreparationMs(ctx, field) case "compilationMs": return ec.fieldContext_DataframeQueryDiagnostics_compilationMs(ctx, field) + case "sourceQueryMs": + return ec.fieldContext_DataframeQueryDiagnostics_sourceQueryMs(ctx, field) case "arangoQueryMs": return ec.fieldContext_DataframeQueryDiagnostics_arangoQueryMs(ctx, field) case "rowMaterializationMs": diff --git a/generated/graphql/graph/executor/schema.generated.go b/generated/graphql/graph/executor/schema.generated.go index 7a4f3155a..477761451 100644 --- a/generated/graphql/graph/executor/schema.generated.go +++ b/generated/graphql/graph/executor/schema.generated.go @@ -2337,6 +2337,29 @@ func (ec *executionContext) fieldContext_DataframeQueryDiagnostics_compilationMs return graphql.NewScalarFieldContext("DataframeQueryDiagnostics", field, false, false, errors.New("field of type Float does not have child fields")) } +func (ec *executionContext) _DataframeQueryDiagnostics_sourceQueryMs(ctx context.Context, field graphql.CollectedField, obj *model.DataframeQueryDiagnostics) (ret graphql.Marshaler) { + return graphql.ResolveField( + ctx, + ec.OperationContext, + field, + func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return ec.fieldContext_DataframeQueryDiagnostics_sourceQueryMs(ctx, field) + }, + func(ctx context.Context) (any, error) { + return obj.SourceQueryMs, nil + }, + nil, + func(ctx context.Context, selections ast.SelectionSet, v float64) graphql.Marshaler { + return ec.marshalNFloat2float64(ctx, selections, v) + }, + true, + true, + ) +} +func (ec *executionContext) fieldContext_DataframeQueryDiagnostics_sourceQueryMs(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + return graphql.NewScalarFieldContext("DataframeQueryDiagnostics", field, false, false, errors.New("field of type Float does not have child fields")) +} + func (ec *executionContext) _DataframeQueryDiagnostics_arangoQueryMs(ctx context.Context, field graphql.CollectedField, obj *model.DataframeQueryDiagnostics) (ret graphql.Marshaler) { return graphql.ResolveField( ctx, @@ -11122,6 +11145,11 @@ func (ec *executionContext) _DataframeQueryDiagnostics(ctx context.Context, sel if out.Values[i] == graphql.Null { out.Invalids++ } + case "sourceQueryMs": + out.Values[i] = ec._DataframeQueryDiagnostics_sourceQueryMs(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } case "arangoQueryMs": out.Values[i] = ec._DataframeQueryDiagnostics_arangoQueryMs(ctx, field, obj) if out.Values[i] == graphql.Null { diff --git a/generated/graphql/graph/model/models.go b/generated/graphql/graph/model/models.go index b46d26c79..048bc76f8 100644 --- a/generated/graphql/graph/model/models.go +++ b/generated/graphql/graph/model/models.go @@ -131,6 +131,7 @@ type DataframeQueryDiagnostics struct { InputResolutionMs float64 `json:"inputResolutionMs"` RequestPreparationMs float64 `json:"requestPreparationMs"` CompilationMs float64 `json:"compilationMs"` + SourceQueryMs float64 `json:"sourceQueryMs"` ArangoQueryMs float64 `json:"arangoQueryMs"` RowMaterializationMs float64 `json:"rowMaterializationMs"` ResultAssemblyMs float64 `json:"resultAssemblyMs"` diff --git a/generated/loomapi/request_boundary_test.go b/generated/loomapi/request_boundary_test.go index e77ae6e00..e5a8923e2 100644 --- a/generated/loomapi/request_boundary_test.go +++ b/generated/loomapi/request_boundary_test.go @@ -5,15 +5,6 @@ import ( "testing" ) -// TestRequestBodySchemasRejectUnknownMembersAtGeneratedBoundary pins the OpenAPI -// promise that every request body schema (additionalProperties: false) stays -// closed at the generated HTTP boundary, mirroring strict_json.go's contract: -// "keep mutation requests closed at the generated HTTP boundary". An ignored -// unknown member turns a client typo (e.g. a mistyped filter or outputId) into -// a silently dropped field instead of the documented 400 (the shared error -// handler renders binder rejections as INVALID_REQUEST). The -// update-explorer-config body is an x-go-type alias of authoringv2.Workspace -// and is waived in strict_json.go pending that type-boundary decision. func TestRequestBodySchemasRejectUnknownMembersAtGeneratedBoundary(t *testing.T) { requests := map[string]func() any{ "ArtifactPrepareRequest": func() any { return &ArtifactPrepareRequest{} }, @@ -62,10 +53,6 @@ func TestRequestBodySchemasRejectUnknownMembersAtGeneratedBoundary(t *testing.T) } } -// TestNewlyClosedRequestBodiesStillDecodeMinimalPayloads guards against -// over-strictness: the request types that only gained strictness (no semantic -// rules) must still decode their empty envelope; required-field validation -// stays with the documented handler validators. func TestNewlyClosedRequestBodiesStillDecodeMinimalPayloads(t *testing.T) { minimalAllowed := []func() any{ func() any { return &ArtifactPrepareRequest{} }, @@ -97,19 +84,6 @@ func TestNewlyClosedRequestBodiesStillDecodeMinimalPayloads(t *testing.T) { } } -// TestLookupSourceBinderPairingMatchesCanonicalAcceptSet pins the -// strict_json.go LookupSource discriminator against the single authority on -// writable source semantics, authoringv2 ColumnSource.validate: the binder's -// early XOR checks must neither accept a payload the domain layer would -// reject nor narrow a payload the domain accepts (a rejected round-86 variant -// blanket-blocked the identifier arm and broke the UPDATE_COLUMN round-trip -// of resolver-produced identifierBySystem columns). The identifier binding is -// accepted alone, mirroring the writable branch in semantic_types.go, with -// the kind==identifierBySystem gate and ValidateIdentifierBinding left to the -// domain authority the lookup object alone cannot see; the binder is -// deliberately one notch narrower (an explicit empty/blank match or path -// pointer next to identifier is rejected here although TrimSpace would make -// it empty at the domain). func TestLookupSourceBinderPairingMatchesCanonicalAcceptSet(t *testing.T) { accept := []string{ `{"match":"category","path":"category.coding[]","projectionMode":"VALUE"}`, diff --git a/generated/loomapi/strict_json.go b/generated/loomapi/strict_json.go index 0353b115a..f4160ca8b 100644 --- a/generated/loomapi/strict_json.go +++ b/generated/loomapi/strict_json.go @@ -7,11 +7,6 @@ import ( "io" ) -// strictDecodeGenerated keeps documented request bodies closed at the generated -// HTTP boundary (mutation requests plus read/search POST bodies whose schemas -// declare additionalProperties:false). The generated Fiber binder otherwise -// silently drops unknown JSON members before the authoring domain can enforce -// its semantics-version guard. func strictDecodeGenerated(raw []byte, target any) error { decoder := json.NewDecoder(bytes.NewReader(raw)) decoder.DisallowUnknownFields() @@ -118,15 +113,6 @@ func (value *ContributorPredicate) UnmarshalJSON(raw []byte) error { return nil } -// LookupSource enforces the single-meaning pairing of authoringv2 -// (semantic_types.go) before a payload crosses into the domain. The typed -// identifier binding mirrors the domain's writable shape: accepted alone (the -// kind gate and ValidateIdentifierBinding stay the authority there) and -// rejected in any mix, so a payload can never reach the decoder in a form -// that one arm would silently drop or that a later revalidation would reject. -// The arm is kept a notch narrower than the domain on pointer presence: an -// explicit blank match/path alongside identifier is rejected here even though -// the domain would TrimSpace it away. func (value *LookupSource) UnmarshalJSON(raw []byte) error { type wire LookupSource var decoded wire @@ -499,15 +485,6 @@ func (value *SemanticInventoryBrowseRequest) UnmarshalJSON(raw []byte) error { return nil } -// Review follow-up (round 83 adversarial review F1): the remaining plain -// request-body schemas bound by the generated strict adapter. The 9th gap -// reported there, the update-explorer-config body, is an x-go-type alias of -// authoringv2.Workspace whose strictness would also affect persisted-draft -// decoding; closing that boundary needs a type-level decision and is waived -// here with a backlog entry (docs/EXPLORER_BACKLOG.md). - -// AssessRowChangeRequest keeps the documented additionalProperties:false contract at the -// generated HTTP boundary; required-field validation stays in the handlers. func (value *AssessRowChangeRequest) UnmarshalJSON(raw []byte) error { type wire AssessRowChangeRequest var decoded wire @@ -518,8 +495,6 @@ func (value *AssessRowChangeRequest) UnmarshalJSON(raw []byte) error { return nil } -// CandidateSearchRequest keeps the documented additionalProperties:false contract at the -// generated HTTP boundary; required-field validation stays in the handlers. func (value *CandidateSearchRequest) UnmarshalJSON(raw []byte) error { type wire CandidateSearchRequest var decoded wire @@ -530,8 +505,6 @@ func (value *CandidateSearchRequest) UnmarshalJSON(raw []byte) error { return nil } -// CellTraceRequest keeps the documented additionalProperties:false contract at the -// generated HTTP boundary; required-field validation stays in the handlers. func (value *CellTraceRequest) UnmarshalJSON(raw []byte) error { type wire CellTraceRequest var decoded wire @@ -542,8 +515,6 @@ func (value *CellTraceRequest) UnmarshalJSON(raw []byte) error { return nil } -// PopulationMappingRequest keeps the documented additionalProperties:false contract at the -// generated HTTP boundary; required-field validation stays in the handlers. func (value *PopulationMappingRequest) UnmarshalJSON(raw []byte) error { type wire PopulationMappingRequest var decoded wire @@ -554,8 +525,6 @@ func (value *PopulationMappingRequest) UnmarshalJSON(raw []byte) error { return nil } -// PreviewRequest keeps the documented additionalProperties:false contract at the -// generated HTTP boundary; required-field validation stays in the handlers. func (value *PreviewRequest) UnmarshalJSON(raw []byte) error { type wire PreviewRequest var decoded wire @@ -566,8 +535,6 @@ func (value *PreviewRequest) UnmarshalJSON(raw []byte) error { return nil } -// PublishRequest keeps the documented additionalProperties:false contract at the -// generated HTTP boundary; required-field validation stays in the handlers. func (value *PublishRequest) UnmarshalJSON(raw []byte) error { type wire PublishRequest var decoded wire @@ -578,8 +545,6 @@ func (value *PublishRequest) UnmarshalJSON(raw []byte) error { return nil } -// ReconcileRequest keeps the documented additionalProperties:false contract at the -// generated HTTP boundary; required-field validation stays in the handlers. func (value *ReconcileRequest) UnmarshalJSON(raw []byte) error { type wire ReconcileRequest var decoded wire @@ -590,8 +555,6 @@ func (value *ReconcileRequest) UnmarshalJSON(raw []byte) error { return nil } -// RowLineageRequest keeps the documented additionalProperties:false contract at the -// generated HTTP boundary; required-field validation stays in the handlers. func (value *RowLineageRequest) UnmarshalJSON(raw []byte) error { type wire RowLineageRequest var decoded wire diff --git a/go.mod b/go.mod index 0022a029c..f74619715 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/getkin/kin-openapi v0.142.0 github.com/gofiber/fiber/v3 v3.4.0 github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.12.0 github.com/oapi-codegen/runtime v1.6.0 github.com/vektah/gqlparser/v2 v2.5.36 go.yaml.in/yaml/v3 v3.0.5 @@ -40,6 +41,9 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.3 // indirect github.com/kkdai/maglev v0.2.0 // indirect github.com/klauspost/compress v1.19.1 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect @@ -66,6 +70,6 @@ require ( golang.org/x/crypto v0.54.0 // indirect golang.org/x/net v0.57.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.40.0 // indirect + golang.org/x/text v0.41.0 // indirect google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/go.sum b/go.sum index 97c20d353..60dbc8afc 100644 --- a/go.sum +++ b/go.sum @@ -60,8 +60,6 @@ github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvA github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= -github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gofiber/fiber/v3 v3.4.0 h1:F0aND4vwZF7dR7cbvSwFQQEpBU902XHKWxrLsFBkVqw= github.com/gofiber/fiber/v3 v3.4.0/go.mod h1:nAhJfdxUIJJph2tPWPmqWf8QDIN2iiqQiQf3lENZpdk= github.com/gofiber/schema v1.8.3 h1:06ZedxIYjngzc0095PYy7uWnFnbRflWFpikvZH61fDc= @@ -79,6 +77,14 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.12.0 h1:hWwax8YiGT0YiN50U1oIDn5kgR7KC2tG3rmavwpPkxY= +github.com/jackc/pgx/v5 v5.12.0/go.mod h1:u2Fi6no03HCSBYE+ZVdLxbwdu3xyi2vU+K/YNJioSds= +github.com/jackc/puddle/v2 v2.2.3 h1:f8HoREISb3l5eO9coPgK7U3hKLhWGokOFVcThl5jrao= +github.com/jackc/puddle/v2 v2.2.3/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/kkdai/maglev v0.2.0 h1:w6DCW0kAA6fstZqXkrBrlgIC3jeIRXkjOYea/m6EK/Y= github.com/kkdai/maglev v0.2.0/go.mod h1:d+mt8Lmt3uqi9aRb/BnPjzD0fy+ETs1vVXiGRnqHVZ4= @@ -133,6 +139,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -143,8 +150,6 @@ github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/urfave/cli/v3 v3.10.1 h1:7Kx9H50hrHbRbyxgO1KP6/BcbiGRz0uYh5YyQ30JEEY= -github.com/urfave/cli/v3 v3.10.1/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.73.0 h1:ocTOORnBWtJ+P8t/6wAjdkchMzdfHmWx2VD/DPbgZ7s= @@ -165,18 +170,14 @@ golang.org/x/arch v0.29.0 h1:8sSET5wB0+exBm0FGmOtdHMqjlRdV2DRD3/IV6OZgho= golang.org/x/arch v0.29.0/go.mod h1:0X+GdSIP+kL5wPmpK7sdkEVTt2XoYP0cSjQSbZBwOi8= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= -golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= -golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/gqlgen.yml b/gqlgen.yml index 1847a37b6..071ce548f 100644 --- a/gqlgen.yml +++ b/gqlgen.yml @@ -18,9 +18,6 @@ resolver: package: resolver skip_mod_tidy: true -# The generated JSON scalar unmarshaller needs the post-generation correction -# in cmd/gqlgenfix. Skip gqlgen's pre-generation package validation so that -# correction can be applied before the package is compiled. skip_validation: true struct_tag: gqlgen diff --git a/internal/acceptance/fixture.go b/internal/acceptance/fixture.go index 4b5d3b611..62c283606 100644 --- a/internal/acceptance/fixture.go +++ b/internal/acceptance/fixture.go @@ -30,7 +30,6 @@ const ( DefaultFetchTimeout = 5 * time.Minute ) -// FixtureDigest identifies both a fixture lock and its content-addressed cache directory. type FixtureDigest string type RunID string @@ -48,9 +47,6 @@ type SelectionRule struct { Limit int `json:"limit"` } -// FixtureLock is intentionally metadata-only. Resources contains the exact -// IDs, upstream version IDs, and canonical payload digests needed to detect -// drift without redistributing potentially restricted FHIR payloads. type FixtureLock struct { Version int `json:"version"` Endpoint string `json:"endpoint"` @@ -207,9 +203,6 @@ func (f *Fetcher) Acquire(ctx context.Context) (ValidatedFixture, error) { if err := os.MkdirAll(f.CacheDir, 0o755); err != nil { return ValidatedFixture{}, fmt.Errorf("create fixture cache: %w", err) } - // Every process writes a unique temporary directory. The lock is held only - // in-process; the rename is atomic across processes and an already-created - // valid directory wins a race harmlessly. tmp, err := os.MkdirTemp(f.CacheDir, ".fixture-") if err != nil { return ValidatedFixture{}, fmt.Errorf("create fixture staging directory: %w", err) @@ -661,9 +654,6 @@ func minInt(a, b int) int { return b } -// RefreshLock discovers a new metadata-only lock. It is intentionally an -// explicit operation: normal acceptance runs call LoadLock and never rewrite -// the checked-in oracle or lock after an upstream response changes. func RefreshLock(ctx context.Context, endpoint, studyID, path string, client *http.Client, limits Limits) (FixtureLock, error) { limits = limits.normalize() endpoint = strings.TrimRight(strings.TrimSpace(endpoint), "/") diff --git a/internal/acceptance/performance.go b/internal/acceptance/performance.go index 20a8f1d1f..f2c53b436 100644 --- a/internal/acceptance/performance.go +++ b/internal/acceptance/performance.go @@ -16,10 +16,6 @@ const ( ProbeFloorSeconds = 0.1 ) -// PerformanceStageNames is deliberately kept in the same order as the -// acceptance contract. These are end-to-end stages, so adding a new timed -// stage here is an explicit contract change rather than an accidental report -// field becoming a gate. var PerformanceStageNames = []string{ "generation_upload", "explorer_publish", @@ -38,17 +34,9 @@ type PerformanceObservation struct { RepeatCurrentSeconds float64 `json:"repeat_current_seconds,omitempty"` RepeatRatio float64 `json:"repeat_ratio,omitempty"` RepeatSuspected bool `json:"repeat_suspected,omitempty"` - // ReverseSuspected is retained as a wire-compatible name for consumers of - // the initial comparator. It is always the result of comparing the repeat - // current run against the repeat base run when ComparePerformanceRuns is - // used. - ReverseSuspected bool `json:"reverse_suspected"` + ReverseSuspected bool `json:"reverse_suspected"` } -// PerformanceStageSeconds extracts only the four stages covered by the -// performance contract. A missing stage remains absent, allowing callers to -// report a malformed/unsupported base explicitly instead of treating zero as -// a very fast run. func PerformanceStageSeconds(stages []StageReport) map[string]float64 { allowed := make(map[string]struct{}, len(PerformanceStageNames)) for _, name := range PerformanceStageNames { @@ -76,11 +64,6 @@ func performanceFloor(name string) (float64, bool) { } } -// ComparePerformanceRuns applies the two-part regression gate. The first -// pair is the initial (ordered) base/current run. If that pair is suspected, -// callers run both variants again in the opposite order with fresh databases -// and pass that pair as repeatBase/repeatCurrent. Confirmation is therefore -// always against the repeat's base, not a value observed before the repeat. func ComparePerformanceRuns(base, current, repeatBase, repeatCurrent map[string]float64) []PerformanceObservation { names := map[string]bool{} for _, name := range PerformanceStageNames { @@ -122,9 +105,6 @@ func ComparePerformanceRuns(base, current, repeatBase, repeatCurrent map[string] return result } -// ComparePerformance is retained for callers that already have a reverse -// current observation but no reverse base. New callers should use -// ComparePerformanceRuns so confirmation is based on the repeat's base. func ComparePerformance(base, current, reverse map[string]float64) []PerformanceObservation { return ComparePerformanceRuns(base, current, base, reverse) } diff --git a/internal/acceptance/runner.go b/internal/acceptance/runner.go index a0292841c..1dd5f5f8b 100644 --- a/internal/acceptance/runner.go +++ b/internal/acceptance/runner.go @@ -114,9 +114,6 @@ var ( errRunnerUninitialized = errors.New("acceptance runner constructed without New") ) -// validateRequired holds the invariants New enforces at construction. Run -// re-checks them at entry so a Runner built as a struct literal (bypassing -// New) faults with a named error instead of dereferencing nil fields. func (cfg Config) validateRequired() error { if cfg.Target == nil { return errTargetRequired diff --git a/internal/acceptance/scenario.go b/internal/acceptance/scenario.go index a25f24575..27a386c07 100644 --- a/internal/acceptance/scenario.go +++ b/internal/acceptance/scenario.go @@ -20,10 +20,9 @@ import ( "strings" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" dfpublication "github.com/calypr/loom/internal/dataframe/publication" - publicationarango "github.com/calypr/loom/internal/dataframe/publication/arango" "github.com/calypr/loom/internal/explorer/authoringv2" - arangostore "github.com/calypr/loom/internal/store/arango" clickhousestore "github.com/calypr/loom/internal/store/clickhouse" ) @@ -249,11 +248,12 @@ func compareSummary(payload map[string]any, want map[string]int) error { } func directArangoCounts(ctx context.Context, cfg ScenarioConfig) (map[string]int, error) { - client, err := arangostore.Open(ctx, cfg.Connections.ArangoURL, cfg.Namespace.ArangoDatabase) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: cfg.Connections.ArangoURL, Database: cfg.Namespace.ArangoDatabase}) if err != nil { return nil, fmt.Errorf("open acceptance ArangoDB: %w", err) } - defer client.Close(ctx) + defer arangoBackend.Close(ctx) + client := arangoBackend.Client result := map[string]int{} for typ := range cfg.Fixture.Counts { count := 0 @@ -570,8 +570,6 @@ func decodeAcceptanceCatalog(state map[string]any, project, explorerID, generati RepeatedBoundaries: candidate.RepeatedBoundaries, }) } - // These scope fields are omitted from the public catalog representation; - // bind them to the request and the Explorer created above before validation. if catalog.SourceGeneration != generation { return empty, fmt.Errorf("Builder catalog generation %q does not match acceptance generation %q", catalog.SourceGeneration, generation) } @@ -581,12 +579,6 @@ func decodeAcceptanceCatalog(state map[string]any, project, explorerID, generati return catalog, nil } -// The acceptance migration reads only node identity and field-candidate -// identity/type/repetition metadata. Decode that public projection explicitly: -// CatalogSnapshot cannot be unmarshaled directly because ConstructionChoice -// contains a closed interface source. Complete and any explicit truncated -// flag remain mandatory guards so this reduced view cannot consume a partial -// catalog. type acceptanceCatalogWire struct { AuthorizationScopeDigest string `json:"authorizationScopeDigest"` Candidates []acceptanceCatalogCandidateWire `json:"candidates"` @@ -668,15 +660,12 @@ func successfulExecution(value map[string]any, pub publication) error { } func verifyClickHouse(ctx context.Context, cfg ScenarioConfig, pub publication, oracle Oracle) (map[string]any, error) { - arangoClient, err := arangostore.Open(ctx, cfg.Connections.ArangoURL, cfg.Namespace.ArangoDatabase) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: cfg.Connections.ArangoURL, Database: cfg.Namespace.ArangoDatabase}) if err != nil { return nil, fmt.Errorf("open publication registry: %w", err) } - defer arangoClient.Close(ctx) - registry, err := publicationarango.New(arangoClient) - if err != nil { - return nil, err - } + defer arangoBackend.Close(ctx) + registry := arangoBackend.PublishedRegistry record, err := registry.GetExecution(ctx, pub.ExecutionID) if err != nil { return nil, fmt.Errorf("resolve publication execution %s: %w", pub.ExecutionID, err) diff --git a/internal/acceptance/scenario_test.go b/internal/acceptance/scenario_test.go index a9a607d05..f08abc86b 100644 --- a/internal/acceptance/scenario_test.go +++ b/internal/acceptance/scenario_test.go @@ -284,7 +284,6 @@ func TestMigrateLegacyWorkspaceForCurrentAPIUsesScopedCatalogAndReusesPublishBod t.Fatalf("migrated body predicates = %d, want 7", got) } - // Match RunScenario's idempotency path: publish the one selected body twice. for range 2 { if _, err := publishWorkspace(context.Background(), cfg, client, migratedBody); err != nil { t.Fatal(err) diff --git a/internal/api/authpolicy/policy.go b/internal/api/authpolicy/policy.go index 276d7102c..3cf64cea9 100644 --- a/internal/api/authpolicy/policy.go +++ b/internal/api/authpolicy/policy.go @@ -1,5 +1,3 @@ -// Package authpolicy centralizes authorization-sentinel classification while -// retaining operation-specific concealment and project-level policy. package authpolicy import ( @@ -20,9 +18,6 @@ const ( OperationConcealedLookup Operation = "concealed_lookup" ) -// Classify maps shared authorization and dependency sentinels according to -// operation context. Unknown errors are returned unchanged for the caller's -// broader dependency policy. func Classify(err error, operation Operation) error { if err == nil { return nil diff --git a/internal/api/bulk/load/cross_project_dataset_authorization_e2e_test.go b/internal/api/bulk/load/cross_project_dataset_authorization_e2e_test.go index 95c795a63..04e9fa49b 100644 --- a/internal/api/bulk/load/cross_project_dataset_authorization_e2e_test.go +++ b/internal/api/bulk/load/cross_project_dataset_authorization_e2e_test.go @@ -17,14 +17,6 @@ import ( "github.com/calypr/loom/internal/ingest" ) -// F-B27 durable control: scoped dataset-mutation gates must be project-bound. -// Threat model matches F-B20/F-B21: calypr/scoped-principal deployments -// (basic/noAuth installs AllowAll and is the single-operator posture). The -// gate under test is the production authscope.ScopeResolver decision that -// CreateDatasetGeneration (operations.go:60) and ActivateDatasetGeneration -// (operations.go:89) call as their ONLY authorization; ingest-side activation -// (ingest/generation_orchestration.go Activate) has no principal at all. - type stubResourceAccess struct{ writePaths []string } func (s stubResourceAccess) GetAllowedResources(_ context.Context, _, _, _ string) ([]string, error) { @@ -196,9 +188,6 @@ func TestCrossProjectDatasetActivateGateIsProjectBound(t *testing.T) { } } -// Contrast controls (green in both pre-fix and repaired postures): the gate -// exists and legitimate owners keep working. - func TestDatasetPublishControlsSameProjectOwnerAndEmptyGrant(t *testing.T) { ownerLoaderCalls := 0 loader := func(_ context.Context, _ GenerationLoadRequest, _ ingest.EventSink) (ingest.LoadSummary, error) { @@ -222,10 +211,6 @@ func TestDatasetPublishControlsSameProjectOwnerAndEmptyGrant(t *testing.T) { } } -// Bootstrap guard for REPAIR-8: a project with no recorded catalog paths yet -// (first-ever generation) must stay publishable — the repair may only require -// membership among EXISTING project paths, never forbid empty catalogs. - func TestDatasetPublishNewProjectBootstrapRemainsPossible(t *testing.T) { loaderCalls := 0 loader := func(_ context.Context, _ GenerationLoadRequest, _ ingest.EventSink) (ingest.LoadSummary, error) { diff --git a/internal/api/bulk/load/errors.go b/internal/api/bulk/load/errors.go index 73bb4d778..3e713f9cf 100644 --- a/internal/api/bulk/load/errors.go +++ b/internal/api/bulk/load/errors.go @@ -8,8 +8,6 @@ import ( "github.com/calypr/loom/internal/ingest" ) -// normalizeError translates ingest-owned failures at the ingest HTTP boundary -// so the shared HTTP package does not depend on the loader implementation. func normalizeError(err error) error { if err == nil { return nil diff --git a/internal/api/bulk/load/fb22_ingress_control_test.go b/internal/api/bulk/load/fb22_ingress_control_test.go index c8bf01e26..e5153c70c 100644 --- a/internal/api/bulk/load/fb22_ingress_control_test.go +++ b/internal/api/bulk/load/fb22_ingress_control_test.go @@ -13,35 +13,6 @@ import ( "github.com/calypr/loom/internal/ingest" ) -// F-7R3 durable control (REPAIR-18 U2 / REPAIR-18b): CreateDatasetGeneration -// ingress (operations.go) must refuse an auth_resource_path that -// authscope.NormalizeAuthResourcePath would silently blank (control bytes, -// e.g. "a\x00b") in EVERY authorizer posture — no silent fallback. -// -// The guard landed at the ingress (REPAIR-18b): after the trim and BEFORE -// AuthorizeWrite, -// -// if authResourcePath != "" && authscope.NormalizeAuthResourcePath(authResourcePath) == "" { -// return nil, dataframeerrors.NewError(dataframeerrors.CodeInvalidRequest, "") -// } -// -// A clean non-empty path normalizes to itself (guard is a no-op); a genuine -// empty stays "" and remains accepted as the legitimate single-operator -// posture; only a non-empty value that collapses to "" (control bytes and -// other all-junk encodings) is refused with CodeInvalidRequest, reaching the -// AllowAll posture (shipped noAuth/single-operator), which never calls the -// ScopeResolver, as well as the scoped posture, where the guard now fires -// before authorizer consultation. -// -// PRE-FIX behavior, pinned here as history: under AllowAll, AuthorizeWrite -// returned nil, operations.go normalized the control-byte value to "" and -// ingested it with no error (CONFIRMED-PENDING, VERDICT=CONFIRMED-PENDING in -// the pre-guard runs of this file). Under scoped authorization the control -// byte was refused as forbidden by the resolver instead, i.e. the two -// postures diverged on the same input. The guards below pin the unified -// rejection; the genuine-empty controls pin that the guard did NOT swallow -// the single-operator posture while "fixing" it. - func TestFb22ScopedIngressDeniesControlByteAuthResourcePath(t *testing.T) { var requests []GenerationLoadRequest loader := func(_ context.Context, req GenerationLoadRequest, _ ingest.EventSink) (ingest.LoadSummary, error) { @@ -81,12 +52,6 @@ func TestFb22ScopedIngressDeniesEmptyAuthResourcePath(t *testing.T) { stubReleases{}, ) - // Empty-string direct case: the ingress guard must leave the genuine - // empty encoding untouched (pre-authorization it is the legitimate - // single-operator posture), so the scoped refusal here still comes from - // the resolver's own membership requirement, as forbidden — NOT from the - // control-byte guard. If this flips to INVALID_REQUEST the guard started - // swallowing genuine-empty instead of only blankable non-empty values. _, err := handler.CreateDatasetGeneration(context.Background(), "projA", "g1", multipartGenerationBodyOwner(t, ""), fb22Principal()) if !errors.Is(err, authscope.ErrForbidden) { t.Fatalf("VERDICT=UNEXPECTED: scoped empty auth path must stay a forbidden refusal from the authorizer, got %v", err) @@ -132,10 +97,6 @@ func TestFb22AllowAllIngressRejectsControlByteAuthResourcePath(t *testing.T) { _, err := handler.CreateDatasetGeneration(context.Background(), "P1", "g1", multipartGenerationBodyOwner(t, "a\x00b"), fb22Principal()) - // REPAIR-18b contract: the shipped single-operator/noAuth posture must - // refuse the control-byte path with CodeInvalidRequest at the ingress, - // not blank it to "" and ingest (pre-guard behavior pinned in the file - // header as CONFIRMED-PENDING). userErr, ok := dataframeerrors.AsUserError(err) if !ok || userErr.Code() != string(dataframeerrors.CodeInvalidRequest) { t.Fatalf("VERDICT=REGRESSION: AllowAll control-byte auth path must be refused with INVALID_REQUEST before authorizer/ingest, got %v", err) @@ -144,10 +105,6 @@ func TestFb22AllowAllIngressRejectsControlByteAuthResourcePath(t *testing.T) { t.Fatalf("VERDICT=REGRESSION: refused control-byte publish reached ingest (silent blank shipped): %+v", captured) } - // Non-vacuity, same mechanism as TestFb22ScopedIngressAllowsLegitAuthResourcePath: - // the same handler and recorded loader MUST still ingest a control-free - // publish exactly once, so the zero-ingest assertion above is a real - // refusal of this input, not a loader that never runs. if _, err := handler.CreateDatasetGeneration(context.Background(), "P1", "g2", multipartGenerationBodyOwner(t, ""), fb22Principal()); err != nil { t.Fatalf("VERDICT=UNEXPECTED: allow-all control publish (genuine empty) refused, gate is vacuous: %v", err) } @@ -164,12 +121,6 @@ func TestFb22AllowAllIngressEmptyAuthResourcePathPosture(t *testing.T) { return ingest.LoadSummary{}, nil }) - // The genuine-empty encoding is the legitimate single-operator posture: - // with no authorizer the empty path is accepted and stored. This must - // survive the ingress guard — the trim keeps whitespace-only form values - // in this accepted shape, and the guard's authResourcePath != "" clause - // exists precisely so genuine-empty is never rejected as a blanked - // control byte. _, err := handler.CreateDatasetGeneration(context.Background(), "P1", "g1", multipartGenerationBodyOwner(t, ""), fb22Principal()) if err != nil { t.Fatalf("VERDICT=UNEXPECTED: AllowAll genuine empty path must be accepted, got %v", err) @@ -209,28 +160,3 @@ func fb22AllowAllHandler(t *testing.T, loader GenerationLoader) *Handler { } return handler } - -// RESOLVED(REPAIR-18 U2 / F-7R3 -> REPAIR-18b): The contract "no silent -// fallback at boundaries" required CreateDatasetGeneration to REJECT an -// auth_resource_path carrying a control byte in EVERY authorizer posture, -// not to let authscope.NormalizeAuthResourcePath blank it at the post- -// authorization store step. Under the AllowAll posture (shipped -// noAuth/single-operator) the ScopeResolver is never consulted, so the -// reject landed at the ingress, before the value reaches the authorizer or -// is normalized and staged — NOT inside internal/authscope/scope_resolver.go -// (REPAIR-15's file), which only gates the scoped posture. The landed guard -// (operations.go, after the trim and before AuthorizeWrite): -// -// if authResourcePath != "" && authscope.NormalizeAuthResourcePath(authResourcePath) == "" { -// return nil, dataframeerrors.NewError(dataframeerrors.CodeInvalidRequest, "") -// } -// -// A clean non-empty path normalizes to itself (guard is a no-op); a genuine -// empty stays "", matching the accepted single-operator posture (pinned by -// the two AllowAll and scoped genuine-empty controls above, including the -// whitespace post-trim shape); only a non-empty value collapsing to "" is -// refused with CodeInvalidRequest — reaching every posture, AllowAll -// included. The pre-guard divergence (scoped=forbidden vs -// AllowAll=blank-and-ingest, VERDICT=CONFIRMED-PENDING) is preserved in the -// file header and in /tmp/opencode/fb18b-red.log as the RED evidence this -// guard was validated against. diff --git a/internal/api/bulk/load/form_spillover_test.go b/internal/api/bulk/load/form_spillover_test.go index 0b4815704..a67a0e551 100644 --- a/internal/api/bulk/load/form_spillover_test.go +++ b/internal/api/bulk/load/form_spillover_test.go @@ -1,39 +1,5 @@ package load -// Durable regression for the bulk-load multipart spillover leak (discovery -// round 73) and for the single-write staging contract that replaced the -// intermediate copy. -// -// The handler used to parse the request with multipart.Reader.ReadForm(32<<20): -// every byte beyond the 32 MiB in-memory budget was spooled to a temporary -// file under os.TempDir, and the shipped handler disposed only its own staged -// copy (defer os.RemoveAll(stagedDir)) and never called form.RemoveAll, so -// every dataset-generation load larger than the budget left full-size copies -// in /tmp for the container's lifetime: unbounded disk growth under normal -// authenticated use. That leak was repaired by adding the disposal (commit -// d2ff3d037) and then removed at the source: CreateDatasetGeneration now parses -// with stageGenerationBundle, which streams each file part once into the -// loader's own staging directory, so no multipart-* spill file is created at -// all. -// -// The assertions are behavior-level and cover three shapes: -// - after a load whose payload provably exceeds the old spillover budget -// (two 20 MiB .ndjson files through the real handler, with the loader -// faked), TMPDIR holds no orphan multipart temp files and the staged -// loader copies survive intact with their content; -// - at load time the staged directory is the only copy of the payload on -// disk, and the peak bytes in the temp root equal the payload exactly — -// what a second write would necessarily exceed; -// - the error-path companion covers the validation-rejection returns, which -// parsed first and returned before any staging cleanup existed. -// -// TestCreateDatasetGenerationTemporaryBytesBeforeAfter puts the shipped -// ReadForm path and the streaming path side by side on the same bounded -// payload and reports the raw bytes and elapsed time for each. It asserts only -// the storage claim (the duplicate copy is gone); it deliberately does not -// assert a speedup, because elapsed time on a tmpfs-scale scratch directory is -// not evidence about production hosts. - import ( "bytes" "context" @@ -52,20 +18,10 @@ import ( "github.com/calypr/loom/internal/ingest" ) -// spilloverFileBytes is 20 MiB per file; two files total 40 MiB, which is -// strictly greater than the handler's historical 32<<20 ReadForm budget, so -// the old path spooled at least one whole file to disk regardless of how the -// budget was apportioned across parts. const spilloverFileBytes = 20 * 1024 * 1024 -// measurementReps is how many times each path runs in the before/after -// comparison. Enough to show the numbers are stable, cheap enough to stay a -// focused test. const measurementReps = 3 -// spilloverRunner verifies staged integrity at load time — after the handler -// returns, its cleanup removes the staged directory by design — and records -// what else occupied the temp root while the loader ran. type spilloverRunner struct { project string stagedAt []int @@ -100,16 +56,12 @@ func (r *spilloverRunner) RunGeneration(_ context.Context, req GenerationLoadReq return ingest.LoadSummary{Files: 2}, nil } -// tempRootSnapshot is the temp root as one actor saw it: the bytes in -// multipart spill files, and everything outside the staged bundle. type tempRootSnapshot struct { spillBytes int64 spillNames []string otherBytes int64 } -// snapshotTempRoot measures the temp root while stagingDir still holds the -// payload. Everything outside the staged bundle is a candidate duplicate. func snapshotTempRoot(stagingDir string) (tempRootSnapshot, error) { var snapshot tempRootSnapshot root := os.TempDir() @@ -135,7 +87,6 @@ func snapshotTempRoot(stagingDir string) (tempRootSnapshot, error) { return snapshot, nil } -// dirBytes sums the sizes of every regular file at or below path. func dirBytes(path string) (int64, error) { var total int64 err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { @@ -150,9 +101,6 @@ func dirBytes(path string) (int64, error) { return total, err } -// tempRootSampler polls the temp root while a request is in flight and keeps -// the high-water mark. It reports peak occupancy, not bytes written; the -// deterministic evidence is the loader-time snapshot. type tempRootSampler struct { root string peak atomic.Int64 @@ -179,15 +127,11 @@ func newTempRootSampler(root string) *tempRootSampler { return sampler } -// stop ends sampling and returns the high-water mark. The sampler runs for the -// whole request, so a peak it never reaches is a peak that never existed; the -// deterministic at-load snapshot carries the byte-equality assertions. func (s *tempRootSampler) stop() int64 { s.closed.Do(func() { close(s.done) }) return s.peak.Load() } -// stagingRun is one measured request through one staging path. type stagingRun struct { path string rep int @@ -206,8 +150,6 @@ func (r stagingRun) String() string { float64(r.elapsed.Microseconds())/1000) } -// measureStagingRun drives the 40 MiB payload through one path with a private -// temp root, sampling the temp root while the request is in flight. func measureStagingRun(t *testing.T, path string, rep int, baseline bool) stagingRun { t.Helper() tempDir := t.TempDir() @@ -317,8 +259,6 @@ func TestCreateDatasetGenerationLeaksNoSpilloverTempFiles(t *testing.T) { t.Fatalf("multipart spillover temp files orphaned in TMPDIR after load: %v", orphans) } - // Positive companion: the loader saw both staged copies intact at load - // time — staging disposes of nothing the load consumes. if runner.project != "P1" || len(runner.stagedAt) != 2 { t.Fatalf("loader view of the bundle: project=%s staged=%v", runner.project, runner.stagedAt) } @@ -328,10 +268,6 @@ func TestCreateDatasetGenerationLeaksNoSpilloverTempFiles(t *testing.T) { } } - // Single-write proof: while the loader ran, the payload existed exactly - // once on disk. The old path (ReadForm + io.Copy into the staging - // directory) held the 40 MiB staged bundle AND a ~20 MiB multipart spill - // file simultaneously, and the spill alone trips these assertions. if runner.spillsAtLoad != nil { t.Fatalf("duplicate multipart spill present while the loader ran: %v", runner.spillsAtLoad) } @@ -345,22 +281,9 @@ func TestCreateDatasetGenerationLeaksNoSpilloverTempFiles(t *testing.T) { t.Logf("SPLILLOVER_DISPOSED_ORPHANS=0 STAGED_AT_LOAD_TIME=2 DUPLICATE_TEMP_BYTES=0 loaderProject=%s", runner.project) } -// TestCreateDatasetGenerationTemporaryBytesBeforeAfter is the reproducible -// before/after measurement for this change. The same 40 MiB payload goes -// through the shipped ReadForm path and the streaming path, alternating, so -// neither ordering nor a warm page cache can decide the result. -// -// Asserted (storage): the streaming path never creates a multipart spill and -// its temp root holds exactly the staged payload; the shipped path does create -// a full-size duplicate, and its temp root holds strictly more than the -// payload. -// -// Not asserted (time): elapsed time is recorded and printed raw only. A -// speedup is not claimed from this test. func TestCreateDatasetGenerationTemporaryBytesBeforeAfter(t *testing.T) { var before, after []stagingRun for rep := 1; rep <= measurementReps; rep++ { - // Alternate the order so any ordering effect lands on both paths. if rep%2 == 0 { after = append(after, measureStagingRun(t, "streaming", rep, false)) } @@ -436,8 +359,6 @@ func medianMillis(runs []stagingRun) float64 { return values[len(values)/2] } -// tempFSType names the filesystem backing the measurement directory, so a -// recorded number can be read against its environment. func tempFSType() string { if data, err := os.ReadFile("/proc/mounts"); err == nil { root := os.TempDir() @@ -463,9 +384,6 @@ func TestCreateDatasetGenerationSpilloverDisposedOnValidationRejection(t *testin handler := spilloverHandler(t, runner) body, boundary := spilloverMultipart(t, true) - // Empty generation: the handler reads (and spilled, on the old path) the - // form before the validation return, so this path leaked even earlier than - // the success path. if _, err := handler.CreateDatasetGeneration(context.Background(), "P1", " ", multipart.NewReader(bytes.NewReader(body), boundary), nil); err == nil { t.Fatal("expected validation rejection for blank generation") } diff --git a/internal/api/bulk/load/generation_staging.go b/internal/api/bulk/load/generation_staging.go index 74906acc1..874416482 100644 --- a/internal/api/bulk/load/generation_staging.go +++ b/internal/api/bulk/load/generation_staging.go @@ -13,15 +13,9 @@ import ( ) const ( - // generationFormMaxMemory is the budget the handler historically passed to - // multipart.Reader.ReadForm. readForm adds a fixed 10 MiB reserve on top of - // it for non-file parts, and both limits decide which parts stay in memory. generationFormMaxMemory = int64(32 << 20) formMemoryReserve = int64(10 << 20) - // Budget accounting mirrored from mime/multipart's readForm so the - // streaming stager makes the same in-memory-vs-streaming, message-too-large, - // part-count and header-count decisions as the ReadForm call it replaces. formMapEntryOverhead = 200 formFileHeaderSize = 100 formMIMEHeaderBaseSize = 400 @@ -30,15 +24,9 @@ const ( formDefaultMaxHeaderValues = 10000 formCopyBufferSize = 32 * 1024 - // generationFileField is the multipart field the loader bundle is read - // from; file parts under any other field name are not part of the bundle. generationFileField = "file" ) -// formLimits are the parse limits mime/multipart's readForm applies, including -// the ones an operator can retune through GODEBUG. Reading them here keeps the -// stager and the parser it replaces in lockstep when a deployment sets -// multipartmaxparts or multipartmaxheaders. type formLimits struct { maxFileMemoryBytes int64 maxMemoryBytes int64 @@ -53,8 +41,6 @@ func generationFormLimits() formLimits { maxParts: formDefaultMaxParts, maxHeaders: formDefaultMaxHeaderValues, } - // Same rule as readForm: an override only applies when it parses, so a - // malformed value keeps the default instead of tightening the limit to zero. if value, err := strconv.Atoi(godebugValue("multipartmaxparts")); err == nil && value >= 0 { limits.maxParts = value } @@ -64,9 +50,6 @@ func generationFormLimits() formLimits { return limits } -// godebugValue returns the last value assigned to name in $GODEBUG, mirroring -// how internal/godebug (not importable) resolves a setting: comma-separated -// pairs, later ones winning, with any "#pattern" bisect suffix stripped. func godebugValue(name string) string { value, found := "", false for _, setting := range strings.Split(os.Getenv("GODEBUG"), ",") { @@ -83,35 +66,20 @@ func godebugValue(name string) string { return value } -// multipartParseError marks a failure raised while parsing the multipart -// stream. mime/multipart returns unexported errors for these, and the handler -// must classify them as invalid input rather than as a staging fault. type multipartParseError struct{ err error } func (e *multipartParseError) Error() string { return e.err.Error() } func (e *multipartParseError) Unwrap() error { return e.err } -// generationBundle is a parsed generation upload: the scalar form values in -// ReadForm's first-value-wins shape, plus the bundle already staged in the -// directory the loader consumes. -// -// stagingErr carries the first file-part fault — an invalid name, a duplicate -// name, or a failed staged write — and is reported only by ParseStagingError, -// which the handler calls after scalar validation and authorization. That is -// where the replaced path raised it: ReadForm parsed the entire body before -// stageGenerationFiles ever inspected a name, so a denied or otherwise invalid -// request reported its scalar-level or transport-level fault instead of the -// file fault. type generationBundle struct { Values map[string][]string StagedDir string - fileParts int // bundle-field file parts, in multipart order, valid or not + fileParts int pending []pendingGenerationFile stagingErr error } -// ParseStagingError returns the deferred file-part fault, or nil. func (b *generationBundle) ParseStagingError() error { if b == nil { return nil @@ -124,9 +92,6 @@ type pendingGenerationFile struct { content []byte } -// Cleanup removes every staged file. It is safe to call more than once and on -// a nil bundle, and it is the only disposal an upload needs: no intermediate -// multipart temporary copy exists to remove. func (b *generationBundle) Cleanup() { if b == nil { return @@ -138,9 +103,6 @@ func (b *generationBundle) Cleanup() { } } -// FileCount is the number of file parts under the bundle field, in multipart -// order. It counts parts the stager later rejects by name, as -// len(form.File["file"]) did. func (b *generationBundle) FileCount() int { if b == nil { return 0 @@ -148,30 +110,12 @@ func (b *generationBundle) FileCount() int { return b.fileParts } -// noteFault records the first file-part fault. Later faults are already -// shadowed by it, as they were when stageGenerationFiles returned at the first -// offending header. func (b *generationBundle) noteFault(err error) { if b.stagingErr == nil { b.stagingErr = err } } -// stageGenerationBundle parses a multipart generation upload in one pass and -// writes every file part straight into the directory the loader consumes, so -// an uploaded byte reaches temporary storage once. It mirrors mime/multipart -// readForm's memory budgeting, part and header limits, value accumulation and -// error values; the deliberate difference is that a part past the budget -// readForm spooled to disk is streamed into the staged file instead of into a -// multipart-* temporary file that a second copy then read back. -// -// A part that stays within the budget is buffered and written when the next -// scalar field arrives or at the end of the stream, which is no later than the -// point where the replaced path handed the same bytes to the stager. File-part -// faults are deferred to ParseStagingError so the handler keeps the shipped -// precedence: parse, scalar validation, authorization, then the file fault. -// Whatever the failure, Cleanup leaves nothing behind. A nil reader yields a -// nil bundle and a nil error, which is how the handler saw an absent body. func stageGenerationBundle(body *multipart.Reader) (*generationBundle, error) { if body == nil { return nil, nil @@ -185,16 +129,10 @@ func stageGenerationBundle(body *multipart.Reader) (*generationBundle, error) { seen := make(map[string]struct{}) var copyBuf []byte - // A parse failure disposes of what has been staged so far. A file-part - // fault does not: it is reported after authorization, and the handler's - // cleanup covers the bundle either way. failParse := func(err error) (*generationBundle, error) { bundle.Cleanup() return nil, &multipartParseError{err} } - // discard drains a part that will not be staged, keeping readForm's budget - // accounting for it intact. readForm returned a failed drain of an - // over-budget part as a parse error, so this does too. discard := func(part *multipart.Part) error { _, err := io.Copy(io.Discard, part) part.Close() @@ -220,8 +158,6 @@ func stageGenerationBundle(body *multipart.Reader) (*generationBundle, error) { part.Close() continue } - // Several values for one key share a map entry, but a uniform - // per-value overhead keeps the accounting identical to readForm. maxMemoryBytes -= int64(len(name)) + formMapEntryOverhead if maxMemoryBytes < 0 { part.Close() @@ -241,8 +177,6 @@ func stageGenerationBundle(body *multipart.Reader) (*generationBundle, error) { return failParse(errors.New("multipart: message too large")) } bundle.Values[name] = append(bundle.Values[name], value.String()) - // A scalar field means every earlier file is fully visible to the - // handler, so writing them now is no later than the replaced path. if err := bundle.flush(); err != nil { bundle.noteFault(err) } @@ -262,17 +196,11 @@ func stageGenerationBundle(body *multipart.Reader) (*generationBundle, error) { return failParse(errors.New("multipart: message too large")) } - // Only the "file" field is the bundle; file parts under any other name - // are charged against the budget, as ReadForm charged them, then - // discarded. bundled := name == generationFileField if bundled { bundle.fileParts++ } - // readForm read and budgeted every part's content before any file name - // was inspected, so the charge applies to parts the bundle rejects: - // their size still decides whether a later part trips the limit. var content bytes.Buffer n, err := io.CopyN(&content, part, maxFileMemoryBytes+1) if err != nil && err != io.EOF { @@ -301,8 +229,6 @@ func stageGenerationBundle(body *multipart.Reader) (*generationBundle, error) { } } if !stage { - // readForm still read this part's content, so a failing read is a - // parse failure here too rather than something to swallow. if err := discard(part); err != nil { return failParse(err) } @@ -313,9 +239,6 @@ func stageGenerationBundle(body *multipart.Reader) (*generationBundle, error) { bundle.pending = append(bundle.pending, pendingGenerationFile{name: stagedName, content: content.Bytes()}) continue } - // Past the budget ReadForm kept on disk: stream the rest into the - // staged file, the only copy the loader will read. The part stays - // open until the remainder has been copied. dst, err := bundle.create(stagedName) if err != nil { bundle.noteFault(err) @@ -331,8 +254,6 @@ func stageGenerationBundle(body *multipart.Reader) (*generationBundle, error) { if copyBuf == nil { copyBuf = make([]byte, formCopyBufferSize) } - // os.File.ReadFrom would allocate its own buffer; writerOnly keeps - // io.Copy on ours, as readForm does. type writerOnly struct{ io.Writer } _, copyErr := io.CopyBuffer(writerOnly{dst}, part, copyBuf) part.Close() @@ -352,7 +273,6 @@ func stageGenerationBundle(body *multipart.Reader) (*generationBundle, error) { return bundle, nil } -// create opens a staged file for a part too large to buffer. func (b *generationBundle) create(name string) (*os.File, error) { if err := b.ensureDir(); err != nil { return nil, err @@ -360,9 +280,6 @@ func (b *generationBundle) create(name string) (*os.File, error) { return os.Create(filepath.Join(b.StagedDir, name)) } -// flush writes the buffered in-memory parts to the staged directory. Once a -// file-part fault is recorded the bundle cannot load, so nothing more is -// written; Cleanup disposes of whatever reached disk beforehand. func (b *generationBundle) flush() error { if len(b.pending) == 0 || b.stagingErr != nil { return nil @@ -400,8 +317,6 @@ func writeStagedFile(path string, content []byte) error { return dst.Close() } -// stagedFileName applies the shipped bundle contract: a bare .ndjson name with -// no path components, unique within one request. func stagedFileName(filename string) (string, error) { name := filepath.Base(filename) if name == "." || name == "" || name != filename || filepath.Ext(name) != ".ndjson" { @@ -410,8 +325,6 @@ func stagedFileName(filename string) (string, error) { return name, nil } -// mimeHeaderSize mirrors the per-part metadata charge mime/multipart's -// readForm applies to its memory budget. func mimeHeaderSize(header textproto.MIMEHeader) int64 { size := int64(formMIMEHeaderBaseSize) for key, values := range header { diff --git a/internal/api/bulk/load/operations.go b/internal/api/bulk/load/operations.go index a1f3b3f05..52d2463c7 100644 --- a/internal/api/bulk/load/operations.go +++ b/internal/api/bulk/load/operations.go @@ -32,14 +32,7 @@ func (h *Handler) GetDatasetGenerationStatus(ctx context.Context, project, gener return status, nil } -// CreateDatasetGeneration parses and stages a generated multipart request, -// then runs the generation import operation. func (h *Handler) CreateDatasetGeneration(ctx context.Context, project, generation string, body *multipart.Reader, principal *authscope.Principal) (*GenerationLoadResult, error) { - // The stager writes each file part into the directory the loader consumes, - // so an uploaded byte is written to temporary storage once and there is no - // ReadForm spillover copy to dispose of. bundle.Cleanup is the only - // disposal, registered first so it runs after every consumer; it is a - // no-op on the nil bundle produced by an absent body. bundle, err := stageGenerationBundle(body) if bundle != nil { defer bundle.Cleanup() @@ -75,9 +68,6 @@ func (h *Handler) CreateDatasetGeneration(ctx context.Context, project, generati if err := h.authz.AuthorizeWrite(ctx, principal, project, authResourcePath); err != nil { return nil, dataframeerrors.Wrap(err, dataframeerrors.CodeForbidden, "") } - // The file-part fault surfaces here, not during parsing: the replaced path - // ran ReadForm to completion and only then inspected names, so an invalid - // name never outranked a scalar fault or a denied request. if err := bundle.ParseStagingError(); err != nil { switch { case errors.Is(err, os.ErrInvalid): diff --git a/internal/api/bulk/load/operations_test.go b/internal/api/bulk/load/operations_test.go index c0e298ab2..e6b4a4e81 100644 --- a/internal/api/bulk/load/operations_test.go +++ b/internal/api/bulk/load/operations_test.go @@ -50,13 +50,6 @@ func TestCreateDatasetGenerationStagesCompleteBundle(t *testing.T) { } } -// createDatasetGenerationBaseline is the shipped pre-change handler body -// (git show HEAD~1:internal/api/bulk/load/operations.go, ReadForm + a second -// io.Copy into a separate staging directory), and stageGenerationFilesBaseline -// is its staging helper. They exist only so the differential tests below can -// compare the streaming stager against the behavior it replaced — same typed -// error, same loader-visible bytes — instead of against a description of it. -// They are never wired into a route. func (h *Handler) createDatasetGenerationBaseline(ctx context.Context, project, generation string, body *multipart.Reader, principal *authscope.Principal) (*GenerationLoadResult, error) { if body == nil { return nil, fiber.NewError(fiber.StatusUnsupportedMediaType) @@ -149,7 +142,6 @@ func stageGenerationFilesBaseline(headers []*multipart.FileHeader) (string, erro return dir, nil } -// orderedPart is one multipart part in the exact order the client sent it. type orderedPart struct { field string filename string @@ -187,7 +179,6 @@ func orderedMultipart(t *testing.T, parts []orderedPart) ([]byte, string) { return body.Bytes(), writer.Boundary() } -// recordingGenerationRunner captures the loader-visible bundle and can fail. type recordingGenerationRunner struct { req GenerationLoadRequest contents map[string]string @@ -232,9 +223,6 @@ func allowAllHandler(t *testing.T, runner *recordingGenerationRunner) *Handler { return handler } -// assertNoTemporaryUploadFiles pins the cleanup contract: after the handler -// returns, the redirected temp root holds no staged directory and no -// multipart spillover file. func assertNoTemporaryUploadFiles(t *testing.T, tempDir string) { t.Helper() entries, err := os.ReadDir(tempDir) @@ -253,11 +241,6 @@ func assertNoTemporaryUploadFiles(t *testing.T, tempDir string) { } } -// TestCreateDatasetGenerationStagesFilesThatArriveBeforeScalarFields pins the -// compatibility-sensitive ordering contract: files first in multipart order -// must still reach the loader byte-identical, and the scalar fields that -// follow them (including a repeat value, where the first one wins) must still -// reach the handler. func TestCreateDatasetGenerationStagesFilesThatArriveBeforeScalarFields(t *testing.T) { tempDir := t.TempDir() t.Setenv("TMPDIR", tempDir) @@ -294,10 +277,6 @@ func TestCreateDatasetGenerationStagesFilesThatArriveBeforeScalarFields(t *testi assertNoTemporaryUploadFiles(t, tempDir) } -// TestCreateDatasetGenerationRejectsBundleWithoutStagingAnything pins the -// file-name and duplicate contract plus its cleanup: every rejection is a -// typed error and leaves nothing on disk, including a file already streamed -// into the staging directory before the rejecting part arrived. func TestCreateDatasetGenerationRejectsBundleWithoutStagingAnything(t *testing.T) { large := bytes.Repeat([]byte("y"), 20*1024*1024) for _, tc := range []struct { @@ -356,9 +335,6 @@ func TestCreateDatasetGenerationRejectsBundleWithoutStagingAnything(t *testing.T } } -// TestCreateDatasetGenerationMalformedAndAbsentBodyAreInvalidInput pins the -// pre-fix classification of parse failures: an absent or unparseable body is -// never a staging fault, and a truncated stream leaves nothing behind. func TestCreateDatasetGenerationMalformedAndAbsentBodyAreInvalidInput(t *testing.T) { t.Run("absent body", func(t *testing.T) { handler := allowAllHandler(t, &recordingGenerationRunner{}) @@ -391,7 +367,6 @@ func TestCreateDatasetGenerationMalformedAndAbsentBodyAreInvalidInput(t *testing }) } -// assertFiberStatus pins the transport-level error the route maps to a status. func assertFiberStatus(t *testing.T, err error, status int) { t.Helper() var fiberErr *fiber.Error @@ -400,8 +375,6 @@ func assertFiberStatus(t *testing.T, err error, status int) { } } -// generationOutcome is everything the caller and the loader can observe, so -// the two staging paths can be compared as black boxes. type generationOutcome struct { errIsNil bool errCode string @@ -440,10 +413,6 @@ func outcomeOf(result *GenerationLoadResult, err error, runner *recordingGenerat return outcome } -// runGenerationOnTempRoot drives one request through one staging path with a -// private temp root, and returns the observable outcome. The temp root must be -// genuinely in effect: the leftover check is only meaningful if the staging -// paths could have written there. func runGenerationOnTempRoot(t *testing.T, parts []orderedPart, truncate int, baseline bool, authz authscope.Authorizer, loaderErr error) generationOutcome { t.Helper() tempDir := t.TempDir() @@ -476,20 +445,9 @@ func runGenerationOnTempRoot(t *testing.T, parts []orderedPart, truncate int, ba return outcome } -// TestCreateDatasetGenerationStreamingPathMatchesShippedPath is the -// differential check behind the single-write change: every payload shape that -// matters — multipart order, scalar semantics, filename validation, duplicates, -// non-bundle file fields, malformed input, denied authorization, loader -// failure — is pushed through both the streaming stager and the shipped -// ReadForm-plus-second-copy path it replaced, and the loader-visible bytes, the -// typed error, and the loader request must agree exactly. func TestCreateDatasetGenerationStreamingPathMatchesShippedPath(t *testing.T) { small := []byte("{\"resourceType\":\"Patient\",\"id\":\"1\"}\n") - // Two of these exceed the 32 MiB ReadForm file budget, which is what forced a - // multipart-* spill in the shipped path. overBudget := bytes.Repeat([]byte("z"), 20*1024*1024) - // One of these exceeds it on its own, so it is written to the staged - // directory before any later part can reject the request. pastBudget := bytes.Repeat([]byte("z"), 33*1024*1024) cases := []struct { @@ -546,10 +504,6 @@ func TestCreateDatasetGenerationStreamingPathMatchesShippedPath(t *testing.T) { }, }, { - // pastBudget is above the 32 MiB file budget on its own, so the - // streaming path has already written it into the staged directory - // before the rejecting part arrives — the handler cannot clean that - // up (it never receives a bundle), so the stager must. name: "staged oversized file rejected by a later duplicate", parts: []orderedPart{ {field: "file", filename: "Patient.ndjson", content: pastBudget}, @@ -572,8 +526,6 @@ func TestCreateDatasetGenerationStreamingPathMatchesShippedPath(t *testing.T) { truncate: 8, }, { - // A scalar field flushes the buffered files, so the rejecting part - // arrives while the staged directory already exists on disk. name: "flushed files rejected by a later duplicate", parts: []orderedPart{ {field: "file", filename: "Patient.ndjson", content: small}, @@ -592,8 +544,6 @@ func TestCreateDatasetGenerationStreamingPathMatchesShippedPath(t *testing.T) { truncate: 8, }, { - // The oversized part is already on disk and the scalar that follows - // pushes the parse past the memory budget. name: "staged oversized file past the scalar budget", parts: []orderedPart{ {field: "file", filename: "Patient.ndjson", content: pastBudget}, @@ -602,8 +552,6 @@ func TestCreateDatasetGenerationStreamingPathMatchesShippedPath(t *testing.T) { }, }, { - // The oversized scalar is the final part, so only its own budget - // charge can refuse it — no later part re-triggers the check. name: "oversized scalar value as the last part", parts: []orderedPart{ {field: "file", filename: "Patient.ndjson", content: small}, @@ -611,22 +559,14 @@ func TestCreateDatasetGenerationStreamingPathMatchesShippedPath(t *testing.T) { }, }, { - // Metadata alone exhausts the budget: 500 scalar parts with 96 KiB - // field names cost ~47 MiB of in-memory accounting while staying under - // the part limit, which is what readForm refuses. name: "metadata charges exhaust the memory budget", parts: metadataHeavyParts(500, 96*1024), }, { - // Same shape for file parts, where readForm additionally charges the - // MIME header size and file-header overhead. name: "file part metadata charges exhaust the memory budget", parts: metadataHeavyFileParts(500, 96*1024), }, { - // Boundary case for the fixed per-part map-entry overhead: 1000 parts - // whose field names alone sit just inside the budget, so only the - // 200-byte-per-part charge readForm applies can push it over. name: "per-part overhead crosses the memory budget", parts: metadataHeavyParts(1000, 43950), }, @@ -704,9 +644,6 @@ func TestCreateDatasetGenerationStreamingPathMatchesShippedPath(t *testing.T) { deny: true, }, { - // The shipped path only discovered a bad file name inside - // stageGenerationFiles, which ran after AuthorizeWrite, so a denied - // caller got FORBIDDEN rather than the file fault. name: "denied authorization with an invalid file name", parts: []orderedPart{ {field: "file", filename: "Patient.txt", content: small}, @@ -722,8 +659,6 @@ func TestCreateDatasetGenerationStreamingPathMatchesShippedPath(t *testing.T) { deny: true, }, { - // Authorization is still decided by the parsed scalar fields, so a - // bundle that fails staging must not bypass the authorizer. name: "denied authorization with a bad name after a staged oversized file", parts: []orderedPart{ {field: "file", filename: "Patient.ndjson", content: pastBudget}, @@ -733,13 +668,10 @@ func TestCreateDatasetGenerationStreamingPathMatchesShippedPath(t *testing.T) { deny: true, }, { - // The shipped path parsed every part before validating, so the parser - // limit outranks a file fault even when the bad name arrives first. name: "invalid file name plus too many parts", parts: tooManyPartsWithBadName(), }, { - // Likewise the scalar checks outrank a file fault. name: "invalid file name plus invalid defer activation", parts: []orderedPart{ {field: "file", filename: "Patient.txt", content: small}, @@ -762,8 +694,6 @@ func TestCreateDatasetGenerationStreamingPathMatchesShippedPath(t *testing.T) { }, }, { - // stageGenerationFiles returned at the first offending header, so - // the earlier fault decides: invalid name, not the later duplicate. name: "invalid file name before a duplicate", parts: []orderedPart{ {field: "file", filename: "Patient.txt", content: small}, @@ -789,10 +719,6 @@ func TestCreateDatasetGenerationStreamingPathMatchesShippedPath(t *testing.T) { }, }, { - // readForm charged every file part's content against the memory budget - // whether or not its name was usable, so an oversized invalid-name - // part can push a later scalar past the limit. The file fault must not - // outrank that parse failure. name: "oversized invalid-name part then a scalar past the budget", parts: []orderedPart{ {field: "file", filename: "Patient.txt", content: pastBudget}, @@ -810,8 +736,6 @@ func TestCreateDatasetGenerationStreamingPathMatchesShippedPath(t *testing.T) { }, }, { - // A rejected part that is itself past the file budget is still - // accounted for, exactly as readForm accounted for the spill it made. name: "oversized non-bundle file part then a scalar past the budget", parts: []orderedPart{ {field: "other", filename: "junk.txt", content: pastBudget}, @@ -820,10 +744,6 @@ func TestCreateDatasetGenerationStreamingPathMatchesShippedPath(t *testing.T) { }, }, { - // 200 in-memory non-bundle file parts of 160000 bytes each cost - // 32 MiB of the readForm budget, so a following 13 MiB scalar - // crosses the limit. Skip that charge and the same request parses - // cleanly, which is a different error class. name: "many non-bundle file parts then a scalar past the budget", parts: nonBundleFillerParts(200, 160000, 13*1024*1024), }, @@ -844,8 +764,6 @@ func TestCreateDatasetGenerationStreamingPathMatchesShippedPath(t *testing.T) { } } -// metadataHeavyParts builds n scalar parts whose field names alone push the -// parse past the readForm memory budget, staying under the part limit. func metadataHeavyParts(n, nameLen int) []orderedPart { name := strings.Repeat("n", nameLen) parts := make([]orderedPart, 0, n+1) @@ -855,8 +773,6 @@ func metadataHeavyParts(n, nameLen int) []orderedPart { return parts } -// metadataHeavyFileParts builds n file parts whose names and MIME headers push -// the parse past the readForm memory budget. func metadataHeavyFileParts(n, nameLen int) []orderedPart { name := strings.Repeat("n", nameLen) parts := make([]orderedPart, 0, n+1) @@ -866,8 +782,6 @@ func metadataHeavyFileParts(n, nameLen int) []orderedPart { return parts } -// tooManyPartsWithBadName puts an invalid file name ahead of a request that -// trips the part-count limit, so the two failure classes compete. func tooManyPartsWithBadName() []orderedPart { parts := []orderedPart{{field: "file", filename: "Patient.txt", content: []byte("nope\n")}} for i := 0; i <= generationFormLimits().maxParts; i++ { @@ -876,10 +790,6 @@ func tooManyPartsWithBadName() []orderedPart { return parts } -// nonBundleFillerParts builds n file parts under a non-bundle field, each -// bytes long, followed by a scalar of scalarBytes. readForm charged every one -// of those parts against its memory budget, which decides whether the scalar -// trips ErrMessageTooLarge. func nonBundleFillerParts(n, bytesPerPart, scalarBytes int) []orderedPart { parts := make([]orderedPart, 0, n+2) filler := bytes.Repeat([]byte("f"), bytesPerPart) @@ -925,10 +835,6 @@ func compareOutcomes(want, got generationOutcome) string { return strings.Join(diffs, "\n") } -// TestCreateDatasetGenerationParserLimitsMatchShippedPath pins the DoS limits -// the streaming stager inherits from readForm: a request past the part count -// and a scalar value past the memory budget must be refused the same way, with -// nothing staged. func TestCreateDatasetGenerationParserLimitsMatchShippedPath(t *testing.T) { small := []byte("{\"resourceType\":\"Patient\",\"id\":\"1\"}\n") authz := authscope.AllowAllAuthorizer{} @@ -947,8 +853,6 @@ func TestCreateDatasetGenerationParserLimitsMatchShippedPath(t *testing.T) { }) t.Run("scalar value budget", func(t *testing.T) { - // A scalar past maxMemory (32 MiB + the 10 MiB non-file reserve) is what - // readForm refuses with ErrMessageTooLarge. oversized := bytes.Repeat([]byte("z"), 45*1024*1024) parts := []orderedPart{ {field: "huge", content: oversized}, @@ -979,10 +883,6 @@ func TestCreateDatasetGenerationParserLimitsMatchShippedPath(t *testing.T) { }) } -// TestCreateDatasetGenerationRejectedAuthorizationAndLoaderFailureLeaveNoFiles -// pins that authorization still runs before the loader, that a loader failure -// keeps its identity through the handler, and that both paths dispose of the -// staged bundle. func TestCreateDatasetGenerationRejectedAuthorizationAndLoaderFailureLeaveNoFiles(t *testing.T) { t.Run("forbidden", func(t *testing.T) { tempDir := t.TempDir() diff --git a/internal/api/bulk/load/service.go b/internal/api/bulk/load/service.go index ab4d82236..f8634215a 100644 --- a/internal/api/bulk/load/service.go +++ b/internal/api/bulk/load/service.go @@ -73,7 +73,6 @@ func (r IngestRunner) RunGeneration(ctx context.Context, req GenerationLoadReque opts.DeferActivation = req.DeferActivation opts.Truncate = false opts.EventSink = sink - // StageOnly is a CLI mode. The HTTP API stages through DeferActivation. opts.StageOnly = false return ingest.Load(ctx, opts) } @@ -167,9 +166,6 @@ func (s *Service) runGeneration(ctx context.Context, req GenerationLoadRequest) return &GenerationLoadResult{Project: req.Project, Generation: req.Generation, AuthResourcePath: req.AuthResourcePath, SubmittedBy: req.SubmittedBy, Summary: &summary, Activated: !req.DeferActivation}, nil } -// activateGeneration performs the release switch only after the exact -// dataframe bundle supplied by the caller is durably successful and remains -// the active pointer for its recipe/generation namespace. func (s *Service) activateGeneration(ctx context.Context, project, generation, executionID string) error { fail := func(err error) error { if err != nil { @@ -208,10 +204,6 @@ func (s *Service) activateGeneration(ctx context.Context, project, generation, e return fail(fmt.Errorf("read generation release: %w", err)) } if !manifest.IsStaged() { - // Domain state signal: activate requires a staged manifest. The - // bulk-load boundary (normalizeError) classifies this sentinel as - // GENERATION_NOT_READY (409); a raw error here served an opaque 500 - // on the documented 409 arm of the activate route. return fail(fmt.Errorf("generation %s/%s is not READY (state %s): %w", project, generation, manifest.State, publication.ErrGenerationNotReady)) } if err := s.generationActivator.Activate(ctx, manifest); err != nil { diff --git a/internal/api/bulk/load/service_test.go b/internal/api/bulk/load/service_test.go index 421738951..400915323 100644 --- a/internal/api/bulk/load/service_test.go +++ b/internal/api/bulk/load/service_test.go @@ -295,11 +295,6 @@ func activationManifest(t *testing.T, project, generation string) publication.Ma return manifest } -// Reviewer-proven defect: the documented 409 arm of -// POST /datasets/{project}/generations/{generation}/activate served an opaque -// 500. Activation of a not-yet-staged generation must be a typed -// GENERATION_NOT_READY conflict end to end: service guard -> domain sentinel -// -> normalizeError boundary -> MapDataframeError 409 non-retryable. func TestActivateGenerationLoadingManifestMapsTo409(t *testing.T) { ref, err := publication.NewRef("project-a", "generation-a") if err != nil { diff --git a/internal/api/columncapabilities/capabilities.go b/internal/api/columncapabilities/capabilities.go index 251dc4bd8..5e0db0be0 100644 --- a/internal/api/columncapabilities/capabilities.go +++ b/internal/api/columncapabilities/capabilities.go @@ -1,10 +1,7 @@ -// Package columncapabilities defines the transport-neutral capability policy -// derived from a published ClickHouse column type. package columncapabilities import "strings" -// Tuple is the shared selector capability contract for a published column. type Tuple struct { Logical string Nullable bool @@ -14,9 +11,6 @@ type Tuple struct { Aggregatable bool } -// FromClickHouse classifies nested Nullable and Array wrappers before -// applying the primitive type policy. Wrapper order is intentionally -// insignificant because ClickHouse can emit either nesting order. func FromClickHouse(clickHouseType string) Tuple { typ := strings.TrimSpace(clickHouseType) result := Tuple{} diff --git a/internal/api/graphql/graph/dataframe/aggregates.go b/internal/api/graphql/graph/dataframe/aggregates.go index 8d4e34837..97f4deede 100644 --- a/internal/api/graphql/graph/dataframe/aggregates.go +++ b/internal/api/graphql/graph/dataframe/aggregates.go @@ -45,10 +45,6 @@ func AggregationsJSON(result dfmaterialization.AggregationsResult) (json.RawMess return json.Marshal(dfmaterialization.NormalizeAggregationResults(result.Aggregations)) } -// GraphQL launches sibling resolvers concurrently, but a large selection set -// can take several milliseconds to enqueue its final aggregate call. The -// fallback exists only for directive-skipped fields where expectedCalls cannot -// be reached; normal operations dispatch immediately at the exact count. const aggregateFallbackDelay = 10 * time.Millisecond type aggregateOperationStateKey struct{} diff --git a/internal/api/graphql/graph/dataframe/aggregates_test.go b/internal/api/graphql/graph/dataframe/aggregates_test.go index e1e7df75e..277cb81ad 100644 --- a/internal/api/graphql/graph/dataframe/aggregates_test.go +++ b/internal/api/graphql/graph/dataframe/aggregates_test.go @@ -102,10 +102,6 @@ func enqueueAndAwait(t *testing.T, wait *sync.WaitGroup, state *aggregateOperati }() } -// The 10ms fallback debounce may legitimately split the burst under -// preemption; wall-clock single-batch behavior is not a product guarantee. -// Synchronize on completion and pin the observable contract: every call -// resolves and each call is dispatched exactly once across all batches. func TestAggregateSchedulerDispatchesExpectedConcurrentCallsOnce(t *testing.T) { const burstCalls = 158 state, recorder := newAggregateSchedulerHarness(t, burstCalls) diff --git a/internal/api/graphql/graph/dataframe/export.go b/internal/api/graphql/graph/dataframe/export.go index 2ceed854a..20a1ce785 100644 --- a/internal/api/graphql/graph/dataframe/export.go +++ b/internal/api/graphql/graph/dataframe/export.go @@ -11,7 +11,6 @@ import ( dfmaterialization "github.com/calypr/loom/internal/dataframe/published" ) -// ExportDataframe streams one authorized project's published dataframe. func (s *Service) ExportDataframe(ctx context.Context, project string, request dfmaterialization.ExportRequest, out io.Writer) error { if s.reader == nil { return dataframeerrors.NewError(dataframeerrors.CodeBackendUnavailable, "", dataframeerrors.WithRetryable(true)) diff --git a/internal/api/graphql/graph/dataframe/fb22_principal_auth_path_control_test.go b/internal/api/graphql/graph/dataframe/fb22_principal_auth_path_control_test.go index c3c227702..2140cc2ce 100644 --- a/internal/api/graphql/graph/dataframe/fb22_principal_auth_path_control_test.go +++ b/internal/api/graphql/graph/dataframe/fb22_principal_auth_path_control_test.go @@ -9,35 +9,15 @@ import ( dfmaterialization "github.com/calypr/loom/internal/dataframe/published" ) -// F-B22 Tier-1 durable control (REPAIR-18 / U1): the resolver-less -// authorizeMaterialization branch in this package's service.go copied -// principal.AuthResourcePaths RAW into projectAccess.authResourcePaths. That -// slice is handed verbatim to PageRequest.AuthResourcePaths (rows.go Rows and -// export.go), which becomes the ClickHouse `auth_resource_path IN ?` bind list -// in published/page.go (and Stream), bypassing the boundary-normalization -// contract that repair-7 applied at -// internal/api/graphql/graph/query/input_resolution.go via -// authscope.NormalizeAuthResourcePaths. The reviewer's threat: a principal -// that mixes a control-byte entry into its declared paths reaches the IN-list -// with that raw NUL entry, so a NUL-bearing grant still participates in the -// \x00-joined scope digest that folds two distinct restricted grants onto one -// digest (finding F-B22). -// -// The archived CONFIRMED shape (raw copy, no normalization) is preserved -// verbatim below as the failing branch, so a regression re-opens loudly the -// same way the sibling queryapi REPAIR-7 control does. - func TestAuthorizeMaterializationDropsControlCharPrincipalPathBeforeClickHouseINList(t *testing.T) { - service := NewService(Config{}) // nil scopeResolver selects the resolver-less :85 branch + service := NewService(Config{}) principal := &authscope.Principal{Subject: "user", Projects: []string{"P1"}, AuthResourcePaths: []string{"a\x00b", "p1"}} - value := dfmaterialization.Materialization{Project: "P1"} // restricted: ScopeUnrestricted == false + value := dfmaterialization.Materialization{Project: "P1"} access, err := service.authorizeMaterialization(context.Background(), principal, value) if err != nil { t.Fatalf("VERDICT=UNEXPECTED resolver-less restricted authorization errored: %v", err) } - // access.authResourcePaths is the exact slice bound to - // PageRequest.AuthResourcePaths -> `auth_resource_path IN ?` (page.go:77). if reflect.DeepEqual(access.authResourcePaths, []string{"a\x00b", "p1"}) { t.Fatalf("VERDICT=CONFIRMED (pre-fix archived shape): principal.AuthResourcePaths were copied raw into projectAccess; the ClickHouse auth_resource_path IN ? list is %q and still carries the NUL entry, so a NUL-bearing grant reaches the \\x00-join digest", access.authResourcePaths) } @@ -50,12 +30,8 @@ func TestAuthorizeMaterializationDropsControlCharPrincipalPathBeforeClickHouseIN t.Logf("VERDICT=REPAIRED CH auth_resource_path IN-list = %q; boundary normalization dropped the NUL-bearing entry", access.authResourcePaths) } -// A principal whose only declared path is control-byte garbage must fail -// closed at the resolver-less boundary instead of reaching the IN-list as a -// restricted-but-empty scope (which the AQL layer could mistake for an -// unrestricted scope) or as a raw control-byte grant. func TestAuthorizeMaterializationGarbageOnlyPrincipalPathFailsClosed(t *testing.T) { - service := NewService(Config{}) // nil scopeResolver selects the resolver-less :85 branch + service := NewService(Config{}) principal := &authscope.Principal{Subject: "user", Projects: []string{"P1"}, AuthResourcePaths: []string{"a\x00b"}} value := dfmaterialization.Materialization{Project: "P1"} diff --git a/internal/api/graphql/graph/dataframe/model.go b/internal/api/graphql/graph/dataframe/model.go index 8a1797036..b394b5ed1 100644 --- a/internal/api/graphql/graph/dataframe/model.go +++ b/internal/api/graphql/graph/dataframe/model.go @@ -48,9 +48,6 @@ func Model(value materialization.Materialization) *model.DataframeMaterializatio return result } -// ColumnFromPhysical adapts candidate execution metadata before activation. -// It intentionally mirrors the published materialization representation while -// retaining semantic identity for browser-side configuration. func ColumnFromPhysical(column publication.PhysicalColumn) *model.DataframeColumn { return ColumnModel(materialization.Column{Name: column.Name, SemanticPath: column.SemanticPath, ClickHouse: column.ClickHouse, LogicalType: column.LogicalType, Nullable: column.Nullable, Repeated: column.Repeated}) } diff --git a/internal/api/graphql/graph/dataframe/rows.go b/internal/api/graphql/graph/dataframe/rows.go index 12164ee30..5130acde7 100644 --- a/internal/api/graphql/graph/dataframe/rows.go +++ b/internal/api/graphql/graph/dataframe/rows.go @@ -10,8 +10,6 @@ import ( dfmaterialization "github.com/calypr/loom/internal/dataframe/published" ) -// ProjectDatasets exposes the current output dataframes for one authorized -// project. It is the only dataset-discovery surface. func (s *Service) ProjectDatasets(ctx context.Context, projectID string) ([]dfmaterialization.Materialization, error) { if s.reader == nil { return nil, readerUnavailable() diff --git a/internal/api/graphql/graph/dataframe/service.go b/internal/api/graphql/graph/dataframe/service.go index 070b5f1a5..011e04f90 100644 --- a/internal/api/graphql/graph/dataframe/service.go +++ b/internal/api/graphql/graph/dataframe/service.go @@ -1,6 +1,3 @@ -// Package dataframe adapts one project's published ClickHouse -// dataframe outputs to GraphQL. Project identity is explicit on every read; -// this package never discovers or combines data from other projects. package dataframe import ( @@ -82,12 +79,6 @@ func (s *Service) authorizeMaterialization(ctx context.Context, principal *auths if len(principal.AuthResourcePaths) == 0 { return projectAccess{}, dataframeerrors.NewError(dataframeerrors.CodeDatasetNotFound, "") } - // F-B22 Tier-1: this slice flows verbatim to PageRequest.AuthResourcePaths - // and the ClickHouse `auth_resource_path IN ?` bind list (published/page.go). - // Normalize at this boundary with the same authscope helper repair-7 applied - // at query/input_resolution.go, so control bytes, stray slashes, and - // duplicate spellings can never reach the scope-digest join; a garbage-only - // principal fails closed instead of widening. normalizedPaths := authscope.NormalizeAuthResourcePaths(principal.AuthResourcePaths) if len(normalizedPaths) == 0 { return projectAccess{}, dataframeerrors.NewError(dataframeerrors.CodeDatasetNotFound, "") diff --git a/internal/api/graphql/graph/doc.go b/internal/api/graphql/graph/doc.go index 6a9b0d3c3..d9e6db960 100644 --- a/internal/api/graphql/graph/doc.go +++ b/internal/api/graphql/graph/doc.go @@ -1,7 +1 @@ -// Package graphqlapi owns Loom's primary GraphQL transport. -// -// HTTP wiring and error presentation live here. Query construction lives in -// query, dataframe access and response mapping in dataframe, resolver wiring in -// resolver, and the schema source in schema. Generated GraphQL types, executors, -// and resolver bindings live outside internal under generated/graphql/graph. package graphqlapi diff --git a/internal/api/graphql/graph/errors.go b/internal/api/graphql/graph/errors.go index dc2b4241c..02448825b 100644 --- a/internal/api/graphql/graph/errors.go +++ b/internal/api/graphql/graph/errors.go @@ -7,9 +7,6 @@ import ( "github.com/vektah/gqlparser/v2/gqlerror" ) -// presentError converts a service error to the GraphQL error shape used by -// the frontend. The original error remains attached to gqlerror.Error.Err so -// server logging and errors.Is/errors.As retain their normal behavior. func presentError(err error, requestID string) *gqlerror.Error { if err == nil { return nil @@ -30,8 +27,6 @@ func presentError(err error, requestID string) *gqlerror.Error { func presentGraphQLError(err error, requestID string) *gqlerror.Error { var gqlErr *gqlerror.Error if errors.As(err, &gqlErr) && gqlErr != nil { - // Parser and validation errors are already safe and actionable. Preserve - // their path/location while giving clients one stable non-retryable code. if gqlErr.Err == nil { copy := *gqlErr if len(copy.Extensions) == 0 { @@ -55,8 +50,6 @@ func presentGraphQLError(err error, requestID string) *gqlerror.Error { return presentError(err, requestID) } -// extensionsForError returns a fresh map suitable for a GraphQL error -// response. It deliberately exposes only the stable semantic contract. func extensionsForError(err error, requestID string) map[string]any { userErr := dataframeerrors.Normalize(err) if userErr == nil { diff --git a/internal/api/graphql/graph/handler.go b/internal/api/graphql/graph/handler.go index c26424662..e682c0aa6 100644 --- a/internal/api/graphql/graph/handler.go +++ b/internal/api/graphql/graph/handler.go @@ -342,9 +342,6 @@ func graphqlFailureStatus(body []byte) int { } data := bytes.TrimSpace(payload.Data) if len(data) > 0 && !bytes.Equal(data, []byte("null")) { - // GraphQL can legitimately return useful partial data alongside field - // errors. Keep HTTP 200 for that case; only failed operations are lifted - // into the HTTP error contract. return 0 } status := http.StatusBadRequest diff --git a/internal/api/graphql/graph/http_integration_test.go b/internal/api/graphql/graph/http_integration_test.go index f2143f205..3411d55fa 100644 --- a/internal/api/graphql/graph/http_integration_test.go +++ b/internal/api/graphql/graph/http_integration_test.go @@ -3,6 +3,7 @@ package graphqlapi_test import ( "context" "encoding/json" + "github.com/calypr/loom/internal/dataframe/compiler" "net/http" "net/http/httptest" "strings" @@ -111,7 +112,13 @@ func TestProjectDataframeDatasetsResolverIsWired(t *testing.T) { func TestGraphQLRunDataframeMutation(t *testing.T) { dfService := dataframeexecution.NewService(dataframeexecution.ServiceConfig{ - QueryRows: func(ctx context.Context, query string, _ int, bindVars map[string]any, visit func(map[string]any) error) error { + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + query, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + if !strings.Contains(query, "LET child_set_") || !strings.Contains(query, "LENGTH(child_set_") { t.Fatalf("expected physical child-set query, got:\n%s", query) } @@ -119,7 +126,7 @@ func TestGraphQLRunDataframeMutation(t *testing.T) { }, }) graphResolver := graphresolver.NewResolver(graphresolver.ResolverConfig{ - DataframeQuery: queryapi.Config{ + DataframeQuery: queryapi.Config{CompileTarget: compiler.ArangoTarget(), ActiveManifestResolver: testActiveManifestResolver{}, DiscoverFields: func(ctx context.Context, opts catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) { switch opts.ResourceType { @@ -200,7 +207,13 @@ func TestGraphQLRunDataframeMutation(t *testing.T) { func TestGraphQLRunDataframeTraversalBuilder(t *testing.T) { dfService := dataframeexecution.NewService(dataframeexecution.ServiceConfig{ - QueryRows: func(ctx context.Context, query string, _ int, bindVars map[string]any, visit func(map[string]any) error) error { + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + query, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + if !strings.Contains(query, "LET child_set_") || !strings.Contains(query, "LENGTH(child_set_") { t.Fatalf("expected physical child-set query, got:\n%s", query) } @@ -213,7 +226,7 @@ func TestGraphQLRunDataframeTraversalBuilder(t *testing.T) { }, }) graphResolver := graphresolver.NewResolver(graphresolver.ResolverConfig{ - DataframeQuery: queryapi.Config{ + DataframeQuery: queryapi.Config{CompileTarget: compiler.ArangoTarget(), ActiveManifestResolver: testActiveManifestResolver{}, DiscoverFields: func(ctx context.Context, opts catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) { switch opts.ResourceType { diff --git a/internal/api/graphql/graph/query/active_generation.go b/internal/api/graphql/graph/query/active_generation.go index e170daab7..555e15074 100644 --- a/internal/api/graphql/graph/query/active_generation.go +++ b/internal/api/graphql/graph/query/active_generation.go @@ -7,9 +7,6 @@ import ( publication "github.com/calypr/loom/internal/dataset" ) -// resolveActiveGeneration pins builder-side catalog discovery to the exact -// READY manifest selected for this request. Without a manifest, the primary -// unversioned catalog namespace remains the fallback. func (s *Service) resolveActiveGeneration(ctx context.Context, project string) (string, error) { if s == nil || s.activeManifestResolver == nil { return "", nil diff --git a/internal/api/graphql/graph/query/active_generation_test.go b/internal/api/graphql/graph/query/active_generation_test.go index 52fb10160..1932a7531 100644 --- a/internal/api/graphql/graph/query/active_generation_test.go +++ b/internal/api/graphql/graph/query/active_generation_test.go @@ -2,6 +2,7 @@ package queryapi import ( "context" + "github.com/calypr/loom/internal/dataframe/compiler" "strings" "testing" @@ -82,14 +83,19 @@ func TestActiveGenerationPropagatesThroughBuilderCatalogEntrypoints(t *testing.T return []catalog.PopulatedReference{{FromType: "Patient", Label: "subject_Patient", ToType: "Specimen", EdgeCount: 1}}, nil } dataframes := dataframeexecution.NewService(dataframeexecution.ServiceConfig{ - QueryRows: func(_ context.Context, _ string, _ int, bindVars map[string]any, _ func(map[string]any) error) error { + SourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, bindVars, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + if got := bindVars["dataset_generation"]; got != generation { t.Fatalf("dataframe execution generation = %#v, want %q", got, generation) } return nil }, }) - service := NewService(Config{ + service := NewService(Config{CompileTarget: compiler.ArangoTarget(), ActiveManifestResolver: active, DiscoverFields: discoverFields, DiscoverReferences: discoverReferences, @@ -130,7 +136,7 @@ func TestActiveGenerationPropagatesThroughBuilderCatalogEntrypoints(t *testing.T } func TestResolveActiveGenerationFallsBackToUnversionedCatalog(t *testing.T) { - service := NewService(Config{ActiveManifestResolver: noActiveManifestResolver{}}) + service := NewService(Config{CompileTarget: compiler.ArangoTarget(), ActiveManifestResolver: noActiveManifestResolver{}}) generation, err := service.resolveActiveGeneration(context.Background(), "project-a") if err != nil { diff --git a/internal/api/graphql/graph/query/auth_scope_test.go b/internal/api/graphql/graph/query/auth_scope_test.go index f67262099..ba82ad7e7 100644 --- a/internal/api/graphql/graph/query/auth_scope_test.go +++ b/internal/api/graphql/graph/query/auth_scope_test.go @@ -2,6 +2,7 @@ package queryapi import ( "context" + "github.com/calypr/loom/internal/dataframe/compiler" "testing" "github.com/calypr/loom/generated/graphql/graph/model" @@ -36,17 +37,19 @@ func TestRunPreservesRestrictedEmptyScopeIntoDataframeService(t *testing.T) { preparedCatalogCalls := 0 dataframes := dataframeexecution.NewService(dataframeexecution.ServiceConfig{ - // This intentionally has no ScopeResolver. The one-shot GraphQL recipe - // is already catalog- and scope-resolved before reaching execution, so - // execution must not perform a second recipe preparation pass. - QueryRows: func(_ context.Context, _ string, _ int, bindVars map[string]any, _ func(map[string]any) error) error { + SourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, bindVars, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + if got, ok := bindVars["auth_resource_paths_unrestricted"].(bool); !ok || got { t.Fatalf("dataframe AQL unrestricted bind = %#v, want false", bindVars["auth_resource_paths_unrestricted"]) } return nil }, }) - service := NewService(Config{ + service := NewService(Config{CompileTarget: compiler.ArangoTarget(), ScopeResolver: resolver, ActiveManifestResolver: &builderActiveManifestResolver{manifest: builderReadyManifest(t, "P1", "generation-1")}, DiscoverFields: func(_ context.Context, options catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) { @@ -76,7 +79,7 @@ func TestIntrospectKeepsRestrictedEmptyCatalogMode(t *testing.T) { resolver := dataframeBuilderRestrictedEmptyScopeResolver() fieldCalls := 0 referenceCalls := 0 - service := NewService(Config{ + service := NewService(Config{CompileTarget: compiler.ArangoTarget(), ScopeResolver: resolver, ActiveManifestResolver: &builderActiveManifestResolver{manifest: builderReadyManifest(t, "P1", "generation-1")}, DiscoverFields: func(_ context.Context, options catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) { diff --git a/internal/api/graphql/graph/query/doc.go b/internal/api/graphql/graph/query/doc.go index 5d8d064e9..ba02d10a1 100644 --- a/internal/api/graphql/graph/query/doc.go +++ b/internal/api/graphql/graph/query/doc.go @@ -1,4 +1 @@ -// Package queryapi adapts GraphQL dataframe requests to Loom's dataframe -// execution layer. It owns input mapping, catalog discovery, validation, -// templates, and direct Arango-backed dataframe execution. package queryapi diff --git a/internal/api/graphql/graph/query/errors.go b/internal/api/graphql/graph/query/errors.go index 3efb9b2b6..2ea6f5687 100644 --- a/internal/api/graphql/graph/query/errors.go +++ b/internal/api/graphql/graph/query/errors.go @@ -8,9 +8,6 @@ import ( "github.com/calypr/loom/internal/dataframe/recipe" ) -// classifyError is the one transport-neutral classification boundary for -// query services. Drivers and Fence errors are kept as causes, while public -// callers receive a stable code and never a driver message. func classifyError(err error) error { if err == nil { return nil diff --git a/internal/api/graphql/graph/query/fhir_service.go b/internal/api/graphql/graph/query/fhir_service.go index d0d465e17..d789a10c5 100644 --- a/internal/api/graphql/graph/query/fhir_service.go +++ b/internal/api/graphql/graph/query/fhir_service.go @@ -1,10 +1,5 @@ package queryapi -// This file contains the transport-neutral FHIR document query facade. It -// deliberately delegates planning, authorization, generation resolution, and -// execution to the same dataframe recipe/compiler/execution path as the rest of -// GraphQL; no AQL or Arango client is constructed here. - import ( "context" "crypto/sha256" @@ -26,9 +21,6 @@ const ( FHIRMaxReadLimit = 10000 ) -// FHIRListRequest is the canonical request used by generated GraphQL root -// adapters. Filters are the existing FhirFilterInput values; there is no -// second filter language for this API. type FHIRListRequest struct { Project string ResourceType string @@ -36,17 +28,12 @@ type FHIRListRequest struct { Limit int } -// FHIRListResult contains full resource envelopes. The runtime row shape is -// intentionally hidden from GraphQL callers; each map is decoded by the -// generated resource registry in the resolver layer. type FHIRListResult struct { Resources []map[string]any Diagnostics any ReadContext FHIRReadContext } -// FHIRReadContext pins reference lookups to the authorization decision and -// active dataset generation used by the owning root resource. type FHIRReadContext struct { Project string DatasetGeneration string @@ -54,9 +41,6 @@ type FHIRReadContext struct { Scope authscope.ReadScope } -// ListFHIR executes one whole-document FHIR read through the production -// semantic and physical compiler. The Document expression is lowered by the -// compiler to an envelope containing payload, id, resourceType, and _key. func (s *Service) ListFHIR(ctx context.Context, request FHIRListRequest) (*FHIRListResult, error) { request, err := normalizeFHIRListRequest(request) if err != nil { @@ -66,9 +50,6 @@ func (s *Service) ListFHIR(ctx context.Context, request FHIRListRequest) (*FHIRL return nil, dataframeerrors.NewError(dataframeerrors.CodeInvalidLimit, "limit must not exceed 10000 without project write access") } - // Reuse the existing GraphQL input preparation. Besides resolving active - // READY generation and authorization scope, this validates fieldRef - // provenance and preserves all existing filter/operator semantics. in := model.FhirDataframeInput{ Project: request.Project, RootResourceType: request.ResourceType, @@ -88,8 +69,6 @@ func (s *Service) ListFHIR(ctx context.Context, request FHIRListRequest) (*FHIRL return s.listFHIRPrepared(ctx, request, prepared, read) } -// ListFHIRInContext executes an internal reference batch against the exact -// generation and read scope authorized for its owning resource. func (s *Service) ListFHIRInContext(ctx context.Context, read FHIRReadContext, request FHIRListRequest) (*FHIRListResult, error) { request.Project = read.Project normalized, err := normalizeFHIRListRequest(request) @@ -119,8 +98,6 @@ func (s *Service) listFHIRPrepared(ctx context.Context, request FHIRListRequest, if len(bundle.Outputs) != 1 { return nil, dataframeerrors.NewError(dataframeerrors.CodeInvalidRequest, "") } - // The whole-document expression is backend-neutral. Compiler support is - // additive to the existing selector/literal/call expression nodes. bundle.Outputs[0].Fields = []recipe.Field{{ Name: "resource", Expr: recipe.Expression{Document: &recipe.DocumentRef{Context: "root"}}, @@ -135,7 +112,7 @@ func (s *Service) listFHIRPrepared(ctx context.Context, request FHIRListRequest, if err != nil { return nil, queryInvalid(dataframeerrors.CodeInvalidRequest, err) } - compiled, err := compileSingleQuery(resolved, request.Limit) + compiled, err := compileSingleQuery(s.compileTarget, resolved, request.Limit) if err != nil { return nil, queryInvalid(dataframeerrors.CodeInvalidRequest, err) } @@ -151,8 +128,6 @@ func (s *Service) listFHIRPrepared(ctx context.Context, request FHIRListRequest, } envelope, ok := value.(map[string]any) if !ok { - // Some test/runtime adapters return JSON-shaped values. Normalize - // those through JSON without exposing backend implementation details. data, marshalErr := json.Marshal(value) if marshalErr != nil { return nil, dataframeerrors.Wrap(marshalErr, dataframeerrors.CodeResourceDecodeFailed, "") @@ -164,9 +139,6 @@ func (s *Service) listFHIRPrepared(ctx context.Context, request FHIRListRequest, if envelope == nil { continue } - // Normalize the storage envelope for generated FHIR models. The - // compiler emits payload plus envelope metadata under `key`; overlay - // id/resourceType and intentionally omit all internal keys. if payload, ok := envelope["payload"].(map[string]any); ok { overlay := make(map[string]any, len(payload)+2) for key, value := range payload { diff --git a/internal/api/graphql/graph/query/fhir_service_test.go b/internal/api/graphql/graph/query/fhir_service_test.go index d029e3079..16aaea62e 100644 --- a/internal/api/graphql/graph/query/fhir_service_test.go +++ b/internal/api/graphql/graph/query/fhir_service_test.go @@ -2,6 +2,7 @@ package queryapi import ( "context" + "github.com/calypr/loom/internal/dataframe/compiler" "testing" "github.com/calypr/loom/internal/authscope" @@ -22,7 +23,7 @@ func (f fhirLimitResourceAccess) GetAllowedResources(_ context.Context, _ string } func TestFHIRLimitRequiresProjectWriteAccessAboveReadCap(t *testing.T) { - if _, err := NewService(Config{}).ListFHIR(context.Background(), FHIRListRequest{ + if _, err := NewService(Config{CompileTarget: compiler.ArangoTarget()}).ListFHIR(context.Background(), FHIRListRequest{ Project: "P1", ResourceType: "Patient", Limit: FHIRMaxReadLimit + 1, }); err == nil { t.Fatal("ListFHIR() accepted an uncapped read without a scope resolver") @@ -45,14 +46,20 @@ func TestFHIRLimitRequiresProjectWriteAccessAboveReadCap(t *testing.T) { return []string{"example-allowed"}, nil }, }) - service := NewService(Config{ + service := NewService(Config{CompileTarget: compiler.ArangoTarget(), ScopeResolver: scopeResolver, ActiveManifestResolver: &builderActiveManifestResolver{manifest: builderReadyManifest(t, "P1", "generation-1")}, DiscoverFields: func(context.Context, catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) { return nil, nil }, Dataframes: dataframeexecution.NewService(dataframeexecution.ServiceConfig{ - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil }, }), diff --git a/internal/api/graphql/graph/query/graph_service.go b/internal/api/graphql/graph/query/graph_service.go index f9dc3aec5..6f3391471 100644 --- a/internal/api/graphql/graph/query/graph_service.go +++ b/internal/api/graphql/graph/query/graph_service.go @@ -1,11 +1,5 @@ package queryapi -// Graph mode is a deliberately small adapter over the dataframe semantic -// pipeline. It owns the transport contract (required-by-default traversals, -// bounded lookahead pagination, and public resource sanitization); route -// validation, generation/scope resolution, and AQL lowering remain in the -// canonical compiler. - import ( "context" "encoding/json" @@ -25,9 +19,6 @@ const ( FhirGraphMaxSteps = 32 ) -// FHIRGraphQuery is transport-neutral and intentionally mirrors only the -// explicit graph DSL. GraphQL resolvers can map generated gqlgen models into -// this type without exposing compiler or storage details. type FHIRGraphQuery struct { Project string AuthResourcePaths []string @@ -76,8 +67,6 @@ type FHIRGraphResult struct { Diagnostics any } -// FHIRGraphExplanation is intentionally sanitized. The AQL text, bind vars, -// collection names, and authorization predicates never cross this boundary. type FHIRGraphExplanation struct { SourceGeneration string RootResourceType string @@ -167,9 +156,6 @@ func graphTraversalInputs(in []FHIRGraphTraversal) []*model.FhirTraversalStepInp return out } -// RunFHIRGraph compiles and executes one graph request. The compiler facade -// returns a single physical query whose RETURN operation performs the global -// path union/dedupe/sort and limit+1 lookahead. func (s *Service) RunFHIRGraph(ctx context.Context, input FHIRGraphQuery) (*FHIRGraphResult, error) { normalized, err := normalizeFHIRGraphQuery(input) if err != nil { @@ -190,7 +176,7 @@ func (s *Service) RunFHIRGraph(ctx context.Context, input FHIRGraphQuery) (*FHIR if err != nil { return nil, queryInvalid(dataframeerrors.CodeInvalidRequest, err) } - compiled, err := compiler.CompileResolvedGraphQueryWithPolicy(resolved, normalized.Limit, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := compiler.CompileResolvedGraphQuery(s.compileTarget, resolved, normalized.Limit, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { return nil, queryInvalid(dataframeerrors.CodeInvalidRequest, err) } @@ -215,9 +201,6 @@ func (s *Service) RunFHIRGraph(ctx context.Context, input FHIRGraphQuery) (*FHIR return &FHIRGraphResult{SourceGeneration: generation, Paths: paths, ReturnedCount: len(paths), PageInfo: FHIRGraphPageInfo{HasMore: hasMore}, Diagnostics: runtimeResult.Diagnostics}, nil } -// ExplainFHIRGraph performs semantic/compiler validation. Live explain is an -// optional compiler operation and must only issue Arango EXPLAIN; the facade -// intentionally exposes no physical query text. func (s *Service) ExplainFHIRGraph(ctx context.Context, input FHIRGraphQuery, live bool) (*FHIRGraphExplanation, error) { normalized, err := normalizeFHIRGraphQuery(input) if err != nil { @@ -236,17 +219,11 @@ func (s *Service) ExplainFHIRGraph(ctx context.Context, input FHIRGraphQuery, li if err != nil { return nil, queryInvalid(dataframeerrors.CodeInvalidRequest, err) } - // The compiler facade performs semantic/physical validation without - // executing the result query. When requested, the runtime issues only an - // Arango EXPLAIN call; this transport contract never exposes AQL or bind - // variables. - compiled, err := compiler.CompileResolvedGraphQueryWithPolicy(resolved, normalized.Limit, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := compiler.CompileResolvedGraphQuery(s.compileTarget, resolved, normalized.Limit, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { return nil, err } if live { - // Explain only: this invokes Arango's EXPLAIN endpoint and never opens a - // result cursor. The assessment is intentionally not returned here. if s.explain == nil { return nil, dataframeerrors.NewError(dataframeerrors.CodeBackendUnavailable, "", dataframeerrors.WithRetryable(true)) } diff --git a/internal/api/graphql/graph/query/input_resolution.go b/internal/api/graphql/graph/query/input_resolution.go index 32b24053d..3daff1612 100644 --- a/internal/api/graphql/graph/query/input_resolution.go +++ b/internal/api/graphql/graph/query/input_resolution.go @@ -12,8 +12,6 @@ import ( "github.com/calypr/loom/internal/projectid" ) -// prepareRunInput resolves field references and returns the effective scope -// and selected generation alongside the GraphQL-shaped input. func (s *Service) prepareRunInput(ctx context.Context, input model.FhirDataframeInput) (model.FhirDataframeInput, authscope.ReadScope, string, error) { input.Project = projectid.Canonical(input.Project) if input.Project == "" { @@ -208,10 +206,6 @@ func (s *Service) resolveReadScopeForGeneration(ctx context.Context, principal * returnScope, err := s.scopeResolver.ResolveReadScopeForGeneration(ctx, principal, project, datasetGeneration, requested) return returnScope, queryBackend(err) } - // F-B22 Tier-1: with no resolver wired, client- and IdP-supplied paths - // become \x00-joined scope digests downstream. Normalize at this boundary - // so control bytes, stray slashes, and duplicate spellings can never - // form a scope; an all-garbage non-empty request is refused, not widened. normalizedRequested := authscope.NormalizeAuthResourcePaths(requested) if len(requested) > 0 && len(normalizedRequested) == 0 { return authscope.ReadScope{}, dataframeerrors.NewError(dataframeerrors.CodeInvalidRequest, "") diff --git a/internal/api/graphql/graph/query/introspection_test.go b/internal/api/graphql/graph/query/introspection_test.go index fb3b2cb70..ebf580e01 100644 --- a/internal/api/graphql/graph/query/introspection_test.go +++ b/internal/api/graphql/graph/query/introspection_test.go @@ -2,6 +2,7 @@ package queryapi import ( "context" + "github.com/calypr/loom/internal/dataframe/compiler" "testing" "github.com/calypr/loom/internal/authscope" @@ -12,7 +13,7 @@ func TestServiceIntrospectUsesCallerScope(t *testing.T) { var gotRefOpts catalog.PopulatedReferenceOptions var gotFieldOpts []catalog.PopulatedFieldOptions - service := NewService(Config{ + service := NewService(Config{CompileTarget: compiler.ArangoTarget(), ActiveManifestResolver: &builderActiveManifestResolver{manifest: builderReadyManifest(t, "P1", "generation-1")}, DiscoverReferences: func(ctx context.Context, opts catalog.PopulatedReferenceOptions) ([]catalog.PopulatedReference, error) { gotRefOpts = opts @@ -68,7 +69,7 @@ func TestServiceIntrospectUsesCallerScope(t *testing.T) { } func TestServiceIntrospectRejectsUnauthorizedScope(t *testing.T) { - service := NewService(Config{}) + service := NewService(Config{CompileTarget: compiler.ArangoTarget()}) ctx := authscope.ContextWithPrincipal(context.Background(), &authscope.Principal{ Subject: "u1", Projects: []string{"P1"}, @@ -86,7 +87,7 @@ func TestServiceIntrospectRejectsUnauthorizedScope(t *testing.T) { } func TestServiceIntrospectRejectsUnauthorizedProject(t *testing.T) { - service := NewService(Config{}) + service := NewService(Config{CompileTarget: compiler.ArangoTarget()}) ctx := authscope.ContextWithPrincipal(context.Background(), &authscope.Principal{ Subject: "u1", Projects: []string{"P1"}, diff --git a/internal/api/graphql/graph/query/postgres_query_test.go b/internal/api/graphql/graph/query/postgres_query_test.go new file mode 100644 index 000000000..1d33a47e8 --- /dev/null +++ b/internal/api/graphql/graph/query/postgres_query_test.go @@ -0,0 +1,782 @@ +package queryapi + +import ( + "context" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/calypr/loom/generated/graphql/graph/model" + "github.com/calypr/loom/internal/authscope" + backendspostgres "github.com/calypr/loom/internal/backends/postgres" + "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/catalog" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + publication "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/ingest" + "github.com/jackc/pgx/v5/pgxpool" +) + +func qRoutePostgreSQLCompileTarget(t *testing.T) compiler.Target { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("verify_qroute_source") + if err != nil { + t.Fatalf("bind PostgreSQL storage placement: %v", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("select PostgreSQL compile target: %v", err) + } + return target +} + +func TestFhirDataframeNilDiscoveryRefusesTyped(t *testing.T) { + rowPortCalls := 0 + dataframes := dataframeexecution.NewService(dataframeexecution.ServiceConfig{ + SourceRows: func(context.Context, dataframeexecution.RowsRequest, dataframeexecution.RowVisitor) error { + rowPortCalls++ + return nil + }, + }) + service := NewService(Config{ + CompileTarget: qRoutePostgreSQLCompileTarget(t), + Dataframes: dataframes, + DiscoverFields: nil, + DiscoverReferences: nil, + }) + + result, err := service.Run(context.Background(), queryapiGenderInput(), nil) + if err == nil { + t.Fatalf("Run with unwired discovery ports returned %+v; it must fail closed, not degrade", result) + } + if rowPortCalls != 0 { + t.Fatalf("unwired discovery still contacted the row port %d times: the refusal must precede any backend read", rowPortCalls) + } + var typed *dataframeerrors.Error + if !errors.As(err, &typed) { + t.Fatalf("discovery refusal %v is not a typed dataframe error", err) + } + if typed.Code() != string(dataframeerrors.CodeBackendUnavailable) { + t.Fatalf("discovery refusal code = %q, want %q", typed.Code(), dataframeerrors.CodeBackendUnavailable) + } + if result != nil { + t.Fatalf("typed refusal must not carry a partial result: %+v", result) + } +} + +const ( + qrLiveProject = "qroute-p1" + qrLiveGenA = "generation-a" + qrLiveGenB = "generation-b" + qrLoadPath = "/fhir" +) + +type qrLiveFixture struct { + dsn string + srcSchema string + relSchema string + database string + scrub func(any) string + bootstrap *backendspostgres.Runtime +} + +var ( + qrDBPattern = regexp.MustCompile(`^verify_qroute_q[0-9]+_[0-9a-f]+$`) + qrCensusSibling = regexp.MustCompile(`^verify_qroute_[0-9]+_[0-9a-f]+$`) + qrOwnedMu sync.Mutex + qrOwned = map[string]qrMint{} + qrStrayMarks = []string{} + qrBaselineMu sync.Mutex + qrBaseline int64 + qrBaselineSet bool + qrBaselineErr error +) + +type qrMint struct { + leg string + dropped bool +} + +var qrMintInventory = map[string]int{ + "TestFhirDataframePostgreSQLLiveExactScalarRows": 1, + "TestFhirDataframePostgreSQLLiveRestrictedPrincipalSeesNoRows": 1, + "TestFhirDataframePostgreSQLLiveActiveGenerationExcludesStaged": 1, + "TestFhirDataframePostgreSQLLiveBackendOutageTyped": 1, + "TestFhirDataframePostgreSQLLivePreCancellationTyped": 1, + "TestFhirDataframePostgreSQLLiveGarbagePrincipalPathsFailClosed": 1, +} + +func qrRegister(db, leg string) { + qrOwnedMu.Lock() + defer qrOwnedMu.Unlock() + qrOwned[db] = qrMint{leg: leg} +} + +func qrMarkDropped(db string) { + qrOwnedMu.Lock() + defer qrOwnedMu.Unlock() + rec, ok := qrOwned[db] + if !ok { + qrStrayMarks = append(qrStrayMarks, db) + return + } + rec.dropped = true + qrOwned[db] = rec +} + +func qrCaptureBaseline(t *testing.T, raw string) { + t.Helper() + qrBaselineMu.Lock() + defer qrBaselineMu.Unlock() + if qrBaselineSet { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + qrBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname ~ '^verify_qroute_q[0-9]+_[0-9a-f]+$'"). + Scan(&count); err != nil { + qrBaselineErr = fmt.Errorf("baseline count: %s", scrubQRStatic(err, raw)) + return + } + qrBaseline, qrBaselineSet = count, true +} + +func scrubQRStatic(err error, raw string) string { + s := err.Error() + if u, perr := url.Parse(raw); perr == nil { + secrets := []string{raw} + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + } + return s +} + +func qrTestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL first query route live legs") + } + return raw +} + +func qrRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) +} + +func newQRLiveFixture(t *testing.T) *qrLiveFixture { + t.Helper() + raw := qrTestPostgresURL(t) + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + + ctx, cancel := context.WithTimeout(context.Background(), 420*time.Second) + t.Cleanup(cancel) + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %s", scrub(err)) + } + t.Cleanup(admin.Close) + + database := fmt.Sprintf("verify_qroute_q%d_%s", time.Now().UnixNano(), qrRandomHex(t)) + if !qrDBPattern.MatchString(database) { + t.Fatalf("mint name %q violates the owned verify_qroute_q pattern", database) + } + qrRegister(database, t.Name()) + qrCaptureBaseline(t, raw) + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+database+`"`); err != nil { + qrMarkDropped(database) + t.Fatalf("create isolated database: %s", scrub(err)) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(context.Background()), + `DROP DATABASE IF EXISTS "`+database+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + return + } + qrMarkDropped(database) + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + } + u.Path = "/" + database + dsn := u.String() + secrets = append(secrets, dsn) + + f := &qrLiveFixture{dsn: dsn, database: database, scrub: scrub, + srcSchema: fmt.Sprintf("verify_qroute_q%d_%s_src", time.Now().UnixNano(), qrRandomHex(t)), + relSchema: fmt.Sprintf("verify_qroute_q%d_%s_rel", time.Now().UnixNano(), qrRandomHex(t))} + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect to isolated database: %s", scrub(err)) + } + for _, schema := range []string{f.srcSchema, f.relSchema} { + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + iso.Close() + t.Fatalf("create series schema: %s", scrub(err)) + } + } + iso.Close() + + bootstrap, err := backendspostgres.Open(ctx, control.Config{ + DSN: dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 30 * time.Second, + Schemas: control.SeriesSchemas{ + control.SeriesSource: f.srcSchema, + control.SeriesDatasetRelease: f.relSchema, + }, + Setup: control.SetupMigrate, + }) + if err != nil { + t.Fatalf("bootstrap fixture series: %s", scrub(err)) + } + if bootstrap.FieldProfiles == nil || bootstrap.Inventory == nil || bootstrap.Relationships == nil || bootstrap.Contributions == nil { + _ = bootstrap.Close() + t.Fatal("fixture runtime assembled without catalog evidence ports") + } + t.Cleanup(func() { _ = bootstrap.Close() }) + f.bootstrap = bootstrap + return f +} + +func (f *qrLiveFixture) load(t *testing.T, project, generation string, files map[string][]string) { + t.Helper() + dir := t.TempDir() + for name, lines := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil { + t.Fatalf("write staging fixture: %v", err) + } + } + ref, err := publication.NewRef(project, generation) + if err != nil { + t.Fatalf("build dataset ref: %v", err) + } + opts := ingest.LoadOptions{ + Schema: qrRepoPath(t, "schemas", "graph-fhir.json"), + MetaDir: dir, + Project: project, + Dataset: &ref, + AuthResourcePath: qrLoadPath, + StageOnly: true, + } + pg := ingest.PGSourceOptions{ + DSN: f.dsn, Schema: f.srcSchema, ManifestSchema: f.relSchema, + ApplicationName: "verify-qroute-live", + Catalog: ingest.GenerationCatalogWriters{ + FieldProfiles: f.bootstrap.FieldProfiles, + Inventory: f.bootstrap.Inventory, + Relationships: f.bootstrap.Relationships, + Contributions: f.bootstrap.Contributions, + }, + } + if _, err := ingest.LoadPostgresGeneration(context.Background(), opts, pg); err != nil { + t.Fatalf("load %s/%s: %s", project, generation, f.scrub(err)) + } +} + +func (f *qrLiveFixture) activate(t *testing.T, project, generation string) { + t.Helper() + ref, err := publication.NewRef(project, generation) + if err != nil { + t.Fatalf("build dataset ref: %v", err) + } + pg := ingest.PGSourceOptions{ + DSN: f.dsn, Schema: f.srcSchema, ManifestSchema: f.relSchema, + ApplicationName: "verify-qroute-live-activate", + } + if err := ingest.ActivatePostgresGeneration(context.Background(), ref, pg); err != nil { + t.Fatalf("activate %s/%s: %s", project, generation, f.scrub(err)) + } +} + +func (f *qrLiveFixture) serviceRuntime(t *testing.T) *backendspostgres.Runtime { + t.Helper() + rt, err := backendspostgres.Open(context.Background(), control.Config{ + DSN: f.dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 30 * time.Second, + Schemas: control.SeriesSchemas{ + control.SeriesSource: f.srcSchema, + control.SeriesDatasetRelease: f.relSchema, + }, + Setup: control.SetupNone, + }) + if err != nil { + t.Fatalf("open service runtime: %s", f.scrub(err)) + } + t.Cleanup(func() { _ = rt.Close() }) + return rt +} + +func qrRepoPath(t *testing.T, elems ...string) string { + t.Helper() + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("locate test file for repository root") + } + dir := filepath.Dir(thisFile) + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return filepath.Join(append([]string{dir}, elems...)...) + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("no go.mod found above the test file") + } + dir = parent + } +} + +func qrPatientLine(t *testing.T, id, gender, birthDate string) string { + t.Helper() + encoded, err := json.Marshal(map[string]any{ + "resourceType": "Patient", "id": id, "gender": gender, "birthDate": birthDate, + }) + if err != nil { + t.Fatalf("marshal patient fixture: %v", err) + } + return string(encoded) +} + +func qrPatientInput(paths []string) *model.FhirDataframeInput { + input := &model.FhirDataframeInput{ + Project: qrLiveProject, + RootResourceType: "Patient", + AuthResourcePaths: paths, + } + for _, name := range []string{"birthDate", "gender"} { + input.RootFields = append(input.RootFields, &model.FhirFieldSelectInput{ + Name: name, Selector: &model.FhirFieldSelectorInput{ValuePath: name}, + }) + } + return input +} + +func qrDispatchGuardedRows(execute dataframeexecution.SourceRows) dataframeexecution.SourceRows { + return func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + if _, _, err := request.Command.PostgreSQLParameters(); err != nil { + return err + } + return execute(ctx, request, visit) + } +} + +type qrCounters struct { + rowCalls atomic.Int64 + discoverCalls atomic.Int64 +} + +func qrService(t *testing.T, f *qrLiveFixture, rt *backendspostgres.Runtime, counters *qrCounters) *Service { + t.Helper() + binding, err := ir.NewPostgresStorageBinding(f.srcSchema) + if err != nil { + t.Fatalf("bind PostgreSQL storage placement: %v", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("select PostgreSQL compile target: %v", err) + } + countedRows := func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + counters.rowCalls.Add(1) + return qrDispatchGuardedRows(rt.SourceRows)(ctx, request, visit) + } + dataframes := dataframeexecution.NewService(dataframeexecution.ServiceConfig{ + SourceRows: countedRows, + }) + cachedDiscovery := catalog.NewCache().DiscoverFields(rt.DiscoverFields) + countedDiscovery := func(ctx context.Context, options catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) { + counters.discoverCalls.Add(1) + return cachedDiscovery(ctx, options) + } + return NewService(Config{ + CompileTarget: target, + Dataframes: dataframes, + ScopeResolver: nil, + ActiveManifestResolver: rt.Manifests, + DiscoverFields: countedDiscovery, + }) +} + +func TestFhirDataframePostgreSQLLiveExactScalarRows(t *testing.T) { + f := newQRLiveFixture(t) + f.load(t, qrLiveProject, qrLiveGenA, map[string][]string{"Patient.ndjson": { + qrPatientLine(t, "patient-ada", "female", "1815-12-10"), + qrPatientLine(t, "patient-alan", "male", "1912-06-23"), + }}) + f.activate(t, qrLiveProject, qrLiveGenA) + var counters qrCounters + service := qrService(t, f, f.serviceRuntime(t), &counters) + + result, err := service.Run(context.Background(), *qrPatientInput(nil), nil) + if err != nil { + t.Fatalf("live scoped read failed: %s", f.scrub(err)) + } + if counters.discoverCalls.Load() == 0 { + t.Fatal("discovery-backed query never consulted the backend read lane") + } + if counters.rowCalls.Load() == 0 { + t.Fatal("exact-rows leg never contacted the row port") + } + if result == nil || result.RowCount != 2 || len(result.Rows) != 2 { + t.Fatalf("result = %+v, want exactly 2 rows", result) + } + gotColumns := append([]string(nil), result.Columns...) + sort.Strings(gotColumns) + if want := []string{"_key", "birthDate", "gender"}; !reflect.DeepEqual(gotColumns, want) { + t.Fatalf("columns = %v, want %v", result.Columns, want) + } + wantRows := []map[string]string{ + {"_key": "patient-ada", "birthDate": "1815-12-10", "gender": "female"}, + {"_key": "patient-alan", "birthDate": "1912-06-23", "gender": "male"}, + } + var gotRows []map[string]string + for _, row := range result.Rows { + decoded := map[string]string{} + for name, value := range row { + var scalar string + switch v := value.(type) { + case json.RawMessage: + if err := json.Unmarshal(v, &scalar); err != nil { + t.Fatalf("column %q is not a JSON string (%s): %v", name, v, err) + } + case string: + scalar = v + default: + t.Fatalf("column %q came back as %T, want raw JSON or string from the row port", name, value) + } + decoded[name] = scalar + } + gotRows = append(gotRows, decoded) + } + sort.Slice(gotRows, func(i, j int) bool { return gotRows[i]["_key"] < gotRows[j]["_key"] }) + if !reflect.DeepEqual(gotRows, wantRows) { + t.Fatalf("rows = %#v, want the exact loaded scalars %#v", gotRows, wantRows) + } + if result.Diagnostics.Total <= 0 { + t.Fatalf("diagnostics missing: %+v", result.Diagnostics) + } +} + +func TestFhirDataframePostgreSQLLiveRestrictedPrincipalSeesNoRows(t *testing.T) { + f := newQRLiveFixture(t) + f.load(t, qrLiveProject, qrLiveGenA, map[string][]string{"Patient.ndjson": { + qrPatientLine(t, "patient-ada", "female", "1815-12-10"), + qrPatientLine(t, "patient-alan", "male", "1912-06-23"), + }}) + f.activate(t, qrLiveProject, qrLiveGenA) + var counters qrCounters + service := qrService(t, f, f.serviceRuntime(t), &counters) + + control, err := service.Run(context.Background(), *qrPatientInput(nil), nil) + if err != nil || control == nil || control.RowCount != 2 { + t.Fatalf("unrestricted control = %+v err=%v, want 2 rows (fixture is not vacuous)", control, f.scrub(err)) + } + + restrictedCtx := authscope.ContextWithPrincipal(context.Background(), &authscope.Principal{ + Subject: "verify_qroute_nurse", AuthResourcePaths: []string{"/fhir-nurse"}, + }) + result, err := service.Run(restrictedCtx, *qrPatientInput([]string{"/fhir-nurse"}), nil) + if err != nil { + t.Fatalf("restricted read failed: %s", f.scrub(err)) + } + if result == nil || result.RowCount != 0 || len(result.Rows) != 0 { + t.Fatalf("restricted read = %+v, want 0 rows with a success envelope", result) + } +} + +func TestFhirDataframePostgreSQLLiveActiveGenerationExcludesStaged(t *testing.T) { + f := newQRLiveFixture(t) + f.load(t, qrLiveProject, qrLiveGenA, map[string][]string{"Patient.ndjson": { + qrPatientLine(t, "patient-ada", "female", "1815-12-10"), + }}) + f.load(t, qrLiveProject, qrLiveGenB, map[string][]string{"Patient.ndjson": { + qrPatientLine(t, "patient-zoe", "female", "1955-03-04"), + }}) + f.activate(t, qrLiveProject, qrLiveGenA) + var counters qrCounters + service := qrService(t, f, f.serviceRuntime(t), &counters) + + result, err := service.Run(context.Background(), *qrPatientInput(nil), nil) + if err != nil { + t.Fatalf("active-generation read failed: %s", f.scrub(err)) + } + if result == nil || result.RowCount != 1 { + t.Fatalf("active-generation read = %+v, want exactly the 1 gen-A row", result) + } + for _, row := range result.Rows { + if strings.Contains(fmt.Sprintf("%v", row), "1955-03-04") { + t.Fatalf("staged gen-B leaked into the active-generation read: %#v", row) + } + } +} + +func TestFhirDataframePostgreSQLLiveBackendOutageTyped(t *testing.T) { + f := newQRLiveFixture(t) + f.load(t, qrLiveProject, qrLiveGenA, map[string][]string{"Patient.ndjson": { + qrPatientLine(t, "patient-ada", "female", "1815-12-10"), + }}) + f.activate(t, qrLiveProject, qrLiveGenA) + var counters qrCounters + rt := f.serviceRuntime(t) + service := qrService(t, f, rt, &counters) + + control, err := service.Run(context.Background(), *qrPatientInput(nil), nil) + if err != nil || control == nil || control.RowCount != 1 { + t.Fatalf("alive control = %+v err=%v, want 1 row before the outage", control, f.scrub(err)) + } + if err := rt.Close(); err != nil { + t.Fatalf("close pool for outage: %s", f.scrub(err)) + } + _, err = service.Run(context.Background(), *qrPatientInput(nil), nil) + if err == nil { + t.Fatal("read after pool close unexpectedly succeeded") + } + var typed *dataframeerrors.Error + if !errors.As(err, &typed) { + t.Fatalf("outage error %v is not a typed dataframe error", err) + } + if typed.Code() != string(dataframeerrors.CodeBackendUnavailable) { + t.Fatalf("outage code = %q, want %q", typed.Code(), dataframeerrors.CodeBackendUnavailable) + } + if !typed.Retryable() { + t.Fatal("a closed pool must stay retry-classified (a fresh pool serves the next attempt)") + } +} + +func TestFhirDataframePostgreSQLLivePreCancellationTyped(t *testing.T) { + f := newQRLiveFixture(t) + f.load(t, qrLiveProject, qrLiveGenA, map[string][]string{"Patient.ndjson": { + qrPatientLine(t, "patient-ada", "female", "1815-12-10"), + }}) + f.activate(t, qrLiveProject, qrLiveGenA) + var counters qrCounters + service := qrService(t, f, f.serviceRuntime(t), &counters) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := service.Run(ctx, *qrPatientInput(nil), nil) + if err == nil { + t.Fatal("pre-cancelled read unexpectedly succeeded") + } + var typed *dataframeerrors.Error + if !errors.As(err, &typed) { + t.Fatalf("cancellation error %v is not a typed dataframe error", err) + } + t.Logf("cancellation classification witness: code=%q retryable=%v", typed.Code(), typed.Retryable()) +} + +func TestQRouteLiveCensusVocabularyCorrespondence(t *testing.T) { + qrHex := func() string { + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("sample suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) + } + for i := 0; i < 200; i++ { + name := fmt.Sprintf("verify_qroute_q%d_%s", time.Now().UnixNano(), qrHex()) + if !qrDBPattern.MatchString(name) { + t.Fatalf("mint-shaped name %q does not match the own census regex", name) + } + if qrCensusSibling.MatchString(name) { + t.Fatalf("own-vocabulary name %q matched internal/server's census regex: parallel census race is back", name) + } + } + ownPositives := []string{"verify_qroute_q1731234567890_0123456789ab"} + ownNegatives := []string{ + "verify_qroute_q_zz", + "verify_qroute_1_x", + "verify_qroute_zz", + "verify_qroute_1731234567890_0123456789ab", + "verify_qroute_qq1731234567890_0123456789ab", + "verify_qroute_q1731234567890_0123456789AB", + "verify_qroute_q_0123456789ab", + "verify_qroute_q1731234567890_", + "verify_qroute_q1731234567890_0123456789ab_src", + "verify_qroute_q1731234567890_z123456789ab", + "xverify_qroute_q1731234567890_0123456789ab", + } + for _, name := range ownPositives { + if !qrDBPattern.MatchString(name) { + t.Errorf("positive %q must match the own census regex", name) + } + if qrCensusSibling.MatchString(name) { + t.Errorf("positive %q must not match the sibling census regex", name) + } + } + for _, name := range ownNegatives { + if qrDBPattern.MatchString(name) { + t.Errorf("never-minted shape %q matched the own census regex: the census can now hide foreign churn under this file's name", name) + } + } + for i := 0; i < 200; i++ { + sibling := fmt.Sprintf("verify_qroute_%d_%s", time.Now().UnixNano(), qrHex()) + if qrDBPattern.MatchString(sibling) { + t.Fatalf("sibling-vocabulary name %q matched this file's census regex", sibling) + } + } +} + +func TestFhirDataframePostgreSQLLiveGarbagePrincipalPathsFailClosed(t *testing.T) { + f := newQRLiveFixture(t) + f.load(t, qrLiveProject, qrLiveGenA, map[string][]string{"Patient.ndjson": { + qrPatientLine(t, "patient-ada", "female", "1815-12-10"), + }}) + f.activate(t, qrLiveProject, qrLiveGenA) + var counters qrCounters + service := qrService(t, f, f.serviceRuntime(t), &counters) + + control, err := service.Run(context.Background(), *qrPatientInput(nil), nil) + if err != nil || control == nil || control.RowCount != 1 { + t.Fatalf("alive control = %+v err=%v, want 1 row before the refusal", control, f.scrub(err)) + } + counters.rowCalls.Store(0) + counters.discoverCalls.Store(0) + + garbageCtx := authscope.ContextWithPrincipal(context.Background(), &authscope.Principal{ + Subject: "verify_qroute_garbage", AuthResourcePaths: []string{"\x00\x01\x02"}, + }) + _, err = service.Run(garbageCtx, *qrPatientInput(nil), nil) + if err == nil { + t.Fatal("garbage principal unexpectedly served rows") + } + var typed *dataframeerrors.Error + if !errors.As(err, &typed) { + t.Fatalf("garbage-principal refusal %v is not a typed dataframe error", err) + } + if typed.Code() != string(dataframeerrors.CodeForbidden) { + t.Fatalf("garbage-principal code = %q, want %q", typed.Code(), dataframeerrors.CodeForbidden) + } + if counters.rowCalls.Load() != 0 || counters.discoverCalls.Load() != 0 { + t.Fatalf("garbage-principal refusal reached the backend: rows=%d discovery=%d, want 0/0", + counters.rowCalls.Load(), counters.discoverCalls.Load()) + } +} + +func TestZZQRouteLiveZeroLeftoverFixtures(t *testing.T) { + raw := qrTestPostgresURL(t) + if !qrBaselineSet { + t.Skip("no live fixture was created in this run; nothing to audit") + } + if qrBaselineErr != nil { + t.Fatalf("baseline was never captured: %v", qrBaselineErr) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for audit: %s", scrubQRStatic(err, raw)) + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname ~ '^verify_qroute_q[0-9]+_[0-9a-f]+$'"). + Scan(&count); err != nil { + t.Fatalf("count leftovers: %s", scrubQRStatic(err, raw)) + } + qrOwnedMu.Lock() + registered, outstanding, strays := 0, []string{}, append([]string(nil), qrStrayMarks...) + perLeg := map[string]int{} + unmatched := []string{} + for db, rec := range qrOwned { + registered++ + perLeg[rec.leg]++ + if !rec.dropped { + outstanding = append(outstanding, db) + } + if !qrDBPattern.MatchString(db) { + unmatched = append(unmatched, db) + } + } + qrOwnedMu.Unlock() + if len(outstanding) > 0 { + t.Fatalf("owned fixtures still live at audit: %v", outstanding) + } + if len(strays) > 0 { + t.Fatalf("drop marks for names this process never minted: %v", strays) + } + if len(unmatched) > 0 { + t.Fatalf("registered mints outside the census regex: %v", unmatched) + } + for leg, want := range qrMintInventory { + if got := perLeg[leg]; got != want { + t.Errorf("mint inventory: leg %q minted %d databases, inventoried %d", leg, got, want) + } + delete(perLeg, leg) + } + for leg, got := range perLeg { + t.Errorf("mint inventory: leg %q minted %d databases but is not inventoried", leg, got) + } + if count != qrBaseline { + rows, err := admin.Query(ctx, + "SELECT datname FROM pg_database WHERE datname ~ '^verify_qroute_q[0-9]+_[0-9a-f]+$' ORDER BY datname") + names := []string{} + if err == nil { + for rows.Next() { + var name string + if err := rows.Scan(&name); err == nil { + names = append(names, name) + } + } + rows.Close() + } + t.Fatalf("own sub-namespace census: live=%d baseline=%d registered=%d names=%v", + count, qrBaseline, registered, names) + } +} diff --git a/internal/api/graphql/graph/query/project_identity_test.go b/internal/api/graphql/graph/query/project_identity_test.go index 2e3fdd9a7..a7d4ea12d 100644 --- a/internal/api/graphql/graph/query/project_identity_test.go +++ b/internal/api/graphql/graph/query/project_identity_test.go @@ -2,6 +2,7 @@ package queryapi import ( "context" + "github.com/calypr/loom/internal/dataframe/compiler" "reflect" "testing" @@ -39,7 +40,7 @@ func TestProjectFormsShareOneShotCatalogAndIntrospectionIdentity(t *testing.T) { t.Fatalf("catalog auth paths = %#v, want %#v", paths, []string{scopePath}) } } - service := NewService(Config{ + service := NewService(Config{CompileTarget: compiler.ArangoTarget(), ScopeResolver: scopeResolver, ActiveManifestResolver: active, DiscoverFields: func(_ context.Context, options catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) { @@ -59,7 +60,12 @@ func TestProjectFormsShareOneShotCatalogAndIntrospectionIdentity(t *testing.T) { return []catalog.PopulatedReference{}, nil }, Dataframes: dataframeexecution.NewService(dataframeexecution.ServiceConfig{ - QueryRows: func(_ context.Context, _ string, _ int, bindVars map[string]any, _ func(map[string]any) error) error { + SourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, bindVars, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + if unrestricted, ok := bindVars["auth_resource_paths_unrestricted"].(bool); !ok || unrestricted { t.Fatalf("dataframe scope bypass = %#v, want false", bindVars["auth_resource_paths_unrestricted"]) } diff --git a/internal/api/graphql/graph/query/read_scope_ingress_control_chars_e2e_test.go b/internal/api/graphql/graph/query/read_scope_ingress_control_chars_e2e_test.go index 9dadf29eb..0b9a94760 100644 --- a/internal/api/graphql/graph/query/read_scope_ingress_control_chars_e2e_test.go +++ b/internal/api/graphql/graph/query/read_scope_ingress_control_chars_e2e_test.go @@ -3,6 +3,7 @@ package queryapi import ( "context" "errors" + "github.com/calypr/loom/internal/dataframe/compiler" "reflect" "strings" "testing" @@ -11,12 +12,6 @@ import ( dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" ) -// F-B22 Tier-1 durable control for the resolver-less read-scope fallback: -// client- and IdP-supplied paths become \x00-joined scope digests -// downstream (fhirReadScopeDigest, explorer capability digests), so they -// must traverse ingest normalization first. Archived CONFIRMED wording is -// preserved verbatim as the REGRESSION branch of each case. - func errorcode(t *testing.T, err error) string { t.Helper() if err == nil { @@ -30,7 +25,7 @@ func errorcode(t *testing.T, err error) string { } func TestClientSuppliedControlCharPathRefused(t *testing.T) { - svc := NewService(Config{}) + svc := NewService(Config{CompileTarget: compiler.ArangoTarget()}) scope, err := svc.resolveReadScopeForGeneration(context.Background(), nil, "proj-a", "gen-1", []string{"x\x00y"}) if scope.Mode == authscope.ReadScopeRestricted && reflect.DeepEqual(scope.AuthResourcePaths, []string{"x\x00y"}) && err == nil { t.Fatalf("VERDICT=CONFIRMED (pre-fix archived shape): client-supplied path with NUL was copied raw into a restricted scope (AuthResourcePaths=%q); the \x00-join digest fold of distinct restricted grants remains constructible from request bytes", scope.AuthResourcePaths) @@ -45,7 +40,7 @@ func TestClientSuppliedControlCharPathRefused(t *testing.T) { } func TestClientSuppliedMixedPathsNormalized(t *testing.T) { - svc := NewService(Config{}) + svc := NewService(Config{CompileTarget: compiler.ArangoTarget()}) scope, err := svc.resolveReadScopeForGeneration(context.Background(), nil, "proj-a", "gen-1", []string{"/c-d/", "a-b", "junk\x01x", "a-b"}) if err != nil { t.Fatalf("VERDICT=UNEXPECTED err=%v", err) @@ -60,7 +55,7 @@ func TestClientSuppliedMixedPathsNormalized(t *testing.T) { } func TestPrincipalGarbageOnlyPathsFailClosed(t *testing.T) { - svc := NewService(Config{}) + svc := NewService(Config{CompileTarget: compiler.ArangoTarget()}) principal := &authscope.Principal{AuthResourcePaths: []string{"a\x00b"}} scope, err := svc.resolveReadScopeForGeneration(context.Background(), principal, "proj-a", "gen-1", nil) if scope.Mode == authscope.ReadScopeRestricted && reflect.DeepEqual(scope.AuthResourcePaths, []string{"a\x00b"}) && err == nil { @@ -76,7 +71,7 @@ func TestPrincipalGarbageOnlyPathsFailClosed(t *testing.T) { } func TestMembershipComparedInNormalizedSpace(t *testing.T) { - svc := NewService(Config{}) + svc := NewService(Config{CompileTarget: compiler.ArangoTarget()}) principal := &authscope.Principal{AuthResourcePaths: []string{"a-b"}} scope, err := svc.resolveReadScopeForGeneration(context.Background(), principal, "proj-a", "gen-1", []string{"/a-b/"}) if err != nil && strings.HasPrefix(errorcode(t, err), string(dataframeerrors.CodeForbidden)) { diff --git a/internal/api/graphql/graph/query/recipe_mapping.go b/internal/api/graphql/graph/query/recipe_mapping.go index cc23e0423..7fcf497f7 100644 --- a/internal/api/graphql/graph/query/recipe_mapping.go +++ b/internal/api/graphql/graph/query/recipe_mapping.go @@ -1,9 +1,5 @@ package queryapi -// This file is the transport-only GraphQL -> recipe boundary. It copies the -// already resolved GraphQL selections into the canonical recipe wire types; -// semantic validation and compilation remain in the recipe/semantic packages. - import ( "fmt" "strings" @@ -13,14 +9,6 @@ import ( "github.com/calypr/loom/internal/dataframe/spec" ) -// RecipeBundleFromInput creates the ephemeral one-output recipe used by a -// one-shot GraphQL dataframe request. Runtime scope (project, generation, -// authorization, and limit) is deliberately not copied into the bundle; the -// caller supplies it through recipe.RuntimeBindings. -// -// Field references must have been resolved by PrepareRunInput first. Keeping -// that boundary explicit prevents an unresolved catalog name from becoming an -// executable selector or an accidental storage-specific path. func RecipeBundleFromInput(in model.FhirDataframeInput) (recipe.Bundle, error) { if strings.TrimSpace(in.RootResourceType) == "" { return recipe.Bundle{}, fmt.Errorf("rootResourceType is required") diff --git a/internal/api/graphql/graph/query/recipe_mapping_test.go b/internal/api/graphql/graph/query/recipe_mapping_test.go index f0d451cac..baba5cbdc 100644 --- a/internal/api/graphql/graph/query/recipe_mapping_test.go +++ b/internal/api/graphql/graph/query/recipe_mapping_test.go @@ -2,6 +2,7 @@ package queryapi import ( "context" + "github.com/calypr/loom/internal/dataframe/compiler" "testing" "github.com/calypr/loom/generated/graphql/graph/model" @@ -112,7 +113,7 @@ func TestGraphQLRecipeResolutionUsesScopedCatalogBeforeSemanticTyping(t *testing t.Fatal(err) } var got catalog.PopulatedFieldOptions - service := NewService(Config{DiscoverFields: func(_ context.Context, options catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) { + service := NewService(Config{CompileTarget: compiler.ArangoTarget(), DiscoverFields: func(_ context.Context, options catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) { got = options return []catalog.PopulatedField{{ResourceType: "Patient", Path: "gender", Kind: "scalar"}}, nil }}) diff --git a/internal/api/graphql/graph/query/service.go b/internal/api/graphql/graph/query/service.go index 1c3a53eb3..9dc2d6c34 100644 --- a/internal/api/graphql/graph/query/service.go +++ b/internal/api/graphql/graph/query/service.go @@ -17,31 +17,29 @@ import ( ) type Service struct { + compileTarget compiler.Target discoverReferencesFn func(context.Context, catalog.PopulatedReferenceOptions) ([]catalog.PopulatedReference, error) discoverFieldsFn func(context.Context, catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) dataframes *dataframeexecution.Service scopeResolver *authscope.ScopeResolver activeManifestResolver publication.ActiveResolver - explain func(context.Context, dataframeexecution.CompiledQuery) error - // now is the single clock seam for the run-phase diagnostics so a test can - // hold the backend open without racing wall-clock speed thresholds. - now func() time.Time + explain func(context.Context, compiler.CompiledQuery) error + now func() time.Time } type Config struct { - DiscoverReferences func(context.Context, catalog.PopulatedReferenceOptions) ([]catalog.PopulatedReference, error) - DiscoverFields func(context.Context, catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) - Dataframes *dataframeexecution.Service - ScopeResolver *authscope.ScopeResolver - // ActiveManifestResolver is optional. When present, builder catalog - // discovery and recipe preparation resolve one READY active generation - // before inspecting any fields or relationship routes. + DiscoverReferences func(context.Context, catalog.PopulatedReferenceOptions) ([]catalog.PopulatedReference, error) + DiscoverFields func(context.Context, catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) + Dataframes *dataframeexecution.Service + CompileTarget compiler.Target + ScopeResolver *authscope.ScopeResolver ActiveManifestResolver publication.ActiveResolver - Explain func(context.Context, dataframeexecution.CompiledQuery) error + Explain func(context.Context, compiler.CompiledQuery) error } func NewService(cfg Config) *Service { service := &Service{ + compileTarget: cfg.CompileTarget, scopeResolver: cfg.ScopeResolver, activeManifestResolver: cfg.ActiveManifestResolver, explain: cfg.Explain, @@ -64,8 +62,6 @@ func catalogUnavailable() error { return dataframeerrors.NewError(dataframeerrors.CodeBackendUnavailable, "", dataframeerrors.WithRetryable(true)) } -// clockNow is the run-phase diagnostics clock. Production uses the wall clock; -// tests substitute a controlled clock to assert phase boundaries. func (s *Service) clockNow() time.Time { if s.now != nil { return s.now() @@ -99,13 +95,13 @@ func (s *Service) resolveRecipe(ctx context.Context, bundle recipe.Bundle, bindi return semantic.ResolveRecipePlan(plan, "", bindings.DatasetGeneration) } -func compileSingleQuery(resolved semantic.ResolvedRecipePlan, limit int) (dataframeexecution.CompiledQuery, error) { - queries, err := compiler.CompileResolvedRecipePlanWithPolicy(resolved, limit, ir.DefaultPhysicalOptimizationPolicy()) +func compileSingleQuery(target compiler.Target, resolved semantic.ResolvedRecipePlan, limit int) (compiler.CompiledQuery, error) { + queries, err := compiler.CompileResolvedRecipePlan(target, resolved, limit, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { - return dataframeexecution.CompiledQuery{}, err + return compiler.CompiledQuery{}, err } if len(queries) != 1 { - return dataframeexecution.CompiledQuery{}, dataframeerrors.NewError(dataframeerrors.CodeInvalidRequest, "") + return compiler.CompiledQuery{}, dataframeerrors.NewError(dataframeerrors.CodeInvalidRequest, "") } return queries[0], nil } @@ -144,13 +140,10 @@ func (s *Service) Run(ctx context.Context, input model.FhirDataframeInput, limit } requestPreparationDuration := s.clockNow().Sub(preparationStarted) compileStarted := s.clockNow() - compiled, err := compileSingleQuery(resolved, rowLimit) + compiled, err := compileSingleQuery(s.compileTarget, resolved, rowLimit) if err != nil { return nil, queryInvalidErrorOrBackend(err) } - // The compilation interval closes at the compiler boundary: the backend - // query and row materialization below are already reported as their own - // diagnostics and must not be double-charged here. compilationDuration := s.clockNow().Sub(compileStarted) result, err := s.dataframes.RunCompiled(ctx, compiled) if err != nil { diff --git a/internal/api/graphql/graph/query/service_test.go b/internal/api/graphql/graph/query/service_test.go index e3bf3793a..35a3f4436 100644 --- a/internal/api/graphql/graph/query/service_test.go +++ b/internal/api/graphql/graph/query/service_test.go @@ -3,6 +3,7 @@ package queryapi import ( "context" "errors" + "github.com/calypr/loom/internal/dataframe/compiler" "reflect" "sync" "testing" @@ -16,20 +17,10 @@ import ( ) const ( - // queryapiBackendDelay is charged through the controlled Service clock, so - // it is huge without slowing the test. queryapiBackendDelay = 90 * time.Second - // queryapiCursorDelay is charged by the synctest bubble clock: in a bubble, - // time only advances while every goroutine is durably blocked, so the held - // cursor is an exact virtual quantum billed to the query span, independent - // of machine speed. - queryapiCursorDelay = 9 * time.Second + queryapiCursorDelay = 9 * time.Second ) -// controlledClock is the deterministic clock behind Service.now. Each read -// moves the clock by one fixed step, so every Run-owned phase reports a fixed -// non-zero duration without racing wall-clock speed; tests advance it -// explicitly to charge a known delay to exactly one region. type controlledClock struct { mu sync.Mutex time time.Time @@ -53,12 +44,6 @@ func (c *controlledClock) advance(d time.Duration) { c.time = c.time.Add(d) } -// queryapiBackend is the single execution seam for these tests. It records the -// compiled query it received and can hold the cursor open for a declared -// quantum, so the query-region duration is a declared constant rather than a -// measured race. fieldCalls proves a request reached request preparation even -// when it later failed, so a backend call count of zero is a boundary claim, -// not a tautology. type queryapiBackend struct { mu sync.Mutex calls int @@ -72,10 +57,12 @@ type queryapiBackend struct { enteredOnce sync.Once } -// queryRows blocks for cursorDelay before producing any row, which is the -// execution service's ArangoQuery span. Inside a synctest bubble that block is -// an exact virtual quantum. -func (b *queryapiBackend) queryRows(_ context.Context, query string, _ int, bindVars map[string]any, visit func(map[string]any) error) error { +func (b *queryapiBackend) queryRows(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + query, bindVars, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + b.mu.Lock() b.calls++ b.queries = append(b.queries, query) @@ -102,9 +89,6 @@ func (b *queryapiBackend) queryRows(_ context.Context, query string, _ int, bind return nil } -// begin opens the entered signal. Call it inside a synctest bubble, before the -// Run under test, so waiting on the cursor is a durably blocking bubble -// operation rather than a wait on an outside channel. func (b *queryapiBackend) begin() { b.mu.Lock() defer b.mu.Unlock() @@ -112,8 +96,6 @@ func (b *queryapiBackend) begin() { b.enteredOnce = sync.Once{} } -// waitEntered blocks until the cursor has been entered, or fails the test if -// Run finished without ever reaching the backend. func (b *queryapiBackend) waitEntered(t *testing.T, done <-chan runOutcome) { t.Helper() b.mu.Lock() @@ -159,7 +141,7 @@ func queryapiGenderInput() model.FhirDataframeInput { func queryapiService(t *testing.T, backend *queryapiBackend, clock *controlledClock) *Service { t.Helper() - service := NewService(Config{ + service := NewService(Config{CompileTarget: compiler.ArangoTarget(), ActiveManifestResolver: &builderActiveManifestResolver{manifest: builderReadyManifest(t, "P1", "generation-1")}, DiscoverFields: func(_ context.Context, options catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) { backend.mu.Lock() @@ -172,7 +154,7 @@ func queryapiService(t *testing.T, backend *queryapiBackend, clock *controlledCl return []catalog.PopulatedField{}, nil } }, - Dataframes: dataframeexecution.NewService(dataframeexecution.ServiceConfig{QueryRows: backend.queryRows}), + Dataframes: dataframeexecution.NewService(dataframeexecution.ServiceConfig{SourceRows: backend.queryRows}), }) if clock != nil { service.now = clock.now @@ -180,14 +162,11 @@ func queryapiService(t *testing.T, backend *queryapiBackend, clock *controlledCl return service } -// runOutcome carries a Run result across the goroutine boundary. type runOutcome struct { result *dataframeexecution.Result err error } -// runAsync starts Run on its own goroutine so the test goroutine can advance -// the controlled clock while the cursor is held open. func runAsync(service *Service, input model.FhirDataframeInput) <-chan runOutcome { done := make(chan runOutcome, 1) go func() { @@ -197,11 +176,6 @@ func runAsync(service *Service, input model.FhirDataframeInput) <-chan runOutcom return done } -// TestRunDoesNotChargeQueryExecutionToCompilation is the boundary gate. A -// cursor held open for a declared quantum must be charged to the query and -// total durations and never to Compilation. The cursor block runs inside a -// synctest bubble, so its whole duration is an exact virtual quantum: the -// assertions compare declared durations, never a wall-clock speed threshold. func TestRunDoesNotChargeQueryExecutionToCompilation(t *testing.T) { baselineBackend := &queryapiBackend{rows: queryapiGenderRows()} baseline, err := queryapiService(t, baselineBackend, newControlledClock()).Run(context.Background(), queryapiGenderInput(), nil) @@ -221,8 +195,6 @@ func TestRunDoesNotChargeQueryExecutionToCompilation(t *testing.T) { heldBackend.begin() done := runAsync(heldService, queryapiGenderInput()) heldBackend.waitEntered(t, done) - // The compiler returned before the cursor was entered, so this advance - // charges the query region and the total, never the compile interval. heldClock.advance(queryapiBackendDelay) outcome := <-done @@ -236,8 +208,6 @@ func TestRunDoesNotChargeQueryExecutionToCompilation(t *testing.T) { heldDiagnostics = outcome.result.Diagnostics baselineDiagnostics := baseline.Diagnostics - // The compilation interval must be invariant to how long the cursor is - // held open, and non-zero so invariance cannot be met by dropping the phase. if baselineDiagnostics.Compilation <= 0 { t.Fatalf("baseline compilation = %v, want a non-zero compile interval", baselineDiagnostics.Compilation) } @@ -248,22 +218,16 @@ func TestRunDoesNotChargeQueryExecutionToCompilation(t *testing.T) { t.Fatalf("compilation %v still covers the %v backend hold", heldDiagnostics.Compilation, queryapiBackendDelay) } - // The same hold must be visible to the separately measured query - // diagnostics: the bubble charged the whole cursor quantum there, and - // none of it moved into the per-row materialization span. The two spans - // must account for the quantum exactly, so a phase cannot both absorb - // and hide it. - if heldDiagnostics.ArangoQuery != queryapiCursorDelay { - t.Fatalf("held arango query = %v, want exactly the %v held cursor quantum", heldDiagnostics.ArangoQuery, queryapiCursorDelay) + if heldDiagnostics.SourceQuery != queryapiCursorDelay { + t.Fatalf("held arango query = %v, want exactly the %v held cursor quantum", heldDiagnostics.SourceQuery, queryapiCursorDelay) } - if charged := heldDiagnostics.ArangoQuery + heldDiagnostics.RowMaterialization; charged != queryapiCursorDelay { - t.Fatalf("cursor quantum %v charged %v across query %v and materialization %v", queryapiCursorDelay, charged, heldDiagnostics.ArangoQuery, heldDiagnostics.RowMaterialization) + if charged := heldDiagnostics.SourceQuery + heldDiagnostics.RowMaterialization; charged != queryapiCursorDelay { + t.Fatalf("cursor quantum %v charged %v across query %v and materialization %v", queryapiCursorDelay, charged, heldDiagnostics.SourceQuery, heldDiagnostics.RowMaterialization) } - if heldDiagnostics.ArangoQuery >= heldDiagnostics.Total { - t.Fatalf("held arango query %v is not bounded by total %v", heldDiagnostics.ArangoQuery, heldDiagnostics.Total) + if heldDiagnostics.SourceQuery >= heldDiagnostics.Total { + t.Fatalf("held arango query %v is not bounded by total %v", heldDiagnostics.SourceQuery, heldDiagnostics.Total) } - // The same delay must land in the total. if want := baselineDiagnostics.Total + queryapiBackendDelay; heldDiagnostics.Total != want { t.Fatalf("held total = %v, want %v (baseline %v + cursor delay %v)", heldDiagnostics.Total, want, baselineDiagnostics.Total, queryapiBackendDelay) } @@ -276,7 +240,7 @@ func TestRunDoesNotChargeQueryExecutionToCompilation(t *testing.T) { if owned := diagnostics.InputResolution + diagnostics.RequestPreparation + diagnostics.Compilation; diagnostics.Total < owned { t.Fatalf("total %v is below the sum %v of the Run-owned phases", diagnostics.Total, owned) } - if diagnostics.ArangoQuery < 0 || diagnostics.RowMaterialization < 0 || diagnostics.ResultAssembly < 0 { + if diagnostics.SourceQuery < 0 || diagnostics.RowMaterialization < 0 || diagnostics.ResultAssembly < 0 { t.Fatalf("negative backend diagnostics: %+v", diagnostics) } } @@ -288,10 +252,6 @@ func TestRunDoesNotChargeQueryExecutionToCompilation(t *testing.T) { } } -// TestRunChargesCursorQuantumOnceAcrossRows pins the other half of the query -// diagnostic: the held cursor is one span billed once to ArangoQuery, however -// many rows it delivers, and it never leaks into the per-row materialization -// span. The quantum is an exact bubble-time constant. func TestRunChargesCursorQuantumOnceAcrossRows(t *testing.T) { backend := &queryapiBackend{ rows: []map[string]any{ @@ -318,8 +278,8 @@ func TestRunChargesCursorQuantumOnceAcrossRows(t *testing.T) { t.Fatalf("row count = %d, want the 3 delivered rows", outcome.result.RowCount) } diagnostics := outcome.result.Diagnostics - if diagnostics.ArangoQuery != queryapiCursorDelay { - t.Fatalf("arango query = %v, want the single %v cursor quantum, not one per row", diagnostics.ArangoQuery, queryapiCursorDelay) + if diagnostics.SourceQuery != queryapiCursorDelay { + t.Fatalf("arango query = %v, want the single %v cursor quantum, not one per row", diagnostics.SourceQuery, queryapiCursorDelay) } if diagnostics.RowMaterialization >= queryapiCursorDelay { t.Fatalf("row materialization = %v, want the cursor quantum charged only to the query span", diagnostics.RowMaterialization) @@ -327,9 +287,6 @@ func TestRunChargesCursorQuantumOnceAcrossRows(t *testing.T) { }) } -// TestRunPreservesResultAndSeparateDiagnostics confirms the boundary fix keeps -// the public row shape and the separately reported query, materialization, and -// plan diagnostics intact. func TestRunPreservesResultAndSeparateDiagnostics(t *testing.T) { backend := &queryapiBackend{rows: queryapiGenderRows()} result, err := queryapiService(t, backend, newControlledClock()).Run(context.Background(), queryapiGenderInput(), nil) @@ -352,7 +309,7 @@ func TestRunPreservesResultAndSeparateDiagnostics(t *testing.T) { t.Fatalf("backend generation bind = %#v, want generation-1", got) } diagnostics := result.Diagnostics - if diagnostics.ArangoQuery < 0 || diagnostics.RowMaterialization < 0 || diagnostics.ResultAssembly < 0 { + if diagnostics.SourceQuery < 0 || diagnostics.RowMaterialization < 0 || diagnostics.ResultAssembly < 0 { t.Fatalf("negative backend diagnostics: %+v", diagnostics) } if diagnostics.Plan.Fingerprint == "" { @@ -363,9 +320,6 @@ func TestRunPreservesResultAndSeparateDiagnostics(t *testing.T) { } } -// TestRunFailedCompileNeverInvokesBackend pins the compile boundary: a request -// that reaches the compiler and fails there must not reach the backend, and the -// error keeps its existing classification. func TestRunFailedCompileNeverInvokesBackend(t *testing.T) { for _, test := range []struct { name string @@ -373,9 +327,6 @@ func TestRunFailedCompileNeverInvokesBackend(t *testing.T) { wantCode dataframeerrors.ErrorCode }{ { - // The compiler, not the recipe validator, rejects this projection - // name; the failure is a compiler-internal fault under the current - // classification contract. name: "compiler rejects reserved projection name", input: model.FhirDataframeInput{ Project: "P1", RootResourceType: "Patient", @@ -384,7 +335,6 @@ func TestRunFailedCompileNeverInvokesBackend(t *testing.T) { wantCode: dataframeerrors.CodeBackendUnavailable, }, { - // The request-preparation boundary rejects this before compilation. name: "preparation rejects cross-grain row grain", input: func() model.FhirDataframeInput { grain := "specimen" @@ -404,8 +354,6 @@ func TestRunFailedCompileNeverInvokesBackend(t *testing.T) { if result != nil { t.Fatalf("Run() result = %#v, want nil on failure", result) } - // The request really reached catalog-backed preparation, so the zero - // backend count below is a boundary claim rather than an early exit. if backend.catalogCallCount() == 0 { t.Fatal("catalog discovery was never called, want a request that reached preparation") } @@ -423,8 +371,6 @@ func TestRunFailedCompileNeverInvokesBackend(t *testing.T) { } } -// TestRunBackendErrorKeepsClassification confirms the compiler-boundary change -// did not absorb backend failures into the compile path. func TestRunBackendErrorKeepsClassification(t *testing.T) { t.Run("unknown cause becomes retryable backend unavailable", func(t *testing.T) { backend := &queryapiBackend{err: errors.New("cursor exploded")} @@ -454,9 +400,6 @@ func TestRunBackendErrorKeepsClassification(t *testing.T) { }) } -// TestQueryDiagnosticsFieldShape pins the diagnostic field set the public -// response is rendered from, so closing the compile interval cannot smuggle in -// a reporting expansion. func TestQueryDiagnosticsFieldShape(t *testing.T) { structure := reflect.TypeOf(dataframeexecution.QueryDiagnostics{}) got := make([]string, 0, structure.NumField()) @@ -465,7 +408,7 @@ func TestQueryDiagnosticsFieldShape(t *testing.T) { } want := []string{ "InputResolution", "RequestPreparation", "Compilation", - "ArangoQuery", "RowMaterialization", "ResultAssembly", "Total", "Plan", + "SourceQuery", "SourceBackend", "RowMaterialization", "ResultAssembly", "Total", "Plan", } if !reflect.DeepEqual(got, want) { t.Fatalf("QueryDiagnostics fields = %#v, want %#v", got, want) diff --git a/internal/api/graphql/graph/resolver/dataframe_diagnostics.go b/internal/api/graphql/graph/resolver/dataframe_diagnostics.go index 2c90e78cc..1fc034109 100644 --- a/internal/api/graphql/graph/resolver/dataframe_diagnostics.go +++ b/internal/api/graphql/graph/resolver/dataframe_diagnostics.go @@ -4,21 +4,26 @@ import ( "time" "github.com/calypr/loom/generated/graphql/graph/model" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" ) func dataframeDiagnostics(in dataframeexecution.QueryDiagnostics) *model.DataframeQueryDiagnostics { - return &model.DataframeQueryDiagnostics{ + out := &model.DataframeQueryDiagnostics{ InputResolutionMs: milliseconds(in.InputResolution), RequestPreparationMs: milliseconds(in.RequestPreparation), CompilationMs: milliseconds(in.Compilation), - ArangoQueryMs: milliseconds(in.ArangoQuery), + SourceQueryMs: milliseconds(in.SourceQuery), RowMaterializationMs: milliseconds(in.RowMaterialization), ResultAssemblyMs: milliseconds(in.ResultAssembly), TotalMs: milliseconds(in.Total), Plan: compilerPlanDiagnostics(in.Plan), } + if in.SourceBackend == command.CompiledBackendAQL { + out.ArangoQueryMs = milliseconds(in.SourceQuery) + } + return out } func milliseconds(value time.Duration) float64 { return float64(value) / float64(time.Millisecond) } diff --git a/internal/api/graphql/graph/resolver/dataframe_diagnostics_test.go b/internal/api/graphql/graph/resolver/dataframe_diagnostics_test.go new file mode 100644 index 000000000..85c8c2830 --- /dev/null +++ b/internal/api/graphql/graph/resolver/dataframe_diagnostics_test.go @@ -0,0 +1,38 @@ +package resolver + +import ( + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" +) + +func TestDataframeDiagnosticsProjectsSourceQueryMsOnBothBackends(t *testing.T) { + const sourceQuery = 1234567 * time.Microsecond + for _, lane := range []struct { + name string + tag command.CompiledBackend + wantArango float64 + }{ + {name: "AQL", tag: command.CompiledBackendAQL, wantArango: 1234.567}, + {name: "PostgreSQL", tag: command.CompiledBackendPostgreSQL, wantArango: 0}, + {name: "untagged fail-closed to zero", tag: "", wantArango: 0}, + } { + t.Run(lane.name, func(t *testing.T) { + out := dataframeDiagnostics(dataframeexecution.QueryDiagnostics{ + SourceQuery: sourceQuery, + SourceBackend: lane.tag, + }) + if out.SourceQueryMs != 1234.567 { + t.Fatalf("sourceQueryMs = %v, want the source interval in ms", out.SourceQueryMs) + } + if out.ArangoQueryMs != lane.wantArango { + t.Fatalf("%s: arangoQueryMs = %v, want %v", lane.name, out.ArangoQueryMs, lane.wantArango) + } + if lane.tag == command.CompiledBackendAQL && out.ArangoQueryMs != out.SourceQueryMs { + t.Fatal("the legacy arangoQueryMs projection must equal sourceQueryMs byte-for-byte on the AQL lane") + } + }) + } +} diff --git a/internal/api/graphql/graph/resolver/fhir_reference_loader.go b/internal/api/graphql/graph/resolver/fhir_reference_loader.go index 8e58e4677..e8fa02f23 100644 --- a/internal/api/graphql/graph/resolver/fhir_reference_loader.go +++ b/internal/api/graphql/graph/resolver/fhir_reference_loader.go @@ -30,8 +30,6 @@ type fhirReferenceOwner struct { depth int } -// fhirReferenceLookup includes the versioned-reference version so two versions -// of one logical resource can never share a cached or pending result. type fhirReferenceLookup struct { scope fhirReferenceScopeKey target, id, version string @@ -42,8 +40,6 @@ type fhirReferenceResult struct { } type fhirReferenceWaiter struct{ ch chan fhirReferenceResult } -// fhirReferenceLoader batches only for the lifetime of one GraphQL operation. -// Keeping it request-local prevents cross-user cache and authorization leaks. type fhirReferenceLoader struct { resolver *Resolver @@ -76,7 +72,6 @@ func fhirReferenceLoaderFromContext(ctx context.Context, resolver *Resolver) *fh if loader, ok := ctx.Value(fhirReferenceLoaderKey{}).(*fhirReferenceLoader); ok { return loader } - // Direct resolver tests do not pass through HTTP middleware. return &fhirReferenceLoader{ resolver: resolver, owners: make(map[*fhir.Reference]fhirReferenceOwner), @@ -160,7 +155,6 @@ func (l *fhirReferenceLoader) load(ctx context.Context, lookup fhirReferenceLook l.pending[lookup] = append(l.pending[lookup], waiter) if !l.timerSet { l.timerSet = true - // ponytail: fixed 1ms coalescing window; tune only if profiling shows missed sibling batches. time.AfterFunc(time.Millisecond, func() { l.dispatch(ctx) }) } l.mu.Unlock() @@ -288,17 +282,11 @@ type parsedFHIRReference struct { contained bool } -// selectFHIRReferenceRow picks the row satisfying one reference out of the rows -// returned for its logical id. Unversioned references keep the existing -// selected-generation row; versioned references must match meta.versionId -// exactly, so a versioned reference never resolves to a different version. func selectFHIRReferenceRow(rows []map[string]any, version string) (map[string]any, bool) { if len(rows) == 0 { return nil, false } if version == "" { - // Preserve the pre-version semantics: the last row the backend returned - // for the logical id wins. return rows[len(rows)-1], true } for _, row := range rows { @@ -318,9 +306,6 @@ func fhirResourceVersionID(resource map[string]any) string { return version } -// validFHIRReferenceVersion mirrors the generated Meta.versionId pattern -// '^[A-Za-z0-9\-.]{1,64}$'; a version that cannot be a versionId can never -// match a stored row and is rejected as malformed syntax. func validFHIRReferenceVersion(version string) bool { if version == "" || len(version) > 64 { return false diff --git a/internal/api/graphql/graph/resolver/fhir_reference_loader_test.go b/internal/api/graphql/graph/resolver/fhir_reference_loader_test.go index 0ecd0b5d9..b73d1e3bd 100644 --- a/internal/api/graphql/graph/resolver/fhir_reference_loader_test.go +++ b/internal/api/graphql/graph/resolver/fhir_reference_loader_test.go @@ -2,6 +2,7 @@ package resolver import ( "context" + "github.com/calypr/loom/internal/dataframe/compiler" "strings" "sync" "testing" @@ -226,10 +227,6 @@ func TestResolveFHIRReferenceMissingTargetIsNotResolved(t *testing.T) { } } -// TestFHIRReferenceBatchKeepsVersionedAndUnversionedLookupsApart drives the -// batch path with one versioned and one unversioned lookup for the same -// logical id: both must be answered from a single query and neither may -// receive the other resource version. func TestFHIRReferenceBatchKeepsVersionedAndUnversionedLookupsApart(t *testing.T) { harness := newFHIRReferenceTestHarness(t, []fhirReferenceTestRow{ {id: "123", version: "4", gender: "female"}, @@ -269,9 +266,6 @@ func TestFHIRReferenceBatchKeepsVersionedAndUnversionedLookupsApart(t *testing.T } } -// TestResolveFHIRReferenceSiblingVersionsResolveIndependently mirrors sibling -// field resolvers in one operation asking for two versions of one logical -// resource plus an absent version; each must get its own outcome. func TestResolveFHIRReferenceSiblingVersionsResolveIndependently(t *testing.T) { harness := newFHIRReferenceTestHarness(t, []fhirReferenceTestRow{ {id: "123", version: "4", gender: "female"}, @@ -337,9 +331,12 @@ type fhirReferenceTestProbe struct { rows []fhirReferenceTestRow } -// queryRows emulates the compiled id-IN reference query against one pinned -// dataset generation, including the authorization filter and row limit. -func (p *fhirReferenceTestProbe) queryRows(_ context.Context, _ string, _ int, bindVars map[string]any, visit func(map[string]any) error) error { +func (p *fhirReferenceTestProbe) queryRows(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, bindVars, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + generation, _ := bindVars["dataset_generation"].(string) ids := fhirReferenceTestBindStrings(bindVars["root_filter_1_value"]) unrestricted, _ := bindVars["auth_resource_paths_unrestricted"].(bool) @@ -423,13 +420,10 @@ func newFHIRReferenceTestHarness(t *testing.T, rows []fhirReferenceTestRow) *fhi return newFHIRReferenceTestHarnessWithScope(t, rows, nil) } -// newFHIRReferenceTestHarnessWithScope serves rows from one fake storage -// generation and registers roots under the given authorized read scope, so -// reference lookups exercise the real generation and scope pinning. func newFHIRReferenceTestHarnessWithScope(t *testing.T, rows []fhirReferenceTestRow, scope *authscope.ReadScope) *fhirReferenceTestHarness { t.Helper() probe := &fhirReferenceTestProbe{rows: rows} - service := queryapi.NewService(queryapi.Config{ + service := queryapi.NewService(queryapi.Config{CompileTarget: compiler.ArangoTarget(), ScopeResolver: authscope.NewScopeResolver(authscope.ScopeResolverConfig{ ResourceAccess: fhirReferenceTestResourceAccess{}, ListExistingAuthResourcePaths: func(context.Context, catalog.AuthResourcePathOptions) ([]string, error) { @@ -442,7 +436,7 @@ func newFHIRReferenceTestHarnessWithScope(t *testing.T, rows []fhirReferenceTest }}, nil }, Dataframes: dataframeexecution.NewService(dataframeexecution.ServiceConfig{ - QueryRows: probe.queryRows, + SourceRows: probe.queryRows, }), }) return &fhirReferenceTestHarness{ @@ -456,8 +450,6 @@ func (h *fhirReferenceTestHarness) context() context.Context { return withFHIRReferenceLoader(context.Background(), h.resolver) } -// subject registers a root document carrying one subject reference under a -// fresh operation-local loader, the way the typed root resolver does. func (h *fhirReferenceTestHarness) subject(ref string) (context.Context, *fhir.Reference) { reference := ref root := &fhir.DocumentReference{Subject: &fhir.Reference{Reference: &reference}} diff --git a/internal/api/graphql/graph/resolver/fhir_resolver_helpers.go b/internal/api/graphql/graph/resolver/fhir_resolver_helpers.go index aa92a6623..33d4df8d1 100644 --- a/internal/api/graphql/graph/resolver/fhir_resolver_helpers.go +++ b/internal/api/graphql/graph/resolver/fhir_resolver_helpers.go @@ -12,10 +12,6 @@ import ( dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" ) -// resolveFHIRField is used by gqlgen's generated autobind adapters for the -// generated FHIR structs. It honors both JSON and gqlgen tags (including -// primitive-extension names such as _birthDate) and keeps missing historical -// fields nullable instead of panicking. func resolveFHIRField[T any](obj any, name string) (T, error) { var zero T v := reflect.ValueOf(obj) @@ -51,9 +47,6 @@ func resolveFHIRField[T any](obj any, name string) (T, error) { return zero, nil } -// listFHIR is shared by the generated exact-capitalization root adapters. It -// keeps generated methods as thin type-conversion wrappers while all query -// planning and execution remains in queryapi.Service. func (r *queryResolver) listFHIR(ctx context.Context, project string, filters []*model.FhirFilterInput, requestedLimit *int, out any, resourceType string) error { limit := 25 if requestedLimit != nil { diff --git a/internal/api/graphql/graph/resolver/operation_context.go b/internal/api/graphql/graph/resolver/operation_context.go index 250cebabc..2a10145c6 100644 --- a/internal/api/graphql/graph/resolver/operation_context.go +++ b/internal/api/graphql/graph/resolver/operation_context.go @@ -7,8 +7,6 @@ import ( "github.com/vektah/gqlparser/v2/ast" ) -// WithOperationContext installs all request-scoped loaders and aggregate -// scheduling state at gqlgen's operation boundary. func (r *Resolver) WithOperationContext(ctx context.Context) context.Context { ctx = withFHIRReferenceLoader(ctx, r) if r.dataframes != nil { diff --git a/internal/api/graphql/graph/resolver/recipe_control_graphql.go b/internal/api/graphql/graph/resolver/recipe_control_graphql.go index 1ea65226f..bbe41fe0b 100644 --- a/internal/api/graphql/graph/resolver/recipe_control_graphql.go +++ b/internal/api/graphql/graph/resolver/recipe_control_graphql.go @@ -254,9 +254,6 @@ func fullRecipeResult(result recipePreviewView, name string) (*model.DataframeRe }, nil } -// dataframeCSV serializes one logical output using the compiler-resolved -// column order. Repeated and object values are represented as JSON in one CSV -// cell so the flat export remains lossless. func dataframeCSV(columns []string, rows []map[string]any) (string, error) { var buffer bytes.Buffer writer := csv.NewWriter(&buffer) diff --git a/internal/api/graphql/graph/resolver/recipe_execution_malformed_persisted_scope_test.go b/internal/api/graphql/graph/resolver/recipe_execution_malformed_persisted_scope_test.go index 5d56ac205..4f3444456 100644 --- a/internal/api/graphql/graph/resolver/recipe_execution_malformed_persisted_scope_test.go +++ b/internal/api/graphql/graph/resolver/recipe_execution_malformed_persisted_scope_test.go @@ -12,28 +12,6 @@ import ( materialization "github.com/calypr/loom/internal/dataframe/publication" ) -// Durable R4 control for the production GraphQL execution reader. -// -// Reproduction. A persisted execution declares auth_resource_paths -// ["p1","foreign\x00secret"]. A project member granted only p1 was served the -// execution metadata, because normalizeAuthResourcePathList drops the malformed -// non-blank entry and the surviving path covered the truncated declaration. -// ReadScope.CoversExecutionPaths now refuses a declaration holding any -// malformed non-blank entry, so the reader's existing restricted-mode gate -// conceals it. These cases run the real NewAuthorizedRecipeExecutionReader and -// the real authscope resolver, driving only its declared ScopeResolverConfig -// seam, against durable metadata fixtures. -// -// Contract pinned here: -// - malformed persisted scope -> the concealed RECIPE_EXECUTION_NOT_FOUND -// user error, byte-identical to a missing id, with zero metadata disclosure -// and zero leak of the malformed bytes; -// - permitted normal execution -> still fully readable (no over-denial); -// - legitimate blank declarations -> unchanged pre-existing policy; -// - backend faults -> unchanged classification (retryable -// BACKEND_UNAVAILABLE for store/catalog faults, not-found for the durable -// sentinel) instead of collapsing into the concealed 404. - const ( mscHomeProject = "P" mscGeneration = "g-msc" @@ -48,9 +26,6 @@ const ( mscOutputRowCount = int64(424242) ) -// mscStore is the durable-metadata fixture for the reader's own -// RecipeExecutionStore seam. Unknown ids yield the shipped not-found sentinel, -// so a concealed denial can be compared against a genuinely missing execution. type mscStore struct { executions map[string]materialization.BundleExecution fault error @@ -92,8 +67,6 @@ func mscStoreWith(id string, declared []string) *mscStore { return &mscStore{executions: map[string]materialization.BundleExecution{id: mscBundle(id, declared)}} } -// mscProbe drives the production resolver's declared catalog seam so a -// restricted scope resolves without a live Fence or database. type mscProbe struct { existing []string calls []catalog.AuthResourcePathOptions @@ -115,8 +88,6 @@ func mscReader(store RecipeExecutionStore, probe *mscProbe) RecipeExecutionReade return NewAuthorizedRecipeExecutionReader(store, resolver) } -// mscMember returns a genuine project member (the shipped authorizer admits it) -// whose only grant is the clean path. func mscMember(grants ...string) *authscope.Principal { return &authscope.Principal{Subject: "operator-msc", Projects: []string{mscHomeProject}, AuthResourcePaths: grants} } @@ -163,8 +134,6 @@ func mscVerdict(t *testing.T, id, verdict, observed string) { t.Logf("R4-GRAPHQL %s | VERDICT=%s | OBSERVED=%s", id, verdict, observed) } -// TestGraphQLRecipeExecutionConcealsMalformedPersistedScope is the required -// regression for acceptance index 1. func TestGraphQLRecipeExecutionConcealsMalformedPersistedScope(t *testing.T) { t.Run("mixed-valid-and-nul-declaration-is-concealed-for-a-member-granted-only-p1", func(t *testing.T) { store := mscStoreWith(mscExecID, []string{mscGrantedPath, mscMalformedNUL}) @@ -172,8 +141,6 @@ func TestGraphQLRecipeExecutionConcealsMalformedPersistedScope(t *testing.T) { reader := mscReader(store, probe) principal := mscMember(mscGrantedPath) - // Precondition: the caller really is a member the shipped authorizer admits, - // so the refusal below cannot be a membership denial. if err := authscope.AuthorizeProject(principal, mscHomeProject, false); err != nil { t.Fatalf("precondition: shipped AuthorizeProject must admit this member, got %v", err) } @@ -186,7 +153,6 @@ func TestGraphQLRecipeExecutionConcealsMalformedPersistedScope(t *testing.T) { if leaked := mscLeakedMetadata(err, got); leaked != "" { t.Fatalf("concealed refusal disclosed metadata (%s): code=%q message=%q", leaked, userErr.Code(), userErr.Error()) } - // Indistinguishable from a genuinely missing execution id. missingGot, missingErr := reader(mscContext(principal), mscMissingID) missingUserErr := mscAssertConcealed(t, missingGot, missingErr) if missingUserErr.Code() != userErr.Code() || missingUserErr.Retryable() != userErr.Retryable() { @@ -311,9 +277,6 @@ func TestGraphQLRecipeExecutionConcealsMalformedPersistedScope(t *testing.T) { }) t.Run("malformed-scope-refusal-is-not-a-membership-or-store-artifact", func(t *testing.T) { - // The same store and probe serve the permitted read; only the persisted - // declaration changes. A disclosure-only difference between the two runs - // proves the malformed entry is what denies the read. clean := mscReader(mscStoreWith(mscExecID, []string{mscGrantedPath}), &mscProbe{existing: []string{mscGrantedPath}}) got, err := clean(mscContext(mscMember(mscGrantedPath)), mscExecID) mscAssertReadable(t, got, err, "baseline-clean-declaration") @@ -338,8 +301,6 @@ func TestGraphQLRecipeExecutionConcealsMalformedPersistedScope(t *testing.T) { }) } -// mscAssertFaultClassification pins that a dependency fault keeps its existing -// semantic classification instead of collapsing into the concealed not-found. func mscAssertFaultClassification(t *testing.T, got *RecipeExecution, err error, wantCode dataframeerrors.ErrorCode, wantRetryable bool, cause string) { t.Helper() if got != nil { @@ -358,9 +319,6 @@ func mscAssertFaultClassification(t *testing.T, got *RecipeExecution, err error, } } -// mscLeakedMetadata returns a non-empty description when a response or error -// carries execution metadata (or the malformed declaration bytes) that a -// concealed refusal must never expose. func mscLeakedMetadata(err error, got *RecipeExecution) string { if got != nil { return "payload present" diff --git a/internal/api/graphql/graph/resolver/recipe_execution_malformed_scope_oracle_probe_test.go b/internal/api/graphql/graph/resolver/recipe_execution_malformed_scope_oracle_probe_test.go index 71da585ae..a8b2d6c4c 100644 --- a/internal/api/graphql/graph/resolver/recipe_execution_malformed_scope_oracle_probe_test.go +++ b/internal/api/graphql/graph/resolver/recipe_execution_malformed_scope_oracle_probe_test.go @@ -10,27 +10,6 @@ import ( materialization "github.com/calypr/loom/internal/dataframe/publication" ) -// Adversarial probes on top of TestGraphQLRecipeExecutionConcealsMalformedPersistedScope. -// -// Probe A (denial-class indistinguishability). The repair routes the R4 -// malformed-persisted-scope case into the reader's existing concealed -// RECIPE_EXECUTION_NOT_FOUND arm. The attack: can a caller use the reader as -// an oracle to tell "this execution's persisted declaration is malformed" from -// "this execution does not exist", "I am not a member", "I lack one declared -// path", or "this execution declares no non-blank path"? Every one of those -// denials must carry one and the same envelope (payload nil, identical code, -// retryability and message), while the backend-fault arms must stay outside -// that envelope — otherwise the concealed class is either leaky or a -// catch-all that hides faults. -// -// Probe B (shared-resolver-cache oracle). The production ScopeResolver caches -// the project+generation path listing. A malformed-scope execution and a clean -// execution can share that key, so the attack is to interleave reads and try -// to poison coverage in either direction: a prior clean read must not make the -// malformed execution readable, and a prior malformed denial must not make the -// clean execution deniable. Verdicts and envelopes must be stable across every -// interleaving on one shared resolver. - type msoEnvelope struct { code string retryable bool @@ -70,12 +49,10 @@ func TestGraphQLRecipeExecutionMalformedScopeDenialIsIndistinguishableFromEveryD return reader(mscContext(member), mscMissingID) }, "non-member": func() (*RecipeExecution, error) { - // Same execution, same grants, but not a member of project P. outsider := &authscope.Principal{Subject: "operator-outsider", Projects: []string{"P-ELSEWHERE"}, AuthResourcePaths: []string{mscGrantedPath}} return reader(mscContext(outsider), mscExecID) }, "restricted-member-missing-declared-path": func() (*RecipeExecution, error) { - // Declares [p1, p2]; this member holds only p1. return reader(mscContext(member), mscMultiPathExecID) }, "blank-declaration": func() (*RecipeExecution, error) { @@ -100,7 +77,6 @@ func TestGraphQLRecipeExecutionMalformedScopeDenialIsIndistinguishableFromEveryD } } - // The fault arms must NOT collapse into the concealed envelope. faulty := mscStoreWith(mscExecID, []string{mscGrantedPath, mscMalformedNUL}) faulty.fault = errors.New("arango connection reset") faultReader := mscReader(faulty, &mscProbe{existing: []string{mscGrantedPath}}) @@ -124,8 +100,6 @@ func TestGraphQLRecipeExecutionMalformedScopeSharedResolverCacheIsNotAnOracle(t mscExecID: mscBundle(mscExecID, []string{mscGrantedPath, mscMalformedNUL}), "msc-clean": mscBundle("msc-clean", []string{mscGrantedPath}), }} - // One resolver, one cache, both executions share project P + generation - // g-msc, so the listing entry is written by whichever read lands first. probe := &mscProbe{existing: []string{mscGrantedPath}} reader := mscReader(&store, probe) member := mscMember(mscGrantedPath) @@ -146,8 +120,6 @@ func TestGraphQLRecipeExecutionMalformedScopeSharedResolverCacheIsNotAnOracle(t } } - // Every interleaving order converges to the same verdicts; the shared - // cache never poisons coverage in either direction. cleanReadable("cold clean-first") malformedDenied("after clean read") cleanReadable("after malformed denial") @@ -158,7 +130,6 @@ func TestGraphQLRecipeExecutionMalformedScopeSharedResolverCacheIsNotAnOracle(t malformedDenied("steady-state malformed") } - // Reversing the priming order on a fresh resolver must be identical. freshProbe := &mscProbe{existing: []string{mscGrantedPath}} freshReader := mscReader(&store, freshProbe) malformedDeniedWith := func(reader RecipeExecutionReader, why string) { @@ -174,8 +145,6 @@ func TestGraphQLRecipeExecutionMalformedScopeSharedResolverCacheIsNotAnOracle(t t.Fatalf("a prior malformed denial must not deny the clean sibling, got payload=%#v err=%v", got, err) } - // A member who holds every path the CLEAN sibling declares still cannot - // read the malformed execution through the warmed cache. wide := mscMember(mscGrantedPath, "p2") if _, err := freshReader(mscContext(wide), "msc-clean"); err != nil { t.Fatalf("wide member must read the clean sibling: %v", err) diff --git a/internal/api/graphql/graph/resolver/recipe_execution_reader.go b/internal/api/graphql/graph/resolver/recipe_execution_reader.go index 15ed32d6f..11b3807fa 100644 --- a/internal/api/graphql/graph/resolver/recipe_execution_reader.go +++ b/internal/api/graphql/graph/resolver/recipe_execution_reader.go @@ -11,10 +11,6 @@ import ( materialization "github.com/calypr/loom/internal/dataframe/publication" ) -// RecipeExecutionStore is the durable subset needed by the GraphQL execution -// status query. Keeping this interface narrower than BundleCatalog makes the -// transport adapter usable with both the production Arango registry and small -// test fakes. type RecipeExecutionStore interface { GetExecution(context.Context, string) (materialization.BundleExecution, error) } @@ -68,18 +64,6 @@ func NewAuthorizedRecipeExecutionReader(store RecipeExecutionStore, scopeResolve return nil, dataframeerrors.Wrap(err, dataframeerrors.CodeInternalError, "") } rowCount := int(output.RowCount) - // REPAIR-20c (phantom trio): the output error MIRROR of the single - // execution-level message applies only while the output itself is - // not healthy. The lease-lost reconciliation (bundle_store.go - // :1207-1214) fails the execution while READY outputs stay - // untouched, and the per-output sanitizer would stamp a healthy - // fully-published page with its own INTERNAL_ERROR trio whose - // retryability contradicts the execution truth (top-level state - // FAILED, errorCode PUBLICATION_LEASE_LOST, retryable=true). - // Healthy = the durable success states READY/PUBLISHED. Genuine - // per-output failures (failPhase bundle_store.go:989-995) are - // FAILED or persist a own FailureCode, and still project - // unconditionally through ErrorCode/ErrorRetryable/Phase below. mirroredError := "" if outputState != "READY" && outputState != "PUBLISHED" { mirroredError = execution.Error diff --git a/internal/api/graphql/graph/resolver/recipe_execution_reader_authz_e2e_test.go b/internal/api/graphql/graph/resolver/recipe_execution_reader_authz_e2e_test.go index c553d43af..f0e29ff2b 100644 --- a/internal/api/graphql/graph/resolver/recipe_execution_reader_authz_e2e_test.go +++ b/internal/api/graphql/graph/resolver/recipe_execution_reader_authz_e2e_test.go @@ -15,38 +15,6 @@ import ( materialization "github.com/calypr/loom/internal/dataframe/publication" ) -// This file is the durable F-B20 authorization control for the production -// GraphQL execution reader. -// -// History. The original suite characterized the shipped defect: NewAuthorizedRecipeExecutionReader -// performed no project-membership assertion at all (the entire block was gated -// on a non-nil scope resolver) and its restricted-mode gate compared path -// COUNTS rather than sets, so a caller whose project allowlist excluded the -// execution's project could disclose cross-project execution metadata by -// holding a colliding auth-path string, and any caller without a resolver was -// served unconditionally. The repair makes the project assertion unconditional -// (authscope.AuthorizeProject, the shipped semantics that keep the anonymous -// single-operator deployment allowed by design) and replaces the count -// comparison with the shared scope.CoversExecutionPaths set rule in both the -// GraphQL reader and its REST sibling internal/server/openapi_recipe_execution.go. -// -// Contract verified here: an authenticated principal whose Projects excludes -// the execution's project is refused with the same concealed -// RECIPE_EXECUTION_NOT_FOUND result as a missing id, before the scope resolver -// is consulted; a member principal keeps full read access; the restricted-mode -// gate is a normalized path-SET rule (duplicates or blank variants alone can -// no longer deny a caller authorized on every declared path, and a restricted -// execution that declares no non-blank path can no longer be served); -// a principal-less request keeps the shipped anonymous posture. -// -// Why the scope resolver is not a hand-written fake. The constructor takes the -// concrete *authscope.ScopeResolver, so no fake type can be substituted. Cases -// B and C build the production authscope.NewScopeResolver and control only its -// declared seams, which is what internal/authscope/scope_test.go does too. -// -// REST sibling behaviour coverage lives in package server -// (openapi_recipe_execution_test.go); this file keeps source-level quotes -// proving the twin carries the same unconditional membership call and set gate. const ( fb20ForeignProject = "Q" fb20ForeignGeneration = "gQ" @@ -71,16 +39,6 @@ const ( fb20SourceTwin = "../../../../server/openapi_recipe_execution.go" ) -// Denial origin is proven behaviourally: catalog consultation counts -// (fb20ScopeProbe.calls) distinguish the membership short-circuit from the -// resolver path, so the concealed user-facing result stays bit-identical -// regardless of which gate refused. - -// fb20QuotedFacts are the exact source fragments this control is built on. -// The membership call is pinned outside the resolver-conditional block in -// BOTH twins, the restricted gate is the shared set rule, and the archived -// count comparison must stay absent. Any disappearance is drift and must fail -// the suite rather than let a stale guarantee pass. var fb20QuotedFacts = []struct { name string source string @@ -143,10 +101,6 @@ var fb20QuotedFacts = []struct { }, } -// fb20ArchivedPreFixFacts record the repaired defect. Their absence from both -// twins is required: these are the shipped shapes the F-B20 reproduction -// demonstrated (unconditional cross-tenant disclosure with a nil resolver, and -// count-blind restricted-mode acceptance). var fb20ArchivedPreFixFacts = []struct { name string source string @@ -167,9 +121,6 @@ var fb20ArchivedPreFixFacts = []struct { }, } -// fb20ExecutionStore is the smallest fake for the reader's own RecipeExecutionStore -// interface. It yields one published, ready-output execution that belongs to -// the foreign project. type fb20ExecutionStore struct { execution materialization.BundleExecution err error @@ -184,10 +135,6 @@ func (s *fb20ExecutionStore) GetExecution(_ context.Context, id string) (materia return s.execution, nil } -// fb20ScopeProbe drives the declared seams of authscope.ScopeResolverConfig so -// the production resolver can resolve restricted scopes without a live Fence -// or a catalog. Call history also proves whether the membership gate precedes -// any resolver consultation. type fb20ScopeProbe struct { existingByProject map[string][]string calls []catalog.AuthResourcePathOptions @@ -233,10 +180,6 @@ func fb20ReadScopeResolver(t *testing.T, probe *fb20ScopeProbe) *authscope.Scope }) } -// fb20RequireRepairedGuards is STEP 0. It pins every quoted fragment and -// requires the archived pre-fix shapes to remain absent from the production -// twins. A regression that restores either fails here with an explicit -// message instead of letting the behavioural cases below pass on drift. func fb20RequireRepairedGuards(t *testing.T) { t.Helper() contents := make(map[string]string, len(fb20QuotedFacts)) @@ -282,9 +225,6 @@ func fb20LineOf(source, needle string) int { return strings.Count(source[:index], "\n") + 1 } -// fb20AssertConcealedDenial asserts the refused read is indistinguishable from -// a missing execution: no payload, the concealed user-facing code, and the -// forbidden condition for callers that inspect the chain. func fb20AssertConcealedDenial(t *testing.T, got *RecipeExecution, err error) string { t.Helper() if got != nil { @@ -304,10 +244,6 @@ func fb20AssertConcealedDenial(t *testing.T, got *RecipeExecution, err error) st return "code=" + userErr.Code() } -// fb20AssertMembershipDeniedByTheRealAuthorizer proves the caller genuinely -// lacks project membership using the shipped AuthorizeProject. Without it, a -// denial result would be vacuous because the "foreign" caller might not have -// been authorized in the first place. func fb20AssertMembershipDeniedByTheRealAuthorizer(t *testing.T, principal *authscope.Principal) { t.Helper() err := authscope.AuthorizeProject(principal, fb20ForeignProject, false) @@ -364,10 +300,6 @@ func fb20Itoa(value *int) string { return strconv.Itoa(*value) } -// Case A: the archived primary defect arm. scopeResolver == nil (server.go -// passes auth.scopeResolver, nil whenever no Fence is configured) and a -// principal whose project allowlist holds only "P". Pre-fix this disclosed -// foreign metadata unconditionally; the repaired reader must conceal. func TestFB20NilScopeResolverRefusesForeignPrincipal(t *testing.T) { fb20RequireRepairedGuards(t) store := &fb20ExecutionStore{execution: fb20ForeignBundle([]string{fb20CallerPath})} @@ -393,10 +325,6 @@ func TestFB20NilScopeResolverRefusesForeignPrincipal(t *testing.T) { " and errors.Is(err, authscope.ErrForbidden); the anonymous (principal-less) posture stays allowed by design") } -// Case A2: a scope resolver IS configured but the caller carries neither -// auth_resource_paths nor a bearer token, so the production resolver would -// treat the caller as unrestricted. Pre-fix the unrestricted mode skipped the -// reader gate entirely; now membership alone refuses. func TestFB20ResolverPresentWithoutCallerGrantsIsRefused(t *testing.T) { fb20RequireRepairedGuards(t) store := &fb20ExecutionStore{execution: fb20ForeignBundle([]string{fb20CallerPath})} @@ -422,10 +350,6 @@ func TestFB20ResolverPresentWithoutCallerGrantsIsRefused(t *testing.T) { "; zero listExisting calls prove the ordering (archived shape required unrestricted mode to fall through)") } -// Case B1: the exact archived acceptance arm. A restricted-mode caller holds -// the very auth-path string the foreign execution carries while its project -// allowlist excludes the foreign project. Pre-fix the reader accepted on the -// path string alone. func TestFB20RestrictedPathStringAloneNoLongerAdmits(t *testing.T) { fb20RequireRepairedGuards(t) store := &fb20ExecutionStore{execution: fb20ForeignBundle([]string{fb20CallerPath})} @@ -439,8 +363,6 @@ func TestFB20RestrictedPathStringAloneNoLongerAdmits(t *testing.T) { } ctx := authscope.ContextWithPrincipal(context.Background(), principal) - // Proof the acceptance arm is real: the resolver itself would authorize - // the held path for the foreign generation (it did pre-fix). scope, scopeErr := resolver.ResolveReadScopeForGeneration(ctx, principal, fb20ForeignProject, fb20ForeignGeneration, []string{fb20CallerPath}) if scopeErr != nil { t.Fatalf("precondition: production resolver should authorize the held path, got %v", scopeErr) @@ -467,9 +389,6 @@ func TestFB20RestrictedPathStringAloneNoLongerAdmits(t *testing.T) { "authorized, yet the reader refused with "+denied+" before any catalog call") } -// Case B2: a member principal whose grants miss the execution's path stays -// concealed through the resolver; a non-member is refused before the resolver. -// Both arrive indistinguishable. func TestFB20PathMismatchStaysConcealedForMemberAndNonMember(t *testing.T) { fb20RequireRepairedGuards(t) alienPrincipal := &authscope.Principal{ @@ -477,8 +396,6 @@ func TestFB20PathMismatchStaysConcealedForMemberAndNonMember(t *testing.T) { Projects: []string{fb20ForeignProject}, AuthResourcePaths: []string{fb20AlienPath}, } - // Control: the member/missing-path denial must come from the resolver, so - // the non-member denial below cannot silently share the wrong source. storeM := &fb20ExecutionStore{execution: fb20ForeignBundle([]string{fb20CallerPath})} probeM := &fb20ScopeProbe{existingByProject: map[string][]string{fb20ForeignProject: {fb20AlienPath}}} readerM := NewAuthorizedRecipeExecutionReader(storeM, fb20ReadScopeResolver(t, probeM)) @@ -510,11 +427,6 @@ func TestFB20PathMismatchStaysConcealedForMemberAndNonMember(t *testing.T) { "remain indistinguishable to callers") } -// Case C: the count-compare's over-rejection side finding and the fail-closed -// blank-declaration rule. Pre-fix, recipe_execution_reader.go:40 denied a -// caller authorized on every distinct path purely because raw and normalized -// counts differed; a restricted execution with no non-blank path was servable -// whenever the restricted scope's count matched. func TestFB20SetRuleReplacesCountCompare(t *testing.T) { fb20RequireRepairedGuards(t) t.Run("authorized-caller-duplicate-declaration-is-served", func(t *testing.T) { @@ -563,9 +475,6 @@ func TestFB20SetRuleReplacesCountCompare(t *testing.T) { }) } -// TestFB20PreventionPrecedentsAndRESTTwinStayQuoted keeps the contrast evidence -// honest and records the REST sibling's repaired posture. Its live behaviour is -// covered by package server tests. func TestFB20PreventionPrecedentsAndRESTTwinStayQuoted(t *testing.T) { fb20RequireRepairedGuards(t) service, err := os.ReadFile(fb20SourceService) diff --git a/internal/api/graphql/graph/resolver/recipe_execution_twin_wire_parity_test.go b/internal/api/graphql/graph/resolver/recipe_execution_twin_wire_parity_test.go index 2f45bfd3a..d04156c6a 100644 --- a/internal/api/graphql/graph/resolver/recipe_execution_twin_wire_parity_test.go +++ b/internal/api/graphql/graph/resolver/recipe_execution_twin_wire_parity_test.go @@ -14,20 +14,6 @@ import ( materialization "github.com/calypr/loom/internal/dataframe/publication" ) -// REPAIR-20b (adversarial review ses_e65cad76effe): absorbed twin-side half -// of the reviewer's parity harness (/tmp/r20-out/harness/ -// r20_parity_twin_wire_test.go). This is the twin-side pin for the shared -// failure-field oracle: for each reviewed durable shape, the REAL reader + -// executionModel wire (the GraphQL twin consumers get exactly this) must -// equal the reviewer-committed golden. The REST-side matrix -// (internal/server/recipe_execution_etl_failure_fields_e2e_test.go) compares -// the generated strict-stack body against the same oracle, so any drift in -// executionModel / dataframeapi.PersistedFailure / -// errors.SanitizePersistedFailure breaks this file, and any REST divergence -// breaks that one. Shapes and golden live in testdata/r20_failure_parity/ -// (canonical copy of the reviewer's /tmp/r20-out artifacts; the server test -// carries the identical fixture pair). - type r20TwinStore struct { byID map[string]materialization.BundleExecution } @@ -99,9 +85,6 @@ func r20JSONEqual(path string, left, right any) string { } } -// TestR20TwinWireMatchesReviewedGolden drives every reviewed shape through -// the production reader + executionModel — the genuine twin pipeline — and -// requires the marshaled GraphQL wire to equal the reviewer golden verbatim. func TestR20TwinWireMatchesReviewedGolden(t *testing.T) { var shapes []json.RawMessage if err := json.Unmarshal(r20ReadTestdata(t, "shapes.json"), &shapes); err != nil { @@ -161,21 +144,6 @@ func TestR20TwinWireMatchesReviewedGolden(t *testing.T) { } } -// r20AssertHealthyOutputsPublishNoFailureTrio pins the promise authority -// (openapi_recipe_execution.go:95-97, twin side = the all-nil -// PersistedFailure arm at recipe_control_graphql.go:319 feeding -// model.go:73-75): a durable output in a successful state (READY/PUBLISHED) -// that persisted no own failure code publishes NO error/errorCode/ -// errorRetryable keys, even when the EXECUTION page is FAILED. The shape-5 -// lease-lost reconciliation (bundle_store.go:1207-1214) sets only -// execution-level fields and leaves outputs untouched, so the reader's -// execution.Error mirror (recipe_execution_reader.go:72) must not become a -// healthy output's own trio: a fully-published page would assert its own -// failure with a sanitizer-inferred errorRetryable that contradicts the -// execution truth (PUBLICATION_LEASE_LOST, retryable=true). Genuine -// per-output failures always persist a own FailureCode (failPhase -// bundle_store.go:989-995 is the sole per-output failure writer) or carry an -// unhealthy state, and both keep projecting. func r20AssertHealthyOutputsPublishNoFailureTrio(t *testing.T, shapeName string, durable materialization.BundleExecution, liveWire map[string]any) { t.Helper() outputs, ok := liveWire["outputs"].([]any) @@ -200,12 +168,6 @@ func r20AssertHealthyOutputsPublishNoFailureTrio(t *testing.T, shapeName string, } } -// r20GenuineOutputFailureRecord is a healthy execution page carrying a -// genuine per-output failure stamped exactly the way the sole production -// per-output failure writer persists it (failPhase bundle_store.go:989-995): -// the output's OWN FailureCode/FailureRetryable/FailurePhase are persisted. -// Execution-level error fields stay empty so the only possible trio source -// at the output is the output's own persisted failure state. func r20GenuineOutputFailureRecord(outputState materialization.BundleState) materialization.BundleExecution { return materialization.BundleExecution{ ID: "shape-genuine-" + string(outputState), @@ -225,15 +187,6 @@ func r20GenuineOutputFailureRecord(outputState materialization.BundleState) mate } } -// TestR20GenuineOutputFailureProjectsDespiteHealthyExecution is the -// suppression-mutant guard for REPAIR-20c. A "suppress all output errors" -// repair (keyed on the execution page being healthy, or on the output state -// alone even when a own failure code was persisted) must NOT pass: the -// genuine per-output failure — (a) output FAILED while the execution is -// PUBLISHED with no execution-level error, and (b) a READY output that -// persisted its own failure code (shape-6 direction, pinned independently of -// the golden anchor) — must still project the full trio plus phase through -// the real reader + executionModel. func TestR20GenuineOutputFailureProjectsDespiteHealthyExecution(t *testing.T) { principal := &authscope.Principal{Subject: "operator-P", Projects: []string{"P"}} ctx := authscope.ContextWithPrincipal(context.Background(), principal) diff --git a/internal/api/graphql/graph/resolver/resolver.go b/internal/api/graphql/graph/resolver/resolver.go index 08eee1839..015fc1d79 100644 --- a/internal/api/graphql/graph/resolver/resolver.go +++ b/internal/api/graphql/graph/resolver/resolver.go @@ -15,9 +15,6 @@ import ( dataset "github.com/calypr/loom/internal/dataset" ) -// RecipeControl is the transport-neutral semantic control plane used by the -// recipe GraphQL fields. It intentionally returns logical plans only; no -// AQL, SQL, table names, or credentials cross this boundary. type RecipeControl interface { Validate(context.Context, string, recipe.RuntimeBindings) (dataframeexecution.Validation, error) Explain(context.Context, string, recipe.RuntimeBindings) (semantic.RecipePlanExplanation, error) @@ -27,9 +24,6 @@ type RecipeControl interface { Run(context.Context, string, recipe.RuntimeBindings) (dataframeexecution.Preview, error) } -// RecipeExecution is the stable logical execution record exposed by GraphQL. -// Implementations may persist additional physical metadata, but adapters must -// not return it here. type RecipeExecution struct { ID string Name string @@ -60,9 +54,6 @@ type RecipeExecutionOutput struct { type RecipeExecutionReader func(context.Context, string) (*RecipeExecution, error) -// RecipeAuthorizer resolves request bindings against the caller's project -// grants. GraphQL operation type is intentionally irrelevant: preview/run are -// reads even though they are mutations, while publication is a write. type RecipeAuthorizer interface { AuthorizeRead(context.Context, recipe.RuntimeBindings) (recipe.RuntimeBindings, error) AuthorizeWrite(context.Context, recipe.RuntimeBindings) (recipe.RuntimeBindings, error) diff --git a/internal/api/graphql/graph/resolver/schema.resolvers.go b/internal/api/graphql/graph/resolver/schema.resolvers.go index 29ffc07a1..0b4ece972 100644 --- a/internal/api/graphql/graph/resolver/schema.resolvers.go +++ b/internal/api/graphql/graph/resolver/schema.resolvers.go @@ -35,10 +35,7 @@ func (r *mutationResolver) RunFhirDataframe(ctx context.Context, input model.Fhi }, nil } -// RunDataframeRecipe executes a registered recipe completely and returns all -// selected rows. This is intentionally separate from preview and from the -// streaming publication mutation; callers should use materialization for -// large production exports. +// RunDataframeRecipe is the resolver for the runDataframeRecipe field. func (r *mutationResolver) RunDataframeRecipe(ctx context.Context, input model.RunDataframeRecipeInput) (*model.DataframeRecipeResult, error) { bindings, err := graphqlRecipeBindings(input.Bindings) if err != nil { diff --git a/internal/api/graphql/graph/schema/schema.graphqls b/internal/api/graphql/graph/schema/schema.graphqls index d1e4f4a3e..263c9acb4 100644 --- a/internal/api/graphql/graph/schema/schema.graphqls +++ b/internal/api/graphql/graph/schema/schema.graphqls @@ -453,10 +453,6 @@ enum FhirValueMode { DISTINCT } -# Typed, storage-independent dataframe predicates. `select` is a FHIR path -# (or a resolved catalog selector), while `fieldRef` is optional frontend -# provenance used during scoped field resolution. Values are a strict tagged -# union; exactly one member matching `kind` is accepted by recipe validation. enum FhirFilterOperator { EQUALS NOT_EQUALS @@ -507,8 +503,6 @@ input FhirFilterInput { select: String! fieldRef: String operator: FhirFilterOperator! - # Scalar selectors do not accept an array quantifier. Repeated selectors - # must opt into ANY, ALL, or NONE explicitly after schema resolution. quantifier: FhirFilterQuantifier values: [FhirFilterValueInput!]! = [] } @@ -606,9 +600,6 @@ input FhirDataframeInput { cursor: String } -# Explicit, bounded FHIR graph traversal. Graph mode defaults each edge to -# REQUIRED (unlike the dataframe builder's OPTIONAL default) and returns -# complete node paths rather than flattened rows. input FhirGraphQueryInput { project: String! authResourcePaths: [String!] @@ -676,13 +667,11 @@ type FhirDataframeResult { diagnostics: DataframeQueryDiagnostics! } -# Server-side timings for one dataframe request. `arangoQueryMs` excludes -# Loom's per-row flattening/delivery time; together those two fields identify -# whether the database plan or result materialization is dominant. type DataframeQueryDiagnostics { inputResolutionMs: Float! requestPreparationMs: Float! compilationMs: Float! + sourceQueryMs: Float! arangoQueryMs: Float! rowMaterializationMs: Float! resultAssemblyMs: Float! @@ -690,8 +679,6 @@ type DataframeQueryDiagnostics { plan: DataframeCompilerPlanDiagnostics! } -# Compiler facts for deciding which AQL rewrite to implement next. These are -# not Arango estimates; pair them with a live PROFILE for operator-level cost. type DataframeCompilerPlanDiagnostics { traversalSets: Int! sharedTraversalCount: Int! diff --git a/internal/api/http/backend_error_boundary_contract_test.go b/internal/api/http/backend_error_boundary_contract_test.go new file mode 100644 index 000000000..635c1d87f --- /dev/null +++ b/internal/api/http/backend_error_boundary_contract_test.go @@ -0,0 +1,200 @@ +package httpapi + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/calypr/loom/internal/backends/postgres/control" + "net/http" + "strings" + "testing" + + "github.com/arangodb/go-driver/v2/arangodb/shared" + arangofactory "github.com/calypr/loom/internal/backends/arango" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +func classifiedArangoFailure(t *testing.T, driverErr error) error { + t.Helper() + classified, ok := arangofactory.TranslateQueryError(fmt.Errorf("arango query: %w", driverErr)) + if !ok { + t.Fatalf("adapter fixture was not classified: %v", driverErr) + } + return classified +} + +func classifiedPostgresFailure(t *testing.T, driverErr error) error { + t.Helper() + translated, ok := control.TranslateError(fmt.Errorf("postgres rows: query: %w", driverErr)) + if !ok { + t.Fatalf("boundary fixture was not classified: %v", driverErr) + } + return translated +} + +func mustBody(t *testing.T, mapped MappedError) string { + t.Helper() + encoded, err := json.Marshal(mapped.Body) + if err != nil { + t.Fatal(err) + } + return string(encoded) +} + +func TestBackendErrorBoundaryContract(t *testing.T) { + t.Run("map_consumes_only_loom_errors", func(t *testing.T) { + rawArango := fmt.Errorf("arango query: %w", shared.ArangoError{HasError: true, Code: 500, ErrorNum: 4, ErrorMessage: "boom hunter2"}) + mapped := MapDataframeError(rawArango, "req-raw") + if mapped.Status != http.StatusInternalServerError || mapped.Body.Error.Code != "INTERNAL_ERROR" { + t.Fatalf("raw vendor error mapped to %d %q; transport must not interpret vendor failures", mapped.Status, mapped.Body.Error.Code) + } + if body := mustBody(t, mapped); strings.Contains(body, "hunter2") || strings.Contains(body, "boom") { + t.Fatalf("raw vendor detail leaked: %s", body) + } + rawPG := &pgconn.PgError{Code: "XX001", Message: "internal server catastrophe hunter2"} + mapped = MapDataframeError(rawPG, "req-raw-pg") + if mapped.Body.Error.Code != "INTERNAL_ERROR" { + t.Fatalf("raw PG error interpreted by transport: %q", mapped.Body.Error.Code) + } + if body := mustBody(t, mapped); strings.Contains(body, "hunter2") || strings.Contains(body, "catastrophe") { + t.Fatalf("raw PG detail leaked: %s", body) + } + }) + + t.Run("equivalent_backend_failures_map_identically", func(t *testing.T) { + arangoMapped := MapDataframeError(classifiedArangoFailure(t, shared.ArangoError{ + HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, ErrorMessage: "AQL: query would use more memory than allowed", + }), "req-a") + pgMapped := MapDataframeError(classifiedPostgresFailure(t, &pgconn.PgError{ + Code: "57014", SeverityUnlocalized: "ERROR", Message: "canceling statement due to statement_timeout", + }), "req-p") + if arangoMapped.Status != http.StatusServiceUnavailable || pgMapped.Status != http.StatusServiceUnavailable { + t.Fatalf("statuses = %d/%d, want 503/503", arangoMapped.Status, pgMapped.Status) + } + if arangoMapped.Body.Error.Retryable || pgMapped.Body.Error.Retryable { + t.Fatal("neither backend's limit failure may advertise a blanket retry") + } + if pgMapped.Body.Error.Code != "QUERY_RESOURCE_LIMIT_EXCEEDED" { + t.Fatalf("PG cancel code = %q, want QUERY_RESOURCE_LIMIT_EXCEEDED", pgMapped.Body.Error.Code) + } + for _, mapped := range []MappedError{arangoMapped, pgMapped} { + body := mustBody(t, mapped) + for _, forbidden := range []string{"AQL", "statement_timeout", "SELECT", "bind", "hunter2"} { + if strings.Contains(strings.ToLower(body), strings.ToLower(forbidden)) { + t.Fatalf("public payload leaked %q: %s", forbidden, body) + } + } + } + }) + + t.Run("public_details_stay_safe", func(t *testing.T) { + classified := classifiedArangoFailure(t, shared.ArangoError{ + HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, + ErrorMessage: `AQL: memory cap hit while evaluating {"token":"secret-token-value","password":"hunter2"}`, + }) + mapped := MapDataframeError(classified, "req-safe") + body := mustBody(t, mapped) + for _, forbidden := range []string{"secret-token-value", "hunter2", "AQL", "memory cap"} { + if strings.Contains(strings.ToLower(body), strings.ToLower(forbidden)) { + t.Fatalf("public payload leaked %q: %s", forbidden, body) + } + } + if mapped.Body.Error.Details["backend"] != "arangodb" || mapped.Body.Error.Details["resource"] != "query_memory" { + t.Fatalf("safe details missing: %v", mapped.Body.Error.Details) + } + }) + + t.Run("cancellation_and_deadline_identity_through_wrapping", func(t *testing.T) { + canceled := fmt.Errorf("stream aborted: %w", context.Canceled) + mapped := MapDataframeError(canceled, "req-c") + if mapped.Body.Error.Code != "CLIENT_CANCELED" || mapped.Status != 499 { + t.Fatalf("canceled = %d %q", mapped.Status, mapped.Body.Error.Code) + } + deadline := fmt.Errorf("stream deadline: %w", context.DeadlineExceeded) + mapped = MapDataframeError(deadline, "req-d") + if mapped.Body.Error.Code != "BACKEND_UNAVAILABLE" || !mapped.Body.Error.Retryable { + t.Fatalf("deadline = %d %q retryable=%t", mapped.Status, mapped.Body.Error.Code, mapped.Body.Error.Retryable) + } + pgCanceled := fmt.Errorf("acquire: %w: %w", control.ErrConnect, context.Canceled) + translated, ok := control.TranslateError(pgCanceled) + if !ok || !errors.Is(translated, context.Canceled) { + t.Fatalf("boundary lost cancellation identity: %v", translated) + } + mapped = MapDataframeError(translated, "req-pc") + if mapped.Body.Error.Retryable { + t.Fatal("caller cancellation must never be retryable") + } + }) + + t.Run("unique_violation_never_becomes_a_cas_conflict", func(t *testing.T) { + translated := classifiedPostgresFailure(t, &pgconn.PgError{ + Code: "23505", SeverityUnlocalized: "ERROR", + Message: `duplicate key value violates unique constraint "uq_cas_revision"`, + ConstraintName: "uq_cas_revision", + }) + if errors.Is(translated, control.ErrUniqueConflict) { + t.Fatal("unproven unique violation presented as domain conflict") + } + mapped := MapDataframeError(translated, "req-u") + if mapped.Body.Error.Code != "INTERNAL_ERROR" || mapped.Body.Error.Retryable { + t.Fatalf("unique violation mapped = %d %q retryable=%t", mapped.Status, mapped.Body.Error.Code, mapped.Body.Error.Retryable) + } + body := mustBody(t, mapped) + for _, forbidden := range []string{"uq_cas_revision", "duplicate key", "violates unique constraint"} { + if strings.Contains(strings.ToLower(body), strings.ToLower(forbidden)) { + t.Fatalf("public payload leaked %q: %s", forbidden, body) + } + } + if _, ok := control.AsUniqueConflict(translated); !ok { + t.Fatal("targeted upgrade path lost") + } + }) + + t.Run("ambiguous_commit_is_distinguishable_from_proven_rollback", func(t *testing.T) { + proven, ok := control.TranslateCommitError(fmt.Errorf("commit: %w", pgx.ErrTxCommitRollback)) + if !ok { + t.Fatal("proven rollback not classified") + } + if errors.Is(proven, control.ErrCommitOutcomeUnknown) { + t.Fatal("proven rollback labeled ambiguous") + } + ambiguous, ok := control.TranslateCommitError(fmt.Errorf("commit: %w", + &pgconn.PgError{Code: "57P01", SeverityUnlocalized: "FATAL", Message: "terminating connection due to administrator command"})) + if !ok { + t.Fatal("mid-commit session death not classified") + } + if !errors.Is(ambiguous, control.ErrCommitOutcomeUnknown) { + t.Fatal("mid-commit session death must be outcome-unknown") + } + provenMapped := MapDataframeError(proven, "req-proven") + ambiguousMapped := MapDataframeError(ambiguous, "req-amb") + if provenMapped.Body.Error.Retryable || ambiguousMapped.Body.Error.Retryable { + t.Fatal("no commit outcome may advertise a blanket retry") + } + if ambiguousMapped.Body.Error.Code != provenMapped.Body.Error.Code { + t.Fatal("public codes unexpectedly differ") + } + }) + + t.Run("serialization_conflicts_stay_policy_only", func(t *testing.T) { + translated := classifiedPostgresFailure(t, &pgconn.PgError{Code: "40001", SeverityUnlocalized: "ERROR", Message: "could not serialize access due to concurrent update"}) + if !control.RetryableTransactionConflict(translated) { + t.Fatal("operation policy recognition lost") + } + userErr, isUser := dataframeerrors.AsUserError(translated) + if !isUser || userErr.Retryable() { + t.Fatalf("boundary marked serialization failure blanket-retryable: %#v", translated) + } + mapped := MapDataframeError(translated, "req-s") + if mapped.Body.Error.Retryable { + t.Fatal("transport must not advertise retry for transaction conflicts") + } + body := mustBody(t, mapped) + if strings.Contains(strings.ToLower(body), "could not serialize access") { + t.Fatalf("public payload leaked the server message: %s", body) + } + }) +} diff --git a/internal/api/http/doc.go b/internal/api/http/doc.go index ad926ed8f..c15357247 100644 --- a/internal/api/http/doc.go +++ b/internal/api/http/doc.go @@ -1,3 +1 @@ -// Package httpapi owns shared HTTP transport concerns: Fiber setup, middleware, -// health checks, and error mapping. Individual APIs register their own routes. package httpapi diff --git a/internal/api/http/errors.go b/internal/api/http/errors.go index a5f1bdf8f..f2ea16048 100644 --- a/internal/api/http/errors.go +++ b/internal/api/http/errors.go @@ -13,8 +13,6 @@ import ( "github.com/gofiber/fiber/v3" ) -// ErrorResponse is the transport-neutral JSON payload the HTTP adapter can -// write before a streaming response has begun. type ErrorResponse struct { Error HTTPErrorBody `json:"error"` } @@ -28,18 +26,12 @@ type HTTPErrorBody struct { RequestID string `json:"requestId,omitempty"` } -// MappedError is the status and safe body chosen for a semantic error. It is -// intentionally independent of Fiber so the same mapping can be tested and -// reused by the export route and future HTTP adapters. type MappedError struct { Status int Body ErrorResponse Cause error } -// MapDataframeError maps a service error without inspecting its text. Unknown -// errors are always redacted to INTERNAL_ERROR. The original cause is kept -// only for operator logging. func MapDataframeError(err error, requestID string) MappedError { if err == nil { return MappedError{} @@ -108,8 +100,6 @@ func MapDataframeError(err error, requestID string) MappedError { } } -// StatusForErrorCode exposes the same semantic HTTP classification used by -// REST handlers to transports that already have a stable Loom error code. func StatusForErrorCode(code string) int { return statusForCode(normalizeHTTPCode(code), 0) } @@ -137,9 +127,6 @@ type httpCodePolicy struct { Status int } -// httpCodePolicies is the transport registry. Domain ErrorCode declarations -// are checked against this table by TestHTTPCodePolicyCoversEveryPublicError. -// Entries with no special transport status intentionally use HTTP 400. var httpCodePolicies = map[string]httpCodePolicy{ "PROJECT_REQUIRED": {http.StatusBadRequest}, "ROOT_RESOURCE_TYPE_REQUIRED": {http.StatusBadRequest}, "UNAUTHORIZED_PROJECT": {http.StatusForbidden}, "UNKNOWN_FIELD": {http.StatusBadRequest}, "FIELD_NOT_POPULATED": {http.StatusBadRequest}, "INVALID_TRAVERSAL": {http.StatusBadRequest}, "UNSAFE_TRAVERSAL_ROUTE": {http.StatusBadRequest}, diff --git a/internal/api/http/errors_test.go b/internal/api/http/errors_test.go index fd733eae9..7f8fc7d6b 100644 --- a/internal/api/http/errors_test.go +++ b/internal/api/http/errors_test.go @@ -1,7 +1,10 @@ package httpapi import ( + "encoding/json" "errors" + "fmt" + "github.com/calypr/loom/internal/backends/postgres/control" "go/ast" "go/parser" "go/token" @@ -14,6 +17,7 @@ import ( dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" "github.com/gofiber/fiber/v3" + "github.com/jackc/pgx/v5/pgconn" ) func TestHTTPCodePolicyCoversEveryPublicError(t *testing.T) { @@ -227,3 +231,97 @@ func TestMapDataframeErrorDescribesUnknownAdapterUserCode(t *testing.T) { } } } + +func TestMapDataframeErrorRedactsPostgresDriverCause(t *testing.T) { + canaries := []string{ + "reb_http_canary_select", "reb_http_schema", "reb_http_table", + "reb_http_ux", "reb_http_bind_value", "reb_http_user", "reb_http_pass", "reb_http_host", + } + secretPgErr := &pgconn.PgError{ + Severity: "ERROR", Code: "23505", + Message: `duplicate key value violates unique constraint "reb_http_ux" with reb_http_canary_select`, + Detail: `Key (col)=(reb_http_bind_value) already exists on reb_http_table.`, + Hint: "check reb_http_schema", + SchemaName: "reb_http_schema", + TableName: "reb_http_table", + ConstraintName: "reb_http_ux", + } + + assertJSONClean := func(what string, mapped MappedError) string { + t.Helper() + body, err := json.Marshal(mapped.Body) + if err != nil { + t.Fatalf("marshal %s: %v", what, err) + } + for _, forbidden := range canaries { + if strings.Contains(string(body), forbidden) { + t.Fatalf("%s payload leaks %q: %s", what, forbidden, body) + } + } + return string(body) + } + + t.Run("untranslated driver error stays redacted", func(t *testing.T) { + mapped := MapDataframeError(fmt.Errorf("seam: %w", secretPgErr), "req-pg-raw") + if mapped.Status != http.StatusInternalServerError || mapped.Body.Error.Code != "INTERNAL_ERROR" { + t.Fatalf("mapped = %#v, want redacted 500 INTERNAL_ERROR", mapped) + } + assertJSONClean("raw 23505", mapped) + }) + + t.Run("translated connectivity failure is retryable 503", func(t *testing.T) { + failure := &control.ConnectError{Op: "open", Err: fmt.Errorf( + "failed to connect to `user=reb_http_user password=reb_http_pass host=reb_http_host`: refused")} + translated, ok := control.TranslateError(failure) + if !ok { + t.Fatal("connect failure not translated") + } + mapped := MapDataframeError(translated, "req-pg-connect") + body := assertJSONClean("connect", mapped) + if mapped.Status != http.StatusServiceUnavailable || mapped.Body.Error.Code != "BACKEND_UNAVAILABLE" || !mapped.Body.Error.Retryable { + t.Fatalf("mapped = %s, want retryable 503 BACKEND_UNAVAILABLE", body) + } + if mapped.Body.Error.Details["backend"] != "postgres" { + t.Fatalf("details lack postgres tag: %#v", mapped.Body.Error.Details) + } + }) + + t.Run("translated serialization failure is never blanket retryable", func(t *testing.T) { + translated, ok := control.TranslateError(fmt.Errorf("seam: %w", + &pgconn.PgError{Severity: "ERROR", Code: "40001", Message: "reb_http_canary_select serialize conflict on reb_http_table"})) + if !ok { + t.Fatal("40001 not translated") + } + mapped := MapDataframeError(translated, "req-pg-ssi") + body := assertJSONClean("40001", mapped) + if mapped.Body.Error.Retryable || mapped.Status != http.StatusInternalServerError { + t.Fatalf("mapped = %s, want non-retryable 500 (retry is an operation-policy decision)", body) + } + }) + + t.Run("translated ambiguous commit is never retryable", func(t *testing.T) { + translated, ok := control.TranslateCommitError(fmt.Errorf("commit: %w", + &pgconn.PgError{Severity: "FATAL", Code: "57P01", Message: "reb_http_canary_select terminating reb_http_table"})) + if !ok { + t.Fatal("ambiguous commit not translated") + } + mapped := MapDataframeError(translated, "req-pg-commit") + body := assertJSONClean("commit", mapped) + if mapped.Body.Error.Retryable { + t.Fatalf("mapped = %s, ambiguous commit must never advertise retry", body) + } + }) + + t.Run("translated query cancel is non-retryable limit", func(t *testing.T) { + translated, ok := control.TranslateError(fmt.Errorf("seam: %w", + &pgconn.PgError{Severity: "ERROR", Code: "57014", Message: "reb_http_canary_select canceling statement"})) + if !ok { + t.Fatal("57014 not translated") + } + mapped := MapDataframeError(translated, "req-pg-cancel") + body := assertJSONClean("57014", mapped) + if mapped.Body.Error.Code != "QUERY_RESOURCE_LIMIT_EXCEEDED" || mapped.Body.Error.Retryable { + t.Fatalf("mapped = %s, want non-retryable QUERY_RESOURCE_LIMIT_EXCEEDED", body) + } + }) +} diff --git a/internal/api/http/health_readiness_test.go b/internal/api/http/health_readiness_test.go new file mode 100644 index 000000000..9cf4355ff --- /dev/null +++ b/internal/api/http/health_readiness_test.go @@ -0,0 +1,226 @@ +package httpapi + +import ( + "context" + "errors" + "net/http" + "sync/atomic" + "testing" + "time" + + "github.com/calypr/loom/internal/authscope" +) + +func newHealthServer(t *testing.T, cfg HTTPConfig) *HTTPServer { + t.Helper() + cfg.Authorizer = authscope.AllowAllAuthorizer{} + server, err := NewHTTPServer(cfg) + if err != nil { + t.Fatal(err) + } + return server +} + +func TestReadinessConsultsBothChecksIndependently(t *testing.T) { + tests := []struct { + name string + coreErr, clickHouseErr error + clickHouseEnabled bool + wantStatus int + wantOverall, wantCore string + wantDataframe string + wantCoreInvoked bool + wantClickHouseInvoked bool + }{ + {name: "both down", coreErr: errors.New("core down"), clickHouseErr: errors.New("clickhouse down"), clickHouseEnabled: true, + wantStatus: http.StatusServiceUnavailable, wantOverall: "not_ready", wantCore: "not_ready", wantDataframe: "backend_unavailable", + wantCoreInvoked: true, wantClickHouseInvoked: true}, + {name: "core down clickhouse up", coreErr: errors.New("core down"), clickHouseEnabled: true, + wantStatus: http.StatusServiceUnavailable, wantOverall: "not_ready", wantCore: "not_ready", wantDataframe: "ready", + wantCoreInvoked: true, wantClickHouseInvoked: true}, + {name: "core down clickhouse disabled", coreErr: errors.New("core down"), + wantStatus: http.StatusServiceUnavailable, wantOverall: "not_ready", wantCore: "not_ready", wantDataframe: "disabled", + wantCoreInvoked: true, wantClickHouseInvoked: false}, + {name: "core up clickhouse down", clickHouseErr: errors.New("clickhouse down"), clickHouseEnabled: true, + wantStatus: http.StatusServiceUnavailable, wantOverall: "not_ready", wantCore: "ready", wantDataframe: "backend_unavailable", + wantCoreInvoked: true, wantClickHouseInvoked: true}, + {name: "both up", clickHouseEnabled: true, + wantStatus: http.StatusOK, wantOverall: "ready", wantCore: "ready", wantDataframe: "ready", + wantCoreInvoked: true, wantClickHouseInvoked: true}, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + var coreInvoked, clickHouseInvoked atomic.Bool + server := newHealthServer(t, HTTPConfig{ + CoreReadyCheck: func(context.Context) error { + coreInvoked.Store(true) + return tt.coreErr + }, + ClickHouseEnabled: tt.clickHouseEnabled, + ClickHouseReadyCheck: func(context.Context) error { + clickHouseInvoked.Store(true) + return tt.clickHouseErr + }, + }) + body, status := server.Readiness(context.Background()) + dataframe, _ := body["dataframe"].(string) + if status != tt.wantStatus || body["status"] != tt.wantOverall || body["core"] != tt.wantCore || dataframe != tt.wantDataframe { + t.Fatalf("status=%d body=%#v", status, body) + } + if coreInvoked.Load() != tt.wantCoreInvoked { + t.Fatalf("core invoked=%v, want %v", coreInvoked.Load(), tt.wantCoreInvoked) + } + if clickHouseInvoked.Load() != tt.wantClickHouseInvoked { + t.Fatalf("clickhouse invoked=%v, want %v", clickHouseInvoked.Load(), tt.wantClickHouseInvoked) + } + }) + } +} + +func TestReadinessHungCoreDoesNotFabricateClickHouseVerdict(t *testing.T) { + coreRelease := make(chan struct{}) + t.Cleanup(func() { close(coreRelease) }) + var clickHouseCtxLive, clickHouseInvoked atomic.Bool + server := newHealthServer(t, HTTPConfig{ + CoreReadyCheck: func(ctx context.Context) error { + <-ctx.Done() + <-coreRelease + return ctx.Err() + }, + ClickHouseEnabled: true, + ClickHouseReadyCheck: func(ctx context.Context) error { + clickHouseInvoked.Store(true) + select { + case <-ctx.Done(): + clickHouseCtxLive.Store(false) + default: + clickHouseCtxLive.Store(true) + } + return nil + }, + }) + started := time.Now() + body, status := server.Readiness(context.Background()) + if !clickHouseInvoked.Load() { + t.Fatal("ClickHouse check was never consulted while core hung") + } + if !clickHouseCtxLive.Load() { + t.Fatal("ClickHouse check received an already-expired context: core hang consumed the shared budget") + } + if status != http.StatusServiceUnavailable || body["status"] != "not_ready" || body["core"] != "not_ready" { + t.Fatalf("hung core must report not_ready, got status=%d body=%#v", status, body) + } + if dataframe, _ := body["dataframe"].(string); dataframe != "ready" { + t.Fatalf("ClickHouse verdict must be its own (ready), got %#v", body) + } + if elapsed := time.Since(started); elapsed > 3*time.Second { + t.Fatalf("readiness waited %s; per-check budgets must bound the probe", elapsed) + } +} + +func TestLivenessNeverConsultsChecksOnAnyLane(t *testing.T) { + var checks atomic.Int64 + server := newHealthServer(t, HTTPConfig{ + CoreReadyCheck: func(context.Context) error { checks.Add(1); return errors.New("backend down") }, + ClickHouseEnabled: true, + ClickHouseReadyCheck: func(context.Context) error { checks.Add(1); return errors.New("clickhouse down") }, + }) + if _, status := server.Readiness(context.Background()); status != http.StatusServiceUnavailable { + t.Fatal("fixture must start with both dependencies failing") + } + before := checks.Load() + body, status := server.Liveness(context.Background()) + if status != http.StatusOK || body["status"] != "live" { + t.Fatalf("liveness status=%d body=%#v", status, body) + } + if got := checks.Load() - before; got != 0 { + t.Fatalf("liveness consulted %d dependency checks; must be DB-free", got) + } +} + +func TestReadinessCoreStateClassifierReportsSafeTokens(t *testing.T) { + classifier := func(err error) string { + switch { + case errors.Is(err, errSchemaDown): + return "schema_mismatch" + case errors.Is(err, errConnectivityDown): + return "connectivity_down" + default: + return "core_state_unknown" + } + } + for _, tt := range []struct { + name string + coreErr error + wantState string + }{ + {name: "schema mismatch", coreErr: errSchemaDown, wantState: "schema_mismatch"}, + {name: "connectivity", coreErr: errConnectivityDown, wantState: "connectivity_down"}, + {name: "unclassified", coreErr: errors.New("plain fault"), wantState: "core_state_unknown"}, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + server := newHealthServer(t, HTTPConfig{ + CoreReadyCheck: func(context.Context) error { return tt.coreErr }, + CoreState: classifier, + ClickHouseEnabled: true, + ClickHouseReadyCheck: func(context.Context) error { return nil }, + }) + body, status := server.Readiness(context.Background()) + if status != http.StatusServiceUnavailable || body["core"] != "not_ready" { + t.Fatalf("status=%d body=%#v", status, body) + } + if body["core_state"] != tt.wantState { + t.Fatalf("core_state=%v, want %s (body=%#v)", body["core_state"], tt.wantState, body) + } + if text, ok := body["core_state"].(string); !ok || len(text) > 64 { + t.Fatalf("core_state must be a short fixed token, got %#v", body["core_state"]) + } + }) + } +} + +func TestReadinessWithoutClassifierKeepsPublishedBodies(t *testing.T) { + server := newHealthServer(t, HTTPConfig{ + CoreReadyCheck: func(context.Context) error { return errors.New("arangodb unreachable") }, + ClickHouseEnabled: true, + ClickHouseReadyCheck: func(context.Context) error { return nil }, + }) + body, status := server.Readiness(context.Background()) + if status != http.StatusServiceUnavailable { + t.Fatalf("status=%d", status) + } + if _, present := body["core_state"]; present { + t.Fatalf("classifier-less (Arango-mode) body carries core_state: %#v", body) + } + if body["status"] != "not_ready" || body["core"] != "not_ready" || body["dataframe"] != "ready" { + t.Fatalf("unexpected readiness body %#v", body) + } + if _, present := body["core_state"]; present { + t.Fatalf("cached Health must not gain core_state either: %#v", body) + } + degraded := newHealthServer(t, HTTPConfig{ + ClickHouseEnabled: true, + ClickHouseReadyCheck: func(context.Context) error { return errors.New("clickhouse unavailable") }, + }) + hb, hs := degraded.Health(context.Background()) + if hs != http.StatusOK || hb["status"] != "degraded" || hb["dataframe"] != "backend_unavailable" { + t.Fatalf("health status=%d body=%#v", hs, hb) + } + if _, present := hb["core_state"]; present { + t.Fatalf("classifier-less Health body carries core_state: %#v", hb) + } +} + +var ( + errSchemaDown = errors.New("ledger checksum mismatch") + errConnectivityDown = &credentialBearingError{} +) + +type credentialBearingError struct{} + +func (credentialBearingError) Error() string { + return `postgres: ping: dial tcp postgres://user:s3cr3t@db:5432/loom: connection refused` +} +func (credentialBearingError) Is(target error) bool { return target == errConnectivityDown } diff --git a/internal/api/http/http_test.go b/internal/api/http/http_test.go index b79937d36..35a0cdc42 100644 --- a/internal/api/http/http_test.go +++ b/internal/api/http/http_test.go @@ -51,7 +51,7 @@ func TestReadinessRequiresCoreAndClickHouse(t *testing.T) { wantOverall, wantCore, wantDataframe string }{ {name: "ready", clickHouseEnabled: true, wantStatus: http.StatusOK, wantOverall: "ready", wantCore: "ready", wantDataframe: "ready"}, - {name: "core unavailable", coreErr: errors.New("core unavailable"), clickHouseEnabled: true, wantStatus: http.StatusServiceUnavailable, wantOverall: "not_ready", wantCore: "not_ready"}, + {name: "core unavailable", coreErr: errors.New("core unavailable"), clickHouseEnabled: true, wantStatus: http.StatusServiceUnavailable, wantOverall: "not_ready", wantCore: "not_ready", wantDataframe: "ready"}, {name: "ClickHouse unavailable", clickHouseErr: errors.New("ClickHouse unavailable"), clickHouseEnabled: true, wantStatus: http.StatusServiceUnavailable, wantOverall: "not_ready", wantCore: "ready", wantDataframe: "backend_unavailable"}, {name: "ClickHouse disabled", wantStatus: http.StatusOK, wantOverall: "ready", wantCore: "ready", wantDataframe: "disabled"}, } diff --git a/internal/api/http/middleware.go b/internal/api/http/middleware.go index 8c143dec2..cbab2e403 100644 --- a/internal/api/http/middleware.go +++ b/internal/api/http/middleware.go @@ -15,12 +15,6 @@ import ( const maxLoggedErrorResponseBytes = 32 * 1024 -// errorResponseLogAttrs extracts the useful, server-owned diagnostic fields -// from responses that were already written by a handler. Authoring routes -// intentionally write their 4xx response and return nil, so the Fiber error -// handler cannot log the underlying diagnostic. Keeping this extraction in -// the request middleware gives every route the same failure visibility while -// bounding the raw packet retained in logs. func errorResponseLogAttrs(body []byte) []any { if len(body) == 0 { return nil @@ -118,8 +112,6 @@ func (s *HTTPServer) loggingMiddleware(c fiber.Ctx) error { } func (s *HTTPServer) authenticationMiddleware(c fiber.Ctx) error { - // Health probes must remain available before credentials are configured and - // are deliberately not project/data APIs. switch c.Path() { case "/health", "/livez", "/readyz": return c.Next() diff --git a/internal/api/http/routes.go b/internal/api/http/routes.go index b4f47b117..8f0c2b192 100644 --- a/internal/api/http/routes.go +++ b/internal/api/http/routes.go @@ -2,6 +2,7 @@ package httpapi import ( "context" + "errors" "time" "github.com/gofiber/fiber/v3" @@ -11,8 +12,6 @@ func (s *HTTPServer) register() { s.app.Use(s.requestIDMiddleware, s.recoveryMiddleware, s.loggingMiddleware, s.authenticationMiddleware) } -// Health computes the cached dependency health response for generated HTTP -// routes without requiring a Fiber context. func (s *HTTPServer) Health(parent context.Context) (map[string]any, int) { s.healthMu.Lock() defer s.healthMu.Unlock() @@ -41,22 +40,83 @@ func (s *HTTPServer) Readiness(parent context.Context) (map[string]any, int) { return healthBody(result), result.httpStatus } -func (s *HTTPServer) checkDependencies(ctx context.Context) healthResult { +const ( + readinessCheckBudget = 2 * time.Second + readinessCollectGrace = 500 * time.Millisecond +) + +var errReadinessCheckTimedOut = errors.New("readiness check exceeded its budget") + +func runBoundedCheck(parent context.Context, check func(context.Context) error) <-chan error { + out := make(chan error, 1) + go func() { + ctx, cancel := context.WithTimeout(parent, readinessCheckBudget) + defer cancel() + out <- check(ctx) + }() + return out +} + +func awaitCheck(parent context.Context, results <-chan error, deadlineAt time.Time) error { + select { + case err := <-results: + return err + default: + } + if parent.Err() != nil { + return context.Cause(parent) + } + remaining := time.Until(deadlineAt) + if remaining <= 0 { + return errReadinessCheckTimedOut + } + timer := time.NewTimer(remaining) + defer timer.Stop() + select { + case err := <-results: + return err + case <-timer.C: + return errReadinessCheckTimedOut + case <-parent.Done(): + return context.Cause(parent) + } +} + +func (s *HTTPServer) checkDependencies(parent context.Context) healthResult { result := healthResult{status: "ready", core: "ready", dataframe: "ready", httpStatus: fiber.StatusOK} + var coreErr, clickHouseErr error + var coreResults, clickHouseResults <-chan error if s.coreReadyCheck != nil { - if err := s.coreReadyCheck(ctx); err != nil { - s.logger.Error("core readiness check failed", "error", err) - return healthResult{status: "not_ready", core: "not_ready", httpStatus: fiber.StatusServiceUnavailable} + coreResults = runBoundedCheck(parent, s.coreReadyCheck) + } + if s.clickHouseEnabled && s.clickHouseReadyCheck != nil { + clickHouseResults = runBoundedCheck(parent, s.clickHouseReadyCheck) + } + deadlineAt := time.Now().Add(readinessCheckBudget + readinessCollectGrace) + if coreResults != nil { + coreErr = awaitCheck(parent, coreResults, deadlineAt) + } + if clickHouseResults != nil { + clickHouseErr = awaitCheck(parent, clickHouseResults, deadlineAt) + } + if coreErr != nil { + s.logger.Error("core readiness check failed", "error", coreErr) + result.core = "not_ready" + if s.coreState != nil { + result.coreState = s.coreState(coreErr) } } if !s.clickHouseEnabled { result.dataframe = "disabled" + } else if clickHouseErr != nil { + s.logger.Error("dataframe readiness check failed", "error", clickHouseErr) + result.dataframe = "backend_unavailable" } - if s.clickHouseEnabled && s.clickHouseReadyCheck != nil { - if err := s.clickHouseReadyCheck(ctx); err != nil { - s.logger.Error("dataframe readiness check failed", "error", err) - result.status, result.dataframe = "degraded", "backend_unavailable" - } + switch { + case result.core != "ready": + result.status, result.httpStatus = "not_ready", fiber.StatusServiceUnavailable + case result.dataframe == "backend_unavailable": + result.status, result.dataframe = "degraded", "backend_unavailable" } return result } @@ -66,5 +126,8 @@ func healthBody(result healthResult) map[string]any { if result.dataframe != "" { body["dataframe"] = result.dataframe } + if result.coreState != "" { + body["core_state"] = result.coreState + } return body } diff --git a/internal/api/http/server.go b/internal/api/http/server.go index 83bb72281..5b14dea99 100644 --- a/internal/api/http/server.go +++ b/internal/api/http/server.go @@ -21,6 +21,7 @@ type HTTPConfig struct { CoreReadyCheck func(context.Context) error ClickHouseReadyCheck func(context.Context) error ClickHouseEnabled bool + CoreState func(error) string } type HTTPServer struct { @@ -31,6 +32,7 @@ type HTTPServer struct { coreReadyCheck func(context.Context) error clickHouseReadyCheck func(context.Context) error clickHouseEnabled bool + coreState func(error) string healthMu sync.Mutex lastHealth time.Time lastHealthResult healthResult @@ -38,6 +40,7 @@ type HTTPServer struct { type healthResult struct { status, core, dataframe string + coreState string httpStatus int } @@ -75,6 +78,7 @@ func NewHTTPServer(cfg HTTPConfig) (*HTTPServer, error) { coreReadyCheck: cfg.CoreReadyCheck, clickHouseReadyCheck: cfg.ClickHouseReadyCheck, clickHouseEnabled: cfg.ClickHouseEnabled, + coreState: cfg.CoreState, } app := fiber.New(fiber.Config{ BodyLimit: cfg.BodyLimit, diff --git a/internal/api/recipe/discovery.go b/internal/api/recipe/discovery.go index e834ff648..ef08dc5d6 100644 --- a/internal/api/recipe/discovery.go +++ b/internal/api/recipe/discovery.go @@ -1,5 +1,3 @@ -// Package recipeapi owns transport-neutral adapters used by recipe -// resolution entrypoints. package recipeapi import ( @@ -12,15 +10,10 @@ import ( "github.com/calypr/loom/internal/dataframe/recipe/schema" ) -// FieldDiscovery adapts the scoped field catalog to recipe schema resolution. -// Both one-shot and stored-recipe callers use this policy so metadata and -// pivot declarations cannot drift between transports. type FieldDiscovery struct { Read func(context.Context, catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) } -// NewFieldDiscovery returns the shared catalog adapter used by every -// recipe-resolution entrypoint. func NewFieldDiscovery(read func(context.Context, catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error)) schema.Discovery { return FieldDiscovery{Read: read} } @@ -35,10 +28,6 @@ func (d FieldDiscovery) Fields(ctx context.Context, scope schema.Scope, resource value := false unrestricted = &value default: - // Fail closed. authscope.Covers denies unknown modes, and a nil - // pointer here would silently fall through the catalog's legacy - // "empty paths mean unrestricted" compatibility rule — the two - // consumers of one persisted scope must agree on the fail posture. value := false unrestricted = &value } diff --git a/internal/api/recipe/discovery_auth_mode_test.go b/internal/api/recipe/discovery_auth_mode_test.go index a1ff55887..045e97c89 100644 --- a/internal/api/recipe/discovery_auth_mode_test.go +++ b/internal/api/recipe/discovery_auth_mode_test.go @@ -1,15 +1,5 @@ package recipeapi -// Durable authorization-posture assertions for the discovery bridge -// (Round 78). schema.Scope.AuthScopeMode is a persisted string that reaches -// this adapter through materialization/execution documents. The catalog -// resolves a nil AuthResourcePathsUnrestricted through its legacy -// compatibility rule — empty paths then mean UNRESTRICTED — while -// authscope.Covers is pinned (covers_execution_paths_control_test) to DENY -// unknown modes. One persisted scope value must not fail open in one -// consumer and closed in the other: an unrecognized mode has to arrive at -// the catalog as an explicit restricted request. - import ( "context" "testing" @@ -53,9 +43,6 @@ func TestFieldDiscoveryEveryAuthScopeModeArrivesExplicit(t *testing.T) { if *opts.AuthResourcePathsUnrestricted != tc.want { t.Fatalf("mode %q: unrestricted=%v want %v", tc.name, *opts.AuthResourcePathsUnrestricted, tc.want) } - // The consumer-side fact the pointer exists to control: with the - // bridge's output, an empty path list must resolve to exactly the - // intended posture, never to the legacy default. if effective := catalog.EffectiveAuthResourcePathsUnrestricted(opts.AuthResourcePaths, opts.AuthResourcePathsUnrestricted); effective != tc.want { t.Fatalf("mode %q: effective=%v want %v", tc.name, effective, tc.want) } @@ -63,8 +50,6 @@ func TestFieldDiscoveryEveryAuthScopeModeArrivesExplicit(t *testing.T) { } func TestFieldDiscoveryUnknownModeMatchesCoversDeny(t *testing.T) { - // authscope-side pinned truth: unknown mode is not unrestricted for - // Covers either. The bridge must not diverge on the same value. unknown := authscope.ReadScope{AuthResourcePaths: nil, Mode: authscope.ReadScopeMode("garbage")} if unknown.Unrestricted() { t.Fatal("authscope.IsUnrestricted(garbage) must be false") diff --git a/internal/authscope/all_generations_cache_key_test.go b/internal/authscope/all_generations_cache_key_test.go index 050d8205b..6e3e3a8e5 100644 --- a/internal/authscope/all_generations_cache_key_test.go +++ b/internal/authscope/all_generations_cache_key_test.go @@ -11,32 +11,8 @@ import ( "github.com/calypr/loom/internal/catalog" ) -// F-B27 durable controls (REPAIR-15). The project-write-binding entry that -// backs AuthorizeWrite must live in its own cache namespace, separated by a -// discriminator on the key type and not by a reserved string. Read-path -// datasetGeneration values arrive from request payloads and are only -// TrimSpace'd (catalog.NormalizeDatasetGeneration), unlike publish-time -// validateOpaqueIdentifier, so any byte sequence is an ordinary generation -// key on the read side. That is exactly why a string sentinel was unsafe: a -// restricted caller could address the write-binding entry directly. - -// fb27ctlSentinelGeneration is the literal that the first F-B27 fix used as -// its all-generations cache sentinel. The promoted probe below replays it as -// the poison payload. const fb27ctlSentinelGeneration = "\x00all-generations" -// fb27ctlLister is a catalog-discovery test double. It records every invocation -// so the controls below can prove which decisions re-consulted the catalog. It -// is SHARED mutable state across a test's resolver calls, which is why the -// recording is mutex-guarded: ScopeResolver invokes its lister OUTSIDE the -// cache lock (scope_resolver.go:378,421), so a distinct-key burst on a cold -// cache drives several goroutines into list() concurrently — the counters-map -// flake class this helper shares with the pre-hygiene authscope doubles (the -// per-subtest recorder standard recorded in -// covers_execution_paths_control_test.go:102-105). Guarding here keeps a -// -count>1 / concurrent driver race-clean in addition to the already-serial -// single-threaded controls. The behind callback is invoked outside the lock so -// it never runs while the recorder is held. type fb27ctlLister struct { mu sync.Mutex calls int @@ -53,23 +29,18 @@ func (l *fb27ctlLister) list(_ context.Context, opts catalog.AuthResourcePathOpt return result, err } -// count returns the number of recorded invocations under the recorder lock. func (l *fb27ctlLister) count() int { l.mu.Lock() defer l.mu.Unlock() return l.calls } -// snapshotCalls returns a copy of every recorded options value under the -// recorder lock, safe to read while other goroutines keep invoking list(). func (l *fb27ctlLister) snapshotCalls() []catalog.AuthResourcePathOptions { l.mu.Lock() defer l.mu.Unlock() return append([]catalog.AuthResourcePathOptions(nil), l.opts...) } -// lastCall returns the most recently recorded options value under the recorder -// lock. func (l *fb27ctlLister) lastCall() (catalog.AuthResourcePathOptions, bool) { l.mu.Lock() defer l.mu.Unlock() @@ -90,23 +61,11 @@ func fb27ctlPrincipal(paths ...string) *Principal { return &Principal{Subject: "u", AuthResourcePaths: append([]string(nil), paths...)} } -// TestFB27SentinelGenerationPoisonCannotReachAuthorizeWrite promotes the -// adversarial probe TestFB27RevSentinelCollisionPoisonsWriteBinding into the -// repo suite. A restricted principal performs a read whose datasetGeneration -// is the reserved all-generations literal, so an empty path list lands under -// {victim, "\x00all-generations"}, and then asks to publish a path it holds -// for another project into that victim. Repaired contract: the poisoned entry -// is never served to AuthorizeWrite, the victim's project lister is consulted -// on the decision, and the write stays forbidden for the TTL. func TestFB27SentinelGenerationPoisonCannotReachAuthorizeWrite(t *testing.T) { l := &fb27ctlLister{behind: func(opts catalog.AuthResourcePathOptions) ([]string, error) { if opts.AllGenerations { - // The victim genuinely records this path: the exact state that - // made the pre-repair path-membership check exploitable. return []string{"projB/data"}, nil } - // No real catalog holds a control-character generation, because - // publish rejects it, so a generation-scoped listing is empty. return nil, nil }} r := fb27ctlResolver(l) @@ -147,26 +106,9 @@ func TestFB27SentinelGenerationPoisonCannotReachAuthorizeWrite(t *testing.T) { t.Logf("VERDICT=REPAIRED sentinel-literal generation %q stayed an ordinary generation key; the write decision re-listed projB (%d fresh call) and denied /projA/data", fb27ctlSentinelGeneration, l.count()-callsAfterPoison) } -// TestFB27ConcurrentDistinctKeyBurstKeepsSharedRecorderConsistent is the -// durability control for the counters-map flake class: it drives ONE shared -// fb27ctlLister from many goroutines at once. ScopeResolver consults the lister -// outside its cache lock (scope_resolver.go:378,421), so a cold-cache burst -// across DISTINCT project/generation keys — which never collide in the resolver -// cache — runs list() concurrently on the shared recorder. Under the pre-hygiene -// unlocked counter this is exactly where `go test -race` flagged the shared -// calls/opts mutation; the recorder mutex added above makes it race-clean. The -// assertions confirm the shared control stayed internally consistent (one -// recording per invocation, no lost/torn appends) while the resolvers returned -// the correct per-key answers. The behind callback is read-only so the only -// shared mutable state exercised here is the recorder under test. func TestFB27ConcurrentDistinctKeyBurstKeepsSharedRecorderConsistent(t *testing.T) { const goroutines = 64 l := &fb27ctlLister{behind: func(opts catalog.AuthResourcePathOptions) ([]string, error) { - // Read-only dispatch: an all-generations lookup (the write-binding - // namespace) yields the held path; a generation-scoped read yields - // none. Deterministic per opts only — the behind closure touches no - // shared state, so the only shared mutable state under test is the - // recorder itself. if opts.AllGenerations { return []string{opts.Project + "/data"}, nil } @@ -182,9 +124,6 @@ func TestFB27ConcurrentDistinctKeyBurstKeepsSharedRecorderConsistent(t *testing. go func(i int) { defer wg.Done() project := "proj" + strconv.Itoa(i) - // Distinct project per goroutine guarantees distinct resolver cache - // keys on a cold cache, so BOTH lookups below are concurrent lister - // misses that funnel into the shared recorder at once. if _, err := r.listExistingPaths(ctx, project, "gen-"+strconv.Itoa(i)); err != nil { errCh <- err return @@ -201,8 +140,6 @@ func TestFB27ConcurrentDistinctKeyBurstKeepsSharedRecorderConsistent(t *testing. t.Fatalf("VERDICT=UNEXPECTED concurrent resolver error = %v", err) } - // Exactly one generation-scoped read plus one all-generations listing per - // goroutine landed in the recorder, with none lost or torn. if got := l.count(); got != 2*goroutines { t.Fatalf("VERDICT=UNEXPECTED lost/torn recorder increments: count=%d want=%d", got, 2*goroutines) } diff --git a/internal/authscope/auth_path_control_ingress_test.go b/internal/authscope/auth_path_control_ingress_test.go index a42716575..694a4de33 100644 --- a/internal/authscope/auth_path_control_ingress_test.go +++ b/internal/authscope/auth_path_control_ingress_test.go @@ -6,13 +6,6 @@ import ( "testing" ) -// F-B22 Tier-1 durable control: auth resource paths carrying NUL or any -// control byte must be rejected at ingest, so scope digests that join -// normalized paths with \x00 can never fold two distinct restricted grants. -// Pre-fix code accepted these paths verbatim at every site; archived -// CONFIRMED text is kept verbatim as the REGRESSION branch so any regression -// fails loudly again. - func TestNormalizeAuthResourcePathRejectsControlCharacters(t *testing.T) { cases := []struct { name string @@ -56,10 +49,6 @@ func TestCalyprKeyExtractionRejectsControlCharacters(t *testing.T) { } func TestNormalizedPathListDropsControlCharEntries(t *testing.T) { - // Archived fold pair from the F-B22 discovery: under the pre-fix - // ingest these two distinct grant lists survived normalization and, - // joined with \x00 by the explorer encoders (explorer_capability.go - // and lifecycle/policy.go), folded to the same digest. left := normalizeAuthResourcePathList([]string{"a\x00b", "c"}) right := normalizeAuthResourcePathList([]string{"a", "b\x00c"}) join := func(paths []string) string { return strings.Join(paths, "\x00") } @@ -86,8 +75,6 @@ func TestNormalizedListDropsControlCharEntriesStable(t *testing.T) { t.Logf("VERDICT=REPAIRED NormalizeAuthResourcePaths=%q (deduped, normalized, control-character entries dropped)", got) } -// normalizeAuthResourcePathListRaw reproduces the archived pre-fix behavior -// for shape discrimination only (no product dependency). func normalizeAuthResourcePathListRaw(paths []string) []string { seen := map[string]struct{}{} out := []string{} diff --git a/internal/authscope/auth_resource_path_options_conflict_test.go b/internal/authscope/auth_resource_path_options_conflict_test.go index 361806b40..618fe5084 100644 --- a/internal/authscope/auth_resource_path_options_conflict_test.go +++ b/internal/authscope/auth_resource_path_options_conflict_test.go @@ -2,126 +2,11 @@ package authscope import ( "context" - "encoding/json" - "errors" - "strings" "testing" "github.com/calypr/loom/internal/catalog" - catalogarango "github.com/calypr/loom/internal/catalog/arango" - store "github.com/calypr/loom/internal/store/arango" ) -// F-2 durable control for the F-B27 all-generations discovery (REPAIR-15). -// Auth-path discovery asks for either one dataset generation or the union over -// all of them, never both: the all-generations query ignores @dataset_generation, -// so accepting both would silently discard the caller's selection. This control -// sits in the F-B27 family because authscope is the only producer of -// AllGenerations requests, which makes the boundary part of its contract. The -// fake Arango client keeps it hermetic. - -type fb27ctlArangoClient struct { - queries []string - vars []map[string]any -} - -func (c *fb27ctlArangoClient) QueryRows(_ context.Context, query string, _ int, vars map[string]any, _ store.RowVisitor) error { - c.queries = append(c.queries, query) - copied := make(map[string]any, len(vars)) - for k, v := range vars { - copied[k] = v - } - c.vars = append(c.vars, copied) - return nil -} - -func (c *fb27ctlArangoClient) InsertBatchRaw(context.Context, string, []json.RawMessage, bool, string) error { - panic("unused by the auth path discovery controls") -} - -func (c *fb27ctlArangoClient) ExecuteAQL(context.Context, string, map[string]any) error { - panic("unused by the auth path discovery controls") -} - -func (c *fb27ctlArangoClient) CollectionExists(context.Context, string) (bool, error) { - panic("unused by the auth path discovery controls") -} - -func (c *fb27ctlArangoClient) Bootstrap(context.Context, store.BootstrapSpec) error { - panic("unused by the auth path discovery controls") -} - -// fb27ctlDiscover runs one discovery against a fresh fake client and asserts -// the guard/query boundary: a rejected request must not reach Arango, an -// accepted one must issue exactly one query. -func fb27ctlDiscover(t *testing.T, opts catalog.AuthResourcePathOptions) ([]string, error) { - t.Helper() - client := &fb27ctlArangoClient{} - s, err := catalogarango.New(client) - if err != nil { - t.Fatalf("VERDICT=UNEXPECTED store construction = %v", err) - } - paths, err := s.DiscoverExistingAuthResourcePaths(context.Background(), opts) - if err != nil { - if len(client.queries) != 0 { - t.Fatalf("VERDICT=UNEXPECTED rejected request reached Arango (%d queries) for %+v: %v", len(client.queries), opts, err) - } - return paths, err - } - if len(client.queries) != 1 { - t.Fatalf("VERDICT=UNEXPECTED accepted request issued %d queries for %+v, want 1", len(client.queries), opts) - } - if paths == nil { - t.Fatalf("VERDICT=UNEXPECTED accepted request returned nil paths for %+v", opts) - } - return paths, nil -} - -func TestFB27AllGenerationsWithSelectedGenerationIsRejected(t *testing.T) { - for _, generation := range []string{"gen-1", " gen-1 ", "sha256:0000000000000000000000000000000000000000000000000000000000000000"} { - _, err := fb27ctlDiscover(t, catalog.AuthResourcePathOptions{ - Project: "projA", - DatasetGeneration: generation, - AllGenerations: true, - }) - if err == nil { - t.Fatalf("VERDICT=CONFIRMED (pre-fix archived shape): AllGenerations with datasetGeneration %q was accepted, so discovery silently ignored the selected generation", generation) - } - if !errors.Is(err, catalogarango.ErrAllGenerationsWithGeneration) { - t.Fatalf("VERDICT=UNEXPECTED conflict error must wrap ErrAllGenerationsWithGeneration, got %v", err) - } - for _, want := range []string{"AllGenerations", "datasetGeneration", generation} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("VERDICT=UNEXPECTED conflict error %q must name the conflicting option and its value %q", err, want) - } - } - } - t.Logf("VERDICT=REPAIRED AllGenerations plus a selected generation is refused before any query") -} - -func TestFB27SupportedAuthResourcePathOptionShapesStillDiscover(t *testing.T) { - cases := []struct { - name string - opts catalog.AuthResourcePathOptions - }{ - {"all generations with empty generation", catalog.AuthResourcePathOptions{Project: "projA", AllGenerations: true}}, - {"all generations with whitespace only generation", catalog.AuthResourcePathOptions{Project: "projA", DatasetGeneration: " ", AllGenerations: true}}, - {"generation scoped", catalog.AuthResourcePathOptions{Project: "projA", DatasetGeneration: " gen-1 "}}, - {"legacy null namespace", catalog.AuthResourcePathOptions{Project: "projA"}}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if _, err := fb27ctlDiscover(t, tc.opts); err != nil { - t.Fatalf("VERDICT=UNEXPECTED valid options %+v rejected: %v", tc.opts, err) - } - }) - } -} - -// TestFB27ResolverNeverEmitsTheConflictingShape pins the producer side of the -// same boundary: every all-generations request the resolver makes carries an -// empty generation, and every generation-bearing request leaves AllGenerations -// false, so the rejected combination is unreachable from the authz path. func TestFB27ResolverNeverEmitsTheConflictingShape(t *testing.T) { l := &fb27ctlLister{behind: func(opts catalog.AuthResourcePathOptions) ([]string, error) { if opts.AllGenerations && catalog.NormalizeDatasetGeneration(opts.DatasetGeneration) != "" { diff --git a/internal/authscope/authentication.go b/internal/authscope/authentication.go index 901d66297..4fe683075 100644 --- a/internal/authscope/authentication.go +++ b/internal/authscope/authentication.go @@ -9,9 +9,6 @@ import ( "strings" ) -// BasicAuthenticator authenticates the standalone/operator mode. A basic -// principal is deliberately unrestricted: Basic is a process-level operator -// credential, not a substitute for a per-project Fence ACL. type BasicAuthenticator struct { Username string Password string @@ -40,17 +37,12 @@ func (a BasicAuthenticator) Authenticate(_ context.Context, headers map[string][ func constantTimeEqual(left, right string) bool { leftBytes, rightBytes := []byte(left), []byte(right) if len(leftBytes) != len(rightBytes) { - // Still perform a comparison to avoid making the common length mismatch - // an immediately observable fast path. _ = subtle.ConstantTimeCompare(leftBytes, rightBytes) return false } return subtle.ConstantTimeCompare(leftBytes, rightBytes) == 1 } -// CalyprAuthenticator is the strict HTTP boundary for Fence-backed mode. It -// validates presence and shape here; the ScopeResolver performs the signed -// resource authorization lookup using the same bearer header. type CalyprAuthenticator struct{} func (CalyprAuthenticator) Authenticate(_ context.Context, headers map[string][]string) (*Principal, error) { diff --git a/internal/authscope/authentication_test.go b/internal/authscope/authentication_test.go index 0b5408fe0..960cdf9b2 100644 --- a/internal/authscope/authentication_test.go +++ b/internal/authscope/authentication_test.go @@ -158,10 +158,6 @@ func bearerTokenWithIssuer(issuer string) string { return "Bearer e30." + encodedIssuer + ".signature" } -// bearerTokenWithIssuerAndNonce mints a distinct bearer token for the same -// issuer so each call hashes to its own cache key, the way a renewing client -// does. The nonce is echoed back as the granted resource by -// fenceGrantHandler so a test can tell which snapshot a lookup served. func bearerTokenWithIssuerAndNonce(issuer, nonce string) string { encodedIssuer := base64.RawURLEncoding.EncodeToString([]byte(`{"iss":"` + issuer + `"}`)) return "Bearer e30." + encodedIssuer + ".sig-" + nonce @@ -178,8 +174,6 @@ func fenceGrantHandler(calls *atomic.Int32) http.HandlerFunc { } } -// fenceGrantedResource is the resource fenceGrantHandler advertises for a -// token minted with the given nonce. func fenceGrantedResource(nonce string) string { return "/programs/example/projects/sig-" + nonce } @@ -240,7 +234,6 @@ func TestFenceUserAccessClientEvictsExpiredSnapshotAndRefetches(t *testing.T) { if calls.Load() != 2 { t.Fatalf("Fence calls = %d, want a fresh fetch after expiry", calls.Load()) } - // The expired snapshot is replaced, not accumulated. if got := fenceCacheLen(client); got != 1 { t.Fatalf("retained entries = %d, want 1 after refetch", got) } @@ -274,7 +267,6 @@ func TestFenceUserAccessClientCapsRetainedSnapshotsAcrossTokenChurn(t *testing.T t.Fatalf("retained entries = %d, want the configured bound %d", got, bound) } - // The most recent token is still cached; the churned-away oldest is not. before := calls.Load() if _, err := client.GetAllowedResources(context.Background(), newest, "read", "*"); err != nil { t.Fatalf("GetAllowedResources(newest) error = %v", err) @@ -330,8 +322,6 @@ func TestFenceUserAccessClientDropsExpiredEntriesOnInsert(t *testing.T) { t.Fatalf("retained entries = %d, want 3 before expiry", got) } - // Churned-away tokens are never looked up again, so only the insert-time - // sweep can release them; the cap alone would leave them retained. clock.advance(2 * time.Minute) if _, err := client.GetAllowedResources(context.Background(), bearerTokenWithIssuerAndNonce(serverURL, "d"), "read", "*"); err != nil { t.Fatalf("GetAllowedResources(d) error = %v", err) @@ -358,7 +348,6 @@ func TestFenceUserAccessClientEvictionDoesNotMixGrantsAcrossTokens(t *testing.T) if got := fenceCacheLen(client); got != 2 { t.Fatalf("retained entries = %d, want the bound 2", got) } - // Evicting A must leave B's own grants intact. before := calls.Load() snapshotB, err := client.getResourceAccess(context.Background(), tokenB) if err != nil { @@ -377,8 +366,6 @@ func TestFenceUserAccessClientEvictionDoesNotMixGrantsAcrossTokens(t *testing.T) t.Fatalf("B snapshot = %#v, want no grants from another token", snapshotB) } - // Reads hand out independent copies: mutating a returned snapshot must not - // reach the cached one, for the same token or any other. snapshotB[fenceGrantedResource("b")][0].Service = "tampered" delete(snapshotB, fenceGrantedResource("b")) snapshotB["/programs/example/projects/injected"] = []resourceAccessRecord{{Method: "write", Service: "*"}} @@ -410,8 +397,6 @@ func TestFenceUserAccessClientDropsExpiredEntryWhenRefetchFails(t *testing.T) { t.Fatal("snapshot absent before expiry") } - // The expired snapshot can never be served again, so the lookup that - // discovers it must release the slot even when the refresh fails. clock.advance(2 * time.Minute) backend.down.Store(true) if _, err := client.GetAllowedResources(context.Background(), token, "read", "*"); err == nil { @@ -421,7 +406,6 @@ func TestFenceUserAccessClientDropsExpiredEntryWhenRefetchFails(t *testing.T) { t.Fatalf("retained entries = %d, want the expired snapshot released", got) } - // Recovery refetches and repopulates the released slot. backend.down.Store(false) resources, err := client.GetAllowedResources(context.Background(), token, "read", "*") if err != nil || len(resources) != 1 || resources[0] != fenceGrantedResource("stale") { @@ -432,8 +416,6 @@ func TestFenceUserAccessClientDropsExpiredEntryWhenRefetchFails(t *testing.T) { } } -// fenceTransport fronts the test Fence server so a refresh can be failed on -// demand without tearing the server down. type fenceTransport struct { down atomic.Bool } diff --git a/internal/authscope/covers_execution_paths_control_test.go b/internal/authscope/covers_execution_paths_control_test.go index 728037d61..4286e09f6 100644 --- a/internal/authscope/covers_execution_paths_control_test.go +++ b/internal/authscope/covers_execution_paths_control_test.go @@ -7,24 +7,12 @@ import ( "github.com/calypr/loom/internal/catalog" ) -// Durable coverage control for CoversExecutionPaths (F-B20 review finding F-2). -// The subset-membership loop in scope_resolver.go is the only code that -// denies an execution declaring a path the restricted scope does not hold. -// The adversarial re-review (mutant C: membership loop neutered, guards kept) -// showed NO shipped repo test discriminated it. These controls replicate the -// reviewer probe semantics and pin every case where the subset loop is the -// sole discriminator. - const fb20CoversSentinelPath = "\x00all-generations" func fb20CoversRestricted(paths ...string) ReadScope { return ReadScope{AuthResourcePaths: paths, Mode: ReadScopeRestricted} } -// Subset-loop SOLE-DISCRIMINATOR cases: every one of these must flip when the -// membership loop is removed (mutant C). The mode and declared-empty guards -// alone answer every other cell of the truth table, so a test suite that -// passes here while missing these leaves the gate unguarded. func TestCoversExecutionPathsSubsetLoopIsOnlyDiscriminatorControl(t *testing.T) { cases := []struct { name string @@ -53,9 +41,6 @@ func TestCoversExecutionPathsSubsetLoopIsOnlyDiscriminatorControl(t *testing.T) } } -// Full truth table replicating the reviewer probe: passes within the subset -// rules (duplicates, whitespace/slash normalization, blank-only declared), -// mode matrix fail-closed, and control-character refusal upstream. func TestCoversExecutionPathsSubsetTruthTableControl(t *testing.T) { cases := []struct { name string @@ -99,10 +84,6 @@ func TestCoversExecutionPathsSubsetTruthTableControl(t *testing.T) { } } -// Resolver-driven cross-generation control: a scope resolved for one dataset -// generation must never cover an execution declared in a sibling generation. -// Each subtest builds its own resolver and lister so no mutable state is -// shared across subtests (race-clean by construction). func TestCoversExecutionPathsGenerationBoundControl(t *testing.T) { cases := []struct { name string @@ -146,11 +127,6 @@ func TestCoversExecutionPathsGenerationBoundControl(t *testing.T) { } } -// Companion pin to unknown-mode-direct-denies above: the OTHER consumer -// posture. Unrestricted() must report true ONLY for the explicit -// unrestricted enum; empty and garbage modes must not gain the bypass at -// any consumer of a persisted scope (R78 fail-posture alignment: the -// catalog discovery bridge now agrees with this table). func TestUnrestrictedRequiresExplicitUnrestrictedMode(t *testing.T) { for _, tc := range []struct { name string diff --git a/internal/authscope/doc.go b/internal/authscope/doc.go index 7b0545aec..256ffc3ea 100644 --- a/internal/authscope/doc.go +++ b/internal/authscope/doc.go @@ -1,3 +1 @@ -// Package authscope owns shared request principal context, bearer auth, -// and auth-resource-path scope resolution used by read and write surfaces. package authscope diff --git a/internal/authscope/fence_resource_access.go b/internal/authscope/fence_resource_access.go index c95a2e39d..f11fc5f5c 100644 --- a/internal/authscope/fence_resource_access.go +++ b/internal/authscope/fence_resource_access.go @@ -16,14 +16,6 @@ import ( "time" ) -// defaultFenceAccessCacheEntries bounds how many parsed Fence snapshots one -// client retains. A snapshot only has to survive one TTL, so the useful working -// set is the number of distinct bearer tokens seen within a TTL. Token churn -// keeps minting sha256 keys that are never read again, and the TTL only decides -// whether a snapshot is served, never whether its memory is released: the entry -// cap is the memory bound, and the expiry sweeps on read and on insert are what -// keep churned-away snapshots from holding slots. Past the cap the least -// recently used token loses its slot and simply refetches. const defaultFenceAccessCacheEntries = 1024 type FenceUserAccessClient struct { @@ -40,9 +32,7 @@ type FenceUserAccessClient struct { type cachedResourceAccess struct { snapshot map[string][]resourceAccessRecord expiresAt time.Time - // seq orders cache touches (inserts and served hits) so cap eviction is - // deterministic and never depends on map iteration order. - seq uint64 + seq uint64 } func NewFenceUserAccessClientWithTrustedIssuer(client *http.Client, trustedIssuer string, cacheTTL time.Duration) (*FenceUserAccessClient, error) { @@ -63,9 +53,6 @@ func newFenceUserAccessClient(client *http.Client, trustedIssuer string, cacheTT return newFenceUserAccessClientWithCachePolicy(client, trustedIssuer, cacheTTL, defaultFenceAccessCacheEntries, nil) } -// newFenceUserAccessClientWithCachePolicy is the seam used by tests to pin the -// entry bound and the clock. A maxEntries below one disables retention -// instead of admitting nothing. func newFenceUserAccessClientWithCachePolicy(client *http.Client, trustedIssuer string, cacheTTL time.Duration, maxEntries int, now func() time.Time) *FenceUserAccessClient { if client == nil { client = http.DefaultClient @@ -93,9 +80,6 @@ func (c *FenceUserAccessClient) currentTime() time.Time { return time.Now() } -// ValidateTrustedFenceIssuer requires an absolute HTTP(S) issuer URL without -// userinfo, query, or fragment. The URL path is allowed because some issuers -// expose their user endpoint below a base path. func ValidateTrustedFenceIssuer(issuer string) error { _, err := parseTrustedFenceIssuer(issuer) return err @@ -163,15 +147,11 @@ func (c *FenceUserAccessClient) getResourceAccess(ctx context.Context, authoriza if cached, ok := c.cache[key]; ok { if now.Before(cached.expiresAt) { snapshot := cloneResourceAccess(cached.snapshot) - // A served hit makes this snapshot the most recently used, so - // cap eviction discards churned-away tokens first. cached.seq = c.nextAccessSeqLocked() c.cache[key] = cached c.mu.Unlock() return snapshot, nil } - // Expired entries are never served, so release the slot instead of - // leaving token churn to accumulate dead snapshots. delete(c.cache, key) } c.mu.Unlock() @@ -216,11 +196,6 @@ func (c *FenceUserAccessClient) getResourceAccess(ctx context.Context, authoriza return snapshot, nil } -// storeResourceAccessLocked installs a fresh snapshot valid until now+TTL and -// holds the cache within maxCacheEntries: snapshots past their TTL are dropped -// first (they can never be served again), then the least-recently-used entries -// until the cap is met. Eviction removes whole per-token keys, so one token can -// never lose grants belonging to another. Callers must hold c.mu. func (c *FenceUserAccessClient) storeResourceAccessLocked(key string, snapshot map[string][]resourceAccessRecord, now time.Time) { c.cache[key] = cachedResourceAccess{snapshot: cloneResourceAccess(snapshot), expiresAt: now.Add(c.cacheTTL), seq: c.nextAccessSeqLocked()} for token, entry := range c.cache { @@ -308,11 +283,6 @@ func NormalizeAuthResourcePath(value string) string { if value == "" { return "" } - // Control characters (including NUL) must never participate in scope - // paths: every scope-digest encoder joins normalized paths with \x00, - // so a NUL inside one declared path folds distinct restricted grant - // lists onto one digest (finding F-B22). Such declarations are refused - // at ingest; consumers already treat an empty normalization as absent. if containsAuthPathControlChar(value) { return "" } @@ -487,17 +457,10 @@ func normalizeResourcePath(resource string) string { return strings.TrimSuffix(resource, "/") } -// NormalizeAuthResourcePaths is the exported view of ingest-time scope-path -// normalization, for callers that build restricted read-scope paths without a -// wired ScopeResolver. Entries normalizing to empty or containing control -// characters are dropped (see the F-B22 note in NormalizeAuthResourcePath). func NormalizeAuthResourcePaths(paths []string) []string { return normalizeAuthResourcePathList(paths) } -// containsAuthPathControlChar reports whether value holds an ASCII control -// byte (C0 range or DEL). Those bytes never legitimately appear in -// authorization resource paths and they enable the F-B22 digest fold. func containsAuthPathControlChar(value string) bool { for i := 0; i < len(value); i++ { if value[i] < 0x20 || value[i] == 0x7f { diff --git a/internal/authscope/persisted_scope_fails_closed_test.go b/internal/authscope/persisted_scope_fails_closed_test.go index 53f7f2aa8..cb7a76e6d 100644 --- a/internal/authscope/persisted_scope_fails_closed_test.go +++ b/internal/authscope/persisted_scope_fails_closed_test.go @@ -9,23 +9,6 @@ import ( "github.com/calypr/loom/internal/catalog" ) -// Durable control for the persisted-declaration fails-closed rule (R4). -// -// Reproduction. A stored execution declares ["p1","foreign\x00secret"] — a -// persisted auth_resource_paths list whose second entry is malformed. -// normalizeAuthResourcePathList drops that entry, so the declaration collapsed -// onto the single grantable path "p1" and a member holding only p1 was served -// execution metadata. The malformed entry asserts a path the system can never -// name, list, or grant (NormalizeAuthResourcePath refuses forbidden C0/DEL -// bytes precisely because the \x00-joined scope digests would fold distinct -// grant lists), so a declaration containing one is not a resolvable -// authorization at all and must be refused rather than silently truncated. -// -// Scope of the rule. Only the DECLARATION a read is authorized against gets -// the stricter treatment. Incoming caller/grant normalization is deliberately -// unchanged (see TestGrantNormalizationRemainsUnchanged), and whitespace-only -// entries keep their long-standing "absent" meaning. - const ( msGrantedPath = "p1" msMalformedNUL = "foreign\x00secret" @@ -35,10 +18,6 @@ func msRestricted(paths ...string) ReadScope { return ReadScope{AuthResourcePaths: paths, Mode: ReadScopeRestricted} } -// TestCoversExecutionPathsRejectsInvalidNonblankDeclaration is the required -// regression: a mixed valid-plus-invalid declaration is refused outright, every -// forbidden C0/DEL control byte is refused, and both the valid covered policy -// and the legitimate blank policy keep their pre-existing answers. func TestCoversExecutionPathsRejectsInvalidNonblankDeclaration(t *testing.T) { cases := []struct { name string @@ -46,8 +25,6 @@ func TestCoversExecutionPathsRejectsInvalidNonblankDeclaration(t *testing.T) { declared []string want bool }{ - // The confirmed R4 reproduction: the surviving valid path alone must not - // cover a declaration that also asserts a malformed path. {"r4-mixed-valid-and-nul-denies", msRestricted(msGrantedPath), []string{msGrantedPath, msMalformedNUL}, false}, {"r4-mixed-order-reversed-denies", msRestricted(msGrantedPath), []string{msMalformedNUL, msGrantedPath}, false}, {"r4-mixed-with-malformed-entry-in-scope-too-denies", msRestricted(msGrantedPath, msMalformedNUL), []string{msGrantedPath, msMalformedNUL}, false}, @@ -65,22 +42,18 @@ func TestCoversExecutionPathsRejectsInvalidNonblankDeclaration(t *testing.T) { {"multi-slash-entry-normalizes-empty-and-denies", msRestricted(msGrantedPath), []string{msGrantedPath, "//"}, false}, {"malformed-entry-besides-foreign-path-denies", msRestricted(msGrantedPath, "p2"), []string{msGrantedPath, "p2", msMalformedNUL}, false}, - // Valid covered declarations keep serving exactly as before. {"valid-single-still-serves", msRestricted(msGrantedPath), []string{msGrantedPath}, true}, {"valid-multi-covered-still-serves", msRestricted(msGrantedPath, "p2"), []string{msGrantedPath, "p2"}, true}, {"valid-multi-covered-duplicates-and-slashes-still-serves", msRestricted(msGrantedPath, "p2"), []string{msGrantedPath, "p2", "p2", " /p2/ "}, true}, {"valid-entry-outside-declaration-still-serves", msRestricted(msGrantedPath, "p9"), []string{msGrantedPath}, true}, {"trailing-tab-is-trimmed-whitespace-serves", msRestricted(msGrantedPath), []string{msGrantedPath, msGrantedPath + "\t"}, true}, - // The legitimate blank policy is untouched: whitespace-only entries - // (including the C0 whitespace bytes TrimSpace recognizes) mean absent. {"blank-only-declaration-still-denies", msRestricted(msGrantedPath), []string{"", " "}, false}, {"valid-plus-blank-still-serves", msRestricted(msGrantedPath), []string{msGrantedPath, "", " ", "\t", "\n", "\r", "\v", "\f"}, true}, {"nil-declaration-still-denies", msRestricted(msGrantedPath), nil, false}, {"empty-declaration-still-denies", msRestricted(msGrantedPath), []string{}, false}, {"foreign-valid-path-still-denies", msRestricted(msGrantedPath), []string{"p2"}, false}, - // Malformed declarations fail closed under every non-restricted mode too. {"unrestricted-mode-malformed-denies", ReadScope{AuthResourcePaths: []string{msGrantedPath}, Mode: ReadScopeUnrestricted}, []string{msGrantedPath, msMalformedNUL}, false}, {"empty-mode-malformed-denies", ReadScope{AuthResourcePaths: []string{msGrantedPath}}, []string{msGrantedPath, msMalformedNUL}, false}, {"unknown-mode-malformed-denies", ReadScope{AuthResourcePaths: []string{msGrantedPath}, Mode: ReadScopeMode("garbage")}, []string{msGrantedPath, msMalformedNUL}, false}, @@ -95,16 +68,6 @@ func TestCoversExecutionPathsRejectsInvalidNonblankDeclaration(t *testing.T) { }) } - // Exhaustive forbidden-control sweep: for every C0 byte and DEL, an interior - // occurrence must refuse the whole mixed declaration, and a bare - // non-whitespace control entry must refuse it too. Whitespace-only entries - // stay legitimate blanks, so they are asserted not to flip the decision. - // The swept set is exactly what containsAuthPathControlChar refuses: - // 0x00-0x1f plus 0x7f. Printable bytes must not be asserted to refuse, and - // 0x20 is excluded because a bare space entry is a legitimate blank (pinned - // by the table cases above) whose padded forms are ordinary foreign paths — - // asserting refusal there would pass through the subset-membership loop - // rather than the malformed-entry rule this sweep exists to pin. t.Run("every-forbidden-control-byte-refuses-declaration", func(t *testing.T) { forbidden := make([]byte, 0, 0x21) for b := 0x00; b <= 0x1f; b++ { @@ -137,11 +100,6 @@ func TestCoversExecutionPathsRejectsInvalidNonblankDeclaration(t *testing.T) { }) } -// TestCoversExecutionPathsMalformedRefusalIsSoleDiscriminator pins that the -// malformed-entry refusal is what denies the R4 read: the identical scope covers -// the declaration with the malformed entry removed and with blank padding -// instead, so the denial cannot come from the mode guard, the blank guard, or -// the subset-membership loop. func TestCoversExecutionPathsMalformedRefusalIsSoleDiscriminator(t *testing.T) { scope := msRestricted(msGrantedPath) if !scope.CoversExecutionPaths([]string{msGrantedPath}) { @@ -155,11 +113,6 @@ func TestCoversExecutionPathsMalformedRefusalIsSoleDiscriminator(t *testing.T) { } } -// TestCoversExecutionPathsResolverBackedMalformedDeclarationDrivesTheGate runs -// the decision through the production resolver rather than a hand-built scope: -// a restricted caller granted p1, a catalog that records p1, and an execution -// declaring the mixed valid-plus-NUL list. The resolver echoes back the clean -// subset, so only the declaration-side refusal can deny the read. func TestCoversExecutionPathsResolverBackedMalformedDeclarationDrivesTheGate(t *testing.T) { lister := func(_ context.Context, opts catalog.AuthResourcePathOptions) ([]string, error) { if opts.Project != "legacy-project" || opts.DatasetGeneration != "g1" { @@ -189,10 +142,6 @@ func TestCoversExecutionPathsResolverBackedMalformedDeclarationDrivesTheGate(t * } } -// TestGrantNormalizationRemainsUnchanged guards the stated non-goal: the -// stricter rule applies to persisted execution declarations only. Incoming -// caller/grant normalization keeps dropping what it always dropped instead of -// refusing the whole grant list. func TestGrantNormalizationRemainsUnchanged(t *testing.T) { if got := NormalizeAuthResourcePaths([]string{msGrantedPath, msMalformedNUL, "", " /p2/ "}); strings.Join(got, "|") != "p1|p2" { t.Fatalf("grant normalization changed: got %q, want p1|p2 (malformed entries dropped, not fatal)", strings.Join(got, "|")) diff --git a/internal/authscope/principal.go b/internal/authscope/principal.go index 4ae7e99a6..9d29f2519 100644 --- a/internal/authscope/principal.go +++ b/internal/authscope/principal.go @@ -8,18 +8,12 @@ import ( "github.com/calypr/loom/internal/projectid" ) -// Transport-neutral authorization conditions. Callers classify these -// sentinels at the API boundary instead of matching driver/Fence messages. var ( ErrUnauthenticated = errors.New("authentication is required") ErrForbidden = errors.New("authorization denied") ErrAuthorizationBackendUnavailable = errors.New("authorization backend unavailable") ) -// Permission is the method recorded by Fence for a resource grant. -// Values are intentionally normalized at the authorization boundary so Loom -// accepts both the lowercase values emitted by current Gen3 services and -// uppercase values used by some deployments. type Permission string const ( diff --git a/internal/authscope/probe_double_normalize_test.go b/internal/authscope/probe_double_normalize_test.go index 7bdbf2351..828f7dac5 100644 --- a/internal/authscope/probe_double_normalize_test.go +++ b/internal/authscope/probe_double_normalize_test.go @@ -9,14 +9,6 @@ import ( "github.com/calypr/loom/internal/catalog" ) -// probeIdempotentNormalizations enumerates every normalization applied to the -// persisted declared paths and proves each one is idempotent. CoversExecution- -// Paths feeds a declaration through normalizeDeclaredExecutionPaths and then -// normalizeAuthResourcePathList, and separately passes the caller grant list -// through normalizeAuthResourcePathList, so "a second pass drops an entry the -// first pass kept" would let a declaration silently lose a path and be covered -// by a narrower grant. Idempotence of every pass rules that class of bug out: -// normalizing an already-normalized value is a fixed point. func TestProbeDeclaredPathNormalizationIsIdempotent(t *testing.T) { cases := []struct { name string @@ -52,11 +44,6 @@ func TestProbeDeclaredPathNormalizationIsIdempotent(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { once, malformedOnce := normalizeDeclaredExecutionPaths(tc.paths) - // A refused declaration never re-enters the pipeline (the gate - // denies on the spot), so re-feeding a malformed first-pass *output* - // is not a system flow. The real invariants are: the refusal is - // deterministic on the original declaration, and a kept output is a - // fixed point. if _, again := normalizeDeclaredExecutionPaths(tc.paths); again != malformedOnce { t.Fatalf("declared-path refusal is not deterministic for %#v", tc.paths) } @@ -75,8 +62,6 @@ func TestProbeDeclaredPathNormalizationIsIdempotent(t *testing.T) { } } - // The list normalizer is idempotent too (it is the second pass on - // kept entries, and the sole pass on grants). listOnce := normalizeAuthResourcePathList(tc.paths) listTwice := normalizeAuthResourcePathList(listOnce) if len(listOnce) != len(listTwice) { @@ -88,9 +73,6 @@ func TestProbeDeclaredPathNormalizationIsIdempotent(t *testing.T) { } } - // Each surviving declared path is itself a fixed point of the - // single-value normalizer, i.e. the declaration-side gate and the - // grant-side normalizer agree on identical spellings. for _, path := range once { if again := NormalizeAuthResourcePath(path); again != path { t.Fatalf("declared path %q is not a fixed point: re-normalized to %q", path, again) @@ -100,10 +82,6 @@ func TestProbeDeclaredPathNormalizationIsIdempotent(t *testing.T) { } } -// TestProbeIdempotencePreservesCoverageDecisions locks the security-relevant -// consequence: because normalization is idempotent, re-passing both sides of -// the coverage gate cannot turn a denial into a coverage (no entry can be -// "dropped by the second pass" to make a declaration look covered). func TestProbeIdempotencePreservesCoverageDecisions(t *testing.T) { cases := []struct { name string @@ -155,9 +133,6 @@ func TestProbeIdempotencePreservesCoverageDecisions(t *testing.T) { if got := tc.scope.CoversExecutionPaths(tc.declared); got != tc.wantCovers { t.Fatalf("CoversExecutionPaths(%q) = %v, want %v", tc.declared, got, tc.wantCovers) } - // The decision must be stable when the declaration is re-rendered - // through the same normalization the gate itself uses: a second - // pass can neither smuggle a denial into coverage nor the reverse. reparsed, malformed := normalizeDeclaredExecutionPaths(tc.declared) if malformed { reparsed = append([]string(nil), tc.declared...) @@ -165,8 +140,6 @@ func TestProbeIdempotencePreservesCoverageDecisions(t *testing.T) { if got := tc.scope.CoversExecutionPaths(reparsed); got != tc.wantCovers { t.Fatalf("CoversExecutionPaths on re-normalized declaration %q = %v, want %v", reparsed, got, tc.wantCovers) } - // Same again through the grant-side normalizer, proving neither - // normalization order can flip a coverage decision. listPass := normalizeAuthResourcePathList(tc.declared) if tc.wantCovers { if got := tc.scope.CoversExecutionPaths(listPass); !got { @@ -177,24 +150,6 @@ func TestProbeIdempotencePreservesCoverageDecisions(t *testing.T) { } } -// TestProbeUnicodeWhitespaceAndC1GapCharacterization characterizes the two -// unicode corners of the fails-closed rule and proves neither is a disclosure -// path. -// -// Corner 1 (unicode whitespace): strings.TrimSpace implements unicode.IsSpace, -// so NBSP/NEL/U+2000-series/U+3000 padding is dropped as "absent" on BOTH the -// declaration side and the grant side (both route through the same helpers). -// Because the trimming is symmetric, a declaration padded with a unicode space -// and a grant padded with the same (or ASCII) space resolve to the identical -// key — coverage can neither leak nor over-deny from padding asymmetry. -// -// Corner 2 (C1 controls, e.g. U+0086): not unicode whitespace, and no byte is -// <0x20 or ==0x7f, so NormalizeAuthResourcePath keeps such an entry verbatim -// instead of refusing it — the "gap" versus the C0/DEL rule. The gap is safe -// precisely because the entry is KEPT, not dropped: covering it requires a -// grant holding the byte-identical path, so the default is denial. Disclosure -// would need the entry to be silently dropped, which only happens for blanks -// (legitimate) and C0/DEL-or-empty normalizations (now refused). func TestProbeUnicodeWhitespaceAndC1GapCharacterization(t *testing.T) { restricted := func(paths ...string) ReadScope { return ReadScope{Mode: ReadScopeRestricted, AuthResourcePaths: paths} @@ -210,8 +165,6 @@ func TestProbeUnicodeWhitespaceAndC1GapCharacterization(t *testing.T) { if !restricted(pad + "p1" + pad).CoversExecutionPaths([]string{"p1"}) { t.Fatalf("padding %q: padded grant must cover the clean declaration (symmetric trim)", pad) } - // A padded padding-only declaration stays the blank policy: no - // non-blank path -> not a resolved authorization. if restricted("p1").CoversExecutionPaths([]string{pad}) { t.Fatalf("padding %q alone must stay blank policy and deny", pad) } @@ -221,18 +174,12 @@ func TestProbeUnicodeWhitespaceAndC1GapCharacterization(t *testing.T) { t.Run("c1-controls-are-kept-never-dropped", func(t *testing.T) { for _, marker := range []string{"\u0080", "\u0086", "\u009f", "\u200b", "\u2060"} { sneaky := "p1" + marker + "secret" - // Kept verbatim: the normalizer preserves the C1 bytes rather than - // dropping the entry. if got := NormalizeAuthResourcePath(sneaky); got != sneaky { t.Fatalf("entry %q must survive normalization verbatim, got %q", sneaky, got) } - // Denial is the default: a grant of the clean prefix does not cover - // the C1-bearing declaration. if restricted("p1").CoversExecutionPaths([]string{"p1", sneaky}) { t.Fatalf("DISCLOSURE: clean grants covered C1-bearing declaration %q", sneaky) } - // And a mixed declaration stays denied even when every other entry - // is granted. if restricted("p1", "p2").CoversExecutionPaths([]string{"p1", sneaky, "p2"}) { t.Fatalf("DISCLOSURE: grants covered mixed declaration carrying %q", sneaky) } @@ -240,15 +187,6 @@ func TestProbeUnicodeWhitespaceAndC1GapCharacterization(t *testing.T) { }) } -// TestProbeUnrestrictedCallerStillTraversesCoverageGate pins the mechanism -// behind the R4 reader verdict for callers that hold no scope grants at all -// (restricted == false in resolveCallerPaths). Such a caller is not bypassed: -// a non-empty declared list still yields a Restricted scope carrying the -// normalized declaration, so the reader's restricted-coverage gate runs and -// CoversExecutionPaths decides. A malformed declaration therefore stays -// unreadable for everyone, and the truncated normalized list can never be -// laundered into a row filter for it. A blank-only declaration keeps the -// legitimate Unrestricted mode. func TestProbeUnrestrictedCallerStillTraversesCoverageGate(t *testing.T) { newResolver := func() *ScopeResolver { return NewScopeResolver(ScopeResolverConfig{ @@ -257,7 +195,6 @@ func TestProbeUnrestrictedCallerStillTraversesCoverageGate(t *testing.T) { }, }) } - // No grants and no Fence client: resolveCallerPaths reports restricted=false. unrestricted := &Principal{Subject: "root", Projects: []string{"P"}} t.Run("malformed-declaration-routes-through-the-gate", func(t *testing.T) { diff --git a/internal/authscope/scope_cache_key_namespace_test.go b/internal/authscope/scope_cache_key_namespace_test.go index eef8c5ddd..ab612fe92 100644 --- a/internal/authscope/scope_cache_key_namespace_test.go +++ b/internal/authscope/scope_cache_key_namespace_test.go @@ -9,14 +9,6 @@ import ( "github.com/calypr/loom/internal/catalog" ) -// Key-namespace durable controls for the F-B27 project-write-binding cache -// (REPAIR-15). The cache key carries an explicit all-generations -// discriminator, so a generation-scoped entry can never answer an -// all-generations lookup and vice versa, and the reserved-literal generation -// string is just one more ordinary generation key. - -// fb27ctlSeed writes a cache entry directly so a control can prove which -// namespace a decision came from by the value it returns. func fb27ctlSeed(r *ScopeResolver, project, datasetGeneration string, allGenerations bool, paths ...string) { r.mu.Lock() defer r.mu.Unlock() @@ -132,8 +124,6 @@ func TestFB27LiteralSentinelGenerationIsAnOrdinaryKey(t *testing.T) { t.Fatalf("VERDICT=UNEXPECTED the literal must reach the catalog as a plain generation, got %+v", got) } - // The all-generations entry is untouched by the literal-keyed read entry: - // it needs its own listing and returns the project union. allGenerations, err := resolver.listExistingPathsAllGenerations(ctx, "projA") if err != nil { t.Fatalf("VERDICT=UNEXPECTED all-generations listing = %v", err) @@ -180,8 +170,6 @@ func TestFB27AuthoritativeProjectBindingProducerUsesModeTrueOnly(t *testing.T) { t.Fatalf("VERDICT=UNEXPECTED warm write must reuse the mode=true entry, calls=%d", l.calls) } - // The same project and empty generation must still be a distinct - // (mode,false) lookup: it re-lists and intersects against that result. scope, err := resolver.ResolveReadScopeForGeneration(ctx, principal, "HTAN_INT/cohort", "", nil) if err != nil { t.Fatalf("VERDICT=UNEXPECTED empty-generation read scope = %v", err) diff --git a/internal/authscope/scope_resolver.go b/internal/authscope/scope_resolver.go index c1658572b..793df38e4 100644 --- a/internal/authscope/scope_resolver.go +++ b/internal/authscope/scope_resolver.go @@ -30,13 +30,6 @@ type ScopeResolver struct { cache map[scopeCacheKey]cachedPaths } -// ReadScopeMode records whether an effective read scope may bypass the -// auth_resource_path predicate. It is deliberately separate from the path -// list: a restricted caller can legitimately resolve to zero paths, and that -// must never be confused with an unrestricted caller. -// -// The empty value is reserved for legacy callers that carry only a path list. -// Those callers retain the historical rule that an empty list is unrestricted. type ReadScopeMode string const ( @@ -44,29 +37,15 @@ const ( ReadScopeRestricted ReadScopeMode = "restricted" ) -// ReadScope is the authorization result that downstream catalog and dataframe -// code must propagate together. AuthResourcePaths may be empty in either mode; -// Mode is therefore authoritative for the AQL bypass bind variable. type ReadScope struct { AuthResourcePaths []string Mode ReadScopeMode } -// Unrestricted reports whether this scope may bypass auth_resource_path -// filtering. Unknown modes are treated as restricted so an invalid internal -// value cannot widen access. func (s ReadScope) Unrestricted() bool { return s.Mode == ReadScopeUnrestricted } -// CoversExecutionPaths reports whether this restricted read scope authorizes -// every non-blank auth resource path an execution declares. A scope that -// declares no paths, or an execution that declares only blank paths, is not a -// resolved authorization for that execution. A declaration holding a malformed -// non-blank entry is likewise not a resolvable authorization: see -// normalizeDeclaredExecutionPaths. The two execution-status twins use this -// single set rule, instead of a path-count comparison, as their defense-in-depth -// gate against resolver drift. func (s ReadScope) CoversExecutionPaths(executionAuthResourcePaths []string) bool { if s.Mode != ReadScopeRestricted { return false @@ -87,22 +66,6 @@ func (s ReadScope) CoversExecutionPaths(executionAuthResourcePaths []string) boo return true } -// normalizeDeclaredExecutionPaths renders the auth resource paths an execution -// declares for the coverage gate. -// -// Whitespace-only entries keep their long-standing meaning of "absent" and are -// dropped, so a declaration that carries blank padding alongside real paths -// still resolves. A non-blank entry that NormalizeAuthResourcePath refuses -// (a forbidden C0/DEL control byte, or bytes that normalize away to nothing, -// e.g. "/") is different in kind: it is a malformed persisted declaration, not -// an absent one. Dropping it would let the surviving paths cover a declaration -// that asserts a path the system can never name or grant, so the whole -// declaration is refused (the boolean result) and the caller conceals. The -// malformed bytes are never echoed, only the refusal. -// -// Incoming caller/grant normalization deliberately stays on the other helper -// (normalizeAuthResourcePathList); this stricter rule applies to the persisted -// declaration a read is being authorized against. func normalizeDeclaredExecutionPaths(paths []string) ([]string, bool) { kept := make([]string, 0, len(paths)) for _, path := range paths { @@ -118,8 +81,6 @@ func normalizeDeclaredExecutionPaths(paths []string) ([]string, bool) { return normalizeAuthResourcePathList(kept), false } -// Clone returns an independent copy suitable for passing between request -// layers without sharing a resolver-owned path slice. func (s ReadScope) Clone() ReadScope { return ReadScope{ AuthResourcePaths: cloneStrings(s.AuthResourcePaths), @@ -132,28 +93,10 @@ type cachedPaths struct { expiresAt time.Time } -// scopeCacheKey keeps authorization-path discovery isolated between immutable -// dataset generations. A project can legitimately have a different set of -// populated auth_resource_path values after a reload, so caching by project -// alone would otherwise leak stale paths into a new generation's catalog and -// dataframe queries. type scopeCacheKey struct { project string datasetGeneration string - // allGenerations separates the single project-level entry (the - // auth_resource_path union that AuthorizeWrite uses to bind a write to its - // target project, F-B27) from the per-generation entries that read and - // write scope resolution use. The discriminator is part of the key type, so - // the two namespaces cannot overlap: no datasetGeneration value, and in - // particular not the "\x00all-generations" literal that the first F-B27 fix - // reserved inside this keyspace, can address the project-level entry. That - // matters because a caller chooses datasetGeneration: read-path generations - // arrive on request payloads and catalog.NormalizeDatasetGeneration only - // TrimSpace's them (the charset gate, publish's validateOpaqueIdentifier, - // runs on the write side only). Such a generation is therefore just an - // ordinary allGenerations=false key that the project-write-binding lookup - // never consults. - allGenerations bool + allGenerations bool } type BearerTokenAuthenticator struct{} @@ -194,19 +137,10 @@ func (a ScopeAuthorizer) AuthorizeWrite(ctx context.Context, principal *Principa return a.Resolver.AuthorizeWrite(ctx, principal, project, authResourcePath) } -// ResolveReadScope returns the effective read authorization mode and paths. -// In particular, it preserves a restricted-empty intersection instead of -// encoding it as an empty slice that a later AQL layer could mistake for an -// unrestricted scope. func (r *ScopeResolver) ResolveReadScope(ctx context.Context, principal *Principal, project string, requested []string) (ReadScope, error) { return r.ResolveReadScopeForGeneration(ctx, principal, project, "", requested) } -// ResolveReadScopeForGeneration resolves an authorization scope against the -// populated auth-resource paths for exactly one dataset generation. An empty -// generation preserves the legacy null-generation namespace; it never means -// every generation. Callers that have selected an active manifest must use -// this method so the scope cache and catalog reads remain generation-aligned. func (r *ScopeResolver) ResolveReadScopeForGeneration(ctx context.Context, principal *Principal, project, datasetGeneration string, requested []string) (ReadScope, error) { callerPaths, restricted, err := r.resolveCallerPaths(ctx, principal, PermissionRead, "*") if err != nil { @@ -217,9 +151,6 @@ func (r *ScopeResolver) ResolveReadScopeForGeneration(ctx context.Context, princ if len(normalizedRequested) == 0 { return ReadScope{Mode: ReadScopeUnrestricted}, nil } - // An unrestricted caller can still deliberately narrow a read to a - // requested subset. That subset is a real AQL constraint, not an - // authorization bypass. return ReadScope{AuthResourcePaths: normalizedRequested, Mode: ReadScopeRestricted}, nil } @@ -266,12 +197,6 @@ func (r *ScopeResolver) AuthorizeWrite(ctx context.Context, principal *Principal normalized := NormalizeAuthResourcePath(authResourcePath) for _, path := range callerPaths { if path == normalized { - // F-B27: path membership alone lets a principal holding one - // project's path publish/activate into any other project, - // because `project` never constrains this decision. Bind the - // grant to the target project: the path must be one the project - // itself has recorded. A project with no catalog paths yet keeps - // first-generation bootstrap (no evidence to compare against). projectPaths, err := r.listExistingPathsAllGenerations(ctx, project) if err != nil { return err @@ -290,13 +215,6 @@ func (r *ScopeResolver) AuthorizeWrite(ctx context.Context, principal *Principal return fmt.Errorf("%w: auth_resource_path %q is outside caller scope for project %q", ErrForbidden, normalized, project) } -// AuthorizeReadProject verifies project-level read access without attempting -// to resolve a dataset-generation scope. Explorer lifecycle metadata is -// stored independently of a dataframe generation, so using -// ResolveReadScope here would intersect the caller's Fence grants with the -// empty-generation catalog namespace and incorrectly reject valid users. -// Generation-specific catalog and dataframe callers must continue using -// ResolveReadScopeForGeneration. func (r *ScopeResolver) AuthorizeReadProject(ctx context.Context, principal *Principal, project string) error { callerPaths, restricted, err := r.resolveCallerPaths(ctx, principal, PermissionRead, "*") if err != nil { @@ -314,10 +232,6 @@ func (r *ScopeResolver) AuthorizeReadProject(ctx context.Context, principal *Pri return fmt.Errorf("%w: caller has no read access to project %q", ErrForbidden, project) } -// ResolveWriteScopeForGeneration proves that the caller may publish or mutate -// data for the selected project. It mirrors read-scope resolution but asks -// Fence for write permission and never treats an empty intersection as an -// unrestricted result. func (r *ScopeResolver) ResolveWriteScopeForGeneration(ctx context.Context, principal *Principal, project, datasetGeneration string, requested []string) (ReadScope, error) { callerPaths, restricted, err := r.resolveCallerPaths(ctx, principal, PermissionWrite, "*") if err != nil { @@ -394,9 +308,6 @@ func (r *ScopeResolver) listExistingPaths(ctx context.Context, project, datasetG if project == "" { return []string{}, nil } - // A generation-scoped lookup is the false half of the discriminator, so it - // can never be answered from (nor overwrite) the project-write-binding - // entry, whatever bytes the caller supplied as its generation. key := scopeCacheKey{project: project, datasetGeneration: datasetGeneration, allGenerations: false} now := time.Now() r.mu.RLock() @@ -425,16 +336,6 @@ func (r *ScopeResolver) listExistingPaths(ctx context.Context, project, datasetG return paths, nil } -// listExistingPathsAllGenerations returns the project's recorded auth paths -// across every dataset generation for project-level write authorization -// (F-B27). It is the only writer of an allGenerations=true cache key, whose -// generation half stays empty because the listing is not scoped to one -// generation. The discriminator, not a reserved generation string, is what -// keeps this entry unaddressable from the caller-supplied generation keyspace -// (see scopeCacheKey), so the collision class that let a restricted caller -// poison the project binding is eliminated structurally. It fails closed like -// the generation-scoped path so an unavailable catalog cannot silently -// downgrade the project binding to the vulnerable pre-repair posture. func (r *ScopeResolver) listExistingPathsAllGenerations(ctx context.Context, project string) ([]string, error) { project = projectid.Canonical(project) if project == "" { @@ -485,9 +386,6 @@ func (r *ScopeResolver) InvalidateProject(project string) { r.mu.Unlock() } -// InvalidateGeneration removes a single project/generation auth-path entry. -// It is useful after a generation-specific catalog rebuild without evicting -// the active scope cache for every other immutable generation in the project. func (r *ScopeResolver) InvalidateGeneration(project, datasetGeneration string) { key := scopeCacheKey{ project: projectid.Canonical(project), diff --git a/internal/authscope/scope_test.go b/internal/authscope/scope_test.go index d34993cb9..84505073e 100644 --- a/internal/authscope/scope_test.go +++ b/internal/authscope/scope_test.go @@ -87,9 +87,6 @@ func TestScopeResolverResolveReadScopeKeepsRestrictedEmptyIntersection(t *testin resources: []string{"/programs/EllrottLab/projects/GDC_Data"}, }, ListExistingAuthResourcePaths: func(context.Context, catalog.AuthResourcePathOptions) ([]string, error) { - // The caller has access to GDC_Data, but this project currently has - // no matching catalog scope. This must be a deny-all scope, not a - // no-filter scope. return []string{"Another-Project"}, nil }, }) @@ -146,8 +143,6 @@ func TestScopeResolverCachesExistingPathsPerNormalizedGeneration(t *testing.T) { t.Fatalf("generation-b scope = %#v, want restricted empty", second) } - // The normalized generation-a lookup must use its own cache entry instead - // of reusing generation-b's empty intersection. again, err := resolver.ResolveReadScopeForGeneration(context.Background(), principal, "P1", "generation-a", nil) if err != nil { t.Fatalf("cached ResolveReadScopeForGeneration(generation-a) error = %v", err) diff --git a/internal/backends/arango/aliases.go b/internal/backends/arango/aliases.go new file mode 100644 index 000000000..659ff130a --- /dev/null +++ b/internal/backends/arango/aliases.go @@ -0,0 +1,121 @@ +package arango + +import ( + catalogadapter "github.com/calypr/loom/internal/backends/arango/internal/catalog" + chartifactadapter "github.com/calypr/loom/internal/backends/arango/internal/chartifact" + datasetadapter "github.com/calypr/loom/internal/backends/arango/internal/dataset" + exploreradapter "github.com/calypr/loom/internal/backends/arango/internal/explorer" + publicationadapter "github.com/calypr/loom/internal/backends/arango/internal/publication" + recipeadapter "github.com/calypr/loom/internal/backends/arango/internal/recipeexec" + store "github.com/calypr/loom/internal/backends/arango/internal/store" +) + +type ( + ConnectionOptions = Config + + Client = store.Client + RowVisitor = store.RowVisitor + BootstrapSpec = store.BootstrapSpec + CollectionSpec = store.CollectionSpec + TransactionCollections = store.TransactionCollections + TransactionFunc = store.TransactionFunc + PreviewCoveringIndex = store.PreviewCoveringIndex + + ExplainRequest = store.ExplainRequest + ExplainOptions = store.ExplainOptions + ExplainResult = store.ExplainResult + ExplainPlan = store.ExplainPlan + ExplainNode = store.ExplainNode + ExplainIndexes = store.ExplainIndexes + ExplainIndex = store.ExplainIndex + ExplainCollection = store.ExplainCollection + ExplainWarning = store.ExplainWarning + ExplainStats = store.ExplainStats + ExplainIndexUse = store.ExplainIndexUse + ExplainAssessment = store.ExplainAssessment + ExplainPlanEstimate = store.ExplainPlanEstimate + ExplainCollectionScan = store.ExplainCollectionScan + ExplainIndexSummary = store.ExplainIndexSummary + ExplainIndexLocation = store.ExplainIndexLocation + + ProfileRequest = store.ProfileRequest + ProfileOptions = store.ProfileOptions + ProfileResult = store.ProfileResult + ProfileExtra = store.ProfileExtra + ProfileStats = store.ProfileStats + ProfileNode = store.ProfileNode + ProfilePhases = store.ProfilePhases + ProfileNodeSummary = store.ProfileNodeSummary + ProfileSummary = store.ProfileSummary + ProfileNodeGroup = store.ProfileNodeGroup + + CatalogStore = catalogadapter.Store + DatasetStore = datasetadapter.Store + ExplorerStore = exploreradapter.Store + ExplorerCapabilitySnapshotStore = exploreradapter.CapabilitySnapshotStore + ExplorerTableShapeCapabilityRepository = exploreradapter.TableShapeCapabilityRepository + RecipeRegistry = recipeadapter.Registry + RecipeRevisionRegistry = recipeadapter.RevisionRegistry + PublishedRegistry = publicationadapter.Registry + ArtifactCatalog = chartifactadapter.Catalog +) + +var ( + WithQueryMemoryLimit = store.WithQueryMemoryLimit + QueryMemoryLimitFromContext = store.QueryMemoryLimitFromContext + DiagnosticCause = store.DiagnosticCause + TranslateQueryError = store.TranslateQueryError + DatabaseExists = store.DatabaseExists + ErrClientClosed = store.ErrClientClosed + IsArangoErrorResponse = store.IsArangoErrorResponse + IsArangoPermissionResponse = store.IsArangoPermissionResponse + AssessExplainResult = store.AssessExplainResult + SummarizeProfile = store.SummarizeProfile + + NewCatalogStore = catalogadapter.New + NewDatasetStore = datasetadapter.New + NewExplorerStore = exploreradapter.New + NewCapabilitySnapshotStore = exploreradapter.NewCapabilitySnapshotStore + NewTableShapeCapabilityRepository = exploreradapter.NewTableShapeCapabilityRepository + NewRecipeRegistry = recipeadapter.New + NewRevisionRegistry = recipeadapter.NewRevisionRegistry + NewPublishedRegistry = publicationadapter.New + NewArtifactCatalog = chartifactadapter.NewCatalog + + DatasetBootstrapSpec = datasetadapter.BootstrapSpec + ExplorerBootstrapSpec = exploreradapter.BootstrapSpec + RecipeBootstrapSpec = recipeadapter.BootstrapSpec + RevisionBootstrapSpec = recipeadapter.RevisionBootstrapSpec + PublishedBootstrapSpec = publicationadapter.BootstrapSpec + ArtifactBootstrapSpec = chartifactadapter.BootstrapSpec + + DatasetActiveDocumentKey = datasetadapter.ActiveDocumentKey + DatasetCollectionSpecs = datasetadapter.CollectionSpecs + DatasetCompareAndSwapActivate = datasetadapter.CompareAndSwapActivateRelease + DatasetCompareAndSwapPublish = datasetadapter.CompareAndSwapPublishRelease + ErrManifestAlreadyExists = datasetadapter.ErrManifestAlreadyExists + + ErrAllGenerationsWithGeneration = catalogadapter.ErrAllGenerationsWithGeneration + ErrSemanticInventoryClaimed = catalogadapter.ErrSemanticInventoryClaimed + ErrSemanticInventoryClaimLost = catalogadapter.ErrSemanticInventoryClaimLost +) + +const ( + DatasetLifecycleCollection = datasetadapter.LifecycleCollection + CatalogSemanticCoverageCollection = catalogadapter.SemanticCoverageCollection + ExplorersCollection = exploreradapter.ExplorersCollection + DraftRevisionsCollection = exploreradapter.DraftRevisionsCollection + RevisionsCollection = exploreradapter.RevisionsCollection + CompilationReceiptsCollection = exploreradapter.CompilationReceiptsCollection + SelectionsCollection = exploreradapter.SelectionsCollection + SelectionMembersCollection = exploreradapter.SelectionMembersCollection + TableShapeCapabilitiesCollection = exploreradapter.TableShapeCapabilitiesCollection + CapabilitySnapshotCollection = exploreradapter.CapabilitySnapshotCollection + RecipeCollection = recipeadapter.Collection + RecipeRevisionCollection = recipeadapter.RevisionCollection + BundleExecutionsCollection = publicationadapter.BundleExecutionsCollection + BundlePointersCollection = publicationadapter.BundlePointersCollection + BundleLeasesCollection = publicationadapter.BundleLeasesCollection + ExecutionReadPinsCollection = publicationadapter.ExecutionReadPinsCollection + PrivateArtifactsCollection = chartifactadapter.PrivateArtifactsCollection +) diff --git a/internal/backends/arango/doc.go b/internal/backends/arango/doc.go new file mode 100644 index 000000000..8ec2fcae5 --- /dev/null +++ b/internal/backends/arango/doc.go @@ -0,0 +1 @@ +package arango diff --git a/internal/backends/arango/internal/catalog/all_generations_auth_path_conflict_test.go b/internal/backends/arango/internal/catalog/all_generations_auth_path_conflict_test.go new file mode 100644 index 000000000..c421871f9 --- /dev/null +++ b/internal/backends/arango/internal/catalog/all_generations_auth_path_conflict_test.go @@ -0,0 +1,107 @@ +package arango + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + store "github.com/calypr/loom/internal/backends/arango/internal/store" + "github.com/calypr/loom/internal/catalog" +) + +type fb27ctlArangoClient struct { + queries []string + vars []map[string]any +} + +func (c *fb27ctlArangoClient) QueryRows(_ context.Context, query string, _ int, vars map[string]any, _ store.RowVisitor) error { + c.queries = append(c.queries, query) + copied := make(map[string]any, len(vars)) + for k, v := range vars { + copied[k] = v + } + c.vars = append(c.vars, copied) + return nil +} + +func (c *fb27ctlArangoClient) InsertBatchRaw(context.Context, string, []json.RawMessage, bool, string) error { + panic("unused by the auth path discovery controls") +} + +func (c *fb27ctlArangoClient) ExecuteAQL(context.Context, string, map[string]any) error { + panic("unused by the auth path discovery controls") +} + +func (c *fb27ctlArangoClient) CollectionExists(context.Context, string) (bool, error) { + panic("unused by the auth path discovery controls") +} + +func (c *fb27ctlArangoClient) Bootstrap(context.Context, store.BootstrapSpec) error { + panic("unused by the auth path discovery controls") +} + +func fb27ctlDiscover(t *testing.T, opts catalog.AuthResourcePathOptions) ([]string, error) { + t.Helper() + client := &fb27ctlArangoClient{} + s, err := New(client) + if err != nil { + t.Fatalf("VERDICT=UNEXPECTED store construction = %v", err) + } + paths, err := s.DiscoverExistingAuthResourcePaths(context.Background(), opts) + if err != nil { + if len(client.queries) != 0 { + t.Fatalf("VERDICT=UNEXPECTED rejected request reached Arango (%d queries) for %+v: %v", len(client.queries), opts, err) + } + return paths, err + } + if len(client.queries) != 1 { + t.Fatalf("VERDICT=UNEXPECTED accepted request issued %d queries for %+v, want 1", len(client.queries), opts) + } + if paths == nil { + t.Fatalf("VERDICT=UNEXPECTED accepted request returned nil paths for %+v", opts) + } + return paths, nil +} + +func TestFB27AllGenerationsWithSelectedGenerationIsRejected(t *testing.T) { + for _, generation := range []string{"gen-1", " gen-1 ", "sha256:0000000000000000000000000000000000000000000000000000000000000000"} { + _, err := fb27ctlDiscover(t, catalog.AuthResourcePathOptions{ + Project: "projA", + DatasetGeneration: generation, + AllGenerations: true, + }) + if err == nil { + t.Fatalf("VERDICT=CONFIRMED (pre-fix archived shape): AllGenerations with datasetGeneration %q was accepted, so discovery silently ignored the selected generation", generation) + } + if !errors.Is(err, ErrAllGenerationsWithGeneration) { + t.Fatalf("VERDICT=UNEXPECTED conflict error must wrap ErrAllGenerationsWithGeneration, got %v", err) + } + for _, want := range []string{"AllGenerations", "datasetGeneration", generation} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("VERDICT=UNEXPECTED conflict error %q must name the conflicting option and its value %q", err, want) + } + } + } + t.Logf("VERDICT=REPAIRED AllGenerations plus a selected generation is refused before any query") +} + +func TestFB27SupportedAuthResourcePathOptionShapesStillDiscover(t *testing.T) { + cases := []struct { + name string + opts catalog.AuthResourcePathOptions + }{ + {"all generations with empty generation", catalog.AuthResourcePathOptions{Project: "projA", AllGenerations: true}}, + {"all generations with whitespace only generation", catalog.AuthResourcePathOptions{Project: "projA", DatasetGeneration: " ", AllGenerations: true}}, + {"generation scoped", catalog.AuthResourcePathOptions{Project: "projA", DatasetGeneration: " gen-1 "}}, + {"legacy null namespace", catalog.AuthResourcePathOptions{Project: "projA"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := fb27ctlDiscover(t, tc.opts); err != nil { + t.Fatalf("VERDICT=UNEXPECTED valid options %+v rejected: %v", tc.opts, err) + } + }) + } +} diff --git a/internal/catalog/arango/all_generations_project_binding_integration_test.go b/internal/backends/arango/internal/catalog/all_generations_project_binding_integration_test.go similarity index 78% rename from internal/catalog/arango/all_generations_project_binding_integration_test.go rename to internal/backends/arango/internal/catalog/all_generations_project_binding_integration_test.go index 6601b4d04..0b3fb7ed6 100644 --- a/internal/catalog/arango/all_generations_project_binding_integration_test.go +++ b/internal/backends/arango/internal/catalog/all_generations_project_binding_integration_test.go @@ -13,24 +13,11 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + storearango "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/catalog" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) -// Durable control for the F-B27 project-binding regression: the all-generations -// discovery query is a deliberate cross-generation union (the scoped query -// treats a null @dataset_generation as "row has no generation", not "any"), so -// DiscoverExistingAuthResourcePaths must not pass dataset_generation as a bind -// when AllGenerations sets the union AQL (which declares only @project). The -// undeclared bind made Arango reject every restricted-scope direct write to a -// project with recorded catalog paths, and authoringWriteDirect masked the -// storage fault as 403. A generation FILTER on the union query is likewise -// forbidden: under AllGenerations validation refuses a non-empty generation, a -// null bind would match only generation-less rows, an empty result would hit -// the first-generation bootstrap branch of AuthorizeWrite, and the F-B27 -// cross-project publish bypass would re-open. - const genBindTestDatabasePrefix = "loom_catalog_genbind_" func openGenBindHarness(t *testing.T, budget time.Duration) (context.Context, *storearango.Client) { @@ -97,9 +84,6 @@ func genBindDiscoveryErrorShape(t *testing.T, err error, label string) { if err == nil { return } - // The pre-fix shape: every project-binding lookup died on the undeclared - // bind. Any regression that resurrects passing dataset_generation to the - // union AQL must fail here rather than resurface as a masked 403 upstream. if strings.Contains(err.Error(), "was not declared in the query") { t.Fatalf("%s: undeclared bind parameter returned from Arango (pre-fix F-B27 regression): %v", label, err) } @@ -169,14 +153,8 @@ func TestAuthorizeWriteProjectBindingAgainstArangoRecordedCatalogPaths(t *testin emptyProject := "genbind-empty-" + uid alpha, beta := "gen-alpha-"+uid, "gen-beta-"+uid - // The bound project records its paths in separate generations: only the - // cross-generation union can authorize them, which is what keeps the - // fix's DROP-the-bind decision (vs declare-and-filter on a null bind) - // load-bearing under mutation. insertGenBindCatalogRow(t, ctx, client, "genbind-bound-a", boundProject, &alpha, "grants/recorded") insertGenBindCatalogRow(t, ctx, client, "genbind-bound-b", boundProject, &beta, "grants/recorded-cross-gen") - // The F-B27 witness: a path recorded by a DIFFERENT project that the same - // principal also holds. Path membership alone used to authorize this. insertGenBindCatalogRow(t, ctx, client, "genbind-other-x", otherProject, &alpha, "grants/foreign") resolver := authscope.NewScopeResolver(authscope.ScopeResolverConfig{ @@ -191,25 +169,17 @@ func TestAuthorizeWriteProjectBindingAgainstArangoRecordedCatalogPaths(t *testin } requestCtx := authscope.ContextWithPrincipal(ctx, principal) - // A restricted path that the target project itself recorded (one per - // generation): the union lookup must succeed, not 500/fail-closed-deny. if err := resolver.AuthorizeWrite(requestCtx, principal, boundProject, "grants/recorded"); err != nil { t.Fatalf("AuthorizeWrite recorded path (generation %s): %v", alpha, err) } if err := resolver.AuthorizeWrite(requestCtx, principal, boundProject, "grants/recorded-cross-gen"); err != nil { t.Fatalf("AuthorizeWrite recorded path held only in a different generation (union requirement): %v", err) } - // The F-B27 guarantee still holds: a path the caller holds but THIS - // project never recorded is refused for the right reason (ErrForbidden), - // never an Arango bind fault and never a silent allow. err = resolver.AuthorizeWrite(requestCtx, principal, boundProject, "grants/foreign") genBindDiscoveryErrorShape(t, err, "AuthorizeWrite foreign path") if !errors.Is(err, authscope.ErrForbidden) { t.Fatalf("AuthorizeWrite path recorded only by another project err=%v; want %v (F-B27 project binding)", err, authscope.ErrForbidden) } - // The first-generation bootstrap stays available for projects with no - // recorded catalog paths at all: the union lookup must return an empty - // list (clean allow via the zero-row branch), not a bind error. bootstrapErr := resolver.AuthorizeWrite(requestCtx, principal, emptyProject, "grants/unrecorded") genBindDiscoveryErrorShape(t, bootstrapErr, "AuthorizeWrite bootstrap project") if bootstrapErr != nil { diff --git a/internal/catalog/arango/codec.go b/internal/backends/arango/internal/catalog/codec.go similarity index 100% rename from internal/catalog/arango/codec.go rename to internal/backends/arango/internal/catalog/codec.go diff --git a/internal/backends/arango/internal/catalog/evidence.go b/internal/backends/arango/internal/catalog/evidence.go new file mode 100644 index 000000000..405929dbb --- /dev/null +++ b/internal/backends/arango/internal/catalog/evidence.go @@ -0,0 +1,59 @@ +package arango + +import ( + "context" + "sort" + + "github.com/calypr/loom/internal/catalog" + fhirschema "github.com/calypr/loom/internal/fhir/schema" +) + +func (s *Store) ReadResourceInventory(ctx context.Context, opts catalog.ResourceInventoryOptions) (catalog.ResourceInventoryResult, error) { + return s.DiscoverResourceInventory(ctx, opts) +} +func (s *Store) ReadRelationshipObservations(ctx context.Context, opts catalog.RelationshipObservationOptions) (catalog.RelationshipObservationResult, error) { + return s.DiscoverRelationshipObservations(ctx, opts) +} +func (s *Store) ReadFieldEnrichment(ctx context.Context, opts catalog.FieldEnrichmentOptions) (catalog.FieldEnrichmentResult, error) { + return s.DiscoverFieldEnrichment(ctx, opts) +} + +func evidenceResourceTypes(requested []string) ([]string, []catalog.EvidenceDiagnostic) { + if len(requested) == 0 { + return fhirschema.ResourceTypes(), nil + } + seen := make(map[string]struct{}, len(requested)) + diagnostics := make([]catalog.EvidenceDiagnostic, 0) + for _, requestedType := range requested { + canonical, ok := fhirschema.ConcreteResourceType(requestedType) + if !ok { + diagnostics = append(diagnostics, catalog.EvidenceDiagnostic{Code: "INVALID_RESOURCE_TYPE", Message: "requested resource type is not a concrete generated FHIR resource", ResourceType: requestedType}) + continue + } + seen[canonical] = struct{}{} + } + result := make([]string, 0, len(seen)) + for resourceType := range seen { + result = append(result, resourceType) + } + sort.Strings(result) + return result, diagnostics +} + +func inventoryObservationLess(a, b catalog.ResourceInventoryObservation) bool { + for _, pair := range [][2]string{{a.Project, b.Project}, {a.DatasetGeneration, b.DatasetGeneration}, {a.AuthResourcePath, b.AuthResourcePath}, {a.ResourceType, b.ResourceType}} { + if pair[0] != pair[1] { + return pair[0] < pair[1] + } + } + return a.DocumentCount < b.DocumentCount +} + +func fieldLess(a, b catalog.FieldEnrichmentObservation) bool { + for _, pair := range [][2]string{{a.Project, b.Project}, {a.DatasetGeneration, b.DatasetGeneration}, {a.AuthResourcePath, b.AuthResourcePath}, {a.ResourceType, b.ResourceType}, {a.Path, b.Path}, {a.Kind, b.Kind}} { + if pair[0] != pair[1] { + return pair[0] < pair[1] + } + } + return a.DocCount < b.DocCount +} diff --git a/internal/catalog/arango/fields.go b/internal/backends/arango/internal/catalog/fields.go similarity index 94% rename from internal/catalog/arango/fields.go rename to internal/backends/arango/internal/catalog/fields.go index 829eac870..dd9c1bcab 100644 --- a/internal/catalog/arango/fields.go +++ b/internal/backends/arango/internal/catalog/fields.go @@ -36,9 +36,6 @@ func (s *Store) DiscoverFields(ctx context.Context, opts catalog.PopulatedFieldO return rows, err } -// DiscoverFieldEnrichment reads profiler rows as optional, fail-closed -// enrichment. A successful empty result is valid evidence; a query failure or -// bounded distinct-value set is explicitly unavailable/incomplete. func (s *Store) DiscoverFieldEnrichment(ctx context.Context, opts catalog.FieldEnrichmentOptions) (catalog.FieldEnrichmentResult, error) { if opts.CursorBatch <= 0 { opts.CursorBatch = 1000 @@ -86,9 +83,6 @@ func (s *Store) DiscoverFieldEnrichment(ctx context.Context, opts catalog.FieldE } field.ResourceType = resourceType result.Values = append(result.Values, catalog.FieldEnrichmentObservation{Project: field.Project, DatasetGeneration: field.DatasetGeneration, AuthResourcePath: field.AuthResourcePath, ResourceType: field.ResourceType, Path: field.Path, Kind: field.Kind, DocCount: field.DocCount, MaxItems: field.MaxItems, SampleCount: field.SampleCount, DistinctValues: append([]string(nil), field.DistinctValues...), DistinctTruncated: field.DistinctTruncated, ExtensionValues: append([]catalog.ExtensionValueObservation(nil), field.ExtensionValues...), SemanticObservations: append([]catalog.SemanticObservation(nil), field.SemanticObservations...), PivotCandidate: field.PivotCandidate, PivotKind: field.PivotKind, PivotColumns: append([]string(nil), field.PivotColumns...), PivotFamily: field.PivotFamily, PivotColumnSelect: field.PivotColumnSelect, PivotValueSelect: field.PivotValueSelect, PivotItemSource: field.PivotItemSource, PivotItemResourceType: field.PivotItemResourceType, PivotValueSelectors: append([]string(nil), field.PivotValueSelectors...)}) - // DistinctTruncated is per-field suggestion metadata. It is retained - // on the observation but does not make the complete enrichment artifact - // unusable; only an artifact-level marker can fail the snapshot. if truncated, ok := row["enrichment_truncated"].(bool); ok && truncated { result.Truncated = true } @@ -106,7 +100,7 @@ func (s *Store) DiscoverFieldEnrichment(ctx context.Context, opts catalog.FieldE } sort.Slice(result.Values, func(i, j int) bool { return fieldLess(result.Values[i], result.Values[j]) }) result.Digest, _ = catalog.FieldEnrichmentDigest(result.Values, result.FieldCoverageKnown, result.Complete, result.Truncated) - result.Status = evidenceStatus(result.Available, result.Complete, result.Truncated, len(result.Values)) + result.Status = catalog.EvidenceStatusFor(result.Available, result.Complete, result.Truncated, len(result.Values)) return result, nil } diff --git a/internal/catalog/arango/relationships.go b/internal/backends/arango/internal/catalog/relationships.go similarity index 88% rename from internal/catalog/arango/relationships.go rename to internal/backends/arango/internal/catalog/relationships.go index b0e6e6e89..fba1f25b1 100644 --- a/internal/catalog/arango/relationships.go +++ b/internal/backends/arango/internal/catalog/relationships.go @@ -8,9 +8,9 @@ import ( "time" "github.com/bytedance/sonic" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/catalog" fhirschema "github.com/calypr/loom/internal/fhir/schema" - store "github.com/calypr/loom/internal/store/arango" ) func (s *Store) DiscoverReferences(ctx context.Context, opts catalog.PopulatedReferenceOptions) ([]catalog.PopulatedReference, error) { @@ -87,10 +87,6 @@ func (s *Store) AuditRelationshipEdges(ctx context.Context, opts catalog.Relatio return summary, nil } -// DiscoverRelationshipObservations returns both storage and Builder views of -// each aggregate. The catalog stores physical fhir_edge direction; Builder -// usually traverses its inverse, so dropping the storage provenance here is a -// correctness bug rather than a presentation detail. func (s *Store) DiscoverRelationshipObservations(ctx context.Context, opts catalog.RelationshipObservationOptions) (catalog.RelationshipObservationResult, error) { if opts.CursorBatch <= 0 { opts.CursorBatch = 1000 @@ -108,7 +104,7 @@ func (s *Store) DiscoverRelationshipObservations(ctx context.Context, opts catal } else { vars["resource_types"] = nil } - counts := make(map[relationshipObservationKey]int64) + counts := make(map[catalog.RelationshipEvidenceKey]int64) var queryErr error err := s.client.QueryRows(ctx, relationshipObservationsAQL, opts.CursorBatch, vars, func(row map[string]any) error { storageFrom := stringValue(row["storage_from_type"]) @@ -124,13 +120,12 @@ func (s *Store) DiscoverRelationshipObservations(ctx context.Context, opts catal result.Diagnostics = append(result.Diagnostics, catalog.EvidenceDiagnostic{Code: "INVALID_RELATIONSHIP_ENDPOINT", Message: "relationship endpoint is not a concrete generated FHIR resource", StorageFromType: storageFrom, Label: label, StorageToType: storageTo, Count: count}) return nil } - observation, ok, diagnostic := relationshipObservation(stringValue(row["project"]), stringValue(row["dataset_generation"]), stringValue(row["auth_resource_path"]), from, label, to, count) + observation, ok, diagnostic := catalog.ProjectRelationshipObservation(stringValue(row["project"]), stringValue(row["dataset_generation"]), stringValue(row["auth_resource_path"]), from, label, to, count) if !ok { result.Diagnostics = append(result.Diagnostics, diagnostic) return nil } - key := relationshipObservationKey{project: observation.Project, generation: observation.DatasetGeneration, auth: observation.AuthResourcePath, storageFrom: observation.StorageFromType, label: observation.Label, storageTo: observation.StorageToType, builderFrom: observation.BuilderFromType, builderTo: observation.BuilderToType, builderDirection: observation.BuilderDirection} - counts[key] += observation.EdgeCount + counts[catalog.RelationshipEvidenceKeyFor(observation)] += observation.EdgeCount return nil }) queryErr = err @@ -140,11 +135,11 @@ func (s *Store) DiscoverRelationshipObservations(ctx context.Context, opts catal return result, queryErr } for key, count := range counts { - result.Values = append(result.Values, catalog.RelationshipObservation{Project: key.project, DatasetGeneration: key.generation, AuthResourcePath: key.auth, FromType: key.builderFrom, Label: key.label, ToType: key.builderTo, EdgeCount: count, StorageFromType: key.storageFrom, StorageToType: key.storageTo, StorageDirection: "OUTBOUND", BuilderFromType: key.builderFrom, BuilderToType: key.builderTo, BuilderDirection: key.builderDirection}) + result.Values = append(result.Values, catalog.RelationshipObservationFromKey(key, count)) } - sort.Slice(result.Values, func(i, j int) bool { return relationshipLess(result.Values[i], result.Values[j]) }) + sort.Slice(result.Values, func(i, j int) bool { return catalog.RelationshipEvidenceLess(result.Values[i], result.Values[j]) }) result.Digest, _ = catalog.RelationshipObservationDigest(result.Values) - result.Status = evidenceStatus(result.Available, result.Complete, result.Truncated, len(result.Values)) + result.Status = catalog.EvidenceStatusFor(result.Available, result.Complete, result.Truncated, len(result.Values)) return result, nil } @@ -230,9 +225,6 @@ func (s *Store) RebuildRelationshipCatalog(ctx context.Context, opts catalog.Rel return catalog.RelationshipRebuildSummary{}, err } vars := map[string]any{"project": opts.Project, "dataset_generation": generation(opts.DatasetGeneration), "auth_resource_paths": opts.AuthResourcePaths, "auth_resource_paths_unrestricted": catalog.EffectiveAuthResourcePathsUnrestricted(opts.AuthResourcePaths, opts.AuthResourcePathsUnrestricted), "resource_types": fhirschema.ResourceTypes()} - // Write-only drain: the query's RETURN 1 rows are scalar, which the - // cursor row decoder rejects (map[string]any), so this must not go - // through QueryRows. if err := s.client.ExecuteAQL(ctx, relationshipClearAQL, vars); err != nil { return catalog.RelationshipRebuildSummary{}, err } diff --git a/internal/catalog/arango/resource_inventory.go b/internal/backends/arango/internal/catalog/resource_inventory.go similarity index 98% rename from internal/catalog/arango/resource_inventory.go rename to internal/backends/arango/internal/catalog/resource_inventory.go index 53b518524..bf2d6e123 100644 --- a/internal/catalog/arango/resource_inventory.go +++ b/internal/backends/arango/internal/catalog/resource_inventory.go @@ -6,8 +6,8 @@ import ( "errors" "fmt" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/catalog" - arangostore "github.com/calypr/loom/internal/store/arango" ) var ErrResourceInventoryBuildUnavailable = errors.New("resource inventory build is unavailable; run arango-fhir-proto rebuild-resource-inventory --generation ID") diff --git a/internal/catalog/arango/resource_inventory_integration_test.go b/internal/backends/arango/internal/catalog/resource_inventory_integration_test.go similarity index 98% rename from internal/catalog/arango/resource_inventory_integration_test.go rename to internal/backends/arango/internal/catalog/resource_inventory_integration_test.go index c7659f98f..df6622f19 100644 --- a/internal/catalog/arango/resource_inventory_integration_test.go +++ b/internal/backends/arango/internal/catalog/resource_inventory_integration_test.go @@ -12,8 +12,8 @@ import ( "testing" "time" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/catalog" - arangostore "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -80,8 +80,6 @@ func TestResourceInventoryPersistsAndScopesCountsAcrossStoreReopen(t *testing.T) t.Fatal(err) } defer client.Close(context.Background()) - // This proof owns its fixture prerequisites even on an empty test database. - // Do not truncate existing collections: fixture rows remain UUID-scoped. if err := client.Bootstrap(ctx, arangostore.BootstrapSpec{Collections: []arangostore.CollectionSpec{ {Name: "Observation"}, {Name: "Patient"}, }}); err != nil { diff --git a/internal/catalog/arango/resource_inventory_unit_test.go b/internal/backends/arango/internal/catalog/resource_inventory_unit_test.go similarity index 99% rename from internal/catalog/arango/resource_inventory_unit_test.go rename to internal/backends/arango/internal/catalog/resource_inventory_unit_test.go index e1fc035f2..4648d3ebf 100644 --- a/internal/catalog/arango/resource_inventory_unit_test.go +++ b/internal/backends/arango/internal/catalog/resource_inventory_unit_test.go @@ -8,8 +8,8 @@ import ( "strings" "testing" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/catalog" - store "github.com/calypr/loom/internal/store/arango" ) type resourceInventoryWrite struct { diff --git a/internal/catalog/arango/resources.go b/internal/backends/arango/internal/catalog/resources.go similarity index 79% rename from internal/catalog/arango/resources.go rename to internal/backends/arango/internal/catalog/resources.go index 862360d89..f2535cae2 100644 --- a/internal/catalog/arango/resources.go +++ b/internal/backends/arango/internal/catalog/resources.go @@ -10,10 +10,6 @@ import ( "github.com/calypr/loom/internal/catalog" ) -// ErrAllGenerationsWithGeneration reports the unsupported option combination -// of AllGenerations with a selected dataset generation. The all-generations -// discovery query ignores @dataset_generation, so accepting both would -// silently drop the caller's generation selection (F-2). var ErrAllGenerationsWithGeneration = errors.New("AllGenerations conflicts with a selected datasetGeneration") func (s *Store) DiscoverExistingAuthResourcePaths(ctx context.Context, opts catalog.AuthResourcePathOptions) ([]string, error) { @@ -27,13 +23,6 @@ func (s *Store) DiscoverExistingAuthResourcePaths(ctx context.Context, opts cata query := existingAuthResourcePathsAQL bindVars := map[string]any{"project": opts.Project, "dataset_generation": generation(opts.DatasetGeneration)} if opts.AllGenerations { - // The union query deliberately omits the dataset_generation filter (see - // existingAuthResourcePathsAllGenerationsAQL), so it declares only - // @project. Passing @dataset_generation here would make Arango reject - // the undeclared bind; declaring and using it instead would turn the - // union into a null filter (AllGenerations keeps the generation empty) - // and silently re-open the F-B27 cross-project bypass via the - // zero-path bootstrap branch. Drop the bind with the filter. query = existingAuthResourcePathsAllGenerationsAQL delete(bindVars, "dataset_generation") } @@ -50,9 +39,6 @@ func (s *Store) DiscoverExistingAuthResourcePaths(ctx context.Context, opts cata return result, nil } -// DiscoverResourceInventory reads the publish-last inventory projection. -// Field profiler rows are deliberately not consulted: a populated field is -// not evidence that a resource collection is a valid row root. func (s *Store) DiscoverResourceInventory(ctx context.Context, opts catalog.ResourceInventoryOptions) (catalog.ResourceInventoryResult, error) { if opts.CursorBatch <= 0 { opts.CursorBatch = 1000 @@ -110,7 +96,7 @@ func (s *Store) DiscoverResourceInventory(ctx context.Context, opts catalog.Reso authPathsUnrestricted := catalog.EffectiveAuthResourcePathsUnrestricted(opts.AuthResourcePaths, opts.AuthResourcePathsUnrestricted) if !authPathsUnrestricted && len(opts.AuthResourcePaths) == 0 { result.Digest, _ = catalog.ResourceInventoryDigest(result.Values) - result.Status = evidenceStatus(result.Available, result.Complete, result.Truncated, len(result.Values)) + result.Status = catalog.EvidenceStatusFor(result.Available, result.Complete, result.Truncated, len(result.Values)) return result, nil } counts := make(map[string]int64, len(resourceTypes)) @@ -151,7 +137,7 @@ func (s *Store) DiscoverResourceInventory(ctx context.Context, opts catalog.Reso return inventoryObservationLess(result.Values[i], result.Values[j]) }) result.Digest, _ = catalog.ResourceInventoryDigest(result.Values) - result.Status = evidenceStatus(result.Available, result.Complete, result.Truncated, len(result.Values)) + result.Status = catalog.EvidenceStatusFor(result.Available, result.Complete, result.Truncated, len(result.Values)) return result, nil } @@ -171,19 +157,6 @@ FOR d IN fhir_field_catalog SORT auth_resource_path RETURN { auth_resource_path: auth_resource_path }` -// The scoped filter treats a null @dataset_generation as "rows without a -// generation" (Arango == null matches missing), not "any generation". -// Project-level write authorization needs the union across generations, so it -// opts in via AuthResourcePathOptions.AllGenerations instead of changing the -// semantics of the existing generation-scoped query (F-B27). -// ValidateAuthResourcePathOptions rejects option combinations whose meaning -// the discovery queries cannot express. AllGenerations asks for the union over -// every dataset generation while the generation-scoped query binds -// @dataset_generation, so a non-empty generation alongside AllGenerations is -// contradictory and must be refused rather than silently ignored. Whitespace -// only generations follow the optional-field convention and remain valid, as -// the project-level write authorization caller (authscope) sends exactly that -// shape. func ValidateAuthResourcePathOptions(opts catalog.AuthResourcePathOptions) error { if opts.AllGenerations && catalog.NormalizeDatasetGeneration(opts.DatasetGeneration) != "" { return fmt.Errorf("%w: datasetGeneration %q must be empty when AllGenerations is set", ErrAllGenerationsWithGeneration, opts.DatasetGeneration) diff --git a/internal/catalog/arango/semantic_coded_category_integration_test.go b/internal/backends/arango/internal/catalog/semantic_coded_category_integration_test.go similarity index 96% rename from internal/catalog/arango/semantic_coded_category_integration_test.go rename to internal/backends/arango/internal/catalog/semantic_coded_category_integration_test.go index 3e1f005b6..db3dd59fb 100644 --- a/internal/catalog/arango/semantic_coded_category_integration_test.go +++ b/internal/backends/arango/internal/catalog/semantic_coded_category_integration_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/catalog" - arangostore "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -92,8 +92,6 @@ func TestSemanticInventoryPagesAndResolvesCodedCategoryFamilies(t *testing.T) { code := fmt.Sprintf("large-%03d", i) addCategoryCode("scope-a", code, "Large category "+code, largeSystem, largeVersion) } - // This code sorts after the ordinary members and is matched by code because - // its display is absent. It must still select the complete family. lateCodeOnlyWitness := "zz-late-code-only-witness" addCategoryCode("scope-a", lateCodeOnlyWitness, "", largeSystem, largeVersion) addCategoryCode("scope-b", "scope-b-private", "AAA scope-b private category", largeSystem, largeVersion) @@ -101,8 +99,6 @@ func TestSemanticInventoryPagesAndResolvesCodedCategoryFamilies(t *testing.T) { smallDisplays := []string{"Small category alpha", "Small category beta"} addCategoryCode("scope-a", "small-alpha", smallDisplays[0], "urn:c01:small", "v1") addCategoryCode("scope-a", "small-beta", smallDisplays[1], "urn:c01:small", "v1") - // Keep the fallback in a one-code family so its representative example is - // deterministic while still exercising indexed family aggregation. const fallbackSystem = "urn:c01:fallback" addCategoryCode("scope-a", "small-fallback-code", "", fallbackSystem, "v1") addCategoryCode("scope-a", "other-system-code", "Other system", "urn:c01:other", "v1") @@ -219,8 +215,6 @@ func TestSemanticInventoryPagesAndResolvesCodedCategoryFamilies(t *testing.T) { if resolvedSmall.FamilyID != smallRef.FamilyID || resolvedSmall.Population != 2 { t.Fatalf("resolved small category family = %+v; want both matching Coding rows", resolvedSmall) } - // Family DTOs intentionally carry one deterministic representative example; - // the population and descriptor cover the full system/version family. assertSemanticCategoryExamples(t, resolvedSmall, []string{smallDisplays[0]}) resolvedFallback, err := store.ResolveSemanticInventorySelections(ctx, catalog.SemanticInventoryResolveOptions{ Project: project, diff --git a/internal/catalog/arango/semantic_coverage.go b/internal/backends/arango/internal/catalog/semantic_coverage.go similarity index 90% rename from internal/catalog/arango/semantic_coverage.go rename to internal/backends/arango/internal/catalog/semantic_coverage.go index a5ce729a8..85ce8a7cb 100644 --- a/internal/catalog/arango/semantic_coverage.go +++ b/internal/backends/arango/internal/catalog/semantic_coverage.go @@ -12,8 +12,8 @@ import ( "strings" "sync" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/catalog" - store "github.com/calypr/loom/internal/store/arango" ) const ( @@ -41,58 +41,6 @@ var ( ErrSemanticCoverageResourceExpectation = errors.New("semantic coverage resource source-set expectation is invalid or incomplete") ) -type SemanticCoverageBuildState string - -const ( - SemanticCoverageBuildBuilding SemanticCoverageBuildState = "BUILDING" - SemanticCoverageBuildFailed SemanticCoverageBuildState = "FAILED" - SemanticCoverageBuildUnavailable SemanticCoverageBuildState = "UNAVAILABLE" - SemanticCoverageBuildComplete SemanticCoverageBuildState = "COMPLETE" - SemanticCoverageBuildPartial SemanticCoverageBuildState = "PARTIAL" - SemanticCoverageBuildMissing SemanticCoverageBuildState = "MISSING" -) - -type SemanticCoverageCandidateStatus string - -const ( - SemanticCoverageCandidateGroupable SemanticCoverageCandidateStatus = "GROUPABLE" - SemanticCoverageCandidateKeepRaw SemanticCoverageCandidateStatus = "KEEP_RAW" - SemanticCoverageCandidateUnavailable SemanticCoverageCandidateStatus = "UNAVAILABLE" -) - -type SemanticCoverageCandidatePath struct { - ResourceType string - CanonicalPath string -} - -type SemanticCoverageLookupOptions struct { - Project string - DatasetGeneration string - BuildID string - SchemaIdentity string - SourceKind string - AuthResourcePaths []string - AuthResourcePathsUnrestricted bool - Candidates []SemanticCoverageCandidatePath -} - -type SemanticCoverageCandidateResult struct { - ResourceType string - CanonicalPath string - CoverageID string - Status SemanticCoverageCandidateStatus - ResourceState SemanticCoverageBuildState - ResourceReason string - Reason catalog.SemanticCoverageReason - References []catalog.SemanticCoverageReference -} - -type SemanticCoverageLookupResult struct { - State SemanticCoverageBuildState - Reason string - Candidates []SemanticCoverageCandidateResult -} - type SemanticCoverageBuildIdentity struct { Project string DatasetGeneration string @@ -116,8 +64,6 @@ type SemanticCoverageSourcePartitionExpectation struct { SourceSetDigest string } -// SemanticCoverageSourceSetAccumulator keeps fixed-size count and digest -// state per authorization partition; it never retains source identifiers. type SemanticCoverageSourceSetAccumulator struct { mu sync.Mutex resourceType string @@ -235,9 +181,7 @@ func semanticCoverageResourceDigest(resourceType string, partitions []SemanticCo type SemanticCoverageWriteStats struct { Rows int64 SourceSetsReplaced int64 - // SerializedBytes counts encoded JSON document bytes acknowledged by the - // writer; it does not estimate Arango storage or index overhead. - SerializedBytes int64 + SerializedBytes int64 } type SemanticCoveragePage struct { @@ -246,48 +190,48 @@ type SemanticCoveragePage struct { } type semanticCoverageStoredDocument struct { - Key string `json:"_key"` - RecordKind string `json:"record_kind"` - BuildState SemanticCoverageBuildState `json:"build_state,omitempty"` - ResourceState SemanticCoverageBuildState `json:"resource_state,omitempty"` - CoverageStorageFormatVersion int `json:"coverage_storage_format_version,omitempty"` - BuildReason string `json:"build_reason,omitempty"` - SourceSetDigest string `json:"source_set_digest,omitempty"` - SourcePartitionCount int64 `json:"source_partition_count,omitempty"` - MaterializedPartitionCount int64 `json:"materialized_partition_count,omitempty"` - SourceEnumerationComplete bool `json:"source_enumeration_complete,omitempty"` - CoverageRuleVersion int `json:"coverage_rule_version,omitempty"` - SummaryKind string `json:"summary_kind,omitempty"` - OwnerCanonicalPath string `json:"owner_canonical_path,omitempty"` - Project string `json:"project"` - DatasetGeneration string `json:"dataset_generation"` - BuildID string `json:"build_id"` - SourceKind string `json:"source_kind"` - AuthResourcePath string `json:"auth_resource_path"` - SchemaIdentity string `json:"schema_identity"` - ResourceType string `json:"resource_type"` - ResourceBoundarySignature string `json:"resource_boundary_signature,omitempty"` - SourceID string `json:"source_id"` - CanonicalPath string `json:"canonical_path"` - ExpectedResourceTypes []string `json:"expected_resource_types,omitempty"` - Counts map[string]int64 `json:"counts,omitempty"` - Paths []semanticCoverageEnvelopePath `json:"paths,omitempty"` - Path *semanticCoverageStoredPath `json:"path,omitempty"` - Diagnostics []semanticCoverageStoredDiagnostic `json:"diagnostics,omitempty"` - SourceRecordCount int64 `json:"source_record_count,omitempty"` - ExpectedSourceCount int64 `json:"expected_source_count,omitempty"` - MaterializedSourceCount int64 `json:"materialized_source_count,omitempty"` - SummaryCount int64 `json:"summary_count,omitempty"` - Complete bool `json:"complete"` - CountsTruncated bool `json:"counts_truncated,omitempty"` - FacetsTruncated bool `json:"facets_truncated,omitempty"` - ReferencesTruncated bool `json:"references_truncated,omitempty"` - DiagnosticLocatorsTruncated bool `json:"diagnostic_locators_truncated,omitempty"` - ReconciliationTruncated bool `json:"reconciliation_truncated,omitempty"` - IncompleteGaps []string `json:"incomplete_gaps,omitempty"` - Groupable bool `json:"groupable,omitempty"` - Status SemanticCoverageCandidateStatus `json:"status,omitempty"` - Reason string `json:"reason,omitempty"` + Key string `json:"_key"` + RecordKind string `json:"record_kind"` + BuildState catalog.SemanticCoverageBuildState `json:"build_state,omitempty"` + ResourceState catalog.SemanticCoverageBuildState `json:"resource_state,omitempty"` + CoverageStorageFormatVersion int `json:"coverage_storage_format_version,omitempty"` + BuildReason string `json:"build_reason,omitempty"` + SourceSetDigest string `json:"source_set_digest,omitempty"` + SourcePartitionCount int64 `json:"source_partition_count,omitempty"` + MaterializedPartitionCount int64 `json:"materialized_partition_count,omitempty"` + SourceEnumerationComplete bool `json:"source_enumeration_complete,omitempty"` + CoverageRuleVersion int `json:"coverage_rule_version,omitempty"` + SummaryKind string `json:"summary_kind,omitempty"` + OwnerCanonicalPath string `json:"owner_canonical_path,omitempty"` + Project string `json:"project"` + DatasetGeneration string `json:"dataset_generation"` + BuildID string `json:"build_id"` + SourceKind string `json:"source_kind"` + AuthResourcePath string `json:"auth_resource_path"` + SchemaIdentity string `json:"schema_identity"` + ResourceType string `json:"resource_type"` + ResourceBoundarySignature string `json:"resource_boundary_signature,omitempty"` + SourceID string `json:"source_id"` + CanonicalPath string `json:"canonical_path"` + ExpectedResourceTypes []string `json:"expected_resource_types,omitempty"` + Counts map[string]int64 `json:"counts,omitempty"` + Paths []semanticCoverageEnvelopePath `json:"paths,omitempty"` + Path *semanticCoverageStoredPath `json:"path,omitempty"` + Diagnostics []semanticCoverageStoredDiagnostic `json:"diagnostics,omitempty"` + SourceRecordCount int64 `json:"source_record_count,omitempty"` + ExpectedSourceCount int64 `json:"expected_source_count,omitempty"` + MaterializedSourceCount int64 `json:"materialized_source_count,omitempty"` + SummaryCount int64 `json:"summary_count,omitempty"` + Complete bool `json:"complete"` + CountsTruncated bool `json:"counts_truncated,omitempty"` + FacetsTruncated bool `json:"facets_truncated,omitempty"` + ReferencesTruncated bool `json:"references_truncated,omitempty"` + DiagnosticLocatorsTruncated bool `json:"diagnostic_locators_truncated,omitempty"` + ReconciliationTruncated bool `json:"reconciliation_truncated,omitempty"` + IncompleteGaps []string `json:"incomplete_gaps,omitempty"` + Groupable bool `json:"groupable,omitempty"` + Status catalog.SemanticCoverageCandidateStatus `json:"status,omitempty"` + Reason string `json:"reason,omitempty"` } type semanticCoverageStoredPath struct { @@ -349,8 +293,6 @@ type semanticCoverageStoredDiagnostic struct { Explanation string `json:"explanation,omitempty"` } -// SemanticCoverageCollectionSpec describes the persistent indexes shared by -// generation ingestion and retained-source backfill. func SemanticCoverageCollectionSpec() store.CollectionSpec { return store.CollectionSpec{ Name: SemanticCoverageCollection, @@ -634,8 +576,6 @@ func semanticCoverageStringCounts(counts map[catalog.SemanticCoveragePresence]in return result } -// writeSemanticCoverageDocuments replaces deterministic rows in bounded -// batches, so replaying a source before its checkpoint advances is idempotent. func (s *Store) writeSemanticCoverageDocuments(ctx context.Context, documents []json.RawMessage, batchSize int) (SemanticCoverageWriteStats, error) { if len(documents) == 0 { return SemanticCoverageWriteStats{}, nil @@ -692,18 +632,13 @@ func semanticCoverageBatchSize(requested int) int { return min(requested, semanticCoverageMaximumBatchSize) } -// BeginSemanticCoverageBuild publishes BUILDING before any source-set replay -// can invalidate previously completed proof for this exact build. func (s *Store) BeginSemanticCoverageBuild(ctx context.Context, identity SemanticCoverageBuildIdentity) error { if err := validateSemanticCoverageBuildIdentity(identity); err != nil { return err } - return s.writeSemanticCoverageBuildMarker(ctx, identity, SemanticCoverageBuildBuilding, "", 0, 0, 0) + return s.writeSemanticCoverageBuildMarker(ctx, identity, catalog.SemanticCoverageBuildBuilding, "", 0, 0, 0) } -// BeginSemanticCoverageResource invalidates one resource publication before -// its sources or summaries change. Other resource certificates remain stored, -// but the enclosing global BUILDING marker keeps all lookups fail-closed. func (s *Store) BeginSemanticCoverageResource(ctx context.Context, identity SemanticCoverageBuildIdentity, resourceType string) error { if err := validateSemanticCoverageBuildIdentity(identity); err != nil { return err @@ -711,7 +646,7 @@ func (s *Store) BeginSemanticCoverageResource(ctx context.Context, identity Sema if strings.TrimSpace(resourceType) == "" { return ErrInvalidSemanticCoverageRead } - if err := s.writeSemanticCoverageResourceMarker(ctx, identity, resourceType, SemanticCoverageBuildBuilding, "", SemanticCoverageResourceExpectation{}, 0, 0); err != nil { + if err := s.writeSemanticCoverageResourceMarker(ctx, identity, resourceType, catalog.SemanticCoverageBuildBuilding, "", SemanticCoverageResourceExpectation{}, 0, 0); err != nil { return err } if err := s.client.ExecuteAQL(ctx, deleteSemanticCoverageResourceSummariesAQL, map[string]interface{}{ @@ -728,22 +663,16 @@ func (s *Store) MarkSemanticCoverageResourceMissing(ctx context.Context, identit if err := s.BeginSemanticCoverageResource(ctx, identity, resourceType); err != nil { return err } - return s.writeSemanticCoverageResourceMarker(ctx, identity, resourceType, SemanticCoverageBuildMissing, "SOURCE_COLLECTION_MISSING", SemanticCoverageResourceExpectation{ResourceType: resourceType}, 0, 0) + return s.writeSemanticCoverageResourceMarker(ctx, identity, resourceType, catalog.SemanticCoverageBuildMissing, "SOURCE_COLLECTION_MISSING", SemanticCoverageResourceExpectation{ResourceType: resourceType}, 0, 0) } -// FailSemanticCoverageBuild keeps execution failures distinct from missing -// source proof so readers can report the real failed state. func (s *Store) FailSemanticCoverageBuild(ctx context.Context, identity SemanticCoverageBuildIdentity, reason string) error { if err := validateSemanticCoverageBuildIdentity(identity); err != nil { return err } - return s.writeSemanticCoverageBuildMarker(ctx, identity, SemanticCoverageBuildFailed, boundedSemanticCoverageReason(reason), 0, 0, 0) + return s.writeSemanticCoverageBuildMarker(ctx, identity, catalog.SemanticCoverageBuildFailed, boundedSemanticCoverageReason(reason), 0, 0, 0) } -// FinalizeSemanticCoverageBuild publishes the global build marker only after -// every expected resource certificate is present and valid. Resource -// summaries and authorization partitions are written by -// CompleteSemanticCoverageResource before their COMPLETE markers. func (s *Store) FinalizeSemanticCoverageBuild( ctx context.Context, identity SemanticCoverageBuildIdentity, @@ -757,10 +686,10 @@ func (s *Store) FinalizeSemanticCoverageBuild( return ErrSemanticCoverageResourceExpectation } if identity.SchemaIdentity == "" { - return s.writeSemanticCoverageBuildMarker(ctx, identity, SemanticCoverageBuildUnavailable, "SOURCE_SCHEMA_IDENTITY_UNAVAILABLE", 0, 0, 0, resources) + return s.writeSemanticCoverageBuildMarker(ctx, identity, catalog.SemanticCoverageBuildUnavailable, "SOURCE_SCHEMA_IDENTITY_UNAVAILABLE", 0, 0, 0, resources) } if len(resources) == 0 { - return s.writeSemanticCoverageBuildMarker(ctx, identity, SemanticCoverageBuildUnavailable, "EXPECTED_RESOURCE_SET_EMPTY", 0, 0, 0, resources) + return s.writeSemanticCoverageBuildMarker(ctx, identity, catalog.SemanticCoverageBuildUnavailable, "EXPECTED_RESOURCE_SET_EMPTY", 0, 0, 0, resources) } markers := make(map[string]semanticCoverageStoredDocument, len(resources)) @@ -811,22 +740,22 @@ func (s *Store) FinalizeSemanticCoverageBuild( return fmt.Errorf("read semantic coverage authorization partitions: %w", err) } - state := SemanticCoverageBuildComplete + state := catalog.SemanticCoverageBuildComplete reason := "" var expectedSources, materializedSources, summaryCount int64 for _, resourceType := range resources { marker, found := markers[resourceType] if !found { - state = SemanticCoverageBuildPartial + state = catalog.SemanticCoverageBuildPartial if reason == "" { reason = "RESOURCE_CERTIFICATES_INCOMPLETE" } continue } - if marker.ResourceState != SemanticCoverageBuildComplete || !marker.SourceEnumerationComplete || + if marker.ResourceState != catalog.SemanticCoverageBuildComplete || !marker.SourceEnumerationComplete || marker.SourceSetDigest == "" || marker.ExpectedSourceCount != marker.MaterializedSourceCount || marker.SourcePartitionCount != marker.MaterializedPartitionCount { - state = SemanticCoverageBuildPartial + state = catalog.SemanticCoverageBuildPartial if reason == "" { reason = marker.BuildReason if reason == "" { @@ -839,13 +768,13 @@ func (s *Store) FinalizeSemanticCoverageBuild( materializedSources += marker.MaterializedSourceCount summaryCount += marker.SummaryCount if partitionCounts[resourceType] != marker.MaterializedPartitionCount || partitionCounts[resourceType] != marker.SourcePartitionCount { - state = SemanticCoverageBuildPartial + state = catalog.SemanticCoverageBuildPartial if reason == "" { reason = "RESOURCE_PARTITION_SET_MISMATCH" } } if partitionReason := incompletePartitions[resourceType]; partitionReason != "" { - state = SemanticCoverageBuildPartial + state = catalog.SemanticCoverageBuildPartial if reason == "" { reason = partitionReason } @@ -907,16 +836,16 @@ func (s *Store) CompleteSemanticCoverageResource( } }() if identity.SchemaIdentity == "" { - return s.writeSemanticCoverageResourceMarker(ctx, identity, expectation.ResourceType, SemanticCoverageBuildUnavailable, "SOURCE_SCHEMA_IDENTITY_UNAVAILABLE", expectation, 0, 0) + return s.writeSemanticCoverageResourceMarker(ctx, identity, expectation.ResourceType, catalog.SemanticCoverageBuildUnavailable, "SOURCE_SCHEMA_IDENTITY_UNAVAILABLE", expectation, 0, 0) } if !expectation.SourceExists { if expectation.SourceCount != 0 || len(expectation.Partitions) != 0 { return ErrSemanticCoverageResourceExpectation } - return s.writeSemanticCoverageResourceMarker(ctx, identity, expectation.ResourceType, SemanticCoverageBuildMissing, "SOURCE_COLLECTION_MISSING", expectation, 0, 0) + return s.writeSemanticCoverageResourceMarker(ctx, identity, expectation.ResourceType, catalog.SemanticCoverageBuildMissing, "SOURCE_COLLECTION_MISSING", expectation, 0, 0) } if !expectation.Terminal { - return s.writeSemanticCoverageResourceMarker(ctx, identity, expectation.ResourceType, SemanticCoverageBuildUnavailable, "SOURCE_ENUMERATION_NOT_TERMINAL", expectation, 0, 0) + return s.writeSemanticCoverageResourceMarker(ctx, identity, expectation.ResourceType, catalog.SemanticCoverageBuildUnavailable, "SOURCE_ENUMERATION_NOT_TERMINAL", expectation, 0, 0) } if err := validateSemanticCoverageResourceExpectation(expectation); err != nil { failureReason = "SOURCE_EXPECTATION_INVALID" @@ -972,7 +901,7 @@ func (s *Store) CompleteSemanticCoverageResource( return fmt.Errorf("write semantic coverage resource partitions for %s: %w", expectation.ResourceType, err) } return s.writeSemanticCoverageResourceMarker( - ctx, identity, expectation.ResourceType, SemanticCoverageBuildComplete, "", expectation, + ctx, identity, expectation.ResourceType, catalog.SemanticCoverageBuildComplete, "", expectation, actualSourceCount, int64(len(materializer.partitionSources)), stats.Rows+partitionStats.Rows, ) } @@ -984,7 +913,7 @@ func (s *Store) FailSemanticCoverageResource(ctx context.Context, identity Seman if strings.TrimSpace(resourceType) == "" { return ErrInvalidSemanticCoverageRead } - return s.writeSemanticCoverageResourceMarker(ctx, identity, resourceType, SemanticCoverageBuildFailed, boundedSemanticCoverageReason(reason), SemanticCoverageResourceExpectation{ResourceType: resourceType}, 0, 0) + return s.writeSemanticCoverageResourceMarker(ctx, identity, resourceType, catalog.SemanticCoverageBuildFailed, boundedSemanticCoverageReason(reason), SemanticCoverageResourceExpectation{ResourceType: resourceType}, 0, 0) } func validateSemanticCoverageResourceExpectation(expectation SemanticCoverageResourceExpectation) error { @@ -1041,14 +970,14 @@ func (s *Store) CompleteRetainedSemanticCoverageResource( return certificationSourceRows, err } if identity.SchemaIdentity == "" { - return certificationSourceRows, s.writeSemanticCoverageResourceMarker(ctx, identity, resourceType, SemanticCoverageBuildUnavailable, "SOURCE_SCHEMA_IDENTITY_UNAVAILABLE", SemanticCoverageResourceExpectation{ResourceType: resourceType}, 0, 0) + return certificationSourceRows, s.writeSemanticCoverageResourceMarker(ctx, identity, resourceType, catalog.SemanticCoverageBuildUnavailable, "SOURCE_SCHEMA_IDENTITY_UNAVAILABLE", SemanticCoverageResourceExpectation{ResourceType: resourceType}, 0, 0) } exists, err := s.client.CollectionExists(ctx, resourceType) if err != nil { return certificationSourceRows, errors.Join(fmt.Errorf("check retained source collection %s: %w", resourceType, err), s.FailSemanticCoverageResource(context.WithoutCancel(ctx), identity, resourceType, "SOURCE_COLLECTION_READ_FAILED")) } if !exists { - return certificationSourceRows, s.writeSemanticCoverageResourceMarker(ctx, identity, resourceType, SemanticCoverageBuildMissing, "SOURCE_COLLECTION_MISSING", SemanticCoverageResourceExpectation{ResourceType: resourceType}, 0, 0) + return certificationSourceRows, s.writeSemanticCoverageResourceMarker(ctx, identity, resourceType, catalog.SemanticCoverageBuildMissing, "SOURCE_COLLECTION_MISSING", SemanticCoverageResourceExpectation{ResourceType: resourceType}, 0, 0) } revisionBefore, err := semanticCoverageCollectionRevision(ctx, s.client, resourceType) if err != nil { @@ -1059,13 +988,13 @@ func (s *Store) CompleteRetainedSemanticCoverageResource( for { page, readErr := s.ReadRetainedSemanticInventoryPage(ctx, identity.Project, identity.DatasetGeneration, resourceType, afterKey, pageSize) if errors.Is(readErr, ErrRetainedSemanticSourceMissing) { - return certificationSourceRows, s.writeSemanticCoverageResourceMarker(ctx, identity, resourceType, SemanticCoverageBuildMissing, "SOURCE_COLLECTION_MISSING", SemanticCoverageResourceExpectation{ResourceType: resourceType}, 0, 0) + return certificationSourceRows, s.writeSemanticCoverageResourceMarker(ctx, identity, resourceType, catalog.SemanticCoverageBuildMissing, "SOURCE_COLLECTION_MISSING", SemanticCoverageResourceExpectation{ResourceType: resourceType}, 0, 0) } if readErr != nil { return certificationSourceRows, errors.Join(fmt.Errorf("enumerate retained source collection %s: %w", resourceType, readErr), s.FailSemanticCoverageResource(context.WithoutCancel(ctx), identity, resourceType, "SOURCE_ENUMERATION_FAILED")) } if !page.SourceExists { - return certificationSourceRows, s.writeSemanticCoverageResourceMarker(ctx, identity, resourceType, SemanticCoverageBuildMissing, "SOURCE_COLLECTION_MISSING", SemanticCoverageResourceExpectation{ResourceType: resourceType}, 0, 0) + return certificationSourceRows, s.writeSemanticCoverageResourceMarker(ctx, identity, resourceType, catalog.SemanticCoverageBuildMissing, "SOURCE_COLLECTION_MISSING", SemanticCoverageResourceExpectation{ResourceType: resourceType}, 0, 0) } for _, source := range page.Rows { certificationSourceRows++ @@ -1133,7 +1062,7 @@ func semanticCoverageBuildBindVars(identity SemanticCoverageBuildIdentity) map[s func (s *Store) writeSemanticCoverageBuildMarker( ctx context.Context, identity SemanticCoverageBuildIdentity, - state SemanticCoverageBuildState, + state catalog.SemanticCoverageBuildState, reason string, expectedSources, materializedSources, summaryCount int64, expectedResourceTypes ...[]string, @@ -1165,7 +1094,7 @@ func (s *Store) writeSemanticCoverageResourceMarker( ctx context.Context, identity SemanticCoverageBuildIdentity, resourceType string, - state SemanticCoverageBuildState, + state catalog.SemanticCoverageBuildState, reason string, expectation SemanticCoverageResourceExpectation, materializedSources int64, @@ -1343,9 +1272,6 @@ func newSemanticCoverageMaterializer(identity SemanticCoverageBuildIdentity) *se } } -// addEnvelope folds all rows for a path and all facets for an exact leaf path -// within one source before updating build summaries. This keeps source counts -// from multiplying when one source contributes several occurrence kinds. func (m *semanticCoverageMaterializer) addEnvelope(source semanticCoverageStoredDocument) error { if err := m.addSource(source); err != nil { return err @@ -1697,19 +1623,19 @@ func (m *semanticCoverageMaterializer) summaryDocument(key semanticCoverageSumma document.Groupable = aggregate.allGroupable && m.partitionComplete[partition] && len(aggregate.references) > 0 switch { case document.Groupable: - document.Status = SemanticCoverageCandidateGroupable + document.Status = catalog.SemanticCoverageCandidateGroupable case !m.partitionComplete[partition]: - document.Status = SemanticCoverageCandidateUnavailable + document.Status = catalog.SemanticCoverageCandidateUnavailable document.Reason = string(m.partitionReason[partition]) if document.Reason == "" { document.Reason = string(catalog.SemanticCoverageReasonIncompleteSource) } case document.CountsTruncated || document.FacetsTruncated || document.ReferencesTruncated || (document.Path != nil && (document.Path.CountsTruncated || document.Path.FacetsTruncated || document.Path.ReferencesTruncated)): - document.Status = SemanticCoverageCandidateKeepRaw + document.Status = catalog.SemanticCoverageCandidateKeepRaw document.Reason = string(catalog.SemanticCoverageReasonTruncatedEvidence) case aggregate.anyKeepRaw || !aggregate.allGroupable: - document.Status = SemanticCoverageCandidateKeepRaw + document.Status = catalog.SemanticCoverageCandidateKeepRaw document.Reason = string(aggregate.reason) if aggregate.referencesFull { document.Reason = string(catalog.SemanticCoverageReasonTruncatedEvidence) @@ -1718,7 +1644,7 @@ func (m *semanticCoverageMaterializer) summaryDocument(key semanticCoverageSumma document.Reason = string(catalog.SemanticCoverageReasonUnrepresentedMetadata) } default: - document.Status = SemanticCoverageCandidateKeepRaw + document.Status = catalog.SemanticCoverageCandidateKeepRaw document.Reason = string(catalog.SemanticCoverageReasonUnrepresentedMetadata) } refs := make([]catalog.SemanticCoverageReference, 0, len(aggregate.references)) @@ -1732,7 +1658,7 @@ func (m *semanticCoverageMaterializer) summaryDocument(key semanticCoverageSumma document.Path.References = semanticCoverageStoredReferences(refs) } document.Key = semanticCoverageDocumentKey("semantic-coverage/v1", m.identity.Project, m.identity.DatasetGeneration, m.identity.BuildID, m.identity.SourceKind, m.identity.SchemaIdentity, "summary", key.summaryKind, key.authResourcePath, key.resourceType, key.canonicalPath, key.boundarySignature) - if document.Status != SemanticCoverageCandidateGroupable && document.Path != nil { + if document.Status != catalog.SemanticCoverageCandidateGroupable && document.Path != nil { document.Path.References = nil } return document @@ -1811,9 +1737,9 @@ func (s *Store) writeSemanticCoveragePartitions(ctx context.Context, materialize SourceRecordCount: materializer.partitionSources[partitionKey], SourceSetDigest: semanticCoveragePartitionDigest(partition.resourceType, partition.authResourcePath, materializer.partitionDigests[partitionKey]), Complete: complete, Groupable: complete, } if complete { - document.Status = SemanticCoverageCandidateGroupable + document.Status = catalog.SemanticCoverageCandidateGroupable } else { - document.Status = SemanticCoverageCandidateUnavailable + document.Status = catalog.SemanticCoverageCandidateUnavailable document.Reason = string(materializer.partitionReason[partitionKey]) if document.Reason == "" { document.Reason = string(catalog.SemanticCoverageReasonIncompleteSource) @@ -1978,9 +1904,6 @@ func validateSemanticCoverageDocument(document json.RawMessage) error { return nil } -// semanticCoverageDocumentKey hashes length-delimited identity parts. This -// keeps source/path keys deterministic without collisions from delimiters in -// FHIR paths or source locators. func semanticCoverageDocumentKey(parts ...string) string { hash := sha256.New() var length [4]byte @@ -1992,9 +1915,6 @@ func semanticCoverageDocumentKey(parts ...string) string { return "scv1_" + hex.EncodeToString(hash.Sum(nil)) } -// readSemanticCoveragePage reads one exact build and authorization scope in -// stable key order. The caller controls whether its authorized path set is -// unrestricted; this method never falls back to another build. func (s *Store) readSemanticCoveragePage( ctx context.Context, project string, @@ -2204,13 +2124,10 @@ type semanticCoverageSourcePresence struct { SourceExists bool `json:"source_exists"` } -// LookupSemanticCoverage reads build, resource, partition, source-presence and -// exact facet proof in one bounded AQL snapshot. The caller's auth scope is -// already resolved; no cross-build or cross-scope fallback is performed. -func (s *Store) LookupSemanticCoverage(ctx context.Context, options SemanticCoverageLookupOptions) (SemanticCoverageLookupResult, error) { - result := SemanticCoverageLookupResult{State: SemanticCoverageBuildUnavailable, Candidates: unavailableSemanticCoverageCandidates(options.Candidates, catalog.SemanticCoverageReasonIncompleteSource)} +func (s *Store) LookupSemanticCoverage(ctx context.Context, options catalog.SemanticCoverageLookupOptions) (catalog.SemanticCoverageLookupResult, error) { + result := catalog.SemanticCoverageLookupResult{State: catalog.SemanticCoverageBuildUnavailable, Candidates: unavailableSemanticCoverageCandidates(options.Candidates, catalog.SemanticCoverageReasonIncompleteSource)} if err := validateSemanticCoverageLookupOptions(options); err != nil { - return SemanticCoverageLookupResult{}, err + return catalog.SemanticCoverageLookupResult{}, err } if options.SchemaIdentity == "" { result.Reason = "SOURCE_SCHEMA_IDENTITY_UNAVAILABLE" @@ -2266,7 +2183,7 @@ func (s *Store) LookupSemanticCoverage(ctx context.Context, options SemanticCove return nil }) if err != nil { - return SemanticCoverageLookupResult{}, fmt.Errorf("read semantic coverage lookup snapshot: %w", err) + return catalog.SemanticCoverageLookupResult{}, fmt.Errorf("read semantic coverage lookup snapshot: %w", err) } if !found || snapshot.Build == nil { result.Reason = "COVERAGE_BUILD_MISSING" @@ -2281,11 +2198,11 @@ func (s *Store) LookupSemanticCoverage(ctx context.Context, options SemanticCove result.State = marker.BuildState result.Reason = marker.BuildReason if marker.CoverageRuleVersion != catalog.SemanticCoverageRuleVersion || marker.CoverageStorageFormatVersion != catalog.SemanticCoverageStorageFormatVersion { - result.State = SemanticCoverageBuildUnavailable + result.State = catalog.SemanticCoverageBuildUnavailable result.Reason = "COVERAGE_FORMAT_VERSION_MISMATCH" return result, nil } - if result.State != SemanticCoverageBuildComplete && result.State != SemanticCoverageBuildPartial { + if result.State != catalog.SemanticCoverageBuildComplete && result.State != catalog.SemanticCoverageBuildPartial { return result, nil } if len(snapshot.Resources) > len(resourceTypes) || len(snapshot.Partitions) > semanticCoverageLookupMaximumPartitions || len(snapshot.Facets) > semanticCoverageLookupMaximumRows || @@ -2346,9 +2263,9 @@ func (s *Store) LookupSemanticCoverage(ctx context.Context, options SemanticCove result.Candidates = unavailableSemanticCoverageCandidates(options.Candidates, catalog.SemanticCoverageReasonIncompleteSource) return result, nil } - facetsByCandidate := make(map[SemanticCoverageCandidatePath][]semanticCoverageStoredDocument, len(options.Candidates)) + facetsByCandidate := make(map[catalog.SemanticCoverageCandidatePath][]semanticCoverageStoredDocument, len(options.Candidates)) for _, facet := range snapshot.Facets { - key := SemanticCoverageCandidatePath{ResourceType: facet.ResourceType, CanonicalPath: facet.CanonicalPath} + key := catalog.SemanticCoverageCandidatePath{ResourceType: facet.ResourceType, CanonicalPath: facet.CanonicalPath} if facet.RecordKind != "summary" || facet.SummaryKind != "facet" || facet.Key != semanticCoverageSummaryDocumentKey(identity, facet) || facet.Project != identity.Project || facet.DatasetGeneration != identity.DatasetGeneration || facet.BuildID != identity.BuildID || @@ -2359,22 +2276,22 @@ func (s *Store) LookupSemanticCoverage(ctx context.Context, options SemanticCove facetsByCandidate[key] = append(facetsByCandidate[key], facet) } - result.Candidates = make([]SemanticCoverageCandidateResult, 0, len(options.Candidates)) + result.Candidates = make([]catalog.SemanticCoverageCandidateResult, 0, len(options.Candidates)) for _, candidate := range options.Candidates { - candidateResult := SemanticCoverageCandidateResult{ResourceType: candidate.ResourceType, CanonicalPath: candidate.CanonicalPath} + candidateResult := catalog.SemanticCoverageCandidateResult{ResourceType: candidate.ResourceType, CanonicalPath: candidate.CanonicalPath} candidateResult.CoverageID = semanticCoverageDocumentKey("semantic-coverage-absent/v1", identity.Project, identity.DatasetGeneration, identity.BuildID, identity.SourceKind, identity.SchemaIdentity, candidate.ResourceType, candidate.CanonicalPath) resource, exists := resourceMarkers[candidate.ResourceType] if !containsSortedSemanticCoverageResource(marker.ExpectedResourceTypes, candidate.ResourceType) { - candidateResult.Status = SemanticCoverageCandidateUnavailable - candidateResult.ResourceState = SemanticCoverageBuildUnavailable + candidateResult.Status = catalog.SemanticCoverageCandidateUnavailable + candidateResult.ResourceState = catalog.SemanticCoverageBuildUnavailable candidateResult.ResourceReason = "RESOURCE_NOT_IN_EXPECTED_MANIFEST" candidateResult.Reason = catalog.SemanticCoverageReasonIncompleteSource result.Candidates = append(result.Candidates, candidateResult) continue } if !exists { - candidateResult.Status = SemanticCoverageCandidateUnavailable - candidateResult.ResourceState = SemanticCoverageBuildUnavailable + candidateResult.Status = catalog.SemanticCoverageCandidateUnavailable + candidateResult.ResourceState = catalog.SemanticCoverageBuildUnavailable candidateResult.ResourceReason = "RESOURCE_CERTIFICATE_MISSING" candidateResult.Reason = catalog.SemanticCoverageReasonIncompleteSource result.Candidates = append(result.Candidates, candidateResult) @@ -2382,8 +2299,8 @@ func (s *Store) LookupSemanticCoverage(ctx context.Context, options SemanticCove } candidateResult.ResourceState = resource.ResourceState candidateResult.ResourceReason = resource.BuildReason - if resource.ResourceState != SemanticCoverageBuildComplete { - candidateResult.Status = SemanticCoverageCandidateUnavailable + if resource.ResourceState != catalog.SemanticCoverageBuildComplete { + candidateResult.Status = catalog.SemanticCoverageCandidateUnavailable if resource.BuildReason == "" { candidateResult.ResourceReason = "RESOURCE_CERTIFICATE_INCOMPLETE" } @@ -2394,8 +2311,8 @@ func (s *Store) LookupSemanticCoverage(ctx context.Context, options SemanticCove if !resource.SourceEnumerationComplete || resource.SourceSetDigest == "" || resource.ExpectedSourceCount != resource.MaterializedSourceCount || resource.SourcePartitionCount != resource.MaterializedPartitionCount { - candidateResult.Status = SemanticCoverageCandidateUnavailable - candidateResult.ResourceState = SemanticCoverageBuildFailed + candidateResult.Status = catalog.SemanticCoverageCandidateUnavailable + candidateResult.ResourceState = catalog.SemanticCoverageBuildFailed candidateResult.ResourceReason = "RESOURCE_CERTIFICATE_INCOMPLETE" candidateResult.Reason = catalog.SemanticCoverageReasonIncompleteSource result.Candidates = append(result.Candidates, candidateResult) @@ -2404,7 +2321,7 @@ func (s *Store) LookupSemanticCoverage(ctx context.Context, options SemanticCove if options.AuthResourcePathsUnrestricted { partitions := partitionsByResource[candidate.ResourceType] if int64(len(partitions)) != resource.SourcePartitionCount { - candidateResult.Status = SemanticCoverageCandidateUnavailable + candidateResult.Status = catalog.SemanticCoverageCandidateUnavailable candidateResult.ResourceReason = "RESOURCE_PARTITION_CERTIFICATE_INCOMPLETE" candidateResult.Reason = catalog.SemanticCoverageReasonIncompleteSource result.Candidates = append(result.Candidates, candidateResult) @@ -2412,7 +2329,7 @@ func (s *Store) LookupSemanticCoverage(ctx context.Context, options SemanticCove } for authPath, partition := range partitions { if !validSemanticCoveragePartition(partition) || !sourceProofByPartition[semanticCoveragePartitionKey(authPath, candidate.ResourceType)] { - candidateResult.Status = SemanticCoverageCandidateUnavailable + candidateResult.Status = catalog.SemanticCoverageCandidateUnavailable candidateResult.ResourceReason = "RESOURCE_PARTITION_CERTIFICATE_INCOMPLETE" candidateResult.Reason = catalog.SemanticCoverageReasonIncompleteSource break @@ -2425,25 +2342,25 @@ func (s *Store) LookupSemanticCoverage(ctx context.Context, options SemanticCove sourceKey := semanticCoveragePartitionKey(authPath, candidate.ResourceType) sourceExists, hasSourceProof := sourceProofByPartition[sourceKey] if !hasSourceProof || sourceExists != hasPartition || (hasPartition && !validSemanticCoveragePartition(partition)) { - candidateResult.Status = SemanticCoverageCandidateUnavailable + candidateResult.Status = catalog.SemanticCoverageCandidateUnavailable candidateResult.ResourceReason = "RESOURCE_PARTITION_CERTIFICATE_INCOMPLETE" candidateResult.Reason = catalog.SemanticCoverageReasonIncompleteSource break } } } - if candidateResult.Status == SemanticCoverageCandidateUnavailable { + if candidateResult.Status == catalog.SemanticCoverageCandidateUnavailable { result.Candidates = append(result.Candidates, candidateResult) continue } - candidateResult.Status = SemanticCoverageCandidateKeepRaw + candidateResult.Status = catalog.SemanticCoverageCandidateKeepRaw candidateResult.Reason = catalog.SemanticCoverageReasonUnrepresentedMetadata rows := facetsByCandidate[candidate] if len(rows) == 0 { result.Candidates = append(result.Candidates, candidateResult) continue } - candidateResult.Status = SemanticCoverageCandidateGroupable + candidateResult.Status = catalog.SemanticCoverageCandidateGroupable candidateResult.Reason = "" candidateResult.References = []catalog.SemanticCoverageReference{} ids := make([]string, 0, len(rows)) @@ -2452,9 +2369,9 @@ func (s *Store) LookupSemanticCoverage(ctx context.Context, options SemanticCove for _, row := range rows { ids = append(ids, row.Key) switch row.Status { - case SemanticCoverageCandidateGroupable: + case catalog.SemanticCoverageCandidateGroupable: if row.Path == nil || !row.Groupable || row.CountsTruncated || row.FacetsTruncated || row.ReferencesTruncated || row.Path.CountsTruncated || row.Path.FacetsTruncated || row.Path.ReferencesTruncated { - candidateResult.Status = SemanticCoverageCandidateUnavailable + candidateResult.Status = catalog.SemanticCoverageCandidateUnavailable reasons[catalog.SemanticCoverageReasonTruncatedEvidence] = struct{}{} continue } @@ -2466,30 +2383,30 @@ func (s *Store) LookupSemanticCoverage(ctx context.Context, options SemanticCove references[semanticCoverageReferenceKey(converted)] = converted } } - case SemanticCoverageCandidateKeepRaw: - if candidateResult.Status != SemanticCoverageCandidateUnavailable { - candidateResult.Status = SemanticCoverageCandidateKeepRaw + case catalog.SemanticCoverageCandidateKeepRaw: + if candidateResult.Status != catalog.SemanticCoverageCandidateUnavailable { + candidateResult.Status = catalog.SemanticCoverageCandidateKeepRaw } if row.Reason != "" { reasons[catalog.SemanticCoverageReason(row.Reason)] = struct{}{} } - case SemanticCoverageCandidateUnavailable: - candidateResult.Status = SemanticCoverageCandidateUnavailable + case catalog.SemanticCoverageCandidateUnavailable: + candidateResult.Status = catalog.SemanticCoverageCandidateUnavailable if row.Reason != "" { reasons[catalog.SemanticCoverageReason(row.Reason)] = struct{}{} } default: - candidateResult.Status = SemanticCoverageCandidateUnavailable + candidateResult.Status = catalog.SemanticCoverageCandidateUnavailable reasons[catalog.SemanticCoverageReasonIncompleteSource] = struct{}{} } } sort.Strings(ids) candidateResult.CoverageID = semanticCoverageDocumentKey(append([]string{"semantic-coverage-result/v1"}, ids...)...) - if candidateResult.Status == SemanticCoverageCandidateGroupable && len(references) == 0 { - candidateResult.Status = SemanticCoverageCandidateKeepRaw + if candidateResult.Status == catalog.SemanticCoverageCandidateGroupable && len(references) == 0 { + candidateResult.Status = catalog.SemanticCoverageCandidateKeepRaw reasons[catalog.SemanticCoverageReasonUnrepresentedMetadata] = struct{}{} } - if candidateResult.Status == SemanticCoverageCandidateGroupable { + if candidateResult.Status == catalog.SemanticCoverageCandidateGroupable { for _, reference := range references { candidateResult.References = append(candidateResult.References, reference) } @@ -2506,7 +2423,7 @@ func (s *Store) LookupSemanticCoverage(ctx context.Context, options SemanticCove } func validSemanticCoveragePartition(partition semanticCoverageStoredDocument) bool { - return partition.Complete && partition.Groupable && partition.Status == SemanticCoverageCandidateGroupable && + return partition.Complete && partition.Groupable && partition.Status == catalog.SemanticCoverageCandidateGroupable && partition.SourceRecordCount > 0 && partition.SourceSetDigest != "" } @@ -2528,14 +2445,14 @@ const semanticCoverageLookupMaximumAuthPaths = 100 const semanticCoverageLookupMaximumRows = semanticCoverageLookupMaximumCandidates * semanticCoverageLookupMaximumAuthPaths const semanticCoverageLookupMaximumPartitions = semanticCoverageMaximumPartitions -func validateSemanticCoverageLookupOptions(options SemanticCoverageLookupOptions) error { +func validateSemanticCoverageLookupOptions(options catalog.SemanticCoverageLookupOptions) error { if strings.TrimSpace(options.Project) == "" || catalog.NormalizeDatasetGeneration(options.DatasetGeneration) == "" || strings.TrimSpace(options.BuildID) == "" || strings.TrimSpace(options.SourceKind) == "" || len(options.Candidates) == 0 || len(options.Candidates) > semanticCoverageLookupMaximumCandidates || (!options.AuthResourcePathsUnrestricted && (len(options.AuthResourcePaths) == 0 || len(options.AuthResourcePaths) > semanticCoverageLookupMaximumAuthPaths)) { return ErrInvalidSemanticCoverageRead } - seen := make(map[SemanticCoverageCandidatePath]struct{}, len(options.Candidates)) + seen := make(map[catalog.SemanticCoverageCandidatePath]struct{}, len(options.Candidates)) for _, candidate := range options.Candidates { if strings.TrimSpace(candidate.ResourceType) == "" || strings.TrimSpace(candidate.CanonicalPath) == "" { return ErrInvalidSemanticCoverageRead @@ -2553,16 +2470,16 @@ func validateSemanticCoverageLookupOptions(options SemanticCoverageLookupOptions return nil } -func unavailableSemanticCoverageCandidates(candidates []SemanticCoverageCandidatePath, reason catalog.SemanticCoverageReason) []SemanticCoverageCandidateResult { - result := make([]SemanticCoverageCandidateResult, 0, len(candidates)) +func unavailableSemanticCoverageCandidates(candidates []catalog.SemanticCoverageCandidatePath, reason catalog.SemanticCoverageReason) []catalog.SemanticCoverageCandidateResult { + result := make([]catalog.SemanticCoverageCandidateResult, 0, len(candidates)) for _, candidate := range candidates { result = append(result, unavailableSemanticCoverageCandidate(candidate, reason)) } return result } -func unavailableSemanticCoverageCandidate(candidate SemanticCoverageCandidatePath, reason catalog.SemanticCoverageReason) SemanticCoverageCandidateResult { - return SemanticCoverageCandidateResult{ResourceType: candidate.ResourceType, CanonicalPath: candidate.CanonicalPath, Status: SemanticCoverageCandidateUnavailable, Reason: reason} +func unavailableSemanticCoverageCandidate(candidate catalog.SemanticCoverageCandidatePath, reason catalog.SemanticCoverageReason) catalog.SemanticCoverageCandidateResult { + return catalog.SemanticCoverageCandidateResult{ResourceType: candidate.ResourceType, CanonicalPath: candidate.CanonicalPath, Status: catalog.SemanticCoverageCandidateUnavailable, Reason: reason} } func singleSemanticCoverageReason(reasons map[catalog.SemanticCoverageReason]struct{}) catalog.SemanticCoverageReason { diff --git a/internal/catalog/arango/semantic_coverage_envelope.go b/internal/backends/arango/internal/catalog/semantic_coverage_envelope.go similarity index 92% rename from internal/catalog/arango/semantic_coverage_envelope.go rename to internal/backends/arango/internal/catalog/semantic_coverage_envelope.go index 790122079..10ec1b744 100644 --- a/internal/catalog/arango/semantic_coverage_envelope.go +++ b/internal/backends/arango/internal/catalog/semantic_coverage_envelope.go @@ -12,9 +12,6 @@ const ( semanticCoverageEnvelopeRecordKind = "source_envelope_v1" ) -// semanticCoverageSourceEnvelope is the sole persisted source-level coverage -// document. The key names the physical source locator; auth scope is mutable -// source data so replay can replace it atomically. type semanticCoverageSourceEnvelope struct { Key string `json:"_key"` RecordKind string `json:"record_kind"` @@ -46,8 +43,6 @@ type semanticCoverageEnvelopePath struct { Path *semanticCoverageStoredPath `json:"path"` } -// Explanations are intentionally omitted from persisted diagnostics. They are -// presentation text and must not become a channel for source values. type semanticCoverageEnvelopeDiagnostic struct { Code string `json:"code"` CanonicalPath string `json:"canonical_path,omitempty"` @@ -55,10 +50,6 @@ type semanticCoverageEnvelopeDiagnostic struct { Reason string `json:"reason,omitempty"` } -// EncodeSemanticCoverageSourceEnvelope returns one bounded JSON document for -// a single source contribution. It persists only aggregated metadata, paths, -// facets, references, and diagnostics; ephemeral emission evidence and source -// values are not part of the DTO. func EncodeSemanticCoverageSourceEnvelope(buildID string, contribution catalog.SemanticCoverageSourceContribution) (json.RawMessage, error) { if err := validateSemanticCoverageContribution(buildID, contribution); err != nil { return nil, err diff --git a/internal/catalog/arango/semantic_coverage_envelope_test.go b/internal/backends/arango/internal/catalog/semantic_coverage_envelope_test.go similarity index 100% rename from internal/catalog/arango/semantic_coverage_envelope_test.go rename to internal/backends/arango/internal/catalog/semantic_coverage_envelope_test.go diff --git a/internal/catalog/arango/semantic_coverage_resource_integration_test.go b/internal/backends/arango/internal/catalog/semantic_coverage_resource_integration_test.go similarity index 83% rename from internal/catalog/arango/semantic_coverage_resource_integration_test.go rename to internal/backends/arango/internal/catalog/semantic_coverage_resource_integration_test.go index 68cebb8c3..55a3c6584 100644 --- a/internal/catalog/arango/semantic_coverage_resource_integration_test.go +++ b/internal/backends/arango/internal/catalog/semantic_coverage_resource_integration_test.go @@ -11,8 +11,8 @@ import ( "testing" "time" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/catalog" - arangostore "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -22,8 +22,6 @@ func TestSemanticCoverageResourceArangoPartialScopeEmptyAndReplay(t *testing.T) t.Fatalf("begin resource coverage build failed (%v)", err) } - // These physical source locators represent retained records with the same - // logical FHIR id. The source envelope key must keep both physical records. specimenA := semanticCoverageResourceContribution(identity, "Specimen", "Patient/scope-a", "retained:Specimen/physical-a/logical-shared", "type", "type.coding[].code", true) specimenB := semanticCoverageResourceContribution(identity, "Specimen", "Patient/scope-b", "retained:Specimen/physical-b/logical-shared", "type", "type.coding[].code", false) specimenContributions := []catalog.SemanticCoverageSourceContribution{specimenA, specimenB} @@ -51,14 +49,14 @@ func TestSemanticCoverageResourceArangoPartialScopeEmptyAndReplay(t *testing.T) t.Fatalf("finalize partial global coverage audit failed (%v)", err) } - specimenCandidate := SemanticCoverageCandidatePath{ResourceType: "Specimen", CanonicalPath: "type.coding[].code"} - lookup := func(t *testing.T, paths []string, unrestricted bool, candidate SemanticCoverageCandidatePath) SemanticCoverageLookupResult { + specimenCandidate := catalog.SemanticCoverageCandidatePath{ResourceType: "Specimen", CanonicalPath: "type.coding[].code"} + lookup := func(t *testing.T, paths []string, unrestricted bool, candidate catalog.SemanticCoverageCandidatePath) catalog.SemanticCoverageLookupResult { t.Helper() - result, err := adapter.LookupSemanticCoverage(ctx, SemanticCoverageLookupOptions{ + result, err := adapter.LookupSemanticCoverage(ctx, catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: identity.BuildID, SchemaIdentity: identity.SchemaIdentity, SourceKind: identity.SourceKind, AuthResourcePaths: paths, AuthResourcePathsUnrestricted: unrestricted, - Candidates: []SemanticCoverageCandidatePath{candidate}, + Candidates: []catalog.SemanticCoverageCandidatePath{candidate}, }) if err != nil { t.Fatalf("lookup resource-scoped coverage failed (%v)", err) @@ -67,8 +65,8 @@ func TestSemanticCoverageResourceArangoPartialScopeEmptyAndReplay(t *testing.T) } aOnly := lookup(t, []string{"Patient/scope-a"}, false, specimenCandidate) - if aOnly.State != SemanticCoverageBuildPartial || aOnly.Candidates[0].ResourceState != SemanticCoverageBuildComplete || - aOnly.Candidates[0].Status != SemanticCoverageCandidateGroupable || len(aOnly.Candidates[0].References) == 0 { + if aOnly.State != catalog.SemanticCoverageBuildPartial || aOnly.Candidates[0].ResourceState != catalog.SemanticCoverageBuildComplete || + aOnly.Candidates[0].Status != catalog.SemanticCoverageCandidateGroupable || len(aOnly.Candidates[0].References) == 0 { t.Fatalf("complete Specimen auth-A candidate under partial audit = %+v, want exact groupable proof", aOnly) } for _, test := range []struct { @@ -81,50 +79,50 @@ func TestSemanticCoverageResourceArangoPartialScopeEmptyAndReplay(t *testing.T) } { t.Run(test.name, func(t *testing.T) { result := lookup(t, test.paths, test.unrestricted, specimenCandidate) - if result.State != SemanticCoverageBuildPartial || result.Candidates[0].ResourceState != SemanticCoverageBuildComplete || - result.Candidates[0].Status == SemanticCoverageCandidateGroupable || len(result.Candidates[0].References) != 0 { + if result.State != catalog.SemanticCoverageBuildPartial || result.Candidates[0].ResourceState != catalog.SemanticCoverageBuildComplete || + result.Candidates[0].Status == catalog.SemanticCoverageCandidateGroupable || len(result.Candidates[0].References) != 0 { t.Fatalf("combined Specimen auth candidate = %+v, want no groupable proof", result) } }) } - task := lookup(t, []string{"Patient/scope-a"}, false, SemanticCoverageCandidatePath{ResourceType: "Task", CanonicalPath: "code.coding[].code"}) - if task.State != SemanticCoverageBuildPartial || task.Candidates[0].ResourceState != SemanticCoverageBuildMissing || - task.Candidates[0].ResourceReason != "SOURCE_COLLECTION_MISSING" || task.Candidates[0].Status != SemanticCoverageCandidateUnavailable || len(task.Candidates[0].References) != 0 { + task := lookup(t, []string{"Patient/scope-a"}, false, catalog.SemanticCoverageCandidatePath{ResourceType: "Task", CanonicalPath: "code.coding[].code"}) + if task.State != catalog.SemanticCoverageBuildPartial || task.Candidates[0].ResourceState != catalog.SemanticCoverageBuildMissing || + task.Candidates[0].ResourceReason != "SOURCE_COLLECTION_MISSING" || task.Candidates[0].Status != catalog.SemanticCoverageCandidateUnavailable || len(task.Candidates[0].References) != 0 { t.Fatalf("missing Task resource candidate = %+v, want distinct MISSING certificate and no refs", task) } - patient := lookup(t, nil, true, SemanticCoverageCandidatePath{ResourceType: "Patient", CanonicalPath: "identifier.type.coding[].code"}) - if patient.State != SemanticCoverageBuildPartial || patient.Candidates[0].ResourceState != SemanticCoverageBuildComplete || - patient.Candidates[0].Status != SemanticCoverageCandidateKeepRaw || len(patient.Candidates[0].References) != 0 { + patient := lookup(t, nil, true, catalog.SemanticCoverageCandidatePath{ResourceType: "Patient", CanonicalPath: "identifier.type.coding[].code"}) + if patient.State != catalog.SemanticCoverageBuildPartial || patient.Candidates[0].ResourceState != catalog.SemanticCoverageBuildComplete || + patient.Candidates[0].Status != catalog.SemanticCoverageCandidateKeepRaw || len(patient.Candidates[0].References) != 0 { t.Fatalf("existing empty Patient resource candidate = %+v, want COMPLETE certificate with raw absence", patient) } - staleBuild, err := adapter.LookupSemanticCoverage(ctx, SemanticCoverageLookupOptions{ + staleBuild, err := adapter.LookupSemanticCoverage(ctx, catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: "stale-build", SchemaIdentity: identity.SchemaIdentity, SourceKind: identity.SourceKind, - AuthResourcePaths: []string{"Patient/scope-a"}, Candidates: []SemanticCoverageCandidatePath{specimenCandidate}, + AuthResourcePaths: []string{"Patient/scope-a"}, Candidates: []catalog.SemanticCoverageCandidatePath{specimenCandidate}, }) - if err != nil || staleBuild.State != SemanticCoverageBuildUnavailable || staleBuild.Reason != "BUILD_ID_MISMATCH" || len(staleBuild.Candidates[0].References) != 0 { + if err != nil || staleBuild.State != catalog.SemanticCoverageBuildUnavailable || staleBuild.Reason != "BUILD_ID_MISMATCH" || len(staleBuild.Candidates[0].References) != 0 { t.Fatalf("stale build lookup = %+v, err=%v; want unavailable identity proof", staleBuild, err) } - staleSchema, err := adapter.LookupSemanticCoverage(ctx, SemanticCoverageLookupOptions{ + staleSchema, err := adapter.LookupSemanticCoverage(ctx, catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: identity.BuildID, SchemaIdentity: strings.Repeat("c", 64), SourceKind: identity.SourceKind, - AuthResourcePaths: []string{"Patient/scope-a"}, Candidates: []SemanticCoverageCandidatePath{specimenCandidate}, + AuthResourcePaths: []string{"Patient/scope-a"}, Candidates: []catalog.SemanticCoverageCandidatePath{specimenCandidate}, }) - if err != nil || staleSchema.State != SemanticCoverageBuildUnavailable || len(staleSchema.Candidates[0].References) != 0 { + if err != nil || staleSchema.State != catalog.SemanticCoverageBuildUnavailable || len(staleSchema.Candidates[0].References) != 0 { t.Fatalf("stale schema lookup = %+v, err=%v; want unavailable identity proof", staleSchema, err) } if err := adapter.BeginSemanticCoverageBuild(ctx, identity); err != nil { t.Fatalf("begin resource replay failed (%v)", err) } - building, err := adapter.LookupSemanticCoverage(ctx, SemanticCoverageLookupOptions{ + building, err := adapter.LookupSemanticCoverage(ctx, catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: identity.BuildID, SchemaIdentity: identity.SchemaIdentity, SourceKind: identity.SourceKind, - AuthResourcePaths: []string{"Patient/scope-a"}, Candidates: []SemanticCoverageCandidatePath{specimenCandidate}, + AuthResourcePaths: []string{"Patient/scope-a"}, Candidates: []catalog.SemanticCoverageCandidatePath{specimenCandidate}, }) - if err != nil || building.State != SemanticCoverageBuildBuilding || len(building.Candidates[0].References) != 0 { + if err != nil || building.State != catalog.SemanticCoverageBuildBuilding || len(building.Candidates[0].References) != 0 { t.Fatalf("in-progress resource replay lookup = %+v, err=%v; want BUILDING and no refs", building, err) } if err := adapter.BeginSemanticCoverageResource(ctx, identity, "Specimen"); err != nil { @@ -156,13 +154,13 @@ func TestSemanticCoverageResourceArangoPartialScopeEmptyAndReplay(t *testing.T) } assertSemanticCoverageEnvelopes(t, ctx, client, identity, replayedContributions) oldPartition := lookup(t, []string{"Patient/scope-a"}, false, specimenCandidate) - if oldPartition.State != SemanticCoverageBuildPartial || oldPartition.Candidates[0].ResourceState != SemanticCoverageBuildComplete || - oldPartition.Candidates[0].Status == SemanticCoverageCandidateGroupable || len(oldPartition.Candidates[0].References) != 0 { + if oldPartition.State != catalog.SemanticCoverageBuildPartial || oldPartition.Candidates[0].ResourceState != catalog.SemanticCoverageBuildComplete || + oldPartition.Candidates[0].Status == catalog.SemanticCoverageCandidateGroupable || len(oldPartition.Candidates[0].References) != 0 { t.Fatalf("old authorization partition after source auth-path drift = %+v, want no old proof", oldPartition) } replayed := lookup(t, []string{"Patient/scope-c"}, false, specimenCandidate) - if replayed.State != SemanticCoverageBuildPartial || replayed.Candidates[0].ResourceState != SemanticCoverageBuildComplete || - replayed.Candidates[0].Status != SemanticCoverageCandidateGroupable || len(replayed.Candidates[0].References) == 0 { + if replayed.State != catalog.SemanticCoverageBuildPartial || replayed.Candidates[0].ResourceState != catalog.SemanticCoverageBuildComplete || + replayed.Candidates[0].Status != catalog.SemanticCoverageCandidateGroupable || len(replayed.Candidates[0].References) == 0 { t.Fatalf("replayed Specimen auth-A proof = %+v, want current groupable proof", replayed) } for _, test := range []struct { @@ -175,7 +173,7 @@ func TestSemanticCoverageResourceArangoPartialScopeEmptyAndReplay(t *testing.T) } { t.Run(test.name, func(t *testing.T) { result := lookup(t, test.paths, test.unrestricted, specimenCandidate) - if result.Candidates[0].Status == SemanticCoverageCandidateGroupable || len(result.Candidates[0].References) != 0 { + if result.Candidates[0].Status == catalog.SemanticCoverageCandidateGroupable || len(result.Candidates[0].References) != 0 { t.Fatalf("combined auth candidate after replacement = %+v, want no groupable proof", result) } }) @@ -235,15 +233,15 @@ FOR d IN fhir_semantic_coverage if err := adapter.FinalizeSemanticCoverageBuild(ctx, identity, []string{"Specimen"}); err != nil { t.Fatalf("finalize global audit around failed Specimen resource failed (%v)", err) } - result, err := adapter.LookupSemanticCoverage(ctx, SemanticCoverageLookupOptions{ + result, err := adapter.LookupSemanticCoverage(ctx, catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: identity.BuildID, SchemaIdentity: identity.SchemaIdentity, SourceKind: identity.SourceKind, AuthResourcePaths: []string{"Patient/scope-a"}, - Candidates: []SemanticCoverageCandidatePath{{ResourceType: "Specimen", CanonicalPath: "type.coding[].code"}}, + Candidates: []catalog.SemanticCoverageCandidatePath{{ResourceType: "Specimen", CanonicalPath: "type.coding[].code"}}, }) - if err != nil || result.State != SemanticCoverageBuildPartial || result.Candidates[0].ResourceState != SemanticCoverageBuildFailed || + if err != nil || result.State != catalog.SemanticCoverageBuildPartial || result.Candidates[0].ResourceState != catalog.SemanticCoverageBuildFailed || result.Candidates[0].ResourceReason == "" || - result.Candidates[0].Status != SemanticCoverageCandidateUnavailable || len(result.Candidates[0].References) != 0 { + result.Candidates[0].Status != catalog.SemanticCoverageCandidateUnavailable || len(result.Candidates[0].References) != 0 { t.Fatalf("mismatched Specimen source-envelope lookup = %+v, err=%v; want FAILED candidate with no refs", result, err) } } diff --git a/internal/catalog/arango/semantic_coverage_test.go b/internal/backends/arango/internal/catalog/semantic_coverage_test.go similarity index 90% rename from internal/catalog/arango/semantic_coverage_test.go rename to internal/backends/arango/internal/catalog/semantic_coverage_test.go index f963f0d77..c9e0ced84 100644 --- a/internal/catalog/arango/semantic_coverage_test.go +++ b/internal/backends/arango/internal/catalog/semantic_coverage_test.go @@ -10,8 +10,8 @@ import ( "strings" "testing" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/catalog" - store "github.com/calypr/loom/internal/store/arango" ) func TestSemanticCoverageCollectionHasEnvelopeAndCertificateIndexes(t *testing.T) { @@ -349,7 +349,7 @@ func TestWriteSemanticCoverageBatchesAtomicSourceEnvelopes(t *testing.T) { t.Fatal("resource certification accepted a partial envelope batch") } for _, document := range interrupted.documents { - if document.RecordKind == "resource" && document.ResourceState == SemanticCoverageBuildComplete { + if document.RecordKind == "resource" && document.ResourceState == catalog.SemanticCoverageBuildComplete { t.Fatalf("partial envelope batch published COMPLETE resource marker: %+v", document) } } @@ -413,8 +413,6 @@ func TestWriteSemanticCoverageKeepsDistinctRetainedKeysWithSameLogicalFHIRIDSepa second := semanticCoverageFixtureContribution() second.Identity.AuthResourcePath = "Patient/scope-b" second.Identity.SourceID = "retained:Observation/arango-key-b" - // These retained rows intentionally model distinct Arango _keys whose FHIR - // payloads share the same logical `id`; source identity must use _key only. sharedLogicalFHIRID := "observation-duplicate" if sharedLogicalFHIRID == "" { t.Fatal("fixture requires a shared logical FHIR id") @@ -501,13 +499,13 @@ func TestSemanticCoverageBuildMaterializesExactAuthAndBuildSummaries(t *testing. } completeSemanticCoverageMemoryBuild(t, adapter, identity, contribution) marker := client.documents[semanticCoverageBuildKey(identity)] - if marker.BuildState != SemanticCoverageBuildComplete || marker.MaterializedSourceCount != 1 || marker.SummaryCount != 4 { + if marker.BuildState != catalog.SemanticCoverageBuildComplete || marker.MaterializedSourceCount != 1 || marker.SummaryCount != 4 { t.Fatalf("coverage marker = %+v, want complete owner and exact leaf summaries for one source", marker) } - options := SemanticCoverageLookupOptions{ + options := catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: buildID, SchemaIdentity: identity.SchemaIdentity, SourceKind: identity.SourceKind, - AuthResourcePaths: []string{contribution.Identity.AuthResourcePath}, Candidates: []SemanticCoverageCandidatePath{{ + AuthResourcePaths: []string{contribution.Identity.AuthResourcePath}, Candidates: []catalog.SemanticCoverageCandidatePath{{ ResourceType: contribution.Identity.ResourceType, CanonicalPath: "coding[].code", }}, } @@ -515,7 +513,7 @@ func TestSemanticCoverageBuildMaterializesExactAuthAndBuildSummaries(t *testing. if err != nil { t.Fatalf("lookup exact coverage: %v", err) } - if lookup.State != SemanticCoverageBuildComplete || len(lookup.Candidates) != 1 || lookup.Candidates[0].Status != SemanticCoverageCandidateGroupable || len(lookup.Candidates[0].References) != 2 { + if lookup.State != catalog.SemanticCoverageBuildComplete || len(lookup.Candidates) != 1 || lookup.Candidates[0].Status != catalog.SemanticCoverageCandidateGroupable || len(lookup.Candidates[0].References) != 2 { t.Fatalf("exact lookup = %+v, want one groupable candidate with typed refs", lookup) } if lookup.Candidates[0].CanonicalPath != "coding[].code" { @@ -528,30 +526,30 @@ func TestSemanticCoverageBuildMaterializesExactAuthAndBuildSummaries(t *testing. } absent := options - absent.Candidates = []SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: "optional.absent"}} + absent.Candidates = []catalog.SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: "optional.absent"}} lookup, err = adapter.LookupSemanticCoverage(context.Background(), absent) - if err != nil || lookup.Candidates[0].Status != SemanticCoverageCandidateKeepRaw || lookup.Candidates[0].Reason != catalog.SemanticCoverageReasonUnrepresentedMetadata { + if err != nil || lookup.Candidates[0].Status != catalog.SemanticCoverageCandidateKeepRaw || lookup.Candidates[0].Reason != catalog.SemanticCoverageReasonUnrepresentedMetadata { t.Fatalf("proven absent facet lookup = %+v err=%v, want KEEP_RAW after complete source partition", lookup, err) } metadata := options - metadata.Candidates = []SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: "coding[].userSelected"}} + metadata.Candidates = []catalog.SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: "coding[].userSelected"}} lookup, err = adapter.LookupSemanticCoverage(context.Background(), metadata) - if err != nil || lookup.Candidates[0].Status != SemanticCoverageCandidateKeepRaw || lookup.Candidates[0].Reason != catalog.SemanticCoverageReasonUnrepresentedMetadata || len(lookup.Candidates[0].References) != 0 { + if err != nil || lookup.Candidates[0].Status != catalog.SemanticCoverageCandidateKeepRaw || lookup.Candidates[0].Reason != catalog.SemanticCoverageReasonUnrepresentedMetadata || len(lookup.Candidates[0].References) != 0 { t.Fatalf("raw metadata facet lookup = %+v err=%v, want KEEP_RAW without inherited owner refs", lookup, err) } unauthorized := options unauthorized.AuthResourcePaths = []string{"Patient/other"} lookup, err = adapter.LookupSemanticCoverage(context.Background(), unauthorized) - if err != nil || len(lookup.Candidates) != 1 || lookup.Candidates[0].Status != SemanticCoverageCandidateKeepRaw || len(lookup.Candidates[0].References) != 0 { + if err != nil || len(lookup.Candidates) != 1 || lookup.Candidates[0].Status != catalog.SemanticCoverageCandidateKeepRaw || len(lookup.Candidates[0].References) != 0 { t.Fatalf("source-free authorized scope lookup = %+v err=%v, want proven KEEP_RAW without refs", lookup, err) } otherBuild := options otherBuild.BuildID = "stale-build" lookup, err = adapter.LookupSemanticCoverage(context.Background(), otherBuild) - if err != nil || lookup.State != SemanticCoverageBuildUnavailable || lookup.Reason != "BUILD_ID_MISMATCH" || lookup.Candidates[0].Status != SemanticCoverageCandidateUnavailable { + if err != nil || lookup.State != catalog.SemanticCoverageBuildUnavailable || lookup.Reason != "BUILD_ID_MISMATCH" || lookup.Candidates[0].Status != catalog.SemanticCoverageCandidateUnavailable { t.Fatalf("stale build lookup = %+v err=%v, want unavailable exact-build result", lookup, err) } } @@ -582,23 +580,23 @@ func TestSemanticCoverageIncompleteReplayCannotReuseCompletedMarkerOrSummary(t * } client.events = nil client.insertAttempts = 0 - client.failInsertAt = 1 // The atomic envelope replacement fails before it can be acknowledged. + client.failInsertAt = 1 contribution.Paths[0].CanonicalPath = "changedPath" if _, err := adapter.WriteSemanticCoverageContributions(context.Background(), buildID, []catalog.SemanticCoverageSourceContribution{contribution}, 10); err == nil { t.Fatal("interrupted source-set replay returned nil") } marker := client.documents[semanticCoverageBuildKey(identity)] - if marker.BuildState != SemanticCoverageBuildBuilding { + if marker.BuildState != catalog.SemanticCoverageBuildBuilding { t.Fatalf("marker after interrupted replay = %q, want BUILDING", marker.BuildState) } - lookup, err := adapter.LookupSemanticCoverage(context.Background(), SemanticCoverageLookupOptions{ + lookup, err := adapter.LookupSemanticCoverage(context.Background(), catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: buildID, SchemaIdentity: identity.SchemaIdentity, SourceKind: identity.SourceKind, - AuthResourcePaths: []string{contribution.Identity.AuthResourcePath}, Candidates: []SemanticCoverageCandidatePath{{ + AuthResourcePaths: []string{contribution.Identity.AuthResourcePath}, Candidates: []catalog.SemanticCoverageCandidatePath{{ ResourceType: contribution.Identity.ResourceType, CanonicalPath: "component[].code", }}, }) - if err != nil || lookup.State != SemanticCoverageBuildBuilding || len(lookup.Candidates[0].References) != 0 { + if err != nil || lookup.State != catalog.SemanticCoverageBuildBuilding || len(lookup.Candidates[0].References) != 0 { t.Fatalf("lookup during replay = %+v err=%v, want BUILDING with no replacement refs", lookup, err) } page, err := adapter.readSemanticCoveragePage(context.Background(), identity.Project, identity.DatasetGeneration, buildID, identity.SourceKind, identity.SchemaIdentity, semanticCoverageEnvelopeRecordKind, []string{contribution.Identity.AuthResourcePath}, false, "", 10) @@ -638,19 +636,19 @@ func TestSemanticCoverageMaterializationKeepsUnauthorizedPartitionSeparate(t *te t.Fatal(err) } completeSemanticCoverageMemoryBuild(t, adapter, identity, allowed, unauthorized) - candidate := SemanticCoverageCandidatePath{ResourceType: "Observation", CanonicalPath: "coding[].code"} - restricted, err := adapter.LookupSemanticCoverage(context.Background(), SemanticCoverageLookupOptions{ + candidate := catalog.SemanticCoverageCandidatePath{ResourceType: "Observation", CanonicalPath: "coding[].code"} + restricted, err := adapter.LookupSemanticCoverage(context.Background(), catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: buildID, SchemaIdentity: identity.SchemaIdentity, - SourceKind: identity.SourceKind, AuthResourcePaths: []string{"Patient/allowed"}, Candidates: []SemanticCoverageCandidatePath{candidate}, + SourceKind: identity.SourceKind, AuthResourcePaths: []string{"Patient/allowed"}, Candidates: []catalog.SemanticCoverageCandidatePath{candidate}, }) - if err != nil || restricted.Candidates[0].Status != SemanticCoverageCandidateGroupable || len(restricted.Candidates[0].References) == 0 { + if err != nil || restricted.Candidates[0].Status != catalog.SemanticCoverageCandidateGroupable || len(restricted.Candidates[0].References) == 0 { t.Fatalf("restricted authorized lookup = %+v err=%v, want only allowed groupable proof", restricted, err) } - unrestricted, err := adapter.LookupSemanticCoverage(context.Background(), SemanticCoverageLookupOptions{ + unrestricted, err := adapter.LookupSemanticCoverage(context.Background(), catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: buildID, SchemaIdentity: identity.SchemaIdentity, - SourceKind: identity.SourceKind, AuthResourcePathsUnrestricted: true, Candidates: []SemanticCoverageCandidatePath{candidate}, + SourceKind: identity.SourceKind, AuthResourcePathsUnrestricted: true, Candidates: []catalog.SemanticCoverageCandidatePath{candidate}, }) - if err != nil || unrestricted.Candidates[0].Status != SemanticCoverageCandidateKeepRaw || len(unrestricted.Candidates[0].References) != 0 { + if err != nil || unrestricted.Candidates[0].Status != catalog.SemanticCoverageCandidateKeepRaw || len(unrestricted.Candidates[0].References) != 0 { t.Fatalf("unrestricted mixed partition lookup = %+v err=%v, want KEEP_RAW with no cross-partition refs", unrestricted, err) } } @@ -674,12 +672,12 @@ func TestFacetProofTruncationKeepsExactCandidateRaw(t *testing.T) { t.Fatalf("write truncated facet source: %v", err) } completeSemanticCoverageMemoryBuild(t, adapter, identity, contribution) - lookup, err := adapter.LookupSemanticCoverage(context.Background(), SemanticCoverageLookupOptions{ + lookup, err := adapter.LookupSemanticCoverage(context.Background(), catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: buildID, SchemaIdentity: identity.SchemaIdentity, SourceKind: identity.SourceKind, AuthResourcePaths: []string{contribution.Identity.AuthResourcePath}, - Candidates: []SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: "coding[].code"}}, + Candidates: []catalog.SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: "coding[].code"}}, }) - if err != nil || lookup.Candidates[0].Status != SemanticCoverageCandidateKeepRaw || lookup.Candidates[0].Reason != catalog.SemanticCoverageReasonTruncatedEvidence || len(lookup.Candidates[0].References) != 0 { + if err != nil || lookup.Candidates[0].Status != catalog.SemanticCoverageCandidateKeepRaw || lookup.Candidates[0].Reason != catalog.SemanticCoverageReasonTruncatedEvidence || len(lookup.Candidates[0].References) != 0 { t.Fatalf("truncated exact facet lookup = %+v err=%v, want KEEP_RAW/TRUNCATED_EVIDENCE without refs", lookup, err) } } @@ -709,13 +707,13 @@ func TestSemanticCoverageExactFacetIgnoresRawSiblingAndOwnerDisposition(t *testi } completeSemanticCoverageMemoryBuild(t, adapter, identity, contribution) - lookup := func(path string) SemanticCoverageCandidateResult { + lookup := func(path string) catalog.SemanticCoverageCandidateResult { t.Helper() - result, err := adapter.LookupSemanticCoverage(context.Background(), SemanticCoverageLookupOptions{ + result, err := adapter.LookupSemanticCoverage(context.Background(), catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: buildID, SchemaIdentity: identity.SchemaIdentity, SourceKind: identity.SourceKind, AuthResourcePaths: []string{contribution.Identity.AuthResourcePath}, - Candidates: []SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: path}}, + Candidates: []catalog.SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: path}}, }) if err != nil { t.Fatal(err) @@ -723,15 +721,15 @@ func TestSemanticCoverageExactFacetIgnoresRawSiblingAndOwnerDisposition(t *testi return result.Candidates[0] } code := lookup("coding[].code") - if code.Status != SemanticCoverageCandidateGroupable || len(code.References) == 0 { + if code.Status != catalog.SemanticCoverageCandidateGroupable || len(code.References) == 0 { t.Fatalf("semantic code facet = %+v, want GROUPABLE with exact refs despite raw owner/sibling", code) } displayResult := lookup("coding[].display") - if displayResult.Status != SemanticCoverageCandidateGroupable || len(displayResult.References) == 0 { + if displayResult.Status != catalog.SemanticCoverageCandidateGroupable || len(displayResult.References) == 0 { t.Fatalf("semantic display facet = %+v, want GROUPABLE with exact refs despite raw owner/sibling", displayResult) } userSelected := lookup("coding[].userSelected") - if userSelected.Status != SemanticCoverageCandidateKeepRaw || userSelected.Reason != catalog.SemanticCoverageReasonUnrepresentedMetadata || len(userSelected.References) != 0 { + if userSelected.Status != catalog.SemanticCoverageCandidateKeepRaw || userSelected.Reason != catalog.SemanticCoverageReasonUnrepresentedMetadata || len(userSelected.References) != 0 { t.Fatalf("raw userSelected sibling = %+v, want KEEP_RAW without inherited refs", userSelected) } } @@ -763,19 +761,19 @@ func TestIncompletePathlessPartitionCannotBorrowProofFromAnotherAuthorizedPartit t.Fatalf("write source partitions: %v", err) } completeSemanticCoverageMemoryBuild(t, adapter, identity, complete, incomplete) - candidate := SemanticCoverageCandidatePath{ResourceType: complete.Identity.ResourceType, CanonicalPath: "coding[].code"} - restricted, err := adapter.LookupSemanticCoverage(context.Background(), SemanticCoverageLookupOptions{ + candidate := catalog.SemanticCoverageCandidatePath{ResourceType: complete.Identity.ResourceType, CanonicalPath: "coding[].code"} + restricted, err := adapter.LookupSemanticCoverage(context.Background(), catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: buildID, SchemaIdentity: identity.SchemaIdentity, - SourceKind: identity.SourceKind, AuthResourcePaths: []string{complete.Identity.AuthResourcePath}, Candidates: []SemanticCoverageCandidatePath{candidate}, + SourceKind: identity.SourceKind, AuthResourcePaths: []string{complete.Identity.AuthResourcePath}, Candidates: []catalog.SemanticCoverageCandidatePath{candidate}, }) - if err != nil || restricted.Candidates[0].Status != SemanticCoverageCandidateGroupable || len(restricted.Candidates[0].References) == 0 { + if err != nil || restricted.Candidates[0].Status != catalog.SemanticCoverageCandidateGroupable || len(restricted.Candidates[0].References) == 0 { t.Fatalf("complete auth partition lookup = %+v err=%v, want isolated complete proof", restricted, err) } - unrestricted, err := adapter.LookupSemanticCoverage(context.Background(), SemanticCoverageLookupOptions{ + unrestricted, err := adapter.LookupSemanticCoverage(context.Background(), catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: buildID, SchemaIdentity: identity.SchemaIdentity, - SourceKind: identity.SourceKind, AuthResourcePathsUnrestricted: true, Candidates: []SemanticCoverageCandidatePath{candidate}, + SourceKind: identity.SourceKind, AuthResourcePathsUnrestricted: true, Candidates: []catalog.SemanticCoverageCandidatePath{candidate}, }) - if err != nil || unrestricted.Candidates[0].Status != SemanticCoverageCandidateUnavailable || len(unrestricted.Candidates[0].References) != 0 { + if err != nil || unrestricted.Candidates[0].Status != catalog.SemanticCoverageCandidateUnavailable || len(unrestricted.Candidates[0].References) != 0 { t.Fatalf("mixed complete/pathless partition lookup = %+v err=%v, want unavailable without borrowed proof", unrestricted, err) } } @@ -803,15 +801,15 @@ func TestEmptySchemaCoverageBuildAndLookupRemainUnavailable(t *testing.T) { t.Fatalf("finalize empty-schema build: %v", err) } marker := client.documents[semanticCoverageBuildKey(identity)] - if marker.BuildState != SemanticCoverageBuildUnavailable || marker.BuildReason != "SOURCE_SCHEMA_IDENTITY_UNAVAILABLE" { + if marker.BuildState != catalog.SemanticCoverageBuildUnavailable || marker.BuildReason != "SOURCE_SCHEMA_IDENTITY_UNAVAILABLE" { t.Fatalf("empty-schema build marker = %+v, want exact unavailable reason", marker) } - lookup, err := adapter.LookupSemanticCoverage(context.Background(), SemanticCoverageLookupOptions{ + lookup, err := adapter.LookupSemanticCoverage(context.Background(), catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: buildID, SourceKind: identity.SourceKind, AuthResourcePathsUnrestricted: true, - Candidates: []SemanticCoverageCandidatePath{{ResourceType: "Observation", CanonicalPath: "coding[].code"}}, + Candidates: []catalog.SemanticCoverageCandidatePath{{ResourceType: "Observation", CanonicalPath: "coding[].code"}}, }) - if err != nil || lookup.State != SemanticCoverageBuildUnavailable || lookup.Reason != "SOURCE_SCHEMA_IDENTITY_UNAVAILABLE" || lookup.Candidates[0].Status != SemanticCoverageCandidateUnavailable || len(lookup.Candidates[0].References) != 0 { + if err != nil || lookup.State != catalog.SemanticCoverageBuildUnavailable || lookup.Reason != "SOURCE_SCHEMA_IDENTITY_UNAVAILABLE" || lookup.Candidates[0].Status != catalog.SemanticCoverageCandidateUnavailable || len(lookup.Candidates[0].References) != 0 { t.Fatalf("empty-schema lookup = %+v err=%v, want unavailable without references", lookup, err) } } @@ -861,12 +859,12 @@ func TestSemanticCoverageCandidateLookupConsidersEveryBoundarySignature(t *testi if len(seenBoundarySignatures) != 2 || !seenBoundarySignatures[""] || !seenBoundarySignatures[subtype.Paths[0].ResourceBoundarySignature] { t.Fatalf("facet summary boundary signatures = %#v, want ordinary and subtype scopes", seenBoundarySignatures) } - lookup, err := adapter.LookupSemanticCoverage(context.Background(), SemanticCoverageLookupOptions{ + lookup, err := adapter.LookupSemanticCoverage(context.Background(), catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: buildID, SchemaIdentity: identity.SchemaIdentity, SourceKind: identity.SourceKind, AuthResourcePaths: []string{ordinary.Identity.AuthResourcePath}, - Candidates: []SemanticCoverageCandidatePath{{ResourceType: "Observation", CanonicalPath: "coding[].code"}}, + Candidates: []catalog.SemanticCoverageCandidatePath{{ResourceType: "Observation", CanonicalPath: "coding[].code"}}, }) - if err != nil || lookup.Candidates[0].Status != SemanticCoverageCandidateKeepRaw || len(lookup.Candidates[0].References) != 0 { + if err != nil || lookup.Candidates[0].Status != catalog.SemanticCoverageCandidateKeepRaw || len(lookup.Candidates[0].References) != 0 { t.Fatalf("candidate lookup across ordinary and subtype summaries = %+v err=%v, want nonempty-boundary signature to block inherited grouping", lookup, err) } } @@ -934,13 +932,13 @@ func TestSemanticCoverageLookupRejectsStorageFormatMismatch(t *testing.T) { marker.CoverageStorageFormatVersion++ client.documents[semanticCoverageBuildKey(identity)] = marker - lookup, err := adapter.LookupSemanticCoverage(context.Background(), SemanticCoverageLookupOptions{ + lookup, err := adapter.LookupSemanticCoverage(context.Background(), catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: buildID, SchemaIdentity: identity.SchemaIdentity, SourceKind: identity.SourceKind, AuthResourcePaths: []string{contribution.Identity.AuthResourcePath}, - Candidates: []SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: "coding[].code"}}, + Candidates: []catalog.SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: "coding[].code"}}, }) - if err != nil || lookup.State != SemanticCoverageBuildUnavailable || lookup.Reason != "COVERAGE_FORMAT_VERSION_MISMATCH" || len(lookup.Candidates[0].References) != 0 { + if err != nil || lookup.State != catalog.SemanticCoverageBuildUnavailable || lookup.Reason != "COVERAGE_FORMAT_VERSION_MISMATCH" || len(lookup.Candidates[0].References) != 0 { t.Fatalf("format-mismatch lookup = %+v err=%v, want unavailable with no references", lookup, err) } } @@ -969,15 +967,15 @@ func TestSemanticCoverageReadFailureCannotPublishComplete(t *testing.T) { } client.queryErr = nil marker := client.documents[semanticCoverageBuildKey(identity)] - if marker.BuildState != SemanticCoverageBuildBuilding { + if marker.BuildState != catalog.SemanticCoverageBuildBuilding { t.Fatalf("marker after failed materialization read = %q, want BUILDING", marker.BuildState) } - lookup, err := adapter.LookupSemanticCoverage(context.Background(), SemanticCoverageLookupOptions{ + lookup, err := adapter.LookupSemanticCoverage(context.Background(), catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: buildID, SchemaIdentity: identity.SchemaIdentity, SourceKind: identity.SourceKind, AuthResourcePaths: []string{contribution.Identity.AuthResourcePath}, - Candidates: []SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: "coding[].code"}}, + Candidates: []catalog.SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: "coding[].code"}}, }) - if err != nil || lookup.State != SemanticCoverageBuildBuilding || len(lookup.Candidates[0].References) != 0 { + if err != nil || lookup.State != catalog.SemanticCoverageBuildBuilding || len(lookup.Candidates[0].References) != 0 { t.Fatalf("lookup after failed materialization read = %+v err=%v, want BUILDING without refs", lookup, err) } } @@ -1008,15 +1006,15 @@ func TestSemanticCoverageSummaryWriteFailureCannotPublishComplete(t *testing.T) client.failInsertAt = 0 client.insertAttempts = 0 marker := client.documents[semanticCoverageBuildKey(identity)] - if marker.BuildState != SemanticCoverageBuildBuilding { + if marker.BuildState != catalog.SemanticCoverageBuildBuilding { t.Fatalf("marker after failed summary write = %q, want BUILDING", marker.BuildState) } - lookup, err := adapter.LookupSemanticCoverage(context.Background(), SemanticCoverageLookupOptions{ + lookup, err := adapter.LookupSemanticCoverage(context.Background(), catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: buildID, SchemaIdentity: identity.SchemaIdentity, SourceKind: identity.SourceKind, AuthResourcePaths: []string{contribution.Identity.AuthResourcePath}, - Candidates: []SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: "coding[].code"}}, + Candidates: []catalog.SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: "coding[].code"}}, }) - if err != nil || lookup.State != SemanticCoverageBuildBuilding || len(lookup.Candidates[0].References) != 0 { + if err != nil || lookup.State != catalog.SemanticCoverageBuildBuilding || len(lookup.Candidates[0].References) != 0 { t.Fatalf("lookup after failed summary write = %+v err=%v, want BUILDING without refs", lookup, err) } } @@ -1024,7 +1022,7 @@ func TestSemanticCoverageSummaryWriteFailureCannotPublishComplete(t *testing.T) func TestCompleteRetainedSemanticCoverageResourceReturnsCertificationSourceRows(t *testing.T) { client := newSemanticCoverageMemoryClient() authPath := "Patient/authorized" - client.retainedSources["Observation"] = []RetainedSemanticInventoryRow{{ + client.retainedSources["Observation"] = []catalog.RetainedSemanticInventoryRow{{ Key: "source-1", AuthResourcePath: &authPath, Payload: json.RawMessage(`{"id":"observation-1"}`), }} adapter, err := New(client) @@ -1048,7 +1046,7 @@ func TestCompleteRetainedSemanticCoverageResourceReturnsCertificationSourceRows( t.Fatalf("retained certification rows=%d err=%v, want one enumerated source row", certificationRows, err) } marker := client.documents[semanticCoverageResourceKey(identity, "Observation")] - if marker.ResourceState != SemanticCoverageBuildComplete || marker.MaterializedSourceCount != certificationRows { + if marker.ResourceState != catalog.SemanticCoverageBuildComplete || marker.MaterializedSourceCount != certificationRows { t.Fatalf("retained resource marker = %+v, want one source and COMPLETE", marker) } } @@ -1064,20 +1062,20 @@ func TestSemanticCoverageLookupPropagatesErrorsAndSeparatesBuildStates(t *testin if err := adapter.BeginSemanticCoverageBuild(context.Background(), identity); err != nil { t.Fatal(err) } - options := SemanticCoverageLookupOptions{ + options := catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: buildID, SchemaIdentity: identity.SchemaIdentity, SourceKind: identity.SourceKind, AuthResourcePathsUnrestricted: true, - Candidates: []SemanticCoverageCandidatePath{{ResourceType: "Observation", CanonicalPath: "code"}}, + Candidates: []catalog.SemanticCoverageCandidatePath{{ResourceType: "Observation", CanonicalPath: "code"}}, } lookup, err := adapter.LookupSemanticCoverage(context.Background(), options) - if err != nil || lookup.State != SemanticCoverageBuildBuilding || lookup.Candidates[0].Status != SemanticCoverageCandidateUnavailable { + if err != nil || lookup.State != catalog.SemanticCoverageBuildBuilding || lookup.Candidates[0].Status != catalog.SemanticCoverageCandidateUnavailable { t.Fatalf("building lookup = %+v err=%v", lookup, err) } if err := adapter.FailSemanticCoverageBuild(context.Background(), identity, "injected source read failure"); err != nil { t.Fatal(err) } lookup, err = adapter.LookupSemanticCoverage(context.Background(), options) - if err != nil || lookup.State != SemanticCoverageBuildFailed || lookup.Reason != "injected source read failure" { + if err != nil || lookup.State != catalog.SemanticCoverageBuildFailed || lookup.Reason != "injected source read failure" { t.Fatalf("failed lookup = %+v err=%v", lookup, err) } client.queryErr = errors.New("injected query error") @@ -1093,12 +1091,12 @@ func TestSemanticCoverageUnrestrictedLookupBindsAnEmptyAuthArray(t *testing.T) { t.Fatal(err) } project, generation := "project", "generation" - _, err = adapter.LookupSemanticCoverage(context.Background(), SemanticCoverageLookupOptions{ + _, err = adapter.LookupSemanticCoverage(context.Background(), catalog.SemanticCoverageLookupOptions{ Project: project, DatasetGeneration: generation, BuildID: catalog.SemanticInventoryBuildID(project, generation), SchemaIdentity: "schema", SourceKind: catalog.SemanticInventorySourceRetained, AuthResourcePathsUnrestricted: true, - Candidates: []SemanticCoverageCandidatePath{{ResourceType: "Specimen", CanonicalPath: "status"}}, + Candidates: []catalog.SemanticCoverageCandidatePath{{ResourceType: "Specimen", CanonicalPath: "status"}}, }) if err != nil { t.Fatalf("unrestricted lookup: %v", err) @@ -1138,16 +1136,16 @@ func TestWriteSemanticCoverageFailureKeepsBuildUnavailableAfterReplayStarts(t *t if err == nil || !strings.Contains(err.Error(), "replace semantic coverage source envelopes") || stats.Rows != 0 { t.Fatalf("failed source replacement stats=%#v err=%v, want atomic envelope failure", stats, err) } - if marker := client.documents[semanticCoverageBuildKey(identity)]; marker.BuildState != SemanticCoverageBuildBuilding { + if marker := client.documents[semanticCoverageBuildKey(identity)]; marker.BuildState != catalog.SemanticCoverageBuildBuilding { t.Fatalf("build marker after failed replay = %+v, want BUILDING", marker) } - lookup, err := adapter.LookupSemanticCoverage(context.Background(), SemanticCoverageLookupOptions{ + lookup, err := adapter.LookupSemanticCoverage(context.Background(), catalog.SemanticCoverageLookupOptions{ Project: identity.Project, DatasetGeneration: identity.DatasetGeneration, BuildID: identity.BuildID, SchemaIdentity: identity.SchemaIdentity, SourceKind: identity.SourceKind, AuthResourcePaths: []string{contribution.Identity.AuthResourcePath}, - Candidates: []SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: "coding[].code"}}, + Candidates: []catalog.SemanticCoverageCandidatePath{{ResourceType: contribution.Identity.ResourceType, CanonicalPath: "coding[].code"}}, }) - if err != nil || lookup.State != SemanticCoverageBuildBuilding || len(lookup.Candidates[0].References) != 0 { + if err != nil || lookup.State != catalog.SemanticCoverageBuildBuilding || len(lookup.Candidates[0].References) != 0 { t.Fatalf("lookup after failed replay = %+v err=%v, want BUILDING and no refs", lookup, err) } } @@ -1370,7 +1368,7 @@ func (c *semanticCoverageTestClient) Bootstrap(_ context.Context, spec store.Boo type semanticCoverageMemoryClient struct { documents map[string]semanticCoverageStoredDocument - retainedSources map[string][]RetainedSemanticInventoryRow + retainedSources map[string][]catalog.RetainedSemanticInventoryRow events []string insertAttempts int failInsertAt int @@ -1380,7 +1378,7 @@ type semanticCoverageMemoryClient struct { func newSemanticCoverageMemoryClient() *semanticCoverageMemoryClient { return &semanticCoverageMemoryClient{ documents: make(map[string]semanticCoverageStoredDocument), - retainedSources: make(map[string][]RetainedSemanticInventoryRow), + retainedSources: make(map[string][]catalog.RetainedSemanticInventoryRow), } } @@ -1392,7 +1390,7 @@ func (c *semanticCoverageMemoryClient) QueryRows(_ context.Context, query string collection := vars["@collection"].(string) afterKey := vars["after_key"].(string) limit := vars["limit"].(int) - rows := append([]RetainedSemanticInventoryRow(nil), c.retainedSources[collection]...) + rows := append([]catalog.RetainedSemanticInventoryRow(nil), c.retainedSources[collection]...) sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key }) count := 0 for _, source := range rows { diff --git a/internal/catalog/arango/semantic_inventory.go b/internal/backends/arango/internal/catalog/semantic_inventory.go similarity index 95% rename from internal/catalog/arango/semantic_inventory.go rename to internal/backends/arango/internal/catalog/semantic_inventory.go index b3d3127a6..4118d7cce 100644 --- a/internal/catalog/arango/semantic_inventory.go +++ b/internal/backends/arango/internal/catalog/semantic_inventory.go @@ -9,9 +9,9 @@ import ( "strings" "time" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/catalog" fhirschema "github.com/calypr/loom/internal/fhir/schema" - store "github.com/calypr/loom/internal/store/arango" ) var ( @@ -23,20 +23,6 @@ var ( ErrRetainedSemanticSourceMissing = errors.New("retained semantic inventory source is missing despite field-profile evidence") ) -type RetainedSemanticInventoryRow struct { - Key string `json:"key"` - AuthResourcePath *string `json:"auth_resource_path"` - Payload json.RawMessage `json:"payload"` -} - -type RetainedSemanticInventoryPage struct { - Rows []RetainedSemanticInventoryRow - HasMore bool - SourceExists bool -} - -// PrepareSemanticInventoryBackfill bootstraps only the derived inventory -// collections. Retained resource collections are never created or changed. func (s *Store) PrepareSemanticInventoryBackfill(ctx context.Context) error { return s.client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ {Name: catalog.SemanticInventoryCollection, Indexes: [][]string{ @@ -53,29 +39,26 @@ func (s *Store) PrepareSemanticInventoryBackfill(ctx context.Context) error { }}) } -// ReadRetainedSemanticInventoryPage returns at most limit retained source -// vertices in stable _key order, without loading a resource collection into -// application memory. -func (s *Store) ReadRetainedSemanticInventoryPage(ctx context.Context, project, generation, collection, afterKey string, limit int) (RetainedSemanticInventoryPage, error) { +func (s *Store) ReadRetainedSemanticInventoryPage(ctx context.Context, project, generation, collection, afterKey string, limit int) (catalog.RetainedSemanticInventoryPage, error) { canonical, ok := fhirschema.ConcreteResourceType(collection) if !ok || canonical != collection || strings.TrimSpace(project) == "" || catalog.NormalizeDatasetGeneration(generation) == "" || limit < 1 || limit > 1000 { - return RetainedSemanticInventoryPage{}, ErrInvalidSemanticInventoryPage + return catalog.RetainedSemanticInventoryPage{}, ErrInvalidSemanticInventoryPage } exists, err := s.client.CollectionExists(ctx, collection) if err != nil { - return RetainedSemanticInventoryPage{}, err + return catalog.RetainedSemanticInventoryPage{}, err } if !exists { profileEvidence, err := s.retainedSourceFieldProfileExists(ctx, project, generation, collection) if err != nil { - return RetainedSemanticInventoryPage{}, err + return catalog.RetainedSemanticInventoryPage{}, err } if profileEvidence { - return RetainedSemanticInventoryPage{}, fmt.Errorf("%w: %s/%s/%s", ErrRetainedSemanticSourceMissing, project, generation, collection) + return catalog.RetainedSemanticInventoryPage{}, fmt.Errorf("%w: %s/%s/%s", ErrRetainedSemanticSourceMissing, project, generation, collection) } - return RetainedSemanticInventoryPage{Rows: []RetainedSemanticInventoryRow{}}, nil + return catalog.RetainedSemanticInventoryPage{Rows: []catalog.RetainedSemanticInventoryRow{}}, nil } - page := RetainedSemanticInventoryPage{Rows: make([]RetainedSemanticInventoryRow, 0, limit), SourceExists: true} + page := catalog.RetainedSemanticInventoryPage{Rows: make([]catalog.RetainedSemanticInventoryRow, 0, limit), SourceExists: true} err = s.client.QueryRows(ctx, retainedSemanticInventorySourcePageAQL, limit+1, map[string]interface{}{ "@collection": collection, "project": project, @@ -83,7 +66,7 @@ func (s *Store) ReadRetainedSemanticInventoryPage(ctx context.Context, project, "after_key": afterKey, "limit": limit + 1, }, func(row map[string]any) error { - var source RetainedSemanticInventoryRow + var source catalog.RetainedSemanticInventoryRow if err := decodeInventoryRow(row, &source); err != nil { return err } @@ -94,7 +77,7 @@ func (s *Store) ReadRetainedSemanticInventoryPage(ctx context.Context, project, return nil }) if err != nil { - return RetainedSemanticInventoryPage{}, fmt.Errorf("read retained %s source page: %w", collection, err) + return catalog.RetainedSemanticInventoryPage{}, fmt.Errorf("read retained %s source page: %w", collection, err) } if len(page.Rows) > limit { page.Rows = page.Rows[:limit] @@ -103,10 +86,10 @@ func (s *Store) ReadRetainedSemanticInventoryPage(ctx context.Context, project, if len(page.Rows) == 0 && afterKey == "" { profileEvidence, err := s.retainedSourceFieldProfileExists(ctx, project, generation, collection) if err != nil { - return RetainedSemanticInventoryPage{}, err + return catalog.RetainedSemanticInventoryPage{}, err } if profileEvidence { - return RetainedSemanticInventoryPage{}, fmt.Errorf("%w: %s/%s/%s has no retained rows", ErrRetainedSemanticSourceMissing, project, generation, collection) + return catalog.RetainedSemanticInventoryPage{}, fmt.Errorf("%w: %s/%s/%s has no retained rows", ErrRetainedSemanticSourceMissing, project, generation, collection) } } return page, nil @@ -127,10 +110,6 @@ func (s *Store) retainedSourceFieldProfileExists(ctx context.Context, project, g return found, err } -// ClaimSemanticInventoryBackfill atomically acquires or resumes the retained -// source scanner. A complete build is returned with claimed=false and is not -// rewritten, which prevents file and retained-vertex contribution sets from -// being mixed by a second scan. func (s *Store) ClaimSemanticInventoryBackfill(ctx context.Context, build catalog.SemanticInventoryBuild, token string, lease time.Duration) (catalog.SemanticInventoryBuild, bool, error) { if err := catalog.ValidateSemanticInventoryBuild(build); err != nil { return catalog.SemanticInventoryBuild{}, false, err @@ -176,9 +155,6 @@ func (s *Store) ClaimSemanticInventoryBackfill(ctx context.Context, build catalo return catalog.SemanticInventoryBuild{}, false, ErrSemanticInventoryClaimed } -// RefreshSemanticInventorySourceAvailability upgrades only the source-coverage -// metadata of an unchanged completed retained build. Callers must independently -// prove the generation's source counts before requesting this compare-and-set. func (s *Store) RefreshSemanticInventorySourceAvailability(ctx context.Context, expected catalog.SemanticInventoryBuild) (catalog.SemanticInventoryBuild, bool, error) { if err := catalog.ValidateSemanticInventoryBuild(expected); err != nil { return catalog.SemanticInventoryBuild{}, false, err @@ -509,9 +485,6 @@ func (s *Store) PageSemanticInventory(ctx context.Context, opts catalog.Semantic return page, nil } -// ResolveSemanticInventorySelections resolves only requested catalog identities -// under the caller's current authorization scope. It never pages or returns the -// wider inventory to the authoring path. func (s *Store) ResolveSemanticInventorySelections(ctx context.Context, opts catalog.SemanticInventoryResolveOptions) (catalog.SemanticInventoryResolveResult, error) { selectionCount := len(opts.References) + len(opts.FamilyReferences) if strings.TrimSpace(opts.Project) == "" || catalog.NormalizeDatasetGeneration(opts.DatasetGeneration) == "" || opts.AuthResourcePathsUnrestricted == nil || selectionCount == 0 || selectionCount > 100 { diff --git a/internal/catalog/arango/semantic_inventory_category_test.go b/internal/backends/arango/internal/catalog/semantic_inventory_category_test.go similarity index 100% rename from internal/catalog/arango/semantic_inventory_category_test.go rename to internal/backends/arango/internal/catalog/semantic_inventory_category_test.go diff --git a/internal/catalog/arango/semantic_inventory_integration_test.go b/internal/backends/arango/internal/catalog/semantic_inventory_integration_test.go similarity index 98% rename from internal/catalog/arango/semantic_inventory_integration_test.go rename to internal/backends/arango/internal/catalog/semantic_inventory_integration_test.go index 01c7c5507..9459adca3 100644 --- a/internal/catalog/arango/semantic_inventory_integration_test.go +++ b/internal/backends/arango/internal/catalog/semantic_inventory_integration_test.go @@ -16,8 +16,8 @@ import ( "testing" "time" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/catalog" - arangostore "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -206,8 +206,6 @@ func TestSemanticInventoryPersistsAndPagesAcrossAuthorizedPaths(t *testing.T) { t.Fatalf("restricted failed page leaked progress: %+v", failedPage.Build) } - // Replay the same source files: per-source event keys replace their prior - // rows. The source files live in t.TempDir and the collection is never cleared. for _, fixture := range fixtures { if err := ingestSemanticInventoryFixture(ctx, store, project, generation, fixture.scope, fixtureRoot, fixtureFiles[fixture.scope]); err != nil { t.Fatal(err) @@ -383,9 +381,6 @@ func TestSemanticInventoryFallbackPolicySurvivesMaterializationAndReadback(t *te } } -// This mirrors the binding-v2 digest input and length-prefix encoding, then -// verifies itself against a contribution from the real semantic emitter -// before creating a second ordered-policy identity for the storage test. func semanticBindingIDForIntegrationTest(observation catalog.SemanticObservation) string { parts := []string{ "semantic-binding/v2", diff --git a/internal/catalog/arango/semantic_inventory_selection_test.go b/internal/backends/arango/internal/catalog/semantic_inventory_selection_test.go similarity index 100% rename from internal/catalog/arango/semantic_inventory_selection_test.go rename to internal/backends/arango/internal/catalog/semantic_inventory_selection_test.go diff --git a/internal/catalog/arango/semantic_inventory_source_availability_test.go b/internal/backends/arango/internal/catalog/semantic_inventory_source_availability_test.go similarity index 100% rename from internal/catalog/arango/semantic_inventory_source_availability_test.go rename to internal/backends/arango/internal/catalog/semantic_inventory_source_availability_test.go diff --git a/internal/catalog/arango/store.go b/internal/backends/arango/internal/catalog/store.go similarity index 77% rename from internal/catalog/arango/store.go rename to internal/backends/arango/internal/catalog/store.go index b9c9972c0..0bacaed89 100644 --- a/internal/catalog/arango/store.go +++ b/internal/backends/arango/internal/catalog/store.go @@ -1,11 +1,10 @@ -// Package arango contains the Arango persistence adapter for catalog facts. package arango import ( "context" "fmt" - store "github.com/calypr/loom/internal/store/arango" + store "github.com/calypr/loom/internal/backends/arango/internal/store" ) type client interface { diff --git a/internal/catalog/arango/store_test.go b/internal/backends/arango/internal/catalog/store_test.go similarity index 99% rename from internal/catalog/arango/store_test.go rename to internal/backends/arango/internal/catalog/store_test.go index 6afaf6cde..d4b5cd728 100644 --- a/internal/catalog/arango/store_test.go +++ b/internal/backends/arango/internal/catalog/store_test.go @@ -11,8 +11,8 @@ import ( "testing" "time" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/catalog" - store "github.com/calypr/loom/internal/store/arango" ) type fakeClient struct { @@ -376,7 +376,7 @@ func TestReadEmptyInitialRetainedSourceFailsWhenGenerationFieldProfileProvesPopu t.Fatalf("empty profiled source error = %v, want ErrRetainedSemanticSourceMissing", err) } if !strings.Contains(retainedSemanticInventoryFieldProfileAQL, "RETURN {doc_count: d.doc_count}") { - t.Fatalf("field-profile query must return an object row for QueryRows: %s", retainedSemanticInventoryFieldProfileAQL) + t.Fatalf("field-profile query must return an object row for SourceRows: %s", retainedSemanticInventoryFieldProfileAQL) } } diff --git a/internal/dataframe/execution/chartifact/arango/catalog.go b/internal/backends/arango/internal/chartifact/catalog.go similarity index 87% rename from internal/dataframe/execution/chartifact/arango/catalog.go rename to internal/backends/arango/internal/chartifact/catalog.go index a856bfbc8..d3a8c1182 100644 --- a/internal/dataframe/execution/chartifact/arango/catalog.go +++ b/internal/backends/arango/internal/chartifact/catalog.go @@ -1,6 +1,3 @@ -// Package arango stores private ClickHouse artifact manifests and leases in a -// dedicated Arango collection. It does not register artifacts as published -// outputs or update any publication pointer. package arango import ( @@ -11,8 +8,8 @@ import ( "strings" "time" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/dataframe/execution/chartifact" - arangostore "github.com/calypr/loom/internal/store/arango" ) const ( @@ -22,8 +19,6 @@ const ( manifestTimeLayout = "2006-01-02T15:04:05.000000000Z" ) -// Catalog implements chartifact.Catalog using single-document conditional -// AQL writes. The Arango document key is the immutable ArtifactID. type Catalog struct { client arangostore.RowQueryer } @@ -44,9 +39,6 @@ func BootstrapSpec() arangostore.BootstrapSpec { }}} } -// Create inserts an immutable manifest only when its ArtifactID is unused. -// Concurrent duplicate creates can produce an Arango write-conflict error; -// neither path replaces an existing manifest. func (c *Catalog) Create(ctx context.Context, manifest chartifact.Manifest) error { if err := requireContext(ctx); err != nil { return err @@ -79,9 +71,6 @@ RETURN {created: true}`, queryBatchSize, map[string]any{ return nil } -// Update changes only progress fields. Active states cannot shorten a lease; -// cleanup-pending is allowed to make a table immediately reclaimable after the -// owner has stopped using it. An expired or stolen lease cannot be updated. func (c *Catalog) Update(ctx context.Context, artifactID, owner string, progress chartifact.Progress) error { if err := requireContext(ctx); err != nil { return err @@ -127,8 +116,6 @@ RETURN {updated: true}`, queryBatchSize, map[string]any{ return nil } -// Renew extends an active, unexpired lease owned by owner. It never shortens -// an existing lease and never resurrects an expired one. func (c *Catalog) Renew(ctx context.Context, artifactID, owner string, until time.Time) (bool, error) { if err := requireContext(ctx); err != nil { return false, err @@ -160,8 +147,6 @@ RETURN {renewed: true}`, queryBatchSize, map[string]any{ return renewed, nil } -// ListExpired returns a bounded, stable-order page of manifests whose lease is -// expired at or before expiredBefore. Cleanup claims recheck this condition. func (c *Catalog) ListExpired(ctx context.Context, expiredBefore time.Time, limit int) ([]chartifact.Manifest, error) { if err := requireContext(ctx); err != nil { return nil, err @@ -198,9 +183,6 @@ RETURN manifest`, queryBatchSize, map[string]any{ return manifests, nil } -// ClaimCleanup atomically changes an expired manifest's owner and state. The -// old lease timestamp is rechecked in the write query so a concurrent renewer -// wins cleanly; only one cleanup owner can claim a given expired manifest. func (c *Catalog) ClaimCleanup(ctx context.Context, artifactID, owner string, expiredBefore, until time.Time) (chartifact.Manifest, bool, error) { if err := requireContext(ctx); err != nil { return chartifact.Manifest{}, false, err @@ -246,7 +228,6 @@ RETURN {manifest: NEW}`, queryBatchSize, map[string]any{ return *claimed, true, nil } -// Delete removes a cleanup-pending manifest only for its current owner. func (c *Catalog) Delete(ctx context.Context, artifactID, owner string) error { if err := requireContext(ctx); err != nil { return err @@ -278,8 +259,6 @@ RETURN {deleted: true}`, queryBatchSize, map[string]any{ return nil } -// ReleaseCleanup makes a failed cleanup immediately reclaimable, but only -// while the caller still owns the cleanup lease. func (c *Catalog) ReleaseCleanup(ctx context.Context, artifactID, owner string, until time.Time) error { if err := requireContext(ctx); err != nil { return err @@ -338,9 +317,6 @@ func manifestDocument(manifest chartifact.Manifest) (map[string]any, error) { return nil, err } document["_key"] = manifest.ArtifactID - // time.Time's default JSON encoding omits trailing fractional zeros. These - // fields are compared and sorted lexicographically by AQL, so persist a - // fixed-width UTC representation to keep string order chronological. document["leaseUntil"] = formatManifestTime(manifest.LeaseUntil) document["createdAt"] = formatManifestTime(manifest.CreatedAt) document["updatedAt"] = formatManifestTime(manifest.UpdatedAt) diff --git a/internal/dataframe/execution/chartifact/arango/catalog_arango_test.go b/internal/backends/arango/internal/chartifact/catalog_arango_test.go similarity index 95% rename from internal/dataframe/execution/chartifact/arango/catalog_arango_test.go rename to internal/backends/arango/internal/chartifact/catalog_arango_test.go index 481ce0f77..c9dc6826c 100644 --- a/internal/dataframe/execution/chartifact/arango/catalog_arango_test.go +++ b/internal/backends/arango/internal/chartifact/catalog_arango_test.go @@ -10,14 +10,11 @@ import ( "github.com/arangodb/go-driver/v2/arangodb/shared" "github.com/calypr/loom/internal/authscope" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/dataframe/execution/chartifact" - arangostore "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) -// This opt-in test exercises Arango's conditional document writes. It uses -// unique keys in the dedicated private manifest collection and never truncates -// or drops that collection. func TestCatalogLeaseArbitrationAgainstArango(t *testing.T) { url, database := os.Getenv("LOOM_TEST_ARANGO_URL"), os.Getenv("LOOM_TEST_ARANGO_DATABASE") if url == "" || database == "" { @@ -108,9 +105,6 @@ func TestCatalogLeaseArbitrationAgainstArango(t *testing.T) { t.Fatalf("same-state cleanup retry did not make artifact reclaimable: %v", err) } - // Concurrent claimers race on the same expired manifest. The AQL update - // predicate must permit one owner at most; the loser may observe no row or - // an Arango write conflict, but must never also own the manifest. type claimResult struct { manifest chartifact.Manifest ok bool @@ -137,7 +131,6 @@ func TestCatalogLeaseArbitrationAgainstArango(t *testing.T) { for result := range results { if result.err != nil { if shared.IsArangoErrorWithErrorNum(result.err, shared.ErrArangoConflict) { - // A stale revision is a valid losing outcome for the conditional write. continue } t.Fatalf("cleanup claim failed outside expected revision conflict: %v", result.err) @@ -276,8 +269,6 @@ func TestCatalogRenewalAndExpiredCleanupClaimHaveOneWinnerAgainstArango(t *testi } } - // A cleanup-owner removal racing a reclaim must either remove the row - // before transfer or lose the revision/owner check after transfer. deleteRaceNow := time.Now().UTC() manifest.State = chartifact.StateCleanupPending manifest.LeaseOwner = writer diff --git a/internal/dataframe/execution/chartifact/arango/catalog_test.go b/internal/backends/arango/internal/chartifact/catalog_test.go similarity index 99% rename from internal/dataframe/execution/chartifact/arango/catalog_test.go rename to internal/backends/arango/internal/chartifact/catalog_test.go index 1df315082..d6d616421 100644 --- a/internal/dataframe/execution/chartifact/arango/catalog_test.go +++ b/internal/backends/arango/internal/chartifact/catalog_test.go @@ -9,8 +9,8 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/dataframe/execution/chartifact" - arangostore "github.com/calypr/loom/internal/store/arango" clickhousestore "github.com/calypr/loom/internal/store/clickhouse" ) diff --git a/internal/backends/arango/internal/dataset/collections.go b/internal/backends/arango/internal/dataset/collections.go new file mode 100644 index 000000000..833c22546 --- /dev/null +++ b/internal/backends/arango/internal/dataset/collections.go @@ -0,0 +1,21 @@ +package arango + +import arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" + +const ( + LifecycleCollection = "loom_dataset_lifecycle" + + manifestRecordType = "manifest" + activeRecordType = "active_generation" + snapshotRecordType = "snapshot_generation" + releaseRecordType = "project_release" + activeReleaseRecordType = "active_project_release" +) + +func CollectionSpecs() []arangostore.CollectionSpec { + return []arangostore.CollectionSpec{{Name: LifecycleCollection}} +} + +func BootstrapSpec() arangostore.BootstrapSpec { + return arangostore.BootstrapSpec{Collections: CollectionSpecs()} +} diff --git a/internal/backends/arango/internal/dataset/doc.go b/internal/backends/arango/internal/dataset/doc.go new file mode 100644 index 000000000..8ec2fcae5 --- /dev/null +++ b/internal/backends/arango/internal/dataset/doc.go @@ -0,0 +1 @@ +package arango diff --git a/internal/dataset/arango/document_codec.go b/internal/backends/arango/internal/dataset/document_codec.go similarity index 96% rename from internal/dataset/arango/document_codec.go rename to internal/backends/arango/internal/dataset/document_codec.go index cf566e498..d79589d08 100644 --- a/internal/dataset/arango/document_codec.go +++ b/internal/backends/arango/internal/dataset/document_codec.go @@ -153,9 +153,6 @@ func manifestDocumentKey(ref publication.Ref) string { return documentKey("manifest", ref.Project, ref.Generation) } -// ManifestDocumentKey and ActiveDocumentKey deliberately expose only the -// minimum composite-transaction identities needed by Loom-owned aggregates. -// Callers must not reimplement the private document-key algorithm. func ManifestDocumentKey(ref publication.Ref) string { return manifestDocumentKey(ref) } func activeDocumentKey(project string) string { return documentKey("active", project) } diff --git a/internal/dataset/arango/generation_activation_cas_e2e_test.go b/internal/backends/arango/internal/dataset/generation_activation_cas_e2e_test.go similarity index 56% rename from internal/dataset/arango/generation_activation_cas_e2e_test.go rename to internal/backends/arango/internal/dataset/generation_activation_cas_e2e_test.go index 31e65990c..52f9d333b 100644 --- a/internal/dataset/arango/generation_activation_cas_e2e_test.go +++ b/internal/backends/arango/internal/dataset/generation_activation_cas_e2e_test.go @@ -1,104 +1,5 @@ package arango -// VERIFICATION-ONLY artifact for FINDINGS.md Round 11 candidate F-B17. It never -// mutates product code, and it touches no other verification artifact (in this -// package revision_inflation_e2e_test.go is owned by another worker; the -// clickhouse package is out of scope). It creates a fully owned ArangoDB -// database (loom_verify_gac_) through the production store.Open path and -// drops it with an HTTP DELETE in t.Cleanup, logging removal. -// -// CLAIM UNDER TEST (F-B17): Store.Activate's AQL is a CAS-less blind patch of -// the shared active_generation pointer, and the manifest-generation activation -// path is a check-then-act (TOCTOU) with no revalidation, so a cross-generation -// activation always wins with err=nil. -// -// SOURCE READING (verbatim, pinned at runtime by gacCitations below): -// -// internal/dataset/arango/manifest_lifecycle.go:318-337 activateAQL: -// LET candidate = FIRST(... FILTER manifest.state == @ready_state OR staged -// FILTER manifest.schemaIdentity == @schema_identity ...) -// LET active = DOCUMENT(@@lifecycle_collection, @active_key) -// FILTER candidate != null / active != null / recordType / project -// UPDATE active WITH { dataset: candidate.dataset, manifestKey: candidate._key } -// IN @@lifecycle_collection -// OPTIONS { ignoreRevs: false, mergeObjects: false } -// RETURN { dataset: NEW.dataset } -// -// There is NO FILTER on the pointer's current value (no -// `FILTER active.dataset == @candidate.dataset`, no -// `FILTER active.manifestKey != @candidate_key`), NO expected-revision bind -// (the release path's `@expected_revision` has no counterpart here) and NO -// lease. `ignoreRevs: false` cannot supply the missing compare: the patch -// carries no `_rev`, so intra-query optimistic locking has nothing to -// compare (and even when it fires, it only covers the DOCUMENT()->UPDATE -// sliver *inside one statement*, never the read-then-act window in Go). -// internal/dataset/arango/manifest_lifecycle.go:198-204 (read-then-act -// idempotency pre-check, a separate HTTP round trip before the write): -// if active, err := s.ResolveActiveManifest(ctx, candidate.Dataset.Project); err == nil { -// if manifestIdentityEqual(active, candidate) { -// return nil -// } -// } -// -// The ONLY rejection Activate can return for a live staged candidate is -// :242-243 ErrActivationConflict, whose message proves it is about the -// CANDIDATE, never about the pointer having moved: "candidate %s/%s was not a -// persisted staged manifest with a valid active pointer". -// -// THREE WRITERS OF THE SHARED active_generation POINTER (parent-enumerated from -// source, not parroted: every INSERT/UPDATE against LifecycleCollection): -// 1. activateAQL UPDATE (manifest_lifecycle.go:334) <- THIS claim (F-B17), the -// manifest-generation activation path driven here. -// 2. activateReleaseAQL generationPointer patch (release_store.go:170-173) <- -// F-B13's already-confirmed lane (CompareAndSwapActivateRelease); NOT driven -// here, kept distinct on purpose. -// 3. composite repository activation (internal/explorer/arango/activation.go:98 -// via activationUpdateAQL :36-39) — same {dataset, manifestKey} patch, with -// no OPTIONS at all (ignoreRevs defaults to true). -// -// All three patch the pointer without comparing its previous value. -// -// REACHABILITY / LAYER HONESTY. The bulk-load Service -// (internal/api/bulk/load/service.go) cannot be driven from this package: its -// activateGeneration needs the dataframe release/execution pipeline plus the -// load pipeline, and it is a thin caller of the store surface under test. So the -// bulk-load pattern is replicated at exactly the line the claim names — -// service.go:206 ReadManifest (resolve candidate) / :210 IsStaged gate / :213 -// Activate — with no active-pointer read between :206 and :213 (the :199-204 -// gate it does perform compares the DATAFRAME execution pointer, not the -// generation pointer). LEG B is a DETERMINISTIC single-goroutine interleaving of -// two competing activations, not two racing OS threads: the store holds no lock -// and no compare, so there is no race to win — the interleaving is exactly what -// concurrent bulk loads produce, and the deterministic form keeps the evidence -// reproducible. That this is a simulation of the interleaving (and not a -// thread-scheduler artifact) is residual uncertainty recorded in the verdict. -// -// PRE-STATED EXPECTATIONS (declared BEFORE any Activate call executes): -// - LEG A (last-writer-wins / silent flip, cross-generation): -// Activate(G1) -> pointer names G1, err=nil. -// Activate(G2) -> pointer names G2, err=nil, _rev churns. -// Activate(G1) AGAIN (switch back to a previously-active generation) -> -// PRE-STATED EXPECT (BUG): err=nil, NO ErrActivationConflict, pointer back -// at G1, _rev churned. The pointer follows the last caller with no conflict. -// - LEG B (lost update / check-then-act): reader resolves active (sees G1), -// reader resolves its candidate manifest (service.go:206), THEN a competing -// activation moves the pointer to another generation, THEN the reader calls -// Activate (service.go:213). PRE-STATED EXPECT (BUG): err=nil, pointer lands -// on the reader's candidate, and the competing activation is silently -// discarded — no error, no revalidation of the stale read. -// - LEG C (sound leg, control): re-Activate the SAME already-active generation -// twice. PRE-STATED EXPECT (SOUND): both calls hit the idempotent no-op -// branch (manifest_lifecycle.go:198-204), err=nil, exactly ONE round trip, -// NO activateAQL leg, and pointer _rev does NOT churn. Same-generation -// re-activation must be observably different from cross-generation flipping. -// - VERDICT: CONFIRMED if any cross-generation overwrite is blind (err=nil, -// pointer moved, no compare anywhere) => t.Fatalf FAILED-ON-PURPOSE quoting -// activateAQL in full and contrasting the release-path CAS shape at -// release_store.go:167 (`FILTER currentRevision == @expected_revision OR -// alreadyActive`), which is the shape a fix would take. PREVENTED if a -// cross-generation Activate is refused => log the refusing guard, PASS. -// Anything else => INCONCLUSIVE. - import ( "context" "encoding/json" @@ -112,47 +13,35 @@ import ( "testing" "time" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" publication "github.com/calypr/loom/internal/dataset" - arangostore "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) const gacDatabasePrefix = "loom_verify_gac_" -// --------------------------------------------------------------------------- -// citation pinning: the verdict must not silently survive a refactor -// --------------------------------------------------------------------------- - type gacCitation struct { file string - line int // 1-based + line int want string } var gacCitations = []gacCitation{ - // the read-then-act idempotency pre-check (the "check" half of check-then-act) - {"internal/dataset/arango/manifest_lifecycle.go", 198, `if active, err := s.ResolveActiveManifest(ctx, candidate.Dataset.Project); err == nil {`}, - {"internal/dataset/arango/manifest_lifecycle.go", 199, `if manifestIdentityEqual(active, candidate) {`}, - {"internal/dataset/arango/manifest_lifecycle.go", 200, `return nil`}, - {"internal/dataset/arango/manifest_lifecycle.go", 202, `} else if !errors.Is(err, publication.ErrNoActiveGeneration) {`}, - // the only rejection shape: about the candidate, never about the pointer moving - {"internal/dataset/arango/manifest_lifecycle.go", 243, `return fmt.Errorf("%w: candidate %s/%s was not a persisted staged manifest with a valid active pointer", ErrActivationConflict, candidate.Dataset.Project, candidate.Dataset.Generation)`}, - // the CAS-less blind patch (the "act" half) — the line that must lack a compare - {"internal/dataset/arango/manifest_lifecycle.go", 334, `UPDATE active WITH { dataset: candidate.dataset, manifestKey: candidate._key } IN @@lifecycle_collection`}, - {"internal/dataset/arango/manifest_lifecycle.go", 335, `OPTIONS { ignoreRevs: false, mergeObjects: false }`}, - // the fix shape that exists elsewhere in the same package (contrast) - {"internal/dataset/arango/release_store.go", 167, `FILTER currentRevision == @expected_revision OR alreadyActive`}, - // bulk-load check-then-act: candidate resolved, only IsStaged rechecked, then Activate - {"internal/api/bulk/load/service.go", 206, `manifest, err := s.generationActivator.ReadManifest(ctx, ref)`}, - {"internal/api/bulk/load/service.go", 210, `if !manifest.IsStaged() {`}, - {"internal/api/bulk/load/service.go", 213, `if err := s.generationActivator.Activate(ctx, manifest); err != nil {`}, - // third writer of the same pointer document (writer inventory) - {"internal/explorer/arango/activation.go", 98, `if err := updateActivationDocument(txCtx, tx, datasetarango.LifecycleCollection, activeKey, map[string]any{"dataset": dataset, "manifestKey": manifestKey}); err != nil {`}, - {"internal/explorer/arango/activation.go", 38, `UPDATE d WITH @patch IN @@c`}, + {"internal/backends/arango/internal/dataset/manifest_lifecycle.go", 191, `if active, err := s.ResolveActiveManifest(ctx, candidate.Dataset.Project); err == nil {`}, + {"internal/backends/arango/internal/dataset/manifest_lifecycle.go", 192, `if manifestIdentityEqual(active, candidate) {`}, + {"internal/backends/arango/internal/dataset/manifest_lifecycle.go", 193, `return nil`}, + {"internal/backends/arango/internal/dataset/manifest_lifecycle.go", 195, `} else if !errors.Is(err, publication.ErrNoActiveGeneration) {`}, + {"internal/backends/arango/internal/dataset/manifest_lifecycle.go", 236, `return fmt.Errorf("%w: candidate %s/%s was not a persisted staged manifest with a valid active pointer", ErrActivationConflict, candidate.Dataset.Project, candidate.Dataset.Generation)`}, + {"internal/backends/arango/internal/dataset/manifest_lifecycle.go", 327, `UPDATE active WITH { dataset: candidate.dataset, manifestKey: candidate._key } IN @@lifecycle_collection`}, + {"internal/backends/arango/internal/dataset/manifest_lifecycle.go", 328, `OPTIONS { ignoreRevs: false, mergeObjects: false }`}, + {"internal/backends/arango/internal/dataset/release_store.go", 197, `FILTER currentRevision == @expected_revision OR alreadyActive`}, + {"internal/api/bulk/load/service.go", 202, `manifest, err := s.generationActivator.ReadManifest(ctx, ref)`}, + {"internal/api/bulk/load/service.go", 206, `if !manifest.IsStaged() {`}, + {"internal/api/bulk/load/service.go", 209, `if err := s.generationActivator.Activate(ctx, manifest); err != nil {`}, + {"internal/backends/arango/internal/explorer/activation.go", 94, `if err := updateActivationDocument(txCtx, tx, datasetarango.LifecycleCollection, activeKey, map[string]any{"dataset": dataset, "manifestKey": manifestKey}); err != nil {`}, + {"internal/backends/arango/internal/explorer/activation.go", 38, `UPDATE d WITH @patch IN @@c`}, } -// gacActivateAQLMustNotHold: a compare against the pointer's current value (or a -// revision/lease token) would make this activation a CAS. None may be present. var gacActivateAQLMustNotHold = []string{ "active.dataset ==", "active.dataset !=", @@ -166,7 +55,6 @@ var gacActivateAQLMustNotHold = []string{ "lease", } -// gacActivateAQLMustHold keeps the absence check from passing vacuously. var gacActivateAQLMustHold = []string{ "UPDATE active WITH { dataset: candidate.dataset, manifestKey: candidate._key } IN @@lifecycle_collection", "OPTIONS { ignoreRevs: false, mergeObjects: false }", @@ -212,8 +100,6 @@ func gacCheckCitations(t *testing.T) { if citation.line < 1 || citation.line > len(lines) { t.Fatalf("citation drift: %s has %d lines, cited :%d == %q", citation.file, len(lines), citation.line, citation.want) } - // Drift-proof: require the exact cited statement verbatim somewhere - // in the file; the contract is the text, not the line number. found := false for _, line := range lines { if strings.TrimSpace(line) == strings.TrimSpace(citation.want) { @@ -227,12 +113,9 @@ func gacCheckCitations(t *testing.T) { } } -// gacShippedActivateAQL re-reads the shipped activateAQL const from source so the -// FAIL message quotes the file a reviewer opens, and proves it equals the const -// the binary actually compiles. func gacShippedActivateAQL(t *testing.T) string { t.Helper() - lines := gacReadSource(t, "internal/dataset/arango/manifest_lifecycle.go") + lines := gacReadSource(t, "internal/backends/arango/internal/dataset/manifest_lifecycle.go") start, end := -1, -1 for i, line := range lines { if strings.HasPrefix(strings.TrimSpace(line), "const activateAQL = `") { @@ -253,8 +136,6 @@ func gacShippedActivateAQL(t *testing.T) string { return shipped } -// gacActivateAQLHasNoCompare is the "no revalidation" half of the verdict, -// evaluated on the query the binary executes (never on a paraphrase). func gacActivateAQLHasNoCompare(t *testing.T) (bool, []string) { t.Helper() missing := make([]string, 0, len(gacActivateAQLMustHold)) @@ -272,24 +153,13 @@ func gacActivateAQLHasNoCompare(t *testing.T) (bool, []string) { present = append(present, forbidden) } } - // Scope note, checked rather than asserted in prose: the ONLY - // `active.dataset`-style consistency filter in this file lives in readActiveAQL - // (:307 `FILTER manifest.dataset == active.dataset`) — the READ path. The WRITE - // path has none. - source := strings.Join(gacReadSource(t, "internal/dataset/arango/manifest_lifecycle.go"), "\n") + source := strings.Join(gacReadSource(t, "internal/backends/arango/internal/dataset/manifest_lifecycle.go"), "\n") if !strings.Contains(source, "FILTER manifest.dataset == active.dataset") { t.Fatalf("readActiveAQL lost its consistency filter; re-audit the read/write asymmetry claim") } return len(present) == 0, present } -// --------------------------------------------------------------------------- -// owned-database harness (skip-clean without LOOM_TEST_ARANGO_URL) -// --------------------------------------------------------------------------- - -// gacRecorder records every AQL statement the production Store issues, so the -// run can show the read-then-act split into separate round trips and prove the -// idempotent branch performs no write leg. type gacRecorder struct { inner arangostore.RowQueryer queries []string @@ -355,10 +225,6 @@ func gacOpen(t *testing.T, budget time.Duration) (context.Context, *arangostore. } t.Logf("GAC_DATABASE_REMOVED=%s", name) }) - // Only the dataset lifecycle collection is bootstrapped: F-B17 is the - // manifest-generation activation path. The dataframe-publication collections - // (F-B13's release-CAS lane) and the explorer collections are deliberately - // NOT bootstrapped, so this artifact cannot drift into another claim's scope. if err := client.Bootstrap(ctx, BootstrapSpec()); err != nil { t.Fatalf("bootstrap dataset lifecycle collections: %v", err) } @@ -370,9 +236,6 @@ func gacOpen(t *testing.T, budget time.Duration) (context.Context, *arangostore. return ctx, client, lifecycle, recorder } -// gacLoadStaged creates a LOADING manifest and transitions it to STAGED through -// the production store, WITHOUT activating: Activate is the subject under test, -// so the pointer is only ever moved by the calls the legs make deliberately. func gacLoadStaged(t *testing.T, ctx context.Context, lifecycle *Store, project, generation, digest string) publication.Manifest { t.Helper() ref, err := publication.NewRef(project, generation) @@ -405,10 +268,6 @@ func gacLoadStaged(t *testing.T, ctx context.Context, lifecycle *Store, project, return staged } -// --------------------------------------------------------------------------- -// raw pointer observation (full .doc, including _id/_rev) -// --------------------------------------------------------------------------- - func gacJSON(t *testing.T, value any) string { t.Helper() raw, err := json.Marshal(value) @@ -418,9 +277,6 @@ func gacJSON(t *testing.T, value any) string { return string(raw) } -// gacPointerDoc reads the active_generation pointer document by the very key the -// production store derives and writes (document_codec.go:161 activeDocumentKey), -// so the OBSERVED document is provably the one Activate patches. func gacPointerDoc(t *testing.T, ctx context.Context, client *arangostore.Client, project string) map[string]any { t.Helper() key := activeDocumentKey(project) @@ -448,8 +304,6 @@ func gacPointerDoc(t *testing.T, ctx context.Context, client *arangostore.Client return doc } -// gacPointerRoleCount is an independent cross-check that the key-read document is -// the pointer the production readers consult (they select it by recordType). func gacPointerRoleCount(t *testing.T, ctx context.Context, client *arangostore.Client, project string) int { t.Helper() count := 0 @@ -481,10 +335,6 @@ func gacPointerLine(doc map[string]any) string { gacField(doc, "_id"), gacField(doc, "_rev"), doc["recordType"], doc["project"], gacPointerGeneration(doc), gacField(doc, "manifestKey")) } -// --------------------------------------------------------------------------- -// the claim -// --------------------------------------------------------------------------- - type gacFlip struct { label string candidate string @@ -515,7 +365,6 @@ func TestGenerationActivationBlindlyFlipsSharedPointerAgainstArango(t *testing.T const g2 = "gac-fb17-gen-2" const g3 = "gac-fb17-gen-3" - // --- step 1: three STAGED manifests with distinct digests, no activation yet. staged := map[string]publication.Manifest{} for _, spec := range []struct{ generation, digest string }{ {g1, strings.Repeat("a", 64)}, @@ -557,8 +406,6 @@ func TestGenerationActivationBlindlyFlipsSharedPointerAgainstArango(t *testing.T return flip } - // --- LEG A: last-writer-wins / silent flip across generations. - // PRE-STATED (declared now, before any of these three calls execute): t.Log("PRE-STATED EXPECT LEG A.1: Activate(G1) -> err=nil, pointer.dataset.generation == G1 (first real activation).") t.Log("PRE-STATED EXPECT LEG A.2 (BUG): Activate(G2) -> err=nil, NO ErrActivationConflict, pointer.dataset.generation == G2, _rev churns: a cross-generation flip is accepted blindly.") t.Log("PRE-STATED EXPECT LEG A.3 (BUG): Activate(G1) AGAIN after G2 -> err=nil, NO ErrActivationConflict, pointer.dataset.generation == G1 again, _rev churns: last writer wins, silent flip back, no conflict error of any kind.") @@ -569,15 +416,12 @@ func TestGenerationActivationBlindlyFlipsSharedPointerAgainstArango(t *testing.T a2 := runActivate("A.2 G2 over G1", g2) a3 := runActivate("A.3 G1 again over G2", g1) - // --- LEG B: lost update / check-then-act (bulk-load pattern at - // service.go:206 resolve candidate -> :210 IsStaged -> :213 Activate). t.Log("PRE-STATED EXPECT LEG B (BUG): reader resolves the active generation (observes G1 after LEG A), resolves its candidate G3 manifest (service.go:206) and only rechecks IsStaged (:210); a competing Activate(G2) moves the pointer in between; the reader's Activate(G3) (:213) then returns err=nil, the pointer lands on G3, and the competing G2 activation is silently discarded. No error, no revalidation of the stale read.") readerObserved, readerObserveErr := lifecycle.ResolveActiveManifest(ctx, project) t.Logf("OBSERVED [B.1 reader resolves active]: generation=%q err=%v (this is what the bulk-load caller sees before it picks a candidate)", readerObserved.Dataset.Generation, readerObserveErr) if readerObserveErr != nil || readerObserved.Dataset.Generation != g1 { t.Fatalf("PRECONDITION BROKEN: reader's initial active resolve should observe G1=%s, got %q err=%v", g1, readerObserved.Dataset.Generation, readerObserveErr) } - // Candidate resolution exactly as service.go:206 (ReadManifest) + :210 gate. candidateRef, refErr := publication.NewRef(project, g3) if refErr != nil { t.Fatalf("new ref: %v", refErr) @@ -587,19 +431,15 @@ func TestGenerationActivationBlindlyFlipsSharedPointerAgainstArango(t *testing.T t.Fatalf("PRECONDITION BROKEN: service.go:206-equivalent ReadManifest(G3) failed: %+v err=%v", readerCandidate, candidateErr) } t.Logf("OBSERVED [B.2 reader resolves candidate]: ReadManifest(G3).IsStaged=%v state=%s (the only thing rechecked before :213 is IsStaged)", readerCandidate.IsStaged(), readerCandidate.State) - // Interleaved competing activation: the pointer moves after the reader read. competitor := runActivate("B.3 competing Activate(G2)", g2) if competitor.err != nil || gacPointerGeneration(competitor.after) != g2 { t.Fatalf("PRECONDITION BROKEN: interleaved Activate(G2) failed (err=%v) or did not name G2 (pointer=%q)", competitor.err, gacPointerGeneration(competitor.after)) } - // The reader now activates the candidate chosen from its STALE read. - // (readerCandidate is the value ReadManifest returned before the pointer moved.) reader := runActivate("B.4 stale reader Activate(G3)", g3) - lost := gacPointerGeneration(competitor.after) // G2, silently discarded + lost := gacPointerGeneration(competitor.after) t.Logf("OBSERVED [B.5 lost update]: reader read active=%q at B.1; pointer moved to %q at B.3; reader's Activate(G3) err=%v landed %q => %q activation was overwritten with no error and no revalidation", readerObserved.Dataset.Generation, lost, reader.err, gacPointerGeneration(reader.after), lost) - // --- LEG C: sound leg, re-Activate the SAME already-active generation twice. t.Log("PRE-STATED EXPECT LEG C (SOUND): with G3 already active, Activate(G3) is the idempotent no-op branch of manifest_lifecycle.go:198-204 (`if active, err := s.ResolveActiveManifest(...); err == nil { if manifestIdentityEqual(active, candidate) { return nil } }`): err=nil, exactly ONE round trip (the pre-check read), ZERO activateAQL write legs, and pointer _rev does NOT churn. Same-generation re-activation must look different from LEG A/B cross-generation flips.") revBefore := gacField(gacPointerDoc(t, ctx, client, project), "_rev") c1 := runActivate("C.1 same generation again", g3) @@ -613,11 +453,9 @@ func TestGenerationActivationBlindlyFlipsSharedPointerAgainstArango(t *testing.T t.Logf("OBSERVED LEG C (sound control, as pre-stated): same-generation re-activation is the :198-204 no-op branch — err=nil, 1 round trip, 0 write legs, _rev stable at %s across both calls.", revBefore) } - // Independent cross-check that the key-read pointer is the one readers use. roleMatches := gacPointerRoleCount(t, ctx, client, project) t.Logf("OBSERVED pointer role-read cross-check: %d document(s) match recordType==\"active_generation\" AND project==%s (the selector used by readActiveAQL)", roleMatches, project) - // --- verdict, entirely from OBSERVED values. crossGeneration := []gacFlip{a2, a3, competitor, reader} blindCount, refused := 0, make([]string, 0, len(crossGeneration)) for _, flip := range crossGeneration { @@ -633,7 +471,7 @@ func TestGenerationActivationBlindlyFlipsSharedPointerAgainstArango(t *testing.T len(crossGeneration), blindCount, len(refused), noCompare, forbiddenPresent) if blindCount >= 1 && noCompare { - t.Fatalf("FAILED-ON-PURPOSE: F-B17 CONFIRMED against real Arango (database owned+dropped by this test). %d/%d cross-generation Store.Activate calls blindly overwrote the shared active_generation pointer with err=nil and no revalidation of any kind: A.2 G1->G2 err=%v, A.3 G2->G1(err=%v), B.3 ->G2 err=%v, B.4 stale reader ->G3 err=%v (the competing %q activation was silently discarded after the reader's resolve; the pointer had already moved). The pointer's previous value is never compared: the missing guard is at internal/dataset/arango/manifest_lifecycle.go:334-335 — `UPDATE active WITH { dataset: candidate.dataset, manifestKey: candidate._key } IN @@lifecycle_collection OPTIONS { ignoreRevs: false, mergeObjects: false }` — which lacks any `FILTER active.dataset == @candidate.dataset` (and any `FILTER active.manifestKey != ...`, expected-revision bind, or lease token; forbidden tokens present in the shipped query: %v). ignoreRevs:false supplies no protection here because no _rev is submitted with the patch. The read-then-act half is manifest_lifecycle.go:198-204 (`if active, err := s.ResolveActiveManifest(ctx, candidate.Dataset.Project); err == nil { if manifestIdentityEqual(active, candidate) { return nil } }`) — a separate HTTP round trip whose result the write never re-reads or re-checks (OBSERVED roundTrips for a cross-generation Activate = %d, i.e. pre-check read then blind write); the bulk-load caller adds no protection either (service.go:206 resolves the candidate, :210 rechecks only IsStaged, :213 activates). The single rejection this path can raise, manifest_lifecycle.go:243 ErrActivationConflict, is about the CANDIDATE (\"was not a persisted staged manifest with a valid active pointer\"), never about the pointer having moved — OBSERVED ErrActivationConflict=%v on the cross-generation flips.\nFIX SHAPE (already present in this package for the release pointer): internal/dataset/arango/release_store.go:167 `FILTER currentRevision == @expected_revision OR alreadyActive` — a compare against the pointer's current state whose failure yields ErrReleaseActivationConflict instead of a silent write; activateAQL needs the equivalent (FILTER on active.dataset/active.manifestKey, or a submitted _rev / revision token) instead of the unconditional UPDATE.\nDISTINCTNESS: this is the manifest-generation activation writer (activateAQL), not F-B13's CompareAndSwapActivateRelease generationPointer patch (release_store.go:170-173), which was never called here (no publication/explorer collections bootstrapped). Third writer on record, not driven: internal/explorer/arango/activation.go:98 + :38.\nSOUND LEG (correctly behaving, not part of the finding): same-generation re-activation hit the idempotent branch, err=nil with 0 write legs and stable _rev (%s) => held=%v.\nSHIPPED activateAQL (internal/dataset/arango/manifest_lifecycle.go:318-337) quoted verbatim:\n%s", + t.Fatalf("FAILED-ON-PURPOSE: F-B17 CONFIRMED against real Arango (database owned+dropped by this test). %d/%d cross-generation Store.Activate calls blindly overwrote the shared active_generation pointer with err=nil and no revalidation of any kind: A.2 G1->G2 err=%v, A.3 G2->G1(err=%v), B.3 ->G2 err=%v, B.4 stale reader ->G3 err=%v (the competing %q activation was silently discarded after the reader's resolve; the pointer had already moved). The pointer's previous value is never compared: the missing guard is at internal/backends/arango/internal/dataset/manifest_lifecycle.go:334-335 — `UPDATE active WITH { dataset: candidate.dataset, manifestKey: candidate._key } IN @@lifecycle_collection OPTIONS { ignoreRevs: false, mergeObjects: false }` — which lacks any `FILTER active.dataset == @candidate.dataset` (and any `FILTER active.manifestKey != ...`, expected-revision bind, or lease token; forbidden tokens present in the shipped query: %v). ignoreRevs:false supplies no protection here because no _rev is submitted with the patch. The read-then-act half is manifest_lifecycle.go:198-204 (`if active, err := s.ResolveActiveManifest(ctx, candidate.Dataset.Project); err == nil { if manifestIdentityEqual(active, candidate) { return nil } }`) — a separate HTTP round trip whose result the write never re-reads or re-checks (OBSERVED roundTrips for a cross-generation Activate = %d, i.e. pre-check read then blind write); the bulk-load caller adds no protection either (service.go:206 resolves the candidate, :210 rechecks only IsStaged, :213 activates). The single rejection this path can raise, manifest_lifecycle.go:243 ErrActivationConflict, is about the CANDIDATE (\"was not a persisted staged manifest with a valid active pointer\"), never about the pointer having moved — OBSERVED ErrActivationConflict=%v on the cross-generation flips.\nFIX SHAPE (already present in this package for the release pointer): internal/backends/arango/internal/dataset/release_store.go:167 `FILTER currentRevision == @expected_revision OR alreadyActive` — a compare against the pointer's current state whose failure yields ErrReleaseActivationConflict instead of a silent write; activateAQL needs the equivalent (FILTER on active.dataset/active.manifestKey, or a submitted _rev / revision token) instead of the unconditional UPDATE.\nDISTINCTNESS: this is the manifest-generation activation writer (activateAQL), not F-B13's CompareAndSwapActivateRelease generationPointer patch (release_store.go:170-173), which was never called here (no publication/explorer collections bootstrapped). Third writer on record, not driven: internal/backends/arango/internal/explorer/activation.go:98 + :38.\nSOUND LEG (correctly behaving, not part of the finding): same-generation re-activation hit the idempotent branch, err=nil with 0 write legs and stable _rev (%s) => held=%v.\nSHIPPED activateAQL (internal/backends/arango/internal/dataset/manifest_lifecycle.go:318-337) quoted verbatim:\n%s", blindCount, len(crossGeneration), a2.err, a3.err, competitor.err, reader.err, lost, forbiddenPresent, a2.roundTrips, a2.conflict, revBefore, soundLegHeld, shippedActivateAQL) } if blindCount == 0 && len(refused) > 0 { diff --git a/internal/dataset/arango/manifest_lifecycle.go b/internal/backends/arango/internal/dataset/manifest_lifecycle.go similarity index 95% rename from internal/dataset/arango/manifest_lifecycle.go rename to internal/backends/arango/internal/dataset/manifest_lifecycle.go index c1cc2cf3f..ac0536b9f 100644 --- a/internal/dataset/arango/manifest_lifecycle.go +++ b/internal/backends/arango/internal/dataset/manifest_lifecycle.go @@ -5,8 +5,8 @@ import ( "errors" "fmt" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" publication "github.com/calypr/loom/internal/dataset" - arangostore "github.com/calypr/loom/internal/store/arango" ) const ( @@ -50,8 +50,6 @@ func (s *Store) CreateManifest(ctx context.Context, manifest publication.Manifes return publication.Manifest{}, err } bindVars := lifecycleBindVars(manifest.Dataset.Project) - // Arango rejects undeclared bind parameters. The create query derives the - // project and active record from the immutable documents themselves. delete(bindVars, "project") delete(bindVars, "active_record_type") bindVars["manifest"] = document @@ -190,11 +188,6 @@ func (s *Store) Activate(ctx context.Context, candidate publication.Manifest) er if err := s.validate(); err != nil { return err } - // Activation is intentionally idempotent. Gecko may retry a publish after - // the active-generation pointer was already switched; doing another - // compare-and-swap update in that case creates an unnecessary revision race - // and can surface PUBLICATION_CONFLICT even though the requested generation - // is already active. if active, err := s.ResolveActiveManifest(ctx, candidate.Dataset.Project); err == nil { if manifestIdentityEqual(active, candidate) { return nil diff --git a/internal/dataset/arango/release_policy_change_e2e_test.go b/internal/backends/arango/internal/dataset/release_policy_change_e2e_test.go similarity index 87% rename from internal/dataset/arango/release_policy_change_e2e_test.go rename to internal/backends/arango/internal/dataset/release_policy_change_e2e_test.go index fd4d4ee73..74416e5f1 100644 --- a/internal/dataset/arango/release_policy_change_e2e_test.go +++ b/internal/backends/arango/internal/dataset/release_policy_change_e2e_test.go @@ -13,29 +13,13 @@ import ( "testing" "time" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/dataset" - arangostore "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) -// R2 durable control: a required->optional reclassification of the SAME selector -// under the SAME execution is a genuinely different immutable snapshot. The -// pre-widening releaseID hashed only project/generation plus each publication's -// (Selector.Key(), ExecutionID), so SaveRelease's full-content comparison -// rejected the new candidate with "release already exists with different -// content" while the identity said the document already existed. This test runs -// the production ReleaseService.Create/Activate against a real Arango and -// asserts the policy change saves AND activates a distinct immutable candidate, -// that the superseded candidate keeps its bytes and _rev, that identical -// retries converge, and that a pre-widening stored id transitions at most once. - const rpcDatabasePrefix = "loom_verify_rpc_" -// rpcHistoricalReleaseID replays the pre-widening digest exactly as shipped -// (project and generation joined by a bare NUL, then one NUL-joined -// (Selector.Key(), ExecutionID) group per publication). It is the fault -// premise: the required and optional snapshots collapse HERE and must not -// collapse in production releaseID. func rpcHistoricalReleaseID(release dataset.ProjectRelease) string { var b strings.Builder b.WriteString(release.Project) @@ -133,9 +117,6 @@ func rpcStageGeneration(t *testing.T, ctx context.Context, lifecycle *Store, pro } } -// rpcRawRelease reads the persisted project_release document by release id -// through raw AQL, so the assertions observe stored bytes and _rev rather than -// the decoded view. func rpcRawRelease(t *testing.T, ctx context.Context, client *arangostore.Client, id string) map[string]any { t.Helper() var doc map[string]any @@ -185,8 +166,6 @@ func rpcPointer(t *testing.T, ctx context.Context, client *arangostore.Client, p return releaseID, revision, rev } -// rpcCanonicalContent renders the persisted immutable content, excluding the -// document metadata, the derived id and the first-creation timestamp. func rpcCanonicalContent(t *testing.T, doc map[string]any) string { t.Helper() content := map[string]any{} @@ -228,7 +207,6 @@ func TestReleaseRequiredPolicyChangeArango(t *testing.T) { return &dataset.ReleaseService{Manifests: lifecycle, Releases: lifecycle, Verifier: verifier, Required: required, Now: func() time.Time { return verifiedAt }} } - // Phase 1: the publication is required. requiredActive, err := newService(selector).Activate(ctx, dataset.ActivationRequest{Project: project, Generation: generation, ExpectedRevision: 0}) if err != nil || requiredActive.Revision != 1 { t.Fatalf("required activation = %#v, %v (want revision 1)", requiredActive, err) @@ -240,7 +218,6 @@ func TestReleaseRequiredPolicyChangeArango(t *testing.T) { t.Fatalf("phase 1 persisted required = %#v, want true", got) } - // Fault premise: the pre-widening digest cannot tell the two snapshots apart. if rpcHistoricalReleaseID(requiredActive.Release) != rpcHistoricalReleaseID(dataset.ProjectRelease{ Project: project, Generation: generation, Publications: []dataset.ReleasePublication{{Selector: selector, ExecutionID: "execution-r2", Generation: generation}}, @@ -248,8 +225,6 @@ func TestReleaseRequiredPolicyChangeArango(t *testing.T) { t.Fatalf("fault premise broken: the historical framing must fold the required/optional pair") } - // Phase 2: the same selector and execution, now optional. This must save a - // distinct immutable candidate and activate it. optionalActive, err := newService().Activate(ctx, dataset.ActivationRequest{Project: project, Generation: generation, ExpectedRevision: 1, OptionalSelectors: []dataset.DataframeSelector{selector}}) if err != nil { t.Fatalf("optional reclassification did not save and activate: %v", err) @@ -271,7 +246,6 @@ func TestReleaseRequiredPolicyChangeArango(t *testing.T) { t.Fatalf("REGRESSION: the two candidates stored identical content: %s", requiredContent) } - // The superseded candidate is untouched: same bytes, same _rev. reReadRequired := rpcRawRelease(t, ctx, client, requiredActive.Release.ID) if got := rpcCanonicalContent(t, reReadRequired); got != requiredContent { t.Fatalf("REGRESSION: the required candidate content mutated: got %s want %s", got, requiredContent) @@ -283,8 +257,6 @@ func TestReleaseRequiredPolicyChangeArango(t *testing.T) { t.Fatalf("REGRESSION: the superseded candidate lost required=true: %#v", got) } - // Immutable-content comparison is NOT weakened: mutating the persisted - // snapshot under its own id must still be refused. tampered := requiredActive.Release tampered.Publications = append([]dataset.ReleasePublication(nil), tampered.Publications...) tampered.Publications[0].Required = false @@ -292,8 +264,6 @@ func TestReleaseRequiredPolicyChangeArango(t *testing.T) { if _, err := lifecycle.SaveRelease(ctx, tampered); err == nil || !strings.Contains(err.Error(), "already exists") { t.Fatalf("tampered re-save of the required candidate = %v, want the different-content refusal", err) } - // Re-saving byte-identical content (drifted createdAt only) is a no-op that - // keeps the original document bytes. drifted := requiredActive.Release drifted.CreatedAt = verifiedAt.Add(42 * time.Minute) saved, err := lifecycle.SaveRelease(ctx, drifted) @@ -307,8 +277,6 @@ func TestReleaseRequiredPolicyChangeArango(t *testing.T) { t.Fatalf("REGRESSION: identical re-save churned the immutable candidate") } - // Identical retries of the optional snapshot: no revision bump, no repeated - // identities, no new documents, no pointer churn. optionalRequest := dataset.ActivationRequest{Project: project, Generation: generation, ExpectedRevision: 2, OptionalSelectors: []dataset.DataframeSelector{selector}} pointerID, pointerRevision, pointerRev := rpcPointer(t, ctx, client, project) if pointerID != optionalActive.Release.ID { @@ -338,9 +306,6 @@ func TestReleaseRequiredPolicyChangeArango(t *testing.T) { t.Fatalf("REGRESSION: identical retries churned the active pointer: %s/%v/%s -> %s/%v/%s", pointerID, pointerRevision, pointerRev, id, revision, rev) } - // Upgrade boundary: a candidate saved AND activated by the pre-widening - // binary keeps its old id; the widened framing transitions it exactly once - // and every later retry converges. legacyProbe := dataset.ProjectRelease{Project: project, Generation: generation, GitCommit: generation, Publications: []dataset.ReleasePublication{{Selector: selector, ExecutionID: "execution-r2", Generation: generation, VerifiedAt: verifiedAt}}} legacyID := rpcHistoricalReleaseID(legacyProbe) diff --git a/internal/dataset/arango/release_store.go b/internal/backends/arango/internal/dataset/release_store.go similarity index 90% rename from internal/dataset/arango/release_store.go rename to internal/backends/arango/internal/dataset/release_store.go index 32ec6e1b8..61d69f1e5 100644 --- a/internal/dataset/arango/release_store.go +++ b/internal/backends/arango/internal/dataset/release_store.go @@ -6,10 +6,12 @@ import ( "fmt" "github.com/arangodb/go-driver/v2/arangodb/shared" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/dataset" - arangostore "github.com/calypr/loom/internal/store/arango" ) +const casConflictRetryBudget = 8 + func (s *Store) SaveRelease(ctx context.Context, release dataset.ProjectRelease) (dataset.ProjectRelease, error) { document, err := lifecycleDocument(release) if err != nil { @@ -64,15 +66,10 @@ func (s *Store) CompareAndSwapActivateRelease(ctx context.Context, release datas return CompareAndSwapActivateRelease(ctx, s.client, release, expectedRevision) } -// CompareAndSwapActivateRelease commits the active Explorer revision and -// matching dataset release in one Arango transaction. func CompareAndSwapActivateRelease(ctx context.Context, client arangostore.RowQueryer, release dataset.ProjectRelease, expectedRevision int64) (dataset.ActiveRelease, error) { return compareAndSwapActivateRelease(ctx, client, release, expectedRevision, false) } -// CompareAndSwapPublishRelease activates a release produced by an Explorer -// publication. The publication is only current when the active generation -// pointer still identifies its source manifest. func CompareAndSwapPublishRelease(ctx context.Context, client arangostore.RowQueryer, release dataset.ProjectRelease, expectedRevision int64) (dataset.ActiveRelease, error) { return compareAndSwapActivateRelease(ctx, client, release, expectedRevision, true) } @@ -97,20 +94,24 @@ func compareAndSwapActivateRelease(ctx context.Context, client arangostore.RowQu binds["require_current_generation"] = requireCurrentGeneration binds["staged_state"] = string(dataset.StateStaged) binds["ready_state"] = string(dataset.StateReady) + const conflictRetries = casConflictRetryBudget var active *dataset.ActiveRelease - err = client.QueryRows(ctx, activateReleaseAQL, metadataBatchSize, binds, func(row map[string]any) error { - decoded, err := decodeValue[dataset.ActiveRelease](row) - if err != nil { - return err + for attempt := 0; ; attempt++ { + active = nil + err = client.QueryRows(ctx, activateReleaseAQL, metadataBatchSize, binds, func(row map[string]any) error { + decoded, err := decodeValue[dataset.ActiveRelease](row) + if err != nil { + return err + } + active = &decoded + return nil + }) + if err == nil { + break + } + if shared.IsArangoErrorWithErrorNum(err, shared.ErrArangoConflict) && attempt < conflictRetries && ctx.Err() == nil { + continue } - active = &decoded - return nil - }) - if err != nil { - // Concurrent release CAS operations can both observe the same revision - // before Arango serializes their pointer writes. Arango reports the loser - // as error 1200 (write-write conflict), so preserve the repository's - // typed CAS contract for callers instead of leaking a backend failure. if shared.IsArangoErrorWithErrorNum(err, shared.ErrArangoConflict) { return dataset.ActiveRelease{}, fmt.Errorf("%w: concurrent active-pointer update", dataset.ErrReleaseActivationConflict) } diff --git a/internal/backends/arango/internal/dataset/release_store_cas_retry_test.go b/internal/backends/arango/internal/dataset/release_store_cas_retry_test.go new file mode 100644 index 000000000..2fd672ba9 --- /dev/null +++ b/internal/backends/arango/internal/dataset/release_store_cas_retry_test.go @@ -0,0 +1,127 @@ +package arango + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "testing" + + "github.com/arangodb/go-driver/v2/arangodb/shared" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" + publication "github.com/calypr/loom/internal/dataset" +) + +type casConflictRowQueryer struct { + conflicts int + calls int + rows []map[string]any + conflictErr error + otherErr error + visitedBinds []map[string]any +} + +func (q *casConflictRowQueryer) QueryRows(_ context.Context, _ string, _ int, binds map[string]interface{}, visitor arangostore.RowVisitor) error { + q.calls++ + q.visitedBinds = append(q.visitedBinds, binds) + if q.calls <= q.conflicts { + if q.conflictErr != nil { + return q.conflictErr + } + return shared.ArangoError{HasError: true, Code: 409, ErrorNum: shared.ErrArangoConflict, ErrorMessage: "write-write conflict"} + } + if q.otherErr != nil { + return q.otherErr + } + for _, row := range q.rows { + if err := visitor(row); err != nil { + return err + } + } + return nil +} + +func casConvergedRow() map[string]any { + encoded, err := json.Marshal(publication.ActiveRelease{ + Release: publication.ProjectRelease{ID: "ab10-release-2", Project: "ab10-p2", Generation: "gen-a", GitCommit: "gen-a"}, + Revision: 2, + }) + if err != nil { + panic(err) + } + var row map[string]any + if err := json.Unmarshal(encoded, &row); err != nil { + panic(err) + } + return row +} + +func casTestRelease() publication.ProjectRelease { + return publication.ProjectRelease{ID: "ab10-release-2", Project: "ab10-p2", Generation: "gen-a", GitCommit: "gen-a"} +} + +func TestCompareAndSwapActivateReleaseRetriesConflictAbort(t *testing.T) { + for _, tc := range []struct { + name string + conflicts int + wantCalls int + }{ + {name: "no abort", conflicts: 0, wantCalls: 1}, + {name: "three aborts then converge", conflicts: 3, wantCalls: 4}, + {name: "budget exactly reached", conflicts: casConflictRetryBudget, wantCalls: casConflictRetryBudget + 1}, + } { + t.Run(tc.name, func(t *testing.T) { + queryer := &casConflictRowQueryer{conflicts: tc.conflicts, rows: []map[string]any{casConvergedRow()}} + active, err := CompareAndSwapActivateRelease(context.Background(), queryer, casTestRelease(), 1) + if err != nil { + t.Fatalf("CAS after %d aborts: %v", tc.conflicts, err) + } + if active.Revision != 2 || active.Release.ID != "ab10-release-2" { + t.Fatalf("CAS converged on (%s, revision %d), want (ab10-release-2, revision 2)", active.Release.ID, active.Revision) + } + if queryer.calls != tc.wantCalls { + t.Fatalf("driver visited %d times, want %d", queryer.calls, tc.wantCalls) + } + for i, binds := range queryer.visitedBinds { + if binds["expected_revision"] != int64(1) || binds["next_revision"] != int64(2) { + t.Fatalf("retry %d changed the CAS inputs: %v/%v", i, binds["expected_revision"], binds["next_revision"]) + } + } + }) + } +} + +func TestCompareAndSwapActivateReleaseExhaustedBudgetStaysTypedConflict(t *testing.T) { + queryer := &casConflictRowQueryer{conflicts: casConflictRetryBudget + 3} + _, err := CompareAndSwapActivateRelease(context.Background(), queryer, casTestRelease(), 1) + if !errors.Is(err, publication.ErrReleaseActivationConflict) { + t.Fatalf("exhausted retry budget returned %v, want the typed activation conflict", err) + } + if queryer.calls != casConflictRetryBudget+1 { + t.Fatalf("driver visited %d times, want the budget plus the first attempt (%d)", queryer.calls, casConflictRetryBudget+1) + } +} + +func TestCompareAndSwapActivateReleaseCancelStopsRetryAfterFirstAbort(t *testing.T) { + queryer := &casConflictRowQueryer{conflicts: casConflictRetryBudget + 3} + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + _, err := CompareAndSwapActivateRelease(cancelled, queryer, casTestRelease(), 1) + if !errors.Is(err, publication.ErrReleaseActivationConflict) { + t.Fatalf("cancelled CAS returned %v, want the typed activation conflict", err) + } + if queryer.calls != 1 { + t.Fatalf("cancelled CAS retried %d times, want 1", queryer.calls) + } +} + +func TestCompareAndSwapActivateReleaseNonConflictErrorIsNotRetried(t *testing.T) { + queryer := &casConflictRowQueryer{otherErr: fmt.Errorf("boom")} + _, err := CompareAndSwapActivateRelease(context.Background(), queryer, casTestRelease(), 1) + if err == nil || !errors.Is(err, queryer.otherErr) { + t.Fatalf("plain failure returned %v, want the wrapped driver error", err) + } + if queryer.calls != 1 { + t.Fatalf("plain failure retried %d times, want 1", queryer.calls) + } +} diff --git a/internal/dataset/arango/revision_inflation_e2e_test.go b/internal/backends/arango/internal/dataset/revision_inflation_e2e_test.go similarity index 86% rename from internal/dataset/arango/revision_inflation_e2e_test.go rename to internal/backends/arango/internal/dataset/revision_inflation_e2e_test.go index 7298412d1..e88ea9e64 100644 --- a/internal/dataset/arango/revision_inflation_e2e_test.go +++ b/internal/backends/arango/internal/dataset/revision_inflation_e2e_test.go @@ -11,25 +11,11 @@ import ( "testing" "time" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/dataset" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) -// F-B15 durable control (product repaired in REPAIR-5/Round 31; the red arm -// stays armed as the regression witness and was observed PREVENTED against -// the real store in Round 36): idempotent re-activation through the -// pre-repair production ReleaseService.Activate path was NOT a no-op. Each identical -// activation mints revision+1 into the sha-derived release ID -// (internal/dataset/release.go:227,273-280) so the AQL alreadyActive guard -// (internal/dataset/arango/release_store.go:158-159) can never match for -// callers that re-enter via Create, and both pointer documents churn while the -// release CONTENT (project/generation/gitCommit/publications/verifications) is -// byte-identical. Recorded CONFIRMED in -// verification/publication-bundle/FINDINGS.md Round 10 (runs 22ffa92b and -// 97d6ab0f); this file replaces the broken truncated re-write (quarantined at -// /tmp/opencode/revision_inflation_broken_a3a58c18.go). - const riDatabasePrefix = "loom_verify_ri_" type riNoopVerifier struct{} @@ -138,8 +124,6 @@ func riObserve(t *testing.T, ctx context.Context, client *store.Client, svc *dat if uErr := json.Unmarshal(raw, &m); uErr != nil { t.Fatalf("unmarshal active release: %v", uErr) } - // Exclude derived, revision-dependent / wall-clock fields: content - // identity means project/generation/gitCommit/publications/verifications. delete(m, "createdAt") delete(m, "id") canonical, mErr := json.Marshal(m) @@ -238,7 +222,7 @@ func TestRevisionInflationAgainstArango(t *testing.T) { switch { case inflated && contentIdentical: - t.Fatalf("FAILED-ON-PURPOSE CONFIRMED: 4 identical activations of %q through the production ReleaseService.Activate path (server.go live-revision pattern) are NOT no-ops: revision %d -> %d, release ID %s -> %s, project_release docs %d -> %d, active_project_release._rev %s -> %s, active_generation._rev %s -> %s (releaseRevision %v -> %v), WHILE the release content is byte-identical (%s). Layer: ReleaseService.Create mints release.ID = releaseID(release, currentRevision+1) (internal/dataset/release.go:227,273-280; sha over project/generation/REVISION/publications); the alreadyActive short-circuit requires re-submitting the SAME releaseId (internal/dataset/arango/release_store.go:158-159) which the Create-then-CAS callers never do. Harm: every repeat activation click mints a fresh immutable release document + revision forever (unbounded Arango lifecycle growth, audit inflation, phantom release-ID churn for ID-keyed consumers).", + t.Fatalf("FAILED-ON-PURPOSE CONFIRMED: 4 identical activations of %q through the production ReleaseService.Activate path (server.go live-revision pattern) are NOT no-ops: revision %d -> %d, release ID %s -> %s, project_release docs %d -> %d, active_project_release._rev %s -> %s, active_generation._rev %s -> %s (releaseRevision %v -> %v), WHILE the release content is byte-identical (%s). Layer: ReleaseService.Create mints release.ID = releaseID(release, currentRevision+1) (internal/dataset/release.go:227,273-280; sha over project/generation/REVISION/publications); the alreadyActive short-circuit requires re-submitting the SAME releaseId (internal/backends/arango/internal/dataset/release_store.go:209-210) which the Create-then-CAS callers never do. Harm: every repeat activation click mints a fresh immutable release document + revision forever (unbounded Arango lifecycle growth, audit inflation, phantom release-ID churn for ID-keyed consumers).", generation, baseline.revision, last.revision, baseline.id, last.id, baseline.releaseDocs, last.releaseDocs, baseline.relDocRev, last.relDocRev, baseline.genDocRev, last.genDocRev, baseline.genDocRevNum, last.genDocRevNum, last.content) case !inflated && contentIdentical: t.Log("PREVENTED (sound): identical re-activation is a true no-op — revision, release ID, doc count, and both pointer _revs stable with byte-identical content.") diff --git a/internal/dataset/arango/store_test.go b/internal/backends/arango/internal/dataset/store_test.go similarity index 98% rename from internal/dataset/arango/store_test.go rename to internal/backends/arango/internal/dataset/store_test.go index ea396891c..daca4d79d 100644 --- a/internal/dataset/arango/store_test.go +++ b/internal/backends/arango/internal/dataset/store_test.go @@ -11,8 +11,8 @@ import ( "time" "github.com/arangodb/go-driver/v2/arangodb/shared" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" publication "github.com/calypr/loom/internal/dataset" - arangostore "github.com/calypr/loom/internal/store/arango" ) func TestBootstrapHasNoSecondaryIndexes(t *testing.T) { @@ -64,8 +64,6 @@ func TestTransitionAndResolveManifest(t *testing.T) { func TestActivateOnlyRevisionChecksActivePointer(t *testing.T) { ready := fixtureManifest(t, "project-a", "generation-a", publication.StateReady) - // The first query is the idempotency read. An empty result means there is - // no active manifest, so activation proceeds to the compare-and-swap query. fake := &fakeQueryClient{responses: [][]map[string]any{{}, {{"dataset": jsonObject(t, ready.Dataset)}}}} store := mustStore(t, fake) if err := store.Activate(context.Background(), ready); err != nil { diff --git a/internal/explorer/arango/activation.go b/internal/backends/arango/internal/explorer/activation.go similarity index 95% rename from internal/explorer/arango/activation.go rename to internal/backends/arango/internal/explorer/activation.go index f5242f93f..9e0ad9bb1 100644 --- a/internal/explorer/arango/activation.go +++ b/internal/backends/arango/internal/explorer/activation.go @@ -5,11 +5,11 @@ import ( "fmt" "time" - publicationarango "github.com/calypr/loom/internal/dataframe/publication/arango" + datasetarango "github.com/calypr/loom/internal/backends/arango/internal/dataset" + publicationarango "github.com/calypr/loom/internal/backends/arango/internal/publication" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/dataset" - datasetarango "github.com/calypr/loom/internal/dataset/arango" "github.com/calypr/loom/internal/explorer" - store "github.com/calypr/loom/internal/store/arango" ) const activationReadAQL = `LET owner = DOCUMENT(@@explorers, @explorerKey) @@ -49,10 +49,6 @@ type activationState struct { prior map[string]any } -// ActivateRepositoryGeneration is the repository-only composite visibility -// switch. It validates all pointers first, then performs each single-collection -// modification inside one Arango transaction so no partial activation is -// visible if any update fails. func (s *Store) ActivateRepositoryGeneration(ctx context.Context, project, generation, revisionID string) error { return s.client.WithTransaction(ctx, store.TransactionCollections{ Read: []string{publicationarango.BundleExecutionsCollection}, diff --git a/internal/explorer/arango/capability_snapshot_store.go b/internal/backends/arango/internal/explorer/capability_snapshot_store.go similarity index 88% rename from internal/explorer/arango/capability_snapshot_store.go rename to internal/backends/arango/internal/explorer/capability_snapshot_store.go index 46b18e158..92443ebb9 100644 --- a/internal/explorer/arango/capability_snapshot_store.go +++ b/internal/backends/arango/internal/explorer/capability_snapshot_store.go @@ -6,18 +6,12 @@ import ( "fmt" "strings" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer/capability" - store "github.com/calypr/loom/internal/store/arango" ) -// CapabilitySnapshotCollection is intentionally separate from the legacy -// Explorer collections. Registration in the server bootstrap should include -// this persistent, non-truncating collection. const CapabilitySnapshotCollection = "loom_explorer_capability_snapshots" -// CapabilitySnapshotCollectionSpec is exported so the server's shared -// bootstrap registry can append it without coupling this adapter to the -// legacy collection list. func CapabilitySnapshotCollectionSpec() store.CollectionSpec { return store.CollectionSpec{ Name: CapabilitySnapshotCollection, @@ -28,9 +22,6 @@ func CapabilitySnapshotCollectionSpec() store.CollectionSpec { } } -// CapabilitySnapshotStore persists the complete canonical snapshot under its -// content address. Insert-ignore is deliberate: retries never execute an -// UPDATE, so an immutable document's payload and revision remain untouched. type CapabilitySnapshotStore struct{ client client } func NewCapabilitySnapshotStore(c client) (*CapabilitySnapshotStore, error) { @@ -151,8 +142,6 @@ func capabilitySnapshotKey(token string) string { } func canonicalSnapshotDocument(snapshot capability.Snapshot, key string) (map[string]any, error) { - // CanonicalPayload is the exact payload covered by Token. Add Token back to - // that normalized object so Arango stores a canonical *full* snapshot. var doc map[string]any if err := json.Unmarshal(snapshot.CanonicalPayload(), &doc); err != nil { return nil, err diff --git a/internal/explorer/arango/capability_snapshot_store_test.go b/internal/backends/arango/internal/explorer/capability_snapshot_store_test.go similarity index 98% rename from internal/explorer/arango/capability_snapshot_store_test.go rename to internal/backends/arango/internal/explorer/capability_snapshot_store_test.go index cc355aed7..ff620bcf3 100644 --- a/internal/explorer/arango/capability_snapshot_store_test.go +++ b/internal/backends/arango/internal/explorer/capability_snapshot_store_test.go @@ -7,8 +7,8 @@ import ( "strings" "testing" + storepkg "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer/capability" - storepkg "github.com/calypr/loom/internal/store/arango" ) type capabilitySnapshotClient struct { diff --git a/internal/explorer/arango/codec.go b/internal/backends/arango/internal/explorer/codec.go similarity index 80% rename from internal/explorer/arango/codec.go rename to internal/backends/arango/internal/explorer/codec.go index d4aa58ea9..5dfda4136 100644 --- a/internal/explorer/arango/codec.go +++ b/internal/backends/arango/internal/explorer/codec.go @@ -15,10 +15,6 @@ func decode[T any](value any) (T, error) { return out, err } -// normalizeUpdatedAt keeps reads tolerant of the numeric Arango timestamp -// written by the short-lived last-write-wins regression. New writes use the -// normal JSON time string representation, but existing documents must remain -// readable so they can be repaired by a subsequent draft save. func normalizeUpdatedAt(value any) any { row, ok := value.(map[string]any) if !ok { diff --git a/internal/explorer/arango/collections.go b/internal/backends/arango/internal/explorer/collections.go similarity index 83% rename from internal/explorer/arango/collections.go rename to internal/backends/arango/internal/explorer/collections.go index 7d590cf84..5ff529b63 100644 --- a/internal/explorer/arango/collections.go +++ b/internal/backends/arango/internal/explorer/collections.go @@ -1,7 +1,6 @@ -// Package arango persists Loom Explorer aggregate records. package arango -import store "github.com/calypr/loom/internal/store/arango" +import store "github.com/calypr/loom/internal/backends/arango/internal/store" const ( ExplorersCollection = "loom_explorers" @@ -15,14 +14,9 @@ const ( ExplicitGroupMembershipsCollection = "loom_explorer_explicit_group_memberships" InterpretationLibrariesCollection = "loom_explorer_interpretation_libraries" InterpretationRevisionsCollection = "loom_explorer_interpretation_revisions" - // LegacyRepositoryConfigsCollection is read only. It remains in bootstrap - // for one compatibility window so startup can migrate old default-owner - // pointers into loom_explorers; no current workflow writes it. - LegacyRepositoryConfigsCollection = "loom_repository_explorer_configs" + LegacyRepositoryConfigsCollection = "loom_repository_explorer_configs" ) -// CollectionSpecs is deliberately non-truncating: both draft history pointers -// and immutable revision/share URLs must survive normal server bootstrap. func CollectionSpecs() []store.CollectionSpec { return []store.CollectionSpec{ {Name: ExplorersCollection, Indexes: [][]string{{"project", "explorerId"}}}, diff --git a/internal/explorer/arango/collections_test.go b/internal/backends/arango/internal/explorer/collections_test.go similarity index 100% rename from internal/explorer/arango/collections_test.go rename to internal/backends/arango/internal/explorer/collections_test.go diff --git a/internal/explorer/arango/cross_generation_publish_e2e_test.go b/internal/backends/arango/internal/explorer/cross_generation_publish_e2e_test.go similarity index 74% rename from internal/explorer/arango/cross_generation_publish_e2e_test.go rename to internal/backends/arango/internal/explorer/cross_generation_publish_e2e_test.go index 81cb68cc6..f20867130 100644 --- a/internal/explorer/arango/cross_generation_publish_e2e_test.go +++ b/internal/backends/arango/internal/explorer/cross_generation_publish_e2e_test.go @@ -1,51 +1,5 @@ package arango -// VERIFICATION-ONLY artifact owned by the F-B13 cross-generation publish audit. -// It never mutates product code. It creates a fully owned ArangoDB database -// (loom_verify_cgp_) through the production store.Open path and drops it -// with an HTTP DELETE in t.Cleanup, logging removal. -// -// F-B13 regression: publishing a retained OLD-generation Explorer receipt -// after a NEWER generation has been activated must not repoint the project's -// active_generation to the old generation. An explicit generation activation -// first selects the rollback target and remains valid. -// The publish preflight does not compare receipt.SourceGeneration to the active -// generation pointer. The release CAS now supplies that transactional check. -// -// - internal/explorer/lifecycle/publish.go:33 only checks the receipt's -// capability snapshot is self-consistent with receipt.SourceGeneration, -// never the active pointer. -// - internal/explorer/lifecycle/publish.go:58 validates only that -// receipt.SourceGeneration is a staged manifest (ValidateReleaseGeneration -// -> ReadManifest(...).IsStaged), never that it is the active generation. -// - internal/dataset/arango/release_store.go:153-179 (activateReleaseAQL) is -// reached by datasetarango.CompareAndSwapPublishRelease at -// internal/explorer/arango/publication.go:57, the exact call the Explorer -// publish binding makes. It resolves `manifest` from the RELEASE's own -// generation (manifest_key from release.Generation, release_store.go:80) -// and patches the shared active_generation pointer only when its project, -// generation and manifest key still match the selected manifest. -// -// REACHABILITY / LAYER HONESTY. The full lifecycle.Service.Publish and the -// Explorer capability resolver live in internal/server (compileExplorerReceipt, -// explorerCapabilityResolver). import "internal/server" would form an import -// cycle for this package (internal/server already imports internal/explorer/ -// arango), so the compile step and the receipt-authority gate are NOT driven; -// they are replicated at the level the hypothesis actually needs. Store. -// PublishAuthoring (publication.go:13) IS reachable in this package, but it -// requires a production-valid explorer.CompilationReceipt that can only be -// produced by the internal/server compile stack, and it merely wraps the same -// datasetarango.CompareAndSwapPublishRelease call at publication.go:57 plus -// unrelated explorer-revision bookkeeping. So this test drives the real -// production binding surface that owns the bug: dataset.ReleaseService.Create -// (the publish-path preflight at release.go:130-229) plus the real -// datasetarango.CompareAndSwapPublishRelease (the CAS publication.go:57 -// invokes), against real ArangoDB. This is the release-CAS / publish-binding -// layer, labelled as such in the verdict. The ResolveForExecution vs -// ResolveForCompilation asymmetry lives in the server file; the equivalent -// asymmetry that repoints the pointer here is that the binding path resolves -// the manifest for release.Generation instead of the active generation. - import ( "context" "encoding/json" @@ -59,22 +13,20 @@ import ( "testing" "time" + datasetarango "github.com/calypr/loom/internal/backends/arango/internal/dataset" + publicationarango "github.com/calypr/loom/internal/backends/arango/internal/publication" + store "github.com/calypr/loom/internal/backends/arango/internal/store" dataframepublication "github.com/calypr/loom/internal/dataframe/publication" - publicationarango "github.com/calypr/loom/internal/dataframe/publication/arango" "github.com/calypr/loom/internal/dataframe/published" "github.com/calypr/loom/internal/dataset" - datasetarango "github.com/calypr/loom/internal/dataset/arango" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) const cgpDatabasePrefix = "loom_verify_cgp_" -// cgpCitation pins the load-bearing source lines the verdict relies on. A drift -// fails the run so a stale conclusion cannot silently survive a refactor. type cgpCitation struct { file string - line int // 1-based + line int want string } @@ -82,17 +34,15 @@ var cgpCitations = []cgpCitation{ {"internal/explorer/lifecycle/publish.go", 33, `if err != nil || strings.TrimSpace(snapshot.Identity.Generation) != strings.TrimSpace(receipt.SourceGeneration) {`}, {"internal/explorer/lifecycle/publish.go", 58, `if err := s.config.ValidateReleaseGeneration(ctx, projectid.Legacy(receipt.Project), receipt.SourceGeneration); err != nil {`}, {"internal/explorer/lifecycle/publish.go", 87, `release, expectedReleaseRevision, err := s.config.PrepareRelease(ctx, projectid.Legacy(receipt.Project), receipt.SourceGeneration, selectorsForBundle(receipt.Bundle))`}, - {"internal/explorer/arango/publication.go", 57, `if _, err := datasetarango.CompareAndSwapPublishRelease(txCtx, tx, release, expectedReleaseRevision); err != nil {`}, - {"internal/dataset/arango/release_store.go", 178, `LET generationPointer = DOCUMENT(@@lifecycle_collection, @active_key)`}, - {"internal/dataset/arango/release_store.go", 192, ` AND generationPointer.dataset.generation == @generation`}, - {"internal/dataset/arango/release_store.go", 202, ` {document: generationPointer, patch: {dataset: manifest.dataset, manifestKey: manifest._key, releaseId: @release_id, releaseRevision: selectedRevision}}`}, - {"internal/server/explorer_capability.go", 177, `func (r *explorerCapabilityResolver) ResolveForExecution(ctx context.Context, project, token string) (lifecycle.AuthorizedCapability, error) {`}, - {"internal/server/explorer_capability.go", 208, `func (r *explorerCapabilityResolver) ResolveForCompilation(ctx context.Context, project, token string) (lifecycle.AuthorizedCapability, error) {`}, - {"internal/server/explorer_capability.go", 218, `if authorized.Snapshot.Identity.Generation != manifest.Dataset.Generation {`}, + {"internal/backends/arango/internal/explorer/publication.go", 57, `if _, err := datasetarango.CompareAndSwapPublishRelease(txCtx, tx, release, expectedReleaseRevision); err != nil {`}, + {"internal/backends/arango/internal/dataset/release_store.go", 179, `LET generationPointer = DOCUMENT(@@lifecycle_collection, @active_key)`}, + {"internal/backends/arango/internal/dataset/release_store.go", 193, ` AND generationPointer.dataset.generation == @generation`}, + {"internal/backends/arango/internal/dataset/release_store.go", 203, ` {document: generationPointer, patch: {dataset: manifest.dataset, manifestKey: manifest._key, releaseId: @release_id, releaseRevision: selectedRevision}}`}, + {"internal/server/explorer_capability.go", 162, `func (r *explorerCapabilityResolver) ResolveForExecution(ctx context.Context, project, token string) (lifecycle.AuthorizedCapability, error) {`}, + {"internal/server/explorer_capability.go", 189, `func (r *explorerCapabilityResolver) ResolveForCompilation(ctx context.Context, project, token string) (lifecycle.AuthorizedCapability, error) {`}, + {"internal/server/explorer_capability.go", 199, `if authorized.Snapshot.Identity.Generation != manifest.Dataset.Generation {`}, } -// cgpAbsentIdentifiers prove the Explorer publish preflight does not substitute -// its own active-generation lookup for the transactional store guard. type cgpAbsentIdentifier struct { file string identifier string @@ -154,10 +104,6 @@ func cgpCheckCitations(t *testing.T) { } } -// cgpVerifier mirrors the production publicationVerificationStore -// (internal/server/release_verifier.go:18-34) exactly, over the real ArangoDB -// publication Registry, so the ReleaseService binding gate consumes a genuine -// PUBLISHED/queryable execution rather than a hand-made success. type cgpVerifier struct { executions dataframepublication.ExactExecutionCatalog } @@ -183,9 +129,6 @@ func (s cgpVerifier) VerifyPublication(ctx context.Context, project, generation var _ dataset.PublicationVerifier = cgpVerifier{} -// cgpOpen is the env-gated owned-database harness (skip-clean when -// LOOM_TEST_ARANGO_URL is unset). It bootstraps only the dataset lifecycle and -// dataframe-publication collections, which is all the F-B13 binding path needs. func cgpOpen(t *testing.T, budget time.Duration) (context.Context, *store.Client, *datasetarango.Store, *publicationarango.Registry) { t.Helper() endpoint := os.Getenv("LOOM_TEST_ARANGO_URL") @@ -241,10 +184,6 @@ func cgpOpen(t *testing.T, budget time.Duration) (context.Context, *store.Client return ctx, client, lifecycle, registry } -// cgpLoadAndActivate drives the real dataset lifecycle Store: create a LOADING -// manifest, transition it to STAGED (graph-loaded), and Activate it so the -// shared active_generation pointer moves to this generation. Returns the staged -// manifest. Mirrors the dataset store tests' load/activate surface. func cgpLoadAndActivate(t *testing.T, ctx context.Context, lifecycle *datasetarango.Store, project, generation, digest string) dataset.Manifest { t.Helper() ref, err := dataset.NewRef(project, generation) @@ -278,12 +217,6 @@ func cgpLoadAndActivate(t *testing.T, ctx context.Context, lifecycle *datasetara return staged } -// cgpSeedPublished writes one PUBLISHED, queryable execution through the -// production lease + SaveExecution path and names it via CompareAndSwapPointer, -// mirroring publication_rows_integration_test.go / dangling_binding_e2e_test.go. -// recipe/version/output are the selector identity that a generation releases; -// unique isolates the execution id/key/lease per generation so two generations -// publishing the SAME selector do not collide on _key. func cgpSeedPublished(t *testing.T, ctx context.Context, registry *publicationarango.Registry, project, generation, recipe, version, output, unique string) (dataframepublication.BundleExecution, dataset.DataframeSelector) { t.Helper() id := "cgp-exec-" + unique @@ -324,9 +257,6 @@ func cgpSeedPublished(t *testing.T, ctx context.Context, registry *publicationar return execution, selector } -// cgpActiveGenerationPointer reads the shared active_generation pointer doc -// directly from the lifecycle collection by role, not by a replicated key, so -// the OBSERVED pointer cannot be biased by a key-deribration guess. func cgpActiveGenerationPointer(t *testing.T, ctx context.Context, client *store.Client, project string) map[string]any { t.Helper() var pointer map[string]any @@ -367,8 +297,6 @@ func cgpMaterializationDigest(values []published.Materialization) string { return "[" + strings.Join(summaries, " ") + "]" } -// TestStaleReceiptPublishDowngradesActiveGenerationAgainstArango is the F-B13 -// finding-assert at the release-CAS / publish-binding layer. func TestStaleReceiptPublishDowngradesActiveGenerationAgainstArango(t *testing.T) { ctx, client, lifecycle, registry := cgpOpen(t, 180*time.Second) cgpCheckCitations(t) @@ -380,28 +308,17 @@ func TestStaleReceiptPublishDowngradesActiveGenerationAgainstArango(t *testing.T const recipe = "fb13-recipe" const output = "out" - // Step 2: PUBLISHED, queryable execution + pointer for BOTH generations. - // Both generations release the same selector identity (recipe/version/output); - // only the generation and the execution id/lease differ. The shared selector - // lets one ReleaseService.Required set resolve in either generation, exactly - // as publication_rows_integration_test.go binds recipe+translationVersion+output. _, selectorG1 := cgpSeedPublished(t, ctx, registry, project, g1, recipe, version, output, "g1") cgpSeedPublished(t, ctx, registry, project, g2, recipe, version, output, "g2") t.Logf("OBSERVED publications: G1 selector=%+v (exec id cgp-exec-g1) G2 selector identical recipe (exec id cgp-exec-g2)", selectorG1) - // Step 1: two loaded+activated generations G1 then G2 (production lifecycle - // store load/activate). Deterministic ordering, single writer. cgpLoadAndActivate(t, ctx, lifecycle, project, g1, strings.Repeat("a", 64)) cgpLoadAndActivate(t, ctx, lifecycle, project, g2, strings.Repeat("b", 64)) - // Step 4 (baseline, PRE-STATED): active generation reads == G2. t.Log(`PRE-STATED EXPECT: baseline active generation reads == G2 (activation is last-writer-wins; G2 activated second).`) readerManifest := &published.Reader{Catalog: registry, ActiveManifestResolver: lifecycle} readerRelease := &published.Reader{Catalog: registry, ActiveReleaseResolver: lifecycle} - // The general release activation is allowed to select a generation. Create - // G1 first, then switch to G2 through ReleaseService.Activate to prove the - // same CAS remains available for legitimate generation changes. releaseService := dataset.ReleaseService{ Manifests: lifecycle, Releases: lifecycle, Verifier: cgpVerifier{executions: registry}, @@ -433,10 +350,6 @@ func TestStaleReceiptPublishDowngradesActiveGenerationAgainstArango(t *testing.T t.Fatalf("BASELINE PRECONDITION BROKEN: discovery must report only G2 generation; manifest=%s release=%s", cgpMaterializationDigest(manifestList), cgpMaterializationDigest(releaseList)) } - // Step 5b: the F-B13 action. Publish the retained G1 receipt post-G2 - // activation through the same strict current-generation CAS used by the - // production Explorer PublishAuthoring transaction. The current release - // revision is read dynamically so the revision CAS itself is not stale. current, err := lifecycle.ReadActiveRelease(ctx, project) if err != nil { t.Fatalf("read current active release before G1 publish: %v", err) @@ -450,10 +363,6 @@ func TestStaleReceiptPublishDowngradesActiveGenerationAgainstArango(t *testing.T } t.Logf("OBSERVED G1 release candidate: id=%s generation=%s bindings=%d (Create preflight REJECTED nothing though G2 is active)", g1Release.ID, g1Release.Generation, len(g1Release.Publications)) - // G1 was an active release before G2 and its immutable candidate is retained. - // Publishing that retained candidate must fail the current-generation CAS. - // The general ReleaseService activation remains the explicit generation - // selection path and must still permit rollback to G1. t.Log("PRE-STATED EXPECT: the retained G1 candidate conflicts on the publish CAS while G2 is active; the general ReleaseService activation can explicitly roll back to G1.") casActive, casErr := datasetarango.CompareAndSwapPublishRelease(ctx, client, g1Release, current.Revision) @@ -469,7 +378,6 @@ func TestStaleReceiptPublishDowngradesActiveGenerationAgainstArango(t *testing.T t.Logf("OBSERVED post-CAS CurrentProjectDatasets manifest-path=%s err=%v", cgpMaterializationDigest(postManifestList), postManifestListErr) t.Logf("OBSERVED post-CAS CurrentProjectDatasets release-path=%s err=%v", cgpMaterializationDigest(postReleaseList), postReleaseListErr) - // Step 6 verdict, entirely from OBSERVED values. silentlyRepointed := casErr == nil && observedGeneration == g1 && postManifestErr == nil && postManifest.Dataset.Generation == g1 && postActive.Release.Generation == g1 prevented := errors.Is(casErr, dataset.ErrReleaseActivationConflict) && observedGeneration == g2 && postManifestErr == nil && postManifest.Dataset.Generation == g2 && postReleaseErr == nil && postActive.Release.Generation == g2 diff --git a/internal/explorer/arango/draft_conflict_test.go b/internal/backends/arango/internal/explorer/draft_conflict_test.go similarity index 96% rename from internal/explorer/arango/draft_conflict_test.go rename to internal/backends/arango/internal/explorer/draft_conflict_test.go index a82ab84a3..54a71a1c9 100644 --- a/internal/explorer/arango/draft_conflict_test.go +++ b/internal/backends/arango/internal/explorer/draft_conflict_test.go @@ -7,8 +7,8 @@ import ( "testing" "github.com/arangodb/go-driver/v2/arangodb/shared" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" - store "github.com/calypr/loom/internal/store/arango" ) type draftTransactionFailureClient struct { diff --git a/internal/explorer/arango/draft_transaction_formal_replay_test.go b/internal/backends/arango/internal/explorer/draft_transaction_formal_replay_test.go similarity index 99% rename from internal/explorer/arango/draft_transaction_formal_replay_test.go rename to internal/backends/arango/internal/explorer/draft_transaction_formal_replay_test.go index 1308e6459..4410f88c4 100644 --- a/internal/explorer/arango/draft_transaction_formal_replay_test.go +++ b/internal/backends/arango/internal/explorer/draft_transaction_formal_replay_test.go @@ -13,8 +13,8 @@ import ( "time" "github.com/arangodb/go-driver/v2/arangodb/shared" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" - store "github.com/calypr/loom/internal/store/arango" ) const draftTransactionRevisionID = "draft_revision_5a5bd747c995d2a79aba154d30be769c0fd16d0329b5eb8ed60efa0f073e02d8" diff --git a/internal/explorer/arango/drafts.go b/internal/backends/arango/internal/explorer/drafts.go similarity index 98% rename from internal/explorer/arango/drafts.go rename to internal/backends/arango/internal/explorer/drafts.go index 26ad8abcf..a5c13c29b 100644 --- a/internal/explorer/arango/drafts.go +++ b/internal/backends/arango/internal/explorer/drafts.go @@ -10,8 +10,8 @@ import ( "time" "github.com/arangodb/go-driver/v2/arangodb/shared" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" - store "github.com/calypr/loom/internal/store/arango" ) func (s *Store) SaveDraft(ctx context.Context, e explorer.Explorer, expected int64, expectedDigest ...string) (*explorer.Explorer, error) { diff --git a/internal/explorer/arango/drafts_integration_test.go b/internal/backends/arango/internal/explorer/drafts_integration_test.go similarity index 99% rename from internal/explorer/arango/drafts_integration_test.go rename to internal/backends/arango/internal/explorer/drafts_integration_test.go index f88f28c3f..ae7f0a4cc 100644 --- a/internal/explorer/arango/drafts_integration_test.go +++ b/internal/backends/arango/internal/explorer/drafts_integration_test.go @@ -15,11 +15,11 @@ import ( "time" "github.com/arangodb/go-driver/v2/arangodb/shared" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" "github.com/calypr/loom/internal/explorer/authoringv2" "github.com/calypr/loom/internal/explorer/capability" "github.com/calypr/loom/internal/explorer/lifecycle" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) diff --git a/internal/explorer/arango/drafts_test.go b/internal/backends/arango/internal/explorer/drafts_test.go similarity index 98% rename from internal/explorer/arango/drafts_test.go rename to internal/backends/arango/internal/explorer/drafts_test.go index 071bfa0f7..b6587107c 100644 --- a/internal/explorer/arango/drafts_test.go +++ b/internal/backends/arango/internal/explorer/drafts_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" + storepkg "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" - storepkg "github.com/calypr/loom/internal/store/arango" ) type draftHistoryClient struct { @@ -130,7 +130,7 @@ func TestSaveDraftPersistsExactPreimageAtomicallyAndPreservesActiveRevision(t *t updated := previous updated.DraftConfig = json.RawMessage(`{"documents":[{"outputId":"patients","construction":{"steps":[{"id":"pivot","outputs":[{"id":"col_age","name":"age"}]},{"id":"derive","outputs":[{"id":"col_ratio","name":"ratio"}]}]}}]}`) - updated.DraftVersion = 999 // The adapter owns the monotonic CAS version. + updated.DraftVersion = 999 updated.DraftDigest = "sha256:new-draft" updated.DraftSnapshotToken = "snapshot-new" updated.DraftSourceGeneration = "generation-new" diff --git a/internal/explorer/arango/explicit_group_storage_integration_test.go b/internal/backends/arango/internal/explorer/explicit_group_storage_integration_test.go similarity index 99% rename from internal/explorer/arango/explicit_group_storage_integration_test.go rename to internal/backends/arango/internal/explorer/explicit_group_storage_integration_test.go index 3a137c4c4..db889f11d 100644 --- a/internal/explorer/arango/explicit_group_storage_integration_test.go +++ b/internal/backends/arango/internal/explorer/explicit_group_storage_integration_test.go @@ -10,9 +10,9 @@ import ( "testing" "time" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" "github.com/calypr/loom/internal/projectid" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -35,7 +35,6 @@ func TestExplicitGroupStorageAgainstArango(t *testing.T) { t.Fatal(err) } - // Repository metadata receives canonical projects from the service boundary. project := projectid.Canonical("loom_explicit_group_test_" + uuid.NewString()) source, sourceWriter, err := persistExplicitGroupTestSelection(ctx, t, persistence, project) if err != nil { diff --git a/internal/explorer/arango/explicit_groups.go b/internal/backends/arango/internal/explorer/explicit_groups.go similarity index 99% rename from internal/explorer/arango/explicit_groups.go rename to internal/backends/arango/internal/explorer/explicit_groups.go index b36868ef3..a518aac54 100644 --- a/internal/explorer/arango/explicit_groups.go +++ b/internal/backends/arango/internal/explorer/explicit_groups.go @@ -12,8 +12,8 @@ import ( "time" "github.com/arangodb/go-driver/v2/arangodb/shared" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" - store "github.com/calypr/loom/internal/store/arango" ) const ( @@ -454,9 +454,6 @@ func readExplicitGroupSourceSelection(ctx context.Context, queryer store.RowQuer return source, nil } -// validateExplicitGroupSourceMemberships runs inside each append/completion -// boundary. It checks every relation against the immutable source selection -// before any group-membership row can be inserted or the revision completed. func validateExplicitGroupSourceMemberships(ctx context.Context, queryer store.RowQueryer, header explorer.ExplicitGroupRevision, source explorer.SelectionRevision, memberships []explorer.ExplicitGroupMembership) error { if err := header.ValidateSourceSelection(source); err != nil { return err diff --git a/internal/explorer/arango/explicit_groups_test.go b/internal/backends/arango/internal/explorer/explicit_groups_test.go similarity index 98% rename from internal/explorer/arango/explicit_groups_test.go rename to internal/backends/arango/internal/explorer/explicit_groups_test.go index e2105a7ce..e87d5f21c 100644 --- a/internal/explorer/arango/explicit_groups_test.go +++ b/internal/backends/arango/internal/explorer/explicit_groups_test.go @@ -8,8 +8,8 @@ import ( "testing" "time" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" - store "github.com/calypr/loom/internal/store/arango" ) func TestPutExplicitGroupDefinitionsUsesObjectRows(t *testing.T) { diff --git a/internal/explorer/arango/explorers.go b/internal/backends/arango/internal/explorer/explorers.go similarity index 100% rename from internal/explorer/arango/explorers.go rename to internal/backends/arango/internal/explorer/explorers.go diff --git a/internal/explorer/arango/interpretations.go b/internal/backends/arango/internal/explorer/interpretations.go similarity index 99% rename from internal/explorer/arango/interpretations.go rename to internal/backends/arango/internal/explorer/interpretations.go index 5988c27e0..7773d5eba 100644 --- a/internal/explorer/arango/interpretations.go +++ b/internal/backends/arango/internal/explorer/interpretations.go @@ -4,8 +4,8 @@ import ( "context" "fmt" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" - store "github.com/calypr/loom/internal/store/arango" ) var _ explorer.InterpretationRepository = (*Store)(nil) diff --git a/internal/explorer/arango/interpretations_test.go b/internal/backends/arango/internal/explorer/interpretations_test.go similarity index 98% rename from internal/explorer/arango/interpretations_test.go rename to internal/backends/arango/internal/explorer/interpretations_test.go index 06bdea7e5..528c7449d 100644 --- a/internal/explorer/arango/interpretations_test.go +++ b/internal/backends/arango/internal/explorer/interpretations_test.go @@ -7,9 +7,9 @@ import ( "strings" "testing" + storepkg "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" "github.com/calypr/loom/internal/explorer/authoringv2" - storepkg "github.com/calypr/loom/internal/store/arango" ) type interpretationClient struct { @@ -130,7 +130,6 @@ func TestInterpretationArangoCreateRetryCollisionParentConflictAndIsolation(t *t if _, err := adapter.CreateInterpretationRevision(ctx, stale, nil); !errors.Is(err, explorer.ErrInterpretationParentConflict) { t.Fatalf("stale root error=%v", err) } - // A persisted key collision with different immutable identity fails closed. prepared, err := explorer.PrepareInterpretationRevision(interpretationTestRevision("project-a")) if err != nil { t.Fatal(err) diff --git a/internal/explorer/arango/publication.go b/internal/backends/arango/internal/explorer/publication.go similarity index 95% rename from internal/explorer/arango/publication.go rename to internal/backends/arango/internal/explorer/publication.go index e41f997a6..0c7297879 100644 --- a/internal/explorer/arango/publication.go +++ b/internal/backends/arango/internal/explorer/publication.go @@ -4,10 +4,10 @@ import ( "context" "time" + datasetarango "github.com/calypr/loom/internal/backends/arango/internal/dataset" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/dataset" - datasetarango "github.com/calypr/loom/internal/dataset/arango" "github.com/calypr/loom/internal/explorer" - store "github.com/calypr/loom/internal/store/arango" ) func (s *Store) PublishAuthoring(ctx context.Context, receipt explorer.CompilationReceipt, revision explorer.Revision, release dataset.ProjectRelease, expectedReleaseRevision int64) (*explorer.Revision, error) { diff --git a/internal/explorer/arango/receipt_store_test.go b/internal/backends/arango/internal/explorer/receipt_store_test.go similarity index 99% rename from internal/explorer/arango/receipt_store_test.go rename to internal/backends/arango/internal/explorer/receipt_store_test.go index 3bdfcbb01..f36c04947 100644 --- a/internal/explorer/arango/receipt_store_test.go +++ b/internal/backends/arango/internal/explorer/receipt_store_test.go @@ -9,9 +9,9 @@ import ( "testing" "github.com/arangodb/go-driver/v2/arangodb/shared" + storepkg "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" - storepkg "github.com/calypr/loom/internal/store/arango" ) type receiptClient struct { diff --git a/internal/explorer/arango/receipts.go b/internal/backends/arango/internal/explorer/receipts.go similarity index 100% rename from internal/explorer/arango/receipts.go rename to internal/backends/arango/internal/explorer/receipts.go diff --git a/internal/explorer/arango/repository.go b/internal/backends/arango/internal/explorer/repository.go similarity index 85% rename from internal/explorer/arango/repository.go rename to internal/backends/arango/internal/explorer/repository.go index 69246b87e..d96222c78 100644 --- a/internal/explorer/arango/repository.go +++ b/internal/backends/arango/internal/explorer/repository.go @@ -13,9 +13,6 @@ import ( "github.com/calypr/loom/internal/projectid" ) -// legacyRepositoryConfig contains only the fields needed to recover the -// canonical default Explorer owner. Immutable generated state already lives in -// the referenced revision and is deliberately not decoded or copied here. type legacyRepositoryConfig struct { Project string `json:"project"` Workspace json.RawMessage `json:"workspace"` @@ -24,9 +21,6 @@ type legacyRepositoryConfig struct { UpdatedAt time.Time `json:"updatedAt"` } -// MigrateLegacyRepositoryConfigs is an idempotent startup migration. It -// recovers only missing repository owners and never overwrites a current owner -// or deletes the legacy source document. func (s *Store) MigrateLegacyRepositoryConfigs(ctx context.Context) error { legacy := []legacyRepositoryConfig{} err := s.client.QueryRows(ctx, `FOR d IN @@c FILTER d.activeRevisionId != null AND d.activeRevisionId != "" RETURN d`, 1000, map[string]any{"@c": LegacyRepositoryConfigsCollection}, func(row map[string]any) error { diff --git a/internal/explorer/arango/repository_test.go b/internal/backends/arango/internal/explorer/repository_test.go similarity index 97% rename from internal/explorer/arango/repository_test.go rename to internal/backends/arango/internal/explorer/repository_test.go index 412b08bb7..765bcfb17 100644 --- a/internal/explorer/arango/repository_test.go +++ b/internal/backends/arango/internal/explorer/repository_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - storepkg "github.com/calypr/loom/internal/store/arango" + storepkg "github.com/calypr/loom/internal/backends/arango/internal/store" ) type migrationClient struct { diff --git a/internal/explorer/arango/revisions.go b/internal/backends/arango/internal/explorer/revisions.go similarity index 97% rename from internal/explorer/arango/revisions.go rename to internal/backends/arango/internal/explorer/revisions.go index c03df1bd1..d67571c2c 100644 --- a/internal/explorer/arango/revisions.go +++ b/internal/backends/arango/internal/explorer/revisions.go @@ -102,8 +102,6 @@ func (s *Store) GetRevision(ctx context.Context, id string) (*explorer.Revision, return out, nil } -// FailRevision records the only non-activation revision transition performed -// outside the atomic activation workflows. func (s *Store) FailRevision(ctx context.Context, id string, diagnostics []explorer.Diagnostic) (*explorer.Revision, error) { now := time.Now().UTC() patch := map[string]any{"status": explorer.RevisionFailed, "diagnostics": diagnostics, "failedAt": now} diff --git a/internal/explorer/arango/selection_references.go b/internal/backends/arango/internal/explorer/selection_references.go similarity index 83% rename from internal/explorer/arango/selection_references.go rename to internal/backends/arango/internal/explorer/selection_references.go index d360471f4..e3ce7db3c 100644 --- a/internal/explorer/arango/selection_references.go +++ b/internal/backends/arango/internal/explorer/selection_references.go @@ -13,11 +13,6 @@ import ( const selectionReferenceBatchSize = 1000 -// ValidateSelectionReferences verifies explicit selection references against -// the concrete resource collection for the requested immutable generation. -// It intentionally runs in the storage adapter: HTTP callers must not be -// able to turn guessed IDs into persisted members by skipping the active -// generation and auth_resource_path predicates. func (s *Store) ValidateSelectionReferences(ctx context.Context, project, generation string, scope authscope.ReadScope, refs []explorer.ResourceRef) error { project = projectid.Canonical(project) generation = strings.TrimSpace(generation) @@ -80,10 +75,6 @@ func (s *Store) ValidateSelectionReferences(ctx context.Context, project, genera if _, ok := found[id]; ok { continue } - // Under a restricted scope, absence is deliberately reported as a - // stale/scope error rather than silently filtering the reference. - // This covers both unauthorized IDs and IDs from a changed active - // generation without disclosing which predicate failed. if !scope.Unrestricted() { return fmt.Errorf("%w: resource reference is unavailable in the authorized scope", explorer.ErrResourceRefScopeMismatch) } diff --git a/internal/explorer/arango/selection_references_test.go b/internal/backends/arango/internal/explorer/selection_references_test.go similarity index 98% rename from internal/explorer/arango/selection_references_test.go rename to internal/backends/arango/internal/explorer/selection_references_test.go index cf82a87f2..8f80c5800 100644 --- a/internal/explorer/arango/selection_references_test.go +++ b/internal/backends/arango/internal/explorer/selection_references_test.go @@ -7,8 +7,8 @@ import ( "testing" "github.com/calypr/loom/internal/authscope" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" - store "github.com/calypr/loom/internal/store/arango" ) type selectionReferenceClient struct { diff --git a/internal/explorer/arango/selection_scale_integration_test.go b/internal/backends/arango/internal/explorer/selection_scale_integration_test.go similarity index 90% rename from internal/explorer/arango/selection_scale_integration_test.go rename to internal/backends/arango/internal/explorer/selection_scale_integration_test.go index 3436da3d3..c9d3dd242 100644 --- a/internal/explorer/arango/selection_scale_integration_test.go +++ b/internal/backends/arango/internal/explorer/selection_scale_integration_test.go @@ -7,15 +7,11 @@ import ( "testing" "time" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) -// TestSelectionStorageScaleAgainstArango measures the production selection -// path at the plan's probe sizes. It deliberately uses the same Begin, -// Append, Digest, and Complete methods as lifecycle requests; no fake store -// or in-memory digest substitute is involved. func TestSelectionStorageScaleAgainstArango(t *testing.T) { url, database := os.Getenv("LOOM_TEST_ARANGO_URL"), os.Getenv("LOOM_TEST_ARANGO_DATABASE") if url == "" || database == "" { diff --git a/internal/explorer/arango/selection_storage_integration_test.go b/internal/backends/arango/internal/explorer/selection_storage_integration_test.go similarity index 95% rename from internal/explorer/arango/selection_storage_integration_test.go rename to internal/backends/arango/internal/explorer/selection_storage_integration_test.go index 32cd6f627..4790e867f 100644 --- a/internal/explorer/arango/selection_storage_integration_test.go +++ b/internal/backends/arango/internal/explorer/selection_storage_integration_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -50,8 +50,6 @@ func TestSelectionStorageAgainstArango(t *testing.T) { } refs := []explorer.SelectionMember{{Ref: explorer.ResourceRef{Project: header.Project, Generation: header.Generation, ResourceType: header.ResourceType, ID: "files001"}}, {Ref: explorer.ResourceRef{Project: header.Project, Generation: header.Generation, ResourceType: header.ResourceType, ID: "files002"}}, {Ref: explorer.ResourceRef{Project: header.Project, Generation: header.Generation, ResourceType: header.ResourceType, ID: "files001"}}} - // BeginSelection reaps expired staging leases, allowing an idempotent - // retry to recover both empty and partially populated crashed attempts. expiredEmptyID := "selection-expired-empty-" + uuid.NewString() expiredEmpty := header expiredEmpty.ID, expiredEmpty.IdempotencyKey = expiredEmptyID, uuid.NewString() @@ -135,9 +133,6 @@ func TestSelectionStorageAgainstArango(t *testing.T) { t.Fatalf("changed idempotency intent err=%v", err) } - // A stale retry may race a live writer, but its abort must not remove the - // live writer's staged members. Repeat this against production Arango - // transactions so a write-skew implementation cannot pass by luck. for i := 0; i < 50; i++ { raceID := "selection-owner-race-" + uuid.NewString() owner := uuid.NewString() diff --git a/internal/explorer/arango/selections.go b/internal/backends/arango/internal/explorer/selections.go similarity index 93% rename from internal/explorer/arango/selections.go rename to internal/backends/arango/internal/explorer/selections.go index 45cb4ee43..925fb21a9 100644 --- a/internal/explorer/arango/selections.go +++ b/internal/backends/arango/internal/explorer/selections.go @@ -8,8 +8,8 @@ import ( "strings" "time" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" - store "github.com/calypr/loom/internal/store/arango" ) const ( @@ -34,10 +34,6 @@ func (s *Store) BeginSelection(ctx context.Context, selection explorer.Selection if selection.Complete { return nil, fmt.Errorf("selection must begin incomplete") } - // A crashed creator leaves a staging header behind. Reap only expired - // leases, in a bounded batch, before idempotency lookup so a retry with the - // same key can claim a clean immutable revision. Active writers are - // protected by writerExpiresAt and are never touched here. if err := s.CleanupSelectionStaging(ctx, time.Now().UTC(), selectionCleanupLimit); err != nil { return nil, err } @@ -95,9 +91,6 @@ func (s *Store) AppendSelectionMembers(ctx context.Context, selectionID, writerT if len(members) == 0 { return nil, nil } - // A batch is intentionally bounded by the lifecycle caller. The adapter - // inserts each member with an immutable key so retries are idempotent and - // never require buffering the complete membership. inserted := make([]explorer.SelectionMember, 0, len(members)) err := s.client.WithTransaction(ctx, store.TransactionCollections{Write: []string{SelectionsCollection, SelectionMembersCollection}}, func(txCtx context.Context, tx store.RowQueryer) error { var header *explorer.SelectionRevision @@ -157,9 +150,6 @@ IN @@c RETURN {key: NEW._key, inserted: OLD == null}`, len(docs), map[string]any{"@c": SelectionMembersCollection, "docs": docs}, func(row map[string]any) error { key, _ := row["key"].(string) value, _ := row["inserted"].(bool) - // A duplicate key can occur within one parameter batch. Arango - // reports the first UPSERT as inserted and the retry as existing; - // preserve the fact that this call created the member. created[key] = created[key] || value return nil }); err != nil { @@ -378,9 +368,6 @@ func (s *Store) readSelection(ctx context.Context, query string, binds map[strin return result, nil } -// CleanupSelectionStaging removes bounded abandoned staging records. It is -// deliberately explicit so startup callers can budget cleanup work rather -// than scanning all selections into memory. func (s *Store) CleanupSelectionStaging(ctx context.Context, before time.Time, limit int) error { if limit <= 0 { limit = 100 diff --git a/internal/explorer/arango/selections_cleanup_e2e_test.go b/internal/backends/arango/internal/explorer/selections_cleanup_e2e_test.go similarity index 86% rename from internal/explorer/arango/selections_cleanup_e2e_test.go rename to internal/backends/arango/internal/explorer/selections_cleanup_e2e_test.go index b194bd4b9..af732ceb6 100644 --- a/internal/explorer/arango/selections_cleanup_e2e_test.go +++ b/internal/backends/arango/internal/explorer/selections_cleanup_e2e_test.go @@ -12,26 +12,12 @@ import ( "testing" "time" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" "github.com/calypr/loom/internal/explorer/lifecycle" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) -// These tests drive the real selection staging lifecycle against a private -// ArangoDB database and only ever call production Store methods. They exist to -// test two claims about CleanupSelectionStaging (selections.go:384) instead of -// trusting its comment at selections.go:37-40: -// -// H3-a: a writer that is legitimately mid-operation but whose writer lease has -// lapsed is never touched by another session's BeginSelection. -// H3-b: cleanup cannot destroy a completed selection, because it omits the -// complete != true guard that AbortSelection (selections.go:290) and -// CompleteSelection (selections.go:242) both carry. -// -// Databases use the loom_exv_ prefix so this suite's residue is checkable -// independently of other verification workers. - const selectionCleanupDatabasePrefix = "loom_exv_" type selectionCleanupDocument struct { @@ -130,12 +116,6 @@ RETURN {missing: d == null, state: d == null ? null : d.state, complete: d != nu return info } -// selectionCleanupExpireLease rewinds writerExpiresAt into the past. Production -// renews the lease only inside AppendSelectionMembers (selections.go:168), so an -// idle gap past the 10m lease has exactly this end state. The existing harness -// uses the same direct patch with the literal 0 -// (selection_storage_integration_test.go:29-34); this variant keeps the value -// explicit about how far past the lease the document is. func selectionCleanupExpireLease(t *testing.T, ctx context.Context, client *store.Client, selectionID string, lapsedFor time.Duration) int64 { t.Helper() expired := time.Now().UTC().Add(-selectionWriterLease - lapsedFor).UnixMilli() @@ -146,9 +126,6 @@ func selectionCleanupExpireLease(t *testing.T, ctx context.Context, client *stor return expired } -// selectionCleanupAssertNoOrphans fails when staged members exist whose header -// is gone. Cleanup removes members and the header in one AQL query, so an orphan -// set is the observable signature of a torn reap. func selectionCleanupAssertNoOrphans(t *testing.T, ctx context.Context, client *store.Client, label string) { t.Helper() var orphans int64 @@ -193,17 +170,10 @@ func selectionCleanupMembers(project, generation, resourceType, prefix string, c return members } -// TestSelectionCleanupStagingReapsLeaseLapsedWriterMidDigest reproduces, in -// production order, the un-renewed window that lifecycle/selection.go:393-419 -// creates between the last AppendSelectionMembers and CompleteSelection, then -// lets a second session call BeginSelection (which unconditionally runs -// CleanupSelectionStaging at selections.go:41). func TestSelectionCleanupStagingReapsLeaseLapsedWriterMidDigest(t *testing.T) { ctx, client, persistence := selectionCleanupOpen(t, 300*time.Second) const project = "loom_exv_cleanup_mid_digest" - // Case 0: harness proof. A genuinely crashed creator (STAGING header, lease - // lapsed) must be reaped, or the H3-a result below would be vacuous. crashedID := "selection-cleanup-crashed-" + uuid.NewString() crashedToken := uuid.NewString() if _, err := persistence.BeginSelection(ctx, selectionCleanupHeader(project, crashedID, uuid.NewString(), time.Now().UTC()), crashedToken); err != nil { @@ -224,8 +194,6 @@ func TestSelectionCleanupStagingReapsLeaseLapsedWriterMidDigest(t *testing.T) { } selectionCleanupAssertNoOrphans(t, ctx, client, "crashed-reap") - // Case 1: control. A writer inside the lease that is mid-digest must survive - // another session's BeginSelection and must still be able to complete. liveID := "selection-cleanup-live-" + uuid.NewString() liveToken := uuid.NewString() liveMembers := selectionCleanupMembers(project, "generation-a", "DocumentReference", "live", 3) @@ -235,8 +203,6 @@ func TestSelectionCleanupStagingReapsLeaseLapsedWriterMidDigest(t *testing.T) { if inserted, err := persistence.AppendSelectionMembers(ctx, liveID, liveToken, liveMembers); err != nil || len(inserted) != 3 { t.Fatalf("live writer append inserted=%d err=%v", len(inserted), err) } - // The un-renewed window: digest runs, no lease refresh happens, then a second - // session calls BeginSelection before this writer completes. liveDigest, liveCount, _, err := persistence.DigestSelectionMembers(ctx, project, liveID) if err != nil || liveCount != 3 { t.Fatalf("live writer digest count=%d err=%v", liveCount, err) @@ -257,8 +223,6 @@ func TestSelectionCleanupStagingReapsLeaseLapsedWriterMidDigest(t *testing.T) { t.Logf("SELECTION_CLEANUP_LIVE_WRITER_COMPLETED digest=%s memberCount=%d", liveCompleted.MembershipDigest, liveCompleted.MemberCount) selectionCleanupAssertNoOrphans(t, ctx, client, "live-writer") - // Case 2: the hypothesis. Identical to case 1, except the lease lapsed while - // the writer sat in the un-renewed digest phase with no timeout violation. idleID := "selection-cleanup-idle-" + uuid.NewString() idleToken := uuid.NewString() if _, err := persistence.BeginSelection(ctx, selectionCleanupHeader(project, idleID, uuid.NewString(), time.Now().UTC()), idleToken); err != nil { @@ -302,16 +266,10 @@ func TestSelectionCleanupStagingReapsLeaseLapsedWriterMidDigest(t *testing.T) { selectionCleanupAssertNoOrphans(t, ctx, client, "tail-begin") } -// TestSelectionCleanupStagingDestroysCompletedStagingHeader probes the missing -// complete != true guard at selections.go:388-400 against the sibling guards in -// AbortSelection (selections.go:290) and CompleteSelection (selections.go:242), -// and separately measures whether production code can reach that state at all. func TestSelectionCleanupStagingDestroysCompletedStagingHeader(t *testing.T) { ctx, client, persistence := selectionCleanupOpen(t, 240*time.Second) const project = "loom_exv_cleanup_completed" - // Reachability probe A: BeginSelection rejects a complete revision, so the - // immutable staging insert cannot carry complete=true. rejected := selectionCleanupHeader(project, "selection-cleanup-reject-"+uuid.NewString(), uuid.NewString(), time.Now().UTC()) rejected.Complete = true rejected.MembershipDigest = strings.Repeat("a", 64) @@ -324,8 +282,6 @@ func TestSelectionCleanupStagingDestroysCompletedStagingHeader(t *testing.T) { t.Errorf("BeginSelection accepted a complete revision; state=STAGING with complete=true is directly reachable") } - // Reachability probe B: run the whole production lifecycle and confirm no - // document ever lands in {state:"STAGING", complete:true}. fullID := "selection-cleanup-full-" + uuid.NewString() fullToken := uuid.NewString() fullMembers := selectionCleanupMembers(project, "generation-a", "DocumentReference", "full", 4) @@ -369,9 +325,6 @@ func TestSelectionCleanupStagingDestroysCompletedStagingHeader(t *testing.T) { t.Errorf("completed selection destroyed by cleanup: %#v", survivor) } - // Guard comparison: a document forced into {STAGING, complete:true} by raw - // AQL (no production path does this) is refused by Abort and Complete but not - // by Cleanup. const guardedID = "selection-cleanup-guarded" guardedToken := "guarded-writer-token" completedAt := time.Now().UTC().Add(-3 * time.Hour) @@ -445,10 +398,6 @@ func TestSelectionCleanupStagingDestroysCompletedStagingHeader(t *testing.T) { selectionCleanupAssertNoOrphans(t, ctx, client, "guarded-cleanup") } -// TestSelectionCleanupStagingLeaseGapAtScale quantifies the un-renewed gap that -// DigestSelectionMembers (selections.go:181-227) can create at the production -// row bound, so the 10m lease claim is judged against measured work rather than -// an assumed constant. func TestSelectionCleanupStagingLeaseGapAtScale(t *testing.T) { ctx, _, persistence := selectionCleanupOpen(t, 25*time.Minute) size := int(lifecycle.DefaultSelectionMaxRows) @@ -489,8 +438,6 @@ func selectionCleanupScaleRun(t *testing.T, ctx context.Context, persistence *St batches++ } appendElapsed := time.Since(appendStart) - // The lease clock last ticked inside the final AppendSelectionMembers - // (selections.go:168). Everything recorded after this line is un-renewed work. unrenewedStart := time.Now() digest, count, bytes, err := persistence.DigestSelectionMembers(ctx, project, selectionID) if err != nil { diff --git a/internal/explorer/arango/store.go b/internal/backends/arango/internal/explorer/store.go similarity index 92% rename from internal/explorer/arango/store.go rename to internal/backends/arango/internal/explorer/store.go index 32a56de37..ef414d240 100644 --- a/internal/explorer/arango/store.go +++ b/internal/backends/arango/internal/explorer/store.go @@ -6,8 +6,8 @@ import ( "encoding/hex" "fmt" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" - store "github.com/calypr/loom/internal/store/arango" ) type client interface { diff --git a/internal/explorer/arango/store_test.go b/internal/backends/arango/internal/explorer/store_test.go similarity index 99% rename from internal/explorer/arango/store_test.go rename to internal/backends/arango/internal/explorer/store_test.go index 526470ed3..c55968789 100644 --- a/internal/explorer/arango/store_test.go +++ b/internal/backends/arango/internal/explorer/store_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" + storepkg "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer" - storepkg "github.com/calypr/loom/internal/store/arango" ) func TestDecodeNormalizesNumericUpdatedAt(t *testing.T) { diff --git a/internal/explorer/arango/table_shape_capabilities.go b/internal/backends/arango/internal/explorer/table_shape_capabilities.go similarity index 99% rename from internal/explorer/arango/table_shape_capabilities.go rename to internal/backends/arango/internal/explorer/table_shape_capabilities.go index d5f8d1608..ce12f748d 100644 --- a/internal/explorer/arango/table_shape_capabilities.go +++ b/internal/backends/arango/internal/explorer/table_shape_capabilities.go @@ -5,8 +5,8 @@ import ( "encoding/json" "fmt" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer/tableshapecap" - store "github.com/calypr/loom/internal/store/arango" ) const TableShapeCapabilitiesCollection = "loom_explorer_table_shape_capabilities" @@ -63,8 +63,6 @@ func (r *TableShapeCapabilityRepository) PutCatalog(ctx context.Context, receipt if stored != nil { return *stored, nil } - // Insert-ignore can return no row for an already present short key. Resolve - // only through the caller's full binding; an out-of-scope key is a collision. got, getErr := r.GetCatalog(ctx, receipt.Binding, receipt.ID) if getErr != nil { return tableshapecap.CatalogReceipt{}, fmt.Errorf("%w: %s", tableshapecap.ErrIdentityClash, receipt.ID) diff --git a/internal/explorer/arango/table_shape_capabilities_test.go b/internal/backends/arango/internal/explorer/table_shape_capabilities_test.go similarity index 99% rename from internal/explorer/arango/table_shape_capabilities_test.go rename to internal/backends/arango/internal/explorer/table_shape_capabilities_test.go index a1a315489..c1c1bacdb 100644 --- a/internal/explorer/arango/table_shape_capabilities_test.go +++ b/internal/backends/arango/internal/explorer/table_shape_capabilities_test.go @@ -6,8 +6,8 @@ import ( "strings" "testing" + storepkg "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/explorer/tableshapecap" - storepkg "github.com/calypr/loom/internal/store/arango" ) func tableShapeTestCatalog(t *testing.T, binding tableshapecap.Binding) tableshapecap.CatalogReceipt { @@ -182,8 +182,6 @@ func TestTableShapeCapabilityRepositoryIsCreateOnceAndTenantScoped(t *testing.T) other := tableShapeTestCatalog(t, binding) other.Columns[0].Label = "Different label" - // Simulate an existing short-key collision after insert-ignore by storing a - // different valid receipt under the incoming outer key. other, err = tableshapecap.NewCatalogReceipt(binding, other.Columns, other.Availability, other.Choices, tableshapecap.SavedShapeSummary{}, other.CreatedAt) if err != nil { t.Fatal(err) diff --git a/internal/dataframe/publication/arango/bundle_registry.go b/internal/backends/arango/internal/publication/bundle_registry.go similarity index 96% rename from internal/dataframe/publication/arango/bundle_registry.go rename to internal/backends/arango/internal/publication/bundle_registry.go index ca59ce11c..f0e22d5e0 100644 --- a/internal/dataframe/publication/arango/bundle_registry.go +++ b/internal/backends/arango/internal/publication/bundle_registry.go @@ -10,8 +10,8 @@ import ( "time" "github.com/arangodb/go-driver/v2/arangodb/shared" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/dataframe/publication" - arangostore "github.com/calypr/loom/internal/store/arango" ) const ( @@ -238,9 +238,6 @@ func (r *Registry) GetPointer(ctx context.Context, name string) (publication.Bun return *found, nil } -// CompareAndSwapPointer performs the visibility update in Arango. The query -// returns a small object so the generic QueryRows client can inspect whether -// the expected version won the race. func (r *Registry) CompareAndSwapPointer(ctx context.Context, name, expected, next string) error { var result struct { Updated bool `json:"updated"` @@ -269,8 +266,6 @@ RETURN {updated: true}`, r.batchSize, map[string]interface{}{"@collection": Bund return nil } -// PublishExecution atomically commits both the visibility pointer and the -// PUBLISHED metadata. A failed compare-and-swap leaves both documents intact. func (r *Registry) PublishExecution(ctx context.Context, name, expected string, execution publication.BundleExecution) error { data, err := json.Marshal(execution) if err != nil { @@ -280,11 +275,6 @@ func (r *Registry) PublishExecution(ctx context.Context, name, expected string, if err := json.Unmarshal(data, &executionDoc); err != nil { return err } - // The state UPDATE and the pointer UPSERT execute inside ONE Arango - // transaction so no observer (the reconcile sweep above all) can ever see - // a PUBLISHED execution whose visibility pointer does not yet name it. - // A failed compare-and-swap -- or any mid-request failure -- rolls the - // whole thing back: F-B12/F-B16 (two-answers class) root repair. txClient, ok := r.client.(arangostore.TransactionalRowClient) if !ok { return fmt.Errorf("bundle publish requires a transactional Arango client") diff --git a/internal/dataframe/publication/arango/bundle_registry_test.go b/internal/backends/arango/internal/publication/bundle_registry_test.go similarity index 99% rename from internal/dataframe/publication/arango/bundle_registry_test.go rename to internal/backends/arango/internal/publication/bundle_registry_test.go index 02bbd5ca5..ca3b61750 100644 --- a/internal/dataframe/publication/arango/bundle_registry_test.go +++ b/internal/backends/arango/internal/publication/bundle_registry_test.go @@ -11,8 +11,8 @@ import ( "time" "github.com/arangodb/go-driver/v2/arangodb/shared" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/dataframe/publication" - arangostore "github.com/calypr/loom/internal/store/arango" ) type publishCaptureClient struct { diff --git a/internal/backends/arango/internal/publication/citation_gate_test.go b/internal/backends/arango/internal/publication/citation_gate_test.go new file mode 100644 index 000000000..bb03eb016 --- /dev/null +++ b/internal/backends/arango/internal/publication/citation_gate_test.go @@ -0,0 +1,88 @@ +package arango + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestCitationLineMismatchAcceptsExactCitedLine(t *testing.T) { + lines := []string{"package main", "", "wantToken := 1", "}"} + if mismatch := citationLineMismatch(lines, 3, "wantToken := 1"); mismatch != "" { + t.Fatalf("exact cited line rejected: %s", mismatch) + } +} + +func TestCitationLineMismatchTrimsSurroundingWhitespace(t *testing.T) { + lines := []string{"\t\twantToken := 1\t "} + if mismatch := citationLineMismatch(lines, 1, " wantToken := 1 "); mismatch != "" { + t.Fatalf("whitespace-equivalent cited line rejected: %s", mismatch) + } +} + +func TestCitationLineMismatchFailsOnWrongLine(t *testing.T) { + lines := []string{"package main", "wantToken := 1", "", "", "other := 2"} + if mismatch := citationLineMismatch(lines, 5, "wantToken := 1"); mismatch == "" { + t.Fatal("want string present at line 2 but cited at line 5 must fail: weak-gate forgery survived") + } +} + +func TestCitationLineMismatchFailsOnNearMissContent(t *testing.T) { + lines := []string{"wantToken := 1"} + if mismatch := citationLineMismatch(lines, 1, "wantToken := 11"); mismatch == "" { + t.Fatal("near-miss content at the cited line must fail") + } +} + +func TestCitationLineMismatchFailsPastEOF(t *testing.T) { + lines := []string{"only", "two"} + for _, line := range []int{3, 4, 1000} { + if mismatch := citationLineMismatch(lines, line, "two"); mismatch == "" { + t.Fatalf("cited :%d past EOF must fail", line) + } + } +} + +func TestCitationLineMismatchFailsNonPositiveLine(t *testing.T) { + lines := []string{"only"} + for _, line := range []int{0, -1, -100} { + if mismatch := citationLineMismatch(lines, line, "only"); mismatch == "" { + t.Fatalf("cited :%d must fail as non-positive", line) + } + } +} + +func TestDanglingCitationTableStrictAtCitedLines(t *testing.T) { + root := danglingRepoRoot(t) + checkCitationTableStrict(t, root, len(danglingCitations), func(i int) (string, int, string) { + return danglingCitations[i].file, danglingCitations[i].line, danglingCitations[i].want + }) +} + +func TestRrdCitationTableStrictAtCitedLines(t *testing.T) { + root := danglingRepoRoot(t) + checkCitationTableStrict(t, root, len(rrdCitations), func(i int) (string, int, string) { + return rrdCitations[i].file, rrdCitations[i].line, rrdCitations[i].want + }) +} + +func checkCitationTableStrict(t *testing.T, root string, n int, at func(int) (string, int, string)) { + t.Helper() + contents := map[string][]string{} + for i := 0; i < n; i++ { + file, line, want := at(i) + lines, ok := contents[file] + if !ok { + raw, err := os.ReadFile(filepath.Join(root, file)) + if err != nil { + t.Fatalf("citation source %s unreadable: %v", file, err) + } + lines = strings.Split(string(raw), "\n") + contents[file] = lines + } + if mismatch := citationLineMismatch(lines, line, want); mismatch != "" { + t.Fatalf("citation drift: %s %s (want %q)", file, mismatch, want) + } + } +} diff --git a/internal/dataframe/publication/arango/dangling_binding_e2e_test.go b/internal/backends/arango/internal/publication/dangling_binding_e2e_test.go similarity index 65% rename from internal/dataframe/publication/arango/dangling_binding_e2e_test.go rename to internal/backends/arango/internal/publication/dangling_binding_e2e_test.go index c6bee495a..da9b6fb69 100644 --- a/internal/dataframe/publication/arango/dangling_binding_e2e_test.go +++ b/internal/backends/arango/internal/publication/dangling_binding_e2e_test.go @@ -14,102 +14,41 @@ import ( "testing" "time" + datasetstore "github.com/calypr/loom/internal/backends/arango/internal/dataset" + store "github.com/calypr/loom/internal/backends/arango/internal/store" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" "github.com/calypr/loom/internal/dataframe/publication" "github.com/calypr/loom/internal/dataframe/published" "github.com/calypr/loom/internal/dataset" - datasetstore "github.com/calypr/loom/internal/dataset/arango" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) -// These opt-in tests adjudicate candidate F-B7 against a real ArangoDB in a -// fully owned database (loom_verify_db_, dropped in t.Cleanup): -// -// F-B7 contract: a broken optional release binding must not hide a healthy -// current output from dataset discovery. Required bindings remain fail-closed. -// -// Mechanism as claimed by the brief: "internal/dataframe/published/current.go:193-257: -// Resolve loads each binding via loadPublishedExecutionForRead, and -// execution == nil || !publishedOutputMatches(execution, binding) -> all = false -// -> ALL bindings recorded stale (:202-211); caller then falls back to stale -// snapshot at internal/dataset/release.go:204-213. Also verifyPublishedExecution -// (arango:188-196) rechecks pointer naming." -// -// LINE-EXACT SOURCE AUDIT (verified 2026-11-01 on branch feature/deep-clean): -// none of those identifiers exist anywhere in the repository: -// -// loadPublishedExecutionForRead -> 0 matches (grep repo-wide) -// publishedOutputMatches -> 0 matches -// "all = false" in published -> 0 matches -// verifyPublishedExecution -> 0 matches (entire arango package) -// requiredReleaseBindings -> 0 matches (internal/dataset/release.go ends at :280) -// releaseBindingsFrom -> 0 matches -// -// The real read path has no Resolve and no stale re-marking at read time. It -// skips only typed deterministic failures for non-required active bindings; -// required bindings and unclassified backend failures remain fail-closed. -// -// internal/dataframe/published/current.go:190-222 activeReleaseDatasets -// :203 `if binding.Stale || binding.Generation != active.Release.Generation {` -// :204 continue // pre-marked-stale bindings skipped -// :211-216 value, err := r.releaseMaterialization(...); unavailable optional bindings continue -// // required or unknown failures still abort the list -// :220-222 if len(result) == 0 { return DatasetNotFound } -// internal/dataframe/published/current.go:74-92 activeReleaseDataset (single selector) -// :86-87 skips non-matching/stale bindings BEFORE materialization -> a binding -// that cannot resolve cannot affect a different selector's answer -// internal/dataframe/published/current.go:94-111 releaseMaterialization -// :98-101 missing execution -> typed unavailable-binding error -// :104-105 identity mismatch -> typed unavailable-binding error -// :107-115 no queryable output -> typed unavailable-binding error -// -// Stale is only ever written at release CREATE (internal/dataset/release.go): -// :185-188 required selector verification failure -> recorded into `failed` -// :192-193 `return ProjectRelease{}, &ReleaseRequirementsError{...}` (create-time ERROR) -// :209-210 carried-forward prior bindings: prior.Required = false; prior.Stale = prior.Generation != ref.Generation -// :254-256 publicationBinding: Stale = result.Generation != generation (false for fresh) -// -// The active snapshot at internal/dataset/arango/release_store.go:144-150 -// (readActiveReleaseAQL) has no stale fallback either: it reads exactly the -// pointer's releaseKey document or yields ErrNoActiveRelease (release_store.go:56-57). -// -// So the claimed whole-dataset STALE demotion cannot be confirmed as stated. -// What these tests measure is whether the real code still produces the -// product-level harm F-B7 fears (one bad binding degrading a healthy binding's -// fresh answer) through the mechanism the code actually uses. - const danglingBindingDatabasePrefix = "loom_verify_db_" -// danglingCitation pins the load-bearing source lines tested above. If the -// source drifts, the citation check fails the test so stale evidence cannot -// quietly survive a refactor. type danglingCitation struct { file string - line int // 1-based + line int want string } var danglingCitations = []danglingCitation{ - {"internal/dataframe/published/current.go", 88, "if binding.Selector != selector || binding.Stale || binding.Generation != active.Release.Generation {"}, - {"internal/dataframe/published/current.go", 97, "execution, err := r.Catalog.GetExecution(ctx, binding.ExecutionID)"}, - {"internal/dataframe/published/current.go", 99, "if errors.Is(err, bundlepublication.ErrBundleNotFound) {"}, - {"internal/dataframe/published/current.go", 102, "return Materialization{}, fmt.Errorf(\"resolve active dataframe execution: %w\", err)"}, - {"internal/dataframe/published/current.go", 115, "return Materialization{}, fmt.Errorf(\"%w: active dataframe execution %q has no queryable output %q\", errReleaseBindingUnavailable, binding.ExecutionID, binding.Selector.Output)"}, - {"internal/dataframe/published/current.go", 208, "if binding.Stale || binding.Generation != active.Release.Generation {"}, - {"internal/dataframe/published/current.go", 211, "value, err := r.releaseMaterialization(ctx, active.Release, binding)"}, - {"internal/dataframe/published/current.go", 212, "if err != nil {"}, - {"internal/dataframe/published/current.go", 213, "if !binding.Required && errors.Is(err, errReleaseBindingUnavailable) {"}, - {"internal/dataframe/published/current.go", 216, "return nil, err"}, - {"internal/dataset/release.go", 193, "return ProjectRelease{}, &ReleaseRequirementsError{Verifications: failed}"}, - {"internal/dataset/release.go", 210, "prior.Stale = prior.Generation != ref.Generation"}, - {"internal/dataset/release.go", 255, "return ReleasePublication{Selector: result.Selector, ExecutionID: result.ExecutionID, Generation: result.Generation, Required: required, Stale: result.Generation != generation, VerifiedAt: result.VerifiedAt}"}, - {"internal/api/graphql/graph/dataframe/rows.go", 27, "values, err := s.reader.CurrentProjectDatasets(ctx, legacyProjectID(projectID))"}, - {"internal/api/graphql/graph/dataframe/rows.go", 29, "return nil, mapReaderError(err)"}, + {"internal/dataframe/published/current.go", 85, "if binding.Selector != selector || binding.Stale || binding.Generation != active.Release.Generation {"}, + {"internal/dataframe/published/current.go", 94, "execution, err := r.Catalog.GetExecution(ctx, binding.ExecutionID)"}, + {"internal/dataframe/published/current.go", 96, "if errors.Is(err, bundlepublication.ErrBundleNotFound) {"}, + {"internal/dataframe/published/current.go", 99, "return Materialization{}, fmt.Errorf(\"resolve active dataframe execution: %w\", err)"}, + {"internal/dataframe/published/current.go", 112, "return Materialization{}, fmt.Errorf(\"%w: active dataframe execution %q has no queryable output %q\", errReleaseBindingUnavailable, binding.ExecutionID, binding.Selector.Output)"}, + {"internal/dataframe/published/current.go", 203, "if binding.Stale || binding.Generation != active.Release.Generation {"}, + {"internal/dataframe/published/current.go", 206, "value, err := r.releaseMaterialization(ctx, active.Release, binding)"}, + {"internal/dataframe/published/current.go", 207, "if err != nil {"}, + {"internal/dataframe/published/current.go", 208, "if !binding.Required && errors.Is(err, errReleaseBindingUnavailable) {"}, + {"internal/dataframe/published/current.go", 211, "return nil, err"}, + {"internal/dataset/release.go", 185, "return ProjectRelease{}, &ReleaseRequirementsError{Verifications: failed}"}, + {"internal/dataset/release.go", 202, "prior.Stale = prior.Generation != ref.Generation"}, + {"internal/dataset/release.go", 247, "return ReleasePublication{Selector: result.Selector, ExecutionID: result.ExecutionID, Generation: result.Generation, Required: required, Stale: result.Generation != generation, VerifiedAt: result.VerifiedAt}"}, + {"internal/api/graphql/graph/dataframe/rows.go", 25, "values, err := s.reader.CurrentProjectDatasets(ctx, legacyProjectID(projectID))"}, + {"internal/api/graphql/graph/dataframe/rows.go", 27, "return nil, mapReaderError(err)"}, } -// danglingAbsentIdentifiers are the identifiers the brief claims exist; the -// files named in the hypothesis must NOT contain them. type danglingAbsentIdentifier struct { file string identifier string @@ -119,7 +58,7 @@ var danglingAbsentIdentifiers = []danglingAbsentIdentifier{ {"internal/dataframe/published/current.go", "loadPublishedExecutionForRead"}, {"internal/dataframe/published/current.go", "publishedOutputMatches"}, {"internal/dataframe/published/current.go", "all = false"}, - {"internal/dataframe/publication/arango/bundle_registry.go", "verifyPublishedExecution"}, + {"internal/backends/arango/internal/publication/bundle_registry.go", "verifyPublishedExecution"}, {"internal/dataset/release.go", "requiredReleaseBindings"}, {"internal/dataset/release.go", "releaseBindingsFrom"}, } @@ -142,6 +81,16 @@ func danglingRepoRoot(t *testing.T) string { } } +func citationLineMismatch(lines []string, line int, want string) string { + if line < 1 || line > len(lines) { + return fmt.Sprintf("cited :%d is past EOF (file has %d lines)", line, len(lines)) + } + if strings.TrimSpace(lines[line-1]) != strings.TrimSpace(want) { + return fmt.Sprintf("cited :%d holds %q, want %q", line, lines[line-1], want) + } + return "" +} + func danglingCheckCitations(t *testing.T) { t.Helper() root := danglingRepoRoot(t) @@ -156,19 +105,8 @@ func danglingCheckCitations(t *testing.T) { lines = strings.Split(string(raw), "\n") contents[c.file] = lines } - if c.line < 1 || c.line > len(lines) { - t.Fatalf("citation drift: %s has %d lines, cited :%d == %q", c.file, len(lines), c.line, c.want) - } - // Drift-proof: the contract is the EXACT source text, not the pin. - found := false - for _, line := range lines { - if strings.TrimSpace(line) == strings.TrimSpace(c.want) { - found = true - break - } - } - if !found { - t.Fatalf("citation drift: %s no longer contains %q (cited :%d)", c.file, c.want, c.line) + if mismatch := citationLineMismatch(lines, c.line, c.want); mismatch != "" { + t.Fatalf("citation drift: %s %s (want %q)", c.file, mismatch, c.want) } } for _, a := range danglingAbsentIdentifiers { @@ -182,12 +120,6 @@ func danglingCheckCitations(t *testing.T) { } } -// danglingDatasetDocumentKey replicates -// internal/dataset/arango/document_codec.go:164-172 (documentKey) with the -// domain from manifest_lifecycle.go:14 so this package can address the -// active-release pointer directly. Fidelity is not taken on faith: every use -// is round-tripped through the PRODUCTION datasetstore.Store.ReadActiveRelease, -// so a mismatched key surfaces as ErrNoActiveRelease and fails the test. func danglingDatasetDocumentKey(kind string, values ...string) string { hash := sha256.New() _, _ = hash.Write([]byte("loom.datasetstore.v1")) @@ -198,12 +130,6 @@ func danglingDatasetDocumentKey(kind string, values ...string) string { return kind + "_" + hex.EncodeToString(hash.Sum(nil)) } -// danglingOpen mirrors the owned-database env-gated harness of -// name_latest_binding_e2e_test.go:103-150 (skip when LOOM_TEST_ARANGO_URL is -// unset, store.Open creates loom_verify_db_ - see internal/store/arango/ -// client.go:130-142 - and HTTP DELETE /_api/database drops it in t.Cleanup). -// It additionally bootstraps the dataset lifecycle collection so the real -// datasetstore.Store and published.Reader run against the same database. func danglingOpen(t *testing.T, budget time.Duration) (context.Context, *store.Client, *Registry, *datasetstore.Store) { t.Helper() endpoint := os.Getenv("LOOM_TEST_ARANGO_URL") @@ -267,11 +193,6 @@ type danglingExecSpec struct { output string } -// danglingSeedExecution writes one PUBLISHED execution through the production -// lease + SaveExecution path (bundle_registry.go:26-66), as the sibling -// F-B6 harness does; outputs carry VerifiedAt/PhysicalTable so -// BundleOutputRecord.Queryable() (bundle.go:147-149) is true without relying -// on Canonicalize Legacy backfill. func danglingSeedExecution(t *testing.T, ctx context.Context, registry *Registry, spec danglingExecSpec) publication.BundleExecution { t.Helper() key := "dbind-key-" + spec.id @@ -310,13 +231,6 @@ func danglingSeedExecution(t *testing.T, ctx context.Context, registry *Registry return execution } -// danglingSeedRelease saves an immutable release through the PRODUCTION -// datasetstore.Store.SaveRelease (release_store.go:12-39) and then activates -// it by patching the active_release placeholder that SaveRelease inserted -// (release_store.go:138, manifest_lifecycle.go:85-87). Production -// ReadActiveRelease (release_store.go:41-60, readActiveReleaseAQL:144-150) is -// the gate: it must return the released bindings, which also validates the -// replicated document-key derivation. func danglingSeedRelease(t *testing.T, ctx context.Context, client *store.Client, lifecycle *datasetstore.Store, project, generation string, bindings []dataset.ReleasePublication) dataset.ProjectRelease { t.Helper() release := dataset.ProjectRelease{ @@ -353,10 +267,6 @@ RETURN {activated: true}`, 32, map[string]interface{}{ return saved } -// danglingPatchBindingStale directly rewrites the bindings array of the saved -// release document (binding-level "recorded stale" - the state the hypothesised -// all=false path would have produced). Direct doc write is authorized by the -// brief; nothing on the read side recomputes this flag. func danglingPatchBindingStale(t *testing.T, ctx context.Context, client *store.Client, release dataset.ProjectRelease) { t.Helper() releaseKey := danglingDatasetDocumentKey("release", release.ID) @@ -376,10 +286,6 @@ RETURN {patched: true}`, 32, map[string]interface{}{ } } -// danglingCorruptDropOutputs removes the PublishedOutput row from the -// execution document. PublishedOutput is a row of the execution document's -// embedded outputs array (bundle.go:168; BootstrapSpec registers no separate -// output collection), so deleting the row is UPDATE outputs=[]. func danglingCorruptDropOutputs(t *testing.T, ctx context.Context, client *store.Client, executionID string) { t.Helper() removed := false @@ -395,8 +301,6 @@ RETURN {removed: true}`, 32, map[string]interface{}{"@collection": BundleExecuti } } -// danglingCorruptDeleteExecution makes the binding dangling: the execution -// document no longer exists while the release still binds its ID. func danglingCorruptDeleteExecution(t *testing.T, ctx context.Context, client *store.Client, executionID string) { t.Helper() removed := false @@ -442,30 +346,6 @@ func danglingSelector(output string, recipe string) published.DataframeSelector return published.DataframeSelector{Recipe: recipe, TranslationVersion: "v1", Output: output} } -// TestBrokenOptionalBindingDoesNotHideHealthyDatasetAgainstArango is the F-B7 regression. -// -// Pre-stated independent expectations (derived from the line-exact audit -// above, declared BEFORE running): -// -// E1 Fresh two-binding release: CurrentProjectDatasets returns exactly 2 -// materializations sorted by Name ("out-a" then "out-b"), State READY, -// ID ":" (types.go:58-80), err == nil. -// E2 Un-naming the bad execution's visibility pointer changes nothing on -// either read path, because releaseMaterialization (current.go:94-111) -// never calls GetPointer. The claim "verifyPublishedExecution -// (arango:188-196) rechecks pointer naming" is false: no such function. -// E3 Deleting the bad optional binding's PublishedOutput row: the list keeps -// the healthy materialization and omits only the unavailable optional one. -// E4 Deleting the whole bad optional execution has the same result. Required -// bindings and transient catalog failures still abort the whole list. -// E5 The single-selector path activeReleaseDataset (current.go:85-90) skips -// the broken OTHER binding by selector before materializing, so the -// healthy selector keeps resolving fresh independently => per-viewer -// degradation DISPROVEN; the list and single-selector paths stay independent. -// -// Verdict rules: CONFIRMED iff one bad binding demonstrably degrades the -// healthy binding's freshness or forces stale fallback in the list path; -// DISPROVEN iff the healthy binding reports fresh independently. func TestBrokenOptionalBindingDoesNotHideHealthyDatasetAgainstArango(t *testing.T) { danglingCheckCitations(t) ctx, client, registry, lifecycle := danglingOpen(t, 120*time.Second) @@ -475,7 +355,6 @@ func TestBrokenOptionalBindingDoesNotHideHealthyDatasetAgainstArango(t *testing. healthyExec := danglingSeedExecution(t, ctx, registry, danglingExecSpec{id: "dbind-exec-healthy", project: project, generation: generation, recipe: "dbind-recipe-a", output: "out-a"}) badExec := danglingSeedExecution(t, ctx, registry, danglingExecSpec{id: "dbind-exec-bad", project: project, generation: generation, recipe: "dbind-recipe-b", output: "out-b"}) - // Pointer naming per the production visibility path (bundle.go:88-97). if err := registry.CompareAndSwapPointer(ctx, healthyExec.PointerName(), "", healthyExec.ID); err != nil { t.Fatalf("point healthy pointer: %v", err) } @@ -493,7 +372,6 @@ func TestBrokenOptionalBindingDoesNotHideHealthyDatasetAgainstArango(t *testing. healthySelector := danglingSelector("out-a", "dbind-recipe-a") badSelector := danglingSelector("out-b", "dbind-recipe-b") - // E1 t.Log("EXPECT E1: fresh list = 2 materializations [out-a out-b], State READY, err==nil") list, err := reader.CurrentProjectDatasets(ctx, project) if err != nil || len(list) != 2 || list[0].Name != "out-a" || list[1].Name != "out-b" { @@ -504,7 +382,6 @@ func TestBrokenOptionalBindingDoesNotHideHealthyDatasetAgainstArango(t *testing. } t.Logf("OBSERVED E1 fresh baseline: %s", danglingMaterializationDigest(list)) - // E2: un-name the bad pointer (pointer -> a non-existent execution). if err := registry.CompareAndSwapPointer(ctx, badExec.PointerName(), badExec.ID, "dbind-orphan-target"); err != nil { t.Fatalf("un-name bad pointer: %v", err) } @@ -518,7 +395,6 @@ func TestBrokenOptionalBindingDoesNotHideHealthyDatasetAgainstArango(t *testing. } t.Logf("OBSERVED E2 pointer-unnamed control: %s (pointer re-check claim has no code behind it)", danglingMaterializationDigest(list)) - // E3: delete the PublishedOutput row of the bad optional execution. danglingCorruptDropOutputs(t, ctx, client, badExec.ID) t.Log("EXPECT E3: healthy out-a remains in the list; only unavailable optional out-b is omitted") list, err = reader.CurrentProjectDatasets(ctx, project) @@ -528,7 +404,6 @@ func TestBrokenOptionalBindingDoesNotHideHealthyDatasetAgainstArango(t *testing. t.Errorf("F-B7 regression: broken optional out-b hid healthy out-a: list=%s err=%v", danglingMaterializationDigest(list), err) } - // E5a: despite E3, healthy single-selector read must be independent. t.Log("EXPECT E5: healthy single read unaffected by the OTHER binding's corruption") single, err := reader.CurrentProjectDataset(ctx, project, healthySelector) if err != nil || single.ID != healthyExec.ID+":out-a" || single.PhysicalTable != "ch_dbind_exec_healthy" { @@ -536,7 +411,6 @@ func TestBrokenOptionalBindingDoesNotHideHealthyDatasetAgainstArango(t *testing. } t.Logf("OBSERVED E5a: healthy single read while binding out-b corrupt: {Name=%q ID=%q PhysicalTable=%q} -> per-viewer demotion DISPROVEN", single.Name, single.ID, single.PhysicalTable) - // E4: now delete the bad execution document entirely (true dangling optional binding). danglingCorruptDeleteExecution(t, ctx, client, badExec.ID) t.Log("EXPECT E4: healthy out-a remains listed after dangling optional out-b is removed") list, err = reader.CurrentProjectDatasets(ctx, project) @@ -549,14 +423,12 @@ func TestBrokenOptionalBindingDoesNotHideHealthyDatasetAgainstArango(t *testing. t.Fatalf("E4 precondition: bad execution should be absent: err=%v", err) } - // E5b: bad single-selector read errors (it can never serve stale). if _, err := reader.CurrentProjectDataset(ctx, project, badSelector); err == nil { t.Fatalf("E5b violated: dangling binding resolver returned success") } else { t.Logf("OBSERVED E5b: bad selector read err=[%s] userCode=%q (loud error, never stale data)", danglingErrorChain(err), danglingUserErrorCode(err)) } - // E5c: healthy single selector STILL fine after full dangling state. if single, err := reader.CurrentProjectDataset(ctx, project, healthySelector); err != nil || single.ID != healthyExec.ID+":out-a" { t.Fatalf("E5c violated: healthy single read degraded by foreign dangling binding: %+v err=%v", single, err) } else { @@ -566,42 +438,6 @@ func TestBrokenOptionalBindingDoesNotHideHealthyDatasetAgainstArango(t *testing. t.Logf("F-B7 prevented: unavailable optional bindings were omitted while the healthy binding remained current; droppedOutputHealthy=%v danglingHealthy=%v", droppedOutputListKeepsHealthy, danglingListKeepsHealthy) } -// TestRequiredBindingSilentDemotion adjudicates the required-set variant: -// "requiredReleaseBindings/releaseBindingsFrom (release.go:349-390): if a -// binding with Required=true gets stale, does the caller error or silently -// serve stale?" Those functions do not exist (release.go ends at :280). The -// real required verification happens at create time: release.go:181-193 makes -// an unverifiable selector a ReleaseRequirementsError, and carried-forward -// prior bindings lose Required at :209. At read time stale/generation filters -// run first; Required only preserves fail-closed behavior for a deterministic -// unavailable active binding instead of omitting it as optional. -// -// Pre-stated independent expectations (declared BEFORE running): -// -// R1 required=true binding carries Stale=true in the release doc: -// CurrentProjectDatasets returns ONLY the non-stale sibling, err==nil: -// the required binding is SILENTLY dropped (no error, no -// ErrReleaseRequirementsUnmet) - current.go:203-204 + 212 (result -// non-empty). Caller behavior = silent skip, NOT error. -// R2 Single read of the required-stale selector: UserError DATASET_NOT_FOUND -// (current.go:86-91 skip + :91), NOT a required-verification error, and -// no stale data is served. -// R3 ALL bindings stale: CurrentProjectDatasets -> UserError -// DATASET_NOT_FOUND (current.go:212-214). Silent empty, still no -// requirements error at read time. -// R4 Control: required=true + Stale=false but execution corrupted -> -// CurrentProjectDatasets still errors, unlike the broken optional binding. -// R5 Flipping Required true/false on a pre-marked stale binding remains -// behaviorally inert because the stale filter runs before materialization. -// -// Verdict: "silently serve stale" is DISPROVEN if the reader never returns a -// stale/generation-mismatched value and instead skips it; "caller errors on -// required-stale" is DISPROVEN iff R1/R3 show nil error. Both are read-side -// correctness properties, so the test PASSES when the reader behaves as the -// audit predicts; the verbatim caller behavior is logged as the report's -// evidence. Behavior is PASS (correct): requirement enforcement lives at -// release.go:181-193 (create), and the reader refuses to serve stale or -// foreign-generation data by construction (current.go:99-101). func TestRequiredBindingSilentDemotion(t *testing.T) { danglingCheckCitations(t) ctx, client, registry, lifecycle := danglingOpen(t, 120*time.Second) @@ -626,24 +462,18 @@ func TestRequiredBindingSilentDemotion(t *testing.T) { t.Fatalf("R0 baseline broken: list=%s err=%v", danglingMaterializationDigest(list), err) } - // R4 control first, while Stale=false: corrupt the REQUIRED binding's - // execution and confirm the caller ERRORS loudly (not silent stale). danglingCorruptDropOutputs(t, ctx, client, requiredExec.ID) t.Log("EXPECT R4: required=true + Stale=false + corrupted execution -> CurrentProjectDatasets returns error (not silent stale)") var r4ErrChain string list, err = reader.CurrentProjectDatasets(ctx, project) r4Loud := err != nil && len(list) == 0 && strings.Contains(danglingErrorChain(err), "has no queryable output") r4ErrChain = danglingErrorChain(err) - // Repair the execution for the stale-flag cases by deleting it: the - // stale=true path (R1-R3) never touches the catalog for the stale - // binding, so deletion keeps the two mechanisms cleanly separated. danglingCorruptDeleteExecution(t, ctx, client, requiredExec.ID) if !r4Loud { t.Fatalf("R4 violated: corrupted required binding did not force a loud whole-list error: list=%s err=[%s]", danglingMaterializationDigest(list), r4ErrChain) } t.Logf("OBSERVED R4 verbatim: corrupted required binding -> caller behavior = ERROR, chain=[%s] userCode=%q", r4ErrChain, danglingUserErrorCode(err)) - // R1: mark the required binding stale in the release document. staleBindings := []dataset.ReleasePublication{ {Selector: requiredSelector, ExecutionID: requiredExec.ID, Generation: generation, Required: true, Stale: true, VerifiedAt: time.Now().UTC()}, {Selector: optionalSelector, ExecutionID: optionalExec.ID, Generation: generation, Required: false, Stale: false, VerifiedAt: time.Now().UTC()}, @@ -658,7 +488,6 @@ func TestRequiredBindingSilentDemotion(t *testing.T) { t.Fatalf("R1 violated: expected silently-skipped required-stale binding (nil error, [out-o]); got list=%s err=%v", danglingMaterializationDigest(list), err) } - // R5: Required does not override the pre-marked stale filter. inertBindings := make([]dataset.ReleasePublication, len(staleBindings)) copy(inertBindings, staleBindings) inertBindings[0].Required = false @@ -669,7 +498,6 @@ func TestRequiredBindingSilentDemotion(t *testing.T) { } t.Log("OBSERVED R5: Required=true->false on the stale binding changed nothing; stale bindings are filtered before the Required error policy") - // R2: single read of the required-stale selector. t.Log("EXPECT R2: single required-stale selector -> UserError code DATASET_NOT_FOUND, NOT ErrReleaseRequirementsUnmet; no stale value served") single, err := reader.CurrentProjectDataset(ctx, project, requiredSelector) code := danglingUserErrorCode(err) @@ -678,7 +506,6 @@ func TestRequiredBindingSilentDemotion(t *testing.T) { } t.Logf("OBSERVED R2 verbatim: required-stale single read -> err=[%s] userCode=%q -> silently serves NOT-FOUND (never stale data)", danglingErrorChain(err), code) - // R3: all bindings stale. allStale := make([]dataset.ReleasePublication, len(staleBindings)) copy(allStale, staleBindings) allStale[1].Stale = true diff --git a/internal/dataframe/publication/arango/execution_read_pin_integration_test.go b/internal/backends/arango/internal/publication/execution_read_pin_integration_test.go similarity index 94% rename from internal/dataframe/publication/arango/execution_read_pin_integration_test.go rename to internal/backends/arango/internal/publication/execution_read_pin_integration_test.go index 1d8197c68..0c7e1fe1e 100644 --- a/internal/dataframe/publication/arango/execution_read_pin_integration_test.go +++ b/internal/backends/arango/internal/publication/execution_read_pin_integration_test.go @@ -8,12 +8,10 @@ import ( "time" "github.com/arangodb/go-driver/v2/arangodb/shared" - store "github.com/calypr/loom/internal/store/arango" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/google/uuid" ) -// This test is opt-in because it needs a real Arango instance. It exercises -// the production AQL pin state document rather than a query-string fake. func TestExecutionReadPinArbitrationAgainstArango(t *testing.T) { url, database := os.Getenv("LOOM_TEST_ARANGO_URL"), os.Getenv("LOOM_TEST_ARANGO_DATABASE") if url == "" || database == "" { @@ -73,8 +71,6 @@ func TestExecutionReadPinArbitrationAgainstArango(t *testing.T) { } _ = registry.ReleaseExecutionReadPin(ctx, executionID, expiredOwner) - // Both contenders start from an unclaimed state. The shared state document - // must make the outcomes mutually exclusive, even when requests overlap. for race := 0; race < 50; race++ { tracedExecution := "pin-race-" + uuid.NewString() tracedReader, tracedCleanup := "reader-"+uuid.NewString(), "cleanup-"+uuid.NewString() diff --git a/internal/dataframe/publication/arango/name_latest_binding_e2e_test.go b/internal/backends/arango/internal/publication/name_latest_binding_e2e_test.go similarity index 84% rename from internal/dataframe/publication/arango/name_latest_binding_e2e_test.go rename to internal/backends/arango/internal/publication/name_latest_binding_e2e_test.go index ab79185c0..985c79d1a 100644 --- a/internal/dataframe/publication/arango/name_latest_binding_e2e_test.go +++ b/internal/backends/arango/internal/publication/name_latest_binding_e2e_test.go @@ -12,26 +12,12 @@ import ( "testing" "time" + store "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/dataframe/publication" "github.com/calypr/loom/internal/dataset" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) -// These opt-in tests reproduce the registry-level discriminator gap behind -// candidate F-B6 against a real ArangoDB in a fully owned database: -// -// Registry.FindExecutionBySelector (bundle_registry.go:174-182) filters only -// project/generation/name/translationVersion/output plus state PUBLISHED|READY -// and returns publishedAt-DESC LIMIT 1. It never binds Key, RecipeDigest, -// SchemaDigest, or ReceiptID, so two PUBLISHED executions for the same -// selector are indistinguishable to the verifier path -// (internal/server/release_verifier.go:18-35 -> dataset/release.go:181-190), -// which echoes whatever execution the finder returned. -// -// Databases use the loom_verify_nlb_ prefix so this suite's residue is -// checkable independently of other verification workers. - const nameBindingDatabasePrefix = "loom_verify_nlb_" const nameBindingSortAQLMarker = "SORT doc.publishedAt DESC" @@ -93,11 +79,6 @@ func (c *nameBindingRecorder) lastSelectorQuery(t *testing.T) nameBindingQuery { return records[len(records)-1] } -// nameBindingOpen mirrors the owned-database harness used by -// internal/explorer/arango/selections_cleanup_e2e_test.go: opt-in via -// LOOM_TEST_ARANGO_URL, uuid-suffixed name under the suite prefix, created via -// store.Open (client.go:111-151 creates the database when absent), dropped by -// HTTP DELETE /_api/database/ in t.Cleanup. func nameBindingOpen(t *testing.T, budget time.Duration) (context.Context, *store.Client, *Registry, *nameBindingRecorder) { t.Helper() endpoint := os.Getenv("LOOM_TEST_ARANGO_URL") @@ -147,11 +128,6 @@ func nameBindingOpen(t *testing.T, budget time.Duration) (context.Context, *stor return ctx, client, registry, recorder } -// nameBindingSeedPublished writes one PUBLISHED execution through the -// production catalog path: AcquireBundleLease then SaveExecution. SaveExecution -// is lease-fenced (bundle_registry.go:44-51 requires a live lease for the -// execution Key), so production lease acquisition is the honest seed, matching -// the lease-guarded flow the fake in bundle_registry_test.go asserts. func nameBindingSeedPublished(t *testing.T, ctx context.Context, registry *Registry, spec nameBindingExecSpec) publication.BundleExecution { t.Helper() owner := "nlb-publisher-" + spec.id @@ -217,8 +193,6 @@ func nameBindingQuoteAQL(t *testing.T, recorded nameBindingQuery) { t.Logf("FIND_EXECUTION_BY_SELECTOR_BIND_VALUES=%#v", recorded.bindVars) } -// nameBindingMissingIdentityFields are the candidate F-B6 inputs that the -// parent verified are absent from the selector AQL and its bind variables. var nameBindingMissingIdentityFields = []string{"recipeDigest", "RecipeDigest", "schemaDigest", "SchemaDigest", "receiptId", "ReceiptID", "doc.key", "doc.Key"} func nameBindingDiscriminatorsPresent(recorded nameBindingQuery) []string { @@ -240,11 +214,6 @@ func nameBindingDiscriminatorsPresent(recorded nameBindingQuery) []string { return present } -// TestFindExecutionBySelectorIgnoresRecipeIdentityAgainstArango is the -// finding-assert for candidate F-B6: two PUBLISHED executions share -// project/generation/recipe/translationVersion/output but carry distinct -// Key/RecipeDigest/SchemaDigest/ReceiptID. The production finder must return -// the publishedAt-newest doc no matter which identity the caller intended. func TestFindExecutionBySelectorIgnoresRecipeIdentityAgainstArango(t *testing.T) { ctx, _, registry, recorder := nameBindingOpen(t, 60*time.Second) @@ -297,11 +266,6 @@ func TestFindExecutionBySelectorIgnoresRecipeIdentityAgainstArango(t *testing.T) } } -// TestFindExecutionBySelectorLegacyCaseAsymmetryAgainstArango reproduces the -// brief's Bug-3: the selector AQL matches capitalized Project/Name/... in the -// FILTER but SORT only reads lowercase publishedAt/readyAt/updatedAt. A legacy -// capitalized doc storing the newest publish time in "PublishedAt" (capital P) -// is FILTER-visible but sorts as if it had no publish time. func TestFindExecutionBySelectorLegacyCaseAsymmetryAgainstArango(t *testing.T) { ctx, client, registry, recorder := nameBindingOpen(t, 60*time.Second) @@ -318,11 +282,6 @@ func TestFindExecutionBySelectorLegacyCaseAsymmetryAgainstArango(t *testing.T) { receiptID: "receipt-case-lower", publishedAt: t1, }) - // The legacy doc keeps lowercase "state" and the "outputs" array key - // because the AQL FILTER has no capitalised branch for either - // (bundle_registry.go:177-178), so a doc without them would be invisible - // rather than merely mis-sorted. Everything else, including "PublishedAt", - // uses the legacy capitalised shape the FILTER accepts via its OR branches. legacyDoc := map[string]any{ "_key": "nlb-case-legacy", "id": "nlb-case-legacy", @@ -380,11 +339,6 @@ func TestFindExecutionBySelectorLegacyCaseAsymmetryAgainstArango(t *testing.T) { } } -// nameBindingReplicatedVerificationStore is a line-for-line replica of -// internal/server/release_verifier.go:14-35 (publicationVerificationStore). -// The original is unreachable from this package because internal/server/ -// server.go:35 imports this package, so importing server from this test would -// be an import cycle; the brief authorises replicating these ~15 lines. type nameBindingReplicatedVerificationStore struct { executions publication.ExactExecutionCatalog } @@ -442,12 +396,6 @@ func (r *nameBindingReleaseRepository) savedRelease() dataset.ProjectRelease { return r.saved } -// TestVerifierToBindingChainEchoAgainstArango drives the seam the candidate -// says is load-bearing: the (replicated) publicationVerificationStore over the -// production registry, then production dataset.ReleaseService.Create, whose -// publicationBinding echoes verification.ExecutionID into the release that -// PublishAuthoring CAS-activates (explorer/arango/publication.go:57). It shows -// the binding follows the newest doc while a distinct intended execution exists. func TestVerifierToBindingChainEchoAgainstArango(t *testing.T) { ctx, _, registry, recorder := nameBindingOpen(t, 60*time.Second) @@ -482,8 +430,6 @@ func TestVerifierToBindingChainEchoAgainstArango(t *testing.T) { return } - // The intended distinct execution must still exist for the echo to mean - // anything: it is shadowed at the finder, not absent. storedIntended, err := registry.GetExecution(ctx, intended.ID) if err != nil { t.Fatalf("intended execution %s missing: %v", intended.ID, err) diff --git a/internal/dataframe/publication/arango/publish_atomicity_e2e_test.go b/internal/backends/arango/internal/publication/publish_atomicity_e2e_test.go similarity index 73% rename from internal/dataframe/publication/arango/publish_atomicity_e2e_test.go rename to internal/backends/arango/internal/publication/publish_atomicity_e2e_test.go index df6d5b8b0..f74e203ee 100644 --- a/internal/dataframe/publication/arango/publish_atomicity_e2e_test.go +++ b/internal/backends/arango/internal/publication/publish_atomicity_e2e_test.go @@ -12,47 +12,13 @@ import ( "github.com/google/uuid" + pubarango "github.com/calypr/loom/internal/backends/arango/internal/publication" + storearango "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/dataframe/publication" - pubarango "github.com/calypr/loom/internal/dataframe/publication/arango" - storearango "github.com/calypr/loom/internal/store/arango" ) -// Repair evidence for F-B16 (Round 11 CONFIRMED, FINDINGS.md): the execution -// UPDATE and the visibility-pointer UPSERT of Registry.PublishExecution were -// two AQL legs whose intermediate state was externally observable, so a -// crash/failure between the legs left an execution PUBLISHED while its -// pointer was never written ("two-answers" class: the name-latest selector -// served the orphan while the bulk-load activation gate refused it). The -// repair commits both legs inside ONE Arango transaction. -// -// The promise under test is the production comment on PublishExecution -// ("atomically commits both the visibility pointer and the PUBLISHED -// metadata. A failed compare-and-swap leaves both documents intact."): -// -// A. Legacy split control (pre-fix shape: the two legs as two separate -// requests, stopping after leg 1). The intermediate PUBLISHED-without- -// pointer state MUST be observable. This is the before-fail witness AND -// the sensitivity proof: the probe really sees a split when one exists. -// B. Repaired path under mid-failure: the publish transaction is aborted -// by the harness AFTER the statement succeeded inside it (models any -// failure after Arango applied writes-to-date). Nothing may remain -// observable: state stays RUNNING, pointer stays absent; and a clean -// retry then lands both halves together. Pre-fix code observable the -// half state here -- proved by arm A executing those same statements. -// C. Clean publish lands together; a stale compare-and-swap is refused -// with both documents intact. -// -// Opt-in (SKIP-clean without LOOM_TEST_ARANGO_URL): -// -// docker exec loom-dev-qwen-current-loom-api-1 sh -c \ -// 'cd /workspace && LOOM_TEST_ARANGO_URL=http://arangodb:8529 \ -// go test -count=1 -run TestPublishExecutionAtomicity -v ./internal/dataframe/publication/arango/' - const patDatabasePrefix = "loom_verify_patom_" -// patAbortError is returned by the harness from INSIDE the publish -// transaction callback after the publish statement already succeeded there; -// go-driver aborts (rolls back) the transaction when the callback errors. var patAbortError = errors.New("injected abort after successful publish statement") func patOpen(t *testing.T) (*storearango.Client, *pubarango.Registry) { @@ -103,8 +69,6 @@ func patOpen(t *testing.T) (*storearango.Client, *pubarango.Registry) { return client, registry } -// patSeed creates a RUNNING execution with a live lease through production -// registry entry points only. func patSeed(ctx context.Context, t *testing.T, registry *pubarango.Registry, execID, bundleKey, pointerName string) publication.BundleExecution { t.Helper() acquired, err := registry.AcquireBundleLease(ctx, bundleKey, "publisher-a", time.Now().UTC().Add(time.Hour)) @@ -127,8 +91,6 @@ func patSeed(ctx context.Context, t *testing.T, registry *pubarango.Registry, ex return execution } -// patObserve reads exactly what outside readers read: GetExecution for the -// state, GetPointer for the bulk-load activation gate's surface. func patObserve(ctx context.Context, t *testing.T, registry *pubarango.Registry, execID, pointerName string) (string, string) { t.Helper() execution, err := registry.GetExecution(ctx, execID) @@ -146,17 +108,10 @@ func TestPublishExecutionAtomicity(t *testing.T) { ctx := context.Background() t.Run("A_legacy_split_is_observable", func(t *testing.T) { - // Pre-fix product shape quoted from bundle_registry.go @ dc9016bc: - // execution UPDATE then pointer UPSERT in one non-transactional AQL; - // the crash between the legs is modelled by running leg 1 as its own - // request and stopping. client, registry := patOpen(t) execution := patSeed(ctx, t, registry, "pat-exec-a", "pat-bundle-a", "pat-project\x00pat-generation\x00pat-recipe-a") doc := patDoc(t, execution, publication.BundlePublished) applied := false - // Leg 1 verbatim from the pre-fix statement (@ dc9016bc) minus the - // pointer leg; @name/@updatedAt stay parked in LET bindings exactly - // as the sibling ppp harness does for the skipped leg. if err := client.QueryRows(ctx, `LET execution = DOCUMENT(@@executions, @executionKey) LET lease = DOCUMENT(@@leases, @leaseKey) LET pointerNameParam = @name @@ -229,8 +184,6 @@ RETURN {updated: true}`, 32, map[string]interface{}{ if state != string(publication.BundlePublished) || pointerErr != "nil" { t.Fatalf("clean publish must land both halves: state=%q pointerErr=%q", state, pointerErr) } - // Stale CAS: pointer already names pat-exec-c; a publish expecting - // something else must be refused and change nothing. rival := execution rival.ID = "pat-exec-c2" rival.State = publication.BundleRunning @@ -271,10 +224,6 @@ func patDoc(t *testing.T, execution publication.BundleExecution, state publicati return doc } -// patTxAborter delegates every call to the real client but aborts the first -// transaction AFTER its callback completed successfully, so the callback's -// writes (applied inside the transaction) are rolled back -- the exact class -// of mid-request failure that split the pre-fix publish. type patTxAborter struct { *storearango.Client armed bool diff --git a/internal/dataframe/publication/arango/registry.go b/internal/backends/arango/internal/publication/registry.go similarity index 92% rename from internal/dataframe/publication/arango/registry.go rename to internal/backends/arango/internal/publication/registry.go index dba865ed4..783bf3583 100644 --- a/internal/dataframe/publication/arango/registry.go +++ b/internal/backends/arango/internal/publication/registry.go @@ -3,7 +3,7 @@ package arango import ( "fmt" - arangostore "github.com/calypr/loom/internal/store/arango" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" ) type Client = arangostore.RowBatchClient diff --git a/internal/dataframe/publication/arango/release_dangle_reconcile_e2e_test.go b/internal/backends/arango/internal/publication/release_dangle_reconcile_e2e_test.go similarity index 68% rename from internal/dataframe/publication/arango/release_dangle_reconcile_e2e_test.go rename to internal/backends/arango/internal/publication/release_dangle_reconcile_e2e_test.go index 686183feb..576628d37 100644 --- a/internal/dataframe/publication/arango/release_dangle_reconcile_e2e_test.go +++ b/internal/backends/arango/internal/publication/release_dangle_reconcile_e2e_test.go @@ -15,23 +15,19 @@ import ( "testing" "time" + datasetstore "github.com/calypr/loom/internal/backends/arango/internal/dataset" + store "github.com/calypr/loom/internal/backends/arango/internal/store" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" "github.com/calypr/loom/internal/dataframe/publication" pubclickhouse "github.com/calypr/loom/internal/dataframe/publication/clickhouse" "github.com/calypr/loom/internal/dataframe/published" "github.com/calypr/loom/internal/dataset" - datasetstore "github.com/calypr/loom/internal/dataset/arango" - store "github.com/calypr/loom/internal/store/arango" storeclickhouse "github.com/calypr/loom/internal/store/clickhouse" "github.com/google/uuid" ) -// rrdTablePrefix names the physical ClickHouse tables this suite owns, so the -// residue check greps exactly `ch_rrd%` in system.tables. const rrdTablePrefix = "ch_rrd" -// rrdDatabasePrefix names the owned Arango database; nothing outside it is -// created or dropped. const rrdDatabasePrefix = "loom_verify_rrd_" type rrdCitation struct { @@ -40,47 +36,32 @@ type rrdCitation struct { want string } -// rrdCitations pin the load-bearing source lines this test depends on. If a -// refactor moves any of them the guard fails the test BEFORE the env gate, so -// stale evidence cannot quietly survive a source change (mirrors the sibling -// dangling_binding_e2e_test.go harness). var rrdCitations = []rrdCitation{ - // The pointer guard reads the visibility pointer OUTSIDE the publish - // sequence and only skips the drop when the pointer already names the - // execution. An absent pointer (the two-doc publish window) proceeds. - {"internal/dataframe/publication/clickhouse/bundle_store.go", 1068, "pointer, pointerErr := s.catalog.GetPointer(cleanupCtx, execution.PointerName())"}, - {"internal/dataframe/publication/clickhouse/bundle_store.go", 1069, "if pointerErr != nil && !errors.Is(pointerErr, publication.ErrBundleNotFound) {"}, - {"internal/dataframe/publication/clickhouse/bundle_store.go", 1073, "if pointerErr == nil && pointer.ExecutionID == execution.ID {"}, - // The physical drop and the FAILED demotion of the candidate. - {"internal/dataframe/publication/clickhouse/bundle_store.go", 1084, "if err := s.clickHouse.DropTable(leaseCtx, output.PhysicalTable); err != nil {"}, - {"internal/dataframe/publication/clickhouse/bundle_store.go", 1089, "execution.State = publication.BundleFailed"}, - {"internal/dataframe/publication/clickhouse/bundle_store.go", 1097, "if err := s.catalog.SaveExecution(leaseCtx, execution, reconcilerID); err != nil {"}, - // The release read path resolves a binding via GetExecution and never - // consults the pointer; a demoted/non-successful execution is an ERROR. - {"internal/dataframe/published/current.go", 95, "execution, err := r.Catalog.GetExecution(ctx, binding.ExecutionID)"}, - {"internal/dataframe/published/current.go", 99, "if execution.ID != binding.ExecutionID || execution.Project != release.Project || execution.DatasetGeneration != binding.Generation || !execution.State.Successful() {"}, - {"internal/dataframe/published/current.go", 105, "return Materialization{}, fmt.Errorf(\"%w: active dataframe execution %q does not match release %q\", errReleaseBindingUnavailable, binding.ExecutionID, release.ID)"}, - // One failing binding aborts the WHOLE CurrentProjectDatasets response. + {"internal/dataframe/publication/clickhouse/bundle_store.go", 1106, "pointer, pointerErr := s.catalog.GetPointer(cleanupCtx, execution.PointerName())"}, + {"internal/dataframe/publication/clickhouse/bundle_store.go", 1107, "if pointerErr != nil && !errors.Is(pointerErr, publication.ErrBundleNotFound) {"}, + {"internal/dataframe/publication/clickhouse/bundle_store.go", 1111, "if pointerErr == nil && pointer.ExecutionID == execution.ID {"}, + {"internal/dataframe/publication/clickhouse/bundle_store.go", 1143, "if err := s.clickHouse.DropTable(leaseCtx, output.PhysicalTable); err != nil {"}, + {"internal/dataframe/publication/clickhouse/bundle_store.go", 1148, "execution.State = publication.BundleFailed"}, + {"internal/dataframe/publication/clickhouse/bundle_store.go", 1159, "if err := s.catalog.SaveExecution(leaseCtx, execution, reconcilerID); err != nil {"}, + {"internal/dataframe/published/current.go", 94, "execution, err := r.Catalog.GetExecution(ctx, binding.ExecutionID)"}, + {"internal/dataframe/published/current.go", 101, "if execution.ID != binding.ExecutionID || execution.Project != release.Project || execution.DatasetGeneration != binding.Generation || !execution.State.Successful() {"}, + {"internal/dataframe/published/current.go", 102, "return Materialization{}, fmt.Errorf(\"%w: active dataframe execution %q does not match release %q\", errReleaseBindingUnavailable, binding.ExecutionID, release.ID)"}, {"internal/dataframe/published/current.go", 206, "value, err := r.releaseMaterialization(ctx, active.Release, binding)"}, {"internal/dataframe/published/current.go", 207, "if err != nil {"}, - {"internal/dataframe/published/current.go", 208, "return nil, err"}, - // The cleanup claim filters on pins/cleanup only, never a release binding. - {"internal/dataframe/publication/arango/bundle_registry.go", 431, "FILTER LENGTH(active) == 0 AND (cleanup == null OR cleanup.expiresAt < @now OR cleanup.ownerId == @owner)"}, - // Release activation pins nothing: the only production read-pin callers. - {"internal/dataframe/published/read_pins.go", 63, "owned, err := pins.AcquireExecutionReadPin(ctx, id, owner, time.Now().UTC().Add(ttl))"}, - {"internal/dataframe/published/export.go", 135, "acquired, err := pins.AcquireExecutionReadPin(ctx, result.ExecutionID, result.ReaderID, expires)"}, + {"internal/dataframe/published/current.go", 211, "return nil, err"}, + {"internal/backends/arango/internal/publication/bundle_registry.go", 421, "FILTER LENGTH(active) == 0 AND (cleanup == null OR cleanup.expiresAt < @now OR cleanup.ownerId == @owner)"}, + {"internal/dataframe/published/read_pins.go", 59, "owned, err := pins.AcquireExecutionReadPin(ctx, id, owner, time.Now().UTC().Add(ttl))"}, + {"internal/dataframe/published/export.go", 115, "acquired, err := pins.AcquireExecutionReadPin(ctx, result.ExecutionID, result.ReaderID, expires)"}, } -// rrdAbsentIdentifiers are names whose absence the hypothesis depends on. type rrdAbsent struct { file string identifier string } var rrdAbsentIdentifiers = []rrdAbsent{ - // release activation must not be a read-pin caller. - {"internal/dataset/arango/release_store.go", "AcquireExecutionReadPin"}, - {"internal/dataset/arango/release_store.go", "ReadPin"}, + {"internal/backends/arango/internal/dataset/release_store.go", "AcquireExecutionReadPin"}, + {"internal/backends/arango/internal/dataset/release_store.go", "ReadPin"}, } func rrdRepoRoot(t *testing.T) string { @@ -115,21 +96,8 @@ func rrdCheckCitations(t *testing.T) { lines = strings.Split(string(raw), "\n") contents[c.file] = lines } - if c.line < 1 || c.line > len(lines) { - t.Fatalf("citation drift: %s has %d lines, cited :%d == %q", c.file, len(lines), c.line, c.want) - } - // Drift-proof: the contract is the EXACT source text, not the pin. A - // repaired file shifts lines constantly; require the cited statement - // to still exist verbatim somewhere and only note pin drift. - found := false - for _, line := range lines { - if strings.TrimSpace(line) == strings.TrimSpace(c.want) { - found = true - break - } - } - if !found { - t.Fatalf("citation drift: %s no longer contains %q (cited :%d)", c.file, c.want, c.line) + if mismatch := citationLineMismatch(lines, c.line, c.want); mismatch != "" { + t.Fatalf("citation drift: %s %s (want %q)", c.file, mismatch, c.want) } } for _, a := range rrdAbsentIdentifiers { @@ -141,8 +109,7 @@ func rrdCheckCitations(t *testing.T) { t.Fatalf("hypothesis invariant broken: %q unexpectedly present in %s (release activation must not read-pin)", a.identifier, a.file) } } - // ClaimExecutionCleanup body carries no release/lifecycle binding signal. - raw, err := os.ReadFile(filepath.Join(root, "internal/dataframe/publication/arango/bundle_registry.go")) + raw, err := os.ReadFile(filepath.Join(root, "internal/backends/arango/internal/publication/bundle_registry.go")) if err != nil { t.Fatalf("citation source bundle_registry.go unreadable: %v", err) } @@ -158,11 +125,6 @@ func rrdCheckCitations(t *testing.T) { } } -// rrdDatasetDocumentKey replicates internal/dataset/arango/document_codec.go -// documentKey (kind + sha256 over the loom.datasetstore.v1 salt) so this -// package can address the active-release pointer that SaveRelease inserted. -// Fidelity is validated by a production ReadActiveRelease round-trip: a wrong -// key surfaces as ErrNoActiveRelease and fails the test. func rrdDatasetDocumentKey(kind string, values ...string) string { hash := sha256.New() _, _ = hash.Write([]byte("loom.datasetstore.v1")) @@ -192,12 +154,6 @@ type rrdHarness struct { stageErr error } -// rrdOpen is the env-gated owned-database harness. It mirrors the sibling -// dangling/name-latest harnesses (t.Skip when LOOM_TEST_ARANGO_URL is unset, -// an owned loom_verify_rrd_ database dropped via HTTP DELETE in -// t.Cleanup) AND additionally opens a REAL ClickHouse client (matching -// internal/store/clickhouse/integration_test.go env names) so the physical -// tables are genuinely created and genuinely dropped by production DropTable. func rrdOpen(t *testing.T) *rrdHarness { t.Helper() endpoint := strings.TrimSpace(os.Getenv("LOOM_TEST_ARANGO_URL")) @@ -283,8 +239,6 @@ func rrdOpen(t *testing.T) *rrdHarness { t.Fatalf("ping clickhouse: %v", err) } h.ch = ch - // Drop the two owned physical tables no matter how the test ends, and - // report exactly which tables are still present AFTER the drop. t.Cleanup(func() { cleanupCtx, stop := context.WithTimeout(context.Background(), 30*time.Second) defer stop() @@ -322,9 +276,6 @@ func rrdSelect(output, recipe string) published.DataframeSelector { return published.DataframeSelector{Recipe: recipe, TranslationVersion: "v1", Output: output} } -// rrdSeedHealthy publishes the healthy execution for real: a genuinely -// queryable PUBLISHED row plus a real ClickHouse table plus its visibility -// pointer. Reconcile leaves it alone (PUBLISHED is not an enumerated state). func (h *rrdHarness) seedHealthy(t *testing.T) { t.Helper() owner := "rrd-pub-healthy-" + h.suffix @@ -353,11 +304,6 @@ func (h *rrdHarness) seedHealthy(t *testing.T) { h.healthy = exec } -// rrdSeedDangle writes the candidate as RUNNING (an enumerated state) with a -// queryable-shaped output row carrying the physical table, then leaves its -// bundle lease EXPIRED so the reconciler can claim it (the "stale RUNNING with -// an expired lease" precondition). No pointer is written: that is the whole -// point of the guard. func (h *rrdHarness) seedDangle(t *testing.T) { t.Helper() owner := "rrd-pub-dangle-" + h.suffix @@ -376,16 +322,12 @@ func (h *rrdHarness) seedDangle(t *testing.T) { if err := h.registry.SaveExecution(h.ctx, exec, owner); err != nil { t.Fatalf("dangle SaveExecution: %v", err) } - // Make the seed-held lease expired (same owner refresh) and leave it in - // place: reconcileExecution's AcquireBundleLease steals an expired lease. if claimed, err := h.registry.AcquireBundleLease(h.ctx, h.dangle.Key, owner, time.Now().UTC().Add(-time.Minute)); err != nil || !claimed { t.Fatalf("dangle expire lease: claimed=%v err=%v", claimed, err) } h.dangle = exec } -// rrdCreatePhysicalTable creates the real MergeTree table so the reconciler's -// DropTable has something real to physically remove. func (h *rrdHarness) createPhysicalTable(t *testing.T, table string) { t.Helper() if err := h.ch.CreateTable(h.ctx, table, []storeclickhouse.Column{{Name: "__loom_row_id", Type: "Int64"}}); err != nil { @@ -410,10 +352,6 @@ func (h *rrdHarness) listRRDTables(t *testing.T) []string { return names } -// rrdActivateRelease saves an immutable release that binds BOTH executions and -// activates it by patching the active_release placeholder SaveRelease created -// (release_store.go saveReleaseAQL:132-142 / readActiveReleaseAQL:144-150). It -// is validated by production ReadActiveRelease, so a bad key/round-trip fails. func (h *rrdHarness) activateRelease(t *testing.T, bindings []dataset.ReleasePublication) dataset.ProjectRelease { t.Helper() release := dataset.ProjectRelease{ @@ -451,12 +389,6 @@ RETURN {activated: true}`, 32, map[string]interface{}{ return saved } -// rrdStagePublicDocVisibleWithoutPointer writes the candidate's exec doc to a -// queryable PUBLISHED row via a direct AQL patch. This is PublishExecution's -// FIRST doc of its two-doc (exec-state + pointer) publication: the exec-state -// is made visible while the visibility POINTER doc is deliberately left -// ABSENT, exactly the non-atomic window that lets reconcileExecution's -// pre-claim GetPointer guard observe nothing to skip on. func (h *rrdHarness) stagePublishedDocWithoutPointer(t *testing.T) { t.Helper() stamp := time.Now().UTC() @@ -495,12 +427,6 @@ RETURN {updated: true}`, 32, map[string]interface{}{ } } -// rrdCatalog is the seam hook. It embeds the real *Registry and overrides -// exactly the exported enumerate/candidate cleanup claim: reconcileExecution's -// pre-claim pointer guard has ALREADY passed on the absent pointer by the -// time ClaimExecutionCleanup runs, so staging the racing publish + release -// Create+Activate here reproduces the F-B12 interleaving deterministically, -// with NO goroutine. Every other capability is the production AQL verbatim. type rrdCatalog struct { *Registry fired bool @@ -533,62 +459,10 @@ func rrdUserErrorCode(err error) string { return "" } -// TestReconcileDanglesActiveReleaseAgainstArangoClickHouse adjudicates -// candidate F-B12 end-to-end against a real ArangoDB (fully owned -// loom_verify_rrd_ database, HTTP-DELETE dropped) AND a real ClickHouse -// (owned ch_rrd* MergeTree tables, physically dropped by production DropTable). -// -// HYPOTHESIS F-B12: a reconcile pass that enumerated a stale RUNNING execution -// can physically DropTable its output rows and demote it to FAILED AFTER a -// PUBLISHED state + release Create+Activate landed, because the pointer guard -// (the reconcileExecution pre-claim pointer guard) reads the pointer outside the publish sequence, -// PublishExecution writes exec-state and pointer as two docs (so the pointer -// can be absent while the exec is visible), release activation pins nothing -// (only read_pins.go:63 and export.go:135 call AcquireExecutionReadPin), and -// ClaimExecutionCleanup (bundle_registry.go:431) has no release-binding signal. -// The release read path (current.go:94-96 -> GetExecution) then total-fails -// CurrentProjectDatasets (current.go:206-208). -// -// The interleaving is staged by ordering the docs by hand: the candidate enters -// Reconcile as RUNNING (real enumerate via the exported VisitExecutionPages), -// reconcileExecution reads the ABSENT pointer and decides to drop, and INSIDE -// the exported ClaimExecutionCleanup seam the racing publish makes the exec doc -// a queryable PUBLISHED row (pointer still absent - the two-doc window) and a -// release binding BOTH the healthy and the candidate execution is Created + -// Activated. Real production AQL then claims cleanup, DropTables the real CH -// table, and demotes to FAILED. -// -// Pre-stated expectations (declared BEFORE running): -// -// O-ENUM the exported RUNNING enumerate yields exactly {dangle} and NOT {healthy}. -// O-MID mid-seam CurrentProjectDatasets = the 2 materializations, err == nil -// (both PUBLISHED + queryable, both real tables present). -// O-CONF after Reconcile: candidate State == FAILED, Outputs == []; the candidate's -// real CH table was physically DropTable'd (system.tables shows it gone); -// the ACTIVE release STILL binds the candidate ExecutionID; and -// CurrentProjectDatasets returns nil + error (the whole list, including -// the still-healthy binding, is lost) => FAILED-ON-PURPOSE / CONFIRMED. -// O-DISP the drop is prevented (candidate still PUBLISHED, or its table still -// present, or CurrentProjectDatasets still serves) => PASS, and we quote -// the exact line that stopped it. -// -// STATUS after Repair-1 (Round 26): PublishExecution is now transactional, so -// the intermediate state this artifact stages (PUBLISHED row with an ABSENT -// pointer) is no longer product-reachable through the publish path itself — -// it models (a) legacy rows written pre-repair and (b) non-lease-guarded -// PUBLISHED writers (F-B13 activateReleaseAQL patch class), and the reconciler -// holding the lease from the reconcileExecution claim prevents a lease-fenced racer -// (model caveat in ReconcileDangleTrust.tla, m-dangle-atomic-current). The -// drop path STILL lacks the release-binding re-check, so this artifact -// legitimately remains FAILED-ON-PURPOSE: Repair-1b (re-check pointer AND -// active-release bindings before DropTable/demote — the shape validated by -// m-dangle-atomic-recheck) converts it through the O-DISP branch with no -// assertion rewrite. Do not delete or skip this red; it is the open defect. func TestReconcileDanglesActiveReleaseAgainstArangoClickHouse(t *testing.T) { rrdCheckCitations(t) h := rrdOpen(t) - // Seed everything for real. h.seedHealthy(t) h.seedDangle(t) h.createPhysicalTable(t, h.healthyTable) @@ -601,8 +475,6 @@ func TestReconcileDanglesActiveReleaseAgainstArangoClickHouse(t *testing.T) { t.Fatalf("precondition: both owned tables must exist, got %v", before) } - // (2a) Capture the enumerate/candidate state through the EXPORTED enumerate - // (not a goroutine): the same VisitExecutionPages Reconcile drives per state. t.Log("EXPECT O-ENUM: exported RUNNING enumerate yields exactly the dangle execution (healthy is PUBLISHED, never enumerated)") candidates := rrdEnumerate(t, h.registry, publication.BundleRunning) var candidateStates []string @@ -619,18 +491,12 @@ func TestReconcileDanglesActiveReleaseAgainstArangoClickHouse(t *testing.T) { t.Fatalf("O-ENUM violated: healthy PUBLISHED execution was enumerated as a drop candidate: %v", candidateStates) } - // Independent healthy baseline: prove a PUBLISHED + queryable execution - // served by the active release WOULD resolve, so any later whole-list - // failure is provably caused by the dangle drop, not by a bad seed. reader := &published.Reader{Catalog: h.registry, ActiveReleaseResolver: h.lifecycle} readerHarnessForBaseline := func(bindings []dataset.ReleasePublication) { h.activateRelease(t, bindings) } - // Bindings reused for the mid-seam release. Capture the healthy baseline - // first using a throwaway release activation to confirm the healthy binding - // alone resolves (this activation is later overwritten by the seam). healthyOnly := []dataset.ReleasePublication{ {Selector: rrdSelect("out-a", "rrd-recipe-a"), ExecutionID: h.healthy.ID, Generation: h.generation, VerifiedAt: time.Now().UTC()}, } @@ -641,19 +507,11 @@ func TestReconcileDanglesActiveReleaseAgainstArangoClickHouse(t *testing.T) { t.Fatalf("healthy-baseline violated: a genuine PUBLISHED binding must resolve: %d %+v err=%v", len(healthBase), healthBase, baseErr) } - // Build the store over the seam-wrapped catalog. The candidate is RUNNING - // with an expired lease, so reconcileExecution proceeds to the drop path. - // Lease TTL stays at its 2-minute default so the fenced SaveExecution(FAILED) - // at the claim step actually lands (the demotion must be durable, not a lost fence). wrapper := &rrdCatalog{Registry: h.registry} wrapper.onRun = func(executionID string) { if executionID != h.dangle.ID { return } - // This runs AFTER reconcileExecution's GetPointer guard has already - // returned ABSENT (so the guard has nothing to skip on) and BEFORE the - // real claim + drop + demotion. Simulate PublishExecution's first doc + - // the release Create+Activate landing on top of the in-flight reconcile. h.stagePublishedDocWithoutPointer(t) bindings := []dataset.ReleasePublication{ {Selector: rrdSelect("out-a", "rrd-recipe-a"), ExecutionID: h.healthy.ID, Generation: h.generation, VerifiedAt: time.Now().UTC()}, @@ -661,8 +519,6 @@ func TestReconcileDanglesActiveReleaseAgainstArangoClickHouse(t *testing.T) { } h.activateRelease(t, bindings) - // Mid-seam read: both bindings genuinely resolvable right now (candidate - // published + queryable, its real table not yet dropped). mid, midErr := reader.CurrentProjectDatasets(h.ctx, h.project) h.midSeamErrChain = rrdErrorChain(midErr) for _, m := range mid { @@ -677,13 +533,8 @@ func TestReconcileDanglesActiveReleaseAgainstArangoClickHouse(t *testing.T) { if err != nil { t.Fatalf("NewBundleStore: %v", err) } - // Production wiring mirror (server.go): the reconciler re-checks the - // ACTIVE release bindings immediately before any drop (Repair-1b). store.SetActiveReleaseReader(h.lifecycle) - // The candidate's pointer is absent when the pass arms: the guard reads - // nothing to skip on. This is the guard's failure mode (an absent pointer is - // treated as "not currently advertised" -> proceed), not an error. t.Log("EXPECT pre-reconcile: candidate pointer is ABSENT, so the pre-claim pointer guard cannot skip the drop") if p, err := h.registry.GetPointer(h.ctx, h.dangle.PointerName()); !errors.Is(err, publication.ErrBundleNotFound) { t.Fatalf("precondition: candidate pointer must be absent before reconcile, got %+v err=%v", p, err) @@ -705,7 +556,6 @@ func TestReconcileDanglesActiveReleaseAgainstArangoClickHouse(t *testing.T) { return } - // OBSERVED after reconcile. execNow, execErr := h.registry.GetExecution(h.ctx, h.dangle.ID) if execErr != nil { t.Fatalf("read candidate execution after reconcile: %v", execErr) @@ -737,14 +587,9 @@ func TestReconcileDanglesActiveReleaseAgainstArangoClickHouse(t *testing.T) { releaseBindsDangle, releaseBindsHealthy, rrdBoundIDs(active), active.Revision) t.Logf("OBSERVED final CurrentProjectDatasets: list=%d err=[%s] userCode=%q", len(finalList), rrdErrorChain(finalErr), rrdUserErrorCode(finalErr)) - // Independent confirmation that the whole-list loss is attributable to the - // dangle binding, not to the healthy one: the healthy row + table survive. healthNow, healthNowErr := h.registry.GetExecution(h.ctx, h.healthy.ID) healthyStillServes := healthNowErr == nil && healthNow.State.Successful() && healthyPresent - // Prove the release-read path never consults the pointer: write the - // candidate's pointer LAST (PublishExecution's second doc eventually - // completes) naming the now-FAILED execution. The read still must fail. if err := h.registry.CompareAndSwapPointer(h.ctx, h.dangle.PointerName(), "", h.dangle.ID); err != nil { t.Fatalf("release-dangle pointer-last write: %v", err) } @@ -773,8 +618,6 @@ func TestReconcileDanglesActiveReleaseAgainstArangoClickHouse(t *testing.T) { prevent, detail, execNow.State, len(execNow.Outputs), dangleGone, releaseBindsDangle, len(finalList), rrdErrorChain(finalErr)) } - // Residue: after this function the cleanup handlers drop the tables; log the - // live state so a reviewer sees exactly what remains to be reported. t.Logf("RRD_RESIDUE_CLICKHOUSE_TABLES=%v", h.listRRDTables(t)) } @@ -808,8 +651,6 @@ func rrdBoundIDs(active dataset.ActiveRelease) string { return strings.Join(parts, " ") } -// rrdDiagnose names the exact load-bearing line that prevented the drop so a -// DISPROVEN verdict cites the mechanism rather than merely asserting it. func rrdDiagnose(exec publication.BundleExecution, execErr error, dangleGone, healthyPresent, releaseBindsDangle bool, list []published.Materialization, listErr, reconcileErr error) (string, string) { switch { case dangleGone && exec.State == publication.BundleFailed && releaseBindsDangle: diff --git a/internal/dataframe/recipe/exec/arango/registry.go b/internal/backends/arango/internal/recipeexec/registry.go similarity index 96% rename from internal/dataframe/recipe/exec/arango/registry.go rename to internal/backends/arango/internal/recipeexec/registry.go index 32f9f3568..75be749a1 100644 --- a/internal/dataframe/recipe/exec/arango/registry.go +++ b/internal/backends/arango/internal/recipeexec/registry.go @@ -1,5 +1,3 @@ -// Package arango provides the durable exec.Store adapter used by Loom -// deployments that already persist dataset metadata in ArangoDB. package arango import ( @@ -9,8 +7,8 @@ import ( "encoding/json" "fmt" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/dataframe/recipe/exec" - arangostore "github.com/calypr/loom/internal/store/arango" ) const Collection = "loom_dataframe_recipes" diff --git a/internal/dataframe/recipe/exec/arango/registry_test.go b/internal/backends/arango/internal/recipeexec/registry_test.go similarity index 98% rename from internal/dataframe/recipe/exec/arango/registry_test.go rename to internal/backends/arango/internal/recipeexec/registry_test.go index 814c08b44..7d41234ca 100644 --- a/internal/dataframe/recipe/exec/arango/registry_test.go +++ b/internal/backends/arango/internal/recipeexec/registry_test.go @@ -8,9 +8,9 @@ import ( "strings" "testing" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/recipe/exec" - arangostore "github.com/calypr/loom/internal/store/arango" ) type legacyRegistryClient struct { diff --git a/internal/dataframe/recipe/exec/arango/revisions.go b/internal/backends/arango/internal/recipeexec/revisions.go similarity index 96% rename from internal/dataframe/recipe/exec/arango/revisions.go rename to internal/backends/arango/internal/recipeexec/revisions.go index ed09efba8..1d91511cd 100644 --- a/internal/dataframe/recipe/exec/arango/revisions.go +++ b/internal/backends/arango/internal/recipeexec/revisions.go @@ -11,16 +11,14 @@ import ( "strings" "time" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/recipe/exec" - arangostore "github.com/calypr/loom/internal/store/arango" ) const RevisionCollection = "loom_dataframe_recipe_revisions" -// RevisionRegistry persists immutable project recipe revisions in the same -// Arango database/client as the legacy name-addressed recipe registry. type RevisionRegistry struct { client Client batchSize int diff --git a/internal/dataframe/recipe/exec/arango/revisions_test.go b/internal/backends/arango/internal/recipeexec/revisions_test.go similarity index 97% rename from internal/dataframe/recipe/exec/arango/revisions_test.go rename to internal/backends/arango/internal/recipeexec/revisions_test.go index f2fe90f62..f0bb73f34 100644 --- a/internal/dataframe/recipe/exec/arango/revisions_test.go +++ b/internal/backends/arango/internal/recipeexec/revisions_test.go @@ -7,9 +7,9 @@ import ( "strings" "testing" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" "github.com/calypr/loom/internal/dataframe/recipe" - arangostore "github.com/calypr/loom/internal/store/arango" ) type revisionClient struct { @@ -51,8 +51,6 @@ func (c *revisionClient) QueryRows(_ context.Context, query string, _ int, vars continue } if strings.Contains(query, "RETURN doc.digest") { - // The driver cannot decode scalar AQL rows into map[string]any; - // a scalar projection must fail exactly like the real cursor. return errors.New("read arango query cursor: json: cannot unmarshal string into Go value of type map[string]interface {}") } if err := visit(doc); err != nil { diff --git a/internal/dataframe/recipe/exec/arango/revisions_user_faults_test.go b/internal/backends/arango/internal/recipeexec/revisions_user_faults_test.go similarity index 78% rename from internal/dataframe/recipe/exec/arango/revisions_user_faults_test.go rename to internal/backends/arango/internal/recipeexec/revisions_user_faults_test.go index 398733da6..4a1906d26 100644 --- a/internal/dataframe/recipe/exec/arango/revisions_user_faults_test.go +++ b/internal/backends/arango/internal/recipeexec/revisions_user_faults_test.go @@ -7,21 +7,11 @@ import ( "strings" "testing" + arangostore "github.com/calypr/loom/internal/backends/arango/internal/store" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" "github.com/calypr/loom/internal/dataframe/recipe" - arangostore "github.com/calypr/loom/internal/store/arango" ) -// A registered recipe name must stay re-registerable with correct parent -// chaining. The currentDigest AQL projection returned a bare scalar -// (RETURN doc.digest); the arango driver cursor decoder requires object rows, -// so EVERY second and later registration of a name died on a raw driver -// decode error and surfaced as INTERNAL_ERROR 500 (retryable=false) at the -// GraphQL boundary — recipe authoring save loops could never complete. -// The fake below reproduces the driver contract (only object rows decode); -// the legacy fake in revisions_test.go hand-synthesized object rows for any -// projection, which is why this defect shipped green. - type driverFaithfulClient struct { docs []map[string]any } @@ -59,7 +49,6 @@ func (c *driverFaithfulClient) QueryRows(_ context.Context, query string, _ int, continue } if scalar { - // Real driver behavior: scalar rows fail to decode into map[string]any. return errors.New("read arango query cursor: json: cannot unmarshal string into Go value of type map[string]interface {}") } row := map[string]any{"digest": doc["digest"], "project": doc["project"], "name": doc["name"]} @@ -93,8 +82,6 @@ func TestSecondRegisterOfExistingNameIsClassifiedNotInternal(t *testing.T) { if err != nil { t.Fatalf("first registration failed: %v", err) } - // Exact duplicate without parent: the current-parent contract must be - // reported as a user fault (STALE_CURSOR), never as an untyped internal. _, err = registerForTest(t, reg, "rows-a", "1", "") userErr, ok := classifiedUserFault(err) if !ok { @@ -103,11 +90,9 @@ func TestSecondRegisterOfExistingNameIsClassifiedNotInternal(t *testing.T) { if userErr.Code() != "STALE_CURSOR" { t.Fatalf("duplicate registration code = %s, want STALE_CURSOR", userErr.Code()) } - // Stale parent likewise stays a typed user fault. if _, err := registerForTest(t, reg, "rows-a", "2", strings.Repeat("f", 64)); !isStaleCursor(err) { t.Fatalf("stale parent must be STALE_CURSOR, got %v", err) } - // Correct parent chains normally. child, err := registerForTest(t, reg, "rows-a", "2", first.Digest) if err != nil { t.Fatalf("correct-parent child registration failed: %v", err) diff --git a/internal/store/arango/client.go b/internal/backends/arango/internal/store/client.go similarity index 82% rename from internal/store/arango/client.go rename to internal/backends/arango/internal/store/client.go index 09de608e5..1ea8a751a 100644 --- a/internal/store/arango/client.go +++ b/internal/backends/arango/internal/store/client.go @@ -11,6 +11,7 @@ import ( "net/http" "strings" "sync" + "sync/atomic" "time" driver "github.com/arangodb/go-driver/v2/arangodb" @@ -24,83 +25,56 @@ type Client struct { rawURL string dbName string client *http.Client + transport *http.Transport previewIndexMu sync.Mutex + closed atomic.Bool +} + +var ErrClientClosed = errors.New("arango: client closed") + +func (c *Client) checkOpen() error { + if c == nil { + return fmt.Errorf("%w: nil client", ErrClientClosed) + } + if c.closed.Load() { + return fmt.Errorf("%w: %s/%s", ErrClientClosed, c.rawURL, c.dbName) + } + return nil } const previewCursorCleanupTimeout = time.Second type queryMemoryLimitContextKey struct{} -// WithQueryMemoryLimit adds a per-query ArangoDB memory cap. Callers that do -// not set this option keep the server's configured query behavior. func WithQueryMemoryLimit(ctx context.Context, bytes int64) context.Context { return context.WithValue(ctx, queryMemoryLimitContextKey{}, bytes) } -// RowQueryer is the narrow row-query surface shared by Arango adapters. It is -// also the only capability exposed to transaction callbacks. type RowQueryer interface { QueryRows(context.Context, string, int, map[string]interface{}, RowVisitor) error } -// BatchInserter is the narrow raw-document write surface shared by Arango -// adapters. type BatchInserter interface { InsertBatchRaw(context.Context, string, []json.RawMessage, bool, string) error } -// AQLExecutor is the narrow surface used for non-row AQL statements. type AQLExecutor interface { ExecuteAQL(context.Context, string, map[string]interface{}) error } -// RowBatchClient is the narrow composite used by registries that only need -// query and raw-document batch-write capabilities. type RowBatchClient interface { RowQueryer BatchInserter } -// TransactionalRowClient is the composite used by registry writes that must -// commit several documents atomically (e.g. the bundle publish CAS). Requiring -// it at the call site makes a silently non-transactional publish impossible. type TransactionalRowClient interface { RowBatchClient WithTransaction(context.Context, TransactionCollections, TransactionFunc) error } -// IsQueryMemoryLimitExceeded reports whether ArangoDB rejected a query because -// it exceeded the configured query memory limit. -func IsQueryMemoryLimitExceeded(err error) bool { - if !IsQueryResourceLimitExceeded(err) { - return false - } - ok, arangoErr := shared.IsArangoError(err) - return ok && strings.Contains(strings.ToLower(arangoErr.ErrorMessage), "memory") -} - -// IsQueryResourceLimitExceeded reports whether ArangoDB rejected a query at a -// configured resource limit. Callers can use IsQueryMemoryLimitExceeded first -// when they need the narrower diagnosis. -func IsQueryResourceLimitExceeded(err error) bool { - return shared.IsArangoErrorWithErrorNum(err, shared.ErrResourceLimit) -} - -// IsQueryOutOfMemory reports whether ArangoDB ran out of memory while -// executing a query. This is distinct from rejecting a query at a configured -// memory limit. -func IsQueryOutOfMemory(err error) bool { - return shared.IsArangoErrorWithErrorNum(err, shared.ErrOutOfMemory) -} - -// IsQueryUserAssertion reports whether AQL aborted through ASSERT with the -// exact stable code emitted by Loom's typed physical plan. -func IsQueryUserAssertion(err error, code string) bool { - if strings.TrimSpace(code) == "" { - return false - } - ok, arangoErr := shared.IsArangoError(err) - return ok && arangoErr.ErrorNum == shared.ErrQueryUserAssert && strings.Contains(arangoErr.ErrorMessage, "AQL: "+code) +func QueryMemoryLimitFromContext(ctx context.Context) (int64, bool) { + bytes, ok := ctx.Value(queryMemoryLimitContextKey{}).(int64) + return bytes, ok } type TransactionCollections struct { @@ -116,6 +90,14 @@ var bufferPool = sync.Pool{ }, } +func DatabaseExists(ctx context.Context, url, database string) (bool, error) { + conn := connection.NewHttpConnection(connection.HttpConfiguration{ + Endpoint: connection.NewRoundRobinEndpoints([]string{url}), + }) + client := driver.NewClient(conn) + return client.DatabaseExists(ctx, database) +} + func Open(ctx context.Context, url, database string) (*Client, error) { transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -151,14 +133,18 @@ func Open(ctx context.Context, url, database string) (*Client, error) { httpClient := &http.Client{Transport: transport} return &Client{ - db: db, - rawURL: url, - dbName: database, - client: httpClient, + db: db, + rawURL: url, + dbName: database, + client: httpClient, + transport: transport, }, nil } func (c *Client) Bootstrap(ctx context.Context, spec BootstrapSpec) error { + if err := c.checkOpen(); err != nil { + return err + } total := len(spec.Collections) for i, collection := range spec.Collections { reportBootstrap(spec, "go_bootstrap_collection_start", map[string]any{ @@ -218,6 +204,9 @@ func (c *Client) Bootstrap(ctx context.Context, spec BootstrapSpec) error { } func (c *Client) InsertBatchRaw(ctx context.Context, collection string, docs []json.RawMessage, overwrite bool, writeAPI string) error { + if err := c.checkOpen(); err != nil { + return err + } if len(docs) == 0 { return nil } @@ -231,20 +220,24 @@ func (c *Client) InsertBatchRaw(ctx context.Context, collection string, docs []j } func (c *Client) QueryRows(ctx context.Context, query string, batchSize int, bindVars map[string]interface{}, visit RowVisitor) error { + if err := c.checkOpen(); err != nil { + return err + } return queryRows(ctx, c.db, query, batchSize, bindVars, visit) } -// QueryRowsWithMaxRuntime executes a row query with an ArangoDB server-side -// runtime limit. It is intended for bounded previews whose client request may -// be canceled while the server is still evaluating the query. func (c *Client) QueryRowsWithMaxRuntime(ctx context.Context, query string, batchSize int, bindVars map[string]interface{}, maxRuntime time.Duration, visit RowVisitor) error { + if err := c.checkOpen(); err != nil { + return err + } return queryRowsWithMaxRuntime(ctx, c.db, query, batchSize, bindVars, maxRuntime, visit) } -// CollectionRevision returns ArangoDB's collection data revision. Callers can -// use it to validate cached read results against intervening document writes. func (c *Client) CollectionRevision(ctx context.Context, name string) (string, error) { - if c == nil || c.db == nil || strings.TrimSpace(name) == "" { + if err := c.checkOpen(); err != nil { + return "", err + } + if c.db == nil || strings.TrimSpace(name) == "" { return "", fmt.Errorf("Arango collection revision requires a client and collection") } collection, err := c.db.GetCollection(ctx, name, nil) @@ -261,10 +254,11 @@ func (c *Client) CollectionRevision(ctx context.Context, name string) (string, e return properties.Revision, nil } -// CollectionCount returns Arango's collection cardinality metadata without -// scanning its documents. func (c *Client) CollectionCount(ctx context.Context, name string) (int64, error) { - if c == nil || c.db == nil || strings.TrimSpace(name) == "" { + if err := c.checkOpen(); err != nil { + return 0, err + } + if c.db == nil || strings.TrimSpace(name) == "" { return 0, fmt.Errorf("Arango collection count requires a client and collection") } collection, err := c.db.GetCollection(ctx, name, nil) @@ -312,6 +306,9 @@ func queryRowsWithMaxRuntime(ctx context.Context, queryer driver.DatabaseQuery, } func (c *Client) WithTransaction(ctx context.Context, collections TransactionCollections, fn TransactionFunc) error { + if err := c.checkOpen(); err != nil { + return err + } if fn == nil { return fmt.Errorf("Arango transaction callback is required") } @@ -337,6 +334,7 @@ func queryRows(ctx context.Context, queryer driver.DatabaseQuery, query string, } func queryRowsWithOptions(ctx context.Context, queryer driver.DatabaseQuery, query string, options *driver.QueryOptions, visit RowVisitor, closeTimeout time.Duration) (resultErr error) { + defer func() { resultErr = translateQueryErrorAtSeam(resultErr) }() if err := ctx.Err(); err != nil { return err } @@ -391,13 +389,16 @@ func queryRowsWithOptions(ctx context.Context, queryer driver.DatabaseQuery, que } func (c *Client) CollectionExists(ctx context.Context, name string) (bool, error) { + if err := c.checkOpen(); err != nil { + return false, err + } return c.db.CollectionExists(ctx, name) } -// ExecuteAQL runs a write/query statement and drains its cursor. It is kept -// deliberately small so durable registries can perform compare-and-swap -// updates without exposing the Arango driver through their public APIs. func (c *Client) ExecuteAQL(ctx context.Context, query string, bindVars map[string]interface{}) error { + if err := c.checkOpen(); err != nil { + return err + } cursor, err := c.db.Query(ctx, query, &driver.QueryOptions{BatchSize: 32, BindVars: bindVars}) if err != nil { return err @@ -416,6 +417,17 @@ func (c *Client) ExecuteAQL(ctx context.Context, query string, bindVars map[stri } func (c *Client) Close(ctx context.Context) error { + if c == nil { + return nil + } + if c.closed.CompareAndSwap(false, true) { + if c.client != nil { + c.client.CloseIdleConnections() + } + if c.transport != nil { + c.transport.CloseIdleConnections() + } + } return nil } diff --git a/internal/store/arango/client_query_memory_limit_test.go b/internal/backends/arango/internal/store/client_query_memory_limit_test.go similarity index 100% rename from internal/store/arango/client_query_memory_limit_test.go rename to internal/backends/arango/internal/store/client_query_memory_limit_test.go diff --git a/internal/store/arango/client_queryrows_test.go b/internal/backends/arango/internal/store/client_queryrows_test.go similarity index 93% rename from internal/store/arango/client_queryrows_test.go rename to internal/backends/arango/internal/store/client_queryrows_test.go index 700d25c58..9e590a627 100644 --- a/internal/store/arango/client_queryrows_test.go +++ b/internal/backends/arango/internal/store/client_queryrows_test.go @@ -149,27 +149,27 @@ func TestIsQueryMemoryLimitExceededRecognizesWrappedArangoResourceLimit(t *testi } err := fmt.Errorf("arango query: %w", driverErr) - if !IsQueryMemoryLimitExceeded(err) { - t.Fatalf("IsQueryMemoryLimitExceeded(%v) = false, want true", err) + if !isQueryMemoryLimitExceeded(err) { + t.Fatalf("isQueryMemoryLimitExceeded(%v) = false, want true", err) } - if !IsQueryResourceLimitExceeded(err) { - t.Fatalf("IsQueryResourceLimitExceeded(%v) = false, want true", err) + if !isQueryResourceLimitExceeded(err) { + t.Fatalf("isQueryResourceLimitExceeded(%v) = false, want true", err) } - if IsQueryMemoryLimitExceeded(errors.New("unrelated query failure")) { + if isQueryMemoryLimitExceeded(errors.New("unrelated query failure")) { t.Fatal("unrelated query failure was classified as a memory-limit error") } outOfMemory := shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrOutOfMemory, ErrorMessage: "out of memory"} - if IsQueryMemoryLimitExceeded(outOfMemory) { + if isQueryMemoryLimitExceeded(outOfMemory) { t.Fatal("database out-of-memory was classified as a configured query-memory limit") } - if !IsQueryOutOfMemory(outOfMemory) { + if !isQueryOutOfMemory(outOfMemory) { t.Fatal("database out-of-memory was not recognized") } otherResource := shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, ErrorMessage: "resource limit exceeded"} - if IsQueryMemoryLimitExceeded(otherResource) { + if isQueryMemoryLimitExceeded(otherResource) { t.Fatal("non-memory resource limit was classified as a memory limit") } - if !IsQueryResourceLimitExceeded(otherResource) { + if !isQueryResourceLimitExceeded(otherResource) { t.Fatal("non-memory resource limit was not recognized") } } @@ -179,10 +179,10 @@ func TestIsQueryUserAssertionRecognizesOnlyTheRequestedStableCode(t *testing.T) HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: RELATIONSHIP_CARDINALITY_VIOLATION (while executing)", }) - if !IsQueryUserAssertion(err, "RELATIONSHIP_CARDINALITY_VIOLATION") { + if !isQueryUserAssertion(err, "RELATIONSHIP_CARDINALITY_VIOLATION") { t.Fatal("stable user assertion was not recognized") } - if IsQueryUserAssertion(err, "SOME_OTHER_ASSERTION") || IsQueryUserAssertion(errors.New("RELATIONSHIP_CARDINALITY_VIOLATION"), "RELATIONSHIP_CARDINALITY_VIOLATION") { + if isQueryUserAssertion(err, "SOME_OTHER_ASSERTION") || isQueryUserAssertion(errors.New("RELATIONSHIP_CARDINALITY_VIOLATION"), "RELATIONSHIP_CARDINALITY_VIOLATION") { t.Fatal("unrelated error was classified as the requested user assertion") } } diff --git a/internal/store/arango/client_revision_test.go b/internal/backends/arango/internal/store/client_revision_test.go similarity index 100% rename from internal/store/arango/client_revision_test.go rename to internal/backends/arango/internal/store/client_revision_test.go diff --git a/internal/backends/arango/internal/store/errors.go b/internal/backends/arango/internal/store/errors.go new file mode 100644 index 000000000..06ae976a8 --- /dev/null +++ b/internal/backends/arango/internal/store/errors.go @@ -0,0 +1,140 @@ +package arango + +import ( + "errors" + "fmt" + "strings" + + "github.com/arangodb/go-driver/v2/arangodb/shared" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" +) + +func isQueryMemoryLimitExceeded(err error) bool { + if !isQueryResourceLimitExceeded(err) { + return false + } + ok, arangoErr := shared.IsArangoError(err) + return ok && strings.Contains(strings.ToLower(arangoErr.ErrorMessage), "memory") +} + +func isQueryResourceLimitExceeded(err error) bool { + return shared.IsArangoErrorWithErrorNum(err, shared.ErrResourceLimit) +} + +func isQueryOutOfMemory(err error) bool { + return shared.IsArangoErrorWithErrorNum(err, shared.ErrOutOfMemory) +} + +func isQueryUserAssertion(err error, code string) bool { + if strings.TrimSpace(code) == "" { + return false + } + ok, arangoErr := shared.IsArangoError(err) + return ok && arangoErr.ErrorNum == shared.ErrQueryUserAssert && strings.Contains(arangoErr.ErrorMessage, "AQL: "+code) +} + +func TranslateQueryError(err error) (translated error, ok bool) { + if err == nil { + return nil, false + } + if _, done := dataframeerrors.AsUserError(err); done { + return err, true + } + switch { + case isQueryUserAssertion(err, string(dataframeerrors.CodeTablePivotCellCardinality)): + return dataframeerrors.Wrap(err, dataframeerrors.CodeTablePivotCellCardinality, ""), true + case isQueryUserAssertion(err, string(dataframeerrors.CodeTablePivotUnlistedCategory)): + return dataframeerrors.Wrap(err, dataframeerrors.CodeTablePivotUnlistedCategory, ""), true + case isQueryUserAssertion(err, string(dataframeerrors.CodeConstructionRowValueMultipleValues)): + return dataframeerrors.Wrap(err, dataframeerrors.CodeConstructionRowValueMultipleValues, ""), true + case isQueryUserAssertion(err, string(dataframeerrors.CodeConstructionExpansionEmpty)): + return dataframeerrors.Wrap(err, dataframeerrors.CodeConstructionExpansionEmpty, ""), true + case isQueryUserAssertion(err, string(dataframeerrors.CodeExplicitGroupUnassignedMember)): + return dataframeerrors.Wrap(err, dataframeerrors.CodeExplicitGroupUnassignedMember, ""), true + case isQueryUserAssertion(err, string(dataframeerrors.CodeRelationshipCardinalityViolation)): + return dataframeerrors.Wrap(err, dataframeerrors.CodeRelationshipCardinalityViolation, ""), true + case isQueryUserAssertion(err, string(dataframeerrors.CodeTemporalAnchorInvalid)): + return dataframeerrors.Wrap(err, dataframeerrors.CodeTemporalAnchorInvalid, ""), true + case isQueryUserAssertion(err, string(dataframeerrors.CodeTemporalPrecisionUnsupported)): + return dataframeerrors.Wrap(err, dataframeerrors.CodeTemporalPrecisionUnsupported, ""), true + case isQueryUserAssertion(err, string(dataframeerrors.CodeTemporalTieAmbiguous)): + return dataframeerrors.Wrap(err, dataframeerrors.CodeTemporalTieAmbiguous, ""), true + case isQueryUserAssertion(err, string(dataframeerrors.CodeUnitIdentityUnknown)): + return dataframeerrors.Wrap(err, dataframeerrors.CodeUnitIdentityUnknown, ""), true + case isQueryUserAssertion(err, string(dataframeerrors.CodeUnitDimensionIncompatible)): + return dataframeerrors.Wrap(err, dataframeerrors.CodeUnitDimensionIncompatible, ""), true + case isQueryMemoryLimitExceeded(err): + return dataframeerrors.Wrap( + err, + dataframeerrors.CodeQueryMemoryLimitExceeded, + "", + dataframeerrors.WithDetails(map[string]any{"backend": "arangodb", "resource": "query_memory"}), + ), true + case isQueryResourceLimitExceeded(err): + return dataframeerrors.Wrap( + err, + dataframeerrors.CodeQueryResourceLimitExceeded, + "", + dataframeerrors.WithDetails(map[string]any{"backend": "arangodb", "resource": "query"}), + ), true + case isQueryOutOfMemory(err): + return dataframeerrors.Wrap( + err, + dataframeerrors.CodeQueryBackendOutOfMemory, + "", + dataframeerrors.WithDetails(map[string]any{"backend": "arangodb", "resource": "memory"}), + ), true + default: + return err, false + } +} + +func translateQueryErrorAtSeam(err error) error { + translated, ok := TranslateQueryError(err) + if !ok { + return err + } + return translated +} + +func arangoErrorFrom(err error) (shared.ArangoError, bool) { + if err == nil { + return shared.ArangoError{}, false + } + var value shared.ArangoError + if errors.As(err, &value) { + return value, true + } + var pointer *shared.ArangoError + if errors.As(err, &pointer) && pointer != nil { + return *pointer, true + } + return shared.ArangoError{}, false +} + +func IsArangoErrorResponse(err error) bool { + _, ok := arangoErrorFrom(err) + return ok +} + +func IsArangoPermissionResponse(err error) bool { + arangoErr, ok := arangoErrorFrom(err) + if !ok { + return false + } + return arangoErr.Code == shared.ErrHttpUnauthorized || + arangoErr.Code == shared.ErrHttpForbidden || + arangoErr.ErrorNum == shared.ErrForbidden +} + +func DiagnosticCause(err error) (string, bool) { + arangoErr, ok := arangoErrorFrom(err) + if !ok { + return "", false + } + cause := fmt.Sprintf("ArangoDB error number %d (HTTP %d)", arangoErr.ErrorNum, arangoErr.Code) + if arangoErr.ErrorMessage != "" { + cause += ": " + arangoErr.ErrorMessage + } + return cause, true +} diff --git a/internal/backends/arango/internal/store/errors_test.go b/internal/backends/arango/internal/store/errors_test.go new file mode 100644 index 000000000..b201ee2a8 --- /dev/null +++ b/internal/backends/arango/internal/store/errors_test.go @@ -0,0 +1,353 @@ +package arango + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + driver "github.com/arangodb/go-driver/v2/arangodb" + "github.com/arangodb/go-driver/v2/arangodb/shared" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" +) + +type translateGolden struct { + name string + driverErr shared.ArangoError + wantCode string + wantPublic string + wantDetails map[string]any +} + +var translateGoldens = []translateGolden{ + { + name: "pivot_cell_cardinality", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: TABLE_PIVOT_CELL_CARDINALITY (while executing)"}, + wantCode: "TABLE_PIVOT_CELL_CARDINALITY", + wantPublic: "More than one record matched a Pivot cell; choose how to handle duplicates or filter the input rows.", + }, + { + name: "pivot_unlisted_category", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: TABLE_PIVOT_UNLISTED_CATEGORY (while executing)"}, + wantCode: "TABLE_PIVOT_UNLISTED_CATEGORY", + wantPublic: "Pivot found an unlisted category; select all discovered categories or filter rows before Pivot.", + }, + { + name: "grouped_row_multiple_values", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: CONSTRUCTION_ROW_VALUE_MULTIPLE_VALUES (while executing)"}, + wantCode: "CONSTRUCTION_ROW_VALUE_MULTIPLE_VALUES", + wantPublic: "Some grouped records have different values for this column. Keep all distinct values or choose more specific grouping fields.", + }, + { + name: "construction_expansion_empty", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: CONSTRUCTION_EXPANSION_EMPTY: related expansion construction related_1 has no related records for row row-17 (while executing)"}, + wantCode: "CONSTRUCTION_EXPANSION_EMPTY", + wantPublic: "At least one row has an empty list or no matching related records. Choose 'Drop the original row' or 'Keep the row with a missing item' for lists, or 'Leave that current row out' or 'Keep that current row once, with no related record ID' for related records.", + }, + { + name: "explicit_group_unassigned_member", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: EXPLICIT_GROUP_UNASSIGNED_MEMBER (while executing)"}, + wantCode: "EXPLICIT_GROUP_UNASSIGNED_MEMBER", + wantPublic: "Some records do not belong to a group. Choose 'Leave out records without a group' or 'Put records without a group in their own group', or assign them to a group.", + }, + { + name: "relationship_cardinality_violation", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: RELATIONSHIP_CARDINALITY_VIOLATION (while executing)"}, + wantCode: "RELATIONSHIP_CARDINALITY_VIOLATION", + wantPublic: "more than one value matched a feature that requires zero or one", + }, + { + name: "temporal_anchor_invalid", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: TEMPORAL_ANCHOR_INVALID (while executing)"}, + wantCode: "TEMPORAL_ANCHOR_INVALID", + wantPublic: "the row date required by a date-aware feature is missing or is not a complete instant", + }, + { + name: "temporal_precision_unsupported", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: TEMPORAL_PRECISION_UNSUPPORTED (while executing)"}, + wantCode: "TEMPORAL_PRECISION_UNSUPPORTED", + wantPublic: "a record date required by a date-aware feature is missing or is not a complete instant", + }, + { + name: "temporal_tie_ambiguous", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: TEMPORAL_TIE_AMBIGUOUS (while executing)"}, + wantCode: "TEMPORAL_TIE_AMBIGUOUS", + wantPublic: "multiple values share the selected date; choose how equal dates should be resolved", + }, + { + name: "unit_identity_unknown", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: UNIT_IDENTITY_UNKNOWN (while executing)"}, + wantCode: "UNIT_IDENTITY_UNKNOWN", + wantPublic: "a measurement has no approved source unit identity", + }, + { + name: "unit_dimension_incompatible", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: UNIT_DIMENSION_INCOMPATIBLE (while executing)"}, + wantCode: "UNIT_DIMENSION_INCOMPATIBLE", + wantPublic: "a measurement unit is dimensionally incompatible with the feature target", + }, + { + name: "query_memory_limit", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, ErrorMessage: "AQL: query would use more memory than allowed"}, + wantCode: "QUERY_MEMORY_LIMIT_EXCEEDED", + wantPublic: "the dataframe query exceeded the database's configured memory limit", + wantDetails: map[string]any{"backend": "arangodb", "resource": "query_memory"}, + }, + { + name: "query_resource_limit", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, ErrorMessage: "resource limit exceeded"}, + wantCode: "QUERY_RESOURCE_LIMIT_EXCEEDED", + wantPublic: "the dataframe query exceeded a configured database resource limit", + wantDetails: map[string]any{"backend": "arangodb", "resource": "query"}, + }, + { + name: "backend_out_of_memory", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrOutOfMemory, ErrorMessage: "out of memory"}, + wantCode: "QUERY_BACKEND_OUT_OF_MEMORY", + wantPublic: "the dataframe database ran out of memory while executing the query", + wantDetails: map[string]any{"backend": "arangodb", "resource": "memory"}, + }, +} + +func TestTranslateQueryErrorGoldenTablePreservesPreMoveSemantics(t *testing.T) { + if len(translateGoldens) != 14 { + t.Fatalf("golden table has %d branches, want 14 (11 assertions + 3 resource limits)", len(translateGoldens)) + } + for _, golden := range translateGoldens { + t.Run(golden.name, func(t *testing.T) { + wrapped := fmt.Errorf("arango query: %w", golden.driverErr) + translated, ok := TranslateQueryError(wrapped) + if !ok { + t.Fatalf("TranslateQueryError(%v) = ok false, want classified", golden.driverErr) + } + userErr, isUser := dataframeerrors.AsUserError(translated) + if !isUser { + t.Fatalf("translated error is not a Loom user error: %v", translated) + } + if userErr.Code() != golden.wantCode { + t.Fatalf("code = %q, want %q", userErr.Code(), golden.wantCode) + } + if got := dataframeerrors.PublicMessage(translated); got != golden.wantPublic { + t.Fatalf("public message = %q, want %q", got, golden.wantPublic) + } + if userErr.Retryable() { + t.Fatal("classified error must not be retryable") + } + if golden.wantDetails == nil { + if len(userErr.Details()) != 0 { + t.Fatalf("details = %v, want none", userErr.Details()) + } + } else if !reflect.DeepEqual(userErr.Details(), golden.wantDetails) { + t.Fatalf("details = %v, want %v", userErr.Details(), golden.wantDetails) + } + if !errors.Is(translated, golden.driverErr) { + t.Fatal("driver cause lost through errors.Is") + } + var fromChain shared.ArangoError + if !errors.As(translated, &fromChain) || fromChain.ErrorNum != golden.driverErr.ErrorNum { + t.Fatalf("driver cause lost through errors.As: %v", fromChain) + } + }) + } +} + +func TestTranslateQueryErrorNeverInfersDomainCodeFromUnrecognizedAssert(t *testing.T) { + unrecognized := shared.ArangoError{ + HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, + ErrorMessage: "AQL: row expansion construction expand_tags has no items for row row-17 (while executing)", + } + if got, ok := TranslateQueryError(unrecognized); ok || got != error(unrecognized) { + t.Fatalf("TranslateQueryError() = (%v, %t), want unrecognized ASSERT returned untouched", got, ok) + } + textOnly := errors.New("AQL: RELATIONSHIP_CARDINALITY_VIOLATION") + if got, ok := TranslateQueryError(textOnly); ok || got != error(textOnly) { + t.Fatalf("text-only error classified: (%v, %t)", got, ok) + } +} + +func TestTranslateQueryErrorIdempotenceAndPassThrough(t *testing.T) { + t.Run("nil", func(t *testing.T) { + if translated, ok := TranslateQueryError(nil); translated != nil || ok { + t.Fatalf("TranslateQueryError(nil) = (%v, %t), want (nil, false)", translated, ok) + } + }) + t.Run("existing_user_error_untouched", func(t *testing.T) { + userErr := dataframeerrors.NewError(dataframeerrors.CodeInvalidLimit, "") + translated, ok := TranslateQueryError(userErr) + if !ok || translated != error(userErr) { + t.Fatalf("existing user error rewritten: (%v, %t)", translated, ok) + } + }) + t.Run("unrecognized_identity_preserved", func(t *testing.T) { + want := errors.New("visitor failed") + translated, ok := TranslateQueryError(want) + if ok || translated != want { + t.Fatalf("unrecognized error changed: (%v, %t)", translated, ok) + } + }) + t.Run("context_sentinels_pass_through", func(t *testing.T) { + for _, sentinel := range []error{context.Canceled, context.DeadlineExceeded} { + if translated, ok := TranslateQueryError(sentinel); ok || translated != sentinel { + t.Fatalf("context sentinel %v classified: (%v, %t)", sentinel, translated, ok) + } + wrapped := fmt.Errorf("wrap: %w", sentinel) + translated, ok := TranslateQueryError(wrapped) + if ok || !errors.Is(translated, sentinel) { + t.Fatalf("wrapped context sentinel changed identity: (%v, %t)", translated, ok) + } + } + }) + t.Run("double_application_stable", func(t *testing.T) { + driverErr := shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, ErrorMessage: "AQL: query would use more memory than allowed"} + once, ok := TranslateQueryError(fmt.Errorf("arango query: %w", driverErr)) + if !ok { + t.Fatal("first classification failed") + } + twice, ok := TranslateQueryError(once) + if !ok || twice != once { + t.Fatalf("double application changed the error: %#v vs %#v", twice, once) + } + }) +} + +func resourceLimitDriverError() shared.ArangoError { + return shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, ErrorMessage: "AQL: query would use more memory than allowed"} +} + +func wantClassifiedResourceLimit(t *testing.T, err error, path string) { + t.Helper() + if err == nil { + t.Fatalf("%s: error lost", path) + } + userErr, ok := dataframeerrors.AsUserError(err) + if !ok || userErr.Code() != string(dataframeerrors.CodeQueryMemoryLimitExceeded) { + t.Fatalf("%s: error %#v not classified by the adapter seam", path, err) + } + if !errors.Is(err, resourceLimitDriverError()) { + t.Fatalf("%s: driver cause lost through errors.Is", path) + } +} + +func TestQueryRowsClassifiesAtSeam(t *testing.T) { + queryer := &fakeQueryer{err: fmt.Errorf("driver: %w", resourceLimitDriverError())} + err := queryRows(context.Background(), queryer, "RETURN 1", 1, nil, func(map[string]any) error { return nil }) + wantClassifiedResourceLimit(t, err, "QueryRows") +} + +func TestQueryRowsWithMaxRuntimeClassifiesAtSeam(t *testing.T) { + queryer := &fakeQueryer{err: fmt.Errorf("driver: %w", resourceLimitDriverError())} + err := queryRowsWithMaxRuntime(context.Background(), queryer, "RETURN 1", 1, nil, time.Second, func(map[string]any) error { return nil }) + wantClassifiedResourceLimit(t, err, "QueryRowsWithMaxRuntime") +} + +type fakeDatabase struct { + driver.Database + queryer driver.DatabaseQuery +} + +func (d fakeDatabase) Query(ctx context.Context, _ string, options *driver.QueryOptions) (driver.Cursor, error) { + return d.queryer.Query(ctx, "", options) +} + +func TestTransactionClientQueryRowsClassifiesAtSeam(t *testing.T) { + queryer := &fakeQueryer{err: fmt.Errorf("driver: %w", resourceLimitDriverError())} + client := fakeDatabase{queryer: queryer} + transaction := transactionClient{queryer: client} + err := transaction.QueryRows(context.Background(), "RETURN 1", 1, nil, func(map[string]any) error { return nil }) + wantClassifiedResourceLimit(t, err, "transaction RowQueryer.QueryRows") +} + +func TestClientQueryRowsClassifiesThroughExportedSurface(t *testing.T) { + queryer := &fakeQueryer{err: fmt.Errorf("driver: %w", resourceLimitDriverError())} + client := &Client{db: fakeDatabase{queryer: queryer}} + err := client.QueryRows(context.Background(), "RETURN 1", 1, nil, func(map[string]any) error { return nil }) + wantClassifiedResourceLimit(t, err, "Client.QueryRows") +} + +func TestClientQueryRowsWithMaxRuntimeClassifiesThroughExportedSurface(t *testing.T) { + queryer := &fakeQueryer{err: fmt.Errorf("driver: %w", resourceLimitDriverError())} + client := &Client{db: fakeDatabase{queryer: queryer}} + err := client.QueryRowsWithMaxRuntime(context.Background(), "RETURN 1", 1, nil, time.Second, func(map[string]any) error { return nil }) + wantClassifiedResourceLimit(t, err, "Client.QueryRowsWithMaxRuntime") +} + +func TestQueryErrorBoundaryJoinsIncludeClassification(t *testing.T) { + readErr := resourceLimitDriverError() + cursor := &fakeCursor{rows: []map[string]any{{"id": "1"}}, readErr: fmt.Errorf("read: %w", readErr), closeErr: errors.New("close failed")} + queryer := &fakeQueryer{cursor: cursor} + err := queryRows(context.Background(), queryer, "RETURN 1", 1, nil, func(map[string]any) error { return nil }) + wantClassifiedResourceLimit(t, err, "joined read+close") + if !errors.Is(err, cursor.closeErr) { + t.Fatal("close failure lost") + } +} + +func TestTranslateQueryErrorNeverLeaksVendorMessageIntoPublicError(t *testing.T) { + driverErr := shared.ArangoError{ + HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, + ErrorMessage: `AQL: memory limit exceeded while binding {"secret":"hunter2","bind":"RETURN SECRET_DO_NOT_RENDER"}`, + } + translated, ok := TranslateQueryError(fmt.Errorf("arango query: %w", driverErr)) + if !ok { + t.Fatal("driver failure not classified") + } + encoded, err := json.Marshal(translated) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"hunter2", "SECRET_DO_NOT_RENDER", "RETURN", "bind"} { + if strings.Contains(string(encoded), forbidden) || strings.Contains(translated.Error(), forbidden) { + t.Fatalf("public rendering leaked %q: %s / %v", forbidden, encoded, translated) + } + } +} + +func TestDiagnosticCauseGoldenStrings(t *testing.T) { + t.Run("value_with_message", func(t *testing.T) { + err := fmt.Errorf("arango query: %w", shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, ErrorMessage: "boom"}) + cause, ok := DiagnosticCause(err) + if !ok { + t.Fatal("DiagnosticCause missed the driver error") + } + if want := "ArangoDB error number 32 (HTTP 500): boom"; cause != want { + t.Fatalf("cause = %q, want %q", cause, want) + } + }) + t.Run("pointer_without_message", func(t *testing.T) { + driverErr := &shared.ArangoError{HasError: true, Code: 400, ErrorNum: 1023} + cause, ok := DiagnosticCause(fmt.Errorf("wrapped: %w", driverErr)) + if !ok { + t.Fatal("pointer driver error missed") + } + if want := "ArangoDB error number 1023 (HTTP 400)"; cause != want { + t.Fatalf("cause = %q, want %q", cause, want) + } + }) + t.Run("typed_nil_pointer_absent", func(t *testing.T) { + if _, ok := DiagnosticCause(fmt.Errorf("wrapped: %w", (*shared.ArangoError)(nil))); ok { + t.Fatal("typed-nil pointer must not yield a diagnostic cause") + } + }) + t.Run("non_arango_absent", func(t *testing.T) { + if _, ok := DiagnosticCause(errors.New("foreign failure")); ok { + t.Fatal("foreign error yielded an ArangoDB diagnostic cause") + } + if _, ok := DiagnosticCause(nil); ok { + t.Fatal("nil yielded a diagnostic cause") + } + }) +} + +func TestQueryMemoryLimitFromContextRoundTrips(t *testing.T) { + if _, ok := QueryMemoryLimitFromContext(context.Background()); ok { + t.Fatal("unset context reported a memory limit") + } + ctx := WithQueryMemoryLimit(context.Background(), 42) + if limit, ok := QueryMemoryLimitFromContext(ctx); !ok || limit != 42 { + t.Fatalf("limit = %d/%t, want 42/true", limit, ok) + } +} diff --git a/internal/store/arango/explain.go b/internal/backends/arango/internal/store/explain.go similarity index 82% rename from internal/store/arango/explain.go rename to internal/backends/arango/internal/store/explain.go index 4c3ab6d63..f6c52fb52 100644 --- a/internal/store/arango/explain.go +++ b/internal/backends/arango/internal/store/explain.go @@ -8,8 +8,6 @@ import ( "sort" ) -// ExplainRequest is the portable JSON body accepted by ArangoDB's AQL -// explain endpoint. It intentionally has no dependency on the Arango driver. type ExplainRequest struct { Query string `json:"query"` BindVars map[string]any `json:"bindVars,omitempty"` @@ -26,7 +24,6 @@ type OptimizerOptions struct { Rules []string `json:"rules,omitempty"` } -// ExplainResult models both single-plan and all-plans responses. type ExplainResult struct { Plan *ExplainPlan `json:"plan,omitempty"` Plans []ExplainPlan `json:"plans,omitempty"` @@ -56,9 +53,6 @@ type ExplainNode struct { IndexCoversOutProjections *bool `json:"indexCoversOutProjections,omitempty"` } -// ExplainIndexes accepts the shapes emitted by different ArangoDB plan nodes. -// Index nodes use an array, while traversal and optimizer nodes can emit a -// single object or an object keyed by an internal index role. type ExplainIndexes []ExplainIndex func (indexes *ExplainIndexes) UnmarshalJSON(data []byte) error { @@ -75,10 +69,6 @@ func (indexes *ExplainIndexes) UnmarshalJSON(data []byte) error { return nil } -// decodeExplainIndexes accepts every index shape emitted by the explain API: -// a direct index object, an array, or nested traversal-index containers such -// as {"base": [...], "levels": {...}}. Nested non-index metadata is ignored; -// a malformed top-level indexes value still returns an error. func decodeExplainIndexes(data []byte, strict bool) ([]ExplainIndex, error) { data = bytes.TrimSpace(data) if len(data) == 0 || bytes.Equal(data, []byte("null")) { @@ -166,8 +156,6 @@ type ExplainStats struct { PeakMemoryUsage uint64 `json:"peakMemoryUsage,omitempty"` } -// ExplainIndexUse associates an index with the plan node and collection that -// use it. Collection falls back to the node collection when omitted by Arango. type ExplainIndexUse struct { Plan int NodeID int64 @@ -184,8 +172,6 @@ type explainEnvelope struct { Code int `json:"code,omitempty"` } -// ParseExplainResult decodes an Arango explain response, rejects trailing JSON, -// and promotes Arango error envelopes to Go errors. func ParseExplainResult(data []byte) (ExplainResult, error) { decoder := json.NewDecoder(bytes.NewReader(data)) var envelope explainEnvelope @@ -204,13 +190,6 @@ func ParseExplainResult(data []byte) (ExplainResult, error) { return envelope.ExplainResult, nil } -// ExtractPlanIndexes returns deterministic, deduplicated index uses from the -// single plan followed by any alternative plans. - -// explainIndexCollection resolves the collection that owns an index across -// Arango's node shapes. Traversal nodes typically omit `collection` and put -// the edge collection in `edgeCollections`, while ordinary IndexNodes use -// `collection` directly. func explainIndexCollection(node ExplainNode, index ExplainIndex) string { if index.Collection != "" { return index.Collection diff --git a/internal/store/arango/explain_assessment.go b/internal/backends/arango/internal/store/explain_assessment.go similarity index 86% rename from internal/store/arango/explain_assessment.go rename to internal/backends/arango/internal/store/explain_assessment.go index c051f9fd3..19639b778 100644 --- a/internal/store/arango/explain_assessment.go +++ b/internal/backends/arango/internal/store/explain_assessment.go @@ -5,9 +5,6 @@ import ( "strings" ) -// ExplainAssessment is a deterministic compiler-facing summary of one Arango -// explain response. Plan 0 is the primary plan when Result.Plan is present; -// alternative plans follow in response order. type ExplainAssessment struct { Plans []ExplainPlanEstimate FullCollectionScans []ExplainCollectionScan @@ -22,17 +19,12 @@ type ExplainPlanEstimate struct { EstimatedNrItems float64 } -// ExplainCollectionScan records an EnumerateCollectionNode. Such a node is a -// full collection enumeration indicator even when a different node elsewhere -// in the plan uses an index. type ExplainCollectionScan struct { Plan int NodeID int64 Collection string } -// ExplainIndexSummary groups equivalent indexes across plan nodes while -// retaining every plan/node use. type ExplainIndexSummary struct { Collection string ID string @@ -47,8 +39,6 @@ type ExplainIndexLocation struct { NodeID int64 } -// AssessExplainResult converts parsed explain data into stable findings without -// issuing queries or depending on an Arango client. func AssessExplainResult(result ExplainResult) ExplainAssessment { plans := explainPlans(result) assessment := ExplainAssessment{ diff --git a/internal/store/arango/explain_assessment_test.go b/internal/backends/arango/internal/store/explain_assessment_test.go similarity index 100% rename from internal/store/arango/explain_assessment_test.go rename to internal/backends/arango/internal/store/explain_assessment_test.go diff --git a/internal/store/arango/explain_client.go b/internal/backends/arango/internal/store/explain_client.go similarity index 88% rename from internal/store/arango/explain_client.go rename to internal/backends/arango/internal/store/explain_client.go index 1fc9a5a0c..19f8f82c9 100644 --- a/internal/store/arango/explain_client.go +++ b/internal/backends/arango/internal/store/explain_client.go @@ -11,10 +11,10 @@ import ( "strings" ) -// Explain submits a typed AQL explain request to this client's database. It is -// intentionally separate from QueryRows so compiler tooling can inspect a plan -// without executing a dataframe query. func (c *Client) Explain(ctx context.Context, request ExplainRequest) (ExplainResult, error) { + if err := c.checkOpen(); err != nil { + return ExplainResult{}, err + } if strings.TrimSpace(request.Query) == "" { return ExplainResult{}, fmt.Errorf("AQL explain query is required") } diff --git a/internal/store/arango/explain_client_test.go b/internal/backends/arango/internal/store/explain_client_test.go similarity index 100% rename from internal/store/arango/explain_client_test.go rename to internal/backends/arango/internal/store/explain_client_test.go diff --git a/internal/store/arango/explain_test.go b/internal/backends/arango/internal/store/explain_test.go similarity index 100% rename from internal/store/arango/explain_test.go rename to internal/backends/arango/internal/store/explain_test.go diff --git a/internal/store/arango/options.go b/internal/backends/arango/internal/store/options.go similarity index 100% rename from internal/store/arango/options.go rename to internal/backends/arango/internal/store/options.go diff --git a/internal/store/arango/population_profile_integration_test.go b/internal/backends/arango/internal/store/population_profile_integration_test.go similarity index 96% rename from internal/store/arango/population_profile_integration_test.go rename to internal/backends/arango/internal/store/population_profile_integration_test.go index 5868bc999..fd10c302b 100644 --- a/internal/store/arango/population_profile_integration_test.go +++ b/internal/backends/arango/internal/store/population_profile_integration_test.go @@ -31,10 +31,6 @@ type populationProfileCase struct { MembershipDriven ProfileSummary `json:"membershipDriven"` } -// TestPopulationPlanProfilesAgainstArango compares the former target-scan -// semijoin with the membership-driven root access path used by the compiler. -// It owns uniquely named collections and is opt-in because it inserts a -// synthetic corpus into a disposable development database. func TestPopulationPlanProfilesAgainstArango(t *testing.T) { if os.Getenv("LOOM_POPULATION_PROFILE") == "" { t.Skip("set LOOM_POPULATION_PROFILE=1 to run population plan profiling") diff --git a/internal/store/arango/preview_index.go b/internal/backends/arango/internal/store/preview_index.go similarity index 89% rename from internal/store/arango/preview_index.go rename to internal/backends/arango/internal/store/preview_index.go index c0ad2ea7d..cd7c8bdd9 100644 --- a/internal/store/arango/preview_index.go +++ b/internal/backends/arango/internal/store/preview_index.go @@ -21,8 +21,6 @@ const previewCoveringIndexCleanupTimeout = 3 * time.Second var ErrPreviewCoveringIndexLimit = errors.New("preview covering index limit reached") -// PreviewCoveringIndex is a verified compiler-owned persistent index that -// may be used to cover a bounded preview without creating a new index. type PreviewCoveringIndex struct { Collection string Name string @@ -42,10 +40,11 @@ type previewIndexReplacement struct { storedValues []string } -// ListPreviewCoveringIndexes returns only persistent, non-sparse, -// non-unique indexes owned by the preview compiler for the named collection. func (c *Client) ListPreviewCoveringIndexes(ctx context.Context, collection string) ([]PreviewCoveringIndex, error) { - if c == nil || c.db == nil || collection == "" || !validIndexPath(collection) { + if err := c.checkOpen(); err != nil { + return nil, err + } + if c.db == nil || collection == "" || !validIndexPath(collection) { return nil, fmt.Errorf("invalid preview covering collection") } col, err := c.db.GetCollection(ctx, collection, nil) @@ -81,44 +80,43 @@ func previewCoveringIndexDefinitions(collection string, indexes []driver.IndexRe return definitions } -// EnsurePreviewCoveringIndex adds one bounded, compiler-selected covering index. -// The caller's AQL hint is non-forcing, so previews continue on the canonical -// plan if index preparation is unavailable. func (c *Client) EnsurePreviewCoveringIndex(ctx context.Context, collection, name string, fields []string) error { + if err := c.checkOpen(); err != nil { + return err + } return c.ensurePreviewCoveringIndex(ctx, collection, name, fields, nil) } -// EnsurePreviewCoveringIndexWithStoredValues provisions a compiler-owned -// projection index. Stored values are validated and verified as part of the -// exact index definition because Arango's ensure operation does not distinguish -// otherwise identical field tuples by stored-value configuration. func (c *Client) EnsurePreviewCoveringIndexWithStoredValues(ctx context.Context, collection, name string, fields, storedValues []string) error { - if c == nil || !validPreviewCoveringIndexWithStoredValues(collection, name, fields, storedValues) { + if err := c.checkOpen(); err != nil { + return err + } + if !validPreviewCoveringIndexWithStoredValues(collection, name, fields, storedValues) { return fmt.Errorf("invalid preview covering index specification") } return c.ensurePreviewCoveringIndexDefinition(ctx, collection, name, fields, storedValues, nil) } -// EnsurePreviewCoveringIndexWithStoredValuesReplacing provisions a -// compiler-owned stored-values index and replaces only the exact older -// compiler-owned definition supplied by the same plan. func (c *Client) EnsurePreviewCoveringIndexWithStoredValuesReplacing(ctx context.Context, collection, name string, fields, storedValues []string, supersededName string, supersededFields, supersededStoredValues []string) error { replacement := &previewIndexReplacement{ name: supersededName, fields: append([]string(nil), supersededFields...), storedValues: append([]string(nil), supersededStoredValues...), } - if c == nil || !validPreviewCoveringIndexStoredValuesReplacement(collection, name, fields, storedValues, replacement) { + if err := c.checkOpen(); err != nil { + return err + } + if !validPreviewCoveringIndexStoredValuesReplacement(collection, name, fields, storedValues, replacement) { return fmt.Errorf("invalid preview covering index replacement specification") } return c.ensurePreviewCoveringIndexDefinition(ctx, collection, name, fields, storedValues, replacement) } -// EnsurePreviewCoveringIndexReplacing provisions a related-category index and, -// only at the per-collection cap, may replace the exact compiler-named legacy -// category index supplied by the same plan. func (c *Client) EnsurePreviewCoveringIndexReplacing(ctx context.Context, collection, name string, fields []string, supersededName string, supersededFields []string) error { replacement := &previewIndexReplacement{name: supersededName, fields: append([]string(nil), supersededFields...)} - if c == nil || !validPreviewCoveringIndexReplacement(collection, name, fields, replacement) { + if err := c.checkOpen(); err != nil { + return err + } + if !validPreviewCoveringIndexReplacement(collection, name, fields, replacement) { return fmt.Errorf("invalid preview covering index replacement specification") } return c.ensurePreviewCoveringIndex(ctx, collection, name, fields, replacement) @@ -291,8 +289,6 @@ func validPreviewCoveringIndexStoredValuesReplacement(collection, name string, f replacement.name == previewCoveringIndexNameWithStoredValues(collection, replacement.fields, replacement.storedValues) } -// previewCoveringIndexName is kept in lockstep with the compiler's stable -// field-only name derivation so replacement is limited to an exact owned spec. func previewCoveringIndexName(collection string, fields []string) string { digest := sha256.Sum256([]byte(collection + "\x00" + strings.Join(fields, "\x00"))) return previewCoveringIndexPrefix + hex.EncodeToString(digest[:8]) diff --git a/internal/store/arango/preview_index_test.go b/internal/backends/arango/internal/store/preview_index_test.go similarity index 100% rename from internal/store/arango/preview_index_test.go rename to internal/backends/arango/internal/store/preview_index_test.go diff --git a/internal/store/arango/profile.go b/internal/backends/arango/internal/store/profile.go similarity index 82% rename from internal/store/arango/profile.go rename to internal/backends/arango/internal/store/profile.go index 8630b0a57..d6c884a67 100644 --- a/internal/store/arango/profile.go +++ b/internal/backends/arango/internal/store/profile.go @@ -7,9 +7,6 @@ import ( "sort" ) -// ProfileRequest is the parameterized body accepted by ArangoDB's cursor -// endpoint. Profile is intentionally opt-in because profiling adds execution -// overhead and is not appropriate for normal dataframe requests. type ProfileRequest struct { Query string `json:"query"` BindVars map[string]any `json:"bindVars,omitempty"` @@ -23,10 +20,6 @@ type ProfileOptions struct { Optimizer OptimizerOptions `json:"optimizer,omitempty"` } -// ProfileResult is the first cursor response returned by a profiled query. -// ArangoDB places profiling information under extra. Result is retained as -// raw JSON because profile is a diagnostic API and must support arbitrary row -// shapes without coupling this package to dataframe models. type ProfileResult struct { Result []json.RawMessage `json:"result,omitempty"` HasMore bool `json:"hasMore,omitempty"` @@ -70,8 +63,6 @@ type ProfileNode struct { Runtime float64 `json:"runtime,omitempty"` } -// ProfilePhases contains the stable phase names emitted by ArangoDB. New -// server versions may add phases; unknown fields are intentionally ignored. type ProfilePhases struct { Initializing float64 `json:"initializing,omitempty"` Parsing float64 `json:"parsing,omitempty"` @@ -83,8 +74,6 @@ type ProfilePhases struct { Finalizing float64 `json:"finalizing,omitempty"` } -// ProfileNodeSummary is a deterministic compact view suitable for logs and -// benchmark artifacts. Node type comes from the profiled execution plan. type ProfileNodeSummary struct { ID int64 Type string @@ -112,9 +101,6 @@ type ProfileNodeGroup struct { Runtime float64 } -// ParseProfileResult decodes an Arango cursor response and rejects malformed -// or error envelopes. It accepts profile level 1 (phase timings) and level 2 -// (plan plus per-node timings). func ParseProfileResult(data []byte) (ProfileResult, error) { decoder := json.NewDecoder(bytes.NewReader(data)) var result ProfileResult @@ -130,9 +116,6 @@ func ParseProfileResult(data []byte) (ProfileResult, error) { return result, nil } -// SummarizeProfile joins runtime node statistics to the returned execution -// plan and groups them by node type. Ordering is stable: nodes and groups are -// sorted by descending runtime and then by ID/type. func SummarizeProfile(result ProfileResult) ProfileSummary { summary := ProfileSummary{ ScannedFull: result.Extra.Stats.ScannedFull, diff --git a/internal/store/arango/profile_client.go b/internal/backends/arango/internal/store/profile_client.go similarity index 80% rename from internal/store/arango/profile_client.go rename to internal/backends/arango/internal/store/profile_client.go index c79fb7653..1f4475d7e 100644 --- a/internal/store/arango/profile_client.go +++ b/internal/backends/arango/internal/store/profile_client.go @@ -11,12 +11,10 @@ import ( "strings" ) -// Profile submits a parameterized AQL query to ArangoDB's cursor endpoint -// with profiling enabled. It returns the first cursor response, which contains -// the complete profile metadata under extra. Callers should use a sufficiently -// large BatchSize when they need to profile a complete result export; normal -// dataframe execution remains on QueryRows and is never profiled implicitly. func (c *Client) Profile(ctx context.Context, request ProfileRequest) (ProfileResult, error) { + if err := c.checkOpen(); err != nil { + return ProfileResult{}, err + } if strings.TrimSpace(request.Query) == "" { return ProfileResult{}, fmt.Errorf("AQL profile query is required") } diff --git a/internal/store/arango/profile_client_test.go b/internal/backends/arango/internal/store/profile_client_test.go similarity index 100% rename from internal/store/arango/profile_client_test.go rename to internal/backends/arango/internal/store/profile_client_test.go diff --git a/internal/store/arango/profile_integration_test.go b/internal/backends/arango/internal/store/profile_integration_test.go similarity index 93% rename from internal/store/arango/profile_integration_test.go rename to internal/backends/arango/internal/store/profile_integration_test.go index c3c8427f0..d26250884 100644 --- a/internal/store/arango/profile_integration_test.go +++ b/internal/backends/arango/internal/store/profile_integration_test.go @@ -9,10 +9,6 @@ import ( "time" ) -// TestProfileCorpusAgainstArango is opt-in because it reads a provisioned -// Arango/META database. The queries are deliberately generic physical shapes, -// not compiler or GDC fixtures: each protects a class of AQL operation that a -// FHIR dataframe compiler must be able to emit for any schema route. func TestProfileCorpusAgainstArango(t *testing.T) { if os.Getenv("LOOM_COMPILER_ARANGO_INTEGRATION") == "" { t.Skip("set LOOM_COMPILER_ARANGO_INTEGRATION=1 to run Arango profile corpus") @@ -77,11 +73,6 @@ func TestProfileCorpusAgainstArango(t *testing.T) { } } -// TestCompoundEdgeIndexExplainAgainstArango audits the endpoint-first -// persistent indexes without mutating the database. Equality filters should -// select the complete inbound/outbound compound index. Multi-type filters are -// recorded as an observation because Arango may choose the edge index for one -// direction when the IN predicate is less selective. func TestCompoundEdgeIndexExplainAgainstArango(t *testing.T) { if os.Getenv("LOOM_COMPILER_ARANGO_INTEGRATION") == "" { t.Skip("set LOOM_COMPILER_ARANGO_INTEGRATION=1 to run Arango index audit") @@ -180,11 +171,6 @@ func TestCompoundEdgeIndexExplainAgainstArango(t *testing.T) { } } -// TestReceiptPreviewExplainAgainstArango is an opt-in contract smoke test for -// the canonical query shapes emitted from immutable compilation receipts. It -// deliberately asserts only that Arango can produce a plan (and records index -// observations); physical index choice can vary with the provisioned corpus -// and optimizer version. func TestReceiptPreviewExplainAgainstArango(t *testing.T) { if os.Getenv("LOOM_COMPILER_ARANGO_INTEGRATION") == "" { t.Skip("set LOOM_COMPILER_ARANGO_INTEGRATION=1 to run receipt preview EXPLAIN fixtures") diff --git a/internal/store/arango/profile_test.go b/internal/backends/arango/internal/store/profile_test.go similarity index 100% rename from internal/store/arango/profile_test.go rename to internal/backends/arango/internal/store/profile_test.go diff --git a/internal/store/arango/types.go b/internal/backends/arango/internal/store/types.go similarity index 100% rename from internal/store/arango/types.go rename to internal/backends/arango/internal/store/types.go diff --git a/internal/backends/arango/open.go b/internal/backends/arango/open.go new file mode 100644 index 000000000..e7acdd54d --- /dev/null +++ b/internal/backends/arango/open.go @@ -0,0 +1,106 @@ +package arango + +import ( + "context" + "fmt" + + catalogadapter "github.com/calypr/loom/internal/backends/arango/internal/catalog" + datasetadapter "github.com/calypr/loom/internal/backends/arango/internal/dataset" + exploreradapter "github.com/calypr/loom/internal/backends/arango/internal/explorer" + publicationadapter "github.com/calypr/loom/internal/backends/arango/internal/publication" + recipeadapter "github.com/calypr/loom/internal/backends/arango/internal/recipeexec" + store "github.com/calypr/loom/internal/backends/arango/internal/store" +) + +var ErrInvalidConfig = fmt.Errorf("arango: invalid configuration") + +var ErrBootstrap = fmt.Errorf("arango: bootstrap failed") + +type Config struct { + URL string + Database string +} + +type Runtime struct { + Client *store.Client + Catalog *catalogadapter.Store + Dataset *datasetadapter.Store + Explorer *exploreradapter.Store + ExplorerCapabilitySnapshots *exploreradapter.CapabilitySnapshotStore + RecipeRegistry *recipeadapter.Registry + RecipeRevisions *recipeadapter.RevisionRegistry + PublishedRegistry *publicationadapter.Registry +} + +func Open(ctx context.Context, cfg Config) (*Runtime, error) { + if cfg.URL == "" { + return nil, fmt.Errorf("%w: url is required", ErrInvalidConfig) + } + if cfg.Database == "" { + return nil, fmt.Errorf("%w: database is required", ErrInvalidConfig) + } + client, err := store.Open(ctx, cfg.URL, cfg.Database) + if err != nil { + return nil, fmt.Errorf("open dataset lifecycle store: %w", err) + } + if err := client.Bootstrap(ctx, datasetadapter.BootstrapSpec()); err != nil { + _ = client.Close(ctx) + return nil, fmt.Errorf("%w: bootstrap dataset lifecycle store: %w", ErrBootstrap, err) + } + if err := client.Bootstrap(ctx, exploreradapter.BootstrapSpec()); err != nil { + _ = client.Close(ctx) + return nil, fmt.Errorf("%w: bootstrap Explorer store: %w", ErrBootstrap, err) + } + runtime := &Runtime{Client: client} + if err := runtime.assemble(client); err != nil { + _ = client.Close(ctx) + return nil, err + } + return runtime, nil +} + +func (r *Runtime) assemble(client *store.Client) error { + recipeRegistry, err := recipeadapter.New(client) + if err != nil { + return fmt.Errorf("create recipe registry: %w", err) + } + r.RecipeRegistry = recipeRegistry + datasetStore, err := datasetadapter.New(client) + if err != nil { + return fmt.Errorf("create dataset lifecycle store: %w", err) + } + r.Dataset = datasetStore + catalogStore, err := catalogadapter.New(client) + if err != nil { + return fmt.Errorf("create catalog store: %w", err) + } + r.Catalog = catalogStore + publishedRegistry, err := publicationadapter.New(client) + if err != nil { + return fmt.Errorf("create published dataframe registry: %w", err) + } + r.PublishedRegistry = publishedRegistry + recipeRevisions, err := recipeadapter.NewRevisionRegistry(client) + if err != nil { + return fmt.Errorf("create recipe revision registry: %w", err) + } + r.RecipeRevisions = recipeRevisions + explorerStore, err := exploreradapter.New(client) + if err != nil { + return fmt.Errorf("create Explorer store: %w", err) + } + r.Explorer = explorerStore + capabilitySnapshots, err := exploreradapter.NewCapabilitySnapshotStore(client) + if err != nil { + return fmt.Errorf("create Explorer capability snapshot store: %w", err) + } + r.ExplorerCapabilitySnapshots = capabilitySnapshots + return nil +} + +func (r *Runtime) Close(ctx context.Context) error { + if r == nil || r.Client == nil { + return nil + } + return r.Client.Close(ctx) +} diff --git a/internal/backends/arango/ownership_contract_test.go b/internal/backends/arango/ownership_contract_test.go new file mode 100644 index 000000000..074716f0c --- /dev/null +++ b/internal/backends/arango/ownership_contract_test.go @@ -0,0 +1,674 @@ +package arango_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "go/parser" + "go/token" + + "github.com/calypr/loom/internal/backends/arango" +) + +const modulePrefix = "github.com/calypr/loom/" + +var retiredAdapterPaths = []string{ + "internal/store/arango", + "internal/catalog/arango", + "internal/dataset/arango", + "internal/explorer/arango", + "internal/dataframe/recipe/exec/arango", + "internal/dataframe/publication/arango", + "internal/dataframe/execution/chartifact/arango", +} + +const contractTestRelPath = "internal/backends/arango/ownership_contract_test.go" + +var retiredPathPendingAllowlist = []struct { + name string + prefix string + reason string +}{} + +func retiredPathTextOffenders(t *testing.T, dir string, exempt ...string) []string { + t.Helper() + var offenders []string + walkErr := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("walk %s: %w", path, err) + } + if entry.IsDir() { + switch entry.Name() { + case ".git", "node_modules", "testdata": + return fs.SkipDir + } + return nil + } + if !strings.HasSuffix(entry.Name(), ".go") { + return nil + } + rel, relErr := filepath.Rel(dir, path) + if relErr != nil { + return fmt.Errorf("rel %s: %w", path, relErr) + } + for _, prefix := range exempt { + if strings.HasPrefix(rel, prefix) { + return nil + } + } + content, readErr := os.ReadFile(path) + if readErr != nil { + return fmt.Errorf("read %s: %w", path, readErr) + } + for _, retired := range retiredAdapterPaths { + if strings.Contains(string(content), retired) { + offenders = append(offenders, rel+" cites "+retired) + } + } + return nil + }) + if walkErr != nil { + t.Fatal(walkErr) + } + return offenders +} + +func repoRoot(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed; cannot locate the repository root") + } + root := filepath.Dir(file) + for i := 0; i < 3; i++ { + root = filepath.Dir(root) + } + if fi, err := os.Stat(filepath.Join(root, "go.mod")); err != nil || fi.IsDir() { + t.Fatalf("resolved root %q has no go.mod: %v", root, err) + } + return root +} + +func fileImports(t *testing.T, path string) []string { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, path, nil, parser.ImportsOnly) + if err != nil { + t.Fatalf("parse %s: %v", path, err) + } + imports := make([]string, 0, len(file.Imports)) + for _, imported := range file.Imports { + imports = append(imports, strings.Trim(imported.Path.Value, `"`)) + } + return imports +} + +func importOffenders(t *testing.T, dir string, includeTests bool, forbidden []string) []string { + t.Helper() + var offenders []string + walkErr := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("walk %s: %w", path, err) + } + if entry.IsDir() { + if entry.Name() == ".git" || entry.Name() == "node_modules" { + return fs.SkipDir + } + return nil + } + name := entry.Name() + if !strings.HasSuffix(name, ".go") || (!includeTests && strings.HasSuffix(name, "_test.go")) { + return nil + } + for _, importPath := range fileImports(t, path) { + for _, ban := range forbidden { + if importPath == ban || strings.HasPrefix(importPath, ban+"/") { + offenders = append(offenders, path+" imports "+importPath) + } + } + } + return nil + }) + if walkErr != nil { + t.Fatal(walkErr) + } + return offenders +} + +func TestArangoFactoryOwnershipContract(t *testing.T) { + root := repoRoot(t) + + t.Run("public_factory_imports_no_driver_or_postgres", func(t *testing.T) { + forbidden := []string{ + modulePrefix + "internal/backends/postgres", + "github.com/arangodb", + "github.com/jackc/pgx", + } + factoryDir := filepath.Join(root, "internal", "backends", "arango") + for _, name := range []string{"open.go", "aliases.go", "doc.go"} { + for _, offender := range importOffendersInFile(t, filepath.Join(factoryDir, name), forbidden) { + t.Errorf("public factory file leaks: %s", offender) + } + } + t.Run("non_vacuity_planted_leak_bleeds", func(t *testing.T) { + dir := t.TempDir() + victim := filepath.Join(dir, "victim_driver_leak.go") + if err := os.WriteFile(victim, []byte("package arango\n\nimport \"github.com/arangodb/go-driver\"\n\nvar _ = driver.Client{}\n"), 0o600); err != nil { + t.Fatal(err) + } + offenders := importOffendersInFile(t, victim, forbidden) + if len(offenders) == 0 || !strings.Contains(offenders[0], "victim_driver_leak.go") { + t.Fatalf("scanner did not name the planted victim: %v", offenders) + } + }) + }) + + t.Run("arangodb_driver_confinement_register", func(t *testing.T) { + type registerEntry struct { + prefix string + claimedLive bool + } + register := []registerEntry{ + {prefix: "internal/backends/arango/internal/", claimedLive: true}, + {prefix: "scripts/local-preview-index-migration.go", claimedLive: true}, + } + hits := make([]int, len(register)) + var offenders []string + scanProductionGo(t, root, func(rel, abs string) { + for _, importPath := range fileImports(t, abs) { + if importPath != "github.com/arangodb/go-driver" && !strings.HasPrefix(importPath, "github.com/arangodb/") { + continue + } + allowed := false + for i, entry := range register { + if strings.HasPrefix(rel, entry.prefix) { + allowed = true + hits[i]++ + } + } + if !allowed { + offenders = append(offenders, rel+" imports "+importPath) + } + } + }) + if len(offenders) != 0 { + t.Fatalf("files outside the register import the ArangoDB driver: %v", offenders) + } + for i, entry := range register { + if entry.claimedLive && hits[i] == 0 { + t.Errorf("register entry %q claims a live driver offender but none was found — update the register deliberately", entry.prefix) + } + } + }) + + t.Run("no_outside_production_file_imports_backend_privates", func(t *testing.T) { + forbidden := []string{modulePrefix + "internal/backends/arango/internal/"} + var offenders []string + scanProductionGo(t, root, func(rel, abs string) { + if strings.HasPrefix(rel, "internal/backends/arango/") { + return + } + for _, importPath := range fileImports(t, abs) { + if strings.HasPrefix(importPath, forbidden[0]) { + offenders = append(offenders, rel+" imports "+importPath) + } + } + }) + if len(offenders) != 0 { + t.Fatalf("production files reach backend privates directly: %v", offenders) + } + }) + + t.Run("zero_references_to_retired_adapter_paths", func(t *testing.T) { + var offenders []string + for _, retired := range retiredAdapterPaths { + for _, offender := range importOffenders(t, root, true, []string{modulePrefix + retired}) { + offenders = append(offenders, offender) + } + } + if len(offenders) != 0 { + t.Fatalf("retired adapter paths still imported: %v", offenders) + } + exempt := []string{contractTestRelPath} + for _, zone := range retiredPathPendingAllowlist { + exempt = append(exempt, zone.prefix) + } + for _, offender := range retiredPathTextOffenders(t, root, exempt...) { + offenders = append(offenders, offender) + } + if len(offenders) != 0 { + t.Fatalf("retired adapter paths still cited: %v", offenders) + } + t.Run("non_vacuity_planted_retired_import_bleeds", func(t *testing.T) { + dir := t.TempDir() + victim := filepath.Join(dir, "victim_retired_import.go") + if err := os.WriteFile(victim, []byte("package x\n\nimport \"github.com/calypr/loom/internal/store/arango\"\n\nvar _ = arango.Client{}\n"), 0o600); err != nil { + t.Fatal(err) + } + offenders := importOffenders(t, dir, true, []string{modulePrefix + "internal/store/arango"}) + if len(offenders) == 0 || !strings.Contains(offenders[0], "victim_retired_import.go") { + t.Fatalf("scanner did not name the planted victim: %v", offenders) + } + }) + t.Run("non_vacuity_planted_retired_comment_bleeds", func(t *testing.T) { + dir := t.TempDir() + victim := filepath.Join(dir, "victim_retired_comment.go") + if err := os.WriteFile(victim, []byte("package x\n\n// Parity contract (internal/explorer/arango/receipts.go): insert-once.\nfunc f() {}\n"), 0o600); err != nil { + t.Fatal(err) + } + found := retiredPathTextOffenders(t, dir) + if len(found) == 0 || !strings.Contains(found[0], "victim_retired_comment.go") { + t.Fatalf("text scanner did not name the planted comment victim: %v", found) + } + }) + for _, zone := range retiredPathPendingAllowlist { + t.Run("allowlist_load_bearing_"+zone.name, func(t *testing.T) { + var named []string + for _, offender := range retiredPathTextOffenders(t, root, contractTestRelPath) { + if strings.HasPrefix(offender, zone.prefix) { + named = append(named, offender) + } + } + if len(named) == 0 { + t.Errorf("allowlist zone %q (%s) names no current offender — repoint the citations and prune this entry", zone.prefix, zone.reason) + } + }) + } + }) + + t.Run("config_validation_refuses_invalid_shapes", func(t *testing.T) { + ctx := context.Background() + cases := []struct { + name string + cfg arango.Config + }{ + {"missing url", arango.Config{Database: "db"}}, + {"empty url", arango.Config{URL: "", Database: "db"}}, + {"missing database", arango.Config{URL: "http://127.0.0.1:1"}}, + {"empty database", arango.Config{URL: "http://127.0.0.1:1", Database: ""}}, + } + for _, tc := range cases { + runtime, err := arango.Open(ctx, tc.cfg) + if !errors.Is(err, arango.ErrInvalidConfig) { + t.Errorf("%s: err = %v, want ErrInvalidConfig", tc.name, err) + } + if runtime != nil { + t.Errorf("%s: half-initialized runtime escaped validation", tc.name) + } + } + if _, err := arango.Open(ctx, arango.Config{URL: "http://127.0.0.1:1", Database: "loom_ownership_control"}); err == nil { + t.Fatal("Open against a dead endpoint succeeded") + } else if errors.Is(err, arango.ErrInvalidConfig) { + t.Fatalf("complete config rejected by validation instead of the dial: %v", err) + } + }) + + t.Run("bootstrap_parity_and_index_attestation", func(t *testing.T) { + url := requireLiveArango(t) + db := mintDatabase(t, url, "parity") + ctx := context.Background() + backend, err := arango.Open(ctx, arango.Config{URL: url, Database: db}) + if err != nil { + t.Fatalf("factory Open: %v", err) + } + if backend.Client == nil || backend.Catalog == nil || backend.Dataset == nil || backend.Explorer == nil || + backend.ExplorerCapabilitySnapshots == nil || backend.RecipeRegistry == nil || backend.RecipeRevisions == nil || backend.PublishedRegistry == nil { + t.Fatal("factory runtime assembled with a nil port") + } + specs := map[string]arango.BootstrapSpec{ + "dataset": arango.DatasetBootstrapSpec(), + "explorer": arango.ExplorerBootstrapSpec(), + "recipe": arango.RecipeBootstrapSpec(), + "recipeRev": arango.RevisionBootstrapSpec(), + "bundle": arango.PublishedBootstrapSpec(), + "chartifact": arango.ArtifactBootstrapSpec(), + } + for name, spec := range specs { + if err := backend.Client.Bootstrap(ctx, spec); err != nil { + t.Fatalf("bootstrap %s: %v", name, err) + } + for _, collection := range spec.Collections { + indexes := liveIndexes(t, url, db, collection.Name) + for _, fields := range collection.Indexes { + if !hasIndexCovering(indexes, fields) { + t.Errorf("%s/%s lacks an index covering %v (live: %v)", name, collection.Name, fields, indexes) + } + } + } + } + for name, spec := range specs { + if err := backend.Client.Bootstrap(ctx, spec); err != nil { + t.Fatalf("re-bootstrap %s: %v", name, err) + } + } + if _, err := backend.Client.CollectionExists(ctx, arango.DatasetLifecycleCollection); err != nil { + t.Fatalf("dataset lifecycle collection after Open: %v", err) + } + if err := backend.Client.Close(context.Background()); err != nil { + t.Fatalf("close: %v", err) + } + }) + + t.Run("representative_query_and_control_ops", func(t *testing.T) { + url := requireLiveArango(t) + db := mintDatabase(t, url, "queryops") + ctx := context.Background() + backend, err := arango.Open(ctx, arango.Config{URL: url, Database: db}) + if err != nil { + t.Fatalf("factory Open: %v", err) + } + t.Cleanup(func() { _ = backend.Close(context.Background()) }) + client := backend.Client + rows := 0 + if err := client.QueryRows(ctx, "FOR i IN RANGE(1, 5) RETURN {value: @factor * i}", 2, map[string]any{"factor": 3}, func(row map[string]any) error { + rows++ + if v, ok := row["value"].(float64); !ok || v != 3*float64(rows) { + return fmt.Errorf("row %d = %v", rows, row) + } + return nil + }); err != nil || rows != 5 { + t.Fatalf("QueryRows: rows=%d err=%v", rows, err) + } + limited := arango.WithQueryMemoryLimit(ctx, 64*1024*1024) + if limit, ok := arango.QueryMemoryLimitFromContext(limited); !ok || limit != 64*1024*1024 { + t.Fatalf("query-memory limit context did not pass through: %d/%v", limit, ok) + } + count := 0 + if err := client.QueryRows(limited, "FOR i IN RANGE(1, 3) RETURN {i: i}", 10, nil, func(map[string]any) error { count++; return nil }); err != nil || count != 3 { + t.Fatalf("memory-limited QueryRows: count=%d err=%v", count, err) + } + if err := client.QueryRowsWithMaxRuntime(ctx, "RETURN {ok: true}", 10, nil, 30*time.Second, func(map[string]any) error { return nil }); err != nil { + t.Fatalf("QueryRowsWithMaxRuntime: %v", err) + } + badErr := client.QueryRows(ctx, "FOR x IN NO_SUCH_COLLECTION RETURN x", 10, nil, func(map[string]any) error { return nil }) + if badErr == nil { + t.Fatal("invalid AQL accepted") + } + if cause, ok := arango.DiagnosticCause(badErr); !ok || cause == "" { + t.Fatalf("vendor diagnostic lost at the boundary: err=%v cause=%q ok=%v", badErr, cause, ok) + } + }) + + t.Run("post_open_failure_cleanup_and_double_close", func(t *testing.T) { + if runtime, err := arango.Open(context.Background(), arango.Config{URL: "http://127.0.0.1:1", Database: "loom_ownership_unreachable"}); err == nil || runtime != nil { + t.Fatalf("Open against dead endpoint returned runtime=%v err=%v", runtime, err) + } + url := requireLiveArango(t) + db := mintDatabase(t, url, "failure") + backend, err := arango.Open(context.Background(), arango.Config{URL: url, Database: db}) + if err != nil { + t.Fatalf("factory Open: %v", err) + } + probe := 0 + if err := backend.Client.QueryRows(context.Background(), "RETURN {ok: true}", 10, nil, func(map[string]any) error { probe++; return nil }); err != nil || probe != 1 { + _ = backend.Close(context.Background()) + t.Fatalf("pre-close probe query: rows=%d err=%v", probe, err) + } + if err := backend.Client.Bootstrap(context.Background(), arango.BootstrapSpec{Collections: []arango.CollectionSpec{{Name: "invalid name with spaces", Indexes: [][]string{{"field"}}}}}); err == nil { + _ = backend.Close(context.Background()) + t.Fatal("bootstrap with invalid collection name succeeded") + } + if err := backend.Close(context.Background()); err != nil { + t.Fatalf("first close: %v", err) + } + if err := backend.Close(context.Background()); err != nil { + t.Fatalf("repeated close must be safe: %v", err) + } + for _, cell := range closedEntryCalls() { + t.Run(cell.name, func(t *testing.T) { + if err := cell.call(backend.Client); !errors.Is(err, arango.ErrClientClosed) { + t.Fatalf("post-close %s: err=%v, want ErrClientClosed", cell.name, err) + } + }) + } + }) + + t.Run("nil_receiver_is_rejected_on_every_exported_entry_path", func(t *testing.T) { + var client *arango.Client + for _, cell := range closedEntryCalls() { + t.Run(cell.name, func(t *testing.T) { + defer func() { + if recovered := recover(); recovered != nil { + t.Fatalf("nil-receiver %s panicked: %v", cell.name, recovered) + } + }() + if err := cell.call(client); !errors.Is(err, arango.ErrClientClosed) { + t.Fatalf("nil-receiver %s: err=%v, want ErrClientClosed", cell.name, err) + } + }) + } + }) + + t.Run("missing_database_auto_provision_is_not_a_leak", func(t *testing.T) { + url := requireLiveArango(t) + name := mintDatabase(t, url, "autoprovision") + backend, err := arango.Open(context.Background(), arango.Config{URL: url, Database: name}) + if err != nil { + t.Fatalf("Open on absent own-mint database: %v", err) + } + if err := backend.Client.Bootstrap(context.Background(), arango.BootstrapSpec{Collections: []arango.CollectionSpec{{Name: "ownership_attest", Indexes: [][]string{{"field"}}}}}); err != nil { + t.Fatalf("bootstrap on auto-provisioned database: %v", err) + } + if err := backend.Close(context.Background()); err != nil { + t.Fatalf("close: %v", err) + } + if err := backend.Close(context.Background()); err != nil { + t.Fatalf("repeated close must be safe: %v", err) + } + again, err := arango.Open(context.Background(), arango.Config{URL: url, Database: name}) + if err != nil { + t.Fatalf("re-Open provisioned database: %v", err) + } + if err := again.Close(context.Background()); err != nil { + t.Fatalf("re-Open close: %v", err) + } + }) + +} + +func importOffendersInFile(t *testing.T, path string, forbidden []string) []string { + t.Helper() + var offenders []string + for _, importPath := range fileImports(t, path) { + for _, ban := range forbidden { + if importPath == ban || strings.HasPrefix(importPath, ban+"/") { + offenders = append(offenders, path+" imports "+importPath) + } + } + } + return offenders +} + +func scanProductionGo(t *testing.T, root string, fn func(relPath, absPath string)) { + t.Helper() + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("walk %s: %w", path, err) + } + if entry.IsDir() { + if entry.Name() == ".git" || entry.Name() == "node_modules" { + return fs.SkipDir + } + return nil + } + name := entry.Name() + if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + return nil + } + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + return fmt.Errorf("rel %s: %w", path, relErr) + } + fn(filepath.ToSlash(rel), path) + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +func requireLiveArango(t *testing.T) string { + t.Helper() + url := os.Getenv("LOOM_TEST_ARANGO_URL") + if url == "" { + t.Skip("LOOM_TEST_ARANGO_URL must point to the supplied ArangoDB 3.12 service (the runner lane asserts this never skips)") + } + return url +} + +func closedEntryCalls() []struct { + name string + call func(*arango.Client) error +} { + ctx := context.Background() + visit := func(map[string]any) error { return nil } + fields := []string{"project", "dataset_generation", "auth_resource_path", "_key"} + indexName := "loom_pivot_preview_0123456789abcdef" + return []struct { + name string + call func(*arango.Client) error + }{ + {"Bootstrap", func(c *arango.Client) error { return c.Bootstrap(ctx, arango.BootstrapSpec{}) }}, + {"InsertBatchRaw", func(c *arango.Client) error { + return c.InsertBatchRaw(ctx, "Observation", []json.RawMessage{json.RawMessage(`{"_key":"ownership-witness"}`)}, false, "document") + }}, + {"QueryRows", func(c *arango.Client) error { + return c.QueryRows(ctx, "RETURN {ok: true}", 10, nil, visit) + }}, + {"QueryRowsWithMaxRuntime", func(c *arango.Client) error { + return c.QueryRowsWithMaxRuntime(ctx, "RETURN {ok: true}", 10, nil, time.Second, visit) + }}, + {"CollectionRevision", func(c *arango.Client) error { + _, err := c.CollectionRevision(ctx, "Observation") + return err + }}, + {"CollectionCount", func(c *arango.Client) error { + _, err := c.CollectionCount(ctx, "Observation") + return err + }}, + {"WithTransaction", func(c *arango.Client) error { + return c.WithTransaction(ctx, arango.TransactionCollections{Read: []string{"Observation"}}, nil) + }}, + {"CollectionExists", func(c *arango.Client) error { + _, err := c.CollectionExists(ctx, "Observation") + return err + }}, + {"ExecuteAQL", func(c *arango.Client) error { return c.ExecuteAQL(ctx, "RETURN {ok: true}", nil) }}, + {"Explain", func(c *arango.Client) error { + _, err := c.Explain(ctx, arango.ExplainRequest{Query: "RETURN 1"}) + return err + }}, + {"Profile", func(c *arango.Client) error { + _, err := c.Profile(ctx, arango.ProfileRequest{Query: "RETURN 1"}) + return err + }}, + {"ListPreviewCoveringIndexes", func(c *arango.Client) error { + _, err := c.ListPreviewCoveringIndexes(ctx, "Observation") + return err + }}, + {"EnsurePreviewCoveringIndex", func(c *arango.Client) error { + return c.EnsurePreviewCoveringIndex(ctx, "Observation", indexName, fields) + }}, + {"EnsurePreviewCoveringIndexWithStoredValues", func(c *arango.Client) error { + return c.EnsurePreviewCoveringIndexWithStoredValues(ctx, "Observation", indexName, fields, []string{"payload.witness"}) + }}, + {"EnsurePreviewCoveringIndexWithStoredValuesReplacing", func(c *arango.Client) error { + return c.EnsurePreviewCoveringIndexWithStoredValuesReplacing(ctx, "Observation", indexName, fields, []string{"payload.witness"}, indexName+"x", fields, []string{"payload.witness"}) + }}, + {"EnsurePreviewCoveringIndexReplacing", func(c *arango.Client) error { + return c.EnsurePreviewCoveringIndexReplacing(ctx, "Observation", indexName, fields, indexName+"x", fields) + }}, + } +} + +func mintDatabaseName(t *testing.T, kind string) string { + t.Helper() + return fmt.Sprintf("loom_ownership_%s_%d_%d", kind, os.Getpid(), time.Now().UnixNano()) +} + +func mintDatabase(t *testing.T, url, kind string) string { + t.Helper() + name := mintDatabaseName(t, kind) + t.Cleanup(func() { + if err := dropDatabase(url, name); err != nil { + t.Errorf("drop own-mint database %s: %v", name, err) + } + }) + return name +} + +func dropDatabase(url, name string) error { + req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/_db/_system/_api/database/%s", strings.TrimRight(url, "/"), name), nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + var result struct { + Result bool `json:"result"` + Error bool `json:"error"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return fmt.Errorf("decode drop response (status %d): %w", resp.StatusCode, err) + } + return nil +} + +type liveIndex struct { + Type string `json:"type"` + Fields []string `json:"fields"` + Unique bool `json:"unique"` +} + +func liveIndexes(t *testing.T, url, db, collection string) []liveIndex { + t.Helper() + resp, err := http.Get(fmt.Sprintf("%s/_db/%s/_api/index?collection=%s", strings.TrimRight(url, "/"), db, collection)) + if err != nil { + t.Fatalf("index listing %s: %v", collection, err) + } + defer resp.Body.Close() + var listing struct { + Error bool `json:"error"` + Code int `json:"code"` + Indexes []liveIndex `json:"indexes"` + } + if err := json.NewDecoder(resp.Body).Decode(&listing); err != nil { + t.Fatalf("decode index listing %s: %v", collection, err) + } + if listing.Error || listing.Code != 200 { + t.Fatalf("index listing %s returned error (code %d)", collection, listing.Code) + } + return listing.Indexes +} + +func hasIndexCovering(indexes []liveIndex, fields []string) bool { +outer: + for _, idx := range indexes { + if idx.Type == "primary" { + continue + } + if len(idx.Fields) != len(fields) { + continue + } + for i := range fields { + if idx.Fields[i] != fields[i] { + continue outer + } + } + return true + } + return false +} diff --git a/internal/backends/postgres/control/capability_compose_test.go b/internal/backends/postgres/control/capability_compose_test.go new file mode 100644 index 000000000..037b4bf6c --- /dev/null +++ b/internal/backends/postgres/control/capability_compose_test.go @@ -0,0 +1,235 @@ +package control + +import ( + "context" + "errors" + "fmt" + "net/url" + "reflect" + "strings" + "testing" + "time" + + catalogstore "github.com/calypr/loom/internal/backends/postgres/internal/catalog" + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestRuntimeCatalogSatisfiesCapabilityEvidenceReader(t *testing.T) { + s := newCTLService(t, true) + runtime := s.runtime + + full := reflect.ValueOf(runtime.Catalog).Pointer() + for name, port := range map[string]any{ + "Inventory": runtime.Inventory, + "FieldProfiles": runtime.FieldProfiles, + "Relationships": runtime.Relationships, + } { + if port == nil || reflect.ValueOf(port).IsNil() { + t.Fatalf("runtime.%s is nil with the source series assigned", name) + } + if reflect.ValueOf(port).Pointer() != full { + t.Fatalf("runtime.%s is not the same Store as runtime.Catalog", name) + } + } + + unrestricted := catalog.ExplicitAuthResourcePathsUnrestricted(false) + opts := catalog.CapabilityEvidenceOptions{ + Project: crProject, DatasetGeneration: crGenA, + AuthResourcePaths: []string{"/fhir/clinical"}, + AuthResourcePathsUnrestricted: unrestricted, + } + + evidence, err := func() (evidence catalog.CapabilityEvidence, err error) { + defer func() { + if rec := recover(); rec != nil { + t.Fatalf("ReadCapabilityEvidence(runtime.Catalog) panicked on the migrated schema: %v", rec) + } + }() + return catalog.ReadCapabilityEvidence(s.ctx, runtime.Catalog, opts) + }() + if err == nil { + t.Fatal("a migrated-but-unloaded generation must not report a complete aggregate") + } + if !errors.Is(err, catalogstore.ErrResourceInventoryBuildUnavailable) { + t.Fatalf("inventory leg must refuse with the typed build-unavailable error, got %v", err) + } + if evidence.ResourceInventory.Available || evidence.ResourceInventory.Status != catalog.EvidenceUnavailable { + t.Fatalf("inventory leg posture = available=%v status=%v, want typed unavailable", + evidence.ResourceInventory.Available, evidence.ResourceInventory.Status) + } + if !evidence.Relationships.Available || len(evidence.Relationships.Values) != 0 { + t.Fatalf("relationship leg on an empty schema = available=%v values=%d, want available/empty", + evidence.Relationships.Available, len(evidence.Relationships.Values)) + } + if evidence.FieldEnrichment.Available || evidence.FieldEnrichment.Status != catalog.EvidenceUnavailable { + t.Fatalf("enrichment leg posture = available=%v status=%v, want typed unavailable", + evidence.FieldEnrichment.Available, evidence.FieldEnrichment.Status) + } + + s.writeProfiles(t) + evidence, err = func() (evidence catalog.CapabilityEvidence, err error) { + defer func() { + if rec := recover(); rec != nil { + t.Fatalf("ReadCapabilityEvidence(runtime.Catalog) panicked after seeding: %v", rec) + } + }() + return catalog.ReadCapabilityEvidence(s.ctx, runtime.Catalog, opts) + }() + if err == nil || !errors.Is(err, catalogstore.ErrResourceInventoryBuildUnavailable) { + t.Fatalf("only the inventory leg may still refuse after seeding, got %v", err) + } + if !evidence.FieldEnrichment.Available || len(evidence.FieldEnrichment.Values) == 0 { + t.Fatalf("enrichment leg must answer through the composed store: available=%v values=%d", + evidence.FieldEnrichment.Available, len(evidence.FieldEnrichment.Values)) + } +} + +func TestRuntimeCatalogSatisfiesCapabilityEvidenceReaderSixSeries(t *testing.T) { + s := newCTLServiceSixSeries(t) + runtime := s.runtime + if runtime.Catalog == nil { + t.Fatal("six-series runtime must assemble Catalog") + } + unrestricted := catalog.ExplicitAuthResourcePathsUnrestricted(false) + opts := catalog.CapabilityEvidenceOptions{ + Project: crProject, DatasetGeneration: crGenA, + AuthResourcePaths: []string{"/fhir/clinical"}, + AuthResourcePathsUnrestricted: unrestricted, + } + evidence, err := func() (evidence catalog.CapabilityEvidence, err error) { + defer func() { + if rec := recover(); rec != nil { + t.Fatalf("ReadCapabilityEvidence(runtime.Catalog) panicked on the six-series runtime: %v", rec) + } + }() + return catalog.ReadCapabilityEvidence(s.ctx, runtime.Catalog, opts) + }() + if !errors.Is(err, catalogstore.ErrResourceInventoryBuildUnavailable) { + t.Fatalf("six-series aggregate must refuse with the typed build-unavailable error, got %v", err) + } + if evidence.ResourceInventory.Available || evidence.ResourceInventory.Status != catalog.EvidenceUnavailable { + t.Fatalf("inventory leg posture = available=%v status=%v, want typed unavailable", + evidence.ResourceInventory.Available, evidence.ResourceInventory.Status) + } + if !evidence.Relationships.Available || len(evidence.Relationships.Values) != 0 { + t.Fatalf("relationship leg on an empty six-series schema = available=%v values=%d, want available/empty", + evidence.Relationships.Available, len(evidence.Relationships.Values)) + } + if evidence.FieldEnrichment.Available || evidence.FieldEnrichment.Status != catalog.EvidenceUnavailable { + t.Fatalf("enrichment leg posture = available=%v status=%v, want typed unavailable", + evidence.FieldEnrichment.Available, evidence.FieldEnrichment.Status) + } + s.writeProfiles(t) + evidence, err = func() (evidence catalog.CapabilityEvidence, err error) { + defer func() { + if rec := recover(); rec != nil { + t.Fatalf("ReadCapabilityEvidence(runtime.Catalog) panicked after seeding: %v", rec) + } + }() + return catalog.ReadCapabilityEvidence(s.ctx, runtime.Catalog, opts) + }() + if err == nil || !errors.Is(err, catalogstore.ErrResourceInventoryBuildUnavailable) { + t.Fatalf("only the inventory leg may still refuse after seeding, got %v", err) + } + if !evidence.FieldEnrichment.Available || len(evidence.FieldEnrichment.Values) == 0 { + t.Fatalf("six-series enrichment leg must answer through the composed store: available=%v values=%d", + evidence.FieldEnrichment.Available, len(evidence.FieldEnrichment.Values)) + } +} + +func TestRuntimeWithoutSourceSeriesKeepsNilPorts(t *testing.T) { + s := newCTLService(t, false) + runtime := s.runtime + if runtime.Catalog != nil { + t.Fatal("Catalog must stay nil without the source series") + } + if runtime.Inventory != nil || runtime.FieldProfiles != nil || runtime.Relationships != nil { + t.Fatal("narrow catalog ports must stay nil without the source series") + } +} + +func newCTLServiceSixSeries(t *testing.T) *ctlService { + t.Helper() + raw := ctlTestPostgresURL(t) + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + secrets := []string{raw} + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + str := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + str = strings.ReplaceAll(str, secret, "") + } + } + return str + } + fail := func(step string, err error) { t.Fatalf("%s: %s", step, scrub(err)) } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + suffix := fmt.Sprintf("%x_%s", time.Now().UnixNano(), ctlRandomHex(t)[:8]) + dbName := "pgctl6_it_" + suffix + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + var count int + if err := admin.QueryRow(context.WithoutCancel(ctx), + "SELECT COUNT(*) FROM pg_database WHERE datname = $1", dbName).Scan(&count); err != nil { + t.Errorf("audit isolated database: %s", scrub(err)) + } else if count != 0 { + t.Errorf("leftover isolated database %s after cleanup", dbName) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + schemas := SeriesSchemas{} + for _, series := range AllSeries() { + schema := "pgctl6_it_" + suffix + "_" + string(series) + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema "+string(series), err) + } + schemas[series] = schema + } + + runtime, err := Open(ctx, Config{ + DSN: dsn, MaxConns: 4, MinConns: 0, AcquireTimeout: 30 * time.Second, + Schemas: schemas, Setup: SetupMigrate, + ApplicationName: ApplicationNamePrefix + "verify-capcompose-ctl6", + }) + if err != nil { + fail("open six-series control runtime", err) + } + t.Cleanup(func() { _ = runtime.Close() }) + return &ctlService{ctx: ctx, runtime: runtime, scrub: scrub, fail: fail} +} diff --git a/internal/backends/postgres/control/capability_evidence_assembly_test.go b/internal/backends/postgres/control/capability_evidence_assembly_test.go new file mode 100644 index 000000000..f4e6df458 --- /dev/null +++ b/internal/backends/postgres/control/capability_evidence_assembly_test.go @@ -0,0 +1,91 @@ +package control + +import ( + "testing" + + "github.com/calypr/loom/internal/catalog" +) + +func (s *ctlService) seedInventory(t *testing.T) { + t.Helper() + if s.runtime.Inventory == nil { + t.Fatal("runtime assembled without the inventory rebuild port") + } + if _, err := s.runtime.Inventory.RebuildResourceInventory(s.ctx, catalog.ResourceInventoryRebuildOptions{ + Project: crProject, DatasetGeneration: crGenA, + ResourceTypes: capabilityEvidenceTypes, + }); err != nil { + s.fail("rebuild resource inventory", err) + } +} + +var capabilityEvidenceTypes = []string{"Patient", "Observation"} + +func capabilityEvidenceOptions() catalog.CapabilityEvidenceOptions { + unrestricted := true + return catalog.CapabilityEvidenceOptions{ + Project: crProject, + DatasetGeneration: crGenA, + AuthResourcePathsUnrestricted: &unrestricted, + ResourceTypes: capabilityEvidenceTypes, + } +} + +func TestRuntimeCatalogServesAllThreeEvidenceSources(t *testing.T) { + s := newCTLService(t, true) + s.writeProfiles(t) + s.seedInventory(t) + if s.runtime.Catalog == nil { + t.Fatal("runtime assembled without the catalog capability-evidence reader") + } + + opts := capabilityEvidenceOptions() + evidence, err := catalog.ReadCapabilityEvidence(s.ctx, s.runtime.Catalog, opts) + if err != nil { + s.fail("read capability evidence", err) + } + if !evidence.ResourceInventory.Available || !evidence.Relationships.Available || !evidence.FieldEnrichment.Available { + t.Errorf("aggregate evidence reported an unavailable source: inventory=%v relationships=%v enrichment=%v, want all available", + evidence.ResourceInventory.Available, evidence.Relationships.Available, evidence.FieldEnrichment.Available) + } + + if _, err := s.runtime.Catalog.DiscoverResourceInventory(s.ctx, opts); err != nil { + s.fail("discover resource inventory", err) + } + if _, err := s.runtime.Catalog.DiscoverRelationshipObservations(s.ctx, opts); err != nil { + s.fail("discover relationship observations", err) + } + enrichment, err := s.runtime.Catalog.DiscoverFieldEnrichment(s.ctx, opts) + if err != nil { + s.fail("discover field enrichment", err) + } + if len(enrichment.Values) == 0 { + t.Errorf("field enrichment served no observations after the runtime's own write path stored profiles for %s/%s, want the written evidence", crProject, crGenA) + } +} + +func TestRuntimeCatalogServesScopedEvidenceUnderAuthPaths(t *testing.T) { + s := newCTLService(t, true) + s.writeProfiles(t) + s.seedInventory(t) + + unrestricted := false + opts := catalog.CapabilityEvidenceOptions{ + Project: crProject, + DatasetGeneration: crGenA, + AuthResourcePathsUnrestricted: &unrestricted, + AuthResourcePaths: []string{"/fhir/clinical"}, + ResourceTypes: capabilityEvidenceTypes, + } + evidence, err := catalog.ReadCapabilityEvidence(s.ctx, s.runtime.Catalog, opts) + if err != nil { + s.fail("read scoped capability evidence", err) + } + if !evidence.Relationships.Available || !evidence.FieldEnrichment.Available { + t.Errorf("scoped evidence reported an unavailable source: relationships=%v enrichment=%v, want both available", + evidence.Relationships.Available, evidence.FieldEnrichment.Available) + } + if len(evidence.FieldEnrichment.Values) == 0 { + t.Errorf("scoped field enrichment served no observations for /fhir/clinical, want the profiles written there") + } +} diff --git a/internal/backends/postgres/control/catalog_read.go b/internal/backends/postgres/control/catalog_read.go new file mode 100644 index 000000000..fb8c17b5a --- /dev/null +++ b/internal/backends/postgres/control/catalog_read.go @@ -0,0 +1,48 @@ +package control + +import ( + "context" + "errors" + "fmt" + + catalogstore "github.com/calypr/loom/internal/backends/postgres/internal/catalog" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" +) + +var ErrSourceSeriesUnassigned = errors.New("postgres backend: source series is unassigned; catalog reads are unavailable") + +func (r *Runtime) DiscoverFields(ctx context.Context, opts catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) { + store, err := r.catalogReadStore() + if err != nil { + return nil, err + } + return store.DiscoverFields(ctx, opts) +} + +func (r *Runtime) DiscoverExistingAuthResourcePaths(ctx context.Context, opts catalog.AuthResourcePathOptions) ([]string, error) { + store, err := r.catalogReadStore() + if err != nil { + return nil, err + } + return store.DiscoverExistingAuthResourcePaths(ctx, opts) +} + +func (r *Runtime) catalogReadStore() (*catalogstore.FieldProfileAdapter, error) { + if r.isClosed() { + return nil, fmt.Errorf("%w: discover catalog fields", ErrRuntimeClosed) + } + if r.sourceSchema == "" { + return nil, fmt.Errorf("%w: assign Schemas[%q] to read catalog discovery facts", + ErrSourceSeriesUnassigned, SeriesSource) + } + repo, err := pgstore.NewFieldProfileRepository(r.client, r.sourceSchema) + if err != nil { + return nil, fmt.Errorf("postgres backend: compose field profile read repository: %w", err) + } + store, err := catalogstore.NewFieldProfileAdapter(repo) + if err != nil { + return nil, fmt.Errorf("postgres backend: compose field profile read store: %w", err) + } + return store, nil +} diff --git a/internal/backends/postgres/control/catalog_read_test.go b/internal/backends/postgres/control/catalog_read_test.go new file mode 100644 index 000000000..c37ecaba0 --- /dev/null +++ b/internal/backends/postgres/control/catalog_read_test.go @@ -0,0 +1,349 @@ +package control + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "net/url" + "os" + "reflect" + "strings" + "testing" + "time" + + catalogstore "github.com/calypr/loom/internal/backends/postgres/internal/catalog" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + crProject = "ctl-p1" + crGenA = "generation-a" +) + +type ctlService struct { + ctx context.Context + runtime *Runtime + scrub func(any) string + fail func(string, error) +} + +func ctlTestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL control catalog read tests") + } + return raw +} + +func ctlRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) +} + +func newCTLService(t *testing.T, assignSource bool) *ctlService { + t.Helper() + raw := ctlTestPostgresURL(t) + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + secrets := []string{raw} + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { t.Fatalf("%s: %s", step, scrub(err)) } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + suffix := fmt.Sprintf("%d_%s", time.Now().UnixNano(), ctlRandomHex(t)) + dbName := "pgctl_it_" + suffix + srcSchema := "pgctl_it_" + suffix + "_src" + relSchema := "pgctl_it_" + suffix + "_rel" + + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + var count int + if err := admin.QueryRow(context.WithoutCancel(ctx), + "SELECT COUNT(*) FROM pg_database WHERE datname = $1", dbName).Scan(&count); err != nil { + t.Errorf("audit isolated database: %s", scrub(err)) + } else if count != 0 { + t.Errorf("leftover isolated database %s after cleanup", dbName) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + defer iso.Close() + for _, schema := range []string{srcSchema, relSchema} { + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema "+schema, err) + } + } + + schemas := SeriesSchemas{SeriesDatasetRelease: relSchema} + if assignSource { + schemas[SeriesSource] = srcSchema + } + runtime, err := Open(ctx, Config{ + DSN: dsn, MaxConns: 4, MinConns: 0, AcquireTimeout: 30 * time.Second, + Schemas: schemas, Setup: SetupMigrate, + ApplicationName: ApplicationNamePrefix + "verify-qroute-ctl", + }) + if err != nil { + fail("open control runtime", err) + } + t.Cleanup(func() { _ = runtime.Close() }) + return &ctlService{ctx: ctx, runtime: runtime, scrub: scrub, fail: fail} +} + +func ctlFieldDoc(path, authPath, resourceType string, docCount int64) catalog.FieldCatalogDocument { + return ctlFieldDocAt(crProject, crGenA, path, authPath, resourceType, docCount) +} + +func ctlFieldDocAt(project, generation, path, authPath, resourceType string, docCount int64) catalog.FieldCatalogDocument { + complete := true + return catalog.FieldCatalogDocument{ + Key: resourceType + "|" + path, + Project: project, DatasetGeneration: generation, AuthResourcePath: authPath, + ResourceType: resourceType, Path: path, Kind: "scalar", + DocCount: docCount, EnrichmentComplete: &complete, + } +} + +func (s *ctlService) writeProfiles(t *testing.T) { + t.Helper() + if s.runtime.FieldProfiles == nil { + t.Fatal("runtime assembled without the field-profile write port") + } + batches := []struct { + scope pgstore.FieldProfileScope + docs []catalog.FieldCatalogDocument + }{ + {pgstore.FieldProfileScope{Project: crProject, DatasetGeneration: crGenA, + AuthResourcePath: "/fhir/clinical", ResourceType: "Patient"}, + []catalog.FieldCatalogDocument{ + ctlFieldDoc("gender", "/fhir/clinical", "Patient", 3), + ctlFieldDoc("birthDate", "/fhir/clinical", "Patient", 3), + }}, + {pgstore.FieldProfileScope{Project: crProject, DatasetGeneration: crGenA, + AuthResourcePath: "/fhir/finance", ResourceType: "Observation"}, + []catalog.FieldCatalogDocument{ + ctlFieldDoc("status", "/fhir/finance", "Observation", 9), + }}, + } + for _, batch := range batches { + if err := s.runtime.FieldProfiles.WriteFieldProfile(s.ctx, batch.scope, batch.docs, 100); err != nil { + s.fail("write field profile "+batch.scope.ResourceType, err) + } + } + completeFalse, truncatedFalse := false, false + markerScope := pgstore.FieldProfileScope{ + Project: crProject, DatasetGeneration: crGenA, + AuthResourcePath: "/fhir/research", ResourceType: "Patient", + } + err := s.runtime.FieldProfiles.WriteFieldProfile(s.ctx, markerScope, []catalog.FieldCatalogDocument{{ + Key: "Patient|status", RecordType: catalog.FieldProfileStatusRecordType, + Project: crProject, DatasetGeneration: crGenA, AuthResourcePath: "/fhir/research", + ResourceType: "Patient", EnrichmentComplete: &completeFalse, EnrichmentTruncated: truncatedFalse, + }}, 100) + if err != nil { + s.fail("write marker field profile", err) + } +} + +func (s *ctlService) writeCrossGenerationProfiles(t *testing.T) { + t.Helper() + if s.runtime.FieldProfiles == nil { + t.Fatal("runtime assembled without the field-profile write port") + } + batches := []struct { + scope pgstore.FieldProfileScope + docs []catalog.FieldCatalogDocument + }{ + {pgstore.FieldProfileScope{Project: crProject, DatasetGeneration: "generation-b", + AuthResourcePath: "/fhir/clinical", ResourceType: "Patient"}, + []catalog.FieldCatalogDocument{ + ctlFieldDocAt(crProject, "generation-b", "gender", "/fhir/clinical", "Patient", 2), + }}, + {pgstore.FieldProfileScope{Project: crProject, DatasetGeneration: "generation-b", + AuthResourcePath: "/fhir/archive", ResourceType: "Observation"}, + []catalog.FieldCatalogDocument{ + ctlFieldDocAt(crProject, "generation-b", "status", "/fhir/archive", "Observation", 5), + }}, + {pgstore.FieldProfileScope{Project: crProject, DatasetGeneration: "", + AuthResourcePath: "/fhir/legacy", ResourceType: "Patient"}, + []catalog.FieldCatalogDocument{ + ctlFieldDocAt(crProject, "", "gender", "/fhir/legacy", "Patient", 1), + }}, + {pgstore.FieldProfileScope{Project: "ctl-other", DatasetGeneration: crGenA, + AuthResourcePath: "/fhir/foreign", ResourceType: "Patient"}, + []catalog.FieldCatalogDocument{ + ctlFieldDocAt("ctl-other", crGenA, "gender", "/fhir/foreign", "Patient", 1), + }}, + } + for _, batch := range batches { + if err := s.runtime.FieldProfiles.WriteFieldProfile(s.ctx, batch.scope, batch.docs, 100); err != nil { + s.fail("write cross-generation field profile "+batch.scope.ResourceType, err) + } + } +} + +func TestRuntimeCatalogReadsOverSourceAssignment(t *testing.T) { + s := newCTLService(t, true) + s.writeProfiles(t) + + fields, err := s.runtime.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: crProject, DatasetGeneration: crGenA, ResourceType: "Patient", + AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + }) + if err != nil { + s.fail("runtime discover fields", err) + } + if len(fields) != 2 || fields[0].Path != "birthDate" || fields[1].Path != "gender" { + t.Fatalf("runtime discovery = %#v, want the two Patient fields in declared order", fields) + } + for _, f := range fields { + if f.Project != crProject || f.DatasetGeneration != crGenA || f.ResourceType != "Patient" || f.Kind != "scalar" { + t.Fatalf("runtime discovery lost identity on %s: %+v", f.Path, f) + } + } + + restricted, err := s.runtime.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: crProject, DatasetGeneration: crGenA, ResourceType: "Patient", + AuthResourcePaths: []string{"/fhir/finance"}, + }) + if err != nil { + s.fail("restricted runtime discovery", err) + } + if len(restricted) != 0 { + t.Fatalf("restricted runtime discovery leaked rows: %#v", restricted) + } + + paths, err := s.runtime.DiscoverExistingAuthResourcePaths(s.ctx, catalog.AuthResourcePathOptions{ + Project: crProject, DatasetGeneration: crGenA, + }) + if err != nil { + s.fail("runtime auth-path listing", err) + } + if want := []string{"/fhir/clinical", "/fhir/finance", "/fhir/research"}; !reflect.DeepEqual(paths, want) { + t.Fatalf("runtime auth paths = %v, want %v", paths, want) + } +} + +func TestRuntimeAllGenerationsListingIsCrossGenerationUnion(t *testing.T) { + s := newCTLService(t, true) + s.writeProfiles(t) + s.writeCrossGenerationProfiles(t) + + union, err := s.runtime.DiscoverExistingAuthResourcePaths(s.ctx, catalog.AuthResourcePathOptions{ + Project: crProject, AllGenerations: true, + }) + if err != nil { + s.fail("all-generations runtime listing", err) + } + want := []string{"/fhir/archive", "/fhir/clinical", "/fhir/finance", "/fhir/legacy", "/fhir/research"} + if !reflect.DeepEqual(union, want) { + t.Fatalf("all-generations union = %v, want %v", union, want) + } + for _, p := range union { + if p == "/fhir/foreign" { + t.Fatalf("foreign project path crossed the bind: %v", union) + } + } + + if _, err := s.runtime.DiscoverExistingAuthResourcePaths(s.ctx, catalog.AuthResourcePathOptions{ + Project: crProject, DatasetGeneration: crGenA, AllGenerations: true, + }); !errors.Is(err, catalogstore.ErrAllGenerationsWithGeneration) { + t.Fatalf("runtime F-2 conflict = %v, want %v", err, catalogstore.ErrAllGenerationsWithGeneration) + } +} + +func TestRuntimeAllGenerationsListingUnrecordedProjectIsHonestEmpty(t *testing.T) { + s := newCTLService(t, true) + s.writeProfiles(t) + empty, err := s.runtime.DiscoverExistingAuthResourcePaths(s.ctx, catalog.AuthResourcePathOptions{ + Project: "ctl-unrecorded", AllGenerations: true, + }) + if err != nil { + t.Fatalf("unrecorded project must list an honest empty, got error: %v", err) + } + if empty == nil || len(empty) != 0 { + t.Fatalf("unrecorded project union = %#v, want a non-nil empty slice", empty) + } +} + +func TestRuntimeCatalogReadsRefusedWithoutSourceSeries(t *testing.T) { + s := newCTLService(t, false) + _, err := s.runtime.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: crProject, AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + }) + if !errors.Is(err, ErrSourceSeriesUnassigned) { + t.Fatalf("unassigned-source DiscoverFields error = %v, want %v", err, ErrSourceSeriesUnassigned) + } + if text := err.Error(); strings.Contains(text, "postgres://") || strings.Contains(text, "postgresql://") || strings.Contains(text, "@") { + t.Fatalf("configuration refusal carries connection material: %q", text) + } + if _, err := s.runtime.DiscoverExistingAuthResourcePaths(s.ctx, catalog.AuthResourcePathOptions{ + Project: crProject, + }); !errors.Is(err, ErrSourceSeriesUnassigned) { + t.Fatalf("unassigned-source listing error = %v, want %v", err, ErrSourceSeriesUnassigned) + } + if _, err := s.runtime.DiscoverExistingAuthResourcePaths(s.ctx, catalog.AuthResourcePathOptions{ + Project: crProject, AllGenerations: true, + }); !errors.Is(err, ErrSourceSeriesUnassigned) { + t.Fatalf("unassigned-source all-generations listing error = %v, want %v", err, ErrSourceSeriesUnassigned) + } +} + +func TestRuntimeCatalogReadsAfterCloseTyped(t *testing.T) { + s := newCTLService(t, true) + if err := s.runtime.Close(); err != nil { + s.fail("close runtime", err) + } + if _, err := s.runtime.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: crProject, AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + }); !errors.Is(err, ErrRuntimeClosed) { + t.Fatalf("closed-runtime DiscoverFields error = %v, want %v", err, ErrRuntimeClosed) + } +} diff --git a/internal/backends/postgres/control/config.go b/internal/backends/postgres/control/config.go new file mode 100644 index 000000000..da17da869 --- /dev/null +++ b/internal/backends/postgres/control/config.go @@ -0,0 +1,98 @@ +package control + +import ( + "fmt" + "regexp" + "strings" + "time" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" +) + +type Series string + +const ( + SeriesSource Series = "source" + SeriesExplorerControl Series = "explorer_control" + SeriesPublicationControl Series = "publication_control" + SeriesDatasetRelease Series = "dataset_release" + SeriesRecipeChains Series = "recipe_chains" + SeriesRecipeExecRegistries Series = "recipe_exec_registries" +) + +func AllSeries() []Series { + return []Series{ + SeriesSource, + SeriesExplorerControl, + SeriesPublicationControl, + SeriesDatasetRelease, + SeriesRecipeChains, + SeriesRecipeExecRegistries, + } +} + +type SeriesSchemas map[Series]string + +type SetupPolicy string + +const ( + SetupMigrate SetupPolicy = "migrate" + SetupNone SetupPolicy = "none" +) + +const ApplicationNamePrefix = "loom-pg-" + +var applicationNamePattern = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`) + +type Config struct { + DSN string + MaxConns int32 + MinConns int32 + AcquireTimeout time.Duration + ApplicationName string + Schemas SeriesSchemas + Setup SetupPolicy +} + +func (c Config) Validate() error { + storeCfg := pgstore.Config{ + DSN: c.DSN, + MaxConns: c.MaxConns, + MinConns: c.MinConns, + AcquireTimeout: c.AcquireTimeout, + } + if err := storeCfg.Validate(); err != nil { + return fmt.Errorf("%w: %w", ErrInvalidConfig, err) + } + if c.ApplicationName != "" { + if !strings.HasPrefix(c.ApplicationName, ApplicationNamePrefix) { + return fmt.Errorf("%w: ApplicationName %q must carry the own-prefix %q so connection witnesses stay unambiguous", + ErrInvalidConfig, c.ApplicationName, ApplicationNamePrefix) + } + if !applicationNamePattern.MatchString(c.ApplicationName) { + return fmt.Errorf("%w: ApplicationName %q must match %s", ErrInvalidConfig, c.ApplicationName, applicationNamePattern) + } + } + switch c.Setup { + case SetupMigrate, SetupNone: + default: + return fmt.Errorf("%w: Setup %q is not one of %q, %q; the schema setup policy must be explicit", + ErrInvalidConfig, string(c.Setup), SetupMigrate, SetupNone) + } + for series, schema := range c.Schemas { + if strings.TrimSpace(schema) == "" { + return fmt.Errorf("%w: series %q is assigned a blank schema name", ErrSeriesAssignment, series) + } + if !pgstore.ValidSchemaName(schema) { + return fmt.Errorf("%w: series %q schema %q must match [a-z_][a-z0-9_]{0,62}", ErrInvalidSchemaName, series, schema) + } + } + assignment := pgstore.Assignment{} + for series, schema := range c.Schemas { + assignment[pgstore.SeriesID(series)] = schema + } + if err := pgstore.ValidateAssignment(assignment); err != nil { + return fmt.Errorf("%w: %w", ErrSeriesAssignment, err) + } + return nil +} diff --git a/internal/backends/postgres/control/errors.go b/internal/backends/postgres/control/errors.go new file mode 100644 index 000000000..20bd394e9 --- /dev/null +++ b/internal/backends/postgres/control/errors.go @@ -0,0 +1,62 @@ +package control + +import ( + "errors" + + datasetstore "github.com/calypr/loom/internal/backends/postgres/internal/dataset" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" +) + +var ( + ErrInvalidConfig = errors.New("postgres backend: invalid config") + ErrSeriesAssignment = errors.New("postgres backend: invalid series assignment") + ErrBootstrap = errors.New("postgres backend: bootstrap failed") + ErrRuntimeClosed = errors.New("postgres backend: runtime closed") +) + +var ( + ErrConnect = pgstore.ErrConnect + ErrUnsupportedVersion = pgstore.ErrUnsupportedVersion + ErrUniqueConflict = pgstore.ErrUniqueConflict + ErrCommitOutcomeUnknown = pgstore.ErrCommitOutcomeUnknown + ErrBatchWrite = pgstore.ErrBatchWrite + + ErrManifestAlreadyExists = datasetstore.ErrManifestAlreadyExists + ErrManifestTransitionConflict = datasetstore.ErrManifestTransitionConflict + ErrActivationConflict = datasetstore.ErrActivationConflict + + ErrMigrationPending = pgstore.ErrMigrationPending + ErrMigrationChecksumMismatch = pgstore.ErrMigrationChecksumMismatch + ErrFutureSchemaVersion = pgstore.ErrFutureSchemaVersion + ErrMigrationMissing = pgstore.ErrMigrationMissing + ErrMigrationOutOfOrder = pgstore.ErrMigrationOutOfOrder +) + +type ( + DriverCondition = pgstore.DriverCondition + TranslatedError = pgstore.TranslatedError + ConnectError = pgstore.ConnectError + UniqueConflictError = pgstore.UniqueConflictError +) + +var ( + ConditionAbortedTransaction = pgstore.ConditionAbortedTransaction + ConditionAdminShutdown = pgstore.ConditionAdminShutdown + ConditionCannotConnectNow = pgstore.ConditionCannotConnectNow + ConditionCommitRejected = pgstore.ConditionCommitRejected + ConditionConnectionFailure = pgstore.ConditionConnectionFailure + ConditionConnectionLost = pgstore.ConditionConnectionLost + ConditionCrashShutdown = pgstore.ConditionCrashShutdown + ConditionDeadlockDetected = pgstore.ConditionDeadlockDetected + ConditionQueryCanceled = pgstore.ConditionQueryCanceled + ConditionRequestCanceled = pgstore.ConditionRequestCanceled + ConditionSerializationFailure = pgstore.ConditionSerializationFailure + ConditionServerRejected = pgstore.ConditionServerRejected + ConditionSessionTerminated = pgstore.ConditionSessionTerminated + ConditionTransactionClosed = pgstore.ConditionTransactionClosed + ConditionUniqueViolation = pgstore.ConditionUniqueViolation + TranslateError = pgstore.TranslateError + TranslateCommitError = pgstore.TranslateCommitError + AsUniqueConflict = pgstore.AsUniqueConflict + RetryableTransactionConflict = pgstore.RetryableTransactionConflict +) diff --git a/internal/backends/postgres/control/runtime.go b/internal/backends/postgres/control/runtime.go new file mode 100644 index 000000000..27a660d92 --- /dev/null +++ b/internal/backends/postgres/control/runtime.go @@ -0,0 +1,497 @@ +package control + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "net/url" + "strings" + "sync" + + catalogstore "github.com/calypr/loom/internal/backends/postgres/internal/catalog" + chartifactstore "github.com/calypr/loom/internal/backends/postgres/internal/chartifact" + datasetstore "github.com/calypr/loom/internal/backends/postgres/internal/dataset" + explorerstore "github.com/calypr/loom/internal/backends/postgres/internal/explorer" + publicationstore "github.com/calypr/loom/internal/backends/postgres/internal/publication" + recipeexecstore "github.com/calypr/loom/internal/backends/postgres/internal/recipeexec" + recipesstore "github.com/calypr/loom/internal/backends/postgres/internal/recipes" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "github.com/calypr/loom/internal/backends/postgres/internal/redact" + + publicationcontract "github.com/calypr/loom/internal/dataset" +) + +type Rows interface { + Next() bool + Scan(dest ...any) error + Values() ([]any, error) + Err() error + Close() +} + +type Row interface { + Scan(dest ...any) error +} + +type Runtime struct { + Manifests publicationcontract.ManifestLifecycle + FieldProfiles catalog.FieldProfileWriter + Inventory catalog.InventoryRebuilder + Relationships catalog.RelationshipDiscoverer + Contributions catalog.SemanticContributor + Catalog *catalogstore.Store + Releases *datasetstore.ReleaseStore + SemanticReads *catalogstore.SemanticReadRepository + + Explorer *explorerstore.Store + Publish *explorerstore.PublishStore + Activate *explorerstore.ActivationStore + Interpretations *explorerstore.InterpretationStore + ExplicitGroups *explorerstore.ExplicitGroupStore + CapabilitySnapshots *explorerstore.CapabilitySnapshotStore + TableShapes *explorerstore.TableShapeCapabilityStore + Bundles *publicationstore.BundleCatalog + Chartifacts *chartifactstore.Catalog + RecipeRevisions *recipesstore.RevisionRegistry + RecipeExecutions *recipeexecstore.Registry + + client *pgstore.Client + sourceSchema string + datasetSchema string + schemas map[Series]string + closeOne sync.Once + closeErr error + closed bool + mu sync.Mutex +} + +var ( + _ publicationcontract.ManifestLifecycle = (*datasetstore.Store)(nil) + _ catalog.FieldProfileWriter = (*catalogstore.Store)(nil) + _ catalog.InventoryRebuilder = (*catalogstore.Store)(nil) + _ catalog.RelationshipDiscoverer = (*catalogstore.Store)(nil) + _ catalog.CapabilityEvidenceReader = (*catalogstore.Store)(nil) + _ catalog.SemanticContributor = (*catalogstore.SemanticContributionRepository)(nil) + _ catalog.InventoryRebuilder = (*catalogstore.InventoryAdapter)(nil) + _ catalog.FieldProfileWriter = (*catalogstore.FieldProfileAdapter)(nil) + _ catalog.RelationshipDiscoverer = (*catalogstore.RelationshipAdapter)(nil) +) + +func Open(ctx context.Context, cfg Config) (*Runtime, error) { + if err := cfg.Validate(); err != nil { + return nil, err + } + applicationName := cfg.ApplicationName + if applicationName == "" { + nonce, err := connectionNonce() + if err != nil { + return nil, fmt.Errorf("%w: generate application name: %v", ErrInvalidConfig, err) + } + applicationName = ApplicationNamePrefix + nonce + } + password := redact.ExtractDSNPassword(cfg.DSN) + + client, err := pgstore.Open(ctx, pgstore.Config{ + DSN: withApplicationName(cfg.DSN, applicationName), + MaxConns: cfg.MaxConns, + MinConns: cfg.MinConns, + AcquireTimeout: cfg.AcquireTimeout, + }) + if err != nil { + return nil, redactWrap(ErrBootstrap, "open postgres source/control pool", err, password) + } + + rt := &Runtime{client: client} + if len(cfg.Schemas) > 0 { + rt.schemas = make(map[Series]string, len(cfg.Schemas)) + for series, schema := range cfg.Schemas { + rt.schemas[series] = schema + } + } + if schema, assigned := cfg.Schemas[SeriesSource]; assigned { + rt.sourceSchema = schema + } + if schema, assigned := cfg.Schemas[SeriesDatasetRelease]; assigned { + rt.datasetSchema = schema + } + if err := rt.setup(ctx, cfg); err != nil { + rt.Close() + return nil, err + } + if err := rt.assembleControlPorts(cfg); err != nil { + rt.Close() + return nil, err + } + if err := rt.assembleDataPlanePorts(cfg); err != nil { + rt.Close() + return nil, err + } + return rt, nil +} + +func (r *Runtime) setup(ctx context.Context, cfg Config) error { + if cfg.Setup == SetupNone { + return nil + } + password := redact.ExtractDSNPassword(cfg.DSN) + for _, series := range AllSeries() { + schema, assigned := cfg.Schemas[series] + if !assigned { + continue + } + var err error + switch series { + case SeriesSource: + _, err = pgstore.MigrateSchema(ctx, r.client, schema) + case SeriesExplorerControl: + _, err = explorerstore.MigrateSchema(ctx, r.client, schema) + case SeriesPublicationControl: + _, err = publicationstore.MigrateSchema(ctx, r.client, schema) + case SeriesDatasetRelease: + _, err = datasetstore.MigrateSchema(ctx, r.client, schema) + case SeriesRecipeChains: + _, err = recipesstore.MigrateSchema(ctx, r.client, schema) + case SeriesRecipeExecRegistries: + _, err = recipeexecstore.MigrateSchema(ctx, r.client, schema) + } + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("schema setup series %q in %q under policy %q", series, schema, cfg.Setup), err, password) + } + } + return nil +} + +func (r *Runtime) assembleControlPorts(cfg Config) error { + password := redact.ExtractDSNPassword(cfg.DSN) + if schema, assigned := cfg.Schemas[SeriesDatasetRelease]; assigned { + store, err := datasetstore.New(r.client, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble dataset manifest lifecycle in %q", schema), err, password) + } + r.Manifests = store + } + schema, assigned := cfg.Schemas[SeriesSource] + if !assigned { + return nil + } + profileRepo, err := pgstore.NewFieldProfileRepository(r.client, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble field profile repository in %q", schema), err, password) + } + inventoryRepo, err := pgstore.NewInventoryRepository(r.client, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble inventory repository in %q", schema), err, password) + } + relationshipsRepo, err := catalogstore.NewRelationshipEvidenceRepository(r.client, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble relationship evidence repository in %q", schema), err, password) + } + catalogStore, err := catalogstore.NewFullStore(inventoryRepo, profileRepo, relationshipsRepo) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble catalog store in %q", schema), err, password) + } + r.FieldProfiles = catalogStore + r.Inventory = catalogStore + r.Catalog = catalogStore + r.Relationships = catalogStore + contributions, err := catalogstore.NewSemanticContributionRepository(r.client, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble semantic contribution repository in %q", schema), err, password) + } + r.Contributions = contributions + semanticReads, err := catalogstore.NewSemanticReadRepository(r.client, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble semantic read repository in %q", schema), err, password) + } + r.SemanticReads = semanticReads + return nil +} + +type leasedDB struct { + client *pgstore.Client +} + +func (d *leasedDB) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { + return d.client.Query(ctx, sql, args...) +} + +func (d *leasedDB) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row { + return d.client.QueryRow(ctx, sql, args...) +} + +func (d *leasedDB) Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) { + conn, err := d.client.Acquire(ctx) + if err != nil { + return pgconn.CommandTag{}, err + } + defer conn.Release() + return conn.Exec(ctx, sql, args...) +} + +func (d *leasedDB) Begin(ctx context.Context) (pgx.Tx, error) { + conn, err := d.client.Acquire(ctx) + if err != nil { + return nil, err + } + tx, err := conn.Begin(ctx) + if err != nil { + conn.Release() + return nil, err + } + return &leasedTx{Tx: tx, releaseOne: &sync.Once{}, release: conn.Release}, nil +} + +type leasedTx struct { + pgx.Tx + releaseOne *sync.Once + release func() +} + +func (t *leasedTx) Commit(ctx context.Context) error { + err := t.Tx.Commit(ctx) + t.releaseOne.Do(t.release) + return err +} + +func (t *leasedTx) Rollback(ctx context.Context) error { + err := t.Tx.Rollback(ctx) + t.releaseOne.Do(t.release) + return err +} + +func (r *Runtime) assembleDataPlanePorts(cfg Config) error { + password := redact.ExtractDSNPassword(cfg.DSN) + dataDB := &leasedDB{client: r.client} + if schema, assigned := cfg.Schemas[SeriesDatasetRelease]; assigned { + releases, err := datasetstore.NewReleaseStore(r.client, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble dataset release repository in %q", schema), err, password) + } + r.Releases = releases + } + if schema, assigned := cfg.Schemas[SeriesExplorerControl]; assigned { + explorer, err := explorerstore.New(r.client, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble Explorer store in %q", schema), err, password) + } + r.Explorer = explorer + interpretations, err := explorerstore.NewInterpretationStore(r.client, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble interpretation store in %q", schema), err, password) + } + r.Interpretations = interpretations + explicitGroups, err := explorerstore.NewExplicitGroupStore(r.client, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble explicit group store in %q", schema), err, password) + } + r.ExplicitGroups = explicitGroups + snapshots, err := explorerstore.NewCapabilitySnapshotStore(r.client, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble capability snapshot store in %q", schema), err, password) + } + r.CapabilitySnapshots = snapshots + tableShapes, err := explorerstore.NewTableShapeCapabilityStore(r.client, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble table-shape capability store in %q", schema), err, password) + } + r.TableShapes = tableShapes + if datasetSchema, datasetAssigned := cfg.Schemas[SeriesDatasetRelease]; datasetAssigned { + publish, err := explorerstore.NewPublishStore(r.client, schema, datasetSchema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble Explorer publish store across %q and %q", schema, datasetSchema), err, password) + } + r.Publish = publish + if publicationSchema, publicationAssigned := cfg.Schemas[SeriesPublicationControl]; publicationAssigned { + activate, err := explorerstore.NewActivationStore(r.client, schema, datasetSchema, publicationSchema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble Explorer activation store across %q, %q, and %q", schema, datasetSchema, publicationSchema), err, password) + } + r.Activate = activate + } + } + } + if schema, assigned := cfg.Schemas[SeriesPublicationControl]; assigned { + bundles, err := publicationstore.NewBundleCatalog(dataDB, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble bundle catalog in %q", schema), err, password) + } + r.Bundles = bundles + chartifacts, err := chartifactstore.NewCatalog(dataDB, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble private artifact catalog in %q", schema), err, password) + } + r.Chartifacts = chartifacts + } + if schema, assigned := cfg.Schemas[SeriesRecipeChains]; assigned { + revisions, err := recipesstore.New(dataDB, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble recipe revision registry in %q", schema), err, password) + } + r.RecipeRevisions = revisions + } + if schema, assigned := cfg.Schemas[SeriesRecipeExecRegistries]; assigned { + executions, err := recipeexecstore.New(dataDB, schema) + if err != nil { + return redactWrap(ErrBootstrap, fmt.Sprintf("assemble recipe execution registry in %q", schema), err, password) + } + r.RecipeExecutions = executions + } + return nil +} + +func (r *Runtime) NewPublicationReconciler(releases publicationcontract.ActiveReleaseResolver) (*publicationstore.Reconciler, error) { + if r.isClosed() { + return nil, ErrRuntimeClosed + } + if r.Bundles == nil { + return nil, fmt.Errorf("%w: publication_control series is not assigned; the reconciler needs the bundle registry and its pin arbiter", ErrInvalidConfig) + } + return publicationstore.NewReconciler(r.Bundles.Registry, r.Bundles.Pins(), releases) +} + +func (r *Runtime) ConnSource() ConnSource { return r.client } + +func (r *Runtime) PoolClient() *Client { + if r.isClosed() { + return nil + } + return r.client +} + +func (r *Runtime) Query(ctx context.Context, sql string, args ...any) (Rows, error) { + if r.isClosed() { + return nil, ErrRuntimeClosed + } + return r.client.Query(ctx, sql, args...) +} + +func (r *Runtime) QueryRow(ctx context.Context, sql string, args ...any) Row { + if r.isClosed() { + return closedRow{} + } + return r.client.QueryRow(ctx, sql, args...) +} + +type closedRow struct{} + +func (closedRow) Scan(dest ...any) error { return ErrRuntimeClosed } + +func (r *Runtime) MigrateDatasetReleaseSchema(ctx context.Context) (MigrateResult, error) { + if r.isClosed() { + return MigrateResult{}, ErrRuntimeClosed + } + if r.datasetSchema == "" { + return MigrateResult{}, fmt.Errorf("%w: dataset_release series is not assigned; manifest-lane migrations need its schema", ErrInvalidConfig) + } + return datasetstore.MigrateSchema(ctx, r.client, r.datasetSchema) +} + +func (r *Runtime) MigrateSchema(ctx context.Context) (MigrateResult, error) { + if r.isClosed() { + return MigrateResult{}, ErrRuntimeClosed + } + if r.sourceSchema == "" { + return MigrateResult{}, fmt.Errorf("%w: source series schema is not assigned", ErrInvalidConfig) + } + return pgstore.MigrateSchema(ctx, r.client, r.sourceSchema) +} + +func (r *Runtime) Health(ctx context.Context) error { + if r.isClosed() { + return ErrRuntimeClosed + } + return r.client.Ping(ctx) +} + +func (r *Runtime) Readiness(ctx context.Context) error { + if err := r.Health(ctx); err != nil { + return err + } + for _, series := range AllSeries() { + schema, assigned := r.schemas[series] + if !assigned || strings.TrimSpace(schema) == "" { + continue + } + var err error + switch series { + case SeriesSource: + err = pgstore.VerifySchema(ctx, r.client, schema) + case SeriesExplorerControl: + err = explorerstore.VerifySchema(ctx, r.client, schema) + case SeriesPublicationControl: + err = publicationstore.VerifySchema(ctx, r.client, schema) + case SeriesDatasetRelease: + err = datasetstore.VerifySchema(ctx, r.client, schema) + case SeriesRecipeChains: + err = recipesstore.VerifySchema(ctx, r.client, schema) + case SeriesRecipeExecRegistries: + err = recipeexecstore.VerifySchema(ctx, r.client, schema) + } + if err != nil { + return fmt.Errorf("postgres backend: schema readiness for series %q: %w", series, err) + } + } + return nil +} + +func (r *Runtime) Close() error { + r.closeOne.Do(func() { + r.mu.Lock() + r.closed = true + r.mu.Unlock() + r.client.Close() + }) + return r.closeErr +} + +func (r *Runtime) isClosed() bool { + r.mu.Lock() + defer r.mu.Unlock() + return r.closed +} + +func connectionNonce() (string, error) { + buf := make([]byte, 6) + if _, err := rand.Read(buf); err != nil { + return "", err + } + return hex.EncodeToString(buf), nil +} + +func withApplicationName(dsn, applicationName string) string { + if u, err := url.Parse(dsn); err == nil && u.Scheme != "" { + q := u.Query() + if q.Get("application_name") != "" { + return dsn + } + q.Set("application_name", applicationName) + u.RawQuery = q.Encode() + return u.String() + } + if strings.Contains(strings.ToLower(dsn), "application_name=") { + return dsn + } + return dsn + " application_name=" + applicationName +} + +type redactedError struct { + msg string + cause error +} + +func (e *redactedError) Error() string { return e.msg } +func (e *redactedError) Unwrap() error { return e.cause } + +func redactWrap(class error, op string, cause error, secret string) error { + text := redact.ScrubSecretText(fmt.Sprintf("%v: %v", op, cause), secret) + return &redactedError{ + msg: text, + cause: errors.Join(class, cause), + } +} diff --git a/internal/backends/postgres/control/vocabulary.go b/internal/backends/postgres/control/vocabulary.go new file mode 100644 index 000000000..60f89a478 --- /dev/null +++ b/internal/backends/postgres/control/vocabulary.go @@ -0,0 +1,78 @@ +package control + +import ( + "context" + + datasetstore "github.com/calypr/loom/internal/backends/postgres/internal/dataset" + explorerstore "github.com/calypr/loom/internal/backends/postgres/internal/explorer" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + + "github.com/calypr/loom/internal/catalog" +) + +type ConnSource = pgstore.ConnSource + +type FieldProfileScope = catalog.FieldProfileScope + +type ( + ResourceBatch = pgstore.ResourceBatch + ResourceBatchResult = pgstore.ResourceBatchResult + EdgeBatch = pgstore.EdgeBatch + EdgeBatchResult = pgstore.EdgeBatchResult +) + +const ( + MaxBatchResources = pgstore.MaxBatchResources + MaxBatchEdges = pgstore.MaxBatchEdges + MaxBatchIDLength = pgstore.MaxBatchIDLength + MaxFieldProfileWriteBatch = pgstore.MaxFieldProfileWriteBatch + MaxInventoryWriteBatch = pgstore.MaxInventoryWriteBatch +) + +func NewResourceWriter(src ConnSource, schema string) (*ResourceWriter, error) { + return pgstore.NewResourceWriter(src, schema) +} + +func NewEdgeWriter(src ConnSource, schema string) (*EdgeWriter, error) { + return pgstore.NewEdgeWriter(src, schema) +} + +type ( + ResourceWriter = pgstore.ResourceWriter + EdgeWriter = pgstore.EdgeWriter +) + +func GenerationAdvisoryLockKey(parts ...string) int64 { + return pgstore.GenerationAdvisoryLockKey(parts...) +} + +func WithAdvisoryLock(ctx context.Context, src ConnSource, key int64, work func() error) error { + return pgstore.WithAdvisoryLock(ctx, src, key, work) +} + +type MigrateResult = pgstore.MigrateResult + +var CommitOutcomeUnknown = pgstore.CommitOutcomeUnknown + +type ( + Client = pgstore.Client + ClientConfig = pgstore.Config +) + +func OpenPool(ctx context.Context, cfg ClientConfig) (*Client, error) { + return pgstore.Open(ctx, cfg) +} + +var OwnerMigrations = explorerstore.OwnerMigrations + +var CheckServerVersion = pgstore.CheckServerVersion + +func MigrateSchema(ctx context.Context, client *Client, schema string) (MigrateResult, error) { + return pgstore.MigrateSchema(ctx, client, schema) +} + +func MigrateDatasetReleaseSchema(ctx context.Context, client *Client, schema string) (MigrateResult, error) { + return datasetstore.MigrateSchema(ctx, client, schema) +} + +var ErrResourceConflict = pgstore.ErrResourceConflict diff --git a/internal/backends/postgres/control/vocabulary_control.go b/internal/backends/postgres/control/vocabulary_control.go new file mode 100644 index 000000000..bd61e00c3 --- /dev/null +++ b/internal/backends/postgres/control/vocabulary_control.go @@ -0,0 +1,190 @@ +package control + +import ( + catalogstore "github.com/calypr/loom/internal/backends/postgres/internal/catalog" + chartifactstore "github.com/calypr/loom/internal/backends/postgres/internal/chartifact" + datasetstore "github.com/calypr/loom/internal/backends/postgres/internal/dataset" + explorerstore "github.com/calypr/loom/internal/backends/postgres/internal/explorer" + publicationstore "github.com/calypr/loom/internal/backends/postgres/internal/publication" + recipeexecstore "github.com/calypr/loom/internal/backends/postgres/internal/recipeexec" + recipesstore "github.com/calypr/loom/internal/backends/postgres/internal/recipes" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + + publicationcontract "github.com/calypr/loom/internal/dataset" +) + +var ( + ErrBatchIdentityConflict = pgstore.ErrBatchIdentityConflict + ErrBatchRowConflict = pgstore.ErrBatchRowConflict + ErrEdgeSourceMissing = pgstore.ErrEdgeSourceMissing + ErrInvalidDSN = pgstore.ErrInvalidDSN + ErrInvalidPoolConfig = pgstore.ErrInvalidPoolConfig +) + +type ( + BatchResourceConflictError = pgstore.BatchResourceConflictError + BatchEdgeConflictError = pgstore.BatchEdgeConflictError + BatchIdentityConflictError = pgstore.BatchIdentityConflictError + EdgeSourceMissingError = pgstore.EdgeSourceMissingError +) + +var ( + ErrManifestNotFound = datasetstore.ErrManifestNotFound + ErrInvalidSchemaName = datasetstore.ErrInvalidSchemaName +) + +type ( + EdgeOccurrenceRow = pgstore.EdgeOccurrenceRow + EdgeScope = pgstore.EdgeScope +) + +type ( + ResourceRepository = pgstore.ResourceRepository + EdgeRepository = pgstore.EdgeRepository +) + +func NewResourceRepository(src ConnSource, schema string) (*ResourceRepository, error) { + return pgstore.NewResourceRepository(src, schema) +} + +func NewEdgeRepository(src ConnSource, schema string) (*EdgeRepository, error) { + return pgstore.NewEdgeRepository(src, schema) +} + +type ( + FieldProfileCoverage = pgstore.FieldProfileCoverage + FieldProfileFilter = pgstore.FieldProfileFilter + InventorySourceGroup = pgstore.InventorySourceGroup + InventoryCountScope = pgstore.InventoryCountScope +) + +type ( + FieldProfileStore = catalogstore.FieldProfileStore + InventoryStore = catalogstore.InventoryStore + RelationshipStore = catalogstore.RelationshipStore +) + +type ( + RelationshipScope = catalogstore.RelationshipScope + RelationshipOccurrenceGroup = catalogstore.RelationshipOccurrenceGroup + SemanticReadRepository = catalogstore.SemanticReadRepository +) + +func NewSemanticReadRepository(src ConnSource, schema string) (*SemanticReadRepository, error) { + return catalogstore.NewSemanticReadRepository(src, schema) +} + +type ( + FieldProfileRepository = pgstore.FieldProfileRepository + InventoryRepository = pgstore.InventoryRepository +) + +func NewFieldProfileRepository(src ConnSource, schema string) (*FieldProfileRepository, error) { + return pgstore.NewFieldProfileRepository(src, schema) +} + +func NewInventoryRepository(src ConnSource, schema string) (*InventoryRepository, error) { + return pgstore.NewInventoryRepository(src, schema) +} + +type ( + CatalogInventoryAdapter = catalogstore.InventoryAdapter + CatalogFieldProfileAdapter = catalogstore.FieldProfileAdapter + CatalogRelationshipAdapter = catalogstore.RelationshipAdapter +) + +func NewCatalogInventoryAdapter(inventory InventoryStore) (*CatalogInventoryAdapter, error) { + return catalogstore.NewInventoryAdapter(inventory) +} + +func NewCatalogFieldProfileAdapter(profiles FieldProfileStore) (*CatalogFieldProfileAdapter, error) { + return catalogstore.NewFieldProfileAdapter(profiles) +} + +func NewCatalogRelationshipEvidenceRepository(src ConnSource, schema string) (*catalogstore.RelationshipEvidenceRepository, error) { + return catalogstore.NewRelationshipEvidenceRepository(src, schema) +} + +func NewCatalogRelationshipAdapter(relationships RelationshipStore) (*CatalogRelationshipAdapter, error) { + return catalogstore.NewRelationshipAdapter(relationships) +} + +type SemanticContributionRepository = catalogstore.SemanticContributionRepository + +func NewSemanticContributionRepository(src ConnSource, schema string) (*SemanticContributionRepository, error) { + return catalogstore.NewSemanticContributionRepository(src, schema) +} + +type ( + ExplorerStore = explorerstore.Store + ExplicitGroupStore = explorerstore.ExplicitGroupStore +) + +func NewExplorerStore(client *Client, schema string) (*ExplorerStore, error) { + return explorerstore.New(client, schema) +} + +func NewExplicitGroupStore(client *Client, schema string) (*ExplicitGroupStore, error) { + return explorerstore.NewExplicitGroupStore(client, schema) +} + +type ( + PublishStore = explorerstore.PublishStore + ActivationStore = explorerstore.ActivationStore + InterpretationStore = explorerstore.InterpretationStore + CapabilitySnapshotStore = explorerstore.CapabilitySnapshotStore + TableShapeCapabilityStore = explorerstore.TableShapeCapabilityStore + DatasetReleaseStore = datasetstore.ReleaseStore + BundleCatalog = publicationstore.BundleCatalog + BundleRegistry = publicationstore.Registry + BundleReadPins = publicationstore.ReadPins + PublicationReconciler = publicationstore.Reconciler + PhysicalDropTable = publicationstore.PhysicalDropTable + RecipeExecRegistry = recipeexecstore.Registry + RecipeRevisionRegistry = recipesstore.RevisionRegistry + ChartifactCatalog = chartifactstore.Catalog +) + +func NewPublishStore(client *Client, explorerSchema, datasetSchema string) (*PublishStore, error) { + return explorerstore.NewPublishStore(client, explorerSchema, datasetSchema) +} + +func NewActivationStore(client *Client, explorerSchema, datasetSchema, publicationSchema string) (*ActivationStore, error) { + return explorerstore.NewActivationStore(client, explorerSchema, datasetSchema, publicationSchema) +} + +func NewInterpretationStore(client *Client, schema string) (*InterpretationStore, error) { + return explorerstore.NewInterpretationStore(client, schema) +} + +func NewCapabilitySnapshotStore(client *Client, schema string) (*CapabilitySnapshotStore, error) { + return explorerstore.NewCapabilitySnapshotStore(client, schema) +} + +func NewTableShapeCapabilityStore(client *Client, schema string) (*TableShapeCapabilityStore, error) { + return explorerstore.NewTableShapeCapabilityStore(client, schema) +} + +func NewDatasetReleaseStore(src ConnSource, schema string) (*DatasetReleaseStore, error) { + return datasetstore.NewReleaseStore(src, schema) +} + +func NewBundleCatalog(client *Client, schema string) (*BundleCatalog, error) { + return publicationstore.NewBundleCatalog(&leasedDB{client: client}, schema) +} + +func NewPublicationReconciler(registry *BundleRegistry, pins *BundleReadPins, releases publicationcontract.ActiveReleaseResolver) (*PublicationReconciler, error) { + return publicationstore.NewReconciler(registry, pins, releases) +} + +func NewRecipeExecRegistry(client *Client, schema string) (*RecipeExecRegistry, error) { + return recipeexecstore.New(&leasedDB{client: client}, schema) +} + +func NewRecipeRevisionRegistry(client *Client, schema string) (*RecipeRevisionRegistry, error) { + return recipesstore.New(&leasedDB{client: client}, schema) +} + +func NewChartifactCatalog(client *Client, schema string) (*ChartifactCatalog, error) { + return chartifactstore.NewCatalog(&leasedDB{client: client}, schema) +} diff --git a/internal/backends/postgres/doc.go b/internal/backends/postgres/doc.go new file mode 100644 index 000000000..bf560bea2 --- /dev/null +++ b/internal/backends/postgres/doc.go @@ -0,0 +1 @@ +package postgres diff --git a/internal/backends/postgres/engine_postgres_command_boundary_pg18_test.go b/internal/backends/postgres/engine_postgres_command_boundary_pg18_test.go new file mode 100644 index 000000000..20851709c --- /dev/null +++ b/internal/backends/postgres/engine_postgres_command_boundary_pg18_test.go @@ -0,0 +1,227 @@ +package postgres_test + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + dataframepostgres "github.com/calypr/loom/internal/backends/postgres/internal/rows" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/execution" +) + +const verifyArch01Rounds = 10 + +func TestVerifyArch01PostgreSQLDispatchZeroChurnRealServer(t *testing.T) { + s := newExec02Service(t) + s.migrate(t) + s.seedFixture(t, "fhir_resource") + queries := 0 + var mu sync.Mutex + counting := &exec02CountingQuerier{inner: s.iso, mu: &mu, count: &queries} + executor, err := dataframepostgres.NewExecutor(counting) + if err != nil { + t.Fatalf("NewExecutor: %v", err) + } + config := exec02EngineConfig(s) + config.SourceRows = func(ctx context.Context, request execution.RowsRequest, visit execution.RowVisitor) error { + if _, _, err := request.Command.PostgreSQLParameters(); err != nil { + return err + } + return executor.SourceRows(ctx, request, visit) + } + engine, err := execution.New(config) + if err != nil { + t.Fatalf("New: %v", err) + } + if _, err := engine.Run(s.ctx, exec02Resolved("resources", "P1", "gen-a", false)); err != nil { + t.Fatalf("warm-up Run: %v", err) + } + mu.Lock() + warmQueries := queries + mu.Unlock() + if warmQueries != 1 { + t.Fatalf("warm-up stream cost %d statement round-trips, want exactly 1", warmQueries) + } + + scanWitnessSQL := `SELECT COALESCE(seq_tup_read, 0) + COALESCE(idx_tup_fetch, 0) FROM pg_stat_user_tables WHERE schemaname = $1 AND relname = 'fhir_resource'` + readServed := func() int64 { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + var served int64 + if err := s.iso.QueryRow(s.ctx, scanWitnessSQL, s.schema).Scan(&served); err != nil { + t.Fatalf("read scan-churn witness: %s", s.scrub(err)) + } + last := served + for { + if time.Now().After(deadline) { + return last + } + time.Sleep(50 * time.Millisecond) + var current int64 + if err := s.iso.QueryRow(s.ctx, scanWitnessSQL, s.schema).Scan(¤t); err != nil { + t.Fatalf("read scan-churn witness: %s", s.scrub(err)) + } + if current != last { + return current + } + } + } + beforeCalibration := readServed() + if _, err := engine.Run(s.ctx, exec02Resolved("resources", "P1", "gen-a", false)); err != nil { + t.Fatalf("calibration Run: %v", err) + } + afterCalibration := readServed() + perStream := afterCalibration - beforeCalibration + if perStream <= 0 { + t.Fatalf("calibration dispatch served %d fixture tuples, want a positive witness", perStream) + } + const measured = 4 + for i := 0; i < measured; i++ { + if _, err := engine.Run(s.ctx, exec02Resolved("resources", "P1", "gen-a", false)); err != nil { + t.Fatalf("stream %d: %v", i, err) + } + } + mu.Lock() + measuredQueries := queries - warmQueries + mu.Unlock() + if measuredQueries != 1+measured { + t.Fatalf("%d dispatched commands cost %d statement round-trips, want exactly %d (zero churn)", + 1+measured, measuredQueries, 1+measured) + } + if served := readServed(); served-afterCalibration != int64(measured)*perStream { + t.Fatalf("the %d measured streams served %d fixture tuples, want exactly %d (%d per dispatch, zero churn)", + measured, served-afterCalibration, int64(measured)*perStream, perStream) + } +} + +func TestVerifyArch01PostgreSQLBoundaryRaceDistinctBackendsOverlapRounds(t *testing.T) { + s := newExec02Service(t) + s.migrate(t) + s.seedFixture(t, "fhir_resource") + for round := 0; round < verifyArch01Rounds; round++ { + var bothInFlight sync.WaitGroup + bothInFlight.Add(2) + barrier := func() { + bothInFlight.Done() + bothInFlight.Wait() + } + type sideResult struct { + backend int64 + err error + } + leftDone := make(chan sideResult, 1) + rightDone := make(chan sideResult, 1) + go func() { + ctx, cancel := context.WithTimeout(s.ctx, 60*time.Second) + defer cancel() + backend, _, err := s.raceStream(ctx, fmt.Sprintf("boundary-round-%d-left", round), barrier) + leftDone <- sideResult{backend, err} + }() + go func() { + ctx, cancel := context.WithTimeout(s.ctx, 60*time.Second) + defer cancel() + backend, _, err := s.raceStream(ctx, fmt.Sprintf("boundary-round-%d-right", round), barrier) + rightDone <- sideResult{backend, err} + }() + left, right := <-leftDone, <-rightDone + if left.err != nil { + t.Fatalf("round %d left: %v", round, left.err) + } + if right.err != nil { + t.Fatalf("round %d right: %v", round, right.err) + } + if left.backend == right.backend || left.backend <= 0 || right.backend <= 0 { + t.Fatalf("round %d: concurrent dispatches shared a backend (%d, %d), want two distinct server pids", + round, left.backend, right.backend) + } + } +} + +func TestVerifyArch01PostgreSQLNamedRefusalsTouchNoServer(t *testing.T) { + s := newExec02Service(t) + s.migrate(t) + s.seedFixture(t, "fhir_resource") + queries := 0 + var mu sync.Mutex + counting := &exec02CountingQuerier{inner: s.iso, mu: &mu, count: &queries} + executor, err := dataframepostgres.NewExecutor(counting) + if err != nil { + t.Fatalf("NewExecutor: %v", err) + } + + crossTag, err := command.NewAQLCommand("FOR d IN x RETURN d", nil) + if err != nil { + t.Fatalf("forge AQL command: %v", err) + } + if err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: crossTag, DeclaredColumns: []string{"id"}}, func(map[string]any) error { return nil }); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("adapter-level cross tag: err = %v, want %v", err, command.ErrCompiledCommandBackendMismatch) + } + if err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: command.CompiledCommand{}, DeclaredColumns: []string{"id"}}, func(map[string]any) error { return nil }); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("adapter-level zero tag: err = %v, want %v", err, command.ErrCompiledCommandBackendNotRegistered) + } + + mu.Lock() + touched := queries + mu.Unlock() + if touched != 0 { + t.Fatalf("refused commands reached the real server %d time(s), want 0", touched) + } +} + +func TestVerifyArch01PostgreSQLCrashBeforeDriverNeverReachesServer(t *testing.T) { + s := newExec02Service(t) + s.migrate(t) + s.seedFixture(t, "fhir_resource") + queries := 0 + var mu sync.Mutex + counting := &exec02CountingQuerier{inner: s.iso, mu: &mu, count: &queries} + executor, err := dataframepostgres.NewExecutor(counting) + if err != nil { + t.Fatalf("NewExecutor: %v", err) + } + config := exec02EngineConfig(s) + config.SourceRows = func(ctx context.Context, request execution.RowsRequest, visit execution.RowVisitor) error { + if _, _, err := request.Command.PostgreSQLParameters(); err != nil { + return err + } + crashCtx, cancel := context.WithCancel(ctx) + cancel() + if err := crashCtx.Err(); err != nil { + return err + } + return executor.SourceRows(crashCtx, request, visit) + } + engine, err := execution.New(config) + if err != nil { + t.Fatalf("New: %v", err) + } + summary, err := engine.PreviewOutput(s.ctx, exec02Resolved("resources", "P1", "gen-a", false), + execution.PreviewRequest{Output: "resources", Limit: 10}, func(map[string]any) error { return nil }) + if err == nil || !errors.Is(err, context.Canceled) { + t.Fatalf("crashed dispatch must surface cancellation: %v", err) + } + if summary.Complete || summary.Truncated { + t.Fatalf("crashed preview must carry neither completion marker: %+v", summary) + } + mu.Lock() + touched := queries + mu.Unlock() + if touched != 0 { + t.Fatalf("crashed dispatch reached the driver %d time(s), want 0", touched) + } + clean, err := execution.New(exec02EngineConfig(s)) + if err != nil { + t.Fatalf("New clean: %v", err) + } + rows, err := clean.Run(s.ctx, exec02Resolved("resources", "P1", "gen-a", false)) + if err != nil { + t.Fatalf("post-crash Run: %v", err) + } + if len(rows["resources"]) != 3 { + t.Fatalf("post-crash stream carried %d rows, want 3", len(rows["resources"])) + } +} diff --git a/internal/backends/postgres/engine_postgres_pg18_test.go b/internal/backends/postgres/engine_postgres_pg18_test.go new file mode 100644 index 000000000..a8eab0e6d --- /dev/null +++ b/internal/backends/postgres/engine_postgres_pg18_test.go @@ -0,0 +1,1192 @@ +package postgres_test + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + dataframepostgres "github.com/calypr/loom/internal/backends/postgres/internal/rows" + storepg "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/execution" + "github.com/calypr/loom/internal/dataframe/recipe" + exec "github.com/calypr/loom/internal/dataframe/recipe/exec" + "github.com/calypr/loom/internal/dataframe/semantic" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + exec02Prefix = "verify_exec02_" + exec02NameFmt = exec02Prefix + "%d_%s" +) + +var exec02NamePattern = regexp.MustCompile(`^verify_exec02_[0-9]+_[0-9a-f]+$`) + +var ( + exec02OwnedMu sync.Mutex + exec02OwnedByName = map[string]string{} +) + +func exec02Register(db, schema string) { + exec02OwnedMu.Lock() + defer exec02OwnedMu.Unlock() + exec02OwnedByName[db] = schema +} + +func exec02RegistrySnapshot() map[string]string { + exec02OwnedMu.Lock() + defer exec02OwnedMu.Unlock() + out := make(map[string]string, len(exec02OwnedByName)) + for k, v := range exec02OwnedByName { + out[k] = v + } + return out +} + +var ( + exec02BaselineOnce sync.Once + exec02BaselineNames []string + exec02BaselineErr error +) + +func exec02CaptureBaseline(raw string) { + exec02BaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec02BaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec02\\_%' ESCAPE '\\'") + if err != nil { + exec02BaselineErr = fmt.Errorf("read baseline: %w", err) + return + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + exec02BaselineErr = fmt.Errorf("scan baseline: %w", err) + return + } + exec02BaselineNames = append(exec02BaselineNames, name) + } + }) +} + +func exec02URL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 dispatch integration tests") + } + return raw +} + +type exec02Service struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + schema string + dsn string + target compiler.Target + scrub func(any) string + fail func(step string, err error) + executor *dataframepostgres.Executor + lastRace atomic.Int64 +} + +func (s *exec02Service) lastRacePID() int64 { return s.lastRace.Load() } + +func newExec02Service(t *testing.T) *exec02Service { + t.Helper() + raw := exec02URL(t) + exec02CaptureBaseline(raw) + if exec02BaselineErr != nil { + t.Fatalf("capture pg_database baseline: %v", exec02BaselineErr) + } + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, + "SELECT current_setting('server_version_num')::int8, current_setting('server_version')"). + Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if err := storepg.CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required PostgreSQL 18 backend: %v", err) + } + + dbName := fmt.Sprintf(exec02NameFmt, time.Now().UnixNano(), exec02RandomHex(t)) + if !exec02NamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + var created int64 + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", dbName).Scan(&created); err != nil { + fail("verify isolated database creation", err) + } + if created != 1 { + t.Fatalf("isolated database %q did not verifiably come into existence", dbName) + } + exec02Register(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(exec02NameFmt, time.Now().UnixNano(), exec02RandomHex(t)) + if !exec02NamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + exec02Register(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + executor, err := dataframepostgres.NewExecutor(iso) + if err != nil { + fail("create PostgreSQL row executor", err) + } + binding, err := ir.NewPostgresStorageBinding(schema) + if err != nil { + fail("bind PostgreSQL storage placement", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + fail("select PostgreSQL compile target", err) + } + return &exec02Service{ctx: ctx, admin: admin, iso: iso, schema: schema, dsn: dsn, target: target, scrub: scrub, fail: fail, executor: executor} +} + +func exec02RandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +func (s *exec02Service) qualifiedTable(name string) string { + return `"` + s.schema + `"."` + name + `"` +} + +func exec02ForbidArango() execution.Config { + return execution.Config{ + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + SourceRows: func(_ context.Context, request execution.RowsRequest, _ execution.RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + + return errors.New("AQL QueryRows must never run for a PostgreSQL-served plan") + }, + PreviewSourceRows: func(_ context.Context, request execution.RowsRequest, _ execution.RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + + return errors.New("AQL PreviewQueryRows must never run for a PostgreSQL-served plan") + }, + PreparePreviewIndex: func(context.Context, compiler.PreviewCoveringIndexSpec) error { + return errors.New("preparePreviewIndex must never run for a PostgreSQL-served plan") + }, + PreviewCoveringIndexes: func(context.Context, string) ([]execution.PreviewCollectionIndex, error) { + return nil, errors.New("previewCoveringIndexes must never run for a PostgreSQL-served plan") + }, + PreviewCollectionRevision: func(context.Context, string) (string, error) { + return "", errors.New("previewCollectionRevision must never run for a PostgreSQL-served plan") + }, + PreviewCollectionScans: func(context.Context, command.CompiledCommand, string) (execution.PreviewCollectionScanEvidence, error) { + return execution.PreviewCollectionScanEvidence{}, errors.New("previewCollectionScans must never run for a PostgreSQL-served plan") + }, + PreviewCollectionCount: func(context.Context, string) (int64, error) { + return 0, errors.New("previewCollectionCount must never run for a PostgreSQL-served plan") + }, + ClickHouseQueryRows: func(context.Context, string, []string, func(map[string]any) error, ...any) error { + return errors.New("ClickHouse callbacks must never run for a PostgreSQL-served plan") + }, + } +} + +func exec02EngineConfig(s *exec02Service) execution.Config { + config := exec02ForbidArango() + config.CompileTarget = s.target + config.PreviewSourceRows = nil + config.SourceRows = func(ctx context.Context, request execution.RowsRequest, visit execution.RowVisitor) error { + if _, _, err := request.Command.PostgreSQLParameters(); err != nil { + return err + } + return s.executor.SourceRows(ctx, request, visit) + } + return config +} + +func exec02Resolved(outputName, project, generation string, withBackend bool) execution.Resolved { + columns := []string{"project", "dataset_generation", "resource_type", "logical_id", "auth_resource_path"} + schema := []lower.CompiledOutputColumn{ + {ID: "project", Name: "project", Label: "project", Kind: "string", Cardinality: "one"}, + {ID: "dataset_generation", Name: "dataset_generation", Label: "dataset_generation", Kind: "string", Cardinality: "one"}, + {ID: "resource_type", Name: "resource_type", Label: "resource_type", Kind: "string", Cardinality: "one"}, + {ID: "logical_id", Name: "logical_id", Label: "logical_id", Kind: "string", Cardinality: "one"}, + {ID: "auth_resource_path", Name: "auth_resource_path", Label: "auth_resource_path", Kind: "string", Cardinality: "one"}, + } + projections := make([]ir.PhysicalProjection, 0, len(columns)+1) + for _, column := range columns { + projections = append(projections, ir.PhysicalProjection{ + Name: column, + Value: ir.PhysicalValue{Variable: "root", Path: []string{column}}, + }) + } + if withBackend { + backend := ir.PhysicalProjection{Name: "backend", Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Patient", + Selector: exec02WitnessSelector(), + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + }, Presence: &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + Paths: [][]string{{"gender"}}, + }} + projections = append([]ir.PhysicalProjection{backend}, projections...) + columns = append([]string{"backend"}, columns...) + schema = append([]lower.CompiledOutputColumn{{ID: "backend", Name: "backend", Label: "backend", Kind: "string", Cardinality: "one"}}, schema...) + } + output := lower.CompiledRecipeOutput{ + Name: outputName, + RootResourceType: "Patient", + Columns: columns, + OutputSchema: schema, + RowIdentity: &spec.RowIdentity{Fields: []string{"logical_id"}}, + Plan: ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": "Patient", + "project": project, + "dataset_generation": generation, + "auth_resource_paths": []string{"FHIR/Patient"}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + }, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"project"}}, Right: &ir.PhysicalValue{BindKey: "project"}}}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"dataset_generation"}}, Right: &ir.PhysicalValue{BindKey: "dataset_generation"}}}}, + {Kind: ir.PhysicalDerivedLetOp, DerivedLet: &ir.PhysicalDerivedLet{Variable: "root_auth_allowed", Operator: "AUTH_RESOURCE_PATH_ALLOWED", Inputs: []ir.PhysicalValue{ + {Variable: "root", Path: []string{"auth_resource_path"}}, + {BindKey: "auth_resource_paths"}, + {BindKey: "auth_resource_paths_unrestricted"}, + }}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root_auth_allowed"}, Right: &ir.PhysicalValue{BindKey: "scope_allowed"}}}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}}, + }, + }, + } + return execution.Resolved{ + Bundle: recipe.Bundle{Name: outputName}, + Semantic: semantic.ResolvedRecipePlan{ + SemanticPlan: semantic.RecipePlan{Bindings: recipe.RuntimeBindings{ + Project: project, DatasetGeneration: generation, OutputNames: []string{outputName}, + }}, + }, + Compiled: lower.CompiledRecipe{Outputs: []lower.CompiledRecipeOutput{output}}, + StoredRecipeDigest: "exec02-real-" + outputName, + } +} + +func (s *exec02Service) migrate(t *testing.T) { + t.Helper() + client, err := storepg.Open(s.ctx, storepg.Config{ + DSN: s.dsn, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 2 * time.Second, + }) + if err != nil { + s.fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + defer client.Close() + if _, err := storepg.MigrateSchema(s.ctx, client, s.schema); err != nil { + s.fail("migrate owned schema", err) + } +} + +func (s *exec02Service) seedFixture(t *testing.T, name string) { + t.Helper() + if name != "fhir_resource" { + t.Fatalf("exec02 fixtures seed only the bound resource placement, got %q", name) + } + repo, err := storepg.NewResourceRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + rows := []struct { + project, gen, rtype, logical, auth string + payload string + }{ + {"P1", "gen-a", "Patient", "patient-ada", "FHIR/Patient", `{"name": "Ada", "middle": null}`}, + {"P1", "gen-a", "Patient", "patient-grace", "FHIR/Patient", `{"name": "Grace"}`}, + {"P1", "gen-a", "Patient", "patient-alan", "FHIR/Patient", `null`}, + {"P1", "gen-b", "Patient", "patient-other-gen", "FHIR/Patient", `{"name": "Other"}`}, + {"other", "gen-a", "Patient", "patient-other-project", "FHIR/Patient", `{"name": "Intruder"}`}, + } + for _, row := range rows { + _, err := repo.InsertResource(s.ctx, source.Resource{ + Project: row.project, + DatasetGeneration: row.gen, + ResourceType: row.rtype, + LogicalID: row.logical, + AuthResourcePath: row.auth, + Payload: json.RawMessage(row.payload), + }) + if err != nil { + s.fail("insert fixture row", err) + } + } +} + +func (s *exec02Service) countRows(t *testing.T, name string) int64 { + t.Helper() + var count int64 + if err := s.iso.QueryRow(s.ctx, `SELECT count(*) FROM `+s.qualifiedTable(name)).Scan(&count); err != nil { + s.fail("count fixture rows", err) + } + return count +} + +func (s *exec02Service) convergePreview(engine *execution.Engine, resolved execution.Resolved, limit int, newVisitor func() func(map[string]any) error) (execution.PreviewSummary, error) { + var summary execution.PreviewSummary + var err error + for attempt := 1; attempt <= 6; attempt++ { + if attempt > 1 { + select { + case <-time.After(150 * time.Millisecond): + case <-s.ctx.Done(): + return summary, s.ctx.Err() + } + } + summary, err = engine.PreviewOutput(s.ctx, resolved, execution.PreviewRequest{Output: "resources", Limit: limit}, newVisitor()) + if err == nil { + return summary, nil + } + } + return summary, err +} + +func exec02DigestRows(rows []map[string]any) string { + encoded, err := json.Marshal(rows) + if err != nil { + return "unencodable:" + err.Error() + } + return string(encoded) +} + +func exec02WitnessSelector() spec.Selector { + selector, err := spec.ParseSelector("gender") + if err != nil { + panic(err) + } + return selector +} + +func exec02WitnessPID(value any) (int64, bool) { + text, ok := value.(string) + if !ok { + var raw []byte + switch typed := value.(type) { + case json.RawMessage: + raw = typed + case []byte: + raw = typed + default: + return 0, false + } + if err := json.Unmarshal(raw, &text); err != nil { + return 0, false + } + } + if !strings.HasPrefix(text, "pg_") { + return 0, false + } + pid, err := strconv.ParseInt(strings.TrimPrefix(text, "pg_"), 10, 64) + if err != nil || pid <= 0 { + return 0, false + } + return pid, true +} + +func exec02StableRowIdentity(parts ...any) string { + encoded, err := json.Marshal(parts) + if err != nil { + panic(err) + } + sum := sha256.Sum256(encoded) + return hex.EncodeToString(sum[:]) +} + +func exec02AsInt64(v any) (int64, bool) { + switch value := v.(type) { + case int64: + return value, true + case int32: + return int64(value), true + default: + return 0, false + } +} + +func exec02GoldenRows(withIdentity bool) []map[string]any { + rows := []map[string]any{ + { + "id": int64(1), "logical_id": "patient-ada", "score": 3.5, "active": true, + "payload": json.RawMessage(`{"name": "Ada", "middle": null}`), "note": "note-ada", + }, + { + "id": int64(3), "logical_id": "patient-alan", "score": 1.125, "active": true, + "payload": json.RawMessage(`null`), "note": "note-alan", + }, + { + "id": int64(2), "logical_id": "patient-grace", "score": 2.25, "active": false, + "payload": json.RawMessage(`{"name": "Grace"}`), "note": nil, + }, + } + if withIdentity { + for _, row := range rows { + row["__loom_row_id"] = exec02StableRowIdentity(row["logical_id"]) + } + } + return rows +} + +func exec02ScopedRowsPublic() []map[string]any { + return []map[string]any{ + { + "project": "P1", "dataset_generation": "gen-a", "resource_type": "Patient", + "logical_id": "patient-ada", "auth_resource_path": "FHIR/Patient", + }, + { + "project": "P1", "dataset_generation": "gen-a", "resource_type": "Patient", + "logical_id": "patient-alan", "auth_resource_path": "FHIR/Patient", + }, + { + "project": "P1", "dataset_generation": "gen-a", "resource_type": "Patient", + "logical_id": "patient-grace", "auth_resource_path": "FHIR/Patient", + }, + } +} + +func exec02ScopedRows() []map[string]any { + rows := exec02ScopedRowsPublic() + for _, row := range rows { + row["__loom_row_id"] = exec02StableRowIdentity(row["logical_id"]) + } + return rows +} + +func TestPostgresEngineExecutesCompiledCommandThroughRealAdapter(t *testing.T) { + s := newExec02Service(t) + s.migrate(t) + s.seedFixture(t, "fhir_resource") + engine, err := execution.New(exec02EngineConfig(s)) + if err != nil { + t.Fatalf("New: %v", err) + } + streams, err := engine.Streams(s.ctx, exec02Resolved("resources", "P1", "gen-a", false)) + if err != nil { + t.Fatalf("Streams: %v", err) + } + if len(streams) != 1 { + t.Fatalf("streams = %d, want 1", len(streams)) + } + var got []map[string]any + result, err := streams[0].Stream(s.ctx, func(row map[string]any) error { + got = append(got, row) + return nil + }) + if err != nil { + t.Fatalf("Stream: %v", err) + } + if result.RowCount != 3 || len(got) != 3 { + t.Fatalf("rows=%d visited=%d, want 3/3 (project/generation scoping leaked or was not applied)", result.RowCount, len(got)) + } + if got[1]["logical_id"] == "patient-other-project" || got[1]["logical_id"] == "patient-other-gen" { + t.Fatalf("scoped execution leaked another project/generation: %v", got) + } + if want := exec02DigestRows(exec02ScopedRows()); exec02DigestRows(got) != want { + t.Fatalf("streamed witness drifted from the committed resources:\n got %s\nwant %s", exec02DigestRows(got), want) + } + for _, row := range got { + for _, column := range []string{"project", "dataset_generation", "resource_type", "logical_id", "auth_resource_path"} { + if _, ok := row[column].(string); !ok { + t.Fatalf("scope column %q must decode as a string: %#v", column, row[column]) + } + } + } +} + +func TestPostgresEnginePreviewBoundsCompletionOnRealServer(t *testing.T) { + s := newExec02Service(t) + s.migrate(t) + s.seedFixture(t, "fhir_resource") + engine, err := execution.New(exec02EngineConfig(s)) + if err != nil { + t.Fatalf("New: %v", err) + } + resolved := exec02Resolved("resources", "P1", "gen-a", false) + + var bounded []map[string]any + summary, err := engine.PreviewOutput(s.ctx, resolved, + execution.PreviewRequest{Output: "resources", Limit: 2}, func(row map[string]any) error { + bounded = append(bounded, row) + return nil + }) + if err != nil { + t.Fatalf("bounded PreviewOutput: %v", err) + } + if summary.RowCount != 2 || summary.Complete || !summary.Truncated { + t.Fatalf("limit-2 preview on 3 rows: rowcount=%d complete=%v truncated=%v", summary.RowCount, summary.Complete, summary.Truncated) + } + if bounded[0]["logical_id"] != "patient-ada" || bounded[1]["logical_id"] != "patient-alan" { + t.Fatalf("preview order drifted: %v", bounded) + } + + var all []map[string]any + summary, err = engine.PreviewOutput(s.ctx, resolved, + execution.PreviewRequest{Output: "resources", Limit: 10}, func(row map[string]any) error { + all = append(all, row) + return nil + }) + if err != nil { + t.Fatalf("full PreviewOutput: %v", err) + } + if summary.RowCount != 3 || !summary.Complete || summary.Truncated { + t.Fatalf("limit-10 preview on 3 rows: rowcount=%d complete=%v truncated=%v", summary.RowCount, summary.Complete, summary.Truncated) + } +} + +func TestPostgresEnginePreviewCancellationRealServer(t *testing.T) { + s := newExec02Service(t) + s.migrate(t) + s.seedFixture(t, "fhir_resource") + engine, err := execution.New(exec02EngineConfig(s)) + if err != nil { + t.Fatalf("New: %v", err) + } + resolved := exec02Resolved("resources", "P1", "gen-a", false) + + ctx, cancel := context.WithCancel(s.ctx) + defer cancel() + visits := 0 + summary, err := engine.PreviewOutput(ctx, resolved, + execution.PreviewRequest{Output: "resources", Limit: 10}, func(row map[string]any) error { + visits++ + cancel() + return nil + }) + if err == nil { + t.Fatalf("canceled preview must not complete: summary=%+v", summary) + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled preview error must satisfy errors.Is(context.Canceled): %v", err) + } + if visits != 1 { + t.Fatalf("visitor continued after cancellation: %d visits", visits) + } + if summary.Complete || summary.Truncated { + t.Fatalf("canceled preview emitted a completion marker: %+v", summary) + } + + var witness []map[string]any + rerun, err := s.convergePreview(engine, resolved, 10, func() func(map[string]any) error { + witness = nil + return func(row map[string]any) error { + witness = append(witness, row) + return nil + } + }) + if err != nil { + t.Fatalf("post-abort rerun: %v", err) + } + if !rerun.Complete || rerun.Truncated { + t.Fatalf("post-abort rerun must converge complete: %+v", rerun) + } + if want := exec02DigestRows(exec02ScopedRowsPublic()); exec02DigestRows(witness) != want { + t.Fatalf("post-abort witness differs from committed golden:\n got %s\nwant %s", exec02DigestRows(witness), want) + } +} + +func TestPostgresEngineTwoRealConnectionRaceDistinctBackends(t *testing.T) { + s := newExec02Service(t) + s.migrate(t) + s.seedFixture(t, "fhir_resource") + + ctxForRound := func() context.Context { + ctx, cancel := context.WithTimeout(s.ctx, 60*time.Second) + _ = cancel + return ctx + } + var stableWitness string + for round := 0; round < 10; round++ { + type sideResult struct { + backend int64 + digest string + err error + } + var bothInFlight sync.WaitGroup + bothInFlight.Add(2) + barrier := func() { + bothInFlight.Done() + bothInFlight.Wait() + } + leftDone := make(chan sideResult, 1) + rightDone := make(chan sideResult, 1) + go func() { + backend, digest, err := s.raceStream(ctxForRound(), fmt.Sprintf("round-%d-left", round), barrier) + leftDone <- sideResult{backend, digest, err} + }() + go func() { + backend, digest, err := s.raceStream(ctxForRound(), fmt.Sprintf("round-%d-right", round), barrier) + rightDone <- sideResult{backend, digest, err} + }() + left, right := <-leftDone, <-rightDone + if left.err != nil { + t.Fatal(left.err) + } + if right.err != nil { + t.Fatal(right.err) + } + if left.backend == right.backend { + t.Fatalf("round %d: both concurrent streams ran on the same backend pid %d", round, left.backend) + } + if left.digest != right.digest { + t.Fatalf("round %d: concurrent witnesses differ:\nleft %s\nright %s", round, left.digest, right.digest) + } + if stableWitness == "" { + stableWitness = left.digest + } else if left.digest != stableWitness { + t.Fatalf("round %d: witness drifted across rounds:\n got %s\nwant %s", round, left.digest, stableWitness) + } + } + + abortCtx, abortCancel := context.WithCancel(s.ctx) + if _, _, err := s.raceStreamAborted(abortCtx, abortCancel); err != nil { + t.Fatalf("aborting stream: %v", err) + } + abortCancel() + var backend int64 + var digest string + var runErr error + for attempt := 1; attempt <= 6; attempt++ { + if attempt > 1 { + select { + case <-time.After(150 * time.Millisecond): + case <-s.ctx.Done(): + t.Fatalf("post-abort convergence window expired: %v", s.ctx.Err()) + } + } + backend, digest, runErr = s.raceStream(ctxForRound(), fmt.Sprintf("post-abort-%d", attempt), nil) + if runErr == nil { + break + } + } + if runErr != nil { + t.Fatal(runErr) + } + if backend <= 0 || digest != stableWitness { + t.Fatalf("post-abort witness differs: pid=%d\n got %s\nwant %s", backend, digest, stableWitness) + } +} + +func TestPostgresEngineParameterizedValuesCannotChangeShape(t *testing.T) { + s := newExec02Service(t) + s.migrate(t) + s.seedFixture(t, "fhir_resource") + before := s.countRows(t, "fhir_resource") + injection := `x'; DROP TABLE "` + s.schema + `"."fhir_resource"; --` + engine, err := execution.New(exec02EngineConfig(s)) + if err != nil { + t.Fatalf("New: %v", err) + } + rows, err := engine.Run(s.ctx, exec02Resolved("resources", injection, "gen-a", false)) + if err != nil { + t.Fatalf("Run with injection payload: %v", err) + } + if len(rows["resources"]) != 0 { + t.Fatalf("injection payload matched %d rows; it must remain a value", len(rows["resources"])) + } + if after := s.countRows(t, "fhir_resource"); after != before { + t.Fatalf("table contents changed under a bound payload: %d rows, want %d", after, before) + } + streams, err := engine.Streams(s.ctx, exec02Resolved("resources", "P1", "gen-a", false)) + if err != nil { + t.Fatalf("golden Streams: %v", err) + } + if len(streams) != 1 { + t.Fatalf("golden streams = %d, want 1", len(streams)) + } + var golden []map[string]any + if _, err := streams[0].Stream(s.ctx, func(row map[string]any) error { + golden = append(golden, row) + return nil + }); err != nil { + t.Fatalf("golden Stream: %v", err) + } + if want := exec02DigestRows(exec02ScopedRows()); exec02DigestRows(golden) != want { + t.Fatalf("golden witness changed after injection probe:\n got %s\nwant %s", exec02DigestRows(golden), want) + } +} + +func (s *exec02Service) raceStream(ctx context.Context, label string, overlap func()) (int64, string, error) { + conn, err := s.iso.Acquire(ctx) + if err != nil { + return 0, "", fmt.Errorf("%s acquire: %w", label, err) + } + defer conn.Release() + tx, err := conn.Begin(ctx) + if err != nil { + return 0, "", fmt.Errorf("%s begin: %w", label, err) + } + defer tx.Rollback(context.WithoutCancel(ctx)) + var pid int64 + if err := tx.QueryRow(ctx, `SELECT pg_backend_pid()`).Scan(&pid); err != nil { + return 0, "", fmt.Errorf("%s backend pid: %w", label, err) + } + if _, err := tx.Exec(ctx, + `INSERT INTO `+s.qualifiedTable("fhir_resource")+` (project, dataset_generation, resource_type, logical_id, auth_resource_path, payload, recorded_at) + VALUES ($1,$2,$3,$4,$5,$6::jsonb, now())`, + "P1", "gen-a", "Patient", fmt.Sprintf("witness-%d", pid), "FHIR/Patient", fmt.Sprintf(`{"gender": "pg_%d"}`, pid)); err != nil { + return 0, "", fmt.Errorf("%s insert witness: %w", label, err) + } + queries := 0 + var mu sync.Mutex + streamExecutor, err := dataframepostgres.NewExecutor(&exec02CountingQuerier{inner: tx, mu: &mu, count: &queries}) + if err != nil { + return 0, "", fmt.Errorf("%s executor: %w", label, err) + } + config := exec02EngineConfig(s) + config.SourceRows = func(ctx context.Context, request execution.RowsRequest, visit execution.RowVisitor) error { + if _, _, err := request.Command.PostgreSQLParameters(); err != nil { + return err + } + return streamExecutor.SourceRows(ctx, request, visit) + } + engine, err := execution.New(config) + if err != nil { + return 0, "", fmt.Errorf("%s engine: %w", label, err) + } + streams, err := engine.Streams(ctx, exec02Resolved("resources", "P1", "gen-a", true)) + if err != nil { + return 0, "", fmt.Errorf("%s Streams: %w", label, err) + } + if len(streams) != 1 { + return 0, "", fmt.Errorf("%s streams = %d, want 1", label, len(streams)) + } + var backend int64 + var rows []map[string]any + if _, err := streams[0].Stream(ctx, func(row map[string]any) error { + if witnessed, ok := exec02WitnessPID(row["backend"]); ok && backend == 0 { + backend = witnessed + if overlap != nil { + overlap() + } + } + delete(row, "backend") + if logical, _ := row["logical_id"].(string); !strings.HasPrefix(logical, "witness-") { + rows = append(rows, row) + } + return nil + }); err != nil { + return 0, "", fmt.Errorf("%s Stream: %w", label, err) + } + if backend != pid { + return 0, "", fmt.Errorf("%s witnessed backend %d, seeded %d", label, backend, pid) + } + s.lastRace.Store(pid) + return backend, exec02DigestRows(rows), nil +} + +type exec02CountingQuerier struct { + inner dataframepostgres.Querier + mu *sync.Mutex + count *int +} + +func (c *exec02CountingQuerier) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { + c.mu.Lock() + *c.count++ + c.mu.Unlock() + return c.inner.Query(ctx, sql, args...) +} + +func TestPostgresEngineCrossTagRefusedBeforeServer(t *testing.T) { + s := newExec02Service(t) + s.migrate(t) + s.seedFixture(t, "fhir_resource") + queries := 0 + var mu sync.Mutex + counting := &exec02CountingQuerier{inner: s.iso, mu: &mu, count: &queries} + executor, err := dataframepostgres.NewExecutor(counting) + if err != nil { + t.Fatalf("NewExecutor: %v", err) + } + config := exec02ForbidArango() + config.CompileTarget = compiler.ArangoTarget() + config.SourceRows = func(ctx context.Context, request execution.RowsRequest, visit execution.RowVisitor) error { + return executor.SourceRows(ctx, request, visit) + } + engine, err := execution.New(config) + if err != nil { + t.Fatalf("New: %v", err) + } + _, err = engine.Run(s.ctx, exec02Resolved("resources", "P1", "gen-a", false)) + if !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("cross-tag dispatch must report the exact mismatch sentinel: %v", err) + } + if !strings.Contains(err.Error(), "refusing to pass AQL query to the PostgreSQL statement executor") { + t.Fatalf("refusal must name the routing defect exactly: %v", err) + } + mu.Lock() + touched := queries + mu.Unlock() + if touched != 0 { + t.Fatalf("refused cross-tag command reached the real server %d time(s)", touched) + } + + controlEngine, err := execution.New(exec02EngineConfig(s)) + if err != nil { + t.Fatalf("New control: %v", err) + } + rows, err := controlEngine.Run(s.ctx, exec02Resolved("resources", "P1", "gen-a", false)) + if err != nil { + t.Fatalf("control Run: %v", err) + } + if len(rows["resources"]) != 3 { + t.Fatalf("control streamed %d rows, want 3", len(rows["resources"])) + } +} + +func TestPostgresEngineBackendKillConverges(t *testing.T) { + s := newExec02Service(t) + s.migrate(t) + s.seedFixture(t, "fhir_resource") + + _, baseline, err := s.raceStream(s.ctx, "kill-baseline", nil) + if err != nil { + t.Fatalf("baseline stream: %v", err) + } + if want := exec02DigestRows(exec02ScopedRows()); baseline != want { + t.Fatalf("baseline witness differs from committed resources:\n got %s\nwant %s", baseline, want) + } + + var killed bool + if err := s.admin.QueryRow(s.ctx, `SELECT pg_terminate_backend($1)`, s.lastRacePID()).Scan(&killed); err != nil { + s.fail("pg_terminate_backend between runs", err) + } + if !killed { + t.Fatalf("pg_terminate_backend reported no kill for %d", s.lastRacePID()) + } + converge := func(label string) string { + t.Helper() + var digest string + var runErr error + for attempt := 1; attempt <= 6; attempt++ { + if attempt > 1 { + select { + case <-time.After(150 * time.Millisecond): + case <-s.ctx.Done(): + t.Fatalf("%s convergence window expired: %v", label, s.ctx.Err()) + } + } + _, digest, runErr = s.raceStream(s.ctx, fmt.Sprintf("%s-%d", label, attempt), nil) + if runErr == nil { + return digest + } + } + t.Fatalf("%s never converged: %v", label, runErr) + return "" + } + if got := converge("post-kill"); got != baseline { + t.Fatalf("post-kill witness differs from committed golden:\n got %s\nwant %s", got, baseline) + } + + streamSQL := `SELECT pg_backend_pid() AS backend, logical_id, repeat('x', 500000) AS blob, pg_sleep(0.25) IS NOT NULL AS tick FROM ` + s.qualifiedTable("fhir_resource") + ` WHERE project = $1 AND dataset_generation = $2 ORDER BY resource_seq` + compiledCommand, err := command.NewPostgreSQLCommand(streamSQL, []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "P1"}, + {Type: command.PostgreSQLParameterText, Value: "gen-a"}, + }) + if err != nil { + t.Fatalf("build kill command: %v", err) + } + adapterExecutor, err := dataframepostgres.NewExecutor(s.iso) + if err != nil { + t.Fatalf("adapter executor: %v", err) + } + var midPid int64 + var midKilled bool + midErr := adapterExecutor.SourceRows(s.ctx, execution.RowsRequest{ + Command: compiledCommand, + DeclaredColumns: []string{"backend", "logical_id", "blob", "tick"}, + }, func(row map[string]any) error { + if midPid != 0 { + return nil + } + pid, ok := exec02AsInt64(row["backend"]) + if !ok || pid <= 0 { + return errors.New("stream row carried no server backend pid witness") + } + midPid = pid + if err := s.admin.QueryRow(s.ctx, `SELECT pg_terminate_backend($1)`, pid).Scan(&midKilled); err != nil { + return fmt.Errorf("pg_terminate_backend live backend %d: %w", pid, err) + } + return nil + }) + if midPid == 0 { + t.Fatal("mid-stream run never witnessed its serving backend") + } + if !midKilled { + t.Fatalf("pg_terminate_backend(%d) reported no kill for the live backend", midPid) + } + if midErr == nil { + t.Fatalf("stream survived termination of its live backend; crash was not injected (pid %d)", midPid) + } + + faultyConfig := exec02EngineConfig(s) + faultyConfig.SourceRows = func(ctx context.Context, request execution.RowsRequest, visit execution.RowVisitor) error { + visited := 0 + return adapterExecutor.SourceRows(ctx, request, func(row map[string]any) error { + if err := visit(row); err != nil { + return err + } + visited++ + if visited == 1 { + return errors.New("injected row-port failure after first visit") + } + return nil + }) + } + faultyEngine, err := execution.New(faultyConfig) + if err != nil { + t.Fatalf("New faulty: %v", err) + } + faultySummary, faultyErr := faultyEngine.PreviewOutput(s.ctx, + exec02Resolved("resources", "P1", "gen-a", false), + execution.PreviewRequest{Output: "resources", Limit: 10}, func(map[string]any) error { return nil }) + if faultyErr == nil { + t.Fatal("port failure after first visit must fail the preview") + } + if faultySummary.Complete || faultySummary.Truncated { + t.Fatalf("failed port run carried a completion marker: %+v", faultySummary) + } + + if got := converge("post-crash"); got != baseline { + t.Fatalf("post-crash witness differs from committed golden:\n got %s\nwant %s", got, baseline) + } +} + +func TestZZZeroLeftoverExec02PostgresFixtures(t *testing.T) { + raw := exec02URL(t) + exec02CaptureBaseline(raw) + if exec02BaselineErr != nil { + t.Fatalf("capture baseline: %v", exec02BaselineErr) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer admin.Close() + + for db := range exec02RegistrySnapshot() { + var exists bool + if err := admin.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_database WHERE datname = $1)`, db).Scan(&exists); err != nil { + t.Fatalf("check registry member %s: %v", db, err) + } + if exists { + t.Fatalf("created database %s was not dropped", db) + } + } + + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec02\\_%' ESCAPE '\\'") + if err != nil { + t.Fatalf("prefix scan: %v", err) + } + var survivors []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("prefix scan scan: %v", err) + } + survivors = append(survivors, name) + } + rows.Close() + if len(survivors) != len(exec02BaselineNames) { + t.Fatalf("verify_exec02_* survivors under own prefix: got %v, baseline %v", survivors, exec02BaselineNames) + } + baselineSet := map[string]bool{} + for _, name := range exec02BaselineNames { + baselineSet[name] = true + } + for _, name := range survivors { + if !baselineSet[name] { + t.Fatalf("unregistered verify_exec02_* survivor (fatal): %s", name) + } + } +} + +func (s *exec02Service) raceStreamAborted(ctx context.Context, cancel context.CancelFunc) (int64, string, error) { + conn, err := s.iso.Acquire(ctx) + if err != nil { + return 0, "", fmt.Errorf("abort acquire: %w", err) + } + defer conn.Release() + queries := 0 + var mu sync.Mutex + streamExecutor, err := dataframepostgres.NewExecutor(&exec02CountingQuerier{inner: conn, mu: &mu, count: &queries}) + if err != nil { + return 0, "", fmt.Errorf("abort executor: %w", err) + } + config := exec02EngineConfig(s) + config.SourceRows = func(ctx context.Context, request execution.RowsRequest, visit execution.RowVisitor) error { + if _, _, err := request.Command.PostgreSQLParameters(); err != nil { + return err + } + return streamExecutor.SourceRows(ctx, request, visit) + } + engine, err := execution.New(config) + if err != nil { + return 0, "", fmt.Errorf("abort engine: %w", err) + } + streams, err := engine.Streams(ctx, exec02Resolved("resources", "P1", "gen-a", true)) + if err != nil { + return 0, "", fmt.Errorf("abort Streams: %w", err) + } + if len(streams) != 1 { + return 0, "", fmt.Errorf("abort streams = %d, want 1", len(streams)) + } + var backend int64 + firstRow := true + if _, err := streams[0].Stream(ctx, func(row map[string]any) error { + if witnessed, ok := exec02WitnessPID(row["backend"]); ok && backend == 0 { + backend = witnessed + } + if firstRow { + firstRow = false + cancel() + } + return nil + }); err != nil && !errors.Is(err, context.Canceled) { + return 0, "", fmt.Errorf("abort Stream: %w", err) + } + return backend, "", nil +} + +type invalidRecipeRegistry struct{} + +func (invalidRecipeRegistry) LoadRecipe(context.Context, string) (exec.Entry, error) { + return exec.Entry{Bundle: recipe.Bundle{ + RecipeSchemaVersion: 1, + Name: "default", + TranslationVersion: "test", + Outputs: []recipe.Output{{ + Name: "Patient", RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{{Name: "missing", Expr: recipe.Expression{Select: "root.missing"}}}, + }}, + }}, nil +} + +func (r invalidRecipeRegistry) LoadRecipeVersion(ctx context.Context, name, _ string) (exec.Entry, error) { + return r.LoadRecipe(ctx, name) +} diff --git a/internal/backends/postgres/explain_postgres_test.go b/internal/backends/postgres/explain_postgres_test.go new file mode 100644 index 000000000..d5c487632 --- /dev/null +++ b/internal/backends/postgres/explain_postgres_test.go @@ -0,0 +1,1518 @@ +package postgres_test + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + dataframepostgres "github.com/calypr/loom/internal/backends/postgres/internal/rows" + storepg "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + "github.com/calypr/loom/internal/dataframe/execution" + "github.com/calypr/loom/internal/dataframe/recipe" + recipeexec "github.com/calypr/loom/internal/dataframe/recipe/exec" + "github.com/calypr/loom/internal/dataframe/semantic" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + exec10SentinelTable = "pat_SENTINEL-TBL-9c14" + exec10SentinelIndex = "pat_resource_seq_SENTINEL-IDX-4e02" + exec10SentinelColumn = "note_SENTINEL-COL-7a51" + exec10SentinelLiteral = "SENTINEL-LIT-bf23" + exec10SentinelBind = "SENTINEL-BIND-13ad" + exec10SentinelSchema = "schema_SENTINEL-SCHEMA-de90" + exec10SentinelMissing = "missing_relation_SENTINEL-TBL-a0ff" +) + +func exec10Sentinels() []string { + return []string{ + exec10SentinelTable, + exec10SentinelIndex, + exec10SentinelColumn, + exec10SentinelLiteral, + exec10SentinelBind, + exec10SentinelSchema, + exec10SentinelMissing, + } +} + +func exec10HostileSQL(qualifiedTable string) string { + return fmt.Sprintf(`SELECT "%s" FROM %s WHERE "%s" = $1 AND "%s"::text <> '%s' ORDER BY resource_seq`, + exec10SentinelColumn, qualifiedTable, exec10SentinelColumn, exec10SentinelColumn, exec10SentinelLiteral) +} + +func exec10HostileParameters() []command.PostgreSQLParameter { + return []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: exec10SentinelBind}, + } +} + +func exec10HostileCommand(t *testing.T, qualifiedTable string) command.CompiledCommand { + t.Helper() + compiledCommand, err := command.NewPostgreSQLCommand(exec10HostileSQL(qualifiedTable), exec10HostileParameters()) + if err != nil { + t.Fatalf("build hostile PostgreSQL command: %v", err) + } + return compiledCommand +} + +func exec10AssertSentinelFree(t *testing.T, surface string, value string, extraSecrets ...string) { + t.Helper() + for _, sentinel := range exec10Sentinels() { + if strings.Contains(value, sentinel) { + t.Fatalf("%s leaks hostile sentinel %q: %s", surface, sentinel, exec10Clip(value)) + } + } + for _, secret := range extraSecrets { + if secret != "" && strings.Contains(value, secret) { + t.Fatalf("%s leaks secret material: %s", surface, exec10Clip(value)) + } + } +} + +func exec10Clip(value string) string { + if len(value) > 600 { + return value[:600] + "…" + } + return value +} + +func TestPostgresExplainBuildCommandClosedDispatch(t *testing.T) { + pgCommand := exec10HostileCommand(t, `"public"."some_table"`) + + explainCommand, err := execution.BuildPostgreSQLExplainCommand(compiler.CompiledQuery{Command: pgCommand}) + if err != nil { + t.Fatalf("execution.BuildPostgreSQLExplainCommand: %v", err) + } + if explainCommand.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("EXPLAIN command must stay PostgreSQL-tagged, got %q", explainCommand.Backend()) + } + sqlText, args, err := explainCommand.PostgreSQLParameters() + if err != nil { + t.Fatalf("EXPLAIN command dispatch: %v", err) + } + if !strings.HasPrefix(sqlText, "EXPLAIN (FORMAT JSON)\n") { + t.Fatalf("EXPLAIN prefix missing: %q", exec10Clip(sqlText)) + } + baseText, baseArgs, err := pgCommand.PostgreSQLParameters() + if err != nil { + t.Fatalf("base command dispatch: %v", err) + } + if sqlText != "EXPLAIN (FORMAT JSON)\n"+baseText { + t.Fatalf("EXPLAIN wrapper must only prepend the fixed prefix") + } + if len(args) != len(baseArgs) || args[0].Value != exec10SentinelBind { + t.Fatalf("ordered arguments must be carried unchanged: %+v", args) + } + args[0].Value = "MUTATED" + again, _, err := explainCommand.PostgreSQLParameters() + _ = again + if err != nil { + t.Fatalf("re-dispatch: %v", err) + } + _, rebound, err := explainCommand.PostgreSQLParameters() + if err != nil || rebound[0].Value != exec10SentinelBind { + t.Fatalf("argument slice must be a copy: %+v err=%v", rebound, err) + } + + aqlCommand, err := command.NewAQLCommand("FOR d IN c RETURN d", nil) + if err != nil { + t.Fatalf("AQL control command: %v", err) + } + if _, err := execution.BuildPostgreSQLExplainCommand(compiler.CompiledQuery{Command: aqlCommand}); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("AQL-tagged command into PostgreSQL EXPLAIN: got %v, want ErrCompiledCommandBackendMismatch", err) + } + if _, err := execution.BuildPostgreSQLExplainCommand(compiler.CompiledQuery{}); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("zero-tag command into PostgreSQL EXPLAIN: got %v, want ErrCompiledCommandBackendNotRegistered", err) + } + unknown, err := command.NewPostgreSQLCommand("SELECT 1", nil) + if err != nil { + t.Fatalf("unknown-tag build: %v", err) + } + _ = unknown +} + +func exec10HostilePlanJSON(t *testing.T) string { + t.Helper() + root := map[string]any{ + "Node Type": "Bitmap Heap Scan", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": exec10SentinelTable, + "Alias": exec10SentinelTable, + "Startup Cost": 0.0, + "Total Cost": 113.45, + "Plan Rows": 2, + "Plan Width": 34, + "Recheck Cond": fmt.Sprintf(`(%s ?? 'note')`, exec10SentinelColumn), + "Filter": fmt.Sprintf(`((%s ->> 'note' = '%s'::text) AND (%s::text <> '%s'::text))`, + exec10SentinelColumn, exec10SentinelBind, exec10SentinelColumn, exec10SentinelLiteral), + "Output": []string{exec10SentinelColumn}, + "Plans": []any{map[string]any{ + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Scan Direction": "Forward", + "Index Name": exec10SentinelIndex, + "Relation Name": exec10SentinelTable, + "Startup Cost": 0.27, + "Total Cost": 8.29, + "Plan Rows": 2, + "Plan Width": 0, + "Index Cond": fmt.Sprintf(`(%s ->> 'note') = '%s'::text`, exec10SentinelColumn, exec10SentinelBind), + "Planning Time": 0.42, + }}, + } + encoded, err := json.Marshal([]map[string]any{{"Plan": root}}) + if err != nil { + t.Fatalf("build hostile plan JSON: %v", err) + } + return string(encoded) +} + +func TestPostgresExplainSanitizerKeepsOnlyAgreedFields(t *testing.T) { + raw := json.RawMessage(exec10HostilePlanJSON(t)) + commandForExplain := exec10HostileCommand(t, `"public"."t"`) + assessment, err := execution.PostgreSQLExplainPlan(context.Background(), func(context.Context, command.CompiledCommand) (json.RawMessage, error) { + return raw, nil + }, compiler.CompiledQuery{Command: commandForExplain}) + if assessment.Backend != string(command.CompiledBackendPostgreSQL) { + t.Fatalf("backend tag: got %q", assessment.Backend) + } + if len(assessment.Nodes) != 2 { + t.Fatalf("nodes: %+v", assessment.Nodes) + } + root := assessment.Nodes[0] + if root.Ordinal != 0 || root.ParentOrdinal != -1 || root.Depth != 0 || root.NodeType != "Bitmap Heap Scan" { + t.Fatalf("root node structure: %+v", root) + } + if root.Estimate != (execution.PostgreSQLPlanEstimate{StartupCost: 0, TotalCost: 113.45, PlanRows: 2, PlanWidth: 34}) { + t.Fatalf("root estimates: %+v", root.Estimate) + } + if assessment.RootEstimate != root.Estimate { + t.Fatalf("root estimate mirror: %+v", assessment.RootEstimate) + } + child := assessment.Nodes[1] + if child.Ordinal != 1 || child.ParentOrdinal != 0 || child.Depth != 1 || child.NodeType != "Index Scan" { + t.Fatalf("child node structure: %+v", child) + } + if len(assessment.FullScans) != 0 { + t.Fatalf("no full scan in this plan: %+v", assessment.FullScans) + } + if len(assessment.IndexScans) != 1 || assessment.IndexScans[0] != (execution.PostgreSQLScanEvidence{Ordinal: 1, NodeType: "Index Scan", ScanKind: execution.PostgreSQLScanIndex}) { + t.Fatalf("index classification: %+v", assessment.IndexScans) + } + rendered, err := json.Marshal(assessment) + if err != nil { + t.Fatalf("marshal: %v", err) + } + exec10AssertSentinelFree(t, "sanitized assessment JSON", string(rendered)) + if strings.Contains(string(rendered), "Filter") || strings.Contains(string(rendered), "Relation") || + strings.Contains(string(rendered), "Index Name") || strings.Contains(string(rendered), "Output") { + t.Fatalf("sanitized assessment carries plan-text keys: %s", exec10Clip(string(rendered))) + } +} + +func TestPostgresExplainPlanErrorMapping(t *testing.T) { + pgCommand := exec10HostileCommand(t, `"public"."t"`) + compiled := compiler.CompiledQuery{Command: pgCommand} + + t.Run("executor failure is typed backend unavailable with cause chain", func(t *testing.T) { + pgErr := &pgconn.PgError{Code: "42P01", Message: "relation \"" + exec10SentinelMissing + "\" does not exist"} + wrapped := &dataframepostgres.QueryError{Op: "query", Err: pgErr} + _, err := execution.PostgreSQLExplainPlan(context.Background(), func(context.Context, command.CompiledCommand) (json.RawMessage, error) { + return nil, wrapped + }, compiled) + if err == nil { + t.Fatal("executor failure swallowed") + } + userErr, ok := dataframeerrors.AsUserError(err) + if !ok { + t.Fatalf("error not typed: %v", err) + } + if userErr.Code() != string(dataframeerrors.CodeBackendUnavailable) { + t.Fatalf("code: %s", userErr.Code()) + } + if !userErr.Retryable() { + t.Fatal("executor failures stay retryable until the store SQLSTATE classifier (#101) refines them") + } + if !errors.Is(err, dataframepostgres.ErrQuery) { + t.Fatalf("cause chain lost: %v", err) + } + var target *pgconn.PgError + if !errors.As(err, &target) || target.Code != "42P01" { + t.Fatalf("pgconn.PgError unreachable through the typed wrapper: %v", err) + } + exec10AssertSentinelFree(t, "user error message", userErr.Error()) + exec10AssertSentinelFree(t, "user error details", fmt.Sprint(userErr.Details())) + }) + + t.Run("canceled context maps to client canceled", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := execution.PostgreSQLExplainPlan(ctx, func(context.Context, command.CompiledCommand) (json.RawMessage, error) { + return nil, context.Canceled + }, compiled) + userErr, ok := dataframeerrors.AsUserError(err) + if !ok || userErr.Code() != string(dataframeerrors.CodeClientCanceled) { + t.Fatalf("canceled mapping: %v", err) + } + }) + + t.Run("deadline maps to retryable backend unavailable", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) + defer cancel() + time.Sleep(5 * time.Millisecond) + _, err := execution.PostgreSQLExplainPlan(ctx, func(context.Context, command.CompiledCommand) (json.RawMessage, error) { + return nil, context.DeadlineExceeded + }, compiled) + userErr, ok := dataframeerrors.AsUserError(err) + if !ok || userErr.Code() != string(dataframeerrors.CodeBackendUnavailable) || !userErr.Retryable() { + t.Fatalf("deadline mapping: %v", err) + } + }) + + t.Run("plan shape violation is non-retryable and sentinel-free", func(t *testing.T) { + hostileRaw := json.RawMessage(exec10HostilePlanJSON(t)[:40]) + _, err := execution.PostgreSQLExplainPlan(context.Background(), func(context.Context, command.CompiledCommand) (json.RawMessage, error) { + return hostileRaw, nil + }, compiled) + userErr, ok := dataframeerrors.AsUserError(err) + if !ok || userErr.Code() != string(dataframeerrors.CodeBackendUnavailable) || userErr.Retryable() { + t.Fatalf("shape mapping: %v", err) + } + details, _ := json.Marshal(userErr.Details()) + exec10AssertSentinelFree(t, "shape error surface", userErr.Error()+" "+string(details)) + }) + + t.Run("nil executor fails closed without fallback", func(t *testing.T) { + if _, err := execution.PostgreSQLExplainPlan(context.Background(), nil, compiled); err == nil || + !strings.Contains(err.Error(), "never fall back") { + t.Fatalf("nil executor: %v", err) + } + }) +} + +type exec10Registry struct{ entry recipeexec.Entry } + +func (r exec10Registry) LoadRecipe(context.Context, string) (recipeexec.Entry, error) { + return r.entry, nil +} +func (r exec10Registry) LoadRecipeVersion(ctx context.Context, name, _ string) (recipeexec.Entry, error) { + return r.LoadRecipe(ctx, name) +} + +func exec10Bundle() recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: 1, + Name: "exec10", + TranslationVersion: "test", + Outputs: []recipe.Output{{ + Name: "resources", RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{{Name: "id", Expr: recipe.Expression{Select: "root.id"}}}, + }}, + } +} + +func exec10Bindings() recipe.RuntimeBindings { + return recipe.RuntimeBindings{Project: "P1", DatasetGeneration: "G1", PreviewLimit: 25, OutputNames: []string{"resources"}} +} + +type exec10Callbacks struct { + aqlCalls int + pgCalls int +} + +func exec10Engine(t *testing.T, callbacks *exec10Callbacks, target compiler.Target) *execution.Engine { + t.Helper() + config := execution.Config{ + CompileTarget: target, + Registry: exec10Registry{entry: recipeexec.Entry{Bundle: exec10Bundle()}}, + SourceRows: func(_ context.Context, request execution.RowsRequest, _ execution.RowVisitor) error { + command := request.Command + if _, _, err := command.AQLParameters(); err != nil { + return err + } + callbacks.aqlCalls++ + return nil + }, + PreviewSourceRows: func(_ context.Context, request execution.RowsRequest, _ execution.RowVisitor) error { + command := request.Command + if _, _, err := command.AQLParameters(); err != nil { + return err + } + callbacks.aqlCalls++ + return nil + }, + ClickHouseQueryRows: func(context.Context, string, []string, func(map[string]any) error, ...any) error { + return errors.New("ClickHouse callbacks must never run for exec10 plans") + }, + } + engine, err := execution.New(config) + if err != nil { + t.Fatalf("execution.New: %v", err) + } + return engine +} + +func exec10PostgreSQLTarget(t *testing.T) compiler.Target { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("loom") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("PostgreSQLTarget: %v", err) + } + return target +} + +func exec10ArgsCarrySentinel(args []command.PostgreSQLParameter) bool { + for _, arg := range args { + if fmt.Sprintf("%v", arg.Value) == exec10SentinelBind { + return true + } + if raw, ok := arg.Value.(json.RawMessage); ok && strings.Contains(string(raw), exec10SentinelBind) { + return true + } + } + return false +} + +func exec10PGResolved(bindings recipe.RuntimeBindings) execution.Resolved { + unrestricted := len(bindings.AuthResourcePaths) == 0 + output := lower.CompiledRecipeOutput{ + Name: "resources", + RootResourceType: "Patient", + Columns: []string{"id"}, + OutputSchema: []lower.CompiledOutputColumn{ + {ID: "id", Name: "id", Label: "id", Kind: "string", Cardinality: "one"}, + }, + RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, + Plan: ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": "Patient", + "project": bindings.Project, + "dataset_generation": bindings.DatasetGeneration, + "auth_resource_paths": append([]string(nil), bindings.AuthResourcePaths...), + "auth_resource_paths_unrestricted": unrestricted, + "scope_allowed": true, + }, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"project"}}, Right: &ir.PhysicalValue{BindKey: "project"}}}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"dataset_generation"}}, Right: &ir.PhysicalValue{BindKey: "dataset_generation"}}}}, + {Kind: ir.PhysicalDerivedLetOp, DerivedLet: &ir.PhysicalDerivedLet{Variable: "root_auth_allowed", Operator: "AUTH_RESOURCE_PATH_ALLOWED", Inputs: []ir.PhysicalValue{ + {Variable: "root", Path: []string{"auth_resource_path"}}, + {BindKey: "auth_resource_paths"}, + {BindKey: "auth_resource_paths_unrestricted"}, + }}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root_auth_allowed"}, Right: &ir.PhysicalValue{BindKey: "scope_allowed"}}}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + }, + }, + } + return execution.Resolved{ + Bundle: recipe.Bundle{Name: "exec10"}, + Semantic: semantic.ResolvedRecipePlan{ + SemanticPlan: semantic.RecipePlan{Bindings: bindings}, + }, + Compiled: lower.CompiledRecipe{Outputs: []lower.CompiledRecipeOutput{output}}, + StoredRecipeDigest: "exec10-pg-resolved", + } +} + +func exec10HostileBindings() recipe.RuntimeBindings { + bindings := exec10Bindings() + bindings.AuthResourcePaths = []string{exec10SentinelBind} + return bindings +} + +func TestPostgresExplainCompileOnlyWithoutDatabase(t *testing.T) { + callbacks := &exec10Callbacks{} + engine := exec10Engine(t, callbacks, compiler.ArangoTarget()) + control := execution.Control{Engine: engine} + result, err := control.ExplainPhysical(context.Background(), "exec10", exec10Bindings(), false) + if err != nil { + t.Fatalf("compile-only execution.ExplainPhysical: %v", err) + } + if len(result.Outputs) != 1 { + t.Fatalf("outputs: %+v", result.Outputs) + } + item := result.Outputs[0] + if item.Name != "resources" || item.PlanFingerprint == "" || len(item.Columns) == 0 { + t.Fatalf("compile-only item: %+v", item) + } + if item.Live != nil || item.LivePostgreSQL != nil { + t.Fatalf("compile-only run must carry no live assessment: %+v", item) + } + if callbacks.aqlCalls != 0 || callbacks.pgCalls != 0 { + t.Fatalf("compile-only run touched a backend adapter: %+v", callbacks) + } + rendered, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + exec10AssertSentinelFree(t, "compile-only explanation JSON", string(rendered)) +} + +func exec10ArangoAssessment() execution.ExplainAssessment { + return execution.ExplainAssessment{ + Plans: []execution.ExplainPlanEstimate{ + {Plan: 0, EstimatedCost: 12.5, EstimatedNrItems: 3}, + {Plan: 1, EstimatedCost: 100.25, EstimatedNrItems: 10}, + }, + FullCollectionScans: []execution.ExplainCollectionScan{{Plan: 1, NodeID: 4, Collection: "Patient"}}, + Indexes: []execution.ExplainIndexSummary{{ + Collection: "Patient", ID: "Patient/idx1", Name: "idx1", Type: "skiplist", + Fields: []string{"id"}, Uses: []execution.ExplainIndexLocation{{Plan: 0, NodeID: 2}}, + }}, + Warnings: []execution.ExplainWarning{{Code: 1004, Message: "full collection scan"}}, + AppliedOptimizerRules: []string{"move-calculations-up"}, + } +} + +const exec10AQLGoldenJSON = `{"Outputs":[{"Name":"resources","PlanFingerprint":"354fb4517532d86fe9258a91307aebecf6d5868c5c16731ce3c1e0f49042a58b","Columns":["id"],"Diagnostics":{"Fingerprint":"cc21da1c6642c5ebc16063694e08f47962de98ef2cd81c12c3c4dc081537c586","TraversalSets":0,"EndpointTraversalCount":0,"NativeTraversalCount":0,"TraversalStrategies":null,"SharedTraversalCount":0,"RequiredMatchReuseCount":0,"ScopedSharingCandidateGroups":0,"ScopedSharingCandidateSets":0,"PotentialSharingOpportunityGroups":0,"PotentialSharingOpportunitySets":0,"RichSourceReuse":null,"RichConsumerGroups":null,"ExpressionBindingCount":0,"SharedKeyedMapCount":0,"ObjectLookupConsumerCount":0,"OptimizationPolicy":{"Policy":"conservative-structural-v1","Enabled":true,"MinimumSavings":1,"RuleStates":[{"Rule":"traversal_sharing","Enabled":true,"Reason":"enabled; candidate-specific decisions are reported separately"},{"Rule":"prepared_selectors","Enabled":false,"Reason":"disabled until implemented and profile-gated"},{"Rule":"compact_set_projection","Enabled":true,"Reason":"enabled; candidate-specific decisions are reported separately"},{"Rule":"endpoint_traversal","Enabled":true,"Reason":"enabled; candidate-specific decisions are reported separately"}],"Decisions":null}},"Live":{"Plans":[{"Plan":0,"EstimatedCost":12.5,"EstimatedNrItems":3},{"Plan":1,"EstimatedCost":100.25,"EstimatedNrItems":10}],"FullCollectionScans":[{"Plan":1,"NodeID":4,"Collection":"Patient"}],"Indexes":[{"Collection":"Patient","ID":"Patient/idx1","Name":"idx1","Type":"skiplist","Fields":["id"],"Uses":[{"Plan":0,"NodeID":2}]}],"Warnings":[{"Code":1004,"Message":"full collection scan"}],"AppliedOptimizerRules":["move-calculations-up"]}}]}` + +func exec10ContractProjection(result execution.PhysicalExplanation) string { + type output struct { + Name string + PlanFingerprint string + Columns []string + Diagnostics ir.CompilerPlanDiagnostics + Live *execution.ExplainAssessment + } + type explanation struct { + Outputs []output + } + view := explanation{Outputs: make([]output, 0, len(result.Outputs))} + for _, item := range result.Outputs { + view.Outputs = append(view.Outputs, output{ + Name: item.Name, PlanFingerprint: item.PlanFingerprint, + Columns: item.Columns, Diagnostics: item.Diagnostics, Live: item.Live, + }) + } + encoded, err := json.Marshal(view) + if err != nil { + panic(err) + } + return string(encoded) +} + +func TestPostgresExplainArangoSurfaceByteFieldCompatible(t *testing.T) { + callbacks := &exec10Callbacks{} + engine := exec10Engine(t, callbacks, compiler.ArangoTarget()) + fixture := exec10ArangoAssessment() + control := execution.Control{ + Engine: engine, + ExplainConnection: func(context.Context, compiler.CompiledQuery) (execution.ExplainAssessment, error) { + return fixture, nil + }, + ExplainPostgreSQLConnection: func(context.Context, compiler.CompiledQuery) (execution.PostgreSQLPlanAssessment, error) { + return execution.PostgreSQLPlanAssessment{}, errors.New("the PostgreSQL callback must never run for an AQL plan") + }, + } + result, err := control.ExplainPhysical(context.Background(), "exec10", exec10Bindings(), true) + if err != nil { + t.Fatalf("live AQL execution.ExplainPhysical: %v", err) + } + item := result.Outputs[0] + if item.Live == nil || item.LivePostgreSQL != nil { + t.Fatalf("AQL live assessment placement: %+v", item) + } + if len(item.Live.Plans) != 2 || len(item.Live.FullCollectionScans) != 1 || + len(item.Live.Indexes) != 1 || len(item.Live.Warnings) != 1 || + len(item.Live.AppliedOptimizerRules) != 1 { + t.Fatalf("assessment fields: %+v", item.Live) + } + if item.Live.Indexes[0].Uses[0] != (execution.ExplainIndexLocation{Plan: 0, NodeID: 2}) { + t.Fatalf("assessment nested fields: %+v", item.Live.Indexes[0]) + } + rendered := exec10ContractProjection(result) + if rendered != exec10AQLGoldenJSON { + t.Fatalf("Arango response contract drifted:\n got %s\nwant %s", rendered, exec10AQLGoldenJSON) + } + if item.LivePostgreSQL != nil { + t.Fatal("AQL output must not carry a PostgreSQL assessment") + } +} + +func TestPostgresExplainTagDispatchIsMutuallyExclusive(t *testing.T) { + callbacks := &exec10Callbacks{} + engine := exec10Engine(t, callbacks, exec10PostgreSQLTarget(t)) + bindings := exec10HostileBindings() + + var received []command.CompiledBackend + var captured command.CompiledCommand + control := execution.Control{ + Engine: engine, + ExplainConnection: func(context.Context, compiler.CompiledQuery) (execution.ExplainAssessment, error) { + t.Fatal("the Arango policy surface must never assess a PostgreSQL-tagged command") + return execution.ExplainAssessment{}, nil + }, + ExplainPostgreSQLConnection: func(_ context.Context, compiled compiler.CompiledQuery) (execution.PostgreSQLPlanAssessment, error) { + received = append(received, compiled.Command.Backend()) + captured = compiled.Command + return execution.PostgreSQLPlanAssessment{ + Backend: string(command.CompiledBackendPostgreSQL), + Nodes: []execution.PostgreSQLPlanNode{{Ordinal: 0, ParentOrdinal: -1, NodeType: "Index Scan", Estimate: execution.PostgreSQLPlanEstimate{StartupCost: 0, TotalCost: 1, PlanRows: 1, PlanWidth: 8}}}, + RootEstimate: execution.PostgreSQLPlanEstimate{StartupCost: 0, TotalCost: 1, PlanRows: 1, PlanWidth: 8}, + }, nil + }, + } + resolved := exec10PGResolved(bindings) + result, err := control.ExplainResolved(context.Background(), resolved, true) + if err != nil { + t.Fatalf("live PostgreSQL execution.ExplainResolved: %v", err) + } + if len(received) != 1 || received[0] != command.CompiledBackendPostgreSQL { + t.Fatalf("PostgreSQL callback invocations: %v", received) + } + capturedSQL, capturedArgs, err := captured.PostgreSQLParameters() + if err != nil { + t.Fatalf("captured command dispatch: %v", err) + } + if strings.HasPrefix(capturedSQL, "EXPLAIN") { + t.Fatalf("callback must receive the executable command, EXPLAIN is the runner's job: %q", exec10Clip(capturedSQL)) + } + if !exec10ArgsCarrySentinel(capturedArgs) { + t.Fatalf("hostile bind value must reach the command as a typed parameter: %+v", capturedArgs) + } + item := result.Outputs[0] + if item.Live != nil || item.LivePostgreSQL == nil { + t.Fatalf("live assessment placement: %+v", item) + } + compileOnly, err := control.ExplainResolved(context.Background(), exec10PGResolved(bindings), false) + if err != nil { + t.Fatalf("compile-only execution.ExplainResolved: %v", err) + } + if compileOnly.Outputs[0].PlanFingerprint != item.PlanFingerprint { + t.Fatalf("PlanFingerprint drifted between live and compile-only runs: %s vs %s", + item.PlanFingerprint, compileOnly.Outputs[0].PlanFingerprint) + } + if len(item.Columns) != 1 || item.Columns[0] != "id" { + t.Fatalf("columns: %+v", item.Columns) + } + rendered, _ := json.Marshal(result) + exec10AssertSentinelFree(t, "PostgreSQL explanation JSON", string(rendered)) + + aqlCallbacks := &exec10Callbacks{} + aqlEngine := exec10Engine(t, aqlCallbacks, compiler.ArangoTarget()) + aqlSurfaceHits := 0 + pgSurfaceHits := 0 + aqlControl := execution.Control{ + Engine: aqlEngine, + ExplainConnection: func(context.Context, compiler.CompiledQuery) (execution.ExplainAssessment, error) { + aqlSurfaceHits++ + return execution.ExplainAssessment{}, nil + }, + ExplainPostgreSQLConnection: func(context.Context, compiler.CompiledQuery) (execution.PostgreSQLPlanAssessment, error) { + pgSurfaceHits++ + return execution.PostgreSQLPlanAssessment{}, errors.New("must not run") + }, + } + if _, err := aqlControl.ExplainPhysical(context.Background(), "exec10", exec10Bindings(), true); err != nil { + t.Fatalf("AQL paired control: %v", err) + } + if aqlSurfaceHits != 1 || pgSurfaceHits != 0 { + t.Fatalf("paired routing: aql=%d pg=%d", aqlSurfaceHits, pgSurfaceHits) + } +} + +func TestPostgresExplainMissingCallbackRefusals(t *testing.T) { + callbacks := &exec10Callbacks{} + engine := exec10Engine(t, callbacks, exec10PostgreSQLTarget(t)) + + resolved := exec10PGResolved(exec10Bindings()) + only := execution.Control{Engine: engine} + _, err := only.ExplainResolved(context.Background(), resolved, true) + if err == nil || err.Error() != "live physical recipe Explain requires a PostgreSQL connection" { + t.Fatalf("missing PostgreSQL callback: %v", err) + } + + armed := execution.Control{Engine: engine, ExplainPostgreSQLConnection: func(context.Context, compiler.CompiledQuery) (execution.PostgreSQLPlanAssessment, error) { + return execution.PostgreSQLPlanAssessment{Backend: string(command.CompiledBackendPostgreSQL)}, nil + }} + if _, err := armed.ExplainResolved(context.Background(), resolved, true); err != nil { + t.Fatalf("armed PostgreSQL callback must proceed: %v", err) + } + + aqlEngine := exec10Engine(t, &exec10Callbacks{}, compiler.ArangoTarget()) + aqlOnly := execution.Control{Engine: aqlEngine} + if _, err := aqlOnly.ExplainPhysical(context.Background(), "exec10", exec10Bindings(), true); err == nil || + err.Error() != "live physical recipe Explain requires an Arango connection" { + t.Fatalf("missing Arango callback message drift: %v", err) + } + pgTouched := false + aqlOnlyPG := execution.Control{Engine: aqlEngine, ExplainPostgreSQLConnection: func(context.Context, compiler.CompiledQuery) (execution.PostgreSQLPlanAssessment, error) { + pgTouched = true + return execution.PostgreSQLPlanAssessment{}, nil + }} + if _, err := aqlOnlyPG.ExplainPhysical(context.Background(), "exec10", exec10Bindings(), true); err == nil || + err.Error() != "live physical recipe Explain requires an Arango connection" { + t.Fatalf("AQL refusal must not be diverted to the PostgreSQL callback: %v", err) + } + if pgTouched { + t.Fatal("AQL command reached the PostgreSQL callback") + } +} + +func TestPostgresExplainControlPostgreSQLErrorIsTyped(t *testing.T) { + callbacks := &exec10Callbacks{} + engine := exec10Engine(t, callbacks, exec10PostgreSQLTarget(t)) + sentinelError := fmt.Errorf("backend exploded while planning %s against %s", exec10SentinelTable, exec10SentinelSchema) + control := execution.Control{Engine: engine, ExplainPostgreSQLConnection: func(context.Context, compiler.CompiledQuery) (execution.PostgreSQLPlanAssessment, error) { + return execution.PostgreSQLPlanAssessment{}, sentinelError + }} + _, err := control.ExplainResolved(context.Background(), exec10PGResolved(exec10Bindings()), true) + userErr, ok := dataframeerrors.AsUserError(err) + if !ok { + t.Fatalf("callback failure not typed: %v", err) + } + if userErr.Code() != string(dataframeerrors.CodeBackendUnavailable) { + t.Fatalf("code: %s", userErr.Code()) + } + if !errors.Is(err, sentinelError) { + t.Fatal("cause chain lost") + } + details, _ := json.Marshal(userErr.Details()) + exec10AssertSentinelFree(t, "control error surface", userErr.Error()+" "+string(details)) +} + +func TestPostgresExplainFullSurfaceSentinelSweep(t *testing.T) { + callbacks := &exec10Callbacks{} + engine := exec10Engine(t, callbacks, exec10PostgreSQLTarget(t)) + bindings := exec10HostileBindings() + + var runnerSaw command.CompiledCommand + control := execution.Control{Engine: engine, ExplainPostgreSQLConnection: func(_ context.Context, compiled compiler.CompiledQuery) (execution.PostgreSQLPlanAssessment, error) { + return execution.PostgreSQLExplainPlan(context.Background(), func(_ context.Context, explainCommand command.CompiledCommand) (json.RawMessage, error) { + runnerSaw = explainCommand + return json.RawMessage(exec10HostilePlanJSON(t)), nil + }, compiled) + }} + result, err := control.ExplainResolved(context.Background(), exec10PGResolved(bindings), true) + if err != nil { + t.Fatalf("live PostgreSQL execution.ExplainResolved: %v", err) + } + runnerSQL, runnerArgs, err := runnerSaw.PostgreSQLParameters() + if err != nil { + t.Fatalf("runner command was not PostgreSQL-tagged: %v", err) + } + if !strings.HasPrefix(runnerSQL, "EXPLAIN (FORMAT JSON)\n") { + t.Fatalf("runner must receive the EXPLAIN-wrapped command: %q", exec10Clip(runnerSQL)) + } + if !exec10ArgsCarrySentinel(runnerArgs) { + t.Fatalf("ordered typed arguments must survive the EXPLAIN wrap: %+v", runnerArgs) + } + rendered, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + exec10AssertSentinelFree(t, "full explanation JSON", string(rendered)) + if !strings.Contains(string(rendered), "Index Scan") || !strings.Contains(string(rendered), "Bitmap Heap Scan") { + t.Fatalf("sanitized node types missing: %s", exec10Clip(string(rendered))) + } + if !strings.Contains(string(rendered), "113.45") { + t.Fatalf("root estimate missing: %s", exec10Clip(string(rendered))) + } + if strings.Contains(string(rendered), exec10HostileSQL(`"public"."t"`)) { + t.Fatal("statement text leaked into the explanation") + } +} + +const ( + exec10Prefix = "verify_exec10_" + exec10NameFmt = exec10Prefix + "%d_%s" +) + +var exec10NamePattern = regexp.MustCompile(`^verify_exec10_[0-9]+_[0-9a-f]+$`) + +var ( + exec10OwnedMu sync.Mutex + exec10OwnedDBs = map[string]string{} + exec10OwnedRoles = map[string]bool{} +) + +func exec10RegisterDB(db, schema string) { + exec10OwnedMu.Lock() + defer exec10OwnedMu.Unlock() + exec10OwnedDBs[db] = schema +} + +func exec10RegisterRole(role string) { + exec10OwnedMu.Lock() + defer exec10OwnedMu.Unlock() + exec10OwnedRoles[role] = true +} + +func exec10RegistrySnapshot() (map[string]string, map[string]bool) { + exec10OwnedMu.Lock() + defer exec10OwnedMu.Unlock() + dbs := make(map[string]string, len(exec10OwnedDBs)) + for k, v := range exec10OwnedDBs { + dbs[k] = v + } + roles := make(map[string]bool, len(exec10OwnedRoles)) + for k, v := range exec10OwnedRoles { + roles[k] = v + } + return dbs, roles +} + +var ( + exec10BaselineOnce sync.Once + exec10BaselineDBs []string + exec10BaselineRoles []string + exec10BaselineErr error +) + +func exec10CaptureBaseline(raw string) { + exec10BaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec10BaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec10\\_%' ESCAPE '\\'") + if err != nil { + exec10BaselineErr = fmt.Errorf("read database baseline: %w", err) + return + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + exec10BaselineErr = fmt.Errorf("scan database baseline: %w", err) + return + } + exec10BaselineDBs = append(exec10BaselineDBs, name) + } + rows.Close() + roles, err := admin.Query(ctx, "SELECT rolname FROM pg_roles WHERE rolname LIKE 'verify\\_exec10\\_%' ESCAPE '\\'") + if err != nil { + exec10BaselineErr = fmt.Errorf("read role baseline: %w", err) + return + } + for roles.Next() { + var name string + if err := roles.Scan(&name); err != nil { + roles.Close() + exec10BaselineErr = fmt.Errorf("scan role baseline: %w", err) + return + } + exec10BaselineRoles = append(exec10BaselineRoles, name) + } + roles.Close() + }) +} + +func exec10URL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 Explain integration tests") + } + return raw +} + +type exec10Service struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + racePool *pgxpool.Pool + database string + schema string + dsn string + scrub func(any) string + fail func(step string, err error) +} + +func newExec10Service(t *testing.T) *exec10Service { + t.Helper() + raw := exec10URL(t) + exec10CaptureBaseline(raw) + if exec10BaselineErr != nil { + t.Fatalf("capture pg_database/pg_roles baseline: %v", exec10BaselineErr) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + lockConn, err := admin.Acquire(ctx) + if err != nil { + fail("acquire serialization connection", err) + } + if _, err := lockConn.Exec(ctx, `SELECT pg_advisory_lock(hashtext('loom/pg-exec-10/live-mutation')::bigint)`); err != nil { + lockConn.Release() + fail("take PG mutation serialization lock", err) + } + t.Cleanup(func() { + _, _ = lockConn.Exec(context.WithoutCancel(ctx), `SELECT pg_advisory_unlock(hashtext('loom/pg-exec-10/live-mutation')::bigint)`) + lockConn.Release() + }) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, + "SELECT current_setting('server_version_num')::int8, current_setting('server_version')"). + Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if err := storepg.CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required PostgreSQL 18 backend: %v", err) + } + + dbName := fmt.Sprintf(exec10NameFmt, time.Now().UnixNano(), exec10RandomHex(t)) + if !exec10NamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + var created int64 + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", dbName).Scan(&created); err != nil { + fail("verify isolated database creation", err) + } + if created != 1 { + t.Fatalf("isolated database %q did not verifiably come into existence", dbName) + } + exec10RegisterDB(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn, dbName) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + racePool, err := pgxpool.NewWithConfig(ctx, func() *pgxpool.Config { + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + fail("parse race pool config", err) + } + cfg.MaxConns = 6 + cfg.MinConns = 2 + return cfg + }()) + if err != nil { + fail("create race pool", err) + } + t.Cleanup(racePool.Close) + + schema := fmt.Sprintf(exec10NameFmt, time.Now().UnixNano(), exec10RandomHex(t)) + if !exec10NamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + exec10RegisterDB(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + return &exec10Service{ctx: ctx, admin: admin, iso: iso, racePool: racePool, database: dbName, schema: schema, dsn: dsn, scrub: scrub, fail: fail} +} + +func exec10RandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +func (s *exec10Service) postgreSQLTarget(t *testing.T) compiler.Target { + t.Helper() + binding, err := ir.NewPostgresStorageBinding(s.schema) + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("PostgreSQLTarget: %v", err) + } + return target +} + +func (s *exec10Service) migrate(t *testing.T) { + t.Helper() + client, err := storepg.Open(s.ctx, storepg.Config{ + DSN: s.dsn, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 2 * time.Second, + }) + if err != nil { + s.fail("open pg-store client", err) + } + defer client.Close() + if _, err := storepg.MigrateSchema(s.ctx, client, s.schema); err != nil { + s.fail("migrate owned schema", err) + } +} + +func (s *exec10Service) qualifiedTable(name string) string { + return `"` + s.schema + `"."` + name + `"` +} + +func (s *exec10Service) exec(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := s.iso.Exec(s.ctx, sql, args...); err != nil { + s.fail("fixture exec", err) + } +} + +func (s *exec10Service) seedHostileFixture(t *testing.T) string { + t.Helper() + table := s.qualifiedTable(exec10SentinelTable) + s.exec(t, `CREATE TABLE `+table+` (resource_seq bigint, "`+exec10SentinelColumn+`" text)`) + s.exec(t, `CREATE INDEX "`+exec10SentinelIndex+`" ON `+table+` (resource_seq)`) + s.exec(t, `INSERT INTO `+table+` SELECT i::bigint, 'note-' || i::text FROM generate_series(1, 500) AS i`) + s.exec(t, `ANALYZE `+table) + return table +} + +func exec10SequentialCommand(table string) command.CompiledCommand { + sqlText := fmt.Sprintf(`SELECT "%s" FROM %s WHERE "%s" = $1`, exec10SentinelColumn, table, exec10SentinelColumn) + compiledCommand, err := command.NewPostgreSQLCommand(sqlText, exec10HostileParameters()) + if err != nil { + panic(err) + } + return compiledCommand +} + +func exec10IndexedCommand(table string) command.CompiledCommand { + sqlText := fmt.Sprintf(`SELECT resource_seq FROM %s WHERE resource_seq = $1 ORDER BY resource_seq`, table) + compiledCommand, err := command.NewPostgreSQLCommand(sqlText, []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterBigint, Value: int64(42)}, + }) + if err != nil { + panic(err) + } + return compiledCommand +} + +func exec10PoolRunner(t *testing.T, querier dataframepostgres.Querier) (execution.PostgreSQLPlanJSONExecutor, *int64) { + t.Helper() + executor, err := dataframepostgres.NewExecutor(querier) + if err != nil { + t.Fatalf("NewExecutor: %v", err) + } + var queries int64 + return func(ctx context.Context, command command.CompiledCommand) (json.RawMessage, error) { + atomic.AddInt64(&queries, 1) + var plans []string + if err := executor.SourceRows(ctx, execution.RowsRequest{Command: command, DeclaredColumns: []string{"QUERY PLAN"}}, func(row map[string]any) error { + switch plan := row["QUERY PLAN"].(type) { + case string: + plans = append(plans, plan) + case []byte: + plans = append(plans, string(plan)) + case json.RawMessage: + plans = append(plans, string(plan)) + default: + return fmt.Errorf("PostgreSQL EXPLAIN returned plan text as %T", row["QUERY PLAN"]) + } + return nil + }); err != nil { + return nil, err + } + if len(plans) != 1 { + return nil, fmt.Errorf("PostgreSQL EXPLAIN must return exactly one plan row, got %d", len(plans)) + } + return json.RawMessage(plans[0]), nil + }, &queries +} + +var exec10NodePattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9 ]{0,63}$`) + +func exec10AssertSanitizedIndexed(t *testing.T, assessment execution.PostgreSQLPlanAssessment) { + t.Helper() + if assessment.Backend != string(command.CompiledBackendPostgreSQL) { + t.Fatalf("backend: %q", assessment.Backend) + } + if len(assessment.Nodes) == 0 || assessment.Nodes[0].Ordinal != 0 || assessment.Nodes[0].ParentOrdinal != -1 { + t.Fatalf("nodes: %+v", assessment.Nodes) + } + for _, node := range assessment.Nodes { + if !exec10NodePattern.MatchString(node.NodeType) { + t.Fatalf("kept node type outside the operator vocabulary: %q", node.NodeType) + } + if node.Estimate.TotalCost < node.Estimate.StartupCost || node.Estimate.PlanRows < 0 || node.Estimate.PlanWidth < 0 { + t.Fatalf("implausible estimates: %+v", node) + } + } + if len(assessment.IndexScans) == 0 { + t.Fatalf("indexed fixture query must classify at least one index scan: %+v", assessment) + } +} + +func exec10AssertSanitizedSequential(t *testing.T, assessment execution.PostgreSQLPlanAssessment) { + t.Helper() + if len(assessment.FullScans) == 0 { + t.Fatalf("sequential fixture query must classify at least one sequential scan: %+v", assessment) + } + for _, evidence := range assessment.FullScans { + if evidence.NodeType != "Seq Scan" && evidence.NodeType != "Sample Scan" { + t.Fatalf("unexpected sequential scan operator: %+v", evidence) + } + } +} + +func TestPostgresExplainLivePG18IndexedAndSequentialPlans(t *testing.T) { + s := newExec10Service(t) + table := s.seedHostileFixture(t) + runner, _ := exec10PoolRunner(t, s.iso) + secrets := []string{s.database, s.schema, exec10SentinelTable, exec10SentinelIndex, exec10SentinelColumn} + + indexed, err := execution.PostgreSQLExplainPlan(s.ctx, runner, compiler.CompiledQuery{Command: exec10IndexedCommand(table)}) + if err != nil { + s.fail("indexed live Explain", err) + } + exec10AssertSanitizedIndexed(t, indexed) + renderedIndexed, _ := json.Marshal(indexed) + exec10AssertSentinelFree(t, "live indexed assessment", string(renderedIndexed), secrets...) + + sequential, err := execution.PostgreSQLExplainPlan(s.ctx, runner, compiler.CompiledQuery{Command: exec10SequentialCommand(table)}) + if err != nil { + s.fail("sequential live Explain", err) + } + exec10AssertSanitizedSequential(t, sequential) + renderedSequential, _ := json.Marshal(sequential) + exec10AssertSentinelFree(t, "live sequential assessment", string(renderedSequential), secrets...) + + hostile, err := execution.PostgreSQLExplainPlan(s.ctx, runner, compiler.CompiledQuery{Command: exec10HostileCommand(t, table)}) + if err != nil { + s.fail("hostile live Explain", err) + } + if len(hostile.Nodes) == 0 { + t.Fatalf("hostile plan produced no nodes") + } + renderedHostile, _ := json.Marshal(hostile) + exec10AssertSentinelFree(t, "live hostile assessment", string(renderedHostile), secrets...) + if strings.Contains(string(renderedHostile), "Filter") || strings.Contains(string(renderedHostile), `"Index Cond"`) { + t.Fatalf("plan-text keys survived sanitization: %s", exec10Clip(string(renderedHostile))) + } + + s.migrate(t) + callbacks := &exec10Callbacks{} + engine := exec10Engine(t, callbacks, s.postgreSQLTarget(t)) + control := execution.Control{Engine: engine, ExplainPostgreSQLConnection: func(ctx context.Context, compiled compiler.CompiledQuery) (execution.PostgreSQLPlanAssessment, error) { + return execution.PostgreSQLExplainPlan(ctx, runner, compiled) + }} + resolved := exec10PGResolved(exec10Bindings()) + result, err := control.ExplainResolved(s.ctx, resolved, true) + if err != nil { + s.fail("control live execution.ExplainResolved", err) + } + item := result.Outputs[0] + if item.LivePostgreSQL == nil || item.Live != nil { + t.Fatalf("control live placement: %+v", item) + } + compileOnly, err := control.ExplainResolved(s.ctx, exec10PGResolved(exec10Bindings()), false) + if err != nil { + s.fail("control compile-only execution.ExplainResolved", err) + } + if compileOnly.Outputs[0].PlanFingerprint != item.PlanFingerprint { + t.Fatalf("fingerprint drifted between live and compile-only runs: %s vs %s", + item.PlanFingerprint, compileOnly.Outputs[0].PlanFingerprint) + } + renderedResult, _ := json.Marshal(result) + exec10AssertSentinelFree(t, "control live explanation", string(renderedResult), secrets...) +} + +func TestPostgresExplainLivePG18PermissionAndAbsence(t *testing.T) { + s := newExec10Service(t) + table := s.seedHostileFixture(t) + conn, err := s.iso.Acquire(s.ctx) + if err != nil { + s.fail("acquire permission-test connection", err) + } + defer conn.Release() + runner, queryCount := exec10PoolRunner(t, conn) + compiledCommand := exec10IndexedCommand(table) + + if _, err := execution.PostgreSQLExplainPlan(s.ctx, runner, compiler.CompiledQuery{Command: compiledCommand}); err != nil { + s.fail("owner EXPLAIN", err) + } + + missingSQL := `SELECT resource_seq FROM "` + s.schema + `"."` + exec10SentinelMissing + `" WHERE resource_seq = $1` + missingCommand, err := command.NewPostgreSQLCommand(missingSQL, []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterBigint, Value: int64(1)}, + }) + if err != nil { + t.Fatalf("build missing-relation command: %v", err) + } + _, err = execution.PostgreSQLExplainPlan(s.ctx, runner, compiler.CompiledQuery{Command: missingCommand}) + s.assertTypedPgError(t, err, "42P01", "undefined relation") + exec10AssertSentinelFree(t, "undefined relation error surface", exec10UserSurface(err)) + + role := fmt.Sprintf(exec10Prefix+"r_%d_%s", time.Now().UnixNano(), exec10RandomHex(t)) + if !regexp.MustCompile(`^verify_exec10_r_[0-9]+_[0-9a-f]+$`).MatchString(role) { + t.Fatalf("role name %q outside the owned pattern", role) + } + s.exec(t, `CREATE ROLE "`+role+`" NOSUPERUSER NOLOGIN`) + exec10RegisterRole(role) + t.Cleanup(func() { + if _, err := s.iso.Exec(context.WithoutCancel(s.ctx), `REVOKE ALL ON `+table+` FROM "`+role+`"`); err != nil { + t.Errorf("revoke table from role %s: %s", role, s.scrub(err)) + } + if _, err := s.iso.Exec(context.WithoutCancel(s.ctx), `REVOKE USAGE ON SCHEMA "`+s.schema+`" FROM "`+role+`"`); err != nil { + t.Errorf("revoke schema from role %s: %s", role, s.scrub(err)) + } + if _, err := s.iso.Exec(context.WithoutCancel(s.ctx), `DROP ROLE IF EXISTS "`+role+`"`); err != nil { + t.Errorf("drop role %s: %s", role, s.scrub(err)) + } + }) + + setRole := func() { + if _, err := conn.Exec(s.ctx, `SET ROLE "`+role+`"`); err != nil { + s.fail("set role", err) + } + } + resetRole := func() { + if _, err := conn.Exec(context.WithoutCancel(s.ctx), `RESET ROLE`); err != nil { + s.fail("reset role", err) + } + } + defer resetRole() + + setRole() + _, err = execution.PostgreSQLExplainPlan(s.ctx, runner, compiler.CompiledQuery{Command: compiledCommand}) + s.assertTypedPgError(t, err, "42501", "permission-denied EXPLAIN at schema level") + exec10AssertSentinelFree(t, "permission error surface", exec10UserSurface(err), s.schema, exec10SentinelTable) + resetRole() + + s.exec(t, `GRANT USAGE ON SCHEMA "`+s.schema+`" TO "`+role+`"`) + setRole() + _, err = execution.PostgreSQLExplainPlan(s.ctx, runner, compiler.CompiledQuery{Command: compiledCommand}) + s.assertTypedPgError(t, err, "42501", "permission-denied EXPLAIN at table level") + resetRole() + + s.exec(t, `GRANT SELECT ON `+table+` TO "`+role+`"`) + setRole() + granted, err := execution.PostgreSQLExplainPlan(s.ctx, runner, compiler.CompiledQuery{Command: compiledCommand}) + resetRole() + if err != nil { + s.fail("granted-role EXPLAIN (paired control)", err) + } + exec10AssertSanitizedIndexed(t, granted) + + before := atomic.LoadInt64(queryCount) + aqlCommand, err := command.NewAQLCommand("FOR d IN c RETURN d", nil) + if err != nil { + t.Fatal(err) + } + if _, err := execution.PostgreSQLExplainPlan(s.ctx, runner, compiler.CompiledQuery{Command: aqlCommand}); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("cross-tag EXPLAIN: %v", err) + } + if after := atomic.LoadInt64(queryCount); after != before { + t.Fatalf("cross-tag command reached the server: queryCount %d -> %d", before, after) + } +} + +func exec10UserSurface(err error) string { + var builder strings.Builder + builder.WriteString(fmt.Sprintf("%v", err)) + if userErr, ok := dataframeerrors.AsUserError(err); ok { + builder.WriteString(" | code=" + userErr.Code()) + builder.WriteString(" | message=" + userErr.Error()) + details, _ := json.Marshal(userErr.Details()) + builder.WriteString(" | details=" + string(details)) + } + return builder.String() +} + +func (s *exec10Service) assertTypedPgError(t *testing.T, err error, wantSQLState, what string) { + t.Helper() + if err == nil { + t.Fatalf("%s: expected failure, got nil", what) + } + userErr, ok := dataframeerrors.AsUserError(err) + if !ok { + t.Fatalf("%s: error not typed: %v", what, err) + } + if userErr.Code() != string(dataframeerrors.CodeInternalError) { + t.Fatalf("%s: code %s, want INTERNAL_ERROR from the store/postgres boundary classification", what, userErr.Code()) + } + var translated *storepg.TranslatedError + if !errors.As(err, &translated) || translated.Condition != storepg.ConditionServerRejected || translated.SQLState != wantSQLState { + t.Fatalf("%s: boundary classification missing for SQLSTATE %s: %v", what, wantSQLState, err) + } +} + +func TestPostgresExplainLivePG18ConcurrentRounds(t *testing.T) { + s := newExec10Service(t) + table := s.seedHostileFixture(t) + runner, _ := exec10PoolRunner(t, s.racePool) + compiledCommand := exec10IndexedCommand(table) + + baselineAssessment, err := execution.PostgreSQLExplainPlan(s.ctx, runner, compiler.CompiledQuery{Command: compiledCommand}) + if err != nil { + s.fail("serialized baseline EXPLAIN", err) + } + baseline, _ := json.Marshal(baselineAssessment) + + pidSQL := fmt.Sprintf(`SELECT pg_backend_pid()::bigint AS backend, resource_seq, pg_sleep(0.05) AS dwell FROM %s WHERE resource_seq = $1`, table) + pidCommand, err := command.NewPostgreSQLCommand(pidSQL, []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterBigint, Value: int64(42)}, + }) + if err != nil { + t.Fatalf("build backend-pid witness command: %v", err) + } + pidExecutor, err := dataframepostgres.NewExecutor(s.racePool) + if err != nil { + s.fail("pid witness executor", err) + } + + const ( + rounds = 10 + concurrentPerRound = 3 + ) + var peakInFlight int64 + for round := 0; round < rounds; round++ { + var barrier sync.WaitGroup + var ready sync.WaitGroup + start := make(chan struct{}) + var inFlight int64 + assessments := make([]string, concurrentPerRound) + pids := make([]int64, concurrentPerRound) + errs := make([]error, 2*concurrentPerRound) + ready.Add(2 * concurrentPerRound) + barrier.Add(2 * concurrentPerRound) + for i := 0; i < concurrentPerRound; i++ { + go func(i int) { + defer barrier.Done() + ready.Done() + <-start + cur := atomic.AddInt64(&inFlight, 1) + for { + old := atomic.LoadInt64(&peakInFlight) + if cur <= old || atomic.CompareAndSwapInt64(&peakInFlight, old, cur) { + break + } + } + defer atomic.AddInt64(&inFlight, -1) + assessment, err := execution.PostgreSQLExplainPlan(s.ctx, runner, compiler.CompiledQuery{Command: compiledCommand}) + if err != nil { + errs[i] = err + return + } + encoded, err := json.Marshal(assessment) + if err != nil { + errs[i] = err + return + } + assessments[i] = string(encoded) + }(i) + go func(i int) { + defer barrier.Done() + ready.Done() + <-start + cur := atomic.AddInt64(&inFlight, 1) + for { + old := atomic.LoadInt64(&peakInFlight) + if cur <= old || atomic.CompareAndSwapInt64(&peakInFlight, old, cur) { + break + } + } + defer atomic.AddInt64(&inFlight, -1) + var seen []int64 + err := pidExecutor.SourceRows(s.ctx, execution.RowsRequest{ + Command: pidCommand, + DeclaredColumns: []string{"backend", "resource_seq", "dwell"}, + }, func(row map[string]any) error { + if pid, ok := exec02AsInt64(row["backend"]); ok { + seen = append(seen, pid) + } + return nil + }) + if err != nil { + errs[concurrentPerRound+i] = err + return + } + if len(seen) != 1 { + errs[concurrentPerRound+i] = fmt.Errorf("paired witness query visited %d rows, want 1", len(seen)) + return + } + pids[i] = seen[0] + }(i) + } + ready.Wait() + close(start) + barrier.Wait() + for i, err := range errs { + if err != nil { + s.fail(fmt.Sprintf("round %d concurrent operation %d", round, i), err) + } + } + seenPIDs := map[int64]bool{} + for _, pid := range pids { + if pid <= 0 { + t.Fatalf("round %d paired witness pid %d is not a real backend", round, pid) + } + if seenPIDs[pid] { + t.Fatalf("round %d paired witnesses repeated backend pid %d: %v", round, pid, pids) + } + seenPIDs[pid] = true + } + if peakInFlight < 2 { + t.Fatalf("round %d never observed concurrent in-flight operations (peak %d)", round, peakInFlight) + } + for i, assessment := range assessments { + if assessment != string(baseline) { + t.Fatalf("round %d concurrent assessment %d churned from the serialized baseline:\n got %s\nwant %s", round, i, assessment, baseline) + } + } + } + if peakInFlight < 2 { + t.Fatalf("no round achieved positive overlap (peak %d)", peakInFlight) + } + afterAssessment, err := execution.PostgreSQLExplainPlan(s.ctx, runner, compiler.CompiledQuery{Command: compiledCommand}) + if err != nil { + s.fail("post-race serialized EXPLAIN", err) + } + if after, _ := json.Marshal(afterAssessment); string(after) != string(baseline) { + t.Fatalf("post-race assessment churned:\n got %s\nwant %s", after, baseline) + } + secrets := []string{s.database, s.schema} + exec10AssertSentinelFree(t, "concurrent baseline assessment", string(baseline), secrets...) +} + +func TestZZZeroLeftoverExec10PostgresFixtures(t *testing.T) { + raw := exec10URL(t) + exec10CaptureBaseline(raw) + if exec10BaselineErr != nil { + t.Fatalf("capture baseline: %v", exec10BaselineErr) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer admin.Close() + + dbs, roles := exec10RegistrySnapshot() + for db := range dbs { + var exists bool + if err := admin.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_database WHERE datname = $1)`, db).Scan(&exists); err != nil { + t.Fatalf("check registry member %s: %v", db, err) + } + if exists { + t.Fatalf("created database %s was not dropped", db) + } + } + for role := range roles { + var exists bool + if err := admin.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_roles WHERE rolname = $1)`, role).Scan(&exists); err != nil { + t.Fatalf("check registry member role %s: %v", role, err) + } + if exists { + t.Fatalf("created role %s was not dropped", role) + } + } + + survivorCheck := func(query string, baseline []string, kind string) { + rows, err := admin.Query(ctx, query) + if err != nil { + t.Fatalf("%s prefix scan: %v", kind, err) + } + var survivors []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("%s prefix scan read: %v", kind, err) + } + survivors = append(survivors, name) + } + rows.Close() + if len(survivors) != len(baseline) { + t.Fatalf("%s verify_exec10_* survivors beyond baseline: got %v, baseline %v", kind, survivors, baseline) + } + baselineSet := map[string]bool{} + for _, name := range baseline { + baselineSet[name] = true + } + for _, name := range survivors { + if !baselineSet[name] { + t.Fatalf("unregistered verify_exec10_* %s survivor (fatal): %s", kind, name) + } + } + } + survivorCheck("SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec10\\_%' ESCAPE '\\'", exec10BaselineDBs, "database") + survivorCheck("SELECT rolname FROM pg_roles WHERE rolname LIKE 'verify\\_exec10\\_%' ESCAPE '\\'", exec10BaselineRoles, "role") +} diff --git a/internal/backends/postgres/internal/catalog/capability_compose_pg_test.go b/internal/backends/postgres/internal/catalog/capability_compose_pg_test.go new file mode 100644 index 000000000..36c45efef --- /dev/null +++ b/internal/backends/postgres/internal/catalog/capability_compose_pg_test.go @@ -0,0 +1,437 @@ +package postgres + +import ( + "context" + "errors" + "reflect" + "testing" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + fhirschema "github.com/calypr/loom/internal/fhir/schema" + "github.com/jackc/pgx/v5" +) + +const ( + ccProject = "cap-compose-p" + ccGen = "cap-compose-g" + ccAuthPath = "/programs/loom/projects/cap-compose-p/bound" + ccOtherPath = "/programs/loom/projects/cap-compose-p/foreign" +) + +type ccInventoryWrapper struct { + real *storepostgres.InventoryRepository + + scanSourceCalls int + insertCountsCalls int + publishBuildCalls int + readBuildCalls int + readCountSums int +} + +func (w *ccInventoryWrapper) ScanSourceCounts(ctx context.Context, project, datasetGeneration, resourceType string) ([]storepostgres.InventorySourceGroup, error) { + w.scanSourceCalls++ + return w.real.ScanSourceCounts(ctx, project, datasetGeneration, resourceType) +} + +func (w *ccInventoryWrapper) InsertCounts(ctx context.Context, rows []catalog.ResourceInventoryCount, batchSize int) error { + w.insertCountsCalls++ + return w.real.InsertCounts(ctx, rows, batchSize) +} + +func (w *ccInventoryWrapper) PublishBuild(ctx context.Context, build catalog.ResourceInventoryBuild) error { + w.publishBuildCalls++ + return w.real.PublishBuild(ctx, build) +} + +func (w *ccInventoryWrapper) ReadBuild(ctx context.Context, project, datasetGeneration string) (catalog.ResourceInventoryBuild, bool, error) { + w.readBuildCalls++ + return w.real.ReadBuild(ctx, project, datasetGeneration) +} + +func (w *ccInventoryWrapper) ReadCountSums(ctx context.Context, scope storepostgres.InventoryCountScope) (map[string]int64, error) { + w.readCountSums++ + return w.real.ReadCountSums(ctx, scope) +} + +type ccProfileWrapper struct { + real *storepostgres.FieldProfileRepository + + upsertCalls int + readRowsCalls int + readFilteredCalls int + readCoverageCalls int + listDistinctCalls int +} + +func (w *ccProfileWrapper) UpsertFieldProfileBatch(ctx context.Context, scope storepostgres.FieldProfileScope, docs []catalog.FieldCatalogDocument, coverage storepostgres.FieldProfileCoverage, batchSize int) error { + w.upsertCalls++ + return w.real.UpsertFieldProfileBatch(ctx, scope, docs, coverage, batchSize) +} + +func (w *ccProfileWrapper) ReadFieldProfileRows(ctx context.Context, scope storepostgres.FieldProfileScope) ([]catalog.FieldCatalogDocument, error) { + w.readRowsCalls++ + return w.real.ReadFieldProfileRows(ctx, scope) +} + +func (w *ccProfileWrapper) ReadFieldProfileRowsFiltered(ctx context.Context, filter storepostgres.FieldProfileFilter) ([]catalog.FieldCatalogDocument, error) { + w.readFilteredCalls++ + return w.real.ReadFieldProfileRowsFiltered(ctx, filter) +} + +func (w *ccProfileWrapper) ReadProfileCoverage(ctx context.Context, filter storepostgres.FieldProfileFilter) ([]storepostgres.FieldProfileCoverage, error) { + w.readCoverageCalls++ + return w.real.ReadProfileCoverage(ctx, filter) +} + +func (w *ccProfileWrapper) ListDistinctAuthResourcePaths(ctx context.Context, project string) ([]string, error) { + w.listDistinctCalls++ + return w.real.ListDistinctAuthResourcePaths(ctx, project) +} + +type ccRelationshipWrapper struct { + real *RelationshipEvidenceRepository + + scanCalls int +} + +func (w *ccRelationshipWrapper) ScanRelationshipOccurrences(ctx context.Context, scope RelationshipScope) ([]RelationshipOccurrenceGroup, error) { + w.scanCalls++ + return w.real.ScanRelationshipOccurrences(ctx, scope) +} + +func (w *ccRelationshipWrapper) reset() { w.scanCalls = 0 } + +func (w *ccInventoryWrapper) reset() { + w.scanSourceCalls, w.insertCountsCalls, w.publishBuildCalls, w.readBuildCalls, w.readCountSums = 0, 0, 0, 0, 0 +} + +func (w *ccProfileWrapper) reset() { + w.upsertCalls, w.readRowsCalls, w.readFilteredCalls, w.readCoverageCalls, w.listDistinctCalls = 0, 0, 0, 0, 0 +} + +func ccRepos(t *testing.T, s *fieldProfileService) (*ccInventoryWrapper, *ccProfileWrapper, *ccRelationshipWrapper) { + t.Helper() + inventory, err := storepostgres.NewInventoryRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new inventory repository", err) + } + profiles, err := storepostgres.NewFieldProfileRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new field profile repository", err) + } + relationships, err := NewRelationshipEvidenceRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new relationship evidence repository", err) + } + return &ccInventoryWrapper{real: inventory}, &ccProfileWrapper{real: profiles}, &ccRelationshipWrapper{real: relationships} +} + +func ccSeedGeneration(t *testing.T, s *fieldProfileService) { + t.Helper() + invRepo, err := storepostgres.NewInventoryRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("seed: inventory repository", err) + } + profRepo, err := storepostgres.NewFieldProfileRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("seed: field profile repository", err) + } + for _, row := range []struct{ rtype, id, authPath string }{ + {"Patient", "p1", ccAuthPath}, + {"Observation", "o1", ccAuthPath}, + {"Observation", "o2", ccOtherPath}, + } { + s.exec(t, "INSERT INTO "+pgx.Identifier{s.schema, "fhir_resource"}.Sanitize()+ + " (project, dataset_generation, resource_type, logical_id, auth_resource_path)"+ + " VALUES ($1, $2, $3, $4, $5)", + ccProject, ccGen, row.rtype, row.id, row.authPath) + } + invStore, err := NewInventoryAdapter(invRepo) + if err != nil { + s.fail("seed: inventory store", err) + } + if _, err := invStore.RebuildResourceInventory(s.ctx, catalog.ResourceInventoryRebuildOptions{ + Project: ccProject, DatasetGeneration: ccGen, ResourceTypes: fhirschema.ResourceTypes(), BatchSize: 2, + }); err != nil { + s.fail("seed: rebuild inventory", err) + } + profStore, err := NewFieldProfileAdapter(profRepo) + if err != nil { + s.fail("seed: profile store", err) + } + if err := profStore.WriteFieldProfile(s.ctx, fdScope(ccProject, ccGen, ccAuthPath, "Patient"), + []catalog.FieldCatalogDocument{fdDoc(ccProject, ccGen, ccAuthPath, "Patient", "gender", 1, "female")}, 100); err != nil { + s.fail("seed: write field profile", err) + } + s.exec(t, "INSERT INTO "+pgx.Identifier{s.schema, "fhir_edge_occurrence"}.Sanitize()+ + " (project, dataset_generation, auth_resource_path, label,"+ + " source_resource_type, source_logical_id, target_resource_type, target_logical_id,"+ + " source_path, source_ordinal)"+ + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + ccProject, ccGen, ccAuthPath, "subject_Patient", "Condition", "cond-cap1", "Patient", "p1", "subject", int64(0)) +} + +func ccEvidenceOptions() catalog.CapabilityEvidenceOptions { + return catalog.CapabilityEvidenceOptions{ + Project: ccProject, DatasetGeneration: ccGen, + AuthResourcePaths: []string{ccAuthPath}, + AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(false), + } +} + +func TestFullStoreCapabilityEvidenceReadsEachLegOnce(t *testing.T) { + s := newFieldProfileService(t) + ccSeedGeneration(t, s) + inventory, profiles, relationships := ccRepos(t, s) + full, err := NewFullStore(inventory, profiles, relationships) + if err != nil { + s.fail("new full store", err) + } + plainInv, err := storepostgres.NewInventoryRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("baseline inventory repository", err) + } + plainProf, err := storepostgres.NewFieldProfileRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("baseline profile repository", err) + } + plainRel, err := NewRelationshipEvidenceRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("baseline relationship repository", err) + } + plain, err := NewFullStore(plainInv, plainProf, plainRel) + if err != nil { + s.fail("new baseline full store", err) + } + opts := ccEvidenceOptions() + want, err := catalog.ReadCapabilityEvidence(s.ctx, plain, opts) + if err != nil { + s.fail("baseline capability evidence", err) + } + + inventory.reset() + profiles.reset() + relationships.reset() + got, err := catalog.ReadCapabilityEvidence(s.ctx, full, opts) + if err != nil { + s.fail("capability evidence over full store", err) + } + if !got.ResourceInventory.Available || !got.Relationships.Available || !got.FieldEnrichment.Available { + t.Fatalf("a seeded generation must answer every leg: inventory=%v relationships=%v enrichment=%v", + got.ResourceInventory.Available, got.Relationships.Available, got.FieldEnrichment.Available) + } + if !reflect.DeepEqual(got.ResourceInventory, want.ResourceInventory) || + !reflect.DeepEqual(got.Relationships, want.Relationships) || + !reflect.DeepEqual(got.FieldEnrichment, want.FieldEnrichment) { + t.Fatalf("full-store evidence diverges from the real-repository baseline:\n got %+v\nwant %+v", got, want) + } + if len(got.Relationships.Values) == 0 || len(got.FieldEnrichment.Values) == 0 { + t.Fatalf("seeded legs must carry values: rel=%d enr=%d", + len(got.Relationships.Values), len(got.FieldEnrichment.Values)) + } + counts := map[string]int64{} + for _, v := range got.ResourceInventory.Values { + counts[v.ResourceType] += v.DocumentCount + } + if counts["Patient"] != 1 || counts["Observation"] != 1 { + t.Fatalf("inventory must reflect the seeded rows: Patient=%d Observation=%d, want 1/1 (o2 lives on the foreign path)", + counts["Patient"], counts["Observation"]) + } + if inventory.readBuildCalls != 1 || inventory.readCountSums != 1 { + t.Fatalf("inventory leg boundary calls: readBuild=%d readCountSums=%d, want 1/1", inventory.readBuildCalls, inventory.readCountSums) + } + if inventory.scanSourceCalls != 0 || inventory.insertCountsCalls != 0 || inventory.publishBuildCalls != 0 { + t.Fatalf("inventory reads must not touch write calls: scan=%d insert=%d publish=%d", + inventory.scanSourceCalls, inventory.insertCountsCalls, inventory.publishBuildCalls) + } + if relationships.scanCalls != 1 { + t.Fatalf("relationship leg boundary calls: scan=%d, want exactly 1", relationships.scanCalls) + } + if profiles.readFilteredCalls != 1 || profiles.readCoverageCalls != 0 { + t.Fatalf("enrichment leg boundary calls: filtered=%d coverage=%d, want 1/0", + profiles.readFilteredCalls, profiles.readCoverageCalls) + } + if profiles.upsertCalls != 0 || profiles.readRowsCalls != 0 || profiles.listDistinctCalls != 0 { + t.Fatalf("enrichment reads must not touch other profile calls: upsert=%d rows=%d distinct=%d", + profiles.upsertCalls, profiles.readRowsCalls, profiles.listDistinctCalls) + } +} + +func TestNarrowAdaptersAnswerTheirLegAndNeverTouchForeignLegs(t *testing.T) { + s := newFieldProfileService(t) + ccSeedGeneration(t, s) + inventory, profiles, relationships := ccRepos(t, s) + + inventoryOnly, err := NewInventoryAdapter(inventory) + if err != nil { + s.fail("inventory-only adapter", err) + } + profileOnly, err := NewFieldProfileAdapter(profiles) + if err != nil { + s.fail("profile-only adapter", err) + } + relationshipOnly, err := NewRelationshipAdapter(relationships) + if err != nil { + s.fail("relationship-only adapter", err) + } + + type probe struct { + name string + call func() error + } + unrestricted := catalog.ExplicitAuthResourcePathsUnrestricted(false) + + probes := []probe{ + {"inventory-only: discover inventory", func() error { + res, err := inventoryOnly.DiscoverResourceInventory(s.ctx, catalog.ResourceInventoryOptions{ + Project: ccProject, DatasetGeneration: ccGen, AuthResourcePaths: []string{ccAuthPath}, AuthResourcePathsUnrestricted: unrestricted}) + if err != nil { + return errors.New("DiscoverResourceInventory: " + errText(err)) + } + if !res.Available || res.Status != catalog.EvidenceAvailable || len(res.Values) == 0 { + return errors.New("inventory adapter must answer the seeded generation with available complete evidence") + } + return nil + }}, + {"inventory-only: read alias", func() error { + res, err := inventoryOnly.ReadResourceInventory(s.ctx, catalog.ResourceInventoryOptions{ + Project: ccProject, DatasetGeneration: ccGen, AuthResourcePaths: []string{ccAuthPath}, AuthResourcePathsUnrestricted: unrestricted}) + if err != nil { + return errors.New("ReadResourceInventory: " + errText(err)) + } + if !res.Available || len(res.Values) == 0 { + return errors.New("ReadResourceInventory alias must answer the seeded generation") + } + return nil + }}, + {"profile-only: enrichment", func() error { + res, err := profileOnly.DiscoverFieldEnrichment(s.ctx, catalog.FieldEnrichmentOptions{ + Project: ccProject, DatasetGeneration: ccGen, AuthResourcePaths: []string{ccAuthPath}, AuthResourcePathsUnrestricted: unrestricted}) + if err != nil { + return errors.New("DiscoverFieldEnrichment: " + errText(err)) + } + if !res.Available || len(res.Values) == 0 { + return errors.New("profile adapter must answer enrichment for the seeded scope") + } + return nil + }}, + {"profile-only: field discovery", func() error { + fields, err := profileOnly.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: ccProject, DatasetGeneration: ccGen, AuthResourcePaths: []string{ccAuthPath}, AuthResourcePathsUnrestricted: unrestricted}) + if err != nil { + return errors.New("DiscoverFields: " + errText(err)) + } + if len(fields) == 0 { + return errors.New("DiscoverFields must answer the seeded scope") + } + return nil + }}, + {"profile-only: auth path listing", func() error { + paths, err := profileOnly.DiscoverExistingAuthResourcePaths(s.ctx, catalog.AuthResourcePathOptions{Project: ccProject, AllGenerations: true}) + if err != nil { + return errors.New("DiscoverExistingAuthResourcePaths: " + errText(err)) + } + if len(paths) != 1 || paths[0] != ccAuthPath { + return errors.New("DiscoverExistingAuthResourcePaths must list only the seeded auth path") + } + return nil + }}, + {"profile-only: read profile", func() error { + docs, err := profileOnly.ReadFieldProfile(s.ctx, fdScope(ccProject, ccGen, ccAuthPath, "Patient")) + if err != nil { + return errors.New("ReadFieldProfile: " + errText(err)) + } + if len(docs) == 0 { + return errors.New("ReadFieldProfile must answer the seeded scope") + } + return nil + }}, + {"relationship-only: discover", func() error { + res, err := relationshipOnly.DiscoverRelationshipObservations(s.ctx, catalog.RelationshipObservationOptions{ + Project: ccProject, DatasetGeneration: ccGen, AuthResourcePaths: []string{ccAuthPath}, AuthResourcePathsUnrestricted: unrestricted}) + if err != nil { + return errors.New("DiscoverRelationshipObservations: " + errText(err)) + } + if !res.Available || len(res.Values) == 0 { + return errors.New("relationship adapter must answer the seeded generation") + } + return nil + }}, + {"relationship-only: read alias", func() error { + res, err := relationshipOnly.ReadRelationshipObservations(s.ctx, catalog.RelationshipObservationOptions{ + Project: ccProject, DatasetGeneration: ccGen, AuthResourcePaths: []string{ccAuthPath}, AuthResourcePathsUnrestricted: unrestricted}) + if err != nil { + return errors.New("ReadRelationshipObservations: " + errText(err)) + } + if !res.Available || len(res.Values) == 0 { + return errors.New("ReadRelationshipObservations alias must answer the seeded generation") + } + return nil + }}, + } + + inventory.reset() + profiles.reset() + relationships.reset() + for _, p := range probes { + func() { + defer func() { + if rec := recover(); rec != nil { + t.Fatalf("%s: panic on an assembled adapter leg: %v", p.name, rec) + } + }() + if err := p.call(); err != nil { + t.Fatalf("%s: %v", p.name, err) + } + }() + } + if inventory.readBuildCalls != 2 || inventory.readCountSums != 2 { + t.Fatalf("inventory adapter boundary calls: readBuild=%d readCountSums=%d, want 2/2", + inventory.readBuildCalls, inventory.readCountSums) + } + if inventory.scanSourceCalls != 0 || inventory.insertCountsCalls != 0 || inventory.publishBuildCalls != 0 { + t.Fatalf("inventory reads must not touch write calls: scan=%d insert=%d publish=%d", + inventory.scanSourceCalls, inventory.insertCountsCalls, inventory.publishBuildCalls) + } + if profiles.upsertCalls != 0 || profiles.readCoverageCalls != 0 { + t.Fatalf("profile reads must not upsert or fall back to coverage: upsert=%d coverage=%d", + profiles.upsertCalls, profiles.readCoverageCalls) + } + if profiles.readFilteredCalls != 2 || profiles.readRowsCalls != 1 || profiles.listDistinctCalls != 1 { + t.Fatalf("profile adapter boundary calls: filtered=%d rows=%d distinct=%d, want 2/1/1", + profiles.readFilteredCalls, profiles.readRowsCalls, profiles.listDistinctCalls) + } + if relationships.scanCalls != 2 { + t.Fatalf("relationship adapter boundary calls: scan=%d, want exactly 2", relationships.scanCalls) + } +} + +func errText(err error) string { + if err == nil { + return "expected typed refusal, got nil" + } + return err.Error() +} + +func TestNewFullStoreRequiresEveryRepository(t *testing.T) { + s := newFieldProfileService(t) + inventory, profiles, relationships := ccRepos(t, s) + if _, err := NewFullStore(nil, profiles.real, relationships.real); err == nil { + t.Fatal("missing inventory must be refused") + } + if _, err := NewFullStore(inventory.real, nil, relationships.real); err == nil { + t.Fatal("missing profiles must be refused") + } + if _, err := NewFullStore(inventory.real, profiles.real, nil); err == nil { + t.Fatal("missing relationships must be refused") + } + full, err := NewFullStore(inventory.real, profiles.real, relationships.real) + if err != nil { + s.fail("full store with all repositories", err) + } + if full == nil { + t.Fatal("full store is nil") + } + var _ catalog.CapabilityEvidenceReader = full +} diff --git a/internal/backends/postgres/internal/catalog/capability_types_test.go b/internal/backends/postgres/internal/catalog/capability_types_test.go new file mode 100644 index 000000000..da8c66cf5 --- /dev/null +++ b/internal/backends/postgres/internal/catalog/capability_types_test.go @@ -0,0 +1,224 @@ +package postgres + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/calypr/loom/internal/catalog" +) + +var ( + _ catalog.CapabilityEvidenceReader = (*Store)(nil) + _ catalog.InventoryRebuilder = (*InventoryAdapter)(nil) + _ catalog.FieldProfileWriter = (*FieldProfileAdapter)(nil) + _ catalog.RelationshipDiscoverer = (*RelationshipAdapter)(nil) +) + +type capabilityLeg struct { + adapterType reflect.Type + ownMethods []string + foreignMethod []string +} + +func TestPartialAdaptersCannotBePassedAsCapabilityEvidenceReaders(t *testing.T) { + readerType := reflect.TypeOf((*catalog.CapabilityEvidenceReader)(nil)).Elem() + legs := []capabilityLeg{ + { + adapterType: reflect.TypeOf((*InventoryAdapter)(nil)), + ownMethods: []string{ + "RebuildResourceInventory", "ReadResourceInventory", "DiscoverResourceInventory", + }, + foreignMethod: []string{ + "WriteFieldProfile", "ReadFieldProfile", "DiscoverFieldEnrichment", + "DiscoverFields", "DiscoverExistingAuthResourcePaths", + "DiscoverRelationshipObservations", "ReadRelationshipObservations", + }, + }, + { + adapterType: reflect.TypeOf((*FieldProfileAdapter)(nil)), + ownMethods: []string{ + "WriteFieldProfile", "ReadFieldProfile", "DiscoverFieldEnrichment", + "DiscoverFields", "DiscoverExistingAuthResourcePaths", + }, + foreignMethod: []string{ + "RebuildResourceInventory", "ReadResourceInventory", "DiscoverResourceInventory", + "DiscoverRelationshipObservations", "ReadRelationshipObservations", + }, + }, + { + adapterType: reflect.TypeOf((*RelationshipAdapter)(nil)), + ownMethods: []string{ + "DiscoverRelationshipObservations", "ReadRelationshipObservations", + }, + foreignMethod: []string{ + "RebuildResourceInventory", "ReadResourceInventory", "DiscoverResourceInventory", + "WriteFieldProfile", "ReadFieldProfile", "DiscoverFieldEnrichment", + "DiscoverFields", "DiscoverExistingAuthResourcePaths", + }, + }, + } + for _, leg := range legs { + if leg.adapterType.Implements(readerType) { + t.Fatalf("%s must not satisfy catalog.CapabilityEvidenceReader", leg.adapterType) + } + for _, method := range leg.ownMethods { + if _, ok := leg.adapterType.MethodByName(method); !ok { + t.Fatalf("%s must serve its own capability %s", leg.adapterType, method) + } + } + for _, method := range leg.foreignMethod { + if _, ok := leg.adapterType.MethodByName(method); ok { + t.Fatalf("%s must not advertise foreign capability %s", leg.adapterType, method) + } + } + } + fullType := reflect.TypeOf((*Store)(nil)) + if !fullType.Implements(readerType) { + t.Fatal("*Store must satisfy catalog.CapabilityEvidenceReader") + } + for _, leg := range legs { + for _, method := range leg.ownMethods { + if _, ok := fullType.MethodByName(method); !ok { + t.Fatalf("*Store must keep the composed capability %s", method) + } + } + } +} + +func TestAdapterConstructorsRefuseNilBoundaries(t *testing.T) { + if _, err := NewInventoryAdapter(nil); err == nil { + t.Fatal("NewInventoryAdapter(nil) accepted a nil persistence boundary") + } + if _, err := NewFieldProfileAdapter(nil); err == nil { + t.Fatal("NewFieldProfileAdapter(nil) accepted a nil persistence boundary") + } + if _, err := NewRelationshipAdapter(nil); err == nil { + t.Fatal("NewRelationshipAdapter(nil) accepted a nil persistence boundary") + } +} + +func TestZeroValueAdaptersRefuseTypedNeverPanic(t *testing.T) { + ctx := context.Background() + type probe struct { + name string + call func() error + } + probes := []probe{ + {"inventory-only: rebuild", func() error { + _, err := (&InventoryAdapter{}).RebuildResourceInventory(ctx, catalog.ResourceInventoryRebuildOptions{ + Project: "p", DatasetGeneration: "g", ResourceTypes: []string{"Patient"}, BatchSize: 2}) + return refusalProbe(err) + }}, + {"inventory-only: discover", func() error { + res, err := (&InventoryAdapter{}).DiscoverResourceInventory(ctx, catalog.ResourceInventoryOptions{Project: "p", DatasetGeneration: "g"}) + if err = refusalProbe(err); err != nil { + return err + } + return typedUnavailableResourceInventory(res) + }}, + {"inventory-only: read alias", func() error { + res, err := (&InventoryAdapter{}).ReadResourceInventory(ctx, catalog.ResourceInventoryOptions{Project: "p", DatasetGeneration: "g"}) + if err = refusalProbe(err); err != nil { + return err + } + return typedUnavailableResourceInventory(res) + }}, + {"profile-only: write", func() error { + return refusalProbe((&FieldProfileAdapter{}).WriteFieldProfile(ctx, fdScope("p", "g", "/a", "Patient"), nil, 100)) + }}, + {"profile-only: read", func() error { + _, err := (&FieldProfileAdapter{}).ReadFieldProfile(ctx, fdScope("p", "g", "/a", "Patient")) + return refusalProbe(err) + }}, + {"profile-only: enrichment", func() error { + res, err := (&FieldProfileAdapter{}).DiscoverFieldEnrichment(ctx, catalog.FieldEnrichmentOptions{Project: "p", DatasetGeneration: "g"}) + if err = refusalProbe(err); err != nil { + return err + } + if res.Available || res.Status != catalog.EvidenceUnavailable || len(res.Values) != 0 { + return errors.New("enrichment refusal must be typed unavailable with no values") + } + return nil + }}, + {"profile-only: discover fields", func() error { + _, err := (&FieldProfileAdapter{}).DiscoverFields(ctx, catalog.PopulatedFieldOptions{Project: "p", DatasetGeneration: "g"}) + return refusalProbe(err) + }}, + {"profile-only: auth path listing", func() error { + _, err := (&FieldProfileAdapter{}).DiscoverExistingAuthResourcePaths(ctx, catalog.AuthResourcePathOptions{Project: "p", AllGenerations: true}) + return refusalProbe(err) + }}, + {"relationship-only: discover", func() error { + res, err := (&RelationshipAdapter{}).DiscoverRelationshipObservations(ctx, catalog.RelationshipObservationOptions{Project: "p", DatasetGeneration: "g"}) + if err = refusalProbe(err); err != nil { + return err + } + if res.Available || res.Status != catalog.EvidenceUnavailable || len(res.Values) != 0 { + return errors.New("relationship refusal must be typed unavailable with no values") + } + return nil + }}, + {"relationship-only: read alias", func() error { + res, err := (&RelationshipAdapter{}).ReadRelationshipObservations(ctx, catalog.RelationshipObservationOptions{Project: "p", DatasetGeneration: "g"}) + if err = refusalProbe(err); err != nil { + return err + } + if res.Available || res.Status != catalog.EvidenceUnavailable || len(res.Values) != 0 { + return errors.New("relationship alias refusal must be typed unavailable with no values") + } + return nil + }}, + {"composed zero store: every leg", func() error { + unassembled := &Store{} + _, err := unassembled.RebuildResourceInventory(ctx, catalog.ResourceInventoryRebuildOptions{ + Project: "p", DatasetGeneration: "g", ResourceTypes: []string{"Patient"}, BatchSize: 2}) + if err = refusalProbe(err); err != nil { + return err + } + res, err := unassembled.DiscoverResourceInventory(ctx, catalog.ResourceInventoryOptions{Project: "p", DatasetGeneration: "g"}) + if err = refusalProbe(err); err != nil { + return err + } + if err := typedUnavailableResourceInventory(res); err != nil { + return err + } + _, err = unassembled.ReadFieldProfile(ctx, fdScope("p", "g", "/a", "Patient")) + if err = refusalProbe(err); err != nil { + return err + } + _, err = unassembled.DiscoverRelationshipObservations(ctx, catalog.RelationshipObservationOptions{Project: "p", DatasetGeneration: "g"}) + return refusalProbe(err) + }}, + } + for _, p := range probes { + func() { + defer func() { + if rec := recover(); rec != nil { + t.Fatalf("%s: panic instead of typed refusal: %v", p.name, rec) + } + }() + if err := p.call(); err != nil { + t.Fatalf("%s: %v", p.name, err) + } + }() + } +} + +func refusalProbe(err error) error { + if err == nil { + return errors.New("expected typed refusal, got nil") + } + if !errors.Is(err, ErrStoreLegNotAssembled) { + return errors.New("refusal must wrap ErrStoreLegNotAssembled, got " + err.Error()) + } + return nil +} + +func typedUnavailableResourceInventory(res catalog.ResourceInventoryResult) error { + if res.Available || res.Status != catalog.EvidenceUnavailable || len(res.Values) != 0 { + return errors.New("inventory refusal must be typed unavailable with no values") + } + return nil +} diff --git a/internal/backends/postgres/internal/catalog/field_discovery.go b/internal/backends/postgres/internal/catalog/field_discovery.go new file mode 100644 index 000000000..72c9f8abb --- /dev/null +++ b/internal/backends/postgres/internal/catalog/field_discovery.go @@ -0,0 +1,107 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "sort" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + fhirschema "github.com/calypr/loom/internal/fhir/schema" +) + +var ErrAllGenerationsWithGeneration = errors.New("AllGenerations conflicts with a selected datasetGeneration") + +func (s *FieldProfileAdapter) DiscoverFields(ctx context.Context, opts catalog.PopulatedFieldOptions) ([]catalog.PopulatedField, error) { + if err := requireLeg("field-profile", s.profiles != nil, "DiscoverFields"); err != nil { + return nil, err + } + fields := []catalog.PopulatedField{} + datasetGeneration := catalog.NormalizeDatasetGeneration(opts.DatasetGeneration) + authPathsUnrestricted := catalog.EffectiveAuthResourcePathsUnrestricted(opts.AuthResourcePaths, opts.AuthResourcePathsUnrestricted) + if !authPathsUnrestricted && len(opts.AuthResourcePaths) == 0 { + return fields, nil + } + var authPaths []string + if !authPathsUnrestricted { + authPaths = append([]string(nil), opts.AuthResourcePaths...) + } + resourceTypes := []string{opts.ResourceType} + if opts.ResourceType == "" { + resourceTypes = fhirschema.ResourceTypes() + } + filter := storepostgres.FieldProfileFilter{ + Project: opts.Project, + DatasetGeneration: datasetGeneration, + ResourceTypes: append([]string(nil), resourceTypes...), + AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(authPathsUnrestricted), + AuthResourcePaths: authPaths, + } + rows, err := s.profiles.ReadFieldProfileRowsFiltered(ctx, filter) + if err != nil { + return nil, fmt.Errorf("postgres catalog: discover fields %s/%s: %w", opts.Project, datasetGeneration, err) + } + for _, doc := range rows { + if doc.RecordType == catalog.FieldProfileStatusRecordType { + continue + } + if opts.PivotOnly && !doc.PivotCandidate { + continue + } + resourceType, ok := fhirschema.ConcreteResourceType(doc.ResourceType) + if !ok { + continue + } + fields = append(fields, populatedFieldFromProfileDocument(doc, resourceType)) + } + return fields, nil +} + +func (s *FieldProfileAdapter) DiscoverExistingAuthResourcePaths(ctx context.Context, opts catalog.AuthResourcePathOptions) ([]string, error) { + if err := requireLeg("field-profile", s.profiles != nil, "DiscoverExistingAuthResourcePaths"); err != nil { + return nil, err + } + datasetGeneration := catalog.NormalizeDatasetGeneration(opts.DatasetGeneration) + if opts.AllGenerations { + if datasetGeneration != "" { + return nil, fmt.Errorf("%w: datasetGeneration %q must be empty when AllGenerations is set", + ErrAllGenerationsWithGeneration, opts.DatasetGeneration) + } + paths, err := s.profiles.ListDistinctAuthResourcePaths(ctx, opts.Project) + if err != nil { + return nil, fmt.Errorf("postgres catalog: discover existing auth paths %s (all generations): %w", opts.Project, err) + } + sort.Strings(paths) + return paths, nil + } + filter := storepostgres.FieldProfileFilter{ + Project: opts.Project, + DatasetGeneration: datasetGeneration, + AllResourceTypes: true, + AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + } + rows, err := s.profiles.ReadFieldProfileRowsFiltered(ctx, filter) + if err != nil { + return nil, fmt.Errorf("postgres catalog: discover existing auth paths %s/%s: %w", opts.Project, datasetGeneration, err) + } + result := make([]string, 0, 16) + seen := make(map[string]struct{}, 16) + for _, doc := range rows { + path := doc.AuthResourcePath + if path == "" { + continue + } + if _, duplicate := seen[path]; duplicate { + continue + } + seen[path] = struct{}{} + result = append(result, path) + } + sort.Strings(result) + return result, nil +} + +func populatedFieldFromProfileDocument(doc catalog.FieldCatalogDocument, resourceType string) catalog.PopulatedField { + return catalog.PopulatedField(fieldObservationFromProfileDocument(doc, resourceType)) +} diff --git a/internal/backends/postgres/internal/catalog/field_discovery_test.go b/internal/backends/postgres/internal/catalog/field_discovery_test.go new file mode 100644 index 000000000..d51b22b93 --- /dev/null +++ b/internal/backends/postgres/internal/catalog/field_discovery_test.go @@ -0,0 +1,533 @@ +package postgres + +import ( + "context" + "errors" + "reflect" + "testing" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" +) + +const ( + fdProject = "fd-p1" + fdGenA = "generation-a" + fdPath1 = "/fhir/clinical" + fdPath2 = "/fhir/finance" +) + +func fdDoc(project, generation, authPath, resourceType, path string, docCount int64, distinct ...string) catalog.FieldCatalogDocument { + complete := true + return catalog.FieldCatalogDocument{ + Key: resourceType + "|" + path, + Project: project, DatasetGeneration: generation, AuthResourcePath: authPath, + ResourceType: resourceType, Path: path, Kind: "scalar", + DocCount: docCount, SampleCount: len(distinct), DistinctValues: append([]string(nil), distinct...), + EnrichmentComplete: &complete, + } +} + +type fdRecordingStore struct { + real *storepostgres.FieldProfileRepository + filteredCalls int + lastFilter storepostgres.FieldProfileFilter + injectReadError error + distinctCalls int + lastDistinctScope string + injectDistinctErr error + reverseDistinctOrder bool +} + +func (w *fdRecordingStore) UpsertFieldProfileBatch(ctx context.Context, scope storepostgres.FieldProfileScope, docs []catalog.FieldCatalogDocument, coverage storepostgres.FieldProfileCoverage, batchSize int) error { + return w.real.UpsertFieldProfileBatch(ctx, scope, docs, coverage, batchSize) +} + +func (w *fdRecordingStore) ReadFieldProfileRows(ctx context.Context, scope storepostgres.FieldProfileScope) ([]catalog.FieldCatalogDocument, error) { + return w.real.ReadFieldProfileRows(ctx, scope) +} + +func (w *fdRecordingStore) ReadFieldProfileRowsFiltered(ctx context.Context, filter storepostgres.FieldProfileFilter) ([]catalog.FieldCatalogDocument, error) { + w.filteredCalls++ + w.lastFilter = filter + if w.injectReadError != nil { + return nil, w.injectReadError + } + return w.real.ReadFieldProfileRowsFiltered(ctx, filter) +} + +func (w *fdRecordingStore) ReadProfileCoverage(ctx context.Context, filter storepostgres.FieldProfileFilter) ([]storepostgres.FieldProfileCoverage, error) { + return w.real.ReadProfileCoverage(ctx, filter) +} + +func (w *fdRecordingStore) ListDistinctAuthResourcePaths(ctx context.Context, project string) ([]string, error) { + w.distinctCalls++ + w.lastDistinctScope = project + if w.injectDistinctErr != nil { + return nil, w.injectDistinctErr + } + paths, err := w.real.ListDistinctAuthResourcePaths(ctx, project) + if err != nil { + return nil, err + } + if w.reverseDistinctOrder { + for i, j := 0, len(paths)-1; i < j; i, j = i+1, j-1 { + paths[i], paths[j] = paths[j], paths[i] + } + } + return paths, nil +} + +func fdWrite(t *testing.T, s *fieldProfileService, scope storepostgres.FieldProfileScope, docs ...catalog.FieldCatalogDocument) { + t.Helper() + store := s.adapter(t) + if err := store.WriteFieldProfile(s.ctx, scope, docs, 100); err != nil { + s.fail("write field profile "+scope.ResourceType+" "+scope.AuthResourcePath, err) + } +} + +func fdScope(project, generation, authPath, resourceType string) storepostgres.FieldProfileScope { + return storepostgres.FieldProfileScope{Project: project, DatasetGeneration: generation, AuthResourcePath: authPath, ResourceType: resourceType} +} + +func fdUnrestricted() *bool { return catalog.ExplicitAuthResourcePathsUnrestricted(true) } + +func TestDiscoverFieldsPostgreSQLExactSetOrderAndMapping(t *testing.T) { + s := newFieldProfileService(t) + full := fdDoc(fdProject, fdGenA, fdPath1, "Patient", "gender", 3, "female", "male") + full.MaxItems = 1 + full.DistinctTruncated = true + full.ExtensionValues = []catalog.ExtensionValueObservation{{ + URL: "http://ext/gender-identity", SourcePath: "gender.extension", + ValuePath: "valueCoding.code", ValueType: "code", URLPath: []string{"http://ext/root", "http://ext/gender-identity"}, + }} + full.SemanticObservations = []catalog.SemanticObservation{{ + SchemaVersion: 1, Status: "observed", Population: 2, + Examples: []string{"female"}, ExtensionURLPath: []string{"http://ext/root"}, + }} + fdWrite(t, s, fdScope(fdProject, fdGenA, fdPath1, "Patient"), + full, + fdDoc(fdProject, fdGenA, fdPath1, "Patient", "active", 5), + fdDoc(fdProject, fdGenA, fdPath1, "Patient", "birthDate", 3), + func() catalog.FieldCatalogDocument { + markerComplete := true + return catalog.FieldCatalogDocument{Key: "Patient|status", RecordType: catalog.FieldProfileStatusRecordType, + Project: fdProject, DatasetGeneration: fdGenA, AuthResourcePath: fdPath1, ResourceType: "Patient", + EnrichmentComplete: &markerComplete} + }(), + ) + fdWrite(t, s, fdScope(fdProject, fdGenA, fdPath2, "Patient"), fdDoc(fdProject, fdGenA, fdPath2, "Patient", "deceasedBoolean", 1)) + fdWrite(t, s, fdScope(fdProject, fdGenA, fdPath1, "Observation"), fdDoc(fdProject, fdGenA, fdPath1, "Observation", "status", 9)) + + store := s.adapter(t) + fields, err := store.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: fdProject, DatasetGeneration: fdGenA, AuthResourcePathsUnrestricted: fdUnrestricted(), + }) + if err != nil { + s.fail("discover fields", err) + } + + wantOrder := []struct{ resourceType, path, authPath string }{ + {"Observation", "status", fdPath1}, + {"Patient", "active", fdPath1}, + {"Patient", "birthDate", fdPath1}, + {"Patient", "deceasedBoolean", fdPath2}, + {"Patient", "gender", fdPath1}, + } + if len(fields) != len(wantOrder) { + t.Fatalf("discovered %d fields, want %d: %#v", len(fields), len(wantOrder), fields) + } + for i, want := range wantOrder { + got := fields[i] + if got.ResourceType != want.resourceType || got.Path != want.path || got.AuthResourcePath != want.authPath { + t.Fatalf("order position %d = %s.%s (%q), want %s.%s (%q): full order %#v", + i, got.ResourceType, got.Path, got.AuthResourcePath, want.resourceType, want.path, want.authPath, fields) + } + if got.Project != fdProject || got.DatasetGeneration != fdGenA || got.Kind != "scalar" { + t.Fatalf("field %s.%s lost identity: %+v", got.ResourceType, got.Path, got) + } + } + t.Logf("registered PG discovery order witness (resource_type, path): %v", wantOrder) + + var gender *catalog.PopulatedField + for i := range fields { + if fields[i].Path == "gender" { + gender = &fields[i] + } + } + if gender == nil { + t.Fatal("gender missing from discovery") + } + if gender.DocCount != 3 || gender.MaxItems != 1 || gender.SampleCount != 2 || + !gender.DistinctTruncated || !reflect.DeepEqual(gender.DistinctValues, []string{"female", "male"}) { + t.Fatalf("gender scalar columns lost in mapping: %+v", gender) + } + if !reflect.DeepEqual(gender.ExtensionValues, full.ExtensionValues) { + t.Fatalf("extension values lost in mapping: got %#v want %#v", gender.ExtensionValues, full.ExtensionValues) + } + if len(gender.SemanticObservations) != 1 || gender.SemanticObservations[0].Status != "observed" || + gender.SemanticObservations[0].Population != 2 || + !reflect.DeepEqual(gender.SemanticObservations[0].Examples, []string{"female"}) { + t.Fatalf("semantic observations lost in mapping: %#v", gender.SemanticObservations) + } + + gender.DistinctValues[0] = "poisoned" + reread, err := store.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: fdProject, DatasetGeneration: fdGenA, AuthResourcePathsUnrestricted: fdUnrestricted(), + }) + if err != nil { + s.fail("re-read fields", err) + } + for _, f := range reread { + if f.Path == "gender" && reflect.DeepEqual(f.DistinctValues, []string{"female", "male"}) == false { + t.Fatalf("returned view aliases persisted state: %#v", f.DistinctValues) + } + } +} + +func TestDiscoverFieldsRestrictedEmptyMakesZeroRoundTrips(t *testing.T) { + s := newFieldProfileService(t) + wrapper := &fdRecordingStore{real: s.realRepository(t)} + store, err := NewFieldProfileAdapter(wrapper) + if err != nil { + s.fail("new field profile store", err) + } + fields, err := store.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: fdProject, DatasetGeneration: fdGenA, + AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(false), + }) + if err != nil { + s.fail("restricted-empty discovery", err) + } + if len(fields) != 0 { + t.Fatalf("restricted-empty returned %d fields: must be empty: %#v", len(fields), fields) + } + if wrapper.filteredCalls != 0 { + t.Fatalf("restricted-empty issued %d store calls: the no-query law was violated", wrapper.filteredCalls) + } +} + +func TestDiscoverFieldsUnrestrictedIgnoresStrayPaths(t *testing.T) { + s := newFieldProfileService(t) + wrapper := &fdRecordingStore{real: s.realRepository(t)} + store, err := NewFieldProfileAdapter(wrapper) + if err != nil { + s.fail("new field profile store", err) + } + if _, err := store.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: fdProject, DatasetGeneration: fdGenA, + AuthResourcePathsUnrestricted: fdUnrestricted(), + AuthResourcePaths: []string{fdPath1, "not/authorized"}, + }); err != nil { + s.fail("unrestricted discovery", err) + } + if wrapper.filteredCalls != 1 { + t.Fatalf("store calls = %d, want exactly 1", wrapper.filteredCalls) + } + if wrapper.lastFilter.AuthResourcePathsUnrestricted == nil || !*wrapper.lastFilter.AuthResourcePathsUnrestricted { + t.Fatalf("filter lost unrestricted mode: %+v", wrapper.lastFilter) + } + if len(wrapper.lastFilter.AuthResourcePaths) != 0 { + t.Fatalf("unrestricted read carried stray paths %v: they must be dropped, mirroring the reference adapter", wrapper.lastFilter.AuthResourcePaths) + } +} + +func TestDiscoverFieldsNamedTypePivotAndConcreteLaws(t *testing.T) { + s := newFieldProfileService(t) + fdWrite(t, s, fdScope(fdProject, fdGenA, fdPath1, "Patient"), + fdDoc(fdProject, fdGenA, fdPath1, "Patient", "gender", 3), + func() catalog.FieldCatalogDocument { + d := fdDoc(fdProject, fdGenA, fdPath1, "Patient", "maritalStatus", 2) + d.PivotCandidate = true + d.PivotKind = "CODED_PIVOT" + d.PivotColumns = []string{"married", "single"} + return d + }(), + ) + fdWrite(t, s, fdScope(fdProject, fdGenA, fdPath1, "FakeResource"), + fdDoc(fdProject, fdGenA, fdPath1, "FakeResource", "anything", 1)) + + store := s.adapter(t) + + named, err := store.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: fdProject, DatasetGeneration: fdGenA, ResourceType: "Patient", + AuthResourcePathsUnrestricted: fdUnrestricted(), + }) + if err != nil { + s.fail("named-type discovery", err) + } + if len(named) != 2 { + t.Fatalf("named-type discovery returned %#v, want the two Patient fields", named) + } + + pivots, err := store.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: fdProject, DatasetGeneration: fdGenA, ResourceType: "Patient", PivotOnly: true, + AuthResourcePathsUnrestricted: fdUnrestricted(), + }) + if err != nil { + s.fail("pivot-only discovery", err) + } + if len(pivots) != 1 || pivots[0].Path != "maritalStatus" || + pivots[0].PivotKind != "CODED_PIVOT" || !reflect.DeepEqual(pivots[0].PivotColumns, []string{"married", "single"}) { + t.Fatalf("pivot-only discovery = %#v, want exactly the pivot candidate", pivots) + } + + fake, err := store.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: fdProject, DatasetGeneration: fdGenA, ResourceType: "FakeResource", + AuthResourcePathsUnrestricted: fdUnrestricted(), + }) + if err != nil { + s.fail("non-concrete discovery", err) + } + if len(fake) != 0 { + t.Fatalf("non-concrete rows surfaced as fields: %#v", fake) + } +} + +func TestDiscoverFieldsStoreFailureIsWrappedNotEmpty(t *testing.T) { + s := newFieldProfileService(t) + boom := errors.New("injected filtered read failure") + wrapper := &fdRecordingStore{real: s.realRepository(t), injectReadError: boom} + store, err := NewFieldProfileAdapter(wrapper) + if err != nil { + s.fail("new field profile store", err) + } + fields, err := store.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: fdProject, DatasetGeneration: fdGenA, AuthResourcePathsUnrestricted: fdUnrestricted(), + }) + if err == nil { + t.Fatal("store failure was swallowed into a successful read") + } + if fields != nil { + t.Fatalf("failed discovery still returned fields: %#v", fields) + } + if !errors.Is(err, boom) { + t.Fatalf("store failure lost its identity in wrapping: %v", err) + } +} + +func TestDiscoverExistingAuthResourcePathsSortedDistinctIncludingMarker(t *testing.T) { + s := newFieldProfileService(t) + fdWrite(t, s, fdScope(fdProject, fdGenA, fdPath2, "Observation"), fdDoc(fdProject, fdGenA, fdPath2, "Observation", "status", 4)) + fdWrite(t, s, fdScope(fdProject, fdGenA, fdPath1, "Patient"), + fdDoc(fdProject, fdGenA, fdPath1, "Patient", "gender", 3), + fdDoc(fdProject, fdGenA, fdPath1, "Patient", "birthDate", 3)) + truncated := false + complete := false + fdWrite(t, s, fdScope(fdProject, fdGenA, "/fhir/research", "Patient"), + catalog.FieldCatalogDocument{ + Key: "Patient|status", RecordType: catalog.FieldProfileStatusRecordType, + Project: fdProject, DatasetGeneration: fdGenA, AuthResourcePath: "/fhir/research", + ResourceType: "Patient", EnrichmentComplete: &complete, EnrichmentTruncated: truncated, + }) + fdWrite(t, s, fdScope(fdProject, fdGenA, "/fhir/custom", "CustomCarePlan"), + fdDoc(fdProject, fdGenA, "/fhir/custom", "CustomCarePlan", "status", 2)) + fdWrite(t, s, fdScope("fd-other", fdGenA, "/fhir/foreign", "Patient"), fdDoc("fd-other", fdGenA, "/fhir/foreign", "Patient", "gender", 1)) + fdWrite(t, s, fdScope(fdProject, "generation-z", fdPath1, "Patient"), fdDoc(fdProject, "generation-z", fdPath1, "Patient", "gender", 1)) + + store := s.adapter(t) + paths, err := store.DiscoverExistingAuthResourcePaths(s.ctx, catalog.AuthResourcePathOptions{ + Project: fdProject, DatasetGeneration: fdGenA, + }) + if err != nil { + s.fail("auth-path listing", err) + } + want := []string{fdPath1, "/fhir/custom", fdPath2, "/fhir/research"} + if !reflect.DeepEqual(paths, want) { + t.Fatalf("auth paths = %v, want sorted distinct %v", paths, want) + } +} + +func TestDiscoverExistingAuthResourcePathsAllGenerationsConflictRefusedNoRoundTrips(t *testing.T) { + s := newFieldProfileService(t) + wrapper := &fdRecordingStore{real: s.realRepository(t)} + store, err := NewFieldProfileAdapter(wrapper) + if err != nil { + s.fail("new field profile store", err) + } + + _, err = store.DiscoverExistingAuthResourcePaths(s.ctx, catalog.AuthResourcePathOptions{ + Project: fdProject, DatasetGeneration: fdGenA, AllGenerations: true, + }) + if !errors.Is(err, ErrAllGenerationsWithGeneration) { + t.Fatalf("AllGenerations+generation error = %v, want %v", err, ErrAllGenerationsWithGeneration) + } + if wrapper.filteredCalls != 0 || wrapper.distinctCalls != 0 { + t.Fatalf("conflict refused after %d filtered + %d distinct store calls; want zero of both", + wrapper.filteredCalls, wrapper.distinctCalls) + } +} + +func TestDiscoverExistingAuthResourcePathsAllGenerationsIsCrossGenerationUnion(t *testing.T) { + s := newFieldProfileService(t) + fdWrite(t, s, fdScope(fdProject, fdGenA, fdPath1, "Patient"), + fdDoc(fdProject, fdGenA, fdPath1, "Patient", "gender", 3), + fdDoc(fdProject, fdGenA, fdPath1, "Patient", "birthDate", 3)) + fdWrite(t, s, fdScope(fdProject, fdGenA, fdPath2, "Observation"), fdDoc(fdProject, fdGenA, fdPath2, "Observation", "status", 4)) + fdWrite(t, s, fdScope(fdProject, "generation-z", fdPath1, "Patient"), fdDoc(fdProject, "generation-z", fdPath1, "Patient", "gender", 1)) + fdWrite(t, s, fdScope(fdProject, "generation-z", "/fhir/archive", "Observation"), fdDoc(fdProject, "generation-z", "/fhir/archive", "Observation", "status", 2)) + fdWrite(t, s, fdScope(fdProject, "", "/fhir/legacy", "Patient"), fdDoc(fdProject, "", "/fhir/legacy", "Patient", "gender", 1)) + truncated := false + complete := false + fdWrite(t, s, fdScope(fdProject, fdGenA, "/fhir/research", "Patient"), + catalog.FieldCatalogDocument{ + Key: "Patient|status", RecordType: catalog.FieldProfileStatusRecordType, + Project: fdProject, DatasetGeneration: fdGenA, AuthResourcePath: "/fhir/research", + ResourceType: "Patient", EnrichmentComplete: &complete, EnrichmentTruncated: truncated, + }) + fdWrite(t, s, fdScope(fdProject, "generation-z", "/fhir/custom", "CustomCarePlan"), + fdDoc(fdProject, "generation-z", "/fhir/custom", "CustomCarePlan", "status", 2)) + fdWrite(t, s, fdScope(fdProject, fdGenA, "", "Patient"), fdDoc(fdProject, fdGenA, "", "Patient", "gender", 1)) + fdWrite(t, s, fdScope("fd-other", fdGenA, "/fhir/foreign", "Patient"), fdDoc("fd-other", fdGenA, "/fhir/foreign", "Patient", "gender", 1)) + + wrapper := &fdRecordingStore{real: s.realRepository(t)} + store, err := NewFieldProfileAdapter(wrapper) + if err != nil { + s.fail("new field profile store", err) + } + + paths, err := store.DiscoverExistingAuthResourcePaths(s.ctx, catalog.AuthResourcePathOptions{ + Project: fdProject, AllGenerations: true, + }) + if err != nil { + s.fail("all-generations listing", err) + } + want := []string{"/fhir/archive", fdPath1, "/fhir/custom", fdPath2, "/fhir/legacy", "/fhir/research"} + if !reflect.DeepEqual(paths, want) { + t.Fatalf("all-generations union = %v, want sorted distinct %v", paths, want) + } + if wrapper.distinctCalls != 1 || wrapper.lastDistinctScope != fdProject { + t.Fatalf("union issued %d distinct reads (project %q); want exactly 1 for %q", + wrapper.distinctCalls, wrapper.lastDistinctScope, fdProject) + } + if wrapper.filteredCalls != 0 { + t.Fatalf("union issued %d filtered reads; the union is one distinct read, not per-generation composition", + wrapper.filteredCalls) + } + + ws, err := store.DiscoverExistingAuthResourcePaths(s.ctx, catalog.AuthResourcePathOptions{ + Project: fdProject, DatasetGeneration: " ", AllGenerations: true, + }) + if err != nil { + s.fail("whitespace-generation all-generations listing", err) + } + if !reflect.DeepEqual(ws, want) { + t.Fatalf("whitespace-generation union = %v, want %v", ws, want) + } + + scoped, err := store.DiscoverExistingAuthResourcePaths(s.ctx, catalog.AuthResourcePathOptions{ + Project: fdProject, DatasetGeneration: "generation-z", + }) + if err != nil { + s.fail("scoped generation-z listing", err) + } + if wantZ := []string{"/fhir/archive", fdPath1, "/fhir/custom"}; !reflect.DeepEqual(scoped, wantZ) { + t.Fatalf("scoped generation-z = %v, want %v", scoped, wantZ) + } +} + +func TestDiscoverExistingAuthResourcePathsAllGenerationsAdapterOwnsByteOrder(t *testing.T) { + s := newFieldProfileService(t) + for _, path := range []string{"/fhir/Beta", "/fhir/_x", "/fhir/alpha", "/fhir/Ω"} { + fdWrite(t, s, fdScope(fdProject, fdGenA, path, "Patient"), fdDoc(fdProject, fdGenA, path, "Patient", "gender", 1)) + } + wrapper := &fdRecordingStore{real: s.realRepository(t), reverseDistinctOrder: true} + store, err := NewFieldProfileAdapter(wrapper) + if err != nil { + s.fail("new field profile store", err) + } + paths, err := store.DiscoverExistingAuthResourcePaths(s.ctx, catalog.AuthResourcePathOptions{ + Project: fdProject, AllGenerations: true, + }) + if err != nil { + s.fail("all-generations listing", err) + } + if want := []string{"/fhir/Beta", "/fhir/_x", "/fhir/alpha", "/fhir/Ω"}; !reflect.DeepEqual(paths, want) { + t.Fatalf("adapter output = %v, want ascending byte order %v", paths, want) + } + if wrapper.distinctCalls != 1 { + t.Fatalf("boundary reads = %d, want exactly 1", wrapper.distinctCalls) + } +} + +func TestDiscoverExistingAuthResourcePathsAllGenerationsUnrecordedProjectIsHonestEmpty(t *testing.T) { + s := newFieldProfileService(t) + fdWrite(t, s, fdScope("fd-other", fdGenA, "/fhir/foreign", "Patient"), fdDoc("fd-other", fdGenA, "/fhir/foreign", "Patient", "gender", 1)) + + wrapper := &fdRecordingStore{real: s.realRepository(t)} + store, err := NewFieldProfileAdapter(wrapper) + if err != nil { + s.fail("new field profile store", err) + } + + paths, err := store.DiscoverExistingAuthResourcePaths(s.ctx, catalog.AuthResourcePathOptions{ + Project: "fd-unrecorded", AllGenerations: true, + }) + if err != nil { + t.Fatalf("unrecorded project must list an honest empty, got error: %v", err) + } + if paths == nil || len(paths) != 0 { + t.Fatalf("unrecorded project union = %#v, want a non-nil empty slice", paths) + } + if wrapper.distinctCalls != 1 || wrapper.lastDistinctScope != "fd-unrecorded" { + t.Fatalf("honest empty came from %d distinct reads (project %q); it must be a real zero-row read", + wrapper.distinctCalls, wrapper.lastDistinctScope) + } +} + +func TestDiscoverExistingAuthResourcePathsAllGenerationsStoreFailureIsWrappedNotEmpty(t *testing.T) { + s := newFieldProfileService(t) + boom := errors.New("injected distinct listing failure") + wrapper := &fdRecordingStore{real: s.realRepository(t), injectDistinctErr: boom} + store, err := NewFieldProfileAdapter(wrapper) + if err != nil { + s.fail("new field profile store", err) + } + paths, err := store.DiscoverExistingAuthResourcePaths(s.ctx, catalog.AuthResourcePathOptions{ + Project: fdProject, AllGenerations: true, + }) + if err == nil { + t.Fatal("store failure was swallowed into a successful listing") + } + if paths != nil { + t.Fatalf("failed listing still returned paths: %#v", paths) + } + if !errors.Is(err, boom) { + t.Fatalf("store failure lost its identity in wrapping: %v", err) + } +} + +func TestDiscoverFieldsLegacyNullGenerationIsolation(t *testing.T) { + s := newFieldProfileService(t) + fdWrite(t, s, fdScope(fdProject, "", fdPath1, "Patient"), fdDoc(fdProject, "", fdPath1, "Patient", "gender", 1)) + fdWrite(t, s, fdScope(fdProject, fdGenA, fdPath1, "Patient"), fdDoc(fdProject, fdGenA, fdPath1, "Patient", "gender", 1)) + + store := s.adapter(t) + legacy, err := store.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: fdProject, AuthResourcePathsUnrestricted: fdUnrestricted(), + }) + if err != nil { + s.fail("legacy discovery", err) + } + if len(legacy) != 1 || legacy[0].DatasetGeneration != "" { + t.Fatalf("legacy read crossed into a generation: %#v", legacy) + } + padded, err := store.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: fdProject, DatasetGeneration: " " + fdGenA + " ", AuthResourcePathsUnrestricted: fdUnrestricted(), + }) + if err != nil { + s.fail("padded discovery", err) + } + if len(padded) != 1 || padded[0].DatasetGeneration != fdGenA { + t.Fatalf("padded generation selection = %#v, want exactly the generation-a row", padded) + } + absent, err := store.DiscoverFields(s.ctx, catalog.PopulatedFieldOptions{ + Project: fdProject, DatasetGeneration: "generation-b", AuthResourcePathsUnrestricted: fdUnrestricted(), + }) + if err != nil { + s.fail("absent-generation discovery", err) + } + if len(absent) != 0 { + t.Fatalf("unprofiled generation matched stored rows: %#v", absent) + } +} diff --git a/internal/backends/postgres/internal/catalog/field_profile.go b/internal/backends/postgres/internal/catalog/field_profile.go new file mode 100644 index 000000000..e350237de --- /dev/null +++ b/internal/backends/postgres/internal/catalog/field_profile.go @@ -0,0 +1,267 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + fhirschema "github.com/calypr/loom/internal/fhir/schema" +) + +var ErrFieldProfileWrite = errors.New("postgres catalog: field profile write failed") + +var ErrInvalidFieldProfile = errors.New("postgres catalog: invalid field profile") + +var ErrFieldProfileUnavailable = errors.New( + "field profile is unavailable; profile this scope against the PostgreSQL store before reading field enrichment") + +type FieldProfileUnavailableError struct { + Project string + DatasetGeneration string + Reason string +} + +func (e *FieldProfileUnavailableError) Error() string { + return fmt.Sprintf("%v: project %q generation %q: %s", + ErrFieldProfileUnavailable, e.Project, e.DatasetGeneration, e.Reason) +} + +func (e *FieldProfileUnavailableError) Unwrap() error { + return ErrFieldProfileUnavailable +} + +type FieldProfileWriteError struct { + Scope storepostgres.FieldProfileScope + Err error +} + +func (e *FieldProfileWriteError) Error() string { + return fmt.Sprintf("postgres catalog: field profile write failed for %s/%s (auth path %q, type %s): %v; "+ + "the batch was atomic, nothing it named was replaced", + e.Scope.Project, e.Scope.DatasetGeneration, e.Scope.AuthResourcePath, e.Scope.ResourceType, e.Err) +} + +func (e *FieldProfileWriteError) Unwrap() []error { + return []error{ErrFieldProfileWrite, e.Err} +} + +func (s *FieldProfileAdapter) WriteFieldProfile(ctx context.Context, scope storepostgres.FieldProfileScope, docs []catalog.FieldCatalogDocument, batchSize int) error { + if err := requireLeg("field-profile", s.profiles != nil, "WriteFieldProfile"); err != nil { + return err + } + scope.DatasetGeneration = catalog.NormalizeDatasetGeneration(scope.DatasetGeneration) + if err := validateFieldProfileBatchScope(scope, docs); err != nil { + return err + } + ordered, coverage := prepareFieldProfileBatch(scope, docs) + if err := s.profiles.UpsertFieldProfileBatch(ctx, scope, ordered, coverage, batchSize); err != nil { + return &FieldProfileWriteError{Scope: scope, Err: err} + } + return nil +} + +func (s *FieldProfileAdapter) ReadFieldProfile(ctx context.Context, scope storepostgres.FieldProfileScope) ([]catalog.FieldCatalogDocument, error) { + if err := requireLeg("field-profile", s.profiles != nil, "ReadFieldProfile"); err != nil { + return nil, err + } + scope.DatasetGeneration = catalog.NormalizeDatasetGeneration(scope.DatasetGeneration) + if strings.TrimSpace(scope.Project) == "" || strings.TrimSpace(scope.ResourceType) == "" { + return nil, fmt.Errorf("%w: project and resource type are required", ErrInvalidFieldProfile) + } + docs, err := s.profiles.ReadFieldProfileRows(ctx, scope) + if err != nil { + return nil, fmt.Errorf("postgres catalog: read field profile %s/%s (auth path %q, type %s): %w", + scope.Project, scope.DatasetGeneration, scope.AuthResourcePath, scope.ResourceType, err) + } + return docs, nil +} + +func (s *FieldProfileAdapter) DiscoverFieldEnrichment(ctx context.Context, opts catalog.FieldEnrichmentOptions) (catalog.FieldEnrichmentResult, error) { + resourceTypes, diagnostics := evidenceResourceTypes(opts.ResourceTypes) + result := catalog.FieldEnrichmentResult{Values: []catalog.FieldEnrichmentObservation{}, Available: true, Complete: true, Diagnostics: diagnostics} + if err := requireLeg("field-profile", s.profiles != nil, "DiscoverFieldEnrichment"); err != nil { + return unavailableFieldEnrichment(result, err.Error()), err + } + sawExplicitCoverage, sawLegacyCoverage := false, false + recordCoverage := func(complete *bool) { + if complete != nil { + sawExplicitCoverage = true + if !*complete { + result.Complete = false + } + } else { + sawLegacyCoverage = true + } + } + datasetGeneration := catalog.NormalizeDatasetGeneration(opts.DatasetGeneration) + authPathsUnrestricted := catalog.EffectiveAuthResourcePathsUnrestricted(opts.AuthResourcePaths, opts.AuthResourcePathsUnrestricted) + if !authPathsUnrestricted && len(opts.AuthResourcePaths) == 0 { + result.Digest, _ = catalog.FieldEnrichmentDigest(result.Values, result.FieldCoverageKnown, result.Complete, result.Truncated) + result.Status = catalog.EvidenceStatusFor(result.Available, result.Complete, result.Truncated, len(result.Values)) + return result, nil + } + var authPaths []string + if !authPathsUnrestricted { + authPaths = append([]string(nil), opts.AuthResourcePaths...) + } + filter := storepostgres.FieldProfileFilter{ + Project: opts.Project, + DatasetGeneration: datasetGeneration, + ResourceTypes: append([]string(nil), resourceTypes...), + AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(authPathsUnrestricted), + AuthResourcePaths: authPaths, + } + + rows, err := s.profiles.ReadFieldProfileRowsFiltered(ctx, filter) + if err != nil { + return unavailableFieldEnrichment(result, err.Error()), fmt.Errorf("postgres catalog: read field enrichment %s/%s: %w", opts.Project, datasetGeneration, err) + } + for _, doc := range rows { + if doc.RecordType == catalog.FieldProfileStatusRecordType { + recordCoverage(doc.EnrichmentComplete) + if doc.EnrichmentTruncated { + result.Truncated = true + } + continue + } + resourceType, ok := fhirschema.ConcreteResourceType(doc.ResourceType) + if !ok { + result.Diagnostics = append(result.Diagnostics, catalog.EvidenceDiagnostic{Code: "INVALID_FIELD_RESOURCE_TYPE", Message: "field enrichment resource type is not concrete/generated", ResourceType: doc.ResourceType}) + continue + } + result.Values = append(result.Values, fieldObservationFromProfileDocument(doc, resourceType)) + if doc.EnrichmentTruncated { + result.Truncated = true + } + recordCoverage(doc.EnrichmentComplete) + } + if len(rows) == 0 { + coverage, err := s.profiles.ReadProfileCoverage(ctx, filter) + if err != nil { + return unavailableFieldEnrichment(result, err.Error()), fmt.Errorf("postgres catalog: read field profile coverage %s/%s: %w", opts.Project, datasetGeneration, err) + } + if len(coverage) == 0 { + return unavailableFieldEnrichment(result, "field profile was never persisted for the requested scope"), + &FieldProfileUnavailableError{Project: opts.Project, DatasetGeneration: datasetGeneration, Reason: "no persisted profile or coverage marker for the requested scope"} + } + for _, marker := range coverage { + recordCoverage(marker.EnrichmentComplete) + if marker.EnrichmentTruncated { + result.Truncated = true + } + } + } + result.FieldCoverageKnown = sawExplicitCoverage && !sawLegacyCoverage + if result.Truncated { + result.Complete = false + } + sort.Slice(result.Values, func(i, j int) bool { return fieldEnrichmentLess(result.Values[i], result.Values[j]) }) + result.Digest, _ = catalog.FieldEnrichmentDigest(result.Values, result.FieldCoverageKnown, result.Complete, result.Truncated) + result.Status = catalog.EvidenceStatusFor(result.Available, result.Complete, result.Truncated, len(result.Values)) + return result, nil +} + +func fieldObservationFromProfileDocument(doc catalog.FieldCatalogDocument, resourceType string) catalog.FieldEnrichmentObservation { + return catalog.FieldEnrichmentObservation{ + Project: doc.Project, DatasetGeneration: doc.DatasetGeneration, AuthResourcePath: doc.AuthResourcePath, + ResourceType: resourceType, Path: doc.Path, Kind: doc.Kind, + DocCount: doc.DocCount, MaxItems: doc.MaxItems, SampleCount: doc.SampleCount, + DistinctValues: append([]string(nil), doc.DistinctValues...), DistinctTruncated: doc.DistinctTruncated, + ExtensionValues: append([]catalog.ExtensionValueObservation(nil), doc.ExtensionValues...), + SemanticObservations: append([]catalog.SemanticObservation(nil), doc.SemanticObservations...), + PivotCandidate: doc.PivotCandidate, PivotKind: doc.PivotKind, + PivotColumns: append([]string(nil), doc.PivotColumns...), + PivotFamily: doc.PivotFamily, PivotColumnSelect: doc.PivotColumnSelect, PivotValueSelect: doc.PivotValueSelect, + PivotItemSource: doc.PivotItemSource, PivotItemResourceType: doc.PivotItemResourceType, + PivotValueSelectors: append([]string(nil), doc.PivotValueSelectors...), + } +} + +func validateFieldProfileBatchScope(scope storepostgres.FieldProfileScope, docs []catalog.FieldCatalogDocument) error { + if strings.TrimSpace(scope.Project) == "" || strings.TrimSpace(scope.ResourceType) == "" { + return fmt.Errorf("%w: scope project and resource type are required", ErrInvalidFieldProfile) + } + seen := make(map[string]struct{}, len(docs)) + var ( + firstSeen bool + firstTruncated bool + firstComplete *bool + ) + for i := range docs { + doc := docs[i] + if doc.Project != scope.Project || doc.DatasetGeneration != scope.DatasetGeneration || + doc.AuthResourcePath != scope.AuthResourcePath || doc.ResourceType != scope.ResourceType { + return fmt.Errorf("%w: document %d scope (%s/%s/%s/%s) diverges from batch scope (%s/%s/%s/%s); one batch persists exactly one profiler scope", + ErrInvalidFieldProfile, i, + doc.Project, doc.DatasetGeneration, doc.AuthResourcePath, doc.ResourceType, + scope.Project, scope.DatasetGeneration, scope.AuthResourcePath, scope.ResourceType) + } + identity := doc.RecordType + "\x00" + doc.Path + if _, dup := seen[identity]; dup { + return fmt.Errorf("%w: document %d repeats identity path %q in one batch", ErrInvalidFieldProfile, i, doc.Path) + } + seen[identity] = struct{}{} + if doc.SampleCount != len(doc.DistinctValues) { + return fmt.Errorf("%w: document %d (path %q) sample_count is %d but retains %d distinct values; the profiler sets sample_count to the retained value count", + ErrInvalidFieldProfile, i, doc.Path, doc.SampleCount, len(doc.DistinctValues)) + } + if !firstSeen { + firstSeen = true + firstTruncated = doc.EnrichmentTruncated + firstComplete = doc.EnrichmentComplete + } else if firstTruncated != doc.EnrichmentTruncated || + (firstComplete == nil) != (doc.EnrichmentComplete == nil) || + (firstComplete != nil && doc.EnrichmentComplete != nil && *firstComplete != *doc.EnrichmentComplete) { + return fmt.Errorf("%w: document %d enrichment flags (complete=%v truncated=%v) disagree with the batch's first document (complete=%v truncated=%v); one profiler run reports one completeness state", + ErrInvalidFieldProfile, i, boolPtr(doc.EnrichmentComplete), doc.EnrichmentTruncated, boolPtr(firstComplete), firstTruncated) + } + } + return nil +} + +func prepareFieldProfileBatch(scope storepostgres.FieldProfileScope, docs []catalog.FieldCatalogDocument) ([]catalog.FieldCatalogDocument, storepostgres.FieldProfileCoverage) { + ordered := append([]catalog.FieldCatalogDocument(nil), docs...) + sort.SliceStable(ordered, func(i, j int) bool { + if ordered[i].RecordType != ordered[j].RecordType { + return ordered[i].RecordType < ordered[j].RecordType + } + return ordered[i].Path < ordered[j].Path + }) + coverage := storepostgres.FieldProfileCoverage{FieldProfileScope: scope} + if len(ordered) == 0 { + complete := true + coverage.EnrichmentComplete = &complete + return ordered, coverage + } + coverage.EnrichmentComplete = ordered[0].EnrichmentComplete + coverage.EnrichmentTruncated = ordered[0].EnrichmentTruncated + return ordered, coverage +} + +func unavailableFieldEnrichment(result catalog.FieldEnrichmentResult, message string) catalog.FieldEnrichmentResult { + result.Values = []catalog.FieldEnrichmentObservation{} + result.Available, result.Complete, result.Status = false, false, catalog.EvidenceUnavailable + result.Diagnostics = append(result.Diagnostics, catalog.EvidenceDiagnostic{Code: "FIELD_ENRICHMENT_UNAVAILABLE", Message: message}) + return result +} + +func fieldEnrichmentLess(a, b catalog.FieldEnrichmentObservation) bool { + for _, pair := range [][2]string{ + {a.Project, b.Project}, {a.DatasetGeneration, b.DatasetGeneration}, + {a.AuthResourcePath, b.AuthResourcePath}, {a.ResourceType, b.ResourceType}, + {a.Path, b.Path}, {a.Kind, b.Kind}, + } { + if pair[0] != pair[1] { + return pair[0] < pair[1] + } + } + return a.DocCount < b.DocCount +} + +func boolPtr(value *bool) bool { + return value != nil && *value +} diff --git a/internal/backends/postgres/internal/catalog/field_profile_test.go b/internal/backends/postgres/internal/catalog/field_profile_test.go new file mode 100644 index 000000000..4ae9ea0c0 --- /dev/null +++ b/internal/backends/postgres/internal/catalog/field_profile_test.go @@ -0,0 +1,1083 @@ +package postgres + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "testing" + "time" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +var pgcat01NamePattern = regexp.MustCompile(`^pgcat01_it_[0-9]+_[0-9a-f]+$`) + +type fieldProfileService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func fieldProfileTestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 integration tests") + } + return raw +} + +func fieldProfileRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) +} + +func fieldProfileScrubber(raw string) func(any) string { + secrets := []string{raw} + if u, err := url.Parse(raw); err == nil && u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + return func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } +} + +func newFieldProfileService(t *testing.T) *fieldProfileService { + t.Helper() + raw := fieldProfileTestPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var dbName, schema string + + t.Cleanup(func() { + var dbCount int + if err := admin.QueryRow(context.WithoutCancel(ctx), + "SELECT COUNT(*) FROM pg_database WHERE datname = $1", dbName).Scan(&dbCount); err != nil { + t.Errorf("audit isolated database %s: %s", dbName, scrub(err)) + } else if dbCount != 0 { + t.Errorf("leftover isolated database %s after cleanup", dbName) + } + var schemaCount int + if err := admin.QueryRow(context.WithoutCancel(ctx), + "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1", schema).Scan(&schemaCount); err != nil { + t.Errorf("audit owned schema %s: %s", schema, scrub(err)) + } else if schemaCount != 0 { + t.Errorf("leftover owned schema %s in the service database after cleanup", schema) + } + }) + + dbName = fmt.Sprintf("pgcat01_it_%d_%s", time.Now().UnixNano(), fieldProfileRandomHex(t)) + if !pgcat01NamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema = fmt.Sprintf("pgcat01_it_%d_%s", time.Now().UnixNano(), fieldProfileRandomHex(t)) + if !pgcat01NamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + service := &fieldProfileService{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} + if _, err := storepostgres.MigrateSchema(ctx, service.openClient(t), schema); err != nil { + fail("MigrateSchema", err) + } + return service +} + +func (s *fieldProfileService) openClient(t *testing.T) *storepostgres.Client { + t.Helper() + client, err := storepostgres.Open(s.ctx, storepostgres.Config{ + DSN: s.dsn, + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *fieldProfileService) realRepository(t *testing.T) *storepostgres.FieldProfileRepository { + t.Helper() + repo, err := storepostgres.NewFieldProfileRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new field profile repository", err) + } + return repo +} + +func (s *fieldProfileService) adapter(t *testing.T) *FieldProfileAdapter { + t.Helper() + store, err := NewFieldProfileAdapter(s.realRepository(t)) + if err != nil { + s.fail("new field profile store", err) + } + return store +} + +func (s *fieldProfileService) exec(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := s.iso.Exec(s.ctx, sql, args...); err != nil { + s.fail("fixture exec", err) + } +} + +func (s *fieldProfileService) profileFQN() string { + return pgx.Identifier{s.schema, "fhir_field_profile"}.Sanitize() +} + +func (s *fieldProfileService) coverageFQN() string { + return pgx.Identifier{s.schema, "fhir_field_profile_coverage"}.Sanitize() +} + +type profileRowWitness struct { + Path string + RecordType string + Kind string + ProfileKey string + DocCount int64 + MaxItems int64 + SampleCount int64 + DistinctTruncated bool + EnrichmentTruncated bool + EnrichmentComplete *bool + DistinctLength int + ExtensionLength int + SemanticLength int +} + +func (s *fieldProfileService) witnessRows(t *testing.T, project, generation string, args ...any) []profileRowWitness { + t.Helper() + all := append([]any{project, generation}, args...) + rows, err := s.iso.Query(s.ctx, "SELECT path, record_type, kind, profile_key, doc_count, max_items, sample_count,"+ + " distinct_truncated, enrichment_truncated, enrichment_complete,"+ + " COALESCE(array_length(distinct_values, 1), 0),"+ + " COALESCE(jsonb_array_length(extension_values), 0),"+ + " COALESCE(jsonb_array_length(semantic_observations), 0)"+ + " FROM "+s.profileFQN()+" WHERE project = $1 AND dataset_generation = $2"+argsSQL(args), all...) + if err != nil { + s.fail("witness rows", err) + } + defer rows.Close() + var out []profileRowWitness + for rows.Next() { + var w profileRowWitness + if err := rows.Scan(&w.Path, &w.RecordType, &w.Kind, &w.ProfileKey, &w.DocCount, &w.MaxItems, &w.SampleCount, + &w.DistinctTruncated, &w.EnrichmentTruncated, &w.EnrichmentComplete, + &w.DistinctLength, &w.ExtensionLength, &w.SemanticLength); err != nil { + s.fail("witness rows", err) + } + out = append(out, w) + } + if err := rows.Err(); err != nil { + s.fail("witness rows", err) + } + return out +} + +func argsSQL(args []any) string { + var b strings.Builder + next := 3 + for range args { + switch next { + case 3: + b.WriteString(" AND auth_resource_path = $" + fmt.Sprint(next)) + case 4: + b.WriteString(" AND resource_type = $" + fmt.Sprint(next)) + } + next++ + } + return b.String() +} + +type coverageWitness struct { + AuthResourcePath string + ResourceType string + EnrichmentIsNull bool + EnrichmentComplete *bool + EnrichmentTruncated bool +} + +func (s *fieldProfileService) witnessCoverage(t *testing.T, project, generation string) []coverageWitness { + t.Helper() + rows, err := s.iso.Query(s.ctx, "SELECT auth_resource_path, resource_type, enrichment_complete IS NULL,"+ + " enrichment_complete, enrichment_truncated FROM "+s.coverageFQN()+ + " WHERE project = $1 AND dataset_generation = $2 ORDER BY auth_resource_path, resource_type", project, generation) + if err != nil { + s.fail("witness coverage", err) + } + defer rows.Close() + var out []coverageWitness + for rows.Next() { + var w coverageWitness + if err := rows.Scan(&w.AuthResourcePath, &w.ResourceType, &w.EnrichmentIsNull, &w.EnrichmentComplete, &w.EnrichmentTruncated); err != nil { + s.fail("witness coverage", err) + } + out = append(out, w) + } + if err := rows.Err(); err != nil { + s.fail("witness coverage", err) + } + return out +} + +func (s *fieldProfileService) witnessRowCount(t *testing.T, project string) int { + t.Helper() + var count int + if err := s.iso.QueryRow(s.ctx, "SELECT COUNT(*) FROM "+s.profileFQN()+" WHERE project = $1", project).Scan(&count); err != nil { + s.fail("witness row count", err) + } + return count +} + +func (s *fieldProfileService) countJSONB(t *testing.T, project, resourceType, path, column, predicate string, args ...any) int { + t.Helper() + sql := "SELECT COUNT(*) FROM " + s.profileFQN() + ", jsonb_array_elements(" + column + ") e" + + " WHERE project = $1 AND resource_type = $2" + all := []any{project, resourceType} + if path != "*" { + sql += " AND path = $3" + all = append(all, path) + } + sql += " AND " + predicate + all = append(all, args...) + var count int + if err := s.iso.QueryRow(s.ctx, sql, all...).Scan(&count); err != nil { + s.fail("witness jsonb elements", err) + } + return count +} + +func fieldProfileScope(project, generation, authPath, resourceType string) storepostgres.FieldProfileScope { + return storepostgres.FieldProfileScope{Project: project, DatasetGeneration: generation, AuthResourcePath: authPath, ResourceType: resourceType} +} + +func patientPayload(gender, birthDate string) map[string]any { + return map[string]any{ + "resourceType": "Patient", + "gender": gender, + "active": true, + "birthDate": birthDate, + "name": []any{ + map[string]any{"family": "Smith", "given": []any{"Ann", "Q"}}, + map[string]any{"family": "Ö’brien‑X"}, + }, + "extension": []any{ + map[string]any{"url": "urn:ext:a", "valueString": "alpha"}, + map[string]any{"url": "urn:ext:b", "valueInteger": 7}, + }, + } +} + +func observationPayload() map[string]any { + return map[string]any{ + "resourceType": "Observation", + "status": "final", + "code": map[string]any{ + "coding": []any{map[string]any{"system": "urn:root", "code": "root-code", "display": "Root label"}}, + "text": "Root text", + }, + "valueQuantity": map[string]any{"value": 42.5, "unit": "kg"}, + "component": []any{ + map[string]any{ + "code": map[string]any{"coding": []any{ + map[string]any{"system": "urn:study:A", "code": "shared"}, + map[string]any{"system": "urn:decoy", "code": "decoy"}, + }}, + "valueQuantity": map[string]any{"value": 111.0, "unit": "cm"}, + }, + map[string]any{ + "code": map[string]any{"coding": []any{map[string]any{"system": "urn:study:B", "code": "shared"}}}, + "valueQuantity": map[string]any{"value": 222.0, "unit": "cm"}, + }, + }, + "extension": []any{map[string]any{ + "url": "parent", + "extension": []any{map[string]any{"url": "leaf", "valueString": "left-only"}}, + }}, + } +} + +func conditionPayload(label string) map[string]any { + return map[string]any{ + "resourceType": "Condition", + "code": map[string]any{"text": label}, + "subject": map[string]any{"reference": "Patient/p1"}, + } +} + +func profilerDocuments(t *testing.T, project, generation, authPath, resourceType string, limits catalog.ProfileLimits, payloads ...map[string]any) []catalog.FieldCatalogDocument { + t.Helper() + profiler := catalog.NewProfilerForGenerationWithLimits(project, generation, authPath, resourceType, nil, limits) + for _, payload := range payloads { + if err := profiler.ObservePayload(payload, map[string]float64{}); err != nil { + t.Fatalf("observe payload: %v", err) + } + } + return profiler.Documents() +} + +func boolPointer(v bool) *bool { return &v } + +func TestFieldProfileProfilerRoundTripPreservesAllPublicFields(t *testing.T) { + s := newFieldProfileService(t) + store := s.adapter(t) + + patientScope := fieldProfileScope("pgcat01-round", "g1", "tenant/a", "Patient") + patientDocs := profilerDocuments(t, "pgcat01-round", "g1", "tenant/a", "Patient", catalog.DefaultProfileLimits(), + patientPayload("male", "2001-02-03"), patientPayload("female", "1999-12-31")) + if len(patientDocs) == 0 { + t.Fatal("profiler emitted no documents") + } + if err := store.WriteFieldProfile(s.ctx, patientScope, patientDocs, 2); err != nil { + t.Fatalf("write patient profile: %v", err) + } + got, err := store.ReadFieldProfile(s.ctx, patientScope) + if err != nil { + t.Fatalf("read patient profile: %v", err) + } + if !reflect.DeepEqual(got, patientDocs) { + t.Fatalf("patient profile round-trip drifted\n got=%#v\nwant=%#v", got, patientDocs) + } + + obsScope := fieldProfileScope("pgcat01-round", "g1", "tenant/a", "Observation") + obsDocs := profilerDocuments(t, "pgcat01-round", "g1", "tenant/a", "Observation", catalog.DefaultProfileLimits(), observationPayload()) + if err := store.WriteFieldProfile(s.ctx, obsScope, obsDocs, 0); err != nil { + t.Fatalf("write observation profile: %v", err) + } + gotObs, err := store.ReadFieldProfile(s.ctx, obsScope) + if err != nil { + t.Fatalf("read observation profile: %v", err) + } + if !reflect.DeepEqual(gotObs, obsDocs) { + t.Fatalf("observation profile round-trip drifted\n got=%#v\nwant=%#v", gotObs, obsDocs) + } + + if n := s.countJSONB(t, "pgcat01-round", "Observation", "component[]", "semantic_observations", + "e->'key'->>'system' = $4 AND e->'examples' @> $5::jsonb", "urn:study:A", `["111"]`); n != 1 { + t.Fatalf("A↔111 correlated elements = %d, want exactly 1", n) + } + if n := s.countJSONB(t, "pgcat01-round", "Observation", "component[]", "semantic_observations", + "e->'key'->>'system' = $4 AND e->'examples' @> $5::jsonb", "urn:study:B", `["222"]`); n != 1 { + t.Fatalf("B↔222 correlated elements = %d, want exactly 1", n) + } + if n := s.countJSONB(t, "pgcat01-round", "Observation", "component[]", "semantic_observations", + "(e->'key'->>'system' = $4 AND e->'examples' @> $5::jsonb) OR (e->'key'->>'system' = $6 AND e->'examples' @> $7::jsonb)", + "urn:study:A", `["222"]`, "urn:study:B", `["111"]`); n != 0 { + t.Fatalf("cross-combined semantic elements = %d, want 0 (key and value must stay correlated)", n) + } + if n := s.countJSONB(t, "pgcat01-round", "Patient", "extension[].url", "extension_values", + "e->>'url' = $4 AND e->>'value_path' = $5 AND e->>'value_type' = $6", "urn:ext:a", "valueString", "string"); n != 1 { + t.Fatalf("urn:ext:a→valueString elements = %d, want exactly 1", n) + } + if n := s.countJSONB(t, "pgcat01-round", "Patient", "extension[].url", "extension_values", + "e->>'url' = $4 AND e->>'value_path' = $5 AND e->>'value_type' = $6", "urn:ext:b", "valueInteger", "integer"); n != 1 { + t.Fatalf("urn:ext:b→valueInteger elements = %d, want exactly 1", n) + } + if n := s.countJSONB(t, "pgcat01-round", "Patient", "extension[].url", "extension_values", + "(e->>'url' = $4 AND e->>'value_path' = $5) OR (e->>'url' = $6 AND e->>'value_path' = $7)", + "urn:ext:a", "valueInteger", "urn:ext:b", "valueString"); n != 0 { + t.Fatalf("cross-combined extension elements = %d, want 0 (URL and value[x] must stay correlated)", n) + } + if n := s.countJSONB(t, "pgcat01-round", "Observation", "*", "semantic_observations", + "e->'extension_url_path' @> $3::jsonb AND e->'examples' @> $4::jsonb", `["parent","leaf"]`, `["left-only"]`); n == 0 { + t.Fatalf("nested extension ancestry observation not persisted as one correlated object") + } + + truncLimits := catalog.ProfileLimits{MaxDistinctValuesPerField: 2, MaxDistinctValueBytes: 64, MaxFields: 3, MaxPivotColumnsPerField: 8, MaxExtensionValuesPerField: 8, MaxShapePlans: 8, MaxRetainedBytes: 4096} + truncScope := fieldProfileScope("pgcat01-round", "g1", "tenant/a", "Condition") + truncDocs := profilerDocuments(t, "pgcat01-round", "g1", "tenant/a", "Condition", truncLimits, + conditionPayload("Angina"), conditionPayload("Sepsis"), conditionPayload("Stroke")) + sawDistinctTruncated, sawEnrichmentTruncated := false, false + for _, doc := range truncDocs { + sawDistinctTruncated = sawDistinctTruncated || doc.DistinctTruncated + sawEnrichmentTruncated = sawEnrichmentTruncated || doc.EnrichmentTruncated + } + if !sawDistinctTruncated || !sawEnrichmentTruncated { + t.Fatalf("truncated fixture lost its truncation signal before persisting: distinct=%v enrichment=%v docs=%#v", + sawDistinctTruncated, sawEnrichmentTruncated, truncDocs) + } + if err := store.WriteFieldProfile(s.ctx, truncScope, truncDocs, 0); err != nil { + t.Fatalf("write truncated profile: %v", err) + } + gotTrunc, err := store.ReadFieldProfile(s.ctx, truncScope) + if err != nil { + t.Fatalf("read truncated profile: %v", err) + } + if !reflect.DeepEqual(gotTrunc, truncDocs) { + t.Fatalf("truncated profile round-trip drifted\n got=%#v\nwant=%#v", gotTrunc, truncDocs) + } + for _, w := range s.witnessRows(t, "pgcat01-round", "g1", "tenant/a", "Condition") { + if !w.EnrichmentTruncated || w.EnrichmentComplete == nil || *w.EnrichmentComplete { + t.Fatalf("witness truncated row %q kept clean flags: %+v", w.Path, w) + } + } + + handScope := fieldProfileScope("pgcat01-round", "g1", "tenant/a", "Practitioner") + handKey := "handcraft-practitioner-active" + handDocs := []catalog.FieldCatalogDocument{{ + Key: handKey, + Project: "pgcat01-round", + DatasetGeneration: "g1", + AuthResourcePath: "tenant/a", + ResourceType: "Practitioner", + Path: "active", + Kind: "scalar", + DocCount: 0, + MaxItems: 0, + SampleCount: 0, + DistinctTruncated: true, + EnrichmentTruncated: false, + PivotCandidate: false, + ExtensionValues: []catalog.ExtensionValueObservation{{ + URL: "urn:hand", SourcePath: "extension[]", ValuePath: "valueBoolean", ValueType: "boolean", + URLPath: []string{"parent-url"}, + }}, + SemanticObservations: []catalog.SemanticObservation{{ + SchemaVersion: 4, + Source: catalog.SemanticObservationSource{Canonical: "urn:hand#active"}, + Key: catalog.SemanticObservationKey{Selector: "active", System: "urn:sys"}, + Value: catalog.SemanticObservationValue{ + Selector: "active", Type: "boolean", + FallbackSelectors: []string{"active.extension"}, + FallbackOnBlankString: true, + }, + ObservedUnits: []string{"unit-1"}, + ObservedUnitsTruncated: true, + Completeness: catalog.SemanticIncomplete, + Status: "SUPPORTED", + Population: 3, + Examples: []string{"true", "false"}, + ExamplesTruncated: true, + RuleHint: catalog.SemanticRuleHintCodedValueV1, + RuleVersion: "9", + }}, + }} + if err := store.WriteFieldProfile(s.ctx, handScope, handDocs, 0); err != nil { + t.Fatalf("write hand-built profile: %v", err) + } + gotHand, err := store.ReadFieldProfile(s.ctx, handScope) + if err != nil { + t.Fatalf("read hand-built profile: %v", err) + } + if !reflect.DeepEqual(gotHand, handDocs) { + t.Fatalf("hand-built profile round-trip drifted\n got=%#v\nwant=%#v", gotHand, handDocs) + } + handWitnesses := s.witnessRows(t, "pgcat01-round", "g1", "tenant/a", "Practitioner") + if len(handWitnesses) != 1 { + t.Fatalf("hand-built witnesses = %v, want one row", handWitnesses) + } + if handWitnesses[0].EnrichmentComplete != nil { + t.Fatalf("missing completeness claim was coerced: %+v", handWitnesses[0]) + } + if !handWitnesses[0].DistinctTruncated || handWitnesses[0].EnrichmentTruncated { + t.Fatalf("false/empty booleans did not round-trip literally: %+v", handWitnesses[0]) + } + if handWitnesses[0].SampleCount != 0 || handWitnesses[0].DistinctLength != 0 { + t.Fatalf("zero counts did not round-trip literally: %+v", handWitnesses[0]) + } + if gotHand[0].SemanticObservations[0].ExamplesTruncated != true || gotHand[0].SemanticObservations[0].ObservedUnitsTruncated != true { + t.Fatalf("bounded-example truncation metadata lost: %#v", gotHand[0].SemanticObservations[0]) + } + + for _, w := range s.witnessRows(t, "pgcat01-round", "g1", "tenant/a", "Patient") { + if w.Path != "gender" { + continue + } + if w.Kind != "scalar" || w.DocCount != 2 || w.SampleCount != 2 || w.DistinctLength != 2 || + w.DistinctTruncated || w.EnrichmentTruncated || w.EnrichmentComplete == nil || !*w.EnrichmentComplete { + t.Fatalf("gender witness = %+v, want exact literal row", w) + } + if !strings.HasPrefix(w.ProfileKey, "gfc_") { + t.Fatalf("generation-qualified profile key %q lost its gfc_ identity", w.ProfileKey) + } + } +} + +func TestFieldProfileUpsertReplacesExactIdentityOnly(t *testing.T) { + s := newFieldProfileService(t) + store := s.adapter(t) + + type scopeSnapshot struct { + scope storepostgres.FieldProfileScope + docs []catalog.FieldCatalogDocument + } + scopes := []storepostgres.FieldProfileScope{ + fieldProfileScope("pgcat01-up", "g1", "a", "Patient"), + fieldProfileScope("pgcat01-up", "g1", "b", "Patient"), + fieldProfileScope("pgcat01-up", "g2", "a", "Patient"), + fieldProfileScope("pgcat01-up", "g1", "a", "Observation"), + fieldProfileScope("pgcat02-neighbor", "g1", "a", "Patient"), + } + var snapshots []scopeSnapshot + for _, scope := range scopes { + docs := profilerDocuments(t, scope.Project, scope.DatasetGeneration, scope.AuthResourcePath, scope.ResourceType, + catalog.DefaultProfileLimits(), patientPayload("male", "2001-02-03")) + if err := store.WriteFieldProfile(s.ctx, scope, docs, 0); err != nil { + t.Fatalf("seed %v: %v", scope, err) + } + got, err := store.ReadFieldProfile(s.ctx, scope) + if err != nil { + t.Fatalf("seed read %v: %v", scope, err) + } + if !reflect.DeepEqual(got, docs) { + t.Fatalf("seed read drifted for %v", scope) + } + snapshots = append(snapshots, scopeSnapshot{scope: scope, docs: got}) + } + totalBefore := 0 + for _, project := range []string{"pgcat01-up", "pgcat02-neighbor"} { + totalBefore += s.witnessRowCount(t, project) + } + + replacement := []catalog.FieldCatalogDocument{{ + Key: "gfc_replaced_gender", + Project: "pgcat01-up", + DatasetGeneration: "g1", + AuthResourcePath: "a", + ResourceType: "Patient", + Path: "gender", + Kind: "codeable_concept", + DocCount: 42, + MaxItems: 7, + SampleCount: 1, + DistinctValues: []string{"zzz-replaced"}, + DistinctTruncated: true, + EnrichmentComplete: boolPointer(false), + EnrichmentTruncated: true, + PivotCandidate: true, + PivotKind: "codeable_concept_display_value", + PivotColumns: []string{"replaced-column"}, + }} + if err := store.WriteFieldProfile(s.ctx, scopes[0], replacement, 0); err != nil { + t.Fatalf("replace identity: %v", err) + } + got, err := store.ReadFieldProfile(s.ctx, scopes[0]) + if err != nil { + t.Fatalf("read replaced scope: %v", err) + } + var gender *catalog.FieldCatalogDocument + for i := range got { + if got[i].Path == "gender" { + gender = &got[i] + } + } + if gender == nil { + t.Fatalf("replaced identity vanished: %#v", got) + } + if gender.Kind != "codeable_concept" || gender.DocCount != 42 || gender.MaxItems != 7 || + gender.SampleCount != 1 || !reflect.DeepEqual(gender.DistinctValues, []string{"zzz-replaced"}) || + !gender.DistinctTruncated || gender.EnrichmentComplete == nil || *gender.EnrichmentComplete || + !gender.EnrichmentTruncated || !gender.PivotCandidate || + !reflect.DeepEqual(gender.PivotColumns, []string{"replaced-column"}) || gender.Key != "gfc_replaced_gender" { + t.Fatalf("replacement did not fully overwrite the identity: %#v", gender) + } + witnesses := s.witnessRows(t, "pgcat01-up", "g1", "a", "Patient") + genderRows := 0 + for _, w := range witnesses { + if w.Path == "gender" { + genderRows++ + } + } + if genderRows != 1 { + t.Fatalf("upsert left %d rows for the replaced identity, want exactly 1", genderRows) + } + + for i := 1; i < len(snapshots); i++ { + again, err := store.ReadFieldProfile(s.ctx, snapshots[i].scope) + if err != nil { + t.Fatalf("re-read %v: %v", snapshots[i].scope, err) + } + if !reflect.DeepEqual(again, snapshots[i].docs) { + t.Fatalf("sibling identity %v was disturbed by the upsert\n got=%#v\nwant=%#v", snapshots[i].scope, again, snapshots[i].docs) + } + } + totalAfter := 0 + for _, project := range []string{"pgcat01-up", "pgcat02-neighbor"} { + totalAfter += s.witnessRowCount(t, project) + } + if totalAfter != totalBefore { + t.Fatalf("row count moved %d → %d; upsert must only replace, never append or drop", totalBefore, totalAfter) + } + + if err := store.WriteFieldProfile(s.ctx, scopes[0], replacement, 0); err != nil { + t.Fatalf("re-apply replacement batch: %v", err) + } + again, err := store.ReadFieldProfile(s.ctx, scopes[0]) + if err != nil { + t.Fatalf("read after re-apply: %v", err) + } + if !reflect.DeepEqual(again, got) { + t.Fatalf("re-applying the identical batch changed the row\n got=%#v\nwant=%#v", again, got) + } +} + +func TestFieldProfileProfileStatesRemainDistinguishable(t *testing.T) { + s := newFieldProfileService(t) + store := s.adapter(t) + + completeScope := fieldProfileScope("pgcat01-complete", "g1", "a", "Observation") + completeDocs := profilerDocuments(t, "pgcat01-complete", "g1", "a", "Observation", catalog.DefaultProfileLimits(), observationPayload()) + if err := store.WriteFieldProfile(s.ctx, completeScope, completeDocs, 0); err != nil { + t.Fatalf("write complete: %v", err) + } + truncatedScope := fieldProfileScope("pgcat01-truncated", "g1", "a", "Condition") + truncDocs := profilerDocuments(t, "pgcat01-truncated", "g1", "a", "Condition", + catalog.ProfileLimits{MaxFields: 1, MaxDistinctValuesPerField: 64, MaxDistinctValueBytes: 64, MaxPivotColumnsPerField: 8, MaxExtensionValuesPerField: 8, MaxShapePlans: 8, MaxRetainedBytes: 4096}, + conditionPayload("Angina")) + if err := store.WriteFieldProfile(s.ctx, truncatedScope, truncDocs, 0); err != nil { + t.Fatalf("write truncated: %v", err) + } + emptyTruncatedScope := fieldProfileScope("pgcat01-empty-truncated", "g1", "a", "Condition") + statusDocs := profilerDocuments(t, "pgcat01-empty-truncated", "g1", "a", "Condition", + catalog.ProfileLimits{MaxFields: 64, MaxDistinctValuesPerField: 64, MaxDistinctValueBytes: 64, MaxPivotColumnsPerField: 8, MaxExtensionValuesPerField: 8, MaxShapePlans: 1, MaxRetainedBytes: 1}, + conditionPayload("Angina")) + if len(statusDocs) != 1 || statusDocs[0].RecordType != catalog.FieldProfileStatusRecordType { + t.Fatalf("expected exactly one profile_status record, got %#v", statusDocs) + } + if err := store.WriteFieldProfile(s.ctx, emptyTruncatedScope, statusDocs, 0); err != nil { + t.Fatalf("write status: %v", err) + } + emptyScope := fieldProfileScope("pgcat01-empty", "g1", "a", "Observation") + if err := store.WriteFieldProfile(s.ctx, emptyScope, nil, 0); err != nil { + t.Fatalf("write known-empty: %v", err) + } + + type expectation struct { + project string + resourceType string + wantStatus catalog.EvidenceStatus + wantComplete bool + wantCoverage bool + wantValues bool + wantUnavailable bool + unavailablePart string + } + cases := []expectation{ + {project: "pgcat01-complete", resourceType: "Observation", wantStatus: catalog.EvidenceAvailable, wantComplete: true, wantCoverage: true, wantValues: true}, + {project: "pgcat01-truncated", resourceType: "Condition", wantStatus: catalog.EvidenceTruncated, wantComplete: false, wantCoverage: true, wantValues: true}, + {project: "pgcat01-empty-truncated", resourceType: "Condition", wantStatus: catalog.EvidenceTruncated, wantComplete: false, wantCoverage: true, wantValues: false}, + {project: "pgcat01-empty", resourceType: "Observation", wantStatus: catalog.EvidenceEmpty, wantComplete: true, wantCoverage: true, wantValues: false}, + {project: "pgcat01-never", resourceType: "Observation", wantStatus: catalog.EvidenceUnavailable, wantComplete: false, wantCoverage: false, wantValues: false, wantUnavailable: true, unavailablePart: "no persisted profile or coverage marker"}, + } + for _, tc := range cases { + result, err := store.DiscoverFieldEnrichment(s.ctx, catalog.FieldEnrichmentOptions{ + Project: tc.project, DatasetGeneration: "g1", ResourceTypes: []string{tc.resourceType}, + AuthResourcePathsUnrestricted: boolPointer(true), + }) + if tc.wantUnavailable { + if err == nil || !errors.Is(err, ErrFieldProfileUnavailable) { + t.Fatalf("%s: error = %v, want errors.Is ErrFieldProfileUnavailable", tc.project, err) + } + var typed *FieldProfileUnavailableError + if !errors.As(err, &typed) { + t.Fatalf("%s: error %v is not a typed *FieldProfileUnavailableError", tc.project, err) + } + if !strings.Contains(typed.Reason, tc.unavailablePart) { + t.Fatalf("%s: unavailable reason %q lacks %q", tc.project, typed.Reason, tc.unavailablePart) + } + } else if err != nil { + t.Fatalf("%s: discover: %v", tc.project, err) + } + if result.Status != tc.wantStatus || result.Complete != tc.wantComplete || + result.FieldCoverageKnown != tc.wantCoverage || (len(result.Values) > 0) != tc.wantValues { + t.Fatalf("%s: result = status %v complete %v coverage %v values %d, want %v/%v/%v/%v", + tc.project, result.Status, result.Complete, result.FieldCoverageKnown, len(result.Values), + tc.wantStatus, tc.wantComplete, tc.wantCoverage, tc.wantValues) + } + if tc.wantStatus == catalog.EvidenceTruncated && !result.Truncated { + t.Fatalf("%s: truncated status without the truncation flag", tc.project) + } + } + + if rows := s.witnessRows(t, "pgcat01-empty", "g1"); len(rows) != 0 { + t.Fatalf("known-empty scope persisted field rows: %v", rows) + } + coverage := s.witnessCoverage(t, "pgcat01-empty", "g1") + if len(coverage) != 1 || coverage[0].EnrichmentIsNull || *coverage[0].EnrichmentComplete != true || coverage[0].EnrichmentTruncated { + t.Fatalf("known-empty coverage marker = %v, want exactly one complete/untruncated marker", coverage) + } + if len(s.witnessCoverage(t, "pgcat01-never", "g1")) != 0 { + t.Fatal("never-written scope has a coverage marker") + } + statusWitness := s.witnessRows(t, "pgcat01-empty-truncated", "g1", "a", "Condition") + if len(statusWitness) != 1 || statusWitness[0].RecordType != catalog.FieldProfileStatusRecordType || + statusWitness[0].Path != "" || statusWitness[0].EnrichmentComplete == nil || *statusWitness[0].EnrichmentComplete || + !statusWitness[0].EnrichmentTruncated { + t.Fatalf("profile_status witness = %v, want one pathless complete=false truncated=true marker", statusWitness) + } + if !strings.HasPrefix(statusWitness[0].ProfileKey, "fpe_") { + t.Fatalf("profile_status key %q lost its fpe_ identity", statusWitness[0].ProfileKey) + } + + restrictedEmpty, err := store.DiscoverFieldEnrichment(s.ctx, catalog.FieldEnrichmentOptions{ + Project: "pgcat01-complete", DatasetGeneration: "g1", ResourceTypes: []string{"Observation"}, + AuthResourcePathsUnrestricted: boolPointer(false), AuthResourcePaths: []string{}, + }) + if err != nil { + t.Fatalf("restricted-empty discover: %v", err) + } + if restrictedEmpty.Status != catalog.EvidenceEmpty || !restrictedEmpty.Available || !restrictedEmpty.Complete { + t.Fatalf("restricted-empty result = %#v", restrictedEmpty) + } + + otherAuth, err := store.DiscoverFieldEnrichment(s.ctx, catalog.FieldEnrichmentOptions{ + Project: "pgcat01-empty", DatasetGeneration: "g1", ResourceTypes: []string{"Observation"}, + AuthResourcePathsUnrestricted: boolPointer(false), AuthResourcePaths: []string{"other-path"}, + }) + if err == nil || !errors.Is(err, ErrFieldProfileUnavailable) { + t.Fatalf("foreign auth path read = %#v err=%v, want typed unavailable", otherAuth, err) + } + if otherAuth.Status != catalog.EvidenceUnavailable || otherAuth.Available || otherAuth.Complete { + t.Fatalf("foreign auth path result = %#v", otherAuth) + } +} + +func TestFieldProfileDiscoveryScopesAuthPathsAndDigests(t *testing.T) { + s := newFieldProfileService(t) + store := s.adapter(t) + + for _, scope := range []storepostgres.FieldProfileScope{ + fieldProfileScope("pgcat01-disc", "g1", "a", "Patient"), + fieldProfileScope("pgcat01-disc", "g1", "b", "Patient"), + fieldProfileScope("pgcat01-disc", "g1", "a", "Observation"), + } { + var payloads []map[string]any + if scope.ResourceType == "Patient" { + payloads = []map[string]any{patientPayload("male", "2001-02-03")} + if scope.AuthResourcePath == "b" { + payloads = []map[string]any{patientPayload("female", "1988-08-08")} + } + } else { + payloads = []map[string]any{observationPayload()} + } + docs := profilerDocuments(t, scope.Project, scope.DatasetGeneration, scope.AuthResourcePath, scope.ResourceType, catalog.DefaultProfileLimits(), payloads...) + if err := store.WriteFieldProfile(s.ctx, scope, docs, 0); err != nil { + t.Fatalf("seed %v: %v", scope, err) + } + } + + discover := func(restricted bool, paths []string, types ...string) (catalog.FieldEnrichmentResult, error) { + return store.DiscoverFieldEnrichment(s.ctx, catalog.FieldEnrichmentOptions{ + Project: "pgcat01-disc", DatasetGeneration: "g1", ResourceTypes: types, + AuthResourcePathsUnrestricted: boolPointer(!restricted), AuthResourcePaths: paths, + }) + } + + aOnly, err := discover(true, []string{"a"}, "Patient") + if err != nil || aOnly.Status != catalog.EvidenceAvailable { + t.Fatalf("restricted [a] = %#v err=%v", aOnly, err) + } + for _, value := range aOnly.Values { + if value.AuthResourcePath != "a" || value.ResourceType != "Patient" { + t.Fatalf("restricted [a] leaked scope: %+v", value) + } + } + bOnly, err := discover(true, []string{"b"}, "Patient") + if err != nil { + t.Fatalf("restricted [b]: %v", err) + } + if aOnly.Digest == bOnly.Digest || aOnly.Digest == "" || bOnly.Digest == "" { + t.Fatalf("different authorization scopes share a digest: %q vs %q", aOnly.Digest, bOnly.Digest) + } + both, err := discover(true, []string{"a", "b"}, "Patient") + if err != nil { + t.Fatalf("restricted [a b]: %v", err) + } + if len(both.Values) != len(aOnly.Values)+len(bOnly.Values) { + t.Fatalf("restricted [a b] values %d != %d + %d", len(both.Values), len(aOnly.Values), len(bOnly.Values)) + } + unrestricted, err := discover(false, nil, "Observation", "Patient") + if err != nil || unrestricted.Status != catalog.EvidenceAvailable { + t.Fatalf("unrestricted = %#v err=%v", unrestricted, err) + } + if len(unrestricted.Values) <= len(both.Values) { + t.Fatalf("unrestricted read did not widen beyond the restricted union: %d vs %d", len(unrestricted.Values), len(both.Values)) + } + for i := 1; i < len(unrestricted.Values); i++ { + if !fieldEnrichmentLess(unrestricted.Values[i-1], unrestricted.Values[i]) { + t.Fatalf("values not ordered at %d: %+v then %+v", i, unrestricted.Values[i-1], unrestricted.Values[i]) + } + } + again, err := store.DiscoverFieldEnrichment(s.ctx, catalog.FieldEnrichmentOptions{ + Project: "pgcat01-disc", DatasetGeneration: "g1", ResourceTypes: []string{"Patient", "Observation"}, + }) + if err != nil { + t.Fatalf("legacy-default read: %v", err) + } + if again.Digest != unrestricted.Digest { + t.Fatalf("digest unstable across equivalent reads: %q vs %q", again.Digest, unrestricted.Digest) + } +} + +func TestFieldProfileWriteRejectionsLeaveNothingBehind(t *testing.T) { + s := newFieldProfileService(t) + store := s.adapter(t) + + baseDoc := func(mutate func(*catalog.FieldCatalogDocument)) catalog.FieldCatalogDocument { + doc := catalog.FieldCatalogDocument{ + Key: "reject-key", Project: "pgcat01-reject", DatasetGeneration: "g1", + AuthResourcePath: "a", ResourceType: "Patient", Path: "gender", Kind: "scalar", + DocCount: 1, SampleCount: 1, DistinctValues: []string{"male"}, + EnrichmentComplete: boolPointer(true), + } + mutate(&doc) + return doc + } + cases := []struct { + name string + docs []catalog.FieldCatalogDocument + sentinel error + }{ + {"scope divergence", []catalog.FieldCatalogDocument{baseDoc(func(d *catalog.FieldCatalogDocument) { d.AuthResourcePath = "other" })}, ErrInvalidFieldProfile}, + {"project divergence", []catalog.FieldCatalogDocument{baseDoc(func(d *catalog.FieldCatalogDocument) { d.Project = "pgcat01-other" })}, ErrInvalidFieldProfile}, + {"generation divergence", []catalog.FieldCatalogDocument{baseDoc(func(d *catalog.FieldCatalogDocument) { d.DatasetGeneration = "g2" })}, ErrInvalidFieldProfile}, + {"duplicate identity", []catalog.FieldCatalogDocument{baseDoc(func(*catalog.FieldCatalogDocument) {}), baseDoc(func(d *catalog.FieldCatalogDocument) { d.DocCount = 2 })}, ErrInvalidFieldProfile}, + {"inconsistent completeness", []catalog.FieldCatalogDocument{ + baseDoc(func(*catalog.FieldCatalogDocument) {}), + baseDoc(func(d *catalog.FieldCatalogDocument) { + d.Path = "active" + d.Key = "reject-key-2" + d.EnrichmentComplete = boolPointer(false) + }), + }, ErrInvalidFieldProfile}, + {"sample count disagreement", []catalog.FieldCatalogDocument{baseDoc(func(d *catalog.FieldCatalogDocument) { d.SampleCount = 5 })}, ErrInvalidFieldProfile}, + {"blank key", []catalog.FieldCatalogDocument{baseDoc(func(d *catalog.FieldCatalogDocument) { d.Key = "" })}, storepostgres.ErrInvalidFieldProfileRow}, + {"field without kind", []catalog.FieldCatalogDocument{baseDoc(func(d *catalog.FieldCatalogDocument) { d.Kind = "" })}, storepostgres.ErrInvalidFieldProfileRow}, + {"status with path", []catalog.FieldCatalogDocument{baseDoc(func(d *catalog.FieldCatalogDocument) { d.RecordType = catalog.FieldProfileStatusRecordType })}, storepostgres.ErrInvalidFieldProfileRow}, + {"unknown record type", []catalog.FieldCatalogDocument{baseDoc(func(d *catalog.FieldCatalogDocument) { d.RecordType = "wat" })}, storepostgres.ErrInvalidFieldProfileRow}, + {"negative doc count", []catalog.FieldCatalogDocument{baseDoc(func(d *catalog.FieldCatalogDocument) { d.DocCount = -1 })}, storepostgres.ErrInvalidFieldProfileRow}, + } + for _, tc := range cases { + err := store.WriteFieldProfile(s.ctx, fieldProfileScope("pgcat01-reject", "g1", "a", "Patient"), tc.docs, 0) + if err == nil || !errors.Is(err, tc.sentinel) { + t.Fatalf("%s: error = %v, want errors.Is %v", tc.name, err, tc.sentinel) + } + if s.witnessRowCount(t, "pgcat01-reject") != 0 { + t.Fatalf("%s: rejected batch left rows behind", tc.name) + } + if len(s.witnessCoverage(t, "pgcat01-reject", "g1")) != 0 { + t.Fatalf("%s: rejected batch left a coverage marker behind", tc.name) + } + } + mixed := []catalog.FieldCatalogDocument{ + baseDoc(func(*catalog.FieldCatalogDocument) {}), + baseDoc(func(d *catalog.FieldCatalogDocument) { d.Path = "active"; d.Key = "reject-key-2"; d.SampleCount = 9 }), + } + if err := store.WriteFieldProfile(s.ctx, fieldProfileScope("pgcat01-reject", "g1", "a", "Patient"), mixed, 0); !errors.Is(err, ErrInvalidFieldProfile) { + t.Fatalf("mixed batch error = %v, want ErrInvalidFieldProfile", err) + } + if s.witnessRowCount(t, "pgcat01-reject") != 0 { + t.Fatal("mixed batch persisted the valid half") + } + + validDocs := []catalog.FieldCatalogDocument{baseDoc(func(d *catalog.FieldCatalogDocument) { d.Key = "keep-me-key" })} + if err := store.WriteFieldProfile(s.ctx, fieldProfileScope("pgcat01-reject", "g1", "a", "Patient"), validDocs, 0); err != nil { + t.Fatalf("valid write: %v", err) + } + badBatch := []catalog.FieldCatalogDocument{ + baseDoc(func(d *catalog.FieldCatalogDocument) { d.Key = "keep-me-key"; d.DocCount = 99 }), + baseDoc(func(d *catalog.FieldCatalogDocument) { d.Path = "active"; d.Key = "doomed-key"; d.Kind = "" }), + } + if err := store.WriteFieldProfile(s.ctx, fieldProfileScope("pgcat01-reject", "g1", "a", "Patient"), badBatch, 0); !errors.Is(err, storepostgres.ErrInvalidFieldProfileRow) { + t.Fatalf("atomic rejection error = %v, want ErrInvalidFieldProfileRow", err) + } + after, err := store.ReadFieldProfile(s.ctx, fieldProfileScope("pgcat01-reject", "g1", "a", "Patient")) + if err != nil { + t.Fatalf("read after atomic rejection: %v", err) + } + if len(after) != 1 || after[0].DocCount != 1 || after[0].Key != "keep-me-key" { + t.Fatalf("atomic rejection disturbed the stored state: %#v", after) + } + + mustCheckViolation := func(step, sql string) { + t.Helper() + _, err := s.iso.Exec(s.ctx, sql) + var pgErr *pgconn.PgError + if err == nil || !errors.As(err, &pgErr) || pgErr.Code != "23514" { + t.Fatalf("%s: error = %v, want CHECK violation 23514", step, err) + } + } + mustCheckViolation("status row with a path", "INSERT INTO "+s.profileFQN()+ + " (project, dataset_generation, auth_resource_path, resource_type, path, profile_key, record_type, kind,"+ + " doc_count, max_items, sample_count, distinct_truncated, enrichment_truncated, pivot_candidate, pivot_kind,"+ + " pivot_family, pivot_column_selector, pivot_value_selector, pivot_item_source, pivot_item_resource_type)"+ + " VALUES ('pgcat01-dbcontract','','','Patient','oops','k','profile_status','scalar',0,0,0,false,false,false,'','','','','','')") + mustCheckViolation("negative count", "INSERT INTO "+s.profileFQN()+ + " (project, dataset_generation, auth_resource_path, resource_type, path, profile_key, record_type, kind,"+ + " doc_count, max_items, sample_count, distinct_truncated, enrichment_truncated, pivot_candidate, pivot_kind,"+ + " pivot_family, pivot_column_selector, pivot_value_selector, pivot_item_source, pivot_item_resource_type)"+ + " VALUES ('pgcat01-dbcontract','','','Patient','gender','k','','scalar',-1,0,0,false,false,false,'','','','','','')") + mustCheckViolation("jsonb scalar subrecord", "INSERT INTO "+s.profileFQN()+ + " (project, dataset_generation, auth_resource_path, resource_type, path, profile_key, record_type, kind,"+ + " doc_count, max_items, sample_count, distinct_truncated, enrichment_truncated, pivot_candidate, pivot_kind,"+ + " pivot_family, pivot_column_selector, pivot_value_selector, pivot_item_source, pivot_item_resource_type, extension_values)"+ + " VALUES ('pgcat01-dbcontract','','','Patient','gender','k','','scalar',0,0,0,false,false,false,'','','','','','','42'::jsonb)") + + if _, err := NewFieldProfileAdapter(nil); err == nil { + t.Fatal("NewFieldProfileAdapter(nil) accepted a nil persistence boundary") + } + if _, err := storepostgres.NewFieldProfileRepository(s.openClient(t), "Bad-Name"); !errors.Is(err, storepostgres.ErrInvalidSchemaName) { + t.Fatalf("NewFieldProfileRepository invalid schema = %v, want ErrInvalidSchemaName", err) + } + repo := s.realRepository(t) + if _, err := repo.ReadFieldProfileRowsFiltered(s.ctx, storepostgres.FieldProfileFilter{ + Project: "x", ResourceTypes: []string{"Patient"}, + }); !errors.Is(err, storepostgres.ErrInvalidFieldProfileScope) { + t.Fatalf("nil unrestricted flag = %v, want ErrInvalidFieldProfileScope", err) + } + if _, err := repo.ReadFieldProfileRowsFiltered(s.ctx, storepostgres.FieldProfileFilter{ + Project: "x", ResourceTypes: []string{"Patient"}, + AuthResourcePathsUnrestricted: boolPointer(true), AuthResourcePaths: []string{"a"}, + }); !errors.Is(err, storepostgres.ErrInvalidFieldProfileScope) { + t.Fatalf("unrestricted with paths = %v, want ErrInvalidFieldProfileScope", err) + } +} + +func TestZZFieldProfileZeroLeftovers(t *testing.T) { + raw := fieldProfileTestPostgresURL(t) + scrub := fieldProfileScrubber(raw) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %s", scrub(err)) + } + defer admin.Close() + + var dbLeftovers []string + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'pgcat01\\_it\\_%' ESCAPE '\\' ORDER BY datname") + if err != nil { + t.Fatalf("audit isolated databases: %s", scrub(err)) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("audit isolated databases: %s", scrub(err)) + } + dbLeftovers = append(dbLeftovers, name) + } + rows.Close() + if err := rows.Err(); err != nil { + t.Fatalf("audit isolated databases: %s", scrub(err)) + } + if len(dbLeftovers) != 0 { + t.Errorf("leftover isolated databases owned by this package: %v", dbLeftovers) + } + + var schemaLeftovers []string + rows, err = admin.Query(ctx, "SELECT nspname FROM pg_namespace WHERE nspname LIKE 'pgcat01\\_it\\_%' ESCAPE '\\' ORDER BY nspname") + if err != nil { + t.Fatalf("audit owned schemas: %s", scrub(err)) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("audit owned schemas: %s", scrub(err)) + } + schemaLeftovers = append(schemaLeftovers, name) + } + rows.Close() + if err := rows.Err(); err != nil { + t.Fatalf("audit owned schemas: %s", scrub(err)) + } + if len(schemaLeftovers) != 0 { + t.Errorf("leftover owned schemas owned by this package: %v", schemaLeftovers) + } +} diff --git a/internal/backends/postgres/internal/catalog/field_projection_test.go b/internal/backends/postgres/internal/catalog/field_projection_test.go new file mode 100644 index 000000000..563dfcb25 --- /dev/null +++ b/internal/backends/postgres/internal/catalog/field_projection_test.go @@ -0,0 +1,261 @@ +package postgres + +import ( + "context" + "reflect" + "testing" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" +) + +type projectionStubStore struct { + rows []catalog.FieldCatalogDocument + filteredCalls int + coverageCalls int + upsertCalls int + rowsCalls int + distinctCalls int +} + +func (s *projectionStubStore) UpsertFieldProfileBatch(ctx context.Context, scope storepostgres.FieldProfileScope, docs []catalog.FieldCatalogDocument, coverage storepostgres.FieldProfileCoverage, batchSize int) error { + s.upsertCalls++ + return nil +} + +func (s *projectionStubStore) ReadFieldProfileRows(ctx context.Context, scope storepostgres.FieldProfileScope) ([]catalog.FieldCatalogDocument, error) { + s.rowsCalls++ + return nil, nil +} + +func (s *projectionStubStore) ReadFieldProfileRowsFiltered(ctx context.Context, filter storepostgres.FieldProfileFilter) ([]catalog.FieldCatalogDocument, error) { + s.filteredCalls++ + return s.rows, nil +} + +func (s *projectionStubStore) ReadProfileCoverage(ctx context.Context, filter storepostgres.FieldProfileFilter) ([]storepostgres.FieldProfileCoverage, error) { + s.coverageCalls++ + return nil, nil +} + +func (s *projectionStubStore) ListDistinctAuthResourcePaths(ctx context.Context, project string) ([]string, error) { + s.distinctCalls++ + return nil, nil +} + +func projectionFullDoc() catalog.FieldCatalogDocument { + doc := fdDoc(fdProject, fdGenA, fdPath1, "Patient", "gender", 3, "female", "male") + doc.MaxItems = 1 + doc.DistinctTruncated = true + doc.ExtensionValues = []catalog.ExtensionValueObservation{{ + URL: "http://ext/gender-identity", SourcePath: "gender.extension", + ValuePath: "valueCoding.code", ValueType: "code", + URLPath: []string{"http://ext/root", "http://ext/gender-identity"}, + }} + doc.SemanticObservations = []catalog.SemanticObservation{{ + SchemaVersion: 1, Status: "observed", Population: 2, + Examples: []string{"female"}, ExtensionURLPath: []string{"http://ext/root"}, + ObservedUnits: []string{"kg"}, + }} + doc.PivotCandidate = true + doc.PivotKind = "codes" + doc.PivotColumns = []string{"code.coding"} + doc.PivotFamily = "observation" + doc.PivotColumnSelect = "code" + doc.PivotValueSelect = "value" + doc.PivotItemSource = "item" + doc.PivotItemResourceType = "Observation" + doc.PivotValueSelectors = []string{"valueQuantity", "valueString"} + doc.EnrichmentTruncated = true + return doc +} + +func projectionExpected() catalog.FieldEnrichmentObservation { + return catalog.FieldEnrichmentObservation{ + Project: fdProject, DatasetGeneration: fdGenA, AuthResourcePath: fdPath1, + ResourceType: "Patient", Path: "gender", Kind: "scalar", + DocCount: 3, MaxItems: 1, SampleCount: 2, + DistinctValues: []string{"female", "male"}, DistinctTruncated: true, + ExtensionValues: []catalog.ExtensionValueObservation{{ + URL: "http://ext/gender-identity", SourcePath: "gender.extension", + ValuePath: "valueCoding.code", ValueType: "code", + URLPath: []string{"http://ext/root", "http://ext/gender-identity"}, + }}, + SemanticObservations: []catalog.SemanticObservation{{ + SchemaVersion: 1, Status: "observed", Population: 2, + Examples: []string{"female"}, ExtensionURLPath: []string{"http://ext/root"}, + ObservedUnits: []string{"kg"}, + }}, + PivotCandidate: true, PivotKind: "codes", PivotColumns: []string{"code.coding"}, + PivotFamily: "observation", PivotColumnSelect: "code", PivotValueSelect: "value", + PivotItemSource: "item", PivotItemResourceType: "Observation", + PivotValueSelectors: []string{"valueQuantity", "valueString"}, + } +} + +func projectionStatusDoc() catalog.FieldCatalogDocument { + complete := true + return catalog.FieldCatalogDocument{ + Key: "Patient|status", RecordType: catalog.FieldProfileStatusRecordType, + Project: fdProject, DatasetGeneration: fdGenA, AuthResourcePath: fdPath1, ResourceType: "Patient", + EnrichmentComplete: &complete, + } +} + +func projectionInvalidDoc() catalog.FieldCatalogDocument { + return fdDoc(fdProject, fdGenA, fdPath1, "Resource", "reference", 1) +} + +func TestFieldProjectionFullFieldEquivalenceBothReadSurfaces(t *testing.T) { + stub := &projectionStubStore{rows: []catalog.FieldCatalogDocument{ + projectionFullDoc(), projectionStatusDoc(), projectionInvalidDoc(), + }} + store, err := NewFieldProfileAdapter(stub) + if err != nil { + t.Fatalf("new field profile adapter: %v", err) + } + expected := projectionExpected() + + fields, err := store.DiscoverFields(context.Background(), catalog.PopulatedFieldOptions{ + Project: fdProject, DatasetGeneration: fdGenA, AuthResourcePathsUnrestricted: fdUnrestricted(), + }) + if err != nil { + t.Fatalf("discover fields: %v", err) + } + if len(fields) != 1 { + t.Fatalf("discovery rows = %d, want only the fully populated data row: %#v", len(fields), fields) + } + if want := catalog.PopulatedField(expected); !reflect.DeepEqual(fields[0], want) { + t.Fatalf("discovery projection lost or altered a field:\n got %+v\nwant %+v", fields[0], want) + } + + enrichment, err := store.DiscoverFieldEnrichment(context.Background(), catalog.FieldEnrichmentOptions{ + Project: fdProject, DatasetGeneration: fdGenA, ResourceTypes: []string{"Patient"}, + AuthResourcePathsUnrestricted: fdUnrestricted(), + }) + if err != nil { + t.Fatalf("discover field enrichment: %v", err) + } + if len(enrichment.Values) != 1 { + t.Fatalf("enrichment rows = %d, want only the fully populated data row: %#v", len(enrichment.Values), enrichment.Values) + } + if !reflect.DeepEqual(enrichment.Values[0], expected) { + t.Fatalf("enrichment projection lost or altered a field:\n got %+v\nwant %+v", enrichment.Values[0], expected) + } + if !enrichment.Available || enrichment.Complete || !enrichment.Truncated || !enrichment.FieldCoverageKnown { + t.Fatalf("enrichment posture = available=%v complete=%v truncated=%v coverageKnown=%v, want true/false/true/true", + enrichment.Available, enrichment.Complete, enrichment.Truncated, enrichment.FieldCoverageKnown) + } + if want := catalog.EvidenceStatusFor(true, false, true, 1); enrichment.Status != want { + t.Fatalf("enrichment status = %v, want %v", enrichment.Status, want) + } + wantDigest, digestErr := catalog.FieldEnrichmentDigest([]catalog.FieldEnrichmentObservation{expected}, true, false, true) + if digestErr != nil { + t.Fatalf("expected digest: %v", digestErr) + } + if enrichment.Digest != wantDigest { + t.Fatalf("enrichment digest = %q, want %q", enrichment.Digest, wantDigest) + } + if len(enrichment.Diagnostics) != 1 || + enrichment.Diagnostics[0].Code != "INVALID_FIELD_RESOURCE_TYPE" || + enrichment.Diagnostics[0].ResourceType != "Resource" { + t.Fatalf("invalid-typed row must be refused with exactly one typed diagnostic, got %+v", enrichment.Diagnostics) + } + + if stub.filteredCalls != 2 || stub.coverageCalls != 0 || stub.upsertCalls != 0 || + stub.rowsCalls != 0 || stub.distinctCalls != 0 { + t.Fatalf("projection boundary calls: filtered=%d coverage=%d upsert=%d rows=%d distinct=%d, want 2/0/0/0/0", + stub.filteredCalls, stub.coverageCalls, stub.upsertCalls, stub.rowsCalls, stub.distinctCalls) + } +} + +func poisonProjectionObservation(obs *catalog.FieldEnrichmentObservation) { + obs.DistinctValues[0] = "poisoned" + obs.DistinctValues = append(obs.DistinctValues, "appended") + obs.PivotColumns[0] = "poisoned" + obs.PivotColumns = append(obs.PivotColumns, "appended") + obs.PivotValueSelectors[0] = "poisoned" + obs.PivotValueSelectors = append(obs.PivotValueSelectors, "appended") + obs.ExtensionValues[0] = catalog.ExtensionValueObservation{} + obs.SemanticObservations[0] = catalog.SemanticObservation{} +} + +func TestFieldProjectionCopiedSlicesIsolatePersistedState(t *testing.T) { + stub := &projectionStubStore{rows: []catalog.FieldCatalogDocument{projectionFullDoc(), projectionStatusDoc()}} + store, err := NewFieldProfileAdapter(stub) + if err != nil { + t.Fatalf("new field profile adapter: %v", err) + } + expected := projectionExpected() + pristine := []catalog.FieldCatalogDocument{projectionFullDoc(), projectionStatusDoc()} + + fields, err := store.DiscoverFields(context.Background(), catalog.PopulatedFieldOptions{ + Project: fdProject, DatasetGeneration: fdGenA, AuthResourcePathsUnrestricted: fdUnrestricted(), + }) + if err != nil { + t.Fatalf("discover fields: %v", err) + } + if len(fields) != 1 { + t.Fatalf("discovery rows = %d, want 1", len(fields)) + } + discovered := catalog.FieldEnrichmentObservation(fields[0]) + poisonProjectionObservation(&discovered) + if !reflect.DeepEqual(stub.rows, pristine) { + t.Fatalf("mutating a discovery result altered persisted state: %+v", stub.rows) + } + rereadFields, err := store.DiscoverFields(context.Background(), catalog.PopulatedFieldOptions{ + Project: fdProject, DatasetGeneration: fdGenA, AuthResourcePathsUnrestricted: fdUnrestricted(), + }) + if err != nil { + t.Fatalf("re-read discover fields: %v", err) + } + if len(rereadFields) != 1 || !reflect.DeepEqual(rereadFields[0], catalog.PopulatedField(expected)) { + t.Fatalf("re-read discovery result diverged after poisoning the first result: %#v", rereadFields) + } + + enrichment, err := store.DiscoverFieldEnrichment(context.Background(), catalog.FieldEnrichmentOptions{ + Project: fdProject, DatasetGeneration: fdGenA, ResourceTypes: []string{"Patient"}, + AuthResourcePathsUnrestricted: fdUnrestricted(), + }) + if err != nil { + t.Fatalf("discover field enrichment: %v", err) + } + if len(enrichment.Values) != 1 { + t.Fatalf("enrichment rows = %d, want 1", len(enrichment.Values)) + } + poisonProjectionObservation(&enrichment.Values[0]) + if !reflect.DeepEqual(stub.rows, pristine) { + t.Fatalf("mutating an enrichment result altered persisted state: %+v", stub.rows) + } + reread, err := store.DiscoverFieldEnrichment(context.Background(), catalog.FieldEnrichmentOptions{ + Project: fdProject, DatasetGeneration: fdGenA, ResourceTypes: []string{"Patient"}, + AuthResourcePathsUnrestricted: fdUnrestricted(), + }) + if err != nil { + t.Fatalf("re-read discover field enrichment: %v", err) + } + if len(reread.Values) != 1 || !reflect.DeepEqual(reread.Values[0], expected) { + t.Fatalf("re-read enrichment result diverged after poisoning the first result: %#v", reread.Values) + } + + if stub.coverageCalls != 0 || stub.upsertCalls != 0 || stub.rowsCalls != 0 || stub.distinctCalls != 0 { + t.Fatalf("projection boundary calls: coverage=%d upsert=%d rows=%d distinct=%d, want 0/0/0/0", + stub.coverageCalls, stub.upsertCalls, stub.rowsCalls, stub.distinctCalls) + } +} + +func TestFieldProjectionDomainStructsRemainConvertible(t *testing.T) { + enrichment := reflect.TypeOf(catalog.FieldEnrichmentObservation{}) + populated := reflect.TypeOf(catalog.PopulatedField{}) + if enrichment.NumField() != populated.NumField() { + t.Fatalf("field counts diverged: FieldEnrichmentObservation=%d PopulatedField=%d", + enrichment.NumField(), populated.NumField()) + } + for i := range enrichment.NumField() { + left, right := enrichment.Field(i), populated.Field(i) + if left.Name != right.Name || left.Type != right.Type || left.Tag != right.Tag { + t.Fatalf("field %d diverged: %s %s %q vs %s %s %q", + i, left.Name, left.Type, left.Tag, right.Name, right.Type, right.Tag) + } + } +} diff --git a/internal/backends/postgres/internal/catalog/relationships.go b/internal/backends/postgres/internal/catalog/relationships.go new file mode 100644 index 000000000..ba1e18daa --- /dev/null +++ b/internal/backends/postgres/internal/catalog/relationships.go @@ -0,0 +1,225 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + fhirschema "github.com/calypr/loom/internal/fhir/schema" + "github.com/jackc/pgx/v5" +) + +const ( + edgeOccurrenceTableName = "fhir_edge_occurrence" + relationshipEvidenceCountColumn = "occurrence_count" +) + +var ErrInvalidRelationshipScope = errors.New("postgres catalog: invalid relationship scope") + +type RelationshipScope struct { + Project string + DatasetGeneration string + AuthResourcePathsUnrestricted *bool + AuthResourcePaths []string +} + +type RelationshipOccurrenceGroup struct { + AuthResourcePath string + StorageSourceResourceType string + StorageTargetResourceType string + Label string + OccurrenceCount int64 +} + +type RelationshipStore interface { + ScanRelationshipOccurrences(ctx context.Context, scope RelationshipScope) ([]RelationshipOccurrenceGroup, error) +} + +var _ RelationshipStore = (*RelationshipEvidenceRepository)(nil) + +type RelationshipEvidenceRepository struct { + src storepostgres.ConnSource + tableFQN string +} + +func NewRelationshipEvidenceRepository(src storepostgres.ConnSource, schema string) (*RelationshipEvidenceRepository, error) { + if src == nil { + return nil, errors.New("postgres: nil relationship evidence connection source") + } + if !storepostgres.ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", storepostgres.ErrInvalidSchemaName, schema) + } + return &RelationshipEvidenceRepository{ + src: src, + tableFQN: pgx.Identifier{schema, edgeOccurrenceTableName}.Sanitize(), + }, nil +} + +type relationshipAggregateShape struct { + SQL string + Args []any +} + +func bindRelationshipAggregate(tableFQN string, scope RelationshipScope) (relationshipAggregateShape, error) { + if strings.TrimSpace(scope.Project) == "" { + return relationshipAggregateShape{}, fmt.Errorf("%w: project is blank", ErrInvalidRelationshipScope) + } + if scope.AuthResourcePathsUnrestricted == nil { + return relationshipAggregateShape{}, fmt.Errorf( + "%w: AuthResourcePathsUnrestricted must be set explicitly; there is no implicit unrestricted mode", + ErrInvalidRelationshipScope) + } + if *scope.AuthResourcePathsUnrestricted { + if len(scope.AuthResourcePaths) > 0 { + return relationshipAggregateShape{}, fmt.Errorf( + "%w: unrestricted mode must not carry auth resource paths (%d given)", + ErrInvalidRelationshipScope, len(scope.AuthResourcePaths)) + } + } + sql := "SELECT auth_resource_path, source_resource_type, label, target_resource_type," + + " COUNT(*) AS " + relationshipEvidenceCountColumn + + " FROM " + tableFQN + + " WHERE project = $1 AND dataset_generation = $2" + args := []any{scope.Project, scope.DatasetGeneration} + if !*scope.AuthResourcePathsUnrestricted { + sql += " AND auth_resource_path = ANY($" + fmt.Sprint(len(args)+1) + "::text[])" + args = append(args, scope.AuthResourcePaths) + } + sql += " GROUP BY auth_resource_path, source_resource_type, label, target_resource_type" + + " ORDER BY auth_resource_path, source_resource_type, label, target_resource_type" + return relationshipAggregateShape{SQL: sql, Args: args}, nil +} + +func (r *RelationshipEvidenceRepository) ScanRelationshipOccurrences( + ctx context.Context, scope RelationshipScope) ([]RelationshipOccurrenceGroup, error) { + bound, err := bindRelationshipAggregate(r.tableFQN, scope) + if err != nil { + return nil, err + } + conn, err := r.src.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres: acquire relationship evidence connection: %w", err) + } + defer conn.Release() + + rows, err := conn.Query(ctx, bound.SQL, bound.Args...) + if err != nil { + return nil, fmt.Errorf("postgres: aggregate relationship occurrences: %w", err) + } + defer rows.Close() + + var out []RelationshipOccurrenceGroup + for rows.Next() { + var group RelationshipOccurrenceGroup + if err := rows.Scan(&group.AuthResourcePath, &group.StorageSourceResourceType, + &group.Label, &group.StorageTargetResourceType, &group.OccurrenceCount); err != nil { + return nil, fmt.Errorf("postgres: aggregate relationship occurrences: %w", err) + } + if group.OccurrenceCount < 1 { + return nil, fmt.Errorf("postgres: aggregate relationship occurrences: group %s/%s/%s at path %q has count %d", + group.StorageSourceResourceType, group.Label, group.StorageTargetResourceType, + group.AuthResourcePath, group.OccurrenceCount) + } + out = append(out, group) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("postgres: aggregate relationship occurrences: %w", err) + } + return out, nil +} + +func (s *RelationshipAdapter) DiscoverRelationshipObservations(ctx context.Context, opts catalog.RelationshipObservationOptions) (catalog.RelationshipObservationResult, error) { + resourceTypes, diagnostics := evidenceResourceTypes(opts.ResourceTypes) + result := catalog.RelationshipObservationResult{Values: []catalog.RelationshipObservation{}, Available: true, Complete: true, Diagnostics: diagnostics} + if err := requireLeg("relationship-evidence", s.relationships != nil, "DiscoverRelationshipObservations"); err != nil { + return unavailableRelationshipEvidence(result, err.Error()), err + } + datasetGeneration := catalog.NormalizeDatasetGeneration(opts.DatasetGeneration) + + unrestricted := catalog.EffectiveAuthResourcePathsUnrestricted(opts.AuthResourcePaths, opts.AuthResourcePathsUnrestricted) + if !unrestricted && len(opts.AuthResourcePaths) == 0 { + result.Digest, _ = catalog.RelationshipObservationDigest(result.Values) + result.Status = catalog.EvidenceStatusFor(result.Available, result.Complete, result.Truncated, len(result.Values)) + return result, nil + } + + scope := RelationshipScope{ + Project: opts.Project, + DatasetGeneration: datasetGeneration, + AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(unrestricted), + } + if !unrestricted { + scope.AuthResourcePaths = append([]string(nil), opts.AuthResourcePaths...) + } + + requestedTypes := len(opts.ResourceTypes) > 0 + var requestedSet map[string]struct{} + if requestedTypes { + requestedSet = make(map[string]struct{}, len(resourceTypes)) + for _, resourceType := range resourceTypes { + requestedSet[resourceType] = struct{}{} + } + } + + groups, err := s.relationships.ScanRelationshipOccurrences(ctx, scope) + if err != nil { + return unavailableRelationshipEvidence(result, err.Error()), + fmt.Errorf("read relationship occurrence evidence: %w", err) + } + + counts := make(map[catalog.RelationshipEvidenceKey]int64) + for _, group := range groups { + storageFrom := group.StorageSourceResourceType + label := group.Label + storageTo := group.StorageTargetResourceType + from, fromOK := fhirschema.ConcreteResourceType(storageFrom) + to, toOK := fhirschema.ConcreteResourceType(storageTo) + if !fromOK || !toOK { + if !requestedTypes { + result.Diagnostics = append(result.Diagnostics, catalog.EvidenceDiagnostic{ + Code: "INVALID_RELATIONSHIP_ENDPOINT", + Message: "relationship endpoint is not a concrete generated FHIR resource", + StorageFromType: storageFrom, Label: label, StorageToType: storageTo, + Count: group.OccurrenceCount, + }) + } + continue + } + if requestedTypes { + if _, ok := requestedSet[from]; !ok { + continue + } + if _, ok := requestedSet[to]; !ok { + continue + } + } + observation, ok, diagnostic := catalog.ProjectRelationshipObservation(opts.Project, datasetGeneration, group.AuthResourcePath, from, label, to, group.OccurrenceCount) + if !ok { + result.Diagnostics = append(result.Diagnostics, diagnostic) + continue + } + counts[catalog.RelationshipEvidenceKeyFor(observation)] += observation.EdgeCount + } + for key, count := range counts { + result.Values = append(result.Values, catalog.RelationshipObservationFromKey(key, count)) + } + sort.Slice(result.Values, func(i, j int) bool { return catalog.RelationshipEvidenceLess(result.Values[i], result.Values[j]) }) + result.Digest, _ = catalog.RelationshipObservationDigest(result.Values) + result.Status = catalog.EvidenceStatusFor(result.Available, result.Complete, result.Truncated, len(result.Values)) + return result, nil +} + +func (s *RelationshipAdapter) ReadRelationshipObservations(ctx context.Context, opts catalog.RelationshipObservationOptions) (catalog.RelationshipObservationResult, error) { + return s.DiscoverRelationshipObservations(ctx, opts) +} + +func unavailableRelationshipEvidence(result catalog.RelationshipObservationResult, message string) catalog.RelationshipObservationResult { + result.Values = []catalog.RelationshipObservation{} + result.Available, result.Complete, result.Status = false, false, catalog.EvidenceUnavailable + result.Diagnostics = append(result.Diagnostics, catalog.EvidenceDiagnostic{Code: "RELATIONSHIP_EVIDENCE_UNAVAILABLE", Message: message}) + return result +} diff --git a/internal/backends/postgres/internal/catalog/relationships_pg_test.go b/internal/backends/postgres/internal/catalog/relationships_pg_test.go new file mode 100644 index 000000000..371a7b595 --- /dev/null +++ b/internal/backends/postgres/internal/catalog/relationships_pg_test.go @@ -0,0 +1,1214 @@ +package postgres + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "sync" + "testing" + "time" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +const relationshipFixturePrefix = "verify_pgcat03_" + +var relationshipFixtureNamePattern = regexp.MustCompile(`^verify_pgcat03_[0-9]+_[0-9a-f]+$`) + +var ( + relationshipOwnedMu sync.Mutex + relationshipOwnedByName = map[string]string{} + relationshipCreatedTotal int + relationshipBaselineDBs = -1 + relationshipBaselineError string +) + +func TestMain(m *testing.M) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw != "" { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + if pool, err := pgxpool.New(ctx, raw); err == nil { + var count int + if err := pool.QueryRow(ctx, + "SELECT COUNT(*) FROM pg_database WHERE datname ~ $1", relationshipFixtureNamePattern.String()).Scan(&count); err != nil { + relationshipBaselineError = err.Error() + } else { + relationshipBaselineDBs = count + } + pool.Close() + } else { + relationshipBaselineError = err.Error() + } + cancel() + } + os.Exit(m.Run()) +} + +func relationshipTestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 integration tests") + } + return raw +} + +func relationshipRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) +} + +type relationshipEvidenceService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func newRelationshipEvidenceService(t *testing.T) *relationshipEvidenceService { + t.Helper() + raw := relationshipTestPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var dbName, schema string + + t.Cleanup(func() { + var dbCount int + if err := admin.QueryRow(context.WithoutCancel(ctx), + "SELECT COUNT(*) FROM pg_database WHERE datname = $1", dbName).Scan(&dbCount); err != nil { + t.Errorf("audit isolated database %s: %s", dbName, scrub(err)) + } else if dbCount != 0 { + t.Errorf("leftover isolated database %s after cleanup", dbName) + } + }) + + dbName = fmt.Sprintf("verify_pgcat03_%d_%s", time.Now().UnixNano(), relationshipRandomHex(t)) + if !relationshipFixtureNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + relationshipOwnedMu.Lock() + relationshipOwnedByName[dbName] = "" + relationshipCreatedTotal++ + relationshipOwnedMu.Unlock() + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + relationshipOwnedMu.Lock() + delete(relationshipOwnedByName, dbName) + relationshipOwnedMu.Unlock() + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema = fmt.Sprintf("verify_pgcat03_%d_%s", time.Now().UnixNano(), relationshipRandomHex(t)) + if !relationshipFixtureNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + relationshipOwnedMu.Lock() + relationshipOwnedByName[dbName] = schema + relationshipOwnedMu.Unlock() + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + service := &relationshipEvidenceService{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} + if _, err := storepostgres.MigrateSchema(ctx, service.openClient(t), schema); err != nil { + fail("MigrateSchema", err) + } + return service +} + +func (s *relationshipEvidenceService) openClient(t *testing.T) *storepostgres.Client { + t.Helper() + client, err := storepostgres.Open(s.ctx, storepostgres.Config{ + DSN: s.dsn, + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *relationshipEvidenceService) openSingleConnPool(t *testing.T, name string) *pgxpool.Pool { + t.Helper() + pool, err := pgxpool.New(s.ctx, s.dsn) + if err != nil { + s.fail("open "+name, err) + } + pool.Close() + cfg, err := pgxpool.ParseConfig(s.dsn) + if err != nil { + s.fail("parse "+name+" config", err) + } + cfg.MaxConns = 1 + pool, err = pgxpool.NewWithConfig(s.ctx, cfg) + if err != nil { + s.fail("open "+name, err) + } + t.Cleanup(pool.Close) + return pool +} + +func backendPID(t *testing.T, pool *pgxpool.Pool) int32 { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + conn, err := pool.Acquire(ctx) + if err != nil { + t.Fatalf("acquire pool for pid: %v", err) + } + defer conn.Release() + var pid int32 + if err := conn.QueryRow(ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + t.Fatalf("read pg_backend_pid: %v", err) + } + return pid +} + +func (s *relationshipEvidenceService) edgeTable() string { + return pgx.Identifier{s.schema, "fhir_edge_occurrence"}.Sanitize() +} + +func (s *relationshipEvidenceService) resourceTable() string { + return pgx.Identifier{s.schema, "fhir_resource"}.Sanitize() +} + +func (s *relationshipEvidenceService) exec(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := s.iso.Exec(s.ctx, sql, args...); err != nil { + s.fail("fixture exec", err) + } +} + +func (s *relationshipEvidenceService) seedResource(t *testing.T, project, generation, resourceType, logicalID, authPath string) { + t.Helper() + s.exec(t, "INSERT INTO "+s.resourceTable()+ + " (project, dataset_generation, resource_type, logical_id, auth_resource_path)"+ + " VALUES ($1, $2, $3, $4, $5)", + project, generation, resourceType, logicalID, authPath) +} + +func (s *relationshipEvidenceService) seedOccurrence(t *testing.T, project, generation, authPath, sourceType, sourceID, label, targetType, targetID, sourcePath string, ordinal int64) { + t.Helper() + s.exec(t, "INSERT INTO "+s.edgeTable()+ + " (project, dataset_generation, auth_resource_path, label,"+ + " source_resource_type, source_logical_id, target_resource_type, target_logical_id,"+ + " source_path, source_ordinal)"+ + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + project, generation, authPath, label, sourceType, sourceID, targetType, targetID, sourcePath, ordinal) +} + +func (s *relationshipEvidenceService) witnessRowCount(t *testing.T, project, generation, authPath, sourceType, label, targetType string) int64 { + t.Helper() + var count int64 + if err := s.iso.QueryRow(s.ctx, "SELECT COUNT(*) FROM "+s.edgeTable()+ + " WHERE project = $1 AND dataset_generation = $2 AND auth_resource_path = $3"+ + " AND source_resource_type = $4 AND label = $5 AND target_resource_type = $6", + project, generation, authPath, sourceType, label, targetType).Scan(&count); err != nil { + s.fail("witness occurrence rows", err) + } + return count +} + +func (s *relationshipEvidenceService) newStore(t *testing.T) *RelationshipAdapter { + t.Helper() + repo, err := NewRelationshipEvidenceRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new relationship evidence repository", err) + } + store, err := NewRelationshipAdapter(repo) + if err != nil { + s.fail("new relationship evidence store", err) + } + return store +} + +func requireValuesByteIdentical(t *testing.T, got []catalog.RelationshipObservation, want []catalog.RelationshipObservation) { + t.Helper() + gotJSON, err := json.Marshal(got) + if err != nil { + t.Fatalf("marshal got: %v", err) + } + wantJSON, err := json.Marshal(want) + if err != nil { + t.Fatalf("marshal want: %v", err) + } + if !bytes.Equal(gotJSON, wantJSON) { + t.Fatalf("row witness mismatch\n got: %s\nwant: %s", gotJSON, wantJSON) + } +} + +func observation(project, generation, auth, storageFrom, label, storageTo string, count int64, builderFrom, builderTo, builderDirection string) catalog.RelationshipObservation { + return catalog.RelationshipObservation{ + Project: project, DatasetGeneration: generation, AuthResourcePath: auth, + FromType: builderFrom, Label: label, ToType: builderTo, EdgeCount: count, + StorageFromType: storageFrom, StorageToType: storageTo, StorageDirection: "OUTBOUND", + BuilderFromType: builderFrom, BuilderToType: builderTo, BuilderDirection: builderDirection, + } +} + +func relationshipOpts(project, generation string, unrestricted *bool, paths []string, types ...string) catalog.RelationshipObservationOptions { + return catalog.RelationshipObservationOptions{ + Project: project, DatasetGeneration: generation, + AuthResourcePathsUnrestricted: unrestricted, + AuthResourcePaths: paths, + ResourceTypes: types, + } +} + +func TestRelationshipObservationsExactOccurrenceCountsOnRealPostgreSQL(t *testing.T) { + s := newRelationshipEvidenceService(t) + store := s.newStore(t) + const project, generation = "pgcat03-p1", "g1" + + s.seedResource(t, project, generation, "Condition", "cond-src1", "clinic/a") + s.seedResource(t, project, generation, "Condition", "cond-src2", "clinic/a") + s.seedResource(t, project, "g2", "Condition", "cond-src1", "clinic/a") + s.seedResource(t, "pgcat03-p2", generation, "Condition", "cond-src1", "clinic/a") + + for ordinal := int64(0); ordinal < 3; ordinal++ { + s.seedOccurrence(t, project, generation, "clinic/a", "Condition", "cond-src1", "subject_Patient", "Patient", "pat-t1", "", ordinal) + } + s.seedOccurrence(t, project, generation, "clinic/a", "Condition", "cond-src2", "subject_Patient", "Patient", "pat-t1", "entry", 0) + s.seedOccurrence(t, "pgcat03-p2", generation, "clinic/a", "Condition", "cond-src1", "subject_Patient", "Patient", "pat-t1", "", 0) + s.seedOccurrence(t, "pgcat03-p2", generation, "clinic/a", "Condition", "cond-src1", "subject_Patient", "Patient", "pat-t1", "", 1) + s.seedOccurrence(t, project, "g2", "clinic/a", "Condition", "cond-src1", "subject_Patient", "Patient", "pat-t1", "", 0) + s.seedOccurrence(t, project, "g2", "clinic/a", "Condition", "cond-src1", "subject_Patient", "Patient", "pat-t1", "", 1) + s.seedOccurrence(t, project, "g2", "clinic/a", "Condition", "cond-src1", "subject_Patient", "Patient", "pat-t1", "", 2) + s.seedResource(t, project, generation, "Condition", "cond-src3", "other") + for ordinal := int64(0); ordinal < 5; ordinal++ { + s.seedOccurrence(t, project, generation, "other", "Condition", "cond-src3", "subject_Patient", "Patient", "pat-t1", "", ordinal) + } + + if got := s.witnessRowCount(t, project, generation, "clinic/a", "Condition", "subject_Patient", "Patient"); got != 4 { + t.Fatalf("fixture witness drifted: clinic/a rows = %d, want 4", got) + } + + result, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil)) + if err != nil { + t.Fatalf("unrestricted read: %v", err) + } + wantInScope := []catalog.RelationshipObservation{ + observation(project, generation, "clinic/a", "Condition", "subject_Patient", "Patient", 4, "Patient", "Condition", "INBOUND"), + } + wantUnrestricted := []catalog.RelationshipObservation{ + observation(project, generation, "clinic/a", "Condition", "subject_Patient", "Patient", 4, "Patient", "Condition", "INBOUND"), + observation(project, generation, "other", "Condition", "subject_Patient", "Patient", 5, "Patient", "Condition", "INBOUND"), + } + requireValuesByteIdentical(t, result.Values, wantUnrestricted) + if result.Status != catalog.EvidenceAvailable || !result.Available || !result.Complete || len(result.Diagnostics) != 0 { + t.Fatalf("unrestricted result = %+v, want available complete with no diagnostics", result) + } + if result.Digest != relationshipExactCountsDigest { + t.Fatalf("exact-counts digest = %q, want golden %q", result.Digest, relationshipExactCountsDigest) + } + + var dedupGroups int + var dedupTotal int64 + if err := s.iso.QueryRow(s.ctx, "SELECT COUNT(*), COALESCE(SUM(1),0) FROM (SELECT DISTINCT source_resource_type, label, target_resource_type FROM "+s.edgeTable()+ + " WHERE project = $1 AND dataset_generation = $2) d", project, generation).Scan(&dedupGroups, &dedupTotal); err != nil { + s.fail("dedup control", err) + } + if dedupGroups != 1 || dedupTotal != 1 { + t.Fatalf("dedup control unexpectedly not collapsed: groups=%d total=%d", dedupGroups, dedupTotal) + } + + var flattened int64 + if err := s.iso.QueryRow(s.ctx, "SELECT COUNT(*) FROM "+s.edgeTable()+ + " WHERE project = $1 AND dataset_generation = $2", project, generation).Scan(&flattened); err != nil { + s.fail("flatten control", err) + } + if flattened != 9 { + t.Fatalf("flatten control = %d, want 9 (4 in-scope + 5 in other)", flattened) + } + + restricted, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(false), []string{"clinic/a"})) + if err != nil { + t.Fatalf("restricted read: %v", err) + } + requireValuesByteIdentical(t, restricted.Values, wantInScope) + + for round := 0; round < 3; round++ { + again, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil)) + if err != nil { + t.Fatalf("repeat read: %v", err) + } + if again.Digest != result.Digest { + t.Fatalf("read %d digest %q != first %q", round, again.Digest, result.Digest) + } + } +} + +func TestRelationshipObservationsAuthModesCompleteAndEmptyOnRealPostgreSQL(t *testing.T) { + s := newRelationshipEvidenceService(t) + store := s.newStore(t) + const project, generation = "pgcat03-auth", "g1" + + s.seedResource(t, project, generation, "Condition", "cond-a", "") + s.seedResource(t, project, generation, "Condition", "cond-b", "clinic/a") + s.seedResource(t, project, generation, "BodyStructure", "bs-c", "clinic/b") + s.seedResource(t, project, generation, "Condition", "cond-c", "clinic/c") + + for ordinal := int64(0); ordinal < 2; ordinal++ { + s.seedOccurrence(t, project, generation, "", "Condition", "cond-a", "subject_Patient", "Patient", "pat-1", "", ordinal) + } + for ordinal := int64(0); ordinal < 3; ordinal++ { + s.seedOccurrence(t, project, generation, "clinic/a", "Condition", "cond-b", "subject_Patient", "Patient", "pat-1", "", ordinal) + } + s.seedOccurrence(t, project, generation, "clinic/b", "BodyStructure", "bs-c", "patient", "Patient", "pat-1", "", 0) + for ordinal := int64(0); ordinal < 4; ordinal++ { + s.seedOccurrence(t, project, generation, "clinic/c", "Condition", "cond-c", "subject_Patient", "Patient", "pat-1", "", ordinal) + } + + all, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil)) + if err != nil { + t.Fatalf("unrestricted: %v", err) + } + wantAll := []catalog.RelationshipObservation{ + observation(project, generation, "", "Condition", "subject_Patient", "Patient", 2, "Patient", "Condition", "INBOUND"), + observation(project, generation, "clinic/a", "Condition", "subject_Patient", "Patient", 3, "Patient", "Condition", "INBOUND"), + observation(project, generation, "clinic/b", "BodyStructure", "patient", "Patient", 1, "Patient", "BodyStructure", "INBOUND"), + observation(project, generation, "clinic/c", "Condition", "subject_Patient", "Patient", 4, "Patient", "Condition", "INBOUND"), + } + requireValuesByteIdentical(t, all.Values, wantAll) + if all.Status != catalog.EvidenceAvailable || !all.Available || !all.Complete { + t.Fatalf("unrestricted status = %+v", all) + } + if all.Digest != relationshipAuthModesDigest { + t.Fatalf("auth-modes digest = %q, want golden %q", all.Digest, relationshipAuthModesDigest) + } + + one, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(false), []string{"clinic/a"})) + if err != nil { + t.Fatalf("one-path: %v", err) + } + requireValuesByteIdentical(t, one.Values, wantAll[1:2]) + + multi, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(false), []string{"clinic/a", "clinic/b"})) + if err != nil { + t.Fatalf("multi-path: %v", err) + } + requireValuesByteIdentical(t, multi.Values, []catalog.RelationshipObservation{wantAll[1], wantAll[2]}) + + empty, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(false), nil)) + if err != nil { + t.Fatalf("restricted-empty: %v", err) + } + if empty.Status != catalog.EvidenceEmpty || !empty.Available || !empty.Complete || len(empty.Values) != 0 { + t.Fatalf("restricted-empty result = %+v, want available complete empty", empty) + } + if total := s.witnessRowCount(t, project, generation, "clinic/c", "Condition", "subject_Patient", "Patient"); total != 4 { + t.Fatalf("fixture witness drifted: clinic/c rows = %d, want 4", total) + } + + repo, err := NewRelationshipEvidenceRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new repository", err) + } + groups, err := repo.ScanRelationshipOccurrences(s.ctx, RelationshipScope{ + Project: project, DatasetGeneration: generation, + AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(false), + }) + if err != nil { + t.Fatalf("restricted-empty repository scan: %v", err) + } + if len(groups) != 0 { + t.Fatalf("restricted-empty scan widened: %+v", groups) + } +} + +func TestRelationshipObservationsInvalidEndpointsDiagnosedRawRowsRemain(t *testing.T) { + s := newRelationshipEvidenceService(t) + store := s.newStore(t) + const project, generation = "pgcat03-invalid", "g1" + + s.seedResource(t, project, generation, "NotAThing", "bad-src", "clinic/a") + s.seedResource(t, project, generation, "Resource", "abstract-src", "clinic/a") + s.seedResource(t, project, generation, "Patient", "pat-src", "clinic/a") + s.seedResource(t, project, generation, "Condition", "cond-good", "clinic/a") + + s.seedOccurrence(t, project, generation, "clinic/a", "NotAThing", "bad-src", "subject_Patient", "Patient", "pat-t", "", 0) + s.seedOccurrence(t, project, generation, "clinic/a", "NotAThing", "bad-src", "subject_Patient", "Patient", "pat-t", "", 1) + s.seedOccurrence(t, project, generation, "clinic/a", "Resource", "abstract-src", "subject_Patient", "Patient", "pat-t", "", 0) + s.seedOccurrence(t, project, generation, "clinic/a", "Patient", "pat-src", "not_a_generated_label", "Practitioner", "prac-t", "", 0) + s.seedOccurrence(t, project, generation, "clinic/a", "Condition", "cond-good", "subject_Patient", "Patient", "pat-t", "", 0) + + result, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil)) + if err != nil { + t.Fatalf("read: %v", err) + } + want := []catalog.RelationshipObservation{ + observation(project, generation, "clinic/a", "Condition", "subject_Patient", "Patient", 1, "Patient", "Condition", "INBOUND"), + } + requireValuesByteIdentical(t, result.Values, want) + if result.Status != catalog.EvidenceAvailable || !result.Available { + t.Fatalf("result with a valid subset must stay available: %+v", result) + } + if len(result.Diagnostics) != 3 { + t.Fatalf("diagnostics = %+v, want three", result.Diagnostics) + } + endpointCodes, traversalCodes := 0, 0 + for _, d := range result.Diagnostics { + switch d.Code { + case "INVALID_RELATIONSHIP_ENDPOINT": + endpointCodes++ + if d.Message != "relationship endpoint is not a concrete generated FHIR resource" { + t.Errorf("endpoint diagnostic message = %q", d.Message) + } + switch d.StorageFromType { + case "NotAThing": + if d.StorageToType != "Patient" || d.Label != "subject_Patient" || d.Count != 2 { + t.Errorf("NotAThing diagnostic = %+v", d) + } + case "Resource": + if d.StorageToType != "Patient" || d.Label != "subject_Patient" || d.Count != 1 { + t.Errorf("Resource diagnostic = %+v", d) + } + default: + t.Errorf("unexpected endpoint diagnostic %q", d.StorageFromType) + } + case "UNSUPPORTED_GENERATED_TRAVERSAL": + traversalCodes++ + if d.Message != "relationship endpoints and label are not a supported generated traversal" || + d.StorageFromType != "Patient" || d.StorageToType != "Practitioner" || + d.Label != "not_a_generated_label" || d.Count != 1 { + t.Errorf("traversal diagnostic = %+v", d) + } + default: + t.Errorf("unexpected diagnostic code %q", d.Code) + } + } + if endpointCodes != 2 || traversalCodes != 1 { + t.Fatalf("diagnostic split endpoints=%d traversals=%d, want 2/1", endpointCodes, traversalCodes) + } + + var rawRows int + var invalidRows int + if err := s.iso.QueryRow(s.ctx, "SELECT COUNT(*) FROM "+s.edgeTable()+ + " WHERE project = $1 AND dataset_generation = $2", project, generation).Scan(&rawRows); err != nil { + s.fail("raw witness", err) + } + if err := s.iso.QueryRow(s.ctx, "SELECT COUNT(*) FROM "+s.edgeTable()+ + " WHERE project = $1 AND dataset_generation = $2 AND source_resource_type IN ('NotAThing','Resource')"+ + " AND label = 'subject_Patient' AND target_resource_type = 'Patient'", project, generation).Scan(&invalidRows); err != nil { + s.fail("raw invalid witness", err) + } + if rawRows != 5 || invalidRows != 3 { + t.Fatalf("raw rows after diagnosed read: total=%d invalid=%d, want 5/3", rawRows, invalidRows) + } + + subset, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil, "condition", "Patient")) + if err != nil { + t.Fatalf("subset read: %v", err) + } + requireValuesByteIdentical(t, subset.Values, want) + for _, d := range subset.Diagnostics { + if d.Code == "INVALID_RELATIONSHIP_ENDPOINT" || d.Code == "UNSUPPORTED_GENERATED_TRAVERSAL" { + t.Fatalf("subset read diagnosed rows outside the requested types: %+v", d) + } + } +} + +func TestRelationshipObservationsFilteredReadKeepsCaseVariantRows(t *testing.T) { + s := newRelationshipEvidenceService(t) + store := s.newStore(t) + const project, generation = "pgcat03-case", "g1" + + s.seedResource(t, project, generation, "condition", "cond-lower", "clinic/a") + s.seedResource(t, project, generation, "Condition", "cond-mixed", "clinic/a") + s.seedOccurrence(t, project, generation, "clinic/a", "condition", "cond-lower", "subject_Patient", "Patient", "pat-x", "", 0) + s.seedOccurrence(t, project, generation, "clinic/a", "condition", "cond-lower", "subject_Patient", "Patient", "pat-x", "", 1) + s.seedOccurrence(t, project, generation, "clinic/a", "Condition", "cond-mixed", "subject_Patient", "PatIent", "pat-x", "", 0) + + want := []catalog.RelationshipObservation{ + observation(project, generation, "clinic/a", "Condition", "subject_Patient", "Patient", 3, "Patient", "Condition", "INBOUND"), + } + + unfiltered, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil)) + if err != nil { + t.Fatalf("unfiltered read: %v", err) + } + requireValuesByteIdentical(t, unfiltered.Values, want) + + for _, requested := range [][]string{{"Condition", "Patient"}, {"condition", "patient"}} { + filtered, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil, requested...)) + if err != nil { + t.Fatalf("filtered read %v: %v", requested, err) + } + requireValuesByteIdentical(t, filtered.Values, want) + for _, d := range filtered.Diagnostics { + if d.Code != "INVALID_RESOURCE_TYPE" { + t.Fatalf("filtered read %v diagnosed rows it should match: %+v", requested, d) + } + } + } + + missed, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil, "BodyStructure", "Patient")) + if err != nil { + t.Fatalf("non-matching filter: %v", err) + } + if len(missed.Values) != 0 || missed.Status != catalog.EvidenceEmpty { + t.Fatalf("non-matching filter returned %+v status %s, want empty", missed.Values, missed.Status) + } + + var lowerSources, mixedTargets int + if err := s.iso.QueryRow(s.ctx, "SELECT COUNT(*) FROM "+s.edgeTable()+ + " WHERE project = $1 AND dataset_generation = $2 AND source_resource_type = 'condition'", project, generation).Scan(&lowerSources); err != nil { + s.fail("raw lower witness", err) + } + if err := s.iso.QueryRow(s.ctx, "SELECT COUNT(*) FROM "+s.edgeTable()+ + " WHERE project = $1 AND dataset_generation = $2 AND target_resource_type = 'PatIent'", project, generation).Scan(&mixedTargets); err != nil { + s.fail("raw mixed witness", err) + } + if lowerSources != 2 || mixedTargets != 1 { + t.Fatalf("raw spellings rewritten: lower=%d mixed=%d, want 2/1", lowerSources, mixedTargets) + } +} + +func TestRelationshipObservationsFilteredReadByteIdenticalToUnfilteredOnWhitespaceVariants(t *testing.T) { + s := newRelationshipEvidenceService(t) + store := s.newStore(t) + const project, generation = "pgcat03-ws", "g1" + + sources := []struct{ resourceType, logicalID string }{ + {"Condition\t", "ws-s0"}, + {"\rCondition", "ws-s1"}, + {"Condition", "ws-s2"}, + {" \tCondition\u00a0", "ws-s3"}, + {"condition", "ws-s4"}, + {"Bodyſtructure", "ws-s5"}, + } + for _, src := range sources { + s.seedResource(t, project, generation, src.resourceType, src.logicalID, "clinic/a") + } + s.seedOccurrence(t, project, generation, "clinic/a", "Condition\t", "ws-s0", "subject_Patient", "Patient", "ws-t", "", 0) + s.seedOccurrence(t, project, generation, "clinic/a", "\rCondition", "ws-s1", "subject_Patient", "Patient", "ws-t", "", 0) + s.seedOccurrence(t, project, generation, "clinic/a", "Condition", "ws-s2", "subject_Patient", "\u00a0Patient", "ws-t", "", 0) + s.seedOccurrence(t, project, generation, "clinic/a", " \tCondition\u00a0", "ws-s3", "subject_Patient", "\nPatient", "ws-t", "", 0) + s.seedOccurrence(t, project, generation, "clinic/a", "condition", "ws-s4", "subject_Patient", "PatIent", "ws-t", "", 0) + s.seedOccurrence(t, project, generation, "clinic/a", "Bodyſtructure", "ws-s5", "patient", "Patient", "ws-t", "", 0) + + unfiltered, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil)) + if err != nil { + t.Fatalf("unfiltered: %v", err) + } + unfilteredJSON, err := json.Marshal(unfiltered.Values) + if err != nil { + t.Fatalf("marshal unfiltered: %v", err) + } + if len(unfiltered.Diagnostics) != 0 { + t.Fatalf("unfiltered diagnostics = %+v, want none (every endpoint canonicalizes under the Go canonicalizer)", unfiltered.Diagnostics) + } + want := []catalog.RelationshipObservation{ + observation(project, generation, "clinic/a", "BodyStructure", "patient", "Patient", 1, "Patient", "BodyStructure", "INBOUND"), + observation(project, generation, "clinic/a", "Condition", "subject_Patient", "Patient", 5, "Patient", "Condition", "INBOUND"), + } + requireValuesByteIdentical(t, unfiltered.Values, want) + + for _, requested := range [][]string{ + {"Condition", "Patient", "BodyStructure"}, + {" condition ", "patient", "\tbodyStructure\u00a0"}, + } { + filtered, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil, requested...)) + if err != nil { + t.Fatalf("filtered %q: %v", requested, err) + } + filteredJSON, err := json.Marshal(filtered.Values) + if err != nil { + t.Fatalf("marshal filtered: %v", err) + } + if !bytes.Equal(filteredJSON, unfilteredJSON) { + t.Fatalf("filtered %q vs unfiltered not byte-identical\n got: %s\nwant: %s", requested, filteredJSON, unfilteredJSON) + } + if filtered.Digest != unfiltered.Digest || filtered.Status != unfiltered.Status { + t.Fatalf("filtered %q digest/status %q/%s != unfiltered %q/%s", + requested, filtered.Digest, filtered.Status, unfiltered.Digest, unfiltered.Status) + } + for _, d := range filtered.Diagnostics { + t.Fatalf("filtered %q carried diagnostics: %+v", requested, d) + } + } + + missed, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil, "Observation", "Patient")) + if err != nil { + t.Fatalf("non-matching filter: %v", err) + } + if len(missed.Values) != 0 || missed.Status != catalog.EvidenceEmpty { + t.Fatalf("non-matching filter returned %d values status %s, want 0/empty", len(missed.Values), missed.Status) + } + allInvalid, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil, "Nope\u00a9", "Patient")) + if err != nil { + t.Fatalf("all-invalid filter: %v", err) + } + if len(allInvalid.Values) != 0 || allInvalid.Status != catalog.EvidenceEmpty || len(allInvalid.Diagnostics) != 1 || + allInvalid.Diagnostics[0].Code != "INVALID_RESOURCE_TYPE" || allInvalid.Diagnostics[0].ResourceType != "Nope\u00a9" { + t.Fatalf("all-invalid filter result = %+v", allInvalid) + } + + rows, err := s.iso.Query(s.ctx, "SELECT source_resource_type, target_resource_type, COUNT(*) FROM "+s.edgeTable()+ + " WHERE project = $1 AND dataset_generation = $2 GROUP BY 1,2", project, generation) + if err != nil { + s.fail("raw spelling witness", err) + } + defer rows.Close() + seen := map[string]int64{} + for rows.Next() { + var source, target string + var count int64 + if err := rows.Scan(&source, &target, &count); err != nil { + s.fail("raw spelling witness scan", err) + } + seen[source+"\x00"+target] = count + } + if err := rows.Err(); err != nil { + s.fail("raw spelling witness rows", err) + } + wantRaw := map[string]int64{ + " \tCondition\u00a0\x00\nPatient": 1, + "Condition\x00\u00a0Patient": 1, + "Condition\t\x00Patient": 1, + "\rCondition\x00Patient": 1, + "Bodyſtructure\x00Patient": 1, + "condition\x00PatIent": 1, + } + if len(seen) != len(wantRaw) { + t.Fatalf("raw rows = %#v, want %#v", seen, wantRaw) + } + for key, count := range wantRaw { + if seen[key] != count { + t.Fatalf("raw row %q = %d, want %d", key, seen[key], count) + } + } +} + +func TestRelationshipObservationsBuilderDirectionDerivedFromRegistryOnRealPostgreSQL(t *testing.T) { + s := newRelationshipEvidenceService(t) + store := s.newStore(t) + const project, generation = "pgcat03-direction", "g1" + + s.seedResource(t, project, generation, "Condition", "cond-d", "clinic/a") + s.seedResource(t, project, generation, "BodyStructure", "bs-d", "clinic/a") + + edgeRepo, err := storepostgres.NewEdgeRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new edge repository", err) + } + for ordinal := int64(0); ordinal < 2; ordinal++ { + if _, err := edgeRepo.InsertEdge(s.ctx, storepostgres.EdgeOccurrenceRow{ + Project: project, DatasetGeneration: generation, AuthResourcePath: "clinic/a", + Label: "subject_Patient", SourceResourceType: "Condition", SourceLogicalID: "cond-d", + TargetResourceType: "Patient", TargetLogicalID: "pat-d", + SourcePath: "entry", SourceOrdinal: ordinal, + }); err != nil { + s.fail("insert repeated occurrence", err) + } + } + if _, err := edgeRepo.InsertEdge(s.ctx, storepostgres.EdgeOccurrenceRow{ + Project: project, DatasetGeneration: generation, AuthResourcePath: "clinic/a", + Label: "patient", SourceResourceType: "BodyStructure", SourceLogicalID: "bs-d", + TargetResourceType: "Patient", TargetLogicalID: "pat-d", + SourcePath: "", SourceOrdinal: 0, + }); err != nil { + s.fail("insert patient occurrence", err) + } + + result, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil)) + if err != nil { + t.Fatalf("read: %v", err) + } + want := []catalog.RelationshipObservation{ + { + Project: project, DatasetGeneration: generation, AuthResourcePath: "clinic/a", + FromType: "Patient", Label: "patient", ToType: "BodyStructure", EdgeCount: 1, + StorageFromType: "BodyStructure", StorageToType: "Patient", StorageDirection: "OUTBOUND", + BuilderFromType: "Patient", BuilderToType: "BodyStructure", BuilderDirection: "INBOUND", + }, + { + Project: project, DatasetGeneration: generation, AuthResourcePath: "clinic/a", + FromType: "Patient", Label: "subject_Patient", ToType: "Condition", EdgeCount: 2, + StorageFromType: "Condition", StorageToType: "Patient", StorageDirection: "OUTBOUND", + BuilderFromType: "Patient", BuilderToType: "Condition", BuilderDirection: "INBOUND", + }, + } + requireValuesByteIdentical(t, result.Values, want) + if result.Digest != relationshipDirectionDigest { + t.Fatalf("direction digest = %q, want golden %q", result.Digest, relationshipDirectionDigest) + } + + rows, err := s.iso.Query(s.ctx, "SELECT source_resource_type, label, target_resource_type, COUNT(*) FROM "+s.edgeTable()+ + " WHERE project = $1 AND dataset_generation = $2 GROUP BY 1,2,3 ORDER BY 1,2,3", project, generation) + if err != nil { + s.fail("storage witness", err) + } + defer rows.Close() + type storageRow struct { + from, label, to string + count int64 + } + var seen []storageRow + for rows.Next() { + var r storageRow + if err := rows.Scan(&r.from, &r.label, &r.to, &r.count); err != nil { + s.fail("storage witness scan", err) + } + seen = append(seen, r) + } + if err := rows.Err(); err != nil { + s.fail("storage witness rows", err) + } + wantStorage := []storageRow{ + {from: "BodyStructure", label: "patient", to: "Patient", count: 1}, + {from: "Condition", label: "subject_Patient", to: "Patient", count: 2}, + } + if len(seen) != len(wantStorage) { + t.Fatalf("storage witness rows = %+v, want %+v", seen, wantStorage) + } + for i := range seen { + if seen[i] != wantStorage[i] { + t.Fatalf("storage witness row %d = %+v, want %+v", i, seen[i], wantStorage[i]) + } + } +} + +func TestRelationshipObservationsConcurrentIdenticalOccurrenceRace(t *testing.T) { + s := newRelationshipEvidenceService(t) + const project, generation = "pgcat03-conflict", "g1" + s.seedResource(t, project, generation, "Condition", "cond-race", "clinic/a") + + poolA := s.openSingleConnPool(t, "writer A") + poolB := s.openSingleConnPool(t, "writer B") + pidA, pidB := backendPID(t, poolA), backendPID(t, poolB) + if pidA == pidB { + t.Fatalf("race connections share backend pid %d", pidA) + } + repoA, err := storepostgres.NewEdgeRepository(poolA, s.schema) + if err != nil { + s.fail("edge repository A", err) + } + repoB, err := storepostgres.NewEdgeRepository(poolB, s.schema) + if err != nil { + s.fail("edge repository B", err) + } + adapterRepo, err := NewRelationshipEvidenceRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("aggregation repository", err) + } + store, err := NewRelationshipAdapter(adapterRepo) + if err != nil { + s.fail("evidence store", err) + } + + const rounds = 12 + for round := 1; round <= rounds; round++ { + rec := storepostgres.EdgeOccurrenceRow{ + Project: project, DatasetGeneration: generation, AuthResourcePath: "clinic/a", + Label: "subject_Patient", SourceResourceType: "Condition", SourceLogicalID: "cond-race", + TargetResourceType: "Patient", TargetLogicalID: "pat-race", + SourcePath: "", SourceOrdinal: int64(round), + } + results := make([]error, 2) + var seqs [2]int64 + start := make(chan struct{}) + var wg sync.WaitGroup + race := func(index int, repo *storepostgres.EdgeRepository) { + defer wg.Done() + <-start + seq, err := repo.InsertEdge(s.ctx, rec) + results[index], seqs[index] = err, seq + } + wg.Add(2) + go race(0, repoA) + go race(1, repoB) + close(start) + wg.Wait() + + winnerA := results[0] == nil + loserOK := results[1] != nil && errors.Is(results[1], storepostgres.ErrEdgeConflict) + loserA := results[1] == nil + loserBOK := results[0] != nil && errors.Is(results[0], storepostgres.ErrEdgeConflict) + if !(winnerA && loserOK) && !(loserA && loserBOK) { + t.Fatalf("round %d: race outcomes A=%v B=%v, want exactly one success and one ErrEdgeConflict", + round, results[0], results[1]) + } + if winnerA && seqs[0] == 0 || loserA && seqs[1] == 0 { + t.Fatalf("round %d: winner returned seq 0: A=%d B=%d", round, seqs[0], seqs[1]) + } + if got := s.witnessRowCount(t, project, generation, "clinic/a", "Condition", "subject_Patient", "Patient"); got != int64(round) { + t.Fatalf("round %d: stored rows = %d, want %d (identity never stored twice)", round, got, round) + } + result, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil)) + if err != nil { + t.Fatalf("round %d read: %v", round, err) + } + requireValuesByteIdentical(t, result.Values, []catalog.RelationshipObservation{ + observation(project, generation, "clinic/a", "Condition", "subject_Patient", "Patient", int64(round), "Patient", "Condition", "INBOUND"), + }) + } +} + +func TestRelationshipObservationsConcurrentAbortConvergenceRace(t *testing.T) { + s := newRelationshipEvidenceService(t) + const project, generation = "pgcat03-abort", "g1" + s.seedResource(t, project, generation, "Condition", "cond-w", "clinic/a") + + poolW := s.openSingleConnPool(t, "aborting writer") + poolR := s.openSingleConnPool(t, "reader") + pidW, pidR := backendPID(t, poolW), backendPID(t, poolR) + if pidW == pidR { + t.Fatalf("reader and writer share backend pid %d", pidW) + } + adapterRepo, err := NewRelationshipEvidenceRepository(poolR, s.schema) + if err != nil { + s.fail("aggregation repository", err) + } + store, err := NewRelationshipAdapter(adapterRepo) + if err != nil { + s.fail("evidence store", err) + } + + witness := func(count int64) []catalog.RelationshipObservation { + if count == 0 { + return []catalog.RelationshipObservation{} + } + return []catalog.RelationshipObservation{ + observation(project, generation, "clinic/a", "Condition", "subject_Patient", "Patient", count, "Patient", "Condition", "INBOUND"), + } + } + readValues := func(step string) []catalog.RelationshipObservation { + result, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil)) + if err != nil { + t.Fatalf("%s read: %v", step, err) + } + return result.Values + } + + const rounds = 12 + insertSQL := "INSERT INTO " + s.edgeTable() + + " (project, dataset_generation, auth_resource_path, label," + + " source_resource_type, source_logical_id, target_resource_type, target_logical_id," + + " source_path, source_ordinal) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)" + for round := 1; round <= rounds; round++ { + wconn, err := poolW.Acquire(s.ctx) + if err != nil { + t.Fatalf("round %d acquire writer: %v", round, err) + } + tx, err := wconn.BeginTx(s.ctx, pgx.TxOptions{}) + if err != nil { + wconn.Release() + t.Fatalf("round %d begin: %v", round, err) + } + if _, err := tx.Exec(s.ctx, insertSQL, project, generation, "clinic/a", "subject_Patient", + "Condition", "cond-w", "Patient", "pat-w", "", int64(round)); err != nil { + _ = tx.Rollback(s.ctx) + wconn.Release() + t.Fatalf("round %d staged insert: %v", round, err) + } + var inflightPID int32 + if err := tx.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&inflightPID); err != nil { + _ = tx.Rollback(s.ctx) + wconn.Release() + t.Fatalf("round %d in-flight pid: %v", round, err) + } + if inflightPID != pidW { + _ = tx.Rollback(s.ctx) + wconn.Release() + t.Fatalf("writer backend changed: %d != %d", inflightPID, pidW) + } + var readerPID int32 + if err := poolR.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&readerPID); err != nil { + _ = tx.Rollback(s.ctx) + wconn.Release() + t.Fatalf("round %d reader pid: %v", round, err) + } + if readerPID == inflightPID { + _ = tx.Rollback(s.ctx) + wconn.Release() + t.Fatalf("reader and writer share backend pid %d mid-flight", readerPID) + } + requireValuesByteIdentical(t, readValues("mid-flight"), witness(int64(round-1))) + + if err := tx.Rollback(s.ctx); err != nil { + wconn.Release() + t.Fatalf("round %d rollback: %v", round, err) + } + requireValuesByteIdentical(t, readValues("post-abort"), witness(int64(round-1))) + + tx2, err := wconn.BeginTx(s.ctx, pgx.TxOptions{}) + if err != nil { + wconn.Release() + t.Fatalf("round %d begin commit-tx: %v", round, err) + } + if _, err := tx2.Exec(s.ctx, insertSQL, project, generation, "clinic/a", "subject_Patient", + "Condition", "cond-w", "Patient", "pat-w", "", int64(round)); err != nil { + _ = tx2.Rollback(s.ctx) + wconn.Release() + t.Fatalf("round %d committed insert: %v", round, err) + } + if err := tx2.Commit(s.ctx); err != nil { + wconn.Release() + t.Fatalf("round %d commit: %v", round, err) + } + wconn.Release() + + requireValuesByteIdentical(t, readValues("post-commit"), witness(int64(round))) + } + + if got := s.witnessRowCount(t, project, generation, "clinic/a", "Condition", "subject_Patient", "Patient"); got != rounds { + t.Fatalf("direct SQL row count = %d, want %d", got, rounds) + } + final, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil)) + if err != nil { + t.Fatalf("final read: %v", err) + } + if final.Digest != relationshipAbortRaceDigest { + t.Fatalf("abort-race final digest = %q, want golden %q", final.Digest, relationshipAbortRaceDigest) + } +} + +func TestRelationshipEvidenceIdentityGuardDropNonVacuity(t *testing.T) { + s := newRelationshipEvidenceService(t) + store := s.newStore(t) + const project, generation = "pgcat03-guard", "g1" + + s.seedResource(t, project, generation, "Condition", "cond-g", "clinic/a") + s.seedOccurrence(t, project, generation, "clinic/a", "Condition", "cond-g", "subject_Patient", "Patient", "pat-g", "", 0) + + duplicateSQL := "INSERT INTO " + s.edgeTable() + + " (project, dataset_generation, auth_resource_path, label," + + " source_resource_type, source_logical_id, target_resource_type, target_logical_id," + + " source_path, source_ordinal) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)" + duplicateArgs := []any{project, generation, "clinic/a", "subject_Patient", "Condition", "cond-g", "Patient", "pat-g", "", int64(0)} + + _, err := s.iso.Exec(s.ctx, duplicateSQL, duplicateArgs...) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "23505" || pgErr.ConstraintName != "fhir_edge_occurrence_identity_ux" { + t.Fatalf("duplicate insert with guard active: %v, want 23505 from fhir_edge_occurrence_identity_ux", err) + } + + if _, err := s.iso.Exec(s.ctx, "ALTER TABLE "+s.edgeTable()+ + " DROP CONSTRAINT fhir_edge_occurrence_identity_ux"); err != nil { + s.fail("drop identity guard", err) + } + if _, err := s.iso.Exec(s.ctx, duplicateSQL, duplicateArgs...); err != nil { + t.Fatalf("duplicate insert after guard drop was refused (%v); the guard was not what prevented it", err) + } + + result, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil)) + if err != nil { + t.Fatalf("guard-dropped read: %v", err) + } + requireValuesByteIdentical(t, result.Values, []catalog.RelationshipObservation{ + observation(project, generation, "clinic/a", "Condition", "subject_Patient", "Patient", 2, "Patient", "Condition", "INBOUND"), + }) + + if _, err := s.iso.Exec(s.ctx, "DELETE FROM "+s.edgeTable()+ + " WHERE ctid = (SELECT ctid FROM "+s.edgeTable()+ + " WHERE project = $1 AND dataset_generation = $2 AND auth_resource_path = $3 AND label = $4"+ + " AND source_resource_type = $5 AND source_logical_id = $6 AND target_resource_type = $7"+ + " AND target_logical_id = $8 AND source_path = $9 AND source_ordinal = $10 LIMIT 1)", + project, generation, "clinic/a", "subject_Patient", "Condition", "cond-g", "Patient", "pat-g", "", int64(0)); err != nil { + s.fail("remove guard-free duplicate", err) + } + if _, err := s.iso.Exec(s.ctx, "ALTER TABLE "+s.edgeTable()+ + " ADD CONSTRAINT fhir_edge_occurrence_identity_ux UNIQUE (project, dataset_generation,"+ + " source_resource_type, source_logical_id, label, target_resource_type, target_logical_id,"+ + " source_path, source_ordinal)"); err != nil { + s.fail("restore identity guard", err) + } + _, err = s.iso.Exec(s.ctx, duplicateSQL, duplicateArgs...) + if !errors.As(err, &pgErr) || pgErr.Code != "23505" || pgErr.ConstraintName != "fhir_edge_occurrence_identity_ux" { + t.Fatalf("duplicate insert after guard restore: %v, want 23505 from fhir_edge_occurrence_identity_ux", err) + } + var live int + if err := s.iso.QueryRow(s.ctx, + "SELECT COUNT(*) FROM pg_constraint WHERE conrelid = '"+s.edgeTable()+"'::regclass AND conname = 'fhir_edge_occurrence_identity_ux'").Scan(&live); err != nil { + s.fail("guard catalog witness", err) + } + if live != 1 { + t.Fatalf("restored guard catalog witness = %d, want 1", live) + } + restored, err := store.DiscoverRelationshipObservations(s.ctx, relationshipOpts(project, generation, + catalog.ExplicitAuthResourcePathsUnrestricted(true), nil)) + if err != nil { + t.Fatalf("restored-guard read: %v", err) + } + requireValuesByteIdentical(t, restored.Values, []catalog.RelationshipObservation{ + observation(project, generation, "clinic/a", "Condition", "subject_Patient", "Patient", 1, "Patient", "Condition", "INBOUND"), + }) +} + +func TestZZZeroLeftoverRelationshipEvidenceFixtures(t *testing.T) { + raw := relationshipTestPostgresURL(t) + relationshipOwnedMu.Lock() + owned := make(map[string]string, len(relationshipOwnedByName)) + for db, schema := range relationshipOwnedByName { + owned[db] = schema + } + created := relationshipCreatedTotal + relationshipOwnedMu.Unlock() + if created == 0 && relationshipBaselineDBs == 0 { + t.Skip("this process created no verify_pgcat03_* fixtures") + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service for audit: %v", err) + } + defer admin.Close() + + for db, schema := range owned { + if !strings.HasPrefix(db, relationshipFixturePrefix) { + t.Fatalf("registry holds foreign database %q", db) + } + var count int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Errorf("audit owned database %s: %v", db, err) + continue + } + if count != 0 { + t.Errorf("leftover owned database %s after all cleanups", db) + if schema != "" { + u, err := url.Parse(raw) + if err == nil { + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + db + if probe, err := pgxpool.New(ctx, u.String()); err == nil { + var schemaCount int + if err := probe.QueryRow(ctx, "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1", schema).Scan(&schemaCount); err != nil { + t.Errorf("audit owned schema %s in %s: %v", schema, db, err) + } else if schemaCount != 0 { + t.Errorf("leftover owned schema %s in leftover database %s", schema, db) + } + probe.Close() + } + } + } + } + } + + if relationshipBaselineDBs < 0 { + t.Errorf("pg_database baseline was not captured at process start (%s); exact-name audit above still ran", relationshipBaselineError) + return + } + var liveCount int + if err := admin.QueryRow(ctx, + "SELECT COUNT(*) FROM pg_database WHERE datname ~ $1", relationshipFixtureNamePattern.String()).Scan(&liveCount); err != nil { + t.Errorf("audit %s namespace: %v", relationshipFixturePrefix, err) + return + } + if liveCount != relationshipBaselineDBs { + t.Errorf("%s pg_database count = %d, want baseline %d", relationshipFixturePrefix, liveCount, relationshipBaselineDBs) + } +} + +const ( + relationshipExactCountsDigest = "sha256:762e10da27eea1702d9951611c95c4d772e9a1ed999d5b0b2db5c8b360749815" + relationshipAuthModesDigest = "sha256:74b528be7584396bba3341b76b4dd13cb63dc374b36916a6e8b68e8a9258f2e1" + relationshipDirectionDigest = "sha256:9b0305fa63d4f4be3cc1524da0d079e403f3090144e8ff0d793eb5ea92bd1cce" + relationshipAbortRaceDigest = "sha256:8f2f7979f86364ecebed93778191aeb9bf241b2c10da097d9fe5f132ce0cde17" +) diff --git a/internal/backends/postgres/internal/catalog/relationships_test.go b/internal/backends/postgres/internal/catalog/relationships_test.go new file mode 100644 index 000000000..9f34e40e3 --- /dev/null +++ b/internal/backends/postgres/internal/catalog/relationships_test.go @@ -0,0 +1,517 @@ +package postgres + +import ( + "context" + "errors" + "strings" + "testing" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5/pgxpool" +) + +type stubRelationshipStore struct { + calls int + scopes []RelationshipScope + groups []RelationshipOccurrenceGroup + err error +} + +func (s *stubRelationshipStore) ScanRelationshipOccurrences(_ context.Context, scope RelationshipScope) ([]RelationshipOccurrenceGroup, error) { + s.calls++ + s.scopes = append(s.scopes, scope) + if s.err != nil { + return nil, s.err + } + return s.groups, nil +} + +func stubStore(t *testing.T, stub *stubRelationshipStore) *RelationshipAdapter { + t.Helper() + store, err := NewRelationshipAdapter(stub) + if err != nil { + t.Fatalf("NewRelationshipAdapter: %v", err) + } + return store +} + +func TestRelationshipEvidenceStoreRequiresBoundary(t *testing.T) { + if _, err := NewRelationshipAdapter(nil); err == nil { + t.Fatal("nil relationship store accepted") + } + if _, err := NewRelationshipEvidenceRepository(nil, "rel_it"); err == nil { + t.Fatal("nil connection source accepted") + } + if _, err := NewRelationshipEvidenceRepository(discardConnSource{}, "Bad-Name"); err == nil { + t.Fatal("invalid schema name accepted") + } else if !errors.Is(err, storepostgres.ErrInvalidSchemaName) { + t.Fatalf("invalid schema name error = %v, want ErrInvalidSchemaName", err) + } +} + +type discardConnSource struct{} + +func (discardConnSource) Acquire(context.Context) (*pgxpool.Conn, error) { + return nil, errors.New("validation must reject before acquiring") +} + +func TestRelationshipEvidenceScopeValidationRejectsBeforeQuery(t *testing.T) { + repo, err := NewRelationshipEvidenceRepository(discardConnSource{}, "rel_it") + if err != nil { + t.Fatalf("NewRelationshipEvidenceRepository: %v", err) + } + trueValue := true + cases := []struct { + name string + scope RelationshipScope + }{ + {"blank project", RelationshipScope{Project: " ", DatasetGeneration: "g", AuthResourcePathsUnrestricted: &trueValue}}, + {"nil unrestricted flag", RelationshipScope{Project: "p", DatasetGeneration: "g"}}, + {"unrestricted with paths", RelationshipScope{Project: "p", DatasetGeneration: "g", AuthResourcePathsUnrestricted: &trueValue, AuthResourcePaths: []string{"a"}}}, + } + for _, tc := range cases { + _, err := repo.ScanRelationshipOccurrences(context.Background(), tc.scope) + if !errors.Is(err, ErrInvalidRelationshipScope) { + t.Errorf("%s: ScanRelationshipOccurrences error = %v, want ErrInvalidRelationshipScope", tc.name, err) + } + } +} + +func TestRelationshipEvidenceAggregateSQLGoldens(t *testing.T) { + const tableFQN = `"rel_it"."fhir_edge_occurrence"` + const groupedTail = " GROUP BY auth_resource_path, source_resource_type, label, target_resource_type" + + " ORDER BY auth_resource_path, source_resource_type, label, target_resource_type" + trueValue, falseValue := true, false + + cases := []struct { + name string + scope RelationshipScope + wantSQL string + wantArgs []any + }{ + { + name: "unrestricted without type filter", + scope: RelationshipScope{Project: "proj", DatasetGeneration: "g1", AuthResourcePathsUnrestricted: &trueValue}, + wantSQL: `SELECT auth_resource_path, source_resource_type, label, target_resource_type, COUNT(*) AS occurrence_count FROM "rel_it"."fhir_edge_occurrence" WHERE project = $1 AND dataset_generation = $2` + groupedTail, + wantArgs: []any{"proj", "g1"}, + }, + { + name: "restricted one path", + scope: RelationshipScope{Project: "proj", DatasetGeneration: "g1", AuthResourcePathsUnrestricted: &falseValue, AuthResourcePaths: []string{"clinic/a"}}, + wantSQL: `SELECT auth_resource_path, source_resource_type, label, target_resource_type, COUNT(*) AS occurrence_count FROM "rel_it"."fhir_edge_occurrence" WHERE project = $1 AND dataset_generation = $2 AND auth_resource_path = ANY($3::text[])` + groupedTail, + wantArgs: []any{"proj", "g1", []string{"clinic/a"}}, + }, + { + name: "restricted multiple paths", + scope: RelationshipScope{Project: "proj", DatasetGeneration: "", AuthResourcePathsUnrestricted: &falseValue, AuthResourcePaths: []string{"clinic/a", "clinic/b"}}, + wantSQL: `SELECT auth_resource_path, source_resource_type, label, target_resource_type, COUNT(*) AS occurrence_count FROM "rel_it"."fhir_edge_occurrence" WHERE project = $1 AND dataset_generation = $2 AND auth_resource_path = ANY($3::text[])` + groupedTail, + wantArgs: []any{"proj", "", []string{"clinic/a", "clinic/b"}}, + }, + } + for _, tc := range cases { + bound, err := bindRelationshipAggregate(tableFQN, tc.scope) + if err != nil { + t.Errorf("%s: bindRelationshipAggregate: %v", tc.name, err) + continue + } + if bound.SQL != tc.wantSQL { + t.Errorf("%s: SQL mismatch\n got: %s\nwant: %s", tc.name, bound.SQL, tc.wantSQL) + } + if len(bound.Args) != len(tc.wantArgs) { + t.Errorf("%s: %d args, want %d", tc.name, len(bound.Args), len(tc.wantArgs)) + continue + } + for i := range bound.Args { + if !argsEqual(bound.Args[i], tc.wantArgs[i]) { + t.Errorf("%s: arg %d = %#v, want %#v", tc.name, i, bound.Args[i], tc.wantArgs[i]) + } + } + } +} + +func argsEqual(a, b any) bool { + as, aOK := a.([]string) + bs, bOK := b.([]string) + if aOK || bOK { + if !aOK || !bOK || len(as) != len(bs) { + return false + } + for i := range as { + if as[i] != bs[i] { + return false + } + } + return true + } + return a == b +} + +func TestRelationshipEvidenceScopeBindingNeverFlattensAuthPaths(t *testing.T) { + stub := &stubRelationshipStore{} + store := stubStore(t, stub) + _, err := store.DiscoverRelationshipObservations(context.Background(), catalog.RelationshipObservationOptions{ + Project: "proj", + DatasetGeneration: " g1 ", + AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(false), + AuthResourcePaths: []string{"clinic/a", "clinic/b"}, + }) + if err != nil { + t.Fatalf("DiscoverRelationshipObservations: %v", err) + } + if stub.calls != 1 || len(stub.scopes) != 1 { + t.Fatalf("scans = %d, want exactly 1", stub.calls) + } + got := stub.scopes[0] + if got.Project != "proj" || got.DatasetGeneration != "g1" { + t.Fatalf("scope = %+v, want project proj generation g1 (normalized)", got) + } + if got.AuthResourcePathsUnrestricted == nil || *got.AuthResourcePathsUnrestricted { + t.Fatalf("restricted scope arrived unrestricted: %+v", got) + } + if strings.Join(got.AuthResourcePaths, "|") != "clinic/a|clinic/b" { + t.Fatalf("auth paths = %#v, want exactly the two authorized paths", got.AuthResourcePaths) + } +} + +func TestRelationshipEvidenceRestrictedEmptyReadsNothing(t *testing.T) { + stub := &stubRelationshipStore{groups: []RelationshipOccurrenceGroup{ + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "Condition", Label: "subject_Patient", StorageTargetResourceType: "Patient", OccurrenceCount: 3}, + }} + store := stubStore(t, stub) + result, err := store.DiscoverRelationshipObservations(context.Background(), catalog.RelationshipObservationOptions{ + Project: "proj", + DatasetGeneration: "g1", + AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(false), + }) + if err != nil { + t.Fatalf("DiscoverRelationshipObservations: %v", err) + } + if stub.calls != 0 { + t.Fatalf("restricted-empty read ran %d scans, want no query at all", stub.calls) + } + if result.Status != catalog.EvidenceEmpty || !result.Available || !result.Complete || len(result.Values) != 0 { + t.Fatalf("restricted-empty result = %+v, want available complete empty with no values", result) + } + if result.Digest != "sha256:74234e98afe7498fb5daf1f36ac2d78acc339464f950703b8c019892f982b90b" { + t.Fatalf("empty digest = %q", result.Digest) + } +} + +func TestRelationshipEvidenceLegacyCompatRuleIsShared(t *testing.T) { + stub := &stubRelationshipStore{} + store := stubStore(t, stub) + if _, err := store.DiscoverRelationshipObservations(context.Background(), catalog.RelationshipObservationOptions{ + Project: "proj", DatasetGeneration: "g1", + }); err != nil { + t.Fatalf("legacy unrestricted read: %v", err) + } + got := stub.scopes[0] + if got.AuthResourcePathsUnrestricted == nil || !*got.AuthResourcePathsUnrestricted || len(got.AuthResourcePaths) != 0 { + t.Fatalf("absent flag with no paths must be unrestricted: %+v", got) + } + + stub2 := &stubRelationshipStore{} + store2 := stubStore(t, stub2) + if _, err := store2.DiscoverRelationshipObservations(context.Background(), catalog.RelationshipObservationOptions{ + Project: "proj", DatasetGeneration: "g1", AuthResourcePaths: []string{"clinic/a"}, + }); err != nil { + t.Fatalf("legacy restricted read: %v", err) + } + got2 := stub2.scopes[0] + if got2.AuthResourcePathsUnrestricted == nil || *got2.AuthResourcePathsUnrestricted { + t.Fatalf("absent flag with paths must stay restricted: %+v", got2) + } + if strings.Join(got2.AuthResourcePaths, "|") != "clinic/a" { + t.Fatalf("legacy restricted paths = %#v", got2.AuthResourcePaths) + } + + stub3 := &stubRelationshipStore{} + store3 := stubStore(t, stub3) + if _, err := store3.DiscoverRelationshipObservations(context.Background(), catalog.RelationshipObservationOptions{ + Project: "proj", DatasetGeneration: "g1", + AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + AuthResourcePaths: []string{"clinic/a"}, + }); err != nil { + t.Fatalf("explicit unrestricted read: %v", err) + } + got3 := stub3.scopes[0] + if got3.AuthResourcePathsUnrestricted == nil || !*got3.AuthResourcePathsUnrestricted || len(got3.AuthResourcePaths) != 0 { + t.Fatalf("explicit unrestricted must win over stray paths: %+v", got3) + } +} + +func TestRelationshipEvidencePreservesStorageAndBuilderDirection(t *testing.T) { + stub := &stubRelationshipStore{groups: []RelationshipOccurrenceGroup{ + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "Condition", Label: "subject_Patient", StorageTargetResourceType: "Patient", OccurrenceCount: 3}, + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "BodyStructure", Label: "patient", StorageTargetResourceType: "Patient", OccurrenceCount: 1}, + }} + store := stubStore(t, stub) + result, err := store.DiscoverRelationshipObservations(context.Background(), catalog.RelationshipObservationOptions{ + Project: "proj", + DatasetGeneration: "g1", + AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + }) + if err != nil { + t.Fatalf("DiscoverRelationshipObservations: %v", err) + } + if len(result.Values) != 2 { + t.Fatalf("values = %+v, want 2", result.Values) + } + for _, value := range result.Values { + if value.StorageDirection != "OUTBOUND" { + t.Errorf("storage direction = %q, want OUTBOUND", value.StorageDirection) + } + if value.BuilderDirection != "INBOUND" { + t.Errorf("builder direction = %q, want INBOUND", value.BuilderDirection) + } + if value.StorageFromType == value.StorageToType { + t.Errorf("endpoints collapsed: %+v", value) + } + if value.FromType != value.BuilderFromType || value.ToType != value.BuilderToType { + t.Errorf("Builder-facing aliases diverge: %+v", value) + } + switch value.Label { + case "subject_Patient": + if value.StorageFromType != "Condition" || value.StorageToType != "Patient" || + value.BuilderFromType != "Patient" || value.BuilderToType != "Condition" || value.EdgeCount != 3 { + t.Errorf("subject_Patient observation = %+v", value) + } + case "patient": + if value.StorageFromType != "BodyStructure" || value.StorageToType != "Patient" || + value.BuilderFromType != "Patient" || value.BuilderToType != "BodyStructure" || value.EdgeCount != 1 { + t.Errorf("patient observation = %+v", value) + } + default: + t.Errorf("unexpected label %q", value.Label) + } + } + if result.Digest != "sha256:3a90f45976b58876882254602a6d9bbcc884f021a8f62cdbc468982915737083" { + t.Fatalf("direction fixture digest = %q", result.Digest) + } + if result.Status != catalog.EvidenceAvailable || !result.Available || !result.Complete { + t.Fatalf("status = %+v, want available complete", result) + } +} + +func TestRelationshipEvidenceCasingIsCanonicalizedNotDiagnosed(t *testing.T) { + stub := &stubRelationshipStore{groups: []RelationshipOccurrenceGroup{ + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "condition", Label: "subject_Patient", StorageTargetResourceType: "patient", OccurrenceCount: 2}, + }} + store := stubStore(t, stub) + result, err := store.DiscoverRelationshipObservations(context.Background(), catalog.RelationshipObservationOptions{ + Project: "proj", DatasetGeneration: "g1", AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + }) + if err != nil { + t.Fatalf("DiscoverRelationshipObservations: %v", err) + } + if len(result.Diagnostics) != 0 { + t.Fatalf("diagnostics = %+v, want none", result.Diagnostics) + } + if len(result.Values) != 1 || result.Values[0].StorageFromType != "Condition" || result.Values[0].StorageToType != "Patient" { + t.Fatalf("values = %+v, want canonicalized Condition/Patient", result.Values) + } +} + +func TestRelationshipEvidenceDiagnosesInvalidEndpointsAndTraversals(t *testing.T) { + stub := &stubRelationshipStore{groups: []RelationshipOccurrenceGroup{ + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "NotAThing", Label: "subject_Patient", StorageTargetResourceType: "Patient", OccurrenceCount: 4}, + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "Resource", Label: "subject_Patient", StorageTargetResourceType: "Patient", OccurrenceCount: 1}, + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "Patient", Label: "not_a_generated_label", StorageTargetResourceType: "Practitioner", OccurrenceCount: 7}, + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "Condition", Label: "subject_Patient", StorageTargetResourceType: "Patient", OccurrenceCount: 2}, + }} + store := stubStore(t, stub) + result, err := store.DiscoverRelationshipObservations(context.Background(), catalog.RelationshipObservationOptions{ + Project: "proj", DatasetGeneration: "g1", AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + }) + if err != nil { + t.Fatalf("DiscoverRelationshipObservations: %v", err) + } + if len(result.Values) != 1 || result.Values[0].StorageFromType != "Condition" { + t.Fatalf("invalid evidence leaked into values: %+v", result.Values) + } + if result.Status != catalog.EvidenceAvailable || !result.Available { + t.Fatalf("result with a valid subset must stay available: %+v", result) + } + if len(result.Diagnostics) != 3 { + t.Fatalf("diagnostics = %+v, want three", result.Diagnostics) + } + for _, d := range result.Diagnostics { + switch d.StorageFromType { + case "NotAThing", "Resource": + if d.Code != "INVALID_RELATIONSHIP_ENDPOINT" || + d.Message != "relationship endpoint is not a concrete generated FHIR resource" || + d.Label != "subject_Patient" || d.StorageToType != "Patient" || + (d.StorageFromType == "NotAThing" && d.Count != 4) || (d.StorageFromType == "Resource" && d.Count != 1) { + t.Errorf("endpoint diagnostic = %+v", d) + } + case "Patient": + if d.Code != "UNSUPPORTED_GENERATED_TRAVERSAL" || + d.Message != "relationship endpoints and label are not a supported generated traversal" || + d.Label != "not_a_generated_label" || d.StorageToType != "Practitioner" || d.Count != 7 { + t.Errorf("traversal diagnostic = %+v", d) + } + default: + t.Errorf("unexpected diagnostic %q", d.StorageFromType) + } + } +} + +func TestRelationshipEvidenceNonPositiveCountIsDiagnosed(t *testing.T) { + observation, ok, diagnostic := catalog.ProjectRelationshipObservation("proj", "g1", "clinic/a", "Condition", "subject_Patient", "Patient", 0) + if ok || observation.Label != "" { + t.Fatalf("zero count accepted: %+v", observation) + } + if diagnostic.Code != "INVALID_RELATIONSHIP_COUNT" || + diagnostic.Message != "relationship evidence count must be positive" || + diagnostic.StorageFromType != "Condition" || diagnostic.StorageToType != "Patient" || diagnostic.Label != "subject_Patient" { + t.Fatalf("count diagnostic = %+v", diagnostic) + } +} + +func TestRelationshipEvidenceRepeatedOccurrencesAndDirectionsStaySeparate(t *testing.T) { + stub := &stubRelationshipStore{groups: []RelationshipOccurrenceGroup{ + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "Condition", Label: "subject_Patient", StorageTargetResourceType: "Patient", OccurrenceCount: 2}, + {AuthResourcePath: "clinic/b", StorageSourceResourceType: "Condition", Label: "subject_Patient", StorageTargetResourceType: "Patient", OccurrenceCount: 5}, + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "Patient", Label: "subject_Patient", StorageTargetResourceType: "Condition", OccurrenceCount: 3}, + }} + store := stubStore(t, stub) + result, err := store.DiscoverRelationshipObservations(context.Background(), catalog.RelationshipObservationOptions{ + Project: "proj", DatasetGeneration: "g1", AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + }) + if err != nil { + t.Fatalf("DiscoverRelationshipObservations: %v", err) + } + if len(result.Values) != 3 { + t.Fatalf("values = %+v, want three distinct observations", result.Values) + } + seen := map[string]int64{} + for _, v := range result.Values { + seen[v.AuthResourcePath+"|"+v.StorageFromType+"->"+v.StorageToType] = v.EdgeCount + } + if seen["clinic/a|Condition->Patient"] != 2 || seen["clinic/b|Condition->Patient"] != 5 || seen["clinic/a|Patient->Condition"] != 3 { + t.Fatalf("cross-scope or cross-direction sum detected: %#v", seen) + } + for i := 1; i < len(result.Values); i++ { + if catalog.RelationshipEvidenceLess(result.Values[i], result.Values[i-1]) { + t.Fatalf("values not sorted at %d: %+v", i, result.Values) + } + } +} + +func TestRelationshipEvidenceReadAliasDelegates(t *testing.T) { + stub := &stubRelationshipStore{groups: []RelationshipOccurrenceGroup{ + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "Condition", Label: "subject_Patient", StorageTargetResourceType: "Patient", OccurrenceCount: 1}, + }} + store := stubStore(t, stub) + opts := catalog.RelationshipObservationOptions{ + Project: "proj", DatasetGeneration: "g1", AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + } + direct, err := store.DiscoverRelationshipObservations(context.Background(), opts) + if err != nil { + t.Fatalf("discover: %v", err) + } + aliased, err := store.ReadRelationshipObservations(context.Background(), opts) + if err != nil { + t.Fatalf("alias: %v", err) + } + if stub.calls != 2 { + t.Fatalf("alias must delegate a real read, calls = %d", stub.calls) + } + if direct.Digest != aliased.Digest || len(aliased.Values) != 1 || aliased.Values[0].EdgeCount != 1 { + t.Fatalf("alias result diverges: %q vs %q", direct.Digest, aliased.Digest) + } +} + +func TestRelationshipEvidenceUnavailablePropagatesError(t *testing.T) { + cause := errors.New("synthetic scan failure") + stub := &stubRelationshipStore{err: cause} + store := stubStore(t, stub) + result, err := store.DiscoverRelationshipObservations(context.Background(), catalog.RelationshipObservationOptions{ + Project: "proj", DatasetGeneration: "g1", AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + }) + if err == nil || !errors.Is(err, cause) { + t.Fatalf("error = %v, want wrapped scan failure", err) + } + if result.Available || result.Complete || result.Status != catalog.EvidenceUnavailable || len(result.Values) != 0 { + t.Fatalf("unavailable result = %+v", result) + } + found := false + for _, d := range result.Diagnostics { + if d.Code == "RELATIONSHIP_EVIDENCE_UNAVAILABLE" && strings.Contains(d.Message, "synthetic scan failure") { + found = true + } + } + if !found { + t.Fatalf("missing unavailable diagnostic: %+v", result.Diagnostics) + } +} + +func variantGroups() []RelationshipOccurrenceGroup { + return []RelationshipOccurrenceGroup{ + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "Condition", Label: "subject_Patient", StorageTargetResourceType: "Patient", OccurrenceCount: 1}, + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "condition", Label: "subject_Patient", StorageTargetResourceType: "PatIent", OccurrenceCount: 1}, + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "Condition\t", Label: "subject_Patient", StorageTargetResourceType: "Patient", OccurrenceCount: 1}, + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "\rCondition", Label: "subject_Patient", StorageTargetResourceType: "Patient", OccurrenceCount: 1}, + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "Condition\u00a0", Label: "subject_Patient", StorageTargetResourceType: "\u00a0Patient", OccurrenceCount: 1}, + {AuthResourcePath: "clinic/a", StorageSourceResourceType: " \tCondition\u00a0", Label: "subject_Patient", StorageTargetResourceType: "\nPatient", OccurrenceCount: 1}, + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "BodyStructure", Label: "patient", StorageTargetResourceType: "Patient", OccurrenceCount: 1}, + {AuthResourcePath: "clinic/a", StorageSourceResourceType: "NotAThing", Label: "subject_Patient", StorageTargetResourceType: "Patient", OccurrenceCount: 9}, + } +} + +func TestRelationshipEvidenceTypeRequestCanonicalizesAndDiagnoses(t *testing.T) { + stub := &stubRelationshipStore{groups: variantGroups()} + store := stubStore(t, stub) + result, err := store.DiscoverRelationshipObservations(context.Background(), catalog.RelationshipObservationOptions{ + Project: "proj", DatasetGeneration: "g1", AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + ResourceTypes: []string{"condition", "Patient", "NotAThing"}, + }) + if err != nil { + t.Fatalf("DiscoverRelationshipObservations: %v", err) + } + if len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != "INVALID_RESOURCE_TYPE" || result.Diagnostics[0].ResourceType != "NotAThing" { + t.Fatalf("diagnostics = %+v, want one INVALID_RESOURCE_TYPE for NotAThing", result.Diagnostics) + } + if len(result.Values) != 1 || result.Values[0].StorageFromType != "Condition" || result.Values[0].EdgeCount != 6 { + t.Fatalf("filtered values = %+v, want the six canonical Condition->Patient variants summed, with the BodyStructure and NotAThing groups silently out", result.Values) + } + + unfiltered, err := store.DiscoverRelationshipObservations(context.Background(), catalog.RelationshipObservationOptions{ + Project: "proj", DatasetGeneration: "g1", AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + }) + if err != nil { + t.Fatalf("unfiltered: %v", err) + } + full, err := store.DiscoverRelationshipObservations(context.Background(), catalog.RelationshipObservationOptions{ + Project: "proj", DatasetGeneration: "g1", AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + ResourceTypes: []string{"Condition", "Patient", "BodyStructure"}, + }) + if err != nil { + t.Fatalf("full-set filtered: %v", err) + } + if full.Digest != unfiltered.Digest || len(full.Values) != 2 { + t.Fatalf("filtered full-set digest %q != unfiltered %q (values %v)", full.Digest, unfiltered.Digest, full.Values) + } + + stub2 := &stubRelationshipStore{groups: variantGroups()} + store2 := stubStore(t, stub2) + emptyResult, err := store2.DiscoverRelationshipObservations(context.Background(), catalog.RelationshipObservationOptions{ + Project: "proj", DatasetGeneration: "g1", AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + ResourceTypes: []string{"NotAThing"}, + }) + if err != nil { + t.Fatalf("all-invalid request: %v", err) + } + if stub2.calls != 1 { + t.Fatalf("all-invalid request must still scan once, calls = %d", stub2.calls) + } + if len(emptyResult.Values) != 0 || emptyResult.Status != catalog.EvidenceEmpty { + t.Fatalf("all-invalid result = %+v, want empty", emptyResult) + } + for _, d := range emptyResult.Diagnostics { + if d.Code == "INVALID_RELATIONSHIP_ENDPOINT" { + t.Fatalf("requested read must not diagnose rows outside the set: %+v", d) + } + } +} diff --git a/internal/backends/postgres/internal/catalog/resource_inventory.go b/internal/backends/postgres/internal/catalog/resource_inventory.go new file mode 100644 index 000000000..31de9bf02 --- /dev/null +++ b/internal/backends/postgres/internal/catalog/resource_inventory.go @@ -0,0 +1,244 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + fhirschema "github.com/calypr/loom/internal/fhir/schema" +) + +var ErrResourceInventoryBuildUnavailable = errors.New( + "resource inventory build is unavailable; run rebuild-resource-inventory --generation ID against the PostgreSQL store") + +var ErrResourceInventoryWrite = errors.New("postgres catalog: resource inventory write failed") + +type ResourceInventoryUnavailableError struct { + Project string + DatasetGeneration string + Reason string +} + +func (e *ResourceInventoryUnavailableError) Error() string { + return fmt.Sprintf("%v: project %q generation %q: %s", + ErrResourceInventoryBuildUnavailable, e.Project, e.DatasetGeneration, e.Reason) +} + +func (e *ResourceInventoryUnavailableError) Unwrap() error { + return ErrResourceInventoryBuildUnavailable +} + +type ResourceInventoryWriteError struct { + Op string + Project string + DatasetGeneration string + BuildID string + Err error +} + +func (e *ResourceInventoryWriteError) Error() string { + return fmt.Sprintf("postgres catalog: resource inventory rebuild failed while writing %s for build %s (%s/%s): %v; "+ + "the build was not published and the prior marker still governs reads", + e.Op, e.BuildID, e.Project, e.DatasetGeneration, e.Err) +} + +func (e *ResourceInventoryWriteError) Unwrap() []error { + return []error{ErrResourceInventoryWrite, e.Err} +} + +func (s *InventoryAdapter) RebuildResourceInventory(ctx context.Context, opts catalog.ResourceInventoryRebuildOptions) (catalog.ResourceInventoryRebuildSummary, error) { + if err := requireLeg("inventory", s.inventory != nil, "RebuildResourceInventory"); err != nil { + return catalog.ResourceInventoryRebuildSummary{}, err + } + datasetGeneration := catalog.NormalizeDatasetGeneration(opts.DatasetGeneration) + if opts.Project == "" || datasetGeneration == "" { + return catalog.ResourceInventoryRebuildSummary{}, fmt.Errorf("resource inventory rebuild requires project and exact dataset generation") + } + resourceTypes, diagnostics := evidenceResourceTypes(opts.ResourceTypes) + if len(diagnostics) > 0 { + return catalog.ResourceInventoryRebuildSummary{}, fmt.Errorf("resource inventory rebuild has invalid resource types: %s", diagnostics[0].ResourceType) + } + if len(resourceTypes) == 0 { + return catalog.ResourceInventoryRebuildSummary{}, fmt.Errorf("resource inventory rebuild requires generated resource types") + } + batchSize := opts.BatchSize + if batchSize <= 0 { + batchSize = 1000 + } + + counts := make([]catalog.ResourceInventoryCount, 0, len(resourceTypes)) + for _, resourceType := range resourceTypes { + if err := ctx.Err(); err != nil { + return catalog.ResourceInventoryRebuildSummary{}, err + } + groups, err := s.inventory.ScanSourceCounts(ctx, opts.Project, datasetGeneration, resourceType) + if err != nil { + return catalog.ResourceInventoryRebuildSummary{}, fmt.Errorf("scan retained %s rows for resource inventory: %w", resourceType, err) + } + if len(groups) == 0 { + counts = append(counts, catalog.ResourceInventoryCount{ + Project: opts.Project, DatasetGeneration: datasetGeneration, + ResourceType: resourceType, DocumentCount: 0, + }) + continue + } + for _, group := range groups { + if group.DocumentCount < 0 { + return catalog.ResourceInventoryRebuildSummary{}, fmt.Errorf("decode %s resource inventory count: negative count", resourceType) + } + counts = append(counts, catalog.ResourceInventoryCount{ + Project: opts.Project, DatasetGeneration: datasetGeneration, + AuthResourcePath: group.AuthResourcePath, ResourceType: resourceType, DocumentCount: group.DocumentCount, + }) + } + } + build, countRows, err := catalog.NewResourceInventoryBuild(opts.Project, datasetGeneration, resourceTypes, counts) + if err != nil { + return catalog.ResourceInventoryRebuildSummary{}, fmt.Errorf("prepare resource inventory build: %w", err) + } + if err := s.inventory.InsertCounts(ctx, countRows, batchSize); err != nil { + return catalog.ResourceInventoryRebuildSummary{}, &ResourceInventoryWriteError{ + Op: "counts", Project: opts.Project, DatasetGeneration: datasetGeneration, BuildID: build.BuildID, Err: err, + } + } + if err := s.inventory.PublishBuild(ctx, build); err != nil { + return catalog.ResourceInventoryRebuildSummary{}, &ResourceInventoryWriteError{ + Op: "marker", Project: opts.Project, DatasetGeneration: datasetGeneration, BuildID: build.BuildID, Err: err, + } + } + var documentCount int64 + for _, count := range countRows { + documentCount += count.DocumentCount + } + return catalog.ResourceInventoryRebuildSummary{ + BuildID: build.BuildID, CoverageDigest: build.CoverageDigest, ResourceTypeCount: build.ResourceTypeCount, + CountRowCount: build.CountRowCount, DocumentCount: documentCount, + }, nil +} + +func (s *InventoryAdapter) ReadResourceInventory(ctx context.Context, opts catalog.ResourceInventoryOptions) (catalog.ResourceInventoryResult, error) { + return s.DiscoverResourceInventory(ctx, opts) +} + +func (s *InventoryAdapter) DiscoverResourceInventory(ctx context.Context, opts catalog.ResourceInventoryOptions) (catalog.ResourceInventoryResult, error) { + resourceTypes, diagnostics := evidenceResourceTypes(opts.ResourceTypes) + result := catalog.ResourceInventoryResult{Values: make([]catalog.ResourceInventoryObservation, 0, len(resourceTypes)), Available: true, Complete: true, Diagnostics: diagnostics} + if err := requireLeg("inventory", s.inventory != nil, "DiscoverResourceInventory"); err != nil { + return unavailableResourceInventory(result, err.Error()), err + } + datasetGeneration := catalog.NormalizeDatasetGeneration(opts.DatasetGeneration) + + build, found, err := s.inventory.ReadBuild(ctx, opts.Project, datasetGeneration) + if err != nil { + return unavailableResourceInventory(result, err.Error()), fmt.Errorf("read resource inventory marker: %w", err) + } + if !found { + return unavailableResourceInventory(result, "completed resource inventory build marker is missing"), + &ResourceInventoryUnavailableError{Project: opts.Project, DatasetGeneration: datasetGeneration, Reason: "marker is missing"} + } + coveredTypes := make(map[string]struct{}, len(build.ResourceTypes)) + for _, resourceType := range build.ResourceTypes { + coveredTypes[resourceType] = struct{}{} + } + canonicalBuildTypes, typeDiagnostics := evidenceResourceTypes(build.ResourceTypes) + typesAreCanonical := len(typeDiagnostics) == 0 && len(canonicalBuildTypes) == len(build.ResourceTypes) + if typesAreCanonical { + for i := range canonicalBuildTypes { + if canonicalBuildTypes[i] != build.ResourceTypes[i] { + typesAreCanonical = false + break + } + } + } + coverageDigest, coverageErr := catalog.ResourceInventoryCoverageDigest(build.ResourceTypes) + if build.Key != catalog.ResourceInventoryBuildKey(opts.Project, datasetGeneration) || + build.RecordType != catalog.ResourceInventoryBuildRecordType || build.Project != opts.Project || + catalog.NormalizeDatasetGeneration(build.DatasetGeneration) != datasetGeneration || + !strings.HasPrefix(build.BuildID, "sha256:") || len(build.BuildID) != len("sha256:")+64 || + build.CoverageDigest == "" || build.CoverageDigest != coverageDigest || + coverageErr != nil || !typesAreCanonical || build.ResourceTypeCount != len(build.ResourceTypes) || + build.CountRowCount < build.ResourceTypeCount || !build.Complete { + return unavailableResourceInventory(result, "resource inventory build marker is stale or incomplete"), + &ResourceInventoryUnavailableError{Project: opts.Project, DatasetGeneration: datasetGeneration, Reason: "stale or incomplete marker"} + } + for _, resourceType := range resourceTypes { + if _, ok := coveredTypes[resourceType]; !ok { + return unavailableResourceInventory(result, "resource inventory build does not cover requested resource type "+resourceType), + &ResourceInventoryUnavailableError{Project: opts.Project, DatasetGeneration: datasetGeneration, Reason: "build does not cover " + resourceType} + } + } + authPathsUnrestricted := catalog.EffectiveAuthResourcePathsUnrestricted(opts.AuthResourcePaths, opts.AuthResourcePathsUnrestricted) + if !authPathsUnrestricted && len(opts.AuthResourcePaths) == 0 { + result.Digest, _ = catalog.ResourceInventoryDigest(result.Values) + result.Status = catalog.EvidenceStatusFor(result.Available, result.Complete, result.Truncated, len(result.Values)) + return result, nil + } + var authPaths []string + if !authPathsUnrestricted { + authPaths = append([]string(nil), opts.AuthResourcePaths...) + } + sums, err := s.inventory.ReadCountSums(ctx, storepostgres.InventoryCountScope{ + Project: opts.Project, + DatasetGeneration: datasetGeneration, + BuildID: build.BuildID, + ResourceTypes: append([]string(nil), resourceTypes...), + AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(authPathsUnrestricted), + AuthResourcePaths: authPaths, + }) + if err != nil { + return unavailableResourceInventory(result, err.Error()), fmt.Errorf("read resource inventory counts: %w", err) + } + for _, resourceType := range resourceTypes { + result.Values = append(result.Values, catalog.ResourceInventoryObservation{ + Project: opts.Project, DatasetGeneration: datasetGeneration, ResourceType: resourceType, DocumentCount: sums[resourceType], + }) + } + sort.Slice(result.Values, func(i, j int) bool { + return inventoryObservationLess(result.Values[i], result.Values[j]) + }) + result.Digest, _ = catalog.ResourceInventoryDigest(result.Values) + result.Status = catalog.EvidenceStatusFor(result.Available, result.Complete, result.Truncated, len(result.Values)) + return result, nil +} + +func unavailableResourceInventory(result catalog.ResourceInventoryResult, message string) catalog.ResourceInventoryResult { + result.Values = []catalog.ResourceInventoryObservation{} + result.Available, result.Complete, result.Status = false, false, catalog.EvidenceUnavailable + result.Diagnostics = append(result.Diagnostics, catalog.EvidenceDiagnostic{Code: "RESOURCE_INVENTORY_UNAVAILABLE", Message: message}) + return result +} + +func evidenceResourceTypes(requested []string) ([]string, []catalog.EvidenceDiagnostic) { + if len(requested) == 0 { + return fhirschema.ResourceTypes(), nil + } + seen := make(map[string]struct{}, len(requested)) + diagnostics := make([]catalog.EvidenceDiagnostic, 0) + for _, requestedType := range requested { + canonical, ok := fhirschema.ConcreteResourceType(requestedType) + if !ok { + diagnostics = append(diagnostics, catalog.EvidenceDiagnostic{Code: "INVALID_RESOURCE_TYPE", Message: "requested resource type is not a concrete generated FHIR resource", ResourceType: requestedType}) + continue + } + seen[canonical] = struct{}{} + } + result := make([]string, 0, len(seen)) + for resourceType := range seen { + result = append(result, resourceType) + } + sort.Strings(result) + return result, diagnostics +} + +func inventoryObservationLess(a, b catalog.ResourceInventoryObservation) bool { + for _, pair := range [][2]string{{a.Project, b.Project}, {a.DatasetGeneration, b.DatasetGeneration}, {a.AuthResourcePath, b.AuthResourcePath}, {a.ResourceType, b.ResourceType}} { + if pair[0] != pair[1] { + return pair[0] < pair[1] + } + } + return a.DocumentCount < b.DocumentCount +} diff --git a/internal/backends/postgres/internal/catalog/resource_inventory_test.go b/internal/backends/postgres/internal/catalog/resource_inventory_test.go new file mode 100644 index 000000000..c7350dafd --- /dev/null +++ b/internal/backends/postgres/internal/catalog/resource_inventory_test.go @@ -0,0 +1,751 @@ +package postgres + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "testing" + "time" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +var pgcat02CatalogNamePattern = regexp.MustCompile(`^pgcat02_it_[0-9]+_[0-9a-f]+$`) + +type catalogInventoryService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func catalogTestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 integration tests") + } + return raw +} + +func catalogRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) +} + +func newCatalogInventoryService(t *testing.T) *catalogInventoryService { + t.Helper() + raw := catalogTestPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var dbName, schema string + + t.Cleanup(func() { + var dbCount int + if err := admin.QueryRow(context.WithoutCancel(ctx), + "SELECT COUNT(*) FROM pg_database WHERE datname = $1", dbName).Scan(&dbCount); err != nil { + t.Errorf("audit isolated database %s: %s", dbName, scrub(err)) + } else if dbCount != 0 { + t.Errorf("leftover isolated database %s after cleanup", dbName) + } + var schemaCount int + if err := admin.QueryRow(context.WithoutCancel(ctx), + "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1", schema).Scan(&schemaCount); err != nil { + t.Errorf("audit owned schema %s: %s", schema, scrub(err)) + } else if schemaCount != 0 { + t.Errorf("leftover owned schema %s in the service database after cleanup", schema) + } + }) + + dbName = fmt.Sprintf("pgcat02_it_%d_%s", time.Now().UnixNano(), catalogRandomHex(t)) + if !pgcat02CatalogNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema = fmt.Sprintf("pgcat02_it_%d_%s", time.Now().UnixNano(), catalogRandomHex(t)) + if !pgcat02CatalogNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + service := &catalogInventoryService{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} + if _, err := storepostgres.MigrateSchema(ctx, service.openClient(t), schema); err != nil { + fail("MigrateSchema", err) + } + return service +} + +func (s *catalogInventoryService) openClient(t *testing.T) *storepostgres.Client { + t.Helper() + client, err := storepostgres.Open(s.ctx, storepostgres.Config{ + DSN: s.dsn, + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *catalogInventoryService) realRepository(t *testing.T) *storepostgres.InventoryRepository { + t.Helper() + repo, err := storepostgres.NewInventoryRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new inventory repository", err) + } + return repo +} + +func (s *catalogInventoryService) exec(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := s.iso.Exec(s.ctx, sql, args...); err != nil { + s.fail("fixture exec", err) + } +} + +func (s *catalogInventoryService) seedResource(t *testing.T, project, generation, resourceType, logicalID, authPath string) { + t.Helper() + s.exec(t, "INSERT INTO "+pgx.Identifier{s.schema, "fhir_resource"}.Sanitize()+ + " (project, dataset_generation, resource_type, logical_id, auth_resource_path)"+ + " VALUES ($1, $2, $3, $4, $5)", + project, generation, resourceType, logicalID, authPath) +} + +type catalogCountWitness struct { + CountKey string + BuildID string + ResourceType string + AuthResourcePath string + DocumentCount int64 +} + +func (s *catalogInventoryService) witnessCountRows(t *testing.T, buildID string) []catalogCountWitness { + t.Helper() + rows, err := s.iso.Query(s.ctx, "SELECT count_key, build_id, resource_type, auth_resource_path, document_count"+ + " FROM "+pgx.Identifier{s.schema, "fhir_resource_inventory_count"}.Sanitize()+ + " WHERE build_id = $1 ORDER BY resource_type, auth_resource_path", buildID) + if err != nil { + s.fail("witness count rows", err) + } + defer rows.Close() + var out []catalogCountWitness + for rows.Next() { + var w catalogCountWitness + if err := rows.Scan(&w.CountKey, &w.BuildID, &w.ResourceType, &w.AuthResourcePath, &w.DocumentCount); err != nil { + s.fail("witness count rows", err) + } + out = append(out, w) + } + if err := rows.Err(); err != nil { + s.fail("witness count rows", err) + } + return out +} + +type catalogMarkerWitness struct { + MarkerKey string + BuildID string + Complete bool + CountRowCount int +} + +func (s *catalogInventoryService) witnessMarkers(t *testing.T, project, generation string) []catalogMarkerWitness { + t.Helper() + rows, err := s.iso.Query(s.ctx, "SELECT marker_key, build_id, complete, count_row_count"+ + " FROM "+pgx.Identifier{s.schema, "fhir_resource_inventory_build"}.Sanitize()+ + " WHERE project = $1 AND dataset_generation = $2", project, generation) + if err != nil { + s.fail("witness markers", err) + } + defer rows.Close() + var out []catalogMarkerWitness + for rows.Next() { + var w catalogMarkerWitness + if err := rows.Scan(&w.MarkerKey, &w.BuildID, &w.Complete, &w.CountRowCount); err != nil { + s.fail("witness markers", err) + } + out = append(out, w) + } + if err := rows.Err(); err != nil { + s.fail("witness markers", err) + } + return out +} + +func (s *catalogInventoryService) totals(result catalog.ResourceInventoryResult) map[string]int64 { + got := map[string]int64{} + for _, value := range result.Values { + got[value.ResourceType] = value.DocumentCount + } + return got +} + +type faultedInventory struct { + *storepostgres.InventoryRepository + calls catalogInventoryCalls + + failOnInsert int + splitBeforeFail bool + injected error + + failOnPublish int +} + +type catalogInventoryCalls struct { + scan int + insert int + publish int + readBuild int + readSums int +} + +func (f *faultedInventory) ScanSourceCounts(ctx context.Context, project, datasetGeneration, resourceType string) ([]storepostgres.InventorySourceGroup, error) { + f.calls.scan++ + return f.InventoryRepository.ScanSourceCounts(ctx, project, datasetGeneration, resourceType) +} + +func (f *faultedInventory) InsertCounts(ctx context.Context, rows []catalog.ResourceInventoryCount, batchSize int) error { + f.calls.insert++ + if f.failOnInsert > 0 && f.calls.insert == f.failOnInsert { + if f.splitBeforeFail && len(rows) > 1 { + if err := f.InventoryRepository.InsertCounts(ctx, rows[:1], batchSize); err != nil { + return err + } + } + return f.injected + } + return f.InventoryRepository.InsertCounts(ctx, rows, batchSize) +} + +func (f *faultedInventory) PublishBuild(ctx context.Context, build catalog.ResourceInventoryBuild) error { + f.calls.publish++ + if f.failOnPublish > 0 && f.calls.publish == f.failOnPublish { + return f.injected + } + return f.InventoryRepository.PublishBuild(ctx, build) +} + +func (f *faultedInventory) ReadBuild(ctx context.Context, project, datasetGeneration string) (catalog.ResourceInventoryBuild, bool, error) { + f.calls.readBuild++ + return f.InventoryRepository.ReadBuild(ctx, project, datasetGeneration) +} + +func (f *faultedInventory) ReadCountSums(ctx context.Context, scope storepostgres.InventoryCountScope) (map[string]int64, error) { + f.calls.readSums++ + return f.InventoryRepository.ReadCountSums(ctx, scope) +} + +func requireUnavailable(t *testing.T, result catalog.ResourceInventoryResult, err error, wantReasonPart string) { + t.Helper() + if err == nil || !errors.Is(err, ErrResourceInventoryBuildUnavailable) { + t.Fatalf("error = %v, want errors.Is ErrResourceInventoryBuildUnavailable", err) + } + var typed *ResourceInventoryUnavailableError + if !errors.As(err, &typed) { + t.Fatalf("error %v is not a typed *ResourceInventoryUnavailableError", err) + } + if !strings.Contains(typed.Reason, wantReasonPart) { + t.Fatalf("unavailable reason = %q, want it to contain %q", typed.Reason, wantReasonPart) + } + if result.Available || result.Complete || result.Status != catalog.EvidenceUnavailable || len(result.Values) != 0 { + t.Fatalf("unavailable result = %#v", result) + } +} + +func (s *catalogInventoryService) literalFixture(t *testing.T, project, generation string) { + t.Helper() + s.seedResource(t, project, generation, "Patient", "p1", "a") + s.seedResource(t, project, generation, "Patient", "p2", "a") + s.seedResource(t, project, generation, "Patient", "p3", "b") + s.seedResource(t, project, generation, "Patient", "p4", "b") + s.seedResource(t, project, generation, "Patient", "p5", "b") + s.seedResource(t, project, generation, "Patient", "p6", "b") + s.seedResource(t, project, generation, "Patient", "p7", "b") + s.seedResource(t, project, generation, "Practitioner", "r1", "") +} + +func inventoryRebuildOptions(project, generation string, types ...string) catalog.ResourceInventoryRebuildOptions { + return catalog.ResourceInventoryRebuildOptions{ + Project: project, DatasetGeneration: generation, ResourceTypes: types, BatchSize: 2, + } +} + +func inventoryReadOptions(project, generation string, unrestricted bool, paths []string, types ...string) catalog.ResourceInventoryOptions { + flag := unrestricted + return catalog.ResourceInventoryOptions{ + Project: project, DatasetGeneration: generation, ResourceTypes: types, + AuthResourcePathsUnrestricted: &flag, AuthResourcePaths: paths, + } +} + +func TestResourceInventoryRebuildGroupsSourceRowsAndPublishesMarkerLast(t *testing.T) { + s := newCatalogInventoryService(t) + faulted := &faultedInventory{InventoryRepository: s.realRepository(t)} + adapter, err := NewInventoryAdapter(faulted) + if err != nil { + t.Fatal(err) + } + s.literalFixture(t, "p", "g") + + summary, err := adapter.RebuildResourceInventory(s.ctx, inventoryRebuildOptions("p", "g", "Observation", "Patient", "Practitioner")) + if err != nil { + t.Fatalf("RebuildResourceInventory: %v", err) + } + if summary.BuildID == "" || summary.CoverageDigest == "" || + summary.ResourceTypeCount != 3 || summary.CountRowCount != 4 || summary.DocumentCount != 8 { + t.Fatalf("summary = %#v", summary) + } + + if faulted.calls.scan != 3 { + t.Fatalf("source scans = %d, want one per declared type", faulted.calls.scan) + } + rows := s.witnessCountRows(t, summary.BuildID) + want := []catalogCountWitness{ + {BuildID: summary.BuildID, ResourceType: "Observation", AuthResourcePath: "", DocumentCount: 0}, + {BuildID: summary.BuildID, ResourceType: "Patient", AuthResourcePath: "a", DocumentCount: 2}, + {BuildID: summary.BuildID, ResourceType: "Patient", AuthResourcePath: "b", DocumentCount: 5}, + {BuildID: summary.BuildID, ResourceType: "Practitioner", AuthResourcePath: "", DocumentCount: 1}, + } + if len(rows) != len(want) { + t.Fatalf("committed rows = %v, want %d rows", rows, len(want)) + } + for i := range want { + if rows[i].BuildID != want[i].BuildID || rows[i].ResourceType != want[i].ResourceType || + rows[i].AuthResourcePath != want[i].AuthResourcePath || rows[i].DocumentCount != want[i].DocumentCount { + t.Fatalf("row %d = %v, want %v", i, rows[i], want[i]) + } + if !strings.HasPrefix(rows[i].CountKey, "ric_") { + t.Fatalf("row %d count key %q is not a ric_ identity", i, rows[i].CountKey) + } + } + + markers := s.witnessMarkers(t, "p", "g") + if len(markers) != 1 { + t.Fatalf("markers = %v, want one", markers) + } + if markers[0].BuildID != summary.BuildID || !markers[0].Complete || markers[0].CountRowCount != 4 { + t.Fatalf("marker = %+v", markers[0]) + } + if markers[0].MarkerKey != catalog.ResourceInventoryBuildKey("p", "g") { + t.Fatalf("marker key = %q, want %q", markers[0].MarkerKey, catalog.ResourceInventoryBuildKey("p", "g")) + } + + result, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", true, nil, "Observation", "Patient", "Practitioner")) + if err != nil || result.Status != catalog.EvidenceAvailable { + t.Fatalf("inventory = %#v err=%v", result, err) + } + if fmt.Sprint(s.totals(result)) != fmt.Sprint(map[string]int64{"Practitioner": 1, "Observation": 0, "Patient": 7}) { + t.Fatalf("totals = %v", s.totals(result)) + } +} + +func TestResourceInventoryCountWriteFailureKeepsPriorReadAndExposesTypedErrors(t *testing.T) { + s := newCatalogInventoryService(t) + faulted := &faultedInventory{InventoryRepository: s.realRepository(t)} + adapter, err := NewInventoryAdapter(faulted) + if err != nil { + t.Fatal(err) + } + s.literalFixture(t, "p", "g") + + first, err := adapter.RebuildResourceInventory(s.ctx, inventoryRebuildOptions("p", "g", "Observation", "Patient", "Practitioner")) + if err != nil { + t.Fatalf("first rebuild: %v", err) + } + before, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", true, nil, "Observation", "Patient", "Practitioner")) + if err != nil { + t.Fatalf("first read: %v", err) + } + beforeDigest := before.Digest + + s.seedResource(t, "p", "g", "Patient", "p8", "a") + injected := errors.New("injected count-write failure") + faulted.failOnInsert = 2 + faulted.splitBeforeFail = true + faulted.injected = injected + + _, err = adapter.RebuildResourceInventory(s.ctx, inventoryRebuildOptions("p", "g", "Observation", "Patient", "Practitioner")) + var writeErr *ResourceInventoryWriteError + if err == nil || !errors.As(err, &writeErr) || !errors.Is(err, ErrResourceInventoryWrite) { + t.Fatalf("failed rebuild error = %v, want *ResourceInventoryWriteError", err) + } + if writeErr.Op != "counts" || !errors.Is(err, injected) { + t.Fatalf("write error op = %q or injected cause missing: %v", writeErr.Op, err) + } + if !strings.Contains(err.Error(), "not published") { + t.Fatalf("write error is not actionable: %v", err) + } + + failedBuild, _, err := catalog.NewResourceInventoryBuild("p", "g", + []string{"Observation", "Patient", "Practitioner"}, + []catalog.ResourceInventoryCount{ + {Project: "p", DatasetGeneration: "g", AuthResourcePath: "", ResourceType: "Observation", DocumentCount: 0}, + {Project: "p", DatasetGeneration: "g", AuthResourcePath: "a", ResourceType: "Patient", DocumentCount: 3}, + {Project: "p", DatasetGeneration: "g", AuthResourcePath: "b", ResourceType: "Patient", DocumentCount: 5}, + {Project: "p", DatasetGeneration: "g", AuthResourcePath: "", ResourceType: "Practitioner", DocumentCount: 1}, + }) + if err != nil { + t.Fatalf("derive failed build identity: %v", err) + } + if rows := s.witnessCountRows(t, failedBuild.BuildID); len(rows) != 1 || rows[0].ResourceType != "Observation" || rows[0].DocumentCount != 0 { + t.Fatalf("partial rows of the failed build = %v, want the committed first row", rows) + } + + after := s.witnessMarkers(t, "p", "g") + if len(after) != 1 || after[0].BuildID != first.BuildID || !after[0].Complete { + t.Fatalf("marker after failure = %+v, want the first build intact", after) + } + intact, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", true, nil, "Observation", "Patient", "Practitioner")) + if err != nil || intact.Status != catalog.EvidenceAvailable || intact.Digest != beforeDigest { + t.Fatalf("read after failure = %#v err=%v, want the prior result", intact, err) + } + if fmt.Sprint(s.totals(intact)) != fmt.Sprint(map[string]int64{"Practitioner": 1, "Observation": 0, "Patient": 7}) { + t.Fatalf("totals after failure = %v, want the prior totals", s.totals(intact)) + } + faulted.calls = catalogInventoryCalls{} + if _, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", true, nil, "Patient")); err != nil { + t.Fatalf("read: %v", err) + } + if faulted.calls.readSums != 1 { + t.Fatalf("count reads = %d, want exactly one (marker-bound)", faulted.calls.readSums) + } + + s.seedResource(t, "p", "g2", "Patient", "q1", "z") + faulted.failOnInsert = faulted.calls.insert + 1 + faulted.splitBeforeFail = false + _, err = adapter.RebuildResourceInventory(s.ctx, inventoryRebuildOptions("p", "g2", "Patient")) + if err == nil || !errors.Is(err, ErrResourceInventoryWrite) || !errors.Is(err, injected) { + t.Fatalf("fresh rebuild error = %v, want injected write failure", err) + } + result, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g2", true, nil, "Patient")) + requireUnavailable(t, result, err, "marker is missing") + if markers := s.witnessMarkers(t, "p", "g2"); len(markers) != 0 { + t.Fatalf("markers after failed first rebuild = %+v, want none", markers) + } + expected, _, err := catalog.NewResourceInventoryBuild("p", "g2", []string{"Patient"}, + []catalog.ResourceInventoryCount{{Project: "p", DatasetGeneration: "g2", AuthResourcePath: "z", ResourceType: "Patient", DocumentCount: 1}}) + if err != nil { + t.Fatalf("derive expected build: %v", err) + } + if rows := s.witnessCountRows(t, expected.BuildID); len(rows) != 0 { + t.Fatalf("count rows committed despite a fully failed count write: %v", rows) + } +} + +func TestResourceInventoryMarkerPublishFailureKeepsPriorMarkerReadable(t *testing.T) { + s := newCatalogInventoryService(t) + faulted := &faultedInventory{InventoryRepository: s.realRepository(t)} + adapter, err := NewInventoryAdapter(faulted) + if err != nil { + t.Fatal(err) + } + s.literalFixture(t, "p", "g") + + first, err := adapter.RebuildResourceInventory(s.ctx, inventoryRebuildOptions("p", "g", "Observation", "Patient", "Practitioner")) + if err != nil { + t.Fatalf("first rebuild: %v", err) + } + + s.seedResource(t, "p", "g", "Observation", "o1", "a") + injected := errors.New("injected marker-publish failure") + faulted.failOnPublish = 2 + faulted.injected = injected + _, err = adapter.RebuildResourceInventory(s.ctx, inventoryRebuildOptions("p", "g", "Observation", "Patient", "Practitioner")) + var writeErr *ResourceInventoryWriteError + if err == nil || !errors.As(err, &writeErr) || writeErr.Op != "marker" || !errors.Is(err, injected) { + t.Fatalf("publish-failure error = %v, want marker write failure", err) + } + + if markers := s.witnessMarkers(t, "p", "g"); len(markers) != 1 || markers[0].BuildID != first.BuildID { + t.Fatalf("markers after publish failure = %+v, want only the first marker", markers) + } + result, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", true, nil, "Observation", "Patient", "Practitioner")) + if err != nil || fmt.Sprint(s.totals(result)) != fmt.Sprint(map[string]int64{"Practitioner": 1, "Observation": 0, "Patient": 7}) { + t.Fatalf("read after publish failure = %v err=%v, want prior totals", s.totals(result), err) + } + + faulted.failOnPublish = 0 + third, err := adapter.RebuildResourceInventory(s.ctx, inventoryRebuildOptions("p", "g", "Observation", "Patient", "Practitioner")) + if err != nil { + t.Fatalf("retry rebuild: %v", err) + } + markers := s.witnessMarkers(t, "p", "g") + if len(markers) != 1 || markers[0].BuildID != third.BuildID { + t.Fatalf("markers after retry = %+v, want only the third build", markers) + } + result, err = adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", true, nil, "Observation", "Patient", "Practitioner")) + if err != nil || fmt.Sprint(s.totals(result)) != fmt.Sprint(map[string]int64{"Practitioner": 1, "Observation": 1, "Patient": 7}) { + t.Fatalf("read after retry = %v err=%v", s.totals(result), err) + } +} + +func TestResourceInventoryRepeatedBuildIsStableAndIsolatesOtherScopes(t *testing.T) { + s := newCatalogInventoryService(t) + faulted := &faultedInventory{InventoryRepository: s.realRepository(t)} + adapter, err := NewInventoryAdapter(faulted) + if err != nil { + t.Fatal(err) + } + s.literalFixture(t, "p", "g") + + types := []string{"Observation", "Patient", "Practitioner"} + first, err := adapter.RebuildResourceInventory(s.ctx, inventoryRebuildOptions("p", "g", types...)) + if err != nil { + t.Fatalf("first rebuild: %v", err) + } + second, err := adapter.RebuildResourceInventory(s.ctx, inventoryRebuildOptions("p", "g", types...)) + if err != nil { + t.Fatalf("second rebuild: %v", err) + } + if first.BuildID != second.BuildID || first.CoverageDigest != second.CoverageDigest || + first.CountRowCount != second.CountRowCount || first.DocumentCount != second.DocumentCount { + t.Fatalf("repeated build identity changed: %#v vs %#v", first, second) + } + if rows := s.witnessCountRows(t, first.BuildID); len(rows) != 4 { + t.Fatalf("rows after repeat = %d, want 4", len(rows)) + } + if markers := s.witnessMarkers(t, "p", "g"); len(markers) != 1 { + t.Fatalf("markers after repeat = %+v, want one", markers) + } + + s.seedResource(t, "p", "g2", "Patient", "q1", "z") + s.seedResource(t, "p", "g2", "Patient", "q2", "z") + s.seedResource(t, "p", "g2", "Patient", "q3", "y") + other, err := adapter.RebuildResourceInventory(s.ctx, inventoryRebuildOptions("p", "g2", "Patient")) + if err != nil { + t.Fatalf("g2 rebuild: %v", err) + } + if other.BuildID == first.BuildID { + t.Fatal("different generation produced the same build id") + } + + g, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", true, nil, types...)) + if err != nil || fmt.Sprint(s.totals(g)) != fmt.Sprint(map[string]int64{"Practitioner": 1, "Observation": 0, "Patient": 7}) { + t.Fatalf("g read after sibling rebuild = %v err=%v", s.totals(g), err) + } + gPathZ, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", false, []string{"z"}, "Patient")) + if err != nil || fmt.Sprint(s.totals(gPathZ)) != fmt.Sprint(map[string]int64{"Patient": 0}) { + t.Fatalf("g restricted [z] = %v err=%v, want Patient=0", s.totals(gPathZ), err) + } + g2PathA, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g2", false, []string{"a"}, "Patient")) + if err != nil || fmt.Sprint(s.totals(g2PathA)) != fmt.Sprint(map[string]int64{"Patient": 0}) { + t.Fatalf("g2 restricted [a] = %v err=%v, want Patient=0", s.totals(g2PathA), err) + } + g2, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g2", true, nil, "Patient")) + if err != nil || fmt.Sprint(s.totals(g2)) != fmt.Sprint(map[string]int64{"Patient": 3}) { + t.Fatalf("g2 read = %v err=%v", s.totals(g2), err) + } +} + +func TestResourceInventoryReadSumsOnlyAuthorizedPaths(t *testing.T) { + s := newCatalogInventoryService(t) + faulted := &faultedInventory{InventoryRepository: s.realRepository(t)} + adapter, err := NewInventoryAdapter(faulted) + if err != nil { + t.Fatal(err) + } + s.literalFixture(t, "p", "g") + summary, err := adapter.RebuildResourceInventory(s.ctx, inventoryRebuildOptions("p", "g", "Observation", "Patient", "Practitioner")) + if err != nil { + t.Fatalf("rebuild: %v", err) + } + + s.exec(t, "INSERT INTO "+pgx.Identifier{s.schema, "fhir_resource_inventory_count"}.Sanitize()+ + " (count_key, build_id, project, dataset_generation, resource_type, auth_resource_path, document_count)"+ + " VALUES ($1, $2, $3, $4, $5, $6, $7)", + "ric_forged_unmarked", "sha256:"+strings.Repeat("a", 64), "p", "g", "Patient", "a", int64(999999)) + + restrictedA, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", false, []string{"a"}, "Observation", "Patient", "Practitioner")) + if err != nil || restrictedA.Status != catalog.EvidenceAvailable { + t.Fatalf("restricted [a] = %#v err=%v", restrictedA, err) + } + if fmt.Sprint(s.totals(restrictedA)) != fmt.Sprint(map[string]int64{"Practitioner": 0, "Observation": 0, "Patient": 2}) { + t.Fatalf("restricted [a] totals = %v, want only path-a evidence (Patient 2, unscoped rows excluded)", s.totals(restrictedA)) + } + restrictedB, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", false, []string{"b"}, "Patient")) + if err != nil || fmt.Sprint(s.totals(restrictedB)) != fmt.Sprint(map[string]int64{"Patient": 5}) { + t.Fatalf("restricted [b] totals = %v err=%v, want Patient 5", s.totals(restrictedB), err) + } + restrictedBoth, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", false, []string{"a", "b"}, "Patient")) + if err != nil || fmt.Sprint(s.totals(restrictedBoth)) != fmt.Sprint(map[string]int64{"Patient": 7}) { + t.Fatalf("restricted [a b] totals = %v err=%v, want Patient 7", s.totals(restrictedBoth), err) + } + unrestricted, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", true, nil, "Observation", "Patient", "Practitioner")) + if err != nil || fmt.Sprint(s.totals(unrestricted)) != fmt.Sprint(map[string]int64{"Practitioner": 1, "Observation": 0, "Patient": 7}) { + t.Fatalf("unrestricted totals = %v err=%v", s.totals(unrestricted), err) + } + unrestrictedAgain, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", true, nil, "Observation", "Patient", "Practitioner")) + if err != nil || unrestrictedAgain.Digest != unrestricted.Digest || unrestricted.Digest == "" { + t.Fatalf("digest unstable: %q vs %q err=%v", unrestricted.Digest, unrestrictedAgain.Digest, err) + } + + retry, err := adapter.RebuildResourceInventory(s.ctx, inventoryRebuildOptions("p", "g", "Observation", "Patient", "Practitioner")) + if err != nil { + t.Fatalf("rebuild over forged scope: %v", err) + } + if retry.BuildID != summary.BuildID { + t.Fatalf("rebuild identity drifted: %s vs %s", retry.BuildID, summary.BuildID) + } + after, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", true, nil, "Patient")) + if err != nil || fmt.Sprint(s.totals(after)) != fmt.Sprint(map[string]int64{"Patient": 7}) { + t.Fatalf("read after forged-scope rebuild = %v err=%v", s.totals(after), err) + } +} + +func TestResourceInventoryRestrictedEmptyScopeReturnsNoRows(t *testing.T) { + s := newCatalogInventoryService(t) + faulted := &faultedInventory{InventoryRepository: s.realRepository(t)} + adapter, err := NewInventoryAdapter(faulted) + if err != nil { + t.Fatal(err) + } + s.literalFixture(t, "p", "g") + if _, err := adapter.RebuildResourceInventory(s.ctx, inventoryRebuildOptions("p", "g", "Patient")); err != nil { + t.Fatalf("rebuild: %v", err) + } + + faulted.calls = catalogInventoryCalls{} + result, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", false, []string{}, "Patient")) + if err != nil { + t.Fatalf("restricted empty scope: %v", err) + } + if !result.Available || !result.Complete || result.Status != catalog.EvidenceEmpty || len(result.Values) != 0 { + t.Fatalf("restricted empty result = %#v", result) + } + if faulted.calls.readSums != 0 || faulted.calls.readBuild != 1 { + t.Fatalf("restricted empty scope queried counts: %+v", faulted.calls) + } +} + +func TestResourceInventoryStaleMarkerIsUnavailable(t *testing.T) { + s := newCatalogInventoryService(t) + faulted := &faultedInventory{InventoryRepository: s.realRepository(t)} + adapter, err := NewInventoryAdapter(faulted) + if err != nil { + t.Fatal(err) + } + + faulted.calls = catalogInventoryCalls{} + result, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", true, nil, "Patient")) + requireUnavailable(t, result, err, "marker is missing") + if faulted.calls.readSums != 0 { + t.Fatalf("missing marker queried counts: %+v", faulted.calls) + } + + s.literalFixture(t, "p", "g") + summary, err := adapter.RebuildResourceInventory(s.ctx, inventoryRebuildOptions("p", "g", "Observation", "Patient")) + if err != nil { + t.Fatalf("rebuild: %v", err) + } + + s.exec(t, "UPDATE "+pgx.Identifier{s.schema, "fhir_resource_inventory_build"}.Sanitize()+ + " SET complete = false WHERE project = $1 AND dataset_generation = $2", "p", "g") + faulted.calls = catalogInventoryCalls{} + result, err = adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", true, nil, "Patient")) + requireUnavailable(t, result, err, "stale or incomplete marker") + if faulted.calls.readSums != 0 { + t.Fatalf("incomplete marker queried counts: %+v", faulted.calls) + } + if rows := s.witnessCountRows(t, summary.BuildID); len(rows) != 3 { + t.Fatalf("count rows vanished under an incomplete marker: %v", rows) + } + + s.exec(t, "UPDATE "+pgx.Identifier{s.schema, "fhir_resource_inventory_build"}.Sanitize()+ + " SET complete = true, coverage_digest = $3 WHERE project = $1 AND dataset_generation = $2", + "p", "g", "sha256:"+strings.Repeat("0", 64)) + result, err = adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", true, nil, "Patient")) + requireUnavailable(t, result, err, "stale or incomplete marker") + + s.exec(t, "UPDATE "+pgx.Identifier{s.schema, "fhir_resource_inventory_build"}.Sanitize()+ + " SET complete = true, coverage_digest = $3 WHERE project = $1 AND dataset_generation = $2", + "p", "g", summary.CoverageDigest) + result, err = adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", true, nil, "Practitioner")) + requireUnavailable(t, result, err, "does not cover Practitioner") + ok, err := adapter.DiscoverResourceInventory(s.ctx, inventoryReadOptions("p", "g", true, nil, "Patient")) + if err != nil || ok.Status != catalog.EvidenceAvailable { + t.Fatalf("covered-type read after restore = %#v err=%v", ok, err) + } +} diff --git a/internal/backends/postgres/internal/catalog/semantic_build.go b/internal/backends/postgres/internal/catalog/semantic_build.go new file mode 100644 index 000000000..fd2d5e3da --- /dev/null +++ b/internal/backends/postgres/internal/catalog/semantic_build.go @@ -0,0 +1,434 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + "unicode/utf8" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + semanticInventoryBuildTableName = "fhir_semantic_inventory_build" + maxSemanticInventoryDiagnosticBytes = 512 + defaultSemanticInventoryDiagnostic = "backfill failed" +) + +var ( + ErrSemanticInventoryClaimed = errors.New("postgres: semantic inventory build is already claimed") + ErrSemanticInventoryClaimLost = errors.New("postgres: semantic inventory build claim was lost") + ErrUnsupportedSemanticInventoryBuild = errors.New("postgres: unsupported semantic inventory build request") + ErrInvalidSemanticInventoryBuild = errors.New("postgres: invalid semantic inventory build") +) + +type SemanticInventoryUnsupportedError struct { + Case string + Detail string +} + +func (e *SemanticInventoryUnsupportedError) Error() string { + return fmt.Sprintf("%v: %s: %s", ErrUnsupportedSemanticInventoryBuild, e.Case, e.Detail) +} + +func (e *SemanticInventoryUnsupportedError) Unwrap() error { + return ErrUnsupportedSemanticInventoryBuild +} + +type SemanticInventoryClaimedError struct { + Project string + DatasetGeneration string + BuildID string + State catalog.SemanticInventoryState +} + +func (e *SemanticInventoryClaimedError) Error() string { + return fmt.Sprintf("%v: %s/%s build %s is in state %q under a live lease; "+ + "the lease must expire or be released before a new claimant can take it", + ErrSemanticInventoryClaimed, e.Project, e.DatasetGeneration, e.BuildID, e.State) +} + +func (e *SemanticInventoryClaimedError) Unwrap() error { return ErrSemanticInventoryClaimed } + +type SemanticInventoryClaimLostError struct { + Project string + DatasetGeneration string + BuildID string + Reason string +} + +func (e *SemanticInventoryClaimLostError) Error() string { + return fmt.Sprintf("%v: %s/%s build %s: %s", + ErrSemanticInventoryClaimLost, e.Project, e.DatasetGeneration, e.BuildID, e.Reason) +} + +func (e *SemanticInventoryClaimLostError) Unwrap() error { return ErrSemanticInventoryClaimLost } + +type SemanticBuildRepository struct { + src storepostgres.ConnSource + buildFQN string +} + +func NewSemanticBuildRepository(src storepostgres.ConnSource, schema string) (*SemanticBuildRepository, error) { + if src == nil { + return nil, errors.New("postgres: nil semantic build connection source") + } + if !storepostgres.ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", storepostgres.ErrInvalidSchemaName, schema) + } + return &SemanticBuildRepository{ + src: src, + buildFQN: pgx.Identifier{schema, semanticInventoryBuildTableName}.Sanitize(), + }, nil +} + +func (r *SemanticBuildRepository) ClaimSemanticInventoryBackfill(ctx context.Context, build catalog.SemanticInventoryBuild, token string, lease time.Duration) (catalog.SemanticInventoryBuild, bool, error) { + if err := validateSemanticInventoryBuildIdentity(build); err != nil { + return catalog.SemanticInventoryBuild{}, false, err + } + normalizeSemanticBuildRequest(&build) + if build.SourceKind != catalog.SemanticInventorySourceRetained { + return catalog.SemanticInventoryBuild{}, false, &SemanticInventoryUnsupportedError{ + Case: "source_kind", + Detail: fmt.Sprintf("claim requires %q, got %q; the file-sourced profiler path is not persisted by this slice", catalog.SemanticInventorySourceRetained, build.SourceKind), + } + } + if strings.TrimSpace(token) == "" { + return catalog.SemanticInventoryBuild{}, false, &SemanticInventoryUnsupportedError{Case: "claim token", Detail: "blank claim token"} + } + if lease <= 0 { + return catalog.SemanticInventoryBuild{}, false, &SemanticInventoryUnsupportedError{Case: "lease", Detail: fmt.Sprintf("lease must be positive, got %s", lease)} + } + now := time.Now() + sql := "INSERT INTO " + r.buildFQN + + " (build_key, project, dataset_generation, build_id, state, observation_schema," + + " inventory_schema, entry_index_version, rule_version, source_kind, source_availability," + + " auth_resource_path, scanned_resources, checkpoint, checkpoint_collection, checkpoint_key," + + " lease_token, lease_expires_at, diagnostics)" + + " VALUES ($1, $2, $3, $4, 'running', $5, $6, $7, $8, $9, 'unknown'," + + " $10, 0, '', '', '', $11, $12, '{}')" + + " ON CONFLICT (project, dataset_generation, build_id) DO UPDATE SET" + + " state = 'running'," + + " source_kind = EXCLUDED.source_kind," + + " source_availability = CASE WHEN " + semanticInventoryBuildTableName + ".source_kind = EXCLUDED.source_kind" + + " THEN " + semanticInventoryBuildTableName + ".source_availability ELSE 'unknown' END," + + " scanned_resources = CASE WHEN " + semanticInventoryBuildTableName + ".source_kind = EXCLUDED.source_kind" + + " THEN " + semanticInventoryBuildTableName + ".scanned_resources ELSE 0 END," + + " checkpoint = ''," + + " checkpoint_collection = CASE WHEN " + semanticInventoryBuildTableName + ".source_kind = EXCLUDED.source_kind" + + " THEN " + semanticInventoryBuildTableName + ".checkpoint_collection ELSE '' END," + + " checkpoint_key = CASE WHEN " + semanticInventoryBuildTableName + ".source_kind = EXCLUDED.source_kind" + + " THEN " + semanticInventoryBuildTableName + ".checkpoint_key ELSE '' END," + + " diagnostics = '{}'," + + " lease_token = EXCLUDED.lease_token," + + " lease_expires_at = EXCLUDED.lease_expires_at," + + " updated_at = now()" + + " WHERE " + semanticInventoryBuildTableName + ".state <> 'complete'" + + " AND (" + semanticInventoryBuildTableName + ".lease_expires_at <= $13" + + " OR " + semanticInventoryBuildTableName + ".lease_token = $14)" + + " RETURNING " + semanticInventoryBuildColumns + var out catalog.SemanticInventoryBuild + var claimed bool + err := r.withConn(ctx, func(conn *pgxpool.Conn) error { + err := conn.QueryRow(ctx, sql, + build.Key, build.Project, build.DatasetGeneration, build.BuildID, + build.ObservationSchema, build.InventorySchema, build.EntryIndexVersion, build.RuleVersion, + build.SourceKind, build.AuthResourcePath, token, now.Add(lease).UnixMilli(), now.UnixMilli(), token, + ).Scan(scanSemanticInventoryBuild(&out)...) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return classifySemanticBuildWrite("claim", build, err) + } + claimed = true + return nil + }) + if err != nil { + return catalog.SemanticInventoryBuild{}, false, err + } + if claimed { + normalizeReadBuild(&out, build.Project, build.DatasetGeneration, build.BuildID) + return out, true, nil + } + current, found, err := r.ReadSemanticInventoryBuild(ctx, build.Project, build.DatasetGeneration, build.BuildID) + if err != nil { + return catalog.SemanticInventoryBuild{}, false, err + } + if !found { + return catalog.SemanticInventoryBuild{}, false, fmt.Errorf( + "postgres: claim semantic inventory build %s/%s: row vanished between claim and re-read", + build.Project, build.DatasetGeneration) + } + if current.State == catalog.SemanticInventoryComplete { + return current, false, nil + } + return catalog.SemanticInventoryBuild{}, false, &SemanticInventoryClaimedError{ + Project: build.Project, DatasetGeneration: build.DatasetGeneration, + BuildID: build.BuildID, State: current.State, + } +} + +func (r *SemanticBuildRepository) ReadSemanticInventoryBuild(ctx context.Context, project, datasetGeneration, buildID string) (catalog.SemanticInventoryBuild, bool, error) { + if strings.TrimSpace(project) == "" || strings.TrimSpace(buildID) == "" { + return catalog.SemanticInventoryBuild{}, false, &SemanticInventoryUnsupportedError{ + Case: "scope", Detail: "project and build id are required for a scoped read", + } + } + datasetGeneration = catalog.NormalizeDatasetGeneration(datasetGeneration) + sql := "SELECT " + semanticInventoryBuildColumns + " FROM " + r.buildFQN + + " WHERE project = $1 AND dataset_generation = $2 AND build_id = $3" + var out catalog.SemanticInventoryBuild + var found bool + err := r.withConn(ctx, func(conn *pgxpool.Conn) error { + err := conn.QueryRow(ctx, sql, project, datasetGeneration, buildID).Scan(scanSemanticInventoryBuild(&out)...) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return fmt.Errorf("postgres: read semantic inventory build %s/%s: %w", project, datasetGeneration, err) + } + found = true + return nil + }) + if err != nil { + return catalog.SemanticInventoryBuild{}, false, err + } + if !found { + return catalog.SemanticInventoryBuild{}, false, nil + } + normalizeReadBuild(&out, project, datasetGeneration, buildID) + return out, true, nil +} + +func (r *SemanticBuildRepository) AdvanceSemanticInventoryBackfill(ctx context.Context, build catalog.SemanticInventoryBuild, token string, expected, next catalog.SemanticInventoryCheckpoint, scanned int64, lease time.Duration) (catalog.SemanticInventoryBuild, error) { + if err := validateSemanticInventoryTransitionRequest(build, token, lease, true); err != nil { + return catalog.SemanticInventoryBuild{}, err + } + normalizeSemanticBuildRequest(&build) + if err := validateSemanticInventoryCheckpoint(expected, "expected", false); err != nil { + return catalog.SemanticInventoryBuild{}, err + } + if err := validateSemanticInventoryCheckpoint(next, "next", true); err != nil { + return catalog.SemanticInventoryBuild{}, err + } + if err := validateSemanticInventorySourceAvailability(build.SourceAvailability); err != nil { + return catalog.SemanticInventoryBuild{}, err + } + now := time.Now() + sql := "UPDATE " + r.buildFQN + " SET" + + " checkpoint_collection = $8, checkpoint_key = $9," + + " scanned_resources = $10, source_availability = $11," + + " lease_expires_at = $12, diagnostics = '{}', updated_at = now()" + + " WHERE project = $1 AND dataset_generation = $2 AND build_id = $3" + + " AND state = 'running' AND lease_token = $4 AND lease_expires_at > $5" + + " AND checkpoint_collection = $6 AND checkpoint_key = $7" + + " RETURNING " + semanticInventoryBuildColumns + return r.executeGuarded(ctx, "advance", build, sql, token, &expected, + build.Project, build.DatasetGeneration, build.BuildID, token, now.UnixMilli(), + expected.Collection, expected.Key, next.Collection, next.Key, max(scanned, 0), + build.SourceAvailability, now.Add(lease).UnixMilli()) +} + +func (r *SemanticBuildRepository) CompleteSemanticInventoryBackfill(ctx context.Context, build catalog.SemanticInventoryBuild, token string, expected catalog.SemanticInventoryCheckpoint, scanned int64) (catalog.SemanticInventoryBuild, error) { + if err := validateSemanticInventoryTransitionRequest(build, token, 0, false); err != nil { + return catalog.SemanticInventoryBuild{}, err + } + normalizeSemanticBuildRequest(&build) + if err := validateSemanticInventoryCheckpoint(expected, "expected", false); err != nil { + return catalog.SemanticInventoryBuild{}, err + } + if err := validateSemanticInventorySourceAvailability(build.SourceAvailability); err != nil { + return catalog.SemanticInventoryBuild{}, err + } + now := time.Now() + sql := "UPDATE " + r.buildFQN + " SET" + + " state = 'complete', scanned_resources = $8, source_availability = $9," + + " lease_token = '', lease_expires_at = 0, diagnostics = '{}', updated_at = now()" + + " WHERE project = $1 AND dataset_generation = $2 AND build_id = $3" + + " AND state = 'running' AND lease_token = $4 AND lease_expires_at > $5" + + " AND checkpoint_collection = $6 AND checkpoint_key = $7" + + " RETURNING " + semanticInventoryBuildColumns + return r.executeGuarded(ctx, "complete", build, sql, token, &expected, + build.Project, build.DatasetGeneration, build.BuildID, token, now.UnixMilli(), + expected.Collection, expected.Key, max(scanned, 0), build.SourceAvailability) +} + +func (r *SemanticBuildRepository) FailSemanticInventoryBackfill(ctx context.Context, build catalog.SemanticInventoryBuild, token, diagnostic string) error { + if err := validateSemanticInventoryTransitionRequest(build, token, 0, false); err != nil { + return err + } + normalizeSemanticBuildRequest(&build) + if text := strings.TrimSpace(diagnostic); text != "" { + if len(text) > maxSemanticInventoryDiagnosticBytes { + cut := []byte(text[:maxSemanticInventoryDiagnosticBytes]) + for len(cut) > 0 && !utf8.Valid(cut) { + cut = cut[:len(cut)-1] + } + text = string(cut) + } + diagnostic = text + } else { + diagnostic = defaultSemanticInventoryDiagnostic + } + sql := "UPDATE " + r.buildFQN + " SET" + + " state = 'failed', lease_token = '', lease_expires_at = 0," + + " diagnostics = $5::text[], updated_at = now()" + + " WHERE project = $1 AND dataset_generation = $2 AND build_id = $3" + + " AND state = 'running' AND lease_token = $4" + + " RETURNING " + semanticInventoryBuildColumns + _, err := r.executeGuarded(ctx, "fail", build, sql, token, nil, + build.Project, build.DatasetGeneration, build.BuildID, token, []string{diagnostic}) + return err +} + +func (r *SemanticBuildRepository) executeGuarded(ctx context.Context, op string, build catalog.SemanticInventoryBuild, sql, token string, expected *catalog.SemanticInventoryCheckpoint, args ...any) (catalog.SemanticInventoryBuild, error) { + var out catalog.SemanticInventoryBuild + matched := false + err := r.withConn(ctx, func(conn *pgxpool.Conn) error { + err := conn.QueryRow(ctx, sql, args...).Scan(scanSemanticInventoryBuild(&out)...) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return classifySemanticBuildWrite(op, build, err) + } + matched = true + return nil + }) + if err != nil { + return catalog.SemanticInventoryBuild{}, err + } + if matched { + normalizeReadBuild(&out, build.Project, build.DatasetGeneration, build.BuildID) + return out, nil + } + current, found, readErr := r.ReadSemanticInventoryBuild(ctx, build.Project, build.DatasetGeneration, build.BuildID) + if readErr != nil { + return catalog.SemanticInventoryBuild{}, readErr + } + if !found { + return catalog.SemanticInventoryBuild{}, &SemanticInventoryClaimLostError{ + Project: build.Project, DatasetGeneration: build.DatasetGeneration, + BuildID: build.BuildID, Reason: "row is missing", + } + } + return catalog.SemanticInventoryBuild{}, &SemanticInventoryClaimLostError{ + Project: build.Project, DatasetGeneration: build.DatasetGeneration, + BuildID: build.BuildID, Reason: semanticBuildGuardMissReason(current, token, expected), + } +} + +func semanticBuildGuardMissReason(current catalog.SemanticInventoryBuild, token string, expected *catalog.SemanticInventoryCheckpoint) string { + if current.State != catalog.SemanticInventoryRunning { + return fmt.Sprintf("state is %q, not running", current.State) + } + if current.LeaseToken != token { + return fmt.Sprintf("lease token is held by another claimant (expiry %d)", current.LeaseExpiresAt) + } + if current.LeaseExpiresAt <= time.Now().UnixMilli() { + return fmt.Sprintf("lease expired at %d", current.LeaseExpiresAt) + } + if expected != nil && (current.SourceCheckpoint.Collection != expected.Collection || current.SourceCheckpoint.Key != expected.Key) { + return fmt.Sprintf("checkpoint is %s/%s, expected %s/%s", + current.SourceCheckpoint.Collection, current.SourceCheckpoint.Key, expected.Collection, expected.Key) + } + return "row changed concurrently; re-read matches every guard" +} + +const semanticInventoryBuildColumns = "build_key, state, observation_schema, inventory_schema," + + " entry_index_version, rule_version, source_kind, source_availability, auth_resource_path," + + " scanned_resources, checkpoint, checkpoint_collection, checkpoint_key, lease_token," + + " lease_expires_at, diagnostics" + +func scanSemanticInventoryBuild(out *catalog.SemanticInventoryBuild) []any { + return []any{ + &out.Key, &out.State, &out.ObservationSchema, &out.InventorySchema, + &out.EntryIndexVersion, &out.RuleVersion, &out.SourceKind, &out.SourceAvailability, + &out.AuthResourcePath, &out.ScannedResources, &out.Checkpoint, + &out.SourceCheckpoint.Collection, &out.SourceCheckpoint.Key, &out.LeaseToken, + &out.LeaseExpiresAt, &out.Diagnostics, + } +} + +func normalizeReadBuild(out *catalog.SemanticInventoryBuild, project, datasetGeneration, buildID string) { + out.Project = project + out.DatasetGeneration = datasetGeneration + out.BuildID = buildID +} + +func (r *SemanticBuildRepository) withConn(ctx context.Context, fn func(*pgxpool.Conn) error) error { + conn, err := r.src.Acquire(ctx) + if err != nil { + return fmt.Errorf("postgres: acquire semantic build connection: %w", err) + } + defer conn.Release() + return fn(conn) +} + +func classifySemanticBuildWrite(op string, build catalog.SemanticInventoryBuild, err error) error { + if storepostgres.IsUniqueConstraintViolation(err, semanticInventoryBuildTableName+"_key_ux") { + return fmt.Errorf("postgres: semantic inventory build key %s collides across scopes (corrupted sib_ derivation): %w", + build.Key, err) + } + return fmt.Errorf("postgres: %s semantic inventory build %s/%s: %w", op, build.Project, build.DatasetGeneration, err) +} + +func validateSemanticInventoryBuildIdentity(build catalog.SemanticInventoryBuild) error { + if err := catalog.ValidateSemanticInventoryBuild(build); err != nil { + return fmt.Errorf("%w: %v", ErrInvalidSemanticInventoryBuild, err) + } + return nil +} + +func normalizeSemanticBuildRequest(build *catalog.SemanticInventoryBuild) { + build.DatasetGeneration = catalog.NormalizeDatasetGeneration(build.DatasetGeneration) +} + +func validateSemanticInventoryTransitionRequest(build catalog.SemanticInventoryBuild, token string, lease time.Duration, requireLease bool) error { + if err := validateSemanticInventoryBuildIdentity(build); err != nil { + return err + } + if strings.TrimSpace(token) == "" { + return &SemanticInventoryUnsupportedError{Case: "claim token", Detail: "blank claim token"} + } + if requireLease && lease <= 0 { + return &SemanticInventoryUnsupportedError{Case: "lease", Detail: fmt.Sprintf("lease renewal must be positive, got %s", lease)} + } + if lease < 0 { + return &SemanticInventoryUnsupportedError{Case: "lease", Detail: fmt.Sprintf("lease must not be negative, got %s", lease)} + } + return nil +} + +func validateSemanticInventoryCheckpoint(cp catalog.SemanticInventoryCheckpoint, name string, requirePresent bool) error { + if (cp.Collection == "") != (cp.Key == "") { + return &SemanticInventoryUnsupportedError{ + Case: "checkpoint", + Detail: fmt.Sprintf("%s checkpoint must set both collection and key, or neither (collection=%q key=%q)", name, cp.Collection, cp.Key), + } + } + if requirePresent && cp.Collection == "" { + return &SemanticInventoryUnsupportedError{ + Case: "checkpoint", + Detail: "next checkpoint must name a collection and key; completion uses CompleteSemanticInventoryBackfill", + } + } + return nil +} + +func validateSemanticInventorySourceAvailability(availability string) error { + switch availability { + case catalog.SemanticInventorySourceAvailabilityUnknown, + catalog.SemanticInventorySourceAvailabilityVerified, + catalog.SemanticInventorySourceAvailabilityUnproven: + return nil + default: + return &SemanticInventoryUnsupportedError{Case: "source_availability", Detail: fmt.Sprintf("unknown value %q", availability)} + } +} diff --git a/internal/backends/postgres/internal/catalog/semantic_build_test.go b/internal/backends/postgres/internal/catalog/semantic_build_test.go new file mode 100644 index 000000000..65c960999 --- /dev/null +++ b/internal/backends/postgres/internal/catalog/semantic_build_test.go @@ -0,0 +1,1253 @@ +package postgres + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "sync" + "testing" + "time" + "unicode/utf8" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +var pgcat04NamePattern = regexp.MustCompile(`^pgcat04_it_[0-9]+_[0-9a-f]+$`) + +var ( + semanticBuildOwnedMu sync.Mutex + semanticBuildOwnedByName = map[string]string{} +) + +func semanticBuildRegister(db, schema string) { + semanticBuildOwnedMu.Lock() + defer semanticBuildOwnedMu.Unlock() + semanticBuildOwnedByName[db] = schema +} + +type semanticBuildService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func semanticBuildTestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 integration tests") + } + return raw +} + +func semanticBuildRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) +} + +func newSemanticBuildService(t *testing.T) *semanticBuildService { + t.Helper() + raw := semanticBuildTestPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var dbName, schema string + + dbName = fmt.Sprintf("pgcat04_it_%d_%s", time.Now().UnixNano(), semanticBuildRandomHex(t)) + if !pgcat04NamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + semanticBuildRegister(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema = fmt.Sprintf("pgcat04_it_%d_%s", time.Now().UnixNano(), semanticBuildRandomHex(t)) + if !pgcat04NamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + semanticBuildRegister(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + service := &semanticBuildService{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} + if _, err := storepostgres.MigrateSchema(ctx, service.openClient(t), schema); err != nil { + fail("MigrateSchema", err) + } + return service +} + +func (s *semanticBuildService) openClient(t *testing.T) *storepostgres.Client { + t.Helper() + client, err := storepostgres.Open(s.ctx, storepostgres.Config{ + DSN: s.dsn, + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *semanticBuildService) repository(t *testing.T) *SemanticBuildRepository { + t.Helper() + repo, err := NewSemanticBuildRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new semantic build repository", err) + } + return repo +} + +func (s *semanticBuildService) exec(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := s.iso.Exec(s.ctx, sql, args...); err != nil { + s.fail("fixture exec", err) + } +} + +func (s *semanticBuildService) buildFQN() string { + return pgx.Identifier{s.schema, "fhir_semantic_inventory_build"}.Sanitize() +} + +type semanticBuildRow struct { + BuildKey string + Project string + DatasetGeneration string + BuildID string + State string + ObservationSchema int + InventorySchema int + EntryIndexVersion int + RuleVersion int + SourceKind string + SourceAvailability string + AuthResourcePath string + ScannedResources int64 + Checkpoint string + CheckpointCollection string + CheckpointKey string + LeaseToken string + LeaseExpiresAt int64 + Diagnostics []string + UpdatedAt time.Time +} + +func sameSemanticRow(a, b semanticBuildRow) bool { + return fmt.Sprint(a) == fmt.Sprint(b) +} + +func (s *semanticBuildService) witnessBuild(t *testing.T, project, generation, buildID string) (semanticBuildRow, bool) { + t.Helper() + var w semanticBuildRow + err := s.iso.QueryRow(s.ctx, "SELECT build_key, project, dataset_generation, build_id, state,"+ + " observation_schema, inventory_schema, entry_index_version, rule_version, source_kind,"+ + " source_availability, auth_resource_path, scanned_resources, checkpoint, checkpoint_collection,"+ + " checkpoint_key, lease_token, lease_expires_at, diagnostics, updated_at"+ + " FROM "+s.buildFQN()+" WHERE project = $1 AND dataset_generation = $2 AND build_id = $3", + project, generation, buildID).Scan( + &w.BuildKey, &w.Project, &w.DatasetGeneration, &w.BuildID, &w.State, + &w.ObservationSchema, &w.InventorySchema, &w.EntryIndexVersion, &w.RuleVersion, &w.SourceKind, + &w.SourceAvailability, &w.AuthResourcePath, &w.ScannedResources, &w.Checkpoint, &w.CheckpointCollection, + &w.CheckpointKey, &w.LeaseToken, &w.LeaseExpiresAt, &w.Diagnostics, &w.UpdatedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return semanticBuildRow{}, false + } + if err != nil { + s.fail("witness semantic build row", err) + } + return w, true +} + +func (s *semanticBuildService) requireNoRow(t *testing.T, project, generation, buildID, what string) { + t.Helper() + if row, found := s.witnessBuild(t, project, generation, buildID); found { + t.Fatalf("%s: unexpected row %+v", what, row) + } +} + +func (s *semanticBuildService) seedSemanticBuildLiteral(t *testing.T, w semanticBuildRow) { + t.Helper() + diagnostics := w.Diagnostics + if diagnostics == nil { + diagnostics = []string{} + } + s.exec(t, "INSERT INTO "+s.buildFQN()+ + " (build_key, project, dataset_generation, build_id, state, observation_schema,"+ + " inventory_schema, entry_index_version, rule_version, source_kind, source_availability,"+ + " auth_resource_path, scanned_resources, checkpoint, checkpoint_collection, checkpoint_key,"+ + " lease_token, lease_expires_at, diagnostics)"+ + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19::text[])", + w.BuildKey, w.Project, w.DatasetGeneration, w.BuildID, w.State, w.ObservationSchema, + w.InventorySchema, w.EntryIndexVersion, w.RuleVersion, w.SourceKind, w.SourceAvailability, + w.AuthResourcePath, w.ScannedResources, w.Checkpoint, w.CheckpointCollection, w.CheckpointKey, + w.LeaseToken, w.LeaseExpiresAt, diagnostics) +} + +func (s *semanticBuildService) trySeedSemanticBuild(t *testing.T, w semanticBuildRow) error { + t.Helper() + diagnostics := w.Diagnostics + if diagnostics == nil { + diagnostics = []string{} + } + _, err := s.iso.Exec(s.ctx, "INSERT INTO "+s.buildFQN()+ + " (build_key, project, dataset_generation, build_id, state, observation_schema,"+ + " inventory_schema, entry_index_version, rule_version, source_kind, source_availability,"+ + " auth_resource_path, scanned_resources, checkpoint, checkpoint_collection, checkpoint_key,"+ + " lease_token, lease_expires_at, diagnostics)"+ + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19::text[])", + w.BuildKey, w.Project, w.DatasetGeneration, w.BuildID, w.State, w.ObservationSchema, + w.InventorySchema, w.EntryIndexVersion, w.RuleVersion, w.SourceKind, w.SourceAvailability, + w.AuthResourcePath, w.ScannedResources, w.Checkpoint, w.CheckpointCollection, w.CheckpointKey, + w.LeaseToken, w.LeaseExpiresAt, diagnostics) + return err +} + +func semanticBuildFixture(project, generation, authPath string) catalog.SemanticInventoryBuild { + build := catalog.NewSemanticInventoryBuild(project, generation, authPath) + build.SourceKind = catalog.SemanticInventorySourceRetained + return build +} + +func literalBuildRowFor(project, generation, state string) semanticBuildRow { + return semanticBuildRow{ + BuildKey: catalog.SemanticInventoryBuildKey(project, generation), + Project: project, + DatasetGeneration: catalog.NormalizeDatasetGeneration(generation), + BuildID: catalog.SemanticInventoryBuildID(project, generation), + State: state, + ObservationSchema: catalog.SemanticObservationLegacyV1SchemaVersion, + InventorySchema: catalog.SemanticInventorySchemaVersion, + EntryIndexVersion: catalog.SemanticInventoryEntryIndexVersion, + RuleVersion: catalog.SemanticObservationLegacyV1RuleVersion, + SourceKind: catalog.SemanticInventorySourceRetained, + SourceAvailability: catalog.SemanticInventorySourceAvailabilityUnproven, + AuthResourcePath: "fhir/Composition?type=hub", + ScannedResources: 42, + Checkpoint: "legacy-string-checkpoint", + CheckpointCollection: "Observation", + CheckpointKey: "obs-0009", + LeaseToken: "", + LeaseExpiresAt: 0, + Diagnostics: []string{"first diagnostic", "second diagnostic"}, + } +} + +func compareDomainBuildToRow(t *testing.T, got catalog.SemanticInventoryBuild, row semanticBuildRow) { + t.Helper() + want := catalog.SemanticInventoryBuild{ + Key: row.BuildKey, + Project: row.Project, + DatasetGeneration: row.DatasetGeneration, + BuildID: row.BuildID, + State: catalog.SemanticInventoryState(row.State), + ObservationSchema: row.ObservationSchema, + InventorySchema: row.InventorySchema, + EntryIndexVersion: row.EntryIndexVersion, + RuleVersion: row.RuleVersion, + SourceKind: row.SourceKind, + SourceAvailability: row.SourceAvailability, + AuthResourcePath: row.AuthResourcePath, + ScannedResources: row.ScannedResources, + Checkpoint: row.Checkpoint, + SourceCheckpoint: catalog.SemanticInventoryCheckpoint{Collection: row.CheckpointCollection, Key: row.CheckpointKey}, + LeaseToken: row.LeaseToken, + LeaseExpiresAt: row.LeaseExpiresAt, + Diagnostics: row.Diagnostics, + } + if got.Key != want.Key || got.Project != want.Project || got.DatasetGeneration != want.DatasetGeneration || + got.BuildID != want.BuildID || got.State != want.State || + got.ObservationSchema != want.ObservationSchema || got.InventorySchema != want.InventorySchema || + got.EntryIndexVersion != want.EntryIndexVersion || got.RuleVersion != want.RuleVersion || + got.SourceKind != want.SourceKind || got.SourceAvailability != want.SourceAvailability || + got.AuthResourcePath != want.AuthResourcePath || got.ScannedResources != want.ScannedResources || + got.Checkpoint != want.Checkpoint || got.SourceCheckpoint != want.SourceCheckpoint || + got.LeaseToken != want.LeaseToken || got.LeaseExpiresAt != want.LeaseExpiresAt || + strings.Join(got.Diagnostics, "|") != strings.Join(want.Diagnostics, "|") { + t.Fatalf("repository read = %#v, want stored row %#v", got, want) + } +} + +func TestSemanticInventoryBuildClaimRoundTripsEveryField(t *testing.T) { + s := newSemanticBuildService(t) + repo := s.repository(t) + build := semanticBuildFixture("p", "g", "fhir/Composition?type=hub") + + before := time.Now().UnixMilli() + claimedBuild, ok, err := repo.ClaimSemanticInventoryBackfill(s.ctx, build, "tok-owner", 30*time.Minute) + after := time.Now().UnixMilli() + if err != nil || !ok { + t.Fatalf("claim = ok %v err %v, want claimed", ok, err) + } + if claimedBuild.State != catalog.SemanticInventoryRunning || claimedBuild.LeaseToken != "tok-owner" { + t.Fatalf("claim returned %#v, want the running claimed build", claimedBuild) + } + + row, found := s.witnessBuild(t, "p", "g", build.BuildID) + if !found { + t.Fatal("claimed build was not stored") + } + if row.BuildKey != build.Key || row.Project != "p" || row.DatasetGeneration != "g" || row.BuildID != build.BuildID { + t.Fatalf("identity columns = %+v", row) + } + if !strings.HasPrefix(row.BuildKey, "sib_") || len(row.BuildKey) != len("sib_")+64 { + t.Fatalf("build key %q is not a sib_ identity", row.BuildKey) + } + if row.State != "running" || row.SourceKind != "retained_vertex" || row.SourceAvailability != "unknown" { + t.Fatalf("lifecycle columns = state %q kind %q availability %q", row.State, row.SourceKind, row.SourceAvailability) + } + if row.ObservationSchema != catalog.SemanticObservationSchemaVersion || + row.InventorySchema != catalog.SemanticInventorySchemaVersion || + row.RuleVersion != catalog.SemanticObservationRuleVersion || row.EntryIndexVersion != 0 { + t.Fatalf("version columns = %+v", row) + } + if row.AuthResourcePath != "fhir/Composition?type=hub" || row.ScannedResources != 0 || row.Checkpoint != "" || + row.CheckpointCollection != "" || row.CheckpointKey != "" || len(row.Diagnostics) != 0 { + t.Fatalf("progress columns = %+v", row) + } + if row.LeaseToken != "tok-owner" || row.LeaseExpiresAt < before+30*60*1000 || row.LeaseExpiresAt > after+30*60*1000 { + t.Fatalf("lease columns = token %q expiry %d, want tok-owner in [%d,%d]", + row.LeaseToken, row.LeaseExpiresAt, before+30*60*1000, after+30*60*1000) + } + + got, found, err := repo.ReadSemanticInventoryBuild(s.ctx, "p", "g", build.BuildID) + if err != nil || !found { + t.Fatalf("read = found %v err %v", found, err) + } + compareDomainBuildToRow(t, got, row) + if got.LeaseToken != "tok-owner" { + t.Fatalf("read lost the lease token: %#v", got) + } + + literal := literalBuildRowFor("p", "g2", "failed") + literal.Diagnostics = []string{"first diagnostic", "second diagnostic"} + s.seedSemanticBuildLiteral(t, literal) + got, found, err = repo.ReadSemanticInventoryBuild(s.ctx, "p", "g2", literal.BuildID) + if err != nil || !found { + t.Fatalf("literal read = found %v err %v", found, err) + } + compareDomainBuildToRow(t, got, literal) + stored, found := s.witnessBuild(t, "p", "g2", literal.BuildID) + if !found || stored.Checkpoint != "legacy-string-checkpoint" || + strings.Join(stored.Diagnostics, "|") != "first diagnostic|second diagnostic" { + t.Fatalf("literal witness = %+v", stored) + } +} + +func TestSemanticInventoryBuildAllSixStatesRoundTrip(t *testing.T) { + s := newSemanticBuildService(t) + repo := s.repository(t) + for i, state := range []string{"unknown", "not_started", "running", "complete", "failed", "invalidated"} { + generation := fmt.Sprintf("state%d", i) + literal := literalBuildRowFor("p", generation, state) + if state == "running" { + literal.LeaseToken = "tok-live" + literal.LeaseExpiresAt = time.Now().Add(time.Hour).UnixMilli() + } + s.seedSemanticBuildLiteral(t, literal) + got, found, err := repo.ReadSemanticInventoryBuild(s.ctx, "p", generation, literal.BuildID) + if err != nil || !found { + t.Fatalf("state %q: read found %v err %v", state, found, err) + } + if got.State != catalog.SemanticInventoryState(state) { + t.Fatalf("state %q read back as %q", state, got.State) + } + compareDomainBuildToRow(t, got, literal) + } +} + +func TestSemanticInventoryBuildSingleClaimantAndStaleTokenCannotAdvanceOrComplete(t *testing.T) { + s := newSemanticBuildService(t) + repo := s.repository(t) + build := semanticBuildFixture("p", "g", "") + + if _, claimed, err := repo.ClaimSemanticInventoryBackfill(s.ctx, build, "tok-a", 30*time.Minute); err != nil || !claimed { + t.Fatalf("first claim = claimed %v err %v", claimed, err) + } + + untouched, _ := s.witnessBuild(t, "p", "g", build.BuildID) + _, claimed, err := repo.ClaimSemanticInventoryBackfill(s.ctx, build, "tok-b", 30*time.Minute) + if claimed || err == nil || !errors.Is(err, ErrSemanticInventoryClaimed) { + t.Fatalf("second claim = claimed %v err %v, want typed claimed error", claimed, err) + } + var claimedErr *SemanticInventoryClaimedError + if !errors.As(err, &claimedErr) || claimedErr.State != catalog.SemanticInventoryRunning { + t.Fatalf("claimed error is not typed with its state: %v", err) + } + after, _ := s.witnessBuild(t, "p", "g", build.BuildID) + if !sameSemanticRow(after, untouched) { + t.Fatalf("rejected claim changed the row:\n before %+v\n after %+v", untouched, after) + } + + staleCheckpoint := catalog.SemanticInventoryCheckpoint{} + next := catalog.SemanticInventoryCheckpoint{Collection: "Patient", Key: "p-1"} + _, staleTokenErr := repo.AdvanceSemanticInventoryBackfill(s.ctx, build, "tok-b", staleCheckpoint, next, 7, time.Minute) + if !errors.Is(staleTokenErr, ErrSemanticInventoryClaimLost) { + t.Fatalf("stale-token advance err = %v, want claim lost", staleTokenErr) + } + var lost *SemanticInventoryClaimLostError + if !errors.As(staleTokenErr, &lost) || !strings.Contains(lost.Reason, "token") { + t.Fatalf("stale-token advance reason = %v, want the token guard", staleTokenErr) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); !sameSemanticRow(row, untouched) { + t.Fatalf("stale-token advance changed the row to %+v", row) + } + if _, err := repo.CompleteSemanticInventoryBackfill(s.ctx, build, "tok-b", staleCheckpoint, 7); !errors.Is(err, ErrSemanticInventoryClaimLost) { + t.Fatalf("stale-token complete err = %v, want claim lost", err) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); !sameSemanticRow(row, untouched) { + t.Fatalf("stale-token complete changed the row to %+v", row) + } + if err := repo.FailSemanticInventoryBackfill(s.ctx, build, "tok-b", "intruder"); !errors.Is(err, ErrSemanticInventoryClaimLost) { + t.Fatalf("stale-token fail err = %v, want claim lost", err) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); !sameSemanticRow(row, untouched) { + t.Fatalf("stale-token fail changed the row to %+v", row) + } + + advanceBefore := time.Now().UnixMilli() + updated, err := repo.AdvanceSemanticInventoryBackfill(s.ctx, build, "tok-a", staleCheckpoint, next, 7, 60*time.Minute) + advanceAfter := time.Now().UnixMilli() + if err != nil { + t.Fatalf("owner advance: %v", err) + } + row, _ := s.witnessBuild(t, "p", "g", build.BuildID) + if row.CheckpointCollection != "Patient" || row.CheckpointKey != "p-1" || row.ScannedResources != 7 { + t.Fatalf("advance did not persist the checkpoint: %+v", row) + } + if row.LeaseToken != "tok-a" || row.LeaseExpiresAt < advanceBefore+60*60*1000 || row.LeaseExpiresAt > advanceAfter+60*60*1000 { + t.Fatalf("advance did not renew the lease to now+60m: expiry %d, want [%d,%d]", + row.LeaseExpiresAt, advanceBefore+60*60*1000, advanceAfter+60*60*1000) + } + if updated.SourceCheckpoint != next || updated.ScannedResources != 7 || updated.LeaseToken != "tok-a" { + t.Fatalf("advance returned %#v", updated) + } + build.SourceAvailability = catalog.SemanticInventorySourceAvailabilityVerified + completed, err := repo.CompleteSemanticInventoryBackfill(s.ctx, build, "tok-a", next, 7) + if err != nil { + t.Fatalf("owner complete: %v", err) + } + row, _ = s.witnessBuild(t, "p", "g", build.BuildID) + if row.State != "complete" || row.LeaseToken != "" || row.LeaseExpiresAt != 0 || len(row.Diagnostics) != 0 || + row.SourceAvailability != "verified" || row.ScannedResources != 7 || + row.CheckpointCollection != "Patient" || row.CheckpointKey != "p-1" { + t.Fatalf("complete transition stored %+v", row) + } + if completed.State != catalog.SemanticInventoryComplete || completed.LeaseToken != "" { + t.Fatalf("complete returned %#v", completed) + } + + returned, claimed, err := repo.ClaimSemanticInventoryBackfill(s.ctx, semanticBuildFixture("p", "g", ""), "tok-c", 30*time.Minute) + if err != nil || claimed { + t.Fatalf("claim on complete = claimed %v err %v, want unclaimed success", claimed, err) + } + if returned.State != catalog.SemanticInventoryComplete || returned.BuildID != build.BuildID { + t.Fatalf("claim on complete returned %#v", returned) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); row.State != "complete" || row.LeaseToken != "" { + t.Fatalf("claim on complete rewrote the marker row: %+v", row) + } + + other := semanticBuildFixture("p", "race", "") + var wg sync.WaitGroup + var mu sync.Mutex + winners, tokens := 0, make([]string, 0, 8) + for i := 0; i < 8; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + token := fmt.Sprintf("tok-race-%d", i) + _, claimedNow, err := repo.ClaimSemanticInventoryBackfill(context.Background(), other, token, time.Hour) + if err != nil && !errors.Is(err, ErrSemanticInventoryClaimed) { + mu.Lock() + t.Errorf("racer %d: %v", i, err) + mu.Unlock() + return + } + mu.Lock() + defer mu.Unlock() + if claimedNow { + winners++ + tokens = append(tokens, token) + } + }(i) + } + wg.Wait() + if winners != 1 { + t.Fatalf("concurrent claims = %d winners, want exactly 1", winners) + } + if row, _ := s.witnessBuild(t, "p", "race", other.BuildID); row.LeaseToken != tokens[0] { + t.Fatalf("stored lease %q is not the winner %q", row.LeaseToken, tokens[0]) + } +} + +func TestSemanticInventoryBuildCheckpointCASRejectsStaleExpected(t *testing.T) { + s := newSemanticBuildService(t) + repo := s.repository(t) + build := semanticBuildFixture("p", "g", "") + if _, claimed, err := repo.ClaimSemanticInventoryBackfill(s.ctx, build, "tok-a", 30*time.Minute); err != nil || !claimed { + t.Fatalf("claim = claimed %v err %v", claimed, err) + } + zero := catalog.SemanticInventoryCheckpoint{} + first := catalog.SemanticInventoryCheckpoint{Collection: "Patient", Key: "p-1"} + if _, err := repo.AdvanceSemanticInventoryBackfill(s.ctx, build, "tok-a", zero, first, 3, 5*time.Minute); err != nil { + t.Fatalf("first advance: %v", err) + } + stored, _ := s.witnessBuild(t, "p", "g", build.BuildID) + + _, staleErr := repo.AdvanceSemanticInventoryBackfill(s.ctx, build, "tok-a", zero, + catalog.SemanticInventoryCheckpoint{Collection: "Patient", Key: "p-2"}, 99, 5*time.Minute) + if !errors.Is(staleErr, ErrSemanticInventoryClaimLost) { + t.Fatalf("stale-expected advance err = %v, want claim lost", staleErr) + } + var lost *SemanticInventoryClaimLostError + if !errors.As(staleErr, &lost) || !strings.Contains(lost.Reason, "checkpoint is Patient/p-1, expected /") { + t.Fatalf("stale-expected reason = %v, want the checkpoint guard naming the stored pair", staleErr) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); !sameSemanticRow(row, stored) { + t.Fatalf("rejected CAS changed the row:\n before %+v\n after %+v", stored, row) + } + + if _, err := repo.AdvanceSemanticInventoryBackfill(s.ctx, build, "tok-a", + catalog.SemanticInventoryCheckpoint{Collection: "Patient", Key: "p-99"}, + catalog.SemanticInventoryCheckpoint{Collection: "Patient", Key: "p-2"}, 99, 5*time.Minute); !errors.Is(err, ErrSemanticInventoryClaimLost) { + t.Fatalf("wrong-key expected advance err = %v, want claim lost", err) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); !sameSemanticRow(row, stored) { + t.Fatalf("rejected wrong-key CAS changed the row to %+v", row) + } + + second := catalog.SemanticInventoryCheckpoint{Collection: "Patient", Key: "p-2"} + if _, err := repo.AdvanceSemanticInventoryBackfill(s.ctx, build, "tok-a", first, second, -5, 5*time.Minute); err != nil { + t.Fatalf("second advance: %v", err) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); row.CheckpointKey != "p-2" || row.ScannedResources != 0 { + t.Fatalf("second advance stored %+v", row) + } + + if _, err := repo.CompleteSemanticInventoryBackfill(s.ctx, build, "tok-a", first, 3); !errors.Is(err, ErrSemanticInventoryClaimLost) { + t.Fatalf("stale-expected complete err = %v, want claim lost", err) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); row.State != "running" { + t.Fatalf("stale-expected complete changed state to %q", row.State) + } +} + +func TestSemanticInventoryBuildFailureAndCompletionDiagnosticSemantics(t *testing.T) { + s := newSemanticBuildService(t) + repo := s.repository(t) + build := semanticBuildFixture("p", "g", "fhir/Patient") + zero := catalog.SemanticInventoryCheckpoint{} + first := catalog.SemanticInventoryCheckpoint{Collection: "Observation", Key: "obs-1"} + + if _, claimed, err := repo.ClaimSemanticInventoryBackfill(s.ctx, build, "tok-a", 30*time.Minute); err != nil || !claimed { + t.Fatalf("claim = claimed %v err %v", claimed, err) + } + build.SourceAvailability = catalog.SemanticInventorySourceAvailabilityUnproven + if _, err := repo.AdvanceSemanticInventoryBackfill(s.ctx, build, "tok-a", zero, first, 11, 30*time.Minute); err != nil { + t.Fatalf("advance: %v", err) + } + + if err := repo.FailSemanticInventoryBackfill(s.ctx, build, "tok-a", "page read failed: timeout"); err != nil { + t.Fatalf("fail: %v", err) + } + row, _ := s.witnessBuild(t, "p", "g", build.BuildID) + if row.State != "failed" || row.LeaseToken != "" || row.LeaseExpiresAt != 0 { + t.Fatalf("fail did not release the lease: %+v", row) + } + if len(row.Diagnostics) != 1 || row.Diagnostics[0] != "page read failed: timeout" { + t.Fatalf("fail diagnostics = %v, want exactly the one failure message", row.Diagnostics) + } + if row.ScannedResources != 11 || row.CheckpointCollection != "Observation" || row.CheckpointKey != "obs-1" || + row.AuthResourcePath != "fhir/Patient" || row.SourceAvailability != "unproven" { + t.Fatalf("fail lost preserved fields: %+v", row) + } + got, found, err := repo.ReadSemanticInventoryBuild(s.ctx, "p", "g", build.BuildID) + if err != nil || !found || len(got.Diagnostics) != 1 || got.Diagnostics[0] != "page read failed: timeout" { + t.Fatalf("fail read-back = %#v found %v err %v", got, found, err) + } + + if _, claimed, err := repo.ClaimSemanticInventoryBackfill(s.ctx, semanticBuildFixture("p", "g", "fhir/Patient"), "tok-a", 30*time.Minute); err != nil || !claimed { + t.Fatalf("re-claim after failure = claimed %v err %v (a failed build must be resumable)", claimed, err) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); row.State != "running" || row.ScannedResources != 11 || + row.CheckpointCollection != "Observation" || row.CheckpointKey != "obs-1" { + t.Fatalf("re-claim after failure lost preserved progress: %+v", row) + } + if err := repo.FailSemanticInventoryBackfill(s.ctx, build, "tok-a", " "); err != nil { + t.Fatalf("blank fail: %v", err) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); len(row.Diagnostics) != 1 || row.Diagnostics[0] != "backfill failed" { + t.Fatalf("blank diagnostic stored %v, want the default", row.Diagnostics) + } + + if _, claimed, err := repo.ClaimSemanticInventoryBackfill(s.ctx, semanticBuildFixture("p", "g", "fhir/Patient"), "tok-a", 30*time.Minute); err != nil || !claimed { + t.Fatalf("re-claim 2 = claimed %v err %v", claimed, err) + } + if err := repo.FailSemanticInventoryBackfill(s.ctx, build, "tok-a", strings.Repeat("x", 600)); err != nil { + t.Fatalf("long fail: %v", err) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); len(row.Diagnostics) != 1 || len(row.Diagnostics[0]) != 512 { + t.Fatalf("long diagnostic not capped at 512 bytes: %d", len(row.Diagnostics[0])) + } + + if err := repo.FailSemanticInventoryBackfill(s.ctx, build, "tok-a", "again"); !errors.Is(err, ErrSemanticInventoryClaimLost) { + t.Fatalf("double fail err = %v, want claim lost", err) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); row.State != "failed" || row.Diagnostics[0] != strings.Repeat("x", 512) { + t.Fatalf("double fail changed the row: %+v", row) + } + + if _, claimed, err := repo.ClaimSemanticInventoryBackfill(s.ctx, semanticBuildFixture("p", "g", "fhir/Patient"), "tok-b", 30*time.Minute); err != nil || !claimed { + t.Fatalf("re-claim after failure = claimed %v err %v", claimed, err) + } + build.SourceAvailability = catalog.SemanticInventorySourceAvailabilityUnproven + if _, err := repo.CompleteSemanticInventoryBackfill(s.ctx, build, "tok-b", first, 11); err != nil { + t.Fatalf("complete: %v", err) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); row.State != "complete" || len(row.Diagnostics) != 0 || + row.SourceAvailability != "unproven" || row.ScannedResources != 11 { + t.Fatalf("complete lost diagnostic/availability meaning: %+v", row) + } + if err := repo.FailSemanticInventoryBackfill(s.ctx, build, "tok-b", "too late"); !errors.Is(err, ErrSemanticInventoryClaimLost) { + t.Fatalf("fail after complete err = %v, want claim lost", err) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); row.State != "complete" || len(row.Diagnostics) != 0 { + t.Fatalf("fail after complete changed the row: %+v", row) + } +} + +func TestSemanticInventoryBuildLeaseExpiryBlocksAdvanceButNotFailure(t *testing.T) { + s := newSemanticBuildService(t) + repo := s.repository(t) + build := semanticBuildFixture("p", "g", "") + zero := catalog.SemanticInventoryCheckpoint{} + first := catalog.SemanticInventoryCheckpoint{Collection: "Patient", Key: "p-1"} + + if _, claimed, err := repo.ClaimSemanticInventoryBackfill(s.ctx, build, "tok-a", 30*time.Minute); err != nil || !claimed { + t.Fatalf("claim = claimed %v err %v", claimed, err) + } + s.exec(t, "UPDATE "+s.buildFQN()+" SET lease_expires_at = $4 WHERE project = $1 AND dataset_generation = $2 AND build_id = $3", + "p", "g", build.BuildID, time.Now().Add(-time.Minute).UnixMilli()) + stored, _ := s.witnessBuild(t, "p", "g", build.BuildID) + + _, expiredErr := repo.AdvanceSemanticInventoryBackfill(s.ctx, build, "tok-a", zero, first, 1, time.Minute) + if !errors.Is(expiredErr, ErrSemanticInventoryClaimLost) { + t.Fatalf("expired advance err = %v, want claim lost", expiredErr) + } + if !strings.Contains(expiredErr.Error(), "lease expired") { + t.Fatalf("expired advance reason = %v, want the expiry guard", expiredErr) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); !sameSemanticRow(row, stored) { + t.Fatalf("expired-lease advance changed the row to %+v", row) + } + + reclaimed, claimed, err := repo.ClaimSemanticInventoryBackfill(s.ctx, semanticBuildFixture("p", "g", ""), "tok-b", 30*time.Minute) + if err != nil || !claimed { + t.Fatalf("stale-lease takeover = claimed %v err %v", claimed, err) + } + if reclaimed.LeaseToken != "tok-b" { + t.Fatalf("takeover returned lease %#v", reclaimed) + } + if err := repo.FailSemanticInventoryBackfill(s.ctx, build, "tok-a", "zombie cleanup"); !errors.Is(err, ErrSemanticInventoryClaimLost) { + t.Fatalf("dead-token fail err = %v, want claim lost", err) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); row.State != "running" || row.LeaseToken != "tok-b" { + t.Fatalf("dead-token fail changed the row: %+v", row) + } + + s.exec(t, "UPDATE "+s.buildFQN()+" SET lease_expires_at = $4 WHERE project = $1 AND dataset_generation = $2 AND build_id = $3", + "p", "g", build.BuildID, time.Now().Add(-time.Minute).UnixMilli()) + if err := repo.FailSemanticInventoryBackfill(s.ctx, build, "tok-b", "cleanup after expiry"); err != nil { + t.Fatalf("expired-holder fail: %v", err) + } + if row, _ := s.witnessBuild(t, "p", "g", build.BuildID); row.State != "failed" || row.LeaseToken != "" || + len(row.Diagnostics) != 1 || row.Diagnostics[0] != "cleanup after expiry" { + t.Fatalf("expired-holder fail stored %+v", row) + } +} + +func TestSemanticInventoryBuildReclaimResumesProgressAndClearsVolatileFields(t *testing.T) { + s := newSemanticBuildService(t) + repo := s.repository(t) + build := semanticBuildFixture("p", "g", "fhir/Composition") + zero := catalog.SemanticInventoryCheckpoint{} + first := catalog.SemanticInventoryCheckpoint{Collection: "Patient", Key: "p-1"} + + if _, claimed, err := repo.ClaimSemanticInventoryBackfill(s.ctx, build, "tok-a", 30*time.Minute); err != nil || !claimed { + t.Fatalf("claim = claimed %v err %v", claimed, err) + } + if _, err := repo.AdvanceSemanticInventoryBackfill(s.ctx, build, "tok-a", zero, first, 3, 30*time.Minute); err != nil { + t.Fatalf("advance: %v", err) + } + s.exec(t, "UPDATE "+s.buildFQN()+" SET checkpoint = $4, diagnostics = ARRAY['stale']::text[]"+ + " WHERE project = $1 AND dataset_generation = $2 AND build_id = $3", + "p", "g", build.BuildID, "volatile-string-checkpoint") + if seeded, _ := s.witnessBuild(t, "p", "g", build.BuildID); seeded.Checkpoint != "volatile-string-checkpoint" || + len(seeded.Diagnostics) != 1 || seeded.Diagnostics[0] != "stale" { + t.Fatalf("volatile seed did not land: %+v", seeded) + } + + resumed, claimed, err := repo.ClaimSemanticInventoryBackfill(s.ctx, semanticBuildFixture("p", "g", "OTHER/Path"), "tok-a", 30*time.Minute) + if err != nil || !claimed { + t.Fatalf("resume = claimed %v err %v", claimed, err) + } + row, _ := s.witnessBuild(t, "p", "g", build.BuildID) + if row.State != "running" || row.ScannedResources != 3 || row.CheckpointCollection != "Patient" || row.CheckpointKey != "p-1" { + t.Fatalf("resume lost progress: %+v", row) + } + if row.Checkpoint != "" || len(row.Diagnostics) != 0 || row.SourceAvailability != "unknown" { + t.Fatalf("resume kept volatile state: %+v", row) + } + if row.AuthResourcePath != "fhir/Composition" { + t.Fatalf("resume overwrote the stored auth path: %+v", row) + } + if resumed.ScannedResources != 3 || resumed.SourceCheckpoint != first { + t.Fatalf("resume returned %#v", resumed) + } + + mixed := literalBuildRowFor("p", "g-mixed", "running") + mixed.SourceKind = catalog.SemanticInventorySourceFile + mixed.SourceAvailability = catalog.SemanticInventorySourceAvailabilityVerified + mixed.LeaseToken = "tok-old" + mixed.LeaseExpiresAt = time.Now().Add(-time.Minute).UnixMilli() + s.seedSemanticBuildLiteral(t, mixed) + if _, claimed, err := repo.ClaimSemanticInventoryBackfill(s.ctx, semanticBuildFixture("p", "g-mixed", ""), "tok-new", 30*time.Minute); err != nil || !claimed { + t.Fatalf("kind-switch claim = claimed %v err %v", claimed, err) + } + row, _ = s.witnessBuild(t, "p", "g-mixed", mixed.BuildID) + if row.SourceKind != "retained_vertex" || row.ScannedResources != 0 || row.CheckpointCollection != "" || + row.CheckpointKey != "" || row.SourceAvailability != "unknown" || row.LeaseToken != "tok-new" { + t.Fatalf("kind switch did not restart progress: %+v", row) + } +} + +func TestSemanticInventoryBuildUnsupportedRequestsAreTypedRefusals(t *testing.T) { + s := newSemanticBuildService(t) + repo := s.repository(t) + + fileBuild := catalog.NewSemanticInventoryBuild("p", "file-kind", "") + _, fileClaimed, fileErr := repo.ClaimSemanticInventoryBackfill(s.ctx, fileBuild, "tok-a", time.Minute) + if !errors.Is(fileErr, ErrUnsupportedSemanticInventoryBuild) || fileClaimed { + t.Fatalf("file-kind claim = claimed %v err %v, want typed refusal", fileClaimed, fileErr) + } + var unsupported *SemanticInventoryUnsupportedError + if !errors.As(fileErr, &unsupported) || unsupported.Case != "source_kind" { + t.Fatalf("file-kind refusal is not typed by case: %v", fileErr) + } + s.requireNoRow(t, "p", "file-kind", fileBuild.BuildID, "file-kind claim stored a row") + + if _, _, err := repo.ClaimSemanticInventoryBackfill(s.ctx, semanticBuildFixture("p", "g", ""), " ", time.Minute); !errors.Is(err, ErrUnsupportedSemanticInventoryBuild) { + t.Fatalf("blank-token claim err = %v, want typed refusal", err) + } + if _, _, err := repo.ClaimSemanticInventoryBackfill(s.ctx, semanticBuildFixture("p", "g2", ""), "tok-a", 0); !errors.Is(err, ErrUnsupportedSemanticInventoryBuild) { + t.Fatalf("zero-lease claim err = %v, want typed refusal", err) + } + s.requireNoRow(t, "p", "g", catalog.SemanticInventoryBuildID("p", "g"), "blank-token claim stored a row") + s.requireNoRow(t, "p", "g2", catalog.SemanticInventoryBuildID("p", "g2"), "zero-lease claim stored a row") + + broken := semanticBuildFixture("p", "g3", "") + broken.BuildID = strings.Repeat("0", 64) + if _, _, err := repo.ClaimSemanticInventoryBackfill(s.ctx, broken, "tok-a", time.Minute); !errors.Is(err, ErrInvalidSemanticInventoryBuild) { + t.Fatalf("broken identity err = %v, want invalid-build sentinel", err) + } + + build := semanticBuildFixture("p", "g4", "") + half := catalog.SemanticInventoryCheckpoint{Collection: "Patient"} + if _, _, err := repo.ClaimSemanticInventoryBackfill(s.ctx, build, "tok-a", time.Minute); err != nil { + t.Fatalf("setup claim: %v", err) + } + if _, err := repo.AdvanceSemanticInventoryBackfill(s.ctx, build, "tok-a", half, half, 1, time.Minute); !errors.Is(err, ErrUnsupportedSemanticInventoryBuild) { + t.Fatalf("half checkpoint advance err = %v, want typed refusal", err) + } + if _, err := repo.AdvanceSemanticInventoryBackfill(s.ctx, build, "tok-a", + catalog.SemanticInventoryCheckpoint{}, catalog.SemanticInventoryCheckpoint{}, 1, time.Minute); !errors.Is(err, ErrUnsupportedSemanticInventoryBuild) { + t.Fatalf("zero next advance err = %v, want typed refusal", err) + } + build.SourceAvailability = "sort-of-verified" + if _, err := repo.AdvanceSemanticInventoryBackfill(s.ctx, build, "tok-a", + catalog.SemanticInventoryCheckpoint{}, catalog.SemanticInventoryCheckpoint{Collection: "Patient", Key: "p"}, 1, time.Minute); !errors.Is(err, ErrUnsupportedSemanticInventoryBuild) { + t.Fatalf("unknown availability advance err = %v, want typed refusal", err) + } + if _, err := repo.AdvanceSemanticInventoryBackfill(s.ctx, build, "tok-a", + catalog.SemanticInventoryCheckpoint{}, catalog.SemanticInventoryCheckpoint{Collection: "Patient", Key: "p"}, 1, 0); !errors.Is(err, ErrUnsupportedSemanticInventoryBuild) { + t.Fatalf("zero-lease advance err = %v, want typed refusal", err) + } + if err := repo.FailSemanticInventoryBackfill(s.ctx, build, "", "no token"); !errors.Is(err, ErrUnsupportedSemanticInventoryBuild) { + t.Fatalf("blank-token fail err = %v, want typed refusal", err) + } + stored, _ := s.witnessBuild(t, "p", "g4", build.BuildID) + if stored.ScannedResources != 0 || stored.CheckpointCollection != "" || stored.SourceAvailability != "unknown" { + t.Fatalf("refused advances changed the row: %+v", stored) + } + if _, _, err := repo.ReadSemanticInventoryBuild(s.ctx, "", "g", "b"); !errors.Is(err, ErrUnsupportedSemanticInventoryBuild) { + t.Fatalf("blank-scope read err = %v, want typed refusal", err) + } + + _, missingErr := repo.AdvanceSemanticInventoryBackfill(s.ctx, semanticBuildFixture("p", "never", ""), "tok-a", + catalog.SemanticInventoryCheckpoint{}, catalog.SemanticInventoryCheckpoint{Collection: "Patient", Key: "p"}, 1, time.Minute) + if !errors.Is(missingErr, ErrSemanticInventoryClaimLost) { + t.Fatalf("missing-row advance err = %v, want claim lost", missingErr) + } + var lost *SemanticInventoryClaimLostError + if !errors.As(missingErr, &lost) || lost.Reason != "row is missing" { + t.Fatalf("missing-row reason = %v", missingErr) + } + s.requireNoRow(t, "p", "never", catalog.SemanticInventoryBuildID("p", "never"), "advance created a row") +} + +func TestSemanticInventoryBuildFailureTruncatesDiagnosticOnRuneBoundary(t *testing.T) { + s := newSemanticBuildService(t) + repo := s.repository(t) + build := semanticBuildFixture("p", "utf8-truncation", "") + claimed, ok, err := repo.ClaimSemanticInventoryBackfill(s.ctx, build, "tok-owner", 30*time.Minute) + if err != nil || !ok { + t.Fatalf("claim = claimed %v err %v", ok, err) + } + diagnostic := strings.Repeat("a", 511) + strings.Repeat("€", 30) + if err := repo.FailSemanticInventoryBackfill(s.ctx, claimed, "tok-owner", diagnostic); err != nil { + s.fail("FailSemanticInventoryBackfill with multi-byte diagnostic", err) + } + row, found := s.witnessBuild(t, "p", "utf8-truncation", claimed.BuildID) + if !found { + t.Fatal("multi-byte diagnostic failure did not record the build") + } + if row.State != "failed" || row.LeaseToken != "" || row.LeaseExpiresAt != 0 { + t.Fatalf("split-rune diagnostic lost the failure: %+v", row) + } + if len(row.Diagnostics) != 1 { + t.Fatalf("diagnostics = %#v, want exactly one", row.Diagnostics) + } + stored := row.Diagnostics[0] + if !utf8.ValidString(stored) { + t.Fatalf("stored diagnostic is not valid UTF-8: %d bytes", len(stored)) + } + if len(stored) > 512 || stored != strings.Repeat("a", 511) { + t.Fatalf("stored diagnostic = %q (%d bytes), want the complete-rune 511-byte prefix", stored, len(stored)) + } +} + +func TestSemanticInventoryBuildPaddedGenerationCannotForkIdentity(t *testing.T) { + s := newSemanticBuildService(t) + repo := s.repository(t) + const padded = " padded generation " + const clean = "padded generation" + rawBuild := func(generation string) catalog.SemanticInventoryBuild { + b := catalog.NewSemanticInventoryBuild("p", generation, "") + b.SourceKind = catalog.SemanticInventorySourceRetained + b.DatasetGeneration = generation + return b + } + + build := rawBuild(padded) + claimed, ok, err := repo.ClaimSemanticInventoryBackfill(s.ctx, build, "tok-owner", 30*time.Minute) + if err != nil || !ok { + t.Fatalf("claim = claimed %v err %v", ok, err) + } + + cleanBuild := rawBuild(clean) + if _, secondOk, secondErr := repo.ClaimSemanticInventoryBackfill(s.ctx, cleanBuild, "tok-other", 30*time.Minute); secondOk || !errors.Is(secondErr, ErrSemanticInventoryClaimed) { + t.Fatalf("clean-spelling claim = claimed %v err %v, want ErrSemanticInventoryClaimed", secondOk, secondErr) + } + + if _, found, readErr := repo.ReadSemanticInventoryBuild(s.ctx, "p", clean, claimed.BuildID); readErr != nil || !found { + t.Fatalf("clean-spelling read = found %v err %v", found, readErr) + } + zero := catalog.SemanticInventoryCheckpoint{} + first := catalog.SemanticInventoryCheckpoint{Collection: "Observation", Key: "obs-1"} + if _, advErr := repo.AdvanceSemanticInventoryBackfill(s.ctx, cleanBuild, "tok-owner", zero, first, 3, 30*time.Minute); advErr != nil { + t.Fatalf("clean-spelling advance: %v", advErr) + } + if _, compErr := repo.CompleteSemanticInventoryBackfill(s.ctx, build, "tok-owner", first, 3); compErr != nil { + t.Fatalf("padded-spelling complete: %v", compErr) + } + + var rowCount int + var storedGeneration string + if err := s.iso.QueryRow(s.ctx, "SELECT COUNT(*), COALESCE(MAX(dataset_generation), '') FROM "+s.buildFQN()+ + " WHERE project = $1 AND dataset_generation IN ($2, $3)", "p", padded, clean). + Scan(&rowCount, &storedGeneration); err != nil { + s.fail("count padded-generation rows", err) + } + if rowCount != 1 || storedGeneration != clean { + t.Fatalf("padded generation forked identity: %d rows, stored %q", rowCount, storedGeneration) + } + row, found := s.witnessBuild(t, "p", clean, claimed.BuildID) + if !found || row.State != "complete" { + t.Fatalf("converged row missing or wrong state: found %v %+v", found, row) + } +} + +const semanticBuildMigrationChecksum = "e69d0dfdd3fc67438b65df7da73ea12edb61324ec8356cbdc4abc1ae5172af3c" + +func TestSemanticInventoryBuildMigrationDoubleAppliesAndRecordsLedgerLiterally(t *testing.T) { + s := newSemanticBuildService(t) + + first, err := storepostgres.MigrateSchema(s.ctx, s.openClient(t), s.schema) + if err != nil { + s.fail("MigrateSchema (probe apply 1)", err) + } + second, err := storepostgres.MigrateSchema(s.ctx, s.openClient(t), s.schema) + if err != nil { + s.fail("MigrateSchema (probe apply 2)", err) + } + if second.LatestVersion != first.LatestVersion { + t.Fatalf("re-apply latest version = %d, want idempotent %d", second.LatestVersion, first.LatestVersion) + } + if len(first.Applied) != len(second.Applied) { + t.Fatalf("re-apply recorded %d rows, want the idempotent %d", len(second.Applied), len(first.Applied)) + } + for i := range first.Applied { + a, b := first.Applied[i], second.Applied[i] + if a.Version != b.Version || a.Name != b.Name || a.Checksum != b.Checksum || !a.AppliedAt.Equal(b.AppliedAt) { + t.Fatalf("re-apply rewrote ledger row %d: %+v vs %+v", i, a, b) + } + } + + var ledgerRows int + var name, checksum string + if err := s.iso.QueryRow(s.ctx, "SELECT COUNT(*), COALESCE(MAX(name), ''), COALESCE(MAX(checksum), '') FROM "+ + pgx.Identifier{s.schema, storepostgres.LedgerTable}.Sanitize()+" WHERE version = $1", int64(6)). + Scan(&ledgerRows, &name, &checksum); err != nil { + s.fail("read ledger for version 6", err) + } + if ledgerRows != 1 { + t.Fatalf("ledger holds %d rows for version 6, want exactly one", ledgerRows) + } + if name != "semantic_inventory_build" { + t.Fatalf("ledger name = %q, want semantic_inventory_build", name) + } + if checksum != semanticBuildMigrationChecksum { + t.Fatalf("ledger checksum = %s, want the pinned %s", checksum, semanticBuildMigrationChecksum) + } + fileBytes, err := os.ReadFile("../store/migrations/000006_semantic_inventory_build.sql") + if err != nil { + t.Fatalf("read migration file: %v", err) + } + sum := sha256.Sum256(fileBytes) + if got := hex.EncodeToString(sum[:]); got != checksum { + t.Fatalf("file sha256sum %s disagrees with ledger checksum %s", got, checksum) + } + + wantConstraints := []string{ + "fhir_semantic_inventory_build_pkey", + "fhir_semantic_inventory_build_key_ux", + "fhir_semantic_inventory_build_key_shape", + "fhir_semantic_inventory_build_project_present", + "fhir_semantic_inventory_build_build_present", + "fhir_semantic_inventory_build_build_shape", + "fhir_semantic_inventory_build_state_known", + "fhir_semantic_inventory_build_versions_sane", + "fhir_semantic_inventory_build_source_kind_known", + "fhir_semantic_inventory_build_availability_known", + "fhir_semantic_inventory_build_scanned_nonnegative", + "fhir_semantic_inventory_build_checkpoint_paired", + "fhir_semantic_inventory_build_lease_paired", + "fhir_semantic_inventory_build_lease_expiry_nonnegative", + } + present := map[string]bool{} + rows, err := s.iso.Query(s.ctx, "SELECT conname FROM pg_constraint WHERE conrelid = $1::regclass", s.buildFQN()) + if err != nil { + s.fail("read table constraints", err) + } + for rows.Next() { + var conname string + if err := rows.Scan(&conname); err != nil { + rows.Close() + s.fail("read table constraints", err) + } + present[conname] = true + } + rows.Close() + if err := rows.Err(); err != nil { + s.fail("read table constraints", err) + } + for _, conname := range wantConstraints { + if !present[conname] { + t.Fatalf("constraint %s is absent from the live table", conname) + } + } +} + +func TestSemanticInventoryBuildMigrationConstraintsRejectViolations(t *testing.T) { + s := newSemanticBuildService(t) + + const ( + checkViolation = "23514" + uniqueViolation = "23505" + ) + base := func(generation string) semanticBuildRow { + row := literalBuildRowFor("p", generation, "running") + row.LeaseToken = "tok-live" + row.LeaseExpiresAt = time.Now().Add(time.Hour).UnixMilli() + return row + } + cases := []struct { + name string + mutate func(*semanticBuildRow) + wantCode string + wantConstraint string + }{ + {"checkpoint collection without key", func(r *semanticBuildRow) { r.CheckpointCollection = "Patient"; r.CheckpointKey = "" }, + checkViolation, "fhir_semantic_inventory_build_checkpoint_paired"}, + {"checkpoint key without collection", func(r *semanticBuildRow) { r.CheckpointCollection = ""; r.CheckpointKey = "p-1" }, + checkViolation, "fhir_semantic_inventory_build_checkpoint_paired"}, + {"lease token with zero expiry", func(r *semanticBuildRow) { r.LeaseToken = "tok-x"; r.LeaseExpiresAt = 0 }, + checkViolation, "fhir_semantic_inventory_build_lease_paired"}, + {"lease expiry without token", func(r *semanticBuildRow) { r.LeaseToken = ""; r.LeaseExpiresAt = time.Now().Add(time.Hour).UnixMilli() }, + checkViolation, "fhir_semantic_inventory_build_lease_paired"}, + {"negative lease expiry under a token", func(r *semanticBuildRow) { r.LeaseToken = "tok-x"; r.LeaseExpiresAt = -5 }, + checkViolation, "fhir_semantic_inventory_build_lease_expiry_nonnegative"}, + {"unknown state", func(r *semanticBuildRow) { r.State = "queued" }, + checkViolation, "fhir_semantic_inventory_build_state_known"}, + {"unknown source kind", func(r *semanticBuildRow) { r.SourceKind = "stream" }, + checkViolation, "fhir_semantic_inventory_build_source_kind_known"}, + {"unknown source availability", func(r *semanticBuildRow) { r.SourceAvailability = "probably" }, + checkViolation, "fhir_semantic_inventory_build_availability_known"}, + {"negative scanned_resources", func(r *semanticBuildRow) { r.ScannedResources = -1 }, + checkViolation, "fhir_semantic_inventory_build_scanned_nonnegative"}, + {"malformed build_key", func(r *semanticBuildRow) { r.BuildKey = "sib_short" }, + checkViolation, "fhir_semantic_inventory_build_key_shape"}, + {"blank project", func(r *semanticBuildRow) { r.Project = "" }, + checkViolation, "fhir_semantic_inventory_build_project_present"}, + {"malformed build_id", func(r *semanticBuildRow) { r.BuildID = strings.Repeat("z", 64) }, + checkViolation, "fhir_semantic_inventory_build_build_shape"}, + {"zero observation_schema", func(r *semanticBuildRow) { r.ObservationSchema = 0 }, + checkViolation, "fhir_semantic_inventory_build_versions_sane"}, + } + for i, tc := range cases { + bad := base(fmt.Sprintf("neg%d", i)) + tc.mutate(&bad) + control := base(fmt.Sprintf("ctrl%d", i)) + s.seedSemanticBuildLiteral(t, control) + if _, found := s.witnessBuild(t, control.Project, control.DatasetGeneration, control.BuildID); !found { + t.Fatalf("%s: control row did not insert — the harness is not proving anything", tc.name) + } + err := s.trySeedSemanticBuild(t, bad) + var pgErr *pgconn.PgError + if err == nil { + t.Fatalf("%s: violating row was ACCEPTED — constraint missing or renamed", tc.name) + } + if !errors.As(err, &pgErr) { + t.Fatalf("%s: violating row failed with a non-database error: %v", tc.name, err) + } + if pgErr.Code != tc.wantCode || pgErr.ConstraintName != tc.wantConstraint { + t.Fatalf("%s: rejected by code=%s constraint=%s, want code=%s constraint=%s", + tc.name, pgErr.Code, pgErr.ConstraintName, tc.wantCode, tc.wantConstraint) + } + s.requireNoRow(t, bad.Project, bad.DatasetGeneration, bad.BuildID, tc.name) + } + + dup := base("dup-pk") + s.seedSemanticBuildLiteral(t, dup) + rewritten := dup + rewritten.ScannedResources = 999 + err := s.trySeedSemanticBuild(t, rewritten) + var pgErr *pgconn.PgError + if err == nil || !errors.As(err, &pgErr) || pgErr.Code != uniqueViolation || pgErr.ConstraintName != "fhir_semantic_inventory_build_pkey" { + t.Fatalf("duplicate identity accepted or misattributed: %v", err) + } + if row, _ := s.witnessBuild(t, dup.Project, dup.DatasetGeneration, dup.BuildID); row.ScannedResources != 42 { + t.Fatalf("duplicate identity replaced the stored row: %+v", row) + } + other := base("dup-key") + stolen := other + stolen.BuildKey = base("dup-pk").BuildKey + err = s.trySeedSemanticBuild(t, stolen) + if err == nil || !errors.As(err, &pgErr) || pgErr.Code != uniqueViolation || pgErr.ConstraintName != "fhir_semantic_inventory_build_key_ux" { + t.Fatalf("reused sib_ build_key accepted or misattributed: %v", err) + } + s.requireNoRow(t, other.Project, other.DatasetGeneration, other.BuildID, "reused build_key") +} + +func TestZZZeroLeftoverSemanticInventoryBuildFixtures(t *testing.T) { + raw := semanticBuildTestPostgresURL(t) + semanticBuildOwnedMu.Lock() + owned := make(map[string]string, len(semanticBuildOwnedByName)) + for db, schema := range semanticBuildOwnedByName { + owned[db] = schema + } + semanticBuildOwnedMu.Unlock() + if len(owned) == 0 { + t.Skip("this process created no pgcat04_it_* fixtures") + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service for audit: %v", err) + } + defer admin.Close() + + for db, schema := range owned { + if !strings.HasPrefix(db, "pgcat04_it_") { + t.Fatalf("registry holds foreign database %q", db) + } + var count int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Errorf("audit owned database %s: %v", db, err) + continue + } + if count != 0 { + t.Errorf("leftover owned database %s after all cleanups", db) + if schema != "" { + u, err := url.Parse(raw) + if err == nil { + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + db + if probe, err := pgxpool.New(ctx, u.String()); err == nil { + var schemaCount int + if err := probe.QueryRow(ctx, "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1", schema).Scan(&schemaCount); err != nil { + t.Errorf("audit owned schema %s in %s: %v", schema, db, err) + } else if schemaCount != 0 { + t.Errorf("leftover owned schema %s in leftover database %s", schema, db) + } + probe.Close() + } + } + } + } + } +} + +func TestIsConcurrentSemanticBuildKeyRaceClassification(t *testing.T) { + keyConstraint := semanticInventoryBuildTableName + "_key_ux" + race := &pgconn.PgError{Code: "23505", ConstraintName: keyConstraint, Message: "duplicate key"} + if !isConcurrentSemanticBuildKeyRace(race) { + t.Fatal("key unique violation must classify as a concurrent build race") + } + if !isConcurrentSemanticBuildKeyRace(fmt.Errorf("claim: %w", race)) { + t.Fatal("wrapped key unique violation must classify as a concurrent build race") + } + if isConcurrentSemanticBuildKeyRace(&pgconn.PgError{Code: "23505", ConstraintName: "fhir_semantic_inventory_build_pkey"}) { + t.Fatal("a different constraint's unique violation is not the build key race") + } + if isConcurrentSemanticBuildKeyRace(errors.New("plain error")) { + t.Fatal("a plain error is not the build key race") + } + for _, input := range []error{ + error((*pgconn.PgError)(nil)), + fmt.Errorf("claim: %w", error((*pgconn.PgError)(nil))), + errors.Join(errors.New("context"), error((*pgconn.PgError)(nil))), + } { + if isConcurrentSemanticBuildKeyRace(input) { + t.Fatalf("typed-nil driver error reported as a build key race: %v", input) + } + } +} + +func TestClassifySemanticBuildWriteKeepsCollisionClassification(t *testing.T) { + build := catalog.SemanticInventoryBuild{Key: "sib_1", Project: "p", DatasetGeneration: "g"} + keyConstraint := semanticInventoryBuildTableName + "_key_ux" + collision := &pgconn.PgError{Code: "23505", ConstraintName: keyConstraint, Message: "duplicate key"} + classified := classifySemanticBuildWrite("claim", build, collision) + if !strings.Contains(classified.Error(), "collides across scopes") { + t.Fatalf("key unique violation not classified as a collision: %v", classified) + } + if !errors.Is(classified, collision) { + t.Fatalf("collision classification lost the driver cause: %v", classified) + } + plain := errors.New("connection reset by peer") + classified = classifySemanticBuildWrite("claim", build, plain) + if !errors.Is(classified, plain) { + t.Fatalf("ordinary cause not preserved: %v", classified) + } + for _, input := range []error{ + error((*pgconn.PgError)(nil)), + fmt.Errorf("claim: %w", error((*pgconn.PgError)(nil))), + } { + classified := classifySemanticBuildWrite("claim", build, input) + if classified == nil { + t.Fatalf("typed-nil driver error classified as no fault: %v", input) + } + if strings.Contains(classified.Error(), "collides across scopes") { + t.Fatalf("typed-nil driver error mistaken for a key collision: %v", classified) + } + } +} diff --git a/internal/backends/postgres/internal/catalog/semantic_contribution.go b/internal/backends/postgres/internal/catalog/semantic_contribution.go new file mode 100644 index 000000000..79792f97e --- /dev/null +++ b/internal/backends/postgres/internal/catalog/semantic_contribution.go @@ -0,0 +1,693 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + fhirschema "github.com/calypr/loom/internal/fhir/schema" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + semanticInventoryContributionTableName = "fhir_semantic_inventory_contribution" + retainedResourceTableName = "fhir_resource" + fieldProfileTableName = "fhir_field_profile" + fieldProfileCoverageTableName = "fhir_field_profile_coverage" + maxRetainedSemanticPageLimit = 1000 + defaultSemanticContributionBatchSize = 500 + maxSemanticContributionChunkSize = 1000 +) + +var ( + ErrSemanticInventoryContributionConflict = errors.New("postgres: semantic inventory contribution conflict") + ErrInvalidSemanticInventoryContribution = errors.New("postgres: invalid semantic inventory contribution") + ErrInvalidRetainedSemanticPageRequest = errors.New("postgres: invalid retained semantic inventory page request") + ErrRetainedSemanticSourceMissing = errors.New("postgres: retained semantic inventory source is missing despite field-profile evidence") + ErrSemanticInventoryStorageMissing = errors.New("postgres: semantic inventory storage is not migrated") + ErrSemanticInventoryEntriesUnsupported = errors.New("postgres: semantic inventory entry materialization is not persisted by this slice") +) + +type SemanticInventoryContributionConflictError struct { + Project string + DatasetGeneration string + BuildID string + SourceKind string + SourceID string + Ordinal int + ContributionKey string + Slot bool + StoredDigest string + ProvidedDigest string +} + +func (e *SemanticInventoryContributionConflictError) Error() string { + if e.Slot { + return fmt.Sprintf("%v: %s/%s build %s kind %q source %s ordinal %d is held by a different contribution key; "+ + "the stored row was not replaced (offered key %s)", + ErrSemanticInventoryContributionConflict, e.Project, e.DatasetGeneration, e.BuildID, + e.SourceKind, e.SourceID, e.Ordinal, e.ContributionKey) + } + return fmt.Sprintf("%v: contribution %s (%s/%s build %s kind %q source %s ordinal %d) already stores digest %s; "+ + "offered digest %s differs; the stored payload was not replaced", + ErrSemanticInventoryContributionConflict, e.ContributionKey, e.Project, e.DatasetGeneration, + e.BuildID, e.SourceKind, e.SourceID, e.Ordinal, e.StoredDigest, e.ProvidedDigest) +} + +func (e *SemanticInventoryContributionConflictError) Unwrap() error { + return ErrSemanticInventoryContributionConflict +} + +type RetainedSemanticSourceMissingError struct { + Project string + DatasetGeneration string + ResourceType string +} + +func (e *RetainedSemanticSourceMissingError) Error() string { + return fmt.Sprintf("%v: %s/%s/%s has no retained rows although field-profile records with doc_count > 0 exist", + ErrRetainedSemanticSourceMissing, e.Project, e.DatasetGeneration, e.ResourceType) +} + +func (e *RetainedSemanticSourceMissingError) Unwrap() error { return ErrRetainedSemanticSourceMissing } + +type SemanticInventoryStorageMissingError struct { + Schema string + Table string +} + +func (e *SemanticInventoryStorageMissingError) Error() string { + return fmt.Sprintf("%v: schema %q has no table %q; run MigrateSchema before the semantic inventory backfill", + ErrSemanticInventoryStorageMissing, e.Schema, e.Table) +} + +func (e *SemanticInventoryStorageMissingError) Unwrap() error { + return ErrSemanticInventoryStorageMissing +} + +type SemanticInventoryEntriesUnsupportedError struct { + Project string + DatasetGeneration string + BuildID string + EntryIndexVersion int +} + +func (e *SemanticInventoryEntriesUnsupportedError) Error() string { + return fmt.Sprintf("%v: build %s/%s (entry index version %d) requires entry materialization, which this "+ + "PostgreSQL slice does not persist; entry rows are a separate slice, never a silent skip", + ErrSemanticInventoryEntriesUnsupported, e.Project, e.DatasetGeneration, e.EntryIndexVersion) +} + +func (e *SemanticInventoryEntriesUnsupportedError) Unwrap() error { + return ErrSemanticInventoryEntriesUnsupported +} + +type SemanticContributionRepository struct { + src storepostgres.ConnSource + schema string + contributionFQN string + buildFQN string + resourceFQN string + fieldProfileFQN string + fieldProfileCovFQN string +} + +func NewSemanticContributionRepository(src storepostgres.ConnSource, schema string) (*SemanticContributionRepository, error) { + if src == nil { + return nil, errors.New("postgres: nil semantic contribution connection source") + } + if !storepostgres.ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", storepostgres.ErrInvalidSchemaName, schema) + } + return &SemanticContributionRepository{ + src: src, + schema: schema, + contributionFQN: pgx.Identifier{schema, semanticInventoryContributionTableName}.Sanitize(), + buildFQN: pgx.Identifier{schema, semanticInventoryBuildTableName}.Sanitize(), + resourceFQN: pgx.Identifier{schema, retainedResourceTableName}.Sanitize(), + fieldProfileFQN: pgx.Identifier{schema, fieldProfileTableName}.Sanitize(), + fieldProfileCovFQN: pgx.Identifier{schema, fieldProfileCoverageTableName}.Sanitize(), + }, nil +} + +func (r *SemanticContributionRepository) PrepareSemanticInventoryBackfill(ctx context.Context) error { + for _, table := range []struct { + fqn string + tableName string + }{ + {r.buildFQN, semanticInventoryBuildTableName}, + {r.contributionFQN, semanticInventoryContributionTableName}, + {r.resourceFQN, retainedResourceTableName}, + } { + exists, err := r.tableExists(ctx, table.fqn) + if err != nil { + return err + } + if !exists { + return &SemanticInventoryStorageMissingError{Schema: r.schema, Table: table.tableName} + } + } + return nil +} + +func (r *SemanticContributionRepository) WriteSemanticInventoryContributions(ctx context.Context, contributions []catalog.SemanticInventoryContribution, batchSize int) error { + if len(contributions) == 0 { + return nil + } + if batchSize <= 0 { + batchSize = defaultSemanticContributionBatchSize + } + if batchSize > maxSemanticContributionChunkSize { + batchSize = maxSemanticContributionChunkSize + } + preparedAll := make([]preparedContribution, 0, len(contributions)) + seenSlot := make(map[string]struct{}, len(contributions)) + for i, contribution := range contributions { + p, err := prepareSemanticContribution(contribution) + if err != nil { + return fmt.Errorf("%w: contribution %d: %v", ErrInvalidSemanticInventoryContribution, i, err) + } + slotKey := semanticContributionSlotKey(p.contribution) + if _, dup := seenSlot[slotKey]; dup { + first := seenSlotFirst(preparedAll, slotKey) + return &SemanticInventoryContributionConflictError{ + Project: first.Project, DatasetGeneration: first.DatasetGeneration, + BuildID: first.BuildID, SourceKind: first.SourceKind, + SourceID: first.SourceID, Ordinal: first.Ordinal, + ContributionKey: first.Key, Slot: true, + } + } + seenSlot[slotKey] = struct{}{} + preparedAll = append(preparedAll, p) + } + + for start := 0; start < len(preparedAll); start += batchSize { + end := min(start+batchSize, len(preparedAll)) + if err := r.writeChunk(ctx, preparedAll[start:end]); err != nil { + return err + } + } + return nil +} + +type preparedContribution struct { + contribution catalog.SemanticInventoryContribution + observation []byte + digest string +} + +func (r *SemanticContributionRepository) writeChunk(ctx context.Context, chunk []preparedContribution) error { + var sql strings.Builder + sql.WriteString("INSERT INTO " + r.contributionFQN + + " (contribution_key, project, dataset_generation, build_id, source_kind," + + " auth_resource_path, resource_type, source_id, ordinal, concept_id, binding_id," + + " example, payload_digest, observation)") + sql.WriteString(" VALUES ") + args := make([]any, 0, len(chunk)*14) + distinctKeys := make([]string, 0, len(chunk)) + seenKey := make(map[string]struct{}, len(chunk)) + parameter := 0 + for i := range chunk { + c := chunk[i].contribution + if i > 0 { + sql.WriteString(", ") + } + sql.WriteString("(") + for field := 0; field < 14; field++ { + if field > 0 { + sql.WriteString(", ") + } + parameter++ + sql.WriteString("$" + strconv.Itoa(parameter)) + } + sql.WriteString(")") + args = append(args, + c.Key, c.Project, c.DatasetGeneration, c.BuildID, c.SourceKind, + c.AuthResourcePath, c.ResourceType, c.SourceID, c.Ordinal, + c.ConceptID, c.BindingID, c.Example, chunk[i].digest, chunk[i].observation) + if _, dup := seenKey[c.Key]; !dup { + distinctKeys = append(distinctKeys, c.Key) + seenKey[c.Key] = struct{}{} + } + } + sql.WriteString(" ON CONFLICT DO NOTHING") + + conn, err := r.src.Acquire(ctx) + if err != nil { + return fmt.Errorf("postgres: acquire semantic contribution connection: %w", err) + } + defer conn.Release() + + tag, err := conn.Exec(ctx, sql.String(), args...) + if err != nil { + return fmt.Errorf("postgres: write semantic inventory contributions: %w", err) + } + if tag.RowsAffected() == int64(len(chunk)) { + return nil + } + stored, err := r.readContributionDigests(ctx, conn, distinctKeys) + if err != nil { + return err + } + holders, err := r.readContributionSlotHolders(ctx, conn, chunk) + if err != nil { + return err + } + for i := range chunk { + p := &chunk[i] + c := p.contribution + storedDigest, keyStored := stored[c.Key] + holder, slotHeld := holders[semanticContributionSlotKey(c)] + if !keyStored { + if !slotHeld || holder == c.Key { + return fmt.Errorf( + "postgres: semantic inventory contribution %s vanished between insert and classification; retry the batch", + c.Key) + } + return &SemanticInventoryContributionConflictError{ + Project: c.Project, DatasetGeneration: c.DatasetGeneration, + BuildID: c.BuildID, SourceKind: c.SourceKind, + SourceID: c.SourceID, Ordinal: c.Ordinal, + ContributionKey: c.Key, Slot: true, + } + } + if storedDigest != p.digest { + return &SemanticInventoryContributionConflictError{ + Project: c.Project, DatasetGeneration: c.DatasetGeneration, + BuildID: c.BuildID, SourceKind: c.SourceKind, + SourceID: c.SourceID, Ordinal: c.Ordinal, + ContributionKey: c.Key, + StoredDigest: storedDigest, ProvidedDigest: p.digest, + } + } + if !slotHeld || holder != c.Key { + return &SemanticInventoryContributionConflictError{ + Project: c.Project, DatasetGeneration: c.DatasetGeneration, + BuildID: c.BuildID, SourceKind: c.SourceKind, + SourceID: c.SourceID, Ordinal: c.Ordinal, + ContributionKey: c.Key, Slot: true, + StoredDigest: storedDigest, ProvidedDigest: p.digest, + } + } + } + return nil +} + +func (r *SemanticContributionRepository) readContributionSlotHolders( + ctx context.Context, conn *pgxpool.Conn, chunk []preparedContribution, +) (map[string]string, error) { + var query strings.Builder + query.WriteString("SELECT project, dataset_generation, build_id, source_kind, source_id, ordinal, contribution_key FROM " + + r.contributionFQN + + " WHERE (project, dataset_generation, build_id, source_kind, source_id, ordinal) IN (VALUES ") + args := make([]any, 0, len(chunk)*6) + parameter := 0 + for i := range chunk { + if i > 0 { + query.WriteString(", ") + } + c := chunk[i].contribution + query.WriteString("(") + for field := 0; field < 6; field++ { + if field > 0 { + query.WriteString(", ") + } + parameter++ + if field == 5 { + query.WriteString("$" + strconv.Itoa(parameter) + "::int") + } else { + query.WriteString("$" + strconv.Itoa(parameter) + "::text") + } + } + query.WriteString(")") + args = append(args, c.Project, c.DatasetGeneration, c.BuildID, c.SourceKind, c.SourceID, c.Ordinal) + } + query.WriteString(")") + + rows, err := conn.Query(ctx, query.String(), args...) + if err != nil { + return nil, fmt.Errorf("postgres: classify semantic inventory contribution slots: %w", err) + } + defer rows.Close() + holders := make(map[string]string, len(chunk)) + for rows.Next() { + var c catalog.SemanticInventoryContribution + var heldKey string + if err := rows.Scan(&c.Project, &c.DatasetGeneration, &c.BuildID, &c.SourceKind, + &c.SourceID, &c.Ordinal, &heldKey); err != nil { + return nil, fmt.Errorf("postgres: scan semantic inventory contribution slot: %w", err) + } + holders[semanticContributionSlotKey(c)] = heldKey + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("postgres: read semantic inventory contribution slots: %w", err) + } + return holders, nil +} + +func (r *SemanticContributionRepository) readContributionDigests(ctx context.Context, conn *pgxpool.Conn, keys []string) (map[string]string, error) { + rows, err := conn.Query(ctx, + "SELECT contribution_key, payload_digest FROM "+r.contributionFQN+" WHERE contribution_key = ANY($1::text[])", + keys) + if err != nil { + return nil, fmt.Errorf("postgres: classify semantic inventory contributions: %w", err) + } + defer rows.Close() + out := make(map[string]string, len(keys)) + for rows.Next() { + var key, digest string + if err := rows.Scan(&key, &digest); err != nil { + return nil, fmt.Errorf("postgres: classify semantic inventory contributions: %w", err) + } + out[key] = digest + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("postgres: classify semantic inventory contributions: %w", err) + } + return out, nil +} + +func (r *SemanticContributionRepository) ReadRetainedSemanticInventoryPage(ctx context.Context, project, generation, collection, afterKey string, limit int) (catalog.RetainedSemanticInventoryPage, error) { + if canonical, ok := fhirschema.ConcreteResourceType(collection); !ok || canonical != collection { + return catalog.RetainedSemanticInventoryPage{}, fmt.Errorf( + "%w: collection %q is not a canonical concrete resource type", ErrInvalidRetainedSemanticPageRequest, collection) + } + if strings.TrimSpace(project) == "" { + return catalog.RetainedSemanticInventoryPage{}, fmt.Errorf("%w: project is blank", ErrInvalidRetainedSemanticPageRequest) + } + if limit < 1 || limit > maxRetainedSemanticPageLimit { + return catalog.RetainedSemanticInventoryPage{}, fmt.Errorf("%w: limit is %d, want 1..%d", + ErrInvalidRetainedSemanticPageRequest, limit, maxRetainedSemanticPageLimit) + } + generation = catalog.NormalizeDatasetGeneration(generation) + + var sql strings.Builder + sql.WriteString("SELECT logical_id, auth_resource_path, payload FROM " + r.resourceFQN + + " WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3") + args := []any{project, generation, collection} + if afterKey != "" { + sql.WriteString(` AND logical_id COLLATE "C" > $4 COLLATE "C"`) + args = append(args, afterKey) + } + sql.WriteString(fmt.Sprintf(` ORDER BY logical_id COLLATE "C" LIMIT $%d`, len(args)+1)) + args = append(args, limit+1) + + page := catalog.RetainedSemanticInventoryPage{Rows: make([]catalog.RetainedSemanticInventoryRow, 0, limit)} + err := r.withConn(ctx, func(conn *pgxpool.Conn) error { + rows, err := conn.Query(ctx, sql.String(), args...) + if err != nil { + return fmt.Errorf("postgres: read retained %s source page: %w", collection, err) + } + defer rows.Close() + for rows.Next() { + var source catalog.RetainedSemanticInventoryRow + var payload []byte + if err := rows.Scan(&source.Key, &source.AuthResourcePath, &payload); err != nil { + return fmt.Errorf("postgres: read retained %s source page: %w", collection, err) + } + if source.Key == "" { + return fmt.Errorf("postgres: retained %s source row is missing logical_id", collection) + } + if payload != nil { + source.Payload = json.RawMessage(payload) + } + page.Rows = append(page.Rows, source) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("postgres: read retained %s source page: %w", collection, err) + } + return nil + }) + if err != nil { + return catalog.RetainedSemanticInventoryPage{}, err + } + if len(page.Rows) > limit { + page.Rows = page.Rows[:limit] + page.HasMore = true + } + if len(page.Rows) == 0 && afterKey == "" { + evidence, err := r.retainedSourceEvidence(ctx, project, generation, collection) + if err != nil { + return catalog.RetainedSemanticInventoryPage{}, err + } + switch evidence { + case retainedEvidenceDocuments: + return catalog.RetainedSemanticInventoryPage{}, &RetainedSemanticSourceMissingError{ + Project: project, DatasetGeneration: generation, ResourceType: collection, + } + case retainedEvidenceCoverage: + page.SourceExists = true + case retainedEvidenceNone: + page.SourceExists = false + } + } else { + page.SourceExists = true + } + return page, nil +} + +type retainedEvidenceKind int + +const ( + retainedEvidenceNone retainedEvidenceKind = iota + retainedEvidenceCoverage + retainedEvidenceDocuments +) + +func (r *SemanticContributionRepository) retainedSourceEvidence(ctx context.Context, project, generation, resourceType string) (retainedEvidenceKind, error) { + documents, err := r.scopedEvidenceExists(ctx, r.fieldProfileFQN, project, generation, resourceType, + " AND record_type = '' AND doc_count > 0") + if err != nil { + return retainedEvidenceNone, err + } + if documents { + return retainedEvidenceDocuments, nil + } + coverage, err := r.scopedEvidenceExists(ctx, r.fieldProfileCovFQN, project, generation, resourceType, "") + if err != nil { + return retainedEvidenceNone, err + } + if coverage { + return retainedEvidenceCoverage, nil + } + return retainedEvidenceNone, nil +} + +func (r *SemanticContributionRepository) scopedEvidenceExists(ctx context.Context, fqn, project, generation, resourceType, extra string) (bool, error) { + exists, err := r.tableExists(ctx, fqn) + if err != nil { + return false, err + } + if !exists { + return false, nil + } + var found bool + err = r.withConn(ctx, func(conn *pgxpool.Conn) error { + err := conn.QueryRow(ctx, + "SELECT true FROM "+fqn+ + " WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3"+extra+" LIMIT 1", + project, generation, resourceType).Scan(&found) + if errors.Is(err, pgx.ErrNoRows) { + found = false + return nil + } + if err != nil { + if storepostgres.IsUndefinedTable(err) { + found = false + return nil + } + return fmt.Errorf("postgres: probe semantic inventory source evidence %s: %w", fqn, err) + } + found = true + return nil + }) + if err != nil { + return false, err + } + return found, nil +} + +func (r *SemanticContributionRepository) EnsureSemanticInventoryEntries(ctx context.Context, build catalog.SemanticInventoryBuild) (catalog.SemanticInventoryBuild, error) { + if err := validateSemanticInventoryBuildIdentity(build); err != nil { + return catalog.SemanticInventoryBuild{}, err + } + return catalog.SemanticInventoryBuild{}, &SemanticInventoryEntriesUnsupportedError{ + Project: build.Project, + DatasetGeneration: catalog.NormalizeDatasetGeneration(build.DatasetGeneration), + BuildID: build.BuildID, + EntryIndexVersion: build.EntryIndexVersion, + } +} + +func (r *SemanticContributionRepository) tableExists(ctx context.Context, fqn string) (bool, error) { + var exists bool + err := r.withConn(ctx, func(conn *pgxpool.Conn) error { + if err := conn.QueryRow(ctx, "SELECT to_regclass($1) IS NOT NULL", fqn).Scan(&exists); err != nil { + return fmt.Errorf("postgres: check semantic inventory table %s: %w", fqn, err) + } + return nil + }) + if err != nil { + return false, err + } + return exists, nil +} + +func (r *SemanticContributionRepository) withConn(ctx context.Context, fn func(*pgxpool.Conn) error) error { + conn, err := r.src.Acquire(ctx) + if err != nil { + return fmt.Errorf("postgres: acquire semantic contribution connection: %w", err) + } + defer conn.Release() + return fn(conn) +} + +func prepareSemanticContribution(contribution catalog.SemanticInventoryContribution) (preparedContribution, error) { + contribution.DatasetGeneration = catalog.NormalizeDatasetGeneration(contribution.DatasetGeneration) + if !isLowerHex64(contribution.Key) { + return preparedContribution{}, fmt.Errorf("contribution key %q is not 64 lowercase hex characters", contribution.Key) + } + if contribution.Project == "" || contribution.DatasetGeneration == "" || contribution.BuildID == "" || + contribution.SourceID == "" || contribution.ConceptID == "" || contribution.BindingID == "" || + contribution.ResourceType == "" { + return preparedContribution{}, errors.New("contribution is missing stable identity") + } + if contribution.Ordinal < 0 { + return preparedContribution{}, fmt.Errorf("contribution ordinal is %d, want >= 0", contribution.Ordinal) + } + switch contribution.SourceKind { + case catalog.SemanticInventorySourceFile, catalog.SemanticInventorySourceRetained: + default: + return preparedContribution{}, fmt.Errorf("contribution source kind %q is unknown", contribution.SourceKind) + } + if contribution.BuildID != catalog.SemanticInventoryBuildID(contribution.Project, contribution.DatasetGeneration) { + return preparedContribution{}, errors.New("contribution build does not match its generation") + } + observation, err := json.Marshal(contribution.Observation) + if err != nil { + return preparedContribution{}, fmt.Errorf("encode contribution observation: %w", err) + } + if len(observation) == 0 || observation[0] != '{' { + return preparedContribution{}, errors.New("contribution observation does not encode as a JSON object") + } + return preparedContribution{ + contribution: contribution, + observation: observation, + digest: semanticContributionPayloadDigest(contribution, observation), + }, nil +} + +func semanticContributionPayloadDigest(c catalog.SemanticInventoryContribution, observation []byte) string { + h := sha256.New() + put := func(s string) { + h.Write([]byte(strconv.Itoa(len(s)))) + h.Write([]byte{':'}) + h.Write([]byte(s)) + } + put("loom.semantic-contribution/v1\n") + put(c.Project) + put(c.DatasetGeneration) + put(c.BuildID) + put(c.SourceKind) + put(c.AuthResourcePath) + put(c.ResourceType) + put(c.SourceID) + put("o" + strconv.Itoa(c.Ordinal)) + put(c.ConceptID) + put(c.BindingID) + put(c.Example) + put(string(observation)) + return hex.EncodeToString(h.Sum(nil)) +} + +func semanticContributionSlotKey(c catalog.SemanticInventoryContribution) string { + frame := func(s string) string { return strconv.Itoa(len(s)) + ":" + s } + return frame(c.Project) + frame(c.DatasetGeneration) + frame(c.BuildID) + + frame(c.SourceKind) + frame(c.SourceID) + frame(strconv.Itoa(c.Ordinal)) +} + +func seenSlotFirst(prepared []preparedContribution, slotKey string) catalog.SemanticInventoryContribution { + for _, p := range prepared { + if semanticContributionSlotKey(p.contribution) == slotKey { + return p.contribution + } + } + return catalog.SemanticInventoryContribution{} +} + +func isLowerHex64(s string) bool { + if len(s) != 64 { + return false + } + for i := 0; i < len(s); i++ { + ch := s[i] + if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') { + return false + } + } + return true +} + +type SemanticInventoryBackfillStore struct { + contributions *SemanticContributionRepository + builds *SemanticBuildRepository +} + +func NewSemanticInventoryBackfillStore(contributions *SemanticContributionRepository, builds *SemanticBuildRepository) (*SemanticInventoryBackfillStore, error) { + if contributions == nil { + return nil, errors.New("postgres: nil semantic contribution repository") + } + if builds == nil { + return nil, errors.New("postgres: nil semantic build repository") + } + return &SemanticInventoryBackfillStore{contributions: contributions, builds: builds}, nil +} + +func (s *SemanticInventoryBackfillStore) PrepareSemanticInventoryBackfill(ctx context.Context) error { + return s.contributions.PrepareSemanticInventoryBackfill(ctx) +} + +func (s *SemanticInventoryBackfillStore) ReadRetainedSemanticInventoryPage(ctx context.Context, project, generation, collection, afterKey string, limit int) (catalog.RetainedSemanticInventoryPage, error) { + return s.contributions.ReadRetainedSemanticInventoryPage(ctx, project, generation, collection, afterKey, limit) +} + +func (s *SemanticInventoryBackfillStore) ClaimSemanticInventoryBackfill(ctx context.Context, build catalog.SemanticInventoryBuild, token string, lease time.Duration) (catalog.SemanticInventoryBuild, bool, error) { + out, claimed, err := s.builds.ClaimSemanticInventoryBackfill(ctx, build, token, lease) + if isConcurrentSemanticBuildKeyRace(err) { + out, claimed, err = s.builds.ClaimSemanticInventoryBackfill(ctx, build, token, lease) + } + return out, claimed, err +} + +func isConcurrentSemanticBuildKeyRace(err error) bool { + return storepostgres.IsUniqueConstraintViolation(err, semanticInventoryBuildTableName+"_key_ux") +} + +func (s *SemanticInventoryBackfillStore) AdvanceSemanticInventoryBackfill(ctx context.Context, build catalog.SemanticInventoryBuild, token string, expected, next catalog.SemanticInventoryCheckpoint, scanned int64, lease time.Duration) (catalog.SemanticInventoryBuild, error) { + return s.builds.AdvanceSemanticInventoryBackfill(ctx, build, token, expected, next, scanned, lease) +} + +func (s *SemanticInventoryBackfillStore) CompleteSemanticInventoryBackfill(ctx context.Context, build catalog.SemanticInventoryBuild, token string, expected catalog.SemanticInventoryCheckpoint, scanned int64) (catalog.SemanticInventoryBuild, error) { + return s.builds.CompleteSemanticInventoryBackfill(ctx, build, token, expected, scanned) +} + +func (s *SemanticInventoryBackfillStore) FailSemanticInventoryBackfill(ctx context.Context, build catalog.SemanticInventoryBuild, token, diagnostic string) error { + return s.builds.FailSemanticInventoryBackfill(ctx, build, token, diagnostic) +} + +func (s *SemanticInventoryBackfillStore) WriteSemanticInventoryContributions(ctx context.Context, contributions []catalog.SemanticInventoryContribution, batchSize int) error { + return s.contributions.WriteSemanticInventoryContributions(ctx, contributions, batchSize) +} + +func (s *SemanticInventoryBackfillStore) EnsureSemanticInventoryEntries(ctx context.Context, build catalog.SemanticInventoryBuild) (catalog.SemanticInventoryBuild, error) { + return s.contributions.EnsureSemanticInventoryEntries(ctx, build) +} diff --git a/internal/backends/postgres/internal/catalog/semantic_contribution_test.go b/internal/backends/postgres/internal/catalog/semantic_contribution_test.go new file mode 100644 index 000000000..a646ad843 --- /dev/null +++ b/internal/backends/postgres/internal/catalog/semantic_contribution_test.go @@ -0,0 +1,2285 @@ +package postgres_test + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + catalogstore "github.com/calypr/loom/internal/backends/postgres/internal/catalog" + "net/url" + "os" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + publication "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/ingest" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +var pgcat05NamePattern = regexp.MustCompile(`^verify_pgcat05_it_[0-9]+_[0-9a-f]+$`) + +var ( + pgcat05OwnedMu sync.Mutex + pgcat05OwnedByName = map[string]string{} +) + +func pgcat05Register(db, schema string) { + pgcat05OwnedMu.Lock() + defer pgcat05OwnedMu.Unlock() + pgcat05OwnedByName[db] = schema +} + +type pgcat05Service struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func pgcat05TestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 integration tests") + } + return raw +} + +func pgcat05RandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) +} + +func newPgcat05Service(t *testing.T) *pgcat05Service { + t.Helper() + raw := pgcat05TestPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var dbName, schema string + + dbName = fmt.Sprintf("verify_pgcat05_it_%d_%s", time.Now().UnixNano(), pgcat05RandomHex(t)) + if !pgcat05NamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + pgcat05Register(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema = fmt.Sprintf("verify_pgcat05_it_%d_%s", time.Now().UnixNano(), pgcat05RandomHex(t)) + if !pgcat05NamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + pgcat05Register(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + service := &pgcat05Service{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} + if _, err := storepostgres.MigrateSchema(ctx, service.openClient(t), schema); err != nil { + fail("MigrateSchema", err) + } + return service +} + +func (s *pgcat05Service) openClient(t *testing.T) *storepostgres.Client { + t.Helper() + client, err := storepostgres.Open(s.ctx, storepostgres.Config{ + DSN: s.dsn, + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *pgcat05Service) openDedicatedClient(t *testing.T, label string) (*storepostgres.Client, int32) { + t.Helper() + client, err := storepostgres.Open(s.ctx, storepostgres.Config{ + DSN: s.dsn, + MaxConns: 1, + MinConns: 1, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + s.fail("open dedicated client "+label, err) + } + t.Cleanup(client.Close) + conn, err := client.Acquire(s.ctx) + if err != nil { + s.fail("acquire dedicated connection "+label, err) + } + defer conn.Release() + var pid int32 + if err := conn.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + s.fail("witness pg_backend_pid "+label, err) + } + if pid <= 0 { + t.Fatalf("pg_backend_pid() for %s returned %d, want a live backend", label, pid) + } + return client, pid +} + +func (s *pgcat05Service) contributionRepository(t *testing.T) *catalogstore.SemanticContributionRepository { + t.Helper() + repo, err := catalogstore.NewSemanticContributionRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new semantic contribution repository", err) + } + return repo +} + +func (s *pgcat05Service) buildRepository(t *testing.T) *catalogstore.SemanticBuildRepository { + t.Helper() + repo, err := catalogstore.NewSemanticBuildRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new semantic build repository", err) + } + return repo +} + +func (s *pgcat05Service) backfillStore(t *testing.T) *catalogstore.SemanticInventoryBackfillStore { + t.Helper() + store, err := catalogstore.NewSemanticInventoryBackfillStore(s.contributionRepository(t), s.buildRepository(t)) + if err != nil { + s.fail("new semantic inventory backfill store", err) + } + return store +} + +func (s *pgcat05Service) exec(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := s.iso.Exec(s.ctx, sql, args...); err != nil { + s.fail("fixture exec", err) + } +} + +func (s *pgcat05Service) contributionFQN() string { + return pgx.Identifier{s.schema, "fhir_semantic_inventory_contribution"}.Sanitize() +} + +func (s *pgcat05Service) resourceFQN() string { + return pgx.Identifier{s.schema, "fhir_resource"}.Sanitize() +} + +func (s *pgcat05Service) buildFQN() string { + return pgx.Identifier{s.schema, "fhir_semantic_inventory_build"}.Sanitize() +} + +func (s *pgcat05Service) fieldProfileFQN() string { + return pgx.Identifier{s.schema, "fhir_field_profile"}.Sanitize() +} + +func (s *pgcat05Service) coverageFQN() string { + return pgx.Identifier{s.schema, "fhir_field_profile_coverage"}.Sanitize() +} + +func (s *pgcat05Service) ledgerFQN() string { + return pgx.Identifier{s.schema, storepostgres.LedgerTable}.Sanitize() +} + +type contributionLiteral struct { + Key string + Project string + DatasetGeneration string + BuildID string + SourceKind string + AuthResourcePath string + ResourceType string + SourceID string + Ordinal int + ConceptID string + BindingID string + Example string + PayloadDigest string + Observation string +} + +type contributionWitness struct { + contributionLiteral + RecordedAt time.Time +} + +func pgcat05CanonicalJSON(s string) string { + var v any + if err := json.Unmarshal([]byte(s), &v); err != nil { + return s + } + b, err := json.Marshal(v) + if err != nil { + return s + } + return string(b) +} + +func (l contributionLiteral) normalized() contributionLiteral { + l.Observation = pgcat05CanonicalJSON(l.Observation) + return l +} + +func (w contributionWitness) normalized() contributionWitness { + w.Observation = pgcat05CanonicalJSON(w.Observation) + return w +} + +func sameContributionLiteral(a, b contributionLiteral) bool { + return a.normalized() == b.normalized() +} + +func (w contributionWitness) sameAs(o contributionWitness) bool { + return fmt.Sprint(w.normalized()) == fmt.Sprint(o.normalized()) +} + +const contributionWitnessColumns = "contribution_key, project, dataset_generation, build_id, source_kind," + + " auth_resource_path, resource_type, source_id, ordinal, concept_id, binding_id, example," + + " payload_digest, observation::text, recorded_at" + +func (s *pgcat05Service) witnessContribution(t *testing.T, key string) (contributionWitness, bool) { + t.Helper() + var w contributionWitness + err := s.iso.QueryRow(s.ctx, "SELECT "+contributionWitnessColumns+" FROM "+s.contributionFQN()+" WHERE contribution_key = $1", key).Scan( + &w.Key, &w.Project, &w.DatasetGeneration, &w.BuildID, &w.SourceKind, + &w.AuthResourcePath, &w.ResourceType, &w.SourceID, &w.Ordinal, &w.ConceptID, + &w.BindingID, &w.Example, &w.PayloadDigest, &w.Observation, &w.RecordedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return contributionWitness{}, false + } + if err != nil { + s.fail("witness contribution row", err) + } + return w, true +} + +func (s *pgcat05Service) requireNoContribution(t *testing.T, key, what string) { + t.Helper() + if row, found := s.witnessContribution(t, key); found { + t.Fatalf("%s: unexpected row %+v", what, row) + } +} + +func (s *pgcat05Service) countContributions(t *testing.T) int64 { + t.Helper() + var count int64 + if err := s.iso.QueryRow(s.ctx, "SELECT COUNT(*) FROM "+s.contributionFQN()).Scan(&count); err != nil { + s.fail("count contributions", err) + } + return count +} + +func (s *pgcat05Service) contributionSeedSQL() string { + return "INSERT INTO " + s.contributionFQN() + + " (contribution_key, project, dataset_generation, build_id, source_kind," + + " auth_resource_path, resource_type, source_id, ordinal, concept_id, binding_id," + + " example, payload_digest, observation)" + + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14::jsonb)" +} + +func (s *pgcat05Service) seedContributionLiteral(t *testing.T, l contributionLiteral) { + t.Helper() + s.exec(t, s.contributionSeedSQL(), l.Key, l.Project, l.DatasetGeneration, l.BuildID, l.SourceKind, + l.AuthResourcePath, l.ResourceType, l.SourceID, l.Ordinal, l.ConceptID, l.BindingID, + l.Example, l.PayloadDigest, l.Observation) +} + +func (s *pgcat05Service) trySeedContributionLiteral(t *testing.T, l contributionLiteral) error { + t.Helper() + _, err := s.iso.Exec(s.ctx, s.contributionSeedSQL(), l.Key, l.Project, l.DatasetGeneration, l.BuildID, + l.SourceKind, l.AuthResourcePath, l.ResourceType, l.SourceID, l.Ordinal, l.ConceptID, + l.BindingID, l.Example, l.PayloadDigest, l.Observation) + return err +} + +func (l contributionLiteral) toWitness(recordedAt time.Time) contributionWitness { + return contributionWitness{contributionLiteral: l, RecordedAt: recordedAt} +} + +func (s *pgcat05Service) seedRetainedResource(t *testing.T, project, generation, resourceType, logicalID, authPath string, payload []byte) { + t.Helper() + var payloadValue any + if payload != nil { + payloadValue = []byte(payload) + } + s.exec(t, "INSERT INTO "+s.resourceFQN()+ + " (project, dataset_generation, resource_type, logical_id, auth_resource_path, payload)"+ + " VALUES ($1, $2, $3, $4, $5, $6::jsonb)", + project, generation, resourceType, logicalID, authPath, payloadValue) +} + +func (s *pgcat05Service) seedFieldProfileDoc(t *testing.T, project, generation, authPath, resourceType, path string, docCount int64) { + t.Helper() + s.exec(t, "INSERT INTO "+s.fieldProfileFQN()+ + " (project, dataset_generation, auth_resource_path, resource_type, path, profile_key,"+ + " record_type, kind, doc_count, max_items, sample_count, distinct_truncated,"+ + " enrichment_truncated, pivot_candidate, pivot_kind, pivot_family,"+ + " pivot_column_selector, pivot_value_selector, pivot_item_source, pivot_item_resource_type)"+ + " VALUES ($1, $2, $3, $4, $5, $6, '', $7, $8, 0, 0, false, false, false, '', '', '', '', '', '')", + project, generation, authPath, resourceType, path, "fpk_"+resourceType+"_"+path, "string", docCount) +} + +func (s *pgcat05Service) seedCoverageMarker(t *testing.T, project, generation, authPath, resourceType string) { + t.Helper() + s.exec(t, "INSERT INTO "+s.coverageFQN()+ + " (project, dataset_generation, auth_resource_path, resource_type, enrichment_truncated)"+ + " VALUES ($1, $2, $3, $4, false)", + project, generation, authPath, resourceType) +} + +func (s *pgcat05Service) witnessBuildRow(t *testing.T, project, generation, buildID string) (catalog.SemanticInventoryBuild, bool) { + t.Helper() + repo := s.buildRepository(t) + build, found, err := repo.ReadSemanticInventoryBuild(s.ctx, project, generation, buildID) + if err != nil { + t.Fatalf("witness build row %s/%s/%s: %v", project, generation, buildID, err) + } + return build, found +} + +const ( + pgcat05GoldenProject = "pgcat05-golden" + pgcat05GoldenGeneration = "gen-a" +) + +type goldenContribution struct { + key string + conceptID string + bindingID string + example string + digest string + sourceID string + authPath string + resourceType string + observation string + payload string +} + +var pgcat05ObservationCodeObs = `{"schema_version":4,"source":{"canonical":"Observation.code.coding[]","type":"Observation","path":"code.coding[]"},"key":{"selector":"code.coding[]","system":"urn:obs-05","version":"v1","code":"loinc-42"},"value":{"selector":"code.coding[].display","type":"string","fallback_selectors":["code.coding[].code"],"fallback_on_blank_string":true},"logical_type":"string","completeness":"complete","status":"SUPPORTED","population":1,"examples":["loinc-42"],"examples_truncated":false,"rule_hint":"CODED_CONCEPT_V1","rule_version":"5"}` + +var pgcat05Goldens = map[string]goldenContribution{ + "obs-a": { + key: "800ae5168dd3f11d3ef7dc957d4f89cd572e225812e699a0feb088d8de0c480c", + conceptID: "ec572479ea77eabd0c8014f33810f458e9f009a27adb8fd77fb9fab3121ad62f", + bindingID: "b498c06c58e286f4cc9ceb74942c8a7872ad49b0220e65f0ea754fd936f4a53d", + example: "loinc-42", + digest: "e95374423254d82ba79ee0cc3305e952f34539b36af2a159feccf2281653ac17", + sourceID: "retained:Observation/obs-a", + authPath: "fhir/Observation?code=urn:obs-05|loinc-42", + resourceType: "Observation", + observation: pgcat05ObservationCodeObs, + payload: `{"resourceType":"Observation","code":{"coding":[{"system":"urn:obs-05","version":"v1","code":"loinc-42"}]}}`, + }, + "obs-b": { + key: "52536e2cb03cb6b0ffbd00fd2b519bc540d185317d400dfc9ed033c1991f3327", + conceptID: "ec572479ea77eabd0c8014f33810f458e9f009a27adb8fd77fb9fab3121ad62f", + bindingID: "b498c06c58e286f4cc9ceb74942c8a7872ad49b0220e65f0ea754fd936f4a53d", + example: "loinc-42", + digest: "11f404e19e6be133fc6212b99b2717f43c01a835eab01ca67580ed39aace094e", + sourceID: "retained:Observation/obs-b", + authPath: "", + resourceType: "Observation", + observation: pgcat05ObservationCodeObs, + payload: `{"resourceType":"Observation","code":{"coding":[{"system":"urn:obs-05","version":"v1","code":"loinc-42"}]}}`, + }, + "obs-c": { + key: "443978452a52b2b02b0d11dd9910114eae49143defaaf522312bc8c95a4c0626", + conceptID: "974b766eafbd1eb88624d676ed271537f253d149dc4afb1ae4dbc0c9e2a8f5d7", + bindingID: "699570acfab3e420beae8c57bd32d81f1fca5c4b62482f28a64a62f3ff8457af", + example: "3.5", + digest: "6e5581a2e45fa58099a69662429eb8ac618d515c8b583d0b141815d5b32d0f96", + sourceID: "retained:Observation/obs-c", + authPath: "", + resourceType: "Observation", + observation: `{"schema_version":4,"source":{"canonical":"Observation.code","type":"Observation","path":"code"},"key":{"selector":"code.coding[]","system":"urn:obs-05","version":"v2","code":"loinc-99"},"value":{"selector":"valueQuantity.value","type":"decimal"},"choice_arm":"valueQuantity","logical_type":"decimal","observed_units":["kg"],"completeness":"complete","status":"SUPPORTED","population":1,"examples":["3.5"],"examples_truncated":false,"rule_hint":"CODED_VALUE_V1","rule_version":"5"}`, + payload: `{"resourceType":"Observation","code":{"coding":[{"system":"urn:obs-05","version":"v2","code":"loinc-99"}]},"valueQuantity":{"value":3.5,"unit":"kg"}}`, + }, + "obs-d": { + key: "d75c5c4b985d9f6a8c01fe007901cca521c088cb8869a313e16e098e6acd74b8", + conceptID: "a0987c1620992c6d9df2adba885c019209707cbcd73145837b732ae4b4c884ac", + bindingID: "b498c06c58e286f4cc9ceb74942c8a7872ad49b0220e65f0ea754fd936f4a53d", + example: "loinc-77", + digest: "caf01406cc621e22e085d7cb7795e0e63a10b0a867fc834b5302c26d992b4a28", + sourceID: "retained:Observation/obs-d", + authPath: "fhir/Encounter?type=visit", + resourceType: "Observation", + observation: `{"schema_version":4,"source":{"canonical":"Observation.code.coding[]","type":"Observation","path":"code.coding[]"},"key":{"selector":"code.coding[]","system":"urn:obs-05","version":"v3","code":"loinc-77"},"value":{"selector":"code.coding[].display","type":"string","fallback_selectors":["code.coding[].code"],"fallback_on_blank_string":true},"logical_type":"string","completeness":"complete","status":"SUPPORTED","population":1,"examples":["loinc-77"],"examples_truncated":false,"rule_hint":"CODED_CONCEPT_V1","rule_version":"5"}`, + payload: `{"resourceType":"Observation","code":{"coding":[{"system":"urn:obs-05","version":"v3","code":"loinc-77"}]}}`, + }, + "obs-e": { + key: "7f44d967925c2ff0929492f44522c90119de8532470a8c2336ab5aade9333ef6", + conceptID: "ec572479ea77eabd0c8014f33810f458e9f009a27adb8fd77fb9fab3121ad62f", + bindingID: "b498c06c58e286f4cc9ceb74942c8a7872ad49b0220e65f0ea754fd936f4a53d", + example: "loinc-42", + digest: "3e3d2c7e97641a0224a6dcf7032aaf32e94cdd1d5a8b207d264ddefcc810366f", + sourceID: "retained:Observation/obs-e", + authPath: "", + resourceType: "Observation", + observation: pgcat05ObservationCodeObs, + payload: `{"resourceType":"Observation","code":{"coding":[{"system":"urn:obs-05","version":"v1","code":"loinc-42"}]}}`, + }, + "obs-f": { + key: "33b6804e9386d95119b04eea4767ad781e92106f5307b48600c376775b4975a2", + conceptID: "8c5fa05f46d6a6a401ac7ce93c06dcdb2a209234d181213ec78156e0455bbd37", + bindingID: "699570acfab3e420beae8c57bd32d81f1fca5c4b62482f28a64a62f3ff8457af", + example: "7", + digest: "a4b9c92c27b80c4ac1982436e9448f4171e080ed047b64cacebc9959c8f09b85", + sourceID: "retained:Observation/obs-f", + authPath: "", + resourceType: "Observation", + observation: `{"schema_version":4,"source":{"canonical":"Observation.code","type":"Observation","path":"code"},"key":{"selector":"code.coding[]","system":"urn:obs-05","version":"v9","code":"loinc-12345"},"value":{"selector":"valueQuantity.value","type":"decimal"},"choice_arm":"valueQuantity","logical_type":"decimal","observed_units":["mg/dL"],"completeness":"complete","status":"SUPPORTED","population":1,"examples":["7"],"examples_truncated":false,"rule_hint":"CODED_VALUE_V1","rule_version":"5"}`, + payload: `{"resourceType":"Observation","status":"final","code":{"coding":[{"system":"urn:obs-05","version":"v9","code":"loinc-12345"}]},"valueQuantity":{"value":7,"unit":"mg/dL"}}`, + }, +} + +func pgcat05GoldenIDs() []string { + ids := make([]string, 0, len(pgcat05Goldens)) + for id := range pgcat05Goldens { + ids = append(ids, id) + } + sort.Strings(ids) + return ids +} + +func pgcat05EmitContribution(t *testing.T, project, generation, authPath, resourceType, sourceID, payload string) []catalog.SemanticInventoryContribution { + t.Helper() + var decoded map[string]any + if err := json.Unmarshal([]byte(payload), &decoded); err != nil { + t.Fatalf("decode golden payload: %v", err) + } + emitter := catalog.NewSemanticInventoryEmitter(project, generation) + var out []catalog.SemanticInventoryContribution + if err := emitter.ObservePayload(decoded, resourceType, authPath, sourceID, func(c catalog.SemanticInventoryContribution) { + out = append(out, c) + }); err != nil { + t.Fatalf("emit contributions: %v", err) + } + return out +} + +func pgcat05GoldensBatch(t *testing.T, ids []string) []catalog.SemanticInventoryContribution { + t.Helper() + var batch []catalog.SemanticInventoryContribution + for _, id := range ids { + g := pgcat05Goldens[id] + emitted := pgcat05EmitContribution(t, pgcat05GoldenProject, pgcat05GoldenGeneration, + g.authPath, g.resourceType, g.sourceID, g.payload) + if len(emitted) != 1 { + t.Fatalf("golden %s: emitter produced %d contributions, want exactly 1", id, len(emitted)) + } + batch = append(batch, emitted[0]) + } + return batch +} + +func pgcat05ExpectedLiteral(g goldenContribution) contributionLiteral { + return contributionLiteral{ + Key: g.key, Project: pgcat05GoldenProject, DatasetGeneration: pgcat05GoldenGeneration, + BuildID: catalog.SemanticInventoryBuildID(pgcat05GoldenProject, pgcat05GoldenGeneration), + SourceKind: catalog.SemanticInventorySourceRetained, + AuthResourcePath: g.authPath, ResourceType: g.resourceType, SourceID: g.sourceID, + Ordinal: 0, ConceptID: g.conceptID, BindingID: g.bindingID, Example: g.example, + PayloadDigest: g.digest, Observation: g.observation, + } +} + +func TestSemanticContributionEmitterGoldenAnchor(t *testing.T) { + for _, id := range pgcat05GoldenIDs() { + g := pgcat05Goldens[id] + emitted := pgcat05EmitContribution(t, pgcat05GoldenProject, pgcat05GoldenGeneration, + g.authPath, g.resourceType, g.sourceID, g.payload) + if len(emitted) != 1 { + t.Fatalf("%s: emitter produced %d contributions, want exactly 1", id, len(emitted)) + } + c := emitted[0] + if c.Key != g.key { + t.Fatalf("%s: contribution key = %s, golden %s", id, c.Key, g.key) + } + if c.ConceptID != g.conceptID || c.BindingID != g.bindingID || c.Example != g.example { + t.Fatalf("%s: concept/binding/example = %s/%s/%q, golden %s/%s/%q", + id, c.ConceptID, c.BindingID, c.Example, g.conceptID, g.bindingID, g.example) + } + observation, err := json.Marshal(c.Observation) + if err != nil { + t.Fatalf("%s: encode observation: %v", id, err) + } + var goldenObs catalog.SemanticObservation + if err := json.Unmarshal([]byte(g.observation), &goldenObs); err != nil { + t.Fatalf("%s: decode golden observation: %v", id, err) + } + if !reflect.DeepEqual(c.Observation, goldenObs) { + t.Fatalf("%s: observation %#v differs from golden %#v", id, c.Observation, goldenObs) + } + if digest := semanticContributionPayloadDigest(c, observation); digest != g.digest { + t.Fatalf("%s: payload digest = %s, golden %s", id, digest, g.digest) + } + } +} + +func TestSemanticContributionWriteRoundTripsEveryColumn(t *testing.T) { + s := newPgcat05Service(t) + repo := s.contributionRepository(t) + + ids := pgcat05GoldenIDs() + if err := repo.WriteSemanticInventoryContributions(s.ctx, pgcat05GoldensBatch(t, ids), 0); err != nil { + s.fail("write golden batch", err) + } + if got := s.countContributions(t); got != int64(len(ids)) { + t.Fatalf("contribution count = %d, want %d", got, len(ids)) + } + for _, id := range ids { + g := pgcat05Goldens[id] + w, found := s.witnessContribution(t, g.key) + if !found { + t.Fatalf("%s: contribution %s missing after write", id, g.key) + } + want := pgcat05ExpectedLiteral(g) + if !sameContributionLiteral(w.contributionLiteral, want) { + t.Fatalf("%s: stored row = %+v, want %+v", id, w.contributionLiteral, want) + } + var equal bool + if err := s.iso.QueryRow(s.ctx, "SELECT observation = $1::jsonb FROM "+s.contributionFQN()+ + " WHERE contribution_key = $2", g.observation, g.key).Scan(&equal); err != nil { + s.fail("witness observation jsonb equality", err) + } + if !equal { + t.Fatalf("%s: stored observation is not jsonb-equal to the golden literal: %s", id, w.Observation) + } + if w.RecordedAt.IsZero() { + t.Fatalf("%s: recorded_at is zero", id) + } + } +} + +func TestSemanticContributionReplayIsIdempotentByteIdentical(t *testing.T) { + s := newPgcat05Service(t) + repo := s.contributionRepository(t) + + ids := pgcat05GoldenIDs() + batch := pgcat05GoldensBatch(t, ids) + if err := repo.WriteSemanticInventoryContributions(s.ctx, batch, 2); err != nil { + s.fail("initial write", err) + } + before := map[string]contributionWitness{} + for _, id := range ids { + w, found := s.witnessContribution(t, pgcat05Goldens[id].key) + if !found { + t.Fatalf("missing row after initial write: %s", id) + } + before[id] = w + } + + if err := repo.WriteSemanticInventoryContributions(s.ctx, pgcat05GoldensBatch(t, ids), 0); err != nil { + s.fail("replay write", err) + } + if got := s.countContributions(t); got != int64(len(ids)) { + t.Fatalf("count after replay = %d, want %d (replay must not duplicate)", got, len(ids)) + } + for _, id := range ids { + after, found := s.witnessContribution(t, pgcat05Goldens[id].key) + if !found { + t.Fatalf("row vanished after replay: %s", id) + } + if !after.sameAs(before[id]) { + t.Fatalf("%s: replay mutated the row:\n before %+v\n after %+v", id, before[id], after) + } + } +} + +func TestSemanticContributionConflictFailsWithoutOverwrite(t *testing.T) { + s := newPgcat05Service(t) + repo := s.contributionRepository(t) + + if err := repo.WriteSemanticInventoryContributions(s.ctx, pgcat05GoldensBatch(t, []string{"obs-a"}), 0); err != nil { + s.fail("initial write", err) + } + before, found := s.witnessContribution(t, pgcat05Goldens["obs-a"].key) + if !found { + t.Fatal("seed row missing") + } + + tampered := pgcat05GoldensBatch(t, []string{"obs-a"})[0] + tampered.Example = "tampered-example" + tampered.Observation.Examples = []string{"tampered-example"} + + err := repo.WriteSemanticInventoryContributions(s.ctx, []catalog.SemanticInventoryContribution{tampered}, 0) + var conflict *catalogstore.SemanticInventoryContributionConflictError + if !errors.As(err, &conflict) || !errors.Is(err, catalogstore.ErrSemanticInventoryContributionConflict) { + t.Fatalf("conflicting write error = %v, want *SemanticInventoryContributionConflictError", err) + } + if conflict.ContributionKey != pgcat05Goldens["obs-a"].key || conflict.Slot { + t.Fatalf("conflict misattributed: %+v", conflict) + } + if conflict.StoredDigest != pgcat05Goldens["obs-a"].digest { + t.Fatalf("conflict names stored digest %s, want golden %s", conflict.StoredDigest, pgcat05Goldens["obs-a"].digest) + } + if conflict.ProvidedDigest == conflict.StoredDigest || !isLowerHex64(conflict.ProvidedDigest) { + t.Fatalf("conflict provided digest = %q, want a differing 64-hex digest", conflict.ProvidedDigest) + } + for _, other := range []error{catalogstore.ErrInvalidSemanticInventoryContribution, catalogstore.ErrSemanticInventoryEntriesUnsupported, catalogstore.ErrRetainedSemanticSourceMissing} { + if errors.Is(err, other) { + t.Fatalf("conflict error misclassifies as %v", other) + } + } + + after, found := s.witnessContribution(t, pgcat05Goldens["obs-a"].key) + if !found { + t.Fatal("conflicting write removed the stored row") + } + if !after.sameAs(before) { + t.Fatalf("conflicting payload overwrote the stored row:\n before %+v\n after %+v", before, after) + } + if got := s.countContributions(t); got != 1 { + t.Fatalf("count after refused conflict = %d, want 1", got) + } +} + +func TestSemanticContributionSlotConflictRefusedWithoutOverwrite(t *testing.T) { + s := newPgcat05Service(t) + repo := s.contributionRepository(t) + + golden := pgcat05Goldens["obs-a"] + s.seedContributionLiteral(t, pgcat05ExpectedLiteral(golden)) + + claimed := pgcat05GoldensBatch(t, []string{"obs-a"})[0] + claimed.AuthResourcePath = "fhir/Observation?code=urn:other" + claimed.Key = strings.Repeat("7", 64) + claimed.Example = golden.example + + err := repo.WriteSemanticInventoryContributions(s.ctx, []catalog.SemanticInventoryContribution{claimed}, 0) + var conflict *catalogstore.SemanticInventoryContributionConflictError + if !errors.As(err, &conflict) || !conflict.Slot { + t.Fatalf("slot conflict error = %v, want a slot-classified conflict", err) + } + if conflict.SourceID != golden.sourceID || conflict.Ordinal != 0 { + t.Fatalf("slot conflict misattributed: %+v", conflict) + } + s.requireNoContribution(t, claimed.Key, "slot-conflicting row") + after, found := s.witnessContribution(t, golden.key) + if !found || !after.sameAs(pgcat05ExpectedLiteral(golden).toWitness(after.RecordedAt)) { + t.Fatalf("slot conflict disturbed the stored row: %+v found=%t", after, found) + } + if !sameContributionLiteral(after.contributionLiteral, pgcat05ExpectedLiteral(golden)) { + t.Fatalf("slot conflict mutated stored columns: %+v", after) + } +} + +func TestSemanticContributionConstraintNegatives(t *testing.T) { + s := newPgcat05Service(t) + + base := func(suffix string) contributionLiteral { + g := pgcat05Goldens["obs-b"] + l := pgcat05ExpectedLiteral(g) + l.Key = fmt.Sprintf("%x", sha256.Sum256([]byte("pgcat05-base/"+suffix))) + l.SourceID = "retained:Observation/neg-" + suffix + l.PayloadDigest = fmt.Sprintf("%064x", len(suffix)+0xbeef0000) + return l + } + const ( + checkViolation = "23514" + uniqueViolation = "23505" + notNullViolation = "23502" + ) + cases := []struct { + name string + mutate func(*contributionLiteral) + wantCode string + wantConstraint string + }{ + {"malformed contribution_key", func(l *contributionLiteral) { l.Key = "key_short" }, checkViolation, "fhir_semantic_inventory_contribution_key_shape"}, + {"uppercase contribution_key", func(l *contributionLiteral) { l.Key = strings.ToUpper(l.Key[:32]) + l.Key[32:] }, checkViolation, "fhir_semantic_inventory_contribution_key_shape"}, + {"malformed payload_digest", func(l *contributionLiteral) { l.PayloadDigest = strings.Repeat("z", 64) }, checkViolation, "fhir_semantic_inventory_contribution_digest_shape"}, + {"malformed build_id", func(l *contributionLiteral) { l.BuildID = strings.Repeat("g", 64) }, checkViolation, "fhir_semantic_inventory_contribution_build_shape"}, + {"blank build_id", func(l *contributionLiteral) { l.BuildID = "" }, checkViolation, "fhir_semantic_inventory_contribution_build_present"}, + {"blank project", func(l *contributionLiteral) { l.Project = "" }, checkViolation, "fhir_semantic_inventory_contribution_project_present"}, + {"blank resource_type", func(l *contributionLiteral) { l.ResourceType = "" }, checkViolation, "fhir_semantic_inventory_contribution_resource_type_present"}, + {"blank source_id", func(l *contributionLiteral) { l.SourceID = "" }, checkViolation, "fhir_semantic_inventory_contribution_source_present"}, + {"blank concept_id", func(l *contributionLiteral) { l.ConceptID = "" }, checkViolation, "fhir_semantic_inventory_contribution_concept_present"}, + {"blank binding_id", func(l *contributionLiteral) { l.BindingID = "" }, checkViolation, "fhir_semantic_inventory_contribution_binding_present"}, + {"negative ordinal", func(l *contributionLiteral) { l.Ordinal = -1 }, checkViolation, "fhir_semantic_inventory_contribution_ordinal_nonnegative"}, + {"unknown source kind", func(l *contributionLiteral) { l.SourceKind = "stream" }, checkViolation, "fhir_semantic_inventory_contribution_source_kind_known"}, + {"observation is a json array", func(l *contributionLiteral) { l.Observation = `[{"a":1}]` }, checkViolation, "fhir_semantic_inventory_contribution_observation_object"}, + {"observation is a json scalar", func(l *contributionLiteral) { l.Observation = `"observation"` }, checkViolation, "fhir_semantic_inventory_contribution_observation_object"}, + } + for i, tc := range cases { + bad := base(fmt.Sprintf("neg%d", i)) + tc.mutate(&bad) + control := base(fmt.Sprintf("ctrl%d", i)) + s.seedContributionLiteral(t, control) + if _, found := s.witnessContribution(t, control.Key); !found { + t.Fatalf("%s: control row did not insert — the harness is not proving anything", tc.name) + } + err := s.trySeedContributionLiteral(t, bad) + var pgErr *pgconn.PgError + if err == nil { + t.Fatalf("%s: violating row was ACCEPTED — constraint missing or renamed", tc.name) + } + if !errors.As(err, &pgErr) { + t.Fatalf("%s: violating row failed with a non-database error: %v", tc.name, err) + } + if pgErr.Code != tc.wantCode || pgErr.ConstraintName != tc.wantConstraint { + t.Fatalf("%s: rejected by code=%s constraint=%s, want code=%s constraint=%s", + tc.name, pgErr.Code, pgErr.ConstraintName, tc.wantCode, tc.wantConstraint) + } + s.requireNoContribution(t, bad.Key, tc.name) + } + + notNull := base("null-obs") + _, err := s.iso.Exec(s.ctx, "INSERT INTO "+s.contributionFQN()+ + " (contribution_key, project, dataset_generation, build_id, source_kind,"+ + " auth_resource_path, resource_type, source_id, ordinal, concept_id, binding_id,"+ + " example, payload_digest, observation)"+ + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, NULL)", + notNull.Key, notNull.Project, notNull.DatasetGeneration, notNull.BuildID, notNull.SourceKind, + notNull.AuthResourcePath, notNull.ResourceType, notNull.SourceID, notNull.Ordinal, + notNull.ConceptID, notNull.BindingID, notNull.Example, notNull.PayloadDigest) + var pgErr *pgconn.PgError + if err == nil || !errors.As(err, &pgErr) || pgErr.Code != notNullViolation || + pgErr.ColumnName != "observation" { + t.Fatalf("NULL observation accepted or misattributed: %v", err) + } + s.requireNoContribution(t, notNull.Key, "NULL observation") + + dupKey := base("dup-key") + s.seedContributionLiteral(t, dupKey) + stolen := dupKey + stolen.SourceID = "retained:Observation/stolen-slot" + stolen.PayloadDigest = fmt.Sprintf("%064x", 0x1234) + err = s.trySeedContributionLiteral(t, stolen) + if err == nil || !errors.As(err, &pgErr) || pgErr.Code != uniqueViolation || + pgErr.ConstraintName != "fhir_semantic_inventory_contribution_key_ux" { + t.Fatalf("reused contribution_key accepted or misattributed: %v", err) + } + if row, found := s.witnessContribution(t, dupKey.Key); !found || row.PayloadDigest != dupKey.PayloadDigest { + t.Fatalf("reused key disturbed the stored row: %+v found=%t", row, found) + } + + dupSlot := base("dup-slot") + s.seedContributionLiteral(t, dupSlot) + rewritten := dupSlot + rewritten.Key = fmt.Sprintf("%064x", 0x9999) + rewritten.PayloadDigest = fmt.Sprintf("%064x", 0x7777) + err = s.trySeedContributionLiteral(t, rewritten) + if err == nil || !errors.As(err, &pgErr) || pgErr.Code != uniqueViolation || + pgErr.ConstraintName != "fhir_semantic_inventory_contribution_pkey" { + t.Fatalf("second key at one slot accepted or misattributed: %v", err) + } + if row, found := s.witnessContribution(t, dupSlot.Key); !found || row.PayloadDigest != dupSlot.PayloadDigest { + t.Fatalf("duplicate slot replaced the stored row: %+v found=%t", row, found) + } +} + +func TestSemanticContributionMigrationContract(t *testing.T) { + s := newPgcat05Service(t) + + const goldenChecksum = "41579b3c86d17c5912195ccaa6ff8655e6a864c01a01f2a23fa1cb4aa6883b02" + + var version int64 + var name, checksum string + var rowCount int + err := s.iso.QueryRow(s.ctx, + "SELECT COUNT(*) FROM "+s.ledgerFQN()+" WHERE version = 9 AND name = 'semantic_inventory_contribution'").Scan(&rowCount) + if err != nil { + s.fail("read ledger", err) + } + if rowCount != 1 { + t.Fatalf("ledger rows for version 9 = %d, want exactly 1", rowCount) + } + if err := s.iso.QueryRow(s.ctx, + "SELECT version, name, checksum FROM "+s.ledgerFQN()+" WHERE version = 9").Scan(&version, &name, &checksum); err != nil { + s.fail("read ledger row", err) + } + if version != 9 || name != "semantic_inventory_contribution" { + t.Fatalf("ledger row = (%d, %q), want (9, %q)", version, name, "semantic_inventory_contribution") + } + if checksum != goldenChecksum { + t.Fatalf("000009 checksum = %s, golden %s", checksum, goldenChecksum) + } + + before := s.witnessLedger(t) + if _, err := storepostgres.MigrateSchema(s.ctx, s.openClient(t), s.schema); err != nil { + s.fail("second MigrateSchema", err) + } + after := s.witnessLedger(t) + if fmt.Sprint(before) != fmt.Sprint(after) { + t.Fatalf("double apply mutated the ledger:\n before %v\n after %v", before, after) + } +} + +func (s *pgcat05Service) witnessLedger(t *testing.T) []storepostgres.LedgerRecord { + t.Helper() + rows, err := s.iso.Query(s.ctx, "SELECT version, name, checksum FROM "+s.ledgerFQN()+" ORDER BY version") + if err != nil { + s.fail("witness ledger", err) + } + defer rows.Close() + var out []storepostgres.LedgerRecord + for rows.Next() { + var rec storepostgres.LedgerRecord + if err := rows.Scan(&rec.Version, &rec.Name, &rec.Checksum); err != nil { + s.fail("scan ledger", err) + } + out = append(out, rec) + } + if err := rows.Err(); err != nil { + s.fail("witness ledger", err) + } + return out +} + +func TestSemanticContributionWriteValidationRefusals(t *testing.T) { + s := newPgcat05Service(t) + repo := s.contributionRepository(t) + + base := pgcat05GoldensBatch(t, []string{"obs-a"})[0] + mutate := func(fn func(*catalog.SemanticInventoryContribution)) catalog.SemanticInventoryContribution { + c := base + fn(&c) + return c + } + cases := []struct { + name string + bad catalog.SemanticInventoryContribution + match func(*testing.T, error) + }{ + {"blank key", mutate(func(c *catalog.SemanticInventoryContribution) { c.Key = "" }), wantInvalidContribution}, + {"short key", mutate(func(c *catalog.SemanticInventoryContribution) { c.Key = c.Key[:63] }), wantInvalidContribution}, + {"uppercase key", mutate(func(c *catalog.SemanticInventoryContribution) { c.Key = strings.ToUpper(c.Key) }), wantInvalidContribution}, + {"blank project", mutate(func(c *catalog.SemanticInventoryContribution) { c.Project = "" }), wantInvalidContribution}, + {"blank generation", mutate(func(c *catalog.SemanticInventoryContribution) { c.DatasetGeneration = " " }), wantInvalidContribution}, + {"blank source id", mutate(func(c *catalog.SemanticInventoryContribution) { c.SourceID = "" }), wantInvalidContribution}, + {"blank concept", mutate(func(c *catalog.SemanticInventoryContribution) { c.ConceptID = "" }), wantInvalidContribution}, + {"blank binding", mutate(func(c *catalog.SemanticInventoryContribution) { c.BindingID = "" }), wantInvalidContribution}, + {"blank resource type", mutate(func(c *catalog.SemanticInventoryContribution) { c.ResourceType = "" }), wantInvalidContribution}, + {"negative ordinal", mutate(func(c *catalog.SemanticInventoryContribution) { c.Ordinal = -1 }), wantInvalidContribution}, + {"unknown source kind", mutate(func(c *catalog.SemanticInventoryContribution) { c.SourceKind = "stream" }), wantInvalidContribution}, + {"build/generation mismatch", mutate(func(c *catalog.SemanticInventoryContribution) { c.BuildID = strings.Repeat("a", 64) }), wantInvalidContribution}, + } + for _, tc := range cases { + err := repo.WriteSemanticInventoryContributions(s.ctx, []catalog.SemanticInventoryContribution{tc.bad}, 0) + tc.match(t, err) + if errors.Is(err, catalogstore.ErrSemanticInventoryContributionConflict) || + errors.Is(err, catalogstore.ErrSemanticInventoryEntriesUnsupported) || + errors.Is(err, catalogstore.ErrRetainedSemanticSourceMissing) { + t.Fatalf("%s: validation refusal misclassifies: %v", tc.name, err) + } + } + if got := s.countContributions(t); got != 0 { + t.Fatalf("validation refusals wrote %d rows, want 0 (refusal must precede any statement)", got) + } + + dup := pgcat05GoldensBatch(t, []string{"obs-a"}) + second := dup[0] + second.Key = strings.Repeat("e", 64) + err := repo.WriteSemanticInventoryContributions(s.ctx, []catalog.SemanticInventoryContribution{dup[0], second}, 0) + var conflict *catalogstore.SemanticInventoryContributionConflictError + if !errors.As(err, &conflict) || !conflict.Slot { + t.Fatalf("in-batch duplicate slot error = %v, want slot conflict", err) + } + if got := s.countContributions(t); got != 0 { + t.Fatalf("in-batch duplicate wrote %d rows, want 0", got) + } + + if err := repo.WriteSemanticInventoryContributions(s.ctx, pgcat05GoldensBatch(t, pgcat05GoldenIDs()), 0); err != nil { + s.fail("control write", err) + } + if got := s.countContributions(t); got != int64(len(pgcat05Goldens)) { + t.Fatalf("control count = %d, want %d", got, len(pgcat05Goldens)) + } +} + +func wantInvalidContribution(t *testing.T, err error) { + t.Helper() + if !errors.Is(err, catalogstore.ErrInvalidSemanticInventoryContribution) { + t.Fatalf("validation error = %v, want ErrInvalidSemanticInventoryContribution", err) + } +} + +var pgcat05AdversarialIDs = []string{ + "Zebra", "apple", "Apple", "10", "2", "-x", "é", + "obs-1", "obs-10", "obs-2", "obs.a", "obs~t", "obscure", "obs_c", + "z-null", "z-jsonnull", +} + +func TestSemanticContributionRetainedPageStableByteOrderKeyset(t *testing.T) { + s := newPgcat05Service(t) + repo := s.contributionRepository(t) + + const project, generation, resourceType = "pgcat05-pages", "gen-p", "Observation" + payload := []byte(pgcat05Goldens["obs-b"].payload) + for _, id := range pgcat05AdversarialIDs { + var rowPayload []byte + switch id { + case "z-null": + rowPayload = nil + case "z-jsonnull": + rowPayload = []byte("null") + default: + rowPayload = payload + } + s.seedRetainedResource(t, project, generation, resourceType, id, "path/"+id, rowPayload) + } + for _, id := range pgcat05AdversarialIDs { + s.seedRetainedResource(t, project+"-other", generation, resourceType, id, "decoy", payload) + s.seedRetainedResource(t, project, generation+"-other", resourceType, id, "decoy", payload) + s.seedRetainedResource(t, project, generation, "Patient", id, "decoy", payload) + } + + var defaultOrder []string + { + rows, err := s.iso.Query(s.ctx, "SELECT logical_id FROM "+s.resourceFQN()+ + " WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3 ORDER BY logical_id", + project, generation, resourceType) + if err != nil { + s.fail("witness default collation order", err) + } + for rows.Next() { + var v string + if err := rows.Scan(&v); err != nil { + rows.Close() + s.fail("scan default collation order", err) + } + defaultOrder = append(defaultOrder, v) + } + rows.Close() + } + want := append([]string(nil), pgcat05AdversarialIDs...) + sort.Strings(want) + if reflect.DeepEqual(defaultOrder, want) { + t.Fatalf("server default collation order matches Go byte order on the adversarial set — the probe is vacuous on this service; expand the id set") + } + boundary := want[len(want)/2] + var icuCount, byteCount int64 + if err := s.iso.QueryRow(s.ctx, "SELECT COUNT(*) FROM "+s.resourceFQN()+ + " WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3 AND logical_id > $4", + project, generation, resourceType, boundary).Scan(&icuCount); err != nil { + s.fail("icu keyset probe", err) + } + if err := s.iso.QueryRow(s.ctx, `SELECT COUNT(*) FROM `+s.resourceFQN()+ + ` WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3 AND logical_id COLLATE "C" > $4 COLLATE "C"`, + project, generation, resourceType, boundary).Scan(&byteCount); err != nil { + s.fail("byte keyset probe", err) + } + goAbove := 0 + for _, id := range want { + if id > boundary { + goAbove++ + } + } + if byteCount != int64(goAbove) { + t.Fatalf("byte-order predicate returned %d rows, Go byte order above %q is %d", byteCount, boundary, goAbove) + } + if icuCount == int64(goAbove) { + t.Fatalf("default-collation predicate agrees with byte order on this set — predicate probe is vacuous; expand the id set") + } + + var walked []string + afterKey := "" + pages := 0 + for { + page, err := repo.ReadRetainedSemanticInventoryPage(s.ctx, project, generation, resourceType, afterKey, 3) + if err != nil { + s.fail("read page", err) + } + pages++ + if !page.SourceExists { + t.Fatalf("page %d: SourceExists=false for a populated scope", pages) + } + if len(page.Rows) > 3 { + t.Fatalf("page %d: %d rows, want at most 3", pages, len(page.Rows)) + } + for _, row := range page.Rows { + walked = append(walked, row.Key) + if row.AuthResourcePath == nil { + t.Fatalf("row %s: auth path is nil; this store never writes SQL NULL auth paths", row.Key) + } + if *row.AuthResourcePath != "path/"+row.Key { + t.Fatalf("row %s: auth path = %q, want %q", row.Key, *row.AuthResourcePath, "path/"+row.Key) + } + switch row.Key { + case "z-null": + if row.Payload != nil { + t.Fatalf("z-null: payload = %s, want SQL NULL (nil RawMessage)", row.Payload) + } + case "z-jsonnull": + if string(row.Payload) != "null" { + t.Fatalf("z-jsonnull: payload = %q, want the explicit JSON null document", row.Payload) + } + default: + var got, expected map[string]any + if err := json.Unmarshal(row.Payload, &got); err != nil { + t.Fatalf("row %s: payload %s does not decode: %v", row.Key, row.Payload, err) + } + if err := json.Unmarshal([]byte(pgcat05Goldens["obs-b"].payload), &expected); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, expected) { + t.Fatalf("row %s: payload %s, want the seeded JSONB %s", row.Key, row.Payload, pgcat05Goldens["obs-b"].payload) + } + } + } + if !page.HasMore { + break + } + if len(page.Rows) == 0 { + t.Fatalf("page %d: HasMore=true with no rows — cursor would stall", pages) + } + nextKey := page.Rows[len(page.Rows)-1].Key + if nextKey <= afterKey { + t.Fatalf("page %d: cursor did not advance (%q -> %q)", pages, afterKey, nextKey) + } + afterKey = nextKey + if pages > len(pgcat05AdversarialIDs) { + t.Fatalf("page walk did not terminate") + } + } + if !reflect.DeepEqual(walked, want) { + t.Fatalf("full walk =\n %v\nwant Go byte order\n %v", walked, want) + } +} + +func TestSemanticContributionRetainedPageEmptySourceDistinctions(t *testing.T) { + s := newPgcat05Service(t) + repo := s.contributionRepository(t) + const generation = "gen-e" + + page, err := repo.ReadRetainedSemanticInventoryPage(s.ctx, "pgcat05-empty-none", generation, "Observation", "", 10) + if err != nil { + s.fail("read unproven scope", err) + } + if page.SourceExists || len(page.Rows) != 0 { + t.Fatalf("never-profiled scope page = %+v, want SourceExists=false with no rows", page) + } + + page, err = repo.ReadRetainedSemanticInventoryPage(s.ctx, "pgcat05-empty-coverage", generation, "Observation", "", 10) + if err != nil { + s.fail("read coverage scope", err) + } + if page.SourceExists { + t.Fatal("scope without any marker must be unproven before seeding (harness is not proving the distinction)") + } + s.seedCoverageMarker(t, "pgcat05-empty-coverage", generation, "", "Observation") + page, err = repo.ReadRetainedSemanticInventoryPage(s.ctx, "pgcat05-empty-coverage", generation, "Observation", "", 10) + if err != nil { + s.fail("read seeded coverage scope", err) + } + if !page.SourceExists || len(page.Rows) != 0 || page.HasMore { + t.Fatalf("known-empty scope page = %+v, want SourceExists=true, no rows, no more", page) + } + s.exec(t, "DELETE FROM "+s.coverageFQN()+" WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3", + "pgcat05-empty-coverage", generation, "Observation") + page, err = repo.ReadRetainedSemanticInventoryPage(s.ctx, "pgcat05-empty-coverage", generation, "Observation", "", 10) + if err != nil { + s.fail("read after marker drop", err) + } + if page.SourceExists { + t.Fatal("known-empty claim survived dropping the coverage marker — the distinction is not marker-driven") + } + + s.seedFieldProfileDoc(t, "pgcat05-empty-docs", generation, "", "Observation", "code.coding", 3) + _, err = repo.ReadRetainedSemanticInventoryPage(s.ctx, "pgcat05-empty-docs", generation, "Observation", "", 10) + var missing *catalogstore.RetainedSemanticSourceMissingError + if !errors.As(err, &missing) || !errors.Is(err, catalogstore.ErrRetainedSemanticSourceMissing) { + t.Fatalf("missing-source read error = %v, want *RetainedSemanticSourceMissingError", err) + } + if missing.ResourceType != "Observation" || missing.Project != "pgcat05-empty-docs" { + t.Fatalf("missing-source error names the wrong scope: %+v", missing) + } + for _, other := range []error{catalogstore.ErrSemanticInventoryStorageMissing, catalogstore.ErrInvalidRetainedSemanticPageRequest, catalogstore.ErrSemanticInventoryContributionConflict} { + if errors.Is(err, other) { + t.Fatalf("missing-source error misclassifies as %v", other) + } + } + s.exec(t, "UPDATE "+s.fieldProfileFQN()+" SET doc_count = 0 WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3", + "pgcat05-empty-docs", generation, "Observation") + page, err = repo.ReadRetainedSemanticInventoryPage(s.ctx, "pgcat05-empty-docs", generation, "Observation", "", 10) + if err != nil { + s.fail("read after doc_count zeroed", err) + } + if page.SourceExists { + t.Fatal("doc_count = 0 was treated as document/coverage evidence") + } + s.exec(t, "INSERT INTO "+s.fieldProfileFQN()+ + " (project, dataset_generation, auth_resource_path, resource_type, path, profile_key,"+ + " record_type, kind, doc_count, max_items, sample_count, distinct_truncated,"+ + " enrichment_truncated, pivot_candidate, pivot_kind, pivot_family,"+ + " pivot_column_selector, pivot_value_selector, pivot_item_source, pivot_item_resource_type)"+ + " VALUES ($1, $2, '', $3, '', 'fpk-status', 'profile_status', '', 9, 0, 0, false, true, false, '', '', '', '', '', '')", + "pgcat05-empty-status", generation, "Observation") + page, err = repo.ReadRetainedSemanticInventoryPage(s.ctx, "pgcat05-empty-status", generation, "Observation", "", 10) + if err != nil { + s.fail("read profile_status-shaped scope", err) + } + if page.SourceExists { + t.Fatal("a profile_status marker row (no fields) must not count as document evidence") + } +} + +func TestSemanticContributionRetainedPageMidWalkEndsCollection(t *testing.T) { + s := newPgcat05Service(t) + repo := s.contributionRepository(t) + const project, generation = "pgcat05-midwalk", "gen-m" + s.seedRetainedResource(t, project, generation, "Observation", "m-a", "", []byte(pgcat05Goldens["obs-b"].payload)) + s.seedRetainedResource(t, project, generation, "Observation", "m-b", "", []byte(pgcat05Goldens["obs-b"].payload)) + s.seedFieldProfileDoc(t, project, generation, "", "Observation", "code.coding", 2) + + first, err := repo.ReadRetainedSemanticInventoryPage(s.ctx, project, generation, "Observation", "", 10) + if err != nil { + s.fail("read first page", err) + } + if !first.SourceExists || len(first.Rows) != 2 { + t.Fatalf("first page = %+v, want two rows", first) + } + end, err := repo.ReadRetainedSemanticInventoryPage(s.ctx, project, generation, "Observation", "m-z", 10) + if err != nil { + t.Fatalf("mid-walk empty read error = %v, want plain end of collection", err) + } + if !end.SourceExists || len(end.Rows) != 0 || end.HasMore { + t.Fatalf("mid-walk empty page = %+v, want SourceExists=true, no rows, no more", end) + } +} + +func TestSemanticContributionRetainedPageRequestValidation(t *testing.T) { + s := newPgcat05Service(t) + repo := s.contributionRepository(t) + + bad := []struct { + name string + project, generation, collection string + limit int + }{ + {"non-canonical collection case", "p", "g", "observation", 10}, + {"unknown collection", "p", "g", "Bogus99", 10}, + {"trailing-space collection", "p", "g", "Observation ", 10}, + {"empty collection", "p", "g", "", 10}, + {"blank project", "", "g", "Observation", 10}, + {"whitespace project", " ", "g", "Observation", 10}, + {"zero limit", "p", "g", "Observation", 0}, + {"negative limit", "p", "g", "Observation", -1}, + {"oversized limit", "p", "g", "Observation", 1001}, + } + for _, tc := range bad { + _, err := repo.ReadRetainedSemanticInventoryPage(s.ctx, tc.project, tc.generation, tc.collection, "", tc.limit) + if !errors.Is(err, catalogstore.ErrInvalidRetainedSemanticPageRequest) { + t.Fatalf("%s: error = %v, want ErrInvalidRetainedSemanticPageRequest", tc.name, err) + } + if errors.Is(err, catalogstore.ErrRetainedSemanticSourceMissing) || errors.Is(err, catalogstore.ErrSemanticInventoryStorageMissing) { + t.Fatalf("%s: validation refusal misclassifies: %v", tc.name, err) + } + } + if _, err := repo.ReadRetainedSemanticInventoryPage(s.ctx, "pgcat05-validate", "g", "Observation", "", 1000); err != nil { + s.fail("control page read", err) + } +} + +func TestSemanticContributionRetainedPageGenerationNormalization(t *testing.T) { + s := newPgcat05Service(t) + repo := s.contributionRepository(t) + const project = "pgcat05-normalize" + payload := []byte(pgcat05Goldens["obs-b"].payload) + s.seedRetainedResource(t, project, "gen-n", "Observation", "clean-1", "", payload) + s.seedRetainedResource(t, project, "gen-n", "Observation", "clean-2", "", payload) + s.seedRetainedResource(t, project, " gen-n ", "Observation", "padded-only", "", payload) + + for _, request := range []string{"gen-n", " gen-n ", "\tgen-n\n"} { + page, err := repo.ReadRetainedSemanticInventoryPage(s.ctx, project, request, "Observation", "", 10) + if err != nil { + s.fail("normalized read", err) + } + keys := []string{} + for _, row := range page.Rows { + keys = append(keys, row.Key) + } + if !reflect.DeepEqual(keys, []string{"clean-1", "clean-2"}) { + t.Fatalf("request %q returned %v, want only the normalized spelling's rows", request, keys) + } + } + var padded string + if err := s.iso.QueryRow(s.ctx, "SELECT logical_id FROM "+s.resourceFQN()+ + " WHERE project = $1 AND dataset_generation = $2", project, " gen-n ").Scan(&padded); err != nil { + s.fail("witness padded row", err) + } + if padded != "padded-only" { + t.Fatalf("padded row missing from the table — the normalization probe was vacuous") + } +} + +func TestSemanticContributionPrepareRefusesMissingTables(t *testing.T) { + s := newPgcat05Service(t) + repo := s.contributionRepository(t) + + rerun := func(name string) { + migrations, err := storepostgres.EmbeddedMigrations() + if err != nil { + s.fail("load migrations", err) + } + for _, m := range migrations { + if m.Name == name { + conn, err := s.iso.Acquire(s.ctx) + if err != nil { + s.fail("acquire rerun connection", err) + } + defer conn.Release() + if _, err := conn.Exec(s.ctx, "SET search_path TO "+pgx.Identifier{s.schema}.Sanitize()+", public"); err != nil { + s.fail("set rerun search_path", err) + } + if _, err := conn.Exec(s.ctx, m.SQL); err != nil { + s.fail("rerun migration "+m.Name, err) + } + return + } + } + t.Fatalf("migration %q is not embedded — the harness lost its restore path", name) + } + + drops := []struct { + table string + migration string + dropSQL string + }{ + {"fhir_semantic_inventory_contribution", "semantic_inventory_contribution", + "DROP TABLE " + s.contributionFQN() + " CASCADE"}, + {"fhir_semantic_inventory_build", "semantic_inventory_build", + "DROP TABLE " + s.buildFQN() + " CASCADE"}, + {"fhir_resource", "fhir_resource", + "DROP TABLE " + s.resourceFQN() + " CASCADE"}, + } + for _, d := range drops { + s.exec(t, d.dropSQL) + err := repo.PrepareSemanticInventoryBackfill(s.ctx) + var missing *catalogstore.SemanticInventoryStorageMissingError + if !errors.As(err, &missing) || !errors.Is(err, catalogstore.ErrSemanticInventoryStorageMissing) { + t.Fatalf("dropped %s: error = %v, want named storage-missing refusal", d.table, err) + } + if missing.Table != d.table { + t.Fatalf("dropped %s: refusal names %q", d.table, missing.Table) + } + if errors.Is(err, catalogstore.ErrSemanticInventoryEntriesUnsupported) || errors.Is(err, catalogstore.ErrRetainedSemanticSourceMissing) { + t.Fatalf("dropped %s: refusal misclassifies: %v", d.table, err) + } + rerun(d.migration) + if err := repo.PrepareSemanticInventoryBackfill(s.ctx); err != nil { + t.Fatalf("restored %s: Prepare error = %v", d.table, err) + } + } +} + +func TestSemanticContributionEntriesRefusalIsNamedNoOp(t *testing.T) { + s := newPgcat05Service(t) + repo := s.contributionRepository(t) + + build := catalog.NewSemanticInventoryBuild(pgcat05GoldenProject, pgcat05GoldenGeneration, "") + build.SourceKind = catalog.SemanticInventorySourceRetained + got, err := repo.EnsureSemanticInventoryEntries(s.ctx, build) + var unsupported *catalogstore.SemanticInventoryEntriesUnsupportedError + if !errors.As(err, &unsupported) || !errors.Is(err, catalogstore.ErrSemanticInventoryEntriesUnsupported) { + t.Fatalf("Ensure error = %v, want named entries-unsupported refusal", err) + } + if got.Key != "" || got.EntryIndexVersion != 0 { + t.Fatalf("refused Ensure returned a mutated build: %+v", got) + } + if unsupported.EntryIndexVersion != 0 || unsupported.BuildID != build.BuildID { + t.Fatalf("refusal does not name the build: %+v", unsupported) + } + if errors.Is(err, catalogstore.ErrSemanticInventoryClaimLost) || errors.Is(err, catalogstore.ErrSemanticInventoryStorageMissing) { + t.Fatalf("entries refusal misclassifies: %v", err) + } + if got := s.countContributions(t); got != 0 { + t.Fatalf("Ensure refusal wrote %d contribution rows, want 0", got) + } + + corrupted := build + corrupted.Key = "sib_not_a_digest" + _, err = repo.EnsureSemanticInventoryEntries(s.ctx, corrupted) + if !errors.Is(err, catalogstore.ErrInvalidSemanticInventoryBuild) { + t.Fatalf("corrupted build Ensure error = %v, want ErrInvalidSemanticInventoryBuild", err) + } + if errors.Is(err, catalogstore.ErrSemanticInventoryEntriesUnsupported) { + t.Fatal("corrupted build must fail as invalid, not merely unsupported") + } +} + +const pgcat05RaceRounds = 10 + +func pgcat05RaceContribution(t *testing.T, project, generation, sourceID, example string) (catalog.SemanticInventoryContribution, contributionLiteral) { + t.Helper() + g := pgcat05Goldens["obs-b"] + emitted := pgcat05EmitContribution(t, project, generation, g.authPath, g.resourceType, sourceID, g.payload) + if len(emitted) != 1 { + t.Fatalf("emitter produced %d contributions for %s, want 1", len(emitted), sourceID) + } + c := emitted[0] + if example != "" { + c.Example = example + c.Observation.Examples = []string{example} + } + digest, observation, err := prepareContributionTest(c) + if err != nil { + t.Fatalf("prepare race contribution: %v", err) + } + literal := contributionLiteral{ + Key: c.Key, Project: c.Project, DatasetGeneration: c.DatasetGeneration, BuildID: c.BuildID, + SourceKind: c.SourceKind, AuthResourcePath: c.AuthResourcePath, ResourceType: c.ResourceType, + SourceID: c.SourceID, Ordinal: c.Ordinal, ConceptID: c.ConceptID, BindingID: c.BindingID, + Example: c.Example, PayloadDigest: digest, Observation: string(observation), + } + return c, literal +} + +func pgcat05RequireDistinctBackends(t *testing.T, pidA, pidB int32) { + t.Helper() + if pidA <= 0 || pidB <= 0 || pidA == pidB { + t.Fatalf("race connections must be distinct live backends, got %d and %d", pidA, pidB) + } +} + +func TestSemanticContributionIdenticalWriteRaceTwoConnections(t *testing.T) { + s := newPgcat05Service(t) + clientA, pidA := s.openDedicatedClient(t, "identical-race A") + clientB, pidB := s.openDedicatedClient(t, "identical-race B") + + repoA, err := catalogstore.NewSemanticContributionRepository(clientA, s.schema) + if err != nil { + s.fail("repository A", err) + } + repoB, err := catalogstore.NewSemanticContributionRepository(clientB, s.schema) + if err != nil { + s.fail("repository B", err) + } + + for round := 0; round < pgcat05RaceRounds; round++ { + pgcat05RequireDistinctBackends(t, pidA, pidB) + var batch []catalog.SemanticInventoryContribution + literals := map[string]contributionLiteral{} + for i := 0; i < 3; i++ { + c, literal := pgcat05RaceContribution(t, "pgcat05-race-ident", "gen", + fmt.Sprintf("retained:Observation/race-%d-%d", round, i), "") + batch = append(batch, c) + literals[c.Key] = literal + } + start := make(chan struct{}) + var writers sync.WaitGroup + errs := make([]error, 2) + for i, repo := range []*catalogstore.SemanticContributionRepository{repoA, repoB} { + writers.Add(1) + go func(i int, repo *catalogstore.SemanticContributionRepository) { + defer writers.Done() + <-start + errs[i] = repo.WriteSemanticInventoryContributions(s.ctx, batch, 0) + }(i, repo) + } + close(start) + writers.Wait() + for i, err := range errs { + if err != nil { + t.Fatalf("round %d writer %d: %v", round, i, err) + } + } + for _, literal := range literals { + w, found := s.witnessContribution(t, literal.Key) + if !found { + t.Fatalf("round %d: row %s missing after race", round, literal.Key) + } + if !sameContributionLiteral(w.contributionLiteral, literal) { + t.Fatalf("round %d: stored row = %+v, want %+v", round, w.contributionLiteral, literal) + } + } + } +} + +func TestSemanticContributionUncommittedWinnerRaceTwoConnections(t *testing.T) { + s := newPgcat05Service(t) + clientA, pidA := s.openDedicatedClient(t, "uncommitted A") + clientB, pidB := s.openDedicatedClient(t, "uncommitted B") + + repoB, err := catalogstore.NewSemanticContributionRepository(clientB, s.schema) + if err != nil { + s.fail("repository B", err) + } + + for round := 0; round < pgcat05RaceRounds; round++ { + pgcat05RequireDistinctBackends(t, pidA, pidB) + commitWinner := round%2 == 0 + sourceID := fmt.Sprintf("retained:Observation/uncommitted-%d", round) + _, literalA := pgcat05RaceContribution(t, "pgcat05-race-uncommitted", "gen", sourceID, "payload-from-A") + contributionB, literalB := pgcat05RaceContribution(t, "pgcat05-race-uncommitted", "gen", sourceID, "payload-from-B") + if contributionB.Key != literalA.Key { + t.Fatalf("round %d: conflicting payloads produced different keys %s vs %s", round, contributionB.Key, literalA.Key) + } + + connA, err := clientA.Acquire(s.ctx) + if err != nil { + s.fail("acquire A", err) + } + tx, err := connA.Begin(s.ctx) + if err != nil { + connA.Release() + s.fail("begin A", err) + } + if _, err := tx.Exec(s.ctx, s.contributionSeedSQL(), + literalA.Key, literalA.Project, literalA.DatasetGeneration, literalA.BuildID, literalA.SourceKind, + literalA.AuthResourcePath, literalA.ResourceType, literalA.SourceID, literalA.Ordinal, + literalA.ConceptID, literalA.BindingID, literalA.Example, literalA.PayloadDigest, literalA.Observation); err != nil { + _ = tx.Rollback(s.ctx) + connA.Release() + s.fail("uncommitted insert A", err) + } + + bDone := make(chan error, 1) + go func() { + bDone <- repoB.WriteSemanticInventoryContributions(s.ctx, []catalog.SemanticInventoryContribution{contributionB}, 0) + }() + + if !s.waitForBackendLockWait(t, pidB, 15*time.Second) { + _ = tx.Rollback(s.ctx) + connA.Release() + t.Fatalf("round %d: writer B never waited on A's uncommitted insert", round) + } + if commitWinner { + if err := tx.Commit(s.ctx); err != nil { + connA.Release() + s.fail("commit A", err) + } + } else { + if err := tx.Rollback(s.ctx); err != nil { + connA.Release() + s.fail("rollback A", err) + } + } + connA.Release() + bErr := <-bDone + + w, found := s.witnessContribution(t, literalA.Key) + if !found { + t.Fatalf("round %d: no row after race", round) + } + if commitWinner { + var conflict *catalogstore.SemanticInventoryContributionConflictError + if !errors.As(bErr, &conflict) { + t.Fatalf("round %d (commit): B error = %v, want typed conflict", round, bErr) + } + if conflict.StoredDigest != literalA.PayloadDigest || conflict.ProvidedDigest != literalB.PayloadDigest { + t.Fatalf("round %d (commit): conflict digests = %s/%s, want %s/%s", + round, conflict.StoredDigest, conflict.ProvidedDigest, literalA.PayloadDigest, literalB.PayloadDigest) + } + if !sameContributionLiteral(w.contributionLiteral, literalA) { + t.Fatalf("round %d (commit): committed winner's row disturbed:\n stored %+v\n want %+v", round, w.contributionLiteral, literalA) + } + } else { + if bErr != nil { + t.Fatalf("round %d (abort): B error = %v, want post-abort convergence to B's row", round, bErr) + } + if !sameContributionLiteral(w.contributionLiteral, literalB) { + t.Fatalf("round %d (abort): row after aborted winner:\n stored %+v\n want %+v", round, w.contributionLiteral, literalB) + } + } + } +} + +func (s *pgcat05Service) waitForBackendLockWait(t *testing.T, pid int32, budget time.Duration) bool { + t.Helper() + deadline := time.Now().Add(budget) + for time.Now().Before(deadline) { + var waiting bool + if err := s.iso.QueryRow(s.ctx, + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE pid = $1 AND state = 'active' AND wait_event_type = 'Lock')", + pid).Scan(&waiting); err != nil { + s.fail("poll pg_stat_activity", err) + } + if waiting { + return true + } + time.Sleep(25 * time.Millisecond) + } + return false +} + +type pgcat05BackfillProbe struct { + *catalogstore.SemanticInventoryBackfillStore + mu sync.Mutex + writeCalls, advanceCalls int + failWriteAt int +} + +func (p *pgcat05BackfillProbe) WriteSemanticInventoryContributions(ctx context.Context, contributions []catalog.SemanticInventoryContribution, batchSize int) error { + p.mu.Lock() + p.writeCalls++ + call := p.writeCalls + fail := p.failWriteAt != 0 && call == p.failWriteAt + p.mu.Unlock() + if fail { + return errors.New("injected contribution write failure") + } + return p.SemanticInventoryBackfillStore.WriteSemanticInventoryContributions(ctx, contributions, batchSize) +} + +func (p *pgcat05BackfillProbe) AdvanceSemanticInventoryBackfill(ctx context.Context, build catalog.SemanticInventoryBuild, token string, expected, next catalog.SemanticInventoryCheckpoint, scanned int64, lease time.Duration) (catalog.SemanticInventoryBuild, error) { + p.mu.Lock() + p.advanceCalls++ + p.mu.Unlock() + return p.SemanticInventoryBackfillStore.AdvanceSemanticInventoryBackfill(ctx, build, token, expected, next, scanned, lease) +} + +func (p *pgcat05BackfillProbe) counts() (int, int) { + p.mu.Lock() + defer p.mu.Unlock() + return p.writeCalls, p.advanceCalls +} + +func (p *pgcat05BackfillProbe) reset() { + p.mu.Lock() + defer p.mu.Unlock() + p.writeCalls, p.advanceCalls = 0, 0 +} + +func pgcat05Manifest(t *testing.T, project, generation string, resourceTypes []string) publication.Manifest { + t.Helper() + ref, err := publication.NewRef(project, generation) + if err != nil { + t.Fatalf("manifest ref: %v", err) + } + schema, err := publication.NewSchemaSnapshot("pgcat05-schema", "4.0.1", strings.Repeat("a", 64), resourceTypes) + if err != nil { + t.Fatalf("schema snapshot: %v", err) + } + manifest, err := publication.NewManifest(ref, schema) + if err != nil { + t.Fatalf("manifest: %v", err) + } + manifest.State = publication.StateStaged + return manifest +} + +func pgcat05BackfillOptions() ingest.SemanticInventoryBackfillOptions { + return ingest.SemanticInventoryBackfillOptions{ + Project: pgcat05GoldenProject, DatasetGeneration: pgcat05GoldenGeneration, + PageSize: 2, BatchSize: 1, + } +} + +func seedPgcat05GoldenSources(t *testing.T, s *pgcat05Service) { + t.Helper() + for _, id := range pgcat05GoldenIDs() { + g := pgcat05Goldens[id] + s.seedRetainedResource(t, pgcat05GoldenProject, pgcat05GoldenGeneration, g.resourceType, id, g.authPath, []byte(g.payload)) + } + s.seedRetainedResource(t, pgcat05GoldenProject, pgcat05GoldenGeneration, "Patient", "pat-a", "", + []byte(`{"resourceType":"Patient","gender":"female"}`)) + s.seedFieldProfileDoc(t, pgcat05GoldenProject, pgcat05GoldenGeneration, "", "Observation", "code.coding", 6) + s.seedFieldProfileDoc(t, pgcat05GoldenProject, pgcat05GoldenGeneration, "", "Patient", "gender", 1) +} + +func pgcat05FullTableWitness(t *testing.T, s *pgcat05Service) []contributionWitness { + t.Helper() + rows, err := s.iso.Query(s.ctx, "SELECT "+contributionWitnessColumns+" FROM "+s.contributionFQN()+" ORDER BY contribution_key") + if err != nil { + s.fail("witness full table", err) + } + defer rows.Close() + var out []contributionWitness + for rows.Next() { + var w contributionWitness + if err := rows.Scan(&w.Key, &w.Project, &w.DatasetGeneration, &w.BuildID, &w.SourceKind, + &w.AuthResourcePath, &w.ResourceType, &w.SourceID, &w.Ordinal, &w.ConceptID, + &w.BindingID, &w.Example, &w.PayloadDigest, &w.Observation, &w.RecordedAt); err != nil { + s.fail("scan full table witness", err) + } + out = append(out, w) + } + if err := rows.Err(); err != nil { + s.fail("witness full table", err) + } + return out +} + +func TestSemanticContributionBackfillRoundTripEndToEnd(t *testing.T) { + s := newPgcat05Service(t) + seedPgcat05GoldenSources(t, s) + probe := &pgcat05BackfillProbe{SemanticInventoryBackfillStore: s.backfillStore(t)} + + manifest := pgcat05Manifest(t, pgcat05GoldenProject, pgcat05GoldenGeneration, []string{"Observation", "Patient"}) + report, err := ingest.BackfillSemanticInventory(s.ctx, probe, manifest, pgcat05BackfillOptions()) + if err != nil { + s.fail("backfill", err) + } + if report.State != catalog.SemanticInventoryComplete || report.ScannedTotal != 7 || report.Checkpoint != (catalog.SemanticInventoryCheckpoint{Collection: "Patient", Key: "pat-a"}) { + t.Fatalf("report = %+v, want complete with 7 scanned and Patient/pat-a checkpoint", report) + } + if report.SourceAvailability != catalog.SemanticInventorySourceAvailabilityVerified || len(report.MissingCollections) != 0 { + t.Fatalf("availability = %q missing=%v, want verified with none", report.SourceAvailability, report.MissingCollections) + } + if writes, advances := probe.counts(); writes != 6 || advances != 4 { + t.Fatalf("probe writes=%d advances=%d, want 6 page chunks over 4 pages", writes, advances) + } + + ids := pgcat05GoldenIDs() + if got := s.countContributions(t); got != int64(len(ids)) { + t.Fatalf("contribution count = %d, want %d", got, len(ids)) + } + for _, id := range ids { + want := pgcat05ExpectedLiteral(pgcat05Goldens[id]) + w, found := s.witnessContribution(t, want.Key) + if !found { + t.Fatalf("%s: golden contribution %s missing after backfill", id, want.Key) + } + if !sameContributionLiteral(w.contributionLiteral, want) { + t.Fatalf("%s: stored %+v, want golden %+v", id, w.contributionLiteral, want) + } + } + build, found := s.witnessBuildRow(t, pgcat05GoldenProject, pgcat05GoldenGeneration, catalog.SemanticInventoryBuildID(pgcat05GoldenProject, pgcat05GoldenGeneration)) + if !found { + t.Fatal("build row missing") + } + if build.State != catalog.SemanticInventoryComplete || build.ScannedResources != 7 || + build.SourceCheckpoint != (catalog.SemanticInventoryCheckpoint{Collection: "Patient", Key: "pat-a"}) || + build.LeaseToken != "" || build.LeaseExpiresAt != 0 || len(build.Diagnostics) != 0 || + build.SourceAvailability != catalog.SemanticInventorySourceAvailabilityVerified { + t.Fatalf("build row = %+v, want complete/7/Patient-pat-a/released/no-diagnostics/verified", build) + } + + before := pgcat05FullTableWitness(t, s) + _, err = ingest.BackfillSemanticInventory(s.ctx, probe, manifest, pgcat05BackfillOptions()) + if !errors.Is(err, catalogstore.ErrSemanticInventoryEntriesUnsupported) { + t.Fatalf("completed version-0 replay error = %v, want named entries refusal", err) + } + after := pgcat05FullTableWitness(t, s) + if !sameContributionWitnesses(before, after) { + t.Fatalf("entries refusal mutated contribution rows:\n before %v\n after %v", before, after) + } + + s.exec(t, "UPDATE "+s.buildFQN()+" SET entry_index_version = $1 WHERE project = $2 AND dataset_generation = $3 AND build_id = $4", + catalog.SemanticInventoryEntryIndexVersion, pgcat05GoldenProject, pgcat05GoldenGeneration, catalog.SemanticInventoryBuildID(pgcat05GoldenProject, pgcat05GoldenGeneration)) + probe.reset() + replay, err := ingest.BackfillSemanticInventory(s.ctx, probe, manifest, pgcat05BackfillOptions()) + if err != nil { + t.Fatalf("version-2 replay: %v", err) + } + if !replay.NoOp || replay.State != catalog.SemanticInventoryComplete { + t.Fatalf("version-2 replay report = %+v, want clean no-op", replay) + } + if writes, advances := probe.counts(); writes != 0 || advances != 0 { + t.Fatalf("no-op replay did work: writes=%d advances=%d", writes, advances) + } + final := pgcat05FullTableWitness(t, s) + if !sameContributionWitnesses(before, final) { + t.Fatalf("replay sequence disturbed rows:\n before %v\n after %v", before, final) + } +} + +func sameContributionWitnesses(a, b []contributionWitness) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if !a[i].sameAs(b[i]) { + return false + } + } + return true +} + +func TestSemanticContributionBackfillFailedBatchKeepsCheckpointAndReplaysIdempotently(t *testing.T) { + s := newPgcat05Service(t) + seedPgcat05GoldenSources(t, s) + probe := &pgcat05BackfillProbe{SemanticInventoryBackfillStore: s.backfillStore(t)} + manifest := pgcat05Manifest(t, pgcat05GoldenProject, pgcat05GoldenGeneration, []string{"Observation", "Patient"}) + options := pgcat05BackfillOptions() + + probe.failWriteAt = 4 + _, err := ingest.BackfillSemanticInventory(s.ctx, probe, manifest, options) + if err == nil || !strings.Contains(err.Error(), "injected contribution write failure") { + t.Fatalf("injected failure error = %v", err) + } + buildID := catalog.SemanticInventoryBuildID(pgcat05GoldenProject, pgcat05GoldenGeneration) + build, found := s.witnessBuildRow(t, pgcat05GoldenProject, pgcat05GoldenGeneration, buildID) + if !found { + t.Fatal("build row missing after failed run") + } + if build.State != catalog.SemanticInventoryFailed || + build.SourceCheckpoint != (catalog.SemanticInventoryCheckpoint{Collection: "Observation", Key: "obs-b"}) || + build.ScannedResources != 2 || build.LeaseToken != "" || build.LeaseExpiresAt != 0 { + t.Fatalf("failed-run build = %+v, want failed at Observation/obs-b with 2 scanned and released lease", build) + } + if len(build.Diagnostics) != 1 || build.Diagnostics[0] != "injected contribution write failure" { + t.Fatalf("failed-run diagnostics = %v, want exactly the injected cause", build.Diagnostics) + } + if writes, advances := probe.counts(); writes != 4 || advances != 1 { + t.Fatalf("failed-run probe writes=%d advances=%d, want 4 attempts with 1 checkpoint advance", writes, advances) + } + for _, id := range []string{"obs-a", "obs-b", "obs-c"} { + if _, found := s.witnessContribution(t, pgcat05Goldens[id].key); !found { + t.Fatalf("%s: committed chunk missing after the failed run", id) + } + } + s.requireNoContribution(t, pgcat05Goldens["obs-d"].key, "failed chunk obs-d") + + crashed, found := s.witnessContribution(t, pgcat05Goldens["obs-c"].key) + if !found { + t.Fatal("crash-window row obs-c missing") + } + crashedWant := pgcat05ExpectedLiteral(pgcat05Goldens["obs-c"]) + if !sameContributionLiteral(crashed.contributionLiteral, crashedWant) { + t.Fatalf("crash-window row = %+v, want %+v", crashed.contributionLiteral, crashedWant) + } + + probe.failWriteAt = 0 + probe.reset() + report, err := ingest.BackfillSemanticInventory(s.ctx, probe, manifest, options) + if err != nil { + s.fail("resumed backfill", err) + } + if report.State != catalog.SemanticInventoryComplete || report.ScannedTotal != 7 { + t.Fatalf("resumed report = %+v", report) + } + resumed, found := s.witnessContribution(t, pgcat05Goldens["obs-c"].key) + if !found || !resumed.sameAs(crashed) { + t.Fatalf("resume disturbed the replayed crash-window row:\n before %+v\n after %+v", crashed, resumed) + } + if writes, advances := probe.counts(); writes != 4 || advances != 3 { + t.Fatalf("resumed probe writes=%d advances=%d, want 4 chunk writes (one a replay) over 3 page advances", writes, advances) + } + for _, id := range pgcat05GoldenIDs() { + want := pgcat05ExpectedLiteral(pgcat05Goldens[id]) + w, found := s.witnessContribution(t, want.Key) + if !found || !sameContributionLiteral(w.contributionLiteral, want) { + t.Fatalf("%s: after resume stored %+v found=%t, want %+v", id, w.contributionLiteral, found, want) + } + } + build, _ = s.witnessBuildRow(t, pgcat05GoldenProject, pgcat05GoldenGeneration, buildID) + if build.State != catalog.SemanticInventoryComplete || + build.SourceCheckpoint != (catalog.SemanticInventoryCheckpoint{Collection: "Patient", Key: "pat-a"}) || + len(build.Diagnostics) != 0 { + t.Fatalf("resumed build = %+v, want complete at Patient/pat-a with diagnostics cleared", build) + } +} + +func TestSemanticContributionBackfillSourceAvailability(t *testing.T) { + s := newPgcat05Service(t) + const generation = "gen-av" + optionsFor := func(project string) ingest.SemanticInventoryBackfillOptions { + return ingest.SemanticInventoryBackfillOptions{Project: project, DatasetGeneration: generation, PageSize: 2, BatchSize: 50} + } + manifestFor := func(project string) publication.Manifest { + return pgcat05Manifest(t, project, generation, []string{"Observation", "Patient"}) + } + seedObservation := func(project string) { + s.seedRetainedResource(t, project, generation, "Observation", "av-a", "", []byte(pgcat05Goldens["obs-b"].payload)) + s.seedFieldProfileDoc(t, project, generation, "", "Observation", "code.coding", 1) + } + + project := "pgcat05-av-unproven" + seedObservation(project) + probe := &pgcat05BackfillProbe{SemanticInventoryBackfillStore: s.backfillStore(t)} + report, err := ingest.BackfillSemanticInventory(s.ctx, probe, manifestFor(project), optionsFor(project)) + if err != nil { + s.fail("unproven backfill", err) + } + if report.State != catalog.SemanticInventoryComplete || + report.SourceAvailability != catalog.SemanticInventorySourceAvailabilityUnproven || + len(report.MissingCollections) != 1 || report.MissingCollections[0] != "Patient" { + t.Fatalf("unproven report = %+v, want complete/unproven/[Patient]", report) + } + build, _ := s.witnessBuildRow(t, project, generation, catalog.SemanticInventoryBuildID(project, generation)) + if build.SourceAvailability != catalog.SemanticInventorySourceAvailabilityUnproven { + t.Fatalf("unproven build availability = %q", build.SourceAvailability) + } + + project = "pgcat05-av-knownempty" + seedObservation(project) + s.seedCoverageMarker(t, project, generation, "", "Patient") + probe = &pgcat05BackfillProbe{SemanticInventoryBackfillStore: s.backfillStore(t)} + report, err = ingest.BackfillSemanticInventory(s.ctx, probe, manifestFor(project), optionsFor(project)) + if err != nil { + s.fail("known-empty backfill", err) + } + if report.State != catalog.SemanticInventoryComplete || + report.SourceAvailability != catalog.SemanticInventorySourceAvailabilityVerified || + len(report.MissingCollections) != 0 { + t.Fatalf("known-empty report = %+v, want complete/verified/no-missing", report) + } + + project = "pgcat05-av-contradict" + seedObservation(project) + s.seedFieldProfileDoc(t, project, generation, "", "Patient", "gender", 4) + probe = &pgcat05BackfillProbe{SemanticInventoryBackfillStore: s.backfillStore(t)} + _, err = ingest.BackfillSemanticInventory(s.ctx, probe, manifestFor(project), optionsFor(project)) + if !errors.Is(err, catalogstore.ErrRetainedSemanticSourceMissing) { + t.Fatalf("contradiction backfill error = %v, want ErrRetainedSemanticSourceMissing", err) + } + if !strings.Contains(err.Error(), "Patient") { + t.Fatalf("contradiction error does not name the collection: %v", err) + } + build, _ = s.witnessBuildRow(t, project, generation, catalog.SemanticInventoryBuildID(project, generation)) + if build.State != catalog.SemanticInventoryFailed || len(build.Diagnostics) != 1 || !strings.Contains(build.Diagnostics[0], "Patient") { + t.Fatalf("contradiction build = %+v, want failed with a Patient diagnostic", build) + } +} + +func TestSemanticContributionBackfillClaimRaceTwoConnections(t *testing.T) { + s := newPgcat05Service(t) + clientA, pidA := s.openDedicatedClient(t, "claim-race A") + clientB, pidB := s.openDedicatedClient(t, "claim-race B") + + const generation = "gen-cr" + for round := 0; round < pgcat05RaceRounds; round++ { + pgcat05RequireDistinctBackends(t, pidA, pidB) + project := fmt.Sprintf("pgcat05-claim-race-%d", round) + for _, id := range []string{"obs-b", "obs-e"} { + g := pgcat05Goldens[id] + s.seedRetainedResource(t, project, generation, g.resourceType, id, g.authPath, []byte(g.payload)) + } + manifest := pgcat05Manifest(t, project, generation, []string{"Observation"}) + options := ingest.SemanticInventoryBackfillOptions{Project: project, DatasetGeneration: generation, PageSize: 1, BatchSize: 1} + + storeA, err := catalogstore.NewSemanticInventoryBackfillStore( + pgcat05Must(catalogstore.NewSemanticContributionRepository(clientA, s.schema)), + pgcat05Must(catalogstore.NewSemanticBuildRepository(clientA, s.schema))) + if err != nil { + s.fail("composite A", err) + } + storeB, err := catalogstore.NewSemanticInventoryBackfillStore( + pgcat05Must(catalogstore.NewSemanticContributionRepository(clientB, s.schema)), + pgcat05Must(catalogstore.NewSemanticBuildRepository(clientB, s.schema))) + if err != nil { + s.fail("composite B", err) + } + + start := make(chan struct{}) + var racers sync.WaitGroup + errs := make([]error, 2) + for i, store := range []*catalogstore.SemanticInventoryBackfillStore{storeA, storeB} { + racers.Add(1) + go func(i int, store *catalogstore.SemanticInventoryBackfillStore) { + defer racers.Done() + <-start + _, errs[i] = ingest.BackfillSemanticInventory(s.ctx, store, manifest, options) + }(i, store) + } + close(start) + racers.Wait() + + completed := 0 + for _, err := range errs { + if err == nil { + completed++ + continue + } + if !errors.Is(err, catalogstore.ErrSemanticInventoryClaimed) && + !errors.Is(err, catalogstore.ErrSemanticInventoryClaimLost) && + !errors.Is(err, catalogstore.ErrSemanticInventoryEntriesUnsupported) { + t.Fatalf("round %d: loser error = %v, want a named refusal", round, err) + } + } + if completed != 1 { + t.Fatalf("round %d: %d backfills completed, want exactly 1 (errors %v)", round, completed, errs) + } + build, found := s.witnessBuildRow(t, project, generation, catalog.SemanticInventoryBuildID(project, generation)) + if !found || build.State != catalog.SemanticInventoryComplete { + t.Fatalf("round %d: build not completed: %+v found=%t", round, build, found) + } + for _, id := range []string{"obs-b", "obs-e"} { + g := pgcat05Goldens[id] + want := pgcat05ExpectedLiteral(g) + want.Project = project + want.DatasetGeneration = catalog.NormalizeDatasetGeneration(generation) + want.BuildID = catalog.SemanticInventoryBuildID(project, generation) + emitted := pgcat05EmitContribution(t, project, generation, g.authPath, g.resourceType, g.sourceID, g.payload) + if len(emitted) != 1 { + t.Fatalf("round %d: emitter drift for %s", round, id) + } + want.Key = emitted[0].Key + preparedDigest, _, prepErr := prepareContributionTest(emitted[0]) + if prepErr != nil { + t.Fatalf("round %d: prepare: %v", round, prepErr) + } + want.PayloadDigest = preparedDigest + w, found := s.witnessContribution(t, want.Key) + if !found || !sameContributionLiteral(w.contributionLiteral, want) { + t.Fatalf("round %d: %s converged to %+v found=%t, want %+v", round, id, w.contributionLiteral, found, want) + } + } + } +} + +func pgcat05Must[T any](v T, err error) T { + if err != nil { + panic(err) + } + return v +} + +func TestZZZeroLeftoverSemanticContributionFixtures(t *testing.T) { + raw := pgcat05TestPostgresURL(t) + pgcat05OwnedMu.Lock() + owned := make(map[string]string, len(pgcat05OwnedByName)) + for db, schema := range pgcat05OwnedByName { + owned[db] = schema + } + pgcat05OwnedMu.Unlock() + if len(owned) == 0 { + t.Skip("this process created no verify_pgcat05_it_* fixtures") + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service for audit: %v", err) + } + defer admin.Close() + + for db, schema := range owned { + if !strings.HasPrefix(db, "verify_pgcat05_it_") { + t.Fatalf("registry holds foreign database %q", db) + } + var count int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Errorf("audit owned database %s: %v", db, err) + continue + } + if count != 0 { + t.Errorf("leftover owned database %s after all cleanups", db) + if schema != "" { + u, err := url.Parse(raw) + if err == nil { + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + db + if probe, err := pgxpool.New(ctx, u.String()); err == nil { + var schemaCount int + if err := probe.QueryRow(ctx, "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1", schema).Scan(&schemaCount); err != nil { + t.Errorf("audit owned schema %s in %s: %v", schema, db, err) + } else if schemaCount != 0 { + t.Errorf("leftover owned schema %s in leftover database %s", schema, db) + } + probe.Close() + } + } + } + } + } + + var prefixLeftovers int + if err := admin.QueryRow(ctx, + "SELECT COUNT(*) FROM pg_database WHERE datname LIKE 'verify_pgcat05_it_%'").Scan(&prefixLeftovers); err != nil { + t.Fatalf("prefix sweep: %v", err) + } + if prefixLeftovers != 0 { + t.Errorf("%d verify_pgcat05_it_* databases survive all cleanups", prefixLeftovers) + } +} + +func TestSemanticContributionLargeBatchClampsChunkCeiling(t *testing.T) { + s := newPgcat05Service(t) + repo := s.contributionRepository(t) + + const rows = 5000 + var goldenObs catalog.SemanticObservation + if err := json.Unmarshal([]byte(pgcat05ObservationCodeObs), &goldenObs); err != nil { + t.Fatalf("decode shared observation template: %v", err) + } + const ( + project = "pgcat05-large-batch" + generation = "gen-large" + ) + buildID := catalog.SemanticInventoryBuildID(project, generation) + + batch := make([]catalog.SemanticInventoryContribution, 0, rows) + for i := 0; i < rows; i++ { + key := fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("pgcat05-large/%d", i)))) + batch = append(batch, catalog.SemanticInventoryContribution{ + Key: key, + Project: project, + DatasetGeneration: generation, + BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceRetained, + ResourceType: "Observation", + SourceID: fmt.Sprintf("retained:Observation/large-%05d", i), + Ordinal: 0, + ConceptID: fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("concept/%d", i)))), + BindingID: fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("binding/%d", i)))), + Example: fmt.Sprintf("example-%d", i), + Observation: goldenObs, + }) + } + + if err := repo.WriteSemanticInventoryContributions(s.ctx, batch, 5000); err != nil { + t.Fatalf("contract-maximum batch of %d rows: %v", rows, err) + } + var count int64 + if err := s.iso.QueryRow(s.ctx, "SELECT COUNT(*) FROM "+s.contributionFQN()+" WHERE project = $1", project).Scan(&count); err != nil { + s.fail("count large-batch rows", err) + } + if count != rows { + t.Fatalf("large-batch rows = %d, want %d", count, rows) + } + + if err := repo.WriteSemanticInventoryContributions(s.ctx, batch, 5000); err != nil { + t.Fatalf("large-batch replay: %v", err) + } + if err := s.iso.QueryRow(s.ctx, "SELECT COUNT(*) FROM "+s.contributionFQN()+" WHERE project = $1", project).Scan(&count); err != nil { + s.fail("recount large-batch rows", err) + } + if count != rows { + t.Fatalf("large-batch rows after replay = %d, want %d (replay must not add rows)", count, rows) + } + for _, i := range []int{0, 1, rows / 2, rows - 2, rows - 1} { + key := fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("pgcat05-large/%d", i)))) + if _, found := s.witnessContribution(t, key); !found { + t.Fatalf("row %d (chunk-crossing sample) missing after clamped write", i) + } + } +} + +func TestSemanticContributionForgedKeyCrossSlotSkipIsClassified(t *testing.T) { + s := newPgcat05Service(t) + repo := s.contributionRepository(t) + + forgedKey := fmt.Sprintf("%x", sha256.Sum256([]byte("pgcat05-forged-key"))) + g := pgcat05Goldens["obs-a"] + + emitInProject := func(project, sourceID string) catalog.SemanticInventoryContribution { + emitted := pgcat05EmitContribution(t, project, "g1", g.authPath, g.resourceType, sourceID, g.payload) + if len(emitted) != 1 { + t.Fatalf("%s: emitter produced %d contributions, want exactly 1", project, len(emitted)) + } + c := emitted[0] + c.Key = forgedKey + return c + } + countIn := func(project string) int64 { + var n int64 + if err := s.iso.QueryRow(s.ctx, "SELECT COUNT(*) FROM "+s.contributionFQN()+" WHERE project = $1", project).Scan(&n); err != nil { + s.fail("count rows for "+project, err) + } + return n + } + + t.Run("in-batch-forged-key-across-slots-refuses-with-typed-conflict", func(t *testing.T) { + a := emitInProject("pgcat05-fk-inbatch-a", "retained:Observation/fk") + b := emitInProject("pgcat05-fk-inbatch-b", "retained:Observation/fk") + err := repo.WriteSemanticInventoryContributions(s.ctx, []catalog.SemanticInventoryContribution{a, b}, 0) + var conflict *catalogstore.SemanticInventoryContributionConflictError + if err == nil { + w, _ := s.witnessContribution(t, forgedKey) + t.Fatalf("SILENT DROP: two distinct-slot contributions under one forged key returned nil; "+ + "only this row stored (count-a=%d count-b=%d row=%+v) — the second payload vanished without a typed conflict", + countIn("pgcat05-fk-inbatch-a"), countIn("pgcat05-fk-inbatch-b"), w) + } + if !errors.As(err, &conflict) && !errors.Is(err, catalogstore.ErrSemanticInventoryContributionConflict) { + t.Fatalf("forged-key in-batch write returned unclassified error: %v", err) + } + if n := countIn("pgcat05-fk-inbatch-b"); n != 0 { + t.Fatalf("forged-key batch wrote into the refused slot: %d rows", n) + } + w, found := s.witnessContribution(t, forgedKey) + if !found { + t.Fatal("forged-key row missing after refused in-batch write") + } + first := a + if w.Project != first.Project { + first = b + } + firstObs, err := json.Marshal(first.Observation) + if err != nil { + t.Fatalf("encode winning observation: %v", err) + } + if w.PayloadDigest != semanticContributionPayloadDigest(first, firstObs) { + t.Fatalf("winning stored row digest = %s, want the winning contribution's digest", w.PayloadDigest) + } + }) + + t.Run("cross-batch-forged-key-positive-control", func(t *testing.T) { + otherForged := fmt.Sprintf("%x", sha256.Sum256([]byte("pgcat05-forged-key-cross-batch"))) + a := emitInProject("pgcat05-fkx-1", "retained:Observation/kx") + a.Key = otherForged + b := emitInProject("pgcat05-fkx-2", "retained:Observation/kx") + b.Key = otherForged + if err := repo.WriteSemanticInventoryContributions(s.ctx, []catalog.SemanticInventoryContribution{a}, 0); err != nil { + t.Fatalf("first forged write: %v", err) + } + err := repo.WriteSemanticInventoryContributions(s.ctx, []catalog.SemanticInventoryContribution{b}, 0) + if !errors.Is(err, catalogstore.ErrSemanticInventoryContributionConflict) { + t.Fatalf("cross-batch forged key with differing payload = %v, want typed conflict", err) + } + if _, ok := s.witnessContribution(t, otherForged); !ok { + t.Fatal("stored forged row vanished") + } + if n := countIn("pgcat05-fkx-2"); n != 0 { + t.Fatalf("conflicting forged row was written: %d", n) + } + }) + + t.Run("honest-replay-still-nil-no-false-conflict", func(t *testing.T) { + c := pgcat05EmitContribution(t, "pgcat05-fk-replay", "g1", g.authPath, g.resourceType, + "retained:Observation/replay", g.payload)[0] + if err := repo.WriteSemanticInventoryContributions(s.ctx, []catalog.SemanticInventoryContribution{c}, 0); err != nil { + t.Fatalf("first honest write: %v", err) + } + before, ok := s.witnessContribution(t, c.Key) + if !ok { + t.Fatal("honest row missing after first write") + } + if err := repo.WriteSemanticInventoryContributions(s.ctx, []catalog.SemanticInventoryContribution{c}, 0); err != nil { + t.Fatalf("honest replay returned %v, want nil (offered slot is held by exactly its own key)", err) + } + after, ok := s.witnessContribution(t, c.Key) + if !ok { + t.Fatal("honest row missing after replay") + } + before.RecordedAt, after.RecordedAt = time.Time{}, time.Time{} + if before != after { + t.Fatalf("replay moved the stored row:\n before %+v\n after %+v", before, after) + } + }) +} + +func prepareContributionTest(contribution catalog.SemanticInventoryContribution) (digest string, observation []byte, err error) { + contribution.DatasetGeneration = catalog.NormalizeDatasetGeneration(contribution.DatasetGeneration) + if !isLowerHex64(contribution.Key) { + return "", nil, fmt.Errorf("contribution key %q is not 64 lowercase hex characters", contribution.Key) + } + if contribution.Project == "" || contribution.DatasetGeneration == "" || contribution.BuildID == "" || + contribution.SourceID == "" || contribution.ConceptID == "" || contribution.BindingID == "" || + contribution.ResourceType == "" { + return "", nil, errors.New("contribution is missing stable identity") + } + if contribution.Ordinal < 0 { + return "", nil, fmt.Errorf("contribution ordinal is %d, want >= 0", contribution.Ordinal) + } + switch contribution.SourceKind { + case catalog.SemanticInventorySourceFile, catalog.SemanticInventorySourceRetained: + default: + return "", nil, fmt.Errorf("contribution source kind %q is unknown", contribution.SourceKind) + } + if contribution.BuildID != catalog.SemanticInventoryBuildID(contribution.Project, contribution.DatasetGeneration) { + return "", nil, errors.New("contribution build does not match its generation") + } + observation, err = json.Marshal(contribution.Observation) + if err != nil { + return "", nil, fmt.Errorf("encode contribution observation: %w", err) + } + if len(observation) == 0 || observation[0] != '{' { + return "", nil, errors.New("contribution observation does not encode as a JSON object") + } + return semanticContributionPayloadDigest(contribution, observation), observation, nil +} +func isLowerHex64(s string) bool { + if len(s) != 64 { + return false + } + for i := 0; i < len(s); i++ { + ch := s[i] + if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') { + return false + } + } + return true +} + +func semanticContributionPayloadDigest(c catalog.SemanticInventoryContribution, observation []byte) string { + h := sha256.New() + put := func(s string) { + h.Write([]byte(strconv.Itoa(len(s)))) + h.Write([]byte{':'}) + h.Write([]byte(s)) + } + put("loom.semantic-contribution/v1\n") + put(c.Project) + put(c.DatasetGeneration) + put(c.BuildID) + put(c.SourceKind) + put(c.AuthResourcePath) + put(c.ResourceType) + put(c.SourceID) + put("o" + strconv.Itoa(c.Ordinal)) + put(c.ConceptID) + put(c.BindingID) + put(c.Example) + put(string(observation)) + return hex.EncodeToString(h.Sum(nil)) +} diff --git a/internal/backends/postgres/internal/catalog/semantic_page.go b/internal/backends/postgres/internal/catalog/semantic_page.go new file mode 100644 index 000000000..08b0c565f --- /dev/null +++ b/internal/backends/postgres/internal/catalog/semantic_page.go @@ -0,0 +1,955 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + maxSemanticInventoryResolveSelections = 100 + semanticCategoryRuleHint = "CODED_CONCEPT_V1" +) + +var ( + ErrInvalidSemanticInventoryPage = errors.New("postgres: invalid semantic inventory page request") + ErrInvalidSemanticInventorySelection = errors.New("postgres: invalid semantic inventory selection request") +) + +type SemanticReadRepository struct { + src storepostgres.ConnSource + schema string + contributionFQN string + buildFQN string +} + +func NewSemanticReadRepository(src storepostgres.ConnSource, schema string) (*SemanticReadRepository, error) { + if src == nil { + return nil, errors.New("postgres: nil semantic read connection source") + } + if !storepostgres.ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", storepostgres.ErrInvalidSchemaName, schema) + } + return &SemanticReadRepository{ + src: src, + schema: schema, + contributionFQN: pgx.Identifier{schema, semanticInventoryContributionTableName}.Sanitize(), + buildFQN: pgx.Identifier{schema, semanticInventoryBuildTableName}.Sanitize(), + }, nil +} + +var ( + _ func(context.Context, catalog.SemanticInventoryPageOptions) (catalog.SemanticInventoryPage, error) = (*SemanticReadRepository)(nil).PageSemanticInventory + _ func(context.Context, catalog.SemanticInventoryResolveOptions) (catalog.SemanticInventoryResolveResult, error) = (*SemanticReadRepository)(nil).ResolveSemanticInventorySelections +) + +func (r *SemanticReadRepository) PageSemanticInventory(ctx context.Context, opts catalog.SemanticInventoryPageOptions) (catalog.SemanticInventoryPage, error) { + if strings.TrimSpace(opts.Project) == "" || catalog.NormalizeDatasetGeneration(opts.DatasetGeneration) == "" || + (opts.SelectionMode != "" && opts.SelectionMode != catalog.SemanticInventoryExactConcept && opts.SelectionMode != catalog.SemanticInventorySystemVersionCategory) { + return catalog.SemanticInventoryPage{}, ErrInvalidSemanticInventoryPage + } + paths := append([]string(nil), opts.AuthResourcePaths...) + sort.Strings(paths) + paths = compactSemanticAuthPaths(paths) + unrestricted := catalog.EffectiveAuthResourcePathsUnrestricted(paths, opts.AuthResourcePathsUnrestricted) + limit := semanticInventoryPageLimit(opts.Limit) + + buildsExist, err := r.tableExists(ctx, r.buildFQN) + if err != nil { + return catalog.SemanticInventoryPage{}, err + } + if !buildsExist { + return catalog.SemanticInventoryPage{Entries: []catalog.SemanticInventoryEntry{}, State: catalog.SemanticInventoryUnknown}, nil + } + buildID := catalog.SemanticInventoryBuildID(opts.Project, opts.DatasetGeneration) + build, found, err := r.readBuildByID(ctx, opts.Project, catalog.NormalizeDatasetGeneration(opts.DatasetGeneration), buildID) + if err != nil { + return catalog.SemanticInventoryPage{}, err + } + if !found { + return catalog.SemanticInventoryPage{Entries: []catalog.SemanticInventoryEntry{}, State: catalog.SemanticInventoryUnknown}, nil + } + normalizeReadBuild(&build, opts.Project, catalog.NormalizeDatasetGeneration(opts.DatasetGeneration), buildID) + + page := catalog.SemanticInventoryPage{Entries: []catalog.SemanticInventoryEntry{}, Build: build, State: build.State} + if !unrestricted { + page.Build.ScannedResources = 0 + page.Build.Checkpoint = "" + page.Build.SourceCheckpoint = catalog.SemanticInventoryCheckpoint{} + page.Build.LeaseToken = "" + page.Build.LeaseExpiresAt = 0 + page.Build.Diagnostics = nil + page.Build.AuthResourcePath = "" + } + if build.State != catalog.SemanticInventoryComplete || build.BuildID == "" { + return page, nil + } + if build.EntryIndexVersion >= catalog.SemanticInventoryEntryIndexVersion { + return catalog.SemanticInventoryPage{}, &SemanticInventoryEntriesUnsupportedError{ + Project: build.Project, + DatasetGeneration: build.DatasetGeneration, + BuildID: build.BuildID, + EntryIndexVersion: build.EntryIndexVersion, + } + } + contributionsExist, err := r.tableExists(ctx, r.contributionFQN) + if err != nil { + return catalog.SemanticInventoryPage{}, err + } + if !contributionsExist { + return catalog.SemanticInventoryPage{Entries: []catalog.SemanticInventoryEntry{}, State: catalog.SemanticInventoryUnknown}, nil + } + afterBindingID, afterSelectionID, err := catalog.DecodeSemanticInventoryCursor(opts.Cursor, opts, build.BuildID) + if err != nil { + return catalog.SemanticInventoryPage{}, err + } + sourceKind := build.SourceKind + if sourceKind == "" { + sourceKind = catalog.SemanticInventorySourceFile + } + + sql, args := semanticPageQuery(r.contributionFQN, semanticPageParams{ + project: opts.Project, + generation: catalog.NormalizeDatasetGeneration(opts.DatasetGeneration), + buildID: build.BuildID, + sourceKind: sourceKind, + unrestricted: unrestricted, + paths: paths, + resourceType: opts.ResourceType, + bindingID: opts.BindingID, + query: opts.Query, + afterBindingID: afterBindingID, + afterSelection: afterSelectionID, + limit: limit, + familyMode: opts.SelectionMode == catalog.SemanticInventorySystemVersionCategory, + }) + rows, err := r.queryPageRows(ctx, sql, args...) + if err != nil { + return catalog.SemanticInventoryPage{}, err + } + + count := 0 + var lastBinding, lastSelection string + for i := range rows { + count++ + if count > limit { + break + } + row := &rows[i] + lastBinding, lastSelection = row.bindingID, row.selectionID + if row.isCategory { + family := row.toFamily(sourceKind, build.ObservationSchema) + family.FamilyID = catalog.SemanticInventoryFamilyID(family.SemanticInventoryFamilyReference) + page.Families = append(page.Families, family) + continue + } + page.Entries = append(page.Entries, row.toEntry(build.ObservationSchema)) + } + if count > limit { + page.NextCursor = catalog.EncodeSemanticInventoryCursor(opts, build.BuildID, lastBinding, lastSelection) + } + return page, nil +} + +type semanticPageParams struct { + project string + generation string + buildID string + sourceKind string + unrestricted bool + paths []string + resourceType string + bindingID string + query string + afterBindingID string + afterSelection string + limit int + familyMode bool +} + +func semanticPageQuery(contributionFQN string, p semanticPageParams) (string, []any) { + args := []any{ + p.project, p.generation, p.buildID, p.sourceKind, p.unrestricted, p.paths, + p.resourceType, p.bindingID, p.query, + p.afterBindingID, p.afterSelection, + p.limit + 1, + } + + var sb strings.Builder + sb.WriteString("WITH candidates AS (SELECT c.binding_id") + lateralMatch := " AND c.concept_id = i.concept_id" + if p.familyMode { + sb.WriteString(", c.concept_id" + + ", " + semanticCategoryPredicate("c") + " AS is_category" + + ", " + semanticFamilySelectionExpr("c") + " AS selection_id") + lateralMatch = " AND ((" + semanticCategoryPredicate("c") + " AND " + + semanticFamilySelectionExpr("c") + " = i.selection_id)" + + " OR (NOT i.is_category AND c.concept_id = i.concept_id))" + } else { + sb.WriteString(", c.concept_id, false AS is_category, c.concept_id AS selection_id") + } + sb.WriteString(" FROM " + contributionFQN + " c WHERE " + semanticScopeClause("c") + + " AND ($7 = '' OR c.resource_type = $7)" + + " AND ($8 = '' OR c.binding_id = $8)" + + " AND ($9 = '' OR " + semanticSearchContains("c", "$9") + "))," + + " identities AS (SELECT binding_id, max(concept_id) AS concept_id, selection_id, is_category FROM candidates WHERE " + + semanticKeysetPredicate("candidates.binding_id", "candidates.selection_id") + + " GROUP BY binding_id, selection_id, is_category" + + " ORDER BY binding_id COLLATE \"C\", selection_id COLLATE \"C\", is_category" + + " LIMIT $12)" + + " SELECT i.binding_id, i.concept_id, i.selection_id, i.is_category, a.* FROM identities i" + + " CROSS JOIN LATERAL (SELECT " + semanticEntryAggregateSelects("c") + + " FROM " + contributionFQN + " c WHERE " + semanticScopeClause("c") + + " AND ($7 = '' OR c.resource_type = $7)" + + " AND c.binding_id = i.binding_id" + lateralMatch + + ") a ORDER BY i.binding_id COLLATE \"C\", i.selection_id COLLATE \"C\", i.is_category") + return sb.String(), args +} + +func semanticResolveReferencesQuery(contributionFQN string, scope resolveScope, references []catalog.SemanticInventoryReference) (string, []any) { + args := scope.args() + var values strings.Builder + values.WriteString("(VALUES ") + for i, reference := range references { + if i > 0 { + values.WriteString(", ") + } + bindingParam := "$" + strconv.Itoa(len(args)+1) + args = append(args, reference.BindingID) + conceptParam := "$" + strconv.Itoa(len(args)+1) + args = append(args, reference.ConceptID) + values.WriteString("(" + bindingParam + "::text, " + conceptParam + "::text, " + strconv.Itoa(i) + ")") + } + values.WriteString(") AS requested(binding_id, concept_id, ord)") + + sql := "SELECT requested.ord, c.binding_id, c.concept_id, c.observation FROM " + values.String() + + " CROSS JOIN LATERAL (SELECT c.observation, c.binding_id, c.concept_id FROM " + contributionFQN + " c WHERE " + + semanticScopeClause("c") + + " AND c.binding_id = requested.binding_id AND c.concept_id = requested.concept_id" + + " ORDER BY c.contribution_key COLLATE \"C\" LIMIT 1) c ORDER BY requested.ord" + return sql, args +} + +func semanticResolveFamiliesQuery(contributionFQN string, scope resolveScope, references []catalog.SemanticInventoryFamilyReference) (string, []any) { + args := scope.args() + var values strings.Builder + values.WriteString("(VALUES ") + for i, reference := range references { + if i > 0 { + values.WriteString(", ") + } + values.WriteString("(") + for field, value := range []string{reference.BindingID, reference.ResourceType, reference.SourcePath, reference.System, reference.Version, reference.SourceKind} { + if field > 0 { + values.WriteString(", ") + } + param := "$" + strconv.Itoa(len(args)+1) + args = append(args, value) + values.WriteString(param + "::text") + } + values.WriteString(", " + strconv.Itoa(i) + ")") + } + values.WriteString(") AS requested(binding_id, resource_type, source_path, system, version, source_kind, ord)") + + join := " LEFT JOIN " + contributionFQN + " c ON " + semanticScopeClause("c") + + " AND c.binding_id = requested.binding_id" + + " AND c.resource_type = requested.resource_type" + + " AND c.observation #>> '{source,type}' = requested.resource_type" + + " AND c.observation #>> '{source,path}' = requested.source_path" + + " AND c.observation #>> '{key,system}' = requested.system" + + " AND " + semanticFamilyVersionMatch("c") + + " AND c.observation #>> '{rule_hint}' = " + sqlStringLiteral(semanticCategoryRuleHint) + + " AND " + semanticExtensionEmptyPredicate("c") + + sql := "SELECT requested.ord, " + semanticFamilyAggregateSelects("c") + " FROM " + values.String() + join + + " WHERE requested.source_kind = $4" + + " GROUP BY requested.ord HAVING COUNT(c.contribution_key) > 0" + + " ORDER BY requested.ord" + return sql, args +} + +func (r *SemanticReadRepository) ResolveSemanticInventorySelections(ctx context.Context, opts catalog.SemanticInventoryResolveOptions) (catalog.SemanticInventoryResolveResult, error) { + selectionCount := len(opts.References) + len(opts.FamilyReferences) + if strings.TrimSpace(opts.Project) == "" || catalog.NormalizeDatasetGeneration(opts.DatasetGeneration) == "" || + opts.AuthResourcePathsUnrestricted == nil || selectionCount == 0 || selectionCount > maxSemanticInventoryResolveSelections { + return catalog.SemanticInventoryResolveResult{}, ErrInvalidSemanticInventorySelection + } + paths := append([]string(nil), opts.AuthResourcePaths...) + sort.Strings(paths) + paths = compactSemanticAuthPaths(paths) + unrestricted := catalog.EffectiveAuthResourcePathsUnrestricted(paths, opts.AuthResourcePathsUnrestricted) + references := make([]catalog.SemanticInventoryReference, 0, len(opts.References)) + seen := make(map[string]struct{}, selectionCount) + for _, reference := range opts.References { + if strings.TrimSpace(reference.ConceptID) == "" || strings.TrimSpace(reference.BindingID) == "" { + return catalog.SemanticInventoryResolveResult{}, ErrInvalidSemanticInventorySelection + } + identity := reference.BindingID + "\x00" + reference.ConceptID + if _, ok := seen[identity]; ok { + return catalog.SemanticInventoryResolveResult{}, ErrInvalidSemanticInventorySelection + } + seen[identity] = struct{}{} + references = append(references, reference) + } + familyReferences := make([]catalog.SemanticInventoryFamilyReference, 0, len(opts.FamilyReferences)) + for _, reference := range opts.FamilyReferences { + if reference.FamilyID != catalog.SemanticInventoryFamilyID(reference) || strings.TrimSpace(reference.BindingID) == "" || + strings.TrimSpace(reference.System) == "" || strings.TrimSpace(reference.ResourceType) == "" || strings.TrimSpace(reference.SourcePath) == "" || + (reference.SourceKind != catalog.SemanticInventorySourceFile && reference.SourceKind != catalog.SemanticInventorySourceRetained) { + return catalog.SemanticInventoryResolveResult{}, ErrInvalidSemanticInventorySelection + } + if _, ok := seen[reference.FamilyID]; ok { + return catalog.SemanticInventoryResolveResult{}, ErrInvalidSemanticInventorySelection + } + seen[reference.FamilyID] = struct{}{} + familyReferences = append(familyReferences, reference) + } + + result := catalog.SemanticInventoryResolveResult{Entries: []catalog.SemanticInventoryEntry{}, State: catalog.SemanticInventoryUnknown} + buildsExist, err := r.tableExists(ctx, r.buildFQN) + if err != nil { + return catalog.SemanticInventoryResolveResult{}, err + } + if !buildsExist { + return result, nil + } + build, storedBuildID, found, err := r.readBuildByKey(ctx, catalog.SemanticInventoryBuildKey(opts.Project, opts.DatasetGeneration)) + if err != nil { + return catalog.SemanticInventoryResolveResult{}, err + } + buildID := catalog.SemanticInventoryBuildID(opts.Project, opts.DatasetGeneration) + if !found || storedBuildID != buildID { + return result, nil + } + normalizeReadBuild(&build, opts.Project, catalog.NormalizeDatasetGeneration(opts.DatasetGeneration), buildID) + result.Build, result.State = build, build.State + if !unrestricted { + result.Build.ScannedResources = 0 + result.Build.Checkpoint = "" + result.Build.SourceCheckpoint = catalog.SemanticInventoryCheckpoint{} + result.Build.LeaseToken = "" + result.Build.LeaseExpiresAt = 0 + result.Build.Diagnostics = nil + result.Build.AuthResourcePath = "" + } + if build.State != catalog.SemanticInventoryComplete { + return result, nil + } + if build.EntryIndexVersion >= catalog.SemanticInventoryEntryIndexVersion { + return catalog.SemanticInventoryResolveResult{}, &SemanticInventoryEntriesUnsupportedError{ + Project: build.Project, + DatasetGeneration: build.DatasetGeneration, + BuildID: build.BuildID, + EntryIndexVersion: build.EntryIndexVersion, + } + } + sourceKind := build.SourceKind + if sourceKind == "" { + sourceKind = catalog.SemanticInventorySourceFile + } + contributionsExist, err := r.tableExists(ctx, r.contributionFQN) + if err != nil { + return catalog.SemanticInventoryResolveResult{}, err + } + if !contributionsExist { + return catalog.SemanticInventoryResolveResult{}, &SemanticInventoryStorageMissingError{ + Schema: r.schema, Table: semanticInventoryContributionTableName, + } + } + + scope := resolveScope{ + project: opts.Project, + generation: catalog.NormalizeDatasetGeneration(opts.DatasetGeneration), + buildID: build.BuildID, + sourceKind: sourceKind, + unrestricted: unrestricted, + paths: paths, + observationSchema: build.ObservationSchema, + } + if len(references) > 0 { + entries, err := r.resolveReferences(ctx, scope, references) + if err != nil { + return catalog.SemanticInventoryResolveResult{}, err + } + result.Entries = entries + } + if len(familyReferences) > 0 { + families, err := r.resolveFamilies(ctx, scope, familyReferences) + if err != nil { + return catalog.SemanticInventoryResolveResult{}, err + } + result.Families = families + } + return result, nil +} + +type resolveScope struct { + project string + generation string + buildID string + sourceKind string + unrestricted bool + paths []string + observationSchema int +} + +func (s resolveScope) args() []any { + return []any{s.project, s.generation, s.buildID, s.sourceKind, s.unrestricted, s.paths} +} + +func (r *SemanticReadRepository) resolveReferences(ctx context.Context, scope resolveScope, references []catalog.SemanticInventoryReference) ([]catalog.SemanticInventoryEntry, error) { + sql, args := semanticResolveReferencesQuery(r.contributionFQN, scope, references) + + entries := make([]*catalog.SemanticInventoryEntry, len(references)) + err := r.withConn(ctx, func(conn *pgxpool.Conn) error { + rows, err := conn.Query(ctx, sql, args...) + if err != nil { + return fmt.Errorf("postgres: resolve semantic inventory selections: %w", err) + } + defer rows.Close() + for rows.Next() { + var ordinal int + var entry catalog.SemanticInventoryEntry + var observation []byte + if err := rows.Scan(&ordinal, &entry.BindingID, &entry.ConceptID, &observation); err != nil { + return fmt.Errorf("postgres: resolve semantic inventory selections: %w", err) + } + if ordinal < 0 || ordinal >= len(entries) { + return fmt.Errorf("postgres: resolve semantic inventory selections: unexpected ordinal %d", ordinal) + } + if err := json.Unmarshal(observation, &entry.Observation); err != nil { + return fmt.Errorf("postgres: decode resolved semantic inventory observation: %w", err) + } + entries[ordinal] = &entry + } + return rows.Err() + }) + if err != nil { + return nil, err + } + out := make([]catalog.SemanticInventoryEntry, 0, len(entries)) + for _, entry := range entries { + if entry != nil { + out = append(out, *entry) + } + } + return out, nil +} + +func (r *SemanticReadRepository) resolveFamilies(ctx context.Context, scope resolveScope, references []catalog.SemanticInventoryFamilyReference) ([]catalog.SemanticInventoryFamily, error) { + sql, args := semanticResolveFamiliesQuery(r.contributionFQN, scope, references) + + families := make([]*catalog.SemanticInventoryFamily, len(references)) + err := r.withConn(ctx, func(conn *pgxpool.Conn) error { + rows, err := conn.Query(ctx, sql, args...) + if err != nil { + return fmt.Errorf("postgres: resolve semantic inventory families: %w", err) + } + defer rows.Close() + for rows.Next() { + var row semanticAggregateRow + if err := rows.Scan(row.scanFamily()...); err != nil { + return fmt.Errorf("postgres: resolve semantic inventory families: %w", err) + } + if row.ord < 0 || row.ord >= len(families) { + return fmt.Errorf("postgres: resolve semantic inventory families: unexpected ordinal %d", row.ord) + } + reference := references[row.ord] + family := row.toFamily(scope.sourceKind, scope.observationSchema) + family.SemanticInventoryFamilyReference = reference + family.Source = catalog.SemanticObservationSource{ + Canonical: family.Source.Canonical, + Type: reference.ResourceType, + Profile: family.Source.Profile, + Path: reference.SourcePath, + } + family.FamilyID = catalog.SemanticInventoryFamilyID(reference) + families[row.ord] = &family + } + return rows.Err() + }) + if err != nil { + return nil, err + } + out := make([]catalog.SemanticInventoryFamily, 0, len(families)) + for _, family := range families { + if family != nil { + out = append(out, *family) + } + } + return out, nil +} + +func (r *SemanticReadRepository) readBuildByID(ctx context.Context, project, generation, buildID string) (catalog.SemanticInventoryBuild, bool, error) { + var build catalog.SemanticInventoryBuild + var found bool + err := r.withConn(ctx, func(conn *pgxpool.Conn) error { + err := conn.QueryRow(ctx, + "SELECT "+semanticInventoryBuildColumns+" FROM "+r.buildFQN+ + " WHERE project = $1 AND dataset_generation = $2 AND build_id = $3", + project, generation, buildID, + ).Scan(scanSemanticInventoryBuild(&build)...) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return fmt.Errorf("postgres: read semantic inventory build for page: %w", err) + } + found = true + return nil + }) + if err != nil { + return catalog.SemanticInventoryBuild{}, false, err + } + return build, found, nil +} + +func (r *SemanticReadRepository) readBuildByKey(ctx context.Context, buildKey string) (catalog.SemanticInventoryBuild, string, bool, error) { + var build catalog.SemanticInventoryBuild + var storedBuildID string + var found bool + err := r.withConn(ctx, func(conn *pgxpool.Conn) error { + err := conn.QueryRow(ctx, + "SELECT "+semanticInventoryBuildColumns+", build_id FROM "+r.buildFQN+" WHERE build_key = $1", + buildKey, + ).Scan(append(scanSemanticInventoryBuild(&build), &storedBuildID)...) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return fmt.Errorf("postgres: read semantic inventory build for resolve: %w", err) + } + found = true + return nil + }) + if err != nil { + return catalog.SemanticInventoryBuild{}, "", false, err + } + return build, storedBuildID, found, nil +} + +func (r *SemanticReadRepository) queryPageRows(ctx context.Context, sql string, args ...any) ([]semanticAggregateRow, error) { + var rows []semanticAggregateRow + err := r.withConn(ctx, func(conn *pgxpool.Conn) error { + result, err := conn.Query(ctx, sql, args...) + if err != nil { + return fmt.Errorf("postgres: page semantic inventory: %w", err) + } + defer result.Close() + for result.Next() { + var row semanticAggregateRow + if err := result.Scan(row.scanPageRow()...); err != nil { + return fmt.Errorf("postgres: page semantic inventory: %w", err) + } + rows = append(rows, row) + } + return result.Err() + }) + if err != nil { + return nil, err + } + return rows, nil +} + +func (r *SemanticReadRepository) tableExists(ctx context.Context, fqn string) (bool, error) { + var exists bool + err := r.withConn(ctx, func(conn *pgxpool.Conn) error { + if err := conn.QueryRow(ctx, "SELECT to_regclass($1) IS NOT NULL", fqn).Scan(&exists); err != nil { + return fmt.Errorf("postgres: check semantic inventory table %s: %w", fqn, err) + } + return nil + }) + if err != nil { + return false, err + } + return exists, nil +} + +func (r *SemanticReadRepository) withConn(ctx context.Context, fn func(*pgxpool.Conn) error) error { + conn, err := r.src.Acquire(ctx) + if err != nil { + return fmt.Errorf("postgres: acquire semantic read connection: %w", err) + } + defer conn.Release() + return fn(conn) +} + +func compactSemanticAuthPaths(paths []string) []string { + if len(paths) < 2 { + return paths + } + write := 1 + for read := 1; read < len(paths); read++ { + if paths[read] != paths[write-1] { + paths[write] = paths[read] + write++ + } + } + return paths[:write] +} + +func semanticInventoryPageLimit(limit int) int { + if limit <= 0 || limit > catalog.SemanticInventoryPageLimit { + return catalog.SemanticInventoryPageLimit + } + return limit +} + +func semanticScopeClause(alias string) string { + return alias + ".project = $1" + + " AND " + alias + ".dataset_generation = $2" + + " AND " + alias + ".build_id = $3" + + " AND " + alias + ".source_kind = $4" + + " AND ($5 OR " + alias + ".auth_resource_path = ANY($6::text[]))" +} + +func semanticKeysetPredicate(bindingColumn, selectionColumn string) string { + return "($10 = ''" + + " OR " + bindingColumn + " COLLATE \"C\" > $10 COLLATE \"C\"" + + " OR (" + bindingColumn + " = $10 COLLATE \"C\"" + + " AND " + selectionColumn + " COLLATE \"C\" > $11 COLLATE \"C\"))" +} + +func semanticLeafText(alias string, path ...string) string { + return "(" + alias + ".observation #>> '" + semanticPath(path) + "')" +} + +func semanticLeafToken(alias string, path ...string) string { + return "COALESCE(" + semanticLeafText(alias, path...) + ", '')" +} + +func semanticLeafJSON(alias string, path ...string) string { + return "(" + alias + ".observation #> '" + semanticPath(path) + "')" +} + +func sqlStringLiteral(value string) string { + return "'" + strings.ReplaceAll(value, "'", "''") + "'" +} + +func semanticPath(path []string) string { + return "{" + strings.Join(path, ",") + "}" +} + +func semanticMinText(alias string, path ...string) string { + return "min(" + semanticLeafText(alias, path...) + " COLLATE \"C\")" +} + +func semanticArrayLeafExpr(alias string, path ...string) string { + leaf := semanticLeafJSON(alias, path...) + return "CASE WHEN jsonb_typeof(" + leaf + ") = 'array' THEN " + leaf + " END" +} + +func semanticMinArray(alias string, path ...string) string { + expr := semanticArrayLeafExpr(alias, path...) + return "(array_agg(" + expr + " ORDER BY " + expr + " ASC NULLS LAST))[1]" +} + +func semanticMaxArray(alias string, path ...string) string { + expr := semanticArrayLeafExpr(alias, path...) + return "(array_agg(" + expr + " ORDER BY " + expr + " DESC NULLS LAST))[1]" +} + +func semanticMinBooleanText(alias string, path ...string) string { + leaf := semanticLeafJSON(alias, path...) + return "min(CASE WHEN jsonb_typeof(" + leaf + ") = 'boolean' THEN " + + semanticLeafText(alias, path...) + " COLLATE \"C\" END)" +} + +func semanticPopulationSum(alias string) string { + leaf := semanticLeafText(alias, "population") + return "coalesce(sum(CASE WHEN jsonb_typeof(" + semanticLeafJSON(alias, "population") + ") = 'number'" + + " AND " + leaf + " ~ '^-?[0-9]+$' THEN (" + leaf + ")::numeric ELSE 0 END), 0)::bigint" +} + +func semanticExampleMin(alias string) string { + return "min(NULLIF(" + alias + ".example, '') COLLATE \"C\")" +} + +func semanticExamplesTruncatedAny(alias string) string { + leaf := semanticLeafJSON(alias, "examples_truncated") + return "max(CASE WHEN " + leaf + " IS NULL OR " + leaf + " = 'false'::jsonb OR " + leaf + " = 'null'::jsonb" + + " THEN 0 ELSE 1 END)" +} + +func semanticExtensionEmptyPredicate(alias string) string { + leaf := semanticLeafJSON(alias, "extension_url_path") + return "(CASE WHEN " + leaf + " IS NULL THEN true" + + " WHEN jsonb_typeof(" + leaf + ") = 'array' THEN jsonb_array_length(" + leaf + ") = 0" + + " ELSE false END)" +} + +func semanticCategoryPredicate(alias string) string { + return "((" + semanticLeafText(alias, "rule_hint") + ") = " + sqlStringLiteral(semanticCategoryRuleHint) + + " AND " + semanticLeafToken(alias, "key", "system") + " <> ''" + + " AND " + semanticExtensionEmptyPredicate(alias) + ")" +} + +func semanticFamilyElement(alias string, path ...string) string { + leaf := semanticLeafJSON(alias, path...) + return "COALESCE(" + leaf + "::text, 'null')" +} + +func semanticFamilyVersionElement(alias string) string { + leaf := semanticLeafJSON(alias, "key", "version") + return "CASE WHEN " + leaf + " IS NULL THEN '\"\"' WHEN " + leaf + " = 'null'::jsonb THEN 'null' ELSE " + leaf + "::text END" +} + +func semanticFamilyVersionMatch(alias string) string { + leaf := semanticLeafJSON(alias, "key", "version") + return "(CASE WHEN " + leaf + " IS NULL THEN '' WHEN " + leaf + " = 'null'::jsonb THEN NULL" + + " ELSE (" + alias + ".observation #>> '{key,version}') END) = requested.version" +} + +func semanticFamilySelectionExpr(alias string) string { + return "CASE WHEN " + semanticCategoryPredicate(alias) + " THEN 'family:['" + + " || " + semanticFamilyElement(alias, "source", "type") + " || ','" + + " || " + semanticFamilyElement(alias, "source", "path") + " || ','" + + " || " + semanticFamilyElement(alias, "key", "system") + " || ','" + + " || " + semanticFamilyVersionElement(alias) + " || ']'" + + " ELSE " + alias + ".concept_id END" +} + +func semanticSearchContains(alias string, queryParam string) string { + search := "concat_ws(' ', " + alias + ".resource_type" + + ", " + semanticLeafText(alias, "source", "path") + + ", " + semanticLeafText(alias, "value", "selector") + + ", " + semanticLeafText(alias, "owning_scope") + + ", " + semanticLeafText(alias, "key", "system") + + ", " + semanticLeafText(alias, "key", "code") + + ", " + semanticLeafText(alias, "key", "display") + ")" + return "strpos(lower(" + search + " COLLATE \"C\"), lower(" + queryParam + " COLLATE \"C\")) > 0" +} + +func semanticEntryAggregateSelects(alias string) string { + return semanticPopulationSum(alias) + " AS agg_population" + + ", " + semanticExampleMin(alias) + " AS agg_example" + + ", " + semanticMinText(alias, "source", "canonical") + " AS agg_canonical" + + ", " + semanticMinText(alias, "source", "type") + " AS agg_source_type" + + ", " + semanticMinText(alias, "source", "profile") + " AS agg_profile" + + ", " + semanticMinText(alias, "source", "path") + " AS agg_path" + + ", " + semanticMinText(alias, "key", "selector") + " AS agg_key_selector" + + ", " + semanticMinText(alias, "key", "system") + " AS agg_key_system" + + ", " + semanticMinText(alias, "key", "version") + " AS agg_key_version" + + ", min(" + semanticLeafToken(alias, "key", "version") + " COLLATE \"C\") AS agg_version_token" + + ", " + semanticMinText(alias, "key", "code") + " AS agg_key_code" + + ", " + semanticMinText(alias, "key", "display") + " AS agg_key_display" + + ", " + semanticMinText(alias, "value", "selector") + " AS agg_value_selector" + + ", " + semanticMinText(alias, "value", "type") + " AS agg_value_type" + + ", " + semanticMinArray(alias, "value", "fallback_selectors") + " AS agg_fallback_selectors" + + ", " + semanticMinBooleanText(alias, "value", "fallback_on_blank_string") + " AS agg_fallback_on_blank" + + ", " + semanticMinText(alias, "owning_scope") + " AS agg_owning_scope" + + ", " + semanticMinArray(alias, "extension_url_path") + " AS agg_extension_url_path" + + ", " + semanticMinText(alias, "choice_arm") + " AS agg_choice_arm" + + ", " + semanticMinText(alias, "logical_type") + " AS agg_logical_type" + + ", " + semanticMinArray(alias, "observed_units") + " AS agg_units_min" + + ", " + semanticMaxArray(alias, "observed_units") + " AS agg_units_max" + + ", " + semanticMinText(alias, "completeness") + " AS agg_completeness" + + ", " + semanticMinText(alias, "status") + " AS agg_status" + + ", " + semanticMinText(alias, "rule_hint") + " AS agg_rule_hint" + + ", " + semanticMinText(alias, "rule_version") + " AS agg_rule_version" + + ", " + semanticExamplesTruncatedAny(alias) + " AS agg_examples_truncated" +} + +func semanticFamilyAggregateSelects(alias string) string { + return semanticPopulationSum(alias) + " AS agg_population" + + ", " + semanticExampleMin(alias) + " AS agg_example" + + ", " + semanticMinText(alias, "source", "canonical") + " AS agg_canonical" + + ", " + semanticMinText(alias, "source", "type") + " AS agg_source_type" + + ", " + semanticMinText(alias, "source", "profile") + " AS agg_profile" + + ", " + semanticMinText(alias, "source", "path") + " AS agg_path" + + ", " + semanticMinText(alias, "key", "selector") + " AS agg_key_selector" + + ", " + semanticMinText(alias, "key", "system") + " AS agg_key_system" + + ", min(" + semanticLeafToken(alias, "key", "version") + " COLLATE \"C\") AS agg_version_token" + + ", " + semanticMinText(alias, "value", "selector") + " AS agg_value_selector" + + ", " + semanticMinText(alias, "value", "type") + " AS agg_value_type" + + ", " + semanticMinArray(alias, "value", "fallback_selectors") + " AS agg_fallback_selectors" + + ", " + semanticMinBooleanText(alias, "value", "fallback_on_blank_string") + " AS agg_fallback_on_blank" + + ", " + semanticMinText(alias, "owning_scope") + " AS agg_owning_scope" + + ", " + semanticMinText(alias, "logical_type") + " AS agg_logical_type" + + ", " + semanticMinText(alias, "completeness") + " AS agg_completeness" + + ", " + semanticMinText(alias, "status") + " AS agg_status" + + ", " + semanticMinText(alias, "rule_version") + " AS agg_rule_version" + + ", " + semanticExamplesTruncatedAny(alias) + " AS agg_examples_truncated" +} + +type semanticAggregateRow struct { + ord int + bindingID string + conceptID string + selectionID string + isCategory bool + population int64 + example *string + canonical *string + sourceType *string + profile *string + path *string + keySelector *string + keySystem *string + keyVersion *string + versionToken *string + keyCode *string + keyDisplay *string + valueSelector *string + valueType *string + fallbackSelectors []byte + fallbackOnBlank *string + owningScope *string + extensionURLPath []byte + choiceArm *string + logicalType *string + unitsMin []byte + unitsMax []byte + completeness *string + status *string + ruleHint *string + ruleVersion *string + examplesTruncated int +} + +func (row *semanticAggregateRow) scanPageRow() []any { + return []any{ + &row.bindingID, &row.conceptID, &row.selectionID, &row.isCategory, + &row.population, &row.example, &row.canonical, &row.sourceType, + &row.profile, &row.path, &row.keySelector, &row.keySystem, + &row.keyVersion, &row.versionToken, &row.keyCode, &row.keyDisplay, + &row.valueSelector, &row.valueType, &row.fallbackSelectors, &row.fallbackOnBlank, + &row.owningScope, &row.extensionURLPath, &row.choiceArm, &row.logicalType, + &row.unitsMin, &row.unitsMax, &row.completeness, &row.status, + &row.ruleHint, &row.ruleVersion, &row.examplesTruncated, + } +} + +func (row *semanticAggregateRow) scanFamily() []any { + return []any{ + &row.ord, + &row.population, &row.example, &row.canonical, &row.sourceType, + &row.profile, &row.path, &row.keySelector, &row.keySystem, + &row.versionToken, &row.valueSelector, &row.valueType, + &row.fallbackSelectors, &row.fallbackOnBlank, &row.owningScope, + &row.logicalType, &row.completeness, &row.status, &row.ruleVersion, + &row.examplesTruncated, + } +} + +func semanticStringOrEmpty(value *string) string { + if value == nil { + return "" + } + return *value +} + +func semanticStringList(raw []byte) []string { + if len(raw) == 0 { + return nil + } + var out []string + if err := json.Unmarshal(raw, &out); err != nil { + return nil + } + return out +} + +func (row *semanticAggregateRow) toEntry(observationSchema int) catalog.SemanticInventoryEntry { + entry := catalog.SemanticInventoryEntry{ + ConceptID: row.conceptID, + BindingID: row.bindingID, + Observation: catalog.SemanticObservation{ + SchemaVersion: observationSchema, + Source: catalog.SemanticObservationSource{ + Canonical: semanticStringOrEmpty(row.canonical), + Type: semanticStringOrEmpty(row.sourceType), + Profile: semanticStringOrEmpty(row.profile), + Path: semanticStringOrEmpty(row.path), + }, + Key: catalog.SemanticObservationKey{ + Selector: semanticStringOrEmpty(row.keySelector), + System: semanticStringOrEmpty(row.keySystem), + Version: semanticStringOrEmpty(row.keyVersion), + Code: semanticStringOrEmpty(row.keyCode), + Display: semanticStringOrEmpty(row.keyDisplay), + }, + Value: catalog.SemanticObservationValue{ + Selector: semanticStringOrEmpty(row.valueSelector), + Type: semanticStringOrEmpty(row.valueType), + FallbackSelectors: semanticStringList(row.fallbackSelectors), + FallbackOnBlankString: semanticStringOrEmpty(row.fallbackOnBlank) == "true", + }, + OwningScope: semanticStringOrEmpty(row.owningScope), + ExtensionURLPath: semanticStringList(row.extensionURLPath), + ChoiceArm: semanticStringOrEmpty(row.choiceArm), + LogicalType: semanticStringOrEmpty(row.logicalType), + ObservedUnits: []string{}, + Completeness: catalog.SemanticObservationCompleteness(semanticStringOrEmpty(row.completeness)), + Status: semanticStringOrEmpty(row.status), + Population: row.population, + Examples: []string{}, + ExamplesTruncated: row.examplesTruncated > 0 || row.population > 1, + RuleHint: semanticStringOrEmpty(row.ruleHint), + RuleVersion: semanticStringOrEmpty(row.ruleVersion), + }, + } + if units := semanticStringList(row.unitsMin); units != nil { + entry.Observation.ObservedUnits = units + } + entry.Observation.ObservedUnitsTruncated = !equalJSONArray(row.unitsMin, row.unitsMax) + if example := semanticStringOrEmpty(row.example); example != "" { + entry.Observation.Examples = []string{example} + } + return entry +} + +func (row *semanticAggregateRow) toFamily(sourceKind string, observationSchema int) catalog.SemanticInventoryFamily { + family := catalog.SemanticInventoryFamily{ + SemanticInventoryFamilyReference: catalog.SemanticInventoryFamilyReference{ + BindingID: row.bindingID, + SourceKind: sourceKind, + ResourceType: semanticStringOrEmpty(row.sourceType), + SourcePath: semanticStringOrEmpty(row.path), + System: semanticStringOrEmpty(row.keySystem), + Version: semanticStringOrEmpty(row.versionToken), + }, + Source: catalog.SemanticObservationSource{ + Canonical: semanticStringOrEmpty(row.canonical), + Type: semanticStringOrEmpty(row.sourceType), + Profile: semanticStringOrEmpty(row.profile), + Path: semanticStringOrEmpty(row.path), + }, + KeySelector: semanticStringOrEmpty(row.keySelector), + Value: catalog.SemanticObservationValue{ + Selector: semanticStringOrEmpty(row.valueSelector), + Type: semanticStringOrEmpty(row.valueType), + FallbackSelectors: semanticStringList(row.fallbackSelectors), + FallbackOnBlankString: semanticStringOrEmpty(row.fallbackOnBlank) == "true", + }, + OwningScope: semanticStringOrEmpty(row.owningScope), + LogicalType: semanticStringOrEmpty(row.logicalType), + SchemaVersion: observationSchema, + RuleVersion: semanticStringOrEmpty(row.ruleVersion), + Completeness: catalog.SemanticObservationCompleteness(semanticStringOrEmpty(row.completeness)), + Status: semanticStringOrEmpty(row.status), + Population: row.population, + Examples: []string{}, + ExamplesTruncated: row.examplesTruncated > 0 || row.population > 1, + } + if example := semanticStringOrEmpty(row.example); example != "" { + family.Examples = []string{example} + } + return family +} + +func equalJSONArray(left, right []byte) bool { + if len(left) == 0 && len(right) == 0 { + return true + } + return string(left) == string(right) +} diff --git a/internal/backends/postgres/internal/catalog/semantic_page_test.go b/internal/backends/postgres/internal/catalog/semantic_page_test.go new file mode 100644 index 000000000..0c71bb437 --- /dev/null +++ b/internal/backends/postgres/internal/catalog/semantic_page_test.go @@ -0,0 +1,2339 @@ +package postgres + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "sync" + "testing" + "time" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +const pgcat06FixturePrefix = "verify_cat06_" + +var pgcat06NamePattern = regexp.MustCompile(`^verify_cat06_[0-9]+_[0-9a-f]+$`) + +var ( + pgcat06OwnedMu sync.Mutex + pgcat06OwnedByName = map[string]string{} +) + +func pgcat06Register(db, schema string) { + pgcat06OwnedMu.Lock() + defer pgcat06OwnedMu.Unlock() + pgcat06OwnedByName[db] = schema +} + +type pgcat06Service struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func pgcat06TestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 integration tests") + } + return raw +} + +func pgcat06RandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) +} + +func newPgcat06Service(t *testing.T) *pgcat06Service { + t.Helper() + raw := pgcat06TestPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var dbName, schema string + + dbName = fmt.Sprintf("verify_cat06_%d_%s", time.Now().UnixNano(), pgcat06RandomHex(t)) + if !pgcat06NamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + pgcat06Register(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema = fmt.Sprintf("verify_cat06_%d_%s", time.Now().UnixNano(), pgcat06RandomHex(t)) + if !pgcat06NamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + pgcat06Register(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + service := &pgcat06Service{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} + if _, err := storepostgres.MigrateSchema(ctx, service.openClient(t), schema); err != nil { + fail("MigrateSchema", err) + } + return service +} + +func (s *pgcat06Service) openClient(t *testing.T) *storepostgres.Client { + t.Helper() + client, err := storepostgres.Open(s.ctx, storepostgres.Config{ + DSN: s.dsn, + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *pgcat06Service) openDedicatedClient(t *testing.T, label string) (*storepostgres.Client, int32) { + t.Helper() + client, err := storepostgres.Open(s.ctx, storepostgres.Config{ + DSN: s.dsn, + MaxConns: 1, + MinConns: 1, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + s.fail("open dedicated client "+label, err) + } + t.Cleanup(client.Close) + pid := s.witnessClientPid(t, client, label) + return client, pid +} + +func (s *pgcat06Service) witnessClientPid(t *testing.T, client *storepostgres.Client, label string) int32 { + t.Helper() + conn, err := client.Acquire(s.ctx) + if err != nil { + s.fail("acquire connection "+label, err) + } + defer conn.Release() + var pid int32 + if err := conn.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + s.fail("witness pg_backend_pid "+label, err) + } + if pid <= 0 { + t.Fatalf("pg_backend_pid() for %s returned %d, want a live backend", label, pid) + } + return pid +} + +func (s *pgcat06Service) readRepository(t *testing.T) *SemanticReadRepository { + t.Helper() + repo, err := NewSemanticReadRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new semantic read repository", err) + } + return repo +} + +func (s *pgcat06Service) contributionRepository(t *testing.T) *SemanticContributionRepository { + t.Helper() + repo, err := NewSemanticContributionRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new semantic contribution repository", err) + } + return repo +} + +func (s *pgcat06Service) exec(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := s.iso.Exec(s.ctx, sql, args...); err != nil { + s.fail("fixture exec", err) + } +} + +func (s *pgcat06Service) contributionFQN() string { + return pgx.Identifier{s.schema, semanticInventoryContributionTableName}.Sanitize() +} + +func (s *pgcat06Service) buildFQN() string { + return pgx.Identifier{s.schema, semanticInventoryBuildTableName}.Sanitize() +} + +func (s *pgcat06Service) ledgerFQN() string { + return pgx.Identifier{s.schema, storepostgres.LedgerTable}.Sanitize() +} + +func pgcat06Hex64(label string) string { + sum := sha256.Sum256([]byte("pgcat06/" + label)) + return hex.EncodeToString(sum[:]) +} + +type buildSeed struct { + Project string + Generation string + State catalog.SemanticInventoryState + ObservationSchema int + InventorySchema int + EntryIndexVersion int + RuleVersion int + SourceKind string + SourceAvailability string + AuthResourcePath string + ScannedResources int64 + LeaseToken string + LeaseExpiresAt int64 + Diagnostics []string +} + +func pgcat06DefaultBuildSeed(project, generation string) buildSeed { + return buildSeed{ + Project: project, Generation: generation, + State: catalog.SemanticInventoryComplete, + ObservationSchema: catalog.SemanticObservationSchemaVersion, + InventorySchema: catalog.SemanticInventorySchemaVersion, + EntryIndexVersion: 0, + RuleVersion: catalog.SemanticObservationRuleVersion, + SourceKind: catalog.SemanticInventorySourceFile, + SourceAvailability: catalog.SemanticInventorySourceAvailabilityVerified, + ScannedResources: 7, + } +} + +func (s *pgcat06Service) seedBuild(t *testing.T, seed buildSeed) catalog.SemanticInventoryBuild { + t.Helper() + build := catalog.NewSemanticInventoryBuild(seed.Project, seed.Generation, seed.AuthResourcePath) + build.State = seed.State + build.ObservationSchema = seed.ObservationSchema + build.InventorySchema = seed.InventorySchema + build.EntryIndexVersion = seed.EntryIndexVersion + build.RuleVersion = seed.RuleVersion + build.SourceKind = seed.SourceKind + build.SourceAvailability = seed.SourceAvailability + build.ScannedResources = seed.ScannedResources + build.LeaseToken = seed.LeaseToken + build.LeaseExpiresAt = seed.LeaseExpiresAt + diagnostics := seed.Diagnostics + if diagnostics == nil { + diagnostics = []string{} + } + s.exec(t, "INSERT INTO "+s.buildFQN()+ + " (build_key, project, dataset_generation, build_id, state, observation_schema,"+ + " inventory_schema, entry_index_version, rule_version, source_kind, source_availability,"+ + " auth_resource_path, scanned_resources, checkpoint, checkpoint_collection, checkpoint_key,"+ + " lease_token, lease_expires_at, diagnostics)"+ + " VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,'','','',$14,$15,$16::text[])", + build.Key, seed.Project, catalog.NormalizeDatasetGeneration(seed.Generation), build.BuildID, + string(seed.State), seed.ObservationSchema, seed.InventorySchema, seed.EntryIndexVersion, + seed.RuleVersion, seed.SourceKind, seed.SourceAvailability, seed.AuthResourcePath, + seed.ScannedResources, seed.LeaseToken, seed.LeaseExpiresAt, diagnostics) + return build +} + +type contributionSeed struct { + Key string + Project string + Generation string + BuildID string + SourceKind string + AuthResourcePath string + ResourceType string + SourceID string + Ordinal int + ConceptID string + BindingID string + Example string + Observation string +} + +func (s *pgcat06Service) seedContribution(t *testing.T, seed contributionSeed) { + t.Helper() + if seed.Key == "" { + seed.Key = pgcat06Hex64(seed.Project + "/" + seed.Generation + "/" + seed.BuildID + "/" + + seed.SourceKind + "/" + seed.SourceID + "/" + fmt.Sprint(seed.Ordinal) + "/" + seed.ConceptID) + } + if strings.TrimSpace(seed.Observation) == "" || seed.Observation[0] != '{' { + t.Fatalf("contribution seed for %s requires an observation object", seed.ConceptID) + } + s.exec(t, "INSERT INTO "+s.contributionFQN()+ + " (contribution_key, project, dataset_generation, build_id, source_kind, auth_resource_path,"+ + " resource_type, source_id, ordinal, concept_id, binding_id, example, payload_digest, observation)"+ + " VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14::jsonb)", + seed.Key, seed.Project, catalog.NormalizeDatasetGeneration(seed.Generation), seed.BuildID, + seed.SourceKind, seed.AuthResourcePath, seed.ResourceType, seed.SourceID, seed.Ordinal, + seed.ConceptID, seed.BindingID, seed.Example, pgcat06Hex64("digest/"+seed.Key), seed.Observation) +} + +func pageWitness(t *testing.T, page catalog.SemanticInventoryPage) string { + t.Helper() + raw, err := json.Marshal(page) + if err != nil { + t.Fatalf("canonicalize page witness: %v", err) + } + return string(raw) +} + +func resolveWitness(t *testing.T, result catalog.SemanticInventoryResolveResult) string { + t.Helper() + raw, err := json.Marshal(result) + if err != nil { + t.Fatalf("canonicalize resolve witness: %v", err) + } + return string(raw) +} + +type pgcat06CursorPayload struct { + Version int `json:"v"` + RequestDigest string `json:"r"` + BindingID string `json:"b"` + ConceptID string `json:"c"` +} + +func pgcat06DecodeCursor(t *testing.T, cursor string) pgcat06CursorPayload { + t.Helper() + raw, err := base64.RawURLEncoding.DecodeString(cursor) + if err != nil { + t.Fatalf("decode cursor base64: %v", err) + } + var payload pgcat06CursorPayload + if err := json.Unmarshal(raw, &payload); err != nil { + t.Fatalf("decode cursor payload: %v", err) + } + return payload +} + +func pgcat06ObsJSON(t *testing.T, observation catalog.SemanticObservation) string { + t.Helper() + raw, err := json.Marshal(observation) + if err != nil { + t.Fatalf("encode fixture observation: %v", err) + } + return string(raw) +} + +func pgcat06Observation(population int64) catalog.SemanticObservation { + return catalog.SemanticObservation{ + SchemaVersion: catalog.SemanticObservationSchemaVersion, + Source: catalog.SemanticObservationSource{ + Canonical: "Observation.code.coding[]", + Type: "Observation", + Profile: "http://profiles.test/Observation", + Path: "code.coding[]", + }, + Key: catalog.SemanticObservationKey{ + Selector: "code.coding[]", + System: "urn:sys-a", + }, + Value: catalog.SemanticObservationValue{ + Selector: "value.value", + Type: "string", + }, + Completeness: catalog.SemanticComplete, + Status: "SUPPORTED", + Population: population, + } +} + +var errPgcat06DeadSource = errors.New("pgcat06 offline test: database must not be touched") + +type pgcat06DeadSource struct{ hits *int } + +func (d *pgcat06DeadSource) Acquire(context.Context) (*pgxpool.Conn, error) { + if d.hits != nil { + *d.hits++ + } + return nil, errPgcat06DeadSource +} + +func pgcat06OfflineRepo(t *testing.T) (*SemanticReadRepository, *int) { + t.Helper() + hits := 0 + repo, err := NewSemanticReadRepository(&pgcat06DeadSource{hits: &hits}, "verify_cat06_offline") + if err != nil { + t.Fatalf("new offline semantic read repository: %v", err) + } + return repo, &hits +} + +func TestSemanticInventoryPageRequestRefusalsBeforeAnyStatement(t *testing.T) { + repo, hits := pgcat06OfflineRepo(t) + base := func() catalog.SemanticInventoryPageOptions { + return catalog.SemanticInventoryPageOptions{ + Project: "cat06-proj", DatasetGeneration: "gen-1", + AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(true), + } + } + cases := []struct { + name string + mutate func(*catalog.SemanticInventoryPageOptions) + }{ + {"blank project", func(o *catalog.SemanticInventoryPageOptions) { o.Project = " " }}, + {"empty project", func(o *catalog.SemanticInventoryPageOptions) { o.Project = "" }}, + {"blank generation", func(o *catalog.SemanticInventoryPageOptions) { o.DatasetGeneration = " \t " }}, + {"empty generation", func(o *catalog.SemanticInventoryPageOptions) { o.DatasetGeneration = "" }}, + {"unknown selection mode", func(o *catalog.SemanticInventoryPageOptions) { o.SelectionMode = "SOMETHING_ELSE" }}, + } + for _, tc := range cases { + opts := base() + tc.mutate(&opts) + _, err := repo.PageSemanticInventory(context.Background(), opts) + if !errors.Is(err, ErrInvalidSemanticInventoryPage) { + t.Errorf("%s: PageSemanticInventory error = %v, want ErrInvalidSemanticInventoryPage", tc.name, err) + } + } + for _, mode := range []catalog.SemanticInventorySelectionMode{"", catalog.SemanticInventoryExactConcept, catalog.SemanticInventorySystemVersionCategory} { + opts := base() + opts.SelectionMode = mode + if _, err := repo.PageSemanticInventory(context.Background(), opts); !errors.Is(err, errPgcat06DeadSource) { + t.Errorf("mode %q: error = %v, want the dead-source sentinel (validation accepted, storage reached)", mode, err) + } + } + if *hits == 0 { + t.Fatalf("accepted shapes never reached the connection source") + } +} + +func TestSemanticInventoryPageConstructorRefusals(t *testing.T) { + if _, err := NewSemanticReadRepository(nil, "verify_cat06_ok"); err == nil { + t.Fatal("nil connection source accepted") + } + if _, err := NewSemanticReadRepository(&pgcat06DeadSource{}, "Bad-Schema"); !errors.Is(err, storepostgres.ErrInvalidSchemaName) { + t.Fatalf("invalid schema error = %v, want ErrInvalidSchemaName", err) + } + if _, err := NewSemanticReadRepository(&pgcat06DeadSource{}, strings.Repeat("s", 64)); !errors.Is(err, storepostgres.ErrInvalidSchemaName) { + t.Fatalf("over-long schema error = %v, want ErrInvalidSchemaName", err) + } +} + +func TestSemanticInventoryResolveRequestRefusalsBeforeAnyStatement(t *testing.T) { + repo, hits := pgcat06OfflineRepo(t) + family := catalog.SemanticInventoryFamilyReference{ + BindingID: "bnd-a", SourceKind: catalog.SemanticInventorySourceFile, ResourceType: "Observation", + SourcePath: "code.coding[]", System: "urn:sys-a", Version: "v1", + } + family.FamilyID = catalog.SemanticInventoryFamilyID(family) + base := func() catalog.SemanticInventoryResolveOptions { + return catalog.SemanticInventoryResolveOptions{ + Project: "cat06-proj", DatasetGeneration: "gen-1", + AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(false), + AuthResourcePaths: []string{"fhir/Observation?type=x"}, + References: []catalog.SemanticInventoryReference{{ConceptID: "cpt-a", BindingID: "bnd-a"}}, + } + } + cases := []struct { + name string + mutate func(*catalog.SemanticInventoryResolveOptions) + }{ + {"nil authorization decision", func(o *catalog.SemanticInventoryResolveOptions) { o.AuthResourcePathsUnrestricted = nil }}, + {"blank project", func(o *catalog.SemanticInventoryResolveOptions) { o.Project = " " }}, + {"blank generation", func(o *catalog.SemanticInventoryResolveOptions) { o.DatasetGeneration = " " }}, + {"no references", func(o *catalog.SemanticInventoryResolveOptions) { + o.References = nil + o.FamilyReferences = nil + }}, + {"too many references", func(o *catalog.SemanticInventoryResolveOptions) { + refs := make([]catalog.SemanticInventoryReference, 0, maxSemanticInventoryResolveSelections+1) + for i := 0; i <= maxSemanticInventoryResolveSelections; i++ { + refs = append(refs, catalog.SemanticInventoryReference{ + ConceptID: fmt.Sprintf("cpt-%03d", i), BindingID: "bnd-a"}) + } + o.References = refs + }}, + {"blank concept", func(o *catalog.SemanticInventoryResolveOptions) { + o.References = []catalog.SemanticInventoryReference{{ConceptID: " ", BindingID: "bnd-a"}} + }}, + {"blank binding", func(o *catalog.SemanticInventoryResolveOptions) { + o.References = []catalog.SemanticInventoryReference{{ConceptID: "cpt-a", BindingID: " "}} + }}, + {"duplicate reference", func(o *catalog.SemanticInventoryResolveOptions) { + o.References = []catalog.SemanticInventoryReference{ + {ConceptID: "cpt-a", BindingID: "bnd-a"}, + {ConceptID: "cpt-a", BindingID: "bnd-a"}, + } + }}, + {"family id does not re-derive", func(o *catalog.SemanticInventoryResolveOptions) { + o.References = nil + bad := family + bad.FamilyID = strings.Repeat("0", 64) + o.FamilyReferences = []catalog.SemanticInventoryFamilyReference{bad} + }}, + {"family blank system", func(o *catalog.SemanticInventoryResolveOptions) { + o.References = nil + bad := family + bad.System = " " + bad.FamilyID = catalog.SemanticInventoryFamilyID(bad) + o.FamilyReferences = []catalog.SemanticInventoryFamilyReference{bad} + }}, + {"family blank resource type", func(o *catalog.SemanticInventoryResolveOptions) { + o.References = nil + bad := family + bad.ResourceType = "" + bad.FamilyID = catalog.SemanticInventoryFamilyID(bad) + o.FamilyReferences = []catalog.SemanticInventoryFamilyReference{bad} + }}, + {"family blank source path", func(o *catalog.SemanticInventoryResolveOptions) { + o.References = nil + bad := family + bad.SourcePath = " " + bad.FamilyID = catalog.SemanticInventoryFamilyID(bad) + o.FamilyReferences = []catalog.SemanticInventoryFamilyReference{bad} + }}, + {"family unknown source kind", func(o *catalog.SemanticInventoryResolveOptions) { + o.References = nil + bad := family + bad.SourceKind = "stream_of_consciousness" + bad.FamilyID = catalog.SemanticInventoryFamilyID(bad) + o.FamilyReferences = []catalog.SemanticInventoryFamilyReference{bad} + }}, + {"duplicate family reference", func(o *catalog.SemanticInventoryResolveOptions) { + o.References = nil + o.FamilyReferences = []catalog.SemanticInventoryFamilyReference{family, family} + }}, + } + for _, tc := range cases { + opts := base() + tc.mutate(&opts) + _, err := repo.ResolveSemanticInventorySelections(context.Background(), opts) + if !errors.Is(err, ErrInvalidSemanticInventorySelection) { + t.Errorf("%s: ResolveSemanticInventorySelections error = %v, want ErrInvalidSemanticInventorySelection", tc.name, err) + } + } + if _, err := repo.ResolveSemanticInventorySelections(context.Background(), base()); !errors.Is(err, errPgcat06DeadSource) { + t.Fatalf("accepted resolve error = %v, want the dead-source sentinel", err) + } + if *hits == 0 { + t.Fatalf("accepted shapes never reached the connection source") + } +} + +func TestSemanticInventoryCursorMismatchUnderScopeChanges(t *testing.T) { + const buildID = "0000000000000000000000000000000000000000000000000000000000000001" + unrestricted := true + base := catalog.SemanticInventoryPageOptions{ + Project: "cat06-proj", DatasetGeneration: "gen-1", + AuthResourcePathsUnrestricted: &unrestricted, + ResourceType: "Observation", + BindingID: "bnd-a", + Query: "abc", + Limit: 10, + } + cursor := catalog.EncodeSemanticInventoryCursor(base, buildID, "bnd-a", "cpt-z") + if binding, concept, err := catalog.DecodeSemanticInventoryCursor(cursor, base, buildID); err != nil || binding != "bnd-a" || concept != "cpt-z" { + t.Fatalf("unchanged cursor round-trip = (%q, %q, %v)", binding, concept, err) + } + changes := []struct { + name string + mutate func(*catalog.SemanticInventoryPageOptions) + }{ + {"project", func(o *catalog.SemanticInventoryPageOptions) { o.Project = "cat06-other" }}, + {"generation", func(o *catalog.SemanticInventoryPageOptions) { o.DatasetGeneration = "gen-2" }}, + {"resource type", func(o *catalog.SemanticInventoryPageOptions) { o.ResourceType = "Encounter" }}, + {"binding filter", func(o *catalog.SemanticInventoryPageOptions) { o.BindingID = "bnd-b" }}, + {"query", func(o *catalog.SemanticInventoryPageOptions) { o.Query = "abd" }}, + {"limit", func(o *catalog.SemanticInventoryPageOptions) { o.Limit = 11 }}, + {"auth paths added", func(o *catalog.SemanticInventoryPageOptions) { + o.AuthResourcePaths = []string{"fhir/Observation?type=x"} + }}, + {"authorization decision", func(o *catalog.SemanticInventoryPageOptions) { + value := false + o.AuthResourcePathsUnrestricted = &value + }}, + {"selection mode", func(o *catalog.SemanticInventoryPageOptions) { + o.SelectionMode = catalog.SemanticInventorySystemVersionCategory + }}, + } + for _, tc := range changes { + opts := base + tc.mutate(&opts) + if _, _, err := catalog.DecodeSemanticInventoryCursor(cursor, opts, buildID); !errors.Is(err, catalog.ErrSemanticInventoryCursorMismatch) { + t.Errorf("%s: changed-scope cursor error = %v, want ErrSemanticInventoryCursorMismatch", tc.name, err) + } + } + padded := base + padded.DatasetGeneration = " gen-1\t" + if _, _, err := catalog.DecodeSemanticInventoryCursor(cursor, padded, buildID); err != nil { + t.Fatalf("padded generation must normalize onto the same cursor: %v", err) + } + for _, bad := range []string{"!!!not base64!!!", base64.RawURLEncoding.EncodeToString([]byte(`{"v":2,"r":"x","b":"y","c":"z"}`)), base64.RawURLEncoding.EncodeToString([]byte(`{"v":1,"r":"x","b":"","c":"z"}`)), base64.RawURLEncoding.EncodeToString([]byte(`{"v":1,"r":"x","b":"y","c":""}`))} { + if _, _, err := catalog.DecodeSemanticInventoryCursor(bad, base, buildID); !errors.Is(err, catalog.ErrSemanticInventoryCursorMismatch) { + t.Errorf("tampered cursor %q error = %v, want typed mismatch", bad, err) + } + } +} + +func TestSemanticInventoryPageSQLCarriesTheParityContract(t *testing.T) { + scope := semanticPageParams{ + project: "cat06-proj", generation: "gen-1", + buildID: "0000000000000000000000000000000000000000000000000000000000000001", + sourceKind: catalog.SemanticInventorySourceFile, + unrestricted: false, paths: []string{"fhir/Observation?type=x"}, + resourceType: "Observation", bindingID: "bnd-a", query: "abc", + afterBindingID: "bnd-a", afterSelection: "cpt-a", limit: 50, + } + exactSQL, exactArgs := semanticPageQuery(`"s"."fhir_semantic_inventory_contribution"`, scope) + familyScope := scope + familyScope.familyMode = true + familySQL, familyArgs := semanticPageQuery(`"s"."fhir_semantic_inventory_contribution"`, familyScope) + + for name, pair := range map[string]struct { + sql string + args []any + }{ + "exact": {exactSQL, exactArgs}, + "family": {familySQL, familyArgs}, + } { + sql, args := pair.sql, pair.args + if len(args) != 12 { + t.Fatalf("%s: statement binds %d parameters, want 12 (scope $1..$6, filters $7..$9, cursor $10..$11, budget $12)", name, len(args)) + } + if args[11] != 51 { + t.Errorf("%s: budget parameter = %v, want limit+1 = 51", name, args[11]) + } + wants := []string{ + `c.project = $1 AND c.dataset_generation = $2 AND c.build_id = $3 AND c.source_kind = $4`, + `($5 OR c.auth_resource_path = ANY($6::text[]))`, + `($7 = '' OR c.resource_type = $7)`, + `candidates.binding_id COLLATE "C" > $10 COLLATE "C"`, + `candidates.selection_id COLLATE "C" > $11 COLLATE "C"`, + `ORDER BY i.binding_id COLLATE "C", i.selection_id COLLATE "C", i.is_category`, + `LIMIT $12`, + `strpos(lower(concat_ws(' ', c.resource_type,`, + `COLLATE "C"), lower($9 COLLATE "C")) > 0`, + `min(NULLIF(c.example, '') COLLATE "C") AS agg_example`, + `~ '^-?[0-9]+$'`, + } + for _, want := range wants { + if !strings.Contains(sql, want) { + t.Errorf("%s statement missing load-bearing fragment %q", name, want) + } + } + if got := strings.Count(sql, `($7 = '' OR c.resource_type = $7)`); got != 2 { + t.Errorf("%s statement applies the resource-type filter %d times, want 2 (identity + aggregation stages)", name, got) + } + if !strings.Contains(sql, `concat_ws(' ', c.resource_type, (c.observation #>> '{source,path}')`) { + t.Errorf("%s statement must feed raw leaves into concat_ws: %s", name, sql) + } + if strings.Contains(sql, `concat_ws(' ', c.resource_type, COALESCE(`) { + t.Errorf("%s statement keeps empty search slots where the reference skips them", name) + } + } + if strings.Contains(exactSQL, "CODED_CONCEPT_V1") || strings.Contains(exactSQL, "family:[") { + t.Errorf("exact page must not compute family identities at all: %s", exactSQL) + } + if !strings.Contains(exactSQL, `false AS is_category, c.concept_id AS selection_id`) { + t.Errorf("exact page must derive the shared ordering key from concept_id: %s", exactSQL) + } + for _, want := range []string{ + `'family:['`, + `'CODED_CONCEPT_V1'`, + } { + if !strings.Contains(familySQL, want) { + t.Errorf("family statement missing %q", want) + } + } + if strings.Contains(familySQL, "jsonb_build_array") { + t.Errorf("family selection must not use jsonb_build_array (its text form pads commas with spaces)") + } +} + +func TestSemanticInventoryResolveSQLCarriesTheParityContract(t *testing.T) { + scope := resolveScope{ + project: "cat06-proj", generation: "gen-1", + buildID: "0000000000000000000000000000000000000000000000000000000000000001", + sourceKind: catalog.SemanticInventorySourceRetained, unrestricted: false, + paths: []string{"fhir/Observation?type=x"}, observationSchema: catalog.SemanticObservationSchemaVersion, + } + exactSQL, exactArgs := semanticResolveReferencesQuery(`"s"."t"`, scope, []catalog.SemanticInventoryReference{ + {ConceptID: "cpt-a", BindingID: "bnd-a"}, {ConceptID: "cpt-b", BindingID: "bnd-b"}, + }) + if len(exactArgs) != 6+4 { + t.Fatalf("exact resolve bound %d args, want 10", len(exactArgs)) + } + if exactArgs[6] != "bnd-a" || exactArgs[7] != "cpt-a" || exactArgs[8] != "bnd-b" || exactArgs[9] != "cpt-b" { + t.Fatalf("exact resolve args out of request order: %v", exactArgs) + } + for _, want := range []string{ + `(VALUES ($7::text, $8::text, 0), ($9::text, $10::text, 1)) AS requested(binding_id, concept_id, ord)`, + `ORDER BY c.contribution_key COLLATE "C" LIMIT 1) c ORDER BY requested.ord`, + `c.project = $1`, + } { + if !strings.Contains(exactSQL, want) { + t.Errorf("exact resolve missing %q", want) + } + } + + family := catalog.SemanticInventoryFamilyReference{ + BindingID: "bnd-a", SourceKind: catalog.SemanticInventorySourceRetained, ResourceType: "Observation", + SourcePath: "code.coding[]", System: "urn:sys-a", Version: "v1", + } + family.FamilyID = catalog.SemanticInventoryFamilyID(family) + familySQL, familyArgs := semanticResolveFamiliesQuery(`"s"."t"`, scope, []catalog.SemanticInventoryFamilyReference{family}) + if len(familyArgs) != 6+6 { + t.Fatalf("family resolve bound %d args, want 12", len(familyArgs)) + } + for _, want := range []string{ + `LEFT JOIN "s"."t" c ON c.project = $1`, + `CASE WHEN (c.observation #> '{key,version}') IS NULL THEN '' WHEN (c.observation #> '{key,version}') = 'null'::jsonb THEN NULL ELSE (c.observation #>> '{key,version}') END) = requested.version`, + `c.observation #>> '{rule_hint}' = 'CODED_CONCEPT_V1'`, + `GROUP BY requested.ord HAVING COUNT(c.contribution_key) > 0`, + `ORDER BY requested.ord`, + `WHERE requested.source_kind = $4`, + } { + if !strings.Contains(familySQL, want) { + t.Errorf("family resolve missing %q", want) + } + } +} + +const ( + pgcat06Project = "cat06-proj" + pgcat06Generation = "gen-1" + pgcat06PathGlucose = "fhir/Observation?type=glucose" + pgcat06PathVisit = "fhir/Encounter?type=visit" + pgcat06PathSecret = "fhir/Secret?type=x" + pgcat06PathNone = "fhir/Nothing?type=none" +) + +func pgcat06Unrestricted(v bool) *bool { return catalog.ExplicitAuthResourcePathsUnrestricted(v) } + +func TestSemanticInventoryEmitterWriterReaderGoldenAnchor(t *testing.T) { + s := newPgcat06Service(t) + anchorSeed := pgcat06DefaultBuildSeed(pgcat06Project, pgcat06Generation) + anchorSeed.SourceKind = catalog.SemanticInventorySourceRetained + s.seedBuild(t, anchorSeed) + contributions := pgcat05EmitContribution(t, pgcat06Project, pgcat06Generation, + pgcat06PathGlucose, "Observation", "retained:Observation/cat06-anchor", + `{"resourceType":"Observation","code":{"coding":[{"system":"urn:obs-05","version":"v1","code":"loinc-42"}]}}`) + if len(contributions) != 1 { + t.Fatalf("emitter produced %d contributions, want exactly 1", len(contributions)) + } + if contributions[0].ConceptID != "ec572479ea77eabd0c8014f33810f458e9f009a27adb8fd77fb9fab3121ad62f" || + contributions[0].BindingID != "b498c06c58e286f4cc9ceb74942c8a7872ad49b0220e65f0ea754fd936f4a53d" { + t.Fatalf("emitter concept/binding drifted from the pinned anchor: %s/%s", + contributions[0].ConceptID, contributions[0].BindingID) + } + if err := s.contributionRepository(t).WriteSemanticInventoryContributions(s.ctx, contributions, 0); err != nil { + s.fail("write emitter contribution", err) + } + + repo := s.readRepository(t) + opts := catalog.SemanticInventoryPageOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(true), + } + page, err := repo.PageSemanticInventory(s.ctx, opts) + if err != nil { + s.fail("page semantic inventory", err) + } + if len(page.Entries) != 1 || page.NextCursor != "" { + t.Fatalf("anchor page = %+v, want exactly one entry and no cursor", page) + } + want := catalog.SemanticInventoryEntry{ + ConceptID: "ec572479ea77eabd0c8014f33810f458e9f009a27adb8fd77fb9fab3121ad62f", + BindingID: "b498c06c58e286f4cc9ceb74942c8a7872ad49b0220e65f0ea754fd936f4a53d", + Observation: catalog.SemanticObservation{ + SchemaVersion: catalog.SemanticObservationSchemaVersion, + Source: catalog.SemanticObservationSource{ + Canonical: "Observation.code.coding[]", + Type: "Observation", + Path: "code.coding[]", + }, + Key: catalog.SemanticObservationKey{ + Selector: "code.coding[]", System: "urn:obs-05", Version: "v1", Code: "loinc-42", + }, + Value: catalog.SemanticObservationValue{ + Selector: "code.coding[].display", Type: "string", + FallbackSelectors: []string{"code.coding[].code"}, + FallbackOnBlankString: true, + }, + LogicalType: "string", + ObservedUnits: []string{}, + Completeness: catalog.SemanticComplete, + Status: "SUPPORTED", + Population: 1, + Examples: []string{"loinc-42"}, + ExamplesTruncated: false, + RuleHint: "CODED_CONCEPT_V1", + RuleVersion: "5", + }, + } + if !reflect.DeepEqual(page.Entries[0], want) { + t.Fatalf("anchor page entry =\n%#v\nwant\n%#v", page.Entries[0], want) + } + perturbed := want + perturbed.Observation.Population = 2 + if reflect.DeepEqual(page.Entries[0], perturbed) { + t.Fatal("the golden comparison accepted a perturbed population") + } + perturbed2 := want + perturbed2.Observation.Examples = []string{"other"} + if reflect.DeepEqual(page.Entries[0], perturbed2) { + t.Fatal("the golden comparison accepted perturbed examples") + } +} + +func TestSemanticInventoryPageExactGolden(t *testing.T) { + s := newPgcat06Service(t) + build := s.seedBuild(t, func() buildSeed { + seed := pgcat06DefaultBuildSeed(pgcat06Project, pgcat06Generation) + seed.LeaseToken = "lease-live" + seed.LeaseExpiresAt = 1_700_000_000_000 + seed.AuthResourcePath = "fhir/secret/internal" + seed.Diagnostics = []string{"stale-diagnostic"} + return seed + }()) + _ = build + buildID := catalog.SemanticInventoryBuildID(pgcat06Project, pgcat06Generation) + put := func(sourceID string, ordinal int, binding, concept, path, example string, observation catalog.SemanticObservation) { + s.seedContribution(t, contributionSeed{ + Project: pgcat06Project, Generation: pgcat06Generation, BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: path, + ResourceType: "Observation", SourceID: sourceID, Ordinal: ordinal, + ConceptID: concept, BindingID: binding, Example: example, + Observation: pgcat06ObsJSON(t, observation), + }) + } + + alpha := func(pop int64) catalog.SemanticObservation { + o := pgcat06Observation(pop) + o.Key = catalog.SemanticObservationKey{Selector: "code.coding[]", System: "urn:sys-a", Version: "v1", Code: "code-a", Display: "Alpha Display"} + return o + } + beta := func() catalog.SemanticObservation { + o := pgcat06Observation(1) + o.Key = catalog.SemanticObservationKey{Selector: "code.coding[]", System: "urn:sys-b", Version: "v1", Code: "code-b", Display: "Beta"} + return o + } + gammaUnits := func(units []string) catalog.SemanticObservation { + o := pgcat06Observation(1) + o.Key = catalog.SemanticObservationKey{Selector: "code.coding[]", System: "urn:sys-c", Code: "code-c"} + o.ObservedUnits = units + return o + } + + put("src-1", 0, "bnd-alpha", "cpt-alpha", pgcat06PathGlucose, "ex-alpha-1", alpha(3)) + put("src-2", 0, "bnd-alpha", "cpt-alpha", pgcat06PathGlucose, "", alpha(2)) + put("src-3", 0, "bnd-alpha", "cpt-beta", pgcat06PathVisit, "beta-ex", beta()) + put("src-4", 0, "bnd-beta", "cpt-gamma", pgcat06PathGlucose, "g1", gammaUnits([]string{"kg"})) + put("src-4", 1, "bnd-beta", "cpt-gamma", pgcat06PathGlucose, "g2", gammaUnits([]string{"g"})) + + foreign := contributionSeed{ + Project: "cat06-other", Generation: pgcat06Generation, BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: pgcat06PathGlucose, + ResourceType: "Observation", SourceID: "src-f1", Ordinal: 0, + ConceptID: "cpt-alpha", BindingID: "bnd-alpha", Example: "foreign", + Observation: pgcat06ObsJSON(t, alpha(99)), + } + s.seedContribution(t, foreign) + s.seedContribution(t, contributionSeed{ + Project: pgcat06Project, Generation: "gen-2", BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: pgcat06PathGlucose, + ResourceType: "Observation", SourceID: "src-f2", Ordinal: 0, + ConceptID: "cpt-alpha", BindingID: "bnd-alpha", Example: "foreign", + Observation: pgcat06ObsJSON(t, alpha(99)), + }) + s.seedContribution(t, contributionSeed{ + Project: pgcat06Project, Generation: pgcat06Generation, BuildID: pgcat06Hex64("foreign-build"), + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: pgcat06PathGlucose, + ResourceType: "Observation", SourceID: "src-f3", Ordinal: 0, + ConceptID: "cpt-alpha", BindingID: "bnd-alpha", Example: "foreign", + Observation: pgcat06ObsJSON(t, alpha(99)), + }) + s.seedContribution(t, contributionSeed{ + Project: pgcat06Project, Generation: pgcat06Generation, BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceRetained, AuthResourcePath: pgcat06PathGlucose, + ResourceType: "Observation", SourceID: "src-f4", Ordinal: 0, + ConceptID: "cpt-alpha", BindingID: "bnd-alpha", Example: "foreign", + Observation: pgcat06ObsJSON(t, alpha(99)), + }) + s.seedContribution(t, contributionSeed{ + Project: pgcat06Project, Generation: pgcat06Generation, BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: pgcat06PathSecret, + ResourceType: "Observation", SourceID: "src-f5", Ordinal: 0, + ConceptID: "cpt-alpha", BindingID: "bnd-alpha", Example: "secret-ex", + Observation: pgcat06ObsJSON(t, alpha(4)), + }) + + repo := s.readRepository(t) + baseOpts := func() catalog.SemanticInventoryPageOptions { + return catalog.SemanticInventoryPageOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + } + } + + opts := baseOpts() + opts.AuthResourcePathsUnrestricted = pgcat06Unrestricted(true) + page, err := repo.PageSemanticInventory(s.ctx, opts) + if err != nil { + s.fail("unrestricted page", err) + } + if len(page.Entries) != 3 { + t.Fatalf("unrestricted page returned %d entries: %+v", len(page.Entries), page.Entries) + } + wantPopulation := int64(3 + 2 + 4) + got := page.Entries[0] + if got.ConceptID != "cpt-alpha" || got.BindingID != "bnd-alpha" { + t.Fatalf("first identity = %s/%s, want bnd-alpha/cpt-alpha", got.BindingID, got.ConceptID) + } + if got.Observation.Population != wantPopulation || !reflect.DeepEqual(got.Observation.Examples, []string{"ex-alpha-1"}) || + !got.Observation.ExamplesTruncated { + t.Fatalf("cpt-alpha aggregate = population %d examples %#v truncated %v, want population %d [ex-alpha-1] true", + got.Observation.Population, got.Observation.Examples, got.Observation.ExamplesTruncated, + wantPopulation) + } + if page.Entries[1].ConceptID != "cpt-beta" || page.Entries[2].ConceptID != "cpt-gamma" { + t.Fatalf("identity order = %s, %s; want cpt-beta then cpt-gamma", page.Entries[1].ConceptID, page.Entries[2].ConceptID) + } + if page.Entries[1].Observation.Population != 1 || page.Entries[1].Observation.ExamplesTruncated || + !reflect.DeepEqual(page.Entries[1].Observation.Examples, []string{"beta-ex"}) { + t.Fatalf("cpt-beta aggregate = %+v, want population 1, no truncation, examples [beta-ex]", page.Entries[1].Observation) + } + gamma := page.Entries[2].Observation + if !reflect.DeepEqual(gamma.ObservedUnits, []string{"g"}) || !gamma.ObservedUnitsTruncated || + gamma.Population != 2 || !gamma.ExamplesTruncated || !reflect.DeepEqual(gamma.Examples, []string{"g1"}) { + t.Fatalf("cpt-gamma aggregate = %+v, want units [g] truncated, population 2, examples [g1] truncated", gamma) + } + if page.NextCursor != "" { + t.Fatalf("short page must not carry a cursor, got %q", page.NextCursor) + } + if page.Build.LeaseToken != "lease-live" || page.Build.LeaseExpiresAt != 1_700_000_000_000 || + page.Build.ScannedResources != 7 || page.Build.AuthResourcePath != "fhir/secret/internal" || + !reflect.DeepEqual(page.Build.Diagnostics, []string{"stale-diagnostic"}) { + t.Fatalf("unrestricted build internals = %+v", page.Build) + } + + opts = baseOpts() + opts.AuthResourcePathsUnrestricted = pgcat06Unrestricted(false) + opts.AuthResourcePaths = []string{pgcat06PathGlucose} + page, err = repo.PageSemanticInventory(s.ctx, opts) + if err != nil { + s.fail("restricted page", err) + } + if len(page.Entries) != 2 || page.Entries[0].ConceptID != "cpt-alpha" || page.Entries[1].ConceptID != "cpt-gamma" { + t.Fatalf("restricted page identities = %+v", page.Entries) + } + if page.Entries[0].Observation.Population != 5 { + t.Fatalf("restricted cpt-alpha population = %d, want 5 (secret path excluded)", page.Entries[0].Observation.Population) + } + if page.Build.ScannedResources != 0 || page.Build.Checkpoint != "" || page.Build.SourceCheckpoint != (catalog.SemanticInventoryCheckpoint{}) || + page.Build.LeaseToken != "" || page.Build.LeaseExpiresAt != 0 || page.Build.Diagnostics != nil || page.Build.AuthResourcePath != "" { + t.Fatalf("restricted build internals were not scrubbed: %+v", page.Build) + } + if page.Build.State != catalog.SemanticInventoryComplete || page.Build.BuildID == "" { + t.Fatalf("restricted page lost the build identity/state: %+v", page.Build) + } + + opts = baseOpts() + opts.AuthResourcePathsUnrestricted = pgcat06Unrestricted(false) + opts.AuthResourcePaths = []string{pgcat06PathNone} + page, err = repo.PageSemanticInventory(s.ctx, opts) + if err != nil { + s.fail("restricted-empty page", err) + } + if page.Entries == nil || len(page.Entries) != 0 || page.Families != nil || page.State != catalog.SemanticInventoryComplete || + page.NextCursor != "" || page.Build.BuildID == "" { + t.Fatalf("restricted-empty page = %+v, want explicit non-nil empty page with the completed build", page) + } + + put("src-5", 0, "bnd-beta", "cpt-gamma", pgcat06PathGlucose, "g3", gammaUnits([]string{"g"})) + opts = baseOpts() + opts.AuthResourcePathsUnrestricted = pgcat06Unrestricted(true) + page, err = repo.PageSemanticInventory(s.ctx, opts) + if err != nil { + s.fail("page after mutation probe", err) + } + if page.Entries[2].Observation.Population != 3 || page.Entries[0].Observation.Population != 9 { + t.Fatalf("mutation probe changed the wrong aggregates: %+v", page.Entries) + } +} + +const ( + pgcat06EscapedPath = "a\"b\\c\t\u001f" + pgcat06EscapedSystem = "urn:fam\"x" + pgcat06EscapedSelection = `family:["Observation","a\"b\\c\t\u001f","urn:fam\"x","v1"]` +) + +func pgcat06FamilyObs(path, system, version string, population int64) catalog.SemanticObservation { + o := pgcat06Observation(population) + o.Source.Type = "Observation" + o.Source.Path = path + o.Key.Selector = "code.coding[]" + o.Key.System = system + o.Key.Version = version + o.RuleHint = "CODED_CONCEPT_V1" + o.RuleVersion = "5" + return o +} + +func pgcat06PlainObs(population int64) catalog.SemanticObservation { + o := pgcat06Observation(population) + o.Source.Type = "Observation" + o.Source.Path = "value.coding[]" + o.RuleHint = "CODED_VALUE_V1" + o.RuleVersion = "5" + return o +} + +func TestSemanticInventoryPageFamilyGolden(t *testing.T) { + s := newPgcat06Service(t) + s.seedBuild(t, pgcat06DefaultBuildSeed(pgcat06Project, pgcat06Generation)) + buildID := catalog.SemanticInventoryBuildID(pgcat06Project, pgcat06Generation) + put := func(sourceID string, ordinal int, concept, path, system, version, example string, observation catalog.SemanticObservation) { + s.seedContribution(t, contributionSeed{ + Project: pgcat06Project, Generation: pgcat06Generation, BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: pgcat06PathGlucose, + ResourceType: "Observation", SourceID: sourceID, Ordinal: ordinal, + ConceptID: concept, BindingID: "bnd-family", Example: example, + Observation: pgcat06ObsJSON(t, observation), + }) + } + + put("f-1", 0, "cpt-fam-v1-a", "code.coding[]", "urn:family-a", "v1", "f-ex-2", pgcat06FamilyObs("code.coding[]", "urn:family-a", "v1", 2)) + put("f-1", 1, "cpt-fam-v1-b", "code.coding[]", "urn:family-a", "v1", "f-ex-1", pgcat06FamilyObs("code.coding[]", "urn:family-a", "v1", 3)) + put("f-2", 0, "cpt-fam-nv-a", "code.coding[]", "urn:family-a", "", "", pgcat06FamilyObs("code.coding[]", "urn:family-a", "", 1)) + put("f-2", 1, "cpt-fam-nv-b", "code.coding[]", "urn:family-a", "", "", pgcat06FamilyObs("code.coding[]", "urn:family-a", "", 1)) + put("f-3", 0, "cpt-fam-esc", pgcat06EscapedPath, pgcat06EscapedSystem, "v1", "esc-ex", pgcat06FamilyObs(pgcat06EscapedPath, pgcat06EscapedSystem, "v1", 1)) + put("p-1", 0, "0-numeric", "value.coding[]", "urn:sys-a", "", "", pgcat06PlainObs(1)) + put("p-2", 0, "cpt-fam-ext", "code.coding[]", "urn:family-a", "v1", "ext-ex", func() catalog.SemanticObservation { + o := pgcat06FamilyObs("code.coding[]", "urn:family-a", "v1", 4) + o.ExtensionURLPath = []string{"http://ext"} + return o + }()) + put("p-3", 0, "cpt-fam-nosys", "code.coding[]", "", "", "", func() catalog.SemanticObservation { + o := pgcat06FamilyObs("code.coding[]", "", "", 1) + return o + }()) + put("p-4", 0, "cpt-plain", "value.coding[]", "urn:sys-a", "v1", "plain-ex", pgcat06PlainObs(1)) + put("p-5", 0, "family:zz-plain", "value.coding[]", "urn:sys-a", "", "", pgcat06PlainObs(1)) + + repo := s.readRepository(t) + opts := catalog.SemanticInventoryPageOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(true), + SelectionMode: catalog.SemanticInventorySystemVersionCategory, + BindingID: "bnd-family", + } + page, err := repo.PageSemanticInventory(s.ctx, opts) + if err != nil { + s.fail("family page", err) + } + if len(page.Entries) != 5 { + t.Fatalf("family page returned %d plain entries: %+v", len(page.Entries), page.Entries) + } + if len(page.Families) != 3 { + t.Fatalf("family page returned %d families: %+v", len(page.Families), page.Families) + } + gotOrder := []string{ + page.Entries[0].ConceptID, page.Entries[1].ConceptID, + page.Entries[2].ConceptID, page.Entries[3].ConceptID, + "F:" + page.Families[0].FamilyID[:8], "F:" + page.Families[1].FamilyID[:8], "F:" + page.Families[2].FamilyID[:8], + page.Entries[4].ConceptID, + } + wantOrder := []string{ + "0-numeric", "cpt-fam-ext", "cpt-fam-nosys", "cpt-plain", + "F:f32acc4b", "F:0a459082", "F:2827c31b", "family:zz-plain", + } + if !reflect.DeepEqual(gotOrder, wantOrder) { + t.Fatalf("interleaved identity order = %v, want %v", gotOrder, wantOrder) + } + + escaped := catalog.SemanticInventoryFamily{ + SemanticInventoryFamilyReference: catalog.SemanticInventoryFamilyReference{ + FamilyID: "f32acc4bd611d5349f64539c4280579b644d414660bb37c6d0a42a239801dd58", + BindingID: "bnd-family", SourceKind: catalog.SemanticInventorySourceFile, + ResourceType: "Observation", SourcePath: pgcat06EscapedPath, System: pgcat06EscapedSystem, Version: "v1", + }, + Source: catalog.SemanticObservationSource{ + Canonical: "Observation.code.coding[]", Type: "Observation", + Profile: "http://profiles.test/Observation", Path: pgcat06EscapedPath, + }, + KeySelector: "code.coding[]", + Value: catalog.SemanticObservationValue{ + Selector: "value.value", Type: "string", + }, + SchemaVersion: catalog.SemanticObservationSchemaVersion, + RuleVersion: "5", + Completeness: catalog.SemanticComplete, + Status: "SUPPORTED", + Population: 1, + Examples: []string{"esc-ex"}, + ExamplesTruncated: false, + } + if !reflect.DeepEqual(page.Families[0], escaped) { + t.Fatalf("escaped family =\n%#v\nwant\n%#v", page.Families[0], escaped) + } + versionless := escaped + versionless.FamilyID = "0a4590824e29ff0d2f81579035f294e561e7c250ecf10ec956a1d31c3d996c00" + versionless.SourcePath = "code.coding[]" + versionless.System = "urn:family-a" + versionless.Version = "" + versionless.Source.Path = "code.coding[]" + versionless.Population = 2 + versionless.Examples = []string{} + versionless.ExamplesTruncated = true + if !reflect.DeepEqual(page.Families[1], versionless) { + t.Fatalf("versionless family =\n%#v\nwant\n%#v", page.Families[1], versionless) + } + versioned := versionless + versioned.FamilyID = "2827c31b6970c91d02bf2fecde13a646dd8f007f1b4ce66eb67bafaed7becbc8" + versioned.Version = "v1" + versioned.Population = 5 + versioned.Examples = []string{"f-ex-1"} + if !reflect.DeepEqual(page.Families[2], versioned) { + t.Fatalf("versioned family =\n%#v\nwant\n%#v", page.Families[2], versioned) + } + if !reflect.DeepEqual(page.Entries[1].Observation.ExtensionURLPath, []string{"http://ext"}) || + page.Entries[1].Observation.Population != 4 { + t.Fatalf("decoy entry = %+v", page.Entries[1]) + } + + var walkedEntries []catalog.SemanticInventoryEntry + var walkedFamilies []catalog.SemanticInventoryFamily + cursor := "" + identitySequence := []string{} + for rounds := 0; ; rounds++ { + if rounds > 10 { + t.Fatalf("identity walk did not terminate: %v", identitySequence) + } + step := opts + step.Limit = 1 + step.Cursor = cursor + small, err := repo.PageSemanticInventory(s.ctx, step) + if err != nil { + s.fail("family page step", err) + } + if len(small.Entries) > 1 || len(small.Families) > 1 || len(small.Entries)+len(small.Families) != 1 { + t.Fatalf("limit-1 page returned %d entries %d families", len(small.Entries), len(small.Families)) + } + walkedEntries = append(walkedEntries, small.Entries...) + walkedFamilies = append(walkedFamilies, small.Families...) + if small.NextCursor == "" { + if len(small.Entries) == 1 { + identitySequence = append(identitySequence, small.Entries[0].ConceptID) + } else { + identitySequence = append(identitySequence, escapedSelectionOf(small.Families[0])) + } + break + } + payload := pgcat06DecodeCursor(t, small.NextCursor) + if payload.Version != 1 || payload.BindingID != "bnd-family" { + t.Fatalf("cursor payload = %+v, want version 1 and binding bnd-family", payload) + } + if len(small.Families) == 1 && payload.ConceptID != small.Families[0].SourcePath { + if payload.ConceptID != escapedSelectionOf(small.Families[0]) { + t.Fatalf("family cursor selection %q does not re-derive from the family row", payload.ConceptID) + } + } + identitySequence = append(identitySequence, payload.ConceptID) + cursor = small.NextCursor + } + if len(identitySequence) != 8 { + t.Fatalf("identity walk produced %d identities: %v", len(identitySequence), identitySequence) + } + if identitySequence[4] != pgcat06EscapedSelection { + t.Fatalf("cursor-carried family selection bytes = %q, want %q", identitySequence[4], pgcat06EscapedSelection) + } + walked := catalog.SemanticInventoryPage{Entries: walkedEntries, Families: walkedFamilies, State: catalog.SemanticInventoryComplete} + full := catalog.SemanticInventoryPage{Entries: page.Entries, Families: page.Families, State: catalog.SemanticInventoryComplete} + if pageWitness(t, walked) != pageWitness(t, full) { + t.Fatalf("paged walk diverged from the unpaged read:\n%s\nvs\n%s", pageWitness(t, walked), pageWitness(t, full)) + } + + exactOpts := opts + exactOpts.SelectionMode = catalog.SemanticInventoryExactConcept + exact, err := repo.PageSemanticInventory(s.ctx, exactOpts) + if err != nil { + s.fail("exact page over family fixtures", err) + } + if len(exact.Entries) != 10 || exact.Families != nil { + t.Fatalf("exact mode over family fixtures returned %d entries, %d families", len(exact.Entries), len(exact.Families)) + } + + origWitness := pageWitness(t, page) + s.exec(t, "UPDATE "+s.contributionFQN()+ + " SET observation = observation - 'extension_url_path' WHERE concept_id = 'cpt-fam-ext'") + merged, err := repo.PageSemanticInventory(s.ctx, opts) + if err != nil { + s.fail("family page after guard drop", err) + } + if len(merged.Families) != 3 || merged.Families[2].Population != 9 || + !reflect.DeepEqual(merged.Families[2].Examples, []string{"ext-ex"}) { + t.Fatalf("decoy did not merge into its twin family: %+v", merged.Families) + } + if len(merged.Entries) != 4 { + t.Fatalf("guard drop left %d plain entries, want 4", len(merged.Entries)) + } + s.exec(t, "UPDATE "+s.contributionFQN()+ + " SET observation = jsonb_set(observation, '{extension_url_path}', '[\"http://ext\"]'::jsonb) WHERE concept_id = 'cpt-fam-ext'") + restored, err := repo.PageSemanticInventory(s.ctx, opts) + if err != nil { + s.fail("family page after guard restore", err) + } + if pageWitness(t, restored) != origWitness { + t.Fatalf("restored page is not byte-identical to the original witness") + } +} + +func escapedSelectionOf(family catalog.SemanticInventoryFamily) string { + return `family:[` + pgcat06JSONString(family.Source.Type) + "," + pgcat06JSONString(family.SourcePath) + "," + + pgcat06JSONString(family.System) + "," + pgcat06JSONString(family.Version) + `]` +} + +func pgcat06JSONString(value string) string { + var sb strings.Builder + sb.WriteByte('"') + for _, r := range value { + switch r { + case '"': + sb.WriteString(`\"`) + case '\\': + sb.WriteString(`\\`) + case '\t': + sb.WriteString(`\t`) + case '\n': + sb.WriteString(`\n`) + case '\r': + sb.WriteString(`\r`) + default: + if r < 0x20 { + fmt.Fprintf(&sb, `\u%04x`, r) + } else { + sb.WriteRune(r) + } + } + } + sb.WriteByte('"') + return sb.String() +} + +func TestSemanticInventoryResolveExactGolden(t *testing.T) { + s := newPgcat06Service(t) + s.seedBuild(t, pgcat06DefaultBuildSeed(pgcat06Project, pgcat06Generation)) + buildID := catalog.SemanticInventoryBuildID(pgcat06Project, pgcat06Generation) + + low, high := pgcat06Hex64("resolve-a"), pgcat06Hex64("resolve-b") + if low > high { + low, high = high, low + } + obsAt := func(code, display string, population int64) catalog.SemanticObservation { + o := pgcat06Observation(population) + o.Source.Type = "Observation" + o.Key.Selector = "code.coding[]" + o.Key.System = "urn:resolve" + o.Key.Code = code + o.Key.Display = display + o.RuleHint = "CODED_CONCEPT_V1" + o.RuleVersion = "5" + return o + } + lowObs := obsAt("low", "SecretLow", 1) + highObs := obsAt("high", "GlucoseHigh", 2) + foreignObs := obsAt("foreign", "ForeignWinsIfLeaked", 99) + seed := func(key, path, example string, observation catalog.SemanticObservation, sourceID string) { + s.seedContribution(t, contributionSeed{ + Key: key, Project: pgcat06Project, Generation: pgcat06Generation, BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: path, + ResourceType: "Observation", SourceID: sourceID, Ordinal: 0, + ConceptID: "cpt-resolve", BindingID: "bnd-resolve", Example: example, + Observation: pgcat06ObsJSON(t, observation), + }) + } + seed(low, pgcat06PathSecret, "low-ex", lowObs, "res-low") + seed(high, pgcat06PathGlucose, "high-ex", highObs, "res-high") + s.seedContribution(t, contributionSeed{ + Key: pgcat06Hex64("resolve-foreign"), Project: pgcat06Project, Generation: "gen-2", BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: pgcat06PathGlucose, + ResourceType: "Observation", SourceID: "res-f", Ordinal: 0, + ConceptID: "cpt-resolve", BindingID: "bnd-resolve", Example: "f", + Observation: pgcat06ObsJSON(t, foreignObs), + }) + + repo := s.readRepository(t) + request := func(unrestricted bool, paths []string, refs ...catalog.SemanticInventoryReference) catalog.SemanticInventoryResolveResult { + t.Helper() + result, err := repo.ResolveSemanticInventorySelections(s.ctx, catalog.SemanticInventoryResolveOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(unrestricted), + AuthResourcePaths: paths, + References: refs, + }) + if err != nil { + s.fail("resolve selections", err) + } + return result + } + refs := func() []catalog.SemanticInventoryReference { + return []catalog.SemanticInventoryReference{ + {ConceptID: "cpt-missing", BindingID: "bnd-resolve"}, + {ConceptID: "cpt-resolve", BindingID: "bnd-resolve"}, + } + } + + restricted := request(false, []string{pgcat06PathGlucose}, refs()...) + if len(restricted.Entries) != 1 || restricted.Entries[0].ConceptID != "cpt-resolve" || + !reflect.DeepEqual(restricted.Entries[0].Observation, highObs) { + t.Fatalf("restricted resolve = entries %+v state %v build %+v, want the authorized glucose row verbatim", + restricted.Entries, restricted.State, restricted.Build) + } + if restricted.State != catalog.SemanticInventoryComplete || restricted.Build.ScannedResources != 0 || + restricted.Build.LeaseToken != "" || restricted.Build.Diagnostics != nil { + t.Fatalf("restricted resolve build = %+v state %v", restricted.Build, restricted.State) + } + + unrestricted := request(true, nil, refs()...) + if len(unrestricted.Entries) != 1 || !reflect.DeepEqual(unrestricted.Entries[0].Observation, lowObs) { + t.Fatalf("unrestricted resolve = %+v, want the least-key secret row verbatim", unrestricted.Entries) + } + if unrestricted.Build.ScannedResources != 7 { + t.Fatalf("unrestricted resolve must expose build internals verbatim: %+v", unrestricted.Build) + } + if unrestricted.Build.Project != pgcat06Project || unrestricted.Build.DatasetGeneration != pgcat06Generation || + unrestricted.Build.BuildID != catalog.SemanticInventoryBuildID(pgcat06Project, pgcat06Generation) { + t.Fatalf("resolve must return the fully identified build: %+v", unrestricted.Build) + } + + empty := request(true, nil, catalog.SemanticInventoryReference{ConceptID: "cpt-none", BindingID: "bnd-none"}) + if empty.Entries == nil || len(empty.Entries) != 0 || empty.State != catalog.SemanticInventoryComplete { + t.Fatalf("all-unmatched resolve = %+v, want non-nil empty entries with the completed build", empty) + } +} + +func TestSemanticInventoryResolveFamilyGolden(t *testing.T) { + s := newPgcat06Service(t) + s.seedBuild(t, pgcat06DefaultBuildSeed(pgcat06Project, pgcat06Generation)) + buildID := catalog.SemanticInventoryBuildID(pgcat06Project, pgcat06Generation) + + fam := func(path, system, version string, population int64) catalog.SemanticObservation { + o := pgcat06Observation(population) + o.Source.Type = "Observation" + o.Source.Path = path + o.Key.Selector = "code.coding[]" + o.Key.System = system + o.Key.Version = version + o.RuleHint = "CODED_CONCEPT_V1" + o.RuleVersion = "5" + return o + } + put := func(sourceID string, concept, path string, observation catalog.SemanticObservation, authPath string) { + s.seedContribution(t, contributionSeed{ + Project: pgcat06Project, Generation: pgcat06Generation, BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: authPath, + ResourceType: "Observation", SourceID: sourceID, Ordinal: 0, + ConceptID: concept, BindingID: "bnd-resolve-family", + Observation: pgcat06ObsJSON(t, observation), + }) + } + put("r-1", "cpt-rv2-a", "code.coding", fam("code.coding", "urn:sys-f", "v2", 1), pgcat06PathGlucose) + put("r-2", "cpt-rv2-b", "code.coding", fam("code.coding", "urn:sys-f", "v2", 2), pgcat06PathVisit) + put("r-3", "cpt-rv2-c", "code.coding", fam("code.coding", "urn:sys-f", "v2", 100), pgcat06PathSecret) + put("r-4", "cpt-rnv-a", "code.coding", fam("code.coding", "urn:sys-f", "", 5), pgcat06PathGlucose) + put("r-5", "cpt-rv2-ext", "code.coding", func() catalog.SemanticObservation { + o := fam("code.coding", "urn:sys-f", "v2", 77) + o.ExtensionURLPath = []string{"http://ext"} + return o + }(), pgcat06PathGlucose) + put("r-6", "cpt-rv2-rule", "code.coding", func() catalog.SemanticObservation { + o := fam("code.coding", "urn:sys-f", "v2", 77) + o.RuleHint = "CODED_VALUE_V1" + return o + }(), pgcat06PathGlucose) + put("r-7", "cpt-rv2-type", "code.coding", func() catalog.SemanticObservation { + o := fam("code.coding", "urn:sys-f", "v2", 77) + o.Source.Type = "Encounter" + return o + }(), pgcat06PathGlucose) + s.seedContribution(t, contributionSeed{ + Project: pgcat06Project, Generation: pgcat06Generation, BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: pgcat06PathGlucose, + ResourceType: "Observation", SourceID: "r-8", Ordinal: 0, + ConceptID: "cpt-rnull", BindingID: "bnd-resolve-family", + Observation: `{"source":{"type":"Observation","path":"code.coding"},"key":{"selector":"code.coding[]","system":"urn:sys-f","version":null},"rule_hint":"CODED_CONCEPT_V1","population":40}`, + }) + s.seedContribution(t, contributionSeed{ + Project: pgcat06Project, Generation: "gen-2", BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: pgcat06PathGlucose, + ResourceType: "Observation", SourceID: "r-9", Ordinal: 0, + ConceptID: "cpt-rv2-gen2", BindingID: "bnd-resolve-family", + Observation: pgcat06ObsJSON(t, fam("code.coding", "urn:sys-f", "v2", 999)), + }) + + v2Ref := catalog.SemanticInventoryFamilyReference{ + BindingID: "bnd-resolve-family", SourceKind: catalog.SemanticInventorySourceFile, + ResourceType: "Observation", SourcePath: "code.coding", System: "urn:sys-f", Version: "v2", + } + v2Ref.FamilyID = catalog.SemanticInventoryFamilyID(v2Ref) + if v2Ref.FamilyID != "1e2abae9e0d5d0fd601edc065d704e5e8e447d7076ba67cfacd1c3b7b088ca43" { + t.Fatalf("v2 family digest drifted from the pinned literal: %s", v2Ref.FamilyID) + } + nvRef := v2Ref + nvRef.Version = "" + nvRef.FamilyID = catalog.SemanticInventoryFamilyID(nvRef) + if nvRef.FamilyID != "5a9a79e24891c171724106b7b46f237e68abd0e603243bbde139f243e6e0ae1d" { + t.Fatalf("versionless family digest drifted from the pinned literal: %s", nvRef.FamilyID) + } + wrongKind := v2Ref + wrongKind.SourceKind = catalog.SemanticInventorySourceRetained + wrongKind.FamilyID = catalog.SemanticInventoryFamilyID(wrongKind) + missing := v2Ref + missing.System = "urn:sys-nowhere" + missing.FamilyID = catalog.SemanticInventoryFamilyID(missing) + + repo := s.readRepository(t) + result, err := repo.ResolveSemanticInventorySelections(s.ctx, catalog.SemanticInventoryResolveOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(false), + AuthResourcePaths: []string{pgcat06PathGlucose}, + FamilyReferences: []catalog.SemanticInventoryFamilyReference{nvRef, missing, v2Ref, wrongKind}, + }) + if err != nil { + s.fail("resolve families", err) + } + if len(result.Families) != 2 || + result.Families[0].FamilyID != nvRef.FamilyID || result.Families[1].FamilyID != v2Ref.FamilyID { + t.Fatalf("family resolve = %+v, want [versionless, v2] in request order", result.Families) + } + if result.Families[0].Population != 5 || result.Families[1].Population != 1 { + t.Fatalf("restricted family populations = %d, %d; want 5 and 1 (visit/secret rows excluded)", + result.Families[0].Population, result.Families[1].Population) + } + want := catalog.SemanticInventoryFamily{ + SemanticInventoryFamilyReference: v2Ref, + Source: catalog.SemanticObservationSource{ + Canonical: "Observation.code.coding[]", Type: "Observation", + Profile: "http://profiles.test/Observation", Path: "code.coding", + }, + KeySelector: "code.coding[]", + Value: catalog.SemanticObservationValue{ + Selector: "value.value", Type: "string", + }, + SchemaVersion: catalog.SemanticObservationSchemaVersion, + RuleVersion: "5", + Completeness: catalog.SemanticComplete, + Status: "SUPPORTED", + Population: 1, + Examples: []string{}, + ExamplesTruncated: false, + } + if !reflect.DeepEqual(result.Families[1], want) { + t.Fatalf("restricted v2 family =\n%#v\nwant\n%#v", result.Families[1], want) + } + + unrestrictedResult, err := repo.ResolveSemanticInventorySelections(s.ctx, catalog.SemanticInventoryResolveOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(true), + FamilyReferences: []catalog.SemanticInventoryFamilyReference{v2Ref}, + }) + if err != nil { + s.fail("resolve families unrestricted", err) + } + if len(unrestrictedResult.Families) != 1 || unrestrictedResult.Families[0].Population != 103 { + t.Fatalf("unrestricted v2 family population = %+v, want 103 (1+2+100 across all paths)", unrestrictedResult.Families) + } + + s.exec(t, "UPDATE "+s.contributionFQN()+" SET observation = observation #- '{key,version}' WHERE source_id = 'r-8'") + nullProbe, err := repo.ResolveSemanticInventorySelections(s.ctx, catalog.SemanticInventoryResolveOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(false), + AuthResourcePaths: []string{pgcat06PathGlucose}, + FamilyReferences: []catalog.SemanticInventoryFamilyReference{nvRef}, + }) + if err != nil { + s.fail("resolve families after null-leaf mutation", err) + } + if len(nullProbe.Families) != 1 || nullProbe.Families[0].Population != 45 { + t.Fatalf("absent-version row did not join the versionless family: %+v", nullProbe.Families) + } + s.exec(t, "UPDATE "+s.contributionFQN()+ + " SET observation = jsonb_set(observation, '{key,version}', 'null'::jsonb, true) WHERE source_id = 'r-8'") + reNull, err := repo.ResolveSemanticInventorySelections(s.ctx, catalog.SemanticInventoryResolveOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(false), + AuthResourcePaths: []string{pgcat06PathGlucose}, + FamilyReferences: []catalog.SemanticInventoryFamilyReference{nvRef}, + }) + if err != nil { + s.fail("resolve families after null-leaf restore", err) + } + if len(reNull.Families) != 1 || reNull.Families[0].Population != 5 { + t.Fatalf("JSON-null-version row leaked into the versionless family: %+v", reNull.Families) + } +} + +func TestSemanticInventoryStorageStates(t *testing.T) { + s := newPgcat06Service(t) + repo := s.readRepository(t) + pageOpts := catalog.SemanticInventoryPageOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(true), + } + resolveOpts := func() catalog.SemanticInventoryResolveOptions { + return catalog.SemanticInventoryResolveOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(true), + References: []catalog.SemanticInventoryReference{{ConceptID: "cpt-x", BindingID: "bnd-x"}}, + } + } + + if _, err := s.iso.Exec(s.ctx, "DROP TABLE "+s.buildFQN()+" CASCADE"); err != nil { + s.fail("drop build table", err) + } + page, err := repo.PageSemanticInventory(s.ctx, pageOpts) + if err != nil { + s.fail("page without build table", err) + } + if page.Entries == nil || len(page.Entries) != 0 || page.State != catalog.SemanticInventoryUnknown || + !reflect.DeepEqual(page.Build, catalog.SemanticInventoryBuild{}) { + t.Fatalf("page without build table = %+v, want unknown-state empty", page) + } + result, err := repo.ResolveSemanticInventorySelections(s.ctx, resolveOpts()) + if err != nil { + s.fail("resolve without build table", err) + } + if result.Entries == nil || len(result.Entries) != 0 || result.State != catalog.SemanticInventoryUnknown { + t.Fatalf("resolve without build table = %+v, want unknown-state empty", result) + } + + if _, err := s.iso.Exec(s.ctx, "DELETE FROM "+s.ledgerFQN()+" WHERE version >= 6"); err != nil { + s.fail("reset ledger below the build migration", err) + } + if _, err := storepostgres.MigrateSchema(s.ctx, s.openClient(t), s.schema); err != nil { + s.fail("re-migrate schema", err) + } + s.seedBuild(t, pgcat06DefaultBuildSeed(pgcat06Project, pgcat06Generation)) + if _, err := s.iso.Exec(s.ctx, "DROP TABLE "+s.contributionFQN()+" CASCADE"); err != nil { + s.fail("drop contribution table", err) + } + page, err = repo.PageSemanticInventory(s.ctx, pageOpts) + if err != nil { + s.fail("page without contribution table", err) + } + if page.State != catalog.SemanticInventoryUnknown || page.Entries == nil || len(page.Entries) != 0 { + t.Fatalf("page without contribution table = %+v, want the Arango-matching unknown-empty", page) + } + _, err = repo.ResolveSemanticInventorySelections(s.ctx, resolveOpts()) + var missing *SemanticInventoryStorageMissingError + if !errors.As(err, &missing) || !strings.Contains(err.Error(), semanticInventoryContributionTableName) { + t.Fatalf("resolve without contribution table error = %v, want named SemanticInventoryStorageMissingError", err) + } + if _, err := s.iso.Exec(s.ctx, "DELETE FROM "+s.ledgerFQN()+" WHERE version >= 9"); err != nil { + s.fail("reset ledger below the contribution migration", err) + } + if _, err := storepostgres.MigrateSchema(s.ctx, s.openClient(t), s.schema); err != nil { + s.fail("restore contribution table", err) + } + page, err = repo.PageSemanticInventory(s.ctx, pageOpts) + if err != nil { + s.fail("page after table restore", err) + } + if page.State != catalog.SemanticInventoryComplete || page.Entries == nil || len(page.Entries) != 0 { + t.Fatalf("page over restored-but-empty table = %+v, want complete-state empty (distinct from the dropped-table unknown)", page) + } +} + +func TestSemanticInventoryEntryIndexBoundaryRefusesByName(t *testing.T) { + s := newPgcat06Service(t) + seed := pgcat06DefaultBuildSeed(pgcat06Project, pgcat06Generation) + seed.EntryIndexVersion = catalog.SemanticInventoryEntryIndexVersion + s.seedBuild(t, seed) + buildID := catalog.SemanticInventoryBuildID(pgcat06Project, pgcat06Generation) + s.seedContribution(t, contributionSeed{ + Project: pgcat06Project, Generation: pgcat06Generation, BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: pgcat06PathGlucose, + ResourceType: "Observation", SourceID: "idx-1", Ordinal: 0, + ConceptID: "cpt-idx", BindingID: "bnd-idx", Example: "e", + Observation: pgcat06ObsJSON(t, pgcat06Observation(1)), + }) + repo := s.readRepository(t) + pageOpts := catalog.SemanticInventoryPageOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(true), + } + resolveOpts := catalog.SemanticInventoryResolveOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(true), + References: []catalog.SemanticInventoryReference{{ConceptID: "cpt-idx", BindingID: "bnd-idx"}}, + } + + _, err := repo.PageSemanticInventory(s.ctx, pageOpts) + var unsupported *SemanticInventoryEntriesUnsupportedError + if !errors.As(err, &unsupported) || !errors.Is(err, ErrSemanticInventoryEntriesUnsupported) { + t.Fatalf("page at the entry-index boundary error = %v, want named SemanticInventoryEntriesUnsupportedError", err) + } + if unsupported.Project != pgcat06Project || unsupported.DatasetGeneration != pgcat06Generation || + unsupported.BuildID != buildID || unsupported.EntryIndexVersion != catalog.SemanticInventoryEntryIndexVersion { + t.Fatalf("refusal fields = %+v", unsupported) + } + if _, err := repo.ResolveSemanticInventorySelections(s.ctx, resolveOpts); !errors.Is(err, ErrSemanticInventoryEntriesUnsupported) { + t.Fatalf("resolve at the entry-index boundary error = %v, want the named boundary error", err) + } + + s.exec(t, "UPDATE "+s.buildFQN()+" SET entry_index_version = $1", catalog.SemanticInventoryEntryIndexVersion-1) + page, err := repo.PageSemanticInventory(s.ctx, pageOpts) + if err != nil || len(page.Entries) != 1 || page.Entries[0].ConceptID != "cpt-idx" { + t.Fatalf("below the boundary the page must serve: %+v err %v", page, err) + } + if _, err := repo.ResolveSemanticInventorySelections(s.ctx, resolveOpts); err != nil { + t.Fatalf("below the boundary resolve must serve: %v", err) + } + s.exec(t, "UPDATE "+s.buildFQN()+" SET entry_index_version = $1", catalog.SemanticInventoryEntryIndexVersion) + if _, err := repo.PageSemanticInventory(s.ctx, pageOpts); !errors.Is(err, ErrSemanticInventoryEntriesUnsupported) { + t.Fatalf("restored boundary must refuse by name again: %v", err) + } +} + +func pgcat06ContributionInsert(fqn string, seed contributionSeed) (string, []any) { + if seed.Key == "" { + seed.Key = pgcat06Hex64(seed.Project + "/" + seed.Generation + "/" + seed.BuildID + "/" + + seed.SourceKind + "/" + seed.SourceID + "/" + fmt.Sprint(seed.Ordinal) + "/" + seed.ConceptID) + } + return "INSERT INTO " + fqn + + " (contribution_key, project, dataset_generation, build_id, source_kind, auth_resource_path," + + " resource_type, source_id, ordinal, concept_id, binding_id, example, payload_digest, observation)" + + " VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14::jsonb)", + []any{seed.Key, seed.Project, catalog.NormalizeDatasetGeneration(seed.Generation), seed.BuildID, + seed.SourceKind, seed.AuthResourcePath, seed.ResourceType, seed.SourceID, seed.Ordinal, + seed.ConceptID, seed.BindingID, seed.Example, pgcat06Hex64("digest/" + seed.Key), seed.Observation} +} + +func TestSemanticInventoryTwoConnectionReadConsistency(t *testing.T) { + const rounds = 10 + s := newPgcat06Service(t) + s.seedBuild(t, pgcat06DefaultBuildSeed(pgcat06Project, pgcat06Generation)) + buildID := catalog.SemanticInventoryBuildID(pgcat06Project, pgcat06Generation) + s.seedContribution(t, contributionSeed{ + Project: pgcat06Project, Generation: pgcat06Generation, BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: pgcat06PathGlucose, + ResourceType: "Observation", SourceID: "stable-1", Ordinal: 0, + ConceptID: "cpt-stable", BindingID: "bnd-stable", Example: "s", + Observation: pgcat06ObsJSON(t, pgcat06Observation(1)), + }) + + writer, writerPid := s.openDedicatedClient(t, "writer") + readerA, pidA := s.openDedicatedClient(t, "reader-a") + readerB, pidB := s.openDedicatedClient(t, "reader-b") + if writerPid == pidA || writerPid == pidB || pidA == pidB { + t.Fatalf("racing backends share pg_backend_pid: %d %d %d", writerPid, pidA, pidB) + } + repoA, err := NewSemanticReadRepository(readerA, s.schema) + if err != nil { + s.fail("reader A repository", err) + } + repoB, err := NewSemanticReadRepository(readerB, s.schema) + if err != nil { + s.fail("reader B repository", err) + } + pageOpts := catalog.SemanticInventoryPageOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(true), + } + resolveOpts := catalog.SemanticInventoryResolveOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(true), + References: []catalog.SemanticInventoryReference{{ConceptID: "cpt-stable", BindingID: "bnd-stable"}}, + } + baselinePage, err := repoA.PageSemanticInventory(s.ctx, pageOpts) + if err != nil { + s.fail("baseline page", err) + } + baselineResolve, err := repoA.ResolveSemanticInventorySelections(s.ctx, resolveOpts) + if err != nil { + s.fail("baseline resolve", err) + } + expectedPageWitness, expectedResolveWitness := pageWitness(t, baselinePage), resolveWitness(t, baselineResolve) + + for round := 0; round < rounds; round++ { + commit := round >= rounds/2 + for _, pair := range []struct { + label string + client *storepostgres.Client + }{{"writer", writer}, {"reader-a", readerA}, {"reader-b", readerB}} { + if pid := s.witnessClientPid(t, pair.client, pair.label); pid <= 0 { + t.Fatalf("round %d: %s backend is not live", round, pair.label) + } + } + conn, err := writer.Acquire(s.ctx) + if err != nil { + s.fail("acquire writer", err) + } + concept := fmt.Sprintf("cpt-race-%d", round) + sql, args := pgcat06ContributionInsert(s.contributionFQN(), contributionSeed{ + Project: pgcat06Project, Generation: pgcat06Generation, BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: pgcat06PathGlucose, + ResourceType: "Observation", SourceID: fmt.Sprintf("race-%d", round), Ordinal: 0, + ConceptID: concept, BindingID: "bnd-stable", Example: "r", + Observation: pgcat06ObsJSON(t, pgcat06Observation(1)), + }) + if _, err := conn.Exec(s.ctx, "BEGIN"); err != nil { + conn.Release() + s.fail("begin writer transaction", err) + } + if _, err := conn.Exec(s.ctx, sql, args...); err != nil { + conn.Release() + s.fail("insert inside writer transaction", err) + } + for name, repo := range map[string]*SemanticReadRepository{"A": repoA, "B": repoB} { + page, err := repo.PageSemanticInventory(s.ctx, pageOpts) + if err != nil { + conn.Exec(s.ctx, "ROLLBACK") + conn.Release() + s.fail("reader "+name+" page during open transaction", err) + } + if w := pageWitness(t, page); w != expectedPageWitness { + conn.Exec(s.ctx, "ROLLBACK") + conn.Release() + t.Fatalf("round %d: reader %s saw the in-flight row: %s", round, name, w) + } + result, err := repo.ResolveSemanticInventorySelections(s.ctx, resolveOpts) + if err != nil { + conn.Exec(s.ctx, "ROLLBACK") + conn.Release() + s.fail("reader "+name+" resolve during open transaction", err) + } + if w := resolveWitness(t, result); w != expectedResolveWitness { + conn.Exec(s.ctx, "ROLLBACK") + conn.Release() + t.Fatalf("round %d: reader %s resolve witness drifted: %s", round, name, w) + } + } + if commit { + if _, err := conn.Exec(s.ctx, "COMMIT"); err != nil { + conn.Release() + s.fail("commit writer transaction", err) + } + } else { + if _, err := conn.Exec(s.ctx, "ROLLBACK"); err != nil { + conn.Release() + s.fail("rollback writer transaction", err) + } + } + conn.Release() + + pageA, err := repoA.PageSemanticInventory(s.ctx, pageOpts) + if err != nil { + s.fail("post-transaction page A", err) + } + pageB, err := repoB.PageSemanticInventory(s.ctx, pageOpts) + if err != nil { + s.fail("post-transaction page B", err) + } + if wa, wb := pageWitness(t, pageA), pageWitness(t, pageB); wa != wb { + t.Fatalf("round %d: two backends disagree after the transaction:\n%s\nvs\n%s", round, wa, wb) + } + sawCommitted := false + for _, entry := range pageA.Entries { + if entry.ConceptID == concept { + sawCommitted = true + } + } + if sawCommitted != commit { + t.Fatalf("round %d: committed=%v but readers saw the row: %v", round, commit, sawCommitted) + } + if !commit { + if w := pageWitness(t, pageA); w != expectedPageWitness { + t.Fatalf("round %d: aborted write leaked into the readers' witness: %s", round, w) + } + } else { + expectedPageWitness = pageWitness(t, pageA) + } + } +} + +type pgcat06SeamSource struct { + inner storepostgres.ConnSource + mu sync.Mutex + calls int + seamAt int + fired bool + seam func() +} + +func (z *pgcat06SeamSource) Acquire(ctx context.Context) (*pgxpool.Conn, error) { + z.mu.Lock() + z.calls++ + if z.calls == z.seamAt && !z.fired { + z.fired = true + z.seam() + } + z.mu.Unlock() + return z.inner.Acquire(ctx) +} + +func pgcat06PgErrorCode(err error) string { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + return pgErr.Code + } + return "" +} + +func TestSemanticInventoryCrashSeams(t *testing.T) { + s := newPgcat06Service(t) + s.seedBuild(t, pgcat06DefaultBuildSeed(pgcat06Project, pgcat06Generation)) + buildID := catalog.SemanticInventoryBuildID(pgcat06Project, pgcat06Generation) + s.seedContribution(t, contributionSeed{ + Project: pgcat06Project, Generation: pgcat06Generation, BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: pgcat06PathGlucose, + ResourceType: "Observation", SourceID: "seam-1", Ordinal: 0, + ConceptID: "cpt-seam", BindingID: "bnd-seam", Example: "s", + Observation: pgcat06ObsJSON(t, pgcat06Observation(1)), + }) + pageOpts := catalog.SemanticInventoryPageOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(true), + } + baseline, err := s.readRepository(t).PageSemanticInventory(s.ctx, pageOpts) + if err != nil { + s.fail("baseline page", err) + } + baselineWitness := pageWitness(t, baseline) + + type pageOutcome struct { + page catalog.SemanticInventoryPage + err error + } + waitForLockWait := func(t *testing.T, pid int32) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for { + var waiting bool + if err := s.admin.QueryRow(s.ctx, + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE pid = $1 AND wait_event_type = 'Lock')", + pid).Scan(&waiting); err != nil { + s.fail("poll pg_stat_activity", err) + } + if waiting { + return + } + if time.Now().After(deadline) { + t.Fatalf("backend %d never entered a lock wait", pid) + } + select { + case <-s.ctx.Done(): + t.Fatalf("context expired waiting for the lock wait") + case <-time.After(50 * time.Millisecond): + } + } + } + terminate := func(t *testing.T, pid int32) { + t.Helper() + var killed bool + if err := s.admin.QueryRow(s.ctx, "SELECT pg_terminate_backend($1)", pid).Scan(&killed); err != nil { + s.fail("terminate backend", err) + } + if !killed { + t.Fatalf("pg_terminate_backend(%d) reported no live backend", pid) + } + } + + locker, lockerPid := s.openDedicatedClient(t, "lock-holder") + lockConn, err := locker.Acquire(s.ctx) + if err != nil { + s.fail("acquire lock holder", err) + } + if _, err := lockConn.Exec(s.ctx, "BEGIN; LOCK TABLE "+s.contributionFQN()+" IN ACCESS EXCLUSIVE MODE"); err != nil { + lockConn.Release() + s.fail("take table lock", err) + } + reader, readerPid := s.openDedicatedClient(t, "locked-reader") + readerRepo, err := NewSemanticReadRepository(reader, s.schema) + if err != nil { + s.fail("locked reader repository", err) + } + outcome := make(chan pageOutcome, 1) + go func() { + page, err := readerRepo.PageSemanticInventory(s.ctx, pageOpts) + outcome <- pageOutcome{page, err} + }() + waitForLockWait(t, readerPid) + terminate(t, lockerPid) + select { + case res := <-outcome: + if res.err != nil { + t.Fatalf("reader failed after the lock holder was terminated: %s", s.scrub(res.err)) + } + if w := pageWitness(t, res.page); w != baselineWitness { + t.Fatalf("reader after lock-holder crash returned the wrong page: %s", w) + } + case <-time.After(60 * time.Second): + t.Fatalf("reader never completed after the lock holder was terminated") + } + lockConn.Release() + if healedPid := s.witnessClientPid(t, locker, "lock-holder (healed)"); healedPid == lockerPid { + t.Fatalf("crashed lock-holder pool reused backend pid %d", lockerPid) + } + + lockConn, err = locker.Acquire(s.ctx) + if err != nil { + s.fail("re-acquire lock holder", err) + } + if _, err := lockConn.Exec(s.ctx, "BEGIN; LOCK TABLE "+s.contributionFQN()+" IN ACCESS EXCLUSIVE MODE"); err != nil { + lockConn.Release() + s.fail("take table lock (second seam)", err) + } + reader2, reader2Pid := s.openDedicatedClient(t, "blocked-reader") + reader2Repo, err := NewSemanticReadRepository(reader2, s.schema) + if err != nil { + s.fail("blocked reader repository", err) + } + outcome2 := make(chan pageOutcome, 1) + go func() { + page, err := reader2Repo.PageSemanticInventory(s.ctx, pageOpts) + outcome2 <- pageOutcome{page, err} + }() + waitForLockWait(t, reader2Pid) + terminate(t, reader2Pid) + select { + case res := <-outcome2: + if res.err == nil { + t.Fatalf("terminated reader reported success") + } + case <-time.After(60 * time.Second): + t.Fatalf("terminated reader never returned an error") + } + if _, err := lockConn.Exec(s.ctx, "COMMIT"); err != nil { + lockConn.Release() + s.fail("release second lock", err) + } + lockConn.Release() + if healedPid := s.witnessClientPid(t, reader2, "blocked-reader (healed)"); healedPid == reader2Pid { + t.Fatalf("terminated reader pool reused backend pid %d", reader2Pid) + } + page, err := reader2Repo.PageSemanticInventory(s.ctx, pageOpts) + if err != nil { + s.fail("recovered reader page", err) + } + if w := pageWitness(t, page); w != baselineWitness { + t.Fatalf("recovered reader returned the wrong page: %s", w) + } + + dropSeam := func() { + if _, err := s.iso.Exec(s.ctx, "DROP TABLE "+s.contributionFQN()+" CASCADE"); err != nil { + s.fail("drop contribution table at seam", err) + } + } + for _, tc := range []struct { + name string + run func(repo *SemanticReadRepository) error + seam int + }{ + {"page", func(repo *SemanticReadRepository) error { + _, err := repo.PageSemanticInventory(s.ctx, pageOpts) + return err + }, 4}, + {"resolve", func(repo *SemanticReadRepository) error { + _, err := repo.ResolveSemanticInventorySelections(s.ctx, catalog.SemanticInventoryResolveOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(true), + References: []catalog.SemanticInventoryReference{{ConceptID: "cpt-seam", BindingID: "bnd-seam"}}, + }) + return err + }, 4}, + } { + client := s.openClient(t) + seam := &pgcat06SeamSource{inner: client, seamAt: tc.seam, seam: dropSeam} + repo, err := NewSemanticReadRepository(seam, s.schema) + if err != nil { + s.fail("seam repository "+tc.name, err) + } + err = tc.run(repo) + if err == nil { + t.Fatalf("%s: dropped table between check and statement returned no error (disguised empty page)", tc.name) + } + if code := pgcat06PgErrorCode(err); code != "42P01" { + t.Fatalf("%s: dropped-table error code = %q (%s), want 42P01", tc.name, code, s.scrub(err)) + } + if !seam.fired { + t.Fatalf("%s: the seam never fired (statement order changed under the test)", tc.name) + } + if _, err := s.iso.Exec(s.ctx, "DELETE FROM "+s.ledgerFQN()+" WHERE version >= 9"); err != nil { + s.fail("reset ledger after seam", err) + } + if _, err := storepostgres.MigrateSchema(s.ctx, s.openClient(t), s.schema); err != nil { + s.fail("restore contribution table after seam", err) + } + } +} + +func TestSemanticInventoryPlanShapeEvidence(t *testing.T) { + s := newPgcat06Service(t) + s.seedBuild(t, pgcat06DefaultBuildSeed(pgcat06Project, pgcat06Generation)) + buildID := catalog.SemanticInventoryBuildID(pgcat06Project, pgcat06Generation) + seedBulk := func(project, generation, sourceIDPrefix string, count int) { + if _, err := s.iso.Exec(s.ctx, "INSERT INTO "+s.contributionFQN()+ + " (contribution_key, project, dataset_generation, build_id, source_kind, auth_resource_path,"+ + " resource_type, source_id, ordinal, concept_id, binding_id, example, payload_digest, observation)"+ + " SELECT md5($5 || gs::text || 'ka') || md5($5 || gs::text || 'kb'), $1, $2, $3, 'file', $4,"+ + " 'Observation', $5 || gs, 0, 'cpt-' || gs, 'bnd-' || (gs % 25), 'ex-' || gs,"+ + " md5(gs::text || 'da') || md5(gs::text || 'db'), '{\"population\":1,\"rule_hint\":\"CODED_VALUE_V1\",\"source\":{\"type\":\"Observation\",\"path\":\"code.coding[]\"},\"key\":{\"system\":\"urn:bulk\"}}'::jsonb"+ + " FROM generate_series(1, $6) gs", + project, catalog.NormalizeDatasetGeneration(generation), buildID, pgcat06PathGlucose, sourceIDPrefix, count); err != nil { + s.fail("seed bulk contributions", err) + } + } + seedBulk(pgcat06Project, pgcat06Generation, "bulk-", 4000) + seedBulk("cat06-noise", pgcat06Generation, "noise-", 4000) + if _, err := s.iso.Exec(s.ctx, "ANALYZE "+s.contributionFQN()); err != nil { + s.fail("analyze contribution table", err) + } + + explain := func(label, sql string, args ...any) string { + t.Helper() + rows, err := s.iso.Query(s.ctx, "EXPLAIN (FORMAT TEXT) "+sql, args...) + if err != nil { + s.fail("explain "+label, err) + } + defer rows.Close() + var plan strings.Builder + for rows.Next() { + var line string + if err := rows.Scan(&line); err != nil { + s.fail("scan "+label+" plan", err) + } + plan.WriteString(line) + plan.WriteString("\n") + } + if err := rows.Err(); err != nil { + s.fail("read "+label+" plan", err) + } + t.Logf("%s plan:\n%s", label, plan.String()) + return plan.String() + } + + pageSQL, pageArgs := semanticPageQuery(s.contributionFQN(), semanticPageParams{ + project: pgcat06Project, generation: pgcat06Generation, buildID: buildID, + sourceKind: catalog.SemanticInventorySourceFile, unrestricted: true, paths: []string{}, + limit: 50, + }) + if plan := explain("page", pageSQL, pageArgs...); !strings.Contains(plan, "fhir_semantic_inventory_contribution_scope_ix") { + t.Fatalf("page plan does not touch the target-contract scope index") + } + + scope := resolveScope{ + project: pgcat06Project, generation: pgcat06Generation, buildID: buildID, + sourceKind: catalog.SemanticInventorySourceFile, unrestricted: true, paths: []string{}, + observationSchema: catalog.SemanticObservationSchemaVersion, + } + exactSQL, exactArgs := semanticResolveReferencesQuery(s.contributionFQN(), scope, []catalog.SemanticInventoryReference{ + {ConceptID: "cpt-42", BindingID: "bnd-42"}, + }) + if plan := explain("resolve-exact", exactSQL, exactArgs...); !strings.Contains(plan, "fhir_semantic_inventory_contribution") { + t.Fatalf("resolve-exact plan does not scan the contribution table at all") + } + family := catalog.SemanticInventoryFamilyReference{ + BindingID: "bnd-42", SourceKind: catalog.SemanticInventorySourceFile, + ResourceType: "Observation", SourcePath: "code.coding[]", System: "urn:bulk", Version: "", + } + familySQL, familyArgs := semanticResolveFamiliesQuery(s.contributionFQN(), scope, []catalog.SemanticInventoryFamilyReference{family}) + if plan := explain("resolve-family", familySQL, familyArgs...); !strings.Contains(plan, "fhir_semantic_inventory_contribution") { + t.Fatalf("resolve-family plan does not scan the contribution table at all") + } +} + +func TestSemanticInventoryStorageContract(t *testing.T) { + s := newPgcat06Service(t) + var indexDef string + if err := s.iso.QueryRow(s.ctx, + "SELECT indexdef FROM pg_indexes WHERE schemaname = $1 AND tablename = $2"+ + " AND indexname = 'fhir_semantic_inventory_contribution_scope_ix'", + s.schema, semanticInventoryContributionTableName).Scan(&indexDef); err != nil { + s.fail("read scope index definition", err) + } + for _, want := range []string{ + "project", "dataset_generation", "build_id", "source_kind", "binding_id", "concept_id", "auth_resource_path", + } { + if !strings.Contains(indexDef, want) { + t.Fatalf("scope index definition %q missing column %q", indexDef, want) + } + } + if !strings.Contains(indexDef, `(project, dataset_generation, build_id, source_kind, binding_id, concept_id, auth_resource_path)`) { + t.Fatalf("scope index definition %q does not carry the contracted column order", indexDef) + } + + var ledgerVersions []int64 + rows, err := s.iso.Query(s.ctx, "SELECT version FROM "+s.ledgerFQN()+" ORDER BY version") + if err != nil { + s.fail("read migration ledger", err) + } + defer rows.Close() + for rows.Next() { + var v int64 + if err := rows.Scan(&v); err != nil { + s.fail("scan migration ledger", err) + } + ledgerVersions = append(ledgerVersions, v) + } + if err := rows.Err(); err != nil { + s.fail("read migration ledger", err) + } + if len(ledgerVersions) == 0 { + t.Fatal("empty migration ledger after MigrateSchema") + } + for i, v := range ledgerVersions { + if v != int64(i+1) { + t.Fatalf("migration ledger is not contiguous from 1: %v", ledgerVersions) + } + } + if ledgerVersions[len(ledgerVersions)-1] < 9 { + t.Fatalf("the pg-catalog-05 contribution migration (version 9) is not applied: %v", ledgerVersions) + } +} + +func TestZZZeroLeftoverSemanticInventoryPageFixtures(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 integration tests") + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for the fixture audit: %s", err) + } + defer admin.Close() + + pgcat06OwnedMu.Lock() + owned := make([]string, 0, len(pgcat06OwnedByName)) + for db := range pgcat06OwnedByName { + owned = append(owned, db) + } + pgcat06OwnedMu.Unlock() + if len(owned) == 0 { + t.Skip("no fixture databases were created by this run") + } + for _, db := range owned { + var count int + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Fatalf("audit owned database %q: %v", db, err) + } + if count != 0 { + t.Errorf("owned database %q survived the run", db) + } + } + var leftoverDBs int + if err := admin.QueryRow(ctx, + `SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\_cat06\_%'`).Scan(&leftoverDBs); err != nil { + t.Fatalf("audit fixture prefix: %v", err) + } + if leftoverDBs != 0 { + t.Errorf("%d databases still match the private verify_cat06_ fixture prefix", leftoverDBs) + } + var leftoverSchemas int + if err := admin.QueryRow(ctx, + `SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'verify\_cat06\_%'`).Scan(&leftoverSchemas); err != nil { + t.Fatalf("audit fixture schemas: %v", err) + } + if leftoverSchemas != 0 { + t.Errorf("%d schemas still match the private verify_cat06_ fixture prefix", leftoverSchemas) + } +} + +func TestSemanticInventoryPageAggregationHonorsResourceTypeFilter(t *testing.T) { + s := newPgcat06Service(t) + s.seedBuild(t, pgcat06DefaultBuildSeed(pgcat06Project, pgcat06Generation)) + buildID := catalog.SemanticInventoryBuildID(pgcat06Project, pgcat06Generation) + put := func(sourceID, resourceType, binding, concept string, observation catalog.SemanticObservation) { + s.seedContribution(t, contributionSeed{ + Project: pgcat06Project, Generation: pgcat06Generation, BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: pgcat06PathGlucose, + ResourceType: resourceType, SourceID: sourceID, Ordinal: 0, + ConceptID: concept, BindingID: binding, + Observation: pgcat06ObsJSON(t, observation), + }) + } + put("rt-1", "Observation", "bnd-rt", "cpt-rt", pgcat06Observation(1)) + put("rt-2", "Encounter", "bnd-rt", "cpt-rt", pgcat06Observation(99)) + fam := func(population int64) catalog.SemanticObservation { + o := pgcat06FamilyObs("code.coding[]", "urn:rt-family", "v1", population) + return o + } + put("rt-3", "Observation", "bnd-frt", "cpt-frt-a", fam(1)) + put("rt-4", "Encounter", "bnd-frt", "cpt-frt-b", fam(99)) + + repo := s.readRepository(t) + base := catalog.SemanticInventoryPageOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(true), + ResourceType: "Observation", + } + for _, tc := range []struct { + label string + binding string + family bool + }{{"exact", "bnd-rt", false}, {"family", "bnd-frt", true}} { + filtered := base + filtered.BindingID = tc.binding + if tc.family { + filtered.SelectionMode = catalog.SemanticInventorySystemVersionCategory + } + page, err := repo.PageSemanticInventory(s.ctx, filtered) + if err != nil { + s.fail("filtered page "+tc.label, err) + } + if tc.family { + if len(page.Families) != 1 || page.Families[0].Population != 1 || len(page.Entries) != 0 { + t.Fatalf("filtered %s page = entries %+v families %+v, want one family with population 1", + tc.label, page.Entries, page.Families) + } + } else if len(page.Entries) != 1 || page.Entries[0].ConceptID != "cpt-rt" || + page.Entries[0].Observation.Population != 1 || len(page.Families) != 0 { + t.Fatalf("filtered %s page = %+v, want cpt-rt alone with population 1 (Encounter-column sibling excluded)", + tc.label, page.Entries) + } + unfiltered := filtered + unfiltered.ResourceType = "" + page, err = repo.PageSemanticInventory(s.ctx, unfiltered) + if err != nil { + s.fail("unfiltered page "+tc.label, err) + } + if tc.family { + if len(page.Families) != 1 || page.Families[0].Population != 100 { + t.Fatalf("unfiltered %s page = %+v, want one family with population 100", tc.label, page.Families) + } + } else if len(page.Entries) != 1 || page.Entries[0].Observation.Population != 100 { + t.Fatalf("unfiltered %s page = %+v, want population 100 (sibling folds in)", tc.label, page.Entries) + } + } +} + +func TestSemanticInventoryPageQueryFoldingGolden(t *testing.T) { + s := newPgcat06Service(t) + s.seedBuild(t, pgcat06DefaultBuildSeed(pgcat06Project, pgcat06Generation)) + buildID := catalog.SemanticInventoryBuildID(pgcat06Project, pgcat06Generation) + observation := catalog.SemanticObservation{ + SchemaVersion: catalog.SemanticObservationSchemaVersion, + Source: catalog.SemanticObservationSource{ + Type: "Observation", Path: "alpha beta", + }, + Key: catalog.SemanticObservationKey{System: "gamma delta", Code: "code-a"}, + Value: catalog.SemanticObservationValue{Type: "string"}, + Completeness: catalog.SemanticComplete, + Status: "SUPPORTED", + Population: 1, + } + s.seedContribution(t, contributionSeed{ + Project: pgcat06Project, Generation: pgcat06Generation, BuildID: buildID, + SourceKind: catalog.SemanticInventorySourceFile, AuthResourcePath: pgcat06PathGlucose, + ResourceType: "Observation", SourceID: "fold-1", Ordinal: 0, + ConceptID: "cpt-fold", BindingID: "bnd-fold", + Observation: pgcat06ObsJSON(t, observation), + }) + repo := s.readRepository(t) + search := func(query string) int { + t.Helper() + page, err := repo.PageSemanticInventory(s.ctx, catalog.SemanticInventoryPageOptions{ + Project: pgcat06Project, DatasetGeneration: pgcat06Generation, + AuthResourcePathsUnrestricted: pgcat06Unrestricted(true), + Query: query, + }) + if err != nil { + s.fail("search page", err) + } + return len(page.Entries) + } + if got := search("beta gamma"); got != 1 { + t.Fatalf("phrase spanning absent leaves matched %d entries, want 1", got) + } + if got := search("beta zeta"); got != 0 { + t.Fatalf("non-matching phrase matched %d entries, want 0 (negative control)", got) + } + if got := search("alpha beta gamma delta code-a"); got != 1 { + t.Fatalf("full-span phrase matched %d entries, want 1", got) + } +} + +func pgcat05EmitContribution(t *testing.T, project, generation, authPath, resourceType, sourceID, payload string) []catalog.SemanticInventoryContribution { + t.Helper() + var decoded map[string]any + if err := json.Unmarshal([]byte(payload), &decoded); err != nil { + t.Fatalf("decode golden payload: %v", err) + } + emitter := catalog.NewSemanticInventoryEmitter(project, generation) + var out []catalog.SemanticInventoryContribution + if err := emitter.ObservePayload(decoded, resourceType, authPath, sourceID, func(c catalog.SemanticInventoryContribution) { + out = append(out, c) + }); err != nil { + t.Fatalf("emit contributions: %v", err) + } + return out +} diff --git a/internal/backends/postgres/internal/catalog/store.go b/internal/backends/postgres/internal/catalog/store.go new file mode 100644 index 000000000..eb53aef38 --- /dev/null +++ b/internal/backends/postgres/internal/catalog/store.go @@ -0,0 +1,98 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/catalog" +) + +type InventoryStore interface { + ScanSourceCounts(ctx context.Context, project, datasetGeneration, resourceType string) ([]storepostgres.InventorySourceGroup, error) + InsertCounts(ctx context.Context, rows []catalog.ResourceInventoryCount, batchSize int) error + PublishBuild(ctx context.Context, build catalog.ResourceInventoryBuild) error + ReadBuild(ctx context.Context, project, datasetGeneration string) (catalog.ResourceInventoryBuild, bool, error) + ReadCountSums(ctx context.Context, scope storepostgres.InventoryCountScope) (map[string]int64, error) +} + +var _ InventoryStore = (*storepostgres.InventoryRepository)(nil) + +type FieldProfileStore interface { + UpsertFieldProfileBatch(ctx context.Context, scope storepostgres.FieldProfileScope, docs []catalog.FieldCatalogDocument, coverage storepostgres.FieldProfileCoverage, batchSize int) error + ReadFieldProfileRows(ctx context.Context, scope storepostgres.FieldProfileScope) ([]catalog.FieldCatalogDocument, error) + ReadFieldProfileRowsFiltered(ctx context.Context, filter storepostgres.FieldProfileFilter) ([]catalog.FieldCatalogDocument, error) + ReadProfileCoverage(ctx context.Context, filter storepostgres.FieldProfileFilter) ([]storepostgres.FieldProfileCoverage, error) + ListDistinctAuthResourcePaths(ctx context.Context, project string) ([]string, error) +} + +var _ FieldProfileStore = (*storepostgres.FieldProfileRepository)(nil) + +type InventoryAdapter struct { + inventory InventoryStore +} + +func NewInventoryAdapter(inventory InventoryStore) (*InventoryAdapter, error) { + if inventory == nil { + return nil, errors.New("catalog PostgreSQL inventory store is required") + } + return &InventoryAdapter{inventory: inventory}, nil +} + +type FieldProfileAdapter struct { + profiles FieldProfileStore +} + +func NewFieldProfileAdapter(profiles FieldProfileStore) (*FieldProfileAdapter, error) { + if profiles == nil { + return nil, errors.New("catalog PostgreSQL field profile store is required") + } + return &FieldProfileAdapter{profiles: profiles}, nil +} + +type RelationshipAdapter struct { + relationships RelationshipStore +} + +func NewRelationshipAdapter(relationships RelationshipStore) (*RelationshipAdapter, error) { + if relationships == nil { + return nil, errors.New("catalog PostgreSQL relationship evidence store is required") + } + return &RelationshipAdapter{relationships: relationships}, nil +} + +type Store struct { + InventoryAdapter + FieldProfileAdapter + RelationshipAdapter +} + +var ErrStoreLegNotAssembled = errors.New("postgres catalog: store leg is not assembled") + +func requireLeg(leg string, present bool, method string) error { + if !present { + return fmt.Errorf("%w: %s was called without the %s repository", ErrStoreLegNotAssembled, method, leg) + } + return nil +} + +func NewFullStore(inventory InventoryStore, profiles FieldProfileStore, relationships RelationshipStore) (*Store, error) { + inventoryAdapter, err := NewInventoryAdapter(inventory) + if err != nil { + return nil, err + } + profileAdapter, err := NewFieldProfileAdapter(profiles) + if err != nil { + return nil, err + } + relationshipAdapter, err := NewRelationshipAdapter(relationships) + if err != nil { + return nil, err + } + return &Store{ + InventoryAdapter: *inventoryAdapter, + FieldProfileAdapter: *profileAdapter, + RelationshipAdapter: *relationshipAdapter, + }, nil +} diff --git a/internal/backends/postgres/internal/chartifact/catalog.go b/internal/backends/postgres/internal/chartifact/catalog.go new file mode 100644 index 000000000..095cec2db --- /dev/null +++ b/internal/backends/postgres/internal/chartifact/catalog.go @@ -0,0 +1,397 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "github.com/calypr/loom/internal/dataframe/execution/chartifact" +) + +const PrivateArtifactsTable = "chartifact_private_artifacts" + +const ( + cleanupPendingState = string(chartifact.StateCleanupPending) + maxExpiredPageSize = 1000 +) + +var ErrInvalidSchema = errors.New("chartifact postgres: invalid schema name") + +var ErrArtifactExists = errors.New("chartifact postgres: artifact manifest already exists") + +type DB interface { + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} + +type Catalog struct { + db DB + artifacts string +} + +var _ chartifact.Catalog = (*Catalog)(nil) + +func NewCatalog(pool DB, schema string) (*Catalog, error) { + if pool == nil { + return nil, errors.New("chartifact postgres: nil pool") + } + if !storepostgres.ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", ErrInvalidSchema, schema) + } + return &Catalog{ + db: pool, + artifacts: pgx.Identifier{schema, PrivateArtifactsTable}.Sanitize(), + }, nil +} + +func manifestParams(manifest chartifact.Manifest) ([]any, error) { + payload, err := json.Marshal(manifest) + if err != nil { + return nil, fmt.Errorf("chartifact postgres: marshal artifact manifest: %w", err) + } + return []any{ + manifest.ArtifactID, manifest.PhysicalTable, string(manifest.State), + manifest.RowCount, manifest.ByteCount, manifest.LeaseOwner, + manifest.LeaseUntil, manifest.CreatedAt, manifest.UpdatedAt, + manifest.Identity.ExecutionID, manifest.Identity.OutputID, manifest.Identity.StageID, + manifest.Identity.Project, manifest.Identity.DatasetGeneration, + string(manifest.Identity.AuthScopeMode), manifest.Identity.ScopeMode, + manifest.Identity.SchemaDigest, manifest.Identity.ScopeDigest, + string(payload), + }, nil +} + +func (c *Catalog) Create(ctx context.Context, manifest chartifact.Manifest) error { + if err := requireContext(ctx); err != nil { + return err + } + if err := validateCreate(manifest); err != nil { + return fmt.Errorf("chartifact postgres: create manifest %q: %w", manifest.ArtifactID, err) + } + params, err := manifestParams(manifest) + if err != nil { + return err + } + tag, err := c.db.Exec(ctx, + "INSERT INTO "+c.artifacts+" (artifact_id, physical_table, state, row_count, byte_count,"+ + " lease_owner, lease_until, created_at, updated_at, execution_id, output_id, stage_id,"+ + " project, dataset_generation, auth_scope_mode, scope_mode, schema_digest, scope_digest, payload)"+ + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)"+ + " ON CONFLICT (artifact_id) DO NOTHING", + params...) + if err != nil { + return fmt.Errorf("chartifact postgres: insert artifact manifest %q: %w", manifest.ArtifactID, err) + } + if tag.RowsAffected() == 0 { + return fmt.Errorf("%w: artifact manifest %q", ErrArtifactExists, manifest.ArtifactID) + } + return nil +} + +func (c *Catalog) Update(ctx context.Context, artifactID, owner string, progress chartifact.Progress) error { + if err := requireContext(ctx); err != nil { + return err + } + if err := validateUpdate(artifactID, owner, progress); err != nil { + return err + } + now := time.Now().UTC() + leaseJSON, err := jsonTime(progress.LeaseUntil) + if err != nil { + return fmt.Errorf("chartifact postgres: update artifact %q: %w", artifactID, err) + } + updatedJSON, err := jsonTime(progress.UpdatedAt) + if err != nil { + return fmt.Errorf("chartifact postgres: update artifact %q: %w", artifactID, err) + } + cleanupPending := progress.State == chartifact.StateCleanupPending + tag, err := c.db.Exec(ctx, + "UPDATE "+c.artifacts+" AS m SET"+ + " state = $3::text, row_count = $4, byte_count = $5,"+ + " lease_until = CASE WHEN $6::boolean THEN $7::timestamptz"+ + " WHEN m.lease_until > $7::timestamptz THEN m.lease_until ELSE $7::timestamptz END,"+ + " updated_at = $9::timestamptz,"+ + " payload = m.payload || jsonb_build_object("+ + " 'state', $3::text, 'rowCount', $4::bigint, 'byteCount', $5::bigint,"+ + " 'leaseUntil', CASE WHEN $6::boolean THEN $8::jsonb"+ + " WHEN m.lease_until > $7::timestamptz THEN m.payload -> 'leaseUntil' ELSE $8::jsonb END,"+ + " 'updatedAt', $10::jsonb)"+ + " WHERE m.artifact_id = $1 AND m.lease_owner = $2"+ + " AND m.lease_until >= $11::timestamptz"+ + " AND (m.state <> '"+cleanupPendingState+"' OR $3::text = '"+cleanupPendingState+"')", + artifactID, owner, string(progress.State), progress.RowCount, progress.ByteCount, + cleanupPending, progress.LeaseUntil, leaseJSON, progress.UpdatedAt, updatedJSON, now) + if err != nil { + return fmt.Errorf("chartifact postgres: update artifact manifest %q: %w", artifactID, err) + } + if tag.RowsAffected() == 0 { + return chartifact.ErrLeaseLost + } + return nil +} + +func (c *Catalog) Renew(ctx context.Context, artifactID, owner string, until time.Time) (bool, error) { + if err := requireContext(ctx); err != nil { + return false, err + } + if strings.TrimSpace(artifactID) == "" || strings.TrimSpace(owner) == "" || until.IsZero() { + return false, fmt.Errorf("chartifact postgres: renewal requires artifact ID, owner, and lease time") + } + now := time.Now().UTC() + untilJSON, err := jsonTime(until) + if err != nil { + return false, fmt.Errorf("chartifact postgres: renew artifact %q: %w", artifactID, err) + } + nowJSON, err := jsonTime(now) + if err != nil { + return false, fmt.Errorf("chartifact postgres: renew artifact %q: %w", artifactID, err) + } + tag, err := c.db.Exec(ctx, + "UPDATE "+c.artifacts+" AS m SET"+ + " lease_until = CASE WHEN m.lease_until > $3::timestamptz THEN m.lease_until ELSE $3::timestamptz END,"+ + " updated_at = $5::timestamptz,"+ + " payload = m.payload || jsonb_build_object("+ + " 'leaseUntil', CASE WHEN m.lease_until > $3::timestamptz THEN m.payload -> 'leaseUntil' ELSE $4::jsonb END,"+ + " 'updatedAt', $6::jsonb)"+ + " WHERE m.artifact_id = $1 AND m.lease_owner = $2"+ + " AND m.state IN ('CREATING', 'WRITING', 'READY')"+ + " AND m.lease_until >= $7::timestamptz", + artifactID, owner, until, untilJSON, now, nowJSON, now) + if err != nil { + return false, fmt.Errorf("chartifact postgres: renew artifact lease %q: %w", artifactID, err) + } + return tag.RowsAffected() == 1, nil +} + +func (c *Catalog) ListExpired(ctx context.Context, expiredBefore time.Time, limit int) ([]chartifact.Manifest, error) { + if err := requireContext(ctx); err != nil { + return nil, err + } + if expiredBefore.IsZero() { + return nil, fmt.Errorf("chartifact postgres: artifact expiry cutoff is required") + } + if limit <= 0 { + return []chartifact.Manifest{}, nil + } + if limit > maxExpiredPageSize { + limit = maxExpiredPageSize + } + rows, err := c.db.Query(ctx, + "SELECT payload FROM "+c.artifacts+ + " WHERE lease_until <= $1::timestamptz"+ + " ORDER BY lease_until ASC, artifact_id COLLATE \"C\" ASC LIMIT $2", + expiredBefore, limit) + if err != nil { + return nil, fmt.Errorf("chartifact postgres: list expired artifacts: %w", err) + } + defer rows.Close() + manifests := make([]chartifact.Manifest, 0, limit) + for rows.Next() { + var payload []byte + if err := rows.Scan(&payload); err != nil { + return nil, fmt.Errorf("chartifact postgres: read expired artifact row: %w", err) + } + manifest, err := decodeManifest(payload) + if err != nil { + return nil, err + } + manifests = append(manifests, manifest) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("chartifact postgres: list expired artifacts: %w", err) + } + return manifests, nil +} + +func (c *Catalog) ClaimCleanup(ctx context.Context, artifactID, owner string, expiredBefore, until time.Time) (chartifact.Manifest, bool, error) { + if err := requireContext(ctx); err != nil { + return chartifact.Manifest{}, false, err + } + if strings.TrimSpace(artifactID) == "" || strings.TrimSpace(owner) == "" || expiredBefore.IsZero() || !until.After(expiredBefore) { + return chartifact.Manifest{}, false, fmt.Errorf("chartifact postgres: cleanup claim requires ID, owner, and a future cleanup lease") + } + untilJSON, err := jsonTime(until) + if err != nil { + return chartifact.Manifest{}, false, fmt.Errorf("chartifact postgres: claim artifact %q cleanup: %w", artifactID, err) + } + now := time.Now().UTC() + nowJSON, err := jsonTime(now) + if err != nil { + return chartifact.Manifest{}, false, fmt.Errorf("chartifact postgres: claim artifact %q cleanup: %w", artifactID, err) + } + var payload []byte + err = c.db.QueryRow(ctx, + "UPDATE "+c.artifacts+" AS m SET"+ + " state = '"+cleanupPendingState+"', lease_owner = $3::text,"+ + " lease_until = $4::timestamptz, updated_at = $6::timestamptz,"+ + " payload = m.payload || jsonb_build_object("+ + " 'state', '"+cleanupPendingState+"', 'leaseOwner', $3::text,"+ + " 'leaseUntil', $5::jsonb, 'updatedAt', $7::jsonb)"+ + " WHERE m.artifact_id = $1 AND m.lease_until <= $2::timestamptz"+ + " RETURNING payload", + artifactID, expiredBefore, owner, until, untilJSON, now, nowJSON).Scan(&payload) + if errors.Is(err, pgx.ErrNoRows) { + return chartifact.Manifest{}, false, nil + } + if err != nil { + return chartifact.Manifest{}, false, fmt.Errorf("chartifact postgres: claim artifact %q cleanup: %w", artifactID, err) + } + manifest, err := decodeManifest(payload) + if err != nil { + return chartifact.Manifest{}, false, err + } + return manifest, true, nil +} + +func (c *Catalog) Delete(ctx context.Context, artifactID, owner string) error { + if err := requireContext(ctx); err != nil { + return err + } + if strings.TrimSpace(artifactID) == "" || strings.TrimSpace(owner) == "" { + return fmt.Errorf("chartifact postgres: deletion requires artifact ID and owner") + } + tag, err := c.db.Exec(ctx, + "DELETE FROM "+c.artifacts+ + " WHERE artifact_id = $1 AND lease_owner = $2 AND state = '"+cleanupPendingState+"'", + artifactID, owner) + if err != nil { + return fmt.Errorf("chartifact postgres: delete artifact manifest %q: %w", artifactID, err) + } + if tag.RowsAffected() == 0 { + return chartifact.ErrLeaseLost + } + return nil +} + +func (c *Catalog) ReleaseCleanup(ctx context.Context, artifactID, owner string, until time.Time) error { + if err := requireContext(ctx); err != nil { + return err + } + if strings.TrimSpace(artifactID) == "" || strings.TrimSpace(owner) == "" || until.IsZero() { + return fmt.Errorf("chartifact postgres: cleanup release requires artifact ID, owner, and lease time") + } + now := time.Now().UTC() + untilJSON, err := jsonTime(until) + if err != nil { + return fmt.Errorf("chartifact postgres: release artifact %q cleanup: %w", artifactID, err) + } + nowJSON, err := jsonTime(now) + if err != nil { + return fmt.Errorf("chartifact postgres: release artifact %q cleanup: %w", artifactID, err) + } + tag, err := c.db.Exec(ctx, + "UPDATE "+c.artifacts+" AS m SET"+ + " lease_until = $3::timestamptz, updated_at = $5::timestamptz,"+ + " payload = m.payload || jsonb_build_object('leaseUntil', $4::jsonb, 'updatedAt', $6::jsonb)"+ + " WHERE m.artifact_id = $1 AND m.lease_owner = $2 AND m.state = '"+cleanupPendingState+"'", + artifactID, owner, until, untilJSON, now, nowJSON) + if err != nil { + return fmt.Errorf("chartifact postgres: release artifact cleanup lease %q: %w", artifactID, err) + } + if tag.RowsAffected() == 0 { + return chartifact.ErrLeaseLost + } + return nil +} + +func jsonTime(value time.Time) (string, error) { + encoded, err := json.Marshal(value.UTC()) + if err != nil { + return "", fmt.Errorf("encode artifact timestamp: %w", err) + } + return string(encoded), nil +} + +func decodeManifest(payload []byte) (chartifact.Manifest, error) { + var manifest chartifact.Manifest + if err := json.Unmarshal(payload, &manifest); err != nil { + return chartifact.Manifest{}, fmt.Errorf("chartifact postgres: decode stored artifact manifest: %w", err) + } + if strings.TrimSpace(manifest.ArtifactID) == "" { + return chartifact.Manifest{}, errors.New("chartifact postgres: stored artifact manifest has no artifact ID") + } + if manifest.LeaseUntil.IsZero() || manifest.CreatedAt.IsZero() || manifest.UpdatedAt.IsZero() { + return chartifact.Manifest{}, fmt.Errorf("chartifact postgres: stored artifact manifest %q has a nullish timestamp member", manifest.ArtifactID) + } + switch manifest.State { + case chartifact.StateCreating, chartifact.StateWriting, chartifact.StateReady, chartifact.StateCleanupPending: + default: + return chartifact.Manifest{}, fmt.Errorf("chartifact postgres: stored artifact manifest %q has invalid state %q", manifest.ArtifactID, manifest.State) + } + return manifest, nil +} + +func validateCreate(manifest chartifact.Manifest) error { + if strings.TrimSpace(manifest.ArtifactID) == "" || manifest.ArtifactID != strings.TrimSpace(manifest.ArtifactID) { + return fmt.Errorf("manifest requires a trimmed artifact ID") + } + if strings.TrimSpace(manifest.PhysicalTable) == "" || manifest.PhysicalTable != strings.TrimSpace(manifest.PhysicalTable) { + return fmt.Errorf("manifest requires a trimmed physical table name") + } + if strings.TrimSpace(manifest.LeaseOwner) == "" || manifest.LeaseOwner != strings.TrimSpace(manifest.LeaseOwner) { + return fmt.Errorf("manifest requires a trimmed lease owner") + } + if err := validateProgressFields(chartifact.Progress{ + State: manifest.State, RowCount: manifest.RowCount, ByteCount: manifest.ByteCount, + }); err != nil { + return err + } + if manifest.LeaseUntil.IsZero() || manifest.CreatedAt.IsZero() || manifest.UpdatedAt.IsZero() { + return fmt.Errorf("manifest requires lease, created, and updated timestamps") + } + for _, field := range []struct { + name string + value string + }{ + {"execution ID", manifest.Identity.ExecutionID}, + {"output ID", manifest.Identity.OutputID}, + {"stage ID", manifest.Identity.StageID}, + {"project", manifest.Identity.Project}, + {"dataset generation", manifest.Identity.DatasetGeneration}, + } { + if field.value == "" { + return fmt.Errorf("manifest requires a %s", field.name) + } + } + return nil +} + +func validateUpdate(artifactID, owner string, progress chartifact.Progress) error { + if strings.TrimSpace(artifactID) == "" || strings.TrimSpace(owner) == "" { + return fmt.Errorf("chartifact postgres: update requires artifact ID and owner") + } + if err := validateProgressFields(progress); err != nil { + return err + } + if progress.LeaseUntil.IsZero() || progress.UpdatedAt.IsZero() { + return fmt.Errorf("chartifact postgres: update requires lease timestamps") + } + return nil +} + +func validateProgressFields(progress chartifact.Progress) error { + switch progress.State { + case chartifact.StateCreating, chartifact.StateWriting, chartifact.StateReady, chartifact.StateCleanupPending: + default: + return fmt.Errorf("invalid state %q", progress.State) + } + if progress.RowCount < 0 || progress.ByteCount < 0 { + return fmt.Errorf("counts must be nonnegative, got rows %d bytes %d", progress.RowCount, progress.ByteCount) + } + return nil +} + +func requireContext(ctx context.Context) error { + if ctx == nil { + return fmt.Errorf("chartifact postgres operation requires a context") + } + return ctx.Err() +} diff --git a/internal/backends/postgres/internal/chartifact/catalog_manager_test.go b/internal/backends/postgres/internal/chartifact/catalog_manager_test.go new file mode 100644 index 000000000..cd7cb4982 --- /dev/null +++ b/internal/backends/postgres/internal/chartifact/catalog_manager_test.go @@ -0,0 +1,271 @@ +package postgres + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/execution/chartifact" + "github.com/calypr/loom/internal/store/clickhouse" +) + +type recordingClickHouse struct { + mu sync.Mutex + creates []string + inserts int + verified []string + drops []string + dropFailures map[string]int +} + +func (f *recordingClickHouse) CreateTable(_ context.Context, table string, _ []clickhouse.Column) error { + f.mu.Lock() + defer f.mu.Unlock() + f.creates = append(f.creates, table) + return nil +} + +func (f *recordingClickHouse) InsertRows(_ context.Context, _ string, _ []clickhouse.Column, rows []map[string]any) error { + f.mu.Lock() + defer f.mu.Unlock() + f.inserts += len(rows) + return nil +} + +func (f *recordingClickHouse) VerifyOutput(_ context.Context, table string, _ []clickhouse.Column, _ int64) error { + f.mu.Lock() + defer f.mu.Unlock() + f.verified = append(f.verified, table) + return nil +} + +func (f *recordingClickHouse) DropTable(_ context.Context, table string) error { + f.mu.Lock() + defer f.mu.Unlock() + if remaining := f.dropFailures[table]; remaining > 0 { + f.dropFailures[table] = remaining - 1 + return errors.New("fake ClickHouse: injected drop failure") + } + f.drops = append(f.drops, table) + return nil +} + +func (f *recordingClickHouse) failDrops(table string, times int) { + f.mu.Lock() + defer f.mu.Unlock() + if f.dropFailures == nil { + f.dropFailures = map[string]int{} + } + f.dropFailures[table] = times +} + +func (f *recordingClickHouse) dropCount(table string) int { + f.mu.Lock() + defer f.mu.Unlock() + count := 0 + for _, dropped := range f.drops { + if dropped == table { + count++ + } + } + return count +} + +func (f *recordingClickHouse) totalDrops() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.drops) +} + +func managerIdentity() chartifact.Identity { + return chartifact.Identity{ + ExecutionID: "execution-mgr", OutputID: "output-mgr", StageID: "stage-mgr", + Project: "project-mgr", DatasetGeneration: "generation-mgr", + RecipeDigest: "recipe-digest", PlanDigest: "plan-digest", + AuthScopeMode: authscope.ReadScopeUnrestricted, + ScopeMode: chartifact.ScopeModeRows, + } +} + +func managerColumns(t *testing.T) []chartifact.Column { + t.Helper() + columns, err := chartifact.NormalizeSchema([]chartifact.Column{ + {ID: "col-name", Name: "name", SemanticPath: "Patient.name", LogicalType: "string", ClickHouseType: "String"}, + }) + if err != nil { + t.Fatalf("NormalizeSchema = %v", err) + } + return columns +} + +func newManager(t *testing.T, s *catalogService, ch *recordingClickHouse) *chartifact.Manager { + t.Helper() + manager, err := chartifact.New(chartifact.Config{ + Catalog: s.catalog, ClickHouse: ch, + BatchRows: 1000, BatchBytes: 1 << 20, LeaseTTL: 2 * time.Minute, + }) + if err != nil { + t.Fatalf("chartifact.New = %v", err) + } + return manager +} + +func writeOneRow(t *testing.T, writer *chartifact.Writer) { + t.Helper() + if err := writer.Write(context.Background(), map[string]any{ + "__loom_row_id": "row-1", "name": "Ada", + }); err != nil { + t.Fatalf("Write = %v", err) + } +} + +func TestManagerLifecycleThroughPostgresCatalogDropsOnceOutsideTransactions(t *testing.T) { + s := newCatalogService(t) + defer s.guarded.check(t) + ch := &recordingClickHouse{} + manager := newManager(t, s, ch) + + writer, err := manager.Begin(context.Background(), managerIdentity(), managerColumns(t)) + if err != nil { + t.Fatalf("Begin = %v", err) + } + created := s.witnessSingleRow(t) + if created.State != "WRITING" { + t.Fatalf("post-Begin row = %+v, want WRITING", created) + } + table, artifactID := created.PhysicalTable, created.ArtifactID + writeOneRow(t, writer) + artifact, err := writer.Finalize(context.Background()) + if err != nil { + t.Fatalf("Finalize = %v", err) + } + if artifact.Manifest().ArtifactID != artifactID || artifact.Manifest().PhysicalTable != table { + t.Fatalf("finalized artifact %q/%q does not match the stored row %q/%q", + artifact.Manifest().ArtifactID, artifact.Manifest().PhysicalTable, artifactID, table) + } + if row := s.witnessRow(t, artifactID); row.State != "READY" || row.RowCount != 1 || row.ByteCount <= 0 { + t.Fatalf("post-Finalize row = %+v, want READY with one row", row) + } + if ch.inserts != 1 { + t.Fatalf("ClickHouse inserts = %d, want 1", ch.inserts) + } + if len(ch.verified) != 1 || ch.verified[0] != table { + t.Fatalf("VerifyOutput calls = %v, want [%s]", ch.verified, table) + } + + if err := artifact.Release(context.Background()); err != nil { + t.Fatalf("Release = %v", err) + } + if err := artifact.Release(context.Background()); err != nil { + t.Fatalf("repeat Release = %v, want idempotent", err) + } + if got := ch.dropCount(table); got != 1 { + t.Fatalf("ClickHouse drops of %s = %d, want exactly 1 across repeat Release calls", table, got) + } + s.requireNoRow(t, artifactID) +} + +func TestDropFailureLeavesReclaimableManifestForReconcile(t *testing.T) { + s := newCatalogService(t) + defer s.guarded.check(t) + failingCH := &recordingClickHouse{} + manager := newManager(t, s, failingCH) + + writer, err := manager.Begin(context.Background(), managerIdentity(), managerColumns(t)) + if err != nil { + t.Fatalf("Begin = %v", err) + } + created := s.witnessSingleRow(t) + table, artifactID := created.PhysicalTable, created.ArtifactID + failingCH.failDrops(table, 1) + if err := writer.Abort(context.Background()); err == nil { + t.Fatal("Abort with an injected drop failure reported success") + } + row := s.witnessRow(t, artifactID) + if row.State != "CLEANUP_PENDING" || row.PhysicalTable != table { + t.Fatalf("failed drop did not leave a cleanup-pending manifest: %+v", row) + } + if _, ok, err := s.catalog.ClaimCleanup(s.ctx, artifactID, "cleanup-early", + time.Now().UTC().Add(time.Hour), time.Now().UTC().Add(2*time.Hour)); err != nil || !ok { + t.Fatalf("reconciler could not claim the failed-drop manifest = (%t, %v)", ok, err) + } + if err := s.catalog.Delete(s.ctx, artifactID, "stage-not-owner"); !errors.Is(err, chartifact.ErrLeaseLost) { + t.Fatalf("non-owner Delete = %v, want ErrLeaseLost", err) + } + if err := s.catalog.Delete(s.ctx, artifactID, "cleanup-early"); err != nil { + t.Fatalf("cleanup-owner Delete = %v", err) + } + s.requireNoRow(t, artifactID) + if failingCH.dropCount(table) != 0 { + t.Fatalf("failed drop was recorded as successful: %d", failingCH.dropCount(table)) + } + + reconcileCH := &recordingClickHouse{} + reconcileManager := newManager(t, s, reconcileCH) + writer, err = reconcileManager.Begin(context.Background(), managerIdentity(), managerColumns(t)) + if err != nil { + t.Fatalf("second Begin = %v", err) + } + created = s.witnessSingleRow(t) + table, artifactID = created.PhysicalTable, created.ArtifactID + if _, err := s.iso.Exec(s.ctx, "UPDATE "+s.qualified+ + " SET lease_until = $2::timestamptz WHERE artifact_id = $1", + artifactID, time.Now().UTC().Add(-time.Minute)); err != nil { + s.fail("expire fixture lease", err) + } + if err := reconcileManager.Reconcile(context.Background(), time.Now().UTC().Add(time.Minute), 10); err != nil { + t.Fatalf("Reconcile = %v", err) + } + if got := reconcileCH.dropCount(table); got != 1 { + t.Fatalf("Reconcile drops of %s = %d, want 1", table, got) + } + s.requireNoRow(t, artifactID) +} + +func TestReconcileRaceDropsEachExpiredArtifactExactlyOnce(t *testing.T) { + s := newCatalogService(t) + defer s.guarded.check(t) + now := time.Now().UTC() + var tables []string + for _, id := range []string{"race-1", "race-2"} { + manifest := sampleManifestWithLease(id, now.Add(-time.Minute)) + if err := s.catalog.Create(s.ctx, manifest); err != nil { + t.Fatalf("Create %s = %v", id, err) + } + tables = append(tables, manifest.PhysicalTable) + } + leftCH, rightCH := &recordingClickHouse{}, &recordingClickHouse{} + left, right := newManager(t, s, leftCH), newManager(t, s, rightCH) + + var wg sync.WaitGroup + start := make(chan struct{}) + reconcile := func(manager *chartifact.Manager) { + wg.Add(1) + go func() { + defer wg.Done() + <-start + if err := manager.Reconcile(context.Background(), now.Add(time.Minute), 10); err != nil { + s.fail("concurrent reconcile", err) + } + }() + } + reconcile(left) + reconcile(right) + close(start) + wg.Wait() + + for _, table := range tables { + if got := leftCH.dropCount(table) + rightCH.dropCount(table); got != 1 { + t.Fatalf("table %s dropped %d times across replicas, want exactly 1", table, got) + } + } + if leftCH.totalDrops()+rightCH.totalDrops() != len(tables) { + t.Fatalf("total drops = %d, want %d", leftCH.totalDrops()+rightCH.totalDrops(), len(tables)) + } + for _, id := range []string{"race-1", "race-2"} { + s.requireNoRow(t, id) + } +} diff --git a/internal/backends/postgres/internal/chartifact/catalog_test.go b/internal/backends/postgres/internal/chartifact/catalog_test.go new file mode 100644 index 000000000..459bcf951 --- /dev/null +++ b/internal/backends/postgres/internal/chartifact/catalog_test.go @@ -0,0 +1,1164 @@ +package postgres + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/calypr/loom/internal/authscope" + publicationpostgres "github.com/calypr/loom/internal/backends/postgres/internal/publication" + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/dataframe/execution/chartifact" +) + +var pgprivcat01ResourceNamePattern = regexp.MustCompile(`^pgprivcat01_it_[0-9]+_[0-9a-f]+$`) + +func testPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 private artifact catalog integration tests") + } + return raw +} + +func randomHex() string { + buf := make([]byte, 6) + if _, err := rand.Read(buf); err != nil { + panic(err) + } + return hex.EncodeToString(buf) +} + +func truncUS(t time.Time) time.Time { return t.Truncate(time.Microsecond) } + +func jsonTimeLiteral(t *testing.T, value time.Time) string { + t.Helper() + encoded, err := json.Marshal(value.UTC()) + if err != nil { + t.Fatalf("encode expected payload timestamp: %v", err) + } + return string(encoded) +} + +type guardedPool struct { + *pgxpool.Pool + mu sync.Mutex + violation *statementViolation +} + +type statementViolation struct { + op string + sql string +} + +func (g *guardedPool) note(op, sql string) { + head := strings.ToUpper(strings.TrimSpace(sql)) + if strings.HasPrefix(head, "BEGIN") || strings.HasPrefix(head, "START TRANSACTION") { + g.mu.Lock() + if g.violation == nil { + v := statementViolation{op: op, sql: sql} + g.violation = &v + } + g.mu.Unlock() + } +} + +func (g *guardedPool) check(t *testing.T) { + t.Helper() + g.mu.Lock() + defer g.mu.Unlock() + if g.violation != nil { + t.Fatalf("catalog opened a database transaction (%s): %s", g.violation.op, g.violation.sql) + } +} + +func (g *guardedPool) Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) { + g.note("exec", sql) + return g.Pool.Exec(ctx, sql, args...) +} + +func (g *guardedPool) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { + g.note("query", sql) + return g.Pool.Query(ctx, sql, args...) +} + +func (g *guardedPool) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row { + g.note("queryrow", sql) + return g.Pool.QueryRow(ctx, sql, args...) +} + +func (g *guardedPool) Begin(ctx context.Context) (pgx.Tx, error) { + g.note("begin", "BEGIN") + return nil, errors.New("guardedPool: catalog code must never begin a transaction") +} + +var _ DB = (*guardedPool)(nil) + +type catalogService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + isoB *pgxpool.Pool + guarded *guardedPool + schema string + qualified string + catalog *Catalog + scrub func(any) string + fail func(step string, err error) +} + +func newCatalogService(t *testing.T) *catalogService { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgprivcat01_it_%d_%s", time.Now().UnixNano(), randomHex()) + if !pgprivcat01ResourceNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if _, err := admin.Exec(cleanupCtx, + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + config, err := pgxpool.ParseConfig(dsn) + if err != nil { + fail("parse isolated database config", err) + } + config.MaxConns = 1 + isoB, err := pgxpool.NewWithConfig(ctx, config) + if err != nil { + fail("connect second pool to isolated database", err) + } + t.Cleanup(isoB.Close) + + schema := fmt.Sprintf("pgprivcat01_it_%d_%s", time.Now().UnixNano(), randomHex()) + if !pgprivcat01ResourceNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if _, err := iso.Exec(cleanupCtx, + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + s := &catalogService{ + ctx: ctx, admin: admin, iso: iso, isoB: isoB, + schema: schema, + qualified: pgx.Identifier{schema, PrivateArtifactsTable}.Sanitize(), + scrub: scrub, fail: fail, + } + s.migrate(t) + s.guarded = &guardedPool{Pool: iso} + catalog, err := NewCatalog(s.guarded, schema) + if err != nil { + fail("build catalog", err) + } + s.catalog = catalog + return s +} + +func (s *catalogService) migrate(t *testing.T) { + t.Helper() + owner, err := publicationpostgres.OwnerMigrations() + if err != nil { + s.fail("publication owner migrations", err) + } + var history []storepostgres.Migration + for _, m := range owner { + if m.Version == 1 || m.Version == 7 { + history = append(history, m) + } + } + if len(history) != 2 || history[0].Version != 1 || history[0].Name != "ledger" || + history[1].Version != 7 || history[1].Name != "private_clickhouse_artifacts" { + t.Fatalf("publication owner history subset = %+v, want exactly versions {1 ledger, 7 private_clickhouse_artifacts}", owner) + } + runner, err := storepostgres.NewMigrationRunner(s.iso, s.schema, history) + if err != nil { + s.fail("new migration runner", err) + } + result, err := runner.Migrate(s.ctx) + if err != nil { + s.fail("migrate catalog", err) + } + if result.LatestVersion != 7 { + t.Fatalf("latest migration version = %d, want 7", result.LatestVersion) + } + wantLedger := []storepostgres.LedgerRecord{ + {Version: 1, Name: "ledger", Checksum: "db70f7758495f7557b3a694149b80ed6e5d4cb74ea5aa7d59a1a2eaa46fc9cde"}, + {Version: 7, Name: "private_clickhouse_artifacts", Checksum: "e827a5a1515df903a540a76858da6ad8e5afaede01f5c0b096f9f87a9b662cfb"}, + } + if len(result.Applied) != len(wantLedger) { + t.Fatalf("applied history = %+v, want %+v", result.Applied, wantLedger) + } + for i := range wantLedger { + got := result.Applied[i] + if got.Version != wantLedger[i].Version || got.Name != wantLedger[i].Name || got.Checksum != wantLedger[i].Checksum { + t.Fatalf("ledger row %d = %+v, want %+v", i, got, wantLedger[i]) + } + } + var regclass *string + if err := s.iso.QueryRow(s.ctx, `SELECT to_regclass($1)::text`, s.schema+"."+PrivateArtifactsTable).Scan(®class); err != nil { + s.fail("witness to_regclass", err) + } + if regclass == nil || *regclass == "" { + t.Fatalf("table %s was not created by migration 7", PrivateArtifactsTable) + } +} + +type rowWitness struct { + ArtifactID string + PhysicalTable string + State string + RowCount int64 + ByteCount int64 + LeaseOwner string + LeaseUntil time.Time + CreatedAt time.Time + UpdatedAt time.Time + ExecutionID string + OutputID string + StageID string + Project string + DatasetGeneration string + AuthScopeMode string + ScopeMode string + SchemaDigest string + ScopeDigest string + Payload string +} + +func (s *catalogService) witnessRow(t *testing.T, artifactID string) rowWitness { + t.Helper() + var row rowWitness + err := s.iso.QueryRow(s.ctx, + "SELECT artifact_id, physical_table, state, row_count, byte_count, lease_owner,"+ + " lease_until, created_at, updated_at, execution_id, output_id, stage_id, project,"+ + " dataset_generation, auth_scope_mode, scope_mode, schema_digest, scope_digest, payload::text"+ + " FROM "+s.qualified+" WHERE artifact_id = $1", + artifactID).Scan( + &row.ArtifactID, &row.PhysicalTable, &row.State, &row.RowCount, &row.ByteCount, &row.LeaseOwner, + &row.LeaseUntil, &row.CreatedAt, &row.UpdatedAt, &row.ExecutionID, &row.OutputID, &row.StageID, + &row.Project, &row.DatasetGeneration, &row.AuthScopeMode, &row.ScopeMode, &row.SchemaDigest, + &row.ScopeDigest, &row.Payload) + if err != nil { + s.fail("witness row "+artifactID, err) + } + return row +} + +func (s *catalogService) requireNoRow(t *testing.T, artifactID string) { + t.Helper() + var count int + if err := s.iso.QueryRow(s.ctx, + "SELECT count(*) FROM "+s.qualified+" WHERE artifact_id = $1", artifactID).Scan(&count); err != nil { + s.fail("witness absence "+artifactID, err) + } + if count != 0 { + t.Fatalf("artifact %q row still present, want deleted", artifactID) + } +} + +func (s *catalogService) witnessSingleRow(t *testing.T) rowWitness { + t.Helper() + var total int + if err := s.iso.QueryRow(s.ctx, "SELECT count(*) FROM "+s.qualified).Scan(&total); err != nil { + s.fail("witness row count", err) + } + if total != 1 { + t.Fatalf("catalog holds %d rows, want exactly 1", total) + } + var row rowWitness + err := s.iso.QueryRow(s.ctx, + "SELECT artifact_id, physical_table, state, row_count, byte_count, lease_owner,"+ + " lease_until, created_at, updated_at, execution_id, output_id, stage_id, project,"+ + " dataset_generation, auth_scope_mode, scope_mode, schema_digest, scope_digest, payload::text"+ + " FROM "+s.qualified).Scan( + &row.ArtifactID, &row.PhysicalTable, &row.State, &row.RowCount, &row.ByteCount, &row.LeaseOwner, + &row.LeaseUntil, &row.CreatedAt, &row.UpdatedAt, &row.ExecutionID, &row.OutputID, &row.StageID, + &row.Project, &row.DatasetGeneration, &row.AuthScopeMode, &row.ScopeMode, &row.SchemaDigest, + &row.ScopeDigest, &row.Payload) + if err != nil { + s.fail("witness single row", err) + } + return row +} + +func (s *catalogService) payloadMember(t *testing.T, artifactID, member string) string { + t.Helper() + var value string + err := s.iso.QueryRow(s.ctx, + "SELECT (payload->'"+member+"')::text FROM "+s.qualified+" WHERE artifact_id = $1", + artifactID).Scan(&value) + if err != nil { + s.fail("witness payload member "+artifactID+" "+member, err) + } + return value +} + +func (s *catalogService) payloadTextField(t *testing.T, artifactID, member string) string { + t.Helper() + var value *string + err := s.iso.QueryRow(s.ctx, + "SELECT (payload -> '"+member+"')::text FROM "+s.qualified+" WHERE artifact_id = $1", + artifactID).Scan(&value) + if err != nil { + s.fail("witness payload field "+artifactID+" "+member, err) + } + if value == nil { + return "" + } + return *value +} + +func (s *catalogService) rawExec(t *testing.T, sql string, args ...any) pgconn.CommandTag { + t.Helper() + tag, err := s.iso.Exec(s.ctx, sql, args...) + if err != nil { + s.fail("raw exec", err) + } + return tag +} + +func fixedTime() time.Time { + return time.Date(2026, 11, 9, 12, 0, 0, 123456789, time.UTC) +} + +func sampleManifest(artifactID string) chartifact.Manifest { + return sampleManifestWithLease(artifactID, fixedTime().Add(10*time.Minute)) +} + +func sampleManifestWithLease(artifactID string, leaseUntil time.Time) chartifact.Manifest { + created := fixedTime() + return chartifact.Manifest{ + ArtifactID: artifactID, + Identity: chartifact.Identity{ + ExecutionID: "execution-1", OutputID: "output-1", StageID: "stage-1", + Project: "project-a", DatasetGeneration: "generation-1", + RecipeDigest: "recipe-digest", PlanDigest: "plan-digest", + SchemaDigest: "schema-digest", ScopeDigest: "scope-digest", + AuthScopeMode: authscope.ReadScopeRestricted, + AuthResourcePaths: []string{"/Programs/a"}, + ScopeMode: chartifact.ScopeModeRows, + }, + Columns: []chartifact.Column{ + {ID: "loom:row_id", Name: "__loom_row_id", SemanticPath: "loom:row_id", LogicalType: "string", ClickHouseType: "String", Internal: true}, + {ID: "col-name", Name: "name", SemanticPath: "Patient.name", LogicalType: "string", ClickHouseType: "String"}, + }, + PhysicalTable: "loom_private_stage_" + strings.ReplaceAll(artifactID, "-", ""), + State: chartifact.StateCreating, + LeaseOwner: "writer-1", + LeaseUntil: leaseUntil, + CreatedAt: created, + UpdatedAt: created, + } +} + +func idsOf(manifests []chartifact.Manifest) []string { + ids := make([]string, len(manifests)) + for i, manifest := range manifests { + ids[i] = manifest.ArtifactID + } + return ids +} + +func TestNewCatalogRejectsNilPoolAndUnsafeSchemas(t *testing.T) { + if _, err := NewCatalog(nil, "public"); err == nil { + t.Fatal("NewCatalog accepted a nil pool") + } + var pool DB = &guardedPool{} + for _, schema := range []string{"", "Public", "bad-name", "bad name", "a;b", strings.Repeat("s", 64)} { + if _, err := NewCatalog(pool, schema); !errors.Is(err, ErrInvalidSchema) { + t.Fatalf("NewCatalog(%q) = %v, want ErrInvalidSchema", schema, err) + } + } + if _, err := NewCatalog(pool, "pgprivcat01_ok"); err != nil { + t.Fatalf("NewCatalog on a safe schema = %v", err) + } +} + +func TestCatalogMethodsPropagateCancellationBeforeQuerying(t *testing.T) { + s := newCatalogService(t) + defer s.guarded.check(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := s.catalog.Create(ctx, sampleManifest("cancelled-1")); !errors.Is(err, context.Canceled) { + t.Fatalf("Create cancellation = %v, want context.Canceled", err) + } + if _, err := s.catalog.ListExpired(ctx, time.Now().UTC(), 10); !errors.Is(err, context.Canceled) { + t.Fatalf("ListExpired cancellation = %v, want context.Canceled", err) + } + s.guarded.check(t) +} + +func TestCreateStoresLiteralRowAndDuplicatePreservesOriginal(t *testing.T) { + s := newCatalogService(t) + defer s.guarded.check(t) + manifest := sampleManifest("art-create") + if err := s.catalog.Create(s.ctx, manifest); err != nil { + t.Fatalf("Create = %v", err) + } + want := rowWitness{ + ArtifactID: "art-create", PhysicalTable: manifest.PhysicalTable, State: "CREATING", + RowCount: 0, ByteCount: 0, LeaseOwner: "writer-1", + LeaseUntil: truncUS(fixedTime().Add(10 * time.Minute)), + CreatedAt: truncUS(fixedTime()), UpdatedAt: truncUS(fixedTime()), + ExecutionID: "execution-1", OutputID: "output-1", StageID: "stage-1", + Project: "project-a", DatasetGeneration: "generation-1", + AuthScopeMode: "restricted", ScopeMode: "ROW_SCOPED", + SchemaDigest: "schema-digest", ScopeDigest: "scope-digest", + } + got := s.witnessRow(t, "art-create") + got.LeaseUntil, got.CreatedAt, got.UpdatedAt = got.LeaseUntil.UTC(), got.CreatedAt.UTC(), got.UpdatedAt.UTC() + if got.ArtifactID != want.ArtifactID || got.PhysicalTable != want.PhysicalTable || got.State != want.State || + got.RowCount != want.RowCount || got.ByteCount != want.ByteCount || got.LeaseOwner != want.LeaseOwner || + !got.LeaseUntil.Equal(want.LeaseUntil) || !got.CreatedAt.Equal(want.CreatedAt) || !got.UpdatedAt.Equal(want.UpdatedAt) || + got.ExecutionID != want.ExecutionID || got.OutputID != want.OutputID || got.StageID != want.StageID || + got.Project != want.Project || got.DatasetGeneration != want.DatasetGeneration || + got.AuthScopeMode != want.AuthScopeMode || got.ScopeMode != want.ScopeMode || + got.SchemaDigest != want.SchemaDigest || got.ScopeDigest != want.ScopeDigest { + t.Fatalf("literal row mismatch:\n got %+v\nwant %+v", got, want) + } + if payloadTime := s.payloadTextField(t, "art-create", "createdAt"); payloadTime != `"2026-11-09T12:00:00.123456789Z"` { + t.Fatalf("payload createdAt = %s, want the exact nanosecond JSON string", payloadTime) + } + + before := s.witnessRow(t, "art-create") + rogue := sampleManifest("art-create") + rogue.PhysicalTable = "loom_private_stage_rogue" + rogue.Identity.Project = "project-b" + rogue.LeaseOwner = "writer-rogue" + if err := s.catalog.Create(s.ctx, rogue); !errors.Is(err, ErrArtifactExists) { + t.Fatalf("duplicate Create = %v, want ErrArtifactExists", err) + } + after := s.witnessRow(t, "art-create") + if before.Payload != after.Payload || before.PhysicalTable != after.PhysicalTable || + before.Project != after.Project || before.LeaseOwner != after.LeaseOwner { + t.Fatalf("duplicate Create mutated the stored manifest:\n before %+v\n after %+v", before, after) + } + + if err := s.catalog.Create(s.ctx, sampleManifest(" spaced")); err == nil { + t.Fatal("Create accepted an untrimmed artifact ID") + } + bad := sampleManifest("art-bad-state") + bad.State = "SOMETHING_ELSE" + if err := s.catalog.Create(s.ctx, bad); err == nil || !strings.Contains(err.Error(), "invalid state") { + t.Fatalf("Create with invalid state = %v, want explicit refusal", err) + } + negative := sampleManifest("art-negative") + negative.RowCount = -1 + if err := s.catalog.Create(s.ctx, negative); err == nil || !strings.Contains(err.Error(), "nonnegative") { + t.Fatalf("Create with negative count = %v, want explicit refusal", err) + } +} + +func TestCreateRefusesRowsItCouldNeverReadBack(t *testing.T) { + s := newCatalogService(t) + defer s.guarded.check(t) + cases := []struct { + name string + mutate func(*chartifact.Manifest) + }{ + {"zero lease until", func(m *chartifact.Manifest) { m.LeaseUntil = time.Time{} }}, + {"zero created at", func(m *chartifact.Manifest) { m.CreatedAt = time.Time{} }}, + {"zero updated at", func(m *chartifact.Manifest) { m.UpdatedAt = time.Time{} }}, + {"blank physical table", func(m *chartifact.Manifest) { m.PhysicalTable = "" }}, + {"whitespace physical table", func(m *chartifact.Manifest) { m.PhysicalTable = " " }}, + {"untrimmed physical table", func(m *chartifact.Manifest) { m.PhysicalTable = " loom_private_stage_x " }}, + {"blank lease owner", func(m *chartifact.Manifest) { m.LeaseOwner = "" }}, + {"whitespace lease owner", func(m *chartifact.Manifest) { m.LeaseOwner = " " }}, + {"untrimmed lease owner", func(m *chartifact.Manifest) { m.LeaseOwner = " writer-1 " }}, + {"blank execution ID", func(m *chartifact.Manifest) { m.Identity.ExecutionID = "" }}, + {"blank output ID", func(m *chartifact.Manifest) { m.Identity.OutputID = "" }}, + {"blank stage ID", func(m *chartifact.Manifest) { m.Identity.StageID = "" }}, + {"blank project", func(m *chartifact.Manifest) { m.Identity.Project = "" }}, + {"blank dataset generation", func(m *chartifact.Manifest) { m.Identity.DatasetGeneration = "" }}, + } + for index, c := range cases { + artifactID := fmt.Sprintf("art-gate-%d", index) + manifest := sampleManifest(artifactID) + c.mutate(&manifest) + err := s.catalog.Create(s.ctx, manifest) + if err == nil { + t.Fatalf("%s: Create = nil, want a pre-write refusal", c.name) + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + t.Fatalf("%s: Create reached the server with SQLSTATE %s, want a catalog validation refusal", c.name, pgErr.Code) + } + if !strings.Contains(err.Error(), "chartifact postgres: create manifest") { + t.Fatalf("%s: Create = %v, want a catalog validation error", c.name, s.scrub(err)) + } + s.requireNoRow(t, artifactID) + } +} + +func TestCreatePreservesMissingVersusNullVersusEmptyAndFailsOnNullishPayload(t *testing.T) { + s := newCatalogService(t) + defer s.guarded.check(t) + + nilPaths := sampleManifest("scope-nil") + nilPaths.Identity.AuthScopeMode = authscope.ReadScopeUnrestricted + nilPaths.Identity.AuthResourcePaths = nil + emptyPaths := sampleManifest("scope-empty") + emptyPaths.Identity.AuthScopeMode = authscope.ReadScopeUnrestricted + emptyPaths.Identity.AuthResourcePaths = []string{} + for _, manifest := range []chartifact.Manifest{nilPaths, emptyPaths} { + if err := s.catalog.Create(s.ctx, manifest); err != nil { + t.Fatalf("Create %s = %v", manifest.ArtifactID, err) + } + } + if typeof := s.payloadTextField(t, "scope-nil", "identity"); typeof == "" { + t.Fatal("identity member absent") + } + var storedType string + if err := s.iso.QueryRow(s.ctx, + "SELECT jsonb_typeof(payload #> '{identity,AuthResourcePaths}') FROM "+s.qualified+" WHERE artifact_id = $1", + "scope-nil").Scan(&storedType); err != nil { + s.fail("witness nil paths type", err) + } + if storedType != "null" { + t.Fatalf("nil AuthResourcePaths stored as %s, want JSON null", storedType) + } + if err := s.iso.QueryRow(s.ctx, + "SELECT jsonb_typeof(payload #> '{identity,AuthResourcePaths}') FROM "+s.qualified+" WHERE artifact_id = $1", + "scope-empty").Scan(&storedType); err != nil { + s.fail("witness empty paths type", err) + } + if storedType != "array" { + t.Fatalf("empty AuthResourcePaths stored as %s, want JSON array", storedType) + } + got, err := s.catalog.ListExpired(s.ctx, fixedTime().Add(24*time.Hour), 10) + if err != nil { + t.Fatalf("ListExpired = %v", err) + } + byID := map[string]chartifact.Manifest{} + for _, manifest := range got { + byID[manifest.ArtifactID] = manifest + } + nilRead, ok := byID["scope-nil"] + if !ok || nilRead.Identity.AuthResourcePaths != nil { + t.Fatalf("nil paths did not round-trip as nil: %#v", nilRead.Identity.AuthResourcePaths) + } + emptyRead, ok := byID["scope-empty"] + if !ok || emptyRead.Identity.AuthResourcePaths == nil || len(emptyRead.Identity.AuthResourcePaths) != 0 { + t.Fatalf("empty paths did not round-trip as empty non-nil: %#v", emptyRead.Identity.AuthResourcePaths) + } + if !reflect.DeepEqual(nilRead.Columns, nilPaths.Columns) { + t.Fatalf("columns did not round-trip: got %#v want %#v", nilRead.Columns, nilPaths.Columns) + } + if !reflect.DeepEqual(nilRead.Identity, nilPaths.Identity) { + t.Fatalf("identity did not round-trip: got %#v want %#v", nilRead.Identity, nilPaths.Identity) + } + if !nilRead.CreatedAt.Equal(nilPaths.CreatedAt) || !nilRead.LeaseUntil.Equal(nilPaths.LeaseUntil) { + t.Fatalf("nanosecond stamps did not round-trip: created %v lease %v", nilRead.CreatedAt, nilRead.LeaseUntil) + } + + rawTime := fixedTime() + if _, err := s.iso.Exec(s.ctx, + "INSERT INTO "+s.qualified+" (artifact_id, physical_table, state, row_count, byte_count,"+ + " lease_owner, lease_until, created_at, updated_at, execution_id, output_id, stage_id,"+ + " project, dataset_generation, auth_scope_mode, scope_mode, schema_digest, scope_digest, payload)"+ + " VALUES ($1,$2,'READY',0,0,'writer-null', $3, $3, $3,'e','o','s','p','g','unrestricted','','','',"+ + " jsonb_build_object('artifactId','payload-null-lease','leaseUntil', null))", + "payload-null-lease", "loom_private_stage_nulllease", rawTime); err != nil { + s.fail("insert nullish payload fixture", err) + } + if _, err := s.catalog.ListExpired(s.ctx, rawTime.Add(time.Hour), 50); err == nil || + !strings.Contains(err.Error(), "nullish timestamp member") { + t.Fatalf("nullish payload ListExpired = %v, want an explicit refusal", err) + } + + cases := []struct { + name string + sql string + payload string + wantState string + }{ + {"state outside the closed set", `VALUES ($1,'t','BOGUS',0,0,'w',$2,$2,$2,'e','o','s','p','g','unrestricted','','','',$3)`, `{"artifactId":"x"}`, "23514"}, + {"negative row count", `VALUES ($1,'t','READY',-1,0,'w',$2,$2,$2,'e','o','s','p','g','unrestricted','','','',$3)`, `{"artifactId":"x"}`, "23514"}, + {"empty lease owner", `VALUES ($1,'t','READY',0,0,'',$2,$2,$2,'e','o','s','p','g','unrestricted','','','',$3)`, `{"artifactId":"x"}`, "23514"}, + {"non-object payload", `VALUES ($1,'t','READY',0,0,'w',$2,$2,$2,'e','o','s','p','g','unrestricted','','','',$3)`, `[]`, "23514"}, + {"SQL NULL scan column", `VALUES ($1,'t','READY',0,0,'w',$2,$2,$2,'e','o',NULL,'p','g','unrestricted','','','',$3)`, `{"artifactId":"x"}`, "23502"}, + } + for index, c := range cases { + _, err := s.iso.Exec(s.ctx, + "INSERT INTO "+s.qualified+" (artifact_id, physical_table, state, row_count, byte_count,"+ + " lease_owner, lease_until, created_at, updated_at, execution_id, output_id, stage_id,"+ + " project, dataset_generation, auth_scope_mode, scope_mode, schema_digest, scope_digest, payload)"+ + " "+c.sql, + fmt.Sprintf("reject-%d", index), rawTime, c.payload) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != c.wantState { + t.Fatalf("%s: err = %v, want SQLSTATE %s", c.name, s.scrub(err), c.wantState) + } + } +} + +func TestUpdateRequiresLeaseOwnerAndPreservesIdentity(t *testing.T) { + s := newCatalogService(t) + defer s.guarded.check(t) + now := time.Now().UTC() + manifest := sampleManifestWithLease("art-update", now.Add(10*time.Minute)) + if err := s.catalog.Create(s.ctx, manifest); err != nil { + t.Fatalf("Create = %v", err) + } + beforeMembers := map[string]string{} + for _, member := range []string{"identity", "columns", "physicalTable", "artifactId", "createdAt"} { + beforeMembers[member] = s.payloadMember(t, "art-update", member) + } + + if err := s.catalog.Update(s.ctx, "art-update", "writer-2", chartifact.Progress{ + State: chartifact.StateWriting, RowCount: 5, ByteCount: 4096, + LeaseUntil: now.Add(20 * time.Minute), UpdatedAt: now.Add(time.Second), + }); !errors.Is(err, chartifact.ErrLeaseLost) { + t.Fatalf("wrong-owner Update = %v, want ErrLeaseLost", err) + } + if err := s.catalog.Update(s.ctx, "missing-artifact", "writer-1", chartifact.Progress{ + State: chartifact.StateWriting, RowCount: 5, ByteCount: 4096, + LeaseUntil: now.Add(20 * time.Minute), UpdatedAt: now.Add(time.Second), + }); !errors.Is(err, chartifact.ErrLeaseLost) { + t.Fatalf("missing-row Update = %v, want ErrLeaseLost", err) + } + if err := s.catalog.Update(s.ctx, "art-update", "writer-1", chartifact.Progress{ + State: "NOT_A_STATE", RowCount: 5, ByteCount: 4096, + LeaseUntil: now.Add(20 * time.Minute), UpdatedAt: now.Add(time.Second), + }); err == nil || !strings.Contains(err.Error(), "invalid state") { + t.Fatalf("invalid-state Update = %v, want explicit refusal", err) + } + if err := s.catalog.Update(s.ctx, "art-update", "writer-1", chartifact.Progress{ + State: chartifact.StateWriting, RowCount: 5, ByteCount: 4096, LeaseUntil: now.Add(20 * time.Minute), + }); err == nil || !strings.Contains(err.Error(), "lease timestamps") { + t.Fatalf("zero-updatedAt Update = %v, want explicit refusal", err) + } + + progress := chartifact.Progress{ + State: chartifact.StateWriting, RowCount: 5, ByteCount: 4096, + LeaseUntil: now.Add(20 * time.Minute), UpdatedAt: now.Add(time.Second), + } + if err := s.catalog.Update(s.ctx, "art-update", "writer-1", progress); err != nil { + t.Fatalf("owner Update = %v", err) + } + after := s.witnessRow(t, "art-update") + if after.State != "WRITING" || after.RowCount != 5 || after.ByteCount != 4096 || + !after.LeaseUntil.UTC().Equal(truncUS(progress.LeaseUntil)) { + t.Fatalf("progress not stored: %+v", after) + } + beforeRow := s.witnessRow(t, "art-update") + for name, pair := range map[string][2]string{ + "physical_table": {manifest.PhysicalTable, after.PhysicalTable}, + "execution_id": {manifest.Identity.ExecutionID, after.ExecutionID}, + "output_id": {manifest.Identity.OutputID, after.OutputID}, + "stage_id": {manifest.Identity.StageID, after.StageID}, + "project": {manifest.Identity.Project, after.Project}, + "generation": {manifest.Identity.DatasetGeneration, after.DatasetGeneration}, + "auth_scope": {string(manifest.Identity.AuthScopeMode), after.AuthScopeMode}, + "schema_digest": {manifest.Identity.SchemaDigest, after.SchemaDigest}, + "scope_digest": {manifest.Identity.ScopeDigest, after.ScopeDigest}, + "created_at": {truncUS(manifest.CreatedAt).String(), after.CreatedAt.UTC().String()}, + } { + if pair[0] != pair[1] { + t.Errorf("%s changed across Update: %q -> %q", name, pair[0], pair[1]) + } + } + _ = beforeRow + for member, before := range beforeMembers { + if after := s.payloadMember(t, "art-update", member); after != before { + t.Errorf("Update mutated immutable payload member %q:\n before %s\n after %s", member, before, after) + } + } + if state := s.payloadTextField(t, "art-update", "state"); state != `"WRITING"` { + t.Fatalf("payload state = %s, want WRITING", state) + } + if rows := s.payloadTextField(t, "art-update", "rowCount"); rows != "5" { + t.Fatalf("payload rowCount = %s, want the JSON number 5", rows) + } + if lease := s.payloadTextField(t, "art-update", "leaseUntil"); lease != jsonTimeLiteral(t, progress.LeaseUntil) { + t.Fatalf("payload leaseUntil = %s, want the supplied lease", lease) + } +} + +func TestUpdateLeaseIsMonotonicExceptCleanupAndCleanupIsTerminal(t *testing.T) { + s := newCatalogService(t) + defer s.guarded.check(t) + now := time.Now().UTC() + longLease := now.Add(30 * time.Minute) + manifest := sampleManifestWithLease("art-lease", longLease) + if err := s.catalog.Create(s.ctx, manifest); err != nil { + t.Fatalf("Create = %v", err) + } + storedLease := s.payloadTextField(t, "art-lease", "leaseUntil") + + if err := s.catalog.Update(s.ctx, "art-lease", "writer-1", chartifact.Progress{ + State: chartifact.StateReady, RowCount: 5, ByteCount: 4096, + LeaseUntil: now.Add(time.Minute), UpdatedAt: now.Add(time.Second), + }); err != nil { + t.Fatalf("shorten attempt Update = %v", err) + } + after := s.witnessRow(t, "art-lease") + if !after.LeaseUntil.UTC().Equal(truncUS(longLease)) { + t.Fatalf("active Update shortened the lease to %v, want %v", after.LeaseUntil, longLease) + } + if lease := s.payloadTextField(t, "art-lease", "leaseUntil"); lease != storedLease { + t.Fatalf("payload leaseUntil moved on a shortening update: %s != %s", lease, storedLease) + } + if state := s.payloadTextField(t, "art-lease", "state"); state != `"READY"` { + t.Fatalf("payload state = %s, want READY", state) + } + + reclaimAt := now.Add(30 * time.Minute) + if err := s.catalog.Update(s.ctx, "art-lease", "writer-1", chartifact.Progress{ + State: chartifact.StateCleanupPending, RowCount: 5, ByteCount: 4096, + LeaseUntil: reclaimAt, UpdatedAt: now, + }); err != nil { + t.Fatalf("cleanup Update = %v", err) + } + if got := s.witnessRow(t, "art-lease"); !got.LeaseUntil.UTC().Equal(truncUS(reclaimAt)) || got.State != "CLEANUP_PENDING" { + t.Fatalf("cleanup lease not applied: %+v", got) + } + if err := s.catalog.Update(s.ctx, "art-lease", "writer-1", chartifact.Progress{ + State: chartifact.StateWriting, RowCount: 5, ByteCount: 4096, + LeaseUntil: now.Add(40 * time.Minute), UpdatedAt: now, + }); !errors.Is(err, chartifact.ErrLeaseLost) { + t.Fatalf("active transition from cleanup = %v, want ErrLeaseLost", err) + } + shorter := now.Add(-time.Minute) + if err := s.catalog.Update(s.ctx, "art-lease", "writer-1", chartifact.Progress{ + State: chartifact.StateCleanupPending, RowCount: 5, ByteCount: 4096, + LeaseUntil: shorter, UpdatedAt: now, + }); err != nil { + t.Fatalf("cleanup retry = %v", err) + } + if got := s.witnessRow(t, "art-lease"); !got.LeaseUntil.UTC().Equal(truncUS(shorter)) { + t.Fatalf("cleanup retry lease = %v, want %v", got.LeaseUntil, shorter) + } +} + +func TestUpdateAndRenewRejectExpiredLeases(t *testing.T) { + s := newCatalogService(t) + defer s.guarded.check(t) + now := time.Now().UTC() + expired := now.Add(-time.Hour) + manifest := sampleManifestWithLease("art-expired", expired) + if err := s.catalog.Create(s.ctx, manifest); err != nil { + t.Fatalf("Create = %v", err) + } + if err := s.catalog.Update(s.ctx, "art-expired", "writer-1", chartifact.Progress{ + State: chartifact.StateWriting, RowCount: 5, ByteCount: 4096, + LeaseUntil: now.Add(time.Hour), UpdatedAt: now, + }); !errors.Is(err, chartifact.ErrLeaseLost) { + t.Fatalf("expired Update = %v, want ErrLeaseLost", err) + } + if err := s.catalog.Update(s.ctx, "art-expired", "writer-1", chartifact.Progress{ + State: chartifact.StateCleanupPending, RowCount: 5, ByteCount: 4096, + LeaseUntil: now, UpdatedAt: now, + }); !errors.Is(err, chartifact.ErrLeaseLost) { + t.Fatalf("expired cleanup Update = %v, want ErrLeaseLost", err) + } + renewed, err := s.catalog.Renew(s.ctx, "art-expired", "writer-1", now.Add(time.Hour)) + if err != nil || renewed { + t.Fatalf("expired Renew = (%t, %v), want no resurrection", renewed, err) + } + got := s.witnessRow(t, "art-expired") + if !got.LeaseUntil.UTC().Equal(truncUS(expired)) || got.State != "CREATING" || got.LeaseOwner != "writer-1" { + t.Fatalf("expired-row mutation leaked: %+v", got) + } +} + +func TestRenewExtendsOwnedActiveUnexpiredLeaseOnly(t *testing.T) { + s := newCatalogService(t) + defer s.guarded.check(t) + now := time.Now().UTC() + held := now.Add(10 * time.Minute) + manifest := sampleManifestWithLease("art-renew", held) + if err := s.catalog.Create(s.ctx, manifest); err != nil { + t.Fatalf("Create = %v", err) + } + storedLease := s.payloadTextField(t, "art-renew", "leaseUntil") + + if _, err := s.catalog.Renew(s.ctx, "art-renew", "writer-1", time.Time{}); err == nil { + t.Fatal("Renew accepted a zero lease time") + } + if renewed, err := s.catalog.Renew(s.ctx, "art-renew", "not-the-owner", now.Add(time.Hour)); err != nil || renewed { + t.Fatalf("wrong-owner Renew = (%t, %v), want (false, nil)", renewed, err) + } + if renewed, err := s.catalog.Renew(s.ctx, "art-renew", "writer-1", now.Add(time.Minute)); err != nil || !renewed { + t.Fatalf("shortening Renew = (%t, %v), want a no-op success", renewed, err) + } + if got := s.witnessRow(t, "art-renew"); !got.LeaseUntil.UTC().Equal(truncUS(held)) { + t.Fatalf("Renew shortened the lease to %v", got.LeaseUntil) + } + if lease := s.payloadTextField(t, "art-renew", "leaseUntil"); lease != storedLease { + t.Fatalf("payload leaseUntil moved on a shortening renew: %s != %s", lease, storedLease) + } + + later := now.Add(time.Hour) + if renewed, err := s.catalog.Renew(s.ctx, "art-renew", "writer-1", later); err != nil || !renewed { + t.Fatalf("extending Renew = (%t, %v)", renewed, err) + } + if got := s.witnessRow(t, "art-renew"); !got.LeaseUntil.UTC().Equal(truncUS(later)) { + t.Fatalf("extended lease = %v, want %v", got.LeaseUntil, later) + } + if lease := s.payloadTextField(t, "art-renew", "leaseUntil"); lease != jsonTimeLiteral(t, later) { + t.Fatalf("payload leaseUntil = %s, want the extended lease", lease) + } + + if err := s.catalog.Update(s.ctx, "art-renew", "writer-1", chartifact.Progress{ + State: chartifact.StateCleanupPending, LeaseUntil: now.Add(-time.Minute), UpdatedAt: now, + }); err != nil { + t.Fatalf("cleanup Update = %v", err) + } + if renewed, err := s.catalog.Renew(s.ctx, "art-renew", "writer-1", now.Add(2*time.Hour)); err != nil || renewed { + t.Fatalf("cleanup-state Renew = (%t, %v), want (false, nil)", renewed, err) + } +} + +func TestListExpiredIsBoundedAndStableWithByteOrderTieBreak(t *testing.T) { + s := newCatalogService(t) + defer s.guarded.check(t) + base := fixedTime() + expire := func(id string, offset time.Duration) { + manifest := sampleManifestWithLease(id, base.Add(offset)) + if err := s.catalog.Create(s.ctx, manifest); err != nil { + t.Fatalf("Create %s = %v", id, err) + } + } + expire("tie-B", 3*time.Minute) + expire("tie-a", 3*time.Minute) + expire("early-1", 1*time.Minute) + expire("late-1", 30*time.Minute) + active := sampleManifestWithLease("active-1", base.Add(100*time.Hour)) + if err := s.catalog.Create(s.ctx, active); err != nil { + t.Fatalf("Create active = %v", err) + } + + if got, err := s.catalog.ListExpired(s.ctx, time.Time{}, 10); err == nil { + t.Fatalf("zero-cutoff ListExpired = %#v, want an error", got) + } + if got, err := s.catalog.ListExpired(s.ctx, base.Add(10*time.Minute), 0); err != nil || len(got) != 0 { + t.Fatalf("limit<=0 ListExpired = (%#v, %v), want empty", got, err) + } + if got, err := s.catalog.ListExpired(s.ctx, base.Add(10*time.Minute), -3); err != nil || len(got) != 0 { + t.Fatalf("negative-limit ListExpired = (%#v, %v), want empty", got, err) + } + + pageOne, err := s.catalog.ListExpired(s.ctx, base.Add(10*time.Minute), 2) + if err != nil { + t.Fatalf("ListExpired = %v", err) + } + if len(pageOne) != 2 || pageOne[0].ArtifactID != "early-1" || pageOne[1].ArtifactID != "tie-B" { + t.Fatalf("bounded page = %v, want [early-1 tie-B]", idsOf(pageOne)) + } + again, err := s.catalog.ListExpired(s.ctx, base.Add(10*time.Minute), 2) + if err != nil || len(again) != 2 || again[0].ArtifactID != pageOne[0].ArtifactID || again[1].ArtifactID != pageOne[1].ArtifactID { + t.Fatalf("unstable page: %v vs %v (%v)", idsOf(pageOne), idsOf(again), err) + } + full, err := s.catalog.ListExpired(s.ctx, base.Add(30*time.Minute), 10) + if err != nil { + t.Fatalf("full ListExpired = %v", err) + } + if want := []string{"early-1", "tie-B", "tie-a", "late-1"}; !reflect.DeepEqual(idsOf(full), want) { + t.Fatalf("expired page = %v, want %v (inclusive cutoff, byte-order tie)", idsOf(full), want) + } + if full[0].PhysicalTable != sampleManifestWithLease("early-1", base).PhysicalTable || + !reflect.DeepEqual(full[0].Identity, sampleManifest("early-1").Identity) { + t.Fatalf("decoded manifest lost identity or physical table: %#v", full[0]) + } +} + +func TestListExpiredCapsOversizedPages(t *testing.T) { + s := newCatalogService(t) + defer s.guarded.check(t) + base := fixedTime() + tag := s.rawExec(t, + "INSERT INTO "+s.qualified+" (artifact_id, physical_table, state, row_count, byte_count,"+ + " lease_owner, lease_until, created_at, updated_at, execution_id, output_id, stage_id,"+ + " project, dataset_generation, auth_scope_mode, scope_mode, schema_digest, scope_digest, payload)"+ + " SELECT 'cap-' || to_char(g, 'FM0000'), 'loom_private_stage_cap' || to_char(g, 'FM0000'),"+ + " 'READY', 0, 0, 'writer-cap', $1::timestamptz + make_interval(secs => g), $1, $1,"+ + " 'e','o','s','p','g','unrestricted','','','',"+ + " jsonb_build_object('artifactId', 'cap-' || to_char(g, 'FM0000'), 'state', 'READY',"+ + " 'leaseUntil', to_char($1::timestamptz + make_interval(secs => g), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"'),"+ + " 'createdAt', to_char($1, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"'),"+ + " 'updatedAt', to_char($1, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"'))"+ + " FROM generate_series(1, $2) g", + base, 1003) + if tag.RowsAffected() != 1003 { + t.Fatalf("cap fixture inserted %d rows, want 1003", tag.RowsAffected()) + } + got, err := s.catalog.ListExpired(s.ctx, base.Add(time.Hour), maxExpiredPageSize+7) + if err != nil { + t.Fatalf("oversized ListExpired = %v", err) + } + if len(got) != maxExpiredPageSize { + t.Fatalf("page size = %d, want the %d cap", len(got), maxExpiredPageSize) + } + if got[0].ArtifactID != "cap-0001" { + t.Fatalf("capped page does not start at the stable head: %s", got[0].ArtifactID) + } +} + +func TestClaimCleanupRechecksExpiryAndHasOneWinner(t *testing.T) { + s := newCatalogService(t) + defer s.guarded.check(t) + base := fixedTime() + cutoff := base.Add(time.Hour) + claimLease := base.Add(3 * time.Hour) + manifest := sampleManifestWithLease("art-claim", base) + if err := s.catalog.Create(s.ctx, manifest); err != nil { + t.Fatalf("Create = %v", err) + } + unexpired := sampleManifestWithLease("art-unexpired", base.Add(30*time.Minute)) + if err := s.catalog.Create(s.ctx, unexpired); err != nil { + t.Fatalf("Create unexpired = %v", err) + } + + if _, ok, err := s.catalog.ClaimCleanup(s.ctx, "art-missing", "cleanup-x", cutoff, claimLease); err != nil || ok { + t.Fatalf("missing-row claim = (%t, %v), want (false, nil)", ok, err) + } + if _, ok, err := s.catalog.ClaimCleanup(s.ctx, "art-unexpired", "cleanup-x", base, claimLease); err != nil || ok { + t.Fatalf("not-yet-expired claim (lease 12:30 > cutoff 12:00) = (%t, %v), want (false, nil)", ok, err) + } + if _, _, err := s.catalog.ClaimCleanup(s.ctx, "art-claim", "cleanup-x", cutoff, cutoff); err == nil { + t.Fatal("claim with a non-future cleanup lease succeeded") + } + if _, _, err := s.catalog.ClaimCleanup(s.ctx, "art-claim", "", cutoff, claimLease); err == nil { + t.Fatal("claim with a blank owner succeeded") + } + + catalogB, err := NewCatalog(s.isoB, s.schema) + if err != nil { + s.fail("build second catalog", err) + } + var ( + wg sync.WaitGroup + mu sync.Mutex + owners []string + results []bool + winner chartifact.Manifest + start = make(chan struct{}) + ) + run := func(c *Catalog, owner string) { + wg.Add(1) + go func() { + defer wg.Done() + <-start + claimed, ok, err := c.ClaimCleanup(s.ctx, "art-claim", owner, cutoff, claimLease) + if err != nil { + s.fail("concurrent claim "+owner, err) + } + mu.Lock() + defer mu.Unlock() + results = append(results, ok) + if ok { + winner = claimed + owners = append(owners, owner) + } + }() + } + run(s.catalog, "cleanup-1") + run(catalogB, "cleanup-2") + close(start) + wg.Wait() + + if len(results) != 2 || (results[0] == results[1]) { + t.Fatalf("concurrent claims did not have exactly one winner: %v", results) + } + if winner.State != chartifact.StateCleanupPending || + !winner.LeaseUntil.Equal(claimLease) || + !reflect.DeepEqual(winner.Identity, manifest.Identity) || + winner.PhysicalTable != manifest.PhysicalTable { + t.Fatalf("claimed manifest wrong: %#v", winner) + } + row := s.witnessRow(t, "art-claim") + if row.State != "CLEANUP_PENDING" || row.LeaseOwner != owners[0] { + t.Fatalf("stored claim wrong: %+v (winner %s)", row, owners[0]) + } + if lease := s.payloadTextField(t, "art-claim", "leaseOwner"); lease != `"`+owners[0]+`"` { + t.Fatalf("payload leaseOwner = %s, want winner %s", lease, owners[0]) + } + if lease := s.payloadTextField(t, "art-claim", "state"); lease != `"CLEANUP_PENDING"` { + t.Fatalf("payload state = %s, want CLEANUP_PENDING", lease) + } + if _, ok, err := s.catalog.ClaimCleanup(s.ctx, "art-claim", "cleanup-3", cutoff, claimLease); err != nil || ok { + t.Fatalf("claim after the winner moved the lease = (%t, %v), want (false, nil)", ok, err) + } +} + +func TestDeleteAndReleaseCleanupValidateCleanupOwner(t *testing.T) { + s := newCatalogService(t) + defer s.guarded.check(t) + now := time.Now().UTC() + manifest := sampleManifestWithLease("art-delete", now.Add(-time.Hour)) + if err := s.catalog.Create(s.ctx, manifest); err != nil { + t.Fatalf("Create = %v", err) + } + if err := s.catalog.Delete(s.ctx, "art-delete", "writer-1"); !errors.Is(err, chartifact.ErrLeaseLost) { + t.Fatalf("pre-claim Delete = %v, want ErrLeaseLost", err) + } + if err := s.catalog.ReleaseCleanup(s.ctx, "art-delete", "writer-1", now); !errors.Is(err, chartifact.ErrLeaseLost) { + t.Fatalf("pre-claim ReleaseCleanup = %v, want ErrLeaseLost", err) + } + if err := s.catalog.Delete(s.ctx, "", ""); err == nil { + t.Fatal("Delete accepted empty arguments") + } + + claimed, ok, err := s.catalog.ClaimCleanup(s.ctx, "art-delete", "cleanup-1", now, now.Add(time.Hour)) + if err != nil || !ok { + t.Fatalf("ClaimCleanup = (%#v, %t, %v)", claimed, ok, err) + } + if err := s.catalog.Delete(s.ctx, "art-delete", "cleanup-2"); !errors.Is(err, chartifact.ErrLeaseLost) { + t.Fatalf("wrong-owner Delete = %v, want ErrLeaseLost", err) + } + if err := s.catalog.ReleaseCleanup(s.ctx, "art-delete", "cleanup-2", now); !errors.Is(err, chartifact.ErrLeaseLost) { + t.Fatalf("wrong-owner ReleaseCleanup = %v, want ErrLeaseLost", err) + } + releaseAt := now.Add(-time.Minute) + if err := s.catalog.ReleaseCleanup(s.ctx, "art-delete", "cleanup-1", releaseAt); err != nil { + t.Fatalf("owner ReleaseCleanup = %v", err) + } + row := s.witnessRow(t, "art-delete") + if row.State != "CLEANUP_PENDING" || !row.LeaseUntil.UTC().Equal(truncUS(releaseAt)) { + t.Fatalf("release did not make the manifest reclaimable: %+v", row) + } + if lease := s.payloadTextField(t, "art-delete", "leaseUntil"); lease != jsonTimeLiteral(t, releaseAt) { + t.Fatalf("payload leaseUntil = %s, want the released lease", lease) + } + second, ok, err := s.catalog.ClaimCleanup(s.ctx, "art-delete", "cleanup-2", now.Add(time.Minute), now.Add(2*time.Hour)) + if err != nil || !ok { + t.Fatalf("re-claim = (%#v, %t, %v)", second, ok, err) + } + if err := s.catalog.Delete(s.ctx, "art-delete", "cleanup-1"); !errors.Is(err, chartifact.ErrLeaseLost) { + t.Fatalf("stale-owner Delete = %v, want ErrLeaseLost", err) + } + if err := s.catalog.Delete(s.ctx, "art-delete", "cleanup-2"); err != nil { + t.Fatalf("owner Delete = %v", err) + } + s.requireNoRow(t, "art-delete") + + active := sampleManifestWithLease("art-active-delete", now.Add(10*time.Minute)) + if err := s.catalog.Create(s.ctx, active); err != nil { + t.Fatalf("Create active = %v", err) + } + if err := s.catalog.Delete(s.ctx, "art-active-delete", "writer-1"); !errors.Is(err, chartifact.ErrLeaseLost) { + t.Fatalf("active-state Delete = %v, want ErrLeaseLost", err) + } +} + +func TestZZNoLeftoverOwnedResources(t *testing.T) { + raw := testPostgresURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %v", err) + } + defer admin.Close() + + var leftoverDBs int + if err := admin.QueryRow(ctx, + `SELECT count(*) FROM pg_database WHERE datname LIKE 'pgprivcat01\_it\_%' ESCAPE '\'`).Scan(&leftoverDBs); err != nil { + t.Fatalf("count leftover databases: %v", err) + } + if leftoverDBs != 0 { + var names []string + rows, err := admin.Query(ctx, `SELECT datname FROM pg_database WHERE datname LIKE 'pgprivcat01\_it\_%' ESCAPE '\'`) + if err == nil { + for rows.Next() { + var name string + if err := rows.Scan(&name); err == nil { + names = append(names, name) + } + } + rows.Close() + } + t.Fatalf("%d isolated databases left behind: %v", leftoverDBs, names) + } + + var leftoverSchemas int + if err := admin.QueryRow(ctx, + `SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgprivcat01\_it\_%' ESCAPE '\'`).Scan(&leftoverSchemas); err != nil { + t.Fatalf("count leftover schemas: %v", err) + } + if leftoverSchemas != 0 { + t.Fatalf("%d owned schemas left behind in the service database", leftoverSchemas) + } +} diff --git a/internal/backends/postgres/internal/dataset/commands.go b/internal/backends/postgres/internal/dataset/commands.go new file mode 100644 index 000000000..2e8e2742c --- /dev/null +++ b/internal/backends/postgres/internal/dataset/commands.go @@ -0,0 +1,127 @@ +package postgres + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "strings" +) + +type CommandBackend string + +const ( + CommandBackendPostgreSQL CommandBackend = "POSTGRESQL" +) + +type ParameterType string + +const ( + ParameterText ParameterType = "text" + ParameterJSONB ParameterType = "jsonb" +) + +func (t ParameterType) registered() bool { + switch t { + case ParameterText, ParameterJSONB: + return true + default: + return false + } +} + +type Parameter struct { + Type ParameterType + Value any +} + +type Command struct { + backend CommandBackend + sql string + args []Parameter +} + +func (c Command) Backend() CommandBackend { return c.backend } + +var ( + ErrInvalidCommand = errors.New("dataset postgres: invalid command") + ErrCommandBackendMismatch = errors.New("dataset postgres: command backend mismatch") +) + +var ( + commandPlaceholderPattern = regexp.MustCompile(`\$(\d+)(?:::([a-z]+))?`) + commandAnyPlaceholder = regexp.MustCompile(`\$[0-9]`) +) + +func NewPostgreSQLCommand(sql string, args []Parameter) (Command, error) { + if strings.TrimSpace(sql) == "" { + return Command{}, fmt.Errorf("%w: statement SQL is blank", ErrInvalidCommand) + } + if len(args) == 0 && commandAnyPlaceholder.MatchString(sql) { + return Command{}, fmt.Errorf("%w: statement references $n placeholders but declares no arguments", ErrInvalidCommand) + } + for i, arg := range args { + if !arg.Type.registered() { + return Command{}, fmt.Errorf("%w: argument %d has unregistered parameter type %q", ErrInvalidCommand, i+1, string(arg.Type)) + } + if arg.Value == nil { + return Command{}, fmt.Errorf("%w: argument %d is typed %q but carries a nil value; this store binds no SQL NULLs", ErrInvalidCommand, i+1, string(arg.Type)) + } + switch arg.Type { + case ParameterText: + if _, ok := arg.Value.(string); !ok { + return Command{}, fmt.Errorf("%w: argument %d is typed %q but carries %T", ErrInvalidCommand, i+1, string(arg.Type), arg.Value) + } + case ParameterJSONB: + if _, ok := arg.Value.(string); !ok { + return Command{}, fmt.Errorf("%w: argument %d is typed %q but carries %T; pass a marshalled JSON string", ErrInvalidCommand, i+1, string(arg.Type), arg.Value) + } + } + } + seen := make([]bool, len(args)) + for _, match := range commandPlaceholderPattern.FindAllStringSubmatch(sql, -1) { + n, err := strconv.Atoi(match[1]) + if err != nil || n < 1 || n > len(args) { + return Command{}, fmt.Errorf("%w: placeholder $%s is outside the declared argument range 1..%d", ErrInvalidCommand, match[1], len(args)) + } + cast := match[2] + switch args[n-1].Type { + case ParameterJSONB: + if cast != string(ParameterJSONB) { + return Command{}, fmt.Errorf("%w: placeholder $%d binds a %s argument and must appear as $%d::jsonb", ErrInvalidCommand, n, string(ParameterJSONB), n) + } + case ParameterText: + if cast != "" { + return Command{}, fmt.Errorf("%w: placeholder $%d binds a %s argument and must appear bare, not as $%d::%s", ErrInvalidCommand, n, string(ParameterText), n, cast) + } + } + seen[n-1] = true + } + if commandAnyPlaceholder.MatchString(commandPlaceholderPattern.ReplaceAllString(sql, "")) { + return Command{}, fmt.Errorf("%w: statement contains a malformed placeholder", ErrInvalidCommand) + } + for i, used := range seen { + if !used { + return Command{}, fmt.Errorf("%w: argument %d is declared but never referenced by a placeholder", ErrInvalidCommand, i+1) + } + } + return Command{ + backend: CommandBackendPostgreSQL, + sql: sql, + args: append([]Parameter(nil), args...), + }, nil +} + +func (c Command) PostgreSQL() (string, []any, error) { + switch c.backend { + case CommandBackendPostgreSQL: + values := make([]any, 0, len(c.args)) + for _, arg := range c.args { + values = append(values, arg.Value) + } + return c.sql, values, nil + default: + return "", nil, fmt.Errorf("%w: refusing to pass command with backend %q to the PostgreSQL statement executor", + ErrCommandBackendMismatch, string(c.backend)) + } +} diff --git a/internal/backends/postgres/internal/dataset/commands_test.go b/internal/backends/postgres/internal/dataset/commands_test.go new file mode 100644 index 000000000..51ff51f47 --- /dev/null +++ b/internal/backends/postgres/internal/dataset/commands_test.go @@ -0,0 +1,115 @@ +package postgres + +import ( + "context" + "errors" + "strings" + "testing" + + publication "github.com/calypr/loom/internal/dataset" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestNewPostgreSQLCommandValidatesOrderedParameters(t *testing.T) { + good, err := NewPostgreSQLCommand( + "UPDATE t SET state = $3 WHERE project = $1 AND generation = $2 AND schema_identity = $4::jsonb", + []Parameter{ + {Type: ParameterText, Value: "p"}, + {Type: ParameterText, Value: "g"}, + {Type: ParameterText, Value: "STAGED"}, + {Type: ParameterJSONB, Value: `{"schemaSha256":"a"}`}, + }) + if err != nil { + t.Fatalf("good command: %v", err) + } + if got := good.Backend(); got != CommandBackendPostgreSQL { + t.Fatalf("backend = %q", got) + } + sqlText, args, err := good.PostgreSQL() + if err != nil { + t.Fatalf("PostgreSQL dispatch: %v", err) + } + if !strings.Contains(sqlText, "$4::jsonb") { + t.Fatalf("sql = %s", sqlText) + } + if len(args) != 4 || args[0] != "p" || args[3] != `{"schemaSha256":"a"}` { + t.Fatalf("ordered args = %v", args) + } + args[0] = "mutated" + _, again, err := good.PostgreSQL() + if err != nil { + t.Fatal(err) + } + if again[0] != "p" { + t.Fatalf("argument slice is shared with the command: %v", again) + } + + bad := []struct { + name string + sql string + args []Parameter + }{ + {"blank sql", " ", nil}, + {"unregistered type", "SELECT $1", []Parameter{{Type: ParameterType("int4"), Value: "1"}}}, + {"nil value", "SELECT $1", []Parameter{{Type: ParameterText, Value: nil}}}, + {"non-string text", "SELECT $1", []Parameter{{Type: ParameterText, Value: 7}}}, + {"placeholder out of range", "SELECT $2", []Parameter{{Type: ParameterText, Value: "a"}}}, + {"unused argument", "SELECT $1", []Parameter{{Type: ParameterText, Value: "a"}, {Type: ParameterText, Value: "b"}}}, + {"jsonb without cast", "SELECT $1", []Parameter{{Type: ParameterJSONB, Value: "{}"}}}, + {"text with cast", "SELECT $1::text", []Parameter{{Type: ParameterText, Value: "a"}}}, + {"wrong jsonb cast name", "SELECT $1::json", []Parameter{{Type: ParameterJSONB, Value: "{}"}}}, + {"text placeholder carries cast", "SELECT $1::jsonb", []Parameter{{Type: ParameterText, Value: "a"}}}, + } + for _, testCase := range bad { + if _, err := NewPostgreSQLCommand(testCase.sql, testCase.args); !errors.Is(err, ErrInvalidCommand) { + t.Errorf("%s: error = %v, want ErrInvalidCommand", testCase.name, err) + } + } +} + +func TestCommandBackendDispatchFailsClosed(t *testing.T) { + if _, _, err := (Command{}).PostgreSQL(); !errors.Is(err, ErrCommandBackendMismatch) { + t.Fatalf("zero command: %v", err) + } + if _, err := execCommand(context.Background(), nil, Command{}); !errors.Is(err, ErrCommandBackendMismatch) { + t.Fatalf("execCommand with untagged command: %v", err) + } + if err := queryRowCommand(context.Background(), nil, Command{}).Scan(new(string)); !errors.Is(err, ErrCommandBackendMismatch) { + t.Fatalf("queryRowCommand with untagged command: %v", err) + } +} + +func TestNewValidatesInputs(t *testing.T) { + if _, err := New(nil, "public"); !errors.Is(err, ErrNilConnSource) { + t.Fatalf("New(nil) = %v", err) + } + for _, schema := range []string{"", "Public", "a b", "1x", strings.Repeat("a", 64), `x"; DROP TABLE y; --`} { + if _, err := New(fakeConnSource{}, schema); !errors.Is(err, ErrInvalidSchemaName) { + t.Errorf("New(schema %q) = %v, want ErrInvalidSchemaName", schema, err) + } + } + store, err := New(fakeConnSource{}, "pgstore03_it_demo") + if err != nil { + t.Fatalf("New: %v", err) + } + if store.manifestFQN != `"pgstore03_it_demo"."dataset_generation_manifest"` || + store.pointerFQN != `"pgstore03_it_demo"."dataset_active_generation"` { + t.Fatalf("fqns = %s / %s", store.manifestFQN, store.pointerFQN) + } +} + +type fakeConnSource struct{} + +func (fakeConnSource) Acquire(context.Context) (*pgxpool.Conn, error) { + return nil, errors.New("unused") +} + +func TestValidateProjectAndActiveResolverContract(t *testing.T) { + if err := validateProject(""); err == nil { + t.Fatal("blank project accepted") + } + if err := validateProject("ok-project"); err != nil { + t.Fatalf("valid project rejected: %v", err) + } + var _ publication.ActiveResolver = (*Store)(nil) +} diff --git a/internal/backends/postgres/internal/dataset/manifest_lifecycle.go b/internal/backends/postgres/internal/dataset/manifest_lifecycle.go new file mode 100644 index 000000000..95cf7b052 --- /dev/null +++ b/internal/backends/postgres/internal/dataset/manifest_lifecycle.go @@ -0,0 +1,443 @@ +package postgres + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + publication "github.com/calypr/loom/internal/dataset" + "github.com/jackc/pgx/v5" +) + +var _ publication.ActiveResolver = (*Store)(nil) + +func (s *Store) CreateManifest(ctx context.Context, manifest publication.Manifest) (publication.Manifest, error) { + if err := manifest.Validate(); err != nil { + return publication.Manifest{}, fmt.Errorf("create generation manifest: %w", err) + } + if manifest.State != publication.StateLoading { + return publication.Manifest{}, fmt.Errorf("create generation manifest: %w: state must be %s, got %s", publication.ErrInvalidTransition, publication.StateLoading, manifest.State) + } + if err := s.validate(); err != nil { + return publication.Manifest{}, err + } + identity, err := marshalSchemaIdentity(manifest.SchemaIdentity) + if err != nil { + return publication.Manifest{}, fmt.Errorf("create generation manifest: %w", err) + } + command, err := NewPostgreSQLCommand( + "INSERT INTO "+s.manifestFQN+` ( + project, generation, state, schema_identity +) +VALUES ($1, $2, $3, $4::jsonb) +RETURNING project, generation, state, schema_identity::text`, + []Parameter{ + {Type: ParameterText, Value: manifest.Dataset.Project}, + {Type: ParameterText, Value: manifest.Dataset.Generation}, + {Type: ParameterText, Value: string(manifest.State)}, + {Type: ParameterJSONB, Value: identity}, + }) + if err != nil { + return publication.Manifest{}, fmt.Errorf("create generation manifest: %w", err) + } + + conn, err := s.src.Acquire(ctx) + if err != nil { + return publication.Manifest{}, fmt.Errorf("create generation manifest: acquire connection: %w", err) + } + defer conn.Release() + + var ( + project string + generation string + state string + identityDB string + ) + err = queryRowConn(ctx, conn, command).Scan(&project, &generation, &state, &identityDB) + if err != nil { + if storepostgres.IsUniqueViolation(err) { + return publication.Manifest{}, fmt.Errorf("%w: %s/%s", ErrManifestAlreadyExists, manifest.Dataset.Project, manifest.Dataset.Generation) + } + return publication.Manifest{}, fmt.Errorf("create generation manifest: %w", err) + } + created, err := manifestFromRow(project, generation, state, identityDB) + if err != nil { + return publication.Manifest{}, fmt.Errorf("create generation manifest: %w", err) + } + if !reflect.DeepEqual(created, manifest) { + return publication.Manifest{}, fmt.Errorf("%w: created manifest does not match request", ErrUnexpectedStoreResult) + } + return created, nil +} + +func (s *Store) ReadManifest(ctx context.Context, ref publication.Ref) (publication.Manifest, error) { + if err := ref.Validate(); err != nil { + return publication.Manifest{}, fmt.Errorf("read generation manifest: %w", err) + } + if err := s.validate(); err != nil { + return publication.Manifest{}, err + } + command, err := NewPostgreSQLCommand( + "SELECT project, generation, state, schema_identity::text FROM "+s.manifestFQN+` +WHERE project = $1 AND generation = $2`, + []Parameter{ + {Type: ParameterText, Value: ref.Project}, + {Type: ParameterText, Value: ref.Generation}, + }) + if err != nil { + return publication.Manifest{}, fmt.Errorf("read generation manifest: %w", err) + } + + conn, err := s.src.Acquire(ctx) + if err != nil { + return publication.Manifest{}, fmt.Errorf("read generation manifest: acquire connection: %w", err) + } + defer conn.Release() + + var ( + project string + generation string + state string + identityDB string + ) + err = queryRowConn(ctx, conn, command).Scan(&project, &generation, &state, &identityDB) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return publication.Manifest{}, fmt.Errorf("%w: %s/%s", ErrManifestNotFound, ref.Project, ref.Generation) + } + return publication.Manifest{}, fmt.Errorf("read generation manifest: %w", err) + } + manifest, err := manifestFromRow(project, generation, state, identityDB) + if err != nil { + return publication.Manifest{}, fmt.Errorf("read generation manifest: %w", err) + } + if manifest.Dataset != ref { + return publication.Manifest{}, fmt.Errorf("%w: read manifest reference does not match request", ErrUnexpectedStoreResult) + } + return manifest, nil +} + +func (s *Store) TransitionManifest(ctx context.Context, manifest publication.Manifest, next publication.State) (publication.Manifest, error) { + nextManifest, err := manifest.Transition(next) + if err != nil { + return publication.Manifest{}, fmt.Errorf("transition generation manifest: %w", err) + } + if err := s.validate(); err != nil { + return publication.Manifest{}, err + } + identity, err := marshalSchemaIdentity(manifest.SchemaIdentity) + if err != nil { + return publication.Manifest{}, fmt.Errorf("transition generation manifest: %w", err) + } + command, err := NewPostgreSQLCommand( + "UPDATE "+s.manifestFQN+` +SET state = $3, updated_at = now() +WHERE project = $1 AND generation = $2 AND state = $4 AND schema_identity = $5::jsonb +RETURNING project, generation, state, schema_identity::text`, + []Parameter{ + {Type: ParameterText, Value: manifest.Dataset.Project}, + {Type: ParameterText, Value: manifest.Dataset.Generation}, + {Type: ParameterText, Value: string(next)}, + {Type: ParameterText, Value: string(manifest.State)}, + {Type: ParameterJSONB, Value: identity}, + }) + if err != nil { + return publication.Manifest{}, fmt.Errorf("transition generation manifest: %w", err) + } + + var result *publication.Manifest + err = s.inTx(ctx, "transition", func(tx pgx.Tx) error { + var ( + project string + generation string + state string + identityDB string + ) + scanErr := queryRowCommand(ctx, tx, command).Scan(&project, &generation, &state, &identityDB) + if scanErr != nil { + if errors.Is(scanErr, pgx.ErrNoRows) { + return fmt.Errorf("%w: %s/%s was not %s with the expected immutable metadata", + ErrManifestTransitionConflict, manifest.Dataset.Project, manifest.Dataset.Generation, manifest.State) + } + return scanErr + } + decoded, decodeErr := manifestFromRow(project, generation, state, identityDB) + if decodeErr != nil { + return decodeErr + } + if !reflect.DeepEqual(decoded, nextManifest) { + return fmt.Errorf("%w: transition result does not match requested state", ErrUnexpectedStoreResult) + } + result = &decoded + return nil + }) + if err != nil { + return publication.Manifest{}, fmt.Errorf("transition generation manifest: %w", err) + } + return *result, nil +} + +func (s *Store) ResolveActiveManifest(ctx context.Context, project string) (publication.Manifest, error) { + if err := validateProject(project); err != nil { + return publication.Manifest{}, fmt.Errorf("resolve active generation: %w", err) + } + if err := s.validate(); err != nil { + return publication.Manifest{}, err + } + command, err := NewPostgreSQLCommand( + "SELECT m.project, m.generation, m.state, m.schema_identity::text FROM "+s.pointerFQN+` p +JOIN `+s.manifestFQN+` m + ON m.project = p.project AND m.generation = p.generation +WHERE p.project = $1`, + []Parameter{ + {Type: ParameterText, Value: project}, + }) + if err != nil { + return publication.Manifest{}, fmt.Errorf("resolve active generation: %w", err) + } + + conn, err := s.src.Acquire(ctx) + if err != nil { + return publication.Manifest{}, fmt.Errorf("resolve active generation: acquire connection: %w", err) + } + defer conn.Release() + + var ( + generatedProject string + generation string + state string + identityDB string + ) + err = queryRowConn(ctx, conn, command).Scan(&generatedProject, &generation, &state, &identityDB) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return publication.Manifest{}, fmt.Errorf("%w: %s", publication.ErrNoActiveGeneration, project) + } + return publication.Manifest{}, fmt.Errorf("resolve active generation: %w", err) + } + manifest, err := manifestFromRow(generatedProject, generation, state, identityDB) + if err != nil { + return publication.Manifest{}, fmt.Errorf("resolve active generation: %w", err) + } + if manifest.Dataset.Project != project || !manifest.IsStaged() { + return publication.Manifest{}, fmt.Errorf("%w: active pointer and staged manifest are inconsistent", ErrUnexpectedStoreResult) + } + return manifest, nil +} + +func (s *Store) ActivateGeneration(ctx context.Context, candidate publication.Manifest, expected *publication.Ref) error { + if !candidate.IsStaged() { + if err := candidate.Validate(); err != nil { + return fmt.Errorf("activate generation: %w", err) + } + return fmt.Errorf("activate generation: %w: %s is %s", publication.ErrGenerationNotReady, candidate.Dataset.Generation, candidate.State) + } + if expected != nil { + if err := expected.Validate(); err != nil { + return fmt.Errorf("activate generation: %w: expected pointer: %w", publication.ErrInvalidActiveGeneration, err) + } + if expected.Project != candidate.Dataset.Project { + return fmt.Errorf("%w: expected pointer project %q does not match candidate project %q", + publication.ErrInvalidActiveGeneration, expected.Project, candidate.Dataset.Project) + } + } + if err := s.validate(); err != nil { + return err + } + project := candidate.Dataset.Project + generation := candidate.Dataset.Generation + + lockCandidate, err := NewPostgreSQLCommand( + "SELECT state, schema_identity::text FROM "+s.manifestFQN+` +WHERE project = $1 AND generation = $2 +FOR SHARE`, + []Parameter{ + {Type: ParameterText, Value: project}, + {Type: ParameterText, Value: generation}, + }) + if err != nil { + return fmt.Errorf("activate generation: %w", err) + } + lockPointer, err := NewPostgreSQLCommand( + "SELECT generation FROM "+s.pointerFQN+` +WHERE project = $1 +FOR UPDATE`, + []Parameter{ + {Type: ParameterText, Value: project}, + }) + if err != nil { + return fmt.Errorf("activate generation: %w", err) + } + readPointer, err := NewPostgreSQLCommand( + "SELECT generation FROM "+s.pointerFQN+` +WHERE project = $1`, + []Parameter{ + {Type: ParameterText, Value: project}, + }) + if err != nil { + return fmt.Errorf("activate generation: %w", err) + } + updatePointerSQL := "UPDATE " + s.pointerFQN + ` +SET generation = $2, updated_at = now() +WHERE project = $1 AND generation = $3` + insertPointer, err := NewPostgreSQLCommand( + "INSERT INTO "+s.pointerFQN+` (project, generation) +VALUES ($1, $2) +ON CONFLICT DO NOTHING`, + []Parameter{ + {Type: ParameterText, Value: project}, + {Type: ParameterText, Value: generation}, + }) + if err != nil { + return fmt.Errorf("activate generation: %w", err) + } + + return s.inTx(ctx, "activate", func(tx pgx.Tx) error { + var ( + storedState string + storedIdentity string + ) + scanErr := queryRowCommand(ctx, tx, lockCandidate).Scan(&storedState, &storedIdentity) + if scanErr != nil { + if errors.Is(scanErr, pgx.ErrNoRows) { + return fmt.Errorf("%w: candidate %s/%s is not a persisted manifest", ErrActivationConflict, project, generation) + } + return scanErr + } + stored, err := manifestFromRow(project, generation, storedState, storedIdentity) + if err != nil { + return err + } + if !stored.IsStaged() || !reflect.DeepEqual(stored.SchemaIdentity, candidate.SchemaIdentity) { + return fmt.Errorf("%w: candidate %s/%s was not a persisted staged manifest with the expected immutable metadata", + ErrActivationConflict, project, generation) + } + + var pointerGeneration string + pointerErr := queryRowCommand(ctx, tx, lockPointer).Scan(&pointerGeneration) + if pointerErr != nil && !errors.Is(pointerErr, pgx.ErrNoRows) { + return pointerErr + } + hasPointer := pointerErr == nil + + if hasPointer { + if pointerGeneration == generation { + return nil + } + if expected == nil { + return fmt.Errorf("%w: candidate %s/%s expected no active generation, found %s", + ErrActivationConflict, project, generation, pointerGeneration) + } + if expected.Generation != pointerGeneration { + return fmt.Errorf("%w: candidate %s/%s expected active generation %s, found %s", + ErrActivationConflict, project, generation, expected.Generation, pointerGeneration) + } + updatePointer, err := NewPostgreSQLCommand( + updatePointerSQL, + []Parameter{ + {Type: ParameterText, Value: project}, + {Type: ParameterText, Value: generation}, + {Type: ParameterText, Value: expected.Generation}, + }) + if err != nil { + return err + } + tag, err := execCommand(ctx, tx, updatePointer) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return fmt.Errorf("%w: active generation pointer moved during activation", ErrActivationConflict) + } + return nil + } + + if expected != nil { + return fmt.Errorf("%w: candidate %s/%s expected active generation %s, found no active generation", + ErrActivationConflict, project, generation, expected.Generation) + } + tag, err := execCommand(ctx, tx, insertPointer) + if err != nil { + return err + } + if tag.RowsAffected() == 1 { + return nil + } + var concurrent string + rereadErr := queryRowCommand(ctx, tx, readPointer).Scan(&concurrent) + if rereadErr != nil { + if errors.Is(rereadErr, pgx.ErrNoRows) { + return fmt.Errorf("%w: active generation pointer vanished during activation", ErrUnexpectedStoreResult) + } + return rereadErr + } + if concurrent == generation { + return nil + } + return fmt.Errorf("%w: candidate %s/%s expected no active generation, found %s (concurrent activation)", + ErrActivationConflict, project, generation, concurrent) + }) +} + +func (s *Store) inTx(ctx context.Context, step string, body func(pgx.Tx) error) (err error) { + conn, err := s.src.Acquire(ctx) + if err != nil { + return fmt.Errorf("acquire connection: %w", err) + } + defer conn.Release() + + tx, err := conn.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return fmt.Errorf("begin transaction: %w", err) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback(ctx) + } + }() + + if err := body(tx); err != nil { + return err + } + if s.commitFault != nil { + if err := s.commitFault(step); err != nil { + return fmt.Errorf("injected fault before commit: %w", err) + } + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("commit transaction: %w", err) + } + committed = true + return nil +} + +func marshalSchemaIdentity(schema publication.SchemaSnapshot) (string, error) { + encoded, err := json.Marshal(schema) + if err != nil { + return "", fmt.Errorf("encode schema identity: %w", err) + } + return string(encoded), nil +} + +func manifestFromRow(project, generation, state, identityDB string) (publication.Manifest, error) { + var schema publication.SchemaSnapshot + decoder := json.NewDecoder(bytes.NewReader([]byte(identityDB))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&schema); err != nil { + return publication.Manifest{}, fmt.Errorf("%w: stored schema identity is malformed: %v", ErrUnexpectedStoreResult, err) + } + manifest := publication.Manifest{ + Dataset: publication.Ref{Project: project, Generation: generation}, + State: publication.State(state), + SchemaIdentity: schema, + } + if err := manifest.Validate(); err != nil { + return publication.Manifest{}, fmt.Errorf("%w: stored manifest is invalid: %v", ErrUnexpectedStoreResult, err) + } + return manifest, nil +} diff --git a/internal/backends/postgres/internal/dataset/manifest_lifecycle_pg_test.go b/internal/backends/postgres/internal/dataset/manifest_lifecycle_pg_test.go new file mode 100644 index 000000000..81278330b --- /dev/null +++ b/internal/backends/postgres/internal/dataset/manifest_lifecycle_pg_test.go @@ -0,0 +1,884 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "math/rand" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "sync" + "testing" + "time" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + publication "github.com/calypr/loom/internal/dataset" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +var pgstore03ResourceNamePattern = regexp.MustCompile(`^pgstore03_it_[0-9]+_[0-9a-f]+$`) + +func testPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 integration tests") + } + return raw +} + +func randomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +type pgService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func newPGService(t *testing.T) *pgService { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgstore03_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstore03ResourceNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if _, err := admin.Exec(cleanupCtx, + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("pgstore03_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstore03ResourceNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if _, err := iso.Exec(cleanupCtx, + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + return &pgService{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} +} + +func (s *pgService) openClient(t *testing.T) *storepostgres.Client { + t.Helper() + return s.openClientWithBounds(t, 2, 4) +} + +func (s *pgService) openClientWithBounds(t *testing.T, minConns, maxConns int32) *storepostgres.Client { + t.Helper() + client, err := storepostgres.Open(s.ctx, storepostgres.Config{ + DSN: s.dsn, + MaxConns: maxConns, + MinConns: minConns, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *pgService) mustMigrate(t *testing.T, client *storepostgres.Client) { + t.Helper() + result, err := storepostgres.MigrateSchema(s.ctx, client, s.schema) + if err != nil { + s.fail("MigrateSchema", err) + } + embedded, err := storepostgres.EmbeddedMigrations() + if err != nil { + s.fail("EmbeddedMigrations", err) + } + var want *storepostgres.Migration + for i := range embedded { + if embedded[i].Version == 3 { + want = &embedded[i] + } + } + if want == nil || want.Name != "dataset_generation_manifest" { + t.Fatalf("embedded history has no migration 3 named dataset_generation_manifest: %+v", embedded) + } + sum := sha256.Sum256([]byte(want.SQL)) + if want.Checksum != hex.EncodeToString(sum[:]) { + t.Fatalf("migration 3 checksum does not match its SQL: %s", want.Checksum) + } + var found *storepostgres.LedgerRecord + for i := range result.Applied { + if result.Applied[i].Version == 3 { + found = &result.Applied[i] + } + } + if found == nil { + t.Fatalf("ledger lacks migration 3: %+v", result.Applied) + } + if found.Name != want.Name || found.Checksum != want.Checksum { + t.Fatalf("ledger row for migration 3 = %+v, want name %q checksum %q", *found, want.Name, want.Checksum) + } + for _, table := range []string{"dataset_generation_manifest", "dataset_active_generation"} { + var regclass *string + if err := s.iso.QueryRow(s.ctx, `SELECT to_regclass($1)::text`, s.schema+"."+table).Scan(®class); err != nil { + s.fail("witness to_regclass "+table, err) + } + if regclass == nil || *regclass == "" { + t.Fatalf("table %s was not created by migration 3", table) + } + } +} + +func (s *pgService) openStore(t *testing.T) (*Store, *storepostgres.Client) { + t.Helper() + client := s.openClient(t) + s.mustMigrate(t, client) + store, err := New(client, s.schema) + if err != nil { + s.fail("New store", err) + } + return store, client +} + +func (s *pgService) rawExec(t *testing.T, sql string, args ...any) pgconn.CommandTag { + t.Helper() + tag, err := s.iso.Exec(s.ctx, sql, args...) + if err != nil { + s.fail("raw exec", err) + } + return tag +} + +func (s *pgService) dumpManifestRow(t *testing.T, project, generation string) string { + t.Helper() + var state, identity string + err := s.iso.QueryRow(s.ctx, + `SELECT state, schema_identity::text FROM "`+s.schema+`".dataset_generation_manifest WHERE project = $1 AND generation = $2`, + project, generation).Scan(&state, &identity) + if err != nil { + return fmt.Sprintf("", s.scrub(err)) + } + return fmt.Sprintf("state=%s identity=%s", state, identity) +} + +func (s *pgService) assertManifestRowLiteral(t *testing.T, ref publication.Ref, wantState string, wantIdentity publication.SchemaSnapshot) { + t.Helper() + identity, err := json.Marshal(wantIdentity) + if err != nil { + t.Fatal(err) + } + var found bool + err = s.iso.QueryRow(s.ctx, + `SELECT true FROM "`+s.schema+`".dataset_generation_manifest +WHERE project = $1 AND generation = $2 AND state = $3 AND schema_identity = $4::jsonb`, + ref.Project, ref.Generation, wantState, string(identity)).Scan(&found) + if err != nil || !found { + t.Fatalf("manifest row %s/%s does not match literal expectation state=%s identity=%s (actual: %s; err=%v)", + ref.Project, ref.Generation, wantState, identity, + s.dumpManifestRow(t, ref.Project, ref.Generation), s.scrub(err)) + } +} + +type pointerWitness struct { + generation string + ctid string + updatedAt string + exists bool +} + +func (s *pgService) witnessPointer(t *testing.T, project string) pointerWitness { + t.Helper() + var w pointerWitness + err := s.iso.QueryRow(s.ctx, + `SELECT generation, ctid::text, updated_at::text FROM "`+s.schema+`".dataset_active_generation WHERE project = $1`, + project).Scan(&w.generation, &w.ctid, &w.updatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return w + } + if err != nil { + s.fail("witness pointer", err) + } + w.exists = true + return w +} + +func (s *pgService) assertPointerRowLiteral(t *testing.T, project, wantGeneration string) { + t.Helper() + w := s.witnessPointer(t, project) + if wantGeneration == "" { + if w.exists { + t.Fatalf("expected no active-generation pointer for %s, found generation=%s", project, w.generation) + } + return + } + if !w.exists || w.generation != wantGeneration { + t.Fatalf("active-generation pointer for %s = %+v, want generation=%s", project, w, wantGeneration) + } +} + +func mustRef(t *testing.T, project, generation string) publication.Ref { + t.Helper() + ref, err := publication.NewRef(project, generation) + if err != nil { + t.Fatalf("NewRef(%s, %s): %v", project, generation, err) + } + return ref +} + +func mustSnapshot(t *testing.T, generation, digest string) publication.SchemaSnapshot { + t.Helper() + schema, err := publication.NewSchemaSnapshot("urn:pgstore03-"+generation, "R5", digest, []string{"Observation", "Patient"}) + if err != nil { + t.Fatalf("NewSchemaSnapshot(%s): %v", generation, err) + } + return schema +} + +func (s *pgService) mustPersistStaged(t *testing.T, store *Store, project, generation, digest string) publication.Manifest { + t.Helper() + ref := mustRef(t, project, generation) + schema := mustSnapshot(t, generation, digest) + loading, err := publication.NewManifest(ref, schema) + if err != nil { + t.Fatalf("NewManifest(%s/%s): %v", project, generation, err) + } + if _, err := store.CreateManifest(s.ctx, loading); err != nil { + t.Fatalf("CreateManifest(%s/%s): %v", project, generation, err) + } + staged, err := store.TransitionManifest(s.ctx, loading, publication.StateStaged) + if err != nil { + t.Fatalf("TransitionManifest(%s/%s -> STAGED): %v", project, generation, err) + } + if !staged.IsStaged() { + t.Fatalf("manifest %s/%s not staged after transition: %s", project, generation, staged.State) + } + s.assertManifestRowLiteral(t, ref, string(publication.StateStaged), schema) + return staged +} + +const ( + digestA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + digestB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + digestC = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + digestF = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" +) + +func TestManifestLifecyclePersistsLiteralRows(t *testing.T) { + s := newPGService(t) + store, _ := s.openStore(t) + + ref := mustRef(t, "pgstore03-project", "gen-1") + schema := mustSnapshot(t, "gen-1", digestA) + loading, err := publication.NewManifest(ref, schema) + if err != nil { + t.Fatal(err) + } + + created, err := store.CreateManifest(s.ctx, loading) + if err != nil { + t.Fatalf("CreateManifest: %v", err) + } + if !reflect.DeepEqual(created, loading) { + t.Fatalf("CreateManifest returned %+v, want %+v", created, loading) + } + s.assertManifestRowLiteral(t, ref, "LOADING", schema) + + read, err := store.ReadManifest(s.ctx, ref) + if err != nil { + t.Fatalf("ReadManifest: %v", err) + } + if !reflect.DeepEqual(read, loading) { + t.Fatalf("ReadManifest = %+v, want %+v", read, loading) + } + identityBytes, err := json.Marshal(read.SchemaIdentity) + if err != nil { + t.Fatal(err) + } + const wantIdentity = `{"schemaId":"urn:pgstore03-gen-1","fhirVersion":"R5","schemaSha256":"` + digestA + `","generatedResourceTypes":["Observation","Patient"]}` + if string(identityBytes) != wantIdentity { + t.Fatalf("schema identity bytes drifted: %s want %s", identityBytes, wantIdentity) + } + + staged, err := store.TransitionManifest(s.ctx, loading, publication.StateStaged) + if err != nil { + t.Fatalf("TransitionManifest: %v", err) + } + if !reflect.DeepEqual(staged, mustStagedCopy(loading)) { + t.Fatalf("TransitionManifest = %+v", staged) + } + s.assertManifestRowLiteral(t, ref, "STAGED", schema) + + ref2 := mustRef(t, "pgstore03-project", "gen-ready") + schema2 := mustSnapshot(t, "gen-ready", digestB) + loading2, err := publication.NewManifest(ref2, schema2) + if err != nil { + t.Fatal(err) + } + if _, err := store.CreateManifest(s.ctx, loading2); err != nil { + t.Fatalf("CreateManifest gen-ready: %v", err) + } + if _, err := store.TransitionManifest(s.ctx, loading2, publication.StateReady); err != nil { + t.Fatalf("TransitionManifest to READY: %v", err) + } + s.assertManifestRowLiteral(t, ref2, "READY", schema2) +} + +func mustStagedCopy(m publication.Manifest) publication.Manifest { + m.State = publication.StateStaged + return m +} + +func TestCreateManifestDuplicateAndInvalid(t *testing.T) { + s := newPGService(t) + store, _ := s.openStore(t) + + ref := mustRef(t, "pgstore03-project", "gen-dup") + schema := mustSnapshot(t, "gen-dup", digestA) + loading, err := publication.NewManifest(ref, schema) + if err != nil { + t.Fatal(err) + } + if _, err := store.CreateManifest(s.ctx, loading); err != nil { + t.Fatalf("first CreateManifest: %v", err) + } + + if _, err := store.CreateManifest(s.ctx, loading); !errors.Is(err, ErrManifestAlreadyExists) { + t.Fatalf("duplicate create: %v, want ErrManifestAlreadyExists", err) + } + other := loading + other.SchemaIdentity = mustSnapshot(t, "gen-dup", digestB) + if _, err := store.CreateManifest(s.ctx, other); !errors.Is(err, ErrManifestAlreadyExists) { + t.Fatalf("duplicate create with different identity: %v, want ErrManifestAlreadyExists", err) + } + s.assertManifestRowLiteral(t, ref, "LOADING", schema) + + staged := loading + staged.State = publication.StateStaged + if _, err := store.CreateManifest(s.ctx, staged); !errors.Is(err, publication.ErrInvalidTransition) { + t.Fatalf("create staged: %v, want ErrInvalidTransition", err) + } + bad := loading + bad.SchemaIdentity.SchemaSHA256 = "not-a-digest" + if _, err := store.CreateManifest(s.ctx, bad); !errors.Is(err, publication.ErrInvalidManifest) { + t.Fatalf("create with bad digest: %v, want ErrInvalidManifest", err) + } + missing := mustRef(t, "pgstore03-project", "gen-none") + if _, err := store.ReadManifest(s.ctx, missing); !errors.Is(err, publication.ErrManifestNotFound) { + t.Fatalf("ReadManifest missing: %v, want ErrManifestNotFound", err) + } +} + +func TestTransitionManifestTypedConflictAndIllegalEdges(t *testing.T) { + s := newPGService(t) + store, _ := s.openStore(t) + + loading := func(generation, digest string) publication.Manifest { + m, err := publication.NewManifest(mustRef(t, "pgstore03-project", generation), mustSnapshot(t, generation, digest)) + if err != nil { + t.Fatal(err) + } + return m + } + + m := loading("gen-cas", digestA) + if _, err := store.CreateManifest(s.ctx, m); err != nil { + t.Fatal(err) + } + if _, err := store.TransitionManifest(s.ctx, m, publication.StateStaged); err != nil { + t.Fatal(err) + } + if _, err := store.TransitionManifest(s.ctx, m, publication.StateFailed); !errors.Is(err, ErrManifestTransitionConflict) { + t.Fatalf("stale-state transition: %v, want ErrManifestTransitionConflict", err) + } + s.assertManifestRowLiteral(t, m.Dataset, "STAGED", m.SchemaIdentity) + + edited := m + edited.SchemaIdentity = mustSnapshot(t, "gen-cas", digestF) + if _, err := store.TransitionManifest(s.ctx, edited, publication.StateFailed); !errors.Is(err, ErrManifestTransitionConflict) { + t.Fatalf("edited-identity transition: %v, want ErrManifestTransitionConflict", err) + } + s.assertManifestRowLiteral(t, m.Dataset, "STAGED", m.SchemaIdentity) + + if _, err := store.TransitionManifest(s.ctx, m, publication.StateLoading); !errors.Is(err, publication.ErrInvalidTransition) { + t.Fatalf("STAGED -> LOADING: %v, want ErrInvalidTransition", err) + } + failedSource := mustStagedCopy(m) + failedSource.State = publication.StateFailed + if _, err := store.TransitionManifest(s.ctx, failedSource, publication.StateStaged); !errors.Is(err, publication.ErrInvalidTransition) { + t.Fatalf("FAILED -> STAGED: %v, want ErrInvalidTransition", err) + } + + doomed := loading("gen-fail", digestC) + if _, err := store.CreateManifest(s.ctx, doomed); err != nil { + t.Fatal(err) + } + failed, err := store.TransitionManifest(s.ctx, doomed, publication.StateFailed) + if err != nil { + t.Fatalf("LOADING -> FAILED: %v", err) + } + if failed.State != publication.StateFailed { + t.Fatalf("failed manifest state = %s", failed.State) + } + s.assertManifestRowLiteral(t, doomed.Dataset, "FAILED", doomed.SchemaIdentity) +} + +func TestActivateFirstPointerAndIdempotentReActivation(t *testing.T) { + s := newPGService(t) + store, _ := s.openStore(t) + + const project = "pgstore03-project" + staged := s.mustPersistStaged(t, store, project, "gen-1", digestA) + + if _, err := store.ResolveActiveManifest(s.ctx, project); !errors.Is(err, publication.ErrNoActiveGeneration) { + t.Fatalf("resolve before activation: %v, want ErrNoActiveGeneration", err) + } + s.assertPointerRowLiteral(t, project, "") + + if err := store.ActivateGeneration(s.ctx, staged, nil); err != nil { + t.Fatalf("first activation: %v", err) + } + s.assertPointerRowLiteral(t, project, "gen-1") + + before := s.witnessPointer(t, project) + for i := 0; i < 2; i++ { + if err := store.ActivateGeneration(s.ctx, staged, nil); err != nil { + t.Fatalf("idempotent activation %d: %v", i, err) + } + } + after := s.witnessPointer(t, project) + if after != before { + t.Fatalf("idempotent re-activation rewrote the pointer row: before=%+v after=%+v", before, after) + } + + resolved, err := store.ResolveActiveManifest(s.ctx, project) + if err != nil { + t.Fatalf("resolve after activation: %v", err) + } + if !reflect.DeepEqual(resolved, staged) { + t.Fatalf("ResolveActiveManifest = %+v, want %+v", resolved, staged) + } + + resolvedViaResolver, err := publication.ResolveActive(s.ctx, store, project) + if err != nil || !reflect.DeepEqual(resolvedViaResolver, staged) { + t.Fatalf("ResolveActive = %+v, %v", resolvedViaResolver, err) + } + + other := s.mustPersistStaged(t, store, "pgstore03-other", "gen-1", digestB) + if err := store.ActivateGeneration(s.ctx, other, nil); err != nil { + t.Fatalf("activate other project: %v", err) + } + s.assertPointerRowLiteral(t, project, "gen-1") + s.assertPointerRowLiteral(t, "pgstore03-other", "gen-1") +} + +func TestActivateRequiresStagedAndPersistedCandidate(t *testing.T) { + s := newPGService(t) + store, _ := s.openStore(t) + + const project = "pgstore03-project" + + loadingManifest := func(generation, digest string) publication.Manifest { + m, err := publication.NewManifest(mustRef(t, project, generation), mustSnapshot(t, generation, digest)) + if err != nil { + t.Fatal(err) + } + return m + } + if err := store.ActivateGeneration(s.ctx, loadingManifest("gen-loading", digestA), nil); !errors.Is(err, publication.ErrGenerationNotReady) { + t.Fatalf("activate LOADING: %v, want ErrGenerationNotReady", err) + } + s.assertPointerRowLiteral(t, project, "") + + failedSource := loadingManifest("gen-failed", digestB) + if _, err := store.CreateManifest(s.ctx, failedSource); err != nil { + t.Fatal(err) + } + failed, err := store.TransitionManifest(s.ctx, failedSource, publication.StateFailed) + if err != nil { + t.Fatal(err) + } + if err := store.ActivateGeneration(s.ctx, failed, nil); !errors.Is(err, publication.ErrGenerationNotReady) { + t.Fatalf("activate FAILED: %v, want ErrGenerationNotReady", err) + } + s.assertPointerRowLiteral(t, project, "") + + ghost := loadingManifest("gen-ghost", digestC) + ghost.State = publication.StateStaged + if err := store.ActivateGeneration(s.ctx, ghost, nil); !errors.Is(err, ErrActivationConflict) { + t.Fatalf("activate unpersisted: %v, want ErrActivationConflict", err) + } + + staged := s.mustPersistStaged(t, store, project, "gen-corrupt", digestA) + s.rawExec(t, `UPDATE "`+s.schema+`".dataset_generation_manifest SET state = 'FAILED' WHERE project = $1 AND generation = $2`, + project, "gen-corrupt") + if err := store.ActivateGeneration(s.ctx, staged, nil); !errors.Is(err, ErrActivationConflict) { + t.Fatalf("activate stale staged read: %v, want ErrActivationConflict", err) + } + s.assertPointerRowLiteral(t, project, "") + + staged2 := s.mustPersistStaged(t, store, project, "gen-identity", digestB) + tampered := staged2 + tampered.SchemaIdentity.SchemaSHA256 = digestF + if err := store.ActivateGeneration(s.ctx, tampered, nil); !errors.Is(err, ErrActivationConflict) { + t.Fatalf("activate edited identity: %v, want ErrActivationConflict", err) + } + s.assertPointerRowLiteral(t, project, "") + + if _, err := store.ResolveActiveManifest(s.ctx, project); !errors.Is(err, publication.ErrNoActiveGeneration) { + t.Fatalf("precondition: %v", err) + } + wrongProject := mustRef(t, "pgstore03-other", "gen-identity") + if err := store.ActivateGeneration(s.ctx, staged2, &wrongProject); !errors.Is(err, publication.ErrInvalidActiveGeneration) { + t.Fatalf("cross-project expected: %v, want ErrInvalidActiveGeneration", err) + } +} + +func TestActivateCompareAndSwapAgainstPointerValue(t *testing.T) { + s := newPGService(t) + store, _ := s.openStore(t) + + const project = "pgstore03-project" + g0 := s.mustPersistStaged(t, store, project, "gen-0", digestA) + g1 := s.mustPersistStaged(t, store, project, "gen-1", digestB) + g2 := s.mustPersistStaged(t, store, project, "gen-2", digestC) + + if err := store.ActivateGeneration(s.ctx, g0, nil); err != nil { + t.Fatalf("activate gen-0: %v", err) + } + s.assertPointerRowLiteral(t, project, "gen-0") + + wrongExpected := mustRef(t, project, "gen-9") + if err := store.ActivateGeneration(s.ctx, g1, &wrongExpected); !errors.Is(err, ErrActivationConflict) { + t.Fatalf("activate with wrong expected: %v, want ErrActivationConflict", err) + } + s.assertPointerRowLiteral(t, project, "gen-0") + + if err := store.ActivateGeneration(s.ctx, g1, nil); !errors.Is(err, ErrActivationConflict) { + t.Fatalf("activate expecting no pointer: %v, want ErrActivationConflict", err) + } + s.assertPointerRowLiteral(t, project, "gen-0") + + expectedG0 := g0.Dataset + if err := store.ActivateGeneration(s.ctx, g1, &expectedG0); err != nil { + t.Fatalf("activate gen-1 with expected gen-0: %v", err) + } + s.assertPointerRowLiteral(t, project, "gen-1") + + if err := store.ActivateGeneration(s.ctx, g2, &expectedG0); !errors.Is(err, ErrActivationConflict) { + t.Fatalf("activate gen-2 with stale expected: %v, want ErrActivationConflict", err) + } + s.assertPointerRowLiteral(t, project, "gen-1") + + expectedG1 := g1.Dataset + if err := store.ActivateGeneration(s.ctx, g2, &expectedG1); err != nil { + t.Fatalf("activate gen-2 with expected gen-1: %v", err) + } + s.assertPointerRowLiteral(t, project, "gen-2") +} + +func TestConcurrentActivationsSameExpectedYieldOneWinner(t *testing.T) { + s := newPGService(t) + store, _ := s.openStore(t) + + const project = "pgstore03-project" + g0 := s.mustPersistStaged(t, store, project, "gen-0", digestA) + if err := store.ActivateGeneration(s.ctx, g0, nil); err != nil { + t.Fatalf("baseline activation: %v", err) + } + g1 := s.mustPersistStaged(t, store, project, "gen-1", digestB) + g2 := s.mustPersistStaged(t, store, project, "gen-2", digestC) + + candidates := []publication.Manifest{g1, g2} + results := make([]error, len(candidates)) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := range candidates { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + expected := g0.Dataset + results[i] = store.ActivateGeneration(s.ctx, candidates[i], &expected) + }(i) + } + close(start) + wg.Wait() + + winner, loser := -1, -1 + for i, err := range results { + t.Logf("activation %d (candidate %s): err=%v", i, candidates[i].Dataset.Generation, err) + if err == nil { + if winner != -1 { + t.Fatalf("both concurrent activations won: %v", results) + } + winner = i + continue + } + if !errors.Is(err, ErrActivationConflict) { + t.Fatalf("activation %d failed with untyped error: %v", i, err) + } + if loser != -1 { + t.Fatalf("both concurrent activations lost: %v", results) + } + loser = i + } + if winner == -1 || loser == -1 { + t.Fatalf("expected exactly one winner and one typed-conflict loser, got %v", results) + } + s.assertPointerRowLiteral(t, project, candidates[winner].Dataset.Generation) + + fresh := candidates[winner].Dataset + if err := store.ActivateGeneration(s.ctx, candidates[loser], &fresh); err != nil { + t.Fatalf("loser re-activation with fresh expected: %v", err) + } + s.assertPointerRowLiteral(t, project, candidates[loser].Dataset.Generation) +} + +func TestInjectedFailureBeforeCommitLeavesManifestAndPointerUnchanged(t *testing.T) { + s := newPGService(t) + store, client := s.openStore(t) + + const project = "pgstore03-project" + g0 := s.mustPersistStaged(t, store, project, "gen-0", digestA) + g1 := s.mustPersistStaged(t, store, project, "gen-1", digestB) + if err := store.ActivateGeneration(s.ctx, g0, nil); err != nil { + t.Fatalf("baseline activation: %v", err) + } + pointerBefore := s.witnessPointer(t, project) + + store.commitFault = func(string) error { return errors.New("injected commit failure") } + if err := store.ActivateGeneration(s.ctx, g1, &g0.Dataset); err == nil { + t.Fatal("activation with injected commit fault returned nil") + } else if errors.Is(err, ErrActivationConflict) { + t.Fatalf("commit fault misclassified as CAS conflict: %v", err) + } + store.commitFault = nil + + after := s.witnessPointer(t, project) + if after != pointerBefore { + t.Fatalf("faulted activation moved the pointer: before=%+v after=%+v", pointerBefore, after) + } + s.assertPointerRowLiteral(t, project, "gen-0") + s.assertManifestRowLiteral(t, g0.Dataset, "STAGED", g0.SchemaIdentity) + s.assertManifestRowLiteral(t, g1.Dataset, "STAGED", g1.SchemaIdentity) + + doomed := func() publication.Manifest { + m, err := publication.NewManifest(mustRef(t, project, "gen-faulty"), mustSnapshot(t, "gen-faulty", digestC)) + if err != nil { + t.Fatal(err) + } + return m + }() + if _, err := store.CreateManifest(s.ctx, doomed); err != nil { + t.Fatal(err) + } + store.commitFault = func(string) error { return errors.New("injected commit failure") } + if _, err := store.TransitionManifest(s.ctx, doomed, publication.StateStaged); err == nil { + t.Fatal("transition with injected commit fault returned nil") + } + store.commitFault = nil + s.assertManifestRowLiteral(t, doomed.Dataset, "LOADING", doomed.SchemaIdentity) + + client.Close() + reopened, _ := s.openStore(t) + resolved, err := reopened.ResolveActiveManifest(s.ctx, project) + if err != nil || !reflect.DeepEqual(resolved, g0) { + t.Fatalf("resolve after reopen = %+v, %v; want %+v", resolved, err, g0) + } + if read, err := reopened.ReadManifest(s.ctx, g1.Dataset); err != nil || !reflect.DeepEqual(read, g1) { + t.Fatalf("read gen-1 after reopen = %+v, %v", read, err) + } + if read, err := reopened.ReadManifest(s.ctx, doomed.Dataset); err != nil || read.State != publication.StateLoading { + t.Fatalf("read faulted manifest after reopen = %+v, %v; want LOADING", read, err) + } +} + +func TestSchemaConstraintsRejectInvalidRows(t *testing.T) { + s := newPGService(t) + store, _ := s.openStore(t) + + staged := s.mustPersistStaged(t, store, "pgstore03-project", "gen-fk", digestA) + + type negative struct { + name string + sql string + args []any + wantState string + } + identity := `{"schemaSha256":"` + digestA + `","generatedResourceTypes":["Patient"]}` + cases := []negative{ + { + name: "public state names only", + sql: `INSERT INTO "%s".dataset_generation_manifest (project, generation, state, schema_identity) VALUES ('p', 'g', 'BOGUS', $1::jsonb)`, + args: []any{identity}, + wantState: "23514", + }, + { + name: "state is never SQL NULL", + sql: `INSERT INTO "%s".dataset_generation_manifest (project, generation, state, schema_identity) VALUES ('p', 'g', NULL, $1::jsonb)`, + args: []any{identity}, + wantState: "23502", + }, + { + name: "schema identity is never SQL NULL", + sql: `INSERT INTO "%s".dataset_generation_manifest (project, generation, state, schema_identity) VALUES ('p', 'g', 'LOADING', NULL)`, + args: nil, + wantState: "23502", + }, + { + name: "schema identity must be a JSON object, not JSON null", + sql: `INSERT INTO "%s".dataset_generation_manifest (project, generation, state, schema_identity) VALUES ('p', 'g', 'LOADING', 'null'::jsonb)`, + args: nil, + wantState: "23514", + }, + { + name: "pointer requires an existing manifest row", + sql: `INSERT INTO "%s".dataset_active_generation (project, generation) VALUES ('p', 'missing')`, + args: nil, + wantState: "23503", + }, + { + name: "pointer generation is never SQL NULL", + sql: `INSERT INTO "%s".dataset_active_generation (project, generation) VALUES ('p', NULL)`, + args: nil, + wantState: "23502", + }, + } + for _, c := range cases { + _, err := s.iso.Exec(s.ctx, fmt.Sprintf(c.sql, s.schema), c.args...) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != c.wantState { + t.Errorf("%s: err = %v, want SQLSTATE %s", c.name, s.scrub(err), c.wantState) + } + } + + if err := store.ActivateGeneration(s.ctx, staged, nil); err != nil { + t.Fatalf("activate for FK fixture: %v", err) + } + _, err := s.iso.Exec(s.ctx, + `INSERT INTO "`+s.schema+`".dataset_active_generation (project, generation) VALUES ($1, $2)`, + "pgstore03-project", "gen-fk") + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "23505" { + t.Fatalf("duplicate pointer insert: err = %v, want SQLSTATE 23505", s.scrub(err)) + } +} + +func TestZZNoLeftoverOwnedResources(t *testing.T) { + raw := testPostgresURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %v", err) + } + defer admin.Close() + + var leftoverDBs int + if err := admin.QueryRow(ctx, + `SELECT count(*) FROM pg_database WHERE datname LIKE 'pgstore03\_it\_%' ESCAPE '\'`).Scan(&leftoverDBs); err != nil { + t.Fatalf("count leftover databases: %v", err) + } + if leftoverDBs != 0 { + var names []string + rows, err := admin.Query(ctx, `SELECT datname FROM pg_database WHERE datname LIKE 'pgstore03\_it\_%' ESCAPE '\'`) + if err == nil { + for rows.Next() { + var n string + if err := rows.Scan(&n); err == nil { + names = append(names, n) + } + } + rows.Close() + } + t.Fatalf("%d isolated databases left behind: %v", leftoverDBs, names) + } + + var leftoverSchemas int + if err := admin.QueryRow(ctx, + `SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgstore03\_it\_%' ESCAPE '\'`).Scan(&leftoverSchemas); err != nil { + t.Fatalf("count leftover schemas: %v", err) + } + if leftoverSchemas != 0 { + t.Fatalf("%d owned schemas left behind in the service database", leftoverSchemas) + } +} diff --git a/internal/backends/postgres/internal/dataset/migrate.go b/internal/backends/postgres/internal/dataset/migrate.go new file mode 100644 index 000000000..a040f6999 --- /dev/null +++ b/internal/backends/postgres/internal/dataset/migrate.go @@ -0,0 +1,51 @@ +package postgres + +import ( + "context" + "embed" + "errors" + "fmt" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" +) + +//go:embed migrations/*.sql +var migrationsFS embed.FS + +func OwnerMigrations() ([]storepostgres.Migration, error) { + migrations, err := storepostgres.LoadMigrations(migrationsFS, "migrations") + if err != nil { + return nil, fmt.Errorf("dataset postgres: load owner migrations: %w", err) + } + return migrations, nil +} + +func MigrateSchema(ctx context.Context, src storepostgres.ConnSource, schema string) (storepostgres.MigrateResult, error) { + if src == nil { + return storepostgres.MigrateResult{}, errors.New("dataset postgres: nil migration connection source") + } + migrations, err := OwnerMigrations() + if err != nil { + return storepostgres.MigrateResult{}, err + } + runner, err := storepostgres.NewMigrationRunner(src, schema, migrations) + if err != nil { + return storepostgres.MigrateResult{}, err + } + return runner.Migrate(ctx) +} + +func VerifySchema(ctx context.Context, src storepostgres.ConnSource, schema string) error { + if src == nil { + return errors.New("dataset postgres: nil migration connection source") + } + migrations, err := OwnerMigrations() + if err != nil { + return err + } + runner, err := storepostgres.NewMigrationRunner(src, schema, migrations) + if err != nil { + return err + } + return runner.Verify(ctx) +} diff --git a/internal/backends/postgres/internal/dataset/migrations/000001_ledger.sql b/internal/backends/postgres/internal/dataset/migrations/000001_ledger.sql new file mode 100644 index 000000000..13c0efcfc --- /dev/null +++ b/internal/backends/postgres/internal/dataset/migrations/000001_ledger.sql @@ -0,0 +1,17 @@ +-- pg-store-02: create the migration ledger. +-- +-- The ledger is itself a versioned migration so its creation is audited by +-- the same runner that reads it. It stores version, name, and checksum only +-- (plus when it was applied) -- never the migration SQL content. The runner +-- inserts each row only after the migration transaction commits, so a +-- recorded migration always applied successfully. +CREATE TABLE schema_migrations ( + version bigint NOT NULL, + name text NOT NULL, + checksum text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT schema_migrations_pkey PRIMARY KEY (version), + CONSTRAINT schema_migrations_version_positive CHECK (version > 0), + CONSTRAINT schema_migrations_name_shape CHECK (name ~ '^[a-z][a-z0-9_]*$'), + CONSTRAINT schema_migrations_checksum_hex CHECK (checksum ~ '^[0-9a-f]{64}$') +); diff --git a/internal/backends/postgres/internal/dataset/migrations/000003_dataset_generation_manifest.sql b/internal/backends/postgres/internal/dataset/migrations/000003_dataset_generation_manifest.sql new file mode 100644 index 000000000..e7dae84ca --- /dev/null +++ b/internal/backends/postgres/internal/dataset/migrations/000003_dataset_generation_manifest.sql @@ -0,0 +1,62 @@ +-- pg-store-03: dataset generation manifests and the active-generation pointer. +-- +-- One row per dataset generation (project, generation) holds the lifecycle +-- state and the schema identity document that participates in +-- compare-and-swap equality. The active-generation pointer is one row per +-- project: a missing row means no active generation (there is no placeholder +-- row and no SQL NULL standing in for "absent"). Presence of the pointer row +-- is the compare value for activation CAS; the store never moves it outside +-- its transactional compare-and-swap. +-- +-- State values are the public dataset manifest state names, byte-for-byte: +-- LOADING, STAGED, READY (legacy spelling of STAGED), FAILED. schema_identity +-- stores the manifest's schemaIdentity JSON document; SQL NULL is not +-- admitted for any column here -- missing data is a constraint violation, +-- never a silent NULL. +-- +-- DDL is retryable (IF NOT EXISTS) per the pg-store-02 runner contract: a +-- crash between a migration's commit and its ledger record must converge on +-- re-application. +-- +-- Version window: 000002_fhir_resource (pg-store-04) landed on the +-- integration branch alongside this 000003, so the embedded history is the +-- complete {1, 2, 3} chain and the runner's out-of-order policy cannot trap +-- a schema (no commit state ever exposes 000003 without 000002). Per-test +-- fixtures always start from an empty ledger. The runner's pinned-checksum +-- gate in migrate_test.go refuses any edit to this file's bytes without an +-- explicit entry update. +CREATE TABLE IF NOT EXISTS dataset_generation_manifest ( + project text NOT NULL, + generation text NOT NULL, + state text NOT NULL, + schema_identity jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT dataset_generation_manifest_pkey + PRIMARY KEY (project, generation), + CONSTRAINT dataset_generation_manifest_project_nonempty + CHECK (project <> ''), + CONSTRAINT dataset_generation_manifest_generation_nonempty + CHECK (generation <> ''), + CONSTRAINT dataset_generation_manifest_state_public_names + CHECK (state IN ('LOADING', 'STAGED', 'READY', 'FAILED')), + CONSTRAINT dataset_generation_manifest_identity_is_object + CHECK (jsonb_typeof(schema_identity) = 'object') +); + +CREATE TABLE IF NOT EXISTS dataset_active_generation ( + project text NOT NULL, + generation text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT dataset_active_generation_pkey + PRIMARY KEY (project), + CONSTRAINT dataset_active_generation_project_nonempty + CHECK (project <> ''), + CONSTRAINT dataset_active_generation_generation_nonempty + CHECK (generation <> ''), + CONSTRAINT dataset_active_generation_manifest_fkey + FOREIGN KEY (project, generation) + REFERENCES dataset_generation_manifest (project, generation) + ON DELETE RESTRICT +); diff --git a/internal/backends/postgres/internal/dataset/migrations/000008_project_release.sql b/internal/backends/postgres/internal/dataset/migrations/000008_project_release.sql new file mode 100644 index 000000000..0f17966e5 --- /dev/null +++ b/internal/backends/postgres/internal/dataset/migrations/000008_project_release.sql @@ -0,0 +1,97 @@ +-- pg-publication-project-release-cas-v1: immutable ProjectRelease candidates +-- and the active-project-release compare-and-swap pointer. +-- +-- Ownership: this lane owns exactly these two tables. The dataset generation +-- manifest and the active-generation pointer belong to pg-store-03 +-- (000003_dataset_generation_manifest.sql) and are NOT duplicated here; the +-- generation owner participates in release activation through the +-- ReleaseStore's GenerationOwnerAdapter inside the same PostgreSQL +-- transaction, reading its own rows -- no mirrored manifest or generation +-- columns are added to either side. ClickHouse remains the publication row +-- store: no dataframe rows move here, and Arango/AQL stay supported. +-- +-- Version reservation: 000006 and 000007 are coordinator-reserved shared +-- numbers (tests/migrations-leases holds 000006). This fixture is +-- package-local (loaded only by this package's tests through the pg-store-02 +-- runner alongside the embedded ledger bootstrap and migration 3) and takes +-- 000008 so it composes without colliding with the shared reservations. +-- Promotion to a shared migrations directory requires coordinator approval. +-- +-- project_release is the immutable candidate archive: +-- * One row per (project, content-addressed release id). The id is an +-- opaque immutable content label minted by internal/dataset +-- (releaseID, "release_" + SHA-256 hex); this schema deliberately adds +-- no id-format CHECK so pre-widening digests and any future framing +-- change stay readable, matching the reader-side treatment in +-- internal/dataset/release.go. +-- * payload stores the complete ProjectRelease JSON document exactly as +-- saved. It is authoritative on read, so nanosecond timestamps and the +-- JSON null-versus-absent shape survive a save/read round trip +-- unchanged; every column is NOT NULL and no statement binds SQL NULL, +-- so missing, JSON-null, and SQL-NULL stay distinct. +-- * generation and git_commit are scan columns derived from the same +-- struct fields the payload carries; the store verifies column/payload +-- agreement on every read and refuses disagreement as a store +-- inconsistency instead of coercing it. +-- * Rows are never updated or deleted: inactive candidates stay retained +-- for audit, so a lost activation race never destroys the candidate. +-- +-- active_project_release is the active-visibility pointer: +-- * One row per project. A missing row means no active release -- there is +-- no placeholder row and no SQL NULL standing in for "absent", matching +-- the pg-store-03 pointer convention. +-- * revision is the CAS counter compared inside the activation +-- transaction: it starts at 1 (revision >= 1; callers express "no +-- active release yet" as expectedRevision 0, exactly as the Arango +-- baseline's releasePointer == null ? 0 rule does) and only ever moves +-- through the store's compare-and-swap. The foreign key makes "the +-- active pointer names a persisted candidate" a schema invariant, +-- mirroring how 000003 makes the active-generation manifest a schema +-- invariant. +-- +-- DDL is retryable (IF NOT EXISTS) per the pg-store-02 runner contract: a +-- crash between a migration's commit and its ledger record must converge on +-- re-application. +CREATE TABLE IF NOT EXISTS project_release ( + project text NOT NULL, + id text NOT NULL, + generation text NOT NULL, + git_commit text NOT NULL, + payload jsonb NOT NULL, + CONSTRAINT project_release_pkey + PRIMARY KEY (project, id), + CONSTRAINT project_release_project_nonempty + CHECK (project <> ''), + CONSTRAINT project_release_id_nonempty + CHECK (id <> ''), + CONSTRAINT project_release_generation_nonempty + CHECK (generation <> ''), + CONSTRAINT project_release_git_commit_nonempty + CHECK (git_commit <> ''), + CONSTRAINT project_release_payload_is_object + CHECK (jsonb_typeof(payload) = 'object') +); + +-- Candidate lookup by audit scope: which candidates were ever retained for a +-- project+generation, newest identity first by id. +CREATE INDEX IF NOT EXISTS project_release_project_generation + ON project_release (project, generation); + +CREATE TABLE IF NOT EXISTS active_project_release ( + project text NOT NULL, + release_id text NOT NULL, + revision bigint NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT active_project_release_pkey + PRIMARY KEY (project), + CONSTRAINT active_project_release_project_nonempty + CHECK (project <> ''), + CONSTRAINT active_project_release_release_id_nonempty + CHECK (release_id <> ''), + CONSTRAINT active_project_release_revision_positive + CHECK (revision >= 1), + CONSTRAINT active_project_release_release_fkey + FOREIGN KEY (project, release_id) + REFERENCES project_release (project, id) +); diff --git a/internal/backends/postgres/internal/dataset/release_store.go b/internal/backends/postgres/internal/dataset/release_store.go new file mode 100644 index 000000000..7e05bcc42 --- /dev/null +++ b/internal/backends/postgres/internal/dataset/release_store.go @@ -0,0 +1,538 @@ +package postgres + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "reflect" + + publication "github.com/calypr/loom/internal/dataset" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + ProjectReleasesTable = "project_release" + ActiveProjectReleasesTable = "active_project_release" +) + +var ( + ErrNilReleaseConnSource = errors.New("dataset postgres: release store connection source is required") + ErrReleaseContentConflict = errors.New("project release candidate already exists with different content") + ErrReleaseActivationConflict = publication.ErrReleaseActivationConflict +) + +var _ publication.ReleaseRepository = (*ReleaseStore)(nil) + +type ReleaseActivation struct { + Project string + Release publication.ProjectRelease + Revision int64 +} + +type GenerationOwnerAdapter interface { + EnsureGenerationPrerequisites(ctx context.Context, tx pgx.Tx, project, generation string) error + OnReleaseActivated(ctx context.Context, tx pgx.Tx, activation ReleaseActivation) error +} + +type ReleaseStore struct { + src ConnSource + releases string + pointers string + adapter GenerationOwnerAdapter + commitFault func(step string) error +} + +func NewReleaseStore(src ConnSource, schema string) (*ReleaseStore, error) { + if src == nil { + return nil, ErrNilReleaseConnSource + } + if !storepostgres.ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", ErrInvalidSchemaName, schema) + } + s := &ReleaseStore{ + src: src, + releases: pgx.Identifier{schema, ProjectReleasesTable}.Sanitize(), + pointers: pgx.Identifier{schema, ActiveProjectReleasesTable}.Sanitize(), + } + s.adapter = &datasetGenerationOwner{ + manifests: pgx.Identifier{schema, "dataset_generation_manifest"}.Sanitize(), + pointer: pgx.Identifier{schema, "dataset_active_generation"}.Sanitize(), + } + return s, nil +} + +func (s *ReleaseStore) validate() error { + if s == nil || s.src == nil { + return ErrNilReleaseConnSource + } + return nil +} + +func validateRelease(release publication.ProjectRelease) error { + if release.ID == "" { + return fmt.Errorf("project release: candidate id is required") + } + ref := publication.Ref{Project: release.Project, Generation: release.Generation} + if err := ref.Validate(); err != nil { + return fmt.Errorf("project release: %w", err) + } + if release.GitCommit == "" { + return fmt.Errorf("project release: gitCommit is required") + } + for _, binding := range release.Publications { + if err := binding.Selector.Validate(); err != nil { + return fmt.Errorf("project release: publication selector %v: %w", binding.Selector, err) + } + } + for _, verification := range release.RequiredVerifications { + if err := verification.Selector.Validate(); err != nil { + return fmt.Errorf("project release: verification selector %v: %w", verification.Selector, err) + } + } + return nil +} + +func marshalReleasePayload(release publication.ProjectRelease) (string, error) { + normalized := release + if normalized.Publications == nil { + normalized.Publications = []publication.ReleasePublication{} + } + if normalized.RequiredVerifications == nil { + normalized.RequiredVerifications = []publication.ContractVerification{} + } + encoded, err := json.Marshal(normalized) + if err != nil { + return "", fmt.Errorf("encode project release payload: %w", err) + } + return string(encoded), nil +} + +func releaseFromRow(project, id, generation, gitCommit, payloadDB string) (publication.ProjectRelease, error) { + var release publication.ProjectRelease + decoder := json.NewDecoder(bytes.NewReader([]byte(payloadDB))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&release); err != nil { + return publication.ProjectRelease{}, fmt.Errorf("%w: stored release payload is malformed: %v", ErrUnexpectedStoreResult, err) + } + if err := validateRelease(release); err != nil { + return publication.ProjectRelease{}, fmt.Errorf("%w: stored release is invalid: %v", ErrUnexpectedStoreResult, err) + } + if release.Project != project || release.ID != id || release.Generation != generation || release.GitCommit != gitCommit { + return publication.ProjectRelease{}, fmt.Errorf("%w: stored release scan columns disagree with its payload for %s/%s", + ErrUnexpectedStoreResult, project, id) + } + return release, nil +} + +func (s *ReleaseStore) SaveRelease(ctx context.Context, release publication.ProjectRelease) (publication.ProjectRelease, error) { + if err := s.validate(); err != nil { + return publication.ProjectRelease{}, err + } + if err := validateRelease(release); err != nil { + return publication.ProjectRelease{}, err + } + payload, err := marshalReleasePayload(release) + if err != nil { + return publication.ProjectRelease{}, err + } + + insert, err := NewPostgreSQLCommand( + "INSERT INTO "+s.releases+` (project, id, generation, git_commit, payload) +VALUES ($1, $2, $3, $4, $5::jsonb) +ON CONFLICT DO NOTHING +RETURNING payload::text`, + []Parameter{ + {Type: ParameterText, Value: release.Project}, + {Type: ParameterText, Value: release.ID}, + {Type: ParameterText, Value: release.Generation}, + {Type: ParameterText, Value: release.GitCommit}, + {Type: ParameterJSONB, Value: payload}, + }) + if err != nil { + return publication.ProjectRelease{}, fmt.Errorf("save project release: %w", err) + } + + conn, err := s.src.Acquire(ctx) + if err != nil { + return publication.ProjectRelease{}, fmt.Errorf("save project release: acquire connection: %w", err) + } + defer conn.Release() + + var storedPayload string + scanErr := queryRowConn(ctx, conn, insert).Scan(&storedPayload) + if errors.Is(scanErr, pgx.ErrNoRows) { + return s.savedCandidateConflict(ctx, conn, release, payload) + } + if scanErr != nil { + return publication.ProjectRelease{}, fmt.Errorf("save project release: %w", scanErr) + } + + stored, err := releaseFromRow(release.Project, release.ID, release.Generation, release.GitCommit, storedPayload) + if err != nil { + return publication.ProjectRelease{}, fmt.Errorf("save project release: %w", err) + } + var submitted publication.ProjectRelease + if err := json.NewDecoder(bytes.NewReader([]byte(payload))).Decode(&submitted); err != nil { + return publication.ProjectRelease{}, fmt.Errorf("save project release: %w", err) + } + if !reflect.DeepEqual(stored, submitted) { + return publication.ProjectRelease{}, fmt.Errorf("%w: stored release does not match the submitted candidate", ErrUnexpectedStoreResult) + } + return stored, nil +} + +func (s *ReleaseStore) savedCandidateConflict(ctx context.Context, conn *pgxpool.Conn, release publication.ProjectRelease, payload string) (publication.ProjectRelease, error) { + lookup, err := NewPostgreSQLCommand( + "SELECT (payload - 'createdAt') = ($2::jsonb - 'createdAt'), payload::text FROM "+s.releases+` +WHERE project = $1 AND id = $3`, + []Parameter{ + {Type: ParameterText, Value: release.Project}, + {Type: ParameterJSONB, Value: payload}, + {Type: ParameterText, Value: release.ID}, + }) + if err != nil { + return publication.ProjectRelease{}, fmt.Errorf("save project release: %w", err) + } + var ( + compatible bool + storedPayload string + ) + scanErr := queryRowConn(ctx, conn, lookup).Scan(&compatible, &storedPayload) + if errors.Is(scanErr, pgx.ErrNoRows) { + return publication.ProjectRelease{}, fmt.Errorf("%w: release row vanished during the save retry race", ErrUnexpectedStoreResult) + } + if scanErr != nil { + return publication.ProjectRelease{}, fmt.Errorf("save project release: %w", scanErr) + } + if !compatible { + return publication.ProjectRelease{}, fmt.Errorf("%w: %s/%s already stores a different document", + ErrReleaseContentConflict, release.Project, release.ID) + } + stored, err := releaseFromRow(release.Project, release.ID, release.Generation, release.GitCommit, storedPayload) + if err != nil { + return publication.ProjectRelease{}, fmt.Errorf("save project release: %w", err) + } + return stored, nil +} + +func (s *ReleaseStore) ReadActiveRelease(ctx context.Context, project string) (publication.ActiveRelease, error) { + if err := validateProject(project); err != nil { + return publication.ActiveRelease{}, fmt.Errorf("read active release: %w", err) + } + if err := s.validate(); err != nil { + return publication.ActiveRelease{}, err + } + command, err := NewPostgreSQLCommand( + "SELECT r.project, r.id, r.generation, r.git_commit, r.payload::text, p.revision FROM "+s.pointers+` p +JOIN `+s.releases+` r + ON r.project = p.project AND r.id = p.release_id +WHERE p.project = $1`, + []Parameter{ + {Type: ParameterText, Value: project}, + }) + if err != nil { + return publication.ActiveRelease{}, fmt.Errorf("read active release: %w", err) + } + + conn, err := s.src.Acquire(ctx) + if err != nil { + return publication.ActiveRelease{}, fmt.Errorf("read active release: acquire connection: %w", err) + } + defer conn.Release() + + var ( + rowProject, rowID, rowGeneration, rowGitCommit string + storedPayload string + revision int64 + ) + scanErr := queryRowConn(ctx, conn, command).Scan(&rowProject, &rowID, &rowGeneration, &rowGitCommit, &storedPayload, &revision) + if errors.Is(scanErr, pgx.ErrNoRows) { + return publication.ActiveRelease{}, fmt.Errorf("%w: %s", publication.ErrNoActiveRelease, project) + } + if scanErr != nil { + return publication.ActiveRelease{}, fmt.Errorf("read active release: %w", scanErr) + } + release, err := releaseFromRow(rowProject, rowID, rowGeneration, rowGitCommit, storedPayload) + if err != nil { + return publication.ActiveRelease{}, fmt.Errorf("read active release: %w", err) + } + if revision < 1 { + return publication.ActiveRelease{}, fmt.Errorf("%w: active release revision %d is below the stored floor", ErrUnexpectedStoreResult, revision) + } + return publication.ActiveRelease{Release: release, Revision: revision}, nil +} + +func (s *ReleaseStore) CompareAndSwapActivateRelease(ctx context.Context, release publication.ProjectRelease, expectedRevision int64) (publication.ActiveRelease, error) { + if err := s.validate(); err != nil { + return publication.ActiveRelease{}, err + } + if err := validateRelease(release); err != nil { + return publication.ActiveRelease{}, err + } + if expectedRevision < 0 { + return publication.ActiveRelease{}, fmt.Errorf("activate release: expectedRevision must not be negative, got %d", expectedRevision) + } + payload, err := marshalReleasePayload(release) + if err != nil { + return publication.ActiveRelease{}, fmt.Errorf("activate release: %w", err) + } + project := release.Project + generation := release.Generation + + lockCandidate, err := NewPostgreSQLCommand( + "SELECT payload::text, (payload - 'createdAt') = ($3::jsonb - 'createdAt') FROM "+s.releases+` +WHERE project = $1 AND id = $2 +FOR SHARE`, + []Parameter{ + {Type: ParameterText, Value: project}, + {Type: ParameterText, Value: release.ID}, + {Type: ParameterJSONB, Value: payload}, + }) + if err != nil { + return publication.ActiveRelease{}, fmt.Errorf("activate release: %w", err) + } + lockPointer, err := NewPostgreSQLCommand( + "SELECT release_id, revision FROM "+s.pointers+` +WHERE project = $1 +FOR UPDATE`, + []Parameter{ + {Type: ParameterText, Value: project}, + }) + if err != nil { + return publication.ActiveRelease{}, fmt.Errorf("activate release: %w", err) + } + readPointer, err := NewPostgreSQLCommand( + "SELECT release_id, revision FROM "+s.pointers+` +WHERE project = $1`, + []Parameter{ + {Type: ParameterText, Value: project}, + }) + if err != nil { + return publication.ActiveRelease{}, fmt.Errorf("activate release: %w", err) + } + insertPointer, err := NewPostgreSQLCommand( + "INSERT INTO "+s.pointers+` (project, release_id, revision) +VALUES ($1, $2, 1) +ON CONFLICT DO NOTHING`, + []Parameter{ + {Type: ParameterText, Value: project}, + {Type: ParameterText, Value: release.ID}, + }) + if err != nil { + return publication.ActiveRelease{}, fmt.Errorf("activate release: %w", err) + } + + var result *publication.ActiveRelease + err = s.inTx(ctx, "activate-release", func(tx pgx.Tx) error { + var ( + storedPayload string + matches bool + ) + scanErr := queryRowCommand(ctx, tx, lockCandidate).Scan(&storedPayload, &matches) + if errors.Is(scanErr, pgx.ErrNoRows) { + return fmt.Errorf("%w: candidate %s/%s is not a persisted project release", + ErrReleaseActivationConflict, project, release.ID) + } + if scanErr != nil { + return scanErr + } + if !matches { + return fmt.Errorf("%w: candidate %s/%s no longer matches the persisted document", + ErrReleaseActivationConflict, project, release.ID) + } + candidate, err := releaseFromRow(project, release.ID, generation, release.GitCommit, storedPayload) + if err != nil { + return err + } + + if err := s.adapter.EnsureGenerationPrerequisites(ctx, tx, project, generation); err != nil { + return err + } + + var ( + pointerReleaseID string + currentRevision int64 + ) + pointerErr := queryRowCommand(ctx, tx, lockPointer).Scan(&pointerReleaseID, ¤tRevision) + if pointerErr != nil && !errors.Is(pointerErr, pgx.ErrNoRows) { + return pointerErr + } + hasPointer := pointerErr == nil + + if hasPointer && pointerReleaseID == release.ID { + result = &publication.ActiveRelease{Release: candidate, Revision: currentRevision} + return nil + } + + if hasPointer { + if currentRevision != expectedRevision { + return fmt.Errorf("%w: candidate %s/%s expected active revision %d, found %d", + ErrReleaseActivationConflict, project, release.ID, expectedRevision, currentRevision) + } + updatePointer, err := newCASPointer(s.pointers, project, release.ID, expectedRevision) + if err != nil { + return err + } + var committedRevision int64 + casErr := queryRowCommand(ctx, tx, updatePointer).Scan(&committedRevision) + if errors.Is(casErr, pgx.ErrNoRows) { + return fmt.Errorf("%w: active release pointer moved during activation", ErrReleaseActivationConflict) + } + if casErr != nil { + return casErr + } + if committedRevision != expectedRevision+1 { + return fmt.Errorf("%w: active release revision committed as %d, want %d", + ErrUnexpectedStoreResult, committedRevision, expectedRevision+1) + } + result = &publication.ActiveRelease{Release: candidate, Revision: committedRevision} + return s.adapter.OnReleaseActivated(ctx, tx, ReleaseActivation{Project: project, Release: candidate, Revision: committedRevision}) + } + + if expectedRevision != 0 { + return fmt.Errorf("%w: candidate %s/%s expected active revision %d, found no active release", + ErrReleaseActivationConflict, project, release.ID, expectedRevision) + } + tag, err := execCommand(ctx, tx, insertPointer) + if err != nil { + return err + } + if tag.RowsAffected() == 1 { + result = &publication.ActiveRelease{Release: candidate, Revision: 1} + return s.adapter.OnReleaseActivated(ctx, tx, ReleaseActivation{Project: project, Release: candidate, Revision: 1}) + } + var concurrentReleaseID string + var concurrentRevision int64 + rereadErr := queryRowCommand(ctx, tx, readPointer).Scan(&concurrentReleaseID, &concurrentRevision) + if errors.Is(rereadErr, pgx.ErrNoRows) { + return fmt.Errorf("%w: active release pointer vanished during activation", ErrUnexpectedStoreResult) + } + if rereadErr != nil { + return rereadErr + } + if concurrentReleaseID != release.ID { + return fmt.Errorf("%w: candidate %s/%s expected no active release, found %s at revision %d (concurrent activation)", + ErrReleaseActivationConflict, project, release.ID, concurrentReleaseID, concurrentRevision) + } + result = &publication.ActiveRelease{Release: candidate, Revision: concurrentRevision} + return nil + }) + if err != nil { + return publication.ActiveRelease{}, fmt.Errorf("activate release: %w", err) + } + if result == nil { + return publication.ActiveRelease{}, fmt.Errorf("%w: activation produced no verdict", ErrUnexpectedStoreResult) + } + return *result, nil +} + +func newCASPointer(pointersFQN, project, releaseID string, expectedRevision int64) (Command, error) { + return NewPostgreSQLCommand( + "UPDATE "+pointersFQN+` +SET release_id = $2, revision = $3, updated_at = now() +WHERE project = $1 AND revision = $4 +RETURNING revision`, + []Parameter{ + {Type: ParameterText, Value: project}, + {Type: ParameterText, Value: releaseID}, + {Type: ParameterText, Value: fmt.Sprintf("%d", expectedRevision+1)}, + {Type: ParameterText, Value: fmt.Sprintf("%d", expectedRevision)}, + }) +} + +func (s *ReleaseStore) inTx(ctx context.Context, step string, body func(pgx.Tx) error) (err error) { + conn, err := s.src.Acquire(ctx) + if err != nil { + return fmt.Errorf("acquire connection: %w", err) + } + defer conn.Release() + + tx, err := conn.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return fmt.Errorf("begin transaction: %w", err) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback(ctx) + } + }() + + if err := body(tx); err != nil { + return err + } + if s.commitFault != nil { + if err := s.commitFault(step); err != nil { + return fmt.Errorf("injected fault before commit: %w", err) + } + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("commit transaction: %w", err) + } + committed = true + return nil +} + +type datasetGenerationOwner struct { + manifests string + pointer string +} + +var _ GenerationOwnerAdapter = (*datasetGenerationOwner)(nil) + +func (g *datasetGenerationOwner) EnsureGenerationPrerequisites(ctx context.Context, tx pgx.Tx, project, generation string) error { + lockManifest, err := NewPostgreSQLCommand( + "SELECT state FROM "+g.manifests+` +WHERE project = $1 AND generation = $2 +FOR SHARE`, + []Parameter{ + {Type: ParameterText, Value: project}, + {Type: ParameterText, Value: generation}, + }) + if err != nil { + return fmt.Errorf("activate release: %w", err) + } + var state string + scanErr := queryRowCommand(ctx, tx, lockManifest).Scan(&state) + if errors.Is(scanErr, pgx.ErrNoRows) { + return fmt.Errorf("%w: generation %s/%s has no persisted manifest", + ErrReleaseActivationConflict, project, generation) + } + if scanErr != nil { + return fmt.Errorf("activate release: %w", scanErr) + } + if state != string(publication.StateStaged) && state != string(publication.StateReady) { + return fmt.Errorf("%w: generation %s/%s manifest is %s, not %s or %s", + ErrReleaseActivationConflict, project, generation, state, + string(publication.StateStaged), string(publication.StateReady)) + } + + lockPointer, err := NewPostgreSQLCommand( + "SELECT generation FROM "+g.pointer+` +WHERE project = $1 +FOR SHARE`, + []Parameter{ + {Type: ParameterText, Value: project}, + }) + if err != nil { + return fmt.Errorf("activate release: %w", err) + } + var activeGeneration string + scanErr = queryRowCommand(ctx, tx, lockPointer).Scan(&activeGeneration) + if errors.Is(scanErr, pgx.ErrNoRows) { + return fmt.Errorf("%w: project %s has no active generation", + ErrReleaseActivationConflict, project) + } + if scanErr != nil { + return fmt.Errorf("activate release: %w", scanErr) + } + return nil +} + +func (g *datasetGenerationOwner) OnReleaseActivated(context.Context, pgx.Tx, ReleaseActivation) error { + return nil +} diff --git a/internal/backends/postgres/internal/dataset/release_store_pg_test.go b/internal/backends/postgres/internal/dataset/release_store_pg_test.go new file mode 100644 index 000000000..d6fc8fbf2 --- /dev/null +++ b/internal/backends/postgres/internal/dataset/release_store_pg_test.go @@ -0,0 +1,1146 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "net/url" + "reflect" + "regexp" + "strings" + "sync" + "testing" + "time" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + publication "github.com/calypr/loom/internal/dataset" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +var pgpubrel01ResourceNamePattern = regexp.MustCompile(`^pgpubrel01_it_[0-9]+_[0-9a-f]+$`) + +type releaseService struct { + ctx context.Context + admin *pgxpool.Pool + witness *pgxpool.Pool + poolA *pgxpool.Pool + poolB *pgxpool.Pool + pidA int32 + pidB int32 + pidW int32 + schema string + scrub func(any) string + fail func(step string, err error) + + releases *ReleaseStore + releasesB *ReleaseStore + gen *Store +} + +func newReleaseService(t *testing.T) *releaseService { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgpubrel01_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgpubrel01ResourceNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if _, err := admin.Exec(cleanupCtx, + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + newPool := func(label string) *pgxpool.Pool { + config, err := pgxpool.ParseConfig(dsn) + if err != nil { + fail("parse pool config "+label, err) + } + config.MaxConns = 1 + pool, err := pgxpool.NewWithConfig(ctx, config) + if err != nil { + fail("connect pool "+label, err) + } + t.Cleanup(pool.Close) + return pool + } + witness := newPool("witness") + poolA := newPool("writer-a") + poolB := newPool("writer-b") + + schema := fmt.Sprintf("pgpubrel01_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgpubrel01ResourceNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := witness.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if _, err := witness.Exec(cleanupCtx, + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + s := &releaseService{ + ctx: ctx, admin: admin, witness: witness, poolA: poolA, poolB: poolB, + schema: schema, scrub: scrub, fail: fail, + } + s.migrate(t) + s.pidA = s.backendPID(t, poolA, "writer A") + s.pidB = s.backendPID(t, poolB, "writer B") + s.pidW = s.backendPID(t, witness, "witness") + if s.pidA == s.pidB || s.pidA == s.pidW || s.pidB == s.pidW { + t.Fatalf("writer and witness backends are not distinct physical connections: A=%d B=%d W=%d", s.pidA, s.pidB, s.pidW) + } + + releases, err := NewReleaseStore(poolA, schema) + if err != nil { + fail("build release store A", err) + } + s.releases = releases + releasesB, err := NewReleaseStore(poolB, schema) + if err != nil { + fail("build release store B", err) + } + s.releasesB = releasesB + gen, err := New(witness, schema) + if err != nil { + fail("build generation store", err) + } + s.gen = gen + return s +} + +func (s *releaseService) backendPID(t *testing.T, pool *pgxpool.Pool, label string) int32 { + t.Helper() + var pid int32 + if err := pool.QueryRow(s.ctx, `SELECT pg_backend_pid()`).Scan(&pid); err != nil { + s.fail("backend pid "+label, err) + } + if pid <= 0 { + t.Fatalf("%s backend pid = %d, want a positive server pid", label, pid) + } + return pid +} + +func (s *releaseService) migrate(t *testing.T) { + t.Helper() + history, err := OwnerMigrations() + if err != nil { + s.fail("owner migrations", err) + } + if len(history) != 3 || + history[0].Version != 1 || history[0].Name != "ledger" || + history[1].Version != 3 || history[1].Name != "dataset_generation_manifest" || + history[2].Version != 8 || history[2].Name != "project_release" { + t.Fatalf("owner history = %+v, want exactly {1 ledger, 3 dataset_generation_manifest, 8 project_release}", history) + } + runner, err := storepostgres.NewMigrationRunner(s.witness, s.schema, history) + if err != nil { + s.fail("new migration runner", err) + } + result, err := runner.Migrate(s.ctx) + if err != nil { + s.fail("migrate release history", err) + } + if result.LatestVersion != 8 { + t.Fatalf("latest migration version = %d, want 8", result.LatestVersion) + } + wantLedger := []storepostgres.LedgerRecord{ + {Version: 1, Name: "ledger", Checksum: "db70f7758495f7557b3a694149b80ed6e5d4cb74ea5aa7d59a1a2eaa46fc9cde"}, + {Version: 3, Name: "dataset_generation_manifest", Checksum: "06c614680a9c1cd4ba2ef0ac0471053bcb3d8e471c36d3852e70682887a72f5d"}, + {Version: 8, Name: "project_release", Checksum: "1d581ec39731958dd0f79cdab18e92861a380007d3e85d6de4940800378480b1"}, + } + if len(result.Applied) != len(wantLedger) { + t.Fatalf("applied history = %+v, want %+v", result.Applied, wantLedger) + } + for i, want := range wantLedger { + got := result.Applied[i] + if got.Version != want.Version || got.Name != want.Name || got.Checksum != want.Checksum { + t.Fatalf("applied[%d] = {Version:%d Name:%q Checksum:%q}, want {Version:%d Name:%q Checksum:%q}", + i, got.Version, got.Name, got.Checksum, want.Version, want.Name, want.Checksum) + } + } +} + +func (s *releaseService) qualified(table string) string { + return pgx.Identifier{s.schema, table}.Sanitize() +} + +func (s *releaseService) mustExec(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) { + t.Helper() + if _, err := pool.Exec(s.ctx, sql, args...); err != nil { + s.fail("exec "+sql, err) + } +} + +func (s *releaseService) execCount(t *testing.T, sql string, args ...any) int { + t.Helper() + var n int + if err := s.witness.QueryRow(s.ctx, sql, args...).Scan(&n); err != nil { + s.fail("count "+sql, err) + } + return n +} + +func (s *releaseService) mustWitnessBool(t *testing.T, sql string, args ...any) bool { + t.Helper() + var value bool + if err := s.witness.QueryRow(s.ctx, sql, args...).Scan(&value); err != nil { + s.fail("witness "+sql, err) + } + return value +} + +func (s *releaseService) mustWitnessText(t *testing.T, sql string, args ...any) string { + t.Helper() + var value string + if err := s.witness.QueryRow(s.ctx, sql, args...).Scan(&value); err != nil { + s.fail("witness "+sql, err) + } + return value +} + +type pointerRow struct { + project string + releaseID string + revision int64 + createdAt string + updatedAt string + ctid string + exists bool +} + +func (s *releaseService) witnessPointer(project string) pointerRow { + var w pointerRow + err := s.witness.QueryRow(s.ctx, + `SELECT project, release_id, revision, created_at::text, updated_at::text, ctid::text FROM `+s.qualified(ActiveProjectReleasesTable)+` +WHERE project = $1`, project). + Scan(&w.project, &w.releaseID, &w.revision, &w.createdAt, &w.updatedAt, &w.ctid) + if errors.Is(err, pgx.ErrNoRows) { + return w + } + if err != nil { + s.fail("witness pointer "+project, err) + } + w.exists = true + return w +} + +func (s *releaseService) assertPointerAbsent(t *testing.T, project string) { + t.Helper() + if w := s.witnessPointer(project); w.exists { + t.Fatalf("expected no active release pointer for %s, found %+v", project, w) + } +} + +func (s *releaseService) assertPointerLiteral(t *testing.T, project, wantReleaseID string, wantRevision int64) { + t.Helper() + w := s.witnessPointer(project) + if !w.exists || w.project != project || w.releaseID != wantReleaseID || w.revision != wantRevision { + t.Fatalf("active release pointer for %s = %+v, want {project:%s release_id:%s revision:%d}", + project, w, project, wantReleaseID, wantRevision) + } +} + +type candidateRow struct { + project, id, generation, gitCommit, payloadText, ctid string + exists bool +} + +func (s *releaseService) witnessCandidate(project, id string) candidateRow { + var w candidateRow + err := s.witness.QueryRow(s.ctx, + `SELECT project, id, generation, git_commit, payload::text, ctid::text FROM `+s.qualified(ProjectReleasesTable)+` +WHERE project = $1 AND id = $2`, project, id). + Scan(&w.project, &w.id, &w.generation, &w.gitCommit, &w.payloadText, &w.ctid) + if errors.Is(err, pgx.ErrNoRows) { + return w + } + if err != nil { + s.fail("witness candidate "+project+"/"+id, err) + } + w.exists = true + return w +} + +func (s *releaseService) assertCandidateLiteral(t *testing.T, want publication.ProjectRelease) { + t.Helper() + payload, err := marshalReleasePayload(want) + if err != nil { + t.Fatal(err) + } + w := s.witnessCandidate(want.Project, want.ID) + if !w.exists { + t.Fatalf("candidate %s/%s does not exist", want.Project, want.ID) + } + if w.project != want.Project || w.id != want.ID || w.generation != want.Generation || w.gitCommit != want.GitCommit { + t.Fatalf("candidate scan columns = %+v, want {project:%s id:%s generation:%s git_commit:%s}", + w, want.Project, want.ID, want.Generation, want.GitCommit) + } + matches := s.mustWitnessBool(t, + `SELECT payload = $1::jsonb FROM `+s.qualified(ProjectReleasesTable)+` WHERE project = $2 AND id = $3`, + payload, want.Project, want.ID) + if !matches { + t.Fatalf("candidate %s/%s payload does not equal the literal expected document\nwant: %s\ngot: %s", + want.Project, want.ID, payload, w.payloadText) + } +} + +func releaseSelector(recipe, version, output string) publication.DataframeSelector { + return publication.DataframeSelector{Recipe: recipe, TranslationVersion: version, Output: output} +} + +func candidateFixture(project, generation, id string) publication.ProjectRelease { + verified := time.Date(2026, 8, 6, 12, 0, 0, 123456789, time.UTC) + created := time.Date(2026, 8, 1, 9, 30, 0, 0, time.UTC) + return publication.ProjectRelease{ + ID: id, + Project: project, + GitCommit: generation, + Generation: generation, + Publications: []publication.ReleasePublication{ + {Selector: releaseSelector("recipe-main", "v2", "patients"), ExecutionID: "exec-1", Generation: generation, Required: true, VerifiedAt: verified}, + {Selector: releaseSelector("recipe-aux", "v1", "encounters"), ExecutionID: "exec-2", Generation: generation, Required: false, Stale: true, VerifiedAt: verified}, + }, + RequiredVerifications: []publication.ContractVerification{ + {Selector: releaseSelector("recipe-main", "v2", "patients"), ExecutionID: "exec-1", Generation: generation, State: "PUBLISHED", Queryable: true, VerifiedAt: verified}, + {Selector: releaseSelector("recipe-optional", "v1", "bundle"), Generation: generation, Queryable: false, ErrorCode: "RELEASE_REQUIREMENTS_UNMET"}, + }, + CreatedAt: created, + } +} + +func (s *releaseService) stageGeneration(t *testing.T, project, generation, digest string) publication.Manifest { + t.Helper() + ref := mustRef(t, project, generation) + schema := mustSnapshot(t, generation, digest) + loading, err := publication.NewManifest(ref, schema) + if err != nil { + t.Fatalf("NewManifest(%s/%s): %v", project, generation, err) + } + if _, err := s.gen.CreateManifest(s.ctx, loading); err != nil { + s.fail("CreateManifest "+project+"/"+generation, err) + } + staged, err := s.gen.TransitionManifest(s.ctx, loading, publication.StateStaged) + if err != nil { + s.fail("TransitionManifest STAGED "+project+"/"+generation, err) + } + return staged +} + +func (s *releaseService) activateGeneration(t *testing.T, staged publication.Manifest) { + t.Helper() + if err := s.gen.ActivateGeneration(s.ctx, staged, nil); err != nil { + s.fail("ActivateGeneration "+staged.Dataset.Project+"/"+staged.Dataset.Generation, err) + } +} + +func (s *releaseService) mustSave(t *testing.T, store *ReleaseStore, release publication.ProjectRelease) publication.ProjectRelease { + t.Helper() + saved, err := store.SaveRelease(s.ctx, release) + if err != nil { + s.fail(fmt.Sprintf("SaveRelease %s/%s", release.Project, release.ID), err) + } + return saved +} + +func TestNewReleaseStoreGuardsNilSourceAndUnsafeSchema(t *testing.T) { + if _, err := NewReleaseStore(nil, "ok_schema"); !errors.Is(err, ErrNilReleaseConnSource) { + t.Fatalf("NewReleaseStore(nil) = %v, want ErrNilReleaseConnSource", err) + } + for _, schema := range []string{"", "Public", "a;drop", "0starts_with_digit", "has space", "tab\tname", strings.Repeat("a", 64)} { + if _, err := NewReleaseStore((*pgxpool.Pool)(nil), schema); !errors.Is(err, ErrInvalidSchemaName) { + t.Fatalf("NewReleaseStore(schema %q) = %v, want ErrInvalidSchemaName", schema, err) + } + } + if _, err := NewReleaseStore((*pgxpool.Pool)(nil), "ok_schema"); err != nil { + t.Fatalf("NewReleaseStore(valid schema) = %v, want success", err) + } +} + +func TestReleaseSchemaCompositionAndColumnContract(t *testing.T) { + s := newReleaseService(t) + + rows, err := s.witness.Query(s.ctx, + "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = $1 ORDER BY tablename", s.schema) + if err != nil { + s.fail("list tables", err) + } + var tables []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + s.fail("scan table", err) + } + tables = append(tables, name) + } + if err := rows.Err(); err != nil { + s.fail("list tables", err) + } + rows.Close() + want := []string{ + "active_project_release", + "dataset_active_generation", + "dataset_generation_manifest", + "project_release", + "schema_migrations", + } + if strings.Join(tables, ",") != strings.Join(want, ",") { + t.Fatalf("schema tables = %#v, want exactly %#v (no duplicated generation schema, no data-plane tables)", tables, want) + } + + type column struct { + name, typ string + nullable bool + } + for _, wantTable := range []struct { + table string + columns []column + }{ + {ProjectReleasesTable, []column{ + {"project", "text", false}, + {"id", "text", false}, + {"generation", "text", false}, + {"git_commit", "text", false}, + {"payload", "jsonb", false}, + }}, + {ActiveProjectReleasesTable, []column{ + {"project", "text", false}, + {"release_id", "text", false}, + {"revision", "bigint", false}, + {"created_at", "timestamp with time zone", false}, + {"updated_at", "timestamp with time zone", false}, + }}, + } { + rows, err := s.witness.Query(s.ctx, + "SELECT column_name, data_type, is_nullable = 'YES' FROM information_schema.columns"+ + " WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position", + s.schema, wantTable.table) + if err != nil { + s.fail("list columns "+wantTable.table, err) + } + var columns []column + for rows.Next() { + var c column + if err := rows.Scan(&c.name, &c.typ, &c.nullable); err != nil { + s.fail("scan column", err) + } + columns = append(columns, c) + } + if err := rows.Err(); err != nil { + s.fail("list columns "+wantTable.table, err) + } + rows.Close() + if len(columns) != len(wantTable.columns) { + t.Fatalf("%s columns = %+v, want %+v", wantTable.table, columns, wantTable.columns) + } + for i, wantCol := range wantTable.columns { + if columns[i] != wantCol { + t.Fatalf("%s column %d = %+v, want %+v", wantTable.table, i, columns[i], wantCol) + } + } + } +} + +func TestSaveReleasePersistsImmutableCandidate(t *testing.T) { + s := newReleaseService(t) + const project = "pgpubrel01-save" + cand := candidateFixture(project, "gen-1", "release_"+digestA) + + saved := s.mustSave(t, s.releases, cand) + if !reflect.DeepEqual(saved, cand) { + t.Fatalf("SaveRelease = %+v, want %+v", saved, cand) + } + s.assertCandidateLiteral(t, cand) + + s.assertPointerAbsent(t, project) + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(ActiveProjectReleasesTable)); n != 0 { + t.Fatalf("active_project_release rows after save = %d, want 0", n) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified("dataset_active_generation")); n != 0 { + t.Fatalf("active generation rows after save = %d, want 0", n) + } + + for _, absentKey := range []string{"executionId", "state"} { + if present := s.mustWitnessBool(t, + `SELECT (payload -> 'requiredVerifications' -> 1) ? $3 FROM `+s.qualified(ProjectReleasesTable)+` WHERE project = $1 AND id = $2`, + project, cand.ID, absentKey); present { + t.Fatalf("stored payload fabricated a %s key for the zero-value verification field", absentKey) + } + } + if got := s.mustWitnessText(t, + `SELECT (payload -> 'requiredVerifications' -> 1) ->> 'verifiedAt' FROM `+s.qualified(ProjectReleasesTable)+` WHERE project = $1 AND id = $2`, + project, cand.ID); got != "0001-01-01T00:00:00Z" { + t.Fatalf("zero-time verification stored as %q, want the canonical zero instant 0001-01-01T00:00:00Z", got) + } + + retry := cand + retry.CreatedAt = time.Date(2027, 1, 2, 3, 4, 5, 0, time.UTC) + retry.Publications = append([]publication.ReleasePublication(nil), cand.Publications...) + retry.RequiredVerifications = append([]publication.ContractVerification(nil), cand.RequiredVerifications...) + before := s.witnessCandidate(project, cand.ID) + savedRetry := s.mustSave(t, s.releases, retry) + if !savedRetry.CreatedAt.Equal(cand.CreatedAt) { + t.Fatalf("same-content retry returned createdAt %v, want the stored original %v", savedRetry.CreatedAt, cand.CreatedAt) + } + after := s.witnessCandidate(project, cand.ID) + if before.payloadText != after.payloadText || before.ctid != after.ctid { + t.Fatalf("same-content retry rewrote the candidate: before=%+v after=%+v", before, after) + } + + changed := candidateFixture(project, "gen-1", "release_"+digestA) + changed.Publications = append([]publication.ReleasePublication(nil), cand.Publications...) + changed.Publications[0].Required = false + if _, err := s.releases.SaveRelease(s.ctx, changed); !errors.Is(err, ErrReleaseContentConflict) { + t.Fatalf("changed-content save = %v, want ErrReleaseContentConflict", err) + } + afterChange := s.witnessCandidate(project, cand.ID) + if afterChange.payloadText != before.payloadText || afterChange.ctid != before.ctid { + t.Fatalf("refused changed-content save mutated the candidate: before=%+v after=%+v", before, afterChange) + } + + other := candidateFixture(project, "gen-1", "release_"+digestB) + other.Publications = append([]publication.ReleasePublication(nil), cand.Publications...) + other.Publications[1].ExecutionID = "exec-9" + s.mustSave(t, s.releases, other) + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(ProjectReleasesTable)+" WHERE project = $1", project); n != 2 { + t.Fatalf("candidate rows = %d, want 2 (history retained)", n) + } +} + +func TestReleaseActivationAtomicallyUpdatesReleasePointer(t *testing.T) { + s := newReleaseService(t) + const project = "pgpubrel01-activate" + staged := s.stageGeneration(t, project, "gen-1", digestA) + s.activateGeneration(t, staged) + + candA := candidateFixture(project, "gen-1", "release_"+digestA) + s.mustSave(t, s.releases, candA) + + active, err := s.releases.CompareAndSwapActivateRelease(s.ctx, candA, 0) + if err != nil { + s.fail("first activation", err) + } + if !reflect.DeepEqual(active, publication.ActiveRelease{Release: candA, Revision: 1}) { + t.Fatalf("first activation = %+v, want {%+v 1}", active, candA) + } + s.assertPointerLiteral(t, project, candA.ID, 1) + + candB := candidateFixture(project, "gen-1", "release_"+digestB) + candB.Publications = append([]publication.ReleasePublication(nil), candA.Publications...) + candB.Publications[0].ExecutionID = "exec-b" + s.mustSave(t, s.releasesB, candB) + stale := s.witnessPointer(project) + if _, err := s.releasesB.CompareAndSwapActivateRelease(s.ctx, candB, 0); !errors.Is(err, publication.ErrReleaseActivationConflict) { + t.Fatalf("stale-expected activation = %v, want ErrReleaseActivationConflict", err) + } + fresh := s.witnessPointer(project) + if fresh != stale { + t.Fatalf("stale activation changed the pointer: before=%+v after=%+v", stale, fresh) + } + + active, err = s.releasesB.CompareAndSwapActivateRelease(s.ctx, candB, 1) + if err != nil { + s.fail("second activation", err) + } + if !reflect.DeepEqual(active, publication.ActiveRelease{Release: candB, Revision: 2}) { + t.Fatalf("second activation = %+v, want {%+v 2}", active, candB) + } + s.assertPointerLiteral(t, project, candB.ID, 2) + s.assertCandidateLiteral(t, candA) + s.assertCandidateLiteral(t, candB) + + read, err := s.releases.ReadActiveRelease(s.ctx, project) + if err != nil { + s.fail("ReadActiveRelease", err) + } + if !reflect.DeepEqual(read, publication.ActiveRelease{Release: candB, Revision: 2}) { + t.Fatalf("ReadActiveRelease = %+v, want {%+v 2}", read, candB) + } + if _, err := s.releases.ReadActiveRelease(s.ctx, "pgpubrel01-never-active"); !errors.Is(err, publication.ErrNoActiveRelease) { + t.Fatalf("ReadActiveRelease(unprojected) = %v, want ErrNoActiveRelease", err) + } +} + +func TestReleaseActivationEmptyCASResultIsConflict(t *testing.T) { + s := newReleaseService(t) + const project = "pgpubrel01-cas-negative" + + ghost := candidateFixture(project, "gen-ghost", "release_"+digestC) + if _, err := s.releases.CompareAndSwapActivateRelease(s.ctx, ghost, 0); !errors.Is(err, publication.ErrReleaseActivationConflict) { + t.Fatalf("unpersisted candidate = %v, want ErrReleaseActivationConflict", err) + } + s.assertPointerAbsent(t, project) + + cand := candidateFixture(project, "gen-1", "release_"+digestA) + s.mustSave(t, s.releases, cand) + if _, err := s.releases.CompareAndSwapActivateRelease(s.ctx, cand, 0); !errors.Is(err, publication.ErrReleaseActivationConflict) { + t.Fatalf("no manifest = %v, want ErrReleaseActivationConflict", err) + } + s.assertPointerAbsent(t, project) + + ref := mustRef(t, project, "gen-1") + schema := mustSnapshot(t, "gen-1", digestA) + loading, err := publication.NewManifest(ref, schema) + if err != nil { + t.Fatal(err) + } + if _, err := s.gen.CreateManifest(s.ctx, loading); err != nil { + s.fail("CreateManifest", err) + } + if _, err := s.releases.CompareAndSwapActivateRelease(s.ctx, cand, 0); !errors.Is(err, publication.ErrReleaseActivationConflict) { + t.Fatalf("LOADING manifest = %v, want ErrReleaseActivationConflict", err) + } + s.assertPointerAbsent(t, project) + + staged, err := s.gen.TransitionManifest(s.ctx, loading, publication.StateStaged) + if err != nil { + s.fail("TransitionManifest", err) + } + if _, err := s.releases.CompareAndSwapActivateRelease(s.ctx, cand, 0); !errors.Is(err, publication.ErrReleaseActivationConflict) { + t.Fatalf("missing active generation = %v, want ErrReleaseActivationConflict", err) + } + s.assertPointerAbsent(t, project) + + s.activateGeneration(t, staged) + + s.mustExec(t, s.witness, `UPDATE `+s.qualified(ProjectReleasesTable)+ + ` SET payload = jsonb_set(payload, '{publications,0,executionId}', '"tampered"') WHERE project = $1 AND id = $2`, + project, cand.ID) + if _, err := s.releases.CompareAndSwapActivateRelease(s.ctx, cand, 0); !errors.Is(err, publication.ErrReleaseActivationConflict) { + t.Fatalf("tampered candidate = %v, want ErrReleaseActivationConflict", err) + } + s.assertPointerAbsent(t, project) + if _, err := s.releases.SaveRelease(s.ctx, cand); !errors.Is(err, ErrReleaseContentConflict) { + t.Fatalf("resave over tampered payload = %v, want ErrReleaseContentConflict", err) + } + + const project2 = "pgpubrel01-cas-negative-2" + staged2 := s.stageGeneration(t, project2, "gen-1", digestB) + s.activateGeneration(t, staged2) + cand2 := candidateFixture(project2, "gen-1", "release_"+digestB) + s.mustSave(t, s.releases, cand2) + s.mustExec(t, s.witness, `UPDATE `+s.qualified("dataset_generation_manifest")+ + ` SET state = 'FAILED' WHERE project = $1 AND generation = $2`, project2, "gen-1") + if _, err := s.releases.CompareAndSwapActivateRelease(s.ctx, cand2, 0); !errors.Is(err, publication.ErrReleaseActivationConflict) { + t.Fatalf("FAILED manifest = %v, want ErrReleaseActivationConflict", err) + } + s.assertPointerAbsent(t, project2) +} + +func TestActiveReleaseReadSurfacesStoredInconsistency(t *testing.T) { + s := newReleaseService(t) + const project = "pgpubrel01-tamper-read" + staged := s.stageGeneration(t, project, "gen-1", digestA) + s.activateGeneration(t, staged) + cand := candidateFixture(project, "gen-1", "release_"+digestA) + s.mustSave(t, s.releases, cand) + if _, err := s.releases.CompareAndSwapActivateRelease(s.ctx, cand, 0); err != nil { + s.fail("activation", err) + } + + s.mustExec(t, s.witness, `UPDATE `+s.qualified(ProjectReleasesTable)+ + ` SET payload = jsonb_set(payload, '{id}', '"release_forged"') WHERE project = $1 AND id = $2`, + project, cand.ID) + if _, err := s.releases.ReadActiveRelease(s.ctx, project); !errors.Is(err, ErrUnexpectedStoreResult) { + t.Fatalf("read over forged payload = %v, want ErrUnexpectedStoreResult", err) + } +} + +func TestReleaseActivationAlreadyActiveIsZeroWriteAndRevisionStable(t *testing.T) { + s := newReleaseService(t) + const project = "pgpubrel01-idempotent" + staged := s.stageGeneration(t, project, "gen-1", digestA) + s.activateGeneration(t, staged) + cand := candidateFixture(project, "gen-1", "release_"+digestA) + s.mustSave(t, s.releases, cand) + if _, err := s.releases.CompareAndSwapActivateRelease(s.ctx, cand, 0); err != nil { + s.fail("first activation", err) + } + before := s.witnessPointer(project) + + for i, expected := range []int64{1, 0} { + active, err := s.releases.CompareAndSwapActivateRelease(s.ctx, cand, expected) + if err != nil { + t.Fatalf("already-active activation %d (expected %d): %v", i, expected, err) + } + if !reflect.DeepEqual(active, publication.ActiveRelease{Release: cand, Revision: 1}) { + t.Fatalf("already-active activation %d = %+v, want {%+v 1}", i, active, cand) + } + } + after := s.witnessPointer(project) + if after != before { + t.Fatalf("already-active re-activation rewrote the pointer: before=%+v after=%+v", before, after) + } +} + +func TestConcurrentReleaseActivationsSameExpectedYieldOneWinner(t *testing.T) { + s := newReleaseService(t) + const project = "pgpubrel01-race" + staged := s.stageGeneration(t, project, "gen-1", digestA) + s.activateGeneration(t, staged) + + candA := candidateFixture(project, "gen-1", "release_"+digestA) + s.mustSave(t, s.releases, candA) + candB := candidateFixture(project, "gen-1", "release_"+digestB) + candB.Publications = append([]publication.ReleasePublication(nil), candA.Publications...) + candB.Publications[1].ExecutionID = "exec-b" + s.mustSave(t, s.releasesB, candB) + + candidates := []publication.ProjectRelease{candA, candB} + stores := []*ReleaseStore{s.releases, s.releasesB} + results := make([]error, len(candidates)) + winnerIDs := make([]publication.ActiveRelease, len(candidates)) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := range candidates { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + active, err := stores[i].CompareAndSwapActivateRelease(s.ctx, candidates[i], 0) + results[i], winnerIDs[i] = err, active + }(i) + } + close(start) + wg.Wait() + + winner, loser := -1, -1 + for i, err := range results { + t.Logf("activation %d (candidate %s, backend pids A=%d B=%d): err=%v", i, candidates[i].ID, s.pidA, s.pidB, err) + if err == nil { + if winner != -1 { + t.Fatalf("both concurrent activations won: %v", results) + } + winner = i + continue + } + if !errors.Is(err, publication.ErrReleaseActivationConflict) { + t.Fatalf("activation %d failed with untyped error: %v", i, err) + } + if loser != -1 { + t.Fatalf("both concurrent activations lost: %v", results) + } + loser = i + } + if winner == -1 || loser == -1 { + t.Fatalf("expected exactly one winner and one typed-conflict loser, got %v", results) + } + if winnerIDs[winner].Revision != 1 || winnerIDs[winner].Release.ID != candidates[winner].ID { + t.Fatalf("winner verdict = %+v, want revision 1 for %s", winnerIDs[winner], candidates[winner].ID) + } + s.assertPointerLiteral(t, project, candidates[winner].ID, 1) + + if _, err := stores[loser].CompareAndSwapActivateRelease(s.ctx, candidates[loser], 1); err != nil { + s.fail("loser re-activation with fresh expected", err) + } + s.assertPointerLiteral(t, project, candidates[loser].ID, 2) +} + +type observingAdapter struct { + inner GenerationOwnerAdapter + pointers string + + entered chan ReleaseActivation + release chan struct{} + activated []ReleaseActivation + mu sync.Mutex + hookErr error + ensureErr error + ensureOn bool + hookViolation string +} + +func newObservingAdapter(inner GenerationOwnerAdapter, pointersFQN string) *observingAdapter { + return &observingAdapter{ + inner: inner, + pointers: pointersFQN, + entered: make(chan ReleaseActivation, 4), + release: make(chan struct{}), + } +} + +func (a *observingAdapter) EnsureGenerationPrerequisites(ctx context.Context, tx pgx.Tx, project, generation string) error { + if a.ensureOn && a.ensureErr != nil { + return a.ensureErr + } + return a.inner.EnsureGenerationPrerequisites(ctx, tx, project, generation) +} + +func (a *observingAdapter) OnReleaseActivated(ctx context.Context, tx pgx.Tx, activation ReleaseActivation) error { + var ( + seenReleaseID string + seenRevision int64 + ) + err := tx.QueryRow(ctx, + `SELECT release_id, revision FROM `+a.pointers+` WHERE project = $1`, activation.Project). + Scan(&seenReleaseID, &seenRevision) + a.mu.Lock() + if err != nil { + a.hookViolation = fmt.Sprintf("in-transaction pointer read failed: %v", err) + } else if seenReleaseID != activation.Release.ID || seenRevision != activation.Revision { + a.hookViolation = fmt.Sprintf("in-transaction pointer read saw {%s %d}, want {%s %d}", + seenReleaseID, seenRevision, activation.Release.ID, activation.Revision) + } + a.activated = append(a.activated, activation) + a.mu.Unlock() + a.entered <- activation + select { + case <-a.release: + case <-ctx.Done(): + return fmt.Errorf("gate released by context: %w", ctx.Err()) + } + return a.hookErr +} + +func (a *observingAdapter) hookCalls() ([]ReleaseActivation, string) { + a.mu.Lock() + defer a.mu.Unlock() + return append([]ReleaseActivation(nil), a.activated...), a.hookViolation +} + +func TestGenerationOwnerAdapterParticipatesInActivationTransaction(t *testing.T) { + s := newReleaseService(t) + const project = "pgpubrel01-adapter" + staged := s.stageGeneration(t, project, "gen-1", digestA) + s.activateGeneration(t, staged) + cand := candidateFixture(project, "gen-1", "release_"+digestA) + s.mustSave(t, s.releases, cand) + + adapter := newObservingAdapter(s.releases.adapter, s.releases.pointers) + s.releases.adapter = adapter + + type outcome struct { + active publication.ActiveRelease + err error + } + done := make(chan outcome, 1) + go func() { + active, err := s.releases.CompareAndSwapActivateRelease(s.ctx, cand, 0) + done <- outcome{active, err} + }() + + select { + case activation := <-adapter.entered: + if activation.Release.ID != cand.ID || activation.Revision != 1 { + t.Fatalf("adapter saw activation %+v, want candidate %s at revision 1", activation, cand.ID) + } + if w := s.witnessPointer(project); w.exists { + t.Fatalf("witness observed the pointer row before commit: %+v", w) + } + close(adapter.release) + case <-time.After(60 * time.Second): + t.Fatal("activation never reached the generation-owner hook") + } + + select { + case result := <-done: + if result.err != nil { + s.fail("activation with observing adapter", result.err) + } + case <-time.After(60 * time.Second): + t.Fatal("activation never completed") + } + s.assertPointerLiteral(t, project, cand.ID, 1) + if _, violation := adapter.hookCalls(); violation != "" { + t.Fatalf("generation-owner hook did not observe its own uncommitted pointer write inside the activation transaction: %s", violation) + } + + beforeHookCalls, _ := adapter.hookCalls() + if _, err := s.releases.CompareAndSwapActivateRelease(s.ctx, cand, 1); err != nil { + s.fail("already-active re-activation", err) + } + afterHookCalls, _ := adapter.hookCalls() + if len(afterHookCalls) != len(beforeHookCalls) { + t.Fatalf("already-active re-activation invoked the generation-owner hook %d more times, want 0 (zero-write path)", + len(afterHookCalls)-len(beforeHookCalls)) + } + + second := candidateFixture(project, "gen-1", "release_"+digestB) + second.Publications = append([]publication.ReleasePublication(nil), cand.Publications...) + second.Publications[1].ExecutionID = "exec-fault" + s.mustSave(t, s.releases, second) + adapter2 := newObservingAdapter(s.releases.adapter, s.releases.pointers) + adapter2.hookErr = errors.New("injected generation-owner fault") + s.releases.adapter = adapter2 + done2 := make(chan error, 1) + go func() { + _, err := s.releases.CompareAndSwapActivateRelease(s.ctx, second, 1) + done2 <- err + }() + select { + case activation := <-adapter2.entered: + if activation.Revision != 2 { + t.Fatalf("faulted activation hook saw revision %d, want 2", activation.Revision) + } + s.assertPointerLiteral(t, project, cand.ID, 1) + close(adapter2.release) + case <-time.After(60 * time.Second): + t.Fatal("activation never reached the faulted generation-owner hook") + } + var hookErr error + select { + case hookErr = <-done2: + case <-time.After(60 * time.Second): + t.Fatal("activation with faulty hook never completed") + } + if hookErr == nil { + t.Fatal("activation with faulty generation-owner hook = nil, want failure") + } + if errors.Is(hookErr, publication.ErrReleaseActivationConflict) { + t.Fatalf("generation-owner fault misclassified as CAS conflict: %v", hookErr) + } + if _, violation := adapter2.hookCalls(); violation != "" { + t.Fatalf("faulted activation hook did not observe its own uncommitted pointer write: %s", violation) + } + s.assertPointerLiteral(t, project, cand.ID, 1) + s.releases.adapter = adapter + if _, err := s.releases.CompareAndSwapActivateRelease(s.ctx, second, 1); err != nil { + s.fail("activation after hook recovery", err) + } + s.assertPointerLiteral(t, project, second.ID, 2) +} + +func TestInjectedFailureBeforeCommitKeepsPointerUnchanged(t *testing.T) { + s := newReleaseService(t) + const project = "pgpubrel01-commitfault" + staged := s.stageGeneration(t, project, "gen-1", digestA) + s.activateGeneration(t, staged) + cand := candidateFixture(project, "gen-1", "release_"+digestA) + s.mustSave(t, s.releases, cand) + + s.releases.commitFault = func(string) error { return errors.New("injected commit failure") } + if _, err := s.releases.CompareAndSwapActivateRelease(s.ctx, cand, 0); err == nil { + t.Fatal("activation with injected commit fault returned nil") + } else if errors.Is(err, publication.ErrReleaseActivationConflict) { + t.Fatalf("commit fault misclassified as CAS conflict: %v", err) + } + s.releases.commitFault = nil + s.assertPointerAbsent(t, project) + + if _, err := s.releases.CompareAndSwapActivateRelease(s.ctx, cand, 0); err != nil { + s.fail("activation after fault removal", err) + } + s.assertPointerLiteral(t, project, cand.ID, 1) +} + +func TestReleaseStoreFailsClosedWithoutItsTables(t *testing.T) { + s := newReleaseService(t) + emptySchema := fmt.Sprintf("pgpubrel01_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgpubrel01ResourceNamePattern.MatchString(emptySchema) { + t.Fatalf("owned schema name %q does not match the required pattern", emptySchema) + } + s.mustExec(t, s.witness, `CREATE SCHEMA "`+emptySchema+`"`) + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if _, err := s.witness.Exec(cleanupCtx, `DROP SCHEMA IF EXISTS "`+emptySchema+`" CASCADE`); err != nil { + t.Errorf("drop empty schema: %s", s.scrub(err)) + } + }) + orphan, err := NewReleaseStore(s.poolB, emptySchema) + if err != nil { + s.fail("build orphan store", err) + } + cand := candidateFixture("pgpubrel01-orphan", "gen-1", "release_"+digestA) + _, err = orphan.SaveRelease(s.ctx, cand) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "42P01" { + t.Fatalf("save against unmigrated schema = %v, want SQLSTATE 42P01 undefined_table", s.scrub(err)) + } + if _, err := orphan.ReadActiveRelease(s.ctx, "pgpubrel01-orphan"); !errors.As(err, &pgErr) || pgErr.Code != "42P01" { + t.Fatalf("read against unmigrated schema = %v, want SQLSTATE 42P01", s.scrub(err)) + } + if _, err := orphan.CompareAndSwapActivateRelease(s.ctx, cand, 0); !errors.As(err, &pgErr) || pgErr.Code != "42P01" { + t.Fatalf("activate against unmigrated schema = %v, want SQLSTATE 42P01, never a typed conflict or fallback", s.scrub(err)) + } +} + +func TestReleaseSchemaConstraintsRejectInvalidRows(t *testing.T) { + s := newReleaseService(t) + const project = "pgpubrel01-constraints" + cand := candidateFixture(project, "gen-1", "release_"+digestA) + s.mustSave(t, s.releases, cand) + payload, err := marshalReleasePayload(cand) + if err != nil { + t.Fatal(err) + } + + type negative struct { + name string + sql string + args []any + wantState string + } + cases := []negative{ + { + name: "candidate project is never SQL NULL", + sql: `INSERT INTO "%[1]s".project_release (project, id, generation, git_commit, payload) VALUES (NULL, 'r', 'g', 'c', '{}'::jsonb)`, + wantState: "23502", + }, + { + name: "candidate project is never empty", + sql: `INSERT INTO "%[1]s".project_release (project, id, generation, git_commit, payload) VALUES ('', 'r', 'g', 'c', '{}'::jsonb)`, + wantState: "23514", + }, + { + name: "candidate payload is never SQL NULL", + sql: `INSERT INTO "%[1]s".project_release (project, id, generation, git_commit, payload) VALUES ('p', 'r', 'g', 'c', NULL)`, + wantState: "23502", + }, + { + name: "candidate payload JSON null is not an object", + sql: `INSERT INTO "%[1]s".project_release (project, id, generation, git_commit, payload) VALUES ('p', 'r', 'g', 'c', 'null'::jsonb)`, + wantState: "23514", + }, + { + name: "candidate payload JSON array is not an object", + sql: `INSERT INTO "%[1]s".project_release (project, id, generation, git_commit, payload) VALUES ('p', 'r', 'g', 'c', '[]'::jsonb)`, + wantState: "23514", + }, + { + name: "duplicate (project, id) candidates are impossible", + sql: `INSERT INTO "%[1]s".project_release (project, id, generation, git_commit, payload) VALUES ($1, $2, $3, $4, $5::jsonb)`, + args: []any{project, cand.ID, cand.Generation, cand.GitCommit, payload}, + wantState: "23505", + }, + { + name: "pointer revision floor is 1 (0 stands in for absent only as expectedRevision)", + sql: `INSERT INTO "%[1]s".active_project_release (project, release_id, revision) VALUES ('p', 'r', 0)`, + wantState: "23514", + }, + { + name: "pointer revision is never SQL NULL", + sql: `INSERT INTO "%[1]s".active_project_release (project, release_id, revision) VALUES ('p', 'r', NULL)`, + wantState: "23502", + }, + { + name: "pointer release_id is never SQL NULL", + sql: `INSERT INTO "%[1]s".active_project_release (project, release_id, revision) VALUES ('p', NULL, 1)`, + wantState: "23502", + }, + { + name: "pointer must name a persisted candidate", + sql: `INSERT INTO "%[1]s".active_project_release (project, release_id, revision) VALUES ('p', 'release_missing', 1)`, + wantState: "23503", + }, + } + for _, c := range cases { + sql := fmt.Sprintf(c.sql, s.schema) + _, err := s.witness.Exec(s.ctx, sql, c.args...) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != c.wantState { + t.Errorf("%s: err = %v, want SQLSTATE %s", c.name, s.scrub(err), c.wantState) + } + } + + staged := s.stageGeneration(t, project, "gen-1", digestA) + s.activateGeneration(t, staged) + if _, err := s.releases.CompareAndSwapActivateRelease(s.ctx, cand, 0); err != nil { + s.fail("activation for FK fixture", err) + } + if _, err := s.witness.Exec(s.ctx, + `DELETE FROM `+s.qualified(ProjectReleasesTable)+` WHERE project = $1 AND id = $2`, project, cand.ID); err == nil { + t.Fatal("deleting the active candidate = nil, want FK violation") + } else { + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "23503" { + t.Fatalf("delete active candidate: err = %v, want SQLSTATE 23503", s.scrub(err)) + } + } +} + +func TestZZNoLeftoverPgpubrel01Resources(t *testing.T) { + raw := testPostgresURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %v", err) + } + defer admin.Close() + + var leftoverDBs int + if err := admin.QueryRow(ctx, + `SELECT count(*) FROM pg_database WHERE datname LIKE 'pgpubrel01\_it\_%' ESCAPE '\'`).Scan(&leftoverDBs); err != nil { + t.Fatalf("count leftover databases: %v", err) + } + if leftoverDBs != 0 { + var names []string + rows, err := admin.Query(ctx, `SELECT datname FROM pg_database WHERE datname LIKE 'pgpubrel01\_it\_%' ESCAPE '\'`) + if err == nil { + for rows.Next() { + var n string + if err := rows.Scan(&n); err == nil { + names = append(names, n) + } + } + rows.Close() + } + t.Fatalf("%d isolated databases left behind: %v", leftoverDBs, names) + } + + var leftoverSchemas int + if err := admin.QueryRow(ctx, + `SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgpubrel01\_it\_%' ESCAPE '\'`).Scan(&leftoverSchemas); err != nil { + t.Fatalf("count leftover schemas: %v", err) + } + if leftoverSchemas != 0 { + t.Fatalf("%d owned schemas left behind in the service database", leftoverSchemas) + } +} diff --git a/internal/backends/postgres/internal/dataset/store.go b/internal/backends/postgres/internal/dataset/store.go new file mode 100644 index 000000000..d7316a36c --- /dev/null +++ b/internal/backends/postgres/internal/dataset/store.go @@ -0,0 +1,93 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + publication "github.com/calypr/loom/internal/dataset" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +var ( + ErrNilConnSource = errors.New("dataset postgres: generation store connection source is required") + ErrInvalidSchemaName = storepostgres.ErrInvalidSchemaName + ErrManifestAlreadyExists = errors.New("generation manifest already exists") + ErrManifestNotFound = publication.ErrManifestNotFound + ErrManifestTransitionConflict = errors.New("generation manifest transition conflict") + ErrActivationConflict = errors.New("generation activation conflict") + ErrUnexpectedStoreResult = errors.New("unexpected generation store result") +) + +type ConnSource interface { + Acquire(ctx context.Context) (*pgxpool.Conn, error) +} + +var ( + _ ConnSource = (*storepostgres.Client)(nil) + _ ConnSource = (*pgxpool.Pool)(nil) +) + +type Store struct { + src ConnSource + manifestFQN string + pointerFQN string + commitFault func(step string) error +} + +func New(src ConnSource, schema string) (*Store, error) { + if src == nil { + return nil, ErrNilConnSource + } + if !storepostgres.ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", ErrInvalidSchemaName, schema) + } + return &Store{ + src: src, + manifestFQN: pgx.Identifier{schema, "dataset_generation_manifest"}.Sanitize(), + pointerFQN: pgx.Identifier{schema, "dataset_active_generation"}.Sanitize(), + }, nil +} + +func (s *Store) validate() error { + if s == nil || s.src == nil { + return ErrNilConnSource + } + return nil +} + +func execCommand(ctx context.Context, tx pgx.Tx, c Command) (pgconn.CommandTag, error) { + sql, args, err := c.PostgreSQL() + if err != nil { + return pgconn.CommandTag{}, err + } + return tx.Exec(ctx, sql, args...) +} + +func queryRowCommand(ctx context.Context, tx pgx.Tx, c Command) pgx.Row { + sql, args, err := c.PostgreSQL() + if err != nil { + return errRow{err: err} + } + return tx.QueryRow(ctx, sql, args...) +} + +func queryRowConn(ctx context.Context, conn *pgxpool.Conn, c Command) pgx.Row { + sql, args, err := c.PostgreSQL() + if err != nil { + return errRow{err: err} + } + return conn.QueryRow(ctx, sql, args...) +} + +type errRow struct{ err error } + +func (r errRow) Scan(dest ...any) error { return r.err } + +func validateProject(project string) error { + _, err := publication.NewRef(project, "generation-project-validation") + return err +} diff --git a/internal/backends/postgres/internal/explorer/activation.go b/internal/backends/postgres/internal/explorer/activation.go new file mode 100644 index 000000000..33675f130 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/activation.go @@ -0,0 +1,343 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + datasetpostgres "github.com/calypr/loom/internal/backends/postgres/internal/dataset" + publicationpostgres "github.com/calypr/loom/internal/backends/postgres/internal/publication" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + dataframepublication "github.com/calypr/loom/internal/dataframe/publication" + "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/explorer" +) + +const ( + activationTxMaxAttempts = 4 + activationDeadlockRetry = 5 * time.Millisecond + activationPublishedState = string(dataframepublication.BundlePublished) +) + +var ErrInvalidActivationInput = errors.New("postgres explorer store: invalid repository generation activation input") + +type ActivationStore struct { + client *pgstore.Client + explorerSchema string + datasetSchema string + publicationSchema string + stmts activationStatements + crashAt func(context.Context, pgx.Tx, string) error +} + +func NewActivationStore(client *pgstore.Client, explorerSchema, datasetSchema, publicationSchema string) (*ActivationStore, error) { + if client == nil { + return nil, errors.New("postgres explorer store: activation client is required") + } + for _, schema := range []string{explorerSchema, datasetSchema, publicationSchema} { + if !pgstore.ValidSchemaName(schema) { + return nil, fmt.Errorf("postgres explorer store: invalid activation schema %q: must match [a-z_][a-z0-9_]{0,62}", schema) + } + } + return &ActivationStore{ + client: client, + explorerSchema: explorerSchema, + datasetSchema: datasetSchema, + publicationSchema: publicationSchema, + stmts: newActivationStatements(explorerSchema, datasetSchema, publicationSchema), + }, nil +} + +type activationStatements struct { + publish publishStatements + manifestState string + generationPointer string + executionState string +} + +func newActivationStatements(explorerSchema, datasetSchema, publicationSchema string) activationStatements { + manifests := sanitizeTableRef(datasetSchema, "dataset_generation_manifest") + generationPointer := sanitizeTableRef(datasetSchema, "dataset_active_generation") + executions := sanitizeTableRef(publicationSchema, publicationpostgres.BundleExecutionsTable) + return activationStatements{ + publish: newPublishStatements(explorerSchema, datasetSchema), + manifestState: "SELECT state FROM " + manifests + " WHERE project = $1 AND generation = $2 FOR SHARE", + generationPointer: "SELECT generation FROM " + generationPointer + " WHERE project = $1 FOR SHARE", + executionState: "SELECT state FROM " + executions + + " WHERE id = $1 AND project = $2 AND dataset_generation = $3 FOR SHARE", + } +} + +func (a *ActivationStore) ActivateRepositoryGeneration(ctx context.Context, project, generation, revisionID string) error { + if project == "" || generation == "" || revisionID == "" { + return fmt.Errorf("%w: project, generation, and revision ID are required (project %q, generation %q, revision %q)", + ErrInvalidActivationInput, project, generation, revisionID) + } + var lastErr error + for attempt := 1; attempt <= activationTxMaxAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return fmt.Errorf("postgres explorer store: activate repository generation: %w", err) + } + runErr := a.runActivateRepositoryGeneration(ctx, project, generation, revisionID) + if runErr == nil { + return nil + } + if errors.Is(runErr, explorer.ErrDraftConflict) { + return runErr + } + lastErr = runErr + if !isDeadlockError(runErr) || attempt == activationTxMaxAttempts { + break + } + if waitErr := waitForStoreRetry(ctx, time.Duration(attempt)*activationDeadlockRetry); waitErr != nil { + return fmt.Errorf("postgres explorer store: activate repository generation retry: %w", errors.Join(lastErr, waitErr)) + } + } + return fmt.Errorf("postgres explorer store: activate repository generation: %w", lastErr) +} + +func (a *ActivationStore) runActivateRepositoryGeneration(ctx context.Context, project, generation, revisionID string) (err error) { + conn, err := a.client.Acquire(ctx) + if err != nil { + return fmt.Errorf("acquire activation connection: %w", err) + } + defer conn.Release() + + tx, err := conn.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return fmt.Errorf("begin activation transaction: %w", err) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback(ctx) + } + }() + + crash := func(step string) error { + if a.crashAt != nil { + if err := a.crashAt(ctx, tx, step); err != nil { + return fmt.Errorf("injected fault at %s: %w", step, err) + } + } + return nil + } + + candidate, err := a.lockActivationCandidate(ctx, tx, revisionID, project, generation) + if err != nil { + return err + } + if err := a.verifyActivationManifest(ctx, tx, project, generation); err != nil { + return err + } + if err := a.verifyActivationGenerationPointer(ctx, tx, project, generation); err != nil { + return err + } + if err := a.verifyActivationExecution(ctx, tx, candidate, project, generation); err != nil { + return err + } + owner, err := a.lockActivationOwner(ctx, tx, project) + if err != nil { + return err + } + + if candidate.Status == string(explorer.RevisionActive) && owner.ActiveRevisionID == candidate.ID { + if err := crash("pre-commit"); err != nil { + return err + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("commit activation transaction: %w", err) + } + committed = true + return nil + } + if candidate.Status != string(explorer.RevisionReady) && candidate.Status != string(explorer.RevisionFailed) { + return fmt.Errorf("%w: candidate revision %q is %s, not activatable", explorer.ErrDraftConflict, candidate.ID, candidate.Status) + } + + if owner.ActiveRevisionID != "" && owner.ActiveRevisionID != candidate.ID { + if err := a.supersedePriorActivationRevision(ctx, tx, owner); err != nil { + return err + } + if err := crash("prior-supersede"); err != nil { + return err + } + } + + activatedPublication, err := activatedPublicationPayload(candidate) + if err != nil { + return err + } + activatedAt := time.Now().UTC() + activationStatement := a.stmts.publish.candidateActive + if candidate.Status == string(explorer.RevisionFailed) { + activationStatement = a.stmts.publish.candidateRevive + } + var activated storedRevisionRow + if err := scanRevisionRow(tx.QueryRow(ctx, activationStatement, + candidate.ID, activatedAt, []byte(publishDiagnosticsSet), []byte(activatedPublication)), &activated); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("%w: candidate revision %q moved before it could be activated", explorer.ErrDraftConflict, candidate.ID) + } + return fmt.Errorf("activate candidate revision %q: %w", candidate.ID, err) + } + if err := crash("candidate-activate"); err != nil { + return err + } + + var activatedOwner explorer.Explorer + if err := scanOwner(tx.QueryRow(ctx, a.stmts.publish.ownerActivate, + project, "default", candidate.ID, time.Now().UTC()), &activatedOwner); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("%w: owner vanished before activation", explorer.ErrDraftConflict) + } + return fmt.Errorf("activate explorer owner %q/default: %w", project, err) + } + if err := crash("owner-activate"); err != nil { + return err + } + + if err := crash("pre-commit"); err != nil { + return err + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("commit activation transaction: %w", err) + } + committed = true + return nil +} + +func (a *ActivationStore) lockActivationCandidate(ctx context.Context, tx pgx.Tx, revisionID, project, generation string) (*storedRevisionRow, error) { + var candidate storedRevisionRow + if err := scanRevisionRow(tx.QueryRow(ctx, a.stmts.publish.revisionClaim, revisionID), &candidate); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("%w: candidate revision %q is not a stored revision", explorer.ErrDraftConflict, revisionID) + } + return nil, fmt.Errorf("lock candidate revision %q: %w", revisionID, err) + } + if candidate.Project != project || candidate.ExplorerID != "default" || candidate.SourceGeneration != generation { + return nil, fmt.Errorf("%w: candidate revision %q does not belong to repository explorer %q/default at generation %q", + explorer.ErrDraftConflict, revisionID, project, generation) + } + return &candidate, nil +} + +func (a *ActivationStore) verifyActivationManifest(ctx context.Context, tx pgx.Tx, project, generation string) error { + var state string + err := tx.QueryRow(ctx, a.stmts.manifestState, project, generation).Scan(&state) + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("%w: generation %s/%s has no persisted manifest", explorer.ErrDraftConflict, project, generation) + } + if err != nil { + return fmt.Errorf("read activation manifest guard %s/%s: %w", project, generation, err) + } + if state != string(dataset.StateStaged) && state != string(dataset.StateReady) { + return fmt.Errorf("%w: generation %s/%s manifest is %s, not %s or %s", + explorer.ErrDraftConflict, project, generation, state, string(dataset.StateStaged), string(dataset.StateReady)) + } + return nil +} + +func (a *ActivationStore) verifyActivationGenerationPointer(ctx context.Context, tx pgx.Tx, project, generation string) error { + var activeGeneration string + err := tx.QueryRow(ctx, a.stmts.generationPointer, project).Scan(&activeGeneration) + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("%w: project %s has no active generation", explorer.ErrDraftConflict, project) + } + if err != nil { + return fmt.Errorf("read active generation guard for %s: %w", project, err) + } + if activeGeneration != generation { + return fmt.Errorf("%w: active generation of %s is %s, not the activation generation %s", + explorer.ErrDraftConflict, project, activeGeneration, generation) + } + return nil +} + +func (a *ActivationStore) verifyActivationExecution(ctx context.Context, tx pgx.Tx, candidate *storedRevisionRow, project, generation string) error { + materializationID, err := firstActivationMaterializationID(candidate) + if err != nil { + return err + } + if materializationID == "" { + return fmt.Errorf("%w: candidate revision %q has no materialized execution", explorer.ErrDraftConflict, candidate.ID) + } + var state string + err = tx.QueryRow(ctx, a.stmts.executionState, materializationID, project, generation).Scan(&state) + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("%w: materialization %q of candidate revision %q is not a persisted execution for %s/%s", + explorer.ErrDraftConflict, materializationID, candidate.ID, project, generation) + } + if err != nil { + return fmt.Errorf("read activation execution guard %q: %w", materializationID, err) + } + if state != activationPublishedState { + return fmt.Errorf("%w: execution %q is %s, not %s", + explorer.ErrDraftConflict, materializationID, state, activationPublishedState) + } + return nil +} + +func firstActivationMaterializationID(candidate *storedRevisionRow) (string, error) { + if len(candidate.Materializations) == 0 { + return "", nil + } + var materializations []explorer.Materialization + if err := json.Unmarshal(candidate.Materializations, &materializations); err != nil { + return "", fmt.Errorf("%w: stored revision %q materializations are undecodable: %v", + datasetpostgres.ErrUnexpectedStoreResult, candidate.ID, err) + } + if len(materializations) == 0 { + return "", nil + } + return materializations[0].MaterializationID, nil +} + +func (a *ActivationStore) lockActivationOwner(ctx context.Context, tx pgx.Tx, project string) (explorer.Explorer, error) { + var owner explorer.Explorer + err := scanOwner(tx.QueryRow(ctx, a.stmts.publish.ownerGuard, + project, "default", string(explorer.ManagementRepository)), &owner) + if errors.Is(err, pgx.ErrNoRows) { + return explorer.Explorer{}, fmt.Errorf("%w: repository explorer %q/default has no activatable owner row", + explorer.ErrDraftConflict, project) + } + if err != nil { + return explorer.Explorer{}, fmt.Errorf("lock activation owner %q/default: %w", project, err) + } + if owner.Project != project || owner.ExplorerID != "default" { + return explorer.Explorer{}, fmt.Errorf("%w: activation owner row %q/%q is foreign to %q/default", + explorer.ErrDraftConflict, owner.Project, owner.ExplorerID, project) + } + return owner, nil +} + +func (a *ActivationStore) supersedePriorActivationRevision(ctx context.Context, tx pgx.Tx, owner explorer.Explorer) error { + var prior storedRevisionRow + err := scanRevisionRow(tx.QueryRow(ctx, a.stmts.publish.priorLock, owner.ActiveRevisionID), &prior) + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("%w: owner active revision %q is not a stored revision", explorer.ErrDraftConflict, owner.ActiveRevisionID) + } + if err != nil { + return fmt.Errorf("lock prior revision %q: %w", owner.ActiveRevisionID, err) + } + if prior.Project != owner.Project || prior.ExplorerID != owner.ExplorerID { + return fmt.Errorf("%w: owner active revision %q is foreign to %q/%q", + explorer.ErrDraftConflict, owner.ActiveRevisionID, owner.Project, owner.ExplorerID) + } + if prior.Status != string(explorer.RevisionActive) { + return fmt.Errorf("%w: owner active revision %q is %s, not ACTIVE", + explorer.ErrDraftConflict, owner.ActiveRevisionID, prior.Status) + } + var superseded storedRevisionRow + if err := scanRevisionRow(tx.QueryRow(ctx, a.stmts.publish.priorSupersede, prior.ID), &superseded); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("%w: prior revision %q moved before it could be superseded", explorer.ErrDraftConflict, prior.ID) + } + return fmt.Errorf("supersede prior revision %q: %w", prior.ID, err) + } + return nil +} diff --git a/internal/backends/postgres/internal/explorer/activation_e2e_test.go b/internal/backends/postgres/internal/explorer/activation_e2e_test.go new file mode 100644 index 000000000..e211e605b --- /dev/null +++ b/internal/backends/postgres/internal/explorer/activation_e2e_test.go @@ -0,0 +1,1069 @@ +package postgres + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + datasetpostgres "github.com/calypr/loom/internal/backends/postgres/internal/dataset" + publicationpostgres "github.com/calypr/loom/internal/backends/postgres/internal/publication" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + dataframepublication "github.com/calypr/loom/internal/dataframe/publication" + "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/explorer" +) + +var pgactNameRe = regexp.MustCompile(`^verify_pgact_[0-9]+_[0-9a-f]+(?:_(?:ctl|lanes|execs))?$`) + +var ( + pgactBaselineOnce sync.Once + pgactDBBaseline atomic.Int64 + pgactLive atomic.Int64 + pgactSchemaBaseline atomic.Int64 + pgactBaselineFailure atomic.Value +) + +type pgactMigrationPin struct { + version int64 + name string + checksum string +} + +var ( + pgactControlPins = []pgactMigrationPin{ + {6, "explorer_revision", "3f6c44bff72d7837634e8c170f0b97e256e38d35f7c844fad27f4fbda38ecc09"}, + {12, "explorer_revision_activation_revival", "08a61bf6ca66a400112f51463f2c5e1c0a7f4cfbb2d707c0063f87829b1acfe9"}, + } + pgactLanePins = []pgactMigrationPin{ + {3, "dataset_generation_manifest", "06c614680a9c1cd4ba2ef0ac0471053bcb3d8e471c36d3852e70682887a72f5d"}, + } + pgactExecPins = []pgactMigrationPin{ + {4, "publication_bundle_catalog", "adcb4683ffd1611b9c68e7a49aaa4b8670d106d65df927cbb8e11866be224943"}, + } +) + +func pgactCaptureBaseline(admin *pgxpool.Pool) { + pgactBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + var mine, schemas int64 + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname LIKE 'verify_pgact\_%' ESCAPE '\'`).Scan(&mine); err != nil { + pgactBaselineFailure.Store(fmt.Errorf("baseline: query owned database namespace: %w", err)) + return + } + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'verify_pgact\_%' ESCAPE '\'`).Scan(&schemas); err != nil { + pgactBaselineFailure.Store(fmt.Errorf("baseline: query owned schema namespace: %w", err)) + return + } + pgactDBBaseline.Store(mine) + pgactSchemaBaseline.Store(schemas) + }) + if stored := pgactBaselineFailure.Load(); stored != nil { + panic(stored.(error)) + } +} + +func pgactRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +type pgactFixture struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dbName string + ctlSchema string + lanesSchema string + execsSchema string + dsn string + scrub func(any) string + fail func(step string, err error) +} + +func newPGActivationFixture(t *testing.T) *pgactFixture { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 repository-generation activation integration tests") + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + pgactCaptureBaseline(admin) + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("verify_pgact_%d_%s", time.Now().UnixNano(), pgactRandomHex(t)) + if !pgactNameRe.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + t.Cleanup(func() { + var mine int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname = $1`, dbName).Scan(&mine); err != nil { + t.Errorf("leftover audit: query database catalog: %s", scrub(err)) + } + if mine != 0 { + t.Errorf("leftover audit: isolated database %s still exists", dbName) + } + var namespaceDatabases int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname LIKE 'verify_pgact\_%' ESCAPE '\'`).Scan(&namespaceDatabases); err != nil { + t.Errorf("leftover audit: query database namespace: %s", scrub(err)) + } + if want := pgactDBBaseline.Load() + pgactLive.Load(); int64(namespaceDatabases) != want { + t.Errorf("leftover audit: %d verify_pgact_* databases remain, baseline+live is %d", namespaceDatabases, want) + } + var namespaceSchemas int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'verify_pgact\_%' ESCAPE '\'`).Scan(&namespaceSchemas); err != nil { + t.Errorf("leftover audit: query schema namespace: %s", scrub(err)) + } + if int64(namespaceSchemas) != pgactSchemaBaseline.Load() { + t.Errorf("leftover audit: %d verify_pgact_* schemas remain in the service database, baseline is %d", namespaceSchemas, pgactSchemaBaseline.Load()) + } + }) + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + pgactLive.Add(-1) + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + pgactLive.Add(1) + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + if probe, err := url.Parse(dsn); err != nil || probe.Query().Get("dbname") != "" || probe.Path != "/"+dbName { + t.Fatal("isolated test DSN does not provably target the isolated database") + } + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + base := fmt.Sprintf("verify_pgact_%d_%s", time.Now().UnixNano(), pgactRandomHex(t)) + ctlSchema := base + "_ctl" + lanesSchema := base + "_lanes" + execsSchema := base + "_execs" + for _, schema := range []string{ctlSchema, lanesSchema, execsSchema} { + if !pgactNameRe.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema "+schema, err) + } + } + t.Cleanup(func() { + for _, schema := range []string{execsSchema, lanesSchema, ctlSchema} { + if _, err := iso.Exec(context.WithoutCancel(ctx), `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema %s: %s", schema, scrub(err)) + } + } + }) + + return &pgactFixture{ + ctx: ctx, admin: admin, iso: iso, dbName: dbName, + ctlSchema: ctlSchema, lanesSchema: lanesSchema, execsSchema: execsSchema, + dsn: dsn, scrub: scrub, fail: fail, + } +} + +func (f *pgactFixture) migrate(t *testing.T) { + t.Helper() + ctlHistory, err := OwnerMigrations() + if err != nil { + f.fail("explorer owner migrations", err) + } + f.verifyPins(t, "explorer control", ctlHistory, pgactControlPins) + lanesHistory, err := datasetpostgres.OwnerMigrations() + if err != nil { + f.fail("dataset owner migrations", err) + } + f.verifyPins(t, "dataset lane", lanesHistory, pgactLanePins) + execsHistory, err := publicationpostgres.OwnerMigrations() + if err != nil { + f.fail("publication owner migrations", err) + } + f.verifyPins(t, "publication lane", execsHistory, pgactExecPins) + + client := f.newPool(t) + f.migrateHistory(t, client, f.ctlSchema, ctlHistory, 12, []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}) + f.migrateHistory(t, client, f.lanesSchema, lanesHistory, 8, []int64{1, 3, 8}) + f.migrateHistory(t, client, f.execsSchema, execsHistory, 9, []int64{1, 4, 6, 7, 9}) +} + +func (f *pgactFixture) verifyPins(t *testing.T, label string, history []pgstore.Migration, pins []pgactMigrationPin) { + t.Helper() + byVersion := make(map[int64]pgstore.Migration, len(history)) + for _, m := range history { + byVersion[m.Version] = m + } + for _, pin := range pins { + m, ok := byVersion[pin.version] + if !ok { + t.Fatalf("%s owner history lacks version %d (%s): the owning ticket's migration vanished", label, pin.version, pin.name) + } + if m.Name != pin.name || m.Checksum != pin.checksum { + t.Fatalf("%s owner history version %d = {Name:%q Checksum:%q}, want the byte-frozen merge pin {Name:%q Checksum:%q}: the owning ticket's migration drifted", + label, pin.version, m.Name, m.Checksum, pin.name, pin.checksum) + } + } +} + +func (f *pgactFixture) migrateHistory(t *testing.T, client *pgstore.Client, schema string, + history []pgstore.Migration, wantTop int64, wantVersions []int64, +) { + t.Helper() + runner, err := pgstore.NewMigrationRunner(client, schema, history) + if err != nil { + f.fail("new migration runner for "+schema, err) + } + result, err := runner.Migrate(f.ctx) + if err != nil { + f.fail("migrate schema "+schema, err) + } + if len(result.Applied) != len(wantVersions) { + t.Fatalf("schema %s applied %d migrations, want %d", schema, len(result.Applied), len(wantVersions)) + } + if result.LatestVersion != wantTop { + t.Fatalf("schema %s latest version = %d, want %d", schema, result.LatestVersion, wantTop) + } + for i, want := range wantVersions { + if result.Applied[i].Version != want { + t.Fatalf("schema %s ledger position %d = %d, want %d", schema, i, result.Applied[i].Version, want) + } + } +} + +func (f *pgactFixture) newPool(t *testing.T) *pgstore.Client { + t.Helper() + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, MaxConns: 8, MinConns: 0, AcquireTimeout: 10 * time.Second, + }) + if err != nil { + f.fail("open pg-store pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (f *pgactFixture) newStore(t *testing.T, client *pgstore.Client) *Store { + t.Helper() + store, err := New(client, f.ctlSchema) + if err != nil { + f.fail("build Explorer control-plane store", err) + } + return store +} + +func (f *pgactFixture) newPublishStore(t *testing.T, client *pgstore.Client) *PublishStore { + t.Helper() + ps, err := NewPublishStore(client, f.ctlSchema, f.lanesSchema) + if err != nil { + f.fail("build publish store", err) + } + return ps +} + +func (f *pgactFixture) newActivationStore(t *testing.T, client *pgstore.Client) *ActivationStore { + t.Helper() + as, err := NewActivationStore(client, f.ctlSchema, f.lanesSchema, f.execsSchema) + if err != nil { + f.fail("build activation store", err) + } + return as +} + +func (f *pgactFixture) newGenerationStore(t *testing.T, client *pgstore.Client) *datasetpostgres.Store { + t.Helper() + gen, err := datasetpostgres.New(client, f.lanesSchema) + if err != nil { + f.fail("build generation store", err) + } + return gen +} + +func (f *pgactFixture) newReleaseStore(t *testing.T, client *pgstore.Client) *datasetpostgres.ReleaseStore { + t.Helper() + rel, err := datasetpostgres.NewReleaseStore(client, f.lanesSchema) + if err != nil { + f.fail("build release store", err) + } + return rel +} + +func (f *pgactFixture) newRegistry(t *testing.T) *publicationpostgres.Registry { + t.Helper() + registry, err := publicationpostgres.New(f.iso, f.execsSchema) + if err != nil { + f.fail("build publication registry", err) + } + return registry +} + +func (f *pgactFixture) ctlTable(table string) string { + return sanitizeTableRef(f.ctlSchema, table) +} + +func (f *pgactFixture) lanesTable(table string) string { + return sanitizeTableRef(f.lanesSchema, table) +} + +func (f *pgactFixture) execsTable(table string) string { + return sanitizeTableRef(f.execsSchema, table) +} + +func (f *pgactFixture) execRaw(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := f.iso.Exec(f.ctx, sql, args...); err != nil { + f.fail("direct exec: "+sql, err) + } +} + +type pgactRevisionState struct { + Status string + ActivatedAt *time.Time + FailedAt *time.Time + Diagnostics string + Publication string +} + +func (f *pgactFixture) revisionState(t *testing.T, id string) pgactRevisionState { + t.Helper() + var out pgactRevisionState + err := f.iso.QueryRow(f.ctx, "SELECT status, activated_at, failed_at, COALESCE(diagnostics::text, ''), COALESCE(publication::text, '') FROM "+ + f.ctlTable(RevisionsTable)+" WHERE id = $1", id). + Scan(&out.Status, &out.ActivatedAt, &out.FailedAt, &out.Diagnostics, &out.Publication) + if err != nil { + f.fail("read revision state "+id, err) + } + return out +} + +func (f *pgactFixture) ownerActivationState(t *testing.T, project string) (string, time.Time) { + t.Helper() + var activeRevisionID string + var updatedAt time.Time + err := f.iso.QueryRow(f.ctx, "SELECT active_revision_id, updated_at FROM "+ + f.ctlTable(OwnerTable)+" WHERE project = $1 AND explorer_id = 'default'", project). + Scan(&activeRevisionID, &updatedAt) + if err != nil { + f.fail("read owner activation state "+project, err) + } + return activeRevisionID, updatedAt +} + +func (f *pgactFixture) snapshotRevisions(t *testing.T, project string) string { + t.Helper() + var out string + err := f.iso.QueryRow(f.ctx, "SELECT COALESCE(string_agg(row_text, E'\\x1e' ORDER BY row_text), '') FROM (SELECT (to_jsonb(t) - 'activated_at' - 'created_at')::text AS row_text FROM "+ + f.ctlTable(RevisionsTable)+" t WHERE project = $1) s", project).Scan(&out) + if err != nil { + f.fail("snapshot revisions "+project, err) + } + return out +} + +func (f *pgactFixture) snapshotOwners(t *testing.T, project string) string { + t.Helper() + var out string + err := f.iso.QueryRow(f.ctx, "SELECT COALESCE(string_agg(row_text, E'\\x1e' ORDER BY row_text), '') FROM (SELECT (to_jsonb(t) - 'updated_at')::text AS row_text FROM "+ + f.ctlTable(OwnerTable)+" t WHERE project = $1) s", project).Scan(&out) + if err != nil { + f.fail("snapshot owners "+project, err) + } + return out +} + +func pgactRevisionWithMaterialization(id, project, receiptID, tag, materializationID string, withMaterialization bool) explorer.Revision { + revision := pubauthRevision(id, project, "default", receiptID, tag) + if withMaterialization { + revision.Materializations = []explorer.Materialization{{ + OutputID: "output", Output: "output", MaterializationID: materializationID, + Fingerprint: "fp-pgact-" + tag, + }} + } else { + revision.Materializations = nil + } + return revision +} + +func (f *pgactFixture) stageGeneration(t *testing.T, gen *datasetpostgres.Store, project, generation, urn string) dataset.Manifest { + t.Helper() + ref, err := dataset.NewRef(project, generation) + if err != nil { + f.fail("ref", err) + } + snapshot, err := dataset.NewSchemaSnapshot(urn, "4.0.1", pubauthSnapshotSHA256, []string{"Patient"}) + if err != nil { + f.fail("snapshot", err) + } + loading, err := dataset.NewManifest(ref, snapshot) + if err != nil { + f.fail("manifest", err) + } + if _, err := gen.CreateManifest(f.ctx, loading); err != nil { + f.fail("create manifest", err) + } + staged, err := gen.TransitionManifest(f.ctx, loading, dataset.StateStaged) + if err != nil { + f.fail("stage manifest", err) + } + return staged +} + +func (f *pgactFixture) activateGeneration(t *testing.T, gen *datasetpostgres.Store, staged dataset.Manifest, expected *dataset.Ref) { + t.Helper() + if err := gen.ActivateGeneration(f.ctx, staged, expected); err != nil { + f.fail("activate generation", err) + } +} + +func (f *pgactFixture) seedPublishedExecution(t *testing.T, registry *publicationpostgres.Registry, project, generation, executionID, state string) { + t.Helper() + unique := strings.TrimPrefix(executionID, "pgact-exec-") + key := "pgact-key-" + unique + owner := "pgact-publisher-" + unique + claimed, err := registry.AcquireBundleLease(f.ctx, key, owner, time.Now().UTC().Add(5*time.Minute)) + if err != nil || !claimed { + f.fail("acquire publication lease", fmt.Errorf("claimed=%v err=%v", claimed, err)) + } + stamp := time.Now().UTC().Truncate(time.Millisecond) + execution := dataframepublication.BundleExecution{ + ID: executionID, + Key: key, + BundleIdentity: dataframepublication.BundleIdentity{ + Name: "pgact-recipe", TranslationVersion: "v1", OutputName: "output", + Project: project, DatasetGeneration: generation, + RecipeDigest: "sha256:pgact-" + unique + "-r", SchemaDigest: "sha256:pgact-" + unique + "-s", + ReceiptID: "pgact-receipt-" + unique, ScopeDigest: "pgact-scope-" + unique, EngineVersion: "pgact-engine", + }, + State: dataframepublication.BundleState(state), + Outputs: []dataframepublication.BundleOutputRecord{{ + Name: "output", PhysicalTable: "ch_pgact_" + unique, + Selector: dataset.DataframeSelector{Recipe: "pgact-recipe", TranslationVersion: "v1", Output: "output"}, + RowCount: 7, ByteCount: 700, + State: dataframepublication.BundleState(state), VerifiedAt: &stamp, + }}, + CreatedAt: stamp.Add(-time.Minute), + UpdatedAt: stamp, + OwnerID: owner, + } + if state == "PUBLISHED" { + execution.ReadyAt = &stamp + execution.PublishedAt = &stamp + } + if err := registry.SaveExecution(f.ctx, execution, owner); err != nil { + f.fail("save execution "+executionID, err) + } +} + +type pgactWorld struct { + store *Store + publish *PublishStore + activation *ActivationStore + registry *publicationpostgres.Registry + generations *datasetpostgres.Store + releases *datasetpostgres.ReleaseStore + project string + priorID string + candidateID string + executionID string + generation string +} + +func (f *pgactFixture) seedWorld(t *testing.T, client *pgstore.Client, project, tag string, + mutateCandidate func(revision *explorer.Revision), opts func(w *pgactWorld), +) *pgactWorld { + t.Helper() + world := &pgactWorld{ + project: project, + generation: pubauthGeneration, + priorID: "pgact-prior-" + tag, + candidateID: "pgact-candidate-" + tag, + executionID: "pgact-exec-" + tag, + } + world.store = f.newStore(t, client) + world.publish = f.newPublishStore(t, client) + world.activation = f.newActivationStore(t, client) + world.registry = f.newRegistry(t) + world.generations = f.newGenerationStore(t, client) + world.releases = f.newReleaseStore(t, client) + + if _, err := world.store.Create(f.ctx, pubauthOwner(project, "default", explorer.ManagementRepository)); err != nil { + f.fail("create repository owner", err) + } + staged := f.stageGeneration(t, world.generations, project, world.generation, "urn:pgact-"+tag) + f.activateGeneration(t, world.generations, staged, nil) + + release := pubauthRelease(project, world.generation, "pgact-rel-"+tag) + if _, err := world.releases.SaveRelease(f.ctx, release); err != nil { + f.fail("save prior release", err) + } + priorReceipt := pubauthReceipt(t, project, "default", tag+"-prior") + priorRevision := pubauthRevision(world.priorID, project, "default", priorReceipt.ID, tag+"-prior") + priorRevision.Materializations = []explorer.Materialization{{ + OutputID: "output", Output: "output", MaterializationID: "pgact-prior-mat-" + tag, Fingerprint: "fp-pgact-prior", + }} + if _, err := world.publish.PublishAuthoring(f.ctx, priorReceipt, priorRevision, release, 0); err != nil { + f.fail("publish prior revision", err) + } + f.seedPublishedExecution(t, world.registry, project, world.generation, world.executionID, "PUBLISHED") + + candidateReceipt := pubauthReceipt(t, project, "default", tag+"-cand") + candidate := pgactRevisionWithMaterialization(world.candidateID, project, candidateReceipt.ID, tag+"-cand", world.executionID, true) + if mutateCandidate != nil { + mutateCandidate(&candidate) + } + if _, err := world.store.InsertRevision(f.ctx, candidate); err != nil { + f.fail("insert candidate revision", err) + } + if opts != nil { + opts(world) + } + return world +} + +func requireActivationConflict(t *testing.T, f *pgactFixture, err error, label string) { + t.Helper() + if err == nil { + t.Fatalf("%s: activation returned nil, want explorer.ErrDraftConflict", label) + } + if !errors.Is(err, explorer.ErrDraftConflict) { + t.Fatalf("%s: error = %s, want explorer.ErrDraftConflict", label, f.scrub(err)) + } +} + +func TestActivateRepositoryGenerationUsesCompositeGuardsAgainstPostgreSQL(t *testing.T) { + f := newPGActivationFixture(t) + f.migrate(t) + client := f.newPool(t) + + t.Run("happy-flip-supersedes-prior-and-is-idempotent", func(t *testing.T) { + const project = "pgact_happy_01" + world := f.seedWorld(t, client, project, "happy01", nil, nil) + + if err := world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, world.candidateID); err != nil { + f.fail("ActivateRepositoryGeneration", err) + } + candidate := f.revisionState(t, world.candidateID) + if candidate.Status != "ACTIVE" { + t.Fatalf("candidate status = %s, want ACTIVE", candidate.Status) + } + if candidate.ActivatedAt == nil || candidate.ActivatedAt.IsZero() { + t.Fatalf("candidate activated_at = %v, want set", candidate.ActivatedAt) + } + if candidate.FailedAt != nil { + t.Fatalf("candidate failed_at = %v, want NULL", candidate.FailedAt) + } + if candidate.Diagnostics != "[]" { + t.Fatalf("candidate diagnostics = %s, want []", candidate.Diagnostics) + } + var publication struct { + State string `json:"state"` + RevisionID string `json:"revisionId"` + Generation string `json:"generation"` + UpdatedAt time.Time `json:"updatedAt"` + } + if err := json.Unmarshal([]byte(candidate.Publication), &publication); err != nil { + f.fail("decode candidate publication", err) + } + if publication.State != "ACTIVE" || publication.RevisionID != world.candidateID || + publication.Generation != world.generation || publication.UpdatedAt.IsZero() { + t.Fatalf("activation publication = %+v, want ACTIVE/%s/%s/non-zero updatedAt", + publication, world.candidateID, world.generation) + } + if prior := f.revisionState(t, world.priorID); prior.Status != "SUPERSEDED" { + t.Fatalf("prior status = %s, want SUPERSEDED", prior.Status) + } + if active, _ := f.ownerActivationState(t, project); active != world.candidateID { + t.Fatalf("owner active_revision_id = %q, want %q", active, world.candidateID) + } + + before := f.revisionState(t, world.candidateID) + beforeOwner, beforeOwnerUpdated := f.ownerActivationState(t, project) + if err := world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, world.candidateID); err != nil { + f.fail("idempotent replay", err) + } + after := f.revisionState(t, world.candidateID) + afterOwner, afterOwnerUpdated := f.ownerActivationState(t, project) + if before.ActivatedAt == nil || after.ActivatedAt == nil || !before.ActivatedAt.Equal(*after.ActivatedAt) { + t.Fatalf("replay restated activated_at: %v -> %v", before.ActivatedAt, after.ActivatedAt) + } + if beforeOwner != afterOwner || !beforeOwnerUpdated.Equal(afterOwnerUpdated) { + t.Fatalf("replay churned the owner pointer: %q/%v -> %q/%v", beforeOwner, beforeOwnerUpdated, afterOwner, afterOwnerUpdated) + } + }) + + guardCases := []struct { + name string + candidate func(revision *explorer.Revision) + setup func(f *pgactFixture, w *pgactWorld) + }{ + {"no-manifest", nil, func(f *pgactFixture, w *pgactWorld) { + f.execRaw(t, "DELETE FROM "+f.lanesTable("dataset_active_generation")+" WHERE project = $1", w.project) + f.execRaw(t, "DELETE FROM "+f.lanesTable("dataset_generation_manifest")+" WHERE project = $1", w.project) + }}, + {"manifest-failed", nil, func(f *pgactFixture, w *pgactWorld) { + f.execRaw(t, "UPDATE "+f.lanesTable("dataset_generation_manifest")+" SET state = 'FAILED' WHERE project = $1", w.project) + }}, + {"no-active-generation", nil, func(f *pgactFixture, w *pgactWorld) { + f.execRaw(t, "DELETE FROM "+f.lanesTable("dataset_active_generation")+" WHERE project = $1", w.project) + }}, + {"pointer-names-other-generation", nil, func(f *pgactFixture, w *pgactWorld) { + other := w.generation + "_second" + staged := f.stageGeneration(t, w.generations, w.project, other, "urn:pgact-second-"+w.project) + current, err := dataset.NewRef(w.project, w.generation) + if err != nil { + f.fail("second-generation pointer expected ref", err) + } + f.activateGeneration(t, w.generations, staged, ¤t) + }}, + {"execution-missing", nil, func(f *pgactFixture, w *pgactWorld) { + f.execRaw(t, "DELETE FROM "+f.execsTable(publicationpostgres.BundleExecutionsTable)+" WHERE id = $1", w.executionID) + }}, + {"execution-not-published", nil, func(f *pgactFixture, w *pgactWorld) { + f.execRaw(t, "UPDATE "+f.execsTable(publicationpostgres.BundleExecutionsTable)+" SET state = 'READY' WHERE id = $1", w.executionID) + }}, + {"zero-materializations", func(revision *explorer.Revision) { + revision.Materializations = nil + }, nil}, + {"owner-not-repository-mode", nil, func(f *pgactFixture, w *pgactWorld) { + f.execRaw(t, "UPDATE "+f.ctlTable(OwnerTable)+" SET management_mode = 'INTERACTIVE' WHERE project = $1 AND explorer_id = 'default'", w.project) + }}, + {"candidate-wrong-explorer", func(revision *explorer.Revision) { + revision.ExplorerID = "other" + }, nil}, + {"candidate-wrong-generation", func(revision *explorer.Revision) { + revision.SourceGeneration = "pgact_other_gen" + }, nil}, + {"prior-pointer-failed-candidate-revival-is-no-bypass", nil, func(f *pgactFixture, w *pgactWorld) { + if _, err := w.store.FailRevisionFrom(f.ctx, w.priorID, explorer.RevisionActive, + []explorer.Diagnostic{{Severity: "ERROR", Code: "GENERATION_ACTIVATION_FAILED", + Message: "injected prior failure", Retryable: false}}); err != nil { + f.fail("fail prior before FAILED-prior strict probe", err) + } + }}, + } + for _, tc := range guardCases { + t.Run(tc.name, func(t *testing.T) { + project := "pgact_guard_" + strings.ReplaceAll(tc.name, "-", "_") + tag := strings.TrimPrefix(project, "pgact_guard_") + world := f.seedWorld(t, client, project, tag, tc.candidate, func(w *pgactWorld) { + if tc.setup != nil { + tc.setup(f, w) + } + }) + beforeRevisions := f.snapshotRevisions(t, project) + beforeOwners := f.snapshotOwners(t, project) + + err := world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, world.candidateID) + requireActivationConflict(t, f, err, tc.name) + + if after := f.snapshotRevisions(t, project); after != beforeRevisions { + t.Fatalf("%s: guard miss mutated revisions:\nbefore=%s\nafter=%s", tc.name, f.scrub(beforeRevisions), f.scrub(after)) + } + if after := f.snapshotOwners(t, project); after != beforeOwners { + t.Fatalf("%s: guard miss mutated owners:\nbefore=%s\nafter=%s", tc.name, f.scrub(beforeOwners), f.scrub(after)) + } + }) + } + + t.Run("missing-candidate-revision", func(t *testing.T) { + const project = "pgact_missing_candidate" + world := f.seedWorld(t, client, project, "missingcand", nil, nil) + err := world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, "pgact-revision-that-does-not-exist") + requireActivationConflict(t, f, err, "missing-candidate-revision") + }) + + t.Run("dangling-prior-pointer", func(t *testing.T) { + const project = "pgact_dangling_prior" + world := f.seedWorld(t, client, project, "dangling01", nil, func(w *pgactWorld) { + f.execRaw(t, "UPDATE "+f.ctlTable(OwnerTable)+" SET active_revision_id = 'pgact-revision-that-does-not-exist' WHERE project = $1 AND explorer_id = 'default'", w.project) + }) + beforeRevisions := f.snapshotRevisions(t, project) + beforeOwners := f.snapshotOwners(t, project) + err := world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, world.candidateID) + requireActivationConflict(t, f, err, "dangling-prior-pointer") + if after := f.snapshotRevisions(t, project); after != beforeRevisions { + t.Fatalf("dangling prior mutated revisions:\nbefore=%s\nafter=%s", f.scrub(beforeRevisions), f.scrub(after)) + } + if after := f.snapshotOwners(t, project); after != beforeOwners { + t.Fatalf("dangling prior mutated owners:\nbefore=%s\nafter=%s", f.scrub(beforeOwners), f.scrub(after)) + } + if candidate := f.revisionState(t, world.candidateID); candidate.Status != "READY" { + t.Fatalf("candidate status after dangling prior = %s, want READY", candidate.Status) + } + }) + + t.Run("replay-with-orphaned-active-candidate-conflicts", func(t *testing.T) { + const project = "pgact_orphan_active" + world := f.seedWorld(t, client, project, "orphana1", nil, nil) + if err := world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, world.candidateID); err != nil { + f.fail("initial activation", err) + } + f.execRaw(t, "UPDATE "+f.ctlTable(OwnerTable)+" SET active_revision_id = $2 WHERE project = $1 AND explorer_id = 'default'", project, world.priorID) + beforeRevisions := f.snapshotRevisions(t, project) + beforeOwners := f.snapshotOwners(t, project) + + err := world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, world.candidateID) + requireActivationConflict(t, f, err, "replay-with-orphaned-active-candidate-conflicts") + + if after := f.snapshotRevisions(t, project); after != beforeRevisions { + t.Fatalf("orphaned-ACTIVE replay mutated revisions:\nbefore=%s\nafter=%s", f.scrub(beforeRevisions), f.scrub(after)) + } + if after := f.snapshotOwners(t, project); after != beforeOwners { + t.Fatalf("orphaned-ACTIVE replay churned the owner pointer:\nbefore=%s\nafter=%s", f.scrub(beforeOwners), f.scrub(after)) + } + }) + + t.Run("superseded-candidate-conflicts", func(t *testing.T) { + const project = "pgact_superseded_cand" + world := f.seedWorld(t, client, project, "supcan01", nil, nil) + if err := world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, world.candidateID); err != nil { + f.fail("initial activation", err) + } + beforeRevisions := f.snapshotRevisions(t, project) + beforeOwners := f.snapshotOwners(t, project) + + err := world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, world.priorID) + requireActivationConflict(t, f, err, "superseded-candidate-conflicts") + + if after := f.snapshotRevisions(t, project); after != beforeRevisions { + t.Fatalf("superseded-candidate activation mutated revisions:\nbefore=%s\nafter=%s", f.scrub(beforeRevisions), f.scrub(after)) + } + if after := f.snapshotOwners(t, project); after != beforeOwners { + t.Fatalf("superseded-candidate activation churned the owner pointer:\nbefore=%s\nafter=%s", f.scrub(beforeOwners), f.scrub(after)) + } + if candidate := f.revisionState(t, world.candidateID); candidate.Status != "ACTIVE" { + t.Fatalf("current ACTIVE revision downgraded by superseded-candidate attempt: %s", candidate.Status) + } + }) + + t.Run("pointer-already-names-ready-candidate-activates-without-self-supersede", func(t *testing.T) { + const project = "pgact_prior_not_active" + world := f.seedWorld(t, client, project, "priornotact", nil, func(w *pgactWorld) { + f.execRaw(t, "UPDATE "+f.ctlTable(OwnerTable)+" SET active_revision_id = $2 WHERE project = $1 AND explorer_id = 'default'", w.project, w.candidateID) + }) + if err := world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, world.candidateID); err != nil { + f.fail("activation with pointer already naming the READY candidate", err) + } + if candidate := f.revisionState(t, world.candidateID); candidate.Status != "ACTIVE" { + t.Fatalf("candidate status = %s, want ACTIVE", candidate.Status) + } + if active, _ := f.ownerActivationState(t, project); active != world.candidateID { + t.Fatalf("owner active_revision_id = %q, want %q", active, world.candidateID) + } + if prior := f.revisionState(t, world.priorID); prior.Status != "ACTIVE" { + t.Fatalf("prior status = %s, want ACTIVE: supersede follows the pointer, and the pointer named the candidate", prior.Status) + } + }) + + t.Run("failed-candidate-revival-matches-fresh-activation-witnesses", func(t *testing.T) { + const project = "pgact_revival_01" + world := f.seedWorld(t, client, project, "revival1", nil, nil) + if _, err := world.store.FailRevisionFrom(f.ctx, world.candidateID, explorer.RevisionReady, + []explorer.Diagnostic{{Severity: "ERROR", Code: "GENERATION_ACTIVATION_FAILED", + Message: "injected pre-activation failure", Retryable: false}}); err != nil { + f.fail("fail candidate before revival", err) + } + failed := f.revisionState(t, world.candidateID) + if failed.Status != "FAILED" || failed.FailedAt == nil || failed.Diagnostics == "[]" || failed.Diagnostics == "" { + t.Fatalf("pre-revival candidate state = %+v, want FAILED with failed_at set and diagnostics recorded", failed) + } + if err := world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, world.candidateID); err != nil { + f.fail("revival activation of FAILED candidate", err) + } + revived := f.revisionState(t, world.candidateID) + if revived.Status != "ACTIVE" { + t.Fatalf("revived candidate status = %s, want ACTIVE", revived.Status) + } + if revived.ActivatedAt == nil || revived.ActivatedAt.IsZero() { + t.Fatalf("revived candidate activated_at = %v, want set", revived.ActivatedAt) + } + if revived.FailedAt != nil { + t.Fatalf("revived candidate failed_at = %v, want cleared", revived.FailedAt) + } + if revived.Diagnostics != "[]" { + t.Fatalf("revived candidate diagnostics = %s, want []", revived.Diagnostics) + } + var publication struct { + State string `json:"state"` + RevisionID string `json:"revisionId"` + Generation string `json:"generation"` + UpdatedAt time.Time `json:"updatedAt"` + } + if err := json.Unmarshal([]byte(revived.Publication), &publication); err != nil { + f.fail("decode revived publication", err) + } + if publication.State != "ACTIVE" || publication.RevisionID != world.candidateID || + publication.Generation != world.generation || publication.UpdatedAt.IsZero() { + t.Fatalf("revived publication = %+v, want ACTIVE/%s/%s/non-zero updatedAt", + publication, world.candidateID, world.generation) + } + if prior := f.revisionState(t, world.priorID); prior.Status != "SUPERSEDED" { + t.Fatalf("prior status after revival = %s, want SUPERSEDED", prior.Status) + } + if active, _ := f.ownerActivationState(t, project); active != world.candidateID { + t.Fatalf("owner active_revision_id after revival = %q, want %q", active, world.candidateID) + } + + const freshProject = "pgact_revival_fresh" + fresh := f.seedWorld(t, client, freshProject, "revfrsh1", nil, nil) + if err := fresh.activation.ActivateRepositoryGeneration(f.ctx, freshProject, fresh.generation, fresh.candidateID); err != nil { + f.fail("fresh activation witness world", err) + } + freshState := f.revisionState(t, fresh.candidateID) + if freshState.Status != revived.Status || freshState.Diagnostics != revived.Diagnostics || + freshState.FailedAt != nil || freshState.ActivatedAt == nil { + t.Fatalf("revival witnesses diverge from fresh activation: revived=%+v fresh=%+v", revived, freshState) + } + var freshPublication struct { + State string `json:"state"` + Generation string `json:"generation"` + } + if err := json.Unmarshal([]byte(freshState.Publication), &freshPublication); err != nil { + f.fail("decode fresh publication witness", err) + } + if freshPublication.State != publication.State || freshPublication.Generation != publication.Generation { + t.Fatalf("revival publication diverges from fresh activation: revived=%+v fresh=%+v", publication, freshPublication) + } + + before := f.revisionState(t, world.candidateID) + beforeOwner, beforeOwnerUpdated := f.ownerActivationState(t, project) + if err := world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, world.candidateID); err != nil { + f.fail("replay after revival", err) + } + after := f.revisionState(t, world.candidateID) + afterOwner, afterOwnerUpdated := f.ownerActivationState(t, project) + if before.ActivatedAt == nil || after.ActivatedAt == nil || !before.ActivatedAt.Equal(*after.ActivatedAt) { + t.Fatalf("replay after revival restated activated_at: %v -> %v", before.ActivatedAt, after.ActivatedAt) + } + if beforeOwner != afterOwner || !beforeOwnerUpdated.Equal(afterOwnerUpdated) { + t.Fatalf("replay after revival churned the owner pointer: %q/%v -> %q/%v", beforeOwner, beforeOwnerUpdated, afterOwner, afterOwnerUpdated) + } + }) + + t.Run("ready-only-first-cas-never-touches-failed-candidate", func(t *testing.T) { + const project = "pgact_firstcas_readyonly" + world := f.seedWorld(t, client, project, "fstcas01", nil, nil) + if _, err := world.store.FailRevisionFrom(f.ctx, world.candidateID, explorer.RevisionReady, + []explorer.Diagnostic{{Severity: "ERROR", Code: "GENERATION_ACTIVATION_FAILED", + Message: "injected failure for first-CAS pin", Retryable: false}}); err != nil { + f.fail("fail candidate for first-CAS pin", err) + } + tag, err := f.iso.Exec(f.ctx, "UPDATE "+f.ctlTable(RevisionsTable)+ + " SET status = 'ACTIVE', activated_at = $2, failed_at = NULL, diagnostics = '[]'::jsonb"+ + " WHERE id = $1 AND status = 'READY'", world.candidateID, time.Now().UTC()) + if err != nil { + f.fail("ready-only first CAS against FAILED candidate", err) + } + if tag.RowsAffected() != 0 { + t.Fatalf("ready-only first CAS affected %d rows, want 0: revival must ride its own statement", tag.RowsAffected()) + } + if candidate := f.revisionState(t, world.candidateID); candidate.Status != "FAILED" { + t.Fatalf("candidate status after ready-only first CAS = %s, want FAILED", candidate.Status) + } + }) + + t.Run("failed-candidate-guard-refusal-leaves-failed-state-byte-stable", func(t *testing.T) { + const project = "pgact_revival_guard" + world := f.seedWorld(t, client, project, "revguard", nil, nil) + if _, err := world.store.FailRevisionFrom(f.ctx, world.candidateID, explorer.RevisionReady, + []explorer.Diagnostic{{Severity: "ERROR", Code: "GENERATION_ACTIVATION_FAILED", + Message: "injected failure before guard refusal", Retryable: false}}); err != nil { + f.fail("fail candidate before guard refusal", err) + } + f.execRaw(t, "UPDATE "+f.lanesTable("dataset_generation_manifest")+" SET state = 'FAILED' WHERE project = $1", project) + beforeRevisions := f.snapshotRevisions(t, project) + beforeOwners := f.snapshotOwners(t, project) + beforeCandidate := f.revisionState(t, world.candidateID) + + err := world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, world.candidateID) + requireActivationConflict(t, f, err, "guard-refused revival") + + if after := f.snapshotRevisions(t, project); after != beforeRevisions { + t.Fatalf("guard-refused revival mutated revisions:\nbefore=%s\nafter=%s", f.scrub(beforeRevisions), f.scrub(after)) + } + if after := f.snapshotOwners(t, project); after != beforeOwners { + t.Fatalf("guard-refused revival churned owners:\nbefore=%s\nafter=%s", f.scrub(beforeOwners), f.scrub(after)) + } + afterCandidate := f.revisionState(t, world.candidateID) + if afterCandidate.Status != "FAILED" || afterCandidate.FailedAt == nil || + !afterCandidate.FailedAt.Equal(*beforeCandidate.FailedAt) || + afterCandidate.Diagnostics != beforeCandidate.Diagnostics { + t.Fatalf("guard-refused revival disturbed the FAILED candidate: before=%+v after=%+v", beforeCandidate, afterCandidate) + } + if active, _ := f.ownerActivationState(t, project); active != world.priorID { + t.Fatalf("owner pointer after guard-refused revival = %q, want %q", active, world.priorID) + } + }) +} + +func installActivationCrashSeam(as *ActivationStore, gate func(step string, tx pgx.Tx) error) *[]string { + steps := &[]string{} + as.crashAt = func(ctx context.Context, tx pgx.Tx, step string) error { + *steps = append(*steps, step) + if gate != nil { + return gate(step, tx) + } + return nil + } + return steps +} + +func TestActivationUsesTransactionsAndSingleModificationQueriesAgainstPostgreSQL(t *testing.T) { + f := newPGActivationFixture(t) + f.migrate(t) + client := f.newPool(t) + + t.Run("mutations-run-in-one-ordered-leg-sequence", func(t *testing.T) { + const project = "pgact_tx_seq_01" + world := f.seedWorld(t, client, project, "txseq01", nil, nil) + steps := installActivationCrashSeam(world.activation, nil) + if err := world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, world.candidateID); err != nil { + f.fail("ActivateRepositoryGeneration", err) + } + want := []string{"prior-supersede", "candidate-activate", "owner-activate", "pre-commit"} + if strings.Join(*steps, ",") != strings.Join(want, ",") { + t.Fatalf("leg sequence = %v, want %v", *steps, want) + } + }) + + for _, crashStep := range []string{"prior-supersede", "candidate-activate", "owner-activate"} { + t.Run("rollback-at-"+crashStep, func(t *testing.T) { + project := "pgact_tx_" + crashStep + tag := strings.ReplaceAll(crashStep, "-", "") + world := f.seedWorld(t, client, project, tag, nil, nil) + steps := installActivationCrashSeam(world.activation, func(step string, _ pgx.Tx) error { + if step == crashStep { + return errors.New("injected activation fault") + } + return nil + }) + beforeRevisions := f.snapshotRevisions(t, project) + beforeOwners := f.snapshotOwners(t, project) + + err := world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, world.candidateID) + if err == nil || !strings.Contains(err.Error(), "injected activation fault") { + t.Fatalf("%s: error = %s, want the injected fault", crashStep, f.scrub(err)) + } + if strings.Join(*steps, ",") == "" { + t.Fatalf("%s: crash seam never fired", crashStep) + } + if after := f.snapshotRevisions(t, project); after != beforeRevisions { + t.Fatalf("%s: fault left partial revision state:\nbefore=%s\nafter=%s", + crashStep, f.scrub(beforeRevisions), f.scrub(after)) + } + if after := f.snapshotOwners(t, project); after != beforeOwners { + t.Fatalf("%s: fault left partial owner state:\nbefore=%s\nafter=%s", + crashStep, f.scrub(beforeOwners), f.scrub(after)) + } + if candidate := f.revisionState(t, world.candidateID); candidate.Status != "READY" { + t.Fatalf("%s: candidate status after rollback = %s, want READY", crashStep, candidate.Status) + } + if prior := f.revisionState(t, world.priorID); prior.Status != "ACTIVE" { + t.Fatalf("%s: prior status after rollback = %s, want ACTIVE", crashStep, prior.Status) + } + if active, _ := f.ownerActivationState(t, project); active != world.priorID { + t.Fatalf("%s: owner pointer after rollback = %q, want %q", crashStep, active, world.priorID) + } + }) + } + + t.Run("activation-races-failure-never-strands", func(t *testing.T) { + const project = "pgact_cas_race_01" + world := f.seedWorld(t, client, project, "race011", nil, nil) + + start := make(chan struct{}) + var wg sync.WaitGroup + var activationErr, failureErr error + wg.Add(2) + go func() { + defer wg.Done() + <-start + activationErr = world.activation.ActivateRepositoryGeneration(f.ctx, project, world.generation, world.candidateID) + }() + go func() { + defer wg.Done() + <-start + _, failureErr = world.store.FailRevisionFrom(f.ctx, world.candidateID, explorer.RevisionReady, + []explorer.Diagnostic{{Severity: "ERROR", Code: "GENERATION_ACTIVATION_FAILED", Message: "racing failure", Retryable: false}}) + }() + close(start) + wg.Wait() + + candidate := f.revisionState(t, world.candidateID) + active, _ := f.ownerActivationState(t, project) + prior := f.revisionState(t, world.priorID) + if activationErr != nil { + t.Fatalf("activation stranded by a racing failure: %s (failure=%s)", + f.scrub(activationErr), f.scrub(failureErr)) + } + if candidate.Status != "ACTIVE" || candidate.ActivatedAt == nil || candidate.FailedAt != nil { + t.Fatalf("race left the candidate stranded: %+v", candidate) + } + if active != world.candidateID { + t.Fatalf("race left the owner pointer at %q, want %q", active, world.candidateID) + } + if prior.Status != "SUPERSEDED" { + t.Fatalf("race left prior status = %s, want SUPERSEDED", prior.Status) + } + if failureErr != nil && !errors.Is(failureErr, ErrStaleRevisionTransition) { + t.Fatalf("racing failure error = %s, want success-revived or ErrStaleRevisionTransition", f.scrub(failureErr)) + } + }) +} diff --git a/internal/backends/postgres/internal/explorer/activation_test.go b/internal/backends/postgres/internal/explorer/activation_test.go new file mode 100644 index 000000000..651c64ef9 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/activation_test.go @@ -0,0 +1,100 @@ +package postgres + +import ( + "context" + "errors" + "strings" + "testing" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" +) + +func TestNewActivationStoreRequiresClient(t *testing.T) { + if _, err := NewActivationStore(nil, "ctl", "lanes", "execs"); err == nil { + t.Fatal("nil client must be refused") + } +} + +func TestNewActivationStoreRejectsUnsafeSchemas(t *testing.T) { + client := &pgstore.Client{} + for _, placement := range [][3]string{ + {"Ctl", "lanes", "execs"}, + {"ctl", "1anes", "execs"}, + {"ctl", "lanes", "bad-name"}, + {"ctl", "lanes", strings.Repeat("a", 64)}, + {"", "lanes", "execs"}, + } { + if _, err := NewActivationStore(client, placement[0], placement[1], placement[2]); err == nil { + t.Fatalf("placement %v must be refused by the schema identifier law", placement) + } else if !strings.Contains(err.Error(), "invalid activation schema") { + t.Fatalf("placement %v error = %v, want the named schema refusal", placement, err) + } + } +} + +func TestActivateRepositoryGenerationRejectsBlankInput(t *testing.T) { + store, err := NewActivationStore(&pgstore.Client{}, "ctl", "lanes", "execs") + if err != nil { + t.Fatal(err) + } + for _, args := range [][3]string{ + {"", "g", "r"}, + {"p", "", "r"}, + {"p", "g", ""}, + } { + err := store.ActivateRepositoryGeneration(context.Background(), args[0], args[1], args[2]) + if !errors.Is(err, ErrInvalidActivationInput) { + t.Fatalf("args %v error = %v, want ErrInvalidActivationInput before any connection", args, err) + } + } +} + +func TestActivationStatementsCarryTheCompositeGuards(t *testing.T) { + stmts := newActivationStatements("ctl", "lanes", "execs") + required := []struct { + name string + statement string + clauses []string + }{ + {"candidateLock", stmts.publish.revisionClaim, []string{ + `"ctl"."explorer_revisions"`, "WHERE id = $1 FOR UPDATE", + }}, + {"manifestGuard", stmts.manifestState, []string{ + `"lanes"."dataset_generation_manifest"`, "WHERE project = $1 AND generation = $2 FOR SHARE", + }}, + {"generationPointerGuard", stmts.generationPointer, []string{ + `"lanes"."dataset_active_generation"`, "WHERE project = $1 FOR SHARE", + }}, + {"executionGuard", stmts.executionState, []string{ + `"execs"."publication_bundle_executions"`, + "WHERE id = $1 AND project = $2 AND dataset_generation = $3 FOR SHARE", + }}, + {"ownerGuard", stmts.publish.ownerGuard, []string{ + `"ctl"."explorers"`, "management_mode = $3", "FOR UPDATE", + }}, + {"priorSupersede", stmts.publish.priorSupersede, []string{ + "SET status = 'SUPERSEDED'", "WHERE id = $1 AND status = 'ACTIVE'", + }}, + {"candidateActivate", stmts.publish.candidateActive, []string{ + "SET status = 'ACTIVE', activated_at = $2, failed_at = NULL, diagnostics = $3, publication = $4", + "WHERE id = $1 AND status = 'READY'", + }}, + {"candidateRevive", stmts.publish.candidateRevive, []string{ + "SET status = 'ACTIVE', activated_at = $2, failed_at = NULL, diagnostics = $3, publication = $4", + "WHERE id = $1 AND status = 'FAILED'", + }}, + {"ownerActivate", stmts.publish.ownerActivate, []string{ + "SET active_revision_id = $3, updated_at = $4", "WHERE project = $1 AND explorer_id = $2", + }}, + } + for _, check := range required { + for _, clause := range check.clauses { + if !strings.Contains(check.statement, clause) { + t.Fatalf("%s lost its audited clause %q:\n%s", check.name, clause, check.statement) + } + } + } + if got := strings.Count(stmts.publish.candidateActive, "UPDATE "); got != 1 { + t.Fatalf("candidate activation must be one statement, found %d:\n%s", got, stmts.publish.candidateActive) + } +} diff --git a/internal/backends/postgres/internal/explorer/capabilities.go b/internal/backends/postgres/internal/explorer/capabilities.go new file mode 100644 index 000000000..8ef1ea7b8 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/capabilities.go @@ -0,0 +1,755 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/explorer/capability" + "github.com/calypr/loom/internal/explorer/tableshapecap" +) + +type capabilityConn interface { + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} + +var _ capabilityConn = (*pgxpool.Conn)(nil) + +const ( + CapabilitySnapshotsTable = "capability_snapshots" + TableShapeCapabilitiesTable = "table_shape_capabilities" +) + +const ( + kindCatalog = "CATALOG" + kindCategoryScan = "CATEGORY_SCAN" + kindResolution = "RESOLUTION" +) + +var ErrStoreInconsistency = errors.New("postgres explorer store: capability artifact stored inconsistently") + +var snapshotColumns = []string{ + "token", "project", "generation", "authorization_scope_digest", "cache_version", + "schema_digest", "resource_inventory_digest", "relationship_digest", "field_digest", + "shape_digest", "protocol_version", "compiler_version", "traversal_policy_version", + "projection_policy_version", "status", "complete", "truncated", "payload", +} + +type CapabilitySnapshotStore struct { + client *pgstore.Client + insert string + getToken string + getIdent string +} + +var _ capability.Repository = (*CapabilitySnapshotStore)(nil) + +func NewCapabilitySnapshotStore(client *pgstore.Client, schema string) (*CapabilitySnapshotStore, error) { + if client == nil { + return nil, errors.New("postgres explorer store: capability snapshot client is required") + } + if !pgstore.ValidSchemaName(schema) { + return nil, fmt.Errorf("postgres explorer store: invalid schema %q: must match [a-z_][a-z0-9_]{0,62}", schema) + } + table := pgx.Identifier{schema, CapabilitySnapshotsTable}.Sanitize() + cols := strings.Join(snapshotColumns, ", ") + placeholders := make([]string, len(snapshotColumns)) + for i := range snapshotColumns { + placeholders[i] = fmt.Sprintf("$%d", i+1) + } + return &CapabilitySnapshotStore{ + client: client, + insert: "INSERT INTO " + table + " (" + cols + ") VALUES (" + strings.Join(placeholders, ", ") + ") " + + "ON CONFLICT (token) DO NOTHING RETURNING payload", + getToken: "SELECT payload FROM " + table + " WHERE token = $1", + getIdent: "SELECT payload FROM " + table + " WHERE project = $1 AND generation = $2 " + + "AND authorization_scope_digest = $3 AND cache_version = $4 AND schema_digest = $5 " + + "AND resource_inventory_digest = $6 AND relationship_digest = $7 AND field_digest = $8 " + + "AND protocol_version = $9 AND compiler_version = $10 AND traversal_policy_version = $11 " + + "AND projection_policy_version = $12 AND shape_digest = $13 AND status IN ('READY', 'BUILDING', 'FAILED') " + + "ORDER BY (status = 'READY') DESC, token LIMIT 1", + }, nil +} + +func capabilitySnapshotDocument(snapshot capability.Snapshot) ([]byte, error) { + var doc map[string]any + if err := json.Unmarshal(snapshot.CanonicalPayload(), &doc); err != nil { + return nil, fmt.Errorf("postgres explorer store: build canonical snapshot document: %w", err) + } + doc["token"] = snapshot.Token + raw, err := json.Marshal(doc) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: encode canonical snapshot document: %w", err) + } + return raw, nil +} + +func snapshotArgs(snapshot capability.Snapshot, document []byte) []any { + id := snapshot.Identity + return []any{ + snapshot.Token, + id.Project, id.Generation, id.AuthorizationScopeDigest, id.CacheVersion, + id.SchemaDigest, id.ResourceInventoryDigest, id.RelationshipDigest, id.FieldDigest, + id.ShapeDigest, id.ProtocolVersion, id.CompilerVersion, id.TraversalPolicyVersion, + id.ProjectionPolicyVersion, + string(snapshot.Status), snapshot.Complete, snapshot.Truncated, + string(document), + } +} + +func decodeSnapshot(raw []byte) (capability.Snapshot, error) { + var value capability.Snapshot + if err := json.Unmarshal(raw, &value); err != nil { + return capability.Snapshot{}, fmt.Errorf("postgres explorer store: decode capability snapshot: %w", err) + } + if err := capability.ValidateSnapshot(value); err != nil { + return capability.Snapshot{}, err + } + return value, nil +} + +func (s *CapabilitySnapshotStore) Put(ctx context.Context, snapshot capability.Snapshot) (*capability.Snapshot, error) { + if err := capability.ValidateSnapshot(snapshot); err != nil { + return nil, err + } + document, err := capabilitySnapshotDocument(snapshot) + if err != nil { + return nil, err + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire snapshot put connection: %w", err) + } + defer conn.Release() + + var stored []byte + err = conn.QueryRow(ctx, s.insert, snapshotArgs(snapshot, document)...).Scan(&stored) + if errors.Is(err, pgx.ErrNoRows) { + existing, getErr := s.getByTokenRaw(ctx, conn, snapshot.Token) + if getErr != nil { + return nil, getErr + } + if existing.Token != snapshot.Token || + string(existing.CanonicalPayload()) != string(snapshot.CanonicalPayload()) { + return nil, fmt.Errorf("%w: %s", capability.ErrTokenCollision, snapshot.Token) + } + if err := capability.ValidateSnapshot(existing); err != nil { + return nil, err + } + return &existing, nil + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: put capability snapshot %q: %w", snapshot.Token, err) + } + value, err := decodeSnapshot(stored) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: decode just-inserted snapshot %q: %w", snapshot.Token, err) + } + return &value, nil +} + +func (s *CapabilitySnapshotStore) GetByToken(ctx context.Context, token string) (*capability.Snapshot, error) { + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire snapshot get connection: %w", err) + } + defer conn.Release() + return s.getByToken(ctx, conn, token) +} + +func (s *CapabilitySnapshotStore) getByToken(ctx context.Context, conn capabilityConn, token string) (*capability.Snapshot, error) { + value, err := s.getByTokenRaw(ctx, conn, token) + if err != nil { + return nil, err + } + if err := capability.ValidateSnapshot(value); err != nil { + return nil, err + } + return &value, nil +} + +func (s *CapabilitySnapshotStore) getByTokenRaw(ctx context.Context, conn capabilityConn, token string) (capability.Snapshot, error) { + var raw []byte + err := conn.QueryRow(ctx, s.getToken, token).Scan(&raw) + if errors.Is(err, pgx.ErrNoRows) { + return capability.Snapshot{}, capability.ErrNotFound + } + if err != nil { + return capability.Snapshot{}, fmt.Errorf("postgres explorer store: get capability snapshot %q: %w", token, err) + } + var value capability.Snapshot + if err := json.Unmarshal(raw, &value); err != nil { + return capability.Snapshot{}, fmt.Errorf("postgres explorer store: decode capability snapshot %q: %w", token, err) + } + if value.Token != token { + return capability.Snapshot{}, fmt.Errorf("%w: snapshot payload token %q is filed under %q", ErrStoreInconsistency, value.Token, token) + } + return value, nil +} + +func (s *CapabilitySnapshotStore) GetByIdentity(ctx context.Context, identity capability.SnapshotIdentity) (*capability.Snapshot, error) { + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire snapshot identity connection: %w", err) + } + defer conn.Release() + + var raw []byte + err = conn.QueryRow(ctx, s.getIdent, + identity.Project, identity.Generation, identity.AuthorizationScopeDigest, identity.CacheVersion, + identity.SchemaDigest, identity.ResourceInventoryDigest, identity.RelationshipDigest, identity.FieldDigest, + identity.ProtocolVersion, identity.CompilerVersion, identity.TraversalPolicyVersion, + identity.ProjectionPolicyVersion, identity.ShapeDigest).Scan(&raw) + if errors.Is(err, pgx.ErrNoRows) { + return nil, capability.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: get capability snapshot by identity: %w", err) + } + value, err := decodeSnapshot(raw) + if err != nil { + return nil, err + } + if value.Identity != identity { + return nil, fmt.Errorf("%w: snapshot %q carries an identity that disagrees with its mirror columns", + ErrStoreInconsistency, value.Token) + } + return &value, nil +} + +var receiptColumns = []string{ + "id", "kind", "parent_catalog_id", "project", "explorer_id", "output_id", "snapshot_token", + "authorization_scope", "source_generation", "draft_version", "draft_digest", + "base_document_digest", "base_compilation_receipt_id", "output_fingerprint", + "compiler_schema_digest", "content_digest", "created_at", "payload", +} + +type TableShapeCapabilityStore struct { + client *pgstore.Client + insert string + catalog string + scanRow string + resolution string + lookup string +} + +var _ tableshapecap.Repository = (*TableShapeCapabilityStore)(nil) + +func NewTableShapeCapabilityStore(client *pgstore.Client, schema string) (*TableShapeCapabilityStore, error) { + if client == nil { + return nil, errors.New("postgres explorer store: table-shape capability client is required") + } + if !pgstore.ValidSchemaName(schema) { + return nil, fmt.Errorf("postgres explorer store: invalid schema %q: must match [a-z_][a-z0-9_]{0,62}", schema) + } + table := pgx.Identifier{schema, TableShapeCapabilitiesTable}.Sanitize() + cols := strings.Join(receiptColumns, ", ") + placeholders := make([]string, len(receiptColumns)) + for i := range receiptColumns { + placeholders[i] = fmt.Sprintf("$%d", i+1) + } + bindingFilter := "WHERE id = $1 AND kind = $2" + bindingColumnFilterFrom(3) + lookupFilter := "WHERE id = $1 AND kind = 'CATALOG'" + lookupColumnFilterFrom(2) + return &TableShapeCapabilityStore{ + client: client, + insert: "INSERT INTO " + table + " (" + cols + ") VALUES (" + strings.Join(placeholders, ", ") + ") ON CONFLICT (id) DO NOTHING RETURNING payload", + catalog: "SELECT " + cols + " FROM " + table + " " + bindingFilter, + scanRow: "SELECT " + cols + " FROM " + table + " WHERE id = $1 AND kind = $2 AND parent_catalog_id = $3" + bindingColumnFilterFrom(4), + resolution: "SELECT " + cols + " FROM " + table + " WHERE id = $1 AND kind = $2 AND parent_catalog_id = $3" + bindingColumnFilterFrom(4), + lookup: "SELECT " + cols + " FROM " + table + " " + lookupFilter, + }, nil +} + +func bindingColumnFilterFrom(start int) string { + columns := []string{ + "project", "explorer_id", "output_id", "snapshot_token", "authorization_scope", + "source_generation", "draft_version", "draft_digest", "base_document_digest", + "base_compilation_receipt_id", "output_fingerprint", "compiler_schema_digest", + } + parts := make([]string, 0, len(columns)) + for i, column := range columns { + parts = append(parts, fmt.Sprintf("AND %s = $%d", column, start+i)) + } + return " " + strings.Join(parts, " ") +} + +func lookupColumnFilterFrom(start int) string { + columns := []string{ + "project", "explorer_id", "output_id", "snapshot_token", "authorization_scope", + "source_generation", "draft_version", "draft_digest", "base_document_digest", + } + parts := make([]string, 0, len(columns)) + for i, column := range columns { + parts = append(parts, fmt.Sprintf("AND %s = $%d", column, start+i)) + } + return " " + strings.Join(parts, " ") +} + +func bindingArgs(binding tableshapecap.Binding) []any { + return []any{ + binding.Project, binding.ExplorerID, binding.OutputID, binding.SnapshotToken, + binding.AuthorizationScope, binding.SourceGeneration, int64(binding.DraftVersion), + binding.DraftDigest, binding.BaseDocumentDigest, binding.BaseCompilationReceiptID, + binding.OutputFingerprint, binding.CompilerSchemaDigest, + } +} + +func receiptArgs(id, kind, parentCatalogID string, binding tableshapecap.Binding, contentDigest, createdAt string, payload []byte) []any { + args := make([]any, 0, len(receiptColumns)) + args = append(args, id, kind, parentCatalogID) + args = append(args, bindingArgs(binding)...) + args = append(args, contentDigest, createdAt, string(payload)) + return args +} + +type receiptRow struct { + id, kind, parentCatalogID string + binding tableshapecap.Binding + contentDigest, createdAt string + payload []byte +} + +func scanReceiptRow(row rowScanner) (receiptRow, error) { + var ( + stored receiptRow + draftVer int64 + ) + targets := []any{ + &stored.id, &stored.kind, &stored.parentCatalogID, + &stored.binding.Project, &stored.binding.ExplorerID, &stored.binding.OutputID, + &stored.binding.SnapshotToken, &stored.binding.AuthorizationScope, &stored.binding.SourceGeneration, + &draftVer, &stored.binding.DraftDigest, &stored.binding.BaseDocumentDigest, + &stored.binding.BaseCompilationReceiptID, &stored.binding.OutputFingerprint, + &stored.binding.CompilerSchemaDigest, + &stored.contentDigest, &stored.createdAt, &stored.payload, + } + if err := row.Scan(targets...); err != nil { + return receiptRow{}, err + } + stored.binding.DraftVersion = uint64(draftVer) + return stored, nil +} + +func (s *TableShapeCapabilityStore) readCatalogRow(ctx context.Context, conn capabilityConn, binding tableshapecap.Binding, id string) (receiptRow, tableshapecap.CatalogReceipt, error) { + args := append([]any{id, kindCatalog}, bindingArgs(binding)...) + stored, err := scanReceiptRow(conn.QueryRow(ctx, s.catalog, args...)) + if errors.Is(err, pgx.ErrNoRows) { + return receiptRow{}, tableshapecap.CatalogReceipt{}, tableshapecap.ErrNotFound + } + if err != nil { + return receiptRow{}, tableshapecap.CatalogReceipt{}, fmt.Errorf("postgres explorer store: read catalog receipt %q: %w", id, err) + } + receipt, err := tableshapecap.DecodeStrict[tableshapecap.CatalogReceipt](stored.payload) + if err != nil { + return stored, tableshapecap.CatalogReceipt{}, fmt.Errorf("%w: catalog %s payload failed validation: %v", ErrStoreInconsistency, id, err) + } + if stored.id != id || stored.kind != kindCatalog || stored.parentCatalogID != "" || + stored.binding != binding || receipt.ID != id || receipt.Binding != binding || + stored.contentDigest != receipt.ContentDigest || stored.createdAt != receipt.CreatedAt { + return stored, tableshapecap.CatalogReceipt{}, fmt.Errorf("%w: catalog %s mirror columns disagree with its payload", ErrStoreInconsistency, id) + } + return stored, receipt, nil +} + +func (s *TableShapeCapabilityStore) GetCatalog(ctx context.Context, binding tableshapecap.Binding, id string) (tableshapecap.CatalogReceipt, error) { + if err := binding.Validate(); err != nil { + return tableshapecap.CatalogReceipt{}, err + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return tableshapecap.CatalogReceipt{}, fmt.Errorf("postgres explorer store: acquire catalog get connection: %w", err) + } + defer conn.Release() + _, receipt, err := s.readCatalogRow(ctx, conn, binding, id) + return receipt, err +} + +func (s *TableShapeCapabilityStore) GetCatalogForLookup(ctx context.Context, lookup tableshapecap.CatalogLookup, id string) (tableshapecap.CatalogReceipt, error) { + if err := lookup.Validate(); err != nil { + return tableshapecap.CatalogReceipt{}, err + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return tableshapecap.CatalogReceipt{}, fmt.Errorf("postgres explorer store: acquire catalog lookup connection: %w", err) + } + defer conn.Release() + + args := append([]any{id}, lookupArgs(lookup)...) + stored, err := scanReceiptRow(conn.QueryRow(ctx, s.lookup, args...)) + if errors.Is(err, pgx.ErrNoRows) { + return tableshapecap.CatalogReceipt{}, tableshapecap.ErrNotFound + } + if err != nil { + return tableshapecap.CatalogReceipt{}, fmt.Errorf("postgres explorer store: lookup catalog receipt %q: %w", id, err) + } + receipt, err := tableshapecap.DecodeStrict[tableshapecap.CatalogReceipt](stored.payload) + if err != nil { + return tableshapecap.CatalogReceipt{}, fmt.Errorf("%w: catalog %s payload failed validation: %v", ErrStoreInconsistency, id, err) + } + if stored.id != id || stored.kind != kindCatalog || stored.parentCatalogID != "" || + !lookup.Matches(stored.binding) || stored.binding != receipt.Binding || + receipt.ID != id || !lookup.Matches(receipt.Binding) || + stored.contentDigest != receipt.ContentDigest || stored.createdAt != receipt.CreatedAt { + return tableshapecap.CatalogReceipt{}, fmt.Errorf("%w: catalog %s mirror columns disagree with its payload", ErrStoreInconsistency, id) + } + return receipt, nil +} + +func lookupArgs(lookup tableshapecap.CatalogLookup) []any { + return []any{ + lookup.Project, lookup.ExplorerID, lookup.OutputID, lookup.SnapshotToken, + lookup.AuthorizationScope, lookup.SourceGeneration, int64(lookup.DraftVersion), + lookup.DraftDigest, lookup.BaseDocumentDigest, + } +} + +func (s *TableShapeCapabilityStore) PutCatalog(ctx context.Context, receipt tableshapecap.CatalogReceipt) (tableshapecap.CatalogReceipt, error) { + if err := receipt.Validate(); err != nil { + return tableshapecap.CatalogReceipt{}, err + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return tableshapecap.CatalogReceipt{}, fmt.Errorf("postgres explorer store: acquire catalog put connection: %w", err) + } + defer conn.Release() + + storedPayload, err := s.insertReceipt(ctx, conn, receipt.ID, kindCatalog, "", receipt.Binding, + receipt.ContentDigest, receipt.CreatedAt, receipt) + if err != nil { + return tableshapecap.CatalogReceipt{}, err + } + if len(storedPayload) > 0 { + return tableshapecap.DecodeStrict[tableshapecap.CatalogReceipt](storedPayload) + } + _, existing, getErr := s.readCatalogRow(ctx, conn, receipt.Binding, receipt.ID) + if getErr != nil { + if errors.Is(getErr, tableshapecap.ErrNotFound) || errors.Is(getErr, ErrStoreInconsistency) { + return tableshapecap.CatalogReceipt{}, fmt.Errorf("%w: %s", tableshapecap.ErrIdentityClash, receipt.ID) + } + return tableshapecap.CatalogReceipt{}, getErr + } + if err := sameCatalog(existing, receipt); err != nil { + return tableshapecap.CatalogReceipt{}, err + } + return existing, nil +} + +func (s *TableShapeCapabilityStore) insertReceipt(ctx context.Context, conn capabilityConn, id, kind, parentCatalogID string, binding tableshapecap.Binding, contentDigest, createdAt string, receipt any) ([]byte, error) { + payload, err := json.Marshal(receipt) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: marshal %s receipt %q: %w", kind, id, err) + } + var stored []byte + err = conn.QueryRow(ctx, s.insert, receiptArgs(id, kind, parentCatalogID, binding, contentDigest, createdAt, payload)...).Scan(&stored) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: insert %s receipt %q: %w", kind, id, err) + } + return stored, nil +} + +func sameCatalog(existing, want tableshapecap.CatalogReceipt) error { + left, err := existing.CanonicalContent() + if err != nil { + return err + } + right, err := want.CanonicalContent() + if err != nil { + return err + } + if existing.ID != want.ID || existing.ContentDigest != want.ContentDigest || string(left) != string(right) { + return fmt.Errorf("%w: %s", tableshapecap.ErrIdentityClash, want.ID) + } + return nil +} + +func sameCategoryScan(existing, want tableshapecap.CategoryScanReceipt) error { + left, err := existing.CanonicalContent() + if err != nil { + return err + } + right, err := want.CanonicalContent() + if err != nil { + return err + } + if existing.ID != want.ID || existing.ContentDigest != want.ContentDigest || string(left) != string(right) { + return fmt.Errorf("%w: %s", tableshapecap.ErrIdentityClash, want.ID) + } + return nil +} + +func sameResolution(existing, want tableshapecap.ResolutionReceipt) error { + left, err := existing.CanonicalContent() + if err != nil { + return err + } + right, err := want.CanonicalContent() + if err != nil { + return err + } + if existing.ID != want.ID || existing.ContentDigest != want.ContentDigest || string(left) != string(right) { + return fmt.Errorf("%w: %s", tableshapecap.ErrIdentityClash, want.ID) + } + return nil +} + +func (s *TableShapeCapabilityStore) readScanRow(ctx context.Context, conn capabilityConn, binding tableshapecap.Binding, parentCatalogID, id string) (receiptRow, tableshapecap.CategoryScanReceipt, error) { + args := append([]any{id, kindCategoryScan, parentCatalogID}, bindingArgs(binding)...) + stored, err := scanReceiptRow(conn.QueryRow(ctx, s.scanRow, args...)) + if errors.Is(err, pgx.ErrNoRows) { + return receiptRow{}, tableshapecap.CategoryScanReceipt{}, tableshapecap.ErrNotFound + } + if err != nil { + return receiptRow{}, tableshapecap.CategoryScanReceipt{}, fmt.Errorf("postgres explorer store: read category-scan receipt %q: %w", id, err) + } + receipt, err := tableshapecap.DecodeStrict[tableshapecap.CategoryScanReceipt](stored.payload) + if err != nil { + return stored, tableshapecap.CategoryScanReceipt{}, fmt.Errorf("%w: category scan %s payload failed validation: %v", ErrStoreInconsistency, id, err) + } + if stored.id != id || stored.kind != kindCategoryScan || stored.parentCatalogID != parentCatalogID || + stored.binding != binding || receipt.ID != id || receipt.Binding != binding || + receipt.ParentCatalogID != parentCatalogID || + stored.contentDigest != receipt.ContentDigest || stored.createdAt != receipt.CreatedAt { + return stored, tableshapecap.CategoryScanReceipt{}, fmt.Errorf("%w: category scan %s mirror columns disagree with its payload", ErrStoreInconsistency, id) + } + return stored, receipt, nil +} + +func (s *TableShapeCapabilityStore) GetCategoryScan(ctx context.Context, binding tableshapecap.Binding, parentCatalogID, id string) (tableshapecap.CategoryScanReceipt, error) { + if err := binding.Validate(); err != nil { + return tableshapecap.CategoryScanReceipt{}, err + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return tableshapecap.CategoryScanReceipt{}, fmt.Errorf("postgres explorer store: acquire scan get connection: %w", err) + } + defer conn.Release() + found, err := s.getCategoryScan(ctx, conn, binding, parentCatalogID, id) + if err != nil { + return tableshapecap.CategoryScanReceipt{}, err + } + _, parent, err := s.readCatalogRow(ctx, conn, binding, parentCatalogID) + if err != nil { + return tableshapecap.CategoryScanReceipt{}, err + } + if err := tableshapecap.ValidateCategoryScanAgainstCatalog(parent, found); err != nil { + return tableshapecap.CategoryScanReceipt{}, err + } + return found, nil +} + +func (s *TableShapeCapabilityStore) getCategoryScan(ctx context.Context, conn capabilityConn, binding tableshapecap.Binding, parentCatalogID, id string) (tableshapecap.CategoryScanReceipt, error) { + _, receipt, err := s.readScanRow(ctx, conn, binding, parentCatalogID, id) + return receipt, err +} + +func (s *TableShapeCapabilityStore) PutCategoryScan(ctx context.Context, receipt tableshapecap.CategoryScanReceipt) (tableshapecap.CategoryScanReceipt, error) { + if err := receipt.Validate(); err != nil { + return tableshapecap.CategoryScanReceipt{}, err + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return tableshapecap.CategoryScanReceipt{}, fmt.Errorf("postgres explorer store: acquire scan put connection: %w", err) + } + defer conn.Release() + + _, parent, err := s.readCatalogRow(ctx, conn, receipt.Binding, receipt.ParentCatalogID) + if err != nil { + return tableshapecap.CategoryScanReceipt{}, err + } + if err := tableshapecap.ValidateCategoryScanAgainstCatalog(parent, receipt); err != nil { + return tableshapecap.CategoryScanReceipt{}, err + } + storedPayload, err := s.insertReceipt(ctx, conn, receipt.ID, kindCategoryScan, receipt.ParentCatalogID, + receipt.Binding, receipt.ContentDigest, receipt.CreatedAt, receipt) + if err != nil { + return tableshapecap.CategoryScanReceipt{}, err + } + if len(storedPayload) > 0 { + return tableshapecap.DecodeStrict[tableshapecap.CategoryScanReceipt](storedPayload) + } + existing, getErr := s.getCategoryScan(ctx, conn, receipt.Binding, receipt.ParentCatalogID, receipt.ID) + if getErr != nil { + if errors.Is(getErr, tableshapecap.ErrNotFound) || errors.Is(getErr, ErrStoreInconsistency) { + return tableshapecap.CategoryScanReceipt{}, fmt.Errorf("%w: %s", tableshapecap.ErrIdentityClash, receipt.ID) + } + return tableshapecap.CategoryScanReceipt{}, getErr + } + if err := sameCategoryScan(existing, receipt); err != nil { + return tableshapecap.CategoryScanReceipt{}, err + } + return existing, nil +} + +func (s *TableShapeCapabilityStore) readResolutionRow(ctx context.Context, conn capabilityConn, binding tableshapecap.Binding, parentCatalogID, id string) (receiptRow, tableshapecap.ResolutionReceipt, error) { + args := append([]any{id, kindResolution, parentCatalogID}, bindingArgs(binding)...) + stored, err := scanReceiptRow(conn.QueryRow(ctx, s.resolution, args...)) + if errors.Is(err, pgx.ErrNoRows) { + return receiptRow{}, tableshapecap.ResolutionReceipt{}, tableshapecap.ErrNotFound + } + if err != nil { + return receiptRow{}, tableshapecap.ResolutionReceipt{}, fmt.Errorf("postgres explorer store: read resolution receipt %q: %w", id, err) + } + receipt, err := tableshapecap.DecodeStrict[tableshapecap.ResolutionReceipt](stored.payload) + if err != nil { + return stored, tableshapecap.ResolutionReceipt{}, fmt.Errorf("%w: resolution %s payload failed validation: %v", ErrStoreInconsistency, id, err) + } + if stored.id != id || stored.kind != kindResolution || stored.parentCatalogID != parentCatalogID || + stored.binding != binding || receipt.ID != id || receipt.Binding != binding || + receipt.ParentCatalogID != parentCatalogID || + stored.contentDigest != receipt.ContentDigest || stored.createdAt != receipt.CreatedAt { + return stored, tableshapecap.ResolutionReceipt{}, fmt.Errorf("%w: resolution %s mirror columns disagree with its payload", ErrStoreInconsistency, id) + } + return stored, receipt, nil +} + +func (s *TableShapeCapabilityStore) GetResolution(ctx context.Context, binding tableshapecap.Binding, parentCatalogID, id string) (tableshapecap.ResolutionReceipt, error) { + if err := binding.Validate(); err != nil { + return tableshapecap.ResolutionReceipt{}, err + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return tableshapecap.ResolutionReceipt{}, fmt.Errorf("postgres explorer store: acquire resolution get connection: %w", err) + } + defer conn.Release() + found, err := s.getResolution(ctx, conn, binding, parentCatalogID, id) + if err != nil { + return tableshapecap.ResolutionReceipt{}, err + } + if err := s.validateResolutionGraph(ctx, conn, binding, parentCatalogID, found); err != nil { + return tableshapecap.ResolutionReceipt{}, err + } + return found, nil +} + +func (s *TableShapeCapabilityStore) getResolution(ctx context.Context, conn capabilityConn, binding tableshapecap.Binding, parentCatalogID, id string) (tableshapecap.ResolutionReceipt, error) { + _, receipt, err := s.readResolutionRow(ctx, conn, binding, parentCatalogID, id) + return receipt, err +} + +func (s *TableShapeCapabilityStore) validateResolutionGraph(ctx context.Context, conn capabilityConn, binding tableshapecap.Binding, parentCatalogID string, resolution tableshapecap.ResolutionReceipt) error { + _, parent, err := s.readCatalogRow(ctx, conn, binding, parentCatalogID) + if err != nil { + return err + } + prior, err := s.resolutionDependencies(ctx, conn, binding, parentCatalogID, resolution) + if err != nil { + return err + } + if err := tableshapecap.ValidateResolutionAgainstCatalog(parent, resolution, prior...); err != nil { + return err + } + if resolution.Pivot != nil { + scan, err := s.getCategoryScan(ctx, conn, binding, parentCatalogID, resolution.Pivot.CategoryDiscoveryID) + if err != nil { + return err + } + if err := tableshapecap.ValidateResolutionAgainstCategoryScan(parent, scan, resolution); err != nil { + return err + } + } + return nil +} + +func (s *TableShapeCapabilityStore) resolutionDependencies(ctx context.Context, conn capabilityConn, binding tableshapecap.Binding, parentCatalogID string, current tableshapecap.ResolutionReceipt) ([]tableshapecap.ResolutionReceipt, error) { + state := map[string]uint8{current.ID: 1} + ordered := make([]tableshapecap.ResolutionReceipt, 0) + var visit func(string) error + visit = func(id string) error { + switch state[id] { + case 1: + return tableshapecap.ErrInvalid + case 2: + return nil + } + state[id] = 1 + previous, err := s.getResolution(ctx, conn, binding, parentCatalogID, id) + if err != nil { + return err + } + switch previous.Kind { + case tableshapecap.ResolutionPivot: + if previous.Pivot == nil { + return tableshapecap.ErrInvalid + } + case tableshapecap.ResolutionDerived: + if previous.Derived == nil { + return tableshapecap.ErrInvalid + } + if previous.Derived.PivotResolutionID != "" { + if err := visit(previous.Derived.PivotResolutionID); err != nil { + return err + } + } + for _, operand := range []tableshapecap.ResolvedOperand{previous.Derived.Left, previous.Derived.Right} { + if operand.Kind == tableshapecap.ResolvedOperandResolution { + if err := visit(operand.ResolutionID); err != nil { + return err + } + } + } + default: + return tableshapecap.ErrInvalid + } + state[id] = 2 + ordered = append(ordered, previous) + return nil + } + if current.Kind == tableshapecap.ResolutionDerived { + if current.Derived.PivotResolutionID != "" { + if err := visit(current.Derived.PivotResolutionID); err != nil { + return nil, err + } + } + for _, operand := range []tableshapecap.ResolvedOperand{current.Derived.Left, current.Derived.Right} { + if operand.Kind == tableshapecap.ResolvedOperandResolution { + if err := visit(operand.ResolutionID); err != nil { + return nil, err + } + } + } + } + return ordered, nil +} + +func (s *TableShapeCapabilityStore) PutResolution(ctx context.Context, receipt tableshapecap.ResolutionReceipt) (tableshapecap.ResolutionReceipt, error) { + if err := receipt.Validate(); err != nil { + return tableshapecap.ResolutionReceipt{}, err + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return tableshapecap.ResolutionReceipt{}, fmt.Errorf("postgres explorer store: acquire resolution put connection: %w", err) + } + defer conn.Release() + + if err := s.validateResolutionGraph(ctx, conn, receipt.Binding, receipt.ParentCatalogID, receipt); err != nil { + return tableshapecap.ResolutionReceipt{}, err + } + storedPayload, err := s.insertReceipt(ctx, conn, receipt.ID, kindResolution, receipt.ParentCatalogID, + receipt.Binding, receipt.ContentDigest, receipt.CreatedAt, receipt) + if err != nil { + return tableshapecap.ResolutionReceipt{}, err + } + if len(storedPayload) > 0 { + return tableshapecap.DecodeStrict[tableshapecap.ResolutionReceipt](storedPayload) + } + existing, getErr := s.getResolution(ctx, conn, receipt.Binding, receipt.ParentCatalogID, receipt.ID) + if getErr != nil { + if errors.Is(getErr, tableshapecap.ErrNotFound) || errors.Is(getErr, ErrStoreInconsistency) { + return tableshapecap.ResolutionReceipt{}, fmt.Errorf("%w: %s", tableshapecap.ErrIdentityClash, receipt.ID) + } + return tableshapecap.ResolutionReceipt{}, getErr + } + if err := sameResolution(existing, receipt); err != nil { + return tableshapecap.ResolutionReceipt{}, err + } + return existing, nil +} diff --git a/internal/backends/postgres/internal/explorer/capabilities_test.go b/internal/backends/postgres/internal/explorer/capabilities_test.go new file mode 100644 index 000000000..551098f82 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/capabilities_test.go @@ -0,0 +1,1729 @@ +package postgres + +import ( + "bytes" + "context" + + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "reflect" + "regexp" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/explorer/capability" + "github.com/calypr/loom/internal/explorer/tableshapecap" +) + +const ( + pgstate03ImmutableCode = "45501" +) + +var pgstate03ResourceNamePattern = regexp.MustCompile(`^pgstate03_it_[0-9]+_[0-9a-f]+$`) + +type capabilityService struct { + ctx context.Context + admin *pgxpool.Pool + witness *pgstore.Client + pidW int32 + pidA int32 + pidB int32 + schema string + scrub func(any) string + fail func(step string, err error) + + snapshots *CapabilitySnapshotStore + snapshotsB *CapabilitySnapshotStore + receipts *TableShapeCapabilityStore + receiptsB *TableShapeCapabilityStore + owner []pgstore.Migration + fixture pgstore.Migration + ledgerAfter []pgstore.LedgerRecord +} + +func newCapabilityService(t *testing.T) *capabilityService { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgstate03_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstate03ResourceNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if _, err := admin.Exec(cleanupCtx, + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + newClient := func(label string) *pgstore.Client { + client, err := pgstore.Open(ctx, pgstore.Config{ + DSN: dsn, + MaxConns: 1, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + fail("open client "+label, err) + } + t.Cleanup(client.Close) + return client + } + witness := newClient("witness") + clientA := newClient("writer-a") + clientB := newClient("writer-b") + + schema := fmt.Sprintf("pgstate03_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstate03ResourceNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if err := execOn(ctx, witness, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if err := execOn(cleanupCtx, witness, `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + s := &capabilityService{ + ctx: ctx, admin: admin, witness: witness, + schema: schema, scrub: scrub, fail: fail, + } + s.migrate(t) + s.pidW = s.backendPID(t, witness, "witness") + s.pidA = s.backendPID(t, clientA, "writer A") + s.pidB = s.backendPID(t, clientB, "writer B") + if s.pidA == s.pidB || s.pidA == s.pidW || s.pidB == s.pidW { + t.Fatalf("writer and witness backends are not distinct physical connections: A=%d B=%d W=%d", s.pidA, s.pidB, s.pidW) + } + + snapshots, err := NewCapabilitySnapshotStore(clientA, schema) + if err != nil { + fail("build snapshot store A", err) + } + s.snapshots = snapshots + snapshotsB, err := NewCapabilitySnapshotStore(clientB, schema) + if err != nil { + fail("build snapshot store B", err) + } + s.snapshotsB = snapshotsB + receipts, err := NewTableShapeCapabilityStore(clientA, schema) + if err != nil { + fail("build receipt store A", err) + } + s.receipts = receipts + receiptsB, err := NewTableShapeCapabilityStore(clientB, schema) + if err != nil { + fail("build receipt store B", err) + } + s.receiptsB = receiptsB + return s +} + +func execOn(ctx context.Context, client *pgstore.Client, sql string, args ...any) error { + conn, err := client.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + _, err = conn.Exec(ctx, sql, args...) + return err +} + +func queryRowOn[T any](ctx context.Context, client *pgstore.Client, dest *T, sql string, args ...any) error { + conn, err := client.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + return conn.QueryRow(ctx, sql, args...).Scan(dest) +} + +func (s *capabilityService) backendPID(t *testing.T, client *pgstore.Client, label string) int32 { + t.Helper() + var pid int32 + if err := queryRowOn(s.ctx, client, &pid, `SELECT pg_backend_pid()`); err != nil { + s.fail("backend pid "+label, err) + } + if pid <= 0 { + t.Fatalf("%s backend pid = %d, want a positive server pid", label, pid) + } + return pid +} + +func (s *capabilityService) migrate(t *testing.T) { + t.Helper() + history, err := OwnerMigrations() + if err != nil { + s.fail("owner migrations", err) + } + if len(history) < 2 || history[0].Version != 1 || history[0].Name != "ledger" || + history[1].Version != 2 || history[1].Name != "explorer_owner" { + t.Fatalf("owner history = %+v, want ledger(1) + explorer_owner(2) at the front", history) + } + var fixture pgstore.Migration + var fixtureFound bool + for _, m := range history { + if m.Version == 4 { + fixture, fixtureFound = m, true + } + } + if !fixtureFound || !strings.HasPrefix(fixture.Name, "capability_artifacts") { + t.Fatalf("owner history version 4 = %+v (found %v), want capability_artifacts", fixture, fixtureFound) + } + const capabilityArtifactsChecksum = "bec3e49f897b28ef71d7d69a5f7c71bb11d13b7c631ee26724de36a89d639222" + if fixture.Checksum != capabilityArtifactsChecksum { + t.Fatalf("capability artifacts checksum = %s, want the byte-frozen merge pin %s: the migration drifted", + fixture.Checksum, capabilityArtifactsChecksum) + } + runner, err := pgstore.NewMigrationRunner(s.witness, s.schema, history) + if err != nil { + s.fail("new migration runner", err) + } + result, err := runner.Migrate(s.ctx) + if err != nil { + s.fail("migrate capability history", err) + } + s.owner = history + s.fixture = fixture + s.ledgerAfter = result.Applied + + want := ownerRecords(history) + var maxVersion int64 + for _, w := range want { + if w.Version > maxVersion { + maxVersion = w.Version + } + } + if result.LatestVersion != maxVersion { + t.Fatalf("latest migration version = %d, want %d", result.LatestVersion, maxVersion) + } + if len(result.Applied) != len(want) { + t.Fatalf("applied history = %+v, want %+v", result.Applied, want) + } + for i, w := range want { + got := result.Applied[i] + if got.Version != w.Version || got.Name != w.Name || got.Checksum != w.Checksum { + t.Fatalf("applied[%d] = {Version:%d Name:%q Checksum:%q}, want {Version:%d Name:%q Checksum:%q}", + i, got.Version, got.Name, got.Checksum, w.Version, w.Name, w.Checksum) + } + } + for i := 1; i < len(result.Applied); i++ { + if result.Applied[i-1].Version >= result.Applied[i].Version { + t.Fatalf("ledger not strictly ascending: %+v", result.Applied) + } + } +} + +func ownerRecords(history []pgstore.Migration) []pgstore.LedgerRecord { + records := make([]pgstore.LedgerRecord, 0, len(history)) + for _, m := range history { + records = append(records, pgstore.LedgerRecord{Version: m.Version, Name: m.Name, Checksum: m.Checksum}) + } + return records +} + +func tablesCreatedIn(sql string) []string { + var code strings.Builder + for _, line := range strings.Split(sql, "\n") { + if idx := strings.Index(line, "--"); idx >= 0 { + line = line[:idx] + } + code.WriteString(line) + code.WriteString("\n") + } + re := regexp.MustCompile(`(?i)CREATE TABLE IF NOT EXISTS\s+([a-zA-Z_][a-zA-Z0-9_]*)`) + matches := re.FindAllStringSubmatch(code.String(), -1) + tables := make([]string, 0, len(matches)) + for _, m := range matches { + tables = append(tables, strings.ToLower(m[1])) + } + return tables +} + +func (s *capabilityService) qualified(table string) string { + return pgx.Identifier{s.schema, table}.Sanitize() +} + +func (s *capabilityService) mustWitnessExec(t *testing.T, sql string, args ...any) { + t.Helper() + if err := execOn(s.ctx, s.witness, sql, args...); err != nil { + s.fail("witness exec "+sql, err) + } +} + +func (s *capabilityService) witnessExecErr(sql string, args ...any) error { + return execOn(s.ctx, s.witness, sql, args...) +} + +func (s *capabilityService) mustWitnessBool(t *testing.T, sql string, args ...any) bool { + t.Helper() + var value bool + if err := queryRowOn(s.ctx, s.witness, &value, sql, args...); err != nil { + s.fail("witness "+sql, err) + } + return value +} + +func (s *capabilityService) mustWitnessText(t *testing.T, sql string, args ...any) string { + t.Helper() + var value string + if err := queryRowOn(s.ctx, s.witness, &value, sql, args...); err != nil { + s.fail("witness "+sql, err) + } + return value +} + +func (s *capabilityService) execCount(t *testing.T, sql string, args ...any) int { + t.Helper() + var n int + if err := queryRowOn(s.ctx, s.witness, &n, sql, args...); err != nil { + s.fail("count "+sql, err) + } + return n +} + +func (s *capabilityService) forEachWitnessText(t *testing.T, dest *[]string, sql string, args ...any) { + t.Helper() + conn, err := s.witness.Acquire(s.ctx) + if err != nil { + s.fail("acquire text witness", err) + } + defer conn.Release() + rows, err := conn.Query(s.ctx, sql, args...) + if err != nil { + s.fail("query text witness "+sql, err) + } + defer rows.Close() + for rows.Next() { + var value string + if err := rows.Scan(&value); err != nil { + s.fail("scan text witness "+sql, err) + } + *dest = append(*dest, value) + } + if err := rows.Err(); err != nil { + s.fail("iterate text witness "+sql, err) + } +} + +type snapshotRow struct { + token string + identity capability.SnapshotIdentity + status string + complete bool + truncated bool + payloadText string + ctid string + exists bool +} + +func (s *capabilityService) witnessSnapshot(token string) snapshotRow { + var w snapshotRow + conn, err := s.witness.Acquire(s.ctx) + if err != nil { + s.fail("acquire snapshot witness", err) + } + defer conn.Release() + var project, generation, authDigest, cacheVersion, schemaDigest, inventoryDigest, + relationshipDigest, fieldDigest, shapeDigest, protocolVersion, compilerVersion, + traversalVersion, projectionVersion string + err = conn.QueryRow(s.ctx, + `SELECT token, project, generation, authorization_scope_digest, cache_version, schema_digest, + resource_inventory_digest, relationship_digest, field_digest, shape_digest, + protocol_version, compiler_version, traversal_policy_version, projection_policy_version, + status, complete, truncated, payload::text, ctid::text + FROM `+s.qualified(CapabilitySnapshotsTable)+` WHERE token = $1`, token). + Scan(&w.token, &project, &generation, &authDigest, &cacheVersion, &schemaDigest, + &inventoryDigest, &relationshipDigest, &fieldDigest, &shapeDigest, + &protocolVersion, &compilerVersion, &traversalVersion, &projectionVersion, + &w.status, &w.complete, &w.truncated, &w.payloadText, &w.ctid) + if errors.Is(err, pgx.ErrNoRows) { + return w + } + if err != nil { + s.fail("witness snapshot "+token, err) + } + w.exists = true + w.identity = capability.SnapshotIdentity{ + Project: project, Generation: generation, AuthorizationScopeDigest: authDigest, + CacheVersion: cacheVersion, SchemaDigest: schemaDigest, ResourceInventoryDigest: inventoryDigest, + RelationshipDigest: relationshipDigest, FieldDigest: fieldDigest, ShapeDigest: shapeDigest, + ProtocolVersion: protocolVersion, CompilerVersion: compilerVersion, + TraversalPolicyVersion: traversalVersion, ProjectionPolicyVersion: projectionVersion, + } + return w +} + +func (s *capabilityService) assertSnapshotAbsent(t *testing.T, token string) { + t.Helper() + if w := s.witnessSnapshot(token); w.exists { + t.Fatalf("expected no capability_snapshots row for %s, found %+v", token, w) + } +} + +func (s *capabilityService) assertSnapshotLiteral(t *testing.T, want capability.Snapshot) { + t.Helper() + document, err := capabilitySnapshotDocument(want) + if err != nil { + t.Fatal(err) + } + w := s.witnessSnapshot(want.Token) + if !w.exists { + t.Fatalf("capability snapshot %s does not exist", want.Token) + } + if w.token != want.Token || w.identity != want.Identity || + w.status != string(want.Status) || w.complete != want.Complete || w.truncated != want.Truncated { + t.Fatalf("snapshot mirror columns = {token:%s identity:%+v status:%s complete:%v truncated:%v}, want {token:%s identity:%+v status:%s complete:%v truncated:%v}", + w.token, w.identity, w.status, w.complete, w.truncated, + want.Token, want.Identity, want.Status, want.Complete, want.Truncated) + } + if !s.mustWitnessBool(t, + `SELECT payload = $1::jsonb FROM `+s.qualified(CapabilitySnapshotsTable)+` WHERE token = $2`, + string(document), want.Token) { + t.Fatalf("snapshot %s payload does not equal the literal canonical document\nwant: %s\ngot: %s", + want.Token, document, w.payloadText) + } +} + +type witnessedReceipt struct { + id, kind, parentCatalogID string + binding tableshapecap.Binding + contentDigest, createdAt string + payloadText string + ctid string + exists bool +} + +func (s *capabilityService) witnessReceipt(id string) witnessedReceipt { + var w witnessedReceipt + conn, err := s.witness.Acquire(s.ctx) + if err != nil { + s.fail("acquire receipt witness", err) + } + defer conn.Release() + var draftVersion int64 + err = conn.QueryRow(s.ctx, + `SELECT id, kind, parent_catalog_id, project, explorer_id, output_id, snapshot_token, + authorization_scope, source_generation, draft_version, draft_digest, + base_document_digest, base_compilation_receipt_id, output_fingerprint, + compiler_schema_digest, content_digest, created_at, payload::text, ctid::text + FROM `+s.qualified(TableShapeCapabilitiesTable)+` WHERE id = $1`, id). + Scan(&w.id, &w.kind, &w.parentCatalogID, + &w.binding.Project, &w.binding.ExplorerID, &w.binding.OutputID, &w.binding.SnapshotToken, + &w.binding.AuthorizationScope, &w.binding.SourceGeneration, &draftVersion, + &w.binding.DraftDigest, &w.binding.BaseDocumentDigest, &w.binding.BaseCompilationReceiptID, + &w.binding.OutputFingerprint, &w.binding.CompilerSchemaDigest, + &w.contentDigest, &w.createdAt, &w.payloadText, &w.ctid) + if errors.Is(err, pgx.ErrNoRows) { + return w + } + if err != nil { + s.fail("witness receipt "+id, err) + } + w.exists = true + w.binding.DraftVersion = uint64(draftVersion) + return w +} + +func (s *capabilityService) assertReceiptAbsent(t *testing.T, id string) { + t.Helper() + if w := s.witnessReceipt(id); w.exists { + t.Fatalf("expected no table_shape_capabilities row for %s, found %+v", id, w) + } +} + +func (s *capabilityService) assertReceiptLiteral(t *testing.T, id, kind, parentCatalogID string, binding tableshapecap.Binding, contentDigest, createdAt string, want any) { + t.Helper() + payload, err := json.Marshal(want) + if err != nil { + t.Fatal(err) + } + w := s.witnessReceipt(id) + if !w.exists { + t.Fatalf("receipt %s does not exist", id) + } + if w.id != id || w.kind != kind || w.parentCatalogID != parentCatalogID || + w.binding != binding || w.contentDigest != contentDigest || w.createdAt != createdAt { + t.Fatalf("receipt %s mirror columns = %+v, want {id:%s kind:%s parent:%s binding:%+v digest:%s created:%s}", + id, w, id, kind, parentCatalogID, binding, contentDigest, createdAt) + } + if !s.mustWitnessBool(t, + `SELECT payload = $1::jsonb FROM `+s.qualified(TableShapeCapabilitiesTable)+` WHERE id = $2`, + string(payload), id) { + t.Fatalf("receipt %s payload does not equal the literal expected artifact\nwant: %s\ngot: %s", id, payload, w.payloadText) + } +} + +func (s *capabilityService) triggerPresent(t *testing.T, trigger, table string) bool { + t.Helper() + var present bool + err := queryRowOn(s.ctx, s.witness, &present, + `SELECT EXISTS ( + SELECT 1 FROM pg_trigger tg + JOIN pg_class c ON c.oid = tg.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE tg.tgname = $1 AND c.relname = $2 AND n.nspname = $3 + AND NOT tg.tgisinternal)`, + trigger, table, s.schema) + if err != nil { + s.fail("trigger catalog witness "+trigger, err) + } + return present +} + +func (s *capabilityService) guardDropProbe(t *testing.T, trigger, table string, reseed func(), stmt string, args ...any) { + t.Helper() + tableFQN := s.qualified(table) + if !s.triggerPresent(t, trigger, table) { + t.Fatalf("immutability trigger %s missing from %s before probe", trigger, table) + } + err := s.witnessExecErr(stmt, args...) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != pgstate03ImmutableCode { + t.Fatalf("guarded violating write on %s = %v, want SQLSTATE %s", table, s.scrub(err), pgstate03ImmutableCode) + } + s.mustWitnessExec(t, `DROP TRIGGER `+trigger+` ON `+tableFQN) + if s.triggerPresent(t, trigger, table) { + t.Fatalf("trigger %s still present after DROP", trigger) + } + conn, acquireErr := s.witness.Acquire(s.ctx) + if acquireErr != nil { + s.fail("acquire unguarded probe connection", acquireErr) + } + tag, execErr := conn.Exec(s.ctx, stmt, args...) + conn.Release() + if execErr != nil || tag.RowsAffected() != 1 { + t.Fatalf("same violating write ran with the trigger dropped on %s: tag=%v err=%v; the guard was not what blocked it", + table, tag, s.scrub(execErr)) + } + if reseed != nil { + reseed() + } + s.mustWitnessExec(t, `CREATE TRIGGER `+trigger+` BEFORE UPDATE OR DELETE ON `+tableFQN+ + ` FOR EACH ROW EXECUTE FUNCTION `+s.qualified("loom_capability_artifact_immutable")+`()`) + if !s.triggerPresent(t, trigger, table) { + t.Fatalf("immutability trigger %s not restored on %s", trigger, table) + } + err = s.witnessExecErr(stmt, args...) + if !errors.As(err, &pgErr) || pgErr.Code != pgstate03ImmutableCode { + t.Fatalf("violating write after trigger restore = %v, want SQLSTATE %s", s.scrub(err), pgstate03ImmutableCode) + } +} + +func ptr[T any](value T) *T { return &value } + +func richSnapshot(project, generation, token string) capability.Snapshot { + identity := capability.SnapshotIdentity{ + Project: project, Generation: generation, + AuthorizationScopeDigest: "sha256:scope-" + token, + CacheVersion: "v2", + SchemaDigest: "sha256:schema-" + token, + ResourceInventoryDigest: "sha256:inventory-" + token, + RelationshipDigest: "sha256:relationship-" + token, + FieldDigest: "sha256:field-" + token, + ShapeDigest: "sha256:shape-" + token, + ProtocolVersion: "proto-3", CompilerVersion: "compiler-9", + TraversalPolicyVersion: "traversal-2", ProjectionPolicyVersion: "projection-4", + } + policy := capability.Policy{ + Route: capability.RoutePolicy{Version: "route-1", MaxHops: 4, AllowsRepeatedEdges: true, AllowsSelfLoops: false}, + Projection: capability.ProjectionPolicy{Version: "projection-1", Modes: []capability.ProjectionMode{capability.ProjectionScalar, capability.ProjectionArray, capability.ProjectionDistinctArray}, SuggestionLimit: 5}, + } + nodes := []capability.Node{ + {ID: "node-patient", ResourceType: "Patient", RowRootEligible: true, RowGrain: "document", Populated: true, DocumentCount: 7, SupportedOperations: []capability.Operation{capability.OperationSelect, capability.OperationFilter}}, + {ID: "node-obs", ResourceType: "Observation", Populated: true, DocumentCount: 3, SupportedOperations: []capability.Operation{capability.OperationChart}, CapabilityVersion: "cap-1"}, + } + edges := []capability.Edge{ + {ID: "edge-1", FromNodeID: "node-patient", ToNodeID: "node-obs", Label: "has", SourceResourceType: "Patient", TargetResourceType: "Observation", ObservedEdgeCount: 5, AllowsRepeatedTarget: true}, + } + candidates := []capability.Candidate{ + { + ID: "cand-1", NodeID: "node-obs", ResourceType: "Observation", FieldPath: "value.codeableConcept.text", + Label: "Code text", LogicalType: "string", Cardinality: "many", + RepeatedBoundaries: []capability.RepeatedBoundary{{Path: "value.codeableConcept", MaxItems: 3}}, + ProjectionModes: []capability.ProjectionMode{capability.ProjectionScalar}, + FilterOperators: []capability.FilterOperator{capability.FilterEquals, capability.FilterIn}, + ChartAggregations: []capability.ChartAggregation{capability.ChartCount}, + SupportedOperations: []capability.Operation{capability.OperationSort}, + AggregateOperations: []capability.AggregateOperationCapability{{Operation: capability.AggregateCount, RowContext: capability.AggregateRowsRecords, Supported: true}}, + Observed: true, + ObservedDocumentCount: 2, Populated: true, + SuggestedValues: []string{"alpha", "", "zulu"}, + SuggestionsComplete: false, SuggestionsTruncated: true, + ConceptCandidates: []capability.ConceptCandidate{ + {SourceResourceType: "Observation", SourcePath: "category", Code: "vitals", Display: "Vital", Completeness: "PARTIAL", Status: "UNRESOLVED", Population: 1, Examples: []string{"x"}, ObservedUnits: []string{"kg"}, ObservedUnitsTruncated: true}, + }, + }, + } + diagnostics := []capability.Diagnostic{ + {Severity: "WARNING", Stage: "scan", Code: "TRUNCATED_EXAMPLES", Message: "examples truncated", Details: map[string]any{"limit": float64(2), "note": "probe"}, Retryable: true}, + } + snapshot := capability.NewSnapshot(identity, policy, capability.StatusReady, true, false, nodes, edges, candidates, diagnostics) + snapshot.Availability = &capability.Availability{ + Readiness: capability.AvailabilityReadiness{ + Status: "READY", ResourceInventory: capability.AvailabilitySourceReadiness{Status: "READY"}, + Relationships: capability.AvailabilitySourceReadiness{Status: "READY"}, + FieldProfiles: capability.AvailabilitySourceReadiness{Status: "DEGRADED", Reason: "profiles stale"}, + }, + Resources: []capability.ObservedResource{{ResourceType: "Observation", DocumentCount: 3, Populated: true}}, + Fields: []capability.ObservedField{{ResourceType: "Observation", Path: "value", ShapeKind: "object", PathObservedCount: 3, Examples: []string{"a"}, ExamplesTruncated: true, SemanticObservations: []capability.ConceptCandidate{{SourceResourceType: "Observation", Completeness: "FULL", Status: "MATCHED", Population: 2}}}}, + Relationships: []capability.ObservedRelationship{{SourceResourceType: "Patient", TargetResourceType: "Observation", Label: "has", StorageDirection: "out", BuilderDirection: "forward", ObservedEdgeCount: 5}}, + } + snapshot.AuditNodes = []capability.Node{{ID: "audit-node", ResourceType: "Audit", Populated: false}} + snapshot.AuditEdges = []capability.Edge{{ID: "audit-edge", FromNodeID: "audit-node", ToNodeID: "node-patient", Label: "audits", SourceResourceType: "Audit", TargetResourceType: "Patient"}} + snapshot.AuditCandidates = []capability.Candidate{{ID: "audit-cand", NodeID: "audit-node", ResourceType: "Audit", FieldPath: "agent", Label: "Agent", LogicalType: "string"}} + sum := sha256.Sum256(snapshot.CanonicalPayload()) + snapshot.Token = "sha256:" + hex.EncodeToString(sum[:]) + return snapshot +} + +func testBinding(project string) tableshapecap.Binding { + return tableshapecap.Binding{ + Project: project, ExplorerID: "explorer-a", OutputID: "output-a", SnapshotToken: "snapshot-a", + AuthorizationScope: "scope-a", SourceGeneration: "generation-a", DraftVersion: 7, DraftDigest: "draft-a", + BaseDocumentDigest: "document-a", BaseCompilationReceiptID: "compilation-a", OutputFingerprint: "output-fingerprint-a", + CompilerSchemaDigest: "schema-a", + } +} + +func testCatalog(t *testing.T, binding tableshapecap.Binding, createdAt string) tableshapecap.CatalogReceipt { + t.Helper() + columns := []tableshapecap.PublicColumn{ + {Key: "patient", Label: "Patient", LogicalType: tableshapecap.LogicalString}, + {Key: "category", Label: "Category", LogicalType: tableshapecap.LogicalString, Nullable: true}, + {Key: "value", Label: "Value", LogicalType: tableshapecap.LogicalDecimal, Nullable: true, UnitIdentity: "mg"}, + } + roles := []tableshapecap.ChoiceRole{ + tableshapecap.RolePivotGroup, tableshapecap.RolePivotCategory, tableshapecap.RolePivotValue, + tableshapecap.RoleUnpivotInput, tableshapecap.RoleDerivedOperand, tableshapecap.RoleDerivedOperator, + tableshapecap.RolePolicyDuplicate, tableshapecap.RolePolicyMissing, tableshapecap.RolePolicyUnlisted, + tableshapecap.RolePolicyUnpivotNull, tableshapecap.RolePolicyDerivedMissing, tableshapecap.RolePolicyDivisionByZero, + } + availability := make([]tableshapecap.RoleAvailability, 0, len(roles)) + for _, role := range roles { + availability = append(availability, tableshapecap.RoleAvailability{Role: role, State: tableshapecap.AvailabilitySupported}) + } + choices := tableshapecap.CatalogChoices{ + Columns: []tableshapecap.ColumnChoice{ + {Role: tableshapecap.RolePivotGroup, ColumnKey: "patient"}, + {Role: tableshapecap.RolePivotCategory, ColumnKey: "category"}, + {Role: tableshapecap.RolePivotValue, ColumnKey: "value"}, + {Role: tableshapecap.RoleUnpivotInput, ColumnKey: "category"}, + {Role: tableshapecap.RoleUnpivotInput, ColumnKey: "value"}, + }, + Operators: []tableshapecap.OperatorChoice{{Role: tableshapecap.RoleDerivedOperator, Operator: "ADD"}}, + Policies: []tableshapecap.PolicyChoice{ + {Role: tableshapecap.RolePolicyDuplicate, PolicyID: "ERROR"}, + {Role: tableshapecap.RolePolicyMissing, PolicyID: "NULL"}, + {Role: tableshapecap.RolePolicyUnlisted, PolicyID: "ERROR"}, + {Role: tableshapecap.RolePolicyUnpivotNull, PolicyID: "PRESERVE"}, + {Role: tableshapecap.RolePolicyDerivedMissing, PolicyID: "ERROR"}, + {Role: tableshapecap.RolePolicyDivisionByZero, PolicyID: "ERROR"}, + }, + Operands: []tableshapecap.OperandChoice{{Role: tableshapecap.RoleDerivedOperand, Operand: tableshapecap.OperandRef{Kind: tableshapecap.OperandColumn, ColumnKey: "value"}}}, + } + receipt, err := tableshapecap.NewCatalogReceipt(binding, columns, availability, choices, + tableshapecap.SavedShapeSummary{ReshapeKind: "PIVOT", DerivedColumns: 2, ShapeDigest: "shape-digest-1"}, createdAt) + if err != nil { + t.Fatalf("NewCatalogReceipt: %v", err) + } + return receipt +} + +func scanValues(t *testing.T) []tableshapecap.Scalar { + t.Helper() + return []tableshapecap.Scalar{ + tableshapecap.StringScalar(""), tableshapecap.IntegerScalar(0), + tableshapecap.DecimalScalar(2.5), tableshapecap.BooleanScalar(false), + tableshapecap.NullScalar(), tableshapecap.MissingScalar(), + } +} + +func testScan(t *testing.T, catalog tableshapecap.CatalogReceipt, createdAt string) tableshapecap.CategoryScanReceipt { + t.Helper() + byRole := map[tableshapecap.ChoiceRole]string{} + for _, choice := range catalog.Choices.Columns { + if _, ok := byRole[choice.Role]; !ok { + byRole[choice.Role] = choice.ID + } + } + values := scanValues(t) + valuesDigest, err := tableshapecap.CategoryValuesDigest(values) + if err != nil { + t.Fatal(err) + } + proof := tableshapecap.CategoryProof{ + Complete: true, DistinctCount: len(values), MaxCategories: 256, ValuesDigest: valuesDigest, + SourceGeneration: catalog.Binding.SourceGeneration, OutputFingerprint: catalog.Binding.OutputFingerprint, + ScanFingerprint: "scan-fingerprint-v1", QueryProof: "query-proof-v1", + } + scan, err := tableshapecap.NewCategoryScanReceipt(catalog.Binding, catalog.ID, + byRole[tableshapecap.RolePivotCategory], byRole[tableshapecap.RolePivotValue], values, proof, createdAt) + if err != nil { + t.Fatalf("NewCategoryScanReceipt: %v", err) + } + return scan +} + +func testPivot(t *testing.T, catalog tableshapecap.CatalogReceipt, scan tableshapecap.CategoryScanReceipt, createdAt string) tableshapecap.ResolutionReceipt { + t.Helper() + byRole := map[tableshapecap.ChoiceRole]string{} + for _, choice := range catalog.Choices.Columns { + if _, ok := byRole[choice.Role]; !ok { + byRole[choice.Role] = choice.ID + } + } + policies := map[tableshapecap.ChoiceRole]string{} + for _, choice := range catalog.Choices.Policies { + policies[choice.Role] = choice.ID + } + pivot := &tableshapecap.PivotResolution{ + GroupColumnChoiceIDs: []string{byRole[tableshapecap.RolePivotGroup]}, + CategoryColumnChoiceID: byRole[tableshapecap.RolePivotCategory], + ValueColumnChoiceID: byRole[tableshapecap.RolePivotValue], + CategoryDiscoveryID: scan.ID, + DuplicatePolicyChoiceID: policies[tableshapecap.RolePolicyDuplicate], + MissingPolicyChoiceID: policies[tableshapecap.RolePolicyMissing], + UnlistedPolicyChoiceID: policies[tableshapecap.RolePolicyUnlisted], + CategoryProof: scan.Proof, + } + for i, category := range scan.Categories { + pivot.Categories = append(pivot.Categories, tableshapecap.FrozenCategory{ + ChoiceID: category.ChoiceID, Value: category.Value, + OutputColumn: fmt.Sprintf("category_%d", i+1), OutputLabel: fmt.Sprintf("Category %d", i+1), + }) + } + resolution, err := tableshapecap.NewResolutionReceipt(catalog.Binding, catalog.ID, + tableshapecap.ResolutionPivot, pivot, nil, nil, createdAt) + if err != nil { + t.Fatalf("NewResolutionReceipt pivot: %v", err) + } + return resolution +} + +func testDerived(t *testing.T, catalog tableshapecap.CatalogReceipt, prior string, name string, createdAt string) tableshapecap.ResolutionReceipt { + t.Helper() + operatorID := catalog.Choices.Operators[0].ID + baseChoiceID := catalog.Choices.Operands[0].ID + missingPolicyID := "" + for _, choice := range catalog.Choices.Policies { + if choice.Role == tableshapecap.RolePolicyDerivedMissing { + missingPolicyID = choice.ID + } + } + left := tableshapecap.ResolvedOperand{Kind: tableshapecap.ResolvedOperandCatalogChoice, ChoiceID: baseChoiceID} + if prior != "" { + zero := 0 + left = tableshapecap.ResolvedOperand{Kind: tableshapecap.ResolvedOperandResolution, ResolutionID: prior, OutputIndex: &zero} + } + resolution, err := tableshapecap.NewResolutionReceipt(catalog.Binding, catalog.ID, + tableshapecap.ResolutionDerived, nil, nil, &tableshapecap.DerivedResolution{ + Output: tableshapecap.NamedOutput{Name: name, Label: strings.ToUpper(name[:1]) + name[1:]}, + OperatorChoiceID: operatorID, + Left: left, + Right: tableshapecap.ResolvedOperand{Kind: tableshapecap.ResolvedOperandLiteral, Literal: ptr(tableshapecap.DecimalScalar(2.5))}, + Result: tableshapecap.TypeFact{LogicalType: tableshapecap.LogicalDecimal, UnitIdentity: "mg"}, + MissingPolicyChoiceID: missingPolicyID, + DivisionByZeroPolicyChoiceID: "", + }, createdAt) + if err != nil { + t.Fatalf("NewResolutionReceipt derived %s: %v", name, err) + } + return resolution +} + +func TestNewCapabilityStoresRejectNilClientAndUnsafeSchema(t *testing.T) { + for _, schema := range []string{"", "Public", "a;drop", "0starts_with_digit", "has space", "tab\tname", strings.Repeat("a", 64)} { + if _, err := NewCapabilitySnapshotStore(nil, schema); err == nil { + t.Fatalf("NewCapabilitySnapshotStore(nil, %q) = nil error, want failure", schema) + } + if _, err := NewTableShapeCapabilityStore(nil, schema); err == nil { + t.Fatalf("NewTableShapeCapabilityStore(nil, %q) = nil error, want failure", schema) + } + } + if _, err := NewCapabilitySnapshotStore(&pgstore.Client{}, "ok_schema"); err != nil { + t.Fatalf("NewCapabilitySnapshotStore(valid) = %v, want success", err) + } + if _, err := NewTableShapeCapabilityStore(&pgstore.Client{}, "ok_schema"); err != nil { + t.Fatalf("NewTableShapeCapabilityStore(valid) = %v, want success", err) + } +} + +func TestCapabilitySchemaCompositionColumnContractAndGuards(t *testing.T) { + s := newCapabilityService(t) + + var tables []string + s.forEachWitnessText(t, &tables, + "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = $1 ORDER BY tablename", s.schema) + declared := map[string]bool{"schema_migrations": true} + for _, m := range s.owner { + for _, table := range tablesCreatedIn(m.SQL) { + declared[table] = true + } + } + for _, table := range tablesCreatedIn(s.fixture.SQL) { + declared[table] = true + } + var want []string + for table := range declared { + want = append(want, table) + } + sort.Strings(want) + if strings.Join(tables, ",") != strings.Join(want, ",") { + t.Fatalf("schema tables = %#v, want exactly %#v", tables, want) + } + + type column struct { + name, typ string + nullable bool + } + for _, wantTable := range []struct { + table string + columns []column + }{ + {CapabilitySnapshotsTable, []column{ + {"token", "text", false}, + {"project", "text", false}, + {"generation", "text", false}, + {"authorization_scope_digest", "text", false}, + {"cache_version", "text", false}, + {"schema_digest", "text", false}, + {"resource_inventory_digest", "text", false}, + {"relationship_digest", "text", false}, + {"field_digest", "text", false}, + {"shape_digest", "text", false}, + {"protocol_version", "text", false}, + {"compiler_version", "text", false}, + {"traversal_policy_version", "text", false}, + {"projection_policy_version", "text", false}, + {"status", "text", false}, + {"complete", "boolean", false}, + {"truncated", "boolean", false}, + {"payload", "jsonb", false}, + }}, + {TableShapeCapabilitiesTable, []column{ + {"id", "text", false}, + {"kind", "text", false}, + {"parent_catalog_id", "text", false}, + {"project", "text", false}, + {"explorer_id", "text", false}, + {"output_id", "text", false}, + {"snapshot_token", "text", false}, + {"authorization_scope", "text", false}, + {"source_generation", "text", false}, + {"draft_version", "bigint", false}, + {"draft_digest", "text", false}, + {"base_document_digest", "text", false}, + {"base_compilation_receipt_id", "text", false}, + {"output_fingerprint", "text", false}, + {"compiler_schema_digest", "text", false}, + {"content_digest", "text", false}, + {"created_at", "text", false}, + {"payload", "jsonb", false}, + }}, + } { + conn, acquireErr := s.witness.Acquire(s.ctx) + if acquireErr != nil { + s.fail("acquire column witness", acquireErr) + } + rows, err := conn.Query(s.ctx, + "SELECT column_name, data_type, is_nullable = 'YES' FROM information_schema.columns"+ + " WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position", + s.schema, wantTable.table) + if err != nil { + rows.Close() + conn.Release() + s.fail("list columns "+wantTable.table, err) + } + var columns []column + for rows.Next() { + var c column + if err := rows.Scan(&c.name, &c.typ, &c.nullable); err != nil { + rows.Close() + conn.Release() + s.fail("scan column", err) + } + columns = append(columns, c) + } + rows.Close() + conn.Release() + if len(columns) != len(wantTable.columns) { + t.Fatalf("%s columns = %+v, want %+v", wantTable.table, columns, wantTable.columns) + } + for i, wantCol := range wantTable.columns { + if columns[i] != wantCol { + t.Fatalf("%s column %d = %+v, want %+v", wantTable.table, i, columns[i], wantCol) + } + } + } + + if !s.triggerPresent(t, "capability_snapshots_immutable", CapabilitySnapshotsTable) { + t.Fatal("capability_snapshots_immutable trigger missing") + } + if !s.triggerPresent(t, "table_shape_capabilities_immutable", TableShapeCapabilitiesTable) { + t.Fatal("table_shape_capabilities_immutable trigger missing") + } + var functionPresent bool + if err := queryRowOn(s.ctx, s.witness, &functionPresent, + `SELECT to_regprocedure($1) IS NOT NULL`, s.schema+".loom_capability_artifact_immutable()"); err != nil { + s.fail("guard function witness", err) + } + if !functionPresent { + t.Fatal("loom_capability_artifact_immutable() guard function missing") + } + for _, index := range []string{ + "capability_snapshots_identity_ix", "capability_snapshots_status_ix", + "table_shape_capabilities_binding_ix", "table_shape_capabilities_parent_ix", + } { + var present bool + if err := queryRowOn(s.ctx, s.witness, &present, + `SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = $1 AND indexname = $2)`, s.schema, index); err != nil { + s.fail("index witness "+index, err) + } + if !present { + t.Fatalf("index %s missing", index) + } + } +} + +func TestCapabilitySnapshotPutIdempotentAndCollisionRefused(t *testing.T) { + s := newCapabilityService(t) + snapshot := richSnapshot("pgstate03-snapshots", "generation-1", "alpha") + + created, err := s.snapshots.Put(s.ctx, snapshot) + if err != nil { + s.fail("put snapshot", err) + } + assertSnapshotEquivalent(t, "Put", created, snapshot) + s.assertSnapshotLiteral(t, snapshot) + before := s.witnessSnapshot(snapshot.Token) + + again, err := s.snapshots.Put(s.ctx, snapshot) + if err != nil { + s.fail("idempotent re-put", err) + } + assertSnapshotEquivalent(t, "idempotent re-put", again, snapshot) + after := s.witnessSnapshot(snapshot.Token) + if before.payloadText != after.payloadText || before.ctid != after.ctid { + t.Fatalf("idempotent re-put rewrote the snapshot row: before=%+v after=%+v", before, after) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(CapabilitySnapshotsTable)); n != 1 { + t.Fatalf("snapshot rows = %d, want exactly 1", n) + } + + s.mustWitnessExec(t, `DROP TRIGGER capability_snapshots_immutable ON `+s.qualified(CapabilitySnapshotsTable)) + s.mustWitnessExec(t, `UPDATE `+s.qualified(CapabilitySnapshotsTable)+ + ` SET payload = jsonb_set(payload, '{status}', '"FAILED"') WHERE token = $1`, snapshot.Token) + s.mustWitnessExec(t, `CREATE TRIGGER capability_snapshots_immutable BEFORE UPDATE OR DELETE ON `+ + s.qualified(CapabilitySnapshotsTable)+` FOR EACH ROW EXECUTE FUNCTION `+s.qualified("loom_capability_artifact_immutable")+`()`) + tampered := s.witnessSnapshot(snapshot.Token) + if tampered.ctid == before.ctid || tampered.payloadText == before.payloadText { + t.Fatalf("tampering did not change the stored row: %+v", tampered) + } + if _, err := s.snapshots.Put(s.ctx, snapshot); !errors.Is(err, capability.ErrTokenCollision) { + t.Fatalf("Put over tampered payload = %v, want capability.ErrTokenCollision", err) + } + untouched := s.witnessSnapshot(snapshot.Token) + if untouched.payloadText != tampered.payloadText || untouched.ctid != tampered.ctid { + t.Fatalf("refused Put mutated the stored row: tampered=%+v after=%+v", tampered, untouched) + } + + lying := snapshot + lying.Token = "sha256:" + strings.Repeat("0", 64) + if _, err := s.snapshots.Put(s.ctx, lying); !errors.Is(err, capability.ErrInvalidSnapshot) { + t.Fatalf("Put with foreign token = %v, want capability.ErrInvalidSnapshot", err) + } + s.assertSnapshotAbsent(t, lying.Token) + notActuallyComplete := capability.NewSnapshot(snapshot.Identity, snapshot.Policy, + capability.StatusReady, false, false, nil, nil, nil, nil) + if _, err := s.snapshots.Put(s.ctx, notActuallyComplete); !errors.Is(err, capability.ErrInvalidSnapshot) { + t.Fatalf("Put of incomplete READY = %v, want capability.ErrInvalidSnapshot", err) + } + + if _, err := s.snapshots.GetByToken(s.ctx, snapshot.Token); !errors.Is(err, capability.ErrInvalidSnapshot) { + t.Fatalf("GetByToken over tampered payload = %v, want capability.ErrInvalidSnapshot", err) + } +} + +func TestCapabilitySnapshotIdentityRetrievalIsFullyScoped(t *testing.T) { + s := newCapabilityService(t) + ready := richSnapshot("pgstate03-identity", "generation-1", "beta") + if _, err := s.snapshots.Put(s.ctx, ready); err != nil { + s.fail("put ready snapshot", err) + } + + got, err := s.snapshots.GetByIdentity(s.ctx, ready.Identity) + if err != nil { + s.fail("get by identity", err) + } + assertSnapshotEquivalent(t, "GetByIdentity", got, ready) + + mutations := map[string]func(id *capability.SnapshotIdentity){ + "project": func(id *capability.SnapshotIdentity) { id.Project = "pgstate03-identity-other" }, + "generation": func(id *capability.SnapshotIdentity) { id.Generation += "-other" }, + "authorizationScopeDigest": func(id *capability.SnapshotIdentity) { id.AuthorizationScopeDigest += "-other" }, + "cacheVersion": func(id *capability.SnapshotIdentity) { id.CacheVersion = "v3" }, + "schemaDigest": func(id *capability.SnapshotIdentity) { id.SchemaDigest += "-other" }, + "resourceInventoryDigest": func(id *capability.SnapshotIdentity) { id.ResourceInventoryDigest += "-other" }, + "relationshipDigest": func(id *capability.SnapshotIdentity) { id.RelationshipDigest += "-other" }, + "fieldDigest": func(id *capability.SnapshotIdentity) { id.FieldDigest += "-other" }, + "shapeDigest": func(id *capability.SnapshotIdentity) { id.ShapeDigest += "-other" }, + "protocolVersion": func(id *capability.SnapshotIdentity) { id.ProtocolVersion += "-other" }, + "compilerVersion": func(id *capability.SnapshotIdentity) { id.CompilerVersion += "-other" }, + "traversalPolicyVersion": func(id *capability.SnapshotIdentity) { id.TraversalPolicyVersion += "-other" }, + "projectionPolicyVersion": func(id *capability.SnapshotIdentity) { id.ProjectionPolicyVersion += "-other" }, + } + for name, mutate := range mutations { + foreign := ready.Identity + mutate(&foreign) + if _, err := s.snapshots.GetByIdentity(s.ctx, foreign); !errors.Is(err, capability.ErrNotFound) { + t.Fatalf("GetByIdentity with mutated %s = %v, want capability.ErrNotFound", name, err) + } + } + + if _, err := s.snapshots.GetByToken(s.ctx, "sha256:"+strings.Repeat("f", 64)); !errors.Is(err, capability.ErrNotFound) { + t.Fatalf("GetByToken(unknown) = %v, want capability.ErrNotFound", err) + } + + building := capability.NewSnapshot(ready.Identity, ready.Policy, capability.StatusBuilding, false, false, nil, nil, nil, nil) + if _, err := s.snapshots.Put(s.ctx, building); err != nil { + s.fail("put building snapshot", err) + } + got, err = s.snapshots.GetByIdentity(s.ctx, ready.Identity) + if err != nil { + s.fail("get by identity with building sibling", err) + } + assertSnapshotEquivalent(t, "GetByIdentity with building sibling", got, ready) + diagnostic := richSnapshot("pgstate03-identity-diagnostic", "generation-1", "gamma") + diagnostic.Status = capability.StatusFailed + diagnostic.Complete = false + sum := sha256.Sum256(diagnostic.CanonicalPayload()) + diagnostic.Token = "sha256:" + hex.EncodeToString(sum[:]) + if _, err := s.snapshots.Put(s.ctx, diagnostic); err != nil { + s.fail("put failed snapshot", err) + } + got, err = s.snapshots.GetByIdentity(s.ctx, diagnostic.Identity) + if err != nil { + s.fail("get diagnostic identity", err) + } + assertSnapshotEquivalent(t, "GetByIdentity FAILED", got, diagnostic) + + sibling := richSnapshot("pgstate03-identity", "generation-1", "beta") + sibling.Identity.ShapeDigest = "sha256:shape-beta-sibling" + siblingSum := sha256.Sum256(sibling.CanonicalPayload()) + sibling.Token = "sha256:" + hex.EncodeToString(siblingSum[:]) + if sibling.Token == ready.Token { + t.Fatal("shape-digest sibling did not produce a distinct token") + } + if _, err := s.snapshots.Put(s.ctx, sibling); err != nil { + s.fail("put shape-digest sibling", err) + } + got, err = s.snapshots.GetByIdentity(s.ctx, ready.Identity) + if err != nil { + s.fail("get by identity with shape-digest sibling", err) + } + if got.Token != ready.Token { + t.Fatalf("GetByIdentity(ready) returned sibling token %q, want %q", got.Token, ready.Token) + } + assertSnapshotEquivalent(t, "GetByIdentity ready beside shape sibling", got, ready) + got, err = s.snapshots.GetByIdentity(s.ctx, sibling.Identity) + if err != nil { + s.fail("get by identity for shape-digest sibling", err) + } + if got.Token != sibling.Token { + t.Fatalf("GetByIdentity(sibling) returned token %q, want %q", got.Token, sibling.Token) + } + assertSnapshotEquivalent(t, "GetByIdentity shape-digest sibling", got, sibling) + + ghost := sibling.Identity + ghost.ShapeDigest = "sha256:shape-beta-ghost" + if _, err := s.snapshots.GetByIdentity(s.ctx, ghost); !errors.Is(err, capability.ErrNotFound) { + t.Fatalf("GetByIdentity(ghost shape digest) = %v, want capability.ErrNotFound", err) + } +} + +func TestTableShapeCatalogPutIdempotentAndChangedContentCollides(t *testing.T) { + s := newCapabilityService(t) + binding := testBinding("pgstate03-catalog") + catalog := testCatalog(t, binding, "2026-09-19T00:00:00Z") + + created, err := s.receipts.PutCatalog(s.ctx, catalog) + if err != nil { + s.fail("put catalog", err) + } + if !reflect.DeepEqual(created, catalog) { + t.Fatalf("PutCatalog = %+v, want %+v", created, catalog) + } + s.assertReceiptLiteral(t, catalog.ID, kindCatalog, "", binding, catalog.ContentDigest, catalog.CreatedAt, catalog) + before := s.witnessReceipt(catalog.ID) + + retry := catalog + retry.CreatedAt = "2027-01-02T03:04:05Z" + stored, err := s.receipts.PutCatalog(s.ctx, retry) + if err != nil { + s.fail("idempotent re-put", err) + } + if !reflect.DeepEqual(stored, catalog) { + t.Fatalf("idempotent re-put returned %+v, want the stored %+v", stored, catalog) + } + after := s.witnessReceipt(catalog.ID) + if before.payloadText != after.payloadText || before.ctid != after.ctid || before.createdAt != "2026-09-19T00:00:00Z" { + t.Fatalf("idempotent re-put rewrote the catalog row: before=%+v after=%+v", before, after) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(TableShapeCapabilitiesTable)); n != 1 { + t.Fatalf("receipt rows = %d, want exactly 1", n) + } + + s.mustWitnessExec(t, `DROP TRIGGER table_shape_capabilities_immutable ON `+s.qualified(TableShapeCapabilitiesTable)) + s.mustWitnessExec(t, `UPDATE `+s.qualified(TableShapeCapabilitiesTable)+ + ` SET payload = jsonb_set(payload, '{columns,0,label}', '"tampered"') WHERE id = $1`, catalog.ID) + s.mustWitnessExec(t, `CREATE TRIGGER table_shape_capabilities_immutable BEFORE UPDATE OR DELETE ON `+ + s.qualified(TableShapeCapabilitiesTable)+` FOR EACH ROW EXECUTE FUNCTION `+s.qualified("loom_capability_artifact_immutable")+`()`) + tampered := s.witnessReceipt(catalog.ID) + if tampered.ctid == before.ctid || tampered.payloadText == before.payloadText { + t.Fatalf("tampering did not change the stored row: %+v", tampered) + } + if _, err := s.receipts.PutCatalog(s.ctx, catalog); !errors.Is(err, tableshapecap.ErrIdentityClash) { + t.Fatalf("PutCatalog over tampered payload = %v, want tableshapecap.ErrIdentityClash", err) + } + untouched := s.witnessReceipt(catalog.ID) + if untouched.payloadText != tampered.payloadText || untouched.ctid != tampered.ctid { + t.Fatalf("refused PutCatalog mutated the stored row: tampered=%+v after=%+v", tampered, untouched) + } + + badID := catalog + badID.ID = "tsc_" + strings.Repeat("a", 32) + if _, err := s.receipts.PutCatalog(s.ctx, badID); !errors.Is(err, tableshapecap.ErrInvalid) { + t.Fatalf("PutCatalog with forged ID = %v, want tableshapecap.ErrInvalid", err) + } + s.assertReceiptAbsent(t, badID.ID) +} + +func TestTableShapeReadsRefuseCrossProjectOutputSnapshotAndParent(t *testing.T) { + s := newCapabilityService(t) + binding := testBinding("pgstate03-binding") + catalog := testCatalog(t, binding, "2026-09-19T00:00:00Z") + if _, err := s.receipts.PutCatalog(s.ctx, catalog); err != nil { + s.fail("put catalog", err) + } + scan := testScan(t, catalog, "2026-09-19T00:00:01Z") + if _, err := s.receipts.PutCategoryScan(s.ctx, scan); err != nil { + s.fail("put scan", err) + } + pivot := testPivot(t, catalog, scan, "2026-09-19T00:00:02Z") + if _, err := s.receipts.PutResolution(s.ctx, pivot); err != nil { + s.fail("put pivot", err) + } + + otherSavedShape := catalog.SavedShape + otherSavedShape.DerivedColumns++ + otherReceipt, err := tableshapecap.NewCatalogReceipt(binding, clonePublicColumns(catalog.Columns), + cloneRoleAvailability(catalog.Availability), cloneCatalogChoicesForTest(catalog.Choices), otherSavedShape, "") + if err != nil { + t.Fatal(err) + } + if _, err := s.receipts.PutCatalog(s.ctx, otherReceipt); err != nil { + s.fail("put second catalog", err) + } + + mutations := map[string]func(b *tableshapecap.Binding){ + "project": func(b *tableshapecap.Binding) { b.Project = "pgstate03-binding-other" }, + "explorer": func(b *tableshapecap.Binding) { b.ExplorerID += "-other" }, + "output": func(b *tableshapecap.Binding) { b.OutputID += "-other" }, + "snapshot": func(b *tableshapecap.Binding) { b.SnapshotToken += "-other" }, + "authorizationScope": func(b *tableshapecap.Binding) { b.AuthorizationScope += "-other" }, + "sourceGeneration": func(b *tableshapecap.Binding) { b.SourceGeneration += "-other" }, + "draftVersion": func(b *tableshapecap.Binding) { b.DraftVersion++ }, + "draftDigest": func(b *tableshapecap.Binding) { b.DraftDigest += "-other" }, + "baseDocumentDigest": func(b *tableshapecap.Binding) { b.BaseDocumentDigest += "-other" }, + "baseCompilationReceiptId": func(b *tableshapecap.Binding) { b.BaseCompilationReceiptID += "-other" }, + "outputFingerprint": func(b *tableshapecap.Binding) { b.OutputFingerprint += "-other" }, + "compilerSchemaDigest": func(b *tableshapecap.Binding) { b.CompilerSchemaDigest += "-other" }, + } + for name, mutate := range mutations { + foreign := binding + mutate(&foreign) + if _, err := s.receipts.GetCatalog(s.ctx, foreign, catalog.ID); !errors.Is(err, tableshapecap.ErrNotFound) { + t.Fatalf("GetCatalog with mutated %s = %v, want tableshapecap.ErrNotFound", name, err) + } + if _, err := s.receipts.GetCategoryScan(s.ctx, foreign, catalog.ID, scan.ID); !errors.Is(err, tableshapecap.ErrNotFound) { + t.Fatalf("GetCategoryScan with mutated %s = %v, want tableshapecap.ErrNotFound", name, err) + } + if _, err := s.receipts.GetResolution(s.ctx, foreign, catalog.ID, pivot.ID); !errors.Is(err, tableshapecap.ErrNotFound) { + t.Fatalf("GetResolution with mutated %s = %v, want tableshapecap.ErrNotFound", name, err) + } + } + + lookup := tableshapecap.CatalogLookup{ + Project: binding.Project, ExplorerID: binding.ExplorerID, OutputID: binding.OutputID, + SnapshotToken: binding.SnapshotToken, AuthorizationScope: binding.AuthorizationScope, + SourceGeneration: binding.SourceGeneration, DraftVersion: binding.DraftVersion, + DraftDigest: binding.DraftDigest, BaseDocumentDigest: binding.BaseDocumentDigest, + } + found, err := s.receipts.GetCatalogForLookup(s.ctx, lookup, catalog.ID) + if err != nil { + s.fail("lookup under correct binding", err) + } + if !reflect.DeepEqual(found, catalog) { + t.Fatalf("GetCatalogForLookup = %+v, want %+v", found, catalog) + } + lookupMutations := map[string]func(l *tableshapecap.CatalogLookup){ + "project": func(l *tableshapecap.CatalogLookup) { l.Project = "pgstate03-binding-other" }, + "explorerId": func(l *tableshapecap.CatalogLookup) { l.ExplorerID += "-other" }, + "outputId": func(l *tableshapecap.CatalogLookup) { l.OutputID += "-other" }, + "snapshotToken": func(l *tableshapecap.CatalogLookup) { l.SnapshotToken += "-other" }, + "authorizationScope": func(l *tableshapecap.CatalogLookup) { l.AuthorizationScope += "-other" }, + "sourceGeneration": func(l *tableshapecap.CatalogLookup) { l.SourceGeneration += "-other" }, + "draftVersion": func(l *tableshapecap.CatalogLookup) { l.DraftVersion++ }, + "draftDigest": func(l *tableshapecap.CatalogLookup) { l.DraftDigest += "-other" }, + "baseDocumentDigest": func(l *tableshapecap.CatalogLookup) { l.BaseDocumentDigest += "-other" }, + } + for name, mutate := range lookupMutations { + foreignLookup := lookup + mutate(&foreignLookup) + if _, err := s.receipts.GetCatalogForLookup(s.ctx, foreignLookup, catalog.ID); !errors.Is(err, tableshapecap.ErrNotFound) { + t.Fatalf("GetCatalogForLookup with mutated %s = %v, want tableshapecap.ErrNotFound", name, err) + } + } + + if _, err := s.receipts.GetCategoryScan(s.ctx, binding, otherReceipt.ID, scan.ID); !errors.Is(err, tableshapecap.ErrNotFound) { + t.Fatalf("GetCategoryScan under foreign parent = %v, want tableshapecap.ErrNotFound", err) + } + if _, err := s.receipts.GetResolution(s.ctx, binding, otherReceipt.ID, pivot.ID); !errors.Is(err, tableshapecap.ErrNotFound) { + t.Fatalf("GetResolution under foreign parent = %v, want tableshapecap.ErrNotFound", err) + } + + orphanScan, err := tableshapecap.NewCategoryScanReceipt(binding, otherReceipt.ID, + scan.CategoryColumnChoiceID, scan.ValueColumnChoiceID, scanValues(t), scan.Proof, "") + if err != nil { + t.Fatal(err) + } + if _, err := s.receipts.PutCategoryScan(s.ctx, orphanScan); !errors.Is(err, tableshapecap.ErrInvalid) { + t.Fatalf("PutCategoryScan against foreign existing parent = %v, want tableshapecap.ErrInvalid", err) + } + s.assertReceiptAbsent(t, orphanScan.ID) + + ghostParent := "tsc_" + strings.Repeat("f", 32) + ghostScan, err := tableshapecap.NewCategoryScanReceipt(binding, ghostParent, + scan.CategoryColumnChoiceID, scan.ValueColumnChoiceID, scanValues(t), scan.Proof, "") + if err != nil { + t.Fatal(err) + } + if _, err := s.receipts.PutCategoryScan(s.ctx, ghostScan); !errors.Is(err, tableshapecap.ErrNotFound) { + t.Fatalf("PutCategoryScan against missing parent = %v, want tableshapecap.ErrNotFound", err) + } + s.assertReceiptAbsent(t, ghostScan.ID) +} + +func clonePublicColumns(v []tableshapecap.PublicColumn) []tableshapecap.PublicColumn { + return append([]tableshapecap.PublicColumn(nil), v...) +} +func cloneRoleAvailability(v []tableshapecap.RoleAvailability) []tableshapecap.RoleAvailability { + return append([]tableshapecap.RoleAvailability(nil), v...) +} +func cloneCatalogChoicesForTest(v tableshapecap.CatalogChoices) tableshapecap.CatalogChoices { + out := tableshapecap.CatalogChoices{ + Columns: append([]tableshapecap.ColumnChoice(nil), v.Columns...), + Operators: append([]tableshapecap.OperatorChoice(nil), v.Operators...), + Policies: append([]tableshapecap.PolicyChoice(nil), v.Policies...), + Operands: append([]tableshapecap.OperandChoice(nil), v.Operands...), + } + return out +} + +func TestNestedParentReceiptsAndCanonicalTokensReadBackExactly(t *testing.T) { + s := newCapabilityService(t) + + snapshot := richSnapshot("pgstate03-nested", "generation-nested", "delta") + if _, err := s.snapshots.Put(s.ctx, snapshot); err != nil { + s.fail("put seed snapshot", err) + } + s.assertSnapshotLiteral(t, snapshot) + + binding := testBinding("pgstate03-nested") + binding.SnapshotToken = snapshot.Token + catalog := testCatalog(t, binding, "2026-09-19T00:00:00Z") + if _, err := s.receipts.PutCatalog(s.ctx, catalog); err != nil { + s.fail("put catalog", err) + } + s.assertReceiptLiteral(t, catalog.ID, kindCatalog, "", binding, catalog.ContentDigest, catalog.CreatedAt, catalog) + + scan := testScan(t, catalog, "2026-09-19T00:00:01Z") + if _, err := s.receipts.PutCategoryScan(s.ctx, scan); err != nil { + s.fail("put scan", err) + } + s.assertReceiptLiteral(t, scan.ID, kindCategoryScan, catalog.ID, binding, scan.ContentDigest, scan.CreatedAt, scan) + + pivot := testPivot(t, catalog, scan, "2026-09-19T00:00:02Z") + if _, err := s.receipts.PutResolution(s.ctx, pivot); err != nil { + s.fail("put pivot", err) + } + s.assertReceiptLiteral(t, pivot.ID, kindResolution, catalog.ID, binding, pivot.ContentDigest, pivot.CreatedAt, pivot) + + derivedOne := testDerived(t, catalog, "", "total", "2026-09-19T00:00:03Z") + if _, err := s.receipts.PutResolution(s.ctx, derivedOne); err != nil { + s.fail("put derived one", err) + } + s.assertReceiptLiteral(t, derivedOne.ID, kindResolution, catalog.ID, binding, derivedOne.ContentDigest, derivedOne.CreatedAt, derivedOne) + + derivedTwo := testDerived(t, catalog, derivedOne.ID, "scaled", "2026-09-19T00:00:04Z") + if _, err := s.receipts.PutResolution(s.ctx, derivedTwo); err != nil { + s.fail("put derived two", err) + } + s.assertReceiptLiteral(t, derivedTwo.ID, kindResolution, catalog.ID, binding, derivedTwo.ContentDigest, derivedTwo.CreatedAt, derivedTwo) + + gotCatalog, err := s.receipts.GetCatalog(s.ctx, binding, catalog.ID) + if err != nil || !reflect.DeepEqual(gotCatalog, catalog) { + t.Fatalf("GetCatalog = %+v (%v), want %+v", gotCatalog, err, catalog) + } + gotScan, err := s.receipts.GetCategoryScan(s.ctx, binding, catalog.ID, scan.ID) + if err != nil || !reflect.DeepEqual(gotScan, scan) { + t.Fatalf("GetCategoryScan = %+v (%v), want %+v", gotScan, err, scan) + } + if w := s.witnessReceipt(scan.ID); w.payloadText == "" || !strings.Contains(w.payloadText, scan.Proof.ValuesDigest) { + t.Fatalf("scan payload lost its canonical values digest: %s", w.payloadText) + } + gotPivot, err := s.receipts.GetResolution(s.ctx, binding, catalog.ID, pivot.ID) + if err != nil || !reflect.DeepEqual(gotPivot, pivot) { + t.Fatalf("GetResolution pivot = %+v (%v), want %+v", gotPivot, err, pivot) + } + for i, want := range scan.Categories { + if !reflect.DeepEqual(gotPivot.Pivot.Categories[i].Value, want.Value) { + t.Fatalf("pivot category %d = %+v, want %+v", i, gotPivot.Pivot.Categories[i].Value, want.Value) + } + } + gotDerivedOne, err := s.receipts.GetResolution(s.ctx, binding, catalog.ID, derivedOne.ID) + if err != nil || !reflect.DeepEqual(gotDerivedOne, derivedOne) { + t.Fatalf("GetResolution derived one = %+v (%v), want %+v", gotDerivedOne, err, derivedOne) + } + gotDerivedTwo, err := s.receipts.GetResolution(s.ctx, binding, catalog.ID, derivedTwo.ID) + if err != nil || !reflect.DeepEqual(gotDerivedTwo, derivedTwo) { + t.Fatalf("GetResolution derived two = %+v (%v), want %+v", gotDerivedTwo, err, derivedTwo) + } + + if w := s.witnessReceipt(catalog.ID); !strings.Contains(w.payloadText, snapshot.Token) || w.binding.SnapshotToken != snapshot.Token { + t.Fatalf("catalog binding lost the canonical snapshot token: %+v", w) + } + if v := s.mustWitnessText(t, + `SELECT payload -> 'pivot' ->> 'categoryDiscoveryId' FROM `+s.qualified(TableShapeCapabilitiesTable)+` WHERE id = $1`, + pivot.ID); v != scan.ID { + t.Fatalf("pivot categoryDiscoveryId = %q, want %q", v, scan.ID) + } + if v := s.mustWitnessText(t, + `SELECT payload -> 'derived' -> 'left' ->> 'resolutionId' FROM `+s.qualified(TableShapeCapabilitiesTable)+` WHERE id = $1`, + derivedTwo.ID); v != derivedOne.ID { + t.Fatalf("derived nested resolutionId = %q, want %q", v, derivedOne.ID) + } +} + +func TestConcurrentSameKeyPutsConvergeOnOneImmutableRow(t *testing.T) { + s := newCapabilityService(t) + binding := testBinding("pgstate03-race") + catalog := testCatalog(t, binding, "2026-09-19T00:00:00Z") + snapshot := richSnapshot("pgstate03-race", "generation-race", "epsilon") + + var wg sync.WaitGroup + catalogResults := make([]error, 2) + stores := []*TableShapeCapabilityStore{s.receipts, s.receiptsB} + start := make(chan struct{}) + for i := range stores { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + _, err := stores[i].PutCatalog(s.ctx, catalog) + catalogResults[i] = err + }(i) + } + snapshotResults := make([]error, 2) + snapshotStores := []*CapabilitySnapshotStore{s.snapshots, s.snapshotsB} + for i := range snapshotStores { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + _, err := snapshotStores[i].Put(s.ctx, snapshot) + snapshotResults[i] = err + }(i) + } + close(start) + wg.Wait() + + for i, err := range catalogResults { + t.Logf("catalog put %d (writer backends A=%d B=%d, witness W=%d): err=%v", i, s.pidA, s.pidB, s.pidW, err) + if err != nil { + t.Fatalf("concurrent catalog put %d failed: %v", i, s.scrub(err)) + } + } + for i, err := range snapshotResults { + if err != nil { + t.Fatalf("concurrent snapshot put %d failed: %v", i, s.scrub(err)) + } + } + + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(TableShapeCapabilitiesTable)); n != 1 { + t.Fatalf("catalog rows after race = %d, want exactly 1", n) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(CapabilitySnapshotsTable)); n != 1 { + t.Fatalf("snapshot rows after race = %d, want exactly 1", n) + } + s.assertReceiptLiteral(t, catalog.ID, kindCatalog, "", binding, catalog.ContentDigest, catalog.CreatedAt, catalog) + s.assertSnapshotLiteral(t, snapshot) + + w := s.witnessReceipt(catalog.ID) + snapshotBefore := s.witnessSnapshot(snapshot.Token) + s.mustWitnessExec(t, `DROP TRIGGER table_shape_capabilities_immutable ON `+s.qualified(TableShapeCapabilitiesTable)) + s.mustWitnessExec(t, `UPDATE `+s.qualified(TableShapeCapabilitiesTable)+ + ` SET payload = jsonb_set(payload, '{savedShape,derivedColumns}', '42') WHERE id = $1`, catalog.ID) + s.mustWitnessExec(t, `CREATE TRIGGER table_shape_capabilities_immutable BEFORE UPDATE OR DELETE ON `+ + s.qualified(TableShapeCapabilitiesTable)+` FOR EACH ROW EXECUTE FUNCTION `+s.qualified("loom_capability_artifact_immutable")+`()`) + staleWitness := s.witnessReceipt(catalog.ID) + if staleWitness.ctid == w.ctid { + t.Fatal("stale fixture tampering did not move the row") + } + if _, err := s.receiptsB.PutCatalog(s.ctx, catalog); !errors.Is(err, tableshapecap.ErrIdentityClash) { + t.Fatalf("stale PutCatalog from writer B = %v, want tableshapecap.ErrIdentityClash", err) + } + after := s.witnessReceipt(catalog.ID) + if after.payloadText != staleWitness.payloadText || after.ctid != staleWitness.ctid { + t.Fatalf("refused stale PutCatalog changed the witness row: before=%+v after=%+v", staleWitness, after) + } + snapshotAfter := s.witnessSnapshot(snapshot.Token) + if snapshotAfter.payloadText != snapshotBefore.payloadText || snapshotAfter.ctid != snapshotBefore.ctid { + t.Fatalf("the untouched snapshot moved during the race: before=%+v after=%+v", snapshotBefore, snapshotAfter) + } +} + +func TestCapabilityImmutabilityGuardsAreEnforcedAtDatabaseLevel(t *testing.T) { + s := newCapabilityService(t) + binding := testBinding("pgstate03-immutable") + catalog := testCatalog(t, binding, "2026-09-19T00:00:00Z") + if _, err := s.receipts.PutCatalog(s.ctx, catalog); err != nil { + s.fail("put catalog", err) + } + snapshot := richSnapshot("pgstate03-immutable", "generation-immutable", "zeta") + if _, err := s.snapshots.Put(s.ctx, snapshot); err != nil { + s.fail("put snapshot", err) + } + + beforeReceipt := s.witnessReceipt(catalog.ID) + beforeSnapshot := s.witnessSnapshot(snapshot.Token) + + probeToken := "sha256:" + strings.Repeat("7", 64) + seedProbeSnapshot := func() { + if err := s.witnessExecErr(`INSERT INTO `+s.qualified(CapabilitySnapshotsTable)+ + ` (token, status, complete, truncated, payload) VALUES ($1, 'READY', true, false, '{}'::jsonb)`, probeToken); err != nil { + s.fail("seed probe snapshot", err) + } + } + probeReceiptSQL := `INSERT INTO ` + s.qualified(TableShapeCapabilitiesTable) + ` (id, kind, parent_catalog_id, + project, explorer_id, output_id, snapshot_token, authorization_scope, source_generation, draft_version, + draft_digest, base_document_digest, base_compilation_receipt_id, output_fingerprint, compiler_schema_digest, + content_digest, created_at, payload) + VALUES ('tsr_` + strings.Repeat("9", 32) + `', 'RESOLUTION', 'tsc_` + strings.Repeat("8", 32) + `', + 'p', 'e', 'o', 'st', 'as', 'sg', 1, 'dd', 'bd', 'bc', 'of', 'cs', 'sha256:` + strings.Repeat("e", 64) + + `', '', '{}'::jsonb)` + probeReceiptID := "tsr_" + strings.Repeat("9", 32) + seedProbeReceipt := func() { + if err := s.witnessExecErr(probeReceiptSQL); err != nil { + s.fail("seed probe receipt", err) + } + } + seedProbeSnapshot() + seedProbeReceipt() + + s.guardDropProbe(t, "capability_snapshots_immutable", CapabilitySnapshotsTable, nil, + `UPDATE `+s.qualified(CapabilitySnapshotsTable)+` SET status = 'FAILED' WHERE token = $1`, probeToken) + s.guardDropProbe(t, "capability_snapshots_immutable", CapabilitySnapshotsTable, seedProbeSnapshot, + `DELETE FROM `+s.qualified(CapabilitySnapshotsTable)+` WHERE token = $1`, probeToken) + s.guardDropProbe(t, "table_shape_capabilities_immutable", TableShapeCapabilitiesTable, nil, + `UPDATE `+s.qualified(TableShapeCapabilitiesTable)+` SET content_digest = $2 WHERE id = $1`, + probeReceiptID, "sha256:"+strings.Repeat("0", 64)) + s.guardDropProbe(t, "table_shape_capabilities_immutable", TableShapeCapabilitiesTable, seedProbeReceipt, + `DELETE FROM `+s.qualified(TableShapeCapabilitiesTable)+` WHERE id = $1`, probeReceiptID) + + afterReceipt := s.witnessReceipt(catalog.ID) + if afterReceipt.payloadText != beforeReceipt.payloadText || afterReceipt.ctid != beforeReceipt.ctid { + t.Fatalf("catalog row changed across the guard probes: before=%+v after=%+v", beforeReceipt, afterReceipt) + } + afterSnapshot := s.witnessSnapshot(snapshot.Token) + if afterSnapshot.payloadText != beforeSnapshot.payloadText || afterSnapshot.ctid != beforeSnapshot.ctid { + t.Fatalf("snapshot row changed across the guard probes: before=%+v after=%+v", beforeSnapshot, afterSnapshot) + } +} + +func TestCapabilitySchemaConstraintsRejectInvalidRows(t *testing.T) { + s := newCapabilityService(t) + binding := testBinding("pgstate03-constraints") + catalog := testCatalog(t, binding, "2026-09-19T00:00:00Z") + if _, err := s.receipts.PutCatalog(s.ctx, catalog); err != nil { + s.fail("put catalog", err) + } + snapshot := richSnapshot("pgstate03-constraints", "generation-constraints", "eta") + if _, err := s.snapshots.Put(s.ctx, snapshot); err != nil { + s.fail("put snapshot", err) + } + snapFQN := s.qualified(CapabilitySnapshotsTable) + recFQN := s.qualified(TableShapeCapabilitiesTable) + + type negative struct { + name string + sql string + args []any + wantState string + } + tokenA := "sha256:" + strings.Repeat("a", 64) + tokenC := "sha256:" + strings.Repeat("c", 64) + tokenD := "sha256:" + strings.Repeat("d", 64) + sqlA := `INSERT INTO ` + snapFQN + ` (token, status, complete, truncated, payload) VALUES ($1, $2, $3, $4, $5::jsonb)` + argsA := []any{tokenA, "BUILDING", false, false, `{"token":"` + tokenA + `"}`} + if err := s.witnessExecErr(sqlA, argsA...); err != nil { + s.fail("seed duplicate-token fixture", err) + } + + validReceiptSQL := `INSERT INTO ` + recFQN + ` (id, kind, parent_catalog_id, project, explorer_id, output_id, + snapshot_token, authorization_scope, source_generation, draft_version, draft_digest, base_document_digest, + base_compilation_receipt_id, output_fingerprint, compiler_schema_digest, content_digest, created_at, payload) + VALUES ('tsd_` + strings.Repeat("c", 32) + `', 'CATEGORY_SCAN', 'tsc_` + strings.Repeat("d", 32) + `', + 'p', 'e', 'o', 'st', 'as', 'sg', 1, 'dd', 'bd', 'bc', 'of', 'cs', 'sha256:` + strings.Repeat("e", 64) + `', '', '{}'::jsonb)` + + cases := []negative{ + {name: "snapshot token is never SQL NULL", sql: `INSERT INTO ` + snapFQN + ` (token, status, complete, truncated, payload) VALUES (NULL, 'BUILDING', false, false, '{}'::jsonb)`, wantState: "23502"}, + {name: "snapshot token keeps its content-address shape", sql: `INSERT INTO ` + snapFQN + ` (token, status, complete, truncated, payload) VALUES ('not-a-token', 'BUILDING', false, false, '{}'::jsonb)`, wantState: "23514"}, + {name: "snapshot status is a closed domain", sql: `INSERT INTO ` + snapFQN + ` (token, status, complete, truncated, payload) VALUES ('sha256:` + strings.Repeat("1", 64) + `', 'WEIRD', false, false, '{}'::jsonb)`, wantState: "23514"}, + {name: "READY implies complete and not-truncated", sql: `INSERT INTO ` + snapFQN + ` (token, status, complete, truncated, payload) VALUES ('sha256:` + strings.Repeat("2", 64) + `', 'READY', false, false, '{}'::jsonb)`, wantState: "23514"}, + {name: "READY implies complete and not-truncated (truncated)", sql: `INSERT INTO ` + snapFQN + ` (token, status, complete, truncated, payload) VALUES ('sha256:` + strings.Repeat("3", 64) + `', 'READY', true, true, '{}'::jsonb)`, wantState: "23514"}, + {name: "snapshot payload is never SQL NULL", sql: `INSERT INTO ` + snapFQN + ` (token, status, complete, truncated, payload) VALUES ('sha256:` + strings.Repeat("4", 64) + `', 'BUILDING', false, false, NULL)`, wantState: "23502"}, + {name: "snapshot payload JSON null is not an object", sql: `INSERT INTO ` + snapFQN + ` (token, status, complete, truncated, payload) VALUES ('sha256:` + strings.Repeat("5", 64) + `', 'BUILDING', false, false, 'null'::jsonb)`, wantState: "23514"}, + {name: "snapshot payload JSON array is not an object", sql: `INSERT INTO ` + snapFQN + ` (token, status, complete, truncated, payload) VALUES ('sha256:` + strings.Repeat("6", 64) + `', 'BUILDING', false, false, '[]'::jsonb)`, wantState: "23514"}, + {name: "duplicate tokens are impossible", sql: sqlA, args: argsA, wantState: "23505"}, + + {name: "receipt kind is a closed domain", sql: strings.Replace(validReceiptSQL, "'CATEGORY_SCAN'", "'BOGUS'", 1), wantState: "23514"}, + {name: "catalog kind requires the tsc_ prefix", sql: strings.Replace(strings.Replace(validReceiptSQL, "'CATEGORY_SCAN'", "'CATALOG'", 1), + "'tsd_"+strings.Repeat("c", 32)+"'", "'tsd_"+strings.Repeat("c", 32)+"'", 1), wantState: "23514"}, + {name: "receipt binding columns are never SQL NULL", sql: strings.Replace(validReceiptSQL, "'p', 'e', 'o'", "NULL, 'e', 'o'", 1), wantState: "23502"}, + {name: "receipt binding columns are never empty", sql: strings.Replace(validReceiptSQL, "'p', 'e', 'o'", "'', 'e', 'o'", 1), wantState: "23514"}, + {name: "draft_version is never SQL NULL", sql: strings.Replace(validReceiptSQL, "1, 'dd'", "NULL, 'dd'", 1), wantState: "23502"}, + {name: "draft_version has a floor of 1", sql: strings.Replace(validReceiptSQL, "1, 'dd'", "0, 'dd'", 1), wantState: "23514"}, + {name: "content_digest keeps its sha256 shape", sql: strings.Replace(validReceiptSQL, "'sha256:"+strings.Repeat("e", 64)+"'", "'md5:deadbeef'", 1), wantState: "23514"}, + {name: "non-catalog rows are parent-bound", sql: strings.Replace(validReceiptSQL, "'CATEGORY_SCAN', 'tsc_"+strings.Repeat("d", 32)+"'", "'CATEGORY_SCAN', ''", 1), wantState: "23514"}, + {name: "receipt payload is never SQL NULL", sql: strings.Replace(validReceiptSQL, "'{}'::jsonb", "NULL", 1), wantState: "23502"}, + {name: "receipt payload JSON scalar is not an object", sql: strings.Replace(validReceiptSQL, "'{}'::jsonb", "'7'::jsonb", 1), wantState: "23514"}, + {name: "duplicate receipt IDs are impossible", sql: `INSERT INTO ` + recFQN + ` (id, kind, parent_catalog_id, project, explorer_id, output_id, + snapshot_token, authorization_scope, source_generation, draft_version, draft_digest, base_document_digest, + base_compilation_receipt_id, output_fingerprint, compiler_schema_digest, content_digest, created_at, payload) + SELECT id, kind, parent_catalog_id, project, explorer_id, output_id, snapshot_token, authorization_scope, + source_generation, draft_version, draft_digest, base_document_digest, base_compilation_receipt_id, + output_fingerprint, compiler_schema_digest, content_digest, created_at, payload FROM ` + recFQN + ` WHERE id = $1`, + args: []any{catalog.ID}, wantState: "23505"}, + } + for _, c := range cases { + err := s.witnessExecErr(c.sql, c.args...) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != c.wantState { + t.Errorf("%s: err = %v, want SQLSTATE %s", c.name, s.scrub(err), c.wantState) + } + } + + if err := s.witnessExecErr(`INSERT INTO `+snapFQN+` (token, status, complete, truncated, payload) VALUES ($1, 'BUILDING', false, true, '{}'::jsonb)`, tokenC); err != nil { + t.Errorf("control: legal BUILDING/truncated snapshot insert refused: %s", s.scrub(err)) + } + if err := s.witnessExecErr(`INSERT INTO `+snapFQN+` (token, status, complete, truncated, payload) VALUES ($1, 'FAILED', true, true, '{}'::jsonb)`, tokenD); err != nil { + t.Errorf("control: legal FAILED snapshot insert refused: %s", s.scrub(err)) + } + if err := s.witnessExecErr(validReceiptSQL); err != nil { + t.Errorf("control: legal category-scan row insert refused: %s", s.scrub(err)) + } + if n := s.execCount(t, "SELECT count(*) FROM "+snapFQN+" WHERE token = ANY($1)", []string{tokenC, tokenD}); n != 2 { + t.Fatalf("control snapshot rows = %d, want 2", n) + } +} + +func TestCapabilityStoresFailClosedWithoutTheirTables(t *testing.T) { + s := newCapabilityService(t) + emptySchema := fmt.Sprintf("pgstate03_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstate03ResourceNamePattern.MatchString(emptySchema) { + t.Fatalf("owned schema name %q does not match the required pattern", emptySchema) + } + s.mustWitnessExec(t, `CREATE SCHEMA "`+emptySchema+`"`) + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if err := execOn(cleanupCtx, s.witness, `DROP SCHEMA IF EXISTS "`+emptySchema+`" CASCADE`); err != nil { + t.Errorf("drop empty schema: %s", s.scrub(err)) + } + }) + orphanSnapshots, err := NewCapabilitySnapshotStore(s.witness, emptySchema) + if err != nil { + s.fail("build orphan snapshot store", err) + } + orphanReceipts, err := NewTableShapeCapabilityStore(s.witness, emptySchema) + if err != nil { + s.fail("build orphan receipt store", err) + } + + binding := testBinding("pgstate03-unmigrated") + catalog := testCatalog(t, binding, "") + scan := testScan(t, catalog, "") + pivot := testPivot(t, catalog, scan, "") + lookup := tableshapecap.CatalogLookup{ + Project: binding.Project, ExplorerID: binding.ExplorerID, OutputID: binding.OutputID, + SnapshotToken: binding.SnapshotToken, AuthorizationScope: binding.AuthorizationScope, + SourceGeneration: binding.SourceGeneration, DraftVersion: binding.DraftVersion, + DraftDigest: binding.DraftDigest, BaseDocumentDigest: binding.BaseDocumentDigest, + } + snapshot := richSnapshot("pgstate03-unmigrated", "generation-unmigrated", "theta") + + operations := []struct { + name string + run func() error + }{ + {"snapshot put", func() error { _, err := orphanSnapshots.Put(s.ctx, snapshot); return err }}, + {"snapshot get by token", func() error { _, err := orphanSnapshots.GetByToken(s.ctx, snapshot.Token); return err }}, + {"snapshot get by identity", func() error { _, err := orphanSnapshots.GetByIdentity(s.ctx, snapshot.Identity); return err }}, + {"catalog put", func() error { _, err := orphanReceipts.PutCatalog(s.ctx, catalog); return err }}, + {"catalog get", func() error { _, err := orphanReceipts.GetCatalog(s.ctx, binding, catalog.ID); return err }}, + {"catalog lookup", func() error { _, err := orphanReceipts.GetCatalogForLookup(s.ctx, lookup, catalog.ID); return err }}, + {"scan put", func() error { _, err := orphanReceipts.PutCategoryScan(s.ctx, scan); return err }}, + {"scan get", func() error { + _, err := orphanReceipts.GetCategoryScan(s.ctx, binding, catalog.ID, scan.ID) + return err + }}, + {"resolution put", func() error { _, err := orphanReceipts.PutResolution(s.ctx, pivot); return err }}, + {"resolution get", func() error { _, err := orphanReceipts.GetResolution(s.ctx, binding, catalog.ID, pivot.ID); return err }}, + } + for _, op := range operations { + err := op.run() + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "42P01" { + t.Fatalf("%s against unmigrated schema = %v, want SQLSTATE 42P01, never a typed miss or fallback", + op.name, s.scrub(err)) + } + } +} + +func TestZZNoLeftoverPgstate03Resources(t *testing.T) { + raw := testPostgresURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %v", err) + } + defer admin.Close() + + var leftoverDBs int + if err := admin.QueryRow(ctx, + `SELECT count(*) FROM pg_database WHERE datname LIKE 'pgstate03\_it\_%' ESCAPE '\'`).Scan(&leftoverDBs); err != nil { + t.Fatalf("count leftover databases: %v", err) + } + if leftoverDBs != 0 { + var names []string + rows, err := admin.Query(ctx, `SELECT datname FROM pg_database WHERE datname LIKE 'pgstate03\_it\_%' ESCAPE '\'`) + if err == nil { + for rows.Next() { + var n string + if err := rows.Scan(&n); err == nil { + names = append(names, n) + } + } + rows.Close() + } + t.Fatalf("%d isolated databases left behind: %v", leftoverDBs, names) + } + + var leftoverSchemas int + if err := admin.QueryRow(ctx, + `SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgstate03\_it\_%' ESCAPE '\'`).Scan(&leftoverSchemas); err != nil { + t.Fatalf("count leftover schemas: %v", err) + } + if leftoverSchemas != 0 { + t.Fatalf("%d owned schemas left behind in the service database", leftoverSchemas) + } +} + +func assertSnapshotEquivalent(t *testing.T, label string, got *capability.Snapshot, want capability.Snapshot) { + t.Helper() + if got == nil { + t.Fatalf("%s = nil, want the stored snapshot", label) + } + if got.Token != want.Token { + t.Fatalf("%s token = %q, want %q", label, got.Token, want.Token) + } + gotPayload := got.CanonicalPayload() + sum := sha256.Sum256(gotPayload) + if got.Token != "sha256:"+hex.EncodeToString(sum[:]) { + t.Fatalf("%s returned a snapshot whose token does not bind its canonical payload", label) + } + if !bytes.Equal(gotPayload, want.CanonicalPayload()) { + t.Fatalf("%s returned canonical payload bytes that differ from the stored artifact", label) + } + if got.Status != want.Status || got.Complete != want.Complete || got.Truncated != want.Truncated { + t.Fatalf("%s status triple = %v/%v/%v, want %v/%v/%v", label, + got.Status, got.Complete, got.Truncated, want.Status, want.Complete, want.Truncated) + } +} diff --git a/internal/backends/postgres/internal/explorer/drafts.go b/internal/backends/postgres/internal/explorer/drafts.go new file mode 100644 index 000000000..247ec2c17 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/drafts.go @@ -0,0 +1,167 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/explorer" +) + +const DraftRevisionsTable = "explorers_draft_revisions" + +const historyIdentityConstraint = "explorers_draft_revisions_identity_pk" + +type draftSQL struct { + preimageRead string + historyInsert string + ownerCAS string + revisionGet string +} + +func newDraftSQL(schema string) draftSQL { + ownerTable := pgx.Identifier{schema, OwnerTable}.Sanitize() + historyTable := pgx.Identifier{schema, DraftRevisionsTable}.Sanitize() + cols := strings.Join(ownerColumns, ", ") + return draftSQL{ + preimageRead: "SELECT " + cols + " FROM " + ownerTable + + " WHERE project = $1 AND explorer_id = $2 AND draft_version = $3" + + " AND ($4 = '' OR draft_digest = $4)", + historyInsert: "INSERT INTO " + historyTable + + " (project, explorer_id, draft_version, id, title, draft_digest, draft_config," + + " snapshot_token, source_generation, authorization_scope_digest, updated_by, updated_at)" + + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", + ownerCAS: "UPDATE " + ownerTable + " SET" + + " title = $4, draft_config = $5, draft_digest = $6, draft_snapshot_token = $7," + + " draft_source_generation = $8, draft_authorization_scope_digest = $9," + + " last_authoring_command_id = $10, last_authoring_command_digest = $11," + + " last_authoring_command_results = $12, updated_by = $13, updated_at = $14," + + " draft_version = $15, previous_draft_revision_id = $16" + + " WHERE project = $1 AND explorer_id = $2 AND draft_version = $3" + + " AND ($17 = '' OR draft_digest = $17)" + + " RETURNING " + cols, + revisionGet: "SELECT id, project, explorer_id, draft_version, title, draft_digest," + + " draft_config, snapshot_token, source_generation, authorization_scope_digest," + + " updated_by, updated_at FROM " + historyTable + + " WHERE project = $1 AND explorer_id = $2 AND id = $3", + } +} + +func draftRevisionKey(project, explorerID string, version int64) string { + h := sha256.New() + for _, part := range []string{project, explorerID, strconv.FormatInt(version, 10)} { + _, _ = h.Write([]byte{0}) + _, _ = h.Write([]byte(part)) + } + return "draft_revision_" + hex.EncodeToString(h.Sum(nil)) +} + +func (s *Store) SaveDraft(ctx context.Context, e explorer.Explorer, expected int64, expectedDigest ...string) (*explorer.Explorer, error) { + digest := "" + if len(expectedDigest) > 0 { + digest = strings.TrimSpace(expectedDigest[0]) + } + e.UpdatedAt = time.Now().UTC() + + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire save-draft connection: %w", err) + } + defer conn.Release() + + tx, err := conn.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: begin save draft %q/%q: %w", e.Project, e.ExplorerID, err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var previous explorer.Explorer + err = scanOwner(tx.QueryRow(ctx, s.drafts.preimageRead, e.Project, e.ExplorerID, expected, digest), &previous) + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrDraftConflict + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: read draft preimage %q/%q: %w", e.Project, e.ExplorerID, err) + } + + revisionID := draftRevisionKey(previous.Project, previous.ExplorerID, previous.DraftVersion) + _, err = tx.Exec(ctx, s.drafts.historyInsert, + previous.Project, previous.ExplorerID, previous.DraftVersion, revisionID, + previous.Title, previous.DraftDigest, jsonParam(previous.DraftConfig), + previous.DraftSnapshotToken, previous.DraftSourceGeneration, + previous.DraftAuthorizationScopeDigest, previous.UpdatedBy, previous.UpdatedAt) + if err != nil { + if pgstore.IsUniqueConstraintViolation(err, historyIdentityConstraint) { + return nil, errors.Join(explorer.ErrDraftConflict, err) + } + return nil, fmt.Errorf("postgres explorer store: persist draft preimage %q/%q version %d: %w", + e.Project, e.ExplorerID, previous.DraftVersion, err) + } + + if s.afterHistoryInsert != nil { + if err := s.afterHistoryInsert(ctx, tx); err != nil { + return nil, fmt.Errorf("postgres explorer store: save draft %q/%q after history insert: %w", + e.Project, e.ExplorerID, err) + } + } + + var updated explorer.Explorer + err = scanOwner(tx.QueryRow(ctx, s.drafts.ownerCAS, + e.Project, e.ExplorerID, expected, + e.Title, jsonParam(e.DraftConfig), e.DraftDigest, e.DraftSnapshotToken, + e.DraftSourceGeneration, e.DraftAuthorizationScopeDigest, + e.LastAuthoringCommandID, e.LastAuthoringCommandDigest, jsonParam(e.LastAuthoringCommandResults), + e.UpdatedBy, e.UpdatedAt, + expected+1, revisionID, + digest), &updated) + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrDraftConflict + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: apply draft CAS %q/%q: %w", e.Project, e.ExplorerID, err) + } + + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("postgres explorer store: commit save draft %q/%q: %w", e.Project, e.ExplorerID, err) + } + return &updated, nil +} + +func (s *Store) GetDraftRevision(ctx context.Context, project, explorerID, revisionID string) (*explorer.DraftRevision, error) { + project = strings.TrimSpace(project) + explorerID = strings.TrimSpace(explorerID) + revisionID = strings.TrimSpace(revisionID) + if project == "" || explorerID == "" || revisionID == "" { + return nil, explorer.ErrNotFound + } + + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire get-draft-revision connection: %w", err) + } + defer conn.Release() + + var out explorer.DraftRevision + err = conn.QueryRow(ctx, s.drafts.revisionGet, project, explorerID, revisionID).Scan( + &out.ID, &out.Project, &out.ExplorerID, &out.DraftVersion, &out.Title, + &out.DraftDigest, &out.DraftConfig, &out.SnapshotToken, + &out.SourceGeneration, &out.AuthorizationScopeDigest, &out.UpdatedBy, &out.UpdatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: read draft revision %q/%q/%q: %w", + project, explorerID, revisionID, err) + } + return &out, nil +} + +var _ explorer.DraftRevisionReader = (*Store)(nil) diff --git a/internal/backends/postgres/internal/explorer/drafts_test.go b/internal/backends/postgres/internal/explorer/drafts_test.go new file mode 100644 index 000000000..73615af98 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/drafts_test.go @@ -0,0 +1,1071 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/url" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/explorer" +) + +var pgstate01NameRe = regexp.MustCompile(`^pgstate01_it_[0-9]+_[0-9a-f]+$`) + +var ( + draftCASOwnedMu sync.Mutex + draftCASOwnedDBs = map[string]string{} +) + +func draftCASRegister(db, schema string) { + draftCASOwnedMu.Lock() + defer draftCASOwnedMu.Unlock() + draftCASOwnedDBs[db] = schema +} + +func newDraftFixture(t *testing.T) *ownerFixture { + t.Helper() + raw := testPostgresURL(t) + + u, err := parseDraftURL(raw) + if err != nil { + t.Fatalf("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL: %v", err) + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + t.Cleanup(cancel) + + admin, err := openDraftPool(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgstate01_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstate01NameRe.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + + t.Cleanup(func() { + var mine int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname = $1`, dbName).Scan(&mine); err != nil { + t.Errorf("leftover audit: query database catalog: %s", scrub(err)) + } + if mine != 0 { + t.Errorf("leftover audit: isolated database %s still exists", dbName) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + draftCASRegister(dbName, "") + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + iso, err := openDraftPool(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("pgstate01_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstate01NameRe.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + draftCASRegister(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + return &ownerFixture{ctx: ctx, admin: admin, iso: iso, dbName: dbName, schema: schema, dsn: dsn, scrub: scrub, fail: fail} +} + +func (f *ownerFixture) historyTable() string { + return pgx.Identifier{f.schema, DraftRevisionsTable}.Sanitize() +} + +func (f *ownerFixture) newPinnedStore(t *testing.T) (*Store, int64) { + t.Helper() + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, + MaxConns: 1, + MinConns: 0, + AcquireTimeout: 15 * time.Second, + }) + if err != nil { + f.fail("open pinned pg-store pool", err) + } + t.Cleanup(client.Close) + store, err := New(client, f.schema) + if err != nil { + f.fail("build Explorer draft store", err) + } + return store, draftClientBackendPID(t, f, client) +} + +func (f *ownerFixture) newPinnedWitness(t *testing.T) (*pgstore.Client, int64) { + t.Helper() + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, + MaxConns: 1, + MinConns: 0, + AcquireTimeout: 15 * time.Second, + }) + if err != nil { + f.fail("open pinned witness pool", err) + } + t.Cleanup(client.Close) + return client, draftClientBackendPID(t, f, client) +} + +func draftClientBackendPID(t *testing.T, f *ownerFixture, client *pgstore.Client) int64 { + t.Helper() + conn, err := client.Acquire(f.ctx) + if err != nil { + f.fail("acquire backend-pid connection", err) + } + defer conn.Release() + var pid int64 + if err := conn.QueryRow(f.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + f.fail("read pg_backend_pid", err) + } + if pid <= 0 { + t.Fatalf("pg_backend_pid() returned %d, want a live backend", pid) + } + return pid +} + +func requireDistinctBackends(t *testing.T, where string, pids ...int64) { + t.Helper() + seen := make(map[int64]string, len(pids)) + for i, pid := range pids { + if prev, dup := seen[pid]; dup { + t.Fatalf("%s: %s and writer %d share backend pid %d; CAS claims require distinct real PG connections", + where, prev, i, pid) + } + seen[pid] = fmt.Sprintf("writer %d", i) + } +} + +const draftCfgOld = `{"documents":[{"outputId":"patients","construction":{"steps":[{"id":"pivot","outputs":[{"id":"col_age","name":"age"}]}]}}]}` + +const draftCfgNew = `{"documents":[{"outputId":"patients","construction":{"steps":[{"id":"pivot","outputs":[{"id":"col_age","name":"age"}]},{"id":"derive","outputs":[{"id":"col_ratio","name":"ratio"}]}]}}]}` + +const draftResultsOK = `{"matchedRows": 12, "traceId": "trc_draft"}` + +var seedUpdatedAt = time.Date(2026, 2, 3, 4, 5, 6, 654321000, time.UTC) + +const seedUpdatedAtText = "2026-02-03 04:05:06.654321" + +func seedDraftOwner(t *testing.T, f *ownerFixture, store *Store, project, id string) explorer.Explorer { + t.Helper() + e := explorer.Explorer{ + Project: project, + ExplorerID: id, + Title: "Patients", + ManagementMode: explorer.ManagementInteractive, + DraftConfig: []byte(draftCfgOld), + DraftVersion: 4, + DraftDigest: "sha256:old-draft", + DraftSnapshotToken: "snapshot-old", + DraftSourceGeneration: "generation-old", + DraftAuthorizationScopeDigest: "scope-old", + PreviousDraftRevisionID: "", + LastAuthoringCommandID: "seed-cmd", + LastAuthoringCommandDigest: "sha256:seed-cmd", + LastAuthoringCommandResults: []byte(draftResultsOK), + ActiveRevisionID: "published-revision-9", + UpdatedBy: "alice", + UpdatedAt: seedUpdatedAt, + } + if _, err := store.Create(f.ctx, e); err != nil { + f.fail("seed draft owner", err) + } + return e +} + +func draftCandidate(seed explorer.Explorer, suffix, digest string) explorer.Explorer { + candidate := seed + candidate.Title = "Patients " + suffix + candidate.DraftConfig = []byte(draftCfgNew) + candidate.DraftVersion = 999 + candidate.DraftDigest = digest + candidate.DraftSnapshotToken = "snapshot-" + suffix + candidate.DraftSourceGeneration = "generation-" + suffix + candidate.DraftAuthorizationScopeDigest = "scope-" + suffix + candidate.PreviousDraftRevisionID = "forged-previous-id" + candidate.LastAuthoringCommandID = "cmd-" + suffix + candidate.LastAuthoringCommandDigest = "sha256:cmd-" + suffix + candidate.LastAuthoringCommandResults = []byte(draftResultsOK) + candidate.ActiveRevisionID = "stale-revision-must-not-overwrite" + candidate.UpdatedBy = "bob" + candidate.UpdatedAt = time.Time{} + return candidate +} + +type literalHistoryRow struct { + ID string + Title string + DraftVersion int64 + DraftDigest string + DraftConfigText string + DraftConfigKind string + SnapshotToken string + SourceGeneration string + AuthorizationScopeDigest string + UpdatedBy string + UpdatedAtText string +} + +func (f *ownerFixture) historyRows(t *testing.T, project, explorerID string) []literalHistoryRow { + t.Helper() + rows, err := f.iso.Query(f.ctx, `SELECT + id, title, draft_version, draft_digest, + COALESCE(draft_config::text, ''), COALESCE(jsonb_typeof(draft_config), 'missing'), + snapshot_token, source_generation, authorization_scope_digest, updated_by, + to_char(updated_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US') + FROM `+f.historyTable()+` WHERE project = $1 AND explorer_id = $2 ORDER BY draft_version`, + project, explorerID) + if err != nil { + f.fail("read history rows", err) + } + defer rows.Close() + out := []literalHistoryRow{} + for rows.Next() { + var r literalHistoryRow + if err := rows.Scan(&r.ID, &r.Title, &r.DraftVersion, &r.DraftDigest, + &r.DraftConfigText, &r.DraftConfigKind, &r.SnapshotToken, &r.SourceGeneration, + &r.AuthorizationScopeDigest, &r.UpdatedBy, &r.UpdatedAtText); err != nil { + f.fail("scan history row", err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + f.fail("read history rows", err) + } + return out +} + +func (f *ownerFixture) historyCount(t *testing.T, project, explorerID string) int { + t.Helper() + var n int + if err := f.iso.QueryRow(f.ctx, + `SELECT count(*) FROM `+f.historyTable()+` WHERE project = $1 AND explorer_id = $2`, + project, explorerID).Scan(&n); err != nil { + f.fail("count history rows", err) + } + return n +} + +func requireExactPreimageRow(t *testing.T, r literalHistoryRow, project, explorerID string, version int64) { + t.Helper() + wantID := draftRevisionKey(project, explorerID, version) + if r.ID != wantID { + t.Fatalf("history id = %q, want derived %q", r.ID, wantID) + } + if r.Title != "Patients" || r.DraftDigest != "sha256:old-draft" || + r.SnapshotToken != "snapshot-old" || r.SourceGeneration != "generation-old" || + r.AuthorizationScopeDigest != "scope-old" || r.UpdatedBy != "alice" || + r.UpdatedAtText != seedUpdatedAtText || r.DraftVersion != version { + t.Fatalf("history preimage scalars changed: %+v", r) + } + if r.DraftConfigKind != "object" { + t.Fatalf("history preimage config kind = %q, want object", r.DraftConfigKind) + } + if !strings.Contains(r.DraftConfigText, `"col_age"`) || strings.Contains(r.DraftConfigText, `"col_ratio"`) { + t.Fatalf("history preimage is not the exact prior workspace: %s", r.DraftConfigText) + } +} + +func TestDraftCASRevisionKeyDerivation(t *testing.T) { + for _, c := range []struct { + project, explorerID string + version int64 + want string + }{ + {"project-a", "patients", 4, "draft_revision_14a3b30bb25b88835660c485a8a8a6fdc4a4e40ec9a12444d2b1c7d283499c00"}, + {"project-a", "patients", 0, "draft_revision_3a07ca4c963c00a696fb0d04080151732edf235771b51d96a8a6a92283b0b88a"}, + {"proj state", "exp/ents", 12345, "draft_revision_e3966c30839b313412cb0b45895a98b51a323953645c56d9a34f5990745fa12e"}, + } { + want := "draft_revision_" + sha256DraftKeyParts(c.project, c.explorerID, fmt.Sprint(c.version)) + if want != c.want { + t.Fatalf("local recomputation disagrees with the pinned literal: %q vs %q", want, c.want) + } + if got := draftRevisionKey(c.project, c.explorerID, c.version); got != c.want { + t.Fatalf("draftRevisionKey(%q,%q,%d) = %q, want %q", c.project, c.explorerID, c.version, got, want) + } + } + if draftRevisionKey("a", "b", 1) == draftRevisionKey("a", "b", 2) { + t.Fatal("version does not change the derived id") + } + if draftRevisionKey("a", "b", 1) == draftRevisionKey("ab", "", 1) { + t.Fatal("unseparated identity parts collide") + } +} + +func sha256DraftKeyParts(parts ...string) string { + h := sha256.New() + for _, part := range parts { + _, _ = h.Write([]byte{0}) + _, _ = h.Write([]byte(part)) + } + return hex.EncodeToString(h.Sum(nil)) +} + +func TestDraftCASStatementSQLContract(t *testing.T) { + store, err := New(&pgstore.Client{}, "loom_control") + if err != nil { + t.Fatalf("New = %v", err) + } + if !strings.Contains(store.drafts.preimageRead, `"loom_control"."explorers"`) || + !strings.Contains(store.drafts.historyInsert, `"loom_control"."explorers_draft_revisions"`) || + !strings.Contains(store.drafts.ownerCAS, `"loom_control"."explorers"`) || + !strings.Contains(store.drafts.revisionGet, `"loom_control"."explorers_draft_revisions"`) { + t.Fatal("every draft statement must bind the schema-qualified, sanitized tables") + } + for _, guard := range []string{ + "WHERE project = $1 AND explorer_id = $2 AND draft_version = $3", + "($4 = '' OR draft_digest = $4)", + } { + if !strings.Contains(store.drafts.preimageRead, guard) { + t.Errorf("preimage read missing guard %q", guard) + } + } + for _, part := range []string{ + "WHERE project = $1 AND explorer_id = $2 AND draft_version = $3", + "($17 = '' OR draft_digest = $17)", + "draft_version = $15", + "previous_draft_revision_id = $16", + "RETURNING", + } { + if !strings.Contains(store.drafts.ownerCAS, part) { + t.Errorf("owner CAS missing %q", part) + } + } + for _, forbidden := range []string{"active_revision_id", "management_mode"} { + setClause := store.drafts.ownerCAS[strings.Index(store.drafts.ownerCAS, "SET"):] + setClause = setClause[:strings.Index(setClause, "WHERE")] + if strings.Contains(setClause, forbidden+" =") { + t.Errorf("owner CAS must never rewrite %s", forbidden) + } + } + if !strings.Contains(store.drafts.historyInsert, "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)") { + t.Error("history insert must use twelve ordered parameters") + } + for _, guard := range []string{"WHERE project = $1 AND explorer_id = $2 AND id = $3"} { + if !strings.Contains(store.drafts.revisionGet, guard) { + t.Errorf("revision read missing scope %q", guard) + } + } +} + +func TestDraftCASStoresExactPreimageAndPreservesPublishedHead(t *testing.T) { + f := newDraftFixture(t) + f.migrate(t) + writer, writerPID := f.newPinnedStore(t) + witness, witnessPID := f.newPinnedWitness(t) + requireDistinctBackends(t, "exact-preimage round trip", writerPID, witnessPID) + + seed := seedDraftOwner(t, f, writer, "proj_pre", "patients") + + before := f.literalRow(t, "proj_pre", "patients") + if before.DraftVersion != 4 || before.DraftDigest != "sha256:old-draft" || + before.PreviousDraftRevisionID != "" || before.ActiveRevisionID != "published-revision-9" || + before.UpdatedAtText != seedUpdatedAtText { + t.Fatalf("seed head drifted before the CAS: %+v", before) + } + + candidate := draftCandidate(seed, "v2", "sha256:new-draft") + stored, err := writer.SaveDraft(f.ctx, candidate, 4, "sha256:old-draft") + if err != nil { + f.fail("save draft", err) + } + if stored.DraftVersion != 5 { + t.Fatalf("stored draftVersion = %d, want 5 (candidate 999 ignored, +1 exactly once)", stored.DraftVersion) + } + if stored.ActiveRevisionID != "published-revision-9" { + t.Fatalf("SaveDraft overwrote the published head with %q", stored.ActiveRevisionID) + } + wantPrevID := draftRevisionKey("proj_pre", "patients", 4) + if stored.PreviousDraftRevisionID != wantPrevID { + t.Fatalf("previousDraftRevisionId = %q, want derived %q", stored.PreviousDraftRevisionID, wantPrevID) + } + if !stored.UpdatedAt.After(seedUpdatedAt) { + t.Fatalf("updatedAt = %s, want the adapter-owned now after the seed literal", stored.UpdatedAt) + } + + after := f.literalRow(t, "proj_pre", "patients") + if after.DraftVersion != 5 || after.DraftDigest != "sha256:new-draft" || + after.DraftSnapshotToken != "snapshot-v2" || after.DraftSourceGeneration != "generation-v2" || + after.DraftAuthzScopeDigest != "scope-v2" || after.Title != "Patients v2" || + after.LastAuthoringCommandID != "cmd-v2" || after.LastAuthoringCommandDigest != "sha256:cmd-v2" || + after.UpdatedBy != "bob" || after.PreviousDraftRevisionID != wantPrevID || + after.ActiveRevisionID != "published-revision-9" || after.ManagementMode != "INTERACTIVE" { + t.Fatalf("winner head wrong: %+v", after) + } + if after.UpdatedAtText == seedUpdatedAtText { + t.Fatal("updatedAt kept the stored preimage instant; the adapter must own it") + } + if after.DraftConfigKind != "object" || !strings.Contains(after.DraftConfigText, `"col_ratio"`) { + t.Fatalf("winner config must carry the candidate workspace: %+v", after) + } + + rows := f.historyRows(t, "proj_pre", "patients") + if len(rows) != 1 { + t.Fatalf("history rows = %d, want exactly 1", len(rows)) + } + requireExactPreimageRow(t, rows[0], "proj_pre", "patients", 4) + + revision, err := func() (*explorer.DraftRevision, error) { + reader, err := New(witness, f.schema) + if err != nil { + f.fail("build reader store", err) + } + return reader.GetDraftRevision(f.ctx, "proj_pre", "patients", wantPrevID) + }() + if err != nil { + f.fail("get draft revision", err) + } + if revision.ID != wantPrevID || revision.Project != "proj_pre" || revision.ExplorerID != "patients" || + revision.DraftVersion != 4 || revision.DraftDigest != "sha256:old-draft" || + revision.Title != "Patients" || revision.SnapshotToken != "snapshot-old" || + revision.SourceGeneration != "generation-old" || revision.AuthorizationScopeDigest != "scope-old" || + revision.UpdatedBy != "alice" || !revision.UpdatedAt.Equal(seedUpdatedAt) { + t.Fatalf("revision read-back = %+v", revision) + } + if !jsonLikeContains(t, revision.DraftConfig, "col_age") || jsonLikeContains(t, revision.DraftConfig, "col_ratio") { + t.Fatalf("revision config is not the exact prior workspace: %s", revision.DraftConfig) + } +} + +func TestDraftCASMissingAndJSONNullDraftConfigAreDistinct(t *testing.T) { + f := newDraftFixture(t) + f.migrate(t) + writer, writerPID := f.newPinnedStore(t) + _, witnessPID := f.newPinnedWitness(t) + requireDistinctBackends(t, "null-state round trip", writerPID, witnessPID) + + seed := seedDraftOwner(t, f, writer, "proj_null", "patients") + if _, err := f.iso.Exec(f.ctx, + `UPDATE `+f.ownerTable()+` SET draft_config = 'null'::jsonb WHERE project = $1 AND explorer_id = $2`, + "proj_null", "patients"); err != nil { + f.fail("set json-null config", err) + } + if row := f.literalRow(t, "proj_null", "patients"); row.DraftConfigKind != "null" { + t.Fatalf("seed config kind = %q, want jsonb 'null' distinct from SQL NULL", row.DraftConfigKind) + } + + candidate := draftCandidate(seed, "cleared", "sha256:cleared") + candidate.DraftConfig = nil + stored, err := writer.SaveDraft(f.ctx, candidate, 4, "sha256:old-draft") + if err != nil { + f.fail("save draft with nil config", err) + } + if stored.DraftConfig != nil { + t.Fatalf("nil candidate config survived as %s", stored.DraftConfig) + } + head := f.literalRow(t, "proj_null", "patients") + if head.DraftConfigKind != "missing" { + t.Fatalf("head config kind = %q, want missing (SQL NULL)", head.DraftConfigKind) + } + rows := f.historyRows(t, "proj_null", "patients") + if len(rows) != 1 || rows[0].DraftConfigKind != "null" { + t.Fatalf("preimage must retain the JSON-null literal: %+v", rows) + } +} + +func TestDraftCASConcurrentSameVersionWritesHaveOneWinnerTwoConnections(t *testing.T) { + f := newDraftFixture(t) + f.migrate(t) + storeA, pidA := f.newPinnedStore(t) + storeB, pidB := f.newPinnedStore(t) + _, witnessPID := f.newPinnedWitness(t) + requireDistinctBackends(t, "concurrent same-version race", pidA, pidB, witnessPID) + + seed := seedDraftOwner(t, f, storeA, "proj_race", "patients") + + type outcome struct { + suffix string + saved *explorer.Explorer + err error + } + start := make(chan struct{}) + results := make(chan outcome, 2) + for i, store := range []*Store{storeA, storeB} { + suffix := fmt.Sprintf("w%d", i) + candidate := draftCandidate(seed, suffix, "sha256:draft-"+suffix) + go func(store *Store, suffix string, candidate explorer.Explorer) { + <-start + saved, err := store.SaveDraft(f.ctx, candidate, 4, "sha256:old-draft") + results <- outcome{suffix: suffix, saved: saved, err: err} + }(store, suffix, candidate) + } + close(start) + + wins, winner, loserErrs := 0, outcome{}, []error{} + for range 2 { + r := <-results + if r.err == nil { + wins++ + winner = r + } else { + if !errors.Is(r.err, explorer.ErrDraftConflict) { + t.Fatalf("loser error = %v, want explorer.ErrDraftConflict", r.err) + } + loserErrs = append(loserErrs, r.err) + } + } + if wins != 1 { + t.Fatalf("same version/digest produced %d winners, want exactly 1", wins) + } + + head := f.literalRow(t, "proj_race", "patients") + if head.DraftVersion != 5 { + t.Fatalf("head draft_version = %d, want 5: two same-version CASes must increment once", head.DraftVersion) + } + if head.LastAuthoringCommandID != "cmd-"+winner.suffix || head.DraftDigest != "sha256:draft-"+winner.suffix { + t.Fatalf("head belongs to %q instead of the winner %q: %+v", head.LastAuthoringCommandID, winner.suffix, head) + } + if head.ActiveRevisionID != "published-revision-9" { + t.Fatalf("race overwrote the published head: %+v", head) + } + if head.PreviousDraftRevisionID != draftRevisionKey("proj_race", "patients", 4) { + t.Fatalf("winner did not link the preimage: %+v", head) + } + + rows := f.historyRows(t, "proj_race", "patients") + if len(rows) != 1 { + t.Fatalf("history rows after the race = %d, want exactly 1", len(rows)) + } + requireExactPreimageRow(t, rows[0], "proj_race", "patients", 4) +} + +func TestDraftCASStaleWriterMapsToConflictWithPreservedDuplicateKeyCause(t *testing.T) { + f := newDraftFixture(t) + f.migrate(t) + storeA, pidA := f.newPinnedStore(t) + storeB, pidB := f.newPinnedStore(t) + requireDistinctBackends(t, "stale-writer cause retention", pidA, pidB) + + seed := seedDraftOwner(t, f, storeA, "proj_cause", "patients") + + inserted := make(chan struct{}) + releaseA := make(chan struct{}) + released := false + release := func() { + if !released { + released = true + close(releaseA) + } + } + t.Cleanup(release) + storeA.afterHistoryInsert = func(ctx context.Context, _ pgx.Tx) error { + close(inserted) + select { + case <-releaseA: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + + type outcome struct { + suffix string + saved *explorer.Explorer + err error + } + resultsA := make(chan outcome, 1) + go func() { + saved, err := storeA.SaveDraft(f.ctx, draftCandidate(seed, "first", "sha256:first"), 4, "sha256:old-draft") + resultsA <- outcome{suffix: "first", saved: saved, err: err} + }() + + select { + case <-inserted: + case <-time.After(30 * time.Second): + release() + t.Fatal("winner writer never reached the post-insert seam") + } + + resultsB := make(chan outcome, 1) + go func() { + saved, err := storeB.SaveDraft(f.ctx, draftCandidate(seed, "stale", "sha256:stale"), 4, "sha256:old-draft") + resultsB <- outcome{suffix: "stale", saved: saved, err: err} + }() + + if !waitForLockWait(f, pidB) { + release() + t.Fatalf("stale writer backend %d never entered a PostgreSQL lock wait behind the uncommitted preimage", pidB) + } + release() + + resA := <-resultsA + if resA.err != nil || resA.saved == nil { + t.Fatalf("first writer must win: %v", resA.err) + } + resB := <-resultsB + if resB.err == nil || resB.saved != nil { + t.Fatalf("stale writer must lose, got saved=%v err=%v", resB.saved, resB.err) + } + if !errors.Is(resB.err, explorer.ErrDraftConflict) { + t.Fatalf("stale writer error = %v, want explorer.ErrDraftConflict", resB.err) + } + var pgErr *pgconn.PgError + if !errors.As(resB.err, &pgErr) { + t.Fatalf("stale writer lost the PostgreSQL cause: %v (%T)", resB.err, resB.err) + } + if pgErr.Code != "23505" || pgErr.ConstraintName != historyIdentityConstraint { + t.Fatalf("stale writer cause = %s/%s, want %s/%s (constraint %q)", pgErr.Code, pgErr.ConstraintName, + "23505", "unique_violation", historyIdentityConstraint) + } + + head := f.literalRow(t, "proj_cause", "patients") + if head.DraftVersion != 5 || head.LastAuthoringCommandID != "cmd-first" { + t.Fatalf("loser changed the head: %+v", head) + } + if rows := f.historyRows(t, "proj_cause", "patients"); len(rows) != 1 || rows[0].Title != "Patients" { + t.Fatalf("loser left history behind: %+v", rows) + } +} + +func waitForLockWait(f *ownerFixture, pid int64) bool { + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + var waiting int + if err := f.admin.QueryRow(f.ctx, + `SELECT count(*) FROM pg_stat_activity WHERE pid = $1 AND state = 'active' AND wait_event_type = 'Lock'`, + pid).Scan(&waiting); err == nil && waiting == 1 { + return true + } + time.Sleep(20 * time.Millisecond) + } + return false +} + +func TestDraftCASStaleRetryAfterWinnerCommitIsRejected(t *testing.T) { + f := newDraftFixture(t) + f.migrate(t) + winnerStore, winnerPID := f.newPinnedStore(t) + staleStore, stalePID := f.newPinnedStore(t) + requireDistinctBackends(t, "stale retry after committed winner", winnerPID, stalePID) + + seed := seedDraftOwner(t, f, winnerStore, "proj_retry", "patients") + if _, err := winnerStore.SaveDraft(f.ctx, draftCandidate(seed, "won", "sha256:won"), 4, "sha256:old-draft"); err != nil { + f.fail("winner save", err) + } + head := f.literalRow(t, "proj_retry", "patients") + if head.DraftVersion != 5 { + t.Fatalf("head after winner = %+v, want draft_version 5", head) + } + if rows := f.historyCount(t, "proj_retry", "patients"); rows != 1 { + t.Fatalf("history after winner = %d, want 1", rows) + } + + stale := draftCandidate(seed, "late", "sha256:late") + if _, err := staleStore.SaveDraft(f.ctx, stale, 4, "sha256:old-draft"); !errors.Is(err, explorer.ErrDraftConflict) { + t.Fatalf("stale retry error = %v, want ErrDraftConflict", err) + } + if _, err := staleStore.SaveDraft(f.ctx, stale, 4); !errors.Is(err, explorer.ErrDraftConflict) { + t.Fatalf("stale version without digest error = %v, want ErrDraftConflict", err) + } + + after := f.literalRow(t, "proj_retry", "patients") + if after != head { + t.Fatalf("rejected stale retry changed the head:\n got %+v\nwant %+v", after, head) + } + if rows := f.historyCount(t, "proj_retry", "patients"); rows != 1 { + t.Fatalf("rejected stale retries persisted history: %d rows", rows) + } + if rows := f.historyRows(t, "proj_retry", "patients"); rows[0].Title != "Patients" { + t.Fatalf("rejected stale retries rewrote the preimage: %+v", rows[0]) + } +} + +func TestDraftCASRollsBackHistoryWhenOwnerCASFailsAfterInsert(t *testing.T) { + f := newDraftFixture(t) + f.migrate(t) + writerStore, writerPID := f.newPinnedStore(t) + takeover, takeoverPID := f.newPinnedWitness(t) + requireDistinctBackends(t, "CAS fails after insert", writerPID, takeoverPID) + + seed := seedDraftOwner(t, f, writerStore, "proj_roll", "patients") + + inserted := make(chan struct{}) + bumped := make(chan struct{}) + released := false + release := func() { + if !released { + released = true + close(bumped) + } + } + t.Cleanup(release) + writerStore.afterHistoryInsert = func(ctx context.Context, _ pgx.Tx) error { + close(inserted) + select { + case <-bumped: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + + type outcome struct { + saved *explorer.Explorer + err error + } + results := make(chan outcome, 1) + go func() { + saved, err := writerStore.SaveDraft(f.ctx, draftCandidate(seed, "phantom", "sha256:phantom"), 4, "sha256:old-draft") + results <- outcome{saved: saved, err: err} + }() + + select { + case <-inserted: + case <-time.After(30 * time.Second): + release() + t.Fatal("writer never reached the post-insert seam") + } + + takeoverConn, err := takeover.Acquire(f.ctx) + if err != nil { + release() + f.fail("acquire takeover connection", err) + } + tag, execErr := takeoverConn.Exec(f.ctx, + `UPDATE `+f.ownerTable()+` SET draft_version = 5, draft_digest = 'sha256:taken-over'`+ + ` WHERE project = $1 AND explorer_id = $2 AND draft_version = 4`, + "proj_roll", "patients") + takeoverConn.Release() + if execErr != nil || tag.RowsAffected() != 1 { + release() + f.fail("takeover bump", fmt.Errorf("rows=%d err=%w", tag.RowsAffected(), execErr)) + } + release() + + res := <-results + if res.err == nil || !errors.Is(res.err, explorer.ErrDraftConflict) { + t.Fatalf("post-insert CAS failure error = %v, want ErrDraftConflict", res.err) + } + if res.saved != nil { + t.Fatalf("failed CAS returned a head: %+v", res.saved) + } + if rows := f.historyRows(t, "proj_roll", "patients"); len(rows) != 0 { + t.Fatalf("aborted writer left history behind: %+v", rows) + } + head := f.literalRow(t, "proj_roll", "patients") + if head.DraftVersion != 5 || head.DraftDigest != "sha256:taken-over" || + head.PreviousDraftRevisionID != "" || head.LastAuthoringCommandID != "seed-cmd" { + t.Fatalf("aborted writer changed the head: %+v", head) + } +} + +func TestDraftCASRollsBackHistoryOnFaultAfterInsert(t *testing.T) { + f := newDraftFixture(t) + f.migrate(t) + writerStore, writerPID := f.newPinnedStore(t) + _, witnessPID := f.newPinnedWitness(t) + requireDistinctBackends(t, "fault after insert", writerPID, witnessPID) + + seed := seedDraftOwner(t, f, writerStore, "proj_fault", "patients") + before := f.literalRow(t, "proj_fault", "patients") + + injected := errors.New("controlled transport failure after history insert") + writerStore.afterHistoryInsert = func(context.Context, pgx.Tx) error { + return injected + } + + saved, err := writerStore.SaveDraft(f.ctx, draftCandidate(seed, "ghost", "sha256:ghost"), 4, "sha256:old-draft") + if !errors.Is(err, injected) { + t.Fatalf("fault error = %v, want the injected cause retained", err) + } + if saved != nil { + t.Fatalf("faulted save returned a head: %+v", saved) + } + if rows := f.historyCount(t, "proj_fault", "patients"); rows != 0 { + t.Fatalf("fault after history insert left %d history rows", rows) + } + if after := f.literalRow(t, "proj_fault", "patients"); after != before { + t.Fatalf("fault after history insert changed the head:\n got %+v\nwant %+v", after, before) + } +} + +func TestDraftCASRevisionMonotonicityAcrossConnections(t *testing.T) { + f := newDraftFixture(t) + f.migrate(t) + writer, writerPID := f.newPinnedStore(t) + readerStore, readerPID := f.newPinnedStore(t) + requireDistinctBackends(t, "monotonic revisions", writerPID, readerPID) + + seed := seedDraftOwner(t, f, writer, "proj_mono", "patients") + + digest := "sha256:old-draft" + title := "Patients" + for i := 0; i < 3; i++ { + expected := int64(4 + i) + nextDigest := fmt.Sprintf("sha256:v%d", 5+i) + suffix := fmt.Sprintf("step%d", i) + candidate := draftCandidate(seed, suffix, nextDigest) + candidate.Title = title + "/" + suffix + stored, err := writer.SaveDraft(f.ctx, candidate, expected, digest) + if err != nil { + f.fail(fmt.Sprintf("monotonic save %d", i), err) + } + if stored.DraftVersion != expected+1 { + t.Fatalf("save %d: draftVersion = %d, want exactly %d (monotonic +1)", i, stored.DraftVersion, expected+1) + } + + head, err := readerStore.Get(f.ctx, "proj_mono", "patients") + if err != nil { + f.fail("monotonic head read", err) + } + if head.DraftVersion != expected+1 || head.DraftDigest != nextDigest { + t.Fatalf("save %d: cross-backend head = %+v", i, head) + } + wantPrev := draftRevisionKey("proj_mono", "patients", expected) + if head.PreviousDraftRevisionID != wantPrev { + t.Fatalf("save %d: previous id = %q, want %q", i, head.PreviousDraftRevisionID, wantPrev) + } + revision, err := readerStore.GetDraftRevision(f.ctx, "proj_mono", "patients", wantPrev) + if err != nil { + f.fail("monotonic revision read", err) + } + if revision.DraftVersion != expected || revision.DraftDigest != digest || revision.Title != title { + t.Fatalf("save %d: preimage = %+v, want version %d digest %q title %q", + i, revision, expected, digest, title) + } + digest = nextDigest + title = candidate.Title + } + + rows := f.historyRows(t, "proj_mono", "patients") + if len(rows) != 3 { + t.Fatalf("history rows = %d, want 3", len(rows)) + } + for i, r := range rows { + if r.DraftVersion != int64(4+i) { + t.Fatalf("history[%d].draft_version = %d, want %d (gap-free, ordered)", i, r.DraftVersion, 4+i) + } + if r.ID != draftRevisionKey("proj_mono", "patients", int64(4+i)) { + t.Fatalf("history[%d].id = %q, want derived key", i, r.ID) + } + wantAuthor := "bob" + if i == 0 { + wantAuthor = "alice" + } + if r.UpdatedBy != wantAuthor { + t.Fatalf("history[%d].updated_by = %q, want the stored-state author %q", i, r.UpdatedBy, wantAuthor) + } + } + var future int + if err := f.iso.QueryRow(f.ctx, `SELECT count(*) FROM `+f.historyTable()+ + ` WHERE project = $1 AND explorer_id = $2 AND draft_version >= 7`, "proj_mono", "patients").Scan(&future); err != nil { + f.fail("probe for future history", err) + } + if future != 0 { + t.Fatalf("%d history rows beyond the last preimage, want 0", future) + } + authors, err := f.iso.Query(f.ctx, `SELECT DISTINCT updated_by FROM `+f.historyTable()+ + ` WHERE project = $1 AND explorer_id = $2 ORDER BY updated_by`, "proj_mono", "patients") + if err != nil { + f.fail("read history authors", err) + } + defer authors.Close() + seen := []string{} + for authors.Next() { + var by string + if err := authors.Scan(&by); err != nil { + f.fail("scan history author", err) + } + seen = append(seen, by) + } + if err := authors.Err(); err != nil { + f.fail("read history authors", err) + } + if len(seen) != 2 || seen[0] != "alice" || seen[1] != "bob" { + t.Fatalf("preimage authors = %v, want exactly {alice, bob}: the seed's author plus the repeated CAS owner, each on their own immutable rows", seen) + } +} + +func TestDraftCASGetDraftRevisionScopesOpaqueIDsToProjectExplorer(t *testing.T) { + f := newDraftFixture(t) + f.migrate(t) + writer, writerPID := f.newPinnedStore(t) + stranger, strangerPID := f.newPinnedStore(t) + requireDistinctBackends(t, "opaque id scoping", writerPID, strangerPID) + + seedA := seedDraftOwner(t, f, writer, "proj_a", "patients") + seedB := seedDraftOwner(t, f, writer, "proj_b", "patients") + seedBY := seedDraftOwner(t, f, writer, "proj_b", "labs") + for _, pair := range []struct { + seed explorer.Explorer + id string + }{{seedA, "proj_a"}, {seedB, "proj_b"}, {seedBY, "proj_b"}} { + if _, err := writer.SaveDraft(f.ctx, draftCandidate(pair.seed, "rev", "sha256:rev-"+pair.id), 4, "sha256:old-draft"); err != nil { + f.fail("seed history", err) + } + } + idA := draftRevisionKey("proj_a", "patients", 4) + idB := draftRevisionKey("proj_b", "patients", 4) + if idA == idB { + t.Fatal("different projects derived the same revision id") + } + + rev, err := stranger.GetDraftRevision(f.ctx, "proj_a", "patients", idA) + if err != nil || rev.ID != idA { + f.fail("legitimate scoped read", err) + } + + if _, err := stranger.GetDraftRevision(f.ctx, "proj_b", "patients", idA); !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("cross-project read of %q = %v, want concealed ErrNotFound", idA, err) + } + if _, err := stranger.GetDraftRevision(f.ctx, "proj_b", "labs", idB); !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("cross-explorer read of %q = %v, want concealed ErrNotFound", idB, err) + } + if _, err := stranger.GetDraftRevision(f.ctx, "proj_a", "patients", "draft_revision_"+strings.Repeat("0", 64)); !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("nonexistent id = %v, want concealed ErrNotFound", err) + } + for _, blank := range []struct{ project, explorerID, revisionID string }{ + {"", "patients", idA}, {"proj_a", "", idA}, {"proj_a", "patients", ""}, + {" ", "patients", idA}, + } { + if _, err := stranger.GetDraftRevision(f.ctx, blank.project, blank.explorerID, blank.revisionID); !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("blank part %#v = %v, want ErrNotFound", blank, err) + } + } + + for _, pair := range []struct { + project, explorerID string + }{ + {"proj_a", "patients"}, {"proj_b", "patients"}, {"proj_b", "labs"}, + } { + if n := f.historyCount(t, pair.project, pair.explorerID); n != 1 { + t.Fatalf("history for %s/%s = %d rows, want the preimage intact at 1", pair.project, pair.explorerID, n) + } + } +} + +func TestZZZeroLeftoverPgStateExplorerDraftCasFixtures(t *testing.T) { + raw := testPostgresURL(t) + draftCASOwnedMu.Lock() + owned := make(map[string]string, len(draftCASOwnedDBs)) + for db, schema := range draftCASOwnedDBs { + owned[db] = schema + } + draftCASOwnedMu.Unlock() + if len(owned) == 0 { + t.Skip("this process created no pgstate01_it_* fixtures") + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := openDraftPool(ctx, raw) + if err != nil { + t.Fatalf("connect to test service for audit: %v", err) + } + defer admin.Close() + + for db, schema := range owned { + if !strings.HasPrefix(db, "pgstate01_it_") { + t.Fatalf("registry holds foreign database %q", db) + } + var count int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Errorf("audit owned database %s: %v", db, err) + continue + } + if count != 0 { + t.Errorf("leftover owned database %s after all cleanups", db) + if schema != "" { + u, err := parseDraftURL(raw) + if err == nil { + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + db + if probe, err := openDraftPool(ctx, u.String()); err == nil { + var schemaCount int + if err := probe.QueryRow(ctx, "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1", schema).Scan(&schemaCount); err != nil { + t.Errorf("audit owned schema %s in %s: %v", schema, db, err) + } else if schemaCount != 0 { + t.Errorf("leftover owned schema %s in leftover database %s", schema, db) + } + probe.Close() + } + } + } + } + } +} + +func parseDraftURL(raw string) (*url.URL, error) { + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + return nil, fmt.Errorf("url is not a postgres:// or postgresql:// URL") + } + return u, nil +} + +func openDraftPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) { + return pgxpool.New(ctx, dsn) +} + +func jsonLikeContains(t *testing.T, raw []byte, needle string) bool { + t.Helper() + if len(raw) == 0 { + return false + } + return strings.Contains(string(raw), needle) +} diff --git a/internal/backends/postgres/internal/explorer/explicit_group_memberships.go b/internal/backends/postgres/internal/explorer/explicit_group_memberships.go new file mode 100644 index 000000000..b7e7eda96 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/explicit_group_memberships.go @@ -0,0 +1,583 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/explorer" +) + +const ( + ExplicitGroupMembershipsTable = "explorer_explicit_group_memberships" + + explicitGroupMembershipPageSize = 1000 +) + +const ( + explicitGroupMembershipPKConstraint = "explorer_explicit_group_memberships_pkey" + explicitGroupMembershipRevisionFK = "explorer_explicit_group_memberships_revision_fkey" + explicitGroupMembershipGroupFK = "explorer_explicit_group_memberships_group_fkey" + explicitGroupRevisionLifecycleValuesCheck = "explorer_explicit_group_revisions_lifecycle_values" +) + +var _ explorer.ExplicitGroupRepository = (*ExplicitGroupStore)(nil) + +var explicitGroupMembershipColumns = []string{ + "revision_id", + "group_id", + "project", + "generation", + "resource_type", + "resource_id", +} + +var explicitGroupMembershipScanColumns = explicitGroupMembershipColumns[1:] + +type explicitGroupMembershipStatements struct { + memberInsert string + memberGet string + memberPage string + memberDeleteRevision string + memberDeleteBatch string + sourceSelection string + sourceMembers string + completeCAS string +} + +func newExplicitGroupMembershipStatements(schema string) explicitGroupMembershipStatements { + members := sanitizeTableRef(schema, ExplicitGroupMembershipsTable) + headers := sanitizeTableRef(schema, ExplicitGroupRevisionsTable) + selHeaders := sanitizeTableRef(schema, SelectionRevisionsTable) + selMembers := sanitizeTableRef(schema, SelectionMembersTable) + mcols := strings.Join(explicitGroupMembershipColumns, ", ") + scols := strings.Join(explicitGroupMembershipScanColumns, ", ") + hcols := strings.Join(explicitGroupColumns, ", ") + return explicitGroupMembershipStatements{ + memberInsert: "INSERT INTO " + members + " (" + mcols + ")" + + " VALUES (" + orderedPlaceholders(len(explicitGroupMembershipColumns)) + ")" + + " ON CONFLICT (" + mcols + ") DO NOTHING" + + " RETURNING " + scols, + memberGet: "SELECT " + scols + " FROM " + members + + " WHERE revision_id = $1 AND group_id = $2 AND project = $3" + + " AND generation = $4 AND resource_type = $5 AND resource_id = $6", + memberPage: "SELECT " + scols + " FROM " + members + + " WHERE revision_id = $1" + + " AND (group_id > $2 OR (group_id = $2 AND resource_id > $3))" + + " ORDER BY group_id, resource_id" + + " LIMIT $4", + memberDeleteRevision: "DELETE FROM " + members + " WHERE revision_id = $1", + memberDeleteBatch: "DELETE FROM " + members + " WHERE revision_id = ANY($1::text[])", + sourceSelection: "SELECT " + strings.Join(selectionReadColumns, ", ") + " FROM " + selHeaders + + " WHERE id = $1 AND project = $2", + sourceMembers: "SELECT " + strings.Join(memberColumns, ", ") + " FROM " + selMembers + + " WHERE selection_id = $1 AND project = $2 AND member_key = ANY($3::text[])", + completeCAS: "UPDATE " + headers + + " SET state = 'COMPLETE', definition_digest = $2, membership_digest = $3," + + " group_count = $4, member_count = $5, completed_at = $6" + + " WHERE id = $1 AND state = 'STAGING' AND writer_token = $7" + + " RETURNING " + hcols, + } +} + +func membershipArgs(revisionID string, m explorer.ExplicitGroupMembership) []any { + return []any{revisionID, string(m.GroupID), m.Ref.Project, m.Ref.Generation, m.Ref.ResourceType, m.Ref.ID} +} + +func scanMembershipRow(row rowScanner, revisionID string) (explorer.ExplicitGroupMembership, error) { + var ( + groupID, project, generation, resourceType, resourceID string + ) + if err := row.Scan(&groupID, &project, &generation, &resourceType, &resourceID); err != nil { + return explorer.ExplicitGroupMembership{}, err + } + membership := explorer.ExplicitGroupMembership{ + GroupID: explorer.ExplicitGroupID(groupID), + Ref: explorer.ResourceRef{ + Project: project, + Generation: generation, + ResourceType: resourceType, + ID: resourceID, + }, + }.Canonical() + if string(membership.GroupID) != groupID || + membership.Ref.Project != project || membership.Ref.Generation != generation || + membership.Ref.ResourceType != resourceType || membership.Ref.ID != resourceID { + return explorer.ExplicitGroupMembership{}, fmt.Errorf( + "%w: membership %q under group %q of revision %q is stored non-canonically", + explorer.ErrCorruptExplicitGroupRevision, resourceID, groupID, revisionID) + } + return membership, nil +} + +func validateExplicitGroupMembershipRow(membership explorer.ExplicitGroupMembership, revisionID explorer.ExplicitGroupRevisionID, header *explorer.ExplicitGroupRevision, groupIDs map[explorer.ExplicitGroupID]struct{}) error { + if _, declared := groupIDs[membership.GroupID]; !declared { + return fmt.Errorf("%w: membership of revision %q names undeclared group %q", + explorer.ErrCorruptExplicitGroupRevision, string(revisionID), string(membership.GroupID)) + } + if err := membership.Ref.Validate(header.Project, header.Generation, header.ResourceType); err != nil { + return fmt.Errorf("%w: %v", explorer.ErrCorruptExplicitGroupRevision, err) + } + return nil +} + +func (s *ExplicitGroupStore) readExplicitGroupSourceSelection(ctx context.Context, conn explicitGroupConn, header *explorer.ExplicitGroupRevision) (*explorer.SelectionRevision, error) { + stored, err := scanSelection(conn.QueryRow(ctx, s.memberStmts.sourceSelection, + header.SourceSelectionRevisionID, header.Project)) + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrExplicitGroupRevisionStaleSource + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: read explicit-group source selection %q of revision %q: %w", + header.SourceSelectionRevisionID, string(header.ID), err) + } + source := stored.revision + if err := header.ValidateSourceSelection(source); err != nil { + return nil, err + } + return &source, nil +} + +func (s *ExplicitGroupStore) validateExplicitGroupSourceMemberships(ctx context.Context, conn explicitGroupConn, header *explorer.ExplicitGroupRevision, source explorer.SelectionRevision, memberships []explorer.ExplicitGroupMembership) error { + if err := header.ValidateSourceSelection(source); err != nil { + return err + } + for start := 0; start < len(memberships); start += explicitGroupMembershipPageSize { + end := start + explicitGroupMembershipPageSize + if end > len(memberships) { + end = len(memberships) + } + batch := memberships[start:end] + keys := make([]string, 0, len(batch)) + want := make(map[string]explorer.ResourceRef, len(batch)) + for _, membership := range batch { + membership = membership.Canonical() + key := selectionMemberKey(header.SourceSelectionRevisionID, membership.Ref) + keys = append(keys, key) + want[key] = membership.Ref + } + found := make(map[string]bool, len(keys)) + rows, err := conn.Query(ctx, s.memberStmts.sourceMembers, + header.SourceSelectionRevisionID, header.Project, keys) + if err != nil { + return fmt.Errorf("postgres explorer store: read source members for revision %q: %w", + string(header.ID), err) + } + for rows.Next() { + var ( + memberKey, mProject, mGeneration, mResourceType, mID string + ordinal int64 + ) + if err := rows.Scan(&memberKey, &mProject, &mGeneration, &mResourceType, &mID, &ordinal); err != nil { + rows.Close() + return fmt.Errorf("postgres explorer store: scan source member for revision %q: %w", + string(header.ID), err) + } + member := explorer.SelectionMember{ + MemberKey: memberKey, + Ref: explorer.ResourceRef{ + Project: mProject, + Generation: mGeneration, + ResourceType: mResourceType, + ID: mID, + }, + } + ref, ok := want[memberKey] + if !ok || member.Ref.Canonical() != ref || member.MemberKey != memberKey { + rows.Close() + return explorer.ErrCorruptExplicitGroupRevision + } + if err := member.Validate(header.Project, header.Generation, header.ResourceType); err != nil { + rows.Close() + return err + } + found[memberKey] = true + } + if err := rows.Err(); err != nil { + rows.Close() + return fmt.Errorf("postgres explorer store: read source members for revision %q: %w", + string(header.ID), err) + } + rows.Close() + for key := range want { + if !found[key] { + return explorer.ErrExplicitGroupMemberNotInSelection + } + } + } + return nil +} + +func (s *ExplicitGroupStore) digestExplicitGroupChildren(ctx context.Context, conn explicitGroupConn, domain *explorer.ExplicitGroupRevision, stored *storedExplicitGroupHeader) (string, string, int64, int64, error) { + groups, err := s.listExplicitGroupDefinitions(ctx, conn, string(domain.ID), stored) + if err != nil { + return "", "", 0, 0, err + } + definitionDigest, err := explorer.ExplicitGroupDefinitionDigest(groups) + if err != nil { + return "", "", 0, 0, err + } + groupIDs := make(map[explorer.ExplicitGroupID]struct{}, len(groups)) + for _, group := range groups { + groupIDs[group.ID] = struct{}{} + } + source, err := s.readExplicitGroupSourceSelection(ctx, conn, domain) + if err != nil { + return "", "", 0, 0, err + } + hash := sha256.New() + explorer.StartExplicitGroupMembershipDigest(hash) + var memberCount int64 + afterGroupID, afterResourceID := "", "" + for { + pageCount := 0 + var page []explorer.ExplicitGroupMembership + rows, err := conn.Query(ctx, s.memberStmts.memberPage, + string(domain.ID), afterGroupID, afterResourceID, explicitGroupMembershipPageSize) + if err != nil { + return "", "", 0, 0, fmt.Errorf("postgres explorer store: stream explicit-group memberships %q: %w", + string(domain.ID), err) + } + for rows.Next() { + membership, err := scanMembershipRow(rows, string(domain.ID)) + if err != nil { + rows.Close() + return "", "", 0, 0, fmt.Errorf("postgres explorer store: scan explicit-group membership %q: %w", + string(domain.ID), err) + } + if err := validateExplicitGroupMembershipRow(membership, domain.ID, domain, groupIDs); err != nil { + rows.Close() + return "", "", 0, 0, err + } + explorer.WriteExplicitGroupMembershipFrame(hash, membership) + memberCount++ + pageCount++ + page = append(page, membership) + afterGroupID, afterResourceID = string(membership.GroupID), membership.Ref.ID + } + if err := rows.Err(); err != nil { + rows.Close() + return "", "", 0, 0, fmt.Errorf("postgres explorer store: stream explicit-group memberships %q: %w", + string(domain.ID), err) + } + rows.Close() + if err := ctx.Err(); err != nil { + return "", "", 0, 0, err + } + if s.membershipPageStreamed != nil { + s.membershipPageStreamed(pageCount) + } + if len(page) > 0 { + if err := s.validateExplicitGroupSourceMemberships(ctx, conn, domain, *source, page); err != nil { + return "", "", 0, 0, err + } + } + if pageCount < explicitGroupMembershipPageSize { + break + } + } + return definitionDigest, hex.EncodeToString(hash.Sum(nil)), int64(len(groups)), memberCount, nil +} + +func (s *ExplicitGroupStore) AppendExplicitGroupMemberships(ctx context.Context, revisionID explorer.ExplicitGroupRevisionID, writerToken string, memberships []explorer.ExplicitGroupMembership) ([]explorer.ExplicitGroupMembership, error) { + revisionID = explorer.ExplicitGroupRevisionID(strings.TrimSpace(string(revisionID))) + writerToken = strings.TrimSpace(writerToken) + if revisionID == "" || writerToken == "" { + return nil, explorer.ErrExplicitGroupRevisionConflict + } + if len(memberships) == 0 { + return nil, nil + } + + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire append-memberships connection: %w", err) + } + defer conn.Release() + + tx, err := conn.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: begin append memberships %q: %w", revisionID, err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var header storedExplicitGroupHeader + err = scanExplicitGroupHeader(tx.QueryRow(ctx, s.stmts.headerClaim, string(revisionID)), &header) + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrExplicitGroupRevisionNotFound + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: lock explicit-group header %q: %w", revisionID, err) + } + if err := guardExplicitGroupWriter(&header, writerToken); err != nil { + return nil, err + } + domain, err := header.toDomain(revisionID) + if err != nil { + return nil, err + } + groups, err := s.listExplicitGroupDefinitions(ctx, tx, string(revisionID), &header) + if err != nil { + return nil, err + } + canonical, err := explorer.CanonicalExplicitGroupMemberships(*domain, groups, memberships) + if err != nil { + return nil, err + } + source, err := s.readExplicitGroupSourceSelection(ctx, tx, domain) + if err != nil { + return nil, err + } + if err := s.validateExplicitGroupSourceMemberships(ctx, tx, domain, *source, canonical); err != nil { + return nil, err + } + + inserted := make([]explorer.ExplicitGroupMembership, 0, len(canonical)) + for _, membership := range canonical { + var ( + groupID, project, generation, resourceType, resourceID string + ) + err := tx.QueryRow(ctx, s.memberStmts.memberInsert, + membershipArgs(string(revisionID), membership)...). + Scan(&groupID, &project, &generation, &resourceType, &resourceID) + if errors.Is(err, pgx.ErrNoRows) { + var ( + sGroupID, sProject, sGeneration, sResourceType, sResourceID string + ) + scanErr := tx.QueryRow(ctx, s.memberStmts.memberGet, + string(revisionID), string(membership.GroupID), membership.Ref.Project, + membership.Ref.Generation, membership.Ref.ResourceType, membership.Ref.ID). + Scan(&sGroupID, &sProject, &sGeneration, &sResourceType, &sResourceID) + if errors.Is(scanErr, pgx.ErrNoRows) || + (scanErr == nil && (sGroupID != string(membership.GroupID) || + sProject != membership.Ref.Project || sGeneration != membership.Ref.Generation || + sResourceType != membership.Ref.ResourceType || sResourceID != membership.Ref.ID)) { + return nil, explorer.ErrExplicitGroupRevisionConflict + } + if scanErr != nil { + return nil, fmt.Errorf("postgres explorer store: reread staged membership of %q under group %q: %w", + revisionID, string(membership.GroupID), scanErr) + } + continue + } + if err != nil { + if pgstore.IsUniqueConstraintViolation(err, explicitGroupMembershipPKConstraint) { + return nil, errors.Join(explorer.ErrExplicitGroupRevisionConflict, err) + } + if pgstore.IsForeignKeyConstraintViolation(err, explicitGroupMembershipGroupFK, explicitGroupMembershipRevisionFK) { + return nil, errors.Join(explorer.ErrExplicitGroupRevisionConflict, err) + } + return nil, fmt.Errorf("postgres explorer store: stage explicit-group membership of %q under group %q: %w", + revisionID, string(membership.GroupID), err) + } + if groupID != string(membership.GroupID) || project != membership.Ref.Project || + generation != membership.Ref.Generation || resourceType != membership.Ref.ResourceType || + resourceID != membership.Ref.ID { + return nil, fmt.Errorf("postgres explorer store: staged membership of %q under group %q reread mismatch", + revisionID, string(membership.GroupID)) + } + inserted = append(inserted, membership) + } + + if s.afterMembershipsInserted != nil { + if err := s.afterMembershipsInserted(ctx, tx); err != nil { + return nil, fmt.Errorf("postgres explorer store: append memberships %q after membership insert: %w", + revisionID, err) + } + } + + if err := s.renewExplicitGroupWriter(ctx, tx, revisionID, writerToken); err != nil { + return nil, err + } + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("postgres explorer store: commit append memberships %q: %w", revisionID, err) + } + return inserted, nil +} + +func (s *ExplicitGroupStore) DigestExplicitGroupRevision(ctx context.Context, project string, revisionID explorer.ExplicitGroupRevisionID) (string, string, int64, int64, error) { + if strings.TrimSpace(string(revisionID)) == "" { + return "", "", 0, 0, explorer.ErrExplicitGroupRevisionNotFound + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return "", "", 0, 0, fmt.Errorf("postgres explorer store: acquire digest-memberships connection: %w", err) + } + defer conn.Release() + + stored, err := s.readHeaderUnlocked(ctx, conn, project, revisionID) + if err != nil { + return "", "", 0, 0, err + } + domain, err := stored.toDomain(revisionID) + if err != nil { + return "", "", 0, 0, err + } + if domain.State == explorer.ExplicitGroupRevisionComplete { + return domain.DefinitionDigest, domain.MembershipDigest, domain.GroupCount, domain.MemberCount, nil + } + return s.digestExplicitGroupChildren(ctx, conn, domain, stored) +} + +func (s *ExplicitGroupStore) CompleteExplicitGroupRevision(ctx context.Context, revisionID explorer.ExplicitGroupRevisionID, writerToken, definitionDigest, membershipDigest string, groupCount, memberCount int64, completedAt time.Time) (*explorer.ExplicitGroupRevision, error) { + revisionID = explorer.ExplicitGroupRevisionID(strings.TrimSpace(string(revisionID))) + writerToken = strings.TrimSpace(writerToken) + definitionDigest, membershipDigest = strings.TrimSpace(definitionDigest), strings.TrimSpace(membershipDigest) + if revisionID == "" || writerToken == "" || definitionDigest == "" || membershipDigest == "" || + groupCount <= 0 || memberCount < 0 { + return nil, explorer.ErrExplicitGroupRevisionIncomplete + } + if completedAt.IsZero() { + completedAt = time.Now().UTC() + } + + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire complete-revision connection: %w", err) + } + defer conn.Release() + + tx, err := conn.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: begin complete revision %q: %w", revisionID, err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var header storedExplicitGroupHeader + err = scanExplicitGroupHeader(tx.QueryRow(ctx, s.stmts.headerClaim, string(revisionID)), &header) + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrExplicitGroupRevisionNotFound + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: lock explicit-group header %q for complete: %w", revisionID, err) + } + domain, err := header.toDomain(revisionID) + if err != nil { + return nil, err + } + if domain.State == explorer.ExplicitGroupRevisionComplete { + if domain.DefinitionDigest != definitionDigest || domain.MembershipDigest != membershipDigest || + domain.GroupCount != groupCount || domain.MemberCount != memberCount { + return nil, explorer.ErrExplicitGroupRevisionConflict + } + return domain, nil + } + if err := guardExplicitGroupWriter(&header, writerToken); err != nil { + return nil, err + } + + actualDefinition, actualMembership, actualGroups, actualMembers, err := + s.digestExplicitGroupChildren(ctx, tx, domain, &header) + if err != nil { + return nil, err + } + if actualDefinition != definitionDigest || actualMembership != membershipDigest || + actualGroups != groupCount || actualMembers != memberCount { + return nil, explorer.ErrExplicitGroupRevisionConflict + } + + if _, err := tx.Exec(ctx, s.stmts.setWriterTokenGUC, writerToken); err != nil { + return nil, fmt.Errorf("postgres explorer store: set writer token guard for complete %q: %w", revisionID, err) + } + var completed storedExplicitGroupHeader + err = scanExplicitGroupHeader(tx.QueryRow(ctx, s.memberStmts.completeCAS, + string(revisionID), definitionDigest, membershipDigest, groupCount, memberCount, + completedAt.UTC(), writerToken), &completed) + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrExplicitGroupRevisionConflict + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: complete explicit-group revision %q: %w", revisionID, err) + } + completedDomain, err := completed.toDomain(revisionID) + if err != nil { + return nil, err + } + + if s.beforeCompleteCommit != nil { + if err := s.beforeCompleteCommit(ctx, tx); err != nil { + return nil, fmt.Errorf("postgres explorer store: complete explicit-group revision %q before commit: %w", + revisionID, err) + } + } + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("postgres explorer store: commit complete explicit-group revision %q: %w", revisionID, err) + } + return completedDomain, nil +} + +func (s *ExplicitGroupStore) VisitExplicitGroupMemberships(ctx context.Context, project string, revisionID explorer.ExplicitGroupRevisionID, after explorer.ExplicitGroupMembershipCursor, pageSize int, visit func(explorer.ExplicitGroupMembership) error) (explorer.ExplicitGroupMembershipCursor, error) { + if visit == nil { + return after, fmt.Errorf("explicit group membership visitor is required") + } + header, err := s.GetExplicitGroupRevision(ctx, project, revisionID) + if err != nil { + return after, err + } + if pageSize <= 0 { + pageSize = explicitGroupMembershipPageSize + } else if pageSize > explicitGroupMembershipPageSize { + pageSize = explicitGroupMembershipPageSize + } + + conn, err := s.client.Acquire(ctx) + if err != nil { + return after, fmt.Errorf("postgres explorer store: acquire visit-memberships connection: %w", err) + } + defer conn.Release() + + stored, err := s.readHeaderUnlocked(ctx, conn, project, revisionID) + if err != nil { + return after, err + } + groups, err := s.listExplicitGroupDefinitions(ctx, conn, string(revisionID), stored) + if err != nil { + return after, err + } + groupIDs := make(map[explorer.ExplicitGroupID]struct{}, len(groups)) + for _, group := range groups { + groupIDs[group.ID] = struct{}{} + } + + page := make([]explorer.ExplicitGroupMembership, 0, pageSize) + rows, err := conn.Query(ctx, s.memberStmts.memberPage, + string(revisionID), string(after.GroupID), after.ResourceID, pageSize) + if err != nil { + return after, fmt.Errorf("postgres explorer store: page explicit-group memberships %q: %w", revisionID, err) + } + for rows.Next() { + membership, err := scanMembershipRow(rows, string(revisionID)) + if err != nil { + rows.Close() + return after, fmt.Errorf("postgres explorer store: scan explicit-group membership page %q: %w", revisionID, err) + } + if err := validateExplicitGroupMembershipRow(membership, revisionID, header, groupIDs); err != nil { + rows.Close() + return after, err + } + page = append(page, membership) + } + if err := rows.Err(); err != nil { + rows.Close() + return after, fmt.Errorf("postgres explorer store: page explicit-group memberships %q: %w", revisionID, err) + } + rows.Close() + + for _, membership := range page { + if err := ctx.Err(); err != nil { + return after, err + } + if err := visit(membership); err != nil { + return after, err + } + after = explorer.ExplicitGroupMembershipCursor{GroupID: membership.GroupID, ResourceID: membership.Ref.ID} + } + return after, nil +} diff --git a/internal/backends/postgres/internal/explorer/explicit_groups.go b/internal/backends/postgres/internal/explorer/explicit_groups.go new file mode 100644 index 000000000..1ae17aec7 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/explicit_groups.go @@ -0,0 +1,761 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/explorer" +) + +const ( + ExplicitGroupRevisionsTable = "explorer_explicit_group_revisions" + ExplicitGroupDefinitionsTable = "explorer_explicit_group_definitions" +) + +const ( + explicitGroupWriterLease = 10 * time.Minute + explicitGroupCleanupLimit = 100 + explicitGroupListLimit = 100 +) + +const ( + gucWriterToken = "loom.explorer_explicit_group_writer_token" + gucAbortToken = "loom.explorer_explicit_group_abort_token" + gucCleanupNow = "loom.explorer_explicit_group_cleanup_now" +) + +const ( + explicitGroupDefinitionPKConstraint = "explorer_explicit_group_definitions_pkey" + explicitGroupDefinitionOrdinalCK = "explorer_explicit_group_definitions_revision_ordinal_key" + explicitGroupRevisionFKConstraint = "explorer_explicit_group_definitions_revision_fkey" + explicitGroupRevisionIdentityCheck = "explorer_explicit_group_revisions_identity_binding" + explicitGroupRevisionRequiredFieldsCK = "explorer_explicit_group_revisions_required_fields" +) + +var explicitGroupColumns = []string{ + "id", + "project", + "generation", + "scope_digest", + "resource_type", + "source_selection_revision_id", + "source_membership_digest", + "definition_digest", + "membership_digest", + "group_count", + "member_count", + "state", + "idempotency_key", + "created_at", + "completed_at", + "writer_token", + "writer_expires_at", +} + +var explicitGroupDefinitionColumns = []string{ + "revision_id", + "group_id", + "label", + "ordinal", +} + +type explicitGroupStatements struct { + headerInsert string + headerGet string + headerGetScoped string + headerClaim string + headerRenew string + headerAbort string + headerDeleteBatch string + defList string + defDeleteBatch string + cleanupSelect string + listComplete string + setWriterTokenGUC string + setAbortTokenGUC string + setCleanupNowGUC string + defInsertBase string +} + +func newExplicitGroupStatements(schema string) explicitGroupStatements { + headerTable := sanitizeTableRef(schema, ExplicitGroupRevisionsTable) + defTable := sanitizeTableRef(schema, ExplicitGroupDefinitionsTable) + hcols := strings.Join(explicitGroupColumns, ", ") + hph := orderedPlaceholders(len(explicitGroupColumns)) + dcols := strings.Join(explicitGroupDefinitionColumns, ", ") + return explicitGroupStatements{ + headerInsert: "INSERT INTO " + headerTable + " (" + hcols + ") VALUES (" + hph + ") " + + "ON CONFLICT (id) DO NOTHING RETURNING " + hcols, + headerGet: "SELECT " + hcols + " FROM " + headerTable + " WHERE id = $1", + headerGetScoped: "SELECT " + hcols + " FROM " + headerTable + + " WHERE project = $1 AND id = $2", + headerClaim: "SELECT " + hcols + " FROM " + headerTable + " WHERE id = $1 FOR UPDATE", + headerRenew: "UPDATE " + headerTable + " SET writer_expires_at = $2 " + + "WHERE id = $1 AND state = 'STAGING' AND writer_token = $3", + headerAbort: "DELETE FROM " + headerTable + + " WHERE id = $1 AND state = 'STAGING' AND writer_token = $2", + headerDeleteBatch: "DELETE FROM " + headerTable + " WHERE id = ANY($1::text[])", + defList: "SELECT group_id, label, ordinal FROM " + defTable + + " WHERE revision_id = $1 ORDER BY ordinal, group_id", + defDeleteBatch: "DELETE FROM " + defTable + " WHERE revision_id = ANY($1::text[])", + cleanupSelect: "SELECT id FROM " + headerTable + + " WHERE state = 'STAGING' AND created_at < $1" + + " AND (writer_expires_at IS NULL OR writer_expires_at < $2)" + + " ORDER BY created_at, id LIMIT $3 FOR UPDATE SKIP LOCKED", + listComplete: "SELECT " + hcols + " FROM " + headerTable + + " WHERE project = $1 AND generation = $2 AND scope_digest = $3" + + " AND resource_type = $4 AND state = 'COMPLETE'" + + " ORDER BY created_at DESC, id LIMIT $5", + setWriterTokenGUC: "SELECT set_config('" + gucWriterToken + "', $1, true)", + setAbortTokenGUC: "SELECT set_config('" + gucAbortToken + "', $1, true)", + setCleanupNowGUC: "SELECT set_config('" + gucCleanupNow + "', $1, true)", + defInsertBase: "INSERT INTO " + defTable + " (" + dcols + ") VALUES ", + } +} + +func (s explicitGroupStatements) defInsertStatement(n int) string { + rows := make([]string, 0, n) + for i := 0; i < n; i++ { + rows = append(rows, fmt.Sprintf("($%d, $%d, $%d, $%d)", 4*i+1, 4*i+2, 4*i+3, 4*i+4)) + } + return s.defInsertBase + strings.Join(rows, ", ") +} + +type explicitGroupConn interface { + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) +} + +var ( + _ explicitGroupConn = (*pgxpool.Conn)(nil) + _ explicitGroupConn = pgx.Tx(nil) +) + +type ExplicitGroupStore struct { + client *pgstore.Client + stmts explicitGroupStatements + memberStmts explicitGroupMembershipStatements + afterDefinitionsInserted func(context.Context, pgx.Tx) error + afterDefinitionsDeleted func(context.Context, pgx.Tx) error + afterMembershipsInserted func(context.Context, pgx.Tx) error + beforeCompleteCommit func(context.Context, pgx.Tx) error + membershipPageStreamed func(rows int) +} + +func NewExplicitGroupStore(client *pgstore.Client, schema string) (*ExplicitGroupStore, error) { + if client == nil { + return nil, errors.New("postgres explorer store: explicit-group client is required") + } + if !pgstore.ValidSchemaName(schema) { + return nil, fmt.Errorf("postgres explorer store: invalid schema %q: must match [a-z_][a-z0-9_]{0,62}", schema) + } + return &ExplicitGroupStore{ + client: client, + stmts: newExplicitGroupStatements(schema), + memberStmts: newExplicitGroupMembershipStatements(schema), + }, nil +} + +type storedExplicitGroupHeader struct { + ID string + Project string + Generation string + ScopeDigest string + ResourceType string + SourceSelectionRevisionID string + SourceMembershipDigest string + DefinitionDigest string + MembershipDigest string + GroupCount int64 + MemberCount int64 + State string + IdempotencyKey string + CreatedAt time.Time + CompletedAt *time.Time + WriterToken *string + WriterExpiresAt *time.Time +} + +func scanExplicitGroupHeader(row rowScanner, out *storedExplicitGroupHeader) error { + return row.Scan( + &out.ID, + &out.Project, + &out.Generation, + &out.ScopeDigest, + &out.ResourceType, + &out.SourceSelectionRevisionID, + &out.SourceMembershipDigest, + &out.DefinitionDigest, + &out.MembershipDigest, + &out.GroupCount, + &out.MemberCount, + &out.State, + &out.IdempotencyKey, + &out.CreatedAt, + &out.CompletedAt, + &out.WriterToken, + &out.WriterExpiresAt, + ) +} + +func headerArgs(revision explorer.ExplicitGroupRevision, writerToken string, writerExpiresAt time.Time) []any { + return []any{ + string(revision.ID), + revision.Project, + revision.Generation, + revision.ScopeDigest, + revision.ResourceType, + revision.SourceSelectionRevisionID, + revision.SourceMembershipDigest, + "", + "", + int64(0), + int64(0), + string(explorer.ExplicitGroupRevisionStaging), + revision.IdempotencyKey, + revision.CreatedAt, + (*time.Time)(nil), + writerToken, + writerExpiresAt, + } +} + +func (h *storedExplicitGroupHeader) toDomain(expectedID explorer.ExplicitGroupRevisionID) (*explorer.ExplicitGroupRevision, error) { + if expectedID != "" && h.ID != string(expectedID) { + return nil, fmt.Errorf("%w: header stored id %q is filed under %q", + explorer.ErrCorruptExplicitGroupRevision, h.ID, string(expectedID)) + } + revision := explorer.ExplicitGroupRevision{ + ID: explorer.ExplicitGroupRevisionID(h.ID), + Project: h.Project, + Generation: h.Generation, + ScopeDigest: h.ScopeDigest, + ResourceType: h.ResourceType, + SourceSelectionRevisionID: h.SourceSelectionRevisionID, + SourceMembershipDigest: h.SourceMembershipDigest, + DefinitionDigest: h.DefinitionDigest, + MembershipDigest: h.MembershipDigest, + GroupCount: h.GroupCount, + MemberCount: h.MemberCount, + State: explorer.ExplicitGroupRevisionState(h.State), + IdempotencyKey: h.IdempotencyKey, + CreatedAt: h.CreatedAt, + CompletedAt: h.CompletedAt, + }.Canonical() + if err := revision.Validate(); err != nil { + return nil, fmt.Errorf("%w: %v", explorer.ErrCorruptExplicitGroupRevision, err) + } + return &revision, nil +} + +func guardExplicitGroupWriter(header *storedExplicitGroupHeader, writerToken string) error { + if header == nil { + return explorer.ErrExplicitGroupRevisionNotFound + } + if header.State != string(explorer.ExplicitGroupRevisionStaging) { + return explorer.ErrExplicitGroupRevisionConflict + } + stored := "" + if header.WriterToken != nil { + stored = strings.TrimSpace(*header.WriterToken) + } + if stored != writerToken { + return explorer.ErrExplicitGroupRevisionConflict + } + return nil +} + +func (s *ExplicitGroupStore) BeginExplicitGroupRevision(ctx context.Context, revision explorer.ExplicitGroupRevision, source explorer.SelectionRevision, writerToken string) (*explorer.ExplicitGroupRevision, error) { + revision = revision.Canonical() + writerToken = strings.TrimSpace(writerToken) + if writerToken == "" { + return nil, explorer.ErrExplicitGroupRevisionConflict + } + if err := revision.Validate(); err != nil { + return nil, err + } + if err := revision.ValidateSourceSelection(source); err != nil { + return nil, err + } + if revision.State != explorer.ExplicitGroupRevisionStaging { + return nil, fmt.Errorf("explicit group revision must begin in staging state") + } + if err := s.CleanupExplicitGroupStaging(ctx, time.Now().UTC(), explicitGroupCleanupLimit); err != nil { + return nil, err + } + + writerExpiresAt := time.Now().UTC().Add(explicitGroupWriterLease) + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire explicit-group begin connection: %w", err) + } + defer conn.Release() + + tx, err := conn.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: begin explicit-group claim %q: %w", revision.ID, err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var stored storedExplicitGroupHeader + err = scanExplicitGroupHeader(tx.QueryRow(ctx, s.stmts.headerInsert, + headerArgs(revision, writerToken, writerExpiresAt)...), &stored) + if err == nil { + domain, err := stored.toDomain(revision.ID) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: decode claimed header %q: %w", revision.ID, err) + } + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("postgres explorer store: commit explicit-group claim %q: %w", revision.ID, err) + } + return domain, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("postgres explorer store: claim explicit-group header %q: %w", revision.ID, err) + } + + var winner storedExplicitGroupHeader + err = scanExplicitGroupHeader(tx.QueryRow(ctx, s.stmts.headerClaim, string(revision.ID)), &winner) + if errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("%w: conflicting explicit-group claim %q vanished before it became visible", + explorer.ErrExplicitGroupRevisionConflict, revision.ID) + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: read conflicting explicit-group header %q: %w", revision.ID, err) + } + winnerDomain, err := winner.toDomain(revision.ID) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: decode conflicting header %q: %w", revision.ID, err) + } + if !sameExplicitGroupRevisionIntent(*winnerDomain, revision) { + return nil, explorer.ErrExplicitGroupRevisionConflict + } + if winnerDomain.State == explorer.ExplicitGroupRevisionStaging { + if err := guardExplicitGroupWriter(&winner, writerToken); err != nil { + return nil, err + } + renewedAt := time.Now().UTC().Add(explicitGroupWriterLease) + if _, err := tx.Exec(ctx, s.stmts.setWriterTokenGUC, writerToken); err != nil { + return nil, fmt.Errorf("postgres explorer store: set writer token guard for begin %q: %w", revision.ID, err) + } + tag, err := tx.Exec(ctx, s.stmts.headerRenew, string(revision.ID), renewedAt, writerToken) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: renew explicit-group writer lease %q: %w", revision.ID, err) + } + if tag.RowsAffected() != 1 { + return nil, explorer.ErrExplicitGroupRevisionConflict + } + } + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("postgres explorer store: commit explicit-group begin %q: %w", revision.ID, err) + } + return winnerDomain, nil +} + +func sameExplicitGroupRevisionIntent(left, right explorer.ExplicitGroupRevision) bool { + left, right = left.Canonical(), right.Canonical() + return left.ID == right.ID && + left.Project == right.Project && + left.Generation == right.Generation && + left.ScopeDigest == right.ScopeDigest && + left.ResourceType == right.ResourceType && + left.SourceSelectionRevisionID == right.SourceSelectionRevisionID && + left.SourceMembershipDigest == right.SourceMembershipDigest && + left.IdempotencyKey == right.IdempotencyKey +} + +func (s *ExplicitGroupStore) PutExplicitGroupDefinitions(ctx context.Context, revisionID explorer.ExplicitGroupRevisionID, writerToken string, groups []explorer.ExplicitGroupDefinition) error { + revisionID = explorer.ExplicitGroupRevisionID(strings.TrimSpace(string(revisionID))) + writerToken = strings.TrimSpace(writerToken) + if revisionID == "" || writerToken == "" { + return explorer.ErrExplicitGroupRevisionConflict + } + canonical, err := explorer.CanonicalExplicitGroupDefinitions(groups) + if err != nil { + return err + } + + conn, err := s.client.Acquire(ctx) + if err != nil { + return fmt.Errorf("postgres explorer store: acquire explicit-group put connection: %w", err) + } + defer conn.Release() + + tx, err := conn.Begin(ctx) + if err != nil { + return fmt.Errorf("postgres explorer store: begin explicit-group put %q: %w", revisionID, err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var header storedExplicitGroupHeader + err = scanExplicitGroupHeader(tx.QueryRow(ctx, s.stmts.headerClaim, string(revisionID)), &header) + if errors.Is(err, pgx.ErrNoRows) { + return explorer.ErrExplicitGroupRevisionNotFound + } + if err != nil { + return fmt.Errorf("postgres explorer store: lock explicit-group header %q: %w", revisionID, err) + } + if err := guardExplicitGroupWriter(&header, writerToken); err != nil { + return err + } + + stored, err := s.listExplicitGroupDefinitions(ctx, tx, string(revisionID), &header) + if err != nil { + return err + } + if len(stored) > 0 { + if !reflect.DeepEqual(stored, canonical) { + return explorer.ErrExplicitGroupRevisionConflict + } + if err := s.renewExplicitGroupWriter(ctx, tx, revisionID, writerToken); err != nil { + return err + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("postgres explorer store: commit explicit-group put %q: %w", revisionID, err) + } + return nil + } + + if len(canonical) > 0 { + args := make([]any, 0, 4*len(canonical)) + for _, group := range canonical { + args = append(args, string(revisionID), string(group.ID), group.Label, group.Ordinal) + } + if _, err := tx.Exec(ctx, s.stmts.defInsertStatement(len(canonical)), args...); err != nil { + if pgstore.IsUniqueConstraintViolation(err, explicitGroupDefinitionPKConstraint, explicitGroupDefinitionOrdinalCK) { + return errors.Join(explorer.ErrExplicitGroupRevisionConflict, err) + } + return fmt.Errorf("postgres explorer store: insert explicit-group definitions for %q: %w", revisionID, err) + } + + if s.afterDefinitionsInserted != nil { + if err := s.afterDefinitionsInserted(ctx, tx); err != nil { + return fmt.Errorf("postgres explorer store: put explicit-group definitions %q after definition insert: %w", + revisionID, err) + } + } + + verified, err := s.listExplicitGroupDefinitions(ctx, tx, string(revisionID), &header) + if err != nil { + return err + } + if !reflect.DeepEqual(verified, canonical) { + return explorer.ErrExplicitGroupRevisionConflict + } + } + if err := s.renewExplicitGroupWriter(ctx, tx, revisionID, writerToken); err != nil { + return err + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("postgres explorer store: commit explicit-group put %q: %w", revisionID, err) + } + return nil +} + +func (s *ExplicitGroupStore) renewExplicitGroupWriter(ctx context.Context, tx pgx.Tx, revisionID explorer.ExplicitGroupRevisionID, writerToken string) error { + if _, err := tx.Exec(ctx, s.stmts.setWriterTokenGUC, writerToken); err != nil { + return fmt.Errorf("postgres explorer store: set writer token guard for %q: %w", revisionID, err) + } + tag, err := tx.Exec(ctx, s.stmts.headerRenew, string(revisionID), + time.Now().UTC().Add(explicitGroupWriterLease), writerToken) + if err != nil { + return fmt.Errorf("postgres explorer store: renew explicit-group writer lease %q: %w", revisionID, err) + } + if tag.RowsAffected() != 1 { + return explorer.ErrExplicitGroupRevisionConflict + } + return nil +} + +func (s *ExplicitGroupStore) listExplicitGroupDefinitions(ctx context.Context, conn explicitGroupConn, revisionID string, header *storedExplicitGroupHeader) ([]explorer.ExplicitGroupDefinition, error) { + groups := make([]explorer.ExplicitGroupDefinition, 0) + rows, err := conn.Query(ctx, s.stmts.defList, revisionID) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: list explicit-group definitions %q: %w", revisionID, err) + } + defer rows.Close() + for rows.Next() { + var ( + groupID string + label string + ordinal int64 + ) + if err := rows.Scan(&groupID, &label, &ordinal); err != nil { + return nil, fmt.Errorf("postgres explorer store: scan explicit-group definition %q: %w", revisionID, err) + } + group := explorer.ExplicitGroupDefinition{ID: explorer.ExplicitGroupID(groupID), Label: label, Ordinal: ordinal} + canonicalGroup := group.Canonical() + if string(canonicalGroup.ID) != groupID || canonicalGroup.Label != label { + return nil, fmt.Errorf("%w: definition %q of revision %q is stored non-canonically", + explorer.ErrCorruptExplicitGroupRevision, groupID, revisionID) + } + groups = append(groups, group) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("postgres explorer store: list explicit-group definitions %q: %w", revisionID, err) + } + if len(groups) == 0 && header.State == string(explorer.ExplicitGroupRevisionStaging) { + return groups, nil + } + canonical, err := explorer.CanonicalExplicitGroupDefinitions(groups) + if err != nil { + return nil, fmt.Errorf("%w: %v", explorer.ErrCorruptExplicitGroupRevision, err) + } + if !reflect.DeepEqual(canonical, groups) { + return nil, explorer.ErrCorruptExplicitGroupRevision + } + if header.State == string(explorer.ExplicitGroupRevisionComplete) { + digest, err := explorer.ExplicitGroupDefinitionDigest(groups) + if err != nil || int64(len(groups)) != header.GroupCount || digest != header.DefinitionDigest { + return nil, explorer.ErrCorruptExplicitGroupRevision + } + } + return groups, nil +} + +func (s *ExplicitGroupStore) AbortExplicitGroupRevision(ctx context.Context, revisionID explorer.ExplicitGroupRevisionID, writerToken string) error { + revisionID = explorer.ExplicitGroupRevisionID(strings.TrimSpace(string(revisionID))) + if revisionID == "" { + return nil + } + writerToken = strings.TrimSpace(writerToken) + if writerToken == "" { + return explorer.ErrExplicitGroupRevisionConflict + } + + conn, err := s.client.Acquire(ctx) + if err != nil { + return fmt.Errorf("postgres explorer store: acquire explicit-group abort connection: %w", err) + } + defer conn.Release() + + tx, err := conn.Begin(ctx) + if err != nil { + return fmt.Errorf("postgres explorer store: begin explicit-group abort %q: %w", revisionID, err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var header storedExplicitGroupHeader + err = scanExplicitGroupHeader(tx.QueryRow(ctx, s.stmts.headerClaim, string(revisionID)), &header) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return fmt.Errorf("postgres explorer store: lock explicit-group header %q: %w", revisionID, err) + } + if err := guardExplicitGroupWriter(&header, writerToken); err != nil { + return err + } + + if _, err := tx.Exec(ctx, s.stmts.setAbortTokenGUC, writerToken); err != nil { + return fmt.Errorf("postgres explorer store: set abort guard for %q: %w", revisionID, err) + } + if _, err := tx.Exec(ctx, s.memberStmts.memberDeleteRevision, string(revisionID)); err != nil { + return fmt.Errorf("postgres explorer store: delete explicit-group memberships for abort %q: %w", revisionID, err) + } + if _, err := tx.Exec(ctx, s.stmts.defDeleteBatch, []string{string(revisionID)}); err != nil { + return fmt.Errorf("postgres explorer store: delete explicit-group definitions for abort %q: %w", revisionID, err) + } + + if s.afterDefinitionsDeleted != nil { + if err := s.afterDefinitionsDeleted(ctx, tx); err != nil { + return fmt.Errorf("postgres explorer store: abort explicit-group revision %q after definition delete: %w", + revisionID, err) + } + } + + tag, err := tx.Exec(ctx, s.stmts.headerAbort, string(revisionID), writerToken) + if err != nil { + return fmt.Errorf("postgres explorer store: delete explicit-group header %q: %w", revisionID, err) + } + if tag.RowsAffected() != 1 { + return explorer.ErrExplicitGroupRevisionConflict + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("postgres explorer store: commit explicit-group abort %q: %w", revisionID, err) + } + return nil +} + +func (s *ExplicitGroupStore) CleanupExplicitGroupStaging(ctx context.Context, before time.Time, limit int) error { + if limit <= 0 { + limit = explicitGroupCleanupLimit + } + now := time.Now().UTC() + + conn, err := s.client.Acquire(ctx) + if err != nil { + return fmt.Errorf("postgres explorer store: acquire explicit-group cleanup connection: %w", err) + } + defer conn.Release() + + tx, err := conn.Begin(ctx) + if err != nil { + return fmt.Errorf("postgres explorer store: begin explicit-group cleanup: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + if _, err := tx.Exec(ctx, s.stmts.setCleanupNowGUC, now.Format(time.RFC3339Nano)); err != nil { + return fmt.Errorf("postgres explorer store: set cleanup guard: %w", err) + } + rows, err := tx.Query(ctx, s.stmts.cleanupSelect, before, now, limit) + if err != nil { + return fmt.Errorf("postgres explorer store: select expired explicit-group headers: %w", err) + } + var expired []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + rows.Close() + return fmt.Errorf("postgres explorer store: scan expired explicit-group headers: %w", err) + } + expired = append(expired, id) + } + rows.Close() + if err := rows.Err(); err != nil { + return fmt.Errorf("postgres explorer store: select expired explicit-group headers: %w", err) + } + if len(expired) == 0 { + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("postgres explorer store: commit empty explicit-group cleanup: %w", err) + } + return nil + } + if _, err := tx.Exec(ctx, s.memberStmts.memberDeleteBatch, expired); err != nil { + return fmt.Errorf("postgres explorer store: delete memberships of expired explicit-group headers: %w", err) + } + if _, err := tx.Exec(ctx, s.stmts.defDeleteBatch, expired); err != nil { + return fmt.Errorf("postgres explorer store: delete definitions of expired explicit-group headers: %w", err) + } + tag, err := tx.Exec(ctx, s.stmts.headerDeleteBatch, expired) + if err != nil { + return fmt.Errorf("postgres explorer store: delete expired explicit-group headers: %w", err) + } + if int(tag.RowsAffected()) != len(expired) { + return fmt.Errorf("postgres explorer store: cleanup removed %d explicit-group headers, selected %d: %w", + tag.RowsAffected(), len(expired), explorer.ErrExplicitGroupRevisionConflict) + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("postgres explorer store: commit explicit-group cleanup: %w", err) + } + return nil +} + +func (s *ExplicitGroupStore) GetExplicitGroupRevision(ctx context.Context, project string, revisionID explorer.ExplicitGroupRevisionID) (*explorer.ExplicitGroupRevision, error) { + if strings.TrimSpace(string(revisionID)) == "" { + return nil, explorer.ErrExplicitGroupRevisionNotFound + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire explicit-group get connection: %w", err) + } + defer conn.Release() + + header, err := s.readHeaderUnlocked(ctx, conn, project, revisionID) + if err != nil { + return nil, err + } + domain, err := header.toDomain(revisionID) + if err != nil { + return nil, err + } + if domain.State != explorer.ExplicitGroupRevisionComplete { + return nil, explorer.ErrExplicitGroupRevisionIncomplete + } + if _, err := s.listExplicitGroupDefinitions(ctx, conn, string(revisionID), header); err != nil { + return nil, err + } + return domain, nil +} + +func (s *ExplicitGroupStore) ListExplicitGroupRevisions(ctx context.Context, project, generation, scopeDigest, resourceType string, limit int) ([]explorer.ExplicitGroupRevision, error) { + if limit <= 0 { + limit = explicitGroupListLimit + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire explicit-group list connection: %w", err) + } + defer conn.Release() + + rows, err := conn.Query(ctx, s.stmts.listComplete, project, generation, scopeDigest, resourceType, limit) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: list explicit-group revisions %q/%q/%q/%q: %w", + project, generation, scopeDigest, resourceType, err) + } + defer rows.Close() + stored := make([]storedExplicitGroupHeader, 0) + for rows.Next() { + var header storedExplicitGroupHeader + if err := scanExplicitGroupHeader(rows, &header); err != nil { + return nil, fmt.Errorf("postgres explorer store: scan explicit-group revision %q: %w", project, err) + } + stored = append(stored, header) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("postgres explorer store: list explicit-group revisions %q: %w", project, err) + } + rows.Close() + revisions := make([]explorer.ExplicitGroupRevision, 0, len(stored)) + for i := range stored { + domain, err := stored[i].toDomain("") + if err != nil { + return nil, err + } + if _, err := s.listExplicitGroupDefinitions(ctx, conn, stored[i].ID, &stored[i]); err != nil { + return nil, err + } + revisions = append(revisions, *domain) + } + return revisions, nil +} + +func (s *ExplicitGroupStore) ListExplicitGroupDefinitions(ctx context.Context, project string, revisionID explorer.ExplicitGroupRevisionID) ([]explorer.ExplicitGroupDefinition, error) { + if strings.TrimSpace(string(revisionID)) == "" { + return nil, explorer.ErrExplicitGroupRevisionNotFound + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire explicit-group definitions connection: %w", err) + } + defer conn.Release() + stored, err := s.readHeaderUnlocked(ctx, conn, project, revisionID) + if err != nil { + return nil, err + } + domain, err := stored.toDomain(revisionID) + if err != nil { + return nil, err + } + if domain.State != explorer.ExplicitGroupRevisionComplete { + return nil, explorer.ErrExplicitGroupRevisionIncomplete + } + return s.listExplicitGroupDefinitions(ctx, conn, string(revisionID), stored) +} + +func (s *ExplicitGroupStore) readHeaderUnlocked(ctx context.Context, conn explicitGroupConn, project string, revisionID explorer.ExplicitGroupRevisionID) (*storedExplicitGroupHeader, error) { + var header storedExplicitGroupHeader + var err error + if strings.TrimSpace(project) == "" { + err = scanExplicitGroupHeader(conn.QueryRow(ctx, s.stmts.headerGet, string(revisionID)), &header) + } else { + err = scanExplicitGroupHeader(conn.QueryRow(ctx, s.stmts.headerGetScoped, strings.TrimSpace(project), string(revisionID)), &header) + } + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrExplicitGroupRevisionNotFound + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: read explicit-group header %q: %w", revisionID, err) + } + return &header, nil +} diff --git a/internal/backends/postgres/internal/explorer/explicit_groups_readpath_test.go b/internal/backends/postgres/internal/explorer/explicit_groups_readpath_test.go new file mode 100644 index 000000000..bc0e45ada --- /dev/null +++ b/internal/backends/postgres/internal/explorer/explicit_groups_readpath_test.go @@ -0,0 +1,269 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "reflect" + "testing" + "time" + + "github.com/calypr/loom/internal/explorer" + "github.com/jackc/pgx/v5/pgxpool" +) + +const e2rvPrefix = "verify_e2rv" + +const ( + e2rvDefDigestGolden = "0711212b6e8e2eee92a289bec97557775e4435545972b8f21c93f03e532f3321" + e2rvMembershipSetGolden = "639035f95e895c5e581f0a5fe66b0dfd20c241549833edbe10f20c87a5daa922" + e2rvMembershipEmptyGoldn = "51a9413396bfc791022568e5db6dbc5001629854b2c17d7b07dfdb35a820ec09" +) + +func newE2RVFixture(t *testing.T) *grpmFixture { + t.Helper() + return newGRPMFixtureWithPrefix(t, e2rvPrefix) +} + +func e2rvSeedDrift(t *testing.T, f *grpmFixture, step, stmt string, args ...any) { + t.Helper() + f.execOnGRPM(t, f.witness, step+": disable definition guard", + `ALTER TABLE `+f.definitionsTable()+` DISABLE TRIGGER explorer_explicit_group_definitions_guard`) + t.Cleanup(func() { + f.execOnGRPM(t, f.witness, step+": re-enable definition guard (cleanup)", + `ALTER TABLE `+f.definitionsTable()+` ENABLE TRIGGER explorer_explicit_group_definitions_guard`) + }) + f.execOnGRPM(t, f.witness, step, stmt, args...) + f.execOnGRPM(t, f.witness, step+": re-enable definition guard", + `ALTER TABLE `+f.definitionsTable()+` ENABLE TRIGGER explorer_explicit_group_definitions_guard`) +} + +func e2rvCompleteHonest(t *testing.T, f *grpmFixture, revision explorer.ExplicitGroupRevision, token string) { + t.Helper() + def, mem, groups, members, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + if err != nil { + f.fail("e2rv digest", err) + } + if _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, + def, mem, groups, members, time.Now().UTC()); err != nil { + f.fail("e2rv completion", err) + } +} + +func e2rvStagedCompleted(t *testing.T, f *grpmFixture, suffix string, seedMembership bool) (explorer.ExplicitGroupRevision, string) { + t.Helper() + revision, token, _ := f.stagedGroupFixture(t, suffix, "r-a", "r-b") + if seedMembership { + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}); err != nil { + f.fail("e2rv seed append", err) + } + } + e2rvCompleteHonest(t, f, revision, token) + return revision, token +} + +func requireE2RVCorrupt(t *testing.T, err error, step string) { + t.Helper() + if !errors.Is(err, explorer.ErrCorruptExplicitGroupRevision) { + t.Fatalf("%s: err = %v, want errors.Is(ErrCorruptExplicitGroupRevision)", step, err) + } +} + +func TestPostgresExplicitGroupRevisionsReadpathServesHealthyByteIdentical(t *testing.T) { + f := newE2RVFixture(t) + f.migrateWithSelections(t) + + revision, _ := e2rvStagedCompleted(t, f, "healthy", true) + + got, err := f.store.GetExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + if err != nil { + f.fail("healthy get", err) + } + if got.ID != revision.ID || got.GroupCount != 2 || got.MemberCount != 1 || + got.DefinitionDigest != e2rvDefDigestGolden || + got.MembershipDigest != e2rvMembershipSetGolden || + got.State != explorer.ExplicitGroupRevisionComplete || got.CompletedAt == nil { + t.Fatalf("healthy Get served %+v, want the frozen gold header", got) + } + if rederived := grpmFrameDigest(t, [2]string{string(grpmGroupA), "r-a"}); rederived != e2rvMembershipSetGolden { + t.Fatalf("membership golden %s disagrees with independent re-derivation %s", e2rvMembershipSetGolden, rederived) + } + + listed, err := f.store.ListExplicitGroupRevisions(f.ctx, grpmProject, grpmGeneration, revision.ScopeDigest, grpmResourceType, 10) + if err != nil { + f.fail("healthy list", err) + } + if len(listed) != 1 || !reflect.DeepEqual(listed[0], *got) { + t.Fatalf("healthy List served %+v, want exactly the Get row", listed) + } + + defs, err := f.store.ListExplicitGroupDefinitions(f.ctx, grpmProject, revision.ID) + if err != nil { + f.fail("healthy list definitions", err) + } + if len(defs) != 2 || string(defs[0].ID) != string(grpmGroupA) || string(defs[1].ID) != string(grpmGroupB) { + t.Fatalf("healthy ListDefinitions served %+v", defs) + } +} + +func TestPostgresExplicitGroupRevisionsGetRefusesCountDrift(t *testing.T) { + f := newE2RVFixture(t) + f.migrateWithSelections(t) + + revision, _ := e2rvStagedCompleted(t, f, "countdrift", true) + if _, err := f.store.GetExplicitGroupRevision(f.ctx, grpmProject, revision.ID); err != nil { + f.fail("pre-drift get (control)", err) + } + + e2rvSeedDrift(t, f, "delete one gold definition", + `DELETE FROM `+f.definitionsTable()+` WHERE revision_id = $1 AND group_id = $2`, + string(revision.ID), string(grpmGroupB)) + + _, err := f.store.GetExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + requireE2RVCorrupt(t, err, "Get over count-drifted children") + + if _, err := f.store.ListExplicitGroupDefinitions(f.ctx, grpmProject, revision.ID); !errors.Is(err, explorer.ErrCorruptExplicitGroupRevision) { + t.Fatalf("ListDefinitions regression: err = %v, want the typed corruption refusal", err) + } + + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.definitionsTable()+ + ` (revision_id, group_id, label, ordinal) VALUES ($1, $2, $3, $4)`, + string(revision.ID), "group-guard-probe", "Probe", int64(53)) + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "only STAGING revisions accept definitions", + "guard-alive probe after the drift toggle") +} + +func TestPostgresExplicitGroupRevisionsGetRefusesDigestOnlyDrift(t *testing.T) { + f := newE2RVFixture(t) + f.migrateWithSelections(t) + + revision, _ := e2rvStagedCompleted(t, f, "digestdrift", false) + healthy, err := f.store.GetExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + if err != nil { + f.fail("pre-drift get (control)", err) + } + if healthy.MembershipDigest != e2rvMembershipEmptyGoldn || healthy.MemberCount != 0 { + t.Fatalf("pre-drift control served membership %q/%d, want the frozen empty set", + healthy.MembershipDigest, healthy.MemberCount) + } + + e2rvSeedDrift(t, f, "tamper one gold label", + `UPDATE `+f.definitionsTable()+` SET label = 'tampered label'`+ + ` WHERE revision_id = $1 AND group_id = $2`, + string(revision.ID), string(grpmGroupA)) + + _, err = f.store.GetExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + requireE2RVCorrupt(t, err, "Get over digest-drifted children (count intact)") +} + +func TestPostgresExplicitGroupRevisionsListRefusesWholePageOnDrift(t *testing.T) { + f := newE2RVFixture(t) + f.migrateWithSelections(t) + + source := f.stageSelection(t, "pagedrift", "r-a", "r-b") + revisionA := f.beginGroupRevision(t, source, "e2rv-key-a", "e2rv-tok-a") + f.putDefinitions(t, revisionA.ID, "e2rv-tok-a") + e2rvCompleteHonest(t, f, revisionA, "e2rv-tok-a") + revisionB := f.beginGroupRevision(t, source, "e2rv-key-b", "e2rv-tok-b") + f.putDefinitions(t, revisionB.ID, "e2rv-tok-b") + e2rvCompleteHonest(t, f, revisionB, "e2rv-tok-b") + + listed, err := f.store.ListExplicitGroupRevisions(f.ctx, grpmProject, grpmGeneration, revisionA.ScopeDigest, grpmResourceType, 10) + if err != nil { + f.fail("healthy two-header list", err) + } + if len(listed) != 2 || + !((listed[0].ID == revisionA.ID && listed[1].ID == revisionB.ID) || + (listed[0].ID == revisionB.ID && listed[1].ID == revisionA.ID)) { + t.Fatalf("healthy page served %+v, want exactly both headers", listed) + } + + e2rvSeedDrift(t, f, "delete a child under header B", + `DELETE FROM `+f.definitionsTable()+` WHERE revision_id = $1 AND group_id = $2`, + string(revisionB.ID), string(grpmGroupB)) + + listed, err = f.store.ListExplicitGroupRevisions(f.ctx, grpmProject, grpmGeneration, revisionA.ScopeDigest, grpmResourceType, 10) + requireE2RVCorrupt(t, err, "List over a page containing one drifted header") + if len(listed) != 0 { + t.Fatalf("refused List still served %d rows -- partial serve re-enacted", len(listed)) + } + + if _, err := f.store.GetExplicitGroupRevision(f.ctx, grpmProject, revisionA.ID); err != nil { + f.fail("healthy sibling get after page refusal", err) + } +} + +func TestPostgresExplicitGroupRevisionsVerifyCostEvidence(t *testing.T) { + f := newE2RVFixture(t) + f.migrateWithSelections(t) + + const headers, groupsPer = 12, 40 + source := f.stageSelection(t, "cost", "r-a", "r-b", "r-c", "r-d", "r-e") + ids := make([]explorer.ExplicitGroupRevisionID, 0, headers) + var scopeDigest string + for h := 0; h < headers; h++ { + token := fmt.Sprintf("e2rv-cost-tok-%d", h) + revision := f.beginGroupRevision(t, source, fmt.Sprintf("e2rv-cost-key-%d", h), token) + defs := make([]explorer.ExplicitGroupDefinition, 0, groupsPer) + for g := 0; g < groupsPer; g++ { + defs = append(defs, explorer.ExplicitGroupDefinition{ + ID: explorer.ExplicitGroupID(fmt.Sprintf("e2rv-cost-group-%03d", g)), + Label: fmt.Sprintf("Cost group %03d", g), + Ordinal: int64(g), + }) + } + if err := f.store.PutExplicitGroupDefinitions(f.ctx, revision.ID, token, defs); err != nil { + f.fail("cost put definitions", err) + } + e2rvCompleteHonest(t, f, revision, token) + ids = append(ids, revision.ID) + scopeDigest = revision.ScopeDigest + } + + start := time.Now() + listed, err := f.store.ListExplicitGroupRevisions(f.ctx, grpmProject, grpmGeneration, scopeDigest, grpmResourceType, 100) + if err != nil { + f.fail("cost list", err) + } + if len(listed) != headers { + t.Fatalf("cost list served %d rows, want %d", len(listed), headers) + } + listMs := time.Since(start).Milliseconds() + + start = time.Now() + for _, id := range ids { + if _, err := f.store.GetExplicitGroupRevision(f.ctx, grpmProject, id); err != nil { + f.fail("cost get", err) + } + } + getMs := time.Since(start).Milliseconds() + + t.Logf("verify-on-read cost evidence: %d headers x %d groups: List %d ms, %d Gets %d ms", + headers, groupsPer, listMs, headers, getMs) + if bound := int64(8000); listMs > bound || getMs > bound { + t.Fatalf("verify-on-read cost runaway: list %d ms, get sweep %d ms, bound %d ms", listMs, getMs, bound) + } +} + +func TestZZZeroLeftoverVerifyE2rvFixtures(t *testing.T) { + raw := pgstate06TestPostgresURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for leftover audit: %v", err) + } + defer admin.Close() + var dbLeftovers int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname LIKE $1 ESCAPE '\'`, + grpmLikePattern(e2rvPrefix)).Scan(&dbLeftovers); err != nil { + t.Fatalf("audit query: %v", err) + } + if dbLeftovers != 0 { + t.Fatalf("%d verify_e2rv_* databases survive the suite", dbLeftovers) + } + if owned := grpmSnapshotRegistry(); len(owned) != 0 { + t.Fatalf("creation registry still holds %d entries: %v", len(owned), owned) + } +} diff --git a/internal/backends/postgres/internal/explorer/explicit_groups_test.go b/internal/backends/postgres/internal/explorer/explicit_groups_test.go new file mode 100644 index 000000000..57b0042fa --- /dev/null +++ b/internal/backends/postgres/internal/explorer/explicit_groups_test.go @@ -0,0 +1,4262 @@ +package postgres + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/explorer" +) + +const ( + pgstate06MigrationVersion = int64(9) + pgstate06MigrationName = "explorer_explicit_groups" + pgstate06MigrationChecksum = "c635cb5ef7c3c04655b4f45b01ba0fe834a7f181f70638f300878caf16c9a14f" + pgstate06CheckViolation = "23514" +) + +var pgstate06NameRe = regexp.MustCompile(`^pgstate06_it_[0-9]+_[0-9a-f]+$`) + +func pgstate06TestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 Explorer explicit-group integration tests") + } + return raw +} + +func pgstate06RandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +type pgState06Fixture struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dbName string + schema string + dsn string + scrub func(any) string + fail func(step string, err error) + + store *ExplicitGroupStore + witness *pgxpool.Pool + witnessID int32 +} + +func newPGState06Fixture(t *testing.T) *pgState06Fixture { + t.Helper() + raw := pgstate06TestPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 360*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgstate06_it_%d_%s", time.Now().UnixNano(), pgstate06RandomHex(t)) + if !pgstate06NameRe.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + + t.Cleanup(func() { + var mine int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname = $1`, dbName).Scan(&mine); err != nil { + t.Errorf("leftover audit: query database catalog: %s", scrub(err)) + } + if mine != 0 { + t.Errorf("leftover audit: isolated database %s still exists", dbName) + } + var namespaceLeftovers int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname LIKE 'pgstate06\_it\_%' ESCAPE '\'`).Scan(&namespaceLeftovers); err != nil { + t.Errorf("leftover audit: query database namespace: %s", scrub(err)) + } + if namespaceLeftovers != 0 { + t.Errorf("leftover audit: %d pgstate06_it_* databases remain", namespaceLeftovers) + } + var schemaLeftovers int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgstate06\_it\_%' ESCAPE '\'`).Scan(&schemaLeftovers); err != nil { + t.Errorf("leftover audit: query schema namespace: %s", scrub(err)) + } + if schemaLeftovers != 0 { + t.Errorf("leftover audit: %d pgstate06_it_* schemas remain in the service database", schemaLeftovers) + } + }) + + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + if probe, err := url.Parse(dsn); err != nil || + probe.Query().Get("dbname") != "" || probe.Path != "/"+dbName { + t.Fatal("isolated test DSN does not provably target the isolated database") + } + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("pgstate06_it_%d_%s", time.Now().UnixNano(), pgstate06RandomHex(t)) + if !pgstate06NameRe.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + f := &pgState06Fixture{ctx: ctx, admin: admin, iso: iso, dbName: dbName, schema: schema, dsn: dsn, scrub: scrub, fail: fail} + f.witness, f.witnessID = f.newDedicatedPool(t) + store, err := f.newStore(t) + if err != nil { + f.fail("build explicit-group store", err) + } + f.store = store + return f +} + +func (f *pgState06Fixture) migrate(t *testing.T) pgstore.MigrateResult { + t.Helper() + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, + MaxConns: 2, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + f.fail("open migration pool", err) + } + defer client.Close() + owner, err := OwnerMigrations() + if err != nil { + f.fail("owner migrations", err) + } + runner, err := pgstore.NewMigrationRunner(client, f.schema, owner) + if err != nil { + f.fail("new migration runner", err) + } + result, err := runner.Migrate(f.ctx) + if err != nil { + f.fail("migrate state schema", err) + } + return result +} + +func (f *pgState06Fixture) newStore(t *testing.T) (*ExplicitGroupStore, error) { + t.Helper() + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + return nil, err + } + t.Cleanup(client.Close) + return NewExplicitGroupStore(client, f.schema) +} + +func (f *pgState06Fixture) newDedicatedStore(t *testing.T) (*ExplicitGroupStore, int32) { + t.Helper() + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, + MaxConns: 1, + MinConns: 1, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + f.fail("open dedicated pg-store pool", err) + } + t.Cleanup(client.Close) + conn, err := client.Acquire(f.ctx) + if err != nil { + f.fail("acquire dedicated pg-store connection", err) + } + var pid int32 + if err := conn.QueryRow(f.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + conn.Release() + f.fail("witness dedicated pg-store backend pid", err) + } + conn.Release() + if pid == 0 { + t.Fatal("dedicated pg-store pool reported backend pid 0") + } + store, err := NewExplicitGroupStore(client, f.schema) + if err != nil { + f.fail("build dedicated explicit-group store", err) + } + return store, pid +} + +func (f *pgState06Fixture) newDedicatedPool(t *testing.T) (*pgxpool.Pool, int32) { + t.Helper() + poolConfig, err := pgxpool.ParseConfig(f.dsn) + if err != nil { + f.fail("parse dedicated raw pool config", err) + } + poolConfig.MaxConns = 1 + poolConfig.MinConns = 1 + pool, err := pgxpool.NewWithConfig(f.ctx, poolConfig) + if err != nil { + f.fail("open dedicated raw pool", err) + } + t.Cleanup(pool.Close) + var pid int32 + if err := pool.QueryRow(f.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + f.fail("witness dedicated raw backend pid", err) + } + if pid == 0 { + t.Fatal("dedicated raw pool reported backend pid 0") + } + return pool, pid +} + +func requireDistinctBackendsPGState06(t *testing.T, pids map[string]int32) { + t.Helper() + seen := map[int32]string{} + for label, pid := range pids { + if pid == 0 { + t.Fatalf("%s reported backend pid 0", label) + } + if other, clash := seen[pid]; clash { + t.Fatalf("%s and %s share backend pid %d, want provably distinct PG connections", label, other, pid) + } + seen[pid] = label + } +} + +func (f *pgState06Fixture) revisionTable() string { + return sanitizeTableRef(f.schema, ExplicitGroupRevisionsTable) +} + +func (f *pgState06Fixture) definitionsTable() string { + return sanitizeTableRef(f.schema, ExplicitGroupDefinitionsTable) +} + +func (f *pgState06Fixture) headerSnapshot(t *testing.T, revisionID string) string { + t.Helper() + var out string + err := f.witness.QueryRow(f.ctx, `SELECT concat_ws(E'\x1f', + id, project, generation, scope_digest, resource_type, + source_selection_revision_id, source_membership_digest, + definition_digest, membership_digest, + group_count::text, member_count::text, state, idempotency_key, + to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), + COALESCE(to_char(completed_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), ''), + COALESCE(writer_token, ''), + COALESCE(to_char(writer_expires_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), '')) + FROM `+f.revisionTable()+` WHERE id = $1`, revisionID).Scan(&out) + if err != nil { + f.fail("header snapshot of "+revisionID, err) + } + return out +} + +func (f *pgState06Fixture) frozenHeaderSnapshot(t *testing.T, revisionID string) string { + t.Helper() + snapshot := f.headerSnapshot(t, revisionID) + parts := strings.Split(snapshot, "\x1f") + if len(parts) != 17 { + t.Fatalf("header snapshot of %s has %d fields, want 17", revisionID, len(parts)) + } + return strings.Join(parts[:15], "\x1f") +} + +func (f *pgState06Fixture) definitionsSnapshot(t *testing.T, revisionID string) string { + t.Helper() + rows, err := f.witness.Query(f.ctx, `SELECT group_id, label, ordinal::text FROM `+ + f.definitionsTable()+` WHERE revision_id = $1 ORDER BY ordinal, group_id`, revisionID) + if err != nil { + f.fail("definitions snapshot of "+revisionID, err) + } + defer rows.Close() + var parts []string + for rows.Next() { + var groupID, label, ordinal string + if err := rows.Scan(&groupID, &label, &ordinal); err != nil { + f.fail("scan definitions snapshot of "+revisionID, err) + } + parts = append(parts, strings.Join([]string{groupID, label, ordinal}, "|")) + } + if err := rows.Err(); err != nil { + f.fail("definitions snapshot of "+revisionID, err) + } + return strings.Join(parts, "\x1e") +} + +func (f *pgState06Fixture) countRows(t *testing.T, table, where string, args ...any) int { + t.Helper() + var count int + if err := f.witness.QueryRow(f.ctx, `SELECT count(*) FROM `+table+` WHERE `+where, args...).Scan(&count); err != nil { + f.fail("count rows in "+table, err) + } + return count +} + +func requirePgErrPGState06(t *testing.T, err error, sqlstate, step string) *pgconn.PgError { + t.Helper() + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + t.Fatalf("%s: error = %v (%T), want *pgconn.PgError", step, err, err) + } + if pgErr.Code != sqlstate { + t.Fatalf("%s: sqlstate = %s, want %s (error: %v)", step, pgErr.Code, sqlstate, err) + } + return pgErr +} + +func requirePgErrConstraintPGState06(t *testing.T, err error, sqlstate, constraint, step string) *pgconn.PgError { + t.Helper() + pgErr := requirePgErrPGState06(t, err, sqlstate, step) + if pgErr.ConstraintName != constraint { + t.Fatalf("%s: constraint = %q, want the named guard %q (message: %s)", step, pgErr.ConstraintName, constraint, pgErr.Message) + } + return pgErr +} + +func requirePgErrTriggerPGState06(t *testing.T, err error, sqlstate, trigger, step string) *pgconn.PgError { + t.Helper() + pgErr := requirePgErrPGState06(t, err, sqlstate, step) + if !strings.Contains(pgErr.Message, trigger) { + t.Fatalf("%s: message %q does not name the guard %q", step, pgErr.Message, trigger) + } + return pgErr +} + +func (f *pgState06Fixture) rerunMigration(t *testing.T, version int64) { + t.Helper() + migrations, err := OwnerMigrations() + if err != nil { + f.fail("load migrations for rerun", err) + } + for _, m := range migrations { + if m.Version == version { + conn, err := f.iso.Acquire(f.ctx) + if err != nil { + f.fail("acquire connection for migration rerun", err) + } + defer conn.Release() + tx, err := conn.Begin(f.ctx) + if err != nil { + f.fail("begin migration rerun transaction", err) + } + if _, err := tx.Exec(f.ctx, + "SET LOCAL search_path TO "+pgx.Identifier{f.schema}.Sanitize()); err != nil { + tx.Rollback(f.ctx) + f.fail("set search_path for migration rerun", err) + } + if _, err := tx.Exec(f.ctx, m.SQL); err != nil { + tx.Rollback(f.ctx) + f.fail(fmt.Sprintf("rerun migration %d/%s", m.Version, m.Name), err) + } + if err := tx.Commit(f.ctx); err != nil { + f.fail("commit migration rerun", err) + } + return + } + } + t.Fatalf("migration version %d is absent from the embedded history", version) +} + +func (f *pgState06Fixture) dropTrigger(t *testing.T, table, trigger string) { + t.Helper() + if _, err := f.witness.Exec(f.ctx, `DROP TRIGGER `+trigger+` ON `+table); err != nil { + f.fail("drop trigger "+trigger+" on "+table, err) + } +} + +func (f *pgState06Fixture) execOn(t *testing.T, pool *pgxpool.Pool, step, sql string, args ...any) { + t.Helper() + if _, err := pool.Exec(f.ctx, sql, args...); err != nil { + f.fail(step+": "+scrubSQLForTest(sql), err) + } +} + +func scrubSQLForTest(sql string) string { + if len(sql) > 160 { + return sql[:160] + "..." + } + return sql +} + +func pgstate06Selection(project string, suffix string) explorer.SelectionRevision { + completedAt := time.Now().UTC() + return explorer.SelectionRevision{ + ID: "pgstate06-selection-" + suffix, + Project: project, + Generation: "generation-a", + ResourceType: "Patient", + Rule: explorer.SelectionRule{Kind: explorer.SelectionRuleExplicit}, + Source: explorer.SelectionSource{Kind: explorer.SelectionSourceExplicit}, + ScopeDigest: "scope-a", + RuleDigest: "selection-rule", + MembershipDigest: "sha256:" + strings.Repeat("a", 64), + MemberCount: 3, + MemberBytes: 30, + Complete: true, + CreatedAt: completedAt, + CompletedAt: &completedAt, + } +} + +func pgstate06Revision(source explorer.SelectionRevision, key string) explorer.ExplicitGroupRevision { + return explorer.ExplicitGroupRevision{ + ID: explorer.ExplicitGroupRevisionIDFor(source.Project, key), + Project: source.Project, + Generation: source.Generation, + ScopeDigest: source.ScopeDigest, + ResourceType: source.ResourceType, + SourceSelectionRevisionID: source.ID, + SourceMembershipDigest: source.MembershipDigest, + State: explorer.ExplicitGroupRevisionStaging, + IdempotencyKey: key, + CreatedAt: time.Now().UTC(), + } +} + +func pgstate06Definitions() []explorer.ExplicitGroupDefinition { + return []explorer.ExplicitGroupDefinition{ + {ID: "group-alpha", Label: "Alpha cohort", Ordinal: 0}, + {ID: "group-empty", Label: "No members", Ordinal: 1}, + {ID: "group-beta", Label: "Beta cohort", Ordinal: 2}, + } +} + +func pgstate06CanonicalDefinitions(t *testing.T) []explorer.ExplicitGroupDefinition { + t.Helper() + canonical, err := explorer.CanonicalExplicitGroupDefinitions(pgstate06Definitions()) + if err != nil { + t.Fatalf("canonical test definitions: %v", err) + } + return canonical +} + +func (f *pgState06Fixture) rawInsertHeader(t *testing.T, pool *pgxpool.Pool, revision explorer.ExplicitGroupRevision, writerToken string, writerExpiresAt *time.Time) { + t.Helper() + f.execOn(t, pool, "raw insert header", + `INSERT INTO `+f.revisionTable()+` (id, project, generation, scope_digest, resource_type,`+ + ` source_selection_revision_id, source_membership_digest, definition_digest, membership_digest,`+ + ` group_count, member_count, state, idempotency_key, created_at, completed_at, writer_token, writer_expires_at)`+ + ` VALUES ($1, $2, $3, $4, $5, $6, $7, '', '', 0, 0, 'STAGING', $8, $9, NULL, $10, $11)`, + string(revision.ID), revision.Project, revision.Generation, revision.ScopeDigest, revision.ResourceType, + revision.SourceSelectionRevisionID, revision.SourceMembershipDigest, revision.IdempotencyKey, + revision.CreatedAt, writerToken, writerExpiresAt) +} + +func (f *pgState06Fixture) rawInsertDefinition(t *testing.T, pool *pgxpool.Pool, revisionID, groupID, label string, ordinal int64) { + t.Helper() + f.execOn(t, pool, "raw insert definition", + `INSERT INTO `+f.definitionsTable()+` (revision_id, group_id, label, ordinal) VALUES ($1, $2, $3, $4)`, + revisionID, groupID, label, ordinal) +} + +func mustErr(t *testing.T, step string, fn func() error, want error) { + t.Helper() + err := fn() + if !errors.Is(err, want) { + t.Fatalf("%s: error = %v, want %v", step, err, want) + } +} + +func mustNil(t *testing.T, step string, fn func() error) { + t.Helper() + if err := fn(); err != nil { + t.Fatalf("%s: error = %v, want success", step, err) + } +} + +func TestPostgresExplicitGroupDefinitionsStoreGuards(t *testing.T) { + f := newPGState06Fixture(t) + f.migrate(t) + + if _, err := NewExplicitGroupStore(nil, "ok_schema"); err == nil { + t.Fatal("NewExplicitGroupStore(nil, ...) = nil error, want failure") + } + for _, bad := range []string{"", "BadName", "has-dash", "1leading", strings.Repeat("x", 64)} { + if _, err := NewExplicitGroupStore(&pgstore.Client{}, bad); err == nil { + t.Fatalf("NewExplicitGroupStore(client, %q) = nil error, want failure", bad) + } + } + if _, err := NewExplicitGroupStore(&pgstore.Client{}, "ok_schema"); err != nil { + t.Fatalf("NewExplicitGroupStore(valid) = %v, want success", err) + } + + bare := fmt.Sprintf("pgstate06_it_%d_%s", time.Now().UnixNano(), pgstate06RandomHex(t)) + if _, err := f.iso.Exec(f.ctx, `CREATE SCHEMA "`+bare+`"`); err != nil { + f.fail("create unmigrated schema", err) + } + client, err := pgstore.Open(f.ctx, pgstore.Config{DSN: f.dsn, MaxConns: 1, MinConns: 0, AcquireTimeout: 5 * time.Second}) + if err != nil { + f.fail("open unmigrated pool", err) + } + t.Cleanup(client.Close) + bareStore, err := NewExplicitGroupStore(client, bare) + if err != nil { + f.fail("build unmigrated store", err) + } + _, err = bareStore.GetExplicitGroupRevision(f.ctx, "p", "pgstate06-missing") + requirePgErrPGState06(t, err, "42P01", "unmapped schema read fails closed") +} + +func TestPostgresExplicitGroupDefinitionsBeginLifecycleAndOwnership(t *testing.T) { + f := newPGState06Fixture(t) + f.migrate(t) + + project := "pgstate06-project" + source := pgstate06Selection(project, "begin-"+pgstate06RandomHex(t)) + key := "begin-" + pgstate06RandomHex(t) + revision := pgstate06Revision(source, key) + token := "writer-" + pgstate06RandomHex(t) + + started, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, token) + if err != nil { + f.fail("begin staging revision", err) + } + if started.ID != revision.ID || started.State != explorer.ExplicitGroupRevisionStaging { + t.Fatalf("started = %+v, want the canonical staging header", started) + } + if started.CreatedAt.IsZero() || started.CompletedAt != nil { + t.Fatalf("started carries completion values: %+v", started) + } + if want := explorer.ExplicitGroupRevisionIDFor(project, key); started.ID != want { + t.Fatalf("revision ID = %s, want self-recomputed %s", started.ID, want) + } + + wantSnapshot := strings.Join([]string{ + string(revision.ID), project, "generation-a", "scope-a", "Patient", + source.ID, source.MembershipDigest, + "", "", "0", "0", "STAGING", key, + }, "\x1f") + snapshot := f.headerSnapshot(t, string(revision.ID)) + if !strings.HasPrefix(snapshot, wantSnapshot) { + t.Fatalf("stored header = %q, want identity/binding/completion-empty prefix %q", snapshot, wantSnapshot) + } + + beforeFull := f.headerSnapshot(t, string(revision.ID)) + beforeFrozen := strings.Join(strings.Split(beforeFull, "\x1f")[:15], "\x1f") + retry, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, token) + if err != nil { + f.fail("same-owner begin retry", err) + } + if retry.ID != started.ID { + t.Fatalf("retry = %+v, want the stored header", retry) + } + afterFull := f.headerSnapshot(t, string(revision.ID)) + afterFrozen := strings.Join(strings.Split(afterFull, "\x1f")[:15], "\x1f") + if beforeFrozen != afterFrozen { + t.Fatalf("same-owner retry changed frozen columns:\nbefore %q\nafter %q", beforeFrozen, afterFrozen) + } + if strings.Split(afterFull, "\x1f")[16] == strings.Split(beforeFull, "\x1f")[16] { + t.Fatal("same-owner retry did not advance the writer lease") + } + + frozenBefore := f.frozenHeaderSnapshot(t, string(revision.ID)) + mustErr(t, "foreign-token begin", func() error { + _, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, "not-the-writer") + return err + }, explorer.ErrExplicitGroupRevisionConflict) + if got := f.frozenHeaderSnapshot(t, string(revision.ID)); got != frozenBefore { + t.Fatalf("foreign-token begin mutated the header:\nbefore %q\nafter %q", frozenBefore, got) + } + mustNil(t, "owner begin after foreign refusal", func() error { + _, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, token) + return err + }) + + forgeKey := "drift-" + pgstate06RandomHex(t) + forge := pgstate06Revision(source, forgeKey) + forgedRow := forge + forgedRow.Generation = "generation-forged" + liveLease := time.Now().UTC().Add(time.Hour) + f.rawInsertHeader(t, f.witness, forgedRow, "forged-writer", &liveLease) + drifted := pgstate06Revision(source, forgeKey) + mustErr(t, "drifted-intent begin", func() error { + _, err := f.store.BeginExplicitGroupRevision(f.ctx, drifted, source, token) + return err + }, explorer.ErrExplicitGroupRevisionConflict) + + mustErr(t, "blank-token begin", func() error { + _, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, " ") + return err + }, explorer.ErrExplicitGroupRevisionConflict) + + completedInput := revision + completedInput.ID = explorer.ExplicitGroupRevisionIDFor(project, key+"-complete") + completedInput.IdempotencyKey = key + "-complete" + completedInput.State = explorer.ExplicitGroupRevisionComplete + completedInput.DefinitionDigest = strings.Repeat("d", 64) + completedInput.MembershipDigest = strings.Repeat("m", 64) + completedInput.GroupCount = 1 + now := time.Now().UTC() + completedInput.CompletedAt = &now + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, completedInput, source, token); err == nil || + !strings.Contains(err.Error(), "must begin in staging state") { + t.Fatalf("completed-state begin = %v, want the staging-state refusal", err) + } + + forged := revision + forged.ID = explorer.ExplicitGroupRevisionID("grouprev_" + strings.Repeat("0", 64)) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, forged, source, token); err == nil || + strings.Contains(err.Error(), "grouprev_0000000000") { + t.Fatalf("forged-ID begin = %v, want the domain identity refusal", err) + } + + bindingCases := []struct { + name string + mutate func(*explorer.SelectionRevision) + }{ + {"selection id", func(s *explorer.SelectionRevision) { s.ID = "other-selection" }}, + {"project", func(s *explorer.SelectionRevision) { s.Project = "other-project" }}, + {"generation", func(s *explorer.SelectionRevision) { s.Generation = "generation-b" }}, + {"resource type", func(s *explorer.SelectionRevision) { s.ResourceType = "Observation" }}, + {"scope digest", func(s *explorer.SelectionRevision) { s.ScopeDigest = "another-scope" }}, + {"membership digest", func(s *explorer.SelectionRevision) { s.MembershipDigest = "sha256:" + strings.Repeat("b", 64) }}, + {"incomplete selection", func(s *explorer.SelectionRevision) { s.Complete = false }}, + } + for _, bindingCase := range bindingCases { + mutated := source + bindingCase.mutate(&mutated) + altKey := "binding-" + bindingCase.name + "-" + pgstate06RandomHex(t) + alt := pgstate06Revision(source, altKey) + _, err := f.store.BeginExplicitGroupRevision(f.ctx, alt, mutated, token) + if !errors.Is(err, explorer.ErrExplicitGroupRevisionStaleSource) { + t.Fatalf("begin with mismatched %s = %v, want ErrExplicitGroupRevisionStaleSource", bindingCase.name, err) + } + if got := f.countRows(t, f.revisionTable(), "id = $1", string(alt.ID)); got != 0 { + t.Fatalf("refused %s binding left %d rows, want 0", bindingCase.name, got) + } + } + okKey := "binding-ok-" + pgstate06RandomHex(t) + ok := pgstate06Revision(source, okKey) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, ok, source, token); err != nil { + f.fail("begin with correct binding after refusals", err) + } + if got := f.countRows(t, f.revisionTable(), "id = $1", string(ok.ID)); got != 1 { + t.Fatalf("correct-binding begin stored %d rows, want 1", got) + } + + reopened, err := f.newStore(t) + if err != nil { + f.fail("reopen store", err) + } + reopenedHeader, err := reopened.BeginExplicitGroupRevision(f.ctx, revision, source, token) + if err != nil { + f.fail("reopen begin retry", err) + } + if reopenedHeader.ID != revision.ID || reopenedHeader.State != explorer.ExplicitGroupRevisionStaging { + t.Fatalf("reopened header = %+v, want the stored staging header", reopenedHeader) + } + if got := f.countRows(t, f.revisionTable(), "id = $1", string(revision.ID)); got != 1 { + t.Fatalf("retry path stored %d headers, want exactly 1", got) + } +} + +func TestPostgresExplicitGroupDefinitionsEmptyDefinitionsSurviveReopen(t *testing.T) { + f := newPGState06Fixture(t) + f.migrate(t) + + project := "pgstate06-project" + source := pgstate06Selection(project, "defs-"+pgstate06RandomHex(t)) + key := "defs-" + pgstate06RandomHex(t) + revision := pgstate06Revision(source, key) + token := "writer-" + pgstate06RandomHex(t) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, token); err != nil { + f.fail("begin", err) + } + + shuffled := []explorer.ExplicitGroupDefinition{ + {ID: "group-beta", Label: "Beta cohort", Ordinal: 2}, + {ID: "group-empty", Label: "No members", Ordinal: 1}, + {ID: "group-alpha", Label: "Alpha cohort", Ordinal: 0}, + } + mustNil(t, "put definitions", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, revision.ID, token, shuffled) + }) + + want := strings.Join([]string{ + "group-alpha|Alpha cohort|0", + "group-empty|No members|1", + "group-beta|Beta cohort|2", + }, "\x1e") + if got := f.definitionsSnapshot(t, string(revision.ID)); got != want { + t.Fatalf("stored definitions = %q, want literal %q", got, want) + } + if got := f.countRows(t, f.definitionsTable(), "revision_id = $1", string(revision.ID)); got != 3 { + t.Fatalf("definition count = %d, want 3", got) + } + + reopened, err := f.newStore(t) + if err != nil { + f.fail("reopen store", err) + } + mustNil(t, "replayed put after reopen", func() error { + return reopened.PutExplicitGroupDefinitions(f.ctx, revision.ID, token, pgstate06Definitions()) + }) + if got := f.definitionsSnapshot(t, string(revision.ID)); got != want { + t.Fatalf("replayed put changed stored definitions:\nbefore %q\nafter %q", want, got) + } + + remixed := []explorer.ExplicitGroupDefinition{ + {ID: "group-empty", Label: " No members ", Ordinal: 1}, + {ID: "group-alpha", Label: "Alpha cohort", Ordinal: 0}, + {ID: "group-beta", Label: "Beta cohort", Ordinal: 2}, + } + mustNil(t, "remixed put", func() error { + return reopened.PutExplicitGroupDefinitions(f.ctx, revision.ID, token, remixed) + }) + if got := f.definitionsSnapshot(t, string(revision.ID)); got != want { + t.Fatalf("remixed put changed stored definitions:\nbefore %q\nafter %q", want, got) + } + + soloKey := "solo-" + pgstate06RandomHex(t) + solo := pgstate06Revision(source, soloKey) + soloToken := "writer-solo-" + pgstate06RandomHex(t) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, solo, source, soloToken); err != nil { + f.fail("begin solo", err) + } + mustNil(t, "put single empty group", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, solo.ID, soloToken, []explorer.ExplicitGroupDefinition{ + {ID: "group-only-empty", Label: "Empty but real", Ordinal: 7}, + }) + }) + if got := f.definitionsSnapshot(t, string(solo.ID)); got != "group-only-empty|Empty but real|7" { + t.Fatalf("empty-group-only revision = %q, want the declared row", got) + } + + emptyKey := "empty-list-" + pgstate06RandomHex(t) + empty := pgstate06Revision(source, emptyKey) + emptyToken := "writer-empty-" + pgstate06RandomHex(t) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, empty, source, emptyToken); err != nil { + f.fail("begin empty-list revision", err) + } + if err := f.store.PutExplicitGroupDefinitions(f.ctx, empty.ID, emptyToken, nil); err == nil { + t.Fatal("put with zero definitions = nil, want the at-least-one-group refusal") + } + if got := f.countRows(t, f.definitionsTable(), "revision_id = $1", string(empty.ID)); got != 0 { + t.Fatalf("refused empty-list put left %d rows, want 0", got) + } + mustNil(t, "legal declaration after refusal", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, empty.ID, emptyToken, []explorer.ExplicitGroupDefinition{ + {ID: "group-first", Label: "First", Ordinal: 0}, + }) + }) + + before := f.definitionsSnapshot(t, string(revision.ID)) + if err := f.store.PutExplicitGroupDefinitions(f.ctx, revision.ID, token, []explorer.ExplicitGroupDefinition{ + {ID: "group-alpha", Label: "Alpha cohort", Ordinal: 0}, + {ID: "group-alpha", Label: "Copycat", Ordinal: 5}, + }); err == nil { + t.Fatal("put with duplicate group ID = nil, want the domain refusal") + } + if err := f.store.PutExplicitGroupDefinitions(f.ctx, revision.ID, token, []explorer.ExplicitGroupDefinition{ + {ID: "group-alpha", Label: "Alpha cohort", Ordinal: 3}, + {ID: "group-beta", Label: "Beta cohort", Ordinal: 3}, + }); err == nil { + t.Fatal("put with duplicate ordinal = nil, want the domain refusal") + } + if got := f.definitionsSnapshot(t, string(revision.ID)); got != before { + t.Fatalf("refused puts mutated stored definitions:\nbefore %q\nafter %q", before, got) + } +} + +func TestPostgresExplicitGroupDefinitionsWriteOnceAndContentDrift(t *testing.T) { + f := newPGState06Fixture(t) + f.migrate(t) + + project := "pgstate06-project" + source := pgstate06Selection(project, "write-once-"+pgstate06RandomHex(t)) + key := "write-once-" + pgstate06RandomHex(t) + revision := pgstate06Revision(source, key) + token := "writer-" + pgstate06RandomHex(t) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, token); err != nil { + f.fail("begin", err) + } + mustNil(t, "put", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, revision.ID, token, pgstate06Definitions()) + }) + want := f.definitionsSnapshot(t, string(revision.ID)) + + driftCases := []struct { + name string + patch func([]explorer.ExplicitGroupDefinition) []explorer.ExplicitGroupDefinition + }{ + {"renamed label", func(g []explorer.ExplicitGroupDefinition) []explorer.ExplicitGroupDefinition { + g[0].Label = "Renamed cohort" + return g + }}, + {"added group", func(g []explorer.ExplicitGroupDefinition) []explorer.ExplicitGroupDefinition { + return append(g, explorer.ExplicitGroupDefinition{ID: "group-extra", Label: "Extra", Ordinal: 9}) + }}, + {"dropped group", func(g []explorer.ExplicitGroupDefinition) []explorer.ExplicitGroupDefinition { + return g[:2] + }}, + {"moved order", func(g []explorer.ExplicitGroupDefinition) []explorer.ExplicitGroupDefinition { + g[2].Ordinal = 3 + return g + }}, + } + for _, driftCase := range driftCases { + drifted, err := explorer.CanonicalExplicitGroupDefinitions(driftCase.patch(pgstate06Definitions())) + if err != nil { + t.Fatalf("canonicalize %s drift fixture: %v", driftCase.name, err) + } + mustErr(t, "put "+driftCase.name+" drift", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, revision.ID, token, drifted) + }, explorer.ErrExplicitGroupRevisionConflict) + if got := f.definitionsSnapshot(t, string(revision.ID)); got != want { + t.Fatalf("%s drift mutated stored definitions:\nbefore %q\nafter %q", driftCase.name, want, got) + } + } + mustNil(t, "identical put after drift refusals", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, revision.ID, token, pgstate06Definitions()) + }) + + mustErr(t, "foreign-token put", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, revision.ID, "intruder", pgstate06Definitions()) + }, explorer.ErrExplicitGroupRevisionConflict) + if got := f.definitionsSnapshot(t, string(revision.ID)); got != want { + t.Fatalf("foreign-token put mutated stored definitions:\nbefore %q\nafter %q", want, got) + } + + mustErr(t, "blank-revision put", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, " ", token, pgstate06Definitions()) + }, explorer.ErrExplicitGroupRevisionConflict) + mustErr(t, "blank-token put", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, revision.ID, " ", pgstate06Definitions()) + }, explorer.ErrExplicitGroupRevisionConflict) + mustErr(t, "unknown-revision put", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, explorer.ExplicitGroupRevisionID("grouprev_"+strings.Repeat("9", 64)), token, pgstate06Definitions()) + }, explorer.ErrExplicitGroupRevisionNotFound) +} + +func TestPostgresExplicitGroupDefinitionsAbortIdempotence(t *testing.T) { + f := newPGState06Fixture(t) + f.migrate(t) + + project := "pgstate06-project" + source := pgstate06Selection(project, "abort-"+pgstate06RandomHex(t)) + + mustNil(t, "abort blank id", func() error { + return f.store.AbortExplicitGroupRevision(f.ctx, " ", "writer") + }) + mustNil(t, "abort missing revision", func() error { + return f.store.AbortExplicitGroupRevision(f.ctx, explorer.ExplicitGroupRevisionID("grouprev_"+strings.Repeat("9", 64)), "writer") + }) + + key := "abort-" + pgstate06RandomHex(t) + revision := pgstate06Revision(source, key) + token := "writer-" + pgstate06RandomHex(t) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, token); err != nil { + f.fail("begin", err) + } + mustNil(t, "put", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, revision.ID, token, pgstate06Definitions()) + }) + + mustErr(t, "abort blank token", func() error { + return f.store.AbortExplicitGroupRevision(f.ctx, revision.ID, " ") + }, explorer.ErrExplicitGroupRevisionConflict) + beforeHeader := f.headerSnapshot(t, string(revision.ID)) + beforeDefs := f.definitionsSnapshot(t, string(revision.ID)) + mustErr(t, "foreign-token abort", func() error { + return f.store.AbortExplicitGroupRevision(f.ctx, revision.ID, "intruder") + }, explorer.ErrExplicitGroupRevisionConflict) + if got := f.headerSnapshot(t, string(revision.ID)); got != beforeHeader { + t.Fatalf("foreign abort mutated the header:\nbefore %q\nafter %q", beforeHeader, got) + } + if got := f.definitionsSnapshot(t, string(revision.ID)); got != beforeDefs { + t.Fatalf("foreign abort mutated the definitions:\nbefore %q\nafter %q", beforeDefs, got) + } + + mustNil(t, "owner abort", func() error { + return f.store.AbortExplicitGroupRevision(f.ctx, revision.ID, token) + }) + if got := f.countRows(t, f.revisionTable(), "id = $1", string(revision.ID)); got != 0 { + t.Fatalf("aborted header count = %d, want 0", got) + } + if got := f.countRows(t, f.definitionsTable(), "revision_id = $1", string(revision.ID)); got != 0 { + t.Fatalf("aborted definitions count = %d, want 0", got) + } + mustNil(t, "owner abort replay", func() error { + return f.store.AbortExplicitGroupRevision(f.ctx, revision.ID, token) + }) + mustNil(t, "anyone abort after removal", func() error { + return f.store.AbortExplicitGroupRevision(f.ctx, revision.ID, "intruder") + }) + + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, "reclaimer"); err != nil { + f.fail("re-claim freed key", err) + } + if got := f.countRows(t, f.revisionTable(), "id = $1", string(revision.ID)); got != 1 { + t.Fatalf("re-claimed header count = %d, want 1", got) + } +} + +func TestPostgresExplicitGroupDefinitionsStagingHiddenFromReads(t *testing.T) { + f := newPGState06Fixture(t) + f.migrate(t) + + project := "pgstate06-project" + source := pgstate06Selection(project, "reads-"+pgstate06RandomHex(t)) + key := "reads-" + pgstate06RandomHex(t) + revision := pgstate06Revision(source, key) + token := "writer-" + pgstate06RandomHex(t) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, token); err != nil { + f.fail("begin", err) + } + + _, err := f.store.GetExplicitGroupRevision(f.ctx, project, revision.ID) + mustErr(t, "get staging header", func() error { + _, err := f.store.GetExplicitGroupRevision(f.ctx, project, revision.ID) + return err + }, explorer.ErrExplicitGroupRevisionIncomplete) + + mustErr(t, "get foreign project", func() error { + _, err := f.store.GetExplicitGroupRevision(f.ctx, "other-project", revision.ID) + return err + }, explorer.ErrExplicitGroupRevisionNotFound) + mustErr(t, "get blank id", func() error { + _, err := f.store.GetExplicitGroupRevision(f.ctx, project, " ") + return err + }, explorer.ErrExplicitGroupRevisionNotFound) + + _, err = f.store.ListExplicitGroupDefinitions(f.ctx, project, revision.ID) + mustErr(t, "list definitions of staging", func() error { + _, err := f.store.ListExplicitGroupDefinitions(f.ctx, project, revision.ID) + return err + }, explorer.ErrExplicitGroupRevisionIncomplete) + mustErr(t, "list definitions blank id", func() error { + _, err := f.store.ListExplicitGroupDefinitions(f.ctx, project, " ") + return err + }, explorer.ErrExplicitGroupRevisionNotFound) + + listed, err := f.store.ListExplicitGroupRevisions(f.ctx, project, "generation-a", "scope-a", "Patient", 0) + if err != nil { + f.fail("list revisions", err) + } + if listed == nil { + t.Fatal("listing returned nil, want empty slice") + } + if len(listed) != 0 { + t.Fatalf("staging header leaked into listing: %+v", listed) + } + if got := f.countRows(t, f.revisionTable(), "id = $1", string(revision.ID)); got != 1 { + t.Fatalf("witness header count = %d, want 1 (hidden by state, not absent)", got) + } +} + +func TestPostgresExplicitGroupDefinitionsCleanupReapsExpiredNeverActive(t *testing.T) { + f := newPGState06Fixture(t) + f.migrate(t) + + project := "pgstate06-project" + source := pgstate06Selection(project, "cleanup-"+pgstate06RandomHex(t)) + now := time.Now().UTC() + past := now.Add(-time.Hour) + expiredLease := now.Add(-time.Minute) + liveLease := now.Add(5 * time.Minute) + + seeds := []struct { + revision explorer.ExplicitGroupRevision + lease *time.Time + }{ + {pgstate06Revision(source, "cl-expired-"+pgstate06RandomHex(t)), &expiredLease}, + {pgstate06Revision(source, "cl-null-"+pgstate06RandomHex(t)), nil}, + {pgstate06Revision(source, "cl-active-"+pgstate06RandomHex(t)), &liveLease}, + {pgstate06Revision(source, "cl-young-"+pgstate06RandomHex(t)), &expiredLease}, + } + expired, nullLease, active, young := seeds[0].revision, seeds[1].revision, seeds[2].revision, seeds[3].revision + for _, seed := range seeds { + f.rawInsertHeader(t, f.witness, explorer.ExplicitGroupRevision{ + ID: seed.revision.ID, Project: seed.revision.Project, Generation: seed.revision.Generation, + ScopeDigest: seed.revision.ScopeDigest, ResourceType: seed.revision.ResourceType, + SourceSelectionRevisionID: seed.revision.SourceSelectionRevisionID, + SourceMembershipDigest: seed.revision.SourceMembershipDigest, + State: explorer.ExplicitGroupRevisionStaging, + IdempotencyKey: seed.revision.IdempotencyKey, + CreatedAt: past, + }, "seed-writer", seed.lease) + } + f.rawInsertDefinition(t, f.witness, string(expired.ID), "group-a", "A", 0) + f.rawInsertDefinition(t, f.witness, string(expired.ID), "group-b", "B", 1) + f.rawInsertDefinition(t, f.witness, string(active.ID), "group-a", "A", 0) + activeDefs := f.definitionsSnapshot(t, string(active.ID)) + activeHeader := f.headerSnapshot(t, string(active.ID)) + + if err := f.store.CleanupExplicitGroupStaging(f.ctx, now, 100); err != nil { + f.fail("cleanup", err) + } + + for _, gone := range []explorer.ExplicitGroupRevisionID{expired.ID, nullLease.ID, young.ID} { + if got := f.countRows(t, f.revisionTable(), "id = $1", string(gone)); got != 0 { + t.Fatalf("expired header %s count = %d, want 0", gone, got) + } + } + if got := f.countRows(t, f.definitionsTable(), "revision_id = $1", string(expired.ID)); got != 0 { + t.Fatalf("expired definitions count = %d, want 0 (cascade)", got) + } + if got := f.headerSnapshot(t, string(active.ID)); got != activeHeader { + t.Fatalf("cleanup touched the active writer:\nbefore %q\nafter %q", activeHeader, got) + } + if got := f.definitionsSnapshot(t, string(active.ID)); got != activeDefs { + t.Fatalf("cleanup touched the active writer's definitions:\nbefore %q\nafter %q", activeDefs, got) + } + + liveKey := "cl-live-" + pgstate06RandomHex(t) + live := pgstate06Revision(source, liveKey) + liveToken := "writer-live-" + pgstate06RandomHex(t) + f.rawInsertHeader(t, f.witness, live, liveToken, &now) + if err := f.store.CleanupExplicitGroupStaging(f.ctx, now, 100); err != nil { + f.fail("cleanup live", err) + } + if got := f.countRows(t, f.revisionTable(), "id = $1", string(live.ID)); got != 1 { + t.Fatalf("row created at the before boundary was reaped, want kept (before bound)") + } + + var bounded []explorer.ExplicitGroupRevisionID + for i := 0; i < 3; i++ { + seed := pgstate06Revision(source, fmt.Sprintf("cl-bound-%d-%s", i, pgstate06RandomHex(t))) + created := now.Add(-time.Duration(3-i) * time.Minute) + f.rawInsertHeader(t, f.witness, explorer.ExplicitGroupRevision{ + ID: seed.ID, Project: seed.Project, Generation: seed.Generation, + ScopeDigest: seed.ScopeDigest, ResourceType: seed.ResourceType, + SourceSelectionRevisionID: seed.SourceSelectionRevisionID, + SourceMembershipDigest: seed.SourceMembershipDigest, + State: explorer.ExplicitGroupRevisionStaging, + IdempotencyKey: seed.IdempotencyKey, + CreatedAt: created, + }, "seed-writer", func() *time.Time { e := created.Add(-time.Second); return &e }()) + bounded = append(bounded, seed.ID) + } + if err := f.store.CleanupExplicitGroupStaging(f.ctx, now.Add(time.Minute), 2); err != nil { + f.fail("bounded cleanup", err) + } + remaining := 0 + for _, id := range bounded { + remaining += f.countRows(t, f.revisionTable(), "id = $1", string(id)) + } + if remaining != 1 { + t.Fatalf("bounded sweep left %d rows, want 1", remaining) + } + if err := f.store.CleanupExplicitGroupStaging(f.ctx, now.Add(time.Minute), 0); err != nil { + f.fail("default-limit cleanup", err) + } + remaining = 0 + for _, id := range bounded { + remaining += f.countRows(t, f.revisionTable(), "id = $1", string(id)) + } + if remaining != 0 { + t.Fatalf("follow-up sweep left %d rows, want 0", remaining) + } + + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, active, source, "seed-writer"); err != nil { + f.fail("active writer begin retry after cleanup", err) + } + mustNil(t, "active writer put after cleanup", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, active.ID, "seed-writer", []explorer.ExplicitGroupDefinition{ + {ID: "group-a", Label: "A", Ordinal: 0}, + }) + }) +} + +func TestPostgresExplicitGroupDefinitionsCorruptionDetection(t *testing.T) { + f := newPGState06Fixture(t) + f.migrate(t) + + project := "pgstate06-project" + source := pgstate06Selection(project, "corrupt-"+pgstate06RandomHex(t)) + token := "writer-" + pgstate06RandomHex(t) + + key := "corrupt-" + pgstate06RandomHex(t) + revision := pgstate06Revision(source, key) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, token); err != nil { + f.fail("begin", err) + } + f.rawInsertDefinition(t, f.witness, string(revision.ID), "group-padded", " padded ", 0) + err := f.store.PutExplicitGroupDefinitions(f.ctx, revision.ID, token, []explorer.ExplicitGroupDefinition{ + {ID: "group-padded", Label: "padded", Ordinal: 0}, + }) + if !errors.Is(err, explorer.ErrCorruptExplicitGroupRevision) { + t.Fatalf("put over non-canonical row = %v, want ErrCorruptExplicitGroupRevision", err) + } + if got := f.countRows(t, f.definitionsTable(), "revision_id = $1", string(revision.ID)); got != 1 { + t.Fatalf("corruption refusal left %d rows, want the single forged row untouched", got) + } + + cleanKey := "corrupt-clean-" + pgstate06RandomHex(t) + clean := pgstate06Revision(source, cleanKey) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, clean, source, token); err != nil { + f.fail("begin clean", err) + } + mustNil(t, "clean put", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, clean.ID, token, []explorer.ExplicitGroupDefinition{ + {ID: "group-padded", Label: "padded", Ordinal: 0}, + }) + }) + + _, err = f.witness.Exec(f.ctx, `UPDATE `+f.revisionTable()+` SET idempotency_key = 'rewritten' WHERE id = $1`, string(clean.ID)) + requirePgErrTriggerPGState06(t, err, pgstate06CheckViolation, + "explorer_explicit_group_revisions guard", "rewriting the idempotency key is frozen content") +} + +func (f *pgState06Fixture) raceSetup(t *testing.T) (storeA, storeB *ExplicitGroupStore) { + t.Helper() + storeA, pidA := f.newDedicatedStore(t) + storeB, pidB := f.newDedicatedStore(t) + requireDistinctBackendsPGState06(t, map[string]int32{"storeA": pidA, "storeB": pidB, "witness": f.witnessID}) + return storeA, storeB +} + +func raceFire(fnA, fnB func() error) (errA, errB error) { + start := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + <-start + errA = fnA() + }() + go func() { + defer wg.Done() + <-start + errB = fnB() + }() + close(start) + wg.Wait() + return errA, errB +} + +func exactlyOneConflict(errA, errB error) bool { + conflictA := errors.Is(errA, explorer.ErrExplicitGroupRevisionConflict) + conflictB := errors.Is(errB, explorer.ErrExplicitGroupRevisionConflict) + return (errA == nil && conflictB) || (conflictA && errB == nil) +} + +func TestPostgresExplicitGroupDefinitionsConcurrentClaimRace(t *testing.T) { + f := newPGState06Fixture(t) + f.migrate(t) + storeA, storeB := f.raceSetup(t) + + project := "pgstate06-project" + source := pgstate06Selection(project, "race-claim-"+pgstate06RandomHex(t)) + + const rounds = 12 + for round := 0; round < rounds; round++ { + key := fmt.Sprintf("race-claim-%d-%s", round, pgstate06RandomHex(t)) + revision := pgstate06Revision(source, key) + tokenA := "race-writer-a-" + pgstate06RandomHex(t) + tokenB := "race-writer-b-" + pgstate06RandomHex(t) + + var headerA, headerB *explorer.ExplicitGroupRevision + errA, errB := raceFire( + func() error { + var err error + headerA, err = storeA.BeginExplicitGroupRevision(f.ctx, revision, source, tokenA) + return err + }, + func() error { + var err error + headerB, err = storeB.BeginExplicitGroupRevision(f.ctx, revision, source, tokenB) + return err + }, + ) + if !exactlyOneConflict(errA, errB) { + t.Fatalf("round %d: claim results errA=%v errB=%v, want one success and one typed conflict", round, errA, errB) + } + + if got := f.countRows(t, f.revisionTable(), "id = $1", string(revision.ID)); got != 1 { + t.Fatalf("round %d: header count = %d, want exactly 1", round, got) + } + var storedToken string + if err := f.witness.QueryRow(f.ctx, `SELECT writer_token FROM `+f.revisionTable()+` WHERE id = $1`, string(revision.ID)).Scan(&storedToken); err != nil { + f.fail("witness claim token", err) + } + if storedToken != tokenA && storedToken != tokenB { + t.Fatalf("round %d: stored writer token %q belongs to neither racer", round, storedToken) + } + winnerToken, loserToken := tokenA, tokenB + if storedToken == tokenB { + winnerToken, loserToken = tokenB, tokenA + } + + before := f.headerSnapshot(t, string(revision.ID)) + mustErr(t, fmt.Sprintf("round %d loser re-claim", round), func() error { + _, err := storeB.BeginExplicitGroupRevision(f.ctx, revision, source, loserToken) + return err + }, explorer.ErrExplicitGroupRevisionConflict) + if got := f.headerSnapshot(t, string(revision.ID)); got != before { + t.Fatalf("round %d: loser re-claim mutated the winner's row:\nbefore %q\nafter %q", round, before, got) + } + + mustNil(t, fmt.Sprintf("round %d winner retry before abort", round), func() error { + _, err := storeA.BeginExplicitGroupRevision(f.ctx, revision, source, winnerToken) + return err + }) + mustNil(t, fmt.Sprintf("round %d winner abort", round), func() error { + return storeA.AbortExplicitGroupRevision(f.ctx, revision.ID, winnerToken) + }) + if got := f.countRows(t, f.revisionTable(), "id = $1", string(revision.ID)); got != 0 { + t.Fatalf("round %d: after abort header count = %d, want 0", round, got) + } + if got := f.countRows(t, f.definitionsTable(), "revision_id = $1", string(revision.ID)); got != 0 { + t.Fatalf("round %d: after abort definition count = %d, want 0", round, got) + } + _ = headerA + _ = headerB + } +} + +func TestPostgresExplicitGroupDefinitionsConcurrentPutRace(t *testing.T) { + f := newPGState06Fixture(t) + f.migrate(t) + storeA, storeB := f.raceSetup(t) + + project := "pgstate06-project" + source := pgstate06Selection(project, "race-put-"+pgstate06RandomHex(t)) + otherSource := pgstate06Selection(project, "race-put-other-"+pgstate06RandomHex(t)) + _ = otherSource + + const rounds = 11 + + wantCanonical := strings.Join([]string{ + "group-alpha|Alpha cohort|0", + "group-empty|No members|1", + "group-beta|Beta cohort|2", + }, "\x1e") + keyIdentical := "race-put-identical-" + pgstate06RandomHex(t) + revisionIdentical := pgstate06Revision(source, keyIdentical) + token := "race-put-owner-" + pgstate06RandomHex(t) + if _, err := storeA.BeginExplicitGroupRevision(f.ctx, revisionIdentical, source, token); err != nil { + f.fail("begin identical-race revision", err) + } + for round := 0; round < rounds; round++ { + errA, errB := raceFire( + func() error { + return storeA.PutExplicitGroupDefinitions(f.ctx, revisionIdentical.ID, token, pgstate06Definitions()) + }, + func() error { + return storeB.PutExplicitGroupDefinitions(f.ctx, revisionIdentical.ID, token, pgstate06Definitions()) + }, + ) + if errA != nil || errB != nil { + t.Fatalf("identical put round %d: errA=%v errB=%v, want both nil (idempotent replay)", round, errA, errB) + } + if got := f.definitionsSnapshot(t, string(revisionIdentical.ID)); got != wantCanonical { + t.Fatalf("identical put round %d: stored definitions = %q, want byte-identical %q", round, got, wantCanonical) + } + } + if got := f.countRows(t, f.definitionsTable(), "revision_id = $1", string(revisionIdentical.ID)); got != 3 { + t.Fatalf("identical puts stored %d definition rows, want exactly 3", got) + } + + shapeA := pgstate06CanonicalDefinitions(t) + shapeB, err := explorer.CanonicalExplicitGroupDefinitions([]explorer.ExplicitGroupDefinition{ + {ID: "group-alpha", Label: "Alpha cohort", Ordinal: 0}, + {ID: "group-empty", Label: "No members at all", Ordinal: 1}, + {ID: "group-beta", Label: "Beta cohort", Ordinal: 2}, + }) + if err != nil { + t.Fatalf("canonicalize shape B: %v", err) + } + renderShape := func(shape []explorer.ExplicitGroupDefinition) string { + parts := make([]string, 0, len(shape)) + for _, g := range shape { + parts = append(parts, fmt.Sprintf("%s|%s|%d", g.ID, g.Label, g.Ordinal)) + } + return strings.Join(parts, "\x1e") + } + for round := 0; round < rounds; round++ { + key := fmt.Sprintf("race-put-conflict-%d-%s", round, pgstate06RandomHex(t)) + revision := pgstate06Revision(source, key) + raceToken := "race-put-token-" + pgstate06RandomHex(t) + if _, err := storeA.BeginExplicitGroupRevision(f.ctx, revision, source, raceToken); err != nil { + f.fail("begin conflict-race revision", err) + } + errA, errB := raceFire( + func() error { return storeA.PutExplicitGroupDefinitions(f.ctx, revision.ID, raceToken, shapeA) }, + func() error { return storeB.PutExplicitGroupDefinitions(f.ctx, revision.ID, raceToken, shapeB) }, + ) + if !exactlyOneConflict(errA, errB) { + t.Fatalf("conflicting put round %d: errA=%v errB=%v, want one success and one typed conflict", round, errA, errB) + } + stored := f.definitionsSnapshot(t, string(revision.ID)) + if stored != renderShape(shapeA) && stored != renderShape(shapeB) { + t.Fatalf("conflicting put round %d: stored definitions = %q, want exactly one canonical shape", round, stored) + } + if got := f.countRows(t, f.definitionsTable(), "revision_id = $1", string(revision.ID)); got != 3 { + t.Fatalf("conflicting put round %d: definition count = %d, want 3 (no interleaved halves)", round, got) + } + mustNil(t, fmt.Sprintf("conflicting put round %d owner abort", round), func() error { + return storeA.AbortExplicitGroupRevision(f.ctx, revision.ID, raceToken) + }) + if got := f.countRows(t, f.definitionsTable(), "revision_id = $1", string(revision.ID)); got != 0 { + t.Fatalf("conflicting put round %d: after abort definition count = %d, want 0", round, got) + } + if got := f.countRows(t, f.revisionTable(), "id = $1", string(revision.ID)); got != 0 { + t.Fatalf("conflicting put round %d: after abort header count = %d, want 0", round, got) + } + } +} + +func TestPostgresExplicitGroupDefinitionsCleanupVersusActiveWriterRace(t *testing.T) { + f := newPGState06Fixture(t) + f.migrate(t) + storeA, storeB := f.raceSetup(t) + + project := "pgstate06-project" + source := pgstate06Selection(project, "race-cleanup-"+pgstate06RandomHex(t)) + + const rounds = 10 + for round := 0; round < rounds; round++ { + key := fmt.Sprintf("race-cleanup-%d-%s", round, pgstate06RandomHex(t)) + revision := pgstate06Revision(source, key) + writerToken := "race-cleanup-writer-" + pgstate06RandomHex(t) + + if _, err := storeA.BeginExplicitGroupRevision(f.ctx, revision, source, writerToken); err != nil { + f.fail("begin racing revision", err) + } + if err := storeA.PutExplicitGroupDefinitions(f.ctx, revision.ID, writerToken, pgstate06Definitions()); err != nil { + f.fail("seed racing definitions", err) + } + headerBefore := f.headerSnapshot(t, string(revision.ID)) + defsBefore := f.definitionsSnapshot(t, string(revision.ID)) + + baitKey := fmt.Sprintf("race-cleanup-bait-%d-%s", round, pgstate06RandomHex(t)) + bait := pgstate06Revision(source, baitKey) + past := time.Now().UTC().Add(-time.Hour) + baitLease := time.Now().UTC().Add(-time.Minute) + f.rawInsertHeader(t, f.witness, explorer.ExplicitGroupRevision{ + ID: bait.ID, Project: bait.Project, Generation: bait.Generation, + ScopeDigest: bait.ScopeDigest, ResourceType: bait.ResourceType, + SourceSelectionRevisionID: bait.SourceSelectionRevisionID, + SourceMembershipDigest: bait.SourceMembershipDigest, + State: explorer.ExplicitGroupRevisionStaging, + IdempotencyKey: bait.IdempotencyKey, + CreatedAt: past, + }, "bait-writer", &baitLease) + f.rawInsertDefinition(t, f.witness, string(bait.ID), "group-bait", "Bait", 0) + + errA, errB := raceFire( + func() error { + if _, err := storeA.BeginExplicitGroupRevision(f.ctx, revision, source, writerToken); err != nil { + return err + } + return storeA.PutExplicitGroupDefinitions(f.ctx, revision.ID, writerToken, pgstate06Definitions()) + }, + func() error { + return storeB.CleanupExplicitGroupStaging(f.ctx, time.Now().UTC(), 100) + }, + ) + if errA != nil { + t.Fatalf("cleanup race round %d: active writer errored: %v", round, errA) + } + if errB != nil { + t.Fatalf("cleanup race round %d: sweeper errored: %v", round, errB) + } + + if got := f.frozenHeaderSnapshot(t, string(revision.ID)); got != strings.Join(strings.Split(headerBefore, "\x1f")[:15], "\x1f") { + t.Fatalf("cleanup race round %d: active writer's frozen fields changed:\nbefore %q\nafter %q", + round, strings.Join(strings.Split(headerBefore, "\x1f")[:15], "\x1f"), got) + } + if got := f.definitionsSnapshot(t, string(revision.ID)); got != defsBefore { + t.Fatalf("cleanup race round %d: active writer's definitions changed:\nbefore %q\nafter %q", round, defsBefore, got) + } + if got := f.countRows(t, f.revisionTable(), "id = $1", string(bait.ID)); got != 0 { + t.Fatalf("cleanup race round %d: bait header count = %d, want 0", round, got) + } + if got := f.countRows(t, f.definitionsTable(), "revision_id = $1", string(bait.ID)); got != 0 { + t.Fatalf("cleanup race round %d: bait definition count = %d, want 0", round, got) + } + mustNil(t, fmt.Sprintf("cleanup race round %d owner abort", round), func() error { + return storeA.AbortExplicitGroupRevision(f.ctx, revision.ID, writerToken) + }) + if got := f.countRows(t, f.revisionTable(), "id = $1", string(revision.ID)); got != 0 { + t.Fatalf("cleanup race round %d: after abort header count = %d, want 0", round, got) + } + } +} + +func TestPostgresExplicitGroupDefinitionsFaultInjectionRollback(t *testing.T) { + f := newPGState06Fixture(t) + f.migrate(t) + + project := "pgstate06-project" + source := pgstate06Selection(project, "fault-"+pgstate06RandomHex(t)) + token := "writer-" + pgstate06RandomHex(t) + + key := "fault-put-" + pgstate06RandomHex(t) + revision := pgstate06Revision(source, key) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, token); err != nil { + f.fail("begin", err) + } + headerBefore := f.headerSnapshot(t, string(revision.ID)) + fault := errors.New("injected put fault") + f.store.afterDefinitionsInserted = func(context.Context, pgx.Tx) error { return fault } + err := f.store.PutExplicitGroupDefinitions(f.ctx, revision.ID, token, pgstate06Definitions()) + if err == nil || !errors.Is(err, fault) { + t.Fatalf("put with injected fault = %v, want the fault to surface", err) + } + f.store.afterDefinitionsInserted = nil + if got := f.countRows(t, f.definitionsTable(), "revision_id = $1", string(revision.ID)); got != 0 { + t.Fatalf("aborted put left %d definition rows, want 0", got) + } + if got := f.headerSnapshot(t, string(revision.ID)); got != headerBefore { + t.Fatalf("aborted put moved the header row:\nbefore %q\nafter %q", headerBefore, got) + } + mustNil(t, "put after fault", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, revision.ID, token, pgstate06Definitions()) + }) + defsAfter := f.definitionsSnapshot(t, string(revision.ID)) + if !strings.Contains(defsAfter, "group-alpha|Alpha cohort|0") || !strings.Contains(defsAfter, "group-empty|No members|1") { + t.Fatalf("post-fault put stored %q, want the canonical set", defsAfter) + } + + keyAbort := "fault-abort-" + pgstate06RandomHex(t) + revisionAbort := pgstate06Revision(source, keyAbort) + tokenAbort := "writer-abort-" + pgstate06RandomHex(t) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, revisionAbort, source, tokenAbort); err != nil { + f.fail("begin abort fixture", err) + } + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, revisionAbort, source, tokenAbort); err != nil { + f.fail("warm lease", err) + } + mustNil(t, "put abort fixture", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, revisionAbort.ID, tokenAbort, pgstate06Definitions()) + }) + headerBeforeAbort := f.headerSnapshot(t, string(revisionAbort.ID)) + defsBeforeAbort := f.definitionsSnapshot(t, string(revisionAbort.ID)) + f.store.afterDefinitionsDeleted = func(context.Context, pgx.Tx) error { return fault } + err = f.store.AbortExplicitGroupRevision(f.ctx, revisionAbort.ID, tokenAbort) + if err == nil || !errors.Is(err, fault) { + t.Fatalf("abort with injected fault = %v, want the fault to surface", err) + } + f.store.afterDefinitionsDeleted = nil + if got := f.headerSnapshot(t, string(revisionAbort.ID)); got != headerBeforeAbort { + t.Fatalf("aborted abort removed the header:\nbefore %q\nafter %q", headerBeforeAbort, got) + } + if got := f.definitionsSnapshot(t, string(revisionAbort.ID)); got != defsBeforeAbort { + t.Fatalf("aborted abort removed definition rows:\nbefore %q\nafter %q", defsBeforeAbort, got) + } + mustNil(t, "abort after fault", func() error { + return f.store.AbortExplicitGroupRevision(f.ctx, revisionAbort.ID, tokenAbort) + }) + if got := f.countRows(t, f.revisionTable(), "id = $1", string(revisionAbort.ID)); got != 0 { + t.Fatalf("post-fault abort left header count = %d, want 0", got) + } + if got := f.countRows(t, f.definitionsTable(), "revision_id = $1", string(revisionAbort.ID)); got != 0 { + t.Fatalf("post-fault abort left definition count = %d, want 0", got) + } +} + +func TestPostgresExplicitGroupDefinitionsMigrationLedgerConvergence(t *testing.T) { + f := newPGState06Fixture(t) + first := f.migrate(t) + + owner, err := OwnerMigrations() + if err != nil { + f.fail("owner migrations", err) + } + + embeddedFiles, embeddedMax := pgstate02EnumerateEmbeddedMigrations(t) + if len(owner) != len(embeddedFiles) { + t.Fatalf("embedded history has %d migrations, want %d (loader and the migrations directory disagree)", len(owner), len(embeddedFiles)) + } + for i, file := range embeddedFiles { + fileVersion, err := pgstate02MigrationFileVersion(file) + if err != nil { + f.fail("parse embedded migration file name", err) + } + if owner[i].Version != fileVersion { + t.Fatalf("loader/directory disagreement: migrations/%s carries version %d, loader reports %d at index %d", + file, fileVersion, owner[i].Version, i) + } + } + + type ledgerRow struct { + version int64 + name string + checksum string + } + readLedger := func() []ledgerRow { + rows, err := f.witness.Query(f.ctx, `SELECT version, name, checksum FROM `+ + sanitizeTableRef(f.schema, "schema_migrations")+` ORDER BY version`) + if err != nil { + f.fail("read ledger", err) + } + defer rows.Close() + var out []ledgerRow + for rows.Next() { + var rec ledgerRow + if err := rows.Scan(&rec.version, &rec.name, &rec.checksum); err != nil { + f.fail("scan ledger", err) + } + out = append(out, rec) + } + if err := rows.Err(); err != nil { + f.fail("read ledger", err) + } + return out + } + ledger := readLedger() + if len(ledger) != len(embeddedFiles) { + t.Fatalf("ledger holds %d rows, want %d (one per embedded migration file)", len(ledger), len(embeddedFiles)) + } + for i, migration := range owner { + recomputed := sha256.Sum256([]byte(migration.SQL)) + hexRecomputed := hex.EncodeToString(recomputed[:]) + if migration.Checksum != hexRecomputed { + t.Fatalf("migration %d/%s loader checksum %s != self-recomputed %s", + migration.Version, migration.Name, migration.Checksum, hexRecomputed) + } + if ledger[i] != (ledgerRow{migration.Version, migration.Name, hexRecomputed}) { + t.Fatalf("ledger[%d] = %+v, want %+v with self-recomputed checksum %s", + i, ledger[i], migration, hexRecomputed) + } + if migration.Version == pgstate06MigrationVersion { + if migration.Name != pgstate06MigrationName { + t.Fatalf("explicit-groups migration name = %q, want %q", migration.Name, pgstate06MigrationName) + } + if hexRecomputed != pgstate06MigrationChecksum { + t.Fatalf("explicit-groups migration checksum = %s, want the byte-frozen pin %s: the migration drifted", + hexRecomputed, pgstate06MigrationChecksum) + } + } + } + + source := pgstate06Selection("pgstate06-project", "ledger-"+pgstate06RandomHex(t)) + key := "ledger-" + pgstate06RandomHex(t) + revision := pgstate06Revision(source, key) + token := "writer-" + pgstate06RandomHex(t) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, token); err != nil { + f.fail("begin ledger fixture", err) + } + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, token); err != nil { + f.fail("renew ledger fixture", err) + } + headerBefore := f.headerSnapshot(t, string(revision.ID)) + + second := f.migrate(t) + if len(first.Applied) != len(second.Applied) || first.LatestVersion != second.LatestVersion { + t.Fatalf("rerun result = %+v, first = %+v, want the identical convergence", second, first) + } + if second.LatestVersion != embeddedMax { + t.Fatalf("latest ledger version = %d, want %d (the maximum version parsed from the embedded migrations directory)", + second.LatestVersion, embeddedMax) + } + if got := readLedger(); len(got) != len(ledger) { + t.Fatalf("ledger after rerun holds %d rows, want %d", len(got), len(ledger)) + } + if got := f.headerSnapshot(t, string(revision.ID)); got != headerBefore { + t.Fatalf("migration rerun mutated staged state:\nbefore %q\nafter %q", headerBefore, got) + } + + pgStoreClient, err := pgstore.Open(f.ctx, pgstore.Config{DSN: f.dsn, MaxConns: 2, MinConns: 0, AcquireTimeout: 10 * time.Second}) + if err != nil { + f.fail("open mixed-history pool", err) + } + defer pgStoreClient.Close() + competing := fmt.Sprintf("pgstate06_it_%d_%s", time.Now().UnixNano(), pgstate06RandomHex(t)) + if _, err := f.iso.Exec(f.ctx, `CREATE SCHEMA "`+competing+`"`); err != nil { + f.fail("create competing schema", err) + } + if _, err := pgstore.MigrateSchema(f.ctx, pgStoreClient, competing); err != nil { + f.fail("bootstrap competing schema with source-store history", err) + } + _, err = MigrateSchema(f.ctx, pgStoreClient, competing) + if !errors.Is(err, pgstore.ErrMigrationChecksumMismatch) { + t.Fatalf("mixed history = %v, want ErrMigrationChecksumMismatch fail-closed", err) + } + if _, err := f.iso.Exec(f.ctx, `DROP SCHEMA IF EXISTS "`+competing+`" CASCADE`); err != nil { + f.fail("drop competing schema", err) + } +} + +func TestZZNoLeftoverPgstate06Resources(t *testing.T) { + raw := pgstate06TestPostgresURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for zero-leftover audit: %v", err) + } + defer admin.Close() + var databases int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname LIKE 'pgstate06\_it\_%' ESCAPE '\'`).Scan(&databases); err != nil { + t.Fatalf("audit databases: %v", err) + } + if databases != 0 { + t.Errorf("zero-leftover audit: %d pgstate06_it_* databases remain", databases) + } + var schemas int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgstate06\_it\_%' ESCAPE '\'`).Scan(&schemas); err != nil { + t.Fatalf("audit schemas: %v", err) + } + if schemas != 0 { + t.Errorf("zero-leftover audit: %d pgstate06_it_* schemas remain in the service database", schemas) + } +} + +func TestPostgresExplicitGroupDefinitionsHeaderDatabaseGuards(t *testing.T) { + f := newPGState06Fixture(t) + f.migrate(t) + + project := "pgstate06-project" + source := pgstate06Selection(project, "guards-"+pgstate06RandomHex(t)) + token := "writer-" + pgstate06RandomHex(t) + + newHeader := func(label string) explorer.ExplicitGroupRevision { + key := "guard-" + label + "-" + pgstate06RandomHex(t) + revision := pgstate06Revision(source, key) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, token); err != nil { + f.fail("begin guard fixture "+label, err) + } + return revision + } + + honest := pgstate06Revision(source, "check-honest-"+pgstate06RandomHex(t)) + f.rawInsertHeader(t, f.witness, honest, "tok", nil) + if got := f.countRows(t, f.revisionTable(), "id = $1", string(honest.ID)); got != 1 { + t.Fatalf("honest raw header count = %d, want 1", got) + } + + forgedID := pgstate06Revision(source, "check-forged-id-"+pgstate06RandomHex(t)) + forgedID.ID = explorer.ExplicitGroupRevisionID("grouprev_" + strings.Repeat("0", 64)) + _, err := f.witness.Exec(f.ctx, `INSERT INTO `+f.revisionTable()+` (id, project, generation, scope_digest, + resource_type, source_selection_revision_id, source_membership_digest, state, idempotency_key, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, 'STAGING', $8, now())`, + string(forgedID.ID), forgedID.Project, forgedID.Generation, forgedID.ScopeDigest, forgedID.ResourceType, + forgedID.SourceSelectionRevisionID, forgedID.SourceMembershipDigest, forgedID.IdempotencyKey) + requirePgErrConstraintPGState06(t, err, pgstate06CheckViolation, + explicitGroupRevisionIdentityCheck, "forged id refused by the named identity-binding CHECK") + + complete := pgstate06Revision(source, "check-complete-"+pgstate06RandomHex(t)) + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.revisionTable()+` (id, project, generation, scope_digest, + resource_type, source_selection_revision_id, source_membership_digest, state, idempotency_key, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, 'COMPLETE', $8, now())`, + string(complete.ID), complete.Project, complete.Generation, complete.ScopeDigest, complete.ResourceType, + complete.SourceSelectionRevisionID, complete.SourceMembershipDigest, complete.IdempotencyKey) + requirePgErrTriggerPGState06(t, err, pgstate06CheckViolation, + "explorer_explicit_group_revisions guard", + "COMPLETE header INSERT refused by the guard: headers are born STAGING (000010 INSERT branch)") + + bogusState := pgstate06Revision(source, "check-bogus-"+pgstate06RandomHex(t)) + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.revisionTable()+` (id, project, generation, scope_digest, + resource_type, source_selection_revision_id, source_membership_digest, state, idempotency_key, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, 'BOGUS', $8, now())`, + string(bogusState.ID), bogusState.Project, bogusState.Generation, bogusState.ScopeDigest, bogusState.ResourceType, + bogusState.SourceSelectionRevisionID, bogusState.SourceMembershipDigest, bogusState.IdempotencyKey) + requirePgErrTriggerPGState06(t, err, pgstate06CheckViolation, + "explorer_explicit_group_revisions guard", + "BOGUS header INSERT refused at the door by the guard (with the trigger in place no UPDATE or INSERT ever reaches the lifecycle CHECK on state; that CHECK's named classification is witnessed in the membership suite's completion-machine test)") + + stagingWithCompletion := pgstate06Revision(source, "check-staging-values-"+pgstate06RandomHex(t)) + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.revisionTable()+` (id, project, generation, scope_digest, + resource_type, source_selection_revision_id, source_membership_digest, definition_digest, state, idempotency_key, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, 'not-empty', 'STAGING', $8, now())`, + string(stagingWithCompletion.ID), stagingWithCompletion.Project, stagingWithCompletion.Generation, + stagingWithCompletion.ScopeDigest, stagingWithCompletion.ResourceType, + stagingWithCompletion.SourceSelectionRevisionID, stagingWithCompletion.SourceMembershipDigest, + stagingWithCompletion.IdempotencyKey) + requirePgErrConstraintPGState06(t, err, pgstate06CheckViolation, + explicitGroupRevisionLifecycleValuesCheck, "staging header with completion values refused by the named lifecycle CHECK") + + blankField := pgstate06Revision(source, "check-required-"+pgstate06RandomHex(t)) + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.revisionTable()+` (id, project, generation, scope_digest, + resource_type, source_selection_revision_id, source_membership_digest, state, idempotency_key, created_at) + VALUES ($1, $2::text, $3::text, $4, $5, $6, $7, 'STAGING', $8, now())`, + string(blankField.ID), blankField.Project, "", blankField.ScopeDigest, blankField.ResourceType, + blankField.SourceSelectionRevisionID, blankField.SourceMembershipDigest, blankField.IdempotencyKey) + requirePgErrConstraintPGState06(t, err, pgstate06CheckViolation, + explicitGroupRevisionRequiredFieldsCK, "blank required field refused by the named CHECK") + + guarded := newHeader("guarded") + frozenUpdate := func(sql string, args ...any) error { + _, err := f.witness.Exec(f.ctx, sql, args...) + return err + } + requirePgErrTriggerPGState06(t, frozenUpdate(`UPDATE `+f.revisionTable()+` SET project = 'hijacked' WHERE id = $1`, string(guarded.ID)), + pgstate06CheckViolation, "explorer_explicit_group_revisions guard", "frozen column UPDATE names its trigger") + requirePgErrTriggerPGState06(t, frozenUpdate(`UPDATE `+f.revisionTable()+` SET state = 'COMPLETE' WHERE id = $1`, string(guarded.ID)), + pgstate06CheckViolation, "explorer_explicit_group_revisions guard", + "direct completion refused by the guarded transition: no writer token held") + requirePgErrTriggerPGState06(t, frozenUpdate(`UPDATE `+f.revisionTable()+` SET completed_at = now() WHERE id = $1`, string(guarded.ID)), + pgstate06CheckViolation, "explorer_explicit_group_revisions guard", "completed_at is frozen content") + requirePgErrTriggerPGState06(t, frozenUpdate(`UPDATE `+f.revisionTable()+` SET writer_expires_at = now() + interval '1 hour' WHERE id = $1`, string(guarded.ID)), + pgstate06CheckViolation, "explorer_explicit_group_revisions guard", "lease renewal without the writer-token GUC is refused") + + frozenBefore := f.frozenHeaderSnapshot(t, string(guarded.ID)) + beforeLease := strings.Split(f.headerSnapshot(t, string(guarded.ID)), "\x1f")[16] + if _, err := f.witness.Exec(f.ctx, `SELECT set_config('`+gucWriterToken+`', $1, false)`, token); err != nil { + f.fail("set writer token GUC on witness", err) + } + if _, err := f.witness.Exec(f.ctx, `UPDATE `+f.revisionTable()+` SET writer_expires_at = now() + interval '2 hours' WHERE id = $1`, string(guarded.ID)); err != nil { + f.fail("authorized lease renewal", err) + } + afterLease := strings.Split(f.headerSnapshot(t, string(guarded.ID)), "\x1f")[16] + if afterLease == beforeLease { + t.Fatal("authorized lease renewal did not move the deadline") + } + if got := f.frozenHeaderSnapshot(t, string(guarded.ID)); got != frozenBefore { + t.Fatalf("authorized renewal changed frozen columns:\nbefore %q\nafter %q", frozenBefore, got) + } + if _, err := f.witness.Exec(f.ctx, `SELECT set_config('`+gucWriterToken+`', NULL, false)`); err != nil { + f.fail("clear writer token GUC on witness", err) + } + + shortenProbe := func(sql string) { + if _, err := f.witness.Exec(f.ctx, `SELECT set_config('`+gucWriterToken+`', $1, false)`, token); err != nil { + f.fail("set writer token GUC", err) + } + _, err := f.witness.Exec(f.ctx, sql, string(guarded.ID)) + requirePgErrTriggerPGState06(t, err, pgstate06CheckViolation, + "explorer_explicit_group_revisions guard", "lease monotonicity guard") + if _, err := f.witness.Exec(f.ctx, `SELECT set_config('`+gucWriterToken+`', NULL, false)`); err != nil { + f.fail("clear writer token GUC", err) + } + } + shortenProbe(`UPDATE ` + f.revisionTable() + ` SET writer_expires_at = '2020-01-01T00:00:00Z'::timestamptz WHERE id = $1`) + shortenProbe(`UPDATE ` + f.revisionTable() + ` SET writer_expires_at = NULL WHERE id = $1`) + + requirePgErrTriggerPGState06(t, frozenUpdate(`DELETE FROM `+f.revisionTable()+` WHERE id = $1`, string(guarded.ID)), + pgstate06CheckViolation, "explorer_explicit_group_revisions guard", "unauthorized DELETE names its trigger") + + f.dropTrigger(t, f.revisionTable(), "explorer_explicit_group_revisions_guard") + unguarded := newHeader("unguarded") + mustNil(t, "unguarded frozen UPDATE succeeds once the guard is dropped", func() error { + _, err := f.witness.Exec(f.ctx, `UPDATE `+f.revisionTable()+` SET source_membership_digest = 'hijacked' WHERE id = $1`, string(unguarded.ID)) + return err + }) + mustNil(t, "unguarded DELETE succeeds once the guard is dropped", func() error { + _, err := f.witness.Exec(f.ctx, `DELETE FROM `+f.revisionTable()+` WHERE id = $1`, string(unguarded.ID)) + return err + }) + if got := f.countRows(t, f.revisionTable(), "id = $1", string(unguarded.ID)); got != 0 { + t.Fatalf("unguarded DELETE left %d rows, want 0 (probe must actually delete)", got) + } + f.rerunMigration(t, pgstate06MigrationVersion) + requirePgErrTriggerPGState06(t, frozenUpdate(`UPDATE `+f.revisionTable()+` SET source_membership_digest = 'hijacked2' WHERE id = $1`, string(guarded.ID)), + pgstate06CheckViolation, "explorer_explicit_group_revisions guard", "rerun migration re-blocks the frozen UPDATE") + requirePgErrTriggerPGState06(t, frozenUpdate(`DELETE FROM `+f.revisionTable()+` WHERE id = $1`, string(guarded.ID)), + pgstate06CheckViolation, "explorer_explicit_group_revisions guard", "rerun migration re-blocks the DELETE") +} + +func TestPostgresExplicitGroupDefinitionsDefinitionDatabaseGuards(t *testing.T) { + f := newPGState06Fixture(t) + f.migrate(t) + + project := "pgstate06-project" + source := pgstate06Selection(project, "defguards-"+pgstate06RandomHex(t)) + token := "writer-" + pgstate06RandomHex(t) + key := "defguards-" + pgstate06RandomHex(t) + revision := pgstate06Revision(source, key) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, token); err != nil { + f.fail("begin", err) + } + mustNil(t, "put", func() error { + return f.store.PutExplicitGroupDefinitions(f.ctx, revision.ID, token, pgstate06Definitions()) + }) + want := f.definitionsSnapshot(t, string(revision.ID)) + + _, err := f.witness.Exec(f.ctx, `INSERT INTO `+f.definitionsTable()+` (revision_id, group_id, label, ordinal) + VALUES ($1, 'group-alpha', 'Dup alpha', 99)`, string(revision.ID)) + requirePgErrConstraintPGState06(t, err, "23505", explicitGroupDefinitionPKConstraint, + "duplicate (revision, group) refused by the named PK constraint") + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.definitionsTable()+` (revision_id, group_id, label, ordinal) + VALUES ($1, 'group-new', 'Ordinal clash', 2)`, string(revision.ID)) + requirePgErrConstraintPGState06(t, err, "23505", explicitGroupDefinitionOrdinalCK, + "duplicate ordinal refused by the named unique constraint") + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.definitionsTable()+` (revision_id, group_id, label, ordinal) + VALUES ($1, 'group-blanklabel', '', 50)`, string(revision.ID)) + requirePgErrConstraintPGState06(t, err, pgstate06CheckViolation, + "explorer_explicit_group_definitions_label_present", "blank label refused by the named CHECK") + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.definitionsTable()+` (revision_id, group_id, label, ordinal) + VALUES ($1, '', 'Blank group id', 51)`, string(revision.ID)) + requirePgErrConstraintPGState06(t, err, pgstate06CheckViolation, + "explorer_explicit_group_definitions_group_id_present", "blank group id refused by the named CHECK") + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.definitionsTable()+` (revision_id, group_id, label, ordinal) + VALUES ($1, 'group-negative', 'Negative ordinal', -1)`, string(revision.ID)) + requirePgErrConstraintPGState06(t, err, pgstate06CheckViolation, + "explorer_explicit_group_definitions_ordinal_non_negative", "negative ordinal refused by the named CHECK") + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.definitionsTable()+` (revision_id, group_id, label, ordinal) + VALUES ('grouprev_' || repeat('f', 64), 'group-orphan', 'Orphan', 52)`) + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "refused: parent revision is absent", + "orphan definition refused by the named guard branch ahead of the FK") + f.rawInsertDefinition(t, f.witness, string(revision.ID), "group-legal-control", "Legal control", 53) + if got := f.countRows(t, f.definitionsTable(), "revision_id = $1 AND group_id = 'group-legal-control'", string(revision.ID)); got != 1 { + t.Fatalf("legal control definition count = %d, want 1", got) + } + + _, err = f.witness.Exec(f.ctx, `UPDATE `+f.definitionsTable()+` SET label = 'Rewritten' + WHERE revision_id = $1 AND group_id = 'group-alpha'`, string(revision.ID)) + requirePgErrTriggerPGState06(t, err, pgstate06CheckViolation, + "explorer_explicit_group_definitions guard", "definition UPDATE names its write-once trigger") + + _, err = f.witness.Exec(f.ctx, `DELETE FROM `+f.definitionsTable()+` + WHERE revision_id = $1 AND group_id = 'group-legal-control'`, string(revision.ID)) + requirePgErrTriggerPGState06(t, err, pgstate06CheckViolation, + "explorer_explicit_group_definitions guard", "unauthorized child DELETE names its trigger") + + if _, err := f.witness.Exec(f.ctx, `SELECT set_config('`+gucAbortToken+`', $1, false)`, token); err != nil { + f.fail("set abort token GUC", err) + } + _, err = f.witness.Exec(f.ctx, `DELETE FROM `+f.revisionTable()+` WHERE id = $1`, string(revision.ID)) + requirePgErrConstraintPGState06(t, err, "23001", explicitGroupRevisionFKConstraint, + "header DELETE blocked by the named FK until children are gone") + if _, err := f.witness.Exec(f.ctx, `DELETE FROM `+f.definitionsTable()+` + WHERE revision_id = $1 AND group_id = 'group-legal-control'`, string(revision.ID)); err != nil { + f.fail("authorized child DELETE", err) + } + if _, err := f.witness.Exec(f.ctx, `SELECT set_config('`+gucWriterToken+`', NULL, false)`); err != nil { + f.fail("clear writer GUC", err) + } + if _, err := f.witness.Exec(f.ctx, `SELECT set_config('`+gucAbortToken+`', NULL, false)`); err != nil { + f.fail("clear abort GUC", err) + } + + f.dropTrigger(t, f.definitionsTable(), "explorer_explicit_group_definitions_guard") + mustNil(t, "unguarded definition UPDATE succeeds", func() error { + _, err := f.witness.Exec(f.ctx, `UPDATE `+f.definitionsTable()+` SET label = 'Rewritten' + WHERE revision_id = $1 AND group_id = 'group-alpha'`, string(revision.ID)) + return err + }) + f.rerunMigration(t, pgstate06MigrationVersion) + _, err = f.witness.Exec(f.ctx, `UPDATE `+f.definitionsTable()+` SET label = 'Rewritten again' + WHERE revision_id = $1 AND group_id = 'group-alpha'`, string(revision.ID)) + requirePgErrTriggerPGState06(t, err, pgstate06CheckViolation, + "explorer_explicit_group_definitions guard", "rerun migration re-blocks the write-once UPDATE") + + _, err = f.witness.Exec(f.ctx, `DELETE FROM `+f.revisionTable()+` WHERE id = $1`, string(revision.ID)) + requirePgErrTriggerPGState06(t, err, pgstate06CheckViolation, + "explorer_explicit_group_revisions guard", "header guard restored by the same rerun") + if got := f.countRows(t, f.revisionTable(), "id = $1", string(revision.ID)); got != 1 { + t.Fatalf("header survived guards count = %d, want 1", got) + } + if got := f.definitionsSnapshot(t, string(revision.ID)); !strings.Contains(got, "group-alpha|Rewritten|0") { + t.Fatalf("definitions after probes = %q, want the witnessed rewrite only", got) + } + _ = want +} + +const ( + grpmMigrationVersion = int64(10) + grpmMigrationName = "explorer_explicit_group_memberships" + grpmMigrationChecksum = "02e0c9ab8bf9864e7d9c82471eacc7406fe2301b025e459784044c4a44489ccd" + grpmDefinitionsGuardVersion = int64(11) + grpmDefinitionsGuardName = "explorer_explicit_group_definition_guard_hardening" + grpmDefinitionsGuardChecksum = "fb45d8f8c312f8f6adb119465da5ed89ac0ddd14929a2c26aa4c917db6d704ec" + grpmSelectionsFixtureName = "selection_revisions" + grpmSelectionsFixtureVersion = int64(8) + grpmSelectionsFixtureChecksum = "562a61ef33786cf674f1c87552e2dc8e70691b34116b7f415d5f81960ed8d65d" + + grpmCheckViolation = "23514" + + grpmProject = "verify-grpm-p" + grpmGeneration = "generation-a" + grpmResourceType = "Patient" + + grpmGroupA = explorer.ExplicitGroupID("verify-grpm-group-a") + grpmGroupB = explorer.ExplicitGroupID("verify-grpm-group-b") + + grpmRaceRounds = 10 + grpmSeamTimeout = 5 * time.Second +) + +func grpmOwnedNameRe(prefix string) *regexp.Regexp { + return regexp.MustCompile(`^` + regexp.QuoteMeta(prefix) + `_[0-9]+_[0-9a-f]+$`) +} + +func grpmLikePattern(prefix string) string { + return strings.ReplaceAll(prefix, "_", `\`) + `\_%` +} + +var ( + grpmRegistryMu sync.Mutex + grpmOwnedRes = map[string]string{} +) + +func grpmRegisterResource(dbName, schema string) { + grpmRegistryMu.Lock() + defer grpmRegistryMu.Unlock() + grpmOwnedRes[dbName] = schema +} + +func grpmDeRegisterResource(dbName string) { + grpmRegistryMu.Lock() + defer grpmRegistryMu.Unlock() + delete(grpmOwnedRes, dbName) +} + +func grpmSnapshotRegistry() map[string]string { + grpmRegistryMu.Lock() + defer grpmRegistryMu.Unlock() + owned := make(map[string]string, len(grpmOwnedRes)) + for k, v := range grpmOwnedRes { + owned[k] = v + } + return owned +} + +type grpmFixture struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dbName string + schema string + dsn string + scrub func(any) string + fail func(step string, err error) + + store *ExplicitGroupStore + selStore *Store + witness *pgxpool.Pool + witnessID int32 +} + +func newGRPMFixture(t *testing.T) *grpmFixture { + t.Helper() + return newGRPMFixtureWithPrefix(t, "verify_grpm") +} + +func newGRPMFixtureWithPrefix(t *testing.T, prefix string) *grpmFixture { + t.Helper() + raw := pgstate06TestPostgresURL(t) + nameRe := grpmOwnedNameRe(prefix) + likePattern := grpmLikePattern(prefix) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 360*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("%s_%d_%s", prefix, time.Now().UnixNano(), pgstate06RandomHex(t)) + if !nameRe.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + + t.Cleanup(func() { + var mine int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname = $1`, dbName).Scan(&mine); err != nil { + t.Errorf("leftover audit: query database catalog: %s", scrub(err)) + } + if mine != 0 { + t.Errorf("leftover audit: isolated database %s still exists", dbName) + } + var namespaceLeftovers int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname LIKE $1 ESCAPE '\'`, likePattern).Scan(&namespaceLeftovers); err != nil { + t.Errorf("leftover audit: query database namespace: %s", scrub(err)) + } + if namespaceLeftovers != 0 { + t.Errorf("leftover audit: %d %s_* databases remain", namespaceLeftovers, prefix) + } + var schemaLeftovers int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_namespace WHERE nspname LIKE $1 ESCAPE '\'`, likePattern).Scan(&schemaLeftovers); err != nil { + t.Errorf("leftover audit: query schema namespace: %s", scrub(err)) + } + if schemaLeftovers != 0 { + t.Errorf("leftover audit: %d %s_* schemas remain in the service database", schemaLeftovers, prefix) + } + grpmDeRegisterResource(dbName) + }) + + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + if probe, err := url.Parse(dsn); err != nil || + probe.Query().Get("dbname") != "" || probe.Path != "/"+dbName { + t.Fatal("isolated test DSN does not provably target the isolated database") + } + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("%s_%d_%s", prefix, time.Now().UnixNano(), pgstate06RandomHex(t)) + if !nameRe.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + grpmRegisterResource(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + f := &grpmFixture{ctx: ctx, admin: admin, iso: iso, dbName: dbName, schema: schema, dsn: dsn, scrub: scrub, fail: fail} + f.witness, f.witnessID = f.newDedicatedPool(t) + f.store, err = f.newStore(t) + if err != nil { + f.fail("build explicit-group store", err) + } + selClient, err := pgstore.Open(f.ctx, pgstore.Config{DSN: f.dsn, MaxConns: 4, MinConns: 0, AcquireTimeout: 10 * time.Second}) + if err != nil { + f.fail("open selection pool", err) + } + t.Cleanup(selClient.Close) + f.selStore, err = New(selClient, f.schema) + if err != nil { + f.fail("build selection store", err) + } + return f +} + +func (f *grpmFixture) migrateWithSelections(t *testing.T) pgstore.MigrateResult { + t.Helper() + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, + MaxConns: 2, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + f.fail("open migration pool", err) + } + defer client.Close() + history, err := OwnerMigrations() + if err != nil { + f.fail("owner migrations", err) + } + fixture, err := requireOwnerMigration(t, history, grpmSelectionsFixtureVersion, grpmSelectionsFixtureName) + if err != nil { + f.fail("selections fixture migration", err) + } + embeddedSum := sha256.Sum256([]byte(fixture.SQL)) + if hex.EncodeToString(embeddedSum[:]) != fixture.Checksum || + fixture.Checksum != grpmSelectionsFixtureChecksum { + t.Fatalf("selections migration checksum %s != embedded recompute %s or the frozen pin %s: the #85-owned migration drifted", + fixture.Checksum, hex.EncodeToString(embeddedSum[:]), grpmSelectionsFixtureChecksum) + } + runner, err := pgstore.NewMigrationRunner(client, f.schema, history) + if err != nil { + f.fail("new migration runner", err) + } + result, err := runner.Migrate(f.ctx) + if err != nil { + f.fail("migrate composed schema", err) + } + return result +} + +func (f *grpmFixture) newStore(t *testing.T) (*ExplicitGroupStore, error) { + t.Helper() + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + return nil, err + } + t.Cleanup(client.Close) + return NewExplicitGroupStore(client, f.schema) +} + +func (f *grpmFixture) newDedicatedStore(t *testing.T) (*ExplicitGroupStore, int32) { + t.Helper() + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, + MaxConns: 1, + MinConns: 1, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + f.fail("open dedicated pg-store pool", err) + } + t.Cleanup(client.Close) + conn, err := client.Acquire(f.ctx) + if err != nil { + f.fail("acquire dedicated pg-store connection", err) + } + var pid int32 + if err := conn.QueryRow(f.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + conn.Release() + f.fail("witness dedicated pg-store backend pid", err) + } + conn.Release() + if pid == 0 { + t.Fatal("dedicated pg-store pool reported backend pid 0") + } + store, err := NewExplicitGroupStore(client, f.schema) + if err != nil { + f.fail("build dedicated explicit-group store", err) + } + return store, pid +} + +func (f *grpmFixture) newDedicatedPool(t *testing.T) (*pgxpool.Pool, int32) { + t.Helper() + poolConfig, err := pgxpool.ParseConfig(f.dsn) + if err != nil { + f.fail("parse dedicated raw pool config", err) + } + poolConfig.MaxConns = 1 + poolConfig.MinConns = 1 + pool, err := pgxpool.NewWithConfig(f.ctx, poolConfig) + if err != nil { + f.fail("open dedicated raw pool", err) + } + t.Cleanup(pool.Close) + var pid int32 + if err := pool.QueryRow(f.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + f.fail("witness dedicated raw backend pid", err) + } + if pid == 0 { + t.Fatal("dedicated raw pool reported backend pid 0") + } + return pool, pid +} + +func requireDistinctBackendsGRPM(t *testing.T, pids map[string]int32) { + t.Helper() + seen := map[int32]string{} + for label, pid := range pids { + if pid == 0 { + t.Fatalf("%s reported backend pid 0", label) + } + if other, clash := seen[pid]; clash { + t.Fatalf("%s and %s share backend pid %d, want provably distinct PG connections", label, other, pid) + } + seen[pid] = label + } +} + +func (f *grpmFixture) revisionTable() string { + return sanitizeTableRef(f.schema, ExplicitGroupRevisionsTable) +} + +func (f *grpmFixture) definitionsTable() string { + return sanitizeTableRef(f.schema, ExplicitGroupDefinitionsTable) +} + +func (f *grpmFixture) membershipsTable() string { + return sanitizeTableRef(f.schema, ExplicitGroupMembershipsTable) +} + +func (f *grpmFixture) grpmHeaderSnapshot(t *testing.T, revisionID string) string { + t.Helper() + var out string + err := f.witness.QueryRow(f.ctx, `SELECT concat_ws(E'\x1f', + id, project, generation, scope_digest, resource_type, + source_selection_revision_id, source_membership_digest, + definition_digest, membership_digest, + group_count::text, member_count::text, state, idempotency_key, + to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), + COALESCE(to_char(completed_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), ''), + COALESCE(writer_token, ''), + COALESCE(to_char(writer_expires_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), '')) + FROM `+f.revisionTable()+` WHERE id = $1`, revisionID).Scan(&out) + if err != nil { + f.fail("header snapshot of "+revisionID, err) + } + return out +} + +func (f *grpmFixture) membershipSnapshot(t *testing.T, revisionID string) string { + t.Helper() + rows, err := f.witness.Query(f.ctx, `SELECT group_id, project, generation, resource_type, resource_id FROM `+ + f.membershipsTable()+` WHERE revision_id = $1`+ + ` ORDER BY group_id, resource_id, project, generation, resource_type`, revisionID) + if err != nil { + f.fail("memberships snapshot of "+revisionID, err) + } + defer rows.Close() + var parts []string + for rows.Next() { + var groupID, project, generation, resourceType, resourceID string + if err := rows.Scan(&groupID, &project, &generation, &resourceType, &resourceID); err != nil { + f.fail("scan memberships snapshot of "+revisionID, err) + } + parts = append(parts, strings.Join([]string{groupID, project, generation, resourceType, resourceID}, "|")) + } + if err := rows.Err(); err != nil { + f.fail("memberships snapshot of "+revisionID, err) + } + return strings.Join(parts, "\x1e") +} + +func (f *grpmFixture) membershipCtidSnapshot(t *testing.T, revisionID string) string { + t.Helper() + rows, err := f.witness.Query(f.ctx, `SELECT ctid::text, group_id, resource_id FROM `+ + f.membershipsTable()+` WHERE revision_id = $1 ORDER BY group_id, resource_id`, revisionID) + if err != nil { + f.fail("ctid memberships snapshot of "+revisionID, err) + } + defer rows.Close() + var parts []string + for rows.Next() { + var ctid, groupID, resourceID string + if err := rows.Scan(&ctid, &groupID, &resourceID); err != nil { + f.fail("scan ctid memberships snapshot of "+revisionID, err) + } + parts = append(parts, ctid+"@"+groupID+"/"+resourceID) + } + if err := rows.Err(); err != nil { + f.fail("ctid memberships snapshot of "+revisionID, err) + } + return strings.Join(parts, "\x1e") +} + +func (f *grpmFixture) countMembers(t *testing.T, revisionID string) int { + t.Helper() + return f.countRowsOn(t, f.membershipsTable(), "revision_id = $1", revisionID) +} + +func (f *grpmFixture) countRowsOn(t *testing.T, table, where string, args ...any) int { + t.Helper() + var count int + if err := f.witness.QueryRow(f.ctx, `SELECT count(*) FROM `+table+` WHERE `+where, args...).Scan(&count); err != nil { + f.fail("count rows in "+table, err) + } + return count +} + +func requirePgErrGRPM(t *testing.T, err error, sqlstate, step string) *pgconn.PgError { + t.Helper() + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + t.Fatalf("%s: error = %v (%T), want *pgconn.PgError", step, err, err) + } + if pgErr.Code != sqlstate { + t.Fatalf("%s: sqlstate = %s, want %s (error: %v)", step, pgErr.Code, sqlstate, err) + } + return pgErr +} + +func requirePgErrConstraintGRPM(t *testing.T, err error, sqlstate, constraint, step string) *pgconn.PgError { + t.Helper() + pgErr := requirePgErrGRPM(t, err, sqlstate, step) + if pgErr.ConstraintName != constraint { + t.Fatalf("%s: constraint = %q, want the named guard %q (message: %s)", step, pgErr.ConstraintName, constraint, pgErr.Message) + } + return pgErr +} + +func requirePgErrTriggerGRPM(t *testing.T, err error, sqlstate, trigger, step string) *pgconn.PgError { + t.Helper() + pgErr := requirePgErrGRPM(t, err, sqlstate, step) + if !strings.Contains(pgErr.Message, trigger) { + t.Fatalf("%s: message %q does not name the guard %q", step, pgErr.Message, trigger) + } + return pgErr +} + +func (f *grpmFixture) rerunMigrationGRPM(t *testing.T, version int64) { + t.Helper() + migrations, err := OwnerMigrations() + if err != nil { + f.fail("load migrations for rerun", err) + } + for _, m := range migrations { + if m.Version == version { + conn, err := f.iso.Acquire(f.ctx) + if err != nil { + f.fail("acquire connection for migration rerun", err) + } + defer conn.Release() + tx, err := conn.Begin(f.ctx) + if err != nil { + f.fail("begin migration rerun transaction", err) + } + if _, err := tx.Exec(f.ctx, + "SET LOCAL search_path TO "+pgx.Identifier{f.schema}.Sanitize()); err != nil { + tx.Rollback(f.ctx) + f.fail("set search_path for migration rerun", err) + } + if _, err := tx.Exec(f.ctx, m.SQL); err != nil { + tx.Rollback(f.ctx) + f.fail(fmt.Sprintf("rerun migration %d/%s", m.Version, m.Name), err) + } + if err := tx.Commit(f.ctx); err != nil { + f.fail("commit migration rerun", err) + } + return + } + } + t.Fatalf("migration version %d is absent from the embedded history", version) +} + +func (f *grpmFixture) dropTriggerGRPM(t *testing.T, table, trigger string) { + t.Helper() + if _, err := f.witness.Exec(f.ctx, `DROP TRIGGER `+trigger+` ON `+table); err != nil { + f.fail("drop trigger "+trigger+" on "+table, err) + } +} + +func (f *grpmFixture) execOnGRPM(t *testing.T, pool *pgxpool.Pool, step, sql string, args ...any) { + t.Helper() + if _, err := pool.Exec(f.ctx, sql, args...); err != nil { + f.fail(step+": "+scrubSQLForTest(sql), err) + } +} + +func (f *grpmFixture) setGUC(t *testing.T, name string, value any) { + t.Helper() + if _, err := f.witness.Exec(f.ctx, `SELECT set_config($1, $2, false)`, name, value); err != nil { + f.fail("set GUC "+name, err) + } +} + +func grpmRef(resourceID string) explorer.ResourceRef { + return explorer.ResourceRef{Project: grpmProject, Generation: grpmGeneration, ResourceType: grpmResourceType, ID: resourceID} +} + +func grpmMembership(group explorer.ExplicitGroupID, resourceID string) explorer.ExplicitGroupMembership { + return explorer.ExplicitGroupMembership{GroupID: group, Ref: grpmRef(resourceID)} +} + +func (f *grpmFixture) stageSelection(t *testing.T, suffix string, resourceIDs ...string) explorer.SelectionRevision { + t.Helper() + selectionID := "verify-grpm-sel-" + suffix + staging := explorer.SelectionRevision{ + ID: selectionID, + Project: grpmProject, + Generation: grpmGeneration, + ResourceType: grpmResourceType, + Rule: explorer.SelectionRule{Kind: explorer.SelectionRuleExplicit}, + Source: explorer.SelectionSource{Kind: explorer.SelectionSourceExplicit}, + ScopeDigest: "verify-grpm-scope", + RuleDigest: "verify-grpm-rule", + CreatedAt: time.Now().UTC(), + } + token := "grpm-sel-writer-" + suffix + if _, err := f.selStore.BeginSelection(f.ctx, staging, token); err != nil { + f.fail("begin source selection "+selectionID, err) + } + if len(resourceIDs) > 0 { + members := make([]explorer.SelectionMember, 0, len(resourceIDs)) + for _, id := range resourceIDs { + members = append(members, explorer.SelectionMember{Ref: grpmRef(id)}) + } + if _, err := f.selStore.AppendSelectionMembers(f.ctx, selectionID, token, members); err != nil { + f.fail("stage source members for "+selectionID, err) + } + } + digest, count, bytes, err := f.selStore.DigestSelectionMembers(f.ctx, grpmProject, selectionID) + if err != nil { + f.fail("digest source selection "+selectionID, err) + } + if _, err := f.selStore.CompleteSelection(f.ctx, selectionID, token, digest, count, bytes, + time.Date(2026, 9, 19, 0, 0, 0, 0, time.UTC)); err != nil { + f.fail("complete source selection "+selectionID, err) + } + stored, err := f.selStore.GetSelection(f.ctx, grpmProject, selectionID) + if err != nil { + f.fail("read stored source selection "+selectionID, err) + } + if !stored.Complete || stored.MembershipDigest != digest || stored.MemberCount != count { + t.Fatalf("stored source selection %q = %+v, want COMPLETE with frozen digest %s and count %d", + selectionID, stored, digest, count) + } + return *stored +} + +func grpmRevision(source explorer.SelectionRevision, key string) explorer.ExplicitGroupRevision { + return explorer.ExplicitGroupRevision{ + ID: explorer.ExplicitGroupRevisionIDFor(grpmProject, key), + Project: grpmProject, + Generation: source.Generation, + ScopeDigest: source.ScopeDigest, + ResourceType: source.ResourceType, + SourceSelectionRevisionID: source.ID, + SourceMembershipDigest: source.MembershipDigest, + State: explorer.ExplicitGroupRevisionStaging, + IdempotencyKey: key, + CreatedAt: time.Now().UTC(), + } +} + +func grpmDefinitions() []explorer.ExplicitGroupDefinition { + return []explorer.ExplicitGroupDefinition{ + {ID: grpmGroupA, Label: "Group A", Ordinal: 0}, + {ID: grpmGroupB, Label: "Group B", Ordinal: 1}, + } +} + +func (f *grpmFixture) beginGroupRevision(t *testing.T, source explorer.SelectionRevision, key, token string) explorer.ExplicitGroupRevision { + t.Helper() + revision := grpmRevision(source, key) + if _, err := f.store.BeginExplicitGroupRevision(f.ctx, revision, source, token); err != nil { + f.fail("begin group revision "+key, err) + } + return revision +} + +func (f *grpmFixture) putDefinitions(t *testing.T, revisionID explorer.ExplicitGroupRevisionID, token string) { + t.Helper() + if err := f.store.PutExplicitGroupDefinitions(f.ctx, revisionID, token, grpmDefinitions()); err != nil { + f.fail("put definitions for "+string(revisionID), err) + } +} + +func (f *grpmFixture) stagedGroupFixture(t *testing.T, suffix string, refs ...string) (explorer.ExplicitGroupRevision, string, explorer.SelectionRevision) { + t.Helper() + source := f.stageSelection(t, suffix, refs...) + token := "grpm-writer-" + suffix + revision := f.beginGroupRevision(t, source, "grpm-key-"+suffix, token) + f.putDefinitions(t, revision.ID, token) + return revision, token, source +} + +func (f *grpmFixture) rawInsertMembership(t *testing.T, pool *pgxpool.Pool, revisionID, groupID, project, generation, resourceType, resourceID string) { + t.Helper() + f.execOnGRPM(t, pool, "raw insert membership", + `INSERT INTO `+f.membershipsTable()+` (revision_id, group_id, project, generation, resource_type, resource_id)`+ + ` VALUES ($1, $2, $3, $4, $5, $6)`, + revisionID, groupID, project, generation, resourceType, resourceID) +} + +func (f *grpmFixture) rawInsertDefinition(t *testing.T, pool *pgxpool.Pool, revisionID, groupID, label string, ordinal int64) { + t.Helper() + f.execOnGRPM(t, pool, "raw insert definition", + `INSERT INTO `+f.definitionsTable()+` (revision_id, group_id, label, ordinal) VALUES ($1, $2, $3, $4)`, + revisionID, groupID, label, ordinal) +} + +func grpmFrameDigest(t *testing.T, pairs ...[2]string) string { + t.Helper() + hash := sha256.New() + writeField := func(value string) { + var length [8]byte + binary.BigEndian.PutUint64(length[:], uint64(len(value))) + if _, err := hash.Write(length[:]); err != nil { + t.Fatalf("frame digest length write: %v", err) + } + if _, err := hash.Write([]byte(value)); err != nil { + t.Fatalf("frame digest value write: %v", err) + } + } + writeField("loom-explicit-group-membership-v1") + for _, pair := range pairs { + writeField(pair[0]) + writeField(grpmProject) + writeField(grpmGeneration) + writeField(grpmResourceType) + writeField(pair[1]) + } + return hex.EncodeToString(hash.Sum(nil)) +} + +func grpmRow(group, resourceID string) string { + return strings.Join([]string{group, grpmProject, grpmGeneration, grpmResourceType, resourceID}, "|") +} + +func grpmRows(rows ...string) string { + return strings.Join(rows, "\x1e") +} + +func grpmSeam(release <-chan struct{}) func(context.Context, pgx.Tx) error { + return func(ctx context.Context, tx pgx.Tx) error { + select { + case <-release: + return nil + case <-time.After(grpmSeamTimeout): + return errors.New("grpm seam deadline exceeded: race schedule broken") + } + } +} + +func grpmHoldCompletion(store *ExplicitGroupStore, ctx context.Context, + revision explorer.ExplicitGroupRevision, token string, + defDigest, memDigest string, groups, members int64) (release chan<- struct{}, complete <-chan error, entered <-chan struct{}) { + releaseComplete := make(chan struct{}) + completeEntered := make(chan struct{}) + var once sync.Once + store.beforeCompleteCommit = func(ctx context.Context, tx pgx.Tx) error { + once.Do(func() { close(completeEntered) }) + select { + case <-releaseComplete: + return nil + case <-time.After(grpmSeamTimeout): + return errors.New("grpm completion seam deadline exceeded: race schedule broken") + } + } + done := make(chan error, 1) + go func() { + _, err := store.CompleteExplicitGroupRevision(ctx, revision.ID, token, + defDigest, memDigest, groups, members, time.Now().UTC()) + done <- err + }() + return releaseComplete, done, completeEntered +} + +func (f *grpmFixture) grpmWatchLockWait(pid int32) (sawLockWait func() bool, stop func()) { + var ( + mu sync.Mutex + seen bool + done = make(chan struct{}) + wg sync.WaitGroup + ) + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-done: + return + default: + } + var waiting int + if err := f.admin.QueryRow(f.ctx, + `SELECT count(*) FROM pg_stat_activity WHERE pid = $1 AND wait_event_type IN ('Lock', 'Transaction')`, + pid).Scan(&waiting); err == nil && waiting > 0 { + mu.Lock() + seen = true + mu.Unlock() + } + time.Sleep(5 * time.Millisecond) + } + }() + return func() bool { + mu.Lock() + defer mu.Unlock() + return seen + }, func() { + close(done) + wg.Wait() + } +} + +func (f *grpmFixture) rawInsertHeaderGRPM(t *testing.T, pool *pgxpool.Pool, revision explorer.ExplicitGroupRevision, writerToken string, writerExpiresAt *time.Time) { + t.Helper() + f.execOnGRPM(t, pool, "raw insert header", + `INSERT INTO `+f.revisionTable()+` (id, project, generation, scope_digest, resource_type,`+ + ` source_selection_revision_id, source_membership_digest, definition_digest, membership_digest,`+ + ` group_count, member_count, state, idempotency_key, created_at, completed_at, writer_token, writer_expires_at)`+ + ` VALUES ($1, $2, $3, $4, $5, $6, $7, '', '', 0, 0, 'STAGING', $8, $9, NULL, $10, $11)`, + string(revision.ID), revision.Project, revision.Generation, revision.ScopeDigest, revision.ResourceType, + revision.SourceSelectionRevisionID, revision.SourceMembershipDigest, revision.IdempotencyKey, + revision.CreatedAt, writerToken, writerExpiresAt) +} + +func TestPostgresExplicitGroupMembershipsAppendOverlapRetryAndScope(t *testing.T) { + f := newGRPMFixture(t) + f.migrateWithSelections(t) + + revision, token, _ := f.stagedGroupFixture(t, "scope", "r-both", "r-a", "r-b") + + batch := []explorer.ExplicitGroupMembership{ + grpmMembership(grpmGroupA, "r-both"), + grpmMembership(grpmGroupA, "r-both"), + grpmMembership(grpmGroupB, "r-both"), + grpmMembership(grpmGroupA, "r-a"), + grpmMembership(grpmGroupB, "r-b"), + } + inserted, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, batch) + if err != nil { + f.fail("append overlapping batch", err) + } + if len(inserted) != 4 { + t.Fatalf("first append reported %d created, want 4: %+v", len(inserted), inserted) + } + want := grpmRows( + grpmRow(string(grpmGroupA), "r-a"), + grpmRow(string(grpmGroupA), "r-both"), + grpmRow(string(grpmGroupB), "r-b"), + grpmRow(string(grpmGroupB), "r-both"), + ) + if got := f.membershipSnapshot(t, string(revision.ID)); got != want { + t.Fatalf("membership rows = %q, want %q", got, want) + } + ctidsBefore := f.membershipCtidSnapshot(t, string(revision.ID)) + + retry, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, batch) + if err != nil { + f.fail("retry overlapping batch", err) + } + if len(retry) != 0 { + t.Fatalf("retry reported %d created, want 0 (insert-once): %+v", len(retry), retry) + } + if got := f.membershipSnapshot(t, string(revision.ID)); got != want { + t.Fatalf("membership rows after retry = %q, want %q", got, want) + } + if got := f.membershipCtidSnapshot(t, string(revision.ID)); got != ctidsBefore { + t.Fatalf("ctids after retry = %q, want byte-identical %q", got, ctidsBefore) + } + + empty, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, nil) + if err != nil || empty != nil { + t.Fatalf("empty batch = (%v, %v), want (nil, nil)", empty, err) + } + if got := f.membershipCtidSnapshot(t, string(revision.ID)); got != ctidsBefore { + t.Fatalf("ctids after empty batch moved: %q", got) + } + + var expires time.Time + if err := f.witness.QueryRow(f.ctx, + `SELECT writer_expires_at FROM `+f.revisionTable()+` WHERE id = $1`, string(revision.ID)).Scan(&expires); err != nil { + f.fail("read writer lease", err) + } + if expires.Before(time.Now().UTC().Add(explicitGroupWriterLease / 2)) { + t.Fatalf("writer lease %v was not renewed forward by the accepted append", expires) + } + + mustErr(t, "digest under foreign project", func() error { + _, _, _, _, err := f.store.DigestExplicitGroupRevision(f.ctx, "verify-grpm-other", revision.ID) + return err + }, explorer.ErrExplicitGroupRevisionNotFound) + mustErr(t, "visit under foreign project", func() error { + _, err := f.store.VisitExplicitGroupMemberships(f.ctx, "verify-grpm-other", revision.ID, + explorer.ExplicitGroupMembershipCursor{}, 10, func(explorer.ExplicitGroupMembership) error { return nil }) + return err + }, explorer.ErrExplicitGroupRevisionNotFound) +} + +func TestPostgresExplicitGroupMembershipsAppendWriterStateAndSourceGuards(t *testing.T) { + f := newGRPMFixture(t) + f.migrateWithSelections(t) + + revision, token, _ := f.stagedGroupFixture(t, "guards", "r-a") + emptyBefore := f.membershipSnapshot(t, string(revision.ID)) + headerBefore := f.grpmHeaderSnapshot(t, string(revision.ID)) + + mustErr(t, "append with foreign writer token", func() error { + _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token+"-wrong", + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}) + return err + }, explorer.ErrExplicitGroupRevisionConflict) + mustErr(t, "append with blank token", func() error { + _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, " ", + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}) + return err + }, explorer.ErrExplicitGroupRevisionConflict) + mustErr(t, "append to blank revision", func() error { + _, err := f.store.AppendExplicitGroupMemberships(f.ctx, " ", token, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}) + return err + }, explorer.ErrExplicitGroupRevisionConflict) + mustErr(t, "append to absent revision", func() error { + _, err := f.store.AppendExplicitGroupMemberships(f.ctx, "verify-grpm-absent-revision", token, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}) + return err + }, explorer.ErrExplicitGroupRevisionNotFound) + if got := f.membershipSnapshot(t, string(revision.ID)); got != emptyBefore { + t.Fatalf("refused appends wrote rows: %q", got) + } + if got := f.grpmHeaderSnapshot(t, string(revision.ID)); got != headerBefore { + t.Fatalf("refused appends moved the header: %q != %q", got, headerBefore) + } + + mustErr(t, "append resource absent from source selection", func() error { + _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-not-in-source")}) + return err + }, explorer.ErrExplicitGroupMemberNotInSelection) + mustErr(t, "append mixed good/bad batch", func() error { + _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{ + grpmMembership(grpmGroupA, "r-a"), + grpmMembership(grpmGroupB, "r-not-in-source"), + }) + return err + }, explorer.ErrExplicitGroupMemberNotInSelection) + if got := f.membershipSnapshot(t, string(revision.ID)); got != emptyBefore { + t.Fatalf("refused source-guard appends wrote rows: %q", got) + } + + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{grpmMembership(explorer.ExplicitGroupID("verify-grpm-group-ghost"), "r-a")}); err == nil || + !strings.Contains(err.Error(), "undeclared explicit group") { + t.Fatalf("append under undeclared group error = %v, want the undeclared-group refusal", err) + } + + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{{GroupID: grpmGroupA, Ref: explorer.ResourceRef{ + Project: "verify-grpm-other", Generation: grpmGeneration, ResourceType: grpmResourceType, ID: "r-a"}}}); err == nil { + t.Fatal("append with foreign project reference was accepted") + } else if errors.Is(err, explorer.ErrExplicitGroupRevisionConflict) || errors.Is(err, explorer.ErrExplicitGroupMemberNotInSelection) { + t.Fatalf("foreign-scope reference got conflict-class error %v, want the domain validation error", err) + } + + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}); err != nil { + f.fail("legal control append", err) + } + if got := f.membershipSnapshot(t, string(revision.ID)); got != grpmRows(grpmRow(string(grpmGroupA), "r-a")) { + t.Fatalf("legal control wrote %q", got) + } + + completed, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, + "0711212b6e8e2eee92a289bec97557775e4435545972b8f21c93f03e532f3321", + grpmFrameDigest(t, [2]string{string(grpmGroupA), "r-a"}), 2, 1, + time.Date(2026, 9, 19, 0, 0, 0, 0, time.UTC)) + if err != nil || completed == nil || completed.State != explorer.ExplicitGroupRevisionComplete { + f.fail("complete before append-after-complete probe", err) + } + mustErr(t, "append after completion", func() error { + _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupB, "r-a")}) + return err + }, explorer.ErrExplicitGroupRevisionConflict) + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.membershipsTable()+ + ` (revision_id, group_id, project, generation, resource_type, resource_id)`+ + ` VALUES ($1, $2, $3, $4, $5, $6)`, string(revision.ID), string(grpmGroupB), + grpmProject, grpmGeneration, grpmResourceType, "r-a") + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "explorer_explicit_group_memberships guard", + "direct insert under COMPLETE parent") +} + +func TestPostgresExplicitGroupMembershipsDigestFrozenAndFencedComplete(t *testing.T) { + f := newGRPMFixture(t) + f.migrateWithSelections(t) + + revision, token, _ := f.stagedGroupFixture(t, "digest", "r-1", "r-2") + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, []explorer.ExplicitGroupMembership{ + grpmMembership(grpmGroupA, "r-1"), grpmMembership(grpmGroupB, "r-1"), + grpmMembership(grpmGroupA, "r-2"), grpmMembership(grpmGroupB, "r-2"), + }); err != nil { + f.fail("seed append", err) + } + const defDigest = "0711212b6e8e2eee92a289bec97557775e4435545972b8f21c93f03e532f3321" + memDigest := grpmFrameDigest(t, + [2]string{string(grpmGroupA), "r-1"}, [2]string{string(grpmGroupA), "r-2"}, + [2]string{string(grpmGroupB), "r-1"}, [2]string{string(grpmGroupB), "r-2"}) + gotDef, gotMem, groups, members, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + if err != nil { + f.fail("staging digest", err) + } + if gotDef != defDigest || gotMem != memDigest || groups != 2 || members != 4 { + t.Fatalf("staging digest = (%s, %s, %d, %d), want (%s, %s, 2, 4)", + gotDef, gotMem, groups, members, defDigest, memDigest) + } + + for name, call := range map[string]func() error{ + "blank token": func() error { + _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, "", defDigest, memDigest, 2, 4, time.Now().UTC()) + return err + }, + "blank digests": func() error { + _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, "", "", 2, 4, time.Now().UTC()) + return err + }, + "zero groups": func() error { + _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, defDigest, memDigest, 0, 4, time.Now().UTC()) + return err + }, + "negative members": func() error { + _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, defDigest, memDigest, 2, -1, time.Now().UTC()) + return err + }, + } { + mustErr(t, "complete with "+name, call, explorer.ErrExplicitGroupRevisionIncomplete) + } + mustErr(t, "complete with foreign token", func() error { + _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token+"-wrong", defDigest, memDigest, 2, 4, time.Now().UTC()) + return err + }, explorer.ErrExplicitGroupRevisionConflict) + mustErr(t, "complete with wrong membership digest", func() error { + _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, defDigest, strings.Repeat("0", 64), 2, 4, time.Now().UTC()) + return err + }, explorer.ErrExplicitGroupRevisionConflict) + if got, _, _, _, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID); err != nil || got != defDigest { + t.Fatalf("digest after refused completion broke: %s (%v)", got, err) + } + if state := f.grpmHeaderSnapshot(t, string(revision.ID)); !strings.Contains(state, "\x1fSTAGING\x1f") { + t.Fatalf("header after refused completion is no longer STAGING: %q", state) + } + mustErr(t, "complete under foreign project id", func() error { + _, err := f.store.CompleteExplicitGroupRevision(f.ctx, "verify-grpm-nope", token, defDigest, memDigest, 2, 4, time.Now().UTC()) + return err + }, explorer.ErrExplicitGroupRevisionNotFound) + + completed, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, defDigest, memDigest, 2, 4, time.Time{}) + if err != nil || completed == nil { + f.fail("honest completion", err) + } + if completed.State != explorer.ExplicitGroupRevisionComplete || + completed.CompletedAt == nil || completed.CompletedAt.IsZero() { + t.Fatalf("completed revision = %+v", completed) + } + if gotDef, gotMem, groups, members, err = f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID); err != nil || + gotDef != defDigest || gotMem != memDigest || groups != 2 || members != 4 { + t.Fatalf("frozen digest = (%s, %s, %d, %d, %v)", gotDef, gotMem, groups, members, err) + } + retry, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, defDigest, memDigest, 2, 4, + time.Date(2026, 9, 19, 0, 0, 0, 0, time.UTC)) + if err != nil || retry == nil { + f.fail("idempotent completion retry", err) + } + if retry.CompletedAt == nil || completed.CompletedAt == nil || !retry.CompletedAt.Equal(*completed.CompletedAt) { + t.Fatalf("idempotent retry moved completed_at: %v != %v", retry.CompletedAt, completed.CompletedAt) + } + mustErr(t, "complete COMPLETE row with different facts", func() error { + _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, defDigest, strings.Repeat("a", 64), 2, 4, time.Now().UTC()) + return err + }, explorer.ErrExplicitGroupRevisionConflict) + mustErr(t, "abort COMPLETE revision", func() error { + return f.store.AbortExplicitGroupRevision(f.ctx, revision.ID, token) + }, explorer.ErrExplicitGroupRevisionConflict) + if got := f.countMembers(t, string(revision.ID)); got != 4 { + t.Fatalf("members after refused abort = %d, want 4", got) + } +} + +func TestPostgresExplicitGroupMembershipsGoldenFactsAndBreakOne(t *testing.T) { + f := newGRPMFixture(t) + f.migrateWithSelections(t) + + revision, token, source := f.stagedGroupFixture(t, "golden", "r-gold-1", "r-gold-2") + if source.ID != "verify-grpm-sel-golden" || source.MembershipDigest == "" || source.MemberCount != 2 { + t.Fatalf("golden source = %+v", source) + } + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, []explorer.ExplicitGroupMembership{ + grpmMembership(grpmGroupA, "r-gold-1"), grpmMembership(grpmGroupA, "r-gold-2"), + grpmMembership(grpmGroupB, "r-gold-1"), grpmMembership(grpmGroupB, "r-gold-2"), + }); err != nil { + f.fail("golden append", err) + } + + const ( + defDigest = "0711212b6e8e2eee92a289bec97557775e4435545972b8f21c93f03e532f3321" + memDigest = "15a005684f311244c87000d13ecbd9f4aa7cde4d4248d6a81c519b82dbbd6e28" + ) + if gotDef, gotMem, groups, members, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID); err != nil || + gotDef != defDigest || gotMem != memDigest || groups != 2 || members != 4 { + t.Fatalf("golden digest = (%s, %s, %d, %d, %v), want the pinned literals", gotDef, gotMem, groups, members, err) + } + if derived := grpmFrameDigest(t, + [2]string{string(grpmGroupA), "r-gold-1"}, [2]string{string(grpmGroupA), "r-gold-2"}, + [2]string{string(grpmGroupB), "r-gold-1"}, [2]string{string(grpmGroupB), "r-gold-2"}); derived != memDigest { + t.Fatalf("golden independent re-derivation %s != literal %s", derived, memDigest) + } + + completedAt := time.Date(2026, 9, 19, 0, 0, 0, 0, time.UTC) + if _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, defDigest, memDigest, 2, 4, completedAt); err != nil { + f.fail("golden completion", err) + } + want := strings.Join([]string{ + string(revision.ID), grpmProject, grpmGeneration, "verify-grpm-scope", grpmResourceType, + "verify-grpm-sel-golden", source.MembershipDigest, + defDigest, memDigest, + "2", "4", "COMPLETE", "grpm-key-golden", + "", "2026-09-19 00:00:00.000000", token, "", + }, "\x1f") + got := f.grpmHeaderSnapshot(t, string(revision.ID)) + parts := strings.Split(got, "\x1f") + wantParts := strings.Split(want, "\x1f") + for i := range wantParts { + if wantParts[i] == "" || wantParts[i] == "" { + continue + } + if parts[i] != wantParts[i] { + t.Fatalf("COMPLETE header field %d = %q, want %q (full: %q)", i, parts[i], wantParts[i], got) + } + } + + flip := func(hexDigest string, pos int) string { + bad := []byte(hexDigest) + if bad[pos] == 'f' { + bad[pos] = '0' + } else { + bad[pos] = 'f' + } + return string(bad) + } + breakCases := []struct { + name string + claimDef string + claimMem string + }{ + {"membership digest flipped", defDigest, flip(memDigest, 40)}, + {"definition digest flipped", flip(defDigest, 7), memDigest}, + } + for _, broken := range breakCases { + broken := broken + mustErr(t, "completion with "+broken.name, func() error { + _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, broken.claimDef, broken.claimMem, 2, 4, completedAt) + return err + }, explorer.ErrExplicitGroupRevisionConflict) + } + singleRev, singleToken, _ := f.stagedGroupFixture(t, "golden-single", "r-single") + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, singleRev.ID, singleToken, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-single")}); err != nil { + f.fail("single append", err) + } + const singleDigest = "f728a0b986129118326ccdd7345695544660bbadc19933c1373c9975a1329338" + if _, gotMem, _, members, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, singleRev.ID); err != nil || + gotMem != singleDigest || members != 1 { + t.Fatalf("single digest = (%s, %d, %v), want (%s, 1, nil)", gotMem, members, err, singleDigest) + } + if derived := grpmFrameDigest(t, [2]string{string(grpmGroupA), "r-single"}); derived != singleDigest { + t.Fatalf("single independent re-derivation %s != literal %s", derived, singleDigest) + } + + overlapRev, overlapToken, _ := f.stagedGroupFixture(t, "golden-overlap", "r-both") + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, overlapRev.ID, overlapToken, + []explorer.ExplicitGroupMembership{ + grpmMembership(grpmGroupA, "r-both"), grpmMembership(grpmGroupB, "r-both"), + }); err != nil { + f.fail("overlap append", err) + } + const overlapDigest = "2b30fa23d892241d12586ab15fc7bec37bbe975c77538ceda3d3eea1ff7b966d" + if _, gotMem, _, members, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, overlapRev.ID); err != nil || + gotMem != overlapDigest || members != 2 { + t.Fatalf("overlap digest = (%s, %d, %v), want (%s, 2, nil)", gotMem, members, err, overlapDigest) + } + + emptyRev, emptyToken, _ := f.stagedGroupFixture(t, "golden-empty") + const emptyDigest = "51a9413396bfc791022568e5db6dbc5001629854b2c17d7b07dfdb35a820ec09" + if _, gotMem, groups, members, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, emptyRev.ID); err != nil || + gotMem != emptyDigest || groups != 2 || members != 0 { + t.Fatalf("empty digest = (%s, %d, %d, %v), want (%s, 2, 0, nil)", gotMem, groups, members, err, emptyDigest) + } + if _, err := f.store.CompleteExplicitGroupRevision(f.ctx, emptyRev.ID, emptyToken, defDigest, emptyDigest, 2, 0, completedAt); err != nil { + f.fail("complete zero-member revision", err) + } +} + +func TestPostgresExplicitGroupMembershipsVisitBoundedResumableCancel(t *testing.T) { + f := newGRPMFixture(t) + f.migrateWithSelections(t) + + const total = 1001 + refs := make([]string, 0, total) + for i := 0; i < total; i++ { + refs = append(refs, fmt.Sprintf("r-%04d", i)) + } + revision, token, _ := f.stagedGroupFixture(t, "visit", refs...) + batch := make([]explorer.ExplicitGroupMembership, 0, total) + for i, ref := range refs { + group := grpmGroupA + if i%2 == 1 { + group = grpmGroupB + } + batch = append(batch, grpmMembership(group, ref)) + } + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, batch); err != nil { + f.fail("append 1001 members", err) + } + + mustErr(t, "visit STAGING revision", func() error { + _, err := f.store.VisitExplicitGroupMemberships(f.ctx, grpmProject, revision.ID, + explorer.ExplicitGroupMembershipCursor{}, 10, func(explorer.ExplicitGroupMembership) error { return nil }) + return err + }, explorer.ErrExplicitGroupRevisionIncomplete) + if _, err := f.store.VisitExplicitGroupMemberships(f.ctx, grpmProject, revision.ID, + explorer.ExplicitGroupMembershipCursor{}, 10, nil); err == nil || + !strings.Contains(err.Error(), "visitor is required") { + t.Fatalf("nil-visitor error = %v, want the required-visitor refusal", err) + } + + var pages []int + f.store.membershipPageStreamed = func(rows int) { pages = append(pages, rows) } + defDigest, memDigest, groups, members, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + f.store.membershipPageStreamed = nil + if err != nil { + f.fail("large digest", err) + } + if groups != 2 || members != total { + t.Fatalf("large digest counts = (%d, %d), want (2, %d)", groups, members, total) + } + if len(pages) == 0 { + t.Fatal("digest streamed no pages") + } + for _, n := range pages { + if n > explicitGroupMembershipPageSize { + t.Fatalf("digest page of %d exceeded the cap %d", n, explicitGroupMembershipPageSize) + } + } + if pages[0] != explicitGroupMembershipPageSize || pages[len(pages)-1] != 1 { + t.Fatalf("digest page shape %v, want first page %d and last page 1", pages, explicitGroupMembershipPageSize) + } + + tx, err := f.witness.Begin(f.ctx) + if err != nil { + f.fail("begin plan-witness transaction", err) + } + defer tx.Rollback(f.ctx) + if _, err := tx.Exec(f.ctx, `SET LOCAL enable_seqscan = off`); err != nil { + f.fail("disable seqscan for plan witness", err) + } + if _, err := tx.Exec(f.ctx, `ANALYZE `+f.membershipsTable()); err != nil { + f.fail("analyze memberships for plan witness", err) + } + explainPlan := func(sql string) string { + t.Helper() + rows, err := tx.Query(f.ctx, `EXPLAIN (FORMAT TEXT) `+sql, string(revision.ID), "", "", 50) + if err != nil { + f.fail("explain membership page", err) + } + defer rows.Close() + var plan strings.Builder + for rows.Next() { + var line string + if err := rows.Scan(&line); err != nil { + f.fail("scan plan line", err) + } + plan.WriteString(line) + plan.WriteByte('\n') + } + if err := rows.Err(); err != nil { + f.fail("read plan lines", err) + } + return plan.String() + } + fixedPlan := explainPlan(f.store.memberStmts.memberPage) + if !strings.Contains(fixedPlan, "explorer_explicit_group_memberships_visit") { + t.Fatalf("bounded page does not use the visit index:\n%s", fixedPlan) + } + if strings.Contains(fixedPlan, "Sort") { + t.Fatalf("bounded page carries a Sort node -- the ORDER BY is not proven by the visit index:\n%s", fixedPlan) + } + tailedPlan := explainPlan(strings.Replace(f.store.memberStmts.memberPage, + "ORDER BY group_id, resource_id", "ORDER BY group_id, resource_id, project, generation, resource_type", 1)) + if !strings.Contains(tailedPlan, "Sort") { + t.Fatalf("plan-shape control: the trailing-scope-column page planned WITHOUT a Sort node -- the no-Sort witness proves nothing:\n%s", tailedPlan) + } + if err := tx.Commit(f.ctx); err != nil { + f.fail("commit plan-witness transaction", err) + } + + if _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, defDigest, memDigest, 2, total, + time.Date(2026, 9, 19, 0, 0, 0, 0, time.UTC)); err != nil { + f.fail("large completion", err) + } + + var seen []explorer.ExplicitGroupMembership + cursor := explorer.ExplicitGroupMembershipCursor{} + calls := 0 + for { + calls++ + if calls > total { + t.Fatal("visit resumption never terminated") + } + next, err := f.store.VisitExplicitGroupMemberships(f.ctx, grpmProject, revision.ID, cursor, 100000, + func(membership explorer.ExplicitGroupMembership) error { + seen = append(seen, membership) + return nil + }) + if err != nil { + f.fail("visit page", err) + } + if next == cursor { + break + } + cursor = next + } + if len(seen) != total { + t.Fatalf("full visit yielded %d members, want %d", len(seen), total) + } + var wantOrder []explorer.ExplicitGroupMembership + for _, group := range []explorer.ExplicitGroupID{grpmGroupA, grpmGroupB} { + for _, m := range batch { + if m.GroupID == group { + wantOrder = append(wantOrder, m) + } + } + } + if len(wantOrder) != total { + t.Fatalf("expected walk set = %d, want %d", len(wantOrder), total) + } + for i, want := range wantOrder { + if seen[i].GroupID != want.GroupID || seen[i].Ref.ID != want.Ref.ID { + t.Fatalf("visit order diverged at %d: got %s/%s want %s/%s", i, + seen[i].GroupID, seen[i].Ref.ID, want.GroupID, want.Ref.ID) + } + } + + visCtx, cancel := context.WithCancel(f.ctx) + visited := 0 + var partial []explorer.ExplicitGroupMembership + cancelledCursor, err := f.store.VisitExplicitGroupMemberships(visCtx, grpmProject, revision.ID, + explorer.ExplicitGroupMembershipCursor{}, 5, func(membership explorer.ExplicitGroupMembership) error { + visited++ + partial = append(partial, membership) + if visited == 3 { + cancel() + } + return nil + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled visit error = %v, want context.Canceled", err) + } + if len(partial) != 3 { + t.Fatalf("cancelled visit called the visitor %d times, want exactly 3", len(partial)) + } + if cancelledCursor.GroupID != partial[2].GroupID || cancelledCursor.ResourceID != partial[2].Ref.ID { + t.Fatalf("cancelled cursor = %+v, want the last visited member %+v", cancelledCursor, partial[2]) + } + resumed := append([]explorer.ExplicitGroupMembership(nil), partial...) + cursor = cancelledCursor + for { + next, err := f.store.VisitExplicitGroupMemberships(f.ctx, grpmProject, revision.ID, cursor, 100000, + func(membership explorer.ExplicitGroupMembership) error { + resumed = append(resumed, membership) + return nil + }) + if err != nil { + f.fail("resumed visit", err) + } + if next == cursor { + break + } + cursor = next + } + if len(resumed) != total { + t.Fatalf("resumed visit total = %d, want %d (no gap, no duplicate)", len(resumed), total) + } + for i := range resumed { + if resumed[i].GroupID != seen[i].GroupID || resumed[i].Ref.ID != seen[i].Ref.ID { + t.Fatalf("resumed walk diverged from full walk at %d", i) + } + } +} + +func TestPostgresExplicitGroupMembershipsDatabaseGuards(t *testing.T) { + f := newGRPMFixture(t) + f.migrateWithSelections(t) + + revision, token, _ := f.stagedGroupFixture(t, "guardsdb", "r-a", "r-b") + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}); err != nil { + f.fail("seed append", err) + } + stored := grpmRows(grpmRow(string(grpmGroupA), "r-a")) + + _, err := f.witness.Exec(f.ctx, `UPDATE `+f.membershipsTable()+` SET resource_id = 'r-c'`+ + ` WHERE revision_id = $1 AND group_id = $2 AND resource_id = $3`, + string(revision.ID), string(grpmGroupA), "r-a") + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "write-once", "membership UPDATE mutation") + if _, err := f.witness.Exec(f.ctx, `UPDATE `+f.membershipsTable()+` SET resource_id = 'r-a'`+ + ` WHERE revision_id = $1 AND group_id = $2 AND resource_id = $3`, + string(revision.ID), string(grpmGroupA), "r-a"); err != nil { + f.fail("no-op membership UPDATE probe", err) + } + + _, err = f.witness.Exec(f.ctx, `DELETE FROM `+f.membershipsTable()+ + ` WHERE revision_id = $1 AND group_id = $2 AND resource_id = $3`, + string(revision.ID), string(grpmGroupA), "r-a") + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "owner-abort or expired-writer-cleanup", + "unauthorized membership DELETE") + if got := f.membershipSnapshot(t, string(revision.ID)); got != stored { + t.Fatalf("refused DELETE moved rows: %q", got) + } + + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.membershipsTable()+ + ` (revision_id, group_id, project, generation, resource_type, resource_id)`+ + ` VALUES ($1, $2, $3, $4, $5, $6)`, + string(revision.ID), string(grpmGroupB), "verify-grpm-other", grpmGeneration, grpmResourceType, "r-b") + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "does not repeat the parent scope", + "foreign-scope membership insert") + + orphan := "verify-grpm-orphan-revision" + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.membershipsTable()+ + ` (revision_id, group_id, project, generation, resource_type, resource_id)`+ + ` VALUES ($1, $2, $3, $4, $5, $6)`, + orphan, string(grpmGroupA), grpmProject, grpmGeneration, grpmResourceType, "r-x") + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "parent revision is absent", "orphan membership insert") + + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.membershipsTable()+ + ` (revision_id, group_id, project, generation, resource_type, resource_id)`+ + ` VALUES ($1, $2, $3, $4, $5, $6)`, + string(revision.ID), "verify-grpm-group-ghost", grpmProject, grpmGeneration, grpmResourceType, "r-b") + requirePgErrConstraintGRPM(t, err, "23503", + explicitGroupMembershipGroupFK, "undeclared-group membership insert") + + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.membershipsTable()+ + ` (revision_id, group_id, project, generation, resource_type, resource_id)`+ + ` VALUES ($1, $2, $3, $4, $5, $6)`, + string(revision.ID), "", grpmProject, grpmGeneration, grpmResourceType, "r-b") + requirePgErrConstraintGRPM(t, err, grpmCheckViolation, + "explorer_explicit_group_memberships_group_id_present", "empty group_id insert") + + _, err = f.witness.Exec(f.ctx, `INSERT INTO `+f.membershipsTable()+ + ` (revision_id, group_id, project, generation, resource_type, resource_id)`+ + ` VALUES ($1, $2, $3, $4, $5, NULL)`, + string(revision.ID), string(grpmGroupB), grpmProject, grpmGeneration, grpmResourceType) + requirePgErrGRPM(t, err, "23502", "NULL resource_id insert") + + f.rawInsertMembership(t, f.witness, string(revision.ID), string(grpmGroupB), grpmProject, grpmGeneration, grpmResourceType, "r-b") + if got := f.membershipSnapshot(t, string(revision.ID)); got != grpmRows(stored, grpmRow(string(grpmGroupB), "r-b")) { + t.Fatalf("mutation-probe insert produced %q", got) + } + + f.dropTriggerGRPM(t, f.membershipsTable(), "explorer_explicit_group_memberships_guard") + f.execOnGRPM(t, f.witness, "guard-dropped delete", `DELETE FROM `+f.membershipsTable()+ + ` WHERE revision_id = $1 AND group_id = $2 AND resource_id = $3`, + string(revision.ID), string(grpmGroupB), "r-b") + f.rerunMigrationGRPM(t, grpmMigrationVersion) + _, err = f.witness.Exec(f.ctx, `DELETE FROM `+f.membershipsTable()+ + ` WHERE revision_id = $1 AND group_id = $2 AND resource_id = $3`, + string(revision.ID), string(grpmGroupA), "r-a") + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "owner-abort or expired-writer-cleanup", + "membership DELETE after guard rerun") +} + +func TestPostgresExplicitGroupMembershipsCompletionMachineDatabaseFacts(t *testing.T) { + f := newGRPMFixture(t) + f.migrateWithSelections(t) + + revision, token, source := f.stagedGroupFixture(t, "machine", "r-a", "r-b") + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{ + grpmMembership(grpmGroupA, "r-a"), grpmMembership(grpmGroupB, "r-b"), + }); err != nil { + f.fail("machine seed append", err) + } + headerBefore := f.grpmHeaderSnapshot(t, string(revision.ID)) + + _, err := f.witness.Exec(f.ctx, `INSERT INTO `+f.revisionTable()+ + ` (id, project, generation, scope_digest, resource_type, source_selection_revision_id,`+ + ` source_membership_digest, definition_digest, membership_digest, group_count, member_count,`+ + ` state, idempotency_key, created_at, completed_at, writer_token, writer_expires_at)`+ + ` VALUES ('verify-grpm-born-complete', $1, $2, 's', 'Patient', 'src', 'md',`+ + ` 'd', 'm', 1, 1, 'COMPLETE', 'k', now(), now(), 'w', now())`, + grpmProject, grpmGeneration) + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "born STAGING", "direct COMPLETE header INSERT") + + _, err = f.witness.Exec(f.ctx, `UPDATE `+f.revisionTable()+ + ` SET state = 'COMPLETE', definition_digest = 'd', membership_digest = 'm',`+ + ` group_count = 2, member_count = 2, completed_at = now() WHERE id = $1`, string(revision.ID)) + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "writer token", "completion without the token GUC") + f.setGUC(t, "loom.explorer_explicit_group_writer_token", token+"-wrong") + _, err = f.witness.Exec(f.ctx, `UPDATE `+f.revisionTable()+ + ` SET state = 'COMPLETE', definition_digest = 'd', membership_digest = 'm',`+ + ` group_count = 2, member_count = 2, completed_at = now() WHERE id = $1`, string(revision.ID)) + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "writer token", "completion with the wrong token GUC") + + f.setGUC(t, "loom.explorer_explicit_group_writer_token", token) + _, err = f.witness.Exec(f.ctx, `UPDATE `+f.revisionTable()+ + ` SET state = 'COMPLETE', definition_digest = 'd', membership_digest = 'm',`+ + ` group_count = 2, member_count = 3, completed_at = now() WHERE id = $1`, string(revision.ID)) + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "claims member_count 3 but 2", "completion claiming a false member_count") + _, err = f.witness.Exec(f.ctx, `UPDATE `+f.revisionTable()+ + ` SET state = 'COMPLETE', definition_digest = 'd', membership_digest = 'm',`+ + ` group_count = 5, member_count = 2, completed_at = now() WHERE id = $1`, string(revision.ID)) + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "claims group_count 5 but 2", "completion claiming a false group_count") + if got := f.grpmHeaderSnapshot(t, string(revision.ID)); got != headerBefore { + t.Fatalf("refused direct completions moved the header: %q != %q", got, headerBefore) + } + + f.execOnGRPM(t, f.witness, "honest guarded direct completion", + `UPDATE `+f.revisionTable()+ + ` SET state = 'COMPLETE', definition_digest = 'd', membership_digest = 'm',`+ + ` group_count = 2, member_count = 2, completed_at = now() WHERE id = $1`, string(revision.ID)) + if state := f.grpmHeaderSnapshot(t, string(revision.ID)); !strings.Contains(state, "\x1fCOMPLETE\x1f") { + t.Fatalf("honest guarded completion did not land: %q", state) + } + _, err = f.witness.Exec(f.ctx, `UPDATE `+f.revisionTable()+` SET membership_digest = 'tampered' WHERE id = $1`, string(revision.ID)) + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "COMPLETE history", "tampering with COMPLETE history") + f.execOnGRPM(t, f.witness, "no-op UPDATE probe", `UPDATE `+f.revisionTable()+` SET state = 'COMPLETE' WHERE id = $1`, string(revision.ID)) + + f.dropTriggerGRPM(t, f.revisionTable(), "explorer_explicit_group_revisions_guard") + f.execOnGRPM(t, f.witness, "set up bogus-state probe", `UPDATE `+f.revisionTable()+ + ` SET writer_expires_at = writer_expires_at WHERE id = $1`, string(revision.ID)) + bogusHeader := grpmRevision(source, "machine-bogus") + bogus := string(bogusHeader.ID) + f.rawInsertHeaderGRPM(t, f.witness, bogusHeader, "bogus-writer", nil) + _, err = f.witness.Exec(f.ctx, `UPDATE `+f.revisionTable()+` SET state = 'BOGUS' WHERE id = $1`, bogus) + requirePgErrConstraintGRPM(t, err, grpmCheckViolation, explicitGroupRevisionLifecycleValuesCheck, + "BOGUS state under the lifecycle CHECK (trigger dropped)") + f.rerunMigrationGRPM(t, grpmMigrationVersion) + _, err = f.witness.Exec(f.ctx, `UPDATE `+f.revisionTable()+` SET state = 'BOGUS' WHERE id = $1`, bogus) + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "the only admitted transition", "BOGUS state after guard rerun") + f.setGUC(t, "loom.explorer_explicit_group_writer_token", "") + other, _, _ := f.stagedGroupFixture(t, "machine-other", "r-a") + _, err = f.witness.Exec(f.ctx, `UPDATE `+f.revisionTable()+ + ` SET state = 'COMPLETE', definition_digest = 'd', membership_digest = 'm',`+ + ` group_count = 2, member_count = 0, completed_at = now() WHERE id = $1`, string(other.ID)) + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "writer token", "tokenless completion after guard rerun") +} + +func TestPostgresExplicitGroupMembershipsAbortCleanupCascade(t *testing.T) { + f := newGRPMFixture(t) + f.migrateWithSelections(t) + + revision, token, _ := f.stagedGroupFixture(t, "abort", "r-a", "r-b") + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, []explorer.ExplicitGroupMembership{ + grpmMembership(grpmGroupA, "r-a"), grpmMembership(grpmGroupB, "r-b"), grpmMembership(grpmGroupA, "r-b"), + }); err != nil { + f.fail("abort seed append", err) + } + if got := f.countMembers(t, string(revision.ID)); got != 3 { + t.Fatalf("abort seed count = %d, want 3", got) + } + mustErr(t, "abort with the wrong token", func() error { + return f.store.AbortExplicitGroupRevision(f.ctx, revision.ID, token+"-wrong") + }, explorer.ErrExplicitGroupRevisionConflict) + if got := f.countMembers(t, string(revision.ID)); got != 3 { + t.Fatalf("refused abort moved memberships: %d", got) + } + if err := f.store.AbortExplicitGroupRevision(f.ctx, revision.ID, token); err != nil { + f.fail("owner abort", err) + } + if got := f.countMembers(t, string(revision.ID)); got != 0 { + t.Fatalf("memberships after abort = %d, want 0 (cascade)", got) + } + if got := f.countRowsOn(t, f.definitionsTable(), "revision_id = $1", string(revision.ID)); got != 0 { + t.Fatalf("definitions after abort = %d, want 0 (cascade)", got) + } + if got := f.countRowsOn(t, f.revisionTable(), "id = $1", string(revision.ID)); got != 0 { + t.Fatalf("header after abort = %d, want 0", got) + } + + live, liveToken, _ := f.stagedGroupFixture(t, "cleanup-live", "r-a") + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, live.ID, liveToken, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}); err != nil { + f.fail("cleanup live seed append", err) + } + expiredLease := time.Now().UTC().Add(-time.Minute) + source := f.stageSelection(t, "cleanup", "r-a") + expired := grpmRevision(source, "grpm-key-cleanup-expired") + f.rawInsertHeaderGRPM(t, f.witness, expired, "grpm-cleanup-writer", &expiredLease) + f.rawInsertDefinition(t, f.witness, string(expired.ID), string(grpmGroupA), "Group A", 0) + f.rawInsertDefinition(t, f.witness, string(expired.ID), string(grpmGroupB), "Group B", 1) + f.rawInsertMembership(t, f.witness, string(expired.ID), string(grpmGroupA), grpmProject, grpmGeneration, grpmResourceType, "r-a") + f.rawInsertMembership(t, f.witness, string(expired.ID), string(grpmGroupB), grpmProject, grpmGeneration, grpmResourceType, "r-a") + + _, err := f.witness.Exec(f.ctx, `DELETE FROM `+f.membershipsTable()+` WHERE revision_id = $1`, string(expired.ID)) + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "owner-abort or expired-writer-cleanup", + "unauthorized membership DELETE on expired staging") + + if err := f.store.CleanupExplicitGroupStaging(f.ctx, time.Now().UTC(), 100); err != nil { + f.fail("expired-writer cleanup", err) + } + if got := f.countRowsOn(t, f.revisionTable(), "id = $1", string(expired.ID)); got != 0 { + t.Fatalf("expired header survived cleanup: %d", got) + } + if got := f.countMembers(t, string(expired.ID)); got != 0 { + t.Fatalf("expired memberships survived cleanup: %d", got) + } + if got := f.countRowsOn(t, f.definitionsTable(), "revision_id = $1", string(expired.ID)); got != 0 { + t.Fatalf("expired definitions survived cleanup: %d", got) + } + if got := f.countMembers(t, string(live.ID)); got != 1 { + t.Fatalf("active writer's memberships were swept: %d, want 1", got) + } + if got := f.countRowsOn(t, f.revisionTable(), "id = $1", string(live.ID)); got != 1 { + t.Fatalf("active writer's header was swept") + } + f.setGUC(t, "loom.explorer_explicit_group_abort_token", liveToken) + _, err = f.witness.Exec(f.ctx, `DELETE FROM `+f.revisionTable()+` WHERE id = $1`, string(live.ID)) + pgErr := requirePgErrGRPM(t, err, "23001", "authorized header DELETE while children remain") + if !strings.Contains(pgErr.Message, "explorer_explicit_group_definitions_revision_fkey") && + !strings.Contains(pgErr.Message, explicitGroupMembershipRevisionFK) { + t.Fatalf("header DELETE refusal %q names neither child RESTRICT foreign key", pgErr.Message) + } + _, err = f.witness.Exec(f.ctx, `DELETE FROM `+f.definitionsTable()+` WHERE revision_id = $1`, string(live.ID)) + pgErr = requirePgErrGRPM(t, err, "23001", "authorized definition DELETE while memberships remain") + if !strings.Contains(pgErr.Message, explicitGroupMembershipGroupFK) { + t.Fatalf("definition DELETE refusal %q does not name the membership group RESTRICT foreign key", pgErr.Message) + } + f.execOnGRPM(t, f.witness, "authorized membership delete", `DELETE FROM `+f.membershipsTable()+` WHERE revision_id = $1`, string(live.ID)) + f.execOnGRPM(t, f.witness, "authorized definition delete", `DELETE FROM `+f.definitionsTable()+` WHERE revision_id = $1`, string(live.ID)) + f.execOnGRPM(t, f.witness, "authorized header delete", `DELETE FROM `+f.revisionTable()+` WHERE id = $1`, string(live.ID)) + f.setGUC(t, "loom.explorer_explicit_group_abort_token", "") +} + +func TestPostgresExplicitGroupMembershipsFaultSeamRollback(t *testing.T) { + f := newGRPMFixture(t) + f.migrateWithSelections(t) + + revision, token, _ := f.stagedGroupFixture(t, "fault-append", "r-a", "r-b") + headerBefore := f.grpmHeaderSnapshot(t, string(revision.ID)) + storeA, _ := f.newDedicatedStore(t) + var insideRows int64 + storeA.afterMembershipsInserted = func(ctx context.Context, tx pgx.Tx) error { + if err := tx.QueryRow(ctx, `SELECT count(*) FROM `+f.membershipsTable()+ + ` WHERE revision_id = $1`, string(revision.ID)).Scan(&insideRows); err != nil { + return err + } + return errors.New("injected crash after membership inserts") + } + _, err := storeA.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, []explorer.ExplicitGroupMembership{ + grpmMembership(grpmGroupA, "r-a"), grpmMembership(grpmGroupB, "r-b"), + }) + if err == nil || !strings.Contains(err.Error(), "injected crash") { + t.Fatalf("append fault error = %v, want the injected crash", err) + } + if insideRows != 2 { + t.Fatalf("the seam saw %d rows inside the transaction, want 2 (the inserts were real)", insideRows) + } + if got := f.membershipSnapshot(t, string(revision.ID)); got != "" { + t.Fatalf("crashed append leaked rows: %q", got) + } + if got := f.grpmHeaderSnapshot(t, string(revision.ID)); got != headerBefore { + t.Fatalf("crashed append moved the header: %q != %q", got, headerBefore) + } + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, []explorer.ExplicitGroupMembership{ + grpmMembership(grpmGroupA, "r-a"), grpmMembership(grpmGroupB, "r-b"), + }); err != nil { + f.fail("honest append after fault", err) + } + + storeB, _ := f.newDedicatedStore(t) + var flippedState string + storeB.beforeCompleteCommit = func(ctx context.Context, tx pgx.Tx) error { + if err := tx.QueryRow(ctx, `SELECT state FROM `+f.revisionTable()+ + ` WHERE id = $1`, string(revision.ID)).Scan(&flippedState); err != nil { + return err + } + return errors.New("injected crash before complete commit") + } + defDigest, memDigest, groups, members, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + if err != nil { + f.fail("pre-fault digest", err) + } + if _, err := storeB.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, + defDigest, memDigest, groups, members, time.Now().UTC()); err == nil || + !strings.Contains(err.Error(), "injected crash") { + t.Fatalf("complete fault error = %v, want the injected crash", err) + } + if flippedState != "COMPLETE" { + t.Fatalf("the seam saw state %q inside the transaction, want COMPLETE (the CAS was real)", flippedState) + } + if got := f.grpmHeaderSnapshot(t, string(revision.ID)); !strings.Contains(got, "\x1fSTAGING\x1f") { + t.Fatalf("crashed completion left the header non-STAGING: %q", got) + } + if got := f.membershipSnapshot(t, string(revision.ID)); got != grpmRows( + grpmRow(string(grpmGroupA), "r-a"), grpmRow(string(grpmGroupB), "r-b")) { + t.Fatalf("crashed completion disturbed memberships: %q", got) + } + if _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, + defDigest, memDigest, groups, members, time.Now().UTC()); err != nil { + f.fail("completion after crash", err) + } +} + +func TestPostgresExplicitGroupMembershipsFencedCompleteRacesAppend(t *testing.T) { + f := newGRPMFixture(t) + f.migrateWithSelections(t) + + storeA, pidA := f.newDedicatedStore(t) + storeB, pidB := f.newDedicatedStore(t) + requireDistinctBackendsGRPM(t, map[string]int32{"appender": pidA, "completer": pidB, "witness": f.witnessID}) + + var appendBlocks, completeBlocks, staleConflicts atomic.Int64 + for round := 0; round < grpmRaceRounds; round++ { + roundStr := fmt.Sprintf("race-%d", round) + revision, token, _ := f.stagedGroupFixture(t, roundStr, "r-a", "r-b", "r-c") + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}); err != nil { + f.fail("race seed append", err) + } + + releaseAppend := make(chan struct{}) + appendEntered := make(chan struct{}) + var appendOnce sync.Once + storeA.afterMembershipsInserted = func(context.Context, pgx.Tx) error { + appendOnce.Do(func() { close(appendEntered) }) + <-releaseAppend + return nil + } + watchB, stopB := f.grpmWatchLockWait(pidB) + type appendOutcome struct { + err error + } + appendDone := make(chan appendOutcome, 1) + go func() { + _, err := storeA.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupB, "r-b")}) + appendDone <- appendOutcome{err} + }() + <-appendEntered + + staleDef, staleMem, groups, members, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + if err != nil { + f.fail("race stale digest", err) + } + completeDone := make(chan error, 1) + go func() { + _, err := storeB.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, + staleDef, staleMem, groups, members, time.Now().UTC()) + completeDone <- err + }() + + locked := false + deadline := time.Now().Add(grpmSeamTimeout) + for !locked && time.Now().Before(deadline) { + locked = watchB() + if locked { + appendBlocks.Add(1) + } + time.Sleep(2 * time.Millisecond) + } + close(releaseAppend) + if err := (<-appendDone).err; err != nil { + f.fail("race append commit", err) + } + completeErr := <-completeDone + stopB() + if !watchB() { + t.Fatalf("round %d: the completer was never observed blocked on the header lock -- the race never actually raced", round) + } + completeBlocks.Add(1) + + if completeErr == nil { + state := f.grpmHeaderSnapshot(t, string(revision.ID)) + if !strings.Contains(state, "\x1fCOMPLETE\x1f") { + t.Fatalf("round %d: completer succeeded on a non-COMPLETE header: %q", round, state) + } + if _, _, _, frozenMembers, derr := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID); derr != nil || frozenMembers != members { + t.Fatalf("round %d: COMPLETE frozen member_count %d != claimed %d", round, frozenMembers, members) + } + continue + } + if !errors.Is(completeErr, explorer.ErrExplicitGroupRevisionConflict) { + f.fail("race completion", completeErr) + } + staleConflicts.Add(1) + state := f.grpmHeaderSnapshot(t, string(revision.ID)) + if !strings.Contains(state, "\x1fSTAGING\x1f") { + t.Fatalf("round %d: conflicted completion left state: %q", round, state) + } + if got := f.membershipSnapshot(t, string(revision.ID)); got != grpmRows( + grpmRow(string(grpmGroupA), "r-a"), grpmRow(string(grpmGroupB), "r-b")) { + t.Fatalf("round %d: post-race membership set = %q", round, got) + } + mustErr(t, "race stale retry", func() error { + _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, strings.Repeat("0", 64), staleMem, groups, members, time.Now().UTC()) + return err + }, explorer.ErrExplicitGroupRevisionConflict) + honestDef, honestMem, honestGroups, honestMembers, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + if err != nil { + f.fail("race honest digest", err) + } + if _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, + honestDef, honestMem, honestGroups, honestMembers, time.Now().UTC()); err != nil { + f.fail("race honest completion", err) + } + if _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, + honestDef, honestMem, honestGroups, honestMembers, time.Now().UTC()); err != nil { + f.fail("race idempotent completion", err) + } + storeA.afterMembershipsInserted = nil + } + if appendBlocks.Load() == 0 { + t.Fatalf("no round ever witnessed the completer blocked behind the appender's lock -- zero-churn control failed") + } + if staleConflicts.Load() == 0 { + t.Fatalf("no round produced the stale-claim conflict -- the completion never fenced a real append") + } +} + +func TestPostgresExplicitGroupMembershipsOneWayMachineRefusesReversal(t *testing.T) { + f := newGRPMFixture(t) + f.migrateWithSelections(t) + + revision, token, _ := f.stagedGroupFixture(t, "oneway", "r-a", "r-b") + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}); err != nil { + f.fail("one-way seed append", err) + } + defDigest, memDigest, groups, members, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + if err != nil { + f.fail("one-way digest", err) + } + if _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, + defDigest, memDigest, groups, members, time.Now().UTC()); err != nil { + f.fail("one-way completion", err) + } + + fullReversal := `UPDATE ` + f.revisionTable() + + ` SET state = 'STAGING', definition_digest = '', membership_digest = '',` + + ` group_count = 0, member_count = 0, completed_at = NULL WHERE id = $1` + stateOnlyReversal := `UPDATE ` + f.revisionTable() + ` SET state = 'STAGING' WHERE id = $1` + + for _, probe := range []struct { + step string + sql string + }{ + {"full COMPLETE->STAGING reversal", fullReversal}, + {"bare COMPLETE->STAGING state flip", stateOnlyReversal}, + } { + _, err := f.witness.Exec(f.ctx, probe.sql, string(revision.ID)) + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "COMPLETE history and may never be modified", + probe.step) + if state := f.grpmHeaderSnapshot(t, string(revision.ID)); !strings.Contains(state, "\x1fCOMPLETE\x1f") { + t.Fatalf("%s: refused yet the header reads %q", probe.step, state) + } + } + + f.dropTriggerGRPM(t, f.revisionTable(), "explorer_explicit_group_revisions_guard") + f.execOnGRPM(t, f.witness, "guard-dropped reversal", fullReversal, string(revision.ID)) + if state := f.grpmHeaderSnapshot(t, string(revision.ID)); !strings.Contains(state, "\x1fSTAGING\x1f") { + t.Fatalf("guard-dropped reversal did not land (header %q) -- the probe is vacuous", state) + } + + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupB, "r-b")}); err != nil { + f.fail("revived append", err) + } + revivedDef, revivedMem, revivedGroups, revivedMembers, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + if err != nil { + f.fail("revived digest", err) + } + if _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, + revivedDef, revivedMem, revivedGroups, revivedMembers, time.Now().UTC()); err != nil { + f.fail("revived completion", err) + } + if state := f.grpmHeaderSnapshot(t, string(revision.ID)); !strings.Contains(state, "\x1fCOMPLETE\x1f") { + t.Fatalf("revived row did not re-complete: %q", state) + } + + f.rerunMigrationGRPM(t, grpmMigrationVersion) + for _, probe := range []struct { + step string + sql string + }{ + {"reversed-after-rerun", fullReversal}, + {"state-flip-after-rerun", stateOnlyReversal}, + } { + _, err := f.witness.Exec(f.ctx, probe.sql, string(revision.ID)) + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "COMPLETE history and may never be modified", + probe.step) + } +} + +func TestPostgresExplicitGroupMembershipsGuardLocksParentUnderCompletionRace(t *testing.T) { + f := newGRPMFixture(t) + f.migrateWithSelections(t) + + storeB, pidB := f.newDedicatedStore(t) + rawPool, rawPID := f.newDedicatedPool(t) + requireDistinctBackendsGRPM(t, map[string]int32{"completer": pidB, "raw": rawPID, "witness": f.witnessID}) + + rawInsertMember := func(table string) string { + return `INSERT INTO ` + table + ` (revision_id, group_id, project, generation, resource_type, resource_id) VALUES ($1, $2, $3, $4, $5, $6)` + } + + var insertBlocked, deleteBlocked, insertRefused, deleteRefused atomic.Int64 + for round := 0; round < grpmRaceRounds; round++ { + roundStr := fmt.Sprintf("toctou-%d", round) + + revision, token, _ := f.stagedGroupFixture(t, roundStr+"-i", "r-a", "r-b") + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}); err != nil { + f.fail("toctou insert seed", err) + } + honestDef, honestMem, groups, members, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + if err != nil { + f.fail("toctou insert digest", err) + } + release, completeDone, entered := grpmHoldCompletion(storeB, f.ctx, revision, token, honestDef, honestMem, groups, members) + <-entered + + watchA, stopA := f.grpmWatchLockWait(rawPID) + insertErr := make(chan error, 1) + go func() { + _, err := rawPool.Exec(f.ctx, rawInsertMember(f.membershipsTable()), + string(revision.ID), string(grpmGroupB), grpmProject, grpmGeneration, grpmResourceType, "r-b") + insertErr <- err + }() + blocked := false + deadline := time.Now().Add(grpmSeamTimeout) + for !blocked && time.Now().Before(deadline) { + if watchA() { + blocked = true + insertBlocked.Add(1) + } + time.Sleep(2 * time.Millisecond) + } + close(release) + if err := <-completeDone; err != nil { + f.fail("toctou insert completion", err) + } + err = <-insertErr + stopA() + if !blocked || !watchA() { + t.Fatalf("round %d: direct-SQL INSERT was never observed queued behind the completing header lock -- the race never actually raced", round) + } + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "only STAGING revisions accept members", + "direct insert authorized against a stale STAGING parent") + if got := f.membershipSnapshot(t, string(revision.ID)); got != grpmRows(grpmRow(string(grpmGroupA), "r-a")) { + t.Fatalf("round %d: refused intruder still changed the set: %q", round, got) + } + frozenDef, frozenMem, _, frozenMembers, derr := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + if derr != nil || frozenDef != honestDef || frozenMem != honestMem || frozenMembers != members { + t.Fatalf("round %d: COMPLETE frozen facts drifted after the refused insert: %v %v %d (wanted %v %v %d)", + round, frozenDef, frozenMem, frozenMembers, honestDef, honestMem, members) + } + insertRefused.Add(1) + + control, controlToken, _ := f.stagedGroupFixture(t, roundStr+"-ic", "r-a", "r-b") + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, control.ID, controlToken, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}); err != nil { + f.fail("toctou insert control seed", err) + } + f.execOnGRPM(t, rawPool, "toctou control insert", rawInsertMember(f.membershipsTable()), + string(control.ID), string(grpmGroupB), grpmProject, grpmGeneration, grpmResourceType, "r-b") + if got := f.membershipSnapshot(t, string(control.ID)); got != grpmRows( + grpmRow(string(grpmGroupA), "r-a"), grpmRow(string(grpmGroupB), "r-b")) { + t.Fatalf("round %d: no-race control insert did not land: %q", round, got) + } + + revision2, token2, _ := f.stagedGroupFixture(t, roundStr+"-d", "r-a", "r-b") + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision2.ID, token2, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}); err != nil { + f.fail("toctou delete seed", err) + } + dDef, dMem, dGroups, dMembers, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision2.ID) + if err != nil { + f.fail("toctou delete digest", err) + } + release2, completeDone2, entered2 := grpmHoldCompletion(storeB, f.ctx, revision2, token2, dDef, dMem, dGroups, dMembers) + <-entered2 + + if _, err := rawPool.Exec(f.ctx, + `SELECT set_config('loom.explorer_explicit_group_abort_token', $1, false)`, token2); err != nil { + f.fail("toctou set abort GUC", err) + } + watchD, stopD := f.grpmWatchLockWait(rawPID) + deleteErr := make(chan error, 1) + go func() { + _, err := rawPool.Exec(f.ctx, `DELETE FROM `+f.membershipsTable()+` WHERE revision_id = $1`, + string(revision2.ID)) + deleteErr <- err + }() + blocked = false + deadline = time.Now().Add(grpmSeamTimeout) + for !blocked && time.Now().Before(deadline) { + if watchD() { + blocked = true + deleteBlocked.Add(1) + } + time.Sleep(2 * time.Millisecond) + } + close(release2) + if err := <-completeDone2; err != nil { + f.fail("toctou delete completion", err) + } + err = <-deleteErr + stopD() + if _, err := rawPool.Exec(f.ctx, + `SELECT set_config('loom.explorer_explicit_group_abort_token', '', false)`); err != nil { + f.fail("toctou reset abort GUC", err) + } + if !blocked || !watchD() { + t.Fatalf("round %d: abort-channel DELETE was never observed queued behind the completing header lock -- the race never actually raced", round) + } + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "DELETE refused: parent revision", + "abort-channel DELETE authorized against a stale STAGING parent") + if got := f.membershipSnapshot(t, string(revision2.ID)); got != grpmRows(grpmRow(string(grpmGroupA), "r-a")) { + t.Fatalf("round %d: refused abort DELETE still removed rows: %q", round, got) + } + if state := f.grpmHeaderSnapshot(t, string(revision2.ID)); !strings.Contains(state, "\x1fCOMPLETE\x1f") { + t.Fatalf("round %d: raced completion left header %q", round, state) + } + deleteRefused.Add(1) + + control2, controlToken2, _ := f.stagedGroupFixture(t, roundStr+"-dc", "r-a") + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, control2.ID, controlToken2, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}); err != nil { + f.fail("toctou delete control seed", err) + } + if _, err := rawPool.Exec(f.ctx, + `SELECT set_config('loom.explorer_explicit_group_abort_token', $1, false)`, controlToken2); err != nil { + f.fail("toctou control set abort GUC", err) + } + f.execOnGRPM(t, rawPool, "toctou control delete", + `DELETE FROM `+f.membershipsTable()+` WHERE revision_id = $1`, string(control2.ID)) + if _, err := rawPool.Exec(f.ctx, + `SELECT set_config('loom.explorer_explicit_group_abort_token', '', false)`); err != nil { + f.fail("toctou control reset abort GUC", err) + } + if got := f.membershipSnapshot(t, string(control2.ID)); got != "" { + t.Fatalf("round %d: no-race control abort DELETE did not land: %q", round, got) + } + + storeB.beforeCompleteCommit = nil + } + if insertBlocked.Load() == 0 || deleteBlocked.Load() == 0 { + t.Fatalf("no round queued the intruding direct-SQL operation behind the completing header lock (insert %d, delete %d) -- the guard is not locking the parent", + insertBlocked.Load(), deleteBlocked.Load()) + } + if insertRefused.Load() == 0 || deleteRefused.Load() == 0 { + t.Fatalf("no round refused the stale-parent intruder (insert %d, delete %d)", + insertRefused.Load(), deleteRefused.Load()) + } +} + +func (f *grpmFixture) grpmDefinitionCount(t *testing.T, revisionID string) int64 { + t.Helper() + var n int64 + if err := f.witness.QueryRow(f.ctx, `SELECT count(*) FROM `+f.definitionsTable()+ + ` WHERE revision_id = $1`, revisionID).Scan(&n); err != nil { + f.fail("independent definition count", err) + } + return n +} + +func TestPostgresExplicitGroupDefinitionsGuardRefusesWritesUnderComplete(t *testing.T) { + f := newGRPMFixture(t) + f.migrateWithSelections(t) + + owner, err := OwnerMigrations() + if err != nil { + f.fail("load owner migrations", err) + } + var pinned bool + for _, m := range owner { + if m.Version == grpmDefinitionsGuardVersion { + pinned = true + if m.Name != grpmDefinitionsGuardName { + t.Fatalf("migration 11 name = %q, want %q", m.Name, grpmDefinitionsGuardName) + } + if got := fmt.Sprintf("%x", sha256.Sum256([]byte(m.SQL))); got != grpmDefinitionsGuardChecksum { + t.Fatalf("000011 embedded bytes drifted: sha256 %s, want %s", got, grpmDefinitionsGuardChecksum) + } + } + } + if !pinned { + t.Fatalf("migration %d (%s) missing from OwnerMigrations", grpmDefinitionsGuardVersion, grpmDefinitionsGuardName) + } + + revision, token, _ := f.stagedGroupFixture(t, "defguard", "r-a", "r-b") + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}); err != nil { + f.fail("definition-guard seed append", err) + } + defDigest, memDigest, groups, members, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + if err != nil { + f.fail("definition-guard digest", err) + } + if _, err := f.store.CompleteExplicitGroupRevision(f.ctx, revision.ID, token, + defDigest, memDigest, groups, members, time.Now().UTC()); err != nil { + f.fail("definition-guard completion", err) + } + + rawDefInsert := func(group string, ordinal int64) error { + _, err := f.witness.Exec(f.ctx, `INSERT INTO `+f.definitionsTable()+ + ` (revision_id, group_id, label, ordinal) VALUES ($1, $2, $3, $4)`, + string(revision.ID), group, "Guard probe", ordinal) + return err + } + rawDefDelete := func(group string) error { + _, err := f.witness.Exec(f.ctx, `DELETE FROM `+f.definitionsTable()+ + ` WHERE revision_id = $1 AND group_id = $2`, string(revision.ID), group) + return err + } + if _, err := f.witness.Exec(f.ctx, + `SELECT set_config('loom.explorer_explicit_group_abort_token', $1, false)`, token); err != nil { + f.fail("definition-guard set abort GUC", err) + } + t.Cleanup(func() { + _, _ = f.witness.Exec(f.ctx, `SELECT set_config('loom.explorer_explicit_group_abort_token', '', false)`) + }) + + requirePgErrTriggerGRPM(t, rawDefInsert("group-c", 2), grpmCheckViolation, + "only STAGING revisions accept definitions", "definition INSERT under COMPLETE") + requirePgErrTriggerGRPM(t, rawDefDelete(string(grpmGroupB)), grpmCheckViolation, + "DELETE of definition", "definition DELETE under COMPLETE") + if n := f.grpmDefinitionCount(t, string(revision.ID)); n != 2 { + t.Fatalf("refused probes changed the stored definition count: %d", n) + } + + f.dropTriggerGRPM(t, f.definitionsTable(), "explorer_explicit_group_definitions_guard") + f.execOnGRPM(t, f.witness, "guard-dropped definition insert", + `INSERT INTO `+f.definitionsTable()+` (revision_id, group_id, label, ordinal) VALUES ($1, $2, $3, $4)`, + string(revision.ID), "group-c", "Guard probe", int64(2)) + f.execOnGRPM(t, f.witness, "guard-dropped definition delete", + `DELETE FROM `+f.definitionsTable()+` WHERE revision_id = $1 AND group_id = $2`, + string(revision.ID), string(grpmGroupB)) + if n := f.grpmDefinitionCount(t, string(revision.ID)); n != 2 { + t.Fatalf("guard-dropped probes did not land as expected: %d rows", n) + } + var frozenCount int64 + if err := f.witness.QueryRow(f.ctx, `SELECT group_count FROM `+f.revisionTable()+ + ` WHERE id = $1`, string(revision.ID)).Scan(&frozenCount); err != nil { + f.fail("read frozen group_count", err) + } + if frozenCount != 2 { + t.Fatalf("frozen group_count drifted under guard drop: %d", frozenCount) + } + + f.rerunMigrationGRPM(t, grpmDefinitionsGuardVersion) + requirePgErrTriggerGRPM(t, rawDefInsert("group-d", 3), grpmCheckViolation, + "only STAGING revisions accept definitions", "definition INSERT after 000011 rerun") + requirePgErrTriggerGRPM(t, rawDefDelete(string(grpmGroupA)), grpmCheckViolation, + "DELETE of definition", "definition DELETE after 000011 rerun") +} + +func TestPostgresExplicitGroupDefinitionsGuardLocksParentUnderCompletionRace(t *testing.T) { + f := newGRPMFixture(t) + f.migrateWithSelections(t) + + storeB, pidB := f.newDedicatedStore(t) + rawPool, rawPID := f.newDedicatedPool(t) + requireDistinctBackendsGRPM(t, map[string]int32{"completer": pidB, "raw": rawPID, "witness": f.witnessID}) + + defCount := func(revisionID string) int64 { + var n int64 + if err := f.witness.QueryRow(f.ctx, `SELECT count(*) FROM `+f.definitionsTable()+ + ` WHERE revision_id = $1`, revisionID).Scan(&n); err != nil { + f.fail("independent definition count", err) + } + return n + } + frozenGroupCount := func(revisionID string) int64 { + var n int64 + if err := f.witness.QueryRow(f.ctx, `SELECT group_count FROM `+f.revisionTable()+ + ` WHERE id = $1`, revisionID).Scan(&n); err != nil { + f.fail("independent group_count read", err) + } + return n + } + + var insertBlocked, deleteBlocked, insertRefused, deleteRefused atomic.Int64 + for round := 0; round < grpmRaceRounds; round++ { + roundStr := fmt.Sprintf("toctou-def-%d", round) + + revision, token, _ := f.stagedGroupFixture(t, roundStr+"-i", "r-a") + defDigest, memDigest, groups, members, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision.ID) + if err != nil { + f.fail("def race insert digest", err) + } + release, completeDone, entered := grpmHoldCompletion(storeB, f.ctx, revision, token, defDigest, memDigest, groups, members) + <-entered + + watchA, stopA := f.grpmWatchLockWait(rawPID) + insertErr := make(chan error, 1) + go func() { + _, err := rawPool.Exec(f.ctx, `INSERT INTO `+f.definitionsTable()+ + ` (revision_id, group_id, label, ordinal) VALUES ($1, $2, $3, $4)`, + string(revision.ID), "group-c", "Race intruder", int64(2)) + insertErr <- err + }() + blocked := false + deadline := time.Now().Add(grpmSeamTimeout) + for !blocked && time.Now().Before(deadline) { + if watchA() { + blocked = true + insertBlocked.Add(1) + } + time.Sleep(2 * time.Millisecond) + } + close(release) + if err := <-completeDone; err != nil { + f.fail("def race insert completion", err) + } + err = <-insertErr + stopA() + if !blocked || !watchA() { + t.Fatalf("round %d: definition INSERT never queued behind the completing header lock -- the race never actually raced", round) + } + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "only STAGING revisions accept definitions", + "definition INSERT authorized against a stale STAGING parent") + revisionID := string(revision.ID) + if n := defCount(revisionID); n != 2 { + t.Fatalf("round %d: refused definition INSERT changed the stored count: %d", round, n) + } + if frozen, stored := frozenGroupCount(revisionID), defCount(revisionID); frozen != stored || frozen != 2 { + t.Fatalf("round %d: served-state drift -- frozen group_count %d vs stored definitions %d", round, frozen, stored) + } + insertRefused.Add(1) + + control, _, _ := f.stagedGroupFixture(t, roundStr+"-ic", "r-a") + f.execOnGRPM(t, rawPool, "def race control insert", `INSERT INTO `+f.definitionsTable()+ + ` (revision_id, group_id, label, ordinal) VALUES ($1, $2, $3, $4)`, + string(control.ID), "group-c", "Race control", int64(2)) + if n := defCount(string(control.ID)); n != 3 { + t.Fatalf("round %d: no-race control INSERT did not land: %d rows", round, n) + } + + revision2, token2, _ := f.stagedGroupFixture(t, roundStr+"-d", "r-a") + dDef, dMem, dGroups, dMembers, err := f.store.DigestExplicitGroupRevision(f.ctx, grpmProject, revision2.ID) + if err != nil { + f.fail("def race delete digest", err) + } + release2, completeDone2, entered2 := grpmHoldCompletion(storeB, f.ctx, revision2, token2, dDef, dMem, dGroups, dMembers) + <-entered2 + + if _, err := rawPool.Exec(f.ctx, + `SELECT set_config('loom.explorer_explicit_group_abort_token', $1, false)`, token2); err != nil { + f.fail("def race set abort GUC", err) + } + watchD, stopD := f.grpmWatchLockWait(rawPID) + deleteErr := make(chan error, 1) + go func() { + _, err := rawPool.Exec(f.ctx, `DELETE FROM `+f.definitionsTable()+ + ` WHERE revision_id = $1 AND group_id = $2`, string(revision2.ID), string(grpmGroupB)) + deleteErr <- err + }() + blocked = false + deadline = time.Now().Add(grpmSeamTimeout) + for !blocked && time.Now().Before(deadline) { + if watchD() { + blocked = true + deleteBlocked.Add(1) + } + time.Sleep(2 * time.Millisecond) + } + close(release2) + if err := <-completeDone2; err != nil { + f.fail("def race delete completion", err) + } + err = <-deleteErr + stopD() + if _, err := rawPool.Exec(f.ctx, + `SELECT set_config('loom.explorer_explicit_group_abort_token', '', false)`); err != nil { + f.fail("def race reset abort GUC", err) + } + if !blocked || !watchD() { + t.Fatalf("round %d: abort-channel definition DELETE never queued behind the completing header lock -- the race never actually raced", round) + } + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "DELETE of definition", + "definition DELETE authorized against a stale STAGING parent") + revisionID2 := string(revision2.ID) + if frozen, stored := frozenGroupCount(revisionID2), defCount(revisionID2); frozen != stored || frozen != 2 { + t.Fatalf("round %d: E1 drift -- frozen group_count %d vs stored definitions %d", round, frozen, stored) + } + deleteRefused.Add(1) + + control2, controlToken2, _ := f.stagedGroupFixture(t, roundStr+"-dc", "r-a") + if _, err := rawPool.Exec(f.ctx, + `SELECT set_config('loom.explorer_explicit_group_abort_token', $1, false)`, controlToken2); err != nil { + f.fail("def race control set abort GUC", err) + } + f.execOnGRPM(t, rawPool, "def race control delete", `DELETE FROM `+f.definitionsTable()+ + ` WHERE revision_id = $1 AND group_id = $2`, string(control2.ID), string(grpmGroupB)) + if _, err := rawPool.Exec(f.ctx, + `SELECT set_config('loom.explorer_explicit_group_abort_token', '', false)`); err != nil { + f.fail("def race control reset abort GUC", err) + } + if n := defCount(string(control2.ID)); n != 1 { + t.Fatalf("round %d: no-race control abort DELETE did not land: %d rows", round, n) + } + + storeB.beforeCompleteCommit = nil + } + if insertBlocked.Load() == 0 || deleteBlocked.Load() == 0 { + t.Fatalf("no round queued the intruding definition statement behind the completing header lock (insert %d, delete %d) -- the guard is not locking the parent", + insertBlocked.Load(), deleteBlocked.Load()) + } + if insertRefused.Load() == 0 || deleteRefused.Load() == 0 { + t.Fatalf("no round refused the stale-parent definition intruder (insert %d, delete %d)", + insertRefused.Load(), deleteRefused.Load()) + } +} + +func TestPostgresExplicitGroupMembershipsMigrationLedgerConvergence(t *testing.T) { + f := newGRPMFixture(t) + result := f.migrateWithSelections(t) + + var versions []int64 + for _, rec := range result.Applied { + versions = append(versions, rec.Version) + } + wantVersions := []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} + if len(versions) != len(wantVersions) { + t.Fatalf("ledger versions = %v, want %v", versions, wantVersions) + } + for i := range wantVersions { + if versions[i] != wantVersions[i] { + t.Fatalf("ledger versions = %v, want %v", versions, wantVersions) + } + } + for _, rec := range result.Applied { + switch rec.Version { + case grpmMigrationVersion: + if rec.Name != grpmMigrationName { + t.Fatalf("ledger version %d name = %q, want %q", rec.Version, rec.Name, grpmMigrationName) + } + if rec.Checksum != grpmMigrationChecksum { + t.Fatalf("ledger checksum for %d = %s, want the frozen pin %s", + rec.Version, rec.Checksum, grpmMigrationChecksum) + } + case grpmSelectionsFixtureVersion: + if rec.Checksum != grpmSelectionsFixtureChecksum { + t.Fatalf("ledger checksum for %d = %s, want the frozen pin %s", + rec.Version, rec.Checksum, grpmSelectionsFixtureChecksum) + } + } + } + + migrations, err := OwnerMigrations() + if err != nil { + f.fail("owner migrations for convergence", err) + } + var found bool + for _, m := range migrations { + if m.Version == grpmMigrationVersion { + found = true + sum := sha256.Sum256([]byte(m.SQL)) + if hex.EncodeToString(sum[:]) != grpmMigrationChecksum || m.Name != grpmMigrationName { + t.Fatalf("embedded membership migration drifted: name %q checksum %s", m.Name, hex.EncodeToString(sum[:])) + } + } + } + if !found { + t.Fatal("the membership migration is absent from the embedded owner history") + } + + client, err := pgstore.Open(f.ctx, pgstore.Config{DSN: f.dsn, MaxConns: 2, MinConns: 0, AcquireTimeout: 10 * time.Second}) + if err != nil { + f.fail("open rerun pool", err) + } + defer client.Close() + rerunHistory, err := OwnerMigrations() + if err != nil { + f.fail("owner migrations for rerun", err) + } + runner, err := pgstore.NewMigrationRunner(client, f.schema, rerunHistory) + if err != nil { + f.fail("new rerun runner", err) + } + rerun, err := runner.Migrate(f.ctx) + if err != nil { + f.fail("convergent rerun", err) + } + if len(rerun.Applied) != len(result.Applied) { + t.Fatalf("rerun ledger length = %d, want %d", len(rerun.Applied), len(result.Applied)) + } + var ledgerRows int + if err := f.witness.QueryRow(f.ctx, `SELECT count(*) FROM `+sanitizeTableRef(f.schema, "schema_migrations")).Scan(&ledgerRows); err != nil { + f.fail("count ledger rows", err) + } + if ledgerRows != len(result.Applied) { + t.Fatalf("ledger holds %d rows, want %d after a convergent rerun", ledgerRows, len(result.Applied)) + } + + revision, token, _ := f.stagedGroupFixture(t, "converge", "r-a") + if _, err := f.store.AppendExplicitGroupMemberships(f.ctx, revision.ID, token, + []explorer.ExplicitGroupMembership{grpmMembership(grpmGroupA, "r-a")}); err != nil { + f.fail("seed append before guard probe", err) + } + _, err = f.witness.Exec(f.ctx, `DELETE FROM `+f.membershipsTable()+` WHERE revision_id = $1`, string(revision.ID)) + requirePgErrTriggerGRPM(t, err, grpmCheckViolation, "owner-abort or expired-writer-cleanup", + "membership DELETE after convergent rerun") +} + +func TestZZZeroLeftoverVerifyGrpmFixtures(t *testing.T) { + raw := pgstate06TestPostgresURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for leftover audit: %v", err) + } + defer admin.Close() + var dbLeftovers int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\_grpm\_%' ESCAPE '\'`).Scan(&dbLeftovers); err != nil { + t.Fatalf("audit query: %v", err) + } + if dbLeftovers != 0 { + t.Fatalf("%d verify_grpm_* databases survive the suite", dbLeftovers) + } + if owned := grpmSnapshotRegistry(); len(owned) != 0 { + t.Fatalf("creation registry still holds %d entries: %v", len(owned), owned) + } +} diff --git a/internal/backends/postgres/internal/explorer/explorers.go b/internal/backends/postgres/internal/explorer/explorers.go new file mode 100644 index 000000000..93fcea68b --- /dev/null +++ b/internal/backends/postgres/internal/explorer/explorers.go @@ -0,0 +1,259 @@ +package postgres + +import ( + "context" + "embed" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/jackc/pgx/v5" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/explorer" +) + +const OwnerTable = "explorers" + +//go:embed migrations/*.sql +var migrationsFS embed.FS + +func OwnerMigrations() ([]pgstore.Migration, error) { + migrations, err := pgstore.LoadMigrations(migrationsFS, "migrations") + if err != nil { + return nil, fmt.Errorf("postgres explorer store: load owner migrations: %w", err) + } + return migrations, nil +} + +func MigrateSchema(ctx context.Context, client *pgstore.Client, schema string) (pgstore.MigrateResult, error) { + if client == nil { + return pgstore.MigrateResult{}, errors.New("postgres explorer store: nil client") + } + migrations, err := OwnerMigrations() + if err != nil { + return pgstore.MigrateResult{}, err + } + runner, err := pgstore.NewMigrationRunner(client, schema, migrations) + if err != nil { + return pgstore.MigrateResult{}, err + } + return runner.Migrate(ctx) +} + +func VerifySchema(ctx context.Context, client *pgstore.Client, schema string) error { + if client == nil { + return errors.New("postgres explorer store: nil client") + } + migrations, err := OwnerMigrations() + if err != nil { + return err + } + runner, err := pgstore.NewMigrationRunner(client, schema, migrations) + if err != nil { + return err + } + return runner.Verify(ctx) +} + +var ownerColumns = []string{ + "project", + "explorer_id", + "title", + "management_mode", + "draft_config", + "draft_version", + "draft_digest", + "draft_snapshot_token", + "draft_source_generation", + "draft_authorization_scope_digest", + "previous_draft_revision_id", + "last_authoring_command_id", + "last_authoring_command_digest", + "last_authoring_command_results", + "active_revision_id", + "updated_by", + "updated_at", +} + +type Store struct { + client *pgstore.Client + insert string + get string + list string + receipt receiptStatements + revision revisionStatements + drafts draftSQL + selections selectionSQL + selectionMembers selectionMemberSQL + afterHistoryInsert func(context.Context, pgx.Tx) error + afterMemberInsert func(context.Context, pgx.Tx) error + beforeCleanupCommit func(context.Context, pgx.Tx) error + beforeCompleteCommit func(context.Context, pgx.Tx) error + memberPageStreamed func(rows int) +} + +func New(client *pgstore.Client, schema string) (*Store, error) { + if client == nil { + return nil, errors.New("postgres explorer store: client is required") + } + if !pgstore.ValidSchemaName(schema) { + return nil, fmt.Errorf("postgres explorer store: invalid schema %q: must match [a-z_][a-z0-9_]{0,62}", schema) + } + table := sanitizeTableRef(schema, OwnerTable) + cols := strings.Join(ownerColumns, ", ") + return &Store{ + client: client, + drafts: newDraftSQL(schema), + selections: newSelectionSQL(schema), + selectionMembers: newSelectionMemberSQL(schema), + insert: "INSERT INTO " + table + " (" + cols + ") VALUES (" + ownerPlaceholders() + ") " + + "ON CONFLICT (project, explorer_id) DO NOTHING RETURNING " + cols, + get: "SELECT " + cols + " FROM " + table + " WHERE project = $1 AND explorer_id = $2", + list: "SELECT " + cols + " FROM " + table + " WHERE project = $1 ORDER BY explorer_id", + receipt: newReceiptStatements(schema), + revision: newRevisionStatements(schema), + }, nil +} + +func sanitizeTableRef(schema, table string) string { + return pgx.Identifier{schema, table}.Sanitize() +} + +func orderedPlaceholders(n int) string { + parts := make([]string, n) + for i := range parts { + parts[i] = fmt.Sprintf("$%d", i+1) + } + return strings.Join(parts, ", ") +} + +func ownerPlaceholders() string { + return orderedPlaceholders(len(ownerColumns)) +} + +func ownerArgs(e explorer.Explorer) []any { + return []any{ + e.Project, + e.ExplorerID, + e.Title, + string(e.ManagementMode), + jsonParam(e.DraftConfig), + e.DraftVersion, + e.DraftDigest, + e.DraftSnapshotToken, + e.DraftSourceGeneration, + e.DraftAuthorizationScopeDigest, + e.PreviousDraftRevisionID, + e.LastAuthoringCommandID, + e.LastAuthoringCommandDigest, + jsonParam(e.LastAuthoringCommandResults), + e.ActiveRevisionID, + e.UpdatedBy, + e.UpdatedAt, + } +} + +func jsonParam(raw json.RawMessage) []byte { + if len(raw) == 0 { + return nil + } + return []byte(raw) +} + +type rowScanner interface { + Scan(dest ...any) error +} + +func scanOwner(row rowScanner, e *explorer.Explorer) error { + var managementMode string + targets := []any{ + &e.Project, + &e.ExplorerID, + &e.Title, + &managementMode, + &e.DraftConfig, + &e.DraftVersion, + &e.DraftDigest, + &e.DraftSnapshotToken, + &e.DraftSourceGeneration, + &e.DraftAuthorizationScopeDigest, + &e.PreviousDraftRevisionID, + &e.LastAuthoringCommandID, + &e.LastAuthoringCommandDigest, + &e.LastAuthoringCommandResults, + &e.ActiveRevisionID, + &e.UpdatedBy, + &e.UpdatedAt, + } + if err := row.Scan(targets...); err != nil { + return err + } + e.ManagementMode = explorer.ManagementMode(managementMode) + return nil +} + +func (s *Store) Create(ctx context.Context, e explorer.Explorer) (*explorer.Explorer, error) { + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire create connection: %w", err) + } + defer conn.Release() + + var out explorer.Explorer + err = scanOwner(conn.QueryRow(ctx, s.insert, ownerArgs(e)...), &out) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrDraftConflict + } + return nil, fmt.Errorf("postgres explorer store: create explorer owner %q/%q: %w", + e.Project, e.ExplorerID, err) + } + return &out, nil +} + +func (s *Store) Get(ctx context.Context, project, id string) (*explorer.Explorer, error) { + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire get connection: %w", err) + } + defer conn.Release() + + var out explorer.Explorer + err = scanOwner(conn.QueryRow(ctx, s.get, project, id), &out) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrNotFound + } + return nil, fmt.Errorf("postgres explorer store: get explorer owner %q/%q: %w", project, id, err) + } + return &out, nil +} + +func (s *Store) List(ctx context.Context, project string) ([]explorer.Explorer, error) { + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire list connection: %w", err) + } + defer conn.Release() + + rows, err := conn.Query(ctx, s.list, project) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: list explorer owners %q: %w", project, err) + } + defer rows.Close() + + out := []explorer.Explorer{} + for rows.Next() { + var e explorer.Explorer + if err := scanOwner(rows, &e); err != nil { + return nil, fmt.Errorf("postgres explorer store: scan explorer owner %q: %w", project, err) + } + out = append(out, e) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("postgres explorer store: list explorer owners %q: %w", project, err) + } + return out, nil +} diff --git a/internal/backends/postgres/internal/explorer/explorers_test.go b/internal/backends/postgres/internal/explorer/explorers_test.go new file mode 100644 index 000000000..9bfc092ea --- /dev/null +++ b/internal/backends/postgres/internal/explorer/explorers_test.go @@ -0,0 +1,949 @@ +package postgres + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/explorer" +) + +const cfgAlpha = `{"title": "Alpha", "filters": [{"column": "day", "op": "eq", "value": "2026-01-01"}], "beta": true, "limit": 25, "note": null}` + +const literalCfgAlphaText = `{"beta": true, "note": null, "limit": 25, "title": "Alpha", "filters": [{"op": "eq", "value": "2026-01-01", "column": "day"}]}` + +const cfgBeta = `{"z": 1, "a": {"deep": [true, false, null]}, "m": "keep"}` + +const literalCfgBetaText = `{"a": {"deep": [true, false, null]}, "m": "keep", "z": 1}` + +const resultsOK = `{"matchedRows": 4021, "traceId": "trc_01"}` + +const literalResultsText = `{"traceId": "trc_01", "matchedRows": 4021}` + +var ownerUpdatedAt = time.Date(2026, 1, 2, 3, 4, 5, 123456000, time.UTC) + +const ownerUpdatedAtText = "2026-01-02 03:04:05.123456" + +func fullOwner(project, id string) explorer.Explorer { + return explorer.Explorer{ + Project: project, + ExplorerID: id, + Title: "Alpha Dashboard", + ManagementMode: explorer.ManagementRepository, + DraftConfig: json.RawMessage(cfgAlpha), + DraftVersion: 3, + DraftDigest: "sha256:aaa", + DraftSnapshotToken: "snap-1", + DraftSourceGeneration: "gen_2026_01", + DraftAuthorizationScopeDigest: "authz-scope-1", + PreviousDraftRevisionID: "draftrev_prev", + LastAuthoringCommandID: "cmd-1", + LastAuthoringCommandDigest: "sha256:cmd", + LastAuthoringCommandResults: json.RawMessage(resultsOK), + ActiveRevisionID: "rev_active", + UpdatedBy: "builder@calypr", + UpdatedAt: ownerUpdatedAt, + } +} + +func jsonEqual(t *testing.T, want, got json.RawMessage, field string) { + t.Helper() + if (want == nil) != (got == nil) { + t.Fatalf("%s: missing-ness changed: want %q got %q", field, want, got) + } + if want == nil { + return + } + if string(want) == "null" || string(got) == "null" { + if !(string(want) == "null" && string(got) == "null") { + t.Fatalf("%s: JSON-null lost: want %q got %q", field, want, got) + } + return + } + var wantValue, gotValue any + if err := json.Unmarshal(want, &wantValue); err != nil { + t.Fatalf("%s: want payload invalid: %v", field, err) + } + if err := json.Unmarshal(got, &gotValue); err != nil { + t.Fatalf("%s: got payload invalid (jsonb loss?): %v", field, err) + } + if !reflect.DeepEqual(wantValue, gotValue) { + t.Fatalf("%s: payload not lossless: want %s got %s", field, want, got) + } +} + +func requireOwnerEqual(t *testing.T, want, got explorer.Explorer, where string) { + t.Helper() + if got.Project != want.Project || got.ExplorerID != want.ExplorerID { + t.Fatalf("%s: identity changed: got %q/%q want %q/%q", where, got.Project, got.ExplorerID, want.Project, want.ExplorerID) + } + if got.Title != want.Title || got.ManagementMode != want.ManagementMode { + t.Fatalf("%s: presentation fields changed: got title=%q mode=%q want title=%q mode=%q", + where, got.Title, got.ManagementMode, want.Title, want.ManagementMode) + } + if got.DraftVersion != want.DraftVersion || got.DraftDigest != want.DraftDigest || + got.DraftSnapshotToken != want.DraftSnapshotToken || + got.DraftSourceGeneration != want.DraftSourceGeneration || + got.DraftAuthorizationScopeDigest != want.DraftAuthorizationScopeDigest || + got.PreviousDraftRevisionID != want.PreviousDraftRevisionID || + got.LastAuthoringCommandID != want.LastAuthoringCommandID || + got.LastAuthoringCommandDigest != want.LastAuthoringCommandDigest || + got.ActiveRevisionID != want.ActiveRevisionID || + got.UpdatedBy != want.UpdatedBy { + t.Fatalf("%s: typed draft/authoring fields changed: got %+v want %+v", where, got, want) + } + jsonEqual(t, want.DraftConfig, got.DraftConfig, where+" draftConfig") + jsonEqual(t, want.LastAuthoringCommandResults, got.LastAuthoringCommandResults, where+" lastAuthoringCommandResults") + if !got.UpdatedAt.Equal(want.UpdatedAt) { + t.Fatalf("%s: updatedAt instant changed: got %s want %s", where, got.UpdatedAt, want.UpdatedAt) + } +} + +var pgstate01v1NameRe = regexp.MustCompile(`^pgstate01v1_it_[0-9]+_[0-9a-f]+$`) + +func testPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 Explorer-owner integration tests") + } + return raw +} + +func randomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +type ownerFixture struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dbName string + schema string + dsn string + scrub func(any) string + fail func(step string, err error) +} + +func newOwnerFixture(t *testing.T) *ownerFixture { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgstate01v1_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstate01v1NameRe.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + + t.Cleanup(func() { + var mine int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname = $1`, dbName).Scan(&mine); err != nil { + t.Errorf("leftover audit: query database catalog: %s", scrub(err)) + } + if mine != 0 { + t.Errorf("leftover audit: isolated database %s still exists", dbName) + } + var namespaceLeftovers int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname LIKE 'pgstate01v1_it_%'`).Scan(&namespaceLeftovers); err != nil { + t.Errorf("leftover audit: query database namespace: %s", scrub(err)) + } + if namespaceLeftovers != 0 { + t.Errorf("leftover audit: %d pgstate01v1_it_* databases remain", namespaceLeftovers) + } + var schemaLeftovers int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgstate01v1_it_%'`).Scan(&schemaLeftovers); err != nil { + t.Errorf("leftover audit: query schema namespace: %s", scrub(err)) + } + if schemaLeftovers != 0 { + t.Errorf("leftover audit: %d pgstate01v1_it_* schemas remain in the service database", schemaLeftovers) + } + }) + + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + if probe, err := url.Parse(dsn); err != nil || + probe.Query().Get("dbname") != "" || probe.Path != "/"+dbName { + t.Fatal("isolated test DSN does not provably target the isolated database") + } + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("pgstate01v1_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstate01v1NameRe.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + return &ownerFixture{ctx: ctx, admin: admin, iso: iso, dbName: dbName, schema: schema, dsn: dsn, scrub: scrub, fail: fail} +} + +func (f *ownerFixture) ownerTable() string { + return pgx.Identifier{f.schema, OwnerTable}.Sanitize() +} + +func (f *ownerFixture) newStore(t *testing.T) *Store { + t.Helper() + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + f.fail("open pg-store pool", err) + } + t.Cleanup(client.Close) + store, err := New(client, f.schema) + if err != nil { + f.fail("build Explorer owner store", err) + } + return store +} + +func (f *ownerFixture) migrate(t *testing.T) pgstore.MigrateResult { + t.Helper() + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, + MaxConns: 2, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + f.fail("open migration pool", err) + } + defer client.Close() + result, err := MigrateSchema(f.ctx, client, f.schema) + if err != nil { + f.fail("migrate owner schema", err) + } + return result +} + +type literalRow struct { + Title string + ManagementMode string + DraftVersion int64 + DraftDigest string + DraftSnapshotToken string + DraftSourceGeneration string + DraftAuthzScopeDigest string + PreviousDraftRevisionID string + LastAuthoringCommandID string + LastAuthoringCommandDigest string + ActiveRevisionID string + UpdatedBy string + UpdatedAtText string + DraftConfigText string + DraftConfigKind string + ResultsText string + ResultsKind string +} + +func (f *ownerFixture) literalRow(t *testing.T, project, id string) literalRow { + t.Helper() + var row literalRow + err := f.iso.QueryRow(f.ctx, `SELECT + title, management_mode, draft_version, draft_digest, draft_snapshot_token, + draft_source_generation, draft_authorization_scope_digest, + previous_draft_revision_id, last_authoring_command_id, last_authoring_command_digest, + active_revision_id, updated_by, + to_char(updated_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), + COALESCE(draft_config::text, ''), COALESCE(jsonb_typeof(draft_config), 'missing'), + COALESCE(last_authoring_command_results::text, ''), COALESCE(jsonb_typeof(last_authoring_command_results), 'missing') + FROM `+f.ownerTable()+` WHERE project = $1 AND explorer_id = $2`, project, id).Scan( + &row.Title, &row.ManagementMode, &row.DraftVersion, &row.DraftDigest, + &row.DraftSnapshotToken, &row.DraftSourceGeneration, &row.DraftAuthzScopeDigest, + &row.PreviousDraftRevisionID, &row.LastAuthoringCommandID, &row.LastAuthoringCommandDigest, + &row.ActiveRevisionID, &row.UpdatedBy, &row.UpdatedAtText, + &row.DraftConfigText, &row.DraftConfigKind, + &row.ResultsText, &row.ResultsKind) + if err != nil { + f.fail("read literal owner row", err) + } + return row +} + +func (f *ownerFixture) ownerCount(t *testing.T, project string) int { + t.Helper() + var n int + if err := f.iso.QueryRow(f.ctx, + `SELECT count(*) FROM `+f.ownerTable()+` WHERE project = $1`, project).Scan(&n); err != nil { + f.fail("count owner rows", err) + } + return n +} + +func (f *ownerFixture) requireLiteralFullOwner(t *testing.T, project, id string) { + t.Helper() + row := f.literalRow(t, project, id) + want := literalRow{ + Title: "Alpha Dashboard", + ManagementMode: "REPOSITORY", + DraftVersion: 3, + DraftDigest: "sha256:aaa", + DraftSnapshotToken: "snap-1", + DraftSourceGeneration: "gen_2026_01", + DraftAuthzScopeDigest: "authz-scope-1", + PreviousDraftRevisionID: "draftrev_prev", + LastAuthoringCommandID: "cmd-1", + LastAuthoringCommandDigest: "sha256:cmd", + ActiveRevisionID: "rev_active", + UpdatedBy: "builder@calypr", + UpdatedAtText: ownerUpdatedAtText, + DraftConfigText: literalCfgAlphaText, + DraftConfigKind: "object", + ResultsText: literalResultsText, + ResultsKind: "object", + } + if row != want { + t.Fatalf("literal owner row mismatch:\n got %+v\nwant %+v", row, want) + } +} + +func requirePgErr(t *testing.T, err error, sqlstate, step string) { + t.Helper() + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + t.Fatalf("%s: error = %v (%T), want *pgconn.PgError", step, err, err) + } + if pgErr.Code != sqlstate { + t.Fatalf("%s: sqlstate = %s, want %s (error: %v)", step, pgErr.Code, sqlstate, err) + } +} + +func TestPostgresExplorerOwnerMigrationHistoryLoads(t *testing.T) { + migrations, err := OwnerMigrations() + if err != nil { + t.Fatalf("OwnerMigrations() = %v", err) + } + embeddedHistoryFiles, _ := pgstate02EnumerateEmbeddedMigrations(t) + if len(migrations) != len(embeddedHistoryFiles) { + t.Fatalf("migration count = %d, want %d (one per file in the embedded migrations directory)", len(migrations), len(embeddedHistoryFiles)) + } + want := []struct { + version int64 + name string + checksum string + }{ + {1, "ledger", "46a99180c6c6c7d41ca13520e020b71827725ec8059cce0142e278fb7577f086"}, + {2, "explorer_owner", "fc85316901b79234695695f972d8abc17d29a726293b0ff6584c96aed1cc7056"}, + {3, "explorer_draft_revision", "4bf4fe35485cb7adc6ea46812b0ae004e89ff935a14f14dd80a300bb3cc86c3f"}, + {4, "capability_artifacts", "bec3e49f897b28ef71d7d69a5f7c71bb11d13b7c631ee26724de36a89d639222"}, + {5, "explorer_compilation_receipt", "f32b3f9c821b5451e64b3d2972fb49e61f4e0c5bfcf88c69bc5b6aa7298fc33a"}, + {6, "explorer_revision", "3f6c44bff72d7837634e8c170f0b97e256e38d35f7c844fad27f4fbda38ecc09"}, + {7, "interpretation_library", "5e308b21a3f567b3066c46458606dd361a9c90646db54ee71618902a544a9ba2"}, + {8, "selection_revisions", "562a61ef33786cf674f1c87552e2dc8e70691b34116b7f415d5f81960ed8d65d"}, + {9, "explorer_explicit_groups", "c635cb5ef7c3c04655b4f45b01ba0fe834a7f181f70638f300878caf16c9a14f"}, + {10, "explorer_explicit_group_memberships", "02e0c9ab8bf9864e7d9c82471eacc7406fe2301b025e459784044c4a44489ccd"}, + {11, "explorer_explicit_group_definition_guard_hardening", "fb45d8f8c312f8f6adb119465da5ed89ac0ddd14929a2c26aa4c917db6d704ec"}, + {12, "explorer_revision_activation_revival", "08a61bf6ca66a400112f51463f2c5e1c0a7f4cfbb2d707c0063f87829b1acfe9"}, + } + if len(want) != len(migrations) { + t.Fatalf("checksum registry holds %d entries, want %d (one per embedded migration file): add the new migration's name and sha256 pin deliberately", len(want), len(migrations)) + } + for i, m := range migrations { + if m.Version != want[i].version || m.Name != want[i].name { + t.Fatalf("migration %d = %d/%s, want %d/%s", + i, m.Version, m.Name, want[i].version, want[i].name) + } + if m.Checksum != want[i].checksum { + t.Fatalf("migration %d/%s checksum = %s, want pinned %s: migration SQL drifted", + m.Version, m.Name, m.Checksum, want[i].checksum) + } + upperSQL := strings.ToUpper(m.SQL) + if !strings.Contains(upperSQL, "CREATE TABLE IF NOT EXISTS") && + !strings.Contains(upperSQL, "CREATE OR REPLACE FUNCTION") { + t.Fatalf("migration %d/%s must be rerun-safe (CREATE TABLE IF NOT EXISTS, or CREATE OR REPLACE FUNCTION for guard hardening); it never truncates", m.Version, m.Name) + } + } + sqlFor := func(name string) string { + for _, m := range migrations { + if m.Name == name { + return m.SQL + } + } + t.Fatalf("migration %s missing", name) + return "" + } + if !strings.Contains(sqlFor("ledger"), "schema_migrations") { + t.Fatal("ledger bootstrap must create the pg-store-02 ledger table") + } + if !strings.Contains(sqlFor("explorer_owner"), `PRIMARY KEY (project, explorer_id)`) { + t.Fatal("owner table must be unique on (project, explorer_id) identity") + } + if !strings.Contains(sqlFor("explorer_compilation_receipt"), `PRIMARY KEY (project, explorer_id, id)`) || + !strings.Contains(sqlFor("explorer_compilation_receipt"), "compilation_receipts_frozen") { + t.Fatal("receipt table must be unique on the receipt identity and ship its frozen trigger") + } + if !strings.Contains(sqlFor("explorer_revision"), "explorer_revisions_guard") { + t.Fatal("revision table must ship its frozen-column, insert-once DELETE, and monotone-transition trigger") + } +} + +func TestPostgresExplorerOwnerStoreValidationAndSQLContract(t *testing.T) { + if _, err := New(nil, "loom"); err == nil { + t.Fatal("New accepted a nil client") + } + for _, bad := range []string{"", " ", "Loom", "1loom", "loom;drop", `loom"x`, "loom x", strings.Repeat("a", 64)} { + if _, err := New(&pgstore.Client{}, bad); err == nil { + t.Errorf("New accepted invalid schema %q", bad) + } else if !strings.Contains(err.Error(), "invalid schema") { + t.Errorf("New(%q) error = %v, want invalid-schema error", bad, err) + } + } + store, err := New(&pgstore.Client{}, "loom_control") + if err != nil { + t.Fatalf("New with a valid schema = %v", err) + } + if !strings.Contains(store.insert, `ON CONFLICT (project, explorer_id) DO NOTHING`) { + t.Error("Create statement must be insert-once: ON CONFLICT (project, explorer_id) DO NOTHING") + } + if !strings.Contains(store.insert, "RETURNING") { + t.Error("Create statement must RETURNING the stored row") + } + if !strings.Contains(store.insert, `"loom_control"."`+OwnerTable+`"`) { + t.Error("Create statement must bind the schema-qualified, sanitized table") + } + if !strings.Contains(store.get, `WHERE project = $1 AND explorer_id = $2`) { + t.Error("Get statement must scope by project AND explorer_id") + } + if !strings.Contains(store.list, `WHERE project = $1 ORDER BY explorer_id`) { + t.Error("List statement must be project-scoped and sorted by explorer_id") + } + for i := 1; i <= len(ownerColumns); i++ { + if !strings.Contains(store.insert, fmt.Sprintf("$%d", i)) { + t.Fatalf("Create statement is missing ordered parameter $%d", i) + } + } + if strings.Contains(store.insert, fmt.Sprintf("$%d", len(ownerColumns)+1)) { + t.Fatalf("Create statement carries more than %d parameters", len(ownerColumns)) + } + if got := len(ownerArgs(fullOwner("p", "e"))); got != len(ownerColumns) { + t.Fatalf("ownerArgs produced %d values, want %d (one per column)", got, len(ownerColumns)) + } +} + +func TestPostgresExplorerOwnerJSONBoundary(t *testing.T) { + if b := jsonParam(nil); b != nil { + t.Errorf("jsonParam(nil) = %q, want SQL NULL", b) + } + if b := jsonParam(json.RawMessage{}); b != nil { + t.Errorf("jsonParam(empty) = %q, want SQL NULL (missing)", b) + } + if b := jsonParam(json.RawMessage("null")); string(b) != "null" { + t.Errorf("jsonParam(null literal) = %q, want to pass through as jsonb 'null'", b) + } +} + +func TestPostgresExplorerOwnerCreateGetRoundTripLiteralRows(t *testing.T) { + f := newOwnerFixture(t) + f.migrate(t) + + writer := f.newStore(t) + want := fullOwner("proj_alpha", "exp_1") + created, err := writer.Create(f.ctx, want) + if err != nil { + f.fail("create owner", err) + } + requireOwnerEqual(t, want, *created, "Create result") + + f.requireLiteralFullOwner(t, "proj_alpha", "exp_1") + if n := f.ownerCount(t, "proj_alpha"); n != 1 { + t.Fatalf("owner row count = %d, want exactly 1", n) + } + + reader := f.newStore(t) + got, err := reader.Get(f.ctx, "proj_alpha", "exp_1") + if err != nil { + f.fail("get owner on fresh pool", err) + } + requireOwnerEqual(t, want, *got, "fresh-pool Get") + + listed, err := reader.List(f.ctx, "proj_alpha") + if err != nil { + f.fail("list owners on fresh pool", err) + } + if len(listed) != 1 { + t.Fatalf("List returned %d owners, want 1", len(listed)) + } + requireOwnerEqual(t, want, listed[0], "fresh-pool List") +} + +func TestPostgresExplorerOwnerCreateDuplicateInsertOnce(t *testing.T) { + f := newOwnerFixture(t) + f.migrate(t) + + writer := f.newStore(t) + first := fullOwner("proj_dup", "exp_same") + if _, err := writer.Create(f.ctx, first); err != nil { + f.fail("create first owner", err) + } + + second := fullOwner("proj_dup", "exp_same") + second.Title = "Overwrite Attempt" + second.ManagementMode = explorer.ManagementInteractive + second.DraftConfig = json.RawMessage(cfgBeta) + second.DraftVersion = 99 + second.DraftDigest = "sha256:evil" + second.LastAuthoringCommandResults = nil + conflict, err := writer.Create(f.ctx, second) + if !errors.Is(err, explorer.ErrDraftConflict) { + t.Fatalf("duplicate Create error = %v, want explorer.ErrDraftConflict", err) + } + if conflict != nil { + t.Fatalf("duplicate Create returned %+v, want nil", conflict) + } + + reader := f.newStore(t) + got, err := reader.Get(f.ctx, "proj_dup", "exp_same") + if err != nil { + f.fail("get owner after duplicate", err) + } + requireOwnerEqual(t, first, *got, "owner after duplicate Create") + f.requireLiteralFullOwner(t, "proj_dup", "exp_same") + + other := fullOwner("proj_other", "exp_same") + if _, err := writer.Create(f.ctx, other); err != nil { + f.fail("create same id under another project", err) + } + if n := f.ownerCount(t, "proj_dup"); n != 1 { + t.Fatalf("proj_dup owner count = %d, want 1", n) + } + if n := f.ownerCount(t, "proj_other"); n != 1 { + t.Fatalf("proj_other owner count = %d, want 1", n) + } + + cols := strings.Join(ownerColumns, ", ") + placeholders := ownerPlaceholders() + _, err = f.iso.Exec(f.ctx, `INSERT INTO `+f.ownerTable()+` (`+cols+`) VALUES (`+placeholders+`)`, + ownerArgs(first)...) + requirePgErr(t, err, "23505", "raw duplicate insert") +} + +func TestPostgresExplorerOwnerGetAndListProjectScope(t *testing.T) { + f := newOwnerFixture(t) + f.migrate(t) + + writer := f.newStore(t) + ids := []string{"default", "Zeta", "alpha", "_x", "beta"} + for _, project := range []string{"proj_scope_a", "proj_scope_b"} { + for _, id := range ids { + if _, err := writer.Create(f.ctx, fullOwner(project, id)); err != nil { + f.fail("create "+project+"/"+id, err) + } + } + } + if _, err := writer.Create(f.ctx, fullOwner("proj_scope_b", "only_b")); err != nil { + f.fail("create only-b owner", err) + } + + reader := f.newStore(t) + + if _, err := reader.Get(f.ctx, "proj_scope_a", "only_b"); !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("cross-project Get error = %v, want explorer.ErrNotFound", err) + } + if _, err := reader.Get(f.ctx, "proj_scope_a", "missing"); !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("missing Get error = %v, want explorer.ErrNotFound", err) + } + if _, err := reader.Get(f.ctx, "proj_scope_b", "only_b"); err != nil { + f.fail("get only-b owner from owning project", err) + } + + listed, err := reader.List(f.ctx, "proj_scope_a") + if err != nil { + f.fail("list proj_scope_a", err) + } + wantOrder := []string{"Zeta", "_x", "alpha", "beta", "default"} + if len(listed) != len(wantOrder) { + t.Fatalf("List returned %d owners, want %d (project-scoped)", len(listed), len(wantOrder)) + } + for i, want := range wantOrder { + if listed[i].ExplorerID != want { + gotIDs := make([]string, len(listed)) + for j, e := range listed { + gotIDs[j] = e.ExplorerID + } + t.Fatalf("List order = %v, want byte order %v", gotIDs, wantOrder) + } + if listed[i].Project != "proj_scope_a" { + t.Fatalf("List leaked project %q", listed[i].Project) + } + } + + empty, err := reader.List(f.ctx, "proj_never_created") + if err != nil { + f.fail("list unknown project", err) + } + if empty == nil || len(empty) != 0 { + t.Fatalf("unknown-project List = %#v, want empty non-nil slice", empty) + } +} + +func TestPostgresExplorerOwnerNullSemantics(t *testing.T) { + f := newOwnerFixture(t) + f.migrate(t) + + writer := f.newStore(t) + + missing := fullOwner("proj_null", "exp_missing") + missing.DraftConfig = nil + missing.LastAuthoringCommandResults = nil + if _, err := writer.Create(f.ctx, missing); err != nil { + f.fail("create missing-payload owner", err) + } + + jsonNull := fullOwner("proj_null", "exp_jsonnull") + jsonNull.DraftConfig = json.RawMessage("null") + jsonNull.LastAuthoringCommandResults = json.RawMessage("null") + if _, err := writer.Create(f.ctx, jsonNull); err != nil { + f.fail("create json-null-payload owner", err) + } + + doc := fullOwner("proj_null", "exp_doc") + if _, err := writer.Create(f.ctx, doc); err != nil { + f.fail("create document-payload owner", err) + } + + reader := f.newStore(t) + + missingRow := f.literalRow(t, "proj_null", "exp_missing") + if missingRow.DraftConfigKind != "missing" || missingRow.DraftConfigText != "" { + t.Fatalf("missing draft_config physical form = %q/%q, want SQL NULL", missingRow.DraftConfigKind, missingRow.DraftConfigText) + } + if missingRow.ResultsKind != "missing" || missingRow.ResultsText != "" { + t.Fatalf("missing results physical form = %q/%q, want SQL NULL", missingRow.ResultsKind, missingRow.ResultsText) + } + jsonNullRow := f.literalRow(t, "proj_null", "exp_jsonnull") + if jsonNullRow.DraftConfigKind != "null" || jsonNullRow.DraftConfigText != "null" { + t.Fatalf("json-null draft_config physical form = %q/%q, want jsonb 'null' (not SQL NULL)", + jsonNullRow.DraftConfigKind, jsonNullRow.DraftConfigText) + } + if jsonNullRow.ResultsKind != "null" || jsonNullRow.ResultsText != "null" { + t.Fatalf("json-null results physical form = %q/%q, want jsonb 'null'", jsonNullRow.ResultsKind, jsonNullRow.ResultsText) + } + docRow := f.literalRow(t, "proj_null", "exp_doc") + if docRow.DraftConfigKind != "object" || docRow.DraftConfigText != literalCfgAlphaText { + t.Fatalf("document draft_config literal = %q/%q, want %q", docRow.DraftConfigKind, docRow.DraftConfigText, literalCfgAlphaText) + } + + gotMissing, err := reader.Get(f.ctx, "proj_null", "exp_missing") + if err != nil { + f.fail("get missing-payload owner", err) + } + if gotMissing.DraftConfig != nil || gotMissing.LastAuthoringCommandResults != nil { + t.Fatalf("missing payloads did not read back nil: %q / %q", gotMissing.DraftConfig, gotMissing.LastAuthoringCommandResults) + } + gotJSONNull, err := reader.Get(f.ctx, "proj_null", "exp_jsonnull") + if err != nil { + f.fail("get json-null-payload owner", err) + } + if string(gotJSONNull.DraftConfig) != "null" || gotJSONNull.DraftConfig == nil { + t.Fatalf("json-null draft_config read back as %q, want RawMessage(\"null\")", gotJSONNull.DraftConfig) + } + if string(gotJSONNull.LastAuthoringCommandResults) != "null" { + t.Fatalf("json-null results read back as %q, want RawMessage(\"null\")", gotJSONNull.LastAuthoringCommandResults) + } + gotDoc, err := reader.Get(f.ctx, "proj_null", "exp_doc") + if err != nil { + f.fail("get document-payload owner", err) + } + requireOwnerEqual(t, doc, *gotDoc, "document-payload read-back") +} + +func TestPostgresExplorerOwnerMigrateRerunPreservesState(t *testing.T) { + f := newOwnerFixture(t) + + first := f.migrate(t) + embeddedFiles, embeddedMax := pgstate02EnumerateEmbeddedMigrations(t) + if first.LatestVersion != embeddedMax || len(first.Applied) != len(embeddedFiles) { + t.Fatalf("first migrate = %+v, want latest %d with %d applied rows (one per embedded migration file; version 4 is the capability ticket's reserved slot)", first, embeddedMax, len(embeddedFiles)) + } + + store := f.newStore(t) + want := fullOwner("proj_rerun", "exp_keep") + if _, err := store.Create(f.ctx, want); err != nil { + f.fail("create owner before rerun", err) + } + + second := f.migrate(t) + third := f.migrate(t) + for _, rerun := range []pgstore.MigrateResult{second, third} { + if rerun.LatestVersion != first.LatestVersion { + t.Fatalf("rerun latest version = %d, want %d", rerun.LatestVersion, first.LatestVersion) + } + if len(rerun.Applied) != len(first.Applied) { + t.Fatalf("rerun recorded %d rows, want %d (no duplicate history)", len(rerun.Applied), len(first.Applied)) + } + for i, rec := range rerun.Applied { + if rec.Version != first.Applied[i].Version || rec.Name != first.Applied[i].Name || + rec.Checksum != first.Applied[i].Checksum { + t.Fatalf("rerun applied[%d] = %+v, want %+v", i, rec, first.Applied[i]) + } + } + } + + migrations, err := OwnerMigrations() + if err != nil { + t.Fatalf("OwnerMigrations() = %v", err) + } + type ledgerRow struct { + Version int64 + Name string + Checksum string + } + var ledger []ledgerRow + rows, err := f.iso.Query(f.ctx, `SELECT version, name, checksum FROM `+ + pgx.Identifier{f.schema, pgstore.LedgerTable}.Sanitize()+` ORDER BY version`) + if err != nil { + f.fail("read ledger", err) + } + for rows.Next() { + var r ledgerRow + if err := rows.Scan(&r.Version, &r.Name, &r.Checksum); err != nil { + rows.Close() + f.fail("scan ledger", err) + } + ledger = append(ledger, r) + } + rows.Close() + if err := rows.Err(); err != nil { + f.fail("read ledger", err) + } + if len(ledger) != len(migrations) { + t.Fatalf("ledger holds %d rows, want %d", len(ledger), len(migrations)) + } + for i, want := range migrations { + if ledger[i] != (ledgerRow{want.Version, want.Name, want.Checksum}) { + t.Fatalf("ledger[%d] = %+v, want %+v", i, ledger[i], want) + } + } + + reader := f.newStore(t) + got, err := reader.Get(f.ctx, "proj_rerun", "exp_keep") + if err != nil { + f.fail("get owner after migrate reruns", err) + } + requireOwnerEqual(t, want, *got, "owner after migrate reruns") + f.requireLiteralFullOwner(t, "proj_rerun", "exp_keep") + + if _, err := MigrateSchema(f.ctx, nil, f.schema); err == nil { + t.Fatal("MigrateSchema accepted a nil client") + } + missingClient, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, MaxConns: 2, MinConns: 0, AcquireTimeout: 10 * time.Second, + }) + if err != nil { + f.fail("open probe pool", err) + } + defer missingClient.Close() + if _, err := MigrateSchema(f.ctx, missingClient, f.schema+"_never_created"); !errors.Is(err, pgstore.ErrInvalidSchemaName) { + t.Fatalf("MigrateSchema on missing schema error = %v, want ErrInvalidSchemaName", err) + } +} + +func TestPostgresExplorerOwnerMixedHistoryIsRefused(t *testing.T) { + f := newOwnerFixture(t) + + f.migrate(t) + store := f.newStore(t) + owner := fullOwner("proj_ownership", "exp_keep") + if _, err := store.Create(f.ctx, owner); err != nil { + f.fail("create owner before mixed-history refusal check", err) + } + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, MaxConns: 2, MinConns: 0, AcquireTimeout: 10 * time.Second, + }) + if err != nil { + f.fail("open source-store pool", err) + } + defer client.Close() + if _, err := pgstore.MigrateSchema(f.ctx, client, f.schema); !errors.Is(err, pgstore.ErrMigrationChecksumMismatch) { + t.Fatalf("source-store history on this package's schema error = %v, want ErrMigrationChecksumMismatch", err) + } + var ledgerVersions []int64 + rows, err := f.iso.Query(f.ctx, `SELECT version FROM `+ + pgx.Identifier{f.schema, pgstore.LedgerTable}.Sanitize()+` ORDER BY version`) + if err != nil { + f.fail("read ledger after refused run", err) + } + for rows.Next() { + var v int64 + if err := rows.Scan(&v); err != nil { + rows.Close() + f.fail("scan ledger", err) + } + ledgerVersions = append(ledgerVersions, v) + } + rows.Close() + if err := rows.Err(); err != nil { + f.fail("read ledger after refused run", err) + } + embeddedFiles, _ := pgstate02EnumerateEmbeddedMigrations(t) + wantLedgerVersions := make([]int64, 0, len(ledgerVersions)) + for _, file := range embeddedFiles { + fileVersion, err := pgstate02MigrationFileVersion(file) + if err != nil { + f.fail("parse embedded migration file name", err) + } + wantLedgerVersions = append(wantLedgerVersions, fileVersion) + } + if len(ledgerVersions) != len(wantLedgerVersions) { + t.Fatalf("ledger versions after refused source-store run = %v, want the embedded history %v (refused before applying)", ledgerVersions, wantLedgerVersions) + } + for i, want := range wantLedgerVersions { + if ledgerVersions[i] != want { + t.Fatalf("ledger versions after refused source-store run = %v, want %v (refused before applying)", ledgerVersions, wantLedgerVersions) + } + } + reader := f.newStore(t) + got, err := reader.Get(f.ctx, "proj_ownership", "exp_keep") + if err != nil { + f.fail("owner must survive the refused mixed-history run", err) + } + requireOwnerEqual(t, owner, *got, "owner after refused mixed-history run") + + pgStoreSchema := f.schema + "_source" + if _, err := f.iso.Exec(f.ctx, `CREATE SCHEMA "`+pgStoreSchema+`"`); err != nil { + f.fail("create source-store schema", err) + } + t.Cleanup(func() { + if _, err := f.iso.Exec(context.WithoutCancel(f.ctx), + `DROP SCHEMA IF EXISTS "`+pgStoreSchema+`" CASCADE`); err != nil { + t.Errorf("drop source-store schema: %s", f.scrub(err)) + } + }) + if _, err := pgstore.MigrateSchema(f.ctx, client, pgStoreSchema); err != nil { + f.fail("apply source-store history to its own schema", err) + } + if _, err := MigrateSchema(f.ctx, client, pgStoreSchema); !errors.Is(err, pgstore.ErrMigrationChecksumMismatch) { + t.Fatalf("this package's history on a source-store schema error = %v, want ErrMigrationChecksumMismatch", err) + } + var ownerTableExists bool + if err := f.iso.QueryRow(f.ctx, + `SELECT to_regclass($1) IS NOT NULL`, + pgx.Identifier{pgStoreSchema, OwnerTable}.Sanitize()).Scan(&ownerTableExists); err != nil { + f.fail("check owner table absence", err) + } + if ownerTableExists { + t.Fatal("refused mixed-history run created the explorers table; the runner must fail before applying") + } +} + +func TestPostgresExplorerOwnerSchemaConstraints(t *testing.T) { + f := newOwnerFixture(t) + f.migrate(t) + + writer := f.newStore(t) + bogusMode := fullOwner("proj_checks", "exp_store_mode") + bogusMode.ManagementMode = explorer.ManagementMode("SOMETHING_ELSE") + _, err := writer.Create(f.ctx, bogusMode) + requirePgErr(t, err, "23514", "store create with unknown management mode") + if errors.Is(err, explorer.ErrDraftConflict) || errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("constraint failure misclassified as owner semantics: %v", err) + } + + cols := strings.Join(ownerColumns, ", ") + placeholders := ownerPlaceholders() + rawInsert := `INSERT INTO ` + f.ownerTable() + ` (` + cols + `) VALUES (` + placeholders + `)` + + bogus := fullOwner("proj_checks", "exp_mode") + bogus.ManagementMode = explorer.ManagementMode("SOMETHING_ELSE") + _, err = f.iso.Exec(f.ctx, rawInsert, ownerArgs(bogus)...) + requirePgErr(t, err, "23514", "unknown management mode") + + negative := fullOwner("proj_checks", "exp_version") + negative.DraftVersion = -1 + _, err = f.iso.Exec(f.ctx, rawInsert, ownerArgs(negative)...) + requirePgErr(t, err, "23514", "negative draft version") + + _, err = f.iso.Exec(f.ctx, rawInsert, + "proj_checks", "exp_json", "t", "REPOSITORY", + "this is not json", int64(1), + "", "", "", "", "", "", "", nil, "", "", ownerUpdatedAt) + requirePgErr(t, err, "22P02", "non-JSON draft payload") + + if n := f.ownerCount(t, "proj_checks"); n != 0 { + t.Fatalf("rejected inserts left %d rows, want 0", n) + } +} diff --git a/internal/backends/postgres/internal/explorer/interpretations.go b/internal/backends/postgres/internal/explorer/interpretations.go new file mode 100644 index 000000000..19b2ab02e --- /dev/null +++ b/internal/backends/postgres/internal/explorer/interpretations.go @@ -0,0 +1,439 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/explorer" +) + +const ( + InterpretationLibrariesTable = "interpretation_libraries" + InterpretationRevisionsTable = "interpretation_revisions" +) + +const ( + interpretationRevisionImmutableTrigger = "interpretation_revisions_immutable" + interpretationLibraryHeadGuardTrigger = "interpretation_libraries_head_guard" + interpretationRevisionGuardFunction = "loom_interpretation_revision_immutable" + interpretationLibraryGuardFunction = "loom_interpretation_library_guard" + + interpretationRevisionPKConstraint = "interpretation_revisions_identity_pk" + interpretationRevisionParentFKConstraint = "interpretation_revisions_parent_same_library_fk" + interpretationLibraryPKConstraint = "interpretation_libraries_identity_pk" + interpretationLibraryHeadFKConstraint = "interpretation_libraries_head_same_library_fk" +) + +var interpretationRevisionColumns = []string{ + "project", + "id", + "library_id", + "parent_revision_id", + "parent_digest", + "content_digest", + "author", + "explanation", + "created_at", + "payload", +} + +var interpretationLibraryColumns = []string{ + "project", + "id", + "head_revision_id", + "head_digest", + "created_at", + "updated_at", +} + +func interpretationRevisionPlaceholders() string { + parts := make([]string, len(interpretationRevisionColumns)) + for i := range interpretationRevisionColumns { + parts[i] = fmt.Sprintf("$%d", i+1) + } + return strings.Join(parts, ", ") +} + +func interpretationLibraryPlaceholders() string { + parts := make([]string, len(interpretationLibraryColumns)) + for i := range interpretationLibraryColumns { + parts[i] = fmt.Sprintf("$%d", i+1) + } + return strings.Join(parts, ", ") +} + +type interpretationSQL struct { + revisionInsert string + revisionGet string + parentDigestLookup string + libraryInsertRoot string + libraryAdvance string + libraryList string +} + +func newInterpretationSQL(schema string) interpretationSQL { + revTable := pgx.Identifier{schema, InterpretationRevisionsTable}.Sanitize() + libTable := pgx.Identifier{schema, InterpretationLibrariesTable}.Sanitize() + revCols := strings.Join(interpretationRevisionColumns, ", ") + libCols := strings.Join(interpretationLibraryColumns, ", ") + return interpretationSQL{ + revisionInsert: "INSERT INTO " + revTable + " (" + revCols + ") VALUES (" + + interpretationRevisionPlaceholders() + ") " + + "ON CONFLICT DO NOTHING RETURNING " + revCols, + revisionGet: "SELECT " + revCols + " FROM " + revTable + + " WHERE project = $1 AND id = $2", + parentDigestLookup: "SELECT content_digest FROM " + revTable + + " WHERE project = $1 AND id = $2 AND library_id = $3", + libraryInsertRoot: "INSERT INTO " + libTable + " (" + libCols + ") VALUES (" + + interpretationLibraryPlaceholders() + ") " + + "ON CONFLICT DO NOTHING RETURNING " + libCols, + libraryAdvance: "UPDATE " + libTable + " SET head_revision_id = $4, head_digest = $5, updated_at = $6" + + " WHERE project = $1 AND id = $2 AND head_revision_id = $3 RETURNING " + libCols, + libraryList: "SELECT " + libCols + " FROM " + libTable + + " WHERE project = $1 ORDER BY id", + } +} + +type InterpretationStore struct { + client *pgstore.Client + sql interpretationSQL + afterRevisionInsert func(context.Context, pgx.Tx) error +} + +var _ explorer.InterpretationRepository = (*InterpretationStore)(nil) + +func NewInterpretationStore(client *pgstore.Client, schema string) (*InterpretationStore, error) { + if client == nil { + return nil, errors.New("postgres explorer store: interpretation client is required") + } + if !pgstore.ValidSchemaName(schema) { + return nil, fmt.Errorf("postgres explorer store: invalid interpretation schema %q: must match [a-z_][a-z0-9_]{0,62}", schema) + } + return &InterpretationStore{client: client, sql: newInterpretationSQL(schema)}, nil +} + +type storedInterpretationRevision struct { + project string + id string + libraryID string + parentRevision *string + parentDigest *string + contentDigest string + author string + explanation string + createdAt time.Time + payload json.RawMessage +} + +func scanInterpretationRevisionRow(row rowScanner, s *storedInterpretationRevision) error { + return row.Scan(&s.project, &s.id, &s.libraryID, &s.parentRevision, &s.parentDigest, + &s.contentDigest, &s.author, &s.explanation, &s.createdAt, &s.payload) +} + +func decodeInterpretationRevision(s *storedInterpretationRevision) (*explorer.InterpretationRevision, error) { + if len(s.payload) == 0 { + return nil, fmt.Errorf("%w: interpretation revision %q has an empty payload", explorer.ErrInterpretationIdentityMismatch, s.id) + } + var revision explorer.InterpretationRevision + if err := json.Unmarshal(s.payload, &revision); err != nil { + return nil, fmt.Errorf("%w: interpretation revision %q payload does not decode: %v", explorer.ErrInterpretationIdentityMismatch, s.id, err) + } + if err := revision.Validate(); err != nil { + return nil, fmt.Errorf("%w: interpretation revision %q failed self-verification: %v", explorer.ErrInterpretationIdentityMismatch, s.id, err) + } + if revision.Project != s.project || s.project == "" { + return nil, fmt.Errorf("%w: interpretation revision %q project mirror disagrees with its payload", explorer.ErrInterpretationIdentityMismatch, s.id) + } + if string(revision.ID) != s.id { + return nil, fmt.Errorf("%w: interpretation revision %q id mirror disagrees with its payload", explorer.ErrInterpretationIdentityMismatch, s.id) + } + if string(revision.LibraryID) != s.libraryID { + return nil, fmt.Errorf("%w: interpretation revision %q library mirror disagrees with its payload", explorer.ErrInterpretationIdentityMismatch, s.id) + } + if string(revision.ContentDigest) != s.contentDigest { + return nil, fmt.Errorf("%w: interpretation revision %q content digest mirror disagrees with its payload", explorer.ErrInterpretationIdentityMismatch, s.id) + } + if revision.Author != s.author || revision.Explanation != s.explanation { + return nil, fmt.Errorf("%w: interpretation revision %q authorship mirrors disagree with its payload", explorer.ErrInterpretationIdentityMismatch, s.id) + } + payloadParent, payloadDigest := "", "" + if revision.ParentRevisionID != nil { + payloadParent = string(*revision.ParentRevisionID) + } + if revision.ParentDigest != nil { + payloadDigest = string(*revision.ParentDigest) + } + mirrorParent, mirrorDigest := "", "" + if s.parentRevision != nil { + mirrorParent = *s.parentRevision + } + if s.parentDigest != nil { + mirrorDigest = *s.parentDigest + } + if payloadParent != mirrorParent || payloadDigest != mirrorDigest { + return nil, fmt.Errorf("%w: interpretation revision %q parent mirrors disagree with its payload", explorer.ErrInterpretationIdentityMismatch, s.id) + } + return &revision, nil +} + +func (s *InterpretationStore) ListInterpretationLibraries(ctx context.Context, rawProject string) ([]explorer.InterpretationLibrary, error) { + project, err := explorer.CanonicalInterpretationProject(rawProject) + if err != nil { + return nil, err + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire list-libraries connection: %w", err) + } + defer conn.Release() + + rows, err := conn.Query(ctx, s.sql.libraryList, project) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: list interpretation libraries %q: %w", project, err) + } + defer rows.Close() + + out := []explorer.InterpretationLibrary{} + for rows.Next() { + var row libRow + if err := scanInterpretationLibraryRow(rows, &row); err != nil { + return nil, fmt.Errorf("postgres explorer store: scan interpretation library %q: %w", project, err) + } + library := explorer.InterpretationLibrary{ + ID: explorer.InterpretationLibraryID(row.id), + Project: row.project, + HeadRevisionID: explorer.InterpretationRevisionID(row.headRevision), + HeadDigest: explorer.InterpretationContentDigest(row.headDigest), + } + if err := validatePersistedInterpretationLibrary(library, project); err != nil { + return nil, fmt.Errorf("postgres explorer store: interpretation library %q failed validation: %w", project, err) + } + out = append(out, library) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("postgres explorer store: list interpretation libraries %q: %w", project, err) + } + return out, nil +} + +func validatePersistedInterpretationLibrary(value explorer.InterpretationLibrary, project string) error { + if value.Project != project { + return fmt.Errorf("persisted interpretation library project %q escaped its scope", value.Project) + } + if _, err := explorer.NewInterpretationLibraryID(string(value.ID)); err != nil { + return err + } + if string(value.HeadRevisionID) == "" { + return fmt.Errorf("interpretation library %q has an empty head", value.ID) + } + if _, err := explorer.NewInterpretationRevisionID(string(value.HeadRevisionID)); err != nil { + return err + } + if _, err := explorer.NewInterpretationContentDigest(string(value.HeadDigest)); err != nil { + return err + } + return nil +} + +func (s *InterpretationStore) GetInterpretationRevision(ctx context.Context, rawProject string, id explorer.InterpretationRevisionID) (*explorer.InterpretationRevision, error) { + project, err := explorer.CanonicalInterpretationProject(rawProject) + if err != nil { + return nil, err + } + if _, err := explorer.NewInterpretationRevisionID(string(id)); err != nil { + return nil, err + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire get-revision connection: %w", err) + } + defer conn.Release() + + var stored storedInterpretationRevision + err = scanInterpretationRevisionRow(conn.QueryRow(ctx, s.sql.revisionGet, project, string(id)), &stored) + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: read interpretation revision %q/%q: %w", project, id, err) + } + revision, err := decodeInterpretationRevision(&stored) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: %w", err) + } + return revision, nil +} + +func (s *InterpretationStore) CreateInterpretationRevision(ctx context.Context, input explorer.InterpretationRevision, expectedParent *explorer.InterpretationRevisionID) (*explorer.InterpretationRevision, error) { + revision, err := explorer.PrepareInterpretationRevision(input) + if err != nil { + return nil, err + } + expected := "" + if expectedParent != nil { + expected = string(*expectedParent) + } + parent := "" + if revision.ParentRevisionID != nil { + parent = string(*revision.ParentRevisionID) + } + if expected != parent { + return nil, explorer.ErrInterpretationParentConflict + } + preparedEnvelope, err := revision.RevisionIdentityEnvelope() + if err != nil { + return nil, fmt.Errorf("postgres explorer store: build prepared identity envelope: %w", err) + } + + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire create-revision connection: %w", err) + } + defer conn.Release() + + tx, err := conn.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: begin create interpretation revision %q/%q: %w", revision.Project, revision.LibraryID, err) + } + defer func() { _ = tx.Rollback(ctx) }() + + if parent != "" { + var persistedDigest string + err := tx.QueryRow(ctx, s.sql.parentDigestLookup, revision.Project, parent, string(revision.LibraryID)).Scan(&persistedDigest) + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrInterpretationParentConflict + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: read parent interpretation revision %q/%q: %w", revision.Project, parent, err) + } + if revision.ParentDigest == nil || string(*revision.ParentDigest) != persistedDigest { + return nil, explorer.ErrInterpretationParentConflict + } + } + + payload, err := json.Marshal(revision) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: encode interpretation revision payload: %w", err) + } + createdAt := revision.CreatedAt + var stored storedInterpretationRevision + insertErr := scanInterpretationRevisionRow(tx.QueryRow(ctx, s.sql.revisionInsert, + revision.Project, string(revision.ID), string(revision.LibraryID), + textParam(parent), digestParam(revision.ParentDigest), string(revision.ContentDigest), + revision.Author, revision.Explanation, createdAt, json.RawMessage(payload), + ), &stored) + if errors.Is(insertErr, pgx.ErrNoRows) { + var existing storedInterpretationRevision + err := scanInterpretationRevisionRow(tx.QueryRow(ctx, s.sql.revisionGet, revision.Project, string(revision.ID)), &existing) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("postgres explorer store: interpretation revision %q vanished during create: %w", revision.ID, err) + } + return nil, fmt.Errorf("postgres explorer store: re-read interpretation revision %q/%q: %w", revision.Project, revision.ID, err) + } + storedRevision, decodeErr := decodeInterpretationRevision(&existing) + if decodeErr != nil { + return nil, errors.Join(explorer.ErrImmutableInterpretation, decodeErr) + } + existingEnvelope, envelopeErr := storedRevision.RevisionIdentityEnvelope() + if envelopeErr != nil { + return nil, errors.Join(explorer.ErrImmutableInterpretation, envelopeErr) + } + if !sameInterpretationEnvelope(preparedEnvelope, existingEnvelope) { + return nil, explorer.ErrImmutableInterpretation + } + return storedRevision, nil + } + if insertErr != nil { + if pgstore.IsForeignKeyConstraintViolation(insertErr, interpretationRevisionParentFKConstraint) { + return nil, errors.Join(explorer.ErrInterpretationParentConflict, insertErr) + } + return nil, fmt.Errorf("postgres explorer store: insert interpretation revision %q/%q: %w", + revision.Project, revision.ID, insertErr) + } + + if s.afterRevisionInsert != nil { + if err := s.afterRevisionInsert(ctx, tx); err != nil { + return nil, fmt.Errorf("postgres explorer store: create revision %q/%q after insert: %w", + revision.Project, revision.LibraryID, err) + } + } + + var headLib libRow + if expected == "" { + err = scanInterpretationLibraryRow(tx.QueryRow(ctx, s.sql.libraryInsertRoot, + revision.Project, string(revision.LibraryID), string(revision.ID), + string(revision.ContentDigest), createdAt, createdAt), &headLib) + } else { + err = scanInterpretationLibraryRow(tx.QueryRow(ctx, s.sql.libraryAdvance, + revision.Project, string(revision.LibraryID), expected, + string(revision.ID), string(revision.ContentDigest), createdAt), &headLib) + } + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrInterpretationParentConflict + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: advance interpretation library head %q/%q: %w", + revision.Project, revision.LibraryID, err) + } + if headLib.headRevision != string(revision.ID) || headLib.headDigest != string(revision.ContentDigest) { + return nil, fmt.Errorf("postgres explorer store: interpretation library %q/%q head does not bind the accepted revision", + revision.Project, revision.LibraryID) + } + + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("postgres explorer store: commit create interpretation revision %q/%q: %w", + revision.Project, revision.ID, err) + } + created, err := decodeInterpretationRevision(&stored) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: created %w", err) + } + return created, nil +} + +func sameInterpretationEnvelope(a, b map[string]any) bool { + ja, err := json.Marshal(a) + if err != nil { + return false + } + jb, err := json.Marshal(b) + if err != nil { + return false + } + return string(ja) == string(jb) +} + +func textParam(s string) any { + if s == "" { + return nil + } + return s +} + +func digestParam(d *explorer.InterpretationContentDigest) any { + if d == nil { + return nil + } + return string(*d) +} + +type libRow struct { + project string + id string + headRevision string + headDigest string + createdAt time.Time + updatedAt time.Time +} + +func scanInterpretationLibraryRow(row rowScanner, l *libRow) error { + return row.Scan(&l.project, &l.id, &l.headRevision, &l.headDigest, &l.createdAt, &l.updatedAt) +} diff --git a/internal/backends/postgres/internal/explorer/interpretations_test.go b/internal/backends/postgres/internal/explorer/interpretations_test.go new file mode 100644 index 000000000..f81b7b01f --- /dev/null +++ b/internal/backends/postgres/internal/explorer/interpretations_test.go @@ -0,0 +1,1767 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/url" + "reflect" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/explorer" + "github.com/calypr/loom/internal/explorer/authoringv2" +) + +const ( + pgstate04FixtureName = "interpretation_library" + pgstate04FixtureChecksum = "5e308b21a3f567b3066c46458606dd361a9c90646db54ee71618902a544a9ba2" + pgstate04ImmutableCode = "45501" + pgstate04UniqueViolation = "23505" + pgstate04ForeignKey = "23503" + pgstate04CheckViolation = "23514" + + interpRaceRounds = 10 +) + +func pgstate04RequireInterpretationMigration(t *testing.T, history []pgstore.Migration) pgstore.Migration { + t.Helper() + var m pgstore.Migration + var found bool + for _, candidate := range history { + if candidate.Version == 7 { + if found { + t.Fatalf("owner history has two version-7 migrations: %+v", history) + } + m, found = candidate, true + } + } + if !found || m.Name != pgstate04FixtureName { + t.Fatalf("owner history version 7 = %+v (found %v), want exactly 7/%s", m, found, pgstate04FixtureName) + } + if m.Checksum != pgstate04FixtureChecksum { + t.Fatalf("interpretation library checksum = %s, want the byte-frozen merge pin %s: the migration drifted", + m.Checksum, pgstate04FixtureChecksum) + } + return m +} + +var pgstate04NameRe = regexp.MustCompile(`^pgstate04_it_[0-9]+_[0-9a-f]+$`) + +var ( + pgstate04OwnedMu sync.Mutex + pgstate04OwnedDBs = map[string]string{} +) + +func pgstate04Register(db, schema string) { + pgstate04OwnedMu.Lock() + defer pgstate04OwnedMu.Unlock() + pgstate04OwnedDBs[db] = schema +} + +type interpretationService struct { + ctx context.Context + admin *pgxpool.Pool + witness *pgstore.Client + schema string + dsn string + pidW int64 + pidA int64 + pidB int64 + scrub func(any) string + fail func(step string, err error) + + clientA *pgstore.Client + clientB *pgstore.Client + + storeA *InterpretationStore + storeB *InterpretationStore + + ledgerAfter []pgstore.LedgerRecord +} + +func newInterpretationService(t *testing.T) *interpretationService { + t.Helper() + raw := testPostgresURL(t) + + u, err := parseDraftURL(raw) + if err != nil { + t.Fatalf("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL: %v", err) + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgstate04_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstate04NameRe.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + pgstate04Register(dbName, "") + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + if probe, err := url.Parse(dsn); err != nil || + probe.Query().Get("dbname") != "" || probe.Path != "/"+dbName { + t.Fatal("isolated test DSN does not provably target the isolated database") + } + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if _, err := admin.Exec(cleanupCtx, + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + newClient := func(label string) *pgstore.Client { + client, err := pgstore.Open(ctx, pgstore.Config{ + DSN: dsn, + MaxConns: 1, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + fail("open client "+label, err) + } + t.Cleanup(client.Close) + return client + } + witness := newClient("witness") + clientA := newClient("writer-a") + clientB := newClient("writer-b") + + schema := fmt.Sprintf("pgstate04_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstate04NameRe.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if err := interpExecOn(ctx, witness, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + pgstate04Register(dbName, schema) + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if err := interpExecOn(cleanupCtx, witness, `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + s := &interpretationService{ + ctx: ctx, admin: admin, witness: witness, schema: schema, dsn: dsn, + clientA: clientA, clientB: clientB, scrub: scrub, fail: fail, + } + + owner, err := OwnerMigrations() + if err != nil { + fail("owner migrations", err) + } + if len(owner) < 2 || owner[0].Version != 1 || owner[0].Name != "ledger" || + owner[1].Version != 2 || owner[1].Name != "explorer_owner" { + t.Fatalf("owner history = %+v, want ledger(1) + explorer_owner(2) at the front", owner) + } + pgstate04RequireInterpretationMigration(t, owner) + history := append([]pgstore.Migration(nil), owner...) + s.migrateWith(t, history) + + want := []pgstore.LedgerRecord{} + for _, m := range history { + want = append(want, pgstore.LedgerRecord{Version: m.Version, Name: m.Name, Checksum: m.Checksum}) + } + if len(s.ledgerAfter) != len(want) { + t.Fatalf("applied history = %+v, want %+v", s.ledgerAfter, want) + } + for i, w := range want { + got := s.ledgerAfter[i] + if got.Version != w.Version || got.Name != w.Name || got.Checksum != w.Checksum { + t.Fatalf("applied[%d] = %+v, want %+v", i, got, w) + } + } + + s.pidW = s.backendPID(t, witness, "witness") + s.pidA = s.backendPID(t, clientA, "writer A") + s.pidB = s.backendPID(t, clientB, "writer B") + if s.pidA == s.pidB || s.pidA == s.pidW || s.pidB == s.pidW { + t.Fatalf("writer and witness backends are not distinct physical connections: A=%d B=%d W=%d", s.pidA, s.pidB, s.pidW) + } + + storeA, err := NewInterpretationStore(clientA, schema) + if err != nil { + fail("build interpretation store A", err) + } + s.storeA = storeA + storeB, err := NewInterpretationStore(clientB, schema) + if err != nil { + fail("build interpretation store B", err) + } + s.storeB = storeB + return s +} + +func (s *interpretationService) migrateWith(t *testing.T, migrations []pgstore.Migration) { + t.Helper() + runner, err := pgstore.NewMigrationRunner(s.witness, s.schema, migrations) + if err != nil { + s.fail("new migration runner", err) + } + result, err := runner.Migrate(s.ctx) + if err != nil { + s.fail("migrate interpretation history", err) + } + s.ledgerAfter = result.Applied +} + +func (s *interpretationService) backendPID(t *testing.T, client *pgstore.Client, label string) int64 { + t.Helper() + conn, err := client.Acquire(s.ctx) + if err != nil { + s.fail("acquire "+label+" connection", err) + } + defer conn.Release() + var pid int64 + if err := conn.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + s.fail("read pg_backend_pid for "+label, err) + } + if pid <= 0 { + t.Fatalf("pg_backend_pid() for %s returned %d, want a live backend", label, pid) + } + return pid +} + +func (s *interpretationService) requireRoundBackendsDistinct(t *testing.T, where string, round int) { + t.Helper() + a := s.backendPID(t, s.clientA, "writer A") + b := s.backendPID(t, s.clientB, "writer B") + if a != s.pidA || b != s.pidB { + t.Fatalf("%s round %d: writer backend moved (A %d->%d, B %d->%d): race would not be on the witnessed connections", + where, round, s.pidA, a, s.pidB, b) + } + if a == b || a == s.pidW || b == s.pidW { + t.Fatalf("%s round %d: backends not distinct (A=%d B=%d W=%d)", where, round, a, b, s.pidW) + } +} + +func interpExecOn(ctx context.Context, client *pgstore.Client, sql string, args ...any) error { + conn, err := client.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + _, err = conn.Exec(ctx, sql, args...) + return err +} + +func (s *interpretationService) qualified(table string) string { + return pgx.Identifier{s.schema, table}.Sanitize() +} + +func (s *interpretationService) mustWitnessExec(t *testing.T, sql string, args ...any) { + t.Helper() + if err := interpExecOn(s.ctx, s.witness, sql, args...); err != nil { + s.fail("witness exec", err) + } +} + +func (s *interpretationService) witnessExecErr(sql string, args ...any) error { + return interpExecOn(s.ctx, s.witness, sql, args...) +} + +type interpWitnessRevision struct { + project string + id string + libraryID string + parentRevision string + parentDigest string + parentIsNull bool + contentDigest string + author string + explanation string + createdAtText string + payloadText string + payloadKind string + ctid string +} + +func (s *interpretationService) witnessRevision(t *testing.T, project, id string) (interpWitnessRevision, bool) { + t.Helper() + var w interpWitnessRevision + var parentRev, parentDigest *string + err := interpQueryRow(s.ctx, s.witness, + `SELECT project, id, library_id, parent_revision_id, parent_digest, + parent_revision_id IS NULL, + content_digest, author, explanation, + to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), + COALESCE(payload::text, ''), COALESCE(jsonb_typeof(payload), 'missing'), + ctid::text + FROM `+s.qualified(InterpretationRevisionsTable)+` WHERE project = $1 AND id = $2`, + []any{&w.project, &w.id, &w.libraryID, &parentRev, &parentDigest, &w.parentIsNull, + &w.contentDigest, &w.author, &w.explanation, + &w.createdAtText, &w.payloadText, &w.payloadKind, &w.ctid}, + project, id) + if errors.Is(err, pgx.ErrNoRows) { + return interpWitnessRevision{}, false + } + if err != nil { + s.fail("witness revision", err) + } + if parentRev != nil { + w.parentRevision = *parentRev + } + if parentDigest != nil { + w.parentDigest = *parentDigest + } + return w, true +} + +func (s *interpretationService) mustWitnessRevision(t *testing.T, project, id string) interpWitnessRevision { + t.Helper() + w, ok := s.witnessRevision(t, project, id) + if !ok { + t.Fatalf("revision row %s/%s missing from the database", project, id) + } + return w +} + +func (s *interpretationService) revisionCount(t *testing.T, project, library string) int { + t.Helper() + var n int + err := interpQueryRow(s.ctx, s.witness, + `SELECT count(*) FROM `+s.qualified(InterpretationRevisionsTable)+ + ` WHERE project = $1 AND library_id = $2`, []any{&n}, project, library) + if err != nil { + s.fail("count revisions", err) + } + return n +} + +type interpWitnessLibrary struct { + project string + id string + headRevision string + headDigest string + createdAt string + updatedAt string + ctid string +} + +func (s *interpretationService) witnessLibrary(t *testing.T, project, id string) (interpWitnessLibrary, bool) { + t.Helper() + var w interpWitnessLibrary + err := interpQueryRow(s.ctx, s.witness, + `SELECT project, id, head_revision_id, head_digest, + to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), + to_char(updated_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), + ctid::text + FROM `+s.qualified(InterpretationLibrariesTable)+` WHERE project = $1 AND id = $2`, + []any{&w.project, &w.id, &w.headRevision, &w.headDigest, &w.createdAt, &w.updatedAt, &w.ctid}, + project, id) + if errors.Is(err, pgx.ErrNoRows) { + return interpWitnessLibrary{}, false + } + if err != nil { + s.fail("witness library", err) + } + return w, true +} + +func (s *interpretationService) mustWitnessLibrary(t *testing.T, project, id string) interpWitnessLibrary { + t.Helper() + w, ok := s.witnessLibrary(t, project, id) + if !ok { + t.Fatalf("library row %s/%s missing from the database", project, id) + } + return w +} + +func (s *interpretationService) libraryCount(t *testing.T, project, id string) int { + t.Helper() + var n int + err := interpQueryRow(s.ctx, s.witness, + `SELECT count(*) FROM `+s.qualified(InterpretationLibrariesTable)+ + ` WHERE project = $1 AND id = $2`, []any{&n}, project, id) + if err != nil { + s.fail("count libraries", err) + } + return n +} + +func interpQueryRow(ctx context.Context, client *pgstore.Client, sql string, dest []any, args ...any) error { + conn, err := client.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + return conn.QueryRow(ctx, sql, args...).Scan(dest...) +} + +var interpBaseTime = time.Date(2026, 3, 4, 5, 6, 7, 123456000, time.UTC) + +const interpBaseTimeText = "2026-03-04 05:06:07.123456" + +func interpRevision(project, library, author, explanation, rulePath string) explorer.InterpretationRevision { + return explorer.InterpretationRevision{ + Project: project, + LibraryID: explorer.InterpretationLibraryID(library), + Author: author, + Explanation: explanation, + CreatedAt: interpBaseTime, + Rules: []explorer.InterpretationRule{{ + ID: explorer.InterpretationRuleID("rule-" + rulePath), + Match: explorer.InterpretationStructuralMatch{ResourceType: "Observation"}, + Definition: explorer.InterpretationFeatureDefinition{ + Source: authoringv2.ColumnSource{ + Kind: authoringv2.SourceField, + Field: &authoringv2.FieldSource{Path: rulePath, ProjectionMode: "VALUE"}, + }, + }, + }}, + } +} + +func interpChild(parent explorer.InterpretationRevision, explanation, rulePath string) explorer.InterpretationRevision { + child := interpRevision(parent.Project, string(parent.LibraryID), parent.Author, explanation, rulePath) + parentID := parent.ID + parentDigest := parent.ContentDigest + child.ParentRevisionID = &parentID + child.ParentDigest = &parentDigest + return child +} + +func createRevision(t *testing.T, s *interpretationService, store *InterpretationStore, + input explorer.InterpretationRevision, expected *explorer.InterpretationRevisionID) *explorer.InterpretationRevision { + t.Helper() + created, err := store.CreateInterpretationRevision(s.ctx, input, expected) + if err != nil { + s.fail(fmt.Sprintf("create interpretation revision %q/%q", input.Project, input.LibraryID), err) + } + return created +} + +func idRef(id explorer.InterpretationRevisionID) *explorer.InterpretationRevisionID { + return &id +} + +func TestInterpretationMigrationFixtureComposition(t *testing.T) { + owner, err := OwnerMigrations() + if err != nil { + t.Fatalf("OwnerMigrations() = %v", err) + } + m := pgstate04RequireInterpretationMigration(t, owner) + history := append([]pgstore.Migration(nil), owner...) + for i := 1; i < len(history); i++ { + if history[i-1].Version >= history[i].Version { + t.Fatalf("composed history not strictly ascending: %+v", history) + } + } + + statements := make([]string, 0) + for _, line := range strings.Split(m.SQL, "\n") { + if i := strings.Index(line, "--"); i >= 0 { + line = line[:i] + } + statements = append(statements, line) + } + upper := strings.ToUpper(strings.Join(statements, "\n")) + if !strings.Contains(upper, "CREATE TABLE IF NOT EXISTS") { + t.Fatal("fixture must be rerun-safe: CREATE TABLE IF NOT EXISTS") + } + if strings.Contains(upper, "DROP TABLE") || strings.Contains(upper, "TRUNCATE") || strings.Contains(upper, "DROP TRIGGER ") && !strings.Contains(upper, "DROP TRIGGER IF EXISTS") { + t.Fatal("fixture may never drop or truncate state: the runner may retry a committed migration") + } + for _, required := range []string{ + InterpretationRevisionsTable, InterpretationLibrariesTable, + interpretationRevisionPKConstraint, interpretationRevisionParentFKConstraint, + interpretationLibraryPKConstraint, interpretationLibraryHeadFKConstraint, + "interpretation_revisions_id_shape", "interpretation_revisions_content_digest_shape", + "interpretation_revisions_parent_pair", "interpretation_revisions_parent_revision_shape", + "interpretation_revisions_parent_digest_shape", "interpretation_revisions_author_non_empty", + "interpretation_revisions_explanation_non_empty", "interpretation_revisions_payload_object", + "interpretation_revisions_library_id_shape", + "interpretation_libraries_head_revision_shape", "interpretation_libraries_head_digest_shape", + interpretationRevisionImmutableTrigger, interpretationLibraryHeadGuardTrigger, + interpretationRevisionGuardFunction, interpretationLibraryGuardFunction, + `ERRCODE = '` + pgstate04ImmutableCode + `'`, + } { + if !strings.Contains(m.SQL, required) { + t.Errorf("fixture missing named object %q", required) + } + } + for _, trigger := range []string{interpretationRevisionImmutableTrigger, interpretationLibraryHeadGuardTrigger} { + if !strings.Contains(upper, "DROP TRIGGER IF EXISTS "+strings.ToUpper(trigger)) { + t.Errorf("trigger %s lacks a rerun-safe DROP TRIGGER IF EXISTS", trigger) + } + } +} + +func TestInterpretationStatementSQLContract(t *testing.T) { + if _, err := NewInterpretationStore(nil, "loom_control"); err == nil { + t.Fatal("NewInterpretationStore accepted a nil client") + } + for _, bad := range []string{"", " ", "Loom", "1loom", "loom;drop", `loom"x`, "loom x", strings.Repeat("a", 64)} { + if _, err := NewInterpretationStore(&pgstore.Client{}, bad); err == nil { + t.Errorf("NewInterpretationStore accepted invalid schema %q", bad) + } else if !strings.Contains(err.Error(), "invalid interpretation schema") { + t.Errorf("NewInterpretationStore(%q) error = %v, want invalid-schema error", bad, err) + } + } + store, err := NewInterpretationStore(&pgstore.Client{}, "loom_control") + if err != nil { + t.Fatalf("NewInterpretationStore = %v", err) + } + revFQN := `"loom_control".` + `"` + InterpretationRevisionsTable + `"` + libFQN := `"loom_control".` + `"` + InterpretationLibrariesTable + `"` + for _, stmt := range []string{store.sql.revisionInsert, store.sql.revisionGet, store.sql.parentDigestLookup} { + if !strings.Contains(stmt, revFQN) { + t.Errorf("statement %q must bind the schema-qualified revision table", stmt) + } + } + for _, stmt := range []string{store.sql.libraryInsertRoot, store.sql.libraryAdvance, store.sql.libraryList} { + if !strings.Contains(stmt, libFQN) { + t.Errorf("statement %q must bind the schema-qualified library table", stmt) + } + } + if !strings.Contains(store.sql.revisionInsert, "ON CONFLICT DO NOTHING RETURNING") { + t.Error("revision insert must be create-once over every unique index: ON CONFLICT DO NOTHING RETURNING") + } + if strings.Contains(store.sql.revisionInsert, "ON CONFLICT (") { + t.Error("revision insert must not target a single conflict clause: both unique indexes arbitrate the same-content race") + } + for i := 1; i <= len(interpretationRevisionColumns); i++ { + if !strings.Contains(store.sql.revisionInsert, fmt.Sprintf("$%d", i)) { + t.Fatalf("revision insert is missing ordered parameter $%d", i) + } + } + if strings.Contains(store.sql.revisionInsert, fmt.Sprintf("$%d", len(interpretationRevisionColumns)+1)) { + t.Fatalf("revision insert carries more than %d parameters", len(interpretationRevisionColumns)) + } + if !strings.Contains(store.sql.parentDigestLookup, "WHERE project = $1 AND id = $2 AND library_id = $3") { + t.Error("parent lookup must be scoped by project AND id AND library_id") + } + if !strings.Contains(store.sql.revisionGet, "WHERE project = $1 AND id = $2") { + t.Error("revision get must be scoped by project AND id") + } + if !strings.Contains(store.sql.libraryAdvance, "WHERE project = $1 AND id = $2 AND head_revision_id = $3") { + t.Error("head advance must be guarded on the expected head revision") + } + setClause := store.sql.libraryAdvance[strings.Index(store.sql.libraryAdvance, " SET "):] + setClause = setClause[:strings.Index(setClause, " WHERE ")] + for _, forbidden := range []string{"project =", " id =", "created_at ="} { + if strings.Contains(setClause, forbidden) { + t.Errorf("head advance SET must never touch %s; got SET %s", forbidden, setClause) + } + } + if !strings.Contains(store.sql.libraryList, "WHERE project = $1 ORDER BY id") { + t.Error("library list must be project-scoped and byte-order sorted") + } +} + +const ( + interpGoldenProject = "proj_golden" + interpGoldenLibrary = "library-golden" + interpGoldenAuthor = "author-a" + interpGoldenExplanation = "reason-a" + interpGoldenRulePath = "status" + + interpGoldenContentDigest = "sha256:4fa584217cf56fef7d9c24fc858455fcf6da3d0fecdca366d75628f780df74cf" + + interpGoldenRevisionID = "interpretation_1a4506e36aa1182d997fb90ffc3539c411e20208844b7ab8dae2862996ad997c" +) + +const interpGoldenCanonicalContentJSON = `{"applicability":{},"rules":[{"id":"rule-status","match":{"resourceType":"Observation"},"definition":{"source":{"kind":"field","field":{"path":"status","projectionMode":"VALUE"}}}}]}` + +func interpGoldenEnvelopeJSON(contentDigest string) string { + return `{"author":"` + interpGoldenAuthor + + `","contentDigest":"` + contentDigest + + `","explanation":"` + interpGoldenExplanation + + `","libraryId":"` + interpGoldenLibrary + + `","parentDigest":"","parentRevisionId":"","project":"` + interpGoldenProject + `"}` +} + +func sha256Hex(raw string) string { + sum := sha256.Sum256([]byte(raw)) + return hex.EncodeToString(sum[:]) +} + +func TestInterpretationIdentityGoldenLiterals(t *testing.T) { + fixture := interpRevision(interpGoldenProject, interpGoldenLibrary, interpGoldenAuthor, interpGoldenExplanation, interpGoldenRulePath) + + independentDigest := "sha256:" + sha256Hex(interpGoldenCanonicalContentJSON) + if independentDigest != interpGoldenContentDigest { + t.Fatalf("independent content digest recomputation disagrees with the pinned literal:\n got %s\nwant %s", independentDigest, interpGoldenContentDigest) + } + independentID := "interpretation_" + sha256Hex(interpGoldenEnvelopeJSON(interpGoldenContentDigest)) + if independentID != interpGoldenRevisionID { + t.Fatalf("independent revision id recomputation disagrees with the pinned literal:\n got %s\nwant %s", independentID, interpGoldenRevisionID) + } + + prepared, err := explorer.PrepareInterpretationRevision(fixture) + if err != nil { + t.Fatalf("PrepareInterpretationRevision = %v", err) + } + if string(prepared.ContentDigest) != interpGoldenContentDigest { + t.Fatalf("production content digest = %s, want pinned %s", prepared.ContentDigest, interpGoldenContentDigest) + } + if string(prepared.ID) != interpGoldenRevisionID { + t.Fatalf("production revision id = %s, want pinned %s", prepared.ID, interpGoldenRevisionID) + } + if err := prepared.Validate(); err != nil { + t.Fatalf("prepared golden revision failed validation: %v", err) + } + + other := fixture + other.Explanation = "a different proposal" + if otherPrepared, err := explorer.PrepareInterpretationRevision(other); err != nil { + t.Fatalf("prepare variant = %v", err) + } else if otherPrepared.ID == prepared.ID || otherPrepared.ContentDigest == prepared.ContentDigest && otherPrepared.Explanation == prepared.Explanation { + t.Fatal("different explanation produced the same immutable identity") + } + contentDrift := interpRevision(interpGoldenProject, interpGoldenLibrary, interpGoldenAuthor, interpGoldenExplanation, "code") + if contentPrepared, err := explorer.PrepareInterpretationRevision(contentDrift); err != nil { + t.Fatalf("prepare content variant = %v", err) + } else if contentPrepared.ContentDigest == prepared.ContentDigest || contentPrepared.ID == prepared.ID { + t.Fatal("different rule content produced the same immutable content digest or id") + } +} + +func TestInterpretationCreateRootRoundTripLiteralRows(t *testing.T) { + s := newInterpretationService(t) + fixture := interpRevision("proj_round", "library-a", "author-a", "reason-a", interpGoldenRulePath) + prepared, err := explorer.PrepareInterpretationRevision(fixture) + if err != nil { + s.fail("prepare fixture", err) + } + created := createRevision(t, s, s.storeA, fixture, nil) + if created.ID != prepared.ID || created.ContentDigest != prepared.ContentDigest { + t.Fatalf("created revision drifted from the prepared identity: %+v vs %+v", created, prepared) + } + if !created.CreatedAt.Equal(interpBaseTime) { + t.Fatalf("created createdAt = %s, want the literal %s", created.CreatedAt, interpBaseTime) + } + + w := s.mustWitnessRevision(t, "proj_round", string(prepared.ID)) + want := interpWitnessRevision{ + project: "proj_round", id: string(prepared.ID), libraryID: "library-a", + parentRevision: "", parentDigest: "", parentIsNull: true, + contentDigest: string(prepared.ContentDigest), author: "author-a", explanation: "reason-a", + createdAtText: interpBaseTimeText, payloadKind: "object", + } + got := w + got.payloadText, got.ctid = "", "" + want.payloadText, want.ctid = "", "" + if got != want { + t.Fatalf("literal revision row mismatch:\n got %+v\nwant %+v", got, want) + } + for _, fragment := range []string{ + `"contentDigest": "` + string(prepared.ContentDigest) + `"`, + `"id": "` + string(prepared.ID) + `"`, + `"libraryId": "library-a"`, + `"author": "author-a"`, + `"explanation": "reason-a"`, + `"path": "status"`, + } { + if !strings.Contains(w.payloadText, fragment) { + t.Errorf("payload does not bind %s: %s", fragment, w.payloadText) + } + } + + l := s.mustWitnessLibrary(t, "proj_round", "library-a") + if l.headRevision != string(prepared.ID) || l.headDigest != string(prepared.ContentDigest) || + l.createdAt != interpBaseTimeText || l.updatedAt != interpBaseTimeText { + t.Fatalf("literal library row wrong: %+v", l) + } + if n := s.libraryCount(t, "proj_round", "library-a"); n != 1 { + t.Fatalf("library rows = %d, want exactly 1", n) + } + + got2, err := s.storeB.GetInterpretationRevision(s.ctx, "proj_round", prepared.ID) + if err != nil { + s.fail("get revision on second backend", err) + } + if got2.ID != prepared.ID || got2.ContentDigest != prepared.ContentDigest || + got2.Author != "author-a" || got2.Explanation != "reason-a" || + !got2.CreatedAt.Equal(interpBaseTime) || !reflect.DeepEqual(got2.Rules, prepared.Rules) { + t.Fatalf("revision read-back drifted: %+v", got2) + } + if got2.ParentRevisionID != nil || got2.ParentDigest != nil { + t.Fatalf("root revision read back with a parent: %+v", got2) + } + + child := interpChild(*created, "child-reason", "code") + childPrepared, err := explorer.PrepareInterpretationRevision(child) + if err != nil { + s.fail("prepare child", err) + } + childTime := interpBaseTime.Add(90 * time.Second) + child.CreatedAt = childTime + childPrepared.CreatedAt = childTime + if err := childPrepared.Validate(); err != nil { + t.Fatalf("re-timed child failed validation: %v", err) + } + childCreated := createRevision(t, s, s.storeB, child, idRef(created.ID)) + if childCreated.ID != childPrepared.ID { + t.Fatalf("child id drifted") + } + cw := s.mustWitnessRevision(t, "proj_round", string(childPrepared.ID)) + if cw.parentIsNull || cw.parentRevision != string(created.ID) || cw.parentDigest != string(created.ContentDigest) { + t.Fatalf("child parent mirrors wrong: %+v", cw) + } + if cw.createdAtText != "2026-03-04 05:07:37.123456" { + t.Fatalf("child created_at = %s, want the literal", cw.createdAtText) + } + l2 := s.mustWitnessLibrary(t, "proj_round", "library-a") + if l2.headRevision != string(childPrepared.ID) || l2.headDigest != string(childPrepared.ContentDigest) { + t.Fatalf("head did not advance to the child: %+v", l2) + } + if l2.createdAt != interpBaseTimeText || l2.updatedAt != "2026-03-04 05:07:37.123456" { + t.Fatalf("head timestamps wrong (created must stay, updated must follow the child): %+v", l2) + } + if n := s.revisionCount(t, "proj_round", "library-a"); n != 2 { + t.Fatalf("revision rows = %d, want 2 (root + child, both retained)", n) + } + + libraries, err := s.storeA.ListInterpretationLibraries(s.ctx, "proj_round") + if err != nil { + s.fail("list libraries", err) + } + if len(libraries) != 1 || libraries[0].ID != "library-a" || + libraries[0].HeadRevisionID != childPrepared.ID || + libraries[0].HeadDigest != childPrepared.ContentDigest { + t.Fatalf("list = %+v, want the advanced head", libraries) + } +} + +func TestInterpretationSameDigestRetryIsIdempotent(t *testing.T) { + s := newInterpretationService(t) + fixture := interpRevision("proj_retry", "library-a", "author-a", "reason-a", interpGoldenRulePath) + first := createRevision(t, s, s.storeA, fixture, nil) + before := s.mustWitnessRevision(t, "proj_retry", string(first.ID)) + beforeLib := s.mustWitnessLibrary(t, "proj_retry", "library-a") + + second, err := s.storeA.CreateInterpretationRevision(s.ctx, fixture, nil) + if err != nil { + s.fail("same-digest retry", err) + } + if second.ID != first.ID || second.ContentDigest != first.ContentDigest || + second.CreatedAt != first.CreatedAt || second.Explanation != first.Explanation { + t.Fatalf("retry returned a different revision: %+v vs %+v", second, first) + } + + timeShifted := fixture + timeShifted.CreatedAt = interpBaseTime.Add(time.Hour) + third, err := s.storeB.CreateInterpretationRevision(s.ctx, timeShifted, nil) + if err != nil { + s.fail("same-digest retry with shifted time", err) + } + if third.ID != first.ID || third.CreatedAt != first.CreatedAt { + t.Fatalf("shifted-time retry did not return the stored row: %+v vs %+v", third, first) + } + + after := s.mustWitnessRevision(t, "proj_retry", string(first.ID)) + if after != before { + t.Fatalf("same-digest retries changed the revision row:\n got %+v\nwant %+v", after, before) + } + afterLib := s.mustWitnessLibrary(t, "proj_retry", "library-a") + if afterLib != beforeLib { + t.Fatalf("same-digest retries changed the library row:\n got %+v\nwant %+v", afterLib, beforeLib) + } + if n := s.revisionCount(t, "proj_retry", "library-a"); n != 1 { + t.Fatalf("revision rows = %d, want exactly 1", n) + } + if n := s.libraryCount(t, "proj_retry", "library-a"); n != 1 { + t.Fatalf("library rows = %d, want exactly 1", n) + } + + stale := interpRevision("proj_retry", "library-a", "author-b", "a different proposal", "code") + if _, err := s.storeA.CreateInterpretationRevision(s.ctx, stale, nil); !errors.Is(err, explorer.ErrInterpretationParentConflict) { + t.Fatalf("stale root create error = %v, want ErrInterpretationParentConflict", err) + } + if n := s.revisionCount(t, "proj_retry", "library-a"); n != 1 { + t.Fatalf("refused stale root left %d revisions behind", n) + } + + tampered := interpRevision("proj_tamper_retry", "library-a", "author-a", "reason-a", interpGoldenRulePath) + tamperedCreated := createRevision(t, s, s.storeA, tampered, nil) + tamperedPrepared, err := explorer.PrepareInterpretationRevision(tampered) + if err != nil { + s.fail("prepare tampered", err) + } + s.mustWitnessExec(t, `DROP TRIGGER `+interpretationRevisionImmutableTrigger+` ON `+s.qualified(InterpretationRevisionsTable)) + s.mustWitnessExec(t, `UPDATE `+s.qualified(InterpretationRevisionsTable)+ + ` SET payload = jsonb_set(payload, '{explanation}', '"tampered"') WHERE project = $1 AND id = $2`, + "proj_tamper_retry", string(tamperedCreated.ID)) + s.mustWitnessExec(t, `CREATE TRIGGER `+interpretationRevisionImmutableTrigger+` BEFORE UPDATE OR DELETE ON `+ + s.qualified(InterpretationRevisionsTable)+` FOR EACH ROW EXECUTE FUNCTION `+ + s.qualified(interpretationRevisionGuardFunction)+`()`) + if _, err := s.storeB.CreateInterpretationRevision(s.ctx, tampered, nil); !errors.Is(err, explorer.ErrImmutableInterpretation) { + t.Fatalf("create over tampered same-identity row = %v, want ErrImmutableInterpretation", err) + } + tamperedWitness := s.mustWitnessRevision(t, "proj_tamper_retry", string(tamperedPrepared.ID)) + if !strings.Contains(tamperedWitness.payloadText, `"tampered"`) { + t.Fatalf("collision refusal rewrote the tampered witness: %s", tamperedWitness.payloadText) + } + + err = s.witnessAcquireExecErr(`INSERT INTO `+s.qualified(InterpretationRevisionsTable)+ + ` (project, id, library_id, parent_revision_id, parent_digest, content_digest, author, explanation, created_at, payload)`+ + ` VALUES ($1, $2, 'library-x', NULL, NULL, 'sha256:`+strings.Repeat("1", 64)+`', 'a', 'b', $3, '{}'::jsonb)`, + "proj_retry", string(first.ID), interpBaseTime) + requireInterpPgErr(t, err, pgstate04UniqueViolation, interpretationRevisionPKConstraint, "raw same-identity insert") +} + +func (s *interpretationService) witnessAcquireExecErr(sql string, args ...any) error { + return interpExecOn(s.ctx, s.witness, sql, args...) +} + +func requireInterpPgErr(t *testing.T, err error, sqlstate, constraint, step string) { + t.Helper() + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + t.Fatalf("%s: error = %v (%T), want *pgconn.PgError", step, err, err) + } + if pgErr.Code != sqlstate { + t.Fatalf("%s: sqlstate = %s, want %s (error: %v)", step, pgErr.Code, sqlstate, err) + } + if constraint != "" && pgErr.ConstraintName != constraint { + t.Fatalf("%s: constraint = %q, want %q (error: %v)", step, pgErr.ConstraintName, constraint, err) + } +} + +func TestInterpretationConcurrentChildrenSameHeadOneWinner(t *testing.T) { + s := newInterpretationService(t) + winsA, winsB := 0, 0 + for round := 1; round <= interpRaceRounds; round++ { + s.requireRoundBackendsDistinct(t, "same-head child race", round) + project := fmt.Sprintf("proj_race_%d", round) + + root := createRevision(t, s, s.storeA, interpRevision(project, "library-a", "author-a", "root", interpGoldenRulePath), nil) + childA := interpChild(*root, "child-a", "code") + childB := interpChild(*root, "child-b", "value") + prepA, err := explorer.PrepareInterpretationRevision(childA) + if err != nil { + s.fail("prepare child A", err) + } + prepB, err := explorer.PrepareInterpretationRevision(childB) + if err != nil { + s.fail("prepare child B", err) + } + if prepA.ID == prepB.ID { + t.Fatal("distinct children derived the same identity") + } + + type racer struct { + who string + store *InterpretationStore + input explorer.InterpretationRevision + prep explorer.InterpretationRevision + } + racers := []racer{ + {who: "A", store: s.storeA, input: childA, prep: prepA}, + {who: "B", store: s.storeB, input: childB, prep: prepB}, + } + type outcome struct { + who string + rev *explorer.InterpretationRevision + err error + } + start := make(chan struct{}) + results := make(chan outcome, 2) + for _, r := range racers { + go func(r racer) { + <-start + rev, err := r.store.CreateInterpretationRevision(s.ctx, r.input, idRef(root.ID)) + results <- outcome{who: r.who, rev: rev, err: err} + }(r) + } + close(start) + + wins := 0 + var winner outcome + for range 2 { + r := <-results + if r.err == nil { + wins++ + winner = r + } else { + if !errors.Is(r.err, explorer.ErrInterpretationParentConflict) { + t.Fatalf("round %d: loser error = %v, want ErrInterpretationParentConflict", round, r.err) + } + } + } + if wins != 1 { + t.Fatalf("round %d: same expected head produced %d winners, want exactly 1", round, wins) + } + var loser racer + for _, r := range racers { + if r.who == winner.who { + continue + } + loser = r + } + if winner.who == "A" { + winsA++ + } else { + winsB++ + } + + l := s.mustWitnessLibrary(t, project, "library-a") + if l.headRevision != string(winner.rev.ID) || l.headDigest != string(winner.rev.ContentDigest) { + t.Fatalf("round %d: head %+v does not bind the winner %s", round, l, winner.rev.ID) + } + if n := s.revisionCount(t, project, "library-a"); n != 2 { + t.Fatalf("round %d: revision rows = %d, want 2 (root + winner only)", round, n) + } + if _, ok := s.witnessRevision(t, project, string(loser.prep.ID)); ok { + t.Fatalf("round %d: loser %s left a revision row behind", round, loser.who) + } + + w := s.mustWitnessRevision(t, project, string(winner.rev.ID)) + if w.project != project || w.id != string(winner.rev.ID) || + w.parentRevision != string(root.ID) || w.parentDigest != string(root.ContentDigest) || + w.contentDigest != string(winner.rev.ContentDigest) || + w.explanation != winner.rev.Explanation || w.author != winner.rev.Author { + t.Fatalf("round %d: winner row does not match the winner's return: %+v vs %+v", round, w, winner.rev) + } + + if _, err := loser.store.CreateInterpretationRevision(s.ctx, loser.input, idRef(root.ID)); !errors.Is(err, explorer.ErrInterpretationParentConflict) { + t.Fatalf("round %d: stale replay error = %v, want ErrInterpretationParentConflict", round, err) + } + if _, ok := s.witnessRevision(t, project, string(loser.prep.ID)); ok { + t.Fatalf("round %d: stale replay persisted the loser row", round) + } + } + if winsA == 0 || winsB == 0 { + t.Fatalf("over %d real rounds one writer never won (A=%d B=%d): contention not observable", interpRaceRounds, winsA, winsB) + } +} + +func TestInterpretationConcurrentSameContentConverges(t *testing.T) { + s := newInterpretationService(t) + for round := 1; round <= interpRaceRounds; round++ { + s.requireRoundBackendsDistinct(t, "same-content race", round) + project := fmt.Sprintf("proj_same_%d", round) + root := createRevision(t, s, s.storeA, interpRevision(project, "library-a", "author-a", "root", interpGoldenRulePath), nil) + shared := interpChild(*root, "shared-child", "code") + prepared, err := explorer.PrepareInterpretationRevision(shared) + if err != nil { + s.fail("prepare shared child", err) + } + + type outcome struct { + rev *explorer.InterpretationRevision + err error + } + start := make(chan struct{}) + results := make(chan outcome, 2) + for _, store := range []*InterpretationStore{s.storeA, s.storeB} { + go func(store *InterpretationStore) { + <-start + rev, err := store.CreateInterpretationRevision(s.ctx, shared, idRef(root.ID)) + results <- outcome{rev: rev, err: err} + }(store) + } + close(start) + + var revs []*explorer.InterpretationRevision + for range 2 { + r := <-results + if r.err != nil { + t.Fatalf("round %d: same-content racer failed: %v", round, s.scrub(r.err)) + } + revs = append(revs, r.rev) + } + if revs[0].ID != prepared.ID || revs[1].ID != prepared.ID || + revs[0].ContentDigest != prepared.ContentDigest || revs[1].ContentDigest != prepared.ContentDigest { + t.Fatalf("round %d: same-content racers returned %s / %s, want %s", round, revs[0].ID, revs[1].ID, prepared.ID) + } + if !reflect.DeepEqual(*revs[0], *revs[1]) { + t.Fatalf("round %d: same-content racers returned different values:\n%+v\n%+v", round, *revs[0], *revs[1]) + } + + w := s.mustWitnessRevision(t, project, string(prepared.ID)) + second, ok := s.witnessRevision(t, project, string(revs[1].ID)) + if !ok || second.payloadText != w.payloadText || second.ctid != w.ctid { + t.Fatalf("round %d: the two racers did not converge on one byte-identical row: %+v vs %+v", round, w, second) + } + if n := s.revisionCount(t, project, "library-a"); n != 2 { + t.Fatalf("round %d: revision rows = %d, want 2 (root + the single child)", round, n) + } + l := s.mustWitnessLibrary(t, project, "library-a") + if l.headRevision != string(prepared.ID) || l.headDigest != string(prepared.ContentDigest) { + t.Fatalf("round %d: head did not advance exactly once to the child: %+v", round, l) + } + if n := s.libraryCount(t, project, "library-a"); n != 1 { + t.Fatalf("round %d: library rows = %d, want exactly 1", round, n) + } + } +} + +func TestInterpretationFaultAfterRevisionInsertRollsBackAndRetryConverges(t *testing.T) { + s := newInterpretationService(t) + + baselineRoot := createRevision(t, s, s.storeA, interpRevision("proj_fault_base", "library-a", "author-a", "root", interpGoldenRulePath), nil) + baselineChild := createRevision(t, s, s.storeA, interpChild(*baselineRoot, "fault-child", "code"), idRef(baselineRoot.ID)) + baseChildWitness := s.mustWitnessRevision(t, "proj_fault_base", string(baselineChild.ID)) + + injected := errors.New("controlled transport failure after revision insert") + faultStore, err := NewInterpretationStore(s.clientA, s.schema) + if err != nil { + s.fail("build faulted store", err) + } + faultStore.afterRevisionInsert = func(context.Context, pgx.Tx) error { return injected } + + faultProject := "proj_fault_root" + faultRootInput := interpRevision(faultProject, "library-a", "author-a", "root", interpGoldenRulePath) + faultPrepared, err := explorer.PrepareInterpretationRevision(faultRootInput) + if err != nil { + s.fail("prepare fault root", err) + } + if _, err := faultStore.CreateInterpretationRevision(s.ctx, faultRootInput, nil); !errors.Is(err, injected) { + t.Fatalf("faulted root create error = %v, want the injected cause retained", err) + } + if _, ok := s.witnessRevision(t, faultProject, string(faultPrepared.ID)); ok { + t.Fatal("faulted root create left the revision row behind") + } + if _, ok := s.witnessLibrary(t, faultProject, "library-a"); ok { + t.Fatal("faulted root create left the library row behind") + } + + retryStore, err := NewInterpretationStore(s.clientB, s.schema) + if err != nil { + s.fail("build retry store", err) + } + retried := createRevision(t, s, retryStore, faultRootInput, nil) + if retried.ID != faultPrepared.ID { + t.Fatalf("retry returned %s, want the prepared %s", retried.ID, faultPrepared.ID) + } + retryWitness := s.mustWitnessRevision(t, faultProject, string(faultPrepared.ID)) + l := s.mustWitnessLibrary(t, faultProject, "library-a") + if retryWitness.parentIsNull && l.headRevision != string(faultPrepared.ID) { + t.Fatalf("retry did not establish the head: %+v", l) + } + + faultChild := interpChild(*retried, "fault-child", "code") + faultChildPrepared, err := explorer.PrepareInterpretationRevision(faultChild) + if err != nil { + s.fail("prepare fault child", err) + } + if _, err := faultStore.CreateInterpretationRevision(s.ctx, faultChild, idRef(retried.ID)); !errors.Is(err, injected) { + t.Fatalf("faulted child create error = %v, want the injected cause retained", err) + } + if _, ok := s.witnessRevision(t, faultProject, string(faultChildPrepared.ID)); ok { + t.Fatal("faulted child create left the child row behind") + } + lStill := s.mustWitnessLibrary(t, faultProject, "library-a") + if lStill != l { + t.Fatalf("faulted child create moved the head:\n got %+v\nwant %+v", lStill, l) + } + + retriedChild := createRevision(t, s, retryStore, interpChild(*retried, "fault-child", "code"), idRef(retried.ID)) + retryChildWitness := s.mustWitnessRevision(t, faultProject, string(retriedChild.ID)) + if !strings.Contains(retryChildWitness.payloadText, `"fault-child"`) || + !strings.Contains(retryChildWitness.payloadText, `"path": "code"`) || + !strings.Contains(baseChildWitness.payloadText, `"path": "code"`) { + t.Fatalf("retry child payload diverged from the clean baseline:\n %s\n %s", retryChildWitness.payloadText, baseChildWitness.payloadText) + } + if retryChildWitness.contentDigest != string(interpChild(*retried, "fault-child", "code").ContentDigest) { + check, err := explorer.PrepareInterpretationRevision(interpChild(*retried, "fault-child", "code")) + if err != nil || string(check.ContentDigest) != retryChildWitness.contentDigest { + t.Fatalf("retry child content digest %q disagrees with the domain derivation", retryChildWitness.contentDigest) + } + } + lAfter := s.mustWitnessLibrary(t, faultProject, "library-a") + if lAfter.headRevision != string(retriedChild.ID) || lAfter.headDigest != string(retryChildWitness.contentDigest) { + t.Fatalf("post-abort retry did not converge on the head: %+v", lAfter) + } +} + +func TestInterpretationHeadCASLoserRollbackAfterCommittedTakeover(t *testing.T) { + s := newInterpretationService(t) + project := "proj_takeover" + root := createRevision(t, s, s.storeA, interpRevision(project, "library-a", "author-a", "root", interpGoldenRulePath), nil) + + inserted := make(chan struct{}) + takeoverDone := make(chan struct{}) + released := false + release := func() { + if !released { + released = true + close(takeoverDone) + } + } + t.Cleanup(release) + s.storeA.afterRevisionInsert = func(ctx context.Context, _ pgx.Tx) error { + close(inserted) + select { + case <-takeoverDone: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + + type outcome struct { + rev *explorer.InterpretationRevision + err error + } + loserResult := make(chan outcome, 1) + loserChild := interpChild(*root, "loser-child", "value") + loserPrepared, err := explorer.PrepareInterpretationRevision(loserChild) + if err != nil { + s.fail("prepare loser child", err) + } + go func() { + rev, err := s.storeA.CreateInterpretationRevision(s.ctx, loserChild, idRef(root.ID)) + loserResult <- outcome{rev: rev, err: err} + }() + + select { + case <-inserted: + case <-time.After(30 * time.Second): + release() + t.Fatal("loser writer never reached the post-insert seam") + } + + takeoverChild := interpChild(*root, "takeover-child", "code") + takeover, err := s.storeB.CreateInterpretationRevision(s.ctx, takeoverChild, idRef(root.ID)) + if err != nil { + release() + s.fail("takeover create", err) + } + takeoverWitness := s.mustWitnessRevision(t, project, string(takeover.ID)) + takeoverLib := s.mustWitnessLibrary(t, project, "library-a") + release() + + res := <-loserResult + if res.err == nil || !errors.Is(res.err, explorer.ErrInterpretationParentConflict) { + t.Fatalf("post-insert CAS failure error = %v, want ErrInterpretationParentConflict", res.err) + } + if res.rev != nil { + t.Fatalf("failed CAS returned a revision: %+v", res.rev) + } + if _, ok := s.witnessRevision(t, project, string(loserPrepared.ID)); ok { + t.Fatal("aborted loser left its revision row behind") + } + afterWitness := s.mustWitnessRevision(t, project, string(takeover.ID)) + if afterWitness != takeoverWitness { + t.Fatalf("aborted loser disturbed the takeover row:\n got %+v\nwant %+v", afterWitness, takeoverWitness) + } + afterLib := s.mustWitnessLibrary(t, project, "library-a") + if afterLib != takeoverLib { + t.Fatalf("aborted loser disturbed the head:\n got %+v\nwant %+v", afterLib, takeoverLib) + } +} + +func TestInterpretationProjectScopeHidesForeignIDsAndOldRevisionsSurvive(t *testing.T) { + s := newInterpretationService(t) + rootA := createRevision(t, s, s.storeA, interpRevision("proj_scope_a", "library-a", "author-a", "root-a", interpGoldenRulePath), nil) + childA1 := createRevision(t, s, s.storeA, interpChild(*rootA, "child-a1", "code"), idRef(rootA.ID)) + childA2 := createRevision(t, s, s.storeA, interpChild(*childA1, "child-a2", "value"), idRef(childA1.ID)) + rootB := createRevision(t, s, s.storeB, interpRevision("proj_scope_b", "library-a", "author-b", "root-b", interpGoldenRulePath), nil) + + missing, err := s.storeB.GetInterpretationRevision(s.ctx, "proj_scope_b", rootA.ID) + if !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("foreign-project read = %v (%+v), want concealed ErrNotFound", err, missing) + } + nonexistent := "interpretation_" + strings.Repeat("0", 64) + _, errMissing := s.storeB.GetInterpretationRevision(s.ctx, "proj_scope_b", explorer.InterpretationRevisionID(nonexistent)) + if !errors.Is(errMissing, explorer.ErrNotFound) { + t.Fatalf("nonexistent read = %v, want ErrNotFound", errMissing) + } + if err.Error() != errMissing.Error() { + t.Fatalf("foreign ID and missing ID are distinguishable: %q vs %q", err, errMissing) + } + + if _, err := s.storeA.GetInterpretationRevision(s.ctx, "proj_scope_a", rootB.ID); !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("reverse cross-project read = %v, want ErrNotFound", err) + } + if _, err := s.storeA.GetInterpretationRevision(s.ctx, "proj_scope_a", explorer.InterpretationRevisionID("bogus id")); !errors.Is(err, explorer.ErrInvalidInterpretationID) { + t.Fatalf("malformed id read = %v, want ErrInvalidInterpretationID", err) + } + for _, blank := range []string{"", " ", "\t"} { + if _, err := s.storeA.ListInterpretationLibraries(s.ctx, blank); !errors.Is(err, explorer.ErrInvalidInterpretationProject) { + t.Fatalf("blank project list = %v, want ErrInvalidInterpretationProject", err) + } + if _, err := s.storeA.GetInterpretationRevision(s.ctx, blank, rootA.ID); !errors.Is(err, explorer.ErrInvalidInterpretationProject) { + t.Fatalf("blank project get = %v, want ErrInvalidInterpretationProject", err) + } + } + + for _, want := range []*explorer.InterpretationRevision{rootA, childA1, childA2} { + got, err := s.storeB.GetInterpretationRevision(s.ctx, "proj_scope_a", want.ID) + if err != nil { + s.fail("read old revision", err) + } + if got.ID != want.ID || got.ContentDigest != want.ContentDigest { + t.Fatalf("old revision %s did not survive head advancement: %+v", want.ID, got) + } + } + l := s.mustWitnessLibrary(t, "proj_scope_a", "library-a") + if l.headRevision != string(childA2.ID) { + t.Fatalf("head is not at the newest child: %+v", l) + } + if n := s.revisionCount(t, "proj_scope_a", "library-a"); n != 3 { + t.Fatalf("revision rows = %d, want 3 (full immutable chain retained)", n) + } + + listedA, err := s.storeA.ListInterpretationLibraries(s.ctx, "proj_scope_a") + if err != nil { + s.fail("list A", err) + } + if len(listedA) != 1 || listedA[0].Project != "proj_scope_a" { + t.Fatalf("list A leaked: %+v", listedA) + } + empty, err := s.storeA.ListInterpretationLibraries(s.ctx, "proj_never") + if err != nil { + s.fail("list unknown", err) + } + if empty == nil || len(empty) != 0 { + t.Fatalf("unknown-project list = %#v, want empty non-nil slice", empty) + } + + oldTime := interpBaseTime.Add(-48 * time.Hour) + oldChild := interpChild(*childA2, "old-child", "category") + oldChild.CreatedAt = oldTime + oldPrepared, err := explorer.PrepareInterpretationRevision(oldChild) + if err != nil { + s.fail("prepare old-time child", err) + } + createRevision(t, s, s.storeA, oldChild, idRef(childA2.ID)) + lOld := s.mustWitnessLibrary(t, "proj_scope_a", "library-a") + if lOld.headRevision != string(oldPrepared.ID) { + t.Fatalf("older-timestamped child did not advance the head: %+v", lOld) + } + oc := s.mustWitnessRevision(t, "proj_scope_a", string(oldPrepared.ID)) + if oc.createdAtText != "2026-03-02 05:06:07.123456" { + t.Fatalf("old-time child timestamp = %s, want the two-day-earlier literal", oc.createdAtText) + } +} + +func (s *interpretationService) triggerPresent(t *testing.T, trigger, table string) bool { + t.Helper() + var present bool + err := interpQueryRow(s.ctx, s.witness, + `SELECT EXISTS ( + SELECT 1 FROM pg_trigger tg + JOIN pg_class c ON c.oid = tg.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE tg.tgname = $1 AND c.relname = $2 AND n.nspname = $3 + AND NOT tg.tgisinternal)`, + []any{&present}, trigger, table, s.schema) + if err != nil { + s.fail("trigger catalog witness "+trigger, err) + } + return present +} + +func (s *interpretationService) interpGuardDropProbe(t *testing.T, trigger, table, function string, reseed func(), stmt string, args ...any) { + t.Helper() + tableFQN := s.qualified(table) + if !s.triggerPresent(t, trigger, table) { + t.Fatalf("guard trigger %s missing from %s before probe", trigger, table) + } + err := s.witnessExecErr(stmt, args...) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != pgstate04ImmutableCode { + t.Fatalf("guarded violating write on %s = %v, want SQLSTATE %s", table, s.scrub(err), pgstate04ImmutableCode) + } + s.mustWitnessExec(t, `DROP TRIGGER `+trigger+` ON `+tableFQN) + if s.triggerPresent(t, trigger, table) { + t.Fatalf("trigger %s still present after DROP", trigger) + } + conn, acquireErr := s.witness.Acquire(s.ctx) + if acquireErr != nil { + s.fail("acquire unguarded probe connection", acquireErr) + } + tag, execErr := conn.Exec(s.ctx, stmt, args...) + conn.Release() + if execErr != nil || tag.RowsAffected() != 1 { + t.Fatalf("same violating write ran with the trigger dropped on %s: tag=%v err=%v; the guard was not what blocked it", + table, tag, s.scrub(execErr)) + } + if reseed != nil { + reseed() + } + s.mustWitnessExec(t, `CREATE TRIGGER `+trigger+` BEFORE UPDATE OR DELETE ON `+tableFQN+ + ` FOR EACH ROW EXECUTE FUNCTION `+s.qualified(function)+`()`) + if !s.triggerPresent(t, trigger, table) { + t.Fatalf("guard trigger %s not restored on %s", trigger, table) + } + err = s.witnessExecErr(stmt, args...) + if !errors.As(err, &pgErr) || pgErr.Code != pgstate04ImmutableCode { + t.Fatalf("violating write after trigger restore = %v, want SQLSTATE %s", s.scrub(err), pgstate04ImmutableCode) + } +} + +func (s *interpretationService) seedProbeRevision(t *testing.T, project, id, library, digest string) { + t.Helper() + s.mustWitnessExec(t, `INSERT INTO `+s.qualified(InterpretationRevisionsTable)+ + ` (project, id, library_id, parent_revision_id, parent_digest, content_digest, author, explanation, created_at, payload)`+ + ` VALUES ($1, $2, $3, NULL, NULL, $4, 'probe-author', 'probe-explanation', $5, '{}'::jsonb)`, + project, id, library, digest, interpBaseTime) +} + +func TestInterpretationImmutabilityGuardsAreEnforcedAtDatabaseLevel(t *testing.T) { + s := newInterpretationService(t) + + artifact := createRevision(t, s, s.storeA, interpRevision("proj_immutable", "library-a", "author-a", "root", interpGoldenRulePath), nil) + beforeRevision := s.mustWitnessRevision(t, "proj_immutable", string(artifact.ID)) + beforeLibrary := s.mustWitnessLibrary(t, "proj_immutable", "library-a") + + probeProject := "probe_immutable" + probeID := "interpretation_" + "1" + strings.Repeat("0", 63) + probeDigest := "sha256:" + strings.Repeat("2", 64) + s.seedProbeRevision(t, probeProject, probeID, "probe-lib", probeDigest) + + s.interpGuardDropProbe(t, interpretationRevisionImmutableTrigger, InterpretationRevisionsTable, + interpretationRevisionGuardFunction, nil, + `UPDATE `+s.qualified(InterpretationRevisionsTable)+` SET content_digest = $2 WHERE project = $1 AND id = $3`, + probeProject, "sha256:"+strings.Repeat("3", 64), probeID) + reseedProbe := func() { s.seedProbeRevision(t, probeProject, probeID, "probe-lib", probeDigest) } + s.interpGuardDropProbe(t, interpretationRevisionImmutableTrigger, InterpretationRevisionsTable, + interpretationRevisionGuardFunction, reseedProbe, + `DELETE FROM `+s.qualified(InterpretationRevisionsTable)+` WHERE project = $1 AND id = $2`, + probeProject, probeID) + + s.mustWitnessExec(t, `INSERT INTO `+s.qualified(InterpretationLibrariesTable)+ + ` (project, id, head_revision_id, head_digest, created_at, updated_at)`+ + ` VALUES ($1, 'probe-lib', $2, $3, $4, $4)`, + probeProject, probeID, probeDigest, interpBaseTime) + s.interpGuardDropProbe(t, interpretationLibraryHeadGuardTrigger, InterpretationLibrariesTable, + interpretationLibraryGuardFunction, func() { + s.mustWitnessExec(t, `INSERT INTO `+s.qualified(InterpretationLibrariesTable)+ + ` (project, id, head_revision_id, head_digest, created_at, updated_at)`+ + ` VALUES ($1, 'probe-lib', $2, $3, $4, $4)`, + probeProject, probeID, probeDigest, interpBaseTime) + }, + `DELETE FROM `+s.qualified(InterpretationLibrariesTable)+` WHERE project = $1 AND id = $2`, + probeProject, "probe-lib") + + for _, stmt := range []string{ + `UPDATE ` + s.qualified(InterpretationLibrariesTable) + ` SET project = project WHERE project = $1 AND id = $2`, + `UPDATE ` + s.qualified(InterpretationLibrariesTable) + ` SET id = id WHERE project = $1 AND id = $2`, + `UPDATE ` + s.qualified(InterpretationLibrariesTable) + ` SET head_revision_id = head_revision_id WHERE project = $1 AND id = $2`, + } { + s.interpGuardDropProbe(t, interpretationLibraryHeadGuardTrigger, InterpretationLibrariesTable, + interpretationLibraryGuardFunction, nil, stmt, probeProject, "probe-lib") + } + + s.interpGuardDropProbe(t, interpretationLibraryHeadGuardTrigger, InterpretationLibrariesTable, + interpretationLibraryGuardFunction, func() { + s.mustWitnessExec(t, `UPDATE `+s.qualified(InterpretationLibrariesTable)+ + ` SET created_at = $3 WHERE project = $1 AND id = $2`, probeProject, "probe-lib", interpBaseTime) + }, + `UPDATE `+s.qualified(InterpretationLibrariesTable)+ + ` SET created_at = created_at - interval '1 day' WHERE project = $1 AND id = $2`, + probeProject, "probe-lib") + probeID2 := "interpretation_" + "2" + strings.Repeat("0", 63) + probeDigest2 := "sha256:" + strings.Repeat("4", 64) + s.seedProbeRevision(t, probeProject, probeID2, "probe-lib", probeDigest2) + s.mustWitnessExec(t, `UPDATE `+s.qualified(InterpretationLibrariesTable)+ + ` SET head_revision_id = $3, head_digest = $4, updated_at = now()`+ + ` WHERE project = $1 AND id = $2`, + probeProject, "probe-lib", probeID2, probeDigest2) + + afterRevision := s.mustWitnessRevision(t, "proj_immutable", string(artifact.ID)) + if afterRevision != beforeRevision { + t.Fatalf("guard probes disturbed the stored artifact row:\n got %+v\nwant %+v", afterRevision, beforeRevision) + } + afterLibrary := s.mustWitnessLibrary(t, "proj_immutable", "library-a") + if afterLibrary != beforeLibrary { + t.Fatalf("guard probes disturbed the stored library row:\n got %+v\nwant %+v", afterLibrary, beforeLibrary) + } + + tamperProject := "probe_tamper_read" + tamper := createRevision(t, s, s.storeB, interpRevision(tamperProject, "library-a", "author-a", "reason-a", interpGoldenRulePath), nil) + s.mustWitnessExec(t, `DROP TRIGGER `+interpretationRevisionImmutableTrigger+` ON `+s.qualified(InterpretationRevisionsTable)) + s.mustWitnessExec(t, `UPDATE `+s.qualified(InterpretationRevisionsTable)+ + ` SET payload = jsonb_set(payload, '{explanation}', '"tampered"') WHERE project = $1 AND id = $2`, + tamperProject, string(tamper.ID)) + s.mustWitnessExec(t, `CREATE TRIGGER `+interpretationRevisionImmutableTrigger+` BEFORE UPDATE OR DELETE ON `+ + s.qualified(InterpretationRevisionsTable)+` FOR EACH ROW EXECUTE FUNCTION `+ + s.qualified(interpretationRevisionGuardFunction)+`()`) + if _, err := s.storeA.GetInterpretationRevision(s.ctx, tamperProject, tamper.ID); !errors.Is(err, explorer.ErrInterpretationIdentityMismatch) { + t.Fatalf("Get over tampered row = %v, want ErrInterpretationIdentityMismatch", err) + } + + mirrorTamper := createRevision(t, s, s.storeB, interpRevision("probe_tamper_mirror", "library-a", "author-a", "reason-a", interpGoldenRulePath), nil) + s.mustWitnessExec(t, `DROP TRIGGER `+interpretationRevisionImmutableTrigger+` ON `+s.qualified(InterpretationRevisionsTable)) + s.mustWitnessExec(t, `UPDATE `+s.qualified(InterpretationRevisionsTable)+ + ` SET content_digest = $2 WHERE project = $1 AND id = $3`, + "probe_tamper_mirror", "sha256:"+strings.Repeat("9", 64), string(mirrorTamper.ID)) + s.mustWitnessExec(t, `CREATE TRIGGER `+interpretationRevisionImmutableTrigger+` BEFORE UPDATE OR DELETE ON `+ + s.qualified(InterpretationRevisionsTable)+` FOR EACH ROW EXECUTE FUNCTION `+ + s.qualified(interpretationRevisionGuardFunction)+`()`) + if _, err := s.storeA.GetInterpretationRevision(s.ctx, "probe_tamper_mirror", mirrorTamper.ID); !errors.Is(err, explorer.ErrInterpretationIdentityMismatch) { + t.Fatalf("Get over mirror-tampered row = %v, want ErrInterpretationIdentityMismatch", err) + } + if _, err := s.storeA.CreateInterpretationRevision(s.ctx, interpRevision("probe_tamper_mirror", "library-a", "author-a", "reason-a", interpGoldenRulePath), nil); !errors.Is(err, explorer.ErrImmutableInterpretation) { + t.Fatalf("create over mirror-tampered row = %v, want ErrImmutableInterpretation", err) + } +} + +func (s *interpretationService) insertRevisionDirectErr(project, id, library string, parent *string, parentDigest *string, digest, author, explanation string, payload string) error { + return s.witnessExecErr(`INSERT INTO `+s.qualified(InterpretationRevisionsTable)+ + ` (project, id, library_id, parent_revision_id, parent_digest, content_digest, author, explanation, created_at, payload)`+ + ` VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::jsonb)`, + project, id, library, parent, parentDigest, digest, author, explanation, interpBaseTime, payload) +} + +func (s *interpretationService) insertLibraryDirectErr(project, id, head, digest string) error { + return s.witnessExecErr(`INSERT INTO `+s.qualified(InterpretationLibrariesTable)+ + ` (project, id, head_revision_id, head_digest, created_at, updated_at)`+ + ` VALUES ($1, $2, $3, $4, $5, $5)`, + project, id, head, digest, interpBaseTime) +} + +func TestInterpretationSchemaConstraintsRejectInvalidRows(t *testing.T) { + s := newInterpretationService(t) + p := "probe_constraints" + goodID := "interpretation_" + strings.Repeat("a", 64) + goodDigest := "sha256:" + strings.Repeat("b", 64) + parentID := "interpretation_" + strings.Repeat("c", 64) + parentDigestValue := "sha256:" + strings.Repeat("d", 64) + parent := parentID + parentDigest := parentDigestValue + + if err := s.insertRevisionDirectErr(p, parentID, "lib-a", nil, nil, parentDigestValue, "a", "e", "{}"); err != nil { + s.fail("legal control parent", err) + } + if err := s.insertRevisionDirectErr(p, goodID, "lib-a", &parent, &parentDigest, goodDigest, "a", "e", "{}"); err != nil { + s.fail("legal control same-library child", err) + } + if err := s.insertLibraryDirectErr(p, "lib-a", goodID, goodDigest); err != nil { + s.fail("legal control library", err) + } + if err := s.insertRevisionDirectErr(p, "interpretation_"+strings.Repeat("e", 64), "lib-b", nil, nil, "sha256:"+strings.Repeat("f", 64), "a", "e", "{}"); err != nil { + s.fail("legal control sibling library revision", err) + } + + if err := s.insertRevisionDirectErr(p, "interpretation_short", "lib-a", nil, nil, goodDigest, "a", "e", "{}"); true { + requireInterpPgErr(t, err, pgstate04CheckViolation, "interpretation_revisions_id_shape", "bad revision id shape") + } + if err := s.insertRevisionDirectErr(p, "interpretation_"+strings.Repeat("9", 64), "lib-a", nil, nil, "sha256:XYZ", "a", "e", "{}"); true { + requireInterpPgErr(t, err, pgstate04CheckViolation, "interpretation_revisions_content_digest_shape", "bad content digest shape") + } + if err := s.insertRevisionDirectErr(p, "interpretation_"+strings.Repeat("8", 64), "lib-a", &parent, nil, goodDigest, "a", "e", "{}"); true { + requireInterpPgErr(t, err, pgstate04CheckViolation, "interpretation_revisions_parent_pair", "parent without digest") + } + if err := s.insertRevisionDirectErr(p, "interpretation_"+strings.Repeat("7", 64), "lib-a", nil, &parentDigest, goodDigest, "a", "e", "{}"); true { + requireInterpPgErr(t, err, pgstate04CheckViolation, "interpretation_revisions_parent_pair", "digest without parent") + } + if err := s.insertRevisionDirectErr(p, "interpretation_"+strings.Repeat("6", 64), "lib-a", ptr("interpretation_short"), &parentDigest, goodDigest, "a", "e", "{}"); true { + requireInterpPgErr(t, err, pgstate04CheckViolation, "interpretation_revisions_parent_revision_shape", "bad parent id shape") + } + if err := s.insertRevisionDirectErr(p, "interpretation_"+strings.Repeat("5", 64), "lib-a", &parent, ptr("sha256:nope"), goodDigest, "a", "e", "{}"); true { + requireInterpPgErr(t, err, pgstate04CheckViolation, "interpretation_revisions_parent_digest_shape", "bad parent digest shape") + } + if err := s.insertRevisionDirectErr(p, "interpretation_"+strings.Repeat("4", 64), "lib-a", nil, nil, goodDigest, "", "e", "{}"); true { + requireInterpPgErr(t, err, pgstate04CheckViolation, "interpretation_revisions_author_non_empty", "empty author") + } + if err := s.insertRevisionDirectErr(p, "interpretation_"+strings.Repeat("3", 64), "lib-a", nil, nil, goodDigest, "a", "", "{}"); true { + requireInterpPgErr(t, err, pgstate04CheckViolation, "interpretation_revisions_explanation_non_empty", "empty explanation") + } + if err := s.insertRevisionDirectErr(p, "interpretation_"+strings.Repeat("2", 64), "lib-a", nil, nil, goodDigest, "a", "e", "[1,2]"); true { + requireInterpPgErr(t, err, pgstate04CheckViolation, "interpretation_revisions_payload_object", "non-object payload") + } + if err := s.insertRevisionDirectErr(p, "interpretation_"+strings.Repeat("1", 64), "1bad-library", nil, nil, goodDigest, "a", "e", "{}"); true { + requireInterpPgErr(t, err, pgstate04CheckViolation, "interpretation_revisions_library_id_shape", "bad library id shape") + } + + if err := s.insertRevisionDirectErr(p, "interpretation_"+strings.Repeat("f", 63)+"e", "lib-a", + ptr("interpretation_"+strings.Repeat("0", 64)), &parentDigest, goodDigest, "a", "e", "{}"); true { + requireInterpPgErr(t, err, pgstate04ForeignKey, interpretationRevisionParentFKConstraint, "parent does not exist") + } + if err := s.insertRevisionDirectErr(p, "interpretation_"+strings.Repeat("f", 63)+"d", "lib-a", + ptr("interpretation_"+strings.Repeat("e", 64)), &parentDigest, goodDigest, "a", "e", "{}"); true { + requireInterpPgErr(t, err, pgstate04ForeignKey, interpretationRevisionParentFKConstraint, "parent lives in another library") + } + + if err := s.insertLibraryDirectErr(p, "lib-c", "interpretation_"+strings.Repeat("0", 64), goodDigest); true { + requireInterpPgErr(t, err, pgstate04ForeignKey, interpretationLibraryHeadFKConstraint, "head does not exist") + } + if err := s.insertLibraryDirectErr(p, "lib-c", "interpretation_"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("f", 64)); true { + requireInterpPgErr(t, err, pgstate04ForeignKey, interpretationLibraryHeadFKConstraint, "head belongs to another library") + } + if err := s.insertLibraryDirectErr(p, "lib-b", "interpretation_"+strings.Repeat("e", 64), "sha256:bad"); true { + requireInterpPgErr(t, err, pgstate04CheckViolation, "interpretation_libraries_head_digest_shape", "bad head digest shape") + } + + if err := s.insertRevisionDirectErr(p, goodID, "lib-z", nil, nil, "sha256:"+strings.Repeat("c", 64), "z", "z", "{}"); true { + requireInterpPgErr(t, err, pgstate04UniqueViolation, interpretationRevisionPKConstraint, "duplicate revision identity") + } + if err := s.insertLibraryDirectErr(p, "lib-a", "interpretation_"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("f", 64)); true { + requireInterpPgErr(t, err, pgstate04UniqueViolation, interpretationLibraryPKConstraint, "duplicate library identity") + } + + if n := s.revisionCount(t, p, "lib-a"); n != 2 { + t.Fatalf("probe lib-a revisions = %d, want the 2 legal controls", n) + } + if n := s.revisionCount(t, p, "lib-b"); n != 1 { + t.Fatalf("probe lib-b revisions = %d, want the 1 legal control", n) + } + if n := s.libraryCount(t, p, "lib-a"); n != 1 { + t.Fatalf("probe lib-a libraries = %d, want the 1 legal control", n) + } + if n := s.libraryCount(t, p, "lib-c"); n != 0 { + t.Fatalf("refused head-FK inserts created %d lib-c libraries", n) + } +} + +func TestInterpretationCASRefusalsWriteNothingAndFailClosed(t *testing.T) { + s := newInterpretationService(t) + root := createRevision(t, s, s.storeA, interpRevision("proj_cas", "library-a", "author-a", "root", interpGoldenRulePath), nil) + + mismatchChild := interpChild(*root, "mismatch", "code") + if _, err := s.storeA.CreateInterpretationRevision(s.ctx, mismatchChild, nil); !errors.Is(err, explorer.ErrInterpretationParentConflict) { + t.Fatalf("claim/content mismatch error = %v, want ErrInterpretationParentConflict", err) + } + if _, err := s.storeA.CreateInterpretationRevision(s.ctx, interpRevision("proj_cas_mismatch", "library-a", "author-a", "root", interpGoldenRulePath), idRef(root.ID)); !errors.Is(err, explorer.ErrInterpretationParentConflict) { + t.Fatalf("root claiming a parent error = %v, want ErrInterpretationParentConflict", err) + } + if n := s.revisionCount(t, "proj_cas_mismatch", "library-a"); n != 0 { + t.Fatalf("preflight refusals persisted %d revisions", n) + } + if n := s.libraryCount(t, "proj_cas_mismatch", "library-a"); n != 0 { + t.Fatalf("preflight refusals persisted %d libraries", n) + } + + headNow := s.mustWitnessLibrary(t, "proj_cas", "library-a") + orphanRoot := interpRevision("proj_cas", "library-a", "author-a", "orphan-fork", "code") + if _, err := s.storeA.CreateInterpretationRevision(s.ctx, orphanRoot, idRef(explorer.InterpretationRevisionID(headNow.headRevision))); !errors.Is(err, explorer.ErrInterpretationParentConflict) { + t.Fatalf("parentless content claiming the head error = %v, want ErrInterpretationParentConflict", err) + } + if l := s.mustWitnessLibrary(t, "proj_cas", "library-a"); l != headNow { + t.Fatalf("claim/content mismatch forked the head:\n got %+v\nwant %+v", l, headNow) + } + if n := s.revisionCount(t, "proj_cas", "library-a"); n != 1 { + t.Fatalf("claim/content mismatch persisted a row for the orphan fork: %d revisions in proj_cas", n) + } + + drift := interpRevision("proj_cas", "library-a", "author-a", "drift-child", "code") + fakeParentID := root.ID + fakeDigest := explorer.InterpretationContentDigest("sha256:" + strings.Repeat("e", 64)) + drift.ParentRevisionID = &fakeParentID + drift.ParentDigest = &fakeDigest + if _, err := s.storeA.CreateInterpretationRevision(s.ctx, drift, idRef(fakeParentID)); !errors.Is(err, explorer.ErrInterpretationParentConflict) { + t.Fatalf("parent digest drift error = %v, want ErrInterpretationParentConflict", err) + } + if n := s.revisionCount(t, "proj_cas", "library-a"); n != 1 { + t.Fatalf("digest-drift refusal persisted revisions: %d", n) + } + + child := createRevision(t, s, s.storeB, interpChild(*root, "won", "code"), idRef(root.ID)) + headBefore := s.mustWitnessLibrary(t, "proj_cas", "library-a") + stale := interpChild(*root, "late", "value") + if _, err := s.storeA.CreateInterpretationRevision(s.ctx, stale, idRef(root.ID)); !errors.Is(err, explorer.ErrInterpretationParentConflict) { + t.Fatalf("stale-head create error = %v, want ErrInterpretationParentConflict", err) + } + headAfter := s.mustWitnessLibrary(t, "proj_cas", "library-a") + if headAfter != headBefore { + t.Fatalf("stale-head refusal moved the head:\n got %+v\nwant %+v", headAfter, headBefore) + } + if n := s.revisionCount(t, "proj_cas", "library-a"); n != 2 { + t.Fatalf("stale-head refusal persisted a row: %d revisions", n) + } + late := interpChild(*child, "late-but-current", "value") + createRevision(t, s, s.storeA, late, idRef(child.ID)) + + orphan := interpRevision("proj_cas", "library-a", "author-a", "orphan", "code") + missingID := explorer.InterpretationRevisionID("interpretation_" + strings.Repeat("0", 64)) + missingDigest := explorer.InterpretationContentDigest("sha256:" + strings.Repeat("0", 64)) + orphan.ParentRevisionID = &missingID + orphan.ParentDigest = &missingDigest + if _, err := s.storeA.CreateInterpretationRevision(s.ctx, orphan, idRef(missingID)); !errors.Is(err, explorer.ErrInterpretationParentConflict) { + t.Fatalf("missing parent error = %v, want ErrInterpretationParentConflict", err) + } + if n := s.revisionCount(t, "proj_cas", "library-a"); n != 3 { + t.Fatalf("missing-parent refusal persisted a row: %d revisions", n) + } + + badAuthor := interpRevision("proj_cas_bad", "library-a", " ", "reason", "code") + if _, err := s.storeA.CreateInterpretationRevision(s.ctx, badAuthor, nil); err == nil || errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("blank author error = %v, want a typed validation error", err) + } + if _, err := s.storeA.CreateInterpretationRevision(s.ctx, interpRevision("proj_cas_bad", "", "author", "reason", "code"), nil); err == nil { + t.Fatal("blank library create returned success") + } + if n := s.revisionCount(t, "proj_cas_bad", "library-a"); n != 0 { + t.Fatalf("invalid inputs persisted %d rows", n) + } +} + +func TestInterpretationStoresFailClosedWithoutTheirTables(t *testing.T) { + s := newInterpretationService(t) + + owner, err := OwnerMigrations() + if err != nil { + s.fail("owner migrations", err) + } + fixtureApplied := false + for _, rec := range s.ledgerAfter { + if rec.Version == 7 && rec.Name == pgstate04FixtureName { + fixtureApplied = true + } + } + if !fixtureApplied { + t.Fatalf("precondition: owned schema ledger lacks the version-7 %s fixture: %+v", pgstate04FixtureName, s.ledgerAfter) + } + + unmigSchema := s.schema + "_unmig" + if err := interpExecOn(s.ctx, s.witness, `CREATE SCHEMA "`+unmigSchema+`"`); err != nil { + s.fail("create unmigrated schema", err) + } + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + _ = interpExecOn(cleanupCtx, s.witness, `DROP SCHEMA IF EXISTS "`+unmigSchema+`" CASCADE`) + }) + interpretationFree := owner[:0:0] + for _, m := range owner { + if m.Version != 7 { + interpretationFree = append(interpretationFree, m) + } + } + if len(interpretationFree) != len(owner)-1 { + t.Fatalf("owner history lacks the version-7 interpretation-library migration to exclude: %+v", owner) + } + ownerRunner, err := pgstore.NewMigrationRunner(s.witness, unmigSchema, interpretationFree) + if err != nil { + s.fail("build owner-minus-interpretation runner", err) + } + if _, err := ownerRunner.Migrate(s.ctx); err != nil { + s.fail("migrate owner-only schema", err) + } + + store, err := NewInterpretationStore(s.clientA, unmigSchema) + if err != nil { + s.fail("build unmigrated store", err) + } + undefinedTable := func(step string, err error) { + t.Helper() + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "42P01" { + t.Fatalf("%s error = %v, want fail-closed 42P01 undefined_table", step, s.scrub(err)) + } + } + _, err = store.GetInterpretationRevision(s.ctx, "proj_x", explorer.InterpretationRevisionID("interpretation_"+strings.Repeat("0", 64))) + undefinedTable("GetInterpretationRevision", err) + _, err = store.ListInterpretationLibraries(s.ctx, "proj_x") + undefinedTable("ListInterpretationLibraries", err) + _, err = store.CreateInterpretationRevision(s.ctx, interpRevision("proj_x", "library-a", "author", "reason", "code"), nil) + undefinedTable("CreateInterpretationRevision", err) +} + +func TestZZZeroLeftoverPgStateInterpretationFixtures(t *testing.T) { + raw := testPostgresURL(t) + pgstate04OwnedMu.Lock() + owned := make(map[string]string, len(pgstate04OwnedDBs)) + for db, schema := range pgstate04OwnedDBs { + owned[db] = schema + } + pgstate04OwnedMu.Unlock() + if len(owned) == 0 { + t.Skip("this process created no pgstate04_it_* fixtures") + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service for audit: %v", err) + } + defer admin.Close() + + for db, schema := range owned { + if !strings.HasPrefix(db, "pgstate04_it_") { + t.Fatalf("registry holds foreign database %q", db) + } + var count int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Errorf("audit owned database %s: %v", db, err) + continue + } + if count != 0 { + t.Errorf("leftover owned database %s after all cleanups", db) + if schema != "" { + u, err := parseDraftURL(raw) + if err == nil { + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + db + if probe, err := pgxpool.New(ctx, u.String()); err == nil { + var schemaCount int + if err := probe.QueryRow(ctx, "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1", schema).Scan(&schemaCount); err != nil { + t.Errorf("audit owned schema %s in %s: %v", schema, db, err) + } else if schemaCount != 0 { + t.Errorf("leftover owned schema %s in leftover database %s", schema, db) + } + probe.Close() + } + } + } + } + } + + var prefixLeftovers int + if err := admin.QueryRow(ctx, + `SELECT count(*) FROM pg_database WHERE datname LIKE 'pgstate04\_it\_%' ESCAPE '\'`).Scan(&prefixLeftovers); err != nil { + t.Errorf("audit own prefix: %v", err) + } else if prefixLeftovers != 0 { + var names []string + rows, err := admin.Query(ctx, `SELECT datname FROM pg_database WHERE datname LIKE 'pgstate04\_it\_%' ESCAPE '\'`) + if err == nil { + for rows.Next() { + var n string + if err := rows.Scan(&n); err == nil { + names = append(names, n) + } + } + rows.Close() + } + t.Errorf("%d pgstate04_it_* databases left behind: %v", prefixLeftovers, names) + } +} diff --git a/internal/backends/postgres/internal/explorer/migrations/000001_ledger.sql b/internal/backends/postgres/internal/explorer/migrations/000001_ledger.sql new file mode 100644 index 000000000..c3da01ec3 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/migrations/000001_ledger.sql @@ -0,0 +1,20 @@ +-- pg-state-foundation-v1: create the per-schema migration ledger. +-- +-- This mirrors the pg-store-02 ledger contract (internal/store/postgres/ +-- migrations/000001_ledger.sql): version, name, and checksum only, never the +-- SQL body, with the same CHECK shapes. The MigrationRunner reads this table +-- from the target schema, so a schema whose history starts here bootstraps +-- its own ledger audited by the same runner. Unlike the pg-store-02 copy this +-- body is written CREATE TABLE IF NOT EXISTS so a crash in the narrow +-- commit-before-record window converges on retry, as the runner documentation +-- requires of feature migrations. It never drops or truncates anything. +CREATE TABLE IF NOT EXISTS schema_migrations ( + version bigint NOT NULL, + name text NOT NULL, + checksum text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT schema_migrations_pkey PRIMARY KEY (version), + CONSTRAINT schema_migrations_version_positive CHECK (version > 0), + CONSTRAINT schema_migrations_name_shape CHECK (name ~ '^[a-z][a-z0-9_]*$'), + CONSTRAINT schema_migrations_checksum_hex CHECK (checksum ~ '^[0-9a-f]{64}$') +); diff --git a/internal/backends/postgres/internal/explorer/migrations/000002_explorer_owner.sql b/internal/backends/postgres/internal/explorer/migrations/000002_explorer_owner.sql new file mode 100644 index 000000000..2ba9d1e87 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/migrations/000002_explorer_owner.sql @@ -0,0 +1,58 @@ +-- pg-state-foundation-v1: Explorer owner table (the first concrete control +-- schema under the pg-store-02 migration runner). +-- +-- One row is the owner record for exactly one Explorer, identified by the +-- unique (project, explorer_id) pair. Identity columns carry COLLATE "C" so +-- uniqueness and List's ORDER BY explorer_id use exact byte order -- the same +-- deterministic order Arango's byte-order SORT gives -- independent of the +-- database's default collation. +-- +-- Column policy: +-- - Queried/authoritative fields are typed columns (text, bigint, +-- timestamptz). The two json.RawMessage payloads are jsonb. +-- - Go strings have no missing/empty distinction, so optional text columns +-- are NOT NULL DEFAULT '' and round-trip the Go zero value exactly. +-- - draft_config and last_authoring_command_results are nullable jsonb and +-- preserve the three Go states of json.RawMessage losslessly: missing +-- (nil) is SQL NULL, JSON-null is the jsonb value 'null' (IS NULL false), +-- and a document is the document. Nothing is coerced across those +-- states; tests assert each with literal rows. +-- - The draft payload is stored lossless: jsonb keeps every member of the +-- canonical ExplorerConfigV2 packet (key order/whitespace are +-- normalized by the jsonb type itself, never a member or value). +-- Caveat, true of jsonb itself: duplicate object keys collapse +-- last-wins and out-of-range numeric literals fail at parse. Authoring +-- packets are canonically encoded (encoding/json ordering) before they +-- reach this column, and non-JSON input is rejected by PostgreSQL +-- (22P02), pinned by the owner tests. +-- - explorers_management_mode_known accepts only the ManagementMode domain +-- ('REPOSITORY'/'INTERACTIVE'); explorers_draft_version_non_negative +-- keeps the CAS counter in its meaningful range. Constraint violations +-- surface as PostgreSQL CHECK violations (23514), classified separately +-- from duplicate-owner conflicts. +-- - This migration never drops or truncates: rerunning bootstrap on an +-- existing schema is a no-op that preserves owner state. +CREATE TABLE IF NOT EXISTS explorers ( + project text NOT NULL COLLATE "C", + explorer_id text NOT NULL COLLATE "C", + title text NOT NULL, + management_mode text NOT NULL, + draft_config jsonb, + draft_version bigint NOT NULL, + draft_digest text NOT NULL DEFAULT '', + draft_snapshot_token text NOT NULL DEFAULT '', + draft_source_generation text NOT NULL DEFAULT '', + draft_authorization_scope_digest text NOT NULL DEFAULT '', + previous_draft_revision_id text NOT NULL DEFAULT '', + last_authoring_command_id text NOT NULL DEFAULT '', + last_authoring_command_digest text NOT NULL DEFAULT '', + last_authoring_command_results jsonb, + active_revision_id text NOT NULL DEFAULT '', + updated_by text NOT NULL DEFAULT '', + updated_at timestamptz NOT NULL, + CONSTRAINT explorers_pkey PRIMARY KEY (project, explorer_id), + CONSTRAINT explorers_management_mode_known + CHECK (management_mode IN ('REPOSITORY', 'INTERACTIVE')), + CONSTRAINT explorers_draft_version_non_negative + CHECK (draft_version >= 0) +); diff --git a/internal/backends/postgres/internal/explorer/migrations/000003_explorer_draft_revision.sql b/internal/backends/postgres/internal/explorer/migrations/000003_explorer_draft_revision.sql new file mode 100644 index 000000000..97150740d --- /dev/null +++ b/internal/backends/postgres/internal/explorer/migrations/000003_explorer_draft_revision.sql @@ -0,0 +1,57 @@ +-- pg-state-explorer-draft-cas-v1: immutable exact-preimage history for +-- Explorer draft saves. +-- +-- One row is the lossless snapshot of the owner state that a SaveDraft CAS +-- winner replaced: the preimage's draft version, digest, config packet, +-- title, source context, author, and timestamp. Together with the owner row +-- it forms the Undo chain the restore command reads (previous_draft_revision +-- _id on explorers points here). SaveDraft inserts this row and updates the +-- owner inside ONE transaction, so a failed or aborted CAS leaves neither a +-- head change nor a history row (pinned by TestDraftCASRollsBackHistory* on +-- real PG18). +-- +-- Identity policy (parity with the Arango adapter's draftRevisionKey): +-- * The version identity (project, explorer_id, draft_version) is the +-- primary key: exactly one exact preimage may ever exist for one +-- Explorer draft version. Competing same-version writers serialize on +-- the unique index inside PostgreSQL; the loser receives a 23505 that +-- SaveDraft maps to explorer.ErrDraftConflict retaining the +-- *pgconn.PgError cause. The row is append-only: nothing here is ever +-- updated, and the CAS flow never deletes history. +-- * The opaque id is the derived draft_revision_<64 hex> key, unique per +-- (project, explorer_id) so restore lookups resolve it through the +-- scoped unique index and a foreign project/Explorer never sees it. +-- explorers_draft_revision_id_shape keeps ids in the derived shape. +-- * Identity columns carry COLLATE "C" for exact byte order, matching the +-- owner table and Arango's byte-order comparisons. +-- +-- Column policy mirrors 000002_explorer_owner.sql: Go strings have no +-- missing/empty distinction, so optional text columns are NOT NULL DEFAULT +-- ''; draft_config is nullable jsonb preserving the three json.RawMessage +-- states losslessly (missing is SQL NULL, JSON-null is the jsonb value +-- 'null', a document is the document). draft_version >= 0 keeps preimages +-- in the CAS counter's meaningful range (version 0 is the legitimate empty +-- first draft). This migration never drops or truncates: the runner may +-- rerun a migration that committed before its ledger row was written. +CREATE TABLE IF NOT EXISTS explorers_draft_revisions ( + project text NOT NULL COLLATE "C", + explorer_id text NOT NULL COLLATE "C", + draft_version bigint NOT NULL, + id text NOT NULL COLLATE "C", + title text NOT NULL, + draft_digest text NOT NULL DEFAULT '', + draft_config jsonb, + snapshot_token text NOT NULL DEFAULT '', + source_generation text NOT NULL DEFAULT '', + authorization_scope_digest text NOT NULL DEFAULT '', + updated_by text NOT NULL DEFAULT '', + updated_at timestamptz NOT NULL, + CONSTRAINT explorers_draft_revisions_identity_pk + PRIMARY KEY (project, explorer_id, draft_version), + CONSTRAINT explorers_draft_revisions_id_ux + UNIQUE (project, explorer_id, id), + CONSTRAINT explorers_draft_revisions_version_non_negative + CHECK (draft_version >= 0), + CONSTRAINT explorers_draft_revisions_id_shape + CHECK (id ~ '^draft_revision_[0-9a-f]{64}$') +); diff --git a/internal/backends/postgres/internal/explorer/migrations/000004_capability_artifacts.sql b/internal/backends/postgres/internal/explorer/migrations/000004_capability_artifacts.sql new file mode 100644 index 000000000..cd42f0cf1 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/migrations/000004_capability_artifacts.sql @@ -0,0 +1,151 @@ +-- pg-state-immutable-capability-receipts-v1: immutable capability snapshots +-- and table-shape capability receipts for the Explorer control plane. +-- +-- Both tables are fully bound, create-once artifacts. The jsonb payload is +-- authoritative for reads (nanosecond-free canonical snapshot documents and +-- receipt objects with their missing-versus-JSON-null member shapes +-- round-trip untouched); the scan columns mirror the payload for retrieval +-- scopes, constraints, and ordered scans only. Every statement in the +-- adapter writes both from the same validated value, and the adapter +-- re-checks payload-versus-mirror agreement on every read. No row is ever +-- UPDATEd or DELETEd: content identity (token / content digest) is the key, +-- so a changed payload is a new artifact or a typed collision, never a +-- mutation. The version continues the Explorer control-plane fixture history +-- after the 000003 draft-revision reservation (pg-state-explorer-draft-cas) +-- and the DDL is idempotent because the pg-store-02 runner may retry a +-- migration that committed before its ledger row was written. +-- +-- Capability snapshot identity: +-- * token is the content address ("sha256:" + 64 lowercase hex of the +-- canonical snapshot payload). Put is insert-only, so concurrent +-- duplicate puts converge on the existing row; a stored payload that no +-- longer hashes to its token is a typed collision, never a rewrite. +-- * The identity columns mirror SnapshotIdentity exactly (empty string +-- stands in for the omitted cacheVersion/shapeDigest, matching the +-- Arango adapter's NOT_NULL(identity.cacheVersion, "") filter). The +-- status CHECK closes the domain and the READY guard keeps the SQL +-- mirror consistent with validateSnapshot's READY completeness rule; +-- the payload remains the authority that the adapter re-validates. +-- +-- Table-shape receipt identity: +-- * id is the content-derived opaque artifact ID with a kind-bound prefix +-- (tsc_ catalog, tsd_ category scan, tsr_ resolution), so the CHECK +-- below makes a kind/prefix mismatch impossible at the database, not +-- just in the adapter. content_digest is the "sha256:" digest over the +-- receipt's canonical identity bytes. +-- * Every binding column is NOT NULL and non-empty and draft_version is +-- at least 1, mirroring Binding.Validate; retrieval is scoped by the +-- full binding, so an opaque ID is never trusted without its current +-- project/Explorer/output/snapshot/source binding. parent_catalog_id is +-- empty exactly for CATALOG rows. +-- +-- Immutability is enforced here, not only in Go: AFTER the tables, a +-- shared guard function plus BEFORE UPDATE OR DELETE triggers refuse any +-- mutation of an artifact row with SQLSTATE 45501 before it can run. The +-- suite proves the guards are not vacuous by dropping each trigger, +-- repeating the violating statement (which must then succeed), and +-- restoring the trigger. + +CREATE TABLE IF NOT EXISTS capability_snapshots ( + token text PRIMARY KEY + CONSTRAINT capability_snapshots_token_shape CHECK (token ~ '^sha256:[0-9a-f]{64}$'), + project text NOT NULL DEFAULT '', + generation text NOT NULL DEFAULT '', + authorization_scope_digest text NOT NULL DEFAULT '', + cache_version text NOT NULL DEFAULT '', + schema_digest text NOT NULL DEFAULT '', + resource_inventory_digest text NOT NULL DEFAULT '', + relationship_digest text NOT NULL DEFAULT '', + field_digest text NOT NULL DEFAULT '', + shape_digest text NOT NULL DEFAULT '', + protocol_version text NOT NULL DEFAULT '', + compiler_version text NOT NULL DEFAULT '', + traversal_policy_version text NOT NULL DEFAULT '', + projection_policy_version text NOT NULL DEFAULT '', + status text NOT NULL + CONSTRAINT capability_snapshots_status_known CHECK (status IN ('BUILDING', 'READY', 'FAILED')), + complete boolean NOT NULL, + truncated boolean NOT NULL, + payload jsonb NOT NULL + CONSTRAINT capability_snapshots_payload_object CHECK (jsonb_typeof(payload) = 'object'), + CONSTRAINT capability_snapshots_ready_is_usable + CHECK (status <> 'READY' OR (complete AND NOT truncated)) +); + +-- Retrieval mirrors the Arango collection indexes: identity lookup and the +-- status-bounded current-identity lookup. +CREATE INDEX IF NOT EXISTS capability_snapshots_identity_ix + ON capability_snapshots (project, generation, authorization_scope_digest, protocol_version, compiler_version); +CREATE INDEX IF NOT EXISTS capability_snapshots_status_ix + ON capability_snapshots (project, generation, authorization_scope_digest, status); + +CREATE TABLE IF NOT EXISTS table_shape_capabilities ( + id text PRIMARY KEY, + kind text NOT NULL + CONSTRAINT table_shape_capabilities_kind_known CHECK (kind IN ('CATALOG', 'CATEGORY_SCAN', 'RESOLUTION')), + parent_catalog_id text NOT NULL DEFAULT '', + project text NOT NULL, + explorer_id text NOT NULL, + output_id text NOT NULL, + snapshot_token text NOT NULL, + authorization_scope text NOT NULL, + source_generation text NOT NULL, + draft_version bigint NOT NULL, + draft_digest text NOT NULL, + base_document_digest text NOT NULL, + base_compilation_receipt_id text NOT NULL, + output_fingerprint text NOT NULL, + compiler_schema_digest text NOT NULL, + content_digest text NOT NULL + CONSTRAINT table_shape_capabilities_digest_shape CHECK (content_digest ~ '^sha256:[0-9a-f]{64}$'), + created_at text NOT NULL DEFAULT '', + payload jsonb NOT NULL + CONSTRAINT table_shape_capabilities_payload_object CHECK (jsonb_typeof(payload) = 'object'), + CONSTRAINT table_shape_capabilities_id_kind_prefix CHECK ( + (kind = 'CATALOG' AND id ~ '^tsc_[0-9a-f]{32}$') + OR (kind = 'CATEGORY_SCAN' AND id ~ '^tsd_[0-9a-f]{32}$') + OR (kind = 'RESOLUTION' AND id ~ '^tsr_[0-9a-f]{32}$') + ), + CONSTRAINT table_shape_capabilities_parent_bound CHECK ( + (kind = 'CATALOG' AND parent_catalog_id = '') + OR (kind <> 'CATALOG' AND parent_catalog_id <> '') + ), + CONSTRAINT table_shape_capabilities_binding_complete CHECK ( + project <> '' AND explorer_id <> '' AND output_id <> '' + AND snapshot_token <> '' AND authorization_scope <> '' AND source_generation <> '' + AND draft_version >= 1 + AND draft_digest <> '' AND base_document_digest <> '' + AND base_compilation_receipt_id <> '' AND output_fingerprint <> '' + AND compiler_schema_digest <> '' + ) +); + +-- Retrieval mirrors the Arango collection indexes: the fully bound opaque-ID +-- lookup and the parent-scoped kind scan. +CREATE INDEX IF NOT EXISTS table_shape_capabilities_binding_ix + ON table_shape_capabilities (project, explorer_id, output_id, snapshot_token, kind, id); +CREATE INDEX IF NOT EXISTS table_shape_capabilities_parent_ix + ON table_shape_capabilities (project, explorer_id, output_id, parent_catalog_id, kind); + +-- The immutability guard: artifact rows are create-once. Any UPDATE or +-- DELETE -- from any session, including one holding the table owner's +-- rights -- aborts with SQLSTATE 45501 before touching a page. The +-- adapter never issues UPDATE/DELETE against these tables; the trigger is +-- what makes that a database-enforced property rather than a convention. +CREATE OR REPLACE FUNCTION loom_capability_artifact_immutable() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION 'loom: capability artifact rows are immutable (create-once)' + USING ERRCODE = '45501'; +END; +$$; + +DROP TRIGGER IF EXISTS capability_snapshots_immutable ON capability_snapshots; +CREATE TRIGGER capability_snapshots_immutable + BEFORE UPDATE OR DELETE ON capability_snapshots + FOR EACH ROW EXECUTE FUNCTION loom_capability_artifact_immutable(); + +DROP TRIGGER IF EXISTS table_shape_capabilities_immutable ON table_shape_capabilities; +CREATE TRIGGER table_shape_capabilities_immutable + BEFORE UPDATE OR DELETE ON table_shape_capabilities + FOR EACH ROW EXECUTE FUNCTION loom_capability_artifact_immutable(); diff --git a/internal/backends/postgres/internal/explorer/migrations/000005_explorer_compilation_receipt.sql b/internal/backends/postgres/internal/explorer/migrations/000005_explorer_compilation_receipt.sql new file mode 100644 index 000000000..1b9e196ea --- /dev/null +++ b/internal/backends/postgres/internal/explorer/migrations/000005_explorer_compilation_receipt.sql @@ -0,0 +1,72 @@ +-- pg-state-receipt-revision-immutable-v1: compilation receipt table. +-- +-- One row is one durable, content-addressed CompilationReceipt for exactly +-- one Explorer, identified by the unique (project, explorer_id, id) triple. +-- Receipts are insert-once: the Arango adapter's insert-once + scoped-reread +-- contract (internal/explorer/arango/receipts.go) becomes a relational +-- insert-once claim, and the DB itself refuses every later mutation. +-- +-- Column policy: +-- - Identity columns carry COLLATE "C" so uniqueness uses exact byte order, +-- matching the owner table and Arango's byte-order _key semantics. +-- - The digest/generation/recipe-identity fields the adapter re-checks on +-- every conflict (intent_digest, snapshot_token, source_generation, +-- recipe_digest, resolved_recipe_digest, compilation_key) plus the +-- contract columns are projected into typed columns so collision +-- reports name real columns, and every one of them is frozen below. +-- - doc is the full serialized CompilationReceipt as jsonb: lossless for +-- the embedded json.RawMessage tri-state (missing key, the jsonb value +-- 'null', or a document), the canonical recipe Bundle, and the exact +-- RFC3339Nano createdAt string. jsonb normalizes key order/whitespace +-- only (never a member or value); the canonical ReceiptID digest -- not +-- jsonb byte order -- is the content check. +-- - compilation_receipts_doc_object refuses a scalar/array doc (23514); +-- compilation_receipts_format_version_non_negative keeps the format +-- version in range. +-- +-- Immutability is ENFORCED, not documented: the row-level BEFORE UPDATE OR +-- DELETE trigger loom_explorer_receipts_frozen raises a check_violation +-- (SQLSTATE 23514) for every mutation attempt, so digests, generation, +-- recipe content inside doc, and every other column cannot be changed or +-- deleted by any SQL, including direct psql. Tests prove the guard +-- non-vacuously: with the trigger dropped the same violating write succeeds, +-- and recreating it from this migration's text blocks it again. +-- +-- This migration never drops or truncates state: rerunning it on an existing +-- schema (crash in the commit-before-record window) is a no-op convergence. + +CREATE TABLE IF NOT EXISTS compilation_receipts ( + project text NOT NULL COLLATE "C", + explorer_id text NOT NULL COLLATE "C", + id text NOT NULL COLLATE "C", + receipt_format_version bigint NOT NULL, + compiler_contract_version text NOT NULL DEFAULT '', + compilation_key text NOT NULL DEFAULT '', + intent_digest text NOT NULL DEFAULT '', + snapshot_token text NOT NULL DEFAULT '', + source_generation text NOT NULL DEFAULT '', + recipe_digest text NOT NULL DEFAULT '', + resolved_recipe_digest text NOT NULL DEFAULT '', + doc jsonb NOT NULL, + CONSTRAINT compilation_receipts_pkey PRIMARY KEY (project, explorer_id, id), + CONSTRAINT compilation_receipts_format_version_non_negative + CHECK (receipt_format_version >= 0), + CONSTRAINT compilation_receipts_doc_object + CHECK (jsonb_typeof(doc) = 'object') +); + +CREATE OR REPLACE FUNCTION loom_explorer_receipts_frozen() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION + 'compilation_receipts is insert-once: % of receipt %/%/% is refused', + TG_OP, OLD.project, OLD.explorer_id, OLD.id + USING ERRCODE = '23514'; +END; +$$; + +DROP TRIGGER IF EXISTS compilation_receipts_frozen ON compilation_receipts; +CREATE TRIGGER compilation_receipts_frozen + BEFORE UPDATE OR DELETE ON compilation_receipts + FOR EACH ROW + EXECUTE FUNCTION loom_explorer_receipts_frozen(); diff --git a/internal/backends/postgres/internal/explorer/migrations/000006_explorer_revision.sql b/internal/backends/postgres/internal/explorer/migrations/000006_explorer_revision.sql new file mode 100644 index 000000000..2755b4b5f --- /dev/null +++ b/internal/backends/postgres/internal/explorer/migrations/000006_explorer_revision.sql @@ -0,0 +1,171 @@ +-- pg-state-receipt-revision-immutable-v1: immutable Explorer revision table. +-- +-- One row is one Explorer revision, identified by its globally unique id. +-- A revision's frozen content (definition, recipe, digests, source +-- generation, materialization mappings, emitted columns, dataset metadata, +-- quality reports, authoring payloads) can never change after InsertRevision +-- -- the exact immutable boundary the Arango adapter enforces only by +-- refusing an overwrite in Go (internal/explorer/arango/revisions.go +-- sameRevisionContent). Here the DB enforces it too: the row-level BEFORE +-- UPDATE OR DELETE trigger loom_explorer_revision_guard refuses deletion +-- outright (insert-once against direct SQL -- a DELETE plus re-INSERT would +-- otherwise fire no guard on a fresh row) and raises a check_violation +-- (SQLSTATE 23514) whenever a frozen column changes, so no direct SQL can +-- rewrite or remove frozen content. Tests prove the guard non-vacuously: +-- with the trigger dropped the same violating UPDATE and a DELETE succeed, +-- and recreating it from this migration's text blocks both again. +-- +-- Lifecycle transitions are a guarded monotone state machine, also enforced +-- here: only READY->ACTIVE, READY->FAILED, ACTIVE->SUPERSEDED and +-- ACTIVE->FAILED are legal, each in one statement. Re-setting a status +-- (the double transition, e.g. FAILED->FAILED with fresh diagnostics), +-- resurrecting a FAILED/SUPERSEDED revision, un-activating an ACTIVE +-- revision, or mutating mutable columns while keeping the same status is +-- refused with 23514. A no-op UPDATE that changes nothing is a harmless +-- no-op and stays allowed. Go-side CAS (FailRevision/FailRevisionFrom with +-- an expected-status token) refuses the same cases as typed errors without +-- ever writing. +-- +-- Column policy: +-- - Identity/lifecycle scalars are typed columns; identity columns carry +-- COLLATE "C" for exact byte-order uniqueness and index order. +-- - config, authoring_bundle, public_output_contract preserve the Go +-- json.RawMessage tri-state losslessly: missing (nil) is SQL NULL, +-- JSON-null is the jsonb value 'null', a document is the document. +-- Nothing is coerced across those states; tests assert each with +-- literal rows. +-- - materializations, emitted_columns, quality_reports, diagnostics are +-- nullable jsonb arrays: a nil Go slice is SQL NULL (missing), an empty +-- slice is '[]', a populated slice is the array -- each survives a +-- fresh-connection read. dataset and publication are Go value structs +-- and therefore always objects. +-- - recipe is the canonical recipe Bundle (jsonb object) and content_digest +-- is the sha256 content identity of the frozen column set, computed by +-- the adapter over the exact same field set Arango compares +-- (immutableRevision). Same id + same digest is the idempotent replay; +-- same id + different digest is the typed ErrImmutableRevision refusal. +-- The CHECK keeps it a canonical hex digest so no row can arrive +-- without its content witness. +-- - explorer_revisions_status_known closes the status domain +-- (READY/ACTIVE/SUPERSEDED/FAILED). +-- +-- This migration never drops or truncates state: rerunning it on an existing +-- schema (crash in the commit-before-record window) is a no-op convergence; +-- CREATE OR REPLACE FUNCTION plus DROP TRIGGER IF EXISTS + CREATE TRIGGER +-- re-establish the identical guard, never a weakened one. + +CREATE TABLE IF NOT EXISTS explorer_revisions ( + id text NOT NULL COLLATE "C", + project text NOT NULL COLLATE "C", + explorer_id text NOT NULL COLLATE "C", + config jsonb, + authoring_bundle jsonb, + intent_digest text NOT NULL DEFAULT '', + compilation_receipt_id text NOT NULL DEFAULT '', + public_output_contract jsonb, + recipe jsonb NOT NULL, + recipe_digest text NOT NULL DEFAULT '', + resolved_schema_digest text NOT NULL DEFAULT '', + source_generation text NOT NULL DEFAULT '', + materializations jsonb, + emitted_columns jsonb, + dataset jsonb NOT NULL, + quality_reports jsonb, + content_digest text NOT NULL, + status text NOT NULL, + diagnostics jsonb, + publication jsonb NOT NULL, + created_by text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL, + ready_at timestamptz, + activated_at timestamptz, + failed_at timestamptz, + CONSTRAINT explorer_revisions_pkey PRIMARY KEY (id), + CONSTRAINT explorer_revisions_status_known + CHECK (status IN ('READY', 'ACTIVE', 'SUPERSEDED', 'FAILED')), + CONSTRAINT explorer_revisions_content_digest_shape + CHECK (content_digest ~ '^sha256:[0-9a-f]{64}$'), + CONSTRAINT explorer_revisions_recipe_object + CHECK (jsonb_typeof(recipe) = 'object'), + CONSTRAINT explorer_revisions_dataset_object + CHECK (jsonb_typeof(dataset) = 'object'), + CONSTRAINT explorer_revisions_publication_object + CHECK (jsonb_typeof(publication) = 'object'), + CONSTRAINT explorer_revisions_materializations_shape + CHECK (materializations IS NULL OR jsonb_typeof(materializations) = 'array'), + CONSTRAINT explorer_revisions_emitted_columns_shape + CHECK (emitted_columns IS NULL OR jsonb_typeof(emitted_columns) = 'array'), + CONSTRAINT explorer_revisions_quality_reports_shape + CHECK (quality_reports IS NULL OR jsonb_typeof(quality_reports) = 'array'), + CONSTRAINT explorer_revisions_diagnostics_shape + CHECK (diagnostics IS NULL OR jsonb_typeof(diagnostics) = 'array') +); + +CREATE INDEX IF NOT EXISTS explorer_revisions_project_explorer + ON explorer_revisions (project, explorer_id); + +CREATE OR REPLACE FUNCTION loom_explorer_revision_guard() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + -- Insert-once against direct SQL: a DELETE followed by a re-INSERT of + -- different frozen content would fire no guard on the fresh row, so + -- deletions are refused outright. NEW is NULL in a BEFORE DELETE row + -- trigger, so the no-op shortcut and the frozen-column comparisons + -- below cannot be relied on for deletes; the refusal carries its own + -- message instead. + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION + 'explorer_revisions is insert-once: DELETE of revision % is refused', + OLD.id + USING ERRCODE = '23514'; + END IF; + + -- A true no-op (every column identical) changes nothing and stays legal. + IF ROW(NEW.*) IS NOT DISTINCT FROM ROW(OLD.*) THEN + RETURN NEW; + END IF; + + IF NEW.id IS DISTINCT FROM OLD.id + OR NEW.project IS DISTINCT FROM OLD.project + OR NEW.explorer_id IS DISTINCT FROM OLD.explorer_id + OR NEW.config IS DISTINCT FROM OLD.config + OR NEW.authoring_bundle IS DISTINCT FROM OLD.authoring_bundle + OR NEW.intent_digest IS DISTINCT FROM OLD.intent_digest + OR NEW.compilation_receipt_id IS DISTINCT FROM OLD.compilation_receipt_id + OR NEW.public_output_contract IS DISTINCT FROM OLD.public_output_contract + OR NEW.recipe IS DISTINCT FROM OLD.recipe + OR NEW.recipe_digest IS DISTINCT FROM OLD.recipe_digest + OR NEW.resolved_schema_digest IS DISTINCT FROM OLD.resolved_schema_digest + OR NEW.source_generation IS DISTINCT FROM OLD.source_generation + OR NEW.materializations IS DISTINCT FROM OLD.materializations + OR NEW.emitted_columns IS DISTINCT FROM OLD.emitted_columns + OR NEW.dataset IS DISTINCT FROM OLD.dataset + OR NEW.quality_reports IS DISTINCT FROM OLD.quality_reports + OR NEW.content_digest IS DISTINCT FROM OLD.content_digest + OR NEW.created_by IS DISTINCT FROM OLD.created_by + OR NEW.created_at IS DISTINCT FROM OLD.created_at + THEN + RAISE EXCEPTION + 'explorer revision % frozen content changed', OLD.id + USING ERRCODE = '23514'; + END IF; + + IF NOT ( + (OLD.status = 'READY' AND NEW.status IN ('ACTIVE', 'FAILED')) + OR (OLD.status = 'ACTIVE' AND NEW.status IN ('SUPERSEDED', 'FAILED')) + ) THEN + RAISE EXCEPTION + 'illegal explorer revision % status transition % -> %', + OLD.id, OLD.status, NEW.status + USING ERRCODE = '23514'; + END IF; + + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS explorer_revisions_guard ON explorer_revisions; +CREATE TRIGGER explorer_revisions_guard + BEFORE UPDATE OR DELETE ON explorer_revisions + FOR EACH ROW + EXECUTE FUNCTION loom_explorer_revision_guard(); diff --git a/internal/backends/postgres/internal/explorer/migrations/000007_interpretation_library.sql b/internal/backends/postgres/internal/explorer/migrations/000007_interpretation_library.sql new file mode 100644 index 000000000..a7df3a9aa --- /dev/null +++ b/internal/backends/postgres/internal/explorer/migrations/000007_interpretation_library.sql @@ -0,0 +1,157 @@ +-- pg-state-interpretation-library-v1: interpretation library heads and +-- immutable project revisions for the Explorer control plane. +-- +-- Two tables, composed after the owner history (ledger 1, owner 2, draft +-- revision 3) of this same Explorer control-plane fixture lineage: +-- * interpretation_revisions holds the immutable, content-addressed +-- facts. The ID is the domain's interpretation_<64 hex> hash over the +-- identity envelope and content_digest is sha256 over the canonical +-- content, so the same content re-derives the same row and different +-- immutable content can never share an identity. Rows are create-once: +-- a shared guard function plus a BEFORE UPDATE OR DELETE trigger refuse +-- any mutation of a revision row with SQLSTATE 45501, and the suite +-- proves the guards are not vacuous by dropping the trigger, repeating +-- the violating statement (which must then succeed), and restoring it. +-- * interpretation_libraries holds exactly one mutable authoring head per +-- (project, id): the CAS adapter statement advances head_revision_id +-- and head_digest only while the row still carries the expected head. +-- The library guard trigger refuses DELETE outright, refuses any change +-- to the identity columns (project, id, created_at), and refuses an +-- UPDATE that does not advance the head: the head pointer moves, the +-- row never disappears, nothing else is rewritten. +-- +-- Integrity is enforced here, not only in Go: +-- * Identity columns carry COLLATE "C" for exact byte order, matching the +-- owner table and Arango's byte-order comparisons. +-- * Every shape rule the domain enforces is mirrored by a named CHECK: +-- revision id, library id, content digest, parent id/digest, head +-- revision/digest. The parent pair is NULL together (root) or non-NULL +-- together (child): interpretation_revisions_parent_pair. +-- * interpretation_revisions_parent_same_library_fk makes a parent that +-- does not exist in the SAME project AND library a database refusal +-- (23503). The pair is NULL for roots, so the composite key skips +-- them. interpretation_libraries_head_same_library_fk makes a head that +-- points at a nonexistent revision, or at a revision of another +-- library, a database refusal: the browse affordance the lifecycle +-- expands can never dangle by construction. +-- * The jsonb payload is authoritative for reads (the adapter re-verifies +-- payload self-consistency and mirror agreement on every read); the +-- scalar mirrors exist for scope guards, constraints, and ordered scans. +-- jsonb preserves the missing-versus-JSON-null member shapes of the +-- canonical revision document untouched. +-- * The DDL is rerun-safe (IF NOT EXISTS, DROP TRIGGER IF EXISTS before +-- each CREATE TRIGGER): the pg-store-02 runner may retry a migration +-- that committed before its ledger row was written. This migration +-- never drops or truncates tables. + +CREATE TABLE IF NOT EXISTS interpretation_revisions ( + project text NOT NULL COLLATE "C", + id text NOT NULL COLLATE "C", + library_id text NOT NULL COLLATE "C", + parent_revision_id text COLLATE "C", + parent_digest text, + content_digest text NOT NULL, + author text NOT NULL, + explanation text NOT NULL, + created_at timestamptz NOT NULL, + payload jsonb NOT NULL, + CONSTRAINT interpretation_revisions_identity_pk + PRIMARY KEY (project, id), + CONSTRAINT interpretation_revisions_library_id_ux + UNIQUE (project, id, library_id), + CONSTRAINT interpretation_revisions_project_non_empty + CHECK (project <> ''), + CONSTRAINT interpretation_revisions_id_shape + CHECK (id ~ '^interpretation_[0-9a-f]{64}$'), + CONSTRAINT interpretation_revisions_library_id_shape + CHECK (library_id ~ '^[A-Za-z][A-Za-z0-9_.:-]{0,127}$'), + CONSTRAINT interpretation_revisions_content_digest_shape + CHECK (content_digest ~ '^sha256:[0-9a-f]{64}$'), + CONSTRAINT interpretation_revisions_parent_pair + CHECK ((parent_revision_id IS NULL) = (parent_digest IS NULL)), + CONSTRAINT interpretation_revisions_parent_revision_shape + CHECK (parent_revision_id IS NULL OR parent_revision_id ~ '^interpretation_[0-9a-f]{64}$'), + CONSTRAINT interpretation_revisions_parent_digest_shape + CHECK (parent_digest IS NULL OR parent_digest ~ '^sha256:[0-9a-f]{64}$'), + CONSTRAINT interpretation_revisions_author_non_empty + CHECK (author <> ''), + CONSTRAINT interpretation_revisions_explanation_non_empty + CHECK (explanation <> ''), + CONSTRAINT interpretation_revisions_payload_object + CHECK (jsonb_typeof(payload) = 'object'), + CONSTRAINT interpretation_revisions_parent_same_library_fk + FOREIGN KEY (project, parent_revision_id, library_id) + REFERENCES interpretation_revisions (project, id, library_id) +); + +-- Retrieval mirrors the Arango adapter's revision lookup (identity PK) plus +-- the library-scoped chain scans. +CREATE INDEX IF NOT EXISTS interpretation_revisions_library_ix + ON interpretation_revisions (project, library_id, id); + +CREATE TABLE IF NOT EXISTS interpretation_libraries ( + project text NOT NULL COLLATE "C", + id text NOT NULL COLLATE "C", + head_revision_id text NOT NULL COLLATE "C", + head_digest text NOT NULL, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT interpretation_libraries_identity_pk + PRIMARY KEY (project, id), + CONSTRAINT interpretation_libraries_project_non_empty + CHECK (project <> ''), + CONSTRAINT interpretation_libraries_id_shape + CHECK (id ~ '^[A-Za-z][A-Za-z0-9_.:-]{0,127}$'), + CONSTRAINT interpretation_libraries_head_revision_shape + CHECK (head_revision_id ~ '^interpretation_[0-9a-f]{64}$'), + CONSTRAINT interpretation_libraries_head_digest_shape + CHECK (head_digest ~ '^sha256:[0-9a-f]{64}$'), + CONSTRAINT interpretation_libraries_head_same_library_fk + FOREIGN KEY (project, head_revision_id, id) + REFERENCES interpretation_revisions (project, id, library_id) +); + +-- The immutability guard: revision rows are create-once. Any UPDATE or +-- DELETE -- from any session, including one holding the table owner's +-- rights -- aborts with SQLSTATE 45501 before touching a page. The adapter +-- never issues UPDATE/DELETE against these tables; the trigger is what +-- makes that a database-enforced property rather than a convention. +CREATE OR REPLACE FUNCTION loom_interpretation_revision_immutable() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION 'loom: interpretation revision rows are immutable (create-once)' + USING ERRCODE = '45501'; +END; +$$; + +-- The head-advance guard: a library row may only ever advance its head. +-- DELETE is refused outright; UPDATE must keep project, id, and created_at +-- byte-identical and must actually move head_revision_id. The adapter's +-- single CAS statement is the only writer that satisfies this contract. +CREATE OR REPLACE FUNCTION loom_interpretation_library_guard() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'loom: interpretation library heads are never deleted' + USING ERRCODE = '45501'; + END IF; + IF NEW.project IS DISTINCT FROM OLD.project + OR NEW.id IS DISTINCT FROM OLD.id + OR NEW.created_at IS DISTINCT FROM OLD.created_at + OR NEW.head_revision_id = OLD.head_revision_id THEN + RAISE EXCEPTION 'loom: interpretation library identity is stable and every update must advance the head' + USING ERRCODE = '45501'; + END IF; + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS interpretation_revisions_immutable ON interpretation_revisions; +CREATE TRIGGER interpretation_revisions_immutable + BEFORE UPDATE OR DELETE ON interpretation_revisions + FOR EACH ROW EXECUTE FUNCTION loom_interpretation_revision_immutable(); + +DROP TRIGGER IF EXISTS interpretation_libraries_head_guard ON interpretation_libraries; +CREATE TRIGGER interpretation_libraries_head_guard + BEFORE UPDATE OR DELETE ON interpretation_libraries + FOR EACH ROW EXECUTE FUNCTION loom_interpretation_library_guard(); diff --git a/internal/backends/postgres/internal/explorer/migrations/000008_selection_revisions.sql b/internal/backends/postgres/internal/explorer/migrations/000008_selection_revisions.sql new file mode 100644 index 000000000..d9a38ff50 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/migrations/000008_selection_revisions.sql @@ -0,0 +1,197 @@ +-- pg-state-selection-revision-owner-v1: staged selection revision +-- identity, project-scoped idempotency key, and expiring writer owner. +-- +-- Two tables form the relational counterpart of the Arango adapter's +-- SelectionsCollection and SelectionMembersCollection +-- (internal/explorer/arango/selections.go). The version continues the +-- Explorer control-plane fixture history after the 000004 capability +-- reservation (pg-state-immutable-capability-receipts). Per the +-- coordinator allocation, version 8 is the next free Explorer control-plane +-- ledger version (the Explorer ledger and the store-dir ledger are separate +-- tables in separate dirs: the store history's own 000008, +-- internal/store/postgres/migrations/000008_batch_write_ledger.sql, is a +-- different ledger over different schemas, so the shared number is no +-- collision), lands strictly after the 000007 reserved for the +-- capability-artifacts ticket, and one migration covering both tables +-- follows the 000004 two-table precedent. The DDL is +-- idempotent because the pg-store-02 runner may retry a migration that +-- committed before its ledger row was written. It never drops or truncates +-- existing state. +-- +-- Selection revision identity: +-- * id is the global selection ID -- Arango keys these documents by +-- _key == id, so the primary key is id alone, and reads that carry a +-- project (GetSelection) stay scoped by it: a foreign project asking +-- for an id that exists elsewhere gets the concealed missing answer. +-- * (project, generation, resource_type, rule_digest, scope_digest) plus +-- the idempotency key are the identity the adapter rereads after any +-- arbiter skip and compares field by field; a key never rebinds +-- different intent. Identity columns carry COLLATE "C" for exact byte +-- order, matching the member-key walk and Arango's byte-order SORT. +-- * The idempotency key is unique per project via a partial unique +-- index: an empty key is omitted (like the model's omitempty and +-- Arango's missing attribute) and never collides; any non-empty key is +-- exactly one revision per project. That index is the single-winner +-- arbiter making concurrent same-key/same-content begins converge. +-- * state and complete agree by CHECK, a COMPLETE row must carry its +-- frozen membership facts (digest, completed_at), counters are +-- non-negative, and identity fields are non-empty -- BeginSelection's +-- validation closes the same domain at the database. +-- +-- Expiring writer owner: +-- * writer_token/writer_expires_at bind the staged revision to its +-- writer; the adapter claims a foreign row only with a guarded UPDATE +-- whose WHERE re-evaluates writer_expires_at < now in PostgreSQL, so +-- an unexpired writer is unclaimable by construction and expired +-- recovery follows the current lease boundary. +-- +-- Member insert-once: +-- * member_key is the derived immutable key (selection id + "_" + the +-- shared key() hash over project/generation/resource type/id), unique +-- per selection as the primary key; the (selection_id, +-- project, generation, resource_type, id) unique keeps reference +-- identity distinct even against an adapter derivation bug. Retries +-- are idempotent through ON CONFLICT DO NOTHING on the primary key. +-- * The key-shape CHECK keeps every member key in the derived +-- "_<64 lowercase hex>" suffix form. +-- +-- Immutability is enforced here, not only in Go: a shared guard per table +-- plus BEFORE UPDATE triggers refuse identity tampering, completed-fact +-- tampering (completion is one-way), and ANY update of a staged member +-- with SQLSTATE 45501 before it can run. Member DELETE remains legal +-- because abort and bounded staging cleanup remove abandoned rows as +-- whole rows -- member content is never rewritten, only retired. The +-- suite proves the guards are not vacuous by dropping each trigger, +-- repeating the violating statement (which must then succeed), and +-- restoring the trigger. + +CREATE TABLE IF NOT EXISTS selection_revisions ( + id text NOT NULL COLLATE "C", + project text NOT NULL COLLATE "C", + generation text NOT NULL COLLATE "C", + resource_type text NOT NULL COLLATE "C", + rule_digest text NOT NULL COLLATE "C", + scope_digest text NOT NULL COLLATE "C", + idempotency_key text NOT NULL COLLATE "C", + state text NOT NULL, + writer_token text NOT NULL, + writer_expires_at timestamptz NOT NULL, + membership_digest text NOT NULL DEFAULT '', + member_count bigint NOT NULL DEFAULT 0, + member_bytes bigint NOT NULL DEFAULT 0, + complete boolean NOT NULL, + created_at timestamptz NOT NULL, + completed_at timestamptz, + rule jsonb NOT NULL + CONSTRAINT selection_revisions_rule_object CHECK (jsonb_typeof(rule) = 'object'), + source jsonb NOT NULL + CONSTRAINT selection_revisions_source_object CHECK (jsonb_typeof(source) = 'object'), + exclusions jsonb + CONSTRAINT selection_revisions_exclusions_array CHECK (exclusions IS NULL OR jsonb_typeof(exclusions) = 'array'), + CONSTRAINT selection_revisions_id_pk PRIMARY KEY (id), + CONSTRAINT selection_revisions_identity_non_empty + CHECK (id <> '' AND project <> '' AND generation <> '' AND resource_type <> ''), + CONSTRAINT selection_revisions_digests_present + CHECK (rule_digest <> '' AND scope_digest <> ''), + CONSTRAINT selection_revisions_writer_present + CHECK (writer_token <> ''), + CONSTRAINT selection_revisions_state_known + CHECK (state IN ('STAGING', 'COMPLETE')), + CONSTRAINT selection_revisions_complete_matches_state + CHECK ((state = 'COMPLETE') = complete), + CONSTRAINT selection_revisions_complete_facts + CHECK (complete = FALSE OR (membership_digest <> '' AND completed_at IS NOT NULL)), + CONSTRAINT selection_revisions_membership_non_negative + CHECK (member_count >= 0 AND member_bytes >= 0) +); + +-- The project-scoped idempotency arbiter. Partial so an omitted (empty) +-- key never participates: like Arango's filter on a missing attribute, +-- keyless revisions coexist, and any non-empty key is exactly one +-- revision per project. +CREATE UNIQUE INDEX IF NOT EXISTS selection_revisions_project_key_ux + ON selection_revisions (project, idempotency_key) + WHERE idempotency_key <> ''; + +-- Retrieval mirror for the bounded staging cleanup scan (expired STAGING +-- headers by age, then lease). +CREATE INDEX IF NOT EXISTS selection_revisions_staging_ix + ON selection_revisions (state, created_at, writer_expires_at); + +CREATE TABLE IF NOT EXISTS selection_members ( + selection_id text NOT NULL COLLATE "C", + member_key text NOT NULL COLLATE "C", + project text NOT NULL COLLATE "C", + generation text NOT NULL COLLATE "C", + resource_type text NOT NULL COLLATE "C", + id text NOT NULL COLLATE "C", + ordinal bigint NOT NULL DEFAULT 0, + CONSTRAINT selection_members_key_pk + PRIMARY KEY (selection_id, member_key), + CONSTRAINT selection_members_ref_ux + UNIQUE (selection_id, project, generation, resource_type, id), + CONSTRAINT selection_members_identity_non_empty + CHECK (selection_id <> '' AND project <> '' AND generation <> '' + AND resource_type <> '' AND id <> ''), + CONSTRAINT selection_members_key_shape + CHECK (member_key ~ '_[0-9a-f]{64}$') +); + +-- The selection-revision guard: identity columns are insert-once, and a +-- COMPLETE row's state/facts are frozen (completion is one-way). The +-- writer lease and pre-completion staging facts stay live precisely +-- because they are outside the frozen set. +CREATE OR REPLACE FUNCTION loom_selection_revision_immutable() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.id IS DISTINCT FROM OLD.id + OR NEW.project IS DISTINCT FROM OLD.project + OR NEW.generation IS DISTINCT FROM OLD.generation + OR NEW.resource_type IS DISTINCT FROM OLD.resource_type + OR NEW.rule_digest IS DISTINCT FROM OLD.rule_digest + OR NEW.scope_digest IS DISTINCT FROM OLD.scope_digest + OR NEW.idempotency_key IS DISTINCT FROM OLD.idempotency_key + OR NEW.created_at IS DISTINCT FROM OLD.created_at + OR NEW.rule IS DISTINCT FROM OLD.rule + OR NEW.source IS DISTINCT FROM OLD.source + OR NEW.exclusions IS DISTINCT FROM OLD.exclusions + THEN + RAISE EXCEPTION 'loom: selection revision identity columns are immutable (insert-once)' + USING ERRCODE = '45501'; + END IF; + IF OLD.complete AND ( + NEW.state IS DISTINCT FROM OLD.state + OR NEW.complete IS DISTINCT FROM OLD.complete + OR NEW.membership_digest IS DISTINCT FROM OLD.membership_digest + OR NEW.member_count IS DISTINCT FROM OLD.member_count + OR NEW.member_bytes IS DISTINCT FROM OLD.member_bytes + OR NEW.completed_at IS DISTINCT FROM OLD.completed_at + ) THEN + RAISE EXCEPTION 'loom: completed selection facts are immutable (completion is one-way)' + USING ERRCODE = '45501'; + END IF; + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS selection_revisions_immutable ON selection_revisions; +CREATE TRIGGER selection_revisions_immutable + BEFORE UPDATE ON selection_revisions + FOR EACH ROW EXECUTE FUNCTION loom_selection_revision_immutable(); + +-- The member guard: staged membership content is insert-once. Any UPDATE +-- aborts with 45501 before touching a page; removal is lifecycle-only +-- (abort and bounded staging cleanup delete whole rows, they never +-- rewrite one). +CREATE OR REPLACE FUNCTION loom_selection_member_immutable() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION 'loom: staged selection member rows are immutable (insert-once; removal is lifecycle-only)' + USING ERRCODE = '45501'; +END; +$$; + +DROP TRIGGER IF EXISTS selection_members_immutable ON selection_members; +CREATE TRIGGER selection_members_immutable + BEFORE UPDATE ON selection_members + FOR EACH ROW EXECUTE FUNCTION loom_selection_member_immutable(); diff --git a/internal/backends/postgres/internal/explorer/migrations/000009_explorer_explicit_groups.sql b/internal/backends/postgres/internal/explorer/migrations/000009_explorer_explicit_groups.sql new file mode 100644 index 000000000..65a98e812 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/migrations/000009_explorer_explicit_groups.sql @@ -0,0 +1,330 @@ +-- pg-state-explicit-group-definitions-v1: explicit-group revision headers +-- and ordered definition rows. +-- +-- Explicit groups use three separate records (internal/explorer/ +-- explicit_group_repository.go): the revision header, the ordered +-- definition rows, and (a later ticket) membership rows. This migration +-- creates the first two. A definition row is a group's durable identity and +-- order independent of any membership, so a group with no members still +-- exists as a real, ordered group -- "empty" here is a declared group with +-- zero membership, never a missing row. Order is a stored column +-- (ordinal), never decode order or a JSON array: parity_risks forbid +-- encoding definitions only as JSONB. +-- +-- Staging lifecycle is database-enforced, not advisory, via two BEFORE +-- triggers plus closed CHECK domains: +-- +-- * The completion gate: nothing in this slice may mark a header +-- COMPLETE -- completion belongs to the whole later-defined revision +-- contract and requires the membership slice's transactionally +-- verified final digest/count operation. So explorer_explicit_group_ +-- revisions_state_staging CHECKs state = 'STAGING' outright: neither +-- the store nor direct SQL can complete a header, and the membership +-- ticket's migration replaces this constraint together with the +-- guarded STAGING->COMPLETE lifecycle machine it will introduce. +-- Staging rows additionally carry no completion values +-- (explorer_explicit_group_revisions_staging_values), matching the +-- domain Validate rule "staging explicit group revision cannot +-- contain completion values". +-- * The idempotency identity stays honestly derived: the revision ID is +-- the deterministic hash of (project, idempotency key) computed by +-- explorer.ExplicitGroupRevisionIDFor, and +-- explorer_explicit_group_revisions_identity_binding recomputes the +-- exact length-prefixed SHA-256 frame in SQL, so a row whose id does +-- not match its own (project, idempotency_key) pair is refused at +-- insert -- the reservation identity cannot be forged. +-- * Header UPDATEs are a renewal-only channel: the BEFORE UPDATE +-- trigger loom_explorer_explicit_group_revision_guard refuses any +-- change to identity, binding, or completion columns, refuses any +-- change to the writer token itself, and admits writer_lease changes +-- only when the session stamped its local +-- loom.explorer_explicit_group_writer_token GUC with the row's own +-- writer token AND the lease moves strictly forward (never revoked, +-- never shortened -- a shrunken lease would let cleanup kill an +-- active writer). A true no-op UPDATE stays legal. +-- * Header DELETEs are authorization-gated cleanup/abort only: the same +-- trigger refuses a DELETE unless the session either (a) stamped +-- loom.explorer_explicit_group_abort_token with the row's own writer +-- token (owner abort), or (b) stamped +-- loom.explorer_explicit_group_cleanup_now with the sweep clock and +-- the row is expired at that clock (writer_expires_at IS NULL or +-- before it -- the Arango adapter's missing-field tolerance). +-- Active writers are therefore never removable by un-authorized SQL; +-- the store passes only its own real clock, so cleanup converges on +-- "expired at the sweep's clock". +-- * Definition rows are write-once: loom_explorer_explicit_group_ +-- definition_guard refuses every UPDATE, and refuses DELETE unless +-- the parent header authorizes the same operation (its own STAGING +-- state plus a matching abort-token GUC or an expired-at-cleanup-now +-- lease), so children of a completed or foreign-owned revision can +-- never be removed. The FOREIGN KEY ... ON DELETE RESTRICT additionally +-- keeps a header removable only after its definition rows are gone -- +-- the store's abort order, not a caller preference. +-- * Definition shape is a closed domain: group identity non-blank, +-- label non-blank, ordinal non-negative, one row per +-- (revision, group) -- PRIMARY KEY -- and one row per +-- (revision, ordinal) -- UNIQUE -- exactly the invariants the domain +-- CanonicalExplicitGroupDefinitions checks in Go, now also DB facts. +-- +-- Column policy: identity columns carry COLLATE "C" for exact byte-order +-- uniqueness and index order (the Arango adapter's byte-order listing and +-- _key tiebreaks). completed_at and the writer columns are nullable +-- precisely where the domain/adapter tolerates absence (staging never +-- carries completed_at; cleanup tolerates a missing writerExpiresAt). +-- writer_token/writer_expires_at are storage-only lifecycle columns the +-- domain struct never carries, matching the adapter's header document. +-- +-- This migration never drops or truncates state: rerunning it on an +-- existing schema (crash in the commit-before-record window) is a no-op +-- convergence; CREATE OR REPLACE FUNCTION plus DROP TRIGGER IF EXISTS + +-- CREATE TRIGGER re-establish the identical guards, never weakened ones. + +CREATE TABLE IF NOT EXISTS explorer_explicit_group_revisions ( + id text NOT NULL COLLATE "C", + project text NOT NULL COLLATE "C", + generation text NOT NULL COLLATE "C", + scope_digest text NOT NULL COLLATE "C", + resource_type text NOT NULL COLLATE "C", + source_selection_revision_id text NOT NULL COLLATE "C", + source_membership_digest text NOT NULL COLLATE "C", + definition_digest text NOT NULL DEFAULT '', + membership_digest text NOT NULL DEFAULT '', + group_count bigint NOT NULL DEFAULT 0, + member_count bigint NOT NULL DEFAULT 0, + state text NOT NULL, + idempotency_key text NOT NULL COLLATE "C", + created_at timestamptz NOT NULL, + completed_at timestamptz, + writer_token text COLLATE "C", + writer_expires_at timestamptz, + CONSTRAINT explorer_explicit_group_revisions_pkey + PRIMARY KEY (id), + -- Completion gate for this slice (replaced by the membership + -- ticket's guarded lifecycle machine): no header may exist in any + -- state other than STAGING, no matter who writes the statement. + CONSTRAINT explorer_explicit_group_revisions_state_staging + CHECK (state = 'STAGING'), + -- Staging headers carry no completion values, per the domain rule. + CONSTRAINT explorer_explicit_group_revisions_staging_values + CHECK (definition_digest = '' AND membership_digest = '' + AND group_count = 0 AND member_count = 0 + AND completed_at IS NULL), + CONSTRAINT explorer_explicit_group_revisions_required_fields + CHECK (project <> '' AND generation <> '' AND scope_digest <> '' + AND resource_type <> '' AND source_selection_revision_id <> '' + AND source_membership_digest <> '' AND idempotency_key <> ''), + -- The idempotency identity is honestly derived: id equals the + -- length-prefixed SHA-256 over the domain separator, project, and + -- idempotency key -- the exact frame explorer.ExplicitGroupRevisionIDFor + -- digests (8-byte big-endian length per field). + CONSTRAINT explorer_explicit_group_revisions_identity_binding + CHECK (id = 'grouprev_'::text || encode( + sha256( + decode(lpad(to_hex(octet_length('loom-explicit-group-revision-id-v1')), 16, '0'), 'hex') + || 'loom-explicit-group-revision-id-v1'::bytea + || decode(lpad(to_hex(octet_length(project)), 16, '0'), 'hex') + || project::bytea + || decode(lpad(to_hex(octet_length(idempotency_key)), 16, '0'), 'hex') + || idempotency_key::bytea + ), 'hex')) +); + +CREATE INDEX IF NOT EXISTS explorer_explicit_group_revisions_listing + ON explorer_explicit_group_revisions + (project, generation, resource_type, scope_digest, created_at DESC, id); + +CREATE INDEX IF NOT EXISTS explorer_explicit_group_revisions_staging + ON explorer_explicit_group_revisions (created_at, id) + WHERE state = 'STAGING'; + +CREATE TABLE IF NOT EXISTS explorer_explicit_group_definitions ( + revision_id text NOT NULL COLLATE "C", + group_id text NOT NULL COLLATE "C", + label text NOT NULL, + ordinal bigint NOT NULL, + CONSTRAINT explorer_explicit_group_definitions_pkey + PRIMARY KEY (revision_id, group_id), + -- Ordinal is the group list's user-visible order: unique per + -- revision, non-negative -- the adapter's canonicalization invariants + -- as relational constraints. + CONSTRAINT explorer_explicit_group_definitions_revision_ordinal_key + UNIQUE (revision_id, ordinal), + CONSTRAINT explorer_explicit_group_definitions_group_id_present + CHECK (group_id <> ''), + CONSTRAINT explorer_explicit_group_definitions_label_present + CHECK (label <> ''), + CONSTRAINT explorer_explicit_group_definitions_ordinal_non_negative + CHECK (ordinal >= 0), + CONSTRAINT explorer_explicit_group_definitions_revision_fkey + FOREIGN KEY (revision_id) + REFERENCES explorer_explicit_group_revisions (id) + ON DELETE RESTRICT ON UPDATE RESTRICT +); + +CREATE OR REPLACE FUNCTION loom_explorer_explicit_group_revision_guard() RETURNS trigger +LANGUAGE plpgsql AS $$ +DECLARE + writer_token text := current_setting('loom.explorer_explicit_group_writer_token', true); + abort_token text := current_setting('loom.explorer_explicit_group_abort_token', true); + cleanup_now text := current_setting('loom.explorer_explicit_group_cleanup_now', true); +BEGIN + IF TG_OP = 'DELETE' THEN + -- Only ever staging headers: a header in any other state is + -- permanent history (the state domain currently keeps this at + -- STAGING; the guard stands ready for the membership ticket's + -- lifecycle machine). + IF OLD.state <> 'STAGING' THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: DELETE of revision % in state % is refused: only STAGING headers are ever removed', + OLD.id, OLD.state + USING ERRCODE = '23514'; + END IF; + IF abort_token IS NOT NULL AND OLD.writer_token IS NOT NULL + AND abort_token = OLD.writer_token THEN + RETURN OLD; -- owner abort: the token holder ends its own staging revision + END IF; + IF cleanup_now IS NOT NULL + AND (OLD.writer_expires_at IS NULL + OR OLD.writer_expires_at < cleanup_now::timestamptz) THEN + RETURN OLD; -- expired-writer cleanup under the sweep's own clock + END IF; + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: DELETE of revision % is refused without owner-abort or expired-writer-cleanup authorization', + OLD.id + USING ERRCODE = '23514'; + END IF; + + -- UPDATE below. A true no-op changes nothing and stays legal. + IF ROW(NEW.*) IS NOT DISTINCT FROM ROW(OLD.*) THEN + RETURN NEW; + END IF; + + IF NEW.writer_token IS DISTINCT FROM OLD.writer_token THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: writer token of revision % changed', OLD.id + USING ERRCODE = '23514'; + END IF; + + IF NEW.writer_expires_at IS DISTINCT FROM OLD.writer_expires_at THEN + -- Lease renewal is the only admitted UPDATE channel: the session + -- must hold this row's writer token, and the lease only moves + -- forward -- it is never revoked and never shortened, so no + -- update can manufacture an "expired" row for cleanup to kill. + IF NEW.writer_expires_at IS NULL THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: lease of revision % may never be revoked', OLD.id + USING ERRCODE = '23514'; + END IF; + IF writer_token IS NULL OR OLD.writer_token IS NULL + OR writer_token <> OLD.writer_token THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: lease renewal of revision % refused without the row''s writer token', OLD.id + USING ERRCODE = '23514'; + END IF; + IF OLD.writer_expires_at IS NOT NULL + AND NEW.writer_expires_at <= OLD.writer_expires_at THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: lease of revision % may never be shortened', OLD.id + USING ERRCODE = '23514'; + END IF; + END IF; + + -- Identity, binding, and completion columns are frozen. The state + -- column is deliberately absent here: the named CHECK constraint + -- owns the completion gate so its refusal is classified by name. + IF NEW.id IS DISTINCT FROM OLD.id + OR NEW.project IS DISTINCT FROM OLD.project + OR NEW.generation IS DISTINCT FROM OLD.generation + OR NEW.scope_digest IS DISTINCT FROM OLD.scope_digest + OR NEW.resource_type IS DISTINCT FROM OLD.resource_type + OR NEW.source_selection_revision_id IS DISTINCT FROM OLD.source_selection_revision_id + OR NEW.source_membership_digest IS DISTINCT FROM OLD.source_membership_digest + OR NEW.definition_digest IS DISTINCT FROM OLD.definition_digest + OR NEW.membership_digest IS DISTINCT FROM OLD.membership_digest + OR NEW.group_count IS DISTINCT FROM OLD.group_count + OR NEW.member_count IS DISTINCT FROM OLD.member_count + OR NEW.idempotency_key IS DISTINCT FROM OLD.idempotency_key + OR NEW.created_at IS DISTINCT FROM OLD.created_at + OR NEW.completed_at IS DISTINCT FROM OLD.completed_at + THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: frozen content of revision % changed', OLD.id + USING ERRCODE = '23514'; + END IF; + + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS explorer_explicit_group_revisions_guard ON explorer_explicit_group_revisions; +CREATE TRIGGER explorer_explicit_group_revisions_guard + BEFORE UPDATE OR DELETE ON explorer_explicit_group_revisions + FOR EACH ROW + EXECUTE FUNCTION loom_explorer_explicit_group_revision_guard(); + +CREATE OR REPLACE FUNCTION loom_explorer_explicit_group_definition_guard() RETURNS trigger +LANGUAGE plpgsql AS $$ +DECLARE + parent_state text; + parent_writer_token text; + parent_expires_at timestamptz; + abort_token text := current_setting('loom.explorer_explicit_group_abort_token', true); + cleanup_now text := current_setting('loom.explorer_explicit_group_cleanup_now', true); + diag_rows integer; +BEGIN + IF TG_OP = 'UPDATE' THEN + IF ROW(NEW.*) IS NOT DISTINCT FROM ROW(OLD.*) THEN + RETURN NEW; + END IF; + RAISE EXCEPTION + 'explorer_explicit_group_definitions guard: definition % of revision % is write-once', + OLD.group_id, OLD.revision_id + USING ERRCODE = '23514'; + END IF; + + -- DELETE below: only ever as part of an authorized teardown of the + -- parent staging revision. The parent must exist, be STAGING, and + -- carry the same authorization the header guard demands. The parent + -- lookup resolves through TG_TABLE_SCHEMA so the guard behaves + -- identically no matter what search_path the calling session carries + -- (the trigger fires during ordinary store traffic, not just under + -- the migration runner's SET LOCAL search_path). + -- FOUND is unreliable after EXECUTE ... INTO here, so the row count + -- drives the absent-parent branch instead. + EXECUTE format( + 'SELECT state, writer_token, writer_expires_at FROM %I.explorer_explicit_group_revisions WHERE id = $1', + TG_TABLE_SCHEMA + ) INTO parent_state, parent_writer_token, parent_expires_at USING OLD.revision_id; + GET DIAGNOSTICS diag_rows = ROW_COUNT; + IF diag_rows = 0 THEN + RAISE EXCEPTION + 'explorer_explicit_group_definitions guard: DELETE of definition % refused: parent revision % is absent', + OLD.group_id, OLD.revision_id + USING ERRCODE = '23514'; + END IF; + IF parent_state <> 'STAGING' THEN + RAISE EXCEPTION + 'explorer_explicit_group_definitions guard: DELETE of definition % refused: parent revision % is %', + OLD.group_id, OLD.revision_id, parent_state + USING ERRCODE = '23514'; + END IF; + IF abort_token IS NOT NULL AND parent_writer_token IS NOT NULL + AND abort_token = parent_writer_token THEN + RETURN OLD; -- owner abort cascade + END IF; + IF cleanup_now IS NOT NULL + AND (parent_expires_at IS NULL OR parent_expires_at < cleanup_now::timestamptz) THEN + RETURN OLD; -- expired-writer cleanup cascade under the sweep's clock + END IF; + RAISE EXCEPTION + 'explorer_explicit_group_definitions guard: DELETE of definition % of revision % refused without owner-abort or expired-writer-cleanup authorization', + OLD.group_id, OLD.revision_id + USING ERRCODE = '23514'; +END; +$$; + +DROP TRIGGER IF EXISTS explorer_explicit_group_definitions_guard ON explorer_explicit_group_definitions; +CREATE TRIGGER explorer_explicit_group_definitions_guard + BEFORE UPDATE OR DELETE ON explorer_explicit_group_definitions + FOR EACH ROW + EXECUTE FUNCTION loom_explorer_explicit_group_definition_guard(); diff --git a/internal/backends/postgres/internal/explorer/migrations/000010_explorer_explicit_group_memberships.sql b/internal/backends/postgres/internal/explorer/migrations/000010_explorer_explicit_group_memberships.sql new file mode 100644 index 000000000..b9679a7d5 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/migrations/000010_explorer_explicit_group_memberships.sql @@ -0,0 +1,432 @@ +-- pg-state-explicit-group-members-v1: explicit-group membership rows and +-- the guarded STAGING->COMPLETE lifecycle machine the definitions migration +-- explicitly deferred to this ticket. +-- +-- 000009 created the revision header and the ordered definition rows and +-- pinned the header to STAGING outright, promising that "the membership +-- ticket's migration replaces that constraint together with the guarded +-- lifecycle machine it will introduce". This migration is that promise, +-- kept in full: +-- +-- (1) explorer_explicit_group_memberships: one row per +-- (revision, group, reference). The primary key CARRIES THE GROUP ID +-- (revision_id, group_id, project, generation, resource_type, +-- resource_id), so one resource occurring under several GroupIDs is +-- several rows -- the overlap the domain contract requires -- while a +-- duplicate retry of the same (revision, group, reference) collides on +-- the primary key and creates nothing. Every identity column is +-- COLLATE "C" for exact byte-order uniqueness and index order +-- (the Arango adapter's byte-order membership walk). +-- (2) Membership rows are children twice over: a foreign key to the +-- header (ON DELETE RESTRICT -- a header is removable only after its +-- membership rows are gone, the store's abort order, not a caller +-- preference) and a compound foreign key to +-- (revision_id, group_id) in explorer_explicit_group_definitions -- +-- so "membership references an undeclared group" is a database fact +-- (23503), not a Go convention. +-- (3) The membership guard trigger makes membership rows +-- insert-once/whole-row-retired: INSERT is admitted only under a +-- STAGING parent whose project, generation, and resource type the +-- reference repeats exactly (a completed or foreign-scoped revision +-- can never gain members, even by direct SQL); every real UPDATE is +-- refused; DELETE is admitted only as part of an authorized teardown +-- of the parent staging revision (owner-abort or expired-writer +-- cleanup, the same two GUC channels 000009 established). +-- (4) The completion machine replaces 000009's completion gate: +-- DROPping explorer_explicit_group_revisions_state_staging and +-- explorer_explicit_group_revisions_staging_values, then ADDing +-- explorer_explicit_group_revisions_lifecycle_values -- one named +-- constraint that closes BOTH the state domain and the value shape: +-- STAGING carries no completion values, COMPLETE carries non-empty +-- digests, group_count > 0, member_count >= 0, and completed_at, and +-- no other state is representable at all. Headers are still born +-- STAGING only: the header guard's new INSERT branch refuses any +-- other state at the door, and the ONLY path to COMPLETE is the +-- guarded UPDATE below. +-- (5) The header guard's UPDATE channel admits exactly three things: +-- a true no-op, the lease renewal 000009 established (writer-token +-- GUC, strictly forward), and the completion transition -- which is +-- admitted only from STAGING, only for the session holding the row's +-- own writer token, only with the writer columns untouched, only +-- with the identity/binding columns frozen, and only when the +-- claimed group_count and member_count EQUAL the definition and +-- membership rows actually stored for the revision at that moment +-- (counted inside the trigger, so "completion transition verifies +-- the definitions and membership set" is a database fact; the +-- digests themselves are content-addressed hashes the store +-- re-derives inside the same lock-held transaction, the exact +-- counterpart of the Arango adapter's digest-and-CAS pair). COMPLETE +-- rows are frozen history: no non-noop UPDATE ever touches them, +-- and the DELETE branch keeps refusing anything but the STAGING +-- teardown channels, so a completed revision is permanent. +-- +-- The migration reruns converge: CREATE TABLE IF NOT EXISTS / +-- CREATE INDEX IF NOT EXISTS, DROP CONSTRAINT IF EXISTS before every ADD, +-- CREATE OR REPLACE FUNCTION, and DROP TRIGGER IF EXISTS + CREATE TRIGGER +-- re-establish the identical guards (crash in the commit-before-record +-- window repeats harmlessly and never weakens a guard). It never drops or +-- truncates state. + +CREATE TABLE IF NOT EXISTS explorer_explicit_group_memberships ( + revision_id text NOT NULL COLLATE "C", + group_id text NOT NULL COLLATE "C", + project text NOT NULL COLLATE "C", + generation text NOT NULL COLLATE "C", + resource_type text NOT NULL COLLATE "C", + resource_id text NOT NULL COLLATE "C", + -- The unique key includes the group ID so legitimate overlaps survive: + -- one resource may belong to several groups, and only the exact + -- (revision, group, reference) tuple is unique (parity_risks). + CONSTRAINT explorer_explicit_group_memberships_pkey + PRIMARY KEY (revision_id, group_id, project, generation, resource_type, resource_id), + CONSTRAINT explorer_explicit_group_memberships_revision_present + CHECK (revision_id <> ''), + CONSTRAINT explorer_explicit_group_memberships_group_id_present + CHECK (group_id <> ''), + CONSTRAINT explorer_explicit_group_memberships_project_present + CHECK (project <> ''), + CONSTRAINT explorer_explicit_group_memberships_generation_present + CHECK (generation <> ''), + CONSTRAINT explorer_explicit_group_memberships_resource_type_present + CHECK (resource_type <> ''), + CONSTRAINT explorer_explicit_group_memberships_resource_id_present + CHECK (resource_id <> ''), + CONSTRAINT explorer_explicit_group_memberships_revision_fkey + FOREIGN KEY (revision_id) + REFERENCES explorer_explicit_group_revisions (id) + ON DELETE RESTRICT ON UPDATE RESTRICT, + -- A membership may name only a group this very revision declared: the + -- declared-domain rule of CanonicalExplicitGroupMemberships as a + -- relational foreign key. + CONSTRAINT explorer_explicit_group_memberships_group_fkey + FOREIGN KEY (revision_id, group_id) + REFERENCES explorer_explicit_group_definitions (revision_id, group_id) + ON DELETE RESTRICT ON UPDATE RESTRICT +); + +-- The bounded keyset walk (digest stream and visit) pages by +-- (group_id, resource_id) within one revision -- the composite +-- continuation cursor of the Arango adapter's membership visit. +CREATE INDEX IF NOT EXISTS explorer_explicit_group_memberships_visit + ON explorer_explicit_group_memberships (revision_id, group_id, resource_id); + +-- The completion gate, version 2. 000009's two staging-only CHECKs are +-- replaced by one named lifecycle constraint that closes the state domain +-- and both value shapes in a single classifiable object. +ALTER TABLE explorer_explicit_group_revisions + DROP CONSTRAINT IF EXISTS explorer_explicit_group_revisions_state_staging; +ALTER TABLE explorer_explicit_group_revisions + DROP CONSTRAINT IF EXISTS explorer_explicit_group_revisions_staging_values; +ALTER TABLE explorer_explicit_group_revisions + DROP CONSTRAINT IF EXISTS explorer_explicit_group_revisions_lifecycle_values; +ALTER TABLE explorer_explicit_group_revisions + ADD CONSTRAINT explorer_explicit_group_revisions_lifecycle_values + CHECK ( + (state = 'STAGING' + AND definition_digest = '' AND membership_digest = '' + AND group_count = 0 AND member_count = 0 + AND completed_at IS NULL) + OR + (state = 'COMPLETE' + AND definition_digest <> '' AND membership_digest <> '' + AND group_count > 0 AND member_count >= 0 + AND completed_at IS NOT NULL) + ); + +CREATE OR REPLACE FUNCTION loom_explorer_explicit_group_revision_guard() RETURNS trigger +LANGUAGE plpgsql AS $$ +DECLARE + writer_token text := current_setting('loom.explorer_explicit_group_writer_token', true); + abort_token text := current_setting('loom.explorer_explicit_group_abort_token', true); + cleanup_now text := current_setting('loom.explorer_explicit_group_cleanup_now', true); + definition_rows bigint; + membership_rows bigint; +BEGIN + IF TG_OP = 'INSERT' THEN + -- Headers are born STAGING. A COMPLETE row may never be inserted + -- directly: completion is a transition, not an initial state, and + -- only the guarded UPDATE channel below may perform it. (Before + -- this migration the same fact lived in the state_staging CHECK; + -- the lifecycle_values constraint keeps the VALUE shapes closed, + -- this branch keeps birth STAGING-only.) + IF NEW.state <> 'STAGING' THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: INSERT of revision % in state % is refused: headers are born STAGING and only the guarded completion transition may complete one', + NEW.id, NEW.state + USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END IF; + + IF TG_OP = 'DELETE' THEN + -- Only ever staging headers: a COMPLETE header is permanent + -- history, exactly as before this migration. + IF OLD.state <> 'STAGING' THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: DELETE of revision % in state % is refused: only STAGING headers are ever removed', + OLD.id, OLD.state + USING ERRCODE = '23514'; + END IF; + IF abort_token IS NOT NULL AND OLD.writer_token IS NOT NULL + AND abort_token = OLD.writer_token THEN + RETURN OLD; -- owner abort: the token holder ends its own staging revision + END IF; + IF cleanup_now IS NOT NULL + AND (OLD.writer_expires_at IS NULL + OR OLD.writer_expires_at < cleanup_now::timestamptz) THEN + RETURN OLD; -- expired-writer cleanup under the sweep's own clock + END IF; + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: DELETE of revision % is refused without owner-abort or expired-writer-cleanup authorization', + OLD.id + USING ERRCODE = '23514'; + END IF; + + -- UPDATE below. A true no-op changes nothing and stays legal. + IF ROW(NEW.*) IS NOT DISTINCT FROM ROW(OLD.*) THEN + RETURN NEW; + END IF; + + IF OLD.state = 'COMPLETE' THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: revision % is COMPLETE history and may never be modified', + OLD.id + USING ERRCODE = '23514'; + END IF; + + IF NEW.writer_token IS DISTINCT FROM OLD.writer_token THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: writer token of revision % changed', OLD.id + USING ERRCODE = '23514'; + END IF; + + IF NEW.state IS DISTINCT FROM OLD.state THEN + -- The completion channel: STAGING -> COMPLETE, and nothing else is + -- representable here (any other pairing lands on this refusal or + -- on the lifecycle_values CHECK). + IF OLD.state <> 'STAGING' OR NEW.state <> 'COMPLETE' THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: revision % state % -> % is refused: the only admitted transition is STAGING -> COMPLETE', + OLD.id, OLD.state, NEW.state + USING ERRCODE = '23514'; + END IF; + IF writer_token IS NULL OR OLD.writer_token IS NULL + OR writer_token <> OLD.writer_token THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: completion of revision % refused without the row''s writer token', OLD.id + USING ERRCODE = '23514'; + END IF; + IF NEW.writer_expires_at IS DISTINCT FROM OLD.writer_expires_at THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: completion of revision % may not move the writer lease', OLD.id + USING ERRCODE = '23514'; + END IF; + IF NEW.id IS DISTINCT FROM OLD.id + OR NEW.project IS DISTINCT FROM OLD.project + OR NEW.generation IS DISTINCT FROM OLD.generation + OR NEW.scope_digest IS DISTINCT FROM OLD.scope_digest + OR NEW.resource_type IS DISTINCT FROM OLD.resource_type + OR NEW.source_selection_revision_id IS DISTINCT FROM OLD.source_selection_revision_id + OR NEW.source_membership_digest IS DISTINCT FROM OLD.source_membership_digest + OR NEW.idempotency_key IS DISTINCT FROM OLD.idempotency_key + OR NEW.created_at IS DISTINCT FROM OLD.created_at + THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: frozen content of revision % changed during completion', OLD.id + USING ERRCODE = '23514'; + END IF; + -- The transition verifies the definitions and membership set at + -- the database: the claimed counts must equal the child rows that + -- actually exist. The header row is locked by the completing + -- transaction (appends and aborts take the same lock first), so + -- this count is stable; the digests are re-derived by the store + -- inside that same lock. + EXECUTE format( + 'SELECT count(*) FROM %I.explorer_explicit_group_definitions WHERE revision_id = $1', + TG_TABLE_SCHEMA + ) INTO definition_rows USING OLD.id; + EXECUTE format( + 'SELECT count(*) FROM %I.explorer_explicit_group_memberships WHERE revision_id = $1', + TG_TABLE_SCHEMA + ) INTO membership_rows USING OLD.id; + IF NEW.group_count <> definition_rows THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: completion of revision % claims group_count % but % definition rows are stored', + OLD.id, NEW.group_count, definition_rows + USING ERRCODE = '23514'; + END IF; + IF NEW.member_count <> membership_rows THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: completion of revision % claims member_count % but % membership rows are stored', + OLD.id, NEW.member_count, membership_rows + USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END IF; + + -- STAGING stays STAGING: the lease channel of 000009, unchanged. + IF NEW.writer_expires_at IS DISTINCT FROM OLD.writer_expires_at THEN + -- Lease renewal is the only admitted non-completion UPDATE: the + -- session must hold this row's writer token, and the lease only + -- moves forward -- it is never revoked and never shortened, so no + -- update can manufacture an "expired" row for cleanup to kill. + IF NEW.writer_expires_at IS NULL THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: lease of revision % may never be revoked', OLD.id + USING ERRCODE = '23514'; + END IF; + IF writer_token IS NULL OR OLD.writer_token IS NULL + OR writer_token <> OLD.writer_token THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: lease renewal of revision % refused without the row''s writer token', OLD.id + USING ERRCODE = '23514'; + END IF; + IF OLD.writer_expires_at IS NOT NULL + AND NEW.writer_expires_at <= OLD.writer_expires_at THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: lease of revision % may never be shortened', OLD.id + USING ERRCODE = '23514'; + END IF; + END IF; + + -- Identity, binding, and completion columns are frozen while the row + -- is STAGING. The state column is deliberately absent here: the named + -- lifecycle CHECK owns the shape so its refusal is classified by name. + IF NEW.id IS DISTINCT FROM OLD.id + OR NEW.project IS DISTINCT FROM OLD.project + OR NEW.generation IS DISTINCT FROM OLD.generation + OR NEW.scope_digest IS DISTINCT FROM OLD.scope_digest + OR NEW.resource_type IS DISTINCT FROM OLD.resource_type + OR NEW.source_selection_revision_id IS DISTINCT FROM OLD.source_selection_revision_id + OR NEW.source_membership_digest IS DISTINCT FROM OLD.source_membership_digest + OR NEW.definition_digest IS DISTINCT FROM OLD.definition_digest + OR NEW.membership_digest IS DISTINCT FROM OLD.membership_digest + OR NEW.group_count IS DISTINCT FROM OLD.group_count + OR NEW.member_count IS DISTINCT FROM OLD.member_count + OR NEW.idempotency_key IS DISTINCT FROM OLD.idempotency_key + OR NEW.created_at IS DISTINCT FROM OLD.created_at + OR NEW.completed_at IS DISTINCT FROM OLD.completed_at + THEN + RAISE EXCEPTION + 'explorer_explicit_group_revisions guard: frozen content of revision % changed', OLD.id + USING ERRCODE = '23514'; + END IF; + + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS explorer_explicit_group_revisions_guard ON explorer_explicit_group_revisions; +CREATE TRIGGER explorer_explicit_group_revisions_guard + BEFORE INSERT OR UPDATE OR DELETE ON explorer_explicit_group_revisions + FOR EACH ROW + EXECUTE FUNCTION loom_explorer_explicit_group_revision_guard(); + +CREATE OR REPLACE FUNCTION loom_explorer_explicit_group_membership_guard() RETURNS trigger +LANGUAGE plpgsql AS $$ +DECLARE + parent_state text; + parent_project text; + parent_generation text; + parent_resource_type text; + parent_writer_token text; + parent_expires_at timestamptz; + abort_token text := current_setting('loom.explorer_explicit_group_abort_token', true); + cleanup_now text := current_setting('loom.explorer_explicit_group_cleanup_now', true); + diag_rows integer; +BEGIN + -- The parent lookup resolves through TG_TABLE_SCHEMA so the guard + -- behaves identically no matter what search_path the calling session + -- carries (same discipline as the definitions guard). FOUND is + -- unreliable after EXECUTE ... INTO, so the row count drives the + -- absent-parent branch. The lookup locks the parent FOR KEY SHARE -- + -- the row-lock mode a foreign-key check takes, which conflicts with + -- every writer transition (the store claims the header FOR UPDATE) + -- -- so a concurrent completion can never flip the parent between + -- this read and the operation it authorizes: the intruding session + -- queues behind the completing transaction and then re-reads the + -- committed state. A plain SELECT here is a snapshot read and leaves + -- a TOCTOU window in which a direct-SQL INSERT (or an abort-channel + -- DELETE) is authorized against a stale STAGING image while the + -- parent completes in parallel; the store API fences its own paths, + -- but this guard is the last line for sessions that drive the tables + -- directly, which is exactly what it exists for. + EXECUTE format( + 'SELECT state, project, generation, resource_type, writer_token, writer_expires_at' + || ' FROM %I.explorer_explicit_group_revisions WHERE id = $1' + || ' FOR KEY SHARE', + TG_TABLE_SCHEMA + ) INTO parent_state, parent_project, parent_generation, parent_resource_type, + parent_writer_token, parent_expires_at + USING COALESCE(NEW.revision_id, OLD.revision_id); + GET DIAGNOSTICS diag_rows = ROW_COUNT; + IF diag_rows = 0 THEN + RAISE EXCEPTION + 'explorer_explicit_group_memberships guard: operation on membership of revision % refused: parent revision is absent', + COALESCE(NEW.revision_id, OLD.revision_id) + USING ERRCODE = '23514'; + END IF; + + IF TG_OP = 'INSERT' THEN + -- Memberships are staged, never backfilled: a COMPLETED revision + -- can never gain members, and every reference repeats the parent's + -- exact project/generation/resource-type scope (the domain's + -- Ref.Validate as a database fact). + IF parent_state <> 'STAGING' THEN + RAISE EXCEPTION + 'explorer_explicit_group_memberships guard: INSERT refused: parent revision % is % and only STAGING revisions accept members', + COALESCE(NEW.revision_id, OLD.revision_id), parent_state + USING ERRCODE = '23514'; + END IF; + IF NEW.project <> parent_project + OR NEW.generation <> parent_generation + OR NEW.resource_type <> parent_resource_type THEN + RAISE EXCEPTION + 'explorer_explicit_group_memberships guard: INSERT refused for revision %: reference scope %.%.% does not repeat the parent scope %.%.%', + NEW.revision_id, + NEW.project, NEW.generation, NEW.resource_type, + parent_project, parent_generation, parent_resource_type + USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END IF; + + IF TG_OP = 'UPDATE' THEN + IF ROW(NEW.*) IS NOT DISTINCT FROM ROW(OLD.*) THEN + RETURN NEW; + END IF; + RAISE EXCEPTION + 'explorer_explicit_group_memberships guard: membership % of revision % under group % is write-once', + OLD.resource_id, OLD.revision_id, OLD.group_id + USING ERRCODE = '23514'; + END IF; + + -- DELETE below: only ever as part of an authorized teardown of the + -- parent staging revision, under the same two GUC channels the + -- header and definition guards demand. + IF parent_state <> 'STAGING' THEN + RAISE EXCEPTION + 'explorer_explicit_group_memberships guard: DELETE refused: parent revision % is %', + OLD.revision_id, parent_state + USING ERRCODE = '23514'; + END IF; + IF abort_token IS NOT NULL AND parent_writer_token IS NOT NULL + AND abort_token = parent_writer_token THEN + RETURN OLD; -- owner abort cascade + END IF; + IF cleanup_now IS NOT NULL + AND (parent_expires_at IS NULL OR parent_expires_at < cleanup_now::timestamptz) THEN + RETURN OLD; -- expired-writer cleanup cascade under the sweep's clock + END IF; + RAISE EXCEPTION + 'explorer_explicit_group_memberships guard: DELETE of membership % of revision % under group % refused without owner-abort or expired-writer-cleanup authorization', + OLD.resource_id, OLD.revision_id, OLD.group_id + USING ERRCODE = '23514'; +END; +$$; + +DROP TRIGGER IF EXISTS explorer_explicit_group_memberships_guard ON explorer_explicit_group_memberships; +CREATE TRIGGER explorer_explicit_group_memberships_guard + BEFORE INSERT OR UPDATE OR DELETE ON explorer_explicit_group_memberships + FOR EACH ROW + EXECUTE FUNCTION loom_explorer_explicit_group_membership_guard(); diff --git a/internal/backends/postgres/internal/explorer/migrations/000011_explorer_explicit_group_definition_guard_hardening.sql b/internal/backends/postgres/internal/explorer/migrations/000011_explorer_explicit_group_definition_guard_hardening.sql new file mode 100644 index 000000000..1d3535fda --- /dev/null +++ b/internal/backends/postgres/internal/explorer/migrations/000011_explorer_explicit_group_definition_guard_hardening.sql @@ -0,0 +1,121 @@ +-- pg-state-explicit-group-members-v1 (CAS escalation): harden the +-- explicit-group definition guard without touching merged history. +-- +-- Migration 000009 is merged history and is never edited. Its +-- loom_explorer_explicit_group_definition_guard had two holes that the +-- CAS verifier turned into served-state witnesses: +-- +-- * The trigger fired BEFORE UPDATE OR DELETE only, so a direct-SQL +-- definition INSERT never fired the guard at all. The verifier +-- attached a definition to a header mid-completion and served +-- diverged facts. +-- * The DELETE parent lookup was a plain SELECT (a snapshot read), so +-- a definition DELETE racing a completion could be authorized +-- against a stale STAGING image while the parent completed in +-- parallel -- E1 admitted the deletion 3/3 deterministically, +-- leaving a frozen group_count of 2 over one stored definition row. +-- +-- This migration CREATE OR REPLACEs the same-named function with the +-- shape verified for the membership guard in 000010: a single shared +-- parent lookup before any branch decides, locked FOR KEY SHARE (the +-- foreign-key row-lock mode, which conflicts with every writer +-- transition -- the store claims the header FOR UPDATE -- so the +-- intruding session queues behind the completing transaction and +-- re-reads the committed state), plus the missing INSERT branch: +-- definitions are staged, never backfilled, and only a STAGING parent +-- accepts them. The trigger is recreated BEFORE INSERT OR UPDATE OR +-- DELETE. Authorization channels for DELETE are unchanged (STAGING +-- parent plus owner-abort token or expired-writer cleanup). +-- +-- Convergence: CREATE OR REPLACE FUNCTION plus DROP TRIGGER IF EXISTS + +-- CREATE TRIGGER re-establish the identical guard on a rerun, never a +-- weakened one; the migration records no state and never drops or +-- truncates. + +CREATE OR REPLACE FUNCTION loom_explorer_explicit_group_definition_guard() RETURNS trigger +LANGUAGE plpgsql AS $$ +DECLARE + parent_state text; + parent_writer_token text; + parent_expires_at timestamptz; + abort_token text := current_setting('loom.explorer_explicit_group_abort_token', true); + cleanup_now text := current_setting('loom.explorer_explicit_group_cleanup_now', true); + diag_rows integer; +BEGIN + -- Shared parent lookup, placed before every branch so no branch can + -- decide without it. Resolves through TG_TABLE_SCHEMA (search_path + -- discipline, same as the original) and locks the parent row -- the + -- TOCTOU close witnessed in 000010: without the row lock this is a + -- snapshot read, and a direct-SQL INSERT (or an abort-channel + -- DELETE) can be authorized against a parent image that a + -- concurrent completion is about to flip. FOUND is unreliable after + -- EXECUTE ... INTO, so the row count drives the absent-parent + -- branch. + EXECUTE format( + 'SELECT state, writer_token, writer_expires_at FROM %I.explorer_explicit_group_revisions WHERE id = $1' + || ' FOR KEY SHARE', + TG_TABLE_SCHEMA + ) INTO parent_state, parent_writer_token, parent_expires_at + USING COALESCE(NEW.revision_id, OLD.revision_id); + GET DIAGNOSTICS diag_rows = ROW_COUNT; + IF diag_rows = 0 THEN + RAISE EXCEPTION + 'explorer_explicit_group_definitions guard: operation on a definition of revision % refused: parent revision is absent', + COALESCE(NEW.revision_id, OLD.revision_id) + USING ERRCODE = '23514'; + END IF; + + IF TG_OP = 'INSERT' THEN + -- The closed hole: the 000009 trigger never fired here. A + -- completed revision can never gain definitions, staged or + -- otherwise -- same discipline the membership guard's INSERT + -- branch carries. + IF parent_state <> 'STAGING' THEN + RAISE EXCEPTION + 'explorer_explicit_group_definitions guard: INSERT refused: parent revision % is % and only STAGING revisions accept definitions', + COALESCE(NEW.revision_id, OLD.revision_id), parent_state + USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END IF; + + IF TG_OP = 'UPDATE' THEN + IF ROW(NEW.*) IS NOT DISTINCT FROM ROW(OLD.*) THEN + RETURN NEW; + END IF; + RAISE EXCEPTION + 'explorer_explicit_group_definitions guard: definition % of revision % is write-once', + OLD.group_id, OLD.revision_id + USING ERRCODE = '23514'; + END IF; + + -- DELETE below: only ever as part of an authorized teardown of the + -- parent staging revision -- same authorization as 000009, now read + -- under the locked lookup above so the authorization cannot be + -- stale. + IF parent_state <> 'STAGING' THEN + RAISE EXCEPTION + 'explorer_explicit_group_definitions guard: DELETE of definition % refused: parent revision % is %', + OLD.group_id, OLD.revision_id, parent_state + USING ERRCODE = '23514'; + END IF; + IF abort_token IS NOT NULL AND parent_writer_token IS NOT NULL + AND abort_token = parent_writer_token THEN + RETURN OLD; -- owner abort cascade + END IF; + IF cleanup_now IS NOT NULL + AND (parent_expires_at IS NULL OR parent_expires_at < cleanup_now::timestamptz) THEN + RETURN OLD; -- expired-writer cleanup cascade under the sweep's clock + END IF; + RAISE EXCEPTION + 'explorer_explicit_group_definitions guard: DELETE of definition % of revision % refused without owner-abort or expired-writer-cleanup authorization', + OLD.group_id, OLD.revision_id + USING ERRCODE = '23514'; +END; +$$; + +DROP TRIGGER IF EXISTS explorer_explicit_group_definitions_guard ON explorer_explicit_group_definitions; +CREATE TRIGGER explorer_explicit_group_definitions_guard + BEFORE INSERT OR UPDATE OR DELETE ON explorer_explicit_group_definitions + FOR EACH ROW + EXECUTE FUNCTION loom_explorer_explicit_group_definition_guard(); diff --git a/internal/backends/postgres/internal/explorer/migrations/000012_explorer_revision_activation_revival.sql b/internal/backends/postgres/internal/explorer/migrations/000012_explorer_revision_activation_revival.sql new file mode 100644 index 000000000..209d058ca --- /dev/null +++ b/internal/backends/postgres/internal/explorer/migrations/000012_explorer_revision_activation_revival.sql @@ -0,0 +1,130 @@ +-- pg-state-revision-activation-revival-v1 (guard evolution): admit the +-- activation-shaped FAILED->ACTIVE revival without editing merged history. +-- +-- Migration 000006 is merged history and is never edited. Its state machine +-- admits only READY->ACTIVE, READY->FAILED, ACTIVE->SUPERSEDED and +-- ACTIVE->FAILED, so a FAILED revision can never come back. That made one +-- consumer-visible outcome permanent on the PostgreSQL lane that the Arango +-- lane recovers from: the shared lifecycle marks the candidate FAILED on +-- every activation miss and a retried deploy converges onto the same +-- deterministic revision identity (content-digest keyed), so a lost +-- activation race pinned the content identity to a permanent refusal, while +-- the Arango baseline (candidate filter READY-or-FAILED plus the ACTIVE +-- replay arm, clearing failedAt/diagnostics on activation) reactivates on +-- retry. +-- +-- This migration CREATE OR REPLACEs loom_explorer_revision_guard with one +-- additional arm, following the sanctioned guard-evolution precedent of +-- 000011_explorer_explicit_group_definition_guard_hardening: same-named +-- function replaced whole, trigger dropped and recreated, convergence on +-- rerun, never a weakened one, no state recorded, no drops or truncates. +-- +-- The arm is the activation shape only -- strictly narrower than the Arango +-- baseline, which admits READY-or-FAILED blindly: +-- +-- * status must become exactly 'ACTIVE'; +-- * failed_at must be cleared to NULL (the activation write clears it); +-- * activated_at must be set (an activation always stamps it); +-- * diagnostics must be present (activation rewrites the diagnostic set, +-- never nulls it); +-- * ready_at must be unchanged (readiness is a one-time fact). +-- +-- A bare resurrection (FAILED->ACTIVE keeping failed_at, or with no +-- activated_at, or nulling diagnostics) is refused with 23514, and any +-- frozen-column smuggling during the transition is refused by the +-- untouched frozen check that runs first. All other FAILED departures +-- (FAILED->READY, FAILED->FAILED, FAILED->SUPERSEDED) and every transition +-- out of SUPERSEDED stay refused exactly as 000006 had them. The adapter's +-- revival write (candidateRevive, WHERE id = $1 AND status = 'FAILED') is +-- the only store path that produces the admitted shape; the first-CAS +-- statement candidateActive stays READY-only and the authoring publish path +-- is untouched. +-- +-- Convergence: CREATE OR REPLACE FUNCTION plus DROP TRIGGER IF EXISTS + +-- CREATE TRIGGER re-establish the identical guard on a rerun; the migration +-- records no ledger row of its own beyond the runner's and never weakens +-- the law. + +CREATE OR REPLACE FUNCTION loom_explorer_revision_guard() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + -- Insert-once against direct SQL: a DELETE followed by a re-INSERT of + -- different frozen content would fire no guard on the fresh row, so + -- deletions are refused outright. NEW is NULL in a BEFORE DELETE row + -- trigger, so the no-op shortcut and the frozen-column comparisons + -- below cannot be relied on for deletes; the refusal carries its own + -- message instead. + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION + 'explorer_revisions is insert-once: DELETE of revision % is refused', + OLD.id + USING ERRCODE = '23514'; + END IF; + + -- A true no-op (every column identical) changes nothing and stays legal. + IF ROW(NEW.*) IS NOT DISTINCT FROM ROW(OLD.*) THEN + RETURN NEW; + END IF; + + IF NEW.id IS DISTINCT FROM OLD.id + OR NEW.project IS DISTINCT FROM OLD.project + OR NEW.explorer_id IS DISTINCT FROM OLD.explorer_id + OR NEW.config IS DISTINCT FROM OLD.config + OR NEW.authoring_bundle IS DISTINCT FROM OLD.authoring_bundle + OR NEW.intent_digest IS DISTINCT FROM OLD.intent_digest + OR NEW.compilation_receipt_id IS DISTINCT FROM OLD.compilation_receipt_id + OR NEW.public_output_contract IS DISTINCT FROM OLD.public_output_contract + OR NEW.recipe IS DISTINCT FROM OLD.recipe + OR NEW.recipe_digest IS DISTINCT FROM OLD.recipe_digest + OR NEW.resolved_schema_digest IS DISTINCT FROM OLD.resolved_schema_digest + OR NEW.source_generation IS DISTINCT FROM OLD.source_generation + OR NEW.materializations IS DISTINCT FROM OLD.materializations + OR NEW.emitted_columns IS DISTINCT FROM OLD.emitted_columns + OR NEW.dataset IS DISTINCT FROM OLD.dataset + OR NEW.quality_reports IS DISTINCT FROM OLD.quality_reports + OR NEW.content_digest IS DISTINCT FROM OLD.content_digest + OR NEW.created_by IS DISTINCT FROM OLD.created_by + OR NEW.created_at IS DISTINCT FROM OLD.created_at + THEN + RAISE EXCEPTION + 'explorer revision % frozen content changed', OLD.id + USING ERRCODE = '23514'; + END IF; + + -- The 000012 arm: a FAILED revision may return to ACTIVE only through + -- the activation shape, so revival cannot forge an activation witness + -- and cannot bypass the READY path's CAS discipline. Every other + -- departure from FAILED stays illegal. + IF OLD.status = 'FAILED' AND NEW.status = 'ACTIVE' THEN + IF NEW.failed_at IS NULL + AND NEW.activated_at IS NOT NULL + AND NEW.diagnostics IS NOT NULL + AND NEW.ready_at IS NOT DISTINCT FROM OLD.ready_at + THEN + RETURN NEW; + END IF; + RAISE EXCEPTION + 'illegal explorer revision % status transition % -> %: FAILED->ACTIVE is admitted only as the activation-shaped write (failed_at cleared, activated_at set, diagnostics present, ready_at unchanged)', + OLD.id, OLD.status, NEW.status + USING ERRCODE = '23514'; + END IF; + + IF NOT ( + (OLD.status = 'READY' AND NEW.status IN ('ACTIVE', 'FAILED')) + OR (OLD.status = 'ACTIVE' AND NEW.status IN ('SUPERSEDED', 'FAILED')) + ) THEN + RAISE EXCEPTION + 'illegal explorer revision % status transition % -> %', + OLD.id, OLD.status, NEW.status + USING ERRCODE = '23514'; + END IF; + + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS explorer_revisions_guard ON explorer_revisions; +CREATE TRIGGER explorer_revisions_guard + BEFORE UPDATE OR DELETE ON explorer_revisions + FOR EACH ROW + EXECUTE FUNCTION loom_explorer_revision_guard(); diff --git a/internal/backends/postgres/internal/explorer/pgstate02_fixture_test.go b/internal/backends/postgres/internal/explorer/pgstate02_fixture_test.go new file mode 100644 index 000000000..6d534a32e --- /dev/null +++ b/internal/backends/postgres/internal/explorer/pgstate02_fixture_test.go @@ -0,0 +1,410 @@ +package postgres + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" +) + +var pgstate02NameRe = regexp.MustCompile(`^pgstate02_it_[0-9]+_[0-9a-f]+$`) + +func pgstate02TestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 Explorer receipt/revision integration tests") + } + return raw +} + +func pgstate02RandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +type pgState02Fixture struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dbName string + schema string + dsn string + scrub func(any) string + fail func(step string, err error) +} + +func newPGState02Fixture(t *testing.T) *pgState02Fixture { + t.Helper() + raw := pgstate02TestPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgstate02_it_%d_%s", time.Now().UnixNano(), pgstate02RandomHex(t)) + if !pgstate02NameRe.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + + t.Cleanup(func() { + var mine int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname = $1`, dbName).Scan(&mine); err != nil { + t.Errorf("leftover audit: query database catalog: %s", scrub(err)) + } + if mine != 0 { + t.Errorf("leftover audit: isolated database %s still exists", dbName) + } + var namespaceLeftovers int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname LIKE 'pgstate02\_it\_%' ESCAPE '\'`).Scan(&namespaceLeftovers); err != nil { + t.Errorf("leftover audit: query database namespace: %s", scrub(err)) + } + if namespaceLeftovers != 0 { + t.Errorf("leftover audit: %d pgstate02_it_* databases remain", namespaceLeftovers) + } + var schemaLeftovers int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgstate02\_it\_%' ESCAPE '\'`).Scan(&schemaLeftovers); err != nil { + t.Errorf("leftover audit: query schema namespace: %s", scrub(err)) + } + if schemaLeftovers != 0 { + t.Errorf("leftover audit: %d pgstate02_it_* schemas remain in the service database", schemaLeftovers) + } + }) + + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + if probe, err := url.Parse(dsn); err != nil || + probe.Query().Get("dbname") != "" || probe.Path != "/"+dbName { + t.Fatal("isolated test DSN does not provably target the isolated database") + } + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("pgstate02_it_%d_%s", time.Now().UnixNano(), pgstate02RandomHex(t)) + if !pgstate02NameRe.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + return &pgState02Fixture{ctx: ctx, admin: admin, iso: iso, dbName: dbName, schema: schema, dsn: dsn, scrub: scrub, fail: fail} +} + +func (f *pgState02Fixture) receiptTable() string { + return sanitizeTableRef(f.schema, ReceiptsTable) +} + +func (f *pgState02Fixture) revisionTable() string { + return sanitizeTableRef(f.schema, RevisionsTable) +} + +func (f *pgState02Fixture) migrate(t *testing.T) pgstore.MigrateResult { + t.Helper() + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, + MaxConns: 2, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + f.fail("open migration pool", err) + } + defer client.Close() + history, err := OwnerMigrations() + if err != nil { + f.fail("owner migrations", err) + } + runner, err := pgstore.NewMigrationRunner(client, f.schema, history) + if err != nil { + f.fail("new migration runner", err) + } + result, err := runner.Migrate(f.ctx) + if err != nil { + f.fail("migrate state schema", err) + } + return result +} + +func (f *pgState02Fixture) rerunMigration(t *testing.T, version int64) { + t.Helper() + migrations, err := OwnerMigrations() + if err != nil { + f.fail("load migrations for rerun", err) + } + for _, m := range migrations { + if m.Version == version { + conn, err := f.iso.Acquire(f.ctx) + if err != nil { + f.fail("acquire connection for migration rerun", err) + } + defer conn.Release() + tx, err := conn.Begin(f.ctx) + if err != nil { + f.fail("begin migration rerun transaction", err) + } + if _, err := tx.Exec(f.ctx, + "SET LOCAL search_path TO "+pgx.Identifier{f.schema}.Sanitize()); err != nil { + tx.Rollback(f.ctx) + f.fail("set search_path for migration rerun", err) + } + if _, err := tx.Exec(f.ctx, m.SQL); err != nil { + tx.Rollback(f.ctx) + f.fail(fmt.Sprintf("rerun migration %d/%s", m.Version, m.Name), err) + } + if err := tx.Commit(f.ctx); err != nil { + f.fail("commit migration rerun", err) + } + return + } + } + t.Fatalf("migration version %d is absent from the embedded history", version) +} + +func (f *pgState02Fixture) newStore(t *testing.T) *Store { + t.Helper() + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + f.fail("open pg-store pool", err) + } + t.Cleanup(client.Close) + store, err := New(client, f.schema) + if err != nil { + f.fail("build Explorer state store", err) + } + return store +} + +func (f *pgState02Fixture) newDedicatedStore(t *testing.T) (*Store, int32) { + t.Helper() + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, + MaxConns: 1, + MinConns: 1, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + f.fail("open dedicated pg-store pool", err) + } + t.Cleanup(client.Close) + conn, err := client.Acquire(f.ctx) + if err != nil { + f.fail("acquire dedicated pg-store connection", err) + } + defer conn.Release() + var pid int32 + if err := conn.QueryRow(f.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + f.fail("witness dedicated pg-store backend pid", err) + } + if pid == 0 { + t.Fatal("dedicated pg-store pool reported backend pid 0") + } + store, err := New(client, f.schema) + if err != nil { + f.fail("build dedicated Explorer state store", err) + } + return store, pid +} + +func (f *pgState02Fixture) newDedicatedPool(t *testing.T) (*pgxpool.Pool, int32) { + t.Helper() + poolConfig, err := pgxpool.ParseConfig(f.dsn) + if err != nil { + f.fail("parse dedicated raw pool config", err) + } + poolConfig.MaxConns = 1 + poolConfig.MinConns = 1 + pool, err := pgxpool.NewWithConfig(f.ctx, poolConfig) + if err != nil { + f.fail("open dedicated raw pool", err) + } + t.Cleanup(pool.Close) + var pid int32 + if err := pool.QueryRow(f.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + f.fail("witness dedicated raw backend pid", err) + } + if pid == 0 { + t.Fatal("dedicated raw pool reported backend pid 0") + } + return pool, pid +} + +func requireDistinctBackendsPGState02(t *testing.T, pidA, pidB int32, labelA, labelB string) { + t.Helper() + if pidA == pidB { + t.Fatalf("%s and %s share backend pid %d, want two real PG connections", labelA, labelB, pidA) + } +} + +func (f *pgState02Fixture) rowSnapshot(t *testing.T, table, where string, args ...any) string { + t.Helper() + rows, err := f.iso.Query(f.ctx, + `SELECT (to_jsonb(t))::text FROM `+table+` t WHERE `+where, args...) + if err != nil { + f.fail("snapshot row in "+table, err) + } + defer rows.Close() + var out string + var count int + for rows.Next() { + if count > 0 { + t.Fatalf("snapshot matched more than one row in %s", table) + } + if err := rows.Scan(&out); err != nil { + f.fail("scan row snapshot in "+table, err) + } + count++ + } + if err := rows.Err(); err != nil { + f.fail("snapshot rows in "+table, err) + } + if count == 0 { + t.Fatalf("snapshot matched no row in %s (%s)", table, where) + } + return out +} + +func (f *pgState02Fixture) frozenSnapshot(t *testing.T, id string) string { + t.Helper() + var out string + err := f.iso.QueryRow(f.ctx, `SELECT concat_ws(E'\x1f', + id, project, explorer_id, + COALESCE(config::text, ''), + COALESCE(authoring_bundle::text, ''), + intent_digest, compilation_receipt_id, + COALESCE(public_output_contract::text, ''), + recipe::text, recipe_digest, resolved_schema_digest, source_generation, + COALESCE(materializations::text, ''), + COALESCE(emitted_columns::text, ''), + dataset::text, + COALESCE(quality_reports::text, ''), + content_digest, created_by, + to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US')) + FROM `+f.revisionTable()+` WHERE id = $1`, id).Scan(&out) + if err != nil { + f.fail("frozen snapshot of revision "+id, err) + } + return out +} + +func (f *pgState02Fixture) columnKinds(t *testing.T, table, where string, args []any, columns ...string) []string { + t.Helper() + selects := make([]string, len(columns)) + for i, c := range columns { + selects[i] = fmt.Sprintf("COALESCE(jsonb_typeof(%s), 'missing')", c) + } + rows, err := f.iso.Query(f.ctx, "SELECT "+strings.Join(selects, ", ")+" FROM "+table+" WHERE "+where, args...) + if err != nil { + f.fail("read jsonb kinds from "+table, err) + } + defer rows.Close() + var out []string + for rows.Next() { + targets := make([]string, len(columns)) + scanTargets := make([]any, len(columns)) + for i := range targets { + scanTargets[i] = &targets[i] + } + if err := rows.Scan(scanTargets...); err != nil { + f.fail("scan jsonb kinds from "+table, err) + } + out = append(out, targets...) + } + if err := rows.Err(); err != nil { + f.fail("read jsonb kinds from "+table, err) + } + if len(out) != len(columns) { + t.Fatalf("jsonb kinds matched %d rows, want exactly 1 in %s", len(out), table) + } + return out +} + +func requirePgErrPGState02(t *testing.T, err error, sqlstate, step string) *pgconn.PgError { + t.Helper() + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + t.Fatalf("%s: error = %v (%T), want *pgconn.PgError", step, err, err) + } + if pgErr.Code != sqlstate { + t.Fatalf("%s: sqlstate = %s, want %s (error: %v)", step, pgErr.Code, sqlstate, err) + } + return pgErr +} diff --git a/internal/backends/postgres/internal/explorer/pubauth_fixture_test.go b/internal/backends/postgres/internal/explorer/pubauth_fixture_test.go new file mode 100644 index 000000000..2a962d494 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/pubauth_fixture_test.go @@ -0,0 +1,756 @@ +package postgres + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + datasetpostgres "github.com/calypr/loom/internal/backends/postgres/internal/dataset" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/explorer" +) + +var pubauthNameRe = regexp.MustCompile(`^verify_pubauth_[0-9]+_[0-9a-f]+(?:_[a-z]+)?$`) + +var ( + pubauthBaselineOnce sync.Once + pubauthDBBaseline atomic.Int64 + pubauthLive atomic.Int64 + pubauthSchemaBaseline atomic.Int64 + pubauthBaselineFailure atomic.Value +) + +func pubauthCaptureBaseline(admin *pgxpool.Pool) { + pubauthBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + var mine, schemas int64 + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname LIKE 'verify_pubauth\_%' ESCAPE '\'`).Scan(&mine); err != nil { + pubauthBaselineFailure.Store(fmt.Errorf("baseline: query owned database namespace: %w", err)) + return + } + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'verify_pubauth\_%' ESCAPE '\'`).Scan(&schemas); err != nil { + pubauthBaselineFailure.Store(fmt.Errorf("baseline: query owned schema namespace: %w", err)) + return + } + pubauthDBBaseline.Store(mine) + pubauthSchemaBaseline.Store(schemas) + }) + if stored := pubauthBaselineFailure.Load(); stored != nil { + panic(stored.(error)) + } +} + +func pubauthTestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 publish-authoring integration tests") + } + return raw +} + +func pubauthRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +type pubAuthFixture struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dbName string + ctlSchema string + lanesSchema string + dsn string + scrub func(any) string + fail func(step string, err error) +} + +func newPGPubAuthFixture(t *testing.T) *pubAuthFixture { + t.Helper() + raw := pubauthTestPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + pubauthCaptureBaseline(admin) + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("verify_pubauth_%d_%s", time.Now().UnixNano(), pubauthRandomHex(t)) + if !pubauthNameRe.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + + t.Cleanup(func() { + var mine int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname = $1`, dbName).Scan(&mine); err != nil { + t.Errorf("leftover audit: query database catalog: %s", scrub(err)) + } + if mine != 0 { + t.Errorf("leftover audit: isolated database %s still exists", dbName) + } + var namespaceDatabases int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname LIKE 'verify_pubauth\_%' ESCAPE '\'`).Scan(&namespaceDatabases); err != nil { + t.Errorf("leftover audit: query database namespace: %s", scrub(err)) + } + if want := pubauthDBBaseline.Load() + pubauthLive.Load(); int64(namespaceDatabases) != want { + t.Errorf("leftover audit: %d verify_pubauth_* databases remain, baseline+live is %d", + namespaceDatabases, want) + } + var namespaceSchemas int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'verify_pubauth\_%' ESCAPE '\'`).Scan(&namespaceSchemas); err != nil { + t.Errorf("leftover audit: query schema namespace: %s", scrub(err)) + } + if int64(namespaceSchemas) != pubauthSchemaBaseline.Load() { + t.Errorf("leftover audit: %d verify_pubauth_* schemas remain in the service database, baseline is %d", + namespaceSchemas, pubauthSchemaBaseline.Load()) + } + }) + + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + pubauthLive.Add(-1) + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + pubauthLive.Add(1) + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + if probe, err := url.Parse(dsn); err != nil || + probe.Query().Get("dbname") != "" || probe.Path != "/"+dbName { + t.Fatal("isolated test DSN does not provably target the isolated database") + } + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + base := fmt.Sprintf("verify_pubauth_%d_%s", time.Now().UnixNano(), pubauthRandomHex(t)) + ctlSchema := base + "_ctl" + lanesSchema := base + "_lanes" + for _, schema := range []string{ctlSchema, lanesSchema} { + if !pubauthNameRe.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema "+schema, err) + } + } + t.Cleanup(func() { + for _, schema := range []string{lanesSchema, ctlSchema} { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema %s: %s", schema, scrub(err)) + } + } + }) + + return &pubAuthFixture{ + ctx: ctx, admin: admin, iso: iso, dbName: dbName, + ctlSchema: ctlSchema, lanesSchema: lanesSchema, + dsn: dsn, scrub: scrub, fail: fail, + } +} + +type pubauthMigrationPin struct { + version int64 + name string + checksum string +} + +var ( + pubauthControlPins = []pubauthMigrationPin{ + {4, "capability_artifacts", "bec3e49f897b28ef71d7d69a5f7c71bb11d13b7c631ee26724de36a89d639222"}, + {7, "interpretation_library", "5e308b21a3f567b3066c46458606dd361a9c90646db54ee71618902a544a9ba2"}, + {8, "selection_revisions", "562a61ef33786cf674f1c87552e2dc8e70691b34116b7f415d5f81960ed8d65d"}, + } + pubauthLanePins = []pubauthMigrationPin{ + {1, "ledger", "db70f7758495f7557b3a694149b80ed6e5d4cb74ea5aa7d59a1a2eaa46fc9cde"}, + {3, "dataset_generation_manifest", "06c614680a9c1cd4ba2ef0ac0471053bcb3d8e471c36d3852e70682887a72f5d"}, + {8, "project_release", "1d581ec39731958dd0f79cdab18e92861a380007d3e85d6de4940800378480b1"}, + } +) + +func pubauthVerifyPins(t *testing.T, label string, history []pgstore.Migration, pins []pubauthMigrationPin) { + t.Helper() + byVersion := make(map[int64]pgstore.Migration, len(history)) + for _, m := range history { + byVersion[m.Version] = m + } + for _, pin := range pins { + m, ok := byVersion[pin.version] + if !ok { + t.Fatalf("%s owner history lacks version %d (%s): the owning ticket's migration vanished", label, pin.version, pin.name) + } + if m.Name != pin.name || m.Checksum != pin.checksum { + t.Fatalf("%s owner history version %d = {Name:%q Checksum:%q}, want the byte-frozen merge pin {Name:%q Checksum:%q}: the owning ticket's migration drifted", + label, pin.version, m.Name, m.Checksum, pin.name, pin.checksum) + } + } +} + +func (f *pubAuthFixture) migrate(t *testing.T) { + t.Helper() + ctlHistory, err := OwnerMigrations() + if err != nil { + f.fail("explorer owner migrations", err) + } + pubauthVerifyPins(t, "explorer control", ctlHistory, pubauthControlPins) + lanesHistory, err := datasetpostgres.OwnerMigrations() + if err != nil { + f.fail("dataset owner migrations", err) + } + pubauthVerifyPins(t, "dataset lane", lanesHistory, pubauthLanePins) + + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, MaxConns: 2, MinConns: 0, AcquireTimeout: 10 * time.Second, + }) + if err != nil { + f.fail("open migration pool", err) + } + defer client.Close() + + ctl := f.migrateHistory(t, client, f.ctlSchema, ctlHistory, 12, []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}) + lanes := f.migrateHistory(t, client, f.lanesSchema, lanesHistory, 8, []int64{1, 3, 8}) + if ctl.LatestVersion != 12 || lanes.LatestVersion != 8 { + t.Fatalf("ledger tops = %d/%d, want 12/8", ctl.LatestVersion, lanes.LatestVersion) + } +} + +func (f *pubAuthFixture) migrateHistory(t *testing.T, client *pgstore.Client, schema string, + history []pgstore.Migration, wantTop int64, wantVersions []int64, +) pgstore.MigrateResult { + t.Helper() + runner, err := pgstore.NewMigrationRunner(client, schema, history) + if err != nil { + f.fail("new migration runner for "+schema, err) + } + result, err := runner.Migrate(f.ctx) + if err != nil { + f.fail("migrate schema "+schema, err) + } + again, err := runner.Migrate(f.ctx) + if err != nil { + f.fail("re-migrate schema "+schema, err) + } + if len(result.Applied) != len(wantVersions) || len(again.Applied) != len(wantVersions) { + t.Fatalf("schema %s applied %d/%d migrations, want %d", schema, len(result.Applied), len(again.Applied), len(wantVersions)) + } + if result.LatestVersion != wantTop || again.LatestVersion != wantTop { + t.Fatalf("schema %s latest version = %d/%d, want %d", schema, result.LatestVersion, again.LatestVersion, wantTop) + } + for i, want := range wantVersions { + if result.Applied[i].Version != want || again.Applied[i].Version != want { + t.Fatalf("schema %s ledger position %d = %d/%d, want %d", + schema, i, result.Applied[i].Version, again.Applied[i].Version, want) + } + } + return result +} + +func (f *pubAuthFixture) rerunMigration(t *testing.T, version int64) { + t.Helper() + migrations, err := OwnerMigrations() + if err != nil { + f.fail("load migrations for rerun", err) + } + for _, m := range migrations { + if m.Version == version { + conn, err := f.iso.Acquire(f.ctx) + if err != nil { + f.fail("acquire connection for migration rerun", err) + } + defer conn.Release() + tx, err := conn.Begin(f.ctx) + if err != nil { + f.fail("begin migration rerun transaction", err) + } + if _, err := tx.Exec(f.ctx, + "SET LOCAL search_path TO "+pgx.Identifier{f.ctlSchema}.Sanitize()); err != nil { + tx.Rollback(f.ctx) + f.fail("set search_path for migration rerun", err) + } + if _, err := tx.Exec(f.ctx, m.SQL); err != nil { + tx.Rollback(f.ctx) + f.fail(fmt.Sprintf("rerun migration %d/%s", m.Version, m.Name), err) + } + if err := tx.Commit(f.ctx); err != nil { + f.fail("commit migration rerun", err) + } + return + } + } + t.Fatalf("control-plane migration version %d is absent from the embedded history", version) +} + +func (f *pubAuthFixture) newStore(t *testing.T) *Store { + t.Helper() + client := f.newPool(t, 4, 0) + store, err := New(client, f.ctlSchema) + if err != nil { + f.fail("build Explorer control-plane store", err) + } + return store +} + +func (f *pubAuthFixture) newDedicatedStore(t *testing.T) (*Store, int32) { + t.Helper() + client := f.newPool(t, 1, 1) + conn, err := client.Acquire(f.ctx) + if err != nil { + f.fail("acquire dedicated control-plane connection", err) + } + defer conn.Release() + var pid int32 + if err := conn.QueryRow(f.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + f.fail("witness dedicated control-plane backend pid", err) + } + if pid == 0 { + t.Fatal("dedicated control-plane pool reported backend pid 0") + } + store, err := New(client, f.ctlSchema) + if err != nil { + f.fail("build dedicated control-plane store", err) + } + return store, pid +} + +func (f *pubAuthFixture) newPublishStore(t *testing.T) *PublishStore { + t.Helper() + client := f.newPool(t, 4, 0) + ps, err := NewPublishStore(client, f.ctlSchema, f.lanesSchema) + if err != nil { + f.fail("build publish store", err) + } + return ps +} + +func (f *pubAuthFixture) newDedicatedPublishStore(t *testing.T) (*PublishStore, int32) { + t.Helper() + client := f.newPool(t, 1, 1) + conn, err := client.Acquire(f.ctx) + if err != nil { + f.fail("acquire dedicated publish connection", err) + } + defer conn.Release() + var pid int32 + if err := conn.QueryRow(f.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + f.fail("witness dedicated publish backend pid", err) + } + if pid == 0 { + t.Fatal("dedicated publish pool reported backend pid 0") + } + ps, err := NewPublishStore(client, f.ctlSchema, f.lanesSchema) + if err != nil { + f.fail("build dedicated publish store", err) + } + return ps, pid +} + +func (f *pubAuthFixture) newDedicatedPool(t *testing.T) (*pgxpool.Pool, int32) { + t.Helper() + poolConfig, err := pgxpool.ParseConfig(f.dsn) + if err != nil { + f.fail("parse dedicated raw pool config", err) + } + poolConfig.MaxConns = 1 + poolConfig.MinConns = 1 + pool, err := pgxpool.NewWithConfig(f.ctx, poolConfig) + if err != nil { + f.fail("open dedicated raw pool", err) + } + t.Cleanup(pool.Close) + var pid int32 + if err := pool.QueryRow(f.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + f.fail("witness dedicated raw backend pid", err) + } + if pid == 0 { + t.Fatal("dedicated raw pool reported backend pid 0") + } + return pool, pid +} + +func (f *pubAuthFixture) newPool(t *testing.T, maxConns, minConns int32) *pgstore.Client { + t.Helper() + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, MaxConns: maxConns, MinConns: minConns, AcquireTimeout: 10 * time.Second, + }) + if err != nil { + f.fail("open pg-store pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (f *pubAuthFixture) newGenerationStore(t *testing.T) *datasetpostgres.Store { + t.Helper() + client := f.newPool(t, 2, 0) + gen, err := datasetpostgres.New(client, f.lanesSchema) + if err != nil { + f.fail("build generation store", err) + } + return gen +} + +func (f *pubAuthFixture) newReleaseStore(t *testing.T) *datasetpostgres.ReleaseStore { + t.Helper() + client := f.newPool(t, 2, 0) + rel, err := datasetpostgres.NewReleaseStore(client, f.lanesSchema) + if err != nil { + f.fail("build release store", err) + } + return rel +} + +func requireDistinctBackendsPubAuth(t *testing.T, pidA, pidB int32, labelA, labelB string) { + t.Helper() + if pidA == pidB { + t.Fatalf("%s and %s share backend pid %d, want two real PG connections", labelA, labelB, pidA) + } +} + +func (f *pubAuthFixture) ctlTable(table string) string { + return sanitizeTableRef(f.ctlSchema, table) +} + +func (f *pubAuthFixture) lanesTable(table string) string { + return sanitizeTableRef(f.lanesSchema, table) +} + +func (f *pubAuthFixture) snapshot(t *testing.T, table string, mask ...string) string { + t.Helper() + return f.snapshotWhere(t, table, "", nil, mask...) +} + +func (f *pubAuthFixture) snapshotWhere(t *testing.T, table, where string, args []any, mask ...string) string { + t.Helper() + expr := "to_jsonb(t)" + for _, column := range mask { + expr += fmt.Sprintf(" - '%s'", column) + } + query := "SELECT COALESCE(string_agg(row_text, E'\\x1e' ORDER BY row_text), '') FROM (SELECT (" + + expr + ")::text AS row_text FROM " + table + " t" + if where != "" { + query += " WHERE " + where + } + query += ") s" + var out string + if err := f.iso.QueryRow(f.ctx, query, args...).Scan(&out); err != nil { + f.fail("snapshot table "+table, err) + } + return out +} + +func (f *pubAuthFixture) countRows(t *testing.T, table, where string, args ...any) int { + t.Helper() + query := "SELECT count(*) FROM " + table + if where != "" { + query += " WHERE " + where + } + var count int + if err := f.iso.QueryRow(f.ctx, query, args...).Scan(&count); err != nil { + f.fail("count rows in "+table, err) + } + return count +} + +func (f *pubAuthFixture) execRaw(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := f.iso.Exec(f.ctx, sql, args...); err != nil { + f.fail("direct exec: "+sql, err) + } +} + +func (f *pubAuthFixture) locksHeldBy(t *testing.T, pool *pgxpool.Pool, pid int32) map[string]bool { + t.Helper() + rows, err := pool.Query(f.ctx, ` +SELECT c.relname +FROM pg_locks l +JOIN pg_class c ON c.oid = l.relation +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE l.pid = $1 AND l.relation IS NOT NULL AND n.nspname = ANY($2::text[])`, + pid, []string{f.ctlSchema, f.lanesSchema}) + if err != nil { + f.fail("read pg_locks for pid "+fmt.Sprint(pid), err) + } + defer rows.Close() + out := map[string]bool{} + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + f.fail("scan pg_locks row", err) + } + out[name] = true + } + if err := rows.Err(); err != nil { + f.fail("read pg_locks rows", err) + } + return out +} + +func requirePgErrPubAuth(t *testing.T, err error, sqlstate, step string) *pgconn.PgError { + t.Helper() + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + t.Fatalf("%s: error = %v (%T), want *pgconn.PgError", step, err, err) + } + if pgErr.Code != sqlstate { + t.Fatalf("%s: sqlstate = %s, want %s (error: %v)", step, pgErr.Code, sqlstate, err) + } + return pgErr +} + +var ( + pubauthCreatedAt = time.Date(2026, 4, 5, 6, 7, 8, 987654000, time.UTC) + pubauthVerifiedAt = time.Date(2026, 4, 5, 6, 7, 9, 123456000, time.UTC) +) + +const pubauthGeneration = "gen_pubauth_01" + +const pubauthSnapshotSHA256 = "2be6ca3efe274cf87ad45c17a516bc91f154e64de9e557516b973f0d22314341" + +func pubauthOwner(project, explorerID string, mode explorer.ManagementMode) explorer.Explorer { + return explorer.Explorer{ + Project: project, + ExplorerID: explorerID, + Title: "pubauth " + explorerID, + ManagementMode: mode, + DraftVersion: 1, + DraftDigest: "sha256:draft-" + explorerID, + UpdatedBy: "author@calypr", + UpdatedAt: pubauthCreatedAt, + } +} + +const ( + cfgPubauthAlpha = `{"title": "PubAuth", "filters": [{"column": "day", "op": "eq", "value": "2026-01-01"}], "limit": 25}` + cfgPubauthBeta = `{"a": 1, "m": "keep", "z": [true, false, null]}` + cfgPubauthContract = `{"outputs":[{"outputId":"output","columns":[],"lossless":true,"mlReady":true}]}` +) + +func jsonRaw(s string) json.RawMessage { return json.RawMessage(s) } + +func newPubauthBundle(name string) recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: name, + TranslationVersion: "pubauth", + Outputs: []recipe.Output{{ + Name: "output", RootResourceType: "Patient", RowGrain: "resource", + }}, + } +} + +func pubauthReceipt(t *testing.T, project, explorerID, tag string) explorer.CompilationReceipt { + t.Helper() + bundle := newPubauthBundle("pubauth-" + tag) + receipt := explorer.CompilationReceipt{ + ReceiptFormatVersion: explorer.CurrentReceiptFormatVersion, + CompilerContractVersion: explorer.CurrentCompilerContractVersion, + Project: project, + ExplorerID: explorerID, + IntentDigest: "sha256:intent-" + tag, + SnapshotToken: "snapshot-pubauth-" + tag, + AuthorizationScopeDigest: "scope-pubauth", + CapabilitySchemaDigest: "schema-pubauth", + ShapeDigest: "shape-pubauth", + SourceGeneration: pubauthGeneration, + RecipeDigest: "sha256:recipe-" + tag, + ResolvedSchemaDigest: "sha256:resolved-" + tag, + NormalizedBundle: jsonRaw(cfgPubauthAlpha), + Bundle: bundle, + PublicOutputContract: jsonRaw(`{"outputs":[{"outputId":"output","columns":[],"lossless":true,"mlReady":true}]}`), + OutputColumnProvenance: map[string]map[string]string{"output": {"__loom_row_id": "EXPLICIT"}}, + RequestID: "req_" + tag, + CreatedAt: pubauthCreatedAt, + } + var err error + if receipt.ResolvedRecipeDigest, err = bundle.Digest(); err != nil { + t.Fatalf("bundle digest: %v", err) + } + if receipt.OutputContractDigest, err = explorer.CompilationArtifactDigest(receipt.PublicOutputContract); err != nil { + t.Fatalf("output contract digest: %v", err) + } + if receipt.CompilationKey, err = explorer.CompilationKey(receipt); err != nil { + t.Fatalf("compilation key: %v", err) + } + if receipt.ID, err = explorer.ReceiptID(receipt); err != nil { + t.Fatalf("receipt id: %v", err) + } + return receipt +} + +func pubauthManualReceipt(t *testing.T, project, explorerID, tag, id string) explorer.CompilationReceipt { + t.Helper() + receipt := pubauthReceipt(t, project, explorerID, tag) + receipt.ReceiptFormatVersion = 0 + receipt.CompilerContractVersion = "" + receipt.CompilationKey = "" + receipt.ID = id + return receipt +} + +func pubauthRevision(id, project, explorerID, receiptID, tag string) explorer.Revision { + return explorer.Revision{ + ID: id, + Project: project, + ExplorerID: explorerID, + Config: jsonRaw(cfgPubauthAlpha), + AuthoringBundle: jsonRaw(cfgPubauthBeta), + IntentDigest: "sha256:pubauth-intent-" + tag, + CompilationReceiptID: receiptID, + PublicOutputContract: jsonRaw(cfgPubauthContract), + Recipe: newPubauthBundle("pubauth-" + tag), + RecipeDigest: "sha256:pubauth-recipe-" + tag, + ResolvedSchemaDigest: "sha256:pubauth-schema-" + tag, + SourceGeneration: pubauthGeneration, + Materializations: []explorer.Materialization{{ + OutputID: "output", Output: "output", MaterializationID: "mat_pubauth_1", + Fingerprint: "fp-pubauth-1", + }}, + EmittedColumns: []explorer.EmittedColumn{{ + EmissionID: "emit_1", OutputID: "output", PublicColumn: "day", + LogicalType: "date", Shape: "scalar", Filterable: true, + }}, + Dataset: explorer.DatasetMetadata{ + Generation: pubauthGeneration, + SchemaDigest: "sha256:pubauth-dataset", + Outputs: []explorer.DatasetOutput{{Name: "output", State: "QUERYABLE", Queryable: true}}, + }, + Publication: explorer.PublicationMetadata{ + State: string(explorer.RevisionReady), + Generation: pubauthGeneration, + ExecutionID: "exec_pubauth_" + tag, + UpdatedAt: pubauthCreatedAt, + }, + Status: explorer.RevisionReady, + CreatedBy: "author@calypr", + CreatedAt: pubauthCreatedAt, + ReadyAt: timePtr(pubauthCreatedAt), + } +} + +func pubauthRelease(project, generation, id string) dataset.ProjectRelease { + return dataset.ProjectRelease{ + ID: id, + Project: project, + GitCommit: "commit-" + id, + Generation: generation, + Publications: []dataset.ReleasePublication{{ + Selector: dataset.DataframeSelector{Recipe: "recipe-pubauth", TranslationVersion: "v1", Output: "output"}, + ExecutionID: "exec_pubauth_1", Generation: generation, Required: true, VerifiedAt: pubauthVerifiedAt, + }}, + RequiredVerifications: []dataset.ContractVerification{{ + Selector: dataset.DataframeSelector{Recipe: "recipe-pubauth", TranslationVersion: "v1", Output: "output"}, + ExecutionID: "exec_pubauth_1", Generation: generation, State: "PUBLISHED", Queryable: true, VerifiedAt: pubauthVerifiedAt, + }}, + CreatedAt: pubauthCreatedAt, + } +} + +func (f *pubAuthFixture) seedPublishedWorld(t *testing.T, store *Store, project, explorerID, tag string, +) (explorer.CompilationReceipt, explorer.Revision, dataset.ProjectRelease) { + t.Helper() + if _, err := store.Create(f.ctx, pubauthOwner(project, explorerID, explorer.ManagementInteractive)); err != nil { + f.fail("create owner "+project+"/"+explorerID, err) + } + stageAndActivate(t, f, f.newGenerationStore(t), project, pubauthGeneration, nil) + release := pubauthRelease(project, pubauthGeneration, "rel_"+project+"_"+tag) + rel := f.newReleaseStore(t) + if _, err := rel.SaveRelease(f.ctx, release); err != nil { + f.fail("save release candidate "+release.ID, err) + } + receipt := pubauthReceipt(t, project, explorerID, tag) + revision := pubauthRevision("rev_"+project+"_"+tag, project, explorerID, receipt.ID, tag) + return receipt, revision, release +} + +func (f *pubAuthFixture) publishOK(t *testing.T, ps *PublishStore, + receipt explorer.CompilationReceipt, revision explorer.Revision, + release dataset.ProjectRelease, expected int64, +) *explorer.Revision { + t.Helper() + out, err := ps.PublishAuthoring(f.ctx, receipt, revision, release, expected) + if err != nil { + f.fail("PublishAuthoring "+revision.ID, err) + } + return out +} + +func timePtr(v time.Time) *time.Time { return &v } + +func installCrashSeam(ps *PublishStore, gate func(step string, tx pgx.Tx) error) *[]string { + steps := &[]string{} + ps.crashAt = func(ctx context.Context, tx pgx.Tx, step string) error { + *steps = append(*steps, step) + if gate != nil { + return gate(step, tx) + } + return nil + } + return steps +} diff --git a/internal/backends/postgres/internal/explorer/publication.go b/internal/backends/postgres/internal/explorer/publication.go new file mode 100644 index 000000000..2b4c0ee08 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/publication.go @@ -0,0 +1,663 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/calypr/loom/internal/explorer" + + datasetpostgres "github.com/calypr/loom/internal/backends/postgres/internal/dataset" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/dataset" +) + +const ( + publishTxMaxAttempts = 4 + publishDeadlockRetry = 5 * time.Millisecond + publishDiagnosticsSet = "[]" +) + +var ErrInvalidPublicationInput = errors.New("postgres explorer store: invalid publish authoring input") + +var ErrPublishContention = errors.New("postgres explorer store: publish authoring lost every deadlock race") + +type PublishStore struct { + client *pgstore.Client + explorerSchema string + datasetSchema string + stmts publishStatements + crashAt func(context.Context, pgx.Tx, string) error +} + +func NewPublishStore(client *pgstore.Client, explorerSchema, datasetSchema string) (*PublishStore, error) { + if client == nil { + return nil, errors.New("postgres explorer store: publish client is required") + } + for _, schema := range []string{explorerSchema, datasetSchema} { + if !pgstore.ValidSchemaName(schema) { + return nil, fmt.Errorf("postgres explorer store: invalid publish schema %q: must match [a-z_][a-z0-9_]{0,62}", schema) + } + } + return &PublishStore{ + client: client, + explorerSchema: explorerSchema, + datasetSchema: datasetSchema, + stmts: newPublishStatements(explorerSchema, datasetSchema), + }, nil +} + +type publishStatements struct { + receiptInsert string + receiptClaim string + revisionInsert string + revisionClaim string + releaseCandidateLock string + manifestLock string + generationPointer string + releasePointerLock string + releasePointerRead string + releasePointerCAS string + releasePointerInsert string + ownerGuard string + priorLock string + priorSupersede string + candidateActive string + candidateRevive string + ownerActivate string +} + +func newPublishStatements(explorerSchema, datasetSchema string) publishStatements { + ownerTable := sanitizeTableRef(explorerSchema, OwnerTable) + ownerCols := strings.Join(ownerColumns, ", ") + manifests := sanitizeTableRef(datasetSchema, "dataset_generation_manifest") + generationPointer := sanitizeTableRef(datasetSchema, "dataset_active_generation") + releases := sanitizeTableRef(datasetSchema, datasetpostgres.ProjectReleasesTable) + pointers := sanitizeTableRef(datasetSchema, datasetpostgres.ActiveProjectReleasesTable) + return publishStatements{ + receiptInsert: newReceiptStatements(explorerSchema).insert, + receiptClaim: newReceiptStatements(explorerSchema).claim, + revisionInsert: newRevisionStatements(explorerSchema).insert, + revisionClaim: newRevisionStatements(explorerSchema).claim, + releaseCandidateLock: "SELECT payload::text, (payload - 'createdAt') = ($3::jsonb - 'createdAt') FROM " + releases + + " WHERE project = $1 AND id = $2 FOR SHARE", + manifestLock: "SELECT state FROM " + manifests + " WHERE project = $1 AND generation = $2 FOR SHARE", + generationPointer: "SELECT generation FROM " + generationPointer + " WHERE project = $1 FOR SHARE", + releasePointerLock: "SELECT release_id, revision FROM " + pointers + " WHERE project = $1 FOR UPDATE", + releasePointerRead: "SELECT release_id, revision FROM " + pointers + " WHERE project = $1", + releasePointerCAS: "UPDATE " + pointers + " SET release_id = $2, revision = $3, updated_at = now()" + + " WHERE project = $1 AND revision = $4 RETURNING revision", + releasePointerInsert: "INSERT INTO " + pointers + " (project, release_id, revision) VALUES ($1, $2, 1) ON CONFLICT DO NOTHING", + ownerGuard: "SELECT " + ownerCols + " FROM " + ownerTable + + " WHERE project = $1 AND explorer_id = $2 AND management_mode = $3 FOR UPDATE", + priorLock: "SELECT " + strings.Join(revisionColumns, ", ") + " FROM " + sanitizeTableRef(explorerSchema, RevisionsTable) + " WHERE id = $1 FOR UPDATE", + priorSupersede: "UPDATE " + sanitizeTableRef(explorerSchema, RevisionsTable) + " SET status = 'SUPERSEDED' WHERE id = $1 AND status = 'ACTIVE' RETURNING " + strings.Join(revisionColumns, ", "), + candidateActive: "UPDATE " + sanitizeTableRef(explorerSchema, RevisionsTable) + + " SET status = 'ACTIVE', activated_at = $2, failed_at = NULL, diagnostics = $3, publication = $4" + + " WHERE id = $1 AND status = 'READY' RETURNING " + strings.Join(revisionColumns, ", "), + candidateRevive: "UPDATE " + sanitizeTableRef(explorerSchema, RevisionsTable) + + " SET status = 'ACTIVE', activated_at = $2, failed_at = NULL, diagnostics = $3, publication = $4" + + " WHERE id = $1 AND status = 'FAILED' RETURNING " + strings.Join(revisionColumns, ", "), + ownerActivate: "UPDATE " + ownerTable + " SET active_revision_id = $3, updated_at = $4" + + " WHERE project = $1 AND explorer_id = $2 RETURNING " + ownerCols, + } +} + +func (p *PublishStore) PublishAuthoring(ctx context.Context, receipt explorer.CompilationReceipt, revision explorer.Revision, release dataset.ProjectRelease, expectedReleaseRevision int64) (*explorer.Revision, error) { + if err := validateReceiptForStore(receipt); err != nil { + return nil, fmt.Errorf("postgres explorer store: publish authoring: %w", err) + } + if strings.TrimSpace(revision.ID) == "" { + return nil, fmt.Errorf("%w: publish authoring: revision ID is required", ErrInvalidPublicationInput) + } + if revision.Status != explorer.RevisionReady { + return nil, fmt.Errorf("%w: publish authoring: revision %q carries status %q, only a READY revision may be published", + ErrInvalidPublicationInput, revision.ID, string(revision.Status)) + } + if expectedReleaseRevision < 0 { + return nil, fmt.Errorf("%w: publish authoring: expected release revision %d is negative", ErrInvalidPublicationInput, expectedReleaseRevision) + } + contentDigest, err := revisionContentDigest(revision) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: publish authoring: content identity of revision %q: %w", revision.ID, err) + } + payloads, err := encodeRevisionPayloads(revision) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: publish authoring: encode revision %q: %w", revision.ID, err) + } + revisionDiag, err := marshalOrNil(revision.Diagnostics) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: publish authoring: encode revision %q diagnostics: %w", revision.ID, err) + } + receiptDoc, err := json.Marshal(receipt) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: publish authoring: encode compilation receipt %q: %w", receipt.ID, err) + } + releasePayload, err := marshalPublishReleasePayload(release) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: publish authoring: encode project release %q: %w", release.ID, err) + } + + var lastErr error + for attempt := 1; attempt <= publishTxMaxAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("postgres explorer store: publish authoring: %w", err) + } + out, runErr := p.runPublishAuthoring(ctx, receipt, receiptDoc, revision, payloads, revisionDiag, contentDigest, release, releasePayload, expectedReleaseRevision) + if runErr == nil { + return out, nil + } + if isTypedPublicationRefusal(runErr) { + return nil, runErr + } + lastErr = runErr + if !isDeadlockError(runErr) || attempt == publishTxMaxAttempts { + break + } + if waitErr := waitForStoreRetry(ctx, time.Duration(attempt)*publishDeadlockRetry); waitErr != nil { + return nil, fmt.Errorf("postgres explorer store: publish authoring retry: %w", errors.Join(lastErr, waitErr)) + } + } + if lastErr == nil { + lastErr = ErrPublishContention + } + if isDeadlockError(lastErr) { + return nil, fmt.Errorf("postgres explorer store: publish authoring: %w: %w", ErrPublishContention, lastErr) + } + return nil, fmt.Errorf("postgres explorer store: publish authoring: %w", lastErr) +} + +func (p *PublishStore) runPublishAuthoring( + ctx context.Context, + receipt explorer.CompilationReceipt, receiptDoc []byte, + revision explorer.Revision, payloads revisionPayloads, revisionDiag []byte, contentDigest string, + release dataset.ProjectRelease, releasePayload string, + expectedReleaseRevision int64, +) (out *explorer.Revision, err error) { + conn, err := p.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("acquire publish connection: %w", err) + } + defer conn.Release() + + tx, err := conn.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return nil, fmt.Errorf("begin publish transaction: %w", err) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback(ctx) + } + }() + + crash := func(step string) error { + if p.crashAt != nil { + if err := p.crashAt(ctx, tx, step); err != nil { + return fmt.Errorf("injected fault at %s: %w", step, err) + } + } + return nil + } + + if _, err := p.claimReceiptForPublish(ctx, tx, receipt, receiptDoc); err != nil { + return nil, err + } + if err := crash("receipt-insert"); err != nil { + return nil, err + } + + candidateRow, err := p.claimRevisionForPublish(ctx, tx, revision, payloads, revisionDiag, contentDigest) + if err != nil { + return nil, err + } + storedRevision, err := decodeRevision(*candidateRow) + if err != nil { + return nil, err + } + if err := crash("revision-insert"); err != nil { + return nil, err + } + + if _, err := p.activateReleaseInTx(ctx, tx, release, releasePayload, expectedReleaseRevision); err != nil { + if isTypedReleaseConflict(err) { + return nil, err + } + return nil, err + } + if err := crash("release-pointer"); err != nil { + return nil, err + } + + owner, err := p.readPublishOwner(ctx, tx, storedRevision) + if err != nil { + return nil, err + } + replay := storedRevision.Status == explorer.RevisionActive && + owner.ActiveRevisionID == storedRevision.ID + if replay { + if err := crash("pre-commit"); err != nil { + return nil, err + } + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("commit publish transaction: %w", err) + } + committed = true + return storedRevision, nil + } + + if storedRevision.Status != explorer.RevisionReady { + return nil, explorer.ErrDraftConflict + } + + if owner.ActiveRevisionID != "" && owner.ActiveRevisionID != storedRevision.ID { + prior, err := p.lockPriorRevision(ctx, tx, owner, storedRevision) + if err != nil { + return nil, err + } + var superseded storedRevisionRow + if err := scanRevisionRow(tx.QueryRow(ctx, p.stmts.priorSupersede, prior.ID), &superseded); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("%w: prior revision %q moved before it could be superseded", explorer.ErrDraftConflict, prior.ID) + } + return nil, fmt.Errorf("supersede prior revision %q: %w", prior.ID, err) + } + if err := crash("prior-supersede"); err != nil { + return nil, err + } + } + + activatedPublication, err := activatedPublicationPayload(candidateRow) + if err != nil { + return nil, err + } + activatedAt := time.Now().UTC() + var activated storedRevisionRow + if err := scanRevisionRow(tx.QueryRow(ctx, p.stmts.candidateActive, + storedRevision.ID, activatedAt, []byte(publishDiagnosticsSet), []byte(activatedPublication)), &activated); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("%w: candidate revision %q moved before it could be activated", explorer.ErrDraftConflict, storedRevision.ID) + } + return nil, fmt.Errorf("activate candidate revision %q: %w", storedRevision.ID, err) + } + if err := crash("candidate-activate"); err != nil { + return nil, err + } + + var activatedOwner explorer.Explorer + if err := scanOwner(tx.QueryRow(ctx, p.stmts.ownerActivate, + storedRevision.Project, storedRevision.ExplorerID, storedRevision.ID, time.Now().UTC()), &activatedOwner); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("%w: owner vanished before activation", explorer.ErrDraftConflict) + } + return nil, fmt.Errorf("activate explorer owner %q/%q: %w", storedRevision.Project, storedRevision.ExplorerID, err) + } + if err := crash("owner-activate"); err != nil { + return nil, err + } + + out, err = decodeRevision(activated) + if err != nil { + return nil, err + } + if err := crash("pre-commit"); err != nil { + return nil, err + } + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("commit publish transaction: %w", err) + } + committed = true + return out, nil +} + +func (p *PublishStore) claimReceiptForPublish(ctx context.Context, tx pgx.Tx, receipt explorer.CompilationReceipt, doc []byte) (*explorer.CompilationReceipt, error) { + wantKey, err := canonicalReceiptKey(receipt) + if err != nil { + return nil, fmt.Errorf("content identity of compilation receipt %q: %w", receipt.ID, err) + } + for attempt := 1; attempt <= receiptInsertMaxAttempts; attempt++ { + var row storedReceipt + err := scanReceipt(tx.QueryRow(ctx, p.stmts.receiptInsert, compilationReceiptArgs(receipt, doc)...), &row) + if err == nil { + stored, decodeErr := decodeStoredReceipt(row) + if decodeErr != nil { + return nil, decodeErr + } + storedKey, keyErr := canonicalReceiptKey(*stored) + if keyErr != nil || storedKey != wantKey { + return nil, fmt.Errorf("%w: stored receipt %q content identity disagrees with the claim", explorer.ErrCorruptReceipt, row.ID) + } + return stored, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("claim compilation receipt %q: %w", receipt.ID, err) + } + existing, err := p.readReceiptRowInTx(ctx, tx, receipt.Project, receipt.ExplorerID, receipt.ID) + if err != nil { + return nil, err + } + if existing != nil { + stored, decodeErr := decodeStoredReceipt(*existing) + if decodeErr != nil { + return nil, decodeErr + } + storedKey, keyErr := canonicalReceiptKey(*stored) + if keyErr != nil || storedKey != wantKey { + return nil, fmt.Errorf("%w: receipt id %q was reused with different content", explorer.ErrCorruptReceipt, receipt.ID) + } + return stored, nil + } + } + return nil, fmt.Errorf("%w: receipt claim %q/%q/%q vanished before it became visible", explorer.ErrNotFound, receipt.Project, receipt.ExplorerID, receipt.ID) +} + +func (p *PublishStore) claimRevisionForPublish(ctx context.Context, tx pgx.Tx, revision explorer.Revision, payloads revisionPayloads, revisionDiag []byte, contentDigest string) (*storedRevisionRow, error) { + args := revisionArgs(revision, payloads, contentDigest, revisionDiag) + for attempt := 1; attempt <= revisionInsertMaxAttempts; attempt++ { + var row storedRevisionRow + err := scanRevisionRow(tx.QueryRow(ctx, p.stmts.revisionInsert, args...), &row) + if err == nil { + return &row, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("claim revision %q: %w", revision.ID, err) + } + var existing storedRevisionRow + err = scanRevisionRow(tx.QueryRow(ctx, p.stmts.revisionClaim, revision.ID), &existing) + if errors.Is(err, pgx.ErrNoRows) { + continue + } + if err != nil { + return nil, fmt.Errorf("claim revision %q: %w", revision.ID, err) + } + if existing.ContentDigest != contentDigest { + return nil, fmt.Errorf("%w: revision id %q was reused with different content (stored %s, requested %s)", + explorer.ErrImmutableRevision, revision.ID, existing.ContentDigest, contentDigest) + } + return &existing, nil + } + return nil, fmt.Errorf("%w: revision claim %q vanished before it became visible", explorer.ErrNotFound, revision.ID) +} + +func (p *PublishStore) activateReleaseInTx(ctx context.Context, tx pgx.Tx, release dataset.ProjectRelease, releasePayload string, expectedRevision int64) (dataset.ActiveRelease, error) { + var ( + storedPayload string + matches bool + ) + err := tx.QueryRow(ctx, p.stmts.releaseCandidateLock, + release.Project, release.ID, releasePayload).Scan(&storedPayload, &matches) + if errors.Is(err, pgx.ErrNoRows) { + return dataset.ActiveRelease{}, fmt.Errorf("%w: candidate %s/%s is not a persisted project release", + dataset.ErrReleaseActivationConflict, release.Project, release.ID) + } + if err != nil { + return dataset.ActiveRelease{}, fmt.Errorf("lock release candidate %s/%s: %w", release.Project, release.ID, err) + } + if !matches { + return dataset.ActiveRelease{}, fmt.Errorf("%w: candidate %s/%s no longer matches the persisted document", + dataset.ErrReleaseActivationConflict, release.Project, release.ID) + } + candidate, err := publishReleaseFromRow(release.Project, release.ID, release.Generation, release.GitCommit, storedPayload) + if err != nil { + return dataset.ActiveRelease{}, err + } + + var state string + err = tx.QueryRow(ctx, p.stmts.manifestLock, release.Project, release.Generation).Scan(&state) + if errors.Is(err, pgx.ErrNoRows) { + return dataset.ActiveRelease{}, fmt.Errorf("%w: generation %s/%s has no persisted manifest", + dataset.ErrReleaseActivationConflict, release.Project, release.Generation) + } + if err != nil { + return dataset.ActiveRelease{}, fmt.Errorf("lock generation manifest %s/%s: %w", release.Project, release.Generation, err) + } + if state != string(dataset.StateStaged) && state != string(dataset.StateReady) { + return dataset.ActiveRelease{}, fmt.Errorf("%w: generation %s/%s manifest is %s, not %s or %s", + dataset.ErrReleaseActivationConflict, release.Project, release.Generation, state, + string(dataset.StateStaged), string(dataset.StateReady)) + } + + var activeGeneration string + err = tx.QueryRow(ctx, p.stmts.generationPointer, release.Project).Scan(&activeGeneration) + if errors.Is(err, pgx.ErrNoRows) { + return dataset.ActiveRelease{}, fmt.Errorf("%w: project %s has no active generation", + dataset.ErrReleaseActivationConflict, release.Project) + } + if err != nil { + return dataset.ActiveRelease{}, fmt.Errorf("read active generation for %s: %w", release.Project, err) + } + if activeGeneration != release.Generation { + return dataset.ActiveRelease{}, fmt.Errorf("%w: active generation of %s is %s, not the candidate generation %s", + dataset.ErrReleaseActivationConflict, release.Project, activeGeneration, release.Generation) + } + + var ( + pointerReleaseID string + currentRevision int64 + ) + lockErr := tx.QueryRow(ctx, p.stmts.releasePointerLock, release.Project).Scan(&pointerReleaseID, ¤tRevision) + if lockErr != nil && !errors.Is(lockErr, pgx.ErrNoRows) { + return dataset.ActiveRelease{}, fmt.Errorf("lock active release pointer for %s: %w", release.Project, lockErr) + } + if lockErr == nil { + if pointerReleaseID == release.ID { + return dataset.ActiveRelease{Release: candidate, Revision: currentRevision}, nil + } + if currentRevision != expectedRevision { + return dataset.ActiveRelease{}, fmt.Errorf("%w: candidate %s/%s expected active revision %d, found %d", + dataset.ErrReleaseActivationConflict, release.Project, release.ID, expectedRevision, currentRevision) + } + var committedRevision int64 + casErr := tx.QueryRow(ctx, p.stmts.releasePointerCAS, + release.Project, release.ID, expectedRevision+1, expectedRevision).Scan(&committedRevision) + if errors.Is(casErr, pgx.ErrNoRows) { + return dataset.ActiveRelease{}, fmt.Errorf("%w: active release pointer moved during activation", dataset.ErrReleaseActivationConflict) + } + if casErr != nil { + return dataset.ActiveRelease{}, fmt.Errorf("move active release pointer for %s: %w", release.Project, casErr) + } + if committedRevision != expectedRevision+1 { + return dataset.ActiveRelease{}, fmt.Errorf("%w: active release revision committed as %d, want %d", + datasetpostgres.ErrUnexpectedStoreResult, committedRevision, expectedRevision+1) + } + return dataset.ActiveRelease{Release: candidate, Revision: committedRevision}, nil + } + + if expectedRevision != 0 { + return dataset.ActiveRelease{}, fmt.Errorf("%w: candidate %s/%s expected active revision %d, found no active release", + dataset.ErrReleaseActivationConflict, release.Project, release.ID, expectedRevision) + } + tag, err := tx.Exec(ctx, p.stmts.releasePointerInsert, release.Project, release.ID) + if err != nil { + return dataset.ActiveRelease{}, fmt.Errorf("create active release pointer for %s: %w", release.Project, err) + } + if tag.RowsAffected() == 1 { + return dataset.ActiveRelease{Release: candidate, Revision: 1}, nil + } + var concurrentReleaseID string + var concurrentRevision int64 + rereadErr := tx.QueryRow(ctx, p.stmts.releasePointerRead, release.Project).Scan(&concurrentReleaseID, &concurrentRevision) + if errors.Is(rereadErr, pgx.ErrNoRows) { + return dataset.ActiveRelease{}, fmt.Errorf("%w: active release pointer vanished during activation", datasetpostgres.ErrUnexpectedStoreResult) + } + if rereadErr != nil { + return dataset.ActiveRelease{}, fmt.Errorf("reread active release pointer for %s: %w", release.Project, rereadErr) + } + if concurrentReleaseID != release.ID { + return dataset.ActiveRelease{}, fmt.Errorf("%w: candidate %s/%s expected no active release, found %s at revision %d (concurrent activation)", + dataset.ErrReleaseActivationConflict, release.Project, release.ID, concurrentReleaseID, concurrentRevision) + } + return dataset.ActiveRelease{Release: candidate, Revision: concurrentRevision}, nil +} + +func (p *PublishStore) readPublishOwner(ctx context.Context, tx pgx.Tx, candidate *explorer.Revision) (explorer.Explorer, error) { + management := explorer.ManagementInteractive + if candidate.ExplorerID == "default" { + management = explorer.ManagementRepository + } + var owner explorer.Explorer + err := scanOwner(tx.QueryRow(ctx, p.stmts.ownerGuard, + candidate.Project, candidate.ExplorerID, string(management)), &owner) + if errors.Is(err, pgx.ErrNoRows) { + return explorer.Explorer{}, explorer.ErrDraftConflict + } + if err != nil { + return explorer.Explorer{}, fmt.Errorf("read owner guard for %q/%q: %w", candidate.Project, candidate.ExplorerID, err) + } + if owner.Project != candidate.Project || owner.ExplorerID != candidate.ExplorerID { + return explorer.Explorer{}, explorer.ErrDraftConflict + } + return owner, nil +} + +func (p *PublishStore) lockPriorRevision(ctx context.Context, tx pgx.Tx, owner explorer.Explorer, candidate *explorer.Revision) (storedRevisionRow, error) { + var prior storedRevisionRow + err := scanRevisionRow(tx.QueryRow(ctx, p.stmts.priorLock, owner.ActiveRevisionID), &prior) + if errors.Is(err, pgx.ErrNoRows) { + return storedRevisionRow{}, fmt.Errorf("%w: owner active revision %q is not a stored revision", + explorer.ErrDraftConflict, owner.ActiveRevisionID) + } + if err != nil { + return storedRevisionRow{}, fmt.Errorf("lock prior revision %q: %w", owner.ActiveRevisionID, err) + } + if prior.Project != owner.Project || prior.ExplorerID != owner.ExplorerID || + prior.Project != candidate.Project || prior.ExplorerID != candidate.ExplorerID { + return storedRevisionRow{}, fmt.Errorf("%w: owner active revision %q is foreign to %q/%q", + explorer.ErrDraftConflict, owner.ActiveRevisionID, owner.Project, owner.ExplorerID) + } + if prior.Status != string(explorer.RevisionActive) { + return storedRevisionRow{}, fmt.Errorf("%w: owner active revision %q is %s, not ACTIVE", + explorer.ErrDraftConflict, owner.ActiveRevisionID, prior.Status) + } + return prior, nil +} + +func activatedPublicationPayload(candidate *storedRevisionRow) (string, error) { + var stored map[string]json.RawMessage + if len(candidate.Publication) == 0 { + return "", fmt.Errorf("%w: candidate revision %q has no stored publication document", + datasetpostgres.ErrUnexpectedStoreResult, candidate.ID) + } + if err := json.Unmarshal(candidate.Publication, &stored); err != nil || stored == nil { + return "", fmt.Errorf("%w: candidate revision %q publication document is undecodable: %v", + datasetpostgres.ErrUnexpectedStoreResult, candidate.ID, err) + } + updated := make(map[string]json.RawMessage, len(stored)+3) + for key, value := range stored { + updated[key] = value + } + now := time.Now().UTC() + stateValue, err := json.Marshal(string(explorer.RevisionActive)) + if err != nil { + return "", err + } + revisionValue, err := json.Marshal(candidate.ID) + if err != nil { + return "", err + } + updatedValue, err := json.Marshal(now) + if err != nil { + return "", err + } + updated["state"] = stateValue + updated["revisionId"] = revisionValue + updated["updatedAt"] = updatedValue + if _, present := stored["generation"]; !present && candidate.SourceGeneration != "" { + generationValue, err := json.Marshal(candidate.SourceGeneration) + if err != nil { + return "", err + } + updated["generation"] = generationValue + } + encoded, err := json.Marshal(updated) + if err != nil { + return "", fmt.Errorf("encode activation publication for revision %q: %w", candidate.ID, err) + } + return string(encoded), nil +} + +func marshalPublishReleasePayload(release dataset.ProjectRelease) (string, error) { + normalized := release + if normalized.Publications == nil { + normalized.Publications = []dataset.ReleasePublication{} + } + if normalized.RequiredVerifications == nil { + normalized.RequiredVerifications = []dataset.ContractVerification{} + } + encoded, err := json.Marshal(normalized) + if err != nil { + return "", fmt.Errorf("encode project release payload: %w", err) + } + return string(encoded), nil +} + +func publishReleaseFromRow(project, id, generation, gitCommit, storedPayload string) (dataset.ProjectRelease, error) { + var release dataset.ProjectRelease + decoder := json.NewDecoder(strings.NewReader(storedPayload)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&release); err != nil { + return dataset.ProjectRelease{}, fmt.Errorf("%w: stored release payload is malformed: %v", datasetpostgres.ErrUnexpectedStoreResult, err) + } + if release.ID == "" { + return dataset.ProjectRelease{}, fmt.Errorf("%w: stored release candidate id is required", datasetpostgres.ErrUnexpectedStoreResult) + } + ref := dataset.Ref{Project: release.Project, Generation: release.Generation} + if err := ref.Validate(); err != nil { + return dataset.ProjectRelease{}, fmt.Errorf("%w: stored release is invalid: %v", datasetpostgres.ErrUnexpectedStoreResult, err) + } + if release.GitCommit == "" { + return dataset.ProjectRelease{}, fmt.Errorf("%w: stored release gitCommit is required", datasetpostgres.ErrUnexpectedStoreResult) + } + for _, binding := range release.Publications { + if err := binding.Selector.Validate(); err != nil { + return dataset.ProjectRelease{}, fmt.Errorf("%w: stored release publication selector %v: %v", + datasetpostgres.ErrUnexpectedStoreResult, binding.Selector, err) + } + } + for _, verification := range release.RequiredVerifications { + if err := verification.Selector.Validate(); err != nil { + return dataset.ProjectRelease{}, fmt.Errorf("%w: stored release verification selector %v: %v", + datasetpostgres.ErrUnexpectedStoreResult, verification.Selector, err) + } + } + if release.Project != project || release.ID != id || release.Generation != generation || release.GitCommit != gitCommit { + return dataset.ProjectRelease{}, fmt.Errorf("%w: stored release scan columns disagree with its payload for %s/%s", + datasetpostgres.ErrUnexpectedStoreResult, project, id) + } + return release, nil +} + +func (p *PublishStore) readReceiptRowInTx(ctx context.Context, tx pgx.Tx, project, explorerID, id string) (*storedReceipt, error) { + var row storedReceipt + err := scanReceipt(tx.QueryRow(ctx, p.stmts.receiptClaim, project, explorerID, id), &row) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + return nil, fmt.Errorf("read compilation receipt %q/%q/%q: %w", project, explorerID, id, err) + } + return &row, nil +} + +func isTypedReleaseConflict(err error) bool { + return errors.Is(err, dataset.ErrReleaseActivationConflict) || + errors.Is(err, datasetpostgres.ErrUnexpectedStoreResult) +} + +func isTypedPublicationRefusal(err error) bool { + return errors.Is(err, explorer.ErrCorruptReceipt) || + errors.Is(err, explorer.ErrNotFound) || + errors.Is(err, explorer.ErrImmutableRevision) || + errors.Is(err, explorer.ErrDraftConflict) || + isTypedReleaseConflict(err) +} + +func isDeadlockError(err error) bool { + return pgstore.IsDeadlockDetected(err) +} diff --git a/internal/backends/postgres/internal/explorer/publication_golden_test.go b/internal/backends/postgres/internal/explorer/publication_golden_test.go new file mode 100644 index 000000000..739afed6f --- /dev/null +++ b/internal/backends/postgres/internal/explorer/publication_golden_test.go @@ -0,0 +1,158 @@ +package postgres + +import ( + "reflect" + "strings" + "testing" +) + +const ( + pubauthGoldenCtl = "ctlg" + pubauthGoldenLane = "lanesg" + pubauthGoldenTable = `"ctlg"."explorers"` +) + +func pubauthGoldenStatementPins() map[string]string { + return map[string]string{ + "releaseCandidateLock": `SELECT payload::text, (payload - 'createdAt') = ($3::jsonb - 'createdAt') FROM "lanesg"."project_release" WHERE project = $1 AND id = $2 FOR SHARE`, + "manifestLock": `SELECT state FROM "lanesg"."dataset_generation_manifest" WHERE project = $1 AND generation = $2 FOR SHARE`, + "generationPointer": `SELECT generation FROM "lanesg"."dataset_active_generation" WHERE project = $1 FOR SHARE`, + "releasePointerLock": `SELECT release_id, revision FROM "lanesg"."active_project_release" WHERE project = $1 FOR UPDATE`, + "releasePointerRead": `SELECT release_id, revision FROM "lanesg"."active_project_release" WHERE project = $1`, + "releasePointerCAS": `UPDATE "lanesg"."active_project_release" SET release_id = $2, revision = $3, updated_at = now() WHERE project = $1 AND revision = $4 RETURNING revision`, + "releasePointerInsert": `INSERT INTO "lanesg"."active_project_release" (project, release_id, revision) VALUES ($1, $2, 1) ON CONFLICT DO NOTHING`, + "ownerGuard": `SELECT ` + strings.Join(ownerColumns, ", ") + ` FROM "ctlg"."explorers" WHERE project = $1 AND explorer_id = $2 AND management_mode = $3 FOR UPDATE`, + "priorLock": `SELECT ` + strings.Join(revisionColumns, ", ") + ` FROM "ctlg"."explorer_revisions" WHERE id = $1 FOR UPDATE`, + "priorSupersede": `UPDATE "ctlg"."explorer_revisions" SET status = 'SUPERSEDED' WHERE id = $1 AND status = 'ACTIVE' RETURNING ` + strings.Join(revisionColumns, ", "), + "candidateActive": `UPDATE "ctlg"."explorer_revisions" SET status = 'ACTIVE', activated_at = $2, failed_at = NULL, diagnostics = $3, publication = $4 WHERE id = $1 AND status = 'READY' RETURNING ` + strings.Join(revisionColumns, ", "), + "ownerActivate": `UPDATE "ctlg"."explorers" SET active_revision_id = $3, updated_at = $4 WHERE project = $1 AND explorer_id = $2 RETURNING ` + strings.Join(ownerColumns, ", "), + } +} + +func TestPublishAuthoringStatementShapeLiterals(t *testing.T) { + live := newPublishStatements(pubauthGoldenCtl, pubauthGoldenLane) + v := reflect.ValueOf(live) + for name, want := range pubauthGoldenStatementPins() { + field := v.FieldByName(name) + if !field.IsValid() { + t.Fatalf("publish statement %q no longer exists", name) + } + got := field.String() + if got != want { + t.Fatalf("statement %s drifted from its literal pin:\ngot:\n%s\nwant:\n%s", name, got, want) + } + breakOnePin(t, "statement "+name, got, want) + } +} + +func TestPublishAuthoringStatementsCanonicalizeOnlyInGo(t *testing.T) { + s := newPublishStatements(pubauthGoldenCtl, pubauthGoldenLane) + v := reflect.ValueOf(s) + for i := range v.NumField() { + statement := strings.ToLower(v.Field(i).String()) + for _, forbidden := range []string{"trim(", "lower(", "upper(", "btrim(", "initcap(", "regexp_replace("} { + if strings.Contains(statement, forbidden) { + t.Errorf("statement %s applies SQL-side canonicalization %q: canonical forms belong in Go", + v.Type().Field(i).Name, forbidden) + } + } + } +} + +func TestPublishAuthoringStoredWorldGoldenLiterals(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + ps := f.newPublishStore(t) + project, explorerID, tag := "proj_pubauth_golden", "exp_golden", "gold" + receipt, revision, release := f.seedPublishedWorld(t, store, project, explorerID, tag) + f.publishOK(t, ps, receipt, revision, release, 0) + + if receipt.ID != pubauthGoldenReceiptID { + t.Fatalf("golden receipt id = %s, want %s", receipt.ID, pubauthGoldenReceiptID) + } + breakOnePin(t, "golden receipt id", receipt.ID, pubauthGoldenReceiptID) + + var storedDigest, storedStatus, storedReceiptBinding string + if err := f.iso.QueryRow(f.ctx, "SELECT content_digest, status, compilation_receipt_id FROM "+ + f.ctlTable(RevisionsTable)+" WHERE id = $1", revision.ID). + Scan(&storedDigest, &storedStatus, &storedReceiptBinding); err != nil { + f.fail("read golden revision", err) + } + if storedDigest != pubauthGoldenContentDigest { + t.Fatalf("golden content digest = %s, want %s", storedDigest, pubauthGoldenContentDigest) + } + breakOnePin(t, "golden content digest", storedDigest, pubauthGoldenContentDigest) + if storedStatus != "ACTIVE" || storedReceiptBinding != receipt.ID { + t.Fatalf("golden revision = %s/%s, want ACTIVE/%s", storedStatus, storedReceiptBinding, receipt.ID) + } + + var publication map[string]any + if err := f.iso.QueryRow(f.ctx, "SELECT publication FROM "+f.ctlTable(RevisionsTable)+ + " WHERE id = $1", revision.ID).Scan(&publication); err != nil { + f.fail("read golden publication", err) + } + if publication["state"] != "ACTIVE" || publication["revisionId"] != revision.ID || + publication["executionId"] != "exec_pubauth_gold" || + publication["generation"] != pubauthGeneration { + t.Fatalf("golden publication = %v, want the activated shape", publication) + } + if _, ok := publication["updatedAt"]; !ok { + t.Fatal("golden publication lost the updatedAt overwrite") + } + keys := make([]string, 0, len(publication)) + for k := range publication { + keys = append(keys, k) + } + if want := []string{"executionId", "generation", "revisionId", "state", "updatedAt"}; !reflect.DeepEqual(sortedStrings(keys), want) { + t.Fatalf("golden publication keys = %v, want %v", keys, want) + } + + var owner string + if err := f.iso.QueryRow(f.ctx, "SELECT (to_jsonb(t) - 'updated_at' - 'created_at')::text FROM "+ + f.ctlTable(OwnerTable)+" t WHERE project = $1", project).Scan(&owner); err != nil { + f.fail("read golden owner row", err) + } + if owner != pubauthGoldenOwnerRow { + t.Fatalf("golden owner row = %s, want %s", owner, pubauthGoldenOwnerRow) + } + breakOnePin(t, "golden owner row", owner, pubauthGoldenOwnerRow) + + var pointer string + if err := f.iso.QueryRow(f.ctx, "SELECT (to_jsonb(t) - 'updated_at' - 'created_at')::text FROM "+ + f.lanesTable("active_project_release")+" t WHERE project = $1", project).Scan(&pointer); err != nil { + f.fail("read golden pointer row", err) + } + if pointer != pubauthGoldenPointerRow { + t.Fatalf("golden pointer row = %s, want %s", pointer, pubauthGoldenPointerRow) + } + breakOnePin(t, "golden pointer row", pointer, pubauthGoldenPointerRow) +} + +func breakOnePin(t *testing.T, label, stored, want string) { + t.Helper() + if len(want) == 0 { + t.Fatalf("%s: empty pin", label) + } + tampered := want[:len(want)-1] + string(want[len(want)-1]+1) + if stored == tampered { + t.Fatalf("%s: pinned region is vacuous (a one-byte tamper still matched)", label) + } +} + +func sortedStrings(in []string) []string { + out := append([]string(nil), in...) + for i := 1; i < len(out); i++ { + for j := i; j > 0 && out[j] < out[j-1]; j-- { + out[j], out[j-1] = out[j-1], out[j] + } + } + return out +} + +const ( + pubauthGoldenReceiptID = "receipt_20cc875b33f9db6614541f42344022ac7b85a981a443615ca2217d6b672c6320" + pubauthGoldenContentDigest = "sha256:25c52c00e6b86c4bf97d0df1d156942a50fb5f6449726f8eead80d96ef26dd08" + pubauthGoldenOwnerRow = `{"title": "pubauth exp_golden", "project": "proj_pubauth_golden", "updated_by": "author@calypr", "explorer_id": "exp_golden", "draft_config": null, "draft_digest": "sha256:draft-exp_golden", "draft_version": 1, "management_mode": "INTERACTIVE", "active_revision_id": "rev_proj_pubauth_golden_gold", "draft_snapshot_token": "", "draft_source_generation": "", "last_authoring_command_id": "", "previous_draft_revision_id": "", "last_authoring_command_digest": "", "last_authoring_command_results": null, "draft_authorization_scope_digest": ""}` + pubauthGoldenPointerRow = `{"project": "proj_pubauth_golden", "revision": 1, "release_id": "rel_proj_pubauth_golden_gold"}` +) diff --git a/internal/backends/postgres/internal/explorer/publication_guarddrop_test.go b/internal/backends/postgres/internal/explorer/publication_guarddrop_test.go new file mode 100644 index 000000000..e2a83b206 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/publication_guarddrop_test.go @@ -0,0 +1,220 @@ +package postgres + +import ( + "errors" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/calypr/loom/internal/explorer" +) + +func publishGuardWorld(t *testing.T, f *pubAuthFixture, project, explorerID string, +) (revision1ID, revision2ID, release1ID, release2ID string, ps *PublishStore) { + t.Helper() + store := f.newStore(t) + ps = f.newPublishStore(t) + receipt1, revision1, release1 := f.seedPublishedWorld(t, store, project, explorerID, "gd1") + f.publishOK(t, ps, receipt1, revision1, release1, 0) + receipt2, revision2, release2 := secondPublishInputsSameExplorer(t, f, project, explorerID, "gd2") + f.publishOK(t, ps, receipt2, revision2, release2, 1) + return revision1.ID, revision2.ID, release1.ID, release2.ID, ps +} + +func TestPublishAuthoringRevisionStateTriggerIsLoadBearing(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + revision1ID, revision2ID, _, _, _ := publishGuardWorld(t, f, "proj_pubauth_gd_trigger", "exp_gd") + revs := f.ctlTable(RevisionsTable) + + if _, err := f.iso.Exec(f.ctx, "UPDATE "+revs+" SET status = 'READY' WHERE id = $1", revision1ID); err == nil { + t.Fatal("resurrection SUPERSEDED -> READY accepted while the state-machine trigger is installed") + } else { + pgErr := requirePgErrPubAuth(t, err, "23514", "guarded resurrection") + if !strings.Contains(pgErr.Message, "illegal explorer revision") || + !strings.Contains(pgErr.Message, "status transition") { + t.Fatalf("resurrection refused by something other than the state machine: %s", pgErr.Message) + } + } + + if _, err := f.iso.Exec(f.ctx, + "DROP TRIGGER explorer_revisions_guard ON "+revs); err != nil { + f.fail("drop revision state-machine trigger", err) + } + if _, err := f.iso.Exec(f.ctx, "UPDATE "+revs+" SET status = 'READY' WHERE id = $1", revision1ID); err != nil { + f.fail("resurrection without the trigger must succeed", err) + } + var resurrected string + if err := f.iso.QueryRow(f.ctx, "SELECT status FROM "+revs+" WHERE id = $1", revision1ID).Scan(&resurrected); err != nil { + f.fail("read resurrected status", err) + } + if resurrected != "READY" { + t.Fatalf("guard-removal resurrection did not persist: status = %q", resurrected) + } + if _, err := f.iso.Exec(f.ctx, "DELETE FROM "+revs+" WHERE id = $1", revision2ID); err != nil { + f.fail("DELETE without the trigger must succeed", err) + } + if got := f.countRows(t, revs, "id = $1", revision2ID); got != 0 { + t.Fatalf("guard-removal DELETE did not persist: %d rows remain", got) + } + + f.rerunMigration(t, 6) + if _, err := f.iso.Exec(f.ctx, "UPDATE "+revs+" SET status = 'SUPERSEDED' WHERE id = $1", revision1ID); err == nil { + t.Fatal("illegal READY -> SUPERSEDED transition accepted after the trigger was restored from the migration text") + } else { + pgErr := requirePgErrPubAuth(t, err, "23514", "restored state machine") + if !strings.Contains(pgErr.Message, "status transition") { + t.Fatalf("restored refusal came from something other than the state machine: %s", pgErr.Message) + } + } + if _, err := f.iso.Exec(f.ctx, + "UPDATE "+revs+" SET content_digest = content_digest || 'x' WHERE id = $1", revision1ID); err == nil { + t.Fatal("frozen-column tamper accepted after the trigger was restored") + } else { + pgErr := requirePgErrPubAuth(t, err, "23514", "restored frozen guard") + if !strings.Contains(pgErr.Message, "frozen content changed") { + t.Fatalf("restored tamper refusal came from something else: %s", pgErr.Message) + } + } + if _, err := f.iso.Exec(f.ctx, "DELETE FROM "+revs+" WHERE id = $1", revision1ID); err == nil { + t.Fatal("DELETE accepted after the trigger was restored") + } else { + pgErr := requirePgErrPubAuth(t, err, "23514", "restored insert-once guard") + if !strings.Contains(pgErr.Message, "insert-once") { + t.Fatalf("restored DELETE refusal came from something else: %s", pgErr.Message) + } + } +} + +func TestPublishAuthoringCandidateStatusFenceIsLoadBearing(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + revision1ID, _, _, _, ps := publishGuardWorld(t, f, "proj_pubauth_gd_cand", "exp_gd") + + guarded := ps.stmts.candidateActive + fenceless := strings.Replace(guarded, " AND status = 'READY'", "", 1) + if guarded == fenceless { + t.Fatal("fence-removal probe did not change the statement -- probe is vacuous") + } + args := []any{revision1ID, time.Now().UTC(), []byte("[]"), []byte(`{"state":"READY"}`)} + + if _, err := f.iso.Exec(f.ctx, fenceless, args...); err == nil { + t.Fatal("fenceless activation CAS landed while the state-machine trigger was installed") + } else { + pgErr := requirePgErrPubAuth(t, err, "23514", "fenceless activation with trigger") + if !strings.Contains(pgErr.Message, "status transition") { + t.Fatalf("fenceless activation refused by something other than the state machine: %s", pgErr.Message) + } + } + + if _, err := f.iso.Exec(f.ctx, + "DROP TRIGGER explorer_revisions_guard ON "+f.ctlTable(RevisionsTable)); err != nil { + f.fail("drop revision state-machine trigger", err) + } + tag, err := f.iso.Exec(f.ctx, guarded, args...) + if err != nil { + f.fail("guarded activation CAS", err) + } + if tag.RowsAffected() != 0 { + t.Fatalf("guarded activation CAS moved a SUPERSEDED row (%d rows) -- the test world is wrong", tag.RowsAffected()) + } + if _, err := f.iso.Exec(f.ctx, fenceless, args...); err != nil { + f.fail("fenceless activation without the trigger must succeed", err) + } + var status string + if err := f.iso.QueryRow(f.ctx, "SELECT status FROM "+f.ctlTable(RevisionsTable)+" WHERE id = $1", + revision1ID).Scan(&status); err != nil { + f.fail("read fenceless-activated status", err) + } + if status != "ACTIVE" { + t.Fatalf("fence-removal activation did not persist: status = %q", status) + } + f.rerunMigration(t, 6) +} + +func TestPublishAuthoringSupersedeStatusFenceIsLoadBearing(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + revision1ID, _, _, _, ps := publishGuardWorld(t, f, "proj_pubauth_gd_super", "exp_gd") + + guarded := ps.stmts.priorSupersede + fenceless := strings.Replace(guarded, " AND status = 'ACTIVE'", "", 1) + if guarded == fenceless { + t.Fatal("fence-removal probe did not change the statement -- probe is vacuous") + } + tag, err := f.iso.Exec(f.ctx, guarded, revision1ID) + if err != nil { + f.fail("guarded supersede CAS", err) + } + if tag.RowsAffected() != 0 { + t.Fatalf("guarded supersede moved an already-SUPERSEDED row -- the test world is wrong") + } + tag, err = f.iso.Exec(f.ctx, fenceless, revision1ID) + if err != nil { + f.fail("fenceless supersede", err) + } + if tag.RowsAffected() != 1 { + t.Fatalf("fenceless supersede affected %d rows, want 1 (the fence is what refuses)", tag.RowsAffected()) + } +} + +func TestPublishAuthoringReleasePointerCASIsLoadBearing(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + _, _, release1ID, _, ps := publishGuardWorld(t, f, "proj_pubauth_gd_ptr", "exp_gd") + pointers := f.lanesTable("active_project_release") + + guarded := ps.stmts.releasePointerCAS + fenceless := strings.Replace(guarded, " AND revision = $4", "", 1) + if guarded == fenceless { + t.Fatal("fence-removal probe did not change the statement -- probe is vacuous") + } + if err := f.iso.QueryRow(f.ctx, guarded, "proj_pubauth_gd_ptr", release1ID, int64(99), int64(999)).Scan(new(int64)); !errors.Is(err, pgx.ErrNoRows) { + t.Fatalf("guarded pointer CAS accepted a stale expected revision: %v", err) + } + if _, err := f.iso.Exec(f.ctx, fenceless, "proj_pubauth_gd_ptr", release1ID, int64(99)); err != nil { + f.fail("fenceless pointer CAS must succeed", err) + } + var revision int64 + var releaseID string + if err := f.iso.QueryRow(f.ctx, + "SELECT revision, release_id FROM "+pointers+" WHERE project = $1", "proj_pubauth_gd_ptr"). + Scan(&revision, &releaseID); err != nil { + f.fail("read fenced pointer", err) + } + if revision != 99 { + t.Fatalf("fenceless pointer move did not persist: revision = %d", revision) + } +} + +func TestPublishAuthoringOwnerGuardModePredicateIsLoadBearing(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + if _, err := store.Create(f.ctx, pubauthOwner("proj_pubauth_gd_mode", "exp_repo", explorer.ManagementRepository)); err != nil { + f.fail("create REPOSITORY-mode owner", err) + } + ps := f.newPublishStore(t) + + guarded := ps.stmts.ownerGuard + fenceless := strings.Replace(guarded, " AND management_mode = $3", "", 1) + if guarded == fenceless { + t.Fatal("predicate-removal probe did not change the statement -- probe is vacuous") + } + tag, err := f.iso.Exec(f.ctx, guarded, "proj_pubauth_gd_mode", "exp_repo", "INTERACTIVE") + if err != nil { + f.fail("guarded owner guard read", err) + } + if tag.RowsAffected() != 0 { + t.Fatalf("owner guard matched a wrong-mode row (%d rows) -- the test world is wrong", tag.RowsAffected()) + } + tag, err = f.iso.Exec(f.ctx, fenceless, "proj_pubauth_gd_mode", "exp_repo") + if err != nil { + f.fail("predicateless owner guard read", err) + } + if tag.RowsAffected() != 1 { + t.Fatalf("predicateless guard affected %d rows, want 1 -- the mode predicate is what refuses", tag.RowsAffected()) + } +} diff --git a/internal/backends/postgres/internal/explorer/publication_race_test.go b/internal/backends/postgres/internal/explorer/publication_race_test.go new file mode 100644 index 000000000..13d555872 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/publication_race_test.go @@ -0,0 +1,186 @@ +package postgres + +import ( + "context" + "errors" + "strconv" + "sync" + "testing" + + "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/explorer" +) + +const pubAuthRaceRounds = 12 + +type pubAuthAttempt struct { + ctx context.Context + ps *PublishStore + receipt explorer.CompilationReceipt + revision explorer.Revision + release dataset.ProjectRelease + expected int64 + out *explorer.Revision + err error +} + +func runPubAuthRace(t *testing.T, a, b *pubAuthAttempt) { + t.Helper() + start := make(chan struct{}) + var wg sync.WaitGroup + for _, at := range []*pubAuthAttempt{a, b} { + wg.Add(1) + go func(at *pubAuthAttempt) { + defer wg.Done() + <-start + at.out, at.err = at.ps.PublishAuthoring(at.ctx, at.receipt, at.revision, at.release, at.expected) + }(at) + } + close(start) + wg.Wait() +} + +func TestPublishAuthoringRacingIdenticalCandidateConverges(t *testing.T) { + for round := range pubAuthRaceRounds { + t.Run(roundName(round), func(t *testing.T) { + project, explorerID := "proj_pubauth_race_same", "exp_race_same" + tag0, tagRace := "r0", "rx" + + fr := newPGPubAuthFixture(t) + fr.migrate(t) + storeR := fr.newStore(t) + rcpt0, rev0, rel0 := fr.seedPublishedWorld(t, storeR, project, explorerID, tag0) + fr.publishOK(t, fr.newPublishStore(t), rcpt0, rev0, rel0, 0) + + raceReceipt, raceRevision, raceRelease := + secondPublishInputsSameExplorer(t, fr, project, explorerID, tagRace) + psA, pidA := fr.newDedicatedPublishStore(t) + psB, pidB := fr.newDedicatedPublishStore(t) + requireDistinctBackendsPubAuth(t, pidA, pidB, "publisher A", "publisher B") + + a := &pubAuthAttempt{ctx: fr.ctx, ps: psA, receipt: raceReceipt, revision: raceRevision, release: raceRelease, expected: 1} + b := &pubAuthAttempt{ctx: fr.ctx, ps: psB, receipt: raceReceipt, revision: raceRevision, release: raceRelease, expected: 1} + runPubAuthRace(t, a, b) + if a.err != nil { + t.Fatalf("publisher A: %v", a.err) + } + if b.err != nil { + t.Fatalf("publisher B: %v", b.err) + } + if got := fr.countRows(t, fr.lanesTable("active_project_release"), "project = $1 AND revision = 2", project); got != 1 { + t.Fatalf("pointer is not exactly at revision 2 after the race (%d)", got) + } + if got := fr.countRows(t, fr.ctlTable(RevisionsTable), "project = $1", project); got != 2 { + t.Fatalf("%d revision rows after identical race, want 2", got) + } + + fg := newPGPubAuthFixture(t) + fg.migrate(t) + storeG := fg.newStore(t) + gReceipt0, gRevision0, gRelease0 := fg.seedPublishedWorld(t, storeG, project, explorerID, tag0) + fg.publishOK(t, fg.newPublishStore(t), gReceipt0, gRevision0, gRelease0, 0) + gReceipt, gRevision, gRelease := + secondPublishInputsSameExplorer(t, fg, project, explorerID, tagRace) + fg.publishOK(t, fg.newPublishStore(t), gReceipt, gRevision, gRelease, 1) + if raced, solo := fr.snapshotForProject(t, project), fg.snapshotForProject(t, project); raced != solo { + t.Fatalf("identical-candidate race diverged from the solo golden:\nraced:\n%s\nsolo:\n%s", raced, solo) + } + }) + } +} + +func TestPublishAuthoringRacingDifferentCandidatesSingleWinner(t *testing.T) { + for round := range pubAuthRaceRounds { + t.Run(roundName(round), func(t *testing.T) { + project, explorerID := "proj_pubauth_race_diff", "exp_race_diff" + fr := newPGPubAuthFixture(t) + fr.migrate(t) + storeR := fr.newStore(t) + rcpt0, rev0, rel0 := fr.seedPublishedWorld(t, storeR, project, explorerID, "r0") + fr.publishOK(t, fr.newPublishStore(t), rcpt0, rev0, rel0, 0) + + receiptX, revisionX, releaseX := + secondPublishInputsSameExplorer(t, fr, project, explorerID, "candX") + receiptY, revisionY, releaseY := + secondPublishInputsSameExplorer(t, fr, project, explorerID, "candY") + psA, pidA := fr.newDedicatedPublishStore(t) + psB, pidB := fr.newDedicatedPublishStore(t) + requireDistinctBackendsPubAuth(t, pidA, pidB, "publisher X", "publisher Y") + + a := &pubAuthAttempt{ctx: fr.ctx, ps: psA, receipt: receiptX, revision: revisionX, release: releaseX, expected: 1} + b := &pubAuthAttempt{ctx: fr.ctx, ps: psB, receipt: receiptY, revision: revisionY, release: releaseY, expected: 1} + runPubAuthRace(t, a, b) + + loser, winnerName, winnerReleaseID, winnerRevisionID := singleRaceWinner(t, a, b) + if !errors.Is(loser.err, dataset.ErrReleaseActivationConflict) { + t.Fatalf("loser error = %v, want dataset.ErrReleaseActivationConflict", loser.err) + } + if got := fr.countRows(t, fr.ctlTable(RevisionsTable), "id = $1", loser.revision.ID); got != 0 { + t.Fatalf("loser revision row %q survived the lost race", loser.revision.ID) + } + if got := fr.countRows(t, fr.ctlTable(ReceiptsTable), "id = $1 AND project = $2", loser.receipt.ID, project); got != 0 { + t.Fatalf("loser receipt row %q survived the lost race", loser.receipt.ID) + } + var pointerRelease string + var pointerRevision int64 + if err := fr.iso.QueryRow(fr.ctx, "SELECT release_id, revision FROM "+ + fr.lanesTable("active_project_release")+" WHERE project = $1", project). + Scan(&pointerRelease, &pointerRevision); err != nil { + fr.fail("read raced pointer", err) + } + if pointerRelease != winnerReleaseID || pointerRevision != 2 { + t.Fatalf("raced pointer = %s@%d, want %s@2", pointerRelease, pointerRevision, winnerReleaseID) + } + var ownerActive string + if err := fr.iso.QueryRow(fr.ctx, "SELECT active_revision_id FROM "+ + fr.ctlTable(OwnerTable)+" WHERE project = $1", project).Scan(&ownerActive); err != nil { + fr.fail("read raced owner", err) + } + if ownerActive != winnerRevisionID { + t.Fatalf("raced owner = %s, want %s (%s won)", ownerActive, winnerRevisionID, winnerName) + } + + var trueExpected int64 = 2 + loser.expected = trueExpected + if _, err := loser.ps.PublishAuthoring(fr.ctx, loser.receipt, loser.revision, loser.release, loser.expected); err != nil { + t.Fatalf("loser retry: %v", err) + } + fg := newPGPubAuthFixture(t) + fg.migrate(t) + storeG := fg.newStore(t) + gReceipt0, gRevision0, gRelease0 := fg.seedPublishedWorld(t, storeG, project, explorerID, "r0") + fg.publishOK(t, fg.newPublishStore(t), gReceipt0, gRevision0, gRelease0, 0) + firstReceipt, firstRevision, firstRelease := + secondPublishInputsSameExplorer(t, fg, project, explorerID, "candX") + secondReceipt, secondRevision, secondRelease := + secondPublishInputsSameExplorer(t, fg, project, explorerID, "candY") + if winnerName == "X" { + fg.publishOK(t, fg.newPublishStore(t), firstReceipt, firstRevision, firstRelease, 1) + fg.publishOK(t, fg.newPublishStore(t), secondReceipt, secondRevision, secondRelease, 2) + } else { + fg.publishOK(t, fg.newPublishStore(t), secondReceipt, secondRevision, secondRelease, 1) + fg.publishOK(t, fg.newPublishStore(t), firstReceipt, firstRevision, firstRelease, 2) + } + if raced, solo := fr.snapshotForProject(t, project), fg.snapshotForProject(t, project); raced != solo { + t.Fatalf("different-candidate race diverged from the solo golden:\nraced:\n%s\nsolo:\n%s", raced, solo) + } + }) + } +} + +func singleRaceWinner(t *testing.T, a, b *pubAuthAttempt) (loser *pubAuthAttempt, winnerName, winnerReleaseID, winnerRevisionID string) { + t.Helper() + aOK, bOK := a.err == nil, b.err == nil + if aOK && bOK { + t.Fatal("both racing publishers committed: the pointer CAS admitted two winners") + } + if !aOK && !bOK { + t.Fatalf("both racing publishers failed: A=%v B=%v, want exactly one winner", a.err, b.err) + } + if aOK { + return b, "X", a.release.ID, a.revision.ID + } + return a, "Y", b.release.ID, b.revision.ID +} + +func roundName(round int) string { return "round-" + strconv.Itoa(round) } diff --git a/internal/backends/postgres/internal/explorer/publication_retry_test.go b/internal/backends/postgres/internal/explorer/publication_retry_test.go new file mode 100644 index 000000000..c13e2f84f --- /dev/null +++ b/internal/backends/postgres/internal/explorer/publication_retry_test.go @@ -0,0 +1,181 @@ +package postgres + +import ( + "errors" + "fmt" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + datasetpostgres "github.com/calypr/loom/internal/backends/postgres/internal/dataset" + "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/explorer" +) + +func TestPublishAuthoringIsDeadlockErrorClassification(t *testing.T) { + deadlock := &pgconn.PgError{Code: "40P01", Message: "deadlock detected"} + cases := []struct { + name string + err error + want bool + }{ + {"bare 40P01", deadlock, true}, + {"wrapped 40P01", fmt.Errorf("lock prior revision: %w", deadlock), true}, + {"doubly wrapped 40P01", fmt.Errorf("outer: %w", fmt.Errorf("inner: %w", deadlock)), true}, + {"joined with 40P01", errors.Join(errors.New("context"), deadlock), true}, + {"unique violation 23505", &pgconn.PgError{Code: "23505", Message: "duplicate key"}, false}, + {"serialization-adjacent 40001", &pgconn.PgError{Code: "40001"}, false}, + {"plain error", errors.New("connection reset by peer"), false}, + {"nil", nil, false}, + {"typed-nil driver error", error((*pgconn.PgError)(nil)), false}, + {"wrapped typed-nil driver error", fmt.Errorf("publish leg: %w", error((*pgconn.PgError)(nil))), false}, + } + for _, tc := range cases { + if got := isDeadlockError(tc.err); got != tc.want { + t.Errorf("%s: isDeadlockError = %v, want %v", tc.name, got, tc.want) + } + } +} + +func TestPublishAuthoringTypedRefusalClassification(t *testing.T) { + typed := []error{ + explorer.ErrCorruptReceipt, + explorer.ErrNotFound, + explorer.ErrImmutableRevision, + explorer.ErrDraftConflict, + dataset.ErrReleaseActivationConflict, + datasetpostgres.ErrUnexpectedStoreResult, + } + for _, sentinel := range typed { + err := fmt.Errorf("postgres explorer store: publish authoring: leg: %w", sentinel) + if !isTypedPublicationRefusal(err) { + t.Errorf("sentinel %v wrapped is not classified terminal", sentinel) + } + } + for _, nonTyped := range []error{ + errors.New("connection reset"), + &pgconn.PgError{Code: "40P01", Message: "deadlock detected"}, + fmt.Errorf("wrapped: %w", errors.New("io")), + } { + if isTypedPublicationRefusal(nonTyped) { + t.Errorf("%v misclassified as a typed refusal: it must stay visible to the retry loop", nonTyped) + } + } +} + +func injectFault(t *testing.T, ps *PublishStore, nAttempts int, fault error) func() int { + t.Helper() + fires := 0 + steps := installCrashSeam(ps, func(step string, tx pgx.Tx) error { + if step != "receipt-insert" { + return nil + } + fires++ + if fires <= nAttempts { + return fault + } + return nil + }) + return func() int { + n := 0 + for _, s := range *steps { + if s == "receipt-insert" { + n++ + } + } + return n + } +} + +func TestPublishAuthoringDeadlockVictimRetriesOnFreshTransaction(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + project, explorerID, tag := "proj_pubauth_dl_retry", "exp_dl", "dlr" + receipt, revision, release := f.seedPublishedWorld(t, store, project, explorerID, tag) + ps := f.newPublishStore(t) + deadlock := &pgconn.PgError{Code: "40P01", Message: "deadlock detected"} + transactions := injectFault(t, ps, 1, deadlock) + + out, err := ps.PublishAuthoring(f.ctx, receipt, revision, release, 0) + if err != nil { + t.Fatalf("deadlock victim was not recovered: %v", err) + } + if out.Status != explorer.RevisionActive { + t.Fatalf("recovered publish status = %s, want ACTIVE", out.Status) + } + if transactions() != 2 { + t.Fatalf("seam reached %d times, want 2 (one deadlocked attempt + one retry)", transactions()) + } + fg := newPGPubAuthFixture(t) + fg.migrate(t) + gReceipt, gRevision, gRelease := fg.seedPublishedWorld(t, fg.newStore(t), project, explorerID, tag) + fg.publishOK(t, fg.newPublishStore(t), gReceipt, gRevision, gRelease, 0) + if got, want := f.snapshotForProject(t, project), fg.snapshotForProject(t, project); got != want { + t.Fatalf("retried publish diverged from the uninterrupted golden:\nretried:\n%s\ngolden:\n%s", got, want) + } +} + +func TestPublishAuthoringExhaustedDeadlocksSurfaceErrPublishContention(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + project, explorerID, tag := "proj_pubauth_dl_exhaust", "exp_dl", "dlex" + receipt, revision, release := f.seedPublishedWorld(t, store, project, explorerID, tag) + ps := f.newPublishStore(t) + deadlock := &pgconn.PgError{Code: "40P01", Message: "deadlock detected"} + transactions := injectFault(t, ps, publishTxMaxAttempts+1, deadlock) + before := f.snapshotAll(t, false) + + _, err := ps.PublishAuthoring(f.ctx, receipt, revision, release, 0) + if !errors.Is(err, ErrPublishContention) { + t.Fatalf("exhausted deadlocks error = %v, want ErrPublishContention", err) + } + if !isDeadlockError(err) { + t.Fatalf("contention error must still wrap the last 40P01 victim: %v", err) + } + if transactions() != publishTxMaxAttempts { + t.Fatalf("seam reached %d times, want the full budget %d", transactions(), publishTxMaxAttempts) + } + if after := f.snapshotAll(t, false); after != before { + t.Fatalf("every deadlocked attempt must have rolled back completely:\nbefore:\n%s\nafter:\n%s", before, after) + } +} + +func TestPublishAuthoringNonDeadlockFaultIsNotRetried(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + project, explorerID, tag := "proj_pubauth_dl_plain", "exp_dl", "dlp" + receipt, revision, release := f.seedPublishedWorld(t, store, project, explorerID, tag) + ps := f.newPublishStore(t) + transactions := injectFault(t, ps, 1, errors.New("connection reset by peer")) + + _, err := ps.PublishAuthoring(f.ctx, receipt, revision, release, 0) + if err == nil || errors.Is(err, ErrPublishContention) { + t.Fatalf("plain backend fault = %v, want the fault once, unlabeled as contention", err) + } + if transactions() != 1 { + t.Fatalf("non-deadlock fault retried %d times, want 1", transactions()) + } +} + +func TestPublishAuthoringTypedRefusalAtBackendSiteIsTerminal(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + project, explorerID, tag := "proj_pubauth_dl_typed", "exp_dl", "dlt" + receipt, revision, release := f.seedPublishedWorld(t, store, project, explorerID, tag) + ps := f.newPublishStore(t) + typed := fmt.Errorf("store fault while holding a lock: %w", explorer.ErrImmutableRevision) + transactions := injectFault(t, ps, publishTxMaxAttempts+1, typed) + + _, err := ps.PublishAuthoring(f.ctx, receipt, revision, release, 0) + if !errors.Is(err, explorer.ErrImmutableRevision) { + t.Fatalf("injected typed refusal = %v, want ErrImmutableRevision terminal", err) + } + if transactions() != 1 { + t.Fatalf("typed refusal retried %d times, want 1 (terminal by sentinel)", transactions()) + } +} diff --git a/internal/backends/postgres/internal/explorer/publication_test.go b/internal/backends/postgres/internal/explorer/publication_test.go new file mode 100644 index 000000000..fb1abfa2b --- /dev/null +++ b/internal/backends/postgres/internal/explorer/publication_test.go @@ -0,0 +1,973 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + datasetpostgres "github.com/calypr/loom/internal/backends/postgres/internal/dataset" + "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/explorer" +) + +var ( + maskRevision = []string{"activated_at"} + maskOwner = []string{"updated_at"} + maskPointer = []string{"updated_at", "created_at"} + maskManifest = []string{"updated_at", "created_at"} + maskAll = []string{"activated_at", "updated_at", "created_at", "ready_at", "failed_at"} +) + +var publishWitnessTables = []struct { + table string + mask []string +}{ + {ReceiptsTable, nil}, + {RevisionsTable, maskAll}, + {OwnerTable, maskAll}, + {"project_release", nil}, + {"active_project_release", maskPointer}, + {"dataset_generation_manifest", maskManifest}, + {"dataset_active_generation", maskPointer}, +} + +func (f *pubAuthFixture) snapshotAll(t *testing.T, maskClock bool) string { + t.Helper() + var b strings.Builder + for _, w := range publishWitnessTables { + table := w.table + mask := w.mask + switch table { + case ReceiptsTable, RevisionsTable, OwnerTable: + table = f.ctlTable(table) + default: + table = f.lanesTable(table) + } + if !maskClock { + mask = nil + } + fmt.Fprintf(&b, "%s|%s\x1d\n", table, f.snapshot(t, table, mask...)) + } + return b.String() +} + +func TestPublishAuthoringCommitsAllFourLegsAtomically(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + ps := f.newPublishStore(t) + project, explorerID, tag := "proj_pubauth_legs", "exp_legs", "legs" + receipt, revision, release := f.seedPublishedWorld(t, store, project, explorerID, tag) + + started := time.Now().UTC().Add(-time.Minute) + out := f.publishOK(t, ps, receipt, revision, release, 0) + + if out.ID != revision.ID || out.Status != explorer.RevisionActive { + t.Fatalf("returned revision = %q/%q, want %q/ACTIVE", out.ID, out.Status, revision.ID) + } + if out.ActivatedAt == nil || out.ActivatedAt.Before(started) { + t.Fatalf("returned activated_at = %v, want a fresh UTC time", out.ActivatedAt) + } + if out.Publication.State != string(explorer.RevisionActive) || + out.Publication.RevisionID != revision.ID || + out.Publication.ExecutionID != "exec_pubauth_"+tag || + out.Publication.Generation != pubauthGeneration || + out.Publication.UpdatedAt.Before(started) { + t.Fatalf("returned publication metadata = %+v, want activation merge over the stored document", out.Publication) + } + if len(out.Diagnostics) != 0 { + t.Fatalf("returned diagnostics = %+v, want the activation-cleared empty list", out.Diagnostics) + } + + stored, err := store.GetRevisionForExplorer(f.ctx, project, explorerID, revision.ID) + if err != nil { + t.Fatalf("read back revision: %v", err) + } + if stored.Status != explorer.RevisionActive || stored.ActivatedAt == nil || stored.FailedAt != nil { + t.Fatalf("stored revision lifecycle = %q activated=%v failed=%v, want ACTIVE/now/NULL", + stored.Status, stored.ActivatedAt, stored.FailedAt) + } + owner, err := store.Get(f.ctx, project, explorerID) + if err != nil { + t.Fatalf("read back owner: %v", err) + } + if owner.ActiveRevisionID != revision.ID { + t.Fatalf("owner active_revision_id = %q, want %q", owner.ActiveRevisionID, revision.ID) + } + rel := f.newReleaseStore(t) + active, err := rel.ReadActiveRelease(f.ctx, project) + if err != nil { + t.Fatalf("read back active release: %v", err) + } + if active.Release.ID != release.ID || active.Revision != 1 { + t.Fatalf("active release = %s@%d, want %s@1", active.Release.ID, active.Revision, release.ID) + } + if _, err := store.GetCompilationReceiptForExplorer(f.ctx, project, explorerID, receipt.ID); err != nil { + t.Fatalf("read back receipt: %v", err) + } + + var publication map[string]json.RawMessage + if err := f.iso.QueryRow(f.ctx, "SELECT publication FROM "+f.ctlTable(RevisionsTable)+" WHERE id = $1", + revision.ID).Scan(&publication); err != nil { + f.fail("read stored publication", err) + } + keys := make([]string, 0, len(publication)) + for key := range publication { + keys = append(keys, key) + } + wantKeys := "executionId,generation,revisionId,state,updatedAt" + if got := strings.Join(sortedJoin(keys), ","); got != wantKeys { + t.Fatalf("activation publication key set = %s, want %s", got, wantKeys) + } + if string(publication["state"]) != `"ACTIVE"` || string(publication["revisionId"]) != `"`+revision.ID+`"` { + t.Fatalf("activation publication overwrite fields = %s/%s, want ACTIVE/%s", + publication["state"], publication["revisionId"], revision.ID) + } + if want := `"exec_pubauth_` + tag + `"`; string(publication["executionId"]) != want { + t.Fatalf("activation publication lost the stored executionId: got %s, want %s", publication["executionId"], want) + } + var diagKind string + if err := f.iso.QueryRow(f.ctx, "SELECT diagnostics::text FROM "+f.ctlTable(RevisionsTable)+" WHERE id = $1", + revision.ID).Scan(&diagKind); err != nil { + f.fail("read stored diagnostics", err) + } + if diagKind != "[]" { + t.Fatalf("stored diagnostics = %s, want the activation-cleared [] document", diagKind) + } + + var ( + receiptID, revStatus, pubState, activeRevision, pointerRelease string + pointerRevision int64 + ) + err = f.iso.QueryRow(f.ctx, ` +SELECT r.id, v.status, v.publication->>'state', e.active_revision_id, p.release_id, p.revision +FROM `+f.ctlTable(ReceiptsTable)+` r, + `+f.ctlTable(RevisionsTable)+` v, + `+f.ctlTable(OwnerTable)+` e, + `+f.lanesTable("active_project_release")+` p +WHERE r.project = $1 AND r.explorer_id = $2 AND r.id = $3 + AND v.id = $4 AND v.project = $1 AND v.explorer_id = $2 + AND e.project = $1 AND e.explorer_id = $2 + AND p.project = $1`, + project, explorerID, receipt.ID, revision.ID). + Scan(&receiptID, &revStatus, &pubState, &activeRevision, &pointerRelease, &pointerRevision) + if err != nil { + f.fail("single-snapshot four-leg read", err) + } + if receiptID != receipt.ID || revStatus != "ACTIVE" || pubState != "ACTIVE" || + activeRevision != revision.ID || pointerRelease != release.ID || pointerRevision != 1 { + t.Fatalf("four-leg snapshot = %q/%q/%q/%q/%q@%d, want all legs committed consistently", + receiptID, revStatus, pubState, activeRevision, pointerRelease, pointerRevision) + } +} + +func sortedJoin(values []string) []string { + out := append([]string(nil), values...) + for i := 1; i < len(out); i++ { + for j := i; j > 0 && out[j] < out[j-1]; j-- { + out[j], out[j-1] = out[j-1], out[j] + } + } + return out +} + +func TestPublishAuthoringCommitBoundaryVisibility(t *testing.T) { + t.Run("stopped-before-commit-hides-every-leg-and-locks-one-transaction", func(t *testing.T) { + f, ps, writerPID, observer, observerPID, observerStore, observerStorePID, releaseGate, reached := stopPublishAtPreCommit(t) + project, explorerID, tag := "proj_pubauth_boundary", "exp_boundary", "boundary" + store := f.newStore(t) + receipt, revision, release := f.seedPublishedWorld(t, store, project, explorerID, tag) + + done := make(chan error, 1) + go func() { + _, err := ps.PublishAuthoring(f.ctx, receipt, revision, release, 0) + done <- err + }() + + var stoppedPID int32 + select { + case stoppedPID = <-reached: + case <-time.After(30 * time.Second): + t.Fatal("writer never reached the pre-commit seam") + } + if stoppedPID != writerPID { + t.Fatalf("writer transaction runs on backend %d, want the dedicated %d", stoppedPID, writerPID) + } + requireDistinctBackendsPubAuth(t, writerPID, observerPID, "publish writer", "observer") + requireDistinctBackendsPubAuth(t, writerPID, observerStorePID, "publish writer", "observer store") + requireDistinctBackendsPubAuth(t, observerPID, observerStorePID, "observer", "observer store") + + if _, err := observerStore.GetCompilationReceiptForExplorer(f.ctx, project, explorerID, receipt.ID); !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("observer store reads the in-flight receipt before COMMIT: %v", err) + } + if _, err := observerStore.GetRevision(f.ctx, revision.ID); err == nil || !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("observer store reads the in-flight revision before COMMIT: %v", err) + } + + if got := f.countRows(t, f.ctlTable(ReceiptsTable), "project = $1", project); got != 0 { + t.Fatalf("observer sees %d receipt rows before COMMIT, want 0", got) + } + if got := f.countRows(t, f.ctlTable(RevisionsTable), "project = $1", project); got != 0 { + t.Fatalf("observer sees %d revision rows before COMMIT, want 0", got) + } + var ownerActive string + if err := observer.QueryRow(f.ctx, "SELECT active_revision_id FROM "+f.ctlTable(OwnerTable)+" WHERE project = $1", + project).Scan(&ownerActive); err != nil { + f.fail("observer reads owner", err) + } + if ownerActive != "" { + t.Fatalf("observer sees owner active_revision_id = %q before COMMIT, want empty", ownerActive) + } + if got := f.countRows(t, f.lanesTable("active_project_release"), "project = $1", project); got != 0 { + t.Fatalf("observer sees %d active release pointers before COMMIT, want 0", got) + } + + locks := f.locksHeldBy(t, observer, writerPID) + for _, table := range []string{ + ReceiptsTable, RevisionsTable, OwnerTable, + "project_release", "active_project_release", + "dataset_generation_manifest", "dataset_active_generation", + } { + if !locks[table] { + t.Fatalf("writer backend %d holds no lock on %s at the pre-commit seam; publish legs are not in one transaction: %v", + writerPID, table, locks) + } + } + + releaseGate <- nil + if err := <-done; err != nil { + t.Fatalf("PublishAuthoring after commit release: %v", err) + } + var legs int + if err := observer.QueryRow(f.ctx, ` +SELECT (SELECT count(*) FROM `+f.ctlTable(ReceiptsTable)+` WHERE project = $1) + + (SELECT count(*) FROM `+f.ctlTable(RevisionsTable)+` WHERE project = $1 AND status = 'ACTIVE') + + (SELECT count(*) FROM `+f.ctlTable(OwnerTable)+` WHERE project = $1 AND active_revision_id = $2) + + (SELECT count(*) FROM `+f.lanesTable("active_project_release")+` WHERE project = $1 AND revision = 1)`, + project, revision.ID).Scan(&legs); err != nil { + f.fail("observer four-leg read", err) + } + if legs != 4 { + t.Fatalf("post-commit four-leg count = %d, want 4 (all legs visible together)", legs) + } + if _, err := observerStore.GetCompilationReceiptForExplorer(f.ctx, project, explorerID, receipt.ID); err != nil { + t.Fatalf("observer store cannot read the committed receipt: %v", err) + } + stored, err := observerStore.GetRevision(f.ctx, revision.ID) + if err != nil || stored.Status != explorer.RevisionActive { + t.Fatalf("observer store post-commit revision = %v/%v, want ACTIVE", stored, err) + } + }) + + t.Run("abort-after-every-write-rolls-back-every-leg", func(t *testing.T) { + f, ps, _, _, _, _, _, releaseGate, reached := stopPublishAtPreCommit(t) + project, explorerID, tag := "proj_pubauth_abort", "exp_abort", "abort" + store := f.newStore(t) + receipt, revision, release := f.seedPublishedWorld(t, store, project, explorerID, tag) + pre := f.snapshotAll(t, false) + + done := make(chan error, 1) + go func() { + _, err := ps.PublishAuthoring(f.ctx, receipt, revision, release, 0) + done <- err + }() + select { + case <-reached: + case <-time.After(30 * time.Second): + t.Fatal("writer never reached the pre-commit seam") + } + releaseGate <- errors.New("injected crash after every write") + err := <-done + if err == nil { + t.Fatal("PublishAuthoring succeeded despite the injected pre-commit crash") + } + if !strings.Contains(err.Error(), "injected crash after every write") { + t.Fatalf("abort error = %v, want the injected fault surfaced", err) + } + if post := f.snapshotAll(t, false); post != pre { + t.Fatalf("post-abort witness diverged from the pre-run world:\npre:\n%s\npost:\n%s", pre, post) + } + psClean := f.newPublishStore(t) + f.publishOK(t, psClean, receipt, revision, release, 0) + var legs int + if err := f.iso.QueryRow(f.ctx, ` +SELECT (SELECT count(*) FROM `+f.ctlTable(ReceiptsTable)+` WHERE project = $1) + + (SELECT count(*) FROM `+f.ctlTable(RevisionsTable)+` WHERE project = $1 AND status = 'ACTIVE') + + (SELECT count(*) FROM `+f.ctlTable(OwnerTable)+` WHERE project = $1 AND active_revision_id = $2) + + (SELECT count(*) FROM `+f.lanesTable("active_project_release")+` WHERE project = $1 AND revision = 1)`, + project, revision.ID).Scan(&legs); err != nil { + f.fail("retry four-leg read", err) + } + if legs != 4 { + t.Fatalf("post-crash retry four-leg count = %d, want 4", legs) + } + }) +} + +func stopPublishAtPreCommit(t *testing.T) ( + *pubAuthFixture, *PublishStore, int32, *pgxpool.Pool, int32, *Store, int32, chan error, chan int32, +) { + t.Helper() + f := newPGPubAuthFixture(t) + f.migrate(t) + ps, writerPID := f.newDedicatedPublishStore(t) + observer, observerPID := f.newDedicatedPool(t) + observerStore, observerStorePID := f.newDedicatedStore(t) + reached := make(chan int32, 1) + releaseGate := make(chan error, 1) + ps.crashAt = func(ctx context.Context, tx pgx.Tx, step string) error { + if step != "pre-commit" { + return nil + } + var pid int32 + if err := tx.QueryRow(ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + return fmt.Errorf("witness writer backend: %w", err) + } + reached <- pid + return <-releaseGate + } + return f, ps, writerPID, observer, observerPID, observerStore, observerStorePID, releaseGate, reached +} + +func TestPublishAuthoringCrashWindowAtEverySeam(t *testing.T) { + steps := []string{ + "receipt-insert", "revision-insert", "release-pointer", + "prior-supersede", "candidate-activate", "owner-activate", "pre-commit", + } + for _, crashStep := range steps { + t.Run(crashStep, func(t *testing.T) { + project, explorerID := "proj_pubauth_twin", "exp_twin" + fg := newPGPubAuthFixture(t) + fg.migrate(t) + storeG, psG := fg.newStore(t), fg.newPublishStore(t) + fc := newPGPubAuthFixture(t) + fc.migrate(t) + storeC, psC := fc.newStore(t), fc.newPublishStore(t) + + receipt1, revision1, release1G := + fg.seedPublishedWorld(t, storeG, project, explorerID, "twin1") + _, _, release1C := fc.seedPublishedWorld(t, storeC, project, explorerID, "twin1") + fg.publishOK(t, psG, receipt1, revision1, release1G, 0) + fc.publishOK(t, psC, receipt1, revision1, release1C, 0) + + receipt2G, revision2G, release2G := + secondPublishInputsSameExplorer(t, fg, project, explorerID, "twin2") + receipt2C, revision2C, release2C := + secondPublishInputsSameExplorer(t, fc, project, explorerID, "twin2") + pre := fc.snapshotAll(t, false) + + psCrash := fc.newPublishStore(t) + recorded := installCrashSeam(psCrash, func(step string, tx pgx.Tx) error { + if step == crashStep { + return errors.New("injected crash at " + step) + } + return nil + }) + _, err := psCrash.PublishAuthoring(fc.ctx, receipt2C, revision2C, release2C, 1) + if err == nil || !strings.Contains(err.Error(), "injected crash at "+crashStep) { + t.Fatalf("crash at %s: error = %v, want the injected fault surfaced", crashStep, err) + } + want := stepsThrough(steps, crashStep) + if strings.Join(*recorded, ",") != strings.Join(want, ",") { + t.Fatalf("crash at %s: recorded seam steps = %v, want %v", crashStep, *recorded, want) + } + if post := fc.snapshotAll(t, false); post != pre { + t.Fatalf("crash at %s left visible state:\npre:\n%s\npost:\n%s", crashStep, pre, post) + } + + fg.publishOK(t, psG, receipt2G, revision2G, release2G, 1) + fc.publishOK(t, fc.newPublishStore(t), receipt2C, revision2C, release2C, 1) + golden := fg.snapshotForProject(t, project) + recovered := fc.snapshotForProject(t, project) + if golden != recovered { + t.Fatalf("crash at %s: recovered run diverged from the clean golden:\ngolden:\n%s\nrecovered:\n%s", + crashStep, golden, recovered) + } + }) + } +} + +func stepsThrough(steps []string, crashStep string) []string { + for i, s := range steps { + if s == crashStep { + return steps[:i+1] + } + } + panic("unknown crash step " + crashStep) +} + +func (f *pubAuthFixture) snapshotForProject(t *testing.T, project string) string { + t.Helper() + specs := []struct { + table, where string + args []any + mask []string + }{ + {f.ctlTable(ReceiptsTable), "project = $1", []any{project}, nil}, + {f.ctlTable(RevisionsTable), "project = $1", []any{project}, maskAll}, + {f.ctlTable(OwnerTable), "project = $1", []any{project}, maskAll}, + {f.lanesTable("project_release"), "project = $1", []any{project}, nil}, + {f.lanesTable("active_project_release"), "project = $1", []any{project}, maskPointer}, + {f.lanesTable("dataset_generation_manifest"), "project = $1", []any{project}, maskManifest}, + {f.lanesTable("dataset_active_generation"), "project = $1", []any{project}, maskPointer}, + } + var b strings.Builder + for _, s := range specs { + out := f.snapshotWhere(t, s.table, s.where, s.args, s.mask...) + label := strings.NewReplacer(f.ctlSchema, "", f.lanesSchema, "").Replace(s.table) + out = pubauthClockRe.ReplaceAllString(out, `"updatedAt":""`) + fmt.Fprintf(&b, "%s|%s\x1d\n", label, out) + } + return b.String() +} + +var pubauthClockRe = regexp.MustCompile(`"updatedAt": ?"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z"`) + +func TestPublishAuthoringIdempotentReplayIsZeroWrite(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + ps := f.newPublishStore(t) + project, explorerID, tag := "proj_pubauth_replay", "exp_replay", "replay" + receipt, revision, release := f.seedPublishedWorld(t, store, project, explorerID, tag) + + f.publishOK(t, ps, receipt, revision, release, 0) + first := f.snapshotAll(t, false) + + out := f.publishOK(t, ps, receipt, revision, release, 0) + if out.Status != explorer.RevisionActive || out.ID != revision.ID { + t.Fatalf("replay returned %q/%q, want the stored ACTIVE revision", out.ID, out.Status) + } + if post := f.snapshotAll(t, false); post != first { + t.Fatalf("identical replay wrote rows:\nfirst:\n%s\nreplay:\n%s", first, post) + } + + f.publishOK(t, ps, receipt, revision, release, 999) + if post := f.snapshotAll(t, false); post != first { + t.Fatal("already-active replay with a stale expected revision must still be zero-write") + } + + sameContent := receipt + sameContent.RequestID = "req_replay_other" + f.publishOK(t, ps, sameContent, revision, release, 0) + if post := f.snapshotAll(t, false); post != first { + t.Fatal("digest-equivalent receipt replay (RequestID only) must be zero-write") + } + + mutatedRevision := revision + mutatedRevision.IntentDigest = "sha256:mutated" + _, err := ps.PublishAuthoring(f.ctx, receipt, mutatedRevision, release, 0) + if !errors.Is(err, explorer.ErrImmutableRevision) { + t.Fatalf("mutated revision replay error = %v, want explorer.ErrImmutableRevision", err) + } + requireErrContainsPubAuth(t, err, "reused with different content", "mutated revision replay") + if post := f.snapshotAll(t, false); post != first { + t.Fatal("the refused revision mutation touched the stored world") + } + + mutatedRelease := release + mutatedRelease.GitCommit = "commit-mutated" + _, err = ps.PublishAuthoring(f.ctx, receipt, revision, mutatedRelease, 0) + if !errors.Is(err, dataset.ErrReleaseActivationConflict) { + t.Fatalf("mutated release replay error = %v, want dataset.ErrReleaseActivationConflict", err) + } + requireErrContainsPubAuth(t, err, "no longer matches the persisted document", "mutated release replay") + if post := f.snapshotAll(t, false); post != first { + t.Fatal("the refused release mutation touched the stored world") + } + + _, err = ps.PublishAuthoring(f.ctx, + pubauthReceipt(t, "proj_pubauth_other", "exp_replay", "replay_other"), + pubauthRevision(revision.ID, "proj_pubauth_other", "exp_replay", "receipt_other", "replay_other"), + pubauthRelease("proj_pubauth_other", pubauthGeneration, "rel_other"), 0) + if !errors.Is(err, explorer.ErrImmutableRevision) { + t.Fatalf("cross-tenant revision reuse error = %v, want explorer.ErrImmutableRevision", err) + } + + reuseProject := "proj_pubauth_reuse" + _, _, reuseRelease := f.seedPublishedWorld(t, store, reuseProject, "exp_reuse", "reuse") + manualID := "receipt_pubauth_reuse" + firstClaim := pubauthManualReceipt(t, reuseProject, "exp_reuse", "reuse1", manualID) + f.publishOK(t, ps, firstClaim, + pubauthRevision("rev_pubauth_reuse1", reuseProject, "exp_reuse", manualID, "reuse1"), reuseRelease, 0) + reuseBase := f.snapshotAll(t, false) + + reuseClaim := pubauthManualReceipt(t, reuseProject, "exp_reuse", "reuse2", manualID) + _, err = ps.PublishAuthoring(f.ctx, reuseClaim, + pubauthRevision("rev_pubauth_reuse2", reuseProject, "exp_reuse", manualID, "reuse2"), reuseRelease, 0) + if !errors.Is(err, explorer.ErrCorruptReceipt) { + t.Fatalf("reused receipt id error = %v, want explorer.ErrCorruptReceipt", err) + } + requireErrContainsPubAuth(t, err, "reused with different content", "reused receipt id") + if _, insertErr := store.InsertCompilationReceipt(f.ctx, reuseClaim); insertErr == nil || + !errors.Is(insertErr, explorer.ErrCorruptReceipt) || + !strings.Contains(insertErr.Error(), "reused with different content") { + t.Fatalf("standalone insert on the same reused id = %v, want the same typed refusal", insertErr) + } + if post := f.snapshotAll(t, false); post != reuseBase { + t.Fatal("the refused receipt-id reuse touched the stored world") + } +} + +func TestPublishAuthoringVerifiedReceiptLeg(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + ps := f.newPublishStore(t) + + if _, err := ps.PublishAuthoring(f.ctx, explorer.CompilationReceipt{ID: " "}, + pubauthRevision("rev_x", "p", "e", "r", "x"), pubauthRelease("p", "g", "r"), 0); !errors.Is(err, explorer.ErrCorruptReceipt) { + t.Fatalf("blank-id receipt error = %v, want explorer.ErrCorruptReceipt", err) + } + + projectA, projectB := "proj_pubauth_tenantA", "proj_pubauth_tenantB" + for _, project := range []string{projectA, projectB} { + if _, err := store.Create(f.ctx, pubauthOwner(project, "exp_tenant", explorer.ManagementInteractive)); err != nil { + f.fail("create tenant owner "+project, err) + } + } + receiptA := pubauthReceipt(t, projectA, "exp_tenant", "shared") + revisionA := pubauthRevision("rev_tenant_a", projectA, "exp_tenant", receiptA.ID, "shared") + releaseA := seedReleaseLaneOnly(t, f, projectA, "shared") + f.publishOK(t, ps, receiptA, revisionA, releaseA, 0) + + receiptB := pubauthReceipt(t, projectB, "exp_tenant", "shared") + if receiptB.ID == receiptA.ID { + t.Fatal("content-addressed ids must differ across tenants (the tenant frame is inside the digest)") + } + revisionB := pubauthRevision("rev_tenant_b", projectB, "exp_tenant", receiptB.ID, "shared") + releaseB := seedReleaseLaneOnly(t, f, projectB, "shared") + f.publishOK(t, ps, receiptB, revisionB, releaseB, 0) + + if got := f.countRows(t, f.ctlTable(ReceiptsTable), "project = $1", projectA); got != 1 { + t.Fatalf("tenant A receipt rows = %d, want 1", got) + } + if got := f.countRows(t, f.ctlTable(ReceiptsTable), "project = $1", projectB); got != 1 { + t.Fatalf("tenant B receipt rows = %d, want 1", got) + } + + if _, err := store.GetCompilationReceiptForExplorer(f.ctx, projectA, "exp_tenant", receiptB.ID); !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("cross-tenant receipt read error = %v, want explorer.ErrNotFound", err) + } + if _, err := store.GetCompilationReceiptForExplorer(f.ctx, projectB, "exp_tenant", receiptA.ID); !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("reverse cross-tenant receipt read error = %v, want explorer.ErrNotFound", err) + } +} + +func seedReleaseLaneOnly(t *testing.T, f *pubAuthFixture, project, tag string) dataset.ProjectRelease { + t.Helper() + gen := f.newGenerationStore(t) + stageAndActivate(t, f, gen, project, pubauthGeneration, nil) + release := pubauthRelease(project, pubauthGeneration, "rel_"+project+"_"+tag) + if _, err := f.newReleaseStore(t).SaveRelease(f.ctx, release); err != nil { + f.fail("save release "+release.ID, err) + } + return release +} + +func seedLanesOnly(t *testing.T, f *pubAuthFixture, project, explorerID string, +) (explorer.CompilationReceipt, explorer.Revision, dataset.ProjectRelease) { + t.Helper() + release := seedReleaseLaneOnly(t, f, project, "lanes") + receipt := pubauthReceipt(t, project, explorerID, "lanes") + revision := pubauthRevision("rev_"+project+"_lanes", project, explorerID, receipt.ID, "lanes") + return receipt, revision, release +} + +func seedLanesOnlyForExistingOwner(t *testing.T, f *pubAuthFixture, project, explorerID string, +) (explorer.CompilationReceipt, explorer.Revision, dataset.ProjectRelease) { + t.Helper() + return seedLanesOnly(t, f, project, explorerID) +} + +func TestPublishAuthoringImmutableRevisionLeg(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + ps := f.newPublishStore(t) + project, explorerID, tag := "proj_pubauth_immut", "exp_immut", "immut" + receipt, revision, release := f.seedPublishedWorld(t, store, project, explorerID, tag) + f.publishOK(t, ps, receipt, revision, release, 0) + before := f.snapshotAll(t, false) + + for name, mutate := range map[string]func(revision explorer.Revision) explorer.Revision{ + "config": func(r explorer.Revision) explorer.Revision { r.Config = jsonRaw(`{"changed":true}`); return r }, + "recipeName": func(r explorer.Revision) explorer.Revision { r.Recipe.Name = "mutated"; return r }, + "emittedColumn": func(r explorer.Revision) explorer.Revision { + cols := append([]explorer.EmittedColumn(nil), r.EmittedColumns...) + cols[0].EmissionID = "emit_mutated" + r.EmittedColumns = cols + return r + }, + "datasetMetadata": func(r explorer.Revision) explorer.Revision { + r.Dataset.SchemaDigest = "sha256:mutated" + return r + }, + } { + mutated := mutate(revision) + if digest, _ := revisionContentDigest(mutated); digest == func() string { d, _ := revisionContentDigest(revision); return d }() { + t.Fatalf("mutation %q did not move the content digest -- probe is vacuous", name) + } + _, err := ps.PublishAuthoring(f.ctx, receipt, mutated, release, 0) + if !errors.Is(err, explorer.ErrImmutableRevision) { + t.Fatalf("mutation %q: error = %v, want explorer.ErrImmutableRevision", name, err) + } + if post := f.snapshotAll(t, false); post != before { + t.Fatalf("mutation %q: the refused publish touched the stored world", name) + } + } +} + +func TestPublishAuthoringReleaseCASRefusalsRollBackEverything(t *testing.T) { + cases := []struct { + name string + refuse func(t *testing.T, f *pubAuthFixture, project string) + wants []string + }{ + { + name: "candidate-never-persisted", + refuse: func(t *testing.T, f *pubAuthFixture, project string) { + f.execRaw(t, "DELETE FROM "+f.lanesTable("project_release")+" WHERE project = $1", project) + }, + wants: []string{"is not a persisted project release"}, + }, + { + name: "manifest-missing", + refuse: func(t *testing.T, f *pubAuthFixture, project string) { + f.execRaw(t, "DELETE FROM "+f.lanesTable("dataset_active_generation")+" WHERE project = $1", project) + f.execRaw(t, "DELETE FROM "+f.lanesTable("dataset_generation_manifest")+" WHERE project = $1", project) + }, + wants: []string{"has no persisted manifest"}, + }, + { + name: "manifest-loading", + refuse: func(t *testing.T, f *pubAuthFixture, project string) { + f.execRaw(t, "UPDATE "+f.lanesTable("dataset_generation_manifest")+ + " SET state = 'LOADING' WHERE project = $1", project) + }, + wants: []string{"manifest is LOADING, not STAGED or READY"}, + }, + { + name: "no-active-generation", + refuse: func(t *testing.T, f *pubAuthFixture, project string) { + f.execRaw(t, "DELETE FROM "+f.lanesTable("dataset_active_generation")+" WHERE project = $1", project) + }, + wants: []string{"has no active generation"}, + }, + { + name: "pointer-moves", + refuse: func(t *testing.T, f *pubAuthFixture, project string) { + replaced, err := dataset.NewRef(project, "gen_pubauth_01") + if err != nil { + f.fail("expected ref", err) + } + stageAndActivate(t, f, f.newGenerationStore(t), project, "gen_pubauth_02", &replaced) + }, + wants: []string{"not the candidate generation"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + ps := f.newPublishStore(t) + project, explorerID, tag := "proj_pubauth_"+strings.ReplaceAll(tc.name, "-", "_"), "exp_cas", "cas" + receipt, revision, release := f.seedPublishedWorld(t, store, project, explorerID, tag) + tc.refuse(t, f, project) + pre := f.snapshotAll(t, false) + _, err := ps.PublishAuthoring(f.ctx, receipt, revision, release, 0) + if !errors.Is(err, dataset.ErrReleaseActivationConflict) { + t.Fatalf("%s: error = %v, want dataset.ErrReleaseActivationConflict", tc.name, err) + } + for _, want := range tc.wants { + requireErrContainsPubAuth(t, err, want, tc.name) + } + if got := f.countRows(t, f.ctlTable(ReceiptsTable), "project = $1", project); got != 0 { + t.Fatalf("%s: %d receipt rows survived the CAS loss, want 0", tc.name, got) + } + if got := f.countRows(t, f.ctlTable(RevisionsTable), "project = $1", project); got != 0 { + t.Fatalf("%s: %d revision rows survived the CAS loss, want 0", tc.name, got) + } + var ownerActive string + if err := f.iso.QueryRow(f.ctx, "SELECT active_revision_id FROM "+f.ctlTable(OwnerTable)+ + " WHERE project = $1", project).Scan(&ownerActive); err != nil { + f.fail("read owner", err) + } + if ownerActive != "" { + t.Fatalf("%s: owner activated %q despite the CAS loss", tc.name, ownerActive) + } + if post := f.snapshotAll(t, false); post != pre { + t.Fatalf("%s: the refused activation changed the stored world", tc.name) + } + }) + } + + t.Run("stale-expected-revision-conflicts-and-converges-on-retry", func(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + ps := f.newPublishStore(t) + project, explorerID := "proj_pubauth_stale", "exp_stale" + receipt1, revision1, release1 := f.seedPublishedWorld(t, store, project, explorerID, "stale1") + f.publishOK(t, ps, receipt1, revision1, release1, 0) + receipt2, revision2, release2 := secondPublishInputsSameExplorer(t, f, project, explorerID, "stale2") + pointerBefore := f.snapshot(t, f.lanesTable("active_project_release"), maskPointer...) + _, err := ps.PublishAuthoring(f.ctx, receipt2, revision2, release2, 0) + if !errors.Is(err, dataset.ErrReleaseActivationConflict) { + t.Fatalf("stale expected: error = %v, want dataset.ErrReleaseActivationConflict", err) + } + requireErrContainsPubAuth(t, err, "expected active revision 0, found 1", "stale expected") + if got := f.countRows(t, f.ctlTable(RevisionsTable), "id = $1", revision2.ID); got != 0 { + t.Fatalf("stale expected: loser revision row survived the rollback") + } + if got := f.snapshot(t, f.lanesTable("active_project_release"), maskPointer...); got != pointerBefore { + t.Fatal("stale expected: the pointer moved for the loser") + } + f.publishOK(t, ps, receipt2, revision2, release2, 1) + var ownerActive, pointerRelease string + var pointerRevision int64 + if err := f.iso.QueryRow(f.ctx, "SELECT e.active_revision_id, p.release_id, p.revision FROM "+ + f.ctlTable(OwnerTable)+" e, "+f.lanesTable("active_project_release")+ + " p WHERE e.project = $1 AND p.project = $1", project). + Scan(&ownerActive, &pointerRelease, &pointerRevision); err != nil { + f.fail("read converged state", err) + } + if ownerActive != revision2.ID || pointerRelease != release2.ID || pointerRevision != 2 { + t.Fatalf("converged = %s/%s@%d, want %s/%s@2", ownerActive, pointerRelease, pointerRevision, + revision2.ID, release2.ID) + } + var priorStatus string + if err := f.iso.QueryRow(f.ctx, "SELECT status FROM "+f.ctlTable(RevisionsTable)+" WHERE id = $1", + revision1.ID).Scan(&priorStatus); err != nil { + f.fail("read prior status", err) + } + if priorStatus != "SUPERSEDED" { + t.Fatalf("prior status = %s, want SUPERSEDED", priorStatus) + } + }) + + t.Run("initial-pointer-requires-expected-zero", func(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + ps := f.newPublishStore(t) + project, explorerID, tag := "proj_pubauth_initial", "exp_initial", "initial" + receipt, revision, release := f.seedPublishedWorld(t, store, project, explorerID, tag) + _, err := ps.PublishAuthoring(f.ctx, receipt, revision, release, 1) + if !errors.Is(err, dataset.ErrReleaseActivationConflict) { + t.Fatalf("initial expected=1: error = %v, want dataset.ErrReleaseActivationConflict", err) + } + requireErrContainsPubAuth(t, err, "found no active release", "initial expected=1") + if got := f.countRows(t, f.ctlTable(RevisionsTable), "project = $1", project); got != 0 { + t.Fatalf("initial expected=1: %d revision rows survived, want 0", got) + } + }) + + t.Run("f-b13-retained-generation-conflicts-while-another-generation-is-active", func(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + ps := f.newPublishStore(t) + project, explorerID := "proj_pubauth_f13", "exp_f13" + if _, err := store.Create(f.ctx, pubauthOwner(project, explorerID, explorer.ManagementInteractive)); err != nil { + f.fail("create owner", err) + } + gen := f.newGenerationStore(t) + stageAndActivate(t, f, gen, project, "gen_pubauth_01", nil) + releaseG1 := pubauthRelease(project, "gen_pubauth_01", "rel_f13_g1") + if _, err := f.newReleaseStore(t).SaveRelease(f.ctx, releaseG1); err != nil { + f.fail("save G1 candidate", err) + } + replaced, err := dataset.NewRef(project, "gen_pubauth_01") + if err != nil { + f.fail("expected ref", err) + } + stageAndActivate(t, f, gen, project, "gen_pubauth_02", &replaced) + receiptG1 := pubauthReceipt(t, project, explorerID, "f13_g1") + revisionG1 := pubauthRevision("rev_f13_g1", project, explorerID, receiptG1.ID, "f13_g1") + _, err = ps.PublishAuthoring(f.ctx, receiptG1, revisionG1, releaseG1, 0) + if !errors.Is(err, dataset.ErrReleaseActivationConflict) { + t.Fatalf("retained G1 publish: error = %v, want dataset.ErrReleaseActivationConflict", err) + } + requireErrContainsPubAuth(t, err, "active generation of", "retained G1 publish") + requireErrContainsPubAuth(t, err, "not the candidate generation gen_pubauth_01", "retained G1 publish") + if got := f.countRows(t, f.ctlTable(RevisionsTable), "project = $1", project); got != 0 { + t.Fatalf("retained G1 publish wrote %d revision rows, want 0", got) + } + }) +} + +func secondPublishInputsSameExplorer(t *testing.T, f *pubAuthFixture, project, explorerID, tag string, +) (explorer.CompilationReceipt, explorer.Revision, dataset.ProjectRelease) { + t.Helper() + release := pubauthRelease(project, pubauthGeneration, "rel_"+project+"_"+tag) + if _, err := f.newReleaseStore(t).SaveRelease(f.ctx, release); err != nil { + f.fail("save second candidate", err) + } + receipt := pubauthReceipt(t, project, explorerID, tag) + revision := pubauthRevision("rev_"+project+"_"+tag, project, explorerID, receipt.ID, tag) + return receipt, revision, release +} + +func stageAndActivate(t *testing.T, f *pubAuthFixture, gen *datasetpostgres.Store, project, generation string, expected *dataset.Ref) { + t.Helper() + ref, err := dataset.NewRef(project, generation) + if err != nil { + f.fail("ref", err) + } + snapshot, err := dataset.NewSchemaSnapshot("schema-pubauth", "4.0.1", pubauthSnapshotSHA256, []string{"Patient"}) + if err != nil { + f.fail("snapshot", err) + } + loading, err := dataset.NewManifest(ref, snapshot) + if err != nil { + f.fail("manifest", err) + } + if _, err := gen.CreateManifest(f.ctx, loading); err != nil { + f.fail("create manifest", err) + } + staged, err := gen.TransitionManifest(f.ctx, loading, dataset.StateStaged) + if err != nil { + f.fail("stage manifest", err) + } + if err := gen.ActivateGeneration(f.ctx, staged, expected); err != nil { + f.fail("activate generation", err) + } +} + +func TestPublishAuthoringActivationGuardRefusals(t *testing.T) { + t.Run("missing-owner-and-mode-mismatch-are-indistinguishable", func(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + ps := f.newPublishStore(t) + + receipt, revision, release := seedLanesOnly(t, f, "proj_pubauth_noowner", "exp_none") + _, err := ps.PublishAuthoring(f.ctx, receipt, revision, release, 0) + if !errors.Is(err, explorer.ErrDraftConflict) { + t.Fatalf("missing owner: error = %v, want explorer.ErrDraftConflict", err) + } + + wrongModeProject := "proj_pubauth_wrongmode" + if _, err := store.Create(f.ctx, pubauthOwner(wrongModeProject, "exp_wrong", explorer.ManagementRepository)); err != nil { + f.fail("create repository-mode owner", err) + } + receipt2, revision2, release2 := seedLanesOnlyForExistingOwner(t, f, wrongModeProject, "exp_wrong") + _, err2 := ps.PublishAuthoring(f.ctx, receipt2, revision2, release2, 0) + if !errors.Is(err2, explorer.ErrDraftConflict) { + t.Fatalf("mode mismatch: error = %v, want explorer.ErrDraftConflict", err2) + } + if err.Error() != err2.Error() { + t.Fatalf("guard causes must be indistinguishable: %q vs %q", err, err2) + } + + defaultProject := "proj_pubauth_default" + if _, err := store.Create(f.ctx, pubauthOwner(defaultProject, "default", explorer.ManagementInteractive)); err != nil { + f.fail("create interactive default owner", err) + } + receipt3, revision3, release3 := seedLanesOnlyForExistingOwner(t, f, defaultProject, "default") + _, err3 := ps.PublishAuthoring(f.ctx, receipt3, revision3, release3, 0) + if !errors.Is(err3, explorer.ErrDraftConflict) { + t.Fatalf("default Explorer with INTERACTIVE mode: error = %v, want explorer.ErrDraftConflict", err3) + } + + f.execRaw(t, "UPDATE "+f.ctlTable(OwnerTable)+" SET management_mode = 'REPOSITORY' WHERE project = $1", defaultProject) + f.publishOK(t, ps, receipt3, revision3, release3, 0) + }) + + t.Run("superseded-or-failed-candidate-is-a-guard-miss", func(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + ps := f.newPublishStore(t) + project, explorerID := "proj_pubauth_guard2", "exp_guard2" + receipt1, revision1, release1 := f.seedPublishedWorld(t, store, project, explorerID, "g2a") + f.publishOK(t, ps, receipt1, revision1, release1, 0) + receipt2, revision2, release2 := secondPublishInputsSameExplorer(t, f, project, explorerID, "g2b") + f.publishOK(t, ps, receipt2, revision2, release2, 1) + + before := f.snapshotAll(t, false) + _, err := ps.PublishAuthoring(f.ctx, receipt1, revision1, release1, 2) + if !errors.Is(err, explorer.ErrDraftConflict) { + t.Fatalf("superseded candidate: error = %v, want explorer.ErrDraftConflict", err) + } + if post := f.snapshotAll(t, false); post != before { + t.Fatal("superseded candidate republish touched the stored world") + } + + if _, err := store.FailRevision(f.ctx, revision2.ID, []explorer.Diagnostic{{Severity: "ERROR", Code: "X"}}); err != nil { + f.fail("fail revision2", err) + } + before = f.snapshotAll(t, false) + _, err = ps.PublishAuthoring(f.ctx, receipt2, revision2, release2, 2) + if !errors.Is(err, explorer.ErrDraftConflict) { + t.Fatalf("failed candidate: error = %v, want explorer.ErrDraftConflict", err) + } + if post := f.snapshotAll(t, false); post != before { + t.Fatal("failed candidate republish touched the stored world") + } + }) + + t.Run("fail-closed-owner-pointer", func(t *testing.T) { + f := newPGPubAuthFixture(t) + f.migrate(t) + store := f.newStore(t) + ps := f.newPublishStore(t) + project, explorerID := "proj_pubauth_prior", "exp_prior" + receipt1, revision1, release1 := f.seedPublishedWorld(t, store, project, explorerID, "priorA") + f.publishOK(t, ps, receipt1, revision1, release1, 0) + + f.execRaw(t, "UPDATE "+f.ctlTable(OwnerTable)+" SET active_revision_id = 'rev_ghost' WHERE project = $1", project) + receipt2, revision2, release2 := secondPublishInputsSameExplorer(t, f, project, explorerID, "priorB") + before := f.snapshotAll(t, false) + _, err := ps.PublishAuthoring(f.ctx, receipt2, revision2, release2, 1) + if !errors.Is(err, explorer.ErrDraftConflict) { + t.Fatalf("dangling prior: error = %v, want explorer.ErrDraftConflict", err) + } + requireErrContainsPubAuth(t, err, "not a stored revision", "dangling prior") + if post := f.snapshotAll(t, false); post != before { + t.Fatal("dangling prior refusal touched the stored world") + } + f.execRaw(t, "UPDATE "+f.ctlTable(OwnerTable)+" SET active_revision_id = $2 WHERE project = $1", + project, revision1.ID) + f.publishOK(t, ps, receipt2, revision2, release2, 1) + + f.execRaw(t, "UPDATE "+f.ctlTable(OwnerTable)+" SET active_revision_id = $2 WHERE project = $1", + project, revision1.ID) + receipt3, revision3, release3 := secondPublishInputsSameExplorer(t, f, project, explorerID, "priorC") + _, err = ps.PublishAuthoring(f.ctx, receipt3, revision3, release3, 2) + if !errors.Is(err, explorer.ErrDraftConflict) { + t.Fatalf("superseded prior: error = %v, want explorer.ErrDraftConflict", err) + } + requireErrContainsPubAuth(t, err, "not ACTIVE", "superseded prior") + + otherProject := "proj_pubauth_prior_other" + if _, err := store.Create(f.ctx, pubauthOwner(otherProject, "exp_other", explorer.ManagementInteractive)); err != nil { + f.fail("create other owner", err) + } + f.execRaw(t, "UPDATE "+f.ctlTable(OwnerTable)+ + " SET active_revision_id = $2 WHERE project = $1", otherProject, revision1.ID) + receipt4, revision4, release4 := seedLanesOnlyForExistingOwner(t, f, otherProject, "exp_other") + _, err = ps.PublishAuthoring(f.ctx, receipt4, revision4, release4, 0) + if !errors.Is(err, explorer.ErrDraftConflict) { + t.Fatalf("foreign prior: error = %v, want explorer.ErrDraftConflict", err) + } + requireErrContainsPubAuth(t, err, "foreign to", "foreign prior") + }) +} + +func requireErrContainsPubAuth(t *testing.T, err error, want, step string) { + t.Helper() + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("%s: error = %v, want it to mention %q", step, err, want) + } +} diff --git a/internal/backends/postgres/internal/explorer/publication_zz_test.go b/internal/backends/postgres/internal/explorer/publication_zz_test.go new file mode 100644 index 000000000..0ddc61b6e --- /dev/null +++ b/internal/backends/postgres/internal/explorer/publication_zz_test.go @@ -0,0 +1,39 @@ +package postgres + +import ( + "context" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestPublishAuthoringZZNoLeftoverResources(t *testing.T) { + raw := pubauthTestPostgresURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %v", err) + } + defer admin.Close() + + var databases int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE left(datname, 15) = 'verify_pubauth_'", + ).Scan(&databases); err != nil { + t.Fatalf("query database namespace: %v", err) + } + if got, want := int64(databases), pubauthDBBaseline.Load()+pubauthLive.Load(); got != want { + t.Errorf("%d verify_pubauth_* databases remain, baseline+live is %d", got, want) + } + var schemas int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_namespace WHERE left(nspname, 15) = 'verify_pubauth' || '_'", + ).Scan(&schemas); err != nil { + t.Fatalf("query schema namespace: %v", err) + } + if got, want := int64(schemas), pubauthSchemaBaseline.Load(); got != want { + t.Errorf("%d verify_pubauth_* schemas remain in the service database, baseline is %d", got, want) + } +} diff --git a/internal/backends/postgres/internal/explorer/receipts.go b/internal/backends/postgres/internal/explorer/receipts.go new file mode 100644 index 000000000..c4ea6c336 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/receipts.go @@ -0,0 +1,276 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/calypr/loom/internal/explorer" +) + +const ReceiptsTable = "compilation_receipts" + +const ( + receiptInsertMaxAttempts = 4 + receiptConflictRetryDelay = 5 * time.Millisecond +) + +var ErrReceiptClaimLost = errors.New("postgres explorer store: receipt insert lost every claim race to an aborted winner") + +var compilationReceiptColumns = []string{ + "project", + "explorer_id", + "id", + "receipt_format_version", + "compiler_contract_version", + "compilation_key", + "intent_digest", + "snapshot_token", + "source_generation", + "recipe_digest", + "resolved_recipe_digest", + "doc", +} + +type receiptStatements struct { + insert string + claim string + get string +} + +func newReceiptStatements(schema string) receiptStatements { + table := sanitizeTableRef(schema, ReceiptsTable) + cols := strings.Join(compilationReceiptColumns, ", ") + placeholders := orderedPlaceholders(len(compilationReceiptColumns)) + scope := " WHERE project = $1 AND explorer_id = $2 AND id = $3" + return receiptStatements{ + insert: "INSERT INTO " + table + " (" + cols + ") VALUES (" + placeholders + ") " + + "ON CONFLICT (project, explorer_id, id) DO NOTHING RETURNING " + cols, + claim: "SELECT " + cols + " FROM " + table + scope + " FOR UPDATE", + get: "SELECT " + cols + " FROM " + table + scope, + } +} + +type storedReceipt struct { + Project string + ExplorerID string + ID string + ReceiptFormatVersion int64 + CompilerContractVersion string + CompilationKey string + IntentDigest string + SnapshotToken string + SourceGeneration string + RecipeDigest string + ResolvedRecipeDigest string + Doc json.RawMessage +} + +func scanReceipt(row rowScanner, out *storedReceipt) error { + return row.Scan( + &out.Project, + &out.ExplorerID, + &out.ID, + &out.ReceiptFormatVersion, + &out.CompilerContractVersion, + &out.CompilationKey, + &out.IntentDigest, + &out.SnapshotToken, + &out.SourceGeneration, + &out.RecipeDigest, + &out.ResolvedRecipeDigest, + &out.Doc, + ) +} + +func compilationReceiptArgs(receipt explorer.CompilationReceipt, doc []byte) []any { + return []any{ + receipt.Project, + receipt.ExplorerID, + receipt.ID, + int64(receipt.ReceiptFormatVersion), + receipt.CompilerContractVersion, + receipt.CompilationKey, + receipt.IntentDigest, + receipt.SnapshotToken, + receipt.SourceGeneration, + receipt.RecipeDigest, + receipt.ResolvedRecipeDigest, + doc, + } +} + +func validateReceiptForStore(receipt explorer.CompilationReceipt) error { + if strings.TrimSpace(receipt.ID) == "" { + return explorer.ErrCorruptReceipt + } + if receipt.ReceiptFormatVersion != 0 || receipt.CompilerContractVersion != "" || receipt.CompilationKey != "" { + if err := receipt.Validate(); err != nil { + return err + } + } + if strings.HasPrefix(receipt.ID, "receipt_") && len(receipt.ID) == len("receipt_")+sha256.Size*2 { + if _, err := hex.DecodeString(strings.TrimPrefix(receipt.ID, "receipt_")); err != nil { + return explorer.ErrCorruptReceipt + } + if err := receipt.ValidateID(); err != nil { + return explorer.ErrCorruptReceipt + } + } + return nil +} + +func decodeStoredReceipt(row storedReceipt) (*explorer.CompilationReceipt, error) { + var receipt explorer.CompilationReceipt + if err := json.Unmarshal(row.Doc, &receipt); err != nil { + return nil, fmt.Errorf("%w: stored receipt %q document is undecodable: %v", explorer.ErrCorruptReceipt, row.ID, err) + } + if err := validateReceiptForStore(receipt); err != nil { + return nil, fmt.Errorf("%w: stored receipt %q failed validation: %v", explorer.ErrCorruptReceipt, row.ID, err) + } + if receipt.ID != row.ID || receipt.Project != row.Project || receipt.ExplorerID != row.ExplorerID { + return nil, fmt.Errorf("%w: stored receipt %q scope columns disagree with its document", explorer.ErrCorruptReceipt, row.ID) + } + if int64(receipt.ReceiptFormatVersion) != row.ReceiptFormatVersion || + receipt.CompilerContractVersion != row.CompilerContractVersion || + receipt.CompilationKey != row.CompilationKey || + receipt.IntentDigest != row.IntentDigest || + receipt.SnapshotToken != row.SnapshotToken || + receipt.SourceGeneration != row.SourceGeneration || + receipt.RecipeDigest != row.RecipeDigest || + receipt.ResolvedRecipeDigest != row.ResolvedRecipeDigest { + return nil, fmt.Errorf("%w: stored receipt %q digest columns disagree with its document", explorer.ErrCorruptReceipt, row.ID) + } + return &receipt, nil +} + +func (s *Store) InsertCompilationReceipt(ctx context.Context, receipt explorer.CompilationReceipt) (*explorer.CompilationReceipt, error) { + if err := validateReceiptForStore(receipt); err != nil { + return nil, fmt.Errorf("postgres explorer store: insert compilation receipt: %w", err) + } + doc, err := json.Marshal(receipt) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: encode compilation receipt %q: %w", receipt.ID, err) + } + wantKey, err := canonicalReceiptKey(receipt) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: content identity of compilation receipt %q: %w", receipt.ID, err) + } + + var lastErr error + for attempt := 1; attempt <= receiptInsertMaxAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("postgres explorer store: insert compilation receipt %q: %w", receipt.ID, err) + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire receipt insert connection: %w", err) + } + stored, claimErr := s.claimedReceipt(ctx, conn, receipt, doc, wantKey) + conn.Release() + if stored != nil { + return stored, nil + } + lastErr = claimErr + if attempt == receiptInsertMaxAttempts { + break + } + if waitErr := waitForStoreRetry(ctx, time.Duration(attempt)*receiptConflictRetryDelay); waitErr != nil { + return nil, fmt.Errorf("postgres explorer store: retry compilation receipt insert %q: %w", + receipt.ID, errors.Join(lastErr, waitErr)) + } + } + if lastErr == nil { + lastErr = ErrReceiptClaimLost + } + return nil, fmt.Errorf("postgres explorer store: insert compilation receipt %q: %w", receipt.ID, lastErr) +} + +func (s *Store) claimedReceipt(ctx context.Context, conn *pgxpool.Conn, receipt explorer.CompilationReceipt, doc []byte, wantKey string) (*explorer.CompilationReceipt, error) { + var row storedReceipt + err := scanReceipt(conn.QueryRow(ctx, s.receipt.insert, compilationReceiptArgs(receipt, doc)...), &row) + if err == nil { + stored, decodeErr := decodeStoredReceipt(row) + if decodeErr != nil { + return nil, decodeErr + } + storedKey, keyErr := canonicalReceiptKey(*stored) + if keyErr != nil || storedKey != wantKey { + return nil, fmt.Errorf("%w: stored receipt %q content identity disagrees with the claim", explorer.ErrCorruptReceipt, row.ID) + } + return stored, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("postgres explorer store: claim compilation receipt %q: %w", receipt.ID, err) + } + + existing, err := s.readReceiptRow(ctx, conn, s.receipt.claim, receipt.Project, receipt.ExplorerID, receipt.ID) + if err != nil { + return nil, err + } + if existing == nil { + return nil, fmt.Errorf("%w: receipt claim %q/%q/%q vanished before it became visible", + explorer.ErrNotFound, receipt.Project, receipt.ExplorerID, receipt.ID) + } + stored, decodeErr := decodeStoredReceipt(*existing) + if decodeErr != nil { + return nil, decodeErr + } + storedKey, keyErr := canonicalReceiptKey(*stored) + if keyErr != nil || storedKey != wantKey { + return nil, fmt.Errorf("%w: receipt id %q was reused with different content", explorer.ErrCorruptReceipt, receipt.ID) + } + return stored, nil +} + +func canonicalReceiptKey(receipt explorer.CompilationReceipt) (string, error) { + return explorer.ReceiptID(receipt) +} + +func (s *Store) GetCompilationReceiptForExplorer(ctx context.Context, project, explorerID, id string) (*explorer.CompilationReceipt, error) { + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire receipt read connection: %w", err) + } + defer conn.Release() + + row, err := s.readReceiptRow(ctx, conn, s.receipt.get, project, explorerID, id) + if err != nil { + return nil, err + } + if row == nil { + return nil, explorer.ErrNotFound + } + return decodeStoredReceipt(*row) +} + +func (s *Store) readReceiptRow(ctx context.Context, conn *pgxpool.Conn, statement, project, explorerID, id string) (*storedReceipt, error) { + var row storedReceipt + err := scanReceipt(conn.QueryRow(ctx, statement, project, explorerID, id), &row) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + return nil, fmt.Errorf("postgres explorer store: read compilation receipt %q/%q/%q: %w", project, explorerID, id, err) + } + return &row, nil +} + +func waitForStoreRetry(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/internal/backends/postgres/internal/explorer/receipts_test.go b/internal/backends/postgres/internal/explorer/receipts_test.go new file mode 100644 index 000000000..d6c01e862 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/receipts_test.go @@ -0,0 +1,845 @@ +package postgres + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/explorer" +) + +const ( + receiptProject = "proj_receipts" + receiptExplorer = "exp_receipts" +) + +var receiptCreatedAt = time.Date(2026, 3, 4, 5, 6, 7, 123456000, time.UTC) + +const receiptCreatedAtText = "2026-03-04T05:06:07.123456Z" + +func pgstate02ReceiptBundle() recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "state-receipt", + TranslationVersion: "test", + Outputs: []recipe.Output{{ + Name: "output", RootResourceType: "Patient", RowGrain: "resource", + }}, + } +} + +func pgstate02Receipt(t *testing.T, intent, project, explorerID string) explorer.CompilationReceipt { + t.Helper() + bundle := pgstate02ReceiptBundle() + receipt := explorer.CompilationReceipt{ + ReceiptFormatVersion: explorer.CurrentReceiptFormatVersion, + CompilerContractVersion: explorer.CurrentCompilerContractVersion, + Project: project, ExplorerID: explorerID, + IntentDigest: intent, + SnapshotToken: "snapshot-pg-01", + AuthorizationScopeDigest: "scope-pg-01", + CapabilitySchemaDigest: "schema-pg-01", + ShapeDigest: "shape-pg-01", + SourceGeneration: "gen_pg_01", + RecipeDigest: "sha256:recipe-digest", + ResolvedSchemaDigest: "sha256:resolved-schema-pg", + NormalizedBundle: json.RawMessage(cfgAlpha), + Bundle: bundle, + PublicOutputContract: json.RawMessage(`{"outputs":[{"outputId":"output","columns":[],"lossless":true,"mlReady":true}]}`), + OutputColumnProvenance: map[string]map[string]string{"output": {"__loom_row_id": "EXPLICIT"}}, + RequestID: "req_01", + CreatedAt: receiptCreatedAt, + } + var err error + if receipt.ResolvedRecipeDigest, err = bundle.Digest(); err != nil { + t.Fatal(err) + } + if receipt.OutputContractDigest, err = explorer.CompilationArtifactDigest(receipt.PublicOutputContract); err != nil { + t.Fatal(err) + } + if receipt.CompilationKey, err = explorer.CompilationKey(receipt); err != nil { + t.Fatal(err) + } + if receipt.ID, err = explorer.ReceiptID(receipt); err != nil { + t.Fatal(err) + } + return receipt +} + +func pgstate02LegacyReceipt(id, intent string) explorer.CompilationReceipt { + return explorer.CompilationReceipt{ + ID: id, + Project: receiptProject, + ExplorerID: receiptExplorer, + IntentDigest: intent, + SnapshotToken: "snapshot-legacy", + SourceGeneration: "gen_legacy", + NormalizedBundle: json.RawMessage(cfgBeta), + RecipeDigest: "sha256:legacy-recipe", + CreatedAt: receiptCreatedAt, + } +} + +func requireReceiptEqual(t *testing.T, want, got explorer.CompilationReceipt, where string) { + t.Helper() + if got.ID != want.ID || got.Project != want.Project || got.ExplorerID != want.ExplorerID { + t.Fatalf("%s: identity changed: got %q/%q/%q want %q/%q/%q", where, + got.Project, got.ExplorerID, got.ID, want.Project, want.ExplorerID, want.ID) + } + if got.ReceiptFormatVersion != want.ReceiptFormatVersion || + got.CompilerContractVersion != want.CompilerContractVersion || + got.CompilationKey != want.CompilationKey || + got.IntentDigest != want.IntentDigest || + got.SnapshotToken != want.SnapshotToken || + got.SourceGeneration != want.SourceGeneration || + got.RecipeDigest != want.RecipeDigest || + got.ResolvedRecipeDigest != want.ResolvedRecipeDigest || + got.RequestID != want.RequestID { + t.Fatalf("%s: scalar fields changed: got %+v want %+v", where, got, want) + } + if !got.CreatedAt.Equal(want.CreatedAt) { + t.Fatalf("%s: createdAt changed: got %s want %s", where, got.CreatedAt, want.CreatedAt) + } + jsonEqual(t, want.NormalizedBundle, got.NormalizedBundle, where+" normalizedBundle") + jsonEqual(t, want.CompiledConfig, got.CompiledConfig, where+" compiledConfig") + jsonEqual(t, want.PublicOutputContract, got.PublicOutputContract, where+" publicOutputContract") + if len(got.Bundle.Outputs) != len(want.Bundle.Outputs) || + (len(want.Bundle.Outputs) > 0 && got.Bundle.Outputs[0].RootResourceType != want.Bundle.Outputs[0].RootResourceType) { + t.Fatalf("%s: canonical recipe not lossless: got %+v want %+v", where, got.Bundle, want.Bundle) + } +} + +func pgstate02RawReceiptInsert(schema string) string { + statements := newReceiptStatements(schema) + return strings.Replace(statements.insert, + " ON CONFLICT (project, explorer_id, id) DO NOTHING", "", 1) +} + +func (f *pgState02Fixture) rawInsertReceipt(t *testing.T, receipt explorer.CompilationReceipt) { + t.Helper() + doc, err := json.Marshal(receipt) + if err != nil { + f.fail("encode raw receipt fixture", err) + } + _, err = f.iso.Exec(f.ctx, pgstate02RawReceiptInsert(f.schema), compilationReceiptArgs(receipt, doc)...) + if err != nil { + f.fail("raw insert receipt "+receipt.ID, err) + } +} + +type receiptLiteralRow struct { + Project string + ExplorerID string + ID string + ReceiptFormatVersion int64 + CompilerContractVersion string + CompilationKey string + IntentDigest string + SnapshotToken string + SourceGeneration string + RecipeDigest string + ResolvedRecipeDigest string + DocType string + DocRequestID string + DocCreatedAt string + DocIntentDigest string +} + +func (f *pgState02Fixture) receiptLiteralRow(t *testing.T, project, explorerID, id string) receiptLiteralRow { + t.Helper() + var row receiptLiteralRow + err := f.iso.QueryRow(f.ctx, `SELECT + project, explorer_id, id, receipt_format_version, compiler_contract_version, + compilation_key, intent_digest, snapshot_token, source_generation, + recipe_digest, resolved_recipe_digest, + jsonb_typeof(doc), doc->>'requestId', doc->>'createdAt', doc->>'intentDigest' + FROM `+f.receiptTable()+` WHERE project = $1 AND explorer_id = $2 AND id = $3`, + project, explorerID, id).Scan( + &row.Project, &row.ExplorerID, &row.ID, &row.ReceiptFormatVersion, + &row.CompilerContractVersion, &row.CompilationKey, &row.IntentDigest, + &row.SnapshotToken, &row.SourceGeneration, &row.RecipeDigest, + &row.ResolvedRecipeDigest, &row.DocType, &row.DocRequestID, + &row.DocCreatedAt, &row.DocIntentDigest) + if err != nil { + f.fail("read literal receipt row", err) + } + return row +} + +func (f *pgState02Fixture) receiptCount(t *testing.T, project string) int { + t.Helper() + var n int + if err := f.iso.QueryRow(f.ctx, + `SELECT count(*) FROM `+f.receiptTable()+` WHERE project = $1`, project).Scan(&n); err != nil { + f.fail("count receipt rows", err) + } + return n +} + +func TestPostgresReceiptSQLContract(t *testing.T) { + statements := newReceiptStatements("loom_control") + if !strings.Contains(statements.insert, `ON CONFLICT (project, explorer_id, id) DO NOTHING`) { + t.Error("receipt insert must be insert-once: ON CONFLICT (project, explorer_id, id) DO NOTHING") + } + if !strings.Contains(statements.insert, "RETURNING") { + t.Error("receipt insert must RETURNING the stored row") + } + if !strings.Contains(statements.insert, `"loom_control"."`+ReceiptsTable+`"`) { + t.Error("receipt insert must bind the schema-qualified, sanitized table") + } + if !strings.Contains(statements.claim, " FOR UPDATE") { + t.Error("the conflict reread must be FOR UPDATE so it waits for an in-flight winner") + } + for name, statement := range map[string]string{"insert": statements.insert, "claim": statements.claim, "get": statements.get} { + if !strings.Contains(statement, `WHERE project = $1 AND explorer_id = $2 AND id = $3`) && name != "insert" { + t.Errorf("receipt %s statement is not tenant-scoped by project AND explorer_id: %s", name, statement) + } + } + for i := 1; i <= len(compilationReceiptColumns); i++ { + if !strings.Contains(statements.insert, fmt.Sprintf("$%d", i)) { + t.Fatalf("receipt insert is missing ordered parameter $%d", i) + } + } + if strings.Contains(statements.insert, fmt.Sprintf("$%d", len(compilationReceiptColumns)+1)) { + t.Fatalf("receipt insert carries more than %d parameters", len(compilationReceiptColumns)) + } + doc, err := json.Marshal(pgstate02LegacyReceipt("legacy-1", "intent")) + if err != nil { + t.Fatal(err) + } + if got := len(compilationReceiptArgs(pgstate02LegacyReceipt("legacy-1", "intent"), doc)); got != len(compilationReceiptColumns) { + t.Fatalf("receiptArgs produced %d values, want %d (one per column)", got, len(compilationReceiptColumns)) + } +} + +func TestPostgresReceiptInsertValidationIsTyped(t *testing.T) { + if err := validateReceiptForStore(explorer.CompilationReceipt{ID: " "}); !errors.Is(err, explorer.ErrCorruptReceipt) { + t.Fatalf("blank ID error = %v, want ErrCorruptReceipt", err) + } + if err := validateReceiptForStore(explorer.CompilationReceipt{}); !errors.Is(err, explorer.ErrCorruptReceipt) { + t.Fatalf("empty ID error = %v, want ErrCorruptReceipt", err) + } + if err := validateReceiptForStore(pgstate02LegacyReceipt("legacy-ok", "intent")); err != nil { + t.Fatalf("legacy receipt rejected: %v", err) + } + tamperedID := "receipt_" + strings.Repeat("0", 64) + err := validateReceiptForStore(explorer.CompilationReceipt{ID: tamperedID}) + if !errors.Is(err, explorer.ErrCorruptReceipt) { + t.Fatalf("mismatched content-addressed ID error = %v, want ErrCorruptReceipt", err) + } +} + +func TestPostgresReceiptContentIdentityIgnoresRequestMetadata(t *testing.T) { + want := pgstate02Receipt(t, "intent-alpha", receiptProject, receiptExplorer) + replay := want + replay.RequestID = "req_02" + replay.CreatedAt = want.CreatedAt.Add(90 * time.Second) + left, err := explorer.ReceiptID(want) + if err != nil { + t.Fatal(err) + } + right, err := explorer.ReceiptID(replay) + if err != nil { + t.Fatal(err) + } + if left != right { + t.Fatalf("RequestID/CreatedAt leaked into the content identity: %s vs %s", left, right) + } + different := want + different.IntentDigest = "intent-beta" + differentKey, err := explorer.ReceiptID(different) + if err != nil { + t.Fatal(err) + } + if differentKey == left { + t.Fatal("intent digest change did not move the content identity") + } +} + +func TestPostgresReceiptInsertGetLiteralRows(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + + writer := f.newStore(t) + want := pgstate02Receipt(t, "intent-alpha", receiptProject, receiptExplorer) + created, err := writer.InsertCompilationReceipt(f.ctx, want) + if err != nil { + f.fail("insert receipt", err) + } + requireReceiptEqual(t, want, *created, "Insert result") + + row := f.receiptLiteralRow(t, receiptProject, receiptExplorer, want.ID) + if row.Project != receiptProject || row.ExplorerID != receiptExplorer || row.ID != want.ID { + t.Fatalf("literal identity = %q/%q/%q, want %q/%q/%q", row.Project, row.ExplorerID, row.ID, receiptProject, receiptExplorer, want.ID) + } + if row.ReceiptFormatVersion != int64(explorer.CurrentReceiptFormatVersion) { + t.Errorf("literal receipt_format_version = %d, want %d", row.ReceiptFormatVersion, explorer.CurrentReceiptFormatVersion) + } + if row.CompilerContractVersion != explorer.CurrentCompilerContractVersion { + t.Errorf("literal compiler_contract_version = %q, want %q", row.CompilerContractVersion, explorer.CurrentCompilerContractVersion) + } + if row.CompilationKey != want.CompilationKey || row.IntentDigest != "intent-alpha" || + row.SnapshotToken != "snapshot-pg-01" || row.SourceGeneration != "gen_pg_01" || + row.RecipeDigest != "sha256:recipe-digest" || row.ResolvedRecipeDigest != want.ResolvedRecipeDigest { + t.Fatalf("literal digest/generation columns drifted: %+v", row) + } + if row.DocType != "object" { + t.Errorf("doc jsonb_typeof = %q, want object", row.DocType) + } + if row.DocRequestID != "req_01" || row.DocCreatedAt != receiptCreatedAtText || row.DocIntentDigest != "intent-alpha" { + t.Errorf("literal doc members = %q/%q/%q, want req_01/%s/intent-alpha", + row.DocRequestID, row.DocCreatedAt, row.DocIntentDigest, receiptCreatedAtText) + } + if kinds := f.columnKinds(t, f.receiptTable(), "project = $1 AND explorer_id = $2 AND id = $3", + []any{receiptProject, receiptExplorer, want.ID}, "doc"); kinds[0] != "object" { + t.Fatalf("doc kind = %s", kinds[0]) + } + var bundleText string + if err := f.iso.QueryRow(f.ctx, + `SELECT doc->'normalizedBundle'::text FROM `+f.receiptTable()+` WHERE project = $1 AND explorer_id = $2 AND id = $3`, + receiptProject, receiptExplorer, want.ID).Scan(&bundleText); err != nil { + f.fail("read normalizedBundle literal", err) + } + if bundleText != literalCfgAlphaText { + t.Fatalf("normalizedBundle jsonb text = %s, want %s", bundleText, literalCfgAlphaText) + } + if n := f.receiptCount(t, receiptProject); n != 1 { + t.Fatalf("receipt row count = %d, want exactly 1", n) + } + + reader := f.newStore(t) + got, err := reader.GetCompilationReceiptForExplorer(f.ctx, receiptProject, receiptExplorer, want.ID) + if err != nil { + f.fail("get receipt on fresh pool", err) + } + requireReceiptEqual(t, want, *got, "fresh-pool Get") + + if _, err := reader.GetCompilationReceiptForExplorer(f.ctx, "proj_other", receiptExplorer, want.ID); !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("cross-project read error = %v, want ErrNotFound", err) + } + if _, err := reader.GetCompilationReceiptForExplorer(f.ctx, receiptProject, "exp_other", want.ID); !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("cross-explorer read error = %v, want ErrNotFound", err) + } +} + +func TestPostgresReceiptIdempotentSameContent(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + + writer := f.newStore(t) + want := pgstate02Receipt(t, "intent-idem", receiptProject, receiptExplorer) + if _, err := writer.InsertCompilationReceipt(f.ctx, want); err != nil { + f.fail("insert receipt", err) + } + before := f.rowSnapshot(t, f.receiptTable(), "project = $1 AND explorer_id = $2 AND id = $3", + receiptProject, receiptExplorer, want.ID) + + replay := want + replay.RequestID = "req-replay" + replay.CreatedAt = want.CreatedAt.Add(2 * time.Minute) + stored, err := writer.InsertCompilationReceipt(f.ctx, replay) + if err != nil { + f.fail("idempotent replay", err) + } + if !stored.CreatedAt.Equal(want.CreatedAt) || stored.RequestID != "req_01" { + t.Fatalf("replay returned the replay's metadata, not the stored row: createdAt=%s requestId=%q", + stored.CreatedAt, stored.RequestID) + } + after := f.rowSnapshot(t, f.receiptTable(), "project = $1 AND explorer_id = $2 AND id = $3", + receiptProject, receiptExplorer, want.ID) + if before != after { + t.Fatalf("idempotent replay mutated the row:\n before %s\n after %s", before, after) + } + if n := f.receiptCount(t, receiptProject); n != 1 { + t.Fatalf("receipt row count = %d, want exactly 1", n) + } +} + +func TestPostgresReceiptIDReuseDifferentContentRefused(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + writer := f.newStore(t) + + t.Run("legacy id reuse via conflict", func(t *testing.T) { + first := pgstate02LegacyReceipt("legacy-reuse-key", "intent-one") + if _, err := writer.InsertCompilationReceipt(f.ctx, first); err != nil { + f.fail("insert first legacy receipt", err) + } + before := f.rowSnapshot(t, f.receiptTable(), "project = $1 AND explorer_id = $2 AND id = $3", + receiptProject, receiptExplorer, first.ID) + + second := pgstate02LegacyReceipt("legacy-reuse-key", "intent-two") + _, err := writer.InsertCompilationReceipt(f.ctx, second) + if !errors.Is(err, explorer.ErrCorruptReceipt) { + t.Fatalf("different content at existing ID error = %v, want ErrCorruptReceipt", err) + } + after := f.rowSnapshot(t, f.receiptTable(), "project = $1 AND explorer_id = $2 AND id = $3", + receiptProject, receiptExplorer, first.ID) + if before != after { + t.Fatalf("refused reuse mutated the stored row:\n before %s\n after %s", before, after) + } + if n := f.receiptCount(t, receiptProject); n != 1 { + t.Fatalf("receipt row count = %d, want exactly 1", n) + } + stored, err := writer.GetCompilationReceiptForExplorer(f.ctx, receiptProject, receiptExplorer, first.ID) + if err != nil || stored.IntentDigest != "intent-one" { + t.Fatalf("stored receipt after refused reuse = %+v, %v", stored, err) + } + }) + + t.Run("content-addressed tamper refused before write", func(t *testing.T) { + valid := pgstate02Receipt(t, "intent-valid", receiptProject, receiptExplorer) + if _, err := writer.InsertCompilationReceipt(f.ctx, valid); err != nil { + f.fail("insert valid receipt", err) + } + before := f.rowSnapshot(t, f.receiptTable(), "project = $1 AND explorer_id = $2 AND id = $3", + receiptProject, receiptExplorer, valid.ID) + + tampered := valid + tampered.IntentDigest = "intent-tampered" + _, err := writer.InsertCompilationReceipt(f.ctx, tampered) + if err == nil { + t.Fatal("tampered content at a content-addressed ID was accepted") + } + if errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("tampered insert misclassified as missing: %v", err) + } + after := f.rowSnapshot(t, f.receiptTable(), "project = $1 AND explorer_id = $2 AND id = $3", + receiptProject, receiptExplorer, valid.ID) + if before != after { + t.Fatalf("refused tampered insert mutated the row:\n before %s\n after %s", before, after) + } + if n := f.receiptCount(t, receiptProject); n != 2 { + t.Fatalf("receipt row count = %d, want 2 (the valid receipt and the legacy one)", n) + } + }) +} + +func TestPostgresReceiptConcurrentIdenticalWritersTwoConnections(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + + storeA, pidA := f.newDedicatedStore(t) + storeB, pidB := f.newDedicatedStore(t) + requireDistinctBackendsPGState02(t, pidA, pidB, "store A", "store B") + + receipt := pgstate02Receipt(t, "intent-race", receiptProject, receiptExplorer) + start := make(chan struct{}) + var writers sync.WaitGroup + type result struct { + stored *explorer.CompilationReceipt + err error + } + results := make(chan result, 2) + for i := 0; i < 2; i++ { + writers.Add(1) + store := storeA + if i == 1 { + store = storeB + } + go func() { + defer writers.Done() + <-start + stored, err := store.InsertCompilationReceipt(f.ctx, receipt) + results <- result{stored: stored, err: err} + }() + } + close(start) + writers.Wait() + close(results) + + for result := range results { + if result.err != nil { + t.Fatalf("concurrent InsertCompilationReceipt error = %v", result.err) + } + requireReceiptEqual(t, receipt, *result.stored, "concurrent insert result") + } + if n := f.receiptCount(t, receiptProject); n != 1 { + t.Fatalf("receipt row count after race = %d, want exactly 1", n) + } +} + +func TestPostgresReceiptUncommittedWinnerClaimTwoConnections(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + + type claimResult struct { + stored *explorer.CompilationReceipt + err error + } + + rawA, pidA := f.newDedicatedPool(t) + storeB, pidB := f.newDedicatedStore(t) + requireDistinctBackendsPGState02(t, pidA, pidB, "raw connection A", "store B") + + winner := pgstate02Receipt(t, "intent-uncommitted", receiptProject, receiptExplorer) + winner.RequestID = "req_from_A" + winner.CreatedAt = time.Date(2026, 4, 5, 6, 7, 8, 654321000, time.UTC) + loser := winner + loser.RequestID = "req_from_B" + loser.CreatedAt = receiptCreatedAt + winnerDoc, err := json.Marshal(winner) + if err != nil { + t.Fatal(err) + } + rawInsert := pgstate02RawReceiptInsert(f.schema) + + insertBlockedUntil := func(commit bool) (*explorer.CompilationReceipt, error) { + connA, err := rawA.Acquire(f.ctx) + if err != nil { + f.fail("acquire raw connection A", err) + } + defer connA.Release() + tx, err := connA.Begin(f.ctx) + if err != nil { + f.fail("begin raw claim transaction", err) + } + if _, err := tx.Exec(f.ctx, rawInsert, compilationReceiptArgs(winner, winnerDoc)...); err != nil { + tx.Rollback(f.ctx) + f.fail("raw claim insert on A", err) + } + done := make(chan claimResult, 1) + go func() { + stored, err := storeB.InsertCompilationReceipt(f.ctx, loser) + done <- claimResult{stored: stored, err: err} + }() + + observed := false + deadline := time.Now().Add(5 * time.Second) + for !observed && time.Now().Before(deadline) { + var waiting bool + if err := f.iso.QueryRow(f.ctx, + `SELECT count(*) > 0 FROM pg_stat_activity + WHERE pid = $1 AND state = 'active' + AND wait_event_type IN ('Lock', 'Transaction')`, pidB).Scan(&waiting); err != nil { + tx.Rollback(f.ctx) + f.fail("witness wait event", err) + } + if waiting { + observed = true + break + } + select { + case r := <-done: + tx.Rollback(f.ctx) + t.Fatalf("precondition broken: claim finished before waiting on the uncommitted winner (err=%v)", r.err) + default: + } + time.Sleep(10 * time.Millisecond) + } + if !observed { + tx.Rollback(f.ctx) + t.Fatal("store B never blocked on the uncommitted claim on connection A") + } + if commit { + if err := tx.Commit(f.ctx); err != nil { + f.fail("commit raw claim", err) + } + } else { + if err := tx.Rollback(f.ctx); err != nil { + f.fail("rollback raw claim", err) + } + } + select { + case r := <-done: + return r.stored, r.err + case <-time.After(30 * time.Second): + t.Fatal("claim never resolved after the winner resolved") + return nil, nil + } + } + + t.Run("winner commits", func(t *testing.T) { + stored, err := insertBlockedUntil(true) + if err != nil { + f.fail("claim against committed winner", err) + } + requireReceiptEqual(t, winner, *stored, "claim reread of the committed winner") + if stored.RequestID != "req_from_A" { + t.Fatalf("claim returned the loser's row, not the winner's: requestId=%q", stored.RequestID) + } + if n := f.receiptCount(t, receiptProject); n != 1 { + t.Fatalf("receipt row count = %d, want exactly 1", n) + } + }) + + t.Run("winner aborts", func(t *testing.T) { + abortWinner := winner + abortWinner.IntentDigest = "intent-uncommitted-abort" + abortWinner.CompilationKey, err = explorer.CompilationKey(abortWinner) + if err != nil { + t.Fatal(err) + } + abortKey, err := explorer.ReceiptID(abortWinner) + if err != nil { + t.Fatal(err) + } + abortWinner.ID = abortKey + abortDoc, err := json.Marshal(abortWinner) + if err != nil { + t.Fatal(err) + } + abortLoser := abortWinner + abortLoser.RequestID = "req_from_B_abort" + abortLoser.CreatedAt = receiptCreatedAt + + connA, err := rawA.Acquire(f.ctx) + if err != nil { + f.fail("acquire raw connection A (abort case)", err) + } + defer connA.Release() + tx, err := connA.Begin(f.ctx) + if err != nil { + f.fail("begin raw claim transaction (abort case)", err) + } + if _, err := tx.Exec(f.ctx, rawInsert, compilationReceiptArgs(abortWinner, abortDoc)...); err != nil { + tx.Rollback(f.ctx) + f.fail("raw claim insert on A (abort case)", err) + } + done := make(chan claimResult, 1) + go func() { + stored, err := storeB.InsertCompilationReceipt(f.ctx, abortLoser) + done <- claimResult{stored: stored, err: err} + }() + time.Sleep(200 * time.Millisecond) + if err := tx.Rollback(f.ctx); err != nil { + f.fail("rollback raw claim (abort case)", err) + } + select { + case r := <-done: + if r.err != nil { + f.fail("claim after aborted winner", r.err) + } + if r.stored.RequestID != "req_from_B_abort" { + t.Fatalf("aborted winner left a row that is not the retry's own: requestId=%q", r.stored.RequestID) + } + case <-time.After(30 * time.Second): + t.Fatal("claim never resolved after the winner aborted") + } + var createdAt string + if err := f.iso.QueryRow(f.ctx, + `SELECT doc->>'createdAt' FROM `+f.receiptTable()+` WHERE project = $1 AND explorer_id = $2 AND id = $3`, + receiptProject, receiptExplorer, abortWinner.ID).Scan(&createdAt); err != nil { + f.fail("read aborted-case winner createdAt", err) + } + if createdAt != receiptCreatedAtText { + t.Fatalf("aborted winner left createdAt %q, want the retry's own %q", createdAt, receiptCreatedAtText) + } + }) +} + +func TestPostgresReceiptFrozenGuardProvenNonVacuous(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + + receipt := pgstate02Receipt(t, "intent-frozen", receiptProject, receiptExplorer) + f.rawInsertReceipt(t, receipt) + + mutations := []struct { + column string + set string + }{ + {"project", "'proj_intruder'"}, + {"explorer_id", "'exp_intruder'"}, + {"id", "'receipt_intruder'"}, + {"receipt_format_version", "9"}, + {"compiler_contract_version", "'loom.explorer.compiler/v99'"}, + {"compilation_key", "'compile_intruder'"}, + {"intent_digest", "'sha256:intruder'"}, + {"snapshot_token", "'intruder'"}, + {"source_generation", "'gen_intruder'"}, + {"recipe_digest", "'sha256:intruder'"}, + {"resolved_recipe_digest", "'sha256:intruder'"}, + {"doc", `'{"id":"intruder"}'::jsonb`}, + } + where := fmt.Sprintf("project = '%s' AND explorer_id = '%s' AND id = '%s'", + receipt.Project, receipt.ExplorerID, receipt.ID) + + for _, m := range mutations { + _, err := f.iso.Exec(f.ctx, + "UPDATE "+f.receiptTable()+" SET "+m.column+" = "+m.set+" WHERE "+where) + pgErr := requirePgErrPGState02(t, err, "23514", "frozen receipt column "+m.column) + if !strings.Contains(pgErr.Message, "insert-once") { + t.Fatalf("frozen receipt refusal on %s came from something other than the guard: %s", m.column, pgErr.Message) + } + } + if _, err := f.iso.Exec(f.ctx, "DELETE FROM "+f.receiptTable()+" WHERE "+where); err == nil { + t.Fatal("DELETE on a receipt was accepted while the frozen trigger is installed") + } else { + pgErr := requirePgErrPGState02(t, err, "23514", "receipt delete") + if !strings.Contains(pgErr.Message, "insert-once") { + t.Fatalf("receipt delete refusal came from something other than the guard: %s", pgErr.Message) + } + } + + if _, err := f.iso.Exec(f.ctx, + `DROP TRIGGER compilation_receipts_frozen ON `+f.receiptTable()); err != nil { + f.fail("drop frozen trigger", err) + } + if _, err := f.iso.Exec(f.ctx, + "UPDATE "+f.receiptTable()+" SET intent_digest = 'sha256:unfrozen' WHERE "+where); err != nil { + f.fail("mutating without the trigger must succeed", err) + } + var mutated string + if err := f.iso.QueryRow(f.ctx, + `SELECT intent_digest FROM `+f.receiptTable()+` WHERE `+where).Scan(&mutated); err != nil { + f.fail("read mutated intent_digest", err) + } + if mutated != "sha256:unfrozen" { + t.Fatalf("guard-removal probe did not persist: intent_digest = %q", mutated) + } + if _, err := f.iso.Exec(f.ctx, + "UPDATE "+f.receiptTable()+" SET doc = '{\"id\":\"unfrozen\"}'::jsonb WHERE "+where); err != nil { + f.fail("mutating doc without the trigger must succeed", err) + } + var docID string + if err := f.iso.QueryRow(f.ctx, + `SELECT doc->>'id' FROM `+f.receiptTable()+` WHERE `+where).Scan(&docID); err != nil { + f.fail("read mutated doc", err) + } + if docID != "unfrozen" { + t.Fatalf("guard-removal doc probe did not persist: doc->>'id' = %q", docID) + } + if _, err := f.iso.Exec(f.ctx, "DELETE FROM "+f.receiptTable()+" WHERE "+where); err != nil { + f.fail("delete without the trigger must succeed", err) + } + if n := f.receiptCount(t, receiptProject); n != 0 { + t.Fatalf("guard-removal delete did not remove the row, count = %d", n) + } + + f.rerunMigration(t, 5) + f.rawInsertReceipt(t, receipt) + if _, err := f.iso.Exec(f.ctx, + "UPDATE "+f.receiptTable()+" SET intent_digest = 'sha256:refrozen' WHERE "+where); err == nil { + t.Fatal("UPDATE accepted after the guard was restored from the migration text") + } else { + pgErr := requirePgErrPGState02(t, err, "23514", "restored frozen guard") + if !strings.Contains(pgErr.Message, "insert-once") { + t.Fatalf("restored guard refusal came from something else: %s", pgErr.Message) + } + } +} + +func TestPostgresReceiptCorruptRowDetectedOnRead(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + + writer := f.newStore(t) + receipt := pgstate02Receipt(t, "intent-corrupt", receiptProject, receiptExplorer) + if _, err := writer.InsertCompilationReceipt(f.ctx, receipt); err != nil { + f.fail("insert receipt", err) + } + + if _, err := f.iso.Exec(f.ctx, + `DROP TRIGGER compilation_receipts_frozen ON `+f.receiptTable()); err != nil { + f.fail("drop frozen trigger", err) + } + if _, err := f.iso.Exec(f.ctx, `UPDATE `+f.receiptTable()+` + SET doc = doc || '{"intentDigest":"sha256:tampered"}'::jsonb, + intent_digest = 'sha256:tampered' + WHERE project = $1 AND explorer_id = $2 AND id = $3`, + receiptProject, receiptExplorer, receipt.ID); err != nil { + f.fail("tamper receipt document", err) + } + var tampered string + if err := f.iso.QueryRow(f.ctx, + `SELECT doc->>'intentDigest' FROM `+f.receiptTable()+ + ` WHERE project = $1 AND explorer_id = $2 AND id = $3`, + receiptProject, receiptExplorer, receipt.ID).Scan(&tampered); err != nil { + f.fail("read tampered document", err) + } + if tampered != "sha256:tampered" { + t.Fatalf("tamper probe did not persist: %q", tampered) + } + f.rerunMigration(t, 5) + + if _, err := writer.GetCompilationReceiptForExplorer(f.ctx, receiptProject, receiptExplorer, receipt.ID); !errors.Is(err, explorer.ErrCorruptReceipt) { + t.Fatalf("read of tampered receipt error = %v, want ErrCorruptReceipt", err) + } +} + +func TestPostgresReceiptJSONTriStateFreshRead(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + writer := f.newStore(t) + + missing := pgstate02LegacyReceipt("legacy-tri-missing", "intent-missing") + missing.NormalizedBundle = nil + jsonNull := pgstate02LegacyReceipt("legacy-tri-null", "intent-null") + jsonNull.CompiledConfig = json.RawMessage("null") + jsonNull.PublicOutputContract = json.RawMessage("null") + documented := pgstate02LegacyReceipt("legacy-tri-doc", "intent-doc") + documented.CompiledConfig = json.RawMessage(cfgBeta) + documented.PublicOutputContract = json.RawMessage(resultsOK) + + for _, receipt := range []explorer.CompilationReceipt{missing, jsonNull, documented} { + if _, err := writer.InsertCompilationReceipt(f.ctx, receipt); err != nil { + f.fail("insert tri-state receipt "+receipt.ID, err) + } + } + + for _, tc := range []struct { + id string + wantField string + wantKind string + }{ + {"legacy-tri-missing", "compiledConfig", "missing"}, + {"legacy-tri-null", "compiledConfig", "null"}, + {"legacy-tri-doc", "compiledConfig", "object"}, + } { + var has bool + var kind string + if err := f.iso.QueryRow(f.ctx, + `SELECT doc ? 'compiledConfig', COALESCE(jsonb_typeof(doc->'compiledConfig'), 'missing') FROM `+ + f.receiptTable()+` WHERE project = $1 AND explorer_id = $2 AND id = $3`, + receiptProject, receiptExplorer, tc.id).Scan(&has, &kind); err != nil { + f.fail("read tri-state witness for "+tc.id, err) + } + if tc.wantKind == "missing" { + if has || kind != "missing" { + t.Fatalf("%s compiledConfig: key present=%v kind=%s, want absent/missing", tc.id, has, kind) + } + continue + } + if !has || kind != tc.wantKind { + t.Fatalf("%s compiledConfig: key present=%v kind=%s, want present/%s", tc.id, has, kind, tc.wantKind) + } + } + + reader := f.newStore(t) + gotMissing, err := reader.GetCompilationReceiptForExplorer(f.ctx, receiptProject, receiptExplorer, "legacy-tri-missing") + if err != nil { + f.fail("read missing-payload receipt", err) + } + if gotMissing.CompiledConfig != nil || gotMissing.PublicOutputContract != nil { + t.Fatalf("missing payloads revived: config=%q contract=%q", gotMissing.CompiledConfig, gotMissing.PublicOutputContract) + } + gotNull, err := reader.GetCompilationReceiptForExplorer(f.ctx, receiptProject, receiptExplorer, "legacy-tri-null") + if err != nil { + f.fail("read json-null receipt", err) + } + if string(gotNull.CompiledConfig) != "null" || string(gotNull.PublicOutputContract) != "null" { + t.Fatalf("JSON-null payload lost: config=%q contract=%q", gotNull.CompiledConfig, gotNull.PublicOutputContract) + } + gotDoc, err := reader.GetCompilationReceiptForExplorer(f.ctx, receiptProject, receiptExplorer, "legacy-tri-doc") + if err != nil { + f.fail("read documented receipt", err) + } + jsonEqual(t, json.RawMessage(cfgBeta), gotDoc.CompiledConfig, "documented compiledConfig") + jsonEqual(t, json.RawMessage(resultsOK), gotDoc.PublicOutputContract, "documented publicOutputContract") + var nilBundleKind string + if err := f.iso.QueryRow(f.ctx, + `SELECT COALESCE(jsonb_typeof(doc->'normalizedBundle'), 'missing') FROM `+ + f.receiptTable()+` WHERE project = $1 AND explorer_id = $2 AND id = 'legacy-tri-missing'`, + receiptProject, receiptExplorer).Scan(&nilBundleKind); err != nil { + f.fail("read nil bundle kind", err) + } + if nilBundleKind != "null" { + t.Fatalf("nil normalizedBundle jsonb typeof = %s, want null (encoding/json has no omitempty on this field)", nilBundleKind) + } +} diff --git a/internal/backends/postgres/internal/explorer/revisions.go b/internal/backends/postgres/internal/explorer/revisions.go new file mode 100644 index 000000000..df74bbeba --- /dev/null +++ b/internal/backends/postgres/internal/explorer/revisions.go @@ -0,0 +1,521 @@ +package postgres + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/calypr/loom/internal/dataframe/publication" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/explorer" +) + +const RevisionsTable = "explorer_revisions" + +const revisionInsertMaxAttempts = 4 + +var ( + ErrStaleRevisionTransition = errors.New("postgres explorer store: stale explorer revision lifecycle token") + ErrRevisionClaimLost = errors.New("postgres explorer store: revision insert lost every claim race to an aborted winner") + ErrInvalidRevision = errors.New("postgres explorer store: invalid explorer revision") +) + +var revisionColumns = []string{ + "id", + "project", + "explorer_id", + "config", + "authoring_bundle", + "intent_digest", + "compilation_receipt_id", + "public_output_contract", + "recipe", + "recipe_digest", + "resolved_schema_digest", + "source_generation", + "materializations", + "emitted_columns", + "dataset", + "quality_reports", + "content_digest", + "status", + "diagnostics", + "publication", + "created_by", + "created_at", + "ready_at", + "activated_at", + "failed_at", +} + +type revisionStatements struct { + insert string + claim string + get string + getScoped string + failAny string + failToken string +} + +func newRevisionStatements(schema string) revisionStatements { + table := sanitizeTableRef(schema, RevisionsTable) + cols := strings.Join(revisionColumns, ", ") + placeholders := orderedPlaceholders(len(revisionColumns)) + return revisionStatements{ + insert: "INSERT INTO " + table + " (" + cols + ") VALUES (" + placeholders + ") " + + "ON CONFLICT (id) DO NOTHING RETURNING " + cols, + claim: "SELECT " + cols + " FROM " + table + " WHERE id = $1 FOR UPDATE", + get: "SELECT " + cols + " FROM " + table + " WHERE id = $1", + getScoped: "SELECT " + cols + " FROM " + table + " WHERE project = $1 AND explorer_id = $2 AND id = $3", + failAny: "UPDATE " + table + " SET status = 'FAILED', diagnostics = $1, failed_at = $2 " + + "WHERE id = $3 AND status IN ('READY', 'ACTIVE') RETURNING " + cols, + failToken: "UPDATE " + table + " SET status = 'FAILED', diagnostics = $1, failed_at = $2 " + + "WHERE id = $3 AND status = $4 AND status IN ('READY', 'ACTIVE') RETURNING " + cols, + } +} + +type frozenRevision struct { + ID string + Project string + ExplorerID string + Config json.RawMessage + AuthoringBundle json.RawMessage + IntentDigest string + CompilationReceiptID string + PublicOutputContract json.RawMessage + Recipe recipe.Bundle + RecipeDigest string + ResolvedSchemaDigest string + SourceGeneration string + Materializations []explorer.Materialization + EmittedColumns []explorer.EmittedColumn + Dataset explorer.DatasetMetadata + QualityReports []publication.QualityReport +} + +func digestableRaw(raw json.RawMessage) (json.RawMessage, error) { + if len(raw) == 0 { + return json.RawMessage(`{"loomRaw":"missing"}`), nil + } + var compact bytes.Buffer + if err := json.Compact(&compact, raw); err != nil { + return nil, fmt.Errorf("invalid raw payload %q: %w", string(raw), err) + } + if compact.String() == "null" { + return json.RawMessage(`{"loomRaw":"null"}`), nil + } + return json.RawMessage(`{"loomRaw":"document","value":` + compact.String() + `}`), nil +} + +func frozenRevisionContent(revision explorer.Revision) (frozenRevision, error) { + config, err := digestableRaw(revision.Config) + if err != nil { + return frozenRevision{}, fmt.Errorf("config: %w", err) + } + authoring, err := digestableRaw(revision.AuthoringBundle) + if err != nil { + return frozenRevision{}, fmt.Errorf("authoringBundle: %w", err) + } + publicContract, err := digestableRaw(revision.PublicOutputContract) + if err != nil { + return frozenRevision{}, fmt.Errorf("publicOutputContract: %w", err) + } + return frozenRevision{ + ID: revision.ID, Project: revision.Project, ExplorerID: revision.ExplorerID, + Config: config, + AuthoringBundle: authoring, + IntentDigest: revision.IntentDigest, + CompilationReceiptID: revision.CompilationReceiptID, + PublicOutputContract: publicContract, + Recipe: revision.Recipe, + RecipeDigest: revision.RecipeDigest, + ResolvedSchemaDigest: revision.ResolvedSchemaDigest, + SourceGeneration: revision.SourceGeneration, + Materializations: append([]explorer.Materialization(nil), revision.Materializations...), + EmittedColumns: append([]explorer.EmittedColumn(nil), revision.EmittedColumns...), + Dataset: revision.Dataset, + QualityReports: publication.CloneQualityReports(revision.QualityReports), + }, nil +} + +func revisionContentDigest(revision explorer.Revision) (string, error) { + frozen, err := frozenRevisionContent(revision) + if err != nil { + return "", fmt.Errorf("encode frozen revision content: %w", err) + } + raw, err := json.Marshal(frozen) + if err != nil { + return "", fmt.Errorf("encode frozen revision content: %w", err) + } + sum := sha256.Sum256(raw) + return "sha256:" + hex.EncodeToString(sum[:]), nil +} + +func marshalOrNil(v any) ([]byte, error) { + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Slice, reflect.Ptr, reflect.Map: + if rv.IsNil() { + return nil, nil + } + } + return json.Marshal(v) +} + +type storedRevisionRow struct { + ID string + Project string + ExplorerID string + Config json.RawMessage + AuthoringBundle json.RawMessage + IntentDigest string + CompilationReceiptID string + PublicOutputContract json.RawMessage + Recipe json.RawMessage + RecipeDigest string + ResolvedSchemaDigest string + SourceGeneration string + Materializations json.RawMessage + EmittedColumns json.RawMessage + Dataset json.RawMessage + QualityReports json.RawMessage + ContentDigest string + Status string + Diagnostics json.RawMessage + Publication json.RawMessage + CreatedBy string + CreatedAt time.Time + ReadyAt *time.Time + ActivatedAt *time.Time + FailedAt *time.Time +} + +func scanRevisionRow(row rowScanner, out *storedRevisionRow) error { + return row.Scan( + &out.ID, + &out.Project, + &out.ExplorerID, + &out.Config, + &out.AuthoringBundle, + &out.IntentDigest, + &out.CompilationReceiptID, + &out.PublicOutputContract, + &out.Recipe, + &out.RecipeDigest, + &out.ResolvedSchemaDigest, + &out.SourceGeneration, + &out.Materializations, + &out.EmittedColumns, + &out.Dataset, + &out.QualityReports, + &out.ContentDigest, + &out.Status, + &out.Diagnostics, + &out.Publication, + &out.CreatedBy, + &out.CreatedAt, + &out.ReadyAt, + &out.ActivatedAt, + &out.FailedAt, + ) +} + +func decodeRevision(row storedRevisionRow) (*explorer.Revision, error) { + revision := &explorer.Revision{ + ID: row.ID, + Project: row.Project, + ExplorerID: row.ExplorerID, + Config: row.Config, + AuthoringBundle: row.AuthoringBundle, + IntentDigest: row.IntentDigest, + CompilationReceiptID: row.CompilationReceiptID, + PublicOutputContract: row.PublicOutputContract, + RecipeDigest: row.RecipeDigest, + ResolvedSchemaDigest: row.ResolvedSchemaDigest, + SourceGeneration: row.SourceGeneration, + Status: explorer.RevisionStatus(row.Status), + CreatedBy: row.CreatedBy, + CreatedAt: row.CreatedAt, + ReadyAt: row.ReadyAt, + ActivatedAt: row.ActivatedAt, + FailedAt: row.FailedAt, + } + if err := json.Unmarshal(row.Recipe, &revision.Recipe); err != nil { + return nil, fmt.Errorf("postgres explorer store: stored revision %q canonical recipe is undecodable: %w", row.ID, err) + } + if err := json.Unmarshal(row.Dataset, &revision.Dataset); err != nil { + return nil, fmt.Errorf("postgres explorer store: stored revision %q dataset metadata is undecodable: %w", row.ID, err) + } + if err := json.Unmarshal(row.Publication, &revision.Publication); err != nil { + return nil, fmt.Errorf("postgres explorer store: stored revision %q publication metadata is undecodable: %w", row.ID, err) + } + if err := unmarshalOrNil(row.Materializations, &revision.Materializations); err != nil { + return nil, fmt.Errorf("postgres explorer store: stored revision %q materializations are undecodable: %w", row.ID, err) + } + if err := unmarshalOrNil(row.EmittedColumns, &revision.EmittedColumns); err != nil { + return nil, fmt.Errorf("postgres explorer store: stored revision %q emitted columns are undecodable: %w", row.ID, err) + } + if err := unmarshalOrNil(row.QualityReports, &revision.QualityReports); err != nil { + return nil, fmt.Errorf("postgres explorer store: stored revision %q quality reports are undecodable: %w", row.ID, err) + } + if err := unmarshalOrNil(row.Diagnostics, &revision.Diagnostics); err != nil { + return nil, fmt.Errorf("postgres explorer store: stored revision %q diagnostics are undecodable: %w", row.ID, err) + } + return revision, nil +} + +func unmarshalOrNil(raw json.RawMessage, out any) error { + if len(raw) == 0 { + return nil + } + return json.Unmarshal(raw, out) +} + +type revisionPayloads struct { + config []byte + authoringBundle []byte + publicOutputContract []byte + recipe []byte + dataset []byte + qualityReports []byte + publication []byte + materializations []byte + emittedColumns []byte + diagnostics []byte +} + +func encodeRevisionPayloads(revision explorer.Revision) (revisionPayloads, error) { + var payloads revisionPayloads + payloads.config = jsonParam(revision.Config) + payloads.authoringBundle = jsonParam(revision.AuthoringBundle) + payloads.publicOutputContract = jsonParam(revision.PublicOutputContract) + var err error + if payloads.recipe, err = json.Marshal(revision.Recipe); err != nil { + return payloads, fmt.Errorf("encode canonical recipe: %w", err) + } + if payloads.dataset, err = json.Marshal(revision.Dataset); err != nil { + return payloads, fmt.Errorf("encode dataset metadata: %w", err) + } + if payloads.publication, err = json.Marshal(revision.Publication); err != nil { + return payloads, fmt.Errorf("encode publication metadata: %w", err) + } + if payloads.materializations, err = marshalOrNil(revision.Materializations); err != nil { + return payloads, fmt.Errorf("encode materializations: %w", err) + } + if payloads.emittedColumns, err = marshalOrNil(revision.EmittedColumns); err != nil { + return payloads, fmt.Errorf("encode emitted columns: %w", err) + } + if payloads.qualityReports, err = marshalOrNil(revision.QualityReports); err != nil { + return payloads, fmt.Errorf("encode quality reports: %w", err) + } + return payloads, nil +} + +func revisionArgs(revision explorer.Revision, payloads revisionPayloads, contentDigest string, diagnostics []byte) []any { + return []any{ + revision.ID, + revision.Project, + revision.ExplorerID, + payloads.config, + payloads.authoringBundle, + revision.IntentDigest, + revision.CompilationReceiptID, + payloads.publicOutputContract, + payloads.recipe, + revision.RecipeDigest, + revision.ResolvedSchemaDigest, + revision.SourceGeneration, + payloads.materializations, + payloads.emittedColumns, + payloads.dataset, + payloads.qualityReports, + contentDigest, + string(revision.Status), + diagnostics, + payloads.publication, + revision.CreatedBy, + revision.CreatedAt, + revision.ReadyAt, + revision.ActivatedAt, + revision.FailedAt, + } +} + +func (s *Store) InsertRevision(ctx context.Context, revision explorer.Revision) (*explorer.Revision, error) { + if strings.TrimSpace(revision.ID) == "" { + return nil, fmt.Errorf("%w: revision ID is required", ErrInvalidRevision) + } + contentDigest, err := revisionContentDigest(revision) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: content identity of revision %q: %w", revision.ID, err) + } + payloads, err := encodeRevisionPayloads(revision) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: encode revision %q: %w", revision.ID, err) + } + diagnostics, err := marshalOrNil(revision.Diagnostics) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: encode revision %q diagnostics: %w", revision.ID, err) + } + args := revisionArgs(revision, payloads, contentDigest, diagnostics) + + var lastErr error + for attempt := 1; attempt <= revisionInsertMaxAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("postgres explorer store: insert revision %q: %w", revision.ID, err) + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire revision insert connection: %w", err) + } + stored, claimErr := s.claimedRevision(ctx, conn, revision.ID, args, contentDigest) + conn.Release() + if stored != nil { + return stored, nil + } + lastErr = claimErr + if attempt == revisionInsertMaxAttempts { + break + } + if waitErr := waitForStoreRetry(ctx, time.Duration(attempt)*receiptConflictRetryDelay); waitErr != nil { + return nil, fmt.Errorf("postgres explorer store: retry revision insert %q: %w", + revision.ID, errors.Join(lastErr, waitErr)) + } + } + if lastErr == nil { + lastErr = ErrRevisionClaimLost + } + return nil, fmt.Errorf("postgres explorer store: insert revision %q: %w", revision.ID, lastErr) +} + +func (s *Store) claimedRevision(ctx context.Context, conn *pgxpool.Conn, id string, args []any, contentDigest string) (*explorer.Revision, error) { + var row storedRevisionRow + err := scanRevisionRow(conn.QueryRow(ctx, s.revision.insert, args...), &row) + if err == nil { + return decodeRevision(row) + } + if !errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("postgres explorer store: claim revision %q: %w", id, err) + } + + existing, err := s.readRevisionRow(ctx, conn, s.revision.claim, id) + if err != nil { + return nil, err + } + if existing == nil { + return nil, fmt.Errorf("%w: revision claim %q vanished before it became visible", explorer.ErrNotFound, id) + } + if existing.ContentDigest != contentDigest { + return nil, fmt.Errorf("%w: revision id %q was reused with different content (stored %s, requested %s)", + explorer.ErrImmutableRevision, id, existing.ContentDigest, contentDigest) + } + return decodeRevision(*existing) +} + +func (s *Store) GetRevision(ctx context.Context, id string) (*explorer.Revision, error) { + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire revision read connection: %w", err) + } + defer conn.Release() + + row, err := s.readRevisionRow(ctx, conn, s.revision.get, id) + if err != nil { + return nil, err + } + if row == nil { + return nil, explorer.ErrNotFound + } + return decodeRevision(*row) +} + +func (s *Store) GetRevisionForExplorer(ctx context.Context, project, explorerID, id string) (*explorer.Revision, error) { + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire scoped revision read connection: %w", err) + } + defer conn.Release() + + var row storedRevisionRow + err = scanRevisionRow(conn.QueryRow(ctx, s.revision.getScoped, project, explorerID, id), &row) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrNotFound + } + return nil, fmt.Errorf("postgres explorer store: read scoped revision %q/%q/%q: %w", project, explorerID, id, err) + } + return decodeRevision(row) +} + +func (s *Store) readRevisionRow(ctx context.Context, conn *pgxpool.Conn, statement, id string) (*storedRevisionRow, error) { + var row storedRevisionRow + err := scanRevisionRow(conn.QueryRow(ctx, statement, id), &row) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + return nil, fmt.Errorf("postgres explorer store: read revision %q: %w", id, err) + } + return &row, nil +} + +func (s *Store) FailRevision(ctx context.Context, id string, diagnostics []explorer.Diagnostic) (*explorer.Revision, error) { + return s.failRevision(ctx, id, diagnostics, false, "") +} + +func (s *Store) FailRevisionFrom(ctx context.Context, id string, expected explorer.RevisionStatus, diagnostics []explorer.Diagnostic) (*explorer.Revision, error) { + switch expected { + case explorer.RevisionReady, explorer.RevisionActive, explorer.RevisionSuperseded, explorer.RevisionFailed: + default: + return nil, fmt.Errorf("%w: unknown expected status %q", ErrInvalidRevision, string(expected)) + } + return s.failRevision(ctx, id, diagnostics, true, string(expected)) +} + +func (s *Store) failRevision(ctx context.Context, id string, diagnostics []explorer.Diagnostic, useToken bool, expected string) (*explorer.Revision, error) { + diagBytes, err := marshalOrNil(diagnostics) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: encode failure diagnostics for revision %q: %w", id, err) + } + failedAt := time.Now().UTC() + + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire revision fail connection: %w", err) + } + defer conn.Release() + + statement := s.revision.failAny + args := []any{diagBytes, failedAt, id} + if useToken { + statement = s.revision.failToken + args = append(args, expected) + } + + var row storedRevisionRow + err = scanRevisionRow(conn.QueryRow(ctx, statement, args...), &row) + if errors.Is(err, pgx.ErrNoRows) { + existing, readErr := s.readRevisionRow(ctx, conn, s.revision.get, id) + if readErr != nil { + return nil, errors.Join(err, readErr) + } + if existing == nil { + return nil, explorer.ErrNotFound + } + return nil, fmt.Errorf("%w: revision %q is %s", ErrStaleRevisionTransition, id, existing.Status) + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: fail revision %q: %w", id, err) + } + return decodeRevision(row) +} diff --git a/internal/backends/postgres/internal/explorer/revisions_test.go b/internal/backends/postgres/internal/explorer/revisions_test.go new file mode 100644 index 000000000..f912eb9a9 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/revisions_test.go @@ -0,0 +1,1401 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/dataframe/publication" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/explorer" +) + +const ( + revisionProject = "proj_revisions" + revisionExplorer = "exp_revisions" +) + +var revisionCreatedAt = time.Date(2026, 2, 3, 4, 5, 6, 654321000, time.UTC) + +const revisionCreatedAtText = "2026-02-03 04:05:06.654321" + +func pgstate02RevisionBundle(name string) recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: name, + TranslationVersion: "test", + Outputs: []recipe.Output{{ + Name: "output", RootResourceType: "Patient", RowGrain: "resource", + }}, + } +} + +func pgstate02Revision(id, name string) explorer.Revision { + return explorer.Revision{ + ID: id, + Project: revisionProject, + ExplorerID: revisionExplorer, + Config: json.RawMessage(cfgAlpha), + AuthoringBundle: json.RawMessage(cfgBeta), + IntentDigest: "sha256:intent-" + id, + CompilationReceiptID: "receipt_" + id, + PublicOutputContract: json.RawMessage(resultsOK), + Recipe: pgstate02RevisionBundle(name), + RecipeDigest: "sha256:recipe-" + id, + ResolvedSchemaDigest: "sha256:schema-" + id, + SourceGeneration: "gen_pg_01", + Materializations: []explorer.Materialization{{ + OutputID: "output", Output: "output", MaterializationID: "mat_1", + Fingerprint: "fp-1", + Columns: []publication.PhysicalColumn{{ + Name: "day", ClickHouse: "Date", LogicalType: "date", + }}, + }}, + EmittedColumns: []explorer.EmittedColumn{{ + EmissionID: "emit_1", OutputID: "output", PublicColumn: "day", + LogicalType: "date", Shape: "scalar", Filterable: true, + }}, + Dataset: explorer.DatasetMetadata{ + Generation: "gen_pg_01", + SchemaDigest: "sha256:dataset", + Outputs: []explorer.DatasetOutput{{Name: "output", State: "QUERYABLE", Queryable: true}}, + }, + Status: explorer.RevisionReady, + CreatedBy: "author@calypr", + CreatedAt: revisionCreatedAt, + } +} + +func pgstate02Diagnostic(code string) []explorer.Diagnostic { + return []explorer.Diagnostic{{Severity: "ERROR", Code: code, Message: "witnessed failure"}} +} + +func requireRevisionEqual(t *testing.T, want, got explorer.Revision, where string) { + t.Helper() + if got.ID != want.ID || got.Project != want.Project || got.ExplorerID != want.ExplorerID { + t.Fatalf("%s: identity changed: got %q/%q/%q want %q/%q/%q", where, + got.Project, got.ExplorerID, got.ID, want.Project, want.ExplorerID, want.ID) + } + if got.IntentDigest != want.IntentDigest || got.CompilationReceiptID != want.CompilationReceiptID || + got.RecipeDigest != want.RecipeDigest || got.ResolvedSchemaDigest != want.ResolvedSchemaDigest || + got.SourceGeneration != want.SourceGeneration || got.Status != want.Status || + got.CreatedBy != want.CreatedBy { + t.Fatalf("%s: frozen scalars/status changed: got %+v want %+v", where, got, want) + } + if !got.CreatedAt.Equal(want.CreatedAt) { + t.Fatalf("%s: createdAt changed: got %s want %s", where, got.CreatedAt, want.CreatedAt) + } + jsonEqual(t, want.Config, got.Config, where+" config") + jsonEqual(t, want.AuthoringBundle, got.AuthoringBundle, where+" authoringBundle") + jsonEqual(t, want.PublicOutputContract, got.PublicOutputContract, where+" publicOutputContract") + if got.Recipe.Name != want.Recipe.Name || len(got.Recipe.Outputs) != len(want.Recipe.Outputs) { + t.Fatalf("%s: canonical recipe not lossless: got %+v want %+v", where, got.Recipe, want.Recipe) + } + if len(want.Materializations) != len(got.Materializations) { + t.Fatalf("%s: materialization count changed: got %d want %d", where, len(got.Materializations), len(want.Materializations)) + } + if len(want.Materializations) > 0 { + w, g := want.Materializations[0], got.Materializations[0] + if g.OutputID != w.OutputID || g.MaterializationID != w.MaterializationID || + g.Fingerprint != w.Fingerprint || len(g.Columns) != len(w.Columns) || + (len(w.Columns) > 0 && g.Columns[0].Name != w.Columns[0].Name) { + t.Fatalf("%s: materialization mapping not lossless: got %+v want %+v", where, g, w) + } + } + if len(want.EmittedColumns) != len(got.EmittedColumns) || + (len(want.EmittedColumns) > 0 && got.EmittedColumns[0].PublicColumn != want.EmittedColumns[0].PublicColumn) { + t.Fatalf("%s: emitted columns changed: got %+v want %+v", where, got.EmittedColumns, want.EmittedColumns) + } + if got.Dataset.Generation != want.Dataset.Generation || + got.Dataset.SchemaDigest != want.Dataset.SchemaDigest || + len(got.Dataset.Outputs) != len(want.Dataset.Outputs) { + t.Fatalf("%s: dataset metadata changed: got %+v want %+v", where, got.Dataset, want.Dataset) + } + if (want.Diagnostics == nil) != (got.Diagnostics == nil) || + len(want.Diagnostics) != len(got.Diagnostics) || + (len(want.Diagnostics) > 0 && got.Diagnostics[0].Code != want.Diagnostics[0].Code) { + t.Fatalf("%s: diagnostics changed: got %+v want %+v", where, got.Diagnostics, want.Diagnostics) + } +} + +func pgstate02RevisionArgs(revision explorer.Revision) ([]any, string, error) { + digest, err := revisionContentDigest(revision) + if err != nil { + return nil, "", err + } + payloads, err := encodeRevisionPayloads(revision) + if err != nil { + return nil, "", err + } + diagnostics, err := marshalOrNil(revision.Diagnostics) + if err != nil { + return nil, "", err + } + return revisionArgs(revision, payloads, digest, diagnostics), digest, nil +} + +func (f *pgState02Fixture) rawInsertRevision(t *testing.T, revision explorer.Revision) string { + t.Helper() + args, digest, err := pgstate02RevisionArgs(revision) + if err != nil { + f.fail("encode raw revision fixture", err) + } + statements := newRevisionStatements(f.schema) + insert := strings.Replace(statements.insert, + " ON CONFLICT (id) DO NOTHING", "", 1) + if _, err := f.iso.Exec(f.ctx, insert, args...); err != nil { + f.fail("raw insert revision "+revision.ID, err) + } + return digest +} + +type revisionLiteralRow struct { + ID string + Project string + ExplorerID string + ConfigKind string + AuthoringKind string + ContractKind string + MaterializationsKind string + EmittedKind string + DatasetKind string + QualityKind string + DiagnosticsKind string + PublicationKind string + ContentDigest string + Status string + CreatedBy string + CreatedAtText string + FailedAtText string + ActivatedAtText string + ConfigText string +} + +func (f *pgState02Fixture) revisionLiteralRow(t *testing.T, id string) revisionLiteralRow { + t.Helper() + var row revisionLiteralRow + err := f.iso.QueryRow(f.ctx, `SELECT + id, project, explorer_id, + COALESCE(jsonb_typeof(config), 'missing'), + COALESCE(jsonb_typeof(authoring_bundle), 'missing'), + COALESCE(jsonb_typeof(public_output_contract), 'missing'), + COALESCE(jsonb_typeof(materializations), 'missing'), + COALESCE(jsonb_typeof(emitted_columns), 'missing'), + jsonb_typeof(dataset), + COALESCE(jsonb_typeof(quality_reports), 'missing'), + COALESCE(jsonb_typeof(diagnostics), 'missing'), + jsonb_typeof(publication), + content_digest, status, created_by, + to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), + COALESCE(to_char(failed_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), ''), + COALESCE(to_char(activated_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), ''), + COALESCE(config::text, '') + FROM `+f.revisionTable()+` WHERE id = $1`, id).Scan( + &row.ID, &row.Project, &row.ExplorerID, + &row.ConfigKind, &row.AuthoringKind, &row.ContractKind, + &row.MaterializationsKind, &row.EmittedKind, &row.DatasetKind, + &row.QualityKind, &row.DiagnosticsKind, &row.PublicationKind, + &row.ContentDigest, &row.Status, &row.CreatedBy, + &row.CreatedAtText, &row.FailedAtText, &row.ActivatedAtText, + &row.ConfigText) + if err != nil { + f.fail("read literal revision row "+id, err) + } + return row +} + +func (f *pgState02Fixture) revisionCount(t *testing.T, project string) int { + t.Helper() + var n int + if err := f.iso.QueryRow(f.ctx, + `SELECT count(*) FROM `+f.revisionTable()+` WHERE project = $1`, project).Scan(&n); err != nil { + f.fail("count revision rows", err) + } + return n +} + +func TestPostgresRevisionSQLContract(t *testing.T) { + statements := newRevisionStatements("loom_control") + if !strings.Contains(statements.insert, "ON CONFLICT (id) DO NOTHING") { + t.Error("revision insert must be insert-once: ON CONFLICT (id) DO NOTHING") + } + if !strings.Contains(statements.insert, "RETURNING") { + t.Error("revision insert must RETURNING the stored row") + } + if !strings.Contains(statements.insert, `"loom_control"."`+RevisionsTable+`"`) { + t.Error("revision insert must bind the schema-qualified, sanitized table") + } + if !strings.Contains(statements.claim, "WHERE id = $1 FOR UPDATE") { + t.Error("the revision conflict reread must be FOR UPDATE by id") + } + if !strings.Contains(statements.getScoped, "WHERE project = $1 AND explorer_id = $2 AND id = $3") { + t.Error("the scoped revision read must be by project AND explorer_id AND id") + } + if !strings.Contains(statements.failAny, "WHERE id = $3 AND status IN ('READY', 'ACTIVE')") { + t.Error("FailRevision must be a CAS admitting only the two failure-eligible states") + } + if !strings.Contains(statements.failToken, "WHERE id = $3 AND status = $4 AND status IN ('READY', 'ACTIVE')") { + t.Error("FailRevisionFrom must carry the expected-status token in its WHERE clause") + } + if !strings.Contains(statements.failAny, "UPDATE") { + t.Error("FailRevision must be an UPDATE") + } + for i := 1; i <= len(revisionColumns); i++ { + if !strings.Contains(statements.insert, fmt.Sprintf("$%d", i)) { + t.Fatalf("revision insert is missing ordered parameter $%d", i) + } + } + if strings.Contains(statements.insert, fmt.Sprintf("$%d", len(revisionColumns)+1)) { + t.Fatalf("revision insert carries more than %d parameters", len(revisionColumns)) + } + args, digest, err := pgstate02RevisionArgs(pgstate02Revision("rev_contract", "contract")) + if err != nil { + t.Fatal(err) + } + if got := len(args); got != len(revisionColumns) { + t.Fatalf("revisionArgs produced %d values, want %d (one per column)", got, len(revisionColumns)) + } + if !strings.HasPrefix(digest, "sha256:") || len(digest) != len("sha256:")+64 { + t.Fatalf("content digest %q is not sha256:<64 hex>", digest) + } +} + +func TestPostgresRevisionContentDigestBoundaries(t *testing.T) { + base := pgstate02Revision("rev_digest", "digest") + digestOf := func(r explorer.Revision) string { + _, digest, err := pgstate02RevisionArgs(r) + if err != nil { + t.Fatal(err) + } + return digest + } + + lifecycle := base + lifecycle.Status = explorer.RevisionActive + lifecycle.Diagnostics = pgstate02Diagnostic("LATE") + activated := revisionCreatedAt.Add(time.Minute) + lifecycle.ActivatedAt = &activated + if digestOf(lifecycle) != digestOf(base) { + t.Fatal("mutable lifecycle fields leaked into the frozen content digest") + } + + for name, mutate := range map[string]func(*explorer.Revision){ + "config": func(r *explorer.Revision) { r.Config = json.RawMessage(cfgBeta) }, + "authoringBundle": func(r *explorer.Revision) { r.AuthoringBundle = json.RawMessage(cfgAlpha) }, + "publicOutput": func(r *explorer.Revision) { r.PublicOutputContract = json.RawMessage(cfgBeta) }, + "recipe": func(r *explorer.Revision) { r.Recipe = pgstate02RevisionBundle("other") }, + "recipeDigest": func(r *explorer.Revision) { r.RecipeDigest = "sha256:other" }, + "resolvedSchema": func(r *explorer.Revision) { r.ResolvedSchemaDigest = "sha256:other" }, + "sourceGeneration": func(r *explorer.Revision) { r.SourceGeneration = "gen_other" }, + "materializationMap": func(r *explorer.Revision) { r.Materializations[0].MaterializationID = "mat_2" }, + "materializationDrop": func(r *explorer.Revision) { r.Materializations = nil }, + "materializationEmpt": func(r *explorer.Revision) { r.Materializations = []explorer.Materialization{} }, + "emittedColumns": func(r *explorer.Revision) { r.EmittedColumns = nil }, + "dataset": func(r *explorer.Revision) { r.Dataset.Generation = "gen_other" }, + "qualityReports": func(r *explorer.Revision) { r.QualityReports = []publication.QualityReport{{ID: "q1"}} }, + "intent": func(r *explorer.Revision) { r.IntentDigest = "sha256:other" }, + "receipt": func(r *explorer.Revision) { r.CompilationReceiptID = "receipt_other" }, + "project": func(r *explorer.Revision) { r.Project = "proj_intruder" }, + "explorer": func(r *explorer.Revision) { r.ExplorerID = "exp_intruder" }, + "rawKeyOrder": func(r *explorer.Revision) { + r.Config = json.RawMessage(`{"limit": 25, "beta": true, "note": null, "title": "Alpha", "filters": [{"op": "eq", "value": "2026-01-01", "column": "day"}]}`) + }, + } { + mutated := pgstate02Revision("rev_digest", "digest") + mutate(&mutated) + if digestOf(mutated) == digestOf(base) { + t.Fatalf("frozen dimension %s did not move the content digest", name) + } + } +} + +func TestPostgresRevisionTriStateArangoParityReplay(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + writer := f.newStore(t) + + nullish := pgstate02Revision("rev_tri_null", "tri") + nullish.Config = json.RawMessage("null") + nullish.QualityReports = nil + if _, err := writer.InsertRevision(f.ctx, nullish); err != nil { + f.fail("insert explicit-null revision", err) + } + before := f.rowSnapshot(t, f.revisionTable(), "id = $1", nullish.ID) + + missingReplay := nullish + missingReplay.Config = nil + if _, err := writer.InsertRevision(f.ctx, missingReplay); !errors.Is(err, explorer.ErrImmutableRevision) { + t.Fatalf("missing-payload replay against stored JSON-null error = %v, want ErrImmutableRevision (Arango parity)", err) + } + if after := f.rowSnapshot(t, f.revisionTable(), "id = $1", nullish.ID); after != before { + t.Fatalf("refused tri-state replay mutated the stored row:\n before %s\n after %s", before, after) + } + + exact := nullish + stored, err := writer.InsertRevision(f.ctx, exact) + if err != nil { + f.fail("byte-exact explicit-null replay was refused", err) + } + requireRevisionEqual(t, nullish, *stored, "explicit-null idempotent replay") + + emptyQR := nullish + emptyQR.QualityReports = []publication.QualityReport{} + if _, err := writer.InsertRevision(f.ctx, emptyQR); err != nil { + t.Fatalf("nil-vs-empty quality-report replay was refused: %v (Arango replays this idempotently via CloneQualityReports)", err) + } + + missingRow := pgstate02Revision("rev_tri_missing", "tri") + missingRow.Config = nil + if _, err := writer.InsertRevision(f.ctx, missingRow); err != nil { + f.fail("insert missing-payload revision", err) + } + emptyRaw := missingRow + emptyRaw.Config = json.RawMessage{} + if _, err := writer.InsertRevision(f.ctx, emptyRaw); err != nil { + t.Fatalf("empty RawMessage replay against a stored missing payload was refused: %v (omitempty collapses both classes in Arango)", err) + } +} + +func TestPostgresRevisionFailTokenAndIDValidation(t *testing.T) { + store, err := New(&pgstore.Client{}, "loom_control") + if err != nil { + t.Fatal(err) + } + if _, err := store.FailRevisionFrom(context.Background(), "rev", explorer.RevisionStatus("BOGUS"), nil); !errors.Is(err, ErrInvalidRevision) { + t.Fatalf("unknown expected status error = %v, want ErrInvalidRevision", err) + } + if _, err := store.InsertRevision(context.Background(), explorer.Revision{ID: " "}); !errors.Is(err, ErrInvalidRevision) { + t.Fatalf("blank revision ID error = %v, want ErrInvalidRevision", err) + } +} + +func TestPostgresRevisionInsertGetLiteralRows(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + + writer := f.newStore(t) + want := pgstate02Revision("rev_literal", "literal") + created, err := writer.InsertRevision(f.ctx, want) + if err != nil { + f.fail("insert revision", err) + } + requireRevisionEqual(t, want, *created, "Insert result") + + _, digest, err := pgstate02RevisionArgs(want) + if err != nil { + t.Fatal(err) + } + row := f.revisionLiteralRow(t, want.ID) + if row.Project != revisionProject || row.ExplorerID != revisionExplorer || row.ID != want.ID { + t.Fatalf("literal identity = %q/%q/%q", row.Project, row.ExplorerID, row.ID) + } + if row.ContentDigest != digest { + t.Fatalf("literal content_digest = %q, want the Go-computed canonical digest %q", row.ContentDigest, digest) + } + if row.Status != "READY" || row.CreatedBy != "author@calypr" { + t.Fatalf("literal status/created_by = %q/%q", row.Status, row.CreatedBy) + } + if row.CreatedAtText != revisionCreatedAtText { + t.Fatalf("literal created_at = %q, want %q", row.CreatedAtText, revisionCreatedAtText) + } + if row.FailedAtText != "" || row.ActivatedAtText != "" { + t.Fatalf("lifecycle timestamps present at insert: failed=%q activated=%q", row.FailedAtText, row.ActivatedAtText) + } + if row.ConfigText != literalCfgAlphaText { + t.Fatalf("literal config jsonb text = %s, want %s", row.ConfigText, literalCfgAlphaText) + } + for name, kind := range map[string]string{ + "config": "object", "authoring_bundle": "object", "public_output_contract": "object", + "materializations": "array", "emitted_columns": "array", "dataset": "object", + "quality_reports": "missing", "diagnostics": "missing", "publication": "object", + } { + var got string + switch name { + case "config": + got = row.ConfigKind + case "authoring_bundle": + got = row.AuthoringKind + case "public_output_contract": + got = row.ContractKind + case "materializations": + got = row.MaterializationsKind + case "emitted_columns": + got = row.EmittedKind + case "dataset": + got = row.DatasetKind + case "quality_reports": + got = row.QualityKind + case "diagnostics": + got = row.DiagnosticsKind + case "publication": + got = row.PublicationKind + } + if got != kind { + t.Fatalf("literal jsonb kind %s = %q, want %q", name, got, kind) + } + } + if n := f.revisionCount(t, revisionProject); n != 1 { + t.Fatalf("revision row count = %d, want exactly 1", n) + } + + reader := f.newStore(t) + got, err := reader.GetRevision(f.ctx, want.ID) + if err != nil { + f.fail("GetRevision on fresh pool", err) + } + requireRevisionEqual(t, want, *got, "fresh-pool GetRevision") + + scoped, err := reader.GetRevisionForExplorer(f.ctx, revisionProject, revisionExplorer, want.ID) + if err != nil { + f.fail("GetRevisionForExplorer on fresh pool", err) + } + requireRevisionEqual(t, want, *scoped, "fresh-pool GetRevisionForExplorer") + + if _, err := reader.GetRevisionForExplorer(f.ctx, "proj_other", revisionExplorer, want.ID); !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("cross-project scoped read error = %v, want ErrNotFound", err) + } + if _, err := reader.GetRevisionForExplorer(f.ctx, revisionProject, "exp_other", want.ID); !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("cross-explorer scoped read error = %v, want ErrNotFound", err) + } +} + +func TestPostgresRevisionIdempotentSameContent(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + writer := f.newStore(t) + + want := pgstate02Revision("rev_idem", "idem") + if _, err := writer.InsertRevision(f.ctx, want); err != nil { + f.fail("insert revision", err) + } + before := f.rowSnapshot(t, f.revisionTable(), "id = $1", want.ID) + + replay := want + replay.Status = explorer.RevisionActive + replay.Diagnostics = pgstate02Diagnostic("REPLAY") + replay.ActivatedAt = &revisionCreatedAt + stored, err := writer.InsertRevision(f.ctx, replay) + if err != nil { + f.fail("idempotent replay", err) + } + if stored.Status != explorer.RevisionReady || stored.Diagnostics != nil || stored.ActivatedAt != nil { + t.Fatalf("replay returned the replay's lifecycle fields, not the stored row: %+v", stored) + } + after := f.rowSnapshot(t, f.revisionTable(), "id = $1", want.ID) + if before != after { + t.Fatalf("idempotent replay mutated the row:\n before %s\n after %s", before, after) + } + if n := f.revisionCount(t, revisionProject); n != 1 { + t.Fatalf("revision row count = %d, want exactly 1", n) + } +} + +func TestPostgresRevisionSameIDDifferentContentRefused(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + writer := f.newStore(t) + + first := pgstate02Revision("rev_collision", "collision") + if _, err := writer.InsertRevision(f.ctx, first); err != nil { + f.fail("insert first revision", err) + } + before := f.rowSnapshot(t, f.revisionTable(), "id = $1", first.ID) + + second := pgstate02Revision("rev_collision", "collision-other-recipe") + _, err := writer.InsertRevision(f.ctx, second) + if !errors.Is(err, explorer.ErrImmutableRevision) { + t.Fatalf("different content at existing ID error = %v, want ErrImmutableRevision", err) + } + after := f.rowSnapshot(t, f.revisionTable(), "id = $1", first.ID) + if before != after { + t.Fatalf("refused reuse mutated the stored row:\n before %s\n after %s", before, after) + } + + sibling := first + sibling.ID = "rev_collision_sibling" + stored, err := writer.InsertRevision(f.ctx, sibling) + if err != nil { + t.Fatalf("identical frozen content (different ID) was refused: %v", err) + } + siblingArgs, siblingDigest, err := pgstate02RevisionArgs(sibling) + if err != nil { + t.Fatal(err) + } + _ = siblingArgs + var storedDigest string + if err := f.iso.QueryRow(f.ctx, + `SELECT content_digest FROM `+f.revisionTable()+` WHERE id = $1`, sibling.ID). + Scan(&storedDigest); err != nil { + f.fail("read sibling content_digest", err) + } + if storedDigest != siblingDigest { + t.Fatalf("sibling content_digest = %q, want the Go-computed %q", storedDigest, siblingDigest) + } + if _, digestOfFirst, _ := pgstate02RevisionArgs(first); digestOfFirst == siblingDigest { + t.Fatal("distinct row identities must carry distinct digests: frozenRevision covers the ID") + } + if stored.Recipe.Name != "collision" { + t.Fatalf("sibling stored with wrong recipe: %+v", stored.Recipe) + } + if n := f.revisionCount(t, revisionProject); n != 2 { + t.Fatalf("revision row count = %d, want 2", n) + } +} + +func TestPostgresRevisionLifecycleAndDoubleTransition(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + writer := f.newStore(t) + + ready := pgstate02Revision("rev_fail_ready", "fail-ready") + if _, err := writer.InsertRevision(f.ctx, ready); err != nil { + f.fail("insert ready revision", err) + } + frozenBefore := f.frozenSnapshot(t, ready.ID) + rowBefore := f.rowSnapshot(t, f.revisionTable(), "id = $1", ready.ID) + + failed, err := writer.FailRevision(f.ctx, ready.ID, pgstate02Diagnostic("COMPILE_FAILED")) + if err != nil { + f.fail("fail READY revision", err) + } + if failed.Status != explorer.RevisionFailed || failed.FailedAt == nil || + len(failed.Diagnostics) != 1 || failed.Diagnostics[0].Code != "COMPILE_FAILED" { + t.Fatalf("fail result = %+v", failed) + } + if frozenAfter := f.frozenSnapshot(t, ready.ID); frozenAfter != frozenBefore { + t.Fatalf("legal READY->FAILED transition disturbed frozen content:\n before %s\n after %s", frozenBefore, frozenAfter) + } + row := f.revisionLiteralRow(t, ready.ID) + if row.Status != "FAILED" || row.FailedAtText == "" || row.DiagnosticsKind != "array" { + t.Fatalf("literal failed row = %+v", row) + } + + blocked, err := writer.FailRevision(f.ctx, ready.ID, pgstate02Diagnostic("SECOND_FAIL")) + if !errors.Is(err, ErrStaleRevisionTransition) { + t.Fatalf("double FailRevision error = %v (result %+v), want ErrStaleRevisionTransition", err, blocked) + } + postFail := f.rowSnapshot(t, f.revisionTable(), "id = $1", ready.ID) + if postFail == rowBefore { + t.Fatal("the first FailRevision never changed the row; the double-transition witness would be vacuous") + } + if again, err := writer.FailRevision(f.ctx, ready.ID, pgstate02Diagnostic("THIRD_FAIL")); !errors.Is(err, ErrStaleRevisionTransition) || again != nil { + t.Fatalf("second double-transition attempt = %+v, %v", again, err) + } + if after := f.rowSnapshot(t, f.revisionTable(), "id = $1", ready.ID); after != postFail { + t.Fatalf("refused double transition mutated the row:\n want %s\n got %s", postFail, after) + } + + if _, err := writer.FailRevisionFrom(f.ctx, ready.ID, explorer.RevisionReady, pgstate02Diagnostic("STALE")); !errors.Is(err, ErrStaleRevisionTransition) { + t.Fatalf("stale READY token on a FAILED revision error = %v, want ErrStaleRevisionTransition", err) + } + if after := f.rowSnapshot(t, f.revisionTable(), "id = $1", ready.ID); after != postFail { + t.Fatalf("stale token write landed:\n want %s\n got %s", postFail, after) + } + if _, err := writer.FailRevisionFrom(f.ctx, ready.ID, explorer.RevisionFailed, pgstate02Diagnostic("RESURRECT")); !errors.Is(err, ErrStaleRevisionTransition) { + t.Fatalf("FAILED->FAILED token error = %v, want ErrStaleRevisionTransition", err) + } + if after := f.rowSnapshot(t, f.revisionTable(), "id = $1", ready.ID); after != postFail { + t.Fatalf("resurrect token write landed:\n want %s\n got %s", postFail, after) + } + if _, err := writer.FailRevision(f.ctx, "rev_never_created", nil); !errors.Is(err, explorer.ErrNotFound) { + t.Fatalf("missing revision fail error = %v, want ErrNotFound", err) + } + + active := pgstate02Revision("rev_fail_active", "fail-active") + active.Status = explorer.RevisionActive + activated := time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC) + active.ActivatedAt = &activated + if _, err := writer.InsertRevision(f.ctx, active); err != nil { + f.fail("insert ACTIVE revision", err) + } + failedActive, err := writer.FailRevision(f.ctx, active.ID, pgstate02Diagnostic("RUNTIME_FAILED")) + if err != nil || failedActive.Status != explorer.RevisionFailed { + t.Fatalf("fail ACTIVE revision = %+v, %v", failedActive, err) + } + + superseded := pgstate02Revision("rev_superseded", "superseded") + superseded.Status = explorer.RevisionActive + if _, err := writer.InsertRevision(f.ctx, superseded); err != nil { + f.fail("insert revision to supersede", err) + } + if _, err := f.iso.Exec(f.ctx, + `UPDATE `+f.revisionTable()+` SET status = 'SUPERSEDED' WHERE id = $1 AND status = 'ACTIVE'`, + superseded.ID); err != nil { + f.fail("legal ACTIVE->SUPERSEDED transition", err) + } + if _, err := writer.FailRevision(f.ctx, superseded.ID, pgstate02Diagnostic("TOO_LATE")); !errors.Is(err, ErrStaleRevisionTransition) { + t.Fatalf("failing a SUPERSEDED revision error = %v, want ErrStaleRevisionTransition", err) + } +} + +func TestPostgresRevisionStaleTokenRaceTwoConnections(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + + storeA, pidA := f.newDedicatedStore(t) + rawB, pidB := f.newDedicatedPool(t) + requireDistinctBackendsPGState02(t, pidA, pidB, "stale writer (store A)", "activator (raw B)") + + revision := pgstate02Revision("rev_race_token", "race") + if _, err := storeA.InsertRevision(f.ctx, revision); err != nil { + f.fail("insert revision from backend A", err) + } + + activatedAt := time.Date(2026, 5, 2, 12, 0, 0, 0, time.UTC) + tag, err := rawB.Exec(f.ctx, + `UPDATE `+f.revisionTable()+` + SET status = 'ACTIVE', activated_at = $2, + publication = '{"state":"PUBLISHED","generation":"gen_pg_01"}'::jsonb + WHERE id = $1 AND status = 'READY'`, revision.ID, activatedAt) + if err != nil { + f.fail("activation on backend B", err) + } + if tag.RowsAffected() != 1 { + t.Fatalf("activation affected %d rows, want 1", tag.RowsAffected()) + } + activatedWitness := f.rowSnapshot(t, f.revisionTable(), "id = $1", revision.ID) + + if _, err := storeA.FailRevisionFrom(f.ctx, revision.ID, explorer.RevisionReady, + pgstate02Diagnostic("STALE_AFTER_ACTIVATION")); !errors.Is(err, ErrStaleRevisionTransition) { + t.Fatalf("stale-token FailRevisionFrom after a cross-backend activation error = %v, want ErrStaleRevisionTransition", err) + } + if after := f.rowSnapshot(t, f.revisionTable(), "id = $1", revision.ID); after != activatedWitness { + t.Fatalf("the stale writer on backend A mutated the row the activator committed:\n want %s\n got %s", activatedWitness, after) + } + row := f.revisionLiteralRow(t, revision.ID) + if row.Status != "ACTIVE" || row.ActivatedAtText != "2026-05-02 12:00:00.000000" || row.FailedAtText != "" { + t.Fatalf("literal row after refused stale write = %+v", row) + } + + if failed, err := storeA.FailRevision(f.ctx, revision.ID, pgstate02Diagnostic("POST_RACE")); err != nil || failed.Status != explorer.RevisionFailed { + t.Fatalf("post-race legal fail = %+v, %v", failed, err) + } +} + +func TestPostgresRevisionConcurrentFailWritersTwoConnections(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + + storeA, pidA := f.newDedicatedStore(t) + storeB, pidB := f.newDedicatedStore(t) + requireDistinctBackendsPGState02(t, pidA, pidB, "failer A", "failer B") + + revision := pgstate02Revision("rev_race_fail", "race-fail") + if _, err := storeA.InsertRevision(f.ctx, revision); err != nil { + f.fail("insert revision", err) + } + + start := make(chan struct{}) + type failResult struct { + backend string + status explorer.RevisionStatus + err error + } + var writers sync.WaitGroup + results := make(chan failResult, 2) + for i := 0; i < 2; i++ { + writers.Add(1) + store, backend := storeA, "A" + code := "FAILER_A" + if i == 1 { + store, backend, code = storeB, "B", "FAILER_B" + } + go func() { + defer writers.Done() + <-start + _, err := store.FailRevision(f.ctx, revision.ID, pgstate02Diagnostic(code)) + status := explorer.RevisionStatus("") + if err == nil { + status = explorer.RevisionFailed + } + results <- failResult{backend: backend, status: status, err: err} + }() + } + close(start) + writers.Wait() + close(results) + + wins, refusals := 0, 0 + for result := range results { + if result.err == nil { + wins++ + continue + } + if errors.Is(result.err, ErrStaleRevisionTransition) { + refusals++ + continue + } + t.Fatalf("unexpected concurrent fail outcome on backend %s: %v", result.backend, result.err) + } + if wins != 1 || refusals != 1 { + t.Fatalf("concurrent fail results: %d winners, %d stale refusals, want 1 and 1", wins, refusals) + } + codes := f.columnKinds(t, f.revisionTable(), "id = $1", []any{revision.ID}, "diagnostics") + if codes[0] != "array" { + t.Fatalf("diagnostics kind = %s", codes[0]) + } + var diagnosticsText string + if err := f.iso.QueryRow(f.ctx, + `SELECT diagnostics::text FROM `+f.revisionTable()+` WHERE id = $1`, revision.ID). + Scan(&diagnosticsText); err != nil { + f.fail("read committed diagnostics", err) + } + if strings.Contains(diagnosticsText, "FAILER_A") == strings.Contains(diagnosticsText, "FAILER_B") { + t.Fatalf("diagnostics are not exactly one winner's: %s", diagnosticsText) + } +} + +func TestPostgresRevisionTransitionGuardProbe(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + + type probe struct { + name string + setup explorer.Revision + set string + want bool + wantMsg string + } + at := func(r explorer.Revision, status explorer.RevisionStatus, withActivated bool) explorer.Revision { + r.Status = status + if withActivated { + stamp := time.Date(2026, 5, 3, 0, 0, 0, 0, time.UTC) + r.ActivatedAt = &stamp + } + return r + } + failedRow := func(id string) explorer.Revision { + r := at(pgstate02Revision(id, "a"), explorer.RevisionFailed, false) + failedAt := time.Date(2026, 5, 2, 0, 0, 0, 0, time.UTC) + r.FailedAt = &failedAt + r.Diagnostics = pgstate02Diagnostic("GENERATION_ACTIVATION_FAILED") + return r + } + + probes := []probe{ + {"ready->active", at(pgstate02Revision("tr_ready_active", "a"), explorer.RevisionReady, false), + `status = 'ACTIVE', activated_at = '2026-05-04T00:00:00Z'`, true, ""}, + {"ready->failed", at(pgstate02Revision("tr_ready_failed", "a"), explorer.RevisionReady, false), + `status = 'FAILED', failed_at = now()`, true, ""}, + {"active->superseded", at(pgstate02Revision("tr_active_super", "a"), explorer.RevisionActive, true), + `status = 'SUPERSEDED'`, true, ""}, + {"active->failed", at(pgstate02Revision("tr_active_failed", "a"), explorer.RevisionActive, true), + `status = 'FAILED', failed_at = now()`, true, ""}, + {"failed->ready resurrection", at(pgstate02Revision("tr_failed_ready", "a"), explorer.RevisionFailed, false), + `status = 'READY'`, false, "status transition"}, + {"failed->failed double transition", at(pgstate02Revision("tr_failed_failed", "a"), explorer.RevisionFailed, false), + `status = 'FAILED', diagnostics = '[{"code":"AGAIN","message":"again","severity":"ERROR"}]'::jsonb`, + false, "status transition"}, + {"failed->active resurrection", at(pgstate02Revision("tr_failed_active", "a"), explorer.RevisionFailed, false), + `status = 'ACTIVE'`, false, "status transition"}, + {"failed->active activation-shape revival", failedRow("tr_failed_revive"), + `status = 'ACTIVE', activated_at = '2026-05-05T00:00:00Z', failed_at = NULL, diagnostics = '[]'::jsonb`, true, ""}, + {"failed->active sloppy keeps failed_at", failedRow("tr_failed_sloppy_failedat"), + `status = 'ACTIVE', activated_at = '2026-05-05T00:00:00Z', diagnostics = '[]'::jsonb`, + false, "activation-shaped write"}, + {"failed->active missing activated_at", failedRow("tr_failed_no_activated_at"), + `status = 'ACTIVE', failed_at = NULL, diagnostics = '[]'::jsonb`, + false, "activation-shaped write"}, + {"failed->active nulling diagnostics", failedRow("tr_failed_null_diagnostics"), + `status = 'ACTIVE', activated_at = '2026-05-05T00:00:00Z', failed_at = NULL, diagnostics = NULL`, + false, "activation-shaped write"}, + {"failed->active frozen dataset smuggling", failedRow("tr_failed_frozen_smuggle"), + `status = 'ACTIVE', activated_at = '2026-05-05T00:00:00Z', failed_at = NULL, diagnostics = '[]'::jsonb, dataset = '{"generation":"gen_intruder","outputs":[]}'::jsonb`, + false, "frozen content changed"}, + {"superseded->active resurrection", at(pgstate02Revision("tr_super_active", "a"), explorer.RevisionSuperseded, true), + `status = 'ACTIVE'`, false, "status transition"}, + {"superseded->ready resurrection", at(pgstate02Revision("tr_super_ready", "a"), explorer.RevisionSuperseded, true), + `status = 'READY'`, false, "status transition"}, + {"ready->superseded skip", at(pgstate02Revision("tr_ready_super", "a"), explorer.RevisionReady, false), + `status = 'SUPERSEDED'`, false, "status transition"}, + {"active->ready un-activation", at(pgstate02Revision("tr_active_ready", "a"), explorer.RevisionActive, true), + `status = 'READY'`, false, "status transition"}, + {"ready->ready same-status mutation", at(pgstate02Revision("tr_ready_ready", "a"), explorer.RevisionReady, false), + `status = 'READY', diagnostics = '[{"code":"NOPE","message":"nope","severity":"ERROR"}]'::jsonb`, + false, "status transition"}, + } + for _, p := range probes { + f.rawInsertRevision(t, p.setup) + before := f.rowSnapshot(t, f.revisionTable(), "id = $1", p.setup.ID) + _, err := f.iso.Exec(f.ctx, + "UPDATE "+f.revisionTable()+" SET "+p.set+" WHERE id = $1", p.setup.ID) + if p.want { + if err != nil { + f.fail("legal transition "+p.name, err) + } + if after := f.rowSnapshot(t, f.revisionTable(), "id = $1", p.setup.ID); after == before { + t.Fatalf("legal transition %s changed nothing", p.name) + } + continue + } + pgErr := requirePgErrPGState02(t, err, "23514", "illegal transition "+p.name) + if !strings.Contains(pgErr.Message, p.wantMsg) { + t.Fatalf("illegal transition %s refused by something other than the state machine: %s", p.name, pgErr.Message) + } + if after := f.rowSnapshot(t, f.revisionTable(), "id = $1", p.setup.ID); after != before { + t.Fatalf("refused illegal transition %s mutated the row:\n want %s\n got %s", p.name, before, after) + } + } + + row := f.revisionLiteralRow(t, "tr_ready_active") + if row.Status != "ACTIVE" || row.ActivatedAtText != "2026-05-04 00:00:00.000000" { + t.Fatalf("literal activated_at witness = %+v", row) + } + + noop := pgstate02Revision("tr_noop", "a") + f.rawInsertRevision(t, noop) + noopBefore := f.rowSnapshot(t, f.revisionTable(), "id = $1", noop.ID) + tag, err := f.iso.Exec(f.ctx, + `UPDATE `+f.revisionTable()+` SET status = 'READY' WHERE id = $1 AND status = 'READY'`, noop.ID) + if err != nil { + f.fail("no-op UPDATE", err) + } + if tag.RowsAffected() != 1 { + t.Fatalf("no-op UPDATE affected %d rows, want 1 (the guard allows identical rows)", tag.RowsAffected()) + } + if after := f.rowSnapshot(t, f.revisionTable(), "id = $1", noop.ID); after != noopBefore { + t.Fatalf("no-op UPDATE changed the row:\n want %s\n got %s", noopBefore, after) + } +} + +func TestPostgresRevisionFrozenGuardProvenNonVacuous(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + + validDigest := "sha256:" + strings.Repeat("a", 64) + revision := pgstate02Revision("rev_frozen_probe", "frozen") + f.rawInsertRevision(t, revision) + + mutations := []struct { + column string + set string + }{ + {"id", "'rev_intruder'"}, + {"project", "'proj_intruder'"}, + {"explorer_id", "'exp_intruder'"}, + {"config", `'{"mutated": true}'::jsonb`}, + {"authoring_bundle", `'{"mutated": true}'::jsonb`}, + {"public_output_contract", `'{"mutated": true}'::jsonb`}, + {"intent_digest", "'sha256:intruder'"}, + {"compilation_receipt_id", "'receipt_intruder'"}, + {"recipe", `'{"name":"intruder"}'::jsonb`}, + {"recipe_digest", "'sha256:intruder'"}, + {"resolved_schema_digest", "'sha256:intruder'"}, + {"source_generation", "'gen_intruder'"}, + {"materializations", `'[]'::jsonb`}, + {"emitted_columns", `'[]'::jsonb`}, + {"dataset", `'{"generation":"gen_intruder","outputs":[]}'::jsonb`}, + {"quality_reports", `'[{"id":"q_intruder"}]'::jsonb`}, + {"content_digest", "'" + validDigest + "'"}, + {"created_by", "'intruder@calypr'"}, + {"created_at", "'2030-01-01T00:00:00Z'"}, + } + for _, m := range mutations { + _, err := f.iso.Exec(f.ctx, + "UPDATE "+f.revisionTable()+" SET "+m.column+" = "+m.set+" WHERE id = $1", revision.ID) + pgErr := requirePgErrPGState02(t, err, "23514", "frozen revision column "+m.column) + if !strings.Contains(pgErr.Message, "frozen content changed") { + t.Fatalf("frozen refusal on %s came from something other than the guard: %s", m.column, pgErr.Message) + } + } + _, err := f.iso.Exec(f.ctx, + "UPDATE "+f.revisionTable()+" SET status = 'FAILED', recipe_digest = 'sha256:smuggled' WHERE id = $1", + revision.ID) + pgErr := requirePgErrPGState02(t, err, "23514", "smuggled frozen mutation during legal transition") + if !strings.Contains(pgErr.Message, "frozen content changed") { + t.Fatalf("smuggled mutation refused by the wrong guard: %s", pgErr.Message) + } + + if _, err := f.iso.Exec(f.ctx, + "DELETE FROM "+f.revisionTable()+" WHERE id = $1", revision.ID); err == nil { + t.Fatal("DELETE of a revision accepted while the guard is live") + } else { + pgErr := requirePgErrPGState02(t, err, "23514", "live insert-once DELETE guard") + if !strings.Contains(pgErr.Message, "insert-once") { + t.Fatalf("DELETE refusal did not come from the insert-once branch: %s", pgErr.Message) + } + } + + if _, err := f.iso.Exec(f.ctx, + `DROP TRIGGER explorer_revisions_guard ON `+f.revisionTable()); err != nil { + f.fail("drop revision guard", err) + } + if _, err := f.iso.Exec(f.ctx, + "UPDATE "+f.revisionTable()+" SET recipe_digest = 'sha256:unfrozen' WHERE id = $1", revision.ID); err != nil { + f.fail("frozen mutation without the trigger must succeed", err) + } + var mutated string + if err := f.iso.QueryRow(f.ctx, + `SELECT recipe_digest FROM `+f.revisionTable()+` WHERE id = $1`, revision.ID).Scan(&mutated); err != nil { + f.fail("read mutated recipe_digest", err) + } + if mutated != "sha256:unfrozen" { + t.Fatalf("guard-removal probe did not persist: recipe_digest = %q", mutated) + } + resurrect := pgstate02Revision("rev_frozen_resurrect", "frozen") + resurrect.Status = explorer.RevisionFailed + failedAt := time.Date(2026, 5, 5, 0, 0, 0, 0, time.UTC) + resurrect.FailedAt = &failedAt + f.rawInsertRevision(t, resurrect) + if _, err := f.iso.Exec(f.ctx, + "UPDATE "+f.revisionTable()+" SET status = 'READY' WHERE id = $1", resurrect.ID); err != nil { + f.fail("resurrection without the trigger must succeed", err) + } + if row := f.revisionLiteralRow(t, resurrect.ID); row.Status != "READY" { + t.Fatalf("guard-removal resurrection did not persist: status = %q", row.Status) + } + if _, err := f.iso.Exec(f.ctx, + "DELETE FROM "+f.revisionTable()+" WHERE id = $1", resurrect.ID); err != nil { + f.fail("DELETE without the trigger must succeed", err) + } + var remaining int + if err := f.iso.QueryRow(f.ctx, + "SELECT count(*) FROM "+f.revisionTable()+" WHERE id = $1", resurrect.ID).Scan(&remaining); err != nil { + f.fail("read post-delete row count", err) + } + if remaining != 0 { + t.Fatalf("guard-removal DELETE did not persist: %d rows remain", remaining) + } + + f.rerunMigration(t, 6) + f.rerunMigration(t, 12) + if _, err := f.iso.Exec(f.ctx, + "UPDATE "+f.revisionTable()+" SET recipe_digest = 'sha256:refrozen' WHERE id = $1", revision.ID); err == nil { + t.Fatal("frozen mutation accepted after the guard was restored from the migration text") + } else { + pgErr := requirePgErrPGState02(t, err, "23514", "restored frozen guard") + if !strings.Contains(pgErr.Message, "frozen content changed") { + t.Fatalf("restored guard refusal came from something else: %s", pgErr.Message) + } + } + restoredProbe := pgstate02Revision("rev_frozen_restored", "frozen") + restoredProbe.Status = explorer.RevisionFailed + f.rawInsertRevision(t, restoredProbe) + if _, err := f.iso.Exec(f.ctx, + "UPDATE "+f.revisionTable()+" SET status = 'READY' WHERE id = $1", restoredProbe.ID); err == nil { + t.Fatal("resurrection accepted after the guard was restored") + } else { + pgErr := requirePgErrPGState02(t, err, "23514", "restored state machine") + if !strings.Contains(pgErr.Message, "status transition") { + t.Fatalf("restored state-machine refusal came from something else: %s", pgErr.Message) + } + } + if _, err := f.iso.Exec(f.ctx, + "DELETE FROM "+f.revisionTable()+" WHERE id = $1", restoredProbe.ID); err == nil { + t.Fatal("DELETE accepted after the guard was restored") + } else { + pgErr := requirePgErrPGState02(t, err, "23514", "restored insert-once DELETE guard") + if !strings.Contains(pgErr.Message, "insert-once") { + t.Fatalf("restored DELETE refusal did not come from the insert-once branch: %s", pgErr.Message) + } + } + + revivalProbe := pgstate02Revision("rev_frozen_revival", "frozen") + revivalProbe.Status = explorer.RevisionFailed + revivalFailedAt := time.Date(2026, 5, 6, 0, 0, 0, 0, time.UTC) + revivalProbe.FailedAt = &revivalFailedAt + revivalProbe.Diagnostics = pgstate02Diagnostic("GENERATION_ACTIVATION_FAILED") + f.rawInsertRevision(t, revivalProbe) + if _, err := f.iso.Exec(f.ctx, + "UPDATE "+f.revisionTable()+" SET status = 'ACTIVE' WHERE id = $1", revivalProbe.ID); err == nil { + t.Fatal("bare FAILED->ACTIVE resurrection accepted after the guard was restored") + } else { + pgErr := requirePgErrPGState02(t, err, "23514", "restored activation-shape refusal") + if !strings.Contains(pgErr.Message, "activation-shaped write") { + t.Fatalf("restored revival refusal came from something other than the 000012 arm: %s", pgErr.Message) + } + } + if _, err := f.iso.Exec(f.ctx, + "UPDATE "+f.revisionTable()+ + " SET status = 'ACTIVE', activated_at = '2026-05-07T00:00:00Z', failed_at = NULL, diagnostics = '[]'::jsonb WHERE id = $1", + revivalProbe.ID); err != nil { + f.fail("activation-shape revival after the guard was restored", err) + } + if row := f.revisionLiteralRow(t, revivalProbe.ID); row.Status != "ACTIVE" || + row.FailedAtText != "" || row.ActivatedAtText != "2026-05-07 00:00:00.000000" { + t.Fatalf("restored-law revival witness = %+v, want ACTIVE with failed_at cleared and activated_at set", row) + } +} + +func TestPostgresRevisionSchemaConstraints(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + + rawInsert := "INSERT INTO " + f.revisionTable() + " (" + + strings.Join(revisionColumns, ", ") + ") VALUES (" + + orderedPlaceholders(len(revisionColumns)) + ")" + + variants := []struct { + name string + sqlstate string + mutate func([]any) []any + }{ + {"unknown status", "23514", func(a []any) []any { a[17] = "SOMETHING_ELSE"; return a }}, + {"digest shape", "23514", func(a []any) []any { a[16] = "not-a-digest"; return a }}, + {"recipe scalar", "23514", func(a []any) []any { a[8] = []byte("5"); return a }}, + {"materializations map", "23514", func(a []any) []any { a[12] = []byte("{}"); return a }}, + {"quality scalar", "23514", func(a []any) []any { a[15] = []byte("42"); return a }}, + {"diagnostics object", "23514", func(a []any) []any { a[18] = []byte("{}"); return a }}, + {"dataset not null", "23502", func(a []any) []any { a[14] = nil; return a }}, + {"publication not null", "23502", func(a []any) []any { a[19] = nil; return a }}, + } + for _, variant := range variants { + args, _, err := pgstate02RevisionArgs(pgstate02Revision("rev_constraint_"+strings.ReplaceAll(variant.name, " ", "_"), "c")) + if err != nil { + t.Fatal(err) + } + _, err = f.iso.Exec(f.ctx, rawInsert, variant.mutate(args)...) + requirePgErrPGState02(t, err, variant.sqlstate, "constraint variant: "+variant.name) + } + + args, _, err := pgstate02RevisionArgs(pgstate02Revision("rev_constraint_notjson", "c")) + if err != nil { + t.Fatal(err) + } + args[3] = "this is not json" + _, err = f.iso.Exec(f.ctx, rawInsert, args...) + requirePgErrPGState02(t, err, "22P02", "non-JSON config payload") + + if n := f.revisionCount(t, revisionProject); n != 0 { + t.Fatalf("rejected raw inserts left %d rows, want 0", n) + } +} + +func TestPostgresRevisionJSONTriStateFreshRead(t *testing.T) { + f := newPGState02Fixture(t) + f.migrate(t) + writer := f.newStore(t) + + missing := pgstate02Revision("rev_tri_missing", "tri") + missing.Config = nil + missing.AuthoringBundle = nil + missing.PublicOutputContract = nil + missing.Materializations = nil + missing.EmittedColumns = nil + + jsonNull := pgstate02Revision("rev_tri_null", "tri") + jsonNull.Config = json.RawMessage("null") + jsonNull.AuthoringBundle = json.RawMessage("null") + jsonNull.PublicOutputContract = json.RawMessage("null") + + empty := pgstate02Revision("rev_tri_empty", "tri") + empty.Materializations = []explorer.Materialization{} + empty.EmittedColumns = []explorer.EmittedColumn{} + empty.QualityReports = []publication.QualityReport{} + empty.Diagnostics = []explorer.Diagnostic{} + + for _, revision := range []explorer.Revision{missing, jsonNull, empty} { + if _, err := writer.InsertRevision(f.ctx, revision); err != nil { + f.fail("insert tri-state revision "+revision.ID, err) + } + } + + for _, tc := range []struct { + id string + config string + mats string + diag string + }{ + {"rev_tri_missing", "missing", "missing", "missing"}, + {"rev_tri_null", "null", "array", "missing"}, + {"rev_tri_empty", "object", "array", "array"}, + } { + kinds := f.columnKinds(t, f.revisionTable(), "id = $1", []any{tc.id}, + "config", "materializations", "diagnostics") + if kinds[0] != tc.config || kinds[1] != tc.mats || kinds[2] != tc.diag { + t.Fatalf("%s stored kinds = %v, want [%s %s %s]", tc.id, kinds, tc.config, tc.mats, tc.diag) + } + } + var emptyMatsText, emptyDiagText string + if err := f.iso.QueryRow(f.ctx, + `SELECT materializations::text, diagnostics::text FROM `+f.revisionTable()+` WHERE id = 'rev_tri_empty'`). + Scan(&emptyMatsText, &emptyDiagText); err != nil { + f.fail("read empty-array literals", err) + } + if emptyMatsText != "[]" || emptyDiagText != "[]" { + t.Fatalf("empty slices stored as %q/%q, want [] / []", emptyMatsText, emptyDiagText) + } + + reader := f.newStore(t) + gotMissing, err := reader.GetRevision(f.ctx, "rev_tri_missing") + if err != nil { + f.fail("read missing-payload revision", err) + } + if gotMissing.Config != nil || gotMissing.AuthoringBundle != nil || gotMissing.PublicOutputContract != nil { + t.Fatalf("missing payloads revived: config=%q authoring=%q contract=%q", + gotMissing.Config, gotMissing.AuthoringBundle, gotMissing.PublicOutputContract) + } + if gotMissing.Materializations != nil || gotMissing.EmittedColumns != nil { + t.Fatalf("missing slices revived: %+v / %+v", gotMissing.Materializations, gotMissing.EmittedColumns) + } + gotNull, err := reader.GetRevision(f.ctx, "rev_tri_null") + if err != nil { + f.fail("read json-null revision", err) + } + if string(gotNull.Config) != "null" || string(gotNull.AuthoringBundle) != "null" || + string(gotNull.PublicOutputContract) != "null" { + t.Fatalf("JSON-null payloads lost: config=%q authoring=%q contract=%q", + gotNull.Config, gotNull.AuthoringBundle, gotNull.PublicOutputContract) + } + gotEmpty, err := reader.GetRevision(f.ctx, "rev_tri_empty") + if err != nil { + f.fail("read empty-payload revision", err) + } + if gotEmpty.Materializations == nil || len(gotEmpty.Materializations) != 0 || + gotEmpty.EmittedColumns == nil || len(gotEmpty.EmittedColumns) != 0 || + gotEmpty.Diagnostics == nil || len(gotEmpty.Diagnostics) != 0 || + gotEmpty.QualityReports == nil || len(gotEmpty.QualityReports) != 0 { + t.Fatalf("empty slices collapsed to nil: %+v", gotEmpty) + } +} + +var pgstate02EmbeddedMigrationPins = map[string]string{ + "000001_ledger.sql": "46a99180c6c6c7d41ca13520e020b71827725ec8059cce0142e278fb7577f086", + "000002_explorer_owner.sql": "fc85316901b79234695695f972d8abc17d29a726293b0ff6584c96aed1cc7056", + "000003_explorer_draft_revision.sql": "4bf4fe35485cb7adc6ea46812b0ae004e89ff935a14f14dd80a300bb3cc86c3f", + "000004_capability_artifacts.sql": "bec3e49f897b28ef71d7d69a5f7c71bb11d13b7c631ee26724de36a89d639222", + "000005_explorer_compilation_receipt.sql": "f32b3f9c821b5451e64b3d2972fb49e61f4e0c5bfcf88c69bc5b6aa7298fc33a", + "000006_explorer_revision.sql": "3f6c44bff72d7837634e8c170f0b97e256e38d35f7c844fad27f4fbda38ecc09", + "000007_interpretation_library.sql": "5e308b21a3f567b3066c46458606dd361a9c90646db54ee71618902a544a9ba2", + "000008_selection_revisions.sql": "562a61ef33786cf674f1c87552e2dc8e70691b34116b7f415d5f81960ed8d65d", + "000009_explorer_explicit_groups.sql": "c635cb5ef7c3c04655b4f45b01ba0fe834a7f181f70638f300878caf16c9a14f", + "000010_explorer_explicit_group_memberships.sql": "02e0c9ab8bf9864e7d9c82471eacc7406fe2301b025e459784044c4a44489ccd", + "000011_explorer_explicit_group_definition_guard_hardening.sql": "fb45d8f8c312f8f6adb119465da5ed89ac0ddd14929a2c26aa4c917db6d704ec", + "000012_explorer_revision_activation_revival.sql": "08a61bf6ca66a400112f51463f2c5e1c0a7f4cfbb2d707c0063f87829b1acfe9", +} + +func pgstate02MigrationFileVersion(fileName string) (int64, error) { + versionText, _, found := strings.Cut(fileName, "_") + if !found { + return 0, fmt.Errorf("embedded migration %s does not match _.sql", fileName) + } + version, err := strconv.ParseInt(versionText, 10, 64) + if err != nil || version < 1 { + return 0, fmt.Errorf("embedded migration %s has an unparsable version prefix", fileName) + } + return version, nil +} + +func pgstate02EnumerateEmbeddedMigrations(t *testing.T) ([]string, int64) { + t.Helper() + var files []string + var maxVersion int64 + err := fs.WalkDir(migrationsFS, "migrations", func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + version, err := pgstate02MigrationFileVersion(d.Name()) + if err != nil { + return err + } + files = append(files, d.Name()) + if version > maxVersion { + maxVersion = version + } + return nil + }) + if err != nil { + t.Fatalf("enumerate embedded migration directory: %v", err) + } + if len(files) == 0 { + t.Fatal("embedded migration directory holds no migrations") + } + sort.Strings(files) + return files, maxVersion +} + +func pgstate02VerifyEmbeddedChecksumPins(t *testing.T, files []string, history []pgstore.Migration) { + t.Helper() + byVersion := make(map[int64]pgstore.Migration, len(history)) + for _, m := range history { + byVersion[m.Version] = m + } + seen := make(map[string]bool, len(files)) + for _, file := range files { + version, err := pgstate02MigrationFileVersion(file) + if err != nil { + t.Fatal(err) + } + pin, pinned := pgstate02EmbeddedMigrationPins[file] + if !pinned { + t.Fatalf("embedded migration %s has no explicit checksum pin: add its sha256 to pgstate02EmbeddedMigrationPins deliberately -- a new migration must be pinned, never silently trusted", file) + } + m, loaded := byVersion[version] + if !loaded { + t.Fatalf("embedded migration file %s (version %d) is absent from OwnerMigrations(): the loader and the directory disagree", file, version) + } + if m.Checksum != pin { + t.Fatalf("embedded migration %s checksum = %s, want the pinned %s: migration SQL drifted", file, m.Checksum, pin) + } + seen[file] = true + } + for file := range pgstate02EmbeddedMigrationPins { + if !seen[file] { + t.Fatalf("checksum pin for %s has no embedded migration file: deleting a migration must drop its pin deliberately", file) + } + } +} + +func TestPostgresExplorerStateMigrationsRerunConverges(t *testing.T) { + embeddedFiles, embeddedMaxVersion := pgstate02EnumerateEmbeddedMigrations(t) + ownerHistory, err := OwnerMigrations() + if err != nil { + t.Fatal(err) + } + if len(ownerHistory) != len(embeddedFiles) { + t.Fatalf("OwnerMigrations() loaded %d migrations but the embedded directory holds %d files %v", len(ownerHistory), len(embeddedFiles), embeddedFiles) + } + pgstate02VerifyEmbeddedChecksumPins(t, embeddedFiles, ownerHistory) + + wantHistory := append([]pgstore.Migration(nil), ownerHistory...) + sort.Slice(wantHistory, func(i, j int) bool { return wantHistory[i].Version < wantHistory[j].Version }) + wantLatest := embeddedMaxVersion + wantCount := len(wantHistory) + + f := newPGState02Fixture(t) + first := f.migrate(t) + if len(first.Applied) != wantCount || first.LatestVersion != wantLatest { + t.Fatalf("first migration result = %+v, want %d ledger rows ending at version %d (self-derived: %d embedded files %v)", + first, wantCount, wantLatest, len(embeddedFiles), embeddedFiles) + } + + writer := f.newStore(t) + owner := fullOwner("proj_state", "exp_state") + if _, err := writer.Create(f.ctx, owner); err != nil { + f.fail("create owner", err) + } + receipt := pgstate02Receipt(t, "intent-rerun", receiptProject, receiptExplorer) + if _, err := writer.InsertCompilationReceipt(f.ctx, receipt); err != nil { + f.fail("insert receipt", err) + } + revision := pgstate02Revision("rev_rerun", "rerun") + if _, err := writer.InsertRevision(f.ctx, revision); err != nil { + f.fail("insert revision", err) + } + + second := f.migrate(t) + if len(second.Applied) != wantCount || second.LatestVersion != wantLatest { + t.Fatalf("rerun result = %+v, want the identical %d-row ledger ending at version %d (all recorded, nothing re-run)", second, wantCount, wantLatest) + } + for i := range first.Applied { + if first.Applied[i] != second.Applied[i] { + t.Fatalf("rerun disturbed applied[%d]: first = %+v, rerun = %+v (ledger rows must be stable and applied_at unchanged)", i, first.Applied[i], second.Applied[i]) + } + } + + rows, err := f.iso.Query(f.ctx, `SELECT version, name, checksum FROM `+ + sanitizeTableRef(f.schema, "schema_migrations")+` ORDER BY version`) + if err != nil { + f.fail("read ledger", err) + } + type ledgerRow struct { + version int64 + name string + checksum string + } + var ledger []ledgerRow + for rows.Next() { + var rec ledgerRow + if err := rows.Scan(&rec.version, &rec.name, &rec.checksum); err != nil { + rows.Close() + f.fail("scan ledger", err) + } + ledger = append(ledger, rec) + } + rows.Close() + if err := rows.Err(); err != nil { + f.fail("read ledger", err) + } + if len(ledger) != wantCount { + t.Fatalf("ledger holds %d rows, want %d", len(ledger), wantCount) + } + for i, want := range wantHistory { + if ledger[i] != (ledgerRow{want.Version, want.Name, want.Checksum}) { + t.Fatalf("ledger[%d] = %+v, want %+v", i, ledger[i], want) + } + } + + var witness []string + for _, rec := range ledger { + witness = append(witness, fmt.Sprintf("%d/%s", rec.version, rec.name)) + } + t.Logf("double-apply ledger witness: [%s]", strings.Join(witness, " ")) + + reader := f.newStore(t) + if _, err := reader.Get(f.ctx, "proj_state", "exp_state"); err != nil { + f.fail("owner after migrate reruns", err) + } + if _, err := reader.GetCompilationReceiptForExplorer(f.ctx, receiptProject, receiptExplorer, receipt.ID); err != nil { + f.fail("receipt after migrate reruns", err) + } + if _, err := reader.GetRevision(f.ctx, revision.ID); err != nil { + f.fail("revision after migrate reruns", err) + } +} + +func TestZZNoLeftoverPGState02Resources(t *testing.T) { + raw := pgstate02TestPostgresURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %v", err) + } + defer admin.Close() + + var dbLeftovers []string + rows, err := admin.Query(ctx, `SELECT datname FROM pg_database WHERE datname LIKE 'pgstate02\_it\_%' ESCAPE '\' ORDER BY datname`) + if err != nil { + t.Fatalf("audit isolated databases: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("audit isolated databases: %v", err) + } + dbLeftovers = append(dbLeftovers, name) + } + rows.Close() + if err := rows.Err(); err != nil { + t.Fatalf("audit isolated databases: %v", err) + } + if len(dbLeftovers) != 0 { + t.Errorf("leftover isolated databases owned by this ticket: %v", dbLeftovers) + } + + var schemaLeftovers []string + rows, err = admin.Query(ctx, `SELECT nspname FROM pg_namespace WHERE nspname LIKE 'pgstate02\_it\_%' ESCAPE '\' ORDER BY nspname`) + if err != nil { + t.Fatalf("audit owned schemas: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("audit owned schemas: %v", err) + } + schemaLeftovers = append(schemaLeftovers, name) + } + rows.Close() + if err := rows.Err(); err != nil { + t.Fatalf("audit owned schemas: %v", err) + } + if len(schemaLeftovers) != 0 { + t.Errorf("leftover owned schemas owned by this ticket: %v", schemaLeftovers) + } +} diff --git a/internal/backends/postgres/internal/explorer/selection_members.go b/internal/backends/postgres/internal/explorer/selection_members.go new file mode 100644 index 000000000..356f20168 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/selection_members.go @@ -0,0 +1,415 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/explorer" +) + +const ( + selectionDigestPageSize = 1000 + selectionMembersRefConstraint = "selection_members_ref_ux" +) + +func selectionMemberKey(selectionID string, ref explorer.ResourceRef) string { + h := sha256.New() + for _, part := range []string{ref.Project, ref.Generation, ref.ResourceType, ref.ID} { + _, _ = h.Write([]byte{0}) + _, _ = h.Write([]byte(part)) + } + return selectionID + "_" + hex.EncodeToString(h.Sum(nil)) +} + +var memberColumns = []string{ + "member_key", + "project", + "generation", + "resource_type", + "id", + "ordinal", +} + +type selectionMemberSQL struct { + memberInsert string + memberByKey string + memberDigestPage string + memberVisitPage string + leaseRenew string + completeCAS string +} + +func newSelectionMemberSQL(schema string) selectionMemberSQL { + members := pgx.Identifier{schema, SelectionMembersTable}.Sanitize() + headers := pgx.Identifier{schema, SelectionRevisionsTable}.Sanitize() + readCols := strings.Join(selectionReadColumns, ", ") + return selectionMemberSQL{ + memberInsert: "INSERT INTO " + members + + " (selection_id, " + strings.Join(memberColumns, ", ") + ")" + + " VALUES (" + orderedPlaceholders(len(memberColumns)+1) + ")" + + " ON CONFLICT (selection_id, member_key) DO NOTHING" + + " RETURNING " + strings.Join(memberColumns, ", "), + memberByKey: "SELECT " + strings.Join(memberColumns, ", ") + " FROM " + members + + " WHERE selection_id = $1 AND member_key = $2", + memberDigestPage: "SELECT project, generation, resource_type, id FROM " + members + + " WHERE selection_id = $1 AND project = $2 AND generation = $3 AND resource_type = $4 AND id > $5" + + " ORDER BY id LIMIT $6", + memberVisitPage: "SELECT " + strings.Join(memberColumns, ", ") + " FROM " + members + + " WHERE selection_id = $1 AND project = $2 AND member_key > $3" + + " ORDER BY member_key LIMIT $4", + leaseRenew: "UPDATE " + headers + " SET writer_expires_at = $2" + + " WHERE id = $1 AND writer_token = $3 AND state = '" + selectionStagingState + "'", + completeCAS: "UPDATE " + headers + " SET state = '" + selectionCompleteState + "', complete = true," + + " membership_digest = $2, member_count = $3, member_bytes = $4, completed_at = $5" + + " WHERE id = $1 AND state = '" + selectionStagingState + "' AND writer_token = $6" + + " RETURNING " + readCols, + } +} + +func (s *Store) streamMemberFacts(ctx context.Context, q selectionQuerier, project, generation, resourceType, selectionID string) (string, int64, int64, error) { + hash := sha256.New() + var count, byteTotal int64 + afterID := "" + for { + pageCount := 0 + rows, err := q.Query(ctx, s.selectionMembers.memberDigestPage, + selectionID, project, generation, resourceType, afterID, selectionDigestPageSize) + if err != nil { + return "", 0, 0, fmt.Errorf("postgres explorer store: stream selection members %q: %w", selectionID, err) + } + for rows.Next() { + var ref explorer.ResourceRef + if err := rows.Scan(&ref.Project, &ref.Generation, &ref.ResourceType, &ref.ID); err != nil { + rows.Close() + return "", 0, 0, fmt.Errorf("postgres explorer store: scan selection member %q: %w", selectionID, err) + } + ref = ref.Canonical() + explorer.WriteMembershipFrame(hash, ref) + for _, value := range []string{ref.Project, ref.Generation, ref.ResourceType, ref.ID} { + byteTotal += int64(8 + len(value)) + } + count++ + pageCount++ + afterID = ref.ID + } + if err := rows.Err(); err != nil { + rows.Close() + return "", 0, 0, fmt.Errorf("postgres explorer store: iterate selection members %q: %w", selectionID, err) + } + rows.Close() + if s.memberPageStreamed != nil { + s.memberPageStreamed(pageCount) + } + if pageCount < selectionDigestPageSize { + break + } + } + return hex.EncodeToString(hash.Sum(nil)), count, byteTotal, nil +} + +func isMemberRefConflict(err error) bool { + return pgstore.IsUniqueConstraintViolation(err, selectionMembersRefConstraint) +} + +func (s *Store) AppendSelectionMembers(ctx context.Context, selectionID, writerToken string, members []explorer.SelectionMember) ([]explorer.SelectionMember, error) { + selectionID = strings.TrimSpace(selectionID) + writerToken = strings.TrimSpace(writerToken) + if selectionID == "" { + return nil, fmt.Errorf("selection id is required") + } + if writerToken == "" { + return nil, explorer.ErrSelectionConflict + } + if len(members) == 0 { + return nil, nil + } + + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire append-members connection: %w", err) + } + defer conn.Release() + + tx, err := conn.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: begin append members %q: %w", selectionID, err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var storedToken, state string + var complete bool + var project, generation, resourceType string + err = tx.QueryRow(ctx, s.selections.headerForAppend, selectionID). + Scan(&storedToken, &state, &complete, &project, &generation, &resourceType) + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrSelectionNotFound + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: lock selection header %q: %w", selectionID, err) + } + if strings.TrimSpace(storedToken) != writerToken { + return nil, explorer.ErrSelectionConflict + } + if complete || (state != "" && state != selectionStagingState) { + return nil, explorer.ErrSelectionConflict + } + + type pendingMember struct { + memberKey string + member explorer.SelectionMember + } + batch := make([]pendingMember, 0, len(members)) + for _, raw := range members { + member := raw.Canonical() + expectedMemberKey := selectionMemberKey(selectionID, member.Ref) + if member.MemberKey != "" && member.MemberKey != expectedMemberKey { + return nil, explorer.ErrSelectionConflict + } + member.MemberKey = expectedMemberKey + if err := member.Ref.Validate(project, generation, resourceType); err != nil { + return nil, err + } + batch = append(batch, pendingMember{memberKey: expectedMemberKey, member: member}) + } + + inserted := make([]explorer.SelectionMember, 0, len(batch)) + for _, entry := range batch { + var ( + memberKey, mProject, mGeneration, mResourceType, mID string + ordinal int64 + ) + err := tx.QueryRow(ctx, s.selectionMembers.memberInsert, + selectionID, entry.memberKey, entry.member.Ref.Project, entry.member.Ref.Generation, + entry.member.Ref.ResourceType, entry.member.Ref.ID, entry.member.Ordinal). + Scan(&memberKey, &mProject, &mGeneration, &mResourceType, &mID, &ordinal) + if errors.Is(err, pgx.ErrNoRows) { + var skippedKey, sProject, sGeneration, sResourceType, sID string + var sOrdinal int64 + scanErr := tx.QueryRow(ctx, s.selectionMembers.memberByKey, selectionID, entry.memberKey). + Scan(&skippedKey, &sProject, &sGeneration, &sResourceType, &sID, &sOrdinal) + if errors.Is(scanErr, pgx.ErrNoRows) { + continue + } + if scanErr != nil { + return nil, fmt.Errorf("postgres explorer store: reread staged member %q of selection %q: %w", + entry.memberKey, selectionID, scanErr) + } + if skippedKey != entry.memberKey || + sProject != entry.member.Ref.Project || sGeneration != entry.member.Ref.Generation || + sResourceType != entry.member.Ref.ResourceType || sID != entry.member.Ref.ID { + return nil, explorer.ErrSelectionConflict + } + continue + } + if err != nil { + if isMemberRefConflict(err) { + return nil, explorer.ErrSelectionConflict + } + return nil, fmt.Errorf("postgres explorer store: stage member %q of selection %q: %w", + entry.memberKey, selectionID, err) + } + if memberKey != entry.memberKey || mProject != entry.member.Ref.Project || + mGeneration != entry.member.Ref.Generation || mResourceType != entry.member.Ref.ResourceType || + mID != entry.member.Ref.ID || ordinal != entry.member.Ordinal { + return nil, fmt.Errorf("postgres explorer store: staged member %q of selection %q reread mismatch", + entry.memberKey, selectionID) + } + inserted = append(inserted, entry.member) + } + + if s.afterMemberInsert != nil { + if err := s.afterMemberInsert(ctx, tx); err != nil { + return nil, fmt.Errorf("postgres explorer store: append members %q after member insert: %w", + selectionID, err) + } + } + + renewAt := time.Now().UTC().Add(selectionWriterLease) + tag, err := tx.Exec(ctx, s.selectionMembers.leaseRenew, selectionID, renewAt, writerToken) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: renew selection writer lease %q: %w", selectionID, err) + } + if tag.RowsAffected() != 1 { + return nil, explorer.ErrSelectionConflict + } + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("postgres explorer store: commit append members %q: %w", selectionID, err) + } + return inserted, nil +} + +func (s *Store) DigestSelectionMembers(ctx context.Context, project, selectionID string) (string, int64, int64, error) { + selectionID = strings.TrimSpace(selectionID) + project = strings.TrimSpace(project) + if selectionID == "" || project == "" { + return "", 0, 0, explorer.ErrSelectionNotFound + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return "", 0, 0, fmt.Errorf("postgres explorer store: acquire digest-members connection: %w", err) + } + defer conn.Release() + + header, err := s.readSelectionRow(ctx, conn, s.selections.headerScoped, selectionID, project) + if err != nil { + return "", 0, 0, err + } + if header.revision.Complete { + return header.revision.MembershipDigest, header.revision.MemberCount, header.revision.MemberBytes, nil + } + return s.streamMemberFacts(ctx, conn, project, header.revision.Generation, header.revision.ResourceType, selectionID) +} + +func (s *Store) CompleteSelection(ctx context.Context, selectionID, writerToken, digest string, count, bytes int64, completedAt time.Time) (*explorer.SelectionRevision, error) { + selectionID = strings.TrimSpace(selectionID) + writerToken = strings.TrimSpace(writerToken) + if selectionID == "" || strings.TrimSpace(digest) == "" || count < 0 || bytes < 0 { + return nil, explorer.ErrSelectionIncomplete + } + if writerToken == "" { + return nil, explorer.ErrSelectionConflict + } + if completedAt.IsZero() { + completedAt = time.Now().UTC() + } + + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire complete-selection connection: %w", err) + } + defer conn.Release() + + tx, err := conn.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: begin complete selection %q: %w", selectionID, err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var storedToken, state string + var complete bool + var project, generation, resourceType string + err = tx.QueryRow(ctx, s.selections.headerForAppend, selectionID). + Scan(&storedToken, &state, &complete, &project, &generation, &resourceType) + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrSelectionNotFound + } + if err != nil { + return nil, fmt.Errorf("postgres explorer store: lock selection header %q for complete: %w", selectionID, err) + } + if complete { + stored, err := s.readSelectionRow(ctx, tx, s.selections.headerByID, selectionID) + if err != nil { + return nil, err + } + if stored.revision.MembershipDigest != digest || + stored.revision.MemberCount != count || stored.revision.MemberBytes != bytes { + return nil, explorer.ErrSelectionConflict + } + revision := stored.revision + return &revision, nil + } + if strings.TrimSpace(storedToken) != writerToken { + return nil, explorer.ErrSelectionConflict + } + if state != "" && state != selectionStagingState { + return nil, explorer.ErrSelectionConflict + } + + stagedDigest, stagedCount, _, err := s.streamMemberFacts(ctx, tx, project, generation, resourceType, selectionID) + if err != nil { + return nil, err + } + if stagedDigest != digest || stagedCount != count { + return nil, explorer.ErrSelectionIncomplete + } + + winner, err := s.readSelectionRowTx(ctx, tx, s.selectionMembers.completeCAS, + selectionID, digest, count, bytes, completedAt.UTC(), writerToken) + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrSelectionConflict + } + if err != nil { + return nil, err + } + if err := winner.revision.Validate(); err != nil { + return nil, err + } + + if s.beforeCompleteCommit != nil { + if err := s.beforeCompleteCommit(ctx, tx); err != nil { + return nil, fmt.Errorf("postgres explorer store: complete selection %q before commit: %w", + selectionID, err) + } + } + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("postgres explorer store: commit complete selection %q: %w", selectionID, err) + } + revision := winner.revision + return &revision, nil +} + +func (s *Store) VisitSelectionMembers(ctx context.Context, project, selectionID, after string, pageSize int, visit func(explorer.SelectionMember) error) (string, error) { + if visit == nil { + return "", fmt.Errorf("selection member visitor is required") + } + header, err := s.GetSelection(ctx, project, selectionID) + if err != nil { + return "", err + } + if !header.Complete { + return "", explorer.ErrSelectionIncomplete + } + if pageSize <= 0 { + pageSize = selectionDigestPageSize + } else if pageSize > selectionDigestPageSize { + pageSize = selectionDigestPageSize + } + + conn, err := s.client.Acquire(ctx) + if err != nil { + return "", fmt.Errorf("postgres explorer store: acquire visit-members connection: %w", err) + } + defer conn.Release() + + last := strings.TrimSpace(after) + page := make([]explorer.SelectionMember, 0, pageSize) + keys := make([]string, 0, pageSize) + rows, err := conn.Query(ctx, s.selectionMembers.memberVisitPage, selectionID, strings.TrimSpace(project), last, pageSize) + if err != nil { + return last, fmt.Errorf("postgres explorer store: page selection members %q: %w", selectionID, err) + } + for rows.Next() { + var member explorer.SelectionMember + if err := rows.Scan(&member.MemberKey, &member.Ref.Project, &member.Ref.Generation, + &member.Ref.ResourceType, &member.Ref.ID, &member.Ordinal); err != nil { + rows.Close() + return last, fmt.Errorf("postgres explorer store: scan selection member page %q: %w", selectionID, err) + } + member = member.Canonical() + page = append(page, member) + keys = append(keys, member.MemberKey) + } + if err := rows.Err(); err != nil { + rows.Close() + return last, fmt.Errorf("postgres explorer store: iterate selection member page %q: %w", selectionID, err) + } + rows.Close() + for i, member := range page { + if err := ctx.Err(); err != nil { + return last, err + } + if err := visit(member); err != nil { + return last, err + } + last = keys[i] + } + return last, nil +} diff --git a/internal/backends/postgres/internal/explorer/selection_members_test.go b/internal/backends/postgres/internal/explorer/selection_members_test.go new file mode 100644 index 000000000..6eff49f29 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/selection_members_test.go @@ -0,0 +1,1363 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "math" + "net/url" + "reflect" + "regexp" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/explorer" +) + +const ( + selemFixtureName = "selection_revisions" + selemFixtureVersion = int64(8) + selemFixtureChecksum = "562a61ef33786cf674f1c87552e2dc8e70691b34116b7f415d5f81960ed8d65d" + selemImmutableCode = "45501" + selemRefConstraint = "selection_members_ref_ux" + selemRaceRounds = 10 + + selemProject = "verify-selem-p" +) + +const ( + selemGoldenKeySelemX = "325f564f597dacc4e53f2fd41e9a0d0ae3ddcd340c927dd8f1647837d882b231" + selemEmptyMembershipDigest = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +) + +var selemNameRe = regexp.MustCompile(`^verify_selem_[0-9]+_[0-9a-f]+$`) + +var ( + selemOwnedMu sync.Mutex + selemOwnedRes = map[string]string{} + selemExtraSchemas []string +) + +func selemRegister(db, schema string) { + selemOwnedMu.Lock() + defer selemOwnedMu.Unlock() + selemOwnedRes[db] = schema +} + +func selemRegisterExtraSchema(schema string) { + selemOwnedMu.Lock() + defer selemOwnedMu.Unlock() + selemExtraSchemas = append(selemExtraSchemas, schema) +} + +type selemService struct { + ctx context.Context + admin *pgxpool.Pool + witness *pgstore.Client + pidW int32 + pidA int32 + pidB int32 + schema string + scrub func(any) string + fail func(step string, err error) + + storeA *Store + storeB *Store + + owner []pgstore.Migration + fixture pgstore.Migration + ledgerAfter []pgstore.LedgerRecord +} + +func newSelemService(t *testing.T) *selemService { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("verify_selem_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !selemNameRe.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if _, err := admin.Exec(cleanupCtx, + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + newClient := func(label string) *pgstore.Client { + client, err := pgstore.Open(ctx, pgstore.Config{ + DSN: dsn, + MaxConns: 1, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + fail("open client "+label, err) + } + t.Cleanup(client.Close) + return client + } + witness := newClient("witness") + clientA := newClient("writer-a") + clientB := newClient("writer-b") + + schema := fmt.Sprintf("verify_selem_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !selemNameRe.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if err := execOn(ctx, witness, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + selemRegister(dbName, schema) + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if err := execOn(cleanupCtx, witness, `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + s := &selemService{ + ctx: ctx, admin: admin, witness: witness, + schema: schema, scrub: scrub, fail: fail, + } + s.migrate(t) + s.pidW = s.backendPID(t, witness, "witness") + s.pidA = s.backendPID(t, clientA, "writer A") + s.pidB = s.backendPID(t, clientB, "writer B") + if s.pidA == s.pidB || s.pidA == s.pidW || s.pidB == s.pidW { + t.Fatalf("writer and witness backends are not distinct physical connections: A=%d B=%d W=%d", s.pidA, s.pidB, s.pidW) + } + + storeA, err := New(clientA, schema) + if err != nil { + fail("build store A", err) + } + s.storeA = storeA + storeB, err := New(clientB, schema) + if err != nil { + fail("build store B", err) + } + s.storeB = storeB + return s +} + +func (s *selemService) backendPID(t *testing.T, client *pgstore.Client, label string) int32 { + t.Helper() + var pid int32 + if err := queryRowOn(s.ctx, client, &pid, `SELECT pg_backend_pid()`); err != nil { + s.fail("backend pid "+label, err) + } + if pid <= 0 { + t.Fatalf("%s backend pid = %d, want a positive server pid", label, pid) + } + return pid +} + +func (s *selemService) migrate(t *testing.T) { + t.Helper() + owner, err := OwnerMigrations() + if err != nil { + s.fail("owner migrations", err) + } + if len(owner) < 2 || owner[0].Version != 1 || owner[0].Name != "ledger" || + owner[1].Version != 2 || owner[1].Name != "explorer_owner" { + t.Fatalf("owner history = %+v, want ledger(1) + explorer_owner(2) at the front", owner) + } + fixture, err := requireOwnerMigration(t, owner, selemFixtureVersion, selemFixtureName) + if err != nil { + s.fail("selection fixture migration", err) + } + sum := sha256.Sum256([]byte(fixture.SQL)) + if got := hex.EncodeToString(sum[:]); got != fixture.Checksum || got != selemFixtureChecksum { + t.Fatalf("checksum self-recompute = %s, runner computed %s, want the byte-frozen merge pin %s", + got, fixture.Checksum, selemFixtureChecksum) + } + runner, err := pgstore.NewMigrationRunner(s.witness, s.schema, owner) + if err != nil { + s.fail("new migration runner", err) + } + result, err := runner.Migrate(s.ctx) + if err != nil { + s.fail("migrate selection history", err) + } + s.owner = owner + s.fixture = fixture + s.ledgerAfter = result.Applied + + want := make([]pgstore.LedgerRecord, 0, len(owner)) + for _, m := range owner { + want = append(want, pgstore.LedgerRecord{Version: m.Version, Name: m.Name, Checksum: m.Checksum}) + } + var maxVersion int64 + for _, w := range want { + if w.Version > maxVersion { + maxVersion = w.Version + } + } + if result.LatestVersion != maxVersion { + t.Fatalf("latest migration version = %d, want %d", result.LatestVersion, maxVersion) + } + if len(result.Applied) != len(want) { + t.Fatalf("applied history = %+v, want %+v", result.Applied, want) + } + for i, w := range want { + got := result.Applied[i] + if got.Version != w.Version || got.Name != w.Name || got.Checksum != w.Checksum { + t.Fatalf("applied[%d] = {Version:%d Name:%q Checksum:%q}, want {Version:%d Name:%q Checksum:%q}", + i, got.Version, got.Name, got.Checksum, w.Version, w.Name, w.Checksum) + } + } + for i := 1; i < len(result.Applied); i++ { + if result.Applied[i-1].Version >= result.Applied[i].Version { + t.Fatalf("ledger not strictly ascending: %+v", result.Applied) + } + } +} + +func (s *selemService) qualified(table string) string { + return pgx.Identifier{s.schema, table}.Sanitize() +} + +func (s *selemService) headersFQN() string { return s.qualified(SelectionRevisionsTable) } +func (s *selemService) membersFQN() string { return s.qualified(SelectionMembersTable) } + +func (s *selemService) witnessExecErr(sql string, args ...any) error { + return execOn(s.ctx, s.witness, sql, args...) +} + +func (s *selemService) execCount(t *testing.T, sql string, args ...any) int { + t.Helper() + var n int + if err := queryRowOn(s.ctx, s.witness, &n, sql, args...); err != nil { + s.fail("count "+sql, err) + } + return n +} + +type selemWitnessedHeader struct { + id, project, generation, resourceType string + ruleDigest, scopeDigest, idempotencyKey string + state, writerToken, membershipDigest string + writerExpiresText, createdAtText, completeAtText string + ruleText, sourceText, exclusionsText string + memberCount, memberBytes int64 + complete bool + ctid string + exists bool +} + +func (w selemWitnessedHeader) same(o selemWitnessedHeader) bool { return w == o } + +func (s *selemService) witnessHeader(id string) selemWitnessedHeader { + var w selemWitnessedHeader + conn, err := s.witness.Acquire(s.ctx) + if err != nil { + s.fail("acquire header witness", err) + } + defer conn.Release() + err = conn.QueryRow(s.ctx, + `SELECT id, project, generation, resource_type, rule_digest, scope_digest, idempotency_key, + state, writer_token, membership_digest, + writer_expires_at::text, created_at::text, COALESCE(completed_at::text, ''), + rule::text, source::text, COALESCE(exclusions::text, ''), + member_count, member_bytes, complete, ctid::text + FROM `+s.headersFQN()+` WHERE id = $1`, id). + Scan(&w.id, &w.project, &w.generation, &w.resourceType, &w.ruleDigest, &w.scopeDigest, + &w.idempotencyKey, &w.state, &w.writerToken, &w.membershipDigest, + &w.writerExpiresText, &w.createdAtText, &w.completeAtText, + &w.ruleText, &w.sourceText, &w.exclusionsText, + &w.memberCount, &w.memberBytes, &w.complete, &w.ctid) + if errors.Is(err, pgx.ErrNoRows) { + return w + } + if err != nil { + s.fail("witness header "+id, err) + } + w.exists = true + return w +} + +func (s *selemService) assertHeaderStable(t *testing.T, id string, before selemWitnessedHeader) selemWitnessedHeader { + t.Helper() + after := s.witnessHeader(id) + if !after.same(before) { + t.Fatalf("selection %s header moved when it must not have:\nbefore=%+v\nafter =%+v", id, before, after) + } + return after +} + +type selemWitnessedMember struct { + selectionID, memberKey, project, generation, resourceType, id string + ordinal int64 + ctid string + exists bool +} + +func (w selemWitnessedMember) same(o selemWitnessedMember) bool { return w == o } + +func (s *selemService) witnessMember(selectionID, memberKey string) selemWitnessedMember { + var w selemWitnessedMember + conn, err := s.witness.Acquire(s.ctx) + if err != nil { + s.fail("acquire member witness", err) + } + defer conn.Release() + err = conn.QueryRow(s.ctx, + `SELECT selection_id, member_key, project, generation, resource_type, id, ordinal, ctid::text + FROM `+s.membersFQN()+` WHERE selection_id = $1 AND member_key = $2`, selectionID, memberKey). + Scan(&w.selectionID, &w.memberKey, &w.project, &w.generation, &w.resourceType, &w.id, &w.ordinal, &w.ctid) + if errors.Is(err, pgx.ErrNoRows) { + return w + } + if err != nil { + s.fail("witness member "+memberKey, err) + } + w.exists = true + return w +} + +func (s *selemService) memberCount(t *testing.T, selectionID string) int { + t.Helper() + return s.execCount(t, "SELECT count(*) FROM "+s.membersFQN()+" WHERE selection_id = $1", selectionID) +} + +func (s *selemService) storedMembers(t *testing.T, selectionID string) []explorer.SelectionMember { + t.Helper() + conn, err := s.witness.Acquire(s.ctx) + if err != nil { + s.fail("acquire member set witness", err) + } + defer conn.Release() + rows, err := conn.Query(s.ctx, + `SELECT project, generation, resource_type, id FROM `+s.membersFQN()+` + WHERE selection_id = $1 ORDER BY id`, selectionID) + if err != nil { + s.fail("witness member set "+selectionID, err) + } + defer rows.Close() + var out []explorer.SelectionMember + for rows.Next() { + var m explorer.SelectionMember + if err := rows.Scan(&m.Ref.Project, &m.Ref.Generation, &m.Ref.ResourceType, &m.Ref.ID); err != nil { + s.fail("scan member set "+selectionID, err) + } + out = append(out, m) + } + if err := rows.Err(); err != nil { + s.fail("iterate member set "+selectionID, err) + } + return out +} + +func selemHex(t *testing.T) string { return randomHex(t) } + +func selemHeaderFixture(id, key string) explorer.SelectionRevision { + return explorer.SelectionRevision{ + ID: id, Project: selemProject, Generation: "gen-a", ResourceType: "DocumentReference", + Rule: explorer.SelectionRule{Kind: explorer.SelectionRuleExplicit}, + Source: explorer.SelectionSource{ + Kind: explorer.SelectionSourcePublished, Generation: "gen-a", + RevisionID: "rev-1", ReceiptID: "receipt-1", ExecutionID: "exec-1", + OutputID: "out-1", SchemaDigest: "schema-digest-1", + ResourceType: "DocumentReference", SourceIDColumn: "id", + }, + ScopeDigest: "selem-scope-" + id, RuleDigest: "selem-rule-" + id, + IdempotencyKey: key, CreatedAt: time.Now().UTC(), + } +} + +func selemRef(id string) explorer.SelectionMember { + return explorer.SelectionMember{Ref: explorer.ResourceRef{ + Project: selemProject, Generation: "gen-a", ResourceType: "DocumentReference", ID: id, + }} +} + +func selemRefs(ids ...string) []explorer.SelectionMember { + out := make([]explorer.SelectionMember, 0, len(ids)) + for _, id := range ids { + out = append(out, selemRef(id)) + } + return out +} + +func (s *selemService) beginStaging(t *testing.T, id, key, writer string) { + t.Helper() + if _, err := s.storeA.BeginSelection(s.ctx, selemHeaderFixture(id, key), writer); err != nil { + s.fail("begin "+id, err) + } +} + +func TestSelectionMembersBatchIdempotentRetryRefusesConflicts(t *testing.T) { + s := newSelemService(t) + suffix := selemHex(t) + id := "selem-batch-" + suffix + writer := "selem-batch-writer-" + suffix + s.beginStaging(t, id, "selem-batch-key-"+suffix, writer) + + batch := selemRefs("files001", "files002", "files001") + inserted, err := s.storeA.AppendSelectionMembers(s.ctx, id, writer, batch) + if err != nil { + s.fail("first append", err) + } + if len(inserted) != 2 || inserted[0].Ref.ID != "files001" || inserted[1].Ref.ID != "files002" { + t.Fatalf("first append inserted %d members (%v), want exactly [files001 files002]", len(inserted), inserted) + } + mk001 := goldenMemberKey(t, id, selemRef("files001").Ref) + mk002 := goldenMemberKey(t, id, selemRef("files002").Ref) + m001 := s.witnessMember(id, mk001) + m002 := s.witnessMember(id, mk002) + if !m001.exists || !m002.exists { + t.Fatalf("staged members missing: %s=%+v %s=%+v", mk001, m001, mk002, m002) + } + if m001.project != selemProject || m001.generation != "gen-a" || + m001.resourceType != "DocumentReference" || m001.id != "files001" { + t.Fatalf("stored member files001 drifted from its reference: %+v", m001) + } + if n := s.memberCount(t, id); n != 2 { + t.Fatalf("member rows after deduped batch = %d, want 2", n) + } + + retryInserted, err := s.storeA.AppendSelectionMembers(s.ctx, id, writer, batch) + if err != nil { + s.fail("retry append", err) + } + if len(retryInserted) != 0 { + t.Fatalf("retry append inserted %d members, want 0", len(retryInserted)) + } + if a := s.witnessMember(id, mk001); !a.same(m001) { + t.Fatalf("retry moved the stored member rows: before=%+v after=%+v", m001, a) + } + if a := s.witnessMember(id, mk002); !a.same(m002) { + t.Fatalf("retry moved the stored member rows: before=%+v after=%+v", m002, a) + } + if n := s.memberCount(t, id); n != 2 { + t.Fatalf("retry created duplicates: %d rows", n) + } + + mixed := selemRefs("files001", "files003") + inserted, err = s.storeA.AppendSelectionMembers(s.ctx, id, writer, mixed) + if err != nil { + s.fail("mixed append", err) + } + if len(inserted) != 1 || inserted[0].Ref.ID != "files003" { + t.Fatalf("mixed append inserted %+v, want exactly [files003]", inserted) + } + if n := s.memberCount(t, id); n != 3 { + t.Fatalf("member rows after mixed append = %d, want 3", n) + } + + foreignBefore := [3]selemWitnessedMember{ + s.witnessMember(id, mk001), + s.witnessMember(id, mk002), + s.witnessMember(id, goldenMemberKey(t, id, selemRef("files003").Ref)), + } + if _, err := s.storeB.AppendSelectionMembers(s.ctx, id, "foreign-"+suffix, selemRefs("files004")); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("foreign writer append = %v, want ErrSelectionConflict", err) + } + for i, refID := range []string{"files001", "files002", "files003"} { + if after := s.witnessMember(id, goldenMemberKey(t, id, selemRef(refID).Ref)); !after.same(foreignBefore[i]) { + t.Fatalf("foreign writer disturbed stored member %s: %+v", refID, after) + } + } + s.assertMemberRowAbsent(t, id, "files004") + if n := s.memberCount(t, id); n != 3 { + t.Fatalf("foreign append left rows behind: %d, want 3", n) + } + + bad := selemRef("files005") + bad.MemberKey = id + "_" + strings.Repeat("0", 64) + if _, err := s.storeA.AppendSelectionMembers(s.ctx, id, writer, append(selemRefs("files006"), bad)); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("wrong-memberKey append = %v, want ErrSelectionConflict", err) + } + s.assertMemberRowAbsent(t, id, "files005") + s.assertMemberRowAbsent(t, id, "files006") + if n := s.memberCount(t, id); n != 3 { + t.Fatalf("wrong-memberKey batch leaked rows: %d, want 3", n) + } + + if err := s.witnessExecErr( + `INSERT INTO `+s.membersFQN()+` + (selection_id, member_key, project, generation, resource_type, id, ordinal) + VALUES ($1, $2, $3, 'gen-a', 'DocumentReference', 'files007', 0)`, + id, id+"_"+strings.Repeat("f", 64), selemProject); err != nil { + s.fail("seed conflicting member", err) + } + foreignRow := s.witnessMember(id, id+"_"+strings.Repeat("f", 64)) + if !foreignRow.exists { + t.Fatal("seeded conflicting member vanished") + } + if _, err := s.storeA.AppendSelectionMembers(s.ctx, id, writer, selemRefs("files008", "files007")); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("arbiter-conflicting append = %v, want ErrSelectionConflict (classified, not a raw driver error)", err) + } + s.assertMemberRowAbsent(t, id, "files008") + if after := s.witnessMember(id, id+"_"+strings.Repeat("f", 64)); !after.same(foreignRow) { + t.Fatalf("the adapter disturbed the conflicting row: %+v", after) + } + if n := s.memberCount(t, id); n != 4 { + t.Fatalf("member rows after arbiter refusal = %d, want 4", n) + } + + assertPgError(t, "raw ref-ux collision", s.witnessExecErr( + `INSERT INTO `+s.membersFQN()+` + (selection_id, member_key, project, generation, resource_type, id, ordinal) + VALUES ($1, $2, $3, 'gen-a', 'DocumentReference', 'files001', 0)`, + id, id+"_"+strings.Repeat("a", 64), selemProject), "23505", selemRefConstraint) + + if err := s.witnessExecErr( + `INSERT INTO `+s.membersFQN()+` + (selection_id, member_key, project, generation, resource_type, id, ordinal) + VALUES ($1, $2, $3, 'gen-a', 'DocumentReference', 'files009', 0)`, + id, id+"_"+strings.Repeat("b", 64), selemProject); err != nil { + t.Fatalf("control: legal member row refused: %s", s.scrub(err)) + } + + poisonKey := goldenMemberKey(t, id, selemRef("files011").Ref) + if err := s.witnessExecErr( + `INSERT INTO `+s.membersFQN()+` + (selection_id, member_key, project, generation, resource_type, id, ordinal) + VALUES ($1, $2, $3, 'gen-a', 'DocumentReference', 'files012', 0)`, + id, poisonKey, selemProject); err != nil { + s.fail("seed key-poisoned member", err) + } + poison := s.witnessMember(id, poisonKey) + if !poison.exists || poison.id != "files012" { + t.Fatalf("seeded key-poisoned row drifted: %+v", poison) + } + if _, err := s.storeA.AppendSelectionMembers(s.ctx, id, writer, selemRefs("files011")); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("append over a key-poisoned row = %v, want ErrSelectionConflict", err) + } + if after := s.witnessMember(id, poisonKey); !after.same(poison) { + t.Fatalf("the adapter rewrote the poisoned row: %+v", after) + } + if n := s.memberCount(t, id); n != 6 { + t.Fatalf("poisoned-key append leaked rows: %d, want 6 (files001-003 + the three hostile probes, no files011)", n) + } + + if err := s.storeA.AbortSelection(s.ctx, id, writer); err != nil { + s.fail("owner abort", err) + } + if w := s.witnessHeader(id); w.exists { + t.Fatalf("owner abort left the header: %+v", w) + } + if n := s.memberCount(t, id); n != 0 { + t.Fatalf("owner abort left %d member rows", n) + } +} + +func (s *selemService) assertMemberRowAbsent(t *testing.T, id, refID string) { + t.Helper() + mk := goldenMemberKey(t, id, selemRef(refID).Ref) + if w := s.witnessMember(id, mk); w.exists { + t.Fatalf("expected no member row %s, found %+v", mk, w) + } +} + +func TestSelectionMembersCompleteFinalizesUnderWriterFence(t *testing.T) { + s := newSelemService(t) + suffix := selemHex(t) + id := "selem-fence-" + suffix + writer := "selem-fence-writer-" + suffix + s.beginStaging(t, id, "selem-fence-key-"+suffix, writer) + if _, err := s.storeA.AppendSelectionMembers(s.ctx, id, writer, selemRefs("files001", "files002")); err != nil { + s.fail("stage members", err) + } + staged := s.storedMembers(t, id) + wantDigest, wantCount, wantBytes := membershipFacts(staged) + + pristine := s.witnessHeader(id) + if _, err := s.storeB.CompleteSelection(s.ctx, id, "stale-"+suffix, wantDigest, wantCount, wantBytes, time.Now().UTC()); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("foreign-token complete = %v, want ErrSelectionConflict", err) + } + s.assertHeaderStable(t, id, pristine) + + if _, err := s.storeA.AppendSelectionMembers(s.ctx, id, writer, selemRefs("files003")); err != nil { + s.fail("append third member", err) + } + postAppend := s.witnessHeader(id) + if postAppend.state != "STAGING" || postAppend.complete { + t.Fatalf("post-append row should still be STAGING: %+v", postAppend) + } + if _, err := s.storeA.CompleteSelection(s.ctx, id, writer, wantDigest, wantCount, wantBytes, time.Now().UTC()); !errors.Is(err, explorer.ErrSelectionIncomplete) { + t.Fatalf("complete with pre-append facts = %v, want ErrSelectionIncomplete (mismatch leaves staging unusable)", err) + } + s.assertHeaderStable(t, id, postAppend) + if _, err := s.storeA.VisitSelectionMembers(s.ctx, selemProject, id, "", 10, func(explorer.SelectionMember) error { + return nil + }); !errors.Is(err, explorer.ErrSelectionIncomplete) { + t.Fatalf("visit after a refused finalize = %v, want ErrSelectionIncomplete -- staging must be unusable", err) + } + + redigested, realCount, rebytes, err := s.storeA.DigestSelectionMembers(s.ctx, selemProject, id) + if err != nil { + s.fail("re-digest", err) + } + if realCount != 3 || redigested == "" { + t.Fatalf("re-digest = %s/%d err=%v, want 3 members", redigested, realCount, err) + } + if _, err := s.storeA.CompleteSelection(s.ctx, id, writer, redigested, realCount-1, rebytes, time.Now().UTC()); !errors.Is(err, explorer.ErrSelectionIncomplete) { + t.Fatalf("complete with count-1 = %v, want ErrSelectionIncomplete", err) + } + s.assertHeaderStable(t, id, postAppend) + + flipped := redigested[:len(redigested)-1] + if redigested[len(redigested)-1] == '0' { + flipped += "1" + } else { + flipped += "0" + } + if _, err := s.storeA.CompleteSelection(s.ctx, id, writer, flipped, realCount, rebytes, time.Now().UTC()); !errors.Is(err, explorer.ErrSelectionIncomplete) { + t.Fatalf("complete with one-flipped digest = %v, want ErrSelectionIncomplete", err) + } + s.assertHeaderStable(t, id, postAppend) + + for i, refID := range []string{"files001", "files002", "files003"} { + _ = i + mk := goldenMemberKey(t, id, selemRef(refID).Ref) + if w := s.witnessMember(id, mk); !w.exists { + t.Fatalf("refused finalizes disturbed stored member %s: %+v", refID, w) + } + } + + completedAt := time.Date(2026, 9, 19, 0, 0, 0, 0, time.UTC) + completed, err := s.storeA.CompleteSelection(s.ctx, id, writer, redigested, realCount, 0, completedAt) + if err != nil { + s.fail("complete with hint bytes", err) + } + if !completed.Complete || completed.MembershipDigest != redigested || completed.MemberCount != realCount || completed.MemberBytes != 0 { + t.Fatalf("completed = %+v, want frozen %s/%d/0", completed, redigested, realCount) + } + done := s.witnessHeader(id) + if done.state != "COMPLETE" || !done.complete || done.membershipDigest != redigested || + done.memberCount != realCount || done.memberBytes != 0 { + t.Fatalf("frozen witness = %+v, want %s/%d/0 COMPLETE", done, redigested, realCount) + } + if done.completeAtText != "2026-09-19 00:00:00+00" { + t.Fatalf("completed_at literal = %q, want 2026-09-19 00:00:00+00", done.completeAtText) + } + + if gotDigest, gotCount, gotBytes := membershipFacts(s.storedMembers(t, id)); gotDigest != redigested || gotCount != realCount || gotBytes != rebytes { + t.Fatalf("frozen digest/count disagrees with stored membership: frozen %s/%d, stored %s/%d (%d bytes)", + redigested, realCount, gotDigest, gotCount, gotBytes) + } + + retry, err := s.storeB.CompleteSelection(s.ctx, id, writer, redigested, realCount, 0, completedAt) + if err != nil { + s.fail("idempotent complete retry", err) + } + if !retry.Complete || retry.MembershipDigest != redigested { + t.Fatalf("idempotent complete retry = %+v", retry) + } + s.assertHeaderStable(t, id, done) + if _, err := s.storeB.CompleteSelection(s.ctx, id, writer, redigested+"0", realCount, 0, completedAt); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("complete with different facts after COMPLETE = %v, want ErrSelectionConflict", err) + } + s.assertHeaderStable(t, id, done) + + if _, err := s.storeA.CompleteSelection(s.ctx, "selem-never-"+suffix, writer, redigested, realCount, 0, completedAt); !errors.Is(err, explorer.ErrSelectionNotFound) { + t.Fatalf("complete on missing selection = %v, want ErrSelectionNotFound", err) + } +} + +func TestSelectionMembersFencedCompleteRacesAppend(t *testing.T) { + s := newSelemService(t) + if s.pidA == s.pidB { + t.Fatal("fence race requires two distinct backends") + } + suffix := selemHex(t) + + completeWins, appendWins := 0, 0 + for round := 0; round < selemRaceRounds; round++ { + id := fmt.Sprintf("selem-fence-race-%d-%s", round, suffix) + writer := "selem-race-writer-" + suffix + s.beginStaging(t, id, fmt.Sprintf("selem-race-key-%d-%s", round, suffix), writer) + if _, err := s.storeA.AppendSelectionMembers(s.ctx, id, writer, selemRefs("files001")); err != nil { + s.fail("race seed append", err) + } + digest, count, bytes, err := s.storeA.DigestSelectionMembers(s.ctx, selemProject, id) + if err != nil || count != 1 { + s.fail("race seed digest", err) + } + + var appendErr, completeErr error + var wg sync.WaitGroup + seamEntered := make(chan struct{}) + seamRelease := make(chan struct{}) + parkInSeam := func(context.Context, pgx.Tx) error { + close(seamEntered) + select { + case <-seamRelease: + return nil + case <-time.After(5 * time.Second): + return fmt.Errorf("race seam timed out waiting for the rival to launch") + } + } + requireSeamEntered := func(operation string) { + select { + case <-seamEntered: + case <-time.After(5 * time.Second): + s.fail(operation+" reached no in-transaction seam", + fmt.Errorf("the rival never held the header fence; the schedule is broken")) + } + } + wg.Add(2) + if round%2 == 0 { + s.storeB.beforeCompleteCommit = parkInSeam + go func() { + defer wg.Done() + _, completeErr = s.storeB.CompleteSelection(s.ctx, id, writer, digest, count, bytes, time.Now().UTC()) + }() + requireSeamEntered("complete") + go func() { + defer wg.Done() + _, appendErr = s.storeA.AppendSelectionMembers(s.ctx, id, writer, selemRefs("files002")) + }() + time.Sleep(2 * time.Millisecond) + close(seamRelease) + } else { + s.storeA.afterMemberInsert = parkInSeam + go func() { + defer wg.Done() + _, appendErr = s.storeA.AppendSelectionMembers(s.ctx, id, writer, selemRefs("files002")) + }() + requireSeamEntered("append") + go func() { + defer wg.Done() + _, completeErr = s.storeB.CompleteSelection(s.ctx, id, writer, digest, count, bytes, time.Now().UTC()) + }() + time.Sleep(2 * time.Millisecond) + close(seamRelease) + } + wg.Wait() + s.storeA.afterMemberInsert = nil + s.storeB.beforeCompleteCommit = nil + + row := s.witnessHeader(id) + if !row.exists { + t.Fatalf("round %d: header vanished", round) + } + stored := s.storedMembers(t, id) + gotDigest, gotCount, _ := membershipFacts(stored) + if row.complete { + completeWins++ + if row.membershipDigest != gotDigest || row.memberCount != gotCount { + t.Fatalf("round %d: COMPLETE row froze %s/%d but the stored membership is %s/%d (fence violated)", + round, row.membershipDigest, row.memberCount, gotDigest, gotCount) + } + if appendErr == nil { + t.Fatalf("round %d: append succeeded yet the row completed without room for it", round) + } + if !errors.Is(appendErr, explorer.ErrSelectionConflict) { + t.Fatalf("round %d: append against the completed row = %v, want ErrSelectionConflict", round, appendErr) + } + if completeErr != nil { + t.Fatalf("round %d: complete lost and won at once: %v", round, completeErr) + } + if len(stored) != 1 { + t.Fatalf("round %d: completed row stores %d members, want the 1 the claim covered", round, len(stored)) + } + } else { + appendWins++ + if !errors.Is(completeErr, explorer.ErrSelectionIncomplete) { + t.Fatalf("round %d: complete against a moved membership = %v, want ErrSelectionIncomplete", round, completeErr) + } + if appendErr != nil { + t.Fatalf("round %d: append lost although the row stayed STAGING: %v", round, appendErr) + } + if row.state != "STAGING" || row.complete || row.membershipDigest != "" || row.memberCount != 0 { + t.Fatalf("round %d: refused finalize left facts behind: %+v", round, row) + } + if len(stored) != 2 { + t.Fatalf("round %d: staged members = %d, want 2 after the append won", round, len(stored)) + } + if _, err := s.storeA.CompleteSelection(s.ctx, id, writer, gotDigest, gotCount, 0, time.Now().UTC()); err != nil { + t.Fatalf("round %d: correct complete after the refusal = %v", round, err) + } + after := s.witnessHeader(id) + if !after.complete || after.membershipDigest != gotDigest || after.memberCount != gotCount { + t.Fatalf("round %d: post-refusal freeze = %+v, want %s/%d", round, after, gotDigest, gotCount) + } + } + } + if completeWins == 0 || appendWins == 0 { + t.Fatalf("race schedule never exercised both interleavings over %d rounds: complete-won=%d append-won=%d", + selemRaceRounds, completeWins, appendWins) + } +} + +func TestSelectionMembersCompleteCrashWindowConvergence(t *testing.T) { + s := newSelemService(t) + suffix := selemHex(t) + id := "selem-crash-" + suffix + writer := "selem-crash-writer-" + suffix + s.beginStaging(t, id, "selem-crash-key-"+suffix, writer) + if _, err := s.storeA.AppendSelectionMembers(s.ctx, id, writer, selemRefs("files001", "files002")); err != nil { + s.fail("stage members", err) + } + before := s.witnessHeader(id) + memberKeys := []string{} + for _, refID := range []string{"files001", "files002"} { + memberKeys = append(memberKeys, goldenMemberKey(t, id, selemRef(refID).Ref)) + } + members := []selemWitnessedMember{s.witnessMember(id, memberKeys[0]), s.witnessMember(id, memberKeys[1])} + + digest, count, bytes, err := s.storeA.DigestSelectionMembers(s.ctx, selemProject, id) + if err != nil { + s.fail("digest", err) + } + s.storeA.beforeCompleteCommit = func(context.Context, pgx.Tx) error { + return errors.New("injected crash before complete commit") + } + if _, err := s.storeA.CompleteSelection(s.ctx, id, writer, digest, count, bytes, time.Now().UTC()); err == nil { + t.Fatal("crashed complete unexpectedly succeeded") + } else if !strings.Contains(err.Error(), "before commit") { + t.Fatalf("crashed complete error = %v, want the seam-classified wrap", err) + } + s.assertHeaderStable(t, id, before) + for i, mk := range memberKeys { + if after := s.witnessMember(id, mk); !after.same(members[i]) { + t.Fatalf("crashed complete disturbed member %s: %+v", mk, after) + } + } + s.storeA.beforeCompleteCommit = nil + completed, err := s.storeA.CompleteSelection(s.ctx, id, writer, digest, count, bytes, time.Now().UTC()) + if err != nil { + s.fail("post-crash complete", err) + } + if !completed.Complete || completed.MembershipDigest != digest || completed.MemberCount != count { + t.Fatalf("post-crash completed = %+v, want %s/%d", completed, digest, count) + } + after := s.witnessHeader(id) + if after.state != "COMPLETE" || !after.complete || after.membershipDigest != digest || after.memberCount != count { + t.Fatalf("post-crash witness = %+v", after) + } +} + +func TestSelectionMembersVisitBoundedKeysetAndCancellation(t *testing.T) { + s := newSelemService(t) + suffix := selemHex(t) + + id := "selem-visit-" + suffix + writer := "selem-visit-writer-" + suffix + s.beginStaging(t, id, "selem-visit-key-"+suffix, writer) + const total = 300 + batch := make([]explorer.SelectionMember, 0, total) + for i := 0; i < total; i++ { + batch = append(batch, selemRef(fmt.Sprintf("vm%04d", i))) + } + for offset := 0; offset < len(batch); offset += 256 { + end := offset + 256 + if end > len(batch) { + end = len(batch) + } + if _, err := s.storeA.AppendSelectionMembers(s.ctx, id, writer, batch[offset:end]); err != nil { + s.fail("bulk append", err) + } + } + digest, count, bytes, err := s.storeA.DigestSelectionMembers(s.ctx, selemProject, id) + if err != nil || count != total { + s.fail("bulk digest", fmt.Errorf("digest = %s/%d err=%v, want %d members", digest, count, err, total)) + } + if _, err := s.storeA.CompleteSelection(s.ctx, id, writer, digest, count, bytes, time.Now().UTC()); err != nil { + s.fail("complete", err) + } + + stopChurn := make(chan struct{}) + churnDone := make(chan struct{}) + var churnCompleted atomic.Int64 + churnErrCh := make(chan error, 1) + recordChurnErr := func(err error) { + select { + case churnErrCh <- err: + default: + } + } + go func() { + defer close(churnDone) + round := 0 + for { + select { + case <-stopChurn: + return + default: + } + round++ + churnID := fmt.Sprintf("selem-churn-%d-%s", round, suffix) + churnWriter := "selem-churn-writer-" + suffix + if _, err := s.storeB.BeginSelection(s.ctx, selemHeaderFixture(churnID, fmt.Sprintf("selem-churn-key-%d-%s", round, suffix)), churnWriter); err != nil { + recordChurnErr(fmt.Errorf("churn begin round %d: %w", round, err)) + return + } + if _, err := s.storeB.AppendSelectionMembers(s.ctx, churnID, churnWriter, selemRefs(fmt.Sprintf("churn%04d", round))); err != nil { + recordChurnErr(fmt.Errorf("churn append round %d: %w", round, err)) + return + } + if err := s.storeB.AbortSelection(s.ctx, churnID, churnWriter); err != nil { + recordChurnErr(fmt.Errorf("churn abort round %d: %w", round, err)) + return + } + churnCompleted.Add(1) + } + }() + + seen := []string{} + cursor := "" + calls := 0 + for { + before := len(seen) + last, err := s.storeA.VisitSelectionMembers(s.ctx, selemProject, id, cursor, 64, func(member explorer.SelectionMember) error { + seen = append(seen, member.MemberKey) + return nil + }) + if err != nil { + s.fail("visit walk", err) + } + calls++ + visited := len(seen) - before + if visited > 64 { + close(stopChurn) + <-churnDone + t.Fatalf("visit call %d visited %d members, want at most pageSize 64", calls, visited) + } + if last == cursor { + if visited != 0 { + t.Fatalf("visit call %d advanced no cursor yet visited %d members", calls, visited) + } + break + } + if visited == 0 { + t.Fatalf("visit call %d advanced the cursor without visiting any member", calls) + } + cursor = last + } + close(stopChurn) + <-churnDone + var churnErr error + select { + case churnErr = <-churnErrCh: + default: + } + if churnErr != nil { + t.Fatalf("foreign churn failed during the walk (rounds completed before the failure: %d): %v", churnCompleted.Load(), churnErr) + } + if rounds := churnCompleted.Load(); rounds == 0 { + t.Fatal("foreign churn completed zero rounds while the walk paged: keyset stability under churn passed vacuously with no churn") + } + + if len(seen) != total { + t.Fatalf("visit walk collected %d, want %d (no gaps, no duplicates)", len(seen), total) + } + if !sort.StringsAreSorted(seen) { + t.Fatal("visit walk is not in byte order") + } + seenSet := map[string]int{} + for _, k := range seen { + seenSet[k]++ + } + for k, n := range seenSet { + if n != 1 { + t.Fatalf("visit walk repeated member %s %d times", k, n) + } + } + if wantCalls := (total+63)/64 + 1; calls != wantCalls { + t.Fatalf("visit walk took %d calls, want %d (64-bounded pages plus the terminal empty page)", calls, wantCalls) + } + + cancelCtx, cancel := context.WithCancel(s.ctx) + const cancelAfter = 20 + visitedBefore := 0 + var cancelCursor string + seenCancel := []string{} + last, err := s.storeA.VisitSelectionMembers(cancelCtx, selemProject, id, "", 64, func(member explorer.SelectionMember) error { + seenCancel = append(seenCancel, member.MemberKey) + visitedBefore++ + if visitedBefore == cancelAfter { + cancel() + } + return nil + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled walk err = %v, want context.Canceled", err) + } + if visitedBefore != cancelAfter { + t.Fatalf("cancelled walk visited %d members, want exactly %d before the stop", visitedBefore, cancelAfter) + } + cancelCursor = last + if len(seenCancel) != cancelAfter { + t.Fatalf("cancelled walk delivered %d members but visited %d (cursor must trail the visitor exactly)", + len(seenCancel), visitedBefore) + } + if cancelCursor != seenCancel[cancelAfter-1] { + t.Fatalf("cancelled cursor %q must name the last visited member %q", cancelCursor, seenCancel[cancelAfter-1]) + } + resume := []string{} + if _, err := s.storeA.VisitSelectionMembers(s.ctx, selemProject, id, cancelCursor, 64, func(member explorer.SelectionMember) error { + resume = append(resume, member.MemberKey) + return nil + }); err != nil { + s.fail("resumed walk after cancellation", err) + } + if len(resume) != 64 { + t.Fatalf("resumed page visited %d, want the next 64", len(resume)) + } + if resume[0] == cancelCursor { + t.Fatal("resumed walk repeated the cursor member (duplicate)") + } + if i := sort.SearchStrings(seen, cancelCursor); i >= len(seen) || seen[i] != cancelCursor { + t.Fatalf("cancelled cursor %q is not a member of the full walk", cancelCursor) + } + if resume[0] != seen[cancelAfter] { + t.Fatalf("resumed walk started at %q, want exactly the next member %q (gap)", resume[0], seen[cancelAfter]) + } + joined := append(append([]string{}, seenCancel...), resume...) + if !reflect.DeepEqual(joined, seen[:cancelAfter+len(resume)]) { + t.Fatal("the cancelled+resumed sequence diverged from the uninterrupted walk") + } + + bigID := "selem-visit-big-" + suffix + bigWriter := "selem-big-writer-" + suffix + s.beginStaging(t, bigID, "selem-big-key-"+suffix, bigWriter) + const bigTotal = 1200 + bigBatch := make([]explorer.SelectionMember, 0, bigTotal) + for i := 0; i < bigTotal; i++ { + bigBatch = append(bigBatch, selemRef(fmt.Sprintf("bm%04d", i))) + } + for offset := 0; offset < len(bigBatch); offset += 600 { + end := offset + 600 + if end > len(bigBatch) { + end = len(bigBatch) + } + if _, err := s.storeA.AppendSelectionMembers(s.ctx, bigID, bigWriter, bigBatch[offset:end]); err != nil { + s.fail("big append", err) + } + } + bigDigest, bigCount, bigBytes, err := s.storeA.DigestSelectionMembers(s.ctx, selemProject, bigID) + if err != nil || bigCount != bigTotal { + s.fail("big digest", fmt.Errorf("digest = %s/%d err=%v", bigDigest, bigCount, err)) + } + if _, err := s.storeA.CompleteSelection(s.ctx, bigID, bigWriter, bigDigest, bigCount, bigBytes, time.Now().UTC()); err != nil { + s.fail("big complete", err) + } + bigVisited := 0 + if _, err := s.storeA.VisitSelectionMembers(s.ctx, selemProject, bigID, "", math.MaxInt32, func(explorer.SelectionMember) error { + bigVisited++ + return nil + }); err != nil { + s.fail("clamp visit", err) + } + if bigVisited != selectionDigestPageSize { + t.Fatalf("absurd page budget visited %d members in one call, want exactly the %d-row bounded page", + bigVisited, selectionDigestPageSize) + } + + stagingID := "selem-visit-staging-" + suffix + s.beginStaging(t, stagingID, "selem-visit-staging-key-"+suffix, "w-"+suffix) + if _, err := s.storeA.VisitSelectionMembers(s.ctx, selemProject, stagingID, "", 10, func(explorer.SelectionMember) error { + return nil + }); !errors.Is(err, explorer.ErrSelectionIncomplete) { + t.Fatalf("visit on STAGING = %v, want ErrSelectionIncomplete", err) + } + if _, err := s.storeA.VisitSelectionMembers(s.ctx, selemProject, id, "", 10, nil); err == nil || + !strings.Contains(err.Error(), "visitor is required") { + t.Fatalf("nil visitor = %v, want the required-visitor refusal", err) + } + if _, err := s.storeA.VisitSelectionMembers(s.ctx, "other-project", id, "", 10, func(explorer.SelectionMember) error { + return nil + }); !errors.Is(err, explorer.ErrSelectionNotFound) { + t.Fatalf("foreign-project visit = %v, want ErrSelectionNotFound", err) + } +} + +func TestSelectionMembersDigestIsBoundedPagedAndFrozen(t *testing.T) { + s := newSelemService(t) + suffix := selemHex(t) + + ref := explorer.ResourceRef{Project: selemProject, Generation: "gen-a", ResourceType: "DocumentReference", ID: "selem-golden-x"} + if got := selectionMemberKey("selection-golden", ref); got != "selection-golden_"+selemGoldenKeySelemX { + t.Fatalf("member key drifted from the pinned golden:\nadapter %s\ngolden %s", got, "selection-golden_"+selemGoldenKeySelemX) + } + + id := "selem-pages-" + suffix + writer := "selem-pages-writer-" + suffix + s.beginStaging(t, id, "selem-pages-key-"+suffix, writer) + const total = 1001 + batch := make([]explorer.SelectionMember, 0, total) + for i := 0; i < total; i++ { + batch = append(batch, selemRef(fmt.Sprintf("pg%04d", i))) + } + if _, err := s.storeA.AppendSelectionMembers(s.ctx, id, writer, batch); err != nil { + s.fail("bulk append", err) + } + + pages := []int{} + s.storeA.memberPageStreamed = func(rows int) { pages = append(pages, rows) } + + digest, count, bytes, err := s.storeA.DigestSelectionMembers(s.ctx, selemProject, id) + if err != nil { + s.fail("paged digest", err) + } + wantDigest, wantCount, wantBytes := membershipFacts(batch) + if digest != wantDigest || count != wantCount || bytes != wantBytes { + t.Fatalf("paged digest = %s/%d/%d, want %s/%d/%d", digest, count, bytes, wantDigest, wantCount, wantBytes) + } + if len(pages) < 2 { + t.Fatalf("digest of %d members fetched %d pages; a single-page read is a full materialization", total, len(pages)) + } + for i, n := range pages { + if n > selectionDigestPageSize { + t.Fatalf("page %d fetched %d rows, above the %d-row bound", i, n, selectionDigestPageSize) + } + } + wantPages := []int{1000, 1} + if len(pages) != len(wantPages) { + t.Fatalf("pages fetched = %v, want %v (exact keyset pages: 1000 + the short final page)", pages, wantPages) + } + for i := range wantPages { + if pages[i] != wantPages[i] { + t.Fatalf("pages fetched = %v, want %v", pages, wantPages) + } + } + + pages = nil + completed, err := s.storeA.CompleteSelection(s.ctx, id, writer, digest, count, bytes, time.Now().UTC()) + if err != nil { + s.fail("fenced complete", err) + } + if !completed.Complete { + t.Fatalf("completed = %+v", completed) + } + if len(pages) != 2 || pages[0] != 1000 || pages[1] != 1 { + t.Fatalf("fenced finalize did not re-stream under the fence: pages = %v, want [1000 1]", pages) + } + + pages = nil + frozenDigest, frozenCount, _, err := s.storeA.DigestSelectionMembers(s.ctx, selemProject, id) + if err != nil { + s.fail("frozen digest", err) + } + if frozenDigest != digest || frozenCount != count { + t.Fatalf("frozen digest = %s/%d, want %s/%d", frozenDigest, frozenCount, digest, count) + } + if len(pages) != 0 { + t.Fatalf("COMPLETE digest re-streamed %v; frozen facts must be returned as stored", pages) + } + s.storeA.memberPageStreamed = nil + + emptyID := "selem-empty-" + suffix + emptyWriter := "selem-empty-writer-" + suffix + s.beginStaging(t, emptyID, "selem-empty-key-"+suffix, emptyWriter) + emptyDigest, emptyCount, emptyBytes, err := s.storeA.DigestSelectionMembers(s.ctx, selemProject, emptyID) + if err != nil { + s.fail("empty digest", err) + } + if emptyDigest != selemEmptyMembershipDigest || emptyCount != 0 || emptyBytes != 0 { + t.Fatalf("empty digest = %s/%d/%d, want %s/0/0", emptyDigest, emptyCount, emptyBytes, selemEmptyMembershipDigest) + } + if _, err := s.storeA.CompleteSelection(s.ctx, emptyID, emptyWriter, emptyDigest, 0, 0, time.Now().UTC()); err != nil { + t.Fatalf("finalizing the empty membership = %v, want success", err) + } + row := s.witnessHeader(emptyID) + if !row.complete || row.membershipDigest != selemEmptyMembershipDigest || row.memberCount != 0 { + t.Fatalf("empty-selection freeze = %+v", row) + } + if _, err := s.storeA.CompleteSelection(s.ctx, emptyID, emptyWriter, digest, count, bytes, time.Now().UTC()); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("re-complete of the empty row with foreign facts = %v, want ErrSelectionConflict", err) + } +} + +func TestSelectionMembersSQLAndGoldenContract(t *testing.T) { + sql := newSelectionMemberSQL("ctl") + headersFQN := `"ctl"."selection_revisions"` + membersFQN := `"ctl"."selection_members"` + for name, statement := range map[string]string{ + "memberInsert": sql.memberInsert, "memberByKey": sql.memberByKey, + "memberDigestPage": sql.memberDigestPage, "memberVisitPage": sql.memberVisitPage, + "leaseRenew": sql.leaseRenew, "completeCAS": sql.completeCAS, + } { + if !strings.Contains(statement, headersFQN) && !strings.Contains(statement, membersFQN) { + t.Fatalf("%s is not schema-qualified: %s", name, statement) + } + } + if !strings.Contains(sql.memberInsert, "ON CONFLICT (selection_id, member_key) DO NOTHING") { + t.Fatalf("memberInsert must be insert-once idempotent: %s", sql.memberInsert) + } + if got := strings.Count(sql.memberInsert, "$"); got != len(memberColumns)+1 { + t.Fatalf("memberInsert placeholders = %d, want %d", got, len(memberColumns)+1) + } + if !strings.Contains(sql.memberDigestPage, "ORDER BY id LIMIT $6") || + !strings.Contains(sql.memberVisitPage, "ORDER BY member_key LIMIT $4") { + t.Fatalf("both member walks must be keyset-bounded: %q / %q", sql.memberDigestPage, sql.memberVisitPage) + } + if !strings.Contains(sql.completeCAS, "state = 'STAGING'") || !strings.Contains(sql.completeCAS, "writer_token = $6") { + t.Fatalf("completeCAS must be a writer-conditional staging CAS: %s", sql.completeCAS) + } + if !strings.Contains(string(readFixtureDDL(t)), selemRefConstraint) { + t.Fatalf("fixture DDL must define the arbiter the adapter classifies: %s", selemRefConstraint) + } + conflict := &pgconn.PgError{Code: "23505", ConstraintName: selemRefConstraint} + if !isMemberRefConflict(fmt.Errorf("wrapped: %w", conflict)) { + t.Fatal("the named arbiter's refusal must classify as a conflicting member") + } + if isMemberRefConflict(&pgconn.PgError{Code: "23505", ConstraintName: "selection_revisions_id_pk"}) { + t.Fatal("a different constraint's unique violation is not a member conflict") + } + if isMemberRefConflict(errors.New("plain error")) { + t.Fatal("a plain error is not a member conflict") + } + typedNil := error((*pgconn.PgError)(nil)) + if isMemberRefConflict(typedNil) { + t.Fatal("a typed-nil driver error is not a member conflict") + } + if isMemberRefConflict(fmt.Errorf("insert: %w", typedNil)) { + t.Fatal("a wrapped typed-nil driver error is not a member conflict") + } +} + +func readFixtureDDL(t *testing.T) []byte { + t.Helper() + owner, err := OwnerMigrations() + if err != nil { + t.Fatal(err) + } + fixture, err := requireOwnerMigration(t, owner, selemFixtureVersion, selemFixtureName) + if err != nil { + t.Fatal(err) + } + return []byte(fixture.SQL) +} + +func TestZZZeroLeftoverVerifySelemFixtures(t *testing.T) { + raw := testPostgresURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %v", err) + } + defer admin.Close() + + selemOwnedMu.Lock() + databases := make(map[string]string, len(selemOwnedRes)) + for db, schema := range selemOwnedRes { + databases[db] = schema + } + extra := append([]string{}, selemExtraSchemas...) + selemOwnedMu.Unlock() + + for db, schema := range databases { + var count int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname = $1`, db).Scan(&count); err != nil { + t.Fatalf("audit database %s: %v", db, err) + } + if count != 0 { + t.Fatalf("isolated database %s (owning schema %s) was left behind", db, schema) + } + } + for _, schema := range extra { + var present bool + if err := admin.QueryRow(ctx, `SELECT to_regnamespace($1) IS NOT NULL`, schema).Scan(&present); err != nil { + t.Fatalf("audit extra schema %s: %v", schema, err) + } + if present { + t.Fatalf("probe schema %s was left behind in the service database", schema) + } + } + + var strays []string + rows, err := admin.Query(ctx, `SELECT datname FROM pg_database WHERE datname LIKE 'verify\_selem\_%' ESCAPE '\' ORDER BY datname`) + if err != nil { + t.Fatalf("prefix-scan databases: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("scan prefix strays: %v", err) + } + strays = append(strays, "database "+name) + } + if err := rows.Err(); err != nil { + rows.Close() + t.Fatalf("iterate prefix strays: %v", err) + } + rows.Close() + schemaRows, err := admin.Query(ctx, `SELECT n.nspname FROM pg_namespace n WHERE n.nspname LIKE 'verify\_selem\_%' ESCAPE '\' ORDER BY n.nspname`) + if err != nil { + t.Fatalf("prefix-scan schemas: %v", err) + } + for schemaRows.Next() { + var name string + if err := schemaRows.Scan(&name); err != nil { + schemaRows.Close() + t.Fatalf("scan schema strays: %v", err) + } + strays = append(strays, "schema "+name) + } + if err := schemaRows.Err(); err != nil { + schemaRows.Close() + t.Fatalf("iterate schema strays: %v", err) + } + schemaRows.Close() + if len(strays) > 0 { + t.Fatalf("unregistered survivors under the verify_selem_* prefix: %s", strings.Join(strays, ", ")) + } +} diff --git a/internal/backends/postgres/internal/explorer/selections.go b/internal/backends/postgres/internal/explorer/selections.go new file mode 100644 index 000000000..223191014 --- /dev/null +++ b/internal/backends/postgres/internal/explorer/selections.go @@ -0,0 +1,403 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/calypr/loom/internal/explorer" +) + +const ( + SelectionRevisionsTable = "selection_revisions" + SelectionMembersTable = "selection_members" + + selectionStagingState = "STAGING" + selectionCompleteState = "COMPLETE" + selectionWriterLease = 10 * time.Minute + selectionCleanupLimit = 100 +) + +var selectionColumns = []string{ + "id", + "project", + "generation", + "resource_type", + "rule", + "source", + "exclusions", + "scope_digest", + "rule_digest", + "membership_digest", + "member_count", + "member_bytes", + "complete", + "idempotency_key", + "created_at", + "completed_at", +} + +var selectionWriteColumns = append([]string{"state", "writer_token", "writer_expires_at"}, selectionColumns...) + +var selectionReadColumns = append(append([]string{}, selectionColumns...), "writer_token", "writer_expires_at") + +type selectionSQL struct { + beginInsert string + headerByID string + headerScoped string + keyOwner string + leaseClaim string + headerForAppend string + cleanupVictims string + cleanupMembers string + cleanupHeaders string + abortMembers string + abortHeader string +} + +func newSelectionSQL(schema string) selectionSQL { + headers := pgx.Identifier{schema, SelectionRevisionsTable}.Sanitize() + members := pgx.Identifier{schema, SelectionMembersTable}.Sanitize() + readCols := strings.Join(selectionReadColumns, ", ") + writePlaceholders := make([]string, len(selectionWriteColumns)) + for i := range selectionWriteColumns { + writePlaceholders[i] = fmt.Sprintf("$%d", i+1) + } + return selectionSQL{ + beginInsert: "INSERT INTO " + headers + " (" + strings.Join(selectionWriteColumns, ", ") + ") VALUES (" + + strings.Join(writePlaceholders, ", ") + ") ON CONFLICT DO NOTHING", + headerByID: "SELECT " + readCols + " FROM " + headers + " WHERE id = $1", + headerScoped: "SELECT " + readCols + " FROM " + headers + " WHERE id = $1 AND project = $2", + keyOwner: "SELECT id FROM " + headers + + " WHERE project = $1 AND idempotency_key = $2 AND $2 <> ''", + leaseClaim: "UPDATE " + headers + " SET writer_token = $2, writer_expires_at = $3" + + " WHERE id = $1 AND state = '" + selectionStagingState + "'" + + " AND writer_expires_at < $4" + + " RETURNING " + readCols, + headerForAppend: "SELECT writer_token, state, complete, project, generation, resource_type" + + " FROM " + headers + " WHERE id = $1 FOR UPDATE", + cleanupVictims: "SELECT id FROM " + headers + + " WHERE state = '" + selectionStagingState + "' AND created_at < $1 AND writer_expires_at < $2" + + " ORDER BY created_at, id LIMIT $3 FOR UPDATE", + cleanupMembers: "DELETE FROM " + members + " WHERE selection_id = ANY($1)", + cleanupHeaders: "DELETE FROM " + headers + " WHERE id = ANY($1)", + abortMembers: "DELETE FROM " + members + " WHERE selection_id = $1", + abortHeader: "DELETE FROM " + headers + " WHERE id = $1", + } +} + +type selectionRow struct { + revision explorer.SelectionRevision + writerToken string + writerExpires time.Time +} + +func scanSelection(row rowScanner) (selectionRow, error) { + var out selectionRow + var ruleRaw, sourceRaw, exclusionsRaw json.RawMessage + var completedAt *time.Time + targets := []any{ + &out.revision.ID, + &out.revision.Project, + &out.revision.Generation, + &out.revision.ResourceType, + &ruleRaw, + &sourceRaw, + &exclusionsRaw, + &out.revision.ScopeDigest, + &out.revision.RuleDigest, + &out.revision.MembershipDigest, + &out.revision.MemberCount, + &out.revision.MemberBytes, + &out.revision.Complete, + &out.revision.IdempotencyKey, + &out.revision.CreatedAt, + &completedAt, + &out.writerToken, + &out.writerExpires, + } + if err := row.Scan(targets...); err != nil { + return selectionRow{}, err + } + out.revision.CompletedAt = completedAt + if err := json.Unmarshal(ruleRaw, &out.revision.Rule); err != nil { + return selectionRow{}, fmt.Errorf("postgres explorer store: decode selection rule: %w", err) + } + if err := json.Unmarshal(sourceRaw, &out.revision.Source); err != nil { + return selectionRow{}, fmt.Errorf("postgres explorer store: decode selection source: %w", err) + } + if len(exclusionsRaw) > 0 { + if err := json.Unmarshal(exclusionsRaw, &out.revision.Exclusions); err != nil { + return selectionRow{}, fmt.Errorf("postgres explorer store: decode selection exclusions: %w", err) + } + } + out.revision = out.revision.Canonical() + return out, nil +} + +func selectionArgs(s explorer.SelectionRevision, ruleJSON, sourceJSON, exclusionsJSON []byte, writerToken string, expiresAt time.Time) []any { + return []any{ + selectionStagingState, + writerToken, + expiresAt, + s.ID, + s.Project, + s.Generation, + s.ResourceType, + ruleJSON, + sourceJSON, + exclusionsJSON, + s.ScopeDigest, + s.RuleDigest, + s.MembershipDigest, + s.MemberCount, + s.MemberBytes, + s.Complete, + s.IdempotencyKey, + s.CreatedAt, + nil, + } +} + +func (s *Store) BeginSelection(ctx context.Context, selection explorer.SelectionRevision, writerToken string) (*explorer.SelectionRevision, error) { + selection = selection.Canonical() + writerToken = strings.TrimSpace(writerToken) + if writerToken == "" { + return nil, explorer.ErrSelectionConflict + } + if err := selection.Validate(); err != nil { + return nil, err + } + if selection.Complete { + return nil, fmt.Errorf("selection must begin incomplete") + } + if err := s.CleanupSelectionStaging(ctx, time.Now().UTC(), selectionCleanupLimit); err != nil { + return nil, err + } + ruleJSON, err := json.Marshal(selection.Rule) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: encode selection rule: %w", err) + } + sourceJSON, err := json.Marshal(selection.Source) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: encode selection source: %w", err) + } + var exclusionsJSON []byte + if selection.Exclusions != nil { + exclusionsJSON, err = json.Marshal(selection.Exclusions) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: encode selection exclusions: %w", err) + } + } + now := time.Now().UTC() + + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire begin-selection connection: %w", err) + } + defer conn.Release() + + if _, err := conn.Exec(ctx, s.selections.beginInsert, + selectionArgs(selection, ruleJSON, sourceJSON, exclusionsJSON, writerToken, now.Add(selectionWriterLease))...); err != nil { + return nil, fmt.Errorf("postgres explorer store: bind selection revision %q/%q: %w", + selection.Project, selection.ID, err) + } + + stored, err := s.readSelectionRow(ctx, conn, s.selections.headerByID, selection.ID) + if errors.Is(err, explorer.ErrSelectionNotFound) { + var holder string + holderErr := conn.QueryRow(ctx, s.selections.keyOwner, selection.Project, selection.IdempotencyKey).Scan(&holder) + if errors.Is(holderErr, pgx.ErrNoRows) { + return nil, explorer.ErrSelectionNotFound + } + if holderErr != nil { + return nil, fmt.Errorf("postgres explorer store: resolve selection key owner %q/%q: %w", + selection.Project, selection.ID, holderErr) + } + return nil, explorer.ErrSelectionConflict + } + if err != nil { + return nil, err + } + + if stored.revision.Project != selection.Project || + stored.revision.Generation != selection.Generation || + stored.revision.ResourceType != selection.ResourceType || + stored.revision.RuleDigest != selection.RuleDigest || + stored.revision.ScopeDigest != selection.ScopeDigest || + stored.revision.IdempotencyKey != selection.IdempotencyKey { + return nil, explorer.ErrSelectionConflict + } + if stored.revision.Complete { + if err := stored.revision.Validate(); err != nil { + return nil, err + } + revision := stored.revision + return &revision, nil + } + if strings.TrimSpace(stored.writerToken) == writerToken { + revision := stored.revision + return &revision, nil + } + + claimed, err := s.readSelectionRowTx(ctx, conn, s.selections.leaseClaim, + selection.ID, writerToken, now.Add(selectionWriterLease), now) + if errors.Is(err, pgx.ErrNoRows) { + return nil, explorer.ErrSelectionConflict + } + if err != nil { + return nil, err + } + revision := claimed.revision + return &revision, nil +} + +func (s *Store) AbortSelection(ctx context.Context, selectionID, writerToken string) error { + selectionID = strings.TrimSpace(selectionID) + writerToken = strings.TrimSpace(writerToken) + if selectionID == "" { + return nil + } + if writerToken == "" { + return explorer.ErrSelectionConflict + } + + conn, err := s.client.Acquire(ctx) + if err != nil { + return fmt.Errorf("postgres explorer store: acquire abort-selection connection: %w", err) + } + defer conn.Release() + + tx, err := conn.Begin(ctx) + if err != nil { + return fmt.Errorf("postgres explorer store: begin abort selection %q: %w", selectionID, err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var storedToken string + var complete bool + err = tx.QueryRow(ctx, s.selections.headerForAppend, selectionID).Scan(&storedToken, new(string), &complete, new(string), new(string), new(string)) + if errors.Is(err, pgx.ErrNoRows) { + return tx.Commit(ctx) + } + if err != nil { + return fmt.Errorf("postgres explorer store: lock selection header %q for abort: %w", selectionID, err) + } + if complete || strings.TrimSpace(storedToken) != writerToken { + return tx.Commit(ctx) + } + if _, err := tx.Exec(ctx, s.selections.abortMembers, selectionID); err != nil { + return fmt.Errorf("postgres explorer store: remove staged members of selection %q: %w", selectionID, err) + } + if _, err := tx.Exec(ctx, s.selections.abortHeader, selectionID); err != nil { + return fmt.Errorf("postgres explorer store: remove selection header %q: %w", selectionID, err) + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("postgres explorer store: commit abort selection %q: %w", selectionID, err) + } + return nil +} + +func (s *Store) GetSelection(ctx context.Context, project, selectionID string) (*explorer.SelectionRevision, error) { + conn, err := s.client.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres explorer store: acquire get-selection connection: %w", err) + } + defer conn.Release() + + stored, err := s.readSelectionRow(ctx, conn, s.selections.headerScoped, strings.TrimSpace(selectionID), strings.TrimSpace(project)) + if err != nil { + return nil, err + } + if stored.revision.Complete { + if err := stored.revision.Validate(); err != nil { + return nil, err + } + } + revision := stored.revision + return &revision, nil +} + +func (s *Store) CleanupSelectionStaging(ctx context.Context, before time.Time, limit int) error { + if limit <= 0 { + limit = selectionCleanupLimit + } + conn, err := s.client.Acquire(ctx) + if err != nil { + return fmt.Errorf("postgres explorer store: acquire cleanup connection: %w", err) + } + defer conn.Release() + + tx, err := conn.Begin(ctx) + if err != nil { + return fmt.Errorf("postgres explorer store: begin cleanup staging: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + rows, err := tx.Query(ctx, s.selections.cleanupVictims, before.UTC(), time.Now().UTC(), limit) + if err != nil { + return fmt.Errorf("postgres explorer store: select cleanup victims: %w", err) + } + var victims []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + rows.Close() + return fmt.Errorf("postgres explorer store: scan cleanup victims: %w", err) + } + victims = append(victims, id) + } + if err := rows.Err(); err != nil { + rows.Close() + return fmt.Errorf("postgres explorer store: iterate cleanup victims: %w", err) + } + rows.Close() + + if len(victims) > 0 { + if _, err := tx.Exec(ctx, s.selections.cleanupMembers, victims); err != nil { + return fmt.Errorf("postgres explorer store: remove abandoned members: %w", err) + } + if _, err := tx.Exec(ctx, s.selections.cleanupHeaders, victims); err != nil { + return fmt.Errorf("postgres explorer store: remove abandoned headers: %w", err) + } + } + + if s.beforeCleanupCommit != nil { + if err := s.beforeCleanupCommit(ctx, tx); err != nil { + return fmt.Errorf("postgres explorer store: cleanup staging before commit: %w", err) + } + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("postgres explorer store: commit cleanup staging: %w", err) + } + return nil +} + +func (s *Store) readSelectionRow(ctx context.Context, conn selectionQuerier, query string, args ...any) (selectionRow, error) { + stored, err := scanSelection(conn.QueryRow(ctx, query, args...)) + if errors.Is(err, pgx.ErrNoRows) { + return selectionRow{}, explorer.ErrSelectionNotFound + } + if err != nil { + return selectionRow{}, fmt.Errorf("postgres explorer store: read selection revision: %w", err) + } + return stored, nil +} + +func (s *Store) readSelectionRowTx(ctx context.Context, conn selectionQuerier, query string, args ...any) (selectionRow, error) { + stored, err := scanSelection(conn.QueryRow(ctx, query, args...)) + if err != nil { + return selectionRow{}, fmt.Errorf("postgres explorer store: write-and-reread selection revision: %w", err) + } + return stored, nil +} + +type selectionQuerier interface { + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) +} diff --git a/internal/backends/postgres/internal/explorer/selections_test.go b/internal/backends/postgres/internal/explorer/selections_test.go new file mode 100644 index 000000000..cd935477e --- /dev/null +++ b/internal/backends/postgres/internal/explorer/selections_test.go @@ -0,0 +1,1970 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "reflect" + "regexp" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/explorer" +) + +const ( + pgstate05FixtureName = "selection_revisions" + pgstate05FixtureVersion = int64(8) + pgstate05FixtureChecksum = "562a61ef33786cf674f1c87552e2dc8e70691b34116b7f415d5f81960ed8d65d" + pgstate05ImmutableCode = "45501" + pgstate05RaceRounds = 10 + + pgstate05Project = "pgstate05-p" +) + +func requireOwnerMigration(t *testing.T, history []pgstore.Migration, version int64, name string) (pgstore.Migration, error) { + t.Helper() + var found pgstore.Migration + count := 0 + for _, m := range history { + if m.Version == version { + count++ + found = m + } + } + if count != 1 { + return pgstore.Migration{}, fmt.Errorf("owner history has %d migrations at version %d, want exactly 1 (%s): %+v", count, version, name, history) + } + if found.Name != name { + return pgstore.Migration{}, fmt.Errorf("owner history version %d = %q, want %q", version, found.Name, name) + } + return found, nil +} + +const ( + pgstate05GoldenKeyFiles001 = "881744de4921fab22a53cec80560023da4ef4071cbe1fec6130e0222c525724a" + pgstate05GoldenKeyFiles002 = "a88d58e2d139439341429d35e241c7911ed4a42893a9ac38b21a80d70b684f3f" + pgstate05GoldenKeyFiles003 = "0f26d620a48845ee8df20bdc13c8bb56f47479d3953c3e4ccb1965b5fd43279a" +) + +var pgstate05NameRe = regexp.MustCompile(`^pgstate05_it_[0-9]+_[0-9a-f]+$`) + +var ( + pgstate05OwnedMu sync.Mutex + pgstate05OwnedRes = map[string]string{} + pgstate05ExtraSchemas []string +) + +func pgstate05Register(db, schema string) { + pgstate05OwnedMu.Lock() + defer pgstate05OwnedMu.Unlock() + pgstate05OwnedRes[db] = schema +} + +func pgstate05RegisterExtraSchema(schema string) { + pgstate05OwnedMu.Lock() + defer pgstate05OwnedMu.Unlock() + pgstate05ExtraSchemas = append(pgstate05ExtraSchemas, schema) +} + +type selectionService struct { + ctx context.Context + admin *pgxpool.Pool + witness *pgstore.Client + pidW int32 + pidA int32 + pidB int32 + schema string + scrub func(any) string + fail func(step string, err error) + + storeA *Store + storeB *Store + + owner []pgstore.Migration + fixture pgstore.Migration + ledgerAfter []pgstore.LedgerRecord +} + +func newSelectionService(t *testing.T) *selectionService { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgstate05_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstate05NameRe.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if _, err := admin.Exec(cleanupCtx, + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + newClient := func(label string) *pgstore.Client { + client, err := pgstore.Open(ctx, pgstore.Config{ + DSN: dsn, + MaxConns: 1, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + fail("open client "+label, err) + } + t.Cleanup(client.Close) + return client + } + witness := newClient("witness") + clientA := newClient("writer-a") + clientB := newClient("writer-b") + + schema := fmt.Sprintf("pgstate05_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstate05NameRe.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if err := execOn(ctx, witness, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + pgstate05Register(dbName, schema) + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if err := execOn(cleanupCtx, witness, `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + s := &selectionService{ + ctx: ctx, admin: admin, witness: witness, + schema: schema, scrub: scrub, fail: fail, + } + s.migrate(t) + s.pidW = s.backendPID(t, witness, "witness") + s.pidA = s.backendPID(t, clientA, "writer A") + s.pidB = s.backendPID(t, clientB, "writer B") + if s.pidA == s.pidB || s.pidA == s.pidW || s.pidB == s.pidW { + t.Fatalf("writer and witness backends are not distinct physical connections: A=%d B=%d W=%d", s.pidA, s.pidB, s.pidW) + } + + storeA, err := New(clientA, schema) + if err != nil { + fail("build store A", err) + } + s.storeA = storeA + storeB, err := New(clientB, schema) + if err != nil { + fail("build store B", err) + } + s.storeB = storeB + return s +} + +func (s *selectionService) backendPID(t *testing.T, client *pgstore.Client, label string) int32 { + t.Helper() + var pid int32 + if err := queryRowOn(s.ctx, client, &pid, `SELECT pg_backend_pid()`); err != nil { + s.fail("backend pid "+label, err) + } + if pid <= 0 { + t.Fatalf("%s backend pid = %d, want a positive server pid", label, pid) + } + return pid +} + +func (s *selectionService) migrate(t *testing.T) { + t.Helper() + owner, err := OwnerMigrations() + if err != nil { + s.fail("owner migrations", err) + } + if len(owner) < 2 || owner[0].Version != 1 || owner[0].Name != "ledger" || + owner[1].Version != 2 || owner[1].Name != "explorer_owner" { + t.Fatalf("owner history = %+v, want ledger(1) + explorer_owner(2) at the front", owner) + } + fixture, err := requireOwnerMigration(t, owner, pgstate05FixtureVersion, pgstate05FixtureName) + if err != nil { + s.fail("selection-revisions migration", err) + } + sum := sha256.Sum256([]byte(fixture.SQL)) + if got := hex.EncodeToString(sum[:]); got != fixture.Checksum || got != pgstate05FixtureChecksum { + t.Fatalf("checksum self-recompute = %s, runner computed %s, want the byte-frozen merge pin %s", + got, fixture.Checksum, pgstate05FixtureChecksum) + } + runner, err := pgstore.NewMigrationRunner(s.witness, s.schema, owner) + if err != nil { + s.fail("new migration runner", err) + } + result, err := runner.Migrate(s.ctx) + if err != nil { + s.fail("migrate selection history", err) + } + s.owner = owner + s.fixture = fixture + s.ledgerAfter = result.Applied + + want := make([]pgstore.LedgerRecord, 0, len(owner)) + for _, m := range owner { + want = append(want, pgstore.LedgerRecord{Version: m.Version, Name: m.Name, Checksum: m.Checksum}) + } + var maxVersion int64 + for _, w := range want { + if w.Version > maxVersion { + maxVersion = w.Version + } + } + if result.LatestVersion != maxVersion { + t.Fatalf("latest migration version = %d, want %d", result.LatestVersion, maxVersion) + } + if len(result.Applied) != len(want) { + t.Fatalf("applied history = %+v, want %+v", result.Applied, want) + } + for i, w := range want { + got := result.Applied[i] + if got.Version != w.Version || got.Name != w.Name || got.Checksum != w.Checksum { + t.Fatalf("applied[%d] = {Version:%d Name:%q Checksum:%q}, want {Version:%d Name:%q Checksum:%q}", + i, got.Version, got.Name, got.Checksum, w.Version, w.Name, w.Checksum) + } + } + for i := 1; i < len(result.Applied); i++ { + if result.Applied[i-1].Version >= result.Applied[i].Version { + t.Fatalf("ledger not strictly ascending: %+v", result.Applied) + } + } +} + +func (s *selectionService) qualified(table string) string { + return pgx.Identifier{s.schema, table}.Sanitize() +} + +func (s *selectionService) headersFQN() string { return s.qualified(SelectionRevisionsTable) } +func (s *selectionService) membersFQN() string { return s.qualified(SelectionMembersTable) } + +func (s *selectionService) mustWitnessExec(t *testing.T, sql string, args ...any) { + t.Helper() + if err := execOn(s.ctx, s.witness, sql, args...); err != nil { + s.fail("witness exec "+sql, err) + } +} + +func (s *selectionService) witnessExecErr(sql string, args ...any) error { + return execOn(s.ctx, s.witness, sql, args...) +} + +func (s *selectionService) witnessExecTag(t *testing.T, sql string, args ...any) int64 { + t.Helper() + conn, err := s.witness.Acquire(s.ctx) + if err != nil { + s.fail("acquire tag witness", err) + } + defer conn.Release() + tag, err := conn.Exec(s.ctx, sql, args...) + if err != nil { + s.fail("witness tag exec "+sql, err) + } + return tag.RowsAffected() +} + +func (s *selectionService) mustWitnessBool(t *testing.T, sql string, args ...any) bool { + t.Helper() + var value bool + if err := queryRowOn(s.ctx, s.witness, &value, sql, args...); err != nil { + s.fail("witness "+sql, err) + } + return value +} + +func parseWitnessTime(t *testing.T, text string) time.Time { + t.Helper() + for _, layout := range []string{ + "2006-01-02 15:04:05.999999999-07", + "2006-01-02 15:04:05.999999999-0700", + "2006-01-02 15:04:05-07", + } { + if ts, err := time.Parse(layout, text); err == nil { + return ts + } + } + t.Fatalf("cannot parse witness timestamptz text %q", text) + return time.Time{} +} + +func (s *selectionService) execCount(t *testing.T, sql string, args ...any) int { + t.Helper() + var n int + if err := queryRowOn(s.ctx, s.witness, &n, sql, args...); err != nil { + s.fail("count "+sql, err) + } + return n +} + +type witnessedSelection struct { + id, project, generation, resourceType string + ruleDigest, scopeDigest, idempotencyKey string + state, writerToken, membershipDigest string + writerExpiresText, createdAtText, completeAtText string + ruleText, sourceText, exclusionsText string + memberCount, memberBytes int64 + complete bool + ctid string + exists bool +} + +func (w witnessedSelection) same(o witnessedSelection) bool { + return w == o +} + +func (s *selectionService) witnessSelection(id string) witnessedSelection { + var w witnessedSelection + conn, err := s.witness.Acquire(s.ctx) + if err != nil { + s.fail("acquire selection witness", err) + } + defer conn.Release() + err = conn.QueryRow(s.ctx, + `SELECT id, project, generation, resource_type, rule_digest, scope_digest, idempotency_key, + state, writer_token, membership_digest, + writer_expires_at::text, created_at::text, COALESCE(completed_at::text, ''), + rule::text, source::text, COALESCE(exclusions::text, ''), + member_count, member_bytes, complete, ctid::text + FROM `+s.headersFQN()+` WHERE id = $1`, id). + Scan(&w.id, &w.project, &w.generation, &w.resourceType, &w.ruleDigest, &w.scopeDigest, + &w.idempotencyKey, &w.state, &w.writerToken, &w.membershipDigest, + &w.writerExpiresText, &w.createdAtText, &w.completeAtText, + &w.ruleText, &w.sourceText, &w.exclusionsText, + &w.memberCount, &w.memberBytes, &w.complete, &w.ctid) + if errors.Is(err, pgx.ErrNoRows) { + return w + } + if err != nil { + s.fail("witness selection "+id, err) + } + w.exists = true + return w +} + +func (s *selectionService) assertSelectionAbsent(t *testing.T, id string) { + t.Helper() + if w := s.witnessSelection(id); w.exists { + t.Fatalf("expected no selection_revisions row for %s, found %+v", id, w) + } +} + +func (s *selectionService) assertSelectionStable(t *testing.T, id string, before witnessedSelection) witnessedSelection { + t.Helper() + after := s.witnessSelection(id) + if !after.same(before) { + t.Fatalf("selection %s row moved when it must not have:\nbefore=%+v\nafter =%+v", id, before, after) + } + return after +} + +func (s *selectionService) assertHeaderJSONB(t *testing.T, id, column string, want any, isNull bool) { + t.Helper() + if isNull { + if !s.mustWitnessBool(t, `SELECT `+column+` IS NULL FROM `+s.headersFQN()+` WHERE id = $1`, id) { + t.Fatalf("selection %s %s must be SQL NULL", id, column) + } + return + } + document, err := json.Marshal(want) + if err != nil { + t.Fatal(err) + } + if !s.mustWitnessBool(t, `SELECT `+column+` = $1::jsonb FROM `+s.headersFQN()+` WHERE id = $2`, string(document), id) { + w := s.witnessSelection(id) + var text string + switch column { + case "rule": + text = w.ruleText + case "source": + text = w.sourceText + default: + text = w.exclusionsText + } + t.Fatalf("selection %s %s does not equal the literal document\nwant: %s\ngot: %s", id, column, document, text) + } +} + +type witnessedMember struct { + selectionID, memberKey, project, generation, resourceType, id string + ordinal int64 + ctid string + exists bool +} + +func (w witnessedMember) same(o witnessedMember) bool { return w == o } + +func (s *selectionService) witnessMember(selectionID, memberKey string) witnessedMember { + var w witnessedMember + conn, err := s.witness.Acquire(s.ctx) + if err != nil { + s.fail("acquire member witness", err) + } + defer conn.Release() + err = conn.QueryRow(s.ctx, + `SELECT selection_id, member_key, project, generation, resource_type, id, ordinal, ctid::text + FROM `+s.membersFQN()+` WHERE selection_id = $1 AND member_key = $2`, selectionID, memberKey). + Scan(&w.selectionID, &w.memberKey, &w.project, &w.generation, &w.resourceType, &w.id, &w.ordinal, &w.ctid) + if errors.Is(err, pgx.ErrNoRows) { + return w + } + if err != nil { + s.fail("witness member "+memberKey, err) + } + w.exists = true + return w +} + +func (s *selectionService) memberCount(t *testing.T, selectionID string) int { + t.Helper() + return s.execCount(t, "SELECT count(*) FROM "+s.membersFQN()+" WHERE selection_id = $1", selectionID) +} + +func (s *selectionService) assertMemberAbsent(t *testing.T, selectionID, memberKey string) { + t.Helper() + if w := s.witnessMember(selectionID, memberKey); w.exists { + t.Fatalf("expected no member row %s, found %+v", memberKey, w) + } +} + +func pgstate05Hex(t *testing.T) string { return randomHex(t) } + +func selectionHeaderFixture(id, key string) explorer.SelectionRevision { + return explorer.SelectionRevision{ + ID: id, Project: pgstate05Project, Generation: "gen-a", ResourceType: "DocumentReference", + Rule: explorer.SelectionRule{Kind: explorer.SelectionRuleAllMatching, Filters: []explorer.SelectionFilter{ + {Column: "status", Op: "EQ", Value: "final"}, + {Column: "category", Op: "IN", Value: []any{"vitals", "labs"}}, + }}, + Source: explorer.SelectionSource{ + Kind: explorer.SelectionSourcePublished, Generation: "gen-a", + RevisionID: "rev-1", ReceiptID: "receipt-1", ExecutionID: "exec-1", + OutputID: "out-1", SchemaDigest: "schema-digest-1", + ResourceType: "DocumentReference", SourceIDColumn: "id", + }, + ScopeDigest: "scope-" + id, RuleDigest: "rule-" + id, + IdempotencyKey: key, CreatedAt: time.Now().UTC(), + } +} + +func selectionRef(id string) explorer.SelectionMember { + return explorer.SelectionMember{Ref: explorer.ResourceRef{ + Project: pgstate05Project, Generation: "gen-a", ResourceType: "DocumentReference", ID: id, + }} +} + +func goldenMemberKey(t *testing.T, selectionID string, ref explorer.ResourceRef) string { + t.Helper() + h := sha256.New() + for _, part := range []string{ref.Project, ref.Generation, ref.ResourceType, ref.ID} { + _, _ = h.Write([]byte{0}) + _, _ = h.Write([]byte(part)) + } + return selectionID + "_" + hex.EncodeToString(h.Sum(nil)) +} + +func membershipFacts(refs []explorer.SelectionMember) (string, int64, int64) { + hash := sha256.New() + var count, byteTotal int64 + type refKey struct{ p, g, t, i string } + seen := map[refKey]struct{}{} + unique := make([]explorer.ResourceRef, 0, len(refs)) + for _, m := range refs { + k := refKey{m.Ref.Project, m.Ref.Generation, m.Ref.ResourceType, m.Ref.ID} + if _, dup := seen[k]; dup { + continue + } + seen[k] = struct{}{} + unique = append(unique, m.Ref) + } + sort.Slice(unique, func(i, j int) bool { + li := []string{unique[i].Project, unique[i].Generation, unique[i].ResourceType, unique[i].ID} + lj := []string{unique[j].Project, unique[j].Generation, unique[j].ResourceType, unique[j].ID} + for k := range li { + if li[k] != lj[k] { + return li[k] < lj[k] + } + } + return false + }) + for _, ref := range unique { + explorer.WriteMembershipFrame(hash, ref) + for _, value := range []string{ref.Project, ref.Generation, ref.ResourceType, ref.ID} { + byteTotal += int64(8 + len(value)) + } + count++ + } + return hex.EncodeToString(hash.Sum(nil)), count, byteTotal +} + +func assertRevisionEquals(t *testing.T, label string, got *explorer.SelectionRevision, want explorer.SelectionRevision) { + t.Helper() + if got == nil { + t.Fatalf("%s = nil, want the stored revision", label) + } + want = want.Canonical() + if got.ID != want.ID || got.Project != want.Project || got.Generation != want.Generation || + got.ResourceType != want.ResourceType || got.ScopeDigest != want.ScopeDigest || + got.RuleDigest != want.RuleDigest || got.MembershipDigest != want.MembershipDigest || + got.MemberCount != want.MemberCount || got.MemberBytes != want.MemberBytes || + got.Complete != want.Complete || got.IdempotencyKey != want.IdempotencyKey { + t.Fatalf("%s = %+v, want %+v", label, got, want) + } + if !reflect.DeepEqual(got.Rule, want.Rule) { + t.Fatalf("%s rule = %+v, want %+v", label, got.Rule, want.Rule) + } + if got.Source != want.Source { + t.Fatalf("%s source = %+v, want %+v", label, got.Source, want.Source) + } + if !reflect.DeepEqual(got.Exclusions, want.Exclusions) { + t.Fatalf("%s exclusions = %+v, want %+v", label, got.Exclusions, want.Exclusions) + } + if !got.CreatedAt.Truncate(time.Microsecond).Equal(want.CreatedAt.Truncate(time.Microsecond)) { + t.Fatalf("%s createdAt: got %s want %s", label, got.CreatedAt, want.CreatedAt) + } + switch { + case want.CompletedAt == nil && got.CompletedAt != nil: + t.Fatalf("%s completedAt = %v, want missing", label, *got.CompletedAt) + case want.CompletedAt != nil && got.CompletedAt == nil: + t.Fatalf("%s completedAt missing, want %v", label, *want.CompletedAt) + case want.CompletedAt != nil && !got.CompletedAt.Equal(*want.CompletedAt): + t.Fatalf("%s completedAt: got %v want %v", label, *got.CompletedAt, *want.CompletedAt) + } +} + +var selectionHeaderDefaults = []struct{ column, literal string }{ + {"id", "'p5-header-1'"}, + {"project", "'" + pgstate05Project + "'"}, + {"generation", "'gen-a'"}, + {"resource_type", "'DocumentReference'"}, + {"rule_digest", "'rule-1'"}, + {"scope_digest", "'scope-1'"}, + {"idempotency_key", "'key-1'"}, + {"state", "'STAGING'"}, + {"writer_token", "'writer-1'"}, + {"writer_expires_at", "'2099-01-01T00:00:00Z'::timestamptz"}, + {"membership_digest", "''"}, + {"member_count", "0"}, + {"member_bytes", "0"}, + {"complete", "false"}, + {"created_at", "'2026-09-19T00:00:00Z'::timestamptz"}, + {"completed_at", "NULL"}, + {"rule", `'{"kind":"EXPLICIT"}'::jsonb`}, + {"source", `'{"kind":"EXPLICIT_REFS"}'::jsonb`}, + {"exclusions", "NULL"}, +} + +var selectionMemberDefaults = []struct{ column, literal string }{ + {"selection_id", "'p5-header-1'"}, + {"member_key", "'p5-header-1_" + pgstate05GoldenKeyFiles001 + "'"}, + {"project", "'" + pgstate05Project + "'"}, + {"generation", "'gen-a'"}, + {"resource_type", "'DocumentReference'"}, + {"id", "'files001'"}, + {"ordinal", "0"}, +} + +func selectionInsertSQL(columns []struct{ column, literal string }, overrides map[string]string) string { + cols := make([]string, 0, len(columns)) + vals := make([]string, 0, len(columns)) + for _, c := range columns { + lit := c.literal + if override, ok := overrides[c.column]; ok { + lit = override + } + cols = append(cols, c.column) + vals = append(vals, lit) + } + return "INSERT INTO %s (" + strings.Join(cols, ", ") + ") VALUES (" + strings.Join(vals, ", ") + ")" +} + +func (s *selectionService) headerInsertSQL(overrides map[string]string) string { + return fmt.Sprintf(selectionInsertSQL(selectionHeaderDefaults, overrides), s.headersFQN()) +} + +func (s *selectionService) memberInsertSQL(overrides map[string]string) string { + return fmt.Sprintf(selectionInsertSQL(selectionMemberDefaults, overrides), s.membersFQN()) +} + +func assertPgError(t *testing.T, label string, err error, wantState, wantConstraint string) { + t.Helper() + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + t.Fatalf("%s: err = %v, want SQLSTATE %s constraint %s", label, err, wantState, wantConstraint) + } + if pgErr.Code != wantState { + t.Fatalf("%s: SQLSTATE = %s (%s), want %s", label, pgErr.Code, pgErr.Message, wantState) + } + if wantConstraint != "" && pgErr.ConstraintName != wantConstraint { + t.Fatalf("%s: constraint = %q (SQLSTATE %s, message %s), want %q", + label, pgErr.ConstraintName, pgErr.Code, pgErr.Message, wantConstraint) + } +} + +func (s *selectionService) guardDropProbe(t *testing.T, trigger, table, guardFunction string, reseed func(), stmt string, args ...any) { + t.Helper() + tableFQN := s.qualified(table) + if !s.triggerPresent(t, trigger, table) { + t.Fatalf("guard trigger %s missing from %s before probe", trigger, table) + } + err := s.witnessExecErr(stmt, args...) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != pgstate05ImmutableCode { + t.Fatalf("guarded violating write on %s = %v, want SQLSTATE %s", table, s.scrub(err), pgstate05ImmutableCode) + } + s.mustWitnessExec(t, `DROP TRIGGER `+trigger+` ON `+tableFQN) + if s.triggerPresent(t, trigger, table) { + t.Fatalf("trigger %s still present after DROP", trigger) + } + conn, acquireErr := s.witness.Acquire(s.ctx) + if acquireErr != nil { + s.fail("acquire unguarded probe connection", acquireErr) + } + tag, execErr := conn.Exec(s.ctx, stmt, args...) + conn.Release() + if execErr != nil || tag.RowsAffected() != 1 { + t.Fatalf("same violating write did NOT run with the trigger dropped on %s: tag=%v err=%v; the guard was not what blocked it", + table, tag, s.scrub(execErr)) + } + if reseed != nil { + reseed() + } + s.mustWitnessExec(t, `CREATE TRIGGER `+trigger+` BEFORE UPDATE ON `+tableFQN+ + ` FOR EACH ROW EXECUTE FUNCTION `+s.qualified(guardFunction)+`()`) + if !s.triggerPresent(t, trigger, table) { + t.Fatalf("guard trigger %s not restored on %s", trigger, table) + } + err = s.witnessExecErr(stmt, args...) + if !errors.As(err, &pgErr) || pgErr.Code != pgstate05ImmutableCode { + t.Fatalf("violating write after trigger restore = %v, want SQLSTATE %s", s.scrub(err), pgstate05ImmutableCode) + } +} + +func (s *selectionService) triggerPresent(t *testing.T, trigger, table string) bool { + t.Helper() + var present bool + err := queryRowOn(s.ctx, s.witness, &present, + `SELECT EXISTS ( + SELECT 1 FROM pg_trigger tg + JOIN pg_class c ON c.oid = tg.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE tg.tgname = $1 AND c.relname = $2 AND n.nspname = $3 + AND NOT tg.tgisinternal)`, + trigger, table, s.schema) + if err != nil { + s.fail("trigger catalog witness "+trigger, err) + } + return present +} + +func (s *selectionService) guardThenCheckProbe(t *testing.T, trigger, table, guardFunction, stmt string, wantState, wantConstraint string, args ...any) { + t.Helper() + tableFQN := s.qualified(table) + if !s.triggerPresent(t, trigger, table) { + t.Fatalf("guard trigger %s missing from %s before probe", trigger, table) + } + assertPgError(t, "guarded write "+stmt, s.witnessExecErr(stmt, args...), pgstate05ImmutableCode, "") + s.mustWitnessExec(t, `DROP TRIGGER `+trigger+` ON `+tableFQN) + assertPgError(t, "un-guarded write "+stmt, s.witnessExecErr(stmt, args...), wantState, wantConstraint) + s.mustWitnessExec(t, `CREATE TRIGGER `+trigger+` BEFORE UPDATE ON `+tableFQN+ + ` FOR EACH ROW EXECUTE FUNCTION `+s.qualified(guardFunction)+`()`) + if !s.triggerPresent(t, trigger, table) { + t.Fatalf("guard trigger %s not restored on %s", trigger, table) + } + assertPgError(t, "restored-guard write "+stmt, s.witnessExecErr(stmt, args...), pgstate05ImmutableCode, "") +} + +func (s *selectionService) expireLease(t *testing.T, id string) { + t.Helper() + if n := s.witnessExecTag(t, + `UPDATE `+s.headersFQN()+` SET writer_expires_at = '2020-01-01T00:00:00Z'::timestamptz WHERE id = $1`, id); n != 1 { + t.Fatalf("expire lease %s affected %d rows, want 1", id, n) + } +} + +func (s *selectionService) beginConcurrently(storeA *Store, selectionA explorer.SelectionRevision, storeB *Store, selectionB explorer.SelectionRevision, tokenA, tokenB string) (*explorer.SelectionRevision, error, *explorer.SelectionRevision, error) { + var wg sync.WaitGroup + start := make(chan struct{}) + var revA, revB *explorer.SelectionRevision + var errA, errB error + wg.Add(2) + go func() { + defer wg.Done() + <-start + revA, errA = storeA.BeginSelection(s.ctx, selectionA, tokenA) + }() + go func() { + defer wg.Done() + <-start + revB, errB = storeB.BeginSelection(s.ctx, selectionB, tokenB) + }() + close(start) + wg.Wait() + return revA, errA, revB, errB +} + +func TestSelectionStorageAgainstPostgres(t *testing.T) { + s := newSelectionService(t) + suffix := pgstate05Hex(t) + selectionID := "selection-storage-" + suffix + writerToken := "writer-" + suffix + header := selectionHeaderFixture(selectionID, "key-"+suffix) + + started, err := s.storeA.BeginSelection(s.ctx, header, writerToken) + if err != nil { + s.fail("begin selection", err) + } + assertRevisionEquals(t, "begin", started, header) + w := s.witnessSelection(selectionID) + if !w.exists { + t.Fatal("begun selection has no stored row") + } + if w.state != "STAGING" || w.complete || w.writerToken != writerToken { + t.Fatalf("staged witness = %+v, want STAGING/uncomplete under %s", w, writerToken) + } + if w.exclusionsText != "" { + t.Fatalf("missing exclusions must be SQL NULL, got %q", w.exclusionsText) + } + s.assertHeaderJSONB(t, selectionID, "rule", header.Canonical().Rule, false) + s.assertHeaderJSONB(t, selectionID, "source", header.Canonical().Source, false) + s.assertHeaderJSONB(t, selectionID, "exclusions", nil, true) + + retry, err := s.storeB.BeginSelection(s.ctx, header, writerToken) + if err != nil { + s.fail("idempotent begin", err) + } + assertRevisionEquals(t, "retry", retry, header) + s.assertSelectionStable(t, selectionID, w) + + refs := []explorer.SelectionMember{selectionRef("files001"), selectionRef("files002"), selectionRef("files001")} + inserted, err := s.storeA.AppendSelectionMembers(s.ctx, selectionID, writerToken, refs) + if err != nil { + s.fail("append members", err) + } + if len(inserted) != 2 { + t.Fatalf("first append inserted %d, want 2 (deduped batch)", len(inserted)) + } + mk001 := goldenMemberKey(t, selectionID, refs[0].Ref) + mk002 := goldenMemberKey(t, selectionID, refs[1].Ref) + m001 := s.witnessMember(selectionID, mk001) + m002 := s.witnessMember(selectionID, mk002) + if !m001.exists || !m002.exists { + t.Fatalf("staged members missing: %s=%+v %s=%+v", mk001, m001, mk002, m002) + } + if !strings.HasSuffix(m001.memberKey, "_"+pgstate05GoldenKeyFiles001) || + !strings.HasSuffix(m002.memberKey, "_"+pgstate05GoldenKeyFiles002) { + t.Fatalf("member keys %q/%q do not carry the golden hex suffixes", m001.memberKey, m002.memberKey) + } + renewed := s.witnessSelection(selectionID) + if renewed.writerExpiresText == w.writerExpiresText || renewed.writerToken != writerToken { + t.Fatalf("append must renew the lease for the same writer: %q -> %q", w.writerExpiresText, renewed.writerExpiresText) + } + if _, err := s.storeB.AppendSelectionMembers(s.ctx, selectionID, "foreign-"+suffix, refs[:1]); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("foreign writer append = %v, want ErrSelectionConflict", err) + } + retryInserted, err := s.storeA.AppendSelectionMembers(s.ctx, selectionID, writerToken, refs) + if err != nil || len(retryInserted) != 0 { + t.Fatalf("retry append inserted %d err=%v, want 0 rows (idempotent)", len(retryInserted), err) + } + s.assertMemberAbsent(t, "no-such-selection", mk001) + + digest, count, byteTotal, err := s.storeA.DigestSelectionMembers(s.ctx, pgstate05Project, selectionID) + if err != nil { + s.fail("digest members", err) + } + wantDigest, wantCount, wantBytes := membershipFacts(refs) + if digest != wantDigest || count != wantCount || byteTotal != wantBytes { + t.Fatalf("digest = %s/%d/%d, want %s/%d/%d", digest, count, byteTotal, wantDigest, wantCount, wantBytes) + } + + completedAt := time.Date(2026, 9, 19, 0, 0, 0, 0, time.UTC) + completed, err := s.storeA.CompleteSelection(s.ctx, selectionID, writerToken, digest, count, byteTotal, completedAt) + if err != nil { + s.fail("complete selection", err) + } + if !completed.Complete || completed.MembershipDigest != digest { + t.Fatalf("completed = %+v, want complete with digest %s", completed, digest) + } + done := s.witnessSelection(selectionID) + if done.state != "COMPLETE" || !done.complete || done.membershipDigest != digest || + done.memberCount != count || done.memberBytes != byteTotal { + t.Fatalf("completed witness = %+v, want frozen facts %s/%d/%d", done, digest, count, byteTotal) + } + if done.completeAtText == "" { + t.Fatal("completed_at must be set on a COMPLETE row") + } + if _, err := s.storeA.AppendSelectionMembers(s.ctx, selectionID, writerToken, refs[:1]); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("append after complete = %v, want ErrSelectionConflict", err) + } + + again, err := s.storeB.BeginSelection(s.ctx, header, "post-complete-"+suffix) + if err != nil { + s.fail("begin after complete", err) + } + if !again.Complete || again.MembershipDigest != digest { + t.Fatalf("begin after complete = %+v, want the COMPLETE row", again) + } + s.assertSelectionStable(t, selectionID, done) + changed := header + changed.RuleDigest = "different-intent" + if _, err := s.storeB.BeginSelection(s.ctx, changed, "changed-"+suffix); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("changed-intent begin = %v, want ErrSelectionConflict", err) + } + s.assertSelectionStable(t, selectionID, done) + + activeID := "selection-active-" + suffix + active := selectionHeaderFixture(activeID, "active-key-"+suffix) + active.CreatedAt = time.Now().UTC().Add(time.Hour) + activeToken := "active-writer-" + suffix + if _, err := s.storeA.BeginSelection(s.ctx, active, activeToken); err != nil { + s.fail("begin active", err) + } + activeRow := s.witnessSelection(activeID) + if _, err := s.storeB.BeginSelection(s.ctx, active, "usurper-"+suffix); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("active-lease begin = %v, want ErrSelectionConflict (unexpired writer)", err) + } + s.assertSelectionStable(t, activeID, activeRow) + + s.expireLease(t, activeID) + recoveredToken := "recovered-writer-" + suffix + recovered, err := s.storeB.BeginSelection(s.ctx, active, recoveredToken) + if err != nil { + s.fail("expired recovery begin", err) + } + assertRevisionEquals(t, "expired recovery", recovered, active) + rebound := s.witnessSelection(activeID) + oldExpiry := parseWitnessTime(t, activeRow.writerExpiresText) + newExpiry := parseWitnessTime(t, rebound.writerExpiresText) + if rebound.writerToken != recoveredToken || !newExpiry.After(oldExpiry) { + t.Fatalf("expired recovery must rebind a fresh lease: %q -> %q", activeRow.writerExpiresText, rebound.writerExpiresText) + } + if rebound.id != activeRow.id || rebound.generation != activeRow.generation || + rebound.ruleDigest != activeRow.ruleDigest || rebound.scopeDigest != activeRow.scopeDigest || + rebound.idempotencyKey != activeRow.idempotencyKey || rebound.ruleText != activeRow.ruleText || + rebound.sourceText != activeRow.sourceText { + t.Fatalf("expired recovery tampered with identity: before=%+v after=%+v", activeRow, rebound) + } + + expiredMultiID := "selection-expired-multi-" + suffix + expiredMulti := selectionHeaderFixture(expiredMultiID, "expired-multi-key-"+suffix) + expiredWriter := "expired-writer-" + suffix + if _, err := s.storeA.BeginSelection(s.ctx, expiredMulti, expiredWriter); err != nil { + s.fail("begin expired multi", err) + } + expiredRefs := []explorer.SelectionMember{selectionRef("files001"), selectionRef("files003")} + if _, err := s.storeA.AppendSelectionMembers(s.ctx, expiredMultiID, expiredWriter, expiredRefs); err != nil { + s.fail("append expired multi", err) + } + if s.memberCount(t, expiredMultiID) != 2 { + t.Fatalf("expired multi members = %d, want 2", s.memberCount(t, expiredMultiID)) + } + s.expireLease(t, expiredMultiID) + recoveredWriter := "recovered-multi-" + suffix + if _, err := s.storeB.BeginSelection(s.ctx, expiredMulti, recoveredWriter); err != nil { + s.fail("expired multi recovery", err) + } + if n := s.memberCount(t, expiredMultiID); n != 0 { + t.Fatalf("expired multi rows survived cleanup: count=%d", n) + } + fresh := s.witnessSelection(expiredMultiID) + if fresh.writerToken != recoveredWriter || fresh.ctid == "" || !fresh.exists { + t.Fatalf("recovered revision is not a clean fresh insert: %+v", fresh) + } + + multiID := "selection-abort-multi-" + suffix + multi := selectionHeaderFixture(multiID, "abort-multi-key-"+suffix) + multiWriter := "abort-writer-" + suffix + if _, err := s.storeA.BeginSelection(s.ctx, multi, multiWriter); err != nil { + s.fail("begin abort multi", err) + } + if _, err := s.storeA.AppendSelectionMembers(s.ctx, multiID, multiWriter, expiredRefs); err != nil { + s.fail("append abort multi", err) + } + if err := s.storeB.AbortSelection(s.ctx, multiID, "not-the-writer"); err != nil { + s.fail("foreign abort", err) + } + multiRow := s.witnessSelection(multiID) + if !multiRow.exists || s.memberCount(t, multiID) != 2 { + t.Fatalf("foreign abort disturbed the live writer: row=%+v members=%d", multiRow, s.memberCount(t, multiID)) + } + if err := s.storeA.AbortSelection(s.ctx, multiID, multiWriter); err != nil { + s.fail("owner abort", err) + } + s.assertSelectionAbsent(t, multiID) + if n := s.memberCount(t, multiID); n != 0 { + t.Fatalf("owner abort left %d member rows", n) + } + + partialID := "selection-partial-" + suffix + partial := selectionHeaderFixture(partialID, "partial-key-"+suffix) + partialWriter := "partial-writer-" + suffix + if _, err := s.storeA.BeginSelection(s.ctx, partial, partialWriter); err != nil { + s.fail("begin partial", err) + } + mixedRefs := append([]explorer.SelectionMember{selectionRef("files001")}, + explorer.SelectionMember{Ref: explorer.ResourceRef{Project: "other", Generation: "gen-a", ResourceType: "DocumentReference", ID: "files003"}}) + if _, err := s.storeA.AppendSelectionMembers(s.ctx, partialID, partialWriter, mixedRefs); err == nil { + t.Fatal("mixed-project append unexpectedly succeeded") + } + if n := s.memberCount(t, partialID); n != 0 { + t.Fatalf("partial append leaked rows: count=%d", n) + } + + if _, err := s.storeA.GetSelection(s.ctx, "other-project", selectionID); !errors.Is(err, explorer.ErrSelectionNotFound) { + t.Fatalf("foreign-project GetSelection = %v, want ErrSelectionNotFound", err) + } + if _, err := s.storeA.GetSelection(s.ctx, " ", selectionID); !errors.Is(err, explorer.ErrSelectionNotFound) { + t.Fatalf("blank-project GetSelection = %v, want ErrSelectionNotFound", err) + } + if _, err := s.storeA.GetSelection(s.ctx, pgstate05Project, "selection-nope"); !errors.Is(err, explorer.ErrSelectionNotFound) { + t.Fatalf("missing GetSelection = %v, want ErrSelectionNotFound", err) + } + + visited := []string{} + last, err := s.storeA.VisitSelectionMembers(s.ctx, pgstate05Project, selectionID, "", 1, func(member explorer.SelectionMember) error { + visited = append(visited, member.MemberKey) + return nil + }) + if err != nil { + s.fail("visit members", err) + } + if len(visited) != 1 || visited[0] != mk001 || last != mk001 { + t.Fatalf("single visit call at pageSize 1 visited %v with cursor %q, want exactly [%s] with cursor %q", + visited, last, mk001, mk001) + } + visited = nil + last, err = s.storeA.VisitSelectionMembers(s.ctx, pgstate05Project, selectionID, last, 10, func(member explorer.SelectionMember) error { + visited = append(visited, member.MemberKey) + return nil + }) + if err != nil { + s.fail("visit members remainder", err) + } + if len(visited) != 1 || visited[0] != mk002 || last != mk002 { + t.Fatalf("resumed visit visited %v with cursor %q, want exactly [%s] with cursor %q", + visited, last, mk002, mk002) + } + visited = nil + last, err = s.storeA.VisitSelectionMembers(s.ctx, pgstate05Project, selectionID, last, 10, func(member explorer.SelectionMember) error { + visited = append(visited, member.MemberKey) + return nil + }) + if err != nil { + s.fail("visit members empty page", err) + } + if len(visited) != 0 || last != mk002 { + t.Fatalf("empty-page visit = %v with cursor %q, want zero visits and unchanged cursor %q", + visited, last, mk002) + } + if _, err := s.storeA.VisitSelectionMembers(s.ctx, pgstate05Project, activeID, "", 10, func(explorer.SelectionMember) error { + return nil + }); !errors.Is(err, explorer.ErrSelectionIncomplete) { + t.Fatalf("visit on STAGING = %v, want ErrSelectionIncomplete", err) + } + + for round := 0; round < pgstate05RaceRounds; round++ { + id := fmt.Sprintf("selection-race-converge-%d-%s", round, suffix) + key := fmt.Sprintf("race-converge-key-%d-%s", round, suffix) + raceHeader := selectionHeaderFixture(id, key) + sharedToken := "shared-writer-" + suffix + revA, errA, revB, errB := s.beginConcurrently(s.storeA, raceHeader, s.storeB, raceHeader, sharedToken, sharedToken) + if errA != nil || errB != nil { + t.Fatalf("round %d same-content begin convergence: A=%v B=%v (pids A=%d B=%d)", round, errA, errB, s.pidA, s.pidB) + } + if revA.ID != id || revB.ID != id { + t.Fatalf("round %d returned foreign revisions: %+v %+v", round, revA, revB) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.headersFQN()+" WHERE id = $1", id); n != 1 { + t.Fatalf("round %d header rows = %d, want exactly 1", round, n) + } + row := s.witnessSelection(id) + if row.writerToken != sharedToken || row.state != "STAGING" { + t.Fatalf("round %d converged row = %+v", round, row) + } + if err := s.storeB.AbortSelection(s.ctx, id, sharedToken); err != nil { + s.fail("race cleanup abort", err) + } + s.assertSelectionAbsent(t, id) + } + + for round := 0; round < pgstate05RaceRounds; round++ { + id := fmt.Sprintf("selection-race-owner-%d-%s", round, suffix) + key := fmt.Sprintf("race-owner-key-%d-%s", round, suffix) + raceHeader := selectionHeaderFixture(id, key) + revA, errA, revB, errB := s.beginConcurrently(s.storeA, raceHeader, s.storeB, raceHeader, "owner-A-"+suffix, "owner-B-"+suffix) + ownerWins := (errA == nil) != (errB == nil) + if !ownerWins { + t.Fatalf("round %d must produce exactly one writer: A=%v B=%v", round, errA, errB) + } + if errA != nil && !errors.Is(errA, explorer.ErrSelectionConflict) { + t.Fatalf("round %d writer A error = %v, want ErrSelectionConflict", round, errA) + } + if errB != nil && !errors.Is(errB, explorer.ErrSelectionConflict) { + t.Fatalf("round %d writer B error = %v, want ErrSelectionConflict", round, errB) + } + if errA == nil && (revA == nil || revA.ID != id) { + t.Fatalf("round %d winner A returned %+v, want id %s", round, revA, id) + } + if errB == nil && (revB == nil || revB.ID != id) { + t.Fatalf("round %d winner B returned %+v, want id %s", round, revB, id) + } + row := s.witnessSelection(id) + if !row.exists || row.writerToken == "" { + t.Fatalf("round %d winner row missing: %+v", round, row) + } + winnerToken := row.writerToken + if err := s.storeB.AbortSelection(s.ctx, id, winnerToken); err != nil { + s.fail("race owner cleanup", err) + } + s.assertSelectionAbsent(t, id) + } + + for round := 0; round < pgstate05RaceRounds; round++ { + id := fmt.Sprintf("selection-race-abort-%d-%s", round, suffix) + key := fmt.Sprintf("race-abort-key-%d-%s", round, suffix) + raceHeader := selectionHeaderFixture(id, key) + owner := "round-writer-" + suffix + if _, err := s.storeA.BeginSelection(s.ctx, raceHeader, owner); err != nil { + s.fail("race abort begin", err) + } + var appendErr, abortErr error + doneCh := make(chan struct{}, 2) + go func() { + _, appendErr = s.storeA.AppendSelectionMembers(s.ctx, id, owner, []explorer.SelectionMember{selectionRef("files001")}) + doneCh <- struct{}{} + }() + go func() { + abortErr = s.storeB.AbortSelection(s.ctx, id, "stale-"+suffix) + doneCh <- struct{}{} + }() + <-doneCh + <-doneCh + if appendErr != nil || abortErr != nil { + t.Fatalf("round %d abort race append=%v abort=%v", round, appendErr, abortErr) + } + if n := s.memberCount(t, id); n != 1 { + t.Fatalf("round %d stale abort removed or duplicated staged members: count=%d", round, n) + } + mk := goldenMemberKey(t, id, selectionRef("files001").Ref) + mw := s.witnessMember(id, mk) + if !mw.exists || mw.memberKey != mk { + t.Fatalf("round %d staged member drifted: %+v", round, mw) + } + dig, cnt, _, err := s.storeA.DigestSelectionMembers(s.ctx, pgstate05Project, id) + if err != nil || cnt != 1 { + t.Fatalf("round %d post-race digest = %s/%d err=%v", round, dig, cnt, err) + } + if _, err := s.storeA.CompleteSelection(s.ctx, id, owner, dig, cnt, 0, completedAt); err != nil { + t.Fatalf("round %d post-race complete = %v", round, err) + } + completedRow := s.witnessSelection(id) + if err := s.storeB.AbortSelection(s.ctx, id, "stale-"+suffix); err != nil { + s.fail("post-complete foreign abort", err) + } + s.assertSelectionStable(t, id, completedRow) + } +} + +func TestSelectionBeginConcurrentMismatchRefused(t *testing.T) { + s := newSelectionService(t) + suffix := pgstate05Hex(t) + + for round := 0; round < pgstate05RaceRounds; round++ { + key := fmt.Sprintf("mismatch-key-%d-%s", round, suffix) + headerA := selectionHeaderFixture(fmt.Sprintf("selection-mismatch-a-%d-%s", round, suffix), key) + headerB := headerA + headerB.ID = fmt.Sprintf("selection-mismatch-b-%d-%s", round, suffix) + headerB.ScopeDigest = headerB.ScopeDigest + "-b" + headerB.RuleDigest = headerB.RuleDigest + "-b" + revA, errA, revB, errB := s.beginConcurrently(s.storeA, headerA, s.storeB, headerB, "mw-"+suffix, "mw-"+suffix) + _ = revA + _ = revB + accepted := 0 + if errA == nil { + accepted++ + } + if errB == nil { + accepted++ + } + if accepted != 1 { + t.Fatalf("round %d same-key different-id: accepted %d of 2 (A=%v B=%v), want exactly 1", round, accepted, errA, errB) + } + if accepted == 1 && ((errA != nil && !errors.Is(errA, explorer.ErrSelectionConflict)) || + (errB != nil && !errors.Is(errB, explorer.ErrSelectionConflict))) { + t.Fatalf("round %d loser error not a typed conflict: A=%v B=%v", round, errA, errB) + } + winner := headerA + if errA != nil { + winner = headerB + } + w := s.witnessSelection(winner.ID) + if !w.exists { + t.Fatalf("round %d winner row missing", round) + } + s.assertSelectionStable(t, winner.ID, w) + loser := headerB + if errA != nil { + loser = headerA + } + s.assertSelectionAbsent(t, loser.ID) + if err := s.storeA.AbortSelection(s.ctx, winner.ID, "mw-"+suffix); err != nil { + s.fail("mismatch abort", err) + } + s.assertSelectionAbsent(t, winner.ID) + s.assertSelectionAbsent(t, loser.ID) + + id := fmt.Sprintf("selection-mismatch-id-%d-%s", round, suffix) + first := selectionHeaderFixture(id, fmt.Sprintf("mismatch-id-key-%d-%s", round, suffix)) + if _, err := s.storeA.BeginSelection(s.ctx, first, "imw-"+suffix); err != nil { + s.fail("mismatch-id seed", err) + } + base := s.witnessSelection(id) + changed := first + changed.RuleDigest = "tampered-intent" + if _, err := s.storeB.BeginSelection(s.ctx, changed, "imw-"+suffix); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("round %d same-id changed-content begin = %v, want conflict", round, err) + } + s.assertSelectionStable(t, id, base) + changedKey := first + changedKey.IdempotencyKey = "different-key" + if _, err := s.storeB.BeginSelection(s.ctx, changedKey, "imw-"+suffix); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("round %d same-id changed-key begin = %v, want conflict", round, err) + } + s.assertSelectionStable(t, id, base) + if err := s.storeA.AbortSelection(s.ctx, id, "imw-"+suffix); err != nil { + s.fail("mismatch-id abort", err) + } + s.assertSelectionAbsent(t, id) + } +} + +func TestSelectionWriterLeaseBoundaryIsDatabaseEnforced(t *testing.T) { + s := newSelectionService(t) + suffix := pgstate05Hex(t) + id := "selection-lease-boundary-" + suffix + header := selectionHeaderFixture(id, "lease-key-"+suffix) + if _, err := s.storeA.BeginSelection(s.ctx, header, "lease-writer-"+suffix); err != nil { + s.fail("seed begin", err) + } + base := s.witnessSelection(id) + + claimSQL := `UPDATE ` + s.headersFQN() + ` SET writer_token = $2, writer_expires_at = $3` + + ` WHERE id = $1 AND state = 'STAGING' AND writer_expires_at < $4 RETURNING writer_token` + + conn, err := s.witness.Acquire(s.ctx) + if err != nil { + s.fail("acquire claim witness", err) + } + future := time.Now().UTC().Add(time.Hour) + tag, err := conn.Exec(s.ctx, claimSQL, id, "usurper", future, time.Now().UTC()) + conn.Release() + if err != nil { + s.fail("boundary claim (unexpired)", err) + } + if tag.RowsAffected() != 0 { + t.Fatalf("unexpired lease claim affected %d rows, want 0", tag.RowsAffected()) + } + s.assertSelectionStable(t, id, base) + + if _, err := s.storeB.BeginSelection(s.ctx, header, "usurper"); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("adapter begin on live lease = %v, want ErrSelectionConflict", err) + } + s.assertSelectionStable(t, id, base) + + s.expireLease(t, id) + tag, err = func() (pgconn.CommandTag, error) { + c, aerr := s.witness.Acquire(s.ctx) + if aerr != nil { + return pgconn.CommandTag{}, aerr + } + defer c.Release() + return c.Exec(s.ctx, claimSQL, id, "direct-claimer", future.Add(time.Hour*2), time.Now().UTC()) + }() + if err != nil { + s.fail("boundary claim (expired)", err) + } + if tag.RowsAffected() != 1 { + t.Fatalf("expired lease claim affected %d rows, want 1", tag.RowsAffected()) + } + claimed := s.witnessSelection(id) + if claimed.writerToken != "direct-claimer" { + t.Fatalf("expired claim did not rebind the writer: %+v", claimed) + } + if claimed.id != base.id || claimed.generation != base.generation || + claimed.ruleDigest != base.ruleDigest || claimed.scopeDigest != base.scopeDigest || + claimed.idempotencyKey != base.idempotencyKey || claimed.ruleText != base.ruleText { + t.Fatalf("boundary claim tampered with identity columns: %+v", claimed) + } + + if _, err := s.storeB.BeginSelection(s.ctx, header, "second-usurper"); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("adapter begin after direct claim = %v, want ErrSelectionConflict", err) + } + own, err := s.storeA.BeginSelection(s.ctx, header, "direct-claimer") + if err != nil || own.ID != id { + t.Fatalf("owner begin under claimed lease = %+v err=%v", own, err) + } +} + +func TestSelectionDatabaseGuardsAreEnforcedAtDatabaseLevel(t *testing.T) { + s := newSelectionService(t) + suffix := pgstate05Hex(t) + + id := "selection-guard-" + suffix + header := selectionHeaderFixture(id, "guard-key-"+suffix) + if _, err := s.storeA.BeginSelection(s.ctx, header, "guard-writer"); err != nil { + s.fail("seed begin", err) + } + + var ( + origGeneration, origResourceType string + origRuleDigest, origScopeDigest string + origIdempotencyKey string + origCreatedAt time.Time + origRuleText, origSourceText string + origExclusionsText string + ) + func() { + conn, aerr := s.witness.Acquire(s.ctx) + if aerr != nil { + s.fail("acquire pristine capture", aerr) + } + defer conn.Release() + if err := conn.QueryRow(s.ctx, + `SELECT generation, resource_type, rule_digest, scope_digest, idempotency_key, created_at, + rule::text, source::text, COALESCE(exclusions::text, '') + FROM `+s.headersFQN()+` WHERE id = $1`, id). + Scan(&origGeneration, &origResourceType, &origRuleDigest, &origScopeDigest, + &origIdempotencyKey, &origCreatedAt, &origRuleText, &origSourceText, &origExclusionsText); err != nil { + s.fail("capture pristine columns", err) + } + }() + + type tamper struct { + column string + tampered any + restore any + cast string + } + var exclusionsRestore any + if origExclusionsText != "" { + exclusionsRestore = []byte(origExclusionsText) + } + identityTampers := []tamper{ + {"generation", "tampered", origGeneration, ""}, + {"resource_type", "Observation", origResourceType, ""}, + {"rule_digest", "tampered-digest", origRuleDigest, ""}, + {"scope_digest", "tampered-scope", origScopeDigest, ""}, + {"idempotency_key", "tampered-guard-key", origIdempotencyKey, ""}, + {"created_at", time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), origCreatedAt, "::timestamptz"}, + {"rule", []byte(`{"kind":"ALL_MATCHING"}`), []byte(origRuleText), "::jsonb"}, + {"source", []byte(`{"kind":"SELECTION_REVISION"}`), []byte(origSourceText), "::jsonb"}, + {"exclusions", []byte(`[{"project":"p"}]`), exclusionsRestore, "::jsonb"}, + } + for _, tm := range identityTampers { + stmt := `UPDATE ` + s.headersFQN() + ` SET ` + tm.column + ` = $2` + tm.cast + ` WHERE id = $1` + reseed := func(tm tamper) func() { + return func() { + if n := s.witnessExecTag(t, `UPDATE `+s.headersFQN()+` SET `+tm.column+` = $2`+tm.cast+` WHERE id = $1`, id, tm.restore); n != 1 { + t.Fatalf("reseed %s affected %d rows, want 1", tm.column, n) + } + } + }(tm) + s.guardDropProbe(t, "selection_revisions_immutable", SelectionRevisionsTable, "loom_selection_revision_immutable", reseed, stmt, id, tm.tampered) + } + + s.expireLease(t, id) + if n := s.witnessExecTag(t, + `UPDATE `+s.headersFQN()+` SET membership_digest = 'staging-is-live', writer_token = 'guard-writer' WHERE id = $1`, id); n != 1 { + t.Fatalf("staging-fact control write affected %d rows, want 1", n) + } + + digest, count, byteTotal, err := s.storeA.DigestSelectionMembers(s.ctx, pgstate05Project, id) + if err != nil { + s.fail("digest", err) + } + completedAt := time.Date(2026, 9, 19, 0, 0, 0, 0, time.UTC) + if _, err := s.storeA.CompleteSelection(s.ctx, id, "guard-writer", digest, count, byteTotal, completedAt); err != nil { + s.fail("complete", err) + } + frozenTampers := []tamper{ + {"membership_digest", "tampered", digest, ""}, + {"member_count", int64(99), count, ""}, + {"member_bytes", int64(99), byteTotal, ""}, + {"completed_at", time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), completedAt, "::timestamptz"}, + } + for _, tm := range frozenTampers { + stmt := `UPDATE ` + s.headersFQN() + ` SET ` + tm.column + ` = $2` + tm.cast + ` WHERE id = $1` + reseed := func(tm tamper) func() { + return func() { + if n := s.witnessExecTag(t, `UPDATE `+s.headersFQN()+` SET `+tm.column+` = $2`+tm.cast+` WHERE id = $1`, id, tm.restore); n != 1 { + t.Fatalf("reseed %s affected %d rows, want 1", tm.column, n) + } + } + }(tm) + s.guardDropProbe(t, "selection_revisions_immutable", SelectionRevisionsTable, "loom_selection_revision_immutable", reseed, stmt, id, tm.tampered) + } + for _, tamper := range []struct{ column, literal string }{ + {"complete", "false"}, + {"state", "'STAGING'"}, + } { + stmt := `UPDATE ` + s.headersFQN() + ` SET ` + tamper.column + ` = ` + tamper.literal + ` WHERE id = $1` + s.guardThenCheckProbe(t, "selection_revisions_immutable", SelectionRevisionsTable, "loom_selection_revision_immutable", + stmt, "23514", "selection_revisions_complete_matches_state", id) + } + + if _, err := s.storeA.AppendSelectionMembers(s.ctx, id, "guard-writer", []explorer.SelectionMember{selectionRef("files001")}); !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("append after complete = %v, want conflict (row is COMPLETE)", err) + } + memberSelID := "selection-guard-member-" + suffix + mk := goldenMemberKey(t, memberSelID, selectionRef("files001").Ref) + memberSel := selectionHeaderFixture(memberSelID, "guard-member-key-"+suffix) + if _, err := s.storeA.BeginSelection(s.ctx, memberSel, "member-writer"); err != nil { + s.fail("member probe begin", err) + } + if _, err := s.storeA.AppendSelectionMembers(s.ctx, memberSelID, "member-writer", []explorer.SelectionMember{selectionRef("files001")}); err != nil { + s.fail("member probe append", err) + } + mw := s.witnessMember(memberSelID, mk) + if !mw.exists { + t.Fatalf("member probe row missing: %+v", mw) + } + memberUpdate := `UPDATE ` + s.membersFQN() + ` SET ordinal = 42 WHERE selection_id = $1 AND member_key = $2` + reseed := func() { + if n := s.witnessExecTag(t, `UPDATE `+s.membersFQN()+` SET ordinal = 0 WHERE selection_id = $1 AND member_key = $2`, memberSelID, mk); n != 1 { + t.Fatalf("member probe reseed affected %d rows, want 1", n) + } + } + s.guardDropProbe(t, "selection_members_immutable", SelectionMembersTable, "loom_selection_member_immutable", reseed, memberUpdate, memberSelID, mk) + s.guardDropProbe(t, "selection_members_immutable", SelectionMembersTable, "loom_selection_member_immutable", func() { + if n := s.witnessExecTag(t, `UPDATE `+s.membersFQN()+` SET id = 'files001' WHERE selection_id = $1 AND member_key = $2`, memberSelID, mk); n != 1 { + t.Fatalf("member id reseed affected %d rows, want 1", n) + } + }, `UPDATE `+s.membersFQN()+` SET id = 'tampered' WHERE selection_id = $1 AND member_key = $2`, memberSelID, mk) + if n := s.witnessExecTag(t, `DELETE FROM `+s.membersFQN()+` WHERE selection_id = $1 AND member_key = $2`, memberSelID, mk); n != 1 { + t.Fatalf("lifecycle member DELETE affected %d rows, want 1 (guard must not over-block)", n) + } + if !s.triggerPresent(t, "selection_members_immutable", SelectionMembersTable) { + t.Fatal("member guard must remain attached after the legal lifecycle DELETE") + } +} + +func TestSelectionSchemaConstraintsRejectInvalidRows(t *testing.T) { + s := newSelectionService(t) + + if err := s.witnessExecErr(s.headerInsertSQL(nil)); err != nil { + s.fail("seed header", err) + } + if err := s.witnessExecErr(s.memberInsertSQL(nil)); err != nil { + s.fail("seed member", err) + } + + type negative struct { + name string + sql string + wantState string + wantConstraint string + } + cases := []negative{ + {"duplicate selection id", s.headerInsertSQL(map[string]string{"id": "'p5-header-1'", "idempotency_key": "'other-key-1'"}), "23505", "selection_revisions_id_pk"}, + {"duplicate project-scoped idempotency key", s.headerInsertSQL(map[string]string{"id": "'p5-header-dup-key'"}), "23505", "selection_revisions_project_key_ux"}, + {"state is a closed domain", s.headerInsertSQL(map[string]string{"id": "'p5-h-state'", "state": "'BOGUS'"}), "23514", "selection_revisions_state_known"}, + {"complete requires COMPLETE state", s.headerInsertSQL(map[string]string{"id": "'p5-h-mismatch'", "complete": "true", "membership_digest": "'d'", "completed_at": "'2026-09-19T00:00:00Z'::timestamptz"}), "23514", "selection_revisions_complete_matches_state"}, + {"COMPLETE without a membership digest", s.headerInsertSQL(map[string]string{"id": "'p5-h-nodigest'", "state": "'COMPLETE'", "complete": "true", "completed_at": "'2026-09-19T00:00:00Z'::timestamptz"}), "23514", "selection_revisions_complete_facts"}, + {"COMPLETE without completed_at", s.headerInsertSQL(map[string]string{"id": "'p5-h-nocomplete'", "state": "'COMPLETE'", "complete": "true", "membership_digest": "'d'"}), "23514", "selection_revisions_complete_facts"}, + {"negative member_count", s.headerInsertSQL(map[string]string{"id": "'p5-h-negcount'", "member_count": "-1"}), "23514", "selection_revisions_membership_non_negative"}, + {"negative member_bytes", s.headerInsertSQL(map[string]string{"id": "'p5-h-negbytes'", "member_bytes": "-1"}), "23514", "selection_revisions_membership_non_negative"}, + {"empty id", s.headerInsertSQL(map[string]string{"id": "''"}), "23514", "selection_revisions_identity_non_empty"}, + {"empty generation", s.headerInsertSQL(map[string]string{"id": "'p5-h-emptygen'", "generation": "''"}), "23514", "selection_revisions_identity_non_empty"}, + {"missing digests", s.headerInsertSQL(map[string]string{"id": "'p5-h-nodigs'", "rule_digest": "''"}), "23514", "selection_revisions_digests_present"}, + {"writer token is never empty", s.headerInsertSQL(map[string]string{"id": "'p5-h-nowriter'", "writer_token": "''"}), "23514", "selection_revisions_writer_present"}, + {"rule must be a JSON object", s.headerInsertSQL(map[string]string{"id": "'p5-h-scalerule'", "rule": "'5'::jsonb"}), "23514", "selection_revisions_rule_object"}, + {"source JSON null is not an object", s.headerInsertSQL(map[string]string{"id": "'p5-h-nullsrc'", "source": "'null'::jsonb"}), "23514", "selection_revisions_source_object"}, + {"exclusions must be a JSON array", s.headerInsertSQL(map[string]string{"id": "'p5-h-objexcl'", "exclusions": `'{"project":"p"}'::jsonb`}), "23514", "selection_revisions_exclusions_array"}, + {"member id must keep its derived shape", s.memberInsertSQL(map[string]string{"member_key": "'p5-header-1_not-hex'"}), "23514", "selection_members_key_shape"}, + {"duplicate member reference under a different key", s.memberInsertSQL(map[string]string{"member_key": "'p5-header-1_" + pgstate05GoldenKeyFiles002 + "'"}), "23505", "selection_members_ref_ux"}, + {"member reference columns are never empty", s.memberInsertSQL(map[string]string{"selection_id": "'p5-header-2'", "member_key": "'p5-header-2_" + pgstate05GoldenKeyFiles002 + "'", "id": "''"}), "23514", "selection_members_identity_non_empty"}, + } + for _, c := range cases { + assertPgError(t, c.name, s.witnessExecErr(c.sql), c.wantState, c.wantConstraint) + } + + if err := s.witnessExecErr(s.headerInsertSQL(map[string]string{ + "id": "'p5-h-complete-legal'", "idempotency_key": "'key-legal-complete'", + "state": "'COMPLETE'", "complete": "true", + "membership_digest": "'digest'", "member_count": "2", "member_bytes": "3", + "completed_at": "'2026-09-19T00:00:00Z'::timestamptz", + "rule": `'{"kind":"ALL_MATCHING","filters":[{"column":"a","op":"EQ","value":1}]}'::jsonb`, + })); err != nil { + t.Fatalf("control: legal COMPLETE header refused: %s", s.scrub(err)) + } + if err := s.witnessExecErr(s.headerInsertSQL(map[string]string{ + "id": "'p5-h-keyless-1'", "idempotency_key": "''", + })); err != nil { + t.Fatalf("control: first keyless header refused: %s", s.scrub(err)) + } + if err := s.witnessExecErr(s.headerInsertSQL(map[string]string{ + "id": "'p5-h-keyless-2'", "idempotency_key": "''", + })); err != nil { + t.Fatalf("control: second keyless header refused (partial unique must ignore empty keys): %s", s.scrub(err)) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.headersFQN()+" WHERE project = $1 AND idempotency_key = ''", pgstate05Project); n != 2 { + t.Fatalf("keyless headers = %d, want 2", n) + } + if err := s.witnessExecErr(s.memberInsertSQL(map[string]string{ + "selection_id": "'p5-header-2'", "member_key": "'p5-header-2_" + pgstate05GoldenKeyFiles003 + "'", "id": "'files003'", "ordinal": "7", + })); err != nil { + t.Fatalf("control: legal member row refused: %s", s.scrub(err)) + } +} + +func TestSelectionCrashWindowConvergence(t *testing.T) { + s := newSelectionService(t) + suffix := pgstate05Hex(t) + + id := "selection-crash-append-" + suffix + header := selectionHeaderFixture(id, "crash-append-key-"+suffix) + writer := "crash-writer-" + suffix + if _, err := s.storeA.BeginSelection(s.ctx, header, writer); err != nil { + s.fail("crash seed begin", err) + } + base := s.witnessSelection(id) + s.storeB.afterMemberInsert = func(context.Context, pgx.Tx) error { + return errors.New("injected crash after member insert") + } + refs := []explorer.SelectionMember{selectionRef("files001"), selectionRef("files002")} + if _, err := s.storeB.AppendSelectionMembers(s.ctx, id, writer, refs); err == nil { + t.Fatal("crashed append unexpectedly succeeded") + } else if !strings.Contains(err.Error(), "after member insert") { + t.Fatalf("crashed append error = %v, want the seam-classified wrap", err) + } + if n := s.memberCount(t, id); n != 0 { + t.Fatalf("crashed append leaked %d member rows", n) + } + s.assertSelectionStable(t, id, base) + s.storeB.afterMemberInsert = nil + inserted, err := s.storeB.AppendSelectionMembers(s.ctx, id, writer, refs) + if err != nil || len(inserted) != 2 { + t.Fatalf("post-crash retry append inserted %d err=%v, want 2", len(inserted), err) + } + + victimID := "selection-crash-cleanup-" + suffix + victim := selectionHeaderFixture(victimID, "crash-cleanup-key-"+suffix) + if _, err := s.storeA.BeginSelection(s.ctx, victim, "victim-writer-"+suffix); err != nil { + s.fail("victim begin", err) + } + if _, err := s.storeA.AppendSelectionMembers(s.ctx, victimID, "victim-writer-"+suffix, []explorer.SelectionMember{selectionRef("files002")}); err != nil { + s.fail("victim append", err) + } + s.expireLease(t, victimID) + victimRow := s.witnessSelection(victimID) + victimMemberKey := goldenMemberKey(t, victimID, selectionRef("files002").Ref) + victimMember := s.witnessMember(victimID, victimMemberKey) + s.storeA.beforeCleanupCommit = func(context.Context, pgx.Tx) error { + return errors.New("injected crash before cleanup commit") + } + if err := s.storeA.CleanupSelectionStaging(s.ctx, time.Now().UTC(), 100); err == nil { + t.Fatal("crashed cleanup unexpectedly succeeded") + } else if !strings.Contains(err.Error(), "before commit") { + t.Fatalf("crashed cleanup error = %v, want the seam-classified wrap", err) + } + s.assertSelectionStable(t, victimID, victimRow) + if after := s.witnessMember(victimID, victimMemberKey); !after.same(victimMember) { + t.Fatalf("crashed cleanup removed members: %+v", after) + } + s.storeA.beforeCleanupCommit = nil + if err := s.storeA.CleanupSelectionStaging(s.ctx, time.Now().UTC(), 100); err != nil { + s.fail("post-crash cleanup", err) + } + s.assertSelectionAbsent(t, victimID) + if n := s.memberCount(t, victimID); n != 0 { + t.Fatalf("post-crash cleanup left %d members", n) + } + + for round := 0; round < pgstate05RaceRounds; round++ { + raceID := fmt.Sprintf("selection-crash-race-%d-%s", round, suffix) + raceHeader := selectionHeaderFixture(raceID, fmt.Sprintf("crash-race-key-%d-%s", round, suffix)) + raceWriter := "race-writer-" + suffix + if _, err := s.storeA.BeginSelection(s.ctx, raceHeader, raceWriter); err != nil { + s.fail("race begin", err) + } + if _, err := s.storeA.AppendSelectionMembers(s.ctx, raceID, raceWriter, []explorer.SelectionMember{selectionRef("files003")}); err != nil { + s.fail("race append", err) + } + if round%2 == 0 { + s.expireLease(t, raceID) + } + var cleanupErr, appendErr error + done := make(chan struct{}, 2) + go func() { + cleanupErr = s.storeB.CleanupSelectionStaging(s.ctx, time.Now().UTC(), 100) + done <- struct{}{} + }() + go func() { + _, appendErr = s.storeA.AppendSelectionMembers(s.ctx, raceID, raceWriter, []explorer.SelectionMember{selectionRef("files001")}) + done <- struct{}{} + }() + <-done + <-done + if cleanupErr != nil { + s.fail("racing cleanup", cleanupErr) + } + row := s.witnessSelection(raceID) + if !row.exists { + if round%2 != 0 { + t.Fatalf("round %d: live (unexpired) writer was reaped", round) + } + if appendErr != nil && !errors.Is(appendErr, explorer.ErrSelectionNotFound) { + t.Fatalf("round %d append into reaped revision = %v, want typed not-found", round, appendErr) + } + if n := s.memberCount(t, raceID); n != 0 { + t.Fatalf("round %d reaped revision left %d members", round, n) + } + continue + } + if appendErr != nil { + t.Fatalf("round %d append into surviving revision = %v", round, appendErr) + } + if round%2 == 0 && row.writerToken != raceWriter { + t.Fatalf("round %d row survived with foreign writer %+v", round, row) + } + } +} + +func TestSelectionIdentityRereadAfterConflict(t *testing.T) { + s := newSelectionService(t) + suffix := pgstate05Hex(t) + id := "selection-identity-" + suffix + key := "identity-key-" + suffix + stored := selectionHeaderFixture(id, key) + writer := "identity-writer-" + suffix + if _, err := s.storeA.BeginSelection(s.ctx, stored, writer); err != nil { + s.fail("seed begin", err) + } + base := s.witnessSelection(id) + + variants := []struct { + name string + mutate func(*explorer.SelectionRevision) + }{ + {"generation", func(r *explorer.SelectionRevision) { r.Generation = "gen-b"; r.Source.Generation = "gen-b" }}, + {"resource type", func(r *explorer.SelectionRevision) { + r.ResourceType = "Observation" + r.Source.ResourceType = "Observation" + }}, + {"rule digest", func(r *explorer.SelectionRevision) { r.RuleDigest = "other-rule" }}, + {"scope digest", func(r *explorer.SelectionRevision) { r.ScopeDigest = "other-scope" }}, + {"idempotency key", func(r *explorer.SelectionRevision) { r.IdempotencyKey = "other-key" }}, + {"project", func(r *explorer.SelectionRevision) { r.Project = "pgstate05-other" }}, + } + for _, variant := range variants { + probe := stored + probe.Rule = stored.Rule + probe.Source = stored.Source + variant.mutate(&probe) + _, err := s.storeB.BeginSelection(s.ctx, probe, writer) + if !errors.Is(err, explorer.ErrSelectionConflict) { + t.Fatalf("%s tampering begin = %v, want ErrSelectionConflict", variant.name, err) + } + s.assertSelectionStable(t, id, base) + } + + reread, err := s.storeA.GetSelection(s.ctx, pgstate05Project, id) + if err != nil { + s.fail("reread", err) + } + assertRevisionEquals(t, "reread", reread, stored) + + digest, count, byteTotal, err := s.storeA.DigestSelectionMembers(s.ctx, pgstate05Project, id) + if err != nil { + s.fail("digest", err) + } + if _, err := s.storeA.CompleteSelection(s.ctx, id, writer, digest, count, byteTotal, time.Now().UTC()); err != nil { + s.fail("complete", err) + } + completeRow := s.witnessSelection(id) + done, err := s.storeB.BeginSelection(s.ctx, stored, writer) + if err != nil || !done.Complete || done.MembershipDigest != digest { + t.Fatalf("begin after complete = %+v err=%v", done, err) + } + s.assertSelectionStable(t, id, completeRow) +} + +func TestSelectionStoreValidationAndSQLContract(t *testing.T) { + if _, err := New(nil, "ctl"); err == nil { + t.Fatal("New(nil, schema) = nil error, want refusal") + } + if _, err := New(&pgstore.Client{}, "Bad Schema!"); err == nil { + t.Fatal("New(client, \"Bad Schema!\") = nil error, want schema refusal") + } + + sql := newSelectionSQL("ctl") + memberSQL := newSelectionMemberSQL("ctl") + headersFQN := `"ctl"."selection_revisions"` + membersFQN := `"ctl"."selection_members"` + for name, statement := range map[string]string{ + "beginInsert": sql.beginInsert, "headerByID": sql.headerByID, "headerScoped": sql.headerScoped, + "keyOwner": sql.keyOwner, "leaseClaim": sql.leaseClaim, "headerForAppend": sql.headerForAppend, + "memberInsert": memberSQL.memberInsert, "leaseRenew": memberSQL.leaseRenew, "completeCAS": memberSQL.completeCAS, + "memberDigestPage": memberSQL.memberDigestPage, "memberVisitPage": memberSQL.memberVisitPage, + "cleanupVictims": sql.cleanupVictims, "cleanupMembers": sql.cleanupMembers, + "cleanupHeaders": sql.cleanupHeaders, "abortMembers": sql.abortMembers, "abortHeader": sql.abortHeader, + } { + if !strings.Contains(statement, headersFQN) && !strings.Contains(statement, membersFQN) { + t.Fatalf("%s is not schema-qualified: %s", name, statement) + } + } + if got := strings.Count(sql.beginInsert, "$"); got != len(selectionWriteColumns) { + t.Fatalf("beginInsert placeholders = %d, want %d (arguments match by construction)", got, len(selectionWriteColumns)) + } + if !strings.Contains(sql.leaseClaim, "writer_expires_at < $4") { + t.Fatalf("leaseClaim must evaluate the boundary in the database: %s", sql.leaseClaim) + } + if !strings.Contains(memberSQL.completeCAS, "state = 'STAGING'") || !strings.Contains(memberSQL.completeCAS, "writer_token = $6") { + t.Fatalf("completeCAS must be a writer-conditional staging CAS: %s", memberSQL.completeCAS) + } + if !strings.Contains(sql.keyOwner, "$2 <> ''") { + t.Fatalf("keyOwner must never match an empty key: %s", sql.keyOwner) + } + if !strings.Contains(memberSQL.memberInsert, "ON CONFLICT (selection_id, member_key) DO NOTHING") { + t.Fatalf("memberInsert must be insert-once idempotent: %s", memberSQL.memberInsert) + } + if !strings.Contains(sql.cleanupVictims, "FOR UPDATE") { + t.Fatalf("cleanupVictims must lock its victims: %s", sql.cleanupVictims) + } + if got := strings.Count(memberSQL.memberInsert, "$"); got != len(memberColumns)+1 { + t.Fatalf("memberInsert placeholders = %d, want %d", got, len(memberColumns)+1) + } + + for _, tc := range []struct{ id, wantHex string }{ + {"files001", pgstate05GoldenKeyFiles001}, + {"files002", pgstate05GoldenKeyFiles002}, + {"files003", pgstate05GoldenKeyFiles003}, + } { + ref := explorer.ResourceRef{Project: pgstate05Project, Generation: "gen-a", ResourceType: "DocumentReference", ID: tc.id} + adapterKey := selectionMemberKey("selection-1", ref) + golden := goldenMemberKey(t, "selection-1", ref) + if adapterKey != golden { + t.Fatalf("selectionMemberKey(%s) = %s, independent derivation says %s", tc.id, adapterKey, golden) + } + if want := "selection-1_" + tc.wantHex; adapterKey != want { + t.Fatalf("selectionMemberKey(%s) = %s, pinned golden says %s", tc.id, adapterKey, want) + } + } + + ownerMigrations, err := OwnerMigrations() + if err != nil { + t.Fatal(err) + } + fixture, err := requireOwnerMigration(t, ownerMigrations, pgstate05FixtureVersion, pgstate05FixtureName) + if err != nil { + t.Fatal(err) + } + sqlBytes := []byte(fixture.SQL) + sum := sha256.Sum256(sqlBytes) + if fixture.Checksum != hex.EncodeToString(sum[:]) { + t.Fatalf("fixture migration = %+v, want version %d %s at the self-recomputed digest %s", + fixture, pgstate05FixtureVersion, pgstate05FixtureName, hex.EncodeToString(sum[:])) + } + for _, needle := range []string{"selection_revisions", "selection_members", "loom_selection_revision_immutable", "loom_selection_member_immutable", "45501"} { + if !strings.Contains(string(sqlBytes), needle) { + t.Fatalf("fixture DDL must mention %s", needle) + } + } +} + +func TestSelectionDigestPagingAndVisitWalk(t *testing.T) { + s := newSelectionService(t) + suffix := pgstate05Hex(t) + id := "selection-pages-" + suffix + header := selectionHeaderFixture(id, "pages-key-"+suffix) + writer := "pages-writer-" + suffix + if _, err := s.storeA.BeginSelection(s.ctx, header, writer); err != nil { + s.fail("begin", err) + } + const memberCount = 1001 + batch := make([]explorer.SelectionMember, 0, memberCount) + for i := 0; i < memberCount; i++ { + batch = append(batch, selectionRef(fmt.Sprintf("m%04d", i))) + } + inserted, err := s.storeA.AppendSelectionMembers(s.ctx, id, writer, batch) + if err != nil || len(inserted) != memberCount { + t.Fatalf("bulk append inserted %d err=%v, want %d", len(inserted), err, memberCount) + } + digest, count, byteTotal, err := s.storeA.DigestSelectionMembers(s.ctx, pgstate05Project, id) + if err != nil { + s.fail("paging digest", err) + } + wantDigest, wantCount, wantBytes := membershipFacts(batch) + if digest != wantDigest || count != wantCount || byteTotal != wantBytes { + t.Fatalf("paging digest crossed a page boundary incorrectly: got %s/%d/%d want %s/%d/%d", + digest, count, byteTotal, wantDigest, wantCount, wantBytes) + } + if n := s.memberCount(t, id); n != memberCount { + t.Fatalf("stored members = %d, want %d", n, memberCount) + } + if _, err := s.storeA.CompleteSelection(s.ctx, id, writer, digest, count, byteTotal, time.Now().UTC()); err != nil { + s.fail("complete", err) + } + seen := []string{} + cursor := "" + calls := 0 + for { + before := len(seen) + last, err := s.storeA.VisitSelectionMembers(s.ctx, pgstate05Project, id, cursor, 7, func(member explorer.SelectionMember) error { + seen = append(seen, member.MemberKey) + return nil + }) + if err != nil { + s.fail("visit walk", err) + } + calls++ + visitedCount := len(seen) - before + if visitedCount > 7 { + t.Fatalf("visit call %d visited %d members, want at most pageSize 7", calls, visitedCount) + } + if last == cursor { + if visitedCount != 0 { + t.Fatalf("visit call %d advanced no cursor yet visited %d members", calls, visitedCount) + } + break + } + if visitedCount == 0 { + t.Fatalf("visit call %d advanced the cursor without visiting any member", calls) + } + cursor = last + } + if wantCalls := memberCount/7 + 1; calls != wantCalls { + t.Fatalf("visit walk took %d calls, want %d (exact pageSize-7 pages plus the terminal empty page)", calls, wantCalls) + } + if len(seen) != memberCount { + t.Fatalf("visit walk collected %d, want %d", len(seen), memberCount) + } + if !sort.StringsAreSorted(seen) { + t.Fatal("visit walk is not in byte order") + } + digest2, count2, _, err := s.storeA.DigestSelectionMembers(s.ctx, pgstate05Project, id) + if err != nil || digest2 != digest || count2 != count { + t.Fatalf("complete digest reread = %s/%d err=%v, want %s/%d", digest2, count2, err, digest, count) + } +} + +func TestSelectionCleanupBoundaryAndLimit(t *testing.T) { + s := newSelectionService(t) + suffix := pgstate05Hex(t) + + type probe struct { + id string + created string + expires string + expired bool + members int + } + probes := []probe{ + {"p5-cleanup-1-" + suffix, "'2026-09-01T00:00:00Z'::timestamptz", "'2026-09-02T00:00:00Z'::timestamptz", true, 2}, + {"p5-cleanup-2-" + suffix, "'2026-09-02T00:00:00Z'::timestamptz", "'2026-09-03T00:00:00Z'::timestamptz", true, 1}, + {"p5-cleanup-3-" + suffix, "'2026-09-03T00:00:00Z'::timestamptz", "'2099-01-01T00:00:00Z'::timestamptz", false, 1}, + } + memberOf := map[string][]string{} + for i, p := range probes { + if err := s.witnessExecErr(s.headerInsertSQL(map[string]string{ + "id": "'" + p.id + "'", "idempotency_key": "'cleanup-key-" + fmt.Sprint(i) + "'", + "created_at": p.created, "writer_expires_at": p.expires, + })); err != nil { + s.fail("cleanup seed header "+p.id, err) + } + keys := []string{} + for m := 0; m < p.members; m++ { + ref := selectionRef(fmt.Sprintf("c%d-%04d", i, m)) + mk := goldenMemberKey(t, p.id, ref.Ref) + if err := s.witnessExecErr(s.memberInsertSQL(map[string]string{ + "selection_id": "'" + p.id + "'", "member_key": "'" + mk + "'", "id": "'" + ref.Ref.ID + "'", + })); err != nil { + s.fail("cleanup seed member "+mk, err) + } + keys = append(keys, mk) + } + memberOf[p.id] = keys + } + before := map[string]witnessedSelection{} + for _, p := range probes { + before[p.id] = s.witnessSelection(p.id) + } + + if err := s.storeA.CleanupSelectionStaging(s.ctx, time.Now().UTC(), 2); err != nil { + s.fail("bounded cleanup", err) + } + s.assertSelectionAbsent(t, probes[0].id) + s.assertSelectionAbsent(t, probes[1].id) + for _, p := range probes[:2] { + if n := s.memberCount(t, p.id); n != 0 { + t.Fatalf("reaped %s left %d members", p.id, n) + } + } + survivor := s.assertSelectionStable(t, probes[2].id, before[probes[2].id]) + if survivor.writerToken != "writer-1" { + t.Fatalf("survivor drifted: %+v", survivor) + } + for _, mk := range memberOf[probes[2].id] { + if !s.witnessMember(probes[2].id, mk).exists { + t.Fatalf("cleanup removed an unexpired writer's member %s", mk) + } + } + + if err := s.storeB.CleanupSelectionStaging(s.ctx, time.Now().UTC(), 0); err != nil { + s.fail("default-limit cleanup", err) + } + s.assertSelectionStable(t, probes[2].id, before[probes[2].id]) + + abandoned := "selection-cleanup-retry-" + suffix + abandonedKey := "cleanup-retry-key-" + suffix + abandonedHeader := selectionHeaderFixture(abandoned, abandonedKey) + if _, err := s.storeA.BeginSelection(s.ctx, abandonedHeader, "abandoned-writer"); err != nil { + s.fail("abandoned begin", err) + } + s.expireLease(t, abandoned) + retry, err := s.storeB.BeginSelection(s.ctx, abandonedHeader, "retry-writer") + if err != nil { + s.fail("abandoned retry begin", err) + } + assertRevisionEquals(t, "abandoned retry", retry, abandonedHeader) + retryRow := s.witnessSelection(abandoned) + if retryRow.writerToken != "retry-writer" || retryRow.memberCount != 0 { + t.Fatalf("abandoned retry row = %+v, want clean retry-writer ownership", retryRow) + } +} + +func TestSelectionStoresFailClosedWithoutTheirTables(t *testing.T) { + s := newSelectionService(t) + bare := fmt.Sprintf("pgstate05_it_%d_%s", time.Now().UnixNano(), pgstate05Hex(t)) + if !pgstate05NameRe.MatchString(bare) { + t.Fatalf("bare schema name %q does not match the pattern", bare) + } + if err := execOn(s.ctx, s.witness, `CREATE SCHEMA "`+bare+`"`); err != nil { + s.fail("create bare schema", err) + } + pgstate05RegisterExtraSchema(bare) + t.Cleanup(func() { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelCleanup() + if err := execOn(cleanupCtx, s.witness, `DROP SCHEMA IF EXISTS "`+bare+`" CASCADE`); err != nil { + t.Errorf("drop bare schema: %s", s.scrub(err)) + } + }) + bareStore, err := New(s.storeA.client, bare) + if err != nil { + s.fail("build bare store", err) + } + header := selectionHeaderFixture("selection-bare", "bare-key") + _, err = bareStore.BeginSelection(s.ctx, header, "w") + assertPgError(t, "begin on unmigrated schema", err, "42P01", "") + _, err = bareStore.AppendSelectionMembers(s.ctx, "selection-bare", "w", []explorer.SelectionMember{selectionRef("files001")}) + assertPgError(t, "append on unmigrated schema", err, "42P01", "") + _, err = bareStore.GetSelection(s.ctx, pgstate05Project, "selection-bare") + assertPgError(t, "get on unmigrated schema", err, "42P01", "") + _, _, _, err = bareStore.DigestSelectionMembers(s.ctx, pgstate05Project, "selection-bare") + assertPgError(t, "digest on unmigrated schema", err, "42P01", "") + if err := bareStore.CleanupSelectionStaging(s.ctx, time.Now().UTC(), 10); err == nil { + t.Fatal("cleanup on unmigrated schema succeeded, want 42P01") + } + if _, err := bareStore.CompleteSelection(s.ctx, "selection-bare", "w", "d", 1, 1, time.Now().UTC()); err == nil { + t.Fatal("complete on unmigrated schema succeeded, want an error") + } +} + +func TestZZZeroLeftoverPgstate05Fixtures(t *testing.T) { + raw := testPostgresURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %v", err) + } + defer admin.Close() + + pgstate05OwnedMu.Lock() + databases := make(map[string]string, len(pgstate05OwnedRes)) + for db, schema := range pgstate05OwnedRes { + databases[db] = schema + } + extra := append([]string{}, pgstate05ExtraSchemas...) + pgstate05OwnedMu.Unlock() + + for db, schema := range databases { + var count int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname = $1`, db).Scan(&count); err != nil { + t.Fatalf("audit database %s: %v", db, err) + } + if count != 0 { + t.Fatalf("isolated database %s (owning schema %s) was left behind", db, schema) + } + } + for _, schema := range extra { + var present bool + if err := admin.QueryRow(ctx, `SELECT to_regnamespace($1) IS NOT NULL`, schema).Scan(&present); err != nil { + t.Fatalf("audit extra schema %s: %v", schema, err) + } + if present { + t.Fatalf("bare probe schema %s was left behind in the service database", schema) + } + } +} diff --git a/internal/backends/postgres/internal/publication/leases.go b/internal/backends/postgres/internal/publication/leases.go new file mode 100644 index 000000000..da9e1d6b2 --- /dev/null +++ b/internal/backends/postgres/internal/publication/leases.go @@ -0,0 +1,200 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/calypr/loom/internal/dataframe/publication" +) + +const ( + ExecutionCheckpointsTable = "publication_execution_checkpoints" + LeaseGrantsTable = "publication_lease_grants" +) + +var ErrInvalidLeaseKey = errors.New("publication postgres: lease key and owner are required") + +type CheckpointRecord struct { + Key string + Sequence int64 + OwnerID string + Generation int64 + CheckpointedAt time.Time + Payload []byte +} + +type FencedLeases struct { + db DB + leases string + grants string + checkpoints string +} + +func NewLeaseKeeper(pool DB, schema string) (*FencedLeases, error) { + if pool == nil { + return nil, errors.New("publication postgres: nil pool") + } + if !pgstore.ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", ErrInvalidSchema, schema) + } + return &FencedLeases{ + db: pool, + leases: pgx.Identifier{schema, BundleLeasesTable}.Sanitize(), + grants: pgx.Identifier{schema, LeaseGrantsTable}.Sanitize(), + checkpoints: pgx.Identifier{schema, ExecutionCheckpointsTable}.Sanitize(), + }, nil +} + +func (k *FencedLeases) AcquireLease(ctx context.Context, key, owner string, expires time.Time) (int64, bool, error) { + if key == "" || owner == "" { + return 0, false, fmt.Errorf("%w: got key %q owner %q", ErrInvalidLeaseKey, key, owner) + } + now := time.Now().UTC() + tx, err := k.db.Begin(ctx) + if err != nil { + return 0, false, fmt.Errorf("publication postgres: begin acquire fenced lease: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var highWater int64 + if err := tx.QueryRow(ctx, + "INSERT INTO "+k.grants+" (key, last_generation) VALUES ($1, 1)"+ + " ON CONFLICT (key) DO UPDATE SET last_generation = "+LeaseGrantsTable+".last_generation + 1"+ + " RETURNING last_generation", + key).Scan(&highWater); err != nil { + return 0, false, fmt.Errorf("publication postgres: bump fenced lease high-water: %w", err) + } + + var generation int64 + err = tx.QueryRow(ctx, + "INSERT INTO "+k.leases+" (key, owner_id, expires_at, acquired_at, epoch)"+ + " VALUES ($1, $2, $3, $4, $5)"+ + " ON CONFLICT (key) DO UPDATE SET owner_id = $2, expires_at = $3, acquired_at = $4,"+ + " epoch = $5"+ + " WHERE "+BundleLeasesTable+".expires_at < $4 OR "+BundleLeasesTable+".owner_id = $2"+ + " RETURNING epoch", + key, owner, expires, now, highWater).Scan(&generation) + if errors.Is(err, pgx.ErrNoRows) { + return 0, false, nil + } + if err != nil { + return 0, false, fmt.Errorf("publication postgres: acquire fenced lease: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return 0, false, fmt.Errorf("publication postgres: commit acquire fenced lease: %w", err) + } + return generation, true, nil +} + +func (k *FencedLeases) RenewLease(ctx context.Context, key, owner string, generation int64, expires time.Time) (bool, error) { + if key == "" || owner == "" || generation < 1 { + return false, nil + } + tag, err := k.db.Exec(ctx, + "UPDATE "+k.leases+" SET expires_at = $4"+ + " WHERE key = $1 AND owner_id = $2 AND epoch = $3 AND expires_at >= $5", + key, owner, generation, expires, time.Now().UTC()) + if err != nil { + return false, fmt.Errorf("publication postgres: renew fenced lease: %w", err) + } + return tag.RowsAffected() == 1, nil +} + +func (k *FencedLeases) ExpireLease(ctx context.Context, key, owner string, generation int64) (bool, error) { + if key == "" || owner == "" || generation < 1 { + return false, nil + } + retireAt := time.Now().UTC().Add(-time.Microsecond) + tag, err := k.db.Exec(ctx, + "UPDATE "+k.leases+" SET expires_at = $4"+ + " WHERE key = $1 AND owner_id = $2 AND epoch = $3 AND expires_at >= $4", + key, owner, generation, retireAt) + if err != nil { + return false, fmt.Errorf("publication postgres: expire fenced lease: %w", err) + } + return tag.RowsAffected() == 1, nil +} + +func (k *FencedLeases) ReleaseLease(ctx context.Context, key, owner string, generation int64) (bool, error) { + if key == "" || owner == "" || generation < 1 { + return false, nil + } + tag, err := k.db.Exec(ctx, + "DELETE FROM "+k.leases+" WHERE key = $1 AND owner_id = $2 AND epoch = $3", + key, owner, generation) + if err != nil { + return false, fmt.Errorf("publication postgres: release fenced lease: %w", err) + } + return tag.RowsAffected() == 1, nil +} + +func jsonKind(encoded []byte) string { + switch encoded[0] { + case '[': + return "a JSON array" + case '"': + return "a JSON string" + case 't', 'f': + return "a JSON bool" + case 'n': + return "JSON null" + default: + return "a JSON number" + } +} + +func (k *FencedLeases) Checkpoint(ctx context.Context, key, owner string, generation int64, payload any) (int64, error) { + if key == "" || owner == "" || generation < 1 { + return 0, publication.ErrBundleLeaseLost + } + if payload == nil { + return 0, errors.New("publication postgres: checkpoint payload is required") + } + encoded, err := json.Marshal(payload) + if err != nil { + return 0, fmt.Errorf("publication postgres: marshal checkpoint payload: %w", err) + } + if len(encoded) == 0 || encoded[0] != '{' { + return 0, fmt.Errorf("publication postgres: checkpoint payload must be a JSON object, got %s", jsonKind(encoded)) + } + now := time.Now().UTC() + var sequence int64 + err = k.db.QueryRow(ctx, + "WITH next AS (SELECT COALESCE(MAX(sequence), 0) + 1 AS seq FROM "+k.checkpoints+" WHERE key = $1)"+ + " INSERT INTO "+k.checkpoints+" (key, sequence, owner_id, generation, checkpointed_at, payload)"+ + " SELECT $1, next.seq, $2, $3, $4, $5::jsonb FROM next"+ + " WHERE EXISTS (SELECT 1 FROM "+k.leases+" l"+ + " WHERE l.key = $1 AND l.owner_id = $2 AND l.epoch = $3 AND l.expires_at >= $4)"+ + " RETURNING sequence", + key, owner, generation, now, + string(encoded)).Scan(&sequence) + if errors.Is(err, pgx.ErrNoRows) { + return 0, publication.ErrBundleLeaseLost + } + if err != nil { + return 0, fmt.Errorf("publication postgres: checkpoint: %w", err) + } + return sequence, nil +} + +func (k *FencedLeases) LatestCheckpoint(ctx context.Context, key string) (CheckpointRecord, bool, error) { + var record CheckpointRecord + err := k.db.QueryRow(ctx, + "SELECT key, sequence, owner_id, generation, checkpointed_at, payload"+ + " FROM "+k.checkpoints+" WHERE key = $1 ORDER BY sequence DESC LIMIT 1", + key).Scan(&record.Key, &record.Sequence, &record.OwnerID, &record.Generation, + &record.CheckpointedAt, &record.Payload) + if errors.Is(err, pgx.ErrNoRows) { + return CheckpointRecord{}, false, nil + } + if err != nil { + return CheckpointRecord{}, false, fmt.Errorf("publication postgres: latest checkpoint: %w", err) + } + return record, true, nil +} diff --git a/internal/backends/postgres/internal/publication/leases_test.go b/internal/backends/postgres/internal/publication/leases_test.go new file mode 100644 index 000000000..8ee1c3336 --- /dev/null +++ b/internal/backends/postgres/internal/publication/leases_test.go @@ -0,0 +1,873 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/dataframe/publication" +) + +var pgleases01ResourceNamePattern = regexp.MustCompile(`^pgleases01_it_[0-9]+_[0-9a-f]+$`) + +type leaseService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + isoB *pgxpool.Pool + schema string + registry *Registry + keeperA *FencedLeases + keeperB *FencedLeases + scrub func(any) string + fail func(step string, err error) +} + +func newLeaseService(t *testing.T) *leaseService { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgleases01_it_%d_%s", time.Now().UnixNano(), randomHex()) + if !pgleases01ResourceNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + poolA, err := newOwnedPool(ctx, dsn) + if err != nil { + fail("connect pool A to isolated database", err) + } + t.Cleanup(poolA.Close) + poolB, err := newOwnedPool(ctx, dsn) + if err != nil { + fail("connect pool B to isolated database", err) + } + t.Cleanup(poolB.Close) + + schema := fmt.Sprintf("pgleases01_it_%d_%s", time.Now().UnixNano(), randomHex()) + if !pgleases01ResourceNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := poolA.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := poolA.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + s := &leaseService{ctx: ctx, admin: admin, iso: poolA, isoB: poolB, schema: schema, scrub: scrub, fail: fail} + s.migrate(t) + registry, err := New(poolA, schema) + if err != nil { + fail("build registry", err) + } + s.registry = registry + keeperA, err := NewLeaseKeeper(poolA, schema) + if err != nil { + fail("build keeper A", err) + } + s.keeperA = keeperA + keeperB, err := NewLeaseKeeper(poolB, schema) + if err != nil { + fail("build keeper B", err) + } + s.keeperB = keeperB + return s +} + +func newOwnedPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) { + config, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, err + } + config.MaxConns = 1 + return pgxpool.NewWithConfig(ctx, config) +} + +func (s *leaseService) migrate(t *testing.T) { + t.Helper() + history := mustOwnerHistorySubset(t, 1, 4, 6) + runner, err := storepostgres.NewMigrationRunner(s.iso, s.schema, history) + if err != nil { + s.fail("new migration runner", err) + } + result, err := runner.Migrate(s.ctx) + if err != nil { + s.fail("migrate checkpoint history", err) + } + if result.LatestVersion != 6 { + t.Fatalf("latest migration version = %d, want 6", result.LatestVersion) + } + wantLedger := []storepostgres.LedgerRecord{ + {Version: 1, Name: "ledger", Checksum: "db70f7758495f7557b3a694149b80ed6e5d4cb74ea5aa7d59a1a2eaa46fc9cde"}, + {Version: 4, Name: "publication_bundle_catalog", Checksum: "adcb4683ffd1611b9c68e7a49aaa4b8670d106d65df927cbb8e11866be224943"}, + {Version: 6, Name: "publication_lease_checkpoints", Checksum: "5a2fc578d1b24ee048275b060856d23b7e1a35ff251519bfb46bbc4c1e8a917c"}, + } + if len(result.Applied) != len(wantLedger) { + t.Fatalf("applied history = %+v, want %+v", result.Applied, wantLedger) + } + for i, want := range wantLedger { + got := result.Applied[i] + if got.Version != want.Version || got.Name != want.Name || got.Checksum != want.Checksum { + t.Fatalf("applied[%d] = {Version:%d Name:%q Checksum:%q}, want {Version:%d Name:%q Checksum:%q}", + i, got.Version, got.Name, got.Checksum, want.Version, want.Name, want.Checksum) + } + } +} + +func (s *leaseService) mustExec(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) { + t.Helper() + if _, err := pool.Exec(s.ctx, sql, args...); err != nil { + s.fail("exec "+sql, err) + } +} + +func (s *leaseService) execCount(t *testing.T, sql string, args ...any) int { + t.Helper() + var n int + if err := s.isoB.QueryRow(s.ctx, sql, args...).Scan(&n); err != nil { + s.fail("count "+sql, err) + } + return n +} + +func (s *leaseService) mustWitnessText(t *testing.T, sql string, args ...any) string { + t.Helper() + var value string + if err := s.isoB.QueryRow(s.ctx, sql, args...).Scan(&value); err != nil { + s.fail("witness "+sql, err) + } + return value +} + +func (s *leaseService) mustWitnessInt64(t *testing.T, sql string, args ...any) int64 { + t.Helper() + var value int64 + if err := s.isoB.QueryRow(s.ctx, sql, args...).Scan(&value); err != nil { + s.fail("witness "+sql, err) + } + return value +} + +func (s *leaseService) mustWitnessTime(t *testing.T, sql string, args ...any) time.Time { + t.Helper() + var value time.Time + if err := s.isoB.QueryRow(s.ctx, sql, args...).Scan(&value); err != nil { + s.fail("witness "+sql, err) + } + return value +} + +func (s *leaseService) qualified(table string) string { + return pgx.Identifier{s.schema, table}.Sanitize() +} + +func (s *leaseService) acquire(t *testing.T, k *FencedLeases, key, owner string, expires time.Time) int64 { + t.Helper() + generation, held, err := k.AcquireLease(s.ctx, key, owner, expires) + if err != nil || !held { + s.fail(fmt.Sprintf("acquire lease %s/%s", key, owner), fmt.Errorf("held=%v generation=%d err=%w", held, generation, err)) + } + return generation +} + +func (s *leaseService) checkpoint(t *testing.T, k *FencedLeases, key, owner string, generation int64, payload any) int64 { + t.Helper() + sequence, err := k.Checkpoint(s.ctx, key, owner, generation, payload) + if err != nil { + s.fail(fmt.Sprintf("checkpoint %s/%s/gen%d", key, owner, generation), err) + } + return sequence +} + +func (s *leaseService) leaseRow(t *testing.T, key string) (string, int64) { + t.Helper() + var owner string + var epoch int64 + if err := s.isoB.QueryRow(s.ctx, + "SELECT owner_id, epoch FROM "+s.qualified(BundleLeasesTable)+" WHERE key = $1", key). + Scan(&owner, &epoch); err != nil { + s.fail("witness lease "+key, err) + } + return owner, epoch +} + +func (s *leaseService) checkpointCount(t *testing.T, key string) int { + t.Helper() + return s.execCount(t, "SELECT count(*) FROM "+s.qualified(ExecutionCheckpointsTable)+" WHERE key = $1", key) +} + +func futureExpiry() time.Time { return time.Now().UTC().Add(time.Hour) } +func pastExpiry() time.Time { return time.Now().UTC().Add(-time.Hour) } + +func TestNewLeaseKeeperRejectsNilPoolAndUnsafeSchemas(t *testing.T) { + if _, err := NewLeaseKeeper(nil, "ok_schema"); err == nil { + t.Fatal("NewLeaseKeeper(nil pool) = nil error, want rejection") + } + for _, schema := range []string{"", "Public", "a;drop", "0starts_with_digit", "has space", "tab\tname", strings.Repeat("a", 64)} { + if _, err := NewLeaseKeeper(neverUsedPool{}, schema); !errors.Is(err, ErrInvalidSchema) { + t.Fatalf("NewLeaseKeeper(schema %q) = %v, want ErrInvalidSchema", schema, err) + } + } +} + +func TestCheckpointWriteSQLComparesOwnerAndUnexpiredLease(t *testing.T) { + s := newLeaseService(t) + + generation := s.acquire(t, s.keeperA, "exec-cp", "writer-a", futureExpiry()) + if generation != 1 { + t.Fatalf("first acquire generation = %d, want 1", generation) + } + sequence := s.checkpoint(t, s.keeperA, "exec-cp", "writer-a", 1, map[string]any{"stage": "queued"}) + if sequence != 1 { + t.Fatalf("first checkpoint sequence = %d, want 1", sequence) + } + + var ( + gotKey, gotOwner string + gotSequence, gotGeneration int64 + ) + if err := s.isoB.QueryRow(s.ctx, + "SELECT key, sequence, owner_id, generation FROM "+s.qualified(ExecutionCheckpointsTable)+ + " WHERE key = $1 AND sequence = $2", "exec-cp", 1). + Scan(&gotKey, &gotSequence, &gotOwner, &gotGeneration); err != nil { + s.fail("witness checkpoint row", err) + } + if gotKey != "exec-cp" || gotSequence != 1 || gotOwner != "writer-a" || gotGeneration != 1 { + t.Fatalf("checkpoint row = {%s %d %s %d}, want {exec-cp 1 writer-a 1}", + gotKey, gotSequence, gotOwner, gotGeneration) + } + if stage := s.mustWitnessText(t, + "SELECT payload->>'stage' FROM "+s.qualified(ExecutionCheckpointsTable)+" WHERE key = $1 AND sequence = $2", + "exec-cp", 1); stage != "queued" { + t.Fatalf("stored payload stage = %q, want queued", stage) + } + + if _, err := s.keeperA.Checkpoint(s.ctx, "exec-cp", "writer-b", 1, map[string]any{"stage": "steal"}); !errors.Is(err, publication.ErrBundleLeaseLost) { + t.Fatalf("foreign-owner checkpoint = %v, want ErrBundleLeaseLost", err) + } + if _, err := s.keeperA.Checkpoint(s.ctx, "exec-cp", "writer-a", 2, map[string]any{"stage": "stale"}); !errors.Is(err, publication.ErrBundleLeaseLost) { + t.Fatalf("stale-generation checkpoint = %v, want ErrBundleLeaseLost", err) + } + if _, err := s.keeperA.Checkpoint(s.ctx, "exec-cp", "writer-a", 0, map[string]any{"stage": "zero"}); !errors.Is(err, publication.ErrBundleLeaseLost) { + t.Fatalf("zero-generation checkpoint = %v, want ErrBundleLeaseLost", err) + } + if n := s.checkpointCount(t, "exec-cp"); n != 1 { + t.Fatalf("checkpoint rows after refused writes = %d, want 1 (refusals wrote nothing)", n) + } + + generation = s.acquire(t, s.keeperA, "exec-cp-expired", "writer-a", pastExpiry()) + if owner, epoch := s.leaseRow(t, "exec-cp-expired"); owner != "writer-a" || epoch != generation { + t.Fatalf("expired-at-birth lease = {%s %d}, want {writer-a %d}", owner, epoch, generation) + } + if _, err := s.keeperA.Checkpoint(s.ctx, "exec-cp-expired", "writer-a", generation, + map[string]any{"stage": "zombie"}); !errors.Is(err, publication.ErrBundleLeaseLost) { + t.Fatalf("checkpoint under expired lease with matching owner+generation = %v, want ErrBundleLeaseLost", err) + } + if n := s.checkpointCount(t, "exec-cp-expired"); n != 0 { + t.Fatalf("checkpoint rows under expired lease = %d, want 0", n) + } + if owner, epoch := s.leaseRow(t, "exec-cp-expired"); owner != "writer-a" || epoch != generation { + t.Fatalf("refused checkpoint mutated the lease = {%s %d}, want {writer-a %d} untouched", owner, epoch, generation) + } +} + +func TestExpiredLeaseCannotBeRenewedOrResurrectedByOldOwner(t *testing.T) { + s := newLeaseService(t) + + generation := s.acquire(t, s.keeperA, "exec-rn", "owner-a", pastExpiry()) + storedExpiry := s.mustWitnessTime(t, + "SELECT expires_at FROM "+s.qualified(BundleLeasesTable)+" WHERE key = $1", "exec-rn") + if !storedExpiry.Before(time.Now().UTC()) { + t.Fatalf("born-expired lease expires_at = %v, want strictly in the past", storedExpiry) + } + + if renewed, err := s.keeperA.RenewLease(s.ctx, "exec-rn", "owner-a", generation, futureExpiry()); err != nil || renewed { + t.Fatalf("old-owner renew of expired lease = %v, %v; want false, nil", renewed, err) + } + if after := s.mustWitnessTime(t, + "SELECT expires_at FROM "+s.qualified(BundleLeasesTable)+" WHERE key = $1", "exec-rn"); !after.Equal(storedExpiry) { + t.Fatalf("expired lease expires_at after refused renew = %v, want the literal stored %v", after, storedExpiry) + } + if _, epoch := s.leaseRow(t, "exec-rn"); epoch != generation { + t.Fatalf("refused renew bumped epoch = %d, want %d", epoch, generation) + } + + takeover, held, err := s.keeperB.AcquireLease(s.ctx, "exec-rn", "owner-b", futureExpiry()) + if err != nil || !held { + s.fail("takeover of expired lease", fmt.Errorf("held=%v err=%w", held, err)) + } + if takeover != generation+1 { + t.Fatalf("takeover returned generation %d, want %d", takeover, generation+1) + } + if owner, epoch := s.leaseRow(t, "exec-rn"); owner != "owner-b" || epoch != generation+1 { + t.Fatalf("after takeover lease = {%s %d}, want {owner-b %d}", owner, epoch, generation+1) + } + + if _, held, err := s.keeperA.AcquireLease(s.ctx, "exec-rn", "owner-a", futureExpiry()); err != nil || held { + t.Fatalf("old-owner re-acquire against live new owner = held %v err %v; want false, nil", held, err) + } + if owner, epoch := s.leaseRow(t, "exec-rn"); owner != "owner-b" || epoch != generation+1 { + t.Fatalf("after refused resurrection lease = {%s %d}, want the untouched {owner-b %d}", owner, epoch, generation+1) + } +} + +func TestReleaseCannotClearNewOwner(t *testing.T) { + s := newLeaseService(t) + + generation := s.acquire(t, s.keeperA, "exec-rel", "owner-a", futureExpiry()) + s.checkpoint(t, s.keeperA, "exec-rel", "owner-a", generation, map[string]any{"stage": "loading"}) + + expired, err := s.keeperA.ExpireLease(s.ctx, "exec-rel", "owner-a", generation) + if err != nil || !expired { + s.fail("owner expire", fmt.Errorf("expired=%v err=%w", expired, err)) + } + if renewed, err := s.keeperA.RenewLease(s.ctx, "exec-rel", "owner-a", generation, futureExpiry()); err != nil || renewed { + t.Fatalf("renew after own expire = %v, %v; want false, nil", renewed, err) + } + if _, err := s.keeperA.Checkpoint(s.ctx, "exec-rel", "owner-a", generation, map[string]any{"stage": "late"}); !errors.Is(err, publication.ErrBundleLeaseLost) { + t.Fatalf("checkpoint after own expire = %v, want ErrBundleLeaseLost", err) + } + + newGeneration := s.acquire(t, s.keeperB, "exec-rel", "owner-b", futureExpiry()) + if newGeneration != generation+1 { + t.Fatalf("takeover generation = %d, want %d", newGeneration, generation+1) + } + s.checkpoint(t, s.keeperB, "exec-rel", "owner-b", newGeneration, map[string]any{"stage": "reconciling"}) + + if released, err := s.keeperA.ReleaseLease(s.ctx, "exec-rel", "owner-a", generation); err != nil || released { + t.Fatalf("stale-owner release = %v, %v; want false, nil", released, err) + } + if released, err := s.keeperB.ReleaseLease(s.ctx, "exec-rel", "owner-b", generation); err != nil || released { + t.Fatalf("same-owner stale-generation release = %v, %v; want false, nil", released, err) + } + if owner, epoch := s.leaseRow(t, "exec-rel"); owner != "owner-b" || epoch != newGeneration { + t.Fatalf("after refused releases lease = {%s %d}, want the untouched {owner-b %d}", owner, epoch, newGeneration) + } + s.checkpoint(t, s.keeperB, "exec-rel", "owner-b", newGeneration, map[string]any{"stage": "still-mine"}) + + if released, err := s.keeperB.ReleaseLease(s.ctx, "exec-rel", "owner-b", newGeneration); err != nil || !released { + t.Fatalf("owner release = %v, %v; want true, nil", released, err) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(BundleLeasesTable)+" WHERE key = $1", "exec-rel"); n != 0 { + t.Fatalf("lease rows after owner release = %d, want 0", n) + } + if released, err := s.keeperB.ReleaseLease(s.ctx, "exec-rel", "owner-b", newGeneration); err != nil || released { + t.Fatalf("double release = %v, %v; want false, nil", released, err) + } +} + +func TestStalePausedWriterCannotCheckpointAfterTakeoverTwoConnections(t *testing.T) { + s := newLeaseService(t) + + backendA := s.mustWitnessInt64On(t, s.iso, "SELECT pg_backend_pid()") + backendB := s.mustWitnessInt64On(t, s.isoB, "SELECT pg_backend_pid()") + if backendA == backendB { + t.Fatalf("pools share backend pid %d, want two real PG connections", backendA) + } + + writerGeneration := s.acquire(t, s.keeperA, "exec-2c", "stale-writer", futureExpiry()) + s.checkpoint(t, s.keeperA, "exec-2c", "stale-writer", writerGeneration, map[string]any{"stage": "queued"}) + s.acquire(t, s.keeperB, "exec-2c", "stale-writer", pastExpiry()) + + reconcilerGeneration := s.acquire(t, s.keeperB, "exec-2c", "reconciler", futureExpiry()) + if reconcilerGeneration != writerGeneration+2 { + t.Fatalf("reconciler generation = %d, want %d (lapse re-grant +1, takeover +2)", + reconcilerGeneration, writerGeneration+2) + } + s.checkpoint(t, s.keeperB, "exec-2c", "reconciler", reconcilerGeneration, map[string]any{"stage": "dropping-stale-outputs"}) + + if _, err := s.keeperA.Checkpoint(s.ctx, "exec-2c", "stale-writer", writerGeneration, + map[string]any{"stage": "resurrect-me"}); !errors.Is(err, publication.ErrBundleLeaseLost) { + t.Fatalf("stale paused writer checkpoint after takeover = %v, want ErrBundleLeaseLost", err) + } + if renewed, err := s.keeperA.RenewLease(s.ctx, "exec-2c", "stale-writer", writerGeneration, futureExpiry()); err != nil || renewed { + t.Fatalf("stale paused writer renew = %v, %v; want false, nil", renewed, err) + } + if expired, err := s.keeperA.ExpireLease(s.ctx, "exec-2c", "stale-writer", writerGeneration); err != nil || expired { + t.Fatalf("stale paused writer expire = %v, %v; want false, nil", expired, err) + } + if released, err := s.keeperA.ReleaseLease(s.ctx, "exec-2c", "stale-writer", writerGeneration); err != nil || released { + t.Fatalf("stale paused writer release = %v, %v; want false, nil", released, err) + } + + if owner, epoch := s.leaseRow(t, "exec-2c"); owner != "reconciler" || epoch != reconcilerGeneration { + t.Fatalf("lease after takeover = {%s %d}, want the untouched {reconciler %d}", owner, epoch, reconcilerGeneration) + } + s.checkpoint(t, s.keeperB, "exec-2c", "reconciler", reconcilerGeneration, map[string]any{"stage": "finalizing"}) + if n := s.checkpointCount(t, "exec-2c"); n != 3 { + t.Fatalf("checkpoint rows after stale-writer attempts = %d, want exactly 3", n) + } + for _, want := range []struct { + sequence, generation int64 + owner string + }{ + {1, writerGeneration, "stale-writer"}, + {2, reconcilerGeneration, "reconciler"}, + {3, reconcilerGeneration, "reconciler"}, + } { + owner := s.mustWitnessText(t, + "SELECT owner_id FROM "+s.qualified(ExecutionCheckpointsTable)+" WHERE key = $1 AND sequence = $2", + "exec-2c", want.sequence) + generation := s.mustWitnessInt64(t, + "SELECT generation FROM "+s.qualified(ExecutionCheckpointsTable)+" WHERE key = $1 AND sequence = $2", + "exec-2c", want.sequence) + if owner != want.owner || generation != want.generation { + t.Fatalf("checkpoint %d = {%s gen %d}, want {%s gen %d}", + want.sequence, owner, generation, want.owner, want.generation) + } + } + + latest, ok, err := s.keeperB.LatestCheckpoint(s.ctx, "exec-2c") + if err != nil || !ok { + s.fail("latest checkpoint", err) + } + if latest.Sequence != 3 || latest.OwnerID != "reconciler" || latest.Generation != reconcilerGeneration { + t.Fatalf("latest checkpoint = %+v, want sequence 3 reconciler generation %d", latest, reconcilerGeneration) + } + if n := s.execCount(t, + "SELECT count(*) FROM "+s.qualified(ExecutionCheckpointsTable)+" WHERE key = $1 AND owner_id = $2 AND generation = $3", + "exec-2c", "stale-writer", writerGeneration); n != 1 { + t.Fatalf("checkpoints under the retired token = %d, want exactly 1 (the pre-pause one)", n) + } +} + +func (s *leaseService) mustWitnessInt64On(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) int64 { + t.Helper() + var value int64 + if err := pool.QueryRow(s.ctx, sql, args...).Scan(&value); err != nil { + s.fail("witness "+sql, err) + } + return value +} + +func TestGenerationPinningCorrelatesRepeatedOwnerGrants(t *testing.T) { + s := newLeaseService(t) + + first := s.acquire(t, s.keeperA, "exec-gen", "owner-a", futureExpiry()) + second := s.acquire(t, s.keeperA, "exec-gen", "owner-a", futureExpiry()) + third := s.acquire(t, s.keeperA, "exec-gen", "owner-a", futureExpiry()) + if first != 1 || second != 2 || third != 3 { + t.Fatalf("same-owner grant generations = %d/%d/%d, want 1/2/3", first, second, third) + } + if owner, epoch := s.leaseRow(t, "exec-gen"); owner != "owner-a" || epoch != 3 { + t.Fatalf("lease after regrants = {%s %d}, want {owner-a 3}", owner, epoch) + } + + s.checkpoint(t, s.keeperA, "exec-gen", "owner-a", 3, map[string]any{"stage": "current"}) + for _, stale := range []int64{1, 2} { + if _, err := s.keeperA.Checkpoint(s.ctx, "exec-gen", "owner-a", stale, map[string]any{"stage": "stale"}); !errors.Is(err, publication.ErrBundleLeaseLost) { + t.Fatalf("checkpoint on retired generation %d = %v, want ErrBundleLeaseLost", stale, err) + } + if renewed, err := s.keeperA.RenewLease(s.ctx, "exec-gen", "owner-a", stale, futureExpiry()); err != nil || renewed { + t.Fatalf("renew on retired generation %d = %v, %v; want false, nil", stale, renewed, err) + } + if expired, err := s.keeperA.ExpireLease(s.ctx, "exec-gen", "owner-a", stale); err != nil || expired { + t.Fatalf("expire on retired generation %d = %v, %v; want false, nil", stale, expired, err) + } + if released, err := s.keeperA.ReleaseLease(s.ctx, "exec-gen", "owner-a", stale); err != nil || released { + t.Fatalf("release on retired generation %d = %v, %v; want false, nil", stale, released, err) + } + } + if owner, epoch := s.leaseRow(t, "exec-gen"); owner != "owner-a" || epoch != 3 { + t.Fatalf("lease after inert stale tokens = {%s %d}, want the untouched {owner-a 3}", owner, epoch) + } + if n := s.checkpointCount(t, "exec-gen"); n != 1 { + t.Fatalf("checkpoint rows = %d, want 1 (retired tokens wrote nothing)", n) + } + + execution := baseExecution("exec-gen-save", "exec-gen", publication.BundleRunning, + stamps(leaseCreated, leaseUpdated, time.Time{}, time.Time{})) + if err := s.registry.SaveExecution(s.ctx, execution, "owner-a"); err != nil { + s.fail("save under keeper grant", err) + } + if _, err := s.keeperA.ExpireLease(s.ctx, "exec-gen", "owner-a", 3); err != nil { + s.fail("expire current grant", err) + } + if err := s.registry.SaveExecution(s.ctx, execution, "owner-a"); !errors.Is(err, publication.ErrBundleLeaseLost) { + t.Fatalf("SaveExecution after keeper expire = %v, want ErrBundleLeaseLost (shared lease registry)", err) + } +} + +var ( + leaseCreated = baseTime.Add(3 * time.Hour) + leaseUpdated = baseTime.Add(3*time.Hour + time.Minute) +) + +func TestReleasedGenerationIsNeverReissued(t *testing.T) { + s := newLeaseService(t) + const key = "exec-aba" + + first := s.acquire(t, s.keeperA, key, "reconciler", futureExpiry()) + if first != 1 { + t.Fatalf("first grant = %d, want 1", first) + } + s.checkpoint(t, s.keeperA, key, "reconciler", first, map[string]any{"episode": 1}) + + if released, err := s.keeperA.ReleaseLease(s.ctx, key, "reconciler", first); err != nil || !released { + s.fail("release first grant", fmt.Errorf("released=%v err=%w", released, err)) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(BundleLeasesTable)+" WHERE key = $1", key); n != 0 { + t.Fatalf("lease rows after release = %d, want 0", n) + } + + second := s.acquire(t, s.keeperA, key, "reconciler", futureExpiry()) + if second <= first { + t.Fatalf("re-acquire after release reissued generation %d (retired %d); want strictly greater", second, first) + } + if second != 2 { + t.Fatalf("re-acquire generation = %d, want 2 (high-water survives deletion)", second) + } + + if _, err := s.keeperB.Checkpoint(s.ctx, key, "reconciler", first, map[string]any{"episode": "straggler"}); !errors.Is(err, publication.ErrBundleLeaseLost) { + t.Fatalf("straggler checkpoint with re-issued-looking token = %v, want ErrBundleLeaseLost", err) + } + if renewed, err := s.keeperB.RenewLease(s.ctx, key, "reconciler", first, futureExpiry()); err != nil || renewed { + t.Fatalf("straggler renew = %v, %v; want false, nil", renewed, err) + } + if expired, err := s.keeperB.ExpireLease(s.ctx, key, "reconciler", first); err != nil || expired { + t.Fatalf("straggler expire = %v, %v; want false, nil", expired, err) + } + if released, err := s.keeperB.ReleaseLease(s.ctx, key, "reconciler", first); err != nil || released { + t.Fatalf("straggler release = %v, %v; want false, nil", released, err) + } + if owner, epoch := s.leaseRow(t, key); owner != "reconciler" || epoch != second { + t.Fatalf("lease after straggler attempts = {%s %d}, want the untouched {reconciler %d}", owner, epoch, second) + } + if n := s.checkpointCount(t, key); n != 1 { + t.Fatalf("checkpoint rows = %d, want 1 (straggler wrote nothing)", n) + } + if gen := s.mustWitnessInt64(t, + "SELECT generation FROM "+s.qualified(ExecutionCheckpointsTable)+" WHERE key = $1 AND sequence = 1", key); gen != first { + t.Fatalf("episode-1 row generation = %d, want %d", gen, first) + } + + if err := s.registry.ReleaseBundleLease(s.ctx, key, "reconciler"); err != nil { + s.fail("sibling release", err) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(BundleLeasesTable)+" WHERE key = $1", key); n != 0 { + t.Fatalf("lease rows after sibling release = %d, want 0", n) + } + third := s.acquire(t, s.keeperA, key, "reconciler", futureExpiry()) + if third <= second { + t.Fatalf("re-acquire after sibling release reissued generation %d (retired %d); want strictly greater", third, second) + } + if third != 3 { + t.Fatalf("third grant generation = %d, want 3", third) + } + if _, err := s.keeperA.Checkpoint(s.ctx, key, "reconciler", second, map[string]any{"episode": "late-second"}); !errors.Is(err, publication.ErrBundleLeaseLost) { + t.Fatalf("checkpoint on retired second token = %v, want ErrBundleLeaseLost", err) + } + sequence := s.checkpoint(t, s.keeperA, key, "reconciler", third, map[string]any{"episode": 3}) + if sequence != 2 { + t.Fatalf("episode-3 checkpoint sequence = %d, want 2", sequence) + } + if gen := s.mustWitnessInt64(t, + "SELECT generation FROM "+s.qualified(ExecutionCheckpointsTable)+" WHERE key = $1 AND sequence = $2", key, 2); gen != third { + t.Fatalf("episode-3 row generation = %d, want %d", gen, third) + } + + if _, held, err := s.keeperB.AcquireLease(s.ctx, key, "challenger", futureExpiry()); err != nil || held { + t.Fatalf("challenger acquire against live owner = held %v err %v; want false, nil", held, err) + } + if owner, epoch := s.leaseRow(t, key); owner != "reconciler" || epoch != third { + t.Fatalf("lease after refused challenger = {%s %d}, want the untouched {reconciler %d}", owner, epoch, third) + } + if hw := s.mustWitnessInt64(t, + "SELECT last_generation FROM "+s.qualified(LeaseGrantsTable)+" WHERE key = $1", key); hw != third { + t.Fatalf("high-water after refused acquire = %d, want %d (bump rolled back, not burned)", hw, third) + } +} + +func TestCheckpointPayloadPreservesMissingVersusJSONNull(t *testing.T) { + s := newLeaseService(t) + generation := s.acquire(t, s.keeperA, "exec-null", "writer-a", futureExpiry()) + + sequence, err := s.keeperA.Checkpoint(s.ctx, "exec-null", "writer-a", generation, map[string]any{ + "stage": "loading", + "note": nil, + }) + if err != nil || sequence != 1 { + s.fail("null-bearing checkpoint", fmt.Errorf("sequence=%d err=%w", sequence, err)) + } + + if present := s.mustWitnessBool(t, + "SELECT payload ? 'note' FROM "+s.qualified(ExecutionCheckpointsTable)+" WHERE key = $1 AND sequence = 1", + "exec-null"); !present { + t.Fatal("stored payload lost the explicit null member") + } + if kind := s.mustWitnessText(t, + "SELECT jsonb_typeof(payload->'note') FROM "+s.qualified(ExecutionCheckpointsTable)+" WHERE key = $1 AND sequence = 1", + "exec-null"); kind != "null" { + t.Fatalf("stored note type = %q, want null", kind) + } + if present := s.mustWitnessBool(t, + "SELECT payload ? 'absent' FROM "+s.qualified(ExecutionCheckpointsTable)+" WHERE key = $1 AND sequence = 1", + "exec-null"); present { + t.Fatal("stored payload gained a member that was never sent") + } + + record, ok, err := s.keeperA.LatestCheckpoint(s.ctx, "exec-null") + if err != nil || !ok { + s.fail("latest checkpoint", err) + } + if record.Sequence != 1 || record.OwnerID != "writer-a" || record.Generation != generation || record.CheckpointedAt.IsZero() { + t.Fatalf("latest checkpoint record = %+v, want sequence 1 writer-a generation %d with a stored stamp", + record, generation) + } + raw := string(record.Payload) + if !regexp.MustCompile(`"note"\s*:\s*null`).MatchString(raw) || + !regexp.MustCompile(`"stage"\s*:\s*"loading"`).MatchString(raw) { + t.Fatalf("returned payload bytes = %s, want the stored jsonb text with the null preserved", raw) + } + + if _, err := s.keeperA.Checkpoint(s.ctx, "exec-null", "writer-a", generation, nil); err == nil { + t.Fatal("nil payload = nil error, want rejection") + } + if _, err := s.keeperA.Checkpoint(s.ctx, "exec-null", "writer-a", generation, any(nil)); err == nil { + t.Fatal("typed nil payload = nil error, want rejection") + } + + for name, nonObject := range map[string]any{ + "number": 42, + "string": "just text", + "array": []any{1, 2}, + "bool": true, + "null": json.RawMessage("null"), + } { + _, err := s.keeperA.Checkpoint(s.ctx, "exec-null", "writer-a", generation, nonObject) + if err == nil { + t.Fatalf("payload %s = nil error, want rejection", name) + } + if !strings.Contains(err.Error(), "must be a JSON object") { + t.Fatalf("payload %s error = %v, want the promised must-be-a-JSON-object message", name, err) + } + if strings.Contains(err.Error(), "23514") { + t.Fatalf("payload %s surfaced a raw CHECK violation: %v", name, err) + } + } + if n := s.checkpointCount(t, "exec-null"); n != 1 { + t.Fatalf("checkpoint rows after refused payloads = %d, want 1", n) + } + sequence, err = s.keeperA.Checkpoint(s.ctx, "exec-null", "writer-a", generation, map[string]any{"stage": "next"}) + if err != nil || sequence != 2 { + s.fail("post-refusal checkpoint", fmt.Errorf("sequence=%d err=%w", sequence, err)) + } + if n := s.checkpointCount(t, "exec-null"); n != 2 { + t.Fatalf("checkpoint rows = %d, want 2", n) + } +} + +func (s *leaseService) mustWitnessBool(t *testing.T, sql string, args ...any) bool { + t.Helper() + var value bool + if err := s.isoB.QueryRow(s.ctx, sql, args...).Scan(&value); err != nil { + s.fail("witness "+sql, err) + } + return value +} + +func TestAcquireGuardsAndNoFallback(t *testing.T) { + s := newLeaseService(t) + + if _, held, err := s.keeperA.AcquireLease(s.ctx, "", "owner-a", futureExpiry()); err == nil || !errors.Is(err, ErrInvalidLeaseKey) || held { + t.Fatalf("AcquireLease(empty key) = held %v err %v, want ErrInvalidLeaseKey", held, err) + } + if _, held, err := s.keeperA.AcquireLease(s.ctx, "exec-guard", "", futureExpiry()); err == nil || !errors.Is(err, ErrInvalidLeaseKey) || held { + t.Fatalf("AcquireLease(empty owner) = held %v err %v, want ErrInvalidLeaseKey", held, err) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(BundleLeasesTable)+" WHERE key IN ($1, $2)", "", "exec-guard"); n != 0 { + t.Fatalf("lease rows after guarded acquires = %d, want 0", n) + } + + emptySchema := fmt.Sprintf("pgleases01_it_%d_%s", time.Now().UnixNano(), randomHex()) + if !pgleases01ResourceNamePattern.MatchString(emptySchema) { + t.Fatalf("owned schema name %q does not match the required pattern", emptySchema) + } + s.mustExec(t, s.iso, `CREATE SCHEMA "`+emptySchema+`"`) + t.Cleanup(func() { + if _, err := s.iso.Exec(context.WithoutCancel(s.ctx), `DROP SCHEMA IF EXISTS "`+emptySchema+`" CASCADE`); err != nil { + t.Errorf("drop empty schema: %s", s.scrub(err)) + } + }) + orphan, err := NewLeaseKeeper(s.isoB, emptySchema) + if err != nil { + s.fail("build orphan keeper", err) + } + _, _, err = orphan.AcquireLease(s.ctx, "exec-missing", "owner-a", futureExpiry()) + if err == nil { + t.Fatal("acquire against unmigrated schema = nil error, want the missing relation surfaced") + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "42P01" { + t.Fatalf("acquire against unmigrated schema = %v, want SQLSTATE 42P01 undefined_table", s.scrub(err)) + } +} + +func TestCheckpointSchemaCompositionIsMetadataOnly(t *testing.T) { + s := newLeaseService(t) + rows, err := s.isoB.Query(s.ctx, + "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = $1 ORDER BY tablename", s.schema) + if err != nil { + s.fail("list tables", err) + } + var tables []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + s.fail("scan table", err) + } + tables = append(tables, name) + } + if err := rows.Err(); err != nil { + s.fail("list tables", err) + } + rows.Close() + want := []string{ + "publication_bundle_executions", + "publication_bundle_leases", + "publication_bundle_pointers", + "publication_execution_checkpoints", + "publication_lease_grants", + "schema_migrations", + } + if strings.Join(tables, ",") != strings.Join(want, ",") { + t.Fatalf("schema tables = %#v, want exactly %#v (metadata only; no row-store tables)", tables, want) + } + + type column struct { + name, typ string + nullable bool + } + rows, err = s.isoB.Query(s.ctx, + "SELECT column_name, data_type, is_nullable = 'YES' FROM information_schema.columns"+ + " WHERE table_schema = $1 AND table_name = $2 ORDER BY ordinal_position", + s.schema, ExecutionCheckpointsTable) + if err != nil { + s.fail("list columns", err) + } + var columns []column + for rows.Next() { + var c column + if err := rows.Scan(&c.name, &c.typ, &c.nullable); err != nil { + s.fail("scan column", err) + } + columns = append(columns, c) + } + if err := rows.Err(); err != nil { + s.fail("list columns", err) + } + rows.Close() + wantColumns := []column{ + {"key", "text", false}, + {"sequence", "bigint", false}, + {"owner_id", "text", false}, + {"generation", "bigint", false}, + {"checkpointed_at", "timestamp with time zone", false}, + {"payload", "jsonb", false}, + } + if len(columns) != len(wantColumns) { + t.Fatalf("columns = %+v, want %+v", columns, wantColumns) + } + for i, wantCol := range wantColumns { + if columns[i] != wantCol { + t.Fatalf("column %d = %+v, want %+v", i, columns[i], wantCol) + } + } +} + +func TestZeroLeftoverPgleasesResources(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer admin.Close() + var databases int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'pgleases01_it_%'").Scan(&databases); err != nil { + t.Fatalf("count databases: %v", err) + } + if databases != 0 { + t.Fatalf("%d pgleases01_it_* databases still exist", databases) + } + var schemas int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgleases01_it_%'").Scan(&schemas); err != nil { + t.Fatalf("count schemas: %v", err) + } + if schemas != 0 { + t.Fatalf("%d pgleases01_it_* schemas still exist in the service database", schemas) + } +} diff --git a/internal/backends/postgres/internal/publication/migrate.go b/internal/backends/postgres/internal/publication/migrate.go new file mode 100644 index 000000000..c2defd605 --- /dev/null +++ b/internal/backends/postgres/internal/publication/migrate.go @@ -0,0 +1,51 @@ +package postgres + +import ( + "context" + "embed" + "errors" + "fmt" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" +) + +//go:embed migrations/*.sql +var migrationsFS embed.FS + +func OwnerMigrations() ([]pgstore.Migration, error) { + migrations, err := pgstore.LoadMigrations(migrationsFS, "migrations") + if err != nil { + return nil, fmt.Errorf("publication postgres: load owner migrations: %w", err) + } + return migrations, nil +} + +func MigrateSchema(ctx context.Context, src pgstore.ConnSource, schema string) (pgstore.MigrateResult, error) { + if src == nil { + return pgstore.MigrateResult{}, errors.New("publication postgres: nil migration connection source") + } + migrations, err := OwnerMigrations() + if err != nil { + return pgstore.MigrateResult{}, err + } + runner, err := pgstore.NewMigrationRunner(src, schema, migrations) + if err != nil { + return pgstore.MigrateResult{}, err + } + return runner.Migrate(ctx) +} + +func VerifySchema(ctx context.Context, src pgstore.ConnSource, schema string) error { + if src == nil { + return errors.New("publication postgres: nil migration connection source") + } + migrations, err := OwnerMigrations() + if err != nil { + return err + } + runner, err := pgstore.NewMigrationRunner(src, schema, migrations) + if err != nil { + return err + } + return runner.Verify(ctx) +} diff --git a/internal/backends/postgres/internal/publication/migrations/000001_ledger.sql b/internal/backends/postgres/internal/publication/migrations/000001_ledger.sql new file mode 100644 index 000000000..13c0efcfc --- /dev/null +++ b/internal/backends/postgres/internal/publication/migrations/000001_ledger.sql @@ -0,0 +1,17 @@ +-- pg-store-02: create the migration ledger. +-- +-- The ledger is itself a versioned migration so its creation is audited by +-- the same runner that reads it. It stores version, name, and checksum only +-- (plus when it was applied) -- never the migration SQL content. The runner +-- inserts each row only after the migration transaction commits, so a +-- recorded migration always applied successfully. +CREATE TABLE schema_migrations ( + version bigint NOT NULL, + name text NOT NULL, + checksum text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT schema_migrations_pkey PRIMARY KEY (version), + CONSTRAINT schema_migrations_version_positive CHECK (version > 0), + CONSTRAINT schema_migrations_name_shape CHECK (name ~ '^[a-z][a-z0-9_]*$'), + CONSTRAINT schema_migrations_checksum_hex CHECK (checksum ~ '^[0-9a-f]{64}$') +); diff --git a/internal/backends/postgres/internal/publication/migrations/000004_publication_bundle_catalog.sql b/internal/backends/postgres/internal/publication/migrations/000004_publication_bundle_catalog.sql new file mode 100644 index 000000000..8dcf10810 --- /dev/null +++ b/internal/backends/postgres/internal/publication/migrations/000004_publication_bundle_catalog.sql @@ -0,0 +1,81 @@ +-- pg-publication-bundle-catalog-v1: PostgreSQL-side publication bundle catalog. +-- +-- These tables persist execution/output metadata, quality/state metadata, +-- the exact logical pointer registry, and fenced bundle leases for the +-- PostgreSQL source/control backend. They hold metadata only: ClickHouse +-- remains the publication row store and no data rows move here. The version +-- deliberately follows the 000002_*/000003_* migrations owned by the +-- concurrent source-store tickets so the histories interleave without +-- collisions, and the DDL is idempotent because the runner may retry a +-- migration that committed before its ledger row was written. +-- +-- Exact logical identity: +-- * An execution's searchable identity is split into explicit scope columns +-- (project, dataset_generation, recipe_name, translation_version) so an +-- exact lookup is a scope equality on all of them -- never a name-only +-- latest fallback. +-- * The full BundleExecution JSON body lives in payload. Reads decode the +-- payload and canonicalize legacy shapes in memory; nothing rewrites it, +-- so historical records keep their stored values (jsonb normalizes only +-- the lexical JSON form, never field values). +-- * Pointer names are arbitrary domain strings (they may embed NUL from the +-- length-prefixed PointerName encoding, which PostgreSQL text and jsonb +-- both reject), so the row key is the hex SHA-256 of the name -- the same +-- discipline as the Arango pointer documents -- and the exact bytes are +-- round-tripped losslessly through bytea. +CREATE TABLE IF NOT EXISTS publication_bundle_executions ( + id text PRIMARY KEY, + execution_key text NOT NULL, + project text NOT NULL, + dataset_generation text NOT NULL, + recipe_name text NOT NULL, + translation_version text NOT NULL, + state text NOT NULL, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + ready_at timestamptz, + published_at timestamptz, + payload jsonb NOT NULL, + CONSTRAINT publication_bundle_executions_payload_object + CHECK (jsonb_typeof(payload) = 'object') +); + +-- Exact publication lookup scope: project + generation + selector. The +-- published/ready/updated ordering mirrors the Arango exact query; id breaks +-- ties so the "latest" answer is deterministic. +CREATE INDEX IF NOT EXISTS publication_bundle_executions_exact_scope + ON publication_bundle_executions ( + project, dataset_generation, recipe_name, translation_version, + state, published_at DESC NULLS LAST, ready_at DESC NULLS LAST, + updated_at DESC, id + ); + +-- FindExecutionByKey: newest created_at first, id tie-break for determinism. +CREATE INDEX IF NOT EXISTS publication_bundle_executions_key_created + ON publication_bundle_executions (execution_key, created_at DESC, id); + +-- Reconciliation scans and stable keyset pages: (state, updated_at, id). +CREATE INDEX IF NOT EXISTS publication_bundle_executions_state_scan + ON publication_bundle_executions (state, updated_at, id); + +CREATE TABLE IF NOT EXISTS publication_bundle_pointers ( + pointer_key text PRIMARY KEY, + -- Exact pointer-name bytes. PostgreSQL text and jsonb both reject NUL, + -- and a PointerName may legally embed one, so the name lives in bytea. + name_bytes bytea NOT NULL, + execution_id text NOT NULL, + updated_at timestamptz NOT NULL +); + +CREATE TABLE IF NOT EXISTS publication_bundle_leases ( + key text PRIMARY KEY, + owner_id text NOT NULL, + expires_at timestamptz NOT NULL, + acquired_at timestamptz NOT NULL, + -- epoch increments on every grant (including a same-owner reclaim after + -- expiry), correlating repeated ownership of one lease key so a witness + -- can distinguish grants. + epoch bigint NOT NULL DEFAULT 1, + CONSTRAINT publication_bundle_leases_epoch_positive + CHECK (epoch >= 1) +); diff --git a/internal/backends/postgres/internal/publication/migrations/000006_publication_lease_checkpoints.sql b/internal/backends/postgres/internal/publication/migrations/000006_publication_lease_checkpoints.sql new file mode 100644 index 000000000..373a5596c --- /dev/null +++ b/internal/backends/postgres/internal/publication/migrations/000006_publication_lease_checkpoints.sql @@ -0,0 +1,54 @@ +-- pg-publication-leases-checkpoints-v1: owner-conditional checkpoint records +-- for the PostgreSQL publication control plane. +-- +-- Execution owners and reconcilers fence their progress checkpoints with the +-- fenced leases already created by 000004_publication_bundle_catalog.sql; +-- this migration adds only the durable record each fenced checkpoint writes. +-- It holds metadata and progress facts only: ClickHouse remains the +-- publication row store and no data rows move here. The version takes the +-- coordinator reservation 000006_* (000005 belongs to the catalog ticket), +-- and the DDL is idempotent because the runner may retry a migration that +-- committed before its ledger row was written. +-- +-- Checkpoint identity: +-- * A checkpoint belongs to one lease key and pins the exact ownership +-- grant that wrote it: owner_id and generation (the lease epoch at write +-- time) are recorded beside the payload, so a witness can reconstruct +-- which owner-episode produced every checkpoint even after takeovers and +-- same-owner reclaims. +-- * sequence is the per-key monotonic position assigned inside the same +-- guarded write that applies the checkpoint, so ordering across +-- ownership changes is gap-free and a stale token cannot pre-fill it. +-- Grant high-water: the FencedLeases keeper consults and bumps this row on +-- every committed Acquire, so a generation it has handed out is never +-- reissued even after the lease row itself is deleted (by ReleaseLease, by +-- Registry.ReleaseBundleLease, or by any other path). The row is never +-- deleted; last_generation only increases, which is what makes the keeper's +-- generation a monotonic fencing token per key across the lease row's whole +-- lifetime. Registry's own Arango-parity lease methods keep their baseline +-- epoch behaviour and do not participate in this protocol. +CREATE TABLE IF NOT EXISTS publication_lease_grants ( + key text PRIMARY KEY, + last_generation bigint NOT NULL DEFAULT 0 CHECK (last_generation >= 0) +); + +CREATE TABLE IF NOT EXISTS publication_execution_checkpoints ( + key text NOT NULL, + sequence bigint NOT NULL, + owner_id text NOT NULL, + generation bigint NOT NULL, + checkpointed_at timestamptz NOT NULL, + payload jsonb NOT NULL, + PRIMARY KEY (key, sequence), + CONSTRAINT publication_execution_checkpoints_sequence_positive + CHECK (sequence >= 1), + CONSTRAINT publication_execution_checkpoints_generation_positive + CHECK (generation >= 1), + CONSTRAINT publication_execution_checkpoints_payload_object + CHECK (jsonb_typeof(payload) = 'object') +); + +-- Ownership audit: which grant wrote which checkpoints for one key, newest +-- first, matching how a resuming writer finds its last checkpoint. +CREATE INDEX IF NOT EXISTS publication_execution_checkpoints_owner_generation + ON publication_execution_checkpoints (key, owner_id, generation, sequence); diff --git a/internal/backends/postgres/internal/publication/migrations/000007_private_clickhouse_artifacts.sql b/internal/backends/postgres/internal/publication/migrations/000007_private_clickhouse_artifacts.sql new file mode 100644 index 000000000..484b2c832 --- /dev/null +++ b/internal/backends/postgres/internal/publication/migrations/000007_private_clickhouse_artifacts.sql @@ -0,0 +1,81 @@ +-- pg-private-clickhouse-artifact-catalog-v1: temporary private ClickHouse +-- artifact manifests and the cleanup-lease catalog for the PostgreSQL control +-- plane. +-- +-- Physical data tables stay in ClickHouse: this catalog persists only the +-- durable manifest (immutable identity, schema, and physical-table binding) +-- and the mutable progress/lease fields around it. It registers nothing as a +-- published output, never advances a visibility pointer, and is deliberately +-- separate from the published BundleCatalog (tests/migrations/000004) and +-- from the Explorer filesystem ZIP ArtifactStore. The version continues the +-- coordinator-ordered fixture history after the 000006 lease-checkpoint +-- reservation, and the DDL is idempotent because the pg-store-02 runner may +-- retry a migration that committed before its ledger row was written. +-- +-- Manifest identity: +-- * artifact_id is the immutable primary key; Create is insert-only, so a +-- concurrent duplicate converges on the existing row instead of +-- replacing it. The (execution, output, stage, project, generation) +-- identity columns and the physical-table binding mirror the payload and +-- are only ever written by Create; the catalog's Update, Renew, +-- ClaimCleanup, Delete, and ReleaseCleanup statements touch progress and +-- lease fields only, so identity and table fields survive every later +-- write unchanged. +-- * payload is the authoritative jsonb manifest: nanosecond timestamps and +-- the missing-versus-JSON-null member shape round-trip losslessly. The +-- scan columns (state, counts, lease, identity scope) mirror the payload +-- for constraints and ordered scans; every statement writes both from +-- the same values. No manifest member is ever SQL NULL: every column is +-- NOT NULL, so a missing value fails explicitly rather than silently +-- reading back as null. +-- * state is closed to the four domain states; counts are nonnegative; the +-- lease owner, physical table, and identity scope must be present. An +-- unsupported value is refused by the constraint, never coerced. +-- +-- Lease/cleanup scan: ListExpired pages the lease_until index in +-- (lease_until, artifact_id COLLATE "C") order -- the deterministic +-- byte-order tie-break the Arango collection gives _key -- so a bounded page +-- is stable across repeated reconciler passes, and ClaimCleanup rechecks +-- lease_until <= expiredBefore inside the same UPDATE so a concurrent +-- renewer wins and only one cleanup owner ever claims a manifest. +CREATE TABLE IF NOT EXISTS chartifact_private_artifacts ( + artifact_id text PRIMARY KEY, + physical_table text NOT NULL, + state text NOT NULL, + row_count bigint NOT NULL, + byte_count bigint NOT NULL, + lease_owner text NOT NULL, + lease_until timestamptz NOT NULL, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + execution_id text NOT NULL, + output_id text NOT NULL, + stage_id text NOT NULL, + project text NOT NULL, + dataset_generation text NOT NULL, + auth_scope_mode text NOT NULL, + scope_mode text NOT NULL, + schema_digest text NOT NULL, + scope_digest text NOT NULL, + payload jsonb NOT NULL, + CONSTRAINT chartifact_private_artifacts_id_present + CHECK (artifact_id <> '' AND artifact_id = btrim(artifact_id)), + CONSTRAINT chartifact_private_artifacts_table_present + CHECK (physical_table <> '' AND physical_table = btrim(physical_table)), + CONSTRAINT chartifact_private_artifacts_owner_present + CHECK (lease_owner <> '' AND lease_owner = btrim(lease_owner)), + CONSTRAINT chartifact_private_artifacts_state_closed + CHECK (state IN ('CREATING', 'WRITING', 'READY', 'CLEANUP_PENDING')), + CONSTRAINT chartifact_private_artifacts_counts_nonnegative + CHECK (row_count >= 0 AND byte_count >= 0), + CONSTRAINT chartifact_private_artifacts_identity_present + CHECK (execution_id <> '' AND output_id <> '' AND stage_id <> '' + AND project <> '' AND dataset_generation <> ''), + CONSTRAINT chartifact_private_artifacts_payload_object + CHECK (jsonb_typeof(payload) = 'object') +); + +-- Bounded expired-lease scan: the reconciler's page walks this index in +-- lease order; the claim path rechecks the same column inside its UPDATE. +CREATE INDEX IF NOT EXISTS chartifact_private_artifacts_lease_until + ON chartifact_private_artifacts (lease_until); diff --git a/internal/backends/postgres/internal/publication/migrations/000009_publication_read_pins.sql b/internal/backends/postgres/internal/publication/migrations/000009_publication_read_pins.sql new file mode 100644 index 000000000..80cb4891d --- /dev/null +++ b/internal/backends/postgres/internal/publication/migrations/000009_publication_read_pins.sql @@ -0,0 +1,56 @@ +-- pg-publication-read-pins-reconcile-v1: read retention pins and cleanup +-- claims for the PostgreSQL publication control plane. +-- +-- Read pins keep the physical publication tables of an exact execution +-- reader-visible while exact export / row streaming runs; a cleanup claim is +-- the reconciler's exclusive right to remove them. The two operations race by +-- design, so they must serialize: every pin and cleanup transition for one +-- execution locks the single publication_execution_retention anchor row +-- first, which is the one lock point both paths take before any pin or claim +-- row is read or written. This migration holds control facts only: the +-- physical tables live in ClickHouse, which remains the publication row +-- store, and no data rows move here. The version continues the coordinator's +-- cross-directory reservation sequence (000004 catalog, 000006 lease +-- checkpoints, 000007 chartifact/interpretations, 000008 selections), and +-- the DDL is idempotent because the runner may retry a migration that +-- committed before its ledger row was written. +-- +-- Retention identity: +-- * Every fact is keyed by the execution id, the same identity the pin and +-- cleanup APIs address, so pins, claims, and the anchor for one +-- execution always colocate on one lock. +-- * Pins are one row per (execution, owner): a repeated acquire by the same +-- owner refreshes that row rather than duplicating a reader, which is +-- how the Arango pins array already collapses, and the row preserves +-- each owner's own expiry (owner-correlated retention). +-- * Cleanup is at most one live claim per execution: a missing claim row is +-- "no cleanup", which stays distinct from any expiry timestamp; the +-- claim stores its owning reconciler and expiry so renewal and release +-- are owner-conditional and a witness can attribute every claim episode. +-- * No column is nullable-ambiguous: expiry is always a timestamptz, so +-- "absent" is expressed only by row absence and never by a NULL stand-in. +CREATE TABLE IF NOT EXISTS publication_execution_retention ( + execution_id text PRIMARY KEY, + updated_at timestamptz NOT NULL +); + +CREATE TABLE IF NOT EXISTS publication_execution_read_pins ( + execution_id text NOT NULL REFERENCES publication_execution_retention (execution_id) ON DELETE CASCADE, + owner_id text NOT NULL CHECK (owner_id <> ''), + expires_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + PRIMARY KEY (execution_id, owner_id) +); + +-- Active-reader and expiry sweeps for one execution: the cleanup guard scans +-- live pins by (execution, expiry), and every successful arbitration GCs +-- expired pins with the same access shape. +CREATE INDEX IF NOT EXISTS publication_execution_read_pins_expiry + ON publication_execution_read_pins (execution_id, expires_at); + +CREATE TABLE IF NOT EXISTS publication_execution_cleanup_claims ( + execution_id text PRIMARY KEY REFERENCES publication_execution_retention (execution_id) ON DELETE CASCADE, + owner_id text NOT NULL CHECK (owner_id <> ''), + expires_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL +); diff --git a/internal/backends/postgres/internal/publication/publish.go b/internal/backends/postgres/internal/publication/publish.go new file mode 100644 index 000000000..958dc968e --- /dev/null +++ b/internal/backends/postgres/internal/publication/publish.go @@ -0,0 +1,127 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/calypr/loom/internal/dataframe/publication" +) + +type publishAttempt struct { + PointerName string + PointerKey string + ExpectedExecution string + ExecutionID string + OwnerID string + BackendPID uint32 + Statements []string + Converged bool +} + +var publishGate atomic.Pointer[func(publishAttempt) error] + +const publishExecutionSet = "execution_key = $2, project = $3, dataset_generation = $4, recipe_name = $5, " + + "translation_version = $6, state = $7, created_at = $8, updated_at = $9, " + + "ready_at = $10, published_at = $11, payload = $12" + +func (r *Registry) PublishExecution(ctx context.Context, name, expected string, execution publication.BundleExecution) error { + params, err := executionParams(execution) + if err != nil { + return err + } + tx, err := r.db.Begin(ctx) + if err != nil { + return fmt.Errorf("publication postgres: begin publish: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + attempt := publishAttempt{ + PointerName: name, + PointerKey: pointerKey(name), + ExpectedExecution: expected, + ExecutionID: execution.ID, + OwnerID: execution.OwnerID, + } + if conn := tx.Conn(); conn != nil { + attempt.BackendPID = conn.PgConn().PID() + } + record := func(sql string) { attempt.Statements = append(attempt.Statements, sql) } + + fenceSQL := "SELECT true FROM " + r.leases + + " WHERE key = $1 AND owner_id = $2 AND expires_at >= $3 FOR UPDATE" + record(fenceSQL) + var held bool + if err := tx.QueryRow(ctx, fenceSQL, execution.Key, execution.OwnerID, time.Now().UTC()).Scan(&held); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return publication.ErrBundlePointerConflict + } + return fmt.Errorf("publication postgres: check publish lease: %w", err) + } + if !held { + return publication.ErrBundlePointerConflict + } + + lockSQL := "SELECT true FROM " + r.executions + " WHERE id = $1 FOR UPDATE" + record(lockSQL) + var exists bool + if err := tx.QueryRow(ctx, lockSQL, execution.ID).Scan(&exists); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return publication.ErrBundlePointerConflict + } + return fmt.Errorf("publication postgres: check publish execution: %w", err) + } + if !exists { + return publication.ErrBundlePointerConflict + } + + swapped, casSQL, err := compareAndSwapPointer(ctx, tx, r.pointers, name, expected, execution.ID, execution.UpdatedAt) + if err != nil { + return err + } + record(casSQL) + if !swapped { + var current string + pointerReadSQL := "SELECT execution_id FROM " + r.pointers + " WHERE pointer_key = $1" + record(pointerReadSQL) + if err := tx.QueryRow(ctx, pointerReadSQL, attempt.PointerKey).Scan(¤t); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return publication.ErrBundlePointerConflict + } + return fmt.Errorf("publication postgres: read contested pointer: %w", err) + } + if current != execution.ID { + return publication.ErrBundlePointerConflict + } + var matches bool + payloadReadSQL := "SELECT payload = $2::jsonb FROM " + r.executions + " WHERE id = $1" + record(payloadReadSQL) + if err := tx.QueryRow(ctx, payloadReadSQL, execution.ID, params[len(params)-1]).Scan(&matches); err != nil { + return fmt.Errorf("publication postgres: compare publish payload: %w", err) + } + if !matches { + return publication.ErrBundlePointerConflict + } + attempt.Converged = true + } else { + updateSQL := "UPDATE " + r.executions + " SET " + publishExecutionSet + " WHERE id = $1" + record(updateSQL) + if _, err := tx.Exec(ctx, updateSQL, params...); err != nil { + return fmt.Errorf("publication postgres: publish execution: %w", err) + } + } + + if gate := publishGate.Load(); gate != nil { + if err := (*gate)(attempt); err != nil { + return err + } + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("publication postgres: commit publish: %w", err) + } + return nil +} diff --git a/internal/backends/postgres/internal/publication/publish_atomicity_test.go b/internal/backends/postgres/internal/publication/publish_atomicity_test.go new file mode 100644 index 000000000..2f2f31e95 --- /dev/null +++ b/internal/backends/postgres/internal/publication/publish_atomicity_test.go @@ -0,0 +1,1308 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/dataframe/publication" +) + +var pubvisResourceNamePattern = regexp.MustCompile(`^verify_pubvis_it_[0-9]+_[0-9a-f]+$`) + +var ( + pubvisOwnedMu sync.Mutex + pubvisOwned = map[string]string{} +) + +func pubvisRegisterOwned(db, schema string) { + pubvisOwnedMu.Lock() + defer pubvisOwnedMu.Unlock() + pubvisOwned[db] = schema +} + +func auditPublishVisibilityLeftovers() error { + if strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) == "" { + return nil + } + pubvisOwnedMu.Lock() + owned := make(map[string]string, len(pubvisOwned)) + for db, schema := range pubvisOwned { + owned[db] = schema + } + pubvisOwnedMu.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL"))) + if err != nil { + return fmt.Errorf("audit connect: %w", err) + } + defer admin.Close() + + for db, schema := range owned { + if !strings.HasPrefix(db, "verify_pubvis_it_") { + return fmt.Errorf("ownership registry holds foreign database %q", db) + } + var count int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + return fmt.Errorf("audit owned database %s: %w", db, err) + } + if count != 0 { + return fmt.Errorf("leftover owned database %s (owned schema %s) after all cleanups", db, schema) + } + } + + var leftoverDatabases []string + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE left(datname, $1) = $2", len("verify_pubvis_"), "verify_pubvis_") + if err != nil { + return fmt.Errorf("audit database namespace scan: %w", err) + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return fmt.Errorf("audit database namespace scan: %w", err) + } + leftoverDatabases = append(leftoverDatabases, name) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("audit database namespace scan: %w", err) + } + rows.Close() + if len(leftoverDatabases) != 0 { + return fmt.Errorf("leftover databases under this suite's fixture namespace verify_pubvis_* after all cleanups: %v", leftoverDatabases) + } + + var leftoverSchemas []string + schemaRows, err := admin.Query(ctx, "SELECT nspname FROM pg_namespace WHERE left(nspname, $1) = $2", len("verify_pubvis_"), "verify_pubvis_") + if err != nil { + return fmt.Errorf("audit schema namespace scan: %w", err) + } + defer schemaRows.Close() + for schemaRows.Next() { + var name string + if err := schemaRows.Scan(&name); err != nil { + return fmt.Errorf("audit schema namespace scan: %w", err) + } + leftoverSchemas = append(leftoverSchemas, name) + } + if err := schemaRows.Err(); err != nil { + return fmt.Errorf("audit schema namespace scan: %w", err) + } + schemaRows.Close() + if len(leftoverSchemas) != 0 { + return fmt.Errorf("leftover schemas under this suite's fixture namespace verify_pubvis_* after all cleanups: %v", leftoverSchemas) + } + return nil +} + +type pubvisService struct { + ctx context.Context + admin *pgxpool.Pool + pub *pgxpool.Pool + pubB *pgxpool.Pool + reader *pgxpool.Pool + schema string + regA *Registry + regB *Registry + regR *Registry + scrub func(any) string + fail func(step string, err error) +} + +func newPubvisService(t *testing.T) *pubvisService { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { t.Fatalf("%s: %s", step, scrub(err)) } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("verify_pubvis_it_%d_%s", time.Now().UnixNano(), randomHex()) + if !pubvisResourceNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + pubvisRegisterOwned(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + poolA, err := newOwnedPool(ctx, dsn) + if err != nil { + fail("connect publisher pool A", err) + } + t.Cleanup(poolA.Close) + poolB, err := newOwnedPool(ctx, dsn) + if err != nil { + fail("connect publisher pool B", err) + } + t.Cleanup(poolB.Close) + poolR, err := newOwnedPool(ctx, dsn) + if err != nil { + fail("connect reader pool", err) + } + t.Cleanup(poolR.Close) + + schema := fmt.Sprintf("verify_pubvis_it_%d_%s", time.Now().UnixNano(), randomHex()) + if !pubvisResourceNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := poolR.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + pubvisRegisterOwned(dbName, schema) + t.Cleanup(func() { + if _, err := poolR.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + s := &pubvisService{ctx: ctx, admin: admin, pub: poolA, pubB: poolB, reader: poolR, schema: schema, scrub: scrub, fail: fail} + s.migrate(t) + s.regA = s.registry(t, poolA) + s.regB = s.registry(t, poolB) + s.regR = s.registry(t, poolR) + return s +} + +func (s *pubvisService) registry(t *testing.T, pool *pgxpool.Pool) *Registry { + t.Helper() + registry, err := New(pool, s.schema) + if err != nil { + s.fail("build registry", err) + } + return registry +} + +func (s *pubvisService) migrate(t *testing.T) { + t.Helper() + history := mustOwnerHistorySubset(t, 1, 4) + runner, err := storepostgres.NewMigrationRunner(s.reader, s.schema, history) + if err != nil { + s.fail("new migration runner", err) + } + result, err := runner.Migrate(s.ctx) + if err != nil { + s.fail("migrate catalog", err) + } + if result.LatestVersion != 4 || len(result.Applied) != 2 { + t.Fatalf("applied history = %+v, want ledger + publication_bundle_catalog", result.Applied) + } +} + +func (s *pubvisService) wText(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) string { + t.Helper() + var value string + if err := pool.QueryRow(s.ctx, sql, args...).Scan(&value); err != nil { + s.fail("witness "+sql, err) + } + return value +} + +func (s *pubvisService) wBool(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) bool { + t.Helper() + var value bool + if err := pool.QueryRow(s.ctx, sql, args...).Scan(&value); err != nil { + s.fail("witness "+sql, err) + } + return value +} + +func (s *pubvisService) wInt64(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) int64 { + t.Helper() + var value int64 + if err := pool.QueryRow(s.ctx, sql, args...).Scan(&value); err != nil { + s.fail("witness "+sql, err) + } + return value +} + +func (s *pubvisService) backendPID(t *testing.T, pool *pgxpool.Pool, role string) uint32 { + t.Helper() + raw := s.wText(t, pool, "SELECT pg_backend_pid()::text") + var pid uint32 + if _, err := fmt.Sscanf(raw, "%d", &pid); err != nil { + s.fail("parse pg_backend_pid "+role, err) + } + if pid == 0 { + t.Fatalf("%s backend pid is zero", role) + } + return pid +} + +type pubvisPointerRow struct { + PointerKey string + NameHex string + Execution string + UpdatedAt string +} + +func (s *pubvisService) pointerRow(t *testing.T, name string) *pubvisPointerRow { + t.Helper() + var row pubvisPointerRow + err := s.reader.QueryRow(s.ctx, + "SELECT pointer_key, encode(name_bytes, 'hex'), execution_id, "+ + `to_char(updated_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') `+ + "FROM "+s.qualified(BundlePointersTable)+" WHERE pointer_key = $1", + pointerKey(name)).Scan(&row.PointerKey, &row.NameHex, &row.Execution, &row.UpdatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + s.fail("pointer row witness", err) + } + return &row +} + +func (s *pubvisService) pointerValue(t *testing.T, name string) pubvisPointerRow { + t.Helper() + row := s.pointerRow(t, name) + if row == nil { + t.Fatalf("pointer row vanished for %q", name) + } + return *row +} + +func (s *pubvisService) payloadText(t *testing.T, id string) string { + t.Helper() + return s.wText(t, s.reader, + "SELECT payload::text FROM "+s.qualified(BundleExecutionsTable)+" WHERE id = $1", id) +} + +func pubvisSHA256Hex(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} + +func (s *pubvisService) qualified(table string) string { + return pgx.Identifier{s.schema, table}.Sanitize() +} + +func (s *pubvisService) activity(t *testing.T, pid uint32) (string, string) { + t.Helper() + var state, query string + if err := s.reader.QueryRow(s.ctx, + "SELECT state, query FROM pg_stat_activity WHERE pid = $1", pid).Scan(&state, &query); err != nil { + s.fail("pg_stat_activity witness", err) + } + return state, query +} + +const ( + pubvisOwnerA = "pubvis-owner-a" + pubvisOwnerB = "pubvis-owner-b" + pubvisLeaseEnd = "2099-01-01T00:00:00Z" +) + +var ( + pubvisCreated = time.Date(2026, 11, 9, 10, 0, 0, 123456789, time.UTC) + pubvisRunUpdated = time.Date(2026, 11, 9, 10, 0, 30, 987654321, time.UTC) + pubvisReady = time.Date(2026, 11, 9, 10, 0, 45, 555555555, time.UTC) + pubvisPublishedAt = time.Date(2026, 11, 9, 10, 2, 0, 112233445, time.UTC) + pubvisLeaseUntil = time.Date(2099, 1, 1, 0, 0, 0, 0, time.UTC) +) + +func pubvisIdentity() publication.BundleIdentity { + return publication.BundleIdentity{ + Name: "pa:nel", + TranslationVersion: "9:v", + Project: "pubvis-project", + DatasetGeneration: "gen-9", + RecipeDigest: "sha256:recipe-pubvis", + SchemaDigest: "sha256:schema-pubvis", + ReceiptID: "receipt-pubvis-1", + ScopeDigest: "sha256:scope-pubvis", + EngineVersion: "loom-engine-pubvis", + AuthScopeMode: "project", + AuthResourcePaths: []string{"fhir/Observation", "fhir/Patient"}, + } +} + +func pubvisExecution(id string, published bool, owner string) publication.BundleExecution { + identity := pubvisIdentity() + updatedAt := pubvisRunUpdated + if published { + updatedAt = pubvisPublishedAt + } + execution := publication.BundleExecution{ + ID: id, + Key: "bundle-" + id, + BundleIdentity: identity, + State: publication.BundleRunning, + CreatedAt: pubvisCreated, + UpdatedAt: updatedAt, + ReadyAt: &pubvisReady, + OwnerID: owner, + } + if published { + execution.State = publication.BundlePublished + execution.PublishedAt = &pubvisPublishedAt + } + state := execution.State + buildOutput := func(name, physical string, rows, bytes int64) publication.BundleOutputRecord { + record := publication.BundleOutputRecord{ + Name: name, + PhysicalTable: physical, + Selector: execution.Selector(name), + RowCount: rows, + ByteCount: bytes, + State: state, + } + if published { + verified := pubvisPublishedAt + record.VerifiedAt = &verified + } + return record + } + outA := buildOutput("out-a", "phys_pubvis_out_a", 1234, 567890) + outA.Columns = []publication.PhysicalColumn{ + {ID: "c1", Name: "subject", SemanticPath: "Patient.id", ClickHouse: "String", LogicalType: "ResourceIdentifier", Provenance: publication.ColumnExplicit, LoomOwned: true}, + {ID: "c2", Name: "value", ClickHouse: "Nullable(Float64)", LogicalType: "Quantity", Nullable: true, Repeated: true}, + } + outA.SourceRow = &publication.SourceRowMetadata{ResourceType: "Patient", IDColumn: "subject"} + outB := buildOutput("out-b", "phys_pubvis_out_b", 7, 4200) + execution.Outputs = []publication.BundleOutputRecord{outA, outB} + return execution +} + +func pubvisPointerName() string { return pubvisIdentity().PointerName() } + +var pubvisPayloadGoldenSHA256 = map[string]string{ + "pubvis-exec-c": "86887eb9943381092f7188d1b8413c7cd978ef0f26f602c1fd16f6f28f934ce6", + "pubvis-exec-e": "1fbe8e253ce9d56d1ceaeeb874b62f706d6b83c5868ee34e5ac516ca00fe1f6b", + "pubvis-exec-a1": "7e6f2d0605f000aa45092794c84c22a3ed07974a884c825382d5d47616021e3b", + "pubvis-exec-a2": "4952b494ccc0411aa6a75ec11b6d73537e2987fd18de6579b8ae557ff30f5c22", + "pubvis-exec-a3": "eccaa7c904afd898ce25f2250607012c73794bc1cca71d94149852811839506d", + "pubvis-exec-a4": "a0e1c1827a9ff03a29f9c22c6161f7094988d451279ced1520009d0b66aee16b", + "pubvis-exec-a5": "d6e0b2e63ae2f3bce0b7da2caf7dc52b9a7e3e8fd854879f660d13a37cfb0be6", + "pubvis-exec-a6": "37df874d48575881ac18b6cccc289ecccb1caff3aa58054c6a218e24295ba353", + "pubvis-exec-a7": "1e63f2ca86e26d761940df49ead0790f0091967e13654fb02f965f95cf7f8dc5", + "pubvis-exec-a8": "8634ca3c5716638f6c47ff4e183ab5f4caf1b0206d84554c4418ac648f25418d", + "pubvis-exec-a9": "8e69135790e96fb4dcff6acbd9f6150b6012a32b885cdc983579bf8cc5d5e66b", + "pubvis-exec-a10": "def4c3ae5d374060ac8736ee4107bb364ef11f0db7c1ba4fe2989b8c053eb7c5", + "pubvis-exec-a11": "8e4be67e86be743a7c3f6c97f4da9689f7cf080a94eaf07113d28118b946d4b7", + "pubvis-exec-a12": "787e8776fcce894b126a9390c9291b3e686f81b2ef4c5c983d0454aa12e133c8", + "pubvis-exec-b1": "1c4e607f4c9fa889716cba380e0fa4210d4499587e15b3f3dc9cccbdbf0ef2c6", + "pubvis-exec-b2": "385596457a9ba6c88d57f76ee4135d8b1bcd924ef97b6e3126697b0d89eaac8f", + "pubvis-exec-b3": "3796e938f6cc0b28ea8c59e13843482c60f251908c074251afaf6924158b3664", + "pubvis-exec-d-r01-a": "8da08c6f92cbdbfc0a3f53a8fadb068b44080a3f3fbbe38bb65367de615881c8", + "pubvis-exec-d-r01-b": "d650cc34b4bb1d13ee79802d804f8a9b65cf9509825a3667b16fd5827246c346", + "pubvis-exec-d-r02-a": "f546f7824de877ff6ed622bbf7a0a6665759d4a67c485f9d276c36eed3d4c1c3", + "pubvis-exec-d-r02-b": "5bf186433d3bc4358f88b04a506838c2f57857239b8838dbf1b5812df3ffda52", + "pubvis-exec-d-r03-a": "d4b838da0f93a0de5559fa9a8381f82c904cf5af88d20e84458b8aab47a704ea", + "pubvis-exec-d-r03-b": "8c98b91fd65e62c74e0d28169d2f990ed46799eff831211454ecf6f1d0d7d024", + "pubvis-exec-d-r04-a": "d6ae7732cb28c360872518977de7dada5dad563c216942a3cf31ac1b5a27fb0b", + "pubvis-exec-d-r04-b": "a732ad412e6d4ff130e9253685f94cda13e9a53c7acf28fdc9450c89457fdf0f", + "pubvis-exec-d-r05-a": "79ee74d1088eddc9eda84fc6956223bc0e350672578cedf188630c2926959ded", + "pubvis-exec-d-r05-b": "dce5589bb3b5b97edf14efa81b435778871b6c464fb424b8ffe53a9f76ea4059", + "pubvis-exec-d-r06-a": "6e3a46f762070040617809991c07d797db7750d929594d7b800f670dbd18d398", + "pubvis-exec-d-r06-b": "20a4c962bdd6e93c2a28e8524e7ef91ea3dc9487a59336a1847ffeea1a7efac1", + "pubvis-exec-d-r07-a": "239dab003d9159f9fcba733cba8db936a6cf7230372a9a0aab2482582e4bd9c6", + "pubvis-exec-d-r07-b": "f315246bf2489e9e4af3a3ed977d1b2269dabdce4ac3affaee22a68e8d17c5a8", + "pubvis-exec-d-r08-a": "b448df7ab59c4e11476bc44746731c9f009092163f74f0f70fca630283996b42", + "pubvis-exec-d-r08-b": "45d13ce4c82ab1ac17ba278b8b3cf8a9c252286d8b6aeb0ec3063ab9c6192d72", + "pubvis-exec-d-r09-a": "d07aee22f1b2002c908b47ee4e494f90cc5e17d24746b92fe1caf6ee7237884c", + "pubvis-exec-d-r09-b": "4b504423b063e03e84581ef582852dfbe33479bd140e66482b649537810deab8", + "pubvis-exec-d-r10-a": "9b70eb587481cd1c07d2e7e69db2c9be529b27649cedfcd72051b9da47614388", + "pubvis-exec-d-r10-b": "ad22c26f50014d80ef24720f9d8eec4d898e7dfae01a5be227cc68cf76105207", + "pubvis-exec-d-r11-a": "b11d5fe6eb7f2a545da01a7cc594d5586945db7aa94403055d1e97e36acfe258", + "pubvis-exec-d-r11-b": "e41cf9e81d0cbdc5e3da0a1bcde14c5f0433e68fbe30ccbdfc0ac1008bc74b59", + "pubvis-exec-d-r12-a": "45ba181aafc8ca1aeb41916afd4cbb4d9191aa8834bb6b6b3b225f39df7c8173", + "pubvis-exec-d-r12-b": "b47944c243ce7ea639d415fcbb060ab4e6175059130950b338c57d01ffeb3a83", + "pubvis-exec-d-s01-a": "73e1555e6ce658e4f7f44ca484993d6345b0e934e0a771d2e80e7e920727fce1", + "pubvis-exec-d-s01-b": "1287f1a519c1be245f73815b89e510d34f2d162edd5e8fd57cfdcc2d950af008", + "pubvis-exec-d-s02-a": "fd6d82ba3c0f6586aa3240a9a73019e3056f39ad948412924e87e9f82a3ee1ee", + "pubvis-exec-d-s02-b": "56c58b81a2d89db7128bbd99b5f57efb069fc5ed7dd6d3f6f763c5107f2fdb5b", + "pubvis-exec-d-s03-a": "02f39fdda136ea21a1c59500674be16f6f86418e8c1dd8eafd6633b168976ac2", + "pubvis-exec-d-s03-b": "2c3e77ab1a129c4eaf40a0bce682b234f8188a4aabfa47bcd14d6cd47a88a5e2", + "pubvis-exec-d-s04-a": "20d47c404f0e6cf80e8c4292ed490d20bdda26a577d7b0ffdc49e10b68c91f31", + "pubvis-exec-d-s04-b": "e301115f4f7b8cd8917deec689662b8e21924864f0fd4d458a6082243a4ea215", + "pubvis-exec-d-s05-a": "13e56112bf5e904f40c04a800e8a7d6a09b634f83376d8a9b2e05d75f3a819ff", + "pubvis-exec-d-s05-b": "75ee1b8cb6fedd26934af686fe0521dece6f04eb10f784f8f2dd4fb99678b358", + "pubvis-exec-d-s06-a": "728926b87c14895d0efd2e92b6606eaac1faa957fc252da4e064eff4a9fc6c5a", + "pubvis-exec-d-s06-b": "12f094813346d4a949406edcbc8a1862290870ba22779339c6401fe0d4cafc62", + "pubvis-exec-d-s07-a": "6dea14edc91ce2502ae8223d4fd291e5115d34caacebfe507f9bbfd2900547b7", + "pubvis-exec-d-s07-b": "c452d0d6022891ecb32adcf29d93c2ba32c1e1087066595bc5aa1117e66deeb2", + "pubvis-exec-d-s08-a": "d3a34af0f8d4e046bf9a138863e08b44695d6d37c09891e1cb39002ded3adf29", + "pubvis-exec-d-s08-b": "7ca472316f45b25b9c737e14bcfe05fc1b79d717a1e81706a7d07854f22b085a", + "pubvis-exec-d-s09-a": "aa599df3947a7a5e70ff5706738362659ac5e6f7d30b90d10e18b5f27c09f6b8", + "pubvis-exec-d-s09-b": "01e226c23fa91f4061d5bb736a8e986ddf88b28829cc237b69b80aacc0339528", + "pubvis-exec-d-s10-a": "63d3d533bec74ce87d8b16b020e1468d091110abef4b4d4de7c5eafc4c81bdf6", + "pubvis-exec-d-s10-b": "fab861cad1a8be4fd57f18002166ef5a5b89d725eb0d1426e2e952a7f808e470", + "pubvis-exec-d-s11-a": "135716f27fca7695a8f535979b93d89426207003c609a76128ae7b00515ad0af", + "pubvis-exec-d-s11-b": "6170e40cd3ae993091f25bcfef93509fb5b1fafab49cfa60a165729295a1aff2", + "pubvis-exec-d-s12-a": "acb66f303fd581f82512d8d60415b71eda822a83584a77f48286e03e8a42425e", + "pubvis-exec-d-s12-b": "14334971ccc4dfd4d6fb890f4a2c64f095286d6a4352e4c3b03dcc8936c8864d", +} + +func pubvisWantPayloadSHA(t *testing.T, id string) string { + t.Helper() + want, ok := pubvisPayloadGoldenSHA256[id] + if !ok { + t.Fatalf("no golden payload sha256 registered for %q", id) + } + return want +} + +func (s *pubvisService) seed(t *testing.T, reg *Registry, owner string, id string) publication.BundleExecution { + t.Helper() + execution := pubvisExecution(id, false, owner) + claimed, err := reg.AcquireBundleLease(s.ctx, execution.Key, owner, pubvisLeaseUntil) + if err != nil || !claimed { + s.fail("acquire lease "+execution.Key, fmt.Errorf("claimed=%v err=%w", claimed, err)) + } + if err := reg.SaveExecution(s.ctx, execution, owner); err != nil { + s.fail("save execution "+id, err) + } + return execution +} + +func (s *pubvisService) publishClean(t *testing.T, reg *Registry, name, expected, id, owner string) publication.BundleExecution { + t.Helper() + published := pubvisExecution(id, true, owner) + if err := reg.PublishExecution(s.ctx, name, expected, published); err != nil { + s.fail("clean publish "+id, err) + } + return published +} + +type pubvisGate struct { + arrived chan publishAttempt + release chan error +} + +func newPubvisGate() *pubvisGate { + return &pubvisGate{arrived: make(chan publishAttempt, 1), release: make(chan error, 1)} +} + +func pubvisArmGate(t *testing.T, gate *pubvisGate) { + t.Helper() + var once sync.Once + fn := func(attempt publishAttempt) error { + engaged := false + once.Do(func() { engaged = true }) + if !engaged { + return nil + } + attempt.Statements = append([]string(nil), attempt.Statements...) + gate.arrived <- attempt + return <-gate.release + } + previous := publishGate.Swap(&fn) + t.Cleanup(func() { + publishGate.Store(previous) + select { + case gate.release <- nil: + default: + } + select { + case <-gate.arrived: + default: + } + }) +} + +func pubvisWaitAttempt(t *testing.T, gate *pubvisGate) publishAttempt { + t.Helper() + select { + case attempt := <-gate.arrived: + return attempt + case <-time.After(30 * time.Second): + t.Fatal("publish gate never fired within 30s: the gate call is missing or publish returned early") + return publishAttempt{} + } +} + +var errPubvisInjected = errors.New("injected publish transaction failure") + +func pubvisGoldenFence(fqLeases string) string { + return "SELECT true FROM " + fqLeases + " WHERE key = $1 AND owner_id = $2 AND expires_at >= $3 FOR UPDATE" +} + +func pubvisGoldenLock(fqExec string) string { + return "SELECT true FROM " + fqExec + " WHERE id = $1 FOR UPDATE" +} + +func pubvisGoldenCAS(fqPointers string) string { + return "INSERT INTO " + fqPointers + " (pointer_key, name_bytes, execution_id, updated_at)" + + " VALUES ($1, $2, $3, $4)" + + " ON CONFLICT (pointer_key) DO UPDATE SET execution_id = $3, updated_at = $4" + + " WHERE " + BundlePointersTable + ".execution_id = $5" +} + +func pubvisGoldenUpdate(fqExec string) string { + return "UPDATE " + fqExec + " SET execution_key = $2, project = $3, dataset_generation = $4, recipe_name = $5, " + + "translation_version = $6, state = $7, created_at = $8, updated_at = $9, " + + "ready_at = $10, published_at = $11, payload = $12 WHERE id = $1" +} + +func pubvisGoldenPointerRead(fqPointers string) string { + return "SELECT execution_id FROM " + fqPointers + " WHERE pointer_key = $1" +} + +func pubvisGoldenPayloadRead(fqExec string) string { + return "SELECT payload = $2::jsonb FROM " + fqExec + " WHERE id = $1" +} + +func (s *pubvisService) goldenPublishStatements(fqLeases, fqExec, fqPointers string) []string { + return []string{ + pubvisGoldenFence(fqLeases), + pubvisGoldenLock(fqExec), + pubvisGoldenCAS(fqPointers), + pubvisGoldenUpdate(fqExec), + } +} + +func (s *pubvisService) goldenConvergedStatements(fqLeases, fqExec, fqPointers string) []string { + return []string{ + pubvisGoldenFence(fqLeases), + pubvisGoldenLock(fqExec), + pubvisGoldenCAS(fqPointers), + pubvisGoldenPointerRead(fqPointers), + pubvisGoldenPayloadRead(fqExec), + } +} + +func pubvisEqualStrings(t *testing.T, label string, got, want []string) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("%s: %d statements recorded %q, want %d %q", label, len(got), got, len(want), want) + } + for i := range got { + if got[i] != want[i] { + t.Fatalf("%s: statement[%d] drifted\n got: %q\nwant: %q", label, i, got[i], want[i]) + } + } +} + +func pubvisExpectExactPublished(t *testing.T, s *pubvisService, want publication.BundleExecution) { + t.Helper() + got, err := s.regR.GetExecution(s.ctx, want.ID) + if err != nil { + s.fail("get published execution "+want.ID, err) + } + check := func(field string, gotV, wantV any) { + t.Helper() + if !reflect.DeepEqual(gotV, wantV) { + t.Fatalf("published execution %s.%s = %#v, want %#v", want.ID, field, gotV, wantV) + } + } + check("ID", got.ID, want.ID) + check("Key", got.Key, want.Key) + check("State", got.State, want.State) + check("Project", got.Project, want.Project) + check("DatasetGeneration", got.DatasetGeneration, want.DatasetGeneration) + check("Name", got.Name, want.Name) + check("TranslationVersion", got.TranslationVersion, want.TranslationVersion) + check("RecipeDigest", got.RecipeDigest, want.RecipeDigest) + check("SchemaDigest", got.SchemaDigest, want.SchemaDigest) + check("ScopeDigest", got.ScopeDigest, want.ScopeDigest) + check("ReceiptID", got.ReceiptID, want.ReceiptID) + check("EngineVersion", got.EngineVersion, want.EngineVersion) + check("AuthScopeMode", got.AuthScopeMode, want.AuthScopeMode) + check("AuthResourcePaths", got.AuthResourcePaths, want.AuthResourcePaths) + check("OwnerID", got.OwnerID, want.OwnerID) + check("Error", got.Error, want.Error) + check("FailureCode", got.FailureCode, want.FailureCode) + check("CreatedAt", got.CreatedAt.Equal(want.CreatedAt), true) + check("UpdatedAt", got.UpdatedAt.Equal(want.UpdatedAt), true) + check("ReadyAt set", got.ReadyAt != nil && got.ReadyAt.Equal(*want.ReadyAt), true) + check("PublishedAt set", got.PublishedAt != nil && got.PublishedAt.Equal(*want.PublishedAt), true) + if len(got.Outputs) != len(want.Outputs) { + t.Fatalf("published outputs = %d, want %d", len(got.Outputs), len(want.Outputs)) + } + for i := range want.Outputs { + w, g := want.Outputs[i], got.Outputs[i] + check(fmt.Sprintf("outputs[%d].Name", i), g.Name, w.Name) + check(fmt.Sprintf("outputs[%d].PhysicalTable", i), g.PhysicalTable, w.PhysicalTable) + check(fmt.Sprintf("outputs[%d].RowCount", i), g.RowCount, w.RowCount) + check(fmt.Sprintf("outputs[%d].ByteCount", i), g.ByteCount, w.ByteCount) + check(fmt.Sprintf("outputs[%d].State", i), g.State, w.State) + check(fmt.Sprintf("outputs[%d].Selector", i), g.Selector, w.Selector) + check(fmt.Sprintf("outputs[%d].Columns", i), g.Columns, w.Columns) + check(fmt.Sprintf("outputs[%d].SourceRow", i), g.SourceRow, w.SourceRow) + check(fmt.Sprintf("outputs[%d].VerifiedAt set", i), g.VerifiedAt != nil && g.VerifiedAt.Equal(*w.VerifiedAt), true) + } +} + +func TestPublishExecutionAtomicity(t *testing.T) { + t.Run("A_reader_sees_old_pointer_until_commit", func(t *testing.T) { + s := newPubvisService(t) + name := pubvisPointerName() + fqLeases, fqExec, fqPointers := s.qualified(BundleLeasesTable), s.qualified(BundleExecutionsTable), s.qualified(BundlePointersTable) + + s.seed(t, s.regA, pubvisOwnerA, "pubvis-exec-a0") + a0 := s.publishClean(t, s.regA, name, "", "pubvis-exec-a0", pubvisOwnerA) + pubvisExpectExactPublished(t, s, a0) + + const rounds = 12 + for round := 1; round <= rounds; round++ { + id := fmt.Sprintf("pubvis-exec-a%d", round) + previous := fmt.Sprintf("pubvis-exec-a%d", round-1) + s.seed(t, s.regA, pubvisOwnerA, id) + published := pubvisExecution(id, true, pubvisOwnerA) + + pubPID := s.backendPID(t, s.pub, "publisher") + readPID := s.backendPID(t, s.reader, "reader") + if pubPID == readPID { + t.Fatalf("publisher and reader share backend pid %d, want two real PG connections", pubPID) + } + + gate := newPubvisGate() + pubvisArmGate(t, gate) + done := make(chan error, 1) + go func() { done <- s.regA.PublishExecution(s.ctx, name, previous, published) }() + + attempt := pubvisWaitAttempt(t, gate) + if attempt.BackendPID != pubPID { + t.Fatalf("round %d: publish transaction backend pid = %d, want the publisher pool backend %d", round, attempt.BackendPID, pubPID) + } + if attempt.BackendPID == readPID { + t.Fatalf("round %d: publish shares backend %d with the reader", round, readPID) + } + pubvisEqualStrings(t, fmt.Sprintf("round %d statements", round), attempt.Statements, + s.goldenPublishStatements(fqLeases, fqExec, fqPointers)) + + pointer, err := s.regR.GetPointer(s.ctx, name) + if err != nil || pointer.ExecutionID != previous { + t.Fatalf("round %d: reader saw pointer %+v err=%v during publish, want %q", round, pointer, err, previous) + } + reading, err := s.regR.GetExecution(s.ctx, id) + if err != nil || reading.State != publication.BundleRunning { + t.Fatalf("round %d: reader saw execution state %q err=%v during publish, want RUNNING", round, reading.State, err) + } + if row := s.pointerRow(t, name); row == nil || row.Execution != previous { + t.Fatalf("round %d: SQL witness of pointer during publish = %+v, want %q", round, row, previous) + } + if state := s.wText(t, s.reader, + "SELECT payload->>'state' FROM "+fqExec+" WHERE id = $1", id); state != "RUNNING" { + t.Fatalf("round %d: SQL witness of execution during publish = %q, want RUNNING", round, state) + } + txState, lastQuery := s.activity(t, attempt.BackendPID) + if txState != "idle in transaction" { + t.Fatalf("round %d: publish backend state = %q, want idle in transaction", round, txState) + } + if lastQuery != pubvisGoldenUpdate(fqExec) { + t.Fatalf("round %d: publish backend last query drifted from the golden UPDATE:\n got: %q", round, lastQuery) + } + + gate.release <- nil + if err := <-done; err != nil { + s.fail(fmt.Sprintf("round %d publish", round), err) + } + + pointer, err = s.regR.GetPointer(s.ctx, name) + if err != nil || pointer.ExecutionID != id { + t.Fatalf("round %d: reader saw pointer %+v err=%v after commit, want %q", round, pointer, err, id) + } + pubvisExpectExactPublished(t, s, published) + if got := pubvisSHA256Hex(s.payloadText(t, id)); got != pubvisWantPayloadSHA(t, id) { + t.Fatalf("round %d: committed payload digest = %s, want golden %s", round, got, pubvisWantPayloadSHA(t, id)) + } + row := s.pointerRow(t, name) + if row == nil { + t.Fatalf("round %d: pointer row vanished after commit", round) + } + if row.Execution != id { + t.Fatalf("round %d: pointer row execution = %q, want %q", round, row.Execution, id) + } + t.Logf("OBS A round %d: old pointer held until COMMIT; full exact execution after (pids %d/%d)", round, attempt.BackendPID, readPID) + } + + row := s.pointerRow(t, name) + wantName := pubvisPointerName() + sum := sha256.Sum256([]byte(wantName)) + if row == nil || row.PointerKey != hex.EncodeToString(sum[:]) || row.NameHex != hex.EncodeToString([]byte(wantName)) { + t.Fatalf("pointer literals = %+v, want key %s name hex %s", row, hex.EncodeToString(sum[:]), hex.EncodeToString([]byte(wantName))) + } + if n := s.wInt64(t, s.reader, "SELECT count(*) FROM "+fqPointers); n != 1 { + t.Fatalf("pointer rows = %d, want exactly 1 after the whole chain", n) + } + }) + + t.Run("B_injected_failure_exposes_neither_half", func(t *testing.T) { + s := newPubvisService(t) + name := pubvisPointerName() + s.seed(t, s.regA, pubvisOwnerA, "pubvis-exec-b0") + s.publishClean(t, s.regA, name, "", "pubvis-exec-b0", pubvisOwnerA) + + const rounds = 3 + for round := 1; round <= rounds; round++ { + id := fmt.Sprintf("pubvis-exec-b%d", round) + s.seed(t, s.regA, pubvisOwnerA, id) + published := pubvisExecution(id, true, pubvisOwnerA) + + prePointer := s.pointerRow(t, name) + prePayload := s.payloadText(t, id) + if pubvisSHA256Hex(prePayload) == pubvisWantPayloadSHA(t, id) { + t.Fatalf("round %d: fixture already carries the published payload", round) + } + + gate := newPubvisGate() + pubvisArmGate(t, gate) + done := make(chan error, 1) + go func() { done <- s.regA.PublishExecution(s.ctx, name, "pubvis-exec-b0", published) }() + + attempt := pubvisWaitAttempt(t, gate) + pointer, err := s.regR.GetPointer(s.ctx, name) + if err != nil || pointer.ExecutionID != "pubvis-exec-b0" { + t.Fatalf("round %d: reader saw pointer %+v err=%v during the doomed publish", round, pointer, err) + } + gate.release <- errPubvisInjected + err = <-done + if !errors.Is(err, errPubvisInjected) { + t.Fatalf("round %d: injected failure did not surface, got %v", round, err) + } + + postPointer := s.pointerRow(t, name) + if postPointer == nil || *postPointer != *prePointer { + t.Fatalf("round %d: pointer row drifted across the abort: %+v vs %+v", round, postPointer, prePointer) + } + postPayload := s.payloadText(t, id) + if postPayload != prePayload { + t.Fatalf("round %d: execution payload drifted across the abort (first difference at byte %d)", + round, firstDiff(prePayload, postPayload)) + } + if state := s.wText(t, s.reader, + "SELECT payload->>'state' FROM "+s.qualified(BundleExecutionsTable)+" WHERE id = $1", id); state != "RUNNING" { + t.Fatalf("round %d: aborted execution state = %q, want RUNNING", round, state) + } + t.Logf("OBS B round %d: crash window after statements pid=%d rolled back; rows byte-identical", round, attempt.BackendPID) + } + + const finalID = "pubvis-exec-b3" + published := pubvisExecution(finalID, true, pubvisOwnerA) + if err := s.regA.PublishExecution(s.ctx, name, "pubvis-exec-b0", published); err != nil { + s.fail("post-abort convergence publish", err) + } + pubvisExpectExactPublished(t, s, published) + if got := pubvisSHA256Hex(s.payloadText(t, finalID)); got != pubvisWantPayloadSHA(t, finalID) { + t.Fatalf("post-abort payload digest = %s, want golden %s", got, pubvisWantPayloadSHA(t, finalID)) + } + row := s.pointerRow(t, name) + if row == nil || row.Execution != finalID { + t.Fatalf("post-abort pointer row = %+v, want %q", row, finalID) + } + + absentID := "pubvis-exec-b9" + s.seed(t, s.regA, pubvisOwnerA, absentID) + absentName := pubvisPointerName() + "|absent" + gate := newPubvisGate() + pubvisArmGate(t, gate) + done := make(chan error, 1) + go func() { + done <- s.regA.PublishExecution(s.ctx, absentName, "", pubvisExecution(absentID, true, pubvisOwnerA)) + }() + pubvisWaitAttempt(t, gate) + gate.release <- errPubvisInjected + if err := <-done; !errors.Is(err, errPubvisInjected) { + t.Fatalf("absent-pointer abort did not surface the injection, got %v", err) + } + if row := s.pointerRow(t, absentName); row != nil { + t.Fatalf("aborted first publish left a pointer row behind: %+v", row) + } + if _, err := s.regR.GetPointer(s.ctx, absentName); !errors.Is(err, publication.ErrBundleNotFound) { + t.Fatalf("aborted first publish pointer readable: %v, want ErrBundleNotFound", err) + } + if err := s.regA.PublishExecution(s.ctx, absentName, "", pubvisExecution(absentID, true, pubvisOwnerA)); err != nil { + s.fail("recovery publish after absent-pointer abort", err) + } + row = s.pointerRow(t, absentName) + if row == nil || row.Execution != absentID { + t.Fatalf("recovery pointer row = %+v, want %q", row, absentID) + } + }) + + t.Run("C_same_execution_retry_is_idempotent", func(t *testing.T) { + s := newPubvisService(t) + name := pubvisPointerName() + const id = "pubvis-exec-c" + fqLeases, fqExec, fqPointers := s.qualified(BundleLeasesTable), s.qualified(BundleExecutionsTable), s.qualified(BundlePointersTable) + s.seed(t, s.regA, pubvisOwnerA, id) + published := pubvisExecution(id, true, pubvisOwnerA) + + gate := newPubvisGate() + pubvisArmGate(t, gate) + done := make(chan error, 1) + go func() { done <- s.regA.PublishExecution(s.ctx, name, "", published) }() + attempt := pubvisWaitAttempt(t, gate) + if attempt.Converged { + t.Fatal("first publish took the converged path before anything was published") + } + pubvisEqualStrings(t, "first publish", attempt.Statements, s.goldenPublishStatements(fqLeases, fqExec, fqPointers)) + gate.release <- nil + if err := <-done; err != nil { + s.fail("first publish", err) + } + + pointerBefore := s.pointerValue(t, name) + payloadBefore := s.payloadText(t, id) + epochBefore := s.wInt64(t, s.reader, + "SELECT epoch FROM "+fqLeases+" WHERE key = $1", "bundle-"+id) + updatedAtLiteral := pointerBefore.UpdatedAt + if updatedAtLiteral != "2026-11-09T10:02:00.112233Z" { + t.Fatalf("pointer updated_at literal = %q, want the published stamp to microsecond precision", updatedAtLiteral) + } + + gate2 := newPubvisGate() + pubvisArmGate(t, gate2) + done = make(chan error, 1) + go func() { done <- s.regA.PublishExecution(s.ctx, name, "", published) }() + retry := pubvisWaitAttempt(t, gate2) + if !retry.Converged { + t.Fatal("identical retry did not take the converged path") + } + pubvisEqualStrings(t, "converged retry", retry.Statements, s.goldenConvergedStatements(fqLeases, fqExec, fqPointers)) + gate2.release <- nil + if err := <-done; err != nil { + t.Fatalf("identical retry = %v, want nil (idempotent)", err) + } + if after := s.pointerValue(t, name); after != pointerBefore { + t.Fatalf("retry mutated the pointer row: %+v vs %+v", after, pointerBefore) + } + if after := s.payloadText(t, id); after != payloadBefore { + t.Fatalf("retry mutated the payload bytes (first difference at byte %d)", firstDiff(payloadBefore, after)) + } + if epoch := s.wInt64(t, s.reader, "SELECT epoch FROM "+fqLeases+" WHERE key = $1", "bundle-"+id); epoch != epochBefore { + t.Fatalf("retry bumped the lease epoch to %d, want it untouched at %d", epoch, epochBefore) + } + + drifted := published + drifted.PublishedAt = nil + driftedErr := s.regA.PublishExecution(s.ctx, name, "", drifted) + if !errors.Is(driftedErr, publication.ErrBundlePointerConflict) { + t.Fatalf("drifted retry = %v, want ErrBundlePointerConflict", driftedErr) + } + if errors.Is(driftedErr, publication.ErrBundleLeaseLost) || errors.Is(driftedErr, publication.ErrBundleNotFound) { + t.Fatal("drifted retry reported a sibling typed error, want exactly ErrBundlePointerConflict") + } + if after := s.pointerValue(t, name); after != pointerBefore { + t.Fatalf("refused drifted retry mutated the pointer row: %+v vs %+v", after, pointerBefore) + } + if after := s.payloadText(t, id); after != payloadBefore { + t.Fatalf("refused drifted retry mutated the payload (first difference at byte %d)", firstDiff(payloadBefore, after)) + } + + if err := s.regA.PublishExecution(s.ctx, name, "", published); err != nil { + t.Fatalf("exact retry after refusals = %v, want nil", err) + } + if got := pubvisSHA256Hex(s.payloadText(t, id)); got != pubvisWantPayloadSHA(t, id) { + t.Fatalf("canonical payload digest = %s, want golden %s", got, pubvisWantPayloadSHA(t, id)) + } + }) + + t.Run("D_concurrent_publishers_one_typed_winner", func(t *testing.T) { + s := newPubvisService(t) + fqPointers := s.qualified(BundlePointersTable) + fqExec := s.qualified(BundleExecutionsTable) + + runRound := func(tag string, staleLease bool) { + t.Helper() + name := pubvisPointerName() + "|" + tag + idA, idB := "pubvis-exec-d-"+tag+"-a", "pubvis-exec-d-"+tag+"-b" + s.seed(t, s.regA, pubvisOwnerA, idA) + s.seed(t, s.regB, pubvisOwnerB, idB) + if staleLease { + claimed, err := s.regB.AcquireBundleLease(s.ctx, "bundle-"+idB, pubvisOwnerB, time.Now().UTC().Add(-time.Minute)) + if err != nil || !claimed { + s.fail("expire stale lease", fmt.Errorf("claimed=%v err=%w", claimed, err)) + } + } + pubA, pubB := pubvisExecution(idA, true, pubvisOwnerA), pubvisExecution(idB, true, pubvisOwnerB) + + pidA := s.backendPID(t, s.pub, "publisher A") + pidB := s.backendPID(t, s.pubB, "publisher B") + if pidA == pidB { + t.Fatalf("round %s: publishers share backend pid %d, want two real PG connections", tag, pidA) + } + + start := make(chan struct{}) + var wg sync.WaitGroup + var errA, errB error + wg.Add(2) + go func() { defer wg.Done(); <-start; errA = s.regA.PublishExecution(s.ctx, name, "", pubA) }() + go func() { defer wg.Done(); <-start; errB = s.regB.PublishExecution(s.ctx, name, "", pubB) }() + close(start) + wg.Wait() + + winners, losers := []string{}, []string{} + for id, err := range map[string]error{idA: errA, idB: errB} { + switch { + case err == nil: + winners = append(winners, id) + case errors.Is(err, publication.ErrBundlePointerConflict) && + !errors.Is(err, publication.ErrBundleLeaseLost) && + !errors.Is(err, publication.ErrBundleNotFound): + losers = append(losers, id) + default: + t.Fatalf("round %s: publisher %s failed with %v, want exactly ErrBundlePointerConflict or success", tag, id, err) + } + } + if len(winners) != 1 || len(losers) != 1 { + t.Fatalf("round %s: winners=%v losers=%v, want exactly one each", tag, winners, losers) + } + winner, loser := winners[0], losers[0] + row := s.pointerRow(t, name) + if row == nil || row.Execution != winner { + t.Fatalf("round %s: pointer row = %+v, want the winner %q", tag, row, winner) + } + if n := s.wInt64(t, s.reader, "SELECT count(*) FROM "+fqPointers+" WHERE pointer_key = $1", pointerKey(name)); n != 1 { + t.Fatalf("round %s: pointer rows = %d, want 1", tag, n) + } + if state := s.wText(t, s.reader, + "SELECT payload->>'state' FROM "+fqExec+" WHERE id = $1", loser); state != "RUNNING" { + t.Fatalf("round %s: loser execution state = %q, want RUNNING (fully rolled back)", tag, state) + } + if pubvisSHA256Hex(s.payloadText(t, loser)) == pubvisWantPayloadSHA(t, loser) { + t.Fatalf("round %s: loser payload carries the published digest", tag) + } + want := pubvisExecution(winner, true, map[string]string{idA: pubvisOwnerA, idB: pubvisOwnerB}[winner]) + pubvisExpectExactPublished(t, s, want) + regWinner, winnerID := s.regA, idA + if winner == idB { + regWinner, winnerID = s.regB, idB + } + if staleLease && winnerID == idB { + t.Fatal("stale-lease round let the expired fence win") + } + if err := regWinner.PublishExecution(s.ctx, name, "", want); err != nil { + t.Fatalf("round %s: winner retry = %v, want the idempotent nil", tag, err) + } + if after := s.pointerValue(t, name); after != *row { + t.Fatalf("round %s: winner retry mutated the pointer %+v vs %+v", tag, after, *row) + } + t.Logf("OBS D round %s (staleLease=%v): winner %s pid=%d, loser %s refused with the named conflict", + tag, staleLease, winner, map[string]uint32{idA: pidA, idB: pidB}[winner], loser) + } + + for round := 1; round <= 12; round++ { + runRound(fmt.Sprintf("r%02d", round), false) + } + for round := 1; round <= 12; round++ { + runRound(fmt.Sprintf("s%02d", round), true) + } + }) + + t.Run("E_no_clickhouse_surface_inside_the_publish_transaction", func(t *testing.T) { + s := newPubvisService(t) + name := pubvisPointerName() + const id = "pubvis-exec-e" + fqLeases, fqExec, fqPointers := s.qualified(BundleLeasesTable), s.qualified(BundleExecutionsTable), s.qualified(BundlePointersTable) + s.seed(t, s.regA, pubvisOwnerA, id) + published := pubvisExecution(id, true, pubvisOwnerA) + + gate := newPubvisGate() + pubvisArmGate(t, gate) + done := make(chan error, 1) + go func() { done <- s.regA.PublishExecution(s.ctx, name, "", published) }() + attempt := pubvisWaitAttempt(t, gate) + + pubvisEqualStrings(t, "publish statement stream", attempt.Statements, + s.goldenPublishStatements(fqLeases, fqExec, fqPointers)) + + txState, lastQuery := s.activity(t, attempt.BackendPID) + if txState != "idle in transaction" { + t.Fatalf("publish backend state during the window = %q, want idle in transaction", txState) + } + if lastQuery != pubvisGoldenUpdate(fqExec) { + t.Fatalf("publish backend last query is not the golden catalog UPDATE:\n got: %q", lastQuery) + } + + sweep := s.wInt64(t, s.reader, + "SELECT count(*) FROM pg_stat_activity WHERE backend_type = 'client backend' "+ + "AND pid <> pg_backend_pid() AND query ~* 'clickhouse|ch_send|insert into .*ch_'") + if sweep != 0 { + t.Fatalf("%d backends were executing ClickHouse-shaped statements inside the publish window", sweep) + } + + gate.release <- nil + if err := <-done; err != nil { + s.fail("publish after E window", err) + } + pubvisExpectExactPublished(t, s, published) + + method, ok := reflect.TypeOf(&Registry{}).MethodByName("PublishExecution") + if !ok { + t.Fatal("Registry.PublishExecution missing") + } + for i := 0; i < method.Type.NumIn(); i++ { + switch kind := method.Type.In(i).Kind(); kind { + case reflect.Func, reflect.Chan: + t.Fatalf("PublishExecution parameter %d has kind %v: the publish API must take values only", i, kind) + } + } + if method.Type.NumIn() != 5 || method.Type.NumOut() != 1 { + t.Fatalf("PublishExecution signature shape = (%d in, %d out), want (5 in, 1 out)", + method.Type.NumIn(), method.Type.NumOut()) + } + ifaceMethod, ok := reflect.TypeOf((*publication.BundleCatalog)(nil)).Elem().MethodByName("PublishExecution") + if !ok || ifaceMethod.Type.NumIn() != 4 { + t.Fatalf("BundleCatalog.PublishExecution interface shape drifted: %+v", ifaceMethod) + } + }) + + t.Run("F_prefix_split_control_is_observable", func(t *testing.T) { + s := newPubvisService(t) + name := pubvisPointerName() + const id = "pubvis-exec-f" + fqExec := s.qualified(BundleExecutionsTable) + s.seed(t, s.regA, pubvisOwnerA, id) + published := pubvisExecution(id, true, pubvisOwnerA) + params, err := executionParams(published) + if err != nil { + s.fail("build publish params", err) + } + + leg1 := pubvisGoldenUpdate(fqExec) + if _, err := s.pub.Exec(s.ctx, leg1, params...); err != nil { + s.fail("pre-fix leg 1", err) + } + + state, err := s.regR.GetExecution(s.ctx, id) + if err != nil || state.State != publication.BundlePublished { + t.Fatalf("sensitivity failure: pre-fix split execution state = %q err=%v, want PUBLISHED visible", state.State, err) + } + if _, err := s.regR.GetPointer(s.ctx, name); !errors.Is(err, publication.ErrBundleNotFound) { + t.Fatalf("sensitivity failure: probe must not see a pointer in the split state, got %v", err) + } + if row := s.pointerRow(t, name); row != nil { + t.Fatalf("sensitivity failure: pointer row exists in the split state: %+v", row) + } + t.Log("OBS F: the pre-fix split IS observable -- witnesses in A/B/D are sensitive to half-publish") + + if err := s.regA.PublishExecution(s.ctx, name, "", published); err != nil { + s.fail("repaired publish from the split state", err) + } + row := s.pointerRow(t, name) + if row == nil || row.Execution != id { + t.Fatalf("repaired publish pointer row = %+v, want %q", row, id) + } + pubvisExpectExactPublished(t, s, published) + }) + + t.Run("G_payload_goldens_are_load_bearing", func(t *testing.T) { + s := newPubvisService(t) + pinned := 0 + for _, p := range []string{"r", "s"} { + for i := 1; i <= 12; i++ { + tag := fmt.Sprintf("%s%02d", p, i) + for _, suffix := range []string{"a", "b"} { + id := "pubvis-exec-d-" + tag + "-" + suffix + owner := pubvisOwnerA + if suffix == "b" { + owner = pubvisOwnerB + } + name := pubvisPointerName() + "|g" + tag + suffix + s.seed(t, s.regA, owner, id) + published := pubvisExecution(id, true, owner) + if err := s.regA.PublishExecution(s.ctx, name, "", published); err != nil { + s.fail("G publish "+id, err) + } + if got := pubvisSHA256Hex(s.payloadText(t, id)); got != pubvisWantPayloadSHA(t, id) { + t.Fatalf("%s: production-published payload digest = %s, want golden %s -- the pin no longer tracks the write path (owner binding or payload drift)", + id, got, pubvisWantPayloadSHA(t, id)) + } + pubvisExpectExactPublished(t, s, published) + pinned++ + } + } + } + t.Logf("OBS G: %d d-* payload golden pins asserted positively against the production write path (owner-correct bindings)", pinned) + }) +} + +func firstDiff(a, b string) int { + limit := min(len(a), len(b)) + for i := 0; i < limit; i++ { + if a[i] != b[i] { + return i + } + } + return limit +} + +func TestAuditPublishVisibilityForeignFixtureActivity(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("no service configured; nothing could have been created") + } + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %v", err) + } + t.Cleanup(admin.Close) + + foreign := fmt.Sprintf("verify_pubvisx_it_%d_%s", time.Now().UnixNano(), randomHex()) + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+foreign+`"`); err != nil { + t.Fatalf("create foreign fixture database: %v", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(context.Background()), + `DROP DATABASE IF EXISTS "`+foreign+`" WITH (FORCE)`); err != nil { + t.Errorf("drop foreign fixture database: %v", err) + } + }) + + if err := auditPublishVisibilityLeftovers(); err != nil { + t.Errorf("audit with foreign fixture database alive = %v, want nil", err) + } + + var survived int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", foreign).Scan(&survived); err != nil { + t.Fatalf("check foreign fixture database survived the audit: %v", err) + } + if survived != 1 { + t.Fatalf("audit removed foreign fixture database %q", foreign) + } + + if _, err := admin.Exec(ctx, `DROP DATABASE IF EXISTS "`+foreign+`" WITH (FORCE)`); err != nil { + t.Fatalf("remove foreign fixture database: %v", err) + } + if err := auditPublishVisibilityLeftovers(); err != nil { + t.Errorf("audit after foreign fixture database was removed = %v, want nil", err) + } +} + +func TestAuditPublishVisibilityOwnedLeakControls(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("no service configured; nothing could have been created") + } + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %v", err) + } + defer admin.Close() + + t.Run("registered database leak", func(t *testing.T) { + db := fmt.Sprintf("verify_pubvis_it_%d_%s", time.Now().UnixNano(), randomHex()) + if !pubvisResourceNamePattern.MatchString(db) { + t.Fatalf("leak database name %q does not match the required pattern", db) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+db+`"`); err != nil { + t.Fatalf("create leak database: %v", err) + } + pubvisRegisterOwned(db, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(context.Background()), + `DROP DATABASE IF EXISTS "`+db+`" WITH (FORCE)`); err != nil { + t.Errorf("drop leak database: %v", err) + } + }) + err := auditPublishVisibilityLeftovers() + if err == nil || !strings.Contains(err.Error(), "leftover owned database") || !strings.Contains(err.Error(), db) { + t.Fatalf("audit with a registered owned database leak = %v, want a leftover owned database failure naming %q", err, db) + } + }) + + t.Run("unregistered namespace database leak", func(t *testing.T) { + db := fmt.Sprintf("verify_pubvis_it_%d_%s", time.Now().UnixNano(), randomHex()) + if !pubvisResourceNamePattern.MatchString(db) { + t.Fatalf("leak database name %q does not match the required pattern", db) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+db+`"`); err != nil { + t.Fatalf("create leak database: %v", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(context.Background()), + `DROP DATABASE IF EXISTS "`+db+`" WITH (FORCE)`); err != nil { + t.Errorf("drop leak database: %v", err) + } + }) + err := auditPublishVisibilityLeftovers() + if err == nil || !strings.Contains(err.Error(), "leftover databases under this suite's fixture namespace") || !strings.Contains(err.Error(), db) { + t.Fatalf("audit with an unregistered namespace database leak = %v, want a fixture namespace failure naming %q", err, db) + } + }) + + t.Run("namespace schema leak", func(t *testing.T) { + schema := fmt.Sprintf("verify_pubvis_it_%d_%s", time.Now().UnixNano(), randomHex()) + if !pubvisResourceNamePattern.MatchString(schema) { + t.Fatalf("leak schema name %q does not match the required pattern", schema) + } + if _, err := admin.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + t.Fatalf("create leak schema: %v", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(context.Background()), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop leak schema: %v", err) + } + }) + err := auditPublishVisibilityLeftovers() + if err == nil || !strings.Contains(err.Error(), "leftover schemas under this suite's fixture namespace") || !strings.Contains(err.Error(), schema) { + t.Fatalf("audit with a namespace schema leak = %v, want a fixture namespace failure naming %q", err, schema) + } + }) +} + +func TestZZZeroLeftoverPublishVisibilityFixtures(t *testing.T) { + if strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) == "" { + t.Skip("no service configured; nothing could have been created") + } + if err := auditPublishVisibilityLeftovers(); err != nil { + t.Errorf("zero-leftover audit failed: %v", err) + } +} diff --git a/internal/backends/postgres/internal/publication/read_pins.go b/internal/backends/postgres/internal/publication/read_pins.go new file mode 100644 index 000000000..977da3cb1 --- /dev/null +++ b/internal/backends/postgres/internal/publication/read_pins.go @@ -0,0 +1,514 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "strings" + "sync/atomic" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/calypr/loom/internal/dataframe/publication" +) + +const ( + ExecutionRetentionTable = "publication_execution_retention" + ExecutionReadPinsTable = "publication_execution_read_pins" + ExecutionCleanupClaimsTable = "publication_execution_cleanup_claims" + + executionCleanupTTL = 2 * time.Minute +) + +var ErrInvalidCleanupIdentity = errors.New("publication postgres: cleanup execution id and owner are required") + +type readPinAttempt struct { + Op string + ExecutionID string + OwnerID string + BackendPID uint32 + Statements []string + Claimed bool +} + +var readPinGate atomic.Pointer[func(readPinAttempt) error] + +type ReadPins struct { + db DB + retention string + pins string + cleanup string + + claimTTL time.Duration +} + +var _ publication.ExecutionReadPinCatalog = (*ReadPins)(nil) + +type BundleCatalog struct { + *Registry + pins *ReadPins +} + +var ( + _ publication.BundleCatalog = (*BundleCatalog)(nil) + _ publication.ExactExecutionCatalog = (*BundleCatalog)(nil) + _ publication.PagedBundleCatalog = (*BundleCatalog)(nil) + _ publication.ExecutionReadPinCatalog = (*BundleCatalog)(nil) +) + +func NewBundleCatalog(pool DB, schema string) (*BundleCatalog, error) { + registry, err := New(pool, schema) + if err != nil { + return nil, err + } + pins, err := NewReadPins(pool, schema) + if err != nil { + return nil, err + } + return &BundleCatalog{Registry: registry, pins: pins}, nil +} + +func (c *BundleCatalog) Pins() *ReadPins { return c.pins } + +func (c *BundleCatalog) AcquireExecutionReadPin(ctx context.Context, executionID, ownerID string, expiresAt time.Time) (bool, error) { + return c.pins.AcquireExecutionReadPin(ctx, executionID, ownerID, expiresAt) +} + +func (c *BundleCatalog) RenewExecutionReadPin(ctx context.Context, executionID, ownerID string, expiresAt time.Time) (bool, error) { + return c.pins.RenewExecutionReadPin(ctx, executionID, ownerID, expiresAt) +} + +func (c *BundleCatalog) ReleaseExecutionReadPin(ctx context.Context, executionID, ownerID string) error { + return c.pins.ReleaseExecutionReadPin(ctx, executionID, ownerID) +} + +func (c *BundleCatalog) ClaimExecutionCleanup(ctx context.Context, executionID, ownerID string) (bool, error) { + return c.pins.ClaimExecutionCleanup(ctx, executionID, ownerID) +} + +func (c *BundleCatalog) RenewExecutionCleanup(ctx context.Context, executionID, ownerID string, expiresAt time.Time) (bool, error) { + return c.pins.RenewExecutionCleanup(ctx, executionID, ownerID, expiresAt) +} + +func (c *BundleCatalog) ReleaseExecutionCleanup(ctx context.Context, executionID, ownerID string) error { + return c.pins.ReleaseExecutionCleanup(ctx, executionID, ownerID) +} + +func NewReadPins(pool DB, schema string) (*ReadPins, error) { + if pool == nil { + return nil, errors.New("publication postgres: nil pool") + } + if !pgstore.ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", ErrInvalidSchema, schema) + } + return &ReadPins{ + db: pool, + retention: pgx.Identifier{schema, ExecutionRetentionTable}.Sanitize(), + pins: pgx.Identifier{schema, ExecutionReadPinsTable}.Sanitize(), + cleanup: pgx.Identifier{schema, ExecutionCleanupClaimsTable}.Sanitize(), + claimTTL: executionCleanupTTL, + }, nil +} + +type readPinTx struct { + p *ReadPins + tx pgx.Tx + attempt readPinAttempt + recorded []string +} + +func (p *ReadPins) beginTransition(ctx context.Context, op, executionID, owner string) (*readPinTx, error) { + tx, err := p.db.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("publication postgres: begin %s: %w", op, err) + } + t := &readPinTx{p: p, tx: tx, attempt: readPinAttempt{Op: op, ExecutionID: executionID, OwnerID: owner}} + if conn := tx.Conn(); conn != nil { + t.attempt.BackendPID = conn.PgConn().PID() + } + return t, nil +} + +func (t *readPinTx) record(sql string) { + t.recorded = append(t.recorded, sql) + t.attempt.Statements = t.recorded +} + +func (t *readPinTx) exec(ctx context.Context, sql string, args ...any) (int64, error) { + t.record(sql) + tag, err := t.tx.Exec(ctx, sql, args...) + if err != nil { + return 0, fmt.Errorf("publication postgres: %s: %w", t.attempt.Op, err) + } + return tag.RowsAffected(), nil +} + +func (t *readPinTx) queryBool(ctx context.Context, sql string, args ...any) (bool, error) { + t.record(sql) + var value bool + if err := t.tx.QueryRow(ctx, sql, args...).Scan(&value); err != nil { + return false, fmt.Errorf("publication postgres: %s guard: %w", t.attempt.Op, err) + } + return value, nil +} + +func (t *readPinTx) finish(ctx context.Context, claimed bool) (bool, error) { + t.attempt.Claimed = claimed + if gate := readPinGate.Load(); gate != nil { + if err := (*gate)(t.attempt); err != nil { + _ = t.tx.Rollback(ctx) + return false, err + } + } + if !claimed { + if err := t.tx.Rollback(ctx); err != nil { + return false, fmt.Errorf("publication postgres: %s rollback refused transition: %w", t.attempt.Op, err) + } + return false, nil + } + if err := t.tx.Commit(ctx); err != nil { + return false, fmt.Errorf("publication postgres: %s commit: %w", t.attempt.Op, err) + } + return true, nil +} + +func (t *readPinTx) close() { + _ = t.tx.Rollback(context.WithoutCancel(context.Background())) +} + +func (t *readPinTx) ensureAnchor(ctx context.Context, now time.Time) error { + if _, err := t.exec(ctx, + "INSERT INTO "+t.p.retention+" (execution_id, updated_at) VALUES ($1, $2)"+ + " ON CONFLICT (execution_id) DO NOTHING", + t.attempt.ExecutionID, now); err != nil { + return err + } + held, err := t.lockAnchor(ctx) + if err != nil { + return err + } + if !held { + return fmt.Errorf("publication postgres: %s: anchor row vanished for execution %s", t.attempt.Op, t.attempt.ExecutionID) + } + return nil +} + +func (t *readPinTx) lockAnchor(ctx context.Context) (bool, error) { + t.record("SELECT true FROM " + t.p.retention + " WHERE execution_id = $1 FOR UPDATE") + var held bool + err := t.tx.QueryRow(ctx, + "SELECT true FROM "+t.p.retention+" WHERE execution_id = $1 FOR UPDATE", + t.attempt.ExecutionID).Scan(&held) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("publication postgres: %s lock anchor: %w", t.attempt.Op, err) + } + return held, nil +} + +func (t *readPinTx) touchAnchor(ctx context.Context, now time.Time) error { + _, err := t.exec(ctx, + "UPDATE "+t.p.retention+" SET updated_at = $2 WHERE execution_id = $1", + t.attempt.ExecutionID, now) + return err +} + +func (t *readPinTx) guardCleanupFree(ctx context.Context, now time.Time) (bool, error) { + return t.queryBool(ctx, + "SELECT NOT EXISTS (SELECT 1 FROM "+t.p.cleanup+" WHERE execution_id = $1 AND expires_at >= $2)", + t.attempt.ExecutionID, now) +} + +func (t *readPinTx) guardNoActivePins(ctx context.Context, now time.Time) (bool, error) { + return t.queryBool(ctx, + "SELECT NOT EXISTS (SELECT 1 FROM "+t.p.pins+" WHERE execution_id = $1 AND expires_at >= $2)", + t.attempt.ExecutionID, now) +} + +func (t *readPinTx) gcExpiredPins(ctx context.Context, now time.Time) error { + _, err := t.exec(ctx, + "DELETE FROM "+t.p.pins+" WHERE execution_id = $1 AND expires_at < $2", + t.attempt.ExecutionID, now) + return err +} + +func (p *ReadPins) AcquireExecutionReadPin(ctx context.Context, executionID, owner string, expires time.Time) (bool, error) { + executionID, owner = strings.TrimSpace(executionID), strings.TrimSpace(owner) + if executionID == "" || owner == "" { + return false, publication.ErrExecutionReadPinLost + } + now := time.Now().UTC() + t, err := p.beginTransition(ctx, "acquire-pin", executionID, owner) + if err != nil { + return false, err + } + defer t.close() + if err := t.ensureAnchor(ctx, now); err != nil { + return false, err + } + free, err := t.guardCleanupFree(ctx, now) + if err != nil { + return false, err + } + if !free { + return t.finish(ctx, false) + } + if err := t.gcExpiredPins(ctx, now); err != nil { + return false, err + } + if _, err := t.exec(ctx, + "INSERT INTO "+t.p.pins+" (execution_id, owner_id, expires_at, updated_at)"+ + " VALUES ($1, $2, $3, $4)"+ + " ON CONFLICT (execution_id, owner_id) DO UPDATE SET expires_at = $3, updated_at = $4", + executionID, owner, expires, now); err != nil { + return false, err + } + if err := t.touchAnchor(ctx, now); err != nil { + return false, err + } + return t.finish(ctx, true) +} + +func (p *ReadPins) RenewExecutionReadPin(ctx context.Context, executionID, owner string, expires time.Time) (bool, error) { + executionID, owner = strings.TrimSpace(executionID), strings.TrimSpace(owner) + if executionID == "" || owner == "" { + return false, publication.ErrExecutionReadPinLost + } + now := time.Now().UTC() + t, err := p.beginTransition(ctx, "renew-pin", executionID, owner) + if err != nil { + return false, err + } + defer t.close() + exists, err := t.lockAnchor(ctx) + if err != nil { + return false, err + } + if !exists { + return t.finish(ctx, false) + } + free, err := t.guardCleanupFree(ctx, now) + if err != nil { + return false, err + } + if !free { + return t.finish(ctx, false) + } + owned, err := t.queryBool(ctx, + "SELECT EXISTS (SELECT 1 FROM "+t.p.pins+" WHERE execution_id = $1 AND owner_id = $2 AND expires_at >= $3)", + executionID, owner, now) + if err != nil { + return false, err + } + if !owned { + return t.finish(ctx, false) + } + if err := t.gcExpiredPins(ctx, now); err != nil { + return false, err + } + rows, err := t.exec(ctx, + "UPDATE "+t.p.pins+" SET expires_at = $3, updated_at = $4"+ + " WHERE execution_id = $1 AND owner_id = $2", + executionID, owner, expires, now) + if err != nil { + return false, err + } + if rows != 1 { + return false, fmt.Errorf("publication postgres: renew-pin lost pin row under lock: %s/%s", executionID, owner) + } + if err := t.touchAnchor(ctx, now); err != nil { + return false, err + } + return t.finish(ctx, true) +} + +func (p *ReadPins) ReleaseExecutionReadPin(ctx context.Context, executionID, owner string) error { + executionID, owner = strings.TrimSpace(executionID), strings.TrimSpace(owner) + if executionID == "" || owner == "" { + return publication.ErrExecutionReadPinLost + } + now := time.Now().UTC() + t, err := p.beginTransition(ctx, "release-pin", executionID, owner) + if err != nil { + return err + } + defer t.close() + exists, err := t.lockAnchor(ctx) + if err != nil { + return err + } + if !exists { + return t.commitTransition(ctx) + } + if _, err := t.exec(ctx, + "DELETE FROM "+t.p.pins+" WHERE execution_id = $1 AND owner_id = $2", + executionID, owner); err != nil { + return err + } + if err := t.touchAnchor(ctx, now); err != nil { + return err + } + return t.commitTransition(ctx) +} + +func (p *ReadPins) ClaimExecutionCleanup(ctx context.Context, executionID, owner string) (bool, error) { + executionID, owner = strings.TrimSpace(executionID), strings.TrimSpace(owner) + if executionID == "" || owner == "" { + return false, ErrInvalidCleanupIdentity + } + now := time.Now().UTC() + t, err := p.beginTransition(ctx, "claim-cleanup", executionID, owner) + if err != nil { + return false, err + } + defer t.close() + if err := t.ensureAnchor(ctx, now); err != nil { + return false, err + } + noPins, err := t.guardNoActivePins(ctx, now) + if err != nil { + return false, err + } + if !noPins { + return t.finish(ctx, false) + } + var ( + claimOwner string + claimExpires time.Time + claimExists bool + ) + t.record("SELECT owner_id, expires_at FROM " + t.p.cleanup + " WHERE execution_id = $1") + err = t.tx.QueryRow(ctx, + "SELECT owner_id, expires_at FROM "+t.p.cleanup+" WHERE execution_id = $1", + executionID).Scan(&claimOwner, &claimExpires) + if err == nil { + claimExists = true + } else if !errors.Is(err, pgx.ErrNoRows) { + return false, fmt.Errorf("publication postgres: claim-cleanup read claim: %w", err) + } + if claimExists && !claimExpires.Before(now) && claimOwner != owner { + return t.finish(ctx, false) + } + if err := t.gcExpiredPins(ctx, now); err != nil { + return false, err + } + if _, err := t.exec(ctx, + "INSERT INTO "+t.p.cleanup+" (execution_id, owner_id, expires_at, updated_at)"+ + " VALUES ($1, $2, $3, $4)"+ + " ON CONFLICT (execution_id) DO UPDATE SET owner_id = $2, expires_at = $3, updated_at = $4", + executionID, owner, now.Add(p.claimTTL), now); err != nil { + return false, err + } + if err := t.touchAnchor(ctx, now); err != nil { + return false, err + } + return t.finish(ctx, true) +} + +func (p *ReadPins) RenewExecutionCleanup(ctx context.Context, executionID, owner string, expires time.Time) (bool, error) { + executionID, owner = strings.TrimSpace(executionID), strings.TrimSpace(owner) + if executionID == "" || owner == "" { + return false, ErrInvalidCleanupIdentity + } + now := time.Now().UTC() + t, err := p.beginTransition(ctx, "renew-cleanup", executionID, owner) + if err != nil { + return false, err + } + defer t.close() + exists, err := t.lockAnchor(ctx) + if err != nil { + return false, err + } + if !exists { + return t.finish(ctx, false) + } + var ( + claimOwner string + claimExpires time.Time + claimFound bool + ) + t.record("SELECT owner_id, expires_at FROM " + t.p.cleanup + " WHERE execution_id = $1") + err = t.tx.QueryRow(ctx, + "SELECT owner_id, expires_at FROM "+t.p.cleanup+" WHERE execution_id = $1", + executionID).Scan(&claimOwner, &claimExpires) + if err == nil { + claimFound = true + } else if !errors.Is(err, pgx.ErrNoRows) { + return false, fmt.Errorf("publication postgres: renew-cleanup read claim: %w", err) + } + if !claimFound || claimOwner != owner || claimExpires.Before(now) { + return t.finish(ctx, false) + } + noPins, err := t.guardNoActivePins(ctx, now) + if err != nil { + return false, err + } + if !noPins { + return t.finish(ctx, false) + } + rows, err := t.exec(ctx, + "UPDATE "+t.p.cleanup+" SET expires_at = $3, updated_at = $4"+ + " WHERE execution_id = $1 AND owner_id = $2", + executionID, owner, expires, now) + if err != nil { + return false, err + } + if rows != 1 { + return false, fmt.Errorf("publication postgres: renew-cleanup lost claim row under lock: %s/%s", executionID, owner) + } + if err := t.touchAnchor(ctx, now); err != nil { + return false, err + } + return t.finish(ctx, true) +} + +func (p *ReadPins) ReleaseExecutionCleanup(ctx context.Context, executionID, owner string) error { + executionID, owner = strings.TrimSpace(executionID), strings.TrimSpace(owner) + if executionID == "" || owner == "" { + return ErrInvalidCleanupIdentity + } + now := time.Now().UTC() + t, err := p.beginTransition(ctx, "release-cleanup", executionID, owner) + if err != nil { + return err + } + defer t.close() + exists, err := t.lockAnchor(ctx) + if err != nil { + return err + } + if !exists { + return t.commitTransition(ctx) + } + rows, err := t.exec(ctx, + "DELETE FROM "+t.p.cleanup+" WHERE execution_id = $1 AND owner_id = $2", + executionID, owner) + if err != nil { + return err + } + if rows == 1 { + if err := t.touchAnchor(ctx, now); err != nil { + return err + } + } + return t.commitTransition(ctx) +} + +func (t *readPinTx) commitTransition(ctx context.Context) error { + t.attempt.Claimed = true + if gate := readPinGate.Load(); gate != nil { + if err := (*gate)(t.attempt); err != nil { + _ = t.tx.Rollback(ctx) + return err + } + } + if err := t.tx.Commit(ctx); err != nil { + return fmt.Errorf("publication postgres: %s commit: %w", t.attempt.Op, err) + } + return nil +} diff --git a/internal/backends/postgres/internal/publication/read_pins_test.go b/internal/backends/postgres/internal/publication/read_pins_test.go new file mode 100644 index 000000000..33b8fd19b --- /dev/null +++ b/internal/backends/postgres/internal/publication/read_pins_test.go @@ -0,0 +1,1376 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/dataframe/publication" +) + +var rprResourceNamePattern = regexp.MustCompile(`^verify_rpr_it_[0-9]+_[0-9a-f]+$`) + +var ( + rprOwnedMu sync.Mutex + rprOwned = map[string]bool{} +) + +func rprRegisterOwned(db string) { + rprOwnedMu.Lock() + defer rprOwnedMu.Unlock() + rprOwned[db] = true +} + +func auditReadPinsLeftovers() error { + if strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) == "" { + return nil + } + rprOwnedMu.Lock() + owned := make([]string, 0, len(rprOwned)) + for db := range rprOwned { + owned = append(owned, db) + } + rprOwnedMu.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL"))) + if err != nil { + return fmt.Errorf("audit connect: %w", err) + } + defer admin.Close() + + for _, db := range owned { + if !strings.HasPrefix(db, "verify_rpr_it_") { + return fmt.Errorf("ownership registry holds foreign database %q", db) + } + var count int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + return fmt.Errorf("audit owned database %s: %w", db, err) + } + if count != 0 { + return fmt.Errorf("leftover owned database %s after all cleanups", db) + } + } + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE left(datname, $1) = $2", len("verify_rpr_"), "verify_rpr_") + if err != nil { + return fmt.Errorf("audit prefix scan: %w", err) + } + defer rows.Close() + var leftovers []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return fmt.Errorf("audit prefix scan: %w", err) + } + leftovers = append(leftovers, name) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("audit prefix scan: %w", err) + } + if len(leftovers) != 0 { + return fmt.Errorf("leftover databases under this suite's registered prefix verify_rpr_* after all cleanups: %v", leftovers) + } + return nil +} + +type rprService struct { + ctx context.Context + admin *pgxpool.Pool + a *pgxpool.Pool + b *pgxpool.Pool + w *pgxpool.Pool + schema string + + pinsA, pinsB, pinsW *ReadPins + regA, regB, regW *Registry + + scrub func(any) string + fail func(step string, err error) +} + +func newRPRService(t *testing.T) *rprService { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { t.Fatalf("%s: %s", step, scrub(err)) } + + ctx, cancel := context.WithTimeout(context.Background(), 480*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("verify_rpr_it_%d_%s", time.Now().UnixNano(), randomHex()) + if !rprResourceNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + rprRegisterOwned(dbName) + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + poolA, err := newOwnedPool(ctx, dsn) + if err != nil { + fail("connect pool A", err) + } + t.Cleanup(poolA.Close) + poolB, err := newOwnedPool(ctx, dsn) + if err != nil { + fail("connect pool B", err) + } + t.Cleanup(poolB.Close) + poolW, err := newOwnedPool(ctx, dsn) + if err != nil { + fail("connect witness pool", err) + } + t.Cleanup(poolW.Close) + + schema := fmt.Sprintf("verify_rpr_it_%d_%s", time.Now().UnixNano(), randomHex()) + if !rprResourceNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := poolW.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := poolW.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + s := &rprService{ctx: ctx, admin: admin, a: poolA, b: poolB, w: poolW, schema: schema, scrub: scrub, fail: fail} + s.migrate(t) + s.pinsA = s.readPins(t, poolA) + s.pinsB = s.readPins(t, poolB) + s.pinsW = s.readPins(t, poolW) + s.regA = s.registry(t, poolA) + s.regB = s.registry(t, poolB) + s.regW = s.registry(t, poolW) + return s +} + +func (s *rprService) readPins(t *testing.T, pool *pgxpool.Pool) *ReadPins { + t.Helper() + pins, err := NewReadPins(pool, s.schema) + if err != nil { + s.fail("build read pins", err) + } + return pins +} + +func (s *rprService) registry(t *testing.T, pool *pgxpool.Pool) *Registry { + t.Helper() + registry, err := New(pool, s.schema) + if err != nil { + s.fail("build registry", err) + } + return registry +} + +func (s *rprService) migrate(t *testing.T) { + t.Helper() + history := mustOwnerHistorySubset(t, 1, 4, 9) + runner, err := storepostgres.NewMigrationRunner(s.w, s.schema, history) + if err != nil { + s.fail("new migration runner", err) + } + result, err := runner.Migrate(s.ctx) + if err != nil { + s.fail("migrate retention history", err) + } + if result.LatestVersion != 9 || len(result.Applied) != 3 { + t.Fatalf("applied history = %+v, want ledger + publication_bundle_catalog + publication_read_pins", result.Applied) + } + want := []storepostgres.LedgerRecord{ + {Version: 1, Name: "ledger", Checksum: "db70f7758495f7557b3a694149b80ed6e5d4cb74ea5aa7d59a1a2eaa46fc9cde"}, + {Version: 4, Name: "publication_bundle_catalog", Checksum: "adcb4683ffd1611b9c68e7a49aaa4b8670d106d65df927cbb8e11866be224943"}, + {Version: 9, Name: "publication_read_pins", Checksum: "d538a21e219f22e5384d43a777c45cd69eec5dcd968f49e5a8ea2b8fddf4f62d"}, + } + for i, w := range want { + got := result.Applied[i] + if got.Version != w.Version || got.Name != w.Name || got.Checksum != w.Checksum { + t.Fatalf("applied[%d] = {Version:%d Name:%q Checksum:%q}, want {Version:%d Name:%q Checksum:%q}", + i, got.Version, got.Name, got.Checksum, w.Version, w.Name, w.Checksum) + } + } +} + +func TestMigrationsCoexistStoreDir000009AndReadPins000009(t *testing.T) { + raw := testPostgresURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 480*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect admin: %v", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("verify_rpr_it_%d_%s", time.Now().UnixNano(), randomHex()) + if !rprResourceNamePattern.MatchString(dbName) { + t.Fatalf("coexistence database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + t.Fatalf("create coexistence database: %v", err) + } + rprRegisterOwned(dbName) + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(context.Background()), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop coexistence database: %v", err) + } + }) + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + pool, err := newOwnedPool(ctx, u.String()) + if err != nil { + t.Fatalf("connect coexistence database: %v", err) + } + defer pool.Close() + + embedded, err := storepostgres.EmbeddedMigrations() + if err != nil { + t.Fatalf("embedded migrations: %v", err) + } + var ledger, store9 storepostgres.Migration + var ledgerFound, store9Found bool + for _, m := range embedded { + if m.Version == 1 && m.Name == "ledger" { + ledger, ledgerFound = m, true + } + if m.Version == 9 && m.Name == "semantic_inventory_contribution" { + store9, store9Found = m, true + } + } + if !ledgerFound { + t.Fatal("embedded history lacks the version 1 ledger bootstrap") + } + if !store9Found { + t.Fatal("embedded history lacks 000009_semantic_inventory_contribution: catalog-05 must be present for the coexistence witness") + } + catalog := mustOwnerHistorySubset(t, 4) + if len(catalog) != 1 || catalog[0].Version != 4 || catalog[0].Name != "publication_bundle_catalog" { + t.Fatalf("owner history version 4 = %+v, want exactly version 4 publication_bundle_catalog", catalog) + } + retention := mustOwnerHistorySubset(t, 9) + if len(retention) != 1 || retention[0].Version != 9 || retention[0].Name != "publication_read_pins" { + t.Fatalf("owner history version 9 = %+v, want exactly version 9 publication_read_pins", retention) + } + + pubSchema := fmt.Sprintf("verify_rpr_it_%d_%s", time.Now().UnixNano(), randomHex()) + storeSchema := fmt.Sprintf("verify_rpr_it_%d_%s", time.Now().UnixNano()+1, randomHex()) + for _, schema := range []string{pubSchema, storeSchema} { + if !rprResourceNamePattern.MatchString(schema) { + t.Fatalf("coexistence schema name %q does not match the required pattern", schema) + } + if _, err := pool.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + t.Fatalf("create coexistence schema: %v", err) + } + } + + pubRunner, err := storepostgres.NewMigrationRunner(pool, pubSchema, + append(append([]storepostgres.Migration{ledger}, catalog...), retention...)) + if err != nil { + t.Fatalf("publication-space runner: %v", err) + } + pubResult, err := pubRunner.Migrate(ctx) + if err != nil { + t.Fatalf("publication-space migrate: %v", err) + } + wantPub := []storepostgres.LedgerRecord{ + {Version: 1, Name: "ledger", Checksum: ledger.Checksum}, + {Version: 4, Name: "publication_bundle_catalog", Checksum: catalog[0].Checksum}, + {Version: 9, Name: "publication_read_pins", Checksum: retention[0].Checksum}, + } + if pubResult.LatestVersion != 9 || len(pubResult.Applied) != len(wantPub) { + t.Fatalf("publication-space result = %+v, want latest 9 with %+v", pubResult, wantPub) + } + for i, want := range wantPub { + got := pubResult.Applied[i] + if got.Version != want.Version || got.Name != want.Name || got.Checksum != want.Checksum { + t.Fatalf("publication-space applied[%d] = {Version:%d Name:%q Checksum:%q}, want {Version:%d Name:%q Checksum:%q}", + i, got.Version, got.Name, got.Checksum, want.Version, want.Name, want.Checksum) + } + } + storeRunner, err := storepostgres.NewMigrationRunner(pool, storeSchema, embedded) + if err != nil { + t.Fatalf("store-space runner: %v", err) + } + storeResult, err := storeRunner.Migrate(ctx) + if err != nil { + t.Fatalf("store-space migrate: %v", err) + } + maxVersion, maxEntry := embedded[0].Version, embedded[0] + for _, m := range embedded { + if m.Version > maxVersion { + maxVersion, maxEntry = m.Version, m + } + } + if storeResult.LatestVersion != maxVersion || len(storeResult.Applied) != len(embedded) { + t.Fatalf("store-space result latest = %d applied = %d, want %d and %d", storeResult.LatestVersion, len(storeResult.Applied), maxVersion, len(embedded)) + } + last := storeResult.Applied[len(storeResult.Applied)-1] + if last.Version != maxEntry.Version || last.Name != maxEntry.Name || last.Checksum != maxEntry.Checksum { + t.Fatalf("store-space last applied = {Version:%d Name:%q Checksum:%q}, want the maximum embedded entry {Version:%d Name:%q Checksum:%q}", + last.Version, last.Name, last.Checksum, maxEntry.Version, maxEntry.Name, maxEntry.Checksum) + } + + ledgerRows := func(schema string) []string { + rows, err := pool.Query(ctx, "SELECT version, name, checksum FROM "+ + pgx.Identifier{schema, storepostgres.LedgerTable}.Sanitize()+" ORDER BY version") + if err != nil { + t.Fatalf("ledger %s: %v", schema, err) + } + defer rows.Close() + var out []string + for rows.Next() { + var v int64 + var name, checksum string + if err := rows.Scan(&v, &name, &checksum); err != nil { + t.Fatalf("ledger scan %s: %v", schema, err) + } + out = append(out, fmt.Sprintf("%d|%s|%s", v, name, checksum)) + } + if err := rows.Err(); err != nil { + t.Fatalf("ledger rows %s: %v", schema, err) + } + return out + } + pubLedger := ledgerRows(pubSchema) + storeLedger := ledgerRows(storeSchema) + if len(pubLedger) != 3 { + t.Fatalf("publication ledger = %v, want exactly 3 rows", pubLedger) + } + if pubLedger[2] != fmt.Sprintf("9|publication_read_pins|%s", retention[0].Checksum) { + t.Fatalf("publication ledger version 9 = %q, want publication_read_pins with the load checksum", pubLedger[2]) + } + store9Row := "" + for _, row := range storeLedger { + if strings.HasPrefix(row, "9|") { + store9Row = row + } + } + if store9Row != fmt.Sprintf("9|semantic_inventory_contribution|%s", store9.Checksum) { + t.Fatalf("store ledger version 9 = %q, want semantic_inventory_contribution with the load checksum (dual existence against the publication ledger's version 9)", store9Row) + } + if tail := storeLedger[len(storeLedger)-1]; tail != fmt.Sprintf("%d|%s|%s", maxEntry.Version, maxEntry.Name, maxEntry.Checksum) { + t.Fatalf("store ledger tail = %q, want the maximum embedded entry %d|%s|...", tail, maxEntry.Version, maxEntry.Name) + } + for _, probe := range []struct { + schema, table string + }{ + {pubSchema, ExecutionReadPinsTable}, + {pubSchema, ExecutionCleanupClaimsTable}, + {storeSchema, "fhir_semantic_inventory_contribution"}, + } { + var present bool + err := pool.QueryRow(ctx, + "SELECT to_regclass($1) IS NOT NULL", + probe.schema+"."+probe.table).Scan(&present) + if err != nil || !present { + t.Fatalf("migration-created table %s.%s absent after coexistence migration (err %v)", probe.schema, probe.table, err) + } + } + + pubAgain, err := pubRunner.Migrate(ctx) + if err != nil { + t.Fatalf("publication-space replay: %v", err) + } + if pubAgain.LatestVersion != pubResult.LatestVersion || !reflect.DeepEqual(pubAgain.Applied, pubResult.Applied) { + t.Fatalf("publication-space replay = %+v, want identical to first run %+v", pubAgain, pubResult) + } + storeAgain, err := storeRunner.Migrate(ctx) + if err != nil { + t.Fatalf("store-space replay: %v", err) + } + if storeAgain.LatestVersion != storeResult.LatestVersion || !reflect.DeepEqual(storeAgain.Applied, storeResult.Applied) { + t.Fatalf("store-space replay = %+v, want identical to first run %+v", storeAgain, storeResult) + } + if got := ledgerRows(pubSchema); !reflect.DeepEqual(got, pubLedger) { + t.Fatalf("publication ledger changed on replay: %v vs %v", got, pubLedger) + } + if got := ledgerRows(storeSchema); !reflect.DeepEqual(got, storeLedger) { + t.Fatalf("store ledger changed on replay: %v vs %v", got, storeLedger) + } + retentionAgain := mustOwnerHistorySubset(t, 9) + if len(retentionAgain) != len(retention) || retentionAgain[0].Version != retention[0].Version || + retentionAgain[0].Name != retention[0].Name || retentionAgain[0].Checksum != retention[0].Checksum { + t.Fatalf("owner history version 9 reloaded differently: %+v vs %+v", retentionAgain, retention) + } + embeddedAgain, err := storepostgres.EmbeddedMigrations() + if err != nil || !reflect.DeepEqual(embeddedAgain, embedded) { + t.Fatalf("embedded history reloaded differently: %v", err) + } + + if _, err := storepostgres.NewMigrationRunner(pool, pubSchema, + []storepostgres.Migration{store9, retention[0]}); !errors.Is(err, storepostgres.ErrInvalidMigration) || + err == nil || !strings.Contains(err.Error(), "duplicate version 9") { + t.Fatalf("same-schema double 000009 = %v, want ErrInvalidMigration duplicate version 9", err) + } +} + +const tsText = `to_char(%s AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"')` + +func (s *rprService) queryRows(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) []string { + t.Helper() + rows, err := pool.Query(s.ctx, sql, args...) + if err != nil { + s.fail("witness "+sql, err) + } + defer rows.Close() + var out []string + for rows.Next() { + count := len(rows.RawValues()) + cols := make([]any, 0, count) + texts := make([]string, count) + for i := range texts { + cols = append(cols, &texts[i]) + } + if err := rows.Scan(cols...); err != nil { + s.fail("witness scan", err) + } + out = append(out, strings.Join(texts, "|")) + } + if err := rows.Err(); err != nil { + s.fail("witness rows", err) + } + return out +} + +func (s *rprService) pinsRows(t *testing.T, pool *pgxpool.Pool, executionID string) []string { + t.Helper() + return s.queryRows(t, pool, + "SELECT owner_id, "+fmt.Sprintf(tsText, "expires_at")+", "+fmt.Sprintf(tsText, "updated_at")+ + " FROM "+pgx.Identifier{s.schema, ExecutionReadPinsTable}.Sanitize()+ + " WHERE execution_id = $1 ORDER BY owner_id", executionID) +} + +func (s *rprService) claimRow(t *testing.T, pool *pgxpool.Pool, executionID string) string { + t.Helper() + rows := s.queryRows(t, pool, + "SELECT owner_id, "+fmt.Sprintf(tsText, "expires_at")+", "+fmt.Sprintf(tsText, "updated_at")+ + " FROM "+pgx.Identifier{s.schema, ExecutionCleanupClaimsTable}.Sanitize()+ + " WHERE execution_id = $1", executionID) + if len(rows) > 1 { + t.Fatalf("cleanup claims for %s = %v, want at most one row", executionID, rows) + } + if len(rows) == 0 { + return "" + } + return rows[0] +} + +func (s *rprService) anchorRow(t *testing.T, pool *pgxpool.Pool, executionID string) string { + t.Helper() + rows := s.queryRows(t, pool, + "SELECT "+fmt.Sprintf(tsText, "updated_at")+ + " FROM "+pgx.Identifier{s.schema, ExecutionRetentionTable}.Sanitize()+" WHERE execution_id = $1", executionID) + if len(rows) > 1 { + t.Fatalf("anchor rows for %s = %v, want at most one", executionID, rows) + } + if len(rows) == 0 { + return "" + } + return rows[0] +} + +func (s *rprService) stateDigest(t *testing.T, pool *pgxpool.Pool) string { + t.Helper() + retention := pgx.Identifier{s.schema, ExecutionRetentionTable}.Sanitize() + pins := pgx.Identifier{s.schema, ExecutionReadPinsTable}.Sanitize() + cleanup := pgx.Identifier{s.schema, ExecutionCleanupClaimsTable}.Sanitize() + sql := "SELECT md5(" + + "(SELECT COALESCE(string_agg(execution_id || '@' || " + fmt.Sprintf(tsText, "updated_at") + ", E'\\n' ORDER BY execution_id), '') FROM " + retention + ") || '#' || " + + "(SELECT COALESCE(string_agg(execution_id || '@' || owner_id || '@' || " + fmt.Sprintf(tsText, "expires_at") + " || '@' || " + fmt.Sprintf(tsText, "updated_at") + ", E'\\n' ORDER BY execution_id, owner_id), '') FROM " + pins + ") || '#' || " + + "(SELECT COALESCE(string_agg(execution_id || '@' || owner_id || '@' || " + fmt.Sprintf(tsText, "expires_at") + " || '@' || " + fmt.Sprintf(tsText, "updated_at") + ", E'\\n' ORDER BY execution_id, owner_id), '') FROM " + cleanup + "))" + var digest string + if err := pool.QueryRow(s.ctx, sql).Scan(&digest); err != nil { + s.fail("state digest", err) + } + return digest +} + +func (s *rprService) backendPID(t *testing.T, pool *pgxpool.Pool) uint32 { + t.Helper() + var pid uint32 + if err := pool.QueryRow(s.ctx, "SELECT pg_backend_pid()::int").Scan(&pid); err != nil { + s.fail("backend pid", err) + } + return pid +} + +func (s *rprService) seedPinDirect(t *testing.T, executionID, owner string, expires time.Time) { + t.Helper() + pins := pgx.Identifier{s.schema, ExecutionReadPinsTable}.Sanitize() + retention := pgx.Identifier{s.schema, ExecutionRetentionTable}.Sanitize() + if _, err := s.w.Exec(s.ctx, + "INSERT INTO "+retention+" (execution_id, updated_at) VALUES ($1, $2) ON CONFLICT (execution_id) DO NOTHING", + executionID, time.Now().UTC()); err != nil { + s.fail("seed anchor", err) + } + if _, err := s.w.Exec(s.ctx, + "INSERT INTO "+pins+" (execution_id, owner_id, expires_at, updated_at) VALUES ($1, $2, $3, $4)"+ + " ON CONFLICT (execution_id, owner_id) DO UPDATE SET expires_at = $3, updated_at = $4", + executionID, owner, expires.UTC(), time.Now().UTC()); err != nil { + s.fail("seed pin", err) + } +} + +func (s *rprService) seedClaimDirect(t *testing.T, executionID, owner string, expires time.Time) { + t.Helper() + cleanup := pgx.Identifier{s.schema, ExecutionCleanupClaimsTable}.Sanitize() + retention := pgx.Identifier{s.schema, ExecutionRetentionTable}.Sanitize() + if _, err := s.w.Exec(s.ctx, + "INSERT INTO "+retention+" (execution_id, updated_at) VALUES ($1, $2) ON CONFLICT (execution_id) DO NOTHING", + executionID, time.Now().UTC()); err != nil { + s.fail("seed anchor", err) + } + if _, err := s.w.Exec(s.ctx, + "INSERT INTO "+cleanup+" (execution_id, owner_id, expires_at, updated_at) VALUES ($1, $2, $3, $4)"+ + " ON CONFLICT (execution_id) DO UPDATE SET owner_id = $2, expires_at = $3, updated_at = $4", + executionID, owner, expires.UTC(), time.Now().UTC()); err != nil { + s.fail("seed claim", err) + } +} + +func (s *rprService) clearStateDirect(t *testing.T, executionID string) { + t.Helper() + pins := pgx.Identifier{s.schema, ExecutionReadPinsTable}.Sanitize() + cleanup := pgx.Identifier{s.schema, ExecutionCleanupClaimsTable}.Sanitize() + retention := pgx.Identifier{s.schema, ExecutionRetentionTable}.Sanitize() + for _, sql := range []string{ + "DELETE FROM " + pins + " WHERE execution_id = $1", + "DELETE FROM " + cleanup + " WHERE execution_id = $1", + "DELETE FROM " + retention + " WHERE execution_id = $1", + } { + if _, err := s.w.Exec(s.ctx, sql, executionID); err != nil { + s.fail("clear state", err) + } + } +} + +func setReadPinGate(fn func(readPinAttempt) error) { + if fn == nil { + readPinGate.Store(nil) + return + } + readPinGate.Store(&fn) +} + +var ( + rprFarFuture = time.Date(2126, 11, 9, 1, 2, 3, 400000, time.UTC) + rprPast = time.Date(2020, 1, 2, 3, 4, 5, 600000, time.UTC) + rprRenewedExp = time.Date(2127, 5, 6, 7, 8, 9, 100000, time.UTC) +) + +func TestReadPinsGoldenRows(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + setReadPinGate(nil) + defer setReadPinGate(nil) + + const exec = "exec-golden-1" + if got := s.stateDigest(t, s.w); got == "" { + t.Fatal("state digest witness is broken") + } + + if got := s.pinsRows(t, s.w, exec); len(got) != 0 { + t.Fatalf("fresh pins = %v, want none", got) + } + if got := s.claimRow(t, s.w, exec); got != "" { + t.Fatalf("fresh claim = %q, want absent", got) + } + if got := s.anchorRow(t, s.w, exec); got != "" { + t.Fatalf("fresh anchor = %q, want absent", got) + } + + claimed, err := s.pinsA.AcquireExecutionReadPin(ctx, exec, "reader-a", rprFarFuture) + if err != nil || !claimed { + t.Fatalf("acquire reader-a = %v, %v", claimed, err) + } + got := s.pinsRows(t, s.w, exec) + if len(got) != 1 { + t.Fatalf("pins = %v, want one literal row", got) + } + if !strings.HasPrefix(got[0], "reader-a|"+rprFarFuture.Format("2006-01-02T15:04:05.000000Z")+"|") { + t.Fatalf("pins[0] = %q, want literal owner and expires_at %s", got[0], rprFarFuture) + } + if s.anchorRow(t, s.w, exec) == "" { + t.Fatal("anchor row missing after first acquire") + } + anchorFirst := s.anchorRow(t, s.w, exec) + + if claimed, err := s.pinsB.AcquireExecutionReadPin(ctx, exec, "reader-b", rprRenewedExp); err != nil || !claimed { + t.Fatalf("acquire reader-b = %v, %v", claimed, err) + } + got = s.pinsRows(t, s.w, exec) + if len(got) != 2 || + !strings.HasPrefix(got[0], "reader-a|"+rprFarFuture.Format("2006-01-02T15:04:05.000000Z")+"|") || + !strings.HasPrefix(got[1], "reader-b|"+rprRenewedExp.Format("2006-01-02T15:04:05.000000Z")+"|") { + t.Fatalf("pins = %v, want both literal rows", got) + } + + if claimed, err := s.pinsA.AcquireExecutionReadPin(ctx, exec, "reader-a", rprRenewedExp); err != nil || !claimed { + t.Fatalf("re-acquire reader-a = %v, %v", claimed, err) + } + got = s.pinsRows(t, s.w, exec) + if len(got) != 2 || + !strings.HasPrefix(got[0], "reader-a|"+rprRenewedExp.Format("2006-01-02T15:04:05.000000Z")+"|") { + t.Fatalf("pins after re-acquire = %v, want one refreshed reader-a row", got) + } + anchorSecond := s.anchorRow(t, s.w, exec) + if anchorFirst == anchorSecond { + t.Fatalf("anchor updated_at did not move on a successful transition: %q", anchorFirst) + } + + if _, err := s.w.Exec(ctx, + "DELETE FROM "+pgx.Identifier{s.schema, ExecutionReadPinsTable}.Sanitize()+" WHERE execution_id = $1", exec); err != nil { + s.fail("clear pins for trim check", err) + } + if claimed, err := s.pinsA.AcquireExecutionReadPin(ctx, " "+exec+" ", " reader-a ", rprFarFuture); err != nil || !claimed { + t.Fatalf("trim acquire = %v, %v", claimed, err) + } + if got := s.pinsRows(t, s.w, exec); len(got) != 1 || !strings.HasPrefix(got[0], "reader-a|") { + t.Fatalf("trimmed acquire stored %v, want one owner 'reader-a' row", got) + } + + s.seedPinDirect(t, exec, "expired-owner", rprPast) + if got := s.pinsRows(t, s.w, exec); len(got) != 2 { + t.Fatalf("seeded pins = %v, want reader-a + expired-owner", got) + } + if claimed, err := s.pinsA.AcquireExecutionReadPin(ctx, exec, "reader-c", rprFarFuture); err != nil || !claimed { + t.Fatalf("acquire reader-c = %v, %v", claimed, err) + } + got = s.pinsRows(t, s.w, exec) + if len(got) != 2 || !strings.HasPrefix(got[0], "reader-a|") || !strings.HasPrefix(got[1], "reader-c|") { + t.Fatalf("pins after acquire GC = %v, want reader-a and reader-c (expired-owner collected)", got) + } + + s.seedPinDirect(t, exec, "expired-owner", rprPast) + before := s.anchorRow(t, s.w, exec) + time.Sleep(2 * time.Millisecond) + if err := s.pinsA.ReleaseExecutionReadPin(ctx, exec, "no-such-owner"); err != nil { + t.Fatalf("release by non-holder: %v", err) + } + got = s.pinsRows(t, s.w, exec) + if len(got) != 3 || !strings.HasPrefix(got[0], "expired-owner|") { + t.Fatalf("pins after foreign release = %v, want expired-owner retained (non-GC parity)", got) + } + if after := s.anchorRow(t, s.w, exec); after == before { + t.Fatal("anchor updated_at must move on every release while state exists (Arango doc-rewrite parity)") + } + + if claimed, err := s.pinsA.RenewExecutionReadPin(ctx, exec, "reader-a", rprRenewedExp); err != nil || !claimed { + t.Fatalf("renew reader-a = %v, %v", claimed, err) + } + got = s.pinsRows(t, s.w, exec) + if len(got) != 2 || !strings.HasPrefix(got[0], "reader-a|"+rprRenewedExp.Format("2006-01-02T15:04:05.000000Z")+"|") { + t.Fatalf("pins after renew = %v, want reader-a refreshed and expired-owner collected", got) + } + + if _, err := s.w.Exec(ctx, "DELETE FROM "+pgx.Identifier{s.schema, ExecutionReadPinsTable}.Sanitize()+" WHERE execution_id = $1", exec); err != nil { + s.fail("clear live pins before expired-claim", err) + } + s.seedPinDirect(t, exec, "expired-owner", rprPast) + s.pinsA.claimTTL = 90 * time.Second + claimed, err = s.pinsA.ClaimExecutionCleanup(ctx, exec, "sweeper-1") + if err != nil || !claimed { + t.Fatalf("claim over expired pins = %v, %v", claimed, err) + } + if got := s.pinsRows(t, s.w, exec); len(got) != 0 { + t.Fatalf("pins after claim = %v, want all expired pins collected (pins: [] parity)", got) + } + claim := s.claimRow(t, s.w, exec) + if !strings.HasPrefix(claim, "sweeper-1|") { + t.Fatalf("claim row = %q, want owner sweeper-1", claim) + } + parts := strings.Split(claim, "|") + if len(parts) != 3 { + t.Fatalf("claim row = %q, want owner|expires|updated", claim) + } + claimExpiry, err := time.Parse("2006-01-02T15:04:05.000000Z", parts[1]) + if err != nil { + t.Fatalf("claim expiry %q: %v", parts[1], err) + } + if d := time.Until(claimExpiry); d < 85*time.Second || d > 90*time.Second+time.Second { + t.Fatalf("claim expiry in %v, want ~90s (pinned claimTTL)", d) + } + + s.seedPinDirect(t, exec, "straggler", rprPast) + if claimed, err := s.pinsA.RenewExecutionCleanup(ctx, exec, "sweeper-1", rprFarFuture); err != nil || !claimed { + t.Fatalf("renew claim = %v, %v", claimed, err) + } + if got := s.pinsRows(t, s.w, exec); len(got) != 1 || !strings.HasPrefix(got[0], "straggler|") { + t.Fatalf("pins after renew-cleanup = %v, want the straggler row untouched (baseline parity)", got) + } + if claim := s.claimRow(t, s.w, exec); !strings.HasPrefix(claim, "sweeper-1|"+rprFarFuture.Format("2006-01-02T15:04:05.000000Z")+"|") { + t.Fatalf("claim after renew = %q, want owner sweeper-1 with literal renewed expiry", claim) + } + + claimBefore := s.claimRow(t, s.w, exec) + anchorBefore := s.anchorRow(t, s.w, exec) + if err := s.pinsB.ReleaseExecutionCleanup(ctx, exec, "not-the-owner"); err != nil { + t.Fatalf("foreign release-cleanup: %v", err) + } + if got := s.claimRow(t, s.w, exec); got != claimBefore { + t.Fatalf("claim changed by foreign release: %q -> %q", claimBefore, got) + } + if got := s.anchorRow(t, s.w, exec); got != anchorBefore { + t.Fatalf("anchor moved on a foreign (zero-row) release: %q -> %q", anchorBefore, got) + } + if err := s.pinsA.ReleaseExecutionCleanup(ctx, exec, "sweeper-1"); err != nil { + t.Fatalf("owner release-cleanup: %v", err) + } + if got := s.claimRow(t, s.w, exec); got != "" { + t.Fatalf("claim after owner release = %q, want absent", got) + } + + if err := s.pinsA.ReleaseExecutionReadPin(ctx, "exec-never-seen", "reader-x"); err != nil { + t.Fatalf("release-pin with no state: %v", err) + } + if err := s.pinsA.ReleaseExecutionCleanup(ctx, "exec-never-seen", "sweeper-x"); err != nil { + t.Fatalf("release-cleanup with no state: %v", err) + } + if got := s.anchorRow(t, s.w, "exec-never-seen"); got != "" { + t.Fatalf("anchor minted by stateless releases: %q", got) + } + if got := s.pinsRows(t, s.w, "exec-never-seen"); len(got) != 0 { + t.Fatalf("pins minted by stateless releases: %v", got) + } +} + +func TestReadPinsExactNamedRefusals(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + setReadPinGate(nil) + defer setReadPinGate(nil) + + const exec = "exec-refusals" + baseDigest := s.stateDigest(t, s.w) + pinCases := []struct { + op string + call func() error + sentry error + }{ + {"acquire-empty-exec", func() error { _, err := s.pinsA.AcquireExecutionReadPin(ctx, " ", "owner", rprFarFuture); return err }, publication.ErrExecutionReadPinLost}, + {"acquire-empty-owner", func() error { _, err := s.pinsA.AcquireExecutionReadPin(ctx, exec, " ", rprFarFuture); return err }, publication.ErrExecutionReadPinLost}, + {"renew-empty-exec", func() error { _, err := s.pinsA.RenewExecutionReadPin(ctx, "", "owner", rprFarFuture); return err }, publication.ErrExecutionReadPinLost}, + {"release-empty-owner", func() error { return s.pinsA.ReleaseExecutionReadPin(ctx, exec, " ") }, publication.ErrExecutionReadPinLost}, + {"claim-empty-exec", func() error { _, err := s.pinsA.ClaimExecutionCleanup(ctx, "", "sweeper"); return err }, ErrInvalidCleanupIdentity}, + {"renew-cleanup-empty-owner", func() error { _, err := s.pinsA.RenewExecutionCleanup(ctx, exec, "", rprFarFuture); return err }, ErrInvalidCleanupIdentity}, + {"release-cleanup-empty-exec", func() error { return s.pinsA.ReleaseExecutionCleanup(ctx, "", "sweeper") }, ErrInvalidCleanupIdentity}, + } + for _, tc := range pinCases { + if err := tc.call(); !errors.Is(err, tc.sentry) { + t.Errorf("%s: error = %v, want errors.Is %v", tc.op, err, tc.sentry) + } + } + if got := s.stateDigest(t, s.w); got != baseDigest { + t.Fatalf("identity refusals left durable state: digest %q != baseline %q", got, baseDigest) + } + + const pinned = "exec-pinned" + s.seedPinDirect(t, pinned, "reader-a", rprFarFuture) + before := s.stateDigest(t, s.w) + if claimed, err := s.pinsA.ClaimExecutionCleanup(ctx, pinned, "sweeper-1"); err != nil || claimed { + t.Fatalf("claim over active pin = %v, %v, want (false, nil)", claimed, err) + } + if got := s.stateDigest(t, s.w); got != before { + t.Fatalf("refused claim changed state: %q != %q", got, before) + } + const fresh = "exec-fresh-refuse" + s.seedClaimDirect(t, fresh, "sweeper-1", rprFarFuture) + seedDigest := s.stateDigest(t, s.w) + if claimed, err := s.pinsB.AcquireExecutionReadPin(ctx, fresh, "reader-b", rprFarFuture); err != nil || claimed { + t.Fatalf("acquire under live claim = %v, %v, want (false, nil)", claimed, err) + } + if got := s.stateDigest(t, s.w); got != seedDigest { + t.Fatalf("refused acquire changed state: %q != %q", got, seedDigest) + } + + const claimedExec = "exec-claimed" + s.seedClaimDirect(t, claimedExec, "sweeper-1", rprFarFuture) + if claimed, err := s.pinsB.ClaimExecutionCleanup(ctx, claimedExec, "sweeper-2"); err != nil || claimed { + t.Fatalf("foreign claim over live claim = %v, %v, want (false, nil)", claimed, err) + } + if claimed, err := s.pinsA.ClaimExecutionCleanup(ctx, claimedExec, "sweeper-1"); err != nil || !claimed { + t.Fatalf("same-owner re-claim = %v, %v, want (true, nil)", claimed, err) + } + + const renewExec = "exec-renew" + if claimed, err := s.pinsA.RenewExecutionReadPin(ctx, renewExec, "reader-a", rprFarFuture); err != nil || claimed { + t.Fatalf("renew with no state = %v, %v, want (false, nil)", claimed, err) + } + s.seedPinDirect(t, renewExec, "reader-a", rprPast) + if claimed, err := s.pinsA.RenewExecutionReadPin(ctx, renewExec, "reader-a", rprFarFuture); err != nil || claimed { + t.Fatalf("renew of an expired pin = %v, %v, want (false, nil)", claimed, err) + } + if claimed, err := s.pinsA.RenewExecutionReadPin(ctx, renewExec, "reader-b", rprFarFuture); err != nil || claimed { + t.Fatalf("renew of an absent pin under foreign pins = %v, %v, want (false, nil)", claimed, err) + } + if _, err := s.w.Exec(ctx, "DELETE FROM "+pgx.Identifier{s.schema, ExecutionCleanupClaimsTable}.Sanitize()); err != nil { + s.fail("clear claims", err) + } + s.seedClaimDirect(t, renewExec, "sweeper-1", rprFarFuture) + if claimed, err := s.pinsA.RenewExecutionReadPin(ctx, renewExec, "reader-a", rprFarFuture); err != nil || claimed { + t.Fatalf("renew pin under a live claim = %v, %v, want (false, nil)", claimed, err) + } + + if claimed, err := s.pinsA.RenewExecutionCleanup(ctx, "exec-no-claim", "sweeper-1", rprFarFuture); err != nil || claimed { + t.Fatalf("renew-cleanup with no claim = %v, %v", claimed, err) + } + if claimed, err := s.pinsA.RenewExecutionCleanup(ctx, claimedExec, "sweeper-2", rprFarFuture); err != nil || claimed { + t.Fatalf("renew-cleanup foreign owner = %v, %v", claimed, err) + } + s.seedClaimDirect(t, claimedExec, "sweeper-old", rprPast) + if claimed, err := s.pinsA.RenewExecutionCleanup(ctx, claimedExec, "sweeper-old", rprFarFuture); err != nil || claimed { + t.Fatalf("renew-cleanup expired claim = %v, %v", claimed, err) + } + s.seedClaimDirect(t, claimedExec, "sweeper-old", rprFarFuture) + s.seedPinDirect(t, claimedExec, "reader-straggler", rprFarFuture) + if claimed, err := s.pinsA.RenewExecutionCleanup(ctx, claimedExec, "sweeper-old", rprRenewedExp); err != nil || claimed { + t.Fatalf("renew-cleanup over an active straggler pin = %v, %v", claimed, err) + } +} + +func TestReadPinsGoldenStatements(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + defer setReadPinGate(nil) + + q := func(table string) string { + return pgx.Identifier{s.schema, table}.Sanitize() + } + golden := map[string][]string{ + "acquire-pin": { + "INSERT INTO " + q(ExecutionRetentionTable) + " (execution_id, updated_at) VALUES ($1, $2) ON CONFLICT (execution_id) DO NOTHING", + "SELECT true FROM " + q(ExecutionRetentionTable) + " WHERE execution_id = $1 FOR UPDATE", + "SELECT NOT EXISTS (SELECT 1 FROM " + q(ExecutionCleanupClaimsTable) + " WHERE execution_id = $1 AND expires_at >= $2)", + "DELETE FROM " + q(ExecutionReadPinsTable) + " WHERE execution_id = $1 AND expires_at < $2", + "INSERT INTO " + q(ExecutionReadPinsTable) + " (execution_id, owner_id, expires_at, updated_at) VALUES ($1, $2, $3, $4) ON CONFLICT (execution_id, owner_id) DO UPDATE SET expires_at = $3, updated_at = $4", + "UPDATE " + q(ExecutionRetentionTable) + " SET updated_at = $2 WHERE execution_id = $1", + }, + "renew-pin": { + "SELECT true FROM " + q(ExecutionRetentionTable) + " WHERE execution_id = $1 FOR UPDATE", + "SELECT NOT EXISTS (SELECT 1 FROM " + q(ExecutionCleanupClaimsTable) + " WHERE execution_id = $1 AND expires_at >= $2)", + "SELECT EXISTS (SELECT 1 FROM " + q(ExecutionReadPinsTable) + " WHERE execution_id = $1 AND owner_id = $2 AND expires_at >= $3)", + "DELETE FROM " + q(ExecutionReadPinsTable) + " WHERE execution_id = $1 AND expires_at < $2", + "UPDATE " + q(ExecutionReadPinsTable) + " SET expires_at = $3, updated_at = $4 WHERE execution_id = $1 AND owner_id = $2", + "UPDATE " + q(ExecutionRetentionTable) + " SET updated_at = $2 WHERE execution_id = $1", + }, + "release-pin": { + "SELECT true FROM " + q(ExecutionRetentionTable) + " WHERE execution_id = $1 FOR UPDATE", + "DELETE FROM " + q(ExecutionReadPinsTable) + " WHERE execution_id = $1 AND owner_id = $2", + "UPDATE " + q(ExecutionRetentionTable) + " SET updated_at = $2 WHERE execution_id = $1", + }, + "claim-cleanup": { + "INSERT INTO " + q(ExecutionRetentionTable) + " (execution_id, updated_at) VALUES ($1, $2) ON CONFLICT (execution_id) DO NOTHING", + "SELECT true FROM " + q(ExecutionRetentionTable) + " WHERE execution_id = $1 FOR UPDATE", + "SELECT NOT EXISTS (SELECT 1 FROM " + q(ExecutionReadPinsTable) + " WHERE execution_id = $1 AND expires_at >= $2)", + "SELECT owner_id, expires_at FROM " + q(ExecutionCleanupClaimsTable) + " WHERE execution_id = $1", + "DELETE FROM " + q(ExecutionReadPinsTable) + " WHERE execution_id = $1 AND expires_at < $2", + "INSERT INTO " + q(ExecutionCleanupClaimsTable) + " (execution_id, owner_id, expires_at, updated_at) VALUES ($1, $2, $3, $4) ON CONFLICT (execution_id) DO UPDATE SET owner_id = $2, expires_at = $3, updated_at = $4", + "UPDATE " + q(ExecutionRetentionTable) + " SET updated_at = $2 WHERE execution_id = $1", + }, + "renew-cleanup": { + "SELECT true FROM " + q(ExecutionRetentionTable) + " WHERE execution_id = $1 FOR UPDATE", + "SELECT owner_id, expires_at FROM " + q(ExecutionCleanupClaimsTable) + " WHERE execution_id = $1", + "SELECT NOT EXISTS (SELECT 1 FROM " + q(ExecutionReadPinsTable) + " WHERE execution_id = $1 AND expires_at >= $2)", + "UPDATE " + q(ExecutionCleanupClaimsTable) + " SET expires_at = $3, updated_at = $4 WHERE execution_id = $1 AND owner_id = $2", + "UPDATE " + q(ExecutionRetentionTable) + " SET updated_at = $2 WHERE execution_id = $1", + }, + "release-cleanup": { + "SELECT true FROM " + q(ExecutionRetentionTable) + " WHERE execution_id = $1 FOR UPDATE", + "DELETE FROM " + q(ExecutionCleanupClaimsTable) + " WHERE execution_id = $1 AND owner_id = $2", + "UPDATE " + q(ExecutionRetentionTable) + " SET updated_at = $2 WHERE execution_id = $1", + }, + } + + type op struct { + name string + call func() + } + const exec = "exec-golden-sql" + ops := []op{ + {"acquire-pin", func() { must(s.pinsA.AcquireExecutionReadPin(ctx, exec, "reader-a", rprFarFuture))(t) }}, + {"renew-pin", func() { must(s.pinsA.RenewExecutionReadPin(ctx, exec, "reader-a", rprFarFuture))(t) }}, + {"claim-cleanup", func() { + if _, err := s.w.Exec(ctx, "DELETE FROM "+q(ExecutionReadPinsTable)+" WHERE execution_id = $1", exec); err != nil { + s.fail("clear pins", err) + } + must(s.pinsA.ClaimExecutionCleanup(ctx, exec, "sweeper-1"))(t) + }}, + {"renew-cleanup", func() { must(s.pinsA.RenewExecutionCleanup(ctx, exec, "sweeper-1", rprFarFuture))(t) }}, + {"release-cleanup", func() { + if err := s.pinsA.ReleaseExecutionCleanup(ctx, exec, "sweeper-1"); err != nil { + s.fail("release-cleanup", err) + } + }}, + {"release-pin", func() { + must(s.pinsA.AcquireExecutionReadPin(ctx, exec, "reader-a", rprFarFuture))(t) + if err := s.pinsA.ReleaseExecutionReadPin(ctx, exec, "reader-a"); err != nil { + s.fail("release-pin", err) + } + }}, + } + for _, o := range ops { + var attempts []readPinAttempt + setReadPinGate(func(a readPinAttempt) error { + attempts = append(attempts, a) + return nil + }) + o.call() + setReadPinGate(nil) + if len(attempts) == 0 { + t.Fatalf("%s: no gate witness", o.name) + } + a := attempts[len(attempts)-1] + if a.Op != o.name { + t.Fatalf("witness op = %q, want %q", a.Op, o.name) + } + if !a.Claimed { + t.Fatalf("%s witness claimed = false, want true for the golden committed path", o.name) + } + if a.BackendPID == 0 { + t.Fatalf("%s witness has no backend pid", o.name) + } + want := golden[o.name] + if strings.Join(a.Statements, "\n") != strings.Join(want, "\n") { + t.Fatalf("%s statements =\n%v\nwant exactly\n%v", o.name, a.Statements, want) + } + } +} + +func must(claimed bool, err error) func(t *testing.T) { + return func(t *testing.T) { + t.Helper() + if err != nil { + t.Fatalf("operation failed: %v", err) + } + if !claimed { + t.Fatalf("operation refused, want claimed") + } + } +} + +func TestReadPinsClaimAcquireSerializeRace(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + setReadPinGate(nil) + defer setReadPinGate(nil) + + pidA := s.backendPID(t, s.a) + pidB := s.backendPID(t, s.b) + if pidA == pidB { + t.Fatalf("racing connections share backend %d; each pool must own one real backend", pidA) + } + const rounds = 12 + for round := 0; round < rounds; round++ { + exec := fmt.Sprintf("exec-race-ac-_%02d", round) + start := make(chan struct{}) + var ( + pinOK bool + pinErr error + claimOK bool + claimErr error + ) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + <-start + pinOK, pinErr = s.pinsA.AcquireExecutionReadPin(ctx, exec, "reader-a", rprFarFuture) + }() + go func() { + defer wg.Done() + <-start + claimOK, claimErr = s.pinsB.ClaimExecutionCleanup(ctx, exec, "sweeper-1") + }() + close(start) + wg.Wait() + if pinErr != nil || claimErr != nil { + t.Fatalf("round %d: pinErr=%v claimErr=%v", round, pinErr, claimErr) + } + if pinOK && claimOK { + t.Fatalf("round %d: BOTH a live pin and a live cleanup claim were granted for %s -- serialization violated", round, exec) + } + if !pinOK && !claimOK { + t.Fatalf("round %d: neither side won %s (digest %s)", round, exec, s.stateDigest(t, s.w)) + } + if pinOK { + rows := s.pinsRows(t, s.w, exec) + if len(rows) != 1 || !strings.HasPrefix(rows[0], "reader-a|"+rprFarFuture.Format("2006-01-02T15:04:05.000000Z")+"|") || + s.claimRow(t, s.w, exec) != "" { + t.Fatalf("round %d: pin-winner state = pins %v claim %q", round, rows, s.claimRow(t, s.w, exec)) + } + } else { + claim := s.claimRow(t, s.w, exec) + if !strings.HasPrefix(claim, "sweeper-1|") || len(s.pinsRows(t, s.w, exec)) != 0 { + t.Fatalf("round %d: claim-winner state = pins %v claim %q", round, s.pinsRows(t, s.w, exec), claim) + } + } + s.clearStateDirect(t, exec) + must(s.pinsA.AcquireExecutionReadPin(ctx, exec, "reader-a", rprFarFuture))(t) + pinDigest := s.stateDigest(t, s.w) + must(s.pinsA.AcquireExecutionReadPin(ctx, exec, "reader-a", rprFarFuture))(t) + if got := s.stateDigest(t, s.w); got != pinDigest { + rows1 := s.pinsRows(t, s.w, exec) + if len(rows1) != 1 || !strings.HasPrefix(rows1[0], "reader-a|"+rprFarFuture.Format("2006-01-02T15:04:05.000000Z")+"|") { + t.Fatalf("round %d: convergence replay state = %v", round, rows1) + } + } + s.clearStateDirect(t, exec) + } +} + +func TestReadPinsClaimVsClaimRace(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + setReadPinGate(nil) + defer setReadPinGate(nil) + + if pidA, pidB := s.backendPID(t, s.a), s.backendPID(t, s.b); pidA == pidB { + t.Fatalf("racing connections share backend %d", pidA) + } + const rounds = 12 + for round := 0; round < rounds; round++ { + exec := fmt.Sprintf("exec-race-cc-%02d", round) + start := make(chan struct{}) + var ( + okA, okB bool + errA, errB error + ) + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); <-start; okA, errA = s.pinsA.ClaimExecutionCleanup(ctx, exec, "sweeper-a") }() + go func() { defer wg.Done(); <-start; okB, errB = s.pinsB.ClaimExecutionCleanup(ctx, exec, "sweeper-b") }() + close(start) + wg.Wait() + if errA != nil || errB != nil { + t.Fatalf("round %d: %v / %v", round, errA, errB) + } + if okA == okB { + t.Fatalf("round %d: both claims answered %v/%v -- exactly one sweeper may win", round, okA, okB) + } + winner := "sweeper-a" + if !okA { + winner = "sweeper-b" + } + claim := s.claimRow(t, s.w, exec) + if !strings.HasPrefix(claim, winner+"|") { + t.Fatalf("round %d: claim row %q, want winner %s", round, claim, winner) + } + s.clearStateDirect(t, exec) + } +} + +func TestReadPinsAcquireVsAcquireRace(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + setReadPinGate(nil) + defer setReadPinGate(nil) + + if pidA, pidB := s.backendPID(t, s.a), s.backendPID(t, s.b); pidA == pidB { + t.Fatalf("racing connections share backend %d", pidA) + } + const rounds = 12 + for round := 0; round < rounds; round++ { + exec := fmt.Sprintf("exec-race-aa-%02d", round) + start := make(chan struct{}) + var okA, okB bool + var errA, errB error + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + <-start + okA, errA = s.pinsA.AcquireExecutionReadPin(ctx, exec, "reader-a", rprFarFuture) + }() + go func() { + defer wg.Done() + <-start + okB, errB = s.pinsB.AcquireExecutionReadPin(ctx, exec, "reader-b", rprRenewedExp) + }() + close(start) + wg.Wait() + if errA != nil || errB != nil || !okA || !okB { + t.Fatalf("round %d: concurrent distinct-owner acquires must both succeed: %v/%v %v/%v", round, okA, errA, okB, errB) + } + rows := s.pinsRows(t, s.w, exec) + if len(rows) != 2 || + !strings.HasPrefix(rows[0], "reader-a|"+rprFarFuture.Format("2006-01-02T15:04:05.000000Z")+"|") || + !strings.HasPrefix(rows[1], "reader-b|"+rprRenewedExp.Format("2006-01-02T15:04:05.000000Z")+"|") { + t.Fatalf("round %d: converged pins = %v, want both literal rows", round, rows) + } + s.clearStateDirect(t, exec) + } +} + +func TestReadPinsCrashWindows(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + defer setReadPinGate(nil) + + ops := []struct { + name string + seed func(t *testing.T, exec string) + call func(t *testing.T, exec string) + post func(t *testing.T, exec string) + }{ + { + name: "acquire-pin", + seed: func(t *testing.T, exec string) {}, + call: func(t *testing.T, exec string) { + must(s.pinsA.AcquireExecutionReadPin(ctx, exec, "reader-a", rprFarFuture))(t) + }, + post: func(t *testing.T, exec string) { + rows := s.pinsRows(t, s.w, exec) + if len(rows) != 1 || !strings.HasPrefix(rows[0], "reader-a|") { + t.Fatalf("retry converge pins = %v", rows) + } + }, + }, + { + name: "renew-pin", + seed: func(t *testing.T, exec string) { s.seedPinDirect(t, exec, "reader-a", rprFarFuture) }, + call: func(t *testing.T, exec string) { + must(s.pinsA.RenewExecutionReadPin(ctx, exec, "reader-a", rprRenewedExp))(t) + }, + post: func(t *testing.T, exec string) { + rows := s.pinsRows(t, s.w, exec) + if len(rows) != 1 || !strings.HasPrefix(rows[0], "reader-a|"+rprRenewedExp.Format("2006-01-02T15:04:05.000000Z")+"|") { + t.Fatalf("retry converge pins = %v", rows) + } + }, + }, + { + name: "release-pin", + seed: func(t *testing.T, exec string) { s.seedPinDirect(t, exec, "reader-a", rprFarFuture) }, + call: func(t *testing.T, exec string) { + if err := s.pinsA.ReleaseExecutionReadPin(ctx, exec, "reader-a"); err != nil { + t.Fatalf("release-pin: %v", err) + } + }, + post: func(t *testing.T, exec string) { + if rows := s.pinsRows(t, s.w, exec); len(rows) != 0 { + t.Fatalf("retry converge pins = %v", rows) + } + }, + }, + { + name: "claim-cleanup", + seed: func(t *testing.T, exec string) { s.seedPinDirect(t, exec, "expired-owner", rprPast) }, + call: func(t *testing.T, exec string) { + must(s.pinsA.ClaimExecutionCleanup(ctx, exec, "sweeper-1"))(t) + }, + post: func(t *testing.T, exec string) { + if rows := s.pinsRows(t, s.w, exec); len(rows) != 0 { + t.Fatalf("retry converge pins = %v, want GC applied", rows) + } + if claim := s.claimRow(t, s.w, exec); !strings.HasPrefix(claim, "sweeper-1|") { + t.Fatalf("retry converge claim = %q", claim) + } + }, + }, + { + name: "renew-cleanup", + seed: func(t *testing.T, exec string) { s.seedClaimDirect(t, exec, "sweeper-1", rprFarFuture) }, + call: func(t *testing.T, exec string) { + must(s.pinsA.RenewExecutionCleanup(ctx, exec, "sweeper-1", rprRenewedExp))(t) + }, + post: func(t *testing.T, exec string) { + if claim := s.claimRow(t, s.w, exec); !strings.HasPrefix(claim, "sweeper-1|"+rprRenewedExp.Format("2006-01-02T15:04:05.000000Z")+"|") { + t.Fatalf("retry converge claim = %q", claim) + } + }, + }, + { + name: "release-cleanup", + seed: func(t *testing.T, exec string) { s.seedClaimDirect(t, exec, "sweeper-1", rprFarFuture) }, + call: func(t *testing.T, exec string) { + if err := s.pinsA.ReleaseExecutionCleanup(ctx, exec, "sweeper-1"); err != nil { + t.Fatalf("release-cleanup: %v", err) + } + }, + post: func(t *testing.T, exec string) { + if claim := s.claimRow(t, s.w, exec); claim != "" { + t.Fatalf("retry converge claim = %q", claim) + } + }, + }, + } + + for _, o := range ops { + exec := "crash-" + o.name + s.clearStateDirect(t, exec) + o.seed(t, exec) + before := s.stateDigest(t, s.w) + gateErr := errors.New("injected crash after all statements applied") + setReadPinGate(func(readPinAttempt) error { return gateErr }) + _, callErr := runOp(s, ctx, o.name, exec) + setReadPinGate(nil) + if !errors.Is(callErr, gateErr) { + t.Fatalf("%s: injected crash error = %v, want the injected error to surface", o.name, callErr) + } + if got := s.stateDigest(t, s.w); got != before { + t.Fatalf("%s: injected crash left durable state: digest %q != %q", o.name, got, before) + } + o.call(t, exec) + o.post(t, exec) + s.clearStateDirect(t, exec) + } + + for _, o := range ops[:4] { + exec := "precommit-" + o.name + s.clearStateDirect(t, exec) + o.seed(t, exec) + witnessPID := s.backendPID(t, s.w) + type gateState struct { + attempt readPinAttempt + visible []string + claim string + state string + pid uint32 + } + gate := make(chan gateState, 1) + setReadPinGate(func(a readPinAttempt) error { + g := gateState{attempt: a} + g.visible = s.pinsRows(t, s.w, exec) + g.claim = s.claimRow(t, s.w, exec) + rows := s.queryRows(t, s.w, + "SELECT state FROM pg_stat_activity WHERE pid = $1", fmt.Sprint(a.BackendPID)) + if len(rows) != 1 || rows[0] != "idle in transaction" { + g.state = fmt.Sprintf("pg_stat_activity state rows %v", rows) + } else { + g.state = "idle in transaction" + } + g.pid = a.BackendPID + gate <- g + return nil + }) + switch o.name { + case "acquire-pin": + must(s.pinsA.AcquireExecutionReadPin(ctx, exec, "reader-a", rprFarFuture))(t) + case "renew-pin": + must(s.pinsA.RenewExecutionReadPin(ctx, exec, "reader-a", rprRenewedExp))(t) + case "release-pin": + if err := s.pinsA.ReleaseExecutionReadPin(ctx, exec, "reader-a"); err != nil { + s.fail("release-pin precommit", err) + } + case "claim-cleanup": + must(s.pinsA.ClaimExecutionCleanup(ctx, exec, "sweeper-1"))(t) + } + setReadPinGate(nil) + g := <-gate + if g.attempt.BackendPID == 0 || g.attempt.BackendPID == witnessPID { + t.Fatalf("%s: writer backend %d not distinct from witness backend %d", o.name, g.attempt.BackendPID, witnessPID) + } + switch o.name { + case "acquire-pin", "renew-pin": + if g.state != "idle in transaction" { + t.Fatalf("%s: %s", o.name, g.state) + } + if o.name == "acquire-pin" && len(g.visible) != 0 { + t.Fatalf("%s: pre-transition pins visible inside the transaction window: %v", o.name, g.visible) + } + if o.name == "renew-pin" && len(g.visible) == 1 && strings.HasPrefix(g.visible[0], "reader-a|"+rprRenewedExp.Format("2006-01-02T15:04:05.000000Z")) { + t.Fatalf("%s: renewed expiry leaked before commit: %v", o.name, g.visible) + } + case "claim-cleanup": + if g.claim != "" { + t.Fatalf("%s: claim row visible before commit: %q", o.name, g.claim) + } + case "release-pin": + if len(g.visible) != 1 { + t.Fatalf("%s: pin vanished before commit: %v", o.name, g.visible) + } + } + switch o.name { + case "acquire-pin": + if rows := s.pinsRows(t, s.w, exec); len(rows) != 1 || !strings.HasPrefix(rows[0], "reader-a|") { + t.Fatalf("%s: post-commit pins = %v", o.name, rows) + } + case "claim-cleanup": + if claim := s.claimRow(t, s.w, exec); !strings.HasPrefix(claim, "sweeper-1|") { + t.Fatalf("%s: post-commit claim = %q", o.name, claim) + } + } + s.clearStateDirect(t, exec) + } +} + +func runOp(s *rprService, ctx context.Context, name, exec string) (bool, error) { + switch name { + case "acquire-pin": + return s.pinsA.AcquireExecutionReadPin(ctx, exec, "reader-a", rprFarFuture) + case "renew-pin": + return s.pinsA.RenewExecutionReadPin(ctx, exec, "reader-a", rprRenewedExp) + case "release-pin": + return true, s.pinsA.ReleaseExecutionReadPin(ctx, exec, "reader-a") + case "claim-cleanup": + return s.pinsA.ClaimExecutionCleanup(ctx, exec, "sweeper-1") + case "renew-cleanup": + return s.pinsA.RenewExecutionCleanup(ctx, exec, "sweeper-1", rprRenewedExp) + case "release-cleanup": + return true, s.pinsA.ReleaseExecutionCleanup(ctx, exec, "sweeper-1") + } + return false, fmt.Errorf("unknown op %q", name) +} diff --git a/internal/backends/postgres/internal/publication/reconcile.go b/internal/backends/postgres/internal/publication/reconcile.go new file mode 100644 index 000000000..1d597585b --- /dev/null +++ b/internal/backends/postgres/internal/publication/reconcile.go @@ -0,0 +1,517 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "reflect" + "sync/atomic" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + "github.com/calypr/loom/internal/dataframe/publication" + "github.com/calypr/loom/internal/dataset" +) + +type PhysicalDropTable func(ctx context.Context, physicalTable string) error + +const ( + defaultReconcilePageSize = 32 + defaultReconcileLeaseTTL = 2 * time.Minute + defaultCleanupHeartbeatTTL = 2 * time.Minute + defaultCleanupRenewal = 30 * time.Second + defaultReconcileCallTimeout = 10 * time.Second +) + +type ReconcileOutcome string + +const ( + ReconcileReconciled ReconcileOutcome = "reconciled" + ReconcileReconciledSuperseded ReconcileOutcome = "reconciled-superseded" + ReconcileSkippedLeased ReconcileOutcome = "skipped-leased" + ReconcileSkippedPointer ReconcileOutcome = "skipped-pointer" + ReconcileSkippedBound ReconcileOutcome = "skipped-bound" + ReconcileRefusedPinActive ReconcileOutcome = "refused-pin-active" + ReconcileRefusedCleanupOwned ReconcileOutcome = "refused-cleanup-owned" + + reconcileClaimGranted ReconcileOutcome = "claim-granted" +) + +type ReconcileDecision struct { + ExecutionID string + Outcome ReconcileOutcome + Err error +} + +type reconcileClaimAttempt struct { + ExecutionID string + ReconcilerID string + BackendPID uint32 + Statements []string + Outcome ReconcileOutcome +} + +type reconcileFinalizeAttempt struct { + ExecutionID string + ReconcilerID string + DroppedTables []string + Remaining []string +} + +var ( + reconcileClaimGate atomic.Pointer[func(reconcileClaimAttempt) error] + reconcileFinalizeGate atomic.Pointer[func(reconcileFinalizeAttempt) error] + + reconcileBeforeClaim atomic.Pointer[func(executionID string)] +) + +type Reconciler struct { + registry *Registry + pins *ReadPins + releases dataset.ActiveReleaseResolver + + reconcilerID string + identityFixed bool + pageSize int + leaseTTL time.Duration + heartbeatTTL time.Duration + renewalEvery time.Duration + callTimeout time.Duration + + OnDecision func(ReconcileDecision) +} + +func NewReconciler(registry *Registry, pins *ReadPins, releases dataset.ActiveReleaseResolver) (*Reconciler, error) { + if registry == nil || pins == nil { + return nil, errors.New("publication postgres: reconciler requires a registry and a pins arbiter") + } + registryType, pinsType := reflect.TypeOf(registry.db), reflect.TypeOf(pins.db) + if registryType == nil || pinsType == nil { + return nil, errors.New("publication postgres: reconciler registry and pins require a database handle") + } + if registryType != pinsType { + return nil, errors.New("publication postgres: reconciler registry and pins bind different database handles") + } + if !registryType.Comparable() { + return nil, fmt.Errorf("publication postgres: reconciler database handle type %s is not comparable; bind a comparable handle (for example *pgxpool.Pool)", registryType) + } + if registry.db != pins.db { + return nil, errors.New("publication postgres: reconciler registry and pins must bind the same database") + } + return &Reconciler{ + registry: registry, + pins: pins, + releases: releases, + pageSize: defaultReconcilePageSize, + leaseTTL: defaultReconcileLeaseTTL, + heartbeatTTL: defaultCleanupHeartbeatTTL, + renewalEvery: defaultCleanupRenewal, + callTimeout: defaultReconcileCallTimeout, + }, nil +} + +func (rc *Reconciler) fixIdentity(identity string) { + rc.reconcilerID = identity + rc.identityFixed = true +} + +func (rc *Reconciler) SweepStale(ctx context.Context, olderThan time.Time, drop PhysicalDropTable) error { + if drop == nil { + return errors.New("publication postgres: reconciler requires a physical drop hook") + } + sweepID := rc.reconcilerID + if !rc.identityFixed { + sweepID = "reconciler-" + uuid.NewString() + } + states := []publication.BundleState{ + publication.BundleRunning, + publication.BundleValidating, + publication.BundleFailed, + } + if rc.releases != nil { + states = append(states, publication.BundlePublished) + } + for _, state := range states { + process := func(executions []publication.BundleExecution) error { + for _, execution := range executions { + if err := rc.reconcileOne(ctx, execution, sweepID, drop); err != nil { + return err + } + } + return nil + } + if err := rc.registry.VisitExecutionPages(ctx, state, olderThan, rc.pageSize, publication.BundleExecutionPageFunc(process)); err != nil { + return err + } + } + return nil +} + +func (rc *Reconciler) decide(execution publication.BundleExecution, outcome ReconcileOutcome, err error) { + if rc.OnDecision != nil { + rc.OnDecision(ReconcileDecision{ExecutionID: execution.ID, Outcome: outcome, Err: err}) + } +} + +func (rc *Reconciler) reconcileOne(ctx context.Context, execution publication.BundleExecution, sweepID string, drop PhysicalDropTable) error { + superseded := execution.State.Canonical() == publication.BundlePublished + if superseded { + pointer, pointerErr := rc.registry.GetPointer(ctx, execution.PointerName()) + if pointerErr == nil { + if pointer.ExecutionID == execution.ID { + rc.decide(execution, ReconcileSkippedPointer, nil) + return nil + } + } else if !errors.Is(pointerErr, publication.ErrBundleNotFound) { + rc.decide(execution, "", pointerErr) + return pointerErr + } + active, activeErr := rc.releases.ReadActiveRelease(ctx, execution.Project) + if activeErr == nil { + if releaseBindsExecution(active, execution.ID) { + rc.decide(execution, ReconcileSkippedBound, nil) + return nil + } + } else if !errors.Is(activeErr, dataset.ErrNoActiveRelease) { + rc.decide(execution, "", activeErr) + return activeErr + } + } + + claimed, err := rc.registry.AcquireBundleLease(ctx, execution.Key, sweepID, time.Now().UTC().Add(rc.leaseTTL)) + if err != nil { + rc.decide(execution, "", err) + return err + } + if !claimed { + rc.decide(execution, ReconcileSkippedLeased, nil) + return nil + } + releaseLease := func() error { + releaseCtx, cancelRelease := context.WithTimeout(context.WithoutCancel(ctx), rc.callTimeout) + defer cancelRelease() + return rc.registry.ReleaseBundleLease(releaseCtx, execution.Key, sweepID) + } + + cleanupCtx, cancelCleanup := context.WithTimeout(context.WithoutCancel(ctx), rc.callTimeout) + defer cancelCleanup() + pointer, pointerErr := rc.registry.GetPointer(cleanupCtx, execution.PointerName()) + if pointerErr != nil && !errors.Is(pointerErr, publication.ErrBundleNotFound) { + return errors.Join(pointerErr, releaseLease()) + } + if pointerErr == nil && pointer.ExecutionID == execution.ID { + rc.decide(execution, ReconcileSkippedPointer, nil) + return releaseLease() + } + + if hook := reconcileBeforeClaim.Load(); hook != nil { + (*hook)(execution.ID) + } + + outcome, claimErr := rc.claimCleanup(ctx, execution, sweepID) + if claimErr != nil { + return errors.Join(claimErr, releaseLease()) + } + switch outcome { + case ReconcileRefusedPinActive: + err := fmt.Errorf("publication postgres: reconcile %s: %w", execution.ID, publication.ErrExecutionReadPinActive) + rc.decide(execution, outcome, err) + return errors.Join(err, releaseLease()) + case ReconcileRefusedCleanupOwned: + err := fmt.Errorf("publication postgres: reconcile %s: %w", execution.ID, publication.ErrExecutionReadPinActive) + rc.decide(execution, outcome, err) + return errors.Join(err, releaseLease()) + case ReconcileSkippedPointer: + rc.decide(execution, outcome, nil) + return releaseLease() + } + + releaseClaimAndLease := func() error { + claimReleaseCtx, cancelClaimRelease := context.WithTimeout(context.WithoutCancel(ctx), rc.callTimeout) + defer cancelClaimRelease() + return errors.Join( + rc.pins.ReleaseExecutionCleanup(claimReleaseCtx, execution.ID, sweepID), + releaseLease(), + ) + } + + pointerNow, pointerNowErr := rc.registry.GetPointer(cleanupCtx, execution.PointerName()) + if pointerNowErr != nil { + if !errors.Is(pointerNowErr, publication.ErrBundleNotFound) { + return errors.Join(pointerNowErr, releaseClaimAndLease()) + } + } else if pointerNow.ExecutionID == execution.ID { + rc.decide(execution, ReconcileSkippedPointer, nil) + return releaseClaimAndLease() + } + if rc.releases != nil { + active, activeErr := rc.releases.ReadActiveRelease(cleanupCtx, execution.Project) + if activeErr != nil { + if !errors.Is(activeErr, dataset.ErrNoActiveRelease) { + return errors.Join(activeErr, releaseClaimAndLease()) + } + } else if releaseBindsExecution(active, execution.ID) { + rc.decide(execution, ReconcileSkippedBound, nil) + return releaseClaimAndLease() + } + } + + dropCtx, cancelDrop := context.WithCancel(cleanupCtx) + heartbeat := &cleanupHeartbeat{ + pins: rc.pins, + ctx: dropCtx, + cancel: cancelDrop, + done: make(chan struct{}), + executionID: execution.ID, + owner: sweepID, + ttl: rc.heartbeatTTL, + every: rc.renewalEvery, + } + go heartbeat.run() + + var first error + dropped := make([]string, 0, len(execution.Outputs)) + remaining := make([]publication.BundleOutputRecord, 0, len(execution.Outputs)) + for _, output := range execution.Outputs { + if err := drop(dropCtx, output.PhysicalTable); err != nil { + first = errors.Join(first, err) + remaining = append(remaining, output) + continue + } + dropped = append(dropped, output.PhysicalTable) + } + heartbeat.stop() + first = errors.Join(first, heartbeat.renewalErr()) + + if gate := reconcileFinalizeGate.Load(); gate != nil { + attempt := reconcileFinalizeAttempt{ + ExecutionID: execution.ID, + ReconcilerID: sweepID, + DroppedTables: dropped, + Remaining: tableNames(remaining), + } + if err := (*gate)(attempt); err != nil { + rc.decide(execution, "", err) + return errors.Join(err, first) + } + } + + updated := execution + updated.State = publication.BundleFailed + updated.Error = "stale execution reconciled" + if superseded { + updated.Error = "superseded execution reconciled (no pointer or active-release reference)" + } + updated.FailureCode = string(dataframeerrors.CodePublicationLeaseLost) + updated.FailureRetryable = true + updated.UpdatedAt = time.Now().UTC() + updated.Outputs = remaining + updated.OwnerID = "" + updated.LeaseExpiresAt = nil + terminalCtx, cancelTerminal := context.WithTimeout(context.WithoutCancel(ctx), rc.callTimeout) + defer cancelTerminal() + first = errors.Join(first, rc.registry.SaveExecution(terminalCtx, updated, sweepID)) + first = errors.Join(first, rc.pins.ReleaseExecutionCleanup(terminalCtx, execution.ID, sweepID)) + first = errors.Join(first, releaseLease()) + if first != nil { + rc.decide(execution, "", first) + } else if superseded { + rc.decide(execution, ReconcileReconciledSuperseded, nil) + } else { + rc.decide(execution, ReconcileReconciled, nil) + } + return first +} + +func (rc *Reconciler) claimCleanup(ctx context.Context, execution publication.BundleExecution, sweepID string) (ReconcileOutcome, error) { + now := time.Now().UTC() + tx, err := rc.registry.db.Begin(ctx) + if err != nil { + return "", fmt.Errorf("publication postgres: begin reconcile claim: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + attempt := reconcileClaimAttempt{ExecutionID: execution.ID, ReconcilerID: sweepID} + if conn := tx.Conn(); conn != nil { + attempt.BackendPID = conn.PgConn().PID() + } + var statements []string + record := func(sql string) { + statements = append(statements, sql) + attempt.Statements = statements + } + + lockPointerSQL := "SELECT execution_id FROM " + rc.registry.pointers + " WHERE pointer_key = $1 FOR UPDATE" + record(lockPointerSQL) + var pointerHolder string + pointerErr := tx.QueryRow(ctx, lockPointerSQL, pointerKey(execution.PointerName())).Scan(&pointerHolder) + if pointerErr != nil && !errors.Is(pointerErr, pgx.ErrNoRows) { + return "", fmt.Errorf("publication postgres: reconcile claim lock pointer: %w", pointerErr) + } + + ensureAnchorSQL := "INSERT INTO " + rc.pins.retention + " (execution_id, updated_at) VALUES ($1, $2)" + + " ON CONFLICT (execution_id) DO NOTHING" + record(ensureAnchorSQL) + if _, err := tx.Exec(ctx, ensureAnchorSQL, execution.ID, now); err != nil { + return "", fmt.Errorf("publication postgres: reconcile claim ensure anchor: %w", err) + } + lockAnchorSQL := "SELECT true FROM " + rc.pins.retention + " WHERE execution_id = $1 FOR UPDATE" + record(lockAnchorSQL) + var held bool + if err := tx.QueryRow(ctx, lockAnchorSQL, execution.ID).Scan(&held); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return "", fmt.Errorf("publication postgres: reconcile claim anchor vanished for %s", execution.ID) + } + return "", fmt.Errorf("publication postgres: reconcile claim lock anchor: %w", err) + } + + if pointerErr == nil && pointerHolder == execution.ID { + attempt.Outcome = ReconcileSkippedPointer + return rc.finishClaim(ctx, tx, attempt) + } + + activePinSQL := "SELECT EXISTS (SELECT 1 FROM " + rc.pins.pins + " WHERE execution_id = $1 AND expires_at >= $2)" + record(activePinSQL) + var pinned bool + if err := tx.QueryRow(ctx, activePinSQL, execution.ID, now).Scan(&pinned); err != nil { + return "", fmt.Errorf("publication postgres: reconcile claim pin guard: %w", err) + } + if pinned { + attempt.Outcome = ReconcileRefusedPinActive + return rc.finishClaim(ctx, tx, attempt) + } + + readClaimSQL := "SELECT owner_id, expires_at FROM " + rc.pins.cleanup + " WHERE execution_id = $1" + record(readClaimSQL) + var ( + claimOwner string + claimExpires time.Time + claimFound bool + ) + err = tx.QueryRow(ctx, readClaimSQL, execution.ID).Scan(&claimOwner, &claimExpires) + if err == nil { + claimFound = true + } else if !errors.Is(err, pgx.ErrNoRows) { + return "", fmt.Errorf("publication postgres: reconcile claim read claim: %w", err) + } + if claimFound && !claimExpires.Before(now) && claimOwner != sweepID { + attempt.Outcome = ReconcileRefusedCleanupOwned + return rc.finishClaim(ctx, tx, attempt) + } + + gcSQL := "DELETE FROM " + rc.pins.pins + " WHERE execution_id = $1 AND expires_at < $2" + record(gcSQL) + if _, err := tx.Exec(ctx, gcSQL, execution.ID, now); err != nil { + return "", fmt.Errorf("publication postgres: reconcile claim gc pins: %w", err) + } + grantSQL := "INSERT INTO " + rc.pins.cleanup + " (execution_id, owner_id, expires_at, updated_at)" + + " VALUES ($1, $2, $3, $4)" + + " ON CONFLICT (execution_id) DO UPDATE SET owner_id = $2, expires_at = $3, updated_at = $4" + record(grantSQL) + if _, err := tx.Exec(ctx, grantSQL, execution.ID, sweepID, now.Add(rc.heartbeatTTL), now); err != nil { + return "", fmt.Errorf("publication postgres: reconcile claim grant: %w", err) + } + touchSQL := "UPDATE " + rc.pins.retention + " SET updated_at = $2 WHERE execution_id = $1" + record(touchSQL) + if _, err := tx.Exec(ctx, touchSQL, execution.ID, now); err != nil { + return "", fmt.Errorf("publication postgres: reconcile claim touch anchor: %w", err) + } + attempt.Outcome = reconcileClaimGranted + return rc.finishClaim(ctx, tx, attempt) +} + +func (rc *Reconciler) finishClaim(ctx context.Context, tx pgx.Tx, attempt reconcileClaimAttempt) (ReconcileOutcome, error) { + if gate := reconcileClaimGate.Load(); gate != nil { + if err := (*gate)(attempt); err != nil { + return "", err + } + } + if attempt.Outcome != reconcileClaimGranted { + if err := tx.Rollback(ctx); err != nil { + return "", fmt.Errorf("publication postgres: reconcile claim rollback refused decision: %w", err) + } + return attempt.Outcome, nil + } + if err := tx.Commit(ctx); err != nil { + return "", fmt.Errorf("publication postgres: reconcile claim commit: %w", err) + } + return attempt.Outcome, nil +} + +func releaseBindsExecution(active dataset.ActiveRelease, executionID string) bool { + for _, binding := range active.Release.Publications { + if binding.ExecutionID == executionID { + return true + } + } + return false +} + +func tableNames(records []publication.BundleOutputRecord) []string { + out := make([]string, 0, len(records)) + for _, record := range records { + out = append(out, record.PhysicalTable) + } + return out +} + +type cleanupHeartbeat struct { + pins *ReadPins + ctx context.Context + cancel context.CancelFunc + done chan struct{} + executionID string + owner string + ttl time.Duration + every time.Duration + + lost atomic.Bool + err atomic.Pointer[error] +} + +func (h *cleanupHeartbeat) run() { + defer close(h.done) + if h.every <= 0 { + return + } + ticker := time.NewTicker(h.every) + defer ticker.Stop() + for { + select { + case <-h.ctx.Done(): + return + case <-ticker.C: + renewCtx, cancel := context.WithTimeout(context.WithoutCancel(h.ctx), defaultReconcileCallTimeout) + owned, err := h.pins.RenewExecutionCleanup(renewCtx, h.executionID, h.owner, time.Now().UTC().Add(h.ttl)) + cancel() + if err != nil || !owned { + if err == nil { + err = publication.ErrExecutionReadPinLost + } + h.err.Store(&err) + h.lost.Store(true) + h.cancel() + return + } + } + } +} + +func (h *cleanupHeartbeat) stop() { + if h.every <= 0 { + <-h.done + return + } + h.cancel() + <-h.done +} + +func (h *cleanupHeartbeat) renewalErr() error { + if p := h.err.Load(); p != nil { + return errors.Join(publication.ErrExecutionReadPinLost, *p) + } + return nil +} diff --git a/internal/backends/postgres/internal/publication/reconcile_test.go b/internal/backends/postgres/internal/publication/reconcile_test.go new file mode 100644 index 000000000..a0441792c --- /dev/null +++ b/internal/backends/postgres/internal/publication/reconcile_test.go @@ -0,0 +1,1450 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "github.com/calypr/loom/internal/dataframe/publication" + "github.com/calypr/loom/internal/dataset" +) + +var ( + rprStaleStamp = time.Date(2026, 1, 5, 6, 7, 8, 900000, time.UTC) +) + +func setReconcileGates(claim func(reconcileClaimAttempt) error, finalize func(reconcileFinalizeAttempt) error) { + if claim == nil { + reconcileClaimGate.Store(nil) + } else { + reconcileClaimGate.Store(&claim) + } + if finalize == nil { + reconcileFinalizeGate.Store(nil) + } else { + reconcileFinalizeGate.Store(&finalize) + } +} + +type rprReleases struct { + mu sync.Mutex + bound map[string][]string + err error +} + +func newRPRReleases() *rprReleases { + return &rprReleases{bound: map[string][]string{}} +} + +func (f *rprReleases) bind(project string, executionIDs ...string) { + f.mu.Lock() + defer f.mu.Unlock() + f.bound[project] = executionIDs +} + +func (f *rprReleases) failWith(err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.err = err +} + +func (f *rprReleases) ReadActiveRelease(_ context.Context, project string) (dataset.ActiveRelease, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.err != nil { + return dataset.ActiveRelease{}, f.err + } + ids := f.bound[project] + if len(ids) == 0 { + return dataset.ActiveRelease{}, dataset.ErrNoActiveRelease + } + release := dataset.ProjectRelease{} + for _, id := range ids { + release.Publications = append(release.Publications, dataset.ReleasePublication{ExecutionID: id}) + } + return dataset.ActiveRelease{Release: release}, nil +} + +type rprDrops struct { + mu sync.Mutex + calls []string + perTable map[string]error + waitFor <-chan struct{} + hookErr func(table string, ctx context.Context) error +} + +func newRPRDrops() *rprDrops { + return &rprDrops{perTable: map[string]error{}} +} + +var ( + liveTablesMu sync.Mutex + liveTables = map[string]bool{} +) + +func markLiveTables(tables ...string) { + liveTablesMu.Lock() + defer liveTablesMu.Unlock() + for _, tbl := range tables { + liveTables[tbl] = true + } +} + +func (d *rprDrops) hook(ctx context.Context, table string) error { + if d.waitFor != nil { + select { + case <-ctx.Done(): + return ctx.Err() + case <-d.waitFor: + } + } + liveTablesMu.Lock() + existed := liveTables[table] + if existed { + delete(liveTables, table) + } + liveTablesMu.Unlock() + if !existed { + return nil + } + if d.hookErr != nil { + if err := d.hookErr(table, ctx); err != nil { + liveTablesMu.Lock() + liveTables[table] = true + liveTablesMu.Unlock() + d.record(table, err) + return err + } + } + d.record(table, nil) + return nil +} + +func (d *rprDrops) record(table string, err error) { + d.mu.Lock() + defer d.mu.Unlock() + if err == nil { + d.calls = append(d.calls, table) + } else { + d.calls = append(d.calls, table+"!error") + } +} + +func (d *rprDrops) snapshot() []string { + d.mu.Lock() + defer d.mu.Unlock() + return append([]string(nil), d.calls...) +} + +func rprExecution(id, project, name string, state publication.BundleState, outputs int) publication.BundleExecution { + execution := publication.BundleExecution{ + ID: id, + Key: "key-" + id, + BundleIdentity: publication.BundleIdentity{ + Name: name, Project: project, DatasetGeneration: "gen-1", + RecipeDigest: "r-" + id, SchemaDigest: "s-" + id, ScopeDigest: "sc-" + id, EngineVersion: "v1", + }, + State: state, + CreatedAt: rprStaleStamp, + UpdatedAt: rprStaleStamp, + } + for i := 0; i < outputs; i++ { + execution.Outputs = append(execution.Outputs, publication.BundleOutputRecord{ + Name: fmt.Sprintf("out-%d", i), PhysicalTable: fmt.Sprintf("tbl-%s-%d", id, i), State: state, + }) + } + return execution +} + +func (s *rprService) seedStaleExecution(t *testing.T, execution publication.BundleExecution) { + t.Helper() + tables := make([]string, 0, len(execution.Outputs)) + for _, output := range execution.Outputs { + tables = append(tables, output.PhysicalTable) + } + markLiveTables(tables...) + granted, err := s.regW.AcquireBundleLease(s.ctx, execution.Key, "seeder", time.Now().UTC().Add(10*time.Minute)) + if err != nil || !granted { + t.Fatalf("seed lease %s: %v %v", execution.Key, granted, err) + } + if err := s.regW.SaveExecution(s.ctx, execution, "seeder"); err != nil { + t.Fatalf("seed save %s: %v", execution.ID, err) + } + if err := s.regW.ReleaseBundleLease(s.ctx, execution.Key, "seeder"); err != nil { + t.Fatalf("seed release %s: %v", execution.Key, err) + } +} + +func (s *rprService) newReconciler(t *testing.T, releases dataset.ActiveReleaseResolver) *Reconciler { + t.Helper() + rc, err := NewReconciler(s.regA, s.pinsA, releases) + if err != nil { + t.Fatalf("new reconciler: %v", err) + } + rc.fixIdentity("reconciler-test") + rc.heartbeatTTL = 2 * time.Minute + rc.renewalEvery = 0 + return rc +} + +func (s *rprService) leaseRow(t *testing.T, key string) string { + t.Helper() + return strings.Join(s.queryRows(t, s.w, + "SELECT owner_id, "+fmt.Sprintf(tsText, "expires_at")+ + " FROM "+pgx.Identifier{s.schema, BundleLeasesTable}.Sanitize()+" WHERE key = $1", key), "|") +} + +func (s *rprService) executionRow(t *testing.T, id string) (state, payload string) { + t.Helper() + rows := s.queryRows(t, s.w, + "SELECT state, payload::text FROM "+pgx.Identifier{s.schema, BundleExecutionsTable}.Sanitize()+" WHERE id = $1", id) + if len(rows) != 1 { + t.Fatalf("execution rows for %s = %v, want exactly one", id, rows) + } + parts := strings.SplitN(rows[0], "|", 2) + return parts[0], strings.ReplaceAll(parts[1], `": `, `":`) +} + +func waitUntil(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(2 * time.Millisecond) + } + t.Fatalf("condition never held: %s", what) +} + +func TestReconcileSweepReclaimsAbandoned(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + setReconcileGates(nil, nil) + defer setReconcileGates(nil, nil) + + for i := 0; i < 7; i++ { + state := publication.BundleRunning + if i == 5 { + state = publication.BundleValidating + } + if i == 6 { + state = publication.BundleFailed + } + s.seedStaleExecution(t, rprExecution(fmt.Sprintf("exec-sweep-%02d", i), "proj-a", "recipe-sweep", state, 2)) + } + fresh := rprExecution("exec-sweep-fresh", "proj-a", "recipe-sweep", publication.BundleRunning, 1) + fresh.UpdatedAt = time.Now().UTC().Add(time.Hour) + s.seedStaleExecution(t, fresh) + s.seedStaleExecution(t, rprExecution("exec-sweep-queued", "proj-a", "recipe-sweep", publication.BundleQueued, 1)) + + drops := newRPRDrops() + rc := s.newReconciler(t, nil) + rc.pageSize = 3 + var decisions []ReconcileDecision + rc.OnDecision = func(d ReconcileDecision) { decisions = append(decisions, d) } + + if err := rc.SweepStale(ctx, time.Now().UTC(), drops.hook); err != nil { + t.Fatalf("sweep: %v", err) + } + + wantDrops := []string{} + wantDecisions := []ReconcileOutcome{} + for i := 0; i < 7; i++ { + id := fmt.Sprintf("exec-sweep-%02d", i) + wantDrops = append(wantDrops, fmt.Sprintf("tbl-%s-0", id), fmt.Sprintf("tbl-%s-1", id)) + wantDecisions = append(wantDecisions, ReconcileReconciled) + } + if got := drops.snapshot(); strings.Join(got, ",") != strings.Join(wantDrops, ",") { + t.Fatalf("drop calls = %v\nwant exactly %v (keyset order, once per table, none for fresh/queued)", got, wantDrops) + } + if len(decisions) != len(wantDecisions) { + t.Fatalf("decisions = %v, want %d entries", decisions, len(wantDecisions)) + } + for i, want := range wantDecisions { + if decisions[i].Outcome != want { + t.Fatalf("decision[%d] = %v, want %v", i, decisions[i].Outcome, want) + } + } + for i := 0; i < 7; i++ { + id := fmt.Sprintf("exec-sweep-%02d", i) + state, payload := s.executionRow(t, id) + if state != string(publication.BundleFailed) { + t.Fatalf("%s state = %q, want FAILED", id, state) + } + for _, want := range []string{ + `"error":"stale execution reconciled"`, + `"failureCode":"PUBLICATION_LEASE_LOST"`, + `"failureRetryable":true`, + } { + if !strings.Contains(payload, want) { + t.Fatalf("%s payload missing literal %s: %s", id, want, payload) + } + } + if strings.Contains(payload, `"outputs":[{`) { + t.Fatalf("%s payload still lists outputs after full drop: %s", id, payload) + } + if strings.Contains(payload, `"ownerId":"seeder"`) || strings.Contains(payload, `"ownerId":"reconciler-test"`) { + t.Fatalf("%s payload still names an owner: %s", id, payload) + } + if s.leaseRow(t, "key-"+id) != "" { + t.Fatalf("%s bundle lease still present after reconcile", id) + } + if claim := s.claimRow(t, s.w, id); claim != "" { + t.Fatalf("%s cleanup claim still present after reconcile: %q", id, claim) + } + if rows := s.pinsRows(t, s.w, id); len(rows) != 0 { + t.Fatalf("%s pins still present after reconcile: %v", id, rows) + } + } + if state, _ := s.executionRow(t, "exec-sweep-fresh"); state != string(publication.BundleRunning) { + t.Fatalf("fresh execution was swept: state %q", state) + } + if state, _ := s.executionRow(t, "exec-sweep-queued"); state != string(publication.BundleQueued) { + t.Fatalf("queued execution was swept: state %q", state) + } +} + +func TestReconcileDropFailureConvergesIdempotently(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + defer setReconcileGates(nil, nil) + + const id = "exec-dropconv" + tbl0 := "tbl-" + id + "-0" + tbl1 := "tbl-" + id + "-1" + s.seedStaleExecution(t, rprExecution(id, "proj-a", "recipe-dropconv", publication.BundleRunning, 2)) + + failing := newRPRDrops() + failing.hookErr = func(table string, _ context.Context) error { + if table == tbl0 { + return errors.New("drop callback exploded") + } + return nil + } + rc := s.newReconciler(t, nil) + err := rc.SweepStale(ctx, time.Now().UTC(), failing.hook) + if err == nil || !strings.Contains(err.Error(), "drop callback exploded") { + t.Fatalf("sweep 1 error = %v, want the drop callback failure", err) + } + if got := strings.Join(failing.snapshot(), ","); got != tbl0+"!error,"+tbl1 { + t.Fatalf("sweep 1 drop witnesses = %v, want [%s %s] in page order", failing.snapshot(), tbl0+"!error", tbl1) + } + state, payload := s.executionRow(t, id) + if state != string(publication.BundleFailed) { + t.Fatalf("state after failed drop = %q, want FAILED", state) + } + if !strings.Contains(payload, tbl0) || strings.Contains(payload, tbl1) { + t.Fatalf("remaining after failed drop wrong (want %s kept, %s gone): %s", tbl0, tbl1, payload) + } + if claim := s.claimRow(t, s.w, id); claim != "" { + t.Fatalf("failed-drop sweep stranded the cleanup claim: %q", claim) + } + if s.leaseRow(t, "key-"+id) != "" { + t.Fatal("failed-drop sweep stranded the bundle lease") + } + if got := s.pinsRows(t, s.w, id); len(got) != 0 { + t.Fatalf("failed-drop sweep left pin rows: %v", got) + } + + rc2, err := NewReconciler(s.regA, s.pinsA, nil) + if err != nil { + t.Fatalf("new unfixed reconciler: %v", err) + } + converged := newRPRDrops() + if err := rc2.SweepStale(ctx, time.Now().UTC(), converged.hook); err != nil { + t.Fatalf("converging sweep: %v", err) + } + if got := converged.snapshot(); len(got) != 1 || got[0] != tbl0 { + t.Fatalf("converging drop witnesses = %v, want exactly [%s]: the already-dead %s must be a no-op re-drop, not a recounted deletion", got, tbl0, tbl1) + } + state, payload = s.executionRow(t, id) + if state != string(publication.BundleFailed) { + t.Fatalf("state after convergence = %q, want FAILED", state) + } + if strings.Contains(payload, tbl0) || strings.Contains(payload, tbl1) { + t.Fatalf("convergence left outputs in the record: %s", payload) + } + if claim := s.claimRow(t, s.w, id); claim != "" { + t.Fatalf("converging sweep stranded the cleanup claim: %q", claim) + } + if s.leaseRow(t, "key-"+id) != "" { + t.Fatal("converging sweep stranded the bundle lease") + } +} + +func TestReconcileRefusedByLivePin(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + setReconcileGates(nil, nil) + defer setReconcileGates(nil, nil) + + const id = "exec-pin-refuse" + s.seedStaleExecution(t, rprExecution(id, "proj-a", "recipe-pin", publication.BundleRunning, 1)) + must(s.pinsA.AcquireExecutionReadPin(ctx, id, "streaming-reader", rprFarFuture))(t) + before := s.stateDigest(t, s.w) + + drops := newRPRDrops() + rc := s.newReconciler(t, nil) + var decisions []ReconcileDecision + rc.OnDecision = func(d ReconcileDecision) { decisions = append(decisions, d) } + + err := rc.SweepStale(ctx, time.Now().UTC(), drops.hook) + if !errors.Is(err, publication.ErrExecutionReadPinActive) { + t.Fatalf("sweep error = %v, want errors.Is publication.ErrExecutionReadPinActive", err) + } + if len(decisions) != 1 || decisions[0].Outcome != ReconcileRefusedPinActive { + t.Fatalf("decisions = %v, want exactly one refused-pin-active", decisions) + } + if got := drops.snapshot(); len(got) != 0 { + t.Fatalf("drop hook ran despite the live pin: %v", got) + } + if s.leaseRow(t, "key-"+id) != "" { + t.Fatal("bundle lease was not released after the refusal") + } + if claim := s.claimRow(t, s.w, id); claim != "" { + t.Fatalf("cleanup claim was minted despite the pin: %q", claim) + } + if state, payload := s.executionRow(t, id); state != string(publication.BundleRunning) || !strings.Contains(payload, `"updatedAt":"2026-01-05T06:07:08.0009Z"`) { + t.Fatalf("pinned execution changed: state %q payload %s", state, payload) + } + _ = before + + if err := s.pinsA.ReleaseExecutionReadPin(ctx, id, "streaming-reader"); err != nil { + t.Fatalf("reader release: %v", err) + } + rc2 := s.newReconciler(t, nil) + rc2.renewalEvery = 0 + if err := rc2.SweepStale(ctx, time.Now().UTC(), drops.hook); err != nil { + t.Fatalf("convergence sweep: %v", err) + } + if got := drops.snapshot(); len(got) != 1 || got[0] != "tbl-"+id+"-0" { + t.Fatalf("convergence drops = %v, want the single table", got) + } +} + +func TestReconcileForeignClaimRefusesSweep(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + setReconcileGates(nil, nil) + defer setReconcileGates(nil, nil) + + const id = "exec-foreign-claim" + s.seedStaleExecution(t, rprExecution(id, "proj-a", "recipe-claim", publication.BundleRunning, 1)) + s.seedClaimDirect(t, id, "other-sweeper", rprFarFuture) + + drops := newRPRDrops() + rc := s.newReconciler(t, nil) + var decisions []ReconcileDecision + rc.OnDecision = func(d ReconcileDecision) { decisions = append(decisions, d) } + err := rc.SweepStale(ctx, time.Now().UTC(), drops.hook) + if !errors.Is(err, publication.ErrExecutionReadPinActive) { + t.Fatalf("sweep error = %v, want named ErrExecutionReadPinActive", err) + } + if len(decisions) != 1 || decisions[0].Outcome != ReconcileRefusedCleanupOwned { + t.Fatalf("decisions = %v, want refused-cleanup-owned", decisions) + } + if got := drops.snapshot(); len(got) != 0 { + t.Fatalf("drops ran under a foreign claim: %v", got) + } + if claim := s.claimRow(t, s.w, id); !strings.HasPrefix(claim, "other-sweeper|") { + t.Fatalf("foreign claim changed: %q", claim) + } +} + +func TestReconcilePointerSkips(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + setReconcileGates(nil, nil) + defer setReconcileGates(nil, nil) + + const id = "exec-pointer-live" + execution := rprExecution(id, "proj-a", "recipe-pointer", publication.BundleRunning, 1) + s.seedStaleExecution(t, execution) + if err := s.regW.CompareAndSwapPointer(ctx, execution.PointerName(), "", id); err != nil { + t.Fatalf("seed pointer: %v", err) + } + drops := newRPRDrops() + rc := s.newReconciler(t, nil) + var decisions []ReconcileDecision + rc.OnDecision = func(d ReconcileDecision) { decisions = append(decisions, d) } + if err := rc.SweepStale(ctx, time.Now().UTC(), drops.hook); err != nil { + t.Fatalf("sweep: %v", err) + } + if len(decisions) != 1 || decisions[0].Outcome != ReconcileSkippedPointer { + t.Fatalf("decisions = %v, want skipped-pointer", decisions) + } + if got := drops.snapshot(); len(got) != 0 { + t.Fatalf("drops ran for a pointer-bound execution: %v", got) + } + if s.leaseRow(t, "key-"+id) != "" { + t.Fatal("lease not released after pointer skip") + } +} + +func TestReconcileInClaimPointerVeto(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + defer setReconcileGates(nil, nil) + + const id = "exec-inclaim" + execution := rprExecution(id, "proj-a", "recipe-inclaim", publication.BundleRunning, 1) + s.seedStaleExecution(t, execution) + if err := s.regW.CompareAndSwapPointer(ctx, execution.PointerName(), "", "exec-inclaim-other"); err != nil { + t.Fatalf("seed pointer: %v", err) + } + setBeforeClaim(func(executionID string) { + if executionID != id { + return + } + if err := s.regB.CompareAndSwapPointer(ctx, execution.PointerName(), "exec-inclaim-other", id); err != nil { + t.Errorf("pre-claim pointer move: %v", err) + } + }) + + casDone := make(chan error, 1) + casBlocked := make(chan bool, 1) + setReconcileGates(func(attempt reconcileClaimAttempt) error { + if attempt.ExecutionID != id { + return nil + } + if attempt.Outcome == reconcileClaimGranted { + t.Errorf("claim granted for %s despite the pointer naming it: in-transaction veto bypassed", id) + } + return nil + }, nil) + + drops := newRPRDrops() + rc := s.newReconciler(t, nil) + var decisions []ReconcileDecision + rc.OnDecision = func(d ReconcileDecision) { decisions = append(decisions, d) } + if err := rc.SweepStale(ctx, time.Now().UTC(), drops.hook); err != nil { + t.Fatalf("sweep: %v", err) + } + setBeforeClaim(nil) + if len(decisions) != 1 || decisions[0].Outcome != ReconcileSkippedPointer { + t.Fatalf("decisions = %v, want skipped-pointer (in-transaction pointer veto)", decisions) + } + if got := drops.snapshot(); len(got) != 0 { + t.Fatalf("physical drop ran despite the pointer that names the execution: %v", got) + } + if s.leaseRow(t, "key-"+id) != "" { + t.Fatal("lease not released after the pointer veto") + } + if claim := s.claimRow(t, s.w, id); claim != "" { + t.Fatalf("claim minted despite the pointer veto: %q", claim) + } + if state, _ := s.executionRow(t, id); state != string(publication.BundleRunning) { + t.Fatalf("bound execution was mutated: %q", state) + } + _, _ = casDone, casBlocked + + const id2 = "exec-inclaim-2" + execution2 := rprExecution(id2, "proj-a", "recipe-inclaim-2", publication.BundleRunning, 1) + s.seedStaleExecution(t, execution2) + if err := s.regW.CompareAndSwapPointer(ctx, execution2.PointerName(), "", "exec-inclaim-other"); err != nil { + t.Fatalf("seed pointer 2: %v", err) + } + setReconcileGates(func(attempt reconcileClaimAttempt) error { + if attempt.Outcome != reconcileClaimGranted || attempt.ExecutionID != id2 { + return nil + } + go func() { + casDone <- s.regB.CompareAndSwapPointer(ctx, execution2.PointerName(), "exec-inclaim-other", id2) + }() + select { + case err := <-casDone: + casBlocked <- false + t.Errorf("pointer CAS completed while the claim transaction held its lock: %v", err) + case <-time.After(50 * time.Millisecond): + casBlocked <- true + } + return nil + }, nil) + rc2 := s.newReconciler(t, nil) + if err := rc2.SweepStale(ctx, time.Now().UTC(), newRPRDrops().hook); err != nil { + t.Fatalf("sweep 2: %v", err) + } + if !<-casBlocked { + t.Fatal("the claim transaction did not hold a row lock a concurrent publisher queued behind") + } + if err := <-casDone; err != nil { + t.Fatalf("concurrent pointer CAS: %v", err) + } +} + +func setBeforeClaim(fn func(executionID string)) { + if fn == nil { + reconcileBeforeClaim.Store(nil) + return + } + reconcileBeforeClaim.Store(&fn) +} + +func TestReconcileJanitorArmAndBindings(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + setReconcileGates(nil, nil) + defer setReconcileGates(nil, nil) + + const orphan = "exec-janitor-off" + s.seedStaleExecution(t, rprExecution(orphan, "proj-off", "recipe-jan", publication.BundlePublished, 1)) + drops := newRPRDrops() + rc := s.newReconciler(t, nil) + if err := rc.SweepStale(ctx, time.Now().UTC(), drops.hook); err != nil { + t.Fatalf("sweep without releases: %v", err) + } + if got := drops.snapshot(); len(got) != 0 { + t.Fatalf("PUBLISHED arm ran without an active-release reader: %v", got) + } + if state, _ := s.executionRow(t, orphan); state != string(publication.BundlePublished) { + t.Fatalf("orphaned PUBLISHED execution was swept with releases nil: %q", state) + } + + releases := newRPRReleases() + const bound = "exec-janitor-bound" + const free = "exec-janitor-free" + s.seedStaleExecution(t, rprExecution(bound, "proj-a", "recipe-jan", publication.BundlePublished, 1)) + s.seedStaleExecution(t, rprExecution(free, "proj-a", "recipe-jan", publication.BundlePublished, 1)) + releases.bind("proj-a", bound) + + drops2 := newRPRDrops() + rc2 := s.newReconciler(t, releases) + var decisions []ReconcileDecision + rc2.OnDecision = func(d ReconcileDecision) { decisions = append(decisions, d) } + if err := rc2.SweepStale(ctx, time.Now().UTC(), drops2.hook); err != nil { + t.Fatalf("sweep with releases: %v", err) + } + wantOutcomes := []ReconcileOutcome{ReconcileSkippedBound, ReconcileReconciledSuperseded, ReconcileReconciledSuperseded} + if len(decisions) != len(wantOutcomes) { + t.Fatalf("decisions = %v, want %d", decisions, len(wantOutcomes)) + } + for i, want := range wantOutcomes { + if decisions[i].Outcome != want { + t.Fatalf("decision[%d] = %v, want %v", i, decisions[i].Outcome, want) + } + } + if decisions[0].ExecutionID != bound || decisions[1].ExecutionID != free || decisions[2].ExecutionID != orphan { + t.Fatalf("decision order = %v, want [bound free orphan]", decisions) + } + if got := drops2.snapshot(); len(got) != 2 || got[0] != "tbl-"+free+"-0" || got[1] != "tbl-"+orphan+"-0" { + t.Fatalf("drops = %v, want the unbound executions' tables", got) + } + if state, payload := s.executionRow(t, bound); state != string(publication.BundlePublished) { + t.Fatalf("active-release-bound execution swept: %q %s", state, payload) + } + if _, payload := s.executionRow(t, free); !strings.Contains(payload, `"error":"superseded execution reconciled (no pointer or active-release reference)"`) { + t.Fatalf("superseded terminal error missing literal: %s", payload) + } + + releases.failWith(errors.New("release store unavailable")) + const failExec = "exec-janitor-fail" + s.seedStaleExecution(t, rprExecution(failExec, "proj-a", "recipe-jan", publication.BundlePublished, 1)) + rc3 := s.newReconciler(t, releases) + err := rc3.SweepStale(ctx, time.Now().UTC(), newRPRDrops().hook) + if err == nil || !strings.Contains(err.Error(), "release store unavailable") { + t.Fatalf("release-reader failure did not veto fail-closed: %v", err) + } + if state, _ := s.executionRow(t, failExec); state != string(publication.BundlePublished) { + t.Fatalf("execution swept despite release-reader failure: %q", state) + } +} + +func TestReconcileSkipsForeignLease(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + setReconcileGates(nil, nil) + defer setReconcileGates(nil, nil) + + const id = "exec-leased" + execution := rprExecution(id, "proj-a", "recipe-lease", publication.BundleRunning, 1) + s.seedStaleExecution(t, execution) + granted, err := s.regW.AcquireBundleLease(ctx, execution.Key, "worker-1", time.Now().UTC().Add(10*time.Minute)) + if err != nil || !granted { + t.Fatalf("seed worker lease: %v %v", granted, err) + } + drops := newRPRDrops() + rc := s.newReconciler(t, nil) + var decisions []ReconcileDecision + rc.OnDecision = func(d ReconcileDecision) { decisions = append(decisions, d) } + if err := rc.SweepStale(ctx, time.Now().UTC(), drops.hook); err != nil { + t.Fatalf("sweep: %v", err) + } + if len(decisions) != 1 || decisions[0].Outcome != ReconcileSkippedLeased { + t.Fatalf("decisions = %v, want skipped-leased", decisions) + } + if got := drops.snapshot(); len(got) != 0 { + t.Fatalf("sweep deleted tables under a foreign live lease: %v", got) + } + if s.leaseRow(t, execution.Key) == "" { + t.Fatal("foreign lease was removed by the sweep") + } +} + +func TestReconcilePublisherRace(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + setReconcileGates(nil, nil) + defer setReconcileGates(nil, nil) + + if pidA, pidB := s.backendPID(t, s.a), s.backendPID(t, s.b); pidA == pidB { + t.Fatalf("racing connections share backend %d", pidA) + } + const rounds = 12 + for round := 0; round < rounds; round++ { + id := fmt.Sprintf("exec-race-pub-%02d", round) + execution := rprExecution(id, "proj-a", fmt.Sprintf("recipe-race-%02d", round), publication.BundleRunning, 1) + s.seedStaleExecution(t, execution) + + start := make(chan struct{}) + var sweepErr, pubErr error + var wg sync.WaitGroup + wg.Add(2) + drops := newRPRDrops() + rc := s.newReconciler(t, nil) + rc.fixIdentity(fmt.Sprintf("reconciler-race-%02d", round)) + var decisionMu sync.Mutex + var decisions []ReconcileDecision + rc.OnDecision = func(d ReconcileDecision) { + decisionMu.Lock() + defer decisionMu.Unlock() + decisions = append(decisions, d) + } + go func() { + defer wg.Done() + <-start + sweepErr = rc.SweepStale(ctx, time.Now().UTC(), drops.hook) + }() + go func() { + defer wg.Done() + <-start + pub := s.regB + granted, err := pub.AcquireBundleLease(ctx, execution.Key, "publisher", time.Now().UTC().Add(10*time.Minute)) + if err != nil { + pubErr = err + return + } + if !granted { + pubErr = fmt.Errorf("publisher could not take the lease") + return + } + published := execution + published.State = publication.BundlePublished + published.OwnerID = "publisher" + now := time.Now().UTC() + published.UpdatedAt = now + published.PublishedAt = &now + pubErr = pub.PublishExecution(ctx, execution.PointerName(), "", published) + }() + close(start) + wg.Wait() + + _, payload := s.executionRow(t, id) + pointer, pointerErr := s.regW.GetPointer(ctx, execution.PointerName()) + published := pointerErr == nil && pointer.ExecutionID == id && strings.Contains(payload, `"state":"PUBLISHED"`) + decisionMu.Lock() + reconciled := false + for _, d := range decisions { + if d.Outcome == ReconcileReconciled && d.ExecutionID == id { + reconciled = true + } + } + decisionMu.Unlock() + if published && reconciled { + t.Fatalf("round %d: BOTH publisher and sweeper completed (late-publisher over a reconciled row) -- lease discipline violated (sweepErr=%v pubErr=%v payload=%s)", + round, sweepErr, pubErr, payload) + } + if !published && !reconciled { + t.Fatalf("round %d: neither publisher nor sweeper completed (sweepErr=%v pubErr=%v payload=%s)", round, sweepErr, pubErr, payload) + } + if reconciled { + if pubErr == nil { + t.Fatalf("round %d: publisher reported success while the sweeper won: %v", round, pubErr) + } + if got := drops.snapshot(); len(got) != 1 || got[0] != "tbl-"+id+"-0" { + t.Fatalf("round %d: sweep drops = %v", round, got) + } + if pointerErr != nil && !errors.Is(pointerErr, publication.ErrBundleNotFound) { + t.Fatalf("round %d: pointer read: %v", round, pointerErr) + } + } else { + if !published { + t.Fatalf("round %d: neither outcome: %v / %v", round, sweepErr, pubErr) + } + if got := drops.snapshot(); len(got) != 0 { + t.Fatalf("round %d: sweep dropped tables under the publisher's lease: %v", round, got) + } + } + } +} + +func TestReconcileCrashWindows(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + defer setReconcileGates(nil, nil) + + { + const id = "exec-crash-claim" + s.seedStaleExecution(t, rprExecution(id, "proj-a", "recipe-crash", publication.BundleRunning, 1)) + injected := errors.New("injected crash inside the claim transaction") + setReconcileGates(func(reconcileClaimAttempt) error { return injected }, nil) + drops := newRPRDrops() + rc := s.newReconciler(t, nil) + err := rc.SweepStale(ctx, time.Now().UTC(), drops.hook) + setReconcileGates(nil, nil) + if !errors.Is(err, injected) { + t.Fatalf("claim-seam crash error = %v, want the injected error", err) + } + if got := drops.snapshot(); len(got) != 0 { + t.Fatalf("drops ran despite the claim crash: %v", got) + } + if claim := s.claimRow(t, s.w, id); claim != "" { + t.Fatalf("claim survived the total rollback: %q", claim) + } + if anchor := s.anchorRow(t, s.w, id); anchor != "" { + t.Fatalf("anchor survived the total rollback: %q", anchor) + } + if rows := s.pinsRows(t, s.w, id); len(rows) != 0 { + t.Fatalf("pins present after claim rollback: %v", rows) + } + if s.leaseRow(t, "key-"+id) != "" { + t.Fatal("baseline parity: a claim failure releases the just-taken bundle lease") + } + if state, _ := s.executionRow(t, id); state != string(publication.BundleRunning) { + t.Fatalf("execution mutated by the crashed sweep: %q", state) + } + drops2 := newRPRDrops() + rc2 := s.newReconciler(t, nil) + if err := rc2.SweepStale(ctx, time.Now().UTC(), drops2.hook); err != nil { + t.Fatalf("post-abort convergence sweep: %v", err) + } + if got := drops2.snapshot(); len(got) != 1 || got[0] != "tbl-"+id+"-0" { + t.Fatalf("post-abort drops = %v", got) + } + if state, _ := s.executionRow(t, id); state != string(publication.BundleFailed) { + t.Fatalf("post-abort state = %q, want FAILED", state) + } + } + + { + const id = "exec-crash-finalize" + s.seedStaleExecution(t, rprExecution(id, "proj-a", "recipe-crash2", publication.BundleRunning, 1)) + injected := errors.New("injected crash between drop and terminal write") + var droppedWhileCrashed []string + setReconcileGates(nil, func(a reconcileFinalizeAttempt) error { + droppedWhileCrashed = append([]string(nil), a.DroppedTables...) + return injected + }) + drops := newRPRDrops() + rc := s.newReconciler(t, nil) + err := rc.SweepStale(ctx, time.Now().UTC(), drops.hook) + setReconcileGates(nil, nil) + if !errors.Is(err, injected) { + t.Fatalf("finalize-seam crash error = %v, want the injected error", err) + } + if got := drops.snapshot(); len(got) != 1 || got[0] != "tbl-"+id+"-0" { + t.Fatalf("finalize-seam drops = %v, want the applied drop", got) + } + if strings.Join(droppedWhileCrashed, ",") != "tbl-"+id+"-0" { + t.Fatalf("finalize witness drops = %v", droppedWhileCrashed) + } + if claim := s.claimRow(t, s.w, id); !strings.HasPrefix(claim, "reconciler-test|") { + t.Fatalf("abandoned claim must stay active: %q", claim) + } + if state, _ := s.executionRow(t, id); state != string(publication.BundleRunning) { + t.Fatalf("execution must keep its pre-sweep record: %q", state) + } + + drops2 := newRPRDrops() + rc2 := s.newReconciler(t, nil) + if err := rc2.SweepStale(ctx, time.Now().UTC(), drops2.hook); err != nil { + t.Fatalf("fixed-identity refresh convergence: %v", err) + } + if state, _ := s.executionRow(t, id); state != string(publication.BundleFailed) { + t.Fatalf("fixed-identity refresh convergence state = %q", state) + } + + const id3 = "exec-crash-finalize-a2" + s.seedStaleExecution(t, rprExecution(id3, "proj-a", "recipe-crash-a2", publication.BundleRunning, 1)) + setReconcileGates(nil, func(reconcileFinalizeAttempt) error { return injected }) + rcA2, err := NewReconciler(s.regA, s.pinsA, nil) + if err != nil { + t.Fatalf("new unfixed reconciler: %v", err) + } + if err := rcA2.SweepStale(ctx, time.Now().UTC(), newRPRDrops().hook); !errors.Is(err, injected) { + t.Fatalf("crash A2: %v", err) + } + setReconcileGates(nil, nil) + if claim := s.claimRow(t, s.w, id3); claim == "" || strings.HasPrefix(claim, "reconciler-test|") { + t.Fatalf("crashed A2 claim must carry a minted identity: %q", claim) + } + var a2Decisions []ReconcileDecision + rcA2.OnDecision = func(d ReconcileDecision) { a2Decisions = append(a2Decisions, d) } + if err := rcA2.SweepStale(ctx, time.Now().UTC(), newRPRDrops().hook); err != nil { + t.Fatalf("same-instance sweep after same-instance crash: %v", err) + } + sawLeased := false + for _, d := range a2Decisions { + if d.ExecutionID == id3 && d.Outcome == ReconcileSkippedLeased { + sawLeased = true + } + } + if !sawLeased { + t.Fatalf("the next sweep of the same instance must be skipped-leased by the abandoned lease: %v", a2Decisions) + } + if state, _ := s.executionRow(t, id3); state != string(publication.BundleRunning) { + t.Fatalf("abandoned record mutated by a blocked sweep: %q", state) + } + s.queryRows(t, s.w, + "UPDATE "+pgx.Identifier{s.schema, BundleLeasesTable}.Sanitize()+ + " SET expires_at = $1 WHERE key = $2", rprPast, "key-"+id3) + s.queryRows(t, s.w, + "UPDATE "+pgx.Identifier{s.schema, ExecutionCleanupClaimsTable}.Sanitize()+ + " SET expires_at = $1 WHERE execution_id = $2", rprPast, id3) + dropsA2 := newRPRDrops() + if err := rcA2.SweepStale(ctx, time.Now().UTC(), dropsA2.hook); err != nil { + t.Fatalf("same-instance post-expiry convergence: %v", err) + } + if state, payload := s.executionRow(t, id3); state != string(publication.BundleFailed) || + !strings.Contains(payload, `"error":"stale execution reconciled"`) { + t.Fatalf("same-instance post-expiry convergence record = %q %s", state, payload) + } + if claim := s.claimRow(t, s.w, id3); claim != "" { + t.Fatalf("convergence did not consume the claim: %q", claim) + } + if got := dropsA2.snapshot(); len(got) != 0 { + t.Fatalf("post-expiry convergence drops = %v, want none (table died in the crashed sweep)", got) + } + + const id2 = "exec-crash-finalize-b" + s.seedStaleExecution(t, rprExecution(id2, "proj-a", "recipe-crash2", publication.BundleRunning, 1)) + setReconcileGates(nil, func(reconcileFinalizeAttempt) error { return injected }) + rc3 := s.newReconciler(t, nil) + rc3.heartbeatTTL = 5 * time.Minute + rc3.leaseTTL = 80 * time.Millisecond + if err := rc3.SweepStale(ctx, time.Now().UTC(), newRPRDrops().hook); !errors.Is(err, injected) { + t.Fatalf("crash B: %v", err) + } + setReconcileGates(nil, nil) + time.Sleep(120 * time.Millisecond) + foreign := s.newReconciler(t, nil) + foreign.fixIdentity("reconciler-foreign") + if err := foreign.SweepStale(ctx, time.Now().UTC(), newRPRDrops().hook); !errors.Is(err, publication.ErrExecutionReadPinActive) { + t.Fatalf("foreign sweep over the live abandoned claim = %v, want named refusal", err) + } + s.queryRows(t, s.w, + "UPDATE "+pgx.Identifier{s.schema, ExecutionCleanupClaimsTable}.Sanitize()+ + " SET expires_at = $1 WHERE execution_id = $2", rprPast, id2) + drops4 := newRPRDrops() + foreign2 := s.newReconciler(t, nil) + foreign2.fixIdentity("reconciler-foreign-2") + if err := foreign2.SweepStale(ctx, time.Now().UTC(), drops4.hook); err != nil { + t.Fatalf("foreign post-expiry convergence: %v", err) + } + if got := drops4.snapshot(); len(got) != 0 { + t.Fatalf("foreign convergence drops = %v, want none (table died in the crashed sweep)", got) + } + if state, payload := s.executionRow(t, id2); state != string(publication.BundleFailed) || + !strings.Contains(payload, `"error":"stale execution reconciled"`) { + t.Fatalf("foreign convergence record = %q %s", state, payload) + } + if claim := s.claimRow(t, s.w, id2); claim != "" { + t.Fatalf("claim not consumed by convergence: %q", claim) + } + } +} + +func TestReconcileHeartbeatLossStopsDrop(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + setReconcileGates(nil, nil) + defer setReconcileGates(nil, nil) + + const id = "exec-heartbeat-loss" + s.seedStaleExecution(t, rprExecution(id, "proj-a", "recipe-hb", publication.BundleRunning, 1)) + + drops := newRPRDrops() + releaseNow := make(chan struct{}) + drops.waitFor = releaseNow + + rc := s.newReconciler(t, nil) + rc.heartbeatTTL = 200 * time.Millisecond + rc.renewalEvery = 15 * time.Millisecond + errCh := make(chan error, 1) + go func() { + errCh <- rc.SweepStale(ctx, time.Now().UTC(), drops.hook) + }() + waitUntil(t, "cleanup claim granted before the drop completed", func() bool { + return s.claimRow(t, s.w, id) != "" + }) + if err := s.pinsW.ReleaseExecutionCleanup(context.Background(), id, "reconciler-test"); err != nil { + t.Fatalf("external claim release: %v", err) + } + time.Sleep(80 * time.Millisecond) + close(releaseNow) + err := <-errCh + if !errors.Is(err, publication.ErrExecutionReadPinLost) { + t.Fatalf("heartbeat-loss sweep error = %v, want errors.Is ErrExecutionReadPinLost", err) + } + _, payload := s.executionRow(t, id) + if !strings.Contains(payload, `"physicalTable":"tbl-`+id+`-0"`) { + t.Fatalf("remaining outputs lost the un-dropped table: %s", payload) + } + if claim := s.claimRow(t, s.w, id); claim != "" { + t.Fatalf("claim should be gone after the sweep finished its releases: %q", claim) + } +} + +func TestReconcileFirstErrorWins(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + setReconcileGates(nil, nil) + defer setReconcileGates(nil, nil) + + s.seedStaleExecution(t, rprExecution("exec-abort-a", "proj-a", "recipe-abort", publication.BundleRunning, 1)) + s.seedStaleExecution(t, rprExecution("exec-abort-b", "proj-a", "recipe-abort", publication.BundleRunning, 1)) + must(s.pinsA.AcquireExecutionReadPin(ctx, "exec-abort-a", "reader", rprFarFuture))(t) + must(s.pinsA.AcquireExecutionReadPin(ctx, "exec-abort-b", "reader", rprFarFuture))(t) + + drops := newRPRDrops() + rc := s.newReconciler(t, nil) + var decisions []ReconcileDecision + rc.OnDecision = func(d ReconcileDecision) { decisions = append(decisions, d) } + err := rc.SweepStale(ctx, time.Now().UTC(), drops.hook) + if !errors.Is(err, publication.ErrExecutionReadPinActive) { + t.Fatalf("sweep error = %v", err) + } + if len(decisions) != 1 || decisions[0].ExecutionID != "exec-abort-a" { + t.Fatalf("decisions = %v, want exactly the first execution (baseline first-error-wins stranding)", decisions) + } + if state, _ := s.executionRow(t, "exec-abort-b"); state != string(publication.BundleRunning) { + t.Fatalf("second execution was processed after the abort: %q", state) + } +} + +func eqStatementLists(t *testing.T, label string, want, got []string) { + t.Helper() + if len(want) != len(got) { + t.Fatalf("%s: statement count = %d, want %d\ngot: %#v\nwant: %#v", label, len(got), len(want), got, want) + } + for i := range want { + if want[i] != got[i] { + t.Fatalf("%s: statement[%d] mismatch\ngot: %s\nwant: %s", label, i, got[i], want[i]) + } + } +} + +func TestReconcileGoldenClaimStatements(t *testing.T) { + locks := func(s *rprService) (pointer, ensure, anchor, active, read, gc, grant, touch string) { + q := func(table string) string { return pgx.Identifier{s.schema, table}.Sanitize() } + pointer = "SELECT execution_id FROM " + q(BundlePointersTable) + " WHERE pointer_key = $1 FOR UPDATE" + ensure = "INSERT INTO " + q(ExecutionRetentionTable) + " (execution_id, updated_at) VALUES ($1, $2) ON CONFLICT (execution_id) DO NOTHING" + anchor = "SELECT true FROM " + q(ExecutionRetentionTable) + " WHERE execution_id = $1 FOR UPDATE" + active = "SELECT EXISTS (SELECT 1 FROM " + q(ExecutionReadPinsTable) + " WHERE execution_id = $1 AND expires_at >= $2)" + read = "SELECT owner_id, expires_at FROM " + q(ExecutionCleanupClaimsTable) + " WHERE execution_id = $1" + gc = "DELETE FROM " + q(ExecutionReadPinsTable) + " WHERE execution_id = $1 AND expires_at < $2" + grant = "INSERT INTO " + q(ExecutionCleanupClaimsTable) + " (execution_id, owner_id, expires_at, updated_at)" + + " VALUES ($1, $2, $3, $4) ON CONFLICT (execution_id) DO UPDATE SET owner_id = $2, expires_at = $3, updated_at = $4" + touch = "UPDATE " + q(ExecutionRetentionTable) + " SET updated_at = $2 WHERE execution_id = $1" + return + } + + runCapture := func(t *testing.T, prepare func(s *rprService, id string), wantOutcome ReconcileOutcome, wantErr bool) { + t.Helper() + s := newRPRService(t) + ctx := s.ctx + defer setReconcileGates(nil, nil) + pointer, ensure, anchor, active, read, gc, grant, touch := locks(s) + var attempts []reconcileClaimAttempt + setReconcileGates(func(a reconcileClaimAttempt) error { + attempts = append(attempts, a) + return nil + }, nil) + const id = "exec-golden-claim" + s.seedStaleExecution(t, rprExecution(id, "proj-a", "recipe-golden", publication.BundleRunning, 1)) + if prepare != nil { + prepare(s, id) + } + rc := s.newReconciler(t, nil) + err := rc.SweepStale(ctx, time.Now().UTC(), newRPRDrops().hook) + if wantErr && !errors.Is(err, publication.ErrExecutionReadPinActive) { + t.Fatalf("sweep error = %v, want named refusal", err) + } + if !wantErr && err != nil { + t.Fatalf("sweep: %v", err) + } + if len(attempts) != 1 { + t.Fatalf("claim attempts = %d, want exactly one: %#v", len(attempts), attempts) + } + a := attempts[0] + if a.ExecutionID != id || a.Outcome != wantOutcome { + t.Fatalf("attempt = %s/%v, want %s/%v", a.ExecutionID, a.Outcome, id, wantOutcome) + } + if a.ReconcilerID != "reconciler-test" { + t.Fatalf("attempt identity = %q, want reconciler-test", a.ReconcilerID) + } + if a.BackendPID == 0 { + t.Fatal("claim attempt carries no backend witness") + } + switch wantOutcome { + case reconcileClaimGranted: + eqStatementLists(t, "granted", []string{pointer, ensure, anchor, active, read, gc, grant, touch}, a.Statements) + case ReconcileRefusedPinActive: + eqStatementLists(t, "refused-pin", []string{pointer, ensure, anchor, active}, a.Statements) + case ReconcileRefusedCleanupOwned: + eqStatementLists(t, "refused-claim", []string{pointer, ensure, anchor, active, read}, a.Statements) + case ReconcileSkippedPointer: + eqStatementLists(t, "vetoed-pointer", []string{pointer, ensure, anchor}, a.Statements) + } + } + + t.Run("granted", func(t *testing.T) { runCapture(t, nil, reconcileClaimGranted, false) }) + t.Run("refused_by_active_pin", func(t *testing.T) { + runCapture(t, func(s *rprService, id string) { + if granted, err := s.pinsW.AcquireExecutionReadPin(s.ctx, id, "reader-1", rprFarFuture); err != nil || !granted { + t.Fatalf("seed pin: %v %v", granted, err) + } + }, ReconcileRefusedPinActive, true) + }) + t.Run("refused_by_foreign_claim", func(t *testing.T) { + runCapture(t, func(s *rprService, id string) { + if granted, err := s.pinsW.ClaimExecutionCleanup(s.ctx, id, "worker-foreign"); err != nil || !granted { + t.Fatalf("seed claim: %v %v", granted, err) + } + }, ReconcileRefusedCleanupOwned, true) + }) + t.Run("vetoed_by_in_transaction_pointer", func(t *testing.T) { + runCapture(t, func(s *rprService, id string) { + execution := rprExecution(id, "proj-a", "recipe-golden", publication.BundleRunning, 1) + if err := s.regW.CompareAndSwapPointer(s.ctx, execution.PointerName(), "", "exec-golden-other"); err != nil { + t.Fatalf("seed pointer: %v", err) + } + setBeforeClaim(func(executionID string) { + if executionID != id { + return + } + if err := s.regB.CompareAndSwapPointer(s.ctx, execution.PointerName(), "exec-golden-other", id); err != nil { + t.Errorf("pre-claim pointer move: %v", err) + } + }) + }, ReconcileSkippedPointer, false) + defer setBeforeClaim(nil) + }) +} + +func TestReconcileTerminalReleaseSurvivesSlowDrop(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + defer setReconcileGates(nil, nil) + const id = "exec-slowdrop" + s.seedStaleExecution(t, rprExecution(id, "proj-a", "recipe-slowdrop", publication.BundleRunning, 1)) + rc := s.newReconciler(t, nil) + rc.callTimeout = 250 * time.Millisecond + drops := newRPRDrops() + slowDrop := func(ctx context.Context, table string) error { + time.Sleep(450 * time.Millisecond) + return drops.hook(ctx, table) + } + if err := rc.SweepStale(ctx, time.Now().UTC(), slowDrop); err != nil { + t.Fatalf("sweep with slow drop: %v", err) + } + if claim := s.claimRow(t, s.w, id); claim != "" { + t.Fatalf("cleanup claim stranded live after a slow drop: %q", claim) + } + state, payload := s.executionRow(t, id) + if state != string(publication.BundleFailed) || !strings.Contains(payload, `"error":"stale execution reconciled"`) { + t.Fatalf("terminal record not written through the slow drop: %q %s", state, payload) + } + if got := drops.snapshot(); len(got) != 1 || got[0] != "tbl-"+id+"-0" { + t.Fatalf("drops = %v", got) + } + if s.leaseRow(t, "key-"+id) != "" { + t.Fatal("bundle lease stranded after a slow drop") + } +} + +func TestReconcileVetoCleanupReleaseBounded(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + defer setReconcileGates(nil, nil) + defer setReadPinGate(nil) + + const id = "exec-veto-bound" + execution := rprExecution(id, "proj-a", "recipe-vetobound", publication.BundleRunning, 1) + s.seedStaleExecution(t, execution) + releases := newRPRReleases() + releases.bind("proj-a", id) + + rc := s.newReconciler(t, releases) + rc.callTimeout = 900 * time.Millisecond + setReconcileGates(func(attempt reconcileClaimAttempt) error { + if attempt.ExecutionID == id && attempt.Outcome == reconcileClaimGranted { + time.Sleep(650 * time.Millisecond) + } + return nil + }, nil) + setReadPinGate(func(attempt readPinAttempt) error { + if attempt.Op == "release-cleanup" && attempt.ExecutionID == id { + time.Sleep(500 * time.Millisecond) + } + return nil + }) + + started := time.Now() + drops := newRPRDrops() + var decisions []ReconcileDecision + rc.OnDecision = func(d ReconcileDecision) { decisions = append(decisions, d) } + if err := rc.SweepStale(ctx, time.Now().UTC(), drops.hook); err != nil { + t.Fatalf("veto sweep with a delayed claim release: %v", err) + } + if got := time.Since(started); got > 8*time.Second { + t.Fatalf("veto sweep blocked %v, want bounded", got) + } + sawBound := false + for _, d := range decisions { + if d.ExecutionID == id && d.Outcome == ReconcileSkippedBound { + sawBound = true + } + } + if !sawBound { + t.Fatalf("active-release veto expected on the post-claim gate: %v", decisions) + } + if got := drops.snapshot(); len(got) != 0 { + t.Fatalf("veto ran physical drops: %v", got) + } + if claim := s.claimRow(t, s.w, id); claim != "" { + t.Fatalf("veto path stranded the cleanup claim (release died past the pre-claim budget): %q", claim) + } + if s.leaseRow(t, "key-"+id) != "" { + t.Fatal("veto path stranded the bundle lease") + } +} + +func TestReconcileSweepsAreMutuallyExclusive(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + defer setReconcileGates(nil, nil) + const id = "exec-concur" + s.seedStaleExecution(t, rprExecution(id, "proj-a", "recipe-concur", publication.BundleRunning, 1)) + rc, err := NewReconciler(s.regA, s.pinsA, nil) + if err != nil { + t.Fatalf("new reconciler: %v", err) + } + rc.heartbeatTTL = 30 * time.Second + rc.renewalEvery = 20 * time.Millisecond + delayedDrop := func(d *rprDrops) PhysicalDropTable { + return func(ctx context.Context, table string) error { + time.Sleep(300 * time.Millisecond) + return d.hook(ctx, table) + } + } + dropsA, dropsB := newRPRDrops(), newRPRDrops() + start := make(chan struct{}) + var errA, errB error + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + <-start + errA = rc.SweepStale(ctx, time.Now().UTC(), delayedDrop(dropsA)) + }() + go func() { + defer wg.Done() + <-start + errB = rc.SweepStale(ctx, time.Now().UTC(), delayedDrop(dropsB)) + }() + close(start) + wg.Wait() + if errA != nil || errB != nil { + t.Fatalf("concurrent sweeps of one instance must exclude through the lease, not abort each other: A=%v B=%v", errA, errB) + } + total := append(dropsA.snapshot(), dropsB.snapshot()...) + if len(total) != 1 || total[0] != "tbl-"+id+"-0" { + t.Fatalf("drops = %v, want exactly one physical drop across both sweeps", total) + } + if state, _ := s.executionRow(t, id); state != string(publication.BundleFailed) { + t.Fatalf("execution state = %q, want FAILED", state) + } + if claim := s.claimRow(t, s.w, id); claim != "" { + t.Fatalf("claim not consumed: %q", claim) + } + if s.leaseRow(t, "key-"+id) != "" { + t.Fatal("lease not consumed") + } +} + +func TestReconcileMintsFreshIdentityPerSweep(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + defer setReconcileGates(nil, nil) + var attempts []reconcileClaimAttempt + setReconcileGates(func(a reconcileClaimAttempt) error { + attempts = append(attempts, a) + return nil + }, nil) + rc, err := NewReconciler(s.regA, s.pinsA, nil) + if err != nil { + t.Fatalf("new reconciler: %v", err) + } + s.seedStaleExecution(t, rprExecution("exec-id-a", "proj-a", "recipe-ida", publication.BundleRunning, 1)) + if err := rc.SweepStale(ctx, time.Now().UTC(), newRPRDrops().hook); err != nil { + t.Fatalf("sweep 1: %v", err) + } + s.seedStaleExecution(t, rprExecution("exec-id-b", "proj-a", "recipe-idb", publication.BundleRunning, 1)) + if err := rc.SweepStale(ctx, time.Now().UTC(), newRPRDrops().hook); err != nil { + t.Fatalf("sweep 2: %v", err) + } + var first, second string + for _, a := range attempts { + switch a.ExecutionID { + case "exec-id-a": + if first == "" { + first = a.ReconcilerID + } + case "exec-id-b": + second = a.ReconcilerID + } + } + if first == "" || second == "" { + t.Fatalf("claim attempts missing a sweep target: %v", attempts) + } + if first == second || !strings.HasPrefix(first, "reconciler-") || !strings.HasPrefix(second, "reconciler-") { + t.Fatalf("sweep identities must be freshly minted per call: %q vs %q", first, second) + } +} + +type unsafeDB struct { + inner DB + junk map[string]string +} + +func (u unsafeDB) Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) { + return u.inner.Exec(ctx, sql, args...) +} + +func (u unsafeDB) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { + return u.inner.Query(ctx, sql, args...) +} + +func (u unsafeDB) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row { + return u.inner.QueryRow(ctx, sql, args...) +} + +func (u unsafeDB) Begin(ctx context.Context) (pgx.Tx, error) { + return u.inner.Begin(ctx) +} + +func TestNewReconcilerRejectsUnsafeOrMismatchedBindings(t *testing.T) { + s := newRPRService(t) + + junk := map[string]string{"canary": "value"} + regU, err := New(s.w, s.schema) + if err != nil { + t.Fatalf("new registry: %v", err) + } + pinsU, err := NewReadPins(s.w, s.schema) + if err != nil { + t.Fatalf("new pins: %v", err) + } + regU.db = unsafeDB{inner: s.w, junk: junk} + pinsU.db = unsafeDB{inner: s.w, junk: junk} + if _, err := NewReconciler(regU, pinsU, nil); err == nil { + t.Fatal("non-comparable database bindings accepted") + } else if !strings.Contains(err.Error(), "not comparable") { + t.Fatalf("non-comparable binding error = %v, want the comparable refusal", err) + } + + regM, err := New(s.w, s.schema) + if err != nil { + t.Fatalf("new registry: %v", err) + } + pinsM, err := NewReadPins(s.w, s.schema) + if err != nil { + t.Fatalf("new pins: %v", err) + } + regM.db = s.w + pinsM.db = s.b + if _, err := NewReconciler(regM, pinsM, nil); err == nil { + t.Fatal("mismatched comparable pools accepted") + } + + regC, err := New(s.w, s.schema) + if err != nil { + t.Fatalf("new registry: %v", err) + } + pinsC, err := NewReadPins(s.w, s.schema) + if err != nil { + t.Fatalf("new pins: %v", err) + } + if _, err := NewReconciler(regC, pinsC, nil); err != nil { + t.Fatalf("matched bindings rejected: %v", err) + } +} + +func TestBundleCatalogDelegatesPinArbitration(t *testing.T) { + s := newRPRService(t) + ctx := s.ctx + cat, err := NewBundleCatalog(s.w, s.schema) + if err != nil { + t.Fatalf("new bundle catalog: %v", err) + } + if _, ok := any(cat).(publication.ExecutionReadPinCatalog); !ok { + t.Fatal("consumers cannot discover read pins on the bundle catalog value") + } + const id = "exec-catalog-delegation" + granted, err := cat.AcquireExecutionReadPin(ctx, id, "reader-1", rprFarFuture) + if err != nil || !granted { + t.Fatalf("catalog acquire pin = %v, %v", granted, err) + } + if rows := s.pinsRows(t, s.w, id); len(rows) != 1 || !strings.HasPrefix(rows[0], "reader-1|") { + t.Fatalf("acquire not delegated to the arbiter: %v", rows) + } + if _, err := cat.RenewExecutionReadPin(ctx, id, "reader-1", rprFarFuture.Add(time.Hour)); err != nil { + t.Fatalf("catalog renew pin: %v", err) + } + if claimed, err := cat.ClaimExecutionCleanup(ctx, id, "cleanup-1"); err != nil || claimed { + t.Fatalf("catalog claim over a live pin = %v, %v, want (false, nil) baseline refusal", claimed, err) + } + if claim := s.claimRow(t, s.w, id); claim != "" { + t.Fatalf("refused catalog claim minted a claim row: %q", claim) + } + if err := cat.ReleaseExecutionReadPin(ctx, id, "reader-1"); err != nil { + t.Fatalf("catalog release pin: %v", err) + } + if rows := s.pinsRows(t, s.w, id); len(rows) != 0 { + t.Fatalf("release not delegated: %v", rows) + } + granted, err = cat.ClaimExecutionCleanup(ctx, id, "cleanup-1") + if err != nil || !granted { + t.Fatalf("catalog claim after release = %v, %v", granted, err) + } + if _, err := cat.RenewExecutionCleanup(ctx, id, "cleanup-1", rprFarFuture); err != nil { + t.Fatalf("catalog renew cleanup: %v", err) + } + if err := cat.ReleaseExecutionCleanup(ctx, id, "cleanup-1"); err != nil { + t.Fatalf("catalog release cleanup: %v", err) + } + if claim := s.claimRow(t, s.w, id); claim != "" { + t.Fatalf("catalog release cleanup not delegated: %q", claim) + } + if cat.Pins() == nil { + t.Fatal("catalog does not expose its bound arbiter for reconciler wiring") + } +} + +func TestZZZeroLeftoverReadPinsFixtures(t *testing.T) { + if strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) == "" { + t.Skip("no service configured; nothing could have been created") + } + if err := auditReadPinsLeftovers(); err != nil { + t.Errorf("read-pins zero-leftover audit failed: %v", err) + } +} diff --git a/internal/backends/postgres/internal/publication/registry.go b/internal/backends/postgres/internal/publication/registry.go new file mode 100644 index 000000000..d61748c7d --- /dev/null +++ b/internal/backends/postgres/internal/publication/registry.go @@ -0,0 +1,359 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "github.com/calypr/loom/internal/dataframe/publication" +) + +const ( + BundleExecutionsTable = "publication_bundle_executions" + BundlePointersTable = "publication_bundle_pointers" + BundleLeasesTable = "publication_bundle_leases" +) + +var ErrInvalidSchema = errors.New("publication postgres: invalid schema name") + +type execer interface { + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) +} + +type DB interface { + execer + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row + Begin(ctx context.Context) (pgx.Tx, error) +} + +type Registry struct { + db DB + executions string + pointers string + leases string +} + +var ( + _ publication.BundleCatalog = (*Registry)(nil) + _ publication.ExactExecutionCatalog = (*Registry)(nil) + _ publication.PagedBundleCatalog = (*Registry)(nil) +) + +func New(pool DB, schema string) (*Registry, error) { + if pool == nil { + return nil, errors.New("publication postgres: nil pool") + } + if !pgstore.ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", ErrInvalidSchema, schema) + } + return &Registry{ + db: pool, + executions: pgx.Identifier{schema, BundleExecutionsTable}.Sanitize(), + pointers: pgx.Identifier{schema, BundlePointersTable}.Sanitize(), + leases: pgx.Identifier{schema, BundleLeasesTable}.Sanitize(), + }, nil +} + +func pointerKey(name string) string { + sum := sha256.Sum256([]byte(name)) + return hex.EncodeToString(sum[:]) +} + +func executionStates(state publication.BundleState) []string { + states := []string{string(state.Canonical())} + switch state.Canonical() { + case publication.BundleQueued: + states = append(states, string(publication.BundlePending)) + case publication.BundleRunning: + states = append(states, string(publication.BundlePreflight), string(publication.BundleLoading)) + case publication.BundlePublished: + states = append(states, string(publication.BundleReady)) + } + return states +} + +func executionParams(execution publication.BundleExecution) ([]any, error) { + payload, err := json.Marshal(execution) + if err != nil { + return nil, fmt.Errorf("publication postgres: marshal execution payload: %w", err) + } + return []any{ + execution.ID, execution.Key, execution.Project, execution.DatasetGeneration, + execution.Name, execution.TranslationVersion, execution.State, + execution.CreatedAt, execution.UpdatedAt, execution.ReadyAt, execution.PublishedAt, + string(payload), + }, nil +} + +const executionColumnList = "id, execution_key, project, dataset_generation, recipe_name, " + + "translation_version, state, created_at, updated_at, ready_at, published_at, payload" + +func scanExecution(row pgx.Row) (publication.BundleExecution, error) { + var payload []byte + if err := row.Scan(&payload); err != nil { + return publication.BundleExecution{}, err + } + var execution publication.BundleExecution + if err := json.Unmarshal(payload, &execution); err != nil { + return publication.BundleExecution{}, fmt.Errorf("publication postgres: decode execution payload: %w", err) + } + return execution, nil +} + +func (r *Registry) SaveExecution(ctx context.Context, execution publication.BundleExecution, fenceOwner string) error { + if execution.ID == "" || execution.Key == "" || fenceOwner == "" { + return publication.ErrBundleLeaseLost + } + params, err := executionParams(execution) + if err != nil { + return err + } + tx, err := r.db.Begin(ctx) + if err != nil { + return fmt.Errorf("publication postgres: begin save execution: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var held bool + if err := tx.QueryRow(ctx, + "SELECT true FROM "+r.leases+ + " WHERE key = $1 AND owner_id = $2 AND expires_at >= $3 FOR UPDATE", + execution.Key, fenceOwner, time.Now().UTC()).Scan(&held); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return publication.ErrBundleLeaseLost + } + return fmt.Errorf("publication postgres: check save fence: %w", err) + } + if !held { + return publication.ErrBundleLeaseLost + } + + set := "execution_key = EXCLUDED.execution_key, project = EXCLUDED.project, " + + "dataset_generation = EXCLUDED.dataset_generation, recipe_name = EXCLUDED.recipe_name, " + + "translation_version = EXCLUDED.translation_version, state = EXCLUDED.state, " + + "created_at = EXCLUDED.created_at, updated_at = EXCLUDED.updated_at, " + + "ready_at = EXCLUDED.ready_at, published_at = EXCLUDED.published_at, " + + "payload = EXCLUDED.payload" + if _, err := tx.Exec(ctx, + "INSERT INTO "+r.executions+" ("+executionColumnList+") "+ + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) "+ + "ON CONFLICT (id) DO UPDATE SET "+set, + params...); err != nil { + return fmt.Errorf("publication postgres: save execution: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("publication postgres: commit save execution: %w", err) + } + return nil +} + +func (r *Registry) GetExecution(ctx context.Context, id string) (publication.BundleExecution, error) { + execution, err := scanExecution(r.db.QueryRow(ctx, + "SELECT payload FROM "+r.executions+" WHERE id = $1", id)) + if errors.Is(err, pgx.ErrNoRows) { + return publication.BundleExecution{}, publication.ErrBundleNotFound + } + if err != nil { + return publication.BundleExecution{}, err + } + return execution.CanonicalizeLegacy(), nil +} + +func (r *Registry) FindExecutionByKey(ctx context.Context, key string) (publication.BundleExecution, error) { + execution, err := scanExecution(r.db.QueryRow(ctx, + "SELECT payload FROM "+r.executions+ + " WHERE execution_key = $1 ORDER BY created_at DESC, id DESC LIMIT 1", key)) + if errors.Is(err, pgx.ErrNoRows) { + return publication.BundleExecution{}, publication.ErrBundleNotFound + } + if err != nil { + return publication.BundleExecution{}, err + } + return execution.CanonicalizeLegacy(), nil +} + +func (r *Registry) scanExecutionRows(ctx context.Context, state publication.BundleState, before time.Time, afterUpdatedAt time.Time, afterID string, limit int) ([]publication.BundleExecution, time.Time, string, error) { + query := "SELECT payload, updated_at, id FROM " + r.executions + + " WHERE state = ANY($1) AND updated_at < $2" + args := []any{executionStates(state), before} + if limit > 0 { + query += " AND (updated_at > $3 OR (updated_at = $3 AND id > $4))" + args = append(args, afterUpdatedAt, afterID) + args = append(args, limit) + query += fmt.Sprintf(" ORDER BY updated_at ASC, id ASC LIMIT $%d", len(args)) + } else { + query += " ORDER BY updated_at ASC, id ASC" + } + rows, err := r.db.Query(ctx, query, args...) + if err != nil { + return nil, time.Time{}, "", fmt.Errorf("publication postgres: list executions: %w", err) + } + defer rows.Close() + out := []publication.BundleExecution{} + var lastAt time.Time + var lastID string + for rows.Next() { + var payload []byte + if err := rows.Scan(&payload, &lastAt, &lastID); err != nil { + return nil, time.Time{}, "", fmt.Errorf("publication postgres: scan execution: %w", err) + } + var execution publication.BundleExecution + if err := json.Unmarshal(payload, &execution); err != nil { + return nil, time.Time{}, "", fmt.Errorf("publication postgres: decode execution payload: %w", err) + } + out = append(out, execution.CanonicalizeLegacy()) + } + if err := rows.Err(); err != nil { + return nil, time.Time{}, "", fmt.Errorf("publication postgres: list executions: %w", err) + } + return out, lastAt, lastID, nil +} + +func (r *Registry) ListExecutions(ctx context.Context, state publication.BundleState, before time.Time) ([]publication.BundleExecution, error) { + out, _, _, err := r.scanExecutionRows(ctx, state, before, time.Time{}, "", 0) + return out, err +} + +func (r *Registry) VisitExecutionPages(ctx context.Context, state publication.BundleState, before time.Time, pageSize int, visit publication.BundleExecutionPageFunc) error { + if pageSize <= 0 { + return fmt.Errorf("execution page size must be positive") + } + if visit == nil { + return fmt.Errorf("execution page visitor is required") + } + var afterUpdatedAt time.Time + afterID := "" + for { + page, lastAt, lastID, err := r.scanExecutionRows(ctx, state, before, afterUpdatedAt, afterID, pageSize) + if err != nil { + return err + } + if len(page) == 0 { + return nil + } + if err := visit(page); err != nil { + return err + } + if len(page) < pageSize { + return nil + } + if lastID == "" || lastAt.IsZero() { + return fmt.Errorf("execution page has no stable continuation key") + } + afterUpdatedAt, afterID = lastAt, lastID + } +} + +var publishedSelectorStates = []string{string(publication.BundlePublished), string(publication.BundleReady)} + +func (r *Registry) FindExecutionBySelector(ctx context.Context, project, generation string, selector publication.DataframeSelector) (publication.BundleExecution, publication.BundleOutputRecord, error) { + if err := selector.Validate(); err != nil { + return publication.BundleExecution{}, publication.BundleOutputRecord{}, err + } + execution, err := scanExecution(r.db.QueryRow(ctx, + "SELECT payload FROM "+r.executions+ + " WHERE project = $1 AND dataset_generation = $2 AND recipe_name = $3"+ + " AND translation_version = $4 AND state = ANY($5)"+ + " AND EXISTS (SELECT 1 FROM jsonb_array_elements(COALESCE(payload->'outputs', '[]'::jsonb)) AS o"+ + " WHERE o->>'name' = $6 OR o->>'Name' = $6)"+ + " ORDER BY published_at DESC NULLS LAST, ready_at DESC NULLS LAST, updated_at DESC, id DESC"+ + " LIMIT 1", + project, generation, selector.Recipe, selector.TranslationVersion, + publishedSelectorStates, selector.Output)) + if errors.Is(err, pgx.ErrNoRows) { + return publication.BundleExecution{}, publication.BundleOutputRecord{}, publication.ErrBundleNotFound + } + if err != nil { + return publication.BundleExecution{}, publication.BundleOutputRecord{}, err + } + execution = execution.CanonicalizeLegacy() + for _, output := range execution.Outputs { + if output.Name == selector.Output { + return execution, output, nil + } + } + return publication.BundleExecution{}, publication.BundleOutputRecord{}, publication.ErrBundleNotFound +} + +func (r *Registry) GetPointer(ctx context.Context, name string) (publication.BundlePointer, error) { + var ( + nameBytes []byte + executionID string + updatedAt time.Time + ) + err := r.db.QueryRow(ctx, + "SELECT name_bytes, execution_id, updated_at FROM "+r.pointers+" WHERE pointer_key = $1", + pointerKey(name)).Scan(&nameBytes, &executionID, &updatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return publication.BundlePointer{}, publication.ErrBundleNotFound + } + if err != nil { + return publication.BundlePointer{}, err + } + return publication.BundlePointer{Name: string(nameBytes), ExecutionID: executionID, UpdatedAt: updatedAt}, nil +} + +func compareAndSwapPointer(ctx context.Context, e execer, qualifiedPointers, name, expected, next string, updatedAt time.Time) (bool, string, error) { + sql := "INSERT INTO " + qualifiedPointers + " (pointer_key, name_bytes, execution_id, updated_at)" + + " VALUES ($1, $2, $3, $4)" + + " ON CONFLICT (pointer_key) DO UPDATE SET execution_id = $3, updated_at = $4" + + " WHERE " + BundlePointersTable + ".execution_id = $5" + tag, err := e.Exec(ctx, sql, pointerKey(name), []byte(name), next, updatedAt, expected) + if err != nil { + return false, sql, fmt.Errorf("publication postgres: compare-and-swap pointer: %w", err) + } + return tag.RowsAffected() == 1, sql, nil +} + +func (r *Registry) CompareAndSwapPointer(ctx context.Context, name, expected, next string) error { + swapped, _, err := compareAndSwapPointer(ctx, r.db, r.pointers, name, expected, next, time.Now().UTC()) + if err != nil { + return err + } + if !swapped { + return publication.ErrBundlePointerConflict + } + return nil +} + +func (r *Registry) AcquireBundleLease(ctx context.Context, key, owner string, expires time.Time) (bool, error) { + now := time.Now().UTC() + tag, err := r.db.Exec(ctx, + "INSERT INTO "+r.leases+" (key, owner_id, expires_at, acquired_at, epoch)"+ + " VALUES ($1, $2, $3, $4, 1)"+ + " ON CONFLICT (key) DO UPDATE SET owner_id = $2, expires_at = $3, acquired_at = $4,"+ + " epoch = "+BundleLeasesTable+".epoch + 1"+ + " WHERE "+BundleLeasesTable+".expires_at < $4 OR "+BundleLeasesTable+".owner_id = $2", + key, owner, expires, now) + if err != nil { + return false, fmt.Errorf("publication postgres: acquire bundle lease: %w", err) + } + return tag.RowsAffected() == 1, nil +} + +func (r *Registry) RenewBundleLease(ctx context.Context, key, owner string, expires time.Time) (bool, error) { + tag, err := r.db.Exec(ctx, + "UPDATE "+r.leases+" SET expires_at = $3 WHERE key = $1 AND owner_id = $2", + key, owner, expires) + if err != nil { + return false, fmt.Errorf("publication postgres: renew bundle lease: %w", err) + } + return tag.RowsAffected() == 1, nil +} + +func (r *Registry) ReleaseBundleLease(ctx context.Context, key, owner string) error { + if _, err := r.db.Exec(ctx, + "DELETE FROM "+r.leases+" WHERE key = $1 AND owner_id = $2", key, owner); err != nil { + return fmt.Errorf("publication postgres: release bundle lease: %w", err) + } + return nil +} diff --git a/internal/backends/postgres/internal/publication/registry_test.go b/internal/backends/postgres/internal/publication/registry_test.go new file mode 100644 index 000000000..6da47acef --- /dev/null +++ b/internal/backends/postgres/internal/publication/registry_test.go @@ -0,0 +1,1289 @@ +package postgres + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/dataframe/publication" +) + +var pgpubcat01ResourceNamePattern = regexp.MustCompile(`^pgpubcat01_it_[0-9]+_[0-9a-f]+$`) + +func testPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 publication catalog integration tests") + } + return raw +} + +func randomHex() string { + buf := make([]byte, 6) + if _, err := rand.Read(buf); err != nil { + panic(err) + } + return hex.EncodeToString(buf) +} + +type catalogService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + schema string + registry *Registry + scrub func(any) string + fail func(step string, err error) +} + +func newCatalogService(t *testing.T) *catalogService { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgpubcat01_it_%d_%s", time.Now().UnixNano(), randomHex()) + if !pgpubcat01ResourceNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("pgpubcat01_it_%d_%s", time.Now().UnixNano(), randomHex()) + if !pgpubcat01ResourceNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + s := &catalogService{ctx: ctx, admin: admin, iso: iso, schema: schema, scrub: scrub, fail: fail} + s.migrate(t) + registry, err := New(iso, schema) + if err != nil { + fail("build registry", err) + } + s.registry = registry + return s +} + +func (s *catalogService) migrate(t *testing.T) { + t.Helper() + history := mustOwnerHistorySubset(t, 1, 4) + runner, err := storepostgres.NewMigrationRunner(s.iso, s.schema, history) + if err != nil { + s.fail("new migration runner", err) + } + result, err := runner.Migrate(s.ctx) + if err != nil { + s.fail("migrate catalog", err) + } + if result.LatestVersion != 4 { + t.Fatalf("latest migration version = %d, want 4", result.LatestVersion) + } + wantLedger := []storepostgres.LedgerRecord{ + {Version: 1, Name: "ledger", Checksum: "db70f7758495f7557b3a694149b80ed6e5d4cb74ea5aa7d59a1a2eaa46fc9cde"}, + {Version: 4, Name: "publication_bundle_catalog", Checksum: "adcb4683ffd1611b9c68e7a49aaa4b8670d106d65df927cbb8e11866be224943"}, + } + if len(result.Applied) != len(wantLedger) { + t.Fatalf("applied history = %+v, want %+v", result.Applied, wantLedger) + } + for i, want := range wantLedger { + got := result.Applied[i] + if got.Version != want.Version || got.Name != want.Name || got.Checksum != want.Checksum { + t.Fatalf("applied[%d] = {Version:%d Name:%q Checksum:%q}, want {Version:%d Name:%q Checksum:%q}", + i, got.Version, got.Name, got.Checksum, want.Version, want.Name, want.Checksum) + } + } +} + +func mustOwnerHistorySubset(t *testing.T, versions ...int64) []storepostgres.Migration { + t.Helper() + owner, err := OwnerMigrations() + if err != nil { + t.Fatalf("owner migrations: %v", err) + } + want := make(map[int64]struct{}, len(versions)) + for _, v := range versions { + want[v] = struct{}{} + } + seen := make(map[int64]struct{}, len(versions)) + var out []storepostgres.Migration + for _, m := range owner { + if _, ok := want[m.Version]; !ok { + continue + } + if _, dup := seen[m.Version]; dup { + t.Fatalf("owner history has duplicate version %d", m.Version) + } + seen[m.Version] = struct{}{} + out = append(out, m) + } + if len(out) != len(versions) { + t.Fatalf("owner history covers versions %v of %v, want %v", seen, versions, versions) + } + return out +} + +func (s *catalogService) mustExec(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := s.iso.Exec(s.ctx, sql, args...); err != nil { + s.fail("exec "+sql, err) + } +} + +func (s *catalogService) execCount(t *testing.T, sql string, args ...any) int { + t.Helper() + var n int + if err := s.iso.QueryRow(s.ctx, sql, args...).Scan(&n); err != nil { + s.fail("count "+sql, err) + } + return n +} + +func (s *catalogService) witnessText(t *testing.T, sql string, args ...any) (string, bool) { + t.Helper() + var value string + err := s.iso.QueryRow(s.ctx, sql, args...).Scan(&value) + if errors.Is(err, pgx.ErrNoRows) { + return "", false + } + if err != nil { + s.fail("witness "+sql, err) + } + return value, true +} + +func (s *catalogService) mustWitnessText(t *testing.T, sql string, args ...any) string { + t.Helper() + value, ok := s.witnessText(t, sql, args...) + if !ok { + t.Fatalf("witness query returned no rows: %s", sql) + } + return value +} + +func (s *catalogService) mustWitnessBool(t *testing.T, sql string, args ...any) bool { + t.Helper() + var value bool + err := s.iso.QueryRow(s.ctx, sql, args...).Scan(&value) + if err != nil { + s.fail("witness "+sql, err) + } + return value +} + +func (s *catalogService) mustWitnessInt64(t *testing.T, sql string, args ...any) int64 { + t.Helper() + var value int64 + err := s.iso.QueryRow(s.ctx, sql, args...).Scan(&value) + if err != nil { + s.fail("witness "+sql, err) + } + return value +} + +func (s *catalogService) qualified(table string) string { + return pgx.Identifier{s.schema, table}.Sanitize() +} + +func (s *catalogService) acquireLease(t *testing.T, key, owner string, expires time.Time) { + t.Helper() + claimed, err := s.registry.AcquireBundleLease(s.ctx, key, owner, expires) + if err != nil || !claimed { + s.fail(fmt.Sprintf("acquire lease %s/%s", key, owner), fmt.Errorf("claimed=%v err=%w", claimed, err)) + } +} + +func (s *catalogService) saveExecution(t *testing.T, execution publication.BundleExecution, fenceOwner string) { + t.Helper() + if err := s.registry.SaveExecution(s.ctx, execution, fenceOwner); err != nil { + s.fail("save execution "+execution.ID, err) + } +} + +func baseExecution(id, key string, state publication.BundleState, stamps executionStamps) publication.BundleExecution { + createdAt := stamps.created + updatedAt := stamps.updated + execution := publication.BundleExecution{ + ID: id, + Key: key, + BundleIdentity: publication.BundleIdentity{ + Name: "panel", + TranslationVersion: "v1", + Project: "alpha", + DatasetGeneration: "gen-1", + RecipeDigest: "sha256:recipe", + SchemaDigest: "sha256:schema", + ScopeDigest: "sha256:scope", + EngineVersion: "loom-engine-3", + AuthScopeMode: "project", + AuthResourcePaths: []string{"fhir/Patient", "fhir/Observation"}, + }, + State: state, + CreatedAt: createdAt, + UpdatedAt: updatedAt, + ReadyAt: stamps.ready, + } + if stamps.published != nil { + execution.PublishedAt = stamps.published + } + if state.Successful() || state == publication.BundleRunning { + execution.Outputs = []publication.BundleOutputRecord{ + { + Name: "out-a", + PhysicalTable: "phys_alpha_out_a", + Selector: execution.Selector("out-a"), + Columns: []publication.PhysicalColumn{ + {ID: "c1", Name: "subject", SemanticPath: "Patient.id", ClickHouse: "String", LogicalType: "ResourceIdentifier", Provenance: publication.ColumnExplicit, LoomOwned: true}, + {ID: "c2", Name: "value", ClickHouse: "Nullable(Float64)", LogicalType: "Quantity", Nullable: true, Repeated: true}, + }, + RowCount: 1234, + ByteCount: 567890, + State: state.Canonical(), + SourceRow: &publication.SourceRowMetadata{ResourceType: "Patient", IDColumn: "subject"}, + }, + { + Name: "out-b", + PhysicalTable: "phys_alpha_out_b", + Selector: execution.Selector("out-b"), + RowCount: 7, + State: state.Canonical(), + }, + } + if state.Successful() { + verified := updatedAt + for i := range execution.Outputs { + execution.Outputs[i].VerifiedAt = &verified + } + } + } + return execution +} + +type executionStamps struct { + created time.Time + updated time.Time + ready *time.Time + published *time.Time +} + +func stamps(created, updated, ready, published time.Time) executionStamps { + st := executionStamps{created: created, updated: updated} + if !ready.IsZero() { + st.ready = &ready + } + if !published.IsZero() { + st.published = &published + } + return st +} + +var ( + baseTime = time.Date(2026, 11, 9, 12, 0, 0, 0, time.UTC) + createdTime = baseTime + updatedTime = baseTime.Add(time.Minute) + readyTime = baseTime.Add(90 * time.Second) + pubTime = baseTime.Add(2 * time.Minute) +) + +func TestNewRejectsNilPoolAndUnsafeSchemas(t *testing.T) { + if _, err := New(nil, "ok_schema"); err == nil { + t.Fatal("New(nil pool) = nil error, want rejection") + } + for _, schema := range []string{"", "Public", "a;drop", "0starts_with_digit", "has space", "tab\tname", strings.Repeat("a", 64)} { + _, err := New(neverUsedPool{}, schema) + if !errors.Is(err, ErrInvalidSchema) { + t.Fatalf("New(schema %q) = %v, want ErrInvalidSchema", schema, err) + } + } +} + +type neverUsedPool struct{} + +func (neverUsedPool) Exec(context.Context, string, ...any) (pgconn.CommandTag, error) { + return pgconn.CommandTag{}, errors.New("never used") +} +func (neverUsedPool) Query(context.Context, string, ...any) (pgx.Rows, error) { + return nil, errors.New("never used") +} +func (neverUsedPool) QueryRow(context.Context, string, ...any) pgx.Row { + return nil +} +func (neverUsedPool) Begin(context.Context) (pgx.Tx, error) { return nil, errors.New("never used") } + +func TestPointerKeyDigestDiscipline(t *testing.T) { + name := "HTAN_INT-BForePC\x00\x00aced-meta-default" + first := pointerKey(name) + second := pointerKey(name) + if first != second { + t.Fatalf("pointer key is not deterministic: %q != %q", first, second) + } + if !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(first) { + t.Fatalf("pointer key is not a digest-safe hex key: %q", first) + } + if first == pointerKey(name+"\x00") { + t.Fatal("distinct names collide on pointer key") + } +} + +func TestCatalogSchemaHoldsMetadataOnly(t *testing.T) { + s := newCatalogService(t) + rows, err := s.iso.Query(s.ctx, + "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = $1 ORDER BY tablename", s.schema) + if err != nil { + s.fail("list tables", err) + } + var tables []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + s.fail("scan table", err) + } + tables = append(tables, name) + } + if err := rows.Err(); err != nil { + s.fail("list tables", err) + } + rows.Close() + want := []string{ + "publication_bundle_executions", + "publication_bundle_leases", + "publication_bundle_pointers", + "schema_migrations", + } + if !reflect.DeepEqual(tables, want) { + t.Fatalf("catalog schema tables = %#v, want exactly %#v (metadata only; no row-store tables)", tables, want) + } +} + +func TestSaveExecutionRoundTripAndLiteralColumns(t *testing.T) { + s := newCatalogService(t) + s.acquireLease(t, "bundle-key", "publisher-a", time.Now().UTC().Add(time.Minute)) + execution := baseExecution("exec-full", "bundle-key", publication.BundleRunning, stamps(createdTime, updatedTime, readyTime, time.Time{})) + s.saveExecution(t, execution, "publisher-a") + + got, err := s.registry.GetExecution(s.ctx, "exec-full") + if err != nil { + s.fail("get execution", err) + } + wantJSON := mustMarshal(t, execution) + gotJSON := mustMarshal(t, got) + if wantJSON != gotJSON { + t.Fatalf("round-tripped execution differs:\n got %s\nwant %s", gotJSON, wantJSON) + } + + var ( + gotKey, gotProject, gotGeneration, gotRecipe, gotTV, gotState string + gotCreated, gotUpdated time.Time + gotReady, gotPublished *time.Time + ) + err = s.iso.QueryRow(s.ctx, + "SELECT execution_key, project, dataset_generation, recipe_name, translation_version, state,"+ + " created_at, updated_at, ready_at, published_at FROM "+s.qualified(BundleExecutionsTable)+ + " WHERE id = $1", "exec-full"). + Scan(&gotKey, &gotProject, &gotGeneration, &gotRecipe, &gotTV, + &gotState, &gotCreated, &gotUpdated, &gotReady, &gotPublished) + if err != nil { + s.fail("witness columns", err) + } + if gotKey != "bundle-key" || gotProject != "alpha" || gotGeneration != "gen-1" || + gotRecipe != "panel" || gotTV != "v1" || gotState != "RUNNING" { + t.Fatalf("stored scope columns = {%s %s %s %s %s %s}, want {bundle-key alpha gen-1 panel v1 RUNNING}", + gotKey, gotProject, gotGeneration, gotRecipe, gotTV, gotState) + } + if !gotCreated.Equal(createdTime) || !gotUpdated.Equal(updatedTime) { + t.Fatalf("stored stamps = %v/%v, want %v/%v", gotCreated, gotUpdated, createdTime, updatedTime) + } + if gotReady == nil || !gotReady.Equal(readyTime) || gotPublished != nil { + t.Fatalf("stored nullable stamps = ready %v published %v, want ready %v published NULL", + gotReady, gotPublished, readyTime) + } + if payloadState := s.mustWitnessText(t, + "SELECT payload->>'state' FROM "+s.qualified(BundleExecutionsTable)+" WHERE id = $1", "exec-full"); payloadState != "RUNNING" { + t.Fatalf("payload state = %q, want RUNNING", payloadState) + } +} + +func mustMarshal(t *testing.T, v any) string { + t.Helper() + data, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshal %T: %v", v, err) + } + return string(data) +} + +func TestSaveExecutionRequiresActiveFence(t *testing.T) { + s := newCatalogService(t) + execution := baseExecution("exec-fence", "bundle-key", publication.BundleRunning, stamps(createdTime, updatedTime, time.Time{}, time.Time{})) + + noID := execution + noID.ID = "" + noKey := execution + noKey.Key = "" + for name, candidate := range map[string]publication.BundleExecution{"empty id": noID, "empty key": noKey} { + if err := s.registry.SaveExecution(s.ctx, candidate, "publisher-a"); !errors.Is(err, publication.ErrBundleLeaseLost) { + t.Fatalf("SaveExecution(%s) = %v, want ErrBundleLeaseLost", name, err) + } + } + if err := s.registry.SaveExecution(s.ctx, execution, ""); !errors.Is(err, publication.ErrBundleLeaseLost) { + t.Fatalf("SaveExecution(empty owner) = %v, want ErrBundleLeaseLost", err) + } + if err := s.registry.SaveExecution(s.ctx, execution, "publisher-a"); !errors.Is(err, publication.ErrBundleLeaseLost) { + t.Fatalf("SaveExecution(without lease) = %v, want ErrBundleLeaseLost", err) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(BundleExecutionsTable)); n != 0 { + t.Fatalf("executions stored after refused saves = %d, want 0", n) + } + + s.acquireLease(t, "bundle-key", "publisher-a", time.Now().UTC().Add(time.Minute)) + s.saveExecution(t, execution, "publisher-a") + newer := execution + newer.Error = "owned by publisher-b" + s.acquireLease(t, "bundle-key", "publisher-a", time.Now().UTC().Add(-time.Second)) + s.acquireLease(t, "bundle-key", "publisher-b", time.Now().UTC().Add(time.Hour)) + if err := s.registry.SaveExecution(s.ctx, newer, "publisher-a"); !errors.Is(err, publication.ErrBundleLeaseLost) { + t.Fatalf("SaveExecution after takeover = %v, want ErrBundleLeaseLost", err) + } + if payload := s.mustWitnessText(t, + "SELECT COALESCE(payload->>'error', '') FROM "+s.qualified(BundleExecutionsTable)+" WHERE id = $1", "exec-fence"); payload != "" { + t.Fatalf("stale owner mutated payload error = %q, want untouched", payload) + } + s.saveExecution(t, newer, "publisher-b") + if payload := s.mustWitnessText(t, + "SELECT COALESCE(payload->>'error', '') FROM "+s.qualified(BundleExecutionsTable)+" WHERE id = $1", "exec-fence"); payload != "owned by publisher-b" { + t.Fatalf("payload error = %q, want owned by publisher-b", payload) + } +} + +func TestLeaseLifecycleAndRepeatedOwnerCorrelation(t *testing.T) { + s := newCatalogService(t) + leaseTable := s.qualified(BundleLeasesTable) + epoch := func() int64 { + return s.mustWitnessInt64(t, "SELECT epoch FROM "+leaseTable+" WHERE key = $1", "bundle-l") + } + owner := func() string { + return s.mustWitnessText(t, "SELECT owner_id FROM "+leaseTable+" WHERE key = $1", "bundle-l") + } + + claimed, err := s.registry.AcquireBundleLease(s.ctx, "bundle-l", "owner-a", time.Now().UTC().Add(time.Hour)) + if err != nil || !claimed { + s.fail("first acquire", fmt.Errorf("claimed=%v err=%w", claimed, err)) + } + if epoch() != 1 || owner() != "owner-a" { + t.Fatalf("after first acquire: epoch=%d owner=%q, want 1 owner-a", epoch(), owner()) + } + + claimed, err = s.registry.AcquireBundleLease(s.ctx, "bundle-l", "owner-b", time.Now().UTC().Add(time.Hour)) + if err != nil || claimed { + t.Fatalf("acquire held lease = %v, %v; want false, nil", claimed, err) + } + if renewed, err := s.registry.RenewBundleLease(s.ctx, "bundle-l", "owner-b", time.Now().UTC().Add(2*time.Hour)); err != nil || renewed { + t.Fatalf("foreign renew = %v, %v; want false, nil", renewed, err) + } + if err := s.registry.ReleaseBundleLease(s.ctx, "bundle-l", "owner-b"); err != nil { + s.fail("foreign release", err) + } + if owner() != "owner-a" { + t.Fatalf("foreign release removed the fence: owner = %q", owner()) + } + if renewed, err := s.registry.RenewBundleLease(s.ctx, "bundle-l", "owner-a", time.Now().UTC().Add(2*time.Hour)); err != nil || !renewed { + t.Fatalf("owner renew = %v, %v; want true, nil", renewed, err) + } + + if err := s.registry.ReleaseBundleLease(s.ctx, "bundle-l", "owner-a"); err != nil { + s.fail("owner release", err) + } + if n := s.execCount(t, "SELECT count(*) FROM "+leaseTable+" WHERE key = $1", "bundle-l"); n != 0 { + t.Fatalf("lease rows after owner release = %d, want 0", n) + } + + s.acquireLease(t, "bundle-l", "owner-a", time.Now().UTC().Add(time.Hour)) + if epoch() != 1 { + t.Fatalf("re-grant after release epoch = %d, want 1", epoch()) + } + s.acquireLease(t, "bundle-l", "owner-a", time.Now().UTC().Add(-time.Minute)) + if epoch() != 2 || owner() != "owner-a" { + t.Fatalf("same-owner expired reclaim: epoch=%d owner=%q, want 2 owner-a", epoch(), owner()) + } + claimed, err = s.registry.AcquireBundleLease(s.ctx, "bundle-l", "owner-b", time.Now().UTC().Add(time.Hour)) + if err != nil || !claimed { + t.Fatalf("claim after expiry = %v, %v; want true, nil", claimed, err) + } + if epoch() != 3 || owner() != "owner-b" { + t.Fatalf("takeover after expiry: epoch=%d owner=%q, want 3 owner-b", epoch(), owner()) + } +} + +func TestPointerCompareAndSwapLiteralRows(t *testing.T) { + s := newCatalogService(t) + name := "5:alpha8:gen-15:panel2:v1" + key := pointerKey(name) + + if err := s.registry.CompareAndSwapPointer(s.ctx, name, "", "exec-1"); err != nil { + s.fail("initial CAS", err) + } + pointer, err := s.registry.GetPointer(s.ctx, name) + if err != nil { + s.fail("get pointer", err) + } + if pointer.Name != name || pointer.ExecutionID != "exec-1" { + t.Fatalf("pointer = %+v, want name %q exec-1", pointer, name) + } + + if err := s.registry.CompareAndSwapPointer(s.ctx, name, "exec-stale", "exec-2"); !errors.Is(err, publication.ErrBundlePointerConflict) { + t.Fatalf("stale CAS = %v, want ErrBundlePointerConflict", err) + } + if got := s.mustWitnessText(t, + "SELECT execution_id FROM "+s.qualified(BundlePointersTable)+" WHERE pointer_key = $1", key); got != "exec-1" { + t.Fatalf("execution_id after stale CAS = %q, want exec-1", got) + } + + if err := s.registry.CompareAndSwapPointer(s.ctx, name, "exec-1", "exec-2"); err != nil { + s.fail("owner CAS", err) + } + if got := s.mustWitnessText(t, + "SELECT execution_id FROM "+s.qualified(BundlePointersTable)+" WHERE pointer_key = $1", key); got != "exec-2" { + t.Fatalf("execution_id after owner CAS = %q, want exec-2", got) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(BundlePointersTable)+" WHERE pointer_key = $1", key); n != 1 { + t.Fatalf("pointer rows = %d, want 1", n) + } +} + +func TestPointerNamesWithNULRoundTripAndNoCollisionFold(t *testing.T) { + s := newCatalogService(t) + + foldA := publication.BundleIdentity{Project: "p", DatasetGeneration: "g", Name: "a\x00b"} + foldB := publication.BundleIdentity{Project: "p", DatasetGeneration: "g", Name: "a", TranslationVersion: "b"} + if foldA.PointerName() == foldB.PointerName() { + t.Fatal("folded identities share a PointerName; domain encoding regressed") + } + if foldA.Key() == foldB.Key() { + t.Fatal("folded identities share a Key; domain encoding regressed") + } + + nulName := "HTAN_INT-BForePC\x00\x00aced-meta-default" + if err := s.registry.CompareAndSwapPointer(s.ctx, nulName, "", "exec-nul"); err != nil { + s.fail("NUL-name CAS", err) + } + pointer, err := s.registry.GetPointer(s.ctx, nulName) + if err != nil { + s.fail("get NUL pointer", err) + } + if pointer.Name != nulName || pointer.ExecutionID != "exec-nul" { + t.Fatalf("NUL pointer = %+v, want exact byte round trip of %q", pointer, nulName) + } + if raw := s.mustWitnessText(t, + "SELECT encode(name_bytes, 'hex') FROM "+s.qualified(BundlePointersTable)+" WHERE pointer_key = $1", pointerKey(nulName)); raw != hex.EncodeToString([]byte(nulName)) { + t.Fatalf("stored name_bytes = %s, want %s", raw, hex.EncodeToString([]byte(nulName))) + } + + if err := s.registry.CompareAndSwapPointer(s.ctx, foldA.PointerName(), "", "exec-a"); err != nil { + s.fail("fold A CAS", err) + } + if err := s.registry.CompareAndSwapPointer(s.ctx, foldB.PointerName(), "", "exec-b"); err != nil { + s.fail("fold B CAS", err) + } + first, err := s.registry.GetPointer(s.ctx, foldA.PointerName()) + if err != nil || first.ExecutionID != "exec-a" { + t.Fatalf("fold A pointer = %+v, %v; want exec-a", first, err) + } + second, err := s.registry.GetPointer(s.ctx, foldB.PointerName()) + if err != nil || second.ExecutionID != "exec-b" { + t.Fatalf("fold B pointer = %+v, %v; want exec-b", second, err) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(BundlePointersTable)); n != 3 { + t.Fatalf("pointer rows = %d, want 3", n) + } +} + +func TestPublishExecutionCommitsPointerAndStateAtomically(t *testing.T) { + s := newCatalogService(t) + running := baseExecution("exec-pub", "bundle-pub", publication.BundleRunning, stamps(createdTime, updatedTime, readyTime, time.Time{})) + s.acquireLease(t, "bundle-pub", "publisher-a", time.Now().UTC().Add(time.Hour)) + s.saveExecution(t, running, "publisher-a") + + name := "5:alpha8:gen-15:panel2:v1" + published := running + published.OwnerID = "publisher-a" + published.State = publication.BundlePublished + published.UpdatedAt = pubTime + published.PublishedAt = &pubTime + if err := s.registry.PublishExecution(s.ctx, name, "", published); err != nil { + s.fail("publish", err) + } + + got, err := s.registry.GetExecution(s.ctx, "exec-pub") + if err != nil { + s.fail("get published", err) + } + if got.State != publication.BundlePublished || got.PublishedAt == nil || !got.PublishedAt.Equal(pubTime) { + t.Fatalf("published execution = state %q publishedAt %v, want PUBLISHED at %s", got.State, got.PublishedAt, pubTime) + } + pointer, err := s.registry.GetPointer(s.ctx, name) + if err != nil { + s.fail("get pointer", err) + } + if pointer.ExecutionID != "exec-pub" || !pointer.UpdatedAt.Equal(pubTime) { + t.Fatalf("pointer = %+v, want exec-pub updated at %s", pointer, pubTime) + } +} + +func TestPublishExecutionRollsBackEverythingOnConflict(t *testing.T) { + s := newCatalogService(t) + name := "5:alpha8:gen-15:panel2:v1" + + expired := baseExecution("exec-exp", "bundle-exp", publication.BundleRunning, stamps(createdTime, updatedTime, readyTime, time.Time{})) + s.acquireLease(t, "bundle-exp", "publisher-a", time.Now().UTC().Add(time.Hour)) + s.saveExecution(t, expired, "publisher-a") + s.acquireLease(t, "bundle-exp", "publisher-a", time.Now().UTC().Add(-time.Second)) + published := expired + published.OwnerID = "publisher-a" + published.State = publication.BundlePublished + if err := s.registry.PublishExecution(s.ctx, name, "", published); !errors.Is(err, publication.ErrBundlePointerConflict) { + t.Fatalf("publish under expired lease = %v, want ErrBundlePointerConflict", err) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(BundlePointersTable)); n != 0 { + t.Fatalf("pointer rows after expired-fence publish = %d, want 0", n) + } + if state := s.mustWitnessText(t, + "SELECT payload->>'state' FROM "+s.qualified(BundleExecutionsTable)+" WHERE id = $1", "exec-exp"); state != "RUNNING" { + t.Fatalf("execution state after refused publish = %q, want RUNNING (rolled back)", state) + } + + foreign := baseExecution("exec-foreign", "bundle-foreign", publication.BundleRunning, stamps(createdTime, updatedTime, readyTime, time.Time{})) + s.acquireLease(t, "bundle-foreign", "publisher-a", time.Now().UTC().Add(time.Hour)) + s.saveExecution(t, foreign, "publisher-a") + foreignPublished := foreign + foreignPublished.OwnerID = "publisher-b" + if err := s.registry.PublishExecution(s.ctx, name, "", foreignPublished); !errors.Is(err, publication.ErrBundlePointerConflict) { + t.Fatalf("publish by non-owner = %v, want ErrBundlePointerConflict", err) + } + + missing := foreignPublished + missing.OwnerID = "publisher-a" + missing.ID = "exec-never-saved" + if err := s.registry.PublishExecution(s.ctx, name, "", missing); !errors.Is(err, publication.ErrBundlePointerConflict) { + t.Fatalf("publish missing execution = %v, want ErrBundlePointerConflict", err) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(BundlePointersTable)); n != 0 { + t.Fatalf("pointer rows after refused publishes = %d, want 0", n) + } + + ok := foreign + ok.OwnerID = "publisher-a" + ok.State = publication.BundlePublished + ok.PublishedAt = &pubTime + if err := s.registry.PublishExecution(s.ctx, name, "", ok); err != nil { + s.fail("baseline publish", err) + } + stale := ok + stale.ID = "exec-stale" + s.acquireLease(t, "bundle-stale", "publisher-a", time.Now().UTC().Add(time.Hour)) + s.saveExecution(t, stale, "publisher-a") + stale.OwnerID = "publisher-a" + if err := s.registry.PublishExecution(s.ctx, name, "exec-never-current", stale); !errors.Is(err, publication.ErrBundlePointerConflict) { + t.Fatalf("publish with stale expected = %v, want ErrBundlePointerConflict", err) + } + pointer, err := s.registry.GetPointer(s.ctx, name) + if err != nil || pointer.ExecutionID != "exec-foreign" { + t.Fatalf("pointer after stale publish = %+v, %v; want exec-foreign", pointer, err) + } +} + +func TestPublishExecutionConcurrentCASOneWinner(t *testing.T) { + s := newCatalogService(t) + name := "5:alpha8:gen-15:panel2:v1" + key := "bundle-race" + candidateA := baseExecution("exec-race-a", key, publication.BundleRunning, stamps(createdTime, updatedTime, readyTime, time.Time{})) + candidateB := baseExecution("exec-race-b", key, publication.BundleRunning, stamps(createdTime, updatedTime, readyTime, time.Time{})) + s.acquireLease(t, key, "publisher-a", time.Now().UTC().Add(time.Hour)) + s.saveExecution(t, candidateA, "publisher-a") + s.saveExecution(t, candidateB, "publisher-a") + for _, e := range []*publication.BundleExecution{&candidateA, &candidateB} { + e.OwnerID = "publisher-a" + e.State = publication.BundlePublished + e.PublishedAt = &pubTime + } + + start := make(chan struct{}) + var wg sync.WaitGroup + var mu sync.Mutex + outcomes := map[string]error{} + for _, execution := range []publication.BundleExecution{candidateA, candidateB} { + wg.Add(1) + go func(execution publication.BundleExecution) { + defer wg.Done() + <-start + err := s.registry.PublishExecution(s.ctx, name, "", execution) + mu.Lock() + outcomes[execution.ID] = err + mu.Unlock() + }(execution) + } + close(start) + wg.Wait() + + winners := []string{} + for id, err := range outcomes { + if len(outcomes) != 2 { + t.Fatalf("outcomes = %d, want 2", len(outcomes)) + } + if err == nil { + winners = append(winners, id) + continue + } + if !errors.Is(err, publication.ErrBundlePointerConflict) { + t.Fatalf("loser outcome for %s = %v, want ErrBundlePointerConflict", id, err) + } + } + if len(winners) != 1 { + t.Fatalf("winners = %#v, want exactly one of exec-race-a/exec-race-b", winners) + } + pointer, err := s.registry.GetPointer(s.ctx, name) + if err != nil { + s.fail("get race pointer", err) + } + if pointer.ExecutionID != winners[0] { + t.Fatalf("pointer names %q, want the winner %q", pointer.ExecutionID, winners[0]) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(BundlePointersTable)); n != 1 { + t.Fatalf("pointer rows after race = %d, want 1", n) + } +} + +func TestExactLookupIsFullyScoped(t *testing.T) { + s := newCatalogService(t) + selector := publication.DataframeSelector{Recipe: "panel", TranslationVersion: "v1", Output: "out-a"} + + newer := baseExecution("exec-exact-new", "bundle-new", publication.BundlePublished, stamps(createdTime, updatedTime, readyTime, pubTime)) + older := baseExecution("exec-exact-old", "bundle-old", publication.BundlePublished, stamps(createdTime.Add(-time.Hour), updatedTime.Add(-time.Hour), readyTime.Add(-time.Hour), pubTime.Add(-time.Hour))) + sibling := baseExecution("exec-exact-beta", "bundle-beta", publication.BundlePublished, stamps(createdTime, updatedTime, readyTime, pubTime)) + sibling.Project = "beta" + otherGeneration := baseExecution("exec-exact-gen2", "bundle-gen2", publication.BundlePublished, stamps(createdTime, updatedTime, readyTime, pubTime)) + otherGeneration.DatasetGeneration = "gen-2" + otherTranslation := baseExecution("exec-exact-v2", "bundle-v2", publication.BundlePublished, stamps(createdTime, updatedTime, readyTime, pubTime)) + otherTranslation.TranslationVersion = "v2" + runningTwin := baseExecution("exec-exact-run", "bundle-run", publication.BundleRunning, stamps(createdTime.Add(time.Hour), updatedTime.Add(time.Hour), readyTime.Add(time.Hour), time.Time{})) + runningTwin.Outputs = append(runningTwin.Outputs, publication.BundleOutputRecord{ + Name: "out-run", PhysicalTable: "phys_alpha_out_run", + Selector: runningTwin.Selector("out-run"), RowCount: 42, State: publication.BundleRunning, + }) + + for _, execution := range []publication.BundleExecution{newer, older, sibling, otherGeneration, otherTranslation, runningTwin} { + keyOwner := execution.Key + s.acquireLease(t, keyOwner, "publisher-a", time.Now().UTC().Add(time.Hour)) + s.saveExecution(t, execution, "publisher-a") + } + + execution, output, err := s.registry.FindExecutionBySelector(s.ctx, "alpha", "gen-1", selector) + if err != nil { + s.fail("exact lookup", err) + } + if execution.ID != "exec-exact-new" { + t.Fatalf("exact lookup returned %q, want exec-exact-new (newest published_at)", execution.ID) + } + wantOutput := publication.BundleOutputRecord{ + Name: "out-a", PhysicalTable: "phys_alpha_out_a", + Selector: publication.DataframeSelector{Recipe: "panel", TranslationVersion: "v1", Output: "out-a"}, + Columns: newer.Outputs[0].Columns, + RowCount: 1234, ByteCount: 567890, State: publication.BundlePublished, + VerifiedAt: newer.Outputs[0].VerifiedAt, + SourceRow: &publication.SourceRowMetadata{ResourceType: "Patient", IDColumn: "subject"}, + } + if gotJSON, wantJSON := mustMarshal(t, output), mustMarshal(t, wantOutput); gotJSON != wantJSON { + t.Fatalf("exact output =\n %s\nwant\n %s", gotJSON, wantJSON) + } + + type hit struct { + project string + generation string + selector publication.DataframeSelector + wantID string + } + for _, h := range []hit{ + {"beta", "gen-1", selector, "exec-exact-beta"}, + {"alpha", "gen-2", selector, "exec-exact-gen2"}, + {"alpha", "gen-1", publication.DataframeSelector{Recipe: "panel", TranslationVersion: "v2", Output: "out-a"}, "exec-exact-v2"}, + {"alpha", "gen-1", publication.DataframeSelector{Recipe: "panel", TranslationVersion: "v1", Output: "out-b"}, "exec-exact-new"}, + } { + got, _, err := s.registry.FindExecutionBySelector(s.ctx, h.project, h.generation, h.selector) + if err != nil { + t.Fatalf("exact hit %s: %v", h.wantID, err) + } + if got.ID != h.wantID { + t.Fatalf("exact lookup (%s,%s,%+v) = %q, want %q", h.project, h.generation, h.selector, got.ID, h.wantID) + } + } + + type miss struct { + name string + project string + generation string + selector publication.DataframeSelector + } + misses := []miss{ + {"unknown project", "gamma", "gen-1", selector}, + {"swapped scope pair", "beta", "gen-2", selector}, + {"unknown generation", "alpha", "gen-9", selector}, + {"unknown translation version", "alpha", "gen-1", publication.DataframeSelector{Recipe: "panel", TranslationVersion: "v9", Output: "out-a"}}, + {"unknown output", "alpha", "gen-1", publication.DataframeSelector{Recipe: "panel", TranslationVersion: "v1", Output: "out-z"}}, + {"case-sensitive recipe", "alpha", "gen-1", publication.DataframeSelector{Recipe: "Panel", TranslationVersion: "v1", Output: "out-a"}}, + } + for _, m := range misses { + if _, _, err := s.registry.FindExecutionBySelector(s.ctx, m.project, m.generation, m.selector); !errors.Is(err, publication.ErrBundleNotFound) { + t.Fatalf("%s: FindExecutionBySelector = %v, want ErrBundleNotFound", m.name, err) + } + } + + runSelector := publication.DataframeSelector{Recipe: "panel", TranslationVersion: "v1", Output: "out-run"} + if _, _, err := s.registry.FindExecutionBySelector(s.ctx, "alpha", "gen-1", runSelector); !errors.Is(err, publication.ErrBundleNotFound) { + t.Fatalf("running twin visible before publish: %v", err) + } + publishedRun := runningTwin + publishedRun.OwnerID = "publisher-a" + publishedRun.State = publication.BundlePublished + olderPublished := pubTime.Add(-time.Hour) + publishedRun.PublishedAt = &olderPublished + for i := range publishedRun.Outputs { + publishedRun.Outputs[i].State = publication.BundlePublished + publishedRun.Outputs[i].VerifiedAt = &olderPublished + } + if err := s.registry.PublishExecution(s.ctx, "pointer-run", "", publishedRun); err != nil { + s.fail("publish run twin", err) + } + if execution, output, err := s.registry.FindExecutionBySelector(s.ctx, "alpha", "gen-1", runSelector); err != nil || + execution.ID != "exec-exact-run" || output.RowCount != 42 || output.PhysicalTable != "phys_alpha_out_run" { + t.Fatalf("run twin after publish = %q/%+v, %v; want exec-exact-run with out-run metadata", execution.ID, output, err) + } + if execution, _, err := s.registry.FindExecutionBySelector(s.ctx, "alpha", "gen-1", selector); err != nil || execution.ID != "exec-exact-new" { + t.Fatalf("newest after run twin published = %q, %v; want exec-exact-new", execution.ID, err) + } +} + +func TestExactLookupOrdersNullStampsDeterministically(t *testing.T) { + s := newCatalogService(t) + selector := publication.DataframeSelector{Recipe: "panel", TranslationVersion: "v1", Output: "out-a"} + + first := baseExecution("exec-null-early", "bundle-null-1", publication.BundlePublished, stamps(createdTime, updatedTime, time.Time{}, time.Time{})) + second := baseExecution("exec-null-late", "bundle-null-2", publication.BundlePublished, stamps(createdTime, updatedTime.Add(time.Minute), time.Time{}, time.Time{})) + for _, execution := range []publication.BundleExecution{first, second} { + s.acquireLease(t, execution.Key, "publisher-a", time.Now().UTC().Add(time.Hour)) + s.saveExecution(t, execution, "publisher-a") + } + execution, _, err := s.registry.FindExecutionBySelector(s.ctx, "alpha", "gen-1", selector) + if err != nil { + s.fail("null-stamp lookup", err) + } + if execution.ID != "exec-null-late" { + t.Fatalf("null-stamp lookup = %q, want exec-null-late", execution.ID) + } +} + +func TestExactLookupRejectsInvalidSelector(t *testing.T) { + s := newCatalogService(t) + for name, selector := range map[string]publication.DataframeSelector{ + "blank recipe": {Recipe: " ", TranslationVersion: "v1", Output: "out-a"}, + "padded translation": {Recipe: "panel", TranslationVersion: " v1", Output: "out-a"}, + "blank output": {Recipe: "panel", TranslationVersion: "v1", Output: ""}, + } { + _, _, err := s.registry.FindExecutionBySelector(s.ctx, "alpha", "gen-1", selector) + if err == nil || !strings.Contains(err.Error(), "is required and must not have surrounding whitespace") { + t.Fatalf("%s: FindExecutionBySelector = %v, want the selector validation error", name, err) + } + } +} + +const legacyQueuedPayload = `{"id":"legacy-queued","key":"legacy-key","name":"legacy-recipe","translationVersion":"v9","project":"p","datasetGeneration":"g","state":"PENDING","outputs":[{"name":"out","physicalTable":"phys_legacy","state":"LOADING","rowCount":3}],"createdAt":"2026-11-09T10:00:00Z","updatedAt":"2026-11-09T10:05:00Z","readyAt":"2026-11-09T10:06:00Z"}` + +const legacyReadyPayload = `{"id":"legacy-ready","key":"legacy-key","name":"legacy-recipe","translationVersion":"v9","project":"p","datasetGeneration":"g","state":"READY","outputs":[{"name":"out","physicalTable":"phys_legacy","state":"READY","rowCount":3}],"createdAt":"2026-11-09T10:01:00Z","updatedAt":"2026-11-09T10:07:00Z","readyAt":"2026-11-09T10:08:00Z"}` + +func insertLegacyRow(t *testing.T, s *catalogService, id, key, recipe, tv, state, payload string, created, updated time.Time, ready *time.Time) { + t.Helper() + s.mustExec(t, + "INSERT INTO "+s.qualified(BundleExecutionsTable)+ + " (id, execution_key, project, dataset_generation, recipe_name, translation_version,"+ + " state, created_at, updated_at, ready_at, published_at, payload)"+ + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", + id, key, "p", "g", recipe, tv, state, created, updated, ready, nil, payload) +} + +func TestLegacyCanonicalizationOnReadDoesNotMutateHistory(t *testing.T) { + s := newCatalogService(t) + legacyCreated := time.Date(2026, 11, 9, 10, 0, 0, 0, time.UTC) + legacyUpdatedQueued := time.Date(2026, 11, 9, 10, 5, 0, 0, time.UTC) + legacyReadyQueued := time.Date(2026, 11, 9, 10, 6, 0, 0, time.UTC) + legacyUpdatedReady := time.Date(2026, 11, 9, 10, 7, 0, 0, time.UTC) + legacyReadyReady := time.Date(2026, 11, 9, 10, 8, 0, 0, time.UTC) + insertLegacyRow(t, s, "legacy-queued", "legacy-key", "legacy-recipe", "v9", "PENDING", legacyQueuedPayload, legacyCreated, legacyUpdatedQueued, &legacyReadyQueued) + insertLegacyRow(t, s, "legacy-ready", "legacy-key", "legacy-recipe", "v9", "READY", legacyReadyPayload, legacyCreated.Add(time.Minute), legacyUpdatedReady, &legacyReadyReady) + + queued, err := s.registry.GetExecution(s.ctx, "legacy-queued") + if err != nil { + s.fail("get legacy queued", err) + } + if queued.State != publication.BundleQueued || queued.Outputs[0].State != publication.BundleRunning { + t.Fatalf("legacy queued states = %q/%q, want QUEUED/RUNNING", queued.State, queued.Outputs[0].State) + } + wantSelector := publication.DataframeSelector{Recipe: "legacy-recipe", TranslationVersion: "v9", Output: "out"} + if queued.Outputs[0].Selector != wantSelector { + t.Fatalf("legacy derived selector = %+v, want %+v", queued.Outputs[0].Selector, wantSelector) + } + if queued.Outputs[0].VerifiedAt != nil || queued.PublishedAt != nil { + t.Fatalf("legacy queued invented stamps: verified=%v published=%v, want nil", queued.Outputs[0].VerifiedAt, queued.PublishedAt) + } + + readyExec, err := s.registry.GetExecution(s.ctx, "legacy-ready") + if err != nil { + s.fail("get legacy ready", err) + } + if readyExec.State != publication.BundlePublished || readyExec.PublishedAt == nil || !readyExec.PublishedAt.Equal(legacyReadyReady) { + t.Fatalf("legacy ready = %q publishedAt %v, want PUBLISHED at %s", readyExec.State, readyExec.PublishedAt, legacyReadyReady) + } + if readyExec.Outputs[0].VerifiedAt == nil || !readyExec.Outputs[0].VerifiedAt.Equal(legacyReadyReady) { + t.Fatalf("legacy ready verifiedAt = %v, want %s", readyExec.Outputs[0].VerifiedAt, legacyReadyReady) + } + + list, err := s.registry.ListExecutions(s.ctx, publication.BundleQueued, baseTime) + if err != nil { + s.fail("list queued", err) + } + if len(list) != 1 || list[0].ID != "legacy-queued" { + t.Fatalf("queued list = %+v, want only legacy-queued", list) + } + if _, err := s.registry.ListExecutions(s.ctx, publication.BundlePending, baseTime); err != nil { + s.fail("list by compat state", err) + } + + byKey, err := s.registry.FindExecutionByKey(s.ctx, "legacy-key") + if err != nil { + s.fail("find by key", err) + } + if byKey.ID != "legacy-ready" { + t.Fatalf("FindExecutionByKey = %q, want legacy-ready (newer created_at)", byKey.ID) + } + + if execution, _, err := s.registry.FindExecutionBySelector(s.ctx, "p", "g", + publication.DataframeSelector{Recipe: "legacy-recipe", TranslationVersion: "v9", Output: "out"}); err != nil || execution.ID != "legacy-ready" { + t.Fatalf("legacy exact lookup = %q, %v; want legacy-ready", execution.ID, err) + } + + visited := []string{} + if err := s.registry.VisitExecutionPages(s.ctx, publication.BundleQueued, baseTime, 10, func(page []publication.BundleExecution) error { + for _, execution := range page { + visited = append(visited, execution.ID) + } + return nil + }); err != nil { + s.fail("visit queued", err) + } + if len(visited) != 1 || visited[0] != "legacy-queued" { + t.Fatalf("visited queued executions = %#v, want [legacy-queued] via the PENDING synonym", visited) + } + + for id, wantPayload := range map[string]string{"legacy-queued": legacyQueuedPayload, "legacy-ready": legacyReadyPayload} { + if !s.mustWitnessBool(t, + "SELECT payload = $2::jsonb FROM "+s.qualified(BundleExecutionsTable)+" WHERE id = $1", + id, wantPayload) { + got := s.mustWitnessText(t, + "SELECT payload::text FROM "+s.qualified(BundleExecutionsTable)+" WHERE id = $1", id) + t.Fatalf("payload for %s changed value under reads:\n got %s\nwant %s", id, got, wantPayload) + } + } + if s.mustWitnessBool(t, + "SELECT (payload ? 'publishedAt') OR (payload->'outputs'->0 ? 'verifiedAt') FROM "+ + s.qualified(BundleExecutionsTable)+" WHERE id = $1", "legacy-queued") { + t.Fatal("read canonicalization fabricated publishedAt/verifiedAt into the stored legacy row") + } + if state := s.mustWitnessText(t, + "SELECT state FROM "+s.qualified(BundleExecutionsTable)+" WHERE id = $1", "legacy-queued"); state != "PENDING" { + t.Fatalf("legacy state column = %q, want stored PENDING", state) + } +} + +func TestMissingJSONNullAndSQLNullStayDistinct(t *testing.T) { + s := newCatalogService(t) + + saved := baseExecution("exec-nulls", "bundle-nulls", publication.BundleRunning, stamps(createdTime, updatedTime, time.Time{}, time.Time{})) + saved.Outputs = nil + s.acquireLease(t, "bundle-nulls", "publisher-a", time.Now().UTC().Add(time.Hour)) + s.saveExecution(t, saved, "publisher-a") + if s.mustWitnessBool(t, + "SELECT ready_at IS NULL AND published_at IS NULL FROM "+s.qualified(BundleExecutionsTable)+" WHERE id = $1", "exec-nulls") { + } else { + t.Fatal("nil stamps must land as SQL NULL columns") + } + if s.mustWitnessBool(t, + "SELECT NOT (payload ? 'readyAt') AND NOT (payload ? 'publishedAt') FROM "+s.qualified(BundleExecutionsTable)+" WHERE id = $1", "exec-nulls") { + } else { + t.Fatal(`saved payload must omit absent stamp fields (no "readyAt":null)`) + } + + jsonNullPayload := `{"id":"legacy-jsonnull","key":"legacy-jsonnull","name":"legacy-recipe","translationVersion":"v9","project":"p","datasetGeneration":"g","state":"QUEUED","outputs":[],"createdAt":"2026-11-09T10:00:00Z","updatedAt":"2026-11-09T10:05:00Z","readyAt":null,"publishedAt":null}` + insertLegacyRow(t, s, "legacy-jsonnull", "legacy-jsonnull", "legacy-recipe", "v9", "QUEUED", jsonNullPayload, + time.Date(2026, 11, 9, 10, 0, 0, 0, time.UTC), time.Date(2026, 11, 9, 10, 5, 0, 0, time.UTC), nil) + if s.mustWitnessBool(t, + "SELECT ready_at IS NULL AND published_at IS NULL"+ + " AND jsonb_typeof(payload->'readyAt') = 'null' AND jsonb_typeof(payload->'publishedAt') = 'null'"+ + " AND (payload ? 'readyAt') AND (payload ? 'publishedAt') FROM "+ + s.qualified(BundleExecutionsTable)+" WHERE id = $1", "legacy-jsonnull") { + } else { + t.Fatal(`explicit JSON null must survive as JSON null with SQL NULL columns`) + } + execution, err := s.registry.GetExecution(s.ctx, "legacy-jsonnull") + if err != nil { + s.fail("get json-null row", err) + } + if execution.ReadyAt != nil || execution.PublishedAt != nil { + t.Fatalf("JSON null decoded as %v/%v, want nil pointers", execution.ReadyAt, execution.PublishedAt) + } + if !s.mustWitnessBool(t, + "SELECT payload = $2::jsonb FROM "+s.qualified(BundleExecutionsTable)+" WHERE id = $1", + "legacy-jsonnull", jsonNullPayload) { + got := s.mustWitnessText(t, + "SELECT payload::text FROM "+s.qualified(BundleExecutionsTable)+" WHERE id = $1", "legacy-jsonnull") + t.Fatalf("JSON null payload mutated value:\n got %s\nwant %s", got, jsonNullPayload) + } +} + +func TestNULInTextScopeColumnsFailsExplicitly(t *testing.T) { + s := newCatalogService(t) + execution := baseExecution("exec-nul-project", "bundle-nul", publication.BundleRunning, stamps(createdTime, updatedTime, time.Time{}, time.Time{})) + execution.Project = "al\x00pha" + s.acquireLease(t, "bundle-nul", "publisher-a", time.Now().UTC().Add(time.Hour)) + + err := s.registry.SaveExecution(s.ctx, execution, "publisher-a") + if err == nil { + t.Fatal("SaveExecution with NUL in a text column succeeded; PostgreSQL text cannot hold NUL and the registry must surface that") + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "22021" { + t.Fatalf("SaveExecution error = %v, want explicit 22021 invalid_text_representation (no silent fallback)", err) + } + if n := s.execCount(t, "SELECT count(*) FROM "+s.qualified(BundleExecutionsTable)+" WHERE id = $1", "exec-nul-project"); n != 0 { + t.Fatalf("rejected save left %d rows, want 0", n) + } +} + +func TestReadsAndWritesOnMissingRowsReportNotFound(t *testing.T) { + s := newCatalogService(t) + if _, err := s.registry.GetExecution(s.ctx, "no-such-execution"); !errors.Is(err, publication.ErrBundleNotFound) { + t.Fatalf("GetExecution(missing) = %v, want ErrBundleNotFound", err) + } + if _, err := s.registry.FindExecutionByKey(s.ctx, "no-such-key"); !errors.Is(err, publication.ErrBundleNotFound) { + t.Fatalf("FindExecutionByKey(missing) = %v, want ErrBundleNotFound", err) + } + if _, err := s.registry.GetPointer(s.ctx, "no-such-pointer"); !errors.Is(err, publication.ErrBundleNotFound) { + t.Fatalf("GetPointer(missing) = %v, want ErrBundleNotFound", err) + } +} + +func TestListExecutionsEmptyIsNonNilSlice(t *testing.T) { + s := newCatalogService(t) + list, err := s.registry.ListExecutions(s.ctx, publication.BundlePublished, time.Now().UTC().Add(time.Hour)) + if err != nil { + s.fail("list empty", err) + } + if list == nil || len(list) != 0 { + t.Fatalf("empty ListExecutions = %#v, want non-nil empty slice (Arango parity)", list) + } +} + +func TestVisitExecutionPagesStableContinuationAndGuards(t *testing.T) { + s := newCatalogService(t) + type row struct { + id string + updated time.Time + } + rows := []row{ + {"exec-page-e", baseTime.Add(4 * time.Minute)}, + {"exec-page-a", baseTime.Add(5 * time.Minute)}, + {"exec-page-c", baseTime.Add(6 * time.Minute)}, + {"exec-page-b", baseTime.Add(7 * time.Minute)}, + {"exec-page-d", baseTime.Add(8 * time.Minute)}, + } + for _, r := range rows { + execution := baseExecution(r.id, "bundle-page", publication.BundleRunning, stamps(createdTime, r.updated, time.Time{}, time.Time{})) + execution.Outputs = nil + s.acquireLease(t, "bundle-page", "publisher-a", time.Now().UTC().Add(time.Hour)) + s.saveExecution(t, execution, "publisher-a") + } + before := baseTime.Add(time.Hour) + + var pages [][]string + if err := s.registry.VisitExecutionPages(s.ctx, publication.BundleRunning, before, 2, func(page []publication.BundleExecution) error { + ids := []string{} + for _, execution := range page { + ids = append(ids, execution.ID) + } + pages = append(pages, ids) + return nil + }); err != nil { + s.fail("visit pages", err) + } + wantPages := [][]string{ + {"exec-page-e", "exec-page-a"}, + {"exec-page-c", "exec-page-b"}, + {"exec-page-d"}, + } + if !reflect.DeepEqual(pages, wantPages) { + t.Fatalf("pages = %#v, want %#v", pages, wantPages) + } + + count := 0 + if err := s.registry.VisitExecutionPages(s.ctx, publication.BundleRunning, rows[0].updated, 2, func(page []publication.BundleExecution) error { + count += len(page) + return nil + }); err != nil { + s.fail("visit before cutoff", err) + } + if count != 0 { + t.Fatalf("rows before cutoff = %d, want 0 (all updates are later)", count) + } + + visits := 0 + stopErr := errors.New("stop the sweep") + if err := s.registry.VisitExecutionPages(s.ctx, publication.BundleRunning, before, 2, func(page []publication.BundleExecution) error { + visits++ + return stopErr + }); !errors.Is(err, stopErr) { + t.Fatalf("VisitExecutionPages visitor error = %v, want the visitor error itself", err) + } + if visits != 1 { + t.Fatalf("visitor calls after error = %d, want 1 (stop on first error)", visits) + } + + if err := s.registry.VisitExecutionPages(s.ctx, publication.BundleRunning, before, 0, func([]publication.BundleExecution) error { return nil }); err == nil || !strings.Contains(err.Error(), "positive") { + t.Fatalf("VisitExecutionPages(pageSize 0) = %v, want positive-size error", err) + } + if err := s.registry.VisitExecutionPages(s.ctx, publication.BundleRunning, before, 2, nil); err == nil || !strings.Contains(err.Error(), "visitor is required") { + t.Fatalf("VisitExecutionPages(nil visitor) = %v, want required-visitor error", err) + } +} + +func TestMain(m *testing.M) { + code := m.Run() + if err := auditZeroLeftovers(); err != nil { + fmt.Fprintf(os.Stderr, "zero-leftover audit failed: %v\n", err) + code = 1 + } + if err := auditPublishVisibilityLeftovers(); err != nil { + fmt.Fprintf(os.Stderr, "publish-visibility zero-leftover audit failed: %v\n", err) + code = 1 + } + if err := auditReadPinsLeftovers(); err != nil { + fmt.Fprintf(os.Stderr, "read-pins zero-leftover audit failed: %v\n", err) + code = 1 + } + os.Exit(code) +} + +func auditZeroLeftovers() error { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + return fmt.Errorf("connect: %v", err) + } + defer admin.Close() + var databases int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'pgpubcat01_it_%'").Scan(&databases); err != nil { + return fmt.Errorf("count databases: %v", err) + } + if databases != 0 { + return fmt.Errorf("%d pgpubcat01_it_* databases still exist", databases) + } + var schemas int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgpubcat01_it_%'").Scan(&schemas); err != nil { + return fmt.Errorf("count schemas: %v", err) + } + if schemas != 0 { + return fmt.Errorf("%d pgpubcat01_it_* schemas still exist in the service database", schemas) + } + return nil +} diff --git a/internal/backends/postgres/internal/recipeexec/migrations/000001_ledger.sql b/internal/backends/postgres/internal/recipeexec/migrations/000001_ledger.sql new file mode 100644 index 000000000..8f12e38d9 --- /dev/null +++ b/internal/backends/postgres/internal/recipeexec/migrations/000001_ledger.sql @@ -0,0 +1,26 @@ +-- pg-state-recipe-registries-v1: create the per-schema migration ledger. +-- +-- This mirrors the pg-store-02 ledger contract (internal/store/postgres/ +-- migrations/000001_ledger.sql): version, name, and checksum only, never the +-- SQL body, with the same CHECK shapes. The MigrationRunner reads this table +-- from the target schema, so a schema whose history starts here bootstraps +-- its own ledger audited by the same runner. Unlike the pg-store-02 copy this +-- body is written CREATE TABLE IF NOT EXISTS so a crash in the narrow +-- commit-before-record window converges on retry, as the runner documentation +-- requires of feature migrations. It never drops or truncates anything. +-- +-- Both recipe-registries package histories (recipe/exec/postgres and +-- recipe/postgres) embed this file byte-identically, so the version-1 +-- checksum agrees everywhere and one control-plane schema can be migrated by +-- the composed history {1 ledger, 2 recipe_exec_registries, 3 +-- recipe_revision_chains} in version order. +CREATE TABLE IF NOT EXISTS schema_migrations ( + version bigint NOT NULL, + name text NOT NULL, + checksum text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT schema_migrations_pkey PRIMARY KEY (version), + CONSTRAINT schema_migrations_version_positive CHECK (version > 0), + CONSTRAINT schema_migrations_name_shape CHECK (name ~ '^[a-z][a-z0-9_]*$'), + CONSTRAINT schema_migrations_checksum_hex CHECK (checksum ~ '^[0-9a-f]{64}$') +); diff --git a/internal/backends/postgres/internal/recipeexec/migrations/000002_recipe_exec_registries.sql b/internal/backends/postgres/internal/recipeexec/migrations/000002_recipe_exec_registries.sql new file mode 100644 index 000000000..841c8a5c3 --- /dev/null +++ b/internal/backends/postgres/internal/recipeexec/migrations/000002_recipe_exec_registries.sql @@ -0,0 +1,75 @@ +-- pg-state-recipe-registries-v1: immutable name/version recipe registrations +-- and the fenced execution-occurrence usage markers for the PostgreSQL +-- recipe-execution control plane. +-- +-- The registration table mirrors the Arango collection loom_dataframe_recipes +-- (internal/dataframe/recipe/exec/arango/registry.go): one row per +-- (name, translationVersion), the canonical Bundle plus digest in a jsonb +-- payload, and identity columns split out so lookup is a scope equality -- +-- never a name-only "latest" guess beyond the contract's LoadRecipe current +-- read. Identity columns carry COLLATE "C" so uniqueness and LoadRecipe's +-- translationVersion DESC "current" order use exact byte order -- the same +-- deterministic order Arango's byte-order SORT gives -- independent of the +-- database's default collation. +-- +-- Usage marking mirrors the Arango usage probe against +-- loom_dataframe_bundle_executions, but as a first-class occurrence table in +-- this history: the PostgreSQL control plane forbids cross-database +-- atomicity, so an execution occurrence and the version it consumes are +-- fenced through one row lock on the recipe row (see the package +-- documentation). Each durable use is one occurrence row keyed by its own +-- occurrence identity -- retries of one occurrence idempotently reconverge +-- on the same row -- and repeated occurrences by one owner correlate through +-- owner_sequence, assigned under the recipe-row lock so witnesses can order +-- them. +-- +-- Column policy follows the sibling control-plane tickets: +-- * Go strings have no missing/empty distinction, so scope columns are +-- NOT NULL with non-empty CHECKs; empty identity never round-trips. +-- * The payload is authoritative for the exec.Entry shape (Bundle + +-- Digest) and jsonb preserves JSON null versus absent members; the +-- digest column carries the CHECK that it is a lowercase-hex SHA-256 so +-- a garbage digest cannot key a version. +-- * This migration never drops or truncates: rerunning bootstrap on an +-- existing schema is a no-op that preserves registered state, as the +-- runner's commit-before-record retry requires. +CREATE TABLE IF NOT EXISTS recipe_exec_recipes ( + recipe_name text NOT NULL COLLATE "C", + translation_version text NOT NULL COLLATE "C", + digest text NOT NULL, + payload jsonb NOT NULL, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT recipe_exec_recipes_pkey PRIMARY KEY (recipe_name, translation_version), + CONSTRAINT recipe_exec_recipes_name_nonempty CHECK (recipe_name <> ''), + CONSTRAINT recipe_exec_recipes_version_nonempty CHECK (translation_version <> ''), + CONSTRAINT recipe_exec_recipes_digest_hex CHECK (digest ~ '^[0-9a-f]{64}$'), + CONSTRAINT recipe_exec_recipes_payload_object CHECK (jsonb_typeof(payload) = 'object') +); + +-- No extra index for LoadRecipe's "current" read: the primary key already +-- orders (recipe_name, translation_version), so the LIMIT 1 highest-version +-- read is a backward index scan of recipe_exec_recipes_pkey. + +CREATE TABLE IF NOT EXISTS recipe_exec_occurrences ( + recipe_name text NOT NULL COLLATE "C", + translation_version text NOT NULL COLLATE "C", + occurrence_id text NOT NULL COLLATE "C", + owner_id text NOT NULL DEFAULT '', + -- Per-(version, owner) ordinal assigned under the recipe-row lock: + -- repeated occurrences from one owner on one version correlate through + -- it, and it is strictly increasing in recording order. + owner_sequence bigint NOT NULL, + recorded_at timestamptz NOT NULL, + CONSTRAINT recipe_exec_occurrences_pkey + PRIMARY KEY (recipe_name, translation_version, occurrence_id), + CONSTRAINT recipe_exec_occurrences_name_nonempty CHECK (recipe_name <> ''), + CONSTRAINT recipe_exec_occurrences_version_nonempty CHECK (translation_version <> ''), + CONSTRAINT recipe_exec_occurrences_occurrence_nonempty CHECK (occurrence_id <> ''), + CONSTRAINT recipe_exec_occurrences_owner_sequence_positive + CHECK (owner_sequence >= 1) +); + +-- Repeated-owner correlation reads: MAX(owner_sequence) per (version, owner). +CREATE INDEX IF NOT EXISTS recipe_exec_occurrences_owner_sequence + ON recipe_exec_occurrences (recipe_name, translation_version, owner_id, owner_sequence DESC); diff --git a/internal/backends/postgres/internal/recipeexec/registry.go b/internal/backends/postgres/internal/recipeexec/registry.go new file mode 100644 index 000000000..a5a138969 --- /dev/null +++ b/internal/backends/postgres/internal/recipeexec/registry.go @@ -0,0 +1,274 @@ +package postgres + +import ( + "context" + "embed" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/dataframe/recipe/exec" +) + +const ( + RecipesTable = "recipe_exec_recipes" + OccurrencesTable = "recipe_exec_occurrences" +) + +//go:embed migrations/*.sql +var migrationsFS embed.FS + +func RegistryMigrations() ([]pgstore.Migration, error) { + migrations, err := pgstore.LoadMigrations(migrationsFS, "migrations") + if err != nil { + return nil, fmt.Errorf("recipe exec postgres: load registry migrations: %w", err) + } + return migrations, nil +} + +func MigrateSchema(ctx context.Context, src pgstore.ConnSource, schema string) (pgstore.MigrateResult, error) { + if src == nil { + return pgstore.MigrateResult{}, errors.New("recipe exec postgres: nil migration connection source") + } + migrations, err := RegistryMigrations() + if err != nil { + return pgstore.MigrateResult{}, err + } + runner, err := pgstore.NewMigrationRunner(src, schema, migrations) + if err != nil { + return pgstore.MigrateResult{}, err + } + return runner.Migrate(ctx) +} + +func VerifySchema(ctx context.Context, src pgstore.ConnSource, schema string) error { + if src == nil { + return errors.New("recipe exec postgres: nil migration connection source") + } + migrations, err := RegistryMigrations() + if err != nil { + return err + } + runner, err := pgstore.NewMigrationRunner(src, schema, migrations) + if err != nil { + return err + } + return runner.Verify(ctx) +} + +var ErrRecipeVersionConflict = errors.New("recipe version already registered with a different digest") + +type DB interface { + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row + Begin(ctx context.Context) (pgx.Tx, error) +} + +type Registry struct { + db DB + recipes string + occurrences string +} + +var _ exec.Store = (*Registry)(nil) + +func New(pool DB, schema string) (*Registry, error) { + if pool == nil { + return nil, errors.New("recipe exec postgres: nil pool") + } + if !pgstore.ValidSchemaName(schema) { + return nil, fmt.Errorf("recipe exec postgres: invalid schema %q: must match [a-z_][a-z0-9_]{0,62}", schema) + } + return &Registry{ + db: pool, + recipes: pgx.Identifier{schema, RecipesTable}.Sanitize(), + occurrences: pgx.Identifier{schema, OccurrencesTable}.Sanitize(), + }, nil +} + +func entryParams(entry exec.Entry, now time.Time) ([]any, error) { + payload, err := json.Marshal(entry) + if err != nil { + return nil, fmt.Errorf("recipe exec postgres: marshal entry payload: %w", err) + } + return []any{entry.Bundle.Name, entry.Bundle.TranslationVersion, entry.Digest, string(payload), now, now}, nil +} + +func (r *Registry) SaveRecipe(ctx context.Context, entry exec.Entry) error { + now := time.Now().UTC().Truncate(time.Microsecond) + params, err := entryParams(entry, now) + if err != nil { + return err + } + tag, err := r.db.Exec(ctx, + "INSERT INTO "+r.recipes+ + " (recipe_name, translation_version, digest, payload, created_at, updated_at)"+ + " VALUES ($1, $2, $3, $4, $5, $6)"+ + " ON CONFLICT (recipe_name, translation_version) DO NOTHING", + params...) + if err != nil { + return fmt.Errorf("recipe exec postgres: save recipe: %w", err) + } + if tag.RowsAffected() == 1 { + return nil + } + var stored string + err = r.db.QueryRow(ctx, + "SELECT digest FROM "+r.recipes+ + " WHERE recipe_name = $1 AND translation_version = $2 FOR UPDATE", + entry.Bundle.Name, entry.Bundle.TranslationVersion).Scan(&stored) + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("%w: %s@%s vanished during save", exec.ErrRecipeNotFound, + entry.Bundle.Name, entry.Bundle.TranslationVersion) + } + if err != nil { + return fmt.Errorf("recipe exec postgres: inspect saved recipe: %w", err) + } + if stored == entry.Digest { + return nil + } + return fmt.Errorf("%w: %s@%s", ErrRecipeVersionConflict, entry.Bundle.Name, entry.Bundle.TranslationVersion) +} + +func (r *Registry) ReplaceRecipeVersion(ctx context.Context, entry exec.Entry) error { + now := time.Now().UTC().Truncate(time.Microsecond) + payload, err := json.Marshal(entry) + if err != nil { + return fmt.Errorf("recipe exec postgres: marshal entry payload: %w", err) + } + tx, err := r.db.Begin(ctx) + if err != nil { + return fmt.Errorf("recipe exec postgres: begin replace: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var held bool + if err := tx.QueryRow(ctx, + "SELECT true FROM "+r.recipes+ + " WHERE recipe_name = $1 AND translation_version = $2 FOR UPDATE", + entry.Bundle.Name, entry.Bundle.TranslationVersion).Scan(&held); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("%w: %s@%s", exec.ErrRecipeNotFound, + entry.Bundle.Name, entry.Bundle.TranslationVersion) + } + return fmt.Errorf("recipe exec postgres: lock recipe row: %w", err) + } + + var used bool + if err := tx.QueryRow(ctx, + "SELECT EXISTS (SELECT 1 FROM "+r.occurrences+ + " WHERE recipe_name = $1 AND translation_version = $2)", + entry.Bundle.Name, entry.Bundle.TranslationVersion).Scan(&used); err != nil { + return fmt.Errorf("recipe exec postgres: probe usage: %w", err) + } + if used { + return fmt.Errorf("%w: %s@%s", exec.ErrRecipeVersionImmutable, + entry.Bundle.Name, entry.Bundle.TranslationVersion) + } + + set := "digest = $3, payload = $4, updated_at = $5" + if _, err := tx.Exec(ctx, + "UPDATE "+r.recipes+" SET "+set+ + " WHERE recipe_name = $1 AND translation_version = $2", + entry.Bundle.Name, entry.Bundle.TranslationVersion, entry.Digest, + string(payload), now); err != nil { + return fmt.Errorf("recipe exec postgres: replace recipe: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("recipe exec postgres: commit replace: %w", err) + } + return nil +} + +func (r *Registry) LoadRecipe(ctx context.Context, name string) (exec.Entry, error) { + return r.loadEntry(ctx, + "SELECT payload FROM "+r.recipes+ + " WHERE recipe_name = $1"+ + " ORDER BY translation_version DESC LIMIT 1", + name) +} + +func (r *Registry) LoadRecipeVersion(ctx context.Context, name, translationVersion string) (exec.Entry, error) { + return r.loadEntry(ctx, + "SELECT payload FROM "+r.recipes+ + " WHERE recipe_name = $1 AND translation_version = $2", + name, translationVersion) +} + +func (r *Registry) loadEntry(ctx context.Context, query string, args ...any) (exec.Entry, error) { + var payload []byte + err := r.db.QueryRow(ctx, query, args...).Scan(&payload) + if errors.Is(err, pgx.ErrNoRows) { + return exec.Entry{}, exec.ErrRecipeNotFound + } + if err != nil { + return exec.Entry{}, fmt.Errorf("recipe exec postgres: load recipe: %w", err) + } + var entry exec.Entry + if err := json.Unmarshal(payload, &entry); err != nil { + return exec.Entry{}, fmt.Errorf("recipe exec postgres: decode entry payload: %w", err) + } + return entry, nil +} + +func (r *Registry) RecipeVersionUsed(ctx context.Context, name, translationVersion string) (bool, error) { + var used bool + if err := r.db.QueryRow(ctx, + "SELECT EXISTS (SELECT 1 FROM "+r.occurrences+ + " WHERE recipe_name = $1 AND translation_version = $2)", + name, translationVersion).Scan(&used); err != nil { + return false, fmt.Errorf("recipe exec postgres: probe usage: %w", err) + } + return used, nil +} + +func (r *Registry) RecordRecipeExecutionOccurrence(ctx context.Context, name, translationVersion, occurrenceID, ownerID string) error { + if strings.TrimSpace(name) == "" || strings.TrimSpace(translationVersion) == "" || strings.TrimSpace(occurrenceID) == "" { + return errors.New("recipe exec postgres: recipe name, translation version, and occurrence id are required") + } + tx, err := r.db.Begin(ctx) + if err != nil { + return fmt.Errorf("recipe exec postgres: begin record occurrence: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var held bool + if err := tx.QueryRow(ctx, + "SELECT true FROM "+r.recipes+ + " WHERE recipe_name = $1 AND translation_version = $2 FOR UPDATE", + name, translationVersion).Scan(&held); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("%w: %s@%s", exec.ErrRecipeNotFound, name, translationVersion) + } + return fmt.Errorf("recipe exec postgres: lock recipe row for occurrence: %w", err) + } + + var next int64 + if err := tx.QueryRow(ctx, + "SELECT COALESCE(MAX(owner_sequence), 0) + 1 FROM "+r.occurrences+ + " WHERE recipe_name = $1 AND translation_version = $2 AND owner_id = $3", + name, translationVersion, ownerID).Scan(&next); err != nil { + return fmt.Errorf("recipe exec postgres: next owner sequence: %w", err) + } + + if _, err := tx.Exec(ctx, + "INSERT INTO "+r.occurrences+ + " (recipe_name, translation_version, occurrence_id, owner_id, owner_sequence, recorded_at)"+ + " VALUES ($1, $2, $3, $4, $5, $6)"+ + " ON CONFLICT (recipe_name, translation_version, occurrence_id) DO NOTHING", + name, translationVersion, occurrenceID, ownerID, next, + time.Now().UTC().Truncate(time.Microsecond)); err != nil { + return fmt.Errorf("recipe exec postgres: record occurrence: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("recipe exec postgres: commit occurrence: %w", err) + } + return nil +} diff --git a/internal/backends/postgres/internal/recipeexec/registry_test.go b/internal/backends/postgres/internal/recipeexec/registry_test.go new file mode 100644 index 000000000..4ea31bd04 --- /dev/null +++ b/internal/backends/postgres/internal/recipeexec/registry_test.go @@ -0,0 +1,976 @@ +package postgres + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "reflect" + "regexp" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/recipe/exec" +) + +var pgrecip01v1NameRe = regexp.MustCompile(`^pgrecip01v1_it_[0-9]+_[0-9a-f]+$`) + +func testPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 recipe exec-registry integration tests") + } + return raw +} + +func randomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +var createdDatabases sync.Map + +func snapshotCreatedDatabases() []string { + var names []string + createdDatabases.Range(func(k, _ any) bool { + names = append(names, k.(string)) + return true + }) + sort.Strings(names) + return names +} + +type registryFixture struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dbName string + schema string + dsn string + registry *Registry + scrub func(any) string + fail func(step string, err error) +} + +func newRegistryFixture(t *testing.T) *registryFixture { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + if err == nil { + return + } + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgrecip01v1_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgrecip01v1NameRe.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + createdDatabases.Store(dbName, struct{}{}) + + t.Cleanup(func() { + var mine int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname = $1`, dbName).Scan(&mine); err != nil { + t.Errorf("leftover audit: query database catalog: %s", scrub(err)) + } + if mine != 0 { + t.Errorf("leftover audit: isolated database %s still exists", dbName) + } + names := snapshotCreatedDatabases() + var mineLeftovers int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname = ANY($1)`, names).Scan(&mineLeftovers); err != nil { + t.Errorf("leftover audit: query owned database set: %s", scrub(err)) + } + if mineLeftovers != 0 { + t.Errorf("leftover audit: %d databases created by this binary remain: %v", mineLeftovers, names) + } + var schemaLeftovers int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgrecip01v1_it_%'`).Scan(&schemaLeftovers); err != nil { + t.Errorf("leftover audit: query schema namespace: %s", scrub(err)) + } + if schemaLeftovers != 0 { + t.Errorf("leftover audit: %d pgrecip01v1_it_* schemas remain in the service database", schemaLeftovers) + } + }) + + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + if probe, err := url.Parse(dsn); err != nil || + probe.Query().Get("dbname") != "" || probe.Path != "/"+dbName { + t.Fatal("isolated test DSN does not provably target the isolated database") + } + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("pgrecip01v1_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgrecip01v1NameRe.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + f := ®istryFixture{ctx: ctx, admin: admin, iso: iso, dbName: dbName, schema: schema, dsn: dsn, scrub: scrub, fail: fail} + if _, err := MigrateSchema(ctx, iso, schema); err != nil { + f.fail("migrate exec-registry history", err) + } + registry, err := New(iso, schema) + if err != nil { + f.fail("build exec registry", err) + } + f.registry = registry + return f +} + +func (f *registryFixture) recipesTable() string { + return pgx.Identifier{f.schema, RecipesTable}.Sanitize() +} + +func (f *registryFixture) occurrencesTable() string { + return pgx.Identifier{f.schema, OccurrencesTable}.Sanitize() +} + +func (f *registryFixture) count(t *testing.T, query string, args ...any) int { + t.Helper() + var n int + if err := f.iso.QueryRow(f.ctx, query, args...).Scan(&n); err != nil { + f.fail("count", fmt.Errorf("%s: %w", query, err)) + } + return n +} + +func (f *registryFixture) mustText(t *testing.T, query string, args ...any) string { + t.Helper() + var s string + if err := f.iso.QueryRow(f.ctx, query, args...).Scan(&s); err != nil { + f.fail("witness text", fmt.Errorf("%s: %w", query, err)) + } + return s +} + +func (f *registryFixture) storedDigest(t *testing.T, name, version string) string { + t.Helper() + return f.mustText(t, "SELECT digest FROM "+f.recipesTable()+ + " WHERE recipe_name = $1 AND translation_version = $2", name, version) +} + +func (f *registryFixture) mustPgErrorCode(t *testing.T, step, query string, args ...any) string { + t.Helper() + _, err := f.iso.Exec(f.ctx, query, args...) + if err == nil { + t.Fatalf("%s: statement unexpectedly succeeded", step) + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + f.fail(step, err) + } + return pgErr.Code +} + +func (f *registryFixture) waitLockedFor(t *testing.T, done <-chan struct{}, patterns ...string) { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + ors := make([]string, 0, len(patterns)) + for range patterns { + ors = append(ors, "query LIKE $"+fmt.Sprint(len(ors)+1)) + } + query := "SELECT count(*) FROM pg_stat_activity" + + " WHERE state = 'active' AND wait_event_type = 'Lock' AND (" + strings.Join(ors, " OR ") + ")" + args := make([]any, 0, len(patterns)) + for _, p := range patterns { + args = append(args, p) + } + for { + var waiting int + if err := f.admin.QueryRow(f.ctx, query, args...).Scan(&waiting); err != nil { + f.fail("witness lock wait", err) + } + if waiting > 0 { + return + } + select { + case <-done: + t.Fatal("the fenced operation completed before the competing writer committed -- the fence did not serialize") + case <-time.After(10 * time.Millisecond): + } + if time.Now().After(deadline) { + t.Fatal("the fenced operation never blocked on the recipe row lock") + } + } +} + +func canonicalEntry(t *testing.T, name, translationVersion, rowGrain string) exec.Entry { + t.Helper() + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: name, + TranslationVersion: translationVersion, + Outputs: []recipe.Output{{ + Name: "DocumentReference", + RootResourceType: "DocumentReference", + RowGrain: rowGrain, + }}, + } + canonical, err := bundle.CanonicalBundle() + if err != nil { + t.Fatalf("canonicalize bundle: %v", err) + } + digest, err := canonical.Digest() + if err != nil { + t.Fatalf("canonical digest: %v", err) + } + return exec.Entry{Bundle: canonical, Digest: digest} +} + +func TestExecSaveLoadRoundTripAndLiteralRows(t *testing.T) { + f := newRegistryFixture(t) + first := canonicalEntry(t, "files", "1", "document_reference") + second := canonicalEntry(t, "files", "2", "document_reference_v2") + f.fail("save first", f.registry.SaveRecipe(f.ctx, first)) + f.fail("save second", f.registry.SaveRecipe(f.ctx, second)) + + for _, entry := range []exec.Entry{first, second} { + loaded, err := f.registry.LoadRecipeVersion(f.ctx, "files", entry.Bundle.TranslationVersion) + f.fail("load version", err) + if !reflect.DeepEqual(loaded, entry) { + t.Fatalf("loaded entry = %+v, want %+v", loaded, entry) + } + } + + if n := f.count(t, "SELECT count(*) FROM "+f.recipesTable()); n != 2 { + t.Fatalf("registration rows = %d, want 2", n) + } + for _, entry := range []exec.Entry{first, second} { + payload, err := json.Marshal(entry) + f.fail("marshal expected payload", err) + if n := f.count(t, "SELECT count(*) FROM "+f.recipesTable()+ + " WHERE recipe_name = $1 AND translation_version = $2 AND digest = $3 AND payload = $4::jsonb", + entry.Bundle.Name, entry.Bundle.TranslationVersion, entry.Digest, string(payload)); n != 1 { + t.Fatalf("stored row for %s is not the literal registration", entry.Bundle.TranslationVersion) + } + } + + loaded, err := f.registry.LoadRecipe(f.ctx, "files") + f.fail("load current", err) + if loaded.Digest != second.Digest { + t.Fatalf("current digest = %s, want %s", loaded.Digest, second.Digest) + } +} + +func TestExecLoadRecipeCurrentUsesExactByteOrder(t *testing.T) { + f := newRegistryFixture(t) + digests := map[string]string{} + for _, version := range []string{"1", "2", "10"} { + entry := canonicalEntry(t, "files", version, "document_reference") + f.fail("save "+version, f.registry.SaveRecipe(f.ctx, entry)) + digests[version] = entry.Digest + } + loaded, err := f.registry.LoadRecipe(f.ctx, "files") + f.fail("load current", err) + if loaded.Digest != digests["2"] { + t.Fatalf("current digest = %s, want version 2's %s (COLLATE \"C\" byte order)", loaded.Digest, digests["2"]) + } + witness := f.mustText(t, "SELECT translation_version FROM "+f.recipesTable()+ + " WHERE recipe_name = 'files' ORDER BY translation_version DESC LIMIT 1") + if witness != "2" { + t.Fatalf("byte-order witness = %q, want 2", witness) + } +} + +func TestExecLoadMissingReportsRecipeNotFound(t *testing.T) { + f := newRegistryFixture(t) + if _, err := f.registry.LoadRecipe(f.ctx, "never"); !errors.Is(err, exec.ErrRecipeNotFound) { + t.Fatalf("LoadRecipe err = %v, want ErrRecipeNotFound", err) + } + f.fail("seed", f.registry.SaveRecipe(f.ctx, canonicalEntry(t, "files", "1", "document_reference"))) + if _, err := f.registry.LoadRecipeVersion(f.ctx, "files", "missing"); !errors.Is(err, exec.ErrRecipeNotFound) { + t.Fatalf("LoadRecipeVersion err = %v, want ErrRecipeNotFound", err) + } + if _, err := f.registry.LoadRecipeVersion(f.ctx, "other", "1"); !errors.Is(err, exec.ErrRecipeNotFound) { + t.Fatalf("cross-name LoadRecipeVersion err = %v, want ErrRecipeNotFound", err) + } +} + +func TestExecSaveSameDigestIsIdempotent(t *testing.T) { + f := newRegistryFixture(t) + entry := canonicalEntry(t, "files", "1", "document_reference") + f.fail("first save", f.registry.SaveRecipe(f.ctx, entry)) + created := f.mustText(t, "SELECT created_at::text FROM "+f.recipesTable()+ + " WHERE recipe_name = $1 AND translation_version = $2", "files", "1") + f.fail("retry save", f.registry.SaveRecipe(f.ctx, entry)) + if n := f.count(t, "SELECT count(*) FROM "+f.recipesTable()); n != 1 { + t.Fatalf("rows after idempotent retry = %d, want 1", n) + } + after := f.mustText(t, "SELECT created_at::text FROM "+f.recipesTable()+ + " WHERE recipe_name = $1 AND translation_version = $2", "files", "1") + if after != created { + t.Fatalf("idempotent retry rewrote created_at: %s -> %s", created, after) + } +} + +func TestExecConcurrentDivergentFirstSaveHasOneWinner(t *testing.T) { + f := newRegistryFixture(t) + entryA := canonicalEntry(t, "files", "1", "grain-a") + entryB := canonicalEntry(t, "files", "1", "grain-b") + + start := make(chan struct{}) + var wg sync.WaitGroup + var errA, errB error + wg.Add(2) + for i, entry := range []exec.Entry{entryA, entryB} { + go func(i int, entry exec.Entry) { + defer wg.Done() + <-start + err := f.registry.SaveRecipe(f.ctx, entry) + if i == 0 { + errA = err + } else { + errB = err + } + }(i, entry) + } + close(start) + wg.Wait() + + outcomes := []error{errA, errB} + wins, conflicts := 0, 0 + for _, err := range outcomes { + switch { + case err == nil: + wins++ + case errors.Is(err, ErrRecipeVersionConflict): + conflicts++ + default: + t.Fatalf("unexpected save outcome: %v", err) + } + } + if wins != 1 || conflicts != 1 { + t.Fatalf("divergent first-save race: wins=%d conflicts=%d (errA=%v errB=%v), want 1/1", + wins, conflicts, errA, errB) + } + if n := f.count(t, "SELECT count(*) FROM "+f.recipesTable()); n != 1 { + t.Fatalf("rows after race = %d, want 1", n) + } + want := entryA.Digest + if errA != nil { + want = entryB.Digest + } + if got := f.storedDigest(t, "files", "1"); got != want { + t.Fatalf("stored digest = %s, want the winner's %s", got, want) + } +} + +func TestExecConcurrentIdenticalFirstSaveIsIdempotent(t *testing.T) { + f := newRegistryFixture(t) + entry := canonicalEntry(t, "files", "1", "document_reference") + start := make(chan struct{}) + var wg sync.WaitGroup + errs := make([]error, 2) + for i := 0; i < 2; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + errs[i] = f.registry.SaveRecipe(f.ctx, entry) + }(i) + } + close(start) + wg.Wait() + for i, err := range errs { + if err != nil { + t.Fatalf("identical concurrent save %d failed: %v", i, err) + } + } + if n := f.count(t, "SELECT count(*) FROM "+f.recipesTable()); n != 1 { + t.Fatalf("rows = %d, want 1", n) + } +} + +func TestExecReplaceBeforeUseSwapsContentAndKeepsCreatedAt(t *testing.T) { + f := newRegistryFixture(t) + original := canonicalEntry(t, "files", "1", "document_reference") + replacement := canonicalEntry(t, "files", "1", "document_reference_changed") + f.fail("save", f.registry.SaveRecipe(f.ctx, original)) + beforeCreated := f.mustText(t, "SELECT created_at::text FROM "+f.recipesTable()+ + " WHERE recipe_name = $1 AND translation_version = $2", "files", "1") + + time.Sleep(2 * time.Millisecond) + f.fail("replace", f.registry.ReplaceRecipeVersion(f.ctx, replacement)) + + loaded, err := f.registry.LoadRecipeVersion(f.ctx, "files", "1") + f.fail("load after replace", err) + if !reflect.DeepEqual(loaded, replacement) { + t.Fatalf("loaded = %+v, want replaced %+v", loaded, replacement) + } + created := f.mustText(t, "SELECT created_at::text FROM "+f.recipesTable()+ + " WHERE recipe_name = $1 AND translation_version = $2", "files", "1") + if created != beforeCreated { + t.Fatalf("replace moved created_at: %s -> %s", beforeCreated, created) + } + updated := f.mustText(t, "SELECT updated_at::text FROM "+f.recipesTable()+ + " WHERE recipe_name = $1 AND translation_version = $2", "files", "1") + if updated == created { + t.Fatal("replace did not advance updated_at") + } +} + +func TestExecReplaceMissingVersionReportsNotFound(t *testing.T) { + f := newRegistryFixture(t) + err := f.registry.ReplaceRecipeVersion(f.ctx, canonicalEntry(t, "files", "1", "document_reference")) + if !errors.Is(err, exec.ErrRecipeNotFound) { + t.Fatalf("replace on absent version err = %v, want ErrRecipeNotFound", err) + } + if n := f.count(t, "SELECT count(*) FROM "+f.recipesTable()); n != 0 { + t.Fatalf("replace manufactured a row: %d", n) + } +} + +func TestExecOccurrenceMakesVersionImmutable(t *testing.T) { + f := newRegistryFixture(t) + original := canonicalEntry(t, "files", "1", "document_reference") + replacement := canonicalEntry(t, "files", "1", "changed") + f.fail("save", f.registry.SaveRecipe(f.ctx, original)) + + used, err := f.registry.RecipeVersionUsed(f.ctx, "files", "1") + f.fail("probe unused", err) + if used { + t.Fatal("RecipeVersionUsed = true before an occurrence exists") + } + f.fail("record first use", f.registry.RecordRecipeExecutionOccurrence(f.ctx, "files", "1", "exec-1", "worker-a")) + + used, err = f.registry.RecipeVersionUsed(f.ctx, "files", "1") + f.fail("probe used", err) + if !used { + t.Fatal("RecipeVersionUsed = false after an occurrence exists") + } + err = f.registry.ReplaceRecipeVersion(f.ctx, replacement) + if !errors.Is(err, exec.ErrRecipeVersionImmutable) { + t.Fatalf("replace after use err = %v, want ErrRecipeVersionImmutable", err) + } + if got := f.storedDigest(t, "files", "1"); got != original.Digest { + t.Fatalf("immutable version changed: %s != %s", got, original.Digest) + } +} + +func TestExecOccurrenceIdentityIsIdempotentAndOwnersCorrelate(t *testing.T) { + f := newRegistryFixture(t) + f.fail("save", f.registry.SaveRecipe(f.ctx, canonicalEntry(t, "files", "1", "document_reference"))) + f.fail("save other version", f.registry.SaveRecipe(f.ctx, canonicalEntry(t, "files", "2", "document_reference"))) + f.fail("save other recipe", f.registry.SaveRecipe(f.ctx, canonicalEntry(t, "other", "1", "document_reference"))) + + f.fail("record o1", f.registry.RecordRecipeExecutionOccurrence(f.ctx, "files", "1", "o1", "worker-a")) + f.fail("re-record o1", f.registry.RecordRecipeExecutionOccurrence(f.ctx, "files", "1", "o1", "worker-a")) + f.fail("record o2", f.registry.RecordRecipeExecutionOccurrence(f.ctx, "files", "1", "o2", "worker-a")) + f.fail("record o3", f.registry.RecordRecipeExecutionOccurrence(f.ctx, "files", "1", "o3", "worker-b")) + + if n := f.count(t, "SELECT count(*) FROM "+f.occurrencesTable()); n != 3 { + t.Fatalf("occurrence rows = %d, want 3 (o1 recorded once)", n) + } + type seq struct { + id string + seq int64 + } + for _, want := range []seq{{"o1", 1}, {"o2", 2}, {"o3", 1}} { + got := f.mustText(t, "SELECT owner_sequence::text FROM "+f.occurrencesTable()+ + " WHERE recipe_name = $1 AND translation_version = $2 AND occurrence_id = $3", + "files", "1", want.id) + if got != fmt.Sprint(want.seq) { + t.Fatalf("occurrence %s owner_sequence = %s, want %d", want.id, got, want.seq) + } + } + + for _, probe := range []struct { + name, version string + want bool + }{ + {"files", "1", true}, + {"files", "2", false}, + {"other", "1", false}, + } { + used, err := f.registry.RecipeVersionUsed(f.ctx, probe.name, probe.version) + f.fail("probe", err) + if used != probe.want { + t.Fatalf("RecipeVersionUsed(%s,%s) = %v, want %v", probe.name, probe.version, used, probe.want) + } + } + + if err := f.registry.RecordRecipeExecutionOccurrence(f.ctx, "ghost", "1", "o1", "worker-a"); !errors.Is(err, exec.ErrRecipeNotFound) { + t.Fatalf("orphan occurrence err = %v, want ErrRecipeNotFound", err) + } + if err := f.registry.RecordRecipeExecutionOccurrence(f.ctx, "files", "1", " ", "worker-a"); err == nil { + t.Fatal("blank occurrence id accepted") + } +} + +func TestExecPersistentRegistryIdempotentRetryAndImmutableAfterUse(t *testing.T) { + f := newRegistryFixture(t) + registry := &exec.PersistentRegistry{Store: f.registry} + + bundleWithGrain := func(grain string) recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "files", + TranslationVersion: "1", + Outputs: []recipe.Output{{ + Name: "DocumentReference", RootResourceType: "DocumentReference", RowGrain: grain, + }}, + } + } + bundle := bundleWithGrain("document_reference") + first, err := registry.RegisterVersion(f.ctx, bundle) + f.fail("register first", err) + retry, err := registry.RegisterVersion(f.ctx, bundleWithGrain("document_reference")) + f.fail("exact retry", err) + if retry.Digest != first.Digest || !reflect.DeepEqual(retry.Bundle, first.Bundle) { + t.Fatalf("exact retry = %+v, want %+v", retry, first) + } + if n := f.count(t, "SELECT count(*) FROM "+f.recipesTable()); n != 1 { + t.Fatalf("rows after exact retry = %d, want 1", n) + } + + changed := bundleWithGrain("changed") + replaced, err := registry.RegisterVersion(f.ctx, changed) + f.fail("register changed", err) + if replaced.Digest == first.Digest { + t.Fatal("pre-use replacement did not change the digest") + } + + f.fail("use it", f.registry.RecordRecipeExecutionOccurrence(f.ctx, "files", "1", "exec-9", "worker-x")) + third := bundleWithGrain("after-use") + if _, err := registry.RegisterVersion(f.ctx, third); !errors.Is(err, exec.ErrRecipeVersionImmutable) { + t.Fatalf("post-use RegisterVersion err = %v, want ErrRecipeVersionImmutable", err) + } + if got := f.storedDigest(t, "files", "1"); got != replaced.Digest { + t.Fatalf("post-use attempt mutated the version: %s != %s", got, replaced.Digest) + } + again, err := registry.RegisterVersion(f.ctx, bundleWithGrain("changed")) + f.fail("exact retry after use", err) + if again.Digest != replaced.Digest { + t.Fatalf("exact retry after use = %s, want %s", again.Digest, replaced.Digest) + } +} + +func TestExecReplaceFencedAgainstCommittingFirstUse(t *testing.T) { + f := newRegistryFixture(t) + original := canonicalEntry(t, "files", "1", "document_reference") + replacement := canonicalEntry(t, "files", "1", "fenced-out") + f.fail("save", f.registry.SaveRecipe(f.ctx, original)) + + connA, err := pgx.Connect(f.ctx, f.dsn) + f.fail("connect first-use session", err) + defer connA.Close(context.Background()) + txConnA, err := connA.Begin(f.ctx) + f.fail("begin first-use transaction", err) + defer func() { _ = txConnA.Rollback(f.ctx) }() + + if _, err := txConnA.Exec(f.ctx, "SELECT true FROM "+f.recipesTable()+ + " WHERE recipe_name = $1 AND translation_version = $2 FOR UPDATE", "files", "1"); err != nil { + f.fail("first-use row lock", err) + } + if _, err := txConnA.Exec(f.ctx, "INSERT INTO "+f.occurrencesTable()+ + " (recipe_name, translation_version, occurrence_id, owner_id, owner_sequence, recorded_at)"+ + " VALUES ($1, $2, $3, $4, 1, $5)", + "files", "1", "race-use", "worker-race", time.Now().UTC().Truncate(time.Microsecond)); err != nil { + f.fail("first-use insert", err) + } + + done := make(chan struct{}) + result := make(chan error, 1) + go func() { + defer close(done) + result <- f.registry.ReplaceRecipeVersion(f.ctx, replacement) + }() + + f.waitLockedFor(t, done, + "SELECT true FROM "+f.recipesTable()+"%FOR UPDATE", + "UPDATE "+f.recipesTable()+"%") + + f.fail("commit racing first use", txConnA.Commit(f.ctx)) + err = <-result + if !errors.Is(err, exec.ErrRecipeVersionImmutable) { + t.Fatalf("replace racing a committing first use = %v, want ErrRecipeVersionImmutable", err) + } + if got := f.storedDigest(t, "files", "1"); got != original.Digest { + t.Fatalf("fence leaked: digest moved to %s while a use committed underneath", got) + } + if n := f.count(t, "SELECT count(*) FROM "+f.occurrencesTable()+ + " WHERE recipe_name = $1 AND translation_version = $2", "files", "1"); n != 1 { + t.Fatalf("occurrence rows = %d, want 1", n) + } +} + +func TestExecRecordFencedBehindCommittingReplace(t *testing.T) { + f := newRegistryFixture(t) + original := canonicalEntry(t, "files", "1", "document_reference") + replacement := canonicalEntry(t, "files", "1", "replaced-first") + f.fail("save", f.registry.SaveRecipe(f.ctx, original)) + + connA, err := pgx.Connect(f.ctx, f.dsn) + f.fail("connect replacing session", err) + defer connA.Close(context.Background()) + txConnA, err := connA.Begin(f.ctx) + f.fail("begin replacing transaction", err) + defer func() { _ = txConnA.Rollback(f.ctx) }() + + if _, err := txConnA.Exec(f.ctx, "SELECT true FROM "+f.recipesTable()+ + " WHERE recipe_name = $1 AND translation_version = $2 FOR UPDATE", "files", "1"); err != nil { + f.fail("replace row lock", err) + } + if _, err := txConnA.Exec(f.ctx, "UPDATE "+f.recipesTable()+ + " SET digest = $3, updated_at = $4 WHERE recipe_name = $1 AND translation_version = $2", + "files", "1", replacement.Digest, time.Now().UTC().Truncate(time.Microsecond)); err != nil { + f.fail("replace update", err) + } + + done := make(chan struct{}) + result := make(chan error, 1) + go func() { + defer close(done) + result <- f.registry.RecordRecipeExecutionOccurrence(f.ctx, "files", "1", "late-use", "worker-late") + }() + + f.waitLockedFor(t, done, + "SELECT true FROM "+f.recipesTable()+"%FOR UPDATE") + + f.fail("commit racing replace", txConnA.Commit(f.ctx)) + f.fail("occurrence behind replace", <-result) + if got := f.storedDigest(t, "files", "1"); got != replacement.Digest { + t.Fatalf("digest = %s, want the committed replacement %s", got, replacement.Digest) + } + used, err := f.registry.RecipeVersionUsed(f.ctx, "files", "1") + f.fail("probe after ordered use", err) + if !used { + t.Fatal("occurrence committed after the replace must still count as use") + } +} + +func TestExecConcurrentReplaceVersusOccurrenceNeverBothWinWrong(t *testing.T) { + f := newRegistryFixture(t) + for iteration := 0; iteration < 8; iteration++ { + name := fmt.Sprintf("race-%d", iteration) + original := canonicalEntry(t, name, "1", "document_reference") + replacement := canonicalEntry(t, name, "1", "swapped") + f.fail("save race entry", f.registry.SaveRecipe(f.ctx, original)) + + start := make(chan struct{}) + var wg sync.WaitGroup + var replaceErr, recordErr error + wg.Add(2) + go func() { + defer wg.Done() + <-start + replaceErr = f.registry.ReplaceRecipeVersion(f.ctx, replacement) + }() + go func() { + defer wg.Done() + <-start + recordErr = f.registry.RecordRecipeExecutionOccurrence(f.ctx, name, "1", "race-occ", "worker-r") + }() + close(start) + wg.Wait() + + f.fail("occurrence always records", recordErr) + if n := f.count(t, "SELECT count(*) FROM "+f.occurrencesTable()+ + " WHERE recipe_name = $1 AND translation_version = $2", name, "1"); n != 1 { + t.Fatalf("iteration %d: occurrence rows = %d, want 1", iteration, n) + } + switch { + case replaceErr == nil: + if got := f.storedDigest(t, name, "1"); got != replacement.Digest { + t.Fatalf("iteration %d: successful replace stored %s, want %s", iteration, got, replacement.Digest) + } + case errors.Is(replaceErr, exec.ErrRecipeVersionImmutable): + if got := f.storedDigest(t, name, "1"); got != original.Digest { + t.Fatalf("iteration %d: immutable replace still moved the digest", iteration) + } + default: + t.Fatalf("iteration %d: unexpected replace error %v", iteration, replaceErr) + } + } +} + +func TestExecNewGuards(t *testing.T) { + f := newRegistryFixture(t) + if _, err := New(nil, f.schema); err == nil { + t.Fatal("nil pool accepted") + } + for _, bad := range []string{"", "Bad-Schema", "1nvalid", `"; DROP`, strings.Repeat("a", 64)} { + if _, err := New(f.iso, bad); err == nil { + t.Fatalf("unsafe schema %q accepted", bad) + } + } +} + +func TestExecStoreFaultsStayUntyped(t *testing.T) { + f := newRegistryFixture(t) + bareSchema := f.schema + "_bare" + if _, err := f.iso.Exec(f.ctx, `CREATE SCHEMA "`+bareSchema+`"`); err != nil { + f.fail("create bare schema", err) + } + registry, err := New(f.iso, bareSchema) + f.fail("build bare registry", err) + if err := registry.SaveRecipe(f.ctx, canonicalEntry(t, "files", "1", "document_reference")); err == nil { + t.Fatal("save against unmigrated schema succeeded") + } else { + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "42P01" { + t.Fatalf("expected undefined-table store fault, got %v", err) + } + } +} + +func TestExecMigrateSchemaHistoryLedgerAndRerun(t *testing.T) { + f := newRegistryFixture(t) + + history, err := RegistryMigrations() + f.fail("load registry migrations", err) + if len(history) != 2 || history[0].Version != 1 || history[0].Name != "ledger" || + history[1].Version != 2 || history[1].Name != "recipe_exec_registries" { + t.Fatalf("embedded history = %+v, want {1 ledger, 2 recipe_exec_registries}", history) + } + + rows, err := f.iso.Query(f.ctx, + "SELECT version, name, checksum FROM "+ + pgx.Identifier{f.schema, pgstore.LedgerTable}.Sanitize()+" ORDER BY version") + f.fail("read ledger", err) + defer rows.Close() + type rec struct { + version int64 + name string + checksum string + } + var got []rec + for rows.Next() { + var r rec + if err := rows.Scan(&r.version, &r.name, &r.checksum); err != nil { + f.fail("scan ledger", err) + } + got = append(got, r) + } + f.fail("ledger rows", rows.Err()) + if len(got) != 2 { + t.Fatalf("ledger rows = %d, want 2", len(got)) + } + for i, want := range history { + if got[i].version != want.Version || got[i].name != want.Name || got[i].checksum != want.Checksum { + t.Fatalf("ledger[%d] = %+v, want {%d %s %s}", i, got[i], want.Version, want.Name, want.Checksum) + } + } + + entry := canonicalEntry(t, "files", "1", "document_reference") + f.fail("save before rerun", f.registry.SaveRecipe(f.ctx, entry)) + + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + f.fail("open pg-store pool", err) + defer client.Close() + result, err := MigrateSchema(f.ctx, client, f.schema) + f.fail("rerun migrate via production pool", err) + if result.LatestVersion != 2 || len(result.Applied) != 2 { + t.Fatalf("rerun result = %+v, want latest 2 with two applied rows", result) + } + loaded, err := f.registry.LoadRecipeVersion(f.ctx, "files", "1") + f.fail("load after rerun", err) + if loaded.Digest != entry.Digest { + t.Fatalf("state did not survive rerun: %s != %s", loaded.Digest, entry.Digest) + } +} + +func TestExecSchemaConstraintsAndNULRejection(t *testing.T) { + f := newRegistryFixture(t) + valid := canonicalEntry(t, "files", "1", "document_reference") + + if code := f.mustPgErrorCode(t, "garbage digest", + "INSERT INTO "+f.recipesTable()+ + " (recipe_name, translation_version, digest, payload, created_at, updated_at)"+ + " VALUES ('p', '1', 'nope', '{}'::jsonb, now(), now())"); code != "23514" { + t.Fatalf("garbage digest sqlstate = %s, want 23514", code) + } + if code := f.mustPgErrorCode(t, "empty version", + "INSERT INTO "+f.recipesTable()+ + " (recipe_name, translation_version, digest, payload, created_at, updated_at)"+ + " VALUES ('p', '', $1, '{}'::jsonb, now(), now())", valid.Digest); code != "23514" { + t.Fatalf("empty version sqlstate = %s, want 23514", code) + } + if code := f.mustPgErrorCode(t, "scalar payload", + "INSERT INTO "+f.recipesTable()+ + " (recipe_name, translation_version, digest, payload, created_at, updated_at)"+ + " VALUES ('p', '1', $1, '5'::jsonb, now(), now())", valid.Digest); code != "23514" { + t.Fatalf("scalar payload sqlstate = %s, want 23514", code) + } + _, err := f.iso.Exec(f.ctx, "INSERT INTO "+f.recipesTable()+ + " (recipe_name, translation_version, digest, payload, created_at, updated_at)"+ + " VALUES ($1, '1', $2, '{}'::jsonb, now(), now())", "bad\x00name", valid.Digest) + if err == nil { + t.Fatal("NUL-bearing recipe name accepted") + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || !strings.HasPrefix(pgErr.Code, "22") { + t.Fatalf("NUL-bearing name err class = %v, want 22xxx", err) + } + if code := f.mustPgErrorCode(t, "owner sequence zero", + "INSERT INTO "+f.occurrencesTable()+ + " (recipe_name, translation_version, occurrence_id, owner_id, owner_sequence, recorded_at)"+ + " VALUES ('p', '1', 'o1', 'w', 0, now())"); code != "23514" { + t.Fatalf("owner_sequence 0 sqlstate = %s, want 23514", code) + } +} + +func TestExecStatePersistsBeyondOneConnection(t *testing.T) { + f := newRegistryFixture(t) + entry := canonicalEntry(t, "files", "1", "document_reference") + f.fail("save", f.registry.SaveRecipe(f.ctx, entry)) + f.fail("record", f.registry.RecordRecipeExecutionOccurrence(f.ctx, "files", "1", "o1", "worker-a")) + + fresh, err := pgxpool.New(f.ctx, f.dsn) + f.fail("open fresh pool", err) + defer fresh.Close() + second, err := New(fresh, f.schema) + f.fail("build registry on fresh pool", err) + loaded, err := second.LoadRecipeVersion(f.ctx, "files", "1") + f.fail("load on fresh pool", err) + if loaded.Digest != entry.Digest { + t.Fatalf("fresh pool digest = %s, want %s", loaded.Digest, entry.Digest) + } + used, err := second.RecipeVersionUsed(f.ctx, "files", "1") + f.fail("probe on fresh pool", err) + if !used { + t.Fatal("occurrence did not survive to the fresh pool") + } +} + +func TestMain(m *testing.M) { + code := m.Run() + if err := auditZeroLeftovers(); err != nil { + fmt.Fprintf(os.Stderr, "zero-leftover audit failed: %v\n", err) + code = 1 + } + os.Exit(code) +} + +func auditZeroLeftovers() error { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return nil + } + auditScrub := func(format string, args ...any) string { + message := fmt.Sprintf(format, args...) + secrets := []string{raw} + if u, err := url.Parse(raw); err == nil && u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + for _, secret := range secrets { + if secret != "" { + message = strings.ReplaceAll(message, secret, "") + } + } + return message + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + return errors.New(auditScrub("connect: %v", err)) + } + defer admin.Close() + names := snapshotCreatedDatabases() + var databases int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname = ANY($1)", names).Scan(&databases); err != nil { + return errors.New(auditScrub("count databases: %v", err)) + } + if databases != 0 { + return errors.New(auditScrub("%d databases created by this binary still exist: %v", databases, names)) + } + var schemas int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgrecip01v1_it_%'").Scan(&schemas); err != nil { + return errors.New(auditScrub("count schemas: %v", err)) + } + if schemas != 0 { + return errors.New(auditScrub("%d pgrecip01v1_it_* schemas still exist in the service database", schemas)) + } + return nil +} diff --git a/internal/backends/postgres/internal/recipes/migrations/000001_ledger.sql b/internal/backends/postgres/internal/recipes/migrations/000001_ledger.sql new file mode 100644 index 000000000..8f12e38d9 --- /dev/null +++ b/internal/backends/postgres/internal/recipes/migrations/000001_ledger.sql @@ -0,0 +1,26 @@ +-- pg-state-recipe-registries-v1: create the per-schema migration ledger. +-- +-- This mirrors the pg-store-02 ledger contract (internal/store/postgres/ +-- migrations/000001_ledger.sql): version, name, and checksum only, never the +-- SQL body, with the same CHECK shapes. The MigrationRunner reads this table +-- from the target schema, so a schema whose history starts here bootstraps +-- its own ledger audited by the same runner. Unlike the pg-store-02 copy this +-- body is written CREATE TABLE IF NOT EXISTS so a crash in the narrow +-- commit-before-record window converges on retry, as the runner documentation +-- requires of feature migrations. It never drops or truncates anything. +-- +-- Both recipe-registries package histories (recipe/exec/postgres and +-- recipe/postgres) embed this file byte-identically, so the version-1 +-- checksum agrees everywhere and one control-plane schema can be migrated by +-- the composed history {1 ledger, 2 recipe_exec_registries, 3 +-- recipe_revision_chains} in version order. +CREATE TABLE IF NOT EXISTS schema_migrations ( + version bigint NOT NULL, + name text NOT NULL, + checksum text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT schema_migrations_pkey PRIMARY KEY (version), + CONSTRAINT schema_migrations_version_positive CHECK (version > 0), + CONSTRAINT schema_migrations_name_shape CHECK (name ~ '^[a-z][a-z0-9_]*$'), + CONSTRAINT schema_migrations_checksum_hex CHECK (checksum ~ '^[0-9a-f]{64}$') +); diff --git a/internal/backends/postgres/internal/recipes/migrations/000003_recipe_revision_chains.sql b/internal/backends/postgres/internal/recipes/migrations/000003_recipe_revision_chains.sql new file mode 100644 index 000000000..e1f687fcd --- /dev/null +++ b/internal/backends/postgres/internal/recipes/migrations/000003_recipe_revision_chains.sql @@ -0,0 +1,71 @@ +-- pg-state-recipe-registries-v1: project recipe revision chains for the +-- PostgreSQL control plane. +-- +-- The revision table mirrors the Arango collection +-- loom_dataframe_recipe_revisions (internal/dataframe/recipe/exec/arango/ +-- revisions.go): one immutable row per (project, name, digest) carrying the +-- canonical Bundle payload and its parent digest, with the digest being the +-- canonical Bundle digest and therefore a safe optimistic-concurrency and +-- publication address. The heads table makes the chain's CAS explicit: one +-- row per (project, name) names the current head digest and counts every +-- committed extension, correlating repeated extensions of one chain through +-- epoch the way the publication leases correlate repeated owners. +-- +-- Deterministic scope and ordering: +-- * project/name/digest carry COLLATE "C" so tenant/project scoping is +-- exact byte equality and List's (created_at DESC, digest) order is +-- stable under any database collation. +-- * The payload is authoritative for the RecipeRevision shape; the +-- searchable identity lives in columns and cannot disagree with what the +-- adapter wrote (one argument slice populates both). +-- Three-state discipline: +-- * parent_digest is NOT NULL DEFAULT '': the Go string has no missing +-- state, so "no parent" is the empty string (the payload omits the +-- key), never SQL NULL. A parent is either empty or a valid digest -- +-- the CHECK rejects every other shape, so a malformed parent can never +-- masquerade as a chain link. +-- * head_digest is the only nullable column and it is a placeholder +-- state, not data: epoch = 0 with head_digest IS NULL means "the chain +-- row exists to serialize writers but no revision has ever committed"; +-- epoch >= 1 always names a real digest. The pair CHECK makes the two +-- illegal combinations (NULL with epoch >= 1, digest with epoch 0) +-- unrepresentable, and no placeholder row can ever be read back as a +-- revision. +-- This migration never drops or truncates: rerunning bootstrap on an +-- existing schema is a no-op that preserves chain state. +CREATE TABLE IF NOT EXISTS recipe_revisions ( + project text NOT NULL COLLATE "C", + name text NOT NULL COLLATE "C", + digest text NOT NULL COLLATE "C", + parent_digest text NOT NULL DEFAULT '', + payload jsonb NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT recipe_revisions_pkey PRIMARY KEY (project, name, digest), + CONSTRAINT recipe_revisions_project_nonempty CHECK (project <> ''), + CONSTRAINT recipe_revisions_name_nonempty CHECK (name <> ''), + CONSTRAINT recipe_revisions_digest_hex CHECK (digest ~ '^[0-9a-f]{64}$'), + CONSTRAINT recipe_revisions_parent_shape + CHECK (parent_digest = '' OR parent_digest ~ '^[0-9a-f]{64}$'), + CONSTRAINT recipe_revisions_payload_object CHECK (jsonb_typeof(payload) = 'object') +); + +-- List(project, name): every revision of one chain, deterministic newest +-- first with digest as the total-order tie-break. +CREATE INDEX IF NOT EXISTS recipe_revisions_chain_created + ON recipe_revisions (project, name, created_at DESC, digest); + +CREATE TABLE IF NOT EXISTS recipe_revision_heads ( + project text NOT NULL COLLATE "C", + name text NOT NULL COLLATE "C", + head_digest text, + epoch bigint NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT recipe_revision_heads_pkey PRIMARY KEY (project, name), + CONSTRAINT recipe_revision_heads_project_nonempty CHECK (project <> ''), + CONSTRAINT recipe_revision_heads_name_nonempty CHECK (name <> ''), + CONSTRAINT recipe_revision_heads_epoch_nonnegative CHECK (epoch >= 0), + CONSTRAINT recipe_revision_heads_state_consistent CHECK ( + (epoch = 0 AND head_digest IS NULL) + OR (epoch >= 1 AND head_digest IS NOT NULL AND head_digest ~ '^[0-9a-f]{64}$') + ) +); diff --git a/internal/backends/postgres/internal/recipes/revisions.go b/internal/backends/postgres/internal/recipes/revisions.go new file mode 100644 index 000000000..1bf4f9aa9 --- /dev/null +++ b/internal/backends/postgres/internal/recipes/revisions.go @@ -0,0 +1,269 @@ +package postgres + +import ( + "context" + "embed" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + "github.com/calypr/loom/internal/dataframe/recipe" +) + +const ( + RevisionsTable = "recipe_revisions" + HeadsTable = "recipe_revision_heads" +) + +//go:embed migrations/*.sql +var migrationsFS embed.FS + +func ChainMigrations() ([]pgstore.Migration, error) { + migrations, err := pgstore.LoadMigrations(migrationsFS, "migrations") + if err != nil { + return nil, fmt.Errorf("recipe postgres: load revision-chain migrations: %w", err) + } + return migrations, nil +} + +func MigrateSchema(ctx context.Context, src pgstore.ConnSource, schema string) (pgstore.MigrateResult, error) { + if src == nil { + return pgstore.MigrateResult{}, errors.New("recipe postgres: nil migration connection source") + } + migrations, err := ChainMigrations() + if err != nil { + return pgstore.MigrateResult{}, err + } + runner, err := pgstore.NewMigrationRunner(src, schema, migrations) + if err != nil { + return pgstore.MigrateResult{}, err + } + return runner.Migrate(ctx) +} + +func VerifySchema(ctx context.Context, src pgstore.ConnSource, schema string) error { + if src == nil { + return errors.New("recipe postgres: nil migration connection source") + } + migrations, err := ChainMigrations() + if err != nil { + return err + } + runner, err := pgstore.NewMigrationRunner(src, schema, migrations) + if err != nil { + return err + } + return runner.Verify(ctx) +} + +type DB interface { + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row + Begin(ctx context.Context) (pgx.Tx, error) +} + +type RevisionRegistry struct { + db DB + revisions string + heads string +} + +var _ recipe.RevisionStore = (*RevisionRegistry)(nil) + +func New(pool DB, schema string) (*RevisionRegistry, error) { + if pool == nil { + return nil, errors.New("recipe postgres: nil pool") + } + if !pgstore.ValidSchemaName(schema) { + return nil, fmt.Errorf("recipe postgres: invalid schema %q: must match [a-z_][a-z0-9_]{0,62}", schema) + } + return &RevisionRegistry{ + db: pool, + revisions: pgx.Identifier{schema, RevisionsTable}.Sanitize(), + heads: pgx.Identifier{schema, HeadsTable}.Sanitize(), + }, nil +} + +func (r *RevisionRegistry) Register(ctx context.Context, project string, bundle recipe.Bundle, parent string) (recipe.RecipeRevision, error) { + if strings.TrimSpace(project) == "" || strings.TrimSpace(bundle.Name) == "" { + return recipe.RecipeRevision{}, dataframeInvalidRequest("project and recipe name are required") + } + canonical, err := bundle.CanonicalBundle() + if err != nil { + return recipe.RecipeRevision{}, err + } + bundle = canonical + digest, err := bundle.Digest() + if err != nil { + return recipe.RecipeRevision{}, err + } + + tx, err := r.db.Begin(ctx) + if err != nil { + return recipe.RecipeRevision{}, fmt.Errorf("recipe postgres: begin register: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + if _, err := tx.Exec(ctx, + "INSERT INTO "+r.heads+" (project, name, head_digest, epoch, updated_at)"+ + " VALUES ($1, $2, NULL, 0, $3)"+ + " ON CONFLICT (project, name) DO NOTHING", + project, bundle.Name, time.Now().UTC().Truncate(time.Microsecond)); err != nil { + return recipe.RecipeRevision{}, fmt.Errorf("recipe postgres: ensure revision head: %w", err) + } + var ( + head *string + epoch int64 + ) + if err := tx.QueryRow(ctx, + "SELECT head_digest, epoch FROM "+r.heads+ + " WHERE project = $1 AND name = $2 FOR UPDATE", + project, bundle.Name).Scan(&head, &epoch); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return recipe.RecipeRevision{}, fmt.Errorf("recipe postgres: revision head row vanished: %w", err) + } + return recipe.RecipeRevision{}, fmt.Errorf("recipe postgres: lock revision head: %w", err) + } + + if head == nil { + if parent != "" { + return recipe.RecipeRevision{}, dataframeInvalidRequest(fmt.Sprintf( + "recipe revision parent %q provided for first revision", parent)) + } + } else if *head == digest && parent == *head { + revision, err := r.scanRevision(ctx, tx, project, bundle.Name, digest) + if err != nil { + return recipe.RecipeRevision{}, err + } + return revision, nil + } else if parent != *head { + return recipe.RecipeRevision{}, dataframeStaleCursor(fmt.Errorf( + "recipe revision parent %q is not current (current %q)", parent, *head)) + } + + expected := any(nil) + if parent != "" { + expected = parent + } + tag, err := tx.Exec(ctx, + "UPDATE "+r.heads+" SET head_digest = $3, epoch = epoch + 1, updated_at = $4"+ + " WHERE project = $1 AND name = $2"+ + " AND head_digest IS NOT DISTINCT FROM $5", + project, bundle.Name, digest, time.Now().UTC().Truncate(time.Microsecond), expected) + if err != nil { + return recipe.RecipeRevision{}, fmt.Errorf("recipe postgres: advance revision head: %w", err) + } + if tag.RowsAffected() != 1 { + return recipe.RecipeRevision{}, dataframeStaleCursor(fmt.Errorf( + "recipe revision parent %q raced the current head", parent)) + } + + createdAt := time.Now().UTC().Truncate(time.Microsecond) + revision := recipe.RecipeRevision{ + Project: project, + Name: bundle.Name, + Digest: digest, + Parent: parent, + Bundle: bundle, + CreatedAt: createdAt, + } + payload, err := json.Marshal(revision) + if err != nil { + return recipe.RecipeRevision{}, fmt.Errorf("recipe postgres: marshal revision payload: %w", err) + } + if _, err := tx.Exec(ctx, + "INSERT INTO "+r.revisions+ + " (project, name, digest, parent_digest, payload, created_at)"+ + " VALUES ($1, $2, $3, $4, $5, $6)"+ + " ON CONFLICT (project, name, digest) DO UPDATE SET"+ + " parent_digest = EXCLUDED.parent_digest, payload = EXCLUDED.payload,"+ + " created_at = EXCLUDED.created_at", + project, bundle.Name, digest, parent, string(payload), createdAt); err != nil { + return recipe.RecipeRevision{}, fmt.Errorf("recipe postgres: insert revision: %w", err) + } + + stored, err := r.scanRevision(ctx, tx, project, bundle.Name, digest) + if err != nil { + return recipe.RecipeRevision{}, err + } + if err := tx.Commit(ctx); err != nil { + return recipe.RecipeRevision{}, fmt.Errorf("recipe postgres: commit register: %w", err) + } + return stored, nil +} + +func (r *RevisionRegistry) Get(ctx context.Context, project, name, digest string) (recipe.RecipeRevision, error) { + return r.scanRevision(ctx, r.db, project, name, digest) +} + +func (r *RevisionRegistry) List(ctx context.Context, project, name string) ([]recipe.RecipeRevision, error) { + rows, err := r.db.Query(ctx, + "SELECT payload FROM "+r.revisions+ + " WHERE project = $1 AND name = $2"+ + " ORDER BY created_at DESC, digest", + project, name) + if err != nil { + return nil, fmt.Errorf("recipe postgres: list revisions: %w", err) + } + defer rows.Close() + out := []recipe.RecipeRevision{} + for rows.Next() { + var payload []byte + if err := rows.Scan(&payload); err != nil { + return nil, fmt.Errorf("recipe postgres: scan revision: %w", err) + } + revision, err := decodeRevisionPayload(payload) + if err != nil { + return nil, err + } + out = append(out, revision) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("recipe postgres: list revisions: %w", err) + } + return out, nil +} + +type revisionReader interface { + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} + +func (r *RevisionRegistry) scanRevision(ctx context.Context, reader revisionReader, project, name, digest string) (recipe.RecipeRevision, error) { + var payload []byte + err := reader.QueryRow(ctx, + "SELECT payload FROM "+r.revisions+ + " WHERE project = $1 AND name = $2 AND digest = $3", + project, name, digest).Scan(&payload) + if errors.Is(err, pgx.ErrNoRows) { + return recipe.RecipeRevision{}, recipe.ErrRecipeRevisionNotFound + } + if err != nil { + return recipe.RecipeRevision{}, fmt.Errorf("recipe postgres: load revision: %w", err) + } + return decodeRevisionPayload(payload) +} + +func decodeRevisionPayload(payload []byte) (recipe.RecipeRevision, error) { + var revision recipe.RecipeRevision + if err := json.Unmarshal(payload, &revision); err != nil { + return recipe.RecipeRevision{}, fmt.Errorf("recipe postgres: decode revision payload: %w", err) + } + return revision, nil +} + +func dataframeInvalidRequest(message string) error { + return dataframeerrors.Wrap(errors.New(message), dataframeerrors.CodeInvalidRequest, "") +} + +func dataframeStaleCursor(cause error) error { + return dataframeerrors.Wrap(cause, dataframeerrors.CodeStaleCursor, "") +} diff --git a/internal/backends/postgres/internal/recipes/revisions_test.go b/internal/backends/postgres/internal/recipes/revisions_test.go new file mode 100644 index 000000000..dc70f7805 --- /dev/null +++ b/internal/backends/postgres/internal/recipes/revisions_test.go @@ -0,0 +1,929 @@ +package postgres + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "reflect" + "regexp" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + "github.com/calypr/loom/internal/dataframe/recipe" +) + +var pgrecip01v1NameRe = regexp.MustCompile(`^pgrecip01v1_it_[0-9]+_[0-9a-f]+$`) + +func testPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 recipe revision-chain integration tests") + } + return raw +} + +func randomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +var createdDatabases sync.Map + +func snapshotCreatedDatabases() []string { + var names []string + createdDatabases.Range(func(k, _ any) bool { + names = append(names, k.(string)) + return true + }) + sort.Strings(names) + return names +} + +type chainFixture struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dbName string + schema string + dsn string + registry *RevisionRegistry + scrub func(any) string + fail func(step string, err error) +} + +func newChainFixture(t *testing.T) *chainFixture { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + if err == nil { + return + } + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgrecip01v1_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgrecip01v1NameRe.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + createdDatabases.Store(dbName, struct{}{}) + + t.Cleanup(func() { + var mine int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname = $1`, dbName).Scan(&mine); err != nil { + t.Errorf("leftover audit: query database catalog: %s", scrub(err)) + } + if mine != 0 { + t.Errorf("leftover audit: isolated database %s still exists", dbName) + } + names := snapshotCreatedDatabases() + var mineLeftovers int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_database WHERE datname = ANY($1)`, names).Scan(&mineLeftovers); err != nil { + t.Errorf("leftover audit: query owned database set: %s", scrub(err)) + } + if mineLeftovers != 0 { + t.Errorf("leftover audit: %d databases created by this binary remain: %v", mineLeftovers, names) + } + var schemaLeftovers int + if err := admin.QueryRow(ctx, `SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgrecip01v1_it_%'`).Scan(&schemaLeftovers); err != nil { + t.Errorf("leftover audit: query schema namespace: %s", scrub(err)) + } + if schemaLeftovers != 0 { + t.Errorf("leftover audit: %d pgrecip01v1_it_* schemas remain in the service database", schemaLeftovers) + } + }) + + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + if probe, err := url.Parse(dsn); err != nil || + probe.Query().Get("dbname") != "" || probe.Path != "/"+dbName { + t.Fatal("isolated test DSN does not provably target the isolated database") + } + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("pgrecip01v1_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgrecip01v1NameRe.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + f := &chainFixture{ctx: ctx, admin: admin, iso: iso, dbName: dbName, schema: schema, dsn: dsn, scrub: scrub, fail: fail} + if _, err := MigrateSchema(ctx, iso, schema); err != nil { + f.fail("migrate revision-chain history", err) + } + registry, err := New(iso, schema) + if err != nil { + f.fail("build revision registry", err) + } + f.registry = registry + return f +} + +func (f *chainFixture) revisionsTable() string { + return pgx.Identifier{f.schema, RevisionsTable}.Sanitize() +} + +func (f *chainFixture) headsTable() string { + return pgx.Identifier{f.schema, HeadsTable}.Sanitize() +} + +func (f *chainFixture) count(t *testing.T, query string, args ...any) int { + t.Helper() + var n int + if err := f.iso.QueryRow(f.ctx, query, args...).Scan(&n); err != nil { + f.fail("count", fmt.Errorf("%s: %w", query, err)) + } + return n +} + +func (f *chainFixture) mustText(t *testing.T, query string, args ...any) string { + t.Helper() + var s string + if err := f.iso.QueryRow(f.ctx, query, args...).Scan(&s); err != nil { + f.fail("witness text", fmt.Errorf("%s: %w", query, err)) + } + return s +} + +func (f *chainFixture) mustInt64(t *testing.T, query string, args ...any) int64 { + t.Helper() + var n int64 + if err := f.iso.QueryRow(f.ctx, query, args...).Scan(&n); err != nil { + f.fail("witness int", fmt.Errorf("%s: %w", query, err)) + } + return n +} + +func (f *chainFixture) mustPgErrorCode(t *testing.T, step, query string, args ...any) string { + t.Helper() + _, err := f.iso.Exec(f.ctx, query, args...) + if err == nil { + t.Fatalf("%s: statement unexpectedly succeeded", step) + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + f.fail(step, err) + } + return pgErr.Code +} + +func bundleFor(name, translationVersion string) recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: name, + TranslationVersion: translationVersion, + Outputs: []recipe.Output{{ + Name: "DocumentReference", + RootResourceType: "DocumentReference", + RowGrain: "document_reference", + }}, + } +} + +func requireRevisionEqual(t *testing.T, want, got recipe.RecipeRevision, where string) { + t.Helper() + if got.Project != want.Project || got.Name != want.Name || got.Digest != want.Digest || got.Parent != want.Parent { + t.Fatalf("%s: identity changed: got %+v want %+v", where, got, want) + } + if !got.CreatedAt.Equal(want.CreatedAt) { + t.Fatalf("%s: createdAt instant changed: got %s want %s", where, got.CreatedAt, want.CreatedAt) + } + if !reflect.DeepEqual(got.Bundle, want.Bundle) { + t.Fatalf("%s: bundle changed: got %+v want %+v", where, got.Bundle, want.Bundle) + } +} + +func userFault(t *testing.T, err error, want dataframeerrors.ErrorCode) dataframeerrors.UserError { + t.Helper() + if err == nil { + t.Fatalf("expected typed %s, got nil", want) + } + userErr, ok := dataframeerrors.AsUserError(err) + if !ok { + t.Fatalf("expected typed %s, got untyped %v", want, err) + } + if userErr.Code() != string(want) { + t.Fatalf("code=%s want %s (err=%v)", userErr.Code(), want, err) + } + return userErr +} + +func TestRevisionRegistryRoundTripAndLiteralRows(t *testing.T) { + f := newChainFixture(t) + bundle := bundleFor("files", "1") + first, err := f.registry.Register(f.ctx, "project-a", bundle, "") + f.fail("register first", err) + + wantDigest := mustCanonicalDigest(t, bundle) + if first.Digest != wantDigest || first.Project != "project-a" || first.Name != "files" || first.Parent != "" { + t.Fatalf("first revision = %+v, want project-a/files digest %s parent empty", first, wantDigest) + } + + loaded, err := f.registry.Get(f.ctx, "project-a", "files", wantDigest) + f.fail("get first", err) + requireRevisionEqual(t, first, loaded, "round trip") + + list, err := f.registry.List(f.ctx, "project-a", "files") + f.fail("list first", err) + if len(list) != 1 { + t.Fatalf("list length = %d, want 1", len(list)) + } + requireRevisionEqual(t, first, list[0], "list") + + if n := f.count(t, "SELECT count(*) FROM "+f.revisionsTable()); n != 1 { + t.Fatalf("revision rows = %d, want 1", n) + } + if got := f.mustText(t, "SELECT digest FROM "+f.revisionsTable()+" WHERE project = $1 AND name = $2", "project-a", "files"); got != wantDigest { + t.Fatalf("stored digest = %q, want %q", got, wantDigest) + } + if got := f.mustText(t, "SELECT parent_digest FROM "+f.revisionsTable()+" WHERE project = $1 AND name = $2", "project-a", "files"); got != "" { + t.Fatalf("stored parent_digest = %q, want the empty string (Go's missing parent)", got) + } + if present := f.mustText(t, "SELECT (payload ? 'parentDigest')::text FROM "+f.revisionsTable()+" WHERE project = $1 AND name = $2", "project-a", "files"); present != "false" { + t.Fatalf("payload has parentDigest key = %s, want absent (missing parent)", present) + } + payload, err := json.Marshal(recipe.RecipeRevision{ + Project: "project-a", Name: "files", Digest: wantDigest, Parent: "", + Bundle: mustCanonical(t, bundle), CreatedAt: first.CreatedAt, + }) + f.fail("marshal expected payload", err) + if n := f.count(t, "SELECT count(*) FROM "+f.revisionsTable()+ + " WHERE project = $1 AND name = $2 AND digest = $3 AND payload = $4::jsonb", + "project-a", "files", wantDigest, string(payload)); n != 1 { + t.Fatal("stored payload is not exactly the marshaled revision") + } + if epoch := f.mustInt64(t, "SELECT epoch FROM "+f.headsTable()+" WHERE project = $1 AND name = $2", "project-a", "files"); epoch != 1 { + t.Fatalf("head epoch after first register = %d, want 1", epoch) + } + if got := f.mustText(t, "SELECT head_digest FROM "+f.headsTable()+" WHERE project = $1 AND name = $2", "project-a", "files"); got != wantDigest { + t.Fatalf("head digest = %q, want %q", got, wantDigest) + } +} + +func TestRevisionRegistryChainsChildrenAndBumpsEpoch(t *testing.T) { + f := newChainFixture(t) + first, err := f.registry.Register(f.ctx, "project-a", bundleFor("files", "1"), "") + f.fail("register first", err) + second, err := f.registry.Register(f.ctx, "project-a", bundleFor("files", "2"), first.Digest) + f.fail("register child", err) + if second.Parent != first.Digest || second.Digest == first.Digest { + t.Fatalf("child = %+v, want parent %s and a distinct digest", second, first.Digest) + } + if epoch := f.mustInt64(t, "SELECT epoch FROM "+f.headsTable()+" WHERE project = $1 AND name = $2", "project-a", "files"); epoch != 2 { + t.Fatalf("head epoch = %d, want 2 (one committed extension after the first)", epoch) + } + if n := f.count(t, "SELECT count(*) FROM "+f.revisionsTable()+" WHERE project = $1 AND name = $2", "project-a", "files"); n != 2 { + t.Fatalf("revision rows = %d, want 2", n) + } + list, err := f.registry.List(f.ctx, "project-a", "files") + f.fail("list chain", err) + if len(list) != 2 || list[0].Digest != second.Digest || list[1].Digest != first.Digest { + t.Fatalf("list order wrong: %+v", list) + } +} + +func mustCanonical(t *testing.T, bundle recipe.Bundle) recipe.Bundle { + t.Helper() + canonical, err := bundle.CanonicalBundle() + if err != nil { + t.Fatalf("canonicalize bundle: %v", err) + } + return canonical +} + +func mustCanonicalDigest(t *testing.T, bundle recipe.Bundle) string { + t.Helper() + digest, err := mustCanonical(t, bundle).Digest() + if err != nil { + t.Fatalf("canonical digest: %v", err) + } + return digest +} + +func TestRevisionRegistryRejectsParentForFirstRevision(t *testing.T) { + f := newChainFixture(t) + _, err := f.registry.Register(f.ctx, "project-a", bundleFor("files", "1"), strings.Repeat("a", 64)) + userFault(t, err, dataframeerrors.CodeInvalidRequest) + + if n := f.count(t, "SELECT count(*) FROM "+f.revisionsTable()); n != 0 { + t.Fatalf("revision rows = %d, want 0 (no dangling first revision)", n) + } + if n := f.count(t, "SELECT count(*) FROM "+f.headsTable()); n != 0 { + t.Fatalf("head rows = %d, want 0 (placeholder rolled back)", n) + } +} + +func TestRevisionRegistryStaleAndEmptyParentAreTypedStaleCursor(t *testing.T) { + f := newChainFixture(t) + first, err := f.registry.Register(f.ctx, "project-a", bundleFor("files", "1"), "") + f.fail("register first", err) + + _, err = f.registry.Register(f.ctx, "project-a", bundleFor("files", "2"), "") + userErr := userFault(t, err, dataframeerrors.CodeStaleCursor) + if userErr.Retryable() { + t.Fatal("stale parent error must not be retryable") + } + if _, err := f.registry.Register(f.ctx, "project-a", bundleFor("files", "3"), strings.Repeat("f", 64)); userFault(t, err, dataframeerrors.CodeStaleCursor) == nil { + t.Fatal("wrong parent not typed") + } + if n := f.count(t, "SELECT count(*) FROM "+f.revisionsTable()+" WHERE project = $1 AND name = $2", "project-a", "files"); n != 1 { + t.Fatalf("revision rows after rejections = %d, want 1", n) + } + epoch := f.mustInt64(t, "SELECT epoch FROM "+f.headsTable()+" WHERE project = $1 AND name = $2", "project-a", "files") + if epoch != 1 { + t.Fatalf("head epoch after rejections = %d, want 1", epoch) + } + if got := f.mustText(t, "SELECT head_digest FROM "+f.headsTable()+" WHERE project = $1 AND name = $2", "project-a", "files"); got != first.Digest { + t.Fatalf("head digest after rejections = %q, want the untouched %q", got, first.Digest) + } +} + +func TestRevisionRegistryValidatesIdentityBeforeStore(t *testing.T) { + f := newChainFixture(t) + if _, err := f.registry.Register(f.ctx, "project-a", bundleFor("", "1"), ""); userFault(t, err, dataframeerrors.CodeInvalidRequest) == nil { + t.Fatal("missing recipe name not typed") + } + if _, err := f.registry.Register(f.ctx, " ", bundleFor("files", "1"), ""); userFault(t, err, dataframeerrors.CodeInvalidRequest) == nil { + t.Fatal("missing project not typed") + } + if n := f.count(t, "SELECT count(*) FROM "+f.revisionsTable()); n != 0 { + t.Fatalf("rows after invalid requests = %d, want 0", n) + } +} + +func TestRevisionRegistryStoreFaultsStayUntyped(t *testing.T) { + f := newChainFixture(t) + bareSchema := f.schema + "_bare" + if _, err := f.iso.Exec(f.ctx, `CREATE SCHEMA "`+bareSchema+`"`); err != nil { + f.fail("create bare schema", err) + } + registry, err := New(f.iso, bareSchema) + f.fail("build bare registry", err) + _, err = registry.Register(f.ctx, "project-a", bundleFor("files", "1"), "") + if err == nil { + t.Fatal("register against unmigrated schema succeeded") + } + if _, typed := dataframeerrors.AsUserError(err); typed { + t.Fatalf("store fault must stay untyped, got %#v", err) + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "42P01" { + t.Fatalf("expected undefined-table store fault, got %v", err) + } +} + +func TestRevisionRegistryHeadRetryIsIdempotent(t *testing.T) { + f := newChainFixture(t) + bundle := bundleFor("files", "1") + first, err := f.registry.Register(f.ctx, "project-a", bundle, "") + f.fail("register first", err) + + retry, err := f.registry.Register(f.ctx, "project-a", bundle, first.Digest) + f.fail("retry head content", err) + requireRevisionEqual(t, first, retry, "head retry") + + if n := f.count(t, "SELECT count(*) FROM "+f.revisionsTable()); n != 1 { + t.Fatalf("revision rows after head retry = %d, want 1", n) + } + if epoch := f.mustInt64(t, "SELECT epoch FROM "+f.headsTable()+" WHERE project = $1 AND name = $2", "project-a", "files"); epoch != 1 { + t.Fatalf("head epoch after idempotent retry = %d, want 1 (pure retries do not bump)", epoch) + } +} + +func TestRevisionRegistryDeepDuplicateKeepsOriginalRow(t *testing.T) { + f := newChainFixture(t) + first, err := f.registry.Register(f.ctx, "project-a", bundleFor("files", "1"), "") + f.fail("register first", err) + second, err := f.registry.Register(f.ctx, "project-a", bundleFor("files", "2"), first.Digest) + f.fail("register second", err) + + time.Sleep(2 * time.Millisecond) + relinked, err := f.registry.Register(f.ctx, "project-a", bundleFor("files", "1"), second.Digest) + f.fail("relink first content", err) + if relinked.Digest != first.Digest || relinked.Parent != second.Digest { + t.Fatalf("relinked revision = %+v, want digest %s parent %s", relinked, first.Digest, second.Digest) + } + if !relinked.CreatedAt.After(first.CreatedAt) { + t.Fatalf("relink did not restamp the row: %s vs original %s", relinked.CreatedAt, first.CreatedAt) + } + if !reflect.DeepEqual(relinked.Bundle, first.Bundle) { + t.Fatal("relink changed the digest-pinned bundle content") + } + if n := f.count(t, "SELECT count(*) FROM "+f.revisionsTable()); n != 2 { + t.Fatalf("revision rows = %d, want 2 (relink reuses the stored row)", n) + } + if epoch := f.mustInt64(t, "SELECT epoch FROM "+f.headsTable()); epoch != 3 { + t.Fatalf("head epoch = %d, want 3", epoch) + } + if got := f.mustText(t, "SELECT head_digest FROM "+f.headsTable()); got != first.Digest { + t.Fatalf("head digest = %q, want the relinked %q", got, first.Digest) + } +} + +func runTwoRegisters(t *testing.T, f *chainFixture, a, b func() (recipe.RecipeRevision, error)) (recipe.RecipeRevision, error, recipe.RecipeRevision, error) { + t.Helper() + start := make(chan struct{}) + var wg sync.WaitGroup + var ra, rb recipe.RecipeRevision + var ea, eb error + wg.Add(2) + go func() { + defer wg.Done() + <-start + ra, ea = a() + }() + go func() { + defer wg.Done() + <-start + rb, eb = b() + }() + close(start) + wg.Wait() + return ra, ea, rb, eb +} + +func TestRevisionConcurrentChildrenOfOneParentHaveOneWinner(t *testing.T) { + f := newChainFixture(t) + for iteration := 0; iteration < 6; iteration++ { + name := fmt.Sprintf("files-race-%d", iteration) + head, err := f.registry.Register(f.ctx, "project-a", bundleFor(name, "1"), "") + f.fail("register race head", err) + + childA := bundleFor(name, "a") + childB := bundleFor(name, "b") + revA, errA, revB, errB := runTwoRegisters(t, f, + func() (recipe.RecipeRevision, error) { + return f.registry.Register(f.ctx, "project-a", childA, head.Digest) + }, + func() (recipe.RecipeRevision, error) { + return f.registry.Register(f.ctx, "project-a", childB, head.Digest) + }) + + winners := 0 + losers := 0 + for _, pair := range []struct { + rev recipe.RecipeRevision + err error + }{{revA, errA}, {revB, errB}} { + switch { + case pair.err == nil: + winners++ + default: + losers++ + userErr := userFault(t, pair.err, dataframeerrors.CodeStaleCursor) + if userErr.Retryable() { + t.Fatal("concurrent loser must receive a non-retryable typed stale-parent conflict") + } + } + } + if winners != 1 || losers != 1 { + t.Fatalf("iteration %d: winners=%d losers=%d (errA=%v errB=%v), want exactly one winner", + iteration, winners, losers, errA, errB) + } + + if n := f.count(t, "SELECT count(*) FROM "+f.revisionsTable()+" WHERE project = $1 AND name = $2", "project-a", name); n != 2 { + t.Fatalf("iteration %d: revision rows = %d, want 2 (loser left no orphan)", iteration, n) + } + if epoch := f.mustInt64(t, "SELECT epoch FROM "+f.headsTable()+" WHERE project = $1 AND name = $2", "project-a", name); epoch != 2 { + t.Fatalf("iteration %d: head epoch = %d, want 2", iteration, epoch) + } + var winner recipe.RecipeRevision + if errA == nil { + winner = revA + } else { + winner = revB + } + if got := f.mustText(t, "SELECT head_digest FROM "+f.headsTable()+" WHERE project = $1 AND name = $2", "project-a", name); got != winner.Digest { + t.Fatalf("iteration %d: head digest = %q, want winner %q", iteration, got, winner.Digest) + } + } +} + +func TestRevisionConcurrentFirstRegistrationsHaveOneWinner(t *testing.T) { + f := newChainFixture(t) + name := "files-first-race" + _, errA, _, errB := runTwoRegisters(t, f, + func() (recipe.RecipeRevision, error) { + return f.registry.Register(f.ctx, "project-a", bundleFor(name, "a"), "") + }, + func() (recipe.RecipeRevision, error) { + return f.registry.Register(f.ctx, "project-a", bundleFor(name, "b"), "") + }) + + winners, losers := 0, 0 + for _, err := range []error{errA, errB} { + if err == nil { + winners++ + continue + } + losers++ + userFault(t, err, dataframeerrors.CodeStaleCursor) + } + if winners != 1 || losers != 1 { + t.Fatalf("first-registration race: winners=%d losers=%d, want 1/1", winners, losers) + } + if n := f.count(t, "SELECT count(*) FROM "+f.revisionsTable()+" WHERE project = $1 AND name = $2", "project-a", name); n != 1 { + t.Fatalf("revision rows = %d, want 1", n) + } + if epoch := f.mustInt64(t, "SELECT epoch FROM "+f.headsTable()+" WHERE project = $1 AND name = $2", "project-a", name); epoch != 1 { + t.Fatalf("head epoch = %d, want 1", epoch) + } +} + +func TestRevisionGetAndListAreProjectAndNameScoped(t *testing.T) { + f := newChainFixture(t) + first, err := f.registry.Register(f.ctx, "project-a", bundleFor("files", "1"), "") + f.fail("register", err) + + if _, err := f.registry.Get(f.ctx, "project-b", "files", first.Digest); !errors.Is(err, recipe.ErrRecipeRevisionNotFound) { + t.Fatalf("cross-project Get err = %v, want ErrRecipeRevisionNotFound", err) + } + if _, err := f.registry.Get(f.ctx, "project-a", "other", first.Digest); !errors.Is(err, recipe.ErrRecipeRevisionNotFound) { + t.Fatalf("cross-name Get err = %v, want ErrRecipeRevisionNotFound", err) + } + if _, err := f.registry.Get(f.ctx, "project-a", "files", strings.Repeat("0", 64)); !errors.Is(err, recipe.ErrRecipeRevisionNotFound) { + t.Fatalf("missing digest err = %v, want ErrRecipeRevisionNotFound", err) + } + empty, err := f.registry.List(f.ctx, "project-z", "nothing") + f.fail("list unknown", err) + if empty == nil || len(empty) != 0 { + t.Fatalf("unknown-chain list = %#v, want non-nil empty slice", empty) + } + + other, err := f.registry.Register(f.ctx, "project-b", bundleFor("files", "1"), "") + f.fail("register in project-b", err) + if other.Digest != first.Digest { + t.Fatal("identical bundle digests disagree across projects") + } + if _, err := f.registry.Get(f.ctx, "project-b", "files", other.Digest); err != nil { + f.fail("get cross-project twin", err) + } +} + +func TestRevisionListIsDeterministicWithLiteralTies(t *testing.T) { + f := newChainFixture(t) + stamp := func(offset time.Duration) time.Time { + return time.Date(2026, 2, 3, 4, 5, 6, 0, time.UTC).Add(offset) + } + type lit struct { + digest string + created time.Time + } + lits := []lit{ + {strings.Repeat("a", 64), stamp(0)}, + {strings.Repeat("b", 64), stamp(0)}, + {strings.Repeat("c", 64), stamp(0)}, + {strings.Repeat("d", 64), stamp(time.Hour)}, + } + for _, l := range lits { + payload := fmt.Sprintf( + `{"project":"literal-p","name":"literal-r","digest":%q,"bundle":{"recipeSchemaVersion":1,"name":"literal-r","translationVersion":"1","outputs":[{"name":"rows","rootResourceType":"Patient","rowGrain":"patient"}]},"createdAt":%q}`, + l.digest, l.created.Format(time.RFC3339Nano)) + if _, err := f.iso.Exec(f.ctx, + "INSERT INTO "+f.revisionsTable()+ + " (project, name, digest, parent_digest, payload, created_at)"+ + " VALUES ($1, $2, $3, $4, $5, $6)", + "literal-p", "literal-r", l.digest, "", payload, l.created); err != nil { + f.fail("insert literal revision", err) + } + } + list, err := f.registry.List(f.ctx, "literal-p", "literal-r") + f.fail("list literal", err) + want := []string{ + strings.Repeat("d", 64), + strings.Repeat("a", 64), + strings.Repeat("b", 64), + strings.Repeat("c", 64), + } + if len(list) != len(want) { + t.Fatalf("list length = %d, want %d", len(list), len(want)) + } + for i, digest := range want { + if list[i].Digest != digest { + t.Fatalf("list[%d] = %s, want %s (created_at DESC, digest tie-break)", i, list[i].Digest, digest) + } + } +} + +func TestRevisionThreeStatesAndHeadPlaceholderInvariant(t *testing.T) { + f := newChainFixture(t) + first, err := f.registry.Register(f.ctx, "project-a", bundleFor("files", "1"), "") + f.fail("register", err) + + nullDigest := strings.Repeat("9", 64) + payload := fmt.Sprintf( + `{"project":"project-a","name":"files","digest":%q,"parentDigest":null,"bundle":{"recipeSchemaVersion":1,"name":"files","translationVersion":"1","outputs":[{"name":"DocumentReference","rootResourceType":"DocumentReference","rowGrain":"document_reference"}]},"createdAt":"2026-02-03T04:05:06Z"}`, + nullDigest) + if _, err := f.iso.Exec(f.ctx, + "INSERT INTO "+f.revisionsTable()+ + " (project, name, digest, parent_digest, payload, created_at)"+ + " VALUES ($1, $2, $3, $4, $5, $6)", + "project-a", "files", nullDigest, "", payload, time.Date(2026, 2, 3, 4, 5, 6, 0, time.UTC)); err != nil { + f.fail("insert json-null parent row", err) + } + if has := f.mustText(t, "SELECT (payload ? 'parentDigest')::text FROM "+f.revisionsTable()+" WHERE digest = $1", nullDigest); has != "true" { + t.Fatalf("json-null parentDigest key presence = %s, want true (null is not absent)", has) + } + if isNull := f.mustText(t, "SELECT COALESCE(jsonb_typeof(payload -> 'parentDigest'), '') FROM "+ + f.revisionsTable()+" WHERE digest = $1", nullDigest); isNull != "null" { + t.Fatalf("json-null parentDigest typeof = %s, want null (JSON null preserved, distinct from absent)", isNull) + } + if absent := f.mustText(t, "SELECT COALESCE(jsonb_typeof(payload -> 'parentDigest'), '') FROM "+ + f.revisionsTable()+" WHERE digest = $1", first.Digest); absent != "" { + t.Fatalf("adapter-written row parentDigest typeof = %s, want absent (missing, not null)", absent) + } + + if _, err := f.iso.Exec(f.ctx, + "INSERT INTO "+f.headsTable()+" (project, name, head_digest, epoch, updated_at)"+ + " VALUES ($1, $2, NULL, 0, $3)", "placeholder-p", "placeholder-r", time.Now().UTC()); err != nil { + f.fail("insert placeholder head", err) + } + list, err := f.registry.List(f.ctx, "placeholder-p", "placeholder-r") + f.fail("list placeholder chain", err) + if len(list) != 0 { + t.Fatalf("placeholder head leaked %d revisions", len(list)) + } + if code := f.mustPgErrorCode(t, "digest set at epoch 0", + "UPDATE "+f.headsTable()+" SET head_digest = $1 WHERE project = $2 AND name = $3", + strings.Repeat("e", 64), "placeholder-p", "placeholder-r"); code != "23514" { + t.Fatalf("digest-set-at-epoch-0 sqlstate = %s, want 23514", code) + } + if got := f.mustText(t, "SELECT head_digest FROM "+f.headsTable()+" WHERE project = $1 AND name = $2", "project-a", "files"); got != first.Digest { + t.Fatalf("head digest = %q, want the registered %q before the clear attempt", got, first.Digest) + } + if code := f.mustPgErrorCode(t, "digest cleared at epoch 1", + "UPDATE "+f.headsTable()+" SET head_digest = NULL WHERE project = $1 AND name = $2", + "project-a", "files"); code != "23514" { + t.Fatalf("digest-cleared-at-epoch-1 sqlstate = %s, want 23514", code) + } + + if code := f.mustPgErrorCode(t, "garbage digest", + "INSERT INTO "+f.revisionsTable()+ + " (project, name, digest, parent_digest, payload, created_at)"+ + " VALUES ('p', 'n', 'nope', '', '{}'::jsonb, now())"); code != "23514" { + t.Fatalf("garbage digest sqlstate = %s, want 23514", code) + } + if code := f.mustPgErrorCode(t, "garbage parent", + "INSERT INTO "+f.revisionsTable()+ + " (project, name, digest, parent_digest, payload, created_at)"+ + " VALUES ('p', 'n', $1, 'garbage', '{}'::jsonb, now())", strings.Repeat("0", 64)); code != "23514" { + t.Fatalf("garbage parent sqlstate = %s, want 23514", code) + } + if code := f.mustPgErrorCode(t, "empty project", + "INSERT INTO "+f.revisionsTable()+ + " (project, name, digest, parent_digest, payload, created_at)"+ + " VALUES ('', 'n', $1, '', '{}'::jsonb, now())", strings.Repeat("0", 64)); code != "23514" { + t.Fatalf("empty project sqlstate = %s, want 23514", code) + } +} + +func TestRevisionNewGuards(t *testing.T) { + f := newChainFixture(t) + if _, err := New(nil, f.schema); err == nil { + t.Fatal("nil pool accepted") + } + for _, bad := range []string{"", "Bad-Schema", "1nvalid", `"; DROP`, strings.Repeat("a", 64)} { + if _, err := New(f.iso, bad); err == nil { + t.Fatalf("unsafe schema %q accepted", bad) + } + } +} + +func TestRevisionMigrateSchemaHistoryLedgerAndRerun(t *testing.T) { + f := newChainFixture(t) + + history, err := ChainMigrations() + f.fail("load chain migrations", err) + if len(history) != 2 || history[0].Version != 1 || history[0].Name != "ledger" || + history[1].Version != 3 || history[1].Name != "recipe_revision_chains" { + t.Fatalf("embedded history = %+v, want {1 ledger, 3 recipe_revision_chains}", history) + } + + rows, err := f.iso.Query(f.ctx, + "SELECT version, name, checksum FROM "+ + pgx.Identifier{f.schema, pgstore.LedgerTable}.Sanitize()+" ORDER BY version") + f.fail("read ledger", err) + defer rows.Close() + type rec struct { + version int64 + name string + checksum string + } + var got []rec + for rows.Next() { + var r rec + if err := rows.Scan(&r.version, &r.name, &r.checksum); err != nil { + f.fail("scan ledger", err) + } + got = append(got, r) + } + f.fail("ledger rows", rows.Err()) + if len(got) != 2 { + t.Fatalf("ledger rows = %d, want 2", len(got)) + } + for i, want := range history { + if got[i].version != want.Version || got[i].name != want.Name || got[i].checksum != want.Checksum { + t.Fatalf("ledger[%d] = %+v, want {%d %s %s}", i, got[i], want.Version, want.Name, want.Checksum) + } + } + + first, err := f.registry.Register(f.ctx, "project-a", bundleFor("files", "1"), "") + f.fail("register before rerun", err) + client, err := pgstore.Open(f.ctx, pgstore.Config{ + DSN: f.dsn, + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + f.fail("open pg-store pool", err) + defer client.Close() + result, err := MigrateSchema(f.ctx, client, f.schema) + f.fail("rerun migrate via production pool", err) + if result.LatestVersion != 3 || len(result.Applied) != 2 { + t.Fatalf("rerun result = %+v, want latest 3 with two applied rows", result) + } + for i, want := range history { + if result.Applied[i].Version != want.Version || result.Applied[i].Checksum != want.Checksum { + t.Fatalf("rerun applied[%d] = %+v, want %d/%s", i, result.Applied[i], want.Version, want.Checksum) + } + } + loaded, err := f.registry.Get(f.ctx, "project-a", "files", first.Digest) + f.fail("get after rerun", err) + requireRevisionEqual(t, first, loaded, "after rerun") +} + +func TestRevisionSchemaMixedHistoryFailsClosed(t *testing.T) { + f := newChainFixture(t) + + edited := pgstore.Migration{Version: 1, Name: "ledger", SQL: "CREATE TABLE schema_migrations (version bigint);"} + runner, err := pgstore.NewMigrationRunner(f.iso, f.schema, []pgstore.Migration{edited}) + f.fail("build edited-ledger runner", err) + if _, err := runner.Migrate(f.ctx); !errors.Is(err, pgstore.ErrMigrationChecksumMismatch) { + t.Fatalf("edited ledger history err = %v, want ErrMigrationChecksumMismatch", err) + } + + outOfOrder := pgstore.Migration{Version: 2, Name: "foreign_table", SQL: "CREATE TABLE IF NOT EXISTS foreign_table (id text);"} + history, err := ChainMigrations() + f.fail("load chain migrations", err) + runner, err = pgstore.NewMigrationRunner(f.iso, f.schema, []pgstore.Migration{ + history[0], outOfOrder, history[1], + }) + f.fail("build out-of-order runner", err) + if _, err := runner.Migrate(f.ctx); !errors.Is(err, pgstore.ErrMigrationOutOfOrder) { + t.Fatalf("pending version below the recorded maximum err = %v, want ErrMigrationOutOfOrder", err) + } + if exists := f.mustText(t, "SELECT (to_regclass($1) IS NOT NULL)::text", + pgx.Identifier{f.schema, "foreign_table"}.Sanitize()); exists != "false" { + t.Fatalf("refused history created its table: %s", exists) + } +} + +func TestRevisionStatePersistsBeyondOneConnection(t *testing.T) { + f := newChainFixture(t) + first, err := f.registry.Register(f.ctx, "project-a", bundleFor("files", "1"), "") + f.fail("register", err) + + fresh, err := pgxpool.New(f.ctx, f.dsn) + f.fail("open fresh pool", err) + defer fresh.Close() + second, err := New(fresh, f.schema) + f.fail("build registry on fresh pool", err) + loaded, err := second.Get(f.ctx, "project-a", "files", first.Digest) + f.fail("get on fresh pool", err) + requireRevisionEqual(t, first, loaded, "fresh pool") +} + +func TestMain(m *testing.M) { + code := m.Run() + if err := auditZeroLeftovers(); err != nil { + fmt.Fprintf(os.Stderr, "zero-leftover audit failed: %v\n", err) + code = 1 + } + os.Exit(code) +} + +func auditZeroLeftovers() error { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return nil + } + auditScrub := func(format string, args ...any) string { + message := fmt.Sprintf(format, args...) + secrets := []string{raw} + if u, err := url.Parse(raw); err == nil && u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + for _, secret := range secrets { + if secret != "" { + message = strings.ReplaceAll(message, secret, "") + } + } + return message + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + return errors.New(auditScrub("connect: %v", err)) + } + defer admin.Close() + names := snapshotCreatedDatabases() + var databases int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname = ANY($1)", names).Scan(&databases); err != nil { + return errors.New(auditScrub("count databases: %v", err)) + } + if databases != 0 { + return errors.New(auditScrub("%d databases created by this binary still exist: %v", databases, names)) + } + var schemas int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgrecip01v1_it_%'").Scan(&schemas); err != nil { + return errors.New(auditScrub("count schemas: %v", err)) + } + if schemas != 0 { + return errors.New(auditScrub("%d pgrecip01v1_it_* schemas still exist in the service database", schemas)) + } + return nil +} diff --git a/internal/backends/postgres/internal/redact/redact.go b/internal/backends/postgres/internal/redact/redact.go new file mode 100644 index 000000000..b29abd0e0 --- /dev/null +++ b/internal/backends/postgres/internal/redact/redact.go @@ -0,0 +1,60 @@ +package redact + +import ( + "net/url" + "regexp" + "strings" +) + +const credentialRedacted = "[redacted]" + +var secretParamPattern = regexp.MustCompile(`(?i)\b(password|sslpassword)\s*=\s*('[^']*'|[^\s]+)`) + +func RedactDSN(dsn string) string { + if dsn == "" { + return dsn + } + if u, err := url.Parse(dsn); err == nil && u.Scheme != "" && u.User != nil { + if _, hasPassword := u.User.Password(); hasPassword { + schemeEnd := strings.Index(dsn, "://") + if schemeEnd >= 0 { + if at := strings.Index(dsn[schemeEnd+3:], "@"); at >= 0 { + return dsn[:schemeEnd+3] + u.User.Username() + ":" + credentialRedacted + dsn[schemeEnd+3+at:] + } + } + } + return dsn + } + return secretParamPattern.ReplaceAllStringFunc(dsn, func(match string) string { + eq := strings.Index(match, "=") + key := strings.TrimSpace(match[:eq]) + value := match[eq+1:] + if strings.HasPrefix(value, "'") { + return key + "='" + credentialRedacted + "'" + } + return key + "=" + credentialRedacted + }) +} + +func ScrubSecretText(text, secret string) string { + if secret == "" { + return text + } + return strings.ReplaceAll(text, secret, credentialRedacted) +} + +func ExtractDSNPassword(dsn string) string { + if u, err := url.Parse(dsn); err == nil && u.Scheme != "" && u.User != nil { + if password, hasPassword := u.User.Password(); hasPassword { + return password + } + } + match := secretParamPattern.FindStringSubmatch(dsn) + if match == nil { + return "" + } + value := match[2] + value = strings.TrimPrefix(value, "'") + value = strings.TrimSuffix(value, "'") + return value +} diff --git a/internal/backends/postgres/internal/redact/redact_internal_test.go b/internal/backends/postgres/internal/redact/redact_internal_test.go new file mode 100644 index 000000000..c9b9683c2 --- /dev/null +++ b/internal/backends/postgres/internal/redact/redact_internal_test.go @@ -0,0 +1,78 @@ +package redact + +import "strings" + +import "testing" + +func TestDSNRedactionKeepsTheRegisteredSecret(t *testing.T) { + const secret = "Ab06Reg1st3redS3cret!" + cases := []struct { + name string + dsn string + mustKeep string + wantExtracted string + }{ + { + name: "url_form", + dsn: "postgres://loom:" + secret + "@db.internal:5432/loom_source?sslmode=require", + mustKeep: "db.internal", + wantExtracted: secret, + }, + { + name: "keyword_form", + dsn: "host=db.internal port=5432 dbname=loom_source user=loom password=" + secret, + mustKeep: "db.internal", + wantExtracted: secret, + }, + { + name: "keyword_form_quoted", + dsn: "host=db.internal password='" + secret + "'", + mustKeep: "db.internal", + wantExtracted: secret, + }, + { + name: "passfile_stays_visible", + dsn: "host=db.internal password=" + secret + " passfile=/etc/loom/pgpass", + mustKeep: "/etc/loom/pgpass", + wantExtracted: secret, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + redacted := RedactDSN(tc.dsn) + if strings.Contains(redacted, secret) { + t.Fatalf("redaction leaked the registered secret: %s", redacted) + } + if !strings.Contains(redacted, tc.mustKeep) { + t.Fatalf("redaction dropped diagnosis material %q: %s", tc.mustKeep, redacted) + } + if !strings.Contains(redacted, credentialRedacted) { + t.Fatalf("redaction marker missing: %s", redacted) + } + if got := ExtractDSNPassword(tc.dsn); got != tc.wantExtracted { + t.Fatalf("extractDSNPassword = %q, want the registered secret", got) + } + }) + } + + t.Run("alive_control_credential_free_dsn_unchanged", func(t *testing.T) { + plain := "postgres://loom@db.internal:5432/loom_source" + if got := RedactDSN(plain); got != plain { + t.Fatalf("credential-free DSN was altered: %s", got) + } + if got := ExtractDSNPassword(plain); got != "" { + t.Fatalf("credential-free DSN yielded a secret: %q", got) + } + }) + + t.Run("scrub_secret_text", func(t *testing.T) { + const secret = "S3cr3t-Pg-Pass" + dirty := `pgx: failed to connect to host=db.internal user=loom password=` + secret + if strings.Contains(ScrubSecretText(dirty, secret), secret) { + t.Fatalf("scrubSecretText left the secret in: %s", dirty) + } + if ScrubSecretText(dirty, "") != dirty { + t.Fatal("empty secret altered ordinary text") + } + }) +} diff --git a/internal/backends/postgres/internal/rows/rows.go b/internal/backends/postgres/internal/rows/rows.go new file mode 100644 index 000000000..7c29377b1 --- /dev/null +++ b/internal/backends/postgres/internal/rows/rows.go @@ -0,0 +1,320 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/execution" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" +) + +type RowVisitor = execution.RowVisitor + +type Querier interface { + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) +} + +var ( + ErrInvalidExecution = errors.New("postgres rows: invalid execution") + ErrMalformedStatement = errors.New("postgres rows: malformed statement") + ErrInvalidParameter = errors.New("postgres rows: invalid parameter") + ErrColumnContractMismatch = errors.New("postgres rows: column contract mismatch") + ErrQuery = errors.New("postgres rows: query failed") +) + +type QueryError struct { + Op string + Err error +} + +func (e *QueryError) Error() string { + return fmt.Sprintf("postgres rows: %s: %v", e.Op, e.Err) +} + +func (e *QueryError) Unwrap() []error { return []error{ErrQuery, e.Err} } + +type Executor struct { + querier Querier +} + +func NewExecutor(querier Querier) (*Executor, error) { + if querier == nil { + return nil, fmt.Errorf("%w: querier is required", ErrInvalidExecution) + } + return &Executor{querier: querier}, nil +} + +var _ execution.SourceRows = (*Executor)(nil).SourceRows + +func (e *Executor) SourceRows(ctx context.Context, request execution.RowsRequest, visit execution.RowVisitor) error { + if ctx == nil { + return fmt.Errorf("%w: context is required", ErrInvalidExecution) + } + if e == nil || e.querier == nil { + return fmt.Errorf("%w: executor has no querier", ErrInvalidExecution) + } + if visit == nil { + return fmt.Errorf("%w: row visitor is required", ErrInvalidExecution) + } + sqlText, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + columns := request.DeclaredColumns + if err := validateColumns(columns); err != nil { + return err + } + if err := validateStatementShape(sqlText, args); err != nil { + return err + } + bound, err := bindParameters(args) + if err != nil { + return err + } + + rows, err := e.querier.Query(ctx, sqlText, bound...) + if err != nil { + if rows != nil { + rows.Close() + } + return classifyExecution(ctx, "query", err) + } + if rows == nil { + return &QueryError{Op: "query", Err: errors.New("driver returned no result set")} + } + return streamRows(ctx, rows, columns, visit) +} + +func streamRows(ctx context.Context, rows pgx.Rows, columns []string, visit RowVisitor) error { + defer rows.Close() + + var targets []any + for rows.Next() { + if targets == nil { + descriptions := rows.FieldDescriptions() + if err := checkColumnContract(columns, descriptions); err != nil { + return err + } + targets = make([]any, len(columns)) + for i, description := range descriptions { + switch description.DataTypeOID { + case pgtype.JSONOID, pgtype.JSONBOID: + targets[i] = new(json.RawMessage) + default: + targets[i] = new(any) + } + } + } + if err := rows.Scan(targets...); err != nil { + return classifyExecution(ctx, "scan", err) + } + row := make(map[string]any, len(columns)) + for i, column := range columns { + row[column] = decodeColumn(targets[i]) + } + if err := visit(row); err != nil { + return err + } + } + if err := rows.Err(); err != nil { + return classifyExecution(ctx, "rows", err) + } + if targets == nil { + if err := checkColumnContract(columns, rows.FieldDescriptions()); err != nil { + return err + } + } + return nil +} + +func validateColumns(columns []string) error { + if len(columns) == 0 { + return fmt.Errorf("%w: at least one declared result column is required", ErrInvalidExecution) + } + seen := make(map[string]struct{}, len(columns)) + for i, column := range columns { + if strings.TrimSpace(column) == "" { + return fmt.Errorf("%w: declared result column %d is blank", ErrInvalidExecution, i+1) + } + if _, duplicate := seen[column]; duplicate { + return fmt.Errorf("%w: declared result column %q is declared twice", ErrInvalidExecution, column) + } + seen[column] = struct{}{} + } + return nil +} + +func checkColumnContract(columns []string, descriptions []pgconn.FieldDescription) error { + if len(descriptions) != len(columns) { + return fmt.Errorf("%w: statement returns %d column(s), contract declares %d", + ErrColumnContractMismatch, len(descriptions), len(columns)) + } + for i, column := range columns { + got := descriptions[i].Name + if got != column && got != "c"+strconv.Itoa(i) { + return fmt.Errorf("%w: result column %d is %q, contract declares %q", + ErrColumnContractMismatch, i+1, got, column) + } + } + return nil +} + +var statementPlaceholderRE = regexp.MustCompile(`\$\d+`) + +func validateStatementShape(sqlText string, args []command.PostgreSQLParameter) error { + matches := statementPlaceholderRE.FindAllString(sqlText, -1) + if len(args) == 0 { + if len(matches) > 0 { + return fmt.Errorf("%w: statement uses $n placeholders but declares no arguments", ErrMalformedStatement) + } + return nil + } + used := make([]bool, len(args)) + for _, match := range matches { + n, err := strconv.Atoi(match[1:]) + if err != nil || n < 1 || n > len(args) { + return fmt.Errorf("%w: placeholder %s is outside the declared argument range 1..%d", + ErrMalformedStatement, match, len(args)) + } + used[n-1] = true + } + for i, appears := range used { + if !appears { + return fmt.Errorf("%w: argument %d never appears as $%d in the statement", + ErrMalformedStatement, i+1, i+1) + } + } + return nil +} + +func bindParameters(args []command.PostgreSQLParameter) ([]any, error) { + bound := make([]any, 0, len(args)) + for i, arg := range args { + value, err := bindParameter(arg) + if err != nil { + return nil, fmt.Errorf("%w: argument %d declared %q carries %T: %w", + ErrInvalidParameter, i+1, string(arg.Type), arg.Value, err) + } + bound = append(bound, value) + } + return bound, nil +} + +func bindParameter(arg command.PostgreSQLParameter) (any, error) { + switch arg.Type { + case command.PostgreSQLParameterNull: + if arg.Value != nil { + return nil, errors.New("an sql-null parameter must carry a nil value; a JSON null belongs on a json or jsonb parameter") + } + return nil, nil + case command.PostgreSQLParameterText: + value, ok := arg.Value.(string) + if !ok { + return nil, errors.New("a text parameter must carry a string") + } + return value, nil + case command.PostgreSQLParameterBigint: + value, ok := arg.Value.(int64) + if !ok { + return nil, errors.New("a bigint parameter must carry an int64") + } + return value, nil + case command.PostgreSQLParameterDoublePrecision: + value, ok := arg.Value.(float64) + if !ok { + return nil, errors.New("a double precision parameter must carry a float64") + } + return value, nil + case command.PostgreSQLParameterBoolean: + value, ok := arg.Value.(bool) + if !ok { + return nil, errors.New("a boolean parameter must carry a bool") + } + return value, nil + case command.PostgreSQLParameterDate: + value, ok := arg.Value.(time.Time) + if !ok { + return nil, errors.New("a date parameter must carry a time.Time") + } + if value.Hour() != 0 || value.Minute() != 0 || value.Second() != 0 || value.Nanosecond() != 0 { + return nil, errors.New("a date parameter must carry a midnight calendar time; absolute instants belong on the timestamp with time zone type") + } + return value, nil + case command.PostgreSQLParameterTimestamptz: + value, ok := arg.Value.(time.Time) + if !ok { + return nil, errors.New("a timestamp with time zone parameter must carry a time.Time") + } + return value, nil + case command.PostgreSQLParameterJSON, command.PostgreSQLParameterJSONB: + raw, ok := jsonBytes(arg.Value) + if !ok { + return nil, errors.New("a json or jsonb parameter must carry a string or json.RawMessage") + } + if !json.Valid(raw) { + return nil, errors.New("a json or jsonb parameter must carry valid JSON") + } + return json.RawMessage(raw), nil + default: + return nil, fmt.Errorf("unregistered parameter type %q", string(arg.Type)) + } +} + +func jsonBytes(value any) ([]byte, bool) { + switch typed := value.(type) { + case json.RawMessage: + return []byte(typed), true + case string: + return []byte(typed), true + default: + return nil, false + } +} + +func decodeColumn(target any) any { + switch typed := target.(type) { + case *json.RawMessage: + raw := *typed + if raw == nil { + return nil + } + buf := make([]byte, len(raw)) + copy(buf, raw) + return json.RawMessage(buf) + case *any: + value := *typed + if raw, ok := value.([]byte); ok { + buf := make([]byte, len(raw)) + copy(buf, raw) + return buf + } + return value + default: + return nil + } +} + +func classifyExecution(ctx context.Context, op string, err error) error { + if err == nil { + return nil + } + if ctx != nil { + if cause := ctx.Err(); cause != nil && !errors.Is(err, cause) { + return &QueryError{Op: op, Err: cause} + } + } + if translated, ok := storepostgres.TranslateError(err); ok { + return &QueryError{Op: op, Err: translated} + } + return &QueryError{Op: op, Err: err} +} diff --git a/internal/backends/postgres/internal/rows/rows_boundary_pg18_ab03_test.go b/internal/backends/postgres/internal/rows/rows_boundary_pg18_ab03_test.go new file mode 100644 index 000000000..cc25a3cbc --- /dev/null +++ b/internal/backends/postgres/internal/rows/rows_boundary_pg18_ab03_test.go @@ -0,0 +1,468 @@ +package postgres + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "github.com/calypr/loom/internal/dataframe/execution" + "net/url" + "os" + "regexp" + "strconv" + "strings" + "sync" + "testing" + "time" + + storepg "github.com/calypr/loom/internal/backends/postgres/internal/store" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ab03Prefix = "verify_ab03_boundary_" + +var ( + ab03NamePattern = regexp.MustCompile(`^verify_ab03_boundary_[a-z0-9_]+$`) + + ab03OwnedMu sync.Mutex + ab03OwnedDBs []string +) + +func ab03URL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 boundary-through witnesses") + } + return raw +} + +func ab03RandomHex(t *testing.T) string { + t.Helper() + buf := make([]byte, 8) + if _, err := rand.Read(buf); err != nil { + t.Fatalf("generate fixture entropy: %v", err) + } + return hex.EncodeToString(buf) +} + +type ab03Service struct { + ctx context.Context + dsn string + scrub func(any) string + fail func(step string, err error) +} + +func newAB03Service(t *testing.T) *ab03Service { + t.Helper() + raw := ab03URL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNumText string + var version string + if err := admin.QueryRow(ctx, "SHOW server_version_num").Scan(&versionNumText); err != nil { + fail("read server_version_num", err) + } + if err := admin.QueryRow(ctx, "SELECT version()").Scan(&version); err != nil { + fail("read version()", err) + } + versionNum, err := strconv.ParseInt(strings.TrimSpace(versionNumText), 10, 64) + if err != nil { + fail("parse server_version_num", err) + } + if err := storepg.CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required backend: %v", err) + } + + dbName := fmt.Sprintf("%s%s_%s", ab03Prefix, time.Now().Format("20060102150405"), ab03RandomHex(t)) + if !ab03NamePattern.MatchString(dbName) { + t.Fatalf("owned database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create owned database", err) + } + ab03OwnedMu.Lock() + ab03OwnedDBs = append(ab03OwnedDBs, dbName) + ab03OwnedMu.Unlock() + t.Cleanup(func() { + if _, err := admin.Exec(context.Background(), `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop owned database %s: %s", dbName, scrub(err)) + } + }) + + svcDSN := *u + svcDSN.Path = "/" + dbName + return &ab03Service{ctx: ctx, dsn: svcDSN.String(), scrub: scrub, fail: fail} +} + +func (s *ab03Service) ab03Conn(t *testing.T) (*pgx.Conn, *Executor) { + t.Helper() + conn, err := pgx.Connect(s.ctx, s.dsn) + if err != nil { + s.fail("connect owned database", err) + } + t.Cleanup(func() { conn.Close(context.Background()) }) + executor, err := NewExecutor(conn) + if err != nil { + s.fail("build executor", err) + } + return conn, executor +} + +func ab03RequireTranslated(t *testing.T, err error, wantOp string, wantCondition storepg.DriverCondition, wantState string) *storepg.TranslatedError { + t.Helper() + if err == nil { + t.Fatalf("expected failure, got nil") + } + if !errors.Is(err, ErrQuery) { + t.Fatalf("error %v does not match ErrQuery", err) + } + var qe *QueryError + if !errors.As(err, &qe) { + t.Fatalf("error %v is not a *QueryError", err) + } + if !strings.HasPrefix(err.Error(), "postgres rows: "+wantOp+":") { + t.Fatalf("error %q did not surface at seam %q", err, wantOp) + } + var translated *storepg.TranslatedError + if !errors.As(err, &translated) { + t.Fatalf("error %v lost the boundary classification", err) + } + if translated.Condition != wantCondition || translated.SQLState != wantState { + t.Fatalf("boundary classification = (%s, %q), want (%s, %q) in error %v", + translated.Condition, translated.SQLState, wantCondition, wantState, err) + } + return translated +} + +func ab03RequireCause(t *testing.T, err error, translated *storepg.TranslatedError, wantState, wantMessageFragment string) { + t.Helper() + if strings.Contains(err.Error(), wantMessageFragment) { + t.Fatalf("public error %q discloses server text %q", err, wantMessageFragment) + } + var userErr, isUser = dataframeerrors.AsUserError(err) + if isUser && strings.Contains(userErr.Error(), wantMessageFragment) { + t.Fatalf("user error rendering %q discloses server text %q", userErr.Error(), wantMessageFragment) + } + var pgErr *pgconn.PgError + if !errors.As(translated.Cause(), &pgErr) { + t.Fatalf("translated error lost the *pgconn.PgError cause through Cause(): %v", err) + } + if pgErr.SQLState() != wantState { + t.Fatalf("cause SQLSTATE = %q, want %q", pgErr.SQLState(), wantState) + } + if !strings.Contains(pgErr.Message, wantMessageFragment) { + t.Fatalf("cause message %q does not contain %q", pgErr.Message, wantMessageFragment) + } +} + +func ab03RequireCode(t *testing.T, err error, want dataframeerrors.ErrorCode) { + t.Helper() + userErr, ok := dataframeerrors.AsUserError(err) + if !ok { + t.Fatalf("error %v carries no Loom user-error identity", err) + } + if userErr.Code() != string(want) { + t.Fatalf("code = %s, want %s", userErr.Code(), want) + } +} + +func TestAB03BoundaryThroughQueryCanceled(t *testing.T) { + svc := newAB03Service(t) + conn, executor := svc.ab03Conn(t) + _, controlExecutor := svc.ab03Conn(t) + + rows := 0 + if err := controlExecutor.SourceRows(svc.ctx, execution.RowsRequest{Command: pgSQL(t, `SELECT 1::bigint AS one UNION ALL SELECT 2::bigint`), DeclaredColumns: []string{"one"}}, func(map[string]any) error { rows++; return nil }); err != nil { + svc.fail("happy-stream control", err) + } + if rows != 2 { + t.Fatalf("control streamed %d rows, want 2", rows) + } + + if _, err := conn.Exec(svc.ctx, `SET statement_timeout = '80ms'`); err != nil { + svc.fail("set statement_timeout", err) + } + err := executor.SourceRows(svc.ctx, execution.RowsRequest{Command: pgSQL(t, `SELECT pg_sleep(5) AS ab03_sleep_witness`), DeclaredColumns: []string{"ab03_sleep_witness"}}, func(map[string]any) error { return nil }) + translated := ab03RequireTranslated(t, err, "rows", storepg.ConditionQueryCanceled, "57014") + ab03RequireCode(t, err, dataframeerrors.CodeQueryResourceLimitExceeded) + ab03RequireCause(t, err, translated, "57014", "canceling statement due to statement timeout") + if strings.Contains(err.Error(), "pg_sleep") || strings.Contains(err.Error(), "ab03_sleep_witness") { + t.Fatalf("public error %q discloses statement text", err) + } + + reapplied, ok := storepg.TranslateError(err) + if !ok { + t.Fatal("re-applying the boundary to a classified error reported not-owned") + } + if reapplied != err { + t.Fatalf("re-application changed the error identity: %v", reapplied) + } + var again *storepg.TranslatedError + if !errors.As(reapplied, &again) || again != translated { + t.Fatal("re-application changed the classification identity") + } +} + +func TestAB03BoundaryThroughServerRejected(t *testing.T) { + svc := newAB03Service(t) + _, executor := svc.ab03Conn(t) + + const missingRelation = "verify_ab03_missing_relation" + err := executor.SourceRows(svc.ctx, execution.RowsRequest{Command: pgSQL(t, `SELECT id FROM `+missingRelation), DeclaredColumns: []string{"id"}}, func(map[string]any) error { return nil }) + translated := ab03RequireTranslated(t, err, "query", storepg.ConditionServerRejected, "42P01") + ab03RequireCode(t, err, dataframeerrors.CodeInternalError) + ab03RequireCause(t, err, translated, "42P01", `relation "`+missingRelation+`" does not exist`) + + userErr, ok := dataframeerrors.AsUserError(err) + if !ok { + t.Fatal("boundary error carries no user-error identity") + } + for _, key := range []string{"backend", "condition", "sqlstate"} { + if _, present := userErr.Details()[key]; !present { + t.Fatalf("public details missing %q: %#v", key, userErr.Details()) + } + } + if _, leaked := userErr.Details()["message"]; leaked { + t.Fatalf("public details carry a server message: %#v", userErr.Details()) + } +} + +func TestAB03BoundaryThroughAbortedTransaction(t *testing.T) { + svc := newAB03Service(t) + conn, executor := svc.ab03Conn(t) + + if _, err := conn.Exec(svc.ctx, `BEGIN`); err != nil { + svc.fail("begin transaction", err) + } + if _, err := conn.Exec(svc.ctx, `SELECT 1/0 AS ab03_abort`); err == nil { + t.Fatal("abort control statement unexpectedly succeeded") + } + err := executor.SourceRows(svc.ctx, execution.RowsRequest{Command: pgSQL(t, `SELECT 1::bigint AS one`), DeclaredColumns: []string{"one"}}, func(map[string]any) error { return nil }) + translated := ab03RequireTranslated(t, err, "query", storepg.ConditionAbortedTransaction, "25P02") + ab03RequireCode(t, err, dataframeerrors.CodeInternalError) + ab03RequireCause(t, err, translated, "25P02", "current transaction is aborted") + if !storepg.IsAbortedTransaction(err) { + t.Fatalf("adapter error not recognized by IsAbortedTransaction: %v", err) + } + + if _, err := conn.Exec(svc.ctx, `ROLLBACK`); err != nil { + svc.fail("rollback", err) + } + streamed := 0 + if err := executor.SourceRows(svc.ctx, execution.RowsRequest{Command: pgSQL(t, `SELECT 1::bigint AS one`), DeclaredColumns: []string{"one"}}, func(map[string]any) error { streamed++; return nil }); err != nil { + svc.fail("post-rollback control", err) + } + if streamed != 1 { + t.Fatalf("post-rollback control streamed %d rows, want 1", streamed) + } +} + +func TestAB03BoundaryThroughAdminShutdown(t *testing.T) { + svc := newAB03Service(t) + _, executor := svc.ab03Conn(t) + terminator, err := pgx.Connect(svc.ctx, svc.dsn) + if err != nil { + svc.fail("connect terminator", err) + } + defer terminator.Close(context.Background()) + + marker := fmt.Sprintf("/*ab03-terminate-%s*/", ab03RandomHex(t)) + errc := make(chan error, 1) + go func() { + errc <- executor.SourceRows(svc.ctx, execution.RowsRequest{Command: pgSQL(t, `SELECT `+marker+` pg_sleep(10) AS ab03_tick`), DeclaredColumns: []string{"ab03_tick"}}, func(map[string]any) error { return nil }) + }() + + var pid int + for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); { + if err := terminator.QueryRow(svc.ctx, + `SELECT pid FROM pg_stat_activity WHERE state = 'active' AND query LIKE $1 LIMIT 1`, + `%`+marker+`%`).Scan(&pid); err == nil { + break + } + select { + case err := <-errc: + t.Fatalf("in-flight query ended before it could be terminated: %v", err) + default: + } + time.Sleep(20 * time.Millisecond) + } + if pid == 0 { + t.Fatal("never observed the in-flight backend") + } + if _, err := terminator.Exec(svc.ctx, `SELECT pg_terminate_backend($1)`, pid); err != nil { + svc.fail("terminate in-flight backend", err) + } + err = <-errc + translated := ab03RequireTranslated(t, err, "rows", storepg.ConditionAdminShutdown, "57P01") + ab03RequireCode(t, err, dataframeerrors.CodeBackendUnavailable) + if !errors.Is(err, storepg.ErrConnect) { + t.Fatalf("admin shutdown lost the ErrConnect sentinel: %v", err) + } + userErr, ok := dataframeerrors.AsUserError(err) + if !ok || !userErr.Retryable() { + t.Fatalf("admin shutdown must be retryable backend-unavailable: %v", err) + } + ab03RequireCause(t, err, translated, "57P01", "terminating connection due to administrator command") +} + +func TestAB03BoundaryThroughConnectionFailure(t *testing.T) { + svc := newAB03Service(t) + pool, err := pgxpool.New(svc.ctx, `postgres://ab03:ab03@127.0.0.1:1/ab03_none?connect_timeout=2`) + if err != nil { + svc.fail("build unreachable pool", err) + } + defer pool.Close() + executor, err := NewExecutor(pool) + if err != nil { + svc.fail("build pool executor", err) + } + queryCtx, cancel := context.WithTimeout(svc.ctx, 30*time.Second) + defer cancel() + err = executor.SourceRows(queryCtx, execution.RowsRequest{Command: pgSQL(t, `SELECT 1::bigint AS one`), DeclaredColumns: []string{"one"}}, func(map[string]any) error { return nil }) + translated := ab03RequireTranslated(t, err, "query", storepg.ConditionConnectionFailure, "") + ab03RequireCode(t, err, dataframeerrors.CodeBackendUnavailable) + if !errors.Is(err, storepg.ErrConnect) { + t.Fatalf("connection failure lost the ErrConnect sentinel: %v", err) + } + userErr, ok := dataframeerrors.AsUserError(err) + if !ok || !userErr.Retryable() { + t.Fatalf("connection failure must be retryable: %v", err) + } + if strings.Contains(err.Error(), "127.0.0.1") || strings.Contains(err.Error(), "ab03:") { + t.Fatalf("public error %q discloses transport or credential material", err) + } + if translated.Cause() == nil { + t.Fatal("connection failure lost its cause") + } +} + +func TestAB03BoundaryThroughContextCancel(t *testing.T) { + svc := newAB03Service(t) + _, executor := svc.ab03Conn(t) + + ctx, cancel := context.WithCancel(svc.ctx) + errc := make(chan error, 1) + go func() { + errc <- executor.SourceRows(ctx, execution.RowsRequest{Command: pgSQL(t, `SELECT pg_sleep(10) AS one`), DeclaredColumns: []string{"one"}}, func(map[string]any) error { return nil }) + }() + time.Sleep(150 * time.Millisecond) + cancel() + err := <-errc + if err == nil { + t.Fatal("canceled query unexpectedly succeeded") + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled query not Is-compatible with context.Canceled: %v", err) + } + var translated *storepg.TranslatedError + if errors.As(err, &translated) && translated.Condition != storepg.ConditionRequestCanceled { + t.Fatalf("caller cancellation invented vendor condition %s", translated.Condition) + } +} + +func TestZZZeroLeftoverAB03Fixtures(t *testing.T) { + raw := ab03URL(t) + ab03OwnedMu.Lock() + owned := append([]string(nil), ab03OwnedDBs...) + ab03OwnedMu.Unlock() + if len(owned) == 0 { + t.Skip("this process created no verify_ab03_boundary_* fixtures") + } + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service for audit: %v", err) + } + defer admin.Close() + + for _, db := range owned { + if !strings.HasPrefix(db, ab03Prefix) { + t.Fatalf("registry holds foreign database %q", db) + } + var count int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Errorf("audit owned database %s: %v", db, err) + continue + } + if count != 0 { + t.Errorf("leftover owned database %s after all cleanups", db) + } + } + var survivors []string + deadline := time.Now().Add(120 * time.Second) + for { + survivors = nil + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE $1 ESCAPE '\\'", `verify\_ab03\_%`) + if err != nil { + t.Fatalf("prefix audit scan: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("prefix audit scan: %v", err) + } + known := false + for _, db := range owned { + if db == name { + known = true + break + } + } + if !known { + survivors = append(survivors, name) + } + } + if err := rows.Err(); err != nil { + rows.Close() + t.Fatalf("prefix audit scan: %v", err) + } + rows.Close() + if len(survivors) == 0 || time.Now().After(deadline) { + break + } + time.Sleep(2 * time.Second) + } + if len(survivors) != 0 { + t.Errorf("verify_ab03_* survivor databases after grace period: %v", survivors) + } +} diff --git a/internal/backends/postgres/internal/rows/rows_postgres_test.go b/internal/backends/postgres/internal/rows/rows_postgres_test.go new file mode 100644 index 000000000..40efcb887 --- /dev/null +++ b/internal/backends/postgres/internal/rows/rows_postgres_test.go @@ -0,0 +1,1352 @@ +package postgres + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "github.com/calypr/loom/internal/dataframe/execution" + "net/url" + "os" + "regexp" + "strconv" + "strings" + "sync" + "testing" + "time" + + storepg "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + exec01Prefix = "verify_exec01_" + exec01NameFmt = exec01Prefix + "%d_%s" +) + +var exec01NamePattern = regexp.MustCompile(`^verify_exec01_[0-9]+_[0-9a-f]+$`) + +var ( + exec01OwnedMu sync.Mutex + exec01OwnedByName = map[string]string{} +) + +func exec01Register(db, schema string) { + exec01OwnedMu.Lock() + defer exec01OwnedMu.Unlock() + exec01OwnedByName[db] = schema +} + +var ( + exec01BaselineOnce sync.Once + exec01BaselineCount int64 + exec01BaselineErr error +) + +func exec01CaptureBaseline(raw string) { + exec01BaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec01BaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_exec01\\_%' ESCAPE '\\'"). + Scan(&count); err != nil { + exec01BaselineErr = fmt.Errorf("count baseline: %w", err) + return + } + exec01BaselineCount = count + }) +} + +func exec01URL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 integration tests") + } + return raw +} + +type exec01Service struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func newExec01Service(t *testing.T) *exec01Service { + t.Helper() + raw := exec01URL(t) + exec01CaptureBaseline(raw) + if exec01BaselineErr != nil { + t.Fatalf("capture pg_database baseline: %v", exec01BaselineErr) + } + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, + "SELECT current_setting('server_version_num')::int8, current_setting('server_version')"). + Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if err := storepg.CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required backend: %v", err) + } + + dbName := fmt.Sprintf(exec01NameFmt, time.Now().UnixNano(), exec01RandomHex(t)) + if !exec01NamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(exec01NameFmt, time.Now().UnixNano(), exec01RandomHex(t)) + if !exec01NamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + exec01Register(dbName, schema) + return &exec01Service{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} +} + +func exec01RandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +func (s *exec01Service) pool(t *testing.T, maxConns int32, suffix string) *pgxpool.Pool { + t.Helper() + return s.poolOn(t, s.dsn, maxConns, suffix) +} + +func (s *exec01Service) poolOn(t *testing.T, dsn string, maxConns int32, suffix string) *pgxpool.Pool { + t.Helper() + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + s.fail("parse pool config", err) + } + cfg.MaxConns = maxConns + cfg.MinConns = 0 + cfg.ConnConfig.RuntimeParams["application_name"] = exec01Prefix + suffix + pool, err := pgxpool.NewWithConfig(s.ctx, cfg) + if err != nil { + s.fail("build pool", err) + } + t.Cleanup(pool.Close) + return pool +} + +func (s *exec01Service) exec(t *testing.T, pool Querier) *Executor { + t.Helper() + executor, err := NewExecutor(pool) + if err != nil { + s.fail("NewExecutor", err) + } + return executor +} + +func (s *exec01Service) table(name string) string { return `"` + s.schema + `"."` + name + `"` } + +func (s *exec01Service) waitForActive(t *testing.T, app, tag string) int64 { + t.Helper() + var pid int64 + deadline := time.Now().Add(10 * time.Second) + for { + if err := s.admin.QueryRow(s.ctx, + `SELECT pid FROM pg_stat_activity + WHERE application_name = $1 AND query LIKE $2 AND state = 'active' AND pid <> pg_backend_pid() LIMIT 1`, + app, "%"+tag+"%").Scan(&pid); err == nil { + return pid + } + if time.Now().After(deadline) { + s.fail("tagged stream never became active on the server", fmt.Errorf("app %q tag %q", app, tag)) + } + time.Sleep(10 * time.Millisecond) + } +} + +func pgSQL(t *testing.T, sql string, args ...command.PostgreSQLParameter) command.CompiledCommand { + t.Helper() + compiledCommand, err := command.NewPostgreSQLCommand(sql, args) + if err != nil { + t.Fatalf("NewPostgreSQLCommand(%q): %v", sql, err) + } + return compiledCommand +} + +func bigint(v int64) command.PostgreSQLParameter { + return command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: v} +} + +func textParam(v string) command.PostgreSQLParameter { + return command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: v} +} + +func jsonbParam(raw string) command.PostgreSQLParameter { + return command.PostgreSQLParameter{Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(raw)} +} + +func sqlNullParam() command.PostgreSQLParameter { + return command.PostgreSQLParameter{Type: command.PostgreSQLParameterNull, Value: nil} +} + +func expectQueryError(t *testing.T, err error, wantOp, wantState, wantMessage string) { + t.Helper() + if !errors.Is(err, ErrQuery) { + t.Fatalf("error %v does not match ErrQuery", err) + } + var qe *QueryError + if !errors.As(err, &qe) { + t.Fatalf("error %v is not a *QueryError", err) + } + if !strings.HasPrefix(err.Error(), "postgres rows: "+wantOp+":") { + t.Fatalf("error %q did not surface at seam %q", err, wantOp) + } + var translated *storepg.TranslatedError + if !errors.As(err, &translated) { + t.Fatalf("error %v lost the boundary classification", err) + } + if translated.SQLState != wantState { + t.Fatalf("SQLSTATE = %q, want %q (error %v)", translated.SQLState, wantState, err) + } + if strings.Contains(err.Error(), wantMessage) { + t.Fatalf("public error %q discloses the server message %q", err, wantMessage) + } + var pgErr *pgconn.PgError + if !errors.As(translated.Cause(), &pgErr) { + t.Fatalf("translated error lost the *pgconn.PgError cause through Cause(): %v", err) + } + if !strings.Contains(pgErr.Message, wantMessage) { + t.Fatalf("server message %q does not contain %q", pgErr.Message, wantMessage) + } +} + +const ( + goldenLabel1 = "alpha" + goldenLabel2 = "beta" + goldenDate1 = "2026-02-13" + goldenDate2 = "2025-12-31" + goldenTS1 = "2026-02-13T10:20:30Z" + goldenTS2 = "2025-12-31T23:59:59.5Z" + goldenNote2 = "plain note; with punctuation" + goldenMeta1Raw = `{"z": 1,"a":2}` + goldenPayload1Read = `{"a": 1, "b": 2}` + goldenPayload2Read = `[true, null]` + + goldenPunctKill = `'); DROP TABLE exec01_punct; --` + goldenPunctStudents = `Robert'); DROP TABLE students;--` + goldenPunctDollar = `cost $1 plus 'quotes' and "dquotes"` + goldenPunctBackslash = `\\'; SELECT count(*) FROM pg_class; --` + + goldenRaceDigest = "1|alpha|1.5;2|beta|-0.25;3|gamma|7" + goldenRaceDigestDirty = "1|alpha|1001.5;2|beta|-0.25;3|gamma|7" +) + +func goldenDate(t *testing.T, literal string) time.Time { + t.Helper() + parsed, err := time.Parse("2006-01-02", literal) + if err != nil { + t.Fatalf("golden date literal %q: %v", literal, err) + } + return parsed +} + +func goldenInstant(t *testing.T, literal string) time.Time { + t.Helper() + parsed, err := time.Parse(time.RFC3339Nano, literal) + if err != nil { + t.Fatalf("golden timestamp literal %q: %v", literal, err) + } + return parsed +} + +func TestExec01TempTableStreamsExactTypedRows(t *testing.T) { + s := newExec01Service(t) + pool := s.pool(t, 2, "temp") + conn, err := pool.Acquire(s.ctx) + if err != nil { + s.fail("acquire pinned connection", err) + } + defer conn.Release() + executor := s.exec(t, conn) + + if _, err := conn.Exec(s.ctx, `CREATE TEMP TABLE exec01_typed_rows( + id bigint NOT NULL PRIMARY KEY, + label text NOT NULL, + score double precision NOT NULL, + active boolean NOT NULL, + event_date date NOT NULL, + event_ts timestamptz NOT NULL, + payload jsonb NOT NULL, + meta json NOT NULL, + note text)`); err != nil { + s.fail("create temp fixture table", err) + } + if _, err := conn.Exec(s.ctx, `INSERT INTO exec01_typed_rows + (id, label, score, active, event_date, event_ts, payload, meta, note) VALUES + (1, 'alpha', 1.5, true, '2026-02-13', '2026-02-13T10:20:30Z', '{"b": 2, "a": 1}'::jsonb, '{"z": 1,"a":2}'::json, NULL), + (2, 'beta', -0.25, false, '2025-12-31', '2025-12-31T23:59:59.5Z', '[true, null]'::jsonb, 'null'::json, 'plain note; with punctuation')`); err != nil { + s.fail("insert temp fixture rows", err) + } + + columns := []string{"id", "label", "score", "active", "event_date", "event_ts", "payload", "meta", "note"} + compiledCommand := pgSQL(t, `SELECT id, label, score, active, event_date, event_ts, payload, meta, note + FROM exec01_typed_rows WHERE id IN ($1, $2) ORDER BY id`, bigint(1), bigint(2)) + + var visited []map[string]any + if err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: compiledCommand, DeclaredColumns: columns}, func(row map[string]any) error { + visited = append(visited, row) + return nil + }); err != nil { + s.fail("stream typed rows", err) + } + if len(visited) != 2 { + t.Fatalf("visited %d rows, want exactly 2", len(visited)) + } + + for i, row := range visited { + if len(row) != len(columns) { + t.Fatalf("row %d has %d keys, want exactly the %d declared columns", i, len(row), len(columns)) + } + for _, column := range columns { + if _, ok := row[column]; !ok { + t.Fatalf("row %d is missing declared column %q", i, column) + } + } + } + + first, second := visited[0], visited[1] + if first["id"] != int64(1) || second["id"] != int64(2) { + t.Fatalf("ids = %#v, %#v, want int64(1), int64(2)", first["id"], second["id"]) + } + if first["label"] != goldenLabel1 || second["label"] != goldenLabel2 { + t.Fatalf("labels = %#v, %#v, want %q, %q", first["label"], second["label"], goldenLabel1, goldenLabel2) + } + if first["score"] != 1.5 || second["score"] != -0.25 { + t.Fatalf("scores = %#v, %#v, want 1.5, -0.25", first["score"], second["score"]) + } + if first["active"] != true || second["active"] != false { + t.Fatalf("actives = %#v, %#v, want true, false", first["active"], second["active"]) + } + if day, ok := first["event_date"].(time.Time); !ok || !day.Equal(goldenDate(t, goldenDate1)) { + t.Fatalf("event_date row 1 = %#v, want %s", first["event_date"], goldenDate1) + } + if day, ok := second["event_date"].(time.Time); !ok || !day.Equal(goldenDate(t, goldenDate2)) { + t.Fatalf("event_date row 2 = %#v, want %s", second["event_date"], goldenDate2) + } + if stamp, ok := first["event_ts"].(time.Time); !ok || stamp.UTC().Format(time.RFC3339Nano) != goldenTS1 { + t.Fatalf("event_ts row 1 = %#v, want %s", first["event_ts"], goldenTS1) + } + if stamp, ok := second["event_ts"].(time.Time); !ok || stamp.UTC().Format(time.RFC3339Nano) != goldenTS2 { + t.Fatalf("event_ts row 2 = %#v, want %s", second["event_ts"], goldenTS2) + } + if raw, ok := first["payload"].(json.RawMessage); !ok || string(raw) != goldenPayload1Read { + t.Fatalf("payload row 1 = %#v, want json.RawMessage(%q)", first["payload"], goldenPayload1Read) + } + if raw, ok := second["payload"].(json.RawMessage); !ok || string(raw) != goldenPayload2Read { + t.Fatalf("payload row 2 = %#v, want json.RawMessage(%q)", second["payload"], goldenPayload2Read) + } + if raw, ok := first["meta"].(json.RawMessage); !ok || string(raw) != goldenMeta1Raw { + t.Fatalf("meta row 1 = %#v, want verbatim json.RawMessage(%q)", first["meta"], goldenMeta1Raw) + } + if raw, ok := second["meta"].(json.RawMessage); !ok || string(raw) != "null" { + t.Fatalf("meta row 2 = %#v, want JSON null value", second["meta"]) + } + if first["note"] != nil { + t.Fatalf("note row 1 = %#v, want nil for SQL NULL", first["note"]) + } + if second["note"] != goldenNote2 { + t.Fatalf("note row 2 = %#v, want %q", second["note"], goldenNote2) + } + + empty := pgSQL(t, `SELECT id, label, score, active, event_date, event_ts, payload, meta, note + FROM exec01_typed_rows WHERE id IN ($1, $2) ORDER BY id`, bigint(98), bigint(99)) + emptyVisits := 0 + if err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: empty, DeclaredColumns: columns}, func(map[string]any) error { emptyVisits++; return nil }); err != nil { + s.fail("empty control stream", err) + } + if emptyVisits != 0 { + t.Fatalf("empty control visited %d rows, want 0", emptyVisits) + } +} + +func TestExec01BoundPunctuationRemainsAValue(t *testing.T) { + s := newExec01Service(t) + pool := s.pool(t, 4, "punct") + executor := s.exec(t, pool) + + if _, err := s.iso.Exec(s.ctx, `CREATE TABLE `+s.table("exec01_punct")+`( + label text PRIMARY KEY, body text NOT NULL)`); err != nil { + s.fail("create punctuation fixture", err) + } + if _, err := s.iso.Exec(s.ctx, `INSERT INTO `+s.table("exec01_punct")+`(label, body) VALUES ($1, $2), ($3, $4)`, + goldenPunctKill, goldenPunctStudents, goldenPunctDollar, goldenPunctBackslash); err != nil { + s.fail("insert punctuation fixtures", err) + } + + bind := pgSQL(t, `SELECT label, body FROM `+s.table("exec01_punct")+` WHERE label = $1 AND body = $2`, + textParam(goldenPunctKill), textParam(goldenPunctStudents)) + var rows []map[string]any + if err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: bind, DeclaredColumns: []string{"label", "body"}}, func(row map[string]any) error { + rows = append(rows, row) + return nil + }); err != nil { + s.fail("bound punctuation stream", err) + } + if len(rows) != 1 { + t.Fatalf("bound punctuation stream returned %d rows, want 1", len(rows)) + } + if rows[0]["label"] != goldenPunctKill || rows[0]["body"] != goldenPunctStudents { + t.Fatalf("bound values altered: %#v", rows[0]) + } + + dollar := pgSQL(t, `SELECT label, body FROM `+s.table("exec01_punct")+` WHERE label = $1`, textParam(goldenPunctDollar)) + if err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: dollar, DeclaredColumns: []string{"label", "body"}}, func(row map[string]any) error { + if row["label"] != goldenPunctDollar || row["body"] != goldenPunctBackslash { + t.Fatalf("dollar-quoted value altered: %#v", row) + } + return nil + }); err != nil { + s.fail("dollar payload stream", err) + } + + var rowCount int64 + if err := s.iso.QueryRow(s.ctx, `SELECT count(*) FROM `+s.table("exec01_punct")).Scan(&rowCount); err != nil { + s.fail("count punctuation rows", err) + } + if rowCount != 2 { + t.Fatalf("fixture table holds %d rows after bound punctuation, want 2 (shape changed)", rowCount) + } + var killedTable int64 + if err := s.iso.QueryRow(s.ctx, `SELECT count(*) FROM pg_class WHERE relname = 'exec01_punct'`).Scan(&killedTable); err != nil { + s.fail("inspect pg_class", err) + } + if killedTable != 1 { + t.Fatalf("exec01_punct appears %d times in pg_class, want exactly 1", killedTable) + } + + interpolated := pgSQL(t, `SELECT label FROM `+s.table("exec01_punct")+` WHERE label = '`+goldenPunctKill+`'`) + interpolatedVisits := 0 + err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: interpolated, DeclaredColumns: []string{"label"}}, func(map[string]any) error { interpolatedVisits++; return nil }) + expectQueryError(t, err, "query", "42601", `syntax error at or near ")"`) + if interpolatedVisits != 0 { + t.Fatalf("interpolated payload visited %d rows, want 0", interpolatedVisits) + } + + witnessSQL := `SELECT pg_backend_pid() AS backend, pg_sleep(0.3) IS NOT NULL AS slept, $1::text AS payload, repeat(md5('exec01-witness'), 2000) AS pad FROM generate_series(1, 2) g` + witness := pgSQL(t, witnessSQL, textParam(goldenPunctKill)) + witnessVisits := 0 + witnessStarted := time.Now() + err = executor.SourceRows(s.ctx, execution.RowsRequest{Command: witness, DeclaredColumns: []string{"backend", "slept", "payload", "pad"}}, func(row map[string]any) error { + witnessVisits++ + if witnessVisits != 1 { + return nil + } + if elapsed := time.Since(witnessStarted); elapsed < 250*time.Millisecond { + t.Fatalf("witness row 1 arrived after %v; the in-flight window was not real", elapsed) + } + backend, ok := row["backend"].(int32) + if !ok { + t.Fatalf("backend column decoded as %#v", row["backend"]) + } + var activityQuery, state string + if err := s.admin.QueryRow(s.ctx, + `SELECT query, state FROM pg_stat_activity WHERE pid = $1 AND pid <> pg_backend_pid()`, int64(backend)). + Scan(&activityQuery, &state); err != nil { + s.fail("read live activity", err) + } + if state != "active" { + t.Fatalf("stream was not still executing when row 1 reached the visitor: state=%q", state) + } + if !strings.Contains(activityQuery, "$1") { + t.Fatalf("live server statement is not parameterized: %q", activityQuery) + } + if strings.Contains(activityQuery, "DROP TABLE") || strings.Contains(activityQuery, goldenPunctKill) { + t.Fatalf("bound value leaked into the server statement text: %q", activityQuery) + } + if row["payload"] != goldenPunctKill { + t.Fatalf("bound payload altered mid-stream: %#v", row["payload"]) + } + return nil + }) + if err != nil { + s.fail("activity-witness stream", err) + } + if witnessVisits != 2 { + t.Fatalf("activity-witness stream visited %d rows, want 2", witnessVisits) + } +} + +func TestExec01VisitorErrorStopsIterationAndClosesRows(t *testing.T) { + s := newExec01Service(t) + pool := s.pool(t, 2, "visitor") + conn, err := pool.Acquire(s.ctx) + if err != nil { + s.fail("acquire pinned connection", err) + } + defer conn.Release() + executor := s.exec(t, conn) + + if _, err := conn.Exec(s.ctx, `CREATE TEMP TABLE exec01_three(id bigint PRIMARY KEY, label text NOT NULL)`); err != nil { + s.fail("create three-row fixture", err) + } + if _, err := conn.Exec(s.ctx, `INSERT INTO exec01_three(id, label) VALUES (1, 'a'), (2, 'b'), (3, 'c')`); err != nil { + s.fail("insert three rows", err) + } + + streamSQL := `SELECT id, label FROM exec01_three ORDER BY id` + compiledCommand := pgSQL(t, streamSQL) + + boom := errors.New("visitor boom") + visits := 0 + err = executor.SourceRows(s.ctx, execution.RowsRequest{Command: compiledCommand, DeclaredColumns: []string{"id", "label"}}, func(row map[string]any) error { + visits++ + if visits == 2 { + return boom + } + return nil + }) + if err != boom { + t.Fatalf("visitor error altered: %#v", err) + } + if visits != 2 { + t.Fatalf("visited %d rows, want iteration to stop at the second", visits) + } + + var reuse int64 + if err := conn.QueryRow(s.ctx, `SELECT 42::bigint`).Scan(&reuse); err != nil { + s.fail("connection not reusable after visitor stop", err) + } + if reuse != 42 { + t.Fatalf("reuse witness = %d, want 42", reuse) + } + var cursors int64 + if err := conn.QueryRow(s.ctx, + `SELECT count(*) FROM pg_cursors WHERE statement LIKE 'SELECT id, label FROM exec01_three%'`).Scan(&cursors); err != nil { + s.fail("count cursors", err) + } + if cursors != 0 { + t.Fatalf("abandoned stream left %d server-side cursor(s)", cursors) + } + + full := 0 + if err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: compiledCommand, DeclaredColumns: []string{"id", "label"}}, func(map[string]any) error { full++; return nil }); err != nil { + s.fail("full drain control", err) + } + if full != 3 { + t.Fatalf("control drained %d rows, want 3", full) + } +} + +func TestExec01CancellationReturnsCompatibleContextErrors(t *testing.T) { + s := newExec01Service(t) + pool := s.poolOn(t, s.dsn, 3, "cancel") + executor := s.exec(t, pool) + + streamSQL := `SELECT pg_sleep(0.05) IS NULL AS tick, g FROM generate_series(1::bigint, 40::bigint) g` + columns := []string{"tick", "g"} + + t.Run("client cancel during in-flight execution is errors.Is-compatible", func(t *testing.T) { + ctx, cancel := context.WithCancel(s.ctx) + defer cancel() + tagged := `SELECT /*exec01-client-cancel*/ pg_sleep(0.05) IS NULL AS tick, g FROM generate_series(1::bigint, 40::bigint) g` + errc := make(chan error, 1) + go func() { + err := executor.streamTagged(ctx, tagged) + errc <- err + }() + s.waitForActive(t, exec01Prefix+"cancel", "exec01-client-cancel") + cancel() + err := <-errc + if !errors.Is(err, context.Canceled) || !errors.Is(err, ErrQuery) { + t.Fatalf("cancellation error not errors.Is-compatible: %v", err) + } + }) + + t.Run("control without cancel streams every row slowly", func(t *testing.T) { + started := time.Now() + visits := 0 + if err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, streamSQL), DeclaredColumns: columns}, func(row map[string]any) error { + visits++ + if _, ok := row["g"].(int64); !ok { + t.Fatalf("unexpected streamed row: %#v", row) + } + return nil + }); err != nil { + s.fail("uncancelled control", err) + } + elapsed := time.Since(started) + if visits != 40 { + t.Fatalf("control streamed %d rows, want all 40", visits) + } + if elapsed < 1500*time.Millisecond { + t.Fatalf("control stream completed in %v; the in-flight window was not real", elapsed) + } + }) + + t.Run("deadline expiry is errors.Is-compatible", func(t *testing.T) { + tagged := `SELECT /*exec01-deadline*/ pg_sleep(0.05) IS NULL AS tick, g FROM generate_series(1::bigint, 40::bigint) g` + ctx, cancel := context.WithTimeout(s.ctx, 350*time.Millisecond) + defer cancel() + err := executor.streamTagged(ctx, tagged) + if !errors.Is(err, context.DeadlineExceeded) || !errors.Is(err, ErrQuery) { + t.Fatalf("deadline error not errors.Is-compatible: %v", err) + } + }) + + t.Run("already-cancelled context fails at the seam", func(t *testing.T) { + ctx, cancel := context.WithCancel(s.ctx) + cancel() + err := executor.SourceRows(ctx, execution.RowsRequest{Command: pgSQL(t, streamSQL), DeclaredColumns: columns}, func(map[string]any) error { return nil }) + if !errors.Is(err, context.Canceled) || !errors.Is(err, ErrQuery) { + t.Fatalf("pre-cancelled context error not errors.Is-compatible: %v", err) + } + }) + + t.Run("server-side cancel keeps its SQLSTATE cause", func(t *testing.T) { + tagged := `SELECT /*exec01-server-cancel*/ pg_sleep(0.05) IS NULL AS tick, g FROM generate_series(1::bigint, 40::bigint) g` + errc := make(chan error, 1) + go func() { + err := executor.streamTagged(s.ctx, tagged) + errc <- err + }() + pid := s.waitForActive(t, exec01Prefix+"cancel", "exec01-server-cancel") + var cancelled bool + if err := s.admin.QueryRow(s.ctx, `SELECT pg_cancel_backend($1)`, pid).Scan(&cancelled); err != nil { + s.fail("pg_cancel_backend", err) + } + if !cancelled { + t.Fatal("pg_cancel_backend reported no cancellation") + } + err := <-errc + expectQueryError(t, err, "rows", "57014", "canceling statement due to user request") + }) + + var witness int64 + if err := pool.QueryRow(s.ctx, `SELECT 7::bigint`).Scan(&witness); err != nil { + s.fail("pool health after aborts", err) + } + if witness != 7 { + t.Fatalf("pool witness = %d, want 7", witness) + } +} + +func (e *Executor) streamTagged(ctx context.Context, sql string) error { + compiledCommand, err := command.NewPostgreSQLCommand(sql, nil) + if err != nil { + return err + } + return e.SourceRows(ctx, execution.RowsRequest{Command: compiledCommand, DeclaredColumns: []string{"tick", "g"}}, func(map[string]any) error { return nil }) +} + +func TestExec01SQLErrorsRetainIdentifiableCauses(t *testing.T) { + s := newExec01Service(t) + pool := s.pool(t, 3, "sqlerr") + executor := s.exec(t, pool) + + if _, err := s.iso.Exec(s.ctx, `CREATE TABLE `+s.table("exec01_three")+`(id bigint PRIMARY KEY, label text NOT NULL)`); err != nil { + s.fail("create sqlerr fixture", err) + } + if _, err := s.iso.Exec(s.ctx, `INSERT INTO `+s.table("exec01_three")+`(id, label) VALUES (1, 'a'), (2, 'b')`); err != nil { + s.fail("insert sqlerr rows", err) + } + if _, err := s.iso.Exec(s.ctx, `CREATE TABLE `+s.table("exec01_uniq")+`(label text PRIMARY KEY)`); err != nil { + s.fail("create unique fixture", err) + } + if _, err := s.iso.Exec(s.ctx, `INSERT INTO `+s.table("exec01_uniq")+`(label) VALUES ('dup')`); err != nil { + s.fail("seed unique fixture", err) + } + + t.Run("undefined relation keeps SQLSTATE 42P01", func(t *testing.T) { + err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, `SELECT id FROM `+s.table("exec01_nope")+` ORDER BY id`), DeclaredColumns: []string{"id"}}, func(map[string]any) error { return nil }) + expectQueryError(t, err, "query", "42P01", `relation "`+s.schema+`.exec01_nope" does not exist`) + count := 0 + if err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, `SELECT id FROM `+s.table("exec01_three")+` ORDER BY id`), DeclaredColumns: []string{"id"}}, func(map[string]any) error { count++; return nil }); err != nil { + s.fail("42P01 control", err) + } + if count != 2 { + t.Fatalf("control streamed %d rows, want 2", count) + } + }) + + t.Run("syntax error keeps SQLSTATE 42601", func(t *testing.T) { + err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, `SELECT FROM WHERE`), DeclaredColumns: []string{"x"}}, func(map[string]any) error { return nil }) + expectQueryError(t, err, "query", "42601", `syntax error at or near "WHERE"`) + }) + + t.Run("operator type error keeps SQLSTATE 42883", func(t *testing.T) { + err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, `SELECT 1::bigint + $1::text`, textParam("x")), DeclaredColumns: []string{"?"}}, func(map[string]any) error { return nil }) + expectQueryError(t, err, "query", "42883", `operator does not exist: bigint + text`) + var sum int64 + if err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, `SELECT 1::bigint + $1::bigint AS total`, bigint(2)), DeclaredColumns: []string{"total"}}, func(row map[string]any) error { sum, _ = row["total"].(int64); return nil }); err != nil { + s.fail("42883 control", err) + } + if sum != 3 { + t.Fatalf("typed control total = %d, want 3", sum) + } + }) + + t.Run("unique violation via RETURNING keeps constraint identity", func(t *testing.T) { + visits := 0 + err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, `INSERT INTO `+s.table("exec01_uniq")+`(label) VALUES ($1) RETURNING label`, textParam("dup")), DeclaredColumns: []string{"label"}}, func(map[string]any) error { visits++; return nil }) + expectQueryError(t, err, "rows", "23505", `duplicate key value violates unique constraint "exec01_uniq_pkey"`) + if visits != 0 { + t.Fatalf("failing INSERT RETURNING visited %d rows, want 0", visits) + } + var count int64 + if err := s.iso.QueryRow(s.ctx, `SELECT count(*) FROM `+s.table("exec01_uniq")).Scan(&count); err != nil { + s.fail("count unique rows", err) + } + if count != 1 { + t.Fatalf("unique fixture holds %d rows after failed insert, want 1 (statement must not half-apply)", count) + } + returned := "" + if err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, `INSERT INTO `+s.table("exec01_uniq")+`(label) VALUES ($1) RETURNING label`, textParam("fresh")), DeclaredColumns: []string{"label"}}, func(row map[string]any) error { returned, _ = row["label"].(string); return nil }); err != nil { + s.fail("RETURNING control", err) + } + if returned != "fresh" { + t.Fatalf("RETURNING label = %q, want %q", returned, "fresh") + } + }) + + var witness int64 + if err := pool.QueryRow(s.ctx, `SELECT 11::bigint`).Scan(&witness); err != nil { + s.fail("pool health after sql errors", err) + } + if witness != 11 { + t.Fatalf("pool witness = %d, want 11", witness) + } +} + +func TestExec01ThreeStatePreservation(t *testing.T) { + s := newExec01Service(t) + pool := s.pool(t, 3, "states") + executor := s.exec(t, pool) + + if _, err := s.iso.Exec(s.ctx, `CREATE TABLE `+s.table("exec01_states")+`( + id bigint PRIMARY KEY, j jsonb, jj json, t text, ab jsonb)`); err != nil { + s.fail("create three-state fixture", err) + } + + inserted := "" + if err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, `INSERT INTO `+s.table("exec01_states")+`(id, j, jj, t) VALUES ($1, $2, $3, $4) RETURNING id`, + bigint(1), jsonbParam("null"), sqlNullParam(), textParam("null")), DeclaredColumns: []string{"id"}}, func(row map[string]any) error { + id, ok := row["id"].(int64) + if !ok || id != 1 { + t.Fatalf("RETURNING id = %#v, want int64(1)", row["id"]) + } + inserted = "1" + return nil + }); err != nil { + s.fail("three-state insert via adapter", err) + } + if inserted != "1" { + t.Fatal("three-state insert did not stream its RETURNING row") + } + + columns := []string{"j", "jj", "t", "ab", "j_is_null", "j_is_json_null", "j_type", "jj_is_null", "t_is_null", "ab_is_null", "ab_type"} + compiledCommand := pgSQL(t, `SELECT j, jj, t, ab, + j IS NULL AS j_is_null, (j = 'null'::jsonb) AS j_is_json_null, jsonb_typeof(j) AS j_type, + jj IS NULL AS jj_is_null, t IS NULL AS t_is_null, ab IS NULL AS ab_is_null, jsonb_typeof(ab) AS ab_type + FROM `+s.table("exec01_states")+` WHERE id = $1`, bigint(1)) + var row map[string]any + if err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: compiledCommand, DeclaredColumns: columns}, func(r map[string]any) error { row = r; return nil }); err != nil { + s.fail("three-state stream", err) + } + if row == nil { + t.Fatal("three-state row never streamed") + } + + if raw, ok := row["j"].(json.RawMessage); !ok || string(raw) != "null" { + t.Fatalf("jsonb JSON-null = %#v, want json.RawMessage(\"null\")", row["j"]) + } + if row["j_is_null"] != false || row["j_is_json_null"] != true || row["j_type"] != "null" { + t.Fatalf("server witnesses disagree with JSON-null: %#v", row) + } + if row["jj"] != nil || row["jj_is_null"] != true { + t.Fatalf("SQL-null json column = %#v / witness %#v, want nil / true", row["jj"], row["jj_is_null"]) + } + if row["t"] != "null" || row["t_is_null"] != false { + t.Fatalf("text \"null\" = %#v / witness %#v, want \"null\" / false", row["t"], row["t_is_null"]) + } + if row["ab"] != nil || row["ab_is_null"] != true || row["ab_type"] != nil { + t.Fatalf("missing-column state = %#v, want nil / true / nil", row) + } + + counting := &countingQuerier{inner: pool} + countingExecutor, err := NewExecutor(counting) + if err != nil { + s.fail("wrap counting executor", err) + } + err = countingExecutor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, `SELECT id FROM `+s.table("exec01_states")+` WHERE id = $1`, + command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: float64(1)}), DeclaredColumns: []string{"id"}}, func(map[string]any) error { return nil }) + requireNamed(t, err, ErrInvalidParameter, "argument 1", `"bigint"`, "must carry an int64") + if counting.calls != 0 { + t.Fatalf("invalid parameter reached the pool (%d calls)", counting.calls) + } +} + +type countingQuerier struct { + inner Querier + calls int +} + +func (q *countingQuerier) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { + q.calls++ + return q.inner.Query(ctx, sql, args...) +} + +func TestExec01ColumnContractAgainstRealServer(t *testing.T) { + s := newExec01Service(t) + pool := s.pool(t, 3, "contract") + executor := s.exec(t, pool) + + if _, err := s.iso.Exec(s.ctx, `CREATE TABLE `+s.table("exec01_named")+`(id bigint PRIMARY KEY, label text NOT NULL)`); err != nil { + s.fail("create contract fixture", err) + } + if _, err := s.iso.Exec(s.ctx, `INSERT INTO `+s.table("exec01_named")+`(id, label) VALUES (1, 'a')`); err != nil { + s.fail("insert contract row", err) + } + sqlText := `SELECT id, label FROM ` + s.table("exec01_named") + + visits := 0 + visit := func(map[string]any) error { visits++; return nil } + + err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, sqlText), DeclaredColumns: []string{"ID", "label"}}, visit) + requireNamed(t, err, ErrColumnContractMismatch, `column 1 is "id", contract declares "ID"`) + + err = executor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, sqlText), DeclaredColumns: []string{"id"}}, visit) + requireNamed(t, err, ErrColumnContractMismatch, "returns 2 column(s), contract declares 1") + err = executor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, sqlText), DeclaredColumns: []string{"id", "label", "extra"}}, visit) + requireNamed(t, err, ErrColumnContractMismatch, "returns 2 column(s), contract declares 3") + if visits != 0 { + t.Fatalf("broken contracts visited %d rows, want 0", visits) + } + + alias := `SELECT id AS "Id", label AS "Label" FROM ` + s.table("exec01_named") + aliased := map[string]any{} + if err := executor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, alias), DeclaredColumns: []string{"Id", "Label"}}, func(row map[string]any) error { + aliased = row + return nil + }); err != nil { + s.fail("quoted-alias contract", err) + } + if aliased["Id"] != int64(1) || aliased["Label"] != "a" { + t.Fatalf("quoted-alias row = %#v", aliased) + } +} + +func TestExec01TwoRealConnectionRaces(t *testing.T) { + s := newExec01Service(t) + pool := s.pool(t, 6, "race") + executor := s.exec(t, pool) + + if _, err := s.iso.Exec(s.ctx, `CREATE TABLE `+s.table("exec01_race")+`( + id bigint PRIMARY KEY, label text NOT NULL, score double precision NOT NULL)`); err != nil { + s.fail("create race fixture", err) + } + if _, err := s.iso.Exec(s.ctx, `INSERT INTO `+s.table("exec01_race")+`(id, label, score) VALUES + (1, 'alpha', 1.5), (2, 'beta', -0.25), (3, 'gamma', 7)`); err != nil { + s.fail("insert race rows", err) + } + + raceSQL := `SELECT pg_backend_pid() AS backend, id, label, score FROM ` + s.table("exec01_race") + ` ORDER BY id` + raceColumns := []string{"backend", "id", "label", "score"} + + streamRace := func(ctx context.Context) (int64, string, error) { + var backend int64 + var lines []string + err := executor.SourceRows(ctx, execution.RowsRequest{Command: pgSQL(t, raceSQL), DeclaredColumns: raceColumns}, func(row map[string]any) error { + pid, ok := row["backend"].(int32) + if !ok { + return fmt.Errorf("backend column decoded as %#v", row["backend"]) + } + id, _ := row["id"].(int64) + label, _ := row["label"].(string) + score, ok := row["score"].(float64) + if !ok { + return fmt.Errorf("score decoded as %#v", row["score"]) + } + backend = int64(pid) + lines = append(lines, fmt.Sprintf("%d|%s|%s", id, label, strconv.FormatFloat(score, 'f', -1, 64))) + return nil + }) + if err != nil { + return 0, "", err + } + return backend, strings.Join(lines, ";"), nil + } + + requireZeroIdleInTx(t, s) + + t.Run("surface A: concurrent readers vs an aborted writer", func(t *testing.T) { + writer, err := pool.Acquire(s.ctx) + if err != nil { + s.fail("acquire writer", err) + } + defer writer.Release() + writerExecutor, err := NewExecutor(writer) + if err != nil { + s.fail("writer executor", err) + } + + for round := 0; round < 10; round++ { + if _, err := writer.Exec(s.ctx, `BEGIN`); err != nil { + s.fail("writer begin", err) + } + if _, err := writer.Exec(s.ctx, `UPDATE `+s.table("exec01_race")+` SET score = score + 1000 WHERE id = 1`); err != nil { + s.fail("writer update", err) + } + var dirtyLines []string + if err := writerExecutor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, raceSQL), DeclaredColumns: raceColumns}, func(row map[string]any) error { + id, _ := row["id"].(int64) + label, _ := row["label"].(string) + score, _ := row["score"].(float64) + dirtyLines = append(dirtyLines, fmt.Sprintf("%d|%s|%s", id, label, strconv.FormatFloat(score, 'f', -1, 64))) + return nil + }); err != nil { + s.fail("writer in-transaction stream", err) + } else if digest := strings.Join(dirtyLines, ";"); digest != goldenRaceDigestDirty { + s.fail("writer dirty digest", fmt.Errorf("digest = %q, want %q", digest, goldenRaceDigestDirty)) + } + + type result struct { + backend int64 + digest string + err error + } + start := make(chan struct{}) + results := make(chan result, 2) + for i := 0; i < 2; i++ { + go func() { + <-start + backend, digest, err := streamRace(s.ctx) + results <- result{backend, digest, err} + }() + } + close(start) + first := <-results + second := <-results + if _, err := writer.Exec(s.ctx, `ROLLBACK`); err != nil { + s.fail("writer rollback", err) + } + + for _, r := range []result{first, second} { + if r.err != nil { + s.fail("reader during aborted writer", r.err) + } + if r.digest != goldenRaceDigest { + t.Fatalf("round %d: reader digest = %q, want %q", round, r.digest, goldenRaceDigest) + } + } + if first.backend == second.backend { + t.Fatalf("round %d: both readers ran on backend %d; two real connections required", round, first.backend) + } + var writerPID int64 + if err := writer.QueryRow(s.ctx, `SELECT pg_backend_pid()`).Scan(&writerPID); err != nil { + s.fail("writer pid", err) + } + if first.backend == writerPID || second.backend == writerPID { + t.Fatalf("round %d: a reader shared the writer backend %d", round, writerPID) + } + } + requireZeroIdleInTx(t, s) + }) + + t.Run("surface B: cancelled sibling converges to the same witness", func(t *testing.T) { + type result struct { + backend int64 + digest string + err error + } + for round := 0; round < 10; round++ { + sibConn, err := pool.Acquire(s.ctx) + if err != nil { + s.fail("acquire sibling connection", err) + } + var sibBackend int64 + if err := sibConn.QueryRow(s.ctx, `SELECT pg_backend_pid()`).Scan(&sibBackend); err != nil { + sibConn.Release() + s.fail("sibling backend pid", err) + } + sibExecutor, err := NewExecutor(sibConn) + if err != nil { + sibConn.Release() + s.fail("sibling executor", err) + } + + cctx, ccancel := context.WithCancel(s.ctx) + siblingErr := make(chan error, 1) + go func() { + defer ccancel() + err := sibExecutor.SourceRows(cctx, execution.RowsRequest{Command: pgSQL(t, + `SELECT /*exec01-sibling*/ pg_sleep(0.02) IS NULL AS tick, g FROM generate_series(1::bigint, 30::bigint) g`), DeclaredColumns: []string{"tick", "g"}}, func(map[string]any) error { return nil }) + siblingErr <- err + }() + + s.waitForActive(t, exec01Prefix+"race", "exec01-sibling") + r := <-func() chan result { + ch := make(chan result, 1) + go func() { + backend, digest, err := streamRace(s.ctx) + ch <- result{backend, digest, err} + }() + return ch + }() + ccancel() + + sibErr := <-siblingErr + sibConn.Release() + if !errors.Is(sibErr, context.Canceled) || !errors.Is(sibErr, ErrQuery) { + t.Fatalf("round %d: cancelled sibling error not context-compatible: %v", round, sibErr) + } + if r.err != nil { + s.fail("sibling race reader", r.err) + } + if r.digest != goldenRaceDigest { + t.Fatalf("round %d: survivor digest = %q, want byte-identical %q", round, r.digest, goldenRaceDigest) + } + if r.backend == 0 { + t.Fatalf("round %d: survivor backend pid never witnessed", round) + } + if r.backend == sibBackend { + t.Fatalf("round %d: survivor shared the sibling backend %d", round, sibBackend) + } + } + requireZeroIdleInTx(t, s) + }) +} + +func requireZeroIdleInTx(t *testing.T, s *exec01Service) { + t.Helper() + var stuck int64 + if err := s.admin.QueryRow(s.ctx, + `SELECT count(*) FROM pg_stat_activity WHERE application_name LIKE 'verify\_exec01%' ESCAPE '\' AND state = 'idle in transaction'`). + Scan(&stuck); err != nil { + s.fail("count idle-in-transaction sessions", err) + } + if stuck != 0 { + t.Fatalf("%d verify_exec01 session(s) stuck idle in transaction after abort convergence", stuck) + } +} + +func TestExec01CrashWindowAtEverySeam(t *testing.T) { + s := newExec01Service(t) + + if _, err := s.iso.Exec(s.ctx, `CREATE TABLE `+s.table("exec01_crash")+`(id bigint PRIMARY KEY, label text NOT NULL)`); err != nil { + s.fail("create crash fixture", err) + } + if _, err := s.iso.Exec(s.ctx, `INSERT INTO `+s.table("exec01_crash")+`(id, label) VALUES (1, 'a'), (2, 'b'), (3, 'c')`); err != nil { + s.fail("insert crash rows", err) + } + + terminate := func(pid int64) { + t.Helper() + var killed bool + if err := s.admin.QueryRow(s.ctx, `SELECT pg_terminate_backend($1)`, pid).Scan(&killed); err != nil { + s.fail("pg_terminate_backend", err) + } + if !killed { + t.Fatalf("pg_terminate_backend(%d) reported no kill", pid) + } + deadline := time.Now().Add(10 * time.Second) + for { + var alive int64 + if err := s.admin.QueryRow(s.ctx, `SELECT count(*) FROM pg_stat_activity WHERE pid = $1`, pid).Scan(&alive); err != nil { + s.fail("poll terminated session", err) + } + if alive == 0 { + return + } + if time.Now().After(deadline) { + s.fail("terminated session never vanished", fmt.Errorf("pid %d still visible", pid)) + } + time.Sleep(20 * time.Millisecond) + } + } + + convergenceWitness := func(t *testing.T, seam string) { + t.Helper() + witnessPool := s.pool(t, 1, "crash-witness") + witnessExecutor, err := NewExecutor(witnessPool) + if err != nil { + s.fail("witness executor", err) + } + var digest strings.Builder + if err := witnessExecutor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, `SELECT id, label FROM `+s.table("exec01_crash")+` ORDER BY id`), DeclaredColumns: []string{"id", "label"}}, func(row map[string]any) error { + id, _ := row["id"].(int64) + label, _ := row["label"].(string) + digest.WriteString(fmt.Sprintf("%d|%s;", id, label)) + return nil + }); err != nil { + s.fail("post-"+seam+"-convergence stream", err) + } + if want := "1|a;2|b;3|c;"; digest.String() != want { + t.Fatalf("post-%s witness = %q, want byte-identical %q", seam, digest.String(), want) + } + requireZeroIdleInTx(t, s) + } + + t.Run("seam: terminate an in-flight execution", func(t *testing.T) { + seamPool := s.pool(t, 2, "crash-inflight") + seamExecutor, err := NewExecutor(seamPool) + if err != nil { + s.fail("in-flight executor", err) + } + tagged := `SELECT /*exec01-terminate-inflight*/ pg_sleep(0.3) IS NOT NULL AS tick, g FROM generate_series(1::bigint, 15::bigint) g` + errc := make(chan error, 1) + go func() { + err := seamExecutor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, tagged), DeclaredColumns: []string{"tick", "g"}}, func(map[string]any) error { return nil }) + errc <- err + }() + pid := s.waitForActive(t, exec01Prefix+"crash-inflight", "exec01-terminate-inflight") + terminate(pid) + err = <-errc + if !errors.Is(err, ErrQuery) { + t.Fatalf("in-flight termination lost the class: %v", err) + } + var translated *storepg.TranslatedError + if !errors.As(err, &translated) || translated.Condition != storepg.ConditionAdminShutdown || translated.SQLState != "57P01" { + t.Fatalf("in-flight termination lost its boundary classification: %v", err) + } + var pgErr *pgconn.PgError + if !errors.As(translated.Cause(), &pgErr) || pgErr.SQLState() != "57P01" { + t.Fatalf("in-flight termination lost its cause: %v", err) + } + convergenceWitness(t, "in-flight") + }) + + t.Run("seam: visitor stop while the backend dies", func(t *testing.T) { + seamPool := s.pool(t, 2, "crash-visitor") + conn, err := seamPool.Acquire(s.ctx) + if err != nil { + s.fail("acquire visitor stream connection", err) + } + defer conn.Release() + visitorExecutor, err := NewExecutor(conn) + if err != nil { + s.fail("visitor executor", err) + } + streamSQL := `SELECT pg_backend_pid() AS backend, id, label FROM ` + s.table("exec01_crash") + ` ORDER BY id` + boom := errors.New("visitor boom during backend death") + var visits int + err = visitorExecutor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, streamSQL), DeclaredColumns: []string{"backend", "id", "label"}}, func(row map[string]any) error { + visits++ + if visits == 1 { + pid32, _ := row["backend"].(int32) + terminate(int64(pid32)) + } + return boom + }) + if err != boom { + t.Fatalf("visitor error was replaced by the crash: %#v", err) + } + if visits != 1 { + t.Fatalf("visited %d rows, want 1", visits) + } + convergenceWitness(t, "visitor-stop") + }) + + t.Run("seam: scan window crash kills a later row", func(t *testing.T) { + seamPool := s.pool(t, 2, "crash-scan") + conn, err := seamPool.Acquire(s.ctx) + if err != nil { + s.fail("acquire scan stream connection", err) + } + defer conn.Release() + scanExecutor, err := NewExecutor(conn) + if err != nil { + s.fail("scan executor", err) + } + streamSQL := `SELECT pg_backend_pid() AS backend, pg_sleep(0.3) IS NOT NULL AS tick, g, repeat(md5(g::text), 50000) AS blob FROM generate_series(1::bigint, 8::bigint) g` + var visits int + err = scanExecutor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, streamSQL), DeclaredColumns: []string{"backend", "tick", "g", "blob"}}, func(row map[string]any) error { + visits++ + if visits == 1 { + if _, ok := row["blob"].(string); !ok { + t.Fatalf("blob column decoded as %#v", row["blob"]) + } + pid32, _ := row["backend"].(int32) + terminate(int64(pid32)) + } + return nil + }) + if !errors.Is(err, ErrQuery) || err == nil { + t.Fatalf("scan-window crash was not classified as a query failure: %v", err) + } + if visits < 1 || visits >= 8 { + t.Fatalf("scan-window crash visited %d rows; the kill never landed mid-stream", visits) + } + convergenceWitness(t, "scan-window") + }) + + t.Run("seam: query start against a dead connection", func(t *testing.T) { + seamPool := s.pool(t, 2, "crash-start") + conn, err := seamPool.Acquire(s.ctx) + if err != nil { + s.fail("acquire doomed connection", err) + } + defer conn.Release() + var pid int64 + if err := conn.QueryRow(s.ctx, `SELECT pg_backend_pid()`).Scan(&pid); err != nil { + s.fail("pid of doomed connection", err) + } + terminate(pid) + doomedExecutor, err := NewExecutor(conn) + if err != nil { + s.fail("doomed executor", err) + } + startErr := doomedExecutor.SourceRows(s.ctx, execution.RowsRequest{Command: pgSQL(t, `SELECT g FROM generate_series(1::bigint, 3::bigint) g`), DeclaredColumns: []string{"g"}}, func(map[string]any) error { return nil }) + if startErr == nil || !errors.Is(startErr, ErrQuery) { + t.Fatalf("query start on a dead connection was not classified: %v", startErr) + } + convergenceWitness(t, "query-start") + }) +} + +func TestZZZeroLeftoverExec01Fixtures(t *testing.T) { + raw := exec01URL(t) + exec01OwnedMu.Lock() + owned := make(map[string]string, len(exec01OwnedByName)) + for db, schema := range exec01OwnedByName { + owned[db] = schema + } + exec01OwnedMu.Unlock() + if len(owned) == 0 { + t.Skip("this process created no verify_exec01_* fixtures") + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service for audit: %v", err) + } + defer admin.Close() + + for db, schema := range owned { + if !strings.HasPrefix(db, exec01Prefix) { + t.Fatalf("registry holds foreign database %q", db) + } + var count int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Errorf("audit owned database %s: %v", db, err) + continue + } + if count != 0 { + t.Errorf("leftover owned database %s after all cleanups", db) + if schema != "" { + u, err := url.Parse(raw) + if err == nil { + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + db + if probe, err := pgxpool.New(ctx, u.String()); err == nil { + var schemaCount int + if err := probe.QueryRow(ctx, "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1", schema).Scan(&schemaCount); err != nil { + t.Errorf("audit owned schema %s in %s: %v", schema, db, err) + } else if schemaCount != 0 { + t.Errorf("leftover owned schema %s in leftover database %s", schema, db) + } + probe.Close() + } + } + } + } + } + + var visible int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_exec01\\_%' ESCAPE '\\'"). + Scan(&visible); err != nil { + t.Fatalf("audit verify_exec01_* database count: %v", err) + } + if visible != exec01BaselineCount { + t.Errorf("verify_exec01_* databases = %d, want baseline %d", visible, exec01BaselineCount) + } +} diff --git a/internal/backends/postgres/internal/rows/rows_postquery_postgres_test.go b/internal/backends/postgres/internal/rows/rows_postquery_postgres_test.go new file mode 100644 index 000000000..fd63f587e --- /dev/null +++ b/internal/backends/postgres/internal/rows/rows_postquery_postgres_test.go @@ -0,0 +1,505 @@ +package postgres + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/execution" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +const exec03rwPrefix = "verify_exec03v1rw_" + +var exec03rwNamePattern = regexp.MustCompile(`^verify_exec03v1rw_[0-9]+_[0-9a-f]+$`) + +var ( + exec03rwOwnedMu sync.Mutex + exec03rwOwnedByName = map[string]string{} +) + +func exec03rwRegister(db, schema string) { + exec03rwOwnedMu.Lock() + defer exec03rwOwnedMu.Unlock() + exec03rwOwnedByName[db] = schema +} + +func exec03rwRegistrySnapshot() map[string]string { + exec03rwOwnedMu.Lock() + defer exec03rwOwnedMu.Unlock() + out := make(map[string]string, len(exec03rwOwnedByName)) + for db, schema := range exec03rwOwnedByName { + out[db] = schema + } + return out +} + +var ( + exec03rwBaselineOnce sync.Once + exec03rwBaselineNames []string + exec03rwBaselineErr error +) + +func exec03rwCaptureBaseline(raw string) { + exec03rwBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec03rwBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec03v1rw\\_%' ESCAPE '\\'") + if err != nil { + exec03rwBaselineErr = fmt.Errorf("read baseline: %w", err) + return + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + exec03rwBaselineErr = fmt.Errorf("scan baseline: %w", err) + return + } + exec03rwBaselineNames = append(exec03rwBaselineNames, name) + } + }) +} + +func exec03rwRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +func exec03rwScrubberFor(raw string) (func(any) string, func(string)) { + secrets := []string{raw} + if u, err := url.Parse(raw); err == nil && u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + return scrub, func(s string) { secrets = append(secrets, s) } +} + +type exec03rwService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + schema string + scrub func(any) string + fail func(step string, err error) +} + +func newExec03rwService(t *testing.T) *exec03rwService { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 row-adapter post-query tests") + } + scrub, addSecret := exec03rwScrubberFor(raw) + exec03rwCaptureBaseline(raw) + if exec03rwBaselineErr != nil { + t.Fatalf("capture pg_database baseline: %s", scrub(exec03rwBaselineErr)) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + fail := func(step string, err error) { t.Fatalf("%s: %s", step, scrub(err)) } + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, "SELECT current_setting('server_version_num')::int8, current_setting('server_version')").Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if versionNum/10000 != 18 { + t.Fatalf("test service is not PostgreSQL 18: %s", version) + } + + dbName := fmt.Sprintf("%s%d_%s", exec03rwPrefix, time.Now().UnixNano(), exec03rwRandomHex(t)) + if !exec03rwNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + var created int64 + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", dbName).Scan(&created); err != nil { + fail("verify isolated database creation", err) + } + if created != 1 { + t.Fatalf("isolated database %q did not verifiably come into existence", dbName) + } + exec03rwRegister(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + addSecret(dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("%s%d_%s", exec03rwPrefix, time.Now().UnixNano(), exec03rwRandomHex(t)) + if !exec03rwNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + exec03rwRegister(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + return &exec03rwService{ctx: ctx, admin: admin, iso: iso, schema: schema, scrub: scrub, fail: fail} +} + +const ( + exec03rwUUID5Wire = `{"__loom_exact_uuid_operation":"uuid5","__loom_exact_uuid_args":["6ba7b810-9dad-11d1-80b4-00c04fd430c8","www.widgets.com"]}` + exec03rwUUID3Wire = `{"__loom_exact_uuid_operation":"uuid3","__loom_exact_uuid_args":["6ba7b810-9dad-11d1-80b4-00c04fd430c8","www.widgets.com"]}` +) + +func (s *exec03rwService) streamViaAdapter(t *testing.T, sql string, args []command.PostgreSQLParameter, columns []string) []map[string]any { + t.Helper() + tagged, err := command.NewPostgreSQLCommand(sql, args) + if err != nil { + t.Fatalf("NewPostgreSQLCommand: %v", err) + } + executor, err := NewExecutor(s.iso) + if err != nil { + s.fail("create executor", err) + } + var rows []map[string]any + err = executor.SourceRows(s.ctx, execution.RowsRequest{Command: tagged, DeclaredColumns: columns}, func(row map[string]any) error { + rows = append(rows, row) + return nil + }) + if err != nil { + s.fail("SourceRows: "+sql, err) + } + return rows +} + +func TestExec03RowsMarkerJSONBPreservedVerbatim(t *testing.T) { + s := newExec03rwService(t) + ctx := s.ctx + if _, err := s.iso.Exec(ctx, `CREATE TABLE "`+s.schema+`".exec03rw_markers (id bigint PRIMARY KEY, payload jsonb)`); err != nil { + s.fail("create fixture table", err) + } + orderedPair := `{"__loom_postquery_call":"concat","__loom_postquery_args":["a","b"]}` + swappedPair := `{"__loom_postquery_call":"concat","__loom_postquery_args":["b","a"]}` + deep := `{"level":{"list":[` + exec03rwUUID5Wire + `,null,1.5]}}` + for _, row := range []struct { + id int64 + payload string + }{ + {1, `{"gender":` + exec03rwUUID5Wire + `}`}, + {2, `{"op":` + orderedPair + `}`}, + {3, `{"op":` + swappedPair + `}`}, + {4, `{"nested":` + deep + `}`}, + } { + if _, err := s.iso.Exec(ctx, `INSERT INTO "`+s.schema+`".exec03rw_markers (id, payload) VALUES ($1, $2::jsonb)`, row.id, row.payload); err != nil { + s.fail("insert fixture row", err) + } + } + rows := s.streamViaAdapter(t, + `SELECT id, payload FROM "`+s.schema+`".exec03rw_markers ORDER BY id`, + []command.PostgreSQLParameter{}, + []string{"id", "payload"}) + if len(rows) != 4 { + t.Fatalf("visited %d rows, want 4", len(rows)) + } + for _, want := range []struct { + id int64 + column string + }{{1, "gender"}, {2, "op"}, {3, "op"}, {4, "nested"}} { + var canonical string + if err := s.iso.QueryRow(ctx, `SELECT (payload #> $1)::text FROM "`+s.schema+`".exec03rw_markers WHERE id = $2`, []string{want.column}, want.id).Scan(&canonical); err != nil { + s.fail("read canonical witness", err) + } + got, ok := rows[want.id-1]["payload"].(json.RawMessage) + if !ok { + t.Fatalf("row %d payload column has Go type %T, want json.RawMessage", want.id, rows[want.id-1]["payload"]) + } + var visited any + if err := json.Unmarshal(got, &visited); err != nil { + t.Fatalf("visited payload is not valid JSON: %v", err) + } + doc, ok := visited.(map[string]any) + if !ok { + t.Fatalf("visited payload is not an object: %T", visited) + } + encoded, err := json.Marshal(doc[want.column]) + if err != nil { + t.Fatalf("encode visited column: %v", err) + } + if string(encoded) != canonical { + var serverValue any + if err := json.Unmarshal([]byte(canonical), &serverValue); err != nil { + t.Fatalf("server canonical not decodable: %v", err) + } + serverEncoded, err := json.Marshal(serverValue) + if err != nil || string(serverEncoded) != string(encoded) { + t.Fatalf("visited %s for id %d = %s, server canonical (re-encoded) = %s (err %v)", want.column, want.id, encoded, serverEncoded, err) + } + } + } + if string(rows[1]["payload"].(json.RawMessage)) == string(rows[2]["payload"].(json.RawMessage)) { + t.Fatal("ordered marker arguments collapsed: [a,b] and [b,a] visited identically") + } + for i, want := range [][]any{{any("a"), any("b")}, {any("b"), any("a")}} { + var doc struct { + Op struct { + Args []any `json:"__loom_postquery_args"` + } `json:"op"` + } + if err := json.Unmarshal(rows[i+1]["payload"].(json.RawMessage), &doc); err != nil { + t.Fatalf("row %d visited bytes do not decode: %v", i+2, err) + } + if len(doc.Op.Args) != 2 || doc.Op.Args[0] != want[0] || doc.Op.Args[1] != want[1] { + t.Fatalf("row %d ordered arguments drifted to %#v, want %#v", i+2, doc.Op.Args, want) + } + } +} + +func TestExec03RowsThreeStateAroundMarkers(t *testing.T) { + s := newExec03rwService(t) + ctx := s.ctx + if _, err := s.iso.Exec(ctx, `CREATE TABLE "`+s.schema+`".exec03rw_states (id bigint PRIMARY KEY, payload jsonb)`); err != nil { + s.fail("create fixture table", err) + } + for _, row := range []struct { + id int64 + payload any + }{ + {1, nil}, + {2, "null"}, + {3, exec03rwUUID5Wire}, + } { + if _, err := s.iso.Exec(ctx, `INSERT INTO "`+s.schema+`".exec03rw_states (id, payload) VALUES ($1, $2::jsonb)`, row.id, row.payload); err != nil { + s.fail("insert fixture row", err) + } + } + rows := s.streamViaAdapter(t, + `SELECT id, payload FROM "`+s.schema+`".exec03rw_states ORDER BY id`, + []command.PostgreSQLParameter{}, + []string{"id", "payload"}) + if len(rows) != 3 { + t.Fatalf("visited %d rows, want 3", len(rows)) + } + if value, present := rows[0]["payload"]; !present || value != nil { + t.Fatalf("SQL NULL column visited as %#v (present %v), want untyped nil", value, present) + } + raw, ok := rows[1]["payload"].(json.RawMessage) + if !ok { + t.Fatalf("JSON null column visited as %T, want json.RawMessage", rows[1]["payload"]) + } + if string(raw) != "null" { + t.Fatalf("JSON null column bytes = %q, want \"null\"", raw) + } + if marker, ok := rows[2]["payload"].(json.RawMessage); !ok || !strings.Contains(string(marker), "__loom_exact_uuid_operation") { + t.Fatalf("marker column visited as %#v", rows[2]["payload"]) + } + var sqlNullState, jsonNullState string + if err := s.iso.QueryRow(ctx, `SELECT COALESCE(jsonb_typeof(payload), '') FROM "`+s.schema+`".exec03rw_states WHERE id = 1`).Scan(&sqlNullState); err != nil { + s.fail("read sql-null typeof", err) + } + if err := s.iso.QueryRow(ctx, `SELECT COALESCE(jsonb_typeof(payload), '') FROM "`+s.schema+`".exec03rw_states WHERE id = 2`).Scan(&jsonNullState); err != nil { + s.fail("read json-null typeof", err) + } + if sqlNullState != "" || jsonNullState != "null" { + t.Fatalf("server states drifted: sql-null typeof %q, json-null typeof %q", sqlNullState, jsonNullState) + } +} + +func TestExec03RowsMalformedMarkerRefusedServerSide(t *testing.T) { + s := newExec03rwService(t) + ctx := s.ctx + if _, err := s.iso.Exec(ctx, `CREATE TABLE "`+s.schema+`".exec03rw_malformed (id bigint PRIMARY KEY, payload jsonb)`); err != nil { + s.fail("create fixture table", err) + } + _, err := s.iso.Exec(ctx, `INSERT INTO "`+s.schema+`".exec03rw_malformed (id, payload) VALUES ($1, $2::jsonb)`, + int64(1), `{"__loom_exact_uuid_operation":"uuid5"`) + if err == nil { + t.Fatal("malformed marker JSON was accepted into a jsonb column") + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.SQLState() != "22P02" { + s.fail("malformed marker insert must be 22P02", err) + } + if _, err := s.iso.Exec(ctx, `INSERT INTO "`+s.schema+`".exec03rw_malformed (id, payload) VALUES ($1, $2::jsonb)`, + int64(2), exec03rwUUID5Wire); err != nil { + s.fail("well-formed marker control", err) + } + rows := s.streamViaAdapter(t, + `SELECT payload FROM "`+s.schema+`".exec03rw_malformed WHERE id = 2`, + []command.PostgreSQLParameter{}, + []string{"payload"}) + if len(rows) != 1 || string(rows[0]["payload"].(json.RawMessage)) == "" { + t.Fatalf("control row = %v", rows) + } +} + +func TestExec03RowsDeepNestedMarkerPassthrough(t *testing.T) { + s := newExec03rwService(t) + ctx := s.ctx + if _, err := s.iso.Exec(ctx, `CREATE TABLE "`+s.schema+`".exec03rw_deep (id bigint PRIMARY KEY, payload jsonb)`); err != nil { + s.fail("create fixture table", err) + } + const depth = 200 + document := `{"a":` + strings.Repeat(`{"a":`, depth) + exec03rwUUID5Wire + strings.Repeat(`}`, depth) + `}` + if _, err := s.iso.Exec(ctx, `INSERT INTO "`+s.schema+`".exec03rw_deep (id, payload) VALUES ($1, $2::jsonb)`, int64(1), document); err != nil { + s.fail("insert deep fixture", err) + } + rows := s.streamViaAdapter(t, + `SELECT payload FROM "`+s.schema+`".exec03rw_deep WHERE id = 1`, + []command.PostgreSQLParameter{}, + []string{"payload"}) + if len(rows) != 1 { + t.Fatalf("rows = %v", rows) + } + raw, ok := rows[0]["payload"].(json.RawMessage) + if !ok { + t.Fatalf("deep payload visited as %T, want a single json.RawMessage", rows[0]["payload"]) + } + if got := strings.Count(string(raw), `{"a":`); got != depth+1 { + t.Fatalf("visited document carries %d nesting levels, want %d", got, depth+1) + } + if !strings.Contains(string(raw), "__loom_exact_uuid_operation") { + t.Fatal("marker bytes lost in passthrough") + } +} + +func TestExec03RowsVisitedMarkerDoesNotAliasDriverBuffer(t *testing.T) { + marker := []byte(exec03rwUUID5Wire) + rows := &scriptedRows{ + fds: []pgconn.FieldDescription{jsonbFD("payload")}, + rowValues: [][]any{{marker}}, + shared: [][]byte{marker}, + } + querier := &recordingQuerier{rows: rows} + executor, err := NewExecutor(querier) + if err != nil { + t.Fatalf("NewExecutor: %v", err) + } + tagged, err := command.NewPostgreSQLCommand(`SELECT payload FROM t`, []command.PostgreSQLParameter{}) + if err != nil { + t.Fatalf("NewPostgreSQLCommand: %v", err) + } + var visited json.RawMessage + err = executor.SourceRows(context.Background(), execution.RowsRequest{Command: tagged, DeclaredColumns: []string{"payload"}}, func(row map[string]any) error { + raw, ok := row["payload"].(json.RawMessage) + if !ok { + return fmt.Errorf("visited value is %T, want json.RawMessage", row["payload"]) + } + visited = raw + return nil + }) + if err != nil { + t.Fatalf("SourceRows: %v", err) + } + for i := range marker { + marker[i] = 'X' + } + if string(visited) != exec03rwUUID5Wire { + t.Fatalf("retained marker aliased the driver buffer: %s", visited) + } +} + +func TestZZExec03RowsZeroLeftoverFixtures(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 row-adapter post-query tests") + } + scrub, _ := exec03rwScrubberFor(raw) + exec03rwCaptureBaseline(raw) + if exec03rwBaselineErr != nil { + t.Fatalf("capture pg_database baseline for audit: %s", scrub(exec03rwBaselineErr)) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for audit: %s", scrub(err)) + } + defer admin.Close() + owned := exec03rwRegistrySnapshot() + for db := range owned { + var count int + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Fatalf("audit %s: %s", db, scrub(err)) + } + if count != 0 { + t.Errorf("owned fixture database %s survived", db) + } + } + baseline := map[string]bool{} + for _, name := range exec03rwBaselineNames { + baseline[name] = true + } + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec03v1rw\\_%' ESCAPE '\\'") + if err != nil { + t.Fatalf("audit sweep: %s", scrub(err)) + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("audit scan: %s", scrub(err)) + } + if !baseline[name] { + t.Errorf("unregistered survivor under owned prefix: %s", name) + } + } + if err := rows.Err(); err != nil { + t.Fatalf("audit rows: %s", scrub(err)) + } +} diff --git a/internal/backends/postgres/internal/rows/rows_test.go b/internal/backends/postgres/internal/rows/rows_test.go new file mode 100644 index 000000000..966234414 --- /dev/null +++ b/internal/backends/postgres/internal/rows/rows_test.go @@ -0,0 +1,754 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "github.com/calypr/loom/internal/dataframe/execution" + "strings" + "testing" + "time" + + storepostgres "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" +) + +type scriptedRows struct { + fds []pgconn.FieldDescription + rowValues [][]any + nextIndex int + closed bool + closeRuns int + scanErr error + err error + shared [][]byte +} + +func (r *scriptedRows) Close() { r.closed = true; r.closeRuns++ } +func (r *scriptedRows) Err() error { + return r.err +} +func (r *scriptedRows) CommandTag() pgconn.CommandTag { return pgconn.NewCommandTag("SELECT") } +func (r *scriptedRows) FieldDescriptions() []pgconn.FieldDescription { + return r.fds +} +func (r *scriptedRows) Next() bool { + if r.closed { + return false + } + if r.nextIndex < len(r.rowValues) { + r.nextIndex++ + return true + } + r.closed = true + return false +} +func (r *scriptedRows) Scan(dest ...any) error { + if r.scanErr != nil { + return r.scanErr + } + if r.nextIndex < 1 || r.nextIndex > len(r.rowValues) { + return errors.New("scriptedRows.Scan called without a current row") + } + values := r.rowValues[r.nextIndex-1] + if len(values) != len(dest) { + return errors.New("scriptedRows.Scan: destination arity mismatch") + } + for i, target := range dest { + switch typed := target.(type) { + case *any: + *typed = values[i] + case *json.RawMessage: + if values[i] == nil { + *typed = nil + continue + } + raw, ok := values[i].([]byte) + if !ok { + return errors.New("scriptedRows.Scan: json target needs []byte") + } + *typed = json.RawMessage(raw) + default: + return errors.New("scriptedRows.Scan: unsupported destination") + } + } + return nil +} +func (r *scriptedRows) Values() ([]any, error) { + if r.nextIndex < 1 || r.nextIndex > len(r.rowValues) { + return nil, errors.New("scriptedRows.Values called without a current row") + } + return r.rowValues[r.nextIndex-1], nil +} +func (r *scriptedRows) RawValues() [][]byte { return r.shared } +func (r *scriptedRows) Conn() *pgx.Conn { return nil } +func (r *scriptedRows) TypeMap() *pgtype.Map { return pgtype.NewMap() } + +type recordingQuerier struct { + calls int + lastSQL string + lastArgs []any + rows pgx.Rows + queryErr error + rowsEvenOnError bool +} + +func (q *recordingQuerier) Query(_ context.Context, sql string, args ...any) (pgx.Rows, error) { + q.calls++ + q.lastSQL = sql + q.lastArgs = args + if q.queryErr != nil { + if q.rowsEvenOnError { + return q.rows, q.queryErr + } + return nil, q.queryErr + } + return q.rows, nil +} + +func textFD(name string) pgconn.FieldDescription { + return pgconn.FieldDescription{Name: name, DataTypeOID: pgtype.TextOID} +} + +func jsonbFD(name string) pgconn.FieldDescription { + return pgconn.FieldDescription{Name: name, DataTypeOID: pgtype.JSONBOID} +} + +func mustPostgreSQLCommand(t *testing.T, sql string, args ...command.PostgreSQLParameter) command.CompiledCommand { + t.Helper() + compiledCommand, err := command.NewPostgreSQLCommand(sql, args) + if err != nil { + t.Fatalf("NewPostgreSQLCommand(%q): %v", sql, err) + } + return compiledCommand +} + +func requireNamed(t *testing.T, err error, sentinel error, want ...string) { + t.Helper() + if err == nil { + t.Fatalf("expected refusal matching %v, got nil", sentinel) + } + if !errors.Is(err, sentinel) { + t.Fatalf("refusal %v does not match sentinel %v", err, sentinel) + } + for _, fragment := range want { + if !strings.Contains(err.Error(), fragment) { + t.Fatalf("refusal %q does not name %q", err, fragment) + } + } +} + +func TestNewExecutorRejectsNilQuerier(t *testing.T) { + _, err := NewExecutor(nil) + requireNamed(t, err, ErrInvalidExecution, "querier is required") +} + +func TestSourceRowsRequestGuardsRefuseBeforeQuerier(t *testing.T) { + pgSQL := mustPostgreSQLCommand(t, `SELECT id FROM t WHERE id = $1`, + command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(1)}) + validVisit := func(map[string]any) error { return nil } + + cases := []struct { + name string + ctx context.Context + visit RowVisitor + columns []string + wantSentinel error + want []string + }{ + {"nil context", nil, validVisit, []string{"id"}, ErrInvalidExecution, []string{"context is required"}}, + {"nil visitor", context.Background(), nil, []string{"id"}, ErrInvalidExecution, []string{"row visitor is required"}}, + {"no declared columns", context.Background(), validVisit, nil, ErrInvalidExecution, []string{"at least one declared result column"}}, + {"blank declared column", context.Background(), validVisit, []string{"id", " "}, ErrInvalidExecution, []string{"column 2 is blank"}}, + {"duplicate declared column", context.Background(), validVisit, []string{"id", "id"}, ErrInvalidExecution, []string{`declared result column "id" is declared twice`}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + querier := &recordingQuerier{} + executor, err := NewExecutor(querier) + if err != nil { + t.Fatal(err) + } + err = executor.SourceRows(tc.ctx, execution.RowsRequest{Command: pgSQL, DeclaredColumns: tc.columns}, tc.visit) + requireNamed(t, err, tc.wantSentinel, tc.want...) + if querier.calls != 0 { + t.Fatalf("guard let the statement reach the driver (%d calls)", querier.calls) + } + }) + } + + querier := &recordingQuerier{rows: &scriptedRows{fds: []pgconn.FieldDescription{textFD("id")}}} + executor, err := NewExecutor(querier) + if err != nil { + t.Fatal(err) + } + if err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: pgSQL, DeclaredColumns: []string{"id"}}, validVisit); err != nil { + t.Fatalf("control request refused: %v", err) + } + if querier.calls != 1 { + t.Fatalf("control executed %d times, want 1", querier.calls) + } + + var zero Executor + if err := zero.SourceRows(context.Background(), execution.RowsRequest{Command: pgSQL, DeclaredColumns: []string{"id"}}, validVisit); err == nil || !errors.Is(err, ErrInvalidExecution) { + t.Fatalf("zero-value executor did not refuse: %v", err) + } +} + +func TestSourceRowsDispatchIsBackendTagged(t *testing.T) { + visit := func(map[string]any) error { return nil } + const sql = `SELECT id FROM t` + + t.Run("AQL-tagged command is refused by name", func(t *testing.T) { + aql, err := command.NewAQLCommand(`FOR d IN t RETURN d`, map[string]any{"x": 1}) + if err != nil { + t.Fatal(err) + } + querier := &recordingQuerier{} + executor, _ := NewExecutor(querier) + err = executor.SourceRows(context.Background(), execution.RowsRequest{Command: aql, DeclaredColumns: []string{"id"}}, visit) + requireNamed(t, err, command.ErrCompiledCommandBackendMismatch, "refusing to pass AQL query to the PostgreSQL statement executor") + if querier.calls != 0 { + t.Fatalf("rendered AQL reached the PostgreSQL driver (%d calls)", querier.calls) + } + }) + + t.Run("zero-value command is refused as unregistered", func(t *testing.T) { + querier := &recordingQuerier{} + executor, _ := NewExecutor(querier) + err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: command.CompiledCommand{}, DeclaredColumns: []string{"id"}}, visit) + requireNamed(t, err, command.ErrCompiledCommandBackendNotRegistered, `unregistered backend ""`) + if querier.calls != 0 { + t.Fatalf("unregistered tag reached the driver (%d calls)", querier.calls) + } + }) + + querier := &recordingQuerier{rows: &scriptedRows{fds: []pgconn.FieldDescription{textFD("id")}}} + executor, _ := NewExecutor(querier) + if err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: mustPostgreSQLCommand(t, sql), DeclaredColumns: []string{"id"}}, visit); err != nil { + t.Fatalf("tagged PostgreSQL control refused: %v", err) + } + if querier.calls != 1 || querier.lastSQL != sql { + t.Fatalf("control dispatch wrong: calls=%d sql=%q", querier.calls, querier.lastSQL) + } +} + +func TestSourceRowsStatementShapeGuards(t *testing.T) { + visit := func(map[string]any) error { return nil } + bigint := func(v int64) command.PostgreSQLParameter { + return command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: v} + } + text := func(v string) command.PostgreSQLParameter { + return command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: v} + } + + cases := []struct { + name string + sql string + args []command.PostgreSQLParameter + wantName string + }{ + {"placeholder without arguments", `SELECT id FROM t WHERE id = $1`, nil, "declares no arguments"}, + {"placeholder above argument range", `SELECT id FROM t WHERE id = $2`, []command.PostgreSQLParameter{bigint(1)}, "$2 is outside the declared argument range 1..1"}, + {"argument never appears", `SELECT id FROM t WHERE id = $1`, []command.PostgreSQLParameter{bigint(1), text("orphan")}, "argument 2 never appears as $2"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + querier := &recordingQuerier{} + executor, _ := NewExecutor(querier) + compiledCommand := mustPostgreSQLCommand(t, tc.sql, tc.args...) + err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: compiledCommand, DeclaredColumns: []string{"id"}}, visit) + requireNamed(t, err, ErrMalformedStatement, tc.wantName) + if querier.calls != 0 { + t.Fatalf("malformed statement reached the driver (%d calls)", querier.calls) + } + }) + } + + for _, control := range []struct { + sql string + args []command.PostgreSQLParameter + }{ + {`SELECT $1 + $1`, []command.PostgreSQLParameter{bigint(1)}}, + {`SELECT 1`, nil}, + } { + querier := &recordingQuerier{rows: &scriptedRows{fds: []pgconn.FieldDescription{textFD("id")}}} + executor, _ := NewExecutor(querier) + if err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: mustPostgreSQLCommand(t, control.sql, control.args...), DeclaredColumns: []string{"id"}}, visit); err != nil { + t.Fatalf("well-formed statement %q refused: %v", control.sql, err) + } + if querier.calls != 1 { + t.Fatalf("statement %q executed %d times, want 1", control.sql, querier.calls) + } + } +} + +func TestBindParametersCanonicalizesEveryDeclaredType(t *testing.T) { + midnight := time.Date(2026, 2, 13, 0, 0, 0, 0, time.UTC) + instant := time.Date(2026, 2, 13, 10, 20, 30, 0, time.UTC) + nonUTCMidnight := time.Date(2026, 2, 13, 0, 0, 0, 0, time.FixedZone("UTC-5", -5*3600)) + + cases := []struct { + name string + param command.PostgreSQLParameter + want any + }{ + {"sql-null binds nil", command.PostgreSQLParameter{Type: command.PostgreSQLParameterNull, Value: nil}, any(nil)}, + {"text binds its string", command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: "x'y;-- $1"}, any("x'y;-- $1")}, + {"bigint binds int64", command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(42)}, any(int64(42))}, + {"double binds float64", command.PostgreSQLParameter{Type: command.PostgreSQLParameterDoublePrecision, Value: 1.5}, any(1.5)}, + {"boolean binds bool", command.PostgreSQLParameter{Type: command.PostgreSQLParameterBoolean, Value: true}, any(true)}, + {"date binds midnight time", command.PostgreSQLParameter{Type: command.PostgreSQLParameterDate, Value: midnight}, any(midnight)}, + {"date accepts a midnight in its own zone", command.PostgreSQLParameter{Type: command.PostgreSQLParameterDate, Value: nonUTCMidnight}, any(nonUTCMidnight)}, + {"timestamptz binds the instant", command.PostgreSQLParameter{Type: command.PostgreSQLParameterTimestamptz, Value: instant}, any(instant)}, + {"jsonb binds a JSON object", command.PostgreSQLParameter{Type: command.PostgreSQLParameterJSONB, Value: `{"k":1}`}, any(json.RawMessage(`{"k":1}`))}, + {"jsonb binds JSON null as a value", command.PostgreSQLParameter{Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage("null")}, any(json.RawMessage("null"))}, + {"json binds a string value", command.PostgreSQLParameter{Type: command.PostgreSQLParameterJSON, Value: `"s"`}, any(json.RawMessage(`"s"`))}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + querier := &recordingQuerier{rows: &scriptedRows{fds: []pgconn.FieldDescription{textFD("x")}}} + executor, _ := NewExecutor(querier) + compiledCommand := mustPostgreSQLCommand(t, `SELECT $1`, tc.param) + if err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: compiledCommand, DeclaredColumns: []string{"x"}}, func(map[string]any) error { return nil }); err != nil { + t.Fatalf("valid parameter refused: %v", err) + } + if len(querier.lastArgs) != 1 { + t.Fatalf("bound %d arguments, want 1", len(querier.lastArgs)) + } + if !equalBoundValue(querier.lastArgs[0], tc.want) { + t.Fatalf("canonical binding = %#v (%T), want %#v (%T)", querier.lastArgs[0], querier.lastArgs[0], tc.want, tc.want) + } + }) + } +} + +func equalBoundValue(got, want any) bool { + if want == nil { + return got == nil + } + gotRaw, gotIsRaw := got.(json.RawMessage) + wantRaw, wantIsRaw := want.(json.RawMessage) + if gotIsRaw != wantIsRaw { + return false + } + if gotIsRaw { + return string(gotRaw) == string(wantRaw) + } + switch wantTyped := want.(type) { + case time.Time: + gotTyped, ok := got.(time.Time) + return ok && gotTyped.Equal(wantTyped) && gotTyped.Location() == wantTyped.Location() + default: + return got == want + } +} + +func TestBindParametersRefusesEveryTypeViolation(t *testing.T) { + visit := func(map[string]any) error { return nil } + cases := []struct { + name string + param command.PostgreSQLParameter + wantName string + }{ + {"bigint refuses int", command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int(42)}, + "must carry an int64"}, + {"bigint refuses float64 (no driver-side coercion)", command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: float64(42)}, + "must carry an int64"}, + {"text refuses a []byte", command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: []byte("x")}, + "must carry a string"}, + {"double refuses int64", command.PostgreSQLParameter{Type: command.PostgreSQLParameterDoublePrecision, Value: int64(1)}, + "must carry a float64"}, + {"boolean refuses a string", command.PostgreSQLParameter{Type: command.PostgreSQLParameterBoolean, Value: "true"}, + "must carry a bool"}, + {"date refuses an instant with a time-of-day component", command.PostgreSQLParameter{Type: command.PostgreSQLParameterDate, Value: time.Date(2026, 2, 13, 17, 45, 0, 0, time.UTC)}, + "midnight calendar time"}, + {"date refuses a string", command.PostgreSQLParameter{Type: command.PostgreSQLParameterDate, Value: "2026-02-13"}, + "must carry a time.Time"}, + {"timestamptz refuses a string", command.PostgreSQLParameter{Type: command.PostgreSQLParameterTimestamptz, Value: "2026-02-13T00:00:00Z"}, + "must carry a time.Time"}, + {"jsonb refuses invalid JSON", command.PostgreSQLParameter{Type: command.PostgreSQLParameterJSONB, Value: `{"k":`}, + "must carry valid JSON"}, + {"json refuses a wrong Go shape", command.PostgreSQLParameter{Type: command.PostgreSQLParameterJSON, Value: map[string]any{"k": 1}}, + "must carry a string or json.RawMessage"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + querier := &recordingQuerier{} + executor, _ := NewExecutor(querier) + compiledCommand := mustPostgreSQLCommand(t, `SELECT $1`, tc.param) + err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: compiledCommand, DeclaredColumns: []string{"x"}}, visit) + requireNamed(t, err, ErrInvalidParameter, "argument 1", string(tc.param.Type), tc.wantName) + if querier.calls != 0 { + t.Fatalf("invalid parameter reached the driver (%d calls)", querier.calls) + } + }) + } +} + +func TestBindParameterRefusesWhatTheCompilerConstructorAlsoRefuses(t *testing.T) { + cases := []struct { + name string + param command.PostgreSQLParameter + wantName string + }{ + {"sql-null with a non-nil value is refused (no SQL-null/JSON-null flattening)", + command.PostgreSQLParameter{Type: command.PostgreSQLParameterNull, Value: "null"}, + "must carry a nil value"}, + {"unregistered declared type is refused (wrong-name probe)", command.PostgreSQLParameter{Type: "uuid", Value: "x"}, + `unregistered parameter type "uuid"`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := bindParameters([]command.PostgreSQLParameter{tc.param}) + requireNamed(t, err, ErrInvalidParameter, "argument 1", string(tc.param.Type), tc.wantName) + }) + } +} + +func TestSourceRowsEnforcesTheColumnContract(t *testing.T) { + querier := &recordingQuerier{rows: &scriptedRows{ + fds: []pgconn.FieldDescription{textFD("id"), jsonbFD("payload")}, + rowValues: [][]any{ + {[]byte("1")}, + }, + }} + executor, _ := NewExecutor(querier) + visits := 0 + visit := func(map[string]any) error { visits++; return nil } + + err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: mustPostgreSQLCommand(t, `SELECT id, payload FROM t`), DeclaredColumns: []string{"Id", "payload"}}, visit) + requireNamed(t, err, ErrColumnContractMismatch, `column 1 is "id", contract declares "Id"`) + + err = executor.SourceRows(context.Background(), execution.RowsRequest{Command: mustPostgreSQLCommand(t, `SELECT id, payload FROM t`), DeclaredColumns: []string{"id"}}, visit) + requireNamed(t, err, ErrColumnContractMismatch, "returns 2 column(s), contract declares 1") + + if visits != 0 { + t.Fatalf("rows were visited despite a broken contract: %d", visits) + } + rows := querier.rows.(*scriptedRows) + if !rows.closed { + t.Fatal("rows left open after a column contract refusal") + } + + rows2 := &scriptedRows{ + fds: []pgconn.FieldDescription{textFD("id"), jsonbFD("payload")}, + rowValues: [][]any{{[]byte("1"), []byte(`{}`)}}, + } + querier.rows = rows2 + if err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: mustPostgreSQLCommand(t, `SELECT id, payload FROM t`), DeclaredColumns: []string{"id", "payload"}}, visit); err != nil { + t.Fatalf("exact contract refused: %v", err) + } + if visits != 1 { + t.Fatalf("control visited %d rows, want 1", visits) + } + if !rows2.closed { + t.Fatal("control rows left open") + } +} + +func TestSourceRowsColumnContractPositionalAliasException(t *testing.T) { + run := func(t *testing.T, contract []string, aliasNames []string) error { + t.Helper() + fds := make([]pgconn.FieldDescription, 0, len(aliasNames)) + for _, name := range aliasNames { + fds = append(fds, textFD(name)) + } + querier := &recordingQuerier{rows: &scriptedRows{fds: fds}} + executor, _ := NewExecutor(querier) + return executor.SourceRows(context.Background(), + execution.RowsRequest{Command: mustPostgreSQLCommand(t, `SELECT c0, c1 FROM t`), DeclaredColumns: contract}, + func(map[string]any) error { return nil }) + } + + t.Run("PositionalAliasForItsOwnPositionIsAccepted", func(t *testing.T) { + if err := run(t, []string{"a", "b"}, []string{"c0", "c1"}); err != nil { + t.Fatalf("positional alias set refused: %v", err) + } + }) + t.Run("SwappedAliasPositionsAreRefused", func(t *testing.T) { + err := run(t, []string{"a", "b"}, []string{"c1", "c0"}) + requireNamed(t, err, ErrColumnContractMismatch, `result column 1 is "c1", contract declares "a"`) + }) + t.Run("ShiftedAliasPositionsAreRefused", func(t *testing.T) { + err := run(t, []string{"a", "b"}, []string{"c1", "c2"}) + requireNamed(t, err, ErrColumnContractMismatch, `result column 1 is "c1", contract declares "a"`) + }) + t.Run("ZeroPaddedAliasIsRefused", func(t *testing.T) { + err := run(t, []string{"a", "b"}, []string{"c00", "c1"}) + requireNamed(t, err, ErrColumnContractMismatch, `result column 1 is "c00", contract declares "a"`) + }) + t.Run("ForeignNameStillRefused", func(t *testing.T) { + err := run(t, []string{"a", "b"}, []string{"x0", "c1"}) + requireNamed(t, err, ErrColumnContractMismatch, `result column 1 is "x0", contract declares "a"`) + }) +} + +func TestSourceRowsRoutesByBackendTagBeforeDialectValidation(t *testing.T) { + visit := func(map[string]any) error { return nil } + + t.Run("AQLTaggedCommandWithoutDeclaredColumnsReportsMismatchNotDialectError", func(t *testing.T) { + aql, err := command.NewAQLCommand(`FOR d IN t RETURN d`, nil) + if err != nil { + t.Fatalf("NewAQLCommand: %v", err) + } + querier := &recordingQuerier{} + executor, _ := NewExecutor(querier) + err = executor.SourceRows(context.Background(), execution.RowsRequest{Command: aql}, visit) + requireNamed(t, err, command.ErrCompiledCommandBackendMismatch, "refusing to pass AQL query to the PostgreSQL statement executor") + if querier.calls != 0 { + t.Fatalf("misrouted command reached the querier %d time(s)", querier.calls) + } + }) + + t.Run("AQLTaggedCommandWithWellFormedColumnsStillReportsMismatch", func(t *testing.T) { + aql, err := command.NewAQLCommand(`FOR d IN t RETURN d`, nil) + if err != nil { + t.Fatalf("NewAQLCommand: %v", err) + } + querier := &recordingQuerier{} + executor, _ := NewExecutor(querier) + err = executor.SourceRows(context.Background(), execution.RowsRequest{Command: aql, DeclaredColumns: []string{"a", "b"}}, visit) + requireNamed(t, err, command.ErrCompiledCommandBackendMismatch) + if querier.calls != 0 { + t.Fatalf("misrouted command reached the querier %d time(s)", querier.calls) + } + }) + + t.Run("ZeroTagReportsNotRegisteredBeforeColumnsAreInspected", func(t *testing.T) { + querier := &recordingQuerier{} + executor, _ := NewExecutor(querier) + err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: command.CompiledCommand{}}, visit) + requireNamed(t, err, command.ErrCompiledCommandBackendNotRegistered) + if querier.calls != 0 { + t.Fatalf("zero-tag command reached the querier %d time(s)", querier.calls) + } + }) + + t.Run("RoutedPostgreSQLCommandStillValidatesDialect", func(t *testing.T) { + querier := &recordingQuerier{} + executor, _ := NewExecutor(querier) + err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: mustPostgreSQLCommand(t, `SELECT $1`, command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: "x"})}, visit) + requireNamed(t, err, ErrInvalidExecution, "at least one declared result column is required") + if querier.calls != 0 { + t.Fatalf("dialect refusal still reached the querier %d time(s)", querier.calls) + } + }) +} + +func TestSourceRowsDecodesRowsIntoCopiedMaps(t *testing.T) { + midnight := time.Date(2026, 2, 13, 0, 0, 0, 0, time.UTC) + sharedJSON := []byte(`{"k": 1}`) + sharedBytes := []byte("raw") + + rows := &scriptedRows{ + fds: []pgconn.FieldDescription{ + textFD("label"), + {Name: "count", DataTypeOID: pgtype.Int8OID}, + {Name: "ratio", DataTypeOID: pgtype.Float8OID}, + {Name: "flag", DataTypeOID: pgtype.BoolOID}, + {Name: "day", DataTypeOID: pgtype.DateOID}, + jsonbFD("payload"), + {Name: "note", DataTypeOID: pgtype.JSONOID}, + {Name: "blob", DataTypeOID: pgtype.ByteaOID}, + }, + rowValues: [][]any{ + {"alpha", int64(7), 2.5, true, midnight, sharedJSON, nil, sharedBytes}, + {"beta", int64(0), 0.0, false, midnight, []byte("null"), []byte("null"), nil}, + }, + } + querier := &recordingQuerier{rows: rows} + executor, _ := NewExecutor(querier) + + var visited []map[string]any + err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: mustPostgreSQLCommand(t, `SELECT label, count, ratio, flag, day, payload, note, blob FROM t`), DeclaredColumns: []string{"label", "count", "ratio", "flag", "day", "payload", "note", "blob"}}, func(row map[string]any) error { + visited = append(visited, row) + return nil + }) + if err != nil { + t.Fatalf("stream failed: %v", err) + } + if len(visited) != 2 { + t.Fatalf("visited %d rows, want 2", len(visited)) + } + + first := visited[0] + if len(first) != 8 { + t.Fatalf("row key count = %d, want 8", len(first)) + } + if first["label"] != "alpha" || first["count"] != int64(7) || first["ratio"] != 2.5 || first["flag"] != true { + t.Fatalf("first row scalars wrong: %#v", first) + } + if day, ok := first["day"].(time.Time); !ok || !day.Equal(midnight) { + t.Fatalf("day decoded as %#v", first["day"]) + } + payload, ok := first["payload"].(json.RawMessage) + if !ok || string(payload) != `{"k": 1}` { + t.Fatalf("payload decoded as %#v", first["payload"]) + } + if _, isRaw := first["note"].(json.RawMessage); isRaw || first["note"] != nil { + t.Fatalf("SQL NULL json column decoded as %#v, want nil", first["note"]) + } + blob, ok := first["blob"].([]byte) + if !ok || string(blob) != "raw" { + t.Fatalf("blob decoded as %#v", first["blob"]) + } + + second := visited[1] + if note, ok := second["note"].(json.RawMessage); !ok || string(note) != "null" { + t.Fatalf("JSON null decoded as %#v, want json.RawMessage(\"null\")", second["note"]) + } + if payload, ok := second["payload"].(json.RawMessage); !ok || string(payload) != "null" { + t.Fatalf("JSONB null decoded as %#v, want json.RawMessage(\"null\")", second["payload"]) + } + + sharedJSON[1] = 'X' + sharedBytes[0] = 'X' + if string(visited[0]["payload"].(json.RawMessage)) != `{"k": 1}` { + t.Fatal("visited payload aliases the driver buffer") + } + if string(visited[0]["blob"].([]byte)) != "raw" { + t.Fatal("visited blob aliases the driver buffer") + } +} + +func TestSourceRowsVisitorErrorStopsIterationAndClosesRows(t *testing.T) { + boom := errors.New("visitor boom") + rows := &scriptedRows{ + fds: []pgconn.FieldDescription{textFD("id")}, + rowValues: [][]any{{[]byte("1")}, {[]byte("2")}, {[]byte("3")}}, + } + querier := &recordingQuerier{rows: rows} + executor, _ := NewExecutor(querier) + + visits := 0 + err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: mustPostgreSQLCommand(t, `SELECT id FROM t`), DeclaredColumns: []string{"id"}}, func(map[string]any) error { + visits++ + if visits == 2 { + return boom + } + return nil + }) + if !errors.Is(err, boom) || err != boom { + t.Fatalf("visitor error was altered: %#v", err) + } + if visits != 2 { + t.Fatalf("visited %d rows after the visitor stopped; iteration did not stop", visits) + } + if !rows.closed { + t.Fatal("rows left open after a visitor error") + } + + rows2 := &scriptedRows{fds: rows.fds, rowValues: rows.rowValues} + querier.rows = rows2 + full := 0 + if err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: mustPostgreSQLCommand(t, `SELECT id FROM t`), DeclaredColumns: []string{"id"}}, func(map[string]any) error { full++; return nil }); err != nil { + t.Fatalf("control stream failed: %v", err) + } + if full != 3 || !rows2.closed { + t.Fatalf("control drained %d rows (want 3), closed=%v", full, rows2.closed) + } +} + +func TestSourceRowsSeamErrorsKeepClassAndCause(t *testing.T) { + visit := func(map[string]any) error { return nil } + fds := []pgconn.FieldDescription{textFD("id")} + + t.Run("mid-stream rows error keeps the cause", func(t *testing.T) { + pgErr := &pgconn.PgError{Code: "57014", Message: "query_canceled"} + querier := &recordingQuerier{rows: &scriptedRows{fds: fds, err: pgErr}} + executor, _ := NewExecutor(querier) + err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: mustPostgreSQLCommand(t, `SELECT id FROM t`), DeclaredColumns: []string{"id"}}, visit) + requireNamed(t, err, ErrQuery, "rows:") + var translated *storepostgres.TranslatedError + if !errors.As(err, &translated) || translated.Condition != storepostgres.ConditionQueryCanceled || translated.SQLState != "57014" { + t.Fatalf("rows error lost the boundary classification: %v", err) + } + var got *pgconn.PgError + if !errors.As(translated.Cause(), &got) || got.SQLState() != "57014" { + t.Fatalf("translated error lost the server cause through Cause(): %v", err) + } + }) + + t.Run("scan failure keeps class and cause", func(t *testing.T) { + scanErr := errors.New("cannot scan") + querier := &recordingQuerier{rows: &scriptedRows{fds: fds, rowValues: [][]any{{"x"}}, scanErr: scanErr}} + executor, _ := NewExecutor(querier) + err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: mustPostgreSQLCommand(t, `SELECT id FROM t`), DeclaredColumns: []string{"id"}}, visit) + requireNamed(t, err, ErrQuery, "scan:") + if !errors.Is(err, scanErr) { + t.Fatalf("scan error lost the cause: %v", err) + } + }) + + t.Run("cancellation during rows is errors.Is-compatible", func(t *testing.T) { + querier := &recordingQuerier{rows: &scriptedRows{fds: fds, err: context.Canceled}} + executor, _ := NewExecutor(querier) + err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: mustPostgreSQLCommand(t, `SELECT id FROM t`), DeclaredColumns: []string{"id"}}, visit) + if !errors.Is(err, context.Canceled) || !errors.Is(err, ErrQuery) { + t.Fatalf("cancellation not classifiable: %v", err) + } + }) + + t.Run("non-matching driver error after cancel is pinned to the context error", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + transportErr := errors.New("conn closed") + querier := &recordingQuerier{rows: &scriptedRows{fds: fds, err: transportErr}} + executor, _ := NewExecutor(querier) + err := executor.SourceRows(ctx, execution.RowsRequest{Command: mustPostgreSQLCommand(t, `SELECT id FROM t`), DeclaredColumns: []string{"id"}}, visit) + if !errors.Is(err, context.Canceled) || !errors.Is(err, ErrQuery) { + t.Fatalf("post-cancel transport error not pinned to the context error: %v", err) + } + }) + + t.Run("query start error keeps class and cause and closes rows", func(t *testing.T) { + pgErr := &pgconn.PgError{Code: "42P01", Message: `relation "t" does not exist`} + rows := &scriptedRows{fds: fds} + querier := &recordingQuerier{rows: rows, queryErr: pgErr, rowsEvenOnError: true} + executor, _ := NewExecutor(querier) + err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: mustPostgreSQLCommand(t, `SELECT id FROM t`), DeclaredColumns: []string{"id"}}, visit) + requireNamed(t, err, ErrQuery, "query:") + var translated *storepostgres.TranslatedError + if !errors.As(err, &translated) || translated.Condition != storepostgres.ConditionServerRejected || translated.SQLState != "42P01" { + t.Fatalf("query start error lost the boundary classification: %v", err) + } + var got *pgconn.PgError + if !errors.As(translated.Cause(), &got) || got.SQLState() != "42P01" { + t.Fatalf("query start error lost the server cause through Cause(): %v", err) + } + if !rows.closed { + t.Fatal("erroring rows handed back by Query were not closed") + } + }) + + t.Run("defensive: nil rows with nil error is a query failure", func(t *testing.T) { + querier := &recordingQuerier{} + executor, _ := NewExecutor(querier) + err := executor.SourceRows(context.Background(), execution.RowsRequest{Command: mustPostgreSQLCommand(t, `SELECT id FROM t`), DeclaredColumns: []string{"id"}}, visit) + requireNamed(t, err, ErrQuery, "no result set") + }) +} + +func TestSourceRowsBoundarySignature(t *testing.T) { + executor, err := NewExecutor(&recordingQuerier{rows: &scriptedRows{fds: fdsFor("x")}}) + if err != nil { + t.Fatal(err) + } + var boundary execution.SourceRows = executor.SourceRows + if err := boundary(context.Background(), execution.RowsRequest{Command: mustPostgreSQLCommand(t, `SELECT 1`), DeclaredColumns: []string{"x"}}, + func(map[string]any) error { return nil }); err != nil { + t.Fatalf("boundary call failed: %v", err) + } +} + +func fdsFor(names ...string) []pgconn.FieldDescription { + out := make([]pgconn.FieldDescription, len(names)) + for i, name := range names { + out[i] = textFD(name) + } + return out +} diff --git a/internal/backends/postgres/internal/store/advisory.go b/internal/backends/postgres/internal/store/advisory.go new file mode 100644 index 000000000..b29475e60 --- /dev/null +++ b/internal/backends/postgres/internal/store/advisory.go @@ -0,0 +1,49 @@ +package postgres + +import ( + "context" + "fmt" + "hash/fnv" +) + +const generationAdvisoryLockNamespace = "loom/pg-ingest-03/inventory-rebuild/v1" + +func GenerationAdvisoryLockKey(parts ...string) int64 { + h := fnv.New64a() + _, _ = h.Write([]byte(generationAdvisoryLockNamespace)) + for _, part := range parts { + _, _ = h.Write([]byte{0}) + _, _ = h.Write([]byte(part)) + } + return int64(h.Sum64()) +} + +func WithAdvisoryLock(ctx context.Context, src ConnSource, key int64, work func() error) (err error) { + conn, err := src.Acquire(ctx) + if err != nil { + return fmt.Errorf("postgres: acquire advisory lock connection: %w", err) + } + if _, lockErr := conn.Exec(ctx, "SELECT pg_advisory_lock($1)", key); lockErr != nil { + discardPoolConn(conn) + return fmt.Errorf("postgres: acquire advisory lock %d: %w", key, lockErr) + } + defer func() { + unlockCtx, cancelUnlock := context.WithTimeout(context.WithoutCancel(ctx), unlockTimeout) + defer cancelUnlock() + var unlocked bool + unlockErr := conn.QueryRow(unlockCtx, "SELECT pg_advisory_unlock($1)", key).Scan(&unlocked) + if unlockErr != nil || !unlocked { + discardPoolConn(conn) + if err == nil { + if unlockErr != nil { + err = fmt.Errorf("postgres: release advisory lock %d: %w", key, unlockErr) + return + } + err = fmt.Errorf("postgres: release advisory lock %d: reported already unlocked", key) + } + return + } + conn.Release() + }() + return work() +} diff --git a/internal/backends/postgres/internal/store/batch_writer.go b/internal/backends/postgres/internal/store/batch_writer.go new file mode 100644 index 000000000..62cf516fe --- /dev/null +++ b/internal/backends/postgres/internal/store/batch_writer.go @@ -0,0 +1,815 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5" +) + +const ( + batchLedgerTableName = "batch_write_ledger" + MaxBatchResources = 1000 + MaxBatchEdges = 1000 + MaxBatchIDLength = 255 +) + +const ( + stagingResourceTable = "stg_batch_resource" + stagingEdgeTable = "stg_batch_edge" +) + +var ( + ErrInvalidBatch = errors.New("postgres: invalid batch") + ErrBatchIdentityConflict = errors.New("postgres: batch identity conflict") + ErrBatchRowConflict = errors.New("postgres: batch row conflict") + ErrBatchAccounting = errors.New("postgres: batch commit accounting divergence") + ErrBatchWrite = errors.New("postgres: batch write failed") +) + +type BatchIdentityConflictError struct { + BatchID string + LedgerDigest string + ProvidedDigest string +} + +func (e *BatchIdentityConflictError) Error() string { + return fmt.Sprintf("postgres: batch %q already committed with digest %s; "+ + "reusing the identity with digest %s is refused and writes nothing", + e.BatchID, e.LedgerDigest, e.ProvidedDigest) +} + +func (e *BatchIdentityConflictError) Unwrap() error { return ErrBatchIdentityConflict } + +type BatchResourceConflictError struct { + Project string + DatasetGeneration string + ResourceType string + LogicalID string + InBatch bool +} + +func (e *BatchResourceConflictError) Error() string { + where := "an already-stored row" + if e.InBatch { + where = "another row in this batch" + } + return fmt.Sprintf("postgres: batch refused: resource %s/%s/%s/%s duplicates %s with different content; "+ + "the batch wrote nothing and the stored payload is preserved", + e.Project, e.DatasetGeneration, e.ResourceType, e.LogicalID, where) +} + +func (e *BatchResourceConflictError) Unwrap() error { return ErrBatchRowConflict } + +type BatchEdgeConflictError struct { + Project string + DatasetGeneration string + SourceResourceType string + SourceLogicalID string + Label string + TargetResourceType string + TargetLogicalID string + SourcePath string + SourceOrdinal int64 + InBatch bool +} + +func (e *BatchEdgeConflictError) Error() string { + where := "an already-stored occurrence" + if e.InBatch { + where = "another occurrence in this batch" + } + return fmt.Sprintf("postgres: batch refused: edge occurrence %s/%s/%s(%s) -%s-> %s/%s at path %q ordinal %d duplicates %s with different content; "+ + "the batch wrote nothing and the stored occurrence is preserved", + e.Project, e.DatasetGeneration, e.SourceResourceType, e.SourceLogicalID, e.Label, + e.TargetResourceType, e.TargetLogicalID, e.SourcePath, e.SourceOrdinal, where) +} + +func (e *BatchEdgeConflictError) Unwrap() error { return ErrBatchRowConflict } + +type BatchAccountingError struct { + BatchID string + Side string + Expected int64 + Committed int64 +} + +func (e *BatchAccountingError) Error() string { + return fmt.Sprintf("postgres: batch %q %s accounting diverged: expected %d inserted rows, publish returned %d; "+ + "a concurrent writer changed the scope mid-batch and the batch rolled back", + e.BatchID, e.Side, e.Expected, e.Committed) +} + +func (e *BatchAccountingError) Unwrap() error { return ErrBatchAccounting } + +type BatchWriteError struct { + Op string + BatchID string + Err error +} + +func (e *BatchWriteError) Error() string { + return fmt.Sprintf("postgres: batch %q failed during %s: %v", e.BatchID, e.Op, e.Err) +} + +func (e *BatchWriteError) Unwrap() []error { return []error{ErrBatchWrite, e.Err} } + +type Batch struct { + ID string + Project string + DatasetGeneration string + Resources []source.Resource + Edges []EdgeOccurrenceRow +} + +type BatchResult struct { + BatchID string + Digest string + CommittedResources int64 + CommittedEdges int64 + SkippedIdenticalResources int64 + SkippedIdenticalEdges int64 + Replay bool +} + +type BatchWriter struct { + src ConnSource + schema string + resourceTableFQN string + edgeTableFQN string + ledgerFQN string + failHook func(step string) error + inTxProbe func(step, batchID string, tx pgx.Tx) +} + +func NewBatchWriter(src ConnSource, schema string) (*BatchWriter, error) { + if src == nil { + return nil, errors.New("postgres: nil batch connection source") + } + if !ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", ErrInvalidSchemaName, schema) + } + return &BatchWriter{ + src: src, + schema: schema, + resourceTableFQN: pgx.Identifier{schema, resourceTableName}.Sanitize(), + edgeTableFQN: pgx.Identifier{schema, edgeTableName}.Sanitize(), + ledgerFQN: pgx.Identifier{schema, batchLedgerTableName}.Sanitize(), + }, nil +} + +const ( + stepBegin = "begin" + stepReadLedger = "read-ledger" + stepClaimLedger = "claim-ledger" + stepStageResources = "stage-resources" + stepCheckResources = "check-resources" + stepPublishResourc = "publish-resources" + stepStageEdges = "stage-edges" + stepCheckEdges = "check-edges" + stepPublishEdges = "publish-edges" + stepFinalizeLedger = "finalize-ledger" + stepCommit = "commit" +) + +func (w *BatchWriter) WriteBatch(ctx context.Context, batch Batch) (BatchResult, error) { + if err := validateBatch(batch); err != nil { + return BatchResult{}, err + } + digest := batchDigest(batch) + + conn, err := w.src.Acquire(ctx) + if err != nil { + return BatchResult{}, &BatchWriteError{Op: "acquire", BatchID: batch.ID, Err: err} + } + defer conn.Release() + + if err := w.hookFail(stepBegin); err != nil { + return BatchResult{}, &BatchWriteError{Op: "begin", BatchID: batch.ID, Err: err} + } + tx, err := conn.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return BatchResult{}, &BatchWriteError{Op: "begin", BatchID: batch.ID, Err: err} + } + abandoned := true + defer func() { + if abandoned { + _ = tx.Rollback(ctx) + } + }() + + setSearchPath := "SET LOCAL search_path TO " + pgx.Identifier{w.schema}.Sanitize() + if _, err := tx.Exec(ctx, setSearchPath); err != nil { + return BatchResult{}, &BatchWriteError{Op: "search_path", BatchID: batch.ID, Err: err} + } + w.probe(stepBegin, batch.ID, tx) + + if err := w.hookFail(stepReadLedger); err != nil { + return BatchResult{}, &BatchWriteError{Op: "ledger", BatchID: batch.ID, Err: err} + } + if res, done, err := w.checkLedger(ctx, tx, batch, digest); err != nil || done { + return res, err + } + w.probe(stepReadLedger, batch.ID, tx) + + if err := w.hookFail(stepClaimLedger); err != nil { + return BatchResult{}, &BatchWriteError{Op: "ledger", BatchID: batch.ID, Err: err} + } + claimed, err := w.claimLedger(ctx, tx, batch, digest) + if err != nil { + return BatchResult{}, err + } + if !claimed { + res, _, err := w.readLedgerRow(ctx, tx, batch, digest) + return res, err + } + w.probe(stepClaimLedger, batch.ID, tx) + + var committedResources, committedEdges int64 + if len(batch.Resources) > 0 { + committedResources, err = w.writeResourcePhase(ctx, tx, batch) + if err != nil { + return BatchResult{}, err + } + } + if len(batch.Edges) > 0 { + committedEdges, err = w.writeEdgePhase(ctx, tx, batch) + if err != nil { + return BatchResult{}, err + } + } + + if err := w.hookFail(stepFinalizeLedger); err != nil { + return BatchResult{}, &BatchWriteError{Op: "ledger", BatchID: batch.ID, Err: err} + } + if _, err := tx.Exec(ctx, + "UPDATE "+w.ledgerFQN+" SET resource_count = $2, edge_count = $3, committed_at = clock_timestamp() WHERE batch_id = $1", + batch.ID, committedResources, committedEdges); err != nil { + return BatchResult{}, &BatchWriteError{Op: "ledger", BatchID: batch.ID, Err: err} + } + w.probe(stepFinalizeLedger, batch.ID, tx) + + if err := w.hookFail(stepCommit); err != nil { + return BatchResult{}, &BatchWriteError{Op: "commit", BatchID: batch.ID, Err: err} + } + if err := tx.Commit(ctx); err != nil { + return BatchResult{}, &BatchWriteError{Op: "commit", BatchID: batch.ID, Err: err} + } + abandoned = false + + return BatchResult{ + BatchID: batch.ID, + Digest: digest, + CommittedResources: committedResources, + CommittedEdges: committedEdges, + SkippedIdenticalResources: int64(len(batch.Resources)) - committedResources, + SkippedIdenticalEdges: int64(len(batch.Edges)) - committedEdges, + }, nil +} + +func (w *BatchWriter) hookFail(step string) error { + if w.failHook == nil { + return nil + } + return w.failHook(step) +} + +func (w *BatchWriter) probe(step, batchID string, tx pgx.Tx) { + if w.inTxProbe == nil { + return + } + w.inTxProbe(step, batchID, tx) +} + +func (w *BatchWriter) checkLedger(ctx context.Context, tx pgx.Tx, batch Batch, digest string) (BatchResult, bool, error) { + var ledgerDigest string + var ledgerResources, ledgerEdges int64 + err := tx.QueryRow(ctx, + "SELECT digest, resource_count, edge_count FROM "+w.ledgerFQN+" WHERE batch_id = $1", + batch.ID).Scan(&ledgerDigest, &ledgerResources, &ledgerEdges) + if errors.Is(err, pgx.ErrNoRows) { + return BatchResult{}, false, nil + } + if err != nil { + return BatchResult{}, false, &BatchWriteError{Op: "ledger", BatchID: batch.ID, Err: err} + } + res, err := classifyLedgerRow(batch, digest, ledgerDigest, ledgerResources, ledgerEdges) + return res, true, err +} + +func (w *BatchWriter) claimLedger(ctx context.Context, tx pgx.Tx, batch Batch, digest string) (bool, error) { + var claimedID string + err := tx.QueryRow(ctx, + "INSERT INTO "+w.ledgerFQN+ + " (batch_id, project, dataset_generation, digest, resource_count, edge_count) "+ + "VALUES ($1, $2, $3, $4::text, 0, 0) ON CONFLICT (batch_id) DO NOTHING RETURNING batch_id", + batch.ID, batch.Project, batch.DatasetGeneration, digest).Scan(&claimedID) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, &BatchWriteError{Op: "ledger", BatchID: batch.ID, Err: err} + } + return true, nil +} + +func (w *BatchWriter) readLedgerRow(ctx context.Context, tx pgx.Tx, batch Batch, digest string) (BatchResult, bool, error) { + var ledgerDigest string + var ledgerResources, ledgerEdges int64 + err := tx.QueryRow(ctx, + "SELECT digest, resource_count, edge_count FROM "+w.ledgerFQN+" WHERE batch_id = $1", + batch.ID).Scan(&ledgerDigest, &ledgerResources, &ledgerEdges) + if errors.Is(err, pgx.ErrNoRows) { + return BatchResult{}, false, &BatchWriteError{ + Op: "ledger", BatchID: batch.ID, + Err: errors.New("concurrent batch claim vanished before it could be classified; retry the batch"), + } + } + if err != nil { + return BatchResult{}, false, &BatchWriteError{Op: "ledger", BatchID: batch.ID, Err: err} + } + res, err := classifyLedgerRow(batch, digest, ledgerDigest, ledgerResources, ledgerEdges) + return res, true, err +} + +func classifyLedgerRow(batch Batch, digest, ledgerDigest string, ledgerResources, ledgerEdges int64) (BatchResult, error) { + if digest != ledgerDigest { + return BatchResult{}, &BatchIdentityConflictError{ + BatchID: batch.ID, + LedgerDigest: ledgerDigest, + ProvidedDigest: digest, + } + } + return BatchResult{ + BatchID: batch.ID, + Digest: digest, + CommittedResources: ledgerResources, + CommittedEdges: ledgerEdges, + SkippedIdenticalResources: int64(len(batch.Resources)) - ledgerResources, + SkippedIdenticalEdges: int64(len(batch.Edges)) - ledgerEdges, + Replay: true, + }, nil +} + +func (w *BatchWriter) stagingResourceFQN() string { + return pgx.Identifier{"pg_temp", stagingResourceTable}.Sanitize() +} + +func (w *BatchWriter) stagingEdgeFQN() string { + return pgx.Identifier{"pg_temp", stagingEdgeTable}.Sanitize() +} + +func (w *BatchWriter) writeResourcePhase(ctx context.Context, tx pgx.Tx, batch Batch) (int64, error) { + if err := w.hookFail(stepStageResources); err != nil { + return 0, &BatchWriteError{Op: "create-staging", BatchID: batch.ID, Err: err} + } + if err := w.stageResources(ctx, tx, batch); err != nil { + return 0, err + } + w.probe(stepStageResources, batch.ID, tx) + + if err := w.hookFail(stepCheckResources); err != nil { + return 0, &BatchWriteError{Op: "check", BatchID: batch.ID, Err: err} + } + expectedResources, err := w.guardResources(ctx, tx, batch) + if err != nil { + return 0, err + } + w.probe(stepCheckResources, batch.ID, tx) + + if err := w.hookFail(stepPublishResourc); err != nil { + return 0, &BatchWriteError{Op: "publish", BatchID: batch.ID, Err: err} + } + committedResources, err := w.publishResources(ctx, tx, batch.ID) + if err != nil { + return 0, err + } + if committedResources != expectedResources { + return 0, &BatchAccountingError{ + BatchID: batch.ID, Side: "resources", + Expected: expectedResources, Committed: committedResources, + } + } + w.probe(stepPublishResourc, batch.ID, tx) + return committedResources, nil +} + +func (w *BatchWriter) stageResources(ctx context.Context, tx pgx.Tx, batch Batch) error { + const createSQL = "CREATE TEMP TABLE " + stagingResourceTable + " (" + + "ordinal bigint NOT NULL, " + + "project text NOT NULL, " + + "dataset_generation text NOT NULL, " + + "resource_type text NOT NULL, " + + "logical_id text NOT NULL, " + + "auth_resource_path text NOT NULL, " + + "payload text" + + ") ON COMMIT DROP" + if _, err := tx.Exec(ctx, createSQL); err != nil { + return &BatchWriteError{Op: "create-staging", BatchID: batch.ID, Err: err} + } + columns := []string{"ordinal", "project", "dataset_generation", "resource_type", + "logical_id", "auth_resource_path", "payload"} + rows := make([][]any, 0, len(batch.Resources)) + for i, rec := range batch.Resources { + var payload any + if rec.Payload != nil { + payload = string(rec.Payload) + } + rows = append(rows, []any{int64(i), rec.Project, rec.DatasetGeneration, + rec.ResourceType, rec.LogicalID, rec.AuthResourcePath, payload}) + } + n, err := tx.CopyFrom(ctx, pgx.Identifier{"pg_temp", stagingResourceTable}, columns, + pgx.CopyFromRows(rows)) + if err != nil { + return &BatchWriteError{Op: "copy", BatchID: batch.ID, Err: err} + } + if n != int64(len(batch.Resources)) { + return &BatchAccountingError{BatchID: batch.ID, Side: "resources", + Expected: int64(len(batch.Resources)), Committed: n} + } + return nil +} + +func (w *BatchWriter) guardResources(ctx context.Context, tx pgx.Tx, batch Batch) (int64, error) { + stg := w.stagingResourceFQN() + var resourceType, logicalID string + inBatchSQL := "SELECT a.resource_type, a.logical_id FROM " + stg + " a JOIN " + stg + " b" + + " ON a.project = b.project AND a.dataset_generation = b.dataset_generation" + + " AND a.resource_type = b.resource_type AND a.logical_id = b.logical_id" + + " AND (a.auth_resource_path IS DISTINCT FROM b.auth_resource_path" + + " OR (a.payload IS NULL) <> (b.payload IS NULL)" + + " OR a.payload::jsonb IS DISTINCT FROM b.payload::jsonb)" + + " ORDER BY a.ordinal, b.ordinal LIMIT 1" + if err := tx.QueryRow(ctx, inBatchSQL).Scan(&resourceType, &logicalID); err == nil { + return 0, &BatchResourceConflictError{ + Project: batch.Project, DatasetGeneration: batch.DatasetGeneration, + ResourceType: resourceType, LogicalID: logicalID, InBatch: true, + } + } else if !errors.Is(err, pgx.ErrNoRows) { + return 0, &BatchWriteError{Op: "check", BatchID: batch.ID, Err: err} + } + + storedSQL := "SELECT s.resource_type, s.logical_id FROM " + stg + " s JOIN " + w.resourceTableFQN + " r" + + " ON r.project = s.project AND r.dataset_generation = s.dataset_generation" + + " AND r.resource_type = s.resource_type AND r.logical_id = s.logical_id" + + " WHERE s.auth_resource_path IS DISTINCT FROM r.auth_resource_path" + + " OR (s.payload IS NULL) <> (r.payload IS NULL)" + + " OR s.payload::jsonb IS DISTINCT FROM r.payload" + + " ORDER BY s.ordinal LIMIT 1" + if err := tx.QueryRow(ctx, storedSQL).Scan(&resourceType, &logicalID); err == nil { + return 0, &BatchResourceConflictError{ + Project: batch.Project, DatasetGeneration: batch.DatasetGeneration, + ResourceType: resourceType, LogicalID: logicalID, InBatch: false, + } + } else if !errors.Is(err, pgx.ErrNoRows) { + return 0, &BatchWriteError{Op: "check", BatchID: batch.ID, Err: err} + } + + expectedSQL := "SELECT count(*) FROM (" + + " SELECT DISTINCT project, dataset_generation, resource_type, logical_id," + + " auth_resource_path, payload::jsonb AS payload FROM " + stg + + ") d WHERE NOT EXISTS (SELECT 1 FROM " + w.resourceTableFQN + " r" + + " WHERE r.project = d.project AND r.dataset_generation = d.dataset_generation" + + " AND r.resource_type = d.resource_type AND r.logical_id = d.logical_id" + + " AND r.auth_resource_path IS NOT DISTINCT FROM d.auth_resource_path" + + " AND r.payload IS NOT DISTINCT FROM d.payload)" + var expected int64 + if err := tx.QueryRow(ctx, expectedSQL).Scan(&expected); err != nil { + return 0, &BatchWriteError{Op: "check", BatchID: batch.ID, Err: err} + } + return expected, nil +} + +func (w *BatchWriter) publishResources(ctx context.Context, tx pgx.Tx, batchID string) (int64, error) { + sql := "INSERT INTO " + w.resourceTableFQN + + " (project, dataset_generation, resource_type, logical_id, auth_resource_path, payload)" + + " SELECT DISTINCT project, dataset_generation, resource_type, logical_id," + + " auth_resource_path, payload::jsonb FROM " + w.stagingResourceFQN() + + " ON CONFLICT ON CONSTRAINT " + resourceTableName + "_natural_identity_ux DO NOTHING" + + " RETURNING resource_seq" + rows, err := tx.Query(ctx, sql) + if err != nil { + return 0, &BatchWriteError{Op: "publish", BatchID: batchID, Err: err} + } + defer rows.Close() + var n int64 + for rows.Next() { + var seq int64 + if err := rows.Scan(&seq); err != nil { + return 0, &BatchWriteError{Op: "publish", BatchID: batchID, Err: err} + } + n++ + } + if err := rows.Err(); err != nil { + return 0, &BatchWriteError{Op: "publish", BatchID: batchID, Err: err} + } + return n, nil +} + +func (w *BatchWriter) writeEdgePhase(ctx context.Context, tx pgx.Tx, batch Batch) (int64, error) { + if err := w.hookFail(stepStageEdges); err != nil { + return 0, &BatchWriteError{Op: "create-staging", BatchID: batch.ID, Err: err} + } + if err := w.stageEdges(ctx, tx, batch); err != nil { + return 0, err + } + w.probe(stepStageEdges, batch.ID, tx) + + if err := w.hookFail(stepCheckEdges); err != nil { + return 0, &BatchWriteError{Op: "check", BatchID: batch.ID, Err: err} + } + expectedEdges, err := w.guardEdges(ctx, tx, batch) + if err != nil { + return 0, err + } + w.probe(stepCheckEdges, batch.ID, tx) + + if err := w.hookFail(stepPublishEdges); err != nil { + return 0, &BatchWriteError{Op: "publish", BatchID: batch.ID, Err: err} + } + committedEdges, err := w.publishEdges(ctx, tx, batch.ID) + if err != nil { + return 0, err + } + if committedEdges != expectedEdges { + return 0, &BatchAccountingError{ + BatchID: batch.ID, Side: "edges", + Expected: expectedEdges, Committed: committedEdges, + } + } + w.probe(stepPublishEdges, batch.ID, tx) + return committedEdges, nil +} + +func (w *BatchWriter) stageEdges(ctx context.Context, tx pgx.Tx, batch Batch) error { + const createSQL = "CREATE TEMP TABLE " + stagingEdgeTable + " (" + + "ordinal bigint NOT NULL, " + + "project text NOT NULL, " + + "dataset_generation text NOT NULL, " + + "auth_resource_path text NOT NULL, " + + "label text NOT NULL, " + + "source_resource_type text NOT NULL, " + + "source_logical_id text NOT NULL, " + + "target_resource_type text NOT NULL, " + + "target_logical_id text NOT NULL, " + + "source_path text NOT NULL, " + + "source_ordinal bigint NOT NULL, " + + "logical_edge_key text" + + ") ON COMMIT DROP" + if _, err := tx.Exec(ctx, createSQL); err != nil { + return &BatchWriteError{Op: "create-staging", BatchID: batch.ID, Err: err} + } + columns := []string{"ordinal", "project", "dataset_generation", "auth_resource_path", + "label", "source_resource_type", "source_logical_id", "target_resource_type", + "target_logical_id", "source_path", "source_ordinal", "logical_edge_key"} + rows := make([][]any, 0, len(batch.Edges)) + for i, rec := range batch.Edges { + var logicalEdgeKey any + if rec.LogicalEdgeKey != nil { + logicalEdgeKey = *rec.LogicalEdgeKey + } + rows = append(rows, []any{int64(i), rec.Project, rec.DatasetGeneration, + rec.AuthResourcePath, rec.Label, rec.SourceResourceType, rec.SourceLogicalID, + rec.TargetResourceType, rec.TargetLogicalID, rec.SourcePath, rec.SourceOrdinal, + logicalEdgeKey}) + } + n, err := tx.CopyFrom(ctx, pgx.Identifier{"pg_temp", stagingEdgeTable}, columns, + pgx.CopyFromRows(rows)) + if err != nil { + return &BatchWriteError{Op: "copy", BatchID: batch.ID, Err: err} + } + if n != int64(len(batch.Edges)) { + return &BatchAccountingError{BatchID: batch.ID, Side: "edges", + Expected: int64(len(batch.Edges)), Committed: n} + } + return nil +} + +func (w *BatchWriter) edgeIdentityJoin(left, right string) string { + return left + ".project = " + right + ".project" + + " AND " + left + ".dataset_generation = " + right + ".dataset_generation" + + " AND " + left + ".source_resource_type = " + right + ".source_resource_type" + + " AND " + left + ".source_logical_id = " + right + ".source_logical_id" + + " AND " + left + ".label = " + right + ".label" + + " AND " + left + ".target_resource_type = " + right + ".target_resource_type" + + " AND " + left + ".target_logical_id = " + right + ".target_logical_id" + + " AND " + left + ".source_path = " + right + ".source_path" + + " AND " + left + ".source_ordinal = " + right + ".source_ordinal" +} + +func (w *BatchWriter) guardEdges(ctx context.Context, tx pgx.Tx, batch Batch) (int64, error) { + stg := w.stagingEdgeFQN() + + var e EdgeOccurrenceRow + inBatchSQL := "SELECT a.project, a.dataset_generation, a.source_resource_type, a.source_logical_id," + + " a.label, a.target_resource_type, a.target_logical_id, a.source_path, a.source_ordinal" + + " FROM " + stg + " a JOIN " + stg + " b" + + " ON " + w.edgeIdentityJoin("a", "b") + + " AND (a.auth_resource_path IS DISTINCT FROM b.auth_resource_path" + + " OR a.logical_edge_key IS DISTINCT FROM b.logical_edge_key)" + + " ORDER BY a.ordinal, b.ordinal LIMIT 1" + err := tx.QueryRow(ctx, inBatchSQL).Scan(&e.Project, &e.DatasetGeneration, + &e.SourceResourceType, &e.SourceLogicalID, &e.Label, + &e.TargetResourceType, &e.TargetLogicalID, &e.SourcePath, &e.SourceOrdinal) + if err == nil { + return 0, &BatchEdgeConflictError{ + Project: e.Project, DatasetGeneration: e.DatasetGeneration, + SourceResourceType: e.SourceResourceType, SourceLogicalID: e.SourceLogicalID, + Label: e.Label, TargetResourceType: e.TargetResourceType, + TargetLogicalID: e.TargetLogicalID, SourcePath: e.SourcePath, + SourceOrdinal: e.SourceOrdinal, InBatch: true, + } + } + if !errors.Is(err, pgx.ErrNoRows) { + return 0, &BatchWriteError{Op: "check", BatchID: batch.ID, Err: err} + } + + storedSQL := "SELECT s.project, s.dataset_generation, s.source_resource_type, s.source_logical_id," + + " s.label, s.target_resource_type, s.target_logical_id, s.source_path, s.source_ordinal" + + " FROM " + stg + " s JOIN " + w.edgeTableFQN + " e" + + " ON " + w.edgeIdentityJoin("s", "e") + + " WHERE s.auth_resource_path IS DISTINCT FROM e.auth_resource_path" + + " OR s.logical_edge_key IS DISTINCT FROM e.logical_edge_key" + + " ORDER BY s.ordinal LIMIT 1" + err = tx.QueryRow(ctx, storedSQL).Scan(&e.Project, &e.DatasetGeneration, + &e.SourceResourceType, &e.SourceLogicalID, &e.Label, + &e.TargetResourceType, &e.TargetLogicalID, &e.SourcePath, &e.SourceOrdinal) + if err == nil { + return 0, &BatchEdgeConflictError{ + Project: e.Project, DatasetGeneration: e.DatasetGeneration, + SourceResourceType: e.SourceResourceType, SourceLogicalID: e.SourceLogicalID, + Label: e.Label, TargetResourceType: e.TargetResourceType, + TargetLogicalID: e.TargetLogicalID, SourcePath: e.SourcePath, + SourceOrdinal: e.SourceOrdinal, InBatch: false, + } + } + if !errors.Is(err, pgx.ErrNoRows) { + return 0, &BatchWriteError{Op: "check", BatchID: batch.ID, Err: err} + } + + sourceSQL := "SELECT s.source_resource_type, s.source_logical_id FROM " + stg + " s" + + " WHERE NOT EXISTS (SELECT 1 FROM " + w.resourceTableFQN + " src" + + " WHERE src.project = s.project AND src.dataset_generation = s.dataset_generation" + + " AND src.resource_type = s.source_resource_type AND src.logical_id = s.source_logical_id)" + + " ORDER BY s.ordinal LIMIT 1" + var sourceType, sourceID string + err = tx.QueryRow(ctx, sourceSQL).Scan(&sourceType, &sourceID) + if err == nil { + return 0, &EdgeSourceMissingError{ + Project: batch.Project, DatasetGeneration: batch.DatasetGeneration, + SourceResourceType: sourceType, SourceLogicalID: sourceID, + } + } + if !errors.Is(err, pgx.ErrNoRows) { + return 0, &BatchWriteError{Op: "check", BatchID: batch.ID, Err: err} + } + + expectedSQL := "SELECT count(*) FROM (" + + " SELECT DISTINCT project, dataset_generation, auth_resource_path, label," + + " source_resource_type, source_logical_id, target_resource_type, target_logical_id," + + " source_path, source_ordinal, logical_edge_key FROM " + stg + + ") d WHERE NOT EXISTS (SELECT 1 FROM " + w.edgeTableFQN + " e" + + " WHERE " + w.edgeIdentityJoin("d", "e") + + " AND e.auth_resource_path IS NOT DISTINCT FROM d.auth_resource_path" + + " AND e.logical_edge_key IS NOT DISTINCT FROM d.logical_edge_key)" + var expected int64 + if err := tx.QueryRow(ctx, expectedSQL).Scan(&expected); err != nil { + return 0, &BatchWriteError{Op: "check", BatchID: batch.ID, Err: err} + } + return expected, nil +} + +func (w *BatchWriter) publishEdges(ctx context.Context, tx pgx.Tx, batchID string) (int64, error) { + sql := "INSERT INTO " + w.edgeTableFQN + + " (project, dataset_generation, auth_resource_path, label," + + " source_resource_type, source_logical_id, target_resource_type, target_logical_id," + + " source_path, source_ordinal, logical_edge_key)" + + " SELECT DISTINCT project, dataset_generation, auth_resource_path, label," + + " source_resource_type, source_logical_id, target_resource_type, target_logical_id," + + " source_path, source_ordinal, logical_edge_key FROM " + w.stagingEdgeFQN() + " s" + + " WHERE EXISTS (SELECT 1 FROM " + w.resourceTableFQN + " src" + + " WHERE src.project = s.project AND src.dataset_generation = s.dataset_generation" + + " AND src.resource_type = s.source_resource_type AND src.logical_id = s.source_logical_id)" + + " ON CONFLICT ON CONSTRAINT " + edgeTableName + "_identity_ux DO NOTHING" + + " RETURNING edge_seq" + rows, err := tx.Query(ctx, sql) + if err != nil { + return 0, &BatchWriteError{Op: "publish", BatchID: batchID, Err: err} + } + defer rows.Close() + var n int64 + for rows.Next() { + var seq int64 + if err := rows.Scan(&seq); err != nil { + return 0, &BatchWriteError{Op: "publish", BatchID: batchID, Err: err} + } + n++ + } + if err := rows.Err(); err != nil { + return 0, &BatchWriteError{Op: "publish", BatchID: batchID, Err: err} + } + return n, nil +} + +func validateBatch(batch Batch) error { + if strings.TrimSpace(batch.ID) == "" { + return fmt.Errorf("%w: batch id is blank", ErrInvalidBatch) + } + if len(batch.ID) > MaxBatchIDLength { + return fmt.Errorf("%w: batch id is %d bytes, want at most %d", + ErrInvalidBatch, len(batch.ID), MaxBatchIDLength) + } + if strings.TrimSpace(batch.Project) == "" { + return fmt.Errorf("%w: project is blank", ErrInvalidBatch) + } + if len(batch.Resources) == 0 && len(batch.Edges) == 0 { + return fmt.Errorf("%w: batch %q carries no rows", ErrInvalidBatch, batch.ID) + } + if len(batch.Resources) > MaxBatchResources { + return fmt.Errorf("%w: batch %q offers %d resources, want at most %d", + ErrInvalidBatch, batch.ID, len(batch.Resources), MaxBatchResources) + } + if len(batch.Edges) > MaxBatchEdges { + return fmt.Errorf("%w: batch %q offers %d edges, want at most %d", + ErrInvalidBatch, batch.ID, len(batch.Edges), MaxBatchEdges) + } + for i, rec := range batch.Resources { + if rec.Project != batch.Project || rec.DatasetGeneration != batch.DatasetGeneration { + return fmt.Errorf("%w: batch %q resource %d is scoped to %s/%s, want the batch scope %s/%s", + ErrInvalidBatch, batch.ID, i, rec.Project, rec.DatasetGeneration, + batch.Project, batch.DatasetGeneration) + } + if err := validateResourceForInsert(ResourceRow{Resource: rec}); err != nil { + return fmt.Errorf("%w: batch %q resource %d: %w", ErrInvalidBatch, batch.ID, i, err) + } + } + for i, rec := range batch.Edges { + if rec.Project != batch.Project || rec.DatasetGeneration != batch.DatasetGeneration { + return fmt.Errorf("%w: batch %q edge %d is scoped to %s/%s, want the batch scope %s/%s", + ErrInvalidBatch, batch.ID, i, rec.Project, rec.DatasetGeneration, + batch.Project, batch.DatasetGeneration) + } + if err := validateEdgeForInsert(rec); err != nil { + return fmt.Errorf("%w: batch %q edge %d: %w", ErrInvalidBatch, batch.ID, i, err) + } + } + return nil +} + +func batchDigest(batch Batch) string { + h := sha256.New() + put := func(s string) { + h.Write([]byte(strconv.Itoa(len(s)))) + h.Write([]byte{':'}) + h.Write([]byte(s)) + } + putTagged := func(present bool, s string) { + if present { + h.Write([]byte{'V'}) + put(s) + return + } + h.Write([]byte{'N'}) + } + put("loom.batch.v1\n") + put(batch.ID) + put(batch.Project) + put(batch.DatasetGeneration) + put("R" + strconv.Itoa(len(batch.Resources))) + for _, rec := range batch.Resources { + put(rec.Project) + put(rec.DatasetGeneration) + put(rec.ResourceType) + put(rec.LogicalID) + put(rec.AuthResourcePath) + putTagged(rec.Payload != nil, string(rec.Payload)) + } + put("E" + strconv.Itoa(len(batch.Edges))) + for _, rec := range batch.Edges { + put(rec.Project) + put(rec.DatasetGeneration) + put(rec.AuthResourcePath) + put(rec.Label) + put(rec.SourceResourceType) + put(rec.SourceLogicalID) + put(rec.TargetResourceType) + put(rec.TargetLogicalID) + put(rec.SourcePath) + put("o" + strconv.FormatInt(rec.SourceOrdinal, 10)) + putTagged(rec.LogicalEdgeKey != nil, derefString(rec.LogicalEdgeKey)) + } + return hex.EncodeToString(h.Sum(nil)) +} + +func derefString(s *string) string { + if s == nil { + return "" + } + return *s +} diff --git a/internal/backends/postgres/internal/store/batch_writer_test.go b/internal/backends/postgres/internal/store/batch_writer_test.go new file mode 100644 index 000000000..0d1729d04 --- /dev/null +++ b/internal/backends/postgres/internal/store/batch_writer_test.go @@ -0,0 +1,1535 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +var pgstore06NamePattern = regexp.MustCompile(`^pgstore06_it_[0-9]+_[0-9a-f]+$`) + +var ( + batchOwnedMu sync.Mutex + batchOwnedByName = map[string]string{} +) + +func batchRegister(db, schema string) { + batchOwnedMu.Lock() + defer batchOwnedMu.Unlock() + batchOwnedByName[db] = schema +} + +type batchService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func newBatchService(t *testing.T) *batchService { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgstore06_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstore06NamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("pgstore06_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstore06NamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + batchRegister(dbName, schema) + return &batchService{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} +} + +func (s *batchService) migrate(t *testing.T) { + t.Helper() + if _, err := MigrateSchema(s.ctx, s.openClient(t, 4), s.schema); err != nil { + s.fail("MigrateSchema", err) + } +} + +func (s *batchService) openClient(t *testing.T, maxConns int32) *Client { + t.Helper() + client, err := Open(s.ctx, Config{ + DSN: s.dsn, + MaxConns: maxConns, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *batchService) openAppClient(t *testing.T, app string, maxConns int32) *Client { + t.Helper() + u, err := url.Parse(s.dsn) + if err != nil { + s.fail("parse isolated dsn for app client", err) + } + q := u.Query() + q.Set("application_name", app) + u.RawQuery = q.Encode() + client, err := Open(s.ctx, Config{ + DSN: u.String(), + MaxConns: maxConns, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + s.fail("open app client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *batchService) newWriterOn(t *testing.T, src ConnSource) *BatchWriter { + t.Helper() + writer, err := NewBatchWriter(src, s.schema) + if err != nil { + s.fail("new batch writer", err) + } + return writer +} + +func (s *batchService) newWriter(t *testing.T) *BatchWriter { + t.Helper() + return s.newWriterOn(t, s.openClient(t, 4)) +} + +func (s *batchService) qualified(table string) string { + return pgx.Identifier{s.schema, table}.Sanitize() +} + +func (s *batchService) exec(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := s.iso.Exec(s.ctx, sql, args...); err != nil { + s.fail("fixture exec", err) + } +} + +func (s *batchService) mustInt64(t *testing.T, sql string, args ...any) int64 { + t.Helper() + var v int64 + if err := s.iso.QueryRow(s.ctx, sql, args...).Scan(&v); err != nil { + s.fail("witness "+sql, err) + } + return v +} + +func (s *batchService) mustText(t *testing.T, sql string, args ...any) string { + t.Helper() + var v string + if err := s.iso.QueryRow(s.ctx, sql, args...).Scan(&v); err != nil { + s.fail("witness "+sql, err) + } + return v +} + +func (s *batchService) backendPID(t *testing.T, pool *pgxpool.Pool) int64 { + t.Helper() + var pid int64 + if err := pool.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + s.fail("witness pg_backend_pid", err) + } + return pid +} + +const ( + batchProject = "prj-batch" + batchGenA = "gen-a" + batchGenB = "gen-b" +) + +func litResource(resourceType, logicalID, authPath string, payload *string) source.Resource { + rec := source.Resource{ + Project: batchProject, DatasetGeneration: batchGenA, + ResourceType: resourceType, LogicalID: logicalID, AuthResourcePath: authPath, + } + if payload != nil { + rec.Payload = json.RawMessage(*payload) + } + return rec +} + +func litEdge(label, srcType, srcID, tgtType, tgtID, sourcePath string, ordinal int64, + authPath string, key *string) EdgeOccurrenceRow { + return EdgeOccurrenceRow{ + Project: batchProject, DatasetGeneration: batchGenA, AuthResourcePath: authPath, + Label: label, SourceResourceType: srcType, SourceLogicalID: srcID, + TargetResourceType: tgtType, TargetLogicalID: tgtID, + SourcePath: sourcePath, SourceOrdinal: ordinal, LogicalEdgeKey: key, + } +} + +func strPtr(s string) *string { return &s } + +func goldenBatch() Batch { + p1 := `{"resourceType":"Patient","id":"p-1","link":[{"type":{"coding":[{"code":"X"}]},"target":"Organization/o-1"}]}` + p2 := `{"resourceType":"Patient","id":"p-2","meta":{"tag":null}}` + return Batch{ + ID: "pgstore06/golden/0", Project: "prj-golden", DatasetGeneration: "gen-golden-1", + Resources: []source.Resource{ + {Project: "prj-golden", DatasetGeneration: "gen-golden-1", ResourceType: "Patient", LogicalID: "p-1", AuthResourcePath: "/fhir", Payload: json.RawMessage(p1)}, + {Project: "prj-golden", DatasetGeneration: "gen-golden-1", ResourceType: "Patient", LogicalID: "p-2", AuthResourcePath: "", Payload: json.RawMessage(p2)}, + {Project: "prj-golden", DatasetGeneration: "gen-golden-1", ResourceType: "Observation", LogicalID: "obs-9", AuthResourcePath: "/fhir", Payload: nil}, + {Project: "prj-golden", DatasetGeneration: "gen-golden-1", ResourceType: "Provenance", LogicalID: "prov-0", AuthResourcePath: "/fhir", Payload: json.RawMessage(`null`)}, + }, + Edges: []EdgeOccurrenceRow{ + {Project: "prj-golden", DatasetGeneration: "gen-golden-1", AuthResourcePath: "/fhir", + Label: "referenced-by", SourceResourceType: "Patient", SourceLogicalID: "p-1", + TargetResourceType: "Organization", TargetLogicalID: "o-1", + SourcePath: "Patient.link.target", SourceOrdinal: 0, LogicalEdgeKey: strPtr("ek-org-link-1")}, + {Project: "prj-golden", DatasetGeneration: "gen-golden-1", AuthResourcePath: "/fhir", + Label: "referenced-by", SourceResourceType: "Patient", SourceLogicalID: "p-1", + TargetResourceType: "Organization", TargetLogicalID: "o-1", + SourcePath: "Patient.link.target", SourceOrdinal: 1, LogicalEdgeKey: nil}, + }, + } +} + +const goldenDigest = "7ad729506a5590958ed4b7dd8e9ea9eed486cbeedff700d6a51f6170e3faac99" + +func (s *batchService) requireResourceRow(t *testing.T, project, generation, rtype, lid, authPath string, wantPayload *string) { + t.Helper() + var gotAuth string + var isNull bool + var payload *string + err := s.iso.QueryRow(s.ctx, + "SELECT auth_resource_path, payload IS NULL, payload::text FROM "+s.qualified(resourceTableName)+ + " WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3 AND logical_id = $4", + project, generation, rtype, lid).Scan(&gotAuth, &isNull, &payload) + if err != nil { + s.fail("witness resource row "+rtype+"/"+lid, err) + } + if gotAuth != authPath { + t.Fatalf("resource %s/%s auth path = %q, want %q", rtype, lid, gotAuth, authPath) + } + if wantPayload == nil { + if !isNull { + t.Fatalf("resource %s/%s payload = %v, want SQL NULL", rtype, lid, *payload) + } + return + } + if isNull { + t.Fatalf("resource %s/%s payload is SQL NULL, want document %s", rtype, lid, *wantPayload) + } + if !jsonEquivalent(*payload, *wantPayload) { + t.Fatalf("resource %s/%s payload = %v, want %s", rtype, lid, *payload, *wantPayload) + } +} + +func jsonEquivalent(got, want string) bool { + var g, w any + if err := json.Unmarshal([]byte(got), &g); err != nil { + return false + } + if err := json.Unmarshal([]byte(want), &w); err != nil { + return false + } + return reflect.DeepEqual(g, w) +} + +func (s *batchService) requireEdgeRow(t *testing.T, project, generation, label, srcType, srcID, tgtType, tgtID, sourcePath string, + ordinal int64, authPath string, wantKey *string) { + t.Helper() + var gotAuth string + var key *string + var count int + err := s.iso.QueryRow(s.ctx, + "SELECT count(*), max(auth_resource_path), max(logical_edge_key) FROM "+s.qualified(edgeTableName)+ + " WHERE project = $1 AND dataset_generation = $2 AND label = $3"+ + " AND source_resource_type = $4 AND source_logical_id = $5"+ + " AND target_resource_type = $6 AND target_logical_id = $7"+ + " AND source_path = $8 AND source_ordinal = $9", + project, generation, label, srcType, srcID, tgtType, tgtID, sourcePath, ordinal, + ).Scan(&count, &gotAuth, &key) + if err != nil { + s.fail("witness edge row", err) + } + if count != 1 { + t.Fatalf("edge occurrence rows = %d, want exactly 1", count) + } + if gotAuth != authPath { + t.Fatalf("edge auth path = %q, want %q", gotAuth, authPath) + } + if (wantKey == nil) != (key == nil) { + t.Fatalf("edge logical_edge_key = %v, want %v", keyPtrText(key), keyPtrText(wantKey)) + } + if wantKey != nil && key != nil && *wantKey != *key { + t.Fatalf("edge logical_edge_key = %q, want %q", *key, *wantKey) + } +} + +func keyPtrText(p *string) string { + if p == nil { + return "SQL NULL" + } + return fmt.Sprintf("%q", *p) +} + +type ledgerWitness struct { + digest string + resources int64 + edges int64 + project string + generation string + committedAt time.Time +} + +func (s *batchService) witnessLedger(t *testing.T, batchID string) (ledgerWitness, bool) { + t.Helper() + var w ledgerWitness + err := s.iso.QueryRow(s.ctx, + "SELECT digest, resource_count, edge_count, project, dataset_generation, committed_at"+ + " FROM "+s.qualified(batchLedgerTableName)+" WHERE batch_id = $1", batchID). + Scan(&w.digest, &w.resources, &w.edges, &w.project, &w.generation, &w.committedAt) + if errors.Is(err, pgx.ErrNoRows) { + return ledgerWitness{}, false + } + if err != nil { + s.fail("witness ledger row", err) + } + return w, true +} + +func (s *batchService) countIn(t *testing.T, table, project, generation string) int64 { + t.Helper() + return s.mustInt64(t, "SELECT count(*) FROM "+s.qualified(table)+ + " WHERE project = $1 AND dataset_generation = $2", project, generation) +} + +func (s *batchService) requireNothingWritten(t *testing.T, project, generation, batchID string) { + t.Helper() + if n := s.countIn(t, resourceTableName, project, generation); n != 0 { + t.Fatalf("fhir_resource rows for %s/%s = %d, want 0", project, generation, n) + } + if n := s.countIn(t, edgeTableName, project, generation); n != 0 { + t.Fatalf("fhir_edge_occurrence rows for %s/%s = %d, want 0", project, generation, n) + } + if _, found := s.witnessLedger(t, batchID); found { + t.Fatalf("batch_write_ledger holds %q, want no marker", batchID) + } +} + +func TestBatchWriterConstructorRejectsInvalidInputs(t *testing.T) { + if _, err := NewBatchWriter(nil, "ok_schema"); err == nil { + t.Error("nil connection source: want error") + } + for _, bad := range []string{"", "Public", "a.b", "1x", strings.Repeat("a", 64)} { + if _, err := NewBatchWriter(stubConnSource{}, bad); !errors.Is(err, ErrInvalidSchemaName) { + t.Errorf("schema %q: NewBatchWriter = %v, want ErrInvalidSchemaName", bad, err) + } + } +} + +func TestBatchWriterRejectsInvalidBatches(t *testing.T) { + writer, err := NewBatchWriter(stubConnSource{}, "ok_schema") + if err != nil { + t.Fatal(err) + } + key := "" + okResource := litResource("Patient", "p-1", "/fhir", strPtr(`{"id":"p-1"}`)) + okEdge := litEdge("referenced-by", "Patient", "p-1", "Organization", "o-1", "a", 0, "", &key) + + base := func(mutate func(*Batch)) Batch { + b := Batch{ID: "b/1", Project: batchProject, DatasetGeneration: batchGenA} + mutate(&b) + return b + } + for _, tc := range []struct { + name string + batch Batch + }{ + {"blank id", base(func(b *Batch) { b.ID = " "; b.Resources = []source.Resource{okResource} })}, + {"long id", base(func(b *Batch) { + b.ID = strings.Repeat("x", MaxBatchIDLength+1) + b.Resources = []source.Resource{okResource} + })}, + {"blank project", base(func(b *Batch) { b.Project = ""; b.Resources = []source.Resource{okResource} })}, + {"no rows", base(func(*Batch) {})}, + {"too many resources", base(func(b *Batch) { + for i := 0; i <= MaxBatchResources; i++ { + b.Resources = append(b.Resources, okResource) + } + })}, + {"too many edges", base(func(b *Batch) { + for i := 0; i <= MaxBatchEdges; i++ { + b.Edges = append(b.Edges, okEdge) + } + })}, + {"resource scope mismatch", base(func(b *Batch) { + rec := okResource + rec.DatasetGeneration = "other-gen" + b.Resources = []source.Resource{rec} + })}, + {"edge scope mismatch", base(func(b *Batch) { + b.Edges = append(b.Edges, EdgeOccurrenceRow{ + Project: "other-prj", DatasetGeneration: batchGenA, AuthResourcePath: "", Label: "l", + SourceResourceType: "Patient", SourceLogicalID: "p-1", + TargetResourceType: "Organization", TargetLogicalID: "o-1", + }) + })}, + {"invalid json payload", base(func(b *Batch) { + rec := okResource + rec.Payload = json.RawMessage("{oops") + b.Resources = []source.Resource{rec} + })}, + {"blank resource type", base(func(b *Batch) { + rec := okResource + rec.ResourceType = "" + b.Resources = []source.Resource{rec} + })}, + {"caller-supplied edge seq", base(func(b *Batch) { + rec := okEdge + rec.EdgeSeq = 9 + b.Edges = []EdgeOccurrenceRow{rec} + })}, + {"negative edge ordinal", base(func(b *Batch) { + rec := okEdge + rec.SourceOrdinal = -1 + b.Edges = []EdgeOccurrenceRow{rec} + })}, + {"blank edge label", base(func(b *Batch) { + rec := okEdge + rec.Label = "" + b.Edges = []EdgeOccurrenceRow{rec} + })}, + } { + _, err := writer.WriteBatch(context.Background(), tc.batch) + if !errors.Is(err, ErrInvalidBatch) { + t.Errorf("%s: WriteBatch = %v, want ErrInvalidBatch", tc.name, err) + } + } +} + +func TestBatchWriterPublishesGoldenBatchViaStagingWithLedgerAccounting(t *testing.T) { + s := newBatchService(t) + s.migrate(t) + + type probeFact struct { + backend int64 + stgResCount int64 + stgEdgeCnt int64 + pubResCount int64 + pubEdgeCnt int64 + stgEdgeVis bool + txStart time.Time + } + var facts sync.Map + var probeErrs []string + var probeMu sync.Mutex + seenSteps := map[string][]string{} + var seenMu sync.Mutex + + writer := s.newWriterOn(t, s.openClient(t, 1)) + writer.inTxProbe = func(step, batchID string, tx pgx.Tx) { + seenMu.Lock() + seenSteps[step] = append(seenSteps[step], batchID) + seenMu.Unlock() + v, _ := facts.LoadOrStore(batchID, &probeFact{}) + f := v.(*probeFact) + note := func(err error) { + probeMu.Lock() + probeErrs = append(probeErrs, step+": "+err.Error()) + probeMu.Unlock() + } + if err := tx.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&f.backend); err != nil { + note(err) + return + } + switch step { + case stepStageResources: + if err := tx.QueryRow(s.ctx, + "SELECT count(*), coalesce(min(ordinal),-1), coalesce(max(ordinal),-1) FROM "+ + writer.stagingResourceFQN()).Scan(&f.stgResCount, new(int64), new(int64)); err != nil { + note(err) + } + case stepCheckResources: + if err := tx.QueryRow(s.ctx, + "SELECT count(*) FROM "+s.qualified(resourceTableName)+ + " WHERE project = $1 AND dataset_generation = $2", + "prj-golden", "gen-golden-1").Scan(&f.pubResCount); err != nil { + note(err) + } + case stepPublishResourc: + if err := tx.QueryRow(s.ctx, + "SELECT count(*) FROM "+s.qualified(resourceTableName)+ + " WHERE project = $1 AND dataset_generation = $2", + "prj-golden", "gen-golden-1").Scan(&f.pubResCount); err != nil { + note(err) + } + case stepCheckEdges: + if err := tx.QueryRow(s.ctx, + "SELECT count(*) FROM "+s.qualified(edgeTableName)+ + " WHERE project = $1 AND dataset_generation = $2", + "prj-golden", "gen-golden-1").Scan(&f.pubEdgeCnt); err != nil { + note(err) + } + var resNow int64 + if err := tx.QueryRow(s.ctx, + "SELECT count(*) FROM "+s.qualified(resourceTableName)+ + " WHERE project = $1 AND dataset_generation = $2", + "prj-golden", "gen-golden-1").Scan(&resNow); err != nil { + note(err) + } + f.stgEdgeVis = resNow == 4 + case stepPublishEdges: + if err := tx.QueryRow(s.ctx, + "SELECT count(*) FROM "+writer.stagingEdgeFQN()).Scan(&f.stgEdgeCnt); err != nil { + note(err) + } + case stepFinalizeLedger: + if err := tx.QueryRow(s.ctx, "SELECT now()").Scan(&f.txStart); err != nil { + note(err) + } + } + } + + res, err := writer.WriteBatch(s.ctx, goldenBatch()) + if err != nil { + s.fail("WriteBatch golden", err) + } + for _, msg := range probeErrs { + t.Errorf("probe %s", msg) + } + if want := (BatchResult{ + BatchID: "pgstore06/golden/0", Digest: goldenDigest, + CommittedResources: 4, CommittedEdges: 2, + }); res != want { + t.Fatalf("result = %+v, want %+v", res, want) + } + + v, ok := facts.Load("pgstore06/golden/0") + if !ok { + t.Fatal("probe never fired") + } + f := v.(*probeFact) + if f.backend == 0 { + t.Fatal("probe backend pid not captured") + } + if f.stgResCount != 4 { + t.Errorf("staging resources before publish = %d, want 4 (COPY landed rows in staging)", f.stgResCount) + } + if f.pubResCount != 4 { + t.Errorf("fhir_resource count observed mid-publish = %d, want 4", f.pubResCount) + } + if f.pubEdgeCnt != 0 { + t.Errorf("fhir_edge_occurrence count before edge publish = %d, want 0", f.pubEdgeCnt) + } + if !f.stgEdgeVis { + t.Errorf("batch's own resources were not visible to the edge guard inside the transaction") + } + if f.stgEdgeCnt != 2 { + t.Errorf("staging edges before publish = %d, want 2", f.stgEdgeCnt) + } + if seen := seenSteps[stepCheckResources]; len(seen) != 1 { + t.Errorf("check-resources fired %d times, want 1", len(seen)) + } + + p1 := `{"resourceType":"Patient","id":"p-1","link":[{"type":{"coding":[{"code":"X"}]},"target":"Organization/o-1"}]}` + p2 := `{"resourceType":"Patient","id":"p-2","meta":{"tag":null}}` + s.requireResourceRow(t, "prj-golden", "gen-golden-1", "Patient", "p-1", "/fhir", &p1) + s.requireResourceRow(t, "prj-golden", "gen-golden-1", "Patient", "p-2", "", &p2) + s.requireResourceRow(t, "prj-golden", "gen-golden-1", "Observation", "obs-9", "/fhir", nil) + s.requireResourceRow(t, "prj-golden", "gen-golden-1", "Provenance", "prov-0", "/fhir", strPtr(`null`)) + if has := s.mustInt64(t, "SELECT count(*) FROM "+s.qualified(resourceTableName)+ + " WHERE project = $1 AND dataset_generation = $2 AND resource_type = 'Patient'"+ + " AND logical_id = 'p-1' AND payload ? 'meta'", "prj-golden", "gen-golden-1"); has != 0 { + t.Errorf("p-1 payload gained a meta key") + } + if has := s.mustInt64(t, "SELECT count(*) FROM "+s.qualified(resourceTableName)+ + " WHERE project = $1 AND dataset_generation = $2 AND resource_type = 'Patient'"+ + " AND logical_id = 'p-2' AND jsonb_typeof(payload->'meta'->'tag') = 'null'", + "prj-golden", "gen-golden-1"); has != 1 { + t.Errorf("p-2 lost its present-null meta.tag") + } + s.requireEdgeRow(t, "prj-golden", "gen-golden-1", "referenced-by", "Patient", "p-1", + "Organization", "o-1", "Patient.link.target", 0, "/fhir", strPtr("ek-org-link-1")) + s.requireEdgeRow(t, "prj-golden", "gen-golden-1", "referenced-by", "Patient", "p-1", + "Organization", "o-1", "Patient.link.target", 1, "/fhir", nil) + + lw, found := s.witnessLedger(t, "pgstore06/golden/0") + if !found { + t.Fatal("ledger marker missing after successful batch") + } + if lw.digest != goldenDigest || lw.resources != 4 || lw.edges != 2 || + lw.project != "prj-golden" || lw.generation != "gen-golden-1" || lw.committedAt.IsZero() { + t.Fatalf("ledger row = %+v, want digest %s resources 4 edges 2 prj-golden/gen-golden-1", + lw, goldenDigest) + } + if total := s.mustInt64(t, "SELECT count(*) FROM "+s.qualified(batchLedgerTableName)); total != 1 { + t.Errorf("ledger rows = %d, want exactly 1", total) + } + if !lw.committedAt.After(f.txStart) { + t.Errorf("ledger committed_at = %v, want strictly after the batch transaction start %v "+ + "(finalize must stamp clock_timestamp(), not keep the claim-time now() default)", + lw.committedAt, f.txStart) + } + + writerPool := writer.src.(*Client) + conn, err := writerPool.Acquire(s.ctx) + if err != nil { + s.fail("acquire writer pool connection", err) + } + defer conn.Release() + var stagingGone bool + if err := conn.QueryRow(s.ctx, + "SELECT to_regclass('pg_temp.stg_batch_resource') IS NULL"+ + " AND to_regclass('pg_temp.stg_batch_edge') IS NULL").Scan(&stagingGone); err != nil { + s.fail("witness staging drop on writer backend", err) + } + if !stagingGone { + t.Errorf("staging temp tables survived commit on the writer backend") + } +} + +func TestBatchWriterReplayOfIdenticalCommittedBatchIsNoOpTwoConnections(t *testing.T) { + s := newBatchService(t) + s.migrate(t) + + writerA := s.newWriterOn(t, s.openClient(t, 1)) + writerB := s.newWriterOn(t, s.openClient(t, 1)) + var pidA, pidB int64 + captureA := func(step, _ string, tx pgx.Tx) { + if step == stepBegin { + if err := tx.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&pidA); err != nil { + s.fail("witness writer A backend", err) + } + } + } + captureB := func(step, _ string, tx pgx.Tx) { + if step == stepBegin { + if err := tx.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&pidB); err != nil { + s.fail("witness writer B backend", err) + } + } + } + writerA.inTxProbe = captureA + writerB.inTxProbe = captureB + + first, err := writerA.WriteBatch(s.ctx, goldenBatch()) + if err != nil { + s.fail("first WriteBatch", err) + } + if first.Replay || first.CommittedResources != 4 || first.CommittedEdges != 2 { + t.Fatalf("first result = %+v, want a fresh 4/2 commit", first) + } + before, found := s.witnessLedger(t, "pgstore06/golden/0") + if !found { + t.Fatal("ledger marker missing after first commit") + } + + replay, err := writerB.WriteBatch(s.ctx, goldenBatch()) + if err != nil { + s.fail("replay WriteBatch", err) + } + if !replay.Replay || replay.CommittedResources != 4 || replay.CommittedEdges != 2 || + replay.Digest != goldenDigest || replay.BatchID != "pgstore06/golden/0" { + t.Fatalf("replay result = %+v, want replay of 4/2 with digest %s", replay, goldenDigest) + } + if pidA == 0 || pidB == 0 || pidA == pidB { + t.Fatalf("writer backends A=%d B=%d, want two distinct real PG connections", pidA, pidB) + } + after, found := s.witnessLedger(t, "pgstore06/golden/0") + if !found { + t.Fatal("ledger marker vanished after replay") + } + if !after.committedAt.Equal(before.committedAt) || after.digest != before.digest || + after.resources != 4 || after.edges != 2 { + t.Fatalf("ledger changed across replay: before %+v after %+v", before, after) + } + if n := s.countIn(t, resourceTableName, "prj-golden", "gen-golden-1"); n != 4 { + t.Errorf("fhir_resource rows after replay = %d, want still 4 (no-op)", n) + } + if n := s.countIn(t, edgeTableName, "prj-golden", "gen-golden-1"); n != 2 { + t.Errorf("fhir_edge_occurrence rows after replay = %d, want still 2 (no-op)", n) + } +} + +func TestBatchWriterIdentityReuseWithDifferentContentReturnsTypedConflict(t *testing.T) { + s := newBatchService(t) + s.migrate(t) + writer := s.newWriter(t) + + if _, err := writer.WriteBatch(s.ctx, goldenBatch()); err != nil { + s.fail("seed WriteBatch", err) + } + + mutated := goldenBatch() + mutated.Resources[1].Payload = json.RawMessage(`{"resourceType":"Patient","id":"p-2","meta":{"tag":"mutated"}}`) + _, err := writer.WriteBatch(s.ctx, mutated) + var idErr *BatchIdentityConflictError + if !errors.As(err, &idErr) || !errors.Is(err, ErrBatchIdentityConflict) { + t.Fatalf("mutated retry = %v, want *BatchIdentityConflictError", err) + } + if idErr.BatchID != "pgstore06/golden/0" || idErr.LedgerDigest != goldenDigest || + idErr.ProvidedDigest == goldenDigest || len(idErr.ProvidedDigest) != 64 { + t.Fatalf("conflict fields = %+v, want ledger digest %s and a differing provided digest", + idErr, goldenDigest) + } + + lw, found := s.witnessLedger(t, "pgstore06/golden/0") + if !found || lw.digest != goldenDigest || lw.resources != 4 || lw.edges != 2 { + t.Fatalf("ledger after refused retry = %+v, want the untouched golden row", lw) + } + p2 := `{"resourceType":"Patient","id":"p-2","meta":{"tag":null}}` + s.requireResourceRow(t, "prj-golden", "gen-golden-1", "Patient", "p-2", "", &p2) + if n := s.countIn(t, resourceTableName, "prj-golden", "gen-golden-1"); n != 4 { + t.Errorf("fhir_resource rows after refused retry = %d, want still 4", n) + } +} + +func TestBatchWriterTamperedLedgerDigestRefusesIdenticalReplay(t *testing.T) { + s := newBatchService(t) + s.migrate(t) + writer := s.newWriter(t) + + if _, err := writer.WriteBatch(s.ctx, goldenBatch()); err != nil { + s.fail("seed WriteBatch", err) + } + s.exec(t, "UPDATE "+s.qualified(batchLedgerTableName)+" SET digest = $1 WHERE batch_id = $2", + strings.Repeat("0", 64), "pgstore06/golden/0") + _, err := writer.WriteBatch(s.ctx, goldenBatch()) + var idErr *BatchIdentityConflictError + if !errors.As(err, &idErr) || !errors.Is(err, ErrBatchIdentityConflict) { + t.Fatalf("replay after digest tamper = %v, want *BatchIdentityConflictError", err) + } + if idErr.LedgerDigest != strings.Repeat("0", 64) || idErr.ProvidedDigest != goldenDigest { + t.Fatalf("conflict fields = %+v, want the tampered ledger digest and the golden provided digest", idErr) + } + if n := s.countIn(t, resourceTableName, "prj-golden", "gen-golden-1"); n != 4 { + t.Errorf("fhir_resource rows = %d, want the batch to have written nothing", n) + } +} + +func TestBatchWriterInBatchDuplicateResourceIdentityDifferentContentRollsBack(t *testing.T) { + s := newBatchService(t) + s.migrate(t) + writer := s.newWriter(t) + + mk := func() Batch { + return Batch{ID: "prj/dup-inbatch", Project: batchProject, DatasetGeneration: batchGenA, Resources: []source.Resource{ + litResource("Patient", "p-dup", "/fhir", strPtr(`{"id":"p-dup","v":"first"}`)), + litResource("Patient", "p-dup", "/fhir", strPtr(`{"id":"p-dup","v":"second"}`)), + litResource("Patient", "p-ok", "", nil), + }} + } + _, err := writer.WriteBatch(s.ctx, mk()) + var rowErr *BatchResourceConflictError + if !errors.As(err, &rowErr) || !errors.Is(err, ErrBatchRowConflict) { + t.Fatalf("in-batch duplicate = %v, want *BatchResourceConflictError", err) + } + if !rowErr.InBatch || rowErr.ResourceType != "Patient" || rowErr.LogicalID != "p-dup" || + rowErr.Project != batchProject || rowErr.DatasetGeneration != batchGenA { + t.Fatalf("conflict fields = %+v, want the in-batch Patient/p-dup identity in batch scope", rowErr) + } + s.requireNothingWritten(t, batchProject, batchGenA, "prj/dup-inbatch") + + b := mk() + b.Resources = append(b.Resources[:0:2], b.Resources[0], b.Resources[2]) + res, err := writer.WriteBatch(s.ctx, b) + if err != nil { + s.fail("control WriteBatch", err) + } + if res.CommittedResources != 2 || res.Replay { + t.Fatalf("control result = %+v, want a fresh 2-row commit", res) + } +} + +func TestBatchWriterStoredResourceIdentityDifferentContentPreservesPriorRow(t *testing.T) { + s := newBatchService(t) + s.migrate(t) + writer := s.newWriter(t) + + first := Batch{ID: "prj/stored-first", Project: batchProject, DatasetGeneration: batchGenA, Resources: []source.Resource{ + litResource("Patient", "p-1", "/fhir", strPtr(`{"id":"p-1","v":"stored"}`)), + }} + if _, err := writer.WriteBatch(s.ctx, first); err != nil { + s.fail("seed WriteBatch", err) + } + + mutated := first + mutated.ID = "prj/stored-second" + mutated.Resources = []source.Resource{ + litResource("Patient", "p-1", "/fhir", strPtr(`{"id":"p-1","v":"replaced"}`)), + } + _, err := writer.WriteBatch(s.ctx, mutated) + var rowErr *BatchResourceConflictError + if !errors.As(err, &rowErr) || rowErr.InBatch { + t.Fatalf("stored-row duplicate = %v, want stored-row *BatchResourceConflictError", err) + } + prior := `{"id":"p-1","v":"stored"}` + s.requireResourceRow(t, batchProject, batchGenA, "Patient", "p-1", "/fhir", &prior) + + authMoved := first + authMoved.ID = "prj/stored-third" + authMoved.Resources = []source.Resource{ + litResource("Patient", "p-1", "/other", strPtr(`{"id":"p-1","v":"stored"}`)), + } + _, err = writer.WriteBatch(s.ctx, authMoved) + if !errors.As(err, &rowErr) || rowErr.InBatch { + t.Fatalf("auth-path-moved duplicate = %v, want stored-row *BatchResourceConflictError", err) + } + s.requireResourceRow(t, batchProject, batchGenA, "Patient", "p-1", "/fhir", &prior) + + if _, found := s.witnessLedger(t, "prj/stored-second"); found { + t.Errorf("refused batch prj/stored-second left a marker") + } + if _, found := s.witnessLedger(t, "prj/stored-third"); found { + t.Errorf("refused batch prj/stored-third left a marker") + } + if n := s.countIn(t, resourceTableName, batchProject, batchGenA); n != 1 { + t.Errorf("fhir_resource rows = %d, want exactly 1 (the prior row)", n) + } +} + +func TestBatchWriterIdenticalRowsCollapseWithinAndAcrossBatches(t *testing.T) { + s := newBatchService(t) + s.migrate(t) + writer := s.newWriter(t) + + dup := litResource("Patient", "p-dup", "/fhir", strPtr(`{"id":"p-dup"}`)) + b := Batch{ID: "prj/collapse-1", Project: batchProject, DatasetGeneration: batchGenA, Resources: []source.Resource{dup, dup, + litResource("Patient", "p-2", "", nil), + litResource("Patient", "p-2", "", nil), + }} + res, err := writer.WriteBatch(s.ctx, b) + if err != nil { + s.fail("collapse WriteBatch", err) + } + if res.CommittedResources != 2 || res.SkippedIdenticalResources != 2 || res.Replay { + t.Fatalf("result = %+v, want 2 committed and 2 collapsed", res) + } + if n := s.countIn(t, resourceTableName, batchProject, batchGenA); n != 2 { + t.Fatalf("fhir_resource rows = %d, want 2 distinct rows", n) + } + lw, found := s.witnessLedger(t, "prj/collapse-1") + if !found || lw.resources != 2 { + t.Fatalf("ledger = %+v, want resource_count 2 (committed rows, not offered rows)", lw) + } + + b2 := Batch{ID: "prj/collapse-2", Project: batchProject, DatasetGeneration: batchGenA, Resources: []source.Resource{dup, dup}} + res2, err := writer.WriteBatch(s.ctx, b2) + if err != nil { + s.fail("cross-batch replay WriteBatch", err) + } + if res2.CommittedResources != 0 || res2.SkippedIdenticalResources != 2 || res2.Replay { + t.Fatalf("cross-batch result = %+v, want 0 committed 2 skipped fresh batch", res2) + } + if n := s.countIn(t, resourceTableName, batchProject, batchGenA); n != 2 { + t.Errorf("fhir_resource rows after cross-batch identical replay = %d, want still 2", n) + } +} + +func TestBatchWriterEdgeDuplicateOccurrenceDifferentContentPreservesPriorRow(t *testing.T) { + s := newBatchService(t) + s.migrate(t) + writer := s.newWriter(t) + + ownerRow := litResource("Patient", "p-1", "/fhir", strPtr(`{"id":"p-1"}`)) + edge := litEdge("referenced-by", "Patient", "p-1", "Organization", "o-1", "Patient.link.target", 0, "/fhir", strPtr("k1")) + first := Batch{ID: "prj/edge-first", Project: batchProject, DatasetGeneration: batchGenA, + Resources: []source.Resource{ownerRow}, Edges: []EdgeOccurrenceRow{edge}} + if _, err := writer.WriteBatch(s.ctx, first); err != nil { + s.fail("seed edge WriteBatch", err) + } + + reshaped := first + reshaped.ID = "prj/edge-second" + reshaped.Resources = nil + reshaped.Edges = []EdgeOccurrenceRow{ + litEdge("referenced-by", "Patient", "p-1", "Organization", "o-1", "Patient.link.target", 0, "/fhir", strPtr("k2")), + } + _, err := writer.WriteBatch(s.ctx, reshaped) + var edgeErr *BatchEdgeConflictError + if !errors.As(err, &edgeErr) || !errors.Is(err, ErrBatchRowConflict) || edgeErr.InBatch { + t.Fatalf("stored edge duplicate = %v, want stored-row *BatchEdgeConflictError", err) + } + if edgeErr.SourceLogicalID != "p-1" || edgeErr.Label != "referenced-by" || edgeErr.SourceOrdinal != 0 { + t.Fatalf("conflict fields = %+v, want the refused occurrence identity", edgeErr) + } + s.requireEdgeRow(t, batchProject, batchGenA, "referenced-by", "Patient", "p-1", + "Organization", "o-1", "Patient.link.target", 0, "/fhir", strPtr("k1")) + + dupe := Batch{ID: "prj/edge-dupe", Project: batchProject, DatasetGeneration: batchGenA, + Resources: []source.Resource{ownerRow}, + Edges: []EdgeOccurrenceRow{ + litEdge("referenced-by", "Patient", "p-1", "Organization", "o-2", "Patient.link.target", 0, "/fhir", nil), + litEdge("referenced-by", "Patient", "p-1", "Organization", "o-2", "Patient.link.target", 0, "", nil), + }} + _, err = writer.WriteBatch(s.ctx, dupe) + if !errors.As(err, &edgeErr) || !edgeErr.InBatch || edgeErr.TargetLogicalID != "o-2" { + t.Fatalf("in-batch edge duplicate = %v, want in-batch *BatchEdgeConflictError for o-2", err) + } + if n := s.countIn(t, edgeTableName, batchProject, batchGenA); n != 1 { + t.Errorf("edge rows = %d, want exactly 1 (the prior occurrence)", n) + } +} + +func TestBatchWriterEdgeSourceMissingRollsBackWholeBatch(t *testing.T) { + s := newBatchService(t) + s.migrate(t) + writer := s.newWriter(t) + + ghost := Batch{ID: "prj/edge-ghost", Project: batchProject, DatasetGeneration: batchGenA, + Resources: []source.Resource{litResource("Patient", "p-ok", "", nil)}, + Edges: []EdgeOccurrenceRow{ + litEdge("referenced-by", "Patient", "p-ghost", "Organization", "o-1", "a", 0, "", nil), + }} + _, err := writer.WriteBatch(s.ctx, ghost) + var missing *EdgeSourceMissingError + if !errors.As(err, &missing) || !errors.Is(err, ErrEdgeSourceMissing) { + t.Fatalf("missing source = %v, want *EdgeSourceMissingError", err) + } + if missing.Project != batchProject || missing.DatasetGeneration != batchGenA || + missing.SourceResourceType != "Patient" || missing.SourceLogicalID != "p-ghost" { + t.Fatalf("missing-source fields = %+v, want Patient/p-ghost in batch scope", missing) + } + s.requireNothingWritten(t, batchProject, batchGenA, "prj/edge-ghost") + + ghost.ID = "prj/edge-ok" + ghost.Edges[0].SourceLogicalID = "p-ok" + res, err := writer.WriteBatch(s.ctx, ghost) + if err != nil { + s.fail("control edge WriteBatch", err) + } + if res.CommittedResources != 1 || res.CommittedEdges != 1 { + t.Fatalf("control result = %+v, want 1/1", res) + } +} + +func TestBatchWriterInjectedFailureBeforeCommitLeavesNoRowsOrMarker(t *testing.T) { + s := newBatchService(t) + s.migrate(t) + + injected := errors.New("injected failure") + steps := []string{stepBegin, stepReadLedger, stepClaimLedger, stepStageResources, + stepCheckResources, stepPublishResourc, stepStageEdges, stepCheckEdges, + stepPublishEdges, stepFinalizeLedger, stepCommit} + for _, target := range steps { + writer := s.newWriter(t) + writer.failHook = func(step string) error { + if step == target { + return injected + } + return nil + } + _, err := writer.WriteBatch(s.ctx, goldenBatch()) + var writeErr *BatchWriteError + if !errors.As(err, &writeErr) || !errors.Is(err, ErrBatchWrite) || !errors.Is(err, injected) { + t.Fatalf("failure at %s = %v, want *BatchWriteError chaining the injected error", target, err) + } + s.requireNothingWritten(t, "prj-golden", "gen-golden-1", "pgstore06/golden/0") + } + + writer := s.newWriter(t) + res, err := writer.WriteBatch(s.ctx, goldenBatch()) + if err != nil { + s.fail("control WriteBatch", err) + } + if res.Replay || res.CommittedResources != 4 || res.CommittedEdges != 2 { + t.Fatalf("control result = %+v, want a fresh 4/2 commit on the same identity", res) + } + if n := s.countIn(t, resourceTableName, "prj-golden", "gen-golden-1"); n != 4 { + t.Errorf("fhir_resource rows after control = %d, want 4", n) + } + if n := s.countIn(t, edgeTableName, "prj-golden", "gen-golden-1"); n != 2 { + t.Errorf("fhir_edge_occurrence rows after control = %d, want 2", n) + } + if _, found := s.witnessLedger(t, "pgstore06/golden/0"); !found { + t.Error("ledger marker missing after control commit") + } +} + +func TestBatchWriterResultCountsEqualCommittedRows(t *testing.T) { + s := newBatchService(t) + s.migrate(t) + writer := s.newWriter(t) + + s.exec(t, "INSERT INTO "+s.qualified(resourceTableName)+ + " (project, dataset_generation, resource_type, logical_id, auth_resource_path, payload)"+ + " VALUES ($1, $2, 'Patient', 'p-pre', '/fhir', $3::jsonb)", + batchProject, batchGenA, `{"id":"p-pre"}`) + + b := Batch{ID: "prj/counts", Project: batchProject, DatasetGeneration: batchGenA, + Resources: []source.Resource{ + litResource("Patient", "p-pre", "/fhir", strPtr(`{"id":"p-pre"}`)), + litResource("Patient", "p-new", "", nil), + litResource("Patient", "p-new", "", nil), + }, + Edges: []EdgeOccurrenceRow{ + litEdge("referenced-by", "Patient", "p-new", "Observation", "o-1", "a", 0, "", nil), + litEdge("referenced-by", "Patient", "p-new", "Observation", "o-1", "a", 0, "", nil), + }} + res, err := writer.WriteBatch(s.ctx, b) + if err != nil { + s.fail("counted WriteBatch", err) + } + if res.CommittedResources != 1 || res.CommittedEdges != 1 { + t.Fatalf("result = %+v, want 1 committed resource and 1 committed edge", res) + } + if res.SkippedIdenticalResources != 2 || res.SkippedIdenticalEdges != 1 { + t.Fatalf("result = %+v, want 2 skipped resources and 1 skipped edge", res) + } + lw, found := s.witnessLedger(t, "prj/counts") + if !found || lw.resources != res.CommittedResources || lw.edges != res.CommittedEdges { + t.Fatalf("ledger = %+v, want the result's committed counts %d/%d", + lw, res.CommittedResources, res.CommittedEdges) + } + if n := s.countIn(t, resourceTableName, batchProject, batchGenA); n != 2 { + t.Errorf("fhir_resource rows = %d, want 2 (pre-seed + one committed)", n) + } + if n := s.countIn(t, edgeTableName, batchProject, batchGenA); n != 1 { + t.Errorf("fhir_edge_occurrence rows = %d, want 1", n) + } +} + +func TestBatchWriterBoundedCapacity(t *testing.T) { + s := newBatchService(t) + s.migrate(t) + writer := s.newWriter(t) + + full := Batch{ID: "prj/bounded", Project: batchProject, DatasetGeneration: batchGenA} + for i := 0; i < MaxBatchResources; i++ { + full.Resources = append(full.Resources, + litResource("Patient", fmt.Sprintf("b-%04d", i), "", nil)) + } + for i := 0; i < MaxBatchEdges; i++ { + full.Edges = append(full.Edges, litEdge("referenced-by", "Patient", + fmt.Sprintf("b-%04d", i), "Observation", "o-x", "a", int64(i), "", nil)) + } + res, err := writer.WriteBatch(s.ctx, full) + if err != nil { + s.fail("full-capacity WriteBatch", err) + } + if res.CommittedResources != MaxBatchResources || res.CommittedEdges != MaxBatchEdges { + t.Fatalf("full result = %+v, want %d/%d committed", res, MaxBatchResources, MaxBatchEdges) + } + if n := s.countIn(t, resourceTableName, batchProject, batchGenA); n != MaxBatchResources { + t.Errorf("fhir_resource rows = %d, want %d", n, MaxBatchResources) + } + if n := s.countIn(t, edgeTableName, batchProject, batchGenA); n != MaxBatchEdges { + t.Errorf("fhir_edge_occurrence rows = %d, want %d", n, MaxBatchEdges) + } +} + +func TestBatchWriterConcurrentIndependentBatchesOverlapTwoConnections(t *testing.T) { + s := newBatchService(t) + s.migrate(t) + + writer := s.newWriterOn(t, s.openClient(t, 2)) + + arrived := map[string]chan struct{}{ + "prj/conc-a": make(chan struct{}), + "prj/conc-b": make(chan struct{}), + } + var pids sync.Map + var stalled sync.Map + writer.inTxProbe = func(step, batchID string, tx pgx.Tx) { + if step != stepPublishEdges { + return + } + var pid int64 + if err := tx.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + stalled.Store(batchID, true) + return + } + pids.Store(batchID, pid) + close(arrived[batchID]) + other := "prj/conc-b" + if batchID == other { + other = "prj/conc-a" + } + select { + case <-arrived[other]: + case <-time.After(20 * time.Second): + stalled.Store(batchID, true) + } + } + + type outcome struct { + res BatchResult + err error + } + outcomes := make(chan outcome, 2) + go func() { + res, err := writer.WriteBatch(s.ctx, concurrentBatch("prj/conc-a", batchGenA, "ga")) + outcomes <- outcome{res, err} + }() + go func() { + res, err := writer.WriteBatch(s.ctx, concurrentBatch("prj/conc-b", batchGenB, "gb")) + outcomes <- outcome{res, err} + }() + var results [2]BatchResult + for i := 0; i < 2; i++ { + select { + case o := <-outcomes: + if o.err != nil { + s.fail("concurrent WriteBatch", o.err) + } + results[i] = o.res + case <-time.After(60 * time.Second): + t.Fatal("concurrent batches did not finish") + } + } + + for _, id := range []string{"prj/conc-a", "prj/conc-b"} { + if v, loaded := stalled.Load(id); loaded && v.(bool) { + t.Fatalf("batch %s never overlapped the other at the publish step: "+ + "the writer serialized independent batches", id) + } + } + vA, _ := pids.Load("prj/conc-a") + vB, _ := pids.Load("prj/conc-b") + if vA == nil || vB == nil || vA.(int64) == vB.(int64) { + t.Fatalf("concurrent batches ran on backends %v/%v, want two distinct real PG connections", vA, vB) + } + + for _, g := range []struct { + id, gen, prefix string + }{{"prj/conc-a", batchGenA, "ga"}, {"prj/conc-b", batchGenB, "gb"}} { + if n := s.countIn(t, resourceTableName, batchProject, g.gen); n != 2 { + t.Errorf("fhir_resource rows for %s = %d, want 2 (no cross-batch sharing)", g.gen, n) + } + if n := s.countIn(t, edgeTableName, batchProject, g.gen); n != 1 { + t.Errorf("fhir_edge_occurrence rows for %s = %d, want 1", g.gen, n) + } + s.requireResourceRow(t, batchProject, g.gen, "Patient", g.prefix+"-1", "/fhir", + strPtr(`{"id":"`+g.prefix+`-1"}`)) + s.requireResourceRow(t, batchProject, g.gen, "Patient", g.prefix+"-2", "", nil) + lw, found := s.witnessLedger(t, g.id) + if !found || lw.generation != g.gen || lw.resources != 2 || lw.edges != 1 { + t.Errorf("ledger for %s = %+v, want 2/1 in generation %s", g.id, lw, g.gen) + } + } +} + +func concurrentBatch(id, generation, prefix string) Batch { + return Batch{ID: id, Project: batchProject, DatasetGeneration: generation, + Resources: []source.Resource{ + {Project: batchProject, DatasetGeneration: generation, ResourceType: "Patient", + LogicalID: prefix + "-1", AuthResourcePath: "/fhir", + Payload: json.RawMessage(`{"id":"` + prefix + `-1"}`)}, + {Project: batchProject, DatasetGeneration: generation, ResourceType: "Patient", + LogicalID: prefix + "-2", AuthResourcePath: "", Payload: nil}, + }, + Edges: []EdgeOccurrenceRow{ + {Project: batchProject, DatasetGeneration: generation, AuthResourcePath: "", + Label: "referenced-by", SourceResourceType: "Patient", SourceLogicalID: prefix + "-1", + TargetResourceType: "Observation", TargetLogicalID: "o-x", + SourcePath: "a", SourceOrdinal: 0, LogicalEdgeKey: nil}, + }} +} + +func (s *batchService) waitBackendWaiting(t *testing.T, app string) int64 { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for { + var pid int64 + var waitType, waitEvent *string + err := s.iso.QueryRow(s.ctx, + "SELECT pid, wait_event_type, wait_event FROM pg_stat_activity"+ + " WHERE application_name = $1 AND state = 'active'", app). + Scan(&pid, &waitType, &waitEvent) + if err == nil && waitType != nil && *waitType == "Lock" { + return pid + } + if time.Now().After(deadline) { + s.fail("waiting for backend "+app+" to block on a row lock", + fmt.Errorf("backend never entered a lock wait")) + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestBatchWriterConcurrentClaimOfSameBatchCommitsOnceAndReplaysTwoConnections(t *testing.T) { + s := newBatchService(t) + s.migrate(t) + + claimHeld := make(chan struct{}) + release := make(chan struct{}) + var t1PID int64 + var t1ProbeErr error + var t1PIDMu sync.Mutex + writer1 := s.newWriterOn(t, s.openAppClient(t, "pgstore06-t1", 1)) + writer1.inTxProbe = func(step, batchID string, tx pgx.Tx) { + if step != stepClaimLedger { + return + } + t1PIDMu.Lock() + t1ProbeErr = tx.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&t1PID) + t1PIDMu.Unlock() + close(claimHeld) + <-release + } + writer2 := s.newWriterOn(t, s.openAppClient(t, "pgstore06-t2", 1)) + + type outcome struct { + res BatchResult + err error + } + r1 := make(chan outcome, 1) + go func() { + res, err := writer1.WriteBatch(s.ctx, goldenBatch()) + r1 <- outcome{res, err} + }() + + select { + case <-claimHeld: + case o := <-r1: + s.fail("T1 never reached the claim probe", o.err) + } + r2 := make(chan outcome, 1) + go func() { + res, err := writer2.WriteBatch(s.ctx, goldenBatch()) + r2 <- outcome{res, err} + }() + + t2PID := s.waitBackendWaiting(t, "pgstore06-t2") + close(release) + o1 := <-r1 + o2 := <-r2 + t1PIDMu.Lock() + heldT1, probeErr := t1PID, t1ProbeErr + t1PIDMu.Unlock() + if probeErr != nil { + s.fail("witness t1 backend", probeErr) + } + if heldT1 == 0 || heldT1 == t2PID { + t.Fatalf("backends t1=%d t2=%d, want two distinct real PG connections", heldT1, t2PID) + } + if o1.err != nil { + s.fail("T1 WriteBatch", o1.err) + } + if o2.err != nil { + s.fail("T2 WriteBatch", o2.err) + } + if o1.res.Replay || o1.res.CommittedResources != 4 || o1.res.CommittedEdges != 2 { + t.Fatalf("T1 result = %+v, want the fresh 4/2 commit", o1.res) + } + if !o2.res.Replay || o2.res.CommittedResources != 4 || o2.res.CommittedEdges != 2 { + t.Fatalf("T2 result = %+v, want a replay of the original 4/2 accounting", o2.res) + } + if n := s.mustInt64(t, "SELECT count(*) FROM "+s.qualified(batchLedgerTableName)); n != 1 { + t.Errorf("ledger rows = %d, want exactly 1", n) + } + if n := s.countIn(t, resourceTableName, "prj-golden", "gen-golden-1"); n != 4 { + t.Errorf("fhir_resource rows = %d, want exactly 4 (committed once)", n) + } + if n := s.countIn(t, edgeTableName, "prj-golden", "gen-golden-1"); n != 2 { + t.Errorf("fhir_edge_occurrence rows = %d, want exactly 2 (committed once)", n) + } +} + +func TestBatchWriterAbandonedClaimDoesNotBurnIdentityTwoConnections(t *testing.T) { + s := newBatchService(t) + s.migrate(t) + + claimHeld := make(chan struct{}) + release := make(chan struct{}) + injected := errors.New("injected abort") + writer1 := s.newWriterOn(t, s.openAppClient(t, "pgstore06-t1", 1)) + writer1.inTxProbe = func(step, batchID string, tx pgx.Tx) { + if step != stepClaimLedger { + return + } + close(claimHeld) + <-release + } + writer1.failHook = func(step string) error { + if step == stepPublishResourc { + return injected + } + return nil + } + writer2 := s.newWriterOn(t, s.openAppClient(t, "pgstore06-t2", 1)) + + type outcome struct { + res BatchResult + err error + } + r1 := make(chan outcome, 1) + go func() { + res, err := writer1.WriteBatch(s.ctx, goldenBatch()) + r1 <- outcome{res, err} + }() + select { + case <-claimHeld: + case o := <-r1: + s.fail("T1 never reached the claim probe", o.err) + } + r2 := make(chan outcome, 1) + go func() { + res, err := writer2.WriteBatch(s.ctx, goldenBatch()) + r2 <- outcome{res, err} + }() + s.waitBackendWaiting(t, "pgstore06-t2") + close(release) + + o1 := <-r1 + if !errors.Is(o1.err, injected) { + t.Fatalf("T1 = %v, want the injected abort", o1.err) + } + o2 := <-r2 + if o2.err != nil { + s.fail("T2 WriteBatch after takeover", o2.err) + } + if o2.res.Replay || o2.res.CommittedResources != 4 || o2.res.CommittedEdges != 2 { + t.Fatalf("T2 result = %+v, want a fresh 4/2 commit after T1's abandoned claim", o2.res) + } + if n := s.countIn(t, resourceTableName, "prj-golden", "gen-golden-1"); n != 4 { + t.Errorf("fhir_resource rows = %d, want exactly 4", n) + } + lw, found := s.witnessLedger(t, "pgstore06/golden/0") + if !found || lw.digest != goldenDigest { + t.Fatalf("ledger = %+v found=%v, want the golden marker", lw, found) + } +} + +func TestBatchWriterMigrationChecksumSelfVerifies(t *testing.T) { + const fileName = "migrations/000008_batch_write_ledger.sql" + onDisk, err := os.ReadFile(fileName) + if err != nil { + t.Fatalf("read %s: %v", fileName, err) + } + sum := sha256.Sum256(onDisk) + diskChecksum := hex.EncodeToString(sum[:]) + + want, pinned := wantEmbeddedChecksums[8] + if !pinned { + t.Fatal("migration 8 has no pinned checksum in migrate_test.go") + } + if want.name != "batch_write_ledger" || want.checksum != diskChecksum { + t.Fatalf("pin = {Name:%q Checksum:%q}, want {Name:%q Checksum:%q} — the pin must be the "+ + "independently computed sha256 of the file's exact bytes", + want.name, want.checksum, "batch_write_ledger", diskChecksum) + } + for _, m := range mustEmbeddedMigrations(t) { + if m.Version == 8 { + if m.Name != want.name || m.Checksum != diskChecksum { + t.Fatalf("embedded migration 8 = {Name:%q Checksum:%q}, want the disk sha256 %s", + m.Name, m.Checksum, diskChecksum) + } + return + } + } + t.Fatal("migration 8 is absent from the embedded history") +} + +func TestBatchWriterMigrationDoubleApplyAndCrashWindowConverge(t *testing.T) { + s := newBatchService(t) + client := s.openClient(t, 4) + + first, err := MigrateSchema(s.ctx, client, s.schema) + if err != nil { + s.fail("first MigrateSchema", err) + } + if first.LatestVersion != 9 { + t.Fatalf("latest version = %d, want 9", first.LatestVersion) + } + if len(first.Applied) < 8 { + t.Fatalf("applied %d migrations, want at least 8", len(first.Applied)) + } + last := first.Applied[7] + if last.Version != 8 || last.Name != "batch_write_ledger" || + last.Checksum != wantEmbeddedChecksums[8].checksum { + t.Fatalf("applied row 8 = %+v, want the pinned batch_write_ledger row", last) + } + + second, err := MigrateSchema(s.ctx, client, s.schema) + if err != nil { + s.fail("second MigrateSchema", err) + } + if !reflect.DeepEqual(first, second) { + t.Fatalf("double apply changed the result: first %+v second %+v", first, second) + } + if n := s.mustInt64(t, "SELECT count(*) FROM "+s.qualified(LedgerTable)+" WHERE version = 8"); n != 1 { + t.Fatalf("ledger rows for version 8 = %d, want exactly 1 after double apply", n) + } + + s.exec(t, "DELETE FROM "+s.qualified(LedgerTable)+" WHERE version >= 8") + third, err := MigrateSchema(s.ctx, client, s.schema) + if err != nil { + s.fail("third MigrateSchema after lost ledger row", err) + } + type pin struct { + version int64 + name string + checksum string + } + pins := func(r MigrateResult) []pin { + out := make([]pin, 0, len(r.Applied)) + for _, a := range r.Applied { + out = append(out, pin{a.Version, a.Name, a.Checksum}) + } + return out + } + if !reflect.DeepEqual(pins(second), pins(third)) { + t.Fatalf("crash-window retry changed the recorded history: second %v third %v", + pins(second), pins(third)) + } + if n := s.mustInt64(t, + "SELECT count(*) FROM "+s.qualified(LedgerTable)+" WHERE version = 8 AND checksum = $1", + wantEmbeddedChecksums[8].checksum); n != 1 { + t.Fatalf("ledger rows for version 8 after convergence = %d, want exactly 1", n) + } + if n := s.mustInt64(t, + "SELECT count(*) FROM "+s.qualified(LedgerTable)+" WHERE version = 9 AND checksum = $1", + wantEmbeddedChecksums[9].checksum); n != 1 { + t.Fatalf("ledger rows for version 9 after convergence = %d, want exactly 1", n) + } + + writer := s.newWriterOn(t, client) + res, err := writer.WriteBatch(s.ctx, goldenBatch()) + if err != nil { + s.fail("WriteBatch after convergence", err) + } + if res.Replay || res.CommittedResources != 4 || res.CommittedEdges != 2 { + t.Fatalf("result after convergence = %+v, want a fresh 4/2 commit", res) + } +} + +func TestZZZeroLeftoverBatchWriterFixtures(t *testing.T) { + raw := testPostgresURL(t) + batchOwnedMu.Lock() + owned := make(map[string]string, len(batchOwnedByName)) + for db, schema := range batchOwnedByName { + owned[db] = schema + } + batchOwnedMu.Unlock() + if len(owned) == 0 { + t.Skip("this process created no pgstore06_it_* fixtures") + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service for audit: %v", err) + } + defer admin.Close() + + for db, schema := range owned { + if !strings.HasPrefix(db, "pgstore06_it_") { + t.Fatalf("registry holds foreign database %q", db) + } + var count int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Errorf("audit owned database %s: %v", db, err) + continue + } + if count != 0 { + t.Errorf("leftover owned database %s after all cleanups", db) + if schema != "" { + u, err := url.Parse(raw) + if err == nil { + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + db + if probe, err := pgxpool.New(ctx, u.String()); err == nil { + var schemaCount int + if err := probe.QueryRow(ctx, "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1", schema).Scan(&schemaCount); err != nil { + t.Errorf("audit owned schema %s in %s: %v", schema, db, err) + } else if schemaCount != 0 { + t.Errorf("leftover owned schema %s in leftover database %s", schema, db) + } + probe.Close() + } + } + } + } + } +} diff --git a/internal/backends/postgres/internal/store/client.go b/internal/backends/postgres/internal/store/client.go new file mode 100644 index 000000000..6040d8b7c --- /dev/null +++ b/internal/backends/postgres/internal/store/client.go @@ -0,0 +1,215 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "strings" + "sync" + "syscall" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const RequiredMajorVersion = 18 + +var ( + ErrInvalidDSN = errors.New("postgres: invalid dsn") + ErrInvalidPoolConfig = errors.New("postgres: invalid pool config") + ErrConnect = errors.New("postgres: connect failed") + ErrUnsupportedVersion = errors.New("postgres: unsupported server version") +) + +type Config struct { + DSN string + MaxConns int32 + MinConns int32 + AcquireTimeout time.Duration +} + +func (c Config) Validate() error { + if strings.TrimSpace(c.DSN) == "" { + return fmt.Errorf("%w: dsn is blank", ErrInvalidDSN) + } + if c.MaxConns < 1 { + return fmt.Errorf("%w: MaxConns is %d, want >= 1", ErrInvalidPoolConfig, c.MaxConns) + } + if c.MinConns < 0 { + return fmt.Errorf("%w: MinConns is %d, want >= 0", ErrInvalidPoolConfig, c.MinConns) + } + if c.MinConns > c.MaxConns { + return fmt.Errorf("%w: MinConns is %d, want <= MaxConns (%d)", + ErrInvalidPoolConfig, c.MinConns, c.MaxConns) + } + if c.AcquireTimeout <= 0 { + return fmt.Errorf("%w: AcquireTimeout is %v, want > 0", ErrInvalidPoolConfig, c.AcquireTimeout) + } + return nil +} + +type ConnectError struct { + Op string + Err error +} + +func (e *ConnectError) Error() string { + return fmt.Sprintf("postgres: %s: %v", e.Op, e.Err) +} + +func (e *ConnectError) Unwrap() []error { return []error{ErrConnect, e.Err} } + +type UnsupportedVersionError struct { + VersionNum int64 + Version string + Major int +} + +func (e *UnsupportedVersionError) Error() string { + return fmt.Sprintf("postgres: server version %s (%d) is not supported; this store requires PostgreSQL %d", + e.Version, e.VersionNum, RequiredMajorVersion) +} + +func (e *UnsupportedVersionError) Unwrap() error { return ErrUnsupportedVersion } + +func CheckServerVersion(versionNum int64, version string) error { + major := int(versionNum / 10000) + if major != RequiredMajorVersion { + return &UnsupportedVersionError{VersionNum: versionNum, Version: version, Major: major} + } + return nil +} + +type Client struct { + pool *pgxpool.Pool + cfg Config +} + +func Open(ctx context.Context, cfg Config) (*Client, error) { + if err := cfg.Validate(); err != nil { + return nil, err + } + pcfg, err := pgxpool.ParseConfig(cfg.DSN) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidDSN, err) + } + pcfg.MaxConns = cfg.MaxConns + pcfg.MinConns = cfg.MinConns + pool, err := pgxpool.NewWithConfig(ctx, pcfg) + if err != nil { + return nil, &ConnectError{Op: "open", Err: err} + } + c := &Client{pool: pool, cfg: cfg} + if err := c.Ping(ctx); err != nil { + pool.Close() + return nil, err + } + return c, nil +} + +func (c *Client) Ping(ctx context.Context) error { + conn, err := c.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + + var ( + versionNum int64 + version string + ) + err = conn.QueryRow(ctx, + "SELECT current_setting('server_version_num')::int8, current_setting('server_version')", + ).Scan(&versionNum, &version) + if err != nil { + return &ConnectError{Op: "ping", Err: err} + } + return CheckServerVersion(versionNum, version) +} + +func (c *Client) Acquire(ctx context.Context) (*pgxpool.Conn, error) { + ctx, cancel := context.WithTimeout(ctx, c.cfg.AcquireTimeout) + defer cancel() + conn, err := c.pool.Acquire(ctx) + if err != nil { + return nil, &ConnectError{Op: "acquire", Err: err} + } + return conn, nil +} + +func (c *Client) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { + conn, err := c.Acquire(ctx) + if err != nil { + return nil, err + } + rows, err := conn.Query(ctx, sql, args...) + if err != nil { + conn.Release() + return nil, &ConnectError{Op: "query", Err: err} + } + return &leasedRows{Rows: rows, conn: conn}, nil +} + +func (c *Client) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row { + conn, err := c.Acquire(ctx) + if err != nil { + return &queryRowError{err: &ConnectError{Op: "query_row", Err: err}} + } + return &leasedRow{Row: conn.QueryRow(ctx, sql, args...), conn: conn} +} + +type leasedRows struct { + pgx.Rows + conn *pgxpool.Conn + closed sync.Once +} + +func (l *leasedRows) Close() { + l.closed.Do(func() { + l.Rows.Close() + l.conn.Release() + }) +} + +type leasedRow struct { + pgx.Row + conn *pgxpool.Conn + scanned sync.Once +} + +func (r *leasedRow) Scan(dest ...any) error { + err := r.Row.Scan(dest...) + r.scanned.Do(func() { r.conn.Release() }) + return err +} + +type queryRowError struct{ err error } + +func (r *queryRowError) Scan(_ ...any) error { return r.err } + +func (c *Client) Close() { + if c.pool != nil { + c.pool.Close() + } +} + +func IsRefusedConnection(err error) bool { + if err == nil { + return false + } + if errors.Is(err, syscall.ECONNREFUSED) { + return true + } + var opErr *net.OpError + if errors.As(err, &opErr) && opErr.Err != nil { + var sysErr *os.SyscallError + if errors.As(opErr.Err, &sysErr) { + return sysErr.Err == syscall.ECONNREFUSED + } + return strings.Contains(strings.ToLower(opErr.Err.Error()), "connection refused") + } + return false +} diff --git a/internal/backends/postgres/internal/store/client_test.go b/internal/backends/postgres/internal/store/client_test.go new file mode 100644 index 000000000..fded553ce --- /dev/null +++ b/internal/backends/postgres/internal/store/client_test.go @@ -0,0 +1,367 @@ +package postgres + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "net" + "net/url" + "os" + "regexp" + "strings" + "syscall" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func validUnitConfig(dsn string) Config { + return Config{ + DSN: dsn, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 2 * time.Second, + } +} + +func assertRejectedWithoutConnection(t *testing.T, err error, want error) { + t.Helper() + if !errors.Is(err, want) { + t.Fatalf("error = %v (%T), want one matching %v", err, err, want) + } + var ce *ConnectError + if errors.As(err, &ce) { + t.Fatalf("config rejection must not reach the connection layer, got %T: %v", err, err) + } +} + +func TestConfigValidateAcceptsBoundedConfig(t *testing.T) { + if err := validUnitConfig("postgres://loom@127.0.0.1:5432/loom").Validate(); err != nil { + t.Fatalf("valid config rejected: %v", err) + } +} + +func TestConfigValidateRejectsInvalidPoolBounds(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*Config) + }{ + {"zero max conns", func(c *Config) { c.MaxConns = 0 }}, + {"negative max conns", func(c *Config) { c.MaxConns = -3 }}, + {"negative min conns", func(c *Config) { c.MinConns = -1 }}, + {"min conns above max", func(c *Config) { c.MinConns = c.MaxConns + 1 }}, + {"zero acquire timeout", func(c *Config) { c.AcquireTimeout = 0 }}, + {"negative acquire timeout", func(c *Config) { c.AcquireTimeout = -time.Second }}, + } { + cfg := validUnitConfig("postgres://loom@127.0.0.1:5432/loom") + tc.mutate(&cfg) + if err := cfg.Validate(); !errors.Is(err, ErrInvalidPoolConfig) { + t.Errorf("%s: Validate() = %v, want ErrInvalidPoolConfig", tc.name, err) + } + } +} + +func TestOpenRejectsBlankDSNWithoutNetwork(t *testing.T) { + for _, dsn := range []string{"", " ", "\t\n "} { + cfg := validUnitConfig(dsn) + _, err := Open(context.Background(), cfg) + assertRejectedWithoutConnection(t, err, ErrInvalidDSN) + } +} + +func TestOpenRejectsMalformedDSNWithoutNetwork(t *testing.T) { + for _, dsn := range []string{ + "not a dsn at all", + "postgres://loom@127.0.0.1:notaport/loom?sslmode=disable", + "postgresql://loom@127.0.0.1:5432/loom?sslmode=not-a-valid-mode", + } { + cfg := validUnitConfig(dsn) + _, err := Open(context.Background(), cfg) + assertRejectedWithoutConnection(t, err, ErrInvalidDSN) + } +} + +func TestOpenRejectsInvalidPoolBoundsWithoutNetwork(t *testing.T) { + cfg := validUnitConfig("postgres://loom@127.0.0.1:1/loom") + cfg.MaxConns = 0 + _, err := Open(context.Background(), cfg) + assertRejectedWithoutConnection(t, err, ErrInvalidPoolConfig) +} + +func TestCheckServerVersion(t *testing.T) { + for _, num := range []int64{180000, 180006, 189999} { + if err := CheckServerVersion(num, "18"); err != nil { + t.Errorf("CheckServerVersion(%d) = %v, want nil", num, err) + } + } + for _, tc := range []struct { + num int64 + version string + major int + }{ + {170014, "17.4", 17}, + {179999, "17.99", 17}, + {190000, "19.0", 19}, + {90624, "9.6.24", 9}, + {0, "", 0}, + } { + err := CheckServerVersion(tc.num, tc.version) + if !errors.Is(err, ErrUnsupportedVersion) { + t.Errorf("CheckServerVersion(%d) = %v, want ErrUnsupportedVersion", tc.num, err) + continue + } + var uve *UnsupportedVersionError + if !errors.As(err, &uve) { + t.Errorf("CheckServerVersion(%d) error type = %T, want *UnsupportedVersionError", tc.num, err) + continue + } + if uve.Major != tc.major || uve.VersionNum != tc.num { + t.Errorf("CheckServerVersion(%d) = {Major:%d VersionNum:%d}, want {Major:%d VersionNum:%d}", + tc.num, uve.Major, uve.VersionNum, tc.major, tc.num) + } + } +} + +func TestRefusedConnectionReturnsClassifiableError(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen on loopback: %v", err) + } + addr := ln.Addr().String() + if err := ln.Close(); err != nil { + t.Fatalf("close listener: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + cfg := Config{ + DSN: fmt.Sprintf("postgres://poolprobe@%s/probe?sslmode=disable", addr), + MaxConns: 2, + MinConns: 0, + AcquireTimeout: 2 * time.Second, + } + _, err = Open(ctx, cfg) + if err == nil { + t.Fatal("Open against a refused port succeeded, want failure") + } + var ce *ConnectError + if !errors.As(err, &ce) { + t.Fatalf("Open error = %v (%T), want *ConnectError", err, err) + } + if !errors.Is(err, ErrConnect) { + t.Fatalf("Open error does not match ErrConnect: %v", err) + } + if !IsRefusedConnection(err) { + t.Fatalf("Open error not classified as a refused connection: %v", err) + } + if !errors.Is(err, syscall.ECONNREFUSED) { + t.Errorf("refused-connection error chain does not reach syscall.ECONNREFUSED: %v", err) + } +} + +var isolatedDBNameRe = regexp.MustCompile(`^pgstore01_it_[0-9]+_[0-9a-f]+$`) + +func testPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 integration tests") + } + return raw +} + +func randomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +func TestPostgres18PoolIntegration(t *testing.T) { + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + if err := admin.Ping(ctx); err != nil { + fail("ping test service", err) + } + + dbName := fmt.Sprintf("pgstore01_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !isolatedDBNameRe.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(ctx, `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + testDSN := u.String() + secrets = append(secrets, testDSN) + if probe, err := url.Parse(testDSN); err != nil || + probe.Query().Get("dbname") != "" || probe.Path != "/"+dbName { + t.Fatal("isolated test DSN does not provably target the isolated database") + } + + client, err := Open(ctx, Config{ + DSN: testDSN, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 5 * time.Second, + }) + if err != nil { + fail("open pool", err) + } + defer client.Close() + + if err := client.Ping(ctx); err != nil { + fail("ping pool", err) + } + + conn, err := client.Acquire(ctx) + if err != nil { + fail("acquire connection", err) + } + var one int + if err := conn.QueryRow(ctx, "SELECT 1").Scan(&one); err != nil { + conn.Release() + fail("query on acquired connection", err) + } + if one != 1 { + conn.Release() + t.Fatalf("SELECT 1 returned %d, want 1", one) + } + conn.Release() + + versionProbe, err := client.Acquire(ctx) + if err != nil { + fail("acquire version probe connection", err) + } + var versionNum int64 + if err := versionProbe.QueryRow(ctx, + "SELECT current_setting('server_version_num')::int8").Scan(&versionNum); err != nil { + versionProbe.Release() + fail("read server_version_num", err) + } + versionProbe.Release() + if versionNum < 180000 || versionNum > 189999 { + t.Fatalf("service reports server_version_num %d, want a PostgreSQL 18 build", versionNum) + } + + marker := "pgstore01_marker_" + randomHex(t) + markerConn, err := client.Acquire(ctx) + if err != nil { + fail("acquire marker connection", err) + } + _, err = markerConn.Exec(ctx, "CREATE TABLE public."+marker+" (id int)") + markerConn.Release() + if err != nil { + fail("create marker table through pool", err) + } + witnessCfg, err := pgxpool.ParseConfig(raw) + if err != nil { + fail("parse witness config", err) + } + witnessCfg.ConnConfig.Database = dbName + witness, err := pgxpool.NewWithConfig(ctx, witnessCfg) + if err != nil { + fail("connect isolated-database witness", err) + } + defer witness.Close() + var markerVisible bool + if err := witness.QueryRow(ctx, + "SELECT to_regclass('public."+marker+"') IS NOT NULL").Scan(&markerVisible); err != nil { + fail("witness query on isolated database", err) + } + if !markerVisible { + t.Fatalf("marker table is absent from isolated database %s: the pool may not target the isolated database", dbName) + } + + canceledCtx, canceledCancel := context.WithCancel(ctx) + canceledCancel() + _, err = client.Acquire(canceledCtx) + if err == nil { + t.Fatal("Acquire with a cancelled context succeeded, want failure") + } + if !errors.Is(err, ErrConnect) || !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled acquire error = %v, want ConnectError matching context.Canceled", err) + } + + bounded, err := Open(ctx, Config{ + DSN: testDSN, + MaxConns: 1, + MinConns: 0, + AcquireTimeout: 5 * time.Second, + }) + if err != nil { + fail("open bounded pool", err) + } + defer bounded.Close() + held, err := bounded.Acquire(ctx) + if err != nil { + fail("acquire sole bounded connection", err) + } + bounded.cfg.AcquireTimeout = 75 * time.Millisecond + started := time.Now() + _, err = bounded.Acquire(ctx) + waited := time.Since(started) + held.Release() + if err == nil { + t.Fatal("second Acquire on an exhausted pool succeeded, want timeout") + } + if !errors.Is(err, ErrConnect) || !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("exhausted acquire error = %v, want ConnectError matching context.DeadlineExceeded", err) + } + if waited > 5*time.Second { + t.Fatalf("acquire timeout took %v, far beyond the 75ms bound", waited) + } + + client.Close() + bounded.Close() +} diff --git a/internal/backends/postgres/internal/store/commit_outcome.go b/internal/backends/postgres/internal/store/commit_outcome.go new file mode 100644 index 000000000..83e4280a2 --- /dev/null +++ b/internal/backends/postgres/internal/store/commit_outcome.go @@ -0,0 +1,28 @@ +package postgres + +import ( + "context" + "errors" + "net" + "strings" +) + +func CommitOutcomeUnknown(err error) bool { + if err == nil { + return false + } + if pgErr, ok := pgErrorFrom(err); ok { + return strings.HasPrefix(pgErr.Code, "08") || pgErr.Code == sqlStateAdminShutdown || pgErr.Code == sqlStateCrashShutdown + } + if errors.Is(err, ErrCommitOutcomeUnknown) { + return true + } + if errors.Is(err, ErrConnect) || errors.Is(err, ErrBatchWrite) { + return true + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return true + } + var netErr net.Error + return errors.As(err, &netErr) +} diff --git a/internal/backends/postgres/internal/store/edge.go b/internal/backends/postgres/internal/store/edge.go new file mode 100644 index 000000000..2f32d430e --- /dev/null +++ b/internal/backends/postgres/internal/store/edge.go @@ -0,0 +1,341 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + edgeTableName = "fhir_edge_occurrence" + MaxEdgeTraversalLimit = 1000 +) + +const edgeColumns = "edge_seq, project, dataset_generation, auth_resource_path, label, source_resource_type, source_logical_id, target_resource_type, target_logical_id, source_path, source_ordinal, logical_edge_key" + +var ( + ErrEdgeConflict = errors.New("postgres: edge conflict") + ErrEdgeSourceMissing = errors.New("postgres: edge source missing") + ErrInvalidEdge = errors.New("postgres: invalid edge") + ErrInvalidEdgeScope = errors.New("postgres: invalid edge scope") +) + +type EdgeConflictError struct { + Project string + DatasetGeneration string + AuthResourcePath string + Label string + SourceResourceType string + SourceLogicalID string + TargetResourceType string + TargetLogicalID string + SourcePath string + SourceOrdinal int64 +} + +func (e *EdgeConflictError) Error() string { + return fmt.Sprintf("postgres: edge occurrence %s/%s/%s(%s) -%s-> %s/%s at path %q ordinal %d (auth path %q) already exists; "+ + "insert never replaces the stored occurrence", + e.Project, e.DatasetGeneration, e.SourceResourceType, e.SourceLogicalID, e.Label, + e.TargetResourceType, e.TargetLogicalID, e.SourcePath, e.SourceOrdinal, e.AuthResourcePath) +} + +func (e *EdgeConflictError) Unwrap() error { return ErrEdgeConflict } + +type EdgeSourceMissingError struct { + Project string + DatasetGeneration string + SourceResourceType string + SourceLogicalID string +} + +func (e *EdgeSourceMissingError) Error() string { + return fmt.Sprintf("postgres: edge source %s/%s/%s/%s does not exist in this project/dataset_generation scope; "+ + "the source must be stored in the same scope before any of its edges", + e.Project, e.DatasetGeneration, e.SourceResourceType, e.SourceLogicalID) +} + +func (e *EdgeSourceMissingError) Unwrap() error { return ErrEdgeSourceMissing } + +type EdgeScope struct { + Project string + DatasetGeneration string + AuthResourcePathsUnrestricted *bool + AuthResourcePaths []string +} + +type EdgeOccurrenceRow struct { + EdgeSeq int64 + Project string + DatasetGeneration string + AuthResourcePath string + Label string + SourceResourceType string + SourceLogicalID string + TargetResourceType string + TargetLogicalID string + SourcePath string + SourceOrdinal int64 + LogicalEdgeKey *string + TargetResolved bool +} + +type EdgeRepository struct { + src ConnSource + tableFQN string + resourceTableFQN string +} + +func NewEdgeRepository(src ConnSource, schema string) (*EdgeRepository, error) { + if src == nil { + return nil, errors.New("postgres: nil edge connection source") + } + if !ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", ErrInvalidSchemaName, schema) + } + return &EdgeRepository{ + src: src, + tableFQN: pgx.Identifier{schema, edgeTableName}.Sanitize(), + resourceTableFQN: pgx.Identifier{schema, resourceTableName}.Sanitize(), + }, nil +} + +func (r *EdgeRepository) InsertEdge(ctx context.Context, rec EdgeOccurrenceRow) (int64, error) { + if err := validateEdgeForInsert(rec); err != nil { + return 0, err + } + conn, err := r.acquire(ctx) + if err != nil { + return 0, err + } + defer conn.Release() + + sql := "INSERT INTO " + r.tableFQN + + " (project, dataset_generation, auth_resource_path, label," + + " source_resource_type, source_logical_id, target_resource_type, target_logical_id," + + " source_path, source_ordinal, logical_edge_key) " + + "SELECT $1::text, $2::text, $3::text, $4::text, $5::text, $6::text, $7::text, $8::text, $9::text, $10::bigint, $11::text " + + "WHERE EXISTS (SELECT 1 FROM " + r.resourceTableFQN + " s" + + " WHERE s.project = $1 AND s.dataset_generation = $2" + + " AND s.resource_type = $5 AND s.logical_id = $6) " + + "ON CONFLICT ON CONSTRAINT " + edgeTableName + "_identity_ux DO NOTHING " + + "RETURNING edge_seq" + var seq int64 + err = conn.QueryRow(ctx, sql, + rec.Project, rec.DatasetGeneration, rec.AuthResourcePath, rec.Label, + rec.SourceResourceType, rec.SourceLogicalID, rec.TargetResourceType, rec.TargetLogicalID, + rec.SourcePath, rec.SourceOrdinal, rec.LogicalEdgeKey).Scan(&seq) + if errors.Is(err, pgx.ErrNoRows) { + return 0, r.classifyRefusedInsert(ctx, conn, rec) + } + if err != nil { + return 0, fmt.Errorf("postgres: insert edge %s/%s/%s/%s -%s-> %s/%s at path %q ordinal %d: %w", + rec.Project, rec.DatasetGeneration, rec.SourceResourceType, rec.SourceLogicalID, + rec.Label, rec.TargetResourceType, rec.TargetLogicalID, rec.SourcePath, rec.SourceOrdinal, err) + } + return seq, nil +} + +func (r *EdgeRepository) classifyRefusedInsert(ctx context.Context, conn *pgxpool.Conn, rec EdgeOccurrenceRow) error { + var sourceExists bool + if err := conn.QueryRow(ctx, + "SELECT EXISTS (SELECT 1 FROM "+r.resourceTableFQN+ + " WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3 AND logical_id = $4)", + rec.Project, rec.DatasetGeneration, rec.SourceResourceType, rec.SourceLogicalID, + ).Scan(&sourceExists); err != nil { + return fmt.Errorf("postgres: classify refused edge insert: %w", err) + } + if !sourceExists { + return &EdgeSourceMissingError{ + Project: rec.Project, + DatasetGeneration: rec.DatasetGeneration, + SourceResourceType: rec.SourceResourceType, + SourceLogicalID: rec.SourceLogicalID, + } + } + return &EdgeConflictError{ + Project: rec.Project, + DatasetGeneration: rec.DatasetGeneration, + AuthResourcePath: rec.AuthResourcePath, + Label: rec.Label, + SourceResourceType: rec.SourceResourceType, + SourceLogicalID: rec.SourceLogicalID, + TargetResourceType: rec.TargetResourceType, + TargetLogicalID: rec.TargetLogicalID, + SourcePath: rec.SourcePath, + SourceOrdinal: rec.SourceOrdinal, + } +} + +func (r *EdgeRepository) ReadEdgesFromSource(ctx context.Context, scope EdgeScope, + sourceResourceType, sourceLogicalID string, afterSeq int64, limit int) ([]EdgeOccurrenceRow, bool, error) { + where, args, binding, err := r.bindTraversal("source_resource_type", "source_logical_id", + scope, sourceResourceType, sourceLogicalID, afterSeq, limit) + if err != nil { + return nil, false, err + } + if binding.restricted && len(binding.paths) == 0 { + return nil, false, nil + } + return r.queryEdges(ctx, where, args, limit) +} + +func (r *EdgeRepository) ReadEdgesToTarget(ctx context.Context, scope EdgeScope, + targetResourceType, targetLogicalID string, afterSeq int64, limit int) ([]EdgeOccurrenceRow, bool, error) { + where, args, binding, err := r.bindTraversal("target_resource_type", "target_logical_id", + scope, targetResourceType, targetLogicalID, afterSeq, limit) + if err != nil { + return nil, false, err + } + if binding.restricted && len(binding.paths) == 0 { + return nil, false, nil + } + return r.queryEdges(ctx, where, args, limit) +} + +func (r *EdgeRepository) bindTraversal(endpointTypeColumn, endpointIDColumn string, + scope EdgeScope, endpointType, endpointID string, afterSeq int64, limit int) (string, []any, edgeScopeBinding, error) { + binding, err := normalizeEdgeScope(scope) + if err != nil { + return "", nil, binding, err + } + if strings.TrimSpace(endpointType) == "" { + return "", nil, binding, fmt.Errorf("%w: traversal endpoint resource type is blank", ErrInvalidEdge) + } + if strings.TrimSpace(endpointID) == "" { + return "", nil, binding, fmt.Errorf("%w: traversal endpoint logical id is blank", ErrInvalidEdge) + } + if afterSeq < 0 { + return "", nil, binding, fmt.Errorf("%w: traversal cursor is %d, want >= 0", ErrInvalidEdgeScope, afterSeq) + } + if limit < 1 || limit > MaxEdgeTraversalLimit { + return "", nil, binding, fmt.Errorf("%w: traversal limit is %d, want 1..%d", + ErrInvalidEdgeScope, limit, MaxEdgeTraversalLimit) + } + + where := " WHERE project = $1 AND dataset_generation = $2" + + " AND " + endpointTypeColumn + " = $3 AND " + endpointIDColumn + " = $4 AND edge_seq > $5" + args := []any{scope.Project, scope.DatasetGeneration, endpointType, endpointID, afterSeq} + if binding.restricted { + where += " AND auth_resource_path = ANY($6::text[])" + args = append(args, binding.paths) + } + where += fmt.Sprintf(" ORDER BY edge_seq LIMIT $%d", len(args)+1) + args = append(args, limit+1) + return where, args, binding, nil +} + +func (r *EdgeRepository) queryEdges(ctx context.Context, where string, args []any, limit int) ([]EdgeOccurrenceRow, bool, error) { + sql := "SELECT " + edgeColumns + ", EXISTS (SELECT 1 FROM " + r.resourceTableFQN + " t" + + " WHERE t.project = " + r.tableFQN + ".project" + + " AND t.dataset_generation = " + r.tableFQN + ".dataset_generation" + + " AND t.resource_type = " + r.tableFQN + ".target_resource_type" + + " AND t.logical_id = " + r.tableFQN + ".target_logical_id) AS target_resolved" + + " FROM " + r.tableFQN + where + conn, err := r.acquire(ctx) + if err != nil { + return nil, false, err + } + defer conn.Release() + + rows, err := conn.Query(ctx, sql, args...) + if err != nil { + return nil, false, fmt.Errorf("postgres: traverse edges: %w", err) + } + defer rows.Close() + + var out []EdgeOccurrenceRow + for rows.Next() { + rec, err := scanEdgeRow(rows) + if err != nil { + return nil, false, fmt.Errorf("postgres: traverse edges: %w", err) + } + out = append(out, rec) + } + if err := rows.Err(); err != nil { + return nil, false, fmt.Errorf("postgres: traverse edges: %w", err) + } + hasMore := len(out) > limit + if hasMore { + out = out[:limit] + } + return out, hasMore, nil +} + +func (r *EdgeRepository) acquire(ctx context.Context) (*pgxpool.Conn, error) { + conn, err := r.src.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres: acquire edge connection: %w", err) + } + return conn, nil +} + +func scanEdgeRow(row pgx.Row) (EdgeOccurrenceRow, error) { + var rec EdgeOccurrenceRow + var logicalEdgeKey *string + if err := row.Scan( + &rec.EdgeSeq, &rec.Project, &rec.DatasetGeneration, &rec.AuthResourcePath, + &rec.Label, &rec.SourceResourceType, &rec.SourceLogicalID, + &rec.TargetResourceType, &rec.TargetLogicalID, + &rec.SourcePath, &rec.SourceOrdinal, &logicalEdgeKey, &rec.TargetResolved, + ); err != nil { + return EdgeOccurrenceRow{}, err + } + rec.LogicalEdgeKey = logicalEdgeKey + return rec, nil +} + +func validateEdgeForInsert(rec EdgeOccurrenceRow) error { + if rec.EdgeSeq != 0 { + return fmt.Errorf("%w: EdgeSeq is %d; the storage identity is server-assigned, insert requires 0", + ErrInvalidEdge, rec.EdgeSeq) + } + for _, f := range []struct { + name string + value string + }{ + {"project", rec.Project}, + {"label", rec.Label}, + {"source resource type", rec.SourceResourceType}, + {"source logical id", rec.SourceLogicalID}, + {"target resource type", rec.TargetResourceType}, + {"target logical id", rec.TargetLogicalID}, + } { + if strings.TrimSpace(f.value) == "" { + return fmt.Errorf("%w: %s is blank", ErrInvalidEdge, f.name) + } + } + if rec.SourceOrdinal < 0 { + return fmt.Errorf("%w: source ordinal is %d, want >= 0", ErrInvalidEdge, rec.SourceOrdinal) + } + return nil +} + +type edgeScopeBinding struct { + restricted bool + paths []string +} + +func normalizeEdgeScope(scope EdgeScope) (edgeScopeBinding, error) { + if strings.TrimSpace(scope.Project) == "" { + return edgeScopeBinding{}, fmt.Errorf("%w: project is blank", ErrInvalidEdgeScope) + } + if scope.AuthResourcePathsUnrestricted == nil { + return edgeScopeBinding{}, fmt.Errorf( + "%w: AuthResourcePathsUnrestricted must be set explicitly; there is no implicit unrestricted mode", + ErrInvalidEdgeScope) + } + if *scope.AuthResourcePathsUnrestricted { + if len(scope.AuthResourcePaths) > 0 { + return edgeScopeBinding{}, fmt.Errorf( + "%w: unrestricted mode must not carry auth resource paths (%d given)", + ErrInvalidEdgeScope, len(scope.AuthResourcePaths)) + } + return edgeScopeBinding{restricted: false}, nil + } + return edgeScopeBinding{restricted: true, paths: scope.AuthResourcePaths}, nil +} diff --git a/internal/backends/postgres/internal/store/edge_test.go b/internal/backends/postgres/internal/store/edge_test.go new file mode 100644 index 000000000..1345541a3 --- /dev/null +++ b/internal/backends/postgres/internal/store/edge_test.go @@ -0,0 +1,1067 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "net/url" + "reflect" + "regexp" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +var pgstore05EdgeNamePattern = regexp.MustCompile(`^pgstore05_it_[0-9]+_[0-9a-f]+$`) + +type edgeService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func newEdgeService(t *testing.T) *edgeService { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgstore05_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstore05EdgeNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("pgstore05_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstore05EdgeNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + return &edgeService{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} +} + +func (s *edgeService) openClient(t *testing.T) *Client { + t.Helper() + client, err := Open(s.ctx, Config{ + DSN: s.dsn, + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *edgeService) newRepositories(t *testing.T) (*EdgeRepository, *ResourceRepository) { + t.Helper() + edgeRepo, err := NewEdgeRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new edge repository", err) + } + resourceRepo, err := NewResourceRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new resource repository", err) + } + return edgeRepo, resourceRepo +} + +func (s *edgeService) mustMigrateEmbedded(t *testing.T) MigrateResult { + t.Helper() + result, err := MigrateSchema(s.ctx, s.openClient(t), s.schema) + if err != nil { + s.fail("MigrateSchema", err) + } + return result +} + +func (s *edgeService) readLedger(t *testing.T) []LedgerRecord { + t.Helper() + rows, err := s.iso.Query(s.ctx, + `SELECT version, name, checksum, applied_at FROM "`+s.schema+`".`+LedgerTable+` ORDER BY version`) + if err != nil { + s.fail("witness ledger", err) + } + defer rows.Close() + var out []LedgerRecord + for rows.Next() { + var rec LedgerRecord + if err := rows.Scan(&rec.Version, &rec.Name, &rec.Checksum, &rec.AppliedAt); err != nil { + s.fail("witness ledger scan", err) + } + out = append(out, rec) + } + if err := rows.Err(); err != nil { + s.fail("witness ledger rows", err) + } + return out +} + +func (s *edgeService) mustSeedResource(t *testing.T, repo *ResourceRepository, + project, generation, resourceType, logicalID, authPath string) int64 { + t.Helper() + seq, err := repo.InsertResource(s.ctx, source.Resource{ + Project: project, DatasetGeneration: generation, ResourceType: resourceType, + LogicalID: logicalID, AuthResourcePath: authPath, + Payload: []byte(`{"resourceType":"` + resourceType + `","id":"` + logicalID + `"}`), + }) + if err != nil { + s.fail(fmt.Sprintf("seed resource %s/%s/%s/%s", project, generation, resourceType, logicalID), err) + } + return seq +} + +func (s *edgeService) mustInsertEdge(t *testing.T, repo *EdgeRepository, rec EdgeOccurrenceRow) int64 { + t.Helper() + seq, err := repo.InsertEdge(s.ctx, rec) + if err != nil { + s.fail(fmt.Sprintf("insert edge %s/%s/%s/%s -%s-> %s/%s at path %q ordinal %d", + rec.Project, rec.DatasetGeneration, rec.SourceResourceType, rec.SourceLogicalID, + rec.Label, rec.TargetResourceType, rec.TargetLogicalID, rec.SourcePath, rec.SourceOrdinal), err) + } + if seq < 1 { + t.Fatalf("insert returned storage identity %d, want >= 1", seq) + } + return seq +} + +func (s *edgeService) mustReadFromSource(t *testing.T, repo *EdgeRepository, + scope EdgeScope, sourceType, sourceID string, afterSeq int64, limit int) ([]EdgeOccurrenceRow, bool) { + t.Helper() + rows, hasMore, err := repo.ReadEdgesFromSource(s.ctx, scope, sourceType, sourceID, afterSeq, limit) + if err != nil { + s.fail(fmt.Sprintf("source traversal %s/%s/%s/%s after %d", + scope.Project, scope.DatasetGeneration, sourceType, sourceID, afterSeq), err) + } + return rows, hasMore +} + +func (s *edgeService) mustReadToTarget(t *testing.T, repo *EdgeRepository, + scope EdgeScope, targetType, targetID string, afterSeq int64, limit int) ([]EdgeOccurrenceRow, bool) { + t.Helper() + rows, hasMore, err := repo.ReadEdgesToTarget(s.ctx, scope, targetType, targetID, afterSeq, limit) + if err != nil { + s.fail(fmt.Sprintf("target traversal %s/%s/%s/%s after %d", + scope.Project, scope.DatasetGeneration, targetType, targetID, afterSeq), err) + } + return rows, hasMore +} + +func (s *edgeService) countEdges(t *testing.T, rec EdgeOccurrenceRow) int64 { + t.Helper() + var count int64 + if err := s.iso.QueryRow(s.ctx, + `SELECT count(*) FROM "`+s.schema+`".fhir_edge_occurrence + WHERE project = $1 AND dataset_generation = $2 AND source_resource_type = $3 + AND source_logical_id = $4 AND label = $5 AND target_resource_type = $6 + AND target_logical_id = $7 AND source_path = $8 AND source_ordinal = $9`, + rec.Project, rec.DatasetGeneration, rec.SourceResourceType, rec.SourceLogicalID, + rec.Label, rec.TargetResourceType, rec.TargetLogicalID, rec.SourcePath, rec.SourceOrdinal, + ).Scan(&count); err != nil { + s.fail("witness occurrence count", err) + } + return count +} + +func (s *edgeService) countResources(t *testing.T, project, generation, resourceType, logicalID string) int64 { + t.Helper() + var count int64 + if err := s.iso.QueryRow(s.ctx, + `SELECT count(*) FROM "`+s.schema+`".fhir_resource + WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3 AND logical_id = $4`, + project, generation, resourceType, logicalID).Scan(&count); err != nil { + s.fail("witness resource count", err) + } + return count +} + +type edgeRowWitness struct { + EdgeSeq int64 + Project string + DatasetGeneration string + AuthResourcePath string + Label string + SourceResourceType string + SourceLogicalID string + TargetResourceType string + TargetLogicalID string + SourcePath string + SourceOrdinal int64 + LogicalEdgeKey *string + RecordedAtPresent bool +} + +func (s *edgeService) witnessEdgeRows(t *testing.T, + project, generation, sourceType, sourceID, label, targetType, targetID string) []edgeRowWitness { + t.Helper() + rows, err := s.iso.Query(s.ctx, + `SELECT edge_seq, project, dataset_generation, auth_resource_path, label, + source_resource_type, source_logical_id, target_resource_type, target_logical_id, + source_path, source_ordinal, logical_edge_key, recorded_at IS NOT NULL + FROM "`+s.schema+`".fhir_edge_occurrence + WHERE project = $1 AND dataset_generation = $2 + AND source_resource_type = $3 AND source_logical_id = $4 + AND label = $5 AND target_resource_type = $6 AND target_logical_id = $7 + ORDER BY edge_seq`, + project, generation, sourceType, sourceID, label, targetType, targetID) + if err != nil { + s.fail("witness edge rows", err) + } + defer rows.Close() + var out []edgeRowWitness + for rows.Next() { + var w edgeRowWitness + if err := rows.Scan(&w.EdgeSeq, &w.Project, &w.DatasetGeneration, &w.AuthResourcePath, + &w.Label, &w.SourceResourceType, &w.SourceLogicalID, &w.TargetResourceType, + &w.TargetLogicalID, &w.SourcePath, &w.SourceOrdinal, &w.LogicalEdgeKey, &w.RecordedAtPresent); err != nil { + s.fail("witness edge rows scan", err) + } + out = append(out, w) + } + if err := rows.Err(); err != nil { + s.fail("witness edge rows err", err) + } + return out +} + +func edgeRestrictedScope(project, generation string, paths ...string) EdgeScope { + restricted := false + return EdgeScope{ + Project: project, + DatasetGeneration: generation, + AuthResourcePathsUnrestricted: &restricted, + AuthResourcePaths: paths, + } +} + +func edgeUnrestrictedScope(project, generation string) EdgeScope { + unrestricted := true + return EdgeScope{ + Project: project, + DatasetGeneration: generation, + AuthResourcePathsUnrestricted: &unrestricted, + } +} + +func edgeRow(project, generation, authPath, label, srcType, srcID, tgtType, tgtID, path string, ordinal int64) EdgeOccurrenceRow { + return EdgeOccurrenceRow{ + Project: project, DatasetGeneration: generation, AuthResourcePath: authPath, + Label: label, SourceResourceType: srcType, SourceLogicalID: srcID, + TargetResourceType: tgtType, TargetLogicalID: tgtID, + SourcePath: path, SourceOrdinal: ordinal, + } +} + +func TestEdgeValidationRejectsInvalidInserts(t *testing.T) { + repo, err := NewEdgeRepository(stubConnSource{}, "pgstore05_stub") + if err != nil { + t.Fatalf("NewEdgeRepository: %v", err) + } + base := edgeRow("proj", "gen", "practitioner/pr-1", "participant", + "Encounter", "enc-1", "Practitioner", "pr-9", "participant[0]", 0) + for _, tc := range []struct { + name string + mutate func(*EdgeOccurrenceRow) + }{ + {"caller-supplied storage identity", func(r *EdgeOccurrenceRow) { r.EdgeSeq = 7 }}, + {"blank project", func(r *EdgeOccurrenceRow) { r.Project = " " }}, + {"blank label", func(r *EdgeOccurrenceRow) { r.Label = "\t" }}, + {"blank source type", func(r *EdgeOccurrenceRow) { r.SourceResourceType = "" }}, + {"blank source id", func(r *EdgeOccurrenceRow) { r.SourceLogicalID = " " }}, + {"blank target type", func(r *EdgeOccurrenceRow) { r.TargetResourceType = "" }}, + {"blank target id", func(r *EdgeOccurrenceRow) { r.TargetLogicalID = "\n" }}, + {"negative ordinal", func(r *EdgeOccurrenceRow) { r.SourceOrdinal = -1 }}, + } { + rec := base + tc.mutate(&rec) + _, err := repo.InsertEdge(context.Background(), rec) + if !errors.Is(err, ErrInvalidEdge) { + t.Errorf("%s: InsertEdge = %v, want ErrInvalidEdge", tc.name, err) + } + } +} + +func TestEdgeScopeAndTraversalValidation(t *testing.T) { + repo, err := NewEdgeRepository(stubConnSource{}, "pgstore05_stub") + if err != nil { + t.Fatalf("NewEdgeRepository: %v", err) + } + ctx := context.Background() + + if _, err := NewEdgeRepository(nil, "pgstore05_stub"); err == nil { + t.Error("NewEdgeRepository with nil source succeeded, want error") + } + if _, err := NewEdgeRepository(stubConnSource{}, "Bad-Schema"); !errors.Is(err, ErrInvalidSchemaName) { + t.Errorf("NewEdgeRepository with invalid schema = %v, want ErrInvalidSchemaName", err) + } + + scope := EdgeScope{Project: "proj", AuthResourcePaths: []string{"a"}} + if _, _, err := repo.ReadEdgesFromSource(ctx, scope, "Encounter", "enc-1", 0, 10); !errors.Is(err, ErrInvalidEdgeScope) { + t.Errorf("source traversal with nil unrestricted flag = %v, want ErrInvalidEdgeScope", err) + } + if _, _, err := repo.ReadEdgesToTarget(ctx, scope, "Practitioner", "pr-9", 0, 10); !errors.Is(err, ErrInvalidEdgeScope) { + t.Errorf("target traversal with nil unrestricted flag = %v, want ErrInvalidEdgeScope", err) + } + + unrestrictedWithPaths := edgeUnrestrictedScope("proj", "gen") + unrestrictedWithPaths.AuthResourcePaths = []string{"practitioner/pr-1"} + if _, _, err := repo.ReadEdgesFromSource(ctx, unrestrictedWithPaths, "Encounter", "enc-1", 0, 10); !errors.Is(err, ErrInvalidEdgeScope) { + t.Errorf("unrestricted scope carrying paths = %v, want ErrInvalidEdgeScope", err) + } + + if _, _, err := repo.ReadEdgesFromSource(ctx, edgeRestrictedScope(" ", "gen", "a"), "Encounter", "enc-1", 0, 10); !errors.Is(err, ErrInvalidEdgeScope) { + t.Errorf("blank project = %v, want ErrInvalidEdgeScope", err) + } + + if _, _, err := repo.ReadEdgesFromSource(ctx, edgeUnrestrictedScope("proj", "gen"), " ", "enc-1", 0, 10); !errors.Is(err, ErrInvalidEdge) { + t.Errorf("blank source type = %v, want ErrInvalidEdge", err) + } + if _, _, err := repo.ReadEdgesFromSource(ctx, edgeUnrestrictedScope("proj", "gen"), "Encounter", "", 0, 10); !errors.Is(err, ErrInvalidEdge) { + t.Errorf("blank source id = %v, want ErrInvalidEdge", err) + } + if _, _, err := repo.ReadEdgesToTarget(ctx, edgeUnrestrictedScope("proj", "gen"), "", "pr-9", 0, 10); !errors.Is(err, ErrInvalidEdge) { + t.Errorf("blank target type = %v, want ErrInvalidEdge", err) + } + if _, _, err := repo.ReadEdgesToTarget(ctx, edgeUnrestrictedScope("proj", "gen"), "Practitioner", " \t", 0, 10); !errors.Is(err, ErrInvalidEdge) { + t.Errorf("blank target id = %v, want ErrInvalidEdge", err) + } + + for _, limit := range []int{0, -1, MaxEdgeTraversalLimit + 1} { + if _, _, err := repo.ReadEdgesFromSource(ctx, edgeUnrestrictedScope("proj", "gen"), "Encounter", "enc-1", 0, limit); !errors.Is(err, ErrInvalidEdgeScope) { + t.Errorf("limit %d = %v, want ErrInvalidEdgeScope", limit, err) + } + } + if _, _, err := repo.ReadEdgesToTarget(ctx, edgeUnrestrictedScope("proj", "gen"), "Practitioner", "pr-9", -1, 10); !errors.Is(err, ErrInvalidEdgeScope) { + t.Errorf("negative cursor = %v, want ErrInvalidEdgeScope", err) + } +} + +func TestEdgeMigrationCreatesTypedOccurrenceTable(t *testing.T) { + svc := newEdgeService(t) + first := svc.mustMigrateEmbedded(t) + second := svc.mustMigrateEmbedded(t) + if !reflect.DeepEqual(first, second) { + t.Fatalf("re-apply changed the result:\n1st: %+v\n2nd: %+v", first, second) + } + assertLedgerEquals(t, svc.readLedger(t), mustEmbeddedMigrations(t)) + + var recordedName string + if err := svc.iso.QueryRow(svc.ctx, + `SELECT name FROM "`+svc.schema+`".`+LedgerTable+` WHERE version = 4`).Scan(&recordedName); err != nil { + svc.fail("witness version 4 ledger row", err) + } + if recordedName != "edge_occurrence" { + t.Fatalf("version 4 ledger name = %q, want edge_occurrence", recordedName) + } + + type columnWitness struct { + Name string + Type string + NotNull bool + Identity string + } + rows, err := svc.iso.Query(svc.ctx, ` + SELECT a.attname, format_type(a.atttypid, a.atttypmod), a.attnotnull, + CASE WHEN a.attidentity = 'a' THEN 'a' + WHEN a.attidentity = 'd' THEN 'd' + ELSE '' END + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = 'fhir_edge_occurrence' + AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum`, svc.schema) + if err != nil { + svc.fail("witness columns", err) + } + defer rows.Close() + var columns []columnWitness + for rows.Next() { + var col columnWitness + if err := rows.Scan(&col.Name, &col.Type, &col.NotNull, &col.Identity); err != nil { + svc.fail("witness columns scan", err) + } + columns = append(columns, col) + } + if err := rows.Err(); err != nil { + svc.fail("witness columns rows", err) + } + wantColumns := []columnWitness{ + {"edge_seq", "bigint", true, "a"}, + {"project", "text", true, ""}, + {"dataset_generation", "text", true, ""}, + {"auth_resource_path", "text", true, ""}, + {"label", "text", true, ""}, + {"source_resource_type", "text", true, ""}, + {"source_logical_id", "text", true, ""}, + {"target_resource_type", "text", true, ""}, + {"target_logical_id", "text", true, ""}, + {"source_path", "text", true, ""}, + {"source_ordinal", "bigint", true, ""}, + {"logical_edge_key", "text", false, ""}, + {"recorded_at", "timestamp with time zone", true, ""}, + } + if !reflect.DeepEqual(columns, wantColumns) { + t.Fatalf("fhir_edge_occurrence columns = %+v, want %+v", columns, wantColumns) + } + + constraintRows, err := svc.iso.Query(svc.ctx, ` + SELECT con.conname, pg_get_constraintdef(con.oid) + FROM pg_constraint con + JOIN pg_class c ON c.oid = con.conrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = 'fhir_edge_occurrence' + AND con.contype IN ('p', 'u', 'c') + ORDER BY con.conname`, svc.schema) + if err != nil { + svc.fail("witness constraints", err) + } + defer constraintRows.Close() + var constraints [][2]string + for constraintRows.Next() { + var name, def string + if err := constraintRows.Scan(&name, &def); err != nil { + svc.fail("witness constraints scan", err) + } + constraints = append(constraints, [2]string{name, def}) + } + if err := constraintRows.Err(); err != nil { + svc.fail("witness constraints rows", err) + } + wantConstraints := [][2]string{ + {"fhir_edge_occurrence_identity_ux", + `UNIQUE (project, dataset_generation, source_resource_type, source_logical_id, label, target_resource_type, target_logical_id, source_path, source_ordinal)`}, + {"fhir_edge_occurrence_label_present", `CHECK ((label <> ''::text))`}, + {"fhir_edge_occurrence_ordinal_nonnegative", `CHECK ((source_ordinal >= 0))`}, + {"fhir_edge_occurrence_pkey", `PRIMARY KEY (edge_seq)`}, + {"fhir_edge_occurrence_project_present", `CHECK ((project <> ''::text))`}, + {"fhir_edge_occurrence_source_id_present", `CHECK ((source_logical_id <> ''::text))`}, + {"fhir_edge_occurrence_source_type_present", `CHECK ((source_resource_type <> ''::text))`}, + {"fhir_edge_occurrence_target_id_present", `CHECK ((target_logical_id <> ''::text))`}, + {"fhir_edge_occurrence_target_type_present", `CHECK ((target_resource_type <> ''::text))`}, + } + if !reflect.DeepEqual(constraints, wantConstraints) { + t.Fatalf("fhir_edge_occurrence constraints = %+v, want %+v", constraints, wantConstraints) + } + + indexRows, err := svc.iso.Query(svc.ctx, ` + SELECT indexname, indexdef FROM pg_indexes + WHERE schemaname = $1 AND tablename = 'fhir_edge_occurrence' + ORDER BY indexname`, svc.schema) + if err != nil { + svc.fail("witness indexes", err) + } + defer indexRows.Close() + var indexes [][2]string + for indexRows.Next() { + var name, def string + if err := indexRows.Scan(&name, &def); err != nil { + svc.fail("witness indexes scan", err) + } + indexes = append(indexes, [2]string{name, def}) + } + if err := indexRows.Err(); err != nil { + svc.fail("witness indexes rows", err) + } + wantIndexes := [][2]string{ + {"fhir_edge_occurrence_identity_ux", + `CREATE UNIQUE INDEX fhir_edge_occurrence_identity_ux ON ` + svc.schema + `.fhir_edge_occurrence USING btree (project, dataset_generation, source_resource_type, source_logical_id, label, target_resource_type, target_logical_id, source_path, source_ordinal)`}, + {"fhir_edge_occurrence_pkey", + `CREATE UNIQUE INDEX fhir_edge_occurrence_pkey ON ` + svc.schema + `.fhir_edge_occurrence USING btree (edge_seq)`}, + {"fhir_edge_occurrence_source_traversal_ix", + `CREATE INDEX fhir_edge_occurrence_source_traversal_ix ON ` + svc.schema + `.fhir_edge_occurrence USING btree (project, dataset_generation, source_resource_type, source_logical_id, auth_resource_path, edge_seq)`}, + {"fhir_edge_occurrence_target_traversal_ix", + `CREATE INDEX fhir_edge_occurrence_target_traversal_ix ON ` + svc.schema + `.fhir_edge_occurrence USING btree (project, dataset_generation, target_resource_type, target_logical_id, auth_resource_path, edge_seq)`}, + } + if !reflect.DeepEqual(indexes, wantIndexes) { + t.Fatalf("fhir_edge_occurrence indexes = %+v, want %+v", indexes, wantIndexes) + } + + var rowCount int64 + if err := svc.iso.QueryRow(svc.ctx, + `SELECT count(*) FROM "`+svc.schema+`".fhir_edge_occurrence`).Scan(&rowCount); err != nil { + svc.fail("count edges", err) + } + if rowCount != 0 { + t.Fatalf("fresh fhir_edge_occurrence has %d rows, want 0", rowCount) + } +} + +func TestEdgeInsertRequiresSourceInSameScope(t *testing.T) { + svc := newEdgeService(t) + svc.mustMigrateEmbedded(t) + edgeRepo, resourceRepo := svc.newRepositories(t) + + svc.mustSeedResource(t, resourceRepo, "proj-s", "gen-1", "Encounter", "enc-1", "practitioner/pr-1") + svc.mustSeedResource(t, resourceRepo, "proj-s", "gen-2", "Encounter", "enc-late", "practitioner/pr-1") + svc.mustSeedResource(t, resourceRepo, "proj-other", "gen-1", "Encounter", "enc-1", "practitioner/pr-1") + + landed := edgeRow("proj-s", "gen-1", "practitioner/pr-1", "participant", + "Encounter", "enc-1", "Practitioner", "pr-9", "participant[0]", 0) + seq := svc.mustInsertEdge(t, edgeRepo, landed) + if seq < 1 { + t.Fatalf("storage identity %d, want >= 1", seq) + } + + missingCases := []struct { + name string + rec EdgeOccurrenceRow + }{ + {"no source at all", edgeRow("proj-s", "gen-1", "", "participant", + "Encounter", "enc-nowhere", "Practitioner", "pr-9", "participant[0]", 0)}, + {"source only in another generation", edgeRow("proj-s", "gen-1", "", "participant", + "Encounter", "enc-late", "Practitioner", "pr-9", "participant[0]", 0)}, + {"source only in another project", edgeRow("proj-missing", "gen-1", "", "participant", + "Encounter", "enc-1", "Practitioner", "pr-9", "participant[0]", 0)}, + } + for _, tc := range missingCases { + _, err := edgeRepo.InsertEdge(svc.ctx, tc.rec) + if !errors.Is(err, ErrEdgeSourceMissing) { + t.Errorf("%s: InsertEdge = %v, want ErrEdgeSourceMissing", tc.name, err) + continue + } + var missing *EdgeSourceMissingError + if !errors.As(err, &missing) { + t.Errorf("%s: error type = %T, want *EdgeSourceMissingError", tc.name, err) + continue + } + want := &EdgeSourceMissingError{ + Project: tc.rec.Project, + DatasetGeneration: tc.rec.DatasetGeneration, + SourceResourceType: tc.rec.SourceResourceType, + SourceLogicalID: tc.rec.SourceLogicalID, + } + if !reflect.DeepEqual(missing, want) { + t.Errorf("%s: error = %+v, want %+v", tc.name, missing, want) + } + if count := svc.countEdges(t, tc.rec); count != 0 { + t.Errorf("%s: refused insert left %d rows, want 0", tc.name, count) + } + } + + if count := svc.countEdges(t, landed); count != 1 { + t.Fatalf("in-scope occurrence has %d rows, want exactly 1", count) + } +} + +func TestEdgeUnresolvedTargetPreservesIdentityAndResolvesLater(t *testing.T) { + svc := newEdgeService(t) + svc.mustMigrateEmbedded(t) + edgeRepo, resourceRepo := svc.newRepositories(t) + + svc.mustSeedResource(t, resourceRepo, "proj-u", "gen-1", "Observation", "obs-src", "") + rec := edgeRow("proj-u", "gen-1", "", "focus", + "Observation", "obs-src", "Condition", "cond-ghost", "focus", 0) + rec.LogicalEdgeKey = ptr("Observation_obs-src_focus_Condition_cond-ghost") + insertedSeq := svc.mustInsertEdge(t, edgeRepo, rec) + + if svc.countResources(t, "proj-u", "gen-1", "Condition", "cond-ghost") != 0 { + t.Fatal("witness setup: Condition/cond-ghost must not exist yet") + } + witnesses := svc.witnessEdgeRows(t, "proj-u", "gen-1", "Observation", "obs-src", "focus", "Condition", "cond-ghost") + if len(witnesses) != 1 { + t.Fatalf("unresolved target stored %d rows, want exactly 1", len(witnesses)) + } + want := edgeRowWitness{ + EdgeSeq: insertedSeq, + Project: "proj-u", DatasetGeneration: "gen-1", AuthResourcePath: "", + Label: "focus", SourceResourceType: "Observation", SourceLogicalID: "obs-src", + TargetResourceType: "Condition", TargetLogicalID: "cond-ghost", + SourcePath: "focus", SourceOrdinal: 0, + LogicalEdgeKey: ptr("Observation_obs-src_focus_Condition_cond-ghost"), + RecordedAtPresent: true, + } + if !reflect.DeepEqual(witnesses[0], want) { + t.Fatalf("stored edge row = %+v, want %+v", witnesses[0], want) + } + + rows, hasMore := svc.mustReadFromSource(t, edgeRepo, edgeUnrestrictedScope("proj-u", "gen-1"), + "Observation", "obs-src", 0, 10) + if len(rows) != 1 || hasMore { + t.Fatalf("source traversal = %d rows (hasMore=%t), want 1 row", len(rows), hasMore) + } + row := rows[0] + if row.TargetResourceType != "Condition" || row.TargetLogicalID != "cond-ghost" { + t.Errorf("unresolved target identity = %s/%s, want Condition/cond-ghost preserved", + row.TargetResourceType, row.TargetLogicalID) + } + if row.TargetResolved { + t.Errorf("TargetResolved = true before the target resource exists") + } + if row.SourceResourceType != "Observation" || row.SourceLogicalID != "obs-src" || + row.Label != "focus" || row.SourcePath != "focus" || row.SourceOrdinal != 0 { + t.Errorf("occurrence identity drifted on read: %+v", row) + } + if row.LogicalEdgeKey == nil || *row.LogicalEdgeKey != "Observation_obs-src_focus_Condition_cond-ghost" { + t.Errorf("logical edge key = %v, want the stored diagnostic", row.LogicalEdgeKey) + } + + svc.mustSeedResource(t, resourceRepo, "proj-u", "gen-1", "Condition", "cond-ghost", "") + rows, _ = svc.mustReadFromSource(t, edgeRepo, edgeUnrestrictedScope("proj-u", "gen-1"), + "Observation", "obs-src", 0, 10) + if len(rows) != 1 { + t.Fatalf("after target insert, traversal = %d rows, want 1 (no duplicate edge)", len(rows)) + } + if !rows[0].TargetResolved { + t.Errorf("TargetResolved = false after the target resource was stored in scope") + } + if rows[0].EdgeSeq != row.EdgeSeq { + t.Errorf("edge storage identity changed from %d to %d: target insert must not rewrite edges", + row.EdgeSeq, rows[0].EdgeSeq) + } +} + +func TestEdgeRepeatedReferenceMultiplicityPreserved(t *testing.T) { + svc := newEdgeService(t) + svc.mustMigrateEmbedded(t) + edgeRepo, resourceRepo := svc.newRepositories(t) + + svc.mustSeedResource(t, resourceRepo, "proj-m", "gen-1", "Encounter", "enc-1", "") + const collapsedKey = "Encounter_enc-1_participant_Practitioner_pr-1" + first := edgeRow("proj-m", "gen-1", "", "participant", + "Encounter", "enc-1", "Practitioner", "pr-1", "participant[0]", 0) + first.LogicalEdgeKey = ptr(collapsedKey) + second := edgeRow("proj-m", "gen-1", "", "participant", + "Encounter", "enc-1", "Practitioner", "pr-1", "participant[1]", 1) + second.LogicalEdgeKey = ptr(collapsedKey) + + seqA := svc.mustInsertEdge(t, edgeRepo, first) + seqB := svc.mustInsertEdge(t, edgeRepo, second) + if seqA == seqB { + t.Fatalf("both repeated references returned storage identity %d, want two distinct rows", seqA) + } + + witnesses := svc.witnessEdgeRows(t, "proj-m", "gen-1", "Encounter", "enc-1", "participant", "Practitioner", "pr-1") + if len(witnesses) != 2 { + t.Fatalf("repeated reference stored %d rows, want 2: %+v", len(witnesses), witnesses) + } + wantRows := []edgeRowWitness{ + {EdgeSeq: seqA, Project: "proj-m", DatasetGeneration: "gen-1", AuthResourcePath: "", + Label: "participant", SourceResourceType: "Encounter", SourceLogicalID: "enc-1", + TargetResourceType: "Practitioner", TargetLogicalID: "pr-1", + SourcePath: "participant[0]", SourceOrdinal: 0, + LogicalEdgeKey: ptr(collapsedKey), RecordedAtPresent: true}, + {EdgeSeq: seqB, Project: "proj-m", DatasetGeneration: "gen-1", AuthResourcePath: "", + Label: "participant", SourceResourceType: "Encounter", SourceLogicalID: "enc-1", + TargetResourceType: "Practitioner", TargetLogicalID: "pr-1", + SourcePath: "participant[1]", SourceOrdinal: 1, + LogicalEdgeKey: ptr(collapsedKey), RecordedAtPresent: true}, + } + if !reflect.DeepEqual(witnesses, wantRows) { + t.Fatalf("stored rows =\n%+v\nwant\n%+v", witnesses, wantRows) + } + + third := edgeRow("proj-m", "gen-1", "", "participant", + "Encounter", "enc-1", "Practitioner", "pr-1", "participant[0]", 1) + third.LogicalEdgeKey = ptr(collapsedKey) + svc.mustInsertEdge(t, edgeRepo, third) + if got := svc.countEdges(t, third); got != 1 { + t.Fatalf("same-path new-ordinal occurrence has %d rows, want 1", got) + } + + duplicate := first + duplicate.AuthResourcePath = "practitioner/pr-other" + _, err := edgeRepo.InsertEdge(svc.ctx, duplicate) + if !errors.Is(err, ErrEdgeConflict) { + t.Fatalf("exact duplicate insert = %v, want ErrEdgeConflict", err) + } + var conflict *EdgeConflictError + if !errors.As(err, &conflict) { + t.Fatalf("error type = %T, want *EdgeConflictError", err) + } + wantConflict := &EdgeConflictError{ + Project: "proj-m", DatasetGeneration: "gen-1", AuthResourcePath: "practitioner/pr-other", + Label: "participant", SourceResourceType: "Encounter", SourceLogicalID: "enc-1", + TargetResourceType: "Practitioner", TargetLogicalID: "pr-1", + SourcePath: "participant[0]", SourceOrdinal: 0, + } + if !reflect.DeepEqual(conflict, wantConflict) { + t.Errorf("conflict = %+v, want %+v", conflict, wantConflict) + } + if got := svc.countEdges(t, first); got != 1 { + t.Errorf("exact duplicate identity holds %d rows after the conflict, want exactly 1 (no silent upsert)", got) + } + after := svc.witnessEdgeRows(t, "proj-m", "gen-1", "Encounter", "enc-1", "participant", "Practitioner", "pr-1") + if len(after) != 3 { + t.Fatalf("occurrence group has %d rows after conflict, want 3", len(after)) + } + for _, w := range after { + if w.EdgeSeq == seqA && w.AuthResourcePath != "" { + t.Errorf("the conflict replaced the stored auth path with %q", w.AuthResourcePath) + } + } +} + +func TestEdgeTraversalsBindScopeAndOrder(t *testing.T) { + svc := newEdgeService(t) + svc.mustMigrateEmbedded(t) + edgeRepo, resourceRepo := svc.newRepositories(t) + + svc.mustSeedResource(t, resourceRepo, "proj-t", "gen-1", "Encounter", "enc-w", "practitioner/pr-1") + svc.mustSeedResource(t, resourceRepo, "proj-t", "gen-1", "Encounter", "enc-w2", "practitioner/pr-1") + svc.mustSeedResource(t, resourceRepo, "proj-x", "gen-1", "Encounter", "enc-w", "practitioner/pr-1") + svc.mustSeedResource(t, resourceRepo, "proj-t", "gen-2", "Encounter", "enc-w", "practitioner/pr-1") + svc.mustSeedResource(t, resourceRepo, "proj-t", "gen-1", "Patient", "pt-2", "practitioner/pr-1") + svc.mustSeedResource(t, resourceRepo, "proj-t", "gen-1", "Patient", "pt-4", "practitioner/pr-1") + + inScope := []EdgeOccurrenceRow{ + edgeRow("proj-t", "gen-1", "practitioner/pr-1", "have", "Encounter", "enc-w", "Patient", "pt-1", "p", 0), + edgeRow("proj-t", "gen-1", "practitioner/pr-1", "have", "Encounter", "enc-w", "Patient", "pt-2", "p", 0), + edgeRow("proj-t", "gen-1", "practitioner/pr-1", "have", "Encounter", "enc-w", "Patient", "pt-3", "p", 0), + edgeRow("proj-t", "gen-1", "practitioner/pr-1", "have", "Encounter", "enc-w", "Patient", "pt-4", "p", 0), + edgeRow("proj-t", "gen-1", "practitioner/pr-1", "have", "Encounter", "enc-w", "Patient", "pt-5", "p", 0), + } + decoys := []EdgeOccurrenceRow{ + edgeRow("proj-x", "gen-1", "practitioner/pr-1", "have", "Encounter", "enc-w", "Patient", "x1", "p", 0), + edgeRow("proj-t", "gen-2", "practitioner/pr-1", "have", "Encounter", "enc-w", "Patient", "pt-6", "p", 0), + edgeRow("proj-t", "gen-1", "practitioner/pr-forbidden", "have", "Encounter", "enc-w", "Patient", "pt-7", "p", 0), + edgeRow("proj-t", "gen-1", "practitioner/pr-1", "have", "Encounter", "enc-w2", "Patient", "pt-2", "p", 0), + } + insertOrder := []EdgeOccurrenceRow{ + inScope[2], decoys[0], inScope[0], decoys[1], inScope[4], decoys[2], inScope[1], inScope[3], decoys[3], + } + for _, rec := range insertOrder { + svc.mustInsertEdge(t, edgeRepo, rec) + } + + wantPages := [][]string{{"pt-3", "pt-1"}, {"pt-5", "pt-2"}, {"pt-4"}} + scope := edgeRestrictedScope("proj-t", "gen-1", "practitioner/pr-1") + var walked []EdgeOccurrenceRow + cursor := int64(0) + pages := 0 + for { + rows, hasMore := svc.mustReadFromSource(t, edgeRepo, scope, "Encounter", "enc-w", cursor, 2) + if pages < len(wantPages) { + if len(rows) != len(wantPages[pages]) { + t.Fatalf("page %d returned %d rows, want %d", pages, len(rows), len(wantPages[pages])) + } + for i, row := range rows { + if row.TargetLogicalID != wantPages[pages][i] { + t.Fatalf("page %d position %d returned target %q, want %q: the traversal is not ordered by storage identity", + pages, i, row.TargetLogicalID, wantPages[pages][i]) + } + } + } + pages++ + if pages > 10 { + t.Fatal("source walk did not terminate within 10 pages") + } + for _, row := range rows { + if row.Project != "proj-t" || row.DatasetGeneration != "gen-1" { + t.Fatalf("source traversal crossed project/generation: %+v", row) + } + if row.AuthResourcePath != "practitioner/pr-1" { + t.Fatalf("source traversal crossed auth path: %+v", row) + } + if row.EdgeSeq <= cursor { + t.Fatalf("row %+v did not advance past cursor %d", row, cursor) + } + wantResolved := row.TargetLogicalID == "pt-2" || row.TargetLogicalID == "pt-4" + if row.TargetResolved != wantResolved { + t.Errorf("target %s TargetResolved = %t, want %t", row.TargetLogicalID, row.TargetResolved, wantResolved) + } + } + walked = append(walked, rows...) + if len(rows) > 0 { + cursor = rows[len(rows)-1].EdgeSeq + } + if !hasMore { + break + } + } + if pages != len(wantPages) { + t.Errorf("walked %d pages, want %d (five in-scope rows at limit 2)", pages, len(wantPages)) + } + if len(walked) != 5 { + t.Fatalf("source walk returned %d rows, want 5: %+v", len(walked), walked) + } + + rows, hasMore := svc.mustReadFromSource(t, edgeRepo, scope, "Encounter", "enc-w", cursor, 2) + if len(rows) != 0 || hasMore { + t.Errorf("read past the end returned %d rows (hasMore=%t), want empty", len(rows), hasMore) + } + + rows, hasMore, err := edgeRepo.ReadEdgesFromSource(svc.ctx, edgeRestrictedScope("proj-t", "gen-1"), + "Encounter", "enc-w", 0, 10) + if err != nil || len(rows) != 0 || hasMore { + t.Errorf("restricted-empty source traversal = (%d rows, hasMore=%t, err=%v), want empty with no error", + len(rows), hasMore, err) + } + + rows, hasMore = svc.mustReadFromSource(t, edgeRepo, edgeUnrestrictedScope("proj-t", "gen-1"), + "Encounter", "enc-w", 0, MaxEdgeTraversalLimit) + if hasMore { + t.Fatal("unrestricted traversal reported more rows past a full page") + } + if len(rows) != 6 { + t.Fatalf("unrestricted traversal returned %d rows, want 6 (five plus the forbidden-path row)", len(rows)) + } + for _, row := range rows { + if row.Project != "proj-t" || row.DatasetGeneration != "gen-1" { + t.Fatalf("unrestricted traversal crossed project/generation: %+v", row) + } + } + + rows, hasMore = svc.mustReadToTarget(t, edgeRepo, edgeRestrictedScope("proj-t", "gen-1", "practitioner/pr-1"), + "Patient", "pt-2", 0, 10) + if hasMore { + t.Fatal("target traversal reported more rows past a full page") + } + if len(rows) != 2 { + t.Fatalf("target traversal returned %d rows, want 2: %+v", len(rows), rows) + } + wantSources := []string{"enc-w", "enc-w2"} + for i, row := range rows { + if row.SourceLogicalID != wantSources[i] { + t.Errorf("target traversal position %d came from %q, want %q", i, row.SourceLogicalID, wantSources[i]) + } + if row.TargetResourceType != "Patient" || row.TargetLogicalID != "pt-2" || !row.TargetResolved { + t.Errorf("target traversal row wrong: %+v", row) + } + } + + rows, _ = svc.mustReadToTarget(t, edgeRepo, edgeRestrictedScope("proj-t", "gen-1", "practitioner/pr-1"), + "Patient", "pt-6", 0, 10) + if len(rows) != 0 { + t.Errorf("target traversal resolved a gen-2-only decoy edge into gen-1: %+v", rows) + } + rows, _ = svc.mustReadToTarget(t, edgeRepo, edgeRestrictedScope("proj-x", "gen-1", "practitioner/pr-1"), + "Patient", "x1", 0, 10) + if len(rows) != 1 || rows[0].Project != "proj-x" { + t.Errorf("other project sees %d rows for its own decoy, want exactly 1: %+v", len(rows), rows) + } +} + +func TestEdgeTargetResolutionBindsProjectAndGeneration(t *testing.T) { + svc := newEdgeService(t) + svc.mustMigrateEmbedded(t) + edgeRepo, resourceRepo := svc.newRepositories(t) + + svc.mustSeedResource(t, resourceRepo, "proj-r", "gen-1", "Encounter", "enc-r", "") + rec := edgeRow("proj-r", "gen-1", "", "diagnosis", + "Encounter", "enc-r", "Condition", "cond-r", "diagnosis[0]", 0) + svc.mustInsertEdge(t, edgeRepo, rec) + + svc.mustSeedResource(t, resourceRepo, "proj-r", "gen-2", "Condition", "cond-r", "") + svc.mustSeedResource(t, resourceRepo, "proj-other-r", "gen-1", "Condition", "cond-r", "") + + rows, _ := svc.mustReadFromSource(t, edgeRepo, edgeUnrestrictedScope("proj-r", "gen-1"), + "Encounter", "enc-r", 0, 10) + if len(rows) != 1 { + t.Fatalf("traversal returned %d rows, want 1", len(rows)) + } + if rows[0].TargetResolved { + t.Error("TargetResolved = true while the target exists only under another generation/project") + } + + rows, _ = svc.mustReadToTarget(t, edgeRepo, edgeUnrestrictedScope("proj-r", "gen-1"), + "Condition", "cond-r", 0, 10) + if len(rows) != 1 || rows[0].TargetResolved { + t.Errorf("target-directed traversal returned %+v, want the edge unresolved", rows) + } + + svc.mustSeedResource(t, resourceRepo, "proj-r", "gen-1", "Condition", "cond-r", "") + rows, _ = svc.mustReadToTarget(t, edgeRepo, edgeUnrestrictedScope("proj-r", "gen-1"), + "Condition", "cond-r", 0, 10) + if len(rows) != 1 || !rows[0].TargetResolved { + t.Errorf("target-directed traversal returned %+v, want the edge resolved after the in-scope target insert", rows) + } +} + +func TestEdgeLogicalKeyDistinguishesSQLNullFromEmptyString(t *testing.T) { + svc := newEdgeService(t) + svc.mustMigrateEmbedded(t) + edgeRepo, resourceRepo := svc.newRepositories(t) + + svc.mustSeedResource(t, resourceRepo, "proj-k", "gen-1", "Observation", "obs-k", "") + noKey := edgeRow("proj-k", "gen-1", "", "focus", "Observation", "obs-k", "Patient", "k-1", "focus", 0) + emptyKey := edgeRow("proj-k", "gen-1", "", "focus", "Observation", "obs-k", "Patient", "k-2", "focus", 0) + emptyKey.LogicalEdgeKey = ptr("") + svc.mustInsertEdge(t, edgeRepo, noKey) + svc.mustInsertEdge(t, edgeRepo, emptyKey) + + var noKeyIsNull, emptyKeyIsNull bool + var emptyKeyValue *string + if err := svc.iso.QueryRow(svc.ctx, + `SELECT logical_edge_key IS NULL FROM "`+svc.schema+`".fhir_edge_occurrence + WHERE target_logical_id = 'k-1' AND source_logical_id = 'obs-k'`).Scan(&noKeyIsNull); err != nil { + svc.fail("witness absent logical key", err) + } + if err := svc.iso.QueryRow(svc.ctx, + `SELECT logical_edge_key IS NULL, logical_edge_key FROM "`+svc.schema+`".fhir_edge_occurrence + WHERE target_logical_id = 'k-2' AND source_logical_id = 'obs-k'`).Scan(&emptyKeyIsNull, &emptyKeyValue); err != nil { + svc.fail("witness empty logical key", err) + } + if !noKeyIsNull { + t.Error("absent logical key was stored as a value, want SQL NULL") + } + if emptyKeyIsNull || emptyKeyValue == nil || *emptyKeyValue != "" { + t.Errorf("empty-string key witness = (isNull=%t, value=%v), want a present empty string", emptyKeyIsNull, emptyKeyValue) + } + + rows, _ := svc.mustReadFromSource(t, edgeRepo, edgeUnrestrictedScope("proj-k", "gen-1"), + "Observation", "obs-k", 0, 10) + if len(rows) != 2 { + t.Fatalf("traversal returned %d rows, want 2", len(rows)) + } + for _, row := range rows { + switch row.TargetLogicalID { + case "k-1": + if row.LogicalEdgeKey != nil { + t.Errorf("absent key read back as %q, want nil", *row.LogicalEdgeKey) + } + case "k-2": + if row.LogicalEdgeKey == nil || *row.LogicalEdgeKey != "" { + t.Errorf("empty key read back as %v, want pointer to the empty string", row.LogicalEdgeKey) + } + default: + t.Errorf("unexpected row %+v", row) + } + } +} + +func TestEdgeNoLeftovers(t *testing.T) { + raw := testPostgresURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %v", err) + } + defer admin.Close() + + var leftDatabases []string + rows, err := admin.Query(ctx, + `SELECT datname FROM pg_database WHERE datname LIKE 'pgstore05_it_%' ORDER BY datname`) + if err != nil { + t.Fatalf("probe leftover databases: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("probe leftover databases scan: %v", err) + } + leftDatabases = append(leftDatabases, name) + } + if err := rows.Err(); err != nil { + rows.Close() + t.Fatalf("probe leftover databases rows: %v", err) + } + rows.Close() + for _, name := range leftDatabases { + if !pgstore05EdgeNamePattern.MatchString(name) { + t.Fatalf("leftover database %q does not carry this ticket's prefix; refusing to drop it", name) + } + if _, err := admin.Exec(ctx, `DROP DATABASE IF EXISTS "`+name+`" WITH (FORCE)`); err != nil { + t.Errorf("drop leftover database %s: %v", name, err) + } else { + t.Errorf("dropped leftover database %s; an earlier edge test leaked it", name) + } + } + + var leftSchemas []string + rows, err = admin.Query(ctx, + `SELECT nspname FROM pg_namespace WHERE nspname LIKE 'pgstore05_it_%' ORDER BY nspname`) + if err != nil { + t.Fatalf("probe leftover schemas: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("probe leftover schemas scan: %v", err) + } + leftSchemas = append(leftSchemas, name) + } + if err := rows.Err(); err != nil { + rows.Close() + t.Fatalf("probe leftover schemas rows: %v", err) + } + rows.Close() + for _, name := range leftSchemas { + if !pgstore05EdgeNamePattern.MatchString(name) { + t.Fatalf("leftover schema %q does not carry this ticket's prefix; refusing to drop it", name) + } + if _, err := admin.Exec(ctx, `DROP SCHEMA IF EXISTS "`+name+`" CASCADE`); err != nil { + t.Errorf("drop leftover schema %s: %v", name, err) + } else { + t.Errorf("dropped leftover schema %s; an earlier edge test leaked it", name) + } + } +} + +func ptr[T any](v T) *T { return &v } diff --git a/internal/backends/postgres/internal/store/edge_writer.go b/internal/backends/postgres/internal/store/edge_writer.go new file mode 100644 index 000000000..a95c7d1a1 --- /dev/null +++ b/internal/backends/postgres/internal/store/edge_writer.go @@ -0,0 +1,64 @@ +package postgres + +import ( + "context" + "fmt" +) + +type EdgeBatch struct { + ID string + Project string + DatasetGeneration string + Rows []EdgeOccurrenceRow +} + +type EdgeBatchResult struct { + BatchID string + Digest string + Committed int64 + SkippedIdentical int64 + Replay bool +} + +type EdgeWriter struct { + batch *BatchWriter +} + +func NewEdgeWriter(src ConnSource, schema string) (*EdgeWriter, error) { + batch, err := NewBatchWriter(src, schema) + if err != nil { + return nil, err + } + return &EdgeWriter{batch: batch}, nil +} + +func (w *EdgeWriter) WriteEdgeBatch(ctx context.Context, batch EdgeBatch) (EdgeBatchResult, error) { + if len(batch.Rows) == 0 { + return EdgeBatchResult{}, fmt.Errorf("%w: edge batch %q carries no rows", + ErrInvalidBatch, batch.ID) + } + result, err := w.batch.WriteBatch(ctx, Batch{ + ID: batch.ID, + Project: batch.Project, + DatasetGeneration: batch.DatasetGeneration, + Edges: batch.Rows, + }) + if err != nil { + return EdgeBatchResult{}, err + } + if result.CommittedResources != 0 || result.SkippedIdenticalResources != 0 { + return EdgeBatchResult{}, &BatchAccountingError{ + BatchID: batch.ID, + Side: "resources", + Expected: 0, + Committed: result.CommittedResources, + } + } + return EdgeBatchResult{ + BatchID: result.BatchID, + Digest: result.Digest, + Committed: result.CommittedEdges, + SkippedIdentical: result.SkippedIdenticalEdges, + Replay: result.Replay, + }, nil +} diff --git a/internal/backends/postgres/internal/store/edge_writer_test.go b/internal/backends/postgres/internal/store/edge_writer_test.go new file mode 100644 index 000000000..8c46a264d --- /dev/null +++ b/internal/backends/postgres/internal/store/edge_writer_test.go @@ -0,0 +1,1110 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "net/url" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + ing02StorePrefix = "verify_ing02_store_" + ing02StoreFmt = ing02StorePrefix + "%d_%s" + + ing02StoreProject = "prj-ing02-store" + ing02StoreGen = "gen-build-a" + ing02StoreAuth = "/fhir" +) + +var ing02StoreNamePattern = regexp.MustCompile(`^verify_ing02_store_[0-9]+_[0-9a-f]+$`) + +var ( + ing02StoreOwnedMu sync.Mutex + ing02StoreOwnedByName = map[string]string{} +) + +func ing02StoreRegister(db, schema string) { + ing02StoreOwnedMu.Lock() + defer ing02StoreOwnedMu.Unlock() + ing02StoreOwnedByName[db] = schema +} + +var ( + ing02StoreBaselineOnce sync.Once + ing02StoreBaselineCount int64 + ing02StoreBaselineErr error +) + +func ing02StoreCaptureBaseline(raw string) { + ing02StoreBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + ing02StoreBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_ing02\\_store\\_%' ESCAPE '\\'"). + Scan(&count); err != nil { + ing02StoreBaselineErr = fmt.Errorf("count baseline: %w", err) + return + } + ing02StoreBaselineCount = count + }) +} + +type ing02StoreService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func newIng02StoreService(t *testing.T) *ing02StoreService { + t.Helper() + raw := testPostgresURL(t) + ing02StoreCaptureBaseline(raw) + if ing02StoreBaselineErr != nil { + t.Fatalf("capture pg_database baseline: %v", ing02StoreBaselineErr) + } + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf(ing02StoreFmt, time.Now().UnixNano(), randomHex(t)) + if !ing02StoreNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(ing02StoreFmt, time.Now().UnixNano(), randomHex(t)) + if !ing02StoreNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + ing02StoreRegister(dbName, schema) + return &ing02StoreService{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} +} + +func (s *ing02StoreService) migrate(t *testing.T) { + t.Helper() + if _, err := MigrateSchema(s.ctx, s.openClient(t, 4), s.schema); err != nil { + s.fail("MigrateSchema", err) + } +} + +func (s *ing02StoreService) openClient(t *testing.T, maxConns int32) *Client { + t.Helper() + client, err := Open(s.ctx, Config{ + DSN: s.dsn, + MaxConns: maxConns, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *ing02StoreService) openAppClient(t *testing.T, app string, maxConns int32) *Client { + t.Helper() + u, err := url.Parse(s.dsn) + if err != nil { + s.fail("parse isolated dsn for app client", err) + } + q := u.Query() + q.Set("application_name", app) + u.RawQuery = q.Encode() + client, err := Open(s.ctx, Config{ + DSN: u.String(), + MaxConns: maxConns, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + s.fail("open app client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *ing02StoreService) newWriterOn(t *testing.T, src ConnSource) *EdgeWriter { + t.Helper() + writer, err := NewEdgeWriter(src, s.schema) + if err != nil { + s.fail("new edge writer", err) + } + return writer +} + +func (s *ing02StoreService) newWriter(t *testing.T) *EdgeWriter { + t.Helper() + return s.newWriterOn(t, s.openClient(t, 4)) +} + +func (s *ing02StoreService) qualified(table string) string { + return pgx.Identifier{s.schema, table}.Sanitize() +} + +func (s *ing02StoreService) exec(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := s.iso.Exec(s.ctx, sql, args...); err != nil { + s.fail("fixture exec "+sql, err) + } +} + +func (s *ing02StoreService) mustInt64(t *testing.T, sql string, args ...any) int64 { + t.Helper() + var v int64 + if err := s.iso.QueryRow(s.ctx, sql, args...).Scan(&v); err != nil { + s.fail("witness "+sql, err) + } + return v +} + +func (s *ing02StoreService) seedResource(t *testing.T, project, generation, rtype, lid, authPath string) { + t.Helper() + s.exec(t, "INSERT INTO "+s.qualified("fhir_resource")+ + " (project, dataset_generation, resource_type, logical_id, auth_resource_path, payload)"+ + " VALUES ($1, $2, $3, $4, $5, $6::jsonb)", + project, generation, rtype, lid, authPath, + fmt.Sprintf(`{"resourceType": %q, "id": %q}`, rtype, lid)) +} + +func (s *ing02StoreService) countEdges(t *testing.T, project, generation string) int64 { + t.Helper() + return s.mustInt64(t, "SELECT count(*) FROM "+s.qualified("fhir_edge_occurrence")+ + " WHERE project = $1 AND dataset_generation = $2", project, generation) +} + +func (s *ing02StoreService) witnessLedger(t *testing.T, batchID string) (digest string, edges int64, found bool) { + t.Helper() + err := s.iso.QueryRow(s.ctx, + "SELECT digest, edge_count FROM "+s.qualified("batch_write_ledger")+" WHERE batch_id = $1", batchID). + Scan(&digest, &edges) + if errors.Is(err, pgx.ErrNoRows) { + return "", 0, false + } + if err != nil { + s.fail("witness ledger row", err) + } + return digest, edges, true +} + +func (s *ing02StoreService) requireEdgeRow(t *testing.T, project, generation, authPath, label, + srcType, srcID, tgtType, tgtID, sourcePath string, sourceOrdinal int64, wantKey *string, +) { + t.Helper() + var gotAuth string + var keyPresent bool + var key *string + err := s.iso.QueryRow(s.ctx, + "SELECT auth_resource_path, logical_edge_key IS NOT NULL, logical_edge_key FROM "+s.qualified("fhir_edge_occurrence")+ + " WHERE project = $1 AND dataset_generation = $2 AND source_resource_type = $3"+ + " AND source_logical_id = $4 AND label = $5 AND target_resource_type = $6"+ + " AND target_logical_id = $7 AND source_path = $8 AND source_ordinal = $9", + project, generation, srcType, srcID, label, tgtType, tgtID, sourcePath, sourceOrdinal). + Scan(&gotAuth, &keyPresent, &key) + if err != nil { + s.fail(fmt.Sprintf("witness occurrence %s/%s -%s-> %s/%s at %q#%d", + srcType, srcID, label, tgtType, tgtID, sourcePath, sourceOrdinal), err) + } + if gotAuth != authPath { + t.Fatalf("occurrence %s/%s -%s-> %s/%s at %q#%d auth = %q, want %q", + srcType, srcID, label, tgtType, tgtID, sourcePath, sourceOrdinal, gotAuth, authPath) + } + if (wantKey != nil) != keyPresent { + t.Fatalf("occurrence %s/%s -%s-> %s/%s at %q#%d key presence = %v, want %v", + srcType, srcID, label, tgtType, tgtID, sourcePath, sourceOrdinal, keyPresent, wantKey != nil) + } + if wantKey != nil && *key != *wantKey { + t.Fatalf("occurrence %s/%s -%s-> %s/%s at %q#%d key = %q, want %q", + srcType, srcID, label, tgtType, tgtID, sourcePath, sourceOrdinal, *key, *wantKey) + } +} + +func (s *ing02StoreService) edgeFingerprint(t *testing.T, project, generation string) string { + t.Helper() + var fp string + sql := "SELECT coalesce(md5(string_agg(entry, E'\n' ORDER BY edge_seq)),'EMPTY') FROM (" + + "SELECT edge_seq, source_resource_type || '|' || source_logical_id || '|' || label || '|' ||" + + " target_resource_type || '|' || target_logical_id || '|' || source_path || '|' ||" + + " source_ordinal::text || '|' || auth_resource_path || '|' || coalesce(logical_edge_key, '') AS entry" + + " FROM " + s.qualified("fhir_edge_occurrence") + " WHERE project = $1 AND dataset_generation = $2) x" + if err := s.iso.QueryRow(s.ctx, sql, project, generation).Scan(&fp); err != nil { + s.fail("edge fingerprint", err) + } + return fp +} + +func (s *ing02StoreService) poolBackendPID(t *testing.T, client *Client) int64 { + t.Helper() + conn, err := client.Acquire(s.ctx) + if err != nil { + s.fail("acquire for backend pid", err) + } + defer conn.Release() + var pid int64 + if err := conn.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + s.fail("witness pg_backend_pid", err) + } + return pid +} + +func (s *ing02StoreService) waitBackendWaiting(t *testing.T, app string) int64 { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for { + var pid int64 + var waitType, waitEvent *string + err := s.iso.QueryRow(s.ctx, + "SELECT pid, wait_event_type, wait_event FROM pg_stat_activity"+ + " WHERE application_name = $1 AND state = 'active'", app). + Scan(&pid, &waitType, &waitEvent) + if err == nil && waitType != nil && *waitType == "Lock" { + return pid + } + if time.Now().After(deadline) { + s.fail("waiting for backend "+app+" to block on a lock", + fmt.Errorf("backend never entered a lock wait")) + } + time.Sleep(10 * time.Millisecond) + } +} + +func ing02Edge(label, srcType, srcID, tgtType, tgtID, sourcePath string, ordinal int64, key *string) EdgeOccurrenceRow { + return EdgeOccurrenceRow{ + Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, AuthResourcePath: ing02StoreAuth, + Label: label, SourceResourceType: srcType, SourceLogicalID: srcID, + TargetResourceType: tgtType, TargetLogicalID: tgtID, + SourcePath: sourcePath, SourceOrdinal: ordinal, LogicalEdgeKey: key, + } +} + +func duplicateFocusOccurrences() []EdgeOccurrenceRow { + return []EdgeOccurrenceRow{ + ing02Edge("focus_Observation", "Observation", "obs-1", "Observation", "focus-1", "focus", 0, strPtr("cc574af6-dup-key")), + ing02Edge("focus_Observation", "Observation", "obs-1", "Observation", "focus-1", "focus", 1, strPtr("cc574af6-dup-key")), + ing02Edge("focus_observation", "Observation", "focus-1", "Observation", "obs-1", "focus", 0, strPtr("82685ede-rev-key")), + ing02Edge("subject_Patient", "Observation", "obs-1", "Patient", "patient-missing", "subject", 0, nil), + } +} + +func seedDuplicateFocusScope(t *testing.T, s *ing02StoreService, generation string) { + t.Helper() + s.seedResource(t, ing02StoreProject, generation, "Observation", "obs-1", ing02StoreAuth) + s.seedResource(t, ing02StoreProject, generation, "Observation", "focus-1", ing02StoreAuth) +} + +func TestPostgresEdgeWriterConstructorRejectsInvalidInputs(t *testing.T) { + if _, err := NewEdgeWriter(nil, "ok_schema"); err == nil { + t.Fatal("NewEdgeWriter(nil) accepted a nil connection source") + } + if _, err := NewEdgeWriter(stubConnSource{}, "Ok Schema"); err == nil { + t.Fatal("NewEdgeWriter accepted an invalid schema name") + } else if !errors.Is(err, ErrInvalidSchemaName) { + t.Fatalf("NewEdgeWriter schema error = %v, want ErrInvalidSchemaName", err) + } +} + +type deadConnSource struct{} + +func (deadConnSource) Acquire(ctx context.Context) (*pgxpool.Conn, error) { + return nil, errors.New("deadConnSource must not be contacted") +} + +func TestPostgresEdgeWriterRejectsInvalidBatchesBeforeAnyRoundTrip(t *testing.T) { + writer, err := NewEdgeWriter(deadConnSource{}, "ok_schema") + if err != nil { + t.Fatal(err) + } + ok := ing02Edge("l", "Observation", "obs-1", "Patient", "p-1", "subject", 0, strPtr("k")) + + cases := []struct { + name string + batch EdgeBatch + }{ + {"blank id", EdgeBatch{ID: " ", Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, Rows: []EdgeOccurrenceRow{ok}}}, + {"over-long id", EdgeBatch{ID: strings.Repeat("x", MaxBatchIDLength+1), Project: ing02StoreProject, Rows: []EdgeOccurrenceRow{ok}}}, + {"blank project", EdgeBatch{ID: "b/1", Project: "", Rows: []EdgeOccurrenceRow{ok}}}, + {"no rows", EdgeBatch{ID: "b/1", Project: ing02StoreProject, DatasetGeneration: ing02StoreGen}}, + {"no rows with dead pool", EdgeBatch{ID: "b/1", Project: ing02StoreProject, Rows: []EdgeOccurrenceRow{}}}, + {"over capacity", func() EdgeBatch { + rows := make([]EdgeOccurrenceRow, 0, MaxBatchEdges+1) + for i := 0; i <= MaxBatchEdges; i++ { + rec := ok + rec.SourceOrdinal = int64(i) + rows = append(rows, rec) + } + return EdgeBatch{ID: "b/1", Project: ing02StoreProject, Rows: rows} + }()}, + {"row scope mismatch", func() EdgeBatch { + rec := ok + rec.Project = "other-prj" + return EdgeBatch{ID: "b/1", Project: ing02StoreProject, Rows: []EdgeOccurrenceRow{rec}} + }()}, + {"caller-supplied edge seq", func() EdgeBatch { + rec := ok + rec.EdgeSeq = 9 + return EdgeBatch{ID: "b/1", Project: ing02StoreProject, Rows: []EdgeOccurrenceRow{rec}} + }()}, + {"negative ordinal", func() EdgeBatch { + rec := ok + rec.SourceOrdinal = -1 + return EdgeBatch{ID: "b/1", Project: ing02StoreProject, Rows: []EdgeOccurrenceRow{rec}} + }()}, + {"blank label", func() EdgeBatch { + rec := ok + rec.Label = " " + return EdgeBatch{ID: "b/1", Project: ing02StoreProject, Rows: []EdgeOccurrenceRow{rec}} + }()}, + {"blank target id", func() EdgeBatch { + rec := ok + rec.TargetLogicalID = "" + return EdgeBatch{ID: "b/1", Project: ing02StoreProject, Rows: []EdgeOccurrenceRow{rec}} + }()}, + } + for _, tc := range cases { + _, err := writer.WriteEdgeBatch(context.Background(), tc.batch) + if !errors.Is(err, ErrInvalidBatch) { + t.Errorf("%s: WriteEdgeBatch = %v, want ErrInvalidBatch", tc.name, err) + continue + } + if strings.Contains(fmt.Sprint(err), "deadConnSource") { + t.Errorf("%s: refusal contacted the connection source: %v", tc.name, err) + } + } + + if _, err := writer.WriteEdgeBatch(context.Background(), EdgeBatch{ + ID: "b/valid-looking", Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, + Rows: []EdgeOccurrenceRow{ok}, + }); !strings.Contains(fmt.Sprint(err), "deadConnSource") { + t.Fatalf("valid batch against the dead pool = %v, want the dead-source contact error (proof the validators passed)", err) + } +} + +const ( + ing02DuplicateBatchID = "ing02-store/duplicate/0" + ing02DuplicateBatchDigest = "12fe2c7b603cd41b8f0a32f869738598b3f4aca5406ff5ab34cb1411693e38cb" +) + +func TestPostgresEdgeWriterWritesDuplicateReferenceOccurrencesWithExactColumns(t *testing.T) { + s := newIng02StoreService(t) + s.migrate(t) + seedDuplicateFocusScope(t, s, ing02StoreGen) + writer := s.newWriter(t) + + res, err := writer.WriteEdgeBatch(s.ctx, EdgeBatch{ + ID: ing02DuplicateBatchID, Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, + Rows: duplicateFocusOccurrences(), + }) + if err != nil { + s.fail("WriteEdgeBatch", err) + } + if res.BatchID != ing02DuplicateBatchID || res.Committed != 4 || res.SkippedIdentical != 0 || res.Replay { + t.Fatalf("result = %+v, want a fresh 4-row commit", res) + } + if res.Digest != ing02DuplicateBatchDigest { + t.Fatalf("digest = %s, want the golden %s", res.Digest, ing02DuplicateBatchDigest) + } + + s.requireEdgeRow(t, ing02StoreProject, ing02StoreGen, ing02StoreAuth, + "focus_Observation", "Observation", "obs-1", "Observation", "focus-1", "focus", 0, strPtr("cc574af6-dup-key")) + s.requireEdgeRow(t, ing02StoreProject, ing02StoreGen, ing02StoreAuth, + "focus_Observation", "Observation", "obs-1", "Observation", "focus-1", "focus", 1, strPtr("cc574af6-dup-key")) + s.requireEdgeRow(t, ing02StoreProject, ing02StoreGen, ing02StoreAuth, + "focus_observation", "Observation", "focus-1", "Observation", "obs-1", "focus", 0, strPtr("82685ede-rev-key")) + s.requireEdgeRow(t, ing02StoreProject, ing02StoreGen, ing02StoreAuth, + "subject_Patient", "Observation", "obs-1", "Patient", "patient-missing", "subject", 0, nil) + + if n := s.countEdges(t, ing02StoreProject, ing02StoreGen); n != 4 { + t.Fatalf("edge rows = %d, want 4", n) + } + distinct := s.mustInt64(t, "SELECT count(DISTINCT edge_seq) FROM "+s.qualified("fhir_edge_occurrence")+ + " WHERE project = $1 AND dataset_generation = $2", ing02StoreProject, ing02StoreGen) + if distinct != 4 { + t.Fatalf("distinct edge_seq = %d, want 4 — occurrences must be independently addressable", distinct) + } + + digest, edges, found := s.witnessLedger(t, ing02DuplicateBatchID) + if !found || digest != ing02DuplicateBatchDigest || edges != 4 { + t.Fatalf("marker = (%s, %d, found %v), want the golden digest with 4 committed rows", + digest, edges, found) + } + + mutated := duplicateFocusOccurrences() + mutated[1].SourceOrdinal = 7 + _, err = writer.WriteEdgeBatch(s.ctx, EdgeBatch{ + ID: ing02DuplicateBatchID, Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, + Rows: mutated, + }) + var conflict *BatchIdentityConflictError + if !errors.As(err, &conflict) || !errors.Is(err, ErrBatchIdentityConflict) { + t.Fatalf("mutated replay under the golden identity = %v, want *BatchIdentityConflictError", err) + } + if conflict.LedgerDigest != ing02DuplicateBatchDigest || conflict.ProvidedDigest == ing02DuplicateBatchDigest { + t.Fatalf("conflict digests = (%s, %s), want the golden ledger digest against a different provided digest", + conflict.LedgerDigest, conflict.ProvidedDigest) + } + if n := s.countEdges(t, ing02StoreProject, ing02StoreGen); n != 4 { + t.Fatalf("edge rows after refused mutated replay = %d, want the original 4", n) + } + if conflict.LedgerDigest == conflict.ProvidedDigest { + t.Fatal("identity conflict claimed identical digests") + } +} + +func TestPostgresEdgeWriterEmptyReplayCollapseAndLedgerCountsCommitted(t *testing.T) { + s := newIng02StoreService(t) + s.migrate(t) + seedDuplicateFocusScope(t, s, ing02StoreGen) + writer := s.newWriter(t) + + rows := duplicateFocusOccurrences() + first, err := writer.WriteEdgeBatch(s.ctx, EdgeBatch{ + ID: "ing02-store/collapse/first", Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, Rows: rows, + }) + if err != nil || first.Committed != 4 || first.Replay { + s.fail("first commit", err) + } + before := s.edgeFingerprint(t, ing02StoreProject, ing02StoreGen) + + second, err := writer.WriteEdgeBatch(s.ctx, EdgeBatch{ + ID: "ing02-store/collapse/second", Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, Rows: rows, + }) + if err != nil { + s.fail("collapse batch", err) + } + if second.Committed != 0 || second.SkippedIdentical != 4 || second.Replay { + t.Fatalf("collapse result = %+v, want 0 committed / 4 skipped", second) + } + if _, edges, found := s.witnessLedger(t, "ing02-store/collapse/second"); !found || edges != 0 { + t.Fatalf("collapse marker = edges %d found %v, want 0 committed in the ledger", edges, found) + } + if fp := s.edgeFingerprint(t, ing02StoreProject, ing02StoreGen); fp != before { + t.Fatalf("scope drifted on identical re-offer: %s != %s", fp, before) + } + + for attempt := 0; attempt < 3; attempt++ { + replay, err := writer.WriteEdgeBatch(s.ctx, EdgeBatch{ + ID: "ing02-store/collapse/first", Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, Rows: rows, + }) + if err != nil { + s.fail("replay", err) + } + if !replay.Replay || replay.Digest != first.Digest || replay.Committed != first.Committed || + replay.SkippedIdentical != first.SkippedIdentical || replay.BatchID != first.BatchID { + t.Fatalf("replay %d = %+v, want byte-identical accounting of %+v", attempt, replay, first) + } + } + + s.exec(t, "UPDATE "+s.qualified("batch_write_ledger")+" SET digest = $2 WHERE batch_id = $1", + "ing02-store/collapse/first", strings.Repeat("a", 64)) + _, err = writer.WriteEdgeBatch(s.ctx, EdgeBatch{ + ID: "ing02-store/collapse/first", Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, Rows: rows, + }) + var conflict *BatchIdentityConflictError + if !errors.As(err, &conflict) || conflict.LedgerDigest != strings.Repeat("a", 64) { + t.Fatalf("replay after digest tamper = %v, want *BatchIdentityConflictError against the tampered digest", err) + } + if fp := s.edgeFingerprint(t, ing02StoreProject, ing02StoreGen); fp != before { + t.Fatalf("scope drifted on refused replay: %s != %s", fp, before) + } +} + +func TestPostgresEdgeWriterMissingTargetStaysUnresolvedTypedEndpoint(t *testing.T) { + s := newIng02StoreService(t) + s.migrate(t) + seedDuplicateFocusScope(t, s, ing02StoreGen) + s.seedResource(t, ing02StoreProject, ing02StoreGen, "Patient", "patient-known", ing02StoreAuth) + + writer := s.newWriter(t) + res, err := writer.WriteEdgeBatch(s.ctx, EdgeBatch{ + ID: "ing02-store/unresolved/0", Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, + Rows: []EdgeOccurrenceRow{ + ing02Edge("subject_Patient", "Observation", "obs-1", "Patient", "patient-missing", "subject", 0, strPtr("k-miss")), + ing02Edge("subject_Patient", "Observation", "obs-1", "Patient", "patient-known", "subject", 1, strPtr("k-known")), + }, + }) + if err != nil || res.Committed != 2 { + s.fail("unresolved-target batch", err) + } + + repo, err := NewEdgeRepository(s.openClient(t, 2), s.schema) + if err != nil { + s.fail("new edge repository", err) + } + unrestricted := true + rows, hasMore, err := repo.ReadEdgesFromSource(s.ctx, EdgeScope{ + Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, + AuthResourcePathsUnrestricted: &unrestricted, + }, "Observation", "obs-1", 0, 10) + if err != nil { + s.fail("traverse from source", err) + } + if hasMore { + t.Fatal("hasMore on a two-row scope") + } + if len(rows) != 2 { + t.Fatalf("traversed %d rows, want 2", len(rows)) + } + seen := map[string]bool{} + for _, row := range rows { + seen[row.TargetLogicalID] = true + switch row.TargetLogicalID { + case "patient-missing": + if row.TargetResolved { + t.Fatal("missing target reported resolved — resolution must be a read-time fact") + } + case "patient-known": + if !row.TargetResolved { + t.Fatal("stored target reported unresolved — same-scope fhir_resource must resolve it") + } + } + } + if !seen["patient-missing"] || !seen["patient-known"] { + t.Fatalf("traversal lost an endpoint: %+v", seen) + } +} + +func TestPostgresEdgeWriterMissingSourceRollsBackWholeBatch(t *testing.T) { + s := newIng02StoreService(t) + s.migrate(t) + writer := s.newWriter(t) + + _, err := writer.WriteEdgeBatch(s.ctx, EdgeBatch{ + ID: "ing02-store/missing-source/0", Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, + Rows: duplicateFocusOccurrences(), + }) + var missing *EdgeSourceMissingError + if !errors.As(err, &missing) || !errors.Is(err, ErrEdgeSourceMissing) { + s.fail("missing-source batch", fmt.Errorf("err = %v, want *EdgeSourceMissingError", err)) + } + if missing.SourceResourceType != "Observation" || missing.SourceLogicalID != "obs-1" || + missing.Project != ing02StoreProject || missing.DatasetGeneration != ing02StoreGen { + t.Fatalf("typed missing-source = %+v, want Observation/obs-1 under %s/%s", + missing, ing02StoreProject, ing02StoreGen) + } + if n := s.countEdges(t, ing02StoreProject, ing02StoreGen); n != 0 { + t.Fatalf("edge rows after refused batch = %d, want 0 — persisted counts must not increment", n) + } + if _, _, found := s.witnessLedger(t, "ing02-store/missing-source/0"); found { + t.Fatal("refused batch left a ledger marker") + } + + seedDuplicateFocusScope(t, s, ing02StoreGen) + res, err := writer.WriteEdgeBatch(s.ctx, EdgeBatch{ + ID: "ing02-store/missing-source/control", Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, + Rows: duplicateFocusOccurrences(), + }) + if err != nil || res.Committed != 4 || res.Replay { + s.fail("control batch after seeding", err) + } +} + +func TestPostgresEdgeWriterDuplicateOccurrenceDifferentContentPreservesPrior(t *testing.T) { + s := newIng02StoreService(t) + s.migrate(t) + seedDuplicateFocusScope(t, s, ing02StoreGen) + writer := s.newWriter(t) + + if _, err := writer.WriteEdgeBatch(s.ctx, EdgeBatch{ + ID: "ing02-store/conflict/first", Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, + Rows: duplicateFocusOccurrences(), + }); err != nil { + s.fail("first commit", err) + } + before := s.edgeFingerprint(t, ing02StoreProject, ing02StoreGen) + + conflicting := duplicateFocusOccurrences() + conflicting[0].LogicalEdgeKey = strPtr("tampered-key") + conflicting = append(conflicting, + ing02Edge("component_focus", "Observation", "obs-1", "Observation", "component-1", "component", 0, strPtr("k-new"))) + _, err := writer.WriteEdgeBatch(s.ctx, EdgeBatch{ + ID: "ing02-store/conflict/second", Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, + Rows: conflicting, + }) + var conflict *BatchEdgeConflictError + if !errors.As(err, &conflict) || !errors.Is(err, ErrBatchRowConflict) { + s.fail("content-drift batch", fmt.Errorf("err = %v, want *BatchEdgeConflictError", err)) + } + if conflict.InBatch || conflict.SourceLogicalID != "obs-1" || conflict.SourcePath != "focus" || + conflict.SourceOrdinal != 0 || conflict.Label != "focus_Observation" { + t.Fatalf("conflict = %+v, want stored-row occurrence obs-1 focus#0", conflict) + } + if fp := s.edgeFingerprint(t, ing02StoreProject, ing02StoreGen); fp != before { + t.Fatalf("stored scope drifted after the refused batch: %s != %s", fp, before) + } + if n := s.mustInt64(t, "SELECT count(*) FROM "+s.qualified("fhir_edge_occurrence")+ + " WHERE target_logical_id = 'component-1'"); n != 0 { + t.Fatalf("refused batch leaked its fresh row (%d), want whole-batch rollback", n) + } + if _, _, found := s.witnessLedger(t, "ing02-store/conflict/second"); found { + t.Fatal("refused batch left a ledger marker") + } + s.requireEdgeRow(t, ing02StoreProject, ing02StoreGen, ing02StoreAuth, + "focus_Observation", "Observation", "obs-1", "Observation", "focus-1", "focus", 0, strPtr("cc574af6-dup-key")) +} + +func TestPostgresEdgeWriterInjectedFailureAtEverySeamLeavesNoRowsOrMarker(t *testing.T) { + s := newIng02StoreService(t) + s.migrate(t) + seedDuplicateFocusScope(t, s, ing02StoreGen) + + seams := []string{stepBegin, stepReadLedger, stepClaimLedger, + stepStageEdges, stepCheckEdges, stepPublishEdges, stepFinalizeLedger, stepCommit} + injected := errors.New("injected crash") + for _, seam := range seams { + writer := s.newWriter(t) + fired := false + writer.batch.failHook = func(step string) error { + if step == seam { + fired = true + return injected + } + return nil + } + _, err := writer.WriteEdgeBatch(s.ctx, EdgeBatch{ + ID: "ing02-store/crash/" + seam, Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, + Rows: duplicateFocusOccurrences(), + }) + var writeErr *BatchWriteError + if !fired { + t.Fatalf("seam %s never fired the injected failure", seam) + } + if !errors.As(err, &writeErr) || !errors.Is(err, ErrBatchWrite) || !errors.Is(err, injected) { + t.Fatalf("failure at seam %s = %v, want *BatchWriteError chaining the injected error", seam, err) + } + if n := s.countEdges(t, ing02StoreProject, ing02StoreGen); n != 0 { + t.Fatalf("seam %s: edge rows after injected crash = %d, want 0", seam, n) + } + if _, _, found := s.witnessLedger(t, "ing02-store/crash/"+seam); found { + t.Fatalf("seam %s: injected crash left a ledger marker", seam) + } + } + + writer := s.newWriter(t) + res, err := writer.WriteEdgeBatch(s.ctx, EdgeBatch{ + ID: "ing02-store/crash/" + stepCommit, Project: ing02StoreProject, DatasetGeneration: ing02StoreGen, + Rows: duplicateFocusOccurrences(), + }) + if err != nil { + s.fail("control after crash sweep", err) + } + if res.Committed != 4 || res.Replay { + t.Fatalf("control = %+v, want a fresh 4-row commit on the burned identity", res) + } +} + +func TestPostgresEdgeWriterConcurrentIdenticalBatchCommitsOnceTwoConnections(t *testing.T) { + const rounds = 12 + s := newIng02StoreService(t) + s.migrate(t) + + clientA := s.openAppClient(t, "ing02-store-race-a", 1) + clientB := s.openAppClient(t, "ing02-store-race-b", 1) + pidA := s.poolBackendPID(t, clientA) + pidB := s.poolBackendPID(t, clientB) + if pidA == 0 || pidB == 0 || pidA == pidB { + t.Fatalf("race pools share backend %d/%d, want distinct real connections", pidA, pidB) + } + writerA := s.newWriterOn(t, clientA) + writerB := s.newWriterOn(t, clientB) + + for round := 0; round < rounds; round++ { + generation := fmt.Sprintf("gen-race-%d", round) + seedDuplicateFocusScope(t, s, generation) + rows := duplicateFocusOccurrences() + for i := range rows { + rows[i].DatasetGeneration = generation + } + batchID := fmt.Sprintf("ing02-store/race/%d", round) + + start := make(chan struct{}) + type outcome struct { + res EdgeBatchResult + err error + } + ra := make(chan outcome, 1) + rb := make(chan outcome, 1) + race := func(w *EdgeWriter, out chan outcome) { + <-start + res, err := w.WriteEdgeBatch(context.Background(), EdgeBatch{ + ID: batchID, Project: ing02StoreProject, DatasetGeneration: generation, Rows: rows, + }) + out <- outcome{res, err} + } + go race(writerA, ra) + go race(writerB, rb) + close(start) + o1, o2 := <-ra, <-rb + if o1.err != nil { + s.fail(fmt.Sprintf("round %d connection A", round), o1.err) + } + if o2.err != nil { + s.fail(fmt.Sprintf("round %d connection B", round), o2.err) + } + if o1.res.BatchID != o2.res.BatchID || o1.res.Digest != o2.res.Digest || + o1.res.Committed != o2.res.Committed || o1.res.SkippedIdentical != o2.res.SkippedIdentical { + t.Fatalf("round %d results diverged: %+v vs %+v", round, o1.res, o2.res) + } + if !o1.res.Replay && !o2.res.Replay || o1.res.Replay && o2.res.Replay { + t.Fatalf("round %d replays = (%v, %v), want exactly one fresh commit", round, o1.res.Replay, o2.res.Replay) + } + if o1.res.Committed != 4 || o1.res.Digest == "" { + t.Fatalf("round %d shared accounting = %+v, want 4 committed under a real digest", round, o1.res) + } + if n := s.countEdges(t, ing02StoreProject, generation); n != 4 { + t.Fatalf("round %d rows = %d, want exactly 4 (committed once)", round, n) + } + digest, edges, found := s.witnessLedger(t, batchID) + if !found || digest != o1.res.Digest || edges != 4 { + t.Fatalf("round %d marker = (%s, %d, found %v), want the shared digest with 4 rows", + round, digest, edges, found) + } + } +} + +func TestPostgresEdgeWriterConcurrentOccurrenceContentConflictOneWinsTwoConnections(t *testing.T) { + const rounds = 10 + s := newIng02StoreService(t) + s.migrate(t) + + clientA := s.openAppClient(t, "ing02-store-conf-a", 1) + clientB := s.openAppClient(t, "ing02-store-conf-b", 1) + pidA := s.poolBackendPID(t, clientA) + pidB := s.poolBackendPID(t, clientB) + if pidA == 0 || pidB == 0 || pidA == pidB { + t.Fatalf("race pools share backend %d/%d, want distinct real connections", pidA, pidB) + } + writerA := s.newWriterOn(t, clientA) + writerB := s.newWriterOn(t, clientB) + + for round := 0; round < rounds; round++ { + generation := fmt.Sprintf("gen-conf-%d", round) + seedDuplicateFocusScope(t, s, generation) + mk := func(key string) []EdgeOccurrenceRow { + rows := duplicateFocusOccurrences() + for i := range rows { + rows[i].DatasetGeneration = generation + } + rows[0].LogicalEdgeKey = strPtr(key) + rows[1].LogicalEdgeKey = strPtr(key) + rows[2].LogicalEdgeKey = strPtr(key) + return rows + } + + start := make(chan struct{}) + type outcome struct { + res EdgeBatchResult + err error + } + ra := make(chan outcome, 1) + rb := make(chan outcome, 1) + go func() { + <-start + res, err := writerA.WriteEdgeBatch(context.Background(), EdgeBatch{ + ID: fmt.Sprintf("ing02-store/conflict-a/%d", round), Project: ing02StoreProject, + DatasetGeneration: generation, Rows: mk("winner-key-A"), + }) + ra <- outcome{res, err} + }() + go func() { + <-start + res, err := writerB.WriteEdgeBatch(context.Background(), EdgeBatch{ + ID: fmt.Sprintf("ing02-store/conflict-b/%d", round), Project: ing02StoreProject, + DatasetGeneration: generation, Rows: mk("winner-key-B"), + }) + rb <- outcome{res, err} + }() + close(start) + o1, o2 := <-ra, <-rb + + var loser outcome + var winner outcome + var loserKey string + var winnerKey string + switch { + case o1.err == nil && o2.err != nil: + winner, loser = o1, o2 + winnerKey, loserKey = "winner-key-A", "winner-key-B" + case o2.err == nil && o1.err != nil: + winner, loser = o2, o1 + winnerKey, loserKey = "winner-key-B", "winner-key-A" + default: + s.fail(fmt.Sprintf("round %d", round), + fmt.Errorf("expected exactly one commit, got A=%v B=%v", o1.err, o2.err)) + } + var conflict *BatchEdgeConflictError + var accounting *BatchAccountingError + typedRefusal := (errors.As(loser.err, &conflict) && errors.Is(loser.err, ErrBatchRowConflict) && !conflict.InBatch) || + (errors.As(loser.err, &accounting) && errors.Is(loser.err, ErrBatchAccounting) && accounting.Side == "edges") + if !typedRefusal { + t.Fatalf("round %d loser err = %v, want stored-row *BatchEdgeConflictError or edge-side *BatchAccountingError", + round, loser.err) + } + if winner.res.Committed != 4 || winner.res.Replay { + t.Fatalf("round %d winner = %+v, want a fresh 4-row commit", round, winner.res) + } + _, err := writerA.WriteEdgeBatch(context.Background(), EdgeBatch{ + ID: fmt.Sprintf("ing02-store/conflict-retry/%d", round), Project: ing02StoreProject, + DatasetGeneration: generation, Rows: mk(loserKey), + }) + var retryConflict *BatchEdgeConflictError + if !errors.As(err, &retryConflict) || !errors.Is(err, ErrBatchRowConflict) || retryConflict.InBatch { + t.Fatalf("round %d loser retry = %v, want stored-row *BatchEdgeConflictError", round, err) + } + rows := mk(winnerKey) + for _, row := range rows { + if row.LogicalEdgeKey != nil { + s.requireEdgeRow(t, ing02StoreProject, generation, ing02StoreAuth, + row.Label, row.SourceResourceType, row.SourceLogicalID, + row.TargetResourceType, row.TargetLogicalID, row.SourcePath, row.SourceOrdinal, + row.LogicalEdgeKey) + } + } + if n := s.countEdges(t, ing02StoreProject, generation); n != 4 { + t.Fatalf("round %d rows = %d, want exactly 4 — loser rollback plus refused retry store nothing", round, n) + } + } +} + +func TestPostgresEdgeWriterCountsChangeOnlyAtCommitTwoConnections(t *testing.T) { + s := newIng02StoreService(t) + s.migrate(t) + + writerClient := s.openAppClient(t, "ing02-store-commit-w", 1) + writer := s.newWriterOn(t, writerClient) + writerPID := s.poolBackendPID(t, writerClient) + + observer, err := pgxpool.New(s.ctx, s.dsn) + if err != nil { + s.fail("open observer pool", err) + } + t.Cleanup(observer.Close) + obsConn, err := observer.Acquire(s.ctx) + if err != nil { + s.fail("acquire observer connection", err) + } + defer obsConn.Release() + var observerPID int64 + if err := obsConn.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&observerPID); err != nil { + s.fail("witness observer pid", err) + } + if observerPID == writerPID { + t.Fatalf("observer and writer share backend %d, want two distinct real PG connections", observerPID) + } + + const rounds = 12 + for round := 0; round < rounds; round++ { + generation := fmt.Sprintf("gen-visibility-%d", round) + seedDuplicateFocusScope(t, s, generation) + rows := duplicateFocusOccurrences() + for i := range rows { + rows[i].DatasetGeneration = generation + } + batchID := fmt.Sprintf("ing02-store/visibility/%d", round) + + if _, err := obsConn.Exec(s.ctx, "BEGIN"); err != nil { + s.fail("observer begin", err) + } + if _, err := obsConn.Exec(s.ctx, + "LOCK TABLE "+s.qualified("fhir_edge_occurrence")+" IN ACCESS EXCLUSIVE MODE"); err != nil { + s.fail("observer lock", err) + } + + type outcome struct { + res EdgeBatchResult + err error + } + done := make(chan outcome, 1) + go func() { + res, err := writer.WriteEdgeBatch(context.Background(), EdgeBatch{ + ID: batchID, Project: ing02StoreProject, DatasetGeneration: generation, Rows: rows, + }) + done <- outcome{res, err} + }() + + blockedPID := s.waitBackendWaiting(t, "ing02-store-commit-w") + if blockedPID != writerPID || blockedPID == observerPID { + t.Fatalf("round %d blocked backend pid = %d, want the writer backend %d (observer is %d)", + round, blockedPID, writerPID, observerPID) + } + var midRows int64 + if err := obsConn.QueryRow(s.ctx, "SELECT count(*) FROM "+s.qualified("fhir_edge_occurrence")+ + " WHERE project = $1 AND dataset_generation = $2", ing02StoreProject, generation). + Scan(&midRows); err != nil { + s.fail("mid-transaction row witness", err) + } + if midRows != 0 { + t.Fatalf("round %d rows visible mid-transaction = %d, want 0 — counts may only change at commit", round, midRows) + } + if _, _, found := s.witnessLedger(t, batchID); found { + t.Fatalf("round %d ledger marker visible mid-transaction, want none", round) + } + + if _, err := obsConn.Exec(s.ctx, "COMMIT"); err != nil { + s.fail("observer commit", err) + } + var o outcome + select { + case o = <-done: + case <-time.After(60 * time.Second): + s.fail("waiting for batch commit", fmt.Errorf("round %d batch never completed after the lock released", round)) + } + if o.err != nil { + s.fail(fmt.Sprintf("round %d WriteEdgeBatch", round), o.err) + } + if o.res.Committed != 4 || o.res.Replay { + t.Fatalf("round %d result = %+v, want a fresh 4-row commit", round, o.res) + } + if n := s.countEdges(t, ing02StoreProject, generation); n != o.res.Committed { + t.Fatalf("round %d rows after commit = %d, want the reported %d", round, n, o.res.Committed) + } + if _, edges, found := s.witnessLedger(t, batchID); !found || edges != 4 { + t.Fatalf("round %d marker missing or miscounted after commit (found %v, edges %d)", round, found, edges) + } + } +} + +func TestZZZeroLeftoverEdgeWriterFixtures(t *testing.T) { + raw := testPostgresURL(t) + ing02StoreOwnedMu.Lock() + owned := make(map[string]string, len(ing02StoreOwnedByName)) + for db, schema := range ing02StoreOwnedByName { + owned[db] = schema + } + ing02StoreOwnedMu.Unlock() + if len(owned) == 0 { + t.Skip("this process created no verify_ing02_store_* fixtures") + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service for audit: %v", err) + } + defer admin.Close() + + for db, schema := range owned { + if !strings.HasPrefix(db, ing02StorePrefix) { + t.Fatalf("registry holds foreign database %q", db) + } + var count int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Errorf("audit owned database %s: %v", db, err) + continue + } + if count != 0 { + t.Errorf("leftover owned database %s after all cleanups", db) + if schema != "" { + u, err := url.Parse(raw) + if err == nil { + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + db + if probe, err := pgxpool.New(ctx, u.String()); err == nil { + var schemaCount int + if err := probe.QueryRow(ctx, "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1", schema).Scan(&schemaCount); err != nil { + t.Errorf("audit owned schema %s in %s: %v", schema, db, err) + } else if schemaCount != 0 { + t.Errorf("leftover owned schema %s in leftover database %s", schema, db) + } + probe.Close() + } + } + } + } + } + + var visible int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_ing02\\_store\\_%' ESCAPE '\\'"). + Scan(&visible); err != nil { + t.Fatalf("audit verify_ing02_store_* database count: %v", err) + } + if visible != ing02StoreBaselineCount { + t.Errorf("verify_ing02_store_* databases = %d, want baseline %d", visible, ing02StoreBaselineCount) + } +} diff --git a/internal/backends/postgres/internal/store/errors.go b/internal/backends/postgres/internal/store/errors.go new file mode 100644 index 000000000..b0524a1ef --- /dev/null +++ b/internal/backends/postgres/internal/store/errors.go @@ -0,0 +1,418 @@ +package postgres + +import ( + "context" + "database/sql/driver" + "errors" + "fmt" + "strings" + + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +const ( + sqlStateSerializationFailure = "40001" + sqlStateDeadlockDetected = "40P01" + sqlStateUniqueViolation = "23505" + sqlStateForeignKeyViolation = "23503" + sqlStateUndefinedTable = "42P01" + sqlStateAbortedTransaction = "25P02" + sqlStateQueryCanceled = "57014" + sqlStateAdminShutdown = "57P01" + sqlStateCrashShutdown = "57P02" + sqlStateCannotConnectNow = "57P03" + sqlStateIdleInTxTerminated = "25P01" + sqlStateIdleInTxTerminated18 = "25P03" +) + +type DriverCondition string + +const ( + ConditionConnectionFailure DriverCondition = "connection_failure" + ConditionConnectionLost DriverCondition = "connection_lost" + ConditionAdminShutdown DriverCondition = "admin_shutdown" + ConditionCrashShutdown DriverCondition = "crash_shutdown" + ConditionCannotConnectNow DriverCondition = "cannot_connect_now" + ConditionSessionTerminated DriverCondition = "session_terminated" + ConditionQueryCanceled DriverCondition = "query_canceled" + ConditionSerializationFailure DriverCondition = "serialization_failure" + ConditionDeadlockDetected DriverCondition = "deadlock_detected" + ConditionUniqueViolation DriverCondition = "unique_violation" + ConditionAbortedTransaction DriverCondition = "aborted_transaction" + ConditionTransactionClosed DriverCondition = "transaction_closed" + ConditionRequestCanceled DriverCondition = "request_canceled" + ConditionCommitRejected DriverCondition = "commit_rejected" + ConditionServerRejected DriverCondition = "server_rejected" +) + +var ErrCommitOutcomeUnknown = errors.New("postgres: commit outcome unknown") + +var ErrUniqueConflict = errors.New("postgres: unique conflict") + +type TranslatedError struct { + Condition DriverCondition + SQLState string + AmbiguousCommit bool + + cause error + unwraps []error +} + +func (e *TranslatedError) Error() string { + if e == nil { + return "postgres: translated driver failure" + } + if e.SQLState != "" { + return fmt.Sprintf("postgres: %s (SQLSTATE %s)", e.Condition, e.SQLState) + } + return fmt.Sprintf("postgres: %s", e.Condition) +} + +func (e *TranslatedError) Unwrap() []error { + if e == nil { + return nil + } + return e.unwraps +} + +func (e *TranslatedError) Cause() error { + if e == nil { + return nil + } + return e.cause +} + +type translateSpec struct { + condition DriverCondition + code dataframeerrors.ErrorCode + retryable bool + sentinels []error + ambiguous bool +} + +func newTranslated(spec translateSpec, sqlState string, cause error) *TranslatedError { + typedCause := spec.code == dataframeerrors.CodeBackendUnavailable + var typedWrapped error + if typedCause { + typedWrapped = ErrConnect + } + details := map[string]any{"backend": "postgres", "condition": string(spec.condition)} + if sqlState != "" { + details["sqlstate"] = sqlState + } + typed := dataframeerrors.Wrap(typedWrapped, spec.code, "", + dataframeerrors.WithRetryable(spec.retryable), + dataframeerrors.WithDetails(details), + ) + unwraps := make([]error, 0, len(spec.sentinels)+1) + unwraps = append(unwraps, spec.sentinels...) + unwraps = append(unwraps, typed) + return &TranslatedError{ + Condition: spec.condition, + SQLState: sqlState, + AmbiguousCommit: spec.ambiguous, + cause: cause, + unwraps: unwraps, + } +} + +func unavailableSpec(condition DriverCondition) translateSpec { + return translateSpec{ + condition: condition, + code: dataframeerrors.CodeBackendUnavailable, + retryable: true, + sentinels: []error{ErrConnect}, + } +} + +func internalSpec(condition DriverCondition, sentinels ...error) translateSpec { + return translateSpec{ + condition: condition, + code: dataframeerrors.CodeInternalError, + retryable: false, + sentinels: sentinels, + } +} + +func TranslateError(err error) (translated error, ok bool) { + return translate(err, false) +} + +func TranslateCommitError(err error) (translated error, ok bool) { + return translate(err, true) +} + +func translate(err error, commitPhase bool) (error, bool) { + if err == nil { + return nil, false + } + var already *TranslatedError + if errors.As(err, &already) { + return err, true + } + if _, ok := dataframeerrors.AsUserError(err); ok { + return err, true + } + + if errors.Is(err, pgx.ErrTxCommitRollback) { + return newTranslated(internalSpec(ConditionCommitRejected), "", err), true + } + if errors.Is(err, pgx.ErrTxClosed) { + return newTranslated(internalSpec(ConditionTransactionClosed), "", err), true + } + + if pgErr, ok := pgErrorFrom(err); ok { + if commitPhase && commitOutcomeIsUncertain(pgErr) { + spec := internalSpec(shutdownCondition(pgErr.Code), ErrCommitOutcomeUnknown, ErrConnect) + spec.ambiguous = true + return newTranslated(spec, pgErr.Code, err), true + } + return translateSQLState(pgErr, err) + } + + if isPGTransportFailure(err) { + if !commitPhase && errors.Is(err, context.Canceled) { + spec := translateSpec{ + condition: ConditionRequestCanceled, + code: dataframeerrors.CodeClientCanceled, + retryable: false, + sentinels: []error{ErrConnect, context.Canceled}, + } + return newTranslated(spec, "", err), true + } + condition := ConditionConnectionFailure + if errors.Is(err, pgconn.ErrConnClosed) { + condition = ConditionConnectionLost + } + spec := unavailableSpec(condition) + appendContextSentinels(&spec, err) + if commitPhase { + if pgconn.SafeToRetry(err) { + return newTranslated(spec, "", err), true + } + spec = internalSpec(condition, ErrCommitOutcomeUnknown, ErrConnect) + spec.ambiguous = true + appendContextSentinels(&spec, err) + return newTranslated(spec, "", err), true + } + return newTranslated(spec, "", err), true + } + + return err, false +} + +func pgErrorFrom(err error) (*pgconn.PgError, bool) { + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr == nil { + return nil, false + } + return pgErr, true +} + +func translateSQLState(pgErr *pgconn.PgError, raw error) (error, bool) { + switch { + case strings.HasPrefix(pgErr.Code, "08"): + return newTranslated(unavailableSpec(ConditionConnectionLost), pgErr.Code, raw), true + case pgErr.Code == sqlStateAdminShutdown: + return newTranslated(unavailableSpec(ConditionAdminShutdown), pgErr.Code, raw), true + case pgErr.Code == sqlStateCrashShutdown: + return newTranslated(unavailableSpec(ConditionCrashShutdown), pgErr.Code, raw), true + case pgErr.Code == sqlStateCannotConnectNow: + return newTranslated(unavailableSpec(ConditionCannotConnectNow), pgErr.Code, raw), true + case pgErr.Code == sqlStateIdleInTxTerminated, pgErr.Code == sqlStateIdleInTxTerminated18: + return newTranslated(unavailableSpec(ConditionSessionTerminated), pgErr.Code, raw), true + case pgErr.Code == sqlStateQueryCanceled: + return newTranslated(translateSpec{ + condition: ConditionQueryCanceled, + code: dataframeerrors.CodeQueryResourceLimitExceeded, + retryable: false, + }, pgErr.Code, raw), true + case pgErr.Code == sqlStateSerializationFailure: + return newTranslated(internalSpec(ConditionSerializationFailure), pgErr.Code, raw), true + case pgErr.Code == sqlStateDeadlockDetected: + return newTranslated(internalSpec(ConditionDeadlockDetected), pgErr.Code, raw), true + case pgErr.Code == sqlStateUniqueViolation: + return newTranslated(internalSpec(ConditionUniqueViolation), pgErr.Code, raw), true + case pgErr.Code == sqlStateAbortedTransaction: + return newTranslated(internalSpec(ConditionAbortedTransaction), pgErr.Code, raw), true + default: + return newTranslated(internalSpec(ConditionServerRejected), pgErr.Code, raw), true + } +} + +func commitOutcomeIsUncertain(pgErr *pgconn.PgError) bool { + if isSessionEndSQLState(pgErr.Code) { + return true + } + severity := pgErr.SeverityUnlocalized + if severity == "" { + severity = pgErr.Severity + } + return strings.EqualFold(severity, "FATAL") || strings.EqualFold(severity, "PANIC") +} + +func isSessionEndSQLState(code string) bool { + switch code { + case sqlStateAdminShutdown, sqlStateCrashShutdown, sqlStateCannotConnectNow, + sqlStateIdleInTxTerminated, sqlStateIdleInTxTerminated18: + return true + default: + return strings.HasPrefix(code, "08") + } +} + +func shutdownCondition(code string) DriverCondition { + switch code { + case sqlStateAdminShutdown: + return ConditionAdminShutdown + case sqlStateCrashShutdown: + return ConditionCrashShutdown + case sqlStateCannotConnectNow: + return ConditionCannotConnectNow + case sqlStateIdleInTxTerminated, sqlStateIdleInTxTerminated18: + return ConditionSessionTerminated + default: + if strings.HasPrefix(code, "08") { + return ConditionConnectionLost + } + return ConditionServerRejected + } +} + +func isPGTransportFailure(err error) bool { + var clientConnect *ConnectError + if errors.As(err, &clientConnect) { + return true + } + var driverConnect *pgconn.ConnectError + if errors.As(err, &driverConnect) { + return true + } + if errors.Is(err, ErrConnect) || errors.Is(err, pgconn.ErrConnClosed) || errors.Is(err, driver.ErrBadConn) { + return true + } + return false +} + +func appendContextSentinels(spec *translateSpec, err error) { + if errors.Is(err, context.Canceled) { + spec.sentinels = append(spec.sentinels, context.Canceled) + } + if errors.Is(err, context.DeadlineExceeded) { + spec.sentinels = append(spec.sentinels, context.DeadlineExceeded) + } +} + +func IsUniqueViolation(err error) bool { + if err == nil { + return false + } + var translated *TranslatedError + if errors.As(err, &translated) && translated != nil && translated.Condition == ConditionUniqueViolation { + return true + } + if hasSQLState(err, sqlStateUniqueViolation) { + return true + } + var conflict *UniqueConflictError + return errors.As(err, &conflict) +} + +func IsAbortedTransaction(err error) bool { + if err == nil { + return false + } + var translated *TranslatedError + if errors.As(err, &translated) && translated != nil && translated.Condition == ConditionAbortedTransaction { + return true + } + return hasSQLState(err, sqlStateAbortedTransaction) +} + +func RetryableTransactionConflict(err error) bool { + if err == nil { + return false + } + var translated *TranslatedError + if errors.As(err, &translated) && translated != nil { + switch translated.Condition { + case ConditionSerializationFailure, ConditionDeadlockDetected: + return true + } + } + return hasSQLState(err, sqlStateSerializationFailure) || hasSQLState(err, sqlStateDeadlockDetected) +} + +func IsUniqueConstraintViolation(err error, constraints ...string) bool { + return matchesDriverViolation(err, sqlStateUniqueViolation, constraints) +} + +func IsForeignKeyConstraintViolation(err error, constraints ...string) bool { + return matchesDriverViolation(err, sqlStateForeignKeyViolation, constraints) +} + +func IsUndefinedTable(err error) bool { + pgErr, ok := pgErrorFrom(err) + return ok && pgErr.Code == sqlStateUndefinedTable +} + +func IsDeadlockDetected(err error) bool { + pgErr, ok := pgErrorFrom(err) + return ok && pgErr.Code == sqlStateDeadlockDetected +} + +func matchesDriverViolation(err error, code string, constraints []string) bool { + pgErr, ok := pgErrorFrom(err) + if !ok || pgErr.Code != code || len(constraints) == 0 { + return false + } + for _, name := range constraints { + if pgErr.ConstraintName == name { + return true + } + } + return false +} + +func hasSQLState(err error, code string) bool { + if err == nil { + return false + } + if pgErr, ok := pgErrorFrom(err); ok { + return pgErr.Code == code + } + var translated *TranslatedError + if errors.As(err, &translated) && translated != nil { + if translated.SQLState == code { + return true + } + if causePgErr, ok := pgErrorFrom(translated.cause); ok { + return causePgErr.Code == code + } + } + return false +} + +type UniqueConflictError struct { + cause error +} + +func (e *UniqueConflictError) Error() string { + return fmt.Sprintf("postgres: unique conflict (SQLSTATE %s)", sqlStateUniqueViolation) +} + +func (e *UniqueConflictError) Unwrap() []error { + if e == nil { + return nil + } + return []error{ErrUniqueConflict, e.cause} +} + +func AsUniqueConflict(err error) (*UniqueConflictError, bool) { + if !IsUniqueViolation(err) { + return nil, false + } + return &UniqueConflictError{cause: err}, true +} diff --git a/internal/backends/postgres/internal/store/errors_boundary_test.go b/internal/backends/postgres/internal/store/errors_boundary_test.go new file mode 100644 index 000000000..0a36ab29e --- /dev/null +++ b/internal/backends/postgres/internal/store/errors_boundary_test.go @@ -0,0 +1,221 @@ +package postgres + +import ( + "errors" + "fmt" + "testing" + + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5/pgconn" +) + +func boundaryTypedNilPgErrorChain(t *testing.T) map[string]error { + t.Helper() + typedNil := error((*pgconn.PgError)(nil)) + return map[string]error{ + "bare": typedNil, + "wrapped": fmt.Errorf("driver exec: %w", typedNil), + "double_wrap": fmt.Errorf("outer: %w", fmt.Errorf("inner: %w", typedNil)), + "joined": errors.Join(errors.New("context"), typedNil), + } +} + +func TestSQLStatePredicatesClassifyDriverViolations(t *testing.T) { + uniqueMatching := &pgconn.PgError{Code: "23505", ConstraintName: "fhir_resource_inventory_build_key_ux", Message: "duplicate key"} + uniqueOther := &pgconn.PgError{Code: "23505", ConstraintName: "fhir_resource_inventory_count_key_ux", Message: "duplicate key"} + foreignKey := &pgconn.PgError{Code: "23503", ConstraintName: "explorer_explicit_group_memberships_group_fkey", Message: "violates foreign key"} + foreignKeyOther := &pgconn.PgError{Code: "23503", ConstraintName: "other_fkey", Message: "violates foreign key"} + undefinedTable := &pgconn.PgError{Code: "42P01", Message: `relation "t" does not exist`} + deadlock := &pgconn.PgError{Code: "40P01", Message: "deadlock detected"} + serialization := &pgconn.PgError{Code: "40001", Message: "could not serialize"} + plain := errors.New("connection reset by peer") + + cases := []struct { + name string + err error + got bool + want bool + }{ + {"unique matching constraint", uniqueMatching, + IsUniqueConstraintViolation(uniqueMatching, "fhir_resource_inventory_build_key_ux"), true}, + {"unique matching constraint in list", uniqueMatching, + IsUniqueConstraintViolation(uniqueMatching, "a", "fhir_resource_inventory_build_key_ux"), true}, + {"unique other constraint", uniqueOther, + IsUniqueConstraintViolation(uniqueOther, "fhir_resource_inventory_build_key_ux"), false}, + {"unique no constraints listed", uniqueMatching, IsUniqueConstraintViolation(uniqueMatching), false}, + {"unique wrapped", uniqueMatching, + IsUniqueConstraintViolation(fmt.Errorf("exec: %w", uniqueMatching), "fhir_resource_inventory_build_key_ux"), true}, + {"unique joined", uniqueMatching, + IsUniqueConstraintViolation(errors.Join(errors.New("context"), uniqueMatching), "fhir_resource_inventory_build_key_ux"), true}, + {"unique against foreign key predicate", uniqueMatching, + IsForeignKeyConstraintViolation(uniqueMatching, "fhir_resource_inventory_build_key_ux"), false}, + {"foreign key matching", foreignKey, + IsForeignKeyConstraintViolation(foreignKey, "explorer_explicit_group_memberships_group_fkey"), true}, + {"foreign key other constraint", foreignKeyOther, + IsForeignKeyConstraintViolation(foreignKeyOther, "explorer_explicit_group_memberships_group_fkey"), false}, + {"foreign key against unique predicate", foreignKey, + IsUniqueConstraintViolation(foreignKey, "explorer_explicit_group_memberships_group_fkey"), false}, + {"undefined table bare", undefinedTable, IsUndefinedTable(undefinedTable), true}, + {"undefined table wrapped", undefinedTable, IsUndefinedTable(fmt.Errorf("ledger: %w", undefinedTable)), true}, + {"unique is not undefined table", uniqueMatching, IsUndefinedTable(uniqueMatching), false}, + {"deadlock bare", deadlock, IsDeadlockDetected(deadlock), true}, + {"deadlock wrapped", deadlock, IsDeadlockDetected(fmt.Errorf("publish: %w", deadlock)), true}, + {"serialization is not deadlock", serialization, IsDeadlockDetected(serialization), false}, + {"plain error unique", plain, IsUniqueConstraintViolation(plain, "x"), false}, + {"plain error foreign key", plain, IsForeignKeyConstraintViolation(plain, "x"), false}, + {"plain error undefined table", plain, IsUndefinedTable(plain), false}, + {"plain error deadlock", plain, IsDeadlockDetected(plain), false}, + } + translatedUnique, ok := TranslateError(uniqueMatching) + if !ok { + t.Fatalf("unique violation not translated: %v", uniqueMatching) + } + cases = append(cases, + struct { + name string + err error + got bool + want bool + }{"translated error stays outside raw predicates", translatedUnique, + IsUniqueConstraintViolation(translatedUnique, "fhir_resource_inventory_build_key_ux"), false}, + struct { + name string + err error + got bool + want bool + }{"nil unique predicate", nil, IsUniqueConstraintViolation(nil, "x"), false}, + struct { + name string + err error + got bool + want bool + }{"nil deadlock predicate", nil, IsDeadlockDetected(nil), false}, + ) + for _, tc := range cases { + if tc.got != tc.want { + t.Errorf("%s: got %v, want %v", tc.name, tc.got, tc.want) + } + } +} + +func TestSQLStatePredicatesTypedNilDriverErrorDoesNotPanic(t *testing.T) { + for inputName, input := range boundaryTypedNilPgErrorChain(t) { + input := input + t.Run(inputName+"/unique_constraint", func(t *testing.T) { + rebNoPanic(t, "IsUniqueConstraintViolation", func() { + if IsUniqueConstraintViolation(input, "fhir_resource_inventory_build_key_ux") { + t.Fatal("typed-nil driver error reported as unique constraint violation") + } + }) + }) + t.Run(inputName+"/foreign_key", func(t *testing.T) { + rebNoPanic(t, "IsForeignKeyConstraintViolation", func() { + if IsForeignKeyConstraintViolation(input, "explorer_explicit_group_memberships_group_fkey") { + t.Fatal("typed-nil driver error reported as foreign key violation") + } + }) + }) + t.Run(inputName+"/undefined_table", func(t *testing.T) { + rebNoPanic(t, "IsUndefinedTable", func() { + if IsUndefinedTable(input) { + t.Fatal("typed-nil driver error reported as undefined table") + } + }) + }) + t.Run(inputName+"/deadlock", func(t *testing.T) { + rebNoPanic(t, "IsDeadlockDetected", func() { + if IsDeadlockDetected(input) { + t.Fatal("typed-nil driver error reported as deadlock") + } + }) + }) + t.Run(inputName+"/commit_outcome", func(t *testing.T) { + rebNoPanic(t, "CommitOutcomeUnknown", func() { + if CommitOutcomeUnknown(input) { + t.Fatal("typed-nil driver error reported as uncertain commit outcome") + } + }) + }) + t.Run(inputName+"/verify_statement", func(t *testing.T) { + rebNoPanic(t, "verifyStatementError", func() { + wrapped := verifyStatementError("verify ledger", input) + if wrapped == nil { + t.Fatal("typed-nil driver error classified as no fault") + } + var connect *ConnectError + if !errors.As(wrapped, &connect) { + t.Fatalf("typed-nil driver error passed through as a server reply: %#v", wrapped) + } + }) + }) + t.Run(inputName+"/inventory_write", func(t *testing.T) { + rebNoPanic(t, "inventoryWriteError", func() { + classified := inventoryWriteError("write inventory counts", []catalog.ResourceInventoryCount{{ + BuildID: "b1", Project: "p", DatasetGeneration: "g", ResourceType: "Patient", + AuthResourcePath: "/fhir/Patient", Key: "k", + }}, input) + if classified == nil { + t.Fatal("typed-nil driver error classified as no fault") + } + if errors.Is(classified, ErrInventoryCountConflict) { + t.Fatalf("typed-nil driver error mistaken for an inventory count conflict: %v", classified) + } + }) + }) + } +} + +func TestInventoryWriteErrorKeepsConstraintClassification(t *testing.T) { + chunk := []catalog.ResourceInventoryCount{{ + BuildID: "b1", Project: "p", DatasetGeneration: "g", ResourceType: "Patient", + AuthResourcePath: "/fhir/Patient", Key: "count-key-1", + }} + keyConflict := &pgconn.PgError{Code: "23505", ConstraintName: inventoryCountTableName + "_key_ux", Message: "duplicate key"} + classified := inventoryWriteError("write inventory counts", chunk, keyConflict) + var conflict *InventoryCountConflictError + if !errors.As(classified, &conflict) { + t.Fatalf("key constraint violation not upgraded to conflict error: %v", classified) + } + if conflict.BuildID != "b1" || conflict.Requested != "count-key-1" { + t.Fatalf("conflict error lost row identity: %#v", conflict) + } + otherConstraint := &pgconn.PgError{Code: "23505", ConstraintName: "fhir_resource_pkey", Message: "duplicate key"} + classified = inventoryWriteError("write inventory counts", chunk, otherConstraint) + if !errors.Is(classified, ErrInventoryCountConflict) { + t.Fatalf("identity unique violation not classified: %v", classified) + } + if !errors.Is(classified, otherConstraint) { + t.Fatalf("driver cause not preserved for inspection: %v", classified) + } + plain := errors.New("connection reset by peer") + classified = inventoryWriteError("write inventory counts", chunk, plain) + if errors.Is(classified, ErrInventoryCountConflict) { + t.Fatalf("ordinary error mistaken for a count conflict: %v", classified) + } + if !errors.Is(classified, plain) { + t.Fatalf("ordinary cause not preserved: %v", classified) + } +} + +func TestCommitOutcomeUnknownKeepsDriverClassification(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {"connection exception 08006", &pgconn.PgError{Code: "08006", Message: "connection_failure"}, true}, + {"admin shutdown", &pgconn.PgError{Code: "57P01", Message: "admin_shutdown"}, true}, + {"crash shutdown", &pgconn.PgError{Code: "57P02", Message: "crash_shutdown"}, true}, + {"cannot connect now", &pgconn.PgError{Code: "57P03", Message: "cannot_connect_now"}, false}, + {"unique violation", &pgconn.PgError{Code: "23505", Message: "duplicate key"}, false}, + {"wrapped admin shutdown", fmt.Errorf("commit: %w", &pgconn.PgError{Code: "57P01"}), true}, + {"sentinel", ErrCommitOutcomeUnknown, true}, + {"plain error", errors.New("server rejected query"), false}, + {"nil", nil, false}, + } + for _, tc := range cases { + if got := CommitOutcomeUnknown(tc.err); got != tc.want { + t.Errorf("%s: CommitOutcomeUnknown = %v, want %v", tc.name, got, tc.want) + } + } +} diff --git a/internal/backends/postgres/internal/store/errors_test.go b/internal/backends/postgres/internal/store/errors_test.go new file mode 100644 index 000000000..4cb763f08 --- /dev/null +++ b/internal/backends/postgres/internal/store/errors_test.go @@ -0,0 +1,1863 @@ +package postgres + +import ( + "context" + "crypto/rand" + "database/sql/driver" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strconv" + "strings" + "sync" + "syscall" + "testing" + "time" + + httpapi "github.com/calypr/loom/internal/api/http" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + rebCanarySQL = "SELECT reb_canary_select FROM reb_schema_canary.reb_table_canary WHERE reb_col_canary = 'reb_bind_canary'" + rebCanaryDSN = "postgres://rebuser:rebpassword@rebhost:5432/rebdb" + rebCanaryConstraint = "reb_secret_ux" + rebCanarySchema = "reb_secret_schema" + rebCanaryTable = "reb_secret_table" + rebCanaryDetail = "key (reb_col_canary)=(reb_bind_canary) already exists." +) + +var rebForbidden = []string{ + "reb_canary_select", "reb_schema_canary", "reb_table_canary", + "reb_col_canary", "reb_bind_canary", + "rebuser", "rebpassword", "rebhost", "rebdb", + rebCanaryConstraint, rebCanarySchema, rebCanaryTable, + "already exists", "duplicate key", "canceling statement", + "deadlock detected", "could not serialize", "current transaction is aborted", +} + +func rebSecretPgError(severity, code string) *pgconn.PgError { + return &pgconn.PgError{ + Severity: severity, + SeverityUnlocalized: severity, + Code: code, + Message: fmt.Sprintf("leak bait for %s: duplicate key value violates unique constraint %q", code, rebCanaryConstraint), + Detail: rebCanaryDetail, + Hint: "use " + rebCanarySQL, + InternalQuery: rebCanarySQL, + Where: "reb_frame_function() " + rebCanarySQL, + SchemaName: rebCanarySchema, + TableName: rebCanaryTable, + ColumnName: "reb_col_canary", + ConstraintName: rebCanaryConstraint, + } +} + +func rebHttpJSON(t *testing.T, err error) (string, httpapi.MappedError) { + t.Helper() + mapped := httpapi.MapDataframeError(err, "reb-test") + body, err2 := json.Marshal(mapped.Body) + if err2 != nil { + t.Fatalf("marshal mapped body: %v", err2) + } + return string(body), mapped +} + +func rebAssertSafeRendering(t *testing.T, err error) { + t.Helper() + if err == nil { + t.Fatal("nil error handed to rendering assertion") + } + renderings := map[string]string{ + "Error()": err.Error(), + "%v": fmt.Sprintf("%v", err), + "%+v": fmt.Sprintf("%+v", err), + "%s": fmt.Sprintf("%s", err), + "publicMsg": dataframeerrors.PublicMessage(err), + } + body, _ := rebHttpJSON(t, err) + renderings["httpJSON"] = body + if userErr, ok := dataframeerrors.AsUserError(err); ok { + details, _ := json.Marshal(userErr.Details()) + renderings["details"] = string(details) + renderings["userErrText"] = userErr.Error() + } + for surface, text := range renderings { + for _, forbidden := range rebForbidden { + if strings.Contains(text, forbidden) { + t.Fatalf("%s leaks forbidden token %q: %s", surface, forbidden, text) + } + } + } +} + +func rebUserErr(t *testing.T, err error) dataframeerrors.UserError { + t.Helper() + userErr, ok := dataframeerrors.AsUserError(err) + if !ok { + t.Fatalf("translated error carries no typed Loom error: %#v", err) + } + return userErr +} + +func TestTranslatePostgresErrorNilAndPassThrough(t *testing.T) { + if translated, ok := TranslateError(nil); translated != nil || ok { + t.Fatalf("TranslateError(nil) = (%v, %v), want (nil, false)", translated, ok) + } + + foreign := errors.New("private backend details reb-not-postgres") + if translated, ok := TranslateError(foreign); translated != foreign || ok { + t.Fatalf("TranslateError(foreign) = (%v, %v), want unchanged,false so Arango behavior stays untouched", translated, ok) + } + + typed := dataframeerrors.NewError(dataframeerrors.CodeStaleCursor, "") + if translated, ok := TranslateError(typed); translated != typed || !ok { + t.Fatalf("TranslateError(typed) = (%v, %v), want unchanged,true (composition, never re-wrap)", translated, ok) + } + + first, ok := TranslateError(rebSecretPgError("ERROR", sqlStateDeadlockDetected)) + if !ok { + t.Fatal("first translation refused a PostgreSQL error") + } + if _, isTranslated := first.(*TranslatedError); !isTranslated { + t.Fatalf("first translation produced %T, want *TranslatedError", first) + } + if second, ok := TranslateError(first); second != first || !ok { + t.Fatalf("re-translating a translated error = (%v, %v), want identical pass-through", second, ok) + } + if second, ok := TranslateError(fmt.Errorf("reclassified: %w", first)); second == nil || !ok { + t.Fatalf("re-translating a wrapped translated error = (%v, %v), want accepted unchanged", second, ok) + } +} + +func TestTranslatePostgresErrorPredicatesNonVacuity(t *testing.T) { + uniqueErr, ok := TranslateError(rebSecretPgError("ERROR", sqlStateUniqueViolation)) + if !ok { + t.Fatal("unique violation not translated") + } + deadlockErr, ok := TranslateError(rebSecretPgError("ERROR", sqlStateDeadlockDetected)) + if !ok { + t.Fatal("deadlock not translated") + } + abortedErr, ok := TranslateError(rebSecretPgError("ERROR", sqlStateAbortedTransaction)) + if !ok { + t.Fatal("aborted transaction not translated") + } + + if !IsUniqueViolation(uniqueErr) { + t.Fatal("IsUniqueViolation must recognize the unique violation (positive control)") + } + if IsUniqueViolation(deadlockErr) { + t.Fatal("IsUniqueViolation must reject a deadlock") + } + if IsUniqueViolation(nil) || IsUniqueViolation(errors.New("foreign reb-not-postgres")) { + t.Fatal("IsUniqueViolation must reject nil and foreign errors") + } + + if !IsAbortedTransaction(abortedErr) { + t.Fatal("IsAbortedTransaction must recognize 25P02 (positive control)") + } + if IsAbortedTransaction(uniqueErr) { + t.Fatal("IsAbortedTransaction must reject a unique violation") + } + + if !RetryableTransactionConflict(deadlockErr) { + t.Fatal("RetryableTransactionConflict must recognize the deadlock (positive control)") + } + serialErr, _ := TranslateError(rebSecretPgError("ERROR", sqlStateSerializationFailure)) + if !RetryableTransactionConflict(serialErr) { + t.Fatal("RetryableTransactionConflict must recognize a serialization failure") + } + if RetryableTransactionConflict(uniqueErr) { + t.Fatal("RetryableTransactionConflict must reject a unique violation") + } + if RetryableTransactionConflict(errors.New("foreign reb-not-postgres")) { + t.Fatal("RetryableTransactionConflict must reject foreign errors") + } + + if !RetryableTransactionConflict(fmt.Errorf("seam: %w", rebSecretPgError("ERROR", sqlStateSerializationFailure))) { + t.Fatal("RetryableTransactionConflict must see raw unwrapped-chain SQLSTATEs") + } + + if _, ok := TranslateCommitError(errors.New("foreign reb-not-postgres")); ok { + t.Fatal("TranslateCommitError must reject foreign errors") + } + if _, ok := TranslateCommitError(nil); ok { + t.Fatal("TranslateCommitError(nil) must reject nil") + } +} + +func TestTranslatePostgresErrorConnectionConditions(t *testing.T) { + t.Run("client connect failure", func(t *testing.T) { + raw := &ConnectError{Op: "open", Err: fmt.Errorf("dial rebhost:5432: %w", syscall.ECONNREFUSED)} + translated, ok := TranslateError(raw) + if !ok { + t.Fatal("client ConnectError not translated") + } + translatedErr, isType := translated.(*TranslatedError) + if !isType || translatedErr.Condition != ConditionConnectionFailure { + t.Fatalf("condition = %#v, want connection_failure", translated) + } + if !errors.Is(translated, ErrConnect) { + t.Fatal("translated connect failure must still match ErrConnect (pg-store-01 contract)") + } + userErr := rebUserErr(t, translated) + if userErr.Code() != string(dataframeerrors.CodeBackendUnavailable) || !userErr.Retryable() { + t.Fatalf("connect failure = %s retryable=%v, want retryable BACKEND_UNAVAILABLE", userErr.Code(), userErr.Retryable()) + } + _, mapped := rebHttpJSON(t, translated) + if mapped.Status != 503 || !mapped.Body.Error.Retryable { + t.Fatalf("mapped = %#v, want 503 retryable", mapped) + } + rebAssertSafeRendering(t, translated) + }) + + t.Run("acquire deadline", func(t *testing.T) { + raw := &ConnectError{Op: "acquire", Err: context.DeadlineExceeded} + translated, ok := TranslateError(raw) + if !ok { + t.Fatal("acquire deadline not translated") + } + if !errors.Is(translated, context.DeadlineExceeded) { + t.Fatal("acquire deadline must stay Is-visible for lifecycle classification") + } + if !errors.Is(translated, ErrConnect) { + t.Fatal("translated acquire failure must match ErrConnect") + } + userErr := rebUserErr(t, translated) + if userErr.Code() != string(dataframeerrors.CodeBackendUnavailable) || !userErr.Retryable() { + t.Fatalf("acquire deadline = %s retryable=%v, want retryable BACKEND_UNAVAILABLE", userErr.Code(), userErr.Retryable()) + } + rebAssertSafeRendering(t, translated) + }) + + t.Run("acquire canceled", func(t *testing.T) { + raw := &ConnectError{Op: "acquire", Err: fmt.Errorf("wait: %w", context.Canceled)} + translated, ok := TranslateError(raw) + if !ok { + t.Fatal("canceled acquire not translated") + } + if !errors.Is(translated, context.Canceled) { + t.Fatal("canceled acquire must stay Is-visible as context.Canceled") + } + translatedErr := translated.(*TranslatedError) + if translatedErr.Condition != ConditionRequestCanceled { + t.Fatalf("condition = %s, want request_canceled", translatedErr.Condition) + } + userErr := rebUserErr(t, translated) + if userErr.Code() != string(dataframeerrors.CodeClientCanceled) || userErr.Retryable() { + t.Fatalf("canceled acquire = %s retryable=%v, want non-retryable CLIENT_CANCELED", userErr.Code(), userErr.Retryable()) + } + _, mapped := rebHttpJSON(t, translated) + if mapped.Status != 499 { + t.Fatalf("mapped status = %d, want 499", mapped.Status) + } + rebAssertSafeRendering(t, translated) + }) + + t.Run("driver conn closed", func(t *testing.T) { + raw := fmt.Errorf("stream: %w", pgconn.ErrConnClosed) + translated, ok := TranslateError(raw) + if !ok { + t.Fatal("pgconn.ErrConnClosed not translated") + } + if translatedErr, isType := translated.(*TranslatedError); !isType || translatedErr.Condition != ConditionConnectionLost { + t.Fatalf("condition = %#v, want connection_lost", translated) + } + userErr := rebUserErr(t, translated) + if userErr.Code() != string(dataframeerrors.CodeBackendUnavailable) || !userErr.Retryable() { + t.Fatalf("conn closed = %s retryable=%v, want retryable BACKEND_UNAVAILABLE", userErr.Code(), userErr.Retryable()) + } + rebAssertSafeRendering(t, translated) + }) + + t.Run("bad conn", func(t *testing.T) { + raw := fmt.Errorf("pool: %w", driver.ErrBadConn) + translated, ok := TranslateError(raw) + if !ok { + t.Fatal("driver.ErrBadConn not translated") + } + userErr := rebUserErr(t, translated) + if userErr.Code() != string(dataframeerrors.CodeBackendUnavailable) || !userErr.Retryable() { + t.Fatalf("bad conn = %s retryable=%v, want retryable BACKEND_UNAVAILABLE", userErr.Code(), userErr.Retryable()) + } + rebAssertSafeRendering(t, translated) + }) + + t.Run("pure context errors stay owned sentinels", func(t *testing.T) { + canceled := fmt.Errorf("query: %w", context.Canceled) + if translated, ok := TranslateError(canceled); translated != canceled || ok { + t.Fatalf("pure context.Canceled = (%v, %v), want pass-through", translated, ok) + } + deadline := fmt.Errorf("query: %w", context.DeadlineExceeded) + if translated, ok := TranslateError(deadline); translated != deadline || ok { + t.Fatalf("pure context.DeadlineExceeded = (%v, %v), want pass-through", translated, ok) + } + }) +} + +type rebConditionCase struct { + name string + sqlState string + severity string + wantCond DriverCondition + wantCode string + wantRetry bool + wantHTTP int + predicateOn func(error) bool +} + +func TestTranslatePostgresErrorServerConditions(t *testing.T) { + tests := []rebConditionCase{ + { + name: "query_canceled_57014", sqlState: "57014", severity: "ERROR", + wantCond: ConditionQueryCanceled, wantCode: string(dataframeerrors.CodeQueryResourceLimitExceeded), + wantRetry: false, wantHTTP: 503, + }, + { + name: "serialization_40001", sqlState: "40001", severity: "ERROR", + wantCond: ConditionSerializationFailure, wantCode: string(dataframeerrors.CodeInternalError), + wantRetry: false, wantHTTP: 500, + predicateOn: RetryableTransactionConflict, + }, + { + name: "deadlock_40P01", sqlState: "40P01", severity: "ERROR", + wantCond: ConditionDeadlockDetected, wantCode: string(dataframeerrors.CodeInternalError), + wantRetry: false, wantHTTP: 500, + predicateOn: RetryableTransactionConflict, + }, + { + name: "unique_23505", sqlState: "23505", severity: "ERROR", + wantCond: ConditionUniqueViolation, wantCode: string(dataframeerrors.CodeInternalError), + wantRetry: false, wantHTTP: 500, + predicateOn: IsUniqueViolation, + }, + { + name: "aborted_25P02", sqlState: "25P02", severity: "ERROR", + wantCond: ConditionAbortedTransaction, wantCode: string(dataframeerrors.CodeInternalError), + wantRetry: false, wantHTTP: 500, + predicateOn: IsAbortedTransaction, + }, + { + name: "admin_shutdown_57P01", sqlState: "57P01", severity: "FATAL", + wantCond: ConditionAdminShutdown, wantCode: string(dataframeerrors.CodeBackendUnavailable), + wantRetry: true, wantHTTP: 503, + }, + { + name: "crash_shutdown_57P02", sqlState: "57P02", severity: "FATAL", + wantCond: ConditionCrashShutdown, wantCode: string(dataframeerrors.CodeBackendUnavailable), + wantRetry: true, wantHTTP: 503, + }, + { + name: "cannot_connect_now_57P03", sqlState: "57P03", severity: "FATAL", + wantCond: ConditionCannotConnectNow, wantCode: string(dataframeerrors.CodeBackendUnavailable), + wantRetry: true, wantHTTP: 503, + }, + { + name: "connection_failure_08006", sqlState: "08006", severity: "FATAL", + wantCond: ConditionConnectionLost, wantCode: string(dataframeerrors.CodeBackendUnavailable), + wantRetry: true, wantHTTP: 503, + }, + { + name: "idle_in_tx_25P03", sqlState: "25P03", severity: "FATAL", + wantCond: ConditionSessionTerminated, wantCode: string(dataframeerrors.CodeBackendUnavailable), + wantRetry: true, wantHTTP: 503, + }, + { + name: "unclassified_42601", sqlState: "42601", severity: "ERROR", + wantCond: ConditionServerRejected, wantCode: string(dataframeerrors.CodeInternalError), + wantRetry: false, wantHTTP: 500, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + raw := rebSecretPgError(test.severity, test.sqlState) + wrapped := fmt.Errorf("adapter seam: %w", raw) + translated, ok := TranslateError(wrapped) + if !ok { + t.Fatal("server error not translated") + } + translatedErr, isType := translated.(*TranslatedError) + if !isType { + t.Fatalf("translated type = %T, want *TranslatedError", translated) + } + if translatedErr.Condition != test.wantCond { + t.Fatalf("condition = %s, want %s", translatedErr.Condition, test.wantCond) + } + if translatedErr.SQLState != test.sqlState { + t.Fatalf("sqlstate = %s, want %s", translatedErr.SQLState, test.sqlState) + } + if translatedErr.AmbiguousCommit { + t.Fatal("query-path translation must not flag commit ambiguity") + } + if translatedErr.Cause() != wrapped { + t.Fatal("raw cause must stay available to server-side logging via Cause()") + } + userErr := rebUserErr(t, translated) + if userErr.Code() != test.wantCode || userErr.Retryable() != test.wantRetry { + t.Fatalf("typed = %s retryable=%v, want %s retryable=%v", userErr.Code(), userErr.Retryable(), test.wantCode, test.wantRetry) + } + body, mapped := rebHttpJSON(t, translated) + if mapped.Status != test.wantHTTP { + t.Fatalf("HTTP status = %d, want %d (%s)", mapped.Status, test.wantHTTP, body) + } + if mapped.Body.Error.Retryable != test.wantRetry { + t.Fatalf("HTTP retryable = %v, want %v (no blanket retry)", mapped.Body.Error.Retryable, test.wantRetry) + } + if mapped.Body.Error.Details["backend"] != "postgres" { + t.Fatalf("details lack the postgres backend tag: %#v", mapped.Body.Error.Details) + } + if mapped.Body.Error.Details["condition"] != string(test.wantCond) { + t.Fatalf("details condition = %#v, want %s", mapped.Body.Error.Details["condition"], test.wantCond) + } + if translatedErr.SQLState != "" && mapped.Body.Error.Details["sqlstate"] != test.sqlState { + t.Fatalf("details sqlstate = %#v, want %s", mapped.Body.Error.Details["sqlstate"], test.sqlState) + } + if test.predicateOn != nil && !test.predicateOn(translated) { + t.Fatalf("dedicated predicate rejected its own condition") + } + rebAssertSafeRendering(t, translated) + + rawBody, rawMapped := rebHttpJSON(t, wrapped) + if rawMapped.Body.Error.Code != "INTERNAL_ERROR" { + t.Fatalf("raw untranslated mapped = %s, want INTERNAL_ERROR redaction", rawMapped.Body.Error.Code) + } + for _, forbidden := range rebForbidden { + if strings.Contains(rawBody, forbidden) { + t.Fatalf("raw redaction leaked %q in %s", forbidden, rawBody) + } + } + }) + } +} + +func TestTranslatePostgresErrorUniqueConflictIsTargeted(t *testing.T) { + raw := rebSecretPgError("ERROR", sqlStateUniqueViolation) + + translated, ok := TranslateError(raw) + if !ok { + t.Fatal("unique violation not translated") + } + if _, isConflict := translated.(*UniqueConflictError); isConflict { + t.Fatal("generic translation must not produce a domain conflict") + } + userErr := rebUserErr(t, translated) + if strings.Contains(userErr.Code(), "CONFLICT") { + t.Fatalf("generic translation must not advertise a conflict code, got %s", userErr.Code()) + } + + conflict, ok := AsUniqueConflict(fmt.Errorf("cas insert: %w", raw)) + if !ok || conflict == nil { + t.Fatal("targeted CAS path must obtain the unique conflict") + } + if !errors.Is(conflict, ErrUniqueConflict) { + t.Fatal("conflict must match ErrUniqueConflict") + } + rebAssertSafeRendering(t, conflict) + + if _, ok := AsUniqueConflict(rebSecretPgError("ERROR", sqlStateDeadlockDetected)); ok { + t.Fatal("AsUniqueConflict must refuse a deadlock") + } + if _, ok := AsUniqueConflict(errors.New("foreign reb-not-postgres")); ok { + t.Fatal("AsUniqueConflict must refuse foreign errors") + } +} + +type rebSafeToRetryError struct{ error } + +func (rebSafeToRetryError) SafeToRetry() bool { return true } + +func TestTranslatePostgresErrorLocalizedSeverityKeepsCommitAmbiguous(t *testing.T) { + tests := []struct { + name string + severity string + unlocalized string + sqlState string + wantCondition DriverCondition + }{ + {"german_fatal", "SCHWERWIEGEND", "FATAL", sqlStateAdminShutdown, ConditionAdminShutdown}, + {"plain_fatal", "FATAL", "FATAL", sqlStateAdminShutdown, ConditionAdminShutdown}, + {"localized_severity_with_panic_v", "PANIK", "PANIC", sqlStateAdminShutdown, ConditionAdminShutdown}, + {"unlocalized_missing_falls_back", "SCHWERWIEGEND", "", sqlStateAdminShutdown, ConditionAdminShutdown}, + {"fatal_v_on_internal_error_state", "SCHWERWIEGEND", "FATAL", "XX001", ConditionServerRejected}, + {"english_severity_fallback", "FATAL", "", "XX001", ConditionServerRejected}, + {"session_end_57P01_nonfatal_severity", "SCHWERWIEGEND", "ERROR", sqlStateAdminShutdown, ConditionAdminShutdown}, + {"session_end_08006_nonfatal_severity", "FEHLER", "ERROR", "08006", ConditionConnectionLost}, + {"session_end_25P03_nonfatal_severity", "FEHLER", "ERROR", sqlStateIdleInTxTerminated18, ConditionSessionTerminated}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pgErr := rebSecretPgError(test.severity, test.sqlState) + pgErr.SeverityUnlocalized = test.unlocalized + translated, ok := TranslateCommitError(fmt.Errorf("commit: %w", pgErr)) + if !ok { + t.Fatal("localized commit failure not translated") + } + translatedErr := translated.(*TranslatedError) + if !translatedErr.AmbiguousCommit || !errors.Is(translated, ErrCommitOutcomeUnknown) { + t.Fatalf("localized severity defeated the ambiguous-COMMIT guarantee: %#v", translatedErr) + } + if translatedErr.Condition != test.wantCondition { + t.Fatalf("condition = %s, want %s", translatedErr.Condition, test.wantCondition) + } + if userErr := rebUserErr(t, translated); userErr.Retryable() { + t.Fatalf("localized FATAL severity produced a retryable advertisement for an unconfirmed commit: %#v", userErr) + } + _, mapped := rebHttpJSON(t, translated) + if mapped.Body.Error.Retryable { + t.Fatal("HTTP must not advertise retry for a localized FATAL mid-commit failure") + } + rebAssertSafeRendering(t, translated) + }) + } + pgErr := rebSecretPgError("FEHLER", sqlStateUniqueViolation) + pgErr.SeverityUnlocalized = "ERROR" + translated, ok := TranslateCommitError(fmt.Errorf("commit: %w", pgErr)) + if !ok { + t.Fatal("localized commit ERROR not translated") + } + if translatedErr := translated.(*TranslatedError); translatedErr.AmbiguousCommit || errors.Is(translated, ErrCommitOutcomeUnknown) { + t.Fatalf("localized ERROR severity wrongly flagged ambiguous: %#v", translatedErr) + } +} + +var ( + rebTypedNilPgErr *pgconn.PgError + rebTypedNilTranslated *TranslatedError +) + +func rebNoPanic(t *testing.T, what string, fn func()) { + t.Helper() + defer func() { + if r := recover(); r != nil { + t.Fatalf("%s panicked: %v", what, r) + } + }() + fn() +} + +func TestTranslatePostgresErrorTypedNilDriverErrorDoesNotPanic(t *testing.T) { + typedNil := error(rebTypedNilPgErr) + inputs := map[string]func() error{ + "bare": func() error { return typedNil }, + "wrapped": func() error { return fmt.Errorf("driver exec: %w", typedNil) }, + "double_wrap": func() error { return fmt.Errorf("outer: %w", fmt.Errorf("inner: %w", typedNil)) }, + } + for inputName, build := range inputs { + input := build() + t.Run(inputName+"/translate", func(t *testing.T) { + rebNoPanic(t, "TranslateError", func() { + translated, ok := TranslateError(input) + if ok { + t.Fatalf("typed-nil driver error classified as a server condition: %#v", translated) + } + if translated != input { + t.Fatalf("typed-nil driver error not passed through unchanged: %#v", translated) + } + }) + }) + t.Run(inputName+"/translate_commit", func(t *testing.T) { + rebNoPanic(t, "TranslateCommitError", func() { + if _, ok := TranslateCommitError(input); ok { + t.Fatal("typed-nil driver error classified at commit phase") + } + }) + }) + t.Run(inputName+"/is_unique_violation", func(t *testing.T) { + rebNoPanic(t, "IsUniqueViolation", func() { + if IsUniqueViolation(input) { + t.Fatal("typed-nil driver error reported as unique violation") + } + }) + }) + t.Run(inputName+"/is_aborted_transaction", func(t *testing.T) { + rebNoPanic(t, "IsAbortedTransaction", func() { + if IsAbortedTransaction(input) { + t.Fatal("typed-nil driver error reported as aborted transaction") + } + }) + }) + t.Run(inputName+"/retryable_transaction_conflict", func(t *testing.T) { + rebNoPanic(t, "RetryableTransactionConflict", func() { + if RetryableTransactionConflict(input) { + t.Fatal("typed-nil driver error reported as retryable transaction conflict") + } + }) + }) + t.Run(inputName+"/as_unique_conflict", func(t *testing.T) { + rebNoPanic(t, "AsUniqueConflict", func() { + if conflict, ok := AsUniqueConflict(input); ok { + t.Fatalf("typed-nil driver error upgraded to domain conflict: %#v", conflict) + } + }) + }) + t.Run(inputName+"/has_sqlstate", func(t *testing.T) { + rebNoPanic(t, "hasSQLState", func() { + for _, code := range []string{sqlStateUniqueViolation, sqlStateAdminShutdown, "08006", "XX001"} { + if hasSQLState(input, code) { + t.Fatalf("typed-nil driver error reported SQLSTATE %s", code) + } + } + }) + }) + } +} + +func TestTranslatePostgresErrorExoticChainInputsDoNotPanic(t *testing.T) { + typedNilTranslated := error(rebTypedNilTranslated) + typedNilWrapped := fmt.Errorf("exec: %w", rebTypedNilPgErr) + + t.Run("typed_nil_translated_predicate_fields", func(t *testing.T) { + for _, input := range []error{typedNilTranslated, fmt.Errorf("outer: %w", typedNilTranslated)} { + rebNoPanic(t, "predicates over typed-nil TranslatedError", func() { + if IsUniqueViolation(input) || IsAbortedTransaction(input) || RetryableTransactionConflict(input) { + t.Fatal("typed-nil TranslatedError claimed a condition it does not carry") + } + if _, ok := AsUniqueConflict(input); ok { + t.Fatal("typed-nil TranslatedError upgraded to domain conflict") + } + if hasSQLState(input, sqlStateUniqueViolation) { + t.Fatal("typed-nil TranslatedError reported a SQLSTATE") + } + }) + } + }) + + t.Run("translated_with_typed_nil_cause", func(t *testing.T) { + translated := &TranslatedError{ + Condition: ConditionUniqueViolation, + SQLState: "99999", + cause: error(rebTypedNilPgErr), + } + rebNoPanic(t, "hasSQLState over typed-nil cause", func() { + if hasSQLState(translated, sqlStateUniqueViolation) { + t.Fatal("typed-nil cause supplied SQLSTATE 23505") + } + if hasSQLState(translated, "99999") != true { + t.Fatal("translated SQLState field stopped matching its own value") + } + if !IsUniqueViolation(translated) { + t.Fatal("translated condition field stopped driving its predicate") + } + }) + }) + + t.Run("conflict_wrapper_over_typed_nil_driver_error", func(t *testing.T) { + conflict := &UniqueConflictError{cause: typedNilWrapped} + rebNoPanic(t, "predicates over conflict wrapper on typed-nil", func() { + if !IsUniqueViolation(conflict) { + t.Fatal("domain conflict marker no longer satisfies IsUniqueViolation") + } + if hasSQLState(conflict, sqlStateUniqueViolation) { + t.Fatal("typed-nil driver error inside conflict wrapper supplied SQLSTATE 23505") + } + }) + }) + + t.Run("translated_error_still_translates_with_typed_nil_in_chain", func(t *testing.T) { + real, _ := TranslateError(rebSecretPgError("ERROR", sqlStateUniqueViolation)) + chain := fmt.Errorf("wrapped: %w and %v", real, typedNilWrapped) + rebNoPanic(t, "TranslateError over mixed chain", func() { + if out, ok := TranslateError(chain); !ok || out != chain { + t.Fatalf("idempotence broken on mixed chain: ok=%v out=%#v", ok, out) + } + }) + }) +} + +func TestTranslatePostgresErrorIsAsNeverReachesRawDriverCause(t *testing.T) { + cases := []struct { + name string + build func() (translated error, raw *pgconn.PgError) + }{ + {"query_translation", func() (error, *pgconn.PgError) { + raw := rebSecretPgError("ERROR", sqlStateUniqueViolation) + out, _ := TranslateError(raw) + return out, raw + }}, + {"ambiguous_commit", func() (error, *pgconn.PgError) { + raw := rebSecretPgError("FATAL", sqlStateAdminShutdown) + out, _ := TranslateCommitError(fmt.Errorf("commit: %w", raw)) + return out, raw + }}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + translated, raw := test.build() + var pgErr *pgconn.PgError + rebNoPanic(t, "errors.As for driver error", func() { + if errors.As(translated, &pgErr) { + t.Fatalf("errors.As reached the raw driver cause: %#v", pgErr) + } + if pgErr != nil { + t.Fatalf("errors.As wrote a driver pointer without matching: %#v", pgErr) + } + }) + if errors.Is(translated, raw) { + t.Fatal("errors.Is reached the raw driver cause identity") + } + wrapped := fmt.Errorf("outer: %w", translated) + if errors.As(wrapped, &pgErr) || errors.Is(wrapped, raw) { + t.Fatal("raw driver cause reachable through a further wrapper") + } + if _, ok := dataframeerrors.AsUserError(translated); !ok { + t.Fatal("curated typed error no longer reachable through the chain") + } + if test.name == "query_translation" && !IsUniqueViolation(translated) { + t.Fatal("translated condition became unreachable through predicates") + } + if test.name == "ambiguous_commit" && !errors.Is(translated, ErrConnect) { + t.Fatal("curated ErrConnect sentinel became unreachable") + } + rebAssertSafeRendering(t, translated) + }) + } +} + +func TestTranslatePostgresErrorPanicSeverityIsIntrinsicallyAmbiguousCommit(t *testing.T) { + tests := []struct { + name string + severity string + unlocalized string + sqlState string + wantCondition DriverCondition + }{ + {"panic_v_on_internal_error", "KRITISCH", "PANIC", "XX001", ConditionServerRejected}, + {"panic_v_on_unique_violation", "FEHLER", "PANIC", sqlStateUniqueViolation, ConditionServerRejected}, + {"panic_v_on_serialization", "FEHLER", "PANIC", sqlStateSerializationFailure, ConditionServerRejected}, + {"panic_s_fallback_without_v", "PANIC", "", "42P01", ConditionServerRejected}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pgErr := rebSecretPgError(test.severity, test.sqlState) + pgErr.SeverityUnlocalized = test.unlocalized + translated, ok := TranslateCommitError(fmt.Errorf("commit: %w", pgErr)) + if !ok { + t.Fatal("PANIC-severity commit failure not translated") + } + translatedErr := translated.(*TranslatedError) + if !translatedErr.AmbiguousCommit || !errors.Is(translated, ErrCommitOutcomeUnknown) { + t.Fatalf("PANIC severity on a non-session-death SQLSTATE was treated as a definitive commit rejection: %#v", translatedErr) + } + if translatedErr.Condition != test.wantCondition { + t.Fatalf("condition = %s, want %s", translatedErr.Condition, test.wantCondition) + } + if translatedErr.SQLState != test.sqlState { + t.Fatalf("SQLState = %s, want %s", translatedErr.SQLState, test.sqlState) + } + if userErr := rebUserErr(t, translated); userErr.Retryable() { + t.Fatal("PANIC-severity ambiguous commit advertised retry") + } + _, mapped := rebHttpJSON(t, translated) + if mapped.Body.Error.Retryable { + t.Fatal("HTTP advertised retry for a PANIC-severity ambiguous commit") + } + rebAssertSafeRendering(t, translated) + }) + } + pgErr := rebSecretPgError("FEHLER", "XX001") + pgErr.SeverityUnlocalized = "ERROR" + translated, ok := TranslateCommitError(fmt.Errorf("commit: %w", pgErr)) + if !ok { + t.Fatal("ERROR-severity commit failure not translated") + } + if translatedErr := translated.(*TranslatedError); translatedErr.AmbiguousCommit { + t.Fatalf("non-fatal severity wrongly flagged ambiguous commit: %#v", translatedErr) + } +} + +func TestTranslatePostgresErrorClass08CommitAlwaysAmbiguous(t *testing.T) { + tests := []struct { + name string + sqlState string + severity string + unlocalized string + }{ + {"08000_nonfatal", "08000", "FEHLER", "ERROR"}, + {"08001_nonfatal", "08001", "FEHLER", "ERROR"}, + {"08003_nonfatal", "08003", "WARNUNG", "WARNING"}, + {"08006_nonfatal", "08006", "FEHLER", "ERROR"}, + {"08006_fatal", "08006", "FATAL", "FATAL"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pgErr := rebSecretPgError(test.severity, test.sqlState) + pgErr.SeverityUnlocalized = test.unlocalized + translated, ok := TranslateCommitError(fmt.Errorf("commit: %w", pgErr)) + if !ok { + t.Fatal("class-08 commit failure not translated") + } + translatedErr := translated.(*TranslatedError) + if !translatedErr.AmbiguousCommit || !errors.Is(translated, ErrCommitOutcomeUnknown) { + t.Fatalf("class-08 %s at COMMIT was treated as definitive: %#v", test.sqlState, translatedErr) + } + if translatedErr.Condition != ConditionConnectionLost { + t.Fatalf("condition = %s, want connection_lost", translatedErr.Condition) + } + if userErr := rebUserErr(t, translated); userErr.Retryable() { + t.Fatalf("class-08 ambiguous commit advertised retry: %#v", userErr) + } + _, mapped := rebHttpJSON(t, translated) + if mapped.Body.Error.Retryable { + t.Fatal("HTTP advertised retry for an ambiguous class-08 commit") + } + rebAssertSafeRendering(t, translated) + }) + } + pgErr := rebSecretPgError("FEHLER", "XX001") + pgErr.SeverityUnlocalized = "ERROR" + translated, ok := TranslateCommitError(fmt.Errorf("commit: %w", pgErr)) + if !ok { + t.Fatal("control commit failure not translated") + } + if translatedErr := translated.(*TranslatedError); translatedErr.AmbiguousCommit { + t.Fatalf("non-08 ERROR severity wrongly flagged ambiguous: %#v", translatedErr) + } +} + +func TestTranslatePostgresErrorCommitClassification(t *testing.T) { + t.Run("fatal mid-commit is ambiguous", func(t *testing.T) { + raw := fmt.Errorf("commit: %w", rebSecretPgError("FATAL", sqlStateAdminShutdown)) + translated, ok := TranslateCommitError(raw) + if !ok { + t.Fatal("commit FATAL not translated") + } + translatedErr := translated.(*TranslatedError) + if !translatedErr.AmbiguousCommit { + t.Fatal("FATAL at commit must flag the outcome as ambiguous") + } + if !errors.Is(translated, ErrCommitOutcomeUnknown) { + t.Fatal("ambiguous commit must match ErrCommitOutcomeUnknown") + } + if translatedErr.Condition != ConditionAdminShutdown || translatedErr.SQLState != sqlStateAdminShutdown { + t.Fatalf("condition = %s/%s, want admin_shutdown/57P01", translatedErr.Condition, translatedErr.SQLState) + } + userErr := rebUserErr(t, translated) + if userErr.Retryable() { + t.Fatal("an ambiguous commit must never be retryable") + } + _, mapped := rebHttpJSON(t, translated) + if mapped.Body.Error.Retryable || mapped.Status != 500 { + t.Fatalf("ambiguous commit mapped = %#v, want non-retryable 500", mapped) + } + for _, forbidden := range []string{"rolled back", "rollback confirmed", "not written", "no write"} { + if strings.Contains(strings.ToLower(translated.Error()), forbidden) { + t.Fatalf("ambiguous commit rendering claims %q: %s", forbidden, translated.Error()) + } + } + rebAssertSafeRendering(t, translated) + }) + + t.Run("server-answered commit error is definitive", func(t *testing.T) { + raw := fmt.Errorf("commit: %w", rebSecretPgError("ERROR", sqlStateUniqueViolation)) + translated, ok := TranslateCommitError(raw) + if !ok { + t.Fatal("commit ERROR not translated") + } + translatedErr := translated.(*TranslatedError) + if translatedErr.AmbiguousCommit { + t.Fatal("a server-answered commit must not be flagged ambiguous") + } + if errors.Is(translated, ErrCommitOutcomeUnknown) { + t.Fatal("a server-answered commit must not claim unknown outcome") + } + if translatedErr.Condition != ConditionUniqueViolation { + t.Fatalf("condition = %s, want unique_violation", translatedErr.Condition) + } + rebAssertSafeRendering(t, translated) + }) + + t.Run("driver-confirmed rollback is definitive", func(t *testing.T) { + translated, ok := TranslateCommitError(fmt.Errorf("commit: %w", pgx.ErrTxCommitRollback)) + if !ok { + t.Fatal("ErrTxCommitRollback not translated") + } + translatedErr := translated.(*TranslatedError) + if translatedErr.Condition != ConditionCommitRejected || translatedErr.AmbiguousCommit { + t.Fatalf("commit rollback = %#v, want definitive commit_rejected", translatedErr) + } + rebAssertSafeRendering(t, translated) + }) + + t.Run("tx closed is definitive", func(t *testing.T) { + translated, ok := TranslateCommitError(fmt.Errorf("commit: %w", pgx.ErrTxClosed)) + if !ok { + t.Fatal("ErrTxClosed not translated") + } + if translatedErr := translated.(*TranslatedError); translatedErr.Condition != ConditionTransactionClosed { + t.Fatalf("condition = %s, want transaction_closed", translatedErr.Condition) + } + rebAssertSafeRendering(t, translated) + }) + + t.Run("pre-send transport failure is definitive not-sent", func(t *testing.T) { + raw := &ConnectError{Op: "acquire", Err: rebSafeToRetryError{errors.New("rebhost:5432 refused")}} + translated, ok := TranslateCommitError(raw) + if !ok { + t.Fatal("pre-send connect failure not translated") + } + translatedErr := translated.(*TranslatedError) + if translatedErr.AmbiguousCommit { + t.Fatal("a failure the driver guarantees occurred before send cannot be ambiguous") + } + userErr := rebUserErr(t, translated) + if userErr.Code() != string(dataframeerrors.CodeBackendUnavailable) || !userErr.Retryable() { + t.Fatalf("pre-send failure = %s retryable=%v, want retryable BACKEND_UNAVAILABLE", userErr.Code(), userErr.Retryable()) + } + rebAssertSafeRendering(t, translated) + }) + + t.Run("post-send transport failure is ambiguous", func(t *testing.T) { + raw := fmt.Errorf("commit flush: %w", pgconn.ErrConnClosed) + translated, ok := TranslateCommitError(raw) + if !ok { + t.Fatal("post-send conn closed not translated") + } + translatedErr := translated.(*TranslatedError) + if !translatedErr.AmbiguousCommit || !errors.Is(translated, ErrCommitOutcomeUnknown) { + t.Fatalf("post-send failure = %#v, want ambiguous with ErrCommitOutcomeUnknown", translatedErr) + } + if translatedErr.Condition != ConditionConnectionLost { + t.Fatalf("condition = %s, want connection_lost", translatedErr.Condition) + } + _, mapped := rebHttpJSON(t, translated) + if mapped.Body.Error.Retryable { + t.Fatal("ambiguous commit must never advertise retry") + } + rebAssertSafeRendering(t, translated) + }) + + t.Run("canceled commit is ambiguous not client-canceled", func(t *testing.T) { + raw := &ConnectError{Op: "query", Err: fmt.Errorf("flush: %w", context.Canceled)} + translated, ok := TranslateCommitError(raw) + if !ok { + t.Fatal("canceled commit not translated") + } + translatedErr := translated.(*TranslatedError) + if !translatedErr.AmbiguousCommit { + t.Fatal("a commit canceled mid-flight has an unknown outcome") + } + if translatedErr.Condition == ConditionRequestCanceled { + t.Fatal("commit phase must not present caller-cancellation semantics over the unknown write") + } + if !errors.Is(translated, context.Canceled) { + t.Fatal("the caller's cancellation must stay Is-visible") + } + rebAssertSafeRendering(t, translated) + }) +} + +func TestTranslatePostgresErrorConcurrentTranslationSharesNoState(t *testing.T) { + sharedRaw := []error{ + rebSecretPgError("ERROR", sqlStateDeadlockDetected), + rebSecretPgError("FATAL", sqlStateAdminShutdown), + &ConnectError{Op: "acquire", Err: context.DeadlineExceeded}, + fmt.Errorf("stream: %w", pgconn.ErrConnClosed), + } + want := make([]string, len(sharedRaw)) + for i, raw := range sharedRaw { + translated, ok := TranslateError(raw) + if !ok { + t.Fatalf("baseline translation %d refused", i) + } + want[i] = fmt.Sprintf("%s|%s", translated, translated.(*TranslatedError).Condition) + } + + const goroutines = 8 + var wg sync.WaitGroup + results := make([][]string, goroutines) + for worker := 0; worker < goroutines; worker++ { + wg.Add(1) + go func(worker int) { + defer wg.Done() + rounds := make([]string, len(sharedRaw)) + for round := 0; round < 25; round++ { + for i, raw := range sharedRaw { + translated, ok := TranslateError(raw) + if !ok { + results[worker] = []string{fmt.Sprintf("worker %d refused input %d", worker, i)} + return + } + _ = RetryableTransactionConflict(translated) + _ = IsUniqueViolation(translated) + _ = httpapi.MapDataframeError(translated, "race") + rounds[i] = fmt.Sprintf("%s|%s", translated, translated.(*TranslatedError).Condition) + } + } + results[worker] = rounds + }(worker) + } + wg.Wait() + for worker, rounds := range results { + if len(rounds) != len(want) { + t.Fatalf("worker %d: %v", worker, rounds) + } + for i := range want { + if rounds[i] != want[i] { + t.Fatalf("worker %d input %d: %s, want %s (shared-state interference)", worker, i, rounds[i], want[i]) + } + } + } +} + +const ( + rebFixturePrefix = "verify_reb_" + rebFixtureNameFmt = rebFixturePrefix + "%d_%s" +) + +var rebFixturePattern = regexp.MustCompile(`^verify_reb_[0-9]+_[0-9a-f]+$`) + +var ( + rebOwnedMu sync.Mutex + rebOwnedByName = map[string]string{} +) + +func rebRegister(db, schema string) { + rebOwnedMu.Lock() + defer rebOwnedMu.Unlock() + rebOwnedByName[db] = schema +} + +var ( + rebBaselineOnce sync.Once + rebBaselineCount int64 + rebBaselineErr error +) + +func rebCaptureBaseline(raw string) { + rebBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + rebBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_reb\\_%' ESCAPE '\\'"). + Scan(&count); err != nil { + rebBaselineErr = fmt.Errorf("count baseline: %w", err) + return + } + rebBaselineCount = count + }) +} + +func rebURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 error-boundary acceptance tests") + } + return raw +} + +type rebService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + schema string + dsn string + scrub func(any) string + fail func(step string, err error) + canaries []string +} + +func newRebService(t *testing.T) *rebService { + t.Helper() + raw := rebURL(t) + rebCaptureBaseline(raw) + if rebBaselineErr != nil { + t.Fatalf("capture pg_database baseline: %v", rebBaselineErr) + } + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, + "SELECT current_setting('server_version_num')::int8, current_setting('server_version')"). + Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if err := CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required backend: %v", err) + } + + dbName := fmt.Sprintf(rebFixtureNameFmt, time.Now().UnixNano(), rebRandomHex(t)) + if !rebFixturePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(rebFixtureNameFmt, time.Now().UnixNano(), rebRandomHex(t)) + if !rebFixturePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + rebRegister(dbName, schema) + return &rebService{ctx: ctx, admin: admin, iso: iso, schema: schema, dsn: dsn, scrub: scrub, fail: fail} +} + +func rebRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +func (s *rebService) rebQualified(table string) string { + return strconv.Quote(s.schema) + "." + table +} + +func (s *rebService) rebScrubbedRendering(t *testing.T, where string, err error) { + t.Helper() + forbidden := append([]string{}, s.canaries...) + forbidden = append(forbidden, s.dsn, "postgres://", "postgresql://") + if u, uerr := url.Parse(s.dsn); uerr == nil && u.User != nil { + forbidden = append(forbidden, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + forbidden = append(forbidden, pw) + } + } + renderings := map[string]string{ + "Error()": err.Error(), + "%v": fmt.Sprintf("%v", err), + } + body, _ := rebHttpJSON(t, err) + renderings["httpJSON"] = body + if userErr, ok := dataframeerrors.AsUserError(err); ok { + details, _ := json.Marshal(userErr.Details()) + renderings["details"] = string(details) + } + for surface, text := range renderings { + for _, token := range forbidden { + if token != "" && strings.Contains(text, token) { + t.Fatalf("%s: %s leaks a forbidden canary token (%d chars); rendered=%s", + where, surface, len(token), strings.ReplaceAll(s.scrub(text), token, "")) + } + } + } +} + +func rebWitnessSQLState(t *testing.T, err error) string { + t.Helper() + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + t.Fatalf("failure is not a server SQLSTATE error: %v", err) + } + return pgErr.Code +} + +func (s *rebService) client(t *testing.T, maxConns int32) *Client { + t.Helper() + client, err := Open(s.ctx, Config{ + DSN: s.dsn, + MaxConns: maxConns, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + s.fail("open store client", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *rebService) acquire(t *testing.T, client *Client) *pgxpool.Conn { + t.Helper() + conn, err := client.Acquire(s.ctx) + if err != nil { + s.fail("acquire", err) + } + t.Cleanup(conn.Release) + return conn +} + +func TestTranslatePostgresErrorRealServerConditions(t *testing.T) { + t.Run("query_canceled_57014", func(t *testing.T) { + s := newRebService(t) + s.canaries = []string{s.schema} + client := s.client(t, 1) + conn := s.acquire(t, client) + if _, err := conn.Exec(s.ctx, "SET statement_timeout = '80ms'"); err != nil { + s.fail("set statement_timeout", err) + } + _, err := conn.Exec(s.ctx, "SELECT pg_sleep(3)") + if err == nil { + t.Fatal("statement_timeout did not fire: non-vacuity control broken") + } + if code := rebWitnessSQLState(t, err); code != sqlStateQueryCanceled { + t.Fatalf("witnessed SQLSTATE %s, want 57014", code) + } + translated, ok := TranslateError(err) + if !ok { + t.Fatal("57014 not translated") + } + translatedErr := translated.(*TranslatedError) + if translatedErr.Condition != ConditionQueryCanceled { + t.Fatalf("condition = %s, want query_canceled", translatedErr.Condition) + } + userErr := rebUserErr(t, translated) + if userErr.Code() != string(dataframeerrors.CodeQueryResourceLimitExceeded) || userErr.Retryable() { + t.Fatalf("typed = %s retryable=%v, want non-retryable QUERY_RESOURCE_LIMIT_EXCEEDED", userErr.Code(), userErr.Retryable()) + } + s.rebScrubbedRendering(t, "57014", translated) + var one int + if err := conn.QueryRow(s.ctx, "SELECT 1").Scan(&one); err != nil || one != 1 { + t.Fatalf("post-cancel control failed: %v", err) + } + }) + + t.Run("serialization_failure_40001", func(t *testing.T) { + s := newRebService(t) + table := `"reb_ssi_canary_table"` + s.canaries = []string{s.schema, "reb_ssi_canary_table"} + if _, err := s.iso.Exec(s.ctx, "CREATE TABLE "+s.rebQualified(table)+" (id int)"); err != nil { + s.fail("create ssi table", err) + } + if _, err := s.iso.Exec(s.ctx, + "INSERT INTO "+s.rebQualified(table)+" SELECT generate_series(1,100)"); err != nil { + s.fail("seed ssi table", err) + } + tx1, err := s.iso.BeginTx(s.ctx, pgx.TxOptions{IsoLevel: pgx.Serializable}) + if err != nil { + s.fail("begin tx1", err) + } + tx2, err := s.iso.BeginTx(s.ctx, pgx.TxOptions{IsoLevel: pgx.Serializable}) + if err != nil { + s.fail("begin tx2", err) + } + var sum int + if err := tx1.QueryRow(s.ctx, "SELECT sum(id) FROM "+s.rebQualified(table)+" WHERE id <= 50").Scan(&sum); err != nil { + s.fail("tx1 read", err) + } + if err := tx2.QueryRow(s.ctx, "SELECT sum(id) FROM "+s.rebQualified(table)+" WHERE id > 50").Scan(&sum); err != nil { + s.fail("tx2 read", err) + } + if _, err := tx1.Exec(s.ctx, "UPDATE "+s.rebQualified(table)+" SET id = id + 100 WHERE id > 50"); err != nil { + s.fail("tx1 write", err) + } + if _, err := tx2.Exec(s.ctx, "UPDATE "+s.rebQualified(table)+" SET id = id - 100 WHERE id <= 50"); err != nil { + s.fail("tx2 write", err) + } + e1 := tx1.Commit(s.ctx) + e2 := tx2.Commit(s.ctx) + victims := []error{} + for _, e := range []error{e1, e2} { + if e == nil { + continue + } + if code := rebWitnessSQLState(t, e); code != sqlStateSerializationFailure { + t.Fatalf("SSI loser failed with SQLSTATE %s, want 40001", code) + } + victims = append(victims, e) + } + if len(victims) != 1 { + t.Fatalf("expected exactly one SSI loser, got %d (errors: %v / %v)", len(victims), s.scrub(e1), s.scrub(e2)) + } + loser := victims[0] + translated, ok := TranslateError(loser) + if !ok { + t.Fatal("40001 not translated") + } + translatedErr := translated.(*TranslatedError) + if translatedErr.Condition != ConditionSerializationFailure { + t.Fatalf("condition = %s, want serialization_failure", translatedErr.Condition) + } + userErr := rebUserErr(t, translated) + if userErr.Retryable() { + t.Fatal("serialization failure must not be blanket-retryable") + } + if !RetryableTransactionConflict(translated) { + t.Fatal("operation policy must still recognize the retryable conflict") + } + _, mapped := rebHttpJSON(t, translated) + if mapped.Body.Error.Retryable { + t.Fatal("HTTP must not advertise retry for 40001") + } + s.rebScrubbedRendering(t, "40001", translated) + + var shifted int + if err := s.iso.QueryRow(s.ctx, + "SELECT count(*) FROM "+s.rebQualified(table)+" WHERE id > 100").Scan(&shifted); err != nil { + s.fail("witness ssi", err) + } + winnerLow := e1 == nil + if (shifted == 50) != winnerLow { + t.Fatalf("witness shifted=%d, winner-low=%v: concurrent outcome not witnessed", shifted, winnerLow) + } + }) + + t.Run("deadlock_40P01", func(t *testing.T) { + s := newRebService(t) + table := `"reb_deadlock_canary_table"` + s.canaries = []string{s.schema, "reb_deadlock_canary_table"} + if _, err := s.iso.Exec(s.ctx, + "CREATE TABLE "+s.rebQualified(table)+" (id int PRIMARY KEY, v int)"); err != nil { + s.fail("create deadlock table", err) + } + if _, err := s.iso.Exec(s.ctx, + "INSERT INTO "+s.rebQualified(table)+" VALUES (1,0),(2,0)"); err != nil { + s.fail("seed deadlock table", err) + } + tx1, err := s.iso.BeginTx(s.ctx, pgx.TxOptions{}) + if err != nil { + s.fail("begin tx1", err) + } + tx2, err := s.iso.BeginTx(s.ctx, pgx.TxOptions{}) + if err != nil { + s.fail("begin tx2", err) + } + if _, err := tx1.Exec(s.ctx, "UPDATE "+s.rebQualified(table)+" SET v = v + 1 WHERE id = 1"); err != nil { + s.fail("tx1 first lock", err) + } + if _, err := tx2.Exec(s.ctx, "UPDATE "+s.rebQualified(table)+" SET v = v + 1 WHERE id = 2"); err != nil { + s.fail("tx2 first lock", err) + } + blocked := make(chan error, 1) + go func() { + _, err := tx1.Exec(s.ctx, "UPDATE "+s.rebQualified(table)+" SET v = v + 1 WHERE id = 2") + blocked <- err + }() + time.Sleep(400 * time.Millisecond) + _, e2 := tx2.Exec(s.ctx, "UPDATE "+s.rebQualified(table)+" SET v = v + 1 WHERE id = 1") + e1 := <-blocked + var loser error + for _, e := range []error{e1, e2} { + if e == nil { + continue + } + if code := rebWitnessSQLState(t, e); code != sqlStateDeadlockDetected { + t.Fatalf("deadlock loser failed with SQLSTATE %s, want 40P01", code) + } + loser = e + } + if loser == nil { + t.Fatal("no deadlock victim: non-vacuity control broken") + } + translated, ok := TranslateError(loser) + if !ok { + t.Fatal("40P01 not translated") + } + translatedErr := translated.(*TranslatedError) + if translatedErr.Condition != ConditionDeadlockDetected { + t.Fatalf("condition = %s, want deadlock_detected", translatedErr.Condition) + } + if rebUserErr(t, translated).Retryable() { + t.Fatal("deadlock must not be blanket-retryable") + } + if !RetryableTransactionConflict(translated) { + t.Fatal("policy predicate must recognize the deadlock") + } + s.rebScrubbedRendering(t, "40P01", translated) + + var winner, loserTx pgx.Tx + if e1 == nil { + winner, loserTx = tx1, tx2 + } else { + winner, loserTx = tx2, tx1 + } + if err := winner.Commit(s.ctx); err != nil { + s.fail("winner commit", err) + } + _ = loserTx.Rollback(s.ctx) + var row1, row2 int + if err := s.iso.QueryRow(s.ctx, + "SELECT v FROM "+s.rebQualified(table)+" WHERE id = 1").Scan(&row1); err != nil { + s.fail("witness row1", err) + } + if err := s.iso.QueryRow(s.ctx, + "SELECT v FROM "+s.rebQualified(table)+" WHERE id = 2").Scan(&row2); err != nil { + s.fail("witness row2", err) + } + if row1 != 1 || row2 != 1 { + t.Fatalf("witness (row1,row2)=(%d,%d), want (1,1): winner +1/+1, victim 0/0", row1, row2) + } + }) + + t.Run("unique_violation_23505", func(t *testing.T) { + s := newRebService(t) + table := `"reb_unique_canary_table"` + constraint := "reb_unique_canary_ux" + s.canaries = []string{s.schema, "reb_unique_canary_table", constraint, "reb_dup_key_canary"} + if _, err := s.iso.Exec(s.ctx, "CREATE TABLE "+s.rebQualified(table)+ + " (id text CONSTRAINT "+constraint+" UNIQUE)"); err != nil { + s.fail("create unique table", err) + } + if _, err := s.iso.Exec(s.ctx, + "INSERT INTO "+s.rebQualified(table)+" VALUES ('reb_dup_key_canary')"); err != nil { + s.fail("seed unique table", err) + } + _, err := s.iso.Exec(s.ctx, "INSERT INTO "+s.rebQualified(table)+" VALUES ('reb_dup_key_canary')") + if err == nil { + t.Fatal("duplicate key accepted: non-vacuity control broken") + } + if code := rebWitnessSQLState(t, err); code != sqlStateUniqueViolation { + t.Fatalf("witnessed SQLSTATE %s, want 23505", code) + } + translated, ok := TranslateError(err) + if !ok { + t.Fatal("23505 not translated") + } + translatedErr := translated.(*TranslatedError) + if translatedErr.Condition != ConditionUniqueViolation { + t.Fatalf("condition = %s, want unique_violation", translatedErr.Condition) + } + userErr := rebUserErr(t, translated) + if userErr.Code() != string(dataframeerrors.CodeInternalError) || userErr.Retryable() { + t.Fatalf("generic 23505 = %s retryable=%v, want non-retryable INTERNAL_ERROR (no unproven domain conflict)", + userErr.Code(), userErr.Retryable()) + } + s.rebScrubbedRendering(t, "23505 generic", translated) + + conflict, ok := AsUniqueConflict(err) + if !ok || conflict == nil { + t.Fatal("targeted insert path must obtain the proven conflict") + } + s.rebScrubbedRendering(t, "23505 conflict", conflict) + + if _, err := s.iso.Exec(s.ctx, + "INSERT INTO "+s.rebQualified(table)+" VALUES ('reb_other_key_canary')"); err != nil { + t.Fatalf("control insert refused: %s", s.scrub(err)) + } + }) + + t.Run("aborted_transaction_25P02", func(t *testing.T) { + s := newRebService(t) + table := `"reb_abort_canary_table"` + s.canaries = []string{s.schema, "reb_abort_canary_table", "reb_after_abort_canary"} + if _, err := s.iso.Exec(s.ctx, + "CREATE TABLE "+s.rebQualified(table)+" (id text PRIMARY KEY)"); err != nil { + s.fail("create abort table", err) + } + client := s.client(t, 1) + conn := s.acquire(t, client) + tx, err := conn.BeginTx(s.ctx, pgx.TxOptions{}) + if err != nil { + s.fail("begin tx", err) + } + if _, err := tx.Exec(s.ctx, "INSERT INTO "+s.rebQualified(table)+" VALUES ('reb_first_canary')"); err != nil { + s.fail("first insert", err) + } + if _, err := tx.Exec(s.ctx, "INSERT INTO "+s.rebQualified(table)+" VALUES ('reb_first_canary')"); err == nil { + t.Fatal("duplicate accepted inside tx: non-vacuity control broken") + } + _, err = tx.Exec(s.ctx, "INSERT INTO "+s.rebQualified(table)+" VALUES ('reb_after_abort_canary')") + if err == nil { + t.Fatal("statement after abort accepted: non-vacuity control broken") + } + if code := rebWitnessSQLState(t, err); code != sqlStateAbortedTransaction { + t.Fatalf("witnessed SQLSTATE %s, want 25P02", code) + } + translated, ok := TranslateError(err) + if !ok { + t.Fatal("25P02 not translated") + } + translatedErr := translated.(*TranslatedError) + if translatedErr.Condition != ConditionAbortedTransaction { + t.Fatalf("condition = %s, want aborted_transaction", translatedErr.Condition) + } + if !IsAbortedTransaction(translated) { + t.Fatal("policy predicate must recognize the aborted transaction") + } + if strings.Contains(strings.ToLower(translated.Error()), "rollback") { + t.Fatalf("25P02 rendering claims rollback: %s", translated.Error()) + } + s.rebScrubbedRendering(t, "25P02", translated) + if err := tx.Rollback(s.ctx); err != nil { + s.fail("explicit rollback", err) + } + var count int + if err := s.iso.QueryRow(s.ctx, + "SELECT count(*) FROM "+s.rebQualified(table)).Scan(&count); err != nil { + s.fail("witness abort", err) + } + if count != 0 { + t.Fatalf("rows after explicit rollback = %d, want 0", count) + } + }) + + t.Run("admin_shutdown_57P01", func(t *testing.T) { + s := newRebService(t) + s.canaries = []string{s.schema} + client := s.client(t, 2) + victim := s.acquire(t, client) + var pid int64 + if err := victim.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + s.fail("read victim pid", err) + } + if _, err := s.admin.Exec(s.ctx, "SELECT pg_terminate_backend($1)", pid); err != nil { + s.fail("terminate victim", err) + } + time.Sleep(300 * time.Millisecond) + var one int + err := victim.QueryRow(s.ctx, "SELECT pg_sleep(5), 1").Scan(new(string), &one) + if err == nil { + t.Fatal("terminated session kept answering: non-vacuity control broken") + } + if code := rebWitnessSQLState(t, err); code != sqlStateAdminShutdown { + t.Fatalf("witnessed SQLSTATE %s, want 57P01", code) + } + translated, ok := TranslateError(err) + if !ok { + t.Fatal("57P01 not translated") + } + translatedErr := translated.(*TranslatedError) + if translatedErr.Condition != ConditionAdminShutdown { + t.Fatalf("condition = %s, want admin_shutdown", translatedErr.Condition) + } + userErr := rebUserErr(t, translated) + if userErr.Code() != string(dataframeerrors.CodeBackendUnavailable) || !userErr.Retryable() { + t.Fatalf("admin shutdown = %s retryable=%v, want retryable BACKEND_UNAVAILABLE", userErr.Code(), userErr.Retryable()) + } + s.rebScrubbedRendering(t, "57P01", translated) + fresh := s.acquire(t, client) + if err := fresh.QueryRow(s.ctx, "SELECT 1").Scan(&one); err != nil || one != 1 { + t.Fatalf("fresh session failed: %s", s.scrub(err)) + } + }) + + t.Run("connection_lost", func(t *testing.T) { + s := newRebService(t) + s.canaries = []string{s.schema} + client := s.client(t, 2) + victim := s.acquire(t, client) + var pid int64 + if err := victim.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + s.fail("read victim pid", err) + } + if _, err := s.admin.Exec(s.ctx, "SELECT pg_terminate_backend($1)", pid); err != nil { + s.fail("terminate victim", err) + } + time.Sleep(300 * time.Millisecond) + var sink int + _ = victim.QueryRow(s.ctx, "SELECT 1").Scan(&sink) + err := victim.QueryRow(s.ctx, "SELECT 1").Scan(&sink) + if err == nil { + t.Fatal("dead connection still answering: non-vacuity control broken") + } + if !errors.Is(err, pgconn.ErrConnClosed) { + t.Fatalf("dead connection must surface pgconn.ErrConnClosed, got %s", s.scrub(err)) + } + translated, ok := TranslateError(err) + if !ok { + t.Fatal("dead-connection failure not translated") + } + translatedErr := translated.(*TranslatedError) + if translatedErr.Condition != ConditionConnectionLost { + t.Fatalf("condition = %s, want connection_lost", translatedErr.Condition) + } + userErr := rebUserErr(t, translated) + if userErr.Code() != string(dataframeerrors.CodeBackendUnavailable) || !userErr.Retryable() { + t.Fatalf("connection lost = %s retryable=%v, want retryable BACKEND_UNAVAILABLE", userErr.Code(), userErr.Retryable()) + } + s.rebScrubbedRendering(t, "connection_lost", translated) + }) + + t.Run("connection_refused", func(t *testing.T) { + s := newRebService(t) + u, err := url.Parse(s.dsn) + if err != nil { + s.fail("parse dsn", err) + } + u.Host = "127.0.0.1:59999" + _, err = Open(s.ctx, Config{ + DSN: u.String(), + MaxConns: 1, + MinConns: 0, + AcquireTimeout: 5 * time.Second, + }) + if err == nil { + t.Fatal("open against a closed port succeeded: non-vacuity control broken") + } + if !IsRefusedConnection(err) { + t.Fatalf("expected a refused connection, got %s", s.scrub(err)) + } + translated, ok := TranslateError(err) + if !ok { + t.Fatal("connect refused not translated") + } + if !errors.Is(translated, ErrConnect) { + t.Fatal("translated refused must match ErrConnect (pg-store-01 contract)") + } + userErr := rebUserErr(t, translated) + if userErr.Code() != string(dataframeerrors.CodeBackendUnavailable) || !userErr.Retryable() { + t.Fatalf("refused = %s retryable=%v, want retryable BACKEND_UNAVAILABLE", userErr.Code(), userErr.Retryable()) + } + s.rebScrubbedRendering(t, "refused", translated) + if control := s.client(t, 1); control == nil { + t.Fatal("control Open failed") + } + }) + + t.Run("acquire_timeout_under_pool_pressure", func(t *testing.T) { + s := newRebService(t) + s.canaries = []string{s.schema} + client := s.client(t, 1) + held, err := client.Acquire(s.ctx) + if err != nil { + s.fail("hold connection", err) + } + actx, cancel := context.WithTimeout(s.ctx, 150*time.Millisecond) + defer cancel() + _, err = client.Acquire(actx) + if err == nil { + held.Release() + t.Fatal("unbounded acquire succeeded: non-vacuity control broken") + } + if !errors.Is(err, ErrConnect) || !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("acquire timeout = %s, want ErrConnect + DeadlineExceeded", s.scrub(err)) + } + translated, ok := TranslateError(err) + if !ok { + t.Fatal("acquire timeout not translated") + } + if !errors.Is(translated, context.DeadlineExceeded) { + t.Fatal("translated acquire timeout must stay Is-visible as DeadlineExceeded") + } + userErr := rebUserErr(t, translated) + if userErr.Code() != string(dataframeerrors.CodeBackendUnavailable) || !userErr.Retryable() { + t.Fatalf("acquire timeout = %s retryable=%v, want retryable BACKEND_UNAVAILABLE", userErr.Code(), userErr.Retryable()) + } + s.rebScrubbedRendering(t, "acquire timeout", translated) + held.Release() + }) + + t.Run("ambiguous_commit", func(t *testing.T) { + s := newRebService(t) + table := `"reb_commit_canary_table"` + s.canaries = []string{s.schema, "reb_commit_canary_table", "reb_ambiguous_canary"} + if _, err := s.iso.Exec(s.ctx, + "CREATE TABLE "+s.rebQualified(table)+" (id text PRIMARY KEY)"); err != nil { + s.fail("create commit table", err) + } + client := s.client(t, 2) + victim := s.acquire(t, client) + tx, err := victim.BeginTx(s.ctx, pgx.TxOptions{}) + if err != nil { + s.fail("begin tx", err) + } + if _, err := tx.Exec(s.ctx, + "INSERT INTO "+s.rebQualified(table)+" VALUES ('reb_ambiguous_canary')"); err != nil { + s.fail("in-tx insert", err) + } + var pid int64 + if err := victim.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + s.fail("read pid", err) + } + if _, err := s.admin.Exec(s.ctx, "SELECT pg_terminate_backend($1)", pid); err != nil { + s.fail("terminate", err) + } + time.Sleep(300 * time.Millisecond) + err = tx.Commit(s.ctx) + if err == nil { + t.Fatal("commit through a terminated session succeeded: non-vacuity control broken") + } + translated, ok := TranslateCommitError(err) + if !ok { + t.Fatal("commit failure not translated") + } + translatedErr := translated.(*TranslatedError) + if !translatedErr.AmbiguousCommit || !errors.Is(translated, ErrCommitOutcomeUnknown) { + t.Fatalf("commit failure = %#v, want ambiguous with ErrCommitOutcomeUnknown", translatedErr) + } + if userErr := rebUserErr(t, translated); userErr.Retryable() { + t.Fatal("ambiguous commit must never be retryable") + } + _, mapped := rebHttpJSON(t, translated) + if mapped.Body.Error.Retryable { + t.Fatal("ambiguous commit must not advertise retry at HTTP") + } + s.rebScrubbedRendering(t, "ambiguous commit", translated) + for _, forbidden := range []string{"rolled back", "committed", "no write", "not written"} { + if strings.Contains(strings.ToLower(translated.Error()), forbidden) { + t.Fatalf("ambiguous commit rendering claims %q: %s", forbidden, translated.Error()) + } + } + }) + + t.Run("definitive_commit_rejection", func(t *testing.T) { + s := newRebService(t) + table := `"reb_defer_canary_table"` + constraint := "reb_defer_canary_ux" + s.canaries = []string{s.schema, "reb_defer_canary_table", constraint, "reb_defer_dup_canary"} + if _, err := s.iso.Exec(s.ctx, "CREATE TABLE "+s.rebQualified(table)+ + " (id text CONSTRAINT "+constraint+" UNIQUE DEFERRABLE INITIALLY DEFERRED)"); err != nil { + s.fail("create deferred-unique table", err) + } + client := s.client(t, 1) + conn := s.acquire(t, client) + tx, err := conn.BeginTx(s.ctx, pgx.TxOptions{}) + if err != nil { + s.fail("begin tx", err) + } + if _, err := tx.Exec(s.ctx, + "INSERT INTO "+s.rebQualified(table)+" VALUES ('reb_defer_dup_canary')"); err != nil { + s.fail("first deferred insert", err) + } + if _, err := tx.Exec(s.ctx, + "INSERT INTO "+s.rebQualified(table)+" VALUES ('reb_defer_dup_canary')"); err != nil { + s.fail("second deferred insert", err) + } + err = tx.Commit(s.ctx) + if err == nil { + t.Fatal("deferred constraint accepted a duplicate: non-vacuity control broken") + } + translated, ok := TranslateCommitError(err) + if !ok { + t.Fatal("deferred commit rejection not translated") + } + translatedErr := translated.(*TranslatedError) + if translatedErr.AmbiguousCommit { + t.Fatal("a server-answered commit rejection must not be ambiguous") + } + if errors.Is(translated, ErrCommitOutcomeUnknown) { + t.Fatal("a server-answered commit rejection must not claim unknown outcome") + } + if translatedErr.Condition != ConditionUniqueViolation || !IsUniqueViolation(translated) { + t.Fatalf("condition = %s, want unique_violation recognized by the predicate", translatedErr.Condition) + } + s.rebScrubbedRendering(t, "deferred rejection", translated) + }) +} + +func TestZZZeroLeftoverRebPostgresErrorFixtures(t *testing.T) { + raw := rebURL(t) + rebOwnedMu.Lock() + owned := make(map[string]string, len(rebOwnedByName)) + for db, schema := range rebOwnedByName { + owned[db] = schema + } + rebOwnedMu.Unlock() + if len(owned) == 0 { + t.Skip("this process created no verify_reb_* fixtures") + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service for audit: %v", err) + } + defer admin.Close() + + for db, schema := range owned { + if !strings.HasPrefix(db, rebFixturePrefix) { + t.Fatalf("registry holds foreign database %q", db) + } + var count int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Errorf("audit owned database %s: %v", db, err) + continue + } + if count != 0 { + t.Errorf("leftover owned database %s after all cleanups", db) + if schema != "" { + u, err := url.Parse(raw) + if err == nil { + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + db + if probe, err := pgxpool.New(ctx, u.String()); err == nil { + var schemaCount int + if err := probe.QueryRow(ctx, "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1", schema).Scan(&schemaCount); err != nil { + t.Errorf("audit owned schema %s in %s: %v", schema, db, err) + } else if schemaCount != 0 { + t.Errorf("leftover owned schema %s in leftover database %s", schema, db) + } + probe.Close() + } + } + } + } + } + + var visible int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_reb\\_%' ESCAPE '\\'"). + Scan(&visible); err != nil { + t.Fatalf("audit verify_reb_* database count: %v", err) + } + if visible != rebBaselineCount { + t.Errorf("verify_reb_* databases = %d, want baseline %d", visible, rebBaselineCount) + } +} diff --git a/internal/backends/postgres/internal/store/field_profile.go b/internal/backends/postgres/internal/store/field_profile.go new file mode 100644 index 000000000..2abd0a940 --- /dev/null +++ b/internal/backends/postgres/internal/store/field_profile.go @@ -0,0 +1,573 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + fieldProfileTableName = "fhir_field_profile" + fieldProfileCoverageTableName = "fhir_field_profile_coverage" + MaxFieldProfileWriteBatch = 500 + defaultFieldProfileWriteBatch = 50 + fieldProfileColumnCount = 26 +) + +var ( + ErrInvalidFieldProfileRow = errors.New("postgres: invalid field profile row") + ErrInvalidFieldProfileScope = errors.New("postgres: invalid field profile scope") +) + +type FieldProfileScope = catalog.FieldProfileScope + +type FieldProfileFilter struct { + Project string + DatasetGeneration string + ResourceTypes []string + AllResourceTypes bool + AuthResourcePathsUnrestricted *bool + AuthResourcePaths []string +} + +type FieldProfileCoverage struct { + FieldProfileScope + EnrichmentComplete *bool + EnrichmentTruncated bool +} + +var fieldProfileColumns = []string{ + "project", "dataset_generation", "auth_resource_path", "resource_type", "path", + "profile_key", "record_type", "kind", "doc_count", "max_items", "sample_count", + "distinct_values", "distinct_truncated", "enrichment_complete", "enrichment_truncated", + "pivot_candidate", "pivot_kind", "pivot_columns", "pivot_family", "pivot_column_selector", + "pivot_value_selector", "pivot_item_source", "pivot_item_resource_type", "pivot_value_selectors", + "extension_values", "semantic_observations", +} + +var fieldProfileUpdateSet = func() string { + parts := make([]string, 0, len(fieldProfileColumns)-len(scopeColumnNames)) + for _, column := range fieldProfileColumns { + switch column { + case "project", "dataset_generation", "auth_resource_path", "resource_type", "path": + continue + } + parts = append(parts, column+" = EXCLUDED."+column) + } + return strings.Join(parts, ", ") +}() + +var scopeColumnNames = []string{"project", "dataset_generation", "auth_resource_path", "resource_type", "path"} + +type FieldProfileRepository struct { + src ConnSource + profileFQN string + coverageFQN string +} + +func NewFieldProfileRepository(src ConnSource, schema string) (*FieldProfileRepository, error) { + if src == nil { + return nil, errors.New("postgres: nil field profile connection source") + } + if !ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", ErrInvalidSchemaName, schema) + } + return &FieldProfileRepository{ + src: src, + profileFQN: pgx.Identifier{schema, fieldProfileTableName}.Sanitize(), + coverageFQN: pgx.Identifier{schema, fieldProfileCoverageTableName}.Sanitize(), + }, nil +} + +func (r *FieldProfileRepository) UpsertFieldProfileBatch(ctx context.Context, scope FieldProfileScope, docs []catalog.FieldCatalogDocument, coverage FieldProfileCoverage, batchSize int) error { + if err := validateFieldProfileScope(scope); err != nil { + return err + } + if coverage.Project != scope.Project || coverage.DatasetGeneration != scope.DatasetGeneration || + coverage.AuthResourcePath != scope.AuthResourcePath || coverage.ResourceType != scope.ResourceType { + return fmt.Errorf("%w: coverage marker scope %+v diverges from batch scope %+v", ErrInvalidFieldProfileScope, coverage.FieldProfileScope, scope) + } + if batchSize <= 0 { + batchSize = defaultFieldProfileWriteBatch + } + if batchSize > MaxFieldProfileWriteBatch { + return fmt.Errorf("%w: batch size is %d, want 1..%d", ErrInvalidFieldProfileScope, batchSize, MaxFieldProfileWriteBatch) + } + args := make([][]any, 0, len(docs)) + for i := range docs { + if err := validateFieldProfileRow(docs[i]); err != nil { + return fmt.Errorf("postgres: field profile row %d: %w", i, err) + } + if docs[i].Project != scope.Project || docs[i].DatasetGeneration != scope.DatasetGeneration || + docs[i].AuthResourcePath != scope.AuthResourcePath || docs[i].ResourceType != scope.ResourceType { + return fmt.Errorf("%w: row %d scope (%s/%s/%s/%s) diverges from batch scope (%s/%s/%s/%s)", + ErrInvalidFieldProfileScope, i, + docs[i].Project, docs[i].DatasetGeneration, docs[i].AuthResourcePath, docs[i].ResourceType, + scope.Project, scope.DatasetGeneration, scope.AuthResourcePath, scope.ResourceType) + } + encoded, err := fieldProfileRowArgs(docs[i]) + if err != nil { + return fmt.Errorf("postgres: field profile row %d: %w", i, err) + } + args = append(args, encoded) + } + + conn, err := r.acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + + tx, err := conn.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return fmt.Errorf("postgres: begin field profile batch: %w", err) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback(ctx) + } + }() + + for start := 0; start < len(args); start += batchSize { + end := min(start+batchSize, len(args)) + if err := upsertFieldProfileChunk(ctx, tx, r.profileFQN, args[start:end]); err != nil { + return err + } + } + + if _, err := tx.Exec(ctx, + "INSERT INTO "+r.coverageFQN+ + " (project, dataset_generation, auth_resource_path, resource_type, enrichment_complete, enrichment_truncated)"+ + " VALUES ($1, $2, $3, $4, $5, $6)"+ + " ON CONFLICT (project, dataset_generation, auth_resource_path, resource_type) DO UPDATE SET"+ + " enrichment_complete = EXCLUDED.enrichment_complete,"+ + " enrichment_truncated = EXCLUDED.enrichment_truncated,"+ + " updated_at = now()", + coverage.Project, coverage.DatasetGeneration, coverage.AuthResourcePath, coverage.ResourceType, + coverage.EnrichmentComplete, coverage.EnrichmentTruncated); err != nil { + return fmt.Errorf("postgres: upsert field profile coverage %s/%s/%s/%s: %w", + coverage.Project, coverage.DatasetGeneration, coverage.AuthResourcePath, coverage.ResourceType, err) + } + + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("postgres: commit field profile batch: %w", err) + } + committed = true + return nil +} + +func upsertFieldProfileChunk(ctx context.Context, tx pgx.Tx, profileFQN string, chunk [][]any) error { + var sql strings.Builder + sql.WriteString("INSERT INTO " + profileFQN + " (" + strings.Join(fieldProfileColumns, ", ") + ") VALUES ") + args := make([]any, 0, len(chunk)*fieldProfileColumnCount) + for i, row := range chunk { + if i > 0 { + sql.WriteString(", ") + } + sql.WriteString("(") + for p := 1; p <= fieldProfileColumnCount; p++ { + if p > 1 { + sql.WriteString(", ") + } + sql.WriteString("$") + sql.WriteString(strconv.Itoa(i*fieldProfileColumnCount + p)) + } + sql.WriteString(")") + args = append(args, row...) + } + sql.WriteString(" ON CONFLICT (project, dataset_generation, auth_resource_path, resource_type, path) DO UPDATE SET ") + sql.WriteString(fieldProfileUpdateSet) + if _, err := tx.Exec(ctx, sql.String(), args...); err != nil { + return fmt.Errorf("postgres: upsert field profile chunk of %d rows: %w", len(chunk), err) + } + return nil +} + +func (r *FieldProfileRepository) ReadFieldProfileRows(ctx context.Context, scope FieldProfileScope) ([]catalog.FieldCatalogDocument, error) { + if err := validateFieldProfileScope(scope); err != nil { + return nil, err + } + sql := "SELECT " + strings.Join(fieldProfileColumns, ", ") + " FROM " + r.profileFQN + + " WHERE project = $1 AND dataset_generation = $2 AND auth_resource_path = $3 AND resource_type = $4" + + " ORDER BY path" + + conn, err := r.acquire(ctx) + if err != nil { + return nil, err + } + defer conn.Release() + + rows, err := conn.Query(ctx, sql, scope.Project, scope.DatasetGeneration, scope.AuthResourcePath, scope.ResourceType) + if err != nil { + return nil, fmt.Errorf("postgres: read field profiles %s/%s/%s/%s: %w", + scope.Project, scope.DatasetGeneration, scope.AuthResourcePath, scope.ResourceType, err) + } + defer rows.Close() + return scanFieldProfileRows(rows) +} + +func (r *FieldProfileRepository) ReadFieldProfileRowsFiltered(ctx context.Context, filter FieldProfileFilter) ([]catalog.FieldCatalogDocument, error) { + where, args, empty, err := fieldProfileFilterSQL(filter) + if err != nil { + return nil, err + } + if empty { + return []catalog.FieldCatalogDocument{}, nil + } + sql := "SELECT " + strings.Join(fieldProfileColumns, ", ") + " FROM " + r.profileFQN + + " WHERE " + where + " ORDER BY resource_type, path" + + conn, err := r.acquire(ctx) + if err != nil { + return nil, err + } + defer conn.Release() + + rows, err := conn.Query(ctx, sql, args...) + if err != nil { + return nil, fmt.Errorf("postgres: read field profiles %s/%s: %w", filter.Project, filter.DatasetGeneration, err) + } + defer rows.Close() + return scanFieldProfileRows(rows) +} + +func (r *FieldProfileRepository) ReadProfileCoverage(ctx context.Context, filter FieldProfileFilter) ([]FieldProfileCoverage, error) { + where, args, empty, err := fieldProfileFilterSQL(filter) + if err != nil { + return nil, err + } + if empty { + return []FieldProfileCoverage{}, nil + } + sql := "SELECT project, dataset_generation, auth_resource_path, resource_type, enrichment_complete, enrichment_truncated" + + " FROM " + r.coverageFQN + " WHERE " + where + " ORDER BY auth_resource_path, resource_type" + + conn, err := r.acquire(ctx) + if err != nil { + return nil, err + } + defer conn.Release() + + rows, err := conn.Query(ctx, sql, args...) + if err != nil { + return nil, fmt.Errorf("postgres: read field profile coverage %s/%s: %w", filter.Project, filter.DatasetGeneration, err) + } + defer rows.Close() + + out := []FieldProfileCoverage{} + for rows.Next() { + var coverage FieldProfileCoverage + if err := rows.Scan(&coverage.Project, &coverage.DatasetGeneration, &coverage.AuthResourcePath, + &coverage.ResourceType, &coverage.EnrichmentComplete, &coverage.EnrichmentTruncated); err != nil { + return nil, fmt.Errorf("postgres: read field profile coverage %s/%s: %w", filter.Project, filter.DatasetGeneration, err) + } + out = append(out, coverage) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("postgres: read field profile coverage %s/%s: %w", filter.Project, filter.DatasetGeneration, err) + } + return out, nil +} + +func (r *FieldProfileRepository) ListDistinctAuthResourcePaths(ctx context.Context, project string) ([]string, error) { + sql, args, err := distinctAuthResourcePathsSQL(r.profileFQN, project) + if err != nil { + return nil, err + } + conn, err := r.acquire(ctx) + if err != nil { + return nil, err + } + defer conn.Release() + + rows, err := conn.Query(ctx, sql, args...) + if err != nil { + return nil, fmt.Errorf("postgres: list distinct auth resource paths %s: %w", project, err) + } + defer rows.Close() + + out := []string{} + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, fmt.Errorf("postgres: scan distinct auth resource path %s: %w", project, err) + } + out = append(out, path) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("postgres: list distinct auth resource paths %s: %w", project, err) + } + return out, nil +} + +func distinctAuthResourcePathsSQL(profileFQN, project string) (string, []any, error) { + if strings.TrimSpace(project) == "" { + return "", nil, fmt.Errorf("%w: project is blank", ErrInvalidFieldProfileScope) + } + sql := `SELECT DISTINCT auth_resource_path COLLATE "C" FROM ` + profileFQN + + " WHERE project = $1 AND auth_resource_path <> ''" + + " ORDER BY auth_resource_path" + return sql, []any{project}, nil +} + +func fieldProfileFilterSQL(filter FieldProfileFilter) (string, []any, bool, error) { + if strings.TrimSpace(filter.Project) == "" { + return "", nil, false, fmt.Errorf("%w: project is blank", ErrInvalidFieldProfileScope) + } + if filter.AuthResourcePathsUnrestricted == nil { + return "", nil, false, fmt.Errorf( + "%w: AuthResourcePathsUnrestricted must be set explicitly; there is no implicit unrestricted mode", + ErrInvalidFieldProfileScope) + } + if *filter.AuthResourcePathsUnrestricted { + if len(filter.AuthResourcePaths) > 0 { + return "", nil, false, fmt.Errorf( + "%w: unrestricted mode must not carry auth resource paths (%d given)", + ErrInvalidFieldProfileScope, len(filter.AuthResourcePaths)) + } + } else if len(filter.AuthResourcePaths) == 0 { + return "", nil, true, nil + } + if filter.AllResourceTypes { + if len(filter.ResourceTypes) > 0 { + return "", nil, false, fmt.Errorf( + "%w: AllResourceTypes must not accompany a declared resource types list (%d given)", + ErrInvalidFieldProfileScope, len(filter.ResourceTypes)) + } + where := "project = $1 AND dataset_generation = $2" + args := []any{filter.Project, filter.DatasetGeneration} + if !*filter.AuthResourcePathsUnrestricted { + where += " AND auth_resource_path = ANY($3::text[])" + args = append(args, append([]string(nil), filter.AuthResourcePaths...)) + } + return where, args, false, nil + } + if len(filter.ResourceTypes) == 0 { + return "", nil, true, nil + } + where := "project = $1 AND dataset_generation = $2 AND resource_type = ANY($3::text[])" + args := []any{filter.Project, filter.DatasetGeneration, append([]string(nil), filter.ResourceTypes...)} + if !*filter.AuthResourcePathsUnrestricted { + where += " AND auth_resource_path = ANY($4::text[])" + args = append(args, append([]string(nil), filter.AuthResourcePaths...)) + } + return where, args, false, nil +} + +func scanFieldProfileRows(rows pgx.Rows) ([]catalog.FieldCatalogDocument, error) { + out := []catalog.FieldCatalogDocument{} + for rows.Next() { + var doc catalog.FieldCatalogDocument + var extensionRaw, semanticRaw json.RawMessage + targets := []any{ + &doc.Project, &doc.DatasetGeneration, &doc.AuthResourcePath, &doc.ResourceType, &doc.Path, + &doc.Key, &doc.RecordType, &doc.Kind, &doc.DocCount, &doc.MaxItems, &doc.SampleCount, + &doc.DistinctValues, &doc.DistinctTruncated, &doc.EnrichmentComplete, &doc.EnrichmentTruncated, + &doc.PivotCandidate, &doc.PivotKind, &doc.PivotColumns, &doc.PivotFamily, &doc.PivotColumnSelect, + &doc.PivotValueSelect, &doc.PivotItemSource, &doc.PivotItemResourceType, &doc.PivotValueSelectors, + &extensionRaw, &semanticRaw, + } + if err := rows.Scan(targets...); err != nil { + return nil, fmt.Errorf("postgres: scan field profile row %s/%s/%s: %w", + doc.Project, doc.ResourceType, doc.Path, err) + } + if err := decodeFieldProfileJSON(&doc, extensionRaw, semanticRaw); err != nil { + return nil, err + } + out = append(out, doc) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("postgres: read field profile rows: %w", err) + } + return out, nil +} + +func decodeFieldProfileJSON(doc *catalog.FieldCatalogDocument, extensionRaw, semanticRaw json.RawMessage) error { + if len(extensionRaw) > 0 { + if err := json.Unmarshal(extensionRaw, &doc.ExtensionValues); err != nil { + return fmt.Errorf("postgres: decode extension observations for field profile %s/%s/%s: %w", + doc.Project, doc.ResourceType, doc.Path, err) + } + } + if len(semanticRaw) > 0 { + if err := json.Unmarshal(semanticRaw, &doc.SemanticObservations); err != nil { + return fmt.Errorf("postgres: decode semantic observations for field profile %s/%s/%s: %w", + doc.Project, doc.ResourceType, doc.Path, err) + } + } + if len(doc.DistinctValues) == 0 { + doc.DistinctValues = nil + } + if len(doc.PivotColumns) == 0 { + doc.PivotColumns = nil + } + if len(doc.PivotValueSelectors) == 0 { + doc.PivotValueSelectors = nil + } + return nil +} + +func fieldProfileRowArgs(doc catalog.FieldCatalogDocument) ([]any, error) { + extensionValues, err := extensionValuesParam(doc.ExtensionValues) + if err != nil { + return nil, fmt.Errorf("encode extension values: %w", err) + } + semanticObservations, err := semanticObservationsParam(doc.SemanticObservations) + if err != nil { + return nil, fmt.Errorf("encode semantic observations: %w", err) + } + return []any{ + doc.Project, doc.DatasetGeneration, doc.AuthResourcePath, doc.ResourceType, doc.Path, + doc.Key, doc.RecordType, doc.Kind, doc.DocCount, int64(doc.MaxItems), int64(doc.SampleCount), + textArrayParam(doc.DistinctValues), doc.DistinctTruncated, doc.EnrichmentComplete, doc.EnrichmentTruncated, + doc.PivotCandidate, doc.PivotKind, textArrayParam(doc.PivotColumns), doc.PivotFamily, doc.PivotColumnSelect, + doc.PivotValueSelect, doc.PivotItemSource, doc.PivotItemResourceType, textArrayParam(doc.PivotValueSelectors), + extensionValues, semanticObservations, + }, nil +} + +func textArrayParam(values []string) any { + if len(values) == 0 { + return nil + } + return append([]string(nil), values...) +} + +func extensionValuesParam(values []catalog.ExtensionValueObservation) ([]byte, error) { + if values == nil { + return nil, nil + } + encoded := make([]json.RawMessage, 0, len(values)) + for i := range values { + observation, err := json.Marshal(values[i]) + if err != nil { + return nil, err + } + object, err := forceSlicePresence([]string{"url_path"}, map[string][]string{ + "url_path": values[i].URLPath, + }, observation) + if err != nil { + return nil, err + } + encoded = append(encoded, object) + } + return json.Marshal(encoded) +} + +func semanticObservationsParam(observations []catalog.SemanticObservation) ([]byte, error) { + if observations == nil { + return nil, nil + } + encoded := make([]json.RawMessage, 0, len(observations)) + for i := range observations { + observation, err := json.Marshal(observations[i]) + if err != nil { + return nil, err + } + object, err := forceSlicePresence([]string{"extension_url_path", "observed_units", "examples"}, map[string][]string{ + "extension_url_path": observations[i].ExtensionURLPath, + "observed_units": observations[i].ObservedUnits, + "examples": observations[i].Examples, + }, observation) + if err != nil { + return nil, err + } + object, err = forceNestedSlicePresence(object, "value", "fallback_selectors", observations[i].Value.FallbackSelectors) + if err != nil { + return nil, err + } + encoded = append(encoded, object) + } + return json.Marshal(encoded) +} + +func forceSlicePresence(keys []string, values map[string][]string, observation json.RawMessage) (json.RawMessage, error) { + var object map[string]json.RawMessage + if err := json.Unmarshal(observation, &object); err != nil { + return nil, err + } + for _, key := range keys { + if values[key] == nil { + object[key] = json.RawMessage("null") + } else if len(values[key]) == 0 { + object[key] = json.RawMessage("[]") + } + } + return json.Marshal(object) +} + +func forceNestedSlicePresence(observation json.RawMessage, member, key string, value []string) (json.RawMessage, error) { + var object map[string]json.RawMessage + if err := json.Unmarshal(observation, &object); err != nil { + return nil, err + } + nested, ok := object[member] + if !ok { + return observation, nil + } + forced, err := forceSlicePresence([]string{key}, map[string][]string{key: value}, nested) + if err != nil { + return nil, err + } + object[member] = forced + return json.Marshal(object) +} + +func validateFieldProfileScope(scope FieldProfileScope) error { + if strings.TrimSpace(scope.Project) == "" { + return fmt.Errorf("%w: project is blank", ErrInvalidFieldProfileScope) + } + if strings.TrimSpace(scope.ResourceType) == "" { + return fmt.Errorf("%w: resource type is blank", ErrInvalidFieldProfileScope) + } + return nil +} + +func validateFieldProfileRow(doc catalog.FieldCatalogDocument) error { + if strings.TrimSpace(doc.Project) == "" { + return fmt.Errorf("%w: project is blank", ErrInvalidFieldProfileRow) + } + if strings.TrimSpace(doc.ResourceType) == "" { + return fmt.Errorf("%w: resource type is blank", ErrInvalidFieldProfileRow) + } + if strings.TrimSpace(doc.Key) == "" { + return fmt.Errorf("%w: profile key is blank", ErrInvalidFieldProfileRow) + } + switch doc.RecordType { + case "": + if doc.Path == "" { + return fmt.Errorf("%w: field profile has no path (record_type is empty)", ErrInvalidFieldProfileRow) + } + if strings.TrimSpace(doc.Kind) == "" { + return fmt.Errorf("%w: field profile %q has no kind", ErrInvalidFieldProfileRow, doc.Path) + } + case catalog.FieldProfileStatusRecordType: + if doc.Path != "" { + return fmt.Errorf("%w: profile_status marker carries path %q", ErrInvalidFieldProfileRow, doc.Path) + } + default: + return fmt.Errorf("%w: record type %q is neither a field profile nor %q", + ErrInvalidFieldProfileRow, doc.RecordType, catalog.FieldProfileStatusRecordType) + } + if doc.DocCount < 0 || doc.MaxItems < 0 || doc.SampleCount < 0 { + return fmt.Errorf("%w: counts are negative (doc_count=%d, max_items=%d, sample_count=%d)", + ErrInvalidFieldProfileRow, doc.DocCount, doc.MaxItems, doc.SampleCount) + } + return nil +} + +func (r *FieldProfileRepository) acquire(ctx context.Context) (*pgxpool.Conn, error) { + conn, err := r.src.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres: acquire field profile connection: %w", err) + } + return conn, nil +} diff --git a/internal/backends/postgres/internal/store/field_profile_auth_paths_test.go b/internal/backends/postgres/internal/store/field_profile_auth_paths_test.go new file mode 100644 index 000000000..3f92da885 --- /dev/null +++ b/internal/backends/postgres/internal/store/field_profile_auth_paths_test.go @@ -0,0 +1,120 @@ +package postgres + +import ( + "errors" + "reflect" + "slices" + "strings" + "testing" +) + +func TestDistinctAuthResourcePathsSQLGoldenPins(t *testing.T) { + sql, args, err := distinctAuthResourcePathsSQL(`"it_schema"."fhir_field_profile"`, "p1") + if err != nil { + t.Fatalf("render: %v", err) + } + want := `SELECT DISTINCT auth_resource_path COLLATE "C" FROM "it_schema"."fhir_field_profile" WHERE project = $1 AND auth_resource_path <> '' ORDER BY auth_resource_path` + if sql != want { + t.Fatalf("golden SQL drift:\n got %q\nwant %q", sql, want) + } + if !reflect.DeepEqual(args, []any{"p1"}) { + t.Fatalf("golden args drift: %#v", args) + } + if strings.Contains(sql, "dataset_generation") { + t.Fatalf("union render carries a dataset_generation predicate: %q", sql) + } + if !strings.Contains(sql, "auth_resource_path <> ''") { + t.Fatalf("union render left the empty-path door open: %q", sql) + } + if !strings.Contains(sql, `COLLATE "C"`) { + t.Fatalf("union render left the order to the cluster collation: %q", sql) + } + for _, fragment := range []string{"ORDER BY resource_type", "GROUP BY"} { + if strings.Contains(sql, fragment) { + t.Fatalf("union render carries unexpected fragment %q: %q", fragment, sql) + } + } + + for _, blank := range []string{"", " ", "\t"} { + sql, args, err := distinctAuthResourcePathsSQL(`"it_schema"."fhir_field_profile"`, blank) + if err == nil { + t.Fatalf("blank project %q rendered %q (args %#v); want the typed refusal", blank, sql, args) + } + if sql != "" || args != nil { + t.Fatalf("blank project %q refusal still produced sql=%q args=%#v", blank, sql, args) + } + if !errors.Is(err, ErrInvalidFieldProfileScope) { + t.Fatalf("blank project refusal %v must wrap ErrInvalidFieldProfileScope", err) + } + } +} + +func TestListDistinctAuthResourcePathsUnionAcrossGenerations(t *testing.T) { + svc := newMigrationService(t) + if _, err := MigrateSchema(svc.ctx, svc.openClient(t), svc.schema); err != nil { + svc.fail("MigrateSchema", err) + } + repo, err := NewFieldProfileRepository(svc.iso, svc.schema) + if err != nil { + t.Fatalf("NewFieldProfileRepository: %v", err) + } + + const project = "pgauth-allgens" + fixture := []map[string]string{ + {"dataset_generation": "'generation-a'", "auth_resource_path": "'/fhir/clinical'", "path": "'obs.a1'"}, + {"dataset_generation": "'generation-a'", "auth_resource_path": "'/fhir/finance'", "path": "'obs.a2'"}, + {"dataset_generation": "'generation-b'", "auth_resource_path": "'/fhir/clinical'", "path": "'obs.b1'"}, + {"dataset_generation": "'generation-b'", "auth_resource_path": "'/fhir/archive'", "path": "'obs.b2'"}, + {"dataset_generation": "''", "auth_resource_path": "'/fhir/legacy'", "path": "'obs.legacy'"}, + {"dataset_generation": "'generation-a'", "auth_resource_path": "'/fhir/custom'", "resource_type": "'CustomCarePlan'", "path": "'obs.custom'"}, + {"dataset_generation": "'generation-a'", "auth_resource_path": "'/fhir/research'", "record_type": "'profile_status'", "path": "''"}, + {"dataset_generation": "'generation-a'", "auth_resource_path": "''", "path": "'obs.unscoped'"}, + {"project": "'pgauth-other'", "dataset_generation": "'generation-a'", "auth_resource_path": "'/fhir/foreign'", "path": "'obs.foreign'"}, + {"dataset_generation": "'generation-c'", "auth_resource_path": "'/fhir/alpha'", "path": "'obs.c1'"}, + {"dataset_generation": "'generation-c'", "auth_resource_path": "'/fhir/Beta'", "path": "'obs.c2'"}, + {"dataset_generation": "'generation-c'", "auth_resource_path": "'/fhir/_x'", "path": "'obs.c3'"}, + {"dataset_generation": "'generation-c'", "auth_resource_path": "'/fhir/Ω'", "path": "'obs.c4'"}, + } + for _, override := range fixture { + if _, owned := override["project"]; !owned { + override["project"] = fpDDLText(project) + } + fpDDLExec(t, svc, fpDDLProfileStmt(svc, override)) + } + + paths, err := repo.ListDistinctAuthResourcePaths(svc.ctx, project) + if err != nil { + svc.fail("ListDistinctAuthResourcePaths", err) + } + want := []string{"/fhir/Beta", "/fhir/_x", "/fhir/alpha", "/fhir/archive", + "/fhir/clinical", "/fhir/custom", "/fhir/finance", "/fhir/legacy", "/fhir/research", "/fhir/Ω"} + if !reflect.DeepEqual(paths, want) { + t.Fatalf("union = %v, want byte-ordered distinct %v", paths, want) + } + if paths == nil { + t.Fatal("union returned a nil slice") + } + if !slices.IsSortedFunc(paths, func(a, b string) int { return strings.Compare(a, b) }) { + t.Fatalf("union is not ascending byte order: %v", paths) + } + + foreign, err := repo.ListDistinctAuthResourcePaths(svc.ctx, "pgauth-other") + if err != nil { + svc.fail("foreign project listing", err) + } + if !reflect.DeepEqual(foreign, []string{"/fhir/foreign"}) { + t.Fatalf("foreign project union = %v, want [/fhir/foreign]", foreign) + } + + empty, err := repo.ListDistinctAuthResourcePaths(svc.ctx, "pgauth-none") + if err != nil { + svc.fail("unrecorded project listing", err) + } + if empty == nil || len(empty) != 0 { + t.Fatalf("unrecorded project union = %#v, want a non-nil empty slice", empty) + } + + if _, err := repo.ListDistinctAuthResourcePaths(svc.ctx, " "); !errors.Is(err, ErrInvalidFieldProfileScope) { + t.Fatalf("blank project refusal = %v, want ErrInvalidFieldProfileScope", err) + } +} diff --git a/internal/backends/postgres/internal/store/field_profile_ddl_test.go b/internal/backends/postgres/internal/store/field_profile_ddl_test.go new file mode 100644 index 000000000..0e09fd74d --- /dev/null +++ b/internal/backends/postgres/internal/store/field_profile_ddl_test.go @@ -0,0 +1,773 @@ +package postgres + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "slices" + "strings" + "testing" + + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5/pgconn" +) + +const ( + fpDDLProject = "pgcat01-ddl" + fpDDLGen = "g1" + fpDDLType = "Observation" +) + +func fpDDLText(s string) string { + return "'" + strings.ReplaceAll(s, "'", "''") + "'" +} + +var fpDDLProfileDefaults = map[string]string{ + "project": fpDDLText(fpDDLProject), + "dataset_generation": fpDDLText(fpDDLGen), + "auth_resource_path": "''", + "resource_type": fpDDLText(fpDDLType), + "path": "'obs.value'", + "profile_key": "'pgcat01-ddl-key'", + "record_type": "''", + "kind": "'string'", + "doc_count": "0", + "max_items": "0", + "sample_count": "0", + "distinct_values": "NULL", + "distinct_truncated": "false", + "enrichment_complete": "NULL", + "enrichment_truncated": "false", + "pivot_candidate": "false", + "pivot_kind": "''", + "pivot_columns": "NULL", + "pivot_family": "''", + "pivot_column_selector": "''", + "pivot_value_selector": "''", + "pivot_item_source": "''", + "pivot_item_resource_type": "''", + "pivot_value_selectors": "NULL", + "extension_values": "NULL", + "semantic_observations": "NULL", +} + +var fpDDLProfileColumns = []string{ + "project", "dataset_generation", "auth_resource_path", "resource_type", "path", + "profile_key", "record_type", "kind", "doc_count", "max_items", "sample_count", + "distinct_values", "distinct_truncated", "enrichment_complete", "enrichment_truncated", + "pivot_candidate", "pivot_kind", "pivot_columns", "pivot_family", "pivot_column_selector", + "pivot_value_selector", "pivot_item_source", "pivot_item_resource_type", "pivot_value_selectors", + "extension_values", "semantic_observations", +} + +func fpDDLProfileStmt(svc *migrationService, overrides map[string]string) string { + cols := make([]string, 0, len(fpDDLProfileColumns)) + vals := make([]string, 0, len(fpDDLProfileColumns)) + for _, column := range fpDDLProfileColumns { + literal, ok := fpDDLProfileDefaults[column] + if !ok { + panic("fpDDLProfileDefaults lacks column " + column) + } + if override, changed := overrides[column]; changed { + literal = override + } + cols = append(cols, column) + vals = append(vals, literal) + } + return fmt.Sprintf(`INSERT INTO "%s".fhir_field_profile (%s) VALUES (%s)`, + svc.schema, strings.Join(cols, ", "), strings.Join(vals, ", ")) +} + +func fpDDLExec(t *testing.T, svc *migrationService, stmt string) { + t.Helper() + if _, err := svc.iso.Exec(svc.ctx, stmt); err != nil { + svc.fail("exec "+firstStatementWords(stmt), err) + } +} + +func fpDDLCount(t *testing.T, svc *migrationService, table string) int64 { + t.Helper() + var n int64 + if err := svc.iso.QueryRow(svc.ctx, fmt.Sprintf(`SELECT count(*) FROM "%s".%s`, svc.schema, table)).Scan(&n); err != nil { + svc.fail("count "+table, err) + } + return n +} + +func fpDDLExpectViolation(t *testing.T, svc *migrationService, table, stmt string, wantCode, wantConstraint string, wantRows int64) { + t.Helper() + _, err := svc.iso.Exec(svc.ctx, stmt) + if err == nil { + t.Fatalf("statement accepted, want rejection by %s (%s): %s", wantConstraint, wantCode, firstStatementWords(stmt)) + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + svc.fail("non-PgError rejection "+firstStatementWords(stmt), err) + } + accepted := []string{wantConstraint} + if wantConstraint == "" { + accepted = []string{"fhir_field_profile_record_shape", "fhir_field_profile_record_type_known"} + } + if pgErr.Code != wantCode || !slices.Contains(accepted, pgErr.ConstraintName) { + t.Fatalf("rejection = (%s, %q), want (%s, one of %v): %v", + pgErr.Code, pgErr.ConstraintName, wantCode, accepted, svc.scrub(err)) + } + if got := fpDDLCount(t, svc, table); got != wantRows { + t.Fatalf("table %s holds %d rows after refused insert, want %d", table, got, wantRows) + } +} + +func firstStatementWords(stmt string) string { + words := strings.Fields(stmt) + if len(words) > 6 { + words = words[:6] + } + return strings.Join(words, " ") + " ..." +} + +func fpDDLFindMigration(t *testing.T, migrations []Migration, version int64) Migration { + t.Helper() + for _, m := range migrations { + if m.Version == version { + return m + } + } + t.Fatalf("embedded migrations have no version %d", version) + return Migration{} +} + +type fpDDLSchemaFingerprint struct { + Columns []string + Constraints []string + Indexes []string +} + +func fpDDLCollectFingerprint(t *testing.T, svc *migrationService) fpDDLSchemaFingerprint { + t.Helper() + fp := fpDDLSchemaFingerprint{} + fp.Columns = fpDDLStringColumn(t, svc, fmt.Sprintf( + `SELECT column_name || '|' || data_type || '|' || is_nullable || '|' || coalesce(column_default,'') + FROM information_schema.columns + WHERE table_schema = $1 AND table_name IN ('fhir_field_profile', 'fhir_field_profile_coverage') + ORDER BY table_name, ordinal_position`), svc.schema) + fp.Constraints = fpDDLStringColumn(t, svc, fmt.Sprintf( + `SELECT c.conrelid::regclass::text || '|' || c.conname || '|' || pg_get_constraintdef(c.oid) + FROM pg_constraint c + WHERE c.connamespace = $1::regnamespace + AND c.conrelid::regclass::text IN ($1::text||'.fhir_field_profile', $1::text||'.fhir_field_profile_coverage') + ORDER BY 1`), svc.schema) + fp.Indexes = fpDDLStringColumn(t, svc, fmt.Sprintf( + `SELECT tablename || '|' || indexname || '|' || indexdef + FROM pg_indexes + WHERE schemaname = $1 AND tablename IN ('fhir_field_profile', 'fhir_field_profile_coverage') + ORDER BY 1`), svc.schema) + if len(fp.Columns) != 26+7 { + t.Fatalf("fingerprint sees %d columns, want 33 (26 + 7): %v", len(fp.Columns), fp.Columns) + } + return fp +} + +func fpDDLStringColumn(t *testing.T, svc *migrationService, query string, arg any) []string { + t.Helper() + rows, err := svc.iso.Query(svc.ctx, query, arg) + if err != nil { + svc.fail("fingerprint query "+firstStatementWords(query), err) + } + defer rows.Close() + var out []string + for rows.Next() { + var s string + if err := rows.Scan(&s); err != nil { + svc.fail("fingerprint scan", err) + } + out = append(out, s) + } + if err := rows.Err(); err != nil { + svc.fail("fingerprint rows", err) + } + return out +} + +func TestFieldProfileDDLDoubleApplyIsIdempotent(t *testing.T) { + svc := newMigrationService(t) + embedded := mustEmbeddedMigrations(t) + + first, err := MigrateSchema(svc.ctx, svc.openClient(t), svc.schema) + if err != nil { + svc.fail("MigrateSchema", err) + } + if len(first.Applied) != len(embedded) || first.LatestVersion != embedded[len(embedded)-1].Version { + t.Fatalf("first migrate = %+v, want %d applied rows up to version %d", + first, len(embedded), embedded[len(embedded)-1].Version) + } + before := fpDDLCollectFingerprint(t, svc) + + m := fpDDLFindMigration(t, embedded, 7) + if m.Name != "field_profile" { + t.Fatalf("embedded version 7 is %q, want field_profile", m.Name) + } + for i := 0; i < 2; i++ { + if _, err := svc.iso.Exec(svc.ctx, m.SQL); err != nil { + svc.fail("direct re-apply of 000007 DDL", err) + } + } + + second, err := MigrateSchema(svc.ctx, svc.openClient(t), svc.schema) + if err != nil { + svc.fail("re-apply MigrateSchema", err) + } + if !reflect.DeepEqual(first, second) { + t.Errorf("runner re-apply changed the result:\n1st: %+v\n2nd: %+v", first, second) + } + after := fpDDLCollectFingerprint(t, svc) + if !reflect.DeepEqual(before, after) { + t.Errorf("schema changed across double apply:\nbefore: %+v\nafter: %+v", before, after) + } + ledger := svc.readLedger(t) + if len(ledger) != len(embedded) { + t.Fatalf("ledger holds %d rows after double apply, want %d", len(ledger), len(embedded)) + } + var sevens int + for _, rec := range ledger { + if rec.Version == 7 { + sevens++ + } + } + if sevens != 1 { + t.Fatalf("ledger records version 7 %d times, want exactly once", sevens) + } +} + +func TestFieldProfileDDLChecksRejectViolations(t *testing.T) { + svc := newMigrationService(t) + if _, err := MigrateSchema(svc.ctx, svc.openClient(t), svc.schema); err != nil { + svc.fail("MigrateSchema", err) + } + + const ( + codeCheck = "23514" + codeUniq = "23505" + ) + cases := []struct { + name string + table string + stmt string + code string + constraint string + rowsBefore int64 + precedingStmts []string + }{ + {"project_blank", "fhir_field_profile", "", codeCheck, "fhir_field_profile_project_present", 0, nil}, + {"resource_type_blank", "fhir_field_profile", "", codeCheck, "fhir_field_profile_type_present", 0, nil}, + {"profile_key_blank", "fhir_field_profile", "", codeCheck, "fhir_field_profile_key_present", 0, nil}, + {"status_row_with_path", "fhir_field_profile", "", codeCheck, "fhir_field_profile_record_shape", 0, nil}, + {"field_row_without_kind", "fhir_field_profile", "", codeCheck, "fhir_field_profile_record_shape", 0, nil}, + {"record_type_unknown", "fhir_field_profile", "", codeCheck, "", 0, nil}, + {"doc_count_negative", "fhir_field_profile", "", codeCheck, "fhir_field_profile_counts_nonnegative", 0, nil}, + {"max_items_negative", "fhir_field_profile", "", codeCheck, "fhir_field_profile_counts_nonnegative", 0, nil}, + {"sample_count_negative", "fhir_field_profile", "", codeCheck, "fhir_field_profile_counts_nonnegative", 0, nil}, + {"extension_object", "fhir_field_profile", "", codeCheck, "fhir_field_profile_extension_is_array", 0, nil}, + {"extension_json_null", "fhir_field_profile", "", codeCheck, "fhir_field_profile_extension_is_array", 0, nil}, + {"semantic_scalar", "fhir_field_profile", "", codeCheck, "fhir_field_profile_semantic_is_array", 0, nil}, + {"duplicate_identity", "fhir_field_profile", "", codeUniq, "fhir_field_profile_pkey", 1, []string{""}}, + {"coverage_project_blank", "fhir_field_profile_coverage", "", codeCheck, "fhir_field_profile_coverage_project_present", 0, nil}, + {"coverage_type_blank", "fhir_field_profile_coverage", "", codeCheck, "fhir_field_profile_coverage_type_present", 0, nil}, + {"coverage_duplicate", "fhir_field_profile_coverage", "", codeUniq, "fhir_field_profile_coverage_pkey", 1, []string{""}}, + } + + overridesByCase := map[string]map[string]string{ + "project_blank": {"project": "''"}, + "resource_type_blank": {"resource_type": "''"}, + "profile_key_blank": {"profile_key": "''"}, + "status_row_with_path": {"record_type": "'profile_status'", "path": "'obs.value'"}, + "field_row_without_kind": {"record_type": "''", "kind": "''"}, + "record_type_unknown": {"record_type": "'bogus'", "path": "'obs.value'"}, + "doc_count_negative": {"doc_count": "-1"}, + "max_items_negative": {"max_items": "-1"}, + "sample_count_negative": {"sample_count": "-1"}, + "extension_object": {"extension_values": `'{"a":1}'::jsonb`}, + "extension_json_null": {"extension_values": `'null'::jsonb`}, + "semantic_scalar": {"semantic_observations": `'7'::jsonb`}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var stmt string + switch tc.name { + case "duplicate_identity": + fpDDLExec(t, svc, fpDDLProfileStmt(svc, map[string]string{"path": "'obs.dup'"})) + stmt = fpDDLProfileStmt(svc, map[string]string{"path": "'obs.dup'", "profile_key": "'changed-key'"}) + case "coverage_duplicate": + fpDDLExec(t, svc, fpDDLCoverageStmt(svc, nil)) + stmt = fpDDLCoverageStmt(svc, map[string]string{"enrichment_truncated": "true"}) + case "coverage_project_blank": + stmt = fpDDLCoverageStmt(svc, map[string]string{"project": "''"}) + case "coverage_type_blank": + stmt = fpDDLCoverageStmt(svc, map[string]string{"resource_type": "''"}) + default: + overrides, ok := overridesByCase[tc.name] + if !ok { + t.Fatalf("case %q has no overrides", tc.name) + } + stmt = fpDDLProfileStmt(svc, overrides) + } + fpDDLExpectViolation(t, svc, tc.table, stmt, tc.code, tc.constraint, tc.rowsBefore) + }) + } + + t.Run("control_legal_shapes", func(t *testing.T) { + rows, err := svc.iso.Query(svc.ctx, + `SELECT conname FROM pg_constraint + WHERE connamespace = $1::regnamespace + AND conrelid::regclass::text IN ($1::text||'.fhir_field_profile', $1::text||'.fhir_field_profile_coverage')`, + svc.schema) + if err != nil { + svc.fail("constraint catalog witness", err) + } + present := map[string]bool{} + for rows.Next() { + var n string + if err := rows.Scan(&n); err != nil { + rows.Close() + svc.fail("constraint catalog scan", err) + } + present[n] = true + } + rows.Close() + for _, want := range []string{ + "fhir_field_profile_pkey", "fhir_field_profile_project_present", + "fhir_field_profile_type_present", "fhir_field_profile_key_present", + "fhir_field_profile_record_shape", "fhir_field_profile_record_type_known", + "fhir_field_profile_counts_nonnegative", "fhir_field_profile_extension_is_array", + "fhir_field_profile_semantic_is_array", "fhir_field_profile_coverage_pkey", + "fhir_field_profile_coverage_project_present", "fhir_field_profile_coverage_type_present", + } { + if !present[want] { + t.Errorf("constraint %s missing from migrated schema", want) + } + } + + fpDDLExec(t, svc, fpDDLProfileStmt(svc, map[string]string{"path": "''", "record_type": "'profile_status'", "kind": "''"})) + fpDDLExec(t, svc, fpDDLProfileStmt(svc, map[string]string{"path": "'obs.jsonb-null'", "extension_values": "NULL", "semantic_observations": "NULL"})) + fpDDLExec(t, svc, fpDDLProfileStmt(svc, map[string]string{"path": "'obs.jsonb-empty'", "extension_values": `'[]'::jsonb`, "semantic_observations": `'[]'::jsonb`})) + fpDDLExec(t, svc, fpDDLProfileStmt(svc, map[string]string{"path": "''", "record_type": "'profile_status'", "kind": "''", "auth_resource_path": "'cov/status-null'", "enrichment_complete": "NULL"})) + }) +} + +func fpDDLCoverageStmt(svc *migrationService, overrides map[string]string) string { + defaults := map[string]string{ + "project": fpDDLText(fpDDLProject), + "dataset_generation": fpDDLText(fpDDLGen), + "auth_resource_path": "'cov/ddl'", + "resource_type": fpDDLText(fpDDLType), + "enrichment_complete": "NULL", + "enrichment_truncated": "false", + } + cols := []string{"project", "dataset_generation", "auth_resource_path", "resource_type", "enrichment_complete", "enrichment_truncated"} + vals := make([]string, 0, len(cols)) + for _, column := range cols { + literal := defaults[column] + if override, changed := overrides[column]; changed { + literal = override + } + vals = append(vals, literal) + } + return fmt.Sprintf(`INSERT INTO "%s".fhir_field_profile_coverage (%s) VALUES (%s)`, + svc.schema, strings.Join(cols, ", "), strings.Join(vals, ", ")) +} + +func fpDDLTextArrayWitness(t *testing.T, svc *migrationService, column string, paths [3]string) error { + t.Helper() + want := [3][3]bool{ + {true, false, false}, + {false, true, false}, + {false, false, true}, + } + for i, path := range paths { + var isNull, isEmpty, isPopulated bool + if err := svc.iso.QueryRow(svc.ctx, fmt.Sprintf( + `SELECT %s IS NULL, coalesce(%s = '{}'::text[], false), coalesce(%s = ARRAY['a','b']::text[], false) + FROM "%s".fhir_field_profile WHERE project=$1 AND dataset_generation=$2 AND resource_type=$3 AND path=$4`, + column, column, column, svc.schema), + fpDDLProject, fpDDLGen, fpDDLType, path).Scan(&isNull, &isEmpty, &isPopulated); err != nil { + return fmt.Errorf("witness row %s: %w", path, err) + } + got := [3]bool{isNull, isEmpty, isPopulated} + if got != want[i] { + return fmt.Errorf("column %s row %s encodings = %v, want %v (NULL/'{}'/populated must be three distinct states)", + column, path, got, want[i]) + } + } + return nil +} + +func TestFieldProfileTextArrayEncodingsDistinguishThreeStates(t *testing.T) { + svc := newMigrationService(t) + if _, err := MigrateSchema(svc.ctx, svc.openClient(t), svc.schema); err != nil { + svc.fail("MigrateSchema", err) + } + repo, err := NewFieldProfileRepository(svc.iso, svc.schema) + if err != nil { + t.Fatalf("NewFieldProfileRepository: %v", err) + } + + for _, column := range []string{"distinct_values", "pivot_columns", "pivot_value_selectors"} { + t.Run(column, func(t *testing.T) { + paths := [3]string{column + "/enc/null", column + "/enc/empty", column + "/enc/populated"} + encodings := []map[string]string{ + {column: "NULL", "path": fpDDLText(paths[0])}, + {column: "'{}'::text[]", "path": fpDDLText(paths[1])}, + {column: "ARRAY['a','b']::text[]", "path": fpDDLText(paths[2])}, + } + for _, override := range encodings { + fpDDLExec(t, svc, fpDDLProfileStmt(svc, override)) + } + if err := fpDDLTextArrayWitness(t, svc, column, paths); err != nil { + t.Fatalf("real storage failed the three-state witness: %v", err) + } + + unrestricted := true + docs, err := repo.ReadFieldProfileRowsFiltered(svc.ctx, FieldProfileFilter{ + Project: fpDDLProject, DatasetGeneration: fpDDLGen, + ResourceTypes: []string{fpDDLType}, + AuthResourcePathsUnrestricted: &unrestricted, + }) + if err != nil { + svc.fail("ReadFieldProfileRowsFiltered", err) + } + byPath := map[string]catalog.FieldCatalogDocument{} + for _, doc := range docs { + byPath[doc.Path] = doc + } + for _, path := range paths { + if _, ok := byPath[path]; !ok { + t.Fatalf("fixture row %s missing from read-back (read %d rows)", path, len(docs)) + } + } + if got := fpDDLColumnSlice(column, byPath[paths[0]]); got != nil { + t.Errorf("NULL-encoded %s decoded to %#v, want nil", column, got) + } + if got := fpDDLColumnSlice(column, byPath[paths[1]]); got != nil { + t.Errorf("'{}'-encoded %s decoded to %#v, want nil (documented profiler contract)", column, got) + } + if got := fpDDLColumnSlice(column, byPath[paths[2]]); !reflect.DeepEqual(got, []string{"a", "b"}) { + t.Errorf("populated %s decoded to %#v, want [a b]", column, got) + } + + writeScope := FieldProfileScope{Project: fpDDLProject, DatasetGeneration: fpDDLGen, ResourceType: fpDDLType} + doc := fpDDLBaseDoc(column + "/enc/write-nil") + if err := repo.UpsertFieldProfileBatch(svc.ctx, writeScope, []catalog.FieldCatalogDocument{doc}, + FieldProfileCoverage{FieldProfileScope: writeScope}, 0); err != nil { + svc.fail("UpsertFieldProfileBatch nil", err) + } + var storedNull bool + if err := svc.iso.QueryRow(svc.ctx, fmt.Sprintf( + `SELECT %[1]s IS NULL FROM "%s".fhir_field_profile WHERE path=$1`, column, svc.schema), + fmt.Sprintf("%s/enc/write-nil", column), + ).Scan(&storedNull); err != nil { + svc.fail("witness nil write", err) + } + if !storedNull { + t.Errorf("repository wrote a nil %s as non-NULL; the collapse of NULL into '{}' would make never-set and seeded-empty indistinguishable", column) + } + + fpDDLExec(t, svc, fpDDLProfileStmt(svc, map[string]string{ + "path": fpDDLText(column + "/enc/collapsed"), column: "NULL", + })) + collapsedWitnessPaths := [3]string{paths[0], paths[1], column + "/enc/collapsed"} + if _, err := svc.iso.Exec(svc.ctx, fmt.Sprintf( + `UPDATE "%s".fhir_field_profile SET %s = '{}'::text[] WHERE path = $1`, + svc.schema, column), column+"/enc/collapsed"); err != nil { + svc.fail("collapse simulation update", err) + } + if err := fpDDLTextArrayWitness(t, svc, column, collapsedWitnessPaths); err == nil { + t.Fatalf("three-state witness accepted a collapsed '{}' row; the witness is vacuous") + } + }) + } +} + +func fpDDLColumnSlice(column string, doc catalog.FieldCatalogDocument) []string { + switch column { + case "distinct_values": + return doc.DistinctValues + case "pivot_columns": + return doc.PivotColumns + case "pivot_value_selectors": + return doc.PivotValueSelectors + } + panic("unknown text[] column " + column) +} + +func fpDDLBaseDoc(path string) catalog.FieldCatalogDocument { + return catalog.FieldCatalogDocument{ + Key: "pgcat01-ddl-" + path, Project: fpDDLProject, DatasetGeneration: fpDDLGen, + ResourceType: fpDDLType, Path: path, Kind: "string", + } +} + +const ( + fpDDLExtensionPopulated = `[{"url":"http://loom.test/ext/a","source_path":"extension","value_path":"valueString","value_type":"string","url_path":null}]` + fpDDLSemanticPopulated = `[{"schema_version":2,"source":{"canonical":"http://loom.test/cs","type":"coding","path":"code"},"key":{"system":"http://loom.test/cs","code":"C1","display":"One"},"value":{"selector":"code","type":"code","fallback_selectors":null},"completeness":"complete","population":5,"examples_truncated":false}]` +) + +func fpDDLJSONBWitness(t *testing.T, svc *migrationService, column string, paths [3]string) error { + t.Helper() + want := [3][3]bool{ + {true, false, false}, + {false, true, false}, + {false, false, true}, + } + for i, path := range paths { + var isNull, isEmpty, isPopulated bool + if err := svc.iso.QueryRow(svc.ctx, fmt.Sprintf( + `SELECT %[1]s IS NULL, coalesce(%[1]s = '[]'::jsonb, false), coalesce(jsonb_typeof(%[1]s) = 'array' AND %[1]s <> '[]'::jsonb, false) + FROM "%[2]s".fhir_field_profile WHERE project=$1 AND dataset_generation=$2 AND resource_type=$3 AND path=$4`, + column, svc.schema), + fpDDLProject, fpDDLGen, fpDDLType, path).Scan(&isNull, &isEmpty, &isPopulated); err != nil { + return fmt.Errorf("witness row %s: %w", path, err) + } + got := [3]bool{isNull, isEmpty, isPopulated} + if got != want[i] { + return fmt.Errorf("column %s row %s encodings = %v, want %v (NULL/'[]'/populated must be three distinct states)", + column, path, got, want[i]) + } + } + return nil +} + +func TestFieldProfileJSONBEncodingsDistinguishThreeStates(t *testing.T) { + svc := newMigrationService(t) + if _, err := MigrateSchema(svc.ctx, svc.openClient(t), svc.schema); err != nil { + svc.fail("MigrateSchema", err) + } + repo, err := NewFieldProfileRepository(svc.iso, svc.schema) + if err != nil { + t.Fatalf("NewFieldProfileRepository: %v", err) + } + populatedLiteral := map[string]string{ + "extension_values": fpDDLText(fpDDLExtensionPopulated), + "semantic_observations": fpDDLText(fpDDLSemanticPopulated), + } + + for _, column := range []string{"extension_values", "semantic_observations"} { + t.Run(column, func(t *testing.T) { + paths := [3]string{column + "/json/null", column + "/json/empty", column + "/json/populated"} + encodings := []map[string]string{ + {column: "NULL", "path": fpDDLText(paths[0])}, + {column: "'[]'::jsonb", "path": fpDDLText(paths[1])}, + {column: populatedLiteral[column], "path": fpDDLText(paths[2])}, + } + for _, override := range encodings { + fpDDLExec(t, svc, fpDDLProfileStmt(svc, override)) + } + if err := fpDDLJSONBWitness(t, svc, column, paths); err != nil { + t.Fatalf("real storage failed the three-state witness: %v", err) + } + + unrestricted := true + docs, err := repo.ReadFieldProfileRowsFiltered(svc.ctx, FieldProfileFilter{ + Project: fpDDLProject, DatasetGeneration: fpDDLGen, + ResourceTypes: []string{fpDDLType}, + AuthResourcePathsUnrestricted: &unrestricted, + }) + if err != nil { + svc.fail("ReadFieldProfileRowsFiltered", err) + } + byPath := map[string]catalog.FieldCatalogDocument{} + for _, doc := range docs { + byPath[doc.Path] = doc + } + if len(byPath) < 3 { + t.Fatalf("read back %d rows, want at least 3", len(byPath)) + } + if got := fpDDLJSONBLen(column, byPath[paths[0]]); got != 0 || !fpDDLJSONBIsNil(column, byPath[paths[0]]) { + t.Errorf("NULL-encoded %s must decode to a nil slice, got len %d nil=%v", column, got, fpDDLJSONBIsNil(column, byPath[paths[0]])) + } + if fpDDLJSONBIsNil(column, byPath[paths[1]]) || fpDDLJSONBLen(column, byPath[paths[1]]) != 0 { + t.Errorf("'[]'-encoded %s must decode to an empty-but-non-nil slice, got len %d nil=%v", + column, fpDDLJSONBLen(column, byPath[paths[1]]), fpDDLJSONBIsNil(column, byPath[paths[1]])) + } + if fpDDLJSONBLen(column, byPath[paths[2]]) != 1 { + t.Fatalf("populated %s decoded to %d elements, want 1", column, fpDDLJSONBLen(column, byPath[paths[2]])) + } + if column == "extension_values" { + want := catalog.ExtensionValueObservation{ + URL: "http://loom.test/ext/a", SourcePath: "extension", ValuePath: "valueString", ValueType: "string", + } + if got := byPath[paths[2]].ExtensionValues[0]; !reflect.DeepEqual(got, want) { + t.Errorf("populated extension decoded to %+v, want %+v", got, want) + } + } else { + got := byPath[paths[2]].SemanticObservations[0] + if got.SchemaVersion != 2 || got.Key.Code != "C1" || got.Completeness != catalog.SemanticComplete || got.Population != 5 { + t.Errorf("populated semantic observation decoded to %+v", got) + } + } + + writeScope := FieldProfileScope{Project: fpDDLProject, DatasetGeneration: fpDDLGen, ResourceType: fpDDLType} + nilDoc := fpDDLBaseDoc(column + "/json/write-nil") + emptyDoc := fpDDLBaseDoc(column + "/json/write-empty") + popDoc := fpDDLBaseDoc(column + "/json/write-populated") + if column == "extension_values" { + emptyDoc.ExtensionValues = []catalog.ExtensionValueObservation{} + popDoc.ExtensionValues = []catalog.ExtensionValueObservation{ + {URL: "http://loom.test/ext/b", SourcePath: "extension", ValuePath: "valueCode", ValueType: "code"}, + } + } else { + emptyDoc.SemanticObservations = []catalog.SemanticObservation{} + popDoc.SemanticObservations = []catalog.SemanticObservation{ + {SchemaVersion: 2, Key: catalog.SemanticObservationKey{Code: "C2"}, + Completeness: catalog.SemanticPartial, Population: 1}, + } + } + if err := repo.UpsertFieldProfileBatch(svc.ctx, writeScope, + []catalog.FieldCatalogDocument{nilDoc, emptyDoc, popDoc}, + FieldProfileCoverage{FieldProfileScope: writeScope}, 0); err != nil { + svc.fail("UpsertFieldProfileBatch jsonb encodings", err) + } + writePaths := [3]string{column + "/json/write-nil", column + "/json/write-empty", column + "/json/write-populated"} + if err := fpDDLJSONBWitness(t, svc, column, writePaths); err != nil { + t.Fatalf("repository encodings failed the three-state witness: %v", err) + } + + fpDDLExec(t, svc, fpDDLProfileStmt(svc, map[string]string{ + "path": fpDDLText(column + "/json/collapsed"), column: "NULL", + })) + if _, err := svc.iso.Exec(svc.ctx, fmt.Sprintf( + `UPDATE "%s".fhir_field_profile SET %s = '[]'::jsonb WHERE path = $1`, + svc.schema, column), column+"/json/collapsed"); err != nil { + svc.fail("collapse simulation update", err) + } + if err := fpDDLJSONBWitness(t, svc, column, [3]string{paths[0], paths[1], column + "/json/collapsed"}); err == nil { + t.Fatal("three-state witness accepted a collapsed '[]' row; the witness is vacuous") + } + + var raw json.RawMessage + if err := svc.iso.QueryRow(svc.ctx, fmt.Sprintf( + `SELECT %s FROM "%s".fhir_field_profile WHERE path=$1`, column, svc.schema), column+"/json/empty").Scan(&raw); err != nil { + svc.fail("witness raw '[]' row", err) + } + collapsedDoc := catalog.FieldCatalogDocument{} + collapsedDecode := func() { + if column == "extension_values" { + _ = json.Unmarshal(raw, &collapsedDoc.ExtensionValues) + if len(collapsedDoc.ExtensionValues) == 0 { + collapsedDoc.ExtensionValues = nil + } + } else { + _ = json.Unmarshal(raw, &collapsedDoc.SemanticObservations) + if len(collapsedDoc.SemanticObservations) == 0 { + collapsedDoc.SemanticObservations = nil + } + } + } + collapsedDecode() + if !fpDDLJSONBIsNil(column, collapsedDoc) { + t.Fatal("collapsed decoder variant did not collapse '[]' to nil; probe B is vacuous") + } + realDoc := catalog.FieldCatalogDocument{} + if err := decodeFieldProfileJSON(&realDoc, map[string]json.RawMessage{ + "extension_values": raw, "semantic_observations": nil, + }[column], map[string]json.RawMessage{ + "extension_values": nil, "semantic_observations": raw, + }[column]); err != nil { + svc.fail("real decode of '[]' row", err) + } + if fpDDLJSONBIsNil(column, realDoc) { + t.Fatal("the real decoder collapsed the stored '[]' row to nil; the round trip is broken") + } + }) + } +} + +func fpDDLJSONBLen(column string, doc catalog.FieldCatalogDocument) int { + if column == "extension_values" { + return len(doc.ExtensionValues) + } + return len(doc.SemanticObservations) +} + +func fpDDLJSONBIsNil(column string, doc catalog.FieldCatalogDocument) bool { + if column == "extension_values" { + return doc.ExtensionValues == nil + } + return doc.SemanticObservations == nil +} + +func TestFieldProfileCoverageMarkerDistinguishesNeverProfiledFromKnownEmpty(t *testing.T) { + svc := newMigrationService(t) + if _, err := MigrateSchema(svc.ctx, svc.openClient(t), svc.schema); err != nil { + svc.fail("MigrateSchema", err) + } + repo, err := NewFieldProfileRepository(svc.iso, svc.schema) + if err != nil { + t.Fatalf("NewFieldProfileRepository: %v", err) + } + + neverScope := FieldProfileScope{Project: fpDDLProject, DatasetGeneration: fpDDLGen, AuthResourcePath: "cov/never", ResourceType: fpDDLType} + emptyScope := FieldProfileScope{Project: fpDDLProject, DatasetGeneration: fpDDLGen, AuthResourcePath: "cov/known-empty", ResourceType: fpDDLType} + + complete := true + if err := repo.UpsertFieldProfileBatch(svc.ctx, emptyScope, nil, + FieldProfileCoverage{FieldProfileScope: emptyScope, EnrichmentComplete: &complete, EnrichmentTruncated: false}, 0); err != nil { + svc.fail("UpsertFieldProfileBatch known-empty", err) + } + + witness := func() error { + neverRows, err := repo.ReadFieldProfileRows(svc.ctx, neverScope) + if err != nil { + return fmt.Errorf("read never-profiled rows: %w", err) + } + emptyRows, err := repo.ReadFieldProfileRows(svc.ctx, emptyScope) + if err != nil { + return fmt.Errorf("read known-empty rows: %w", err) + } + if len(neverRows) != 0 || len(emptyRows) != 0 { + return fmt.Errorf("both scopes must sit at zero profile rows to test the collapse point, got %d and %d", + len(neverRows), len(emptyRows)) + } + restricted := false + neverCoverage, err := repo.ReadProfileCoverage(svc.ctx, FieldProfileFilter{ + Project: fpDDLProject, DatasetGeneration: fpDDLGen, + ResourceTypes: []string{fpDDLType}, AuthResourcePathsUnrestricted: &restricted, + AuthResourcePaths: []string{"cov/never"}, + }) + if err != nil { + return fmt.Errorf("read never-profiled coverage: %w", err) + } + emptyCoverage, err := repo.ReadProfileCoverage(svc.ctx, FieldProfileFilter{ + Project: fpDDLProject, DatasetGeneration: fpDDLGen, + ResourceTypes: []string{fpDDLType}, AuthResourcePathsUnrestricted: &restricted, + AuthResourcePaths: []string{"cov/known-empty"}, + }) + if err != nil { + return fmt.Errorf("read known-empty coverage: %w", err) + } + if len(neverCoverage) != 0 { + return fmt.Errorf("never-profiled scope carries %d coverage markers, want 0", len(neverCoverage)) + } + if len(emptyCoverage) != 1 { + return fmt.Errorf("known-empty scope carries %d coverage markers, want 1", len(emptyCoverage)) + } + rec := emptyCoverage[0] + if rec.EnrichmentComplete == nil || *rec.EnrichmentComplete != true || rec.EnrichmentTruncated { + return fmt.Errorf("known-empty marker = %+v, want EnrichmentComplete=true truncated=false", rec) + } + return nil + } + + if err := witness(); err != nil { + t.Fatalf("coverage marker witness failed: %v", err) + } + + if _, err := svc.iso.Exec(svc.ctx, fmt.Sprintf( + `DELETE FROM "%s".fhir_field_profile_coverage WHERE auth_resource_path = 'cov/known-empty'`, + svc.schema)); err != nil { + svc.fail("remove marker row", err) + } + if err := witness(); err == nil { + t.Fatal("witness accepted a state where known-empty and never-profiled are indistinguishable; the probe is vacuous") + } +} diff --git a/internal/backends/postgres/internal/store/field_profile_filter_golden_test.go b/internal/backends/postgres/internal/store/field_profile_filter_golden_test.go new file mode 100644 index 000000000..7e240935a --- /dev/null +++ b/internal/backends/postgres/internal/store/field_profile_filter_golden_test.go @@ -0,0 +1,169 @@ +package postgres + +import ( + "errors" + "reflect" + "strings" + "testing" +) + +func goldenBoolPtr(v bool) *bool { return &v } + +func TestFieldProfileFilterSQLGoldenPins(t *testing.T) { + types := []string{"Patient", "Observation"} + paths := []string{"/fhir/clinical", "/fhir/finance"} + cases := []struct { + name string + filter FieldProfileFilter + wantSQL string + wantArgs []any + }{ + { + name: "declared-list restricted keeps the 10d2ce6d render", + filter: FieldProfileFilter{ + Project: "p1", DatasetGeneration: "generation-a", + ResourceTypes: types, + AuthResourcePathsUnrestricted: goldenBoolPtr(false), + AuthResourcePaths: paths, + }, + wantSQL: "project = $1 AND dataset_generation = $2 AND resource_type = ANY($3::text[]) AND auth_resource_path = ANY($4::text[])", + wantArgs: []any{"p1", "generation-a", + append([]string(nil), types...), append([]string(nil), paths...)}, + }, + { + name: "declared-list unrestricted keeps the 10d2ce6d render", + filter: FieldProfileFilter{ + Project: "p1", DatasetGeneration: "", + ResourceTypes: types, + AuthResourcePathsUnrestricted: goldenBoolPtr(true), + }, + wantSQL: "project = $1 AND dataset_generation = $2 AND resource_type = ANY($3::text[])", + wantArgs: []any{"p1", "", append([]string(nil), types...)}, + }, + { + name: "all-types unrestricted renders the reference scope verbatim", + filter: FieldProfileFilter{ + Project: "p1", DatasetGeneration: "generation-a", + AllResourceTypes: true, + AuthResourcePathsUnrestricted: goldenBoolPtr(true), + }, + wantSQL: "project = $1 AND dataset_generation = $2", + wantArgs: []any{"p1", "generation-a"}, + }, + { + name: "all-types restricted shifts auth paths to $3", + filter: FieldProfileFilter{ + Project: "p1", DatasetGeneration: "generation-a", + AllResourceTypes: true, + AuthResourcePathsUnrestricted: goldenBoolPtr(false), + AuthResourcePaths: paths, + }, + wantSQL: "project = $1 AND dataset_generation = $2 AND auth_resource_path = ANY($3::text[])", + wantArgs: []any{"p1", "generation-a", append([]string(nil), paths...)}, + }, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + passes := 1 + if !tc.filter.AllResourceTypes { + passes = 2 + } + for pass := 0; pass < passes; pass++ { + filter := tc.filter + if pass == 1 { + filter.AllResourceTypes = false + } + sql, args, empty, err := fieldProfileFilterSQL(filter) + if err != nil { + t.Fatalf("render (pass %d): %v", pass, err) + } + if empty { + t.Fatalf("render (pass %d): expected a rendered read, got read-nothing", pass) + } + if sql != tc.wantSQL { + t.Fatalf("golden SQL drift (pass %d):\n got %q\nwant %q", pass, sql, tc.wantSQL) + } + if !reflect.DeepEqual(args, tc.wantArgs) { + t.Fatalf("golden args drift (pass %d): got %#v want %#v", pass, args, tc.wantArgs) + } + } + }) + } +} + +func TestFieldProfileFilterSQLNoResourceTypeFragmentAllTypes(t *testing.T) { + for _, unrestricted := range []bool{true, false} { + filter := FieldProfileFilter{ + Project: "p1", DatasetGeneration: "generation-a", + AllResourceTypes: true, + AuthResourcePathsUnrestricted: goldenBoolPtr(unrestricted), + } + if !unrestricted { + filter.AuthResourcePaths = []string{"/fhir/clinical"} + } + sql, _, empty, err := fieldProfileFilterSQL(filter) + if err != nil || empty { + t.Fatalf("all-types door (unrestricted=%v): sql=%q empty=%v err=%v", unrestricted, sql, empty, err) + } + if strings.Contains(sql, "resource_type") { + t.Fatalf("all-types render (unrestricted=%v) carries a residual resource_type predicate: %q", unrestricted, sql) + } + } +} + +func TestFieldProfileFilterSQLAllResourceTypesMutualExclusion(t *testing.T) { + for _, door := range []string{"unrestricted", "restricted"} { + filter := FieldProfileFilter{ + Project: "p1", + DatasetGeneration: "generation-a", + ResourceTypes: []string{"Patient"}, + AllResourceTypes: true, + } + if door == "unrestricted" { + filter.AuthResourcePathsUnrestricted = goldenBoolPtr(true) + } else { + filter.AuthResourcePathsUnrestricted = goldenBoolPtr(false) + filter.AuthResourcePaths = []string{"/fhir/clinical"} + } + sql, args, empty, err := fieldProfileFilterSQL(filter) + if err == nil { + t.Fatalf("%s door: AllResourceTypes + declared list rendered %q (empty=%v, args=%#v); want the typed refusal", door, sql, empty, args) + } + if !strings.Contains(err.Error(), "AllResourceTypes must not accompany a declared resource types list") { + t.Fatalf("%s door: refusal %q must name the conflict", door, err) + } + if !errors.Is(err, ErrInvalidFieldProfileScope) { + t.Fatalf("%s door: refusal %v must wrap ErrInvalidFieldProfileScope", door, err) + } + } +} + +func TestFieldProfileFilterSQLReadNothingDoorsUnchanged(t *testing.T) { + cases := []struct { + name string + filter FieldProfileFilter + }{ + {"declared-list empty types", FieldProfileFilter{ + Project: "p1", ResourceTypes: nil, AuthResourcePathsUnrestricted: goldenBoolPtr(true), + }}, + {"restricted without paths (declared list)", FieldProfileFilter{ + Project: "p1", ResourceTypes: []string{"Patient"}, AuthResourcePathsUnrestricted: goldenBoolPtr(false), + }}, + {"restricted without paths (all types)", FieldProfileFilter{ + Project: "p1", AllResourceTypes: true, AuthResourcePathsUnrestricted: goldenBoolPtr(false), + }}, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + sql, args, empty, err := fieldProfileFilterSQL(tc.filter) + if err != nil { + t.Fatalf("render: %v", err) + } + if !empty || sql != "" || args != nil { + t.Fatalf("read-nothing door returned sql=%q args=%#v empty=%v", sql, args, empty) + } + }) + } +} diff --git a/internal/backends/postgres/internal/store/inventory.go b/internal/backends/postgres/internal/store/inventory.go new file mode 100644 index 000000000..e34fe9d4b --- /dev/null +++ b/internal/backends/postgres/internal/store/inventory.go @@ -0,0 +1,573 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "sort" + "strconv" + "strings" + + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + inventoryCountTableName = "fhir_resource_inventory_count" + inventoryBuildTableName = "fhir_resource_inventory_build" + MaxInventoryWriteBatch = 1000 + defaultInventoryWriteBatch = 1000 +) + +var ( + ErrInvalidInventoryRow = errors.New("postgres: invalid inventory row") + ErrInvalidInventoryScope = errors.New("postgres: invalid inventory scope") + ErrInventoryCountConflict = errors.New("postgres: inventory count conflict") + ErrInventoryPublishGuard = errors.New("postgres: inventory publish guard failed") +) + +type InventorySourceGroup struct { + AuthResourcePath string + DocumentCount int64 +} + +type InventoryCountScope struct { + Project string + DatasetGeneration string + BuildID string + ResourceTypes []string + AuthResourcePathsUnrestricted *bool + AuthResourcePaths []string +} + +type InventoryCountConflictError struct { + BuildID string + Project string + DatasetGeneration string + ResourceType string + AuthResourcePath string + Field string + Stored string + Requested string +} + +func (e *InventoryCountConflictError) Error() string { + return fmt.Sprintf("postgres: inventory count %s/%s (type %s, auth path %q) already holds build %s with %s=%s; "+ + "reinsert requested %s=%s and never replaces the stored row", + e.Project, e.DatasetGeneration, e.ResourceType, e.AuthResourcePath, + e.BuildID, e.Field, e.Stored, e.Field, e.Requested) +} + +func (e *InventoryCountConflictError) Unwrap() error { return ErrInventoryCountConflict } + +type InventoryPublishGuardError struct { + BuildID string + Project string + DatasetGeneration string + ClaimedRowCount int + CommittedRowCount int + MissingResourceTypes []string +} + +func (e *InventoryPublishGuardError) Error() string { + return fmt.Sprintf("postgres: cannot publish inventory marker for %s/%s build %s: "+ + "marker claims %d count rows, %d committed; missing resource types: %s; "+ + "commit every count row first — the marker is published last", + e.Project, e.DatasetGeneration, e.BuildID, + e.ClaimedRowCount, e.CommittedRowCount, strings.Join(e.MissingResourceTypes, ", ")) +} + +func (e *InventoryPublishGuardError) Unwrap() error { return ErrInventoryPublishGuard } + +type InventoryRepository struct { + src ConnSource + countFQN string + buildFQN string + sourceFQN string +} + +func NewInventoryRepository(src ConnSource, schema string) (*InventoryRepository, error) { + if src == nil { + return nil, errors.New("postgres: nil inventory connection source") + } + if !ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", ErrInvalidSchemaName, schema) + } + return &InventoryRepository{ + src: src, + countFQN: pgx.Identifier{schema, inventoryCountTableName}.Sanitize(), + buildFQN: pgx.Identifier{schema, inventoryBuildTableName}.Sanitize(), + sourceFQN: pgx.Identifier{schema, resourceTableName}.Sanitize(), + }, nil +} + +func (r *InventoryRepository) ScanSourceCounts(ctx context.Context, project, datasetGeneration, resourceType string) ([]InventorySourceGroup, error) { + if strings.TrimSpace(project) == "" { + return nil, fmt.Errorf("%w: project is blank", ErrInvalidInventoryScope) + } + if strings.TrimSpace(resourceType) == "" { + return nil, fmt.Errorf("%w: resource type is blank", ErrInvalidInventoryScope) + } + sql := "SELECT auth_resource_path, COUNT(*)::int8 FROM " + r.sourceFQN + + " WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3" + + " GROUP BY auth_resource_path ORDER BY auth_resource_path" + + conn, err := r.acquire(ctx) + if err != nil { + return nil, err + } + defer conn.Release() + + rows, err := conn.Query(ctx, sql, project, datasetGeneration, resourceType) + if err != nil { + return nil, fmt.Errorf("postgres: scan inventory source %s/%s/%s: %w", + project, datasetGeneration, resourceType, err) + } + defer rows.Close() + + var out []InventorySourceGroup + for rows.Next() { + var group InventorySourceGroup + if err := rows.Scan(&group.AuthResourcePath, &group.DocumentCount); err != nil { + return nil, fmt.Errorf("postgres: scan inventory source %s/%s/%s: %w", + project, datasetGeneration, resourceType, err) + } + if group.DocumentCount < 0 { + return nil, fmt.Errorf("postgres: scan inventory source %s/%s/%s: negative count for path %q", + project, datasetGeneration, resourceType, group.AuthResourcePath) + } + out = append(out, group) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("postgres: scan inventory source %s/%s/%s: %w", + project, datasetGeneration, resourceType, err) + } + return out, nil +} + +func (r *InventoryRepository) InsertCounts(ctx context.Context, rows []catalog.ResourceInventoryCount, batchSize int) error { + if len(rows) == 0 { + return fmt.Errorf("%w: no count rows to insert", ErrInvalidInventoryRow) + } + if batchSize <= 0 { + batchSize = defaultInventoryWriteBatch + } + if batchSize > MaxInventoryWriteBatch { + return fmt.Errorf("%w: batch size is %d, want 1..%d", ErrInvalidInventoryScope, batchSize, MaxInventoryWriteBatch) + } + seen := make(map[string]struct{}, len(rows)) + for i, row := range rows { + if err := validateInventoryCountRow(row); err != nil { + return fmt.Errorf("postgres: inventory count row %d: %w", i, err) + } + identity := strings.Join([]string{row.BuildID, row.Project, row.DatasetGeneration, row.ResourceType, row.AuthResourcePath}, "\x00") + if _, dup := seen[identity]; dup { + return fmt.Errorf("%w: duplicate count scope %s/%s in one insert (type %q, auth path %q)", + ErrInvalidInventoryRow, row.Project, row.DatasetGeneration, row.ResourceType, row.AuthResourcePath) + } + seen[identity] = struct{}{} + } + + for start := 0; start < len(rows); start += batchSize { + end := min(start+batchSize, len(rows)) + if err := r.insertCountChunk(ctx, rows[start:end]); err != nil { + return err + } + } + return nil +} + +func (r *InventoryRepository) insertCountChunk(ctx context.Context, chunk []catalog.ResourceInventoryCount) error { + conn, err := r.acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + + tx, err := conn.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return fmt.Errorf("postgres: begin inventory count chunk: %w", err) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback(ctx) + } + }() + + var sql strings.Builder + sql.WriteString("INSERT INTO " + r.countFQN + + " (count_key, build_id, project, dataset_generation, resource_type, auth_resource_path, document_count) VALUES ") + args := make([]any, 0, len(chunk)*7) + for i, row := range chunk { + if i > 0 { + sql.WriteString(", ") + } + base := i * 7 + sql.WriteString("(") + for p := 1; p <= 7; p++ { + if p > 1 { + sql.WriteString(", ") + } + sql.WriteString("$") + sql.WriteString(strconv.Itoa(base + p)) + } + sql.WriteString(")") + args = append(args, row.Key, row.BuildID, row.Project, row.DatasetGeneration, + row.ResourceType, row.AuthResourcePath, row.DocumentCount) + } + sql.WriteString(" ON CONFLICT (build_id, project, dataset_generation, resource_type, auth_resource_path)" + + " DO UPDATE SET document_count = EXCLUDED.document_count" + + " WHERE " + inventoryCountTableName + ".document_count = EXCLUDED.document_count" + + " AND " + inventoryCountTableName + ".count_key = EXCLUDED.count_key" + + " RETURNING count_key") + + returningRows, err := tx.Query(ctx, sql.String(), args...) + if err != nil { + return inventoryWriteError("insert inventory counts", chunk, err) + } + returned := make(map[string]struct{}, len(chunk)) + var returnErr error + for returningRows.Next() { + var key string + if err := returningRows.Scan(&key); err != nil { + returningRows.Close() + return fmt.Errorf("postgres: insert inventory counts: %w", err) + } + returned[key] = struct{}{} + } + returningRows.Close() + returnErr = returningRows.Err() + if returnErr != nil { + return inventoryWriteError("insert inventory counts", chunk, returnErr) + } + + if len(returned) != len(chunk) { + for _, row := range chunk { + if _, ok := returned[row.Key]; ok { + continue + } + storedKey, storedCount, found, err := selectInventoryCountRow(ctx, tx, r.countFQN, row) + if err != nil { + return err + } + if !found { + return fmt.Errorf("postgres: insert inventory counts: row %s vanished before conflict inspection", row.Key) + } + if storedKey != row.Key { + return &InventoryCountConflictError{ + BuildID: row.BuildID, Project: row.Project, DatasetGeneration: row.DatasetGeneration, + ResourceType: row.ResourceType, AuthResourcePath: row.AuthResourcePath, + Field: "count_key", Stored: storedKey, Requested: row.Key, + } + } + if storedCount != row.DocumentCount { + return &InventoryCountConflictError{ + BuildID: row.BuildID, Project: row.Project, DatasetGeneration: row.DatasetGeneration, + ResourceType: row.ResourceType, AuthResourcePath: row.AuthResourcePath, + Field: "document_count", Stored: strconv.FormatInt(storedCount, 10), Requested: strconv.FormatInt(row.DocumentCount, 10), + } + } + } + return fmt.Errorf("postgres: insert inventory counts: %d of %d rows returned without a detectable divergence", + len(returned), len(chunk)) + } + + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("postgres: commit inventory count chunk: %w", err) + } + committed = true + return nil +} + +func (r *InventoryRepository) PublishBuild(ctx context.Context, build catalog.ResourceInventoryBuild) error { + if err := validateInventoryBuildMarker(build); err != nil { + return err + } + + conn, err := r.acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + + tx, err := conn.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return fmt.Errorf("postgres: begin inventory marker publish: %w", err) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback(ctx) + } + }() + + var committedRows int + err = tx.QueryRow(ctx, + "SELECT COUNT(*) FROM "+r.countFQN+" WHERE build_id = $1 AND project = $2 AND dataset_generation = $3", + build.BuildID, build.Project, build.DatasetGeneration).Scan(&committedRows) + if err != nil { + return fmt.Errorf("postgres: inventory publish guard count check: %w", err) + } + + declared := append([]string(nil), build.ResourceTypes...) + sort.Strings(declared) + rows, err := tx.Query(ctx, + "SELECT DISTINCT resource_type FROM "+r.countFQN+" WHERE build_id = $1 AND project = $2 AND dataset_generation = $3", + build.BuildID, build.Project, build.DatasetGeneration) + if err != nil { + return fmt.Errorf("postgres: inventory publish guard coverage check: %w", err) + } + defer rows.Close() + present := make(map[string]struct{}, len(declared)) + for rows.Next() { + var resourceType string + if err := rows.Scan(&resourceType); err != nil { + rows.Close() + return fmt.Errorf("postgres: inventory publish guard coverage check: %w", err) + } + present[resourceType] = struct{}{} + } + rows.Close() + if err := rows.Err(); err != nil { + return fmt.Errorf("postgres: inventory publish guard coverage check: %w", err) + } + var missing []string + for _, resourceType := range declared { + if _, ok := present[resourceType]; !ok { + missing = append(missing, resourceType) + } + } + if committedRows != build.CountRowCount || len(missing) > 0 { + return &InventoryPublishGuardError{ + BuildID: build.BuildID, Project: build.Project, DatasetGeneration: build.DatasetGeneration, + ClaimedRowCount: build.CountRowCount, CommittedRowCount: committedRows, MissingResourceTypes: missing, + } + } + + _, err = tx.Exec(ctx, + "INSERT INTO "+r.buildFQN+ + " (marker_key, project, dataset_generation, build_id, coverage_digest, resource_types,"+ + " resource_type_count, count_row_count, complete)"+ + " VALUES ($1, $2, $3, $4, $5, $6::text[], $7, $8, $9)"+ + " ON CONFLICT (project, dataset_generation) DO UPDATE SET"+ + " marker_key = EXCLUDED.marker_key, build_id = EXCLUDED.build_id,"+ + " coverage_digest = EXCLUDED.coverage_digest, resource_types = EXCLUDED.resource_types,"+ + " resource_type_count = EXCLUDED.resource_type_count, count_row_count = EXCLUDED.count_row_count,"+ + " complete = EXCLUDED.complete, published_at = now()", + build.Key, build.Project, build.DatasetGeneration, build.BuildID, build.CoverageDigest, + declared, build.ResourceTypeCount, build.CountRowCount, build.Complete) + if err != nil { + if IsUniqueConstraintViolation(err, inventoryBuildTableName+"_key_ux") { + return fmt.Errorf("postgres: inventory marker key %s collides across scopes (corrupted derivation): %w", + build.Key, err) + } + return fmt.Errorf("postgres: publish inventory marker %s/%s: %w", build.Project, build.DatasetGeneration, err) + } + + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("postgres: commit inventory marker publish: %w", err) + } + committed = true + return nil +} + +func (r *InventoryRepository) ReadBuild(ctx context.Context, project, datasetGeneration string) (catalog.ResourceInventoryBuild, bool, error) { + if strings.TrimSpace(project) == "" { + return catalog.ResourceInventoryBuild{}, false, fmt.Errorf("%w: project is blank", ErrInvalidInventoryScope) + } + sql := "SELECT marker_key, build_id, coverage_digest, resource_types, resource_type_count, count_row_count, complete" + + " FROM " + r.buildFQN + " WHERE project = $1 AND dataset_generation = $2" + + conn, err := r.acquire(ctx) + if err != nil { + return catalog.ResourceInventoryBuild{}, false, err + } + defer conn.Release() + + var build catalog.ResourceInventoryBuild + err = conn.QueryRow(ctx, sql, project, datasetGeneration).Scan( + &build.Key, &build.BuildID, &build.CoverageDigest, &build.ResourceTypes, + &build.ResourceTypeCount, &build.CountRowCount, &build.Complete, + ) + if errors.Is(err, pgx.ErrNoRows) { + return catalog.ResourceInventoryBuild{}, false, nil + } + if err != nil { + return catalog.ResourceInventoryBuild{}, false, fmt.Errorf("postgres: read inventory marker %s/%s: %w", + project, datasetGeneration, err) + } + build.RecordType = catalog.ResourceInventoryBuildRecordType + build.Project = project + build.DatasetGeneration = datasetGeneration + return build, true, nil +} + +func (r *InventoryRepository) ReadCountSums(ctx context.Context, scope InventoryCountScope) (map[string]int64, error) { + if strings.TrimSpace(scope.Project) == "" { + return nil, fmt.Errorf("%w: project is blank", ErrInvalidInventoryScope) + } + if strings.TrimSpace(scope.BuildID) == "" { + return nil, fmt.Errorf("%w: build id is blank", ErrInvalidInventoryScope) + } + if scope.AuthResourcePathsUnrestricted == nil { + return nil, fmt.Errorf( + "%w: AuthResourcePathsUnrestricted must be set explicitly; there is no implicit unrestricted mode", + ErrInvalidInventoryScope) + } + sums := make(map[string]int64, len(scope.ResourceTypes)) + if *scope.AuthResourcePathsUnrestricted { + if len(scope.AuthResourcePaths) > 0 { + return nil, fmt.Errorf( + "%w: unrestricted mode must not carry auth resource paths (%d given)", + ErrInvalidInventoryScope, len(scope.AuthResourcePaths)) + } + } else if len(scope.AuthResourcePaths) == 0 { + return sums, nil + } + if len(scope.ResourceTypes) == 0 { + return sums, nil + } + + sql := "SELECT resource_type, SUM(document_count)::bigint FROM " + r.countFQN + + " WHERE project = $1 AND dataset_generation = $2 AND build_id = $3 AND resource_type = ANY($4::text[])" + args := []any{scope.Project, scope.DatasetGeneration, scope.BuildID, scope.ResourceTypes} + if !*scope.AuthResourcePathsUnrestricted { + sql += " AND auth_resource_path = ANY($5::text[])" + args = append(args, scope.AuthResourcePaths) + } + sql += " GROUP BY resource_type" + + conn, err := r.acquire(ctx) + if err != nil { + return nil, err + } + defer conn.Release() + + rows, err := conn.Query(ctx, sql, args...) + if err != nil { + return nil, fmt.Errorf("postgres: read inventory counts %s/%s build %s: %w", + scope.Project, scope.DatasetGeneration, scope.BuildID, err) + } + defer rows.Close() + for rows.Next() { + var resourceType string + var total int64 + if err := rows.Scan(&resourceType, &total); err != nil { + return nil, fmt.Errorf("postgres: read inventory counts %s/%s build %s: %w", + scope.Project, scope.DatasetGeneration, scope.BuildID, err) + } + if total < 0 { + return nil, fmt.Errorf("postgres: read inventory counts %s/%s build %s: negative sum for type %q", + scope.Project, scope.DatasetGeneration, scope.BuildID, resourceType) + } + sums[resourceType] = total + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("postgres: read inventory counts %s/%s build %s: %w", + scope.Project, scope.DatasetGeneration, scope.BuildID, err) + } + return sums, nil +} + +func selectInventoryCountRow(ctx context.Context, tx pgx.Tx, countFQN string, row catalog.ResourceInventoryCount) (string, int64, bool, error) { + var storedKey string + var storedCount int64 + err := tx.QueryRow(ctx, + "SELECT count_key, document_count FROM "+countFQN+ + " WHERE build_id = $1 AND project = $2 AND dataset_generation = $3 AND resource_type = $4 AND auth_resource_path = $5", + row.BuildID, row.Project, row.DatasetGeneration, row.ResourceType, row.AuthResourcePath, + ).Scan(&storedKey, &storedCount) + if errors.Is(err, pgx.ErrNoRows) { + return "", 0, false, nil + } + if err != nil { + return "", 0, false, fmt.Errorf("postgres: inspect conflicting inventory count %s: %w", row.Key, err) + } + return storedKey, storedCount, true, nil +} + +func inventoryWriteError(op string, chunk []catalog.ResourceInventoryCount, err error) error { + if pgErr, ok := pgErrorFrom(err); ok && pgErr.Code == sqlStateUniqueViolation { + sample := chunk[0] + if pgErr.ConstraintName == inventoryCountTableName+"_key_ux" { + return &InventoryCountConflictError{ + BuildID: sample.BuildID, Project: sample.Project, DatasetGeneration: sample.DatasetGeneration, + ResourceType: sample.ResourceType, AuthResourcePath: sample.AuthResourcePath, + Field: "count_key", Stored: "", Requested: sample.Key, + } + } + return fmt.Errorf("%w: %s: identity unique violation (%s): %w", + ErrInventoryCountConflict, op, pgErr.ConstraintName, err) + } + return fmt.Errorf("postgres: %s: %w", op, err) +} + +func validateInventoryCountRow(row catalog.ResourceInventoryCount) error { + if row.RecordType != catalog.ResourceInventoryCountRecordType { + return fmt.Errorf("%w: record type is %q, want %q", ErrInvalidInventoryRow, row.RecordType, catalog.ResourceInventoryCountRecordType) + } + if strings.TrimSpace(row.Key) == "" { + return fmt.Errorf("%w: count key is blank", ErrInvalidInventoryRow) + } + if strings.TrimSpace(row.BuildID) == "" { + return fmt.Errorf("%w: build id is blank", ErrInvalidInventoryRow) + } + for _, f := range []struct { + name string + value string + }{ + {"project", row.Project}, + {"resource type", row.ResourceType}, + } { + if strings.TrimSpace(f.value) == "" { + return fmt.Errorf("%w: %s is blank", ErrInvalidInventoryRow, f.name) + } + } + if row.DocumentCount < 0 { + return fmt.Errorf("%w: document count is %d, want >= 0", ErrInvalidInventoryRow, row.DocumentCount) + } + return nil +} + +func validateInventoryBuildMarker(build catalog.ResourceInventoryBuild) error { + if build.RecordType != catalog.ResourceInventoryBuildRecordType { + return fmt.Errorf("%w: record type is %q, want %q", ErrInvalidInventoryRow, build.RecordType, catalog.ResourceInventoryBuildRecordType) + } + if !build.Complete { + return fmt.Errorf("%w: marker claims complete=false; the store publishes only complete markers", ErrInvalidInventoryRow) + } + for _, f := range []struct { + name string + value string + }{ + {"marker key", build.Key}, + {"project", build.Project}, + {"build id", build.BuildID}, + {"coverage digest", build.CoverageDigest}, + } { + if strings.TrimSpace(f.value) == "" { + return fmt.Errorf("%w: %s is blank", ErrInvalidInventoryRow, f.name) + } + } + if len(build.ResourceTypes) == 0 { + return fmt.Errorf("%w: marker declares no resource types", ErrInvalidInventoryRow) + } + if build.ResourceTypeCount != len(build.ResourceTypes) { + return fmt.Errorf("%w: resource type count is %d, want %d", ErrInvalidInventoryRow, build.ResourceTypeCount, len(build.ResourceTypes)) + } + if build.CountRowCount < build.ResourceTypeCount { + return fmt.Errorf("%w: count row count %d is below the declared %d resource types", + ErrInvalidInventoryRow, build.CountRowCount, build.ResourceTypeCount) + } + return nil +} + +func (r *InventoryRepository) acquire(ctx context.Context) (*pgxpool.Conn, error) { + conn, err := r.src.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres: acquire inventory connection: %w", err) + } + return conn, nil +} diff --git a/internal/backends/postgres/internal/store/inventory_test.go b/internal/backends/postgres/internal/store/inventory_test.go new file mode 100644 index 000000000..ac1050e47 --- /dev/null +++ b/internal/backends/postgres/internal/store/inventory_test.go @@ -0,0 +1,620 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "net/url" + "regexp" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/catalog" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +var pgcat02StoreNamePattern = regexp.MustCompile(`^pgcat02_it_[0-9]+_[0-9a-f]+$`) + +type inventoryStoreService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func newInventoryStoreService(t *testing.T) *inventoryStoreService { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var dbName, schema string + + t.Cleanup(func() { + var dbCount int + if err := admin.QueryRow(context.WithoutCancel(ctx), + "SELECT COUNT(*) FROM pg_database WHERE datname = $1", dbName).Scan(&dbCount); err != nil { + t.Errorf("audit isolated database %s: %s", dbName, scrub(err)) + } else if dbCount != 0 { + t.Errorf("leftover isolated database %s after cleanup", dbName) + } + var schemaCount int + if err := admin.QueryRow(context.WithoutCancel(ctx), + "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1", schema).Scan(&schemaCount); err != nil { + t.Errorf("audit owned schema %s: %s", schema, scrub(err)) + } else if schemaCount != 0 { + t.Errorf("leftover owned schema %s in the service database after cleanup", schema) + } + }) + + dbName = fmt.Sprintf("pgcat02_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgcat02StoreNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema = fmt.Sprintf("pgcat02_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgcat02StoreNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + return &inventoryStoreService{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} +} + +func (s *inventoryStoreService) migrate(t *testing.T) { + t.Helper() + if _, err := MigrateSchema(s.ctx, s.openClient(t), s.schema); err != nil { + s.fail("MigrateSchema", err) + } +} + +func (s *inventoryStoreService) openClient(t *testing.T) *Client { + t.Helper() + client, err := Open(s.ctx, Config{ + DSN: s.dsn, + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *inventoryStoreService) newRepository(t *testing.T) *InventoryRepository { + t.Helper() + repo, err := NewInventoryRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new inventory repository", err) + } + return repo +} + +func (s *inventoryStoreService) exec(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := s.iso.Exec(s.ctx, sql, args...); err != nil { + s.fail("fixture exec", err) + } +} + +func (s *inventoryStoreService) seedResource(t *testing.T, project, generation, resourceType, logicalID, authPath string) { + t.Helper() + s.exec(t, "INSERT INTO "+pgx.Identifier{s.schema, resourceTableName}.Sanitize()+ + " (project, dataset_generation, resource_type, logical_id, auth_resource_path)"+ + " VALUES ($1, $2, $3, $4, $5)", + project, generation, resourceType, logicalID, authPath) +} + +type countRowWitness struct { + CountKey string + BuildID string + Project string + DatasetGeneration string + ResourceType string + AuthResourcePath string + DocumentCount int64 +} + +func (s *inventoryStoreService) witnessCountRows(t *testing.T, buildID string) []countRowWitness { + t.Helper() + rows, err := s.iso.Query(s.ctx, "SELECT count_key, build_id, project, dataset_generation, resource_type, auth_resource_path, document_count"+ + " FROM "+pgx.Identifier{s.schema, inventoryCountTableName}.Sanitize()+ + " WHERE build_id = $1 ORDER BY resource_type, auth_resource_path", buildID) + if err != nil { + s.fail("witness count rows", err) + } + defer rows.Close() + var out []countRowWitness + for rows.Next() { + var w countRowWitness + if err := rows.Scan(&w.CountKey, &w.BuildID, &w.Project, &w.DatasetGeneration, + &w.ResourceType, &w.AuthResourcePath, &w.DocumentCount); err != nil { + s.fail("witness count rows", err) + } + out = append(out, w) + } + if err := rows.Err(); err != nil { + s.fail("witness count rows", err) + } + return out +} + +type markerWitness struct { + MarkerKey string + Project string + DatasetGeneration string + BuildID string + CoverageDigest string + ResourceTypes []string + ResourceTypeCount int + CountRowCount int + Complete bool +} + +func (s *inventoryStoreService) witnessMarkers(t *testing.T, project, generation string) []markerWitness { + t.Helper() + rows, err := s.iso.Query(s.ctx, "SELECT marker_key, project, dataset_generation, build_id, coverage_digest,"+ + " resource_types, resource_type_count, count_row_count, complete"+ + " FROM "+pgx.Identifier{s.schema, inventoryBuildTableName}.Sanitize()+ + " WHERE project = $1 AND dataset_generation = $2 ORDER BY marker_key", project, generation) + if err != nil { + s.fail("witness markers", err) + } + defer rows.Close() + var out []markerWitness + for rows.Next() { + var w markerWitness + if err := rows.Scan(&w.MarkerKey, &w.Project, &w.DatasetGeneration, &w.BuildID, &w.CoverageDigest, + &w.ResourceTypes, &w.ResourceTypeCount, &w.CountRowCount, &w.Complete); err != nil { + s.fail("witness markers", err) + } + out = append(out, w) + } + if err := rows.Err(); err != nil { + s.fail("witness markers", err) + } + return out +} + +func mustInventoryBuild(t *testing.T, project, generation string, types []string, counts []catalog.ResourceInventoryCount) (catalog.ResourceInventoryBuild, []catalog.ResourceInventoryCount) { + t.Helper() + build, countRows, err := catalog.NewResourceInventoryBuild(project, generation, types, counts) + if err != nil { + t.Fatalf("NewResourceInventoryBuild: %v", err) + } + return build, countRows +} + +func countFor(project, generation, authPath, resourceType string, documentCount int64) catalog.ResourceInventoryCount { + return catalog.ResourceInventoryCount{ + Project: project, DatasetGeneration: generation, AuthResourcePath: authPath, + ResourceType: resourceType, DocumentCount: documentCount, + } +} + +func explicitUnrestricted(v bool) *bool { return &v } + +func TestInventoryScanSourceCountsGroupsAuthPaths(t *testing.T) { + s := newInventoryStoreService(t) + s.migrate(t) + repo := s.newRepository(t) + + s.seedResource(t, "p", "g", "Patient", "p1", "a") + s.seedResource(t, "p", "g", "Patient", "p2", "a") + s.seedResource(t, "p", "g", "Patient", "p3", "b") + s.seedResource(t, "p", "g", "Patient", "p4", "") + s.seedResource(t, "p", "g2", "Patient", "q1", "a") + s.seedResource(t, "p", "g", "Observation", "o1", "z") + s.seedResource(t, "o", "g", "Patient", "x1", "a") + + groups, err := repo.ScanSourceCounts(s.ctx, "p", "g", "Patient") + if err != nil { + t.Fatalf("ScanSourceCounts: %v", err) + } + want := []InventorySourceGroup{{AuthResourcePath: "", DocumentCount: 1}, {AuthResourcePath: "a", DocumentCount: 2}, {AuthResourcePath: "b", DocumentCount: 1}} + if fmt.Sprint(groups) != fmt.Sprint(want) { + t.Fatalf("Patient groups = %v, want %v", groups, want) + } + + groups, err = repo.ScanSourceCounts(s.ctx, "p", "g", "Encounter") + if err != nil { + t.Fatalf("ScanSourceCounts Encounter: %v", err) + } + if len(groups) != 0 { + t.Fatalf("Encounter groups = %v, want none", groups) + } + + if _, err := repo.ScanSourceCounts(s.ctx, "", "g", "Patient"); !errors.Is(err, ErrInvalidInventoryScope) { + t.Fatalf("blank project error = %v, want ErrInvalidInventoryScope", err) + } + if _, err := repo.ScanSourceCounts(s.ctx, "p", "g", " "); !errors.Is(err, ErrInvalidInventoryScope) { + t.Fatalf("blank type error = %v, want ErrInvalidInventoryScope", err) + } +} + +func TestInventoryInsertCountsIsRepeatStableAndRejectsDivergence(t *testing.T) { + s := newInventoryStoreService(t) + s.migrate(t) + repo := s.newRepository(t) + + build, counts := mustInventoryBuild(t, "p", "g", []string{"Patient", "Observation"}, []catalog.ResourceInventoryCount{ + countFor("p", "g", "a", "Patient", 2), + countFor("p", "g", "b", "Patient", 1), + countFor("p", "g", "", "Observation", 0), + }) + + if err := repo.InsertCounts(s.ctx, counts, 2); err != nil { + t.Fatalf("first InsertCounts: %v", err) + } + rows := s.witnessCountRows(t, build.BuildID) + if len(rows) != 3 { + t.Fatalf("committed count rows = %d, want 3", len(rows)) + } + want := []countRowWitness{ + {CountKey: counts[0].Key, BuildID: build.BuildID, Project: "p", DatasetGeneration: "g", ResourceType: "Observation", AuthResourcePath: "", DocumentCount: 0}, + {CountKey: counts[1].Key, BuildID: build.BuildID, Project: "p", DatasetGeneration: "g", ResourceType: "Patient", AuthResourcePath: "a", DocumentCount: 2}, + {CountKey: counts[2].Key, BuildID: build.BuildID, Project: "p", DatasetGeneration: "g", ResourceType: "Patient", AuthResourcePath: "b", DocumentCount: 1}, + } + if fmt.Sprint(rows) != fmt.Sprint(want) { + t.Fatalf("committed rows = %v, want %v", rows, want) + } + for _, row := range rows { + if !strings.HasPrefix(row.CountKey, "ric_") { + t.Fatalf("count key %q is not a ric_ identity", row.CountKey) + } + } + + if err := repo.InsertCounts(s.ctx, counts, 2); err != nil { + t.Fatalf("repeat InsertCounts: %v", err) + } + if again := s.witnessCountRows(t, build.BuildID); len(again) != 3 { + t.Fatalf("rows after repeat = %d, want 3", len(again)) + } + + s.exec(t, "UPDATE "+pgx.Identifier{s.schema, inventoryCountTableName}.Sanitize()+ + " SET document_count = 99 WHERE count_key = $1", counts[1].Key) + err := repo.InsertCounts(s.ctx, counts, 2) + var conflict *InventoryCountConflictError + if !errors.As(err, &conflict) || !errors.Is(err, ErrInventoryCountConflict) { + t.Fatalf("divergent insert error = %v, want *InventoryCountConflictError", err) + } + if conflict.Field != "document_count" || conflict.Stored != "99" || conflict.Requested != "2" { + t.Fatalf("conflict detail = %+v", conflict) + } + var stored int64 + if err := s.iso.QueryRow(s.ctx, "SELECT document_count FROM "+pgx.Identifier{s.schema, inventoryCountTableName}.Sanitize()+ + " WHERE count_key = $1", counts[1].Key).Scan(&stored); err != nil { + s.fail("read tampered row", err) + } + if stored != 99 { + t.Fatalf("stored count = %d, want the tampered 99 (insert must never replace)", stored) + } + + forged := catalog.ResourceInventoryCount{ + Key: "ric_forged_samecount", RecordType: catalog.ResourceInventoryCountRecordType, + BuildID: counts[0].BuildID, Project: counts[0].Project, DatasetGeneration: counts[0].DatasetGeneration, + AuthResourcePath: counts[0].AuthResourcePath, ResourceType: counts[0].ResourceType, + DocumentCount: counts[0].DocumentCount, + } + err = repo.InsertCounts(s.ctx, []catalog.ResourceInventoryCount{forged}, 10) + var keyConflict *InventoryCountConflictError + if !errors.As(err, &keyConflict) || keyConflict.Field != "count_key" || + keyConflict.Stored != counts[0].Key || keyConflict.Requested != "ric_forged_samecount" { + t.Fatalf("forged-key reinsert error = %v, want typed count_key conflict", err) + } + var storedKey string + if err := s.iso.QueryRow(s.ctx, "SELECT count_key FROM "+pgx.Identifier{s.schema, inventoryCountTableName}.Sanitize()+ + " WHERE build_id = $1 AND project = $2 AND dataset_generation = $3 AND resource_type = $4 AND auth_resource_path = $5", + counts[0].BuildID, counts[0].Project, counts[0].DatasetGeneration, counts[0].ResourceType, counts[0].AuthResourcePath, + ).Scan(&storedKey); err != nil { + s.fail("read stored key", err) + } + if storedKey != counts[0].Key { + t.Fatalf("stored count key = %q, want the authentic %q (never replaced)", storedKey, counts[0].Key) + } + + if err := repo.InsertCounts(s.ctx, []catalog.ResourceInventoryCount{counts[0]}, 10); err != nil { + t.Fatalf("identical single-row reinsert: %v", err) + } + + if err := repo.InsertCounts(s.ctx, nil, 2); !errors.Is(err, ErrInvalidInventoryRow) { + t.Fatalf("empty insert error = %v, want ErrInvalidInventoryRow", err) + } + if err := repo.InsertCounts(s.ctx, counts, MaxInventoryWriteBatch+1); !errors.Is(err, ErrInvalidInventoryScope) { + t.Fatalf("oversized batch error = %v, want ErrInvalidInventoryScope", err) + } + duplicate := append([]catalog.ResourceInventoryCount{}, counts...) + duplicate = append(duplicate, counts[0]) + if err := repo.InsertCounts(s.ctx, duplicate, 10); !errors.Is(err, ErrInvalidInventoryRow) { + t.Fatalf("duplicate scope error = %v, want ErrInvalidInventoryRow", err) + } + negative := counts + negative[0].DocumentCount = -1 + if err := repo.InsertCounts(s.ctx, negative, 10); !errors.Is(err, ErrInvalidInventoryRow) { + t.Fatalf("negative count error = %v, want ErrInvalidInventoryRow", err) + } +} + +func TestInventoryRepositoryRejectsCrossScopeIdentityCollisions(t *testing.T) { + s := newInventoryStoreService(t) + s.migrate(t) + repo := s.newRepository(t) + + _, counts := mustInventoryBuild(t, "p", "g", []string{"Patient"}, + []catalog.ResourceInventoryCount{countFor("p", "g", "a", "Patient", 2)}) + if err := repo.InsertCounts(s.ctx, counts, 10); err != nil { + t.Fatalf("baseline InsertCounts: %v", err) + } + forgedKeyRow := catalog.ResourceInventoryCount{ + Key: counts[0].Key, RecordType: catalog.ResourceInventoryCountRecordType, + BuildID: "sha256:" + strings.Repeat("f", 64), Project: "p", DatasetGeneration: "g", + AuthResourcePath: "z", ResourceType: "Patient", DocumentCount: 1, + } + err := repo.InsertCounts(s.ctx, []catalog.ResourceInventoryCount{forgedKeyRow}, 10) + if !errors.Is(err, ErrInventoryCountConflict) { + t.Fatalf("cross-scope count_key insert error = %v, want ErrInventoryCountConflict", err) + } + + build1, counts1 := mustInventoryBuild(t, "p", "g", []string{"Patient"}, + []catalog.ResourceInventoryCount{countFor("p", "g", "b", "Patient", 1)}) + if err := repo.PublishBuild(s.ctx, build1); !errors.Is(err, ErrInventoryPublishGuard) { + t.Fatalf("publish before own rows = %v, want guard error", err) + } + if err := repo.InsertCounts(s.ctx, counts1, 10); err != nil { + t.Fatalf("commit build1 rows: %v", err) + } + build1.Key = "ribm_forged_cross_scope" + if err := repo.PublishBuild(s.ctx, build1); err != nil { + t.Fatalf("publish build1 with forged key: %v", err) + } + build2, counts2 := mustInventoryBuild(t, "p", "g2", []string{"Patient"}, + []catalog.ResourceInventoryCount{countFor("p", "g2", "c", "Patient", 4)}) + if err := repo.InsertCounts(s.ctx, counts2, 10); err != nil { + t.Fatalf("commit build2 rows: %v", err) + } + build2.Key = "ribm_forged_cross_scope" + err = repo.PublishBuild(s.ctx, build2) + if err == nil || !strings.Contains(err.Error(), "collides across scopes") { + t.Fatalf("cross-scope marker_key publish error = %v, want corrupted-derivation classification", err) + } + if markers := s.witnessMarkers(t, "p", "g2"); len(markers) != 0 { + t.Fatalf("markers for g2 after collision = %+v, want none", markers) + } + markers := s.witnessMarkers(t, "p", "g") + if len(markers) != 1 || markers[0].MarkerKey != "ribm_forged_cross_scope" { + t.Fatalf("first marker damaged by refused publish: %+v", markers) + } +} + +func TestInventoryPublishBuildGuardsUncommittedCounts(t *testing.T) { + s := newInventoryStoreService(t) + s.migrate(t) + repo := s.newRepository(t) + + build, counts := mustInventoryBuild(t, "p", "g", []string{"Patient", "Observation"}, []catalog.ResourceInventoryCount{ + countFor("p", "g", "a", "Patient", 2), + countFor("p", "g", "b", "Patient", 1), + countFor("p", "g", "", "Observation", 0), + }) + + if err := repo.InsertCounts(s.ctx, counts[:1], 10); err != nil { + t.Fatalf("partial InsertCounts: %v", err) + } + err := repo.PublishBuild(s.ctx, build) + var guard *InventoryPublishGuardError + if !errors.As(err, &guard) || !errors.Is(err, ErrInventoryPublishGuard) { + t.Fatalf("premature publish error = %v, want *InventoryPublishGuardError", err) + } + if guard.CommittedRowCount != 1 || guard.ClaimedRowCount != 3 || fmt.Sprint(guard.MissingResourceTypes) != "[Patient]" { + t.Fatalf("guard detail = %+v", guard) + } + if markers := s.witnessMarkers(t, "p", "g"); len(markers) != 0 { + t.Fatalf("markers after guard failure = %v, want none", markers) + } + + if err := repo.InsertCounts(s.ctx, counts[1:2], 10); err != nil { + t.Fatalf("second InsertCounts: %v", err) + } + if err := repo.PublishBuild(s.ctx, build); !errors.Is(err, ErrInventoryPublishGuard) { + t.Fatalf("gap-only publish = %v, want ErrInventoryPublishGuard", err) + } + + if err := repo.InsertCounts(s.ctx, counts[2:], 10); err != nil { + t.Fatalf("final InsertCounts: %v", err) + } + if err := repo.PublishBuild(s.ctx, build); err != nil { + t.Fatalf("PublishBuild: %v", err) + } + markers := s.witnessMarkers(t, "p", "g") + if len(markers) != 1 { + t.Fatalf("markers = %v, want exactly one", markers) + } + wantMarker := markerWitness{ + MarkerKey: build.Key, Project: "p", DatasetGeneration: "g", BuildID: build.BuildID, + CoverageDigest: build.CoverageDigest, ResourceTypes: []string{"Observation", "Patient"}, + ResourceTypeCount: 2, CountRowCount: 3, Complete: true, + } + if fmt.Sprint(markers[0]) != fmt.Sprint(wantMarker) { + t.Fatalf("marker = %v, want %v", markers[0], wantMarker) + } + + got, found, err := repo.ReadBuild(s.ctx, "p", "g") + if err != nil || !found { + t.Fatalf("ReadBuild = %v, %v, err=%v", got, found, err) + } + if got.Key != build.Key || got.BuildID != build.BuildID || got.CoverageDigest != build.CoverageDigest || + !got.Complete || got.ResourceTypeCount != 2 || got.CountRowCount != 3 || + got.RecordType != catalog.ResourceInventoryBuildRecordType || got.Project != "p" || got.DatasetGeneration != "g" { + t.Fatalf("ReadBuild marker = %#v", got) + } + if _, found, err := repo.ReadBuild(s.ctx, "p", "never"); err != nil || found { + t.Fatalf("ReadBuild unknown scope = %v, %v, err=%v; want absent", got, found, err) + } + + if err := repo.PublishBuild(s.ctx, build); err != nil { + t.Fatalf("republish: %v", err) + } + if markers := s.witnessMarkers(t, "p", "g"); len(markers) != 1 { + t.Fatalf("markers after republish = %v, want one", markers) + } + + broken := build + broken.Complete = false + if err := repo.PublishBuild(s.ctx, broken); !errors.Is(err, ErrInvalidInventoryRow) { + t.Fatalf("incomplete marker publish = %v, want ErrInvalidInventoryRow", err) + } +} + +func TestInventoryReadCountSumsBindsScopeAndBuild(t *testing.T) { + s := newInventoryStoreService(t) + s.migrate(t) + repo := s.newRepository(t) + + build, counts := mustInventoryBuild(t, "p", "g", []string{"Patient", "Observation"}, []catalog.ResourceInventoryCount{ + countFor("p", "g", "a", "Patient", 2), + countFor("p", "g", "b", "Patient", 1), + countFor("p", "g", "", "Observation", 0), + }) + if err := repo.InsertCounts(s.ctx, counts, 10); err != nil { + t.Fatalf("InsertCounts: %v", err) + } + if err := repo.PublishBuild(s.ctx, build); err != nil { + t.Fatalf("PublishBuild: %v", err) + } + + s.exec(t, "INSERT INTO "+pgx.Identifier{s.schema, inventoryCountTableName}.Sanitize()+ + " (count_key, build_id, project, dataset_generation, resource_type, auth_resource_path, document_count)"+ + " VALUES ($1, $2, $3, $4, $5, $6, $7)", + "ric_forged_abandoned", "sha256:"+strings.Repeat("a", 64), "p", "g", "Patient", "a", int64(999999)) + s.exec(t, "INSERT INTO "+pgx.Identifier{s.schema, inventoryCountTableName}.Sanitize()+ + " (count_key, build_id, project, dataset_generation, resource_type, auth_resource_path, document_count)"+ + " VALUES ($1, $2, $3, $4, $5, $6, $7)", + "ric_forged_other_project", build.BuildID, "other", "g", "Patient", "a", int64(888888)) + s.exec(t, "INSERT INTO "+pgx.Identifier{s.schema, inventoryCountTableName}.Sanitize()+ + " (count_key, build_id, project, dataset_generation, resource_type, auth_resource_path, document_count)"+ + " VALUES ($1, $2, $3, $4, $5, $6, $7)", + "ric_forged_other_generation", build.BuildID, "p", "other", "Patient", "a", int64(777777)) + + restricted := func(paths []string) map[string]int64 { + t.Helper() + sums, err := repo.ReadCountSums(s.ctx, InventoryCountScope{ + Project: "p", DatasetGeneration: "g", BuildID: build.BuildID, + ResourceTypes: []string{"Patient", "Observation"}, + AuthResourcePathsUnrestricted: explicitUnrestricted(false), + AuthResourcePaths: paths, + }) + if err != nil { + t.Fatalf("restricted ReadCountSums(%v): %v", paths, err) + } + return sums + } + if got := restricted([]string{"a"}); fmt.Sprint(got) != fmt.Sprint(map[string]int64{"Patient": 2}) { + t.Fatalf("restricted [a] sums = %v, want Patient=2 only", got) + } + if got := restricted([]string{"a", "b"}); fmt.Sprint(got) != fmt.Sprint(map[string]int64{"Patient": 3}) { + t.Fatalf("restricted [a b] sums = %v, want Patient=3 only", got) + } + if got := restricted([]string{}); len(got) != 0 { + t.Fatalf("restricted empty sums = %v, want empty", got) + } + + unrestricted, err := repo.ReadCountSums(s.ctx, InventoryCountScope{ + Project: "p", DatasetGeneration: "g", BuildID: build.BuildID, + ResourceTypes: []string{"Patient", "Observation"}, + AuthResourcePathsUnrestricted: explicitUnrestricted(true), + }) + if err != nil { + t.Fatalf("unrestricted ReadCountSums: %v", err) + } + if fmt.Sprint(unrestricted) != fmt.Sprint(map[string]int64{"Patient": 3, "Observation": 0}) { + t.Fatalf("unrestricted sums = %v, want Patient=3 Observation=0", unrestricted) + } + + if _, err := repo.ReadCountSums(s.ctx, InventoryCountScope{ + Project: "p", DatasetGeneration: "g", BuildID: build.BuildID, ResourceTypes: []string{"Patient"}, + }); !errors.Is(err, ErrInvalidInventoryScope) { + t.Fatalf("nil unrestricted flag error = %v, want ErrInvalidInventoryScope", err) + } + if _, err := repo.ReadCountSums(s.ctx, InventoryCountScope{ + Project: "p", DatasetGeneration: "g", BuildID: build.BuildID, ResourceTypes: []string{"Patient"}, + AuthResourcePathsUnrestricted: explicitUnrestricted(true), AuthResourcePaths: []string{"a"}, + }); !errors.Is(err, ErrInvalidInventoryScope) { + t.Fatalf("unrestricted-with-paths error = %v, want ErrInvalidInventoryScope", err) + } + if _, err := repo.ReadCountSums(s.ctx, InventoryCountScope{ + Project: "p", DatasetGeneration: "g", AuthResourcePathsUnrestricted: explicitUnrestricted(true), + }); !errors.Is(err, ErrInvalidInventoryScope) { + t.Fatalf("blank build id error = %v, want ErrInvalidInventoryScope", err) + } +} diff --git a/internal/backends/postgres/internal/store/migrate.go b/internal/backends/postgres/internal/store/migrate.go new file mode 100644 index 000000000..b31e55164 --- /dev/null +++ b/internal/backends/postgres/internal/store/migrate.go @@ -0,0 +1,539 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "embed" + "encoding/hex" + "errors" + "fmt" + "hash/fnv" + "io/fs" + "path" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +//go:embed migrations/*.sql +var embeddedMigrationsFS embed.FS + +const ( + migrationsDir = "migrations" + LedgerTable = "schema_migrations" + advisoryLockNamespace = "loom:pgstore02:migrate:" + unlockTimeout = 5 * time.Second +) + +var ( + migrationFilePattern = regexp.MustCompile(`^([0-9]+)_([a-z][a-z0-9_]*)\.sql$`) + migrationNamePattern = regexp.MustCompile(`^[a-z][a-z0-9_]*$`) + schemaNamePattern = regexp.MustCompile(`^[a-z_][a-z0-9_]{0,62}$`) +) + +var ( + ErrInvalidMigration = errors.New("postgres: invalid migration") + ErrInvalidSchemaName = errors.New("postgres: invalid schema name") + ErrMigrationChecksumMismatch = errors.New("postgres: migration checksum mismatch") + ErrFutureSchemaVersion = errors.New("postgres: future schema version in ledger") + ErrMigrationMissing = errors.New("postgres: applied migration missing from history") + ErrMigrationOutOfOrder = errors.New("postgres: out-of-order migration") + ErrMigrationApply = errors.New("postgres: migration apply failed") + ErrMigrationLockAcquire = errors.New("postgres: migration lock acquisition failed") + ErrLedgerRecord = errors.New("postgres: ledger record failed") + ErrMigrationPending = errors.New("postgres: schema is behind the embedded migration history") +) + +type Migration struct { + Version int64 + Name string + SQL string + Checksum string +} + +func LoadMigrations(fsys fs.FS, dir string) ([]Migration, error) { + entries, err := fs.ReadDir(fsys, dir) + if err != nil { + return nil, fmt.Errorf("%w: read %s: %v", ErrInvalidMigration, dir, err) + } + out := make([]Migration, 0, len(entries)) + seenVersion := make(map[int64]string, len(entries)) + seenName := make(map[string]int64, len(entries)) + for _, entry := range entries { + fileName := entry.Name() + if entry.IsDir() { + return nil, fmt.Errorf("%w: %s/%s: nested directories are not supported", + ErrInvalidMigration, dir, fileName) + } + matches := migrationFilePattern.FindStringSubmatch(fileName) + if matches == nil { + return nil, fmt.Errorf("%w: %s/%s: file name must match _.sql "+ + "with a positive integer version and an [a-z][a-z0-9_]* name", + ErrInvalidMigration, dir, fileName) + } + version, err := strconv.ParseInt(matches[1], 10, 64) + if err != nil || version < 1 { + return nil, fmt.Errorf("%w: %s/%s: version must be a positive integer", + ErrInvalidMigration, dir, fileName) + } + name := matches[2] + if prev, dup := seenVersion[version]; dup { + return nil, fmt.Errorf("%w: %s/%s: version %d already used by %s", + ErrInvalidMigration, dir, fileName, version, prev) + } + if prev, dup := seenName[name]; dup { + return nil, fmt.Errorf("%w: %s/%s: name %q already used by version %d", + ErrInvalidMigration, dir, fileName, name, prev) + } + sql, err := fs.ReadFile(fsys, path.Join(dir, fileName)) + if err != nil { + return nil, fmt.Errorf("%w: %s/%s: read: %v", ErrInvalidMigration, dir, fileName, err) + } + if strings.TrimSpace(string(sql)) == "" { + return nil, fmt.Errorf("%w: %s/%s: empty SQL body", ErrInvalidMigration, dir, fileName) + } + seenVersion[version] = fileName + seenName[name] = version + out = append(out, Migration{ + Version: version, + Name: name, + SQL: string(sql), + Checksum: migrationChecksum(string(sql)), + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].Version < out[j].Version }) + return out, nil +} + +func EmbeddedMigrations() ([]Migration, error) { + return LoadMigrations(embeddedMigrationsFS, migrationsDir) +} + +func migrationChecksum(sql string) string { + sum := sha256.Sum256([]byte(sql)) + return hex.EncodeToString(sum[:]) +} + +func migrationAdvisoryLockKey(schema string) int64 { + h := fnv.New64a() + _, _ = h.Write([]byte(advisoryLockNamespace)) + _, _ = h.Write([]byte{0}) + _, _ = h.Write([]byte(schema)) + return int64(h.Sum64()) +} + +type ConnSource interface { + Acquire(ctx context.Context) (*pgxpool.Conn, error) +} + +var ( + _ ConnSource = (*Client)(nil) + _ ConnSource = (*pgxpool.Pool)(nil) +) + +type LedgerRecord struct { + Version int64 + Name string + Checksum string + AppliedAt time.Time +} + +type MigrateResult struct { + LatestVersion int64 + Applied []LedgerRecord +} + +type MigrationChecksumMismatchError struct { + Version int64 + Name string + LedgerChecksum string + ComputedChecksum string +} + +func (e *MigrationChecksumMismatchError) Error() string { + return fmt.Sprintf("postgres: migration %d (%s) was edited after it was applied: "+ + "ledger checksum %s, embedded checksum %s", + e.Version, e.Name, e.LedgerChecksum, e.ComputedChecksum) +} + +func (e *MigrationChecksumMismatchError) Unwrap() error { return ErrMigrationChecksumMismatch } + +type FutureSchemaVersionError struct { + LedgerVersion int64 + LedgerName string + LatestEmbedded int64 +} + +func (e *FutureSchemaVersionError) Error() string { + return fmt.Sprintf("postgres: ledger contains version %d (%s), above the latest embedded version %d; "+ + "the ledger is ahead of this binary", + e.LedgerVersion, e.LedgerName, e.LatestEmbedded) +} + +func (e *FutureSchemaVersionError) Unwrap() error { return ErrFutureSchemaVersion } + +type MissingEmbeddedMigrationError struct { + Version int64 + LedgerName string +} + +func (e *MissingEmbeddedMigrationError) Error() string { + return fmt.Sprintf("postgres: ledger records version %d (%s), which is missing from the embedded history", + e.Version, e.LedgerName) +} + +func (e *MissingEmbeddedMigrationError) Unwrap() error { return ErrMigrationMissing } + +type OutOfOrderMigrationError struct { + Version int64 + Name string + LatestRecord int64 +} + +func (e *OutOfOrderMigrationError) Error() string { + return fmt.Sprintf("postgres: migration %d (%s) is pending although version %d is already recorded; "+ + "add a new version instead of editing history", + e.Version, e.Name, e.LatestRecord) +} + +func (e *OutOfOrderMigrationError) Unwrap() error { return ErrMigrationOutOfOrder } + +type MigrationApplyError struct { + Op string + Version int64 + Name string + Err error +} + +func (e *MigrationApplyError) Error() string { + return fmt.Sprintf("postgres: migration %d (%s) failed during %s: %v", + e.Version, e.Name, e.Op, e.Err) +} + +func (e *MigrationApplyError) Unwrap() []error { return []error{ErrMigrationApply, e.Err} } + +type MigrationRunner struct { + src ConnSource + schema string + ledgerFQN string + migrations []Migration + panicHook func() +} + +func NewMigrationRunner(src ConnSource, schema string, migrations []Migration) (*MigrationRunner, error) { + if src == nil { + return nil, errors.New("postgres: nil migration connection source") + } + if !ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", ErrInvalidSchemaName, schema) + } + normalized := append([]Migration(nil), migrations...) + for i := range normalized { + m := &normalized[i] + if m.Version < 1 { + return nil, fmt.Errorf("%w: version %d is below 1", ErrInvalidMigration, m.Version) + } + if !migrationNamePattern.MatchString(m.Name) { + return nil, fmt.Errorf("%w: version %d has invalid name %q", ErrInvalidMigration, m.Version, m.Name) + } + if strings.TrimSpace(m.SQL) == "" { + return nil, fmt.Errorf("%w: version %d (%s) has an empty SQL body", ErrInvalidMigration, m.Version, m.Name) + } + m.Checksum = migrationChecksum(m.SQL) + } + sort.Slice(normalized, func(i, j int) bool { return normalized[i].Version < normalized[j].Version }) + seenVersion := make(map[int64]struct{}, len(normalized)) + seenName := make(map[string]struct{}, len(normalized)) + for _, m := range normalized { + if _, dup := seenVersion[m.Version]; dup { + return nil, fmt.Errorf("%w: duplicate version %d", ErrInvalidMigration, m.Version) + } + if _, dup := seenName[m.Name]; dup { + return nil, fmt.Errorf("%w: duplicate name %q", ErrInvalidMigration, m.Name) + } + seenVersion[m.Version] = struct{}{} + seenName[m.Name] = struct{}{} + } + return &MigrationRunner{ + src: src, + schema: schema, + ledgerFQN: pgx.Identifier{schema, LedgerTable}.Sanitize(), + migrations: normalized, + }, nil +} + +func (r *MigrationRunner) Migrate(ctx context.Context) (result MigrateResult, err error) { + conn, err := r.src.Acquire(ctx) + if err != nil { + return MigrateResult{}, fmt.Errorf("postgres: acquire migration connection: %w", err) + } + + key := migrationAdvisoryLockKey(r.schema) + if _, lockErr := conn.Exec(ctx, "SELECT pg_advisory_lock($1)", key); lockErr != nil { + discardPoolConn(conn) + return MigrateResult{}, fmt.Errorf("%w for schema %q: %w", ErrMigrationLockAcquire, r.schema, lockErr) + } + + defer func() { + unlockCtx, cancelUnlock := context.WithTimeout(context.WithoutCancel(ctx), unlockTimeout) + defer cancelUnlock() + var unlocked bool + unlockErr := conn.QueryRow(unlockCtx, "SELECT pg_advisory_unlock($1)", key).Scan(&unlocked) + if unlockErr != nil || !unlocked { + discardPoolConn(conn) + if err == nil { + if unlockErr != nil { + err = fmt.Errorf("postgres: release migration lock for schema %q: %w", r.schema, unlockErr) + return + } + err = fmt.Errorf("postgres: release migration lock for schema %q: reported already unlocked", r.schema) + } + return + } + conn.Release() + }() + + return r.migrateLocked(ctx, conn) +} + +func (r *MigrationRunner) Verify(ctx context.Context) error { + conn, err := r.src.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + + var schemaExists bool + if err := conn.QueryRow(ctx, "SELECT to_regnamespace($1) IS NOT NULL", r.schema).Scan(&schemaExists); err != nil { + return verifyStatementError("verify", err) + } + if !schemaExists { + return fmt.Errorf("%w: schema %q does not exist in this database; the caller owns schema lifecycle", + ErrInvalidSchemaName, r.schema) + } + applied, err := r.readLedger(ctx, conn) + if err != nil { + return verifyStatementError("verify ledger", err) + } + if err := r.validateHistory(applied); err != nil { + return err + } + pending, err := r.pending(applied) + if err != nil { + return err + } + if len(pending) > 0 { + first := pending[0] + return fmt.Errorf("%w: %d pending migration(s), first version %d (%s); schema %q", + ErrMigrationPending, len(pending), first.Version, first.Name, r.schema) + } + return nil +} + +func verifyStatementError(op string, err error) error { + if _, ok := pgErrorFrom(err); ok { + return err + } + return &ConnectError{Op: op, Err: err} +} + +func discardPoolConn(conn *pgxpool.Conn) { + _ = conn.Hijack().Close(context.Background()) +} + +func (r *MigrationRunner) migrateLocked(ctx context.Context, conn *pgxpool.Conn) (MigrateResult, error) { + if r.panicHook != nil { + r.panicHook() + } + var schemaExists bool + if err := conn.QueryRow(ctx, "SELECT to_regnamespace($1) IS NOT NULL", r.schema).Scan(&schemaExists); err != nil { + return MigrateResult{}, fmt.Errorf("postgres: check schema %q: %w", r.schema, err) + } + if !schemaExists { + return MigrateResult{}, fmt.Errorf("%w: schema %q does not exist in this database; the caller owns schema lifecycle", + ErrInvalidSchemaName, r.schema) + } + + applied, err := r.readLedger(ctx, conn) + if err != nil { + return MigrateResult{}, err + } + if err := r.validateHistory(applied); err != nil { + return MigrateResult{}, err + } + + pending, err := r.pending(applied) + if err != nil { + return MigrateResult{}, err + } + for _, m := range pending { + if err := r.applyOne(ctx, conn, m); err != nil { + return MigrateResult{}, err + } + } + + final, err := r.readLedger(ctx, conn) + if err != nil { + return MigrateResult{}, err + } + result := MigrateResult{Applied: final} + if len(final) > 0 { + result.LatestVersion = final[len(final)-1].Version + } + return result, nil +} + +func (r *MigrationRunner) readLedger(ctx context.Context, conn *pgxpool.Conn) ([]LedgerRecord, error) { + rows, err := conn.Query(ctx, + "SELECT version, name, checksum, applied_at FROM "+r.ledgerFQN+" ORDER BY version") + if err != nil { + if pgErr, ok := pgErrorFrom(err); ok && pgErr.Code == sqlStateUndefinedTable { + return nil, nil + } + return nil, fmt.Errorf("postgres: read migration ledger: %w", err) + } + defer rows.Close() + var out []LedgerRecord + for rows.Next() { + var rec LedgerRecord + if err := rows.Scan(&rec.Version, &rec.Name, &rec.Checksum, &rec.AppliedAt); err != nil { + return nil, fmt.Errorf("postgres: scan migration ledger: %w", err) + } + out = append(out, rec) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("postgres: read migration ledger: %w", err) + } + return out, nil +} + +func (r *MigrationRunner) validateHistory(applied []LedgerRecord) error { + byVersion := make(map[int64]Migration, len(r.migrations)) + latestEmbedded := int64(0) + for _, m := range r.migrations { + byVersion[m.Version] = m + if m.Version > latestEmbedded { + latestEmbedded = m.Version + } + } + for _, rec := range applied { + m, known := byVersion[rec.Version] + if !known { + if rec.Version > latestEmbedded { + return &FutureSchemaVersionError{ + LedgerVersion: rec.Version, + LedgerName: rec.Name, + LatestEmbedded: latestEmbedded, + } + } + return &MissingEmbeddedMigrationError{Version: rec.Version, LedgerName: rec.Name} + } + if rec.Checksum != m.Checksum { + return &MigrationChecksumMismatchError{ + Version: rec.Version, + Name: m.Name, + LedgerChecksum: rec.Checksum, + ComputedChecksum: m.Checksum, + } + } + } + return nil +} + +func (r *MigrationRunner) pending(applied []LedgerRecord) ([]Migration, error) { + recorded := make(map[int64]struct{}, len(applied)) + latestRecord := int64(0) + for _, rec := range applied { + recorded[rec.Version] = struct{}{} + if rec.Version > latestRecord { + latestRecord = rec.Version + } + } + var pending []Migration + for _, m := range r.migrations { + if _, done := recorded[m.Version]; done { + continue + } + if m.Version < latestRecord { + return nil, &OutOfOrderMigrationError{ + Version: m.Version, + Name: m.Name, + LatestRecord: latestRecord, + } + } + pending = append(pending, m) + } + return pending, nil +} + +func (r *MigrationRunner) applyOne(ctx context.Context, conn *pgxpool.Conn, m Migration) error { + tx, err := conn.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return &MigrationApplyError{Op: "begin", Version: m.Version, Name: m.Name, Err: err} + } + abandoned := true + defer func() { + if abandoned { + _ = tx.Rollback(ctx) + } + }() + + setSearchPath := "SET LOCAL search_path TO " + pgx.Identifier{r.schema}.Sanitize() + if _, err := tx.Exec(ctx, setSearchPath); err != nil { + return &MigrationApplyError{Op: "search_path", Version: m.Version, Name: m.Name, Err: err} + } + if _, err := tx.Exec(ctx, m.SQL); err != nil { + return &MigrationApplyError{Op: "apply", Version: m.Version, Name: m.Name, Err: err} + } + if err := tx.Commit(ctx); err != nil { + return &MigrationApplyError{Op: "commit", Version: m.Version, Name: m.Name, Err: err} + } + abandoned = false + + if _, err := conn.Exec(ctx, + "INSERT INTO "+r.ledgerFQN+" (version, name, checksum) VALUES ($1, $2, $3)", + m.Version, m.Name, m.Checksum); err != nil { + return fmt.Errorf("%w: migration %d (%s) committed but was not recorded; resolve the ledger before retrying: %w", + ErrLedgerRecord, m.Version, m.Name, err) + } + return nil +} + +func MigrateSchema(ctx context.Context, client *Client, schema string) (MigrateResult, error) { + if client == nil { + return MigrateResult{}, errors.New("postgres: nil client") + } + migrations, err := EmbeddedMigrations() + if err != nil { + return MigrateResult{}, err + } + runner, err := NewMigrationRunner(client, schema, migrations) + if err != nil { + return MigrateResult{}, err + } + return runner.Migrate(ctx) +} + +func ValidSchemaName(schema string) bool { + return schemaNamePattern.MatchString(schema) +} + +func VerifySchema(ctx context.Context, client *Client, schema string) error { + if client == nil { + return errors.New("postgres: nil client") + } + migrations, err := EmbeddedMigrations() + if err != nil { + return err + } + runner, err := NewMigrationRunner(client, schema, migrations) + if err != nil { + return err + } + return runner.Verify(ctx) +} diff --git a/internal/backends/postgres/internal/store/migrate_test.go b/internal/backends/postgres/internal/store/migrate_test.go new file mode 100644 index 000000000..532a32106 --- /dev/null +++ b/internal/backends/postgres/internal/store/migrate_test.go @@ -0,0 +1,911 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "net/url" + "reflect" + "regexp" + "strings" + "sync" + "testing" + "testing/fstest" + "time" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +func migrationMapFS(files map[string]string) fstest.MapFS { + out := fstest.MapFS{} + for name, sql := range files { + out[name] = &fstest.MapFile{Data: []byte(sql)} + } + return out +} + +func mustLoadMigrations(t *testing.T, files map[string]string) []Migration { + t.Helper() + migrations, err := LoadMigrations(migrationMapFS(files), ".") + if err != nil { + t.Fatalf("LoadMigrations: %v", err) + } + return migrations +} + +func mustEmbeddedMigrations(t *testing.T) []Migration { + t.Helper() + migrations, err := EmbeddedMigrations() + if err != nil { + t.Fatalf("EmbeddedMigrations: %v", err) + } + return migrations +} + +func migrationSet(t *testing.T, base []Migration, extra map[string]string) []Migration { + set := append(append([]Migration(nil), base...), mustLoadMigrations(t, extra)...) + return set +} + +func ledgerOnlyMigrations(t *testing.T) []Migration { + t.Helper() + migrations := mustEmbeddedMigrations(t) + if len(migrations) == 0 || migrations[0].Version != 1 || migrations[0].Name != "ledger" { + t.Fatalf("embedded history must begin with the version-1 ledger bootstrap: %+v", migrations) + } + return []Migration{migrations[0]} +} + +type stubConnSource struct{} + +func (stubConnSource) Acquire(context.Context) (*pgxpool.Conn, error) { + return nil, errors.New("stub source is never used") +} + +func TestMigrationLoadParsesChecksumsAndSorts(t *testing.T) { + const alphaSQL = "CREATE TABLE alpha (id bigint PRIMARY KEY);\n" + const betaSQL = "CREATE TABLE beta (id bigint);\n" + got, err := LoadMigrations(migrationMapFS(map[string]string{ + "000002_beta.sql": betaSQL, + "000001_alpha.sql": alphaSQL, + }), ".") + if err != nil { + t.Fatalf("LoadMigrations: %v", err) + } + if len(got) != 2 { + t.Fatalf("loaded %d migrations, want 2", len(got)) + } + if got[0].Version != 1 || got[0].Name != "alpha" || got[0].SQL != alphaSQL { + t.Errorf("first migration = %+v, want version 1 alpha", got[0]) + } + if got[1].Version != 2 || got[1].Name != "beta" { + t.Errorf("second migration = %+v, want version 2 beta", got[1]) + } + if got[0].Checksum != migrationChecksum(alphaSQL) { + t.Errorf("checksum = %s, want sha256 of exact SQL bytes", got[0].Checksum) + } + if len(got[1].Checksum) != 64 || strings.ToLower(got[1].Checksum) != got[1].Checksum { + t.Errorf("checksum %q is not lowercase 64-char hex", got[1].Checksum) + } +} + +func TestMigrationLoadRejectsInvalidFiles(t *testing.T) { + for _, tc := range []struct { + name string + files map[string]string + }{ + {"non-sql file", map[string]string{"README.md": "notes", "000001_alpha.sql": "SELECT 1;"}}, + {"missing underscore", map[string]string{"0001alpha.sql": "SELECT 1;"}}, + {"non-numeric version", map[string]string{"v1_alpha.sql": "SELECT 1;"}}, + {"zero version", map[string]string{"000000_alpha.sql": "SELECT 1;"}}, + {"version overflow", map[string]string{"99999999999999999999_alpha.sql": "SELECT 1;"}}, + {"uppercase name", map[string]string{"000001_Alpha.sql": "SELECT 1;"}}, + {"name with dot", map[string]string{"000001_al pha.sql": "SELECT 1;"}}, + {"duplicate version", map[string]string{"000001_alpha.sql": "SELECT 1;", "000001_beta.sql": "SELECT 2;"}}, + {"duplicate name", map[string]string{"000001_alpha.sql": "SELECT 1;", "000002_alpha.sql": "SELECT 2;"}}, + {"empty sql", map[string]string{"000001_alpha.sql": " \n\t\n"}}, + } { + _, err := LoadMigrations(migrationMapFS(tc.files), ".") + if !errors.Is(err, ErrInvalidMigration) { + t.Errorf("%s: LoadMigrations = %v, want ErrInvalidMigration", tc.name, err) + } + } + if _, err := LoadMigrations(migrationMapFS(nil), "absent"); !errors.Is(err, ErrInvalidMigration) { + t.Errorf("missing directory: LoadMigrations = %v, want ErrInvalidMigration", err) + } +} + +var wantEmbeddedChecksums = map[int64]struct { + name string + checksum string +}{ + 1: {"ledger", "db70f7758495f7557b3a694149b80ed6e5d4cb74ea5aa7d59a1a2eaa46fc9cde"}, + 2: {"fhir_resource", "d6c737db696eea1f9e9b8cd02158d14f98dfae9d9cc1d406b3c03bcc1b9fdaf0"}, + 3: {"dataset_generation_manifest", "06c614680a9c1cd4ba2ef0ac0471053bcb3d8e471c36d3852e70682887a72f5d"}, + 4: {"edge_occurrence", "aca2b36e1b839919434efb5b8e6c3d1939d7c182d5e15898288ac3f1f0ccb79b"}, + 5: {"resource_inventory", "acebb614e6e5c7488f01c2ead47a3389da584bd940d866927816c64fd92e4146"}, + 6: {"semantic_inventory_build", "e69d0dfdd3fc67438b65df7da73ea12edb61324ec8356cbdc4abc1ae5172af3c"}, + 7: {"field_profile", "202245110193213c9ae5183bfdd3a6856e88d405d26c742d5de686775beae41a"}, + 8: {"batch_write_ledger", "cad1877917eb24a7f7d4cb11c578b83f007fff92ceddc3ca79989a71742b05f4"}, + 9: {"semantic_inventory_contribution", "41579b3c86d17c5912195ccaa6ff8655e6a864c01a01f2a23fa1cb4aa6883b02"}, +} + +func TestEmbeddedMigrationsBootstrapsTheLedger(t *testing.T) { + migrations := mustEmbeddedMigrations(t) + if len(migrations) == 0 { + t.Fatal("embedded history is empty; the ledger bootstrap must always be present") + } + first := migrations[0] + if first.Version != 1 || first.Name != "ledger" { + t.Fatalf("bootstrap migration = {Version:%d Name:%q}, want {1 ledger}", first.Version, first.Name) + } + if !strings.Contains(first.SQL, LedgerTable) { + t.Fatalf("bootstrap SQL does not create %s", LedgerTable) + } + for i, m := range migrations { + if i > 0 && migrations[i-1].Version >= m.Version { + t.Fatalf("history is not strictly ascending at index %d: version %d then %d", + i, migrations[i-1].Version, m.Version) + } + want, pinned := wantEmbeddedChecksums[m.Version] + if !pinned { + t.Errorf("migration %d (%s) has no pinned checksum; add sha256sum of its exact bytes to wantEmbeddedChecksums", + m.Version, m.Name) + continue + } + if m.Name != want.name || m.Checksum != want.checksum { + t.Errorf("migration %d = {Name:%q Checksum:%q}, want {Name:%q Checksum:%q}: migration SQL drifted", + m.Version, m.Name, m.Checksum, want.name, want.checksum) + } + } + for version, want := range wantEmbeddedChecksums { + found := false + for _, m := range migrations { + if m.Version == version { + found = true + break + } + } + if !found { + t.Errorf("wantEmbeddedChecksums pins version %d (%s) which is absent from the embedded history", + version, want.name) + } + } +} + +func TestNewMigrationRunnerRejectsInvalidInputs(t *testing.T) { + for _, tc := range []struct { + name string + src ConnSource + schema string + migrations []Migration + want error + }{ + {"nil source", nil, "ok_schema", nil, errors.New("sentinel")}, + {"empty schema", stubConnSource{}, "", nil, ErrInvalidSchemaName}, + {"uppercase schema", stubConnSource{}, "Public", nil, ErrInvalidSchemaName}, + {"dotted schema", stubConnSource{}, "public.other", nil, ErrInvalidSchemaName}, + {"leading digit", stubConnSource{}, "1schema", nil, ErrInvalidSchemaName}, + {"overlong schema", stubConnSource{}, strings.Repeat("a", 64), nil, ErrInvalidSchemaName}, + {"zero version", stubConnSource{}, "ok_schema", + []Migration{{Version: 0, Name: "a", SQL: "SELECT 1;"}}, ErrInvalidMigration}, + {"bad name", stubConnSource{}, "ok_schema", + []Migration{{Version: 1, Name: "Bad-Name", SQL: "SELECT 1;"}}, ErrInvalidMigration}, + {"empty sql", stubConnSource{}, "ok_schema", + []Migration{{Version: 1, Name: "a", SQL: " "}}, ErrInvalidMigration}, + {"duplicate version", stubConnSource{}, "ok_schema", + []Migration{{Version: 2, Name: "a", SQL: "SELECT 1;"}, {Version: 2, Name: "b", SQL: "SELECT 2;"}}, ErrInvalidMigration}, + {"duplicate name", stubConnSource{}, "ok_schema", + []Migration{{Version: 1, Name: "a", SQL: "SELECT 1;"}, {Version: 2, Name: "a", SQL: "SELECT 2;"}}, ErrInvalidMigration}, + } { + _, err := NewMigrationRunner(tc.src, tc.schema, tc.migrations) + if tc.want == ErrInvalidSchemaName || tc.want == ErrInvalidMigration { + if !errors.Is(err, tc.want) { + t.Errorf("%s: NewMigrationRunner = %v, want %v", tc.name, err, tc.want) + } + continue + } + if err == nil { + t.Errorf("%s: NewMigrationRunner succeeded, want error", tc.name) + } + } +} + +func TestNewMigrationRunnerNormalizesAndSorts(t *testing.T) { + runner, err := NewMigrationRunner(stubConnSource{}, "pgstore02_it_x", []Migration{ + {Version: 3, Name: "c", SQL: "SELECT 3;"}, + {Version: 1, Name: "a", SQL: "SELECT 1;"}, + }) + if err != nil { + t.Fatalf("NewMigrationRunner: %v", err) + } + if len(runner.migrations) != 2 || runner.migrations[0].Version != 1 || runner.migrations[1].Version != 3 { + t.Fatalf("history not sorted: %+v", runner.migrations) + } + if runner.migrations[0].Checksum != migrationChecksum("SELECT 1;") { + t.Errorf("checksum not recomputed from SQL: %s", runner.migrations[0].Checksum) + } + if runner.ledgerFQN != `"pgstore02_it_x"."schema_migrations"` { + t.Errorf("ledgerFQN = %s", runner.ledgerFQN) + } +} + +func TestMigrationAdvisoryLockKey(t *testing.T) { + first := migrationAdvisoryLockKey("pgstore02_it_x") + if second := migrationAdvisoryLockKey("pgstore02_it_x"); first != second { + t.Fatalf("key not stable: %d vs %d", first, second) + } + if first == migrationAdvisoryLockKey("pgstore02_it_y") { + t.Fatal("different schemas share an advisory-lock key") + } + if first == 0 { + t.Fatal("advisory-lock key must be nonzero") + } +} + +var pgstore02ResourceNamePattern = regexp.MustCompile(`^pgstore02_it_[0-9]+_[0-9a-f]+$`) + +type migrationService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func newMigrationService(t *testing.T) *migrationService { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgstore02_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstore02ResourceNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("pgstore02_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstore02ResourceNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + return &migrationService{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} +} + +func (s *migrationService) openClient(t *testing.T) *Client { + t.Helper() + return s.openClientWithBounds(t, 1, 3) +} + +func (s *migrationService) openClientWithBounds(t *testing.T, minConns, maxConns int32) *Client { + t.Helper() + client, err := Open(s.ctx, Config{ + DSN: s.dsn, + MaxConns: maxConns, + MinConns: minConns, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *migrationService) newRunner(t *testing.T, src ConnSource, migrations []Migration) *MigrationRunner { + t.Helper() + runner, err := NewMigrationRunner(src, s.schema, migrations) + if err != nil { + t.Fatalf("NewMigrationRunner: %v", err) + } + return runner +} + +func (s *migrationService) readLedger(t *testing.T) []LedgerRecord { + t.Helper() + rows, err := s.iso.Query(s.ctx, + `SELECT version, name, checksum, applied_at FROM "`+s.schema+`".`+LedgerTable+` ORDER BY version`) + if err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "42P01" { + return nil + } + s.fail("witness ledger", err) + } + defer rows.Close() + var out []LedgerRecord + for rows.Next() { + var rec LedgerRecord + if err := rows.Scan(&rec.Version, &rec.Name, &rec.Checksum, &rec.AppliedAt); err != nil { + s.fail("witness ledger scan", err) + } + out = append(out, rec) + } + if err := rows.Err(); err != nil { + s.fail("witness ledger rows", err) + } + return out +} + +func (s *migrationService) regclass(t *testing.T, relation string) string { + t.Helper() + var found *string + err := s.iso.QueryRow(s.ctx, + `SELECT to_regclass($1)::text`, s.schema+"."+relation).Scan(&found) + if err != nil { + s.fail("witness to_regclass "+relation, err) + } + if found == nil { + return "" + } + return *found +} + +func (s *migrationService) mustMigrate(t *testing.T, runner *MigrationRunner) MigrateResult { + t.Helper() + result, err := runner.Migrate(s.ctx) + if err != nil { + s.fail("Migrate", err) + } + return result +} + +func assertLedgerEquals(t *testing.T, got []LedgerRecord, want []Migration) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("ledger has %d rows, want %d: %+v", len(got), len(want), got) + } + seen := make(map[int64]struct{}, len(got)) + for i, rec := range got { + if rec.Version != want[i].Version || rec.Name != want[i].Name || rec.Checksum != want[i].Checksum { + t.Fatalf("ledger row %d = %+v, want {Version:%d Name:%q Checksum:%q}", + i, rec, want[i].Version, want[i].Name, want[i].Checksum) + } + if _, dup := seen[rec.Version]; dup { + t.Fatalf("ledger recorded version %d twice: %+v", rec.Version, got) + } + seen[rec.Version] = struct{}{} + } +} + +func TestMigrationConcurrentCallersApplyExactlyOnce(t *testing.T) { + svc := newMigrationService(t) + set := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{ + "000002_alpha.sql": "CREATE TABLE alpha (id bigint PRIMARY KEY);\n", + "000003_sleepy.sql": "SELECT pg_catalog.pg_sleep(0.75);\nCREATE TABLE beta (id bigint);\n", + }) + + clientA := svc.openClient(t) + clientB := svc.openClient(t) + + pidOf := func(client *Client, tag string) int32 { + conn, err := client.Acquire(svc.ctx) + if err != nil { + svc.fail("acquire "+tag+" probe connection", err) + } + defer conn.Release() + var pid int32 + if err := conn.QueryRow(svc.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + svc.fail("read "+tag+" backend pid", err) + } + return pid + } + if pidA, pidB := pidOf(clientA, "caller A"), pidOf(clientB, "caller B"); pidA == pidB { + t.Fatalf("both callers share backend pid %d; want separate connections", pidA) + } + + runnerA := svc.newRunner(t, clientA, set) + runnerB := svc.newRunner(t, clientB, set) + + start := make(chan struct{}) + var results [2]MigrateResult + var errs [2]error + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + <-start + results[0], errs[0] = runnerA.Migrate(svc.ctx) + }() + go func() { + defer wg.Done() + <-start + results[1], errs[1] = runnerB.Migrate(svc.ctx) + }() + close(start) + + finished := make(chan struct{}) + go func() { + wg.Wait() + close(finished) + }() + select { + case <-finished: + case <-time.After(90 * time.Second): + t.Fatal("concurrent Migrate calls did not finish in 90s; the advisory lock may be deadlocked") + } + for i, err := range errs { + if err != nil { + svc.fail(fmt.Sprintf("concurrent caller %c", 'A'+i), err) + } + } + if !reflect.DeepEqual(results[0], results[1]) { + t.Errorf("callers disagree:\nA: %+v\nB: %+v", results[0], results[1]) + } + if results[0].LatestVersion != 3 || len(results[0].Applied) != 3 { + t.Fatalf("result = {LatestVersion:%d Applied:%d}, want {3 3}", + results[0].LatestVersion, len(results[0].Applied)) + } + assertLedgerEquals(t, svc.readLedger(t), set) + if got := svc.regclass(t, "alpha"); got == "" { + t.Error("alpha absent from the owned schema") + } + if got := svc.regclass(t, "beta"); got == "" { + t.Error("beta absent from the owned schema") + } +} + +func TestMigrationIdempotentReapplyReturnsSameState(t *testing.T) { + svc := newMigrationService(t) + set := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{ + "000002_gamma.sql": "CREATE TABLE gamma (id int);\n", + "000003_note.sql": "ALTER TABLE gamma ADD COLUMN note text;\nCREATE INDEX gamma_note_idx ON gamma (note);\n", + }) + runner := svc.newRunner(t, svc.openClient(t), set) + + first := svc.mustMigrate(t, runner) + second := svc.mustMigrate(t, runner) + third := svc.mustMigrate(t, runner) + + if !reflect.DeepEqual(first, second) || !reflect.DeepEqual(second, third) { + t.Fatalf("re-apply changed the result:\n1st: %+v\n2nd: %+v\n3rd: %+v", first, second, third) + } + if first.LatestVersion != 3 || len(first.Applied) != 3 { + t.Fatalf("result = {LatestVersion:%d Applied:%d}, want {3 3}", first.LatestVersion, len(first.Applied)) + } + assertLedgerEquals(t, svc.readLedger(t), set) +} + +func TestMigrationEditedChecksumFailsBeforeApply(t *testing.T) { + svc := newMigrationService(t) + const plainSQL = "CREATE TABLE keep_me (id int);\n" + const editedSQL = "CREATE TABLE keep_me (id int);\nCREATE TABLE edited_marker (id int);\n" + applied := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{"000002_plain.sql": plainSQL}) + edited := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{"000002_plain.sql": editedSQL}) + + svc.mustMigrate(t, svc.newRunner(t, svc.openClient(t), applied)) + + result, err := svc.newRunner(t, svc.openClient(t), edited).Migrate(svc.ctx) + if !errors.Is(err, ErrMigrationChecksumMismatch) { + t.Fatalf("Migrate with edited history = %v, want ErrMigrationChecksumMismatch", err) + } + if !reflect.DeepEqual(result, MigrateResult{}) { + t.Errorf("failed Migrate returned result %+v, want zero value", result) + } + var mismatch *MigrationChecksumMismatchError + if !errors.As(err, &mismatch) { + t.Fatalf("error type = %T, want *MigrationChecksumMismatchError", err) + } + if mismatch.Version != 2 || mismatch.Name != "plain" { + t.Errorf("mismatch = %+v, want version 2 name plain", mismatch) + } + if mismatch.LedgerChecksum != migrationChecksum(plainSQL) || + mismatch.ComputedChecksum != migrationChecksum(editedSQL) { + t.Errorf("mismatch checksums = {ledger:%s computed:%s}, want the original and edited sha256", + mismatch.LedgerChecksum, mismatch.ComputedChecksum) + } + if got := svc.regclass(t, "edited_marker"); got != "" { + t.Errorf("edited_marker exists (%s); edited migration was applied", got) + } + assertLedgerEquals(t, svc.readLedger(t), applied) +} + +func TestMigrationFutureSchemaVersionFailsBeforeApply(t *testing.T) { + svc := newMigrationService(t) + full := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{ + "000002_alpha.sql": "CREATE TABLE alpha (id int);\n", + "000003_beta.sql": "CREATE TABLE beta (id int);\n", + }) + svc.mustMigrate(t, svc.newRunner(t, svc.openClient(t), full)) + + outdated := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{ + "000002_alpha.sql": "CREATE TABLE alpha (id int);\n", + }) + _, err := svc.newRunner(t, svc.openClient(t), outdated).Migrate(svc.ctx) + if !errors.Is(err, ErrFutureSchemaVersion) { + t.Fatalf("Migrate with a future ledger = %v, want ErrFutureSchemaVersion", err) + } + var future *FutureSchemaVersionError + if !errors.As(err, &future) { + t.Fatalf("error type = %T, want *FutureSchemaVersionError", err) + } + if future.LedgerVersion != 3 || future.LedgerName != "beta" || future.LatestEmbedded != 2 { + t.Errorf("future-version error = %+v, want {LedgerVersion:3 LedgerName:beta LatestEmbedded:2}", future) + } + assertLedgerEquals(t, svc.readLedger(t), full) +} + +func TestMigrationMissingEmbeddedFileFails(t *testing.T) { + svc := newMigrationService(t) + applied := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{ + "000002_alpha.sql": "CREATE TABLE alpha (id int);\n", + "000003_beta.sql": "CREATE TABLE beta (id int);\n", + }) + svc.mustMigrate(t, svc.newRunner(t, svc.openClient(t), applied)) + + shuffled := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{ + "000002_alpha.sql": "CREATE TABLE alpha (id int);\n", + "000004_delta.sql": "CREATE TABLE delta (id int);\n", + }) + _, err := svc.newRunner(t, svc.openClient(t), shuffled).Migrate(svc.ctx) + if !errors.Is(err, ErrMigrationMissing) { + t.Fatalf("Migrate with a vanished migration = %v, want ErrMigrationMissing", err) + } + var missing *MissingEmbeddedMigrationError + if !errors.As(err, &missing) || missing.Version != 3 || missing.LedgerName != "beta" { + t.Fatalf("error = %v (%T), want MissingEmbeddedMigrationError{3 beta}", err, err) + } + if got := svc.regclass(t, "delta"); got != "" { + t.Errorf("delta exists (%s); pending migration ran despite the missing file", got) + } +} + +func TestMigrationOutOfOrderFails(t *testing.T) { + svc := newMigrationService(t) + applied := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{ + "000002_alpha.sql": "CREATE TABLE alpha (id int);\n", + "000004_delta.sql": "CREATE TABLE delta (id int);\n", + }) + svc.mustMigrate(t, svc.newRunner(t, svc.openClient(t), applied)) + + amended := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{ + "000002_alpha.sql": "CREATE TABLE alpha (id int);\n", + "000003_beta.sql": "CREATE TABLE beta (id int);\n", + "000004_delta.sql": "CREATE TABLE delta (id int);\n", + }) + _, err := svc.newRunner(t, svc.openClient(t), amended).Migrate(svc.ctx) + if !errors.Is(err, ErrMigrationOutOfOrder) { + t.Fatalf("Migrate with an out-of-order file = %v, want ErrMigrationOutOfOrder", err) + } + var outOfOrder *OutOfOrderMigrationError + if !errors.As(err, &outOfOrder) || outOfOrder.Version != 3 || outOfOrder.LatestRecord != 4 { + t.Fatalf("error = %v (%T), want OutOfOrderMigrationError{Version:3 LatestRecord:4}", err, err) + } + if got := svc.regclass(t, "beta"); got != "" { + t.Errorf("beta exists (%s); out-of-order migration was applied", got) + } +} + +func TestMigrationFailingDDLRollsBackAndKeepsLedgerIntact(t *testing.T) { + svc := newMigrationService(t) + failing := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{ + "000002_ok.sql": "CREATE TABLE audit_ok (id int);\nINSERT INTO audit_ok VALUES (1);\n", + "000003_bad.sql": "CREATE TABLE rollback_probe (id int);\n" + + "INSERT INTO no_such_table VALUES (1);\n", + }) + result, err := svc.newRunner(t, svc.openClient(t), failing).Migrate(svc.ctx) + if !errors.Is(err, ErrMigrationApply) { + t.Fatalf("Migrate with a failing migration = %v, want ErrMigrationApply", err) + } + if !reflect.DeepEqual(result, MigrateResult{}) { + t.Errorf("failed Migrate returned result %+v, want zero value", result) + } + var applyErr *MigrationApplyError + if !errors.As(err, &applyErr) { + t.Fatalf("error type = %T, want *MigrationApplyError", err) + } + if applyErr.Version != 3 || applyErr.Name != "bad" || applyErr.Op != "apply" { + t.Errorf("apply error = %+v, want {Version:3 Name:bad Op:apply}", applyErr) + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "42P01" { + t.Errorf("apply error does not chain the undefined-table PgError: %v", err) + } + + if got := svc.regclass(t, "rollback_probe"); got != "" { + t.Errorf("rollback_probe exists (%s); failing migration DDL was not rolled back", got) + } + if got := svc.regclass(t, "audit_ok"); got == "" { + t.Error("audit_ok absent; the prior committed migration was disturbed") + } + prior := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{ + "000002_ok.sql": "CREATE TABLE audit_ok (id int);\nINSERT INTO audit_ok VALUES (1);\n", + }) + assertLedgerEquals(t, svc.readLedger(t), prior) + + fixed := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{ + "000002_ok.sql": "CREATE TABLE audit_ok (id int);\nINSERT INTO audit_ok VALUES (1);\n", + "000003_bad.sql": "CREATE TABLE rollback_probe (id int);\n", + }) + recovered := svc.mustMigrate(t, svc.newRunner(t, svc.openClient(t), fixed)) + if recovered.LatestVersion != 3 { + t.Fatalf("recovered LatestVersion = %d, want 3", recovered.LatestVersion) + } + if got := svc.regclass(t, "rollback_probe"); got == "" { + t.Error("rollback_probe absent after the corrected migration applied") + } + assertLedgerEquals(t, svc.readLedger(t), fixed) + var auditRows int + if err := svc.iso.QueryRow(svc.ctx, `SELECT count(*) FROM "`+svc.schema+`".audit_ok`).Scan(&auditRows); err != nil { + svc.fail("count audit_ok rows", err) + } + if auditRows != 1 { + t.Errorf("audit_ok has %d rows, want 1: migration 000002 was replayed", auditRows) + } +} + +func TestMigrationUnconfirmedUnlockDiscardsConnection(t *testing.T) { + svc := newMigrationService(t) + set := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{ + "000002_surrender.sql": "SELECT pg_catalog.pg_advisory_unlock_all();\nCREATE TABLE surrendered (id int);\n", + }) + client := svc.openClientWithBounds(t, 0, 3) + before := client.pool.Stat().TotalConns() + if before == 0 { + t.Fatal("Open's readiness ping must have warmed the pool before the run") + } + + _, err := svc.newRunner(t, client, set).Migrate(svc.ctx) + if err == nil || !strings.Contains(err.Error(), "already unlocked") { + t.Fatalf("Migrate with a lock-surrendering migration = %v, want already-unlocked release error", err) + } + assertLedgerEquals(t, svc.readLedger(t), set) + if acquired := client.pool.Stat().AcquiredConns(); acquired != 0 { + t.Errorf("pool still holds %d acquired connections; the poisoned connection was not discarded", acquired) + } + if total := client.pool.Stat().TotalConns(); total >= before { + t.Errorf("pool physical conns = %d after the unconfirmed unlock, want a drop below the pre-run %d: "+ + "the unproven connection was returned to the pool instead of discarded", total, before) + } + again := svc.mustMigrate(t, svc.newRunner(t, client, set)) + if again.LatestVersion != 2 { + t.Fatalf("follow-up LatestVersion = %d, want 2", again.LatestVersion) + } +} + +func TestMigrationPanicUnderLockReleasesLockAndLeaksNothing(t *testing.T) { + svc := newMigrationService(t) + set := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{ + "000002_item.sql": "CREATE TABLE panicked_on (id int);\n", + }) + injected := errors.New("injected panic under the migration lock") + + client := svc.openClient(t) + runner := svc.newRunner(t, client, set) + runner.panicHook = func() { panic(injected) } + + panicked := func() (recovered any) { + defer func() { recovered = recover() }() + _, _ = runner.Migrate(svc.ctx) + return nil + }() + if recovered, ok := panicked.(error); !ok || !errors.Is(recovered, injected) { + t.Fatalf("panic under the lock was not propagated: %v", panicked) + } + if rows := svc.readLedger(t); len(rows) != 0 { + t.Errorf("ledger has %d rows after the injected panic, want 0", len(rows)) + } + if acquired := client.pool.Stat().AcquiredConns(); acquired != 0 { + t.Errorf("pool still holds %d acquired connections after the panic", acquired) + } + recovered := svc.mustMigrate(t, svc.newRunner(t, svc.openClient(t), set)) + if recovered.LatestVersion != 2 { + t.Fatalf("post-panic LatestVersion = %d, want 2", recovered.LatestVersion) + } + assertLedgerEquals(t, svc.readLedger(t), set) +} + +func waitAdvisoryLock(t *testing.T, svc *migrationService, present bool) { + t.Helper() + query := `SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND granted + AND database = (SELECT oid FROM pg_database WHERE datname = current_database())` + deadline := time.Now().Add(15 * time.Second) + for { + var held int64 + if err := svc.iso.QueryRow(svc.ctx, query).Scan(&held); err != nil { + svc.fail("probe advisory locks", err) + } + if (held > 0) == present { + return + } + if time.Now().After(deadline) { + t.Fatalf("advisory lock presence did not reach %v within 15s (last count %d)", present, held) + } + time.Sleep(20 * time.Millisecond) + } +} + +func TestMigrationContendedAcquireDeadlineIsNamed(t *testing.T) { + svc := newMigrationService(t) + set := migrationSet(t, ledgerOnlyMigrations(t), map[string]string{ + "000002_slow.sql": "SELECT pg_catalog.pg_sleep(3);\nCREATE TABLE slow_one (id int);\n", + }) + + holderClient := svc.openClient(t) + holderRunner := svc.newRunner(t, holderClient, set) + holderDone := make(chan error, 1) + go func() { + _, err := holderRunner.Migrate(svc.ctx) + holderDone <- err + }() + waitAdvisoryLock(t, svc, true) + + loserClient := svc.openClient(t) + loserRunner := svc.newRunner(t, loserClient, set) + shortCtx, cancelShort := context.WithTimeout(svc.ctx, 300*time.Millisecond) + defer cancelShort() + _, err := loserRunner.Migrate(shortCtx) + if !errors.Is(err, ErrMigrationLockAcquire) { + t.Fatalf("contended Migrate = %v, want ErrMigrationLockAcquire", err) + } + var pgErr *pgconn.PgError + if !errors.Is(err, context.DeadlineExceeded) && + !(errors.As(err, &pgErr) && pgErr.Code == "57014") { + t.Fatalf("acquisition error does not chain a deadline or cancellation cause: %v", err) + } + if acquired := loserClient.pool.Stat().AcquiredConns(); acquired != 0 { + t.Errorf("loser pool still holds %d acquired connections; racing connection was not discarded", acquired) + } + + if holderErr := <-holderDone; holderErr != nil { + svc.fail("holder Migrate", holderErr) + } + + started := time.Now() + fresh := svc.mustMigrate(t, svc.newRunner(t, svc.openClient(t), set)) + if waited := time.Since(started); waited > 2*time.Second { + t.Errorf("fresh Migrate waited %v after the holder finished; the aborted attempt left lock residue", waited) + } + if fresh.LatestVersion != 2 { + t.Fatalf("fresh LatestVersion = %d, want 2", fresh.LatestVersion) + } + if _, err := loserRunner.Migrate(svc.ctx); err != nil { + svc.fail("loser client retry after deadline", err) + } + assertLedgerEquals(t, svc.readLedger(t), set) +} + +func TestMigrationSchemaMustExist(t *testing.T) { + svc := newMigrationService(t) + runner, err := NewMigrationRunner(svc.openClient(t), "pgstore02_it_missing", mustEmbeddedMigrations(t)) + if err != nil { + t.Fatalf("NewMigrationRunner: %v", err) + } + if _, err := runner.Migrate(svc.ctx); !errors.Is(err, ErrInvalidSchemaName) { + t.Fatalf("Migrate into a missing schema = %v, want ErrInvalidSchemaName", err) + } +} + +func TestMigrationEmbeddedBootstrapAppliesIdempotently(t *testing.T) { + svc := newMigrationService(t) + embedded := mustEmbeddedMigrations(t) + wantLatest := embedded[len(embedded)-1].Version + first, err := MigrateSchema(svc.ctx, svc.openClient(t), svc.schema) + if err != nil { + svc.fail("MigrateSchema", err) + } + if first.LatestVersion != wantLatest || len(first.Applied) != len(embedded) { + t.Fatalf("full embedded history result = %+v, want latest version %d over %d rows", + first, wantLatest, len(embedded)) + } + second, err := MigrateSchema(svc.ctx, svc.openClient(t), svc.schema) + if err != nil { + svc.fail("re-apply MigrateSchema", err) + } + if !reflect.DeepEqual(first, second) { + t.Errorf("bootstrap re-apply changed the result:\n1st: %+v\n2nd: %+v", first, second) + } + assertLedgerEquals(t, svc.readLedger(t), mustEmbeddedMigrations(t)) +} + +func TestMigrationNoLeftovers(t *testing.T) { + raw := testPostgresURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %v", err) + } + defer admin.Close() + + databases := leftoverNames(t, admin, ctx, + `SELECT datname FROM pg_database WHERE datname LIKE 'pgstore02_it_%' ORDER BY datname`) + if len(databases) != 0 { + t.Errorf("leftover isolated databases: %v", databases) + } + schemas := leftoverNames(t, admin, ctx, + `SELECT n.nspname FROM pg_namespace n WHERE n.nspname LIKE 'pgstore02_it_%' ORDER BY n.nspname`) + if len(schemas) != 0 { + t.Errorf("leftover owned schemas in the service database: %v", schemas) + } +} + +func leftoverNames(t *testing.T, pool *pgxpool.Pool, ctx context.Context, query string) []string { + t.Helper() + rows, err := pool.Query(ctx, query) + if err != nil { + t.Fatalf("leftover probe %q: %v", query, err) + } + defer rows.Close() + var out []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("leftover probe scan: %v", err) + } + out = append(out, name) + } + if err := rows.Err(); err != nil { + t.Fatalf("leftover probe rows: %v", err) + } + return out +} diff --git a/internal/backends/postgres/internal/store/migration_ownership.go b/internal/backends/postgres/internal/store/migration_ownership.go new file mode 100644 index 000000000..769c6f8a5 --- /dev/null +++ b/internal/backends/postgres/internal/store/migration_ownership.go @@ -0,0 +1,232 @@ +package postgres + +import ( + "encoding/json" + "errors" + "fmt" + "sort" +) + +var ( + ErrInvalidAssignment = errors.New("postgres: invalid migration ownership assignment") +) + +type SeriesID string + +const ( + SeriesSource SeriesID = "source" + SeriesExplorerControl SeriesID = "explorer_control" + SeriesPublicationControl SeriesID = "publication_control" + SeriesDatasetRelease SeriesID = "dataset_release" + SeriesRecipeChains SeriesID = "recipe_chains" + SeriesRecipeExecRegistries SeriesID = "recipe_exec_registries" +) + +type HistoryEntry struct { + Version int64 `json:"version"` + Name string `json:"name"` + Checksum string `json:"checksum"` +} + +type HistoryManifest struct { + Series SeriesID `json:"series"` + OwnerPackage string `json:"owner_package"` + Stores []string `json:"stores"` + Entries []HistoryEntry `json:"entries"` +} + +var manifest = []HistoryManifest{ + { + Series: SeriesSource, + OwnerPackage: "internal/backends/postgres/internal/store", + Stores: []string{ + "internal/backends/postgres/internal/store", + "internal/backends/postgres/internal/catalog", + "internal/ingest (generation load, resource/edge writers)", + "internal/dataframe/compiler/render/postgres (storage bindings)", + }, + Entries: []HistoryEntry{ + {1, "ledger", "db70f7758495f7557b3a694149b80ed6e5d4cb74ea5aa7d59a1a2eaa46fc9cde"}, + {2, "fhir_resource", "d6c737db696eea1f9e9b8cd02158d14f98dfae9d9cc1d406b3c03bcc1b9fdaf0"}, + {3, "dataset_generation_manifest", "06c614680a9c1cd4ba2ef0ac0471053bcb3d8e471c36d3852e70682887a72f5d"}, + {4, "edge_occurrence", "aca2b36e1b839919434efb5b8e6c3d1939d7c182d5e15898288ac3f1f0ccb79b"}, + {5, "resource_inventory", "acebb614e6e5c7488f01c2ead47a3389da584bd940d866927816c64fd92e4146"}, + {6, "semantic_inventory_build", "e69d0dfdd3fc67438b65df7da73ea12edb61324ec8356cbdc4abc1ae5172af3c"}, + {7, "field_profile", "202245110193213c9ae5183bfdd3a6856e88d405d26c742d5de686775beae41a"}, + {8, "batch_write_ledger", "cad1877917eb24a7f7d4cb11c578b83f007fff92ceddc3ca79989a71742b05f4"}, + {9, "semantic_inventory_contribution", "41579b3c86d17c5912195ccaa6ff8655e6a864c01a01f2a23fa1cb4aa6883b02"}, + }, + }, + { + Series: SeriesExplorerControl, + OwnerPackage: "internal/backends/postgres/internal/explorer", + Stores: []string{ + "internal/backends/postgres/internal/explorer (owner, drafts, receipts, revisions, capabilities, interpretations, selections, selection members, explicit groups, publish store)", + }, + Entries: []HistoryEntry{ + {1, "ledger", "46a99180c6c6c7d41ca13520e020b71827725ec8059cce0142e278fb7577f086"}, + {2, "explorer_owner", "fc85316901b79234695695f972d8abc17d29a726293b0ff6584c96aed1cc7056"}, + {3, "explorer_draft_revision", "4bf4fe35485cb7adc6ea46812b0ae004e89ff935a14f14dd80a300bb3cc86c3f"}, + {4, "capability_artifacts", "bec3e49f897b28ef71d7d69a5f7c71bb11d13b7c631ee26724de36a89d639222"}, + {5, "explorer_compilation_receipt", "f32b3f9c821b5451e64b3d2972fb49e61f4e0c5bfcf88c69bc5b6aa7298fc33a"}, + {6, "explorer_revision", "3f6c44bff72d7837634e8c170f0b97e256e38d35f7c844fad27f4fbda38ecc09"}, + {7, "interpretation_library", "5e308b21a3f567b3066c46458606dd361a9c90646db54ee71618902a544a9ba2"}, + {8, "selection_revisions", "562a61ef33786cf674f1c87552e2dc8e70691b34116b7f415d5f81960ed8d65d"}, + {9, "explorer_explicit_groups", "c635cb5ef7c3c04655b4f45b01ba0fe834a7f181f70638f300878caf16c9a14f"}, + {10, "explorer_explicit_group_memberships", "02e0c9ab8bf9864e7d9c82471eacc7406fe2301b025e459784044c4a44489ccd"}, + {11, "explorer_explicit_group_definition_guard_hardening", "fb45d8f8c312f8f6adb119465da5ed89ac0ddd14929a2c26aa4c917db6d704ec"}, + {12, "explorer_revision_activation_revival", "08a61bf6ca66a400112f51463f2c5e1c0a7f4cfbb2d707c0063f87829b1acfe9"}, + }, + }, + { + Series: SeriesPublicationControl, + OwnerPackage: "internal/backends/postgres/internal/publication", + Stores: []string{ + "internal/backends/postgres/internal/publication (bundle registry, leases, read pins)", + "internal/backends/postgres/internal/chartifact (private artifact catalog)", + }, + Entries: []HistoryEntry{ + {1, "ledger", "db70f7758495f7557b3a694149b80ed6e5d4cb74ea5aa7d59a1a2eaa46fc9cde"}, + {4, "publication_bundle_catalog", "adcb4683ffd1611b9c68e7a49aaa4b8670d106d65df927cbb8e11866be224943"}, + {6, "publication_lease_checkpoints", "5a2fc578d1b24ee048275b060856d23b7e1a35ff251519bfb46bbc4c1e8a917c"}, + {7, "private_clickhouse_artifacts", "e827a5a1515df903a540a76858da6ad8e5afaede01f5c0b096f9f87a9b662cfb"}, + {9, "publication_read_pins", "d538a21e219f22e5384d43a777c45cd69eec5dcd968f49e5a8ea2b8fddf4f62d"}, + }, + }, + { + Series: SeriesDatasetRelease, + OwnerPackage: "internal/backends/postgres/internal/dataset", + Stores: []string{ + "internal/backends/postgres/internal/dataset (generation manifest, project releases, active-release pointers)", + "internal/backends/postgres/internal/explorer (publish store release lane)", + }, + Entries: []HistoryEntry{ + {1, "ledger", "db70f7758495f7557b3a694149b80ed6e5d4cb74ea5aa7d59a1a2eaa46fc9cde"}, + {3, "dataset_generation_manifest", "06c614680a9c1cd4ba2ef0ac0471053bcb3d8e471c36d3852e70682887a72f5d"}, + {8, "project_release", "1d581ec39731958dd0f79cdab18e92861a380007d3e85d6de4940800378480b1"}, + }, + }, + { + Series: SeriesRecipeChains, + OwnerPackage: "internal/backends/postgres/internal/recipes", + Stores: []string{ + "internal/backends/postgres/internal/recipes (revision chains)", + }, + Entries: []HistoryEntry{ + {1, "ledger", "12d5f68a77f1d4e42023a7100355a1568ff870c71644014454067c9df12099dc"}, + {3, "recipe_revision_chains", "a99114804e9ed55de3a0951c7573a6578529a602a166aa5af7a3d0d2dd685059"}, + }, + }, + { + Series: SeriesRecipeExecRegistries, + OwnerPackage: "internal/backends/postgres/internal/recipeexec", + Stores: []string{ + "internal/backends/postgres/internal/recipeexec (persistent recipe registries)", + }, + Entries: []HistoryEntry{ + {1, "ledger", "12d5f68a77f1d4e42023a7100355a1568ff870c71644014454067c9df12099dc"}, + {2, "recipe_exec_registries", "f8b6a4317268fc135aa4a301bfa5d9e40da96d7ff8ee2686b5d93fa8d4906c8b"}, + }, + }, +} + +func Manifest() []HistoryManifest { + out := make([]HistoryManifest, 0, len(manifest)) + for _, m := range manifest { + out = append(out, m.clone()) + } + sort.Slice(out, func(i, j int) bool { return out[i].Series < out[j].Series }) + return out +} + +func LookupManifest(series SeriesID) (HistoryManifest, bool) { + for _, m := range manifest { + if m.Series == series { + return m.clone(), true + } + } + return HistoryManifest{}, false +} + +func ManifestJSON() ([]byte, error) { + return json.MarshalIndent(Manifest(), "", " ") +} + +func (m HistoryManifest) clone() HistoryManifest { + out := m + out.Stores = append([]string(nil), m.Stores...) + out.Entries = append([]HistoryEntry(nil), m.Entries...) + return out +} + +type Assignment map[SeriesID]string + +type AssignmentConflict struct { + Schema string + SeriesA SeriesID + SeriesB SeriesID + Version int64 + NameA string + ChecksumA string + NameB string + ChecksumB string +} + +func (c *AssignmentConflict) Error() string { + return fmt.Sprintf("postgres: series %s and %s cannot share schema %q: version %d is %s checksum %s in %s but %s checksum %s in %s", + c.SeriesA, c.SeriesB, c.Schema, c.Version, c.NameA, c.ChecksumA[:12], c.SeriesA, c.NameB, c.ChecksumB[:12], c.SeriesB) +} + +func (c *AssignmentConflict) Unwrap() error { return ErrInvalidAssignment } + +func ValidateAssignment(assignment Assignment) error { + bySchema := map[string][]SeriesID{} + for series, schema := range assignment { + if _, ok := LookupManifest(series); !ok { + return fmt.Errorf("%w: unknown series %q", ErrInvalidAssignment, series) + } + if !ValidSchemaName(schema) { + return fmt.Errorf("%w: series %q schema %q must match [a-z_][a-z0-9_]{0,62}", + ErrInvalidAssignment, series, schema) + } + bySchema[schema] = append(bySchema[schema], series) + } + schemas := make([]string, 0, len(bySchema)) + for schema := range bySchema { + schemas = append(schemas, schema) + } + sort.Strings(schemas) + for _, schema := range schemas { + series := bySchema[schema] + sort.Slice(series, func(i, j int) bool { return series[i] < series[j] }) + for i := 0; i < len(series); i++ { + for j := i + 1; j < len(series); j++ { + if conflict := seriesConflict(schema, series[i], series[j]); conflict != nil { + return conflict + } + } + } + } + return nil +} + +func seriesConflict(schema string, a, b SeriesID) *AssignmentConflict { + ma, okA := LookupManifest(a) + mb, okB := LookupManifest(b) + if !okA || !okB { + return nil + } + byVersion := make(map[int64]HistoryEntry, len(ma.Entries)) + for _, e := range ma.Entries { + byVersion[e.Version] = e + } + for _, eb := range mb.Entries { + if ea, shared := byVersion[eb.Version]; shared && (ea.Name != eb.Name || ea.Checksum != eb.Checksum) { + return &AssignmentConflict{ + Schema: schema, SeriesA: a, SeriesB: b, Version: eb.Version, + NameA: ea.Name, ChecksumA: ea.Checksum, NameB: eb.Name, ChecksumB: eb.Checksum, + } + } + } + return nil +} diff --git a/internal/backends/postgres/internal/store/migrations/000001_ledger.sql b/internal/backends/postgres/internal/store/migrations/000001_ledger.sql new file mode 100644 index 000000000..13c0efcfc --- /dev/null +++ b/internal/backends/postgres/internal/store/migrations/000001_ledger.sql @@ -0,0 +1,17 @@ +-- pg-store-02: create the migration ledger. +-- +-- The ledger is itself a versioned migration so its creation is audited by +-- the same runner that reads it. It stores version, name, and checksum only +-- (plus when it was applied) -- never the migration SQL content. The runner +-- inserts each row only after the migration transaction commits, so a +-- recorded migration always applied successfully. +CREATE TABLE schema_migrations ( + version bigint NOT NULL, + name text NOT NULL, + checksum text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT schema_migrations_pkey PRIMARY KEY (version), + CONSTRAINT schema_migrations_version_positive CHECK (version > 0), + CONSTRAINT schema_migrations_name_shape CHECK (name ~ '^[a-z][a-z0-9_]*$'), + CONSTRAINT schema_migrations_checksum_hex CHECK (checksum ~ '^[0-9a-f]{64}$') +); diff --git a/internal/backends/postgres/internal/store/migrations/000002_fhir_resource.sql b/internal/backends/postgres/internal/store/migrations/000002_fhir_resource.sql new file mode 100644 index 000000000..c4dff19f5 --- /dev/null +++ b/internal/backends/postgres/internal/store/migrations/000002_fhir_resource.sql @@ -0,0 +1,67 @@ +-- pg-store-04: scoped FHIR resource persistence in the PostgreSQL source store. +-- +-- One table holds every FHIR resource type (no collection-per-resource +-- emulation). The storage identity fields that are also graph fields on the +-- Arango vertex (project, dataset generation, resource type, logical FHIR id, +-- auth path) become typed columns, and the original resource object is stored +-- as JSONB in payload. PostgreSQL canonicalizes JSONB -- object key order, +-- duplicate keys (last value wins), and numeric literals (1e2 becomes 100) -- +-- so no byte-exact round trip is promised; the guarantees are the semantic +-- ones below. +-- +-- Identity and scope rules mirrored from the Arango model +-- (internal/ingest/generation_identity.go, internal/ingest/documents.go): +-- - The natural identity is (project, dataset_generation, resource_type, +-- logical_id). The generation-qualified Arango _key is derived from +-- exactly these fields, so the same natural identity may never carry a +-- second payload: the UNIQUE constraint makes the database, not the +-- caller, the arbiter of "insert only". +-- - auth_resource_path is a typed, non-null scope value. The empty string +-- is the legacy (unscoped) namespace, mirroring the omitempty vertex +-- field; restricted readers bind it explicitly alongside project and +-- dataset generation. +-- - dataset_generation is a typed, non-null scope value. The empty string +-- is the legacy dataset namespace, never "any generation", matching the +-- catalog contract that readers bind either an exact value or the legacy +-- empty value. +-- - payload is nullable so an absent resource object (SQL NULL) stays +-- distinct from an explicit JSON null document and from JSON values +-- inside a document. JSONB preserves present-but-null keys, empty +-- arrays, and empty objects; a missing key never appears. +-- - resource_seq is the server-assigned storage identity. It is immutable +-- once assigned, so keyset reads ordered by it are stable regardless of +-- later inserts. +-- +-- The DDL is idempotent (IF NOT EXISTS) because the runner records the +-- ledger row after commit and must converge on retry. +CREATE TABLE IF NOT EXISTS fhir_resource ( + resource_seq bigint GENERATED ALWAYS AS IDENTITY, + project text NOT NULL, + dataset_generation text NOT NULL, + resource_type text NOT NULL, + logical_id text NOT NULL, + auth_resource_path text NOT NULL, + payload jsonb, + recorded_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fhir_resource_pkey PRIMARY KEY (resource_seq), + CONSTRAINT fhir_resource_natural_identity_ux + UNIQUE (project, dataset_generation, resource_type, logical_id), + CONSTRAINT fhir_resource_project_present + CHECK (project <> ''), + CONSTRAINT fhir_resource_resource_type_present + CHECK (resource_type <> ''), + CONSTRAINT fhir_resource_logical_id_present + CHECK (logical_id <> '') +); + +-- Scope-scoped keyset scans. A single-path restricted read binds project, +-- dataset generation, and one auth_resource_path, so it walks this index in +-- resource_seq order with no sort. A multi-path restricted read +-- (auth_resource_path = ANY(...)) and the unrestricted mode leave +-- auth_resource_path unbound or multi-valued: PostgreSQL still uses this +-- index for the bound prefix but must sort the fetched rows by resource_seq +-- within the scope. That sort is bounded by the scope's row count and the +-- keyset predicate; revisit the index shape if it ever shows up in capacity +-- profiles. +CREATE INDEX IF NOT EXISTS fhir_resource_scope_seq_ix + ON fhir_resource (project, dataset_generation, auth_resource_path, resource_seq); diff --git a/internal/backends/postgres/internal/store/migrations/000003_dataset_generation_manifest.sql b/internal/backends/postgres/internal/store/migrations/000003_dataset_generation_manifest.sql new file mode 100644 index 000000000..e7dae84ca --- /dev/null +++ b/internal/backends/postgres/internal/store/migrations/000003_dataset_generation_manifest.sql @@ -0,0 +1,62 @@ +-- pg-store-03: dataset generation manifests and the active-generation pointer. +-- +-- One row per dataset generation (project, generation) holds the lifecycle +-- state and the schema identity document that participates in +-- compare-and-swap equality. The active-generation pointer is one row per +-- project: a missing row means no active generation (there is no placeholder +-- row and no SQL NULL standing in for "absent"). Presence of the pointer row +-- is the compare value for activation CAS; the store never moves it outside +-- its transactional compare-and-swap. +-- +-- State values are the public dataset manifest state names, byte-for-byte: +-- LOADING, STAGED, READY (legacy spelling of STAGED), FAILED. schema_identity +-- stores the manifest's schemaIdentity JSON document; SQL NULL is not +-- admitted for any column here -- missing data is a constraint violation, +-- never a silent NULL. +-- +-- DDL is retryable (IF NOT EXISTS) per the pg-store-02 runner contract: a +-- crash between a migration's commit and its ledger record must converge on +-- re-application. +-- +-- Version window: 000002_fhir_resource (pg-store-04) landed on the +-- integration branch alongside this 000003, so the embedded history is the +-- complete {1, 2, 3} chain and the runner's out-of-order policy cannot trap +-- a schema (no commit state ever exposes 000003 without 000002). Per-test +-- fixtures always start from an empty ledger. The runner's pinned-checksum +-- gate in migrate_test.go refuses any edit to this file's bytes without an +-- explicit entry update. +CREATE TABLE IF NOT EXISTS dataset_generation_manifest ( + project text NOT NULL, + generation text NOT NULL, + state text NOT NULL, + schema_identity jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT dataset_generation_manifest_pkey + PRIMARY KEY (project, generation), + CONSTRAINT dataset_generation_manifest_project_nonempty + CHECK (project <> ''), + CONSTRAINT dataset_generation_manifest_generation_nonempty + CHECK (generation <> ''), + CONSTRAINT dataset_generation_manifest_state_public_names + CHECK (state IN ('LOADING', 'STAGED', 'READY', 'FAILED')), + CONSTRAINT dataset_generation_manifest_identity_is_object + CHECK (jsonb_typeof(schema_identity) = 'object') +); + +CREATE TABLE IF NOT EXISTS dataset_active_generation ( + project text NOT NULL, + generation text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT dataset_active_generation_pkey + PRIMARY KEY (project), + CONSTRAINT dataset_active_generation_project_nonempty + CHECK (project <> ''), + CONSTRAINT dataset_active_generation_generation_nonempty + CHECK (generation <> ''), + CONSTRAINT dataset_active_generation_manifest_fkey + FOREIGN KEY (project, generation) + REFERENCES dataset_generation_manifest (project, generation) + ON DELETE RESTRICT +); diff --git a/internal/backends/postgres/internal/store/migrations/000004_edge_occurrence.sql b/internal/backends/postgres/internal/store/migrations/000004_edge_occurrence.sql new file mode 100644 index 000000000..c5bb7a554 --- /dev/null +++ b/internal/backends/postgres/internal/store/migrations/000004_edge_occurrence.sql @@ -0,0 +1,96 @@ +-- pg-store-05: relationship edges as scoped typed occurrences. +-- +-- One table holds every relationship edge (no edge-collection-per-label +-- emulation). Endpoints are stored as typed identities -- resource type plus +-- logical FHIR id -- inside the same project/dataset-generation scope as the +-- resource table (000002_fhir_resource), not as physical document ids, so +-- edges survive the generation-qualified graph-key rewrite that +-- internal/ingest/generation_identity.go applies to Arango documents. +-- +-- Identity rules mirrored from the Arango model +-- (internal/ingest/documents.go EdgeDocument, generation_identity.go): +-- - The Arango edge key is derived from source/target/label, so identical +-- repeated references from one source document collapse to one edge. +-- The PostgreSQL contract deliberately does NOT dedupe endpoints: the +-- occurrence identity is +-- (project, dataset_generation, source_resource_type, source_logical_id, +-- label, target_resource_type, target_logical_id, source_path, +-- source_ordinal). Two identical target/label references from different +-- source path occurrences are two rows; only the fully identical +-- occurrence is a conflict. +-- - source_path and source_ordinal carry the repeated-owner correlation: +-- the path locates the reference inside the owning source document and +-- the ordinal distinguishes identical references at one path. The empty +-- source_path is the unqualified (legacy) namespace, never "any path". +-- - logical_edge_key preserves the caller's logical edge key (the +-- collapsed Arango-style key) as a diagnostic only. It carries no +-- constraint: rows that share a logical_edge_key are exactly the +-- multiplicity an endpoint-keyed store would have collapsed. A SQL NULL +-- means the caller supplied no key, distinct from a present key whose +-- value is the empty string. +-- - The target may be unresolved (a dangling FHIR reference). Its +-- resource type and logical id are stored regardless; resolution is a +-- read-time fact about fhir_resource in the same scope, never a stored +-- flag that can go stale. The source, by contrast, must exist in the +-- same project/dataset_generation scope at insert time; that +-- precondition is enforced by the repository (a plain column pair, not +-- a foreign key, so generation cleanup and bulk writes stay in their +-- owning tickets and no silent cascade can delete edges). +-- - project, dataset_generation, and auth_resource_path are typed, +-- non-null scope columns exactly as in fhir_resource; the empty strings +-- are the legacy namespaces. +-- +-- The DDL is idempotent (IF NOT EXISTS) because the runner records the +-- ledger row after commit and must converge on retry. +CREATE TABLE IF NOT EXISTS fhir_edge_occurrence ( + edge_seq bigint GENERATED ALWAYS AS IDENTITY, + project text NOT NULL, + dataset_generation text NOT NULL, + auth_resource_path text NOT NULL, + label text NOT NULL, + source_resource_type text NOT NULL, + source_logical_id text NOT NULL, + target_resource_type text NOT NULL, + target_logical_id text NOT NULL, + source_path text NOT NULL, + source_ordinal bigint NOT NULL, + logical_edge_key text, + recorded_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fhir_edge_occurrence_pkey PRIMARY KEY (edge_seq), + CONSTRAINT fhir_edge_occurrence_identity_ux + UNIQUE (project, dataset_generation, source_resource_type, + source_logical_id, label, target_resource_type, + target_logical_id, source_path, source_ordinal), + CONSTRAINT fhir_edge_occurrence_project_present + CHECK (project <> ''), + CONSTRAINT fhir_edge_occurrence_label_present + CHECK (label <> ''), + CONSTRAINT fhir_edge_occurrence_source_type_present + CHECK (source_resource_type <> ''), + CONSTRAINT fhir_edge_occurrence_source_id_present + CHECK (source_logical_id <> ''), + CONSTRAINT fhir_edge_occurrence_target_type_present + CHECK (target_resource_type <> ''), + CONSTRAINT fhir_edge_occurrence_target_id_present + CHECK (target_logical_id <> ''), + CONSTRAINT fhir_edge_occurrence_ordinal_nonnegative + CHECK (source_ordinal >= 0) +); + +-- Source-directed traversal. Binds project, dataset generation, the source +-- endpoint identity, and (in restricted mode) one auth_resource_path, then +-- walks edge_seq order with no sort. Like the fhir_resource keyset index, +-- multi-path restricted and unrestricted reads use the bound prefix and sort +-- the fetched rows by edge_seq within the scope. +CREATE INDEX IF NOT EXISTS fhir_edge_occurrence_source_traversal_ix + ON fhir_edge_occurrence (project, dataset_generation, + source_resource_type, source_logical_id, + auth_resource_path, edge_seq); + +-- Target-directed traversal (reverse lookup). Same shape, target-bound: +-- scoped edge lookup by target identity never resolves across +-- project/generation because project and dataset_generation lead the key. +CREATE INDEX IF NOT EXISTS fhir_edge_occurrence_target_traversal_ix + ON fhir_edge_occurrence (project, dataset_generation, + target_resource_type, target_logical_id, + auth_resource_path, edge_seq); diff --git a/internal/backends/postgres/internal/store/migrations/000005_resource_inventory.sql b/internal/backends/postgres/internal/store/migrations/000005_resource_inventory.sql new file mode 100644 index 000000000..03f414a22 --- /dev/null +++ b/internal/backends/postgres/internal/store/migrations/000005_resource_inventory.sql @@ -0,0 +1,88 @@ +-- pg-catalog-02: generation-scoped resource inventory counts with a +-- publish-last completeness marker in the PostgreSQL source store. +-- +-- This mirrors the Arango fhir_resource_inventory collection +-- (internal/catalog/resource_inventory.go, internal/catalog/arango/ +-- resource_inventory.go) with two tables instead of one polymorphic +-- collection: +-- +-- - fhir_resource_inventory_count holds one row per +-- (build_id, project, dataset_generation, resource_type, +-- auth_resource_path) with the exact zero-inclusive document count. +-- Counts from abandoned builds stay invisible because readers bind to +-- the published marker's build id, never to "the latest rows". +-- - fhir_resource_inventory_build holds the one published completeness +-- marker per (project, dataset_generation). A successful rebuild +-- commits every count row first and swaps the marker last; the marker +-- row is the only evidence that a build_id's counts are complete. +-- +-- Identity rules mirrored from the shared contract: +-- - count_key is the content-addressed ric_ identity assigned by +-- catalog.NewResourceInventoryBuild (project, generation, build id, +-- resource type, auth path); UNIQUE makes the database, not the +-- caller, the arbiter of one-row-per-scope. +-- - document_count is a non-negative exact count; zero rows are real +-- evidence ("this generated type is empty in this scope"), never a +-- missing row. +-- - auth_resource_path and dataset_generation are typed, non-null scope +-- values; the empty string is the legacy namespace, never "any". +-- - marker_key is the deterministic ribm_ identity derived from +-- (project, dataset_generation), so one scope holds exactly one +-- published marker and republishing is a replacement, never a second +-- marker. +-- - complete is deliberately un-constrained so a reader-side test can +-- seed an incomplete historical marker; the repository only ever +-- publishes complete markers after its commit guard passes. +-- +-- The DDL is idempotent (IF NOT EXISTS) because the runner records the +-- ledger row after commit and must converge on retry. +CREATE TABLE IF NOT EXISTS fhir_resource_inventory_count ( + count_key text NOT NULL, + build_id text NOT NULL, + project text NOT NULL, + dataset_generation text NOT NULL, + resource_type text NOT NULL, + auth_resource_path text NOT NULL, + document_count bigint NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fhir_resource_inventory_count_pkey + PRIMARY KEY (build_id, project, dataset_generation, resource_type, auth_resource_path), + CONSTRAINT fhir_resource_inventory_count_key_ux + UNIQUE (count_key), + CONSTRAINT fhir_resource_inventory_count_nonnegative + CHECK (document_count >= 0), + CONSTRAINT fhir_resource_inventory_count_build_present + CHECK (build_id <> ''), + CONSTRAINT fhir_resource_inventory_count_project_present + CHECK (project <> ''), + CONSTRAINT fhir_resource_inventory_count_type_present + CHECK (resource_type <> '') +); + +-- Reader path: sums for one published build inside one project/generation, +-- grouped per resource type and filtered by authorization path. +CREATE INDEX IF NOT EXISTS fhir_resource_inventory_count_scope_ix + ON fhir_resource_inventory_count (project, dataset_generation, build_id, resource_type); + +CREATE TABLE IF NOT EXISTS fhir_resource_inventory_build ( + marker_key text NOT NULL, + project text NOT NULL, + dataset_generation text NOT NULL, + build_id text NOT NULL, + coverage_digest text NOT NULL, + resource_types text[] NOT NULL, + resource_type_count integer NOT NULL, + count_row_count integer NOT NULL, + complete boolean NOT NULL, + published_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fhir_resource_inventory_build_pkey + PRIMARY KEY (project, dataset_generation), + CONSTRAINT fhir_resource_inventory_build_key_ux + UNIQUE (marker_key), + CONSTRAINT fhir_resource_inventory_build_project_present + CHECK (project <> ''), + CONSTRAINT fhir_resource_inventory_build_build_present + CHECK (build_id <> ''), + CONSTRAINT fhir_resource_inventory_build_types_covered + CHECK (resource_type_count >= 1 AND count_row_count >= resource_type_count) +); diff --git a/internal/backends/postgres/internal/store/migrations/000006_semantic_inventory_build.sql b/internal/backends/postgres/internal/store/migrations/000006_semantic_inventory_build.sql new file mode 100644 index 000000000..2613d2714 --- /dev/null +++ b/internal/backends/postgres/internal/store/migrations/000006_semantic_inventory_build.sql @@ -0,0 +1,82 @@ +-- pg-catalog-04: semantic-inventory build lifecycle, lease, and typed +-- checkpoint state in the PostgreSQL source store. +-- +-- This mirrors the Arango fhir_semantic_inventory_builds collection +-- (internal/catalog/semantic_inventory.go, internal/catalog/arango/ +-- semantic_inventory.go) as one typed row per build identity, owned by +-- internal/catalog/postgres/semantic_build.go: +-- +-- - One row per (project, dataset_generation, build_id): the content- +-- addressed build identity pins schema/rule/emitter versions, so a +-- version change is a different row, never an in-place edit. +-- - state carries the closed SemanticInventoryState set; the repository +-- only drives not_started/unknown -> running -> complete|failed, and a +-- completed row is never rewritten by a claim (file and retained-vertex +-- contribution sets are never mixed). +-- - lease_token/lease_expires_at (epoch milliseconds, matching +-- catalog.SemanticInventoryBuild.LeaseExpiresAt exactly) implement a +-- fenced lease: both-empty-or-both-set, so a released lease cannot be +-- half-cleared, and every guard is one statement, so exactly one +-- claimant can hold a live lease. +-- - checkpoint_collection/checkpoint_key is the typed keyset cursor that +-- advances only under compare-and-swap on the exact expected pair; the +-- legacy string checkpoint is preserved for round-trip parity with the +-- shared contract. +-- - diagnostics is an ordered text array (a failed build carries its one +-- failure diagnostic; a complete build carries none). +-- +-- Every column is NOT NULL: like migration 000005, a missing domain value +-- is the typed empty string or zero, never SQL NULL, so SQL NULL carries no +-- meaning here and nothing collapses "missing" into "null". The DDL is +-- idempotent (IF NOT EXISTS) because the runner records the ledger row +-- after commit and must converge on retry. +CREATE TABLE IF NOT EXISTS fhir_semantic_inventory_build ( + build_key text NOT NULL, + project text NOT NULL, + dataset_generation text NOT NULL, + build_id text NOT NULL, + state text NOT NULL, + observation_schema integer NOT NULL, + inventory_schema integer NOT NULL, + entry_index_version integer NOT NULL DEFAULT 0, + rule_version integer NOT NULL, + source_kind text NOT NULL, + source_availability text NOT NULL, + auth_resource_path text NOT NULL DEFAULT '', + scanned_resources bigint NOT NULL DEFAULT 0, + checkpoint text NOT NULL DEFAULT '', + checkpoint_collection text NOT NULL DEFAULT '', + checkpoint_key text NOT NULL DEFAULT '', + lease_token text NOT NULL DEFAULT '', + lease_expires_at bigint NOT NULL DEFAULT 0, + diagnostics text[] NOT NULL DEFAULT '{}', + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fhir_semantic_inventory_build_pkey + PRIMARY KEY (project, dataset_generation, build_id), + CONSTRAINT fhir_semantic_inventory_build_key_ux + UNIQUE (build_key), + CONSTRAINT fhir_semantic_inventory_build_key_shape + CHECK (build_key ~ '^sib_[0-9a-f]{64}$'), + CONSTRAINT fhir_semantic_inventory_build_project_present + CHECK (project <> ''), + CONSTRAINT fhir_semantic_inventory_build_build_present + CHECK (build_id <> ''), + CONSTRAINT fhir_semantic_inventory_build_build_shape + CHECK (build_id ~ '^[0-9a-f]{64}$'), + CONSTRAINT fhir_semantic_inventory_build_state_known + CHECK (state IN ('unknown', 'not_started', 'running', 'complete', 'failed', 'invalidated')), + CONSTRAINT fhir_semantic_inventory_build_versions_sane + CHECK (observation_schema >= 1 AND inventory_schema >= 1 AND rule_version >= 1 AND entry_index_version >= 0), + CONSTRAINT fhir_semantic_inventory_build_source_kind_known + CHECK (source_kind IN ('file', 'retained_vertex')), + CONSTRAINT fhir_semantic_inventory_build_availability_known + CHECK (source_availability IN ('unknown', 'verified', 'unproven')), + CONSTRAINT fhir_semantic_inventory_build_scanned_nonnegative + CHECK (scanned_resources >= 0), + CONSTRAINT fhir_semantic_inventory_build_checkpoint_paired + CHECK ((checkpoint_collection = '') = (checkpoint_key = '')), + CONSTRAINT fhir_semantic_inventory_build_lease_paired + CHECK ((lease_token = '') = (lease_expires_at = 0)), + CONSTRAINT fhir_semantic_inventory_build_lease_expiry_nonnegative + CHECK (lease_expires_at >= 0) +); diff --git a/internal/backends/postgres/internal/store/migrations/000007_field_profile.sql b/internal/backends/postgres/internal/store/migrations/000007_field_profile.sql new file mode 100644 index 000000000..696dad69e --- /dev/null +++ b/internal/backends/postgres/internal/store/migrations/000007_field_profile.sql @@ -0,0 +1,153 @@ +-- pg-catalog-01: bounded field-profile records persisted from the write-side +-- catalog.Profiler (internal/catalog/write_profiler.go Documents()) in the +-- PostgreSQL source store, replacing the Arango fhir_field_catalog +-- collection for this backend (internal/catalog/arango/fields.go is the +-- behavioral reference). +-- +-- Two tables instead of one polymorphic Arango collection: +-- +-- - fhir_field_profile holds one row per field-profile identity +-- (project, dataset_generation, auth_resource_path, resource_type, path) +-- plus the scope-level profile_status record (path = '', record_type = +-- 'profile_status') that the profiler emits when enrichment truncated +-- with no fields retained. Query identity is normalized into typed +-- columns; JSONB is used only for the correlated structured subrecords +-- (extension_values, semantic_observations), where each element is one +-- correlated observation and the correlation must never be flattened +-- into independently cross-combinable value sets. +-- - fhir_field_profile_coverage holds one coverage marker per profiler +-- scope (project, dataset_generation, auth_resource_path, resource_type). +-- It is what makes a persisted known-empty profile distinguishable from +-- a scope that was never profiled: the profiler emits zero documents for +-- a clean empty population, so without this marker "empty" and +-- "unavailable" would collapse into the same zero-row read (the Arango +-- backend gets this for free from collection existence; a relational +-- table created by a migration always exists). +-- +-- Fidelity rules pinned by internal/catalog/postgres tests: +-- - Every public field of catalog.FieldCatalogDocument round-trips +-- unchanged, including false booleans and zero counts. +-- - Optional text columns are NOT NULL with no DEFAULT: Go strings have +-- no missing/empty distinction, the repository always supplies all 26 +-- column values, so the Go zero value round-trips exactly, and a direct +-- SQL writer that omits a column gets a NOT NULL violation rather than +-- a silent empty string. +-- - enrichment_complete is a nullable boolean: SQL NULL is the missing +-- *bool (nil), true/false are the explicit states. It is never coerced. +-- - distinct_values, pivot_columns, and pivot_value_selectors are text[]: +-- SQL NULL and '{}' are preserved distinctly in the database, and both +-- decode to a nil Go slice (the profiler only ever emits nil-or-populated +-- via append(nil, ...) copies). The adapter passes nil for an empty Go +-- slice, so a profiler round-trip keeps nil as nil. +-- - extension_values and semantic_observations are jsonb arrays: SQL NULL +-- is the missing (nil) Go slice, '[]' is an empty-but-non-nil slice, +-- and a populated array holds one object per correlated observation. +-- The adapter encodes the Go nil/empty/populated distinction explicitly +-- (encoding/json's omitempty would otherwise erase it inside member +-- slices such as a profiler-retained empty Examples list). A jsonb +-- 'null' or malformed non-array value is refused by the type checks +-- below (which pass on SQL NULL because jsonb_typeof(NULL) is NULL and +-- CHECK accepts unknown). +-- - Identity columns carry COLLATE "C" so the primary key and ORDER BY +-- use exact byte order, matching Arango's byte-order SORT and the +-- profiler's Go byte-order sort of paths. +-- - profile_key is stored as written (the profiler's canonical +-- field-catalog key) and returned unchanged. It is deliberately NOT the +-- primary key: the legacy (empty-generation) key derivation sanitizes +-- its components many-to-one, so the full identity tuple -- not the +-- derived key -- is the one-row-per-profile arbiter. Two distinct +-- scopes are therefore never merged, and upsert replaces exactly one +-- identity. +-- - The DDL is idempotent (IF NOT EXISTS) because the runner records the +-- ledger row after commit and must converge on retry. +-- +-- Versioning: this migration is 000007, not 000006. Version 000006 was +-- reserved for pg-catalog-04's semantic_inventory_build migration while +-- this ticket was in flight, so this migration was numbered 7 and the +-- branch history temporarily skipped version 6 (the runner tolerates the +-- gap). That reservation has been delivered: #73 (eafbf05a) merged +-- 000006_semantic_inventory_build.sql, and the embedded history is the +-- contiguous chain 1..7. The jump in numbering existed because of the +-- runner contract: MigrationRunner.pending() refuses any pending +-- migration whose version is below the highest already-recorded version +-- (*OutOfOrderMigrationError, ErrMigrationOutOfOrder), so once 000007 is +-- applied to a database, 000006 can never be applied there. The merge +-- queue therefore sequenced catalog-04 first: 000006 merged AND was +-- applied to every database before 000007 reached any. (A database that +-- had applied 000007 during the in-flight gap window would have been +-- permanently unable to receive 000006 -- which is why this ticket +-- waited for #73 to land before merging.) +CREATE TABLE IF NOT EXISTS fhir_field_profile ( + project text NOT NULL COLLATE "C", + dataset_generation text NOT NULL COLLATE "C", + auth_resource_path text NOT NULL COLLATE "C", + resource_type text NOT NULL COLLATE "C", + path text NOT NULL COLLATE "C", + profile_key text NOT NULL, + record_type text NOT NULL, + kind text NOT NULL, + doc_count bigint NOT NULL, + max_items bigint NOT NULL, + sample_count bigint NOT NULL, + distinct_values text[], + distinct_truncated boolean NOT NULL, + enrichment_complete boolean, + enrichment_truncated boolean NOT NULL, + pivot_candidate boolean NOT NULL, + pivot_kind text NOT NULL, + pivot_columns text[], + pivot_family text NOT NULL, + pivot_column_selector text NOT NULL, + pivot_value_selector text NOT NULL, + pivot_item_source text NOT NULL, + pivot_item_resource_type text NOT NULL, + pivot_value_selectors text[], + extension_values jsonb, + semantic_observations jsonb, + CONSTRAINT fhir_field_profile_pkey + PRIMARY KEY (project, dataset_generation, auth_resource_path, resource_type, path), + CONSTRAINT fhir_field_profile_project_present + CHECK (project <> ''), + CONSTRAINT fhir_field_profile_type_present + CHECK (resource_type <> ''), + CONSTRAINT fhir_field_profile_key_present + CHECK (profile_key <> ''), + -- One shape per record kind: a field profile always has a path and a + -- kind; the scope-level profile_status marker never does. Mixed shapes + -- (a status row with a path, a field row without a kind) are refused. + CONSTRAINT fhir_field_profile_record_shape + CHECK ((record_type = '' AND path <> '' AND kind <> '') + OR (record_type = 'profile_status' AND path = '')), + CONSTRAINT fhir_field_profile_record_type_known + CHECK (record_type IN ('', 'profile_status')), + CONSTRAINT fhir_field_profile_counts_nonnegative + CHECK (doc_count >= 0 AND max_items >= 0 AND sample_count >= 0), + -- Correlated subrecords are jsonb arrays (or SQL NULL when missing). + CONSTRAINT fhir_field_profile_extension_is_array + CHECK (jsonb_typeof(extension_values) = 'array'), + CONSTRAINT fhir_field_profile_semantic_is_array + CHECK (jsonb_typeof(semantic_observations) = 'array') +); + +-- Reader path: discovery reads bind project, dataset generation, then +-- filter resource types and (in restricted mode) authorization paths. +CREATE INDEX IF NOT EXISTS fhir_field_profile_scope_ix + ON fhir_field_profile (project, dataset_generation, resource_type, auth_resource_path); + +CREATE TABLE IF NOT EXISTS fhir_field_profile_coverage ( + project text NOT NULL COLLATE "C", + dataset_generation text NOT NULL COLLATE "C", + auth_resource_path text NOT NULL COLLATE "C", + resource_type text NOT NULL COLLATE "C", + -- Nullable: SQL NULL is the missing *bool enrichment-completeness + -- claim, exactly like fhir_field_profile.enrichment_complete. + enrichment_complete boolean, + enrichment_truncated boolean NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fhir_field_profile_coverage_pkey + PRIMARY KEY (project, dataset_generation, auth_resource_path, resource_type), + CONSTRAINT fhir_field_profile_coverage_project_present + CHECK (project <> ''), + CONSTRAINT fhir_field_profile_coverage_type_present + CHECK (resource_type <> '') +); diff --git a/internal/backends/postgres/internal/store/migrations/000008_batch_write_ledger.sql b/internal/backends/postgres/internal/store/migrations/000008_batch_write_ledger.sql new file mode 100644 index 000000000..b633b6cc6 --- /dev/null +++ b/internal/backends/postgres/internal/store/migrations/000008_batch_write_ledger.sql @@ -0,0 +1,56 @@ +-- pg-store-06: idempotent COPY batch-write ledger for the PostgreSQL source store. +-- +-- One row per committed bulk write unit (resource and edge rows written with +-- PostgreSQL COPY through transaction-local staging). The row is claimed +-- inside the batch's own transaction, before any row is staged or published, +-- so the ledger marker and the published rows commit or roll back together: +-- a batch that failed before commit leaves neither rows nor marker, and a +-- retry of identical committed input reads the marker and stops. +-- +-- Identity rules: +-- - batch_id is the caller's deterministic batch identity (stable across +-- retries of one write unit). PRIMARY KEY makes the database, not the +-- caller, the arbiter of "this identity has committed once"; concurrent +-- claimants of one identity serialize on the key. +-- - digest is the lowercase hex SHA-256 of the canonical encoding of the +-- batch content (scope, ordered rows, every column, payload bytes or +-- their absence). Replaying the same identity with an equal digest is a +-- no-op; reusing the identity with a different digest is refused by the +-- writer as a typed conflict, so an ambiguous retry can never replace +-- a different payload under one identity. +-- - project and dataset_generation record the batch scope exactly as +-- fhir_resource and fhir_edge_occurrence store it; the empty +-- dataset_generation is the legacy dataset namespace, never "any". +-- - resource_count and edge_count are the exact numbers of rows the +-- batch's publish statements inserted into their scoped tables (not the +-- number offered, which may collapse duplicates or skip identical +-- already-stored rows). Readers get exact commit accounting from here. +-- - committed_at is the wall-clock moment the batch's commit accounting +-- was finalized. The marker row is claimed EARLY in the batch's own +-- transaction, so a DEFAULT now() value would hold the transaction +-- START (PostgreSQL now() returns the transaction snapshot time), which +-- precedes the real commit by the whole batch duration. The writer's +-- finalize statement therefore overwrites the column with +-- clock_timestamp() (the real per-call wall clock) when it records the +-- committed counts; true commit time is unknowable from inside the +-- transaction, and the marker deliberately lands inside the +-- transaction so rows and marker commit or roll back together. The +-- DEFAULT only covers hand-seeded rows. +-- +-- The DDL is idempotent (IF NOT EXISTS) because the runner records the +-- ledger row after commit and must converge on retry. +CREATE TABLE IF NOT EXISTS batch_write_ledger ( + batch_id text NOT NULL, + project text NOT NULL, + dataset_generation text NOT NULL, + digest text NOT NULL, + resource_count bigint NOT NULL DEFAULT 0, + edge_count bigint NOT NULL DEFAULT 0, + committed_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT batch_write_ledger_pkey PRIMARY KEY (batch_id), + CONSTRAINT batch_write_ledger_batch_present CHECK (batch_id <> ''), + CONSTRAINT batch_write_ledger_project_present CHECK (project <> ''), + CONSTRAINT batch_write_ledger_digest_hex CHECK (digest ~ '^[0-9a-f]{64}$'), + CONSTRAINT batch_write_ledger_counts_nonnegative + CHECK (resource_count >= 0 AND edge_count >= 0) +); diff --git a/internal/backends/postgres/internal/store/migrations/000009_semantic_inventory_contribution.sql b/internal/backends/postgres/internal/store/migrations/000009_semantic_inventory_contribution.sql new file mode 100644 index 000000000..1712debc8 --- /dev/null +++ b/internal/backends/postgres/internal/store/migrations/000009_semantic_inventory_contribution.sql @@ -0,0 +1,113 @@ +-- pg-catalog-05: deterministic semantic-inventory contribution persistence +-- for the PostgreSQL source-store retained-source backfill +-- (internal/ingest/semantic_inventory_backfill.go), owned by +-- internal/catalog/postgres/semantic_contribution.go. The Arango behavioral +-- reference is the fhir_semantic_inventory collection written by +-- internal/catalog/arango/semantic_inventory.go WriteSemanticInventory- +-- Contributions. +-- +-- One row per contribution identity. The contribution key is the +-- content-addressed digest the semantic emitter derives from +-- (project, dataset generation, auth path, resource type, build id, source +-- id, ordinal[, source kind]) — so identity is build/source/ordinal scoped, +-- and replaying one generation replaces the same key, never a near-duplicate. +-- +-- Identity and replay rules (mirroring the pg-store-06 batch ledger digest +-- discipline): +-- - contribution_key is the content-addressed identity. UNIQUE makes the +-- database, not the caller, the arbiter of "this identity has written +-- once"; the writer replays it idempotently by payload digest and +-- refuses a differing payload at that identity as a typed conflict +-- instead of overwriting. +-- - The primary key is the (build, source, ordinal) slot — the +-- architecture's contribution identity — so a second, differently keyed +-- contribution can never claim one source slot, and every guard is one +-- statement. +-- - payload_digest is the lowercase hex SHA-256 of the canonical Go +-- encoding of the payload-bearing content (auth path, resource type, +-- concept, binding, example, observation). Equal digest at one key is a +-- no-op replay; a differing digest is refused. PostgreSQL stores the +-- digest as an opaque exact string: no SQL-side re-encoding, trim, or +-- lowering participates in identity comparison (the single source of +-- truth is Go post-canonicalization). +-- - observation is the observation JSONB. PostgreSQL canonicalizes JSONB +-- (key order, duplicate keys, numeric literals), so no byte-exact round +-- trip is promised; the semantic identity of "same payload" is carried +-- by payload_digest, and the observation type check refuses anything +-- that is not a JSON object. +-- - Identity columns carry COLLATE "C" so the primary key, the unique +-- key, and the classification lookups use exact byte order, matching +-- migration 000007's convention and Go string comparison. +-- - Every column is NOT NULL: a missing domain value is the typed empty +-- string or zero, never SQL NULL, so SQL NULL carries no meaning here. +-- +-- The supporting index (project, dataset_generation, build_id, source_kind, +-- binding_id, concept_id, auth_resource_path) mirrors the Arango +-- contribution collection's page index for later bounded catalog page +-- reads; the source index supports per-source replay audits. +-- +-- The trailing CREATE INDEX extends the pg-store-04 fhir_resource table +-- with the byte-order keyset index this slice's retained-source page +-- reader needs: pages walk (project, dataset_generation, resource_type, +-- logical_id COLLATE "C") with no sort, matching the adapter's COLLATE "C" +-- predicate exactly. It is purely additive (IF NOT EXISTS): it modifies no +-- column, constraint, or existing index of that table. +-- +-- The DDL is idempotent (IF NOT EXISTS) because the runner records the +-- ledger row after commit and must converge on retry. +CREATE TABLE IF NOT EXISTS fhir_semantic_inventory_contribution ( + contribution_key text NOT NULL COLLATE "C", + project text NOT NULL COLLATE "C", + dataset_generation text NOT NULL COLLATE "C", + build_id text NOT NULL COLLATE "C", + source_kind text NOT NULL COLLATE "C", + auth_resource_path text NOT NULL COLLATE "C", + resource_type text NOT NULL COLLATE "C", + source_id text NOT NULL COLLATE "C", + ordinal integer NOT NULL, + concept_id text NOT NULL COLLATE "C", + binding_id text NOT NULL COLLATE "C", + example text NOT NULL COLLATE "C", + payload_digest text NOT NULL COLLATE "C", + observation jsonb NOT NULL, + recorded_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fhir_semantic_inventory_contribution_pkey + PRIMARY KEY (project, dataset_generation, build_id, source_kind, source_id, ordinal), + CONSTRAINT fhir_semantic_inventory_contribution_key_ux + UNIQUE (contribution_key), + CONSTRAINT fhir_semantic_inventory_contribution_key_shape + CHECK (contribution_key ~ '^[0-9a-f]{64}$'), + CONSTRAINT fhir_semantic_inventory_contribution_digest_shape + CHECK (payload_digest ~ '^[0-9a-f]{64}$'), + CONSTRAINT fhir_semantic_inventory_contribution_project_present + CHECK (project <> ''), + CONSTRAINT fhir_semantic_inventory_contribution_build_present + CHECK (build_id <> ''), + CONSTRAINT fhir_semantic_inventory_contribution_build_shape + CHECK (build_id ~ '^[0-9a-f]{64}$'), + CONSTRAINT fhir_semantic_inventory_contribution_resource_type_present + CHECK (resource_type <> ''), + CONSTRAINT fhir_semantic_inventory_contribution_source_present + CHECK (source_id <> ''), + CONSTRAINT fhir_semantic_inventory_contribution_concept_present + CHECK (concept_id <> ''), + CONSTRAINT fhir_semantic_inventory_contribution_binding_present + CHECK (binding_id <> ''), + CONSTRAINT fhir_semantic_inventory_contribution_ordinal_nonnegative + CHECK (ordinal >= 0), + CONSTRAINT fhir_semantic_inventory_contribution_source_kind_known + CHECK (source_kind IN ('file', 'retained_vertex')), + CONSTRAINT fhir_semantic_inventory_contribution_observation_object + CHECK (jsonb_typeof(observation) = 'object') +); + +CREATE INDEX IF NOT EXISTS fhir_semantic_inventory_contribution_scope_ix + ON fhir_semantic_inventory_contribution + (project, dataset_generation, build_id, source_kind, binding_id, concept_id, auth_resource_path); + +CREATE INDEX IF NOT EXISTS fhir_semantic_inventory_contribution_source_ix + ON fhir_semantic_inventory_contribution + (project, dataset_generation, build_id, source_kind, source_id); + +CREATE INDEX IF NOT EXISTS fhir_resource_semantic_keyset_ix + ON fhir_resource (project, dataset_generation, resource_type, logical_id COLLATE "C"); diff --git a/internal/backends/postgres/internal/store/ownership_contract_test.go b/internal/backends/postgres/internal/store/ownership_contract_test.go new file mode 100644 index 000000000..87626ed9c --- /dev/null +++ b/internal/backends/postgres/internal/store/ownership_contract_test.go @@ -0,0 +1,1303 @@ +package postgres_test + +import ( + "context" + "crypto/sha256" + "embed" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "sort" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/calypr/loom/internal/authscope" + chartifactpg "github.com/calypr/loom/internal/backends/postgres/internal/chartifact" + datasetpg "github.com/calypr/loom/internal/backends/postgres/internal/dataset" + explorerpg "github.com/calypr/loom/internal/backends/postgres/internal/explorer" + publicationpg "github.com/calypr/loom/internal/backends/postgres/internal/publication" + recipeexecpg "github.com/calypr/loom/internal/backends/postgres/internal/recipeexec" + recipepg "github.com/calypr/loom/internal/backends/postgres/internal/recipes" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + chartifact "github.com/calypr/loom/internal/dataframe/execution/chartifact" + "github.com/calypr/loom/internal/dataframe/publication" + "github.com/calypr/loom/internal/dataframe/recipe" + recipeexec "github.com/calypr/loom/internal/dataframe/recipe/exec" + "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/explorer" + "github.com/calypr/loom/internal/source" +) + +const ( + ab05Prefix = "verify_ab05_" + preRefactorRoot = "testdata/ab05_pre_refactor" +) + +//go:embed migrations/*.sql +var ab05SourceFS embed.FS + +var ab05OwnerHistories = map[pgstore.SeriesID]func() ([]pgstore.Migration, error){ + pgstore.SeriesSource: pgstore.EmbeddedMigrations, + pgstore.SeriesExplorerControl: explorerpg.OwnerMigrations, + pgstore.SeriesPublicationControl: publicationpg.OwnerMigrations, + pgstore.SeriesDatasetRelease: datasetpg.OwnerMigrations, + pgstore.SeriesRecipeChains: recipepg.ChainMigrations, + pgstore.SeriesRecipeExecRegistries: recipeexecpg.RegistryMigrations, +} + +var ab05SignatureTable = map[pgstore.SeriesID]string{ + pgstore.SeriesSource: "fhir_resource", + pgstore.SeriesExplorerControl: explorerpg.OwnerTable, + pgstore.SeriesPublicationControl: publicationpg.BundleLeasesTable, + pgstore.SeriesDatasetRelease: datasetpg.ProjectReleasesTable, + pgstore.SeriesRecipeChains: recipepg.RevisionsTable, + pgstore.SeriesRecipeExecRegistries: recipeexecpg.RecipesTable, +} + +var ab05ChecksumPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + +func ab05ManifestMatchesOwners(t *testing.T) { + t.Helper() + manifest := pgstore.Manifest() + if len(manifest) != len(ab05OwnerHistories) { + t.Fatalf("manifest holds %d series, ownership map knows %d: register the new series in both places", + len(manifest), len(ab05OwnerHistories)) + } + for _, m := range manifest { + history, ok := ab05OwnerHistories[m.Series] + if !ok { + t.Fatalf("series %s has no owner history function in the contract test", m.Series) + } + entries, err := history() + if err != nil { + t.Fatalf("series %s: owner history: %v", m.Series, err) + } + if len(entries) != len(m.Entries) { + t.Fatalf("series %s: owner embeds %d migrations, manifest records %d: append the new version to migration_ownership.go in the same change", + m.Series, len(entries), len(m.Entries)) + } + for i := range entries { + if entries[i].Version != m.Entries[i].Version || entries[i].Name != m.Entries[i].Name || + entries[i].Checksum != m.Entries[i].Checksum { + t.Fatalf("series %s entry %d = %d/%s/%s, manifest records %d/%s/%s: manifest drifted from the embedded bytes", + m.Series, i, entries[i].Version, entries[i].Name, entries[i].Checksum[:12], + m.Entries[i].Version, m.Entries[i].Name, m.Entries[i].Checksum[:12]) + } + if i > 0 && entries[i-1].Version >= entries[i].Version { + t.Fatalf("series %s: owner history not strictly ascending at entry %d", m.Series, i) + } + } + looked, ok := pgstore.LookupManifest(m.Series) + if !ok { + t.Fatalf("LookupManifest(%s) not found", m.Series) + } + if !reflect.DeepEqual(looked, m) { + t.Fatalf("LookupManifest(%s) disagrees with Manifest()", m.Series) + } + looked.Entries[0].Checksum = strings.Repeat("0", 64) + looked.Stores = append(looked.Stores, "tampered") + if again, _ := pgstore.LookupManifest(m.Series); !reflect.DeepEqual(again, m) { + t.Fatalf("LookupManifest(%s) hands out views into the authored record, not copies", m.Series) + } + } + raw, err := pgstore.ManifestJSON() + if err != nil { + t.Fatalf("ManifestJSON: %v", err) + } + var rendered []pgstore.HistoryManifest + if err := json.Unmarshal(raw, &rendered); err != nil { + t.Fatalf("ManifestJSON did not round-trip: %v", err) + } + if !reflect.DeepEqual(rendered, manifest) { + t.Fatal("ManifestJSON rendering differs from Manifest()") + } +} + +func ab05ManifestRecordInvariants(t *testing.T) { + t.Helper() + want := []pgstore.SeriesID{ + pgstore.SeriesDatasetRelease, pgstore.SeriesExplorerControl, pgstore.SeriesPublicationControl, + pgstore.SeriesRecipeChains, pgstore.SeriesRecipeExecRegistries, pgstore.SeriesSource, + } + manifest := pgstore.Manifest() + if len(manifest) != len(want) { + t.Fatalf("series count = %d, want exactly the six production series", len(manifest)) + } + for i, m := range manifest { + if m.Series != want[i] { + t.Fatalf("Manifest() order = %v, want sorted %v", manifest[i].Series, want[i]) + } + if m.OwnerPackage == "" { + t.Fatalf("series %s: no owner package recorded", m.Series) + } + if _, err := os.Stat(filepath.Join(repoRootForOwnerStat(t), m.OwnerPackage)); err != nil { + t.Fatalf("series %s: owner package %s is not a real directory: %v", m.Series, m.OwnerPackage, err) + } + if len(m.Stores) == 0 { + t.Fatalf("series %s: no production stores recorded for this history", m.Series) + } + var prev int64 + for _, e := range m.Entries { + if e.Version <= prev { + t.Fatalf("series %s: entries not strictly ascending at version %d", m.Series, e.Version) + } + prev = e.Version + if e.Name == "" { + t.Fatalf("series %s version %d: empty name", m.Series, e.Version) + } + if !ab05ChecksumPattern.MatchString(e.Checksum) { + t.Fatalf("series %s version %d: checksum %q is not 64 lowercase hex", m.Series, e.Version, e.Checksum) + } + } + } +} + +var ab05MovedFiles = []struct { + preRefactor string + series pgstore.SeriesID + version int64 + name string +}{ + {"tests/migrations-capabilities/000004_capability_artifacts.sql", pgstore.SeriesExplorerControl, 4, "capability_artifacts"}, + {"tests/migrations-interpretations/000007_interpretation_library.sql", pgstore.SeriesExplorerControl, 7, "interpretation_library"}, + {"tests/migrations-selections/000008_selection_revisions.sql", pgstore.SeriesExplorerControl, 8, "selection_revisions"}, + {"tests/migrations/000004_publication_bundle_catalog.sql", pgstore.SeriesPublicationControl, 4, "publication_bundle_catalog"}, + {"tests/migrations-leases/000006_publication_lease_checkpoints.sql", pgstore.SeriesPublicationControl, 6, "publication_lease_checkpoints"}, + {"tests/migrations-chartifact/000007_private_clickhouse_artifacts.sql", pgstore.SeriesPublicationControl, 7, "private_clickhouse_artifacts"}, + {"tests/migrations-readpins/000009_publication_read_pins.sql", pgstore.SeriesPublicationControl, 9, "publication_read_pins"}, + {"internal/dataset/postgres/testdata/migrations-releases/000008_project_release.sql", pgstore.SeriesDatasetRelease, 8, "project_release"}, +} + +func ab05PreRefactorBytesFrozen(t *testing.T) { + t.Helper() + for _, moved := range ab05MovedFiles { + snapshot := filepath.Join(preRefactorRoot, filepath.FromSlash(moved.preRefactor)) + disk, err := os.ReadFile(snapshot) + if err != nil { + t.Fatalf("%s: frozen snapshot missing: %v", snapshot, err) + } + m, ok := pgstore.LookupManifest(moved.series) + if !ok { + t.Fatalf("series %s missing from the manifest", moved.series) + } + var pinned *pgstore.HistoryEntry + for i := range m.Entries { + if m.Entries[i].Version == moved.version { + pinned = &m.Entries[i] + } + } + if pinned == nil || pinned.Name != moved.name { + t.Fatalf("%s: manifest series %s does not pin version %d as %s", + moved.preRefactor, moved.series, moved.version, moved.name) + } + sum := sha256.Sum256(disk) + if got := hex.EncodeToString(sum[:]); got != pinned.Checksum { + t.Fatalf("%s: snapshot sha256 %s != manifest/ledger identity %s for %d/%s", + moved.preRefactor, got[:12], pinned.Checksum[:12], pinned.Version, pinned.Name) + } + entries, err := ab05OwnerHistories[moved.series]() + if err != nil { + t.Fatalf("series %s owner history: %v", moved.series, err) + } + var embedded *pgstore.Migration + for i := range entries { + if entries[i].Version == moved.version { + embedded = &entries[i] + } + } + if embedded == nil || string(disk) != embedded.SQL { + t.Fatalf("%s: frozen snapshot bytes differ from the bytes embedded in %s (the move must change paths, never bytes)", + moved.preRefactor, m.OwnerPackage) + } + } +} + +func ab05SharedBootstrapBytesIdentical(t *testing.T) { + t.Helper() + source, err := pgstore.EmbeddedMigrations() + if err != nil { + t.Fatalf("source history: %v", err) + } + find := func(history []pgstore.Migration, version int64, what string) pgstore.Migration { + t.Helper() + for _, m := range history { + if m.Version == version { + return m + } + } + t.Fatalf("%s: no version %d", what, version) + return pgstore.Migration{} + } + publicationHistory, err := publicationpg.OwnerMigrations() + if err != nil { + t.Fatalf("publication history: %v", err) + } + datasetHistory, err := datasetpg.OwnerMigrations() + if err != nil { + t.Fatalf("dataset history: %v", err) + } + chains, err := recipepg.ChainMigrations() + if err != nil { + t.Fatalf("recipe chains history: %v", err) + } + execRegistries, err := recipeexecpg.RegistryMigrations() + if err != nil { + t.Fatalf("recipe exec history: %v", err) + } + pairs := []struct { + what string + left pgstore.Migration + right pgstore.Migration + }{ + {"publication ledger v1 vs source ledger v1", find(publicationHistory, 1, "publication"), find(source, 1, "source")}, + {"dataset ledger v1 vs source ledger v1", find(datasetHistory, 1, "dataset"), find(source, 1, "source")}, + {"dataset manifest v3 vs source manifest v3", find(datasetHistory, 3, "dataset"), find(source, 3, "source")}, + {"exec ledger v1 vs chains ledger v1", find(execRegistries, 1, "exec"), find(chains, 1, "chains")}, + } + for _, p := range pairs { + if p.left.Name != p.right.Name || p.left.Checksum != p.right.Checksum || p.left.SQL != p.right.SQL { + t.Fatalf("%s: shared bootstrap bytes diverged: %s vs %s", p.what, p.left.Checksum[:12], p.right.Checksum[:12]) + } + } +} + +func ab05ManifestChecksumsNonCircular(t *testing.T) { + t.Helper() + for _, m := range pgstore.Manifest() { + entries, err := ab05OwnerHistories[m.Series]() + if err != nil { + t.Fatalf("series %s: owner history: %v", m.Series, err) + } + if len(entries) != len(m.Entries) { + t.Fatalf("series %s: %d embedded migrations, manifest records %d", m.Series, len(entries), len(m.Entries)) + } + for i := range entries { + sum := sha256.Sum256([]byte(entries[i].SQL)) + recomputed := hex.EncodeToString(sum[:]) + if recomputed != entries[i].Checksum { + t.Fatalf("series %s %d/%s: test-recomputed sha256 %s != loader checksum %s: loader is not hashing the served bytes", + m.Series, entries[i].Version, entries[i].Name, recomputed[:12], entries[i].Checksum[:12]) + } + if recomputed != m.Entries[i].Checksum { + t.Fatalf("series %s %d/%s: test-recomputed sha256 %s != manifest literal %s: the manifest is pinning itself, not these bytes", + m.Series, entries[i].Version, entries[i].Name, recomputed[:12], m.Entries[i].Checksum[:12]) + } + } + if m.Series != pgstore.SeriesSource { + continue + } + for _, e := range m.Entries { + file := fmt.Sprintf("migrations/%06d_%s.sql", e.Version, e.Name) + raw, err := ab05SourceFS.ReadFile(file) + if err != nil { + t.Fatalf("source embed FS read %s: %v", file, err) + } + sum := sha256.Sum256(raw) + if got := hex.EncodeToString(sum[:]); got != e.Checksum { + t.Fatalf("source embed FS %s: sha256 %s != manifest literal %s", file, got[:12], e.Checksum[:12]) + } + } + } +} + +func ab05AssignmentValidation(t *testing.T) { + t.Helper() + type wantConflict struct { + version int64 + a, b pgstore.SeriesID + } + cases := []struct { + name string + assign pgstore.Assignment + errIs error + conflict *wantConflict + }{ + { + name: "all six series on distinct schemas", + assign: pgstore.Assignment{ + pgstore.SeriesSource: "s_source", pgstore.SeriesExplorerControl: "s_explorer", + pgstore.SeriesPublicationControl: "s_pub", pgstore.SeriesDatasetRelease: "s_rel", + pgstore.SeriesRecipeChains: "s_chains", pgstore.SeriesRecipeExecRegistries: "s_exec", + }, + }, + { + name: "empty assignment", + assign: pgstore.Assignment{}, + }, + { + name: "recipe pair may share (identical ledger v1)", + assign: pgstore.Assignment{ + pgstore.SeriesRecipeChains: "s_recipe", pgstore.SeriesRecipeExecRegistries: "s_recipe", + }, + }, + { + name: "publication and dataset release may share (ledger v1 byte-identical)", + assign: pgstore.Assignment{ + pgstore.SeriesPublicationControl: "s_shared", pgstore.SeriesDatasetRelease: "s_shared", + }, + }, + { + name: "identical shared versions on distinct schemas", + assign: pgstore.Assignment{ + pgstore.SeriesRecipeChains: "s_chains_a", pgstore.SeriesRecipeExecRegistries: "s_chains_b", + pgstore.SeriesPublicationControl: "s_pub_a", pgstore.SeriesDatasetRelease: "s_rel_a", + }, + }, + { + name: "source and explorer share: ledger v1 checksums differ", + assign: pgstore.Assignment{ + pgstore.SeriesSource: "x", pgstore.SeriesExplorerControl: "x", + }, + conflict: &wantConflict{1, pgstore.SeriesExplorerControl, pgstore.SeriesSource}, + }, + { + name: "source and dataset share: v1/v3 identical but v8 differs", + assign: pgstore.Assignment{ + pgstore.SeriesSource: "x", pgstore.SeriesDatasetRelease: "x", + }, + conflict: &wantConflict{8, pgstore.SeriesDatasetRelease, pgstore.SeriesSource}, + }, + { + name: "source and publication share: v1 identical but v4 differs", + assign: pgstore.Assignment{ + pgstore.SeriesSource: "x", pgstore.SeriesPublicationControl: "x", + }, + conflict: &wantConflict{4, pgstore.SeriesPublicationControl, pgstore.SeriesSource}, + }, + { + name: "publication and recipe chains share: ledger v1 differs", + assign: pgstore.Assignment{ + pgstore.SeriesPublicationControl: "x", pgstore.SeriesRecipeChains: "x", + }, + conflict: &wantConflict{1, pgstore.SeriesPublicationControl, pgstore.SeriesRecipeChains}, + }, + { + name: "unknown series", + assign: pgstore.Assignment{"warehouse_ledger": "x"}, + errIs: pgstore.ErrInvalidAssignment, + }, + { + name: "uppercase schema", + assign: pgstore.Assignment{pgstore.SeriesSource: "Bad_Name"}, + errIs: pgstore.ErrInvalidAssignment, + }, + { + name: "empty schema", + assign: pgstore.Assignment{pgstore.SeriesSource: ""}, + errIs: pgstore.ErrInvalidAssignment, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := pgstore.ValidateAssignment(tc.assign) + if tc.conflict == nil && tc.errIs == nil { + if err != nil { + t.Fatalf("ValidateAssignment = %v, want nil", err) + } + return + } + if err == nil { + t.Fatalf("ValidateAssignment accepted %v, want rejection", tc.assign) + } + if tc.errIs != nil { + if !errors.Is(err, tc.errIs) { + t.Fatalf("ValidateAssignment error %v does not wrap %v", err, tc.errIs) + } + return + } + var conflict *pgstore.AssignmentConflict + if !errors.As(err, &conflict) { + t.Fatalf("ValidateAssignment error %v is not an *AssignmentConflict", err) + } + if !errors.Is(err, pgstore.ErrInvalidAssignment) { + t.Fatalf("conflict error %v does not wrap ErrInvalidAssignment", err) + } + if conflict.Version != tc.conflict.version || conflict.SeriesA != tc.conflict.a || + conflict.SeriesB != tc.conflict.b { + t.Fatalf("conflict = %s/%s at version %d, want %s/%s at %d", + conflict.SeriesA, conflict.SeriesB, conflict.Version, + tc.conflict.a, tc.conflict.b, tc.conflict.version) + } + if conflict.Schema != tc.assign[tc.conflict.a] { + t.Fatalf("conflict schema = %q, want the shared assignment %q", + conflict.Schema, tc.assign[tc.conflict.a]) + } + if conflict.ChecksumA == conflict.ChecksumB && conflict.NameA == conflict.NameB { + t.Fatalf("conflict names no distinguishing identity: %+v", conflict) + } + }) + } +} + +func ab05PostgresURL(t *testing.T) string { + t.Helper() + raw := os.Getenv("LOOM_TEST_POSTGRES_URL") + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL halves of the ownership contract " + + "(the runner lane asserts this suite runs without skips)") + } + return raw +} + +func ab05Admin(t *testing.T, ctx context.Context) *pgxpool.Pool { + t.Helper() + pool, err := pgxpool.New(ctx, ab05PostgresURL(t)) + if err != nil { + t.Fatalf("admin pool: %v", err) + } + t.Cleanup(pool.Close) + if err := pool.Ping(ctx); err != nil { + t.Fatalf("admin ping: %v", err) + } + return pool +} + +func ab05Expiry(d time.Duration) time.Time { return time.Now().UTC().Add(d) } + +func ab05Schema(t *testing.T, ctx context.Context, admin *pgxpool.Pool, name string) string { + t.Helper() + schema := ab05Prefix + name + if _, err := admin.Exec(ctx, "CREATE SCHEMA "+pgxIdentifier(schema)); err != nil { + t.Fatalf("create schema %s: %v", schema, err) + } + t.Cleanup(func() { + _, _ = admin.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+pgxIdentifier(schema)+" CASCADE") + }) + return schema +} + +func pgxIdentifier(name string) string { + return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` +} + +func ab05ReadLedger(t *testing.T, ctx context.Context, admin *pgxpool.Pool, schema string) []pgstore.LedgerRecord { + t.Helper() + rows, err := admin.Query(ctx, "SELECT version, name, checksum, applied_at FROM "+ + pgxIdentifier(schema)+`.schema_migrations ORDER BY version`) + if err != nil { + t.Fatalf("read ledger of %s: %v", schema, err) + } + defer rows.Close() + var out []pgstore.LedgerRecord + for rows.Next() { + var r pgstore.LedgerRecord + if err := rows.Scan(&r.Version, &r.Name, &r.Checksum, &r.AppliedAt); err != nil { + t.Fatalf("scan ledger row of %s: %v", schema, err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + t.Fatalf("ledger rows of %s: %v", schema, err) + } + return out +} + +func ab05ledgerEqual(t *testing.T, schema string, got []pgstore.LedgerRecord, want []pgstore.HistoryEntry) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("ledger of %s holds %d rows, want %d", schema, len(got), len(want)) + } + for i := range got { + if got[i].Version != want[i].Version || got[i].Name != want[i].Name || got[i].Checksum != want[i].Checksum { + t.Fatalf("ledger row %d of %s = %d/%s/%s, want %d/%s/%s", + i, schema, got[i].Version, got[i].Name, got[i].Checksum[:12], + want[i].Version, want[i].Name, want[i].Checksum[:12]) + } + } +} + +func ab05RegclassExists(t *testing.T, ctx context.Context, admin *pgxpool.Pool, schema, table string) bool { + t.Helper() + var present bool + if err := admin.QueryRow(ctx, "SELECT to_regclass($1) IS NOT NULL", + schema+"."+table).Scan(&present); err != nil { + t.Fatalf("to_regclass %s.%s: %v", schema, table, err) + } + return present +} + +func ab05OwnerMigrate(t *testing.T, ctx context.Context, client *pgstore.Client, admin *pgxpool.Pool, + series pgstore.SeriesID, schema string) (pgstore.MigrateResult, error) { + t.Helper() + switch series { + case pgstore.SeriesSource: + history, err := pgstore.EmbeddedMigrations() + if err != nil { + return pgstore.MigrateResult{}, err + } + runner, err := pgstore.NewMigrationRunner(client, schema, history) + if err != nil { + return pgstore.MigrateResult{}, err + } + return runner.Migrate(ctx) + case pgstore.SeriesExplorerControl: + return explorerpg.MigrateSchema(ctx, client, schema) + case pgstore.SeriesPublicationControl: + return publicationpg.MigrateSchema(ctx, client, schema) + case pgstore.SeriesDatasetRelease: + return datasetpg.MigrateSchema(ctx, client, schema) + case pgstore.SeriesRecipeChains: + return recipepg.MigrateSchema(ctx, client, schema) + case pgstore.SeriesRecipeExecRegistries: + return recipeexecpg.MigrateSchema(ctx, client, schema) + } + t.Fatalf("no owner entry point for series %s", series) + return pgstore.MigrateResult{}, nil +} + +func ab05Client(t *testing.T, ctx context.Context) *pgstore.Client { + t.Helper() + client, err := pgstore.Open(ctx, pgstore.Config{ + DSN: ab05PostgresURL(t), + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + t.Fatalf("open store client: %v", err) + } + t.Cleanup(client.Close) + return client +} + +func ab05FreshBootstrapsConverge(t *testing.T) { + t.Helper() + ctx := context.Background() + admin := ab05Admin(t, ctx) + client := ab05Client(t, ctx) + for _, m := range pgstore.Manifest() { + t.Run(string(m.Series), func(t *testing.T) { + schema := ab05Schema(t, ctx, admin, "boot_"+string(m.Series)) + first, err := ab05OwnerMigrate(t, ctx, client, admin, m.Series, schema) + if err != nil { + t.Fatalf("owner bootstrap of %s: %v", m.Series, err) + } + ab05ledgerEqual(t, schema, first.Applied, m.Entries) + ab05ledgerEqual(t, schema, ab05ReadLedger(t, ctx, admin, schema), m.Entries) + if !ab05RegclassExists(t, ctx, admin, schema, "schema_migrations") || + !ab05RegclassExists(t, ctx, admin, schema, ab05SignatureTable[m.Series]) { + t.Fatalf("series %s: ledger or signature table missing after bootstrap", m.Series) + } + second, err := ab05OwnerMigrate(t, ctx, client, admin, m.Series, schema) + if err != nil { + t.Fatalf("owner rerun of %s: %v", m.Series, err) + } + if !reflect.DeepEqual(first.Applied, second.Applied) { + t.Fatalf("series %s: rerun ledger drifted from first apply", m.Series) + } + }) + } +} + +func ab05RepresentativeDomainOperations(t *testing.T) { + t.Helper() + ctx := context.Background() + admin := ab05Admin(t, ctx) + client := ab05Client(t, ctx) + now := time.Date(2026, 11, 10, 9, 0, 0, 0, time.UTC) + owner := "ab05-owner" + + t.Run("source resource round trip", func(t *testing.T) { + schema := ab05Schema(t, ctx, admin, "op_source") + if _, err := ab05OwnerMigrate(t, ctx, client, admin, pgstore.SeriesSource, schema); err != nil { + t.Fatalf("bootstrap source: %v", err) + } + unrestricted := true + scope := source.ResourceScope{ + Project: "verify-ab05", DatasetGeneration: "gen-1", + AuthResourcePathsUnrestricted: &unrestricted, + } + repo, err := pgstore.NewResourceRepository(client, schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + row := source.Resource{ + Project: scope.Project, DatasetGeneration: scope.DatasetGeneration, + ResourceType: "Patient", LogicalID: "pat-1", AuthResourcePath: "fhir/Patient", + Payload: json.RawMessage(`{"resourceType":"Patient","id":"pat-1"}`), + } + seq, err := repo.InsertResource(ctx, row) + if err != nil { + t.Fatalf("insert resource: %v", err) + } + if seq <= 0 { + t.Fatalf("inserted resource seq = %d, want a server-assigned positive seq", seq) + } + got, err := repo.GetResource(ctx, scope, "Patient", "pat-1") + if err != nil { + t.Fatalf("get resource: %v", err) + } + if got.ResourceSeq != seq || got.LogicalID != "pat-1" || got.AuthResourcePath != "fhir/Patient" { + t.Fatalf("resource round trip = %+v, want seq %d", got, seq) + } + }) + + t.Run("explorer owner create and read", func(t *testing.T) { + schema := ab05Schema(t, ctx, admin, "op_explorer") + if _, err := ab05OwnerMigrate(t, ctx, client, admin, pgstore.SeriesExplorerControl, schema); err != nil { + t.Fatalf("bootstrap explorer: %v", err) + } + store, err := explorerpg.New(client, schema) + if err != nil { + t.Fatalf("explorer store: %v", err) + } + e := explorer.Explorer{ + Project: "verify-ab05", ExplorerID: "exp-1", Title: "Ownership Probe", + ManagementMode: explorer.ManagementRepository, + DraftConfig: json.RawMessage(`{"panels":[]}`), + DraftVersion: 1, + DraftDigest: "sha256:ab05", + DraftSnapshotToken: "snap-1", + DraftSourceGeneration: "gen-1", + DraftAuthorizationScopeDigest: "authz-1", + LastAuthoringCommandID: "cmd-1", + LastAuthoringCommandDigest: "sha256:cmd", + LastAuthoringCommandResults: json.RawMessage(`{"ok":true}`), + UpdatedBy: "ab05@calypr", + UpdatedAt: now, + } + created, err := store.Create(ctx, e) + if err != nil { + t.Fatalf("create explorer: %v", err) + } + if created.Title != e.Title { + t.Fatalf("created explorer title = %q", created.Title) + } + got, err := store.Get(ctx, e.Project, e.ExplorerID) + if err != nil { + t.Fatalf("get explorer: %v", err) + } + if got.DraftDigest != "sha256:ab05" || got.DraftVersion != 1 { + t.Fatalf("explorer round trip = %+v", got) + } + }) + + t.Run("publication control plane registration", func(t *testing.T) { + schema := ab05Schema(t, ctx, admin, "op_pub") + if _, err := ab05OwnerMigrate(t, ctx, client, admin, pgstore.SeriesPublicationControl, schema); err != nil { + t.Fatalf("bootstrap publication: %v", err) + } + registry, err := publicationpg.New(admin, schema) + if err != nil { + t.Fatalf("publication registry: %v", err) + } + key, executionID := "ab05-key", "ab05-exec-1" + held, err := registry.AcquireBundleLease(ctx, key, owner, ab05Expiry(time.Hour)) + if err != nil || !held { + t.Fatalf("acquire bundle lease = (%v, %v), want held", held, err) + } + execution := publication.BundleExecution{ + ID: executionID, Key: key, + BundleIdentity: publication.BundleIdentity{ + Name: "panel", TranslationVersion: "v1", Project: "verify-ab05", + DatasetGeneration: "gen-1", RecipeDigest: "sha256:recipe", SchemaDigest: "sha256:schema", + ScopeDigest: "sha256:scope", EngineVersion: "loom-ab05", AuthScopeMode: "project", + AuthResourcePaths: []string{"fhir/Patient"}, + }, + State: publication.BundleQueued, CreatedAt: now, UpdatedAt: now, ReadyAt: &now, + } + if err := registry.SaveExecution(ctx, execution, owner); err != nil { + t.Fatalf("save execution: %v", err) + } + got, err := registry.GetExecution(ctx, executionID) + if err != nil { + t.Fatalf("get execution: %v", err) + } + if got.State != publication.BundleQueued || got.Project != "verify-ab05" { + t.Fatalf("execution round trip = %+v", got) + } + pins, err := publicationpg.NewReadPins(admin, schema) + if err != nil { + t.Fatalf("read pins: %v", err) + } + pinned, err := pins.AcquireExecutionReadPin(ctx, executionID, owner, ab05Expiry(time.Hour)) + if err != nil || !pinned { + t.Fatalf("acquire execution read pin = (%v, %v), want pinned", pinned, err) + } + catalog, err := chartifactpg.NewCatalog(admin, schema) + if err != nil { + t.Fatalf("artifact catalog: %v", err) + } + artifactID := "ab05-artifact-1" + manifest := chartifact.Manifest{ + ArtifactID: artifactID, + Identity: chartifact.Identity{ + ExecutionID: executionID, OutputID: "output-1", StageID: "stage-1", + Project: "verify-ab05", DatasetGeneration: "gen-1", + RecipeDigest: "recipe-digest", PlanDigest: "plan-digest", + SchemaDigest: "schema-digest", ScopeDigest: "scope-digest", + AuthScopeMode: authscope.ReadScopeRestricted, + AuthResourcePaths: []string{"/Programs/ab05"}, + ScopeMode: chartifact.ScopeModeRows, + }, + Columns: []chartifact.Column{ + {ID: "loom:row_id", Name: "__loom_row_id", SemanticPath: "loom:row_id", + LogicalType: "string", ClickHouseType: "String", Internal: true}, + }, + PhysicalTable: "loom_private_stage_ab05", + State: chartifact.StateCreating, + LeaseOwner: owner, + LeaseUntil: ab05Expiry(time.Hour), + CreatedAt: now, + UpdatedAt: now, + } + if err := catalog.Create(ctx, manifest); err != nil { + t.Fatalf("create private artifact: %v", err) + } + renewed, err := catalog.Renew(ctx, artifactID, owner, ab05Expiry(2*time.Hour)) + if err != nil || !renewed { + t.Fatalf("renew private artifact lease = (%v, %v), want renewed", renewed, err) + } + }) + + t.Run("recipe chain registration", func(t *testing.T) { + schema := ab05Schema(t, ctx, admin, "op_chains") + if _, err := ab05OwnerMigrate(t, ctx, client, admin, pgstore.SeriesRecipeChains, schema); err != nil { + t.Fatalf("bootstrap recipe chains: %v", err) + } + registry, err := recipepg.New(admin, schema) + if err != nil { + t.Fatalf("recipe registry: %v", err) + } + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "ab05-recipe", + TranslationVersion: "v1", + Outputs: []recipe.Output{{ + Name: "DocumentReference", RootResourceType: "DocumentReference", + RowGrain: "document_reference", + }}, + } + if _, err := registry.Register(ctx, "verify-ab05", bundle, ""); err != nil { + t.Fatalf("register revision: %v", err) + } + list, err := registry.List(ctx, "verify-ab05", "ab05-recipe") + if err != nil { + t.Fatalf("list revisions: %v", err) + } + if len(list) != 1 || list[0].Bundle.Name != "ab05-recipe" { + t.Fatalf("revision list = %+v, want the one registered revision", list) + } + }) + + t.Run("recipe exec registry registration", func(t *testing.T) { + schema := ab05Schema(t, ctx, admin, "op_exec") + if _, err := ab05OwnerMigrate(t, ctx, client, admin, pgstore.SeriesRecipeExecRegistries, schema); err != nil { + t.Fatalf("bootstrap recipe exec: %v", err) + } + registry, err := recipeexecpg.New(admin, schema) + if err != nil { + t.Fatalf("exec registry: %v", err) + } + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "ab05-exec-recipe", + TranslationVersion: "v1", + Outputs: []recipe.Output{{ + Name: "DocumentReference", RootResourceType: "DocumentReference", + RowGrain: "document_reference", + }}, + } + canonical, err := bundle.CanonicalBundle() + if err != nil { + t.Fatalf("canonicalize: %v", err) + } + digest, err := canonical.Digest() + if err != nil { + t.Fatalf("digest: %v", err) + } + entry := recipeexec.Entry{Bundle: canonical, Digest: digest} + if err := registry.SaveRecipe(ctx, entry); err != nil { + t.Fatalf("save recipe: %v", err) + } + loaded, err := registry.LoadRecipeVersion(ctx, "ab05-exec-recipe", "v1") + if err != nil { + t.Fatalf("load recipe version: %v", err) + } + if loaded.Digest != digest { + t.Fatalf("loaded digest %s != saved %s", loaded.Digest, digest) + } + }) + + t.Run("dataset release lane typed read", func(t *testing.T) { + schema := ab05Schema(t, ctx, admin, "op_release") + if _, err := ab05OwnerMigrate(t, ctx, client, admin, pgstore.SeriesDatasetRelease, schema); err != nil { + t.Fatalf("bootstrap dataset release: %v", err) + } + releases, err := datasetpg.NewReleaseStore(client, schema) + if err != nil { + t.Fatalf("release store: %v", err) + } + if _, err := releases.ReadActiveRelease(ctx, "verify-ab05"); !errors.Is(err, dataset.ErrNoActiveRelease) { + t.Fatalf("ReadActiveRelease on a fresh schema = %v, want the typed ErrNoActiveRelease "+ + "(a real query against the version-8 tables, not a table-missing failure)", err) + } + }) +} + +func ab05IncompatibleHistoriesRefused(t *testing.T) { + t.Helper() + ctx := context.Background() + admin := ab05Admin(t, ctx) + client := ab05Client(t, ctx) + + t.Run("explorer history onto a source schema", func(t *testing.T) { + schema := ab05Schema(t, ctx, admin, "clash_src_exp") + sourceManifest, _ := pgstore.LookupManifest(pgstore.SeriesSource) + if _, err := ab05OwnerMigrate(t, ctx, client, admin, pgstore.SeriesSource, schema); err != nil { + t.Fatalf("bootstrap source: %v", err) + } + before := ab05ReadLedger(t, ctx, admin, schema) + _, err := ab05OwnerMigrate(t, ctx, client, admin, pgstore.SeriesExplorerControl, schema) + if err == nil { + t.Fatal("explorer history was accepted onto a source-leased schema") + } + if !errors.Is(err, pgstore.ErrMigrationChecksumMismatch) { + t.Fatalf("explorer onto source = %v, want ErrMigrationChecksumMismatch at shared version 1", err) + } + if ab05RegclassExists(t, ctx, admin, schema, explorerpg.OwnerTable) { + t.Fatal("explorer signature table exists after a refused migration: the failure mutated state") + } + ab05ledgerEqual(t, schema, ab05ReadLedger(t, ctx, admin, schema), sourceManifest.Entries) + _ = before + }) + + t.Run("dataset history onto a publication schema", func(t *testing.T) { + schema := ab05Schema(t, ctx, admin, "clash_pub_rel") + pubManifest, _ := pgstore.LookupManifest(pgstore.SeriesPublicationControl) + if _, err := ab05OwnerMigrate(t, ctx, client, admin, pgstore.SeriesPublicationControl, schema); err != nil { + t.Fatalf("bootstrap publication: %v", err) + } + _, err := ab05OwnerMigrate(t, ctx, client, admin, pgstore.SeriesDatasetRelease, schema) + if err == nil { + t.Fatal("dataset release history was accepted onto a publication-control schema") + } + if !errors.Is(err, pgstore.ErrMigrationChecksumMismatch) && + !errors.Is(err, pgstore.ErrFutureSchemaVersion) && + !errors.Is(err, pgstore.ErrMigrationMissing) { + t.Fatalf("dataset onto publication = %v, want a typed ledger-history refusal", err) + } + if ab05RegclassExists(t, ctx, admin, schema, datasetpg.ProjectReleasesTable) { + t.Fatal("project_release exists after a refused migration: the failure mutated state") + } + ab05ledgerEqual(t, schema, ab05ReadLedger(t, ctx, admin, schema), pubManifest.Entries) + }) + + t.Run("source history onto an explorer schema", func(t *testing.T) { + schema := ab05Schema(t, ctx, admin, "clash_exp_src") + expManifest, _ := pgstore.LookupManifest(pgstore.SeriesExplorerControl) + if _, err := ab05OwnerMigrate(t, ctx, client, admin, pgstore.SeriesExplorerControl, schema); err != nil { + t.Fatalf("bootstrap explorer: %v", err) + } + _, err := ab05OwnerMigrate(t, ctx, client, admin, pgstore.SeriesSource, schema) + if err == nil { + t.Fatal("source history was accepted onto an explorer-control schema") + } + if !errors.Is(err, pgstore.ErrMigrationChecksumMismatch) { + t.Fatalf("source onto explorer = %v, want ErrMigrationChecksumMismatch", err) + } + if ab05RegclassExists(t, ctx, admin, schema, "fhir_resource") { + t.Fatal("fhir_resource exists after a refused migration: the failure mutated state") + } + ab05ledgerEqual(t, schema, ab05ReadLedger(t, ctx, admin, schema), expManifest.Entries) + }) +} + +func ab05AssertUpgradeLedger(t *testing.T, schema string, seeded, after []pgstore.LedgerRecord, wantAdded []pgstore.HistoryEntry) { + t.Helper() + seededVersions := make(map[int64]bool, len(seeded)) + for _, r := range seeded { + seededVersions[r.Version] = true + } + shared := make([]pgstore.LedgerRecord, 0, len(seeded)) + added := make([]pgstore.LedgerRecord, 0, len(after)) + for _, r := range after { + if seededVersions[r.Version] { + shared = append(shared, r) + } else { + added = append(added, r) + } + } + if !reflect.DeepEqual(shared, seeded) { + t.Fatalf("ledger of %s: rows present before the upgrade were rewritten or reordered by it (no-rerun rule): got %v, seeded %v", schema, shared, seeded) + } + if len(added) != len(wantAdded) { + t.Fatalf("ledger of %s: upgrade added %d rows, want exactly %d (%v)", schema, len(added), len(wantAdded), wantAdded) + } + for i := range added { + if added[i].Version != wantAdded[i].Version || added[i].Name != wantAdded[i].Name || added[i].Checksum != wantAdded[i].Checksum { + t.Fatalf("ledger of %s: added row %d = %d/%s/%s, want %d/%s/%s", + schema, i, added[i].Version, added[i].Name, added[i].Checksum[:12], + wantAdded[i].Version, wantAdded[i].Name, wantAdded[i].Checksum[:12]) + } + } +} + +func ab05PreRefactorUpgradeConverges(t *testing.T) { + t.Helper() + ctx := context.Background() + admin := ab05Admin(t, ctx) + client := ab05Client(t, ctx) + + loadPreRefactor := func(t *testing.T, dir string) []pgstore.Migration { + t.Helper() + migrations, err := pgstore.LoadMigrations(os.DirFS(preRefactorRoot), dir) + if err != nil { + t.Fatalf("load pre-refactor %s: %v", dir, err) + } + return migrations + } + seed := func(t *testing.T, name string, history []pgstore.Migration) (string, []pgstore.LedgerRecord) { + t.Helper() + sort.Slice(history, func(i, j int) bool { return history[i].Version < history[j].Version }) + schema := ab05Schema(t, ctx, admin, name) + runner, err := pgstore.NewMigrationRunner(client, schema, history) + if err != nil { + t.Fatalf("seed runner %s: %v", name, err) + } + result, err := runner.Migrate(ctx) + if err != nil { + t.Fatalf("seed migrate %s: %v", name, err) + } + return schema, result.Applied + } + now := time.Date(2026, 11, 10, 10, 0, 0, 0, time.UTC) + + t.Run("ExplorerDeployedSchemaRefusesRetroactive478", func(t *testing.T) { + owner, err := explorerpg.OwnerMigrations() + if err != nil { + t.Fatalf("owner history: %v", err) + } + legacy := make([]pgstore.Migration, 0, len(owner)) + moved := map[int64]bool{4: true, 7: true, 8: true} + for _, m := range owner { + if !moved[m.Version] { + legacy = append(legacy, m) + } + } + schema, seeded := seed(t, "up_explorer_refuse", legacy) + if len(seeded) != len(owner)-3 { + t.Fatalf("seeded explorer ledger holds %d rows, want %d (pre-refactor deployed composition)", len(seeded), len(owner)-3) + } + store, err := explorerpg.New(client, schema) + if err != nil { + t.Fatalf("explorer store: %v", err) + } + e := explorer.Explorer{ + Project: "verify-ab05", ExplorerID: "ab05-refuse", Title: "Refusal Survivor", + ManagementMode: explorer.ManagementRepository, + DraftConfig: json.RawMessage(`{"panels":[]}`), + DraftVersion: 1, + DraftDigest: "sha256:ab05-refuse", + DraftSnapshotToken: "snap-r", + DraftSourceGeneration: "gen-1", + DraftAuthorizationScopeDigest: "authz-r", + LastAuthoringCommandID: "cmd-r", + LastAuthoringCommandDigest: "sha256:cmd-r", + LastAuthoringCommandResults: json.RawMessage(`{"ok":true}`), + UpdatedBy: "ab05@calypr", + UpdatedAt: now, + } + if _, err := store.Create(ctx, e); err != nil { + t.Fatalf("create explorer before refusal attempt: %v", err) + } + _, err = explorerpg.MigrateSchema(ctx, client, schema) + if !errors.Is(err, pgstore.ErrMigrationOutOfOrder) { + t.Fatalf("retroactive gap-fill was not refused with ErrMigrationOutOfOrder: %v", err) + } + var outOfOrder *pgstore.OutOfOrderMigrationError + var wantRecord int64 + for _, m := range owner { + if !moved[m.Version] && m.Version > wantRecord { + wantRecord = m.Version + } + } + if !errors.As(err, &outOfOrder) || outOfOrder.Version != 4 || outOfOrder.LatestRecord != wantRecord { + t.Fatalf("refusal = %+v, want pending version 4 against recorded max %d", outOfOrder, wantRecord) + } + if after := ab05ReadLedger(t, ctx, admin, schema); !reflect.DeepEqual(after, seeded) { + t.Fatalf("the refused retroactive migration mutated the deployed ledger: seeded %v, after %v", seeded, after) + } + if got, err := store.Get(ctx, e.Project, e.ExplorerID); err != nil || got.Title != "Refusal Survivor" { + t.Fatalf("explorer row did not survive the refusal: %+v, %v", got, err) + } + }) + + t.Run("ExplorerControlPlanePreRefactorFixtureCompositionUpgrade", func(t *testing.T) { + owner, err := explorerpg.OwnerMigrations() + if err != nil { + t.Fatalf("owner history: %v", err) + } + legacy := make([]pgstore.Migration, 0, len(owner)) + moved := map[int64]bool{4: true, 7: true, 8: true} + for _, m := range owner { + if !moved[m.Version] { + legacy = append(legacy, m) + } + } + legacy = append(legacy, loadPreRefactor(t, "tests/migrations-capabilities")...) + legacy = append(legacy, loadPreRefactor(t, "tests/migrations-interpretations")...) + legacy = append(legacy, loadPreRefactor(t, "tests/migrations-selections")...) + schema, seeded := seed(t, "up_explorer", legacy) + expManifest, _ := pgstore.LookupManifest(pgstore.SeriesExplorerControl) + ab05ledgerEqual(t, schema, seeded, expManifest.Entries) + for _, movedFile := range ab05MovedFiles { + if movedFile.series != pgstore.SeriesExplorerControl { + continue + } + disk, err := os.ReadFile(filepath.Join(preRefactorRoot, filepath.FromSlash(movedFile.preRefactor))) + if err != nil { + t.Fatalf("pre-refactor snapshot %s: %v", movedFile.preRefactor, err) + } + sum := sha256.Sum256(disk) + var row *pgstore.LedgerRecord + for i := range seeded { + if seeded[i].Version == movedFile.version { + row = &seeded[i] + } + } + if row == nil || row.Name != movedFile.name || row.Checksum != hex.EncodeToString(sum[:]) { + t.Fatalf("seeded ledger row for %s does not equal the frozen snapshot identity", movedFile.preRefactor) + } + } + store, err := explorerpg.New(client, schema) + if err != nil { + t.Fatalf("explorer store: %v", err) + } + e := explorer.Explorer{ + Project: "verify-ab05", ExplorerID: "ab05-upgrade", Title: "Upgrade Survivor", + ManagementMode: explorer.ManagementRepository, + DraftConfig: json.RawMessage(`{"panels":[]}`), + DraftVersion: 1, + DraftDigest: "sha256:ab05-up", + DraftSnapshotToken: "snap-up", + DraftSourceGeneration: "gen-1", + DraftAuthorizationScopeDigest: "authz-up", + LastAuthoringCommandID: "cmd-up", + LastAuthoringCommandDigest: "sha256:cmd-up", + LastAuthoringCommandResults: json.RawMessage(`{"ok":true}`), + UpdatedBy: "ab05@calypr", + UpdatedAt: now, + } + if _, err := store.Create(ctx, e); err != nil { + t.Fatalf("create explorer before upgrade: %v", err) + } + after, err := explorerpg.MigrateSchema(ctx, client, schema) + if err != nil { + t.Fatalf("owner upgrade of a fixture-seeded explorer schema: %v", err) + } + ab05ledgerEqual(t, schema, after.Applied, expManifest.Entries) + ab05AssertUpgradeLedger(t, schema, seeded, after.Applied, nil) + if got, err := store.Get(ctx, e.Project, e.ExplorerID); err != nil || got.Title != "Upgrade Survivor" { + t.Fatalf("explorer row did not survive the upgrade: %+v, %v", got, err) + } + twice, err := explorerpg.MigrateSchema(ctx, client, schema) + if err != nil { + t.Fatalf("owner double-apply of explorer schema: %v", err) + } + if !reflect.DeepEqual(twice.Applied, after.Applied) { + t.Fatal("explorer double-apply through the owner drifted the ledger") + } + }) + + t.Run("PublicationControlPlaneUpgrade", func(t *testing.T) { + source, err := pgstore.EmbeddedMigrations() + if err != nil { + t.Fatalf("source history: %v", err) + } + legacy := []pgstore.Migration{source[0]} + for _, dir := range []string{ + "tests/migrations", "tests/migrations-leases", + "tests/migrations-chartifact", "tests/migrations-readpins", + } { + legacy = append(legacy, loadPreRefactor(t, dir)...) + } + schema, seeded := seed(t, "up_pub", legacy) + pubManifest, _ := pgstore.LookupManifest(pgstore.SeriesPublicationControl) + ab05ledgerEqual(t, schema, seeded, pubManifest.Entries) + registry, err := publicationpg.New(admin, schema) + if err != nil { + t.Fatalf("publication registry: %v", err) + } + key := "ab05-up-key" + held, err := registry.AcquireBundleLease(ctx, key, "ab05-owner", ab05Expiry(time.Hour)) + if err != nil || !held { + t.Fatalf("pre-upgrade lease = (%v, %v), want held", held, err) + } + catalog, err := chartifactpg.NewCatalog(admin, schema) + if err != nil { + t.Fatalf("artifact catalog: %v", err) + } + artifactID := "ab05-up-artifact" + manifest := chartifact.Manifest{ + ArtifactID: artifactID, + Identity: chartifact.Identity{ + ExecutionID: "ab05-up-exec", OutputID: "output-1", StageID: "stage-1", + Project: "verify-ab05", DatasetGeneration: "gen-1", + RecipeDigest: "recipe-digest", PlanDigest: "plan-digest", + SchemaDigest: "schema-digest", ScopeDigest: "scope-digest", + AuthScopeMode: authscope.ReadScopeRestricted, + AuthResourcePaths: []string{"/Programs/ab05"}, + ScopeMode: chartifact.ScopeModeRows, + }, + Columns: []chartifact.Column{ + {ID: "loom:row_id", Name: "__loom_row_id", SemanticPath: "loom:row_id", + LogicalType: "string", ClickHouseType: "String", Internal: true}, + }, + PhysicalTable: "loom_private_stage_ab05_up", + State: chartifact.StateCreating, + LeaseOwner: "ab05-owner", + LeaseUntil: ab05Expiry(time.Hour), + CreatedAt: now, + UpdatedAt: now, + } + if err := catalog.Create(ctx, manifest); err != nil { + t.Fatalf("pre-upgrade artifact create: %v", err) + } + after, err := publicationpg.MigrateSchema(ctx, client, schema) + if err != nil { + t.Fatalf("owner upgrade of a fixture-seeded publication schema: %v", err) + } + ab05ledgerEqual(t, schema, after.Applied, pubManifest.Entries) + ab05AssertUpgradeLedger(t, schema, seeded, after.Applied, nil) + renewed, err := catalog.Renew(ctx, artifactID, "ab05-owner", ab05Expiry(2*time.Hour)) + if err != nil || !renewed { + t.Fatalf("private artifact did not survive the upgrade: renew = (%v, %v)", renewed, err) + } + if held, err := registry.AcquireBundleLease(ctx, key, "ab05-owner", ab05Expiry(time.Hour)); err != nil || !held { + t.Fatalf("lease did not survive the upgrade: reacquire = (%v, %v)", held, err) + } + twice, err := publicationpg.MigrateSchema(ctx, client, schema) + if err != nil { + t.Fatalf("owner double-apply of publication schema: %v", err) + } + if !reflect.DeepEqual(twice.Applied, after.Applied) { + t.Fatal("publication double-apply through the owner drifted the ledger") + } + }) + + t.Run("DatasetReleaseLaneUpgrade", func(t *testing.T) { + source, err := pgstore.EmbeddedMigrations() + if err != nil { + t.Fatalf("source history: %v", err) + } + legacy := []pgstore.Migration{source[0], source[2]} + legacy = append(legacy, loadPreRefactor(t, + "internal/dataset/postgres/testdata/migrations-releases")...) + schema, seeded := seed(t, "up_rel", legacy) + relManifest, _ := pgstore.LookupManifest(pgstore.SeriesDatasetRelease) + ab05ledgerEqual(t, schema, seeded, relManifest.Entries) + gen, err := datasetpg.New(client, schema) + if err != nil { + t.Fatalf("dataset store: %v", err) + } + ref, err := dataset.NewRef("verify-ab05", "gen-ab05-up") + if err != nil { + t.Fatalf("NewRef: %v", err) + } + snapshot, err := dataset.NewSchemaSnapshot("urn:ab05-gen-ab05-up", "R5", strings.Repeat("0123456789abcdef", 4), []string{"Observation", "Patient"}) + if err != nil { + t.Fatalf("NewSchemaSnapshot: %v", err) + } + loading, err := dataset.NewManifest(ref, snapshot) + if err != nil { + t.Fatalf("NewManifest: %v", err) + } + if _, err := gen.CreateManifest(ctx, loading); err != nil { + t.Fatalf("CreateManifest before upgrade: %v", err) + } + if _, err := gen.TransitionManifest(ctx, loading, dataset.StateStaged); err != nil { + t.Fatalf("TransitionManifest before upgrade: %v", err) + } + after, err := datasetpg.MigrateSchema(ctx, client, schema) + if err != nil { + t.Fatalf("owner upgrade of a fixture-seeded release schema: %v", err) + } + ab05ledgerEqual(t, schema, after.Applied, relManifest.Entries) + ab05AssertUpgradeLedger(t, schema, seeded, after.Applied, nil) + staged, err := gen.ReadManifest(ctx, ref) + if err != nil || staged.State != dataset.StateStaged { + t.Fatalf("staged generation manifest did not survive the upgrade: %+v, %v", staged, err) + } + twice, err := datasetpg.MigrateSchema(ctx, client, schema) + if err != nil { + t.Fatalf("owner double-apply of release schema: %v", err) + } + if !reflect.DeepEqual(twice.Applied, after.Applied) { + t.Fatal("release double-apply through the owner drifted the ledger") + } + }) +} + +func ab05ZeroSurvivorSweep(t *testing.T) { + t.Helper() + if os.Getenv("LOOM_TEST_POSTGRES_URL") == "" { + t.Skip("no PostgreSQL in this environment; nothing was created to sweep") + } + ctx := context.Background() + admin := ab05Admin(t, ctx) + rows, err := admin.Query(ctx, + "SELECT nspname FROM pg_namespace WHERE nspname LIKE 'verify_ab05\\_%'") + if err != nil { + t.Fatalf("sweep scan: %v", err) + } + var leftovers []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("sweep scan row: %v", err) + } + leftovers = append(leftovers, name) + } + rows.Close() + for _, name := range leftovers { + _, _ = admin.Exec(ctx, "DROP SCHEMA IF EXISTS "+pgxIdentifier(name)+" CASCADE") + } + if len(leftovers) > 0 { + t.Fatalf("zero-survivor sweep found leaked schemas: %v", leftovers) + } +} + +func TestPostgresMigrationOwnershipContract(t *testing.T) { + t.Run("ManifestMatchesEmbeddedOwners", ab05ManifestMatchesOwners) + t.Run("ManifestChecksumsNonCircular", ab05ManifestChecksumsNonCircular) + t.Run("ManifestRecordInvariants", ab05ManifestRecordInvariants) + t.Run("PreRefactorBytesFrozenInOwners", ab05PreRefactorBytesFrozen) + t.Run("SharedBootstrapBytesIdentical", ab05SharedBootstrapBytesIdentical) + t.Run("ValidateAssignmentMatrix", ab05AssignmentValidation) + + t.Run("FreshBootstrapsConverge", ab05FreshBootstrapsConverge) + t.Run("RepresentativeDomainOperations", ab05RepresentativeDomainOperations) + t.Run("IncompatibleHistoriesRefused", ab05IncompatibleHistoriesRefused) + t.Run("PreRefactorUpgradeConverges", ab05PreRefactorUpgradeConverges) + t.Run("TestZZZeroSurvivorSweep", ab05ZeroSurvivorSweep) +} + +func repoRootForOwnerStat(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed; cannot locate the repository root") + } + dir := filepath.Dir(file) + for { + if fi, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil && !fi.IsDir() { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("no go.mod found above the store ownership contract test") + } + dir = parent + } +} diff --git a/internal/backends/postgres/internal/store/resource.go b/internal/backends/postgres/internal/store/resource.go new file mode 100644 index 000000000..04a510fc6 --- /dev/null +++ b/internal/backends/postgres/internal/store/resource.go @@ -0,0 +1,296 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + resourceTableName = "fhir_resource" + MaxResourceKeysetLimit = 1000 +) + +const resourceColumns = "resource_seq, project, dataset_generation, resource_type, logical_id, auth_resource_path, payload" + +type ResourceRow struct { + source.Resource + ResourceSeq int64 +} + +var ( + ErrResourceConflict = errors.New("postgres: resource conflict") + ErrResourceNotFound = errors.New("postgres: resource not found") + ErrInvalidResource = errors.New("postgres: invalid resource") + ErrInvalidResourceScope = errors.New("postgres: invalid resource scope") +) + +type ResourceConflictError struct { + Project string + DatasetGeneration string + ResourceType string + LogicalID string + AuthResourcePath string +} + +func (e *ResourceConflictError) Error() string { + return fmt.Sprintf("postgres: resource %s/%s/%s/%s (auth path %q) already exists; "+ + "insert never replaces the stored payload", + e.Project, e.DatasetGeneration, e.ResourceType, e.LogicalID, e.AuthResourcePath) +} + +func (e *ResourceConflictError) Unwrap() error { return ErrResourceConflict } + +type ResourceNotFoundError struct { + Project string + DatasetGeneration string + ResourceType string + LogicalID string + Restricted bool +} + +func (e *ResourceNotFoundError) Error() string { + return fmt.Sprintf("postgres: no resource %s/%s/%s/%s in scope (restricted=%t)", + e.Project, e.DatasetGeneration, e.ResourceType, e.LogicalID, e.Restricted) +} + +func (e *ResourceNotFoundError) Unwrap() error { return ErrResourceNotFound } + +type ResourceRepository struct { + src ConnSource + tableFQN string +} + +func NewResourceRepository(src ConnSource, schema string) (*ResourceRepository, error) { + if src == nil { + return nil, errors.New("postgres: nil resource connection source") + } + if !ValidSchemaName(schema) { + return nil, fmt.Errorf("%w: %q must match [a-z_][a-z0-9_]{0,62}", ErrInvalidSchemaName, schema) + } + return &ResourceRepository{ + src: src, + tableFQN: pgx.Identifier{schema, resourceTableName}.Sanitize(), + }, nil +} + +func (r *ResourceRepository) InsertResource(ctx context.Context, rec source.Resource) (int64, error) { + if err := validateResourceForInsert(ResourceRow{Resource: rec}); err != nil { + return 0, err + } + var payloadValue any + if rec.Payload != nil { + payloadValue = []byte(rec.Payload) + } + conn, err := r.acquire(ctx) + if err != nil { + return 0, err + } + defer conn.Release() + + sql := "INSERT INTO " + r.tableFQN + " (project, dataset_generation, resource_type, logical_id, auth_resource_path, payload) " + + "VALUES ($1, $2, $3, $4, $5, $6::jsonb) " + + "ON CONFLICT (project, dataset_generation, resource_type, logical_id) DO NOTHING " + + "RETURNING resource_seq" + var seq int64 + err = conn.QueryRow(ctx, sql, + rec.Project, rec.DatasetGeneration, rec.ResourceType, rec.LogicalID, rec.AuthResourcePath, + payloadValue).Scan(&seq) + if errors.Is(err, pgx.ErrNoRows) { + return 0, &ResourceConflictError{ + Project: rec.Project, + DatasetGeneration: rec.DatasetGeneration, + ResourceType: rec.ResourceType, + LogicalID: rec.LogicalID, + AuthResourcePath: rec.AuthResourcePath, + } + } + if err != nil { + return 0, fmt.Errorf("postgres: insert resource %s/%s/%s/%s: %w", + rec.Project, rec.DatasetGeneration, rec.ResourceType, rec.LogicalID, err) + } + return seq, nil +} + +func (r *ResourceRepository) GetResource(ctx context.Context, scope source.ResourceScope, resourceType, logicalID string) (ResourceRow, error) { + binding, err := normalizeResourceScope(scope) + if err != nil { + return ResourceRow{}, err + } + if strings.TrimSpace(resourceType) == "" { + return ResourceRow{}, fmt.Errorf("%w: resource type is blank", ErrInvalidResource) + } + if strings.TrimSpace(logicalID) == "" { + return ResourceRow{}, fmt.Errorf("%w: logical id is blank", ErrInvalidResource) + } + notFound := &ResourceNotFoundError{ + Project: scope.Project, + DatasetGeneration: scope.DatasetGeneration, + ResourceType: resourceType, + LogicalID: logicalID, + Restricted: binding.restricted, + } + if binding.restricted && len(binding.paths) == 0 { + return ResourceRow{}, notFound + } + + sql := "SELECT " + resourceColumns + " FROM " + r.tableFQN + + " WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3 AND logical_id = $4" + args := []any{scope.Project, scope.DatasetGeneration, resourceType, logicalID} + if binding.restricted { + sql += " AND auth_resource_path = ANY($5::text[])" + args = append(args, binding.paths) + } + + conn, err := r.acquire(ctx) + if err != nil { + return ResourceRow{}, err + } + defer conn.Release() + + rec, err := scanResourceRow(conn.QueryRow(ctx, sql, args...)) + if errors.Is(err, pgx.ErrNoRows) { + return ResourceRow{}, notFound + } + if err != nil { + return ResourceRow{}, fmt.Errorf("postgres: read resource %s/%s/%s/%s: %w", + scope.Project, scope.DatasetGeneration, resourceType, logicalID, err) + } + return rec, nil +} + +func (r *ResourceRepository) ReadResources(ctx context.Context, scope source.ResourceScope, afterSeq int64, limit int) ([]ResourceRow, bool, error) { + binding, err := normalizeResourceScope(scope) + if err != nil { + return nil, false, err + } + if afterSeq < 0 { + return nil, false, fmt.Errorf("%w: keyset cursor is %d, want >= 0", ErrInvalidResourceScope, afterSeq) + } + if limit < 1 || limit > MaxResourceKeysetLimit { + return nil, false, fmt.Errorf("%w: keyset limit is %d, want 1..%d", + ErrInvalidResourceScope, limit, MaxResourceKeysetLimit) + } + if binding.restricted && len(binding.paths) == 0 { + return nil, false, nil + } + + sql := "SELECT " + resourceColumns + " FROM " + r.tableFQN + + " WHERE project = $1 AND dataset_generation = $2 AND resource_seq > $3" + args := []any{scope.Project, scope.DatasetGeneration, afterSeq} + if binding.restricted { + sql += " AND auth_resource_path = ANY($4::text[])" + args = append(args, binding.paths) + } + sql += fmt.Sprintf(" ORDER BY resource_seq LIMIT $%d", len(args)+1) + args = append(args, limit+1) + + conn, err := r.acquire(ctx) + if err != nil { + return nil, false, err + } + defer conn.Release() + + rows, err := conn.Query(ctx, sql, args...) + if err != nil { + return nil, false, fmt.Errorf("postgres: keyset read resources %s/%s: %w", + scope.Project, scope.DatasetGeneration, err) + } + defer rows.Close() + + var out []ResourceRow + for rows.Next() { + rec, err := scanResourceRow(rows) + if err != nil { + return nil, false, fmt.Errorf("postgres: keyset read resources %s/%s: %w", + scope.Project, scope.DatasetGeneration, err) + } + out = append(out, rec) + } + if err := rows.Err(); err != nil { + return nil, false, fmt.Errorf("postgres: keyset read resources %s/%s: %w", + scope.Project, scope.DatasetGeneration, err) + } + hasMore := len(out) > limit + if hasMore { + out = out[:limit] + } + return out, hasMore, nil +} + +func (r *ResourceRepository) acquire(ctx context.Context) (*pgxpool.Conn, error) { + conn, err := r.src.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres: acquire resource connection: %w", err) + } + return conn, nil +} + +func scanResourceRow(row pgx.Row) (ResourceRow, error) { + var rec ResourceRow + var payload []byte + if err := row.Scan( + &rec.ResourceSeq, &rec.Project, &rec.DatasetGeneration, + &rec.ResourceType, &rec.LogicalID, &rec.AuthResourcePath, &payload, + ); err != nil { + return ResourceRow{}, err + } + if payload != nil { + rec.Payload = json.RawMessage(payload) + } + return rec, nil +} + +func validateResourceForInsert(rec ResourceRow) error { + if rec.ResourceSeq != 0 { + return fmt.Errorf("%w: ResourceSeq is %d; the storage identity is server-assigned, insert requires 0", + ErrInvalidResource, rec.ResourceSeq) + } + for _, f := range []struct { + name string + value string + }{ + {"project", rec.Project}, + {"resource type", rec.ResourceType}, + {"logical id", rec.LogicalID}, + } { + if strings.TrimSpace(f.value) == "" { + return fmt.Errorf("%w: %s is blank", ErrInvalidResource, f.name) + } + } + if rec.Payload != nil && !json.Valid(rec.Payload) { + return fmt.Errorf("%w: payload is not valid JSON", ErrInvalidResource) + } + return nil +} + +type resourceScopeBinding struct { + restricted bool + paths []string +} + +func normalizeResourceScope(scope source.ResourceScope) (resourceScopeBinding, error) { + if strings.TrimSpace(scope.Project) == "" { + return resourceScopeBinding{}, fmt.Errorf("%w: project is blank", ErrInvalidResourceScope) + } + if scope.AuthResourcePathsUnrestricted == nil { + return resourceScopeBinding{}, fmt.Errorf( + "%w: AuthResourcePathsUnrestricted must be set explicitly; there is no implicit unrestricted mode", + ErrInvalidResourceScope) + } + if *scope.AuthResourcePathsUnrestricted { + if len(scope.AuthResourcePaths) > 0 { + return resourceScopeBinding{}, fmt.Errorf( + "%w: unrestricted mode must not carry auth resource paths (%d given)", + ErrInvalidResourceScope, len(scope.AuthResourcePaths)) + } + return resourceScopeBinding{restricted: false}, nil + } + return resourceScopeBinding{restricted: true, paths: scope.AuthResourcePaths}, nil +} diff --git a/internal/backends/postgres/internal/store/resource_test.go b/internal/backends/postgres/internal/store/resource_test.go new file mode 100644 index 000000000..01392555a --- /dev/null +++ b/internal/backends/postgres/internal/store/resource_test.go @@ -0,0 +1,1023 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "reflect" + "regexp" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +var pgstore04ResourceNamePattern = regexp.MustCompile(`^pgstore04_it_[0-9]+_[0-9a-f]+$`) + +type resourceService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func newResourceService(t *testing.T) *resourceService { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("pgstore04_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstore04ResourceNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("pgstore04_it_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !pgstore04ResourceNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + return &resourceService{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} +} + +func (s *resourceService) openClient(t *testing.T) *Client { + t.Helper() + client, err := Open(s.ctx, Config{ + DSN: s.dsn, + MaxConns: 4, + MinConns: 0, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *resourceService) newRepository(t *testing.T) *ResourceRepository { + t.Helper() + repo, err := NewResourceRepository(s.openClient(t), s.schema) + if err != nil { + s.fail("new resource repository", err) + } + return repo +} + +func (s *resourceService) mustMigrateEmbedded(t *testing.T) MigrateResult { + t.Helper() + result, err := MigrateSchema(s.ctx, s.openClient(t), s.schema) + if err != nil { + s.fail("MigrateSchema", err) + } + return result +} + +func (s *resourceService) readLedger(t *testing.T) []LedgerRecord { + t.Helper() + rows, err := s.iso.Query(s.ctx, + `SELECT version, name, checksum, applied_at FROM "`+s.schema+`".`+LedgerTable+` ORDER BY version`) + if err != nil { + s.fail("witness ledger", err) + } + defer rows.Close() + var out []LedgerRecord + for rows.Next() { + var rec LedgerRecord + if err := rows.Scan(&rec.Version, &rec.Name, &rec.Checksum, &rec.AppliedAt); err != nil { + s.fail("witness ledger scan", err) + } + out = append(out, rec) + } + if err := rows.Err(); err != nil { + s.fail("witness ledger rows", err) + } + return out +} + +func (s *resourceService) mustInsert(t *testing.T, repo *ResourceRepository, rec source.Resource) int64 { + t.Helper() + seq, err := repo.InsertResource(s.ctx, rec) + if err != nil { + s.fail(fmt.Sprintf("insert resource %s/%s/%s/%s", + rec.Project, rec.DatasetGeneration, rec.ResourceType, rec.LogicalID), err) + } + if seq < 1 { + t.Fatalf("insert returned storage identity %d, want >= 1", seq) + } + return seq +} + +func (s *resourceService) mustGet(t *testing.T, repo *ResourceRepository, + scope source.ResourceScope, resourceType, logicalID string) ResourceRow { + t.Helper() + rec, err := repo.GetResource(s.ctx, scope, resourceType, logicalID) + if err != nil { + s.fail(fmt.Sprintf("get resource %s/%s/%s/%s", + scope.Project, scope.DatasetGeneration, resourceType, logicalID), err) + } + return rec +} + +func (s *resourceService) mustRead(t *testing.T, repo *ResourceRepository, + scope source.ResourceScope, afterSeq int64, limit int) ([]ResourceRow, bool) { + t.Helper() + rows, hasMore, err := repo.ReadResources(s.ctx, scope, afterSeq, limit) + if err != nil { + s.fail(fmt.Sprintf("keyset read resources %s/%s after %d", + scope.Project, scope.DatasetGeneration, afterSeq), err) + } + return rows, hasMore +} + +func TestPhysicalResourceRowRefusesOfferedStorageIdentity(t *testing.T) { + base := source.Resource{ + Project: "proj", DatasetGeneration: "gen", ResourceType: "Patient", + LogicalID: "p-1", AuthResourcePath: "practitioner/pr-1", + Payload: json.RawMessage(`{"id":"p-1"}`), + } + if err := validateResourceForInsert(ResourceRow{Resource: base}); err != nil { + t.Errorf("validateResourceForInsert with zero storage identity = %v, want nil", err) + } + err := validateResourceForInsert(ResourceRow{Resource: base, ResourceSeq: 7}) + if !errors.Is(err, ErrInvalidResource) { + t.Fatalf("validateResourceForInsert with offered storage identity = %v, want ErrInvalidResource", err) + } + if !strings.Contains(err.Error(), "server-assigned") { + t.Errorf("refusal %v does not name the server-assigned contract", err) + } +} + +func restrictedScope(project, generation string, paths ...string) source.ResourceScope { + restricted := false + return source.ResourceScope{ + Project: project, + DatasetGeneration: generation, + AuthResourcePathsUnrestricted: &restricted, + AuthResourcePaths: paths, + } +} + +func unrestrictedScope(project, generation string) source.ResourceScope { + unrestricted := true + return source.ResourceScope{ + Project: project, + DatasetGeneration: generation, + AuthResourcePathsUnrestricted: &unrestricted, + } +} + +func payloadObject(t *testing.T, payload json.RawMessage) map[string]json.RawMessage { + t.Helper() + if payload == nil { + t.Fatal("payload is nil (SQL NULL), want a JSON object") + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(payload, &obj); err != nil { + t.Fatalf("payload %s is not a JSON object: %v", payload, err) + } + return obj +} + +func assertPayloadEqual(t *testing.T, got json.RawMessage, want string) { + t.Helper() + var g, w any + if err := json.Unmarshal(got, &g); err != nil { + t.Fatalf("stored payload %s does not parse: %v", got, err) + } + if err := json.Unmarshal([]byte(want), &w); err != nil { + t.Fatalf("want payload %s does not parse: %v", want, err) + } + if !reflect.DeepEqual(g, w) { + t.Fatalf("stored payload %s, want %s", got, want) + } +} + +func assertRowScope(t *testing.T, got ResourceRow, want source.Resource) { + t.Helper() + if got.Project != want.Project || got.DatasetGeneration != want.DatasetGeneration || + got.ResourceType != want.ResourceType || got.LogicalID != want.LogicalID || + got.AuthResourcePath != want.AuthResourcePath { + t.Fatalf("row scope = %+v, want scope %+v", got, want) + } +} + +func TestResourceValidationRejectsInvalidInserts(t *testing.T) { + repo, err := NewResourceRepository(stubConnSource{}, "pgstore04_stub") + if err != nil { + t.Fatalf("NewResourceRepository: %v", err) + } + base := source.Resource{ + Project: "proj", DatasetGeneration: "gen", ResourceType: "Patient", + LogicalID: "p-1", AuthResourcePath: "practitioner/pr-1", + Payload: json.RawMessage(`{"id":"p-1"}`), + } + for _, tc := range []struct { + name string + mutate func(*source.Resource) + }{ + {"blank project", func(r *source.Resource) { r.Project = " " }}, + {"blank resource type", func(r *source.Resource) { r.ResourceType = "" }}, + {"blank logical id", func(r *source.Resource) { r.LogicalID = "\t" }}, + {"malformed json payload", func(r *source.Resource) { r.Payload = json.RawMessage(`{"id":`) }}, + {"whitespace payload", func(r *source.Resource) { r.Payload = json.RawMessage(` `) }}, + } { + rec := base + tc.mutate(&rec) + _, err := repo.InsertResource(context.Background(), rec) + if !errors.Is(err, ErrInvalidResource) { + t.Errorf("%s: InsertResource = %v, want ErrInvalidResource", tc.name, err) + } + } +} + +func TestResourceScopeAndQueryValidation(t *testing.T) { + repo, err := NewResourceRepository(stubConnSource{}, "pgstore04_stub") + if err != nil { + t.Fatalf("NewResourceRepository: %v", err) + } + ctx := context.Background() + + if _, err := NewResourceRepository(nil, "pgstore04_stub"); err == nil { + t.Error("NewResourceRepository with nil source succeeded, want error") + } + if _, err := NewResourceRepository(stubConnSource{}, "Bad-Schema"); !errors.Is(err, ErrInvalidSchemaName) { + t.Errorf("NewResourceRepository with invalid schema = %v, want ErrInvalidSchemaName", err) + } + + scope := source.ResourceScope{Project: "proj", AuthResourcePaths: []string{"a"}} + if _, err := repo.GetResource(ctx, scope, "Patient", "p-1"); !errors.Is(err, ErrInvalidResourceScope) { + t.Errorf("GetResource with nil unrestricted flag = %v, want ErrInvalidResourceScope", err) + } + if _, _, err := repo.ReadResources(ctx, scope, 0, 10); !errors.Is(err, ErrInvalidResourceScope) { + t.Errorf("ReadResources with nil unrestricted flag = %v, want ErrInvalidResourceScope", err) + } + + unrestrictedWithPaths := unrestrictedScope("proj", "gen") + unrestrictedWithPaths.AuthResourcePaths = []string{"practitioner/pr-1"} + if _, err := repo.GetResource(ctx, unrestrictedWithPaths, "Patient", "p-1"); !errors.Is(err, ErrInvalidResourceScope) { + t.Errorf("unrestricted scope carrying paths = %v, want ErrInvalidResourceScope", err) + } + + blank := restrictedScope(" ", "gen", "a") + if _, _, err := repo.ReadResources(ctx, blank, 0, 10); !errors.Is(err, ErrInvalidResourceScope) { + t.Errorf("blank project = %v, want ErrInvalidResourceScope", err) + } + + for _, limit := range []int{0, -1, MaxResourceKeysetLimit + 1} { + if _, _, err := repo.ReadResources(ctx, unrestrictedScope("proj", "gen"), 0, limit); !errors.Is(err, ErrInvalidResourceScope) { + t.Errorf("limit %d = %v, want ErrInvalidResourceScope", limit, err) + } + } + if _, _, err := repo.ReadResources(ctx, unrestrictedScope("proj", "gen"), -1, 10); !errors.Is(err, ErrInvalidResourceScope) { + t.Errorf("negative cursor = %v, want ErrInvalidResourceScope", err) + } + + if _, err := repo.GetResource(ctx, unrestrictedScope("proj", "gen"), " ", "p-1"); !errors.Is(err, ErrInvalidResource) { + t.Errorf("blank resource type = %v, want ErrInvalidResource", err) + } + if _, err := repo.GetResource(ctx, unrestrictedScope("proj", "gen"), "Patient", ""); !errors.Is(err, ErrInvalidResource) { + t.Errorf("blank logical id = %v, want ErrInvalidResource", err) + } +} + +func TestResourceMigrationCreatesTypedResourceTable(t *testing.T) { + svc := newResourceService(t) + first := svc.mustMigrateEmbedded(t) + second := svc.mustMigrateEmbedded(t) + if !reflect.DeepEqual(first, second) { + t.Fatalf("re-apply changed the result:\n1st: %+v\n2nd: %+v", first, second) + } + assertLedgerEquals(t, svc.readLedger(t), mustEmbeddedMigrations(t)) + + var recordedName string + if err := svc.iso.QueryRow(svc.ctx, + `SELECT name FROM "`+svc.schema+`".`+LedgerTable+` WHERE version = 2`).Scan(&recordedName); err != nil { + svc.fail("witness version 2 ledger row", err) + } + if recordedName != "fhir_resource" { + t.Fatalf("version 2 ledger name = %q, want fhir_resource", recordedName) + } + + type columnWitness struct { + Name string + Type string + NotNull bool + Identity string + } + rows, err := svc.iso.Query(svc.ctx, ` + SELECT a.attname, format_type(a.atttypid, a.atttypmod), a.attnotnull, + CASE WHEN a.attidentity = 'a' THEN 'a' + WHEN a.attidentity = 'd' THEN 'd' + ELSE '' END + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = 'fhir_resource' + AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum`, svc.schema) + if err != nil { + svc.fail("witness columns", err) + } + defer rows.Close() + var columns []columnWitness + for rows.Next() { + var col columnWitness + if err := rows.Scan(&col.Name, &col.Type, &col.NotNull, &col.Identity); err != nil { + svc.fail("witness columns scan", err) + } + columns = append(columns, col) + } + if err := rows.Err(); err != nil { + svc.fail("witness columns rows", err) + } + wantColumns := []columnWitness{ + {"resource_seq", "bigint", true, "a"}, + {"project", "text", true, ""}, + {"dataset_generation", "text", true, ""}, + {"resource_type", "text", true, ""}, + {"logical_id", "text", true, ""}, + {"auth_resource_path", "text", true, ""}, + {"payload", "jsonb", false, ""}, + {"recorded_at", "timestamp with time zone", true, ""}, + } + if !reflect.DeepEqual(columns, wantColumns) { + t.Fatalf("fhir_resource columns = %+v, want %+v", columns, wantColumns) + } + + constraintRows, err := svc.iso.Query(svc.ctx, ` + SELECT con.conname, pg_get_constraintdef(con.oid) + FROM pg_constraint con + JOIN pg_class c ON c.oid = con.conrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = 'fhir_resource' + AND con.contype IN ('p', 'u', 'c') + ORDER BY con.conname`, svc.schema) + if err != nil { + svc.fail("witness constraints", err) + } + defer constraintRows.Close() + var constraints [][2]string + for constraintRows.Next() { + var name, def string + if err := constraintRows.Scan(&name, &def); err != nil { + svc.fail("witness constraints scan", err) + } + constraints = append(constraints, [2]string{name, def}) + } + if err := constraintRows.Err(); err != nil { + svc.fail("witness constraints rows", err) + } + wantConstraints := [][2]string{ + {"fhir_resource_logical_id_present", `CHECK ((logical_id <> ''::text))`}, + {"fhir_resource_natural_identity_ux", `UNIQUE (project, dataset_generation, resource_type, logical_id)`}, + {"fhir_resource_pkey", `PRIMARY KEY (resource_seq)`}, + {"fhir_resource_project_present", `CHECK ((project <> ''::text))`}, + {"fhir_resource_resource_type_present", `CHECK ((resource_type <> ''::text))`}, + } + if !reflect.DeepEqual(constraints, wantConstraints) { + t.Fatalf("fhir_resource constraints = %+v, want %+v", constraints, wantConstraints) + } + + indexRows, err := svc.iso.Query(svc.ctx, ` + SELECT indexname, indexdef FROM pg_indexes + WHERE schemaname = $1 AND tablename = 'fhir_resource' + ORDER BY indexname`, svc.schema) + if err != nil { + svc.fail("witness indexes", err) + } + defer indexRows.Close() + var indexes [][2]string + for indexRows.Next() { + var name, def string + if err := indexRows.Scan(&name, &def); err != nil { + svc.fail("witness indexes scan", err) + } + indexes = append(indexes, [2]string{name, def}) + } + if err := indexRows.Err(); err != nil { + svc.fail("witness indexes rows", err) + } + wantIndexes := [][2]string{ + {"fhir_resource_natural_identity_ux", + `CREATE UNIQUE INDEX fhir_resource_natural_identity_ux ON ` + svc.schema + `.fhir_resource USING btree (project, dataset_generation, resource_type, logical_id)`}, + {"fhir_resource_pkey", + `CREATE UNIQUE INDEX fhir_resource_pkey ON ` + svc.schema + `.fhir_resource USING btree (resource_seq)`}, + {"fhir_resource_scope_seq_ix", + `CREATE INDEX fhir_resource_scope_seq_ix ON ` + svc.schema + `.fhir_resource USING btree (project, dataset_generation, auth_resource_path, resource_seq)`}, + {"fhir_resource_semantic_keyset_ix", + `CREATE INDEX fhir_resource_semantic_keyset_ix ON ` + svc.schema + `.fhir_resource USING btree (project, dataset_generation, resource_type, logical_id COLLATE "C")`}, + } + if !reflect.DeepEqual(indexes, wantIndexes) { + t.Fatalf("fhir_resource indexes = %+v, want %+v", indexes, wantIndexes) + } + + var rowCount int64 + if err := svc.iso.QueryRow(svc.ctx, + `SELECT count(*) FROM "`+svc.schema+`".fhir_resource`).Scan(&rowCount); err != nil { + svc.fail("count resources", err) + } + if rowCount != 0 { + t.Fatalf("fresh fhir_resource has %d rows, want 0", rowCount) + } +} + +func TestResourceInsertAndExactReadPreserveFHIRJSON(t *testing.T) { + svc := newResourceService(t) + svc.mustMigrateEmbedded(t) + repo := svc.newRepository(t) + + const payload = `{"resourceType":"Observation","id":"obs-7","status":"final",` + + `"deceased":null,"category":[],"meta":{},"note":{"text":"keep"},"value":42}` + want := source.Resource{ + Project: "proj-f", DatasetGeneration: "gen-1", ResourceType: "Observation", + LogicalID: "obs-7", AuthResourcePath: "practitioner/pr-1", + Payload: json.RawMessage(payload), + } + seq := svc.mustInsert(t, repo, want) + + got := svc.mustGet(t, repo, + restrictedScope("proj-f", "gen-1", "practitioner/pr-1"), "Observation", "obs-7") + if got.ResourceSeq != seq { + t.Errorf("storage identity = %d, want %d assigned at insert", got.ResourceSeq, seq) + } + assertRowScope(t, got, want) + assertPayloadEqual(t, got.Payload, payload) + + obj := payloadObject(t, got.Payload) + raw, ok := obj["deceased"] + if !ok || string(raw) != "null" { + t.Errorf(`key "deceased" = %q (present=%t), want literal JSON null`, raw, ok) + } + raw, ok = obj["category"] + if !ok || string(raw) != "[]" { + t.Errorf(`key "category" = %q (present=%t), want literal []`, raw, ok) + } + raw, ok = obj["meta"] + if !ok || string(raw) != "{}" { + t.Errorf(`key "meta" = %q (present=%t), want literal {}`, raw, ok) + } + raw, ok = obj["value"] + if !ok || string(raw) != "42" { + t.Errorf(`key "value" = %q (present=%t), want literal 42`, raw, ok) + } + if raw, ok := obj["subject"]; ok { + t.Errorf(`absent key "subject" came back as %q; missing key must stay absent`, raw) + } + + var typeof string + if err := svc.iso.QueryRow(svc.ctx, + `SELECT jsonb_typeof(payload) FROM "`+svc.schema+`".fhir_resource WHERE resource_seq = $1`, + seq).Scan(&typeof); err != nil { + svc.fail("witness jsonb_typeof", err) + } + if typeof != "object" { + t.Errorf("jsonb_typeof(payload) = %q, want object", typeof) + } +} + +func TestResourcePayloadDistinguishesSQLNullJSONNullAndMissingKey(t *testing.T) { + svc := newResourceService(t) + svc.mustMigrateEmbedded(t) + repo := svc.newRepository(t) + scope := func() source.ResourceScope { return restrictedScope("proj-n", "gen-1", "") } + + absentSeq := svc.mustInsert(t, repo, source.Resource{ + Project: "proj-n", DatasetGeneration: "gen-1", ResourceType: "Patient", + LogicalID: "n-1", AuthResourcePath: "", + Payload: nil, + }) + jsonNullSeq := svc.mustInsert(t, repo, source.Resource{ + Project: "proj-n", DatasetGeneration: "gen-1", ResourceType: "Patient", + LogicalID: "n-2", AuthResourcePath: "", + Payload: json.RawMessage(`null`), + }) + objectSeq := svc.mustInsert(t, repo, source.Resource{ + Project: "proj-n", DatasetGeneration: "gen-1", ResourceType: "Patient", + LogicalID: "n-3", AuthResourcePath: "", + Payload: json.RawMessage(`{"id":"n-3","a":null}`), + }) + + witness := func(seq int64, wantIsNull bool, wantTypeof *string) { + t.Helper() + var isNull bool + var typeof *string + if err := svc.iso.QueryRow(svc.ctx, + `SELECT payload IS NULL, jsonb_typeof(payload) FROM "`+svc.schema+`".fhir_resource WHERE resource_seq = $1`, + seq).Scan(&isNull, &typeof); err != nil { + svc.fail("witness payload nulls", err) + } + if isNull != wantIsNull { + t.Errorf("seq %d payload IS NULL = %t, want %t", seq, isNull, wantIsNull) + } + if (typeof == nil) != (wantTypeof == nil) { + t.Errorf("seq %d jsonb_typeof = %v, want %v", seq, typeof, wantTypeof) + return + } + if wantTypeof != nil && *typeof != *wantTypeof { + t.Errorf("seq %d jsonb_typeof = %q, want %q", seq, *typeof, *wantTypeof) + } + } + nullType := "null" + objectType := "object" + witness(absentSeq, true, nil) + witness(jsonNullSeq, false, &nullType) + witness(objectSeq, false, &objectType) + + absent := svc.mustGet(t, repo, scope(), "Patient", "n-1") + if absent.Payload != nil { + t.Errorf("SQL NULL payload read back as %q, want nil", absent.Payload) + } + jsonNull := svc.mustGet(t, repo, scope(), "Patient", "n-2") + if string(jsonNull.Payload) != "null" || jsonNull.Payload == nil { + t.Errorf("JSON null payload read back as %q, want the bytes null", jsonNull.Payload) + } + object := svc.mustGet(t, repo, scope(), "Patient", "n-3") + obj := payloadObject(t, object.Payload) + if raw, ok := obj["a"]; !ok || string(raw) != "null" { + t.Errorf(`explicit null key "a" = %q (present=%t), want literal null`, raw, ok) + } + if raw, ok := obj["b"]; ok { + t.Errorf(`missing key "b" came back as %q; missing must stay distinct from null`, raw) + } +} + +func TestResourceDuplicateNaturalIdentityConflict(t *testing.T) { + svc := newResourceService(t) + svc.mustMigrateEmbedded(t) + repo := svc.newRepository(t) + + const original = `{"resourceType":"Patient","id":"dup-1","gender":"female"}` + const different = `{"resourceType":"Patient","id":"dup-1","gender":"male"}` + originalSeq := svc.mustInsert(t, repo, source.Resource{ + Project: "proj-c", DatasetGeneration: "gen-1", ResourceType: "Patient", + LogicalID: "dup-1", AuthResourcePath: "practitioner/pr-1", + Payload: json.RawMessage(original), + }) + + _, err := repo.InsertResource(svc.ctx, source.Resource{ + Project: "proj-c", DatasetGeneration: "gen-1", ResourceType: "Patient", + LogicalID: "dup-1", AuthResourcePath: "practitioner/pr-2", + Payload: json.RawMessage(different), + }) + if !errors.Is(err, ErrResourceConflict) { + t.Fatalf("duplicate insert = %v, want ErrResourceConflict", err) + } + var conflict *ResourceConflictError + if !errors.As(err, &conflict) { + t.Fatalf("error type = %T, want *ResourceConflictError", err) + } + wantConflict := &ResourceConflictError{ + Project: "proj-c", DatasetGeneration: "gen-1", ResourceType: "Patient", + LogicalID: "dup-1", AuthResourcePath: "practitioner/pr-2", + } + if !reflect.DeepEqual(conflict, wantConflict) { + t.Errorf("conflict = %+v, want %+v", conflict, wantConflict) + } + + got := svc.mustGet(t, repo, + restrictedScope("proj-c", "gen-1", "practitioner/pr-1"), "Patient", "dup-1") + if got.ResourceSeq != originalSeq { + t.Errorf("storage identity = %d after conflict, want the original %d: the row was replaced", + got.ResourceSeq, originalSeq) + } + assertPayloadEqual(t, got.Payload, original) + if got.AuthResourcePath != "practitioner/pr-1" { + t.Errorf("auth path = %q after conflict, want practitioner/pr-1", got.AuthResourcePath) + } + + if _, err := repo.InsertResource(svc.ctx, source.Resource{ + Project: "proj-c", DatasetGeneration: "gen-1", ResourceType: "Patient", + LogicalID: "dup-1", AuthResourcePath: "practitioner/pr-1", + Payload: json.RawMessage(original), + }); !errors.Is(err, ErrResourceConflict) { + t.Errorf("identical duplicate insert = %v, want ErrResourceConflict", err) + } + + var count int64 + if err := svc.iso.QueryRow(svc.ctx, + `SELECT count(*) FROM "`+svc.schema+`".fhir_resource + WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3 AND logical_id = $4`, + "proj-c", "gen-1", "Patient", "dup-1").Scan(&count); err != nil { + svc.fail("count natural identity", err) + } + if count != 1 { + t.Errorf("natural identity has %d rows, want 1", count) + } +} + +func TestResourceExactReadBindsScope(t *testing.T) { + svc := newResourceService(t) + svc.mustMigrateEmbedded(t) + repo := svc.newRepository(t) + + seed := []source.Resource{ + {Project: "proj-a", DatasetGeneration: "gen-1", ResourceType: "Patient", LogicalID: "p1", AuthResourcePath: "practitioner/pr-1", Payload: json.RawMessage(`{"id":"p1"}`)}, + {Project: "proj-a", DatasetGeneration: "gen-1", ResourceType: "Patient", LogicalID: "p2", AuthResourcePath: "practitioner/pr-2", Payload: json.RawMessage(`{"id":"p2"}`)}, + {Project: "proj-a", DatasetGeneration: "gen-1", ResourceType: "Patient", LogicalID: "p3", AuthResourcePath: "", Payload: json.RawMessage(`{"id":"p3"}`)}, + {Project: "proj-a", DatasetGeneration: "gen-2", ResourceType: "Patient", LogicalID: "p4", AuthResourcePath: "practitioner/pr-1", Payload: json.RawMessage(`{"id":"p4"}`)}, + {Project: "proj-b", DatasetGeneration: "gen-1", ResourceType: "Patient", LogicalID: "p5", AuthResourcePath: "practitioner/pr-1", Payload: json.RawMessage(`{"id":"p5"}`)}, + {Project: "proj-a", DatasetGeneration: "gen-1", ResourceType: "Encounter", LogicalID: "p1", AuthResourcePath: "practitioner/pr-1", Payload: json.RawMessage(`{"id":"enc-p1"}`)}, + } + for _, rec := range seed { + svc.mustInsert(t, repo, rec) + } + + got := svc.mustGet(t, repo, restrictedScope("proj-a", "gen-1", "practitioner/pr-1"), "Patient", "p1") + assertRowScope(t, got, seed[0]) + assertPayloadEqual(t, got.Payload, `{"id":"p1"}`) + + got = svc.mustGet(t, repo, restrictedScope("proj-a", "gen-1", ""), "Patient", "p3") + assertRowScope(t, got, seed[2]) + assertPayloadEqual(t, got.Payload, `{"id":"p3"}`) + + notFoundCases := []struct { + name string + scope source.ResourceScope + resourceType string + logicalID string + }{ + {"wrong path", restrictedScope("proj-a", "gen-1", "practitioner/pr-2"), "Patient", "p1"}, + {"wrong generation", restrictedScope("proj-a", "gen-1", "practitioner/pr-1"), "Patient", "p4"}, + {"other project", restrictedScope("proj-a", "gen-1", "practitioner/pr-1"), "Patient", "p5"}, + {"other resource type", restrictedScope("proj-a", "gen-1", "practitioner/pr-1"), "Encounter", "nope"}, + {"restricted with no paths", restrictedScope("proj-a", "gen-1"), "Patient", "p1"}, + } + for _, tc := range notFoundCases { + _, err := repo.GetResource(svc.ctx, tc.scope, tc.resourceType, tc.logicalID) + if !errors.Is(err, ErrResourceNotFound) { + t.Errorf("%s: GetResource = %v, want ErrResourceNotFound", tc.name, err) + continue + } + var nf *ResourceNotFoundError + if errors.As(err, &nf) && !nf.Restricted { + t.Errorf("%s: not-found error %+v does not report restricted mode", tc.name, nf) + } + } + + got = svc.mustGet(t, repo, unrestrictedScope("proj-a", "gen-1"), "Patient", "p2") + assertRowScope(t, got, seed[1]) + if _, err := repo.GetResource(svc.ctx, unrestrictedScope("proj-a", "gen-1"), "Patient", "p4"); !errors.Is(err, ErrResourceNotFound) { + t.Errorf("unrestricted read across generations = %v, want ErrResourceNotFound", err) + } + if _, err := repo.GetResource(svc.ctx, unrestrictedScope("proj-a", "gen-1"), "Patient", "p5"); !errors.Is(err, ErrResourceNotFound) { + t.Errorf("unrestricted read across projects = %v, want ErrResourceNotFound", err) + } + + got = svc.mustGet(t, repo, restrictedScope("proj-b", "gen-1", "practitioner/pr-1"), "Patient", "p5") + assertRowScope(t, got, seed[4]) +} + +func TestResourceKeysetReadStableOrderNeverCrossesScope(t *testing.T) { + svc := newResourceService(t) + svc.mustMigrateEmbedded(t) + repo := svc.newRepository(t) + + inScope := []source.Resource{ + {Project: "proj-k", DatasetGeneration: "gen-1", ResourceType: "Patient", LogicalID: "k1", AuthResourcePath: "practitioner/pr-2", Payload: json.RawMessage(`{"id":"k1"}`)}, + {Project: "proj-k", DatasetGeneration: "gen-1", ResourceType: "Patient", LogicalID: "k2", AuthResourcePath: "practitioner/pr-1", Payload: json.RawMessage(`{"id":"k2"}`)}, + {Project: "proj-k", DatasetGeneration: "gen-1", ResourceType: "Patient", LogicalID: "k3", AuthResourcePath: "practitioner/pr-1", Payload: json.RawMessage(`{"id":"k3"}`)}, + {Project: "proj-k", DatasetGeneration: "gen-1", ResourceType: "Observation", LogicalID: "k4", AuthResourcePath: "practitioner/pr-2", Payload: json.RawMessage(`{"id":"k4"}`)}, + {Project: "proj-k", DatasetGeneration: "gen-1", ResourceType: "Patient", LogicalID: "k5", AuthResourcePath: "practitioner/pr-1", Payload: json.RawMessage(`{"id":"k5"}`)}, + } + decoys := []source.Resource{ + {Project: "proj-x", DatasetGeneration: "gen-1", ResourceType: "Patient", LogicalID: "x1", AuthResourcePath: "practitioner/pr-1", Payload: json.RawMessage(`{"id":"x1"}`)}, + {Project: "proj-k", DatasetGeneration: "gen-2", ResourceType: "Patient", LogicalID: "k6", AuthResourcePath: "practitioner/pr-1", Payload: json.RawMessage(`{"id":"k6"}`)}, + {Project: "proj-k", DatasetGeneration: "gen-1", ResourceType: "Patient", LogicalID: "k7", AuthResourcePath: "practitioner/pr-forbidden", Payload: json.RawMessage(`{"id":"k7"}`)}, + {Project: "proj-x", DatasetGeneration: "gen-2", ResourceType: "Patient", LogicalID: "x2", AuthResourcePath: "practitioner/pr-2", Payload: json.RawMessage(`{"id":"x2"}`)}, + } + insertOrder := []source.Resource{ + inScope[2], decoys[0], inScope[0], decoys[1], inScope[4], decoys[2], inScope[1], decoys[3], inScope[3], + } + for _, rec := range insertOrder { + svc.mustInsert(t, repo, rec) + } + wantSeqs := make([]int64, 0, len(inScope)) + for _, rec := range inScope { + got := svc.mustGet(t, repo, restrictedScope("proj-k", "gen-1", "practitioner/pr-1", "practitioner/pr-2"), rec.ResourceType, rec.LogicalID) + wantSeqs = append(wantSeqs, got.ResourceSeq) + } + if len(wantSeqs) != len(inScope) { + t.Fatalf("seeded %d in-scope rows, witnessed %d", len(inScope), len(inScope)) + } + if !(wantSeqs[2] < wantSeqs[0] && wantSeqs[0] < wantSeqs[4] && wantSeqs[4] < wantSeqs[1] && wantSeqs[1] < wantSeqs[3]) { + t.Fatalf("seed storage order %+v is not k3 10 { + t.Fatal("keyset walk did not terminate within 10 pages") + } + for i, rec := range rows { + if i > 0 && rec.ResourceSeq <= rows[i-1].ResourceSeq { + t.Fatalf("page not ascending by storage identity: %+v", rows) + } + if rec.Project != "proj-k" || rec.DatasetGeneration != "gen-1" { + t.Fatalf("page crossed project/generation: %+v", rec) + } + if rec.AuthResourcePath != "practitioner/pr-1" && rec.AuthResourcePath != "practitioner/pr-2" { + t.Fatalf("page crossed auth path: %+v", rec) + } + if rec.ResourceSeq <= cursor { + t.Fatalf("row %+v did not advance past cursor %d", rec, cursor) + } + } + walked = append(walked, rows...) + if len(rows) > 0 { + cursor = rows[len(rows)-1].ResourceSeq + } + if !hasMore { + break + } + } + if pages != len(wantPages) { + t.Errorf("walked %d pages, want %d (five in-scope rows at limit 2)", pages, len(wantPages)) + } + if len(walked) != len(inScope) { + t.Fatalf("walk returned %d rows, want %d: %+v", len(walked), len(inScope), walked) + } + + seenID := make(map[string]int, len(walked)) + for i, rec := range walked { + seenID[rec.ResourceType+"/"+rec.LogicalID]++ + witness := svc.mustGet(t, repo, restrictedScope("proj-k", "gen-1", "practitioner/pr-1", "practitioner/pr-2"), rec.ResourceType, rec.LogicalID) + if rec.ResourceSeq != witness.ResourceSeq { + t.Fatalf("walk position %d has storage identity %d, want the witnessed %d for %s/%s", + i, rec.ResourceSeq, witness.ResourceSeq, rec.ResourceType, rec.LogicalID) + } + if i > 0 && walked[i-1].ResourceSeq >= rec.ResourceSeq { + t.Fatalf("walk is not strictly ascending by storage identity at position %d: %d then %d", + i, walked[i-1].ResourceSeq, rec.ResourceSeq) + } + var want source.Resource + for _, cand := range inScope { + if cand.ResourceType == rec.ResourceType && cand.LogicalID == rec.LogicalID { + want = cand + break + } + } + if want.LogicalID == "" { + t.Fatalf("walk returned an unexpected row %+v", rec) + } + assertRowScope(t, rec, want) + assertPayloadEqual(t, rec.Payload, string(want.Payload)) + } + for _, rec := range inScope { + if seenID[rec.ResourceType+"/"+rec.LogicalID] != 1 { + t.Errorf("in-scope row %s/%s walked %d times, want exactly once", + rec.ResourceType, rec.LogicalID, seenID[rec.ResourceType+"/"+rec.LogicalID]) + } + } + + rows, hasMore := svc.mustRead(t, repo, scope, cursor, 2) + if len(rows) != 0 || hasMore { + t.Errorf("read past the end returned %d rows (hasMore=%t), want empty", len(rows), hasMore) + } + + rows, hasMore, err := repo.ReadResources(svc.ctx, restrictedScope("proj-k", "gen-1"), 0, 10) + if err != nil || len(rows) != 0 || hasMore { + t.Errorf("restricted-empty read = (%d rows, hasMore=%t, err=%v), want empty with no error", + len(rows), hasMore, err) + } + + rows, hasMore = svc.mustRead(t, repo, unrestrictedScope("proj-k", "gen-1"), 0, MaxResourceKeysetLimit) + if hasMore { + t.Fatalf("unrestricted read reported more rows past a full page") + } + if len(rows) != len(inScope)+1 { + t.Fatalf("unrestricted read returned %d rows, want %d (in-scope plus the forbidden-path row)", + len(rows), len(inScope)+1) + } + for _, rec := range rows { + if rec.Project != "proj-k" || rec.DatasetGeneration != "gen-1" { + t.Fatalf("unrestricted read crossed project/generation: %+v", rec) + } + } +} + +func TestResourceKeysetWalkStableUnderInserts(t *testing.T) { + svc := newResourceService(t) + svc.mustMigrateEmbedded(t) + repo := svc.newRepository(t) + scope := restrictedScope("proj-w", "gen-1", "practitioner/pr-1") + + mk := func(id string) source.Resource { + return source.Resource{ + Project: "proj-w", DatasetGeneration: "gen-1", ResourceType: "Patient", + LogicalID: id, AuthResourcePath: "practitioner/pr-1", + Payload: json.RawMessage(`{"id":"` + id + `"}`), + } + } + firstFour := []source.Resource{mk("w3"), mk("w1"), mk("w4"), mk("w2")} + for _, rec := range firstFour { + svc.mustInsert(t, repo, rec) + } + + page1, hasMore := svc.mustRead(t, repo, scope, 0, 2) + if len(page1) != 2 || !hasMore { + t.Fatalf("page1 = %d rows (hasMore=%t), want 2 rows more=true", len(page1), hasMore) + } + if page1[0].LogicalID != "w3" || page1[1].LogicalID != "w1" { + t.Fatalf("page1 returned %s,%s, want w3,w1: the read is not ordered by storage identity (lexical order leaked in)", + page1[0].LogicalID, page1[1].LogicalID) + } + cursor := page1[1].ResourceSeq + + lastTwo := []source.Resource{mk("w6"), mk("w5")} + for _, rec := range lastTwo { + svc.mustInsert(t, repo, rec) + } + + page2, hasMore := svc.mustRead(t, repo, scope, cursor, 2) + if len(page2) != 2 || !hasMore { + t.Fatalf("page2 = %d rows (hasMore=%t), want 2 rows more=true", len(page2), hasMore) + } + if page2[0].LogicalID != "w4" || page2[1].LogicalID != "w2" { + t.Fatalf("page2 returned %s,%s, want w4,w2: the page boundary shifted under concurrent inserts", + page2[0].LogicalID, page2[1].LogicalID) + } + cursor = page2[1].ResourceSeq + + page3, hasMore := svc.mustRead(t, repo, scope, cursor, 2) + if len(page3) != 2 || hasMore { + t.Fatalf("page3 = %d rows (hasMore=%t), want 2 rows more=false", len(page3), hasMore) + } + if page3[0].LogicalID != "w6" || page3[1].LogicalID != "w5" { + t.Fatalf("page3 returned %s,%s, want w6,w5 (new rows must land at the end in storage order)", + page3[0].LogicalID, page3[1].LogicalID) + } + + seen := map[string]int{} + for _, rec := range append(append(append([]ResourceRow(nil), page1...), page2...), page3...) { + seen[rec.LogicalID]++ + } + if len(seen) != 6 { + t.Fatalf("walk saw %d distinct rows, want 6: %v", len(seen), seen) + } + for id, count := range seen { + if count != 1 { + t.Errorf("row %s returned %d times, want exactly once", id, count) + } + } +} + +func TestResourceNoLeftovers(t *testing.T) { + raw := testPostgresURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %v", err) + } + defer admin.Close() + + var leftDatabases []string + rows, err := admin.Query(ctx, + `SELECT datname FROM pg_database WHERE datname LIKE 'pgstore04_it_%' ORDER BY datname`) + if err != nil { + t.Fatalf("probe leftover databases: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("probe leftover databases scan: %v", err) + } + leftDatabases = append(leftDatabases, name) + } + if err := rows.Err(); err != nil { + rows.Close() + t.Fatalf("probe leftover databases rows: %v", err) + } + rows.Close() + for _, name := range leftDatabases { + if !pgstore04ResourceNamePattern.MatchString(name) { + t.Fatalf("leftover database %q does not carry this ticket's prefix; refusing to drop it", name) + } + if _, err := admin.Exec(ctx, `DROP DATABASE IF EXISTS "`+name+`" WITH (FORCE)`); err != nil { + t.Errorf("drop leftover database %s: %v", name, err) + } else { + t.Errorf("dropped leftover database %s; an earlier resource test leaked it", name) + } + } + + var leftSchemas []string + rows, err = admin.Query(ctx, + `SELECT nspname FROM pg_namespace WHERE nspname LIKE 'pgstore04_it_%' ORDER BY nspname`) + if err != nil { + t.Fatalf("probe leftover schemas: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("probe leftover schemas scan: %v", err) + } + leftSchemas = append(leftSchemas, name) + } + if err := rows.Err(); err != nil { + rows.Close() + t.Fatalf("probe leftover schemas rows: %v", err) + } + rows.Close() + for _, name := range leftSchemas { + if !pgstore04ResourceNamePattern.MatchString(name) { + t.Fatalf("leftover schema %q does not carry this ticket's prefix; refusing to drop it", name) + } + if _, err := admin.Exec(ctx, `DROP SCHEMA IF EXISTS "`+name+`" CASCADE`); err != nil { + t.Errorf("drop leftover schema %s: %v", name, err) + } else { + t.Errorf("dropped leftover schema %s; an earlier resource test leaked it", name) + } + } +} diff --git a/internal/backends/postgres/internal/store/resource_writer.go b/internal/backends/postgres/internal/store/resource_writer.go new file mode 100644 index 000000000..28f4e3d10 --- /dev/null +++ b/internal/backends/postgres/internal/store/resource_writer.go @@ -0,0 +1,66 @@ +package postgres + +import ( + "context" + "fmt" + + "github.com/calypr/loom/internal/source" +) + +type ResourceBatch struct { + ID string + Project string + DatasetGeneration string + Rows []source.Resource +} + +type ResourceBatchResult struct { + BatchID string + Digest string + Committed int64 + SkippedIdentical int64 + Replay bool +} + +type ResourceWriter struct { + batch *BatchWriter +} + +func NewResourceWriter(src ConnSource, schema string) (*ResourceWriter, error) { + batch, err := NewBatchWriter(src, schema) + if err != nil { + return nil, err + } + return &ResourceWriter{batch: batch}, nil +} + +func (w *ResourceWriter) WriteResourceBatch(ctx context.Context, batch ResourceBatch) (ResourceBatchResult, error) { + if len(batch.Rows) == 0 { + return ResourceBatchResult{}, fmt.Errorf("%w: resource batch %q carries no rows", + ErrInvalidBatch, batch.ID) + } + result, err := w.batch.WriteBatch(ctx, Batch{ + ID: batch.ID, + Project: batch.Project, + DatasetGeneration: batch.DatasetGeneration, + Resources: batch.Rows, + }) + if err != nil { + return ResourceBatchResult{}, err + } + if result.CommittedEdges != 0 || result.SkippedIdenticalEdges != 0 { + return ResourceBatchResult{}, &BatchAccountingError{ + BatchID: batch.ID, + Side: "edges", + Expected: 0, + Committed: result.CommittedEdges, + } + } + return ResourceBatchResult{ + BatchID: result.BatchID, + Digest: result.Digest, + Committed: result.CommittedResources, + SkippedIdentical: result.SkippedIdenticalResources, + Replay: result.Replay, + }, nil +} diff --git a/internal/backends/postgres/internal/store/resource_writer_test.go b/internal/backends/postgres/internal/store/resource_writer_test.go new file mode 100644 index 000000000..a4af75650 --- /dev/null +++ b/internal/backends/postgres/internal/store/resource_writer_test.go @@ -0,0 +1,1229 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "net/url" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + ing01StorePrefix = "verify_ing01_store_" + ing01NameFmt = ing01StorePrefix + "%d_%s" +) + +var ing01StoreNamePattern = regexp.MustCompile(`^verify_ing01_store_[0-9]+_[0-9a-f]+$`) + +var ( + ing01OwnedMu sync.Mutex + ing01OwnedByName = map[string]string{} +) + +func ing01Register(db, schema string) { + ing01OwnedMu.Lock() + defer ing01OwnedMu.Unlock() + ing01OwnedByName[db] = schema +} + +var ( + ing01BaselineOnce sync.Once + ing01BaselineCount int64 + ing01BaselineErr error +) + +func ing01CaptureBaseline(raw string) { + ing01BaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + ing01BaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_ing01\\_store\\_%' ESCAPE '\\'"). + Scan(&count); err != nil { + ing01BaselineErr = fmt.Errorf("count baseline: %w", err) + return + } + ing01BaselineCount = count + }) +} + +type ing01Service struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func newIngest01Service(t *testing.T) *ing01Service { + t.Helper() + raw := testPostgresURL(t) + ing01CaptureBaseline(raw) + if ing01BaselineErr != nil { + t.Fatalf("capture pg_database baseline: %v", ing01BaselineErr) + } + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf(ing01NameFmt, time.Now().UnixNano(), randomHex(t)) + if !ing01StoreNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(ing01NameFmt, time.Now().UnixNano(), randomHex(t)) + if !ing01StoreNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + ing01Register(dbName, schema) + return &ing01Service{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} +} + +func (s *ing01Service) migrate(t *testing.T) { + t.Helper() + if _, err := MigrateSchema(s.ctx, s.openClient(t, 4), s.schema); err != nil { + s.fail("MigrateSchema", err) + } +} + +func (s *ing01Service) openClient(t *testing.T, maxConns int32) *Client { + t.Helper() + client, err := Open(s.ctx, Config{ + DSN: s.dsn, + MaxConns: maxConns, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *ing01Service) openAppClient(t *testing.T, app string, maxConns int32) *Client { + t.Helper() + u, err := url.Parse(s.dsn) + if err != nil { + s.fail("parse isolated dsn for app client", err) + } + q := u.Query() + q.Set("application_name", app) + u.RawQuery = q.Encode() + client, err := Open(s.ctx, Config{ + DSN: u.String(), + MaxConns: maxConns, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + s.fail("open app client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *ing01Service) newWriterOn(t *testing.T, src ConnSource) *ResourceWriter { + t.Helper() + writer, err := NewResourceWriter(src, s.schema) + if err != nil { + s.fail("new resource writer", err) + } + return writer +} + +func (s *ing01Service) newWriter(t *testing.T) *ResourceWriter { + t.Helper() + return s.newWriterOn(t, s.openClient(t, 4)) +} + +func (s *ing01Service) qualified(table string) string { + return pgx.Identifier{s.schema, table}.Sanitize() +} + +func (s *ing01Service) exec(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := s.iso.Exec(s.ctx, sql, args...); err != nil { + s.fail("fixture exec", err) + } +} + +func (s *ing01Service) mustInt64(t *testing.T, sql string, args ...any) int64 { + t.Helper() + var v int64 + if err := s.iso.QueryRow(s.ctx, sql, args...).Scan(&v); err != nil { + s.fail("witness "+sql, err) + } + return v +} + +func (s *ing01Service) poolBackendPID(t *testing.T, client *Client) int64 { + t.Helper() + conn, err := client.Acquire(s.ctx) + if err != nil { + s.fail("acquire for backend pid", err) + } + defer conn.Release() + var pid int64 + if err := conn.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + s.fail("witness pg_backend_pid", err) + } + return pid +} + +func (s *ing01Service) waitBackendWaiting(t *testing.T, app string) int64 { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for { + var pid int64 + var waitType, waitEvent *string + err := s.iso.QueryRow(s.ctx, + "SELECT pid, wait_event_type, wait_event FROM pg_stat_activity"+ + " WHERE application_name = $1 AND state = 'active'", app). + Scan(&pid, &waitType, &waitEvent) + if err == nil && waitType != nil && *waitType == "Lock" { + return pid + } + if time.Now().After(deadline) { + s.fail("waiting for backend "+app+" to block on a lock", + fmt.Errorf("backend never entered a lock wait")) + } + time.Sleep(10 * time.Millisecond) + } +} + +const ( + ing01Project = "prj-ing01" + ing01GenA = "gen-a" + + ing01PayloadA = `{"resourceType":"Patient","id":"p-1","active":true,"meta":{"tag":null},"link":[{"code":"X"}]}` + ing01PayloadB = `{"resourceType":"Patient","id":"p-1","active":false}` + ing01PayloadKey = `{"id":"p-2","resourceType":"Patient"}` + + ing01PayloadAText = `{"id": "p-1", "link": [{"code": "X"}], "meta": {"tag": null}, "active": true, "resourceType": "Patient"}` + ing01PayloadBText = `{"id": "p-1", "active": false, "resourceType": "Patient"}` + ing01PayloadKeyText = `{"id": "p-2", "resourceType": "Patient"}` + + ing01GoldenBatchID = "ing01/golden/0" + ing01GoldenDigest = "511de6c949331c638d82e53430c523d2fb345b63f53f7bf50f32fa4358aa1ff6" + ing01LegacyBatchID = "ing01/legacy/0" + ing01LegacyDigest = "587b4e7659240ecf902bab11984a63fddd3a2475b227458867e6354c3790f2df" + + ing01FingerprintA = "ddf4b412b24330dcfdf5f84759ed764e" + ing01FingerprintB = "d74ad9a14d6bb357e2c93cee012b3ac0" +) + +func ing01GoldenRows(project, generation string) []source.Resource { + return []source.Resource{ + {Project: project, DatasetGeneration: generation, ResourceType: "Patient", LogicalID: "p-1", + AuthResourcePath: "/fhir", Payload: []byte(ing01PayloadA)}, + {Project: project, DatasetGeneration: generation, ResourceType: "Patient", LogicalID: "p-2", + AuthResourcePath: "", Payload: []byte(ing01PayloadKey)}, + {Project: project, DatasetGeneration: generation, ResourceType: "Observation", LogicalID: "obs-9", + AuthResourcePath: "/fhir", Payload: nil}, + {Project: project, DatasetGeneration: generation, ResourceType: "Provenance", LogicalID: "prov-0", + AuthResourcePath: "/fhir", Payload: []byte(`null`)}, + } +} + +func ing01GoldenBatch(id, project, generation string) ResourceBatch { + return ResourceBatch{ID: id, Project: project, DatasetGeneration: generation, Rows: ing01GoldenRows(project, generation)} +} + +func ing01BatchDigest(b ResourceBatch) string { + return batchDigest(Batch{ + ID: b.ID, + Project: b.Project, + DatasetGeneration: b.DatasetGeneration, + Resources: b.Rows, + }) +} + +type ing01LedgerWitness struct { + digest string + resources int64 + edges int64 + project string + generation string +} + +func (s *ing01Service) witnessLedger(t *testing.T, batchID string) (ing01LedgerWitness, bool) { + t.Helper() + var w ing01LedgerWitness + err := s.iso.QueryRow(s.ctx, + "SELECT digest, resource_count, edge_count, project, dataset_generation FROM "+ + s.qualified(batchLedgerTableName)+" WHERE batch_id = $1", batchID). + Scan(&w.digest, &w.resources, &w.edges, &w.project, &w.generation) + if errors.Is(err, pgx.ErrNoRows) { + return ing01LedgerWitness{}, false + } + if err != nil { + s.fail("witness ledger row", err) + } + return w, true +} + +func (s *ing01Service) countIn(t *testing.T, project, generation string) int64 { + t.Helper() + return s.mustInt64(t, "SELECT count(*) FROM "+s.qualified(resourceTableName)+ + " WHERE project = $1 AND dataset_generation = $2", project, generation) +} + +func (s *ing01Service) scopeFingerprint(t *testing.T, project, generation string) string { + t.Helper() + var fp string + sql := "SELECT coalesce(md5(string_agg(entry, E'\n' ORDER BY resource_type, logical_id, auth_resource_path)),'EMPTY') FROM (" + + "SELECT resource_type, logical_id, auth_resource_path," + + " resource_type || '|' || logical_id || '|' || auth_resource_path || '|' ||" + + " coalesce(payload::text, '') AS entry" + + " FROM " + s.qualified(resourceTableName) + " WHERE project = $1 AND dataset_generation = $2) x" + if err := s.iso.QueryRow(s.ctx, sql, project, generation).Scan(&fp); err != nil { + s.fail("scope fingerprint", err) + } + return fp +} + +func (s *ing01Service) requireResourceRow(t *testing.T, project, generation, rtype, lid, authPath string, wantPayloadText *string) { + t.Helper() + var gotAuth string + var isNull bool + var payload *string + err := s.iso.QueryRow(s.ctx, + "SELECT auth_resource_path, payload IS NULL, payload::text FROM "+s.qualified(resourceTableName)+ + " WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3 AND logical_id = $4", + project, generation, rtype, lid).Scan(&gotAuth, &isNull, &payload) + if err != nil { + s.fail("witness resource row "+rtype+"/"+lid, err) + } + if gotAuth != authPath { + t.Fatalf("resource %s/%s auth path = %q, want %q", rtype, lid, gotAuth, authPath) + } + if wantPayloadText == nil { + if !isNull { + t.Fatalf("resource %s/%s payload = %v, want SQL NULL", rtype, lid, *payload) + } + return + } + if isNull { + t.Fatalf("resource %s/%s payload is SQL NULL, want document %s", rtype, lid, *wantPayloadText) + } + if *payload != *wantPayloadText { + t.Fatalf("resource %s/%s payload text = %s, want %s", rtype, lid, *payload, *wantPayloadText) + } +} + +func (s *ing01Service) requireNothingWritten(t *testing.T, project, generation, batchID string) { + t.Helper() + if n := s.countIn(t, project, generation); n != 0 { + t.Fatalf("fhir_resource rows for %s/%s = %d, want 0", project, generation, n) + } + if _, found := s.witnessLedger(t, batchID); found { + t.Fatalf("batch_write_ledger holds %q, want no marker", batchID) + } +} + +func TestPostgresResourceWriterConstructorRejectsInvalidInputs(t *testing.T) { + if _, err := NewResourceWriter(nil, "verify_ing01_store_ok"); err == nil { + t.Fatal("nil connection source accepted, want refusal") + } + raw := testPostgresURL(t) + client, err := Open(context.Background(), Config{DSN: raw, MaxConns: 1, MinConns: 0, AcquireTimeout: 5 * time.Second}) + if err != nil { + t.Fatalf("open client: %v", err) + } + defer client.Close() + for _, bad := range []string{"", "BadName", "1leading", strings.Repeat("s", 64), "semi;colon"} { + if _, err := NewResourceWriter(client, bad); !errors.Is(err, ErrInvalidSchemaName) { + t.Fatalf("schema %q error = %v, want ErrInvalidSchemaName", bad, err) + } + } +} + +func TestPostgresResourceWriterWritesBoundedBatchWithExactColumnsAndLedgerMarker(t *testing.T) { + s := newIngest01Service(t) + s.migrate(t) + writer := s.newWriter(t) + + batch := ing01GoldenBatch(ing01GoldenBatchID, ing01Project, ing01GenA) + res, err := writer.WriteResourceBatch(s.ctx, batch) + if err != nil { + s.fail("WriteResourceBatch", err) + } + if res.BatchID != batch.ID || res.Replay || res.Committed != 4 || res.SkippedIdentical != 0 { + t.Fatalf("result = %+v, want the golden 4/0 non-replay accounting", res) + } + if res.Digest != ing01GoldenDigest { + t.Fatalf("digest = %s, want the pinned golden %s", res.Digest, ing01GoldenDigest) + } + if got := ing01BatchDigest(batch); got != ing01GoldenDigest { + t.Fatalf("engine digest = %s, want the pinned golden %s", got, ing01GoldenDigest) + } + + s.requireResourceRow(t, ing01Project, ing01GenA, "Patient", "p-1", "/fhir", strPtr(ing01PayloadAText)) + s.requireResourceRow(t, ing01Project, ing01GenA, "Patient", "p-2", "", strPtr(ing01PayloadKeyText)) + s.requireResourceRow(t, ing01Project, ing01GenA, "Observation", "obs-9", "/fhir", nil) + s.requireResourceRow(t, ing01Project, ing01GenA, "Provenance", "prov-0", "/fhir", strPtr("null")) + if n := s.countIn(t, ing01Project, ing01GenA); n != 4 { + t.Fatalf("fhir_resource rows = %d, want 4", n) + } + if fp := s.scopeFingerprint(t, ing01Project, ing01GenA); fp != ing01FingerprintA { + t.Fatalf("scope fingerprint = %s, want the pinned golden %s", fp, ing01FingerprintA) + } + + seqs := s.mustInt64(t, "SELECT count(DISTINCT resource_seq) FROM "+s.qualified(resourceTableName)+ + " WHERE project = $1 AND dataset_generation = $2", ing01Project, ing01GenA) + if seqs != 4 { + t.Fatalf("distinct resource_seq = %d, want 4 server-assigned identities", seqs) + } + + w, found := s.witnessLedger(t, batch.ID) + if !found { + t.Fatal("no ledger marker after commit") + } + if w.digest != ing01GoldenDigest || w.resources != 4 || w.edges != 0 || w.project != ing01Project || w.generation != ing01GenA { + t.Fatalf("ledger marker = %+v, want golden digest with 4/0 under %s/%s", w, ing01Project, ing01GenA) + } +} + +func TestPostgresResourceWriterLegacyEmptyGenerationNamespaceCommits(t *testing.T) { + s := newIngest01Service(t) + s.migrate(t) + writer := s.newWriter(t) + + rows := []source.Resource{{ + Project: ing01Project, DatasetGeneration: "", ResourceType: "Patient", LogicalID: "p-1", + AuthResourcePath: "/fhir", Payload: []byte(ing01PayloadA), + }} + batch := ResourceBatch{ID: ing01LegacyBatchID, Project: ing01Project, DatasetGeneration: "", Rows: rows} + res, err := writer.WriteResourceBatch(s.ctx, batch) + if err != nil { + s.fail("legacy WriteResourceBatch", err) + } + if res.Committed != 1 || res.Replay { + t.Fatalf("legacy result = %+v, want a fresh single-row commit", res) + } + if res.Digest != ing01LegacyDigest || ing01BatchDigest(batch) != ing01LegacyDigest { + t.Fatalf("legacy digest = %s (engine %s), want the pinned literal %s", + res.Digest, ing01BatchDigest(batch), ing01LegacyDigest) + } + s.requireResourceRow(t, ing01Project, "", "Patient", "p-1", "/fhir", strPtr(ing01PayloadAText)) + if n := s.countIn(t, ing01Project, ing01GenA); n != 0 { + t.Fatalf("gen-a rows = %d, want 0 — the legacy namespace must not touch gen-a", n) + } +} + +func TestPostgresResourceWriterIdenticalReplayIsNoOpAndTamperedDigestRefuses(t *testing.T) { + s := newIngest01Service(t) + s.migrate(t) + writer := s.newWriter(t) + + batchID := "ing01/replay/0" + batch := ing01GoldenBatch(batchID, ing01Project, ing01GenA) + first, err := writer.WriteResourceBatch(s.ctx, batch) + if err != nil { + s.fail("first WriteResourceBatch", err) + } + before := s.scopeFingerprint(t, ing01Project, ing01GenA) + + for attempt := 0; attempt < 3; attempt++ { + replay, err := writer.WriteResourceBatch(s.ctx, batch) + if err != nil { + s.fail("replay WriteResourceBatch", err) + } + if !replay.Replay || replay.Committed != first.Committed || replay.SkippedIdentical != first.SkippedIdentical || + replay.Digest != first.Digest || replay.BatchID != first.BatchID { + t.Fatalf("replay attempt %d = %+v, want a byte-identical replay of %+v", attempt, replay, first) + } + if n := s.countIn(t, ing01Project, ing01GenA); n != 4 { + t.Fatalf("rows after replay %d = %d, want 4 — replay must write nothing", attempt, n) + } + } + if fp := s.scopeFingerprint(t, ing01Project, ing01GenA); fp != before { + t.Fatalf("scope fingerprint drifted across replays: %s != %s", fp, before) + } + + tampered := strings.Repeat("ab", 32) + s.exec(t, "UPDATE "+s.qualified(batchLedgerTableName)+" SET digest = $1 WHERE batch_id = $2", + tampered, batchID) + _, err = writer.WriteResourceBatch(s.ctx, batch) + var conflict *BatchIdentityConflictError + if !errors.As(err, &conflict) { + s.fail("replay against tampered digest", fmt.Errorf("err = %v, want *BatchIdentityConflictError", err)) + } + wantDigest := ing01BatchDigest(batch) + if conflict.BatchID != batchID || conflict.LedgerDigest != tampered || conflict.ProvidedDigest != wantDigest { + t.Fatalf("identity conflict = %+v, want tampered ledger digest %s against computed digest %s", + conflict, tampered, wantDigest) + } + if fp := s.scopeFingerprint(t, ing01Project, ing01GenA); fp != before { + t.Fatalf("tampered replay mutated the scope: %s != %s", fp, before) + } +} + +func TestPostgresResourceWriterDuplicateNaturalIdentityDifferentPayloadRefused(t *testing.T) { + s := newIngest01Service(t) + s.migrate(t) + writer := s.newWriter(t) + + if _, err := writer.WriteResourceBatch(s.ctx, ing01GoldenBatch("ing01/dup/first", ing01Project, ing01GenA)); err != nil { + s.fail("first commit", err) + } + storedFingerprint := s.scopeFingerprint(t, ing01Project, ing01GenA) + + rows := append(ing01GoldenRows(ing01Project, ing01GenA), source.Resource{ + Project: ing01Project, DatasetGeneration: ing01GenA, ResourceType: "Encounter", LogicalID: "enc-1", + AuthResourcePath: "/fhir", Payload: []byte(`{"resourceType":"Encounter","id":"enc-1"}`), + }) + rows[0].Payload = []byte(ing01PayloadB) + _, err := writer.WriteResourceBatch(s.ctx, ResourceBatch{ + ID: "ing01/dup/second", Project: ing01Project, DatasetGeneration: ing01GenA, Rows: rows, + }) + var conflict *BatchResourceConflictError + if !errors.As(err, &conflict) { + s.fail("duplicate payload batch", fmt.Errorf("err = %v, want *BatchResourceConflictError", err)) + } + if !errors.Is(err, ErrBatchRowConflict) { + t.Fatalf("duplicate payload error does not match ErrBatchRowConflict: %v", err) + } + if conflict.InBatch { + t.Fatalf("conflict reports in-batch, want the stored-row path: %+v", conflict) + } + if conflict.Project != ing01Project || conflict.DatasetGeneration != ing01GenA || + conflict.ResourceType != "Patient" || conflict.LogicalID != "p-1" { + t.Fatalf("conflict identity = %+v, want %s/%s Patient/p-1", conflict, ing01Project, ing01GenA) + } + + if fp := s.scopeFingerprint(t, ing01Project, ing01GenA); fp != storedFingerprint { + t.Fatalf("stored scope drifted after refused batch: %s != %s", fp, storedFingerprint) + } + if n := s.mustInt64(t, "SELECT count(*) FROM "+s.qualified(resourceTableName)+ + " WHERE resource_type = 'Encounter' AND logical_id = 'enc-1'"); n != 0 { + t.Fatalf("refused batch leaked its fresh Encounter row (count %d), want whole-batch rollback", n) + } + if _, found := s.witnessLedger(t, "ing01/dup/second"); found { + t.Fatal("refused batch left a ledger marker, want none") + } + + rows[0].Payload = []byte(ing01PayloadA) + res, err := writer.WriteResourceBatch(s.ctx, ResourceBatch{ + ID: "ing01/dup/control", Project: ing01Project, DatasetGeneration: ing01GenA, Rows: rows, + }) + if err != nil { + s.fail("control batch", err) + } + if res.Committed != 1 || res.SkippedIdentical != 4 { + t.Fatalf("control result = %+v, want 1 committed (the Encounter) and 4 identical rows skipped", res) + } +} + +func TestPostgresResourceWriterInBatchDuplicateDifferentPayloadRollsBackWholeBatch(t *testing.T) { + s := newIngest01Service(t) + s.migrate(t) + writer := s.newWriter(t) + + rows := []source.Resource{ + {Project: ing01Project, DatasetGeneration: ing01GenA, ResourceType: "Patient", LogicalID: "p-9", + AuthResourcePath: "/fhir", Payload: []byte(`{"resourceType":"Patient","id":"p-9","v":1}`)}, + {Project: ing01Project, DatasetGeneration: ing01GenA, ResourceType: "Patient", LogicalID: "p-10", + AuthResourcePath: "", Payload: []byte(`{"resourceType":"Patient","id":"p-10"}`)}, + {Project: ing01Project, DatasetGeneration: ing01GenA, ResourceType: "Patient", LogicalID: "p-9", + AuthResourcePath: "/fhir", Payload: []byte(`{"resourceType":"Patient","id":"p-9","v":2}`)}, + } + _, err := writer.WriteResourceBatch(s.ctx, ResourceBatch{ + ID: "ing01/inbatch/bad", Project: ing01Project, DatasetGeneration: ing01GenA, Rows: rows, + }) + var conflict *BatchResourceConflictError + if !errors.As(err, &conflict) { + s.fail("in-batch duplicate batch", fmt.Errorf("err = %v, want *BatchResourceConflictError", err)) + } + if !conflict.InBatch || conflict.ResourceType != "Patient" || conflict.LogicalID != "p-9" { + t.Fatalf("in-batch conflict = %+v, want the in-batch Patient/p-9 conflict", conflict) + } + s.requireNothingWritten(t, ing01Project, ing01GenA, "ing01/inbatch/bad") + + rows[2].Payload = []byte(`{"resourceType":"Patient","id":"p-9","v":1}`) + res, err := writer.WriteResourceBatch(s.ctx, ResourceBatch{ + ID: "ing01/inbatch/control", Project: ing01Project, DatasetGeneration: ing01GenA, Rows: rows, + }) + if err != nil { + s.fail("in-batch control", err) + } + if res.Committed != 2 || res.SkippedIdentical != 1 { + t.Fatalf("in-batch control = %+v, want 2 committed and 1 collapsed duplicate", res) + } + if n := s.countIn(t, ing01Project, ing01GenA); n != 2 { + t.Fatalf("rows after collapse = %d, want 2", n) + } +} + +func TestPostgresResourceWriterIdenticalRowsAcrossDistinctBatchesAreIdempotent(t *testing.T) { + s := newIngest01Service(t) + s.migrate(t) + writer := s.newWriter(t) + + batchA := ing01GoldenBatch("ing01/cross/a", ing01Project, ing01GenA) + first, err := writer.WriteResourceBatch(s.ctx, batchA) + if err != nil { + s.fail("first batch", err) + } + if first.Committed != 4 || first.Digest != ing01BatchDigest(batchA) { + t.Fatalf("first = %+v, want 4 committed under the engine digest", first) + } + batchB := ing01GoldenBatch("ing01/cross/b", ing01Project, ing01GenA) + second, err := writer.WriteResourceBatch(s.ctx, batchB) + if err != nil { + s.fail("second batch", err) + } + if second.Replay { + t.Fatalf("second = %+v, want row-level idempotency, not a ledger replay", second) + } + if second.Committed != 0 || second.SkippedIdentical != 4 { + t.Fatalf("second = %+v, want 0 committed and 4 identical rows skipped", second) + } + if n := s.countIn(t, ing01Project, ing01GenA); n != 4 { + t.Fatalf("rows after identical second batch = %d, want 4", n) + } + if _, found := s.witnessLedger(t, batchA.ID); !found { + t.Fatal("first batch marker missing") + } + if w, found := s.witnessLedger(t, batchB.ID); !found || w.resources != 0 || w.digest != ing01BatchDigest(batchB) { + t.Fatalf("second marker = %+v (found %v), want its own engine digest with 0 committed rows", w, found) + } +} + +func TestPostgresResourceWriterRejectsInvalidBatchesBeforeRoundTrip(t *testing.T) { + s := newIngest01Service(t) + s.migrate(t) + + closed := s.openClient(t, 1) + closed.Close() + writer := s.newWriterOn(t, closed) + + good := ing01GoldenRows(ing01Project, "gen-reject") + mutate := func(i int, f func(*source.Resource)) []source.Resource { + rows := ing01GoldenRows(ing01Project, "gen-reject") + f(&rows[i]) + return rows + } + overCapacity := make([]source.Resource, 0, MaxBatchResources+1) + for i := 0; i <= MaxBatchResources; i++ { + overCapacity = append(overCapacity, source.Resource{ + Project: ing01Project, DatasetGeneration: "gen-over", ResourceType: "Patient", + LogicalID: fmt.Sprintf("p-%d", i), AuthResourcePath: "/fhir", + }) + } + longID := strings.Repeat("x", MaxBatchIDLength+1) + + cases := []struct { + name string + batch ResourceBatch + contains string + }{ + {"blank id", ResourceBatch{ID: " ", Project: ing01Project, DatasetGeneration: "gen-reject", Rows: good}, "batch id is blank"}, + {"over-long id", ResourceBatch{ID: longID, Project: ing01Project, DatasetGeneration: "gen-reject", Rows: good}, "at most 255"}, + {"blank project", ResourceBatch{ID: "ing01/reject/project", Project: " ", DatasetGeneration: "gen-reject", Rows: good}, "project is blank"}, + {"no rows", ResourceBatch{ID: "ing01/reject/empty", Project: ing01Project, DatasetGeneration: "gen-reject", Rows: nil}, "carries no rows"}, + {"over capacity", ResourceBatch{ID: "ing01/reject/over", Project: ing01Project, DatasetGeneration: "gen-over", Rows: overCapacity}, "want at most 1000"}, + {"foreign row", ResourceBatch{ID: "ing01/reject/foreign", Project: ing01Project, DatasetGeneration: "gen-reject", + Rows: mutate(0, func(r *source.Resource) { r.DatasetGeneration = "gen-other" })}, "want the batch scope"}, + {"blank logical id", ResourceBatch{ID: "ing01/reject/lid", Project: ing01Project, DatasetGeneration: "gen-reject", + Rows: mutate(1, func(r *source.Resource) { r.LogicalID = " " })}, "logical id is blank"}, + {"invalid json payload", ResourceBatch{ID: "ing01/reject/json", Project: ing01Project, DatasetGeneration: "gen-reject", + Rows: mutate(1, func(r *source.Resource) { r.Payload = []byte(`{"broken":`) })}, "not valid JSON"}, + } + for _, tc := range cases { + _, err := writer.WriteResourceBatch(s.ctx, tc.batch) + if !errors.Is(err, ErrInvalidBatch) { + t.Fatalf("%s: err = %v, want ErrInvalidBatch", tc.name, err) + } + if !strings.Contains(err.Error(), tc.contains) { + t.Fatalf("%s: err = %v, want it to name %q", tc.name, err, tc.contains) + } + if errors.Is(err, ErrBatchWrite) { + t.Fatalf("%s: refusal reached the engine: %v", tc.name, err) + } + } + + for _, generation := range []string{"gen-reject", "gen-over", "gen-other"} { + s.requireNothingWritten(t, ing01Project, generation, "unused-marker-name") + } + for _, id := range []string{"ing01/reject/project", "ing01/reject/empty", "ing01/reject/over", + "ing01/reject/foreign", "ing01/reject/seq", "ing01/reject/lid", "ing01/reject/json"} { + if _, found := s.witnessLedger(t, id); found { + t.Fatalf("refused batch %q left a ledger marker", id) + } + } + + live := s.newWriter(t) + res, err := live.WriteResourceBatch(s.ctx, ResourceBatch{ + ID: "ing01/reject/control", Project: ing01Project, DatasetGeneration: "gen-reject", Rows: good, + }) + if err != nil { + s.fail("control commit", err) + } + if res.Committed != 4 { + t.Fatalf("control = %+v, want 4 committed", res) + } +} + +func TestPostgresResourceWriterCrashWindowAtEveryStepBeforeCommitLeavesNothing(t *testing.T) { + s := newIngest01Service(t) + s.migrate(t) + writer := s.newWriter(t) + + steps := []string{ + stepBegin, stepReadLedger, stepClaimLedger, stepStageResources, + stepCheckResources, stepPublishResourc, stepFinalizeLedger, stepCommit, + } + for i, step := range steps { + batch := ing01GoldenBatch(fmt.Sprintf("ing01/crash/%d", i), ing01Project, ing01GenA) + writer.batch.failHook = func(current string) error { + if current == step { + return fmt.Errorf("injected crash before %s", current) + } + return nil + } + _, err := writer.WriteResourceBatch(s.ctx, batch) + writer.batch.failHook = nil + if err == nil { + t.Fatalf("crash at %s: WriteResourceBatch succeeded, want failure", step) + } + if !errors.Is(err, ErrBatchWrite) { + t.Fatalf("crash at %s: err = %v, want ErrBatchWrite", step, err) + } + s.requireNothingWritten(t, ing01Project, ing01GenA, batch.ID) + + retry, err := writer.WriteResourceBatch(s.ctx, batch) + if err != nil { + s.fail(fmt.Sprintf("retry after crash at %s", step), err) + } + wantDigest := ing01BatchDigest(batch) + if retry.Replay || retry.Committed != 4 || retry.Digest != wantDigest { + t.Fatalf("retry after crash at %s = %+v, want the fresh golden commit under %s", step, retry, wantDigest) + } + if n := s.countIn(t, ing01Project, ing01GenA); n != 4 { + t.Fatalf("rows after retry from crash at %s = %d, want 4", step, n) + } + if w, found := s.witnessLedger(t, batch.ID); !found || w.resources != 4 || w.digest != wantDigest { + t.Fatalf("marker after retry from crash at %s = %+v (found %v), want the golden accounting", step, w, found) + } + s.exec(t, "DELETE FROM "+s.qualified(resourceTableName)+" WHERE project = $1 AND dataset_generation = $2", + ing01Project, ing01GenA) + s.exec(t, "DELETE FROM "+s.qualified(batchLedgerTableName)+" WHERE batch_id = $1", batch.ID) + } +} + +func TestPostgresResourceWriterReportedCountsChangeOnlyAtCommitTwoConnections(t *testing.T) { + s := newIngest01Service(t) + s.migrate(t) + + writerClient := s.openAppClient(t, "ing01-commit-w", 1) + writer := s.newWriterOn(t, writerClient) + writerPID := s.poolBackendPID(t, writerClient) + + observer, err := pgxpool.New(s.ctx, s.dsn) + if err != nil { + s.fail("open observer pool", err) + } + t.Cleanup(observer.Close) + + obsConn, err := observer.Acquire(s.ctx) + if err != nil { + s.fail("acquire observer connection", err) + } + defer obsConn.Release() + var observerPID int64 + if err := obsConn.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&observerPID); err != nil { + s.fail("witness observer pid", err) + } + if observerPID == writerPID { + t.Fatalf("observer and writer share backend %d, want two distinct real PG connections", observerPID) + } + + if _, err := obsConn.Exec(s.ctx, "BEGIN"); err != nil { + s.fail("observer begin", err) + } + if _, err := obsConn.Exec(s.ctx, "LOCK TABLE "+s.qualified(resourceTableName)+" IN ACCESS EXCLUSIVE MODE"); err != nil { + s.fail("observer lock", err) + } + + batch := ing01GoldenBatch("ing01/commit/visibility", ing01Project, ing01GenA) + type outcome struct { + res ResourceBatchResult + err error + } + done := make(chan outcome, 1) + go func() { + res, err := writer.WriteResourceBatch(context.Background(), batch) + done <- outcome{res, err} + }() + + blockedPID := s.waitBackendWaiting(t, "ing01-commit-w") + if blockedPID != writerPID || blockedPID == observerPID { + t.Fatalf("blocked backend pid = %d, want the writer backend %d (observer is %d)", + blockedPID, writerPID, observerPID) + } + + var midRows int64 + if err := obsConn.QueryRow(s.ctx, "SELECT count(*) FROM "+s.qualified(resourceTableName)+ + " WHERE project = $1 AND dataset_generation = $2", ing01Project, ing01GenA).Scan(&midRows); err != nil { + s.fail("mid-transaction row witness", err) + } + if midRows != 0 { + t.Fatalf("rows visible mid-transaction = %d, want 0 — counts may only change at commit", midRows) + } + if _, found := s.witnessLedger(t, batch.ID); found { + t.Fatal("ledger marker visible mid-transaction, want none — marker and rows commit together") + } + + if _, err := obsConn.Exec(s.ctx, "COMMIT"); err != nil { + s.fail("observer commit", err) + } + var o outcome + select { + case o = <-done: + case <-time.After(60 * time.Second): + s.fail("waiting for batch commit", errors.New("batch never completed after the lock was released")) + } + if o.err != nil { + s.fail("WriteResourceBatch", o.err) + } + if o.res.Committed != 4 || o.res.Digest != ing01BatchDigest(batch) { + t.Fatalf("committed result = %+v, want the golden 4-row commit", o.res) + } + if n := s.countIn(t, ing01Project, ing01GenA); n != o.res.Committed { + t.Fatalf("rows after commit = %d, want the reported %d", n, o.res.Committed) + } + if w, found := s.witnessLedger(t, batch.ID); !found || w.resources != 4 { + t.Fatalf("ledger marker = %+v (found %v), want the 4-row accounting after commit", w, found) + } +} + +type ing01RacePair struct { + w1, w2 *ResourceWriter + pid1 *int64 + pid2 *int64 + mu sync.Mutex +} + +func (s *ing01Service) newRacePair(t *testing.T) *ing01RacePair { + t.Helper() + c1 := s.openAppClient(t, fmt.Sprintf("ing01-race-a-%d", time.Now().UnixNano()), 1) + c2 := s.openAppClient(t, fmt.Sprintf("ing01-race-b-%d", time.Now().UnixNano()), 1) + pidA := s.poolBackendPID(t, c1) + pidB := s.poolBackendPID(t, c2) + if pidA == pidB || pidA == 0 || pidB == 0 { + t.Fatalf("race pools share backend %d/%d, want distinct connections", pidA, pidB) + } + pair := &ing01RacePair{pid1: new(int64), pid2: new(int64)} + pair.w1 = s.newWriterOn(t, c1) + pair.w2 = s.newWriterOn(t, c2) + install := func(w *ResourceWriter, slot *int64) { + w.batch.inTxProbe = func(step, _ string, tx pgx.Tx) { + if step != stepBegin { + return + } + pair.mu.Lock() + defer pair.mu.Unlock() + if *slot == 0 { + var pid int64 + if err := tx.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&pid); err == nil { + *slot = pid + } + } + } + } + install(pair.w1, pair.pid1) + install(pair.w2, pair.pid2) + t.Cleanup(func() { + pair.w1.batch.inTxProbe = nil + pair.w2.batch.inTxProbe = nil + }) + return pair +} + +type ing01RaceOutcome struct { + res ResourceBatchResult + err error +} + +func (p *ing01RacePair) raceBoth(s *ing01Service, batchA, batchB ResourceBatch) (ing01RaceOutcome, ing01RaceOutcome) { + p.mu.Lock() + *p.pid1, *p.pid2 = 0, 0 + p.mu.Unlock() + start := make(chan struct{}) + ra := make(chan ing01RaceOutcome, 1) + rb := make(chan ing01RaceOutcome, 1) + go func() { <-start; res, err := p.w1.WriteResourceBatch(s.ctx, batchA); ra <- ing01RaceOutcome{res, err} }() + go func() { <-start; res, err := p.w2.WriteResourceBatch(s.ctx, batchB); rb <- ing01RaceOutcome{res, err} }() + close(start) + return <-ra, <-rb +} + +func (p *ing01RacePair) requireDistinctBackends(t *testing.T, round int) { + t.Helper() + p.mu.Lock() + defer p.mu.Unlock() + if *p.pid1 == 0 || *p.pid2 == 0 || *p.pid1 == *p.pid2 { + t.Fatalf("round %d backends = (%d, %d), want two distinct pg_backend_pid values", + round, *p.pid1, *p.pid2) + } +} + +func TestPostgresResourceWriterConcurrentIdenticalBatchCommitsOnceTwoConnections(t *testing.T) { + const rounds = 12 + s := newIngest01Service(t) + s.migrate(t) + pair := s.newRacePair(t) + + for round := 0; round < rounds; round++ { + generation := fmt.Sprintf("gen-race1-%d", round) + batch := ing01GoldenBatch(fmt.Sprintf("ing01/race1/%d", round), ing01Project, generation) + wantDigest := ing01BatchDigest(batch) + + o1, o2 := pair.raceBoth(s, batch, batch) + pair.requireDistinctBackends(t, round) + if o1.err != nil { + s.fail(fmt.Sprintf("round %d connection A", round), o1.err) + } + if o2.err != nil { + s.fail(fmt.Sprintf("round %d connection B", round), o2.err) + } + + fresh, replays := 0, 0 + for _, r := range []ResourceBatchResult{o1.res, o2.res} { + if r.Replay { + replays++ + } else { + fresh++ + } + if r.Digest != wantDigest || r.Committed != 4 || r.SkippedIdentical != 0 { + t.Fatalf("round %d result = %+v, want the golden 4-row accounting under digest %s", + round, r, wantDigest) + } + } + if fresh != 1 || replays != 1 { + t.Fatalf("round %d had %d fresh and %d replays, want 1/1", round, fresh, replays) + } + if o1.res.Digest != o2.res.Digest || o1.res.Committed != o2.res.Committed || + o1.res.SkippedIdentical != o2.res.SkippedIdentical || o1.res.BatchID != o2.res.BatchID { + t.Fatalf("round %d results diverged: %+v vs %+v", round, o1.res, o2.res) + } + + if n := s.countIn(t, ing01Project, generation); n != 4 { + t.Fatalf("round %d rows = %d, want exactly 4 (committed once)", round, n) + } + if fp := s.scopeFingerprint(t, ing01Project, generation); fp != ing01FingerprintA { + t.Fatalf("round %d fingerprint = %s, want the pinned golden %s", round, fp, ing01FingerprintA) + } + w, found := s.witnessLedger(t, batch.ID) + if !found || w.resources != 4 || w.digest != wantDigest { + t.Fatalf("round %d marker = %+v (found %v), want the golden accounting", round, w, found) + } + } +} + +func TestPostgresResourceWriterConcurrentIdentityReuseDifferentContentRefusesOneTwoConnections(t *testing.T) { + const rounds = 12 + s := newIngest01Service(t) + s.migrate(t) + pair := s.newRacePair(t) + + for round := 0; round < rounds; round++ { + generation := fmt.Sprintf("gen-race2-%d", round) + batchID := fmt.Sprintf("ing01/race2/%d", round) + + contentA := ing01GoldenBatch(batchID, ing01Project, generation) + contentB := ing01GoldenBatch(batchID, ing01Project, generation) + contentB.Rows[0].Payload = []byte(ing01PayloadB) + digestA := ing01BatchDigest(contentA) + digestB := ing01BatchDigest(contentB) + + o1, o2 := pair.raceBoth(s, contentA, contentB) + pair.requireDistinctBackends(t, round) + + type side struct { + outcome ing01RaceOutcome + ownBatch ResourceBatch + ownDigest string + otherDigest string + ownPayload string + ownFingerprint string + } + sides := []side{ + {outcome: o1, ownBatch: contentA, ownDigest: digestA, otherDigest: digestB, + ownPayload: ing01PayloadAText, ownFingerprint: ing01FingerprintA}, + {outcome: o2, ownBatch: contentB, ownDigest: digestB, otherDigest: digestA, + ownPayload: ing01PayloadBText, ownFingerprint: ing01FingerprintB}, + } + var winner *side + var loser *side + for i := range sides { + var conflict *BatchIdentityConflictError + switch { + case errors.As(sides[i].outcome.err, &conflict): + if loser != nil { + t.Fatalf("round %d: both sides got identity conflicts", round) + } + loser = &sides[i] + if conflict.BatchID != batchID || conflict.ProvidedDigest != sides[i].ownDigest || + conflict.LedgerDigest != sides[i].otherDigest { + t.Fatalf("round %d conflict = %+v, want provided %s against ledger %s", + round, conflict, sides[i].ownDigest, sides[i].otherDigest) + } + case sides[i].outcome.err != nil: + s.fail(fmt.Sprintf("round %d unexpected loser classification", round), sides[i].outcome.err) + default: + if winner != nil { + t.Fatalf("round %d: both sides committed", round) + } + winner = &sides[i] + } + } + if winner == nil || loser == nil { + t.Fatalf("round %d outcomes = (%v, %v), want one commit and one *BatchIdentityConflictError", + round, o1.err, o2.err) + } + if winner.outcome.res.Committed != 4 || winner.outcome.res.Digest != winner.ownDigest { + t.Fatalf("round %d winner result = %+v, want the 4-row commit under %s", + round, winner.outcome.res, winner.ownDigest) + } + + if fp := s.scopeFingerprint(t, ing01Project, generation); fp != winner.ownFingerprint { + t.Fatalf("round %d fingerprint = %s, want the winner's pinned %s", round, fp, winner.ownFingerprint) + } + s.requireResourceRow(t, ing01Project, generation, "Patient", "p-1", "/fhir", strPtr(winner.ownPayload)) + if n := s.mustInt64(t, "SELECT count(*) FROM "+s.qualified(batchLedgerTableName)+" WHERE batch_id = $1", batchID); n != 1 { + t.Fatalf("round %d ledger rows for %q = %d, want exactly 1", round, batchID, n) + } + } +} + +func TestPostgresResourceWriterConcurrentDuplicateIdentityDifferentPayloadConvergesTwoConnections(t *testing.T) { + const rounds = 12 + s := newIngest01Service(t) + s.migrate(t) + pair := s.newRacePair(t) + + aPayload := `{"resourceType":"Patient","id":"p-dup","race":"a"}` + bPayload := `{"resourceType":"Patient","id":"p-dup","race":"b"}` + const ( + aPayloadText = `{"id": "p-dup", "race": "a", "resourceType": "Patient"}` + bPayloadText = `{"id": "p-dup", "race": "b", "resourceType": "Patient"}` + ) + + for round := 0; round < rounds; round++ { + generation := fmt.Sprintf("gen-race3-%d", round) + row := func(payload string) source.Resource { + return source.Resource{Project: ing01Project, DatasetGeneration: generation, + ResourceType: "Patient", LogicalID: "p-dup", AuthResourcePath: "/fhir", Payload: []byte(payload)} + } + companion := func(name string) source.Resource { + return source.Resource{Project: ing01Project, DatasetGeneration: generation, + ResourceType: "Observation", LogicalID: "obs-" + name, AuthResourcePath: "", + Payload: []byte(`{"resourceType":"Observation","id":"obs-` + name + `"}`)} + } + batchA := ResourceBatch{ID: fmt.Sprintf("ing01/race3/a/%d", round), Project: ing01Project, + DatasetGeneration: generation, Rows: []source.Resource{row(aPayload), companion("a")}} + batchB := ResourceBatch{ID: fmt.Sprintf("ing01/race3/b/%d", round), Project: ing01Project, + DatasetGeneration: generation, Rows: []source.Resource{row(bPayload), companion("b")}} + + o1, o2 := pair.raceBoth(s, batchA, batchB) + pair.requireDistinctBackends(t, round) + + type side struct { + outcome ing01RaceOutcome + batch ResourceBatch + payloadText string + ownCompanion string + } + sides := []side{ + {outcome: o1, batch: batchA, payloadText: aPayloadText, ownCompanion: "obs-a"}, + {outcome: o2, batch: batchB, payloadText: bPayloadText, ownCompanion: "obs-b"}, + } + var winner, loser *side + for i := range sides { + if sides[i].outcome.err == nil { + if winner != nil { + t.Fatalf("round %d: both batches committed", round) + } + winner = &sides[i] + continue + } + if loser != nil { + t.Fatalf("round %d: both batches refused (%v / %v)", round, o1.err, o2.err) + } + loser = &sides[i] + if !errors.Is(sides[i].outcome.err, ErrBatchRowConflict) && + !errors.Is(sides[i].outcome.err, ErrBatchAccounting) { + s.fail(fmt.Sprintf("round %d loser classification", round), + fmt.Errorf("err = %v, want ErrBatchRowConflict or ErrBatchAccounting", sides[i].outcome.err)) + } + } + if winner == nil || loser == nil || winner.outcome.res.Committed != 2 { + t.Fatalf("round %d outcomes = (%+v/%v, %+v/%v), want one 2-row commit and one typed refusal", + round, o1.res, o1.err, o2.res, o2.err) + } + + s.requireResourceRow(t, ing01Project, generation, "Patient", "p-dup", "/fhir", strPtr(winner.payloadText)) + if n := s.countIn(t, ing01Project, generation); n != 2 { + t.Fatalf("round %d rows = %d, want exactly 2 — the loser's rows must be rolled back", round, n) + } + if n := s.mustInt64(t, "SELECT count(*) FROM "+s.qualified(resourceTableName)+ + " WHERE dataset_generation = $1 AND logical_id = $2", generation, loser.ownCompanion); n != 0 { + t.Fatalf("round %d loser's own companion row %s survived (%d), want whole-batch rollback", + round, loser.ownCompanion, n) + } + if _, found := s.witnessLedger(t, loser.batch.ID); found { + t.Fatalf("round %d loser batch %q left a ledger marker", round, loser.batch.ID) + } + if _, found := s.witnessLedger(t, winner.batch.ID); !found { + t.Fatalf("round %d winner batch %q marker missing", round, winner.batch.ID) + } + } +} + +func TestZZZeroLeftoverPostgresResourceWriterFixtures(t *testing.T) { + raw := testPostgresURL(t) + ing01OwnedMu.Lock() + owned := make(map[string]string, len(ing01OwnedByName)) + for db, schema := range ing01OwnedByName { + owned[db] = schema + } + ing01OwnedMu.Unlock() + if len(owned) == 0 { + t.Skip("this process created no verify_ing01_store_* fixtures") + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service for audit: %v", err) + } + defer admin.Close() + + for db, schema := range owned { + if !strings.HasPrefix(db, ing01StorePrefix) { + t.Fatalf("registry holds foreign database %q", db) + } + var count int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Errorf("audit owned database %s: %v", db, err) + continue + } + if count != 0 { + t.Errorf("leftover owned database %s after all cleanups", db) + if schema != "" { + u, err := url.Parse(raw) + if err == nil { + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + db + if probe, err := pgxpool.New(ctx, u.String()); err == nil { + var schemaCount int + if err := probe.QueryRow(ctx, "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1", schema).Scan(&schemaCount); err != nil { + t.Errorf("audit owned schema %s in %s: %v", schema, db, err) + } else if schemaCount != 0 { + t.Errorf("leftover owned schema %s in leftover database %s", schema, db) + } + probe.Close() + } + } + } + } + } + + var visible int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_ing01\\_store\\_%' ESCAPE '\\'"). + Scan(&visible); err != nil { + t.Fatalf("audit verify_ing01_store_* database count: %v", err) + } + if visible != ing01BaselineCount { + t.Errorf("verify_ing01_store_* databases = %d, want baseline %d", visible, ing01BaselineCount) + } +} diff --git a/internal/backends/postgres/internal/store/schema_name_rule_test.go b/internal/backends/postgres/internal/store/schema_name_rule_test.go new file mode 100644 index 000000000..31b8f7e14 --- /dev/null +++ b/internal/backends/postgres/internal/store/schema_name_rule_test.go @@ -0,0 +1,73 @@ +package postgres + +import ( + "regexp" + "strings" + "testing" +) + +func TestValidSchemaNameVerdictTable(t *testing.T) { + cases := []struct { + schema string + want bool + }{ + {"", false}, + {"a", true}, + {"_", true}, + {strings.Repeat("a", 63), true}, + {strings.Repeat("a", 64), false}, + {"z" + strings.Repeat("_z", 31), true}, + {"z" + strings.Repeat("_z", 32), false}, + {"_mix9_case", true}, + {"a9_", true}, + {"9lead", false}, + {"Upper", false}, + {"A", false}, + {"a.b", false}, + {"a b", false}, + {" a", false}, + {"a ", false}, + {"a-b", false}, + {"a;b", false}, + {"a\"b", false}, + {"a列", false}, + {"ä", false}, + {"a\x00b", false}, + {"\n", false}, + } + for _, tc := range cases { + if got := ValidSchemaName(tc.schema); got != tc.want { + t.Errorf("ValidSchemaName(%q) = %v, want %v", tc.schema, got, tc.want) + } + } +} + +func TestValidSchemaNameMatchesCanonicalRegexp(t *testing.T) { + canonical := regexp.MustCompile(`^[a-z_][a-z0-9_]{0,62}$`) + corpus := []string{ + "", "a", "_", "9", "A", "z_", "a.b", "a b", "a-b", "a\tb", "a\nb", + "a\x00b", "ä", "列", "a列", "9_", "_9", "a9", "A_", "__", "a__b9", + strings.Repeat("a", 62), strings.Repeat("a", 63), strings.Repeat("a", 64), + strings.Repeat("a", 65), "z" + strings.Repeat("_z", 30), "z" + strings.Repeat("_z", 31), + strings.Repeat("_", 63), strings.Repeat("_", 64), + "public", "loom_test_1", "verify_pgcomp_1_abc", + "schema; DROP TABLE x", "a'b", `a"b`, "a\\b", "%", "*", "a%", + } + alphabet := []string{"a", "Z", "9", "_", ".", "-", " ", "\t", "列"} + var build func(prefix string, depth int) + build = func(prefix string, depth int) { + if depth == 0 { + corpus = append(corpus, prefix) + return + } + for _, ch := range alphabet { + build(prefix+ch, depth-1) + } + } + build("", 2) + for _, schema := range corpus { + if want := canonical.MatchString(schema); ValidSchemaName(schema) != want { + t.Fatalf("ValidSchemaName(%q) diverges from canonical regexp: got %v, want %v", schema, !want, want) + } + } +} diff --git a/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/internal/dataset/postgres/testdata/migrations-releases/000008_project_release.sql b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/internal/dataset/postgres/testdata/migrations-releases/000008_project_release.sql new file mode 100644 index 000000000..0f17966e5 --- /dev/null +++ b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/internal/dataset/postgres/testdata/migrations-releases/000008_project_release.sql @@ -0,0 +1,97 @@ +-- pg-publication-project-release-cas-v1: immutable ProjectRelease candidates +-- and the active-project-release compare-and-swap pointer. +-- +-- Ownership: this lane owns exactly these two tables. The dataset generation +-- manifest and the active-generation pointer belong to pg-store-03 +-- (000003_dataset_generation_manifest.sql) and are NOT duplicated here; the +-- generation owner participates in release activation through the +-- ReleaseStore's GenerationOwnerAdapter inside the same PostgreSQL +-- transaction, reading its own rows -- no mirrored manifest or generation +-- columns are added to either side. ClickHouse remains the publication row +-- store: no dataframe rows move here, and Arango/AQL stay supported. +-- +-- Version reservation: 000006 and 000007 are coordinator-reserved shared +-- numbers (tests/migrations-leases holds 000006). This fixture is +-- package-local (loaded only by this package's tests through the pg-store-02 +-- runner alongside the embedded ledger bootstrap and migration 3) and takes +-- 000008 so it composes without colliding with the shared reservations. +-- Promotion to a shared migrations directory requires coordinator approval. +-- +-- project_release is the immutable candidate archive: +-- * One row per (project, content-addressed release id). The id is an +-- opaque immutable content label minted by internal/dataset +-- (releaseID, "release_" + SHA-256 hex); this schema deliberately adds +-- no id-format CHECK so pre-widening digests and any future framing +-- change stay readable, matching the reader-side treatment in +-- internal/dataset/release.go. +-- * payload stores the complete ProjectRelease JSON document exactly as +-- saved. It is authoritative on read, so nanosecond timestamps and the +-- JSON null-versus-absent shape survive a save/read round trip +-- unchanged; every column is NOT NULL and no statement binds SQL NULL, +-- so missing, JSON-null, and SQL-NULL stay distinct. +-- * generation and git_commit are scan columns derived from the same +-- struct fields the payload carries; the store verifies column/payload +-- agreement on every read and refuses disagreement as a store +-- inconsistency instead of coercing it. +-- * Rows are never updated or deleted: inactive candidates stay retained +-- for audit, so a lost activation race never destroys the candidate. +-- +-- active_project_release is the active-visibility pointer: +-- * One row per project. A missing row means no active release -- there is +-- no placeholder row and no SQL NULL standing in for "absent", matching +-- the pg-store-03 pointer convention. +-- * revision is the CAS counter compared inside the activation +-- transaction: it starts at 1 (revision >= 1; callers express "no +-- active release yet" as expectedRevision 0, exactly as the Arango +-- baseline's releasePointer == null ? 0 rule does) and only ever moves +-- through the store's compare-and-swap. The foreign key makes "the +-- active pointer names a persisted candidate" a schema invariant, +-- mirroring how 000003 makes the active-generation manifest a schema +-- invariant. +-- +-- DDL is retryable (IF NOT EXISTS) per the pg-store-02 runner contract: a +-- crash between a migration's commit and its ledger record must converge on +-- re-application. +CREATE TABLE IF NOT EXISTS project_release ( + project text NOT NULL, + id text NOT NULL, + generation text NOT NULL, + git_commit text NOT NULL, + payload jsonb NOT NULL, + CONSTRAINT project_release_pkey + PRIMARY KEY (project, id), + CONSTRAINT project_release_project_nonempty + CHECK (project <> ''), + CONSTRAINT project_release_id_nonempty + CHECK (id <> ''), + CONSTRAINT project_release_generation_nonempty + CHECK (generation <> ''), + CONSTRAINT project_release_git_commit_nonempty + CHECK (git_commit <> ''), + CONSTRAINT project_release_payload_is_object + CHECK (jsonb_typeof(payload) = 'object') +); + +-- Candidate lookup by audit scope: which candidates were ever retained for a +-- project+generation, newest identity first by id. +CREATE INDEX IF NOT EXISTS project_release_project_generation + ON project_release (project, generation); + +CREATE TABLE IF NOT EXISTS active_project_release ( + project text NOT NULL, + release_id text NOT NULL, + revision bigint NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT active_project_release_pkey + PRIMARY KEY (project), + CONSTRAINT active_project_release_project_nonempty + CHECK (project <> ''), + CONSTRAINT active_project_release_release_id_nonempty + CHECK (release_id <> ''), + CONSTRAINT active_project_release_revision_positive + CHECK (revision >= 1), + CONSTRAINT active_project_release_release_fkey + FOREIGN KEY (project, release_id) + REFERENCES project_release (project, id) +); diff --git a/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-capabilities/000004_capability_artifacts.sql b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-capabilities/000004_capability_artifacts.sql new file mode 100644 index 000000000..cd42f0cf1 --- /dev/null +++ b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-capabilities/000004_capability_artifacts.sql @@ -0,0 +1,151 @@ +-- pg-state-immutable-capability-receipts-v1: immutable capability snapshots +-- and table-shape capability receipts for the Explorer control plane. +-- +-- Both tables are fully bound, create-once artifacts. The jsonb payload is +-- authoritative for reads (nanosecond-free canonical snapshot documents and +-- receipt objects with their missing-versus-JSON-null member shapes +-- round-trip untouched); the scan columns mirror the payload for retrieval +-- scopes, constraints, and ordered scans only. Every statement in the +-- adapter writes both from the same validated value, and the adapter +-- re-checks payload-versus-mirror agreement on every read. No row is ever +-- UPDATEd or DELETEd: content identity (token / content digest) is the key, +-- so a changed payload is a new artifact or a typed collision, never a +-- mutation. The version continues the Explorer control-plane fixture history +-- after the 000003 draft-revision reservation (pg-state-explorer-draft-cas) +-- and the DDL is idempotent because the pg-store-02 runner may retry a +-- migration that committed before its ledger row was written. +-- +-- Capability snapshot identity: +-- * token is the content address ("sha256:" + 64 lowercase hex of the +-- canonical snapshot payload). Put is insert-only, so concurrent +-- duplicate puts converge on the existing row; a stored payload that no +-- longer hashes to its token is a typed collision, never a rewrite. +-- * The identity columns mirror SnapshotIdentity exactly (empty string +-- stands in for the omitted cacheVersion/shapeDigest, matching the +-- Arango adapter's NOT_NULL(identity.cacheVersion, "") filter). The +-- status CHECK closes the domain and the READY guard keeps the SQL +-- mirror consistent with validateSnapshot's READY completeness rule; +-- the payload remains the authority that the adapter re-validates. +-- +-- Table-shape receipt identity: +-- * id is the content-derived opaque artifact ID with a kind-bound prefix +-- (tsc_ catalog, tsd_ category scan, tsr_ resolution), so the CHECK +-- below makes a kind/prefix mismatch impossible at the database, not +-- just in the adapter. content_digest is the "sha256:" digest over the +-- receipt's canonical identity bytes. +-- * Every binding column is NOT NULL and non-empty and draft_version is +-- at least 1, mirroring Binding.Validate; retrieval is scoped by the +-- full binding, so an opaque ID is never trusted without its current +-- project/Explorer/output/snapshot/source binding. parent_catalog_id is +-- empty exactly for CATALOG rows. +-- +-- Immutability is enforced here, not only in Go: AFTER the tables, a +-- shared guard function plus BEFORE UPDATE OR DELETE triggers refuse any +-- mutation of an artifact row with SQLSTATE 45501 before it can run. The +-- suite proves the guards are not vacuous by dropping each trigger, +-- repeating the violating statement (which must then succeed), and +-- restoring the trigger. + +CREATE TABLE IF NOT EXISTS capability_snapshots ( + token text PRIMARY KEY + CONSTRAINT capability_snapshots_token_shape CHECK (token ~ '^sha256:[0-9a-f]{64}$'), + project text NOT NULL DEFAULT '', + generation text NOT NULL DEFAULT '', + authorization_scope_digest text NOT NULL DEFAULT '', + cache_version text NOT NULL DEFAULT '', + schema_digest text NOT NULL DEFAULT '', + resource_inventory_digest text NOT NULL DEFAULT '', + relationship_digest text NOT NULL DEFAULT '', + field_digest text NOT NULL DEFAULT '', + shape_digest text NOT NULL DEFAULT '', + protocol_version text NOT NULL DEFAULT '', + compiler_version text NOT NULL DEFAULT '', + traversal_policy_version text NOT NULL DEFAULT '', + projection_policy_version text NOT NULL DEFAULT '', + status text NOT NULL + CONSTRAINT capability_snapshots_status_known CHECK (status IN ('BUILDING', 'READY', 'FAILED')), + complete boolean NOT NULL, + truncated boolean NOT NULL, + payload jsonb NOT NULL + CONSTRAINT capability_snapshots_payload_object CHECK (jsonb_typeof(payload) = 'object'), + CONSTRAINT capability_snapshots_ready_is_usable + CHECK (status <> 'READY' OR (complete AND NOT truncated)) +); + +-- Retrieval mirrors the Arango collection indexes: identity lookup and the +-- status-bounded current-identity lookup. +CREATE INDEX IF NOT EXISTS capability_snapshots_identity_ix + ON capability_snapshots (project, generation, authorization_scope_digest, protocol_version, compiler_version); +CREATE INDEX IF NOT EXISTS capability_snapshots_status_ix + ON capability_snapshots (project, generation, authorization_scope_digest, status); + +CREATE TABLE IF NOT EXISTS table_shape_capabilities ( + id text PRIMARY KEY, + kind text NOT NULL + CONSTRAINT table_shape_capabilities_kind_known CHECK (kind IN ('CATALOG', 'CATEGORY_SCAN', 'RESOLUTION')), + parent_catalog_id text NOT NULL DEFAULT '', + project text NOT NULL, + explorer_id text NOT NULL, + output_id text NOT NULL, + snapshot_token text NOT NULL, + authorization_scope text NOT NULL, + source_generation text NOT NULL, + draft_version bigint NOT NULL, + draft_digest text NOT NULL, + base_document_digest text NOT NULL, + base_compilation_receipt_id text NOT NULL, + output_fingerprint text NOT NULL, + compiler_schema_digest text NOT NULL, + content_digest text NOT NULL + CONSTRAINT table_shape_capabilities_digest_shape CHECK (content_digest ~ '^sha256:[0-9a-f]{64}$'), + created_at text NOT NULL DEFAULT '', + payload jsonb NOT NULL + CONSTRAINT table_shape_capabilities_payload_object CHECK (jsonb_typeof(payload) = 'object'), + CONSTRAINT table_shape_capabilities_id_kind_prefix CHECK ( + (kind = 'CATALOG' AND id ~ '^tsc_[0-9a-f]{32}$') + OR (kind = 'CATEGORY_SCAN' AND id ~ '^tsd_[0-9a-f]{32}$') + OR (kind = 'RESOLUTION' AND id ~ '^tsr_[0-9a-f]{32}$') + ), + CONSTRAINT table_shape_capabilities_parent_bound CHECK ( + (kind = 'CATALOG' AND parent_catalog_id = '') + OR (kind <> 'CATALOG' AND parent_catalog_id <> '') + ), + CONSTRAINT table_shape_capabilities_binding_complete CHECK ( + project <> '' AND explorer_id <> '' AND output_id <> '' + AND snapshot_token <> '' AND authorization_scope <> '' AND source_generation <> '' + AND draft_version >= 1 + AND draft_digest <> '' AND base_document_digest <> '' + AND base_compilation_receipt_id <> '' AND output_fingerprint <> '' + AND compiler_schema_digest <> '' + ) +); + +-- Retrieval mirrors the Arango collection indexes: the fully bound opaque-ID +-- lookup and the parent-scoped kind scan. +CREATE INDEX IF NOT EXISTS table_shape_capabilities_binding_ix + ON table_shape_capabilities (project, explorer_id, output_id, snapshot_token, kind, id); +CREATE INDEX IF NOT EXISTS table_shape_capabilities_parent_ix + ON table_shape_capabilities (project, explorer_id, output_id, parent_catalog_id, kind); + +-- The immutability guard: artifact rows are create-once. Any UPDATE or +-- DELETE -- from any session, including one holding the table owner's +-- rights -- aborts with SQLSTATE 45501 before touching a page. The +-- adapter never issues UPDATE/DELETE against these tables; the trigger is +-- what makes that a database-enforced property rather than a convention. +CREATE OR REPLACE FUNCTION loom_capability_artifact_immutable() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION 'loom: capability artifact rows are immutable (create-once)' + USING ERRCODE = '45501'; +END; +$$; + +DROP TRIGGER IF EXISTS capability_snapshots_immutable ON capability_snapshots; +CREATE TRIGGER capability_snapshots_immutable + BEFORE UPDATE OR DELETE ON capability_snapshots + FOR EACH ROW EXECUTE FUNCTION loom_capability_artifact_immutable(); + +DROP TRIGGER IF EXISTS table_shape_capabilities_immutable ON table_shape_capabilities; +CREATE TRIGGER table_shape_capabilities_immutable + BEFORE UPDATE OR DELETE ON table_shape_capabilities + FOR EACH ROW EXECUTE FUNCTION loom_capability_artifact_immutable(); diff --git a/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-chartifact/000007_private_clickhouse_artifacts.sql b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-chartifact/000007_private_clickhouse_artifacts.sql new file mode 100644 index 000000000..484b2c832 --- /dev/null +++ b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-chartifact/000007_private_clickhouse_artifacts.sql @@ -0,0 +1,81 @@ +-- pg-private-clickhouse-artifact-catalog-v1: temporary private ClickHouse +-- artifact manifests and the cleanup-lease catalog for the PostgreSQL control +-- plane. +-- +-- Physical data tables stay in ClickHouse: this catalog persists only the +-- durable manifest (immutable identity, schema, and physical-table binding) +-- and the mutable progress/lease fields around it. It registers nothing as a +-- published output, never advances a visibility pointer, and is deliberately +-- separate from the published BundleCatalog (tests/migrations/000004) and +-- from the Explorer filesystem ZIP ArtifactStore. The version continues the +-- coordinator-ordered fixture history after the 000006 lease-checkpoint +-- reservation, and the DDL is idempotent because the pg-store-02 runner may +-- retry a migration that committed before its ledger row was written. +-- +-- Manifest identity: +-- * artifact_id is the immutable primary key; Create is insert-only, so a +-- concurrent duplicate converges on the existing row instead of +-- replacing it. The (execution, output, stage, project, generation) +-- identity columns and the physical-table binding mirror the payload and +-- are only ever written by Create; the catalog's Update, Renew, +-- ClaimCleanup, Delete, and ReleaseCleanup statements touch progress and +-- lease fields only, so identity and table fields survive every later +-- write unchanged. +-- * payload is the authoritative jsonb manifest: nanosecond timestamps and +-- the missing-versus-JSON-null member shape round-trip losslessly. The +-- scan columns (state, counts, lease, identity scope) mirror the payload +-- for constraints and ordered scans; every statement writes both from +-- the same values. No manifest member is ever SQL NULL: every column is +-- NOT NULL, so a missing value fails explicitly rather than silently +-- reading back as null. +-- * state is closed to the four domain states; counts are nonnegative; the +-- lease owner, physical table, and identity scope must be present. An +-- unsupported value is refused by the constraint, never coerced. +-- +-- Lease/cleanup scan: ListExpired pages the lease_until index in +-- (lease_until, artifact_id COLLATE "C") order -- the deterministic +-- byte-order tie-break the Arango collection gives _key -- so a bounded page +-- is stable across repeated reconciler passes, and ClaimCleanup rechecks +-- lease_until <= expiredBefore inside the same UPDATE so a concurrent +-- renewer wins and only one cleanup owner ever claims a manifest. +CREATE TABLE IF NOT EXISTS chartifact_private_artifacts ( + artifact_id text PRIMARY KEY, + physical_table text NOT NULL, + state text NOT NULL, + row_count bigint NOT NULL, + byte_count bigint NOT NULL, + lease_owner text NOT NULL, + lease_until timestamptz NOT NULL, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + execution_id text NOT NULL, + output_id text NOT NULL, + stage_id text NOT NULL, + project text NOT NULL, + dataset_generation text NOT NULL, + auth_scope_mode text NOT NULL, + scope_mode text NOT NULL, + schema_digest text NOT NULL, + scope_digest text NOT NULL, + payload jsonb NOT NULL, + CONSTRAINT chartifact_private_artifacts_id_present + CHECK (artifact_id <> '' AND artifact_id = btrim(artifact_id)), + CONSTRAINT chartifact_private_artifacts_table_present + CHECK (physical_table <> '' AND physical_table = btrim(physical_table)), + CONSTRAINT chartifact_private_artifacts_owner_present + CHECK (lease_owner <> '' AND lease_owner = btrim(lease_owner)), + CONSTRAINT chartifact_private_artifacts_state_closed + CHECK (state IN ('CREATING', 'WRITING', 'READY', 'CLEANUP_PENDING')), + CONSTRAINT chartifact_private_artifacts_counts_nonnegative + CHECK (row_count >= 0 AND byte_count >= 0), + CONSTRAINT chartifact_private_artifacts_identity_present + CHECK (execution_id <> '' AND output_id <> '' AND stage_id <> '' + AND project <> '' AND dataset_generation <> ''), + CONSTRAINT chartifact_private_artifacts_payload_object + CHECK (jsonb_typeof(payload) = 'object') +); + +-- Bounded expired-lease scan: the reconciler's page walks this index in +-- lease order; the claim path rechecks the same column inside its UPDATE. +CREATE INDEX IF NOT EXISTS chartifact_private_artifacts_lease_until + ON chartifact_private_artifacts (lease_until); diff --git a/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-interpretations/000007_interpretation_library.sql b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-interpretations/000007_interpretation_library.sql new file mode 100644 index 000000000..a7df3a9aa --- /dev/null +++ b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-interpretations/000007_interpretation_library.sql @@ -0,0 +1,157 @@ +-- pg-state-interpretation-library-v1: interpretation library heads and +-- immutable project revisions for the Explorer control plane. +-- +-- Two tables, composed after the owner history (ledger 1, owner 2, draft +-- revision 3) of this same Explorer control-plane fixture lineage: +-- * interpretation_revisions holds the immutable, content-addressed +-- facts. The ID is the domain's interpretation_<64 hex> hash over the +-- identity envelope and content_digest is sha256 over the canonical +-- content, so the same content re-derives the same row and different +-- immutable content can never share an identity. Rows are create-once: +-- a shared guard function plus a BEFORE UPDATE OR DELETE trigger refuse +-- any mutation of a revision row with SQLSTATE 45501, and the suite +-- proves the guards are not vacuous by dropping the trigger, repeating +-- the violating statement (which must then succeed), and restoring it. +-- * interpretation_libraries holds exactly one mutable authoring head per +-- (project, id): the CAS adapter statement advances head_revision_id +-- and head_digest only while the row still carries the expected head. +-- The library guard trigger refuses DELETE outright, refuses any change +-- to the identity columns (project, id, created_at), and refuses an +-- UPDATE that does not advance the head: the head pointer moves, the +-- row never disappears, nothing else is rewritten. +-- +-- Integrity is enforced here, not only in Go: +-- * Identity columns carry COLLATE "C" for exact byte order, matching the +-- owner table and Arango's byte-order comparisons. +-- * Every shape rule the domain enforces is mirrored by a named CHECK: +-- revision id, library id, content digest, parent id/digest, head +-- revision/digest. The parent pair is NULL together (root) or non-NULL +-- together (child): interpretation_revisions_parent_pair. +-- * interpretation_revisions_parent_same_library_fk makes a parent that +-- does not exist in the SAME project AND library a database refusal +-- (23503). The pair is NULL for roots, so the composite key skips +-- them. interpretation_libraries_head_same_library_fk makes a head that +-- points at a nonexistent revision, or at a revision of another +-- library, a database refusal: the browse affordance the lifecycle +-- expands can never dangle by construction. +-- * The jsonb payload is authoritative for reads (the adapter re-verifies +-- payload self-consistency and mirror agreement on every read); the +-- scalar mirrors exist for scope guards, constraints, and ordered scans. +-- jsonb preserves the missing-versus-JSON-null member shapes of the +-- canonical revision document untouched. +-- * The DDL is rerun-safe (IF NOT EXISTS, DROP TRIGGER IF EXISTS before +-- each CREATE TRIGGER): the pg-store-02 runner may retry a migration +-- that committed before its ledger row was written. This migration +-- never drops or truncates tables. + +CREATE TABLE IF NOT EXISTS interpretation_revisions ( + project text NOT NULL COLLATE "C", + id text NOT NULL COLLATE "C", + library_id text NOT NULL COLLATE "C", + parent_revision_id text COLLATE "C", + parent_digest text, + content_digest text NOT NULL, + author text NOT NULL, + explanation text NOT NULL, + created_at timestamptz NOT NULL, + payload jsonb NOT NULL, + CONSTRAINT interpretation_revisions_identity_pk + PRIMARY KEY (project, id), + CONSTRAINT interpretation_revisions_library_id_ux + UNIQUE (project, id, library_id), + CONSTRAINT interpretation_revisions_project_non_empty + CHECK (project <> ''), + CONSTRAINT interpretation_revisions_id_shape + CHECK (id ~ '^interpretation_[0-9a-f]{64}$'), + CONSTRAINT interpretation_revisions_library_id_shape + CHECK (library_id ~ '^[A-Za-z][A-Za-z0-9_.:-]{0,127}$'), + CONSTRAINT interpretation_revisions_content_digest_shape + CHECK (content_digest ~ '^sha256:[0-9a-f]{64}$'), + CONSTRAINT interpretation_revisions_parent_pair + CHECK ((parent_revision_id IS NULL) = (parent_digest IS NULL)), + CONSTRAINT interpretation_revisions_parent_revision_shape + CHECK (parent_revision_id IS NULL OR parent_revision_id ~ '^interpretation_[0-9a-f]{64}$'), + CONSTRAINT interpretation_revisions_parent_digest_shape + CHECK (parent_digest IS NULL OR parent_digest ~ '^sha256:[0-9a-f]{64}$'), + CONSTRAINT interpretation_revisions_author_non_empty + CHECK (author <> ''), + CONSTRAINT interpretation_revisions_explanation_non_empty + CHECK (explanation <> ''), + CONSTRAINT interpretation_revisions_payload_object + CHECK (jsonb_typeof(payload) = 'object'), + CONSTRAINT interpretation_revisions_parent_same_library_fk + FOREIGN KEY (project, parent_revision_id, library_id) + REFERENCES interpretation_revisions (project, id, library_id) +); + +-- Retrieval mirrors the Arango adapter's revision lookup (identity PK) plus +-- the library-scoped chain scans. +CREATE INDEX IF NOT EXISTS interpretation_revisions_library_ix + ON interpretation_revisions (project, library_id, id); + +CREATE TABLE IF NOT EXISTS interpretation_libraries ( + project text NOT NULL COLLATE "C", + id text NOT NULL COLLATE "C", + head_revision_id text NOT NULL COLLATE "C", + head_digest text NOT NULL, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT interpretation_libraries_identity_pk + PRIMARY KEY (project, id), + CONSTRAINT interpretation_libraries_project_non_empty + CHECK (project <> ''), + CONSTRAINT interpretation_libraries_id_shape + CHECK (id ~ '^[A-Za-z][A-Za-z0-9_.:-]{0,127}$'), + CONSTRAINT interpretation_libraries_head_revision_shape + CHECK (head_revision_id ~ '^interpretation_[0-9a-f]{64}$'), + CONSTRAINT interpretation_libraries_head_digest_shape + CHECK (head_digest ~ '^sha256:[0-9a-f]{64}$'), + CONSTRAINT interpretation_libraries_head_same_library_fk + FOREIGN KEY (project, head_revision_id, id) + REFERENCES interpretation_revisions (project, id, library_id) +); + +-- The immutability guard: revision rows are create-once. Any UPDATE or +-- DELETE -- from any session, including one holding the table owner's +-- rights -- aborts with SQLSTATE 45501 before touching a page. The adapter +-- never issues UPDATE/DELETE against these tables; the trigger is what +-- makes that a database-enforced property rather than a convention. +CREATE OR REPLACE FUNCTION loom_interpretation_revision_immutable() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION 'loom: interpretation revision rows are immutable (create-once)' + USING ERRCODE = '45501'; +END; +$$; + +-- The head-advance guard: a library row may only ever advance its head. +-- DELETE is refused outright; UPDATE must keep project, id, and created_at +-- byte-identical and must actually move head_revision_id. The adapter's +-- single CAS statement is the only writer that satisfies this contract. +CREATE OR REPLACE FUNCTION loom_interpretation_library_guard() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'loom: interpretation library heads are never deleted' + USING ERRCODE = '45501'; + END IF; + IF NEW.project IS DISTINCT FROM OLD.project + OR NEW.id IS DISTINCT FROM OLD.id + OR NEW.created_at IS DISTINCT FROM OLD.created_at + OR NEW.head_revision_id = OLD.head_revision_id THEN + RAISE EXCEPTION 'loom: interpretation library identity is stable and every update must advance the head' + USING ERRCODE = '45501'; + END IF; + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS interpretation_revisions_immutable ON interpretation_revisions; +CREATE TRIGGER interpretation_revisions_immutable + BEFORE UPDATE OR DELETE ON interpretation_revisions + FOR EACH ROW EXECUTE FUNCTION loom_interpretation_revision_immutable(); + +DROP TRIGGER IF EXISTS interpretation_libraries_head_guard ON interpretation_libraries; +CREATE TRIGGER interpretation_libraries_head_guard + BEFORE UPDATE OR DELETE ON interpretation_libraries + FOR EACH ROW EXECUTE FUNCTION loom_interpretation_library_guard(); diff --git a/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-leases/000006_publication_lease_checkpoints.sql b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-leases/000006_publication_lease_checkpoints.sql new file mode 100644 index 000000000..373a5596c --- /dev/null +++ b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-leases/000006_publication_lease_checkpoints.sql @@ -0,0 +1,54 @@ +-- pg-publication-leases-checkpoints-v1: owner-conditional checkpoint records +-- for the PostgreSQL publication control plane. +-- +-- Execution owners and reconcilers fence their progress checkpoints with the +-- fenced leases already created by 000004_publication_bundle_catalog.sql; +-- this migration adds only the durable record each fenced checkpoint writes. +-- It holds metadata and progress facts only: ClickHouse remains the +-- publication row store and no data rows move here. The version takes the +-- coordinator reservation 000006_* (000005 belongs to the catalog ticket), +-- and the DDL is idempotent because the runner may retry a migration that +-- committed before its ledger row was written. +-- +-- Checkpoint identity: +-- * A checkpoint belongs to one lease key and pins the exact ownership +-- grant that wrote it: owner_id and generation (the lease epoch at write +-- time) are recorded beside the payload, so a witness can reconstruct +-- which owner-episode produced every checkpoint even after takeovers and +-- same-owner reclaims. +-- * sequence is the per-key monotonic position assigned inside the same +-- guarded write that applies the checkpoint, so ordering across +-- ownership changes is gap-free and a stale token cannot pre-fill it. +-- Grant high-water: the FencedLeases keeper consults and bumps this row on +-- every committed Acquire, so a generation it has handed out is never +-- reissued even after the lease row itself is deleted (by ReleaseLease, by +-- Registry.ReleaseBundleLease, or by any other path). The row is never +-- deleted; last_generation only increases, which is what makes the keeper's +-- generation a monotonic fencing token per key across the lease row's whole +-- lifetime. Registry's own Arango-parity lease methods keep their baseline +-- epoch behaviour and do not participate in this protocol. +CREATE TABLE IF NOT EXISTS publication_lease_grants ( + key text PRIMARY KEY, + last_generation bigint NOT NULL DEFAULT 0 CHECK (last_generation >= 0) +); + +CREATE TABLE IF NOT EXISTS publication_execution_checkpoints ( + key text NOT NULL, + sequence bigint NOT NULL, + owner_id text NOT NULL, + generation bigint NOT NULL, + checkpointed_at timestamptz NOT NULL, + payload jsonb NOT NULL, + PRIMARY KEY (key, sequence), + CONSTRAINT publication_execution_checkpoints_sequence_positive + CHECK (sequence >= 1), + CONSTRAINT publication_execution_checkpoints_generation_positive + CHECK (generation >= 1), + CONSTRAINT publication_execution_checkpoints_payload_object + CHECK (jsonb_typeof(payload) = 'object') +); + +-- Ownership audit: which grant wrote which checkpoints for one key, newest +-- first, matching how a resuming writer finds its last checkpoint. +CREATE INDEX IF NOT EXISTS publication_execution_checkpoints_owner_generation + ON publication_execution_checkpoints (key, owner_id, generation, sequence); diff --git a/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-readpins/000009_publication_read_pins.sql b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-readpins/000009_publication_read_pins.sql new file mode 100644 index 000000000..80cb4891d --- /dev/null +++ b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-readpins/000009_publication_read_pins.sql @@ -0,0 +1,56 @@ +-- pg-publication-read-pins-reconcile-v1: read retention pins and cleanup +-- claims for the PostgreSQL publication control plane. +-- +-- Read pins keep the physical publication tables of an exact execution +-- reader-visible while exact export / row streaming runs; a cleanup claim is +-- the reconciler's exclusive right to remove them. The two operations race by +-- design, so they must serialize: every pin and cleanup transition for one +-- execution locks the single publication_execution_retention anchor row +-- first, which is the one lock point both paths take before any pin or claim +-- row is read or written. This migration holds control facts only: the +-- physical tables live in ClickHouse, which remains the publication row +-- store, and no data rows move here. The version continues the coordinator's +-- cross-directory reservation sequence (000004 catalog, 000006 lease +-- checkpoints, 000007 chartifact/interpretations, 000008 selections), and +-- the DDL is idempotent because the runner may retry a migration that +-- committed before its ledger row was written. +-- +-- Retention identity: +-- * Every fact is keyed by the execution id, the same identity the pin and +-- cleanup APIs address, so pins, claims, and the anchor for one +-- execution always colocate on one lock. +-- * Pins are one row per (execution, owner): a repeated acquire by the same +-- owner refreshes that row rather than duplicating a reader, which is +-- how the Arango pins array already collapses, and the row preserves +-- each owner's own expiry (owner-correlated retention). +-- * Cleanup is at most one live claim per execution: a missing claim row is +-- "no cleanup", which stays distinct from any expiry timestamp; the +-- claim stores its owning reconciler and expiry so renewal and release +-- are owner-conditional and a witness can attribute every claim episode. +-- * No column is nullable-ambiguous: expiry is always a timestamptz, so +-- "absent" is expressed only by row absence and never by a NULL stand-in. +CREATE TABLE IF NOT EXISTS publication_execution_retention ( + execution_id text PRIMARY KEY, + updated_at timestamptz NOT NULL +); + +CREATE TABLE IF NOT EXISTS publication_execution_read_pins ( + execution_id text NOT NULL REFERENCES publication_execution_retention (execution_id) ON DELETE CASCADE, + owner_id text NOT NULL CHECK (owner_id <> ''), + expires_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + PRIMARY KEY (execution_id, owner_id) +); + +-- Active-reader and expiry sweeps for one execution: the cleanup guard scans +-- live pins by (execution, expiry), and every successful arbitration GCs +-- expired pins with the same access shape. +CREATE INDEX IF NOT EXISTS publication_execution_read_pins_expiry + ON publication_execution_read_pins (execution_id, expires_at); + +CREATE TABLE IF NOT EXISTS publication_execution_cleanup_claims ( + execution_id text PRIMARY KEY REFERENCES publication_execution_retention (execution_id) ON DELETE CASCADE, + owner_id text NOT NULL CHECK (owner_id <> ''), + expires_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL +); diff --git a/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-selections/000008_selection_revisions.sql b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-selections/000008_selection_revisions.sql new file mode 100644 index 000000000..d9a38ff50 --- /dev/null +++ b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations-selections/000008_selection_revisions.sql @@ -0,0 +1,197 @@ +-- pg-state-selection-revision-owner-v1: staged selection revision +-- identity, project-scoped idempotency key, and expiring writer owner. +-- +-- Two tables form the relational counterpart of the Arango adapter's +-- SelectionsCollection and SelectionMembersCollection +-- (internal/explorer/arango/selections.go). The version continues the +-- Explorer control-plane fixture history after the 000004 capability +-- reservation (pg-state-immutable-capability-receipts). Per the +-- coordinator allocation, version 8 is the next free Explorer control-plane +-- ledger version (the Explorer ledger and the store-dir ledger are separate +-- tables in separate dirs: the store history's own 000008, +-- internal/store/postgres/migrations/000008_batch_write_ledger.sql, is a +-- different ledger over different schemas, so the shared number is no +-- collision), lands strictly after the 000007 reserved for the +-- capability-artifacts ticket, and one migration covering both tables +-- follows the 000004 two-table precedent. The DDL is +-- idempotent because the pg-store-02 runner may retry a migration that +-- committed before its ledger row was written. It never drops or truncates +-- existing state. +-- +-- Selection revision identity: +-- * id is the global selection ID -- Arango keys these documents by +-- _key == id, so the primary key is id alone, and reads that carry a +-- project (GetSelection) stay scoped by it: a foreign project asking +-- for an id that exists elsewhere gets the concealed missing answer. +-- * (project, generation, resource_type, rule_digest, scope_digest) plus +-- the idempotency key are the identity the adapter rereads after any +-- arbiter skip and compares field by field; a key never rebinds +-- different intent. Identity columns carry COLLATE "C" for exact byte +-- order, matching the member-key walk and Arango's byte-order SORT. +-- * The idempotency key is unique per project via a partial unique +-- index: an empty key is omitted (like the model's omitempty and +-- Arango's missing attribute) and never collides; any non-empty key is +-- exactly one revision per project. That index is the single-winner +-- arbiter making concurrent same-key/same-content begins converge. +-- * state and complete agree by CHECK, a COMPLETE row must carry its +-- frozen membership facts (digest, completed_at), counters are +-- non-negative, and identity fields are non-empty -- BeginSelection's +-- validation closes the same domain at the database. +-- +-- Expiring writer owner: +-- * writer_token/writer_expires_at bind the staged revision to its +-- writer; the adapter claims a foreign row only with a guarded UPDATE +-- whose WHERE re-evaluates writer_expires_at < now in PostgreSQL, so +-- an unexpired writer is unclaimable by construction and expired +-- recovery follows the current lease boundary. +-- +-- Member insert-once: +-- * member_key is the derived immutable key (selection id + "_" + the +-- shared key() hash over project/generation/resource type/id), unique +-- per selection as the primary key; the (selection_id, +-- project, generation, resource_type, id) unique keeps reference +-- identity distinct even against an adapter derivation bug. Retries +-- are idempotent through ON CONFLICT DO NOTHING on the primary key. +-- * The key-shape CHECK keeps every member key in the derived +-- "_<64 lowercase hex>" suffix form. +-- +-- Immutability is enforced here, not only in Go: a shared guard per table +-- plus BEFORE UPDATE triggers refuse identity tampering, completed-fact +-- tampering (completion is one-way), and ANY update of a staged member +-- with SQLSTATE 45501 before it can run. Member DELETE remains legal +-- because abort and bounded staging cleanup remove abandoned rows as +-- whole rows -- member content is never rewritten, only retired. The +-- suite proves the guards are not vacuous by dropping each trigger, +-- repeating the violating statement (which must then succeed), and +-- restoring the trigger. + +CREATE TABLE IF NOT EXISTS selection_revisions ( + id text NOT NULL COLLATE "C", + project text NOT NULL COLLATE "C", + generation text NOT NULL COLLATE "C", + resource_type text NOT NULL COLLATE "C", + rule_digest text NOT NULL COLLATE "C", + scope_digest text NOT NULL COLLATE "C", + idempotency_key text NOT NULL COLLATE "C", + state text NOT NULL, + writer_token text NOT NULL, + writer_expires_at timestamptz NOT NULL, + membership_digest text NOT NULL DEFAULT '', + member_count bigint NOT NULL DEFAULT 0, + member_bytes bigint NOT NULL DEFAULT 0, + complete boolean NOT NULL, + created_at timestamptz NOT NULL, + completed_at timestamptz, + rule jsonb NOT NULL + CONSTRAINT selection_revisions_rule_object CHECK (jsonb_typeof(rule) = 'object'), + source jsonb NOT NULL + CONSTRAINT selection_revisions_source_object CHECK (jsonb_typeof(source) = 'object'), + exclusions jsonb + CONSTRAINT selection_revisions_exclusions_array CHECK (exclusions IS NULL OR jsonb_typeof(exclusions) = 'array'), + CONSTRAINT selection_revisions_id_pk PRIMARY KEY (id), + CONSTRAINT selection_revisions_identity_non_empty + CHECK (id <> '' AND project <> '' AND generation <> '' AND resource_type <> ''), + CONSTRAINT selection_revisions_digests_present + CHECK (rule_digest <> '' AND scope_digest <> ''), + CONSTRAINT selection_revisions_writer_present + CHECK (writer_token <> ''), + CONSTRAINT selection_revisions_state_known + CHECK (state IN ('STAGING', 'COMPLETE')), + CONSTRAINT selection_revisions_complete_matches_state + CHECK ((state = 'COMPLETE') = complete), + CONSTRAINT selection_revisions_complete_facts + CHECK (complete = FALSE OR (membership_digest <> '' AND completed_at IS NOT NULL)), + CONSTRAINT selection_revisions_membership_non_negative + CHECK (member_count >= 0 AND member_bytes >= 0) +); + +-- The project-scoped idempotency arbiter. Partial so an omitted (empty) +-- key never participates: like Arango's filter on a missing attribute, +-- keyless revisions coexist, and any non-empty key is exactly one +-- revision per project. +CREATE UNIQUE INDEX IF NOT EXISTS selection_revisions_project_key_ux + ON selection_revisions (project, idempotency_key) + WHERE idempotency_key <> ''; + +-- Retrieval mirror for the bounded staging cleanup scan (expired STAGING +-- headers by age, then lease). +CREATE INDEX IF NOT EXISTS selection_revisions_staging_ix + ON selection_revisions (state, created_at, writer_expires_at); + +CREATE TABLE IF NOT EXISTS selection_members ( + selection_id text NOT NULL COLLATE "C", + member_key text NOT NULL COLLATE "C", + project text NOT NULL COLLATE "C", + generation text NOT NULL COLLATE "C", + resource_type text NOT NULL COLLATE "C", + id text NOT NULL COLLATE "C", + ordinal bigint NOT NULL DEFAULT 0, + CONSTRAINT selection_members_key_pk + PRIMARY KEY (selection_id, member_key), + CONSTRAINT selection_members_ref_ux + UNIQUE (selection_id, project, generation, resource_type, id), + CONSTRAINT selection_members_identity_non_empty + CHECK (selection_id <> '' AND project <> '' AND generation <> '' + AND resource_type <> '' AND id <> ''), + CONSTRAINT selection_members_key_shape + CHECK (member_key ~ '_[0-9a-f]{64}$') +); + +-- The selection-revision guard: identity columns are insert-once, and a +-- COMPLETE row's state/facts are frozen (completion is one-way). The +-- writer lease and pre-completion staging facts stay live precisely +-- because they are outside the frozen set. +CREATE OR REPLACE FUNCTION loom_selection_revision_immutable() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.id IS DISTINCT FROM OLD.id + OR NEW.project IS DISTINCT FROM OLD.project + OR NEW.generation IS DISTINCT FROM OLD.generation + OR NEW.resource_type IS DISTINCT FROM OLD.resource_type + OR NEW.rule_digest IS DISTINCT FROM OLD.rule_digest + OR NEW.scope_digest IS DISTINCT FROM OLD.scope_digest + OR NEW.idempotency_key IS DISTINCT FROM OLD.idempotency_key + OR NEW.created_at IS DISTINCT FROM OLD.created_at + OR NEW.rule IS DISTINCT FROM OLD.rule + OR NEW.source IS DISTINCT FROM OLD.source + OR NEW.exclusions IS DISTINCT FROM OLD.exclusions + THEN + RAISE EXCEPTION 'loom: selection revision identity columns are immutable (insert-once)' + USING ERRCODE = '45501'; + END IF; + IF OLD.complete AND ( + NEW.state IS DISTINCT FROM OLD.state + OR NEW.complete IS DISTINCT FROM OLD.complete + OR NEW.membership_digest IS DISTINCT FROM OLD.membership_digest + OR NEW.member_count IS DISTINCT FROM OLD.member_count + OR NEW.member_bytes IS DISTINCT FROM OLD.member_bytes + OR NEW.completed_at IS DISTINCT FROM OLD.completed_at + ) THEN + RAISE EXCEPTION 'loom: completed selection facts are immutable (completion is one-way)' + USING ERRCODE = '45501'; + END IF; + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS selection_revisions_immutable ON selection_revisions; +CREATE TRIGGER selection_revisions_immutable + BEFORE UPDATE ON selection_revisions + FOR EACH ROW EXECUTE FUNCTION loom_selection_revision_immutable(); + +-- The member guard: staged membership content is insert-once. Any UPDATE +-- aborts with 45501 before touching a page; removal is lifecycle-only +-- (abort and bounded staging cleanup delete whole rows, they never +-- rewrite one). +CREATE OR REPLACE FUNCTION loom_selection_member_immutable() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION 'loom: staged selection member rows are immutable (insert-once; removal is lifecycle-only)' + USING ERRCODE = '45501'; +END; +$$; + +DROP TRIGGER IF EXISTS selection_members_immutable ON selection_members; +CREATE TRIGGER selection_members_immutable + BEFORE UPDATE ON selection_members + FOR EACH ROW EXECUTE FUNCTION loom_selection_member_immutable(); diff --git a/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations/000004_publication_bundle_catalog.sql b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations/000004_publication_bundle_catalog.sql new file mode 100644 index 000000000..8dcf10810 --- /dev/null +++ b/internal/backends/postgres/internal/store/testdata/ab05_pre_refactor/tests/migrations/000004_publication_bundle_catalog.sql @@ -0,0 +1,81 @@ +-- pg-publication-bundle-catalog-v1: PostgreSQL-side publication bundle catalog. +-- +-- These tables persist execution/output metadata, quality/state metadata, +-- the exact logical pointer registry, and fenced bundle leases for the +-- PostgreSQL source/control backend. They hold metadata only: ClickHouse +-- remains the publication row store and no data rows move here. The version +-- deliberately follows the 000002_*/000003_* migrations owned by the +-- concurrent source-store tickets so the histories interleave without +-- collisions, and the DDL is idempotent because the runner may retry a +-- migration that committed before its ledger row was written. +-- +-- Exact logical identity: +-- * An execution's searchable identity is split into explicit scope columns +-- (project, dataset_generation, recipe_name, translation_version) so an +-- exact lookup is a scope equality on all of them -- never a name-only +-- latest fallback. +-- * The full BundleExecution JSON body lives in payload. Reads decode the +-- payload and canonicalize legacy shapes in memory; nothing rewrites it, +-- so historical records keep their stored values (jsonb normalizes only +-- the lexical JSON form, never field values). +-- * Pointer names are arbitrary domain strings (they may embed NUL from the +-- length-prefixed PointerName encoding, which PostgreSQL text and jsonb +-- both reject), so the row key is the hex SHA-256 of the name -- the same +-- discipline as the Arango pointer documents -- and the exact bytes are +-- round-tripped losslessly through bytea. +CREATE TABLE IF NOT EXISTS publication_bundle_executions ( + id text PRIMARY KEY, + execution_key text NOT NULL, + project text NOT NULL, + dataset_generation text NOT NULL, + recipe_name text NOT NULL, + translation_version text NOT NULL, + state text NOT NULL, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + ready_at timestamptz, + published_at timestamptz, + payload jsonb NOT NULL, + CONSTRAINT publication_bundle_executions_payload_object + CHECK (jsonb_typeof(payload) = 'object') +); + +-- Exact publication lookup scope: project + generation + selector. The +-- published/ready/updated ordering mirrors the Arango exact query; id breaks +-- ties so the "latest" answer is deterministic. +CREATE INDEX IF NOT EXISTS publication_bundle_executions_exact_scope + ON publication_bundle_executions ( + project, dataset_generation, recipe_name, translation_version, + state, published_at DESC NULLS LAST, ready_at DESC NULLS LAST, + updated_at DESC, id + ); + +-- FindExecutionByKey: newest created_at first, id tie-break for determinism. +CREATE INDEX IF NOT EXISTS publication_bundle_executions_key_created + ON publication_bundle_executions (execution_key, created_at DESC, id); + +-- Reconciliation scans and stable keyset pages: (state, updated_at, id). +CREATE INDEX IF NOT EXISTS publication_bundle_executions_state_scan + ON publication_bundle_executions (state, updated_at, id); + +CREATE TABLE IF NOT EXISTS publication_bundle_pointers ( + pointer_key text PRIMARY KEY, + -- Exact pointer-name bytes. PostgreSQL text and jsonb both reject NUL, + -- and a PointerName may legally embed one, so the name lives in bytea. + name_bytes bytea NOT NULL, + execution_id text NOT NULL, + updated_at timestamptz NOT NULL +); + +CREATE TABLE IF NOT EXISTS publication_bundle_leases ( + key text PRIMARY KEY, + owner_id text NOT NULL, + expires_at timestamptz NOT NULL, + acquired_at timestamptz NOT NULL, + -- epoch increments on every grant (including a same-owner reclaim after + -- expiry), correlating repeated ownership of one lease key so a witness + -- can distinguish grants. + epoch bigint NOT NULL DEFAULT 1, + CONSTRAINT publication_bundle_leases_epoch_positive + CHECK (epoch >= 1) +); diff --git a/internal/backends/postgres/internal/store/verify_classify_test.go b/internal/backends/postgres/internal/store/verify_classify_test.go new file mode 100644 index 000000000..64597ae24 --- /dev/null +++ b/internal/backends/postgres/internal/store/verify_classify_test.go @@ -0,0 +1,94 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "testing" + + "github.com/jackc/pgx/v5/pgconn" +) + +func TestVerifyStatementErrorClassification(t *testing.T) { + for _, tt := range []struct { + name string + err error + wantConnect bool + }{ + {"closed connection", net.ErrClosed, true}, + {"protocol truncation", io.ErrUnexpectedEOF, true}, + {"deadline exceeded wrapped by readLedger", fmt.Errorf("postgres: read migration ledger: %w", context.DeadlineExceeded), true}, + {"refused connection", fmt.Errorf("postgres: read migration ledger: %w", errors.New("dial tcp 127.0.0.1:5432: connect: connection refused")), true}, + {"server-answered undefined column", &pgconn.PgError{Code: "42703", Message: "column \"checksum\" does not exist"}, false}, + {"server-answered permission denied wrapped by readLedger", fmt.Errorf("postgres: read migration ledger: %w", &pgconn.PgError{Code: "42501", Message: "permission denied for table schema_migrations"}), false}, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + got := verifyStatementError("verify ledger", tt.err) + if isConnect := errors.Is(got, ErrConnect); isConnect != tt.wantConnect { + t.Fatalf("errors.Is(err, ErrConnect) = %v, want %v for %v", isConnect, tt.wantConnect, tt.err) + } + if tt.wantConnect { + var ce *ConnectError + if !errors.As(got, &ce) || ce.Op != "verify ledger" || !errors.Is(ce, tt.err) { + t.Fatalf("connectivity fault must join the pool's *ConnectError chain with the cause inspectable, got %#v", got) + } + } else { + if got != tt.err { + t.Fatalf("server-answered fault must pass through verbatim, got %#v want %#v", got, tt.err) + } + var pgErr *pgconn.PgError + if !errors.As(got, &pgErr) { + t.Fatalf("server-answered fault must stay a *pgconn.PgError, got %#v", got) + } + } + }) + } +} + +func TestVerifyLedgerQueryServerFaultIsNotConnectivity(t *testing.T) { + svc := newMigrationService(t) + client := svc.openClient(t) + if _, err := svc.iso.Exec(svc.ctx, + `CREATE TABLE "`+svc.schema+`".schema_migrations (version bigint)`); err != nil { + svc.fail("create wrong-shaped ledger table", err) + } + runner := svc.newRunner(t, client, nil) + err := runner.Verify(svc.ctx) + if err == nil { + t.Fatal("Verify over a wrong-shaped ledger table must fail") + } + if errors.Is(err, ErrConnect) { + t.Fatalf("a server-answered ledger query fault must not enter the ErrConnect chain (live connection is not connectivity_down): %s", svc.scrub(err)) + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + t.Fatalf("Verify must pass a server-answered fault through inspectable as *pgconn.PgError, got %#v", err) + } + if pgErr.Code != "42703" { + t.Fatalf("expected undefined-column 42703 from the wrong-shaped ledger, got %s: %s", pgErr.Code, svc.scrub(pgErr)) + } + if err := client.Ping(svc.ctx); err != nil { + svc.fail("ping after server-answered fault", err) + } +} + +func TestVerifyNamedArmsUnchangedByStatementClassification(t *testing.T) { + svc := newMigrationService(t) + client := svc.openClient(t) + missing := svc.schema + "_absent" + runner, err := NewMigrationRunner(client, missing, nil) + if err != nil { + svc.fail("construct runner for absent schema", err) + } + if err := runner.Verify(svc.ctx); !errors.Is(err, ErrInvalidSchemaName) { + t.Fatalf("absent schema must classify as ErrInvalidSchemaName, got %#v", err) + } + migrations := []Migration{{Version: 1, Name: "pgverify01_probe", SQL: "CREATE TABLE t(x int)"}} + pendingRunner := svc.newRunner(t, client, migrations) + if err := pendingRunner.Verify(svc.ctx); !errors.Is(err, ErrMigrationPending) { + t.Fatalf("empty ledger with a pending migration must classify as ErrMigrationPending, got %#v", err) + } +} diff --git a/internal/backends/postgres/internal/store/verify_readonly_test.go b/internal/backends/postgres/internal/store/verify_readonly_test.go new file mode 100644 index 000000000..c41328f78 --- /dev/null +++ b/internal/backends/postgres/internal/store/verify_readonly_test.go @@ -0,0 +1,83 @@ +package postgres + +import ( + "go/ast" + "go/parser" + "go/token" + "strings" + "testing" +) + +func TestVerifyIsReadOnlyAndLockFree(t *testing.T) { + set := token.NewFileSet() + parsed, err := parser.ParseFile(set, "migrate.go", nil, 0) + if err != nil { + t.Fatalf("parse migrate.go: %v", err) + } + allowed := map[string]bool{ + "Acquire": true, "Release": true, "QueryRow": true, "Query": true, "Scan": true, + "readLedger": true, "validateHistory": true, "pending": true, + "verifyStatementError": true, "pgErrorFrom": true, + "Errorf": true, "Sprintf": true, "len": true, + "Close": true, "Next": true, "Err": true, "append": true, "sort": true, + "Slice": true, "As": true, "make": true, "new": true, + "int64": true, "int": true, "string": true, + } + forbidden := map[string]bool{ + "Exec": true, "ExecFunc": true, "Begin": true, "BeginFunc": true, + "Prepare": true, "CopyFrom": true, "SendBatch": true, "Hijack": true, + "pg_advisory_lock": true, "pg_advisory_xact_lock": true, + } + callees := map[string]bool{ + "Verify": true, "readLedger": true, "validateHistory": true, "pending": true, + } + seen := map[string]bool{} + var bad, suspicious []string + ast.Inspect(parsed, func(n ast.Node) bool { + fn, ok := n.(*ast.FuncDecl) + if !ok || fn.Recv == nil || !callees[fn.Name.Name] { + return true + } + seen[fn.Name.Name] = true + ast.Inspect(fn.Body, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.CallExpr: + var name string + switch f := x.Fun.(type) { + case *ast.SelectorExpr: + name = f.Sel.Name + case *ast.Ident: + name = f.Name + } + if forbidden[name] { + bad = append(bad, fn.Name.Name+"."+name) + } else if !allowed[name] && name != fn.Name.Name { + suspicious = append(suspicious, fn.Name.Name+"."+name) + } + case *ast.BasicLit: + if x.Kind != token.STRING { + return true + } + lit := strings.ToUpper(x.Value) + for _, tell := range []string{"INSERT", "UPDATE", "DELETE", "CREATE ", "ALTER ", "DROP ", "TRUNCATE", "TRUNCATE", "BEGIN", "LOCK", "PG_ADVISORY", "GRANT", "COMMENT ON"} { + if strings.Contains(lit, tell) { + bad = append(bad, fn.Name.Name+" literal "+x.Value) + } + } + } + return true + }) + return true + }) + for name := range callees { + if !seen[name] { + t.Fatalf("callee %s not found in migrate.go; the witness must be re-read, not widened", name) + } + } + if len(bad) > 0 { + t.Fatalf("Verify path touches a write/lock surface: %v", bad) + } + if len(suspicious) > 0 { + t.Fatalf("Verify path calls a method outside the read-only allowlist: %v", suspicious) + } +} diff --git a/internal/backends/postgres/open.go b/internal/backends/postgres/open.go new file mode 100644 index 000000000..a5cf4ba14 --- /dev/null +++ b/internal/backends/postgres/open.go @@ -0,0 +1,63 @@ +package postgres + +import ( + "context" + "fmt" + + "github.com/calypr/loom/internal/backends/postgres/control" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + + rows "github.com/calypr/loom/internal/backends/postgres/internal/rows" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" +) + +type Runtime struct { + *control.Runtime + SourceRows dataframeexecution.SourceRows + PlanJSON dataframeexecution.PostgreSQLPlanJSONExecutor +} + +type Querier = rows.Querier + +func NewPlanJSONExecutor(querier Querier) (dataframeexecution.PostgreSQLPlanJSONExecutor, error) { + if querier == nil { + return nil, fmt.Errorf("%w: nil row querier", control.ErrInvalidConfig) + } + return newPostgreSQLPlanJSONExecutor(querier) +} + +func NewSourceRows(querier Querier) (dataframeexecution.SourceRows, error) { + if querier == nil { + return nil, fmt.Errorf("%w: nil row querier", control.ErrInvalidConfig) + } + executor, err := rows.NewExecutor(querier) + if err != nil { + return nil, err + } + return executor.SourceRows, nil +} + +var _ rows.Querier = (*pgstore.Client)(nil) + +func Open(ctx context.Context, cfg control.Config) (*Runtime, error) { + inner, err := control.Open(ctx, cfg) + if err != nil { + return nil, err + } + querier := inner.PoolClient() + if querier == nil { + _ = inner.Close() + return nil, fmt.Errorf("%w: control runtime did not expose its pool as the row querier", control.ErrBootstrap) + } + executor, err := rows.NewExecutor(querier) + if err != nil { + _ = inner.Close() + return nil, fmt.Errorf("%w: assemble row executor: %w", control.ErrBootstrap, err) + } + planJSON, err := newPostgreSQLPlanJSONExecutor(querier) + if err != nil { + _ = inner.Close() + return nil, fmt.Errorf("%w: assemble plan executor: %w", control.ErrBootstrap, err) + } + return &Runtime{Runtime: inner, SourceRows: executor.SourceRows, PlanJSON: planJSON}, nil +} diff --git a/internal/backends/postgres/open_live_test.go b/internal/backends/postgres/open_live_test.go new file mode 100644 index 000000000..f63819680 --- /dev/null +++ b/internal/backends/postgres/open_live_test.go @@ -0,0 +1,394 @@ +package postgres_test + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/backends/postgres" + "github.com/calypr/loom/internal/backends/postgres/control" + rows "github.com/calypr/loom/internal/backends/postgres/internal/rows" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/execution" +) + +const ab06Namespace = "verify_ab06_" + +func ab06LiveURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 live Open assembly half; the runner lane makes a skip a failure, never acceptance") + } + if u := raw; !strings.HasPrefix(u, "postgres://") && !strings.HasPrefix(u, "postgresql://") { + t.Fatalf("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL, got scheme-prefixed %q", u[:min(len(u), 16)]) + } + return raw +} + +func ab06NewNonce(t *testing.T) string { + t.Helper() + buf := make([]byte, 6) + if _, err := rand.Read(buf); err != nil { + t.Fatalf("generate fixture namespace nonce: %v", err) + } + return hex.EncodeToString(buf) +} + +func ab06RawPool(t *testing.T, ctx context.Context, dsn string) *control.Client { + t.Helper() + pool, err := control.OpenPool(ctx, control.ClientConfig{ + DSN: dsn, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + t.Fatalf("open control pool for fixture administration: %v", err) + } + t.Cleanup(pool.Close) + return pool +} + +func ab06SeriesSchema(nonce string, series control.Series) string { + return ab06Namespace + nonce + "_" + string(series) +} + +func ab06AllSeriesSchemas(nonce string) control.SeriesSchemas { + schemas := make(control.SeriesSchemas, len(control.AllSeries())) + for _, series := range control.AllSeries() { + schemas[series] = ab06SeriesSchema(nonce, series) + } + return schemas +} + +func ab06Exec(t *testing.T, pool *control.Client, sql string) { + t.Helper() + rs, err := pool.Query(context.Background(), sql) + if err != nil { + t.Fatalf("fixture exec %q: %v", sql, err) + } + for rs.Next() { + } + err = rs.Err() + rs.Close() + if err != nil { + t.Fatalf("fixture exec %q: %v", sql, err) + } +} + +func factoryOpenLiveAssembly(t *testing.T) { + t.Run("live_open_assembly", func(t *testing.T) { + dsn := ab06LiveURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + admin := ab06RawPool(t, ctx, dsn) + + nonce := ab06NewNonce(t) + schemas := ab06AllSeriesSchemas(nonce) + for _, series := range control.AllSeries() { + ab06Exec(t, admin, "CREATE SCHEMA "+ab06SeriesSchema(nonce, series)) + } + t.Cleanup(func() { + for _, series := range control.AllSeries() { + ab06Exec(t, admin, "DROP SCHEMA IF EXISTS "+ab06SeriesSchema(nonce, series)+" CASCADE") + } + }) + + appName := "loom-pg-ab06live" + nonce + runtime, err := postgres.Open(ctx, control.Config{ + DSN: dsn, + ApplicationName: appName, + Schemas: schemas, + Setup: control.SetupMigrate, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 10 * time.Second, + }) + if err != nil { + t.Fatalf("postgres.Open over the real server: %v", err) + } + defer runtime.Close() + + if runtime.SourceRows == nil { + t.Fatal("Open returned a Runtime with a nil SourceRows port") + } + if runtime.PlanJSON == nil { + t.Fatal("Open returned a Runtime with a nil PlanJSON port") + } + + if err := runtime.Health(ctx); err != nil { + t.Fatalf("Health on the live runtime: %v", err) + } + + var serverVersion string + if err := runtime.QueryRow(ctx, "SHOW server_version").Scan(&serverVersion); err != nil { + t.Fatalf("read server_version through the public query surface: %v", err) + } + major := serverVersion + if i := strings.IndexAny(major, " \t"); i >= 0 { + major = major[:i] + } + if i := strings.IndexByte(major, '.'); i >= 0 { + major = major[:i] + } + if major != "18" { + t.Fatalf("readiness gate admitted server major %q, want 18 (server_version %q)", major, serverVersion) + } + + migrateResult, err := runtime.MigrateSchema(ctx) + if err != nil { + t.Fatalf("public MigrateSchema witness after Open(SetupMigrate): %v", err) + } + if migrateResult.LatestVersion <= 0 || len(migrateResult.Applied) == 0 { + t.Fatalf("source-series bootstrap not witnessed: %+v", migrateResult) + } + for _, series := range control.AllSeries() { + var ledgerRows int64 + if err := runtime.QueryRow(ctx, fmt.Sprintf( + "SELECT count(*) FROM %s.schema_migrations", ab06SeriesSchema(nonce, series), + )).Scan(&ledgerRows); err != nil { + t.Fatalf("read ledger from series %q schema %s: %v", series, ab06SeriesSchema(nonce, series), err) + } + if ledgerRows == 0 { + t.Fatalf("series %q bootstrap not witnessed: empty ledger in %s", series, ab06SeriesSchema(nonce, series)) + } + } + + var tagged int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_stat_activity WHERE application_name = $1", appName, + ).Scan(&tagged); err != nil { + t.Fatalf("query pg_stat_activity for the own-prefix tag: %v", err) + } + if tagged < 1 { + t.Fatalf("no live backend carried the own-prefix application_name %q (pool self-tagging regressed)", appName) + } + + streamNonce := ab06NewNonce(t) + streamSchemas := ab06AllSeriesSchemas(streamNonce) + for _, series := range control.AllSeries() { + ab06Exec(t, admin, "CREATE SCHEMA "+ab06SeriesSchema(streamNonce, series)) + } + t.Cleanup(func() { + for _, series := range control.AllSeries() { + ab06Exec(t, admin, "DROP SCHEMA IF EXISTS "+ab06SeriesSchema(streamNonce, series)+" CASCADE") + } + }) + streamRuntime, err := postgres.Open(ctx, control.Config{ + DSN: dsn, + ApplicationName: "loom-pg-ab06stream" + streamNonce, + Schemas: streamSchemas, + Setup: control.SetupNone, + MaxConns: 2, + MinConns: 1, + AcquireTimeout: 250 * time.Millisecond, + }) + if err != nil { + t.Fatalf("open streaming runtime (AcquireTimeout 250ms): %v", err) + } + defer streamRuntime.Close() + + const streamRows = 30 + streamCommand, err := command.NewPostgreSQLCommand( + fmt.Sprintf("SELECT generate_series(1, %d)::bigint AS n, pg_sleep(0.02) AS drip", streamRows), nil) + if err != nil { + t.Fatalf("build server-paced stream command: %v", err) + } + seen := 0 + if err := streamRuntime.SourceRows(ctx, execution.RowsRequest{ + Command: streamCommand, + DeclaredColumns: []string{"n", "drip"}, + }, func(row map[string]any) error { + seen++ + return nil + }); err != nil { + t.Fatalf("server-paced stream ran longer than AcquireTimeout and failed (context bound to acquisition, not to the caller): %v", err) + } + if seen != streamRows { + t.Fatalf("streamed %d rows, want %d", seen, streamRows) + } + slowRow := streamRuntime.QueryRow(ctx, "SELECT pg_sleep(0.35), 7::bigint") + var dripped string + var seven int64 + if err := slowRow.Scan(&dripped, &seven); err != nil { + t.Fatalf("QueryRow Scan after a server-side delay past the acquire timeout failed: %v", err) + } + if seven != 7 { + t.Fatalf("slow QueryRow value = %d, want 7", seven) + } + + literalCommand, err := command.NewPostgreSQLCommand( + "SELECT $1::bigint AS n, 'loom'::text AS tag", + []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterBigint, Value: int64(42)}}, + ) + if err != nil { + t.Fatalf("build PG-tagged literal command: %v", err) + } + var streamed []map[string]any + if err := runtime.SourceRows(ctx, execution.RowsRequest{ + Command: literalCommand, + DeclaredColumns: []string{"n", "tag"}, + }, func(row map[string]any) error { + streamed = append(streamed, row) + return nil + }); err != nil { + t.Fatalf("SourceRows over the live server: %v", err) + } + if len(streamed) != 1 { + t.Fatalf("SourceRows streamed %d rows, want exactly 1", len(streamed)) + } + if n, ok := streamed[0]["n"].(int64); !ok || n != 42 { + t.Fatalf("typed literal round trip: row[n] = %#v, want int64(42)", streamed[0]["n"]) + } + if tag, ok := streamed[0]["tag"].(string); !ok || tag != "loom" { + t.Fatalf("typed literal round trip: row[tag] = %#v, want \"loom\"", streamed[0]["tag"]) + } + + if err := runtime.SourceRows(ctx, execution.RowsRequest{ + Command: literalCommand, + DeclaredColumns: []string{"n"}, + }, func(map[string]any) error { return nil }); !errors.Is(err, rows.ErrColumnContractMismatch) { + t.Fatalf("declared-column undercount: err = %v, want %v", err, rows.ErrColumnContractMismatch) + } + + aqlCommand, err := command.NewAQLCommand("FOR d IN c RETURN d", nil) + if err != nil { + t.Fatalf("build AQL-tagged command: %v", err) + } + if err := runtime.SourceRows(ctx, execution.RowsRequest{ + Command: aqlCommand, + DeclaredColumns: []string{"d"}, + }, func(map[string]any) error { return nil }); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("AQL-tagged command into the PostgreSQL port: err = %v, want %v", err, command.ErrCompiledCommandBackendMismatch) + } + + explainCommand, err := command.NewPostgreSQLCommand( + "EXPLAIN (FORMAT JSON) SELECT $1::bigint AS n", + []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterBigint, Value: int64(1)}}, + ) + if err != nil { + t.Fatalf("build EXPLAIN-retagged command: %v", err) + } + plan, err := runtime.PlanJSON(ctx, explainCommand) + if err != nil { + t.Fatalf("PlanJSON over the live server: %v", err) + } + var planDoc []map[string]any + if err := json.Unmarshal(plan, &planDoc); err != nil { + t.Fatalf("PlanJSON did not decode as a JSON array of plan objects: %v (raw %s)", err, plan) + } + if len(planDoc) == 0 { + t.Fatal("PlanJSON returned an empty plan array") + } + + absentNonce := ab06NewNonce(t) + missingSchema := ab06SeriesSchema(absentNonce, control.AllSeries()[0]) + failedTag := "loom-pg-ab06fail" + absentNonce + if _, err := postgres.Open(ctx, control.Config{ + DSN: dsn, + ApplicationName: failedTag, + Schemas: ab06AllSeriesSchemas(absentNonce), + Setup: control.SetupMigrate, + MaxConns: 2, + MinConns: 1, + AcquireTimeout: 10 * time.Second, + }); !errors.Is(err, control.ErrBootstrap) { + t.Fatalf("Open over an absent caller-owned schema: err = %v, want an %v-wrapped refusal", err, control.ErrBootstrap) + } + var created bool + if err := admin.QueryRow(ctx, "SELECT to_regnamespace($1) IS NOT NULL", missingSchema).Scan(&created); err != nil { + t.Fatalf("check absent schema stayed absent: %v", err) + } + if created { + t.Fatalf("setup policy created the unassigned schema %q; the migrate runner must refuse and leave schema lifecycle to the caller", missingSchema) + } + ab06WaitBackendsZero(t, admin, failedTag, "failed Open") + + if err := runtime.Close(); err != nil { + t.Fatalf("first Close: %v", err) + } + if err := runtime.Close(); err != nil { + t.Fatalf("second Close must stay safe, got: %v", err) + } + if err := runtime.Health(ctx); !errors.Is(err, control.ErrRuntimeClosed) { + t.Fatalf("Health after Close: err = %v, want %v", err, control.ErrRuntimeClosed) + } + if _, err := runtime.Query(ctx, "SELECT 1"); !errors.Is(err, control.ErrRuntimeClosed) { + t.Fatalf("Query after Close: err = %v, want %v", err, control.ErrRuntimeClosed) + } + if err := runtime.SourceRows(ctx, execution.RowsRequest{ + Command: literalCommand, + DeclaredColumns: []string{"n", "tag"}, + }, func(map[string]any) error { return nil }); !errors.Is(err, rows.ErrQuery) { + t.Fatalf("SourceRows after Close: err = %v, want an %v-classified refusal", err, rows.ErrQuery) + } + + ab06WaitBackendsZero(t, admin, appName, "Close of the main runtime") + + var remaining int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM information_schema.schemata WHERE schema_name LIKE $1", ab06Namespace+"%", + ).Scan(&remaining); err != nil { + t.Fatalf("count owned-namespace schemas: %v", err) + } + if want := 2 * len(control.AllSeries()); remaining != want { + t.Fatalf("owned namespace holds %d schemas during the run, want exactly %d (this subtest's two fixtures)", remaining, want) + } + }) +} + +func TestZZZeroLeftoverAB06LiveOpenFixtures(t *testing.T) { + dsn := ab06LiveURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + pool := ab06RawPool(t, ctx, dsn) + var leftovers []string + rs, err := pool.Query(ctx, + "SELECT schema_name FROM information_schema.schemata WHERE schema_name LIKE $1 ORDER BY schema_name", + ab06Namespace+"%") + if err != nil { + t.Fatalf("list owned-namespace schemas: %v", err) + } + defer rs.Close() + for rs.Next() { + var name string + if err := rs.Scan(&name); err != nil { + t.Fatalf("scan leftover schema name: %v", err) + } + leftovers = append(leftovers, name) + } + if err := rs.Err(); err != nil { + t.Fatalf("read leftover schema rows: %v", err) + } + if len(leftovers) != 0 { + t.Fatalf("verify_ab06_* schemas survived the suite: %v", leftovers) + } +} + +func ab06WaitBackendsZero(t *testing.T, admin *control.Client, tag, what string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + var leaked int64 + for { + if err := admin.QueryRow(context.Background(), + "SELECT count(*) FROM pg_stat_activity WHERE application_name = $1", tag, + ).Scan(&leaked); err != nil { + t.Fatalf("query pg_stat_activity for tag %q: %v", tag, err) + } + if leaked == 0 { + return + } + if time.Now().After(deadline) { + t.Fatalf("%s leaked %d live backends under tag %q after 10s", what, leaked, tag) + } + time.Sleep(100 * time.Millisecond) + } +} diff --git a/internal/backends/postgres/ownership_contract_test.go b/internal/backends/postgres/ownership_contract_test.go new file mode 100644 index 000000000..cc99254d9 --- /dev/null +++ b/internal/backends/postgres/ownership_contract_test.go @@ -0,0 +1,316 @@ +package postgres_test + +import ( + "context" + "errors" + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/backends/postgres/control" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" + "github.com/calypr/loom/internal/server" +) + +type driverRegisterEntry struct { + prefix string + claimedLive bool +} + +var productionDriverImportAllowlist = []driverRegisterEntry{ + {prefix: "internal/backends/", claimedLive: true}, +} + +func repoRoot(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed; cannot locate the repository root") + } + dir := filepath.Dir(file) + root := dir + for i := 0; i < 3; i++ { + root = filepath.Dir(root) + } + if fi, err := os.Stat(filepath.Join(root, "go.mod")); err != nil || fi.IsDir() { + t.Fatalf("resolved root %q has no go.mod: %v", root, err) + } + return root +} + +func fileImports(t *testing.T, path string) []string { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, path, nil, parser.ImportsOnly) + if err != nil { + t.Fatalf("parse %s: %v", path, err) + } + imports := make([]string, 0, len(file.Imports)) + for _, imported := range file.Imports { + imports = append(imports, strings.Trim(imported.Path.Value, `"`)) + } + return imports +} + +func walkProductionGo(t *testing.T, root string, fn func(relPath, absPath string)) { + t.Helper() + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + t.Fatalf("walk %s: %v", path, err) + } + if entry.IsDir() { + if entry.Name() == ".git" || entry.Name() == "node_modules" { + return fs.SkipDir + } + return nil + } + name := entry.Name() + if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + return nil + } + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + t.Fatalf("rel %s: %v", path, relErr) + } + fn(filepath.ToSlash(rel), path) + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } +} + +func allowedByDriverRegister(relPath string) bool { + for _, entry := range productionDriverImportAllowlist { + if strings.HasPrefix(relPath, entry.prefix) { + return true + } + } + return false +} + +func TestPostgresFactoryOwnershipContract(t *testing.T) { + factoryOpenLiveAssembly(t) + + root := repoRoot(t) + + t.Run("public_factory_surface_is_driver_free", func(t *testing.T) { + factoryDir := filepath.Join(root, "internal", "backends", "postgres") + entries, err := os.ReadDir(factoryDir) + if err != nil { + t.Fatalf("read factory dir: %v", err) + } + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + for _, importPath := range fileImports(t, filepath.Join(factoryDir, name)) { + if strings.HasPrefix(importPath, "github.com/jackc/") { + t.Fatalf("public factory file %s imports database driver %s", name, importPath) + } + } + } + }) + + t.Run("production_driver_imports_confined_to_register", func(t *testing.T) { + var offenders []string + walkProductionGo(t, root, func(relPath, absPath string) { + for _, importPath := range fileImports(t, absPath) { + if strings.HasPrefix(importPath, "github.com/jackc/pgx") && !allowedByDriverRegister(relPath) { + offenders = append(offenders, relPath+" imports "+importPath) + } + } + }) + if len(offenders) != 0 { + t.Fatalf("production files import the PostgreSQL driver outside the register: %v", offenders) + } + + t.Run("register_is_not_stale", func(t *testing.T) { + live := map[string]bool{} + walkProductionGo(t, root, func(relPath, absPath string) { + for _, importPath := range fileImports(t, absPath) { + if strings.HasPrefix(importPath, "github.com/jackc/pgx") { + live[relPath] = true + } + } + }) + for _, entry := range productionDriverImportAllowlist { + if !entry.claimedLive { + continue + } + found := false + for rel := range live { + if strings.HasPrefix(rel, entry.prefix) { + found = true + break + } + } + if !found { + t.Fatalf("register entry %q matches no live driver importer; update the register in the commit that changed driver reach", entry.prefix) + } + } + }) + + t.Run("named_exception_shrank_to_zero_in_the_move_commit", func(t *testing.T) { + var touched []string + walkProductionGo(t, filepath.Join(root, "internal", "ingest"), func(relPath, absPath string) { + for _, importPath := range fileImports(t, absPath) { + if strings.HasPrefix(importPath, "github.com/jackc/pgx") { + touched = append(touched, relPath) + } + } + }) + sort.Strings(touched) + if len(touched) != 0 { + t.Fatalf("ingest driver-touch set = %v, want [] (the named exception was retired in the move commit; driver classes arrive only through the factory)", touched) + } + }) + }) + + t.Run("production_files_outside_backend_never_import_backend_privates", func(t *testing.T) { + var offenders []string + walkProductionGo(t, root, func(relPath, absPath string) { + if strings.HasPrefix(relPath, "internal/backends/postgres/") { + return + } + for _, importPath := range fileImports(t, absPath) { + if strings.HasPrefix(importPath, "github.com/calypr/loom/internal/backends/postgres/internal/") { + offenders = append(offenders, relPath+" imports "+importPath) + } + } + }) + if len(offenders) != 0 { + t.Fatalf("callers reach backend privates: %v", offenders) + } + }) + + t.Run("config_validation_matrix_with_alive_controls", func(t *testing.T) { + base := func() control.Config { + return control.Config{ + DSN: "postgres://loom:secret@localhost:5432/loom", + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 30 * time.Second, + Setup: control.SetupMigrate, + Schemas: control.SeriesSchemas{control.SeriesSource: "loom_source"}, + } + } + if err := base().Validate(); err != nil { + t.Fatalf("baseline valid config rejected: %v", err) + } + cases := []struct { + name string + defect func(*control.Config) + sentinel error + }{ + {"blank_dsn", func(c *control.Config) { c.DSN = " " }, control.ErrInvalidConfig}, + {"zero_max_conns", func(c *control.Config) { c.MaxConns = 0 }, control.ErrInvalidConfig}, + {"negative_min_conns", func(c *control.Config) { c.MinConns = -1 }, control.ErrInvalidConfig}, + {"min_above_max", func(c *control.Config) { c.MinConns = 8 }, control.ErrInvalidConfig}, + {"nonpositive_acquire_timeout", func(c *control.Config) { c.AcquireTimeout = 0 }, control.ErrInvalidConfig}, + {"foreign_application_name_prefix", func(c *control.Config) { c.ApplicationName = "somebody-elses-tag" }, control.ErrInvalidConfig}, + {"unsafe_application_name_characters", func(c *control.Config) { c.ApplicationName = "loom-pg-host=evil" }, control.ErrInvalidConfig}, + {"missing_setup_policy", func(c *control.Config) { c.Setup = "" }, control.ErrInvalidConfig}, + {"unknown_setup_policy", func(c *control.Config) { c.Setup = "yolo" }, control.ErrInvalidConfig}, + {"blank_assigned_schema", func(c *control.Config) { c.Schemas = control.SeriesSchemas{control.SeriesSource: " "} }, control.ErrSeriesAssignment}, + {"unknown_series", func(c *control.Config) { c.Schemas = control.SeriesSchemas{"not_a_series": "ok"} }, control.ErrSeriesAssignment}, + {"unsafe_schema_name", func(c *control.Config) { + c.Schemas = control.SeriesSchemas{control.SeriesSource: "Bad Name; DROP"} + }, control.ErrInvalidSchemaName}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg := base() + tc.defect(&cfg) + err := cfg.Validate() + if !errors.Is(err, tc.sentinel) { + t.Fatalf("defect %s: err = %v, want errors.Is(_, %v)", tc.name, err, tc.sentinel) + } + fixed := base() + switch tc.name { + case "min_above_max": + fixed.MinConns = 1 + } + if err := fixed.Validate(); err != nil { + t.Fatalf("alive control rejected a sound config: %v", err) + } + }) + } + }) + + t.Run("series_set_equals_migration_manifest", func(t *testing.T) { + factory := map[string]bool{} + for _, series := range control.AllSeries() { + factory[string(series)] = true + } + manifest := map[string]bool{} + for _, entry := range pgstore.Manifest() { + manifest[string(entry.Series)] = true + } + for series := range manifest { + if !factory[series] { + t.Fatalf("manifest series %q missing from the factory series set", series) + } + } + for series := range factory { + if !manifest[series] { + t.Fatalf("factory series %q absent from the arch-backend-05 manifest", series) + } + } + }) + + t.Run("server_postgres_gate_stays_closed_named_and_dial_free", func(t *testing.T) { + cfg := server.Config{} + cfg.Server.Backend = "postgres" + cfg.Server.PostgreSQL.Host = "192.0.2.1" + cfg.Server.PostgreSQL.Port = 60543 + cfg.Server.PostgreSQL.Database = "loom_gate_witness" + cfg.Server.PostgreSQL.User = "loom_gate_witness" + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + started := time.Now() + err := server.RunWithConfig(ctx, cfg) + elapsed := time.Since(started) + if !errors.Is(err, server.ErrPostgreSQLRuntimeNotWired) { + t.Fatalf("postgres startup err = %v, want errors.Is(_, ErrPostgreSQLRuntimeNotWired)", err) + } + if elapsed > 1500*time.Millisecond { + t.Fatalf("gate took %v: must refuse before dialing the configured host", elapsed) + } + }) + + t.Run("sabotage_named_victims_bleed", func(t *testing.T) { + dir := t.TempDir() + victims := map[string]string{ + "victim_driver_import.go": "package victim\n\nimport \"github.com/jackc/pgx/v5/pgxpool\"\n\nvar _ = pgxpool.Config{}\n", + } + for name, body := range victims { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + var offenders []string + walkProductionGo(t, dir, func(relPath, absPath string) { + for _, importPath := range fileImports(t, absPath) { + if strings.HasPrefix(importPath, "github.com/jackc/pgx") { + offenders = append(offenders, relPath) + } + } + }) + joined := strings.Join(offenders, "\n") + for name := range victims { + if !strings.Contains(joined, name) { + t.Fatalf("confinement scanner did not name victim %s in:\n%s", name, joined) + } + } + }) +} diff --git a/internal/backends/postgres/plan_executor.go b/internal/backends/postgres/plan_executor.go new file mode 100644 index 000000000..51f08619c --- /dev/null +++ b/internal/backends/postgres/plan_executor.go @@ -0,0 +1,41 @@ +package postgres + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + + rows "github.com/calypr/loom/internal/backends/postgres/internal/rows" +) + +func newPostgreSQLPlanJSONExecutor(querier rows.Querier) (dataframeexecution.PostgreSQLPlanJSONExecutor, error) { + executor, err := rows.NewExecutor(querier) + if err != nil { + return nil, err + } + return func(ctx context.Context, command command.CompiledCommand) (json.RawMessage, error) { + var plans []string + if err := executor.SourceRows(ctx, dataframeexecution.RowsRequest{Command: command, DeclaredColumns: []string{"QUERY PLAN"}}, func(row map[string]any) error { + switch plan := row["QUERY PLAN"].(type) { + case string: + plans = append(plans, plan) + case []byte: + plans = append(plans, string(plan)) + case json.RawMessage: + plans = append(plans, string(plan)) + default: + return fmt.Errorf("PostgreSQL EXPLAIN returned plan text as %T", row["QUERY PLAN"]) + } + return nil + }); err != nil { + return nil, err + } + if len(plans) != 1 { + return nil, fmt.Errorf("PostgreSQL EXPLAIN must return exactly one plan row, got %d", len(plans)) + } + return json.RawMessage(plans[0]), nil + }, nil +} diff --git a/internal/backends/postgres/plan_executor_test.go b/internal/backends/postgres/plan_executor_test.go new file mode 100644 index 000000000..14250cdb6 --- /dev/null +++ b/internal/backends/postgres/plan_executor_test.go @@ -0,0 +1,41 @@ +package postgres + +import ( + "context" + "errors" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/jackc/pgx/v5" +) + +type countingQuerier struct{ called *bool } + +func (q countingQuerier) Query(context.Context, string, ...any) (pgx.Rows, error) { + *q.called = true + return nil, errors.New("unexpected querier call") +} + +func TestPostgreSQLPlanJSONExecutorConstructionFailsClosed(t *testing.T) { + if _, err := newPostgreSQLPlanJSONExecutor(nil); err == nil { + t.Fatal("nil querier must fail closed") + } + aqlCommand, err := command.NewAQLCommand("FOR d IN c RETURN d", nil) + if err != nil { + t.Fatal(err) + } + querierCalled := false + runner, err := newPostgreSQLPlanJSONExecutor(countingQuerier{called: &querierCalled}) + if err != nil { + t.Fatal(err) + } + if _, err := runner(context.Background(), aqlCommand); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("AQL-tagged command into the PostgreSQL runner: %v", err) + } + if querierCalled { + t.Fatal("cross-tag command reached the querier") + } + if _, err := NewPlanJSONExecutor(nil); err == nil { + t.Fatal("nil querier must fail closed at the public seam") + } +} diff --git a/internal/backends/postgres/sentinel_identity_test.go b/internal/backends/postgres/sentinel_identity_test.go new file mode 100644 index 000000000..5f7139b2b --- /dev/null +++ b/internal/backends/postgres/sentinel_identity_test.go @@ -0,0 +1,39 @@ +package postgres + +import ( + "errors" + "testing" + + "github.com/calypr/loom/internal/backends/postgres/control" + datasetstore "github.com/calypr/loom/internal/backends/postgres/internal/dataset" + pgstore "github.com/calypr/loom/internal/backends/postgres/internal/store" +) + +func TestControlSentinelsMatchProbeEmittedIdentities(t *testing.T) { + for _, row := range []struct { + name string + controlSide error + probeSide error + }{ + {"control.ErrRuntimeClosed (control-native; Readiness->Health emits it)", control.ErrRuntimeClosed, control.ErrRuntimeClosed}, + {"control.ErrConnect (ConnectError.Unwrap carries it; Acquire/Ping/verify faults)", control.ErrConnect, pgstore.ErrConnect}, + {"control.ErrUnsupportedVersion (Ping's PostgreSQL-18 gate)", control.ErrUnsupportedVersion, pgstore.ErrUnsupportedVersion}, + {"control.ErrInvalidSchemaName (runner name + absent-schema arms; F1 regression: dataset's copy aliased the store identity)", control.ErrInvalidSchemaName, pgstore.ErrInvalidSchemaName}, + {"control.ErrInvalidSchemaName via the dataset config emitter", control.ErrInvalidSchemaName, datasetstore.ErrInvalidSchemaName}, + {"control.ErrMigrationPending", control.ErrMigrationPending, pgstore.ErrMigrationPending}, + {"control.ErrMigrationChecksumMismatch", control.ErrMigrationChecksumMismatch, pgstore.ErrMigrationChecksumMismatch}, + {"control.ErrFutureSchemaVersion", control.ErrFutureSchemaVersion, pgstore.ErrFutureSchemaVersion}, + {"control.ErrMigrationMissing", control.ErrMigrationMissing, pgstore.ErrMigrationMissing}, + {"control.ErrMigrationOutOfOrder", control.ErrMigrationOutOfOrder, pgstore.ErrMigrationOutOfOrder}, + } { + row := row + t.Run(row.name, func(t *testing.T) { + if !errors.Is(row.probeSide, row.controlSide) { + t.Fatalf("probe-emitted %v does not match the control sentinel: classifier token unreachable (F1 class)", row.name) + } + if !errors.Is(row.controlSide, row.probeSide) { + t.Fatalf("control sentinel %v does not match the probe-emitted identity: alias drift", row.name) + } + }) + } +} diff --git a/internal/catalog/arango/evidence.go b/internal/catalog/arango/evidence.go deleted file mode 100644 index ff683f722..000000000 --- a/internal/catalog/arango/evidence.go +++ /dev/null @@ -1,114 +0,0 @@ -package arango - -import ( - "context" - "sort" - - "github.com/calypr/loom/internal/catalog" - fhirschema "github.com/calypr/loom/internal/fhir/schema" -) - -// Read* aliases make the capability boundary explicit while retaining the -// Discover* naming used by the pre-existing catalog reader. -func (s *Store) ReadResourceInventory(ctx context.Context, opts catalog.ResourceInventoryOptions) (catalog.ResourceInventoryResult, error) { - return s.DiscoverResourceInventory(ctx, opts) -} -func (s *Store) ReadRelationshipObservations(ctx context.Context, opts catalog.RelationshipObservationOptions) (catalog.RelationshipObservationResult, error) { - return s.DiscoverRelationshipObservations(ctx, opts) -} -func (s *Store) ReadFieldEnrichment(ctx context.Context, opts catalog.FieldEnrichmentOptions) (catalog.FieldEnrichmentResult, error) { - return s.DiscoverFieldEnrichment(ctx, opts) -} - -type relationshipObservationKey struct { - project, generation, auth string - storageFrom, label, storageTo string - builderFrom, builderTo, builderDirection string -} - -func evidenceResourceTypes(requested []string) ([]string, []catalog.EvidenceDiagnostic) { - if len(requested) == 0 { - return fhirschema.ResourceTypes(), nil - } - seen := make(map[string]struct{}, len(requested)) - diagnostics := make([]catalog.EvidenceDiagnostic, 0) - for _, requestedType := range requested { - canonical, ok := fhirschema.ConcreteResourceType(requestedType) - if !ok { - diagnostics = append(diagnostics, catalog.EvidenceDiagnostic{Code: "INVALID_RESOURCE_TYPE", Message: "requested resource type is not a concrete generated FHIR resource", ResourceType: requestedType}) - continue - } - seen[canonical] = struct{}{} - } - result := make([]string, 0, len(seen)) - for resourceType := range seen { - result = append(result, resourceType) - } - sort.Strings(result) - return result, diagnostics -} - -func inventoryObservationLess(a, b catalog.ResourceInventoryObservation) bool { - for _, pair := range [][2]string{{a.Project, b.Project}, {a.DatasetGeneration, b.DatasetGeneration}, {a.AuthResourcePath, b.AuthResourcePath}, {a.ResourceType, b.ResourceType}} { - if pair[0] != pair[1] { - return pair[0] < pair[1] - } - } - return a.DocumentCount < b.DocumentCount -} - -func relationshipLess(a, b catalog.RelationshipObservation) bool { - for _, pair := range [][2]string{{a.Project, b.Project}, {a.DatasetGeneration, b.DatasetGeneration}, {a.AuthResourcePath, b.AuthResourcePath}, {a.FromType, b.FromType}, {a.Label, b.Label}, {a.ToType, b.ToType}, {a.StorageFromType, b.StorageFromType}, {a.StorageToType, b.StorageToType}} { - if pair[0] != pair[1] { - return pair[0] < pair[1] - } - } - return a.EdgeCount < b.EdgeCount -} - -func fieldLess(a, b catalog.FieldEnrichmentObservation) bool { - for _, pair := range [][2]string{{a.Project, b.Project}, {a.DatasetGeneration, b.DatasetGeneration}, {a.AuthResourcePath, b.AuthResourcePath}, {a.ResourceType, b.ResourceType}, {a.Path, b.Path}, {a.Kind, b.Kind}} { - if pair[0] != pair[1] { - return pair[0] < pair[1] - } - } - return a.DocCount < b.DocCount -} - -func relationshipObservation(project, datasetGeneration, auth, storageFrom, label, storageTo string, count int64) (catalog.RelationshipObservation, bool, catalog.EvidenceDiagnostic) { - diagnostic := catalog.EvidenceDiagnostic{StorageFromType: storageFrom, Label: label, StorageToType: storageTo, Count: count} - if count <= 0 { - diagnostic.Code = "INVALID_RELATIONSHIP_COUNT" - diagnostic.Message = "relationship evidence count must be positive" - return catalog.RelationshipObservation{}, false, diagnostic - } - // Stored fhir_edge is always _from -> _to. Builder's normal projection is - // the inverse generated traversal, which is physically INBOUND from the - // stored target. Keep a direct generated traversal as a valid fallback for - // labels whose generated schema only exposes the storage orientation. - if _, found, err := fhirschema.ResolveCompilerTraversal(storageTo, label, storageFrom); found && err == nil { - return catalog.RelationshipObservation{Project: project, DatasetGeneration: datasetGeneration, AuthResourcePath: auth, FromType: storageTo, Label: label, ToType: storageFrom, EdgeCount: count, StorageFromType: storageFrom, StorageToType: storageTo, StorageDirection: "OUTBOUND", BuilderFromType: storageTo, BuilderToType: storageFrom, BuilderDirection: "INBOUND"}, true, diagnostic - } - if _, found, err := fhirschema.ResolveCompilerTraversal(storageFrom, label, storageTo); found && err == nil { - return catalog.RelationshipObservation{Project: project, DatasetGeneration: datasetGeneration, AuthResourcePath: auth, FromType: storageFrom, Label: label, ToType: storageTo, EdgeCount: count, StorageFromType: storageFrom, StorageToType: storageTo, StorageDirection: "OUTBOUND", BuilderFromType: storageFrom, BuilderToType: storageTo, BuilderDirection: "OUTBOUND"}, true, diagnostic - } - diagnostic.Code = "UNSUPPORTED_GENERATED_TRAVERSAL" - diagnostic.Message = "relationship endpoints and label are not a supported generated traversal" - return catalog.RelationshipObservation{}, false, diagnostic -} - -func evidenceStatus(available, complete, truncated bool, count int) catalog.EvidenceStatus { - if !available { - return catalog.EvidenceUnavailable - } - if truncated { - return catalog.EvidenceTruncated - } - if !complete { - return catalog.EvidenceIncomplete - } - if count == 0 { - return catalog.EvidenceEmpty - } - return catalog.EvidenceAvailable -} diff --git a/internal/catalog/auth_scope.go b/internal/catalog/auth_scope.go index 9512cface..cbd8efac9 100644 --- a/internal/catalog/auth_scope.go +++ b/internal/catalog/auth_scope.go @@ -1,16 +1,9 @@ package catalog -// ExplicitAuthResourcePathsUnrestricted returns an independent pointer for -// catalog options. The pointer distinguishes a deliberately restricted empty -// scope from legacy callers that provide only an empty path slice. func ExplicitAuthResourcePathsUnrestricted(unrestricted bool) *bool { return &unrestricted } -// EffectiveAuthResourcePathsUnrestricted resolves the compatibility rule for -// catalog callers. New request paths should always pass an explicit value; -// direct legacy callers retain their historical empty-means-unrestricted -// behavior until migrated. func EffectiveAuthResourcePathsUnrestricted(paths []string, explicit *bool) bool { if explicit != nil { return *explicit diff --git a/internal/catalog/backend_contract.go b/internal/catalog/backend_contract.go new file mode 100644 index 000000000..4efc0f74f --- /dev/null +++ b/internal/catalog/backend_contract.go @@ -0,0 +1,26 @@ +package catalog + +import "context" + +type FieldProfileScope struct { + Project string + DatasetGeneration string + AuthResourcePath string + ResourceType string +} + +type FieldProfileWriter interface { + WriteFieldProfile(ctx context.Context, scope FieldProfileScope, docs []FieldCatalogDocument, batchSize int) error +} + +type InventoryRebuilder interface { + RebuildResourceInventory(ctx context.Context, opts ResourceInventoryRebuildOptions) (ResourceInventoryRebuildSummary, error) +} + +type RelationshipDiscoverer interface { + DiscoverRelationshipObservations(ctx context.Context, opts RelationshipObservationOptions) (RelationshipObservationResult, error) +} + +type SemanticContributor interface { + WriteSemanticInventoryContributions(ctx context.Context, contributions []SemanticInventoryContribution, batchSize int) error +} diff --git a/internal/catalog/capability_evidence.go b/internal/catalog/capability_evidence.go index fdb9a1c29..eb4baf1d2 100644 --- a/internal/catalog/capability_evidence.go +++ b/internal/catalog/capability_evidence.go @@ -126,9 +126,6 @@ func cloneFieldEnrichment(in []FieldEnrichmentObservation) []FieldEnrichmentObse return out } -// ReadCapabilityEvidence keeps the three evidence sources independent while -// providing a convenient aggregate for snapshot compilers. It returns the -// first dependency error but still returns all successfully read evidence. func ReadCapabilityEvidence(ctx context.Context, reader CapabilityEvidenceReader, opts CapabilityEvidenceOptions) (CapabilityEvidence, error) { var result CapabilityEvidence var firstErr error diff --git a/internal/catalog/generation.go b/internal/catalog/generation.go index 5aaccc281..3bb9ac7ac 100644 --- a/internal/catalog/generation.go +++ b/internal/catalog/generation.go @@ -2,16 +2,10 @@ package catalog import "strings" -// NormalizeDatasetGeneration applies the optional opaque-generation convention -// at every catalog boundary. It prevents a whitespace-padded generation from -// using a cache key that differs from the AQL bind value. func NormalizeDatasetGeneration(generation string) string { return strings.TrimSpace(generation) } -// HasDatasetGeneration reports whether a request selected a concrete -// generation. Whitespace-only inputs follow the optional-field convention and -// select the legacy null namespace. func HasDatasetGeneration(generation string) bool { return NormalizeDatasetGeneration(generation) != "" } diff --git a/internal/catalog/helpers.go b/internal/catalog/helpers.go index 6e98614bf..a56b0dcaa 100644 --- a/internal/catalog/helpers.go +++ b/internal/catalog/helpers.go @@ -22,16 +22,6 @@ func fieldCatalogKey(project, authResourcePath, resourceType, path string) strin const generationFieldCatalogKeyPrefix = "gfc_" -// fieldCatalogKeyForGeneration returns the persistent catalog document key for -// one profiler identity. The empty-generation branch deliberately calls the -// pre-generation key function unchanged: existing catalogs and callers that -// have not selected a generation must retain their exact legacy key layout. -// -// A non-empty generation uses a SHA-256 digest of every identity component. -// In particular, it must not append a sanitized generation string to the -// legacy key: sanitization is many-to-one (for example, a slash and a space -// both become an underscore) and would allow one immutable generation to -// overwrite another catalog row. func fieldCatalogKeyForGeneration(project, datasetGeneration, authResourcePath, resourceType, path string) string { datasetGeneration = NormalizeDatasetGeneration(datasetGeneration) if datasetGeneration == "" { diff --git a/internal/catalog/limits.go b/internal/catalog/limits.go index 1151a75ef..c7552f1a5 100644 --- a/internal/catalog/limits.go +++ b/internal/catalog/limits.go @@ -1,9 +1,5 @@ package catalog -// ProfileLimits bounds the write-side memory retained while profiling one -// resource file. Catalog discovery is useful only while it remains a bounded -// summary; retaining every value from a high-cardinality field turns an ingest -// into an in-memory copy of the source dataset. type ProfileLimits struct { MaxFields int MaxDistinctValuesPerField int @@ -11,10 +7,7 @@ type ProfileLimits struct { MaxPivotColumnsPerField int MaxExtensionValuesPerField int MaxShapePlans int - // MaxRetainedBytes is an aggregate budget shared by shape plans and all - // profilers using one cache. It bounds the in-memory catalog summary for a - // load even when many workers observe distinct values concurrently. - MaxRetainedBytes int + MaxRetainedBytes int } const ( @@ -27,8 +20,6 @@ const ( DefaultMaxRetainedBytes = 64 << 20 ) -// DefaultProfileLimits are deliberately finite. Callers that need a -// different operational envelope must make that choice explicitly. func DefaultProfileLimits() ProfileLimits { return ProfileLimits{ MaxFields: DefaultMaxProfileFields, @@ -67,8 +58,6 @@ func (l ProfileLimits) normalized() ProfileLimits { return l } -// NormalizeProfileLimits applies the finite defaults to an operational load -// configuration. func NormalizeProfileLimits(l ProfileLimits) ProfileLimits { return l.normalized() } diff --git a/internal/catalog/rebuild.go b/internal/catalog/rebuild.go index ec71b673d..535cb00dc 100644 --- a/internal/catalog/rebuild.go +++ b/internal/catalog/rebuild.go @@ -1,8 +1,5 @@ package catalog -// RelationshipRebuildOptions describes the explicit repair path. Normal -// discovery never uses this scan; operators call it after an old dataset was -// loaded before the relationship catalog existed or after a repair. type RelationshipRebuildOptions struct { Project string DatasetGeneration string @@ -22,9 +19,6 @@ type RelationshipRebuildSummary struct { Seconds float64 `json:"seconds"` } -// RelationshipAuditOptions identifies the immutable edge namespace to audit. -// The audit is read-only and is intentionally separate from catalog rebuild so -// operators can inspect malformed historical data before choosing a repair. type RelationshipAuditOptions struct { Project string DatasetGeneration string diff --git a/internal/catalog/relationship_evidence.go b/internal/catalog/relationship_evidence.go new file mode 100644 index 000000000..304573541 --- /dev/null +++ b/internal/catalog/relationship_evidence.go @@ -0,0 +1,96 @@ +package catalog + +import ( + fhirschema "github.com/calypr/loom/internal/fhir/schema" +) + +type RelationshipEvidenceKey struct { + Project string + DatasetGeneration string + AuthResourcePath string + StorageFromType string + Label string + StorageToType string + BuilderFromType string + BuilderToType string + BuilderDirection string +} + +func RelationshipEvidenceKeyFor(observation RelationshipObservation) RelationshipEvidenceKey { + return RelationshipEvidenceKey{ + Project: observation.Project, + DatasetGeneration: observation.DatasetGeneration, + AuthResourcePath: observation.AuthResourcePath, + StorageFromType: observation.StorageFromType, + Label: observation.Label, + StorageToType: observation.StorageToType, + BuilderFromType: observation.BuilderFromType, + BuilderToType: observation.BuilderToType, + BuilderDirection: observation.BuilderDirection, + } +} + +func RelationshipObservationFromKey(key RelationshipEvidenceKey, count int64) RelationshipObservation { + return RelationshipObservation{ + Project: key.Project, + DatasetGeneration: key.DatasetGeneration, + AuthResourcePath: key.AuthResourcePath, + FromType: key.BuilderFromType, + Label: key.Label, + ToType: key.BuilderToType, + EdgeCount: count, + StorageFromType: key.StorageFromType, + StorageToType: key.StorageToType, + StorageDirection: "OUTBOUND", + BuilderFromType: key.BuilderFromType, + BuilderToType: key.BuilderToType, + BuilderDirection: key.BuilderDirection, + } +} + +func ProjectRelationshipObservation(project, datasetGeneration, auth, storageFrom, label, storageTo string, count int64) (RelationshipObservation, bool, EvidenceDiagnostic) { + return projectRelationshipObservation(fhirschema.ResolveCompilerTraversal, project, datasetGeneration, auth, storageFrom, label, storageTo, count) +} + +func projectRelationshipObservation(resolve func(fromType, edgeLabel, toType string) (fhirschema.CompilerTraversal, bool, error), project, datasetGeneration, auth, storageFrom, label, storageTo string, count int64) (RelationshipObservation, bool, EvidenceDiagnostic) { + diagnostic := EvidenceDiagnostic{StorageFromType: storageFrom, Label: label, StorageToType: storageTo, Count: count} + if count <= 0 { + diagnostic.Code = "INVALID_RELATIONSHIP_COUNT" + diagnostic.Message = "relationship evidence count must be positive" + return RelationshipObservation{}, false, diagnostic + } + if _, found, err := resolve(storageTo, label, storageFrom); found && err == nil { + return RelationshipObservation{Project: project, DatasetGeneration: datasetGeneration, AuthResourcePath: auth, FromType: storageTo, Label: label, ToType: storageFrom, EdgeCount: count, StorageFromType: storageFrom, StorageToType: storageTo, StorageDirection: "OUTBOUND", BuilderFromType: storageTo, BuilderToType: storageFrom, BuilderDirection: "INBOUND"}, true, diagnostic + } + if _, found, err := resolve(storageFrom, label, storageTo); found && err == nil { + return RelationshipObservation{Project: project, DatasetGeneration: datasetGeneration, AuthResourcePath: auth, FromType: storageFrom, Label: label, ToType: storageTo, EdgeCount: count, StorageFromType: storageFrom, StorageToType: storageTo, StorageDirection: "OUTBOUND", BuilderFromType: storageFrom, BuilderToType: storageTo, BuilderDirection: "OUTBOUND"}, true, diagnostic + } + diagnostic.Code = "UNSUPPORTED_GENERATED_TRAVERSAL" + diagnostic.Message = "relationship endpoints and label are not a supported generated traversal" + return RelationshipObservation{}, false, diagnostic +} + +func RelationshipEvidenceLess(a, b RelationshipObservation) bool { + for _, pair := range [][2]string{{a.Project, b.Project}, {a.DatasetGeneration, b.DatasetGeneration}, {a.AuthResourcePath, b.AuthResourcePath}, {a.FromType, b.FromType}, {a.Label, b.Label}, {a.ToType, b.ToType}, {a.StorageFromType, b.StorageFromType}, {a.StorageToType, b.StorageToType}} { + if pair[0] != pair[1] { + return pair[0] < pair[1] + } + } + return a.EdgeCount < b.EdgeCount +} + +func EvidenceStatusFor(available, complete, truncated bool, count int) EvidenceStatus { + if !available { + return EvidenceUnavailable + } + if truncated { + return EvidenceTruncated + } + if !complete { + return EvidenceIncomplete + } + if count == 0 { + return EvidenceEmpty + } + return EvidenceAvailable +} diff --git a/internal/catalog/relationship_evidence_census_test.go b/internal/catalog/relationship_evidence_census_test.go new file mode 100644 index 000000000..8c06e0f12 --- /dev/null +++ b/internal/catalog/relationship_evidence_census_test.go @@ -0,0 +1,42 @@ +package catalog + +import ( + "fmt" + "sort" + "testing" + + generatedschema "github.com/calypr/loom/generated/fhirschema" + fhirschema "github.com/calypr/loom/internal/fhir/schema" +) + +func TestRelationshipRegistryHasNoDirectOnlyTraversals(t *testing.T) { + var directOnly []string + census := 0 + for key, spec := range generatedschema.Traversals { + from, label, to := spec.FromType, spec.EdgeLabel, spec.ToType + if triple(from, label, to) != key { + t.Errorf("generated traversal record under key %q declares inconsistent fields %q", key, triple(from, label, to)) + continue + } + _, directFound, directErr := fhirschema.ResolveCompilerTraversal(from, label, to) + if !directFound || directErr != nil { + continue + } + census++ + if _, inverseFound, inverseErr := fhirschema.ResolveCompilerTraversal(to, label, from); !inverseFound || inverseErr != nil { + directOnly = append(directOnly, triple(from, label, to)) + } + } + if directOnly != nil { + sort.Strings(directOnly) + t.Fatalf("generated relationship registry contains %d direct-only traversal(s) that would silently exercise the relationship-evidence direct-fallback branch: %v — handle deliberately (registry fix, or a live witness for the fallback branch reviewed with the arch-backend-07 owner); the stub variant in TestRelationshipEvidenceProjection is not a substitute for real registry coverage", len(directOnly), directOnly) + } + if census == 0 { + t.Fatal("census selected zero resolvable generated traversals — the registry or the resolver is broken; a vacuous pass is not acceptance") + } + t.Logf("census: %d resolvable generated traversals, all inverse-reachable", census) +} + +func triple(from, label, to string) string { + return fmt.Sprintf("%s|%s|%s", from, label, to) +} diff --git a/internal/catalog/relationship_evidence_test.go b/internal/catalog/relationship_evidence_test.go new file mode 100644 index 000000000..d7de96ed3 --- /dev/null +++ b/internal/catalog/relationship_evidence_test.go @@ -0,0 +1,209 @@ +package catalog + +import ( + "sort" + "testing" + + fhirschema "github.com/calypr/loom/internal/fhir/schema" +) + +func traversalStub(found map[string]bool, errs map[string]error) func(fromType, edgeLabel, toType string) (fhirschema.CompilerTraversal, bool, error) { + return func(fromType, edgeLabel, toType string) (fhirschema.CompilerTraversal, bool, error) { + key := fromType + "|" + edgeLabel + "|" + toType + if !found[key] { + return fhirschema.CompilerTraversal{}, false, nil + } + return fhirschema.CompilerTraversal{FromType: fromType, EdgeLabel: edgeLabel, ToType: toType}, true, errs[key] + } +} + +func TestRelationshipEvidenceProjection(t *testing.T) { + t.Run("inverse projection is the preferred Builder view", func(t *testing.T) { + observation, ok, diagnostic := ProjectRelationshipObservation("proj", "g1", "clinic/a", "Condition", "subject_Patient", "Patient", 3) + if !ok { + t.Fatalf("registry-known triple rejected: %+v", diagnostic) + } + want := RelationshipObservation{ + Project: "proj", DatasetGeneration: "g1", AuthResourcePath: "clinic/a", + FromType: "Patient", Label: "subject_Patient", ToType: "Condition", EdgeCount: 3, + StorageFromType: "Condition", StorageToType: "Patient", StorageDirection: "OUTBOUND", + BuilderFromType: "Patient", BuilderToType: "Condition", BuilderDirection: "INBOUND", + } + if observation != want { + t.Fatalf("observation = %+v, want %+v", observation, want) + } + wantDiagnostic := EvidenceDiagnostic{StorageFromType: "Condition", Label: "subject_Patient", StorageToType: "Patient", Count: 3} + if diagnostic != wantDiagnostic { + t.Fatalf("carried diagnostic = %+v, want %+v", diagnostic, wantDiagnostic) + } + }) + + t.Run("direct-only traversal is a valid fallback projection", func(t *testing.T) { + found := map[string]bool{"Condition|subject_Patient|Patient": true} + observation, ok, diagnostic := projectRelationshipObservation(traversalStub(found, nil), "proj", "g1", "clinic/a", "Condition", "subject_Patient", "Patient", 2) + if !ok { + t.Fatalf("direct-only triple rejected: %+v", diagnostic) + } + want := RelationshipObservation{ + Project: "proj", DatasetGeneration: "g1", AuthResourcePath: "clinic/a", + FromType: "Condition", Label: "subject_Patient", ToType: "Patient", EdgeCount: 2, + StorageFromType: "Condition", StorageToType: "Patient", StorageDirection: "OUTBOUND", + BuilderFromType: "Condition", BuilderToType: "Patient", BuilderDirection: "OUTBOUND", + } + if observation != want { + t.Fatalf("observation = %+v, want %+v", observation, want) + } + }) + + t.Run("inverse wins when both orientations resolve", func(t *testing.T) { + found := map[string]bool{ + "Condition|subject_Patient|Patient": true, + "Patient|subject_Patient|Condition": true, + } + observation, ok, _ := projectRelationshipObservation(traversalStub(found, nil), "proj", "g1", "clinic/a", "Condition", "subject_Patient", "Patient", 1) + if !ok { + t.Fatal("both orientations resolve, projection must succeed") + } + if observation.BuilderDirection != "INBOUND" || observation.BuilderFromType != "Patient" || observation.BuilderToType != "Condition" { + t.Fatalf("inverse-first selection broken: %+v", observation) + } + }) + + t.Run("unsupported traversal is diagnosed with exact bytes", func(t *testing.T) { + _, ok, diagnostic := ProjectRelationshipObservation("proj", "g1", "clinic/a", "Patient", "not_a_generated_label", "Practitioner", 7) + if ok { + t.Fatal("unsupported traversal accepted") + } + want := EvidenceDiagnostic{ + Code: "UNSUPPORTED_GENERATED_TRAVERSAL", + Message: "relationship endpoints and label are not a supported generated traversal", + StorageFromType: "Patient", Label: "not_a_generated_label", StorageToType: "Practitioner", Count: 7, + } + if diagnostic != want { + t.Fatalf("diagnostic = %+v, want %+v", diagnostic, want) + } + found := map[string]bool{"Condition|subject_Patient|Patient": true} + errs := map[string]error{"Condition|subject_Patient|Patient": errStub} + _, ok, diagnostic = projectRelationshipObservation(traversalStub(found, errs), "proj", "g1", "clinic/a", "Condition", "subject_Patient", "Patient", 4) + if ok || diagnostic.Code != "UNSUPPORTED_GENERATED_TRAVERSAL" { + t.Fatalf("erroring resolver: ok=%v diagnostic=%+v", ok, diagnostic) + } + }) + + t.Run("non-positive count is diagnosed with exact bytes", func(t *testing.T) { + for _, count := range []int64{0, -3} { + observation, ok, diagnostic := ProjectRelationshipObservation("proj", "g1", "clinic/a", "Condition", "subject_Patient", "Patient", count) + if ok { + t.Fatalf("count %d accepted", count) + } + if observation != (RelationshipObservation{}) { + t.Fatalf("rejected count %d returned an observation: %+v", count, observation) + } + want := EvidenceDiagnostic{ + Code: "INVALID_RELATIONSHIP_COUNT", + Message: "relationship evidence count must be positive", + StorageFromType: "Condition", Label: "subject_Patient", StorageToType: "Patient", Count: count, + } + if diagnostic != want { + t.Fatalf("count %d diagnostic = %+v, want %+v", count, diagnostic, want) + } + } + }) + + t.Run("merge ordering and digest bytes match the pre-consolidation adapters", func(t *testing.T) { + type aggregate struct { + project, generation, auth, storageFrom, label, storageTo string + count int64 + } + aggregates := []aggregate{ + {"p2", "g1", "clinic/b", "Condition", "subject_Patient", "Patient", 3}, + {"p1", "g1", "clinic/a", "BodyStructure", "patient", "Patient", 1}, + {"p1", "g1", "clinic/a", "Condition", "subject_Patient", "Patient", 5}, + {"p1", "g1", "clinic/a", "Condition", "subject_Patient", "Patient", 2}, + {"p1", "g2", "clinic/a", "Condition", "subject_Patient", "Patient", 3}, + } + counts := make(map[RelationshipEvidenceKey]int64) + for _, a := range aggregates { + observation, ok, diagnostic := ProjectRelationshipObservation(a.project, a.generation, a.auth, a.storageFrom, a.label, a.storageTo, a.count) + if !ok { + t.Fatalf("aggregate %+v rejected: %+v", a, diagnostic) + } + counts[RelationshipEvidenceKeyFor(observation)] += observation.EdgeCount + } + values := make([]RelationshipObservation, 0, len(counts)) + for key, count := range counts { + values = append(values, RelationshipObservationFromKey(key, count)) + } + if len(values) != 4 { + t.Fatalf("merged %d groups, want 4: %+v", len(values), values) + } + sort.Slice(values, func(i, j int) bool { return RelationshipEvidenceLess(values[i], values[j]) }) + for i := 1; i < len(values); i++ { + if !RelationshipEvidenceLess(values[i-1], values[i]) { + t.Fatalf("order not strict at %d: %+v before %+v", i, values[i-1], values[i]) + } + } + wantOrder := []RelationshipObservation{ + {Project: "p1", DatasetGeneration: "g1", AuthResourcePath: "clinic/a", FromType: "Patient", Label: "patient", ToType: "BodyStructure", EdgeCount: 1, StorageFromType: "BodyStructure", StorageToType: "Patient", StorageDirection: "OUTBOUND", BuilderFromType: "Patient", BuilderToType: "BodyStructure", BuilderDirection: "INBOUND"}, + {Project: "p1", DatasetGeneration: "g1", AuthResourcePath: "clinic/a", FromType: "Patient", Label: "subject_Patient", ToType: "Condition", EdgeCount: 7, StorageFromType: "Condition", StorageToType: "Patient", StorageDirection: "OUTBOUND", BuilderFromType: "Patient", BuilderToType: "Condition", BuilderDirection: "INBOUND"}, + {Project: "p1", DatasetGeneration: "g2", AuthResourcePath: "clinic/a", FromType: "Patient", Label: "subject_Patient", ToType: "Condition", EdgeCount: 3, StorageFromType: "Condition", StorageToType: "Patient", StorageDirection: "OUTBOUND", BuilderFromType: "Patient", BuilderToType: "Condition", BuilderDirection: "INBOUND"}, + {Project: "p2", DatasetGeneration: "g1", AuthResourcePath: "clinic/b", FromType: "Patient", Label: "subject_Patient", ToType: "Condition", EdgeCount: 3, StorageFromType: "Condition", StorageToType: "Patient", StorageDirection: "OUTBOUND", BuilderFromType: "Patient", BuilderToType: "Condition", BuilderDirection: "INBOUND"}, + } + for i := range wantOrder { + if values[i] != wantOrder[i] { + t.Fatalf("order[%d] = %+v, want %+v", i, values[i], wantOrder[i]) + } + } + digest, err := RelationshipObservationDigest(values) + if err != nil { + t.Fatalf("digest: %v", err) + } + if want := "sha256:f2391bc882ce18f46fc6877d8eeb40ffd84773597a9a3308ce2a10872783982e"; digest != want { + t.Fatalf("digest = %q, want %q", digest, want) + } + }) + + t.Run("comparator tie-breaks only on the count", func(t *testing.T) { + base := RelationshipObservation{Project: "p", DatasetGeneration: "g", AuthResourcePath: "a", FromType: "Patient", Label: "subject_Patient", ToType: "Condition", StorageFromType: "Condition", StorageToType: "Patient", StorageDirection: "OUTBOUND", BuilderFromType: "Patient", BuilderToType: "Condition", BuilderDirection: "INBOUND"} + low, high := base, base + low.EdgeCount, high.EdgeCount = 5, 9 + if !RelationshipEvidenceLess(low, high) || RelationshipEvidenceLess(high, low) { + t.Fatalf("count tie-break wrong: low 0 { + pivotColumnSelect = deriveObservedPivotSelector(stat.pivotColumnArms, observationPivotColumnArmPrecedence) + } + pivotValueSelect := stat.pivotValueSelect + if len(stat.pivotValueArms) > 0 { + pivotValueSelect = deriveObservedPivotSelector(stat.pivotValueArms, observationPivotValueArmPrecedence) + } out = append(out, FieldCatalogDocument{ Key: fieldCatalogKeyForGeneration(p.project, datasetGeneration, p.authResourcePath, p.resourceType, stat.path), Project: p.project, @@ -326,8 +323,8 @@ func (p *Profiler) Documents() []FieldCatalogDocument { PivotKind: stat.pivotKind, PivotColumns: pivotColumns, PivotFamily: stat.pivotFamily, - PivotColumnSelect: stat.pivotColumnSelect, - PivotValueSelect: stat.pivotValueSelect, + PivotColumnSelect: pivotColumnSelect, + PivotValueSelect: pivotValueSelect, PivotItemSource: stat.pivotItemSource, PivotItemResourceType: stat.pivotItemResourceType, PivotValueSelectors: append([]string(nil), stat.pivotValueSelectors...), @@ -390,6 +387,9 @@ func maxLength(nodes []any, field string) int { func (p *Profiler) ensureStat(field *fieldPlan) *fieldCatalogStats { if stat, ok := p.stats[field.Path]; ok { + if pivotStatNeedsUpgrade(stat, field) { + p.upgradePivotStat(stat, field) + } return stat } if p.limits.MaxFields > 0 && len(p.stats) >= p.limits.MaxFields { @@ -412,19 +412,91 @@ func (p *Profiler) ensureStat(field *fieldPlan) *fieldCatalogStats { extensionValueSet: make(map[string]struct{}), } if field.PivotCandidate { - if spec, ok := fhirschema.DefaultPivotSpec(p.resourceType, field.Path, ""); ok { - stat.pivotFamily = spec.Family - stat.pivotColumnSelect = fhirschema.SelectorExpression(spec.ColumnSelector) - stat.pivotValueSelect = fhirschema.SelectorExpression(spec.ValueSelector) - stat.pivotItemSource = spec.ItemSourcePath - stat.pivotItemResourceType = spec.ItemResourceType - stat.pivotValueSelectors = selectorExpressions(spec.ValueSelectors) - } + p.applyDefaultPivotSpec(stat) } p.stats[field.Path] = stat return stat } +func pivotStatNeedsUpgrade(stat *fieldCatalogStats, field *fieldPlan) bool { + if kindUpgrade(stat.kind, field.Kind) != "" { + return true + } + if !field.PivotCandidate { + return false + } + return !stat.pivotCandidate || stat.pivotKind == "" || !stat.pivotSpecApplied +} + +func kindUpgradeRank(kind string) int { + switch kind { + case fieldKindCodeableConcept: + return 3 + case fieldKindCoding: + return 2 + case fieldKindObject: + return 1 + default: + return 0 + } +} + +func kindUpgrade(current string, incoming string) string { + if kindUpgradeRank(current) < kindUpgradeRank(fieldKindObject) { + return "" + } + if kindUpgradeRank(incoming) < kindUpgradeRank(fieldKindCoding) { + return "" + } + if kindUpgradeRank(incoming) <= kindUpgradeRank(current) { + return "" + } + return incoming +} + +func (p *Profiler) upgradePivotStat(stat *fieldCatalogStats, field *fieldPlan) { + if upgraded := kindUpgrade(stat.kind, field.Kind); upgraded != "" { + stat.kind = upgraded + } + if !field.PivotCandidate { + return + } + stat.pivotCandidate = true + if stat.pivotKind == "" { + stat.pivotKind = field.PivotKind + } + p.applyDefaultPivotSpec(stat) +} + +func (p *Profiler) applyDefaultPivotSpec(stat *fieldCatalogStats) { + if stat.pivotSpecApplied { + return + } + stat.pivotSpecApplied = true + spec, ok := fhirschema.DefaultPivotSpec(p.resourceType, stat.path, "") + if !ok { + return + } + if strings.TrimSpace(stat.pivotFamily) == "" { + stat.pivotFamily = spec.Family + } + if strings.TrimSpace(stat.pivotColumnSelect) == "" { + stat.pivotColumnSelect = fhirschema.SelectorExpression(spec.ColumnSelector) + } + if strings.TrimSpace(stat.pivotValueSelect) == "" { + stat.pivotValueSelect = fhirschema.SelectorExpression(spec.ValueSelector) + } + if strings.TrimSpace(stat.pivotItemSource) == "" { + stat.pivotItemSource = spec.ItemSourcePath + } + if strings.TrimSpace(stat.pivotItemResourceType) == "" { + stat.pivotItemResourceType = spec.ItemResourceType + } + if len(stat.pivotValueSelectors) == 0 { + stat.pivotValueSelectors = selectorExpressions(spec.ValueSelectors) + } +} + func (p *Profiler) RetentionBytes() int { if p.budget != nil { return p.budget.usage() @@ -673,8 +745,6 @@ func (p *Profiler) ensureExtensionURLStat(path string) *fieldCatalogStats { } func extensionValueMapping(path string, value any) (string, string) { - // FHIR complex value[x] payloads and arrays require lossless JSON fallback; - // an empty value path is the resolver's explicit canonical-JSON marker. switch value.(type) { case map[string]any, []any: return "", "string" @@ -703,14 +773,117 @@ func (s *fieldCatalogStats) setPivotDefaults(family string, columnSelector strin if strings.TrimSpace(family) != "" { s.pivotFamily = family } - if strings.TrimSpace(columnSelector) != "" { + if strings.TrimSpace(columnSelector) != "" && len(s.pivotColumnArms) == 0 { s.pivotColumnSelect = columnSelector } - if strings.TrimSpace(valueSelector) != "" { + if strings.TrimSpace(valueSelector) != "" && len(s.pivotValueArms) == 0 { s.pivotValueSelect = valueSelector } } +func (s *fieldCatalogStats) observePivotArms(columnSelector string, valueSelector string) { + if value := strings.TrimSpace(columnSelector); value != "" { + if s.pivotColumnArms == nil { + s.pivotColumnArms = make(map[string]struct{}) + } + s.pivotColumnArms[value] = struct{}{} + } + if value := strings.TrimSpace(valueSelector); value != "" { + if s.pivotValueArms == nil { + s.pivotValueArms = make(map[string]struct{}) + } + s.pivotValueArms[value] = struct{}{} + } +} + +func (s *fieldCatalogStats) mergePivotArmsFrom(other *fieldCatalogStats) { + for value := range other.pivotColumnArms { + if s.pivotColumnArms == nil { + s.pivotColumnArms = make(map[string]struct{}) + } + s.pivotColumnArms[value] = struct{}{} + } + for value := range other.pivotValueArms { + if s.pivotValueArms == nil { + s.pivotValueArms = make(map[string]struct{}) + } + s.pivotValueArms[value] = struct{}{} + } +} + +func (s *fieldCatalogStats) convergePivotEvidence(other *fieldCatalogStats) { + s.mergePivotArmsFrom(other) + if upgraded := kindUpgrade(s.kind, other.kind); upgraded != "" { + s.kind = upgraded + } + if other.pivotCandidate { + s.pivotCandidate = true + } + if pivotKindRank(other.pivotKind) > pivotKindRank(s.pivotKind) { + s.pivotKind = other.pivotKind + } + s.pivotSpecApplied = s.pivotSpecApplied || other.pivotSpecApplied +} + +func pivotKindRank(kind string) int { + switch kind { + case pivotKindObservation: + return 2 + case pivotKindCodeableConcept: + return 1 + default: + return 0 + } +} + +func cloneStringSet(set map[string]struct{}) map[string]struct{} { + if len(set) == 0 { + return nil + } + clone := make(map[string]struct{}, len(set)) + for value := range set { + clone[value] = struct{}{} + } + return clone +} + +var observationPivotColumnArmPrecedence = []string{"code.text", "code.coding[].display"} + +var observationPivotValueArmPrecedence = []string{ + "valueQuantity.value", + "valueCodeableConcept.text", + "valueCodeableConcept.coding[].display", + "valueString", + "valueInteger", + "valueBoolean", + "valueDecimal", + "valueDateTime", + "valueTime", + "valuePeriod.start", + "valuePeriod.end", + "valueRange.low.value", + "valueRange.high.value", + "valueRatio.numerator.value", + "valueRatio.denominator.value", +} + +func deriveObservedPivotSelector(arms map[string]struct{}, precedence []string) string { + for _, candidate := range precedence { + if _, ok := arms[candidate]; ok { + return candidate + } + } + remaining := make([]string, 0, len(arms)) + for arm := range arms { + remaining = append(remaining, arm) + } + slices.Sort(remaining) + if len(remaining) == 0 { + return "" + } + return remaining[0] +} + func (s *fieldCatalogStats) setPivotScope(itemSource, itemResourceType string, valueSelectors []string) { if strings.TrimSpace(itemSource) != "" { s.pivotItemSource = itemSource @@ -847,15 +1020,17 @@ func isLoomMetadataField(path string) bool { func addFieldPlan(fieldMap map[string]*fieldPlan, path string, accessor []pathStep, kind string, pivotCandidate bool, pivotKind string) { if existing, ok := fieldMap[path]; ok { - if existing.Kind == fieldKindObject && (kind == fieldKindCodeableConcept || kind == fieldKindCoding) { - existing.Kind = kind + if upgraded := kindUpgrade(existing.Kind, kind); upgraded != "" { + existing.Kind = upgraded } if existing.Kind == fieldKindArray && kind != fieldKindArray { return } if pivotCandidate { existing.PivotCandidate = true - existing.PivotKind = pivotKind + if pivotKindRank(pivotKind) > pivotKindRank(existing.PivotKind) { + existing.PivotKind = pivotKind + } } return } @@ -928,11 +1103,13 @@ func (p *Profiler) observeObservationCodePivot(payload map[string]any) { } stat.pivotCandidate = true stat.pivotKind = pivotKindObservation + stat.pivotFamily = fhirschema.PivotFamilyObservationCodeValue + p.applyDefaultPivotSpec(stat) columnSelector := "code.coding[].display" if _, hasText := codeValue["text"]; hasText { columnSelector = "code.text" } - stat.setPivotDefaults(fhirschema.PivotFamilyObservationCodeValue, columnSelector, valueSelector) + stat.observePivotArms(columnSelector, valueSelector) for _, col := range codeableConceptColumns(codeValue) { p.addPivotColumnWithLimits(stat, col) } diff --git a/internal/catalog/write_profiler_pivot_determinism_test.go b/internal/catalog/write_profiler_pivot_determinism_test.go new file mode 100644 index 000000000..9697cd4e6 --- /dev/null +++ b/internal/catalog/write_profiler_pivot_determinism_test.go @@ -0,0 +1,315 @@ +package catalog + +import ( + "encoding/json" + "fmt" + "testing" +) + +func pivotDeterminismObservations() []map[string]any { + return []map[string]any{ + { + "code": map[string]any{"coding": []any{map[string]any{"system": "https://bench.example/codes", "code": "QUANTITY", "display": "Quantity Display"}}}, + "valueQuantity": map[string]any{"value": 7.5, "unit": "kg", "system": "http://unitsofmeasure.org", "code": "kg"}, + }, + { + "code": map[string]any{"coding": []any{map[string]any{"system": "https://bench.example/codes", "code": "STRING", "display": "String Display"}}}, + "valueString": "self-reported:no", + }, + { + "code": map[string]any{"coding": []any{map[string]any{"system": "https://bench.example/codes", "code": "CONCEPT_TEXT", "display": "Concept Text Display"}}}, + "valueCodeableConcept": map[string]any{"text": "STAGE-A"}, + }, + { + "code": map[string]any{"coding": []any{map[string]any{"system": "https://bench.example/codes", "code": "CONCEPT_CODING", "display": "Concept Coding Display"}}}, + "valueCodeableConcept": map[string]any{"coding": []any{map[string]any{"system": "https://bench.example/codes", "code": "STAGE-B", "display": "Stage B"}}}, + }, + { + "code": map[string]any{ + "text": "Mixed Code Text", + "coding": []any{map[string]any{"system": "https://bench.example/codes", "code": "MIXED", "display": "Mixed Display"}}, + }, + "valueQuantity": map[string]any{"value": 3, "unit": "cm"}, + "valueCodeableConcept": map[string]any{"text": "STAGE-C"}, + "valueString": "mixed-string", + }, + { + "code": map[string]any{"coding": []any{map[string]any{"system": "https://bench.example/codes", "code": "NO_VALUE", "display": "No Value Display"}}}, + }, + } +} + +func pivotDeterminismPermutations(count int) [][]int { + remaining := make([]int, count) + for index := range remaining { + remaining[index] = index + } + results := [][]int(nil) + var build func(prefix []int, rest []int) + build = func(prefix []int, rest []int) { + if len(rest) == 0 { + permutation := append([]int(nil), prefix...) + results = append(results, permutation) + return + } + for index := range rest { + next := append([]int(nil), rest[:index]...) + next = append(next, rest[index+1:]...) + build(append(append([]int(nil), prefix...), rest[index]), next) + } + } + build(nil, remaining) + return results +} + +func pivotDeterminismProfile(t *testing.T, docs []map[string]any) string { + t.Helper() + profiler := NewProfilerForGeneration("TEST", "pivot-determinism", "pathA", "Observation", NewShapePlanCache()) + for _, doc := range docs { + if err := profiler.ObservePayload(doc, map[string]float64{}); err != nil { + t.Fatalf("observe payload: %v", err) + } + } + encoded, err := json.Marshal(profiler.Documents()) + if err != nil { + t.Fatalf("marshal documents: %v", err) + } + return string(encoded) +} + +func pivotDeterminismFieldDocument(t *testing.T, encoded string, path string) FieldCatalogDocument { + t.Helper() + var docs []FieldCatalogDocument + if err := json.Unmarshal([]byte(encoded), &docs); err != nil { + t.Fatalf("unmarshal documents: %v", err) + } + for _, doc := range docs { + if doc.Path == path { + return doc + } + } + t.Fatalf("no field document for path %q in %s", path, encoded) + return FieldCatalogDocument{} +} + +func TestFieldCatalogObservationPivotDefaultsIgnoreDocumentArrivalOrder(t *testing.T) { + docs := pivotDeterminismObservations() + var canonical string + for permutation, order := range pivotDeterminismPermutations(len(docs)) { + ordered := make([]map[string]any, 0, len(order)) + for _, index := range order { + ordered = append(ordered, docs[index]) + } + encoded := pivotDeterminismProfile(t, ordered) + if permutation == 0 { + canonical = encoded + } + if encoded != canonical { + t.Fatalf("permutation %d (order %v) produced a different stored profile than permutation 0\nfirst: %s\nother: %s", permutation, order, canonical, encoded) + } + } + found := pivotDeterminismFieldDocument(t, canonical, "code") + if found.PivotValueSelect != "valueQuantity.value" { + t.Fatalf("pivot value selector = %q, want %q", found.PivotValueSelect, "valueQuantity.value") + } + if found.PivotColumnSelect != "code.text" { + t.Fatalf("pivot column selector = %q, want %q", found.PivotColumnSelect, "code.text") + } + if found.PivotKind != pivotKindObservation { + t.Fatalf("pivot kind = %q, want %q", found.PivotKind, pivotKindObservation) + } +} + +func TestFieldCatalogObservationPivotValueSelectorFollowsArmPrecedence(t *testing.T) { + cases := []struct { + name string + docs []map[string]any + want string + }{ + { + name: "coding-display-beats-string", + docs: []map[string]any{ + {"code": map[string]any{"coding": []any{map[string]any{"code": "A", "display": "Alpha"}}}, "valueString": "plain"}, + {"code": map[string]any{"coding": []any{map[string]any{"code": "B", "display": "Beta"}}}, "valueCodeableConcept": map[string]any{"coding": []any{map[string]any{"code": "C", "display": "Gamma"}}}}, + }, + want: "valueCodeableConcept.coding[].display", + }, + { + name: "concept-text-beats-coding-display", + docs: []map[string]any{ + {"code": map[string]any{"coding": []any{map[string]any{"code": "B", "display": "Beta"}}}, "valueCodeableConcept": map[string]any{"coding": []any{map[string]any{"code": "C", "display": "Gamma"}}}}, + {"code": map[string]any{"coding": []any{map[string]any{"code": "A", "display": "Alpha"}}}, "valueCodeableConcept": map[string]any{"text": "STAGE-A"}}, + }, + want: "valueCodeableConcept.text", + }, + { + name: "quantity-beats-all", + docs: []map[string]any{ + {"code": map[string]any{"coding": []any{map[string]any{"code": "A", "display": "Alpha"}}}, "valueString": "plain"}, + {"code": map[string]any{"coding": []any{map[string]any{"code": "B", "display": "Beta"}}}, "valueCodeableConcept": map[string]any{"text": "STAGE-A"}}, + {"code": map[string]any{"coding": []any{map[string]any{"code": "C", "display": "Gamma"}}}, "valueQuantity": map[string]any{"value": 1.5}}, + }, + want: "valueQuantity.value", + }, + } + for _, testCase := range cases { + reversed := make([]map[string]any, 0, len(testCase.docs)) + for index := len(testCase.docs) - 1; index >= 0; index-- { + reversed = append(reversed, testCase.docs[index]) + } + forward := pivotDeterminismProfile(t, testCase.docs) + backward := pivotDeterminismProfile(t, reversed) + if forward != backward { + t.Fatalf("%s: arrival order changed the stored profile\nforward: %s\nbackward: %s", testCase.name, forward, backward) + } + found := pivotDeterminismFieldDocument(t, forward, "code") + if found.PivotValueSelect != testCase.want { + t.Fatalf("%s: pivot value selector = %q, want %q", testCase.name, found.PivotValueSelect, testCase.want) + } + } +} + +func pivotDeterminismMergePair(t *testing.T, leftDocs, rightDocs []map[string]any, swap bool) string { + t.Helper() + left := NewProfilerForGeneration("TEST", "pivot-determinism", "pathA", "Observation", NewShapePlanCache()) + right := NewProfilerForGeneration("TEST", "pivot-determinism", "pathA", "Observation", NewShapePlanCache()) + for _, doc := range leftDocs { + if err := left.ObservePayload(doc, map[string]float64{}); err != nil { + t.Fatalf("observe left payload: %v", err) + } + } + for _, doc := range rightDocs { + if err := right.ObservePayload(doc, map[string]float64{}); err != nil { + t.Fatalf("observe right payload: %v", err) + } + } + destination, source := left, right + if swap { + destination, source = right, left + } + if err := destination.Merge(source); err != nil { + t.Fatalf("merge: %v", err) + } + encoded, err := json.Marshal(destination.Documents()) + if err != nil { + t.Fatalf("marshal documents: %v", err) + } + return string(encoded) +} + +func TestFieldCatalogObservationPivotDefaultsStableAcrossMergeOrder(t *testing.T) { + leftDocs := []map[string]any{ + {"code": map[string]any{"coding": []any{map[string]any{"code": "Q", "display": "Quantity"}}}, "valueQuantity": map[string]any{"value": 2}}, + {"code": map[string]any{"coding": []any{map[string]any{"code": "N", "display": "None"}}}}, + } + rightDocs := []map[string]any{ + {"code": map[string]any{"coding": []any{map[string]any{"code": "S", "display": "String"}}}, "valueString": "later"}, + {"code": map[string]any{"coding": []any{map[string]any{"code": "CT", "display": "Concept"}}}, "valueCodeableConcept": map[string]any{"text": "STAGE-A"}}, + } + forward := pivotDeterminismMergePair(t, leftDocs, rightDocs, false) + backward := pivotDeterminismMergePair(t, leftDocs, rightDocs, true) + if forward != backward { + t.Fatalf("merge direction changed the stored profile\nleft-into-right: %s\nright-into-left: %s", forward, backward) + } + found := pivotDeterminismFieldDocument(t, forward, "code") + if found.PivotValueSelect != "valueQuantity.value" { + t.Fatalf("pivot value selector = %q, want %q", found.PivotValueSelect, "valueQuantity.value") + } +} + +func pivotDeterminismCreationOrderRuns(t *testing.T) []string { + emptyCodeWithArm := map[string]any{ + "code": map[string]any{}, + "valueQuantity": map[string]any{"value": 9}, + } + codeShapeWithoutArm := map[string]any{ + "code": map[string]any{"coding": []any{map[string]any{"code": "C", "display": "Concept"}}}, + } + return []string{ + pivotDeterminismProfile(t, []map[string]any{emptyCodeWithArm, codeShapeWithoutArm}), + pivotDeterminismProfile(t, []map[string]any{codeShapeWithoutArm, emptyCodeWithArm}), + pivotDeterminismMergePair(t, []map[string]any{emptyCodeWithArm}, []map[string]any{codeShapeWithoutArm}, false), + pivotDeterminismMergePair(t, []map[string]any{emptyCodeWithArm}, []map[string]any{codeShapeWithoutArm}, true), + } +} + +func pivotDeterminismKindShards() (map[string]any, map[string]any) { + codingDoc := map[string]any{ + "code": map[string]any{"coding": []any{map[string]any{"code": "KIND-CODING", "display": "Coding Kind"}}}, + "valueCodeableConcept": map[string]any{"system": "http://example.org/sys", "code": "LEAF", "display": "Leaf"}, + } + conceptDoc := map[string]any{ + "code": map[string]any{"coding": []any{map[string]any{"code": "KIND-CONCEPT", "display": "Concept Kind"}}}, + "valueCodeableConcept": map[string]any{"text": "CONCEPT"}, + } + return codingDoc, conceptDoc +} + +func pivotDeterminismMergePlans(firstKind string, firstPivot string, secondKind string, secondPivot string) *fieldPlan { + plans := make(map[string]*fieldPlan) + addFieldPlan(plans, "code", nil, firstKind, true, firstPivot) + addFieldPlan(plans, "code", nil, secondKind, true, secondPivot) + return plans["code"] +} + +func TestFieldCatalogPlanPivotKindConvergesAcrossMergeOrder(t *testing.T) { + forward := pivotDeterminismMergePlans(fieldKindCodeableConcept, pivotKindCodeableConcept, fieldKindCodeableConcept, pivotKindObservation) + backward := pivotDeterminismMergePlans(fieldKindCodeableConcept, pivotKindObservation, fieldKindCodeableConcept, pivotKindCodeableConcept) + if forward.PivotKind != pivotKindObservation || backward.PivotKind != pivotKindObservation { + t.Fatalf("plan pivot kind depends on merge order: forward %q, backward %q, want %q", forward.PivotKind, backward.PivotKind, pivotKindObservation) + } + if !forward.PivotCandidate || !backward.PivotCandidate { + t.Fatalf("plan pivot candidate lost: forward %v, backward %v", forward.PivotCandidate, backward.PivotCandidate) + } +} + +func TestFieldCatalogKindConvergesAcrossMergeDirection(t *testing.T) { + codingDoc, conceptDoc := pivotDeterminismKindShards() + forward := pivotDeterminismMergePair(t, []map[string]any{codingDoc}, []map[string]any{conceptDoc}, false) + backward := pivotDeterminismMergePair(t, []map[string]any{codingDoc}, []map[string]any{conceptDoc}, true) + if forward != backward { + t.Fatalf("merge direction changed the stored profile\nleft-into-right: %s\nright-into-left: %s", forward, backward) + } + found := pivotDeterminismFieldDocument(t, forward, "valueCodeableConcept") + if found.Kind != fieldKindCodeableConcept { + t.Fatalf("kind = %q, want %q", found.Kind, fieldKindCodeableConcept) + } +} + +func TestFieldCatalogKindConvergesAcrossDocumentArrivalOrder(t *testing.T) { + codingDoc, conceptDoc := pivotDeterminismKindShards() + forward := pivotDeterminismProfile(t, []map[string]any{codingDoc, conceptDoc}) + backward := pivotDeterminismProfile(t, []map[string]any{conceptDoc, codingDoc}) + if forward != backward { + t.Fatalf("document arrival order changed the stored profile\nforward: %s\nbackward: %s", forward, backward) + } + found := pivotDeterminismFieldDocument(t, forward, "valueCodeableConcept") + if found.Kind != fieldKindCodeableConcept { + t.Fatalf("kind = %q, want %q", found.Kind, fieldKindCodeableConcept) + } +} + +func TestFieldCatalogObservationPivotStatConvergesAcrossCreationOrder(t *testing.T) { + runs := pivotDeterminismCreationOrderRuns(t) + for index := 1; index < len(runs); index++ { + if runs[index] != runs[0] { + t.Fatalf("run %d produced a different stored profile than run 0\nbase: %s\nrun%d: %s", index, runs[0], index, runs[index]) + } + } + found := pivotDeterminismFieldDocument(t, runs[0], "code") + if !found.PivotCandidate { + t.Fatalf("pivot candidate = false, want true in %s", runs[0]) + } + if found.PivotKind != pivotKindObservation { + t.Fatalf("pivot kind = %q, want %q", found.PivotKind, pivotKindObservation) + } + if found.PivotItemResourceType != "Observation" { + t.Fatalf("pivot item resource type = %q, want %q", found.PivotItemResourceType, "Observation") + } + if len(found.PivotValueSelectors) == 0 { + t.Fatalf("pivot value selectors empty, want spec defaults in %s", runs[0]) + } + if fmt.Sprint(found.PivotFamily) != "OBSERVATION_CODE_VALUE" { + t.Fatalf("pivot family = %q, want OBSERVATION_CODE_VALUE", found.PivotFamily) + } +} diff --git a/internal/catalog/write_profiler_test.go b/internal/catalog/write_profiler_test.go index 0744f52ed..6d0364297 100644 --- a/internal/catalog/write_profiler_test.go +++ b/internal/catalog/write_profiler_test.go @@ -459,8 +459,6 @@ func TestGenerationProfilerSeparatesCatalogDocumentsAndKeys(t *testing.T) { t.Fatalf("catalog keys must isolate legacy and each generation: legacy=%q generationA=%q generationB=%q", legacyDocument.Key, generationADocument.Key, generationBDocument.Key) } - // These generation values collide under sanitizeCollectionKey, which is why - // a direct sanitized suffix would not be a safe immutable namespace. if got, want := sanitizeCollectionKey("generation/a"), sanitizeCollectionKey("generation a"); got != want { t.Fatalf("test setup expected sanitized generation collision: %q != %q", got, want) } diff --git a/internal/ci/gofmt_gate_test.go b/internal/ci/gofmt_gate_test.go index fcbeec40d..8a865abe5 100644 --- a/internal/ci/gofmt_gate_test.go +++ b/internal/ci/gofmt_gate_test.go @@ -1,19 +1,5 @@ package ci -// Durable formatting boundary (BS-67-1 class follow-through, Round 78). -// The gofmt-drift class that reached ten tracked files across two waves is -// now asserted dead: every Git-tracked .go file must be gofmt-canonical. -// Tracked-only scope keeps operator working-copy experiments out of the -// gate's face; untracked files are not CI content. -// -// Arms: (1) detector sensitivity — a known non-canonical source is RED -// through the same gofmt binary invocation, proving assertions (2) and (3) -// carry teeth; (2) the repository run is CLEAN; (3) a freshly gofmt'd copy -// of that same source is CLEAN, guarding against a vacuous always-clean -// detector; (4) any non-zero gofmt exit is FATAL — a batch that lists one -// dirty file while another fails to parse (exit 2, nothing listed for it) -// must error, not pass on the listed file's strength. - import ( "bytes" "fmt" @@ -53,10 +39,6 @@ func gofmtList(t *testing.T, gofmt string, files []string) (string, error) { cmd.Stderr = os.Stderr out, err := cmd.Output() dirty.Write(out) - // Any non-zero gofmt exit is fatal regardless of what the batch - // listed: exit 2 with parse errors on stderr and a non-empty list - // for other files in the same batch would otherwise pass the - // broken file silently (reviewer-found shape, parent-reproduced). if err != nil { return "", fmt.Errorf("gofmt batch %d-%d: %w", start, end, err) } @@ -87,7 +69,6 @@ func trackedGoFiles(t *testing.T) []string { func TestGofmtGateTrackedFilesAreCanonical(t *testing.T) { gofmt := gofmtBinary(t) - // Arm 1: sensitivity. Same invocation shape as the repository run. scratch := t.TempDir() dirtyPath := filepath.Join(scratch, "dirty.go") if err := os.WriteFile(dirtyPath, []byte("package scratch\nfunc f( ) int {\nreturn 1\n}\n"), 0o644); err != nil { @@ -101,7 +82,6 @@ func TestGofmtGateTrackedFilesAreCanonical(t *testing.T) { t.Fatalf("sensitivity arm: gofmt did not flag the known-dirty file (output %q); the gate would be vacuous", dirty) } - // Arm 2: repository baseline. files := trackedGoFiles(t) report, err := gofmtList(t, gofmt, files) if err != nil { @@ -111,7 +91,6 @@ func TestGofmtGateTrackedFilesAreCanonical(t *testing.T) { t.Fatalf("gofmt drift in tracked files:\n%s", report) } - // Arm 3: anti-vacuity — the detector clears canonical input too. cleanPath := filepath.Join(scratch, "clean.go") if err := os.WriteFile(cleanPath, []byte("package scratch\n\nfunc f() int {\n\treturn 1\n}\n"), 0o644); err != nil { t.Fatal(err) @@ -124,9 +103,6 @@ func TestGofmtGateTrackedFilesAreCanonical(t *testing.T) { t.Fatalf("gofmt flagged canonical input: %q", clean) } - // Arm 4: exit-code honesty (reviewer-found shape, parent-reproduced): - // a batch mixing a listed-dirty file with an UNPARSEABLE file must - // return an error, never pass on the strength of the listed file. brokenPath := filepath.Join(scratch, "broken.go") if err := os.WriteFile(brokenPath, []byte("package scratch\nfunc f( {\n"), 0o644); err != nil { t.Fatal(err) diff --git a/internal/ci/vet_gate_test.go b/internal/ci/vet_gate_test.go index c559dfed0..832d34781 100644 --- a/internal/ci/vet_gate_test.go +++ b/internal/ci/vet_gate_test.go @@ -1,13 +1,5 @@ package ci -// CI vet boundary: `go test` vets only its fixed analyzer subset, so classes -// outside that subset (copylocks above all) ship green unless a full `go vet` -// gates them. The GitHub node-unit job has no Go toolchain, so this package -// holds the durable gate checks in the go-unit job: repo-wide default-set -// greenness, a fault control proving copylocks actually fires (via -overlay, -// so nothing is written to the repo), its negative control, and the -// scripts/ci/vet.mjs CLI smoke that CI calls. - import ( "encoding/json" "errors" @@ -22,9 +14,6 @@ import ( "time" ) -// R67Nonce keeps probe content unique per run: the vet analyzer result is -// cached on package CONTENT, so a stale cached diagnostic (naming an earlier -// run's temp path) could otherwise replay and mask a regression. const vetProbeSourceTemplate = `package authoringv2 import "sync" @@ -103,18 +92,12 @@ func TestVetGateFiresOnInjectedCopylocksFault(t *testing.T) { if runErr == nil { t.Fatalf("the injected copylocks fault must make the default-set go vet fail (that is the shipped-green class this gate exists for):\n%s", out) } - // vet names the overlay replacement file, so this run's temp dir is the - // current-run proof; combined with the content nonce it is cache-immune. for _, want := range []string{"copies lock value", temp} { if !strings.Contains(out, want) { t.Fatalf("default-set go vet output must contain %q for the injected probe; got:\n%s", want, out) } } - // Negative control: -copylocks=false keeps the default analyzer set minus - // copylocks, so the same fault passing vet identifies copylocks (rather - // than any other analyzer, or a compile failure from the probe) as what - // fired above. out, runErr = runGoVet(t, "-copylocks=false", "-overlay="+overlay, pkg) if runErr != nil { t.Fatalf("with -copylocks=false the same overlay must vet clean; anything failing there is not the copylocks class:\n%v\n%s", runErr, out) diff --git a/internal/dataframe/columntransform/types.go b/internal/dataframe/columntransform/types.go index c03030380..799a4b784 100644 --- a/internal/dataframe/columntransform/types.go +++ b/internal/dataframe/columntransform/types.go @@ -1,5 +1,3 @@ -// Package columntransform defines closed, backend-neutral transformations -// that operate on one already-selected output column value. package columntransform import "fmt" @@ -17,8 +15,6 @@ const ( UnknownKeepOriginal UnknownPolicy = "KEEP_ORIGINAL" ) -// ValueTransformation is a closed union. The selected column remains the -// identity-bearing output; this value only changes its scalar contents. type ValueTransformation struct { Kind Kind `json:"kind"` ExactCategoryRecode *ExactCategoryRecode `json:"exactCategoryRecode,omitempty"` diff --git a/internal/dataframe/compiler/api.go b/internal/dataframe/compiler/api.go index 2727289a2..a20d4fea6 100644 --- a/internal/dataframe/compiler/api.go +++ b/internal/dataframe/compiler/api.go @@ -1,5 +1 @@ -// Package compiler orchestrates dataframe compilation. -// -// Compilation proceeds through spec, semantic, ir, lower, optimize, and -// render/aql, with each layer owning its canonical types and transformations. package compiler diff --git a/internal/dataframe/compiler/backend_leaf_guard_test.go b/internal/dataframe/compiler/backend_leaf_guard_test.go new file mode 100644 index 000000000..6fa78b873 --- /dev/null +++ b/internal/dataframe/compiler/backend_leaf_guard_test.go @@ -0,0 +1,145 @@ +package compiler + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/calypr/loom/scripts/packagepolicy" +) + +func compilerProductionRule(t *testing.T) packagepolicy.Rule { + t.Helper() + rule, ok := packagepolicy.ByLabel("compiler-production: no backend implementations or database drivers") + if !ok { + t.Fatal("authoritative compiler-production rule is missing from the packagepolicy table") + } + return rule +} + +func TestCompilerProductionFilesNeverImportBackendsOrDrivers(t *testing.T) { + rule := compilerProductionRule(t) + + if res := packagepolicy.ScanTarget(".", rule.Label, rule.Owner, rule.Forbidden, rule.RejectNonStdlib, rule.Scope, true); !res.Clean() { + for _, f := range res.ImportFindings() { + t.Error(f.String()) + } + for _, e := range res.Errors { + t.Error(e.String()) + } + t.FailNow() + } + + t.Run("non_vacuity_named_victims_bleed", func(t *testing.T) { + dir := t.TempDir() + victims := map[string]string{ + "victim_backends_import.go": "package compiler\n\nimport \"github.com/calypr/loom/internal/backends/postgres/internal/store\"\n\nvar _ = store.SeriesSource\n", + "victim_pgx_import.go": "package compiler\n\nimport \"github.com/jackc/pgx/v5/pgxpool\"\n\nvar _ = pgxpool.Config{}\n", + "victim_arangodb_import.go": "package compiler\n\nimport \"github.com/arangodb/go-driver\"\n\nvar _ = go-driver.Database\n", + } + for name, body := range victims { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + res := packagepolicy.ScanTarget(dir, rule.Label, rule.Owner, rule.Forbidden, rule.RejectNonStdlib, rule.Scope, true) + if len(res.Errors) != 0 { + t.Fatalf("fixture scan must not raise verification failures: %v", res.Errors) + } + offenders := strings.Join(findingsToStrings(res.ImportFindings()), "\n") + for name, body := range victims { + importPath := importPathOf(t, body) + if !strings.Contains(offenders, name) { + t.Fatalf("guard did not name victim %s in:\n%s", name, offenders) + } + if !strings.Contains(offenders, importPath) { + t.Fatalf("guard did not name the offending import %q for %s in:\n%s", importPath, name, offenders) + } + if !strings.Contains(offenders, "allowed owner") { + t.Fatalf("guard output must name the allowed owner for %s in:\n%s", name, offenders) + } + } + }) + + t.Run("non_vacuity_fires_on_a_real_postgres_adapter", func(t *testing.T) { + repoRoot := repoRootForGuardTest(t) + target := filepath.Join(repoRoot, "internal", "backends", "postgres", "internal", "rows") + res := packagepolicy.ScanTarget(target, rule.Label, rule.Owner, rule.Forbidden, rule.RejectNonStdlib, packagepolicy.AllScopes, true) + if len(res.Errors) != 0 { + t.Fatalf("real-adapter control must run cleanly before it can detect: %v", res.Errors) + } + if !res.HasImportFindings() { + t.Fatal("scanner missed the known PostgreSQL driver imports in internal/backends/postgres/internal/rows") + } + if !anyFindingNames(res.ImportFindings(), "rows.go", "github.com/jackc/pgx") { + t.Fatalf("real-adapter control must produce a concrete rows.go pgx import finding, got %v", res.ImportFindings()) + } + }) + + t.Run("non_vacuity_fires_on_a_real_arango_adapter", func(t *testing.T) { + repoRoot := repoRootForGuardTest(t) + target := filepath.Join(repoRoot, "internal", "backends", "arango", "internal", "store") + res := packagepolicy.ScanTarget(target, rule.Label, rule.Owner, rule.Forbidden, rule.RejectNonStdlib, packagepolicy.AllScopes, true) + if len(res.Errors) != 0 { + t.Fatalf("real-adapter control must run cleanly before it can detect: %v", res.Errors) + } + if !res.HasImportFindings() { + t.Fatal("scanner missed the known ArangoDB driver imports in internal/backends/arango/internal/store") + } + }) + + t.Run("non_vacuity_malformed_input_is_verification_failure", func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "broken.go"), []byte("!!! not go source >>> {{{"), 0o600); err != nil { + t.Fatal(err) + } + res := packagepolicy.ScanTarget(dir, rule.Label, rule.Owner, rule.Forbidden, rule.RejectNonStdlib, rule.Scope, true) + if res.HasImportFindings() { + t.Fatalf("parse failure must never be reported as import detection: %v", res.ImportFindings()) + } + if len(res.Errors) == 0 { + t.Fatal("malformed input must fail verification") + } + }) +} + +func repoRootForGuardTest(t *testing.T) string { + t.Helper() + root, err := packagepolicy.FindRepoRoot(".") + if err != nil { + t.Fatal(err) + } + return root +} + +func findingsToStrings(findings []packagepolicy.Finding) []string { + out := make([]string, 0, len(findings)) + for _, f := range findings { + out = append(out, f.String()) + } + return out +} + +func anyFindingNames(findings []packagepolicy.Finding, fileFragment, importFragment string) bool { + for _, f := range findings { + if strings.Contains(f.File, fileFragment) && strings.Contains(f.Import, importFragment) { + return true + } + } + return false +} + +func importPathOf(t *testing.T, body string) string { + t.Helper() + start := strings.Index(body, `"`) + if start < 0 { + t.Fatalf("fixture body has no import path: %s", body) + } + rest := body[start+1:] + end := strings.Index(rest, `"`) + if end < 0 { + t.Fatalf("fixture body has an unterminated import path: %s", body) + } + return rest[:end] +} diff --git a/internal/dataframe/compiler/backend_ownership_guard_contract_test.go b/internal/dataframe/compiler/backend_ownership_guard_contract_test.go new file mode 100644 index 000000000..c36d189fb --- /dev/null +++ b/internal/dataframe/compiler/backend_ownership_guard_contract_test.go @@ -0,0 +1,348 @@ +package compiler + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/calypr/loom/scripts/packagepolicy" +) + +func guardRule(t *testing.T, label string) packagepolicy.Rule { + t.Helper() + rule, ok := packagepolicy.ByLabel(label) + if !ok { + t.Fatalf("authoritative rule %q is missing from the packagepolicy table", label) + } + return rule +} + +func guardScan(t *testing.T, rule packagepolicy.Rule, target string, scope packagepolicy.Scope) packagepolicy.Result { + t.Helper() + return packagepolicy.ScanTarget(target, rule.Label, rule.Owner, rule.Forbidden, rule.RejectNonStdlib, scope, true) +} + +func guardWrite(t *testing.T, dir, rel, imp string) string { + t.Helper() + path := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + var body string + if imp == "" { + body = "package fixture\n" + } else { + body = "package fixture\n\nimport \"" + imp + "\"\n" + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func mustOldPath(t *testing.T, segments ...string) string { + t.Helper() + candidate := "internal/" + strings.Join(segments, "/") + for _, p := range packagepolicy.OldConcreteImportPaths() { + if p == candidate { + return p + } + } + t.Fatalf("planted old path %q is not in the authoritative retired table", candidate) + return "" +} + +func requireFinding(t *testing.T, res packagepolicy.Result, ruleLabel, victimFile, victimImport string) { + t.Helper() + if len(res.Errors) != 0 { + t.Fatalf("verification failures must never stand in for import detection: %v", res.Errors) + } + for _, f := range res.ImportFindings() { + if strings.Contains(f.File, victimFile) && strings.Contains(f.Import, victimImport) && f.Rule == ruleLabel { + line := f.String() + if !strings.Contains(line, victimFile) || !strings.Contains(line, victimImport) { + t.Fatalf("finding must name the offending file and import: %s", line) + } + if !strings.Contains(line, "allowed owner") || f.Owner == "" { + t.Fatalf("finding must carry allowed-owner guidance: %s", line) + } + return + } + } + t.Fatalf("guard did not report %s importing %s under rule %q; findings: %v", victimFile, victimImport, ruleLabel, res.Findings) +} + +func requireClean(t *testing.T, res packagepolicy.Result) { + t.Helper() + if !res.Clean() { + t.Fatalf("expected a clean fully-established scan; findings %v errors %v", res.Findings, res.Errors) + } + if res.Scanned == 0 { + t.Fatal("clean scan covered zero files: vacuous") + } +} + +func TestBackendOwnershipGuardContract(t *testing.T) { + repoRoot := repoRootForGuardTest(t) + + tableScan := packagepolicy.ScanAll(repoRoot) + for _, f := range tableScan.ImportFindings() { + t.Error(f.String()) + } + for _, e := range tableScan.Errors { + t.Error(e.String()) + } + for _, e := range packagepolicy.CheckAbsences(repoRoot, packagepolicy.RepoAbsences()) { + t.Error(e.String()) + } + for _, rule := range packagepolicy.RepoNoReExport() { + for _, e := range packagepolicy.CheckNoReExport(repoRoot, rule) { + t.Error(e.String()) + } + } + if t.Failed() { + t.Fatal("repository violates the authoritative package boundary table (see failures above)") + } + + compilerRule := guardRule(t, "compiler-production: no backend implementations or database drivers") + neutralRule := guardRule(t, "neutral-domains: no backend implementations or database drivers") + aqlRule := guardRule(t, "render-aql: no lowerer, optimizer, runtime, backends, or drivers") + postgresRenderRule := guardRule(t, "render-postgres: no runtime, backends, stores, or drivers") + oldPathsRule := guardRule(t, "old-concrete-paths: relocated under internal/backends by arch-backend-06/07") + + t.Run("compiler_backends_and_drivers", func(t *testing.T) { + requireClean(t, packagepolicy.ScanRule(repoRoot, compilerRule)) + + dir := t.TempDir() + guardWrite(t, dir, "compiler/backends_leak.go", "github.com/calypr/loom/internal/backends/postgres/internal/store") + guardWrite(t, dir, "compiler/pgx_leak.go", "github.com/jackc/pgx/v5/pgxpool") + guardWrite(t, dir, "compiler/driver_leak.go", "github.com/arangodb/go-driver") + res := guardScan(t, compilerRule, filepath.Join(dir, "compiler"), packagepolicy.Production) + requireFinding(t, res, compilerRule.Label, "backends_leak.go", "internal/backends/postgres/internal/store") + requireFinding(t, res, compilerRule.Label, "pgx_leak.go", "github.com/jackc/pgx") + requireFinding(t, res, compilerRule.Label, "driver_leak.go", "github.com/arangodb") + }) + + t.Run("neutral_domain_backend_import", func(t *testing.T) { + requireClean(t, packagepolicy.ScanRule(repoRoot, neutralRule)) + + dir := t.TempDir() + guardWrite(t, dir, "execution/preview_old.go", "github.com/calypr/loom/"+mustOldPath(t, "store", "arango")) + guardWrite(t, dir, "execution/preview_private.go", "github.com/calypr/loom/internal/backends/arango/internal/explorer") + guardWrite(t, dir, "catalog/rows_old.go", "github.com/calypr/loom/"+mustOldPath(t, "store", "postgres")) + guardWrite(t, dir, "catalog/rows_private.go", "github.com/calypr/loom/internal/backends/postgres/internal/rows") + guardWrite(t, dir, "dataset/factory_leak.go", "github.com/calypr/loom/internal/backends/postgres") + + res := guardScan(t, neutralRule, dir, packagepolicy.Production) + requireFinding(t, res, neutralRule.Label, "preview_private.go", "internal/backends/arango/internal/explorer") + requireFinding(t, res, neutralRule.Label, "rows_private.go", "internal/backends/postgres/internal/rows") + requireFinding(t, res, neutralRule.Label, "factory_leak.go", "internal/backends/postgres") + old := guardScan(t, oldPathsRule, dir, packagepolicy.AllScopes) + requireFinding(t, old, oldPathsRule.Label, "preview_old.go", mustOldPath(t, "store", "arango")) + requireFinding(t, old, oldPathsRule.Label, "rows_old.go", mustOldPath(t, "store", "postgres")) + + for _, rel := range []string{ + "execution/preview_old.go", "execution/preview_private.go", + "catalog/rows_old.go", "catalog/rows_private.go", + "dataset/factory_leak.go", + } { + guardWrite(t, dir, rel, "") + } + requireClean(t, guardScan(t, neutralRule, dir, packagepolicy.Production)) + requireClean(t, guardScan(t, oldPathsRule, dir, packagepolicy.AllScopes)) + }) + + t.Run("renderer_runtime_import", func(t *testing.T) { + requireClean(t, packagepolicy.ScanRule(repoRoot, aqlRule)) + requireClean(t, packagepolicy.ScanRule(repoRoot, postgresRenderRule)) + + dir := t.TempDir() + guardWrite(t, dir, "render/aql/runtime_leak.go", "github.com/calypr/loom/internal/dataframe/execution") + guardWrite(t, dir, "render/postgres/runtime_leak.go", "github.com/calypr/loom/internal/dataframe/execution") + guardWrite(t, dir, "render/aql/root_leak.go", "github.com/calypr/loom/internal/dataframe/compiler") + guardWrite(t, dir, "render/postgres/root_leak.go", "github.com/calypr/loom/internal/dataframe/compiler") + guardWrite(t, dir, "render/aql/clean.go", "") + guardWrite(t, dir, "render/postgres/driver_leak.go", "github.com/jackc/pgx/v5/pgxpool") + guardWrite(t, dir, "render/postgres/store_leak.go", "github.com/calypr/loom/internal/store/clickhouse") + + aqlRes := guardScan(t, aqlRule, filepath.Join(dir, "render/aql"), packagepolicy.Production) + requireFinding(t, aqlRes, aqlRule.Label, "runtime_leak.go", "internal/dataframe/execution") + requireFinding(t, aqlRes, aqlRule.Label, "root_leak.go", "internal/dataframe/compiler") + + pgRes := guardScan(t, postgresRenderRule, filepath.Join(dir, "render/postgres"), packagepolicy.Production) + requireFinding(t, pgRes, postgresRenderRule.Label, "runtime_leak.go", "internal/dataframe/execution") + requireFinding(t, pgRes, postgresRenderRule.Label, "root_leak.go", "internal/dataframe/compiler") + requireFinding(t, pgRes, postgresRenderRule.Label, "driver_leak.go", "github.com/jackc/pgx") + requireFinding(t, pgRes, postgresRenderRule.Label, "store_leak.go", "internal/store/clickhouse") + }) + + t.Run("legitimate_adapter_domain_import", func(t *testing.T) { + dir := t.TempDir() + guardWrite(t, dir, "adapter/rows_adapter.go", "github.com/calypr/loom/internal/catalog") + guardWrite(t, dir, "adapter/recipe_reader.go", "github.com/calypr/loom/internal/dataframe/recipe") + guardWrite(t, dir, "adapter/execution_ports.go", "github.com/calypr/loom/internal/dataframe/execution") + guardWrite(t, dir, "factory/factory.go", "github.com/calypr/loom/internal/dataset") + requireClean(t, guardScan(t, neutralRule, dir, packagepolicy.Production)) + + guardWrite(t, dir, "factory/factory.go", "github.com/calypr/loom/internal/backends/arango/internal/store") + res := guardScan(t, neutralRule, dir, packagepolicy.Production) + requireFinding(t, res, neutralRule.Label, "factory.go", "internal/backends/arango/internal/store") + }) + + t.Run("actual_relocated_adapter_import", func(t *testing.T) { + res := guardScan(t, compilerRule, filepath.Join(repoRoot, "internal", "backends", "postgres", "internal", "rows"), packagepolicy.Production) + if len(res.Errors) != 0 { + t.Fatalf("real-adapter control must run with zero verification failures, got %v", res.Errors) + } + if res.Scanned == 0 { + t.Fatal("real-adapter control scanned zero files: vacuous") + } + var named bool + for _, f := range res.ImportFindings() { + if strings.Contains(f.File, "rows.go") && strings.HasPrefix(f.Import, "github.com/jackc/pgx") { + named = true + } + } + if !named { + t.Fatalf("expected a concrete rows.go pgx import finding, got %v", res.ImportFindings()) + } + + arango := guardScan(t, compilerRule, filepath.Join(repoRoot, "internal", "backends", "arango", "internal", "store"), packagepolicy.Production) + if len(arango.Errors) != 0 || !arango.HasImportFindings() { + t.Fatalf("Arango real-adapter control failed: findings %v errors %v", arango.ImportFindings(), arango.Errors) + } + var driverNamed bool + for _, f := range arango.ImportFindings() { + if strings.HasPrefix(f.Import, "github.com/arangodb") { + driverNamed = true + } + } + if !driverNamed { + t.Fatalf("expected a concrete go-driver import finding in the Arango adapter, got %v", arango.ImportFindings()) + } + }) + + t.Run("missing_or_malformed_input_is_error", func(t *testing.T) { + dir := t.TempDir() + missing := guardScan(t, compilerRule, filepath.Join(dir, "does-not-exist"), packagepolicy.Production) + if missing.HasImportFindings() { + t.Fatalf("missing path produced import findings: %v", missing.ImportFindings()) + } + if !hasErrorKind(missing.Errors, packagepolicy.ErrMissing) { + t.Fatalf("missing path must fail as %s, got %v", packagepolicy.ErrMissing, missing.Errors) + } + + empty := filepath.Join(dir, "empty") + if err := os.MkdirAll(empty, 0o755); err != nil { + t.Fatal(err) + } + vacuous := guardScan(t, compilerRule, empty, packagepolicy.Production) + if vacuous.HasImportFindings() { + t.Fatalf("empty scan produced import findings: %v", vacuous.ImportFindings()) + } + if !hasErrorKind(vacuous.Errors, packagepolicy.ErrVacuous) { + t.Fatalf("empty scan must fail as %s, got %v", packagepolicy.ErrVacuous, vacuous.Errors) + } + + badDir := filepath.Join(dir, "bad") + guardWrite(t, badDir, "broken.go", "") + if err := os.WriteFile(filepath.Join(badDir, "broken.go"), []byte("!!! not go source >>> {{{"), 0o600); err != nil { + t.Fatal(err) + } + malformed := guardScan(t, compilerRule, badDir, packagepolicy.Production) + if malformed.HasImportFindings() { + t.Fatalf("malformed input produced import findings: %v", malformed.ImportFindings()) + } + if !hasErrorKind(malformed.Errors, packagepolicy.ErrParse) { + t.Fatalf("malformed input must fail as %s, got %v", packagepolicy.ErrParse, malformed.Errors) + } + }) + + t.Run("facade_law_control_plane", func(t *testing.T) { + shim := packagepolicy.AbsenceRule{Label: "removed control-forwarding facade", Owner: "import control directly", + Paths: []string{"internal/backends/postgres/aliases.go"}} + dir := t.TempDir() + facadeDir := filepath.Join(dir, "internal/backends/postgres") + if err := os.MkdirAll(facadeDir, 0o755); err != nil { + t.Fatal(err) + } + shimPath := filepath.Join(facadeDir, "aliases.go") + if err := os.WriteFile(shimPath, []byte("package postgres\n"), 0o600); err != nil { + t.Fatal(err) + } + if errs := packagepolicy.CheckAbsences(dir, []packagepolicy.AbsenceRule{shim}); len(errs) == 0 { + t.Fatal("recreated aliases.go must fail the path-absence rule") + } + if err := os.Remove(shimPath); err != nil { + t.Fatal(err) + } + if errs := packagepolicy.CheckAbsences(dir, []packagepolicy.AbsenceRule{shim}); len(errs) != 0 { + t.Fatalf("removal must converge: %v", errs) + } + + root := t.TempDir() + for _, f := range []struct{ rel, body string }{ + {"internal/backends/postgres/control/config.go", "package control\n\nimport \"errors\"\n\ntype Config struct{ N int }\n\nvar ErrInvalid = errors.New(\"x\")\n"}, + {"internal/backends/postgres/factory.go", "package postgres\n\nimport \"github.com/calypr/loom/internal/backends/postgres/control\"\n\nfunc Open(c control.Config) (*Runtime, error) { return nil, nil }\n\ntype Runtime struct{ *control.Runtime }\n"}, + } { + p := filepath.Join(root, f.rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(f.body), 0o600); err != nil { + t.Fatal(err) + } + } + noReExport := packagepolicy.RepoNoReExport() + if len(noReExport) == 0 { + t.Fatal("the postgres-root no-re-export rule is missing from the table") + } + if errs := packagepolicy.CheckNoReExport(root, noReExport[0]); len(errs) != 0 { + t.Fatalf("legal composition must pass the no-re-export check: %v", errs) + } + if err := os.WriteFile(filepath.Join(root, "internal/backends/postgres/aliases.go"), + []byte("package postgres\n\nimport \"github.com/calypr/loom/internal/backends/postgres/control\"\n\ntype Config = control.Config\n"), 0o600); err != nil { + t.Fatal(err) + } + foundAlias := false + for _, e := range packagepolicy.CheckNoReExport(root, noReExport[0]) { + if strings.Contains(e.Reason, "Config") { + foundAlias = true + } + } + if !foundAlias { + t.Fatal("re-exported control.Config from root must be caught") + } + idRule := packagepolicy.AbsenceRule{Label: "removed untyped row-querier bridge", Owner: "typed Querier", + Identifiers: []string{"QuerierAsAny"}, ScanRoots: []string{"internal/backends/postgres"}} + if err := os.WriteFile(filepath.Join(root, "internal/backends/postgres/bridge.go"), + []byte("package postgres\n\nfunc QuerierAsAny() any { return nil }\n"), 0o600); err != nil { + t.Fatal(err) + } + if errs := packagepolicy.CheckAbsences(root, []packagepolicy.AbsenceRule{idRule}); len(errs) == 0 { + t.Fatal("planted QuerierAsAny must fail the identifier-absence rule") + } + if err := os.Remove(filepath.Join(root, "internal/backends/postgres/bridge.go")); err != nil { + t.Fatal(err) + } + + cycleRule := guardRule(t, "postgres-control: cycle-free control plane") + cdir := t.TempDir() + guardWrite(t, cdir, "control/reaches_root.go", "github.com/calypr/loom/internal/backends/postgres") + guardWrite(t, cdir, "control/reaches_rows.go", "github.com/calypr/loom/internal/backends/postgres/internal/rows") + res := guardScan(t, cycleRule, filepath.Join(cdir, "control"), packagepolicy.AllScopes) + requireFinding(t, res, cycleRule.Label, "reaches_root.go", "internal/backends/postgres") + requireFinding(t, res, cycleRule.Label, "reaches_rows.go", "internal/backends/postgres/internal/rows") + requireClean(t, packagepolicy.ScanRule(repoRoot, cycleRule)) + }) +} + +func hasErrorKind(errors []packagepolicy.ScanError, kind packagepolicy.ErrorKind) bool { + for _, e := range errors { + if e.Kind == kind { + return true + } + } + return false +} diff --git a/internal/dataframe/compiler/capability/probe.go b/internal/dataframe/compiler/capability/probe.go index 9aac35adf..553cb63f5 100644 --- a/internal/dataframe/compiler/capability/probe.go +++ b/internal/dataframe/compiler/capability/probe.go @@ -1,10 +1,3 @@ -// Package capability contains compiler-backed capability probes. -// -// A probe is deliberately expressed in schema and semantic terms. It does -// not know about catalog rows, Explorer wire types, or a particular storage -// adapter. A successful probe is an executable proof: the request passed the -// generated schema, semantic validation, physical lowering, optimization, -// and canonical AQL renderer. package capability import ( @@ -24,10 +17,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// Scope is the request provenance carried into every successful proof. The -// same values are placed in the physical plan and consequently in rendered -// bind variables; consumers must not treat a query without this provenance as -// an advertised capability. type Scope struct { Project string DatasetGeneration string @@ -35,9 +24,6 @@ type Scope struct { AuthScopeMode authscope.ReadScopeMode } -// Traversal describes one authoring-direction relationship. The source, -// label, and target are all required so an observed edge cannot be silently -// matched to an ambiguous generated relationship. type Traversal struct { FromResourceType string EdgeLabel string @@ -46,9 +32,6 @@ type Traversal struct { MatchMode spec.TraversalMatchMode } -// Filter describes an optional candidate filter operation. Values are typed -// semantic values, not query fragments. When Values is empty, ProbeCandidate -// supplies a harmless typed sample solely to exercise the compiler path. type Filter struct { Operator spec.FilterOperator Quantifier spec.ArrayQuantifier @@ -56,30 +39,22 @@ type Filter struct { Values []spec.FilterValue } -// Chart describes an optional candidate chart/aggregate operation. type Chart struct { Operation recipe.AggregateOperation RequiredValues []string } -// RootRequest asks whether one concrete generated resource can be a row root. type RootRequest struct { Scope ResourceType string } -// TraversalRequest asks whether one exact relationship can be lowered and -// rendered from a concrete root. It proves storage direction as well as the -// generated logical relationship. type TraversalRequest struct { Scope RootResourceType string Traversal } -// CandidateRequest asks whether a selector and its requested operation can be -// compiled at an occurrence. Route is optional for a zero-hop candidate and -// is a sequence of exact one-hop traversals for a candidate on a related node. type CandidateRequest struct { Scope RootResourceType string @@ -92,8 +67,6 @@ type CandidateRequest struct { Chart *Chart } -// OwnerRecordsRequest asks whether one repeated FHIR owner binding can be -// preserved as an ordered list of structured records at an occurrence. type OwnerRecordsRequest struct { Scope RootResourceType string @@ -103,8 +76,6 @@ type OwnerRecordsRequest struct { Key fhirschema.CorrelatedKey } -// Options controls the explicitly authorized physical rewrite. The default is -// the same conservative policy used by normal recipe compilation. type Options struct { Policy ir.PhysicalOptimizationPolicy } @@ -116,7 +87,6 @@ func (o Options) normalized() Options { return o } -// Rendered is the executable proof and its scope provenance. type Rendered struct { Query string BindVars map[string]any @@ -176,7 +146,6 @@ type Result struct { Rendered Rendered } -// ProbeRoot compiles and renders a zero-hop concrete row root. func ProbeRoot(ctx context.Context, request RootRequest, options ...Options) (Result, error) { if err := contextErr(ctx); err != nil { return Result{}, err @@ -200,8 +169,6 @@ func ProbeRoot(ctx context.Context, request RootRequest, options ...Options) (Re return Result{Root: &capability, Rendered: rendered}, nil } -// ProbeTraversal compiles one exact one-hop route and reports the direction -// selected by the storage-route proof in the shared lowerer. func ProbeTraversal(ctx context.Context, request TraversalRequest, options ...Options) (Result, error) { if strings.TrimSpace(request.Traversal.FromResourceType) == "" { request.Traversal.FromResourceType = request.RootResourceType @@ -235,8 +202,6 @@ func ProbeTraversal(ctx context.Context, request TraversalRequest, options ...Op return Result{Traversal: &capability, Rendered: rendered}, nil } -// ProbeProjection proves the requested projection without probing unrelated -// filter and chart operations during field discovery. func ProbeProjection(ctx context.Context, request CandidateRequest, options ...Options) (Result, error) { prepared, err := prepareCandidate(request) if err != nil { @@ -259,9 +224,6 @@ func ProbeProjection(ctx context.Context, request CandidateRequest, options ...O return Result{Candidate: &metadata, Rendered: rendered}, nil } -// ProbeCandidate compiles a candidate projection, filter, or chart operation. -// It also probes every operation supported by the shared compiler so the -// returned metadata can be used directly by a capability builder. func ProbeCandidate(ctx context.Context, request CandidateRequest, options ...Options) (Result, error) { optionsValue := optionsFor(options) prepared, err := prepareCandidate(request) @@ -331,8 +293,6 @@ func ProbeCandidate(ctx context.Context, request CandidateRequest, options ...Op return Result{Candidate: &metadata, Rendered: rendered}, nil } -// ProbeOwnerRecords compiles the exact binding and terminology key through -// semantic lowering, physical validation, optimization, and AQL rendering. func ProbeOwnerRecords(ctx context.Context, request OwnerRecordsRequest, options ...Options) (Result, error) { if err := contextErr(ctx); err != nil { return Result{}, err diff --git a/internal/dataframe/compiler/capability/schema_fields.go b/internal/dataframe/compiler/capability/schema_fields.go index b0b071a18..93025301d 100644 --- a/internal/dataframe/compiler/capability/schema_fields.go +++ b/internal/dataframe/compiler/capability/schema_fields.go @@ -8,9 +8,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// SchemaFieldDescriptor is structural metadata from the active generated FHIR -// definitions. It deliberately carries neither population observations nor -// compiler-probed operations; route selection owns the exact compiler proof. type SchemaFieldDescriptor struct { ResourceType string `json:"resourceType"` Path string `json:"path"` @@ -19,9 +16,6 @@ type SchemaFieldDescriptor struct { RepeatedPaths []string `json:"repeatedPaths,omitempty"` } -// SchemaFieldDescriptors returns bounded-depth generated primitive paths for -// one concrete resource type. Arrays remain MANY at every nested scalar path, -// and repeated path identities are reported without inferred widths. func SchemaFieldDescriptors(resourceType string) []SchemaFieldDescriptor { canonical, ok := fhirschema.ConcreteResourceType(resourceType) if !ok { diff --git a/internal/dataframe/compiler/category_scan.go b/internal/dataframe/compiler/category_scan.go index 314abe2f4..35fb006b4 100644 --- a/internal/dataframe/compiler/category_scan.go +++ b/internal/dataframe/compiler/category_scan.go @@ -10,10 +10,12 @@ import ( "sort" "strings" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/compiler/optimize" "github.com/calypr/loom/internal/dataframe/compiler/render/aql" + postgresrender "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" "github.com/calypr/loom/internal/dataframe/expression" "github.com/calypr/loom/internal/dataframe/recipe" ) @@ -57,9 +59,6 @@ func CategoryScanRefusalCodeOf(err error) (CategoryScanRefusalCode, bool) { return refusal.Code, true } -// CategoryScanProof binds a scan to the exact finalized output contract and -// executable query. It is immutable compiler output, suitable for inclusion -// in a later signed receipt without reconstructing either plan or schema. type CategoryScanProof struct { Version int Output string @@ -79,13 +78,11 @@ type CategoryScanProof struct { } type CategoryOverflowWitness struct { - Query string - BindVars map[string]any + Command command.CompiledCommand } type CompiledCategoryScanQuery struct { - Query string - BindVars map[string]any + Command command.CompiledCommand PresentColumn string ValueColumn string Proof CategoryScanProof @@ -95,20 +92,25 @@ type CompiledCategoryScanQuery struct { OverflowWitness *CategoryOverflowWitness } -func CompileCategoryScanOutputWithPolicy(output lower.CompiledRecipeOutput, columnName string, maxValues int, policy ir.PhysicalOptimizationPolicy) (CompiledCategoryScanQuery, error) { +func CompileCategoryScanOutput(target Target, output lower.CompiledRecipeOutput, columnName string, maxValues int, policy ir.PhysicalOptimizationPolicy) (CompiledCategoryScanQuery, error) { + if err := target.validate(); err != nil { + return CompiledCategoryScanQuery{}, err + } columnName = strings.TrimSpace(columnName) column, err := categoryScanColumn(output.OutputSchema, columnName) if err != nil { return CompiledCategoryScanQuery{}, err } + if target.backend == command.CompiledBackendPostgreSQL { + return compilePostgreSQLCategoryScan(target, output, output.OutputSchema, column, lower.CompiledOutputColumn{}, "", "", "", maxValues, policy) + } return compileCategoryScan(output, output.OutputSchema, column, lower.CompiledOutputColumn{}, "", "", "", maxValues, policy) } -// CompileCategoryScanStageWithPolicy compiles the exact prefix ending at a -// compiler-owned construction stage and binds the proof to the selected -// category and value columns. The value column is not read by the scan, but is -// included in the proof so a result cannot be reused for another pivot pair. -func CompileCategoryScanStageWithPolicy(output lower.CompiledRecipeOutput, stageID, categoryColumnID, valueColumnID string, maxValues int, policy ir.PhysicalOptimizationPolicy) (CompiledCategoryScanQuery, error) { +func CompileCategoryScanStage(target Target, output lower.CompiledRecipeOutput, stageID, categoryColumnID, valueColumnID string, maxValues int, policy ir.PhysicalOptimizationPolicy) (CompiledCategoryScanQuery, error) { + if err := target.validate(); err != nil { + return CompiledCategoryScanQuery{}, err + } stage, found := compiledStageByID(output.Stages, stageID) if !found { return CompiledCategoryScanQuery{}, &CategoryScanRefusal{Code: CategoryScanStageUnknown, Column: stageID} @@ -160,6 +162,9 @@ func CompileCategoryScanStageWithPolicy(output lower.CompiledRecipeOutput, stage } else if stageID != recipe.ConstructionSourceProjectionID { return CompiledCategoryScanQuery{}, &CategoryScanRefusal{Code: CategoryScanStageUnknown, Column: stageID} } + if target.backend == command.CompiledBackendPostgreSQL { + return compilePostgreSQLCategoryScan(target, stageOutput, stageOutput.OutputSchema, category, value, stageID, category.ID, value.ID, maxValues, policy) + } return compileCategoryScan(stageOutput, stageOutput.OutputSchema, category, value, stageID, category.ID, value.ID, maxValues, policy) } @@ -213,10 +218,6 @@ func compileCategoryScan(output lower.CompiledRecipeOutput, schema []lower.Compi categoryScanPreservesSourceCategories(physical, stageID, categoryColumnID, column) || (stageID == "" && categoryScanDirectRootSource(physical, column)) if sourceProjectionEquivalent { - // The proof remains bound to the requested terminal stage, while the - // category query uses the cheaper source projection after proving that - // every row in the stage prefix still represents a source parent and - // carries this exact root category unchanged. physical.StageSequence = nil physical = narrowSourceCategoryScanProjection(physical, column.Name) } else { @@ -298,13 +299,15 @@ func compileCategoryScan(output lower.CompiledRecipeOutput, schema []lower.Compi witnessBinds := cloneCategoryScanBinds(witness.BindVars) witnessBinds[categoryColumnBind] = column.Name witnessBinds[categoryLimitBind] = maxValues + 1 - overflowWitness = &CategoryOverflowWitness{Query: witnessQuery, BindVars: witnessBinds} + witnessCommand, witnessCommandErr := command.NewAQLCommand(witnessQuery, witnessBinds) + if witnessCommandErr != nil { + return CompiledCategoryScanQuery{}, fmt.Errorf("tag category overflow witness command: %w", witnessCommandErr) + } + overflowWitness = &CategoryOverflowWitness{Command: witnessCommand} } } } if categoryIndex != nil && len(categoryIndex.StoredValues) != 0 { - // The category index is also the direct-root Pivot index for this path; - // do not schedule the older field-only source prewarm alongside it. previewCoveringIndex = nil } @@ -324,7 +327,11 @@ func compileCategoryScan(output lower.CompiledRecipeOutput, schema []lower.Compi QueryFingerprint: queryDigest, PresenceTracked: column.PresenceCompanionName != "" || relatedEligible || categoryScanDirectRootSource(physical, column), } if overflowWitness != nil { - proof.OverflowWitnessFingerprint, err = categoryQueryFingerprint(overflowWitness.Query, overflowWitness.BindVars) + witnessText, witnessBinds, witnessDispatchErr := overflowWitness.Command.AQLParameters() + if witnessDispatchErr != nil { + return CompiledCategoryScanQuery{}, fmt.Errorf("dispatch category overflow witness to AQL fingerprinting: %w", witnessDispatchErr) + } + proof.OverflowWitnessFingerprint, err = categoryQueryFingerprint(witnessText, witnessBinds) if err != nil { return CompiledCategoryScanQuery{}, fmt.Errorf("fingerprint category overflow witness: %w", err) } @@ -339,17 +346,83 @@ func compileCategoryScan(output lower.CompiledRecipeOutput, schema []lower.Compi if err != nil { return CompiledCategoryScanQuery{}, fmt.Errorf("fingerprint category scan proof: %w", err) } + compiledCommand, commandErr := command.NewAQLCommand(query, bindVars) + if commandErr != nil { + return CompiledCategoryScanQuery{}, fmt.Errorf("tag category scan command: %w", commandErr) + } return CompiledCategoryScanQuery{ - Query: query, BindVars: bindVars, PresentColumn: "present", ValueColumn: "value", + Command: compiledCommand, PresentColumn: "present", ValueColumn: "value", Proof: proof, Diagnostics: diagnostics, CategoryIndex: categoryIndex, PreviewCoveringIndex: previewCoveringIndex, OverflowWitness: overflowWitness, }, nil } -// categoryScanPreservesSourceCategories proves that complete distinct values -// at a construction stage equal the values on its source parents. It only -// accepts a direct scalar source projection and a contiguous prefix made of -// parent-preserving RELATED_EXPAND and row-preserving RELATED_FIELD stages. +func compilePostgreSQLCategoryScan(target Target, output lower.CompiledRecipeOutput, schema []lower.CompiledOutputColumn, column, valueColumn lower.CompiledOutputColumn, stageID, categoryColumnID, valueColumnID string, maxValues int, policy ir.PhysicalOptimizationPolicy) (CompiledCategoryScanQuery, error) { + if maxValues < 1 || maxValues > MaxCategoryScanValues { + return CompiledCategoryScanQuery{}, &CategoryScanRefusal{Code: CategoryScanInvalidLimit, Column: column.Name} + } + physical, err := categoryScanPhysicalPlan(output, policy) + if err != nil { + return CompiledCategoryScanQuery{}, err + } + presenceMarkerColumn := categoryScanPresenceMarkerColumn(physical, schema) + if physical.StageSequence != nil { + sourceProjectionEquivalent := stageID == recipe.ConstructionSourceProjectionID || + categoryScanPreservesSourceCategories(physical, stageID, categoryColumnID, column) || + (stageID == "" && categoryScanDirectRootSource(physical, column)) + if !sourceProjectionEquivalent { + return CompiledCategoryScanQuery{}, fmt.Errorf( + "category scan of construction stage %q needs a composed stage-sequence read, which has no registered PostgreSQL category terminal: %w", + stageID, postgresrender.ErrUnsupportedPostgresPlan) + } + physical.StageSequence = nil + physical = narrowSourceCategoryScanProjection(physical, column.Name) + } + physical, err = withGenericPhysicalExecutionWindow(physical, 0) + if err != nil { + return CompiledCategoryScanQuery{}, fmt.Errorf("apply category scan execution window: %w", err) + } + if physical.StageSequence == nil { + physical = withoutCategoryScanRootIdentitySort(physical) + } + rendered, err := postgresrender.RenderCategoryScan(physical, target.postgres, column.Name, presenceMarkerColumn, maxValues) + if err != nil { + return CompiledCategoryScanQuery{}, fmt.Errorf("render postgresql category scan: %w", err) + } + compiledCommand, commandErr := command.NewPostgreSQLCommand(rendered.Values.SQL, rendered.Values.Args) + if commandErr != nil { + return CompiledCategoryScanQuery{}, fmt.Errorf("tag category scan command: %w", commandErr) + } + + diagnostics := physicalPlanDiagnostics(physical) + schemaDigest, err := categoryHash(schema) + if err != nil { + return CompiledCategoryScanQuery{}, fmt.Errorf("fingerprint category scan schema: %w", err) + } + proof := CategoryScanProof{ + Version: 1, Output: output.Name, Column: column.Name, Kind: column.Kind, + Cardinality: column.Cardinality, MaxValues: maxValues, + OutputSchemaDigest: schemaDigest, PlanFingerprint: diagnostics.Fingerprint, + QueryFingerprint: postgreSQLPlanFingerprint(rendered.Values.SQL, rendered.Values.Args, rendered.Values.OutputColumns), + PresenceTracked: true, + } + if stageID != "" { + proof.Version = 2 + proof.StageID = stageID + proof.ColumnID = categoryColumnID + proof.ValueColumnID = valueColumnID + } + proof.Fingerprint, err = categoryHash(proof) + if err != nil { + return CompiledCategoryScanQuery{}, fmt.Errorf("fingerprint category scan proof: %w", err) + } + return CompiledCategoryScanQuery{ + Command: compiledCommand, + PresentColumn: rendered.PresentColumn, ValueColumn: rendered.ValueColumn, + Proof: proof, Diagnostics: diagnostics, + }, nil +} + func categoryScanPreservesSourceCategories(plan ir.PhysicalPlan, stageID, categoryColumnID string, category lower.CompiledOutputColumn) bool { sequence := plan.StageSequence if sequence == nil || stageID == "" || stageID == sequence.SourceStageID || sequence.FinalStageID != stageID || @@ -521,8 +594,6 @@ func categoryScanRootProjectionSource(plan ir.PhysicalPlan) (*ir.PhysicalRootSca correlatedVariables[set.Prepared.Variable] = struct{}{} } default: - // Top-level traversals and unnests change row grain; only nested - // correlated sets are safe before a direct root projection. return nil, nil, false } } @@ -727,7 +798,11 @@ func categoryScanIndexedRootQueries(plan ir.PhysicalPlan, category lower.Compile delete(witnessBinds, categoryLimitBind) bindVars = categoryScanPruneBindVars(bindVars, query) witnessBinds = categoryScanPruneBindVars(witnessBinds, witnessQuery) - return query, bindVars, &CategoryOverflowWitness{Query: witnessQuery, BindVars: witnessBinds}, nil + witnessCommand, witnessCommandErr := command.NewAQLCommand(witnessQuery, witnessBinds) + if witnessCommandErr != nil { + return "", nil, nil, fmt.Errorf("tag category overflow witness command: %w", witnessCommandErr) + } + return query, bindVars, &CategoryOverflowWitness{Command: witnessCommand}, nil } func categoryScanStoredQuantityParentQuery(prefix string, root *ir.PhysicalRootScan, valueExpression string, valuePath []string, bindVars map[string]any, maxValues int) (string, map[string]any, *CategoryOverflowWitness, error) { @@ -768,7 +843,11 @@ func categoryScanStoredQuantityParentQuery(prefix string, root *ir.PhysicalRootS delete(witnessBinds, categoryLimitBind) bindVars = categoryScanPruneBindVars(bindVars, query) witnessBinds = categoryScanPruneBindVars(witnessBinds, witnessQuery) - return query, bindVars, &CategoryOverflowWitness{Query: witnessQuery, BindVars: witnessBinds}, nil + witnessCommand, witnessCommandErr := command.NewAQLCommand(witnessQuery, witnessBinds) + if witnessCommandErr != nil { + return "", nil, nil, fmt.Errorf("tag category overflow witness command: %w", witnessCommandErr) + } + return query, bindVars, &CategoryOverflowWitness{Command: witnessCommand}, nil } func categoryScanUsesStoredQuantityParent(index *PreviewCoveringIndexSpec, valuePath []string) bool { diff --git a/internal/dataframe/compiler/category_scan_arango_test.go b/internal/dataframe/compiler/category_scan_arango_test.go index 45b07bf23..a673ed6da 100644 --- a/internal/dataframe/compiler/category_scan_arango_test.go +++ b/internal/dataframe/compiler/category_scan_arango_test.go @@ -10,11 +10,11 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -25,12 +25,13 @@ func TestConstructionCategoryScanPreservesMissingAndNullAgainstArango(t *testing } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Patient"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Patient"}}}); err != nil { t.Fatal(err) } @@ -126,7 +127,7 @@ func TestConstructionCategoryScanPreservesMissingAndNullAgainstArango(t *testing if err != nil { t.Fatal(err) } - categoryScan, err := CompileCategoryScanStageWithPolicy(compiled.Outputs[0], stageID, "birth_date_id", "patient_id", 10, ir.DefaultPhysicalOptimizationPolicy()) + categoryScan, err := CompileCategoryScanStage(ArangoTarget(), compiled.Outputs[0], stageID, "birth_date_id", "patient_id", 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -135,7 +136,7 @@ func TestConstructionCategoryScanPreservesMissingAndNullAgainstArango(t *testing assertMissingAndNull := func(categoryScan CompiledCategoryScanQuery) { t.Helper() - rows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: categoryScan.Query, BindVars: categoryScan.BindVars}) + rows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: categoryScan.Command}) if len(rows) != 2 { t.Fatalf("category scan returned %d categories, want distinct MISSING and NULL: %#v", len(rows), rows) } @@ -158,17 +159,17 @@ func TestConstructionCategoryScanPreservesMissingAndNullAgainstArango(t *testing equalityScan := compileScan(makeOutput("construction_category_presence_equality", "keep_active_equality", recipe.FilterEquals), "keep_active_equality") rootFilter := "FILTER root.payload.active == @construction_filter_value" stageFilter := "FILTER __loom_construction_input_1.active == @construction_filter_value" - if !strings.Contains(equalityScan.Query, rootFilter) || !strings.Contains(equalityScan.Query, stageFilter) { - t.Fatalf("eligible equality should be copied to the root and retained on its stage:\n%s", equalityScan.Query) + if !strings.Contains(testAQLText(t, equalityScan.Command), rootFilter) || !strings.Contains(testAQLText(t, equalityScan.Command), stageFilter) { + t.Fatalf("eligible equality should be copied to the root and retained on its stage:\n%s", testAQLText(t, equalityScan.Command)) } - if strings.Contains(equalityScan.Query, "MERGE(") || !strings.Contains(equalityScan.Query, "__loom_category_scan_presence") { - t.Fatalf("category scan should carry presence in a scalar marker instead of merging row objects:\n%s", equalityScan.Query) + if strings.Contains(testAQLText(t, equalityScan.Command), "MERGE(") || !strings.Contains(testAQLText(t, equalityScan.Command), "__loom_category_scan_presence") { + t.Fatalf("category scan should carry presence in a scalar marker instead of merging row objects:\n%s", testAQLText(t, equalityScan.Command)) } assertMissingAndNull(equalityScan) existsScan := compileScan(makeOutput("construction_category_presence_exists", "keep_active_exists", recipe.FilterExists), "keep_active_exists") - if strings.Contains(existsScan.Query, "FILTER root.payload.active") || !strings.Contains(existsScan.Query, "FILTER __loom_construction_input_1.active != null") { - t.Fatalf("ineligible EXISTS scan must keep its original staged filter without source pushdown:\n%s", existsScan.Query) + if strings.Contains(testAQLText(t, existsScan.Command), "FILTER root.payload.active") || !strings.Contains(testAQLText(t, existsScan.Command), "FILTER __loom_construction_input_1.active != null") { + t.Fatalf("ineligible EXISTS scan must keep its original staged filter without source pushdown:\n%s", testAQLText(t, existsScan.Command)) } assertMissingAndNull(existsScan) } @@ -180,12 +181,13 @@ func TestRootQuantityCategoryScanAugmentsStoredParentIndexAgainstArango(t *testi } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Observation"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Observation"}}}); err != nil { t.Fatal(err) } @@ -216,8 +218,6 @@ func TestRootQuantityCategoryScanAugmentsStoredParentIndexAgainstArango(t *testi {id: "other-project", resourceType: "Observation", authPath: allowedPath, project: project + "_foreign", generation: generation, includeQuantity: true, quantity: map[string]any{"code": "foreign-project"}}, {id: "other-generation", resourceType: "Observation", authPath: allowedPath, project: project, generation: generation + "_foreign", includeQuantity: true, quantity: map[string]any{"code": "foreign-generation"}}, {id: "denied-scope", resourceType: "Observation", authPath: "/category/denied", project: project, generation: generation, includeQuantity: true, quantity: map[string]any{"code": "denied"}}, - // ResourceType mismatches cannot be produced by the supported FHIR - // ingest path; this guard makes the shared type-ordered index explicit. {id: "wrong-type", resourceType: "Patient", authPath: allowedPath, project: project, generation: generation, includeQuantity: true, quantity: map[string]any{"code": "wrong-type"}}, } documents := make([]json.RawMessage, 0, len(fixtures)) @@ -272,7 +272,7 @@ func TestRootQuantityCategoryScanAugmentsStoredParentIndexAgainstArango(t *testi if err != nil { t.Fatal(err) } - scan, err := CompileCategoryScanStageWithPolicy(compiled.Outputs[0], recipe.ConstructionSourceProjectionID, "category_id", "value_id", 10, ir.DefaultPhysicalOptimizationPolicy()) + scan, err := CompileCategoryScanStage(ArangoTarget(), compiled.Outputs[0], recipe.ConstructionSourceProjectionID, "category_id", "value_id", 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -285,19 +285,19 @@ func TestRootQuantityCategoryScanAugmentsStoredParentIndexAgainstArango(t *testi !reflect.DeepEqual(scan.CategoryIndex.Supersedes.Fields, wantLegacyFields) || !reflect.DeepEqual(scan.CategoryIndex.Supersedes.StoredValues, wantStoredValues) { t.Fatalf("root quantity category index does not exactly augment its stored-parent Pivot index: %+v", scan.CategoryIndex) } - if !strings.Contains(scan.Query, "root.resourceType == @__loom_category_resource_type") || - scan.BindVars["__loom_category_resource_type"] != "Observation" { - t.Fatalf("root category query omitted exact resource type guard: %s\n%#v", scan.Query, scan.BindVars) + if !strings.Contains(testAQLText(t, scan.Command), "root.resourceType == @__loom_category_resource_type") || + testAQLBinds(t, scan.Command)["__loom_category_resource_type"] != "Observation" { + t.Fatalf("root category query omitted exact resource type guard: %s\n%#v", testAQLText(t, scan.Command), testAQLBinds(t, scan.Command)) } - if strings.Contains(scan.Query, "UNION_DISTINCT") || !strings.Contains(scan.Query, "COLLECT ") || - !strings.Contains(scan.Query, "IS_OBJECT(__loom_category_parent)") || !strings.Contains(scan.Query, "HAS(__loom_category_parent") { - t.Fatalf("stored-parent category scan did not use one object-presence grouping query:\n%s", scan.Query) + if strings.Contains(testAQLText(t, scan.Command), "UNION_DISTINCT") || !strings.Contains(testAQLText(t, scan.Command), "COLLECT ") || + !strings.Contains(testAQLText(t, scan.Command), "IS_OBJECT(__loom_category_parent)") || !strings.Contains(testAQLText(t, scan.Command), "HAS(__loom_category_parent") { + t.Fatalf("stored-parent category scan did not use one object-presence grouping query:\n%s", testAQLText(t, scan.Command)) } if scan.PreviewCoveringIndex != nil { t.Fatalf("stored-parent category index should suppress the older field-only prewarm: %+v", scan.PreviewCoveringIndex) } - rows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: scan.Query, BindVars: scan.BindVars}) + rows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: scan.Command}) seen := map[string]bool{} for _, row := range rows { present, ok := row[scan.PresentColumn].(bool) diff --git a/internal/dataframe/compiler/category_scan_filter_pushdown.go b/internal/dataframe/compiler/category_scan_filter_pushdown.go index dc0dbe7fc..bf74209ab 100644 --- a/internal/dataframe/compiler/category_scan_filter_pushdown.go +++ b/internal/dataframe/compiler/category_scan_filter_pushdown.go @@ -6,15 +6,6 @@ import ( "github.com/calypr/loom/internal/dataframe/recipe" ) -// withCategoryScanSourceFilterPushdown duplicates leading construction -// equality filters onto direct root projections. The stage filters remain in -// place as the semantic checks; this copy lets Arango reject nonmatching root -// documents before materializing the full source projection. -// -// Pushdown stops at the first non-filter stage or any filter shape that is not -// a bound scalar equality over a direct pass-through projection. This keeps -// filters from crossing derived expressions, row reshapes, or other work that -// can change evaluation behavior. func withCategoryScanSourceFilterPushdown(plan ir.PhysicalPlan) ir.PhysicalPlan { sequence := plan.StageSequence if sequence == nil || len(sequence.Stages) == 0 || len(plan.Operations) < 2 || diff --git a/internal/dataframe/compiler/category_scan_parent_presence_arango_test.go b/internal/dataframe/compiler/category_scan_parent_presence_arango_test.go index 44d4d4008..0d736afc2 100644 --- a/internal/dataframe/compiler/category_scan_parent_presence_arango_test.go +++ b/internal/dataframe/compiler/category_scan_parent_presence_arango_test.go @@ -11,11 +11,11 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -64,7 +64,7 @@ FOR document IN Observation if !scan.Proof.PresenceTracked || scan.OverflowWitness == nil || scan.Proof.OverflowWitnessFingerprint == "" { t.Fatalf("root category scan proof = %+v, overflow witness present=%t; want tracked parent presence and a fingerprinted witness", scan.Proof, scan.OverflowWitness != nil) } - rows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: scan.Query, BindVars: scan.BindVars}) + rows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: scan.Command}) got := make(map[string]struct{}, len(rows)) for _, row := range rows { present, ok := row[scan.PresentColumn].(bool) @@ -125,13 +125,13 @@ FOR document IN Observation t.Fatalf("overflow scan proof = %+v, witness=%+v; want a presence-aware overflow witness", overflowScan.Proof, overflowScan.OverflowWitness) } witnessRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{ - Query: overflowScan.OverflowWitness.Query, BindVars: overflowScan.OverflowWitness.BindVars, + Command: overflowScan.OverflowWitness.Command, }) if len(witnessRows) != 1 || witnessRows[0][overflowScan.PresentColumn] != false || witnessRows[0][overflowScan.ValueColumn] != nil { t.Fatalf("overflow witness rows = %#v, want one MISSING category witness", witnessRows) } - overflowRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: overflowScan.Query, BindVars: overflowScan.BindVars}) + overflowRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: overflowScan.Command}) if len(overflowRows) != MaxCategoryScanValues+1 { t.Fatalf("category scan returned %d rows at the overflow boundary, want %d (256 values plus one witness row)", len(overflowRows), MaxCategoryScanValues+1) } @@ -168,7 +168,7 @@ FOR document IN Observation func insertRootQuantityCategoryFixtureRows( t *testing.T, ctx context.Context, - client *store.Client, + client *arangofactory.Client, project, generation string, fixtures []struct { id string @@ -236,7 +236,7 @@ func compileRootValueQuantityCodeCategoryScan(t *testing.T, project, generation if err != nil { t.Fatal(err) } - scan, err := CompileCategoryScanStageWithPolicy( + scan, err := CompileCategoryScanStage(ArangoTarget(), compiled.Outputs[0], recipe.ConstructionSourceProjectionID, "category_id", "value_id", MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy(), ) diff --git a/internal/dataframe/compiler/category_scan_related.go b/internal/dataframe/compiler/category_scan_related.go index ef4bd61d1..5c024307f 100644 --- a/internal/dataframe/compiler/category_scan_related.go +++ b/internal/dataframe/compiler/category_scan_related.go @@ -30,10 +30,6 @@ type relatedCategoryPlan struct { index *PreviewCoveringIndexSpec } -// compileRelatedCategoryScan accepts only a direct root source with its -// canonical project, generation, and authorization scope, followed by a -// connected chain of PRESERVE_PARENT related expansions and direct scalar -// related-field helpers. All other plans use the existing category renderer. func compileRelatedCategoryScan( plan ir.PhysicalPlan, stageID, categoryColumnID, valueColumnID string, diff --git a/internal/dataframe/compiler/category_scan_related_collection_identity_test.go b/internal/dataframe/compiler/category_scan_related_collection_identity_test.go index df43ab2c3..aa1034b4e 100644 --- a/internal/dataframe/compiler/category_scan_related_collection_identity_test.go +++ b/internal/dataframe/compiler/category_scan_related_collection_identity_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - store "github.com/calypr/loom/internal/store/arango" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/google/uuid" ) @@ -20,7 +20,7 @@ func TestRelatedCategoryReverseLookupPreservesCrossCollectionResourceTypeMatchAg } ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Specimen"}, {Name: "Patient"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -80,9 +80,6 @@ func TestRelatedCategoryReverseLookupPreservesCrossCollectionResourceTypeMatchAg insertEdge("canonical_root_patient", "Specimen", canonicalRoot, "Patient", canonicalPatient, "Specimen", "Patient") insertEdge("canonical_observation_patient", "Observation", canonicalObservation, "Patient", canonicalPatient, "Observation", "Patient") - // Keep a scoped cross-collection endpoint claiming Patient in the fixture: - // current lookup semantics accept it because its declared type and scope - // match the route, regardless of its actual collection. foreignRoot := insertDocument("Specimen", "Specimen", "root_foreign_collection", nil) foreignPatient := insertDocument("Observation", "Patient", "patient_stored_as_observation", nil) foreignObservation := insertDocument("Observation", "Observation", "observation_foreign_collection", "cross-collection-category") @@ -93,7 +90,7 @@ func TestRelatedCategoryReverseLookupPreservesCrossCollectionResourceTypeMatchAg insertEdge("missing_target", "Observation", missingTargetObservation, "Patient", key("patient_missing"), "Observation", "Patient") _, _, scanned := compileRelatedCategoryRuntimeScans(t, relatedCategoryScanOutput(relatedCategoryScanOptions{}), project, generation, MaxCategoryScanValues) - rows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: scanned.Query, BindVars: scanned.BindVars}) + rows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: scanned.Command}) got := make(map[string]struct{}, len(rows)) for _, row := range rows { present, ok := row[scanned.PresentColumn].(bool) @@ -107,6 +104,6 @@ func TestRelatedCategoryReverseLookupPreservesCrossCollectionResourceTypeMatchAg relatedCategoryValueKey(true, "cross-collection-category"): {}, } if !reflect.DeepEqual(got, want) { - t.Fatalf("related categories = %v, want canonical plus the scoped cross-collection Patient claim and no missing-target category %v; query:\n%s", got, want, scanned.Query) + t.Fatalf("related categories = %v, want canonical plus the scoped cross-collection Patient claim and no missing-target category %v; query:\n%s", got, want, testAQLText(t, scanned.Command)) } } diff --git a/internal/dataframe/compiler/category_scan_related_presence_arango_test.go b/internal/dataframe/compiler/category_scan_related_presence_arango_test.go index 863a58ce4..62753aad6 100644 --- a/internal/dataframe/compiler/category_scan_related_presence_arango_test.go +++ b/internal/dataframe/compiler/category_scan_related_presence_arango_test.go @@ -10,15 +10,15 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/recipe" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) func TestRelatedConstructionPivotTracksMissingNullAndStringAgainstArango(t *testing.T) { ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Specimen"}, {Name: "Patient"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -102,12 +102,9 @@ func TestRelatedConstructionPivotTracksMissingNullAndStringAgainstArango(t *test map[string]any{"code": nil, "value": 20}) addObservation("observation-absent-code", absentCodePatient, project, generation, visible, project, generation, visible, map[string]any{"value": 30}) - // The entire nested valueQuantity parent is absent for this terminal. observationKey := insertResource(project, generation, "Observation", "observation-absent-parent", visible, map[string]any{"status": "final"}) insertEdge(project, generation, "observation-absent-parent-patient", "Observation", observationKey, "Patient", absentParentPatient, visible) - // These linked rows are plausible route candidates but fail one of the - // independent terminal or edge scope predicates. addObservation("observation-foreign-project", stringPatient, foreignProject, generation, visible, project, generation, visible, map[string]any{"code": "forged-project", "value": 901}) addObservation("observation-stale-generation", stringPatient, project, generation+"-old", visible, project, generation, visible, @@ -127,10 +124,7 @@ func TestRelatedConstructionPivotTracksMissingNullAndStringAgainstArango(t *test t.Fatalf("related presence scan proof = %#v, want a complete tracked query without a conclusive-MISSING overflow witness", categoryScan.Proof) } - // This raw traversal is deliberately independent of the compiler plans. It - // models the two PRESERVE_PARENT boundaries and the exact terminal field. - oracle := CompiledQuery{ - Query: `FOR root IN Specimen + oracleCommand := testAQLCommand(`FOR root IN Specimen FILTER root.project == @project AND root.dataset_generation == @generation AND root.auth_resource_path IN @auth_paths AND root._key IN @root_keys LET patients = ( @@ -161,12 +155,11 @@ func TestRelatedConstructionPivotTracksMissingNullAndStringAgainstArango(t *test LET category = present AND terminal != null ? quantity.code : null LET amount = terminal == null ? null : quantity.value RETURN { specimen_id: root.id, present, category, amount } -`, - BindVars: map[string]any{ - "project": project, "generation": generation, "auth_paths": []string{visible}, - "root_keys": []string{rootKeys["root-string"], rootKeys["root-null"], rootKeys["root-absent-code"], rootKeys["root-absent-parent"], rootKeys["root-no-observation"]}, - }, - } +`, map[string]any{ + "project": project, "generation": generation, "auth_paths": []string{visible}, + "root_keys": []string{rootKeys["root-string"], rootKeys["root-null"], rootKeys["root-absent-code"], rootKeys["root-absent-parent"], rootKeys["root-no-observation"]}}, + ) + oracle := CompiledQuery{Command: oracleCommand} rawRows := executeReshapeOracleQuery(t, ctx, client, oracle) if len(rawRows) != 5 { t.Fatalf("independent scoped related rows = %#v, want five visible roots including the preserved no-target row", rawRows) @@ -226,7 +219,7 @@ func TestRelatedConstructionPivotTracksMissingNullAndStringAgainstArango(t *test t.Fatalf("preserved no-target row should remain in its group with NULL Pivot cells: %#v", noTarget) } - discoveredRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: categoryScan.Query, BindVars: categoryScan.BindVars}) + discoveredRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: categoryScan.Command}) gotCategoryKeys := make(map[string]bool, len(discoveredRows)) for _, row := range discoveredRows { present, ok := row[categoryScan.PresentColumn].(bool) @@ -246,7 +239,7 @@ func TestRelatedConstructionPivotTracksMissingNullAndStringAgainstArango(t *test t.Fatalf("compiled category scan returned %d rows, want distinct MISSING, NULL, and STRING d: %#v", len(discoveredRows), discoveredRows) } - pivotQuery, err := CompileRecipeOutputWithPolicy(compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + pivotQuery, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile final typed-MISSING related Pivot: %v", err) } @@ -276,12 +269,10 @@ func TestRelatedConstructionPivotTracksMissingNullAndStringAgainstArango(t *test } } - // A NULL-only authored heading accepts both explicit NULL and the - // PRESERVE_PARENT no-target value without requiring a MISSING category. nullOnlyOutput := relatedPresenceNullOnlyPivotOutput(recipe.PivotUnlistedCategoryError, "root-null", "root-no-observation") nullOnlyCompiled, _, _ := compileRelatedCategoryRuntimeScansWithBindings(t, nullOnlyOutput, bindings, MaxCategoryScanValues) - nullOnlyQuery, err := CompileRecipeOutputWithPolicy(nullOnlyCompiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + nullOnlyQuery, err := CompileRecipeOutput(ArangoTarget(), nullOnlyCompiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile NULL-only related Pivot: %v", err) } @@ -308,19 +299,15 @@ func TestRelatedConstructionPivotTracksMissingNullAndStringAgainstArango(t *test t.Fatalf("NULL-only Pivot cells = %#v, want %#v", gotNullOnly, wantNullOnly) } - // Isolate a MISSING category beside valid NULL rows so ERROR can only be - // justified by the absent code, not by the fixture's STRING d category. nullOnlyErrorOutput := relatedPresenceNullOnlyPivotOutput(recipe.PivotUnlistedCategoryError, "root-null", "root-absent-code", "root-no-observation") nullOnlyErrorCompiled, _, _ := compileRelatedCategoryRuntimeScansWithBindings(t, nullOnlyErrorOutput, bindings, MaxCategoryScanValues) - nullOnlyErrorQuery, err := CompileRecipeOutputWithPolicy(nullOnlyErrorCompiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + nullOnlyErrorQuery, err := CompileRecipeOutput(ArangoTarget(), nullOnlyErrorCompiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile NULL-only related Pivot with unlisted error: %v", err) } - err = client.QueryRows(ctx, nullOnlyErrorQuery.Query, 100, nullOnlyErrorQuery.BindVars, func(map[string]any) error { return nil }) - if err == nil || !strings.Contains(err.Error(), "TABLE_PIVOT_UNLISTED_CATEGORY") { - t.Fatalf("NULL-only Pivot with an absent category returned error %v, want TABLE_PIVOT_UNLISTED_CATEGORY", err) - } + err = client.QueryRows(ctx, testAQLText(t, nullOnlyErrorQuery.Command), 100, testAQLBinds(t, nullOnlyErrorQuery.Command), func(map[string]any) error { return nil }) + requireStableRuntimeCode(t, err, "TABLE_PIVOT_UNLISTED_CATEGORY") } func relatedPresenceNullOnlyPivotOutput(unlistedPolicy recipe.PivotUnlistedCategoryPolicy, rootIDs ...string) recipe.Output { @@ -356,7 +343,7 @@ func TestRelatedConstructionPivotPresencePlanCompiles(t *testing.T) { if !scanned.Proof.PresenceTracked || scanned.OverflowWitness != nil { t.Fatalf("compiled related presence proof = %#v, want tracked category scan without conclusive-MISSING witness", scanned.Proof) } - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile final typed-MISSING Pivot: %v", err) } @@ -366,8 +353,8 @@ func TestRelatedConstructionPivotPresencePlanCompiles(t *testing.T) { presenceColumn = stage.GroupedPivot.CategoryPresenceColumn } } - if presenceColumn == "" || !strings.Contains(query.Query, "NOT (") || !containsBindValue(query.BindVars, presenceColumn) { - t.Fatalf("compiled MISSING Pivot did not consume its typed presence companion %q:\n%s\n%#v", presenceColumn, query.Query, query.BindVars) + if presenceColumn == "" || !strings.Contains(testAQLText(t, query.Command), "NOT (") || !containsBindValue(testAQLBinds(t, query.Command), presenceColumn) { + t.Fatalf("compiled MISSING Pivot did not consume its typed presence companion %q:\n%s\n%#v", presenceColumn, testAQLText(t, query.Command), testAQLBinds(t, query.Command)) } } diff --git a/internal/dataframe/compiler/category_scan_related_test.go b/internal/dataframe/compiler/category_scan_related_test.go index 4e097fc1f..a37ed1c5e 100644 --- a/internal/dataframe/compiler/category_scan_related_test.go +++ b/internal/dataframe/compiler/category_scan_related_test.go @@ -10,21 +10,21 @@ import ( "testing" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) func TestCompileCategoryScanStreamsExactRelatedRouteWithScopeAndStageWitnesses(t *testing.T) { output := compilePopulationMappingOutput(t, relatedCategoryScanOutput(relatedCategoryScanOptions{})) - scanned, err := CompileCategoryScanStageWithPolicy(output, "add_amount", "category_id", "amount_id", MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) + scanned, err := CompileCategoryScanStage(ArangoTarget(), output, "add_amount", "category_id", "amount_id", MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - query := scanned.Query + query := testAQLText(t, scanned.Command) wantRoute := []struct{ from, to, direction string }{ {from: "Specimen", to: "Patient", direction: "OUTBOUND"}, {from: "Patient", to: "Observation", direction: "INBOUND"}, @@ -108,32 +108,32 @@ func TestCompileCategoryScanStreamsExactRelatedRouteWithScopeAndStageWitnesses(t if scanned.CategoryIndex != nil { t.Fatalf("streamed related category query should not synchronously prepare a terminal category index: %+v", scanned.CategoryIndex) } - if !containsCollectionBindValue(scanned.BindVars, "Specimen") || !containsCollectionBindValue(scanned.BindVars, "fhir_edge") { - t.Fatalf("streamed route is missing its typed root or edge collection bind: %#v", scanned.BindVars) + if !containsCollectionBindValue(testAQLBinds(t, scanned.Command), "Specimen") || !containsCollectionBindValue(testAQLBinds(t, scanned.Command), "fhir_edge") { + t.Fatalf("streamed route is missing its typed root or edge collection bind: %#v", testAQLBinds(t, scanned.Command)) } if !strings.Contains(query, "FOR __loom_related_category_stream_document_2 IN @@__loom_related_category_target_collection") || - scanned.BindVars["@__loom_related_category_target_collection"] != "Observation" { - t.Fatalf("valueQuantity.code projection is not using the exact typed Observation primary-key collection lookup: bind=%#v\n%s", scanned.BindVars["@__loom_related_category_target_collection"], query) + testAQLBinds(t, scanned.Command)["@__loom_related_category_target_collection"] != "Observation" { + t.Fatalf("valueQuantity.code projection is not using the exact typed Observation primary-key collection lookup: bind=%#v\n%s", testAQLBinds(t, scanned.Command)["@__loom_related_category_target_collection"], query) } if !strings.Contains(query, "IS_SAME_COLLECTION(@related_expand_1_hop_1_target_type, __loom_related_category_stream_edge_2._from)") || !strings.Contains(query, "LET __loom_related_category_stream_fallback_document_2 = DOCUMENT(__loom_related_category_stream_edge_2._from)") { t.Fatalf("projected category lookup must retain the collection-identity fallback for a resourceType stored in another collection:\n%s", query) } for _, routeValue := range []string{"Specimen", "Patient", "Observation", "subject_Patient", "fhir_edge"} { - if !relatedCategoryHasBindValue(scanned.BindVars, routeValue) { - t.Errorf("typed related route did not bind exact route value %q: %#v", routeValue, scanned.BindVars) + if !relatedCategoryHasBindValue(testAQLBinds(t, scanned.Command), routeValue) { + t.Errorf("typed related route did not bind exact route value %q: %#v", routeValue, testAQLBinds(t, scanned.Command)) } } - assertCategoryScanBindVarsMatchQuery(t, scanned.Query, scanned.BindVars) + assertCategoryScanBindVarsMatchQuery(t, scanned.Command) if scanned.OverflowWitness != nil { if scanned.Proof.OverflowWitnessFingerprint == "" { t.Fatal("overflow witness is not bound into the category proof") } - assertCategoryScanBindVarsMatchQuery(t, scanned.OverflowWitness.Query, scanned.OverflowWitness.BindVars) + assertCategoryScanBindVarsMatchQuery(t, scanned.OverflowWitness.Command) } - assertCategoryScanLimitFollowsStreamedWitnesses(t, scanned.Query) - if scanned.BindVars[categoryLimitBind] != MaxCategoryScanValues+1 { - t.Fatalf("category limit = %#v, want complete-category sentinel %d", scanned.BindVars[categoryLimitBind], MaxCategoryScanValues+1) + assertCategoryScanLimitFollowsStreamedWitnesses(t, testAQLText(t, scanned.Command)) + if testAQLBinds(t, scanned.Command)[categoryLimitBind] != MaxCategoryScanValues+1 { + t.Fatalf("category limit = %#v, want complete-category sentinel %d", testAQLBinds(t, scanned.Command)[categoryLimitBind], MaxCategoryScanValues+1) } if !scanned.Proof.PresenceTracked || !strings.Contains(query, "stream_present") || !strings.Contains(query, "HAS(") { t.Fatalf("exact related-field presence is not bound into the category proof/query: proof=%#v\n%s", scanned.Proof, query) @@ -178,7 +178,7 @@ func TestRelatedCategoryScanRetainsLegacyGenerationRejectionAndFalseScopeFallbac legacy.OptimizedPlan = nil legacy.Plan = ir.ClonePhysicalPlan(legacy.Plan) legacy.Plan.BindVars["dataset_generation"] = nil - if _, err := CompileCategoryScanStageWithPolicy(legacy, "add_amount", "category_id", "amount_id", MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()); err == nil { + if _, err := CompileCategoryScanStage(ArangoTarget(), legacy, "add_amount", "category_id", "amount_id", MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()); err == nil { t.Fatal("nil dataset generation bypassed the existing typed related-field source validation") } @@ -186,18 +186,18 @@ func TestRelatedCategoryScanRetainsLegacyGenerationRejectionAndFalseScopeFallbac denied.OptimizedPlan = nil denied.Plan = ir.ClonePhysicalPlan(denied.Plan) denied.Plan.BindVars["scope_allowed"] = false - deniedScan, err := CompileCategoryScanStageWithPolicy(denied, "add_amount", "category_id", "amount_id", MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) + deniedScan, err := CompileCategoryScanStage(ArangoTarget(), denied, "add_amount", "category_id", "amount_id", MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if !strings.Contains(deniedScan.Query, "__loom_construction_stage_") || strings.Contains(deniedScan.Query, "__loom_related_category_stream_") { - t.Fatalf("scope_allowed=false physical plan bypassed its source guards or selected the streamed specialization:\n%s", deniedScan.Query) + if !strings.Contains(testAQLText(t, deniedScan.Command), "__loom_construction_stage_") || strings.Contains(testAQLText(t, deniedScan.Command), "__loom_related_category_stream_") { + t.Fatalf("scope_allowed=false physical plan bypassed its source guards or selected the streamed specialization:\n%s", testAQLText(t, deniedScan.Command)) } } func TestRelatedCategoryScanMatchesPreservedRowsAndCapsAfterEmptyWitnessAgainstArango(t *testing.T) { ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Specimen"}, {Name: "Patient"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -250,10 +250,6 @@ func TestRelatedCategoryScanMatchesPreservedRowsAndCapsAfterEmptyWitnessAgainstA } } - // s1 and s3 share one populated patient; s1 also has a sibling patient with - // no Observation. s2 has no Patient. The category query may deduplicate the - // shared patient identity, but the ordinary output must preserve each root's - // full row multiplicity and both empty boundaries. s1 := insertResource(project, "Specimen", "s1", map[string]any{"status": "available"}) insertResource(project, "Specimen", "s2", map[string]any{"status": "available"}) s3 := insertResource(project, "Specimen", "s3", map[string]any{"status": "available"}) @@ -297,13 +293,7 @@ func TestRelatedCategoryScanMatchesPreservedRowsAndCapsAfterEmptyWitnessAgainstA if want := map[string]int{"s1": 5, "s2": 1, "s3": 4}; !reflect.DeepEqual(rowsByRoot, want) { t.Fatalf("native output multiplicity by Specimen = %v, want %v; category-scan identity dedup must not affect ordinary rows", rowsByRoot, want) } - // The public no-Pivot output materializes optional related fields as nullable - // values, so its `category` property cannot distinguish a missing source - // path from explicit NULL. Discover chooser categories from an independent - // raw traversal that tracks both PRESERVE_PARENT boundaries and exact path - // presence instead. - rawCategories := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{ - Query: `FOR root IN Specimen + rawCategoriesCommand := testAQLCommand(`FOR root IN Specimen FILTER root.project == @project AND root.dataset_generation == @generation LET patients = ( FOR first_edge IN fhir_edge @@ -334,9 +324,8 @@ func TestRelatedCategoryScanMatchesPreservedRowsAndCapsAfterEmptyWitnessAgainstA LET category_value = category_present ? terminal.payload.valueQuantity.code : null COLLECT present = category_present, value = category_value RETURN { present, value } -`, - BindVars: map[string]any{"project": project, "generation": generation}, - }) +`, map[string]any{"project": project, "generation": generation}) + rawCategories := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: rawCategoriesCommand}) want := map[string]struct{}{ relatedCategoryValueKey(false, nil): {}, relatedCategoryValueKey(true, nil): {}, @@ -353,7 +342,7 @@ func TestRelatedCategoryScanMatchesPreservedRowsAndCapsAfterEmptyWitnessAgainstA if !reflect.DeepEqual(oracleCategories, want) { t.Fatalf("independent raw category states = %v, want explicit MISSING, NULL, and native string: %v", oracleCategories, want) } - actualRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: scanned.Query, BindVars: scanned.BindVars}) + actualRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: scanned.Command}) got := map[string]struct{}{} for _, row := range actualRows { present, ok := row[scanned.PresentColumn].(bool) @@ -366,7 +355,7 @@ func TestRelatedCategoryScanMatchesPreservedRowsAndCapsAfterEmptyWitnessAgainstA t.Fatalf("optimized category scan returned duplicate category rows: rows=%#v distinct=%v", actualRows, got) } if !reflect.DeepEqual(got, want) { - t.Fatalf("optimized categories = %v, independent raw categories = %v; query:\n%s", got, want, scanned.Query) + t.Fatalf("optimized categories = %v, independent raw categories = %v; query:\n%s", got, want, testAQLText(t, scanned.Command)) } if _, found := got[relatedCategoryValueKey(true, "orphan-terminal")]; found { t.Fatalf("orphan terminal leaked into related categories: %v", got) @@ -375,8 +364,6 @@ func TestRelatedCategoryScanMatchesPreservedRowsAndCapsAfterEmptyWitnessAgainstA t.Fatalf("independent raw traversal omitted its expected preserved/explicit NULL category: %v", want) } - // This isolated project has only the category "d" on terminal records, - // so NULL can only come from the empty second-stage expansion. witnessRoot := insertResource(witnessProject, "Specimen", "root", map[string]any{"status": "available"}) witnessFull := insertResource(witnessProject, "Patient", "full", map[string]any{}) witnessEmpty := insertResource(witnessProject, "Patient", "empty", map[string]any{}) @@ -385,7 +372,7 @@ func TestRelatedCategoryScanMatchesPreservedRowsAndCapsAfterEmptyWitnessAgainstA witnessObservation := insertResource(witnessProject, "Observation", "only-code", map[string]any{"valueQuantity": map[string]any{"code": "d"}}) insertEdge(witnessProject, "only-code-patient", "Observation", witnessObservation, "Patient", witnessFull) _, _, witnessScan := compileRelatedCategoryRuntimeScans(t, relatedCategoryScanOutput(relatedCategoryScanOptions{}), witnessProject, generation, MaxCategoryScanValues) - witnessRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: witnessScan.Query, BindVars: witnessScan.BindVars}) + witnessRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: witnessScan.Command}) witnessCategories := map[string]struct{}{} for _, row := range witnessRows { witnessCategories[relatedCategoryValueKey(row[witnessScan.PresentColumn].(bool), row[witnessScan.ValueColumn])] = struct{}{} @@ -397,8 +384,6 @@ func TestRelatedCategoryScanMatchesPreservedRowsAndCapsAfterEmptyWitnessAgainstA t.Fatalf("second-stage PRESERVE_PARENT NULL was lost: %v", witnessCategories) } - // This project has one complete non-NULL chain and one root with no Patient, - // isolating the first PRESERVE_PARENT boundary. firstFullRoot := insertResource(firstStageProject, "Specimen", "full-root", map[string]any{"status": "available"}) insertResource(firstStageProject, "Specimen", "empty-root", map[string]any{"status": "available"}) firstPatient := insertResource(firstStageProject, "Patient", "only-patient", map[string]any{}) @@ -406,7 +391,7 @@ func TestRelatedCategoryScanMatchesPreservedRowsAndCapsAfterEmptyWitnessAgainstA firstObservation := insertResource(firstStageProject, "Observation", "only-code", map[string]any{"valueQuantity": map[string]any{"code": "e"}}) insertEdge(firstStageProject, "only-code-patient", "Observation", firstObservation, "Patient", firstPatient) _, _, firstStageScan := compileRelatedCategoryRuntimeScans(t, relatedCategoryScanOutput(relatedCategoryScanOptions{}), firstStageProject, generation, MaxCategoryScanValues) - firstStageRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: firstStageScan.Query, BindVars: firstStageScan.BindVars}) + firstStageRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: firstStageScan.Command}) firstStageCategories := map[string]struct{}{} for _, row := range firstStageRows { firstStageCategories[relatedCategoryValueKey(row[firstStageScan.PresentColumn].(bool), row[firstStageScan.ValueColumn])] = struct{}{} @@ -418,8 +403,6 @@ func TestRelatedCategoryScanMatchesPreservedRowsAndCapsAfterEmptyWitnessAgainstA t.Fatalf("first-stage PRESERVE_PARENT NULL was lost: %v", firstStageCategories) } - // Exactly 256 distinct terminal strings plus one empty PRESERVE_PARENT - // category must return 257 rows so the scan can report overflow accurately. capSpecimen := insertResource(capProject, "Specimen", "root", map[string]any{"status": "available"}) capFull := insertResource(capProject, "Patient", "full", map[string]any{}) capEmpty := insertResource(capProject, "Patient", "empty", map[string]any{}) @@ -431,7 +414,7 @@ func TestRelatedCategoryScanMatchesPreservedRowsAndCapsAfterEmptyWitnessAgainstA insertEdge(capProject, id+"-patient", "Observation", observation, "Patient", capFull) } _, _, capScan := compileRelatedCategoryRuntimeScans(t, relatedCategoryScanOutput(relatedCategoryScanOptions{}), capProject, generation, MaxCategoryScanValues) - capRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: capScan.Query, BindVars: capScan.BindVars}) + capRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: capScan.Command}) if len(capRows) != MaxCategoryScanValues+1 { t.Fatalf("category scan returned %d rows for 256 string categories plus a preserved empty category; want sentinel %d", len(capRows), MaxCategoryScanValues+1) } @@ -439,7 +422,7 @@ func TestRelatedCategoryScanMatchesPreservedRowsAndCapsAfterEmptyWitnessAgainstA func TestRelatedCategoryScanMatchesRestrictedRootEdgeAndTargetScopeAgainstArango(t *testing.T) { ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Specimen"}, {Name: "Patient"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -516,7 +499,6 @@ func TestRelatedCategoryScanMatchesRestrictedRootEdgeAndTargetScopeAgainstArango addObservation("second-edge-denied", allowedPatient, "second-edge-denied", blocked, allowed) addObservation("terminal-denied", allowedPatient, "terminal-denied", allowed, blocked) insertResource("Observation", "orphan", allowed, map[string]any{"valueQuantity": map[string]any{"code": "orphan"}}) - // The orphan is intentionally not connected to any Patient by subject_Patient. bindings := recipe.RuntimeBindings{ Project: project, SelectionProject: project, DatasetGeneration: generation, @@ -530,8 +512,8 @@ func TestRelatedCategoryScanMatchesRestrictedRootEdgeAndTargetScopeAgainstArango "__loom_related_category_stream_edge_2.auth_resource_path IN @auth_resource_paths", "__loom_related_category_stream_document_2.auth_resource_path IN @auth_resource_paths", } { - if !strings.Contains(scanned.Query, path) { - t.Errorf("related-category query omitted scoped route component %q:\n%s", path, scanned.Query) + if !strings.Contains(testAQLText(t, scanned.Command), path) { + t.Errorf("related-category query omitted scoped route component %q:\n%s", path, testAQLText(t, scanned.Command)) } } want := map[string]struct{}{} @@ -540,7 +522,7 @@ func TestRelatedCategoryScanMatchesRestrictedRootEdgeAndTargetScopeAgainstArango value, present := row["category"] want[relatedCategoryValueKey(present, value)] = struct{}{} } - actualRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: scanned.Query, BindVars: scanned.BindVars}) + actualRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: scanned.Command}) got := map[string]struct{}{} for _, row := range actualRows { present, ok := row[scanned.PresentColumn].(bool) @@ -587,11 +569,11 @@ func compileRelatedCategoryRuntimeScansWithBindings(t *testing.T, definition rec t.Fatal(err) } output := compiled.Outputs[0] - full, err := CompileRecipeOutputWithPolicy(output, bindings, 1024, ir.DefaultPhysicalOptimizationPolicy()) + full, err := CompileRecipeOutput(ArangoTarget(), output, bindings, 1024, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - scanned, err := CompileCategoryScanStageWithPolicy(output, "add_amount", "category_id", "amount_id", maxValues, ir.DefaultPhysicalOptimizationPolicy()) + scanned, err := CompileCategoryScanStage(ArangoTarget(), output, "add_amount", "category_id", "amount_id", maxValues, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -622,22 +604,20 @@ func TestCompileRelatedCategoryScanFallsBackForUnsafeShapes(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { output := compilePopulationMappingOutput(t, relatedCategoryScanOutput(test.options)) - scanned, err := CompileCategoryScanStageWithPolicy(output, test.options.TerminalIDForTest(), "category_id", "amount_id", MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) + scanned, err := CompileCategoryScanStage(ArangoTarget(), output, test.options.TerminalIDForTest(), "category_id", "amount_id", MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if !strings.Contains(scanned.Query, "__loom_construction_stage_") || !strings.Contains(scanned.Query, "related_expand_records") { - t.Fatalf("unsafe shape bypassed the exact construction prefix:\n%s", scanned.Query) + if !strings.Contains(testAQLText(t, scanned.Command), "__loom_construction_stage_") || !strings.Contains(testAQLText(t, scanned.Command), "related_expand_records") { + t.Fatalf("unsafe shape bypassed the exact construction prefix:\n%s", testAQLText(t, scanned.Command)) } - if strings.Contains(scanned.Query, "__loom_related_category_stream_") { - t.Fatalf("unsafe shape selected the streamed related-category specialization:\n%s", scanned.Query) + if strings.Contains(testAQLText(t, scanned.Command), "__loom_related_category_stream_") { + t.Fatalf("unsafe shape selected the streamed related-category specialization:\n%s", testAQLText(t, scanned.Command)) } }) } } -// These tests keep invalid scan requests on the typed refusal path rather than -// allowing the related-route specialization to guess at a schema. func TestCompileRelatedCategoryScanRejectsMismatchedOrInactiveColumnIDs(t *testing.T) { output := compilePopulationMappingOutput(t, relatedCategoryScanOutput(relatedCategoryScanOptions{})) for _, test := range []struct { @@ -647,7 +627,7 @@ func TestCompileRelatedCategoryScanRejectsMismatchedOrInactiveColumnIDs(t *testi {stage: "add_amount", category: "stale_category", value: "amount_id"}, {stage: "add_amount", category: "category_id", value: "stale_value"}, } { - _, err := CompileCategoryScanStageWithPolicy(output, test.stage, test.category, test.value, MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) + _, err := CompileCategoryScanStage(ArangoTarget(), output, test.stage, test.category, test.value, MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) var refusal *CategoryScanRefusal if !errors.As(err, &refusal) { t.Errorf("scan (%q, %q, %q) error = %v, want typed refusal", test.stage, test.category, test.value, err) @@ -697,8 +677,6 @@ type relatedCategoryScanOptions struct { secondEmptyPolicy recipe.ExpansionEmptyPolicy } -// TerminalIDForTest returns the final stage ID for the matching recipe shape. -// It lives on the options type so negative cases scan the same stage they build. func (options relatedCategoryScanOptions) TerminalIDForTest() string { if options.valueFromPatient { return "add_category" diff --git a/internal/dataframe/compiler/category_scan_test.go b/internal/dataframe/compiler/category_scan_test.go index b26479db3..5ec8f8ab0 100644 --- a/internal/dataframe/compiler/category_scan_test.go +++ b/internal/dataframe/compiler/category_scan_test.go @@ -2,12 +2,15 @@ package compiler import ( "errors" + "fmt" "reflect" "strings" "testing" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" + postgresrender "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" "github.com/calypr/loom/internal/dataframe/expression" "github.com/calypr/loom/internal/dataframe/recipe" ) @@ -28,7 +31,7 @@ func TestCompileCategoryScanWrapsCompleteFinalOutputWithBoundedDistinct(t *testi MissingInputPolicy: recipe.MissingInputError, }}, }) - compiled, err := CompileCategoryScanOutputWithPolicy(output, "doubled", 256, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileCategoryScanOutput(ArangoTarget(), output, "doubled", 256, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -37,26 +40,26 @@ func TestCompileCategoryScanWrapsCompleteFinalOutputWithBoundedDistinct(t *testi "__loom_category_row[@__loom_category_column]", "COLLECT __loom_category_group_present", "SORT __loom_category_group_present ASC", "LIMIT @__loom_category_limit", } { - if !strings.Contains(compiled.Query, want) { - t.Fatalf("category query missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Fatalf("category query missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } - if strings.Contains(compiled.Query, "[\"doubled\"]") { - t.Fatalf("category column was interpolated into AQL:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), "[\"doubled\"]") { + t.Fatalf("category column was interpolated into AQL:\n%s", testAQLText(t, compiled.Command)) } - if got := compiled.BindVars[categoryColumnBind]; got != "doubled" { + if got := testAQLBinds(t, compiled.Command)[categoryColumnBind]; got != "doubled" { t.Fatalf("column bind = %#v", got) } - if got := compiled.BindVars[categoryLimitBind]; got != 257 { + if got := testAQLBinds(t, compiled.Command)[categoryLimitBind]; got != 257 { t.Fatalf("limit bind = %#v", got) } - collectAt := strings.LastIndex(compiled.Query, "COLLECT __loom_category_group_present") - limitAt := strings.LastIndex(compiled.Query, "LIMIT @__loom_category_limit") + collectAt := strings.LastIndex(testAQLText(t, compiled.Command), "COLLECT __loom_category_group_present") + limitAt := strings.LastIndex(testAQLText(t, compiled.Command), "LIMIT @__loom_category_limit") if collectAt < 0 || limitAt < collectAt { - t.Fatalf("distinct must precede category limit:\n%s", compiled.Query) + t.Fatalf("distinct must precede category limit:\n%s", testAQLText(t, compiled.Command)) } - if strings.Contains(compiled.Query[:collectAt], "LIMIT @limit") { - t.Fatalf("category scan retained a preview/root limit before distinct:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command)[:collectAt], "LIMIT @limit") { + t.Fatalf("category scan retained a preview/root limit before distinct:\n%s", testAQLText(t, compiled.Command)) } if compiled.Proof.OutputSchemaDigest == "" || compiled.Proof.PlanFingerprint == "" || compiled.Proof.QueryFingerprint == "" || compiled.Proof.Fingerprint == "" { t.Fatalf("incomplete proof: %#v", compiled.Proof) @@ -87,7 +90,7 @@ func TestCompileCategoryScanRefusesNonPublicOrNonScalarColumns(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := CompileCategoryScanOutputWithPolicy(base, test.name, 256, ir.DefaultPhysicalOptimizationPolicy()) + _, err := CompileCategoryScanOutput(ArangoTarget(), base, test.name, 256, ir.DefaultPhysicalOptimizationPolicy()) var refusal *CategoryScanRefusal if !errors.As(err, &refusal) || refusal.Code != test.code { t.Fatalf("error = %#v, want %s", err, test.code) @@ -104,14 +107,14 @@ func TestCompileCategoryScanProofChangesWithSchemaAndSelectedColumn(t *testing.T Name: "Patients", RootResourceType: "Patient", RowGrain: "patient", RootColumnNaming: recipe.RootColumnNamingExact, Fields: []recipe.Field{{Name: "id", Expr: recipe.Expression{Select: "root.id"}}, {Name: "gender", Expr: recipe.Expression{Select: "root.gender"}}}, }) - left, err := CompileCategoryScanOutputWithPolicy(output, "gender", 10, ir.DefaultPhysicalOptimizationPolicy()) + left, err := CompileCategoryScanOutput(ArangoTarget(), output, "gender", 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } rightOutput := output rightOutput.OutputSchema = lower.CloneCompiledOutputSchema(output.OutputSchema) rightOutput.OutputSchema[0].SemanticPath += ":changed" - right, err := CompileCategoryScanOutputWithPolicy(rightOutput, "gender", 10, ir.DefaultPhysicalOptimizationPolicy()) + right, err := CompileCategoryScanOutput(ArangoTarget(), rightOutput, "gender", 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -122,12 +125,12 @@ func TestCompileCategoryScanProofChangesWithSchemaAndSelectedColumn(t *testing.T func TestCompileCategoryScanUsesExactConstructionStagePrefixAndBindsPivotPair(t *testing.T) { output := compilePopulationMappingOutput(t, categoryScanConstructionRecipeOutput()) - scanned, err := CompileCategoryScanStageWithPolicy(output, "derive_total", "status_id", "total_id", 256, ir.DefaultPhysicalOptimizationPolicy()) + scanned, err := CompileCategoryScanStage(ArangoTarget(), output, "derive_total", "status_id", "total_id", 256, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if strings.Count(scanned.Query, "LET __loom_construction_stage_") != 1 || strings.Contains(scanned.Query, "__loom_construction_stage_2") { - t.Fatalf("category scan did not stop at derive_total:\n%s", scanned.Query) + if strings.Count(testAQLText(t, scanned.Command), "LET __loom_construction_stage_") != 1 || strings.Contains(testAQLText(t, scanned.Command), "__loom_construction_stage_2") { + t.Fatalf("category scan did not stop at derive_total:\n%s", testAQLText(t, scanned.Command)) } proof := scanned.Proof if proof.Version != 2 || proof.Output != output.Name || proof.StageID != "derive_total" || proof.ColumnID != "status_id" || proof.ValueColumnID != "total_id" || proof.Column != "status" || proof.MaxValues != 256 { @@ -136,7 +139,7 @@ func TestCompileCategoryScanUsesExactConstructionStagePrefixAndBindsPivotPair(t if proof.OutputSchemaDigest == "" || proof.PlanFingerprint == "" || proof.QueryFingerprint == "" || proof.Fingerprint == "" { t.Fatalf("incomplete stage scan proof: %#v", proof) } - otherPair, err := CompileCategoryScanStageWithPolicy(output, "derive_total", "status_id", "amount_id", 256, ir.DefaultPhysicalOptimizationPolicy()) + otherPair, err := CompileCategoryScanStage(ArangoTarget(), output, "derive_total", "status_id", "amount_id", 256, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -152,7 +155,7 @@ func TestCompileCategoryScanUsesExactConstructionStagePrefixAndBindsPivotPair(t {stage: "derive_total", category: "stale_category", value: "total_id", want: CategoryScanColumnUnknown}, {stage: "derive_total", category: "status_id", value: "stale_value", want: CategoryScanColumnUnknown}, } { - _, err := CompileCategoryScanStageWithPolicy(output, test.stage, test.category, test.value, 256, ir.DefaultPhysicalOptimizationPolicy()) + _, err := CompileCategoryScanStage(ArangoTarget(), output, test.stage, test.category, test.value, 256, ir.DefaultPhysicalOptimizationPolicy()) var refusal *CategoryScanRefusal if !errors.As(err, &refusal) || refusal.Code != test.want { t.Errorf("stage scan (%q, %q, %q) error = %v, want refusal %s", test.stage, test.category, test.value, err, test.want) @@ -185,33 +188,33 @@ func TestCompileCategoryScanTracksPresenceThroughFilterForOrdinaryDiscoveryProbe output.Construction = &construction compiled := compilePopulationMappingOutput(t, output) - scanned, err := CompileCategoryScanStageWithPolicy(compiled, "keep_positive", "status_id", "amount_id", 256, ir.DefaultPhysicalOptimizationPolicy()) + scanned, err := CompileCategoryScanStage(ArangoTarget(), compiled, "keep_positive", "status_id", "amount_id", 256, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - companion, ok := scanned.BindVars["__loom_category_presence_column"].(string) + companion, ok := testAQLBinds(t, scanned.Command)["__loom_category_presence_column"].(string) if !ok || companion == "" || !scanned.Proof.PresenceTracked { - t.Fatalf("ordinary temporary Pivot probe lost source presence across its filtered prefix: proof=%#v binds=%#v\n%s", scanned.Proof, scanned.BindVars, scanned.Query) + t.Fatalf("ordinary temporary Pivot probe lost source presence across its filtered prefix: proof=%#v binds=%#v\n%s", scanned.Proof, testAQLBinds(t, scanned.Command), testAQLText(t, scanned.Command)) } for _, marker := range []string{"__loom_construction_stage_", "FILTER __loom_construction_input_2.total", companion, "__loom_category_presence_column"} { - if !strings.Contains(scanned.Query, marker) { - t.Fatalf("filtered source-presence category query omitted %q:\n%s", marker, scanned.Query) + if !strings.Contains(testAQLText(t, scanned.Command), marker) { + t.Fatalf("filtered source-presence category query omitted %q:\n%s", marker, testAQLText(t, scanned.Command)) } } } func TestCompileCategoryScanCollapsesParentPreservingRelatedPrefixToRootCategories(t *testing.T) { output := compilePopulationMappingOutput(t, categoryScanRelatedRecipeOutput(recipe.ExpansionPreserveParent, false, false)) - scanned, err := CompileCategoryScanStageWithPolicy(output, "add_status", "category_id", "value_id", 256, ir.DefaultPhysicalOptimizationPolicy()) + scanned, err := CompileCategoryScanStage(ArangoTarget(), output, "add_status", "category_id", "value_id", 256, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if strings.Contains(scanned.Query, "__loom_construction_stage_") || strings.Contains(scanned.Query, "related_expand_records") || - strings.Contains(scanned.Query, "related_field_document") { - t.Fatalf("parent-preserving related prefix was not narrowed to the direct root category scan:\n%s", scanned.Query) + if strings.Contains(testAQLText(t, scanned.Command), "__loom_construction_stage_") || strings.Contains(testAQLText(t, scanned.Command), "related_expand_records") || + strings.Contains(testAQLText(t, scanned.Command), "related_field_document") { + t.Fatalf("parent-preserving related prefix was not narrowed to the direct root category scan:\n%s", testAQLText(t, scanned.Command)) } - if strings.Contains(scanned.Query, "LET __loom_category_rows =") { - t.Fatalf("direct root category scan should stream its terminal source projection:\n%s", scanned.Query) + if strings.Contains(testAQLText(t, scanned.Command), "LET __loom_category_rows =") { + t.Fatalf("direct root category scan should stream its terminal source projection:\n%s", testAQLText(t, scanned.Command)) } wantFields := []string{"project", "dataset_generation", "resourceType", "payload.gender", "auth_resource_path"} wantLegacyFields := []string{"project", "dataset_generation", "payload.gender", "auth_resource_path"} @@ -222,9 +225,9 @@ func TestCompileCategoryScanCollapsesParentPreservingRelatedPrefixToRootCategori !reflect.DeepEqual(scanned.CategoryIndex.Supersedes.Fields, wantLegacyFields) { t.Fatalf("direct root category index replacement = %+v, want only its exact legacy category index", scanned.CategoryIndex.Supersedes) } - if !strings.Contains(scanned.Query, "forceIndexHint: false") || !strings.Contains(scanned.Query, scanned.CategoryIndex.Name) || - !strings.Contains(scanned.Query, "root.resourceType == @__loom_category_resource_type") || scanned.BindVars["__loom_category_resource_type"] != "Patient" { - t.Fatalf("direct root category query omitted its non-forcing shared index hint or exact root type guard:\n%s\n%#v", scanned.Query, scanned.BindVars) + if !strings.Contains(testAQLText(t, scanned.Command), "forceIndexHint: false") || !strings.Contains(testAQLText(t, scanned.Command), scanned.CategoryIndex.Name) || + !strings.Contains(testAQLText(t, scanned.Command), "root.resourceType == @__loom_category_resource_type") || testAQLBinds(t, scanned.Command)["__loom_category_resource_type"] != "Patient" { + t.Fatalf("direct root category query omitted its non-forcing shared index hint or exact root type guard:\n%s\n%#v", testAQLText(t, scanned.Command), testAQLBinds(t, scanned.Command)) } if scanned.OverflowWitness == nil { t.Fatal("direct root category scan omitted its exact missing-value witness") @@ -235,21 +238,21 @@ func TestCompileCategoryScanCollapsesParentPreservingRelatedPrefixToRootCategori "FILTER root_scope_allowed == @scope_allowed", "FILTER root.resourceType == @__loom_category_resource_type", } { - if !strings.Contains(scanned.Query, guard) || !strings.Contains(scanned.OverflowWitness.Query, guard) { - t.Errorf("root category query or missing witness lost the exact scope/type guard %q:\nquery:\n%s\nwitness:\n%s", guard, scanned.Query, scanned.OverflowWitness.Query) + if !strings.Contains(testAQLText(t, scanned.Command), guard) || !strings.Contains(testAQLText(t, scanned.OverflowWitness.Command), guard) { + t.Errorf("root category query or missing witness lost the exact scope/type guard %q:\nquery:\n%s\nwitness:\n%s", guard, testAQLText(t, scanned.Command), testAQLText(t, scanned.OverflowWitness.Command)) } } - if scanned.BindVars["@root_collection"] != "Patient" || scanned.BindVars["auth_resource_paths_unrestricted"] != true { - t.Fatalf("root category query changed its typed collection or auth-scope bindings: %#v", scanned.BindVars) + if testAQLBinds(t, scanned.Command)["@root_collection"] != "Patient" || testAQLBinds(t, scanned.Command)["auth_resource_paths_unrestricted"] != true { + t.Fatalf("root category query changed its typed collection or auth-scope bindings: %#v", testAQLBinds(t, scanned.Command)) } - if !strings.Contains(scanned.Query, "LET __loom_category_nonnull = (") || !strings.Contains(scanned.Query, "LET __loom_category_null = (") { - t.Fatalf("direct root category query omitted its indexed null/non-null branches:\n%s", scanned.Query) + if !strings.Contains(testAQLText(t, scanned.Command), "LET __loom_category_nonnull = (") || !strings.Contains(testAQLText(t, scanned.Command), "LET __loom_category_null = (") { + t.Fatalf("direct root category query omitted its indexed null/non-null branches:\n%s", testAQLText(t, scanned.Command)) } - assertCategoryScanBindVarsMatchQuery(t, scanned.Query, scanned.BindVars) - if !strings.Contains(scanned.OverflowWitness.Query, "FILTER NOT (") { + assertCategoryScanBindVarsMatchQuery(t, scanned.Command) + if !strings.Contains(testAQLText(t, scanned.OverflowWitness.Command), "FILTER NOT (") { t.Fatalf("direct root category scan omitted its exact missing-value witness: %+v", scanned.OverflowWitness) } - assertCategoryScanBindVarsMatchQuery(t, scanned.OverflowWitness.Query, scanned.OverflowWitness.BindVars) + assertCategoryScanBindVarsMatchQuery(t, scanned.OverflowWitness.Command) terminalStage, found := compiledStageByID(output.Stages, "add_status") if !found { t.Fatal("compiled output lost terminal related-field stage") @@ -275,7 +278,7 @@ func TestCompileCategoryScanUsesLegacyIndexFallbackWhenRootCollectionTypeIsUnpro } output.Plan.BindVars[root.CollectionBindKey] = "ObservationShadow" - scanned, err := CompileCategoryScanStageWithPolicy( + scanned, err := CompileCategoryScanStage(ArangoTarget(), output, recipe.ConstructionSourceProjectionID, "status_id", "amount_id", 256, ir.DefaultPhysicalOptimizationPolicy(), ) @@ -287,16 +290,17 @@ func TestCompileCategoryScanUsesLegacyIndexFallbackWhenRootCollectionTypeIsUnpro !reflect.DeepEqual(scanned.CategoryIndex.Fields, wantFields) || scanned.CategoryIndex.Supersedes != nil { t.Fatalf("unproven collection/type pair should retain the legacy non-replacement index: %+v", scanned.CategoryIndex) } - if strings.Contains(scanned.Query, "root.resourceType == @__loom_category_resource_type") { - t.Fatalf("unproven collection/type pair received a narrowing type predicate:\n%s", scanned.Query) + if strings.Contains(testAQLText(t, scanned.Command), "root.resourceType == @__loom_category_resource_type") { + t.Fatalf("unproven collection/type pair received a narrowing type predicate:\n%s", testAQLText(t, scanned.Command)) } - if _, exists := scanned.BindVars["__loom_category_resource_type"]; exists { - t.Fatalf("unproven collection/type pair received a resourceType bind: %#v", scanned.BindVars) + if _, exists := testAQLBinds(t, scanned.Command)["__loom_category_resource_type"]; exists { + t.Fatalf("unproven collection/type pair received a resourceType bind: %#v", testAQLBinds(t, scanned.Command)) } } -func assertCategoryScanBindVarsMatchQuery(t *testing.T, query string, bindVars map[string]any) { +func assertCategoryScanBindVarsMatchQuery(t *testing.T, command command.CompiledCommand) { t.Helper() + query, bindVars := testAQLView(t, command) want := map[string]struct{}{} for index := 0; index < len(query); index++ { if query[index] != '@' { @@ -344,12 +348,12 @@ func TestCategoryScanRootCategoryFastPathRejectsUnsafePrefixes(t *testing.T) { output := compilePopulationMappingOutput(t, test.output) stageID := "add_status" categoryID := "category_id" - scanned, err := CompileCategoryScanStageWithPolicy(output, stageID, categoryID, "value_id", 256, ir.DefaultPhysicalOptimizationPolicy()) + scanned, err := CompileCategoryScanStage(ArangoTarget(), output, stageID, categoryID, "value_id", 256, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if !strings.Contains(scanned.Query, "__loom_construction_stage_") || scanned.CategoryIndex != nil { - t.Fatalf("unsafe stage prefix bypassed construction evaluation or got a direct root index:\n%s", scanned.Query) + if !strings.Contains(testAQLText(t, scanned.Command), "__loom_construction_stage_") || scanned.CategoryIndex != nil { + t.Fatalf("unsafe stage prefix bypassed construction evaluation or got a direct root index:\n%s", testAQLText(t, scanned.Command)) } }) } @@ -422,18 +426,18 @@ func categoryScanRelatedRecipeOutput(emptyPolicy recipe.ExpansionEmptyPolicy, wi func TestCompileCategoryScanReturnsOnlyTerminalCategoryAfterFilter(t *testing.T) { output := compilePopulationMappingOutput(t, categoryScanConstructionRecipeOutput()) - scanned, err := CompileCategoryScanStageWithPolicy(output, "keep_positive", "status_id", "total_id", 256, ir.DefaultPhysicalOptimizationPolicy()) + scanned, err := CompileCategoryScanStage(ArangoTarget(), output, "keep_positive", "status_id", "total_id", 256, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - finalAt := strings.Index(scanned.Query, "FOR __loom_construction_final_row IN") - categoryAt := strings.Index(scanned.Query, "FOR __loom_category_row IN __loom_category_rows") + finalAt := strings.Index(testAQLText(t, scanned.Command), "FOR __loom_construction_final_row IN") + categoryAt := strings.Index(testAQLText(t, scanned.Command), "FOR __loom_category_row IN __loom_category_rows") if finalAt < 0 || categoryAt <= finalAt { - t.Fatalf("category scan is missing its terminal output boundary:\n%s", scanned.Query) + t.Fatalf("category scan is missing its terminal output boundary:\n%s", testAQLText(t, scanned.Command)) } - terminal := scanned.Query[finalAt:categoryAt] - if !strings.Contains(scanned.Query[:finalAt], "total") || !strings.Contains(scanned.Query, "__loom_construction_stage_1") || !strings.Contains(scanned.Query, "__loom_construction_stage_2") { - t.Fatalf("category scan dropped the prior derived filter or exact stage prefix:\n%s", scanned.Query) + terminal := testAQLText(t, scanned.Command)[finalAt:categoryAt] + if !strings.Contains(testAQLText(t, scanned.Command)[:finalAt], "total") || !strings.Contains(testAQLText(t, scanned.Command), "__loom_construction_stage_1") || !strings.Contains(testAQLText(t, scanned.Command), "__loom_construction_stage_2") { + t.Fatalf("category scan dropped the prior derived filter or exact stage prefix:\n%s", testAQLText(t, scanned.Command)) } if !strings.Contains(terminal, "status") || strings.Contains(terminal, "amount") || strings.Contains(terminal, "total") { t.Fatalf("terminal category projection was not narrowed safely:\n%s", terminal) @@ -441,13 +445,13 @@ func TestCompileCategoryScanReturnsOnlyTerminalCategoryAfterFilter(t *testing.T) if strings.Contains(terminal, "SORT __loom_construction_final_row.") { t.Fatalf("category scan retained an unneeded terminal row sort:\n%s", terminal) } - if strings.Contains(scanned.Query[:finalAt], "MERGE(") { - t.Fatalf("category scan merged projected objects before its stage filter:\n%s", scanned.Query[:finalAt]) + if strings.Contains(testAQLText(t, scanned.Command)[:finalAt], "MERGE(") { + t.Fatalf("category scan merged projected objects before its stage filter:\n%s", testAQLText(t, scanned.Command)[:finalAt]) } if !strings.Contains(terminal, `"__loom_category_scan_presence": __loom_construction_final_row["__loom_category_scan_presence"]`) { t.Fatalf("terminal category scan did not carry the presence marker through its stage prefix:\n%s", terminal) } - categoryOutput := scanned.Query[categoryAt:] + categoryOutput := testAQLText(t, scanned.Command)[categoryAt:] for _, required := range []string{ "COLLECT __loom_category_group_present = __loom_category_present, __loom_category_group_value = __loom_category_value", "SORT __loom_category_group_present ASC, TYPENAME(__loom_category_group_value) ASC, __loom_category_group_value ASC", @@ -457,8 +461,8 @@ func TestCompileCategoryScanReturnsOnlyTerminalCategoryAfterFilter(t *testing.T) t.Errorf("category scan lost its ordered complete-category contract %q:\n%s", required, categoryOutput) } } - if !strings.Contains(scanned.Query, "root_scope_allowed") || !strings.Contains(scanned.Query, "auth_resource_paths") { - t.Fatalf("narrowed category return lost the source authorization scope:\n%s", scanned.Query) + if !strings.Contains(testAQLText(t, scanned.Command), "root_scope_allowed") || !strings.Contains(testAQLText(t, scanned.Command), "auth_resource_paths") { + t.Fatalf("narrowed category return lost the source authorization scope:\n%s", testAQLText(t, scanned.Command)) } stage, found := compiledStageByID(output.Stages, "keep_positive") if !found { @@ -471,7 +475,7 @@ func TestCompileCategoryScanReturnsOnlyTerminalCategoryAfterFilter(t *testing.T) if scanned.Proof.OutputSchemaDigest != wantSchemaDigest { t.Fatalf("category proof was narrowed with the query return: got %q, want full stage schema %q", scanned.Proof.OutputSchemaDigest, wantSchemaDigest) } - wantQueryDigest, err := categoryQueryFingerprint(scanned.Query, scanned.BindVars) + wantQueryDigest, err := categoryQueryFingerprint(testAQLText(t, scanned.Command), testAQLBinds(t, scanned.Command)) if err != nil { t.Fatal(err) } @@ -525,31 +529,31 @@ func categoryScanConstructionRecipeOutput() recipe.Output { func TestCompileCategoryScanPushesDirectScalarEqualityToSourceAndKeepsFallbacks(t *testing.T) { output := compilePopulationMappingOutput(t, categoryScanFilterTestOutput(recipe.FilterEquals)) - scanned, err := CompileCategoryScanStageWithPolicy(output, "keep_active", "birth_date_id", "patient_id", 32, ir.DefaultPhysicalOptimizationPolicy()) + scanned, err := CompileCategoryScanStage(ArangoTarget(), output, "keep_active", "birth_date_id", "patient_id", 32, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } rootFilter := "FILTER root.payload.active == @construction_filter_value" stageFilter := "FILTER __loom_construction_input_1.active == @construction_filter_value" - if strings.Count(scanned.Query, rootFilter) != 1 || !strings.Contains(scanned.Query, stageFilter) { - t.Fatalf("category scan must duplicate an eligible source equality and retain the stage predicate:\n%s", scanned.Query) + if strings.Count(testAQLText(t, scanned.Command), rootFilter) != 1 || !strings.Contains(testAQLText(t, scanned.Command), stageFilter) { + t.Fatalf("category scan must duplicate an eligible source equality and retain the stage predicate:\n%s", testAQLText(t, scanned.Command)) } - rootFilterAt := strings.Index(scanned.Query, rootFilter) - stageReturnAt := strings.Index(scanned.Query, "RETURN { [@__loom_physical_projection_") + rootFilterAt := strings.Index(testAQLText(t, scanned.Command), rootFilter) + stageReturnAt := strings.Index(testAQLText(t, scanned.Command), "RETURN { [@__loom_physical_projection_") if rootFilterAt < 0 || stageReturnAt < 0 || rootFilterAt > stageReturnAt || - strings.Index(scanned.Query, "root.project == @project") > rootFilterAt || - strings.Index(scanned.Query, "root.dataset_generation == @dataset_generation") > rootFilterAt || - strings.Index(scanned.Query, "root_scope_allowed") > rootFilterAt { - t.Fatalf("source equality must follow project, generation, and authorization scope before source projection:\n%s", scanned.Query) + strings.Index(testAQLText(t, scanned.Command), "root.project == @project") > rootFilterAt || + strings.Index(testAQLText(t, scanned.Command), "root.dataset_generation == @dataset_generation") > rootFilterAt || + strings.Index(testAQLText(t, scanned.Command), "root_scope_allowed") > rootFilterAt { + t.Fatalf("source equality must follow project, generation, and authorization scope before source projection:\n%s", testAQLText(t, scanned.Command)) } ineligibleOutput := compilePopulationMappingOutput(t, categoryScanFilterTestOutput(recipe.FilterExists)) - ineligible, err := CompileCategoryScanStageWithPolicy(ineligibleOutput, "keep_active", "birth_date_id", "patient_id", 32, ir.DefaultPhysicalOptimizationPolicy()) + ineligible, err := CompileCategoryScanStage(ArangoTarget(), ineligibleOutput, "keep_active", "birth_date_id", "patient_id", 32, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if strings.Contains(ineligible.Query, "FILTER root.payload.active") || !strings.Contains(ineligible.Query, "FILTER __loom_construction_input_1.active != null") { - t.Fatalf("ineligible non-equality filter should remain on the staged path:\n%s", ineligible.Query) + if strings.Contains(testAQLText(t, ineligible.Command), "FILTER root.payload.active") || !strings.Contains(testAQLText(t, ineligible.Command), "FILTER __loom_construction_input_1.active != null") { + t.Fatalf("ineligible non-equality filter should remain on the staged path:\n%s", testAQLText(t, ineligible.Command)) } } @@ -603,20 +607,20 @@ func TestCompileSourceProjectionCategoryScanOffersCoveringIndexForExplicitPivotP if output.Plan.StageSequence != nil { t.Fatal("source-only output unexpectedly has a construction stage sequence") } - scanned, err := CompileCategoryScanStageWithPolicy(output, recipe.ConstructionSourceProjectionID, "category_id", "amount_id", 256, ir.DefaultPhysicalOptimizationPolicy()) + scanned, err := CompileCategoryScanStage(ArangoTarget(), output, recipe.ConstructionSourceProjectionID, "category_id", "amount_id", 256, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } index := scanned.PreviewCoveringIndex if index == nil { - t.Fatalf("direct source category discovery did not emit an explicit-pair covering-index spec:\n%s", scanned.Query) + t.Fatalf("direct source category discovery did not emit an explicit-pair covering-index spec:\n%s", testAQLText(t, scanned.Command)) } wantFields := []string{"project", "dataset_generation", "auth_resource_path", "_key", "payload.gender", "payload.id", "payload.multipleBirthInteger"} if index.Collection != "Patient" || !strings.HasPrefix(index.Name, previewCoveringIndexNamePrefix) || !reflect.DeepEqual(index.Fields, wantFields) { t.Fatalf("category scan covering-index metadata = %+v, want collection Patient fields %#v", index, wantFields) } projectionCount := 0 - for key, value := range scanned.BindVars { + for key, value := range testAQLBinds(t, scanned.Command) { if !strings.HasPrefix(key, "__loom_physical_projection_") || !strings.HasSuffix(key, "_name") { continue } @@ -625,16 +629,16 @@ func TestCompileSourceProjectionCategoryScanOffersCoveringIndexForExplicitPivotP t.Fatalf("source category scan retained projection %q=%#v", key, value) } } - if projectionCount != 0 || !strings.Contains(scanned.Query, "COLLECT value = root.payload.gender") || strings.Contains(scanned.Query, "root.payload.id") || strings.Contains(scanned.Query, "root.payload.multipleBirthInteger") { - t.Fatalf("source category scan should group the selected source field directly (found %d projection names):\n%s", projectionCount, scanned.Query) + if projectionCount != 0 || !strings.Contains(testAQLText(t, scanned.Command), "COLLECT value = root.payload.gender") || strings.Contains(testAQLText(t, scanned.Command), "root.payload.id") || strings.Contains(testAQLText(t, scanned.Command), "root.payload.multipleBirthInteger") { + t.Fatalf("source category scan should group the selected source field directly (found %d projection names):\n%s", projectionCount, testAQLText(t, scanned.Command)) } for _, guard := range []string{"root.project == @project", "root.dataset_generation == @dataset_generation", "root.auth_resource_path IN @auth_resource_paths", "FILTER root_scope_allowed == @scope_allowed"} { - if !strings.Contains(scanned.Query, guard) { - t.Fatalf("direct source category scan lost scope guard %q:\n%s", guard, scanned.Query) + if !strings.Contains(testAQLText(t, scanned.Command), guard) { + t.Fatalf("direct source category scan lost scope guard %q:\n%s", guard, testAQLText(t, scanned.Command)) } } - if strings.Contains(scanned.Query, "SORT root._key") { - t.Fatalf("complete category discovery must not sort source rows before grouping categories:\n%s", scanned.Query) + if strings.Contains(testAQLText(t, scanned.Command), "SORT root._key") { + t.Fatalf("complete category discovery must not sort source rows before grouping categories:\n%s", testAQLText(t, scanned.Command)) } sourceStage, found := compiledStageByID(output.Stages, recipe.ConstructionSourceProjectionID) if !found { @@ -650,17 +654,17 @@ func TestCompileSourceProjectionCategoryScanOffersCoveringIndexForExplicitPivotP scanned.Proof.QueryFingerprint == "" || scanned.Proof.Fingerprint == "" { t.Fatalf("narrowed source scan lost its complete stage/pair proof: %#v", scanned.Proof) } - if !scanned.Proof.PresenceTracked || !strings.Contains(scanned.Query, "present") { - t.Fatalf("exact root selector presence is not bound into the category proof/query: proof=%#v\n%s", scanned.Proof, scanned.Query) + if !scanned.Proof.PresenceTracked || !strings.Contains(testAQLText(t, scanned.Command), "present") { + t.Fatalf("exact root selector presence is not bound into the category proof/query: proof=%#v\n%s", scanned.Proof, testAQLText(t, scanned.Command)) } - if !strings.Contains(scanned.Query, "root_scope_allowed") || !strings.Contains(scanned.Query, "auth_resource_paths") { - t.Fatalf("narrowed source scan lost root authorization scope:\n%s", scanned.Query) + if !strings.Contains(testAQLText(t, scanned.Command), "root_scope_allowed") || !strings.Contains(testAQLText(t, scanned.Command), "auth_resource_paths") { + t.Fatalf("narrowed source scan lost root authorization scope:\n%s", testAQLText(t, scanned.Command)) } - if !strings.Contains(scanned.Query, `indexHint: "`+previewCoveringIndexNamePrefix) || !strings.Contains(scanned.Query, "forceIndexHint: false") || strings.Contains(scanned.Query, "forceIndexHint: true") { - t.Fatalf("category scan should use its optional covering index without requiring it:\n%s", scanned.Query) + if !strings.Contains(testAQLText(t, scanned.Command), `indexHint: "`+previewCoveringIndexNamePrefix) || !strings.Contains(testAQLText(t, scanned.Command), "forceIndexHint: false") || strings.Contains(testAQLText(t, scanned.Command), "forceIndexHint: true") { + t.Fatalf("category scan should use its optional covering index without requiring it:\n%s", testAQLText(t, scanned.Command)) } - withoutPair, err := CompileCategoryScanOutputWithPolicy(output, "category", 256, ir.DefaultPhysicalOptimizationPolicy()) + withoutPair, err := CompileCategoryScanOutput(ArangoTarget(), output, "category", 256, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -694,18 +698,236 @@ func TestCompileCategoryScanTracksPresenceThroughExpandForDiscoveryProbe(t *test output.Construction = &construction compiled := compilePopulationMappingOutput(t, output) - scanned, err := CompileCategoryScanStageWithPolicy(compiled, "expand_tags", "status_id", "tag_id", MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) + scanned, err := CompileCategoryScanStage(ArangoTarget(), compiled, "expand_tags", "status_id", "tag_id", MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } if !scanned.Proof.PresenceTracked { - t.Fatalf("source presence was dropped through Expand: proof=%#v\n%s", scanned.Proof, scanned.Query) + t.Fatalf("source presence was dropped through Expand: proof=%#v\n%s", scanned.Proof, testAQLText(t, scanned.Command)) } - companion, ok := scanned.BindVars["__loom_category_presence_column"].(string) + companion, ok := testAQLBinds(t, scanned.Command)["__loom_category_presence_column"].(string) if !ok || companion == "" { - t.Fatalf("Expand category scan did not consume its source presence companion: %#v", scanned.BindVars) + t.Fatalf("Expand category scan did not consume its source presence companion: %#v", testAQLBinds(t, scanned.Command)) } - if !strings.Contains(scanned.Query, companion) { - t.Fatalf("Expand category query omitted presence companion %q:\n%s", companion, scanned.Query) + if !strings.Contains(testAQLText(t, scanned.Command), companion) { + t.Fatalf("Expand category query omitted presence companion %q:\n%s", companion, testAQLText(t, scanned.Command)) + } +} + +func mustPostgreSQLCategoryTarget(t *testing.T) Target { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("verify_sql25v1_compile") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + target, err := PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("PostgreSQLTarget: %v", err) + } + return target +} + +func mustPostgreSQLCategorySQL(t *testing.T, compiledCommand command.CompiledCommand) (string, []command.PostgreSQLParameter) { + t.Helper() + if compiledCommand.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("category command backend = %v, want PostgreSQL", compiledCommand.Backend()) + } + sql, args, err := compiledCommand.PostgreSQLParameters() + if err != nil { + t.Fatalf("PostgreSQLParameters: %v", err) + } + return sql, args +} + +func TestCompileCategoryScanOutputForPostgreSQLTagsTheRegisteredCategoryStatement(t *testing.T) { + output := compilePopulationMappingOutput(t, recipe.Output{ + Name: "Patients", RootResourceType: "Patient", RowGrain: "patient", RootColumnNaming: recipe.RootColumnNamingExact, + Fields: []recipe.Field{ + {Name: "id", Expr: recipe.Expression{Select: "root.id"}}, + {Name: "gender", Expr: recipe.Expression{Select: "root.gender"}}, + }, + }) + scanned, err := CompileCategoryScanOutput(mustPostgreSQLCategoryTarget(t), output, "gender", 64, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatal(err) + } + sql, args := mustPostgreSQLCategorySQL(t, scanned.Command) + for _, want := range []string{ + "__loom_category_source AS", "__loom_category_rows AS", "__loom_category_distinct AS", + `CASE WHEN __loom_category_row."c1" THEN __loom_category_row."c0" END AS "value"`, + `GROUP BY "present", "value"`, `ORDER BY "present" ASC`, "LIMIT $", + } { + if !strings.Contains(sql, want) { + t.Fatalf("PostgreSQL category statement missing %q:\n%s", want, sql) + } + } + if strings.Contains(sql, "@") { + t.Fatalf("AQL bind syntax reached the PostgreSQL category command:\n%s", sql) + } + if strings.Contains(sql, `"gender"`) { + t.Fatalf("the category column name was interpolated into SQL text:\n%s", sql) + } + last := args[len(args)-1] + if want := (command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(65)}); last != want { + t.Fatalf("page bound = %#v, want %#v", last, want) + } + if scanned.PresentColumn != "present" || scanned.ValueColumn != "value" { + t.Fatalf("positional contract = %q/%q", scanned.PresentColumn, scanned.ValueColumn) + } + if scanned.CategoryIndex != nil || scanned.PreviewCoveringIndex != nil { + t.Fatalf("Arango index advice reached the PostgreSQL scan: %#v %#v", scanned.CategoryIndex, scanned.PreviewCoveringIndex) + } + if scanned.OverflowWitness != nil { + t.Fatalf("Arango overflow witness reached the PostgreSQL scan: %#v", scanned.OverflowWitness) + } + if scanned.Proof.Output != "Patients" || scanned.Proof.Column != "gender" || scanned.Proof.MaxValues != 64 || + scanned.Proof.OutputSchemaDigest == "" || scanned.Proof.PlanFingerprint == "" || + scanned.Proof.QueryFingerprint == "" || scanned.Proof.Fingerprint == "" || !scanned.Proof.PresenceTracked { + t.Fatalf("incomplete PostgreSQL category proof: %#v", scanned.Proof) + } + if scanned.Proof.OverflowWitnessFingerprint != "" { + t.Fatalf("proof cites a witness this scan does not carry: %#v", scanned.Proof) + } + again, err := CompileCategoryScanOutput(mustPostgreSQLCategoryTarget(t), output, "gender", 64, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatal(err) + } + againSQL, againArgs := mustPostgreSQLCategorySQL(t, again.Command) + if againSQL != sql || !reflect.DeepEqual(againArgs, args) || again.Proof.Fingerprint != scanned.Proof.Fingerprint { + t.Fatal("PostgreSQL category compilation was not deterministic") + } + other, err := CompileCategoryScanOutput(mustPostgreSQLCategoryTarget(t), output, "id", 64, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatal(err) + } + otherSQL, otherArgs := mustPostgreSQLCategorySQL(t, other.Command) + if other.Proof.Fingerprint == scanned.Proof.Fingerprint || reflect.DeepEqual(otherArgs, args) { + t.Fatalf("proof and arguments were not bound to the selected category column: %#v %#v", other.Proof.Fingerprint, otherArgs) + } + if otherSQL != sql { + t.Fatalf("the registered category template changed between two registered columns:\n%s\n%s", sql, otherSQL) + } +} + +func TestCompileCategoryScanPostgreSQLRefusalsMatchTheAQLPath(t *testing.T) { + target := mustPostgreSQLCategoryTarget(t) + base := lower.CompiledRecipeOutput{OutputSchema: []lower.CompiledOutputColumn{ + {Name: "status", Kind: string(expression.KindString), Cardinality: string(expression.OptionalOne)}, + {Name: "internal", Kind: string(expression.KindString), Cardinality: string(expression.RequiredOne), Internal: true}, + {Name: "identity", Kind: string(expression.KindString), Cardinality: string(expression.RequiredOne), Identity: true}, + {Name: "repeated", Kind: string(expression.KindString), Cardinality: string(expression.Many)}, + {Name: "object", Kind: string(expression.KindObject), Cardinality: string(expression.OptionalOne)}, + {Name: "unknown_kind", Kind: "mystery", Cardinality: string(expression.OptionalOne)}, + }} + for _, test := range []struct { + name string + code CategoryScanRefusalCode + }{ + {"missing", CategoryScanColumnUnknown}, + {"internal", CategoryScanColumnInternal}, + {"identity", CategoryScanColumnIdentity}, + {"repeated", CategoryScanColumnRepeated}, + {"object", CategoryScanColumnUnsupported}, + {"unknown_kind", CategoryScanColumnUnsupported}, + } { + _, pgErr := CompileCategoryScanOutput(target, base, test.name, 256, ir.DefaultPhysicalOptimizationPolicy()) + var refusal *CategoryScanRefusal + if !errors.As(pgErr, &refusal) || refusal.Code != test.code { + t.Fatalf("PostgreSQL refusal for %q = %v, want %s", test.name, pgErr, test.code) + } + if code, ok := CategoryScanRefusalCodeOf(pgErr); !ok || code != test.code { + t.Fatalf("typed PostgreSQL code = %q/%t", code, ok) + } + } + for _, maxValues := range []int{0, MaxCategoryScanValues + 1} { + _, pgErr := CompileCategoryScanOutput(target, base, "status", maxValues, ir.DefaultPhysicalOptimizationPolicy()) + if code, ok := CategoryScanRefusalCodeOf(pgErr); !ok || code != CategoryScanInvalidLimit { + t.Fatalf("PostgreSQL limit refusal for %d = %v", maxValues, pgErr) + } + } +} + +func TestCompileCategoryScanStageForPostgreSQLKeepsTheSourceProjectionAndRefusesComposedStages(t *testing.T) { + compiled := compilePopulationMappingOutput(t, categoryScanConstructionRecipeOutput()) + target := mustPostgreSQLCategoryTarget(t) + + scanned, err := CompileCategoryScanStage(target, compiled, recipe.ConstructionSourceProjectionID, "status_id", "amount_id", 64, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatal(err) + } + sql, args := mustPostgreSQLCategorySQL(t, scanned.Command) + if !strings.Contains(sql, "__loom_category_distinct AS") || !strings.HasSuffix(sql, "LIMIT $6") { + t.Fatalf("source-projection stage scan was not the registered category statement:\n%s", sql) + } + if len(args) == 0 || args[len(args)-1].Value != int64(65) { + t.Fatalf("stage scan bound = %#v", args) + } + if scanned.Proof.Version != 2 || scanned.Proof.StageID != recipe.ConstructionSourceProjectionID || + scanned.Proof.ColumnID != "status_id" || scanned.Proof.ValueColumnID != "amount_id" || + scanned.Proof.Column != "status" || !scanned.Proof.PresenceTracked { + t.Fatalf("stage proof is not bound to the requested pair: %#v", scanned.Proof) + } + if scanned.OverflowWitness != nil { + t.Fatalf("Arango witness reached the PostgreSQL stage scan: %#v", scanned.OverflowWitness) + } + + for _, stageID := range []string{"derive_total", "keep_positive"} { + _, stageErr := CompileCategoryScanStage(target, compiled, stageID, "status_id", "amount_id", 64, ir.DefaultPhysicalOptimizationPolicy()) + if !errors.Is(stageErr, postgresrender.ErrUnsupportedPostgresPlan) { + t.Fatalf("composed stage %q must refuse with the typed capability gap: %v", stageID, stageErr) + } + if strings.Contains(strings.ToLower(stageErr.Error()), "aql") { + t.Fatalf("composed stage refusal must not offer an AQL fallback: %v", stageErr) + } + } +} + +func TestCompileCategoryScanStageForPostgreSQLCollapsesProvenRelatedPrefixToSourceRead(t *testing.T) { + compiled := compilePopulationMappingOutput(t, categoryScanRelatedRecipeOutput(recipe.ExpansionPreserveParent, false, false)) + if _, found := compiledStageByID(compiled.Stages, "add_status"); !found { + t.Fatal("compiled output lost the terminal related-field stage") + } + + scanned, err := CompileCategoryScanStage(mustPostgreSQLCategoryTarget(t), compiled, "add_status", "category_id", "value_id", 256, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("proof-accepted related prefix must render, not refuse: %v", err) + } + sql, args := mustPostgreSQLCategorySQL(t, scanned.Command) + for _, want := range []string{ + "__loom_category_source AS", "__loom_category_rows AS", "__loom_category_distinct AS", + `GROUP BY "present", "value"`, `ORDER BY "present" ASC`, + `END COLLATE "C" ASC, "value" ASC`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("collapsed related-prefix page is not the registered composed statement (missing %q):\n%s", want, sql) + } + } + if !strings.HasSuffix(sql, "LIMIT $"+fmt.Sprint(len(args))) { + t.Fatalf("the bound parameter must be the final placeholder:\n%s\n%#v", sql, args) + } + if scanned.Proof.Version != 2 || scanned.Proof.StageID != "add_status" || + scanned.Proof.ColumnID != "category_id" || scanned.Proof.ValueColumnID != "value_id" || + scanned.Proof.Column != "category" || scanned.Proof.MaxValues != 256 || !scanned.Proof.PresenceTracked { + t.Fatalf("collapsed scan lost its Version-2 stage/pair proof: %#v", scanned.Proof) + } + if terminalStage, _ := compiledStageByID(compiled.Stages, "add_status"); true { + wantDigest, hashErr := categoryHash(terminalStage.Columns) + if hashErr != nil { + t.Fatal(hashErr) + } + if scanned.Proof.OutputSchemaDigest != wantDigest || scanned.Proof.PlanFingerprint == "" || + scanned.Proof.QueryFingerprint == "" || scanned.Proof.Fingerprint == "" { + t.Fatalf("collapsed scan lost its proof identity: %#v", scanned.Proof) + } + } + if scanned.CategoryIndex != nil || scanned.PreviewCoveringIndex != nil || scanned.OverflowWitness != nil { + t.Fatalf("Arango-only execution aids reached the collapsed PostgreSQL scan: %#v %#v %#v", + scanned.CategoryIndex, scanned.PreviewCoveringIndex, scanned.OverflowWitness) + } + + filtered := compilePopulationMappingOutput(t, categoryScanRelatedRecipeOutput(recipe.ExpansionPreserveParent, true, false)) + _, refusalErr := CompileCategoryScanStage(mustPostgreSQLCategoryTarget(t), filtered, "add_status", "category_id", "value_id", 256, ir.DefaultPhysicalOptimizationPolicy()) + if !errors.Is(refusalErr, postgresrender.ErrUnsupportedPostgresPlan) { + t.Fatalf("unproven related prefix must refuse with the typed capability gap: %v", refusalErr) } } diff --git a/internal/dataframe/compiler/cell_trace.go b/internal/dataframe/compiler/cell_trace.go index f8ad641f7..fcfc45039 100644 --- a/internal/dataframe/compiler/cell_trace.go +++ b/internal/dataframe/compiler/cell_trace.go @@ -1,24 +1,23 @@ package compiler import ( + "errors" "fmt" "strings" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/compiler/render/aql" + postgresrender "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" "github.com/calypr/loom/internal/dataframe/spec" ) const MaxCellTraceContributions = 100 const traceReshapeRowID = "__loom_row_id" -// CompiledCellTraceQuery is a specialized terminal over one canonical output -// plan. It retains the exact final value expression and stable row identity; -// only the contributor page is diagnostic-only. type CompiledCellTraceQuery struct { - Query string - BindVars map[string]any + Command command.CompiledCommand ValueColumn string ContributionsColumn string StatusColumn string @@ -26,8 +25,6 @@ type CompiledCellTraceQuery struct { OmissionColumn string IdentityPartsColumn string ExplicitIdentityColumn string - // ExplicitIdentityObject is set only when construction lowering proves the - // final explicit identity is a required compiler-owned object column. ExplicitIdentityObject bool RowIdentity *spec.RowIdentity ContributionOffset int @@ -35,26 +32,31 @@ type CompiledCellTraceQuery struct { Diagnostics ir.CompilerPlanDiagnostics } -func CompileCellTraceOutputWithPolicy(output lower.CompiledRecipeOutput, column string, offset, limit int, policy ir.PhysicalOptimizationPolicy) (CompiledCellTraceQuery, error) { - column = strings.TrimSpace(column) - if column == "" { - return CompiledCellTraceQuery{}, fmt.Errorf("cell trace column is required") +func cellTraceCommand(rendered aql.RenderedPhysicalPlan) (command.CompiledCommand, error) { + compiledCommand, err := command.NewAQLCommand(rendered.Query, rendered.BindVars) + if err != nil { + return command.CompiledCommand{}, fmt.Errorf("tag cell trace command: %w", err) } - if output.RowIdentity == nil { - return CompiledCellTraceQuery{}, fmt.Errorf("cell trace requires a row identity") + return compiledCommand, nil +} + +func CompileCellTraceOutput(target Target, output lower.CompiledRecipeOutput, column string, offset, limit int, policy ir.PhysicalOptimizationPolicy) (CompiledCellTraceQuery, error) { + if err := target.validate(); err != nil { + return CompiledCellTraceQuery{}, err } - if offset < 0 { - return CompiledCellTraceQuery{}, fmt.Errorf("cell trace offset cannot be negative") + if target.backend == command.CompiledBackendPostgreSQL { + return compileCellTracePostgreSQL(target, output, column, offset, limit) } - if limit <= 0 { - limit = 25 + if err := target.rejectPostgreSQL("cell trace compilation"); err != nil { + return CompiledCellTraceQuery{}, err } - if limit > MaxCellTraceContributions { - return CompiledCellTraceQuery{}, fmt.Errorf("cell trace contribution limit exceeds %d", MaxCellTraceContributions) + column, offset, limit, err := normalizeCellTracePage(output, column, offset, limit) + if err != nil { + return CompiledCellTraceQuery{}, err } physical := ir.ClonePhysicalPlan(output.Plan) - _ = policy // The trace deliberately uses the canonical pre-optimization plan. - physical, err := withGenericPhysicalExecutionWindow(physical, 0) + _ = policy + physical, err = withGenericPhysicalExecutionWindow(physical, 0) if err != nil { return CompiledCellTraceQuery{}, fmt.Errorf("apply cell trace execution window: %w", err) } @@ -64,17 +66,57 @@ func CompileCellTraceOutputWithPolicy(output lower.CompiledRecipeOutput, column if len(physical.Operations) == 1 && physical.Operations[0].Kind == ir.PhysicalGroupRowsOp && physical.Operations[0].GroupRows != nil { return compileGroupRowsCellTrace(output, &physical, column, offset, limit) } - terminalIndex, identityParts, explicitIdentity, err := finalPopulationMappingIdentity(physical, output.RowIdentity) + prepared, facts, err := prepareCellTracePhysicalPlan(physical, output, column, offset, limit) if err != nil { return CompiledCellTraceQuery{}, err } + rendered, err := aql.RenderPhysicalPlan(prepared) + if err != nil { + return CompiledCellTraceQuery{}, fmt.Errorf("render cell trace physical plan: %w", err) + } + compiledCommand, commandErr := cellTraceCommand(rendered) + if commandErr != nil { + return CompiledCellTraceQuery{}, commandErr + } + return cellTraceQuery(compiledCommand, output, prepared, facts, offset, limit), nil +} + +func normalizeCellTracePage(output lower.CompiledRecipeOutput, column string, offset, limit int) (string, int, int, error) { + column = strings.TrimSpace(column) + if column == "" { + return "", 0, 0, fmt.Errorf("cell trace column is required") + } + if output.RowIdentity == nil { + return "", 0, 0, fmt.Errorf("cell trace requires a row identity") + } + if offset < 0 { + return "", 0, 0, fmt.Errorf("cell trace offset cannot be negative") + } + if limit <= 0 { + limit = 25 + } + if limit > MaxCellTraceContributions { + return "", 0, 0, fmt.Errorf("cell trace contribution limit exceeds %d", MaxCellTraceContributions) + } + return column, offset, limit, nil +} + +type cellTraceTerminalFacts struct { + explicitIdentity *ir.PhysicalExpression +} + +func prepareCellTracePhysicalPlan(physical ir.PhysicalPlan, output lower.CompiledRecipeOutput, column string, offset, limit int) (ir.PhysicalPlan, cellTraceTerminalFacts, error) { + terminalIndex, identityParts, explicitIdentity, err := finalPopulationMappingIdentity(physical, output.RowIdentity) + if err != nil { + return ir.PhysicalPlan{}, cellTraceTerminalFacts{}, err + } projection, err := traceProjection(physical.Operations[terminalIndex].Return, column) if err != nil { - return CompiledCellTraceQuery{}, err + return ir.PhysicalPlan{}, cellTraceTerminalFacts{}, err } value, err := physicalProjectionExpression(projection) if err != nil { - return CompiledCellTraceQuery{}, fmt.Errorf("trace column %q: %w", column, err) + return ir.PhysicalPlan{}, cellTraceTerminalFacts{}, fmt.Errorf("trace column %q: %w", column, err) } contribution := traceContributionForProjection(physical, value) const offsetBind, limitBind, fetchLimitBind = "cell_trace_offset", "cell_trace_limit", "cell_trace_fetch_limit" @@ -90,28 +132,57 @@ func CompileCellTraceOutputWithPolicy(output lower.CompiledRecipeOutput, column }, } if err := physical.Validate(); err != nil { - return CompiledCellTraceQuery{}, fmt.Errorf("validate cell trace physical plan: %w", err) + return ir.PhysicalPlan{}, cellTraceTerminalFacts{}, fmt.Errorf("validate cell trace physical plan: %w", err) } if err := ir.ValidateGenericPhysicalPlanScope(physical); err != nil { - return CompiledCellTraceQuery{}, fmt.Errorf("verify cell trace physical scope: %w", err) - } - rendered, err := aql.RenderPhysicalPlan(physical) - if err != nil { - return CompiledCellTraceQuery{}, fmt.Errorf("render cell trace physical plan: %w", err) + return ir.PhysicalPlan{}, cellTraceTerminalFacts{}, fmt.Errorf("verify cell trace physical scope: %w", err) } + return physical, cellTraceTerminalFacts{explicitIdentity: explicitIdentity}, nil +} + +func cellTraceQuery(compiledCommand command.CompiledCommand, output lower.CompiledRecipeOutput, prepared ir.PhysicalPlan, facts cellTraceTerminalFacts, offset, limit int) CompiledCellTraceQuery { query := CompiledCellTraceQuery{ - Query: rendered.Query, BindVars: rendered.BindVars, + Command: compiledCommand, ValueColumn: ir.PhysicalCellTraceValueField, ContributionsColumn: ir.PhysicalCellTraceContributionsField, StatusColumn: ir.PhysicalCellTraceStatusField, HasMoreColumn: ir.PhysicalCellTraceHasMoreField, OmissionColumn: ir.PhysicalCellTraceOmissionField, RowIdentity: output.RowIdentity.Clone(), - ContributionOffset: offset, ContributionLimit: limit, Diagnostics: physicalPlanDiagnostics(physical), + ContributionOffset: offset, ContributionLimit: limit, Diagnostics: physicalPlanDiagnostics(prepared), } - if explicitIdentity != nil { + if facts.explicitIdentity != nil { query.ExplicitIdentityColumn = ir.PhysicalCellTraceExplicitIdentityField } else { query.IdentityPartsColumn = ir.PhysicalCellTraceIdentityPartsField } - return query, nil + return query +} + +func compileCellTracePostgreSQL(target Target, output lower.CompiledRecipeOutput, column string, offset, limit int) (CompiledCellTraceQuery, error) { + column, offset, limit, err := normalizeCellTracePage(output, column, offset, limit) + if err != nil { + return CompiledCellTraceQuery{}, err + } + physical, err := withGenericPhysicalExecutionWindow(ir.ClonePhysicalPlan(output.Plan), 0) + if err != nil { + return CompiledCellTraceQuery{}, fmt.Errorf("cell trace compilation is unsupported on the PostgreSQL compile target: %w", + errors.Join(fmt.Errorf("apply cell trace execution window: %w", err), postgresrender.ErrUnsupportedPostgresPlan)) + } + if physical.StageSequence != nil { + return CompiledCellTraceQuery{}, fmt.Errorf("composed construction cell trace is unsupported on the PostgreSQL compile target: %w", postgresrender.ErrUnsupportedPostgresPlan) + } + if len(physical.Operations) == 1 && physical.Operations[0].Kind == ir.PhysicalGroupRowsOp && physical.Operations[0].GroupRows != nil { + return CompiledCellTraceQuery{}, fmt.Errorf("explicit group rows cell trace is unsupported on the PostgreSQL compile target: %w", postgresrender.ErrUnsupportedPostgresPlan) + } + prepared, facts, err := prepareCellTracePhysicalPlan(physical, output, column, offset, limit) + if err != nil { + return CompiledCellTraceQuery{}, fmt.Errorf("cell trace compilation is unsupported on the PostgreSQL compile target: %w", + errors.Join(err, postgresrender.ErrUnsupportedPostgresPlan)) + } + compiledCommand, _, err := target.renderPostgreSQLFinalPlan(prepared) + if err != nil { + return CompiledCellTraceQuery{}, fmt.Errorf("cell trace compilation is unsupported on the PostgreSQL compile target: %w", + errors.Join(err, postgresrender.ErrUnsupportedPostgresPlan)) + } + return cellTraceQuery(compiledCommand, output, prepared, facts, offset, limit), nil } func compileGroupRowsCellTrace(output lower.CompiledRecipeOutput, physical *ir.PhysicalPlan, column string, offset, limit int) (CompiledCellTraceQuery, error) { @@ -144,8 +215,12 @@ func compileGroupRowsCellTrace(output lower.CompiledRecipeOutput, physical *ir.P if err != nil { return CompiledCellTraceQuery{}, fmt.Errorf("render group-row cell trace physical plan: %w", err) } + compiledCommand, commandErr := cellTraceCommand(rendered) + if commandErr != nil { + return CompiledCellTraceQuery{}, commandErr + } return CompiledCellTraceQuery{ - Query: rendered.Query, BindVars: rendered.BindVars, + Command: compiledCommand, ValueColumn: ir.PhysicalCellTraceValueField, ContributionsColumn: ir.PhysicalCellTraceContributionsField, StatusColumn: ir.PhysicalCellTraceStatusField, HasMoreColumn: ir.PhysicalCellTraceHasMoreField, OmissionColumn: ir.PhysicalCellTraceOmissionField, ExplicitIdentityColumn: ir.PhysicalCellTraceExplicitIdentityField, @@ -180,8 +255,12 @@ func compileConstructionCellTrace(output lower.CompiledRecipeOutput, physical *i if err != nil { return CompiledCellTraceQuery{}, fmt.Errorf("render construction cell trace physical plan: %w", err) } + compiledCommand, commandErr := cellTraceCommand(rendered) + if commandErr != nil { + return CompiledCellTraceQuery{}, commandErr + } query := CompiledCellTraceQuery{ - Query: rendered.Query, BindVars: rendered.BindVars, + Command: compiledCommand, ValueColumn: ir.PhysicalCellTraceValueField, ContributionsColumn: ir.PhysicalCellTraceContributionsField, StatusColumn: ir.PhysicalCellTraceStatusField, HasMoreColumn: ir.PhysicalCellTraceHasMoreField, OmissionColumn: ir.PhysicalCellTraceOmissionField, RowIdentity: output.RowIdentity.Clone(), diff --git a/internal/dataframe/compiler/cell_trace_test.go b/internal/dataframe/compiler/cell_trace_test.go index 33cde19e0..44e86488b 100644 --- a/internal/dataframe/compiler/cell_trace_test.go +++ b/internal/dataframe/compiler/cell_trace_test.go @@ -1,14 +1,18 @@ package compiler import ( + "errors" "reflect" "strings" "testing" "github.com/calypr/loom/internal/dataframe/columntransform" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" + postgresrender "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" fhirschema "github.com/calypr/loom/internal/fhir/schema" ) @@ -18,18 +22,18 @@ func TestCompileCellTraceUsesFinalValueAndStableIdentity(t *testing.T) { RootColumnNaming: recipe.RootColumnNamingExact, Fields: []recipe.Field{{Name: "patient_id", Expr: recipe.Expression{Select: "root.id"}}}, }) - compiled, err := CompileCellTraceOutputWithPolicy(output, "patient_id", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileCellTraceOutput(ArangoTarget(), output, "patient_id", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } for _, want := range []string{"root.payload.id", "@cell_trace_offset", "@cell_trace_limit", "[@project, root._key]"} { - if !strings.Contains(compiled.Query, want) { - t.Fatalf("trace query missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Fatalf("trace query missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } for _, field := range []string{ir.PhysicalCellTraceValueField, ir.PhysicalCellTraceContributionsField, ir.PhysicalCellTraceStatusField, ir.PhysicalCellTraceHasMoreField} { - if !containsBindValue(compiled.BindVars, field) { - t.Fatalf("trace field %q missing from binds %#v", field, compiled.BindVars) + if !containsBindValue(testAQLBinds(t, compiled.Command), field) { + t.Fatalf("trace field %q missing from binds %#v", field, testAQLBinds(t, compiled.Command)) } } if compiled.IdentityPartsColumn != ir.PhysicalCellTraceIdentityPartsField || compiled.ExplicitIdentityColumn != "" { @@ -50,7 +54,7 @@ func TestCompileCellTraceForSourceOnlyGroupRowsUsesFinalMemberValueAndObjectIden if output.Plan.StageSequence != nil || len(output.Plan.Operations) != 1 || output.Plan.Operations[0].GroupRows == nil { t.Fatalf("source-only explicit cohort was not compiled as the ordinary GROUP_ROWS terminal: %#v", output.Plan) } - compiled, err := CompileCellTraceOutputWithPolicy(output, "resourceType", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileCellTraceOutput(ArangoTarget(), output, "resourceType", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile CellTrace for ordinary group-row member field: %v", err) } @@ -73,16 +77,16 @@ func TestCompileCellTraceForSourceOnlyGroupRowsUsesFinalMemberValueAndObjectIden "__loom_trace_source_identity.id, value:", "UNIQUE(", } { - if !strings.Contains(compiled.Query, want) { - t.Errorf("typed GROUP_ROWS CellTrace query is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Errorf("typed GROUP_ROWS CellTrace query is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } - if strings.Contains(compiled.Query, "TRACE_CONTRIBUTORS_UNAVAILABLE") { - t.Fatalf("selected scalar member field should carry exact contributor evidence:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), "TRACE_CONTRIBUTORS_UNAVAILABLE") { + t.Fatalf("selected scalar member field should carry exact contributor evidence:\n%s", testAQLText(t, compiled.Command)) } for _, field := range []string{ir.PhysicalCellTraceValueField, ir.PhysicalCellTraceExplicitIdentityField} { - if !containsBindValue(compiled.BindVars, field) { - t.Errorf("group-row CellTrace is missing typed output field %q: %#v", field, compiled.BindVars) + if !containsBindValue(testAQLBinds(t, compiled.Command), field) { + t.Errorf("group-row CellTrace is missing typed output field %q: %#v", field, testAQLBinds(t, compiled.Command)) } } } @@ -134,27 +138,27 @@ func TestCompileSourceOnlyGroupRowsCellTraceMatchesExactOneAndAllMemberValues(t RowValues: []recipe.GroupRowValuePolicy{{ColumnID: test.field.ColumnID, Policy: test.policy}}, }, }) - compiled, err := CompileCellTraceOutputWithPolicy(output, test.field.Name, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileCellTraceOutput(ArangoTarget(), output, test.field.Name, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile %s GROUP_ROWS CellTrace: %v", test.name, err) } for _, marker := range test.want { - if !strings.Contains(compiled.Query, marker) { - t.Errorf("query is missing %q:\n%s", marker, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), marker) { + t.Errorf("query is missing %q:\n%s", marker, testAQLText(t, compiled.Command)) } } for _, marker := range test.unexpected { - if strings.Contains(compiled.Query, marker) { - t.Errorf("query unexpectedly contains %q:\n%s", marker, compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), marker) { + t.Errorf("query unexpectedly contains %q:\n%s", marker, testAQLText(t, compiled.Command)) } } - if strings.Contains(compiled.Query, "TRACE_CONTRIBUTORS_UNAVAILABLE") { - t.Errorf("supported %s output has unavailable contributor evidence:\n%s", test.name, compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), "TRACE_CONTRIBUTORS_UNAVAILABLE") { + t.Errorf("supported %s output has unavailable contributor evidence:\n%s", test.name, testAQLText(t, compiled.Command)) } - if !strings.Contains(compiled.Query, "source.auth_resource_path IN @auth_resource_paths") || - !strings.Contains(compiled.Query, "revision.project == @project") || - !strings.Contains(compiled.Query, "revision.generation == @dataset_generation") { - t.Errorf("%s trace lost its project, generation, or authorization scope:\n%s", test.name, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), "source.auth_resource_path IN @auth_resource_paths") || + !strings.Contains(testAQLText(t, compiled.Command), "revision.project == @project") || + !strings.Contains(testAQLText(t, compiled.Command), "revision.generation == @dataset_generation") { + t.Errorf("%s trace lost its project, generation, or authorization scope:\n%s", test.name, testAQLText(t, compiled.Command)) } }) } @@ -170,13 +174,13 @@ func TestCompileCellTraceExplainsOptionalTraversalBeforeFirstReduction(t *testin Fields: []recipe.Field{{Name: "condition_id", Expr: recipe.Expression{Select: "id"}, ValueMode: recipe.ValueModeFirst}}, }}, }) - compiled, err := CompileCellTraceOutputWithPolicy(output, "condition_id", 5, 10, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileCellTraceOutput(ArangoTarget(), output, "condition_id", 5, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } for _, want := range []string{"FOR __loom_physical_trace_contributor", "IN child_set_1", "TO_ARRAY(", "resourceType:", "resourceId:", "LENGTH(__loom_physical_trace_status_candidates) > 1 ? \"AMBIGUOUS\""} { - if !strings.Contains(compiled.Query, want) { - t.Fatalf("trace query missing pre-reduction contributor evidence %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Fatalf("trace query missing pre-reduction contributor evidence %q:\n%s", want, testAQLText(t, compiled.Command)) } } if compiled.ContributionOffset != 5 || compiled.ContributionLimit != 10 { @@ -195,13 +199,13 @@ func TestCompileCellTraceExplainsRelatedAggregateContributors(t *testing.T) { Aggregates: []recipe.Aggregate{{Name: "maximum_value", Operation: recipe.AggregateMax, Expr: &value}}, }}, }) - compiled, err := CompileCellTraceOutputWithPolicy(output, "maximum_value", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileCellTraceOutput(ArangoTarget(), output, "maximum_value", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } for _, want := range []string{"FOR __loom_physical_trace_contributor", "IN child_set_1", "resourceType:", "resourceId:", ".payload.valueQuantity.value"} { - if !strings.Contains(compiled.Query, want) { - t.Fatalf("trace query missing related aggregate contributor evidence %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Fatalf("trace query missing related aggregate contributor evidence %q:\n%s", want, testAQLText(t, compiled.Command)) } } } @@ -227,17 +231,17 @@ func TestCompileCellTracePreservesAggregateContributorsThroughCategoryRecode(t * }, }}, }) - compiled, err := CompileCellTraceOutputWithPolicy(output, "category", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileCellTraceOutput(ArangoTarget(), output, "category", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } for _, want := range []string{"FOR __loom_physical_trace_contributor", "IN child_set_1", ".payload.valueString", "resourceType:", "resourceId:"} { - if !strings.Contains(compiled.Query, want) { - t.Fatalf("recoded aggregate trace is missing contributor evidence %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Fatalf("recoded aggregate trace is missing contributor evidence %q:\n%s", want, testAQLText(t, compiled.Command)) } } - if containsBindValue(compiled.BindVars, "TRACE_CONTRIBUTORS_UNAVAILABLE") { - t.Fatalf("recoded aggregate trace unexpectedly reports unavailable contributors: %#v", compiled.BindVars) + if containsBindValue(testAQLBinds(t, compiled.Command), "TRACE_CONTRIBUTORS_UNAVAILABLE") { + t.Fatalf("recoded aggregate trace unexpectedly reports unavailable contributors: %#v", testAQLBinds(t, compiled.Command)) } } @@ -252,17 +256,17 @@ func TestCompileCellTraceExplainsNumericAggregateContributors(t *testing.T) { {Name: "mean_value", Operation: recipe.AggregateMean, Expr: &value}, }, }) - compiled, err := CompileCellTraceOutputWithPolicy(output, "sum_value", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileCellTraceOutput(ArangoTarget(), output, "sum_value", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } for _, want := range []string{"NUMERIC_AGGREGATE_NON_NUMERIC", "resourceType:", "resourceId:", "value:", "!= null"} { - if !strings.Contains(compiled.Query, want) { - t.Fatalf("numeric aggregate trace is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Fatalf("numeric aggregate trace is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } - if strings.Contains(compiled.Query, "AMBIGUOUS") { - t.Fatalf("a numeric reduction with explicit contributor enumeration must not be marked lossy:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), "AMBIGUOUS") { + t.Fatalf("a numeric reduction with explicit contributor enumeration must not be marked lossy:\n%s", testAQLText(t, compiled.Command)) } } @@ -285,7 +289,7 @@ func TestCompileWindowedAggregateCellTraceUsesEligibleContributors(t *testing.T) }}, }}, }) - compiled, err := CompileCellTraceOutputWithPolicy(output, "window_sum", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileCellTraceOutput(ArangoTarget(), output, "window_sum", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -295,8 +299,8 @@ func TestCompileWindowedAggregateCellTraceUsesEligibleContributors(t *testing.T) "FOR __loom_physical_trace_numeric_contributor IN FIRST(FOR __loom_physical_contributor_window_scope", "__loom_physical_contributor_window_timestamp", } { - if !strings.Contains(compiled.Query, want) { - t.Fatalf("windowed aggregate trace query missing eligible contributor evidence %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Fatalf("windowed aggregate trace query missing eligible contributor evidence %q:\n%s", want, testAQLText(t, compiled.Command)) } } } @@ -315,13 +319,13 @@ func TestCompileCellTraceUsesOwnerRecordEvidenceAsContributions(t *testing.T) { Key: fhirschema.CorrelatedKey{System: "http://loinc.org", Code: "8302-2"}, }}, }) - compiled, err := CompileCellTraceOutputWithPolicy(output, "height_records", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileCellTraceOutput(ArangoTarget(), output, "height_records", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } for _, want := range []string{"trace_owner_record", ".source.resourceType", ".source.resourceId", "ownerOrdinal", "INVALID_CHOICE_ARM"} { - if !strings.Contains(compiled.Query, want) { - t.Fatalf("owner-record trace query missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Fatalf("owner-record trace query missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } } @@ -351,39 +355,39 @@ func TestCompileCellTraceCarriesPivotAndDerivedLeafLineage(t *testing.T) { t.Fatal(err) } - alpha, err := CompileCellTraceOutputWithPolicy(compiled, "alpha", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + alpha, err := CompileCellTraceOutput(ArangoTarget(), compiled, "alpha", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } for _, want := range []string{"UNIQUE(FLATTEN", "resourceId:", "FILTER", " != null"} { - if !strings.Contains(alpha.Query, want) { - t.Fatalf("pivot trace query missing %q:\n%s", want, alpha.Query) + if !strings.Contains(testAQLText(t, alpha.Command), want) { + t.Fatalf("pivot trace query missing %q:\n%s", want, testAQLText(t, alpha.Command)) } } - if !containsBindValue(alpha.BindVars, ir.PhysicalCellTraceSourceDocumentField) || !containsBindValue(alpha.BindVars, ir.PhysicalCellTraceSourcePresencePrefix+"6") { - t.Fatalf("pivot trace did not bind source document and value-presence fields: %#v", alpha.BindVars) + if !containsBindValue(testAQLBinds(t, alpha.Command), ir.PhysicalCellTraceSourceDocumentField) || !containsBindValue(testAQLBinds(t, alpha.Command), ir.PhysicalCellTraceSourcePresencePrefix+"6") { + t.Fatalf("pivot trace did not bind source document and value-presence fields: %#v", testAQLBinds(t, alpha.Command)) } - group, err := CompileCellTraceOutputWithPolicy(compiled, "group_text", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + group, err := CompileCellTraceOutput(ArangoTarget(), compiled, "group_text", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if !containsBindValue(group.BindVars, "group_text") || !containsBindValue(group.BindVars, ir.PhysicalCellTraceSourcePresencePrefix+"2") { - t.Fatalf("pivot group-key trace does not retain the selected source field:\n%s", group.Query) + if !containsBindValue(testAQLBinds(t, group.Command), "group_text") || !containsBindValue(testAQLBinds(t, group.Command), ir.PhysicalCellTraceSourcePresencePrefix+"2") { + t.Fatalf("pivot group-key trace does not retain the selected source field:\n%s", testAQLText(t, group.Command)) } - derived, err := CompileCellTraceOutputWithPolicy(compiled, "alpha_plus_zero", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + derived, err := CompileCellTraceOutput(ArangoTarget(), compiled, "alpha_plus_zero", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if got := strings.Count(derived.Query, "FOR __loom_physical_trace_pivot_cell_source"); got != 4 { - t.Fatalf("derived trace has %d pivot contributor subqueries, want status/page for alpha and zero:\n%s", got, derived.Query) + if got := strings.Count(testAQLText(t, derived.Command), "FOR __loom_physical_trace_pivot_cell_source"); got != 4 { + t.Fatalf("derived trace has %d pivot contributor subqueries, want status/page for alpha and zero:\n%s", got, testAQLText(t, derived.Command)) } - if strings.Contains(derived.Query, ir.PhysicalCellTraceDerivedOmission) { - t.Fatalf("supported derived lineage has omission:\n%s", derived.Query) + if strings.Contains(testAQLText(t, derived.Command), ir.PhysicalCellTraceDerivedOmission) { + t.Fatalf("supported derived lineage has omission:\n%s", testAQLText(t, derived.Command)) } - if !containsBindValue(derived.BindVars, "alpha") || !containsBindValue(derived.BindVars, "zero") { - t.Fatalf("derived trace lost pivot category bindings: %#v", derived.BindVars) + if !containsBindValue(testAQLBinds(t, derived.Command), "alpha") || !containsBindValue(testAQLBinds(t, derived.Command), "zero") { + t.Fatalf("derived trace lost pivot category bindings: %#v", testAQLBinds(t, derived.Command)) } } @@ -393,35 +397,35 @@ func TestCompileCellTraceDistinguishesUnpivotValueFromGeneratedKey(t *testing.T) if err != nil { t.Fatal(err) } - value, err := CompileCellTraceOutputWithPolicy(compiled, "amount", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + value, err := CompileCellTraceOutput(ArangoTarget(), compiled, "amount", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } for _, want := range []string{"sourceDocument", "sourcePresence", "sourceSupported"} { - if !strings.Contains(value.Query, want) { - t.Fatalf("unpivot trace query missing %q:\n%s", want, value.Query) + if !strings.Contains(testAQLText(t, value.Command), want) { + t.Fatalf("unpivot trace query missing %q:\n%s", want, testAQLText(t, value.Command)) } } - if !containsBindValue(value.BindVars, ir.PhysicalCellTraceSourceDocumentField) { - t.Fatalf("unpivot trace did not bind the source-document field: %#v", value.BindVars) + if !containsBindValue(testAQLBinds(t, value.Command), ir.PhysicalCellTraceSourceDocumentField) { + t.Fatalf("unpivot trace did not bind the source-document field: %#v", testAQLBinds(t, value.Command)) } - passed, err := CompileCellTraceOutputWithPolicy(compiled, "text_value", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + passed, err := CompileCellTraceOutput(ArangoTarget(), compiled, "text_value", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if !containsBindValue(passed.BindVars, ir.PhysicalCellTracePassSourceDocumentField) || !containsBindValue(passed.BindVars, ir.PhysicalCellTracePassSourcePresenceField) { - t.Fatalf("unpivot pass-through trace lost source evidence fields: %#v", passed.BindVars) + if !containsBindValue(testAQLBinds(t, passed.Command), ir.PhysicalCellTracePassSourceDocumentField) || !containsBindValue(testAQLBinds(t, passed.Command), ir.PhysicalCellTracePassSourcePresenceField) { + t.Fatalf("unpivot pass-through trace lost source evidence fields: %#v", testAQLBinds(t, passed.Command)) } - if !containsBindValue(value.BindVars, "group_number") || !containsBindValue(value.BindVars, "numeric_value") { - t.Fatalf("unpivot trace lost selected source columns: %#v", value.BindVars) + if !containsBindValue(testAQLBinds(t, value.Command), "group_number") || !containsBindValue(testAQLBinds(t, value.Command), "numeric_value") { + t.Fatalf("unpivot trace lost selected source columns: %#v", testAQLBinds(t, value.Command)) } - key, err := CompileCellTraceOutputWithPolicy(compiled, "measure", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + key, err := CompileCellTraceOutput(ArangoTarget(), compiled, "measure", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if !strings.Contains(key.Query, ir.PhysicalCellTraceGeneratedKeyOmission) { - t.Fatalf("unpivot key trace omitted its generated-key reason:\n%s", key.Query) + if !strings.Contains(testAQLText(t, key.Command), ir.PhysicalCellTraceGeneratedKeyOmission) { + t.Fatalf("unpivot key trace omitted its generated-key reason:\n%s", testAQLText(t, key.Command)) } } @@ -450,7 +454,7 @@ func TestConstructionCellTraceResolvesDeriveInputsThroughFilterStage(t *testing. func TestCompileConstructionCellTraceReturnsDerivedValueAndStageEvidence(t *testing.T) { output := compilePopulationMappingOutput(t, constructionCellTraceRecipeOutput()) - compiled, err := CompileCellTraceOutputWithPolicy(output, "total", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileCellTraceOutput(ArangoTarget(), output, "total", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -462,15 +466,15 @@ func TestCompileConstructionCellTraceReturnsDerivedValueAndStageEvidence(t *test "@cell_trace_offset, @cell_trace_fetch_limit", "inputStageId:", "inputColumnId:", "outputStageId:", "outputColumnId:", "finalStageId:", "constructionId:", "operation:", } { - if !strings.Contains(compiled.Query, want) { - t.Fatalf("construction cell trace query is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Fatalf("construction cell trace query is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } for _, want := range []string{ "source_projection", "amount_id", "amount", "derive_total", "total_id", "total", "keep_positive", "calc_total", "DERIVE", } { - if !containsBindValue(compiled.BindVars, want) { - t.Errorf("construction trace bind value %q is missing from %#v", want, compiled.BindVars) + if !containsBindValue(testAQLBinds(t, compiled.Command), want) { + t.Errorf("construction trace bind value %q is missing from %#v", want, testAQLBinds(t, compiled.Command)) } } if compiled.IdentityPartsColumn != ir.PhysicalCellTraceIdentityPartsField || compiled.ExplicitIdentityColumn != "" { @@ -501,7 +505,7 @@ func TestConstructionRelatedSourceCellTraceReplaysScopedRoutePerOccurrence(t *te t.Fatal("trace did not retain the exact producer route subplan") } - compiled, err := CompileCellTraceOutputWithPolicy(output, "observation_status", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileCellTraceOutput(ArangoTarget(), output, "observation_status", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -515,23 +519,23 @@ func TestConstructionRelatedSourceCellTraceReplaysScopedRoutePerOccurrence(t *te "LIMIT @cell_trace_offset, @cell_trace_fetch_limit", "NO_MATCH", } { - if !strings.Contains(compiled.Query, want) { - t.Errorf("related-source trace query is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Errorf("related-source trace query is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } - if strings.Contains(compiled.Query, "UNIQUE(") || strings.Contains(compiled.Query, "SORTED_UNIQUE(") || strings.Contains(compiled.Query, ".payload.status != null") { - t.Fatalf("related-source trace must preserve path occurrences and null values without deduplication/filtering:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), "UNIQUE(") || strings.Contains(testAQLText(t, compiled.Command), "SORTED_UNIQUE(") || strings.Contains(testAQLText(t, compiled.Command), ".payload.status != null") { + t.Fatalf("related-source trace must preserve path occurrences and null values without deduplication/filtering:\n%s", testAQLText(t, compiled.Command)) } for _, want := range []string{ "fixture_program-project-a", "generation-a", "subject_Patient", "Patient", "Observation", "resourceType", "resourceId", "value", "outputStageId", "outputColumnId", "observation_status", "add_observation_status", "finalStageId", "RELATED_SOURCE", } { - if !containsBindValue(compiled.BindVars, want) { - t.Errorf("related-source route bind value %q is missing from %#v", want, compiled.BindVars) + if !containsBindValue(testAQLBinds(t, compiled.Command), want) { + t.Errorf("related-source route bind value %q is missing from %#v", want, testAQLBinds(t, compiled.Command)) } } - if !strings.Contains(compiled.Query, "dataset_generation") || !strings.Contains(compiled.Query, "auth_resource_path") { - t.Fatalf("related-source trace lost generation or authorization filters:\n%s", compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), "dataset_generation") || !strings.Contains(testAQLText(t, compiled.Command), "auth_resource_path") { + t.Fatalf("related-source trace lost generation or authorization filters:\n%s", testAQLText(t, compiled.Command)) } if compiled.IdentityPartsColumn != ir.PhysicalCellTraceIdentityPartsField || compiled.ExplicitIdentityColumn != "" || compiled.RowIdentity == nil { t.Fatalf("related-source trace lost exact output row identity: %#v", compiled) @@ -542,7 +546,7 @@ func TestConstructionCellTraceRejectsMalformedPublishedIdentity(t *testing.T) { output := compilePopulationMappingOutput(t, constructionCellTraceRecipeOutput()) output.RowIdentity = output.RowIdentity.Clone() output.RowIdentity.Fields = []string{"project", "not_the_root_key"} - if _, err := CompileCellTraceOutputWithPolicy(output, "total", 0, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil || err.Error() != "construction cell trace row identity does not match the final stage" { + if _, err := CompileCellTraceOutput(ArangoTarget(), output, "total", 0, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil || err.Error() != "construction cell trace row identity does not match the final stage" { t.Fatalf("malformed construction identity error = %v", err) } } @@ -565,7 +569,7 @@ func TestConstructionCellTraceRejectsStructuredGeneratedIdentity(t *testing.T) { output.Plan.StageSequence.Stages[lastStage].OutputColumns[index].Kind = "object" } } - if _, err := CompileCellTraceOutputWithPolicy(output, "alpha", 0, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil || err.Error() != "construction cell trace row identity does not match the final stage" { + if _, err := CompileCellTraceOutput(ArangoTarget(), output, "alpha", 0, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil || err.Error() != "construction cell trace row identity does not match the final stage" { t.Fatalf("structured generated identity error = %v", err) } } @@ -578,11 +582,11 @@ func assertConstructionTraceUsesPreviewIdentity(t *testing.T, output lower.Compi if compiled.RowIdentity == nil || !reflect.DeepEqual(compiled.RowIdentity.Fields, output.RowIdentity.Fields) { t.Fatalf("cell trace identity = %#v, want Preview identity %#v", compiled.RowIdentity, output.RowIdentity) } - if compiled.BindVars["project"] != "fixture_program-project-a" { - t.Fatalf("cell trace project identity bind = %#v, want the same pinned project as Preview", compiled.BindVars["project"]) + if testAQLBinds(t, compiled.Command)["project"] != "fixture_program-project-a" { + t.Fatalf("cell trace project identity bind = %#v, want the same pinned project as Preview", testAQLBinds(t, compiled.Command)["project"]) } - if !strings.Contains(compiled.Query, "[@project, __loom_construction_final_row._key]") { - t.Fatalf("cell trace identity parts do not match Preview project/key order:\n%s", compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), "[@project, __loom_construction_final_row._key]") { + t.Fatalf("cell trace identity parts do not match Preview project/key order:\n%s", testAQLText(t, compiled.Command)) } } @@ -626,16 +630,16 @@ func TestConstructionCellTraceMarksPivotLineageAsUnsupported(t *testing.T) { if len(lineage.RowIdentityFields) != 1 || lineage.RowIdentityFields[0] != "__loom_row_id" { t.Fatalf("pivot trace identity = %#v, want the compiler-generated explicit scalar identity", lineage.RowIdentityFields) } - compiled, err := CompileCellTraceOutputWithPolicy(output, "alpha", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileCellTraceOutput(ArangoTarget(), output, "alpha", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if !strings.Contains(compiled.Query, "__loom_construction_final_row.alpha") || !strings.Contains(compiled.Query, "CONSTRUCTION_TRACE_OPERATION_UNSUPPORTED") { - t.Fatalf("unsupported reshape trace must retain the value and explicit omission:\n%s\n%#v", compiled.Query, compiled.BindVars) + if !strings.Contains(testAQLText(t, compiled.Command), "__loom_construction_final_row.alpha") || !strings.Contains(testAQLText(t, compiled.Command), "CONSTRUCTION_TRACE_OPERATION_UNSUPPORTED") { + t.Fatalf("unsupported reshape trace must retain the value and explicit omission:\n%s\n%#v", testAQLText(t, compiled.Command), testAQLBinds(t, compiled.Command)) } if compiled.ExplicitIdentityColumn != ir.PhysicalCellTraceExplicitIdentityField || compiled.IdentityPartsColumn != "" || - !strings.Contains(compiled.Query, "__loom_construction_final_row.__loom_row_id") { - t.Fatalf("pivot trace must match Preview's literal generated row ID without re-hashing:\n%s\n%#v", compiled.Query, compiled) + !strings.Contains(testAQLText(t, compiled.Command), "__loom_construction_final_row.__loom_row_id") { + t.Fatalf("pivot trace must match Preview's literal generated row ID without re-hashing:\n%s\n%#v", testAQLText(t, compiled.Command), compiled) } } @@ -687,10 +691,283 @@ func TestCompileCellTraceRejectsUnknownOrHiddenColumn(t *testing.T) { Name: "Patients", RootResourceType: "Patient", RowGrain: "patient", Fields: []recipe.Field{{Name: "patient_id", Expr: recipe.Expression{Select: "root.id"}}}, }) - if _, err := CompileCellTraceOutputWithPolicy(output, "missing", 0, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil { + if _, err := CompileCellTraceOutput(ArangoTarget(), output, "missing", 0, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil { t.Fatal("unknown trace column was accepted") } - if _, err := CompileCellTraceOutputWithPolicy(output, "__loom_row_id", 0, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil { + if _, err := CompileCellTraceOutput(ArangoTarget(), output, "__loom_row_id", 0, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil { t.Fatal("internal row identity was accepted as a trace column") } } + +func mustPostgresCellTraceTarget(t *testing.T) Target { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("pgir02_it_unit") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + target, err := PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("PostgreSQLTarget: %v", err) + } + return target +} + +func compilePopulationMappingOutputWithPolicy(t *testing.T, output recipe.Output, policy ir.PhysicalOptimizationPolicy) lower.CompiledRecipeOutput { + t.Helper() + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "population-mapping-test", + TranslationVersion: "population-mapping-test", + Outputs: []recipe.Output{output}, + } + bindings := recipe.RuntimeBindings{Project: "fixture_program-project-a", SelectionProject: "fixture_program/project-a", DatasetGeneration: "generation-a", SelectionMembersCollection: "loom_explorer_selection_members"} + plan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + t.Fatal(err) + } + resolved, err := semantic.ResolveRecipePlan(plan, "scope-a", "generation-a") + if err != nil { + t.Fatal(err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, policy) + if err != nil { + t.Fatal(err) + } + return compiled.Outputs[0] +} + +func sql24PostgreSQLOutput() recipe.Output { + return recipe.Output{ + Name: "Patients", RootResourceType: "Patient", RowGrain: "patient", + RootColumnNaming: recipe.RootColumnNamingExact, + Fields: []recipe.Field{ + {Name: "patient_id", Expr: recipe.Expression{Select: "root.id"}}, + {Name: "gender", Expr: recipe.Expression{Select: "root.gender"}}, + }, + } +} + +func sql24TraversalOutput() recipe.Output { + return recipe.Output{ + Name: "Patients", RootResourceType: "Patient", RowGrain: "patient", + RootColumnNaming: recipe.RootColumnNamingExact, TraversalColumnNaming: recipe.TraversalColumnNamingExact, + Fields: []recipe.Field{{Name: "patient_id", Expr: recipe.Expression{Select: "root.id"}}}, + Traversals: []recipe.Traversal{{ + Name: "subject_Patient", ToResourceType: "Condition", Alias: "condition", + Fields: []recipe.Field{{Name: "condition_id", Expr: recipe.Expression{Select: "id"}, ValueMode: recipe.ValueModeFirst}}, + }}, + } +} + +func sql24PostgresSQL(t *testing.T, compiled CompiledCellTraceQuery) (string, []command.PostgreSQLParameter) { + t.Helper() + sqlText, parameters, err := compiled.Command.PostgreSQLParameters() + if err != nil { + t.Fatalf("compiled cell trace command is not a PostgreSQL command: %v", err) + } + return sqlText, parameters +} + +func TestCompileCellTracePostgreSQLRootExtract(t *testing.T) { + output := compilePopulationMappingOutput(t, sql24PostgreSQLOutput()) + compiled, err := CompileCellTraceOutput(mustPostgresCellTraceTarget(t), output, "gender", 2, 5, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("PostgreSQL root-extract cell trace compile: %v", err) + } + sqlText, parameters := sql24PostgresSQL(t, compiled) + for _, want := range []string{ + `AS "__loom_trace_value"`, `AS "__loom_trace_contributions"`, + `AS "__loom_trace_identity_parts"`, `AS "__loom_trace_status"`, + `AS "__loom_trace_has_more"`, `AS "__loom_trace_omission"`, + `'NO_MATCH'`, `'RECORDED_NULL'`, `'VALUE'`, `false AS "__loom_trace_has_more"`, + `jsonb_build_object('resourceType',`, + } { + if !strings.Contains(sqlText, want) { + t.Fatalf("PostgreSQL root-extract trace is missing %q:\n%s", want, sqlText) + } + } + if compiled.IdentityPartsColumn != ir.PhysicalCellTraceIdentityPartsField || compiled.ExplicitIdentityColumn != "" { + t.Fatalf("trace identity columns = (%q, %q)", compiled.IdentityPartsColumn, compiled.ExplicitIdentityColumn) + } + if compiled.ValueColumn != ir.PhysicalCellTraceValueField || compiled.StatusColumn != ir.PhysicalCellTraceStatusField || + compiled.ContributionsColumn != ir.PhysicalCellTraceContributionsField || + compiled.HasMoreColumn != ir.PhysicalCellTraceHasMoreField || compiled.OmissionColumn != ir.PhysicalCellTraceOmissionField { + t.Fatalf("contract columns do not match the registered field names: %#v", compiled) + } + if compiled.ContributionOffset != 2 || compiled.ContributionLimit != 5 { + t.Fatalf("trace page = %d/%d", compiled.ContributionOffset, compiled.ContributionLimit) + } + if len(parameters) == 0 { + t.Fatal("every scope and page fact must travel as an ordered parameter") + } + if output.Plan.Operations[terminalIndexOf(t, output.Plan)].CellTraceReturn != nil { + t.Fatal("compiling a diagnostic terminal mutated the canonical Preview plan") + } +} + +func TestCompileCellTracePostgreSQLAuthoredIdentityMintsExplicitColumn(t *testing.T) { + output := sql24PostgreSQLOutput() + output.Identity = &recipe.Identity{Name: "row", Expr: recipe.Expression{Select: "root.id"}} + root := compilePopulationMappingOutput(t, output) + compiled, err := CompileCellTraceOutput(mustPostgresCellTraceTarget(t), root, "gender", 0, 5, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("authored-identity cell trace must compile on the PostgreSQL target: %v", err) + } + if compiled.ExplicitIdentityColumn != ir.PhysicalCellTraceExplicitIdentityField || compiled.IdentityPartsColumn != "" { + t.Fatalf("contract columns = (%q, %q), want the explicit identity named by the mint", + compiled.ExplicitIdentityColumn, compiled.IdentityPartsColumn) + } + sqlText, _ := sql24PostgresSQL(t, compiled) + if !strings.Contains(sqlText, ` AS "`+ir.PhysicalCellTraceExplicitIdentityField+`"`) { + t.Fatalf("the minted explicit identity must publish as its contract column:\n%s", sqlText) + } + if strings.Contains(sqlText, ir.PhysicalCellTraceIdentityPartsField) { + t.Fatalf("the identity-parts spelling must stay unpublished under the mint's pick law:\n%s", sqlText) + } + terminalSeen := false + for _, operation := range root.Plan.Operations { + if operation.Kind == ir.PhysicalCellTraceReturnOp { + terminalSeen = true + } + } + if terminalSeen { + t.Fatal("compiling the trace must not mutate the canonical plan") + } +} + +func terminalIndexOf(t *testing.T, plan ir.PhysicalPlan) int { + t.Helper() + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalCellTraceReturnOp { + t.Fatalf("canonical preview plan already carries the evidence terminal") + } + _ = operation + } + return len(plan.Operations) - 1 +} + +func TestCompileCellTracePostgreSQLReducedSetContributors(t *testing.T) { + output := compilePopulationMappingOutputWithPolicy(t, sql24TraversalOutput(), + ir.DefaultPhysicalOptimizationPolicy().WithRule(ir.PhysicalOptimizationRuleEndpointTraversal, false)) + compiled, err := CompileCellTraceOutput(mustPostgresCellTraceTarget(t), output, "condition_id", 5, 10, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("PostgreSQL reduced-set cell trace compile: %v", err) + } + sqlText, parameters := sql24PostgresSQL(t, compiled) + for _, want := range []string{ + `jsonb_agg(jsonb_build_object('resourceType',`, `'resourceId',`, `'value',`, + `AS capped`, `OFFSET $`, `WITH ORDINALITY`, + `'AMBIGUOUS'`, `'NO_MATCH'`, `AS "__loom_trace_has_more"`, + } { + if !strings.Contains(sqlText, want) { + t.Fatalf("PostgreSQL reduced-set trace is missing %q:\n%s", want, sqlText) + } + } + if compiled.ContributionOffset != 5 || compiled.ContributionLimit != 10 { + t.Fatalf("trace page = %d/%d", compiled.ContributionOffset, compiled.ContributionLimit) + } + if compiled.IdentityPartsColumn != ir.PhysicalCellTraceIdentityPartsField { + t.Fatalf("identity column = %q", compiled.IdentityPartsColumn) + } + _ = parameters +} + +func TestCompileCellTracePostgreSQLRefusesEndpointLookupTyped(t *testing.T) { + output := compilePopulationMappingOutput(t, sql24TraversalOutput()) + _, err := CompileCellTraceOutput(mustPostgresCellTraceTarget(t), output, "condition_id", 5, 10, ir.DefaultPhysicalOptimizationPolicy()) + if err == nil || !strings.Contains(err.Error(), "endpoint-lookup") { + t.Fatalf("endpoint-lookup shape must refuse with the spine's typed voice, got: %v", err) + } + if !errors.Is(err, postgresrender.ErrUnsupportedPostgresPlan) { + t.Fatalf("refusal must be the typed unsupported-plan error: %v", err) + } +} + +func TestCompileCellTracePostgreSQLNamedRefusals(t *testing.T) { + target := mustPostgresCellTraceTarget(t) + + groupRows := compilePopulationMappingOutput(t, recipe.Output{ + Name: "GroupedSpecimens", RootResourceType: "Specimen", RowGrain: "groups", + RootColumnNaming: recipe.RootColumnNamingExact, + Fields: []recipe.Field{{Name: "resourceType", ColumnID: "resource_type", Expr: recipe.Expression{Select: "root.resourceType"}}}, + GroupRows: &recipe.GroupRows{ + RevisionID: "grouprev_1", UnassignedMemberPolicy: "EXCLUDE", + RowValues: []recipe.GroupRowValuePolicy{{ColumnID: "resource_type", Policy: recipe.ConstructionRowValueOne}}, + }, + }) + if _, err := CompileCellTraceOutput(target, groupRows, "resourceType", 0, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil || + !strings.Contains(err.Error(), "explicit group rows cell trace is unsupported on the PostgreSQL compile target") { + t.Fatalf("group-rows trace must refuse by name: %v", err) + } + + construction := compilePopulationMappingOutput(t, constructionCellTraceRecipeOutput()) + if construction.Plan.StageSequence == nil { + t.Fatal("construction fixture lost its stage sequence") + } + if _, err := CompileCellTraceOutput(target, construction, "total", 0, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil || + !strings.Contains(err.Error(), "composed construction cell trace is unsupported on the PostgreSQL compile target") { + t.Fatalf("composed-construction trace must refuse by name: %v", err) + } + + aggregateValue := recipe.Expression{Select: "valueQuantity.value"} + aggregate := compilePopulationMappingOutput(t, recipe.Output{ + Name: "Patients", RootResourceType: "Patient", RowGrain: "patient", + RootColumnNaming: recipe.RootColumnNamingExact, TraversalColumnNaming: recipe.TraversalColumnNamingExact, + Fields: []recipe.Field{{Name: "patient_id", Expr: recipe.Expression{Select: "root.id"}}}, + Traversals: []recipe.Traversal{{ + Name: "subject_Patient", ToResourceType: "Observation", Alias: "observation", + Aggregates: []recipe.Aggregate{{Name: "maximum_value", Operation: recipe.AggregateMax, Expr: &aggregateValue}}, + }}, + }) + if _, err := CompileCellTraceOutput(target, aggregate, "maximum_value", 0, 25, ir.DefaultPhysicalOptimizationPolicy()); err == nil || + !strings.Contains(err.Error(), "aggregate, owner-records, call-recode, and prepared-extract values") { + t.Fatalf("aggregate trace must refuse with the terminal-surface voice: %v", err) + } + + root := compilePopulationMappingOutput(t, sql24PostgreSQLOutput()) + if _, err := CompileCellTraceOutput(target, root, "missing", 0, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil { + t.Fatal("unknown trace column was accepted on the PostgreSQL target") + } + if _, err := CompileCellTraceOutput(target, root, "__loom_row_id", 0, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil { + t.Fatal("internal row identity was accepted as a trace column on the PostgreSQL target") + } + if _, err := CompileCellTraceOutput(target, root, "gender", -1, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil || + !strings.Contains(err.Error(), "cell trace offset cannot be negative") { + t.Fatalf("negative offset must refuse with the shared page voice: %v", err) + } + if defaulted, err := CompileCellTraceOutput(target, root, "gender", 0, 0, ir.DefaultPhysicalOptimizationPolicy()); err != nil || + defaulted.ContributionLimit != 25 { + t.Fatalf("limit <= 0 must take the registered default 25 on the shared page contract: %v / %+v", err, defaulted) + } + if _, err := CompileCellTraceOutput(target, root, "gender", 0, MaxCellTraceContributions+1, ir.DefaultPhysicalOptimizationPolicy()); err == nil || + !strings.Contains(err.Error(), "cell trace contribution limit exceeds") { + t.Fatalf("over-max limit must refuse with the shared page voice: %v", err) + } +} + +func TestCompileCellTraceArangoVoiceUnchangedForPostgresArms(t *testing.T) { + root := compilePopulationMappingOutput(t, sql24PostgreSQLOutput()) + compiled, err := CompileCellTraceOutput(ArangoTarget(), root, "gender", 2, 5, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"root.payload.gender", "@cell_trace_offset", "@cell_trace_limit", "LENGTH(__loom_physical_trace_status_candidates) == 0"} { + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Fatalf("AQL root-extract voice lost %q:\n%s", want, testAQLText(t, compiled.Command)) + } + } + + traversal := compilePopulationMappingOutput(t, sql24TraversalOutput()) + compiled, err = CompileCellTraceOutput(ArangoTarget(), traversal, "condition_id", 5, 10, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"FOR __loom_physical_trace_contributor", "IN child_set_1", "LENGTH(__loom_physical_trace_status_candidates) > 1 ? \"AMBIGUOUS\"", "SLICE("} { + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Fatalf("AQL reduced-set voice lost %q:\n%s", want, testAQLText(t, compiled.Command)) + } + } + if compiled.ContributionOffset != 5 || compiled.ContributionLimit != 10 { + t.Fatalf("AQL trace page = %d/%d", compiled.ContributionOffset, compiled.ContributionLimit) + } +} diff --git a/internal/dataframe/compiler/cohort_drop_boundary_arango_test.go b/internal/dataframe/compiler/cohort_drop_boundary_arango_test.go index 4ba1c3330..7ad0c57a6 100644 --- a/internal/dataframe/compiler/cohort_drop_boundary_arango_test.go +++ b/internal/dataframe/compiler/cohort_drop_boundary_arango_test.go @@ -9,24 +9,22 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/projectid" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) -// This fixture proves that a pinned member missing from the current resource -// collection cannot pass a row-dropping construction stage before COHORT_GROUP. func TestPreCohortDropStagesExcludeMissingPinnedMembersAgainstArango(t *testing.T) { ctx, client := openConstructionReshapeArango(t) - collections := []store.CollectionSpec{ + collections := []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, {Name: "loom_explorer_explicit_group_revisions"}, {Name: "loom_explorer_selections"}, {Name: "loom_explorer_explicit_group_definitions"}, {Name: "loom_explorer_explicit_group_memberships"}, {Name: "loom_explorer_selection_members"}, } - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: collections}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: collections}); err != nil { t.Fatal(err) } @@ -190,7 +188,7 @@ func TestPreCohortDropStagesExcludeMissingPinnedMembersAgainstArango(t *testing. if cohort == nil || cohort.PreserveMissingMembers { t.Fatalf("%s PreserveMissingMembers = %#v, want false", name, cohort) } - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile %s cohort query: %v", name, err) } diff --git a/internal/dataframe/compiler/command/command.go b/internal/dataframe/compiler/command/command.go new file mode 100644 index 000000000..478e79f6b --- /dev/null +++ b/internal/dataframe/compiler/command/command.go @@ -0,0 +1,180 @@ +package command + +import ( + "errors" + "fmt" + "reflect" + "strings" +) + +type CompiledBackend string + +const ( + CompiledBackendAQL CompiledBackend = "AQL" + CompiledBackendPostgreSQL CompiledBackend = "POSTGRESQL" +) + +var ErrCompiledCommandBackendMismatch = errors.New("compiled command backend mismatch") + +var ErrCompiledCommandBackendNotRegistered = errors.New("compiled command backend is not registered") + +type AQLCommand struct { + Query string + BindVars map[string]any +} + +type PostgreSQLParameterType string + +const ( + PostgreSQLParameterNull PostgreSQLParameterType = "null" + PostgreSQLParameterText PostgreSQLParameterType = "text" + PostgreSQLParameterBigint PostgreSQLParameterType = "bigint" + PostgreSQLParameterDoublePrecision PostgreSQLParameterType = "double precision" + PostgreSQLParameterBoolean PostgreSQLParameterType = "boolean" + PostgreSQLParameterDate PostgreSQLParameterType = "date" + PostgreSQLParameterTimestamptz PostgreSQLParameterType = "timestamp with time zone" + PostgreSQLParameterJSON PostgreSQLParameterType = "json" + PostgreSQLParameterJSONB PostgreSQLParameterType = "jsonb" +) + +func (t PostgreSQLParameterType) registered() bool { + switch t { + case PostgreSQLParameterNull, PostgreSQLParameterText, PostgreSQLParameterBigint, + PostgreSQLParameterDoublePrecision, PostgreSQLParameterBoolean, PostgreSQLParameterDate, + PostgreSQLParameterTimestamptz, PostgreSQLParameterJSON, PostgreSQLParameterJSONB: + return true + default: + return false + } +} + +type PostgreSQLParameter struct { + Type PostgreSQLParameterType + Value any +} + +type PostgreSQLCommand struct { + SQL string + Args []PostgreSQLParameter +} + +type CompiledCommand struct { + backend CompiledBackend + aql AQLCommand + postgreSQL PostgreSQLCommand +} + +func NewAQLCommand(query string, bindVars map[string]any) (CompiledCommand, error) { + if strings.TrimSpace(query) == "" { + return CompiledCommand{}, fmt.Errorf("AQL compiled command requires rendered query text") + } + return CompiledCommand{ + backend: CompiledBackendAQL, + aql: AQLCommand{Query: query, BindVars: bindVars}, + }, nil +} + +func NewPostgreSQLCommand(sql string, args []PostgreSQLParameter) (CompiledCommand, error) { + if strings.TrimSpace(sql) == "" { + return CompiledCommand{}, fmt.Errorf("PostgreSQL compiled command requires rendered statement SQL") + } + if err := validatePostgreSQLParameters(args); err != nil { + return CompiledCommand{}, err + } + var copied []PostgreSQLParameter + if len(args) != 0 { + copied = append([]PostgreSQLParameter(nil), args...) + } + return CompiledCommand{ + backend: CompiledBackendPostgreSQL, + postgreSQL: PostgreSQLCommand{SQL: sql, Args: copied}, + }, nil +} + +func (c CompiledCommand) Backend() CompiledBackend { return c.backend } + +func (c CompiledCommand) WithAQLBinds(overrides map[string]any) (CompiledCommand, error) { + switch c.backend { + case CompiledBackendAQL: + case CompiledBackendPostgreSQL: + return CompiledCommand{}, fmt.Errorf("%w: %s statements bind positional arguments and have no named binds to override", + ErrCompiledCommandBackendMismatch, string(CompiledBackendPostgreSQL)) + default: + return CompiledCommand{}, fmt.Errorf("%w: cannot override binds on command with unregistered backend %q", + ErrCompiledCommandBackendNotRegistered, string(c.backend)) + } + if len(overrides) == 0 { + return CompiledCommand{}, fmt.Errorf("%w: bind overrides must name at least one existing bind", + ErrCompiledCommandBackendMismatch) + } + binds := make(map[string]any, len(c.aql.BindVars)) + for key, value := range c.aql.BindVars { + binds[key] = value + } + for key, value := range overrides { + current, exists := binds[key] + if !exists { + return CompiledCommand{}, fmt.Errorf("%w: bind %q is not part of the rendered AQL command", + ErrCompiledCommandBackendMismatch, key) + } + if current == nil || value == nil || reflect.TypeOf(current) != reflect.TypeOf(value) { + return CompiledCommand{}, fmt.Errorf("%w: bind %q override type %T does not match the rendered type %T", + ErrCompiledCommandBackendMismatch, key, value, current) + } + binds[key] = value + } + return CompiledCommand{ + backend: CompiledBackendAQL, + aql: AQLCommand{Query: c.aql.Query, BindVars: binds}, + }, nil +} + +func (c CompiledCommand) AQLParameters() (string, map[string]any, error) { + switch c.backend { + case CompiledBackendAQL: + return c.aql.Query, c.aql.BindVars, nil + case CompiledBackendPostgreSQL: + return "", nil, fmt.Errorf("%w: refusing to pass %s statement SQL to the AQL QueryRows(query, bindVars) adapter", + ErrCompiledCommandBackendMismatch, string(CompiledBackendPostgreSQL)) + default: + return "", nil, fmt.Errorf("%w: cannot dispatch command with unregistered backend %q to AQL", + ErrCompiledCommandBackendNotRegistered, string(c.backend)) + } +} + +func (c CompiledCommand) PostgreSQLParameters() (string, []PostgreSQLParameter, error) { + switch c.backend { + case CompiledBackendPostgreSQL: + var copied []PostgreSQLParameter + if len(c.postgreSQL.Args) != 0 { + copied = append([]PostgreSQLParameter(nil), c.postgreSQL.Args...) + } + return c.postgreSQL.SQL, copied, nil + case CompiledBackendAQL: + return "", nil, fmt.Errorf("%w: refusing to pass %s query to the PostgreSQL statement executor", + ErrCompiledCommandBackendMismatch, string(CompiledBackendAQL)) + default: + return "", nil, fmt.Errorf("%w: cannot dispatch command with unregistered backend %q to PostgreSQL", + ErrCompiledCommandBackendNotRegistered, string(c.backend)) + } +} + +func (c CompiledCommand) Dispatch(aqlAdapter func(AQLCommand) error, postgreSQLAdapter func(PostgreSQLCommand) error) error { + switch c.backend { + case CompiledBackendAQL: + if aqlAdapter == nil { + return fmt.Errorf("%w: no AQL adapter registered for %s command", + ErrCompiledCommandBackendNotRegistered, string(CompiledBackendAQL)) + } + return aqlAdapter(c.aql) + case CompiledBackendPostgreSQL: + if postgreSQLAdapter == nil { + return fmt.Errorf("%w: no PostgreSQL adapter registered for %s command", + ErrCompiledCommandBackendNotRegistered, string(CompiledBackendPostgreSQL)) + } + return postgreSQLAdapter(c.postgreSQL) + default: + return fmt.Errorf("%w: cannot dispatch command with unregistered backend %q", + ErrCompiledCommandBackendNotRegistered, string(c.backend)) + } +} diff --git a/internal/dataframe/compiler/command/compiled_command_test.go b/internal/dataframe/compiler/command/compiled_command_test.go new file mode 100644 index 000000000..34adbcd17 --- /dev/null +++ b/internal/dataframe/compiler/command/compiled_command_test.go @@ -0,0 +1,350 @@ +package command_test + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/render/aql" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" +) + +func TestCompiledCommand(t *testing.T) { + t.Run("AQLTagCarriesQueryAndBindMapUnchanged", func(t *testing.T) { + bindVars := map[string]any{"@@collection": "Patient", "limit": 7} + compiledCommand, err := command.NewAQLCommand("FOR d IN @@collection RETURN d", bindVars) + if err != nil { + t.Fatalf("NewAQLCommand: %v", err) + } + if compiledCommand.Backend() != command.CompiledBackendAQL { + t.Fatalf("backend = %q, want %q", compiledCommand.Backend(), command.CompiledBackendAQL) + } + query, gotBinds, err := compiledCommand.AQLParameters() + if err != nil { + t.Fatalf("AQLParameters: %v", err) + } + if query != "FOR d IN @@collection RETURN d" { + t.Fatalf("query = %q", query) + } + if reflect.ValueOf(gotBinds).Pointer() != reflect.ValueOf(bindVars).Pointer() { + t.Fatal("AQL dispatch must carry the exact bind map instance unchanged") + } + if _, _, err := compiledCommand.PostgreSQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("AQL command to PostgreSQL dispatch error = %v, want mismatch", err) + } + }) + + t.Run("PostgreSQLTagCarriesSQLAndOrderedTypedArgs", func(t *testing.T) { + args := []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + {Type: command.PostgreSQLParameterBigint, Value: int64(7)}, + {Type: command.PostgreSQLParameterNull, Value: nil}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage("null")}, + } + compiledCommand, err := command.NewPostgreSQLCommand("SELECT $1, $2, $3, $4", args) + if err != nil { + t.Fatalf("NewPostgreSQLCommand: %v", err) + } + if compiledCommand.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("backend = %q, want %q", compiledCommand.Backend(), command.CompiledBackendPostgreSQL) + } + sql, gotArgs, err := compiledCommand.PostgreSQLParameters() + if err != nil { + t.Fatalf("PostgreSQLParameters: %v", err) + } + if sql != "SELECT $1, $2, $3, $4" { + t.Fatalf("sql = %q", sql) + } + if !reflect.DeepEqual(gotArgs, args) { + t.Fatalf("ordered args changed: %#v", gotArgs) + } + if gotArgs[0].Value != "Patient" || gotArgs[1].Value != int64(7) || + gotArgs[2].Type != command.PostgreSQLParameterNull || gotArgs[2].Value != nil { + t.Fatalf("argument order or null/JSON-null distinction lost: %#v", gotArgs) + } + args[0].Value = "Observation" + if _, reread, err := compiledCommand.PostgreSQLParameters(); err != nil || reread[0].Value != "Patient" { + t.Fatalf("args must be copied at construction: %#v %v", reread, err) + } + gotArgs[0].Value = "Corrupted" + if _, afterMutate, err := compiledCommand.PostgreSQLParameters(); err != nil || afterMutate[0].Value != "Patient" { + t.Fatalf("PostgreSQLParameters must return a fresh copy on every read: %#v %v", afterMutate, err) + } + if _, _, err := compiledCommand.AQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("PostgreSQL command to AQL dispatch error = %v, want mismatch", err) + } else if !strings.Contains(err.Error(), "AQL") { + t.Fatalf("mismatch error must name the refused AQL adapter: %v", err) + } + }) + + t.Run("ExhaustiveDispatchRejectsMismatchedPairsBeforeExecution", func(t *testing.T) { + var zero command.CompiledCommand + if zero.Backend() != command.CompiledBackend("") { + t.Fatalf("zero-value backend = %q", zero.Backend()) + } + if _, _, err := zero.AQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("zero-value AQL dispatch error = %v, want unregistered backend", err) + } + if _, _, err := zero.PostgreSQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("zero-value PostgreSQL dispatch error = %v, want unregistered backend", err) + } + aqlCalls, pgCalls := 0, 0 + if err := zero.Dispatch(func(command.AQLCommand) error { aqlCalls++; return nil }, func(command.PostgreSQLCommand) error { pgCalls++; return nil }); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("zero-value Dispatch error = %v, want unregistered backend", err) + } + if aqlCalls != 0 || pgCalls != 0 { + t.Fatalf("unregistered backend invoked adapters: aql=%d postgresql=%d", aqlCalls, pgCalls) + } + + aqlCommand, err := command.NewAQLCommand("RETURN 1", nil) + if err != nil { + t.Fatalf("NewAQLCommand: %v", err) + } + if err := aqlCommand.Dispatch(nil, func(command.PostgreSQLCommand) error { pgCalls++; return nil }); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("AQL command without AQL adapter error = %v, want backend-not-registered", err) + } + if pgCalls != 0 { + t.Fatal("mismatched Dispatch must not run the other backend's adapter") + } + if err := aqlCommand.Dispatch(func(command.AQLCommand) error { aqlCalls++; return nil }, nil); err != nil { + t.Fatalf("matching Dispatch: %v", err) + } + if aqlCalls != 1 { + t.Fatalf("matching AQL Dispatch adapter calls = %d, want 1", aqlCalls) + } + + pgCommand, err := command.NewPostgreSQLCommand("SELECT 1", nil) + if err != nil { + t.Fatalf("NewPostgreSQLCommand: %v", err) + } + if err := pgCommand.Dispatch(func(command.AQLCommand) error { aqlCalls++; return nil }, nil); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("PostgreSQL command without PostgreSQL adapter error = %v, want backend-not-registered", err) + } + aqlOnly, err := command.NewAQLCommand("RETURN 1", nil) + if err != nil { + t.Fatalf("NewAQLCommand: %v", err) + } + if err := aqlOnly.Dispatch(nil, nil); errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("missing AQL adapter misclassified as mismatch: %v", err) + } + pgOnly, err := command.NewPostgreSQLCommand("SELECT 1", nil) + if err != nil { + t.Fatalf("NewPostgreSQLCommand: %v", err) + } + if err := pgOnly.Dispatch(nil, nil); errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("missing PostgreSQL adapter misclassified as mismatch: %v", err) + } + if aqlCalls != 1 { + t.Fatal("mismatched Dispatch must not run the AQL adapter") + } + }) + + t.Run("AQLRegistrationCarriesRenderedPayloadUnchanged", func(t *testing.T) { + rendered := aql.RenderedPhysicalPlan{Query: "FOR d IN @@collection RETURN d", BindVars: map[string]any{"@@collection": "Patient"}} + compiledCommand, err := command.NewAQLCommand(rendered.Query, rendered.BindVars) + if err != nil { + t.Fatalf("NewAQLCommand(AQL): %v", err) + } + query, bindVars, err := compiledCommand.AQLParameters() + if err != nil || query != rendered.Query || !reflect.DeepEqual(bindVars, rendered.BindVars) { + t.Fatalf("AQL registration changed rendered result: %q vs %q, %v, %v", query, rendered.Query, bindVars, err) + } + var zero command.CompiledCommand + if _, _, err := zero.AQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("zero-tag AQL dispatch error = %v, want not-registered", err) + } + for _, backend := range []command.CompiledBackend{"", "CLICKHOUSE", "aql", "postgresql", "MYSQL"} { + if backend == command.CompiledBackendAQL || backend == command.CompiledBackendPostgreSQL { + t.Fatalf("tag constants changed: %q", backend) + } + } + }) + + t.Run("ConstructorsRejectMalformedTagPayloadPairs", func(t *testing.T) { + if _, err := command.NewAQLCommand(" ", nil); err == nil { + t.Fatal("AQL command with blank query must be rejected") + } + if _, err := command.NewPostgreSQLCommand("", nil); err == nil { + t.Fatal("PostgreSQL command with blank SQL must be rejected") + } + cases := []struct { + name string + args []command.PostgreSQLParameter + }{ + {"unregistered type", []command.PostgreSQLParameter{{Type: "money", Value: "1.00"}}}, + {"nil value under typed parameter", []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterText, Value: nil}}}, + {"value under SQL null", []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterNull, Value: "x"}}}, + } + for _, testCase := range cases { + if _, err := command.NewPostgreSQLCommand("SELECT $1", testCase.args); err == nil { + t.Fatalf("%s: malformed argument accepted", testCase.name) + } + } + }) + + t.Run("RecipePathPublishesOnlyDispatchedAQLCommand", func(t *testing.T) { + bindings := recipe.RuntimeBindings{Project: "command-project", DatasetGeneration: "command-generation"} + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "command_tag", + TranslationVersion: "test", + Outputs: []recipe.Output{{ + Name: "patients", RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{{Name: "gender", ColumnID: "gender", Expr: recipe.Expression{Select: "root.gender"}}}, + }}, + } + plan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + queries, err := compiler.CompileResolvedRecipePlan(compiler.ArangoTarget(), resolved, 7, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("CompileResolvedRecipePlan: %v", err) + } + if len(queries) != 1 { + t.Fatalf("compiled outputs = %d, want 1", len(queries)) + } + compiled := queries[0] + if strings.TrimSpace(testAQLText(t, compiled.Command)) == "" || len(testAQLBinds(t, compiled.Command)) == 0 { + t.Fatalf("recipe path lost rendered command: %#v", compiled) + } + rendered, err := aql.RenderPhysicalPlan(compiled.PhysicalPlan) + if err != nil { + t.Fatalf("render retained plan: %v", err) + } + compiledCommand, err := command.NewAQLCommand(rendered.Query, rendered.BindVars) + if err != nil { + t.Fatalf("NewAQLCommand(AQL): %v", err) + } + query, bindVars, err := compiledCommand.AQLParameters() + if err != nil { + t.Fatalf("AQLParameters: %v", err) + } + if query != testAQLText(t, compiled.Command) || !reflect.DeepEqual(bindVars, testAQLBinds(t, compiled.Command)) { + t.Fatalf("recipe-published command differs from dispatched AQL command: queryMatch=%t bindMatch=%t", + query == testAQLText(t, compiled.Command), reflect.DeepEqual(bindVars, testAQLBinds(t, compiled.Command))) + } + mislabelled, err := command.NewPostgreSQLCommand(testAQLText(t, compiled.Command), []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`{"limit":7}`)}}) + if err != nil { + t.Fatalf("NewPostgreSQLCommand: %v", err) + } + if _, _, err := mislabelled.AQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("PostgreSQL-tagged SQL to AQL adapter error = %v, want mismatch", err) + } + }) +} + +func TestCompiledCommandWithAQLBinds(t *testing.T) { + aqlCommand, err := command.NewAQLCommand("FOR d IN @@collection FILTER d.p == @p LIMIT @size RETURN d", map[string]any{ + "@@collection": "Patient", + "p": "alpha", + "size": 10, + "nilbind": nil, + }) + if err != nil { + t.Fatalf("NewAQLCommand: %v", err) + } + t.Run("OverrideCopiesWithoutMutatingOriginal", func(t *testing.T) { + updated, err := aqlCommand.WithAQLBinds(map[string]any{"size": 25, "p": "beta"}) + if err != nil { + t.Fatalf("WithAQLBinds: %v", err) + } + if updated.Backend() != command.CompiledBackendAQL { + t.Fatalf("override changed the tag to %q", updated.Backend()) + } + query, binds, err := updated.AQLParameters() + if err != nil { + t.Fatalf("AQLParameters: %v", err) + } + if query != "FOR d IN @@collection FILTER d.p == @p LIMIT @size RETURN d" { + t.Fatalf("override changed the query text: %q", query) + } + if binds["size"] != 25 || binds["p"] != "beta" { + t.Fatalf("overrides not applied: %v", binds) + } + if binds["@@collection"] != "Patient" || binds["nilbind"] != nil { + t.Fatalf("untouched binds lost: %v", binds) + } + _, originalBinds, err := aqlCommand.AQLParameters() + if err != nil { + t.Fatalf("original AQLParameters: %v", err) + } + if originalBinds["size"] != 10 || originalBinds["p"] != "alpha" { + t.Fatalf("override mutated the original command binds: %v", originalBinds) + } + if _, originalBinds, err := aqlCommand.AQLParameters(); err != nil { + t.Fatalf("original AQLParameters: %v", err) + } else { + originalBinds["size"] = 999 + } + _, updatedBinds, err := updated.AQLParameters() + if err != nil { + t.Fatalf("recheck AQLParameters: %v", err) + } + if updatedBinds["size"] != 25 { + t.Fatalf("returned command does not own a fresh bind map: %v", updatedBinds) + } + }) + failures := []struct { + name string + command command.CompiledCommand + overrides map[string]any + sentinel error + }{ + {"UnknownKeyRefused", aqlCommand, map[string]any{"missing": 1}, command.ErrCompiledCommandBackendMismatch}, + {"TypeMismatchRefused", aqlCommand, map[string]any{"size": "25"}, command.ErrCompiledCommandBackendMismatch}, + {"NilOverrideRefused", aqlCommand, map[string]any{"size": nil}, command.ErrCompiledCommandBackendMismatch}, + {"NilCurrentValueRefused", aqlCommand, map[string]any{"nilbind": 1}, command.ErrCompiledCommandBackendMismatch}, + {"EmptyOverridesRefused", aqlCommand, map[string]any{}, command.ErrCompiledCommandBackendMismatch}, + {"NilOverridesRefused", aqlCommand, nil, command.ErrCompiledCommandBackendMismatch}, + {"PostgreSQLTagRefused", mustPostgreSQLCommandForBindOverrides(t), map[string]any{"size": 5}, command.ErrCompiledCommandBackendMismatch}, + {"ZeroTagRefused", command.CompiledCommand{}, map[string]any{"size": 5}, command.ErrCompiledCommandBackendNotRegistered}, + } + for _, tc := range failures { + t.Run(tc.name, func(t *testing.T) { + if _, err := tc.command.WithAQLBinds(tc.overrides); !errors.Is(err, tc.sentinel) { + t.Fatalf("WithAQLBinds error = %v, want %v", err, tc.sentinel) + } + }) + } +} + +func mustPostgreSQLCommandForBindOverrides(t *testing.T) command.CompiledCommand { + t.Helper() + compiledCommand, err := command.NewPostgreSQLCommand("SELECT id FROM t WHERE p = $1", []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterText, Value: "alpha"}}) + if err != nil { + t.Fatalf("NewPostgreSQLCommand: %v", err) + } + return compiledCommand +} + +func testAQLView(t *testing.T, compiledCommand command.CompiledCommand) (string, map[string]any) { + t.Helper() + query, bindVars, err := compiledCommand.AQLParameters() + if err != nil { + t.Fatalf("derive AQL parameters from %s command: %v", compiledCommand.Backend(), err) + } + return query, bindVars +} + +func testAQLText(t *testing.T, compiledCommand command.CompiledCommand) string { + t.Helper() + query, _ := testAQLView(t, compiledCommand) + return query +} + +func testAQLBinds(t *testing.T, compiledCommand command.CompiledCommand) map[string]any { + t.Helper() + _, bindVars := testAQLView(t, compiledCommand) + return bindVars +} diff --git a/internal/dataframe/compiler/command/imports_guard_test.go b/internal/dataframe/compiler/command/imports_guard_test.go new file mode 100644 index 000000000..6dabad045 --- /dev/null +++ b/internal/dataframe/compiler/command/imports_guard_test.go @@ -0,0 +1,34 @@ +package command_test + +import ( + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestCommandLeafIsStdlibOnly(t *testing.T) { + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read package directory: %v", err) + } + fset := token.NewFileSet() + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + file, parseErr := parser.ParseFile(fset, filepath.Join(".", name), nil, parser.ImportsOnly) + if parseErr != nil { + t.Fatalf("parse %s: %v", name, parseErr) + } + for _, imported := range file.Imports { + path := strings.Trim(imported.Path.Value, `"`) + if strings.Contains(path, ".") { + t.Errorf("%s imports %q: the command leaf is standard-library-only", name, path) + } + } + } +} diff --git a/internal/dataframe/compiler/command/rendered_postgres.go b/internal/dataframe/compiler/command/rendered_postgres.go new file mode 100644 index 000000000..2a7eb9259 --- /dev/null +++ b/internal/dataframe/compiler/command/rendered_postgres.go @@ -0,0 +1,106 @@ +package command + +import ( + "errors" + "fmt" + "strings" +) + +var ErrRenderedPostgresPlaceholderMismatch = errors.New("rendered postgres plan placeholder mismatch") + +var ErrRenderedPostgresAQLBindSyntax = errors.New("rendered postgres plan contains aql bind syntax") + +type RenderedPostgresPhysicalPlan struct { + SQL string + Args []PostgreSQLParameter + OutputColumns []string +} + +func NewRenderedPostgresPhysicalPlan(sql string, args []PostgreSQLParameter, outputColumns []string) (RenderedPostgresPhysicalPlan, error) { + if strings.TrimSpace(sql) == "" { + return RenderedPostgresPhysicalPlan{}, fmt.Errorf("rendered postgres plan requires statement SQL") + } + if strings.Contains(sql, "@") { + return RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: statement SQL must use $n placeholders only; found an '@' token", ErrRenderedPostgresAQLBindSyntax) + } + if err := validatePostgreSQLParameters(args); err != nil { + return RenderedPostgresPhysicalPlan{}, err + } + if err := validatePostgresPlaceholders(sql, len(args)); err != nil { + return RenderedPostgresPhysicalPlan{}, err + } + var copiedArgs []PostgreSQLParameter + if len(args) != 0 { + copiedArgs = append([]PostgreSQLParameter(nil), args...) + } + copiedColumns := make([]string, len(outputColumns)) + for index, name := range outputColumns { + if strings.TrimSpace(name) == "" { + return RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "rendered postgres plan output column %d is blank", index+1) + } + copiedColumns[index] = name + } + return RenderedPostgresPhysicalPlan{SQL: sql, Args: copiedArgs, OutputColumns: copiedColumns}, nil +} + +func validatePostgreSQLParameters(args []PostgreSQLParameter) error { + for index, argument := range args { + if !argument.Type.registered() { + return fmt.Errorf("PostgreSQL command argument %d has unregistered parameter type %q", index+1, string(argument.Type)) + } + if argument.Type == PostgreSQLParameterNull { + if argument.Value != nil { + return fmt.Errorf("PostgreSQL command argument %d is typed SQL null but carries a value", index+1) + } + continue + } + if argument.Value == nil { + return fmt.Errorf("PostgreSQL command argument %d is typed %q but carries a nil value; use %q for SQL NULL", index+1, string(argument.Type), string(PostgreSQLParameterNull)) + } + } + return nil +} + +func validatePostgresPlaceholders(sql string, argCount int) error { + seen := make([]bool, argCount+1) + maxSeen := 0 + for index := 0; index < len(sql); index++ { + if sql[index] != '$' { + continue + } + start := index + 1 + end := start + for end < len(sql) && sql[end] >= '0' && sql[end] <= '9' { + end++ + } + if end == start { + continue + } + number := 0 + for _, digit := range sql[start:end] { + number = number*10 + int(digit-'0') + } + index = end - 1 + if number < 1 || number > argCount { + return fmt.Errorf("%w: $%d is outside the 1..%d argument range", + ErrRenderedPostgresPlaceholderMismatch, number, argCount) + } + if number > maxSeen { + maxSeen = number + } + seen[number] = true + } + if maxSeen != argCount { + return fmt.Errorf("%w: highest placeholder is $%d but %d arguments were rendered", + ErrRenderedPostgresPlaceholderMismatch, maxSeen, argCount) + } + for number := 1; number <= argCount; number++ { + if !seen[number] { + return fmt.Errorf("%w: argument %d ($%d) is never referenced by the statement", + ErrRenderedPostgresPlaceholderMismatch, number, number) + } + } + return nil +} diff --git a/internal/dataframe/compiler/command/rendered_postgres_test.go b/internal/dataframe/compiler/command/rendered_postgres_test.go new file mode 100644 index 000000000..c830c1402 --- /dev/null +++ b/internal/dataframe/compiler/command/rendered_postgres_test.go @@ -0,0 +1,109 @@ +package command_test + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" +) + +func okPlan(t *testing.T, sql string, args []command.PostgreSQLParameter, columns []string) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := command.NewRenderedPostgresPhysicalPlan(sql, args, columns) + if err != nil { + t.Fatalf("NewRenderedPostgresPhysicalPlan(%q): %v", sql, err) + } + return rendered +} + +func TestRenderedPostgresPlanRequiresSQLAndRejectsAQLBindSyntax(t *testing.T) { + if _, err := command.NewRenderedPostgresPhysicalPlan(" ", nil, nil); err == nil { + t.Fatal("blank SQL accepted") + } + for _, sql := range []string{ + `SELECT @value`, + `FOR d IN @@collection RETURN d`, + `SELECT $1 -- @@ sneaky`, + } { + _, err := command.NewRenderedPostgresPhysicalPlan(sql, []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterText, Value: "x"}}, nil) + if !errors.Is(err, command.ErrRenderedPostgresAQLBindSyntax) { + t.Errorf("SQL %q error = %v, want ErrRenderedPostgresAQLBindSyntax", sql, err) + } + } +} + +func TestRenderedPostgresPlanPlaceholderContract(t *testing.T) { + text := func(v string) command.PostgreSQLParameter { + return command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: v} + } + okPlan(t, `SELECT $1, $1 WHERE $2 = $1`, []command.PostgreSQLParameter{text("a"), text("b")}, nil) + + for _, tc := range []struct { + name string + sql string + args []command.PostgreSQLParameter + }{ + {"out of range", `SELECT $3`, []command.PostgreSQLParameter{text("a")}}, + {"gap skips a number", `SELECT $1, $3`, []command.PostgreSQLParameter{text("a"), text("b"), text("c")}}, + {"unreferenced tail argument", `SELECT $1`, []command.PostgreSQLParameter{text("a"), text("b")}}, + {"zero placeholder", `SELECT $0`, []command.PostgreSQLParameter{text("a")}}, + {"arguments without placeholders", `SELECT 1`, []command.PostgreSQLParameter{text("a")}}, + } { + if _, err := command.NewRenderedPostgresPhysicalPlan(tc.sql, tc.args, nil); !errors.Is(err, command.ErrRenderedPostgresPlaceholderMismatch) { + t.Errorf("%s: error = %v, want ErrRenderedPostgresPlaceholderMismatch", tc.name, err) + } + } + okPlan(t, `SELECT resource_seq FROM "s"."fhir_resource"`, nil, nil) + if _, err := command.NewRenderedPostgresPhysicalPlan(`SELECT $1`, nil, nil); !errors.Is(err, command.ErrRenderedPostgresPlaceholderMismatch) { + t.Fatalf("placeholder without arguments: error = %v", err) + } +} + +func TestRenderedPostgresPlanArgumentValidationMatchesCommandContract(t *testing.T) { + for _, args := range [][]command.PostgreSQLParameter{ + {{Type: "money", Value: "1.00"}}, + {{Type: command.PostgreSQLParameterText, Value: nil}}, + {{Type: command.PostgreSQLParameterNull, Value: "x"}}, + } { + if _, err := command.NewRenderedPostgresPhysicalPlan(`SELECT $1`, args, nil); err == nil { + t.Fatalf("malformed arguments accepted: %#v", args) + } + } + okPlan(t, `SELECT $1`, []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterNull}}, nil) + okPlan(t, `SELECT $1`, []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage("null")}}, nil) +} + +func TestRenderedPostgresPlanCopiesAndValidatesOutputColumns(t *testing.T) { + args := []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterText, Value: "x"}} + columns := []string{"keep me", "second"} + rendered := okPlan(t, `SELECT $1 AS "c0", $1 AS "c1"`, args, columns) + if !reflect.DeepEqual(rendered.OutputColumns, []string{"keep me", "second"}) { + t.Fatalf("output columns = %v", rendered.OutputColumns) + } + args[0].Value = "mutated" + columns[0] = "mutated" + if rendered.Args[0].Value != "x" || rendered.OutputColumns[0] != "keep me" { + t.Fatalf("constructor aliased caller data: %#v %v", rendered.Args, rendered.OutputColumns) + } + for _, columns := range [][]string{{"ok", " "}, {""}} { + if _, err := command.NewRenderedPostgresPhysicalPlan(`SELECT $1`, args, columns); err == nil { + t.Fatalf("blank output column accepted: %v", columns) + } + } +} + +func TestRenderedPostgresPlanSQLCarriesOnlyIdentifiersAndPlaceholders(t *testing.T) { + sql := `SELECT r0."logical_id" AS "c0" FROM "owner_schema"."fhir_resource" AS r0` + + ` WHERE r0."project" = $1 AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($2::jsonb))` + hostile := `x'; DROP TABLE fhir_resource; --` + rendered := okPlan(t, sql, []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: hostile}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["` + hostile + `"]`)}, + }, []string{"logical_id"}) + if strings.Contains(rendered.SQL, hostile) || strings.Contains(rendered.SQL, "DROP") { + t.Fatalf("value leaked into SQL: %s", rendered.SQL) + } +} diff --git a/internal/dataframe/compiler/command_helpers_external_test.go b/internal/dataframe/compiler/command_helpers_external_test.go new file mode 100644 index 000000000..8482107e6 --- /dev/null +++ b/internal/dataframe/compiler/command_helpers_external_test.go @@ -0,0 +1,37 @@ +package compiler_test + +import ( + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" +) + +func testAQLView(t *testing.T, compiledCommand command.CompiledCommand) (string, map[string]any) { + t.Helper() + query, bindVars, err := compiledCommand.AQLParameters() + if err != nil { + t.Fatalf("derive AQL parameters from %s command: %v", compiledCommand.Backend(), err) + } + return query, bindVars +} + +func testAQLCommand(t *testing.T, query string, bindVars map[string]any) command.CompiledCommand { + t.Helper() + compiledCommand, err := command.NewAQLCommand(query, bindVars) + if err != nil { + t.Fatalf("build AQL command for test: %v", err) + } + return compiledCommand +} + +func testAQLText(t *testing.T, compiledCommand command.CompiledCommand) string { + t.Helper() + query, _ := testAQLView(t, compiledCommand) + return query +} + +func testAQLBinds(t *testing.T, compiledCommand command.CompiledCommand) map[string]any { + t.Helper() + _, bindVars := testAQLView(t, compiledCommand) + return bindVars +} diff --git a/internal/dataframe/compiler/command_helpers_test.go b/internal/dataframe/compiler/command_helpers_test.go new file mode 100644 index 000000000..4345bfa27 --- /dev/null +++ b/internal/dataframe/compiler/command_helpers_test.go @@ -0,0 +1,57 @@ +package compiler + +import ( + "github.com/calypr/loom/internal/dataframe/compiler/command" + "strings" + "testing" + + arangofactory "github.com/calypr/loom/internal/backends/arango" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" +) + +func requireStableRuntimeCode(t *testing.T, err error, code string) string { + t.Helper() + if err == nil { + t.Fatalf("error = nil, want stable code %s", code) + } + if userErr, ok := dataframeerrors.AsUserError(err); ok { + if userErr.Code() != code { + t.Fatalf("error = %v carries code %q, want stable code %s", err, userErr.Code(), code) + } + } else if !strings.Contains(err.Error(), code) { + t.Fatalf("error = %v, want stable code %s", err, code) + } + if cause, ok := arangofactory.DiagnosticCause(err); ok { + return cause + } + return err.Error() +} + +func testAQLView(t *testing.T, compiledCommand command.CompiledCommand) (string, map[string]any) { + t.Helper() + query, bindVars, err := compiledCommand.AQLParameters() + if err != nil { + t.Fatalf("dispatch compiled command to AQL view: %v", err) + } + return query, bindVars +} + +func testAQLCommand(query string, bindVars map[string]any) command.CompiledCommand { + compiledCommand, err := command.NewAQLCommand(query, bindVars) + if err != nil { + panic(err) + } + return compiledCommand +} + +func testAQLText(t *testing.T, compiledCommand command.CompiledCommand) string { + t.Helper() + query, _ := testAQLView(t, compiledCommand) + return query +} + +func testAQLBinds(t *testing.T, compiledCommand command.CompiledCommand) map[string]any { + t.Helper() + _, bindVars := testAQLView(t, compiledCommand) + return bindVars +} diff --git a/internal/dataframe/compiler/compiled_command_test.go b/internal/dataframe/compiler/compiled_command_test.go new file mode 100644 index 000000000..3fa4f4ed7 --- /dev/null +++ b/internal/dataframe/compiler/compiled_command_test.go @@ -0,0 +1,285 @@ +package compiler_test + +import ( + "encoding/json" + "errors" + "reflect" + "regexp" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/render/aql" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" +) + +const ( + pgIR01Project = "P1" + pgIR01Generation = "G1" +) + +func pgIR01ResolvedRecipe(t *testing.T) semantic.ResolvedRecipePlan { + t.Helper() + bindings := recipe.RuntimeBindings{Project: pgIR01Project, DatasetGeneration: pgIR01Generation} + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "ir01_check4_boundary", + TranslationVersion: "test", + Outputs: []recipe.Output{{ + Name: "patients", RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{{Name: "gender", ColumnID: "gender", Expr: recipe.Expression{Select: "root.gender"}}}, + }}, + } + plan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + return resolved +} + +func pgIR01AQLQueries(t *testing.T, limit int) []compiler.CompiledQuery { + t.Helper() + queries, err := compiler.CompileResolvedRecipePlan( + compiler.ArangoTarget(), pgIR01ResolvedRecipe(t), limit, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("CompileResolvedRecipePlan(ArangoTarget): %v", err) + } + if len(queries) != 1 { + t.Fatalf("compiled outputs = %d, want 1", len(queries)) + } + return queries +} + +func pgIR01PostgreSQLQuery(t *testing.T) compiler.CompiledQuery { + t.Helper() + output, bindings := archBackend11FingerprintFixture() + query, err := compiler.CompileRecipeOutput( + archBackend11PostgreSQLTarget(t), output, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("CompileRecipeOutput(PostgreSQLTarget): %v", err) + } + return query +} + +func pgIR01PlaceholderScan(t *testing.T, sqlText string) (int, bool) { + t.Helper() + matches := regexp.MustCompile(`\$(\d+)`).FindAllStringSubmatch(sqlText, -1) + seen := map[int]int{} + maxPosition := 0 + for _, match := range matches { + position := 0 + for _, digit := range match[1] { + position = position*10 + int(digit-'0') + } + seen[position]++ + if position > maxPosition { + maxPosition = position + } + } + if len(matches) != maxPosition { + return maxPosition, false + } + for position := 1; position <= maxPosition; position++ { + if seen[position] != 1 { + return maxPosition, false + } + } + return maxPosition, true +} + +func TestCompiledCommand(t *testing.T) { + t.Run("AQLTagCarriesRenderedPayloadUnchanged", func(t *testing.T) { + compiled := pgIR01AQLQueries(t, 7)[0] + if compiled.Command.Backend() != command.CompiledBackendAQL { + t.Fatalf("recipe path tagged the command %q, want %q", compiled.Command.Backend(), command.CompiledBackendAQL) + } + query, binds, err := compiled.Command.AQLParameters() + if err != nil { + t.Fatalf("AQLParameters: %v", err) + } + if strings.TrimSpace(query) == "" || len(binds) == 0 { + t.Fatalf("recipe path published an empty AQL payload: %q %v", query, binds) + } + rendered, err := aql.RenderPhysicalPlan(compiled.PhysicalPlan) + if err != nil { + t.Fatalf("render retained plan: %v", err) + } + if query != rendered.Query { + t.Fatalf("published AQL drifted from the canonical render:\npublished: %q\ncanonical: %q", query, rendered.Query) + } + if !reflect.DeepEqual(binds, rendered.BindVars) { + t.Fatalf("published binds drifted from the canonical render: %v vs %v", binds, rendered.BindVars) + } + _, bindsAgain, err := compiled.Command.AQLParameters() + if err != nil { + t.Fatalf("repeated AQLParameters: %v", err) + } + if reflect.ValueOf(binds).Pointer() != reflect.ValueOf(bindsAgain).Pointer() { + t.Fatal("AQL dispatch must carry the exact bind map instance unchanged") + } + second := pgIR01AQLQueries(t, 7)[0] + query2, binds2, err := second.Command.AQLParameters() + if err != nil { + t.Fatalf("second compile AQLParameters: %v", err) + } + if query2 != query || !reflect.DeepEqual(binds2, binds) { + t.Fatalf("recipe compilation is not deterministic:\n%q vs %q\n%v vs %v", query2, query, binds2, binds) + } + }) + + t.Run("PostgreSQLTagCarriesSQLAndOrderedTypedArgs", func(t *testing.T) { + compiled := pgIR01PostgreSQLQuery(t) + if compiled.Command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("selected PostgreSQL target produced %q", compiled.Command.Backend()) + } + sqlText, args, err := compiled.Command.PostgreSQLParameters() + if err != nil { + t.Fatalf("PostgreSQLParameters: %v", err) + } + if sqlText != archBackend11BaseSQL { + t.Fatalf("statement SQL drifted from the frozen contract:\ngot: %q\nwant: %q", sqlText, archBackend11BaseSQL) + } + if compiled.PlanDiagnostics.Fingerprint != archBackend11BaseFingerprint { + t.Fatalf("plan fingerprint drifted: %q want %q", compiled.PlanDiagnostics.Fingerprint, archBackend11BaseFingerprint) + } + wantTypes := []command.PostgreSQLParameterType{ + command.PostgreSQLParameterText, command.PostgreSQLParameterText, + command.PostgreSQLParameterJSONB, command.PostgreSQLParameterText, command.PostgreSQLParameterText, + } + if len(args) != len(wantTypes) { + t.Fatalf("ordered argument count = %d, want %d: %#v", len(args), len(wantTypes), args) + } + wantValues := []any{"P1", "G1", json.RawMessage(`["FHIR/Patient"]`), "Patient", "{gender}"} + for index := range args { + if args[index].Type != wantTypes[index] { + t.Fatalf("argument %d type = %q, want %q", index, args[index].Type, wantTypes[index]) + } + if args[index].Value == nil { + t.Fatalf("argument %d carries a nil value under the non-null type %q", index, args[index].Type) + } + if !reflect.DeepEqual(args[index].Value, wantValues[index]) { + t.Fatalf("argument %d value = %#v, want %#v", index, args[index].Value, wantValues[index]) + } + } + maxPosition, complete := pgIR01PlaceholderScan(t, sqlText) + if !complete || maxPosition != len(args) { + t.Fatalf("placeholders $1..$%d (complete=%t) do not match %d ordered arguments", maxPosition, complete, len(args)) + } + args[0].Value = "Corrupted" + _, reread, err := compiled.Command.PostgreSQLParameters() + if err != nil || !reflect.DeepEqual(reread[0].Value, "P1") { + t.Fatalf("PostgreSQLParameters must return a fresh copy on every read: %#v %v", reread, err) + } + aqlQuery, aqlBinds, err := compiled.Command.AQLParameters() + if !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("PostgreSQL command to AQL adapter error = %v, want mismatch", err) + } + if aqlQuery != "" || aqlBinds != nil { + t.Fatalf("refused AQL dispatch must return zero values, got %q %v", aqlQuery, aqlBinds) + } + if !strings.Contains(err.Error(), "AQL") { + t.Fatalf("mismatch error must name the refused AQL adapter: %v", err) + } + second := pgIR01PostgreSQLQuery(t) + sql2, args2, err := second.Command.PostgreSQLParameters() + if err != nil { + t.Fatalf("second compile PostgreSQLParameters: %v", err) + } + if sql2 != sqlText || !reflect.DeepEqual(args2, reread) || second.PlanDiagnostics.Fingerprint != compiled.PlanDiagnostics.Fingerprint { + t.Fatalf("PostgreSQL compilation is not deterministic:\n%q vs %q\n%#v vs %#v", sql2, sqlText, args2, reread) + } + }) + + t.Run("ExhaustiveDispatchRejectsMismatchBeforeExecution", func(t *testing.T) { + aqlCommand := pgIR01AQLQueries(t, 7)[0].Command + pgCommand := pgIR01PostgreSQLQuery(t).Command + aqlCalls, pgCalls := 0, 0 + aqlTrap := func(command.AQLCommand) error { aqlCalls++; return nil } + pgTrap := func(command.PostgreSQLCommand) error { pgCalls++; return nil } + + if err := pgCommand.Dispatch(aqlTrap, nil); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("PostgreSQL command without PostgreSQL adapter error = %v, want backend-not-registered", err) + } + if aqlCalls != 0 || pgCalls != 0 { + t.Fatalf("mismatched Dispatch executed adapters: aql=%d postgresql=%d", aqlCalls, pgCalls) + } + if err := aqlCommand.Dispatch(nil, pgTrap); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("AQL command without AQL adapter error = %v, want backend-not-registered", err) + } + if aqlCalls != 0 || pgCalls != 0 { + t.Fatalf("mismatched Dispatch executed adapters: aql=%d postgresql=%d", aqlCalls, pgCalls) + } + var zero command.CompiledCommand + if _, _, err := zero.AQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("zero-tag AQL dispatch error = %v, want backend-not-registered", err) + } + if _, _, err := zero.PostgreSQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("zero-tag PostgreSQL dispatch error = %v, want backend-not-registered", err) + } + if err := zero.Dispatch(aqlTrap, pgTrap); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("zero-tag Dispatch error = %v, want backend-not-registered", err) + } + if aqlCalls != 0 || pgCalls != 0 { + t.Fatalf("zero-tag Dispatch executed adapters: aql=%d postgresql=%d", aqlCalls, pgCalls) + } + if err := aqlCommand.Dispatch(aqlTrap, pgTrap); err != nil { + t.Fatalf("matching AQL Dispatch: %v", err) + } + if aqlCalls != 1 || pgCalls != 0 { + t.Fatalf("AQL Dispatch adapter calls aql=%d postgresql=%d, want 1 and 0", aqlCalls, pgCalls) + } + if err := pgCommand.Dispatch(aqlTrap, pgTrap); err != nil { + t.Fatalf("matching PostgreSQL Dispatch: %v", err) + } + if aqlCalls != 1 || pgCalls != 1 { + t.Fatalf("PostgreSQL Dispatch adapter calls aql=%d postgresql=%d, want 1 and 1", aqlCalls, pgCalls) + } + }) + + t.Run("AQLCallPathRetainsExactBehavior", func(t *testing.T) { + compiled := pgIR01AQLQueries(t, 7)[0] + if compiled.PlanMode != "physical" || compiled.PlanProfile != "generic_fhir_graph_recipe" { + t.Fatalf("AQL plan metadata changed: mode=%q profile=%q", compiled.PlanMode, compiled.PlanProfile) + } + if compiled.Limit != 7 { + t.Fatalf("AQL limit not preserved: %d, want 7", compiled.Limit) + } + if len(compiled.Columns) == 0 || len(compiled.PublicColumns) == 0 { + t.Fatalf("AQL output lost its column contract: columns=%v public=%v", compiled.Columns, compiled.PublicColumns) + } + query, binds, err := compiled.Command.AQLParameters() + if err != nil { + t.Fatalf("AQLParameters: %v", err) + } + if !strings.Contains(query, "FOR root IN") || !strings.Contains(query, "@project") { + t.Fatalf("AQL path drifted from canonical AQL rendering: %q", query) + } + if binds["project"] != pgIR01Project || binds["dataset_generation"] != pgIR01Generation { + t.Fatalf("AQL scope binds lost: %v", binds) + } + hasCollectionBind := false + for key := range binds { + if strings.HasPrefix(key, "@") { + hasCollectionBind = true + } + } + if !hasCollectionBind { + t.Fatalf("AQL collection bind missing: %v", binds) + } + pg := pgIR01PostgreSQLQuery(t) + if pg.PreviewCoveringIndex != nil || pg.PreviewGroupScan != nil || pg.PreviewGroupProjection != nil { + t.Fatalf("PostgreSQL output carries AQL-only preview accelerators: index=%v scan=%v projection=%v", + pg.PreviewCoveringIndex, pg.PreviewGroupScan, pg.PreviewGroupProjection) + } + if _, binds, err := pg.Command.AQLParameters(); err == nil || binds != nil { + t.Fatalf("PostgreSQL command must never expose a named bind map: %v %v", binds, err) + } + }) +} diff --git a/internal/dataframe/compiler/compiled_query.go b/internal/dataframe/compiler/compiled_query.go index cc2c38b60..aa1eab3e0 100644 --- a/internal/dataframe/compiler/compiled_query.go +++ b/internal/dataframe/compiler/compiled_query.go @@ -1,102 +1,64 @@ package compiler import ( + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/spec" ) -// PreviewCoveringIndexSpec describes a bounded compiler-selected persistent -// index that may cover a preview source scan. Runtime owners may provision -// this index before executing the hinted query. type PreviewCoveringIndexSpec struct { - Collection string - Name string - Fields []string - // StoredValues contains payload paths retained for projections by this - // persistent index. It does not change lookup or sort semantics. - StoredValues []string - // Supersedes identifies the exact legacy category index that may be - // replaced if the collection has reached its bounded preview-index cap. - Supersedes *PreviewCoveringIndexReplacement - // PrepareAfterPreview keeps Group index creation out of the query's critical - // path. The query uses a non-forcing hint and remains valid without it. + Collection string + Name string + Fields []string + StoredValues []string + Supersedes *PreviewCoveringIndexReplacement PrepareAfterPreview bool - // pivotGroupKeyPaths enables the Pivot-only two-scan renderer. Other - // preview covering indexes use a single hinted root scan. pivotGroupKeyPaths [][]string } -// PreviewCoveringIndexReplacement is an exact compiler-owned index identity -// eligible for replacement by a more useful index on the same category path. type PreviewCoveringIndexReplacement struct { Name string Fields []string StoredValues []string } -// PreviewGroupScanSpec contains compiler-rendered alternatives for a narrow -// direct-source Group preview. Runtime may choose SequentialQuery only after -// verifying the current scan is non-covering and that the exact scoped source -// count is close to the collection cardinality. type PreviewGroupScanSpec struct { - Collection string - SequentialQuery string - SequentialBindVars map[string]any - ScopeCountQuery string - ScopeCountBindVars map[string]any + Collection string + Sequential command.CompiledCommand + ScopeCount command.CompiledCommand } -// PreviewGroupProjectionSpec marks a direct semantic Group whose complete -// Coding-array projection can be safely factored. Runtime must use the exact -// scoped count evidence before considering the alternative query. type PreviewGroupProjectionSpec struct { - Collection string - ScopeCountQuery string - ScopeCountBindVars map[string]any - ReadOnlyIndex *PreviewGroupProjectionIndexSpec + Collection string + ScopeCount command.CompiledCommand + ReadOnlyIndex *PreviewGroupProjectionIndexSpec } -// PreviewGroupProjectionIndexSpec describes read-only requirements for -// checking whether an existing root index covers the source scope and the -// complete Coding projection. It cannot request index creation. type PreviewGroupProjectionIndexSpec struct { RequiredFields []string ProjectionPath string } -// CompiledQuery is the executable result of the canonical recipe compiler. -// It contains parameterized AQL plus stable metadata for execution, export, -// and diagnostics; it does not expose a transport-specific request builder. type CompiledQuery struct { - // PhysicalPlan is the finalized execution plan after optimization and - // preview-window insertion. It is retained for server-owned execution - // preparation; Query and BindVars remain the rendered public result. - PhysicalPlan ir.PhysicalPlan - Project string - DatasetGeneration string - RootResourceType string - TranslationVersion string - AuthResourcePaths []string - PlanMode string - PlanProfile string - TraversalCount int - OptimizationRules []string - RowIdentity *spec.RowIdentity - Query string - BindVars map[string]any - Columns []string - // OutputSchema is the compiler-owned ordered schema for the finalized - // physical RETURN projections. PublicColumns is the transport-safe view; - // Columns remains the legacy execution metadata used by generic runtime - // callers and may include the stable physical row identity. - OutputSchema []lower.CompiledOutputColumn - PublicColumns []string - PivotFields []string - Limit int - // PartialValidation is true when a preview-only source or terminal Pivot - // group window bounds validation to a deterministic subset of output rows. + PhysicalPlan ir.PhysicalPlan + Project string + DatasetGeneration string + RootResourceType string + TranslationVersion string + AuthResourcePaths []string + PlanMode string + PlanProfile string + TraversalCount int + OptimizationRules []string + RowIdentity *spec.RowIdentity + Command command.CompiledCommand + Columns []string + OutputSchema []lower.CompiledOutputColumn + PublicColumns []string + PivotFields []string + Limit int PartialValidation bool PreviewCoveringIndex *PreviewCoveringIndexSpec PreviewGroupScan *PreviewGroupScanSpec diff --git a/internal/dataframe/compiler/construction_arango_test.go b/internal/dataframe/compiler/construction_arango_test.go index a3f0c84b1..b73710c77 100644 --- a/internal/dataframe/compiler/construction_arango_test.go +++ b/internal/dataframe/compiler/construction_arango_test.go @@ -11,11 +11,11 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -26,12 +26,13 @@ func TestConstructionPivotDeriveFilterUnpivotRowsAgainstArango(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Observation"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Observation"}}}); err != nil { t.Fatal(err) } @@ -95,7 +96,7 @@ func TestConstructionPivotDeriveFilterUnpivotRowsAgainstArango(t *testing.T) { if err != nil { t.Fatal(err) } - query, err := CompileRecipeOutputWithPolicy(compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -135,12 +136,13 @@ func TestRelatedSourceAllMatchesAtSelectedStageAgainstArango(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -149,7 +151,6 @@ func TestRelatedSourceAllMatchesAtSelectedStageAgainstArango(t *testing.T) { project := "loom_related_source_" + uuid.NewString() generation := "generation-related-source" patientKey := func(id string) string { return project + "_" + id } - // Root-grain traces preserve the same canonical project/key identity as Preview. traceRootKey := func(row map[string]any, identityColumn string) string { t.Helper() parts, ok := row[identityColumn].([]any) @@ -284,7 +285,7 @@ func TestRelatedSourceAllMatchesAtSelectedStageAgainstArango(t *testing.T) { if err != nil { t.Fatal(err) } - query, err := CompileRecipeOutputWithPolicy(compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -311,11 +312,11 @@ func TestRelatedSourceAllMatchesAtSelectedStageAgainstArango(t *testing.T) { if !ok || len(gotSparse) != 0 { t.Fatalf("sparse p2 observation statuses = %#v, want empty list", byPatient["p2"]["observation_status"]) } - trace, err := CompileCellTraceOutputWithPolicy(compiled.Outputs[0], "observation_status", 0, 100, ir.DefaultPhysicalOptimizationPolicy()) + trace, err := CompileCellTraceOutput(ArangoTarget(), compiled.Outputs[0], "observation_status", 0, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - traceRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: trace.Query, BindVars: trace.BindVars}) + traceRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: trace.Command}) traces := make(map[string]map[string]any, len(traceRows)) for _, row := range traceRows { traces[traceRootKey(row, trace.IdentityPartsColumn)] = row @@ -348,7 +349,7 @@ func TestRelatedSourceAllMatchesAtSelectedStageAgainstArango(t *testing.T) { } pageSize := 1 - page, err := CompileRecipeOutputPageWithPolicy(compiled.Outputs[0], bindings, pageSize, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled.Outputs[0], bindings, pageSize, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile paged related-source output: %v", err) } @@ -362,10 +363,10 @@ func TestRelatedSourceAllMatchesAtSelectedStageAgainstArango(t *testing.T) { pagedRows := make([]map[string]any, 0, len(patients)) afterKey := "" for pageIndex := 0; pageIndex <= len(patients); pageIndex++ { - keyBinds := cloneBinds(page.RootKeysBindVars) + keyBinds := cloneBinds(testAQLBinds(t, page.RootKeys)) keyBinds[RootPageAfterKeyBind] = afterKey rootKeys := make([]string, 0, pageSize) - if err := client.QueryRows(ctx, page.RootKeysQuery, 100, keyBinds, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, page.RootKeys), 100, keyBinds, func(row map[string]any) error { key, ok := row["_key"].(string) if !ok || key == "" { return fmt.Errorf("root-key page returned invalid _key %v", row["_key"]) @@ -373,7 +374,7 @@ func TestRelatedSourceAllMatchesAtSelectedStageAgainstArango(t *testing.T) { rootKeys = append(rootKeys, key) return nil }); err != nil { - t.Fatalf("execute related-source root-key page %d: %v\n%s", pageIndex, err, page.RootKeysQuery) + t.Fatalf("execute related-source root-key page %d: %v\n%s", pageIndex, err, testAQLText(t, page.RootKeys)) } if len(rootKeys) == 0 { break @@ -381,13 +382,13 @@ func TestRelatedSourceAllMatchesAtSelectedStageAgainstArango(t *testing.T) { if len(rootKeys) > pageSize { t.Fatalf("root-key page returned %d keys, want at most %d", len(rootKeys), pageSize) } - rowBinds := cloneBinds(page.RowsBindVars) + rowBinds := cloneBinds(testAQLBinds(t, page.Rows)) rowBinds[RootPageKeysBind] = rootKeys - if err := client.QueryRows(ctx, page.RowsQuery, 100, rowBinds, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, page.Rows), 100, rowBinds, func(row map[string]any) error { pagedRows = append(pagedRows, row) return nil }); err != nil { - t.Fatalf("execute related-source selected-root page %d: %v\n%s", pageIndex, err, page.RowsQuery) + t.Fatalf("execute related-source selected-root page %d: %v\n%s", pageIndex, err, testAQLText(t, page.Rows)) } afterKey = rootKeys[len(rootKeys)-1] } @@ -463,7 +464,7 @@ func TestRelatedSourceAllMatchesAtSelectedStageAgainstArango(t *testing.T) { if !found { t.Fatalf("%s output schema = %#v", form.name, columns) } - query, err := CompileRecipeOutputWithPolicy(compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -475,11 +476,11 @@ func TestRelatedSourceAllMatchesAtSelectedStageAgainstArango(t *testing.T) { if values["p1"] != form.present || values["p2"] != form.absent { t.Fatalf("%s values = %#v, want p1=%v p2=%v", form.name, values, form.present, form.absent) } - trace, err := CompileCellTraceOutputWithPolicy(compiled.Outputs[0], form.column, 0, 100, ir.DefaultPhysicalOptimizationPolicy()) + trace, err := CompileCellTraceOutput(ArangoTarget(), compiled.Outputs[0], form.column, 0, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - traceRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: trace.Query, BindVars: trace.BindVars}) + traceRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: trace.Command}) for _, row := range traceRows { if traceRootKey(row, trace.IdentityPartsColumn) != patientKey("p1") { continue @@ -513,7 +514,7 @@ func TestRelatedSourceAllMatchesAtSelectedStageAgainstArango(t *testing.T) { if err != nil { t.Fatal(err) } - existsQuery, err := CompileRecipeOutputWithPolicy(existsCompiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + existsQuery, err := CompileRecipeOutput(ArangoTarget(), existsCompiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -523,7 +524,6 @@ func TestRelatedSourceAllMatchesAtSelectedStageAgainstArango(t *testing.T) { existsByPatient[fmt.Sprint(row["patient_id"])] = row } existsStatuses, ok := existsByPatient["p1"]["observation_status"].([]any) - // ALL retains distinct source records even when multiple edges reach one record. if !ok || len(existsStatuses) != 2 { t.Fatalf("EXISTS p1 statuses = %#v, want one status per distinct eligible source record", existsByPatient["p1"]["observation_status"]) } @@ -538,11 +538,11 @@ func TestRelatedSourceAllMatchesAtSelectedStageAgainstArango(t *testing.T) { if !ok || len(existsSparse) != 0 { t.Fatalf("EXISTS sparse p2 statuses = %#v, want empty list", existsByPatient["p2"]["observation_status"]) } - existsTrace, err := CompileCellTraceOutputWithPolicy(existsCompiled.Outputs[0], "observation_status", 0, 100, ir.DefaultPhysicalOptimizationPolicy()) + existsTrace, err := CompileCellTraceOutput(ArangoTarget(), existsCompiled.Outputs[0], "observation_status", 0, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - existsTraceRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: existsTrace.Query, BindVars: existsTrace.BindVars}) + existsTraceRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: existsTrace.Command}) for _, row := range existsTraceRows { if traceRootKey(row, existsTrace.IdentityPartsColumn) != patientKey("p1") { continue @@ -563,12 +563,13 @@ func TestPagedConstructionEqualsFilterReturnsMatchingSourceRootFromArango(t *tes } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Specimen"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Specimen"}}}); err != nil { t.Fatal(err) } @@ -621,12 +622,12 @@ func TestPagedConstructionEqualsFilterReturnsMatchingSourceRootFromArango(t *tes if err != nil { t.Fatal(err) } - page, err := CompileRecipeOutputPageWithPolicy(compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } rootKeys := make([]string, 0, 1) - if err := client.QueryRows(ctx, page.RootKeysQuery, 100, page.RootKeysBindVars, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, page.RootKeys), 100, testAQLBinds(t, page.RootKeys), func(row map[string]any) error { key, ok := row["_key"].(string) if !ok { return fmt.Errorf("root-key query returned invalid _key %#v", row["_key"]) @@ -634,24 +635,24 @@ func TestPagedConstructionEqualsFilterReturnsMatchingSourceRootFromArango(t *tes rootKeys = append(rootKeys, key) return nil }); err != nil { - t.Fatalf("execute construction EQUALS root-key query:\n%s\n%v", page.RootKeysQuery, err) + t.Fatalf("execute construction EQUALS root-key query:\n%s\n%v", testAQLText(t, page.RootKeys), err) } wantKey := fmt.Sprintf("%s_specimen_1", project) if len(rootKeys) != 1 || rootKeys[0] != wantKey { - t.Fatalf("construction EQUALS root keys = %#v, want only %q\n%s", rootKeys, wantKey, page.RootKeysQuery) + t.Fatalf("construction EQUALS root keys = %#v, want only %q\n%s", rootKeys, wantKey, testAQLText(t, page.RootKeys)) } - rowBindVars := make(map[string]any, len(page.RowsBindVars)+1) - for key, value := range page.RowsBindVars { + rowBindVars := make(map[string]any, len(testAQLBinds(t, page.Rows))+1) + for key, value := range testAQLBinds(t, page.Rows) { rowBindVars[key] = value } rowBindVars[RootPageKeysBind] = rootKeys rows := make([]map[string]any, 0, 1) - if err := client.QueryRows(ctx, page.RowsQuery, 100, rowBindVars, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, page.Rows), 100, rowBindVars, func(row map[string]any) error { rows = append(rows, row) return nil }); err != nil { - t.Fatalf("execute selected construction rows query:\n%s\n%v", page.RowsQuery, err) + t.Fatalf("execute selected construction rows query:\n%s\n%v", testAQLText(t, page.Rows), err) } if len(rows) != 1 || rows[0]["specimen_id"] != targetID { t.Fatalf("construction EQUALS rows = %#v, want one row with Specimen ID %q", rows, targetID) diff --git a/internal/dataframe/compiler/construction_coded_group_arango_test.go b/internal/dataframe/compiler/construction_coded_group_arango_test.go index 9ad22a911..fa044a86d 100644 --- a/internal/dataframe/compiler/construction_coded_group_arango_test.go +++ b/internal/dataframe/compiler/construction_coded_group_arango_test.go @@ -10,11 +10,11 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/compiler/render/aql" "github.com/calypr/loom/internal/dataframe/recipe" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -140,7 +140,7 @@ func TestConstructionCodedGroupRowLineagePagesScopedContributorsAgainstArango(t } output := constructionCodedGroupTestOutput() compiled := lowerConstructionOutput(t, output, bindings) - preview, err := CompileRecipeOutputWithPolicy(compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + preview, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -168,7 +168,7 @@ func TestConstructionCodedGroupRowLineagePagesScopedContributorsAgainstArango(t } compilePage := func(rowID string, offset int) CompiledRowLineageQuery { t.Helper() - query, compileErr := CompileRowLineageOutput(compiled, rowID, offset, 1, ir.DefaultPhysicalOptimizationPolicy()) + query, compileErr := CompileRowLineageOutput(ArangoTarget(), compiled, rowID, offset, 1, ir.DefaultPhysicalOptimizationPolicy()) if compileErr != nil { t.Fatalf("compile CODED_GROUP row-lineage page at %d: %v", offset, compileErr) } @@ -176,8 +176,8 @@ func TestConstructionCodedGroupRowLineagePagesScopedContributorsAgainstArango(t "root.project == @project", "root.dataset_generation == @dataset_generation", "root.auth_resource_path IN @auth_resource_paths", } { - if !strings.Contains(query.Query, want) { - t.Errorf("CODED_GROUP row-lineage query lost source scope %q:\n%s", want, query.Query) + if !strings.Contains(testAQLText(t, query.Command), want) { + t.Errorf("CODED_GROUP row-lineage query lost source scope %q:\n%s", want, testAQLText(t, query.Command)) } } return query @@ -235,7 +235,7 @@ func TestConstructionCodedGroupInliningMatchesAuthorizedAndPopulationSourceRowsA project, generation := "loom_coded_group_inline_"+suffix, "generation-coded-group-inline" fixtureCollection := "loom_coded_group_inline_roots_" + suffix populationCollection := "loom_coded_group_inline_members_" + suffix - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: fixtureCollection}, {Name: populationCollection}, }}); err != nil { t.Fatal(err) @@ -252,11 +252,11 @@ func TestConstructionCodedGroupInliningMatchesAuthorizedAndPopulationSourceRowsA } allOutput := constructionCodedGroupTestOutput() allCompiled := lowerConstructionOutput(t, allOutput, bindings) - allQuery, err := CompileRecipeOutputWithPolicy(allCompiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + allQuery, err := CompileRecipeOutput(ArangoTarget(), allCompiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - allQuery.BindVars["@root_collection"] = fixtureCollection + testAQLBinds(t, allQuery.Command)["@root_collection"] = fixtureCollection allSourceIDs := codedGroupSourceProjectionIDs(t, ctx, client, allCompiled, fixtureCollection) allRows := executeReshapeOracleQuery(t, ctx, client, allQuery) assertCodedGroupSourceIDsMatchCodes(t, allRows, allSourceIDs) @@ -272,18 +272,18 @@ func TestConstructionCodedGroupInliningMatchesAuthorizedAndPopulationSourceRowsA selectedBindings.SelectionProject = project selectedBindings.SelectionMembersCollection = populationCollection selectedCompiled := lowerConstructionOutput(t, selectedOutput, selectedBindings) - selectedQuery, err := CompileRecipeOutputWithPolicy(selectedCompiled, selectedBindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + selectedQuery, err := CompileRecipeOutput(ArangoTarget(), selectedCompiled, selectedBindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - selectedQuery.BindVars["@root_collection"] = fixtureCollection - selectedQuery.BindVars["@population_source_collection"] = fixtureCollection + testAQLBinds(t, selectedQuery.Command)["@root_collection"] = fixtureCollection + testAQLBinds(t, selectedQuery.Command)["@population_source_collection"] = fixtureCollection selectedSourceIDs := codedGroupSourceProjectionIDs(t, ctx, client, selectedCompiled, fixtureCollection) selectedRows := executeReshapeOracleQuery(t, ctx, client, selectedQuery) assertCodedGroupSourceIDsMatchCodes(t, selectedRows, selectedSourceIDs) } -func insertCodedGroupResources(t *testing.T, ctx context.Context, client *store.Client, collection, project, generation string, payloads []map[string]any) { +func insertCodedGroupResources(t *testing.T, ctx context.Context, client *arangofactory.Client, collection, project, generation string, payloads []map[string]any) { t.Helper() documents := make([]json.RawMessage, 0, len(payloads)) for _, payload := range payloads { @@ -309,7 +309,7 @@ func codedGroupComponent(code string) map[string]any { }}} } -func insertCodedGroupPopulationMembers(t *testing.T, ctx context.Context, client *store.Client, collection, selectionID, project, generation string, ids []string) { +func insertCodedGroupPopulationMembers(t *testing.T, ctx context.Context, client *arangofactory.Client, collection, selectionID, project, generation string, ids []string) { t.Helper() documents := make([]json.RawMessage, 0, len(ids)) for _, id := range ids { @@ -327,7 +327,7 @@ func insertCodedGroupPopulationMembers(t *testing.T, ctx context.Context, client } } -func codedGroupSourceProjectionIDs(t *testing.T, ctx context.Context, client *store.Client, compiled lower.CompiledRecipeOutput, collection string) []string { +func codedGroupSourceProjectionIDs(t *testing.T, ctx context.Context, client *arangofactory.Client, compiled lower.CompiledRecipeOutput, collection string) []string { t.Helper() plan := ir.ClonePhysicalPlan(compiled.Plan) plan.StageSequence = nil diff --git a/internal/dataframe/compiler/construction_coded_group_execution_test.go b/internal/dataframe/compiler/construction_coded_group_execution_test.go index f38034a41..da950a2a6 100644 --- a/internal/dataframe/compiler/construction_coded_group_execution_test.go +++ b/internal/dataframe/compiler/construction_coded_group_execution_test.go @@ -18,7 +18,7 @@ func TestCodedGroupPreviewWithSourceIdentityKeepsStageInputScoped(t *testing.T) AuthScopeMode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{"BodyStructure/allowed"}, } compiled := lowerConstructionOutput(t, bodyStructureCodedGroupOutput(), bindings) - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -32,26 +32,26 @@ func TestCodedGroupPreviewWithSourceIdentityKeepsStageInputScoped(t *testing.T) "root.payload[\"includedStructure\"]", "AGGREGATE __loom_construction_coded_group_count_1 = SUM(1)", } { - if !strings.Contains(query.Query, expected) { - t.Errorf("inlined coded-group query is missing %q:\n%s", expected, query.Query) + if !strings.Contains(testAQLText(t, query.Command), expected) { + t.Errorf("inlined coded-group query is missing %q:\n%s", expected, testAQLText(t, query.Command)) } } for _, unexpected := range []string{"__loom_construction_source_projection", "DOCUMENT(@@root_collection", "IN __loom_construction_input_1"} { - if strings.Contains(query.Query, unexpected) { - t.Fatalf("direct-root CODED_GROUP query retained fallback fragment %q:\n%s", unexpected, query.Query) + if strings.Contains(testAQLText(t, query.Command), unexpected) { + t.Fatalf("direct-root CODED_GROUP query retained fallback fragment %q:\n%s", unexpected, testAQLText(t, query.Command)) } } - if strings.Contains(query.Query, ir.PreviewSourceResourceIDColumn) { - t.Fatalf("composite CODED_GROUP preview gained a single-source identity projection:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), ir.PreviewSourceResourceIDColumn) { + t.Fatalf("composite CODED_GROUP preview gained a single-source identity projection:\n%s", testAQLText(t, query.Command)) } if got, want := query.PublicColumns, []string{"code_system", "code_version", "code", "source_records"}; !reflect.DeepEqual(got, want) { t.Fatalf("CODED_GROUP preview public columns = %#v, want %#v", got, want) } - if query.BindVars["auth_resource_paths_unrestricted"] != false { - t.Fatalf("inlined source bypassed restricted authorization: %#v", query.BindVars) + if testAQLBinds(t, query.Command)["auth_resource_paths_unrestricted"] != false { + t.Fatalf("inlined source bypassed restricted authorization: %#v", testAQLBinds(t, query.Command)) } - if paths, ok := query.BindVars["auth_resource_paths"].([]string); !ok || !reflect.DeepEqual(paths, []string{"BodyStructure/allowed"}) { - t.Fatalf("inlined source changed the authorized paths: %#v", query.BindVars["auth_resource_paths"]) + if paths, ok := testAQLBinds(t, query.Command)["auth_resource_paths"].([]string); !ok || !reflect.DeepEqual(paths, []string{"BodyStructure/allowed"}) { + t.Fatalf("inlined source changed the authorized paths: %#v", testAQLBinds(t, query.Command)["auth_resource_paths"]) } } @@ -83,7 +83,7 @@ func TestCodedGroupInlinesPopulationSelectedRootScan(t *testing.T) { AuthScopeMode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{"BodyStructure/allowed"}, } compiled := lowerConstructionOutput(t, output, bindings) - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -94,12 +94,12 @@ func TestCodedGroupInlinesPopulationSelectedRootScan(t *testing.T) { "LET root = DOCUMENT(@@root_collection, __loom_physical_population_root_key)", "root.payload[\"includedStructure\"]", } { - if !strings.Contains(query.Query, expected) { - t.Errorf("population-selected coded-group query is missing %q:\n%s", expected, query.Query) + if !strings.Contains(testAQLText(t, query.Command), expected) { + t.Errorf("population-selected coded-group query is missing %q:\n%s", expected, testAQLText(t, query.Command)) } } - if strings.Contains(query.Query, "__loom_construction_source_projection") || strings.Contains(query.Query, "__loom_construction_coded_group_source_id") { - t.Fatalf("population-selected first-stage CODED_GROUP did not inline its physical root scan:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "__loom_construction_source_projection") || strings.Contains(testAQLText(t, query.Command), "__loom_construction_coded_group_source_id") { + t.Fatalf("population-selected first-stage CODED_GROUP did not inline its physical root scan:\n%s", testAQLText(t, query.Command)) } } diff --git a/internal/dataframe/compiler/construction_coded_pivot_test.go b/internal/dataframe/compiler/construction_coded_pivot_test.go index e8ebf1293..bc9f30c98 100644 --- a/internal/dataframe/compiler/construction_coded_pivot_test.go +++ b/internal/dataframe/compiler/construction_coded_pivot_test.go @@ -15,18 +15,18 @@ func TestConstructionCodedPivotPreviewLimitsRootsBeforeCodedExtraction(t *testin t.Fatal("direct-root CODED_PIVOT should allow an indexed source-root preview window") } - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - sortIndex := strings.Index(query.Query, "SORT root.id ASC") - limitIndex := strings.Index(query.Query, "LIMIT @limit") - ownerIndex := strings.Index(query.Query, "coded_pivot_owner") + sortIndex := strings.Index(testAQLText(t, query.Command), "SORT root.id ASC") + limitIndex := strings.Index(testAQLText(t, query.Command), "LIMIT @limit") + ownerIndex := strings.Index(testAQLText(t, query.Command), "coded_pivot_owner") if sortIndex < 0 || limitIndex < 0 || ownerIndex < 0 || !(sortIndex < limitIndex && limitIndex < ownerIndex) { - t.Fatalf("preview must bound roots before running correlated category extraction (sort=%d limit=%d owner=%d):\n%s", sortIndex, limitIndex, ownerIndex, query.Query) + t.Fatalf("preview must bound roots before running correlated category extraction (sort=%d limit=%d owner=%d):\n%s", sortIndex, limitIndex, ownerIndex, testAQLText(t, query.Command)) } - if strings.Contains(query.Query, "__loom_construction_source_projection") || strings.Contains(query.Query, "COLLECT ") { - t.Fatalf("bounded coded preview must stream selected roots through the typed Pivot stage:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "__loom_construction_source_projection") || strings.Contains(testAQLText(t, query.Command), "COLLECT ") { + t.Fatalf("bounded coded preview must stream selected roots through the typed Pivot stage:\n%s", testAQLText(t, query.Command)) } } diff --git a/internal/dataframe/compiler/construction_execution_test.go b/internal/dataframe/compiler/construction_execution_test.go index 22875815c..b3f725cc1 100644 --- a/internal/dataframe/compiler/construction_execution_test.go +++ b/internal/dataframe/compiler/construction_execution_test.go @@ -42,7 +42,7 @@ func TestConstructionPreviewLimitAppliesAfterFinalStage(t *testing.T) { if err != nil { t.Fatal(err) } - queries, err := CompileResolvedRecipePlanWithPolicy(resolved, 3, ir.DefaultPhysicalOptimizationPolicy()) + queries, err := CompileResolvedRecipePlan(ArangoTarget(), resolved, 3, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -50,22 +50,22 @@ func TestConstructionPreviewLimitAppliesAfterFinalStage(t *testing.T) { t.Fatalf("compiled queries = %d, want one", len(queries)) } query := queries[0] - finalRows := strings.Index(query.Query, "FOR __loom_construction_final_row IN") - limit := strings.Index(query.Query, "LIMIT @limit") + finalRows := strings.Index(testAQLText(t, query.Command), "FOR __loom_construction_final_row IN") + limit := strings.Index(testAQLText(t, query.Command), "LIMIT @limit") if finalRows < 0 || limit < finalRows { - t.Fatalf("preview limit is not applied after the final stage rows:\n%s", query.Query) + t.Fatalf("preview limit is not applied after the final stage rows:\n%s", testAQLText(t, query.Command)) } - if got, exists := query.BindVars["limit"]; !exists || got != 3 { + if got, exists := testAQLBinds(t, query.Command)["limit"]; !exists || got != 3 { t.Fatalf("preview limit bind = %#v, exists=%t", got, exists) } - if got := query.BindVars["@root_collection"]; got != "Patient" { + if got := testAQLBinds(t, query.Command)["@root_collection"]; got != "Patient" { t.Fatalf("root collection bind = %#v, want collection-bind form with Patient", got) } - if _, exists := query.BindVars["root_collection"]; exists { + if _, exists := testAQLBinds(t, query.Command)["root_collection"]; exists { t.Fatal("plain root collection bind leaked into executable AQL bind variables") } - if strings.Contains(query.Query[:finalRows], "LIMIT @limit") { - t.Fatalf("preview limit leaked into source scan before construction stages:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command)[:finalRows], "LIMIT @limit") { + t.Fatalf("preview limit leaked into source scan before construction stages:\n%s", testAQLText(t, query.Command)) } } @@ -107,12 +107,12 @@ func TestStreamingMissingCategoryPivotUsesStoredParentProjectionIndex(t *testing if !outputHasCompositeSource(compiled.Outputs[0]) { t.Fatal("production-shaped Pivot lost its composite-source classification under IncludeSourceIdentity") } - query, err := CompileRecipeOutputWithPolicy(compiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } if query.PreviewCoveringIndex == nil { - t.Fatalf("direct-root typed-MISSING Pivot has no stored-values index contract: %s", query.Query) + t.Fatalf("direct-root typed-MISSING Pivot has no stored-values index contract: %s", testAQLText(t, query.Command)) } physical, err := optimize.OptimizePhysicalPlanWithPolicy(compiled.Outputs[0].Plan, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { @@ -176,30 +176,30 @@ func TestStreamingMissingCategoryPivotUsesStoredParentProjectionIndex(t *testing "HAS(__loom_physical_construction_reshape_stream_source_parent, @", "TABLE_PIVOT_UNLISTED_CATEGORY", } { - if !strings.Contains(query.Query, required) { - t.Fatalf("stored-parent streaming query omitted %q:\n%s", required, query.Query) + if !strings.Contains(testAQLText(t, query.Command), required) { + t.Fatalf("stored-parent streaming query omitted %q:\n%s", required, testAQLText(t, query.Command)) } } compiledPivot := compiled.Outputs[0].Plan.StageSequence.Stages[0].GroupedPivot - if streamingTestBindKey(query.BindVars, "reshape_category_column", compiledPivot.CategoryColumn) != "" || - streamingTestBindKey(query.BindVars, "reshape_category_presence_column", compiledPivot.CategoryPresenceColumn) != "" { - t.Fatalf("direct scalar Pivot retained field-name binds: %#v", query.BindVars) + if streamingTestBindKey(testAQLBinds(t, query.Command), "reshape_category_column", compiledPivot.CategoryColumn) != "" || + streamingTestBindKey(testAQLBinds(t, query.Command), "reshape_category_presence_column", compiledPivot.CategoryPresenceColumn) != "" { + t.Fatalf("direct scalar Pivot retained field-name binds: %#v", testAQLBinds(t, query.Command)) } - categoryValue := streamingTestDirectScalarVariable(query.Query, "__loom_physical_construction_reshape_stream_source_parent.code") - presenceValue := streamingTestDirectScalarVariable(query.Query, "IS_OBJECT(__loom_physical_construction_reshape_stream_source_parent)") + categoryValue := streamingTestDirectScalarVariable(testAQLText(t, query.Command), "__loom_physical_construction_reshape_stream_source_parent.code") + presenceValue := streamingTestDirectScalarVariable(testAQLText(t, query.Command), "IS_OBJECT(__loom_physical_construction_reshape_stream_source_parent)") if categoryValue == "" || presenceValue == "" { - t.Fatalf("direct scalar category or presence projection missing: category=%q presence=%q\n%s", categoryValue, presenceValue, query.Query) + t.Fatalf("direct scalar category or presence projection missing: category=%q presence=%q\n%s", categoryValue, presenceValue, testAQLText(t, query.Command)) } missingMatch := fmt.Sprintf("NOT (%s == true)", presenceValue) nullMatch := fmt.Sprintf("(%s == true AND %s == null)", presenceValue, categoryValue) - if !strings.Contains(query.Query, missingMatch) || !strings.Contains(query.Query, nullMatch) { - t.Fatalf("stored-parent projection merged MISSING and explicit NULL categories: missing=%q null=%q\n%s", missingMatch, nullMatch, query.Query) + if !strings.Contains(testAQLText(t, query.Command), missingMatch) || !strings.Contains(testAQLText(t, query.Command), nullMatch) { + t.Fatalf("stored-parent projection merged MISSING and explicit NULL categories: missing=%q null=%q\n%s", missingMatch, nullMatch, testAQLText(t, query.Command)) } - if strings.Contains(query.Query, "HAS(root.payload.valueQuantity.code)") { - t.Fatalf("presence test inspected the leaf instead of its stored parent object:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "HAS(root.payload.valueQuantity.code)") { + t.Fatalf("presence test inspected the leaf instead of its stored parent object:\n%s", testAQLText(t, query.Command)) } - if !strings.Contains(query.Query, " AGGREGATE ") || strings.Contains(query.Query, "INTO __loom_reshape_pivot_group_rows_") { - t.Fatalf("stored-parent preview stopped using complete-population streaming aggregation:\n%s", query.Query) + if !strings.Contains(testAQLText(t, query.Command), " AGGREGATE ") || strings.Contains(testAQLText(t, query.Command), "INTO __loom_reshape_pivot_group_rows_") { + t.Fatalf("stored-parent preview stopped using complete-population streaming aggregation:\n%s", testAQLText(t, query.Command)) } } @@ -212,71 +212,69 @@ func TestRootGroupedPivotPreviewBoundsParentsBeforeRelatedExpansion(t *testing.T pivot.Inputs = []recipe.ConstructionInputRef{{Kind: recipe.ConstructionStepOutputInput, StepID: expand.ID}} output.Construction.Steps = []recipe.ConstructionStep{expand, pivot} query := compileConstructionPivotPreview(t, output, 3) - limit := strings.Index(query.Query, "LIMIT @limit") - stage := strings.Index(query.Query, "LET __loom_construction_stage_1") - if !query.PartialValidation || limit < 0 || stage < limit || !strings.Contains(query.Query[stage:], "COLLECT") { - t.Fatalf("preview must bound parents but retain complete related contributions and grouping:\n%s", query.Query) + limit := strings.Index(testAQLText(t, query.Command), "LIMIT @limit") + stage := strings.Index(testAQLText(t, query.Command), "LET __loom_construction_stage_1") + if !query.PartialValidation || limit < 0 || stage < limit || !strings.Contains(testAQLText(t, query.Command)[stage:], "COLLECT") { + t.Fatalf("preview must bound parents but retain complete related contributions and grouping:\n%s", testAQLText(t, query.Command)) } full := compileConstructionPivotPreview(t, output, 0) - if strings.Contains(full.Query, "LIMIT @limit") || full.PartialValidation { + if strings.Contains(testAQLText(t, full.Command), "LIMIT @limit") || full.PartialValidation { t.Fatal("preview window leaked into full execution") } expand.Operation.RelatedExpand.EmptyPolicy = recipe.ExpansionExclude expand.Outputs[len(expand.Outputs)-1].Nullable = false output.Construction.Steps[0] = expand query = compileConstructionPivotPreview(t, output, 3) - if strings.Contains(query.Query[:strings.Index(query.Query, "LET __loom_construction_stage_1")], "LIMIT @limit") { + if strings.Contains(testAQLText(t, query.Command)[:strings.Index(testAQLText(t, query.Command), "LET __loom_construction_stage_1")], "LIMIT @limit") { t.Fatal("dropping unmatched parents must not use a source-root window") } } func TestConstructionPreviewBoundsProvenRootIDPivotBeforeSourceMaterialization(t *testing.T) { query := compileConstructionPivotPreview(t, constructionPreviewPivotOutput("root.id", false), 3) - sourceStart := strings.Index(query.Query, "LET __loom_construction_source_projection = (") - stageStart := strings.Index(query.Query, "LET __loom_construction_stage_1 = (") + sourceStart := strings.Index(testAQLText(t, query.Command), "LET __loom_construction_source_projection = (") + stageStart := strings.Index(testAQLText(t, query.Command), "LET __loom_construction_stage_1 = (") if sourceStart < 0 || stageStart <= sourceStart { - t.Fatalf("construction query has no source projection followed by terminal stage:\n%s", query.Query) + t.Fatalf("construction query has no source projection followed by terminal stage:\n%s", testAQLText(t, query.Command)) } - sourceQuery := query.Query[sourceStart:stageStart] + sourceQuery := testAQLText(t, query.Command)[sourceStart:stageStart] sortAt := strings.Index(sourceQuery, "SORT root.id ASC") limitAt := strings.Index(sourceQuery, "LIMIT @limit") returnAt := strings.Index(sourceQuery, "RETURN ") if sortAt < 0 || limitAt <= sortAt || returnAt <= limitAt { t.Fatalf("root-id sort/limit did not run before source projection materialization:\n%s", sourceQuery) } - if got, exists := query.BindVars["limit"]; !exists || got != 3 { + if got, exists := testAQLBinds(t, query.Command)["limit"]; !exists || got != 3 { t.Fatalf("preview limit bind = %#v, exists=%t", got, exists) } if !query.PartialValidation { t.Fatal("preview with a source window was not marked as partial validation") } - stageQuery := query.Query[stageStart:] + stageQuery := testAQLText(t, query.Command)[stageStart:] if strings.Contains(stageQuery, "COLLECT") { - t.Fatalf("proven root-ID pivot still materialized group rows:\n%s", query.Query) + t.Fatalf("proven root-ID pivot still materialized group rows:\n%s", testAQLText(t, query.Command)) } if strings.Count(stageQuery, "FOR ") != 2 { - // One FOR iterates source rows inside the stage and one emits final rows. - // Category matching must not add a FOR over group rows per output cell. - t.Fatalf("root-ID pivot emitted per-category group-row scans:\n%s", query.Query) + t.Fatalf("root-ID pivot emitted per-category group-row scans:\n%s", testAQLText(t, query.Command)) } if strings.Contains(stageQuery, "SORT __loom_construction_pivot_group") { - t.Fatalf("root-ID pivot retained an intermediate sort before the final identity sort:\n%s", query.Query) + t.Fatalf("root-ID pivot retained an intermediate sort before the final identity sort:\n%s", testAQLText(t, query.Command)) } for _, validation := range []string{"TYPENAME(", "TABLE_PIVOT_VALUE_TYPE_MISMATCH", "TABLE_PIVOT_CELL_CARDINALITY", "TABLE_PIVOT_UNLISTED_CATEGORY"} { if !strings.Contains(stageQuery, validation) { - t.Fatalf("root-ID pivot dropped %q validation:\n%s", validation, query.Query) + t.Fatalf("root-ID pivot dropped %q validation:\n%s", validation, testAQLText(t, query.Command)) } } full := compileConstructionPivotPreview(t, constructionPreviewPivotOutput("root.id", false), 0) - if strings.Contains(full.Query, "LIMIT @limit") || strings.Contains(full.Query, "SORT root.id ASC") { - t.Fatalf("full construction execution received a preview-only source window:\n%s", full.Query) + if strings.Contains(testAQLText(t, full.Command), "LIMIT @limit") || strings.Contains(testAQLText(t, full.Command), "SORT root.id ASC") { + t.Fatalf("full construction execution received a preview-only source window:\n%s", testAQLText(t, full.Command)) } - if strings.Contains(full.Query, "COLLECT") || strings.Count(full.Query, "FOR ") != 3 { - t.Fatalf("full direct-ID pivot did not retain the one-input-per-group renderer:\n%s", full.Query) + if strings.Contains(testAQLText(t, full.Command), "COLLECT") || strings.Count(testAQLText(t, full.Command), "FOR ") != 3 { + t.Fatalf("full direct-ID pivot did not retain the one-input-per-group renderer:\n%s", testAQLText(t, full.Command)) } - if !strings.Contains(full.Query, "SORT __loom_construction_final_row.__loom_row_id ASC") { - t.Fatalf("full construction output lost deterministic final row-identity ordering:\n%s", full.Query) + if !strings.Contains(testAQLText(t, full.Command), "SORT __loom_construction_final_row.__loom_row_id ASC") { + t.Fatalf("full construction output lost deterministic final row-identity ordering:\n%s", testAQLText(t, full.Command)) } if full.PartialValidation { t.Fatal("full construction execution was marked as partial validation") @@ -287,8 +285,8 @@ func TestConstructionPreviewRootIDProofUsesLoweredProjectionNotSemanticAlias(t * output := constructionPreviewPivotOutput("root.id", false) output.Fields[0].FieldRef = "source-column:specimen-id" query := compileConstructionPivotPreview(t, output, 3) - if !strings.Contains(query.Query, "SORT root.id ASC") || !strings.Contains(query.Query, "LIMIT @limit") { - t.Fatalf("direct lowered root id with opaque semantic identity did not qualify for source window:\n%s", query.Query) + if !strings.Contains(testAQLText(t, query.Command), "SORT root.id ASC") || !strings.Contains(testAQLText(t, query.Command), "LIMIT @limit") { + t.Fatalf("direct lowered root id with opaque semantic identity did not qualify for source window:\n%s", testAQLText(t, query.Command)) } } @@ -330,17 +328,17 @@ func TestConstructionPreviewUsesTerminalPivotWindowWithoutRootIDProof(t *testing } { t.Run(tc.name, func(t *testing.T) { query := compileConstructionPivotPreview(t, tc.output, 3) - if strings.Contains(query.Query, "SORT root.id ASC") { - t.Fatalf("unproven plan received the early source order:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "SORT root.id ASC") { + t.Fatalf("unproven plan received the early source order:\n%s", testAQLText(t, query.Command)) } - if strings.Count(query.Query, "LIMIT @limit") != 2 { - t.Fatalf("terminal Pivot preview should limit complete groups and retain the final limit:\n%s", query.Query) + if strings.Count(testAQLText(t, query.Command), "LIMIT @limit") != 2 { + t.Fatalf("terminal Pivot preview should limit complete groups and retain the final limit:\n%s", testAQLText(t, query.Command)) } if !query.PartialValidation { t.Fatal("preview group window was not marked as partial validation") } - if tc.name == "non-identity group key" && !strings.Contains(query.Query, "COLLECT") { - t.Fatalf("nonunique group key bypassed canonical COLLECT:\n%s", query.Query) + if tc.name == "non-identity group key" && !strings.Contains(testAQLText(t, query.Command), "COLLECT") { + t.Fatalf("nonunique group key bypassed canonical COLLECT:\n%s", testAQLText(t, query.Command)) } }) } @@ -351,23 +349,23 @@ func TestConstructionPreviewTerminalPivotLimitsCompleteGroupsBeforeCellReduction if !preview.PartialValidation { t.Fatal("terminal Pivot preview window was not marked as partial validation") } - selectedStart := strings.Index(preview.Query, "LET __loom_physical_construction_reshape_preview_selected_group_keys = (") + selectedStart := strings.Index(testAQLText(t, preview.Command), "LET __loom_physical_construction_reshape_preview_selected_group_keys = (") if selectedStart < 0 { - t.Fatalf("terminal Pivot preview has no first-pass group selection:\n%s", preview.Query) + t.Fatalf("terminal Pivot preview has no first-pass group selection:\n%s", testAQLText(t, preview.Command)) } - selectedLimit := strings.Index(preview.Query[selectedStart:], "LIMIT @limit") + selectedStart - sourceStart := strings.Index(preview.Query, "LET __loom_construction_source_projection = (") - filterAt := strings.Index(preview.Query, "FILTER root.payload.gender IN __loom_physical_construction_reshape_preview_selected_group_keys") - stageStart := strings.Index(preview.Query, "LET __loom_construction_stage_1 = (") + selectedLimit := strings.Index(testAQLText(t, preview.Command)[selectedStart:], "LIMIT @limit") + selectedStart + sourceStart := strings.Index(testAQLText(t, preview.Command), "LET __loom_construction_source_projection = (") + filterAt := strings.Index(testAQLText(t, preview.Command), "FILTER root.payload.gender IN __loom_physical_construction_reshape_preview_selected_group_keys") + stageStart := strings.Index(testAQLText(t, preview.Command), "LET __loom_construction_stage_1 = (") if selectedLimit <= selectedStart || sourceStart <= selectedLimit || filterAt <= sourceStart || stageStart <= filterAt { - t.Fatalf("preview must select complete group tuples, filter the direct source scan before its wide projection, then run the Pivot stage:\n%s", preview.Query) + t.Fatalf("preview must select complete group tuples, filter the direct source scan before its wide projection, then run the Pivot stage:\n%s", testAQLText(t, preview.Command)) } - keySource := preview.Query[selectedStart:sourceStart] + keySource := testAQLText(t, preview.Command)[selectedStart:sourceStart] if strings.Contains(keySource, "root.payload.multipleBirthInteger") || strings.Contains(keySource, "root.payload.category") || strings.Count(keySource, "RETURN {") != 1 { t.Fatalf("first pass materialized non-key source projections:\n%s", keySource) } - sourceQuery := preview.Query[sourceStart:stageStart] + sourceQuery := testAQLText(t, preview.Command)[sourceStart:stageStart] groupFilterAt := strings.Index(sourceQuery, "FILTER root.payload.gender IN __loom_physical_construction_reshape_preview_selected_group_keys") projectionAt := strings.Index(sourceQuery, "RETURN ") if !strings.Contains(sourceQuery, "root.payload.multipleBirthInteger") || groupFilterAt < 0 || projectionAt <= groupFilterAt { @@ -378,26 +376,26 @@ func TestConstructionPreviewTerminalPivotLimitsCompleteGroupsBeforeCellReduction "root.dataset_generation == @dataset_generation", "root.auth_resource_path IN @auth_resource_paths", } { - if strings.Count(preview.Query, scope) != 2 { - t.Fatalf("source scope %q appears %d times, want both scans:\n%s", scope, strings.Count(preview.Query, scope), preview.Query) + if strings.Count(testAQLText(t, preview.Command), scope) != 2 { + t.Fatalf("source scope %q appears %d times, want both scans:\n%s", scope, strings.Count(testAQLText(t, preview.Command), scope), testAQLText(t, preview.Command)) } } - stage := preview.Query[stageStart:] + stage := testAQLText(t, preview.Command)[stageStart:] collectAt := strings.Index(stage, "COLLECT ") cellValidationAt := strings.Index(stage, "TABLE_PIVOT_CELL_CARDINALITY") if collectAt < 0 || cellValidationAt <= collectAt { t.Fatalf("preview must collect selected groups completely before computing cells:\n%s", stage) } - if strings.Count(stage, "LIMIT @limit") != 1 || strings.Count(preview.Query, "LIMIT @limit") != 2 { - t.Fatalf("expected first-pass group and final output limits:\n%s", preview.Query) + if strings.Count(stage, "LIMIT @limit") != 1 || strings.Count(testAQLText(t, preview.Command), "LIMIT @limit") != 2 { + t.Fatalf("expected first-pass group and final output limits:\n%s", testAQLText(t, preview.Command)) } - if got := preview.BindVars["limit"]; got != 3 { + if got := testAQLBinds(t, preview.Command)["limit"]; got != 3 { t.Fatalf("preview limit bind = %#v, want 3", got) } full := compileConstructionPivotPreview(t, constructionPreviewPivotOutput("root.gender", false), 0) - if full.PartialValidation || strings.Contains(full.Query, "LIMIT @limit") || strings.Contains(full.Query, "reshape_preview_identity") { - t.Fatalf("full execution received the preview-only group window:\n%s", full.Query) + if full.PartialValidation || strings.Contains(testAQLText(t, full.Command), "LIMIT @limit") || strings.Contains(testAQLText(t, full.Command), "reshape_preview_identity") { + t.Fatalf("full execution received the preview-only group window:\n%s", testAQLText(t, full.Command)) } } @@ -405,15 +403,15 @@ func TestConstructionPreviewTwoScanUsesSeparateProjectionBindNamespaces(t *testi output := constructionPreviewPivotOutput("root.gender", false) output.Fields[0], output.Fields[1] = output.Fields[1], output.Fields[0] query := compileConstructionPivotPreview(t, output, 3) - if got := query.BindVars["__loom_physical_preview_key_projection_0_name"]; got != "resource_id" { + if got := testAQLBinds(t, query.Command)["__loom_physical_preview_key_projection_0_name"]; got != "resource_id" { t.Fatalf("first-pass projection-name bind = %#v, want resource_id", got) } - if got := query.BindVars["__loom_physical_projection_0_name"]; got != "_key" { + if got := testAQLBinds(t, query.Command)["__loom_physical_projection_0_name"]; got != "_key" { t.Fatalf("second-pass projection-name bind = %#v, want _key", got) } - if _, ok := query.BindVars["__loom_physical_reshape_preview_source_group_column"]; ok || - !strings.Contains(query.Query, "FILTER root.payload.gender IN __loom_physical_construction_reshape_preview_selected_group_keys") { - t.Fatalf("second-pass filter did not use the compiler-proven direct group source path:\n%s", query.Query) + if _, ok := testAQLBinds(t, query.Command)["__loom_physical_reshape_preview_source_group_column"]; ok || + !strings.Contains(testAQLText(t, query.Command), "FILTER root.payload.gender IN __loom_physical_construction_reshape_preview_selected_group_keys") { + t.Fatalf("second-pass filter did not use the compiler-proven direct group source path:\n%s", testAQLText(t, query.Command)) } } @@ -421,7 +419,7 @@ func TestConstructionPreviewCoveringIndexMetadataAndRootHint(t *testing.T) { query := compileConstructionPivotPreview(t, constructionPreviewPivotOutput("root.gender", false), 3) index := query.PreviewCoveringIndex if index == nil { - t.Fatalf("eligible terminal Pivot preview has no covering-index spec:\n%s", query.Query) + t.Fatalf("eligible terminal Pivot preview has no covering-index spec:\n%s", testAQLText(t, query.Command)) } if index.PrepareAfterPreview { t.Fatal("Pivot covering index must remain prepared before the preview query") @@ -437,11 +435,11 @@ func TestConstructionPreviewCoveringIndexMetadataAndRootHint(t *testing.T) { t.Fatalf("index fields = %#v, want %#v", index.Fields, wantFields) } wantHint := "OPTIONS { indexHint: \"" + index.Name + "\", forceIndexHint: false }" - if strings.Count(query.Query, wantHint) != 2 { - t.Fatalf("both source scans must use the same non-forcing index hint %q:\n%s", wantHint, query.Query) + if strings.Count(testAQLText(t, query.Command), wantHint) != 2 { + t.Fatalf("both source scans must use the same non-forcing index hint %q:\n%s", wantHint, testAQLText(t, query.Command)) } - if strings.Count(query.Query, "FOR root IN @@root_collection "+wantHint) != 2 { - t.Fatalf("index hint was not placed on both root source scans:\n%s", query.Query) + if strings.Count(testAQLText(t, query.Command), "FOR root IN @@root_collection "+wantHint) != 2 { + t.Fatalf("index hint was not placed on both root source scans:\n%s", testAQLText(t, query.Command)) } repeated := compileConstructionPivotPreview(t, constructionPreviewPivotOutput("root.gender", false), 3) @@ -465,8 +463,8 @@ func TestConstructionPreviewCoveringIndexIneligiblePlansStayUnhinted(t *testing. if query.PreviewCoveringIndex != nil { t.Fatalf("ineligible plan produced index metadata: %+v", query.PreviewCoveringIndex) } - if strings.Contains(query.Query, "indexHint:") { - t.Fatalf("ineligible plan received an index hint:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "indexHint:") { + t.Fatalf("ineligible plan received an index hint:\n%s", testAQLText(t, query.Command)) } }) } @@ -476,7 +474,7 @@ func TestConstructionGroupPreviewUsesProjectionFieldsAndOneRootHint(t *testing.T query := compileConstructionOutputQuery(t, constructionGroupRowValuesOutput(), "group-covering-project", "group-covering-generation") index := query.PreviewCoveringIndex if index == nil { - t.Fatalf("eligible terminal Group preview has no covering-index spec:\n%s", query.Query) + t.Fatalf("eligible terminal Group preview has no covering-index spec:\n%s", testAQLText(t, query.Command)) } if index.Collection != "Observation" { t.Fatalf("index collection = %q, want Observation", index.Collection) @@ -489,49 +487,50 @@ func TestConstructionGroupPreviewUsesProjectionFieldsAndOneRootHint(t *testing.T t.Fatalf("index fields = %#v, want %#v", index.Fields, wantFields) } wantHint := "OPTIONS { indexHint: \"" + index.Name + "\", forceIndexHint: false }" - if strings.Count(query.Query, wantHint) != 1 || !strings.Contains(query.Query, "FOR root IN @@root_collection "+wantHint) { - t.Fatalf("Group source should receive one non-forcing root hint %q:\n%s", wantHint, query.Query) + if strings.Count(testAQLText(t, query.Command), wantHint) != 1 || !strings.Contains(testAQLText(t, query.Command), "FOR root IN @@root_collection "+wantHint) { + t.Fatalf("Group source should receive one non-forcing root hint %q:\n%s", wantHint, testAQLText(t, query.Command)) } - if strings.Count(query.Query, "FOR root IN @@root_collection") != 1 { - t.Fatalf("Group preview unexpectedly changed its source scan count:\n%s", query.Query) + if strings.Count(testAQLText(t, query.Command), "FOR root IN @@root_collection") != 1 { + t.Fatalf("Group preview unexpectedly changed its source scan count:\n%s", testAQLText(t, query.Command)) } sequential := query.PreviewGroupScan if sequential == nil || sequential.Collection != index.Collection { t.Fatalf("eligible direct Group preview has no scan alternatives: %+v", sequential) } - if !strings.Contains(sequential.SequentialQuery, "OPTIONS { disableIndex: true }") || strings.Contains(sequential.SequentialQuery, "indexHint:") { - t.Fatalf("sequential alternative did not disable only its direct root index:\n%s", sequential.SequentialQuery) + if !strings.Contains(testAQLText(t, sequential.Sequential), "OPTIONS { disableIndex: true }") || strings.Contains(testAQLText(t, sequential.Sequential), "indexHint:") { + t.Fatalf("sequential alternative did not disable only its direct root index:\n%s", testAQLText(t, sequential.Sequential)) } for _, scope := range []string{ "root.project == @project", "root.dataset_generation == @dataset_generation", "root.auth_resource_path IN @auth_resource_paths", "FILTER root_scope_allowed == @scope_allowed", } { - if !strings.Contains(sequential.ScopeCountQuery, scope) { - t.Fatalf("scope count omitted canonical filter %q:\n%s", scope, sequential.ScopeCountQuery) + if !strings.Contains(testAQLText(t, sequential.ScopeCount), scope) { + t.Fatalf("scope count omitted canonical filter %q:\n%s", scope, testAQLText(t, sequential.ScopeCount)) } } - if !strings.Contains(sequential.ScopeCountQuery, "COLLECT WITH COUNT INTO") || - !strings.Contains(sequential.ScopeCountQuery, "RETURN {count:") || - strings.Contains(sequential.ScopeCountQuery, "payload.note") || strings.Contains(sequential.ScopeCountQuery, "construction_group") { - t.Fatalf("scope count should return a named count without projecting or grouping source values:\n%s", sequential.ScopeCountQuery) + if !strings.Contains(testAQLText(t, sequential.ScopeCount), "COLLECT WITH COUNT INTO") || + !strings.Contains(testAQLText(t, sequential.ScopeCount), "RETURN {count:") || + strings.Contains(testAQLText(t, sequential.ScopeCount), "payload.note") || strings.Contains(testAQLText(t, sequential.ScopeCount), "construction_group") { + t.Fatalf("scope count should return a named count without projecting or grouping source values:\n%s", testAQLText(t, sequential.ScopeCount)) } for _, bind := range []string{"project", "dataset_generation", "auth_resource_paths"} { - if _, ok := sequential.ScopeCountBindVars[bind]; !ok { + if _, ok := testAQLBinds(t, sequential.ScopeCount)[bind]; !ok { t.Fatalf("scope count omitted %q bind", bind) } } existingName := "loom_pivot_preview_existing_compatible" - queryWithExistingIndex, existingBinds, err := RenderPreviewPhysicalPlanWithRootIndexHint(query.PhysicalPlan, existingName) + queryWithExistingIndexCommand, err := RenderPreviewPhysicalPlanWithRootIndexHint(query.PhysicalPlan, existingName) if err != nil { t.Fatalf("render Group preview with existing compatible index: %v", err) } + queryWithExistingIndex, existingBinds := testAQLView(t, queryWithExistingIndexCommand) if !strings.Contains(queryWithExistingIndex, "OPTIONS { indexHint: \""+existingName+"\", forceIndexHint: false }") || strings.Contains(queryWithExistingIndex, "forceIndexHint: true") { t.Fatalf("existing index hint is not non-forcing:\n%s", queryWithExistingIndex) } - if !reflect.DeepEqual(existingBinds, query.BindVars) { - t.Fatalf("existing-index render changed query binds: got=%v want=%v", existingBinds, query.BindVars) + if !reflect.DeepEqual(existingBinds, testAQLBinds(t, query.Command)) { + t.Fatalf("existing-index render changed query binds: got=%v want=%v", existingBinds, testAQLBinds(t, query.Command)) } repeated := compileConstructionOutputQuery(t, constructionGroupRowValuesOutput(), "group-covering-project", "group-covering-generation") @@ -543,14 +542,14 @@ func TestConstructionGroupPreviewUsesProjectionFieldsAndOneRootHint(t *testing.T fullOutput := lowerConstructionOutput(t, constructionGroupRowValuesOutput(), recipe.RuntimeBindings{ Project: "group-covering-project", DatasetGeneration: "group-covering-generation", }) - full, err := CompileRecipeOutputWithPolicy(fullOutput, recipe.RuntimeBindings{ + full, err := CompileRecipeOutput(ArangoTarget(), fullOutput, recipe.RuntimeBindings{ Project: "group-covering-project", DatasetGeneration: "group-covering-generation", }, 0, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if full.PreviewCoveringIndex != nil || strings.Contains(full.Query, "indexHint:") { - t.Fatalf("full Group execution received preview index metadata or a hint: %+v\n%s", full.PreviewCoveringIndex, full.Query) + if full.PreviewCoveringIndex != nil || strings.Contains(testAQLText(t, full.Command), "indexHint:") { + t.Fatalf("full Group execution received preview index metadata or a hint: %+v\n%s", full.PreviewCoveringIndex, testAQLText(t, full.Command)) } } @@ -563,7 +562,7 @@ func TestConstructionGroupScanAlternativeRejectsExtraSourceFilters(t *testing.T) operations := append([]ir.PhysicalOperation(nil), output.Plan.Operations[:len(output.Plan.Operations)-1]...) operations = append(operations, output.Plan.Operations[1], output.Plan.Operations[len(output.Plan.Operations)-1]) output.Plan.Operations = operations - query, err := CompileRecipeOutputWithPolicy(output, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), output, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -587,7 +586,7 @@ func compileConstructionPivotPreview(t *testing.T, output recipe.Output, limit i if err != nil { t.Fatal(err) } - queries, err := CompileResolvedRecipePlanWithPolicy(resolved, limit, ir.DefaultPhysicalOptimizationPolicy()) + queries, err := CompileResolvedRecipePlan(ArangoTarget(), resolved, limit, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } diff --git a/internal/dataframe/compiler/construction_group_expand_arango_test.go b/internal/dataframe/compiler/construction_group_expand_arango_test.go index 08dbef327..54badba8d 100644 --- a/internal/dataframe/compiler/construction_group_expand_arango_test.go +++ b/internal/dataframe/compiler/construction_group_expand_arango_test.go @@ -12,11 +12,11 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -128,7 +128,7 @@ func TestConstructionGroupRowLineagePagesScopedContributorsAgainstArango(t *test AuthResourcePaths: []string{"Observation/visible"}, } output := lowerConstructionOutput(t, constructionMissingKeyPolicyOutput(recipe.ConstructionGroupMissingKeyGroup), bindings) - preview, err := CompileRecipeOutputWithPolicy(output, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + preview, err := CompileRecipeOutput(ArangoTarget(), output, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -143,7 +143,7 @@ func TestConstructionGroupRowLineagePagesScopedContributorsAgainstArango(t *test compilePage := func(rowID string, offset int) CompiledRowLineageQuery { t.Helper() - query, compileErr := CompileRowLineageOutput(output, rowID, offset, 1, ir.DefaultPhysicalOptimizationPolicy()) + query, compileErr := CompileRowLineageOutput(ArangoTarget(), output, rowID, offset, 1, ir.DefaultPhysicalOptimizationPolicy()) if compileErr != nil { t.Fatalf("compile Group row-lineage page at %d: %v", offset, compileErr) } @@ -151,8 +151,8 @@ func TestConstructionGroupRowLineagePagesScopedContributorsAgainstArango(t *test "root.project == @project", "root.dataset_generation == @dataset_generation", "root.auth_resource_path IN @auth_resource_paths", } { - if !strings.Contains(query.Query, want) { - t.Errorf("Group row-lineage query lost source scope %q:\n%s", want, query.Query) + if !strings.Contains(testAQLText(t, query.Command), want) { + t.Errorf("Group row-lineage query lost source scope %q:\n%s", want, testAQLText(t, query.Command)) } } return query @@ -250,7 +250,7 @@ func TestConstructionGroupFilterRowLineagePagesScopedContributorsAgainstArango(t AuthResourcePaths: []string{"Observation/visible"}, } unfiltered := lowerConstructionOutput(t, constructionMissingKeyPolicyOutput(recipe.ConstructionGroupMissingKeyGroup), bindings) - unfilteredPreview, err := CompileRecipeOutputWithPolicy(unfiltered, bindings, 200, ir.DefaultPhysicalOptimizationPolicy()) + unfilteredPreview, err := CompileRecipeOutput(ArangoTarget(), unfiltered, bindings, 200, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -266,7 +266,7 @@ func TestConstructionGroupFilterRowLineagePagesScopedContributorsAgainstArango(t } output := lowerConstructionOutput(t, constructionGroupFilterLineageOutput(104, 103), bindings) - preview, err := CompileRecipeOutputWithPolicy(output, bindings, 200, ir.DefaultPhysicalOptimizationPolicy()) + preview, err := CompileRecipeOutput(ArangoTarget(), output, bindings, 200, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -281,7 +281,7 @@ func TestConstructionGroupFilterRowLineagePagesScopedContributorsAgainstArango(t compilePage := func(compiled lower.CompiledRecipeOutput, id string, offset int) CompiledRowLineageQuery { t.Helper() - query, compileErr := CompileRowLineageOutput(compiled, id, offset, 50, ir.DefaultPhysicalOptimizationPolicy()) + query, compileErr := CompileRowLineageOutput(ArangoTarget(), compiled, id, offset, 50, ir.DefaultPhysicalOptimizationPolicy()) if compileErr != nil { t.Fatalf("compile filtered Group row-lineage page at %d: %v", offset, compileErr) } @@ -289,8 +289,8 @@ func TestConstructionGroupFilterRowLineagePagesScopedContributorsAgainstArango(t "root.project == @project", "root.dataset_generation == @dataset_generation", "root.auth_resource_path IN @auth_resource_paths", "LIMIT @row_lineage_offset, @row_lineage_fetch_limit", } { - if !strings.Contains(query.Query, want) { - t.Errorf("filtered Group row-lineage query lost required scope/page clause %q:\n%s", want, query.Query) + if !strings.Contains(testAQLText(t, query.Command), want) { + t.Errorf("filtered Group row-lineage query lost required scope/page clause %q:\n%s", want, testAQLText(t, query.Command)) } } return query @@ -350,28 +350,28 @@ func TestConstructionGroupFilterRowLineagePagesScopedContributorsAgainstArango(t func TestConstructionGroupCountRowsOnlyQueryAvoidsInputRowBuffers(t *testing.T) { query := compileConstructionOutputQuery(t, constructionCountRowsOnlyOutput(), "construction-count-rows-only", "generation-count-rows-only") - if !strings.Contains(query.Query, "COLLECT WITH COUNT INTO ") { - t.Fatalf("count-only query does not use streaming row count:\n%s", query.Query) + if !strings.Contains(testAQLText(t, query.Command), "COLLECT WITH COUNT INTO ") { + t.Fatalf("count-only query does not use streaming row count:\n%s", testAQLText(t, query.Command)) } for _, bufferedRows := range []string{ "construction_group_input_rows", "construction_group_all_rows", "LET __loom_construction_source_projection = (", } { - if strings.Contains(query.Query, bufferedRows) { - t.Errorf("keyless count-only query retains %q:\n%s", bufferedRows, query.Query) + if strings.Contains(testAQLText(t, query.Command), bufferedRows) { + t.Errorf("keyless count-only query retains %q:\n%s", bufferedRows, testAQLText(t, query.Command)) } } - if strings.Contains(query.Query, "SORT root.id ASC") { - t.Fatalf("keyless count-only query sorts source rows before counting:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "SORT root.id ASC") { + t.Fatalf("keyless count-only query sorts source rows before counting:\n%s", testAQLText(t, query.Command)) } - collectAt := strings.Index(query.Query, "COLLECT WITH COUNT INTO") - limitAt := strings.LastIndex(query.Query, "LIMIT @limit") + collectAt := strings.Index(testAQLText(t, query.Command), "COLLECT WITH COUNT INTO") + limitAt := strings.LastIndex(testAQLText(t, query.Command), "LIMIT @limit") if collectAt < 0 || limitAt < collectAt { - t.Fatalf("preview bound is not applied after the exact keyless count:\n%s", query.Query) + t.Fatalf("preview bound is not applied after the exact keyless count:\n%s", testAQLText(t, query.Command)) } - if strings.Contains(query.Query[:collectAt], "SORT ") { - t.Fatalf("keyless count query sorts source rows before the count:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command)[:collectAt], "SORT ") { + t.Fatalf("keyless count query sorts source rows before the count:\n%s", testAQLText(t, query.Command)) } if query.PartialValidation { t.Fatal("exact keyless count was marked as partial validation") @@ -386,30 +386,30 @@ func TestConstructionGroupCountRowsOnlyQueryAvoidsInputRowBuffers(t *testing.T) "root.dataset_generation == @dataset_generation", "root.auth_resource_path IN @auth_resource_paths", } { - if !strings.Contains(restricted.Query, scopedPredicate) { - t.Errorf("inlined count source lost canonical scope predicate %q:\n%s", scopedPredicate, restricted.Query) + if !strings.Contains(testAQLText(t, restricted.Command), scopedPredicate) { + t.Errorf("inlined count source lost canonical scope predicate %q:\n%s", scopedPredicate, testAQLText(t, restricted.Command)) } } - if restricted.BindVars["auth_resource_paths_unrestricted"] != false { - t.Fatalf("inlined count source bypassed restricted authorization: %#v", restricted.BindVars) + if testAQLBinds(t, restricted.Command)["auth_resource_paths_unrestricted"] != false { + t.Fatalf("inlined count source bypassed restricted authorization: %#v", testAQLBinds(t, restricted.Command)) } - if paths, ok := restricted.BindVars["auth_resource_paths"].([]string); !ok || len(paths) != 1 || paths[0] != "Specimen/visible" { - t.Fatalf("inlined count source changed the authorized paths: %#v", restricted.BindVars["auth_resource_paths"]) + if paths, ok := testAQLBinds(t, restricted.Command)["auth_resource_paths"].([]string); !ok || len(paths) != 1 || paths[0] != "Specimen/visible" { + t.Fatalf("inlined count source changed the authorized paths: %#v", testAQLBinds(t, restricted.Command)["auth_resource_paths"]) } keyedQuery := compileConstructionOutputQuery(t, constructionMissingKeyPolicyOutput(recipe.ConstructionGroupMissingKeyError), "construction-count-rows-only", "generation-count-rows-only") - if !strings.Contains(keyedQuery.Query, "WITH COUNT INTO ") || !strings.Contains(keyedQuery.Query, "CONSTRUCTION_GROUP_MISSING_KEY") { - t.Fatalf("keyed count query did not retain its missing-key error behavior:\n%s", keyedQuery.Query) + if !strings.Contains(testAQLText(t, keyedQuery.Command), "WITH COUNT INTO ") || !strings.Contains(testAQLText(t, keyedQuery.Command), "CONSTRUCTION_GROUP_MISSING_KEY") { + t.Fatalf("keyed count query did not retain its missing-key error behavior:\n%s", testAQLText(t, keyedQuery.Command)) } keyed := constructionMissingKeyPolicyOutput(recipe.ConstructionGroupMissingKeyGroup) keyed.Fields = append(keyed.Fields, recipe.Field{Name: "amount", ColumnID: "amount_id", Expr: recipe.Expression{Select: "root.valueQuantity.value"}}) keyed.Construction.SourceColumns = append(keyed.Construction.SourceColumns, recipe.StageColumn{ID: "amount_id", Name: "amount", Type: "decimal"}) pruned := compileConstructionOutputQuery(t, keyed, "construction-count-rows-only", "generation-count-rows-only") - if !strings.Contains(pruned.Query, "root.payload.status") || strings.Contains(pruned.Query, "root.payload.valueQuantity.value") { - t.Fatalf("keyed count query projected an unused source field:\n%s", pruned.Query) + if !strings.Contains(testAQLText(t, pruned.Command), "root.payload.status") || strings.Contains(testAQLText(t, pruned.Command), "root.payload.valueQuantity.value") { + t.Fatalf("keyed count query projected an unused source field:\n%s", testAQLText(t, pruned.Command)) } - if !strings.Contains(pruned.Query, "SORT __loom_construction_group_key_1 ASC") || strings.Contains(pruned.Query, "SORT __loom_construction_final_row.__loom_row_id") { - t.Fatalf("group query did not preserve its key order without a second final sort:\n%s", pruned.Query) + if !strings.Contains(testAQLText(t, pruned.Command), "SORT __loom_construction_group_key_1 ASC") || strings.Contains(testAQLText(t, pruned.Command), "SORT __loom_construction_final_row.__loom_row_id") { + t.Fatalf("group query did not preserve its key order without a second final sort:\n%s", testAQLText(t, pruned.Command)) } } @@ -425,8 +425,8 @@ func TestConstructionGroupAllRowValuesUsesStreamingTypedAggregates(t *testing.T) "SORTED_UNIQUE((FOR ", "CONSTRUCTION_GROUP_KEY_TYPE_MISMATCH", } { - if !strings.Contains(query.Query, want) { - t.Errorf("streaming Group query missing %q:\n%s", want, query.Query) + if !strings.Contains(testAQLText(t, query.Command), want) { + t.Errorf("streaming Group query missing %q:\n%s", want, testAQLText(t, query.Command)) } } for _, buffered := range []string{ @@ -435,40 +435,40 @@ func TestConstructionGroupAllRowValuesUsesStreamingTypedAggregates(t *testing.T) " INTO __loom_construction_group_rows_", "LENGTH((FOR ", } { - if strings.Contains(query.Query, buffered) { - t.Errorf("streaming Group query retained contributor materialization %q:\n%s", buffered, query.Query) + if strings.Contains(testAQLText(t, query.Command), buffered) { + t.Errorf("streaming Group query retained contributor materialization %q:\n%s", buffered, testAQLText(t, query.Command)) } } - projectionAt := strings.Index(query.Query, "LET __loom_construction_input_1 = {") - collectAt := strings.Index(query.Query, "COLLECT __loom_construction_group_key_") + projectionAt := strings.Index(testAQLText(t, query.Command), "LET __loom_construction_input_1 = {") + collectAt := strings.Index(testAQLText(t, query.Command), "COLLECT __loom_construction_group_key_") if projectionAt < 0 || collectAt < projectionAt { - t.Fatalf("direct source projection was not assigned in the Group scope before COLLECT:\n%s", query.Query) + t.Fatalf("direct source projection was not assigned in the Group scope before COLLECT:\n%s", testAQLText(t, query.Command)) } for _, scopeClause := range []string{ "FILTER root.project == @project", "FILTER root.dataset_generation == @dataset_generation", "FILTER root_scope_allowed == @scope_allowed", } { - if !strings.Contains(query.Query, scopeClause) { - t.Errorf("streamed Group source lost scope clause %q:\n%s", scopeClause, query.Query) + if !strings.Contains(testAQLText(t, query.Command), scopeClause) { + t.Errorf("streamed Group source lost scope clause %q:\n%s", scopeClause, testAQLText(t, query.Command)) } } - if strings.Contains(query.Query, "FOR __loom_selector_") || !strings.Contains(query.Query, "[* FILTER CURRENT.text != null RETURN CURRENT.text]") { - t.Fatalf("repeated source selector did not use a typed inline array expansion:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "FOR __loom_selector_") || !strings.Contains(testAQLText(t, query.Command), "[* FILTER CURRENT.text != null RETURN CURRENT.text]") { + t.Fatalf("repeated source selector did not use a typed inline array expansion:\n%s", testAQLText(t, query.Command)) } oneQuery := compileConstructionOutputQuery(t, constructionGroupOneRowValueOutput(), "construction-group-one-row-value", "generation-group-one-row-value") - if strings.Contains(oneQuery.Query, "construction_source_projection = (") || !strings.Contains(oneQuery.Query, "CONSTRUCTION_ROW_VALUE_MULTIPLE_VALUES") { - t.Fatalf("ONE row values did not use direct-source aggregation with its multiple-value check:\n%s", oneQuery.Query) + if strings.Contains(testAQLText(t, oneQuery.Command), "construction_source_projection = (") || !strings.Contains(testAQLText(t, oneQuery.Command), "CONSTRUCTION_ROW_VALUE_MULTIPLE_VALUES") { + t.Fatalf("ONE row values did not use direct-source aggregation with its multiple-value check:\n%s", testAQLText(t, oneQuery.Command)) } } func TestConstructionGroupStreamingKeepsExpandedInputStage(t *testing.T) { query := compileConstructionOutputQuery(t, constructionExpandGroupOutput(), "construction-group-expanded", "generation-group-expanded") - if !strings.Contains(query.Query, "FOR __loom_construction_input_2 IN __loom_construction_stage_1") { - t.Fatalf("Group after expansion no longer consumes the expanded stage:\n%s", query.Query) + if !strings.Contains(testAQLText(t, query.Command), "FOR __loom_construction_input_2 IN __loom_construction_stage_1") { + t.Fatalf("Group after expansion no longer consumes the expanded stage:\n%s", testAQLText(t, query.Command)) } - if !strings.Contains(query.Query, "construction_source_projection = (") { - t.Fatalf("multi-stage Group bypassed its staged source input:\n%s", query.Query) + if !strings.Contains(testAQLText(t, query.Command), "construction_source_projection = (") { + t.Fatalf("multi-stage Group bypassed its staged source input:\n%s", testAQLText(t, query.Command)) } } @@ -569,16 +569,14 @@ func TestConstructionGroupOneRowValuePreservesDistinctAndErrorSemanticsAgainstAr {"id": "one-multiple-a", "status": "active", "note": []any{map[string]any{"text": "blue"}}}, {"id": "one-multiple-b", "status": "active", "note": []any{map[string]any{"text": "red"}}}, }) - if err := executeConstructionOutputError(t, ctx, client, constructionGroupOneRowValueOutput(), multipleProject, generation); err == nil || !strings.Contains(err.Error(), "CONSTRUCTION_ROW_VALUE_MULTIPLE_VALUES") { - t.Fatalf("Group ONE accepted multiple distinct values, error = %v", err) - } + err := executeConstructionOutputError(t, ctx, client, constructionGroupOneRowValueOutput(), multipleProject, generation) + requireStableRuntimeCode(t, err, "CONSTRUCTION_ROW_VALUE_MULTIPLE_VALUES") insertConstructionReshapeRows(t, ctx, client, invalidTypeProject, generation, []map[string]any{ {"id": "one-invalid-type", "status": "active", "note": []any{map[string]any{"text": 7}}}, }) - if err := executeConstructionOutputError(t, ctx, client, constructionGroupOneRowValueOutput(), invalidTypeProject, generation); err == nil || !strings.Contains(err.Error(), "CONSTRUCTION_ROW_VALUE_TYPE_MISMATCH") { - t.Fatalf("Group ONE accepted an invalid value type, error = %v", err) - } + err = executeConstructionOutputError(t, ctx, client, constructionGroupOneRowValueOutput(), invalidTypeProject, generation) + requireStableRuntimeCode(t, err, "CONSTRUCTION_ROW_VALUE_TYPE_MISMATCH") } func TestConstructionExpandPreserveAndGroupNullKeyRowsAgainstArango(t *testing.T) { @@ -751,11 +749,11 @@ func TestConstructionGroupMissingKeyPoliciesAgainstArango(t *testing.T) { if err != nil { t.Fatalf("compile ERROR recipe: %v", err) } - query, err := CompileRecipeOutputWithPolicy(compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile ERROR query: %v", err) } - if err := client.QueryRows(ctx, query.Query, 500, query.BindVars, func(map[string]any) error { return nil }); err == nil { + if err := client.QueryRows(ctx, testAQLText(t, query.Command), 500, testAQLBinds(t, query.Command), func(map[string]any) error { return nil }); err == nil { t.Fatal("ERROR policy accepted absent or null group keys") } } @@ -913,7 +911,7 @@ func constructionExpandGroupOutput() recipe.Output { return output } -func openConstructionReshapeArango(t *testing.T) (context.Context, *store.Client) { +func openConstructionReshapeArango(t *testing.T) (context.Context, *arangofactory.Client) { t.Helper() url, database := os.Getenv("LOOM_TEST_ARANGO_URL"), os.Getenv("LOOM_TEST_ARANGO_DATABASE") if url == "" || database == "" { @@ -921,18 +919,19 @@ func openConstructionReshapeArango(t *testing.T) (context.Context, *store.Client } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) t.Cleanup(cancel) - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client t.Cleanup(func() { _ = client.Close(context.Background()) }) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Observation"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Observation"}}}); err != nil { t.Fatal(err) } return ctx, client } -func insertConstructionReshapeRows(t *testing.T, ctx context.Context, client *store.Client, project, generation string, payloads []map[string]any) { +func insertConstructionReshapeRows(t *testing.T, ctx context.Context, client *arangofactory.Client, project, generation string, payloads []map[string]any) { t.Helper() documents := make([]json.RawMessage, 0, len(payloads)) for _, payload := range payloads { @@ -955,16 +954,16 @@ func insertConstructionReshapeRows(t *testing.T, ctx context.Context, client *st } } -func executeConstructionOutput(t *testing.T, ctx context.Context, client *store.Client, output recipe.Output, project, generation string) []map[string]any { +func executeConstructionOutput(t *testing.T, ctx context.Context, client *arangofactory.Client, output recipe.Output, project, generation string) []map[string]any { t.Helper() query := compileConstructionOutputQuery(t, output, project, generation) return executeReshapeOracleQuery(t, ctx, client, query) } -func executeConstructionOutputError(t *testing.T, ctx context.Context, client *store.Client, output recipe.Output, project, generation string) error { +func executeConstructionOutputError(t *testing.T, ctx context.Context, client *arangofactory.Client, output recipe.Output, project, generation string) error { t.Helper() query := compileConstructionOutputQuery(t, output, project, generation) - return client.QueryRows(ctx, query.Query, 500, query.BindVars, func(map[string]any) error { return nil }) + return client.QueryRows(ctx, testAQLText(t, query.Command), 500, testAQLBinds(t, query.Command), func(map[string]any) error { return nil }) } func compileConstructionOutputQuery(t *testing.T, output recipe.Output, project, generation string) CompiledQuery { @@ -976,7 +975,7 @@ func compileConstructionOutputQueryWithBindings(t *testing.T, output recipe.Outp t.Helper() bindings.Project, bindings.DatasetGeneration = project, generation compiledOutput := lowerConstructionOutput(t, output, bindings) - query, err := CompileRecipeOutputWithPolicy(compiledOutput, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiledOutput, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile construction query: %v", err) } @@ -1006,14 +1005,14 @@ func TestPreviewSourceIdentityStaysPrivateThroughConstructionExpand(t *testing.T output := constructionExpandOutput() ordinary := compileConstructionOutputQuery(t, output, "source-id-project", "source-id-generation") preview := compileConstructionOutputQueryWithBindings(t, output, "source-id-project", "source-id-generation", recipe.RuntimeBindings{IncludeSourceIdentity: true}) - if strings.Contains(ordinary.Query, ir.PreviewSourceResourceIDColumn) { - t.Fatalf("ordinary compilation gained a source ID projection:\n%s", ordinary.Query) + if strings.Contains(testAQLText(t, ordinary.Command), ir.PreviewSourceResourceIDColumn) { + t.Fatalf("ordinary compilation gained a source ID projection:\n%s", testAQLText(t, ordinary.Command)) } - if !strings.Contains(preview.Query, ir.PreviewSourceResourceIDColumn) { - t.Fatalf("preview compilation did not carry the source ID through EXPAND:\n%s", preview.Query) + if !strings.Contains(testAQLText(t, preview.Command), ir.PreviewSourceResourceIDColumn) { + t.Fatalf("preview compilation did not carry the source ID through EXPAND:\n%s", testAQLText(t, preview.Command)) } - if !strings.Contains(preview.Query, "root.id") { - t.Fatalf("preview projection did not read the root FHIR id in scope:\n%s", preview.Query) + if !strings.Contains(testAQLText(t, preview.Command), "root.id") { + t.Fatalf("preview projection did not read the root FHIR id in scope:\n%s", testAQLText(t, preview.Command)) } if containsString(preview.PublicColumns, ir.PreviewSourceResourceIDColumn) { t.Fatalf("private source ID entered public columns: %#v", preview.PublicColumns) @@ -1065,7 +1064,7 @@ func TestPreviewSourceIdentityStaysPrivateThroughConstructionExpand(t *testing.T stageInputProjections[index] = projectionNames(stage.InputProjections) stageOutputProjections[index] = projectionNames(stage.OutputProjections) } - if _, err := CompileRecipeOutputWithPolicy(compiledOutput, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()); err != nil { + if _, err := CompileRecipeOutput(ArangoTarget(), compiledOutput, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()); err != nil { t.Fatalf("compile query-only clone with source identity: %v", err) } for index, operation := range compiledOutput.Plan.Operations { diff --git a/internal/dataframe/compiler/construction_group_preview_liveness_arango_test.go b/internal/dataframe/compiler/construction_group_preview_liveness_arango_test.go index fcc9d1793..e0c6a4e17 100644 --- a/internal/dataframe/compiler/construction_group_preview_liveness_arango_test.go +++ b/internal/dataframe/compiler/construction_group_preview_liveness_arango_test.go @@ -11,17 +11,17 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/render/aql" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/fhir/schema" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) func TestSourceGroupPruningKeepsCorrelatedPivotALLValuesAgainstArango(t *testing.T) { ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Condition"}, {Name: "Observation"}, {Name: "Specimen"}, {Name: "Patient"}, {Name: "ResearchSubject"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { @@ -160,7 +160,7 @@ func TestSourceGroupPruningKeepsCorrelatedPivotALLValuesAgainstArango(t *testing AuthScopeMode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{"/allowed"}, } compiled := lowerConstructionOutput(t, output, bindings) - preview, err := CompileRecipeOutputWithPolicy(compiled, bindings, 1, ir.DefaultPhysicalOptimizationPolicy()) + preview, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 1, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -169,34 +169,34 @@ func TestSourceGroupPruningKeepsCorrelatedPivotALLValuesAgainstArango(t *testing sequence.Stages[0].Group == nil || len(sequence.Stages[0].Group.RowValues) == 0 { t.Fatalf("limit-1 output is not the exact typed terminal GROUP row-value plan: %#v", preview.PhysicalPlan.StageSequence) } - assertTumorGroupLivenessPreviewFrontier(t, preview.Query) + assertTumorGroupLivenessPreviewFrontier(t, testAQLText(t, preview.Command)) - if strings.Contains(preview.Query, "child_set_3") || strings.Contains(preview.Query, "child_set_4") || strings.Contains(preview.Query, "subject_Patient") { - t.Fatalf("unselected optional Patient→ResearchSubject path survived source projection liveness:\n%s", preview.Query) + if strings.Contains(testAQLText(t, preview.Command), "child_set_3") || strings.Contains(testAQLText(t, preview.Command), "child_set_4") || strings.Contains(testAQLText(t, preview.Command), "subject_Patient") { + t.Fatalf("unselected optional Patient→ResearchSubject path survived source projection liveness:\n%s", testAQLText(t, preview.Command)) } for _, retained := range []string{ "child_set_1_label", "child_set_2_label", "child_set_1_scope_allowed", "child_set_2_scope_allowed", "auth_resource_paths", "dataset_generation", "FILTER __correlation_version == @", } { - if !strings.Contains(preview.Query, retained) && !strings.Contains(fmt.Sprint(preview.BindVars), retained) { + if !strings.Contains(testAQLText(t, preview.Command), retained) && !strings.Contains(fmt.Sprint(testAQLBinds(t, preview.Command)), retained) { t.Errorf("live authorized Correlated Pivot query lost %q", retained) } } versionFilter := "FILTER __correlation_version == @" - versionFilterStart := strings.Index(preview.Query, versionFilter) + versionFilterStart := strings.Index(testAQLText(t, preview.Command), versionFilter) if versionFilterStart < 0 { t.Fatal("compiled correlated Pivot query omitted exact Coding.version matching") } - versionBindKey := strings.FieldsFunc(preview.Query[versionFilterStart+len(versionFilter):], func(r rune) bool { + versionBindKey := strings.FieldsFunc(testAQLText(t, preview.Command)[versionFilterStart+len(versionFilter):], func(r rune) bool { return r == '\n' || r == '\r' || r == '\t' || r == ' ' || r == ')' || r == ';' }) if len(versionBindKey) == 0 { - t.Fatalf("compiled version filter has no bind key: %s", preview.Query[versionFilterStart:]) + t.Fatalf("compiled version filter has no bind key: %s", testAQLText(t, preview.Command)[versionFilterStart:]) } - if got, ok := preview.BindVars[versionBindKey[0]]; !ok || got != "" { + if got, ok := testAQLBinds(t, preview.Command)[versionBindKey[0]]; !ok || got != "" { t.Fatalf("versionless Tumor key bind %q = %#v (present=%t), want exact empty version", versionBindKey[0], got, ok) } - if got := preview.BindVars["auth_resource_paths"]; !reflect.DeepEqual(got, []string{"/allowed"}) { + if got := testAQLBinds(t, preview.Command)["auth_resource_paths"]; !reflect.DeepEqual(got, []string{"/allowed"}) { t.Fatalf("authorization paths bind = %#v, want restricted /allowed scope", got) } rows := executeReshapeOracleQuery(t, ctx, client, preview) @@ -208,9 +208,6 @@ func TestSourceGroupPruningKeepsCorrelatedPivotALLValuesAgainstArango(t *testing t.Fatalf("limit-1 ALL values = %#v, want %#v (late value must survive post-GROUP limit)", got, want) } - // Render the same finalized typed GROUP plan without its preview window. - // This is the generic full-source reference for the optimized root-key - // frontier query above, independent of the fixture's hand-written oracle. genericPlan := clonePhysicalPlan(preview.PhysicalPlan) genericPlan.StageSequence.PreviewLimitBindKey = "" genericRendered, err := aql.RenderPhysicalPlan(genericPlan) @@ -219,8 +216,7 @@ func TestSourceGroupPruningKeepsCorrelatedPivotALLValuesAgainstArango(t *testing } allRows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{ PhysicalPlan: genericPlan, - Query: genericRendered.Query, - BindVars: genericRendered.BindVars, + Command: testAQLCommand(genericRendered.Query, genericRendered.BindVars), }) t.Logf("compiled all-groups AQL rows: %#v", allRows) if len(allRows) != 2 { diff --git a/internal/dataframe/compiler/construction_group_related_summary_arango_test.go b/internal/dataframe/compiler/construction_group_related_summary_arango_test.go index 7ef91569d..1051053e4 100644 --- a/internal/dataframe/compiler/construction_group_related_summary_arango_test.go +++ b/internal/dataframe/compiler/construction_group_related_summary_arango_test.go @@ -8,8 +8,8 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/recipe" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -33,7 +33,7 @@ func TestGroupedRelatedSourceCompilesAfterReshape(t *testing.T) { func TestGroupedRelatedSourceDistinctRecordsAgainstArango(t *testing.T) { ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -107,7 +107,7 @@ func TestGroupedRelatedSourceDistinctRecordsAgainstArango(t *testing.T) { query := compileConstructionOutputQuery(t, output, project, generation) t.Cleanup(func() { if t.Failed() { - t.Logf("query:\n%s\nbindings: %#v", query.Query, query.BindVars) + t.Logf("query:\n%s\nbindings: %#v", testAQLText(t, query.Command), testAQLBinds(t, query.Command)) } }) rows := executeReshapeOracleQuery(t, ctx, client, query) diff --git a/internal/dataframe/compiler/construction_group_source_streaming_test.go b/internal/dataframe/compiler/construction_group_source_streaming_test.go index 8ff4dd276..9809ec875 100644 --- a/internal/dataframe/compiler/construction_group_source_streaming_test.go +++ b/internal/dataframe/compiler/construction_group_source_streaming_test.go @@ -64,9 +64,9 @@ func TestRelatedUnnestCountRowsStreamsSourceProjection(t *testing.T) { t.Fatalf("source unnest = %#v, want one relationship owner route", unnest) } - query, err := CompileRecipeOutputWithPolicy(compiledOutput, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiledOutput, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { - t.Fatalf("CompileRecipeOutputWithPolicy() error = %v", err) + t.Fatalf("CompileRecipeOutput(ArangoTarget(), ) error = %v", err) } for _, want := range []string{ "FILTER root.project == @project", @@ -86,34 +86,34 @@ func TestRelatedUnnestCountRowsStreamsSourceProjection(t *testing.T) { "FILTER ASSERT(__loom_construction_group_key_1 == null OR TYPENAME(__loom_construction_group_key_1) == @", "FILTER expansion_route_1_scope_allowed == @scope_allowed", } { - if !strings.Contains(query.Query, want) { - t.Errorf("streamed related COUNT_ROWS query missing %q:\n%s", want, query.Query) + if !strings.Contains(testAQLText(t, query.Command), want) { + t.Errorf("streamed related COUNT_ROWS query missing %q:\n%s", want, testAQLText(t, query.Command)) } } - if got := query.BindVars["__loom_physical_construction_construction_group_key_type_0"]; got != "string" { + if got := testAQLBinds(t, query.Command)["__loom_physical_construction_construction_group_key_type_0"]; got != "string" { t.Fatalf("group key type bind = %#v, want string", got) } - if strings.Contains(query.Query, "LET __loom_construction_source_projection = (") { - t.Fatalf("related COUNT_ROWS query retained the projected source array:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "LET __loom_construction_source_projection = (") { + t.Fatalf("related COUNT_ROWS query retained the projected source array:\n%s", testAQLText(t, query.Command)) } - if strings.Contains(query.Query, "FOR __loom_construction_input_1 IN __loom_construction_source_projection") { - t.Fatalf("GROUP re-read materialized source rows:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "FOR __loom_construction_input_1 IN __loom_construction_source_projection") { + t.Fatalf("GROUP re-read materialized source rows:\n%s", testAQLText(t, query.Command)) } - if strings.Contains(query.Query, "FILTER __loom_construction_input_1[@__loom_physical_construction_construction_group_key_column_0] != null") { - t.Fatalf("GROUP excluded missing keys instead of grouping null values:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "FILTER __loom_construction_input_1[@__loom_physical_construction_construction_group_key_column_0] != null") { + t.Fatalf("GROUP excluded missing keys instead of grouping null values:\n%s", testAQLText(t, query.Command)) } - projectionAt := strings.Index(query.Query, "LET __loom_construction_input_1 = {") - collectAt := strings.Index(query.Query, "COLLECT __loom_construction_group_key_1 = __loom_construction_input_1[@") - if projectionAt < 0 || collectAt <= projectionAt || !strings.Contains(query.Query[projectionAt:collectAt], "__loom_expansion_node_1.payload.subject.reference") { - t.Fatalf("projected source row does not feed GROUP directly:\n%s", query.Query) + projectionAt := strings.Index(testAQLText(t, query.Command), "LET __loom_construction_input_1 = {") + collectAt := strings.Index(testAQLText(t, query.Command), "COLLECT __loom_construction_group_key_1 = __loom_construction_input_1[@") + if projectionAt < 0 || collectAt <= projectionAt || !strings.Contains(testAQLText(t, query.Command)[projectionAt:collectAt], "__loom_expansion_node_1.payload.subject.reference") { + t.Fatalf("projected source row does not feed GROUP directly:\n%s", testAQLText(t, query.Command)) } - projectionLineEnd := strings.Index(query.Query[projectionAt:], "\n") + projectionAt - if strings.TrimSpace(query.Query[projectionLineEnd:collectAt]) != "" { - t.Fatalf("GROUP is not the next operation after its scalar source projection LET:\n%s", query.Query) + projectionLineEnd := strings.Index(testAQLText(t, query.Command)[projectionAt:], "\n") + projectionAt + if strings.TrimSpace(testAQLText(t, query.Command)[projectionLineEnd:collectAt]) != "" { + t.Fatalf("GROUP is not the next operation after its scalar source projection LET:\n%s", testAQLText(t, query.Command)) } - unnestAt := strings.Index(query.Query, "FOR __loom_physical_unnest_index_0 IN") - traversalAt := strings.Index(query.Query, "FOR __loom_expansion_edge_1 IN") + unnestAt := strings.Index(testAQLText(t, query.Command), "FOR __loom_physical_unnest_index_0 IN") + traversalAt := strings.Index(testAQLText(t, query.Command), "FOR __loom_expansion_edge_1 IN") if traversalAt < 0 || unnestAt <= traversalAt || collectAt <= unnestAt { - t.Fatalf("relationship and UNNEST loops must remain before COUNT_ROWS COLLECT:\n%s", query.Query) + t.Fatalf("relationship and UNNEST loops must remain before COUNT_ROWS COLLECT:\n%s", testAQLText(t, query.Command)) } } diff --git a/internal/dataframe/compiler/construction_pivot_lineage_arango_test.go b/internal/dataframe/compiler/construction_pivot_lineage_arango_test.go index 339963b1c..934bdfab2 100644 --- a/internal/dataframe/compiler/construction_pivot_lineage_arango_test.go +++ b/internal/dataframe/compiler/construction_pivot_lineage_arango_test.go @@ -67,23 +67,20 @@ func TestConstructionPivotRowLineageFailsWhenAnotherGroupViolatesGlobalPolicyAga output := constructionPivotLineageOutput() pivot := output.Construction.Steps[0].Operation.Pivot pivot.DuplicatePolicy, pivot.MissingCellPolicy = test.duplicate, test.missing - if err := executeConstructionOutputError(t, ctx, client, output, project, generation); err == nil || !strings.Contains(err.Error(), test.wantErrorCode) { - t.Fatalf("full Pivot error = %v, want global error %s", err, test.wantErrorCode) - } + err := executeConstructionOutputError(t, ctx, client, output, project, generation) + requireStableRuntimeCode(t, err, test.wantErrorCode) compiledOutput := lowerConstructionOutput(t, output, recipe.RuntimeBindings{Project: project, DatasetGeneration: generation}) rowID, err := json.Marshal([]any{"GROUPED_PIVOT", "row_lineage_pivot", []any{"STRING", "wanted"}}) if err != nil { t.Fatal(err) } - lineage, err := CompileRowLineageOutput(compiledOutput, string(rowID), 0, 5, ir.DefaultPhysicalOptimizationPolicy()) + lineage, err := CompileRowLineageOutput(ArangoTarget(), compiledOutput, string(rowID), 0, 5, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile Pivot row lineage: %v", err) } - err = client.QueryRows(ctx, lineage.Query, 32, lineage.BindVars, func(map[string]any) error { return nil }) - if err == nil || !strings.Contains(err.Error(), test.wantErrorCode) { - t.Fatalf("row lineage error = %v, want global Pivot policy error %s", err, test.wantErrorCode) - } + err = client.QueryRows(ctx, testAQLText(t, lineage.Command), 32, testAQLBinds(t, lineage.Command), func(map[string]any) error { return nil }) + requireStableRuntimeCode(t, err, test.wantErrorCode) }) } } @@ -140,13 +137,13 @@ func TestConstructionPivotRowLineagePagesTypedGroupPreimageAgainstArango(t *test rowID := string(rowIDBytes) compilePage := func(requestedRowID string, offset int) CompiledRowLineageQuery { t.Helper() - query, compileErr := CompileRowLineageOutput(compiledOutput, requestedRowID, offset, 1, ir.DefaultPhysicalOptimizationPolicy()) + query, compileErr := CompileRowLineageOutput(ArangoTarget(), compiledOutput, requestedRowID, offset, 1, ir.DefaultPhysicalOptimizationPolicy()) if compileErr != nil { t.Fatalf("compile Pivot row-lineage page at %d: %v", offset, compileErr) } for _, want := range []string{"root.project == @project", "root.dataset_generation == @dataset_generation", "LIMIT @row_lineage_offset, @row_lineage_fetch_limit"} { - if !strings.Contains(query.Query, want) { - t.Errorf("Pivot row-lineage query lost source scope or bounded paging contract %q:\n%s", want, query.Query) + if !strings.Contains(testAQLText(t, query.Command), want) { + t.Errorf("Pivot row-lineage query lost source scope or bounded paging contract %q:\n%s", want, testAQLText(t, query.Command)) } } return query diff --git a/internal/dataframe/compiler/construction_preview_root_id_filter.go b/internal/dataframe/compiler/construction_preview_root_id_filter.go index 2af4aa5d8..38b1a3250 100644 --- a/internal/dataframe/compiler/construction_preview_root_id_filter.go +++ b/internal/dataframe/compiler/construction_preview_root_id_filter.go @@ -8,11 +8,6 @@ import ( "github.com/calypr/loom/internal/dataframe/recipe" ) -// withConstructionPreviewRootIDFilter adds an indexed root-id equality for a -// first-stage construction filter over the exact root.payload.id projection. -// The construction filter remains in the stage sequence as the value-level -// check. Loom ingestion stores the same unmodified FHIR id in the document id -// and payload.id fields, so this root predicate can safely narrow the scan. func withConstructionPreviewRootIDFilter(output lower.CompiledRecipeOutput, plan ir.PhysicalPlan) ir.PhysicalPlan { filter, ok := constructionPreviewRootIDFilter(output, plan) if !ok { diff --git a/internal/dataframe/compiler/construction_preview_root_id_filter_test.go b/internal/dataframe/compiler/construction_preview_root_id_filter_test.go index a204e786d..229cecd82 100644 --- a/internal/dataframe/compiler/construction_preview_root_id_filter_test.go +++ b/internal/dataframe/compiler/construction_preview_root_id_filter_test.go @@ -21,49 +21,49 @@ func TestConstructionPreviewPushesLeadingRootIDFilterAndRetainsStagePredicate(t } compiled := compileConstructionRootIDTestOutput(t, output, bindings) - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - assertConstructionRootIDFilter(t, query.Query, "Specimen", id) - if got := query.BindVars["construction_filter_value"]; got != id { + assertConstructionRootIDFilter(t, testAQLText(t, query.Command), "Specimen", id) + if got := testAQLBinds(t, query.Command)["construction_filter_value"]; got != id { t.Fatalf("filter bind = %#v, want %q", got, id) } - if query.BindVars["auth_resource_paths_unrestricted"] != false || query.BindVars["scope_allowed"] != true { - t.Fatalf("root ID fast path did not preserve the restricted auth-scope binds: %#v", query.BindVars) + if testAQLBinds(t, query.Command)["auth_resource_paths_unrestricted"] != false || testAQLBinds(t, query.Command)["scope_allowed"] != true { + t.Fatalf("root ID fast path did not preserve the restricted auth-scope binds: %#v", testAQLBinds(t, query.Command)) } - if paths, ok := query.BindVars["auth_resource_paths"].([]string); !ok || len(paths) != 1 || paths[0] != "Specimen/"+id { - t.Fatalf("root ID fast path changed auth paths: %#v", query.BindVars["auth_resource_paths"]) + if paths, ok := testAQLBinds(t, query.Command)["auth_resource_paths"].([]string); !ok || len(paths) != 1 || paths[0] != "Specimen/"+id { + t.Fatalf("root ID fast path changed auth paths: %#v", testAQLBinds(t, query.Command)["auth_resource_paths"]) } - page, err := CompileRecipeOutputPageWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - assertConstructionRootIDFilter(t, page.RootKeysQuery, "Specimen", id) - if !strings.Contains(page.RootKeysQuery, "payload.id") { - t.Fatalf("root-key query lost the exact payload ID recheck:\n%s", page.RootKeysQuery) + assertConstructionRootIDFilter(t, testAQLText(t, page.RootKeys), "Specimen", id) + if !strings.Contains(testAQLText(t, page.RootKeys), "payload.id") { + t.Fatalf("root-key query lost the exact payload ID recheck:\n%s", testAQLText(t, page.RootKeys)) } - if !strings.Contains(page.RowsQuery, "root.id") || !strings.Contains(page.RowsQuery, "@construction_filter_value") { - t.Fatalf("selected-root query lost the root seek or construction predicate:\n%s", page.RowsQuery) + if !strings.Contains(testAQLText(t, page.Rows), "root.id") || !strings.Contains(testAQLText(t, page.Rows), "@construction_filter_value") { + t.Fatalf("selected-root query lost the root seek or construction predicate:\n%s", testAQLText(t, page.Rows)) } - if !strings.Contains(page.RowsQuery, "FILTER __loom_construction_input_1.specimen_id == @construction_filter_value") { - t.Fatalf("selected-root query lost the original source-column filter:\n%s", page.RowsQuery) + if !strings.Contains(testAQLText(t, page.Rows), "FILTER __loom_construction_input_1.specimen_id == @construction_filter_value") { + t.Fatalf("selected-root query lost the original source-column filter:\n%s", testAQLText(t, page.Rows)) } - fullQuery, err := CompileRecipeOutputWithPolicy(compiled, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + fullQuery, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - assertConstructionRootIDFilterNotInjected(t, fullQuery.Query) + assertConstructionRootIDFilterNotInjected(t, testAQLText(t, fullQuery.Command)) fullBindings := bindings fullBindings.PreviewLimit = 0 - fullPage, err := CompileRecipeOutputPageWithPolicy(compiled, fullBindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + fullPage, err := CompileRecipeOutputPage(ArangoTarget(), compiled, fullBindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - assertConstructionRootIDFilterNotInjected(t, fullPage.RootKeysQuery) + assertConstructionRootIDFilterNotInjected(t, testAQLText(t, fullPage.RootKeys)) } func TestConstructionPreviewRootIDFilterRequiresExactPayloadIDProjection(t *testing.T) { @@ -71,15 +71,15 @@ func TestConstructionPreviewRootIDFilterRequiresExactPayloadIDProjection(t *test output.Fields[0].Expr.Select = "root.status" bindings := recipe.RuntimeBindings{Project: "project-a", DatasetGeneration: "generation-a"} compiled := compileConstructionRootIDTestOutput(t, output, bindings) - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if strings.Contains(query.Query, "FILTER root.id == @construction_filter_value") { - t.Fatalf("non-ID source filter was pushed onto root.id:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "FILTER root.id == @construction_filter_value") { + t.Fatalf("non-ID source filter was pushed onto root.id:\n%s", testAQLText(t, query.Command)) } - if !strings.Contains(query.Query, "@construction_filter_value") { - t.Fatalf("original non-ID construction filter was lost:\n%s", query.Query) + if !strings.Contains(testAQLText(t, query.Command), "@construction_filter_value") { + t.Fatalf("original non-ID construction filter was lost:\n%s", testAQLText(t, query.Command)) } } diff --git a/internal/dataframe/compiler/construction_related_count_arango_test.go b/internal/dataframe/compiler/construction_related_count_arango_test.go index adf40bbaf..ab892b018 100644 --- a/internal/dataframe/compiler/construction_related_count_arango_test.go +++ b/internal/dataframe/compiler/construction_related_count_arango_test.go @@ -8,12 +8,12 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/compiler/render/aql" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -22,7 +22,7 @@ func TestRelatedEligibilityCountRowsReverseMatchesForwardOnArango(t *testing.T) t.Skip("Arango query equivalence requires the integration database") } ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "Observation"}, {Name: "Specimen"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -183,7 +183,7 @@ func compileRelatedCountPhysicalPlan(t *testing.T, project, generation string) i return compiled.Outputs[0].Plan } -func executeRelatedCountValue(t *testing.T, ctx context.Context, client *store.Client, query string, binds map[string]any) float64 { +func executeRelatedCountValue(t *testing.T, ctx context.Context, client *arangofactory.Client, query string, binds map[string]any) float64 { t.Helper() var value float64 count := 0 diff --git a/internal/dataframe/compiler/construction_related_expand_arango_test.go b/internal/dataframe/compiler/construction_related_expand_arango_test.go index 2a31c88a7..e038dd41b 100644 --- a/internal/dataframe/compiler/construction_related_expand_arango_test.go +++ b/internal/dataframe/compiler/construction_related_expand_arango_test.go @@ -11,13 +11,13 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/compiler/render/aql" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" "github.com/calypr/loom/internal/dataframe/spec" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -33,13 +33,13 @@ func TestRelatedSourceProjectionLivenessForKeylessCount(t *testing.T) { t.Run(test.name, func(t *testing.T) { query := compileRelatedExpandCountQuery(t, relatedExpandCountOutput(test.groupByFlag)) for _, marker := range []string{"unused_count_column_marker", "unused_list_column_marker", "unused_presence_column_marker"} { - gotMarker := strings.Contains(query.Query, marker) + gotMarker := strings.Contains(testAQLText(t, query.Command), marker) if gotMarker != test.wantMarker { - t.Errorf("related source projection %q included = %t, want %t:\n%s", marker, gotMarker, test.wantMarker, query.Query) + t.Errorf("related source projection %q included = %t, want %t:\n%s", marker, gotMarker, test.wantMarker, testAQLText(t, query.Command)) } } - if !strings.Contains(query.Query, "related_expand_records") || !strings.Contains(query.Query, "WITH COUNT INTO") { - t.Fatalf("query must retain related expansion and row counting:\n%s", query.Query) + if !strings.Contains(testAQLText(t, query.Command), "related_expand_records") || !strings.Contains(testAQLText(t, query.Command), "WITH COUNT INTO") { + t.Fatalf("query must retain related expansion and row counting:\n%s", testAQLText(t, query.Command)) } }) } @@ -50,7 +50,7 @@ func TestRelatedExpandDistinctTerminalRowsAndEmptyPoliciesAgainstArango(t *testi t.Skip("LOOM_TEST_ARANGO_URL and LOOM_TEST_ARANGO_DATABASE must point to the supplied Docker Arango service") } ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -125,7 +125,7 @@ func TestRelatedExpandDistinctTerminalRowsAndEmptyPoliciesAgainstArango(t *testi Project: project, DatasetGeneration: generation, AuthScopeMode: authscope.ReadScopeUnrestricted, } directOutput := lowerConstructionOutput(t, directRelatedExpandOutput(recipe.ExpansionPreserveParent), bindings) - previewQuery, err := CompileRecipeOutputWithPolicy(directOutput, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + previewQuery, err := CompileRecipeOutput(ArangoTarget(), directOutput, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -141,7 +141,7 @@ func TestRelatedExpandDistinctTerminalRowsAndEmptyPoliciesAgainstArango(t *testi for parent, parentRows := range byParent { for _, row := range parentRows { rowID, _ := row["__loom_row_id"].(string) - compiled, err := CompileRowLineageOutput(directOutput, rowID, 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), directOutput, rowID, 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile row lineage for %s: %v", parent, err) } @@ -170,7 +170,7 @@ func TestRelatedExpandDistinctTerminalRowsAndEmptyPoliciesAgainstArango(t *testi stage := directOutput.Plan.StageSequence.Stages[0] constructionID := directOutput.Plan.BindVars[stage.RelatedExpand.ConstructionIDBindKey].(string) forgedRowID := relatedExpandRowID(t, project+"_p1", constructionID, "Observation/"+project+"_forged") - forged, err := CompileRowLineageOutput(directOutput, forgedRowID, 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + forged, err := CompileRowLineageOutput(ArangoTarget(), directOutput, forgedRowID, 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -224,7 +224,7 @@ func TestRelatedExpandDistinctTerminalRowsAndEmptyPoliciesAgainstArango(t *testi t.Run(string(recipe.ExpansionError), func(t *testing.T) { _, query := compileRelatedExpandOracleQuery(t, relatedExpandOracleOutput(recipe.ExpansionError), project, generation) started := time.Now() - err := client.QueryRows(ctx, query.Query, 500, query.BindVars, func(map[string]any) error { return nil }) + err := client.QueryRows(ctx, testAQLText(t, query.Command), 500, testAQLBinds(t, query.Command), func(map[string]any) error { return nil }) t.Logf("RELATED_EXPAND %s query rejected an empty route in %s", recipe.ExpansionError, time.Since(started)) if err == nil { t.Fatalf("ERROR policy returned rows despite active p2 having no related resource") @@ -238,7 +238,7 @@ func TestGroupedRelatedExpandUsesDistinctContributorUnionAndOnwardIdentityAgains t.Skip("LOOM_TEST_ARANGO_URL and LOOM_TEST_ARANGO_DATABASE must point to the supplied Docker Arango service") } ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "Observation"}, {Name: "Specimen"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -350,7 +350,7 @@ func TestGroupedRelatedExpandUsesDistinctContributorUnionAndOnwardIdentityAgains compiled.Plan.StageSequence.Stages[2].RelatedExpand == nil || compiled.Plan.StageSequence.Stages[2].RelatedExpand.AnchorKind != "activeRelatedRecord" { t.Fatalf("expected Group → contributor-set expansion → exact onward identity stages: %#v", compiled.Stages) } - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -398,12 +398,10 @@ func TestGroupedRelatedExpandUsesDistinctContributorUnionAndOnwardIdentityAgains output := groupedRelatedExpansionOutput(recipe.ExpansionError) query := compileConstructionOutputQueryWithBindings(t, output, project, generation, recipe.RuntimeBindings{AuthScopeMode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{"/allowed"}}) - err := client.QueryRows(ctx, query.Query, 100, query.BindVars, func(map[string]any) error { return nil }) - if err == nil { - t.Fatal("ERROR policy returned rows for the empty female contributor group") - } - if !strings.Contains(strings.ToLower(err.Error()), "related expansion") || !strings.Contains(strings.ToLower(err.Error()), "no related records") { - t.Fatalf("ERROR policy failed for an unrelated reason: %v", err) + err := client.QueryRows(ctx, testAQLText(t, query.Command), 100, testAQLBinds(t, query.Command), func(map[string]any) error { return nil }) + cause := requireStableRuntimeCode(t, err, "CONSTRUCTION_EXPANSION_EMPTY") + if !strings.Contains(strings.ToLower(cause), "related expansion") || !strings.Contains(strings.ToLower(cause), "no related records") { + t.Fatalf("ERROR policy failed for an unrelated reason: diagnostic cause = %q", cause) } }) thresholdTwo, thresholdThree := 2, 3 @@ -431,7 +429,7 @@ func TestGroupedRelatedExpandUsesDistinctContributorUnionAndOnwardIdentityAgains compiled.Plan.StageSequence.Stages[1].Kind != ir.PhysicalStageRelatedEligibilityOp { t.Fatalf("expected direct Group -> related eligibility with materialized root contributors: %#v", compiled.Plan.StageSequence.Stages) } - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -536,7 +534,7 @@ func TestRelatedFieldReadsExactExpandedRecordAfterFilterAgainstArango(t *testing t.Skip("LOOM_TEST_ARANGO_URL and LOOM_TEST_ARANGO_DATABASE must point to the supplied Docker Arango service") } ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -663,7 +661,7 @@ func TestRelatedFieldReadsExactExpandedRecordAfterFilterAgainstArango(t *testing if err != nil { t.Fatal(err) } - query, err := CompileRecipeOutputWithPolicy(compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -672,8 +670,8 @@ func TestRelatedFieldReadsExactExpandedRecordAfterFilterAgainstArango(t *testing "related_field_document.dataset_generation == @dataset_generation", "related_field_document.auth_resource_path IN @auth_resource_paths", } { - if !strings.Contains(query.Query, required) { - t.Fatalf("exact field lookup omits scope guard %q: %s", required, query.Query) + if !strings.Contains(testAQLText(t, query.Command), required) { + t.Fatalf("exact field lookup omits scope guard %q: %s", required, testAQLText(t, query.Command)) } } rows := executeReshapeOracleQuery(t, ctx, client, query) @@ -763,21 +761,21 @@ func compileRelatedExpandOracleQuery(t *testing.T, output recipe.Output, project if err != nil { t.Fatalf("compile related-expansion recipe: %v", err) } - query, err := CompileRecipeOutputWithPolicy(compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile related-expansion query: %v", err) } return compiled.Outputs[0], query } -func executeRowLineageOracleQuery(t *testing.T, ctx context.Context, client *store.Client, query CompiledRowLineageQuery) map[string]any { +func executeRowLineageOracleQuery(t *testing.T, ctx context.Context, client *arangofactory.Client, query CompiledRowLineageQuery) map[string]any { t.Helper() rows := make([]map[string]any, 0, 1) - if err := client.QueryRows(ctx, query.Query, 32, query.BindVars, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, query.Command), 32, testAQLBinds(t, query.Command), func(row map[string]any) error { rows = append(rows, row) return nil }); err != nil { - t.Fatalf("execute compiler-generated row lineage query: %v\n%s", err, query.Query) + t.Fatalf("execute compiler-generated row lineage query: %v\n%s", err, testAQLText(t, query.Command)) } if len(rows) != 1 { t.Fatalf("row lineage returned %d rows, want one result: %#v", len(rows), rows) @@ -881,7 +879,7 @@ func compileRelatedExpandCountQuery(t *testing.T, output recipe.Output) Compiled if err != nil { t.Fatalf("compile related expansion count recipe: %v", err) } - query, err := CompileRecipeOutputWithPolicy(compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("render related expansion count recipe: %v", err) } @@ -893,7 +891,7 @@ func TestRelatedExpandRepeatedScalarContributorMatchesWithinTerminalRecordAgains t.Skip("LOOM_TEST_ARANGO_URL and LOOM_TEST_ARANGO_DATABASE must point to the supplied Docker Arango service") } ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -994,8 +992,8 @@ func TestRelatedExpandRepeatedScalarContributorMatchesWithinTerminalRecordAgains } query := compileConstructionOutputQueryWithBindings(t, output, project, generation, bindings) for _, required := range []string{".project == @project", ".dataset_generation == @dataset_generation", "auth_resource_path IN @auth_resource_paths"} { - if !strings.Contains(query.Query, required) { - t.Fatalf("repeated related predicate query omitted scope guard %q: %s", required, query.Query) + if !strings.Contains(testAQLText(t, query.Command), required) { + t.Fatalf("repeated related predicate query omitted scope guard %q: %s", required, testAQLText(t, query.Command)) } } rows := executeReshapeOracleQuery(t, ctx, client, query) @@ -1032,7 +1030,7 @@ func TestTerminalRepeatedPrimitiveSelectorValuesAgainstArango(t *testing.T) { t.Skip("LOOM_TEST_ARANGO_URL and LOOM_TEST_ARANGO_DATABASE must point to the supplied Docker Arango service") } ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Patient"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Patient"}}}); err != nil { t.Fatal(err) } project, generation := "loom_repeated_terminal_"+uuid.NewString(), "generation-repeated-terminal" @@ -1107,7 +1105,7 @@ func TestTerminalRepeatedPrimitiveSelectorValuesAgainstArango(t *testing.T) { if err != nil { t.Fatalf("render %s terminal primitive selector: %v", mode, err) } - rows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: rendered.Query, BindVars: rendered.BindVars}) + rows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: testAQLCommand(rendered.Query, rendered.BindVars)}) if len(rows) != len(documents) { t.Fatalf("%s terminal primitive query returned %d rows, want %d:\n%s", mode, len(rows), len(documents), rendered.Query) } diff --git a/internal/dataframe/compiler/construction_related_group_preview_arango_test.go b/internal/dataframe/compiler/construction_related_group_preview_arango_test.go index 129dde7f2..5cc1c8732 100644 --- a/internal/dataframe/compiler/construction_related_group_preview_arango_test.go +++ b/internal/dataframe/compiler/construction_related_group_preview_arango_test.go @@ -11,9 +11,9 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/recipe" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -26,7 +26,7 @@ func TestRelatedGroupPreviewReportsValidationWindow(t *testing.T) { output.Construction.Steps = output.Construction.Steps[:3] } compiled := lowerConstructionOutput(t, output, bindings) - preview, err := CompileRecipeOutputWithPolicy(compiled, bindings, 2, ir.DefaultPhysicalOptimizationPolicy()) + preview, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 2, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -78,7 +78,7 @@ func TestRelatedGroupPreviewPreservesProjectionAndScopeContracts(t *testing.T) { bindings := recipe.RuntimeBindings{Project: "owned-project", DatasetGeneration: "owned-generation", IncludeRowIdentity: true} compiled := lowerConstructionOutput(t, relatedGroupPreviewOutput(t, bindings), bindings) test.change(&compiled.Plan) - preview, err := CompileRecipeOutputWithPolicy(compiled, bindings, 2, ir.DefaultPhysicalOptimizationPolicy()) + preview, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 2, ir.DefaultPhysicalOptimizationPolicy()) if test.wantError != "" { if err == nil || !strings.Contains(err.Error(), test.wantError) || !strings.Contains(err.Error(), "project scope filter at operation") || !strings.Contains(err.Error(), ".project == @project") { @@ -104,7 +104,7 @@ func TestRelatedGroupPreviewPreservesProjectionAndScopeContracts(t *testing.T) { func TestRelatedGroupPreviewKeepsCompleteContributorsAgainstArango(t *testing.T) { ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Specimen"}, {Name: "Patient"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -191,8 +191,6 @@ func TestRelatedGroupPreviewKeepsCompleteContributorsAgainstArango(t *testing.T) {"blocked_root", "pa"}, {"wrong_generation_root", "pa"}, {"wrong_project_root", "pa"}} { addEdge("Specimen", pair[0], pair[1]) } - // Canonical forward traversal does not check this redundant source-type field. - // A reverse optimization must not introduce that extra predicate. addEdge("Specimen", "s2", "pa_duplicate")["from_type"] = "unexpected-source-type" addEdge("Observation", "rogue_root", "pa")["from_type"] = "Specimen" for _, pair := range [][2]string{{"o1", "pa"}, {"o1", "pa"}, {"o2", "pa"}, {"o3", "pa_duplicate"}, @@ -208,7 +206,7 @@ func TestRelatedGroupPreviewKeepsCompleteContributorsAgainstArango(t *testing.T) AuthScopeMode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{"/allowed"}, IncludeRowIdentity: true} output := relatedGroupPreviewOutput(t, bindings) compiled := lowerConstructionOutput(t, output, bindings) - full, err := CompileRecipeOutputWithPolicy(compiled, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + full, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -229,7 +227,7 @@ func TestRelatedGroupPreviewKeepsCompleteContributorsAgainstArango(t *testing.T) } comparePreview := func(t *testing.T, limit int) { t.Helper() - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, limit, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, limit, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -246,7 +244,7 @@ func TestRelatedGroupPreviewKeepsCompleteContributorsAgainstArango(t *testing.T) terminalOutput := relatedGroupPreviewOutput(t, bindings) terminalOutput.Construction.Steps = terminalOutput.Construction.Steps[:3] terminal := lowerConstructionOutput(t, terminalOutput, bindings) - fullQuery, err := CompileRecipeOutputWithPolicy(terminal, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + fullQuery, err := CompileRecipeOutput(ArangoTarget(), terminal, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -254,7 +252,7 @@ func TestRelatedGroupPreviewKeepsCompleteContributorsAgainstArango(t *testing.T) sort.Slice(fullRows, func(i, j int) bool { return fullRows[i]["__loom_row_id"].(string) < fullRows[j]["__loom_row_id"].(string) }) - previewQuery, err := CompileRecipeOutputWithPolicy(terminal, bindings, 2, ir.DefaultPhysicalOptimizationPolicy()) + previewQuery, err := CompileRecipeOutput(ArangoTarget(), terminal, bindings, 2, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -263,8 +261,6 @@ func TestRelatedGroupPreviewKeepsCompleteContributorsAgainstArango(t *testing.T) t.Fatalf("terminal Group changed complete groups: got %#v, want %#v", previewRows, fullRows[:2]) } }) - // A candidate prefix containing only unreferenced Patients must fall back; - // it must never turn a nonempty dataframe into an empty preview. unreferenced := make([]map[string]any, 0, 20) for i := range 20 { unreferenced = append(unreferenced, resource("Patient", fmt.Sprintf("unused_%d", i), fmt.Sprintf("!unused_%02d", i))) diff --git a/internal/dataframe/compiler/correlated_arango_test.go b/internal/dataframe/compiler/correlated_arango_test.go index aa40b5d4a..c10dc1874 100644 --- a/internal/dataframe/compiler/correlated_arango_test.go +++ b/internal/dataframe/compiler/correlated_arango_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/compiler/render/aql" @@ -20,7 +21,6 @@ import ( "github.com/calypr/loom/internal/dataframe/unit" fhirschema "github.com/calypr/loom/internal/fhir/schema" "github.com/calypr/loom/internal/projectid" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -31,12 +31,13 @@ func TestNumericAggregateLiteralValuesAgainstArango(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Observation"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Observation"}}}); err != nil { t.Fatal(err) } selector, err := spec.ParseSelector("component[].valueInteger") @@ -108,7 +109,7 @@ func TestNumericAggregateLiteralValuesAgainstArango(t *testing.T) { } want := *tc.want if name == "mean_value" { - want /= 3 // 2, 0, and 4 contribute; the null is ignored. + want /= 3 } value, ok := got.(float64) if !ok || math.Abs(value-want) > 1e-9 { @@ -126,12 +127,13 @@ func TestExplicitGroupRowsAgainstArango(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - collections := []store.CollectionSpec{ + collections := []arangofactory.CollectionSpec{ {Name: "Observation"}, {Name: "loom_explorer_explicit_group_revisions"}, {Name: "loom_explorer_explicit_group_definitions"}, @@ -139,7 +141,7 @@ func TestExplicitGroupRowsAgainstArango(t *testing.T) { {Name: "loom_explorer_selections"}, {Name: "loom_explorer_selection_members"}, } - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: collections}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: collections}); err != nil { t.Fatal(err) } project, generation := "group_rows_"+uuid.NewString(), "generation_"+uuid.NewString() @@ -192,11 +194,11 @@ func TestExplicitGroupRowsAgainstArango(t *testing.T) { t.Fatal(compileErr) } rows := make([]map[string]any, 0) - if queryErr := client.QueryRows(ctx, queries[0].Query, 100, queries[0].BindVars, func(row map[string]any) error { + if queryErr := client.QueryRows(ctx, testAQLText(t, queries[0].Command), 100, testAQLBinds(t, queries[0].Command), func(row map[string]any) error { rows = append(rows, row) return nil }); queryErr != nil { - t.Fatalf("execute explicit group rows: %v\n%s", queryErr, queries[0].Query) + t.Fatalf("execute explicit group rows: %v\n%s", queryErr, testAQLText(t, queries[0].Command)) } return rows } @@ -249,19 +251,18 @@ func TestExplicitGroupRowsAgainstArango(t *testing.T) { t.Fatal(err) } unassignedRows := make([]map[string]any, 0) - if err := client.QueryRows(ctx, unassignedQueries[0].Query, 100, unassignedQueries[0].BindVars, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, unassignedQueries[0].Command), 100, testAQLBinds(t, unassignedQueries[0].Command), func(row map[string]any) error { unassignedRows = append(unassignedRows, row) return nil }); err != nil { - t.Fatalf("execute GROUP_AS_UNASSIGNED query: %v\n%s", err, unassignedQueries[0].Query) + t.Fatalf("execute GROUP_AS_UNASSIGNED query: %v\n%s", err, testAQLText(t, unassignedQueries[0].Command)) } if len(unassignedRows) != 4 || unassignedRows[3]["group_id"] != "__loom_unassigned__" { t.Fatalf("unassigned group rows = %#v", unassignedRows) } errorQuery := mustCompileGroupRowsQuery(t, project, generation, revisionID, "ERROR") - if err := client.QueryRows(ctx, errorQuery.Query, 100, errorQuery.BindVars, func(map[string]any) error { return nil }); err == nil || !strings.Contains(err.Error(), "EXPLICIT_GROUP_UNASSIGNED_MEMBER") { - t.Fatalf("ERROR policy query error = %v, want unassigned member assertion", err) - } + err = client.QueryRows(ctx, testAQLText(t, errorQuery.Command), 100, testAQLBinds(t, errorQuery.Command), func(map[string]any) error { return nil }) + requireStableRuntimeCode(t, err, "EXPLICIT_GROUP_UNASSIGNED_MEMBER") } func compileGroupRowsQuery(project, generation, revisionID, policy string) ([]CompiledQuery, error) { @@ -276,7 +277,7 @@ func compileGroupRowsQuery(project, generation, revisionID, policy string) ([]Co if err != nil { return nil, err } - return CompileResolvedRecipePlanWithPolicy(resolved, 100, ir.DefaultPhysicalOptimizationPolicy()) + return CompileResolvedRecipePlan(ArangoTarget(), resolved, 100, ir.DefaultPhysicalOptimizationPolicy()) } func mustCompileGroupRowsQuery(t *testing.T, project, generation, revisionID, policy string) CompiledQuery { @@ -297,7 +298,7 @@ func groupRowsMembership(revisionID, project, generation, groupID, memberID stri } } -func insertGroupRowsFixtureDocs(ctx context.Context, t *testing.T, client *store.Client, collection string, docs []map[string]any) { +func insertGroupRowsFixtureDocs(ctx context.Context, t *testing.T, client *arangofactory.Client, collection string, docs []map[string]any) { t.Helper() raw := make([]json.RawMessage, 0, len(docs)) for _, doc := range docs { @@ -321,12 +322,13 @@ func TestUnitNormalizationLiteralValuesAgainstArango(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Observation"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Observation"}}}); err != nil { t.Fatal(err) } @@ -398,9 +400,7 @@ func TestUnitNormalizationLiteralValuesAgainstArango(t *testing.T) { return nil }) if tc.wantUnknown { - if queryErr == nil || !strings.Contains(queryErr.Error(), "UNIT_IDENTITY_UNKNOWN") { - t.Fatalf("query error = %v, want UNIT_IDENTITY_UNKNOWN\n%s", queryErr, rendered.Query) - } + requireStableRuntimeCode(t, queryErr, "UNIT_IDENTITY_UNKNOWN") return } if queryErr != nil { @@ -424,12 +424,13 @@ func TestExtensionCompilerLiteralValuesAgainstArango(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Observation"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Observation"}}}); err != nil { t.Fatal(err) } project := "loom_extension_" + uuid.NewString() @@ -484,12 +485,13 @@ func TestCorrelatedCompilerLiteralValuesAgainstArango(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Observation"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Observation"}}}); err != nil { t.Fatal(err) } coding := func(system, code string) map[string]any { return map[string]any{"system": system, "code": code} } @@ -583,12 +585,13 @@ func TestOwnerRecordsPreserveRepeatedFHIRValuesAgainstArango(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Observation"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Observation"}}}); err != nil { t.Fatal(err) } @@ -707,12 +710,13 @@ func TestContributorPredicatesRemainFeatureLocalAgainstArango(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -836,12 +840,13 @@ func TestRecipeOccurrenceExpansionLiteralRowsAgainstArango(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "Observation"}, {Name: "Specimen"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -1031,24 +1036,25 @@ func TestRecipeOccurrenceExpansionLiteralRowsAgainstArango(t *testing.T) { if err != nil { t.Fatal(err) } - queryErr := client.QueryRows(ctx, queries[0].Query, 100, queries[0].BindVars, func(map[string]any) error { return nil }) - if queryErr == nil || !strings.Contains(queryErr.Error(), "specimen-owner") || !strings.Contains(queryErr.Error(), emptySpecimenKey) { - t.Fatalf("ERROR policy error = %v, want owner-boundary assertion for occurrence specimen-owner and empty owner %s\n%s", queryErr, emptySpecimenKey, queries[0].Query) + queryErr := client.QueryRows(ctx, testAQLText(t, queries[0].Command), 100, testAQLBinds(t, queries[0].Command), func(map[string]any) error { return nil }) + cause := requireStableRuntimeCode(t, queryErr, "CONSTRUCTION_EXPANSION_EMPTY") + if !strings.Contains(cause, "specimen-owner") || !strings.Contains(cause, emptySpecimenKey) { + t.Fatalf("ERROR policy diagnostic cause = %q, want owner-boundary assertion for occurrence specimen-owner and empty owner %s\n%s", cause, emptySpecimenKey, testAQLText(t, queries[0].Command)) } } -func runRecipeExpansionQuery(t *testing.T, ctx context.Context, client *store.Client, project, generation string, output recipe.Output) []map[string]any { +func runRecipeExpansionQuery(t *testing.T, ctx context.Context, client *arangofactory.Client, project, generation string, output recipe.Output) []map[string]any { t.Helper() queries, err := compileRecipeExpansionQuery(t, project, generation, output) if err != nil { t.Fatal(err) } rows := make([]map[string]any, 0) - if err := client.QueryRows(ctx, queries[0].Query, 100, queries[0].BindVars, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, queries[0].Command), 100, testAQLBinds(t, queries[0].Command), func(row map[string]any) error { rows = append(rows, row) return nil }); err != nil { - t.Fatalf("execute occurrence expansion query: %v\n%s", err, queries[0].Query) + t.Fatalf("execute occurrence expansion query: %v\n%s", err, testAQLText(t, queries[0].Command)) } return rows } @@ -1065,7 +1071,7 @@ func compileRecipeExpansionQuery(t *testing.T, project, generation string, outpu if err != nil { return nil, err } - return CompileResolvedRecipePlanWithPolicy(resolved, 100, ir.DefaultPhysicalOptimizationPolicy()) + return CompileResolvedRecipePlan(ArangoTarget(), resolved, 100, ir.DefaultPhysicalOptimizationPolicy()) } func expansionIdentityFromRow(t *testing.T, row map[string]any) map[string]any { diff --git a/internal/dataframe/compiler/explicit_cohort_values_arango_test.go b/internal/dataframe/compiler/explicit_cohort_values_arango_test.go index 3e57f676d..e395e9a68 100644 --- a/internal/dataframe/compiler/explicit_cohort_values_arango_test.go +++ b/internal/dataframe/compiler/explicit_cohort_values_arango_test.go @@ -10,23 +10,23 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/columntransform" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/projectid" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) func TestRelatedCountAppendPreservesExplicitCohortMembersAgainstArango(t *testing.T) { ctx, client := openConstructionReshapeArango(t) - collections := []store.CollectionSpec{ + collections := []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, {Name: "loom_explorer_explicit_group_revisions"}, {Name: "loom_explorer_selections"}, {Name: "loom_explorer_explicit_group_definitions"}, {Name: "loom_explorer_explicit_group_memberships"}, {Name: "loom_explorer_selection_members"}, } - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: collections}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: collections}); err != nil { t.Fatal(err) } project := "LOOM_COHORT/related-append-" + uuid.NewString() @@ -202,7 +202,7 @@ func TestRelatedCountAppendPreservesExplicitCohortMembersAgainstArango(t *testin execute := func(output recipe.Output) []map[string]any { t.Helper() compiled := lowerConstructionOutput(t, output, bindings) - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile cohort output %q: %v", output.Name, err) } @@ -361,11 +361,11 @@ func TestRelatedCountAppendPreservesExplicitCohortMembersAgainstArango(t *testin func TestExplicitCohortMemberValuesAgainstArango(t *testing.T) { ctx, client := openConstructionReshapeArango(t) collections := []string{"Patient", "loom_explorer_explicit_group_revisions", "loom_explorer_selections", "loom_explorer_explicit_group_definitions", "loom_explorer_explicit_group_memberships", "loom_explorer_selection_members"} - specs := make([]store.CollectionSpec, 0, len(collections)) + specs := make([]arangofactory.CollectionSpec, 0, len(collections)) for _, name := range collections { - specs = append(specs, store.CollectionSpec{Name: name}) + specs = append(specs, arangofactory.CollectionSpec{Name: name}) } - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: specs}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: specs}); err != nil { t.Fatal(err) } project := "LOOM_COHORT/member-values-" + uuid.NewString() @@ -421,7 +421,6 @@ func TestExplicitCohortMemberValuesAgainstArango(t *testing.T) { "payload": map[string]any{"id": id, "gender": "female", "identifier": []any{map[string]any{"value": "shared"}, map[string]any{"value": fmt.Sprintf("value-%d", index)}}}}) } } - // Same FHIR ID outside either dataset boundary must never supply member values. for index, scope := range []map[string]any{{"project": "another-project", "dataset_generation": generation}, {"project": resourceProject, "dataset_generation": "another-generation"}} { scope["_key"], scope["id"], scope["resourceType"], scope["auth_resource_path"] = keyPrefix+fmt.Sprintf("_outside_%d", index), "a", "Patient", "/allowed" scope["payload"] = map[string]any{"id": "leaked", "gender": "male"} @@ -433,7 +432,7 @@ func TestExplicitCohortMemberValuesAgainstArango(t *testing.T) { } bindings := recipe.RuntimeBindings{Project: resourceProject, SelectionProject: project, DatasetGeneration: generation, AuthScopeMode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{"/allowed"}, IncludeRowIdentity: true} compiled := lowerConstructionOutput(t, output, bindings) - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -490,7 +489,7 @@ func TestExplicitCohortMemberValuesAgainstArango(t *testing.T) { Outputs: []recipe.StageColumn{{ID: "ids", Name: "ids", Type: "string"}, {ID: "gender", Name: "gender", Type: "string"}, {ID: "identifiers", Name: "identifiers", Type: "string"}}, }}} compiled := lowerConstructionOutput(t, filtered, bindings) - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -534,7 +533,7 @@ func TestExplicitCohortMemberValuesAgainstArango(t *testing.T) { }, }}} compiled := lowerConstructionOutput(t, filtered, bindings) - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -568,11 +567,11 @@ func TestExplicitCohortMemberValuesAgainstArango(t *testing.T) { groups.RowValues = []recipe.GroupRowValuePolicy{{ColumnID: "ids", Policy: recipe.ConstructionRowValueOne}} one.GroupRows = &groups compiled := lowerConstructionOutput(t, one, bindings) - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - err = client.ExecuteAQL(ctx, query.Query, query.BindVars) + err = client.ExecuteAQL(ctx, testAQLText(t, query.Command), testAQLBinds(t, query.Command)) if err == nil || !strings.Contains(err.Error(), "CONSTRUCTION_ROW_VALUE_MULTIPLE_VALUES") { t.Fatalf("ONE disagreement must preserve its typed repair error: %v", err) } @@ -599,7 +598,7 @@ func TestExplicitCohortMemberValuesAgainstArango(t *testing.T) { recoded.GroupRows = &groups recoded.ColumnTransformations = []recipe.ColumnTransformation{{Column: "ids", Transformation: transformation}} compiled := lowerConstructionOutput(t, recoded, bindings) - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile transformed cohort member values: %v", err) } @@ -648,11 +647,11 @@ func TestExplicitCohortMemberValuesAgainstArango(t *testing.T) { }, }}} compiled := lowerConstructionOutput(t, recoded, bindings) - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if err := client.ExecuteAQL(ctx, query.Query, query.BindVars); err == nil || !strings.Contains(err.Error(), "CATEGORY_RECODE_UNKNOWN_VALUE") { + if err := client.ExecuteAQL(ctx, testAQLText(t, query.Command), testAQLBinds(t, query.Command)); err == nil || !strings.Contains(err.Error(), "CATEGORY_RECODE_UNKNOWN_VALUE") { t.Fatalf("unknown non-null cohort member must retain the transformation error: %v", err) } }) @@ -660,7 +659,7 @@ func TestExplicitCohortMemberValuesAgainstArango(t *testing.T) { denyAll := bindings denyAll.AuthResourcePaths = nil compiled := lowerConstructionOutput(t, output, denyAll) - query, err := CompileRecipeOutputWithPolicy(compiled, denyAll, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, denyAll, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -668,8 +667,6 @@ func TestExplicitCohortMemberValuesAgainstArango(t *testing.T) { if !reflect.DeepEqual(row["ids"], []any{}) || row["gender"] != nil || !reflect.DeepEqual(row["identifiers"], []any{}) { t.Fatalf("restricted-empty authorization leaked cohort member data: %#v", row) } - // A verified pinned selection may retain the old null-payload identity - // for a resource that no longer exists; it cannot carry field values. for _, rawMember := range row["members"].([]any) { member := rawMember.(map[string]any) identity := member["source_identity"].(map[string]any) diff --git a/internal/dataframe/compiler/generic_physical_plan_test.go b/internal/dataframe/compiler/generic_physical_plan_test.go index c08b45773..c9423a0a2 100644 --- a/internal/dataframe/compiler/generic_physical_plan_test.go +++ b/internal/dataframe/compiler/generic_physical_plan_test.go @@ -158,7 +158,7 @@ func TestCompileRecipeOutputPreservesMixedChildDependencyOrder(t *testing.T) { t.Fatal(err) } - compiled, err := CompileRecipeOutputWithPolicy(lower.CompiledRecipeOutput{ + compiled, err := CompileRecipeOutput(ArangoTarget(), lower.CompiledRecipeOutput{ Name: "patients", RootResourceType: "Patient", RowGrain: "patient", @@ -168,10 +168,10 @@ func TestCompileRecipeOutputPreservesMixedChildDependencyOrder(t *testing.T) { t.Fatal(err) } - parentSet := strings.Index(compiled.Query, "LET child_set_1 =") - nestedTraversal := strings.Index(compiled.Query, "FOR __loom_physical_parent_1 IN child_set_1") + parentSet := strings.Index(testAQLText(t, compiled.Command), "LET child_set_1 =") + nestedTraversal := strings.Index(testAQLText(t, compiled.Command), "FOR __loom_physical_parent_1 IN child_set_1") if parentSet < 0 || nestedTraversal < 0 || parentSet >= nestedTraversal { - t.Fatalf("mixed child dependencies rendered out of order:\n%s", compiled.Query) + t.Fatalf("mixed child dependencies rendered out of order:\n%s", testAQLText(t, compiled.Command)) } } diff --git a/internal/dataframe/compiler/graph_execution.go b/internal/dataframe/compiler/graph_execution.go index 787fe5f2c..d5fe23744 100644 --- a/internal/dataframe/compiler/graph_execution.go +++ b/internal/dataframe/compiler/graph_execution.go @@ -3,6 +3,7 @@ package compiler import ( "fmt" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/compiler/optimize" @@ -10,11 +11,13 @@ import ( "github.com/calypr/loom/internal/dataframe/semantic" ) -// CompileResolvedGraphQueryWithPolicy compiles the explicit graph frontend -// through the canonical physical optimizer and renderer. The public limit is -// preserved in metadata while the physical graph return fetches limit+1 rows -// for lookahead-based hasMore calculation. -func CompileResolvedGraphQueryWithPolicy(resolved semantic.ResolvedRecipePlan, limit int, policy ir.PhysicalOptimizationPolicy) (CompiledQuery, error) { +func CompileResolvedGraphQuery(target Target, resolved semantic.ResolvedRecipePlan, limit int, policy ir.PhysicalOptimizationPolicy) (CompiledQuery, error) { + if err := target.validate(); err != nil { + return CompiledQuery{}, err + } + if err := target.rejectPostgreSQL("graph query compilation"); err != nil { + return CompiledQuery{}, err + } if limit < 1 || limit > 10000 { return CompiledQuery{}, fmt.Errorf("graph limit must be between 1 and 10000") } @@ -30,13 +33,17 @@ func CompileResolvedGraphQueryWithPolicy(resolved semantic.ResolvedRecipePlan, l if err != nil { return CompiledQuery{}, fmt.Errorf("render graph physical plan: %w", err) } + compiledCommand, err := command.NewAQLCommand(rendered.Query, rendered.BindVars) + if err != nil { + return CompiledQuery{}, fmt.Errorf("tag graph command: %w", err) + } output := resolved.SemanticPlan.Outputs[0] bindings := resolved.SemanticPlan.Bindings return CompiledQuery{ Project: bindings.Project, DatasetGeneration: normalizeDatasetGeneration(bindings.DatasetGeneration), RootResourceType: output.RootResourceType, AuthResourcePaths: cloneStrings(bindings.AuthResourcePaths), PlanMode: "physical", PlanProfile: "generic_fhir_graph_path", TraversalCount: physicalTraversalCount(physical), - OptimizationRules: recipeOptimizationRules(physical), Query: rendered.Query, BindVars: rendered.BindVars, + OptimizationRules: recipeOptimizationRules(physical), Command: compiledCommand, Columns: []string{"paths"}, PublicColumns: []string{"paths"}, Limit: limit, PlanDiagnostics: physicalPlanDiagnostics(physical), }, nil diff --git a/internal/dataframe/compiler/group_connected_field_source_arango_test.go b/internal/dataframe/compiler/group_connected_field_source_arango_test.go index e9df2e730..38d22bb8c 100644 --- a/internal/dataframe/compiler/group_connected_field_source_arango_test.go +++ b/internal/dataframe/compiler/group_connected_field_source_arango_test.go @@ -9,11 +9,11 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer/capability" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -26,7 +26,7 @@ func TestConstructionGroupRelatedOccurrenceCodingFieldSourceAgainstArango(t *tes for _, codingEmptyPolicy := range []string{"GROUP", "EXCLUDE", "ERROR"} { output := connectedGroupOutputFromJSON(t, source, codingEmptyPolicy) compiled := lowerConstructionOutput(t, output, bindings) - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile connected Coding Group with coding-empty %s policy: %v", codingEmptyPolicy, err) } @@ -34,7 +34,7 @@ func TestConstructionGroupRelatedOccurrenceCodingFieldSourceAgainstArango(t *tes } ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "Specimen"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -205,7 +205,7 @@ func TestConstructionGroupRelatedOccurrenceCodingFieldSourceAgainstArango(t *tes t.Fatalf("connected EXCLUDE changed the absent-route Patient C identity: GROUP=%#v EXCLUDE=%#v", cMissingRoute, cExcluded) } errorQuery := queries["ERROR"] - if err := client.QueryRows(ctx, errorQuery.Query, 100, errorQuery.BindVars, func(map[string]any) error { return nil }); err == nil { + if err := client.QueryRows(ctx, testAQLText(t, errorQuery.Command), 100, testAQLBinds(t, errorQuery.Command), func(map[string]any) error { return nil }); err == nil { t.Fatal("connected ERROR Coding-empty policy accepted the matched B-empty target") } removeDocument := func(collection, key string) { @@ -462,10 +462,10 @@ func connectedCodingWithoutSystem(version, code, display string) map[string]any return map[string]any{"version": version, "code": code, "display": display} } -func executeConnectedGroupQuery(t *testing.T, ctx context.Context, client *store.Client, query CompiledQuery) []map[string]any { +func executeConnectedGroupQuery(t *testing.T, ctx context.Context, client *arangofactory.Client, query CompiledQuery) []map[string]any { t.Helper() rows := make([]map[string]any, 0) - if err := client.QueryRows(ctx, query.Query, 500, query.BindVars, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, query.Command), 500, testAQLBinds(t, query.Command), func(row map[string]any) error { raw, err := json.Marshal(row) if err != nil { return fmt.Errorf("snapshot connected Group result row: %w", err) @@ -477,7 +477,7 @@ func executeConnectedGroupQuery(t *testing.T, ctx context.Context, client *store rows = append(rows, copy) return nil }); err != nil { - t.Fatalf("execute connected Group query: %v\n%s", err, query.Query) + t.Fatalf("execute connected Group query: %v\n%s", err, testAQLText(t, query.Command)) } return rows } @@ -615,7 +615,7 @@ func assertConnectedGroupIdentity(t *testing.T, row map[string]any, patient stri func assertConnectedGroupLineage( t *testing.T, ctx context.Context, - client *store.Client, + client *arangofactory.Client, compiled lower.CompiledRecipeOutput, row map[string]any, project, patientKey string, @@ -625,7 +625,7 @@ func assertConnectedGroupLineage( ) { t.Helper() rowID, _ := row["__loom_row_id"].(string) - query, err := CompileRowLineageOutput(compiled, rowID, 0, 20, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRowLineageOutput(ArangoTarget(), compiled, rowID, 0, 20, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile connected Group lineage for row %#v: %v", row, err) } diff --git a/internal/dataframe/compiler/group_field_source_arango_test.go b/internal/dataframe/compiler/group_field_source_arango_test.go index e246a4671..d7924918b 100644 --- a/internal/dataframe/compiler/group_field_source_arango_test.go +++ b/internal/dataframe/compiler/group_field_source_arango_test.go @@ -10,12 +10,12 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" "github.com/calypr/loom/internal/explorer/capability" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -63,7 +63,7 @@ func TestConstructionGroupExpandedCodingFieldSourceAgainstArango(t *testing.T) { compiled.Stages[0].GroupFieldSourceContext.SourceShapeID != sourceContext.SourceShapeID { t.Fatalf("lowered source descriptor lost its exact pre-Group shape: %#v", compiled.Stages) } - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile expanded Coding Group with %s policy: %v", policy, err) } @@ -72,7 +72,7 @@ func TestConstructionGroupExpandedCodingFieldSourceAgainstArango(t *testing.T) { } ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "fhir_edge", Edge: true}, {Name: "loom_explorer_selection_members"}, }}); err != nil { t.Fatal(err) @@ -255,7 +255,7 @@ func TestConstructionGroupExpandedCodingFieldSourceAgainstArango(t *testing.T) { } } errorQuery := queryByPolicy[recipe.ConstructionGroupMissingKeyError] - if err := client.QueryRows(ctx, errorQuery.Query, 100, errorQuery.BindVars, func(map[string]any) error { return nil }); err == nil { + if err := client.QueryRows(ctx, testAQLText(t, errorQuery.Command), 100, testAQLBinds(t, errorQuery.Command), func(map[string]any) error { return nil }); err == nil { t.Fatal("ERROR expanded Coding Group accepted missing and empty code occurrences") } }) @@ -277,8 +277,8 @@ func TestConstructionGroupExpandedCodingFieldSourceAgainstArango(t *testing.T) { "id": "observation-b", "resourceType": "Observation", "status": "final", "valueCodeableConcept": map[string]any{"coding": []any{test.coding}}, }) - err := client.QueryRows(ctx, queryByPolicy[recipe.ConstructionGroupMissingKeyGroup].Query, 100, - queryByPolicy[recipe.ConstructionGroupMissingKeyGroup].BindVars, func(map[string]any) error { return nil }) + err := client.QueryRows(ctx, testAQLText(t, queryByPolicy[recipe.ConstructionGroupMissingKeyGroup].Command), 100, + testAQLBinds(t, queryByPolicy[recipe.ConstructionGroupMissingKeyGroup].Command), func(map[string]any) error { return nil }) if err == nil || !strings.Contains(err.Error(), "GROUP_KEY_TYPE_MISMATCH") { t.Fatalf("GROUP query with %s = %v, want GROUP_KEY_TYPE_MISMATCH", test.name, err) } @@ -372,7 +372,7 @@ func TestConstructionGroupAncestorRowValuesAgainstArango(t *testing.T) { if err != nil { t.Fatalf("lower ancestor Coding Group with post-Group component RowValue: %v", err) } - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile ancestor Coding Group with component RowValue: %v", err) } @@ -405,7 +405,7 @@ func TestConstructionGroupAncestorRowValuesAgainstArango(t *testing.T) { } ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "fhir_edge", Edge: true}, {Name: "loom_explorer_selection_members"}, }}); err != nil { t.Fatal(err) @@ -855,7 +855,7 @@ func assertExpandedCodingGroupUsesProjectedMembership(t *testing.T, compiled low t.Fatalf("expanded Coding Group has incomplete input row or membership column: stage=%#v coding=%#v", stage, coding) } membershipBind := "" - for key, value := range query.BindVars { + for key, value := range testAQLBinds(t, query.Command) { if value == coding.MembershipColumn && strings.Contains(key, "construction_group_coding_membership_column") { if membershipBind != "" { t.Fatalf("Coding membership column %q has multiple query binds %q and %q", coding.MembershipColumn, membershipBind, key) @@ -867,15 +867,15 @@ func assertExpandedCodingGroupUsesProjectedMembership(t *testing.T, compiled low t.Fatalf("compiled query has no bind for Coding membership column %q", coding.MembershipColumn) } read := fmt.Sprintf("%s[@%s]", stage.InputRowVariable, membershipBind) - readAt := strings.Index(query.Query, read) + readAt := strings.Index(testAQLText(t, query.Command), read) if readAt < 0 { - t.Fatalf("Group query does not read projected Coding membership at %s:\n%s", read, query.Query) + t.Fatalf("Group query does not read projected Coding membership at %s:\n%s", read, testAQLText(t, query.Command)) } - collectAtRelative := strings.Index(query.Query[readAt+len(read):], "COLLECT ") + collectAtRelative := strings.Index(testAQLText(t, query.Command)[readAt+len(read):], "COLLECT ") if collectAtRelative < 0 { - t.Fatalf("Group query does not collect after reading Coding membership:\n%s", query.Query) + t.Fatalf("Group query does not collect after reading Coding membership:\n%s", testAQLText(t, query.Command)) } - groupKeyRead := query.Query[readAt : readAt+len(read)+collectAtRelative] + groupKeyRead := testAQLText(t, query.Command)[readAt : readAt+len(read)+collectAtRelative] if strings.Contains(groupKeyRead, "DOCUMENT(") || strings.Contains(groupKeyRead, ".payload") { t.Fatalf("Group key evaluation re-scans or walks a root payload instead of using the projected Coding membership:\n%s", groupKeyRead) } @@ -969,7 +969,7 @@ func expandedCodingGroupObservation(id string, includeSharedDuplicates bool) map } } -func insertExpandedCodingGroupSelectionMember(t *testing.T, ctx context.Context, client *store.Client, project, generation, selectionID string) { +func insertExpandedCodingGroupSelectionMember(t *testing.T, ctx context.Context, client *arangofactory.Client, project, generation, selectionID string) { t.Helper() document, err := json.Marshal(map[string]any{ "_key": selectionID + "_patient", "selectionId": selectionID, @@ -983,7 +983,7 @@ func insertExpandedCodingGroupSelectionMember(t *testing.T, ctx context.Context, } } -func replaceExpandedCodingGroupObservation(t *testing.T, ctx context.Context, client *store.Client, key string, payload map[string]any) { +func replaceExpandedCodingGroupObservation(t *testing.T, ctx context.Context, client *arangofactory.Client, key string, payload map[string]any) { t.Helper() err := client.ExecuteAQL(ctx, "FOR document IN Observation FILTER document._key == @key UPDATE document WITH {payload: @payload} IN Observation", @@ -994,7 +994,7 @@ func replaceExpandedCodingGroupObservation(t *testing.T, ctx context.Context, cl } } -func insertExpandedCodingGroupResource(t *testing.T, ctx context.Context, client *store.Client, collection, key, project, generation string, payload map[string]any) { +func insertExpandedCodingGroupResource(t *testing.T, ctx context.Context, client *arangofactory.Client, collection, key, project, generation string, payload map[string]any) { t.Helper() document, err := json.Marshal(map[string]any{ "_key": key, "id": payload["id"], "project": project, "project_id": project, @@ -1008,7 +1008,7 @@ func insertExpandedCodingGroupResource(t *testing.T, ctx context.Context, client } } -func insertExpandedCodingGroupEdge(t *testing.T, ctx context.Context, client *store.Client, key, observationKey, patientKey, project, generation string) { +func insertExpandedCodingGroupEdge(t *testing.T, ctx context.Context, client *arangofactory.Client, key, observationKey, patientKey, project, generation string) { t.Helper() document, err := json.Marshal(map[string]any{ "_key": key, "_from": "Observation/" + observationKey, "_to": "Patient/" + patientKey, @@ -1048,7 +1048,7 @@ func assertExpandedCodingGroupOwnerValues(t *testing.T, raw any, want []string) func assertExpandedCodingGroupLineage( t *testing.T, ctx context.Context, - client *store.Client, + client *arangofactory.Client, compiled lower.CompiledRecipeOutput, row map[string]any, project, patientKey string, @@ -1058,7 +1058,7 @@ func assertExpandedCodingGroupLineage( ) { t.Helper() rowID, _ := row["__loom_row_id"].(string) - query, err := CompileRowLineageOutput(compiled, rowID, 0, 20, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRowLineageOutput(ArangoTarget(), compiled, rowID, 0, 20, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile expanded Coding Group lineage for %q: %v", row["coding"], err) } diff --git a/internal/dataframe/compiler/ir/clone.go b/internal/dataframe/compiler/ir/clone.go index 6f7ea64f4..6c27ff8e5 100644 --- a/internal/dataframe/compiler/ir/clone.go +++ b/internal/dataframe/compiler/ir/clone.go @@ -254,9 +254,6 @@ func ClonePhysicalSubplan(subplan PhysicalSubplan) PhysicalSubplan { return clonePhysicalSubplan(subplan) } -// CanonicalExecutionPhysicalPlan returns the renderer-relevant plan contract. -// Compiler diagnostics and semantic source locations are deliberately removed: -// they explain a plan but do not change what the backend executes. func CanonicalExecutionPhysicalPlan(plan PhysicalPlan) PhysicalPlan { out := clonePhysicalPlan(plan) out.Source = PhysicalSource{} diff --git a/internal/dataframe/compiler/ir/dataset_generation.go b/internal/dataframe/compiler/ir/dataset_generation.go index f5ddff782..bf3ce4899 100644 --- a/internal/dataframe/compiler/ir/dataset_generation.go +++ b/internal/dataframe/compiler/ir/dataset_generation.go @@ -11,10 +11,6 @@ func normalizeDatasetGeneration(generation string) string { return strings.TrimSpace(generation) } -// datasetGenerationBindValue makes the absent-generation case explicit for -// every AQL renderer. Binding nil and using equality yields -// `dataset_generation == null`, so legacy rows are isolated from all -// generation-qualified rows by default. func datasetGenerationBindValue(generation string) any { generation = normalizeDatasetGeneration(generation) if generation == "" { diff --git a/internal/dataframe/compiler/ir/diagnostics.go b/internal/dataframe/compiler/ir/diagnostics.go index be867d7f7..32c3efa18 100644 --- a/internal/dataframe/compiler/ir/diagnostics.go +++ b/internal/dataframe/compiler/ir/diagnostics.go @@ -12,13 +12,7 @@ func valueString(value any) string { return fmt.Sprint(value) } -// CompilerPlanDiagnostics describes work the physical renderer will ask AQL to -// perform. It deliberately reports compiler facts, not estimated database -// cost: use it alongside Arango PROFILE to decide which rewrite is worthwhile. type CompilerPlanDiagnostics struct { - // Fingerprint is a deterministic structural identity for the validated - // physical plan. It is safe to expose in Explain because it is a digest, - // never the plan's bind values or authorization paths. Fingerprint string TraversalSets int EndpointTraversalCount int @@ -35,17 +29,9 @@ type CompilerPlanDiagnostics struct { ExpressionBindingCount int SharedKeyedMapCount int ObjectLookupConsumerCount int - // OptimizationPolicy is the explainable decision record for optional - // physical rewrites. It reports both enabled rewrites and conservative - // rejections so rendered AQL is never the only evidence of a decision. - OptimizationPolicy PhysicalOptimizationReport + OptimizationPolicy PhysicalOptimizationReport } -// PhysicalTraversalDecision exposes the route strategy selected by the typed -// physical compiler. It is deliberately metadata-only: no user values or raw -// AQL are included. Native decisions explain whether endpoint lowering was -// disabled by policy or rejected because its validated storage contract was -// unavailable. type PhysicalTraversalDecision struct { SourceVariable string TargetVariable string @@ -59,10 +45,6 @@ type PhysicalTraversalDecision struct { Reason string } -// RichSourceReuse identifies a materialized relationship set that is scanned -// repeatedly by rich projections. A high count does not mean the traversal is -// repeated; it means aggregate/pivot/slice operations each loop over the same -// already-materialized child set. type RichSourceReuse struct { SourceSet string AggregateConsumers int @@ -70,11 +52,6 @@ type RichSourceReuse struct { SliceConsumers int } -// RichConsumerGroup is a renderer-independent compatibility classification -// for rich expressions over one materialized set. A group is eligible only -// when its complete semantic expression signature is identical; source reuse -// alone is not enough to fuse predicates, ordering, pivot reduction, or slice -// limits safely. type RichConsumerGroup struct { SourceSet string Kind PhysicalExpressionKind @@ -292,9 +269,6 @@ func collectRichConsumerGroups(expression *PhysicalExpression, groups map[string groups[key] = &RichConsumerGroup{SourceSet: source, Kind: expression.Kind, Signature: signature, Consumers: 1} } -// physicalTraversalOpportunityKey intentionally ignores scoped filters and -// semantic provenance. It identifies broader neighbor traversals that could -// serve multiple typed children once the scope-safe rewrite exists. func physicalTraversalOpportunityKey(plan PhysicalPlan, set PhysicalSet, setIndex int) string { traversal := set.Subplan.Operations[0].Traversal if traversal == nil { @@ -302,9 +276,6 @@ func physicalTraversalOpportunityKey(plan PhysicalPlan, set PhysicalSet, setInde } unnestScope, err := physicalUnnestScopeIdentityAt(plan, setIndex) if err != nil { - // Diagnostics must remain total even for a malformed candidate. The - // malformed scope is deliberately isolated from valid candidates rather - // than silently grouping it with the root scope. unnestScope = "invalid-unnest-scope" } return traversal.SourceVariable + "|" + string(traversal.Direction) + "|" + valueString(plan.BindVars[traversal.EdgeCollectionBindKey]) + "|" + valueString(plan.BindVars[traversal.EdgeLabelBindKey]) + "|" + traversal.EdgeTargetTypeField + "|unnest=" + unnestScope diff --git a/internal/dataframe/compiler/ir/doc.go b/internal/dataframe/compiler/ir/doc.go index f44bc53dd..0d5883581 100644 --- a/internal/dataframe/compiler/ir/doc.go +++ b/internal/dataframe/compiler/ir/doc.go @@ -1,3 +1 @@ -// Package ir contains the typed physical operation graph and its scope, -// variable, bind, and optimization-diagnostic invariants. package ir diff --git a/internal/dataframe/compiler/ir/physical_combine.go b/internal/dataframe/compiler/ir/physical_combine.go index c2aa3e3c9..15b989cd7 100644 --- a/internal/dataframe/compiler/ir/physical_combine.go +++ b/internal/dataframe/compiler/ir/physical_combine.go @@ -54,9 +54,6 @@ type PhysicalCombineOutputColumn struct { Repeated bool } -// PhysicalClickHouseCombine is the closed semantic/physical contract for a -// constructed-table operation. Published table names and authorization are -// request scoped and are attached only after exact revision resolution. type PhysicalClickHouseCombine struct { Kind PhysicalCombineKind Inputs []PhysicalCombineInputRef @@ -72,22 +69,14 @@ func (combine PhysicalClickHouseCombine) Validate() error { return combine.validate(false, false) } -// ValidateWithPrivateStage is used only by a typed composite physical plan. -// Standalone combines continue to require exact published table revisions. func (combine PhysicalClickHouseCombine) ValidateWithPrivateStage() error { return combine.validate(true, false) } -// ValidateForWorkspaceCompilation accepts typed sibling-output refs after the -// workspace resolver has supplied their schemas. It does not authorize -// execution; ordinary Validate deliberately rejects those unresolved refs. func (combine PhysicalClickHouseCombine) ValidateForWorkspaceCompilation() error { return combine.validate(false, true) } -// ValidateWithWorkspaceArtifacts validates a typed terminal Combine that will -// resolve same-bundle workspace references through server-owned artifacts. -// Standard Validate deliberately continues to reject unresolved references. func (combine PhysicalClickHouseCombine) ValidateWithWorkspaceArtifacts() error { if err := combine.validate(false, true); err != nil { return fmt.Errorf("validate workspace artifact combine: %w", err) @@ -257,11 +246,24 @@ func (combine PhysicalClickHouseCombine) validate(allowPrivateStage, allowWorksp return nil } -// PhysicalClickHousePrefix binds a terminal ClickHouse Combine input to the -// exact last typed AQL stage. A restricted prefix is supported only for one -// immutable authorization path; the renderer emits that bound path as hidden -// row metadata instead of guessing it from result rows. +type PhysicalPrefixSourceBackend string + +const ( + PhysicalPrefixSourceAQL PhysicalPrefixSourceBackend = "AQL" + PhysicalPrefixSourcePostgreSQL PhysicalPrefixSourceBackend = "POSTGRESQL" +) + +func (backend PhysicalPrefixSourceBackend) Validate() error { + switch backend { + case PhysicalPrefixSourceAQL, PhysicalPrefixSourcePostgreSQL: + return nil + default: + return fmt.Errorf("private ClickHouse prefix requires an explicit supported source backend, got %q", string(backend)) + } +} + type PhysicalClickHousePrefix struct { + SourceBackend PhysicalPrefixSourceBackend StageID string AuthScopeMode string AuthResourcePaths []string @@ -270,8 +272,40 @@ type PhysicalClickHousePrefix struct { } func (prefix PhysicalClickHousePrefix) Validate(sequence *PhysicalStageSequence, bindVars map[string]any) error { + switch prefix.SourceBackend { + case PhysicalPrefixSourceAQL: + return prefix.validateSourceAQL(sequence, bindVars) + case PhysicalPrefixSourcePostgreSQL: + return prefix.validateSourcePostgreSQL(sequence, bindVars) + default: + return prefix.SourceBackend.Validate() + } +} + +func (prefix PhysicalClickHousePrefix) validateSourceAQL(sequence *PhysicalStageSequence, bindVars map[string]any) error { + if err := prefix.validateShared(sequence, bindVars, "AQL"); err != nil { + return err + } + if prefix.AuthResourcePathBindKey == "" { + return nil + } + value, ok := bindVars[prefix.AuthResourcePathBindKey].(string) + if !ok || value != prefix.AuthResourcePaths[0] { + return fmt.Errorf("private ClickHouse prefix authorization bind does not match its exact scope") + } + return nil +} + +func (prefix PhysicalClickHousePrefix) validateSourcePostgreSQL(sequence *PhysicalStageSequence, bindVars map[string]any) error { + if prefix.AuthScopeMode != "unrestricted" { + return fmt.Errorf("restricted private ClickHouse prefixes are unsupported on the PostgreSQL source backend: the composed PostgreSQL renderer has no registered bound authorization path column") + } + return prefix.validateShared(sequence, bindVars, "PostgreSQL") +} + +func (prefix PhysicalClickHousePrefix) validateShared(sequence *PhysicalStageSequence, bindVars map[string]any, backend string) error { if sequence == nil || strings.TrimSpace(prefix.StageID) == "" || prefix.StageID != sequence.FinalStageID { - return fmt.Errorf("private ClickHouse prefix must reference the exact final AQL stage") + return fmt.Errorf("private ClickHouse prefix must reference the exact final %s construction stage", backend) } if sequence.CellTraceReturn != nil { return fmt.Errorf("cell-trace plans cannot feed a private ClickHouse Combine") @@ -286,7 +320,7 @@ func (prefix PhysicalClickHousePrefix) Validate(sequence *PhysicalStageSequence, unrestricted, unrestrictedOK := bindVars[physicalScopeAuthPathsUnrestrictedBind].(bool) allowed, allowedOK := bindVars[physicalScopeAllowedBind].(bool) if !pathsOK || !samePhysicalStrings(paths, prefix.AuthResourcePaths) || !unrestrictedOK || unrestricted != (prefix.AuthScopeMode == "unrestricted") || !allowedOK || !allowed { - return fmt.Errorf("private ClickHouse prefix scope differs from the AQL authorization bindings") + return fmt.Errorf("private ClickHouse prefix scope differs from the %s authorization bindings", backend) } for _, column := range sequence.FinalColumns { if column.Name == "auth_resource_path" || column.Name == "project_id" { @@ -298,13 +332,6 @@ func (prefix PhysicalClickHousePrefix) Validate(sequence *PhysicalStageSequence, return fmt.Errorf("%s prefixes cannot feed a private ClickHouse Combine until authorization scope is preserved by grouping", stage.Kind) } } - if prefix.AuthResourcePathBindKey == "" { - return nil - } - value, ok := bindVars[prefix.AuthResourcePathBindKey].(string) - if !ok || value != prefix.AuthResourcePaths[0] { - return fmt.Errorf("private ClickHouse prefix authorization bind does not match its exact scope") - } return nil } @@ -370,8 +397,6 @@ func validatePhysicalCombineKeys(keys []PhysicalCombineKey) error { return nil } -// ResolvedClickHouseTable is populated by the exact revision resolver after -// project, output, schema, scope, and authorization checks succeed. type ResolvedClickHouseTable struct { TableID string RevisionID string @@ -393,9 +418,6 @@ type ResolvedClickHouseTable struct { ScopeEvidenceDigest string } -// ClickHouseArtifactIdentity carries the exact identity used by the private -// artifact writer. Expected and actual values are compared in full before the -// artifact can become a ClickHouse input. type ClickHouseArtifactIdentity struct { ExecutionID string OutputID string diff --git a/internal/dataframe/compiler/ir/physical_combine_scalar.go b/internal/dataframe/compiler/ir/physical_combine_scalar.go index 048faae01..f1863ac6f 100644 --- a/internal/dataframe/compiler/ir/physical_combine_scalar.go +++ b/internal/dataframe/compiler/ir/physical_combine_scalar.go @@ -4,9 +4,6 @@ import ( "strings" ) -// ClickHouseCombineScalarBaseType returns the scalar base type accepted for a -// combine key or append input. KEY_JOIN, MEMBERSHIP, and APPEND accept nullable -// scalars; Membership's ordinary equality treats NULL as a nonmatch. func ClickHouseCombineScalarBaseType(value string, kind PhysicalCombineKind) (string, bool) { physical := strings.TrimSpace(value) if physical == "" { diff --git a/internal/dataframe/compiler/ir/physical_combine_test.go b/internal/dataframe/compiler/ir/physical_combine_test.go index 945d59e78..9d584cd2e 100644 --- a/internal/dataframe/compiler/ir/physical_combine_test.go +++ b/internal/dataframe/compiler/ir/physical_combine_test.go @@ -1,6 +1,7 @@ package ir import ( + "strconv" "strings" "testing" ) @@ -138,3 +139,106 @@ func TestPhysicalClickHouseCombineRejectsAQLPayloadAndUnpinnedInputs(t *testing. t.Fatalf("missing ClickHouse payload error = %v", err) } } + +func prefixDispatchSequence() *PhysicalStageSequence { + return &PhysicalStageSequence{ + SourceStageID: "source_projection", FinalStageID: "final_stage", + FinalRowIdentity: "__loom_row_id", + Stages: []PhysicalConstructionStage{{ID: "final_stage", Kind: PhysicalStageFilterOp}}, + FinalColumns: []PhysicalStageColumn{ + {ID: "__loom_row_id", Name: "__loom_row_id", Kind: "string", Internal: true}, + {ID: "value", Name: "value", Kind: "integer"}, + }, + } +} + +func prefixDispatchBinds(paths []string, unrestricted bool) map[string]any { + return map[string]any{ + physicalScopeAuthPathsBind: append([]string(nil), paths...), + physicalScopeAuthPathsUnrestrictedBind: unrestricted, + physicalScopeAllowedBind: true, + "construction_private_auth_resource_path": "/programs/p1", + } +} + +func TestPhysicalClickHousePrefixRequiresExplicitSourceBackend(t *testing.T) { + sequence := prefixDispatchSequence() + binds := prefixDispatchBinds(nil, true) + untagged := PhysicalClickHousePrefix{StageID: "final_stage", AuthScopeMode: "unrestricted"} + if err := untagged.Validate(sequence, binds); err == nil || !strings.Contains(err.Error(), "explicit supported source backend") { + t.Fatalf("untagged prefix error = %v", err) + } + unknown := untagged + unknown.SourceBackend = PhysicalPrefixSourceBackend("CLICKHOUSE") + if err := unknown.Validate(sequence, binds); err == nil || !strings.Contains(err.Error(), "explicit supported source backend") { + t.Fatalf("unknown-tag prefix error = %v", err) + } + if err := PhysicalPrefixSourceAQL.Validate(); err != nil { + t.Fatalf("AQL tag rejected: %v", err) + } + if err := PhysicalPrefixSourcePostgreSQL.Validate(); err != nil { + t.Fatalf("PostgreSQL tag rejected: %v", err) + } +} + +func TestPhysicalClickHousePrefixDispatchesBothBackendsOnSharedContract(t *testing.T) { + for _, backend := range []PhysicalPrefixSourceBackend{PhysicalPrefixSourceAQL, PhysicalPrefixSourcePostgreSQL} { + prefix := PhysicalClickHousePrefix{SourceBackend: backend, StageID: "final_stage", AuthScopeMode: "unrestricted"} + if err := prefix.Validate(prefixDispatchSequence(), prefixDispatchBinds(nil, true)); err != nil { + t.Fatalf("%s unrestricted prefix: %v", backend, err) + } + if backend == PhysicalPrefixSourceAQL { + restricted := PhysicalClickHousePrefix{ + SourceBackend: backend, StageID: "final_stage", AuthScopeMode: "restricted", + AuthResourcePaths: []string{"/programs/p1"}, IncludeAuthResourcePath: true, + AuthResourcePathBindKey: "construction_private_auth_resource_path", + } + if err := restricted.Validate(prefixDispatchSequence(), prefixDispatchBinds([]string{"/programs/p1"}, false)); err != nil { + t.Fatalf("AQL restricted prefix: %v", err) + } + } + } +} + +func TestPhysicalClickHousePrefixRefusesRestrictedPostgreSQLPrefix(t *testing.T) { + prefix := PhysicalClickHousePrefix{ + SourceBackend: PhysicalPrefixSourcePostgreSQL, StageID: "final_stage", AuthScopeMode: "restricted", + AuthResourcePaths: []string{"/programs/p1"}, IncludeAuthResourcePath: true, + AuthResourcePathBindKey: "construction_private_auth_resource_path", + } + err := prefix.Validate(prefixDispatchSequence(), prefixDispatchBinds([]string{"/programs/p1"}, false)) + if err == nil || !strings.Contains(err.Error(), "unsupported on the PostgreSQL source backend") { + t.Fatalf("restricted PostgreSQL prefix error = %v", err) + } + if err := prefix.ValidateScope(); err != nil { + t.Fatalf("restricted scope shape must stay well-formed for the AQL backend: %v", err) + } +} + +func TestPhysicalClickHousePrefixPostgreSQLScopeGateIsAnAllowlist(t *testing.T) { + for _, mode := range []string{"Restricted", "RESTRICTED", " restricted", "restricted ", "restricted\u00a0", ""} { + t.Run("mode="+strconv.Quote(mode), func(t *testing.T) { + prefix := PhysicalClickHousePrefix{ + SourceBackend: PhysicalPrefixSourcePostgreSQL, StageID: "final_stage", AuthScopeMode: mode, + AuthResourcePaths: []string{"/programs/p1"}, IncludeAuthResourcePath: true, + AuthResourcePathBindKey: "construction_private_auth_resource_path", + } + err := prefix.Validate(prefixDispatchSequence(), prefixDispatchBinds([]string{"/programs/p1"}, false)) + if err == nil || !strings.Contains(err.Error(), "unsupported on the PostgreSQL source backend") { + t.Fatalf("non-canonical restricted mode %q on PostgreSQL error = %v, want the PostgreSQL restricted refusal", mode, err) + } + }) + } +} + +func TestPhysicalClickHousePrefixBackendMismatchUsesBackendNamedContract(t *testing.T) { + prefix := PhysicalClickHousePrefix{SourceBackend: PhysicalPrefixSourcePostgreSQL, StageID: "final_stage", AuthScopeMode: "unrestricted"} + if err := prefix.Validate(prefixDispatchSequence(), prefixDispatchBinds(nil, false)); err == nil || !strings.Contains(err.Error(), "differs from the PostgreSQL authorization bindings") { + t.Fatalf("PostgreSQL scope mismatch error = %v", err) + } + mismatchedStage := prefix + mismatchedStage.StageID = "different_stage" + if err := mismatchedStage.Validate(prefixDispatchSequence(), prefixDispatchBinds(nil, true)); err == nil || !strings.Contains(err.Error(), "exact final PostgreSQL construction stage") { + t.Fatalf("PostgreSQL stage mismatch error = %v", err) + } +} diff --git a/internal/dataframe/compiler/ir/physical_operations.go b/internal/dataframe/compiler/ir/physical_operations.go index 970359a81..b548eb33e 100644 --- a/internal/dataframe/compiler/ir/physical_operations.go +++ b/internal/dataframe/compiler/ir/physical_operations.go @@ -8,35 +8,18 @@ import ( const PhysicalCodedPivotSourcePayloadColumn = "__loom_coded_pivot_source_payload" -// PhysicalPlan is the renderer-independent AQL operation graph produced after -// semantic planning. Operations are ordered because AQL variables have lexical -// scope: an operation may reference only variables introduced before it. type PhysicalSet struct { - Variable string - Kind PhysicalSetKind - Subplan PhysicalSubplan - Unique bool - // Output describes a compact, identity-safe projection of each set item. - // A nil output preserves the full stored document for shared traversal or - // other consumers that have not proved a smaller contract. - Output *PhysicalSetOutput - // Projection describes selector values computed in the original child-set - // subquery. It replaces the payload-bearing second prepared array when all - // downstream consumers have a projection-safe selector contract. - Projection *PhysicalSetProjection - // SourceSetVariable is set for a typed subset over an already materialized - // shared traversal. Such a set does not begin with TRAVERSAL; ItemVariable - // is bound by the renderer while iterating SourceSetVariable. + Variable string + Kind PhysicalSetKind + Subplan PhysicalSubplan + Unique bool + Output *PhysicalSetOutput + Projection *PhysicalSetProjection SourceSetVariable string ItemVariable string - // SortByKey makes the set's node order part of physical semantics. Optional - // relationship materialization must not rely on Arango traversal order. - SortByKey bool - // Reduction contains set-level reductions for direct child projections. It - // is intentionally separate from the identity-bearing set because nested - // traversals still consume every matching child. - Reduction *PhysicalSetReduction - Prepared *PhysicalPreparedSet + SortByKey bool + Reduction *PhysicalSetReduction + Prepared *PhysicalPreparedSet } type PhysicalSetKind string @@ -46,16 +29,12 @@ const ( PhysicalNodePathSetKind PhysicalSetKind = "NODE_PATH" ) -// PhysicalPathNode is a public path node backed by a stored document. Value is -// typed so path lowering cannot smuggle AQL text into the renderer. type PhysicalPathNode struct { Alias string ResourceType string Value PhysicalValue } -// PhysicalPathRelationship carries relationship metadata while keeping raw -// edge documents private to the compiler/runtime. type PhysicalPathRelationship struct { Alias string LabelBindKey string @@ -69,9 +48,6 @@ type PhysicalPathSeed struct { RouteOrder int } -// PhysicalPathExtend appends one depth-one traversal to an existing path -// set. SourcePath identifies the terminal stored document inside each source -// item (empty means the item itself). MatchMode is REQUIRED or OPTIONAL. type PhysicalPathExtend struct { Variable string SourceVariable string @@ -81,9 +57,7 @@ type PhysicalPathExtend struct { Relationship PhysicalPathRelationship MatchMode string RouteOrder int - // Scope contains typed edge/target filters and authorization operations - // evaluated inside the correlated traversal subquery. - Scope []PhysicalOperation + Scope []PhysicalOperation } type PhysicalGraphReturn struct { @@ -91,12 +65,6 @@ type PhysicalGraphReturn struct { LimitBindKey string } -// PhysicalPopulationMappingReturn is an internal final-row witness terminal. -// Members must be the matched selected-member array for the current root row. -// IdentityParts preserve the ordered default identity inputs; ExplicitIdentity -// carries a recipe-defined __loom_row_id expression when present. The -// renderer emits one witness row per (member, identity) pair without changing -// the ordinary return schema. type PhysicalPopulationMappingReturn struct { Members PhysicalExpression IdentityParts []PhysicalPopulationMappingIdentityPart @@ -108,10 +76,6 @@ type PhysicalPopulationMappingIdentityPart struct { Expression PhysicalExpression } -// PhysicalCellTraceReturn is an internal evidence terminal. Value is the -// exact final projection expression. The renderer derives bounded source -// contributors only from this closed expression tree; it never evaluates the -// FHIR document independently of the compiled plan. type PhysicalCellTraceReturn struct { Value PhysicalExpression Contribution *PhysicalCellTraceContribution @@ -123,21 +87,12 @@ type PhysicalCellTraceReturn struct { LimitBindKey string FetchLimitBindKey string OmissionCode string - // RowExists indicates that the terminal is compiled over an existing typed - // output row, so contributor availability does not determine value status. - RowExists bool + RowExists bool } -// PhysicalCellTraceConstruction describes value lineage through an ordered -// construction sequence. The renderer evaluates InputColumns on the selected -// final row and attaches the stable stage and column identities to each item. type PhysicalCellTraceConstruction struct { FinalStageID string RowIdentityColumn string - // RowIdentityFields preserve the ordered public identity contract used by - // Preview. Construction stages can keep a narrower internal row identity - // column (for example _key) while the public identity also includes the - // project bind. RowIdentityFields []string OutputColumnID string OutputColumn string @@ -149,9 +104,6 @@ type PhysicalCellTraceConstruction struct { OmissionCode string } -// PhysicalCellTraceConstructionInput identifies a column read by the -// construction step that produced the requested cell. FinalValueColumn is the -// surviving name for that stable column ID in the final stage schema. type PhysicalCellTraceConstructionInput struct { StageID string ColumnID string @@ -159,8 +111,6 @@ type PhysicalCellTraceConstructionInput struct { FinalValueColumn string } -// PhysicalCellTraceRelatedSource replays the compiler-owned route subplan for -// the exact final row. Each traversed path remains one contribution occurrence. type PhysicalCellTraceRelatedSource struct { InputRowVariable string AnchorColumn string @@ -168,8 +118,6 @@ type PhysicalCellTraceRelatedSource struct { Subplan PhysicalSubplan } -// PhysicalTableShapeExclusionReturn emits source-level exclusions from the -// input rows of one compiler-validated grouped pivot. type PhysicalTableShapeExclusionReturn struct { Pivot PhysicalGroupedPivot OffsetBindKey string @@ -177,9 +125,6 @@ type PhysicalTableShapeExclusionReturn struct { FetchLimitBindKey string } -// PhysicalCellTraceReshape carries the exact source-cell lineage needed to -// explain a table-shape output. It exists only on the diagnostic terminal; -// ordinary dataframe rows and schemas remain unchanged. type PhysicalCellTraceReshape struct { OutputVariable string Sources []PhysicalCellTraceReshapeSource @@ -195,10 +140,6 @@ const ( PhysicalCellTraceUnpivotField PhysicalCellTraceReshapeSourceKind = "UNPIVOT_PASSTHROUGH" ) -// PhysicalCellTraceReshapeSource identifies one source expression that -// contributes to the requested reshaped cell. Pivot sources use the grouped -// rows unless lowering proves one input row per group; unpivot sources are -// carried on the emitted row. type PhysicalCellTraceReshapeSource struct { Kind PhysicalCellTraceReshapeSourceKind GroupRowsVariable string @@ -230,9 +171,6 @@ const ( PhysicalCellTraceDerivedLiteralOnly = "TABLE_SHAPE_DERIVED_HAS_NO_SOURCE_CELL" ) -// PhysicalCellTraceContribution points at the pre-reduction projected set -// already produced by canonical lowering. ValueField is compiler-generated, -// never supplied by a request. type PhysicalCellTraceContribution struct { SetVariable string ValueField string @@ -271,8 +209,6 @@ const ( PhysicalPopulationMappingExplicitIdentityField = "__loom_population_explicit_identity" ) -// PhysicalUnnestEmptyPolicy controls null/empty source behavior. The renderer -// owns the AQL shape; this IR never stores a query fragment. type PhysicalUnnestEmptyPolicy string const ( @@ -281,9 +217,6 @@ const ( PhysicalUnnestPreserveParent PhysicalUnnestEmptyPolicy = "PRESERVE_PARENT" ) -// PhysicalUnnestOwner describes the exact root-to-owner route evaluated before -// the selected repeated value is expanded. An empty Route means the owner is -// the root document. type PhysicalUnnestOwner struct { OccurrenceID string ResourceType string @@ -298,17 +231,11 @@ type PhysicalUnnestRouteStep struct { Scope []PhysicalOperation } -// PhysicalUnnestAncestor retains one repeated item traversed on the way to -// the selected item. It lets correlated projections read sibling fields from -// the exact enclosing item that produced each expanded row. type PhysicalUnnestAncestor struct { StepIndex int Variable string } -// PhysicalUnnest is the single enriched cardinality boundary. It carries the -// typed owner route, source, empty policy, compiler-owned item evidence, and -// any repeated ancestors needed to keep nested projections correlated. type PhysicalUnnest struct { Owner PhysicalUnnestOwner OutputVariable string @@ -319,9 +246,6 @@ type PhysicalUnnest struct { EmptyPolicy PhysicalUnnestEmptyPolicy } -// PhysicalUnnestSortKeys is the compiler-owned stable ordering tuple for an -// expanded row window. Route edges are preserved in route order so two graph -// paths to the same owner remain distinct ordered witnesses. func PhysicalUnnestSortKeys(unnest PhysicalUnnest) []PhysicalValue { keys := []PhysicalValue{{Variable: unnest.Owner.RootVariable, Path: []string{"_key"}}} for _, step := range unnest.Owner.Route { @@ -339,10 +263,6 @@ func PhysicalUnnestSortKeys(unnest PhysicalUnnest) []PhysicalValue { return keys } -// PhysicalSetProjection is a single-materialization selector projection. The -// fields are arrays because selector evaluation preserves repeated FHIR -// values; scalar consumers apply their normal FIRST/FLATTEN semantics when -// reading the projected field. type PhysicalSetProjection struct { Fields []PhysicalSetProjectionField } @@ -355,9 +275,6 @@ type PhysicalSetProjectionField struct { Demand PhysicalSelectorValueDemand } -// PhysicalSelectorValueDemand describes how much of a selector's result a -// projected set must retain. The zero value preserves every value so plans -// built without demand analysis remain conservative. type PhysicalSelectorValueDemand string const ( @@ -365,10 +282,6 @@ const ( PhysicalSelectorFirstValue PhysicalSelectorValueDemand = "FIRST_ONLY" ) -// PhysicalSetReduction is the typed contract for reducing selector slots -// after a set has been materialized. The renderer owns the AQL spelling; the -// IR only describes which projected slot supplies each result and which -// cardinality-preserving reduction applies. type PhysicalSetReduction struct { Variable string SourceSetVariable string @@ -389,10 +302,6 @@ const ( PhysicalSetReductionDistinct PhysicalSetReductionMode = "DISTINCT" ) -// PhysicalSetOutputField names the only stored properties that may survive a -// compact set projection. The graph identity fields preserve nested traversal -// and duplicate-edge semantics; payload is retained only when a downstream -// selector or rich consumer needs it. type PhysicalSetOutputField string const ( @@ -411,25 +320,16 @@ type PhysicalSubplan struct { Captures []string Operations []PhysicalOperation Return PhysicalExpression - // Sort makes an array-valued subplan projection deterministic. It is - // validated in the subplan's local scope and is absent from EXISTS plans. - Sort *PhysicalValue - Unique bool - // DistinctBy groups projected values by one exact identity while preserving - // equal values from different identities. It is used for related ALL forms. + Sort *PhysicalValue + Unique bool DistinctBy *PhysicalValue } -// PhysicalCollectionScan reads a compiler-provided collection inside a -// correlated subplan. A keyed scan iterates one compiler-proven document-key -// set and looks up each document directly instead of scanning the collection. type PhysicalCollectionScan struct { Variable string CollectionBindKey string } -// PhysicalKeySetLookup iterates an exact compiler-owned set of document keys -// and resolves each document from one compiler-pinned collection. type PhysicalKeySetLookup struct { Variable string KeyVariable string @@ -437,9 +337,6 @@ type PhysicalKeySetLookup struct { Keys PhysicalValue } -// PhysicalDocumentLookup resolves an exact hidden _id in one compiler-chosen -// resource collection. The renderer verifies the resolved _id still equals -// the captured value before later scope filters or traversals can use it. type PhysicalDocumentLookup struct { Variable string CollectionBindKey string @@ -447,12 +344,8 @@ type PhysicalDocumentLookup struct { } type PhysicalPredicate struct { - Operator string - Left PhysicalValue - // LeftExpression lets a comparison consume a typed selector extraction. - // Exactly one of Left and LeftExpression is present. This keeps user - // filters out of AQL-shaped strings while the scope predicates can retain - // their compact value-only form. + Operator string + Left PhysicalValue LeftExpression *PhysicalExpression Right *PhysicalValue Quantifier spec.ArrayQuantifier @@ -470,9 +363,6 @@ const ( PhysicalExistsPredicate PhysicalPredicateKind = "EXISTS" ) -// PhysicalPredicateExpression is the typed predicate tree used by rich -// physical operations. Exists contains a bounded correlated subplan; it is -// not a string-shaped LENGTH(FOR ...) compatibility escape hatch. type PhysicalPredicateExpression struct { Kind PhysicalPredicateKind Comparison *PhysicalPredicate @@ -481,61 +371,38 @@ type PhysicalPredicateExpression struct { } type PhysicalFilter struct { - // Predicate is retained for the frozen navigation plan. New lowering must - // use Expression so compound and existence predicates stay typed. Predicate PhysicalPredicate Expression *PhysicalPredicateExpression } -// PhysicalDerivedLet names a derived value. Operator is a compiler-owned -// symbolic operation (for example UNIQUE or LENGTH), never raw AQL. type PhysicalDerivedLet struct { Variable string Operator string Inputs []PhysicalValue } -// PhysicalExpressionLet binds a deterministic expression once in the -// current root-row scope. It is distinct from symbolic derived operators. type PhysicalExpressionLet struct { Variable string Expression PhysicalExpression } -// PhysicalSort carries one or more typed keys for a deterministic execution -// window. Key order is semantic: expanded rows include their occurrence and -// owner-relative item witnesses after the root identity. type PhysicalSort struct { Keys []PhysicalValue } -// PhysicalLimit references a positive integer bind value. Keeping the value -// in BindVars retains the same parameterized execution boundary as filters. type PhysicalLimit struct { BindKey string } type PhysicalProjection struct { - Name string - // Hidden projections are returned by the backend for executor-side - // validation (for example dynamic runtime key metadata) but are omitted - // from public dataframe columns after post-query materialization. - Hidden bool - Value PhysicalValue - Expression *PhysicalExpression - // Presence preserves whether a source property existed before projection. - // It is metadata for row operations such as grouped pivot and is not itself - // emitted in the ordinary public projection. - Presence *PhysicalProjectionPresence - // PresenceOutput emits the exact Presence proof as this projection's boolean - // value. Construction stages use it only for compiler-owned hidden sidecars. + Name string + Hidden bool + Value PhysicalValue + Expression *PhysicalExpression + Presence *PhysicalProjectionPresence PresenceOutput bool } -// PhysicalProjectionPresence is a closed property-presence proof for one or -// more concrete selector paths rooted at the same physical value. Any path -// being present makes the projected value present (for example, a selector -// fallback that supplies the final value). type PhysicalProjectionPresence struct { Source PhysicalValue Paths [][]string @@ -545,26 +412,17 @@ type PhysicalReturn struct { Projections []PhysicalProjection } -// PhysicalGroupedPivot is a typed row operation over one materialized base -// row projection. COLLECT forms groups; categories remain frozen values and -// never become query fragments. type PhysicalGroupedPivot struct { - ConstructionID string - InputRowVariable string - GroupRowsVariable string - OutputRowVariable string - // OneInputRowPerGroup is set only when lowering proves that a direct root - // resource ID is among the group keys and the source plan cannot multiply - // root rows. Renderers can then emit one pivot row per input without COLLECT. - OneInputRowPerGroup bool - InputProjections []PhysicalProjection - GroupKeys []PhysicalGroupedPivotKey - CategoryColumn string - CategoryPresenceColumn string - CategoryPresence *PhysicalProjectionPresence - // CategoryPresenceFromInput marks CategoryPresenceColumn as an already - // materialized boolean sidecar in InputProjections instead of a name for a - // renderer-generated presence projection. + ConstructionID string + InputRowVariable string + GroupRowsVariable string + OutputRowVariable string + OneInputRowPerGroup bool + InputProjections []PhysicalProjection + GroupKeys []PhysicalGroupedPivotKey + CategoryColumn string + CategoryPresenceColumn string + CategoryPresence *PhysicalProjectionPresence CategoryPresenceFromInput bool CategoryType string ValueColumn string @@ -575,27 +433,18 @@ type PhysicalGroupedPivot struct { RootContributorInputMany bool RootContributorOutputColumn string RootContributorVariable string - // CodedCorrelation is present for construction CODED_PIVOT. The renderer - // substitutes each coded category's bound system/code pair into this one - // checked owner/key/value binding, keeping category identity and value in - // the same owner scope. - CodedCorrelation *PhysicalCorrelation - CodedCategories []PhysicalGroupedCodedPivotCategory - // CodedSourceVariable is the selected direct root scan variable. The first - // construction stage inlines that scan so it never materializes full root - // payloads into an intermediate array. - CodedSourceVariable string - ConstructionIDBindKey string - UnlistedEvidenceColumn string - DuplicatePolicy string - MissingCellPolicy string - UnlistedCategoryPolicy string + CodedCorrelation *PhysicalCorrelation + CodedCategories []PhysicalGroupedCodedPivotCategory + CodedSourceVariable string + ConstructionIDBindKey string + UnlistedEvidenceColumn string + DuplicatePolicy string + MissingCellPolicy string + UnlistedCategoryPolicy string } type PhysicalGroupedPivotKey struct { - Column string - // Output permits a construction stage to rename a pass-through group key - // while still resolving the key against the preceding stage's schema. + Column string Output string Variable string Kind string @@ -623,9 +472,6 @@ const ( PhysicalPivotCategoryMissingMatch PhysicalPivotCategoryMatchKind = "MISSING" ) -// PhysicalUnpivot is a terminal cardinality-changing row operation. Input -// order defines emitted key/value rows; identity columns are copied from the -// input row before the selected value key is appended. type PhysicalUnpivot struct { ConstructionID string InputRowVariable string @@ -643,8 +489,6 @@ type PhysicalUnpivot struct { NullRowPolicy string } -// PhysicalUnpivotOutput preserves one input value under its stage output -// name. Empty PreservedOutputs retains the input name for legacy reshapes. type PhysicalUnpivotOutput struct { InputColumn string OutputColumn string @@ -667,5 +511,3 @@ var ( physicalBindKeyPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*$`) physicalPathPartPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) ) - -// Validate enforces the frozen physical-plan invariants without rendering AQL. diff --git a/internal/dataframe/compiler/ir/physical_stage_sequence.go b/internal/dataframe/compiler/ir/physical_stage_sequence.go index c5b1bf93f..3fc476a8a 100644 --- a/internal/dataframe/compiler/ir/physical_stage_sequence.go +++ b/internal/dataframe/compiler/ir/physical_stage_sequence.go @@ -5,34 +5,22 @@ import "github.com/calypr/loom/internal/dataframe/unit" const PreviewSourceResourceIDColumn = "__loom_source_resource_id" type PhysicalStageSequence struct { - SourceStageID string - SourceRowIdentity string - SourceColumns []PhysicalStageColumn - Stages []PhysicalConstructionStage - FinalStageID string - FinalRowIdentity string - FinalColumns []PhysicalStageColumn - PreviewLimitBindKey string - // PreviewSourceWindowByRootID is set only when lowering proves the terminal - // Pivot group key is the direct scalar root FHIR id and the source has root - // row identity. Preview compilation may then bound the root scan before the - // source projection is materialized. Full execution ignores this hint. - PreviewSourceWindowByRootID bool - // PreviewTerminalPivotWindow is set only when a preview ends at a - // nonunique Pivot. Rendering may select complete groups by their final - // grouped-row identity after COLLECT and before computing Pivot cells. - PreviewTerminalPivotWindow bool - // OutputAuthResourcePathBindKey adds the exact bound authorization path to - // the private prefix result as hidden row metadata. It is populated only by - // the typed composite Combine boundary. + SourceStageID string + SourceRowIdentity string + SourceColumns []PhysicalStageColumn + Stages []PhysicalConstructionStage + FinalStageID string + FinalRowIdentity string + FinalColumns []PhysicalStageColumn + PreviewLimitBindKey string + PreviewSourceWindowByRootID bool + PreviewTerminalPivotWindow bool OutputAuthResourcePathBindKey string CellTraceReturn *PhysicalCellTraceReturn RowLineageReturn *PhysicalRowLineageReturn PopulationMappingReturn *PhysicalStagePopulationMappingReturn } -// PhysicalStagePopulationMappingReturn maps the final root-contributor keys -// back to the exact selected member IDs captured in the original source rows. type PhysicalStagePopulationMappingReturn struct { SourceRootKeyColumn string SourceMemberIDsColumn string @@ -41,10 +29,6 @@ type PhysicalStagePopulationMappingReturn struct { RowIdentityColumn string } -// PhysicalRowLineageReturn selects source records for one final construction -// row. Trace carries exact owner identities for authored RELATED_EXPAND stages; -// Group terminals retain their existing typed path. It is diagnostic metadata -// and never attaches contributors to ordinary preview rows. type PhysicalRowLineageReturn struct { RowIDBindKey string OffsetBindKey string @@ -56,9 +40,6 @@ type PhysicalRowLineageReturn struct { Trace *PhysicalRowLineageTrace } -// PhysicalRowLineageTrace carries exact compiler-decoded identities for the -// row-changing construction stages that own authored contributor occurrences. -// Stages are ordered from the source toward the final output row. type PhysicalRowLineageTrace struct { RootKeyBindKey string Stages []PhysicalRowLineageStageMatch @@ -70,23 +51,19 @@ type PhysicalRowLineageStageMatch struct { StageRowIDBindKey string RelatedTerminalIDBindKey string RelatedRowKind string - // IdentityKeyBindKeys carries decoded keys for a GROUP or PIVOT preimage - // owner, in the order of that owner's typed key list. - IdentityKeyBindKeys []string + IdentityKeyBindKeys []string } type PhysicalStageColumn struct { - ID string - Name string - Label string - Kind string - Cardinality string - Nullable bool - Internal bool - Identity bool - RelatedRecordAnchor *PhysicalStageRelatedRecordAnchor - // RootContributorResourceType marks a compiler-owned root identity or - // deduplicated set of root identities retained through a reshape. + ID string + Name string + Label string + Kind string + Cardinality string + Nullable bool + Internal bool + Identity bool + RelatedRecordAnchor *PhysicalStageRelatedRecordAnchor RootContributorResourceType string NormalizedUnit *unit.UnitIdentity PresenceCompanionName string @@ -155,9 +132,6 @@ type PhysicalStageGroup struct { RowValues []PhysicalStageRowValue } -// PhysicalStageCohortGroup inserts a pinned explicit-cohort boundary into an -// ordinary construction sequence. Contributor columns are compiler-proven -// root storage identities retained by the preceding stage. type PhysicalStageCohortGroup struct { Rows PhysicalGroupRows ContributorInputColumn string @@ -167,22 +141,16 @@ type PhysicalStageCohortGroup struct { PreserveMissingMembers bool } -// PhysicalStageRowValue reduces values from contributors retained by a shape -// operation's existing COLLECT. type PhysicalStageRowValue struct { InputColumn string InputKind string InputMany bool - // FieldSource authorizes a hidden scalar input projection from one exact - // source context. Ordinary row values still require a public input column. FieldSource *PhysicalStageGroupFieldSource Output string Policy string Variable string } -// PhysicalStageCodedGroup groups root records by the tuple from one generated -// repeated Coding path. It retains no coding ordinal or display value. type PhysicalStageCodedGroup struct { RootCollectionBindKey string ConstructionIDBindKey string @@ -191,24 +159,21 @@ type PhysicalStageCodedGroup struct { ResourceType string SourceIdentityColumn string GroupRowsVariable string - // SourceRowsUnique is set only after lowering proves that the direct source - // scan yields at most one row per root _key. CODED_GROUP uses it to count - // locally deduplicated tuples with SUM(1), without retaining contributor IDs. - SourceRowsUnique bool - CodingPath string - PathSegments []PhysicalCodedGroupPathSegment - MissingKeyPolicy PhysicalStageGroupMissingKeyPolicy - SystemOutputColumn string - VersionOutputColumn string - CodeOutputColumn string - CountOutputColumn string - SystemVariable string - VersionVariable string - CodeVariable string - CountVariable string - IdentityVariable string - RowValues []PhysicalStageRowValue - RowValueProjections []PhysicalProjection + SourceRowsUnique bool + CodingPath string + PathSegments []PhysicalCodedGroupPathSegment + MissingKeyPolicy PhysicalStageGroupMissingKeyPolicy + SystemOutputColumn string + VersionOutputColumn string + CodeOutputColumn string + CountOutputColumn string + SystemVariable string + VersionVariable string + CodeVariable string + CountVariable string + IdentityVariable string + RowValues []PhysicalStageRowValue + RowValueProjections []PhysicalProjection } type PhysicalCodedGroupPathSegment struct { @@ -233,8 +198,6 @@ type PhysicalStageGroupKey struct { FieldSource *PhysicalStageGroupFieldSource } -// PhysicalStageGroupFieldSource is a tagged sum for local/current source -// selectors and an exact related occurrence expansion. type PhysicalStageGroupFieldSource struct { Kind PhysicalStageGroupFieldSourceKind PhysicalStageGroupCurrentStageFieldSource @@ -284,9 +247,6 @@ type PhysicalStageGroupRelatedOccurrenceFieldSource struct { MembershipColumn string } -// PhysicalStageGroupRelatedOccurrenceAdapter is the one Group-owned route -// expansion shared by every key and RowValue bound to the same related route. -// Its target projection contains only selected scalars or one Coding array. type PhysicalStageGroupRelatedOccurrenceAdapter struct { RouteID string RelatedRecords PhysicalStageRelatedExpand @@ -324,8 +284,6 @@ type PhysicalStageGroupRelatedRouteStep struct { ToResourceType string } -// PhysicalStageGroupCodingKey groups one compiler-proven Coding item by its -// exact system/version/code tuple while keeping display as a separate label. type PhysicalStageGroupCodingKey struct { OutputColumn string InputTupleVariable string @@ -342,8 +300,6 @@ type PhysicalStageGroupCodingKey struct { RelatedSource *PhysicalStageGroupRelatedOccurrenceFieldSource } -// PhysicalStageGroupSemanticKey expands one exact semantic coding family per -// input row. Tuple identity is separate from the public display label. type PhysicalStageGroupSemanticKey struct { OutputColumn string TupleVariable string @@ -384,9 +340,7 @@ type PhysicalStageExpand struct { } type PhysicalStageRelatedSource struct { - AnchorColumnID string - // RootContributorColumn identifies compiler-proven root records when the - // signed row anchor is a shaped identity rather than the root document key. + AnchorColumnID string RootContributorColumn string RootResourceType string OutputColumnID string @@ -400,9 +354,6 @@ type PhysicalStageRelatedSource struct { ContributorPolicy string } -// PhysicalStageRelatedExpand emits one row for each distinct terminal -// resource reached from a retained root key. The terminal _id and parent row -// identity are retained as hidden columns for downstream related-field stages. type PhysicalStageRelatedExpand struct { RouteID string Multiplicity string diff --git a/internal/dataframe/compiler/ir/physical_stage_validation.go b/internal/dataframe/compiler/ir/physical_stage_validation.go index d7ff7803b..05798f67d 100644 --- a/internal/dataframe/compiler/ir/physical_stage_validation.go +++ b/internal/dataframe/compiler/ir/physical_stage_validation.go @@ -398,7 +398,6 @@ func validateCodedPivotSourceRoot(sourceOperations []PhysicalOperation, bindVars } case PhysicalFilterOp, PhysicalDerivedLetOp, PhysicalExpressionLetOp, PhysicalSetOp, PhysicalSortOp, PhysicalLimitOp: - // These source operations preserve root row grain. case PhysicalReturnOp: returns++ if operation.Return == nil || index != len(sourceOperations)-1 { @@ -1804,8 +1803,6 @@ func validatePhysicalStageRelatedCodingKey( return nil } -// PhysicalUnnestExactGroupShapeEqual compares compiler-owned shape facts while -// allowing cloned runtime bindings and operation-local references to differ. func PhysicalUnnestExactGroupShapeEqual(left, right PhysicalUnnest) bool { if left.Owner.OccurrenceID != right.Owner.OccurrenceID || left.Owner.ResourceType != right.Owner.ResourceType || left.Owner.RootVariable != right.Owner.RootVariable || left.Owner.OwnerVariable != right.Owner.OwnerVariable || diff --git a/internal/dataframe/compiler/ir/physical_types.go b/internal/dataframe/compiler/ir/physical_types.go index 6840b104e..a1823359a 100644 --- a/internal/dataframe/compiler/ir/physical_types.go +++ b/internal/dataframe/compiler/ir/physical_types.go @@ -1,53 +1,26 @@ package ir -// PhysicalPlan is the renderer-independent backend plan produced after -// semantic planning. AQL operation graphs retain lexical variable scope; -// ClickHouse construction plans use their own closed typed payload. type PhysicalPlan struct { - Version int - Engine PhysicalEngine - Source PhysicalSource - RootOccurrenceID string - BindVars map[string]any - Operations []PhysicalOperation - // ClickHouseCombine is present only for a terminal exact-revision combine - // plan. It is immutable after lowering and executed by the ClickHouse - // renderer/stream path, never by the AQL renderer. - ClickHouseCombine *PhysicalClickHouseCombine - // ClickHousePrefix is present only when a typed AQL stage sequence feeds one - // private ClickHouse artifact input at the terminal Combine boundary. - ClickHousePrefix *PhysicalClickHousePrefix - // StageSequence composes typed operators over materialized intermediate - // rows. The ordinary operations form the source projection; stage execution - // remains part of this physical plan and shares its bind scope and renderer. - StageSequence *PhysicalStageSequence - // PreviewSourceWindowByRootID permits the one typed root-id sort used to - // bound a proven unique-id Pivot sample after its stage sequence is stripped - // for rendering the underlying source query. + Version int + Engine PhysicalEngine + Source PhysicalSource + RootOccurrenceID string + BindVars map[string]any + Operations []PhysicalOperation + ClickHouseCombine *PhysicalClickHouseCombine + ClickHousePrefix *PhysicalClickHousePrefix + StageSequence *PhysicalStageSequence PreviewSourceWindowByRootID bool - // DeferredExpressionLets are construction-time shared family bindings. - // Lowering appends them after all source sets exist and before RETURN; - // completed plans must have this list empty. - DeferredExpressionLets []PhysicalOperation - // AppliedRules records physical rewrites without exposing renderer - // implementation details to callers. - AppliedRules []string - SharedTraversalCount int - // OptimizationPolicy records every optional rewrite decision made by the - // physical optimizer, including conservative rejections. - OptimizationPolicy PhysicalOptimizationReport - // RequiredMatchReuseCount records duplicate required EXISTS predicates - // removed during physical lowering. It is deliberately separate from - // shared traversal count: required predicates remain pre-window - // semi-joins, while optional sets are post-window materializations. - RequiredMatchReuseCount int + DeferredExpressionLets []PhysicalOperation + AppliedRules []string + SharedTraversalCount int + OptimizationPolicy PhysicalOptimizationReport + RequiredMatchReuseCount int } const PopulationMappingMembersVariable = "__loom_population_members_value" const PhysicalPopulationMappingMembersColumn = "__loom_population_member_ids" -// PhysicalSource retains semantic provenance through physical optimization so -// explain output and compiler errors can point back to user intent. type PhysicalSource struct { RecipeID string TemplateID string @@ -60,59 +33,30 @@ type PhysicalSource struct { type PhysicalOperationKind string const ( - PhysicalRootScanOp PhysicalOperationKind = "ROOT_SCAN" - PhysicalTraversalOp PhysicalOperationKind = "TRAVERSAL" - PhysicalFilterOp PhysicalOperationKind = "FILTER" - PhysicalDerivedLetOp PhysicalOperationKind = "DERIVED_LET" - PhysicalExpressionLetOp PhysicalOperationKind = "EXPRESSION_LET" - // PhysicalSetOp materializes a correlated, array-valued subplan. It is the - // only operation that can introduce a set variable; selectors, aggregates, - // pivots, and slices consume that variable through typed expressions. - PhysicalSetOp PhysicalOperationKind = "SET" - // PhysicalUnnestOp is the cardinality-changing operation used for a - // correlated UNNEST. It introduces an item binding (and optionally an - // ordinality binding) for downstream operations. - PhysicalUnnestOp PhysicalOperationKind = "UNNEST" - // PhysicalSortOp and PhysicalLimitOp describe the root execution window. - // They are intentionally typed so preview ordering and bounds cannot be - // smuggled into an AQL string by a caller. - PhysicalSortOp PhysicalOperationKind = "SORT" - PhysicalLimitOp PhysicalOperationKind = "LIMIT" - PhysicalReturnOp PhysicalOperationKind = "RETURN" - PhysicalPathSeedOp PhysicalOperationKind = "PATH_SEED" - PhysicalPathExtendOp PhysicalOperationKind = "PATH_EXTEND" - PhysicalGraphReturnOp PhysicalOperationKind = "GRAPH_RETURN" - PhysicalCollectionScanOp PhysicalOperationKind = "COLLECTION_SCAN" - // PhysicalKeySetLookupOp iterates an exact array of resource _keys and - // resolves each document without scanning the whole collection. - PhysicalKeySetLookupOp PhysicalOperationKind = "KEY_SET_LOOKUP" - // PhysicalDocumentLookupOp reads one compiler-pinned resource collection - // using an exact active document identity captured from an earlier stage. - PhysicalDocumentLookupOp PhysicalOperationKind = "DOCUMENT_LOOKUP" - // PhysicalPopulationMappingReturnOp expands matched members only after the - // canonical final-row plan has completed. Its witness rows are internal. - PhysicalPopulationMappingReturnOp PhysicalOperationKind = "POPULATION_MAPPING_RETURN" - // PhysicalCellTraceReturnOp replaces the public projection terminal for one - // bounded, receipt-backed explanation request. It evaluates the exact - // compiled value expression and exposes pre-reduction contributors without - // changing ordinary dataframe execution. - PhysicalCellTraceReturnOp PhysicalOperationKind = "CELL_TRACE_RETURN" - // PhysicalTableShapeExclusionReturnOp replaces a grouped pivot and its - // publication projection with a bounded diagnostic over the exact pivot - // input rows. + PhysicalRootScanOp PhysicalOperationKind = "ROOT_SCAN" + PhysicalTraversalOp PhysicalOperationKind = "TRAVERSAL" + PhysicalFilterOp PhysicalOperationKind = "FILTER" + PhysicalDerivedLetOp PhysicalOperationKind = "DERIVED_LET" + PhysicalExpressionLetOp PhysicalOperationKind = "EXPRESSION_LET" + PhysicalSetOp PhysicalOperationKind = "SET" + PhysicalUnnestOp PhysicalOperationKind = "UNNEST" + PhysicalSortOp PhysicalOperationKind = "SORT" + PhysicalLimitOp PhysicalOperationKind = "LIMIT" + PhysicalReturnOp PhysicalOperationKind = "RETURN" + PhysicalPathSeedOp PhysicalOperationKind = "PATH_SEED" + PhysicalPathExtendOp PhysicalOperationKind = "PATH_EXTEND" + PhysicalGraphReturnOp PhysicalOperationKind = "GRAPH_RETURN" + PhysicalCollectionScanOp PhysicalOperationKind = "COLLECTION_SCAN" + PhysicalKeySetLookupOp PhysicalOperationKind = "KEY_SET_LOOKUP" + PhysicalDocumentLookupOp PhysicalOperationKind = "DOCUMENT_LOOKUP" + PhysicalPopulationMappingReturnOp PhysicalOperationKind = "POPULATION_MAPPING_RETURN" + PhysicalCellTraceReturnOp PhysicalOperationKind = "CELL_TRACE_RETURN" PhysicalTableShapeExclusionReturnOp PhysicalOperationKind = "TABLE_SHAPE_EXCLUSION_RETURN" - // PhysicalGroupRowsOp emits one row per definition in a pinned immutable group revision. - PhysicalGroupRowsOp PhysicalOperationKind = "GROUP_ROWS" - // PhysicalGroupedPivotOp groups finalized output rows by authored scalar - // keys and projects the frozen category list into named columns. - PhysicalGroupedPivotOp PhysicalOperationKind = "GROUPED_PIVOT" - // PhysicalUnpivotOp replaces selected scalar columns with one typed key/value - // row per selected input column. - PhysicalUnpivotOp PhysicalOperationKind = "UNPIVOT" + PhysicalGroupRowsOp PhysicalOperationKind = "GROUP_ROWS" + PhysicalGroupedPivotOp PhysicalOperationKind = "GROUPED_PIVOT" + PhysicalUnpivotOp PhysicalOperationKind = "UNPIVOT" ) -// PhysicalOperation is a tagged union. Exactly one payload matching Kind must -// be set. Source can be more specific than the plan-level provenance. type PhysicalOperation struct { Kind PhysicalOperationKind Source PhysicalSource @@ -140,9 +84,6 @@ type PhysicalOperation struct { Unpivot *PhysicalUnpivot } -// PhysicalGroupRows is a typed source and terminal for grouped dataframe rows. -// Membership references remain independent of resource lookup so missing -// source documents do not erase group membership. type PhysicalGroupRows struct { RevisionCollectionBindKey string SelectionCollectionBindKey string @@ -163,9 +104,6 @@ type PhysicalGroupRows struct { CellTrace *PhysicalGroupRowsCellTrace } -// PhysicalGroupRowsCellTrace replaces the public row terminal with a bounded -// trace of one compiler-selected field. The renderer evaluates the field from -// the exact row it would otherwise return, including its stable group identity. type PhysicalGroupRowsCellTrace struct { OutputColumn string Cardinality PhysicalCardinality @@ -174,8 +112,6 @@ type PhysicalGroupRowsCellTrace struct { FetchLimitBindKey string } -// PhysicalGroupMemberValue reduces a checked field over the authorized members -// of one pinned cohort, preserving the cohort's existing row identity. type PhysicalGroupMemberValue struct { Output string Kind string @@ -186,31 +122,15 @@ type PhysicalGroupMemberValue struct { type PhysicalRootScan struct { Variable string CollectionBindKey string - // PageCandidateSeed is an optional, typed candidate-only prefilter for root - // key discovery. It must never replace the exact root predicates that - // follow ROOT_SCAN, and it cannot be combined with Population or CohortSource. PageCandidateSeed *PhysicalRootPageCandidateSeed - // Population replaces the full root collection scan with an indexed scan - // that starts from persisted selection members. The renderer deduplicates - // RootKey before restoring Variable from CollectionBindKey. - Population *PhysicalPopulationRootSource - // CohortSource replaces a full scan with pinned assigned and selection - // members when lowering proves the input rows retain exact root identity. - CohortSource *PhysicalCohortRootSource + Population *PhysicalPopulationRootSource + CohortSource *PhysicalCohortRootSource } -// PhysicalRootPageCandidateSeed starts from a scoped related-resource lookup -// and walks the inverse relationship to candidate roots. It is a superset -// seed only: the root plan must re-evaluate its exact related SET reduction -// and filter before applying the root-key execution window. type PhysicalRootPageCandidateSeed struct { Subplan PhysicalSubplan } -// PhysicalCohortRootSource bounds a typed cohort's source rows to the union -// of explicit memberships and pinned selection members. The latter preserves -// unassigned semantics; the former preserves accepted assignments even if a -// stored membership is outside the source selection. type PhysicalCohortRootSource struct { CohortStageID string CohortInputStageID string @@ -228,11 +148,6 @@ type PhysicalCohortRootSource struct { PolicyBindKey string } -// PhysicalPopulationRootSource describes a membership-driven root scan. The -// member collection has a different trust boundary from FHIR resource -// collections, so its filters are kept separate from the scoped resource -// operations. CollectMembersVariable is empty for ordinary dataframe reads; -// population-mapping compilation sets it to retain member witnesses per root. type PhysicalPopulationRootSource struct { MemberScan PhysicalCollectionScan MemberFilters []PhysicalFilter @@ -250,10 +165,6 @@ const ( PhysicalAny PhysicalTraversalDirection = "ANY" ) -// PhysicalTraversalStrategy selects the execution shape for a validated -// depth-one relationship. Native graph traversal is the conservative -// fallback. EndpointLookup is only legal when storage-route metadata proves -// the endpoint/discriminator fields and their compound index contract. type PhysicalTraversalStrategy string const ( @@ -268,24 +179,11 @@ type PhysicalTraversal struct { Direction PhysicalTraversalDirection EdgeCollectionBindKey string EdgeLabelBindKey string - // SourceTypeBindKey retains the typed FHIR resource at the source endpoint. - // Ordinary forward traversal does not need it, but reverse semijoin plans - // use the opposite edge discriminator to preserve the exact typed route. - SourceTypeBindKey string - TargetTypeBindKey string - // EdgeTargetTypeField is a compiler-owned fhir_edge discriminator used - // alongside TargetTypeBindKey. For a parent-to-child INBOUND route it is - // from_type; for a proven forward OUTBOUND route it is to_type. The node - // resourceType check remains independently mandatory. - EdgeTargetTypeField string - // Strategy is deliberately typed rather than an AQL fragment. Endpoint - // fields are supplied by resolveStorageRoute and validated against the - // direction before the renderer can use them. - Strategy PhysicalTraversalStrategy - EndpointField string - EndpointJoinField string - EndpointIndexFields []string + SourceTypeBindKey string + TargetTypeBindKey string + EdgeTargetTypeField string + Strategy PhysicalTraversalStrategy + EndpointField string + EndpointJoinField string + EndpointIndexFields []string } - -// PhysicalValue is either a variable/path reference or a bind variable. A -// renderer must never interpret Path segments as AQL source text. diff --git a/internal/dataframe/compiler/ir/physical_validation.go b/internal/dataframe/compiler/ir/physical_validation.go index 669bdcf40..cbde72d3c 100644 --- a/internal/dataframe/compiler/ir/physical_validation.go +++ b/internal/dataframe/compiler/ir/physical_validation.go @@ -36,14 +36,14 @@ func (p PhysicalPlan) Validate() error { return nil } if len(p.DeferredExpressionLets) != 0 || p.StageSequence == nil { - return fmt.Errorf("composite ClickHouse combine requires a typed AQL stage sequence and no deferred expressions") + return fmt.Errorf("composite ClickHouse combine requires a typed construction stage sequence and no deferred expressions") } if p.StageSequence.PreviewLimitBindKey != "" { - return fmt.Errorf("bounded AQL preview cannot feed a private ClickHouse artifact") + return fmt.Errorf("bounded preview cannot feed a private ClickHouse artifact") } for _, operation := range p.Operations { if operation.Kind == PhysicalLimitOp { - return fmt.Errorf("bounded AQL prefix cannot feed a private ClickHouse artifact") + return fmt.Errorf("bounded prefix cannot feed a private ClickHouse artifact") } if operation.Kind == PhysicalGroupRowsOp { return fmt.Errorf("GROUP prefixes cannot feed a private ClickHouse Combine until authorization scope is preserved by grouping") @@ -71,7 +71,7 @@ func (p PhysicalPlan) Validate() error { privateStageID = input.PrivateStageID } if privateStageID != p.ClickHousePrefix.StageID { - return fmt.Errorf("composite ClickHouse input does not match the exact AQL prefix stage") + return fmt.Errorf("composite ClickHouse input does not match the exact prefix stage") } default: return fmt.Errorf("unsupported physical engine %q", p.Engine) @@ -371,10 +371,6 @@ func (p PhysicalPlan) Validate() error { return nil } -// ValidateForWorkspaceCompilation validates the closed typed shape of a -// terminal workspace-output Combine. It is a resolver-only boundary: normal -// physical validation and all execution/render entry points still reject the -// unresolved sibling references until server-owned capture is implemented. func (p PhysicalPlan) ValidateForWorkspaceCompilation() error { if p.Engine != PhysicalEngineClickHouse || p.ClickHouseCombine == nil || p.ClickHousePrefix != nil { return p.Validate() diff --git a/internal/dataframe/compiler/ir/physical_validation_expr.go b/internal/dataframe/compiler/ir/physical_validation_expr.go index a74a833fa..77fd5b4ed 100644 --- a/internal/dataframe/compiler/ir/physical_validation_expr.go +++ b/internal/dataframe/compiler/ir/physical_validation_expr.go @@ -261,9 +261,6 @@ func validatePhysicalKeySet(keySet PhysicalKeySet, defined map[string]bool, bind return nil } -// validatePhysicalCall checks the small, backend-neutral operator vocabulary -// accepted by the AQL renderer. Keeping this list here prevents a recipe from -// smuggling arbitrary function or query text through the physical plan. func validatePhysicalCall(call PhysicalCall, defined map[string]bool, bindVars map[string]any) error { name := strings.ToLower(strings.TrimSpace(call.Name)) if name == "" { @@ -312,11 +309,6 @@ func validatePhysicalCall(call PhysicalCall, defined map[string]bool, bindVars m return nil } -// validatePhysicalExpressionObjectCycles protects recursive expression and -// subplan validation from malformed in-memory plans containing pointer cycles. -// JSON decoding cannot produce such cycles, but compiler stages and tests can -// wire pointers directly. The active/visited split permits shared DAG nodes -// while rejecting only true recursion. func validatePhysicalExpressionObjectCycles(expression PhysicalExpression) error { active := map[*PhysicalObject]bool{} visited := map[*PhysicalObject]bool{} diff --git a/internal/dataframe/compiler/ir/physical_validation_refs.go b/internal/dataframe/compiler/ir/physical_validation_refs.go index 4a0a999db..03c96ced5 100644 --- a/internal/dataframe/compiler/ir/physical_validation_refs.go +++ b/internal/dataframe/compiler/ir/physical_validation_refs.go @@ -123,10 +123,6 @@ func validatePhysicalSelector(resourceType string, selector spec.Selector) error return nil } -// schemaDefinitionExists accepts both top-level FHIR resources and generated -// backbone/choice definitions such as GroupMember. A definition may be a -// valid selector source without being a graph collection or traversal node; -// collection and route validation deliberately continue to use HasResource. func schemaDefinitionExists(resourceType string) bool { return fhirschema.DefinitionExists(resourceType) } diff --git a/internal/dataframe/compiler/ir/physical_validation_shapes.go b/internal/dataframe/compiler/ir/physical_validation_shapes.go index c9367bd21..378911ff4 100644 --- a/internal/dataframe/compiler/ir/physical_validation_shapes.go +++ b/internal/dataframe/compiler/ir/physical_validation_shapes.go @@ -54,10 +54,6 @@ func validatePhysicalExpressionProjections(projections []PhysicalExpressionProje } func validatePhysicalPredicateExpression(predicate PhysicalPredicateExpression, defined map[string]bool, bindVars map[string]any) error { - // Filter predicates can contain EXISTS subplans without being wrapped in a - // PhysicalExpression. Run the shared pointer-cycle preflight here as well as - // from validatePhysicalExpression so malformed in-memory plans are rejected - // before recursive structural validation. if err := validatePhysicalExpressionObjectCycles(PhysicalExpression{ Aggregate: &PhysicalAggregate{Predicate: &predicate}, }); err != nil { diff --git a/internal/dataframe/compiler/ir/physical_values.go b/internal/dataframe/compiler/ir/physical_values.go index d59a562ea..cc085c3b3 100644 --- a/internal/dataframe/compiler/ir/physical_values.go +++ b/internal/dataframe/compiler/ir/physical_values.go @@ -6,18 +6,12 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// PhysicalPlan is the renderer-independent AQL operation graph produced after -// semantic planning. Operations are ordered because AQL variables have lexical -// scope: an operation may reference only variables introduced before it. type PhysicalValue struct { Variable string Path []string BindKey string } -// PhysicalCardinality and PhysicalNullBehavior are deliberately explicit on -// every rich expression. A renderer must not infer array/scalar/null behavior -// from where an expression appears in an AQL template. type PhysicalCardinality string const ( @@ -37,9 +31,7 @@ const ( type PhysicalExpressionKind string const ( - PhysicalValueExpression PhysicalExpressionKind = "VALUE" - // PhysicalLiteralExpression is a typed bind-backed constant. Literals are - // kept out of generated AQL so recipe authors cannot inject query source. + PhysicalValueExpression PhysicalExpressionKind = "VALUE" PhysicalLiteralExpression PhysicalExpressionKind = "LITERAL" PhysicalExtractExpression PhysicalExpressionKind = "EXTRACT" PhysicalAggregateExpression PhysicalExpressionKind = "AGGREGATE" @@ -49,25 +41,13 @@ const ( PhysicalObjectLookupExpression PhysicalExpressionKind = "OBJECT_LOOKUP" PhysicalKeyedMapExpression PhysicalExpressionKind = "KEYED_MAP" PhysicalObjectKeysExpression PhysicalExpressionKind = "OBJECT_KEYS" - // PhysicalKeySetExpression returns the sorted distinct keys observed in a - // bounded source. It is executor metadata, not a public dataframe column. - PhysicalKeySetExpression PhysicalExpressionKind = "KEY_SET" - PhysicalObjectExpression PhysicalExpressionKind = "OBJECT" - // PhysicalSubplanExpression evaluates a compiler-owned correlated subquery - // as an array-valued projection, such as row-level population provenance. - PhysicalSubplanExpression PhysicalExpressionKind = "SUBPLAN" - // PhysicalCallExpression represents a recipe-neutral expression function. - // Name is validated against the compiler-owned operator registry and Args - // remain typed expressions; neither contains AQL source text. - PhysicalCallExpression PhysicalExpressionKind = "CALL" - // PhysicalRelatedFieldExpression loads one compiler-selected scalar path - // from a document identified by its exact retained Arango _id. + PhysicalKeySetExpression PhysicalExpressionKind = "KEY_SET" + PhysicalObjectExpression PhysicalExpressionKind = "OBJECT" + PhysicalSubplanExpression PhysicalExpressionKind = "SUBPLAN" + PhysicalCallExpression PhysicalExpressionKind = "CALL" PhysicalRelatedFieldExpression PhysicalExpressionKind = "RELATED_FIELD" ) -// PhysicalSelectorExecutionMode records a schema-proven selector lowering. -// Generic is the safe fallback for predicates, fallbacks, choice paths, and -// unknown cardinality. type PhysicalSelectorExecutionMode string const ( @@ -76,9 +56,6 @@ const ( PhysicalSelectorConditionalArray PhysicalSelectorExecutionMode = "CONDITIONAL_ARRAY" ) -// PhysicalExpression is a closed, renderer-independent value tree. It carries -// no AQL source text: selector paths have already been parsed and all literals -// are represented by bind values. type PhysicalExpression struct { Kind PhysicalExpressionKind Cardinality PhysicalCardinality @@ -100,16 +77,10 @@ type PhysicalExpression struct { RelatedField *PhysicalRelatedField } -// PhysicalLiteral references a value in the plan bind map. BindKey is -// deliberately required even for primitive constants so all recipe data uses -// the same parameterized execution boundary. type PhysicalLiteral struct { BindKey string } -// PhysicalCall is a backend-neutral expression operator. TargetKind is used -// by cast and distinct_present_one to name a logical or table scalar type, -// never a backend type or query fragment. type PhysicalCall struct { Name string Args []PhysicalExpression @@ -120,36 +91,20 @@ type PhysicalRelatedField struct { DocumentID PhysicalValue ResourceType string Path []string - // PresenceOnly asks the exact related-field lookup to return whether the - // selector path exists. An absent or out-of-scope terminal remains present - // for compatibility with RELATED_FIELD's existing null result. PresenceOnly bool } -// PhysicalExtract obtains one FHIR selector from a variable or prior set -// element. ResourceType keeps schema validation available after semantic -// lowering; fallbacks remain ordered value alternatives and are flattened -// with the primary selector before FIRST, ALL, or DISTINCT projection. type PhysicalExtract struct { - Source PhysicalValue - ResourceType string - Selector spec.Selector - Fallbacks []PhysicalSelectorFallback - // Distinct preserves the explicit DISTINCT projection mode after semantic - // lowering. It is meaningful only for an array-valued expression. - Distinct bool - ExecutionMode PhysicalSelectorExecutionMode - // Prepared points at a selector value projected by a prepared child set. - // Source remains the owning set for scope validation and diagnostics. - Prepared *PhysicalPreparedReference - // UnitNormalization retains the original measurement and exact source - // identity selectors until the renderer applies one approved rule per item. + Source PhysicalValue + ResourceType string + Selector spec.Selector + Fallbacks []PhysicalSelectorFallback + Distinct bool + ExecutionMode PhysicalSelectorExecutionMode + Prepared *PhysicalPreparedReference UnitNormalization *PhysicalUnitNormalization } -// PhysicalSelectorFallback is one ordered selector alternative. Keeping its -// source and schema type with the selector lets recipes preserve fallbacks -// whose lexical owner differs from the primary selector's owner. type PhysicalSelectorFallback struct { Source PhysicalValue ResourceType string @@ -165,15 +120,11 @@ type PhysicalUnitNormalization struct { Rules []unit.UnitConversionRule } -// PhysicalPreparedReference identifies one selector column in a prepared set. type PhysicalPreparedReference struct { SetVariable string Field string } -// PhysicalPreparedSet describes selector projections shared by rich -// consumers over one materialized relationship set. Preparation stays -// attached to the correlated set so root row grain cannot change. type PhysicalPreparedSet struct { Variable string SourceSetVariable string @@ -231,13 +182,8 @@ type PhysicalAggregate struct { } type PhysicalPivotMap struct { - Source PhysicalValue - ResourceType string - // ItemSource and ItemResourceType describe a repeated backbone item that - // owns the key/value pair. Keeping this scope in the physical operation is - // what prevents key and value selectors from being flattened independently - // (for example Observation.component[].code paired with the same - // component's value[x]). + Source PhysicalValue + ResourceType string ItemSource spec.Selector ItemResourceType string KeySelector spec.Selector @@ -250,14 +196,9 @@ type PhysicalPivotMap struct { ProjectionMode string PreparedKey *PhysicalPreparedReference PreparedValue *PhysicalPreparedReference - // Correlation is optional for legacy pivots. When present, one owner item - // is iterated first, then its Coding items are matched by system+code in - // the same Coding object before ValueSelector is evaluated. - Correlation *PhysicalCorrelation + Correlation *PhysicalCorrelation } -// PhysicalOwnerRecords emits one ordered record per repeated FHIR owner while -// preserving the surrounding dataframe row grain. type PhysicalOwnerRecords struct { Correlation PhysicalCorrelation OwnerPathBindKey string @@ -265,10 +206,6 @@ type PhysicalOwnerRecords struct { LogicalTypeBindKey string } -// PhysicalCorrelation is the shared typed representation for correlated -// terminology predicates and projections. Selectors are relative to Source's -// owner payload, except SystemSelector and CodeSelector which are relative to -// one item produced by KeySelector. type PhysicalCorrelation struct { Source PhysicalValue ResourceType string @@ -293,10 +230,6 @@ type PhysicalCorrelation struct { SystemBindKey string CodeBindKey string VersionBindKey string - // ExtensionURLSelectors and ExtensionURLBindKeys describe an ancestor- - // aware extension correlation. Each selector is relative to the current - // Extension item and each bind is matched before descending to the next - // nested extension. When non-empty, coding fields above are unused. ExtensionURLSelectors []spec.Selector ExtensionURLBindKeys []string } @@ -335,8 +268,6 @@ type PhysicalKeySet struct { ItemKey PhysicalExpression } -// PhysicalSlice is a bounded, stable, nested projection over a prior set. -// Limit remains a bind variable so callers cannot synthesize query literals. type PhysicalSlice struct { Source PhysicalValue Predicate *PhysicalPredicateExpression @@ -353,8 +284,3 @@ type PhysicalExpressionProjection struct { type PhysicalObject struct { Fields []PhysicalExpressionProjection } - -// PhysicalSet is a correlated, array-valued subplan. Captures are declared -// explicitly so validation can prevent accidental references to parent or -// future variables. The set variable becomes visible only after its subplan -// has validated successfully. diff --git a/internal/dataframe/compiler/ir/policy.go b/internal/dataframe/compiler/ir/policy.go index 4e0a2cca2..ebfdf5bfc 100644 --- a/internal/dataframe/compiler/ir/policy.go +++ b/internal/dataframe/compiler/ir/policy.go @@ -6,30 +6,12 @@ import ( "strings" ) -// PhysicalOptimizationPolicy is the small, explainable policy used by the -// physical optimizer. It deliberately estimates operation work rather than -// pretending to predict Arango's cost; PROFILE remains the authority for -// deciding whether a rewrite is worthwhile on a particular dataset. type PhysicalOptimizationPolicy struct { - // Enabled is false when the policy is disabled explicitly. Disabling the - // policy never disables validation or changes semantic meaning; it prevents - // all optional optimizer families while preserving the unoptimized physical - // execution plan. - Enabled bool - // MinimumSavings is the minimum estimated operation-work reduction required - // before an optional rewrite is applied. The default is one operation. + Enabled bool MinimumSavings int - // RuleOverrides contains explicit per-rule ablation decisions. A nil entry - // preserves the production default: traversal sharing and compact set - // projection are enabled after their live parity/profile gates. The older - // second-pass prepared selector experiment and later optimization families - // remain disabled until their own payload/cost gate has passed. - RuleOverrides map[PhysicalOptimizationRule]bool + RuleOverrides map[PhysicalOptimizationRule]bool } -// PhysicalOptimizationRule names an independently ablatable optimizer family. -// These names are compiler-owned and are intentionally independent of FHIR -// resource names or rendered AQL fragments. type PhysicalOptimizationRule string const ( @@ -46,8 +28,6 @@ var allPhysicalOptimizationRules = []PhysicalOptimizationRule{ PhysicalOptimizationRuleEndpointTraversal, } -// RuleEnabled resolves one rule without mutating the caller's policy. A -// global disable always wins; an explicit override wins over the defaults. func (policy PhysicalOptimizationPolicy) RuleEnabled(rule PhysicalOptimizationRule) bool { if !policy.Enabled { return false @@ -65,8 +45,6 @@ func (policy PhysicalOptimizationPolicy) RuleEnabled(rule PhysicalOptimizationRu } } -// WithRule returns a copy with one named rule explicitly enabled or disabled. -// It is used by benchmark and parity harnesses to change exactly one rule. func (policy PhysicalOptimizationPolicy) WithRule(rule PhysicalOptimizationRule, enabled bool) PhysicalOptimizationPolicy { if policy.RuleOverrides == nil { policy.RuleOverrides = make(map[PhysicalOptimizationRule]bool) @@ -81,9 +59,6 @@ func (policy PhysicalOptimizationPolicy) WithRule(rule PhysicalOptimizationRule, return policy } -// PhysicalOptimizationDecision explains one candidate rewrite. The values are -// intentionally structural and stable across schemas: they count typed -// physical operations, not FHIR resource names or rendered AQL fragments. type PhysicalOptimizationDecision struct { Rule string Enabled bool @@ -94,10 +69,6 @@ type PhysicalOptimizationDecision struct { Reason string } -// PhysicalOptimizationReport is attached to an optimized plan and copied -// into compiler diagnostics. A rejection is useful evidence: it tells callers -// why a seemingly similar route was left unshared instead of silently hiding -// the decision in rendered AQL. type PhysicalOptimizationReport struct { Policy string Enabled bool @@ -106,9 +77,6 @@ type PhysicalOptimizationReport struct { Decisions []PhysicalOptimizationDecision } -// PhysicalOptimizationRuleState reports the resolved state of every known -// optimizer family, including families that had no candidate in this plan. -// Decisions remain reserved for candidate-specific estimates and rewrites. type PhysicalOptimizationRuleState struct { Rule PhysicalOptimizationRule Enabled bool @@ -117,15 +85,6 @@ type PhysicalOptimizationRuleState struct { const physicalOptimizationPolicyName = "conservative-structural-v1" -// DefaultPhysicalOptimizationPolicy returns the production policy. The -// LOOM_PHYSICAL_COST_POLICY environment variable is intentionally a local -// developer switch, not a user-controlled query input. Set it to "off" (or -// "0"/"false") to compare the unshared physical shape; all validation and -// result semantics remain unchanged. Compact set projection includes -// traversal-time selector projection for fallback-free rich consumers; set -// LOOM_PHYSICAL_RULE_COMPACT_PROJECTION=off to compare full-node output. The -// older second-pass prepared selector experiment remains opt-in via -// LOOM_PHYSICAL_RULE_PREPARED_SELECTORS=on. func DefaultPhysicalOptimizationPolicy() PhysicalOptimizationPolicy { policy := PhysicalOptimizationPolicy{Enabled: true, MinimumSavings: 1} switch strings.ToLower(strings.TrimSpace(os.Getenv("LOOM_PHYSICAL_COST_POLICY"))) { @@ -196,33 +155,20 @@ func (report *PhysicalOptimizationReport) AddDecision(decision PhysicalOptimizat report.addDecision(decision) } -// estimateTraversalSharingWork models the prefix and subset operations that -// the rewrite actually changes. Every original set pays for its traversal and -// scope prefix. The shared plan pays for one prefix plus one typed subset per -// consumer. Consumer-specific operations are present in either plan and are -// therefore intentionally excluded from the estimate. func estimateTraversalSharingWork(prefix PhysicalTraversalPrefixDecomposition, candidateSets int) (baseline, optimized, savings int) { if candidateSets < 2 { return 0, 0, 0 } - // Traversal plus the canonical scope block. The decomposition carries the - // scope operation count so this estimate remains independent of a specific - // FHIR route or an incidental renderer variable name. prefixOperations := 1 + prefix.Prefix.ScopeOperationCount if prefix.Prefix.SourceVariable == "" { return 0, 0, 0 } baseline = prefixOperations * candidateSets - optimized = prefixOperations + candidateSets // one typed subset per set + optimized = prefixOperations + candidateSets savings = baseline - optimized return baseline, optimized, savings } -// estimatePreparedSelectorWork uses a deliberately small structural model: -// selector extraction has a source iteration and a value projection, while a -// prepared value pays one projection plus a cheap field read at each consumer. -// It is only used as a lower-bound gate (PROFILE remains authoritative), so a -// selector must have at least two consumers before preparation is allowed. func estimatePreparedSelectorWork(selectorUseCount int) (baseline, optimized, savings int) { if selectorUseCount < 2 { return 0, 0, 0 diff --git a/internal/dataframe/compiler/ir/postgres_storage.go b/internal/dataframe/compiler/ir/postgres_storage.go new file mode 100644 index 000000000..7c5021ac9 --- /dev/null +++ b/internal/dataframe/compiler/ir/postgres_storage.go @@ -0,0 +1,194 @@ +package ir + +import ( + "errors" + "fmt" + "regexp" + "sort" +) + +var ErrUnregisteredPostgresIdentifier = errors.New("unregistered postgres identifier") + +var postgresSchemaNamePattern = regexp.MustCompile(`^[a-z_][a-z0-9_]{0,62}$`) + +type PostgresLogicalTableKind string + +const ( + PostgresResourceLogicalTable PostgresLogicalTableKind = "resource" + PostgresEdgeOccurrenceLogicalTable PostgresLogicalTableKind = "edge_occurrence" +) + +type postgresLogicalTableSpec struct { + physicalTable string + columns map[string]struct{} +} + +var postgresLogicalTables = map[PostgresLogicalTableKind]postgresLogicalTableSpec{ + PostgresResourceLogicalTable: { + physicalTable: "fhir_resource", + columns: columnSet( + "resource_seq", "project", "dataset_generation", "resource_type", + "logical_id", "auth_resource_path", "payload", "recorded_at", + ), + }, + PostgresEdgeOccurrenceLogicalTable: { + physicalTable: "fhir_edge_occurrence", + columns: columnSet( + "edge_seq", "project", "dataset_generation", "auth_resource_path", + "label", "source_resource_type", "source_logical_id", + "target_resource_type", "target_logical_id", "source_path", + "source_ordinal", "logical_edge_key", "recorded_at", + ), + }, +} + +func columnSet(names ...string) map[string]struct{} { + set := make(map[string]struct{}, len(names)) + for _, name := range names { + set[name] = struct{}{} + } + return set +} + +type PostgresTableIdentifier struct { + schema string + table string +} + +func (i PostgresTableIdentifier) Schema() string { return i.schema } + +func (i PostgresTableIdentifier) Table() string { return i.table } + +func (i PostgresTableIdentifier) SQLText() string { + return quotePostgresIdentifier(i.schema) + "." + quotePostgresIdentifier(i.table) +} + +func (i PostgresTableIdentifier) IsRegistered() bool { + if i.schema == "" || i.table == "" { + return false + } + if !postgresSchemaNamePattern.MatchString(i.schema) { + return false + } + for _, spec := range postgresLogicalTables { + if spec.physicalTable == i.table { + return true + } + } + return false +} + +func NewPostgresTableIdentifier(schema string, kind PostgresLogicalTableKind) (PostgresTableIdentifier, error) { + spec, ok := postgresLogicalTables[kind] + if !ok { + return PostgresTableIdentifier{}, fmt.Errorf("%w: logical table kind %q is not registered", + ErrUnregisteredPostgresIdentifier, string(kind)) + } + if !postgresSchemaNamePattern.MatchString(schema) { + return PostgresTableIdentifier{}, fmt.Errorf("%w: schema name %q must match [a-z_][a-z0-9_]{0,62}", + ErrUnregisteredPostgresIdentifier, schema) + } + return PostgresTableIdentifier{schema: schema, table: spec.physicalTable}, nil +} + +type PostgresColumnIdentifier struct { + tableKind PostgresLogicalTableKind + column string +} + +func (c PostgresColumnIdentifier) TableKind() PostgresLogicalTableKind { return c.tableKind } + +func (c PostgresColumnIdentifier) Name() string { return c.column } + +func (c PostgresColumnIdentifier) SQLText() string { return quotePostgresIdentifier(c.column) } + +type PostgresLogicalTable struct { + kind PostgresLogicalTableKind + table PostgresTableIdentifier +} + +func (t PostgresLogicalTable) Kind() PostgresLogicalTableKind { return t.kind } + +func (t PostgresLogicalTable) Identifier() PostgresTableIdentifier { return t.table } + +func (t PostgresLogicalTable) Column(name string) (PostgresColumnIdentifier, error) { + spec, ok := postgresLogicalTables[t.kind] + if !ok { + return PostgresColumnIdentifier{}, fmt.Errorf("%w: logical table %q has no registry entry", + ErrUnregisteredPostgresIdentifier, string(t.kind)) + } + if _, ok := spec.columns[name]; !ok { + return PostgresColumnIdentifier{}, fmt.Errorf("%w: column %q is not a column of logical table %q", + ErrUnregisteredPostgresIdentifier, name, string(t.kind)) + } + return PostgresColumnIdentifier{tableKind: t.kind, column: name}, nil +} + +func (t PostgresLogicalTable) MustColumn(name string) PostgresColumnIdentifier { + column, err := t.Column(name) + if err != nil { + panic(err) + } + return column +} + +type PostgresStorageBinding struct { + Resource PostgresLogicalTable + Edge PostgresLogicalTable +} + +func NewPostgresStorageBinding(schema string) (PostgresStorageBinding, error) { + resource, err := NewPostgresTableIdentifier(schema, PostgresResourceLogicalTable) + if err != nil { + return PostgresStorageBinding{}, fmt.Errorf("resource table: %w", err) + } + edge, err := NewPostgresTableIdentifier(schema, PostgresEdgeOccurrenceLogicalTable) + if err != nil { + return PostgresStorageBinding{}, fmt.Errorf("edge table: %w", err) + } + return PostgresStorageBinding{ + Resource: PostgresLogicalTable{kind: PostgresResourceLogicalTable, table: resource}, + Edge: PostgresLogicalTable{kind: PostgresEdgeOccurrenceLogicalTable, table: edge}, + }, nil +} + +func (b PostgresStorageBinding) Validate() error { + if b.Resource.kind != PostgresResourceLogicalTable || !b.Resource.table.IsRegistered() { + return fmt.Errorf("%w: resource logical table is not registered", ErrUnregisteredPostgresIdentifier) + } + if b.Edge.kind != PostgresEdgeOccurrenceLogicalTable || !b.Edge.table.IsRegistered() { + return fmt.Errorf("%w: edge logical table is not registered", ErrUnregisteredPostgresIdentifier) + } + if b.Resource.table.schema != b.Edge.table.schema { + return fmt.Errorf("%w: resource table schema %q and edge table schema %q must match", + ErrUnregisteredPostgresIdentifier, b.Resource.table.schema, b.Edge.table.schema) + } + return nil +} + +func RegisteredPostgresColumnNames(kind PostgresLogicalTableKind) ([]string, error) { + spec, ok := postgresLogicalTables[kind] + if !ok { + return nil, fmt.Errorf("%w: logical table kind %q is not registered", + ErrUnregisteredPostgresIdentifier, string(kind)) + } + names := make([]string, 0, len(spec.columns)) + for name := range spec.columns { + names = append(names, name) + } + sort.Strings(names) + return names, nil +} + +func quotePostgresIdentifier(name string) string { + out := make([]byte, 0, len(name)+2) + out = append(out, '"') + for index := 0; index < len(name); index++ { + if name[index] == '"' { + out = append(out, '"') + } + out = append(out, name[index]) + } + out = append(out, '"') + return string(out) +} diff --git a/internal/dataframe/compiler/ir/prefix.go b/internal/dataframe/compiler/ir/prefix.go index e44de61a1..f1fae19ca 100644 --- a/internal/dataframe/compiler/ir/prefix.go +++ b/internal/dataframe/compiler/ir/prefix.go @@ -5,14 +5,6 @@ import ( "fmt" ) -// PhysicalTraversalPrefix is the canonical, target-type-independent portion -// of one optional generic traversal set. It describes exactly the work that a -// future optimizer may materialize once for compatible siblings; it is not a -// renderer instruction and has no runtime effect by itself. -// -// NodeVariable and EdgeVariable are deliberately canonical names. Local -// variables from the source set live on PhysicalTraversalSubset, which makes -// alpha-equivalent prefixes compare without relying on lowering counters. type PhysicalTraversalPrefix struct { SourceVariable string Direction PhysicalTraversalDirection @@ -25,14 +17,9 @@ type PhysicalTraversalPrefix struct { AuthUnrestrictedBindKey string ScopeAllowedBindKey string ScopeOperationCount int - // UnnestScopeIdentity identifies the active cardinality-changing scope at - // the point where this set is materialized. It is intentionally derived - // from canonical PhysicalUnnest operations rather than renderer variable - // names. Prefixes on opposite sides of an unnest barrier must never share, - // even when they read the same root variable. - UnnestScopeIdentity string - NodeVariable string - EdgeVariable string + UnnestScopeIdentity string + NodeVariable string + EdgeVariable string } const ( @@ -40,10 +27,6 @@ const ( physicalTraversalPrefixEdgeVariable = "__loom_prefix_edge" ) -// PhysicalTraversalSubset contains the target-type specialization and work -// after the mandatory traversal scope. Its local variables are retained so a -// later rewrite can perform explicit alpha-renaming instead of guessing from -// AQL strings. type PhysicalTraversalSubset struct { TargetTypeBindKey string TargetVariable string @@ -51,9 +34,6 @@ type PhysicalTraversalSubset struct { ConsumerOperations []PhysicalOperation } -// PhysicalTraversalPrefixDecomposition is the only B1 output consumed by a -// future sharing rewrite. PrefixKey is stable across generated local variable -// and target-type bind names, but differs for any scoped physical behavior. type PhysicalTraversalPrefixDecomposition struct { Prefix PhysicalTraversalPrefix PrefixOperations []PhysicalOperation @@ -75,9 +55,6 @@ const ( PhysicalPrefixInvalidTarget PhysicalTraversalPrefixRejectionReason = "INVALID_TARGET_SUBSET" ) -// PhysicalTraversalPrefixError makes rejection intentional and inspectable. -// Optimizers must retain this reason in diagnostics rather than treating an -// ineligible set as an accidental non-match. type PhysicalTraversalPrefixError struct { Reason PhysicalTraversalPrefixRejectionReason Detail string @@ -94,10 +71,6 @@ func rejectPhysicalTraversalPrefix(reason PhysicalTraversalPrefixRejectionReason return &PhysicalTraversalPrefixError{Reason: reason, Detail: fmt.Sprintf(format, args...)} } -// DecomposePhysicalTraversalPrefixAt is the position-aware form used by the -// optimizer and diagnostics. The operation index is required because an -// unnest changes row cardinality for all operations that follow it while the -// source variable may remain the same root binding. func DecomposePhysicalTraversalPrefixAt(plan PhysicalPlan, set PhysicalSet, setIndex int) (PhysicalTraversalPrefixDecomposition, error) { return decomposePhysicalTraversalPrefix(plan, set, setIndex) } @@ -130,9 +103,6 @@ func decomposePhysicalTraversalPrefix(plan PhysicalPlan, set PhysicalSet, setInd return PhysicalTraversalPrefixDecomposition{}, rejectPhysicalTraversalPrefix(PhysicalPrefixInvalidRoute, "%v", err) } - // The prefix is exactly TRAVERSAL followed by edge/node project, - // generation, and auth scope. Consumer predicates start only after this - // block, so they cannot accidentally broaden the shared neighbor set. scope := set.Subplan.Operations[1:7] scopeVariable, err := validateGenericNavigationScopeBlock(scope, traversal.TargetVariable, traversal.EdgeVariable, traversal.TargetVariable) if err != nil { @@ -185,11 +155,6 @@ func decomposePhysicalTraversalPrefix(plan PhysicalPlan, set PhysicalSet, setInd }, nil } -// physicalUnnestScopeIdentityAt returns a deterministic identity for every -// active top-level unnest before operationIndex. A set after an unnest still -// reads the original root binding, so source-variable equality alone is not a -// sufficient sharing key. The expression and join mode are part of the key so -// a rewrite cannot cross a different cardinality/null-preservation contract. func physicalUnnestScopeIdentityAt(plan PhysicalPlan, operationIndex int) (string, error) { if operationIndex < 0 { return "", nil diff --git a/internal/dataframe/compiler/ir/scope.go b/internal/dataframe/compiler/ir/scope.go index d95c47c84..37d86843c 100644 --- a/internal/dataframe/compiler/ir/scope.go +++ b/internal/dataframe/compiler/ir/scope.go @@ -13,17 +13,6 @@ const ( physicalScopeDatasetGenerationField = datasetGenerationField ) -// ValidateGenericPhysicalPlanScope proves the authorization and project-scope -// contract of the navigation-only physical plan built by -// BuildGenericPhysicalPlan. It deliberately validates the physical operation -// graph rather than rendered AQL, so a renderer cannot accidentally hide a -// missing or reordered scope operation. -// -// This is intentionally narrower than PhysicalPlan.Validate: arbitrary -// physical plans may have a different scope strategy, while the generic FHIR -// navigation plan must use the exact project and authorization primitives -// checked here. It also requires an exact dataset-generation predicate for -// each scanned graph document, including the legacy null-generation case. func ValidateGenericPhysicalPlanScope(plan PhysicalPlan) error { if err := plan.Validate(); err != nil { return fmt.Errorf("validate physical plan before verifying generic scope: %w", err) @@ -416,9 +405,6 @@ func physicalScopeLookupResource(description, variable string) physicalScopeReso } } -// physicalScopeWindowEnd returns the first subsequent operation that can -// create another resource or terminate the plan. Scope must be established -// before either happens; otherwise a traversal can observe an unscoped row. func physicalScopeWindowEnd(operations []PhysicalOperation, start int) int { for index := start; index < len(operations); index++ { switch operations[index].Kind { diff --git a/internal/dataframe/compiler/ir/whole_relation_scope_evidence.go b/internal/dataframe/compiler/ir/whole_relation_scope_evidence.go index aa70818a1..a0fb3d54e 100644 --- a/internal/dataframe/compiler/ir/whole_relation_scope_evidence.go +++ b/internal/dataframe/compiler/ir/whole_relation_scope_evidence.go @@ -10,9 +10,6 @@ import ( "github.com/calypr/loom/internal/projectid" ) -// WholeRelationScopeIdentity is compiler-owned identity for a finalized -// whole-relation output. The lowerer supplies OutputID and the digest of its -// final schema after constructing the physical plan. type WholeRelationScopeIdentity struct { OutputID string Project string @@ -22,10 +19,6 @@ type WholeRelationScopeIdentity struct { SchemaDigest string } -// WholeRelationScopeEvidence is an opaque proof that one exact typed physical -// plan reads only the complete authorized source scope and has no bounded -// output window. Issuance always means whole-scope evidence; it does not attest -// row-level auth_resource_path metadata on grouped output rows. type WholeRelationScopeEvidence struct { planFingerprint string identity WholeRelationScopeIdentity @@ -34,11 +27,6 @@ type WholeRelationScopeEvidence struct { digest string } -// NewWholeRelationScopeEvidence issues evidence only for complete unbounded -// typed AQL source plans whose reads have explicit scope contracts. The plan -// fingerprint is computed here so a caller cannot assert a digest for a -// different plan. Cohort and coded group stages use their typed custom-source -// contracts in addition to generic operation-graph scope validation. func NewWholeRelationScopeEvidence(plan PhysicalPlan, identity WholeRelationScopeIdentity) (WholeRelationScopeEvidence, error) { if err := plan.Validate(); err != nil { return WholeRelationScopeEvidence{}, fmt.Errorf("whole-relation evidence physical plan: %w", err) @@ -65,8 +53,6 @@ func NewWholeRelationScopeEvidence(plan PhysicalPlan, identity WholeRelationScop return WholeRelationScopeEvidence{planFingerprint: fingerprint, identity: identity, issuerKind: issuerKind, digest: digest}, nil } -// Matches reports whether this evidence was issued for the exact current plan -// and output identity. It is safe to persist Digest only after this check. func (e WholeRelationScopeEvidence) Matches(plan PhysicalPlan, identity WholeRelationScopeIdentity) bool { if e.digest == "" || e.planFingerprint == "" || physicalPlanFingerprint(plan) != e.planFingerprint || !sameWholeRelationScopeIdentity(e.identity, identity) { @@ -82,8 +68,6 @@ func (e WholeRelationScopeEvidence) Matches(plan PhysicalPlan, identity WholeRel } } -// Digest returns a stable digest of the validated plan fingerprint and exact -// output identity. The zero value returns an empty digest. func (e WholeRelationScopeEvidence) Digest() string { return e.digest } diff --git a/internal/dataframe/compiler/ir/workspace_combine_scope_operand.go b/internal/dataframe/compiler/ir/workspace_combine_scope_operand.go index 6a4ba7b6e..005785b4e 100644 --- a/internal/dataframe/compiler/ir/workspace_combine_scope_operand.go +++ b/internal/dataframe/compiler/ir/workspace_combine_scope_operand.go @@ -16,10 +16,6 @@ const ( workspaceCombinePrivateOperand workspaceCombineOperandKind = "private-aql-prefix" ) -// WorkspaceCombineScopeOperand is opaque provenance for one ordered input to -// a workspace Combine. Callers can only obtain one through a validating -// constructor below; its fields are copied so later resolver mutations cannot -// change the issued evidence. type WorkspaceCombineScopeOperand struct { inputIndex int ref PhysicalCombineInputRef @@ -31,9 +27,6 @@ type WorkspaceCombineScopeOperand struct { resolvedColumns []ResolvedClickHouseColumn } -// NewWorkspaceOutputScopeOperand binds one exact same-workspace output ref to -// the already validated evidence for that output. The source plan may be an -// ordinary AQL plan or a previously composed ClickHouse plan. func NewWorkspaceOutputScopeOperand(inputIndex int, ref PhysicalCombineInputRef, sourcePlan PhysicalPlan, sourceIdentity WholeRelationScopeIdentity, sourceEvidence WholeRelationScopeEvidence) (WorkspaceCombineScopeOperand, error) { if inputIndex < 0 || strings.TrimSpace(ref.WorkspaceOutputID) == "" || ref.WorkspaceOutputID != strings.TrimSpace(ref.WorkspaceOutputID) || ref.TableID != "" || ref.RevisionID != "" || ref.OutputID != "" || ref.PrivateStageID != "" { @@ -60,10 +53,6 @@ func NewWorkspaceOutputScopeOperand(inputIndex int, ref PhysicalCombineInputRef, }, nil } -// NewPublishedWorkspaceScopeOperand binds an exact immutable published -// revision returned by the server resolver to the requested output scope. -// Call it at the resolver boundary, immediately after exact revision lookup, -// reauthorization, and schema validation. func NewPublishedWorkspaceScopeOperand(inputIndex int, ref PhysicalCombineInputRef, resolved ResolvedClickHouseTable, expectedScope WholeRelationScopeIdentity) (WorkspaceCombineScopeOperand, error) { if inputIndex < 0 || strings.TrimSpace(ref.TableID) == "" || ref.TableID != strings.TrimSpace(ref.TableID) || strings.TrimSpace(ref.RevisionID) == "" || ref.RevisionID != strings.TrimSpace(ref.RevisionID) || @@ -108,8 +97,6 @@ func NewPublishedWorkspaceScopeOperand(inputIndex int, ref PhysicalCombineInputR }, nil } -// NewPrivateAQLPrefixScopeOperand binds the composite plan's exact private -// AQL stage to the evidence issued for that complete source query. func NewPrivateAQLPrefixScopeOperand(inputIndex int, ref PhysicalCombineInputRef, sourcePlan PhysicalPlan, sourceIdentity WholeRelationScopeIdentity, sourceEvidence WholeRelationScopeEvidence) (WorkspaceCombineScopeOperand, error) { if inputIndex < 0 || strings.TrimSpace(ref.PrivateStageID) == "" || ref.PrivateStageID != strings.TrimSpace(ref.PrivateStageID) || ref.TableID != "" || ref.RevisionID != "" || ref.OutputID != "" || ref.WorkspaceOutputID != "" { @@ -137,9 +124,6 @@ func NewPrivateAQLPrefixScopeOperand(inputIndex int, ref PhysicalCombineInputRef }, nil } -// NewWorkspaceCombineScopeEvidence composes the exact authorized provenance -// of every ordered input and binds it to the actual optimized typed Combine -// plan and its compiler-final output identity. func NewWorkspaceCombineScopeEvidence(actualPlan PhysicalPlan, finalizedIdentity WholeRelationScopeIdentity, orderedOperands []WorkspaceCombineScopeOperand) (WholeRelationScopeEvidence, error) { if err := validateWorkspaceCombinePhysicalPlan(actualPlan, finalizedIdentity); err != nil { return WholeRelationScopeEvidence{}, err diff --git a/internal/dataframe/compiler/lower/auth_scope.go b/internal/dataframe/compiler/lower/auth_scope.go index ae1645eb3..09ec4c7b5 100644 --- a/internal/dataframe/compiler/lower/auth_scope.go +++ b/internal/dataframe/compiler/lower/auth_scope.go @@ -5,7 +5,6 @@ import ( "github.com/calypr/loom/internal/dataframe/semantic" ) -// effectiveAuthScopeUnrestricted makes resolved request scopes authoritative. func effectiveAuthScopeUnrestricted(paths []string, mode authscope.ReadScopeMode) bool { switch mode { case authscope.ReadScopeUnrestricted: @@ -15,7 +14,6 @@ func effectiveAuthScopeUnrestricted(paths []string, mode authscope.ReadScopeMode case "": return len(paths) == 0 default: - // An invalid internal mode must fail closed rather than bypass scope. return false } } diff --git a/internal/dataframe/compiler/lower/child_sets.go b/internal/dataframe/compiler/lower/child_sets.go index 67fe5b566..081b304e5 100644 --- a/internal/dataframe/compiler/lower/child_sets.go +++ b/internal/dataframe/compiler/lower/child_sets.go @@ -124,22 +124,7 @@ func buildOptionalChildPhysicalSet(physical *ir.PhysicalPlan, setIndex int, pare } projections = append(projections, ir.PhysicalProjection{Name: traversalColumnName(projectionPrefix, slice.Name), Expression: &expression}) } - // A traversal-time projection is the production form of selector reuse. It - // computes selector arrays in the original child subquery and removes the - // payload before materialization. The older prepared-set pass is retained - // only for explicit experiments and is never layered on top of this shape. if policy.RuleEnabled(ir.PhysicalOptimizationRuleCompactProjection) { - // Dynamic maps and deferred pivot families both read the original - // resource payload at the outer return boundary. A selector-only - // projection would discard that payload and leave their lookup/pivot - // evaluation iterating over `child_set_N.payload` (null). Keep the - // compact identity/payload contract for those sets; ordinary selector - // consumers still use the cheaper single-materialization projection. - // - // A pivot's public columns are OBJECT_LOOKUP projections over its - // deferred map, so projectPhysicalChildSet cannot discover its key/value - // selectors from the return projection list. Do not drop payload until a - // correlated pivot-specific projection contract exists. if len(child.DynamicMaps) == 0 && len(child.Pivots) == 0 && len(child.OwnerRecords) == 0 { projectPhysicalChildSet(&set, child.ResourceType, projections) } else { @@ -205,12 +190,6 @@ func buildPhysicalChildRouteScope(physical *ir.PhysicalPlan, child semantic.Sema return scope, nil } -// compactPhysicalSetOutput retains graph identity and only the payload needed -// by downstream selectors/rich consumers. Scope predicates run before this -// projection, so project, generation, and authorization metadata do not need -// to remain in the post-window set. Nested traversal still receives _id, and -// UNIQUE/SORT retain both identity keys to preserve duplicate and ordering -// semantics. func compactPhysicalSetOutput(node semantic.SemanticNode) *ir.PhysicalSetOutput { fields := []ir.PhysicalSetOutputField{ ir.PhysicalSetGraphIDField, @@ -233,10 +212,6 @@ func compactPhysicalSetOutput(node semantic.SemanticNode) *ir.PhysicalSetOutput return &ir.PhysicalSetOutput{Fields: fields} } -// prepareRichChildSet adds a generic selector cache when at least two rich -// consumers read the same child relationship set. The cache is deliberately -// selector-based (not FHIR-resource-specific): any generated resource type -// with repeated aggregate, pivot, or slice selectors can use it. func prepareRichChildSet(set *ir.PhysicalSet, resourceType string, projections []ir.PhysicalProjection, policy ir.PhysicalOptimizationPolicy) { if !policy.RuleEnabled(ir.PhysicalOptimizationRulePreparedSelectors) { return @@ -300,9 +275,6 @@ func prepareRichChildSet(set *ir.PhysicalSet, resourceType string, projections [ } eligible := map[string]string{} fields := make([]ir.PhysicalPreparedField, 0) - // Map iteration order must not influence the physical plan. Stable field - // allocation keeps generated AQL and bind-variable names deterministic, - // which is important for explain fixtures and cache keys. keys := make([]string, 0, len(counts)) for key := range counts { keys = append(keys, key) @@ -331,10 +303,6 @@ func prepareRichChildSet(set *ir.PhysicalSet, resourceType string, projections [ if extract == nil || extract.Source.Variable != set.Variable || extract.UnitNormalization != nil { return } - // Fallback selectors are an ordered fallback chain. A prepared - // primary value alone cannot preserve that contract, so leave these - // extracts on the original renderer path until the prepared schema - // can represent fallback alternatives explicitly. if len(extract.Fallbacks) == 0 { if field, ok := eligible[physicalSelectorIdentity(extract.Selector)]; ok { extract.Prepared = &ir.PhysicalPreparedReference{SetVariable: set.Prepared.Variable, Field: field} @@ -390,10 +358,6 @@ func prepareRichChildSet(set *ir.PhysicalSet, resourceType string, projections [ } } -// projectPhysicalChildSet attaches selector values to the original set item -// rather than creating a second prepared array. A fallback selector is an -// ordered multi-source contract that cannot be represented by one projected -// field, so the entire set conservatively retains its existing payload output. func projectPhysicalChildSet(set *ir.PhysicalSet, resourceType string, projections []ir.PhysicalProjection) { if set == nil || len(projections) == 0 { return diff --git a/internal/dataframe/compiler/lower/correlated_binding.go b/internal/dataframe/compiler/lower/correlated_binding.go index a72c1268b..716888b8a 100644 --- a/internal/dataframe/compiler/lower/correlated_binding.go +++ b/internal/dataframe/compiler/lower/correlated_binding.go @@ -9,10 +9,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// LowerCorrelatedBinding converts the checked schema binding into the one -// physical representation shared by filters and projections. Callers provide -// bound terminology values; selectors and scope are never rendered as user -// supplied query text. func LowerCorrelatedBinding(resourceType string, binding fhirschema.CorrelatedBinding, source ir.PhysicalValue, systemBindKey, codeBindKey string) (ir.PhysicalCorrelation, error) { return LowerCorrelatedBindingWithVersion(resourceType, binding, source, systemBindKey, codeBindKey, "") } @@ -36,10 +32,6 @@ func LowerCorrelatedBindingWithVersion(resourceType string, binding fhirschema.C } choiceArms := append([]string(nil), checked.ChoiceArms...) if len(choiceArms) == 0 { - // A checked value[x] selector names its requested arm even when older - // authoring omitted ChoiceArms. Infer only that structural arm; this is - // not a clinical equivalence and lets runtime data expose another arm as - // INVALID_CHOICE_ARM instead of disappearing as null. valuePath := checked.ValueSelector.CanonicalPath() if first, _, found := strings.Cut(valuePath, "."); found || strings.HasPrefix(valuePath, "value") { if found { @@ -82,10 +74,6 @@ func LowerCorrelatedBindingWithVersion(resourceType string, binding fhirschema.C return correlation, nil } -// LowerExtensionBinding converts the checked ancestor-aware extension shape -// into the same physical correlation IR used by terminology pivots. URL -// literals are bound individually so every ancestor must match in its own -// lexical extension loop before the terminal value is evaluated. func LowerExtensionBinding(resourceType string, binding fhirschema.ExtensionBinding, source ir.PhysicalValue, urlBindKeys []string) (ir.PhysicalCorrelation, error) { checked, err := fhirschema.ValidateExtensionBinding(resourceType, binding) if err != nil { @@ -128,17 +116,10 @@ func LowerExtensionBinding(resourceType string, binding fhirschema.ExtensionBind }, nil } -// LowerCorrelatedPredicate uses the same binding as projection lowering. The -// code system and code are independent bind values but are evaluated against -// the same Coding item by the renderer. func LowerCorrelatedPredicate(physical *ir.PhysicalPlan, resourceType string, binding fhirschema.CorrelatedBinding, source ir.PhysicalValue, code spec.CodeValue) (ir.PhysicalPredicate, error) { return LowerCorrelatedPredicateWithIdentity(physical, resourceType, binding, source, code, "") } -// LowerCorrelatedPredicateWithIdentity is the filter-safe entry point for -// callers lowering more than one correlated predicate over the same source. -// The identity is part of both bind keys, so later filters cannot overwrite an -// earlier system/code pair in the shared physical bind map. func LowerCorrelatedPredicateWithIdentity(physical *ir.PhysicalPlan, resourceType string, binding fhirschema.CorrelatedBinding, source ir.PhysicalValue, code spec.CodeValue, identity string) (ir.PhysicalPredicate, error) { if physical == nil { return ir.PhysicalPredicate{}, fmt.Errorf("physical plan is required") diff --git a/internal/dataframe/compiler/lower/doc.go b/internal/dataframe/compiler/lower/doc.go index 7c5e82141..646e1dfa2 100644 --- a/internal/dataframe/compiler/lower/doc.go +++ b/internal/dataframe/compiler/lower/doc.go @@ -1,4 +1 @@ -// Package lower lowers logical FHIR dataframe plans into typed physical IR. -// It is the only compiler layer allowed to resolve stored graph routes and -// endpoint-versus-native traversal strategies. package lower diff --git a/internal/dataframe/compiler/lower/graph_plan.go b/internal/dataframe/compiler/lower/graph_plan.go index 33d7549c9..4438c3916 100644 --- a/internal/dataframe/compiler/lower/graph_plan.go +++ b/internal/dataframe/compiler/lower/graph_plan.go @@ -14,9 +14,6 @@ const ( graphLookaheadBindKey = "graph_limit" ) -// BuildGraphPhysicalPlan lowers an explicit graph frontend onto the same -// scoped root/traversal lowering used by dataframe recipes. Each route gets -// its own path set, so sibling branches form independent unions. func BuildGraphPhysicalPlan(output semantic.OutputPlan, context semantic.ExecutionContext, limit int, policy ir.PhysicalOptimizationPolicy) (ir.PhysicalPlan, error) { if limit < 1 || limit > 10000 { return ir.PhysicalPlan{}, fmt.Errorf("graph limit must be between 1 and 10000") @@ -73,11 +70,6 @@ func BuildGraphPhysicalPlan(output semantic.OutputPlan, context semantic.Executi }}, }) - // A root-only path is useful when the request has only OPTIONAL branches: - // an optional miss leaves the root eligible and contributes no synthetic - // child. Once any REQUIRED route is declared, however, returning the root - // itself would leak a path the caller did not ask for; required semi-joins - // below still gate root membership. pathSets := make([]string, 0, countGraphTraversals(output.Root)+1) if !graphHasRequiredTraversal(output.Root) { pathSets = append(pathSets, "path_root") @@ -156,10 +148,6 @@ func normalizeGraphMatchModes(node semantic.SemanticNode) semantic.SemanticNode return copy } -// graphRequiredSemiJoinRoot keeps REQUIRED membership proofs for routes that -// are reachable from the root through required ancestors. A required child -// below an OPTIONAL branch cannot disqualify the root when that branch is -// absent; its path set simply contributes no rows. func graphRequiredSemiJoinRoot(node semantic.SemanticNode) semantic.SemanticNode { copy := node copy.Children = make([]semantic.SemanticNode, len(node.Children)) @@ -182,10 +170,6 @@ func graphRequiredSemiJoinNode(node semantic.SemanticNode, ancestorsRequired boo return copy } -// CompileResolvedGraphPlan selects the single graph output from the resolved -// recipe boundary and lowers it to graph/path IR. The graph frontend creates -// one output deliberately; accepting multiple outputs here would make path -// union and lookahead-limit semantics ambiguous. func CompileResolvedGraphPlan(resolved semantic.ResolvedRecipePlan, limit int, policy ir.PhysicalOptimizationPolicy) (ir.PhysicalPlan, error) { if len(resolved.SemanticPlan.Outputs) != 1 { return ir.PhysicalPlan{}, fmt.Errorf("graph query requires exactly one output, got %d", len(resolved.SemanticPlan.Outputs)) diff --git a/internal/dataframe/compiler/lower/group_field_source_context.go b/internal/dataframe/compiler/lower/group_field_source_context.go index 73e1d12de..d868ea7b3 100644 --- a/internal/dataframe/compiler/lower/group_field_source_context.go +++ b/internal/dataframe/compiler/lower/group_field_source_context.go @@ -184,18 +184,12 @@ func groupFieldSourcePrefixIsRowPreserving(plan *ir.PhysicalPlan, root *ir.Physi return false } default: - // Sets, traversals, windows, table reshapes and prior row changing - // stages cannot be silently treated as source occurrence context. return false } } return rootCount == 1 && unnestSeen == unnestCount && returns == 1 } -// Population roots are row-preserving because the typed root renderer groups -// the selected root key before loading the root document. The context checks -// only the source and identity structure; PhysicalPlan validation owns filter, -// resource-scope, and authorization validation. func groupFieldSourcePopulationIsTyped(population *ir.PhysicalPopulationRootSource, rootVariable string) bool { if population == nil { return true @@ -233,10 +227,6 @@ func groupFieldSourcePopulationIsTyped(population *ir.PhysicalPopulationRootSour return population.RootKey.Variable == current && physicalPathEquals(population.RootKey.Path, "_key") } -// A PhysicalSet is an array-valued LET, so a validated direct related-field -// set preserves each outer row. Require its correlated traversal to remain -// anchored at the physical root; sets from other source shapes are not proof -// that root/active occurrence identity survived. func groupFieldSourceRowPreservingSet(operation ir.PhysicalOperation, rootVariable string) bool { set := operation.Set if operation.Kind != ir.PhysicalSetOp || set == nil || set.Variable == "" || rootVariable == "" || diff --git a/internal/dataframe/compiler/lower/group_field_source_related.go b/internal/dataframe/compiler/lower/group_field_source_related.go index faccec976..fd1b066c6 100644 --- a/internal/dataframe/compiler/lower/group_field_source_related.go +++ b/internal/dataframe/compiler/lower/group_field_source_related.go @@ -10,10 +10,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// BindGroupRelatedOccurrenceSource resolves occurrence identities and the -// canonical source-shape digest for one related Group field source. Lifecycle -// choices pass graph route facts here; they do not synthesize occurrence IDs -// or source-shape digests themselves. func BindGroupRelatedOccurrenceSource( context *CompiledGroupFieldSourceContext, source recipe.ConstructionGroupFieldSource, @@ -45,10 +41,6 @@ func BindGroupRelatedOccurrenceSource( return source, nil } -// ResolveGroupRelatedOccurrenceSource is shared by capability issuance and -// durable binding validation. It returns route-prefix occurrence IDs, the -// route ID, and the selected-value source-shape digest from compiler-owned -// source context and exact graph route facts. func ResolveGroupRelatedOccurrenceSource( context *CompiledGroupFieldSourceContext, valueKind string, diff --git a/internal/dataframe/compiler/lower/group_related_occurrence_lower.go b/internal/dataframe/compiler/lower/group_related_occurrence_lower.go index 17c21d336..9c204c24e 100644 --- a/internal/dataframe/compiler/lower/group_related_occurrence_lower.go +++ b/internal/dataframe/compiler/lower/group_related_occurrence_lower.go @@ -442,10 +442,6 @@ func ensureGroupRelatedOccurrenceAdapter( return adapter, nil } -// ensureGroupRelatedRootKeyInput carries the proven source root key through -// the typed Group input row. A streamed source Group otherwise prunes an -// unused source RETURN to {}, leaving the Group-owned route adapter without -// the anchor it needs to re-run the selected route. func ensureGroupRelatedRootKeyInput( plan *ir.PhysicalPlan, stage *ir.PhysicalConstructionStage, diff --git a/internal/dataframe/compiler/lower/helpers.go b/internal/dataframe/compiler/lower/helpers.go index 1b8a6253d..dea3fdbc7 100644 --- a/internal/dataframe/compiler/lower/helpers.go +++ b/internal/dataframe/compiler/lower/helpers.go @@ -72,10 +72,6 @@ func selectorExecutionMode(resourceType string, selector spec.Selector, fallback }) } -// selectorExecutionModeForExpression is the shared selector classifier used -// by expression frontends. It resolves schema metadata once and passes the -// complete physical expression context through the same proof used by -// generic field lowering. func selectorExecutionModeForExpression(resourceType string, selector spec.Selector, fallbacks []spec.Selector) ir.PhysicalSelectorExecutionMode { return selectorExecutionModeWithContext(selectorModeContext{ ResourceType: resourceType, diff --git a/internal/dataframe/compiler/lower/plan_orchestration.go b/internal/dataframe/compiler/lower/plan_orchestration.go index 521acb354..b9d3271c6 100644 --- a/internal/dataframe/compiler/lower/plan_orchestration.go +++ b/internal/dataframe/compiler/lower/plan_orchestration.go @@ -8,17 +8,10 @@ import ( semanticpkg "github.com/calypr/loom/internal/dataframe/semantic" ) -// BuildGenericPhysicalPlanWithPolicy lowers one canonical output with an -// explicit request execution context. Keeping these values separate prevents -// runtime provenance from being mistaken for semantic output structure. func BuildGenericPhysicalPlanWithPolicy(output semanticpkg.OutputPlan, context semanticpkg.ExecutionContext, policy ir.PhysicalOptimizationPolicy) (ir.PhysicalPlan, error) { return buildGenericPhysicalPlanWithPolicy(output, context, policy, nil) } -// buildGenericPhysicalPlanWithPolicy is the shared physical-plan builder. A -// field lowerer is supplied by frontends whose canonical semantic fields are -// richer than selector metadata (for example persisted recipe expressions). -// A nil lowerer retains the selector-only GraphQL/dataframe behavior. func buildGenericPhysicalPlanWithPolicy(output semanticpkg.OutputPlan, context semanticpkg.ExecutionContext, policy ir.PhysicalOptimizationPolicy, lowerer semanticFieldProjectionLowerer) (ir.PhysicalPlan, error) { if strings.TrimSpace(context.Project) == "" { return ir.PhysicalPlan{}, fmt.Errorf("semantic plan project is required") @@ -110,11 +103,6 @@ func buildGenericPhysicalPlanWithPolicy(output semanticpkg.OutputPlan, context s continue } if child.MatchMode.Required() { - // Required routes are represented by the root semi-join emitted - // above for membership, but they may still need a materialized - // child set for selected fields or nested shaping. The second - // physical set is post-window output work; it cannot change root - // membership because the semi-join remains before SORT/LIMIT. if physicalNodeNeedsMaterializedSet(child) { childSetIndex++ childProjectionPrefix := traversalColumnPrefix(output.TraversalColumnNaming, projectionPrefix, child.Alias) @@ -167,9 +155,6 @@ func buildGenericPhysicalPlanWithPolicy(output semanticpkg.OutputPlan, context s } continue } - // Optional children are correlated sets. Keeping them in a LET - // subquery preserves the parent row grain while allowing typed child - // filters and projections to be applied before materialization. childSetIndex++ childProjectionPrefix := traversalColumnPrefix(output.TraversalColumnNaming, projectionPrefix, child.Alias) childBindings[child.Alias] = physicalSemanticBinding{ResourceType: child.ResourceType, Source: ir.PhysicalValue{Variable: fmt.Sprintf("child_set_%d", childSetIndex)}} @@ -209,9 +194,6 @@ func buildGenericPhysicalPlanWithPolicy(output semanticpkg.OutputPlan, context s return physical, nil } -// physicalNodeNeedsMaterializedSet reports whether a node or any optional -// descendant has shaped output. Materializing an otherwise unselected parent -// is necessary to give nested sets a stable correlated source variable. func physicalNodeNeedsMaterializedSet(node semanticpkg.SemanticNode) bool { if len(node.Fields) != 0 || len(node.Filters) != 0 || len(node.Pivots) != 0 || len(node.OwnerRecords) != 0 || len(node.Aggregates) != 0 || len(node.Slices) != 0 || len(node.DynamicMaps) != 0 { return true @@ -240,10 +222,6 @@ func appendProjectScope(operations []ir.PhysicalOperation, variables []string, r return operations } -// appendDatasetGenerationScope applies the same exact generation bind to -// every physical document participating in a scan/traversal. With a nil bind -// value this renders `dataset_generation == null`, deliberately isolating -// legacy documents from later generation-qualified loads. func appendDatasetGenerationScope(operations []ir.PhysicalOperation, variables []string, relationship string, node semanticpkg.SemanticNode) []ir.PhysicalOperation { right := ir.PhysicalValue{BindKey: datasetGenerationBindKey} for _, variable := range variables { diff --git a/internal/dataframe/compiler/lower/population.go b/internal/dataframe/compiler/lower/population.go index 84ccd633e..ab98145a9 100644 --- a/internal/dataframe/compiler/lower/population.go +++ b/internal/dataframe/compiler/lower/population.go @@ -15,10 +15,6 @@ const ( populationSourceCollectionBindKey = "population_source_collection" ) -// configurePopulationRootSource changes the root access path from a complete -// resource collection scan to an indexed selection-member scan. It joins the -// selected FHIR resources by id, then traverses the population route in -// reverse until it reaches the dataframe root type. func configurePopulationRootSource(physical *ir.PhysicalPlan, output semantic.SemanticNode, population *semantic.SemanticPopulation, context semantic.ExecutionContext, policy ir.PhysicalOptimizationPolicy) error { if population == nil { return nil diff --git a/internal/dataframe/compiler/lower/recipe_compile.go b/internal/dataframe/compiler/lower/recipe_compile.go index 4c53ce7e8..327050496 100644 --- a/internal/dataframe/compiler/lower/recipe_compile.go +++ b/internal/dataframe/compiler/lower/recipe_compile.go @@ -1,9 +1,5 @@ package lower -// This file contains the canonical recipe lowering boundary. Persisted -// recipes are a frontend: after resolution each output is lowered to the same -// ir.PhysicalPlan used by the GraphQL dataframe compiler. - import ( "fmt" "strings" @@ -18,108 +14,57 @@ import ( "github.com/calypr/loom/internal/dataframe/unit" ) -// CompiledRecipe is orchestration metadata around one canonical physical plan -// per output. It deliberately contains no recipe-specific traversal, -// projection, or renderer structures. Output order is the persisted recipe -// order and therefore part of the stable materialization contract. type CompiledRecipe struct { - Version int - RecipeDigest string - TranslationVersion string - ResolvedSchemaDigest string - ScopeDigest string - SourceGeneration string - Outputs []CompiledRecipeOutput - // WorkspaceDependencies contains the topologically compiled, unselected - // outputs required by the selected public outputs. It is execution support - // only; public output ordering remains in Outputs. + Version int + RecipeDigest string + TranslationVersion string + ResolvedSchemaDigest string + ScopeDigest string + SourceGeneration string + Outputs []CompiledRecipeOutput WorkspaceDependencies []CompiledRecipeOutput } -// CompiledRecipeOutput is the canonical compiler result for one output. -// Columns is metadata for the stream/materialization layer; all query -// semantics live in Plan. type CompiledRecipeOutput struct { - Name string - TranslationVersion string - RootResourceType string - RowGrain spec.RowGrain - RootColumnNaming recipe.RootColumnNaming - Columns []string - // OutputSchema is the compiler-owned ordered projection schema. It is - // captured from the finalized physical RETURN projections rather than - // reconstructed by a transport adapter from the semantic recipe tree. - // Internal projections remain present here so execution and diagnostics can - // distinguish them from the public dataframe contract. - OutputSchema []CompiledOutputColumn - // WorkspaceOutputSources records compiler-resolved sibling schemas for a - // terminal Combine. These refs are structural only; execution requires a - // server-owned workspace capture before an artifact can be read. + Name string + TranslationVersion string + RootResourceType string + RowGrain spec.RowGrain + RootColumnNaming recipe.RootColumnNaming + Columns []string + OutputSchema []CompiledOutputColumn WorkspaceOutputSources []WorkspaceOutputSource - // ScopeEvidence is present only for a compiler-validated complete, unbounded - // output relation. ScopeEvidenceIdentity is the exact output, scope, and - // finalized schema identity used when issuing it. - ScopeEvidence *ir.WholeRelationScopeEvidence - ScopeEvidenceIdentity *ir.WholeRelationScopeIdentity - // RowIdentity describes the stable semantic identity used by publication - // targets. It is metadata only; the physical plan remains authoritative for - // the returned values. - RowIdentity *spec.RowIdentity - // DynamicColumns is post-query validation metadata. The physical plan - // contains the bounded projections; observed-key/type checks remain above - // the backend execution boundary. - DynamicColumns []DynamicColumnMetadata - // Stages exposes the exact typed schemas and operation capabilities used - // to compile an ordered construction sequence. - Stages []CompiledStageDescriptor - Plan ir.PhysicalPlan - // OptimizedPlan is populated by the request orchestrator after all outputs - // have been lowered. Keeping it separate from Plan lets preview windows be - // rendered repeatedly without re-running the optimizer or lowering stage. - OptimizedPlan *ir.PhysicalPlan + ScopeEvidence *ir.WholeRelationScopeEvidence + ScopeEvidenceIdentity *ir.WholeRelationScopeIdentity + RowIdentity *spec.RowIdentity + DynamicColumns []DynamicColumnMetadata + Stages []CompiledStageDescriptor + Plan ir.PhysicalPlan + OptimizedPlan *ir.PhysicalPlan } -// WorkspaceOutputSource binds an exact same-bundle output ID to its -// compiler-owned finalized schema. It contains no client-declared schema or -// receipt/head identity. type WorkspaceOutputSource struct { InputIndex int OutputID string Schema []CompiledOutputColumn } -// CompiledOutputColumn describes one finalized physical projection. Kind and -// Cardinality are logical, backend-neutral values; Internal projections are -// never exposed by dataframe transports. type CompiledOutputColumn struct { - ID string - Name string - Label string - // SemanticValueKind distinguishes public labels whose row identity is - // carried by hidden semantic tuples rather than by the display scalar. - SemanticValueKind string - // SemanticPath is the stable FHIR/provenance identity for this column. - // Physical names are deliberately excluded so storage renames do not - // invalidate Explorer configuration. - SemanticPath string - Kind string - Cardinality string - Nullable bool - // NormalizedUnit is the compiler-resolved identity of the value in this - // column. It does not describe source conversion rules or selectors. - NormalizedUnit *unit.UnitIdentity - Internal bool - Identity bool - Discovered bool - SourceChild *lineage.SourceChild - // PresenceCompanionName identifies a compiler-owned hidden boolean column - // that preserves whether this scalar existed before stage materialization. - PresenceCompanionName string - // RelatedRecordAnchor marks the hidden exact terminal document identity - // emitted by RELATED_EXPAND. It is copied only by row-preserving stages. - RelatedRecordAnchor *CompiledRelatedRecordAnchor - // RootContributorResourceType marks a compiler-proven root identity or a - // hidden, globally deduplicated set of contributing root identities. + ID string + Name string + Label string + SemanticValueKind string + SemanticPath string + Kind string + Cardinality string + Nullable bool + NormalizedUnit *unit.UnitIdentity + Internal bool + Identity bool + Discovered bool + SourceChild *lineage.SourceChild + PresenceCompanionName string + RelatedRecordAnchor *CompiledRelatedRecordAnchor RootContributorResourceType string } @@ -130,7 +75,6 @@ type CompiledRelatedRecordAnchor struct { TargetResourceType string } -// CloneCompiledOutputSchema copies output metadata, including optional unit identities. func CloneCompiledOutputSchema(columns []CompiledOutputColumn) []CompiledOutputColumn { cloned := append([]CompiledOutputColumn(nil), columns...) for index := range cloned { @@ -163,10 +107,6 @@ type DynamicColumnMetadata struct { Discovered bool } -// outputSelectionError types a rejected caller-supplied output selection. -// Selections arrive through request bindings, so the fault is a non-retryable -// request defect; the requested identifier reaches transports only via the -// typed details map, mirroring execution.previewAdmissionError. func outputSelectionError(message string, details map[string]any) error { options := []dataframeerrors.ErrorOption{dataframeerrors.WithRetryable(false)} if details != nil { @@ -175,14 +115,6 @@ func outputSelectionError(message string, details map[string]any) error { return dataframeerrors.NewError(dataframeerrors.CodeInvalidRequest, message, options...) } -// CompileResolvedRecipePlan lowers every resolved recipe output into the -// canonical physical IR. The optimizer and renderer are intentionally not -// called here: callers can apply an explicit PhysicalOptimizationPolicy and -// execution window at the same boundary used by generic dataframe requests. -// -// Dynamic maps are lowered into bounded named projections below. Their -// observed-key/type checks remain metadata for post-query validation; no -// runtime map-shaped AQL is emitted. func CompileResolvedRecipePlan(resolved semantic.ResolvedRecipePlan, policy ir.PhysicalOptimizationPolicy) (CompiledRecipe, error) { semanticPlan := resolved.SemanticPlan if semanticPlan.Version <= 0 || strings.TrimSpace(semanticPlan.RecipeDigest) == "" { @@ -268,8 +200,6 @@ func CompileResolvedRecipePlan(resolved semantic.ResolvedRecipePlan, policy ir.P compiledByName[output.Name] = compiled resolvedSchemas[output.Name] = CloneCompiledOutputSchema(compiled.OutputSchema) } - // Public output order remains authored order even when the compiler had to - // resolve forward references first. for _, output := range semanticPlan.Outputs { if len(selected) > 0 && !selected[output.Name] { continue diff --git a/internal/dataframe/compiler/lower/recipe_construction.go b/internal/dataframe/compiler/lower/recipe_construction.go index 807904386..79e1af25f 100644 --- a/internal/dataframe/compiler/lower/recipe_construction.go +++ b/internal/dataframe/compiler/lower/recipe_construction.go @@ -20,8 +20,6 @@ const constructionRowID = "__loom_row_id" const sourceExpansionIdentity = "__loom_expansion_identity" const rootContributorSetColumn = "__loom_root_contributor_keys" -// CompiledStageDescriptor is the compiler-owned schema and capability view -// for one exact stage reference. Columns describe the stage output. type CompiledStageDescriptor struct { ID string InputStageID string @@ -35,8 +33,6 @@ type CompiledStageDescriptor struct { GroupFieldSourceContext *CompiledGroupFieldSourceContext } -// CompiledGroupFieldSourceContext is the compiler-owned, immutable source -// shape offered to Group capability discovery and rechecked during lowering. type CompiledGroupFieldSourceContext struct { SourceStageID string SourceShapeID string @@ -118,11 +114,6 @@ type StageOperationCapability struct { Reason string } -// DescribeConstructionSourceStage returns the compiler-owned capability view -// for the exact resolved source schema. It is also used by capability-only -// discovery for a new, zero-column table: the physical compiler has still -// resolved row identity and source columns, but the result is never treated -// as an executable public output. func DescribeConstructionSourceStage(schema []CompiledOutputColumn, rootResourceType string, sourcePlan *ir.PhysicalPlan) (CompiledStageDescriptor, error) { identity := constructionSourceIdentity(schema) if identity == "" { @@ -350,10 +341,6 @@ func appendRecipeConstructionStages(plan *ir.PhysicalPlan, outputName, rootResou retainRootContributors := retainForLaterOperations || retainForPopulation if retainForPopulation && !retainForLaterOperations && (constructionTerminalCodingGroupRequeriesExpandedPreimage(construction.Steps, plan, sourceIdentity, rootResourceType) || constructionTerminalRelatedCodingGroupRequeriesPreimage(construction.Steps, plan, sourceIdentity, rootResourceType)) { - // The terminal expanded CODING_IDENTITY Group's exact occurrence preimage - // is re-queried for row lineage. Its typed root key remains in the source - // schema, while a per-group contributor array would defeat streaming and - // duplicate the exact preimage lookup. retainRootContributors = false } priorSchema, priorIdentity := resolvedSource, sourceIdentity @@ -574,10 +561,6 @@ func constructionCohortAnchorIndex(construction recipe.Construction, cohort *coh return -1, fmt.Errorf("cohort afterStepId %q does not name a construction step", cohort.AfterStepID) } -// constructionOperationCanDropRows identifies successful construction stages -// that can remove an input row. Cohort missing-member preservation follows -// this typed operation policy: a missing source resource cannot satisfy a -// row-dropping stage before the cohort. func constructionOperationCanDropRows(operation recipe.ConstructionOperation) bool { switch operation.Kind { case recipe.ConstructionFilterOp, recipe.ConstructionRelatedEligibilityOp: @@ -589,8 +572,6 @@ func constructionOperationCanDropRows(operation recipe.ConstructionOperation) bo case recipe.ConstructionGroupOp: return operation.Group != nil && operation.Group.MissingKeyPolicy.Normalized() == recipe.ConstructionGroupMissingKeyExclude default: - // ERROR policies abort the whole candidate. They do not successfully - // filter rows; ordinary and coded Pivot have no supported row-drop policy. return false } } @@ -859,8 +840,6 @@ func constructionRootIDPivotSourceEligible(plan *ir.PhysicalPlan, rootResourceTy return false } default: - // Traversals, expansions, and other row-shaping operations must - // remain ahead of any source limit. return false } } @@ -873,8 +852,6 @@ func groupedPivotHasDirectRootIDKey(plan *ir.PhysicalPlan, keys []ir.PhysicalGro if !ok || column.Internal || column.Kind != string(expression.KindString) || column.Cardinality == string(expression.Many) { continue } - // The lowered source projection is authoritative. Catalog fieldRef values - // can give a direct FHIR id an opaque semantic identity. if constructionPreviewProjectionIsRootID(plan, column.Name, rootResourceType) { return true } @@ -1018,9 +995,6 @@ func constructionPivotCategoryColumnIDs(construction recipe.Construction) map[st ids := make(map[string]bool) for _, step := range construction.Steps { if step.Operation.Kind == recipe.ConstructionPivotOp && step.Operation.Pivot != nil { - // Discovery compiles a temporary single-string Pivot before the user - // has selected category keys. Preserve exact source presence for every - // future Pivot category so that this probe can safely offer MISSING. ids[step.Operation.Pivot.CategoryColumnID] = true } } @@ -1117,8 +1091,6 @@ func carryConstructionPresenceCompanions(input, output []CompiledOutputColumn, p Name: companionName, Hidden: true, Value: ir.PhysicalValue{Variable: inputRow, Path: []string{companionName}}, } - // Expand already emits its generated row identity projection; keep the - // presence projection before it so schema and projection order stay aligned. if hasPhysicalProjectionName(projections, constructionRowID) { projections = insertProjectionBeforeIdentity(projections, presenceProjection, constructionRowID) } else { @@ -1964,6 +1936,7 @@ func reconcileConstructionUnpivotSchema(declarations []recipe.StageColumn, compu } for _, column := range computed { if column.Internal && column.Name != constructionRowID { + column.Identity = false result = append(result, column) } } diff --git a/internal/dataframe/compiler/lower/recipe_construction_coded_group.go b/internal/dataframe/compiler/lower/recipe_construction_coded_group.go index dc4bb92ae..625d79096 100644 --- a/internal/dataframe/compiler/lower/recipe_construction_coded_group.go +++ b/internal/dataframe/compiler/lower/recipe_construction_coded_group.go @@ -175,8 +175,6 @@ func constructionCodedGroupSourceRowsUnique(plan *ir.PhysicalPlan) error { } case ir.PhysicalFilterOp, ir.PhysicalDerivedLetOp, ir.PhysicalExpressionLetOp, ir.PhysicalSetOp, ir.PhysicalSortOp, ir.PhysicalLimitOp: - // These operations can remove or annotate rows but cannot add another - // row for the same root identity. case ir.PhysicalReturnOp: returns++ if operation.Return == nil || index != len(plan.Operations)-1 { diff --git a/internal/dataframe/compiler/lower/recipe_construction_coded_pivot.go b/internal/dataframe/compiler/lower/recipe_construction_coded_pivot.go index bd524c11b..fc43a79f9 100644 --- a/internal/dataframe/compiler/lower/recipe_construction_coded_pivot.go +++ b/internal/dataframe/compiler/lower/recipe_construction_coded_pivot.go @@ -182,7 +182,6 @@ func constructionDirectRootRowsUnique(plan *ir.PhysicalPlan, operation string) e } case ir.PhysicalFilterOp, ir.PhysicalDerivedLetOp, ir.PhysicalExpressionLetOp, ir.PhysicalSetOp, ir.PhysicalSortOp, ir.PhysicalLimitOp: - // These operations can remove or annotate rows but cannot add roots. case ir.PhysicalReturnOp: returns++ if item.Return == nil || index != len(plan.Operations)-1 { diff --git a/internal/dataframe/compiler/lower/recipe_construction_combine_metadata.go b/internal/dataframe/compiler/lower/recipe_construction_combine_metadata.go index 182be74c5..696783505 100644 --- a/internal/dataframe/compiler/lower/recipe_construction_combine_metadata.go +++ b/internal/dataframe/compiler/lower/recipe_construction_combine_metadata.go @@ -2,8 +2,6 @@ package lower import "github.com/calypr/loom/internal/dataframe/recipe" -// ConstructionCombineColumnType exposes the lowerer's authoritative physical -// type mapping for receipt-bound workspace schema metadata. func ConstructionCombineColumnType(column recipe.StageColumn) (logical, physical string, err error) { return constructionCombineColumnType(column) } diff --git a/internal/dataframe/compiler/lower/recipe_construction_composite.go b/internal/dataframe/compiler/lower/recipe_construction_composite.go index deb28b701..2bc13d130 100644 --- a/internal/dataframe/compiler/lower/recipe_construction_composite.go +++ b/internal/dataframe/compiler/lower/recipe_construction_composite.go @@ -10,23 +10,25 @@ import ( "github.com/calypr/loom/internal/dataframe/recipe" ) -// ComposeClickHouseCombine adds a terminal typed Combine to an already lowered -// AQL construction prefix. Recipe authoring still decides which mixed plans -// are reachable; this internal compiler seam accepts only an exact typed stage -// reference and one supported authorization scope. -func ComposeClickHouseCombine(prefix ir.PhysicalPlan, combine ir.PhysicalClickHouseCombine, bindings recipe.RuntimeBindings) (ir.PhysicalPlan, error) { +func ComposeClickHouseCombine(prefix ir.PhysicalPlan, combine ir.PhysicalClickHouseCombine, sourceBackend ir.PhysicalPrefixSourceBackend, bindings recipe.RuntimeBindings) (ir.PhysicalPlan, error) { + if err := sourceBackend.Validate(); err != nil { + return ir.PhysicalPlan{}, err + } + if sourceBackend == ir.PhysicalPrefixSourcePostgreSQL && bindings.AuthScopeMode == authscope.ReadScopeRestricted { + return ir.PhysicalPlan{}, fmt.Errorf("restricted private ClickHouse prefixes are unsupported on the PostgreSQL source backend: the composed PostgreSQL renderer has no registered bound authorization path column") + } if prefix.Engine == ir.PhysicalEngineClickHouse || prefix.ClickHouseCombine != nil || prefix.ClickHousePrefix != nil { - return ir.PhysicalPlan{}, fmt.Errorf("ClickHouse prefix must be a standalone AQL physical plan") + return ir.PhysicalPlan{}, fmt.Errorf("ClickHouse prefix must be a standalone construction physical plan") } if prefix.StageSequence == nil { - return ir.PhysicalPlan{}, fmt.Errorf("ClickHouse prefix requires a typed AQL construction stage sequence") + return ir.PhysicalPlan{}, fmt.Errorf("ClickHouse prefix requires a typed construction stage sequence") } if prefix.StageSequence.PreviewLimitBindKey != "" { return ir.PhysicalPlan{}, fmt.Errorf("bounded preview cannot feed a private ClickHouse artifact") } for _, operation := range prefix.Operations { if operation.Kind == ir.PhysicalLimitOp { - return ir.PhysicalPlan{}, fmt.Errorf("bounded AQL prefix cannot feed a private ClickHouse artifact") + return ir.PhysicalPlan{}, fmt.Errorf("bounded prefix cannot feed a private ClickHouse artifact") } } for _, stage := range prefix.StageSequence.Stages { @@ -55,6 +57,7 @@ func ComposeClickHouseCombine(prefix ir.PhysicalPlan, combine ir.PhysicalClickHo composite := ir.ClonePhysicalPlan(prefix) prefixContract := &ir.PhysicalClickHousePrefix{ + SourceBackend: sourceBackend, StageID: prefix.StageSequence.FinalStageID, AuthScopeMode: string(bindings.AuthScopeMode), AuthResourcePaths: append([]string(nil), bindings.AuthResourcePaths...), diff --git a/internal/dataframe/compiler/lower/recipe_construction_composite_test.go b/internal/dataframe/compiler/lower/recipe_construction_composite_test.go index 50783a940..9a9fd7c64 100644 --- a/internal/dataframe/compiler/lower/recipe_construction_composite_test.go +++ b/internal/dataframe/compiler/lower/recipe_construction_composite_test.go @@ -21,7 +21,7 @@ func TestComposeClickHouseCombineRendersExactUnboundedAQLPrefix(t *testing.T) { prefix := compileCompositePrefixTestOutput(t, constructionTestOutput(), bindings) combine := compositeAppendTestPlan(prefix.Plan.StageSequence.FinalStageID) - composite, err := ComposeClickHouseCombine(prefix.Plan, combine, bindings) + composite, err := ComposeClickHouseCombine(prefix.Plan, combine, ir.PhysicalPrefixSourceAQL, bindings) if err != nil { t.Fatalf("ComposeClickHouseCombine() error = %v", err) } @@ -48,7 +48,7 @@ func TestComposeClickHouseCombineRendersExactUnboundedAQLPrefix(t *testing.T) { } mismatchedStage := ir.ClonePhysicalPlan(composite) mismatchedStage.ClickHouseCombine.Inputs[0].PrivateStageID = "different_stage" - if _, err := aql.RenderClickHousePrefix(mismatchedStage); err == nil || !strings.Contains(err.Error(), "does not match the exact AQL prefix stage") { + if _, err := aql.RenderClickHousePrefix(mismatchedStage); err == nil || !strings.Contains(err.Error(), "does not match the exact prefix stage") { t.Fatalf("mismatched composite stage error = %v", err) } } @@ -62,7 +62,7 @@ func TestComposeClickHouseCombineRejectsRestrictedMultiPathPrefix(t *testing.T) prefix := compileCompositePrefixTestOutput(t, constructionTestOutput(), bindings) combine := compositeAppendTestPlan(prefix.Plan.StageSequence.FinalStageID) - if _, err := ComposeClickHouseCombine(prefix.Plan, combine, bindings); err == nil || !strings.Contains(err.Error(), "exactly one immutable authorization path") { + if _, err := ComposeClickHouseCombine(prefix.Plan, combine, ir.PhysicalPrefixSourceAQL, bindings); err == nil || !strings.Contains(err.Error(), "exactly one immutable authorization path") { t.Fatalf("multi-path restricted prefix error = %v", err) } } @@ -90,7 +90,7 @@ func TestComposeClickHouseCombineRejectsGroupPrefix(t *testing.T) { prefix := compileCompositePrefixTestOutput(t, output, bindings) combine := compositeAppendTestPlan(prefix.Plan.StageSequence.FinalStageID) - if _, err := ComposeClickHouseCombine(prefix.Plan, combine, bindings); err == nil || !strings.Contains(err.Error(), "GROUP prefixes cannot feed") { + if _, err := ComposeClickHouseCombine(prefix.Plan, combine, ir.PhysicalPrefixSourceAQL, bindings); err == nil || !strings.Contains(err.Error(), "GROUP prefixes cannot feed") { t.Fatalf("GROUP prefix error = %v", err) } } @@ -137,12 +137,10 @@ func TestComposeClickHouseCombineRejectsCohortGroupPrefixAndIRBypass(t *testing. } } combine := compositeAppendTestPlan(prefix.Plan.StageSequence.FinalStageID) - if _, err := ComposeClickHouseCombine(prefix.Plan, combine, bindings); err == nil || !strings.Contains(err.Error(), "COHORT_GROUP prefixes cannot feed") { + if _, err := ComposeClickHouseCombine(prefix.Plan, combine, ir.PhysicalPrefixSourceAQL, bindings); err == nil || !strings.Contains(err.Error(), "COHORT_GROUP prefixes cannot feed") { t.Fatalf("COHORT_GROUP prefix error = %v", err) } - // Construct a malformed-at-the-public-seam composite directly to ensure - // PhysicalPlan validation independently closes the same authorization gap. forged := ir.ClonePhysicalPlan(prefix.Plan) forged.Engine = ir.PhysicalEngineClickHouse forged.ClickHouseCombine = &combine @@ -150,7 +148,8 @@ func TestComposeClickHouseCombineRejectsCohortGroupPrefixAndIRBypass(t *testing. forged.BindVars[authPathKey] = bindings.AuthResourcePaths[0] forged.StageSequence.OutputAuthResourcePathBindKey = authPathKey forged.ClickHousePrefix = &ir.PhysicalClickHousePrefix{ - StageID: prefix.Plan.StageSequence.FinalStageID, AuthScopeMode: string(bindings.AuthScopeMode), + SourceBackend: ir.PhysicalPrefixSourceAQL, + StageID: prefix.Plan.StageSequence.FinalStageID, AuthScopeMode: string(bindings.AuthScopeMode), AuthResourcePaths: append([]string(nil), bindings.AuthResourcePaths...), IncludeAuthResourcePath: true, AuthResourcePathBindKey: authPathKey, } @@ -382,3 +381,61 @@ func compositeAppendTestPlan(stageID string) ir.PhysicalClickHouseCombine { Outputs: []ir.PhysicalCombineOutputColumn{{ID: "amount", Name: "amount", LogicalType: "integer", ClickHouseType: "Int64"}}, } } + +func TestComposeClickHouseCombineStampsExplicitSourceBackendAndClosesCrossRendering(t *testing.T) { + bindings := recipe.RuntimeBindings{ + Project: "project", DatasetGeneration: "generation", + AuthScopeMode: authscope.ReadScopeUnrestricted, + } + prefix := compileCompositePrefixTestOutput(t, constructionTestOutput(), bindings) + combine := compositeAppendTestPlan(prefix.Plan.StageSequence.FinalStageID) + + composite, err := ComposeClickHouseCombine(prefix.Plan, combine, ir.PhysicalPrefixSourcePostgreSQL, bindings) + if err != nil { + t.Fatalf("PostgreSQL-tagged compose error = %v", err) + } + if composite.ClickHousePrefix.SourceBackend != ir.PhysicalPrefixSourcePostgreSQL { + t.Fatalf("composer stamped backend %q", string(composite.ClickHousePrefix.SourceBackend)) + } + if _, err := aql.RenderClickHousePrefix(composite); err == nil || !strings.Contains(err.Error(), "cannot compile a ClickHouse prefix tagged source backend") { + t.Fatalf("AQL renderer accepted a PostgreSQL-tagged prefix: %v", err) + } + aqlComposite, err := ComposeClickHouseCombine(prefix.Plan, combine, ir.PhysicalPrefixSourceAQL, bindings) + if err != nil { + t.Fatalf("AQL-tagged compose error = %v", err) + } + if _, err := aql.RenderClickHousePrefix(aqlComposite); err != nil { + t.Fatalf("AQL renderer refused its own tag: %v", err) + } +} + +func TestComposeClickHouseCombineRefusesRestrictedPostgreSQLPrefix(t *testing.T) { + bindings := recipe.RuntimeBindings{ + Project: "project", DatasetGeneration: "generation", + AuthScopeMode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{"/programs/p1"}, + IncludeAuthResourcePath: true, + } + prefix := compileCompositePrefixTestOutput(t, constructionTestOutput(), bindings) + combine := compositeAppendTestPlan(prefix.Plan.StageSequence.FinalStageID) + if _, err := ComposeClickHouseCombine(prefix.Plan, combine, ir.PhysicalPrefixSourcePostgreSQL, bindings); err == nil || !strings.Contains(err.Error(), "unsupported on the PostgreSQL source backend") { + t.Fatalf("restricted PostgreSQL prefix error = %v", err) + } + if _, err := ComposeClickHouseCombine(prefix.Plan, combine, ir.PhysicalPrefixSourceAQL, bindings); err != nil { + t.Fatalf("restricted AQL prefix compose error = %v", err) + } +} + +func TestComposeClickHouseCombineRequiresExplicitSourceBackend(t *testing.T) { + bindings := recipe.RuntimeBindings{ + Project: "project", DatasetGeneration: "generation", + AuthScopeMode: authscope.ReadScopeUnrestricted, + } + prefix := compileCompositePrefixTestOutput(t, constructionTestOutput(), bindings) + combine := compositeAppendTestPlan(prefix.Plan.StageSequence.FinalStageID) + if _, err := ComposeClickHouseCombine(prefix.Plan, combine, "", bindings); err == nil || !strings.Contains(err.Error(), "explicit supported source backend") { + t.Fatalf("untagged compose error = %v", err) + } + if _, err := ComposeClickHouseCombine(prefix.Plan, combine, ir.PhysicalPrefixSourceBackend("CLICKHOUSE"), bindings); err == nil || !strings.Contains(err.Error(), "explicit supported source backend") { + t.Fatalf("unknown-tag compose error = %v", err) + } +} diff --git a/internal/dataframe/compiler/lower/recipe_construction_related_eligibility.go b/internal/dataframe/compiler/lower/recipe_construction_related_eligibility.go index f8cb18f14..706c91a13 100644 --- a/internal/dataframe/compiler/lower/recipe_construction_related_eligibility.go +++ b/internal/dataframe/compiler/lower/recipe_construction_related_eligibility.go @@ -11,8 +11,6 @@ import ( "github.com/calypr/loom/internal/dataframe/spec" ) -// RelatedEligibilityAnchorError identifies a saved step whose row anchor no -// longer resolves after an earlier row-shape change. type RelatedEligibilityAnchorError struct{ StepID string } func (e *RelatedEligibilityAnchorError) Error() string { diff --git a/internal/dataframe/compiler/lower/recipe_construction_reshape.go b/internal/dataframe/compiler/lower/recipe_construction_reshape.go index 6bfa64c71..c24cd0460 100644 --- a/internal/dataframe/compiler/lower/recipe_construction_reshape.go +++ b/internal/dataframe/compiler/lower/recipe_construction_reshape.go @@ -352,7 +352,7 @@ func lowerConstructionRowValues( if err != nil { return nil, nil, fmt.Errorf("row value fieldSource: %w", err) } - _ = projection // The exact source projection is installed by the shared helper. + _ = projection } else { var ok bool inputColumn, ok = input[rowValue.InputColumnID] diff --git a/internal/dataframe/compiler/lower/recipe_dynamic_columns.go b/internal/dataframe/compiler/lower/recipe_dynamic_columns.go index 2704594ba..eb6b8dc8e 100644 --- a/internal/dataframe/compiler/lower/recipe_dynamic_columns.go +++ b/internal/dataframe/compiler/lower/recipe_dynamic_columns.go @@ -1,9 +1,5 @@ package lower -// This file contains the canonical recipe lowering boundary. Persisted -// recipes are a frontend: after resolution each output is lowered to the same -// ir.PhysicalPlan used by the GraphQL dataframe compiler. - import ( "fmt" "strings" @@ -15,10 +11,6 @@ import ( "github.com/calypr/loom/internal/dataframe/spec" ) -// CompiledRecipe is orchestration metadata around one canonical physical plan -// per output. It deliberately contains no recipe-specific traversal, -// projection, or renderer structures. Output order is the persisted recipe -// order and therefore part of the stable materialization contract. func appendRecipeDynamicColumns(plan *ir.PhysicalPlan, output semantic.OutputPlan, resolvedColumns map[string][]semantic.ResolvedColumn) ([]DynamicColumnMetadata, error) { dynamics := flattenRecipeDynamicMaps(output.Root) if len(dynamics) == 0 { @@ -136,14 +128,7 @@ func appendRecipeDynamicColumns(plan *ir.PhysicalPlan, output semantic.OutputPla if output.Collision != "overwrite" { return nil, fmt.Errorf("dynamic column %q collides with another output column", outputName) } - // The legacy dataframe flatteners update one row map in declaration - // order. Preserve that contract for an explicit overwrite policy - // instead of rejecting a category/identifier or extension collision. plan.Operations[returnOp].Return.Projections[projectionIndex] = ir.PhysicalProjection{Name: outputName, Expression: &lookup} - // Keep validation metadata for every family sharing this public - // compatibility column. The physical projection is intentionally - // overwritten in declaration order, but runtime-key validation must - // still accept keys frozen for the earlier family. metadata = append(metadata, entry) continue } @@ -222,12 +207,6 @@ func lowerDynamicItemExpression(input semantic.SemanticExpression, bindVars map[ return lowerDynamicItemAST(input.Expression, bindVars, resourceType) } -// lowerDynamicItemAST lowers a dynamic value/key expression after semantic -// checking has already proven every selector. It intentionally handles item -// selectors as raw payload paths: the item resource type is a nested FHIR -// datatype (Extension, Identifier, ObservationComponent, ...), not the parent -// resource type carried by the dynamic map. Re-validating it against the -// parent schema would reject valid generic choice fields. func lowerDynamicItemAST(input expression.Expression, bindVars map[string]any, resourceType string) (ir.PhysicalExpression, error) { if input.Document != nil && strings.EqualFold(strings.TrimSpace(input.Document.Context), "item") { return ir.PhysicalExpression{Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalObjectCardinality, NullBehavior: ir.PhysicalPreserveNull, Value: &ir.PhysicalValue{Variable: "dynamic_item"}}, nil diff --git a/internal/dataframe/compiler/lower/recipe_expression.go b/internal/dataframe/compiler/lower/recipe_expression.go index 833cab2be..eb227df28 100644 --- a/internal/dataframe/compiler/lower/recipe_expression.go +++ b/internal/dataframe/compiler/lower/recipe_expression.go @@ -1,10 +1,5 @@ package lower -// This file is the narrow bridge from the persisted, backend-neutral -// expression AST to the physical expression IR. It deliberately performs no -// output-name or resource-specific dispatch: selectors and calls remain data -// all the way to the renderer. - import ( "fmt" "strings" @@ -15,10 +10,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// LowerRecipeExpression lowers one checked recipe expression into the typed -// physical tree. bindVars receives literal values using deterministic, -// collision-safe keys. resourceType is used to retain schema provenance for -// selector validation; it is not used to select implementation behavior. func LowerRecipeExpression(input expression.Expression, bindVars map[string]any, resourceType string) (ir.PhysicalExpression, error) { if bindVars == nil { return ir.PhysicalExpression{}, fmt.Errorf("bind vars map is required") @@ -38,9 +29,6 @@ func lowerRecipeExpression(input expression.Expression, bindVars map[string]any, case expression.NullEmpty: behavior = ir.PhysicalEmptyOnNull case expression.NullError: - // Physical execution has no error-valued AQL expression. Preserve null - // here and let the semantic checker reject error-sensitive recipes until - // an explicit error channel is added to the execution contract. behavior = ir.PhysicalPreserveNull } result := ir.PhysicalExpression{Cardinality: cardinality, NullBehavior: behavior} @@ -62,10 +50,6 @@ func lowerRecipeExpression(input expression.Expression, bindVars map[string]any, if err != nil { return ir.PhysicalExpression{}, fmt.Errorf("parse selector %q: %w", selectorPath, err) } - // Semantic recipe plans normally carry a checked expression type, but - // the narrow public lowering entrypoint also accepts the persisted AST. - // Recover repetition from generated schema metadata in that case so a - // repeated FHIR selector cannot silently become a scalar physical value. if !input.Type.Valid() { if metadata, ok := fhirschema.ResolveTerminalScalarMetadata(resourceType, selector.CanonicalPath()); ok && metadata.Repeated { cardinality = ir.PhysicalArrayCardinality @@ -102,9 +86,6 @@ func lowerRecipeExpression(input expression.Expression, bindVars map[string]any, call.TargetKind = string(input.Call.Target.Kind) } for index, arg := range input.Call.Args { - // Recipe wire syntax represents cast as (value, target literal), - // while the typed AST stores target in Call.Target. The target is - // metadata, not a runtime physical argument. if call.Name == "cast" && input.Call.Target != nil && index == 1 { continue } @@ -122,9 +103,6 @@ func lowerRecipeExpression(input expression.Expression, bindVars map[string]any, } } -// lowerDocumentRef materializes the storage envelope needed by whole-resource -// consumers while retaining the payload as one nested object. It deliberately -// uses only typed PhysicalValue nodes; no backend query text is embedded here. func lowerDocumentRef(ref expression.DocumentRef, behavior ir.PhysicalNullBehavior) ir.PhysicalExpression { variable := strings.TrimSpace(ref.Context) if variable == "" { diff --git a/internal/dataframe/compiler/lower/recipe_expression_lowering.go b/internal/dataframe/compiler/lower/recipe_expression_lowering.go index 327e4c8dd..ab0eee129 100644 --- a/internal/dataframe/compiler/lower/recipe_expression_lowering.go +++ b/internal/dataframe/compiler/lower/recipe_expression_lowering.go @@ -1,9 +1,5 @@ package lower -// This file contains the canonical recipe lowering boundary. Persisted -// recipes are a frontend: after resolution each output is lowered to the same -// ir.PhysicalPlan used by the GraphQL dataframe compiler. - import ( "fmt" "strings" @@ -15,12 +11,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// recipeFieldProjectionLowerer lowers rich recipe expressions at the generic -// plan boundary, where the traversal walk has established the physical value -// for every lexical alias. Selector-bearing fields deliberately delegate to -// the existing selector lowerer so projection modes, fallbacks, and prepared -// selector behavior remain unchanged. Owner-relative siblings of a nested row -// expansion bind through the exact captured repeated ancestor. func recipeFieldProjectionLowerer(output semantic.OutputPlan) semanticFieldProjectionLowerer { contexts := recipeExpressionContexts(output) return func(physical *ir.PhysicalPlan, node semantic.SemanticNode, index int, field semantic.SemanticField, source ir.PhysicalValue, bindings map[string]physicalSemanticBinding) (ir.PhysicalProjection, error) { @@ -190,9 +180,6 @@ func recipeExpandedAncestorProjection(expansion *semantic.SemanticRowExpansion, return recipeExpandedAncestorSelector(expansion, context, projectionSelector) } -// recipeExpandedAncestorSelector finds the nearest repeated item shared by an -// expanded source and an owner-relative sibling selector, then returns the -// sibling suffix relative to that captured item. func recipeExpandedAncestorSelector(expansion *semantic.SemanticRowExpansion, context string, projectionSelector spec.Selector) (ir.PhysicalUnnestAncestor, spec.Selector, string, bool) { if expansion == nil || context != expansion.Owner.Alias || expansion.Source.Expression.Selector == nil { return ir.PhysicalUnnestAncestor{}, spec.Selector{}, "", false @@ -317,10 +304,6 @@ func rewriteRecipeExpressionVariable(value *ir.PhysicalExpression, from, to stri } } -// rewriteRecipeExpressionBinding applies a physical lexical binding to all -// value-bearing nodes. Extracts also inherit a payload path when the binding -// is a traversed document variable (as opposed to a materialized child set, -// whose renderer intentionally handles set items through item.payload). func rewriteRecipeExpressionBinding(value *ir.PhysicalExpression, from string, source ir.PhysicalValue) { if value == nil { return @@ -377,12 +360,6 @@ func recipeExpressionContexts(output semantic.OutputPlan) map[string]string { return contexts } -// lowerRecipeExpressionScoped is the recipe-expression counterpart to the -// generic lowerer's selector classifier. A recipe expression may combine -// selectors from several lexical bindings (root plus an expanded item), so a -// single resourceType argument is insufficient. Each selector is lowered in -// its binding's generated schema type and item selectors are rooted at the -// item value rather than at a FHIR document payload. func lowerRecipeExpressionScoped(input expression.Expression, bindVars map[string]any, rootResourceType string, contexts map[string]string) (ir.PhysicalExpression, error) { if input.Document != nil { context := strings.TrimSpace(input.Document.Context) diff --git a/internal/dataframe/compiler/lower/recipe_output_test.go b/internal/dataframe/compiler/lower/recipe_output_test.go index 79a7e45e4..14b8bea25 100644 --- a/internal/dataframe/compiler/lower/recipe_output_test.go +++ b/internal/dataframe/compiler/lower/recipe_output_test.go @@ -119,9 +119,6 @@ func TestCompileResolvedRecipePlanTypesOutputSelectionAsInvalidRequest(t *testin if err == nil { t.Fatal("expected rejection") } - // The transport bulk wrappers (Engine.Materialize's ResolutionError, - // recipeGraphQLError) must keep the request defect classified after - // they add their own framing; assert traversal survives wrapping. userErr, ok := dataframeerrors.AsUserError(fmt.Errorf("compile resolved bundle: %w", err)) if !ok { t.Fatalf("selection fault escaped as untyped; transports publish retryable-free INTERNAL_ERROR 500 for a typo: %v", err) diff --git a/internal/dataframe/compiler/lower/recipe_schema.go b/internal/dataframe/compiler/lower/recipe_schema.go index f3dacf3c8..efc1e334f 100644 --- a/internal/dataframe/compiler/lower/recipe_schema.go +++ b/internal/dataframe/compiler/lower/recipe_schema.go @@ -1,9 +1,5 @@ package lower -// This file contains the canonical recipe lowering boundary. Persisted -// recipes are a frontend: after resolution each output is lowered to the same -// ir.PhysicalPlan used by the GraphQL dataframe compiler. - import ( "fmt" "strings" @@ -15,15 +11,6 @@ import ( "github.com/calypr/loom/internal/dataframe/unit" ) -// CompiledRecipe is orchestration metadata around one canonical physical plan -// per output. It deliberately contains no recipe-specific traversal, -// projection, or renderer structures. Output order is the persisted recipe -// order and therefore part of the stable materialization contract. -// recipeOutputSchema is the single schema capture point for recipe outputs. -// Names and order come from the physical RETURN operation after all recipe -// identity and bounded dynamic projections have been appended. Semantic -// metadata is used only to enrich those already-finalized projections with -// logical type/cardinality information. func recipeOutputSchema(plan ir.PhysicalPlan, output semantic.OutputPlan, dynamicMetadata []DynamicColumnMetadata, derivedTypes map[string]derivedColumnMetadata) ([]CompiledOutputColumn, error) { if output.GroupRows != nil { columns := []CompiledOutputColumn{ @@ -84,7 +71,6 @@ func recipeOutputSchema(plan ir.PhysicalPlan, output semantic.OutputPlan, dynami return } if existing, exists := logical[name]; exists { - // Explicit declarations win collisions under overwrite policies. if existing.Discovered && !discovered { logical[name] = CompiledOutputColumn{Name: name, SemanticPath: semanticPath, Kind: kind, Cardinality: cardinality, Nullable: nullable, NormalizedUnit: cloneUnitIdentity(normalizedUnit), Discovered: false} } @@ -213,10 +199,6 @@ func recipeOutputSchema(plan ir.PhysicalPlan, output semantic.OutputPlan, dynami } semanticCounts[path]++ } - // Legacy recipes may intentionally project the same FHIR value more than - // once. Keep those columns addressable by qualifying the ambiguous source - // with the authored projection name. New recipes should use fieldRef when - // they need rename-stable identities for such duplicate projections. for index := range result { if !result[index].Internal && semanticCounts[result[index].SemanticPath] > 1 { result[index].SemanticPath += "#" + result[index].Name @@ -227,9 +209,6 @@ func recipeOutputSchema(plan ir.PhysicalPlan, output semantic.OutputPlan, dynami return nil, nil } -// recipeSemanticPath produces a storage-independent identity. Explicit -// fieldRef values are frontend/catalog provenance and are retained verbatim; -// selector expressions fall back to resource type plus normalized FHIR path. func recipeSemanticPath(rootResource, resource, fieldRef string, expr expression.Expression) string { ref := strings.TrimSpace(fieldRef) if ref != "" { diff --git a/internal/dataframe/compiler/lower/recipe_table_reshape.go b/internal/dataframe/compiler/lower/recipe_table_reshape.go index 2a2dc1e3c..cc442e406 100644 --- a/internal/dataframe/compiler/lower/recipe_table_reshape.go +++ b/internal/dataframe/compiler/lower/recipe_table_reshape.go @@ -146,10 +146,6 @@ func lowerRecipeGroupedPivot(plan *ir.PhysicalPlan, pivot semantic.SemanticGroup MissingCellPolicy: string(pivot.MissingCellPolicy), UnlistedCategoryPolicy: string(pivot.UnlistedCategoryPolicy), } if !categoryNeedsPresence { - // The row's property presence is consumed by sentinel category matching - // only for authored NULL/MISSING keys. Keep the root-stage proof available - // to a category-discovery prefix, but avoid attaching it to ordinary - // string/value Pivot inputs where it would disable bounded preview planning. for index := range physical.InputProjections { if physical.InputProjections[index].Name == pivot.CategoryColumn { physical.InputProjections[index].Presence = nil @@ -230,17 +226,6 @@ func lowerRecipeGroupedPivot(plan *ir.PhysicalPlan, pivot semantic.SemanticGroup column.Internal = false column.Identity = false column.Discovered = false - // Every category cloned from the shared value input must carry a - // distinct compiled column ID: the receipt compiled output schema - // rejects duplicate IDs, which failed every multi-category table-shape - // pivot proposal. The first category keeps the historical clone - // identity byte-for-byte because single-category pivot receipts saved - // with that identity are revalidated against fresh compiles at - // publication; receipts with two or more categories could never have - // persisted pre-fix (validation failed), so clearing the identity on - // later categories introduces no saved-artifact drift. A pivot-produced - // aggregate has no source producer of its own, matching the unpivot - // key/value outputs and the compiler-owned row identity. if index > 0 { column.ID = "" column.SourceChild = nil @@ -379,12 +364,6 @@ func requireTableScalarColumn(name string, schema map[string]CompiledOutputColum return column, nil } -// TableReshapeScalarKind is the authoritative melt/reshape value-column gate: -// it maps a logical column kind to its physical table scalar class and refuses -// every kind the reshape lowerers cannot compile. Explorer source channels -// (for example the UNPIVOT signed-input lane) admit kinds through this single -// decision so an advertised channel can never drift from what the compiler -// actually melts. func TableReshapeScalarKind(kind string) (string, bool) { switch expression.ValueKind(strings.ToLower(strings.TrimSpace(kind))) { case expression.KindString, expression.KindDate, expression.KindDateTime, expression.KindCode, expression.KindUUID: diff --git a/internal/dataframe/compiler/lower/recipe_table_reshape_test.go b/internal/dataframe/compiler/lower/recipe_table_reshape_test.go index 388bcf0db..8a932c1a4 100644 --- a/internal/dataframe/compiler/lower/recipe_table_reshape_test.go +++ b/internal/dataframe/compiler/lower/recipe_table_reshape_test.go @@ -79,12 +79,6 @@ func TestCompileGroupedTablePivotUsesFrozenTypedCategoriesAndOrderedOutputs(t *t } func TestCompileGroupedTablePivotDoesNotDuplicateSourceColumnIDAcrossCategories(t *testing.T) { - // Authoring-compiled outputs carry stable source column IDs on recipe - // fields. Two categories materialized from one shared value input must - // not each claim the value's source producer identity: the receipt - // compiled output schema rejects duplicate column IDs, which surfaced as - // an HTTP 500 on every multi-category table-shape pivot proposal. The - // defect is type-agnostic; only the reducer policies require numerics. for _, test := range []struct { name string valueColumn string @@ -102,9 +96,6 @@ func TestCompileGroupedTablePivotDoesNotDuplicateSourceColumnIDAcrossCategories( categoryKinds: []string{string(expression.KindInteger), string(expression.KindInteger)}, }, { - // Saved receipts from single-category pivots were compiled with - // the clone identity and are revalidated at publication; that - // historical byte identity must not drift. name: "single category keeps saved identity", valueColumn: "value", valueID: "source_shared_value", policy: recipe.PivotDuplicateSum, categories: []recipe.GroupedPivotCategory{ {Key: tableScalarInteger(0), Output: "birth_zero", Label: "Zero"}, @@ -151,9 +142,6 @@ func TestCompileGroupedTablePivotDoesNotDuplicateSourceColumnIDAcrossCategories( t.Fatalf("category output %q missing from schema %#v", category.Output, compiledSchemaNames(compiled.OutputSchema)) } if index == 0 { - // The first category preserves the historical clone - // identity so already-saved single-category pivot - // receipts keep recompiling byte-for-byte. if column.ID != test.valueID { t.Fatalf("first pivot category lost its historical clone identity: got %q want %q: %#v", column.ID, test.valueID, column) } @@ -195,11 +183,6 @@ func assertUniquePivotOutputSchemaIDs(t *testing.T, schema []CompiledOutputColum } func TestLowerRecipeGroupedPivotLaterCategoriesDropSourceIdentityFields(t *testing.T) { - // Construction-lane value columns (recipe_construction.go source-projection - // stage columns) can carry source identity: a producer column ID plus - // lineage fields the root-lane fixtures cannot produce. Later categories - // materialized from the same shared value input must drop every identity - // field; the first category keeps them for saved-receipt byte compatibility. child := &lineage.SourceChild{ Kind: lineage.IndexedValueChild, ParentColumnIDs: []string{"source_root"}, OccurrenceID: "base", SourcePath: "valueQuantity.value", diff --git a/internal/dataframe/compiler/lower/required_match.go b/internal/dataframe/compiler/lower/required_match.go index afec991a5..c04ce6ac2 100644 --- a/internal/dataframe/compiler/lower/required_match.go +++ b/internal/dataframe/compiler/lower/required_match.go @@ -9,10 +9,6 @@ import ( "github.com/calypr/loom/internal/dataframe/spec" ) -// appendRequiredTraversalMatchFilters lowers every REQUIRED semantic route to -// a correlated typed EXISTS subplan. The subplan is deliberately separate -// from optional traversal materialization: it is a root semi-join and must run -// before the root sort/window, while optional traversal sets remain post-limit. func appendRequiredTraversalMatchFilters(physical *ir.PhysicalPlan, root semantic.SemanticNode) error { return appendRequiredTraversalMatchFiltersExcept(physical, root, nil) } @@ -27,11 +23,6 @@ func appendRequiredTraversalMatchFiltersExcept(physical *ir.PhysicalPlan, root s childRequired := child.MatchMode.Required() _, selectedRouteStep := selectedOccurrences[child.OccurrenceID] if childRequired && requiredPrefix && !selectedRouteStep { - // Two required children with the same physical route and typed - // predicates are the same root semi-join even when their aliases - // differ. Deduplicating this exact proof is safe: it does not - // move a predicate across the root execution window or alter any - // optional child materialization. key, err := requiredSemanticRouteKey(root.ResourceType, next) if err != nil { return err @@ -61,10 +52,6 @@ func appendRequiredTraversalMatchFiltersExcept(physical *ir.PhysicalPlan, root s return walk(root, nil, true) } -// requiredSemanticRouteKey intentionally excludes aliases and selection -// shape. A required route only contributes root membership, so aliases cannot -// distinguish two predicates. JSON gives us a deterministic key for typed -// filters without using AQL text or resource-specific knowledge. func requiredSemanticRouteKey(rootResourceType string, route []semantic.SemanticNode) (string, error) { type step struct { ResourceType string diff --git a/internal/dataframe/compiler/lower/root_projection.go b/internal/dataframe/compiler/lower/root_projection.go index 20d9f419b..d7d782af1 100644 --- a/internal/dataframe/compiler/lower/root_projection.go +++ b/internal/dataframe/compiler/lower/root_projection.go @@ -11,10 +11,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// physicalSemanticBinding records the physical value and schema resource -// associated with one semantic lexical binding. Recipe expressions use this -// while lowering so a selector is rooted at the binding that is actually in -// scope (root document, traversal item, set item, or unnest item). type physicalSemanticBinding struct { ResourceType string Source ir.PhysicalValue @@ -31,12 +27,6 @@ func clonePhysicalSemanticBindings(bindings map[string]physicalSemanticBinding) return copy } -// semanticFieldProjectionLowerer is shared by root and child projection -// construction. The generic dataframe compiler supplies nil and uses the -// selector-only lowering below; recipe compilation supplies a direct checked -// expression lowerer. Keeping this at the projection boundary means rich -// expressions are lowered once, while the traversal/set walk still owns -// lexical physical variables. type semanticFieldProjectionLowerer func( physical *ir.PhysicalPlan, node semantic.SemanticNode, @@ -240,9 +230,6 @@ func physicalPivotProjections(physical *ir.PhysicalPlan, resourceType string, so if err != nil { return nil, err } - // Exact correlated pivots match their code against the bounded pivot - // columns array in AQL; unlike filters and owner-record predicates they - // do not use a scalar code bind. Avoid emitting an undeclared/unused bind. correlation.CodeBindKey = "" sharedExpression.Pivot.Correlation = &correlation } @@ -285,11 +272,6 @@ func deferredExpressionVariableExists(physical ir.PhysicalPlan, variable string) return false } -// physicalAggregateExpression lowers one semantic aggregate into the typed -// aggregate IR. A root source is a singleton document; child sources are -// correlated PhysicalSet variables. Selector extraction remains typed so the -// renderer can choose the correct payload iteration without embedding user -// paths in AQL. func physicalAggregateExpression(physical *ir.PhysicalPlan, resourceType string, source ir.PhysicalValue, aggregate semantic.SemanticAggregate, sourceIsSet bool) (ir.PhysicalExpression, error) { op := ir.PhysicalAggregateOperation(strings.ToUpper(strings.TrimSpace(aggregate.Operation))) switch op { @@ -386,19 +368,12 @@ func aggregateProjectionName(aggregate semantic.SemanticAggregate, prefix string return prefix + aggregate.Name } -// physicalSliceExpression lowers a representative slice into a typed, -// bounded projection over either the singleton root document or a correlated -// child set. The source set is already materialized in stable _key order; -// the explicit sort expression below makes that ordering part of the slice -// contract and gives the renderer a deterministic tie-break. func physicalSliceExpression(physical *ir.PhysicalPlan, resourceType string, source ir.PhysicalValue, slice semantic.SemanticSlice) (ir.PhysicalExpression, error) { if slice.Limit <= 0 { return ir.PhysicalExpression{}, fmt.Errorf("slice %q requires positive limit", slice.Name) } limitKey := "slice_" + sanitizeColumnName(source.Variable) + "_" + sanitizeColumnName(slice.Name) + "_limit" physical.BindVars[limitKey] = slice.Limit - // Expressions are anchored to the declared source variable for validation; - // the renderer rebinds them to its per-item loop variable. leftSource := source physicalSlice := ir.PhysicalSlice{ Source: source, @@ -512,10 +487,6 @@ func appendRootPhysicalFilters(physical *ir.PhysicalPlan, root semantic.Semantic } predicate.Right = &ir.PhysicalValue{BindKey: key} } - // Ingestion stores the FHIR id unchanged in both document.id and - // payload.id. Narrow exact ID matches with the indexed document field - // before extracting FHIR values or expanding repeated rows. Retain the - // original predicate below as the value-level check. if selector.CanonicalPath() == "id" && selector.Filter == nil && filter.Quantifier == "" && (filter.Operator == spec.FilterEquals || filter.Operator == spec.FilterIn) { physical.Operations = append(physical.Operations, ir.PhysicalOperation{ @@ -532,8 +503,3 @@ func appendRootPhysicalFilters(physical *ir.PhysicalPlan, root semantic.Semantic } return nil } - -// buildOptionalChildPhysicalSet lowers one root-relative optional traversal. -// The set retains resource documents (rather than pre-shaped maps), allowing -// the renderer to apply each child selector against the document payload at -// the outer return while preserving root row grain. diff --git a/internal/dataframe/compiler/lower/storage_route.go b/internal/dataframe/compiler/lower/storage_route.go index a7d7fe415..2cb5b0fce 100644 --- a/internal/dataframe/compiler/lower/storage_route.go +++ b/internal/dataframe/compiler/lower/storage_route.go @@ -9,29 +9,13 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// ErrUnsupportedStorageRoute identifies a FHIR relationship that is known to -// the generated schema but has not been proven safe for Loom's stored -// fhir_edge layout. Callers must not substitute a different AQL direction for -// this error: every physical direction must have a compiler-owned storage -// contract. var ErrUnsupportedStorageRoute = errors.New("unsupported storage route") -// storageRoute is the compiler-owned bridge from generated FHIR relationship -// metadata to the currently supported stored-edge operation. The generated -// FHIR Direction describes the logical reference, not the Arango direction; -// Direction below is established only by a storage proof in -// resolveStorageRoute. type storageRoute struct { Relationship fhirschema.CompilerTraversal Direction ir.PhysicalTraversalDirection } -// targetEdgeTypeField returns the fhir_edge type discriminator for the node -// reached by this route. It is deliberately direction-aware: forward FHIR -// references store their target in to_type, while the generated builder route -// reads the source-side type through an INBOUND traversal. The node -// resourceType predicate remains mandatory even when this edge predicate is -// used as an index-selectivity hint. func (route storageRoute) targetEdgeTypeField() string { switch route.Direction { case ir.PhysicalInbound: @@ -43,10 +27,6 @@ func (route storageRoute) targetEdgeTypeField() string { } } -// endpointLookupFields describes the generic fhir_edge compound-index -// contract used by the endpoint strategy. It is derived solely from the -// proven storage direction; callers must still retain native traversal when -// the metadata is incomplete. func (route storageRoute) endpointLookupFields() (parentField, joinField string, indexFields []string, ok bool) { switch route.Direction { case ir.PhysicalInbound: @@ -58,16 +38,6 @@ func (route storageRoute) endpointLookupFields() (parentField, joinField string, } } -// resolveStorageRoute accepts generated synthetic reverse routes for which -// the source resource is provably the reference target. The ingester stores -// normal FHIR references as child _from -> parent _to, so a parent -> child -// dataframe route is an INBOUND fhir_edge traversal. -// -// It also accepts generated forward routes when the schema gives an exact -// target hint and an outbound logical reference. EdgeFromGrip (the single -// edge adapter used by ingest) materializes every such reference as source -// _from -> referenced target _to in fhir_edge, so the generated target hint is -// the storage proof; no resource-specific allowlist is needed. func resolveStorageRoute(fromType, edgeLabel, toType string) (storageRoute, error) { if !fhirschema.HasResource(fromType) { return storageRoute{}, unsupportedStorageRouteError(fromType, edgeLabel, toType, "source resource type is not represented by the active generated FHIR schema") @@ -80,8 +50,6 @@ func resolveStorageRoute(fromType, edgeLabel, toType string) (storageRoute, erro if !found { return storageRoute{}, unsupportedStorageRouteError(fromType, edgeLabel, toType, "is not represented by the active generated FHIR schema") } - // LookupTraversal is keyed by this exact tuple, but retain the check so a - // malformed future generated map cannot silently become executable AQL. if spec.FromType != fromType || spec.EdgeLabel != edgeLabel || spec.ToType != toType { return storageRoute{}, unsupportedStorageRouteError(fromType, edgeLabel, toType, "generated traversal metadata does not match the requested route") } @@ -93,9 +61,6 @@ func resolveStorageRoute(fromType, edgeLabel, toType string) (storageRoute, erro return storageRoute{}, unsupportedStorageRouteError(fromType, edgeLabel, toType, "is not represented by the active generated FHIR schema") } - // Only a precise, generated /* target hint proves that - // this tuple is the synthetic parent -> child route. In particular, - // Resource/* is deliberately not evidence for a concrete parent type. if hasExactStorageParentHint(spec.RegexMatch, fromType) { return storageRoute{Relationship: relationship, Direction: ir.PhysicalInbound}, nil } diff --git a/internal/dataframe/compiler/lower/traversal.go b/internal/dataframe/compiler/lower/traversal.go index ecff9e1cf..83a4c32e0 100644 --- a/internal/dataframe/compiler/lower/traversal.go +++ b/internal/dataframe/compiler/lower/traversal.go @@ -7,22 +7,10 @@ import ( "github.com/calypr/loom/internal/dataframe/compiler/ir" ) -// TraversalLoweringRequest is the backend-neutral input to the shared -// schema-derived traversal constructor. The caller supplies semantic resource -// identities and compiler-owned variable names; route direction, edge type -// discriminator, and endpoint fields are resolved from fhirschema through -// resolveStorageRoute. -// -// BindPrefix is part of the physical naming contract. It must be unique in -// the containing plan and stable for a given semantic traversal. It is used -// only to derive bind keys; it is never rendered as AQL source. type TraversalLoweringRequest struct { - FromType string - EdgeLabel string - ToType string - // DirectionOverride is only for a route whose exact stored-edge direction - // was already proven by its caller. It resolves self-type inverse routes, - // where swapping endpoint types cannot distinguish the original direction. + FromType string + EdgeLabel string + ToType string DirectionOverride ir.PhysicalTraversalDirection SourceVariable string TargetVariable string @@ -31,17 +19,11 @@ type TraversalLoweringRequest struct { Policy ir.PhysicalOptimizationPolicy } -// TraversalLoweringResult contains the canonical physical traversal and the -// bind values required by its renderer. type TraversalLoweringResult struct { Traversal ir.PhysicalTraversal BindVars map[string]any } -// BuildPhysicalTraversal resolves one FHIR relationship and constructs the -// canonical physical traversal used by every compiler frontend. It is the -// only lower-layer entry point that should assemble traversal direction, -// target discriminator, endpoint fields, and traversal bind keys. func BuildPhysicalTraversal(request TraversalLoweringRequest) (TraversalLoweringResult, error) { if strings.TrimSpace(request.FromType) == "" { return TraversalLoweringResult{}, fmt.Errorf("traversal source resource type is required") @@ -136,9 +118,6 @@ func isCompilerIdentifier(value string) bool { return true } -// physicalTraversalStrategyForRoute selects an execution strategy only after -// route metadata has been validated. Endpoint lookup is a typed physical -// choice; it does not permit callers to provide endpoint fields or AQL. func physicalTraversalStrategyForRoute(policy ir.PhysicalOptimizationPolicy, route storageRoute) (ir.PhysicalTraversalStrategy, string, string, []string) { if !policy.RuleEnabled(ir.PhysicalOptimizationRuleEndpointTraversal) { return ir.PhysicalTraversalNative, "", "", nil diff --git a/internal/dataframe/compiler/lower/typed_predicate.go b/internal/dataframe/compiler/lower/typed_predicate.go index f5e7a656b..2492d4985 100644 --- a/internal/dataframe/compiler/lower/typed_predicate.go +++ b/internal/dataframe/compiler/lower/typed_predicate.go @@ -8,10 +8,6 @@ import ( "github.com/calypr/loom/internal/dataframe/spec" ) -// lowerTypedPredicate is the one semantic-to-physical lowering boundary for a -// typed filter. It owns selector extraction and literal binding while callers -// supply a scope-specific prefix, ensuring sibling rich consumers cannot -// overwrite one another's bind variables. func lowerTypedPredicate(physical *ir.PhysicalPlan, resourceType string, source ir.PhysicalValue, filter spec.TypedFilter, bindPrefix string) (*ir.PhysicalPredicateExpression, error) { if physical == nil { return nil, fmt.Errorf("physical plan is required") diff --git a/internal/dataframe/compiler/lower/whole_relation_scope.go b/internal/dataframe/compiler/lower/whole_relation_scope.go index a16024e42..84b32a0d3 100644 --- a/internal/dataframe/compiler/lower/whole_relation_scope.go +++ b/internal/dataframe/compiler/lower/whole_relation_scope.go @@ -31,18 +31,12 @@ func issueWholeRelationScopeEvidence(outputID string, plan ir.PhysicalPlan, sche } evidence, err := ir.NewWholeRelationScopeEvidence(plan, identity) if err != nil { - // A typed plan can remain valid for ordinary execution while falling - // outside the stronger sealed-capture proof contract. Omit evidence so - // private grouped capture fails closed without changing query compilation. return nil, nil, nil } return &evidence, &identity, nil } func wholeRelationScopeCandidate(plan ir.PhysicalPlan) bool { - // Ordinary AQL outputs can be captured as complete source relations too. - // Workspace ClickHouse plans receive evidence only after their ordered - // resolver and child-evidence operands have been validated. return plan.Engine == "" || plan.Engine == ir.PhysicalEngineAQL } diff --git a/internal/dataframe/compiler/lower/workspace_combine_scope_evidence_test.go b/internal/dataframe/compiler/lower/workspace_combine_scope_evidence_test.go index c61cef4ec..41685326e 100644 --- a/internal/dataframe/compiler/lower/workspace_combine_scope_evidence_test.go +++ b/internal/dataframe/compiler/lower/workspace_combine_scope_evidence_test.go @@ -137,7 +137,7 @@ func TestWorkspaceCombineScopeEvidenceBindsExactPrivateAQLPrefix(t *testing.T) { privateRef := ir.PhysicalCombineInputRef{PrivateStageID: source.Plan.StageSequence.FinalStageID} publishedRef := ir.PhysicalCombineInputRef{TableID: "4:r1:1v7:published", RevisionID: "revision-1", OutputID: "published"} combine := testWorkspaceCombinePayload("tag_id", privateRef, "source_value", publishedRef) - actualPlan, err := ComposeClickHouseCombine(source.Plan, *combine, bindings) + actualPlan, err := ComposeClickHouseCombine(source.Plan, *combine, ir.PhysicalPrefixSourceAQL, bindings) if err != nil { t.Fatalf("compose exact AQL prefix and Combine: %v", err) } diff --git a/internal/dataframe/compiler/nested_expansion_arango_test.go b/internal/dataframe/compiler/nested_expansion_arango_test.go index b73652d90..47455ec28 100644 --- a/internal/dataframe/compiler/nested_expansion_arango_test.go +++ b/internal/dataframe/compiler/nested_expansion_arango_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "reflect" - "strings" "testing" "time" @@ -76,9 +75,8 @@ func TestNestedRepeatedSourceValuesThroughAuthoredConstructionExpandAgainstArang assertNestedConstructionExpandRows(t, executeReshapeOracleQuery(t, ctx, client, excludedQuery), project, wantNonEmpty) errorQuery := compileConstructionOutputQuery(t, nestedConstructionExpandOutput(recipe.ExpansionError), project, generation) - if err := client.QueryRows(ctx, errorQuery.Query, 500, errorQuery.BindVars, func(map[string]any) error { return nil }); err == nil || !strings.Contains(err.Error(), "CONSTRUCTION_EXPANSION_EMPTY") { - t.Fatalf("ERROR policy over missing/empty nested source lists = %v, want CONSTRUCTION_EXPANSION_EMPTY", err) - } + err := client.QueryRows(ctx, testAQLText(t, errorQuery.Command), 500, testAQLBinds(t, errorQuery.Command), func(map[string]any) error { return nil }) + requireStableRuntimeCode(t, err, "CONSTRUCTION_EXPANSION_EMPTY") } type nestedConstructionExpandExpected struct { diff --git a/internal/dataframe/compiler/operator_oracles_arango_test.go b/internal/dataframe/compiler/operator_oracles_arango_test.go index c0db466c2..eab7e809b 100644 --- a/internal/dataframe/compiler/operator_oracles_arango_test.go +++ b/internal/dataframe/compiler/operator_oracles_arango_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/columntransform" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" @@ -17,7 +18,6 @@ import ( "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" "github.com/calypr/loom/internal/dataframe/unit" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -89,12 +89,13 @@ func TestS04CompiledOperatorOraclesAgainstArango(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -402,10 +403,8 @@ func TestS04CompiledOperatorOraclesAgainstArango(t *testing.T) { windowMalformedTimeCompiled := compileOracleBundle(t, recipe.Bundle{ RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: "s04-window-malformed-time", TranslationVersion: "1", Outputs: []recipe.Output{windowedSummary}, }, projects["window-malformed-time"], generation) - windowMalformedTimeRows, windowMalformedTimeErr := queryOracleOutput(ctx, client, windowMalformedTimeCompiled.Outputs[0]) - if windowMalformedTimeErr == nil || !strings.Contains(windowMalformedTimeErr.Error(), "TEMPORAL_PRECISION_UNSUPPORTED") { - t.Fatalf("windowed malformed-time query returned rows=%#v error=%v; want TEMPORAL_PRECISION_UNSUPPORTED", windowMalformedTimeRows, windowMalformedTimeErr) - } + _, windowMalformedTimeErr := queryOracleOutput(ctx, client, windowMalformedTimeCompiled.Outputs[0]) + requireStableRuntimeCode(t, windowMalformedTimeErr, "TEMPORAL_PRECISION_UNSUPPORTED") rows := executeOracleQuery(t, ctx, client, mainCompiled.Outputs[0]) if len(rows) != len(wants) { t.Fatalf("summary rows = %#v, want %d patient rows", rows, len(wants)) @@ -599,10 +598,8 @@ func TestS04CompiledOperatorOraclesAgainstArango(t *testing.T) { uniqueCompiled := compileOracleBundle(t, recipe.Bundle{ RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: "s04-unique-tie", TranslationVersion: "1", Outputs: []recipe.Output{uniqueTemporal}, }, projects["main"], generation) - uniqueRows, uniqueErr := queryOracleOutput(ctx, client, uniqueCompiled.Outputs[0]) - if uniqueErr == nil || !strings.Contains(uniqueErr.Error(), "TEMPORAL_TIE_AMBIGUOUS") { - t.Fatalf("REQUIRE_UNIQUE temporal query returned rows=%#v error=%v; want TEMPORAL_TIE_AMBIGUOUS", uniqueRows, uniqueErr) - } + _, uniqueErr := queryOracleOutput(ctx, client, uniqueCompiled.Outputs[0]) + requireStableRuntimeCode(t, uniqueErr, "TEMPORAL_TIE_AMBIGUOUS") uniqueWindowed := windowedSummary uniqueWindowed.Name = "unique_windowed_temporal" uniqueAggregate := windowedSummary.Traversals[0].Aggregates[6] @@ -616,10 +613,8 @@ func TestS04CompiledOperatorOraclesAgainstArango(t *testing.T) { uniqueWindowedCompiled := compileOracleBundle(t, recipe.Bundle{ RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: "s04-unique-windowed-tie", TranslationVersion: "1", Outputs: []recipe.Output{uniqueWindowed}, }, projects["main"], generation) - uniqueWindowedRows, uniqueWindowedErr := queryOracleOutput(ctx, client, uniqueWindowedCompiled.Outputs[0]) - if uniqueWindowedErr == nil || !strings.Contains(uniqueWindowedErr.Error(), "TEMPORAL_TIE_AMBIGUOUS") { - t.Fatalf("windowed REQUIRE_UNIQUE query returned rows=%#v error=%v; want TEMPORAL_TIE_AMBIGUOUS for the two eligible Jan 2 items", uniqueWindowedRows, uniqueWindowedErr) - } + _, uniqueWindowedErr := queryOracleOutput(ctx, client, uniqueWindowedCompiled.Outputs[0]) + requireStableRuntimeCode(t, uniqueWindowedErr, "TEMPORAL_TIE_AMBIGUOUS") unsupportedOutput := recipe.Output{ Name: "unsupported_unit", RootResourceType: "Patient", RowGrain: "patient", @@ -630,10 +625,8 @@ func TestS04CompiledOperatorOraclesAgainstArango(t *testing.T) { unsupportedCompiled := compileOracleBundle(t, recipe.Bundle{ RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: "s04-unsupported-unit", TranslationVersion: "1", Outputs: []recipe.Output{unsupportedOutput}, }, projects["unsupported"], generation) - unsupportedRows, unsupportedErr := queryOracleOutput(ctx, client, unsupportedCompiled.Outputs[0]) - if unsupportedErr == nil || !strings.Contains(unsupportedErr.Error(), "UNIT_IDENTITY_UNKNOWN") { - t.Fatalf("unsupported-unit query returned rows=%#v error=%v; want UNIT_IDENTITY_UNKNOWN for UCUM furlong", unsupportedRows, unsupportedErr) - } + _, unsupportedErr := queryOracleOutput(ctx, client, unsupportedCompiled.Outputs[0]) + requireStableRuntimeCode(t, unsupportedErr, "UNIT_IDENTITY_UNKNOWN") missingTimeOutput := recipe.Output{ Name: "missing_time", RootResourceType: "Patient", RowGrain: "patient", @@ -677,7 +670,7 @@ func compileOracleBundle(t *testing.T, bundle recipe.Bundle, project, generation return compiled } -func executeOracleQuery(t *testing.T, ctx context.Context, client *store.Client, output lower.CompiledRecipeOutput) []map[string]any { +func executeOracleQuery(t *testing.T, ctx context.Context, client *arangofactory.Client, output lower.CompiledRecipeOutput) []map[string]any { t.Helper() rows, err := queryOracleOutput(ctx, client, output) if err != nil { @@ -686,7 +679,7 @@ func executeOracleQuery(t *testing.T, ctx context.Context, client *store.Client, return rows } -func queryOracleOutput(ctx context.Context, client *store.Client, output lower.CompiledRecipeOutput) ([]map[string]any, error) { +func queryOracleOutput(ctx context.Context, client *arangofactory.Client, output lower.CompiledRecipeOutput) ([]map[string]any, error) { rendered, err := aql.RenderPhysicalPlan(output.Plan) if err != nil { return nil, err @@ -702,7 +695,7 @@ func queryOracleOutput(ctx context.Context, client *store.Client, output lower.C return rows, err } -func executeOracleTrace(t *testing.T, ctx context.Context, client *store.Client, output lower.CompiledRecipeOutput, column string) map[string][]operatorOracleContributor { +func executeOracleTrace(t *testing.T, ctx context.Context, client *arangofactory.Client, output lower.CompiledRecipeOutput, column string) map[string][]operatorOracleContributor { t.Helper() _, cells := executeOracleTraceResults(t, ctx, client, output, column) contributors := make(map[string][]operatorOracleContributor, len(cells)) @@ -715,18 +708,18 @@ func executeOracleTrace(t *testing.T, ctx context.Context, client *store.Client, return contributors } -func executeOracleTraceResults(t *testing.T, ctx context.Context, client *store.Client, output lower.CompiledRecipeOutput, column string) (CompiledCellTraceQuery, map[string]operatorOracleTraceResult) { +func executeOracleTraceResults(t *testing.T, ctx context.Context, client *arangofactory.Client, output lower.CompiledRecipeOutput, column string) (CompiledCellTraceQuery, map[string]operatorOracleTraceResult) { t.Helper() - trace, err := CompileCellTraceOutputWithPolicy(output, column, 0, 100, ir.DefaultPhysicalOptimizationPolicy()) + trace, err := CompileCellTraceOutput(ArangoTarget(), output, column, 0, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile %s cell trace: %v", column, err) } rows := make([]map[string]any, 0) - if err := client.QueryRows(ctx, trace.Query, 500, trace.BindVars, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, trace.Command), 500, testAQLBinds(t, trace.Command), func(row map[string]any) error { rows = append(rows, row) return nil }); err != nil { - t.Fatalf("execute %s cell trace: %v\n%s", column, err, trace.Query) + t.Fatalf("execute %s cell trace: %v\n%s", column, err, testAQLText(t, trace.Command)) } project, _ := output.Plan.BindVars["project"].(string) return trace, oracleTraceResultsByIdentity(t, rows, trace, project) diff --git a/internal/dataframe/compiler/optimize/doc.go b/internal/dataframe/compiler/optimize/doc.go index 4b760baff..e4599d7ad 100644 --- a/internal/dataframe/compiler/optimize/doc.go +++ b/internal/dataframe/compiler/optimize/doc.go @@ -1,3 +1 @@ -// Package optimize applies explainable, semantics-preserving rewrites to -// typed physical IR. It never parses or edits rendered AQL. package optimize diff --git a/internal/dataframe/compiler/optimize/optimize.go b/internal/dataframe/compiler/optimize/optimize.go index 407db78c8..141b2c6df 100644 --- a/internal/dataframe/compiler/optimize/optimize.go +++ b/internal/dataframe/compiler/optimize/optimize.go @@ -9,13 +9,6 @@ import ( const OptimizerRuleTraversalSharing = "share_identical_traversals" -// OptimizePhysicalPlan applies semantics-preserving physical rewrites using -// the conservative structural cost policy. An unshared plan is always -// available as the correctness oracle, and the local policy switch can be used -// to compare the two physical shapes without changing request semantics. -// OptimizePhysicalPlanWithPolicy is the explicit form used by tests and -// benchmarking tools. The policy decides only whether an optional rewrite is -// enabled; it never relaxes physical-plan validation. func OptimizePhysicalPlanWithPolicy(plan ir.PhysicalPlan, policy ir.PhysicalOptimizationPolicy) (ir.PhysicalPlan, error) { if err := plan.Validate(); err != nil { return ir.PhysicalPlan{}, fmt.Errorf("validate physical plan: %w", err) @@ -30,8 +23,6 @@ func OptimizePhysicalPlanWithPolicy(plan ir.PhysicalPlan, policy ir.PhysicalOpti } decomposition, err := ir.DecomposePhysicalTraversalPrefixAt(out, *op.Set, i) if err != nil { - // Ineligibility is an expected optimizer outcome. The original plan - // remains the execution oracle. continue } decompositions[i] = decomposition @@ -51,7 +42,6 @@ func OptimizePhysicalPlanWithPolicy(plan ir.PhysicalPlan, policy ir.PhysicalOpti Rule: OptimizerRuleTraversalSharing, CandidateSets: len(indices), } - // Sharing is useful only when it removes target-type-specific traversals. types := map[string]bool{} for _, index := range indices { t := decompositions[index].Subset.TargetTypeBindKey @@ -124,9 +114,6 @@ func (route storageRoute) endpointLookupFields() (string, string, []string, bool func sharePhysicalSetGroup(plan *ir.PhysicalPlan, indices []int, types map[string]bool, decompositions map[int]ir.PhysicalTraversalPrefixDecomposition, policy ir.PhysicalOptimizationPolicy) error { first := indices[0] - // The broad set changes its target bind from one type to a type list. Deep - // clone before that change: its first consumer must retain the original - // single target-type bind for the typed subset. originalOperation := ir.ClonePhysicalOperation(plan.Operations[first]) original := *originalOperation.Set t := *original.Subplan.Operations[0].Traversal @@ -146,7 +133,6 @@ func sharePhysicalSetGroup(plan *ir.PhysicalPlan, indices []int, types map[strin for typ := range types { targetTypes = append(targetTypes, typ) } - // Deterministic order is part of plan stability. for i := 0; i < len(targetTypes); i++ { for j := i + 1; j < len(targetTypes); j++ { if targetTypes[j] < targetTypes[i] { @@ -169,33 +155,14 @@ func sharePhysicalSetGroup(plan *ir.PhysicalPlan, indices []int, types map[strin return fmt.Errorf("missing traversal prefix decomposition for set %d", first) } base.Subplan.Operations = ir.ClonePhysicalOperations(decomposition.PrefixOperations) - // Prefix decomposition preserves the original consumer's scalar type - // binding. Restore the broadened discriminator after cloning it so the - // shared traversal can collect every candidate type in this group. base.Subplan.Operations[0].Traversal = &t base.Variable = baseName base.SourceSetVariable, base.ItemVariable = "", "" - // The broad source must remain a full node because sibling consumers may - // require different compact fields. Each typed subset retains its own - // proven output contract below; unioning those contracts into the broad - // traversal would make sharing resource/cardinality dependent. base.Output = nil - // Prepared values belong to typed consumer sets. The broad shared neighbor - // set must contain raw nodes only; consumers prepare their own projections - // after resource-type filtering. base.Prepared = nil - // Traversal-time selector projections are also consumer-specific. Keeping - // the first sibling's projection on the broad heterogeneous set evaluates - // selectors against the wrong resource payload and leaves typed subsets - // with empty values. base.Projection = nil - // The heterogeneous shared neighbor set has no consumer-specific selector - // slots, so it must not retain a reduction belonging to the original typed - // set. base.Reduction = nil base.Subplan.Return = ir.PhysicalExpression{Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalObjectCardinality, NullBehavior: ir.PhysicalPreserveNull, Value: &ir.PhysicalValue{Variable: t.TargetVariable}} - // Rebuild operation stream, inserting the broad traversal immediately before - // the first consumer and replacing each consumer with a typed subset. indexSet := map[int]bool{} for _, i := range indices { indexSet[i] = true @@ -226,9 +193,6 @@ func sharePhysicalSetGroup(plan *ir.PhysicalPlan, indices []int, types map[strin set.SourceSetVariable, set.ItemVariable, set.Subplan = baseName, item, sub set.Unique, set.SortByKey = true, true if set.Prepared != nil { - // The typed consumer now reads the shared subset rather than the - // original traversal set. Rebind the prepared projection's source - // to the consumer set so rendering defines it in the same scope. prepared := *set.Prepared prepared.SourceSetVariable = set.Variable set.Prepared = &prepared diff --git a/internal/dataframe/compiler/optimizer.go b/internal/dataframe/compiler/optimizer.go index 458e781e6..3dfaab593 100644 --- a/internal/dataframe/compiler/optimizer.go +++ b/internal/dataframe/compiler/optimizer.go @@ -1,7 +1,5 @@ package compiler -// Stable optimizer-rule identifiers appear in compiler explain output and -// conformance evidence. They are deliberately independent of AQL text. const ( OptimizerRuleFilterPushdown = "push_down_typed_filters" ) diff --git a/internal/dataframe/compiler/output.go b/internal/dataframe/compiler/output.go index 66eb65964..42f406dcb 100644 --- a/internal/dataframe/compiler/output.go +++ b/internal/dataframe/compiler/output.go @@ -7,9 +7,6 @@ import ( "github.com/calypr/loom/internal/dataframe/compiler/lower" ) -// publicOutputColumns returns the ordered transport schema owned by the -// compiler. Physical identity and executor-validation projections are kept in -// OutputSchema but never leak into dataframe JSON/CSV/publication columns. func publicOutputColumns(schema []lower.CompiledOutputColumn) []string { columns := make([]string, 0, len(schema)) for _, column := range schema { @@ -21,9 +18,6 @@ func publicOutputColumns(schema []lower.CompiledOutputColumn) []string { return columns } -// PublicOutputColumns returns a copy of the compiler-owned transport schema. -// It is used by recipe execution adapters that need to carry the schema with -// streamed rows without re-walking semantic recipe nodes. func PublicOutputColumns(schema []lower.CompiledOutputColumn) []string { return publicOutputColumns(schema) } @@ -64,9 +58,6 @@ func physicalTraversalCount(plan ir.PhysicalPlan) int { return count } -// withGenericPhysicalExecutionWindow inserts the deterministic root ordering -// and optional preview bound before any traversal LET subquery, ensuring an -// expensive optional navigation is evaluated only for selected root rows. func withGenericPhysicalExecutionWindow(plan ir.PhysicalPlan, limit int) (ir.PhysicalPlan, error) { if plan.StageSequence != nil { out := clonePhysicalPlan(plan) @@ -77,9 +68,6 @@ func withGenericPhysicalExecutionWindow(plan ir.PhysicalPlan, limit int) (ir.Phy out.BindVars[genericPhysicalExecutionLimitBind] = limit out.StageSequence.PreviewLimitBindKey = genericPhysicalExecutionLimitBind if out.StageSequence.PreviewSourceWindowByRootID && !insertConstructionPreviewRootIDWindow(&out) { - // Optimizer rewrites may make the source shape ineligible. The - // canonical source path remains correct; only skip this preview - // optimization when its root-scan insertion point is unavailable. out.StageSequence.PreviewSourceWindowByRootID = false } out.StageSequence.PreviewTerminalPivotWindow = previewTerminalPivotWindowEligible(out.StageSequence) @@ -142,9 +130,6 @@ func withGenericPhysicalExecutionWindow(plan ir.PhysicalPlan, limit int) (ir.Phy return out, nil } - // The generic scope verifier defines the root scope as every operation up - // to the first traversal or terminal return. BuildGenericPhysicalPlan has - // already proven that the whole prefix scopes the root correctly. insertAt := physicalScopeWindowEnd(plan.Operations, 1) unnestIndex := -1 for index, operation := range plan.Operations { @@ -156,18 +141,11 @@ func withGenericPhysicalExecutionWindow(plan ir.PhysicalPlan, limit int) (ir.Phy } } if unnestIndex >= 0 { - // A row expansion owns the additional variables in its stable identity - // tuple. Keep the window after that operation, while still before the - // unrelated child SET/traversal operations it is intended to bound. if unnestIndex > insertAt { return ir.PhysicalPlan{}, fmt.Errorf("generic execution window requires row expansion before child SET/traversal operations") } insertAt = unnestIndex + 1 } else { - // Shared expression LETs are deliberately emitted immediately before - // RETURN for root-only plans. Put the root execution window before them so - // AQL can discard rows before evaluating the family maps, while traversal - // plans still insert the window before their first child SET. for insertAt > 1 && insertAt <= len(plan.Operations) && plan.Operations[insertAt-1].Kind == ir.PhysicalExpressionLetOp { insertAt-- } @@ -266,9 +244,7 @@ func insertConstructionPreviewRootIDWindow(plan *ir.PhysicalPlan) bool { { Kind: ir.PhysicalSortOp, Source: ir.PhysicalSource{SemanticNode: plan.Source.SemanticNode, ResourceType: plan.Source.ResourceType, SemanticField: "id"}, - // Loom's FHIR ingest stores the validated logical resource id both - // at the document root and in payload. The root field is indexed. - Sort: &ir.PhysicalSort{Keys: []ir.PhysicalValue{{Variable: root, Path: []string{"id"}}}}, + Sort: &ir.PhysicalSort{Keys: []ir.PhysicalValue{{Variable: root, Path: []string{"id"}}}}, }, { Kind: ir.PhysicalLimitOp, diff --git a/internal/dataframe/compiler/physical_projection_reserved_name_arango_test.go b/internal/dataframe/compiler/physical_projection_reserved_name_arango_test.go index 5371f45da..ad27e2cf2 100644 --- a/internal/dataframe/compiler/physical_projection_reserved_name_arango_test.go +++ b/internal/dataframe/compiler/physical_projection_reserved_name_arango_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/recipe" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -19,10 +19,11 @@ func TestGroupedPivotReservedPhysicalOutputNameAgainstArango(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client t.Cleanup(func() { closeCtx, closeCancel := context.WithTimeout(context.Background(), 10*time.Second) defer closeCancel() @@ -30,7 +31,7 @@ func TestGroupedPivotReservedPhysicalOutputNameAgainstArango(t *testing.T) { t.Errorf("close reserved-name pivot fixture client: %v", err) } }) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Observation"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Observation"}}}); err != nil { t.Fatal(err) } diff --git a/internal/dataframe/compiler/pivot_presence_arango_audit_test.go b/internal/dataframe/compiler/pivot_presence_arango_audit_test.go index a72369f71..63669ed96 100644 --- a/internal/dataframe/compiler/pivot_presence_arango_audit_test.go +++ b/internal/dataframe/compiler/pivot_presence_arango_audit_test.go @@ -10,17 +10,14 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) -// TestPivotPresenceArangoAudit executes the presence matrix against real AQL and -// reports which offered columns would reach MISSING_UNSUPPORTED when the source -// contains absent or explicit-null values. It is diagnostic and env-gated. func TestPivotPresenceArangoAudit(t *testing.T) { url, database := os.Getenv("LOOM_TEST_ARANGO_URL"), os.Getenv("LOOM_TEST_ARANGO_DATABASE") if url == "" || database == "" { @@ -28,12 +25,17 @@ func TestPivotPresenceArangoAudit(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } - defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Patient"}, {Name: "Observation"}}}); err != nil { + client := arangoBackend.Client + t.Cleanup(func() { + if err := client.Close(context.Background()); err != nil { + t.Errorf("close presence audit client: %v", err) + } + }) + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Patient"}, {Name: "Observation"}}}); err != nil { t.Fatal(err) } @@ -83,12 +85,12 @@ FOR document IN %s } audited := appendPresenceAuditPivot(fixture.output, stage, category, *value) compiled := compilePresenceAuditOutput(t, audited, project, generation, []string{allowedPath}) - scanned, err := CompileCategoryScanStageWithPolicy(compiled, stage.ID, category.ID, value.ID, MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) + scanned, err := CompileCategoryScanStage(ArangoTarget(), compiled, stage.ID, category.ID, value.ID, MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Logf("%s\t%s\t%s\tCOMPILE_REFUSED\t%v", fixture.name, stage.ID, category.Name, err) continue } - presentFalse, missing, conclusive, rows, err := executePresenceAuditScan(ctx, client, scanned) + presentFalse, missing, conclusive, rows, err := executePresenceAuditScan(t, ctx, client, scanned) if err != nil { t.Logf("%s\t%s\t%s\tEXECUTION_ERROR\t%v", fixture.name, stage.ID, category.Name, err) continue @@ -167,11 +169,11 @@ func compilePresenceAuditOutput(t *testing.T, output recipe.Output, project, gen return compiled.Outputs[0] } -func executePresenceAuditScan(ctx context.Context, client *store.Client, scanned CompiledCategoryScanQuery) (presentFalse int, missing, conclusive bool, rows int, err error) { +func executePresenceAuditScan(t *testing.T, ctx context.Context, client *arangofactory.Client, scanned CompiledCategoryScanQuery) (presentFalse int, missing, conclusive bool, rows int, err error) { if witness := scanned.OverflowWitness; witness != nil { witnessRows := 0 witnessMissing := false - if err := client.QueryRows(ctx, witness.Query, 100, witness.BindVars, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, witness.Command), 100, testAQLBinds(t, witness.Command), func(row map[string]any) error { present, ok := row[scanned.PresentColumn].(bool) if !ok { return fmt.Errorf("overflow witness has invalid presence marker") @@ -189,7 +191,7 @@ func executePresenceAuditScan(ctx context.Context, client *store.Client, scanned return 0, true, true, witnessRows, nil } } - err = client.QueryRows(ctx, scanned.Query, 100, scanned.BindVars, func(row map[string]any) error { + err = client.QueryRows(ctx, testAQLText(t, scanned.Command), 100, testAQLBinds(t, scanned.Command), func(row map[string]any) error { present, ok := row[scanned.PresentColumn].(bool) if !ok { return fmt.Errorf("category scan row has invalid presence marker") @@ -206,7 +208,7 @@ func executePresenceAuditScan(ctx context.Context, client *store.Client, scanned return presentFalse, presentFalse > 0, false, rows, nil } -func seedPresenceAuditDocuments(t *testing.T, ctx context.Context, client *store.Client, project, generation, authPath string) { +func seedPresenceAuditDocuments(t *testing.T, ctx context.Context, client *arangofactory.Client, project, generation, authPath string) { t.Helper() base := func(key, resourceType string) map[string]any { return map[string]any{ @@ -419,7 +421,7 @@ func seedPresenceAuditDocuments(t *testing.T, ctx context.Context, client *store insertPresenceAuditBatch(t, ctx, client, "Patient", patients) } -func insertPresenceAuditBatch(t *testing.T, ctx context.Context, client *store.Client, collection string, rows []map[string]any) { +func insertPresenceAuditBatch(t *testing.T, ctx context.Context, client *arangofactory.Client, collection string, rows []map[string]any) { t.Helper() documents := make([]json.RawMessage, 0, len(rows)) for _, row := range rows { diff --git a/internal/dataframe/compiler/pivot_presence_audit_test.go b/internal/dataframe/compiler/pivot_presence_audit_test.go index c28ac5d2b..6e87b4aa9 100644 --- a/internal/dataframe/compiler/pivot_presence_audit_test.go +++ b/internal/dataframe/compiler/pivot_presence_audit_test.go @@ -12,8 +12,6 @@ import ( "github.com/calypr/loom/internal/dataframe/recipe" ) -// TestPivotPresenceAudit is a diagnostic matrix, not a regression gate. Run it -// with LOOM_PIVOT_PRESENCE_AUDIT=1 when changing category discovery support. func TestPivotPresenceAudit(t *testing.T) { if os.Getenv("LOOM_PIVOT_PRESENCE_AUDIT") == "" { t.Skip("set LOOM_PIVOT_PRESENCE_AUDIT=1 to print the Pivot category presence matrix") @@ -53,11 +51,11 @@ func TestPivotPresenceAudit(t *testing.T) { } audited := appendPresenceAuditPivot(fixture.output, stage, category, *value) compiled := compilePopulationMappingOutput(t, audited) - result := auditCategoryAtStage(compiled, stage, category) + result := auditCategoryAtStage(t, compiled, stage, category) if fixture.name == "unpivot" && stage.ID == "unpivot" && (category.Name == "amount" || category.Name == "measure") { - scanned, err := CompileCategoryScanStageWithPolicy(compiled, stage.ID, category.ID, value.ID, MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) + scanned, err := CompileCategoryScanStage(ArangoTarget(), compiled, stage.ID, category.ID, value.ID, MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) if err == nil { - for _, line := range strings.Split(scanned.Query, "\n") { + for _, line := range strings.Split(testAQLText(t, scanned.Command), "\n") { if strings.Contains(line, "__loom_category_scan_presence") { t.Logf("MARKER %s/%s: %s", stage.ID, category.Name, strings.TrimSpace(line)) } @@ -96,13 +94,13 @@ func eligibleCategoryColumns(columns []lower.CompiledOutputColumn) []lower.Compi return eligible } -func auditCategoryAtStage(output lower.CompiledRecipeOutput, stage lower.CompiledStageDescriptor, category lower.CompiledOutputColumn) categoryAuditResult { +func auditCategoryAtStage(t *testing.T, output lower.CompiledRecipeOutput, stage lower.CompiledStageDescriptor, category lower.CompiledOutputColumn) categoryAuditResult { lastCode := "" for _, value := range eligibleCategoryColumns(stage.Columns) { if value.ID == category.ID { continue } - scanned, err := CompileCategoryScanStageWithPolicy(output, stage.ID, category.ID, value.ID, MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) + scanned, err := CompileCategoryScanStage(ArangoTarget(), output, stage.ID, category.ID, value.ID, MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { if code, ok := CategoryScanRefusalCodeOf(err); ok { lastCode = string(code) @@ -113,8 +111,8 @@ func auditCategoryAtStage(output lower.CompiledRecipeOutput, stage lower.Compile } return categoryAuditResult{ status: "SUPPORTED", - queryLen: len(scanned.Query), - forLoops: strings.Count(scanned.Query, "FOR "), + queryLen: len(testAQLText(t, scanned.Command)), + forLoops: strings.Count(testAQLText(t, scanned.Command), "FOR "), presence: scanned.Proof.PresenceTracked, indexed: scanned.CategoryIndex != nil || scanned.PreviewCoveringIndex != nil, witness: scanned.OverflowWitness != nil, diff --git a/internal/dataframe/compiler/population_frontier_arango_test.go b/internal/dataframe/compiler/population_frontier_arango_test.go index 0391b74bc..28973321e 100644 --- a/internal/dataframe/compiler/population_frontier_arango_test.go +++ b/internal/dataframe/compiler/population_frontier_arango_test.go @@ -12,11 +12,11 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -26,17 +26,18 @@ func TestPopulationMappingRepeatedRouteDeduplicatesScopedFrontiersAgainstArango( } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) t.Cleanup(cancel) - client, err := store.Open(ctx, os.Getenv("LOOM_TEST_ARANGO_URL"), os.Getenv("LOOM_TEST_ARANGO_DATABASE")) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: os.Getenv("LOOM_TEST_ARANGO_URL"), Database: os.Getenv("LOOM_TEST_ARANGO_DATABASE")}) if err != nil { t.Fatal(err) } - t.Cleanup(func() { _ = client.Close(context.Background()) }) + t.Cleanup(func() { _ = arangoBackend.Close(context.Background()) }) + client := arangoBackend.Client suffix := strings.ReplaceAll(uuid.NewString(), "-", "") observations := "loom_population_frontier_observations_" + suffix patients := "loom_population_frontier_patients_" + suffix edges := "loom_population_frontier_edges_" + suffix members := "loom_population_frontier_members_" + suffix - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: observations}, {Name: patients}, {Name: edges, Edge: true}, {Name: members}, }}); err != nil { t.Fatal(err) @@ -103,8 +104,6 @@ func TestPopulationMappingRepeatedRouteDeduplicatesScopedFrontiersAgainstArango( } } - // Each bad resource is attached by correctly scoped edges. If the resource - // scope is dropped, it leads to an otherwise disconnected root observation. for index, mismatch := range []resource{ {patients, "patient-wrong-project", "patient-wrong-project", "Patient", "other-" + project, generation, "/allowed"}, {patients, "patient-wrong-generation", "patient-wrong-generation", "Patient", project, "other-" + generation, "/allowed"}, @@ -117,8 +116,6 @@ func TestPopulationMappingRepeatedRouteDeduplicatesScopedFrontiersAgainstArango( insertEdge("resource-scope-witness-"+mismatch.key, observationID, mismatch.key, allowed) } - // Each bad edge is the only path from a selected source to its disconnected - // bridge. Its valid bridge edge makes an omitted edge-scope filter observable. for index, mismatch := range []edgeScope{ {project: "other-" + project, generation: generation, authPath: "/allowed"}, {project: project, generation: "other-" + generation, authPath: "/allowed"}, @@ -191,28 +188,28 @@ func TestPopulationMappingRepeatedRouteDeduplicatesScopedFrontiersAgainstArango( t.Fatal(err) } output := compiledRecipe.Outputs[0] - compiled, err := CompilePopulationMappingOutputWithPolicy(output, bindings, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompilePopulationMappingOutput(ArangoTarget(), output, bindings, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - compiled.BindVars["@root_collection"] = observations - compiled.BindVars["@population_source_collection"] = observations - compiled.BindVars["@population_members_collection"] = members + testAQLBinds(t, compiled.Command)["@root_collection"] = observations + testAQLBinds(t, compiled.Command)["@population_source_collection"] = observations + testAQLBinds(t, compiled.Command)["@population_members_collection"] = members for index := range route { - compiled.BindVars[fmt.Sprintf("@population_root_route_%d_edge_collection", index)] = edges + testAQLBinds(t, compiled.Command)[fmt.Sprintf("@population_root_route_%d_edge_collection", index)] = edges } queryCtx, cancel := context.WithTimeout(ctx, 20*time.Second) defer cancel() - profile, err := client.Profile(queryCtx, store.ProfileRequest{ - Query: compiled.Query, BindVars: compiled.BindVars, BatchSize: 256, - Count: true, Options: store.ProfileOptions{Profile: 2}, + profile, err := client.Profile(queryCtx, arangofactory.ProfileRequest{ + Query: testAQLText(t, compiled.Command), BindVars: testAQLBinds(t, compiled.Command), BatchSize: 256, + Count: true, Options: arangofactory.ProfileOptions{Profile: 2}, }) if err != nil { - t.Fatalf("execute scoped 8-hop population mapping query: %v\n%s", err, compiled.Query) + t.Fatalf("execute scoped 8-hop population mapping query: %v\n%s", err, testAQLText(t, compiled.Command)) } if profile.HasMore || profile.Count != 70 || len(profile.Result) != 70 { - t.Fatalf("scoped route returned count=%d rows=%d hasMore=%v stats=%#v binds=%#v, want 70 member/root witnesses:\n%s", profile.Count, len(profile.Result), profile.HasMore, profile.Extra.Stats, compiled.BindVars, compiled.Query) + t.Fatalf("scoped route returned count=%d rows=%d hasMore=%v stats=%#v binds=%#v, want 70 member/root witnesses:\n%s", profile.Count, len(profile.Result), profile.HasMore, profile.Extra.Stats, testAQLBinds(t, compiled.Command), testAQLText(t, compiled.Command)) } wantRoots := make(map[string]struct{}, len(observationIDs)) for _, id := range observationIDs { @@ -255,7 +252,7 @@ func TestPopulationMappingRepeatedRouteDeduplicatesScopedFrontiersAgainstArango( if len(witnesses) != 2 { t.Fatalf("mapped member count = %d, want two accessible selected members", len(witnesses)) } - stats := store.SummarizeProfile(profile) + stats := arangofactory.SummarizeProfile(profile) t.Logf("8-hop overlapping-member query scanned index=%d full=%d peak=%d bytes (%.1f MiB) profile-runtime=%.3fs", stats.ScannedIndex, stats.ScannedFull, stats.PeakMemory, float64(stats.PeakMemory)/(1024*1024), stats.RuntimeSeconds) scanned := stats.ScannedIndex + stats.ScannedFull if scanned > 50000 { diff --git a/internal/dataframe/compiler/population_mapping.go b/internal/dataframe/compiler/population_mapping.go index 48f20ef15..65be69699 100644 --- a/internal/dataframe/compiler/population_mapping.go +++ b/internal/dataframe/compiler/population_mapping.go @@ -4,22 +4,18 @@ import ( "fmt" "strings" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/compiler/optimize" "github.com/calypr/loom/internal/dataframe/compiler/render/aql" + postgresrender "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/spec" ) -// CompiledPopulationMappingQuery is an internal witness query. Its rows are -// (selected member, final row identity) pairs and are not a dataframe schema. -// Default identities are returned as ordered IdentityPartsColumn values for -// the execution layer to hash with RowIdentity. Explicit recipe identities -// use ExplicitIdentityColumn and preserve the compiled identity expression. type CompiledPopulationMappingQuery struct { - Query string - BindVars map[string]any + Command command.CompiledCommand MemberColumn string IdentityPartsColumn string ExplicitIdentityColumn string @@ -27,11 +23,10 @@ type CompiledPopulationMappingQuery struct { Diagnostics ir.CompilerPlanDiagnostics } -// CompilePopulationMappingOutputWithPolicy builds the complete mapping query -// from the canonical final output plan. Unlike ordinary Preview/Publish, it -// has no execution limit: required filters, authored filters, child sets, and -// explicit expansion all run before the internal witness terminal. -func CompilePopulationMappingOutputWithPolicy(output lower.CompiledRecipeOutput, bindings recipe.RuntimeBindings, policy ir.PhysicalOptimizationPolicy) (CompiledPopulationMappingQuery, error) { +func CompilePopulationMappingOutput(target Target, output lower.CompiledRecipeOutput, bindings recipe.RuntimeBindings, policy ir.PhysicalOptimizationPolicy) (CompiledPopulationMappingQuery, error) { + if err := target.validate(); err != nil { + return CompiledPopulationMappingQuery{}, err + } if output.RowIdentity == nil { return CompiledPopulationMappingQuery{}, fmt.Errorf("population mapping requires a row identity") } @@ -51,12 +46,42 @@ func CompilePopulationMappingOutputWithPolicy(output lower.CompiledRecipeOutput, } rowIdentity = effectiveIdentity } + if target.backend == command.CompiledBackendPostgreSQL { + return compilePopulationMappingPostgreSQL(target, physical, rowIdentity) + } rendered, err := aql.RenderPhysicalPlan(physical) if err != nil { return CompiledPopulationMappingQuery{}, fmt.Errorf("render population mapping physical plan: %w", err) } + compiledCommand, commandErr := command.NewAQLCommand(rendered.Query, rendered.BindVars) + if commandErr != nil { + return CompiledPopulationMappingQuery{}, fmt.Errorf("tag population mapping command: %w", commandErr) + } + return CompiledPopulationMappingQuery{ + Command: compiledCommand, + MemberColumn: ir.PhysicalPopulationMappingMemberField, + IdentityPartsColumn: mappingIdentityPartsColumn(physical), + ExplicitIdentityColumn: mappingExplicitIdentityColumn(physical), + RowIdentity: rowIdentity, + Diagnostics: physicalPlanDiagnostics(physical), + }, nil +} + +func compilePopulationMappingPostgreSQL(target Target, physical ir.PhysicalPlan, rowIdentity *spec.RowIdentity) (CompiledPopulationMappingQuery, error) { + membership, err := postgresrender.NewPostgresPopulationRootBinding(target.postgres.Resource.Identifier().Schema()) + if err != nil { + return CompiledPopulationMappingQuery{}, fmt.Errorf("bind population mapping membership schema: %w", err) + } + rendered, err := postgresrender.RenderPopulationMappingPlan(physical, target.postgres, membership) + if err != nil { + return CompiledPopulationMappingQuery{}, fmt.Errorf("render population mapping postgresql plan: %w", err) + } + compiledCommand, commandErr := command.NewPostgreSQLCommand(rendered.SQL, rendered.Args) + if commandErr != nil { + return CompiledPopulationMappingQuery{}, fmt.Errorf("tag population mapping command: %w", commandErr) + } return CompiledPopulationMappingQuery{ - Query: rendered.Query, BindVars: rendered.BindVars, + Command: compiledCommand, MemberColumn: ir.PhysicalPopulationMappingMemberField, IdentityPartsColumn: mappingIdentityPartsColumn(physical), ExplicitIdentityColumn: mappingExplicitIdentityColumn(physical), diff --git a/internal/dataframe/compiler/population_mapping_filter_exists_test.go b/internal/dataframe/compiler/population_mapping_filter_exists_test.go index 747a1c278..9658478a1 100644 --- a/internal/dataframe/compiler/population_mapping_filter_exists_test.go +++ b/internal/dataframe/compiler/population_mapping_filter_exists_test.go @@ -38,7 +38,7 @@ func TestCompilePopulationMappingFilterExistsUsesPinnedRootKey(t *testing.T) { t.Fatalf("compiled construction stages = %#v, want the authored FILTER EXISTS stage", compiledOutput.Plan.StageSequence.Stages) } - mapping, err := CompilePopulationMappingOutputWithPolicy(compiledOutput, populationMappingTestBindings(), ir.DefaultPhysicalOptimizationPolicy()) + mapping, err := CompilePopulationMappingOutput(ArangoTarget(), compiledOutput, populationMappingTestBindings(), ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile population mapping for FILTER EXISTS: %v", err) } @@ -48,9 +48,9 @@ func TestCompilePopulationMappingFilterExistsUsesPinnedRootKey(t *testing.T) { if mapping.IdentityPartsColumn != ir.PhysicalPopulationMappingIdentityPartsField || mapping.ExplicitIdentityColumn != "" { t.Fatalf("mapping witness identity columns = (%q, %q)", mapping.IdentityPartsColumn, mapping.ExplicitIdentityColumn) } - filterAt := strings.Index(mapping.Query, "FILTER __loom_construction_input_1.observation_id != null") - witnessAt := strings.Index(mapping.Query, "FOR __loom_physical_construction_population_mapping_source_row") + filterAt := strings.Index(testAQLText(t, mapping.Command), "FILTER __loom_construction_input_1.observation_id != null") + witnessAt := strings.Index(testAQLText(t, mapping.Command), "FOR __loom_physical_construction_population_mapping_source_row") if filterAt < 0 || witnessAt < 0 || filterAt >= witnessAt { - t.Fatalf("population mapping did not run the authored filter before its final-row witness terminal:\n%s", mapping.Query) + t.Fatalf("population mapping did not run the authored filter before its final-row witness terminal:\n%s", testAQLText(t, mapping.Command)) } } diff --git a/internal/dataframe/compiler/population_mapping_routed_group_arango_test.go b/internal/dataframe/compiler/population_mapping_routed_group_arango_test.go index 3be582cfc..c7be21f5c 100644 --- a/internal/dataframe/compiler/population_mapping_routed_group_arango_test.go +++ b/internal/dataframe/compiler/population_mapping_routed_group_arango_test.go @@ -12,11 +12,11 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -27,13 +27,13 @@ func TestPopulationMappingRoutedCohortPreservesSelectedMembersAgainstArango(t *t observations := "loom_population_routed_observations_" + suffix patients := "loom_population_routed_patients_" + suffix edges := "loom_population_routed_edges_" + suffix - collections := []store.CollectionSpec{ + collections := []arangofactory.CollectionSpec{ {Name: observations}, {Name: patients}, {Name: edges, Edge: true}, {Name: "loom_explorer_explicit_group_revisions"}, {Name: "loom_explorer_selections"}, {Name: "loom_explorer_explicit_group_definitions"}, {Name: "loom_explorer_explicit_group_memberships"}, {Name: "loom_explorer_selection_members"}, } - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: collections}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: collections}); err != nil { t.Fatal(err) } owned := make(map[string][]string) @@ -98,9 +98,6 @@ func TestPopulationMappingRoutedCohortPreservesSelectedMembersAgainstArango(t *t } insert(patients, patientDocs...) - // Each selected Observation is routed to a Patient. The fixture table is - // the source oracle: only the first two rows satisfy every scope check and - // survive the authored patient_id filter; both route to the same Patient. type memberSeed struct { id, patientID string sourceProject, sourceGeneration, auth string @@ -239,11 +236,11 @@ func TestPopulationMappingRoutedCohortPreservesSelectedMembersAgainstArango(t *t bindVars["@population_members_collection"] = "loom_explorer_selection_members" bindVars["@population_root_route_0_edge_collection"] = edges } - preview, err := CompileRecipeOutputWithPolicy(compiledRecipe.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + preview, err := CompileRecipeOutput(ArangoTarget(), compiledRecipe.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile routed cohort preview: %v", err) } - bindCollections(preview.BindVars) + bindCollections(testAQLBinds(t, preview.Command)) previewRows := executeReshapeOracleQuery(t, ctx, client, preview) previewGroups := make(map[string]map[string]any, len(previewRows)) for _, row := range previewRows { @@ -285,15 +282,13 @@ func TestPopulationMappingRoutedCohortPreservesSelectedMembersAgainstArango(t *t if !ok || len(emptyMembers) != 0 { t.Fatalf("empty cohort preview members = %#v, want empty list", emptyPreview["members"]) } - mapping, err := CompilePopulationMappingOutputWithPolicy(compiledRecipe.Outputs[0], bindings, ir.DefaultPhysicalOptimizationPolicy()) + mapping, err := CompilePopulationMappingOutput(ArangoTarget(), compiledRecipe.Outputs[0], bindings, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - bindCollections(mapping.BindVars) - rows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Query: mapping.Query, BindVars: mapping.BindVars}) + bindCollections(testAQLBinds(t, mapping.Command)) + rows := executeReshapeOracleQuery(t, ctx, client, CompiledQuery{Command: mapping.Command}) - // Derive the witness oracle from the fixture rows used above, without using - // the compiler preview or mapping result to decide which members should map. wantMembers := make([]string, 0, len(members)) for _, member := range members { patient := patientsByID[member.patientID] @@ -308,8 +303,8 @@ func TestPopulationMappingRoutedCohortPreservesSelectedMembersAgainstArango(t *t wantIdentity := map[string]any{"group_revision_id": cohortRevisionID, "group_id": "routed"} if len(rows) != len(wantMembers) { t.Logf("ordinary preview rows: %#v", previewRows) - t.Logf("compiled population mapping query:\n%s", mapping.Query) - t.Logf("compiled population mapping bind vars: %#v", mapping.BindVars) + t.Logf("compiled population mapping query:\n%s", testAQLText(t, mapping.Command)) + t.Logf("compiled population mapping bind vars: %#v", testAQLBinds(t, mapping.Command)) const scopedSourceJoin = ` FOR member IN @@members FILTER member.selectionId == @selection_id @@ -386,9 +381,6 @@ FOR target IN @@roots t.Fatalf("seed-derived witness oracle changed unexpectedly: %#v", wantMembers) } - // Exercise original selected-member witnesses across GROUP→PIVOT→cohort. - // The same fixture oracle must still map both selected Observations to the - // exact final cohort identity after two contributor-set reductions. pivotOutput := routedPopulationGroupPivotCohortOutput(output.Population, cohortRevisionID, excludedPatientID) pivotPlan, err := semantic.BuildRecipePlan(recipe.Bundle{ RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: "population-routed-group-pivot-cohort", @@ -424,31 +416,31 @@ FOR target IN @@roots if err != nil { t.Fatalf("compile pre-cohort GROUP→PIVOT recipe: %v", err) } - preCohortPreview, err := CompileRecipeOutputWithPolicy(preCohortCompiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + preCohortPreview, err := CompileRecipeOutput(ArangoTarget(), preCohortCompiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile pre-cohort GROUP→PIVOT preview: %v", err) } - bindCollections(preCohortPreview.BindVars) + bindCollections(testAQLBinds(t, preCohortPreview.Command)) preCohortRows := executeReshapeOracleQuery(t, ctx, client, preCohortPreview) if len(preCohortRows) != 1 || preCohortRows[0]["patient_id"] != "shared" { - t.Fatalf("pre-cohort GROUP→PIVOT rows = %#v, want exactly Patient/shared; query:\n%s", preCohortRows, preCohortPreview.Query) + t.Fatalf("pre-cohort GROUP→PIVOT rows = %#v, want exactly Patient/shared; query:\n%s", preCohortRows, testAQLText(t, preCohortPreview.Command)) } preCohortRow := preCohortRows[0] if got, ok := preCohortRow["male_count"].(float64); !ok || got != 1 { - t.Fatalf("pre-cohort GROUP→PIVOT male count = %#v, want 1 distinct root Patient row; row=%#v query:\n%s", preCohortRow["male_count"], preCohortRow, preCohortPreview.Query) + t.Fatalf("pre-cohort GROUP→PIVOT male count = %#v, want 1 distinct root Patient row; row=%#v query:\n%s", preCohortRow["male_count"], preCohortRow, testAQLText(t, preCohortPreview.Command)) } if got := preCohortRow["female_count"]; got != nil { - t.Fatalf("pre-cohort GROUP→PIVOT female count = %#v, want a null missing cell; query:\n%s", got, preCohortPreview.Query) + t.Fatalf("pre-cohort GROUP→PIVOT female count = %#v, want a null missing cell; query:\n%s", got, testAQLText(t, preCohortPreview.Command)) } if got, ok := preCohortRow["__loom_root_contributor_keys"].([]any); !ok || !reflect.DeepEqual(got, []any{key("patient_shared")}) { - t.Fatalf("pre-cohort GROUP→PIVOT root contributors = %#v, want the exact shared Patient storage key; query:\n%s", preCohortRow["__loom_root_contributor_keys"], preCohortPreview.Query) + t.Fatalf("pre-cohort GROUP→PIVOT root contributors = %#v, want the exact shared Patient storage key; query:\n%s", preCohortRow["__loom_root_contributor_keys"], testAQLText(t, preCohortPreview.Command)) } - pivotPreview, err := CompileRecipeOutputWithPolicy(pivotCompiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + pivotPreview, err := CompileRecipeOutput(ArangoTarget(), pivotCompiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile routed GROUP→PIVOT cohort preview: %v", err) } - bindCollections(pivotPreview.BindVars) + bindCollections(testAQLBinds(t, pivotPreview.Command)) pivotRows := executeReshapeOracleQuery(t, ctx, client, pivotPreview) pivotGroups := make(map[string]map[string]any, len(pivotRows)) for _, row := range pivotRows { @@ -485,17 +477,17 @@ FOR target IN @@roots t.Fatalf("GROUP→PIVOT cohort root contributors = %#v, want the exact shared Patient storage key", routedPivotPreview["__loom_root_contributor_keys"]) } - pivotMapping, err := CompilePopulationMappingOutputWithPolicy(pivotCompiled.Outputs[0], bindings, ir.DefaultPhysicalOptimizationPolicy()) + pivotMapping, err := CompilePopulationMappingOutput(ArangoTarget(), pivotCompiled.Outputs[0], bindings, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile routed GROUP→PIVOT population mapping: %v", err) } - bindCollections(pivotMapping.BindVars) + bindCollections(testAQLBinds(t, pivotMapping.Command)) pivotMappingRows := make([]map[string]any, 0, len(wantMembers)) - if err := client.QueryRows(ctx, pivotMapping.Query, 100, pivotMapping.BindVars, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, pivotMapping.Command), 100, testAQLBinds(t, pivotMapping.Command), func(row map[string]any) error { pivotMappingRows = append(pivotMappingRows, row) return nil }); err != nil { - t.Fatalf("execute routed GROUP→PIVOT population mapping query:\n%s\n%v", pivotMapping.Query, err) + t.Fatalf("execute routed GROUP→PIVOT population mapping query:\n%s\n%v", testAQLText(t, pivotMapping.Command), err) } pivotMappedMembers := make([]string, 0, len(pivotMappingRows)) for _, row := range pivotMappingRows { @@ -645,7 +637,7 @@ func TestPopulationMappingRoutedGroupPivotCohortCompilesFinalIdentity(t *testing if !identityColumnFound || !groupRevisionFound || !groupIDFound { t.Fatalf("cohort final identity is not a required object over required string group identity components: %#v", sequence.FinalColumns) } - mapping, err := CompilePopulationMappingOutputWithPolicy(outputPlan, bindings, ir.DefaultPhysicalOptimizationPolicy()) + mapping, err := CompilePopulationMappingOutput(ArangoTarget(), outputPlan, bindings, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile routed GROUP→PIVOT population mapping: %v", err) } @@ -703,7 +695,7 @@ func TestPopulationMappingRoutedGroupPivotCohortCompilesFinalIdentity(t *testing if _, _, err := finalConstructionPopulationMappingIdentity(candidate.Plan, candidate); err == nil { t.Fatalf("final construction identity accepted forged cohort shape: %#v", candidate.Plan.StageSequence) } - if _, err := CompilePopulationMappingOutputWithPolicy(candidate, bindings, ir.DefaultPhysicalOptimizationPolicy()); err == nil { + if _, err := CompilePopulationMappingOutput(ArangoTarget(), candidate, bindings, ir.DefaultPhysicalOptimizationPolicy()); err == nil { t.Fatalf("population mapping accepted forged cohort identity shape: %#v", candidate.Plan.StageSequence) } }) @@ -728,11 +720,11 @@ func TestPopulationMappingRoutedGroupPivotCohortCompilesFinalIdentity(t *testing if err != nil { t.Fatalf("compile pre-cohort GROUP→PIVOT: %v", err) } - preCohortPreview, err := CompileRecipeOutputWithPolicy(preCohortCompiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + preCohortPreview, err := CompileRecipeOutput(ArangoTarget(), preCohortCompiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("render pre-cohort GROUP→PIVOT preview: %v", err) } - if strings.TrimSpace(preCohortPreview.Query) == "" { + if strings.TrimSpace(testAQLText(t, preCohortPreview.Command)) == "" { t.Fatal("pre-cohort GROUP→PIVOT preview query is empty") } } diff --git a/internal/dataframe/compiler/population_mapping_test.go b/internal/dataframe/compiler/population_mapping_test.go index 0baeac900..70d46d8bc 100644 --- a/internal/dataframe/compiler/population_mapping_test.go +++ b/internal/dataframe/compiler/population_mapping_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/recipe" @@ -20,8 +21,8 @@ func TestCompilePopulationMappingDirectKeepsMemberRowWitnessesInternal(t *testin ResourceType: "Specimen", }, }) - if got := strings.Count(compiled.Query, "FOR population_member IN @@population_members_collection"); got != 1 { - t.Fatalf("mapping member scan count = %d, want 1:\n%s", got, compiled.Query) + if got := strings.Count(testAQLText(t, compiled.Command), "FOR population_member IN @@population_members_collection"); got != 1 { + t.Fatalf("mapping member scan count = %d, want 1:\n%s", got, testAQLText(t, compiled.Command)) } for _, want := range []string{ "COLLECT __loom_physical_population_root_key = population_source._key INTO __loom_physical_population_member_ids = population_member.id", @@ -31,18 +32,18 @@ func TestCompilePopulationMappingDirectKeepsMemberRowWitnessesInternal(t *testin "@__loom_physical_population_mapping_identity_name", "[@project, root._key]", } { - if !strings.Contains(compiled.Query, want) { - t.Fatalf("mapping query is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Fatalf("mapping query is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } - if strings.Contains(compiled.Query, "RETURN { [@__loom_projection") { - t.Fatalf("mapping query used a public projection return:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), "RETURN { [@__loom_projection") { + t.Fatalf("mapping query used a public projection return:\n%s", testAQLText(t, compiled.Command)) } - if !containsBindValue(compiled.BindVars, ir.PhysicalPopulationMappingMemberField) { - t.Fatalf("member witness bind is missing from %#v", compiled.BindVars) + if !containsBindValue(testAQLBinds(t, compiled.Command), ir.PhysicalPopulationMappingMemberField) { + t.Fatalf("member witness bind is missing from %#v", testAQLBinds(t, compiled.Command)) } - if !containsBindValue(compiled.BindVars, ir.PhysicalPopulationMappingIdentityPartsField) { - t.Fatalf("identity-parts witness bind is missing from %#v", compiled.BindVars) + if !containsBindValue(testAQLBinds(t, compiled.Command), ir.PhysicalPopulationMappingIdentityPartsField) { + t.Fatalf("identity-parts witness bind is missing from %#v", testAQLBinds(t, compiled.Command)) } if compiled.IdentityPartsColumn != ir.PhysicalPopulationMappingIdentityPartsField || compiled.ExplicitIdentityColumn != "" { t.Fatalf("default mapping identity columns = (%q, %q)", compiled.IdentityPartsColumn, compiled.ExplicitIdentityColumn) @@ -62,14 +63,14 @@ func TestCompilePopulationMappingExplicitIdentityUsesCompiledExpression(t *testi ResourceType: "Specimen", }, }) - if !strings.Contains(compiled.Query, "@__loom_physical_population_mapping_identity_name]: root.payload.id") || strings.Contains(compiled.Query, "@__loom_physical_population_mapping_identity_name]: [root.payload.id]") { - t.Fatalf("mapping query did not preserve the compiled explicit identity expression:\n%s", compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), "@__loom_physical_population_mapping_identity_name]: root.payload.id") || strings.Contains(testAQLText(t, compiled.Command), "@__loom_physical_population_mapping_identity_name]: [root.payload.id]") { + t.Fatalf("mapping query did not preserve the compiled explicit identity expression:\n%s", testAQLText(t, compiled.Command)) } - if containsBindValue(compiled.BindVars, ir.PhysicalPopulationMappingIdentityPartsField) { - t.Fatalf("explicit mapping unexpectedly rendered default identity parts: %#v", compiled.BindVars) + if containsBindValue(testAQLBinds(t, compiled.Command), ir.PhysicalPopulationMappingIdentityPartsField) { + t.Fatalf("explicit mapping unexpectedly rendered default identity parts: %#v", testAQLBinds(t, compiled.Command)) } - if !containsBindValue(compiled.BindVars, ir.PhysicalPopulationMappingExplicitIdentityField) { - t.Fatalf("explicit identity bind is missing: %#v", compiled.BindVars) + if !containsBindValue(testAQLBinds(t, compiled.Command), ir.PhysicalPopulationMappingExplicitIdentityField) { + t.Fatalf("explicit identity bind is missing: %#v", testAQLBinds(t, compiled.Command)) } if compiled.IdentityPartsColumn != "" || compiled.ExplicitIdentityColumn != ir.PhysicalPopulationMappingExplicitIdentityField { t.Fatalf("explicit mapping identity columns = (%q, %q)", compiled.IdentityPartsColumn, compiled.ExplicitIdentityColumn) @@ -95,12 +96,12 @@ func TestCompilePopulationMappingDefaultIdentityIsStableAcrossInnerOuterExpansio } } } - compiled, err := CompilePopulationMappingOutputWithPolicy(output, recipe.RuntimeBindings{}, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompilePopulationMappingOutput(ArangoTarget(), output, recipe.RuntimeBindings{}, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if !strings.Contains(compiled.Query, "[@project, root._key]") { - t.Fatalf("%s mapping query does not carry canonical project/key identity parts:\n%s", mode, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), "[@project, root._key]") { + t.Fatalf("%s mapping query does not carry canonical project/key identity parts:\n%s", mode, testAQLText(t, compiled.Command)) } if compiled.IdentityPartsColumn != ir.PhysicalPopulationMappingIdentityPartsField || compiled.ExplicitIdentityColumn != "" { t.Fatalf("%s mapping identity columns = (%q, %q)", mode, compiled.IdentityPartsColumn, compiled.ExplicitIdentityColumn) @@ -122,7 +123,7 @@ func TestCompilePopulationMappingRejectsMissingIdentityField(t *testing.T) { }, }) output.RowIdentity.Fields = []string{"project", "missing_identity_field"} - if _, err := CompilePopulationMappingOutputWithPolicy(output, recipe.RuntimeBindings{}, ir.DefaultPhysicalOptimizationPolicy()); err == nil || !strings.Contains(err.Error(), "missing_identity_field") { + if _, err := CompilePopulationMappingOutput(ArangoTarget(), output, recipe.RuntimeBindings{}, ir.DefaultPhysicalOptimizationPolicy()); err == nil || !strings.Contains(err.Error(), "missing_identity_field") { t.Fatalf("missing identity field error = %v", err) } } @@ -187,11 +188,11 @@ func TestCompilePopulationMappingReversedRouteRunsOnceBeforeWitnessTerminal(t *t Route: []recipe.PopulationRouteStep{{ResourceType: "DocumentReference", Relationship: "subject_Specimen"}}, }, }) - if got := strings.Count(compiled.Query, "FOR population_root_edge_0 IN @@population_root_route_0_edge_collection"); got != 1 { - t.Fatalf("mapping route traversal count = %d, want 1:\n%s", got, compiled.Query) + if got := strings.Count(testAQLText(t, compiled.Command), "FOR population_root_edge_0 IN @@population_root_route_0_edge_collection"); got != 1 { + t.Fatalf("mapping route traversal count = %d, want 1:\n%s", got, testAQLText(t, compiled.Command)) } - if strings.Index(compiled.Query, "population_source.id == population_member.id") > strings.Index(compiled.Query, "FOR __loom_physical_population_mapping_member") { - t.Fatalf("mapping witness terminal preceded member matching:\n%s", compiled.Query) + if strings.Index(testAQLText(t, compiled.Command), "population_source.id == population_member.id") > strings.Index(testAQLText(t, compiled.Command), "FOR __loom_physical_population_mapping_member") { + t.Fatalf("mapping witness terminal preceded member matching:\n%s", testAQLText(t, compiled.Command)) } } @@ -208,11 +209,11 @@ func TestCompilePopulationMappingRequiredNavigationRemainsBeforeWitnesses(t *tes ResourceType: "Patient", }, }) - if !strings.Contains(compiled.Query, "required_0_node_0") { - t.Fatalf("mapping query lost required downstream elimination:\n%s", compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), "required_0_node_0") { + t.Fatalf("mapping query lost required downstream elimination:\n%s", testAQLText(t, compiled.Command)) } - if strings.Index(compiled.Query, "required_0_node_0") > strings.Index(compiled.Query, "FOR __loom_physical_population_mapping_member") { - t.Fatalf("required match was evaluated after witnesses:\n%s", compiled.Query) + if strings.Index(testAQLText(t, compiled.Command), "required_0_node_0") > strings.Index(testAQLText(t, compiled.Command), "FOR __loom_physical_population_mapping_member") { + t.Fatalf("required match was evaluated after witnesses:\n%s", testAQLText(t, compiled.Command)) } } @@ -242,24 +243,24 @@ func TestCompilePopulationMappingPreservesInnerAndOuterFinalRowSemantics(t *test t.Fatal("compiled expansion has no UNNEST operation") } } - compiled, err := CompilePopulationMappingOutputWithPolicy(output, recipe.RuntimeBindings{Project: "fixture_program-project-a", SelectionProject: "fixture_program/project-a", DatasetGeneration: "generation-a", SelectionMembersCollection: "loom_explorer_selection_members"}, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompilePopulationMappingOutput(ArangoTarget(), output, recipe.RuntimeBindings{Project: "fixture_program-project-a", SelectionProject: "fixture_program/project-a", DatasetGeneration: "generation-a", SelectionMembersCollection: "loom_explorer_selection_members"}, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if mode == "INNER" && !strings.Contains(compiled.Query, "LET item =") { - t.Fatalf("inner mapping query lost expansion:\n%s", compiled.Query) + if mode == "INNER" && !strings.Contains(testAQLText(t, compiled.Command), "LET item =") { + t.Fatalf("inner mapping query lost expansion:\n%s", testAQLText(t, compiled.Command)) } - if mode == "OUTER" && !strings.Contains(compiled.Query, "LET item =") { - t.Fatalf("outer mapping query lost expansion:\n%s", compiled.Query) + if mode == "OUTER" && !strings.Contains(testAQLText(t, compiled.Command), "LET item =") { + t.Fatalf("outer mapping query lost expansion:\n%s", testAQLText(t, compiled.Command)) } - if mode == "INNER" && !strings.Contains(compiled.Query, "LENGTH(__loom_physical_unnest_source_0) == 0 ? [] : RANGE") { - t.Fatalf("inner mapping query did not preserve inner expansion:\n%s", compiled.Query) + if mode == "INNER" && !strings.Contains(testAQLText(t, compiled.Command), "LENGTH(__loom_physical_unnest_source_0) == 0 ? [] : RANGE") { + t.Fatalf("inner mapping query did not preserve inner expansion:\n%s", testAQLText(t, compiled.Command)) } - if mode == "OUTER" && !strings.Contains(compiled.Query, "LENGTH(__loom_physical_unnest_source_0) == 0 ? [null]") { - t.Fatalf("outer mapping query did not preserve outer expansion:\n%s", compiled.Query) + if mode == "OUTER" && !strings.Contains(testAQLText(t, compiled.Command), "LENGTH(__loom_physical_unnest_source_0) == 0 ? [null]") { + t.Fatalf("outer mapping query did not preserve outer expansion:\n%s", testAQLText(t, compiled.Command)) } - if strings.Index(compiled.Query, "FOR item IN") > strings.Index(compiled.Query, "FOR __loom_physical_population_mapping_member") { - t.Fatalf("%s witness terminal ran before final expansion:\n%s", mode, compiled.Query) + if strings.Index(testAQLText(t, compiled.Command), "FOR item IN") > strings.Index(testAQLText(t, compiled.Command), "FOR __loom_physical_population_mapping_member") { + t.Fatalf("%s witness terminal ran before final expansion:\n%s", mode, testAQLText(t, compiled.Command)) } }) } @@ -316,15 +317,15 @@ func TestCompilePopulationMappingConstructionUsesFinalObjectIdentityAndRootMembe "@__loom_physical_construction_population_mapping_identity_name]: [__loom_physical_construction_population_mapping_selected_identity]", } for _, want := range wants { - if !strings.Contains(compiled.Query, want) { - t.Fatalf("construction population mapping query is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Fatalf("construction population mapping query is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } if compiled.RowIdentity == nil || len(compiled.RowIdentity.Fields) != 1 || compiled.RowIdentity.Fields[0] != "__loom_row_id" || compiled.IdentityPartsColumn != ir.PhysicalPopulationMappingIdentityPartsField || compiled.ExplicitIdentityColumn != "" { t.Fatalf("construction population mapping identity columns = (%#v, %q, %q)", compiled.RowIdentity, compiled.IdentityPartsColumn, compiled.ExplicitIdentityColumn) } - if !containsBindValue(compiled.BindVars, ir.PhysicalPopulationMappingMembersColumn) { + if !containsBindValue(testAQLBinds(t, compiled.Command), ir.PhysicalPopulationMappingMembersColumn) { t.Fatal("construction population mapping source IDs are not projected through the private source mapping column") } } @@ -339,7 +340,7 @@ func TestCompilePopulationMappingOrdinaryGroupRelatedUsesTypedFinalIdentity(t *t t.Fatalf("GROUP→RELATED_SOURCE final identity = %#v", sequence) } - mapping, err := CompilePopulationMappingOutputWithPolicy(compiledOutput, recipe.RuntimeBindings{ + mapping, err := CompilePopulationMappingOutput(ArangoTarget(), compiledOutput, recipe.RuntimeBindings{ Project: "fixture_project", SelectionProject: "fixture/project", DatasetGeneration: "generation-a", SelectionMembersCollection: "loom_explorer_selection_members", }, ir.DefaultPhysicalOptimizationPolicy()) @@ -352,9 +353,9 @@ func TestCompilePopulationMappingOrdinaryGroupRelatedUsesTypedFinalIdentity(t *t if mapping.IdentityPartsColumn != ir.PhysicalPopulationMappingIdentityPartsField || mapping.ExplicitIdentityColumn != "" { t.Fatalf("mapping identity columns = (%q, %q)", mapping.IdentityPartsColumn, mapping.ExplicitIdentityColumn) } - if !strings.Contains(mapping.Query, "COLLECT __loom_physical_construction_population_mapping_selected_member") || - !strings.Contains(mapping.Query, "__loom_construction_final_row[@__loom_physical_construction_population_mapping_final_row_identity]") { - t.Fatalf("mapping query did not bind selected members to the exact final row identity:\n%s", mapping.Query) + if !strings.Contains(testAQLText(t, mapping.Command), "COLLECT __loom_physical_construction_population_mapping_selected_member") || + !strings.Contains(testAQLText(t, mapping.Command), "__loom_construction_final_row[@__loom_physical_construction_population_mapping_final_row_identity]") { + t.Fatalf("mapping query did not bind selected members to the exact final row identity:\n%s", testAQLText(t, mapping.Command)) } } @@ -362,7 +363,7 @@ func TestCompilePopulationMappingConstructionRejectsUnrelatedIdentityFallback(t compiledOutput := compilePopulationMappingOutput(t, populationMappingGroupRelatedOutput()) compiledOutput.RowIdentity = compiledOutput.RowIdentity.Clone() compiledOutput.RowIdentity.Fields = []string{"project", "unrelated_key"} - if _, err := CompilePopulationMappingOutputWithPolicy(compiledOutput, recipe.RuntimeBindings{ + if _, err := CompilePopulationMappingOutput(ArangoTarget(), compiledOutput, recipe.RuntimeBindings{ Project: "fixture_project", SelectionProject: "fixture/project", DatasetGeneration: "generation-a", SelectionMembersCollection: "loom_explorer_selection_members", }, ir.DefaultPhysicalOptimizationPolicy()); err == nil || !strings.Contains(err.Error(), "exact final row identity") { @@ -407,7 +408,7 @@ func TestCompilePopulationMappingConstructionRejectsUnprovenFinalIdentitySchema( } else { test.mutate(&compiledOutput.Plan.StageSequence.FinalColumns[identityIndex]) } - if _, err := CompilePopulationMappingOutputWithPolicy(compiledOutput, populationMappingTestBindings(), ir.DefaultPhysicalOptimizationPolicy()); err == nil { + if _, err := CompilePopulationMappingOutput(ArangoTarget(), compiledOutput, populationMappingTestBindings(), ir.DefaultPhysicalOptimizationPolicy()); err == nil { t.Fatalf("population mapping accepted unproven final identity schema: %#v", compiledOutput.Plan.StageSequence.FinalColumns) } }) @@ -427,7 +428,7 @@ func TestCompilePopulationMappingConstructionRejectsWrongDefaultIdentityFieldCou compiledOutput := compilePopulationMappingOutput(t, populationMappingGroupRelatedOutput()) compiledOutput.RowIdentity = compiledOutput.RowIdentity.Clone() compiledOutput.RowIdentity.Fields = append([]string(nil), test.fields...) - if _, err := CompilePopulationMappingOutputWithPolicy(compiledOutput, populationMappingTestBindings(), ir.DefaultPhysicalOptimizationPolicy()); err == nil || !strings.Contains(err.Error(), "exact final row identity") { + if _, err := CompilePopulationMappingOutput(ArangoTarget(), compiledOutput, populationMappingTestBindings(), ir.DefaultPhysicalOptimizationPolicy()); err == nil || !strings.Contains(err.Error(), "exact final row identity") { t.Fatalf("identity fields %v were accepted or returned the wrong error: %v", test.fields, err) } }) @@ -453,7 +454,7 @@ func populationMappingTestBindings() recipe.RuntimeBindings { func compilePopulationMappingRecipe(t *testing.T, output recipe.Output) CompiledPopulationMappingQuery { t.Helper() compiled := compilePopulationMappingOutput(t, output) - mapping, err := CompilePopulationMappingOutputWithPolicy(compiled, recipe.RuntimeBindings{Project: "fixture_program-project-a", SelectionProject: "fixture_program/project-a", DatasetGeneration: "generation-a", SelectionMembersCollection: "loom_explorer_selection_members"}, ir.DefaultPhysicalOptimizationPolicy()) + mapping, err := CompilePopulationMappingOutput(ArangoTarget(), compiled, recipe.RuntimeBindings{Project: "fixture_program-project-a", SelectionProject: "fixture_program/project-a", DatasetGeneration: "generation-a", SelectionMembersCollection: "loom_explorer_selection_members"}, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -483,3 +484,195 @@ func compilePopulationMappingOutput(t *testing.T, output recipe.Output) lower.Co } return compiled.Outputs[0] } + +func populationMappingPostgreSQLTarget(t *testing.T) Target { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("loom") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + target, err := PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("PostgreSQLTarget: %v", err) + } + return target +} + +func populationMappingPostgreSQLSQL(t *testing.T, compiled CompiledPopulationMappingQuery) (string, []command.PostgreSQLParameter) { + t.Helper() + if compiled.Command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("mapping command backend = %q, want postgresql", string(compiled.Command.Backend())) + } + sql, args, err := compiled.Command.PostgreSQLParameters() + if err != nil { + t.Fatalf("PostgreSQLParameters: %v", err) + } + if sql == "" { + t.Fatal("mapping statement SQL is empty") + } + return sql, args +} + +func TestCompilePopulationMappingPostgreSQLDefaultIdentityWitnesses(t *testing.T) { + output := recipe.Output{ + Name: "Specimens", RootResourceType: "Specimen", RowGrain: "specimen", + Fields: []recipe.Field{{Name: "id", Expr: recipe.Expression{Select: "root.id"}}}, + Population: &recipe.PopulationConstraint{ + SelectionRevisionID: "selection-1", MembershipDigest: "sha256:members", MemberCount: 3, + ResourceType: "Specimen", + }, + } + compiledOutput := compilePopulationMappingOutput(t, output) + bindings := recipe.RuntimeBindings{Project: "fixture_program-project-a", SelectionProject: "fixture_program/project-a", DatasetGeneration: "generation-a", SelectionMembersCollection: "loom_explorer_selection_members"} + postgresQuery, err := CompilePopulationMappingOutput(populationMappingPostgreSQLTarget(t), compiledOutput, bindings, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("postgresql population mapping compile: %v", err) + } + sql, args := populationMappingPostgreSQLSQL(t, postgresQuery) + for _, want := range []string{ + `CROSS JOIN LATERAL jsonb_array_elements_text(`, + `jsonb_typeof(`, + `'[]'::jsonb`, + `jsonb_build_array(`, + `ARRAY_AGG(`, + `COLLATE "C"`, + `"loom"."selection_members"`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("mapping SQL is missing %q:\n%s", want, sql) + } + } + if strings.Contains(sql, "@") { + t.Fatalf("mapping SQL carries AQL bind syntax:\n%s", sql) + } + if len(args) == 0 { + t.Fatal("mapping statement is not parameterized") + } + if strings.Contains(sql, "fixture_program") { + t.Fatalf("mapping SQL inlines bind values:\n%s", sql) + } + aqlQuery, err := CompilePopulationMappingOutput(ArangoTarget(), compiledOutput, bindings, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("aql population mapping control: %v", err) + } + if postgresQuery.MemberColumn != aqlQuery.MemberColumn || + postgresQuery.IdentityPartsColumn != aqlQuery.IdentityPartsColumn || + postgresQuery.ExplicitIdentityColumn != aqlQuery.ExplicitIdentityColumn || + !reflect.DeepEqual(postgresQuery.RowIdentity, aqlQuery.RowIdentity) || + !reflect.DeepEqual(postgresQuery.Diagnostics, aqlQuery.Diagnostics) { + t.Fatalf("postgresql proof metadata drifted from the AQL control:\n pg: %#v\n aql: %#v", postgresQuery, aqlQuery) + } + if postgresQuery.IdentityPartsColumn != ir.PhysicalPopulationMappingIdentityPartsField || postgresQuery.ExplicitIdentityColumn != "" { + t.Fatalf("default mapping identity columns = (%q, %q)", postgresQuery.IdentityPartsColumn, postgresQuery.ExplicitIdentityColumn) + } + second, err := CompilePopulationMappingOutput(populationMappingPostgreSQLTarget(t), compiledOutput, bindings, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("second postgresql compile: %v", err) + } + sql2, args2 := populationMappingPostgreSQLSQL(t, second) + if sql2 != sql || !reflect.DeepEqual(args2, args) { + t.Fatal("population mapping rendering is not deterministic") + } +} + +func TestCompilePopulationMappingPostgreSQLExplicitIdentity(t *testing.T) { + compiled := compilePopulationMappingRecipePostgreSQL(t, recipe.Output{ + Name: "Specimens", RootResourceType: "Specimen", RowGrain: "specimen", + Fields: []recipe.Field{{Name: "id", Expr: recipe.Expression{Select: "root.id"}}}, + Identity: &recipe.Identity{Name: "row", Expr: recipe.Expression{Select: "root.id"}}, + Population: &recipe.PopulationConstraint{ + SelectionRevisionID: "selection-1", MembershipDigest: "sha256:members", MemberCount: 3, + ResourceType: "Specimen", + }, + }) + sql, _ := populationMappingPostgreSQLSQL(t, compiled) + if compiled.IdentityPartsColumn != "" || compiled.ExplicitIdentityColumn != ir.PhysicalPopulationMappingExplicitIdentityField { + t.Fatalf("explicit mapping identity columns = (%q, %q)", compiled.IdentityPartsColumn, compiled.ExplicitIdentityColumn) + } + if !strings.Contains(sql, "#>") { + t.Fatalf("explicit identity did not render through the registered payload path:\n%s", sql) + } +} + +func populationMappingSummaryOutput(keyed bool) recipe.Output { + output := constructionSummaryOutput() + if keyed { + group := output.Construction.Steps[0].Operation.Group + group.Keys = []recipe.ConstructionGroupKey{{InputColumnID: "status_id", OutputColumnID: "status_key_id"}} + output.Construction.Steps[0].Outputs = append([]recipe.StageColumn{{ID: "status_key_id", Name: "status_key"}}, output.Construction.Steps[0].Outputs...) + } + output.Population = &recipe.PopulationConstraint{ + SelectionRevisionID: "selection-summary", MembershipDigest: "sha256:summary-members", MemberCount: 2, + ResourceType: "Observation", + } + return output +} + +func TestCompilePopulationMappingPostgreSQLConstructionTerminal(t *testing.T) { + for _, keyed := range []bool{false, true} { + name := "grand-total" + if keyed { + name = "keyed" + } + t.Run(name, func(t *testing.T) { + compiledOutput := compilePopulationMappingOutput(t, populationMappingSummaryOutput(keyed)) + postgresQuery, err := CompilePopulationMappingOutput(populationMappingPostgreSQLTarget(t), compiledOutput, populationMappingTestBindings(), ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("postgresql construction mapping compile: %v", err) + } + sql, _ := populationMappingPostgreSQLSQL(t, postgresQuery) + for _, want := range []string{ + "WITH ", + `"loom"."selection_members"`, + "rcpool AS (", + "ARRAY_AGG(DISTINCT", + "INNER JOIN __loom_src AS ps ON", + "CROSS JOIN LATERAL jsonb_array_elements_text(", + `GROUP BY s."`, + `COLLATE "C" ASC, mn0.member COLLATE "C" ASC`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("construction mapping SQL is missing %q:\n%s", want, sql) + } + } + if strings.Contains(sql, "@") { + t.Fatalf("construction mapping SQL carries AQL bind syntax:\n%s", sql) + } + if postgresQuery.IdentityPartsColumn != ir.PhysicalPopulationMappingIdentityPartsField { + t.Fatalf("construction mapping identity column = %q", postgresQuery.IdentityPartsColumn) + } + aqlQuery, err := CompilePopulationMappingOutput(ArangoTarget(), compiledOutput, populationMappingTestBindings(), ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("aql construction control: %v", err) + } + if !reflect.DeepEqual(postgresQuery.RowIdentity, aqlQuery.RowIdentity) || !reflect.DeepEqual(postgresQuery.Diagnostics, aqlQuery.Diagnostics) { + t.Fatalf("construction proof metadata drifted from the AQL control") + } + }) + } +} + +func TestCompilePopulationMappingPostgreSQLRefusesMissingRowIdentity(t *testing.T) { + compiledOutput := compilePopulationMappingOutput(t, recipe.Output{ + Name: "Specimens", RootResourceType: "Specimen", RowGrain: "specimen", + Fields: []recipe.Field{{Name: "id", Expr: recipe.Expression{Select: "root.id"}}}, + Population: &recipe.PopulationConstraint{ + SelectionRevisionID: "selection-1", MembershipDigest: "sha256:members", MemberCount: 3, + ResourceType: "Specimen", + }, + }) + compiledOutput.RowIdentity = nil + if _, err := CompilePopulationMappingOutput(populationMappingPostgreSQLTarget(t), compiledOutput, populationMappingTestBindings(), ir.DefaultPhysicalOptimizationPolicy()); err == nil || !strings.Contains(err.Error(), "requires a row identity") { + t.Fatalf("missing row identity error = %v", err) + } +} + +func compilePopulationMappingRecipePostgreSQL(t *testing.T, output recipe.Output) CompiledPopulationMappingQuery { + t.Helper() + compiled := compilePopulationMappingOutput(t, output) + mapping, err := CompilePopulationMappingOutput(populationMappingPostgreSQLTarget(t), compiled, recipe.RuntimeBindings{Project: "fixture_program-project-a", SelectionProject: "fixture_program/project-a", DatasetGeneration: "generation-a", SelectionMembersCollection: "loom_explorer_selection_members"}, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatal(err) + } + return mapping +} diff --git a/internal/dataframe/compiler/postgres_coded_group_compile_test.go b/internal/dataframe/compiler/postgres_coded_group_compile_test.go new file mode 100644 index 000000000..b3d875c16 --- /dev/null +++ b/internal/dataframe/compiler/postgres_coded_group_compile_test.go @@ -0,0 +1,159 @@ +package compiler_test + +import ( + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" +) + +func codedGroup19CompiledOutput(t *testing.T, policy recipe.ConstructionGroupMissingKeyPolicy) (lower.CompiledRecipeOutput, recipe.RuntimeBindings) { + t.Helper() + columns := []recipe.StageColumn{ + {ID: "system_id", Name: "code_system", Type: "string", Nullable: true}, + {ID: "version_id", Name: "code_version", Type: "string", Nullable: true}, + {ID: "code_id", Name: "code", Type: "string", Nullable: true}, + {ID: "count_id", Name: "source_records", Type: "integer"}, + } + output := recipe.Output{ + Name: "coded_group_rows", RootResourceType: "Observation", RowGrain: "resource", + Fields: []recipe.Field{{Name: "resource_id", ColumnID: "resource_id", Expr: recipe.Expression{Select: "root.id"}}}, + Construction: &recipe.Construction{ + Version: 1, SourceColumns: []recipe.StageColumn{{ID: "resource_id", Name: "resource_id"}}, + Steps: []recipe.ConstructionStep{{ + ID: "group_codes", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionCodedGroupOp, CodedGroup: &recipe.ConstructionCodedGroup{ + ConstructionID: "group_codes", + Source: recipe.ConstructionCodedGroupSource{ + OccurrenceID: "base", ResourceType: "Observation", CodingPath: "component[].code.coding[]", + FHIRType: "Coding", Cardinality: "MANY", Shape: "ARRAY", Route: []recipe.ConstructionRelatedRouteStep{}, + }, + MissingKeyPolicy: policy, + SystemOutputColumnID: "system_id", VersionOutputColumnID: "version_id", + CodeOutputColumnID: "code_id", DistinctSourceCountOutputColumnID: "count_id", + }}, Outputs: columns, + }}, + }, + } + bindings := recipe.RuntimeBindings{Project: "P19", DatasetGeneration: "G19", AuthResourcePaths: []string{"FHIR/Observation"}} + bundle := recipe.Bundle{RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: output.Name, TranslationVersion: "test", Outputs: []recipe.Output{output}} + plan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("lower: %v", err) + } + return compiled.Outputs[0], bindings +} + +func TestPostgreSQLCodedGroupCompileContract(t *testing.T) { + binding, err := ir.NewPostgresStorageBinding("loom") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("PostgreSQLTarget: %v", err) + } + output, bindings := codedGroup19CompiledOutput(t, recipe.ConstructionGroupMissingKeyGroup) + query, err := compiler.CompileRecipeOutput(target, output, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("CompileRecipeOutput (PostgreSQL): %v", err) + } + + if query.Command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("command backend = %q, want PostgreSQL", string(query.Command.Backend())) + } + if _, _, err := query.Command.AQLParameters(); err == nil { + t.Fatalf("a PostgreSQL-tagged command must not expose AQL parameters") + } + sql, args, err := query.Command.PostgreSQLParameters() + if err != nil { + t.Fatalf("PostgreSQLParameters: %v", err) + } + for _, want := range []string{"WITH cg_src AS (", "cg_candidates", "cg_tuples", "cg_grouped", `COLLATE "C" ASC`} { + if !strings.Contains(sql, want) { + t.Fatalf("compiled statement is missing %q:\n%s", want, sql) + } + } + if strings.Contains(sql, "@") { + t.Fatalf("compiled statement carries AQL bind syntax:\n%s", sql) + } + if len(args) == 0 { + t.Fatalf("compiled statement declares no bound parameters") + } + texts := 0 + for index, parameter := range args { + switch parameter.Type { + case command.PostgreSQLParameterText: + texts++ + case command.PostgreSQLParameterJSONB: + default: + t.Fatalf("arg %d has unregistered type %q", index, string(parameter.Type)) + } + } + if texts < 8 { + t.Fatalf("text-bound path/key parameters = %d, want at least 8", texts) + } + + wantColumns := []string{"code_system", "code_version", "code", "source_records", "__loom_row_id"} + if strings.Join(query.Columns, ",") != strings.Join(wantColumns, ",") { + t.Fatalf("declared columns = %v, want %v", query.Columns, wantColumns) + } + wantPublic := []string{"code_system", "code_version", "code", "source_records"} + if strings.Join(query.PublicColumns, ",") != strings.Join(wantPublic, ",") { + t.Fatalf("public columns = %v, want %v", query.PublicColumns, wantPublic) + } + + repeat, err := compiler.CompileRecipeOutput(target, output, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("recompile: %v", err) + } + if repeat.PlanDiagnostics.Fingerprint != query.PlanDiagnostics.Fingerprint { + t.Fatalf("fingerprint is not deterministic: %s vs %s", query.PlanDiagnostics.Fingerprint, repeat.PlanDiagnostics.Fingerprint) + } + excludeOutput, excludeBindings := codedGroup19CompiledOutput(t, recipe.ConstructionGroupMissingKeyExclude) + exclude, err := compiler.CompileRecipeOutput(target, excludeOutput, excludeBindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("CompileRecipeOutput (EXCLUDE): %v", err) + } + if exclude.PlanDiagnostics.Fingerprint == query.PlanDiagnostics.Fingerprint { + t.Fatalf("fingerprint must move when the rendered surface moves") + } +} + +func TestPostgreSQLCodedGroupAQLControlFreezesSharedLowering(t *testing.T) { + output, bindings := codedGroup19CompiledOutput(t, recipe.ConstructionGroupMissingKeyError) + query, err := compiler.CompileRecipeOutput(compiler.ArangoTarget(), output, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("CompileRecipeOutput (AQL control): %v", err) + } + if query.Command.Backend() != command.CompiledBackendAQL { + t.Fatalf("control command backend = %q, want AQL", string(query.Command.Backend())) + } + aql, _, err := query.Command.AQLParameters() + if err != nil { + t.Fatalf("AQLParameters: %v", err) + } + for _, want := range []string{"CONSTRUCTION_CODED_GROUP_MISSING_KEY", `"system"`, `"version"`, `"code"`} { + if !strings.Contains(aql, want) { + t.Fatalf("AQL control drifted, missing %q:\n%s", want, aql) + } + } + for _, forbidden := range []string{"WITH cg_src", "jsonb_array_elements", "COLLATE"} { + if strings.Contains(aql, forbidden) { + t.Fatalf("AQL control leaked PostgreSQL renderer text (%q):\n%s", forbidden, aql) + } + } +} diff --git a/internal/dataframe/compiler/postgres_parity_fixture_test.go b/internal/dataframe/compiler/postgres_parity_fixture_test.go new file mode 100644 index 000000000..dd5bc06fb --- /dev/null +++ b/internal/dataframe/compiler/postgres_parity_fixture_test.go @@ -0,0 +1,165 @@ +package compiler_test + +import ( + "errors" + "reflect" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/render/aql" + postgresrender "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" +) + +func TestPostgresParityFixture(t *testing.T) { + bindings := recipe.RuntimeBindings{Project: "parity-project", DatasetGeneration: "parity-generation"} + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "parity_fixture", + TranslationVersion: "test", + Outputs: []recipe.Output{{ + Name: "patients", RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{{Name: "gender", ColumnID: "gender", Expr: recipe.Expression{Select: "root.gender"}}}, + }}, + } + plan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + queries, err := compiler.CompileResolvedRecipePlan(compiler.ArangoTarget(), resolved, 7, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("CompileResolvedRecipePlanWithPolicy: %v", err) + } + if len(queries) != 1 { + t.Fatalf("compiled outputs = %d, want 1", len(queries)) + } + compiled := queries[0] + + t.Run("AQLPathIsUnchanged", func(t *testing.T) { + rendered, err := aql.RenderPhysicalPlan(compiled.PhysicalPlan) + if err != nil { + t.Fatalf("AQL render of the retained plan: %v", err) + } + if rendered.Query != testAQLText(t, compiled.Command) { + t.Fatalf("AQL renderer output drifted from the compiled query") + } + if !strings.Contains(rendered.Query, "@") { + t.Fatalf("expected the AQL path to keep its named bind syntax") + } + compiledCommand, err := command.NewAQLCommand(rendered.Query, rendered.BindVars) + if err != nil { + t.Fatalf("AQL command registration: %v", err) + } + if _, _, err := compiledCommand.PostgreSQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("AQL command to the PostgreSQL executor error = %v, want mismatch", err) + } + }) + + t.Run("CompileSelectionStaysFailClosed", func(t *testing.T) { + if _, err := compiler.CompileResolvedRecipePlan(compiler.Target{}, resolved, 7, ir.DefaultPhysicalOptimizationPolicy()); !errors.Is(err, compiler.ErrTargetNotSelected) { + t.Fatalf("zero-target compile error = %v, want ErrTargetNotSelected", err) + } + if _, err := compiler.PostgreSQLTarget(ir.PostgresStorageBinding{}); err == nil { + t.Fatal("unbound PostgreSQL target accepted") + } + storage, err := ir.NewPostgresStorageBinding("pgir02_it_parity") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + target, err := compiler.PostgreSQLTarget(storage) + if err != nil { + t.Fatalf("PostgreSQLTarget: %v", err) + } + pgQueries, pgErr := compiler.CompileResolvedRecipePlan(target, resolved, 7, ir.DefaultPhysicalOptimizationPolicy()) + if pgErr != nil { + if !errors.Is(pgErr, postgresrender.ErrUnsupportedPostgresPlan) { + t.Fatalf("PostgreSQL compile error = %v, want ErrUnsupportedPostgresPlan", pgErr) + } + return + } + if len(pgQueries) != 1 { + t.Fatalf("PostgreSQL compiled outputs = %d, want 1", len(pgQueries)) + } + if pgQueries[0].Command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("PostgreSQL compile produced tag %q; AQL fallback is forbidden", pgQueries[0].Command.Backend()) + } + }) + + t.Run("PostgreSQLRendererNeverFallsBackToAQL", func(t *testing.T) { + storage, err := ir.NewPostgresStorageBinding("pgir02_it_parity") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + pgRendered, pgErr := postgresrender.RenderPhysicalPlan(compiled.PhysicalPlan, storage) + if pgErr == nil { + if strings.Contains(pgRendered.SQL, "@") { + t.Fatalf("PostgreSQL SQL carries AQL bind syntax: %s", pgRendered.SQL) + } + if pgRendered.SQL == testAQLText(t, compiled.Command) { + t.Fatalf("PostgreSQL renderer returned the AQL query text") + } + return + } + if !errors.Is(pgErr, postgresrender.ErrUnsupportedPostgresPlan) { + t.Fatalf("PostgreSQL render error = %v, want ErrUnsupportedPostgresPlan", pgErr) + } + if strings.Contains(pgErr.Error(), testAQLText(t, compiled.Command)) { + t.Fatalf("PostgreSQL failure path leaked AQL text as a result") + } + }) + + t.Run("PostgreSQLRenderedSpineDispatchesOnlyToPostgreSQL", func(t *testing.T) { + storage, err := ir.NewPostgresStorageBinding("pgir02_it_parity") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + spine := ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": "Patient", + "project": "parity-project", + "dataset_generation": "parity-generation", + "auth_resource_paths": []string{"FHIR/Patient"}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + }, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + }, + } + first, err := postgresrender.RenderPhysicalPlan(spine, storage) + if err != nil { + t.Fatalf("PostgreSQL spine render: %v", err) + } + second, err := postgresrender.RenderPhysicalPlan(spine, storage) + if err != nil || first.SQL != second.SQL || !reflect.DeepEqual(first.Args, second.Args) { + t.Fatalf("parity spine render is not deterministic") + } + if strings.Contains(first.SQL, "@") { + t.Fatalf("PostgreSQL SQL carries AQL bind syntax: %s", first.SQL) + } + compiledCommand, err := command.NewPostgreSQLCommand(first.SQL, first.Args) + if err != nil { + t.Fatalf("PostgreSQL command construction from rendered plan: %v", err) + } + if _, _, err := compiledCommand.AQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("PostgreSQL command to the AQL adapter error = %v, want mismatch", err) + } + sql, args, err := compiledCommand.PostgreSQLParameters() + if err != nil || sql != first.SQL || !reflect.DeepEqual(args, first.Args) { + t.Fatalf("PostgreSQL dispatch round trip lost the rendered result") + } + }) +} diff --git a/internal/dataframe/compiler/postgres_target.go b/internal/dataframe/compiler/postgres_target.go new file mode 100644 index 000000000..e22df213c --- /dev/null +++ b/internal/dataframe/compiler/postgres_target.go @@ -0,0 +1,48 @@ +package compiler + +import ( + "fmt" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" +) + +func compilePostgreSQLRecipeOutput(target Target, output lower.CompiledRecipeOutput, bindings recipe.RuntimeBindings, limit int, physical ir.PhysicalPlan, includePreviewSourceID bool) (CompiledQuery, error) { + tagged, rendered, err := target.renderPostgreSQLFinalPlan(physical) + if err != nil { + return CompiledQuery{}, err + } + if len(rendered.OutputColumns) == 0 { + return CompiledQuery{}, fmt.Errorf("postgresql statement declares no result columns") + } + outputSchema := lower.CloneCompiledOutputSchema(output.OutputSchema) + if includePreviewSourceID { + outputSchema = append(outputSchema, lower.CompiledOutputColumn{ + ID: ir.PreviewSourceResourceIDColumn, Name: ir.PreviewSourceResourceIDColumn, + Label: ir.PreviewSourceResourceIDColumn, SemanticPath: "preview:source_resource_id", + Kind: "string", Cardinality: "optional_one", Nullable: true, Internal: true, + }) + } + publicColumns := postgreSQLPublicColumns(outputSchema, rendered.OutputColumns) + if len(outputSchema) > 0 && len(publicColumns) == 0 { + return CompiledQuery{}, fmt.Errorf("every declared PostgreSQL result column is an internal projection; refusing to publish internal columns in the public row") + } + return CompiledQuery{ + PhysicalPlan: clonePhysicalPlan(physical), + Project: bindings.Project, + DatasetGeneration: normalizeDatasetGeneration(bindings.DatasetGeneration), + RootResourceType: output.RootResourceType, + TranslationVersion: output.TranslationVersion, + AuthResourcePaths: cloneStrings(bindings.AuthResourcePaths), + PlanMode: "physical", + PlanProfile: "generic_fhir_graph_recipe", + RowIdentity: output.RowIdentity.Clone(), + Command: tagged, + Columns: append([]string(nil), rendered.OutputColumns...), + OutputSchema: outputSchema, + PublicColumns: append([]string(nil), publicColumns...), + Limit: limit, + PlanDiagnostics: ir.CompilerPlanDiagnostics{Fingerprint: postgreSQLPlanFingerprint(rendered.SQL, rendered.Args, rendered.OutputColumns)}, + }, nil +} diff --git a/internal/dataframe/compiler/postgres_target_fingerprint_golden_test.go b/internal/dataframe/compiler/postgres_target_fingerprint_golden_test.go new file mode 100644 index 000000000..40ffb8e8d --- /dev/null +++ b/internal/dataframe/compiler/postgres_target_fingerprint_golden_test.go @@ -0,0 +1,136 @@ +package compiler_test + +import ( + "encoding/json" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/spec" +) + +const ( + archBackend11BaseFingerprint = "25f4a1abcf9192edb94f53e905cd596f32d96cba67548f68d5e1ec619bdb4dc5" + archBackend11BaseSQL = "SELECT r0.\"logical_id\" AS \"c0\", (r0.\"payload\" #> $5::text[]) AS \"c1\" " + + "FROM \"loom\".\"fhir_resource\" AS r0 " + + "WHERE r0.\"project\" = $1 AND r0.\"dataset_generation\" = $2 " + + "AND r0.\"auth_resource_path\" = ANY (SELECT jsonb_array_elements_text($3::jsonb)) " + + "AND r0.\"resource_type\" = $4 " + + "ORDER BY r0.\"logical_id\" ASC, r0.\"resource_seq\" ASC" +) + +func archBackend11FingerprintFixture() (lower.CompiledRecipeOutput, recipe.RuntimeBindings) { + bindings := recipe.RuntimeBindings{ + Project: "P1", DatasetGeneration: "G1", OutputNames: []string{"resources"}, + } + output := lower.CompiledRecipeOutput{ + Name: "resources", + RootResourceType: "Patient", + Columns: []string{"id", "name"}, + OutputSchema: []lower.CompiledOutputColumn{ + {ID: "id", Name: "id", Label: "id", Kind: "string", Cardinality: "one"}, + {ID: "name", Name: "name", Label: "name", Kind: "string", Cardinality: "one"}, + }, + RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, + Plan: ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": "Patient", + "project": "P1", + "dataset_generation": "G1", + "auth_resource_paths": []string{"FHIR/Patient"}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + }, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"project"}}, Right: &ir.PhysicalValue{BindKey: "project"}}}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"dataset_generation"}}, Right: &ir.PhysicalValue{BindKey: "dataset_generation"}}}}, + {Kind: ir.PhysicalDerivedLetOp, DerivedLet: &ir.PhysicalDerivedLet{Variable: "root_auth_allowed", Operator: "AUTH_RESOURCE_PATH_ALLOWED", Inputs: []ir.PhysicalValue{ + {Variable: "root", Path: []string{"auth_resource_path"}}, + {BindKey: "auth_resource_paths"}, + {BindKey: "auth_resource_paths_unrestricted"}, + }}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root_auth_allowed"}, Right: &ir.PhysicalValue{BindKey: "scope_allowed"}}}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + {Name: "name", Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Patient", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: "gender"}}}, + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + }, Presence: &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + Paths: [][]string{{"gender"}}, + }}, + }}}, + }, + }, + } + return output, bindings +} + +func TestPostgreSQLPlanFingerprintGoldenMatchesBase648f7c70(t *testing.T) { + binding, err := ir.NewPostgresStorageBinding("loom") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("PostgreSQLTarget: %v", err) + } + output, bindings := archBackend11FingerprintFixture() + query, err := compiler.CompileRecipeOutput(target, output, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("CompileRecipeOutput: %v", err) + } + sql, args, err := query.Command.PostgreSQLParameters() + if err != nil { + t.Fatalf("PostgreSQLParameters: %v", err) + } + if sql != archBackend11BaseSQL { + t.Fatalf("fixture statement drifted from the base capture input:\n got %q\nwant %q", sql, archBackend11BaseSQL) + } + if len(args) != 5 { + t.Fatalf("argument count drifted from the base capture: %d", len(args)) + } + wantArgs := []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "P1"}, + {Type: command.PostgreSQLParameterText, Value: "G1"}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["FHIR/Patient"]`)}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + {Type: command.PostgreSQLParameterText, Value: "{gender}"}, + } + for i := range wantArgs { + if args[i].Type != wantArgs[i].Type { + t.Fatalf("arg %d type: %q want %q", i, args[i].Type, wantArgs[i].Type) + } + if string(marshalProbe(args[i].Value)) != string(marshalProbe(wantArgs[i].Value)) { + t.Fatalf("arg %d value: %v want %v", i, args[i].Value, wantArgs[i].Value) + } + } + if len(query.Columns) != 2 || query.Columns[0] != "id" || query.Columns[1] != "name" { + t.Fatalf("declared column contract drifted from the base capture: %v", query.Columns) + } + if query.PlanDiagnostics.Fingerprint != archBackend11BaseFingerprint { + t.Fatalf("moved PostgreSQL fingerprint diverges from the base 648f7c70 capture: got %s want %s", + query.PlanDiagnostics.Fingerprint, archBackend11BaseFingerprint) + } +} + +func marshalProbe(value any) []byte { + encoded, err := json.Marshal(value) + if err != nil { + return []byte("!marshal-error:" + err.Error()) + } + return encoded +} diff --git a/internal/dataframe/compiler/preview_covering_index.go b/internal/dataframe/compiler/preview_covering_index.go index dea0b7d7a..5da6c548a 100644 --- a/internal/dataframe/compiler/preview_covering_index.go +++ b/internal/dataframe/compiler/preview_covering_index.go @@ -8,6 +8,7 @@ import ( "sort" "strings" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/render/aql" ) @@ -16,15 +17,12 @@ const previewCoveringIndexNamePrefix = "loom_pivot_preview_" var previewCoveringIndexPrefixFields = []string{"project", "dataset_generation", "auth_resource_path"} -// RenderPreviewPhysicalPlanWithRootIndexHint rerenders a compiled preview -// plan with one compiler-owned, non-forcing root-index hint. It is used only -// after runtime index metadata identifies an existing compatible definition. -func RenderPreviewPhysicalPlanWithRootIndexHint(plan ir.PhysicalPlan, indexName string) (string, map[string]any, error) { +func RenderPreviewPhysicalPlanWithRootIndexHint(plan ir.PhysicalPlan, indexName string) (command.CompiledCommand, error) { rendered, err := aql.RenderPhysicalPlanWithRootIndexHint(plan, indexName) if err != nil { - return "", nil, err + return command.CompiledCommand{}, err } - return rendered.Query, rendered.BindVars, nil + return command.NewAQLCommand(rendered.Query, rendered.BindVars) } func previewCoveringIndexSpec(plan ir.PhysicalPlan) *PreviewCoveringIndexSpec { @@ -84,11 +82,6 @@ func previewCoveringIndexSpec(plan ir.PhysicalPlan) *PreviewCoveringIndexSpec { return nil } -// streamingMissingCategoryPivotIndexSpec describes the narrowly-scoped -// covering index used by the direct-root streaming Pivot renderer. The key -// fields retain the existing project, generation, and authorization filters; -// payload values are stored as projections so a non-forcing hinted scan can -// avoid document materialization when ArangoDB can cover the query. func streamingMissingCategoryPivotIndexSpec(plan ir.PhysicalPlan) *PreviewCoveringIndexSpec { sequence := plan.StageSequence if sequence == nil || !sequence.PreviewTerminalPivotWindow || sequence.PreviewLimitBindKey == "" || @@ -229,10 +222,6 @@ func previewCoveringDirectScalarProjectionPath(projection ir.PhysicalProjection, return previewCoveringProjectionPath(projection, rootVariable) } -// streamingPivotPresenceParentPath proves that the missing-category marker is -// a direct nested property and that its parent object can be retained whole in -// storedValues. Keeping that object preserves an explicit null leaf separately -// from an absent leaf; storing only the leaf would encode both as null. func streamingPivotPresenceParentPath(projections []ir.PhysicalProjection, pivot ir.PhysicalGroupedPivot, rootVariable string) ([]string, bool) { var categoryPath []string var presence *ir.PhysicalProjectionPresence @@ -291,16 +280,22 @@ func previewGroupScanSpec(plan ir.PhysicalPlan, covering *PreviewCoveringIndexSp if err != nil { return nil } + scopeCountCommand, err := command.NewAQLCommand(scopeCount.Query, scopeCount.BindVars) + if err != nil { + return nil + } sequential, err := aql.RenderPhysicalPlanWithRootIndexDisabled(plan) if err != nil { return nil } + sequentialCommand, err := command.NewAQLCommand(sequential.Query, sequential.BindVars) + if err != nil { + return nil + } return &PreviewGroupScanSpec{ - Collection: covering.Collection, - SequentialQuery: sequential.Query, - SequentialBindVars: sequential.BindVars, - ScopeCountQuery: scopeCount.Query, - ScopeCountBindVars: scopeCount.BindVars, + Collection: covering.Collection, + Sequential: sequentialCommand, + ScopeCount: scopeCountCommand, } } diff --git a/internal/dataframe/compiler/public_columns_freeze_test.go b/internal/dataframe/compiler/public_columns_freeze_test.go new file mode 100644 index 000000000..7ce6b159a --- /dev/null +++ b/internal/dataframe/compiler/public_columns_freeze_test.go @@ -0,0 +1,92 @@ +package compiler + +import ( + "reflect" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/lower" +) + +func frozenColumns(names ...string) []string { return append([]string(nil), names...) } + +func frozenSchema(specs ...[2]any) []lower.CompiledOutputColumn { + schema := make([]lower.CompiledOutputColumn, 0, len(specs)) + for _, spec := range specs { + schema = append(schema, lower.CompiledOutputColumn{ + Name: spec[0].(string), + Internal: spec[1].(bool), + }) + } + return schema +} + +func TestPostgreSQLPublicColumnsFreezeMatchesBase648f7c70(t *testing.T) { + t.Run("GoldenRecipe", func(t *testing.T) { + got := postgreSQLPublicColumns( + frozenSchema([2]any{"id", false}, [2]any{"name", false}), + frozenColumns("id", "name"), + ) + want := []string{"id", "name"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("golden recipe projection: %v want %v", got, want) + } + }) + + t.Run("SchemaOrderIsAuthoritative", func(t *testing.T) { + got := postgreSQLPublicColumns( + frozenSchema([2]any{"id", false}, [2]any{"name", false}), + frozenColumns("name", "id"), + ) + if !reflect.DeepEqual(got, []string{"id", "name"}) { + t.Fatalf("projection must follow compiler schema order, not declared order: %v", got) + } + }) + + t.Run("InternalProjectionEdgeCase", func(t *testing.T) { + got := postgreSQLPublicColumns( + frozenSchema( + [2]any{"id", false}, + [2]any{"__loom_source_resource_id", true}, + [2]any{"name", false}, + ), + frozenColumns("id", "__loom_source_resource_id", "name"), + ) + if !reflect.DeepEqual(got, []string{"id", "name"}) { + t.Fatalf("internal projection leaked into the public preview projection: %v", got) + } + }) + + t.Run("AllInternalRefused", func(t *testing.T) { + got := postgreSQLPublicColumns( + frozenSchema([2]any{"__loom_internal", true}), + frozenColumns("__loom_internal"), + ) + if len(got) != 0 { + t.Fatalf("all-internal contract must project to empty for caller refusal: %v", got) + } + }) + + t.Run("SchemaColumnNotDeclaredIsDropped", func(t *testing.T) { + got := postgreSQLPublicColumns( + frozenSchema([2]any{"id", false}, [2]any{"secret", false}), + frozenColumns("id"), + ) + if !reflect.DeepEqual(got, []string{"id"}) { + t.Fatalf("undeclared schema column must not publish: %v", got) + } + }) + + t.Run("NoSchemaPublishesDeclared", func(t *testing.T) { + declared := frozenColumns("id", "name") + got := postgreSQLPublicColumns(nil, declared) + if !reflect.DeepEqual(got, declared) { + t.Fatalf("no-schema path must publish the declared contract: %v", got) + } + if len(got) != 0 { + got[0] = "mutated" + if declared[0] != "id" { + t.Fatal("no-schema path must copy the declared contract, not alias it") + } + } + }) +} diff --git a/internal/dataframe/compiler/recipe_execution.go b/internal/dataframe/compiler/recipe_execution.go index 4d90cca2e..627baa849 100644 --- a/internal/dataframe/compiler/recipe_execution.go +++ b/internal/dataframe/compiler/recipe_execution.go @@ -3,6 +3,7 @@ package compiler import ( "fmt" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/compiler/optimize" @@ -26,11 +27,10 @@ func workspaceOutputCaptureRequired(output lower.CompiledRecipeOutput) bool { return false } -// CompileResolvedRecipePlanWithPolicy is the common execution boundary for a -// resolved recipe bundle. Lowering produces canonical physical plans; this -// function applies the generic optimizer, inserts the typed execution window, -// and renders through RenderPhysicalPlan for each output. -func CompileResolvedRecipePlanWithPolicy(resolved semantic.ResolvedRecipePlan, limit int, policy ir.PhysicalOptimizationPolicy) ([]CompiledQuery, error) { +func CompileResolvedRecipePlan(target Target, resolved semantic.ResolvedRecipePlan, limit int, policy ir.PhysicalOptimizationPolicy) ([]CompiledQuery, error) { + if err := target.validate(); err != nil { + return nil, err + } compiled, err := lower.CompileResolvedRecipePlan(resolved, policy) if err != nil { return nil, err @@ -38,7 +38,7 @@ func CompileResolvedRecipePlanWithPolicy(resolved semantic.ResolvedRecipePlan, l queries := make([]CompiledQuery, 0, len(compiled.Outputs)) for _, output := range compiled.Outputs { output.TranslationVersion = resolved.SemanticPlan.TranslationVersion - query, err := CompileRecipeOutputWithPolicy(output, resolved.SemanticPlan.Bindings, limit, policy) + query, err := CompileRecipeOutput(target, output, resolved.SemanticPlan.Bindings, limit, policy) if err != nil { return nil, fmt.Errorf("output %q: %w", output.Name, err) } @@ -47,12 +47,16 @@ func CompileResolvedRecipePlanWithPolicy(resolved semantic.ResolvedRecipePlan, l return queries, nil } -// CompileRecipeOutputWithPolicy applies the common optimizer, execution -// window, and canonical renderer to one already-lowered recipe output. -func CompileRecipeOutputWithPolicy(output lower.CompiledRecipeOutput, bindings recipe.RuntimeBindings, limit int, policy ir.PhysicalOptimizationPolicy) (CompiledQuery, error) { +func CompileRecipeOutput(target Target, output lower.CompiledRecipeOutput, bindings recipe.RuntimeBindings, limit int, policy ir.PhysicalOptimizationPolicy) (CompiledQuery, error) { + if err := target.validate(); err != nil { + return CompiledQuery{}, err + } if workspaceOutputCaptureRequired(output) { return CompiledQuery{}, fmt.Errorf("output %q references same-workspace outputs; server-owned workspace capture is not available", output.Name) } + if output.Plan.ClickHousePrefix != nil || (output.OptimizedPlan != nil && output.OptimizedPlan.ClickHousePrefix != nil) { + return CompiledQuery{}, fmt.Errorf("output %q is a composite ClickHouse plan; compile its source prefix through CompileClickHousePrefix and its combine through the ClickHouse stream path", output.Name) + } var physical ir.PhysicalPlan groupRows := len(output.Plan.Operations) == 1 && output.Plan.Operations[0].Kind == ir.PhysicalGroupRowsOp if groupRows { @@ -87,6 +91,9 @@ func CompileRecipeOutputWithPolicy(output lower.CompiledRecipeOutput, bindings r return CompiledQuery{}, err } } + if target.backend == command.CompiledBackendPostgreSQL { + return compilePostgreSQLRecipeOutput(target, output, bindings, limit, physical, includePreviewSourceID) + } streamingMissingCategoryPivot := canRenderStreamingMissingCategoryPivotPreview(output, bindings, physical) legacyPreviewCoveringIndex := previewCoveringIndexSpec(physical) previewCoveringIndex := legacyPreviewCoveringIndex @@ -123,6 +130,10 @@ func CompileRecipeOutputWithPolicy(output lower.CompiledRecipeOutput, bindings r if err != nil { return CompiledQuery{}, fmt.Errorf("render canonical recipe physical plan: %w", err) } + compiledCommand, err := command.NewAQLCommand(rendered.Query, rendered.BindVars) + if err != nil { + return CompiledQuery{}, fmt.Errorf("tag canonical recipe command: %w", err) + } columns, pivotFields := physicalProjectionMetadata(physical) if len(output.Columns) != 0 { columns = append([]string(nil), output.Columns...) @@ -156,8 +167,7 @@ func CompileRecipeOutputWithPolicy(output lower.CompiledRecipeOutput, bindings r TraversalCount: physicalTraversalCount(physical), RowIdentity: output.RowIdentity.Clone(), OptimizationRules: recipeOptimizationRules(physical), - Query: rendered.Query, - BindVars: rendered.BindVars, + Command: compiledCommand, Columns: columns, OutputSchema: outputSchema, PublicColumns: publicColumns, @@ -172,11 +182,109 @@ func CompileRecipeOutputWithPolicy(output lower.CompiledRecipeOutput, bindings r }, nil } -// canRenderStreamingMissingCategoryPivotPreview only enables the streaming -// renderer for a direct typed-root scan feeding one terminal Pivot. The -// renderer retains the full source query and all scope predicates; this gate -// only excludes shapes whose projections or row membership need an -// intermediate table. +type CompiledClickHousePrefix struct { + Command command.CompiledCommand + OutputColumns []string + Fingerprint string +} + +func CompileClickHousePrefix(target Target, plan ir.PhysicalPlan) (CompiledClickHousePrefix, error) { + if err := target.validate(); err != nil { + return CompiledClickHousePrefix{}, err + } + if plan.Engine != ir.PhysicalEngineClickHouse || plan.ClickHousePrefix == nil || plan.ClickHouseCombine == nil { + return CompiledClickHousePrefix{}, fmt.Errorf("composite ClickHouse plan with a typed prefix is required") + } + if err := plan.Validate(); err != nil { + return CompiledClickHousePrefix{}, fmt.Errorf("validate composite ClickHouse prefix plan: %w", err) + } + switch prefix := plan.ClickHousePrefix; prefix.SourceBackend { + case ir.PhysicalPrefixSourceAQL: + if target.backend != command.CompiledBackendAQL { + return CompiledClickHousePrefix{}, fmt.Errorf("AQL-tagged ClickHouse prefix requires the AQL compile target, got %s", string(target.backend)) + } + rendered, err := aql.RenderClickHousePrefix(plan) + if err != nil { + return CompiledClickHousePrefix{}, fmt.Errorf("render ClickHouse prefix command: %w", err) + } + compiledCommand, err := command.NewAQLCommand(rendered.Query, rendered.BindVars) + if err != nil { + return CompiledClickHousePrefix{}, fmt.Errorf("tag ClickHouse prefix command: %w", err) + } + fingerprint, err := categoryQueryFingerprint(rendered.Query, rendered.BindVars) + if err != nil { + return CompiledClickHousePrefix{}, fmt.Errorf("fingerprint ClickHouse prefix command: %w", err) + } + return CompiledClickHousePrefix{ + Command: compiledCommand, + OutputColumns: clickHousePrefixSchema(plan), + Fingerprint: fingerprint, + }, nil + case ir.PhysicalPrefixSourcePostgreSQL: + if target.backend != command.CompiledBackendPostgreSQL { + return CompiledClickHousePrefix{}, fmt.Errorf("PostgreSQL-tagged ClickHouse prefix requires the PostgreSQL compile target, got %s", string(target.backend)) + } + source := ir.ClonePhysicalPlan(plan) + source.Engine = ir.PhysicalEngineAQL + source.ClickHouseCombine = nil + source.ClickHousePrefix = nil + tagged, rendered, err := target.renderPostgreSQLFinalPlan(source) + if err != nil { + return CompiledClickHousePrefix{}, fmt.Errorf("render ClickHouse prefix command: %w", err) + } + if err := matchClickHousePrefixSchema(plan, rendered.OutputColumns); err != nil { + return CompiledClickHousePrefix{}, err + } + return CompiledClickHousePrefix{ + Command: tagged, + OutputColumns: append([]string(nil), rendered.OutputColumns...), + Fingerprint: postgreSQLPlanFingerprint(rendered.SQL, rendered.Args, rendered.OutputColumns), + }, nil + default: + return CompiledClickHousePrefix{}, prefix.SourceBackend.Validate() + } +} + +func clickHousePrefixSchema(plan ir.PhysicalPlan) []string { + if plan.StageSequence == nil { + return nil + } + columns := make([]string, 0, len(plan.StageSequence.FinalColumns)) + for _, column := range plan.StageSequence.FinalColumns { + columns = append(columns, column.Name) + } + return columns +} + +func matchClickHousePrefixSchema(plan ir.PhysicalPlan, declared []string) error { + if plan.StageSequence == nil { + return fmt.Errorf("PostgreSQL ClickHouse prefix has no typed stage sequence") + } + sequence := plan.StageSequence + expected := clickHousePrefixSchema(plan) + if len(expected) == 0 { + return fmt.Errorf("PostgreSQL ClickHouse prefix declares no typed final schema columns") + } + sawRowIdentity := false + for _, column := range sequence.FinalColumns { + if column.Name == sequence.FinalRowIdentity { + sawRowIdentity = true + } + } + if !sawRowIdentity { + return fmt.Errorf("PostgreSQL ClickHouse prefix typed schema does not carry its row identity column %q", sequence.FinalRowIdentity) + } + if len(declared) != len(expected) { + return fmt.Errorf("PostgreSQL ClickHouse prefix declares %d result columns, want the exact typed prefix schema %v", len(declared), expected) + } + for index := range expected { + if declared[index] != expected[index] { + return fmt.Errorf("PostgreSQL ClickHouse prefix result column %d is %q, want the typed prefix column %q", index, declared[index], expected[index]) + } + } + return nil +} + func canRenderStreamingMissingCategoryPivotPreview(output lower.CompiledRecipeOutput, bindings recipe.RuntimeBindings, plan ir.PhysicalPlan) bool { sequence := plan.StageSequence if sequence == nil || !sequence.PreviewTerminalPivotWindow || sequence.PreviewLimitBindKey == "" || @@ -248,10 +356,6 @@ func canRenderStreamingMissingCategoryPivotPreview(output lower.CompiledRecipeOu return true } -// streamingPivotInputProjectsSourceRows proves that the Pivot's row-shaped -// inputs are a pure projection of the immediately preceding source RETURN. -// Lowering represents that relationship through the stage input variable, -// even when the source itself is a direct root scan. func streamingPivotInputProjectsSourceRows(pivot ir.PhysicalGroupedPivot, inputRowVariable string, sourceNames map[string]struct{}) bool { if inputRowVariable == "" || len(pivot.InputProjections) == 0 { return false @@ -286,12 +390,6 @@ func streamingPivotInputProjectsSourceRows(pivot ir.PhysicalGroupedPivot, inputR return true } -// withoutUnusedTerminalPivotPresenceCompanions keeps category-presence -// sidecars out of ordinary terminal Pivot previews. Lowering retains them in -// the canonical plan so a later category-discovery scan can distinguish an -// absent source property from explicit NULL. A frozen Pivot only consumes -// that proof when one of its categories is MISSING; otherwise carrying the -// hidden boolean through the preview would disable the bounded two-scan path. func withoutUnusedTerminalPivotPresenceCompanions(plan ir.PhysicalPlan) ir.PhysicalPlan { sequence := plan.StageSequence if sequence == nil || len(sequence.Stages) != 1 || sequence.PreviewLimitBindKey == "" || diff --git a/internal/dataframe/compiler/recipe_execution_pg18_test.go b/internal/dataframe/compiler/recipe_execution_pg18_test.go new file mode 100644 index 000000000..a86bc6526 --- /dev/null +++ b/internal/dataframe/compiler/recipe_execution_pg18_test.go @@ -0,0 +1,334 @@ +package compiler_test + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "sort" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/authscope" + control "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + "net/url" + "regexp" +) + +const ir03NamespacePrefix = "verify_ir03v1" + +var ir03ResourceNamePattern = regexp.MustCompile(`^verify_ir03v1_it_[0-9]+_[0-9a-f]+$`) + +func ir03TestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the ClickHouse prefix seam PostgreSQL integration tests") + } + return raw +} + +func ir03RandomHex(t *testing.T) string { + t.Helper() + buf := make([]byte, 6) + if _, err := rand.Read(buf); err != nil { + t.Fatalf("read random bytes: %v", err) + } + return hex.EncodeToString(buf) +} + +type ir03Service struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + client *control.Client + schema string + resources *control.ResourceRepository + scrub func(any) string + fail func(step string, err error) +} + +func newIR03Service(t *testing.T) *ir03Service { + t.Helper() + raw := ir03TestPostgresURL(t) + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + dbName := fmt.Sprintf(ir03NamespacePrefix+"_it_%d_%s", time.Now().UnixNano(), ir03RandomHex(t)) + if !ir03ResourceNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + schema := fmt.Sprintf(ir03NamespacePrefix+"_it_%d_%s", time.Now().UnixNano(), ir03RandomHex(t)) + if !ir03ResourceNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + client, err := control.OpenPool(ctx, control.ClientConfig{DSN: dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 2 * time.Second}) + if err != nil { + fail("open pg-store client (requires a PostgreSQL 18 service)", err) + } + t.Cleanup(client.Close) + if _, err := control.MigrateSchema(ctx, client, schema); err != nil { + fail("migrate owned schema", err) + } + resources, err := control.NewResourceRepository(client, schema) + if err != nil { + fail("build resource repository", err) + } + return &ir03Service{ctx: ctx, admin: admin, iso: iso, client: client, schema: schema, resources: resources, scrub: scrub, fail: fail} +} + +func (s *ir03Service) insertPatient(t *testing.T, project, generation, logicalID, authPath string, payload map[string]any) { + t.Helper() + raw, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + if _, err := s.resources.InsertResource(s.ctx, source.Resource{ + Project: project, DatasetGeneration: generation, ResourceType: "Patient", + LogicalID: logicalID, AuthResourcePath: authPath, Payload: raw, + }); err != nil { + s.fail("insert patient "+logicalID, err) + } +} + +func (s *ir03Service) runCommand(t *testing.T, sqlText string, args []any) ([]string, [][]string) { + t.Helper() + rows, err := s.iso.Query(s.ctx, sqlText, args...) + if err != nil { + s.fail("execute prefix command", err) + } + defer rows.Close() + columns := make([]string, 0) + if desc := rows.FieldDescriptions(); len(desc) > 0 { + for _, field := range desc { + columns = append(columns, string(field.Name)) + } + } + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + s.fail("read prefix command row", err) + } + row := make([]string, len(values)) + for index, value := range values { + row[index] = ir03StringifyCell(value) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + s.fail("finish prefix command", err) + } + return columns, out +} + +func (s *ir03Service) witness(t *testing.T, sql string, args ...any) [][]string { + t.Helper() + rows, err := s.iso.Query(s.ctx, sql, args...) + if err != nil { + s.fail("execute direct-SQL witness", err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + s.fail("read witness row", err) + } + row := make([]string, len(values)) + for index, value := range values { + row[index] = ir03StringifyCell(value) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + s.fail("finish witness", err) + } + return out +} + +func ir03StringifyCell(value any) string { + switch typed := value.(type) { + case nil: + return "" + case string: + return typed + case []byte: + return string(typed) + case pgtype.Numeric: + if !typed.Valid { + return "" + } + text, err := typed.MarshalJSON() + if err != nil { + return fmt.Sprintf("%v", typed) + } + return string(text) + default: + return fmt.Sprintf("%v", typed) + } +} + +func TestCompileClickHousePrefixPostgreSQLExecutesExactRows(t *testing.T) { + service := newIR03Service(t) + bindings := prefixTestBindings(authscope.ReadScopeUnrestricted) + service.insertPatient(t, bindings.Project, bindings.DatasetGeneration, "patient-1", "/programs/p1", map[string]any{"gender": "male"}) + service.insertPatient(t, bindings.Project, bindings.DatasetGeneration, "patient-2", "/programs/p1", map[string]any{"gender": "male"}) + service.insertPatient(t, bindings.Project, bindings.DatasetGeneration, "patient-3", "/programs/p1", map[string]any{"gender": "female"}) + service.insertPatient(t, "other-project", bindings.DatasetGeneration, "patient-9", "/programs/p1", map[string]any{"gender": "male"}) + + composite := prefixTestCompositeWithCombine(t, prefixTestFilterOnlyOutput(), ir.PhysicalPrefixSourcePostgreSQL, bindings, func(stageID string) ir.PhysicalClickHouseCombine { + return prefixTestAppendCombineFor(stageID, "gender_id", "right_gender", "gender", "string", "String") + }) + target, err := compiler.PostgreSQLTarget(mustPostgresBinding(t, service.schema)) + if err != nil { + t.Fatalf("PostgreSQLTarget() error = %v", err) + } + compiledPrefix, err := compiler.CompileClickHousePrefix(target, composite) + if err != nil { + t.Fatalf("CompileClickHousePrefix() error = %v", err) + } + sqlText, parameters, err := compiledPrefix.Command.PostgreSQLParameters() + if err != nil { + t.Fatalf("PostgreSQLParameters() error = %v", err) + } + args := make([]any, len(parameters)) + for index, parameter := range parameters { + args[index] = parameter.Value + } + declared, rows := service.runCommand(t, sqlText, args) + if len(declared) != len(compiledPrefix.OutputColumns) { + t.Fatalf("statement declares %d columns, seam reports %v", len(declared), compiledPrefix.OutputColumns) + } + columnIndex := map[string]int{} + for index, name := range compiledPrefix.OutputColumns { + columnIndex[name] = index + } + rowIDIndex, ok := columnIndex[composite.StageSequence.FinalRowIdentity] + if !ok { + t.Fatalf("row identity column %q is not declared: %v", composite.StageSequence.FinalRowIdentity, declared) + } + var patientIDs, genders []string + seenRowIDs := map[string]bool{} + for _, row := range rows { + patientIDs = append(patientIDs, row[columnIndex["patient_id"]]) + genders = append(genders, row[columnIndex["gender"]]) + if row[rowIDIndex] == "" || row[rowIDIndex] == "" || seenRowIDs[row[rowIDIndex]] { + t.Fatalf("row identity %q is missing or duplicated across rows: %v", row[rowIDIndex], rows) + } + seenRowIDs[row[rowIDIndex]] = true + } + sort.Strings(patientIDs) + sort.Strings(genders) + if strings.Join(patientIDs, ",") != "patient-1,patient-2" { + t.Fatalf("prefix selected patients %v, want patient-1,patient-2 (SQL: %s)", patientIDs, sqlText) + } + if strings.Join(genders, ",") != "male,male" { + t.Fatalf("prefix selected genders %v, want male,male", genders) + } + witness := service.witness(t, + `SELECT logical_id FROM "`+service.schema+`".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' AND payload->>'gender'='male' ORDER BY logical_id`, + bindings.Project, bindings.DatasetGeneration) + if len(witness) != 2 { + t.Fatalf("direct-SQL witness returned %d rows, want 2", len(witness)) + } +} + +func TestMain(m *testing.M) { + code := m.Run() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + os.Exit(code) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fmt.Fprintf(os.Stderr, "zero-leftover audit could not connect: %v\n", err) + os.Exit(1) + } + defer admin.Close() + var databases int + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname LIKE $1", ir03NamespacePrefix+"_%").Scan(&databases); err != nil { + fmt.Fprintf(os.Stderr, "zero-leftover audit failed to count databases: %v\n", err) + os.Exit(1) + } + if databases != 0 { + fmt.Fprintf(os.Stderr, "zero-leftover audit FAILED: %d %s_* databases still exist\n", databases, ir03NamespacePrefix) + code = 1 + } + var schemas int + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_namespace WHERE nspname LIKE $1", ir03NamespacePrefix+"_%").Scan(&schemas); err != nil { + fmt.Fprintf(os.Stderr, "zero-leftover audit failed to count schemas: %v\n", err) + os.Exit(1) + } + if schemas != 0 { + fmt.Fprintf(os.Stderr, "zero-leftover audit FAILED: %d %s_* schemas still exist in the service database\n", schemas, ir03NamespacePrefix) + code = 1 + } + os.Exit(code) +} diff --git a/internal/dataframe/compiler/recipe_execution_test.go b/internal/dataframe/compiler/recipe_execution_test.go new file mode 100644 index 000000000..3bc2a39a7 --- /dev/null +++ b/internal/dataframe/compiler/recipe_execution_test.go @@ -0,0 +1,387 @@ +package compiler_test + +import ( + "encoding/json" + "os" + "reflect" + "strconv" + "strings" + "testing" + + "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" +) + +func prefixTestBindings(mode authscope.ReadScopeMode) recipe.RuntimeBindings { + bindings := recipe.RuntimeBindings{ + Project: "project", DatasetGeneration: "generation", AuthScopeMode: mode, + } + if mode == authscope.ReadScopeRestricted { + bindings.AuthResourcePaths = []string{"/programs/p1"} + bindings.IncludeAuthResourcePath = true + } + return bindings +} + +func prefixTestComposite(t *testing.T, backend ir.PhysicalPrefixSourceBackend, bindings recipe.RuntimeBindings) ir.PhysicalPlan { + return prefixTestCompositeFromOutput(t, prefixTestConstructionOutput(), backend, bindings) +} + +func prefixTestCompositeFromOutput(t *testing.T, output recipe.Output, backend ir.PhysicalPrefixSourceBackend, bindings recipe.RuntimeBindings) ir.PhysicalPlan { + return prefixTestCompositeWithCombine(t, output, backend, bindings, prefixTestAppendCombine) +} + +func prefixTestCompositeWithCombine(t *testing.T, output recipe.Output, backend ir.PhysicalPrefixSourceBackend, bindings recipe.RuntimeBindings, combineFor func(stageID string) ir.PhysicalClickHouseCombine) ir.PhysicalPlan { + t.Helper() + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: "composite-test", + TranslationVersion: "test", Outputs: []recipe.Output{output}, + } + plan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + t.Fatalf("BuildRecipePlan() error = %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, "scope-digest", bindings.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan() error = %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("CompileResolvedRecipePlan() error = %v", err) + } + compiledOutput := compiled.Outputs[0] + combine := combineFor(compiledOutput.Plan.StageSequence.FinalStageID) + composite, err := lower.ComposeClickHouseCombine(compiledOutput.Plan, combine, backend, bindings) + if err != nil { + t.Fatalf("ComposeClickHouseCombine() error = %v", err) + } + return composite +} + +func prefixTestExpectedSchema(composite ir.PhysicalPlan) []string { + columns := make([]string, 0, len(composite.StageSequence.FinalColumns)) + for _, column := range composite.StageSequence.FinalColumns { + columns = append(columns, column.Name) + } + return columns +} + +func TestCompileClickHousePrefixAQLPreservesExactPrefixBytes(t *testing.T) { + composite := prefixTestComposite(t, ir.PhysicalPrefixSourceAQL, prefixTestBindings(authscope.ReadScopeRestricted)) + compiledPrefix, err := compiler.CompileClickHousePrefix(compiler.ArangoTarget(), composite) + if err != nil { + t.Fatalf("CompileClickHousePrefix() error = %v", err) + } + if compiledPrefix.Command.Backend() != command.CompiledBackendAQL { + t.Fatalf("prefix command backend = %q, want AQL", string(compiledPrefix.Command.Backend())) + } + query, binds, err := compiledPrefix.Command.AQLParameters() + if err != nil { + t.Fatalf("AQLParameters() error = %v", err) + } + wantQuery, err := os.ReadFile("testdata/clickhouse_prefix_aql.query") + if err != nil { + t.Fatalf("read golden query: %v", err) + } + if query != string(wantQuery) { + t.Fatalf("AQL prefix bytes drifted from the frozen baseline:\n%s", query) + } + wantBindsJSON, err := os.ReadFile("testdata/clickhouse_prefix_aql.binds.json") + if err != nil { + t.Fatalf("read golden binds: %v", err) + } + var wantBinds map[string]any + if err := json.Unmarshal(wantBindsJSON, &wantBinds); err != nil { + t.Fatalf("decode golden binds: %v", err) + } + gotBindsJSON, err := json.Marshal(binds) + if err != nil { + t.Fatalf("encode binds: %v", err) + } + normalizedWant, err := json.Marshal(wantBinds) + if err != nil { + t.Fatalf("re-encode golden binds: %v", err) + } + if string(gotBindsJSON) != string(normalizedWant) { + t.Fatalf("AQL prefix binds drifted:\n%s", string(gotBindsJSON)) + } + if compiledPrefix.Fingerprint == "" { + t.Fatal("AQL prefix command has no fingerprint") + } + if want := prefixTestExpectedSchema(composite); !reflect.DeepEqual(compiledPrefix.OutputColumns, want) { + t.Fatalf("AQL prefix columns = %v, want the typed final schema %v", compiledPrefix.OutputColumns, want) + } +} + +func TestCompileClickHousePrefixTagAndTargetDispatchIsClosed(t *testing.T) { + unrestricted := prefixTestBindings(authscope.ReadScopeUnrestricted) + postgresTarget, err := compiler.PostgreSQLTarget(mustPostgresBinding(t, "verify_ir03v1_offline")) + if err != nil { + t.Fatalf("PostgreSQLTarget() error = %v", err) + } + cases := []struct { + name string + backend ir.PhysicalPrefixSourceBackend + target compiler.Target + wants string + }{ + {"untagged fails closed", "", compiler.ArangoTarget(), "explicit supported source backend"}, + {"unknown tag fails closed", ir.PhysicalPrefixSourceBackend("CLICKHOUSE"), compiler.ArangoTarget(), "explicit supported source backend"}, + {"AQL tag needs AQL target", ir.PhysicalPrefixSourceAQL, postgresTarget, "requires the AQL compile target"}, + {"PostgreSQL tag needs PostgreSQL target", ir.PhysicalPrefixSourcePostgreSQL, compiler.ArangoTarget(), "requires the PostgreSQL compile target"}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + composite := prefixTestComposite(t, ir.PhysicalPrefixSourceAQL, unrestricted) + composite.ClickHousePrefix = &ir.PhysicalClickHousePrefix{ + SourceBackend: testCase.backend, + StageID: composite.ClickHousePrefix.StageID, + AuthScopeMode: composite.ClickHousePrefix.AuthScopeMode, + AuthResourcePaths: append([]string(nil), composite.ClickHousePrefix.AuthResourcePaths...), + } + if _, err := compiler.CompileClickHousePrefix(testCase.target, composite); err == nil || !strings.Contains(err.Error(), testCase.wants) { + t.Fatalf("error = %v, want %q", err, testCase.wants) + } + }) + } +} + +func TestCompileClickHousePrefixRejectsNonCompositePlans(t *testing.T) { + bindings := prefixTestBindings(authscope.ReadScopeUnrestricted) + plain := prefixTestComposite(t, ir.PhysicalPrefixSourceAQL, bindings) + standalone := ir.ClonePhysicalPlan(plain) + standalone.ClickHousePrefix = nil + if _, err := compiler.CompileClickHousePrefix(compiler.ArangoTarget(), standalone); err == nil || !strings.Contains(err.Error(), "composite ClickHouse plan with a typed prefix is required") { + t.Fatalf("prefix-less plan error = %v", err) + } + aqlOnly := ir.ClonePhysicalPlan(plain) + aqlOnly.Engine = ir.PhysicalEngineAQL + aqlOnly.ClickHouseCombine = nil + aqlOnly.ClickHousePrefix = nil + if _, err := compiler.CompileClickHousePrefix(compiler.ArangoTarget(), aqlOnly); err == nil || !strings.Contains(err.Error(), "composite ClickHouse plan with a typed prefix is required") { + t.Fatalf("AQL-only plan error = %v", err) + } + if _, err := compiler.CompileClickHousePrefix(compiler.Target{}, plain); err != compiler.ErrTargetNotSelected { + t.Fatalf("zero-target error = %v, want ErrTargetNotSelected", err) + } +} + +func TestCompileRecipeOutputRefusesCompositePlans(t *testing.T) { + composite := prefixTestComposite(t, ir.PhysicalPrefixSourceAQL, prefixTestBindings(authscope.ReadScopeRestricted)) + output := lower.CompiledRecipeOutput{Name: "composite", Plan: composite} + _, err := compiler.CompileRecipeOutput(compiler.ArangoTarget(), output, prefixTestBindings(authscope.ReadScopeRestricted), 0, ir.DefaultPhysicalOptimizationPolicy()) + if err == nil || !strings.Contains(err.Error(), "composite ClickHouse plan") { + t.Fatalf("CompileRecipeOutput() composite error = %v", err) + } + optimized := composite + output.OptimizedPlan = &optimized + if _, err := compiler.CompileRecipeOutput(compiler.ArangoTarget(), output, prefixTestBindings(authscope.ReadScopeRestricted), 0, ir.DefaultPhysicalOptimizationPolicy()); err == nil || !strings.Contains(err.Error(), "composite ClickHouse plan") { + t.Fatalf("optimized composite error = %v", err) + } +} + +func TestCompileClickHousePrefixPostgreSQLRendersParameterizedSQL(t *testing.T) { + composite := prefixTestCompositeWithCombine(t, prefixTestFilterOnlyOutput(), ir.PhysicalPrefixSourcePostgreSQL, prefixTestBindings(authscope.ReadScopeUnrestricted), func(stageID string) ir.PhysicalClickHouseCombine { + return prefixTestAppendCombineFor(stageID, "gender_id", "right_gender", "gender", "string", "String") + }) + if composite.ClickHousePrefix.SourceBackend != ir.PhysicalPrefixSourcePostgreSQL { + t.Fatalf("composer did not stamp the PostgreSQL source backend") + } + target, err := compiler.PostgreSQLTarget(mustPostgresBinding(t, "verify_ir03v1_offline")) + if err != nil { + t.Fatalf("PostgreSQLTarget() error = %v", err) + } + compiledPrefix, err := compiler.CompileClickHousePrefix(target, composite) + if err != nil { + t.Fatalf("CompileClickHousePrefix() error = %v", err) + } + if compiledPrefix.Command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("prefix command backend = %q, want POSTGRESQL", string(compiledPrefix.Command.Backend())) + } + sqlText, args, err := compiledPrefix.Command.PostgreSQLParameters() + if err != nil { + t.Fatalf("PostgreSQLParameters() error = %v", err) + } + if !strings.Contains(sqlText, "$1") { + t.Fatalf("PostgreSQL prefix is not parameterized: %s", sqlText) + } + if len(args) == 0 { + t.Fatal("PostgreSQL prefix declares no typed arguments") + } + if strings.Contains(strings.ToUpper(sqlText), "@") { + t.Fatalf("PostgreSQL prefix leaked an AQL named bind: %s", sqlText) + } + if want := prefixTestExpectedSchema(composite); !reflect.DeepEqual(compiledPrefix.OutputColumns, want) { + t.Fatalf("PostgreSQL prefix columns = %v, want the exact typed final schema %v", compiledPrefix.OutputColumns, want) + } + if compiledPrefix.Fingerprint == "" { + t.Fatal("PostgreSQL prefix command has no fingerprint") + } + + broken := ir.ClonePhysicalPlan(composite) + broken.StageSequence.FinalRowIdentity = "__loom_missing_row_id" + if _, err := compiler.CompileClickHousePrefix(target, broken); err == nil || !strings.Contains(err.Error(), "row identity") { + t.Fatalf("missing row identity error = %v", err) + } +} + +func TestCompileClickHousePrefixRefusesDirectlyBuiltRestrictedPostgreSQLPrefix(t *testing.T) { + target, err := compiler.PostgreSQLTarget(mustPostgresBinding(t, "verify_ir03v1_offline")) + if err != nil { + t.Fatalf("PostgreSQLTarget() error = %v", err) + } + base := prefixTestCompositeWithCombine(t, prefixTestFilterOnlyOutput(), ir.PhysicalPrefixSourcePostgreSQL, prefixTestBindings(authscope.ReadScopeUnrestricted), func(stageID string) ir.PhysicalClickHouseCombine { + return prefixTestAppendCombineFor(stageID, "gender_id", "right_gender", "gender", "string", "String") + }) + for _, mode := range []string{"restricted", "Restricted", ""} { + t.Run("mode="+strconv.Quote(mode), func(t *testing.T) { + handBuilt := ir.ClonePhysicalPlan(base) + handBuilt.ClickHousePrefix.AuthScopeMode = mode + handBuilt.ClickHousePrefix.AuthResourcePaths = []string{"/programs/p1"} + handBuilt.ClickHousePrefix.IncludeAuthResourcePath = true + handBuilt.ClickHousePrefix.AuthResourcePathBindKey = "construction_private_auth_resource_path" + handBuilt.BindVars["auth_resource_paths"] = []string{"/programs/p1"} + handBuilt.BindVars["auth_resource_paths_unrestricted"] = false + handBuilt.BindVars["scope_allowed"] = true + handBuilt.BindVars["construction_private_auth_resource_path"] = "/programs/p1" + compiled, err := compiler.CompileClickHousePrefix(target, handBuilt) + if err == nil || !strings.Contains(err.Error(), "unsupported on the PostgreSQL source backend") { + t.Fatalf("hand-built restricted PostgreSQL prefix error = %v, want the restricted refusal", err) + } + if compiled.Fingerprint != "" || compiled.Command.Backend() != "" || len(compiled.OutputColumns) != 0 { + t.Fatalf("hand-built restricted PostgreSQL prefix produced a compiled artifact: backend=%q columns=%v fingerprint=%q", string(compiled.Command.Backend()), compiled.OutputColumns, compiled.Fingerprint) + } + }) + } + if _, err := compiler.CompileClickHousePrefix(target, base); err != nil { + t.Fatalf("composer-built unrestricted PostgreSQL prefix: %v", err) + } +} + +func mustPostgresBinding(t *testing.T, schema string) ir.PostgresStorageBinding { + t.Helper() + binding, err := ir.NewPostgresStorageBinding(schema) + if err != nil { + t.Fatalf("NewPostgresStorageBinding(%q): %v", schema, err) + } + return binding +} + +func prefixTestConstructionOutput() recipe.Output { + groupLabel, categoryLabel, amountLabel := "Group", "Category", "Amount" + integerZero, integerOne := int64(0), int64(1) + stringA, stringB := "a", "b" + filterValue := int64(5) + return recipe.Output{ + Name: "construction_test", RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{ + {Name: "group", ColumnID: "group_id", Label: groupLabel, Expr: recipe.Expression{Select: "gender"}}, + {Name: "category", ColumnID: "category_id", Label: categoryLabel, Expr: recipe.Expression{Select: "multipleBirthInteger"}}, + {Name: "amount", ColumnID: "amount_id", Label: amountLabel, Expr: recipe.Expression{Select: "multipleBirthInteger"}}, + }, + Construction: &recipe.Construction{ + Version: 1, + SourceColumns: []recipe.StageColumn{ + {ID: "group_id", Name: "group", Label: groupLabel}, + {ID: "category_id", Name: "category", Label: categoryLabel}, + {ID: "amount_id", Name: "amount", Label: amountLabel}, + }, + Steps: []recipe.ConstructionStep{ + { + ID: "pivot", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionPivotOp, Pivot: &recipe.ConstructionPivot{ + ConstructionID: "pivot_values", GroupKeyIDs: []string{"group_id"}, CategoryColumnID: "category_id", ValueColumnID: "amount_id", + Categories: []recipe.ConstructionPivotCategory{ + {Key: recipe.TableScalar{Kind: recipe.TableScalarInteger, Integer: &integerZero}, OutputColumnID: "amount_a_id"}, + {Key: recipe.TableScalar{Kind: recipe.TableScalarInteger, Integer: &integerOne}, OutputColumnID: "amount_b_id"}, + }, + DuplicatePolicy: recipe.PivotDuplicateSum, MissingCellPolicy: recipe.PivotMissingCellNull, + UnlistedCategoryPolicy: recipe.PivotUnlistedCategoryError, + }}, + Outputs: []recipe.StageColumn{{ID: "group_id", Name: "group", Label: groupLabel}, {ID: "amount_a_id", Name: "amount_a"}, {ID: "amount_b_id", Name: "amount_b"}}, + }, + { + ID: "derive", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionStepOutputInput, StepID: "pivot"}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionDeriveOp, Derive: &recipe.ConstructionDerive{ + ConstructionID: "derive_total", OutputColumnID: "total_id", Operation: recipe.DerivedAdd, + Left: recipe.ConstructionOperand{Kind: recipe.DerivedColumnOperand, ColumnID: "amount_a_id"}, + Right: recipe.ConstructionOperand{Kind: recipe.DerivedLiteralOperand, Literal: &recipe.DerivedLiteral{Kind: recipe.NumericInteger, Integer: &integerOne}}, + MissingInputPolicy: recipe.MissingInputPropagateNull, + }}, + Outputs: []recipe.StageColumn{{ID: "group_id", Name: "group", Label: groupLabel}, {ID: "amount_a_id", Name: "amount_a"}, {ID: "amount_b_id", Name: "amount_b"}, {ID: "total_id", Name: "total"}}, + }, + { + ID: "filter", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionStepOutputInput, StepID: "derive"}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionFilterOp, Filter: &recipe.ConstructionFilter{ + ColumnID: "total_id", Operator: recipe.FilterGreaterEq, + Values: []recipe.FilterValue{{Kind: recipe.FilterInteger, Integer: &filterValue}}, + }}, + Outputs: []recipe.StageColumn{{ID: "group_id", Name: "group", Label: groupLabel}, {ID: "amount_a_id", Name: "amount_a"}, {ID: "amount_b_id", Name: "amount_b"}, {ID: "total_id", Name: "total"}}, + }, + { + ID: "unpivot", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionStepOutputInput, StepID: "filter"}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionUnpivotOp, Unpivot: &recipe.ConstructionUnpivot{ + ConstructionID: "unpivot_values", Inputs: []recipe.ConstructionUnpivotInput{ + {ColumnID: "amount_a_id", Key: recipe.TableScalar{Kind: recipe.TableScalarString, String: &stringA}}, + {ColumnID: "amount_b_id", Key: recipe.TableScalar{Kind: recipe.TableScalarString, String: &stringB}}, + }, KeyOutputColumnID: "measure_id", ValueOutputColumnID: "amount_id", NullRowPolicy: recipe.UnpivotNullPreserve, + }}, + Outputs: []recipe.StageColumn{{ID: "group_id", Name: "group", Label: groupLabel}, {ID: "total_id", Name: "total"}, {ID: "measure_id", Name: "measure"}, {ID: "amount_id", Name: "amount"}}, + }, + }, + }, + } +} + +func prefixTestFilterOnlyOutput() recipe.Output { + value := "male" + return recipe.Output{ + Name: "filter_prefix", RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{ + {Name: "patient_id", ColumnID: "patient_id", Expr: recipe.Expression{Select: "id"}}, + {Name: "gender", ColumnID: "gender_id", Expr: recipe.Expression{Select: "gender"}}, + }, + Construction: &recipe.Construction{ + Version: 1, + SourceColumns: []recipe.StageColumn{ + {ID: "patient_id", Name: "patient_id", Type: "string"}, + {ID: "gender_id", Name: "gender", Type: "string"}, + }, + Steps: []recipe.ConstructionStep{{ + ID: "source_filter", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionFilterOp, Filter: &recipe.ConstructionFilter{ + ColumnID: "gender_id", Operator: recipe.FilterEquals, + Values: []recipe.FilterValue{{Kind: recipe.FilterString, String: &value}}, + }}, + Outputs: []recipe.StageColumn{ + {ID: "patient_id", Name: "patient_id", Type: "string"}, + {ID: "gender_id", Name: "gender", Type: "string"}, + }, + }}, + }, + } +} + +func prefixTestAppendCombine(stageID string) ir.PhysicalClickHouseCombine { + return prefixTestAppendCombineFor(stageID, "amount_id", "right_amount", "amount", "integer", "Int64") +} + +func prefixTestAppendCombineFor(stageID, inputColumnID, rightColumnID, outputID, logicalType, clickHouseType string) ir.PhysicalClickHouseCombine { + return ir.PhysicalClickHouseCombine{ + Kind: ir.PhysicalCombineAppend, + Inputs: []ir.PhysicalCombineInputRef{ + {PrivateStageID: stageID}, + {TableID: "table-right", RevisionID: "revision-right", OutputID: "right"}, + }, + Projections: []ir.PhysicalCombineProjection{ + {OutputColumnID: outputID, InputIndex: 0, InputColumnID: inputColumnID}, + {OutputColumnID: outputID, InputIndex: 1, InputColumnID: rightColumnID}, + }, + Outputs: []ir.PhysicalCombineOutputColumn{{ID: outputID, Name: outputID, LogicalType: logicalType, ClickHouseType: clickHouseType}}, + } +} diff --git a/internal/dataframe/compiler/recipe_paging.go b/internal/dataframe/compiler/recipe_paging.go index 04fb0568d..b076b9bd1 100644 --- a/internal/dataframe/compiler/recipe_paging.go +++ b/internal/dataframe/compiler/recipe_paging.go @@ -3,6 +3,7 @@ package compiler import ( "fmt" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/compiler/optimize" @@ -18,21 +19,19 @@ const ( rootPageDatasetGenerationBind = "dataset_generation" ) -// CompiledOutputPage contains immutable query templates for bounded root -// execution. RootKeys selects the next root page even when those roots emit no -// output rows. Rows executes the complete output for exactly that page. type CompiledOutputPage struct { - RootKeysQuery string - RootKeysBindVars map[string]any - RowsQuery string - RowsBindVars map[string]any - RowsDiagnostics ir.CompilerPlanDiagnostics + RootKeys command.CompiledCommand + Rows command.CompiledCommand + RowsDiagnostics ir.CompilerPlanDiagnostics } -// CompileRecipeOutputPageWithPolicy builds typed key-discovery and selected- -// root templates. Page binds are compiler-owned and callers may only replace -// their values between executions. -func CompileRecipeOutputPageWithPolicy(output lower.CompiledRecipeOutput, bindings recipe.RuntimeBindings, pageSize int, policy ir.PhysicalOptimizationPolicy) (CompiledOutputPage, error) { +func CompileRecipeOutputPage(target Target, output lower.CompiledRecipeOutput, bindings recipe.RuntimeBindings, pageSize int, policy ir.PhysicalOptimizationPolicy) (CompiledOutputPage, error) { + if err := target.validate(); err != nil { + return CompiledOutputPage{}, err + } + if err := target.rejectPostgreSQL("root-key output paging"); err != nil { + return CompiledOutputPage{}, err + } if workspaceOutputCaptureRequired(output) { return CompiledOutputPage{}, fmt.Errorf("output %q references same-workspace outputs; server-owned workspace capture is not available", output.Name) } @@ -88,9 +87,16 @@ func CompileRecipeOutputPageWithPolicy(output lower.CompiledRecipeOutput, bindin if err != nil { return CompiledOutputPage{}, fmt.Errorf("render selected-root page: %w", err) } + rootKeysCommand, err := command.NewAQLCommand(keys.Query, keys.BindVars) + if err != nil { + return CompiledOutputPage{}, fmt.Errorf("tag root-key page command: %w", err) + } + rowsCommand, err := command.NewAQLCommand(rows.Query, rows.BindVars) + if err != nil { + return CompiledOutputPage{}, fmt.Errorf("tag selected-root page command: %w", err) + } return CompiledOutputPage{ - RootKeysQuery: keys.Query, RootKeysBindVars: keys.BindVars, - RowsQuery: rows.Query, RowsBindVars: rows.BindVars, RowsDiagnostics: physicalPlanDiagnostics(rowsPlan), + RootKeys: rootKeysCommand, Rows: rowsCommand, RowsDiagnostics: physicalPlanDiagnostics(rowsPlan), }, nil } @@ -140,10 +146,6 @@ func rootKeysPagePlan(plan ir.PhysicalPlan, pageSize int) (ir.PhysicalPlan, erro ir.PhysicalOperation{Kind: ir.PhysicalReturnOp, Source: source, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{{Name: "_key", Value: left}}}}, ) out.Operations = append(append([]ir.PhysicalOperation(nil), out.Operations[:insertAt]...), pageOperations...) - // Root-key discovery deliberately stops before the typed construction - // stages. Its RETURN contains only _key, so carrying the final row schema - // through validation would incorrectly require every source-stage column. - // The selected-roots rows plan retains this sequence and executes it fully. out.StageSequence = nil if err := ir.ValidateGenericPhysicalPlanScope(out); err != nil { return ir.PhysicalPlan{}, err @@ -151,13 +153,6 @@ func rootKeysPagePlan(plan ir.PhysicalPlan, pageSize int) (ir.PhysicalPlan, erro return out, nil } -// rootPageConstructionFilters moves only a leading run of FILTER stages over -// source-projection columns into root-key discovery. Their producers and -// filters run after the keyset predicate and before the sort/limit, so a -// root-correlated related set is evaluated only for roots after the cursor. -// Filters after a transforming stage, or filters whose source projection is -// not available directly from the root or one exact root-correlated set, -// remain in the selected-root query. func rootPageConstructionFilters(plan ir.PhysicalPlan) ([]ir.PhysicalOperation, *ir.PhysicalRootPageCandidateSeed) { sequence := plan.StageSequence if sequence == nil || len(sequence.Stages) == 0 || len(plan.Operations) == 0 { @@ -206,8 +201,6 @@ func rootPageConstructionFilters(plan ir.PhysicalPlan) ([]ir.PhysicalOperation, Variable: stage.InputRowVariable, Expression: ir.ClonePhysicalExpression(*projection.Expression), }, }) - // The reduced root query binds the projected value itself, not the - // stage's row object, so the stage column path must be dropped. filter.Predicate.Left = ir.PhysicalValue{Variable: stage.InputRowVariable} } else { filter.Predicate.Left = projection.Value @@ -223,10 +216,6 @@ func rootPageConstructionFilters(plan ir.PhysicalPlan) ([]ir.PhysicalOperation, return operations, candidateSeed } -// rootPageRelatedCandidateSeed builds a candidate-only reverse lookup for a -// positive filter over one directly related FIRST reduction. The root-key -// query retains the original correlated SET and filter, so this subplan may -// overproduce keys but cannot decide row membership. func rootPageRelatedCandidateSeed(plan *ir.PhysicalPlan, stage ir.PhysicalConstructionStage, projection ir.PhysicalProjection, producer *ir.PhysicalOperation, root string) *ir.PhysicalRootPageCandidateSeed { if plan == nil || producer == nil || producer.Set == nil || producer.Set.Projection == nil || producer.Set.Reduction == nil || stage.Filter == nil || stage.Filter.Expression != nil || stage.Filter.Predicate.Operator != "EQUALS" || @@ -482,10 +471,6 @@ func rootPageConstructionFilter(stage ir.PhysicalConstructionStage, projections return *filter, projection, producer, true } -// rootPageProjectionSetProducer finds the single typed set reduction that -// supplies this projection and is correlated directly to the root. Requiring -// the reduction field itself to match prevents a neighboring set or another -// projected value from being pulled into root-key discovery. func rootPageProjectionSetProducer(projection ir.PhysicalProjection, operations []ir.PhysicalOperation, root string) *ir.PhysicalOperation { if projection.Expression != nil || projection.Value.Variable == "" || projection.Value.BindKey != "" || len(projection.Value.Path) != 1 { return nil @@ -635,15 +620,10 @@ func rootPageKeyLookupSafe(plan ir.PhysicalPlan) bool { case ir.PhysicalStageDeriveOp, ir.PhysicalStageFilterOp, ir.PhysicalStageUnpivotOp, ir.PhysicalStageExpandOp, ir.PhysicalStageRelatedSourceOp, ir.PhysicalStageRelatedExpandOp, ir.PhysicalStageRelatedEligibilityOp, ir.PhysicalStageRelatedFieldOp: - // These stages operate independently on each selected root row. case ir.PhysicalStagePivotOp, ir.PhysicalStageGroupOp, ir.PhysicalStageCodedGroupOp, ir.PhysicalStageCohortGroupOp: - // These stages combine rows from different roots, so root-local - // lookup would change their group or category semantics. return false default: - // A new stage kind must prove row-local semantics before using this - // optimization. return false } } @@ -657,27 +637,18 @@ func rootPageKeyLookupSafe(plan ir.PhysicalPlan) bool { case ir.PhysicalTraversalOp, ir.PhysicalFilterOp, ir.PhysicalDerivedLetOp, ir.PhysicalExpressionLetOp, ir.PhysicalSetOp, ir.PhysicalUnnestOp, ir.PhysicalUnpivotOp, ir.PhysicalSortOp, ir.PhysicalLimitOp, ir.PhysicalReturnOp: - // These operations preserve independent execution for each selected - // root, including row-local expansion and projection. default: - // Unknown or specialized operations use the established collection - // filter path until their paging semantics are explicitly verified. return false } } return true } -// rootPageInsertionIndex returns the boundary after scoped root predicates and -// before the first cardinality-changing or navigation operation. func rootPageInsertionIndex(operations []ir.PhysicalOperation) int { - index := 5 // ROOT_SCAN plus the canonical four-operation root scope block. + index := 5 for index < len(operations) { operation := operations[index] if operation.Kind == ir.PhysicalExpressionLetOp { - // A population LET is immediately followed by its root eligibility - // filter. Projection-only LETs are intentionally left out of the - // key-discovery prefix so paging does not evaluate them early. if index+1 >= len(operations) || operations[index+1].Kind != ir.PhysicalFilterOp { break } diff --git a/internal/dataframe/compiler/recipe_paging_related_filter_sequence_test.go b/internal/dataframe/compiler/recipe_paging_related_filter_sequence_test.go index 8d53fb721..d8770eaa1 100644 --- a/internal/dataframe/compiler/recipe_paging_related_filter_sequence_test.go +++ b/internal/dataframe/compiler/recipe_paging_related_filter_sequence_test.go @@ -24,7 +24,7 @@ func TestCompileRecipeOutputPageOrdersMixedRelatedFilterSequences(t *testing.T) pageFilterStep("require_patient", "keep_patient", "patient-id", recipe.FilterExists, "", columns), } page := compileRelatedConstructionPage(t, "root-before-shared-related", output) - seed, rootQuery := splitRootPageCandidateSeed(t, page.RootKeysQuery) + seed, rootQuery := splitRootPageCandidateSeed(t, testAQLText(t, page.RootKeys)) assertPageQueryOrder(t, seed, "child_set_1_node.payload.id == @construction_filter_value_1", "FOR child_set_1_edge IN", @@ -39,8 +39,8 @@ func TestCompileRecipeOutputPageOrdersMixedRelatedFilterSequences(t *testing.T) "SORT root._key ASC", "LIMIT @limit", ) - if count := strings.Count(page.RootKeysQuery, "LET child_set_1 = UNIQUE"); count != 1 { - t.Fatalf("shared related producer count = %d, want 1:\n%s", count, page.RootKeysQuery) + if count := strings.Count(testAQLText(t, page.RootKeys), "LET child_set_1 = UNIQUE"); count != 1 { + t.Fatalf("shared related producer count = %d, want 1:\n%s", count, testAQLText(t, page.RootKeys)) } }) @@ -52,7 +52,7 @@ func TestCompileRecipeOutputPageOrdersMixedRelatedFilterSequences(t *testing.T) pageFilterStep("keep_root", "keep_patient", "root-id", recipe.FilterEquals, rootID, columns), } page := compileRelatedConstructionPage(t, "related-before-root-expression", output) - seed, rootQuery := splitRootPageCandidateSeed(t, page.RootKeysQuery) + seed, rootQuery := splitRootPageCandidateSeed(t, testAQLText(t, page.RootKeys)) assertPageQueryOrder(t, seed, "child_set_1_node.payload.id == @construction_filter_value", "FOR child_set_1_edge IN", @@ -76,7 +76,7 @@ func TestCompileRecipeOutputPageOrdersMixedRelatedFilterSequences(t *testing.T) pageFilterStep("keep_practitioner", "keep_patient", "practitioner-id", recipe.FilterEquals, practitionerID, columns), } page := compileRelatedConstructionPage(t, "independent-related-producers", output) - seed, rootQuery := splitRootPageCandidateSeed(t, page.RootKeysQuery) + seed, rootQuery := splitRootPageCandidateSeed(t, testAQLText(t, page.RootKeys)) assertPageQueryOrder(t, seed, "child_set_1_node.payload.id == @construction_filter_value", "FOR child_set_1_edge IN", @@ -172,7 +172,7 @@ func compileRelatedConstructionPage(t *testing.T, name string, output recipe.Out if err != nil { t.Fatal(err) } - page, err := CompileRecipeOutputPageWithPolicy(compiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } diff --git a/internal/dataframe/compiler/recipe_paging_routed_exists_arango_test.go b/internal/dataframe/compiler/recipe_paging_routed_exists_arango_test.go index eb9bfcf97..5f5307ff4 100644 --- a/internal/dataframe/compiler/recipe_paging_routed_exists_arango_test.go +++ b/internal/dataframe/compiler/recipe_paging_routed_exists_arango_test.go @@ -8,11 +8,11 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -42,13 +42,13 @@ func TestRootPageRoutedExistsKeepsEligibleRootsAcrossCursorAgainstArango(t *test if err != nil { t.Fatalf("lower routed root-page recipe: %v", err) } - page, err := CompileRecipeOutputPageWithPolicy(compiled.Outputs[0], bindings, 1, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled.Outputs[0], bindings, 1, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile routed EXISTS root page: %v", err) } ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Observation"}, {Name: "Patient"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatalf("bootstrap routed root-page fixture collections: %v", err) @@ -130,13 +130,13 @@ func TestRootPageRoutedExistsKeepsEligibleRootsAcrossCursorAgainstArango(t *test } readPage := func(after string) readResult { t.Helper() - keyBinds := make(map[string]any, len(page.RootKeysBindVars)) - for key, value := range page.RootKeysBindVars { + keyBinds := make(map[string]any, len(testAQLBinds(t, page.RootKeys))) + for key, value := range testAQLBinds(t, page.RootKeys) { keyBinds[key] = value } keyBinds[RootPageAfterKeyBind] = after result := readResult{keys: make([]string, 0, 1)} - if err := client.QueryRows(ctx, page.RootKeysQuery, 10, keyBinds, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, page.RootKeys), 10, keyBinds, func(row map[string]any) error { key, ok := row["_key"].(string) if !ok { t.Fatalf("routed EXISTS root-key query returned invalid _key %#v", row["_key"]) @@ -144,7 +144,7 @@ func TestRootPageRoutedExistsKeepsEligibleRootsAcrossCursorAgainstArango(t *test result.keys = append(result.keys, key) return nil }); err != nil { - t.Fatalf("execute routed EXISTS root-key query after %q:\n%s\n%v", after, page.RootKeysQuery, err) + t.Fatalf("execute routed EXISTS root-key query after %q:\n%s\n%v", after, testAQLText(t, page.RootKeys), err) } if len(result.keys) > 1 { t.Fatalf("routed EXISTS root-key page after %q returned %#v, want at most one key", after, result.keys) @@ -153,28 +153,28 @@ func TestRootPageRoutedExistsKeepsEligibleRootsAcrossCursorAgainstArango(t *test return result } - rowBinds := make(map[string]any, len(page.RowsBindVars)+1) - for key, value := range page.RowsBindVars { + rowBinds := make(map[string]any, len(testAQLBinds(t, page.Rows))+1) + for key, value := range testAQLBinds(t, page.Rows) { rowBinds[key] = value } rowBinds[RootPageKeysBind] = result.keys result.rows = make([]map[string]any, 0, 1) - if err := client.QueryRows(ctx, page.RowsQuery, 10, rowBinds, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, page.Rows), 10, rowBinds, func(row map[string]any) error { result.rows = append(result.rows, row) return nil }); err != nil { - t.Fatalf("execute routed EXISTS selected-root query after %q:\n%s\n%v", after, page.RowsQuery, err) + t.Fatalf("execute routed EXISTS selected-root query after %q:\n%s\n%v", after, testAQLText(t, page.Rows), err) } return result } firstPage := readPage("") if !reflect.DeepEqual(firstPage.keys, []string{firstMatchKey}) { - t.Fatalf("first routed EXISTS root-key page = %#v, want only %q\n%s", firstPage.keys, firstMatchKey, page.RootKeysQuery) + t.Fatalf("first routed EXISTS root-key page = %#v, want only %q\n%s", firstPage.keys, firstMatchKey, testAQLText(t, page.RootKeys)) } secondPage := readPage(firstPage.keys[0]) if !reflect.DeepEqual(secondPage.keys, []string{secondMatchKey}) { - t.Fatalf("next routed EXISTS root-key page = %#v, want only %q\n%s", secondPage.keys, secondMatchKey, page.RootKeysQuery) + t.Fatalf("next routed EXISTS root-key page = %#v, want only %q\n%s", secondPage.keys, secondMatchKey, testAQLText(t, page.RootKeys)) } thirdPage := readPage(secondPage.keys[0]) if len(thirdPage.keys) != 0 || len(thirdPage.rows) != 0 { diff --git a/internal/dataframe/compiler/recipe_paging_test.go b/internal/dataframe/compiler/recipe_paging_test.go index 6947712dc..cbe69aeaf 100644 --- a/internal/dataframe/compiler/recipe_paging_test.go +++ b/internal/dataframe/compiler/recipe_paging_test.go @@ -39,30 +39,30 @@ func TestCompileRecipeOutputPageSelectsRootsBeforeExpansion(t *testing.T) { plan.Operations = append(plan.Operations, ir.PhysicalOperation{}) copy(plan.Operations[6:], plan.Operations[5:]) plan.Operations[5] = unnest - page, err := CompileRecipeOutputPageWithPolicy(lower.CompiledRecipeOutput{ + page, err := CompileRecipeOutputPage(ArangoTarget(), lower.CompiledRecipeOutput{ Name: "patients", RootResourceType: "Patient", Plan: plan, }, recipe.RuntimeBindings{Project: "p"}, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if got := page.RootKeysBindVars[RootPageSizeBind]; got != 25 { + if got := testAQLBinds(t, page.RootKeys)[RootPageSizeBind]; got != 25 { t.Fatalf("root page size bind = %#v, want 25", got) } - if strings.Contains(page.RootKeysQuery, "unnest") || strings.Contains(page.RootKeysQuery, "extension") { - t.Fatalf("root-key discovery crossed the expansion boundary:\n%s", page.RootKeysQuery) + if strings.Contains(testAQLText(t, page.RootKeys), "unnest") || strings.Contains(testAQLText(t, page.RootKeys), "extension") { + t.Fatalf("root-key discovery crossed the expansion boundary:\n%s", testAQLText(t, page.RootKeys)) } - keyWindow := strings.Index(page.RowsQuery, "IN SORTED_UNIQUE(@"+RootPageKeysBind+")") - rootLookup := strings.Index(page.RowsQuery, "LET root = DOCUMENT(@@root_collection,") - nullGuard := strings.Index(page.RowsQuery, "FILTER root != null") - unnestIndex := strings.Index(page.RowsQuery, "LET item =") + keyWindow := strings.Index(testAQLText(t, page.Rows), "IN SORTED_UNIQUE(@"+RootPageKeysBind+")") + rootLookup := strings.Index(testAQLText(t, page.Rows), "LET root = DOCUMENT(@@root_collection,") + nullGuard := strings.Index(testAQLText(t, page.Rows), "FILTER root != null") + unnestIndex := strings.Index(testAQLText(t, page.Rows), "LET item =") if keyWindow < 0 || rootLookup < keyWindow || nullGuard < rootLookup || unnestIndex < nullGuard { - t.Fatalf("sorted page-key lookup and missing-root guard must run before UNNEST:\n%s", page.RowsQuery) + t.Fatalf("sorted page-key lookup and missing-root guard must run before UNNEST:\n%s", testAQLText(t, page.Rows)) } - if strings.Contains(page.RowsQuery, "root._key IN @"+RootPageKeysBind) { - t.Fatalf("selected-root rows still scan the root collection and filter its keys:\n%s", page.RowsQuery) + if strings.Contains(testAQLText(t, page.Rows), "root._key IN @"+RootPageKeysBind) { + t.Fatalf("selected-root rows still scan the root collection and filter its keys:\n%s", testAQLText(t, page.Rows)) } - if !strings.Contains(page.RootKeysQuery, "root._key > @"+RootPageAfterKeyBind) { - t.Fatalf("root-key discovery is not a keyset query:\n%s", page.RootKeysQuery) + if !strings.Contains(testAQLText(t, page.RootKeys), "root._key > @"+RootPageAfterKeyBind) { + t.Fatalf("root-key discovery is not a keyset query:\n%s", testAQLText(t, page.RootKeys)) } } @@ -113,33 +113,33 @@ func TestCompileRecipeOutputPageKeepsRelatedStageOutOfRootKeyDiscovery(t *testin if err != nil { t.Fatal(err) } - page, err := CompileRecipeOutputPageWithPolicy(compiled.Outputs[0], bindings, 2, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled.Outputs[0], bindings, 2, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile paged related-source output: %v", err) } - if strings.Contains(page.RootKeysQuery, "observation_status") || strings.Contains(page.RootKeysQuery, "@@related_") { - t.Fatalf("root-key discovery evaluated the related-source stage:\n%s", page.RootKeysQuery) + if strings.Contains(testAQLText(t, page.RootKeys), "observation_status") || strings.Contains(testAQLText(t, page.RootKeys), "@@related_") { + t.Fatalf("root-key discovery evaluated the related-source stage:\n%s", testAQLText(t, page.RootKeys)) } - if !strings.Contains(page.RootKeysQuery, "root._key > @"+RootPageAfterKeyBind) || page.RootKeysBindVars[RootPageSizeBind] != 2 { - t.Fatalf("root-key discovery lost its keyset window: query=%s binds=%#v", page.RootKeysQuery, page.RootKeysBindVars) + if !strings.Contains(testAQLText(t, page.RootKeys), "root._key > @"+RootPageAfterKeyBind) || testAQLBinds(t, page.RootKeys)[RootPageSizeBind] != 2 { + t.Fatalf("root-key discovery lost its keyset window: query=%s binds=%#v", testAQLText(t, page.RootKeys), testAQLBinds(t, page.RootKeys)) } - if !strings.Contains(page.RowsQuery, "IN SORTED_UNIQUE(@"+RootPageKeysBind+")") || - !strings.Contains(page.RowsQuery, "LET root = DOCUMENT(@@root_collection,") || - !strings.Contains(page.RowsQuery, "observation_status") || !strings.Contains(page.RowsQuery, "@@related_0_hop_1_edge_collection") { - t.Fatalf("selected-root page did not retain the related-source stage:\n%s", page.RowsQuery) + if !strings.Contains(testAQLText(t, page.Rows), "IN SORTED_UNIQUE(@"+RootPageKeysBind+")") || + !strings.Contains(testAQLText(t, page.Rows), "LET root = DOCUMENT(@@root_collection,") || + !strings.Contains(testAQLText(t, page.Rows), "observation_status") || !strings.Contains(testAQLText(t, page.Rows), "@@related_0_hop_1_edge_collection") { + t.Fatalf("selected-root page did not retain the related-source stage:\n%s", testAQLText(t, page.Rows)) } } func TestCompileRecipeOutputPageKeepsAggregationOnCollectionFilterPath(t *testing.T) { bindings := recipe.RuntimeBindings{Project: "project-a", DatasetGeneration: "generation-a"} compiled := lowerConstructionOutput(t, constructionOracleOutput(), bindings) - page, err := CompileRecipeOutputPageWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile aggregated output page: %v", err) } - if strings.Contains(page.RowsQuery, "IN SORTED_UNIQUE(@"+RootPageKeysBind+")") || - !strings.Contains(page.RowsQuery, "root._key IN @"+RootPageKeysBind) { - t.Fatalf("aggregated Pivot/Unpivot output must keep the existing root collection filter path:\n%s", page.RowsQuery) + if strings.Contains(testAQLText(t, page.Rows), "IN SORTED_UNIQUE(@"+RootPageKeysBind+")") || + !strings.Contains(testAQLText(t, page.Rows), "root._key IN @"+RootPageKeysBind) { + t.Fatalf("aggregated Pivot/Unpivot output must keep the existing root collection filter path:\n%s", testAQLText(t, page.Rows)) } } @@ -149,36 +149,36 @@ func TestRelatedExpandPreviewPageCapsOnlyTerminalIdentityPartition(t *testing.T) compiled := lowerConstructionOutput(t, output, bindings) bindings.PreviewLimit = 3 - page, err := CompileRecipeOutputPageWithPolicy(compiled, bindings, 10, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled, bindings, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile terminal related-expansion preview page: %v", err) } previewLimitBind := "" - for key, value := range page.RowsBindVars { + for key, value := range testAQLBinds(t, page.Rows) { if strings.HasPrefix(key, "__loom_physical_construction_related_expand_preview_limit") && value == 3 { previewLimitBind = key } } if previewLimitBind == "" { - t.Fatalf("selected-root page has no per-parent preview limit bind: %#v", page.RowsBindVars) + t.Fatalf("selected-root page has no per-parent preview limit bind: %#v", testAQLBinds(t, page.Rows)) } - if !strings.Contains(page.RowsQuery, "SORT TO_STRING([[\"input\",") || !strings.Contains(page.RowsQuery, "[\"related\",") || - !strings.Contains(page.RowsQuery, "LIMIT @"+previewLimitBind) { - t.Fatalf("terminal related candidates were not capped by the exact final row identity:\n%s", page.RowsQuery) + if !strings.Contains(testAQLText(t, page.Rows), "SORT TO_STRING([[\"input\",") || !strings.Contains(testAQLText(t, page.Rows), "[\"related\",") || + !strings.Contains(testAQLText(t, page.Rows), "LIMIT @"+previewLimitBind) { + t.Fatalf("terminal related candidates were not capped by the exact final row identity:\n%s", testAQLText(t, page.Rows)) } - if strings.Contains(page.RowsQuery, "LIMIT @limit") { - t.Fatalf("selected-root rows query gained a global page limit:\n%s", page.RowsQuery) + if strings.Contains(testAQLText(t, page.Rows), "LIMIT @limit") { + t.Fatalf("selected-root rows query gained a global page limit:\n%s", testAQLText(t, page.Rows)) } - if !strings.Contains(page.RowsQuery, "SORTED_UNIQUE(") || !strings.Contains(page.RowsQuery, "SORT __loom_construction_final_row.__loom_row_id ASC") { - t.Fatalf("related deduplication or final identity ordering was removed:\n%s", page.RowsQuery) + if !strings.Contains(testAQLText(t, page.Rows), "SORTED_UNIQUE(") || !strings.Contains(testAQLText(t, page.Rows), "SORT __loom_construction_final_row.__loom_row_id ASC") { + t.Fatalf("related deduplication or final identity ordering was removed:\n%s", testAQLText(t, page.Rows)) } - full, err := CompileRecipeOutputWithPolicy(compiled, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + full, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile full related-expansion output: %v", err) } - if strings.Contains(full.Query, "related_expand_preview_candidate") || strings.Contains(full.Query, "LIMIT @limit") { - t.Fatalf("preview cap leaked into full publication execution:\n%s", full.Query) + if strings.Contains(testAQLText(t, full.Command), "related_expand_preview_candidate") || strings.Contains(testAQLText(t, full.Command), "LIMIT @limit") { + t.Fatalf("preview cap leaked into full publication execution:\n%s", testAQLText(t, full.Command)) } filteredOutput := directRelatedExpandOutput(recipe.ExpansionExclude) @@ -192,12 +192,12 @@ func TestRelatedExpandPreviewPageCapsOnlyTerminalIdentityPartition(t *testing.T) Outputs: append([]recipe.StageColumn(nil), expand.Outputs...), }) filtered := lowerConstructionOutput(t, filteredOutput, recipe.RuntimeBindings{Project: bindings.Project, DatasetGeneration: bindings.DatasetGeneration}) - filteredPage, err := CompileRecipeOutputPageWithPolicy(filtered, bindings, 10, ir.DefaultPhysicalOptimizationPolicy()) + filteredPage, err := CompileRecipeOutputPage(ArangoTarget(), filtered, bindings, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile nonterminal related-expansion page: %v", err) } - if strings.Contains(filteredPage.RowsQuery, "related_expand_preview_limit") || strings.Contains(filteredPage.RowsQuery, "related_expand_preview_candidate") { - t.Fatalf("nonterminal related expansion was capped before its downstream filter:\n%s", filteredPage.RowsQuery) + if strings.Contains(testAQLText(t, filteredPage.Rows), "related_expand_preview_limit") || strings.Contains(testAQLText(t, filteredPage.Rows), "related_expand_preview_candidate") { + t.Fatalf("nonterminal related expansion was capped before its downstream filter:\n%s", testAQLText(t, filteredPage.Rows)) } } @@ -230,11 +230,11 @@ func TestCompileRecipeOutputPageRetainsSinglePopulationComputation(t *testing.T) if err != nil { t.Fatal(err) } - page, err := CompileRecipeOutputPageWithPolicy(compiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - for name, query := range map[string]string{"root keys": page.RootKeysQuery, "rows": page.RowsQuery} { + for name, query := range map[string]string{"root keys": testAQLText(t, page.RootKeys), "rows": testAQLText(t, page.Rows)} { if got := strings.Count(query, "FOR population_member IN @@population_members_collection"); got != 1 { t.Fatalf("%s population member scan count = %d, want 1:\n%s", name, got, query) } @@ -245,10 +245,10 @@ func TestCompileRecipeOutputPageRetainsSinglePopulationComputation(t *testing.T) t.Fatalf("%s ordinary page query materialized population provenance:\n%s", name, query) } } - filterIndex := strings.Index(page.RowsQuery, "root._key IN @"+RootPageKeysBind) - populationIndex := strings.Index(page.RowsQuery, "FOR population_member IN @@population_members_collection") + filterIndex := strings.Index(testAQLText(t, page.Rows), "root._key IN @"+RootPageKeysBind) + populationIndex := strings.Index(testAQLText(t, page.Rows), "FOR population_member IN @@population_members_collection") if populationIndex < 0 || filterIndex < 0 || populationIndex > filterIndex { - t.Fatalf("population eligibility was not evaluated before selected-root paging:\n%s", page.RowsQuery) + t.Fatalf("population eligibility was not evaluated before selected-root paging:\n%s", testAQLText(t, page.Rows)) } } @@ -274,31 +274,31 @@ func TestCompileRecipeOutputPageFiltersConstructionRowsBeforeRootWindow(t *testi if err != nil { t.Fatal(err) } - page, err := CompileRecipeOutputPageWithPolicy(compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } letSuffix := " = root.payload.id" - letValue := strings.Index(page.RootKeysQuery, letSuffix) + letValue := strings.Index(testAQLText(t, page.RootKeys), letSuffix) if letValue < 0 { - t.Fatalf("root-key query lost the source projection for the filtered ID:\n%s", page.RootKeysQuery) + t.Fatalf("root-key query lost the source projection for the filtered ID:\n%s", testAQLText(t, page.RootKeys)) } - letStart := strings.LastIndex(page.RootKeysQuery[:letValue], "\n") + 1 - letVariable := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(page.RootKeysQuery[letStart:letValue]), "LET ")) + letStart := strings.LastIndex(testAQLText(t, page.RootKeys)[:letValue], "\n") + 1 + letVariable := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(testAQLText(t, page.RootKeys)[letStart:letValue]), "LET ")) filterText := "FILTER " + letVariable + " == @construction_filter_value" - filter := strings.Index(page.RootKeysQuery, filterText) - window := strings.Index(page.RootKeysQuery, "SORT root._key ASC") - limit := strings.Index(page.RootKeysQuery, "LIMIT @limit") + filter := strings.Index(testAQLText(t, page.RootKeys), filterText) + window := strings.Index(testAQLText(t, page.RootKeys), "SORT root._key ASC") + limit := strings.Index(testAQLText(t, page.RootKeys), "LIMIT @limit") if letVariable == "" || filter < 0 || window < 0 || limit < 0 || letValue > filter || filter > window || window > limit { - t.Fatalf("construction EQUALS filter must compare the source ID value and run before the keyset page window: let=%q filter=%d sort=%d limit=%d\n%s", letVariable, filter, window, limit, page.RootKeysQuery) + t.Fatalf("construction EQUALS filter must compare the source ID value and run before the keyset page window: let=%q filter=%d sort=%d limit=%d\n%s", letVariable, filter, window, limit, testAQLText(t, page.RootKeys)) } - for name, bindVars := range map[string]map[string]any{"root keys": page.RootKeysBindVars, "selected rows": page.RowsBindVars} { + for name, bindVars := range map[string]map[string]any{"root keys": testAQLBinds(t, page.RootKeys), "selected rows": testAQLBinds(t, page.Rows)} { if got := bindVars["construction_filter_value"]; got != id { t.Fatalf("%s construction filter bind = %#v, want %q", name, got, id) } } - if !strings.Contains(page.RowsQuery, "@construction_filter_value") { - t.Fatalf("selected-root execution lost the construction filter:\n%s", page.RowsQuery) + if !strings.Contains(testAQLText(t, page.Rows), "@construction_filter_value") { + t.Fatalf("selected-root execution lost the construction filter:\n%s", testAQLText(t, page.Rows)) } } @@ -324,17 +324,17 @@ func TestCompileRecipeOutputPageFiltersRelatedConstructionRowsBeforeRootWindow(t if err != nil { t.Fatal(err) } - page, err := CompileRecipeOutputPageWithPolicy(compiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - window := strings.Index(page.RootKeysQuery, "SORT root._key ASC") - limit := strings.Index(page.RootKeysQuery, "LIMIT @limit") - filter := strings.Index(page.RootKeysQuery, "@construction_filter_value") + window := strings.Index(testAQLText(t, page.RootKeys), "SORT root._key ASC") + limit := strings.Index(testAQLText(t, page.RootKeys), "LIMIT @limit") + filter := strings.Index(testAQLText(t, page.RootKeys), "@construction_filter_value") if filter < 0 || window < 0 || limit < 0 || filter > window || window > limit { - t.Fatalf("source-projection Patient.id construction filter must run before the root-key window: filter=%d sort=%d limit=%d\n%s", filter, window, limit, page.RootKeysQuery) + t.Fatalf("source-projection Patient.id construction filter must run before the root-key window: filter=%d sort=%d limit=%d\n%s", filter, window, limit, testAQLText(t, page.RootKeys)) } - if got := page.RootKeysBindVars["construction_filter_value"]; got != patientID { + if got := testAQLBinds(t, page.RootKeys)["construction_filter_value"]; got != patientID { t.Fatalf("root-key related construction filter bind = %#v, want %q", got, patientID) } } @@ -398,19 +398,19 @@ func TestCompileRecipeOutputPageFiltersMissingConstructionRowsBeforeRootWindow(t if err != nil { t.Fatal(err) } - page, err := CompileRecipeOutputPageWithPolicy(compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } rootFilter := "FILTER __loom_construction_input_1 " + tc.comparison - filter := strings.Index(page.RootKeysQuery, rootFilter) - window := strings.Index(page.RootKeysQuery, "SORT root._key ASC") - limit := strings.Index(page.RootKeysQuery, "LIMIT @limit") + filter := strings.Index(testAQLText(t, page.RootKeys), rootFilter) + window := strings.Index(testAQLText(t, page.RootKeys), "SORT root._key ASC") + limit := strings.Index(testAQLText(t, page.RootKeys), "LIMIT @limit") if filter < 0 || window < 0 || limit < 0 || filter > window || window > limit { - t.Fatalf("construction %s filter must run before the root-key page window: filter=%d sort=%d limit=%d\n%s", tc.name, filter, window, limit, page.RootKeysQuery) + t.Fatalf("construction %s filter must run before the root-key page window: filter=%d sort=%d limit=%d\n%s", tc.name, filter, window, limit, testAQLText(t, page.RootKeys)) } - if !strings.Contains(page.RowsQuery, "FILTER __loom_construction_input_1.specimen_id "+tc.comparison) { - t.Fatalf("selected-root execution lost the construction %s filter:\n%s", tc.name, page.RowsQuery) + if !strings.Contains(testAQLText(t, page.Rows), "FILTER __loom_construction_input_1.specimen_id "+tc.comparison) { + t.Fatalf("selected-root execution lost the construction %s filter:\n%s", tc.name, testAQLText(t, page.Rows)) } }) } @@ -443,19 +443,19 @@ func TestCompileRecipeOutputPageDirectRootExistsOrdersPredicateBeforeLimit(t *te t.Fatal(err) } pageSize := 2 - page, err := CompileRecipeOutputPageWithPolicy(compiled.Outputs[0], bindings, pageSize, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled.Outputs[0], bindings, pageSize, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - cursorFilter := strings.Index(page.RootKeysQuery, "FILTER root._key > @"+RootPageAfterKeyBind) - projectedID := strings.Index(page.RootKeysQuery, " = root.payload.id") - existsFilter := strings.Index(page.RootKeysQuery, "FILTER __loom_construction_input_1 != null") - sort := strings.Index(page.RootKeysQuery, "SORT root._key ASC") - limit := strings.Index(page.RootKeysQuery, "LIMIT @"+RootPageSizeBind) + cursorFilter := strings.Index(testAQLText(t, page.RootKeys), "FILTER root._key > @"+RootPageAfterKeyBind) + projectedID := strings.Index(testAQLText(t, page.RootKeys), " = root.payload.id") + existsFilter := strings.Index(testAQLText(t, page.RootKeys), "FILTER __loom_construction_input_1 != null") + sort := strings.Index(testAQLText(t, page.RootKeys), "SORT root._key ASC") + limit := strings.Index(testAQLText(t, page.RootKeys), "LIMIT @"+RootPageSizeBind) if cursorFilter < 0 || projectedID <= cursorFilter || existsFilter <= projectedID || sort <= existsFilter || limit <= sort { - t.Fatalf("compiled direct-root EXISTS query must place its predicate after the cursor and before root-key sorting/limiting: cursor=%d projection=%d exists=%d sort=%d limit=%d\n%s", cursorFilter, projectedID, existsFilter, sort, limit, page.RootKeysQuery) + t.Fatalf("compiled direct-root EXISTS query must place its predicate after the cursor and before root-key sorting/limiting: cursor=%d projection=%d exists=%d sort=%d limit=%d\n%s", cursorFilter, projectedID, existsFilter, sort, limit, testAQLText(t, page.RootKeys)) } - if got := page.RootKeysBindVars[RootPageSizeBind]; got != pageSize { + if got := testAQLBinds(t, page.RootKeys)[RootPageSizeBind]; got != pageSize { t.Fatalf("compiled root page size = %#v, want %d", got, pageSize) } } @@ -520,24 +520,24 @@ func TestCompileRecipeOutputPageRendersEveryValueFilterBeforeRootWindow(t *testi if err != nil { t.Fatal(err) } - page, err := CompileRecipeOutputPageWithPolicy(compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled.Outputs[0], bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - filterIndex := strings.Index(page.RootKeysQuery, "FILTER __loom_construction_input_1 "+tc.comparison) + filterIndex := strings.Index(testAQLText(t, page.RootKeys), "FILTER __loom_construction_input_1 "+tc.comparison) if tc.operator == recipe.FilterContains { - filterIndex = strings.Index(page.RootKeysQuery, "FILTER "+tc.comparison) + filterIndex = strings.Index(testAQLText(t, page.RootKeys), "FILTER "+tc.comparison) } - window := strings.Index(page.RootKeysQuery, "SORT root._key ASC") - limit := strings.Index(page.RootKeysQuery, "LIMIT @limit") + window := strings.Index(testAQLText(t, page.RootKeys), "SORT root._key ASC") + limit := strings.Index(testAQLText(t, page.RootKeys), "LIMIT @limit") if filterIndex < 0 || window < 0 || limit < 0 || filterIndex > window || window > limit { - t.Fatalf("construction %s filter must execute before the root-key window: filter=%d sort=%d limit=%d\n%s", tc.name, filterIndex, window, limit, page.RootKeysQuery) + t.Fatalf("construction %s filter must execute before the root-key window: filter=%d sort=%d limit=%d\n%s", tc.name, filterIndex, window, limit, testAQLText(t, page.RootKeys)) } - if !strings.Contains(page.RowsQuery, "@construction_filter_value") { - t.Fatalf("selected-root execution lost the construction %s predicate:\n%s", tc.name, page.RowsQuery) + if !strings.Contains(testAQLText(t, page.Rows), "@construction_filter_value") { + t.Fatalf("selected-root execution lost the construction %s predicate:\n%s", tc.name, testAQLText(t, page.Rows)) } - for label, bindVars := range map[string]map[string]any{"root keys": page.RootKeysBindVars, "selected rows": page.RowsBindVars} { + for label, bindVars := range map[string]map[string]any{"root keys": testAQLBinds(t, page.RootKeys), "selected rows": testAQLBinds(t, page.Rows)} { if got := bindVars["construction_filter_value"]; !reflect.DeepEqual(got, tc.wantBind) { t.Fatalf("%s %s bind = %#v, want %#v", label, tc.name, got, tc.wantBind) } diff --git a/internal/dataframe/compiler/render/aql/calls.go b/internal/dataframe/compiler/render/aql/calls.go index bcee82eae..343f20407 100644 --- a/internal/dataframe/compiler/render/aql/calls.go +++ b/internal/dataframe/compiler/render/aql/calls.go @@ -106,9 +106,6 @@ func (r *physicalPlanRenderer) renderCall(expression ir.PhysicalExpression) (str if err := require(2); err != nil { return "", err } - // AQL has no JOIN function. Recipe expressions pass the value - // collection first and the delimiter second; CONCAT_SEPARATOR uses the - // inverse argument order and accepts an array as its value operand. return "CONCAT_SEPARATOR(" + args[1] + ", " + args[0] + ")", nil case "cast": if err := require(1); err != nil { @@ -160,9 +157,6 @@ func (r *physicalPlanRenderer) renderCall(expression ir.PhysicalExpression) (str if len(args) < 2 { return "", fmt.Errorf("%s requires a namespace and at least one name", name) } - // AQL cannot reproduce namespace-byte hashing and RFC bit handling - // portably. Return a typed marker for the compiler-owned post-query - // stage instead of emitting a textual hash approximation. return fmt.Sprintf("{\"__loom_exact_uuid_operation\": %q, \"__loom_exact_uuid_args\": [%s]}", name, strings.Join(args, ", ")), nil case "if": if err := require(3); err != nil { @@ -240,12 +234,3 @@ func containsExactUUIDCall(expressions []ir.PhysicalExpression) bool { } return false } - -// renderObject renders a recursively typed object expression. Sorting a copy -// gives equivalent physical plans stable AQL and bind-key allocation without -// changing the semantic field order held by the plan. -// -// The compact dynamic-key literal is used when every field preserves nulls. -// If any field requests OMIT_NULLS, fields are represented as a temporary -// stream and merged so null-valued fields can be removed without evaluating -// their expression twice. diff --git a/internal/dataframe/compiler/render/aql/cell_trace.go b/internal/dataframe/compiler/render/aql/cell_trace.go index 3d95b8182..caa7f228a 100644 --- a/internal/dataframe/compiler/render/aql/cell_trace.go +++ b/internal/dataframe/compiler/render/aql/cell_trace.go @@ -283,8 +283,6 @@ func (r *physicalPlanRenderer) renderReshapeTraceContributors(lineage ir.Physica return "", "", false, "", queryErr } queries = append(queries, query) - // Each output row came from exactly one unpivot input. Do not - // duplicate its evidence once for every configured input column. goto contributorsReady default: return "", "", false, "", fmt.Errorf("unsupported reshape trace source kind %q", source.Kind) diff --git a/internal/dataframe/compiler/render/aql/collections.go b/internal/dataframe/compiler/render/aql/collections.go index 2d6fb4f15..d70a7a7dc 100644 --- a/internal/dataframe/compiler/render/aql/collections.go +++ b/internal/dataframe/compiler/render/aql/collections.go @@ -67,9 +67,6 @@ func (r *physicalPlanRenderer) renderObject(expression ir.PhysicalExpression) (s )`, strings.Join(items, ", ")), nil } -// renderSlice emits a correlated, bounded array projection. Sort and the -// _key tie-break are rendered inside the subquery so representative values -// are deterministic even when traversal order changes. func (r *physicalPlanRenderer) renderSlice(expression ir.PhysicalExpression) (string, error) { slice := expression.Slice if slice == nil { @@ -167,10 +164,6 @@ func slicePreparedVariable(slice *ir.PhysicalSlice) string { return "" } -// renderPivot emits a bounded sparse object keyed by the requested catalog -// columns. Values from all matching resources are combined per key and reduced -// deterministically to the first sorted value while keeping selectors and -// column values typed. func (r *physicalPlanRenderer) renderPivot(expression ir.PhysicalExpression) (string, error) { pivot := expression.Pivot if pivot == nil { @@ -208,12 +201,6 @@ func (r *physicalPlanRenderer) renderPivot(expression ir.PhysicalExpression) (st return "", fmt.Errorf("pivot item source: %w", sourceErr) } itemExpression = r.newInternalVariable("pivot_item_value") - // A selector ending in an iterated array is represented by the - // selector renderer as one subquery row containing that array. Pivot - // item sources need the repeated elements themselves so key/value - // selectors run against each component/backbone item, not against the - // wrapper array. Flatten exactly that one selector-result layer; - // deeper nesting remains owned by the selector's own iteration steps. itemLoop += fmt.Sprintf("\n FOR %s IN FLATTEN(%s)", itemExpression, itemValues) } previousPreparedItem := r.preparedItem @@ -337,10 +324,6 @@ func (r *physicalPlanRenderer) renderCorrelatedPivot(correlation ir.PhysicalCorr codeClause = "" pairKey = "FIRST(@" + columnsBindKey + ")" } - // Reduce repeated Coding aliases within each owner before the outer - // reduction. Distinct owners remain separate, so ALL retains multiplicity - // across components while duplicate Coding entries cannot duplicate a - // component's value. pairs := fmt.Sprintf(`FOR %s IN %s LET __correlation_owner_pairs = ( FOR %s IN %s @@ -388,9 +371,6 @@ func (r *physicalPlanRenderer) renderCorrelatedPivot(correlation ir.PhysicalCorr )`, pairs, value), nil } -// renderPivotSelector evaluates a selector against either the resource -// document or a correlated repeated item. Keeping the item scope explicit -// prevents keys and values from different repeated elements being paired. func (r *physicalPlanRenderer) renderPivotSelector(resourceItem, item string, prepared *ir.PhysicalPreparedReference, selector spec.Selector, itemScoped bool) (string, error) { if prepared != nil { return item + "." + prepared.Field, nil @@ -402,10 +382,6 @@ func (r *physicalPlanRenderer) renderPivotSelector(resourceItem, item string, pr return r.renderSelectorArrayFromSource(source, selector, false, false) } -// renderAggregate emits reductions over either a correlated PhysicalSet or a -// singleton root document. The source is kept typed in the IR; this method is -// the only place that decides the AQL collection expression (`set` versus -// `[root]`). func (r *physicalPlanRenderer) renderAggregate(expression ir.PhysicalExpression) (string, error) { aggregate := expression.Aggregate if aggregate == nil { @@ -499,8 +475,6 @@ func (r *physicalPlanRenderer) renderAggregate(expression ir.PhysicalExpression) return "", fmt.Errorf("unsupported aggregate operation %q", aggregate.Operation) } -// renderAggregateItems is the single predicate/source lowering used by both -// normal aggregate values and targeted cell traces. func (r *physicalPlanRenderer) renderAggregateItems(aggregate *ir.PhysicalAggregate) (string, bool, error) { source, err := r.renderValue(aggregate.Source) if err != nil { @@ -651,9 +625,6 @@ func aggregatePreparedVariable(aggregate *ir.PhysicalAggregate) string { if aggregate.ContributorWindow != nil { return "" } - // Prepared child sets contain raw selector columns. Unit normalization - // needs the original value and its sibling system/code selectors for each - // contributing item, so it must stay at item grain until the reducer. if aggregate.Value != nil && aggregate.Value.Extract != nil && aggregate.Value.Extract.UnitNormalization != nil { return "" } @@ -701,9 +672,6 @@ func (r *physicalPlanRenderer) renderAggregateItemValue(expression ir.PhysicalEx return value, nil } -// renderExtensionCorrelatedPivot keeps each nested extension URL check in the -// loop that owns the next extension object. A leaf URL can therefore never -// match a value beneath a different ancestor URL. func (r *physicalPlanRenderer) renderExtensionCorrelatedPivot(correlation ir.PhysicalCorrelation, columnsBindKey string, stringify, flattenSingle bool, projectionMode string) (string, error) { if columnsBindKey == "" { return "", fmt.Errorf("extension pivot columns bind is required") @@ -761,9 +729,6 @@ func (r *physicalPlanRenderer) renderExtensionCorrelatedPivot(correlation ir.Phy if err != nil { return "", err } - // Aggregate every matched terminal extension before applying the declared - // reduction. Emitting one object per owner would make MERGE overwrite - // repeated matches and would silently turn VALUE multiplicity into FIRST. aggregation := fmt.Sprintf("FOR __extension_result IN [1]\n LET __extension_pairs = (\n %s\n )\n LET __correlation_flat_values = FLATTEN(__extension_pairs[*].values)\n LET __correlation_unsupported_values = FLATTEN(__extension_pairs[*].unsupported)\n FILTER LENGTH(__correlation_flat_values) > 0 OR LENGTH(__correlation_unsupported_values) > 0\n RETURN ", pairs) if flattenSingle { return "FIRST(\n " + aggregation + value + "\n)", nil @@ -771,10 +736,6 @@ func (r *physicalPlanRenderer) renderExtensionCorrelatedPivot(correlation ir.Phy return "MERGE(\n " + aggregation + "{ [FIRST(@" + columnsBindKey + ")]: " + value + " }\n)", nil } -// renderCorrelatedReduction is shared by Coding and Extension correlations so -// every projection mode has one explicit multiplicity and invalid-choice -// contract. ALL/DISTINCT preserve arrays; VALUE reports multiplicity; FIRST is -// deterministic but intentionally lossy. func renderCorrelatedReduction(projectionMode string, stringify bool, primitive string) (string, error) { mode := strings.ToUpper(strings.TrimSpace(projectionMode)) if mode == "" { diff --git a/internal/dataframe/compiler/render/aql/combine_prefix.go b/internal/dataframe/compiler/render/aql/combine_prefix.go index 887589d54..29a0b3398 100644 --- a/internal/dataframe/compiler/render/aql/combine_prefix.go +++ b/internal/dataframe/compiler/render/aql/combine_prefix.go @@ -6,14 +6,13 @@ import ( "github.com/calypr/loom/internal/dataframe/compiler/ir" ) -// RenderClickHousePrefix exposes the typed AQL prefix of a composite physical -// plan as an unbounded stream query. The plan must already have passed the -// composite validator, which binds the AQL sequence's final stage to the -// terminal Combine input and rejects unsupported scope propagation. func RenderClickHousePrefix(plan ir.PhysicalPlan) (RenderedPhysicalPlan, error) { if plan.Engine != ir.PhysicalEngineClickHouse || plan.ClickHousePrefix == nil || plan.ClickHouseCombine == nil { return RenderedPhysicalPlan{}, fmt.Errorf("typed composite ClickHouse plan is required") } + if plan.ClickHousePrefix.SourceBackend != ir.PhysicalPrefixSourceAQL { + return RenderedPhysicalPlan{}, fmt.Errorf("AQL renderer cannot compile a ClickHouse prefix tagged source backend %q", string(plan.ClickHousePrefix.SourceBackend)) + } if err := plan.Validate(); err != nil { return RenderedPhysicalPlan{}, fmt.Errorf("validate composite ClickHouse prefix: %w", err) } diff --git a/internal/dataframe/compiler/render/aql/command_helpers_ext_test.go b/internal/dataframe/compiler/render/aql/command_helpers_ext_test.go new file mode 100644 index 000000000..88d313bc4 --- /dev/null +++ b/internal/dataframe/compiler/render/aql/command_helpers_ext_test.go @@ -0,0 +1,37 @@ +package aql_test + +import ( + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" +) + +func testAQLView(t *testing.T, command command.CompiledCommand) (string, map[string]any) { + t.Helper() + query, bindVars, err := command.AQLParameters() + if err != nil { + t.Fatalf("derive AQL parameters from %s command: %v", command.Backend(), err) + } + return query, bindVars +} + +func testAQLCommand(t *testing.T, query string, bindVars map[string]any) command.CompiledCommand { + t.Helper() + compiledCommand, err := command.NewAQLCommand(query, bindVars) + if err != nil { + t.Fatalf("build AQL command for test: %v", err) + } + return compiledCommand +} + +func testAQLText(t *testing.T, command command.CompiledCommand) string { + t.Helper() + query, _ := testAQLView(t, command) + return query +} + +func testAQLBinds(t *testing.T, command command.CompiledCommand) map[string]any { + t.Helper() + _, bindVars := testAQLView(t, command) + return bindVars +} diff --git a/internal/dataframe/compiler/render/aql/construction_group.go b/internal/dataframe/compiler/render/aql/construction_group.go index 0ee580d26..0aa575ab3 100644 --- a/internal/dataframe/compiler/render/aql/construction_group.go +++ b/internal/dataframe/compiler/render/aql/construction_group.go @@ -38,9 +38,6 @@ func (r *physicalPlanRenderer) renderConstructionGroupStage(stage ir.PhysicalCon } } if group.RootContributorInputColumn != "" { - // Root contributor collection needs the exact grouped rows. Keep the - // streaming COUNT_ROWS shortcut for ordinary Groups, but materialize - // this demanded-only lineage alongside the existing group collection. countRowsOnly = false } aggregateRowValues := countRowsOnly && (input.RowsVariable != "" || input.SourceRowInScope) && constructionGroupRowValuesCanAggregate(group.RowValues) @@ -529,8 +526,6 @@ func (r *physicalPlanRenderer) renderConstructionGroupRelatedOccurrenceInput( NullBehavior: ir.PhysicalEmptyOnNull, Value: &ir.PhysicalValue{Variable: targetVariable, Path: []string{adapter.CodingArrayColumn}}, } - // Preserve one sentinel for both a missing route and an empty Coding - // array, then apply the Coding policy only when a target matched. coding.EmptyPolicy = ir.PhysicalUnnestPreserveParent codingLines, unnestErr := r.renderUnnest(coding, " ", 0, 0) if unnestErr != nil { @@ -700,10 +695,6 @@ func (r *physicalPlanRenderer) renderConstructionGroupSemanticMembership( return lines, memberships, nil } -// renderConstructionGroupSemanticMembershipFromRaw applies the ordinary -// semantic Coding matcher and per-input tuple dedup to one complete raw Coding -// projection. The projection-factorization preview calls the same extractor -// after COLLECT has collapsed identical arrays and retained their weight. func (r *physicalPlanRenderer) renderConstructionGroupSemanticMembershipFromRaw( semantic *ir.PhysicalStageGroupSemanticKey, rawCodings string, @@ -739,9 +730,6 @@ func (r *physicalPlanRenderer) renderConstructionGroupSemanticMembershipFromRaw( fmt.Sprintf(" LET %s = (LENGTH(%s) == 0 ? [{system: null, version: null, code: null, display: null}] : (FOR %s IN %s COLLECT %s = %s AGGREGATE %s = MIN(%s) RETURN {system: (%s != null ? @%s : null), version: (%s != null ? %s : null), code: %s, display: %s}))", local, matched, matchedCoding, matched, code, localCode, display, localDisplay, code, codeSystemBind, code, matchedVersion, code, display), ) - // Local aggregation collapses duplicate semantic tuples within this - // projection. Ordinary Group supplies one source row at a time; factoring - // supplies each distinct projection with its occurrence weight. return lines, local, nil } @@ -807,8 +795,6 @@ func (r *physicalPlanRenderer) renderConstructionGroupAggregate(group *ir.Physic if aggregate.Operation != "SUM" && aggregate.Operation != "MIN" && aggregate.Operation != "MAX" && aggregate.Operation != "MEAN" { return "", fmt.Errorf("unsupported aggregate operation %q", aggregate.Operation) } - // Materialize each group's typed values once, then apply null behavior: - // numeric reducers over no non-null values return null rather than zero. reducer := aggregate.Operation if reducer == "MEAN" { reducer = "SUM" diff --git a/internal/dataframe/compiler/render/aql/construction_related_expand.go b/internal/dataframe/compiler/render/aql/construction_related_expand.go index 8a22759c8..65e163a6d 100644 --- a/internal/dataframe/compiler/render/aql/construction_related_expand.go +++ b/internal/dataframe/compiler/render/aql/construction_related_expand.go @@ -22,9 +22,6 @@ func (r *physicalPlanRenderer) renderConstructionRelatedExpandStage(stage ir.Phy if previewLimitBindKey != "" { rawRecordsVariable := r.newInternalVariable("related_expand_raw_records") candidateVariable := r.newInternalVariable("related_expand_preview_candidate") - // Keeping each parent's first N rows by the final row identity preserves - // the global first N: any discarded row already has N predecessors in - // its own parent partition. lines = append(lines, fmt.Sprintf(" LET %s = %s", rawRecordsVariable, records), fmt.Sprintf(" LET %s = (", related.RelatedRecordsVariable), diff --git a/internal/dataframe/compiler/render/aql/doc.go b/internal/dataframe/compiler/render/aql/doc.go index 3235f0678..7188761f9 100644 --- a/internal/dataframe/compiler/render/aql/doc.go +++ b/internal/dataframe/compiler/render/aql/doc.go @@ -1,3 +1 @@ -// Package aql renders validated physical IR as deterministic, parameterized -// Arango AQL. It does not select FHIR routes or optimizer rules. package aql diff --git a/internal/dataframe/compiler/render/aql/graph_render.go b/internal/dataframe/compiler/render/aql/graph_render.go index 8b13bec39..43903aeee 100644 --- a/internal/dataframe/compiler/render/aql/graph_render.go +++ b/internal/dataframe/compiler/render/aql/graph_render.go @@ -7,10 +7,6 @@ import ( "github.com/calypr/loom/internal/dataframe/compiler/ir" ) -// renderGraphPhysicalPlan is the graph/path counterpart to the dataframe -// navigation layout. It deliberately shares the same root scope operations -// and traversal renderer primitives, but keeps path rows correlated until a -// single final union/dedupe/sort/limit operation. func renderGraphPhysicalPlan(plan ir.PhysicalPlan, collectionKeys map[string]struct{}) (RenderedPhysicalPlan, error) { if len(plan.Operations) < 3 || plan.Operations[0].Kind != ir.PhysicalRootScanOp { return RenderedPhysicalPlan{}, fmt.Errorf("graph renderer requires ROOT_SCAN followed by path operations") @@ -159,8 +155,6 @@ func (r *physicalPlanRenderer) renderGraphReturn(graph ir.PhysicalGraphReturn, r rowSets[index] = fmt.Sprintf("%s[*].s%d", rows, index) } lines := []string{ - // rows[*].sN is one array per root; the outer list adds the route-set - // dimension, so flatten two levels to obtain individual path objects. fmt.Sprintf("LET %s = FLATTEN([%s], 2)", all, strings.Join(rowSets, ", ")), fmt.Sprintf(" FOR %s IN %s", path, all), fmt.Sprintf(" COLLECT __loom_path_identity = %s.__loom_path_identity INTO %s", path, group), diff --git a/internal/dataframe/compiler/render/aql/group_row_value_preview.go b/internal/dataframe/compiler/render/aql/group_row_value_preview.go index 839642d91..94967d6d7 100644 --- a/internal/dataframe/compiler/render/aql/group_row_value_preview.go +++ b/internal/dataframe/compiler/render/aql/group_row_value_preview.go @@ -22,11 +22,6 @@ type groupRowValuePreviewShape struct { keyType string } -// terminalGroupRowValuePreviewWindowFor proves narrow direct-root Group shapes -// whose row values or semantic memberships can be deferred until the first -// ordinary group keys are selected. The frontier scans the same scoped roots, -// groups and orders by the exact direct-root key, and retains every root key -// for those groups. The canonical source and Group then run over those roots. func terminalGroupRowValuePreviewWindowFor( plan, sourcePlan ir.PhysicalPlan, sequence *ir.PhysicalStageSequence, @@ -55,9 +50,6 @@ func terminalGroupRowValuePreviewWindowFor( stage := sequence.Stages[0] group := stage.Group semanticGroup := group != nil && group.SemanticKey != nil - // This frontier intentionally keeps the existing no-hint gate for ordinary - // row-value Groups; only a proven semantic Group may carry a non-forcing - // root hint into its direct-key scan. if options.rootIndexHint != "" && !semanticGroup { return ir.PhysicalPlan{}, groupRowValuePreviewShape{}, false } @@ -116,8 +108,6 @@ func terminalGroupRowValuePreviewWindowFor( return ir.PhysicalPlan{}, groupRowValuePreviewShape{}, false } case ir.PhysicalFilterOp: - // Only the exact project/generation/auth filters in the generic - // root-scope block are safe to reuse in the key frontier. if index >= rootScopeEnd { return ir.PhysicalPlan{}, groupRowValuePreviewShape{}, false } @@ -134,8 +124,6 @@ func terminalGroupRowValuePreviewWindowFor( return ir.PhysicalPlan{}, groupRowValuePreviewShape{}, false } default: - // An UNNEST, traversal, or root window can change which source rows - // contribute to a Group. Those shapes stay on the generic renderer. return ir.PhysicalPlan{}, groupRowValuePreviewShape{}, false } } @@ -223,9 +211,6 @@ func renderGroupRowValuePreviewFrontier( sequence *ir.PhysicalStageSequence, options physicalRenderOptions, ) (groupRowValuePreviewFrontier, error) { - // Root hints are non-forcing. Apply the same hint to the key scan when one - // was selected; the canonical source later resolves the selected keys with - // DOCUMENT, so its root-window scan does not use or require that hint. source, err := renderPhysicalPlanWithOptions(frontierPlan, physicalRenderOptions{ rootIndexHint: options.rootIndexHint, disableRootIndex: options.disableRootIndex, diff --git a/internal/dataframe/compiler/render/aql/group_rows.go b/internal/dataframe/compiler/render/aql/group_rows.go index 3440f437e..22bb26e55 100644 --- a/internal/dataframe/compiler/render/aql/group_rows.go +++ b/internal/dataframe/compiler/render/aql/group_rows.go @@ -314,7 +314,6 @@ func (r *physicalPlanRenderer) renderConstructionCohortGroupStage(stage ir.Physi fmt.Sprintf(" FILTER member.groupId == %s.groupId", definition), fmt.Sprintf(" LET %s = FIRST(FOR resource IN @@%s", source, rows.ResourceCollectionBindKey), fmt.Sprintf(" FILTER resource.id == member.ref.id AND resource.project == @%s AND resource.dataset_generation == member.ref.generation AND resource.resourceType == member.ref.resourceType", rows.ResourceProjectBindKey), - // Prefer a physical resource that survived the prefix when logical IDs repeat. fmt.Sprintf(" SORT (resource._key IN %s) DESC, resource._key ASC", contributors), " RETURN resource)", fmt.Sprintf(" FILTER %s", cohortMemberInputPredicate(source, contributors, cohort.PreserveMissingMembers)), diff --git a/internal/dataframe/compiler/render/aql/navigation_layout.go b/internal/dataframe/compiler/render/aql/navigation_layout.go index b53b640e7..748911ce7 100644 --- a/internal/dataframe/compiler/render/aql/navigation_layout.go +++ b/internal/dataframe/compiler/render/aql/navigation_layout.go @@ -91,17 +91,12 @@ rootPredicates: if keysetSeen { pageExpressionLets = append(pageExpressionLets, operation) } - // Compiler-owned expression LETs may establish a typed value for a - // root predicate. Keep them in the pre-window block so the value is - // computed once per root and before SORT/LIMIT. layout.rootPredicates = append(layout.rootPredicates, operation) index++ case ir.PhysicalSortOp: break rootPredicates case ir.PhysicalSetOp: if !keysetSeen { - // Preserve the ordinary post-window set path for plans without a - // root-page cursor predicate. break rootPredicates } if hasUnusedRootPageSet(pageSets, usedPageSets) { @@ -134,10 +129,6 @@ rootPredicates: } } } - // UNNEST is a cardinality boundary. It is kept after root predicates and - // before the execution window so a row-grain-aware compiler can put a - // database-side LIMIT after expansion. This remains a canonical physical - // operation; the slice only captures the validated renderer layout. for index < last && plan.Operations[index].Kind == ir.PhysicalUnnestOp { if plan.Operations[index].Unnest == nil { return physicalNavigationRenderLayout{}, fmt.Errorf("unnest at operation %d is missing payload", index) @@ -206,7 +197,7 @@ rootPredicates: if operation.Kind != ir.PhysicalTraversalOp { return physicalNavigationRenderLayout{}, fmt.Errorf("generic navigation renderer expected TRAVERSAL at operation %d, got %s", index, operation.Kind) } - const traversalScopeLength = 6 // edge + target project/generation, then auth LET/filter + const traversalScopeLength = 6 if index+traversalScopeLength >= last { return physicalNavigationRenderLayout{}, fmt.Errorf("traversal at operation %d is missing its project/auth scope block", index) } @@ -281,8 +272,6 @@ func validateRootPageRelatedSet(operation ir.PhysicalOperation, rootVariable str return fmt.Errorf("must use one traversal directly from the root") } case ir.PhysicalFilterOp, ir.PhysicalDerivedLetOp, ir.PhysicalExpressionLetOp: - // These retain the child's typed project, generation, authorization, - // and authored filter scopes inside the correlated set. default: return fmt.Errorf("contains unsupported subplan operation %s", operation.Kind) } @@ -473,11 +462,6 @@ func validateGenericNavigationRootLimit(operation ir.PhysicalOperation) error { return nil } -// validateGenericNavigationScopeBlock accepts the exact generation-safe scope -// operations emitted by appendProjectScope, appendDatasetGenerationScope, and -// appendAuthScope. The standalone -// scope verifier is intentionally more flexible; rendering is stricter so it -// can relocate the whole block safely into a LET subquery. func validateGenericNavigationScopeBlock(operations []ir.PhysicalOperation, resourceVariable, edgeVariable, targetVariable string) (string, error) { expectedProjectVariables := []string{resourceVariable} expectedGenerationVariables := []string{resourceVariable} diff --git a/internal/dataframe/compiler/render/aql/navigation_render.go b/internal/dataframe/compiler/render/aql/navigation_render.go index c3a1bdc3b..1bab68a08 100644 --- a/internal/dataframe/compiler/render/aql/navigation_render.go +++ b/internal/dataframe/compiler/render/aql/navigation_render.go @@ -131,16 +131,6 @@ func (r *physicalPlanRenderer) renderUnnestSource(unnest ir.PhysicalUnnest) (str return "(\n " + strings.Join(lines, "\n ") + "\n )", nil } -// renderUnnest lowers the canonical cardinality-changing operation into -// correlated AQL loops. The physical IR deliberately does not contain AQL; -// this is the sole renderer implementation for both recipe-originated and -// generic plans. -// -// A source is evaluated once per parent row and normalized to [] on null. An -// INNER unnest emits no rows for an empty source. OUTER uses a nullable index -// sentinel so it emits exactly one parent row with a null item. Indexed loops -// are used whenever ordinality or OUTER semantics are requested, preserving -// duplicate values and a stable zero-based position. func (r *physicalPlanRenderer) renderUnnest(unnest ir.PhysicalUnnest, indent string, ordinal, depth int) ([]string, error) { source, err := r.renderUnnestSource(unnest) if err != nil { @@ -171,10 +161,6 @@ func (r *physicalPlanRenderer) renderUnnest(unnest ir.PhysicalUnnest, indent str } rawSourceVariable := r.newInternalVariable(fmt.Sprintf("unnest_raw_source_%d", ordinal)) sourceVariable := r.newInternalVariable(fmt.Sprintf("unnest_source_%d", ordinal)) - // Selector extraction preserves one array layer per repeated path. UNNEST - // consumes the resulting collection, so flatten exactly one layer here; - // otherwise a source such as root.member[] becomes [member[]] and emits one - // row per parent rather than one row per member. lines = append(lines, fmt.Sprintf("%sLET %s = %s", baseIndent, rawSourceVariable, source), fmt.Sprintf("%sLET %s = (%s == null ? [] : FLATTEN(%s))", baseIndent, sourceVariable, rawSourceVariable, rawSourceVariable), @@ -239,9 +225,6 @@ func (r *physicalPlanRenderer) renderSet(set ir.PhysicalSet, index int) ([]strin strategy = ir.PhysicalTraversalNative } if strategy == ir.PhysicalTraversalEndpointLookup { - // The endpoint equality is the first edge predicate so Arango can use - // the route's compound endpoint index. The node is materialized only - // after edge scope/type predicates have narrowed the candidate set. lines = append(lines, fmt.Sprintf("%sFOR %s IN @@%s", indent, t.EdgeVariable, t.EdgeCollectionBindKey), fmt.Sprintf("%s FILTER %s.%s == %s._id", indent, t.EdgeVariable, t.EndpointField, parentVariable), @@ -397,11 +380,6 @@ func (r *physicalPlanRenderer) renderPreparedSet(prepared ir.PhysicalPreparedSet } item := r.newInternalVariable("prepared_item") lines := []string{fmt.Sprintf(" LET %s = (", prepared.Variable), fmt.Sprintf(" FOR %s IN %s", item, prepared.SourceSetVariable)} - // Rich consumers may combine a prepared selector with a direct payload - // fallback (slice identity, an unprepared pivot value, or a nested object - // field). Preserve the node-facing fields those consumers already read while - // adding prepared projections; the optimizer can remove this retention only - // after a separate compact-set contract proves it safe. fields := []string{ fmt.Sprintf("_key: %s._key", item), fmt.Sprintf("id: %s.id", item), diff --git a/internal/dataframe/compiler/render/aql/predicates.go b/internal/dataframe/compiler/render/aql/predicates.go index 34123969d..c01122009 100644 --- a/internal/dataframe/compiler/render/aql/predicates.go +++ b/internal/dataframe/compiler/render/aql/predicates.go @@ -57,9 +57,6 @@ func (r *physicalPlanRenderer) renderPredicate(predicate ir.PhysicalPredicate) ( } } -// renderCorrelationPredicate keeps the Coding loop and owner value in one -// lexical scope. A system from Coding[0] can therefore never match a code -// from Coding[1], nor can a value from a different repeated component pass. func (r *physicalPlanRenderer) renderCorrelationPredicate(correlation ir.PhysicalCorrelation) (string, error) { owners, err := r.renderCorrelationOwners(correlation.Source, correlation.OwnerSelector) if err != nil { @@ -140,9 +137,6 @@ func (r *physicalPlanRenderer) renderCorrelationOwners(source ir.PhysicalValue, if err != nil { return "", err } - // A selector whose terminal step iterates an array returns one array - // per source document. Correlation loops over owner items, so flatten - // exactly that selector-result layer before pairing Coding/value data. return "FLATTEN(" + owners + ")", nil } return "[" + raw + "]", nil @@ -177,11 +171,6 @@ func (r *physicalPlanRenderer) renderCorrelationValues(source string, selector s return "FIRST(FOR __correlation_candidate IN [" + strings.Join(values, ", ") + "] FILTER " + eligible + " RETURN __correlation_candidate)", nil } -// renderCorrelationUnsupportedChoiceValues returns values observed on choice -// arms that are not part of the checked binding. Keeping this separate from -// the selected value expression lets projections expose an explicit typed -// incompatibility while predicates reject that row instead of treating it as -// ordinary absence. func (r *physicalPlanRenderer) renderCorrelationUnsupportedChoiceValues(source string, correlation ir.PhysicalCorrelation) (string, error) { if len(correlation.ChoiceSelectors) == 0 || len(correlation.ChoiceArms) == 0 { return "[]", nil @@ -300,9 +289,6 @@ func (r *physicalPlanRenderer) renderPredicateExpression(predicate ir.PhysicalPr } } -// renderExistsSubplan serializes a validated correlated subplan. EXISTS is -// always bounded: relationship matching is a semi-join, never a row-expanding -// traversal, so the renderer appends LIMIT 1 immediately before RETURN. func (r *physicalPlanRenderer) renderExistsSubplan(subplan ir.PhysicalSubplan, indent string) (string, error) { value, err := r.renderSubplan(subplan, indent, true) if err != nil { diff --git a/internal/dataframe/compiler/render/aql/recipe_unpivot_identity_test.go b/internal/dataframe/compiler/render/aql/recipe_unpivot_identity_test.go new file mode 100644 index 000000000..d4166a15a --- /dev/null +++ b/internal/dataframe/compiler/render/aql/recipe_unpivot_identity_test.go @@ -0,0 +1,107 @@ +package aql + +import ( + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" +) + +func TestAQL16ConstructionUnpivotStageSchemaIdentity(t *testing.T) { + score, amount := "score", "amount" + output := recipe.Output{ + Name: "aql16_unpivot_schema", RootResourceType: "Observation", RowGrain: "observation", + Fields: []recipe.Field{ + {Name: "status", ColumnID: "status_id", Expr: recipe.Expression{Select: "root.status"}}, + {Name: "score", ColumnID: "score_id", Expr: recipe.Expression{Select: "root.valueInteger"}}, + {Name: "amount", ColumnID: "amount_id", Expr: recipe.Expression{Select: "root.valueQuantity.value"}}, + }, + Construction: &recipe.Construction{Version: 1, SourceColumns: []recipe.StageColumn{ + {ID: "status_id", Name: "status"}, {ID: "score_id", Name: "score"}, {ID: "amount_id", Name: "amount"}, + }}, + } + output.Construction.Steps = []recipe.ConstructionStep{{ + ID: "unpivot_measures", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionUnpivotOp, Unpivot: &recipe.ConstructionUnpivot{ + ConstructionID: "unpivot_measures_values", + Inputs: []recipe.ConstructionUnpivotInput{ + {ColumnID: "score_id", Key: recipe.TableScalar{Kind: recipe.TableScalarString, String: &score}}, + {ColumnID: "amount_id", Key: recipe.TableScalar{Kind: recipe.TableScalarString, String: &amount}}, + }, + KeyOutputColumnID: "measure_id", ValueOutputColumnID: "measure_value_id", + NullRowPolicy: recipe.UnpivotNullDrop, + }}, + Outputs: []recipe.StageColumn{ + {ID: "status_id", Name: "status"}, {ID: "measure_id", Name: "measure"}, + {ID: "measure_value_id", Name: "measure_value"}, + }, + }} + bindings := recipe.RuntimeBindings{ + Project: "p13-proj", DatasetGeneration: "p13-gen", + AuthResourcePaths: []string{"FHIR/Observation"}, + } + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: output.Name, TranslationVersion: "test", + Outputs: []recipe.Output{output}, + } + if err := bundle.Validate(); err != nil { + t.Fatalf("bundle validate: %v", err) + } + plan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("lower.CompileResolvedRecipePlan: %v", err) + } + if len(compiled.Outputs) != 1 { + t.Fatalf("compiled outputs = %d, want 1", len(compiled.Outputs)) + } + physical := compiled.Outputs[0].Plan + if physical.StageSequence == nil || len(physical.StageSequence.Stages) != 1 { + t.Fatalf("expected one construction stage, got %+v", physical.StageSequence) + } + stage := physical.StageSequence.Stages[0] + if stage.Kind != ir.PhysicalStageUnpivotOp || stage.Unpivot == nil { + t.Fatalf("stage kind = %q, want UNPIVOT payload", string(stage.Kind)) + } + var rootKey *ir.PhysicalStageColumn + identities := 0 + for i := range stage.OutputColumns { + column := &stage.OutputColumns[i] + if column.Identity { + identities++ + if column.Name != stage.RowIdentityColumn { + t.Fatalf("identity column %q is not the named row identity %q: %#v", column.Name, stage.RowIdentityColumn, column) + } + } + if column.Name == "_key" { + rootKey = column + } + } + if stage.RowIdentityColumn != "__loom_row_id" { + t.Fatalf("row identity column = %q, want the generated __loom_row_id", stage.RowIdentityColumn) + } + if identities != 1 { + t.Fatalf("stage schema declares %d identity columns, want exactly the generated row identity: %#v", identities, stage.OutputColumns) + } + if rootKey == nil || !rootKey.Internal || rootKey.Identity { + t.Fatalf("carried root _key must be internal and identity-free: %#v", rootKey) + } + rendered, err := RenderPhysicalPlan(physical) + if err != nil { + t.Fatalf("AQL renderer refused the production-lowered UNPIVOT stage: %v", err) + } + if !strings.Contains(rendered.Query, `"TABLE_UNPIVOT"`) { + t.Fatalf("rendered AQL lacks the TABLE_UNPIVOT identity witness (table_reshape.go:557 posture):\n%s", rendered.Query) + } +} diff --git a/internal/dataframe/compiler/render/aql/related_count.go b/internal/dataframe/compiler/render/aql/related_count.go index a282cc8d2..2d247dfc7 100644 --- a/internal/dataframe/compiler/render/aql/related_count.go +++ b/internal/dataframe/compiler/render/aql/related_count.go @@ -8,9 +8,6 @@ import ( "github.com/calypr/loom/internal/dataframe/compiler/ir" ) -// renderRelatedEligibilityCountRows reverses one exact EXISTS route for -// terminal count and category scans. Every scope predicate is matched against -// the canonical typed plan before this renderer relocates it. func renderRelatedEligibilityCountRows(plan ir.PhysicalPlan, sequence *ir.PhysicalStageSequence, options physicalRenderOptions) (RenderedPhysicalPlan, bool, error) { if stage, sourceProjection, rootVariable, ok := terminalRelatedEligibilityCategoryScanOnly(plan, sequence, options); ok { return renderRelatedEligibilityCategoryScan(plan, sequence, options, stage, sourceProjection, rootVariable) @@ -55,10 +52,6 @@ func renderRelatedEligibilityCountRows(plan ir.PhysicalPlan, sequence *ir.Physic return RenderedPhysicalPlan{Query: query, BindVars: pruneUnusedRuntimeBindVars(renderer.bindVars, query)}, true, nil } -// relatedEligibilityCategoryScanOverflowSentinel is one above the compiler's -// maximum category scan size (currently 256). Reaching it proves overflow; -// returning fewer rows is complete, and returning this many cannot be -// mistaken for a complete category set. const relatedEligibilityCategoryScanOverflowSentinel = 257 type relatedEligibilityReverseFrontier struct { @@ -68,9 +61,6 @@ type relatedEligibilityReverseFrontier struct { traversals []ir.PhysicalTraversal } -// buildRelatedEligibilityReverseFrontier is shared by COUNT_ROWS and the -// category terminal. It only moves an EXISTS filter whose correlated subplan -// passes the same exact endpoint-route and scope proof used by Count. func buildRelatedEligibilityReverseFrontier(plan ir.PhysicalPlan, stage ir.PhysicalConstructionStage, internalPrefix, terminalCollectionPrefix string) (relatedEligibilityReverseFrontier, bool, error) { if stage.Filter == nil || stage.Filter.Expression == nil || stage.Filter.Expression.Kind != ir.PhysicalExistsPredicate || stage.Filter.Expression.Exists == nil { return relatedEligibilityReverseFrontier{}, false, nil @@ -265,8 +255,6 @@ func matchesCategoryScanSourceOperations(operations []ir.PhysicalOperation, root return false } default: - // Any later filter, sort, limit, or row-changing operation could - // affect category membership or its completeness. return false } } @@ -371,10 +359,6 @@ func renderRelatedEligibilityCategoryScan(plan ir.PhysicalPlan, sequence *ir.Phy return RenderedPhysicalPlan{Query: query, BindVars: pruneUnusedRuntimeBindVars(renderer.bindVars, query)}, true, nil } -// RenderRelatedEligibilityCategoryOverflowWitness samples qualifying source -// rows from the same exact route used by the full category scan. The caller -// may conclude overflow when the sample contains more distinct categories -// than the supported limit. func RenderRelatedEligibilityCategoryOverflowWitness(plan ir.PhysicalPlan, columnName, markerColumn string, sampleSize int) (RenderedPhysicalPlan, bool, error) { if sampleSize < 1 { return RenderedPhysicalPlan{}, false, fmt.Errorf("category overflow sample size must be positive") @@ -509,7 +493,7 @@ func relatedCountRoute(plan ir.PhysicalPlan, stage ir.PhysicalConstructionStage, for position < len(subplan.Operations) { operation := subplan.Operations[position] if operation.Kind != ir.PhysicalTraversalOp || operation.Traversal == nil { - return nil, false // Reject contributor predicates and any other extra subplan work. + return nil, false } traversal := *operation.Traversal sourceType, sourceOK := plan.BindVars[traversal.SourceTypeBindKey].(string) diff --git a/internal/dataframe/compiler/render/aql/related_count_category_test.go b/internal/dataframe/compiler/render/aql/related_count_category_test.go index 090b51f88..bbbee3f77 100644 --- a/internal/dataframe/compiler/render/aql/related_count_category_test.go +++ b/internal/dataframe/compiler/render/aql/related_count_category_test.go @@ -63,7 +63,7 @@ func TestRelatedEligibilityCategoryScanUsesScopedReverseFrontier(t *testing.T) { Inputs: []ir.PhysicalValue{{Variable: "root", Path: []string{"payload", "name"}}}, }, } - query, err := compiler.CompileCategoryScanStageWithPolicy(categoryOutput, "eligible_patients", "active", "patient_id", compiler.MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) + query, err := compiler.CompileCategoryScanStage(compiler.ArangoTarget(), categoryOutput, "eligible_patients", "active", "patient_id", compiler.MaxCategoryScanValues, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -77,20 +77,20 @@ func TestRelatedEligibilityCategoryScanUsesScopedReverseFrontier(t *testing.T) { "COLLECT __loom_physical_related_category_group_present = __loom_physical_related_category_category_present, __loom_physical_related_category_group_value = __loom_physical_related_category_category_value", "LIMIT 257", } { - if !strings.Contains(query.Query, want) { - t.Errorf("reverse category query is missing %q:\n%s", want, query.Query) + if !strings.Contains(testAQLText(t, query.Command), want) { + t.Errorf("reverse category query is missing %q:\n%s", want, testAQLText(t, query.Command)) } } - if strings.Contains(query.Query, "LET __loom_construction_source_projection = (") { - t.Fatalf("category scan materialized the source root projection before reversing eligibility:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "LET __loom_construction_source_projection = (") { + t.Fatalf("category scan materialized the source root projection before reversing eligibility:\n%s", testAQLText(t, query.Command)) } - if strings.Contains(query.Query, "unused_source_calculation") { - t.Fatalf("unselected derived LET leaked into the specialized category plan:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "unused_source_calculation") { + t.Fatalf("unselected derived LET leaked into the specialized category plan:\n%s", testAQLText(t, query.Command)) } - if got := query.BindVars["related_eligibility_0_hop_1_source_type"]; got != "Patient" { + if got := testAQLBinds(t, query.Command)["related_eligibility_0_hop_1_source_type"]; got != "Patient" { t.Fatalf("typed source discriminator bind = %#v, want Patient", got) } - if got := query.BindVars["@__loom_related_category_terminal_collection"]; got != "Observation" { + if got := testAQLBinds(t, query.Command)["@__loom_related_category_terminal_collection"]; got != "Observation" { t.Fatalf("terminal collection bind = %#v, want Observation", got) } if query.OverflowWitness == nil || query.Proof.OverflowWitnessFingerprint == "" { @@ -102,11 +102,11 @@ func TestRelatedEligibilityCategoryScanUsesScopedReverseFrontier(t *testing.T) { "LIMIT 1025", "COLLECT present = __loom_witness_present, value = __loom_witness_value", } { - if !strings.Contains(query.OverflowWitness.Query, want) { - t.Errorf("category overflow witness is missing %q:\n%s", want, query.OverflowWitness.Query) + if !strings.Contains(testAQLText(t, query.OverflowWitness.Command), want) { + t.Errorf("category overflow witness is missing %q:\n%s", want, testAQLText(t, query.OverflowWitness.Command)) } } - if strings.Contains(query.OverflowWitness.Query, "unused_source_calculation") { + if strings.Contains(testAQLText(t, query.OverflowWitness.Command), "unused_source_calculation") { t.Fatal("unselected source calculation leaked into the overflow witness") } } diff --git a/internal/dataframe/compiler/render/aql/related_group_preview.go b/internal/dataframe/compiler/render/aql/related_group_preview.go index 95cbf7c87..8183aa06b 100644 --- a/internal/dataframe/compiler/render/aql/related_group_preview.go +++ b/internal/dataframe/compiler/render/aql/related_group_preview.go @@ -37,10 +37,6 @@ type previewGroupIdentityFilter struct { BoundedVariable string } -// relatedGroupPreviewShapeFor accepts only a direct scoped-root expansion -// chain whose first related FHIR ID is the sole COUNT_ROWS Group key. The -// preview can then select complete Group identities through the exact first -// hop and reuse the canonical construction stages for every selected root. func relatedGroupPreviewShapeFor(plan ir.PhysicalPlan, sequence *ir.PhysicalStageSequence, options physicalRenderOptions) (relatedGroupPreviewShape, bool) { if plan.Engine == ir.PhysicalEngineClickHouse || sequence == nil || sequence.PreviewLimitBindKey == "" || sequence.PreviewSourceWindowByRootID || sequence.PreviewTerminalPivotWindow || sequence.CellTraceReturn != nil || @@ -247,10 +243,6 @@ func relatedGroupPreviewExactFirstHop(plan ir.PhysicalPlan, stage ir.PhysicalCon return ir.PhysicalTraversal{}, false } - // Reuse the exact COUNT route proof after converting only the already-verified - // sorted-unique object projection into its terminal _id scalar. The candidate - // scan assumes normal ingestion stores each resourceType in its named vertex - // collection; the reverse root check still verifies the physical root bucket. route := related.RelatedRecords route.Unique = false route.Sort = nil diff --git a/internal/dataframe/compiler/render/aql/render.go b/internal/dataframe/compiler/render/aql/render.go index 4ce1ceff2..c8ea103ef 100644 --- a/internal/dataframe/compiler/render/aql/render.go +++ b/internal/dataframe/compiler/render/aql/render.go @@ -8,14 +8,6 @@ import ( "github.com/calypr/loom/internal/dataframe/compiler/ir" ) -// RenderedPhysicalPlan is an executable AQL representation of a validated -// PhysicalPlan. BindVars is independent of the input plan and uses Arango's -// required "@name" key form for collection bind variables referenced as -// "@@name" in Query. -// -// This renderer covers generic physical navigation and rich expression -// operators emitted by BuildGenericPhysicalPlan. Projection names, including -// nested object field names, remain bind-backed and never become AQL source. type RenderedPhysicalPlan struct { Query string BindVars map[string]any @@ -25,15 +17,10 @@ type RenderedPhysicalPlan struct { var rootIndexHintPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*$`) var previewSourcePathSegmentPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) -// RenderPhysicalPlan renders a validated physical plan to deterministic AQL. -// It keeps data and metadata values out of the generated AQL source. func RenderPhysicalPlan(plan ir.PhysicalPlan) (RenderedPhysicalPlan, error) { return renderPhysicalPlan(plan, "") } -// RenderPhysicalPlanWithRootIndexHint renders one direct root scan with a -// non-forcing persistent-index hint. The hint can improve covering reads, but -// Arango remains free to choose another plan when the index does not help. func RenderPhysicalPlanWithRootIndexHint(plan ir.PhysicalPlan, indexHint string) (RenderedPhysicalPlan, error) { if !rootIndexHintPattern.MatchString(indexHint) { return RenderedPhysicalPlan{}, fmt.Errorf("root index hint %q is not a safe index name", indexHint) @@ -45,10 +32,6 @@ func RenderPhysicalPlanWithRootIndexHint(plan ir.PhysicalPlan, indexHint string) return renderPhysicalPlanWithOptions(plan, physicalRenderOptions{rootIndexHint: indexHint}) } -// RenderPhysicalPlanWithRootIndexDisabled renders a plan with Arango's -// per-FOR full-scan option on its direct root scan. It is reserved for -// compiler-proven preview alternatives; ordinary callers should let Arango -// choose its scan plan. func RenderPhysicalPlanWithRootIndexDisabled(plan ir.PhysicalPlan) (RenderedPhysicalPlan, error) { if len(plan.Operations) == 0 || plan.Operations[0].Kind != ir.PhysicalRootScanOp || plan.Operations[0].RootScan == nil || plan.Operations[0].RootScan.Population != nil { @@ -57,31 +40,19 @@ func RenderPhysicalPlanWithRootIndexDisabled(plan ir.PhysicalPlan) (RenderedPhys return renderPhysicalPlanWithOptions(plan, physicalRenderOptions{disableRootIndex: true}) } -// RenderPhysicalPlanWithSemanticGroupProjectionFactorization renders a narrow -// direct-source semantic Group after identical complete Coding projections -// have been collected with occurrence weights. func RenderPhysicalPlanWithSemanticGroupProjectionFactorization(plan ir.PhysicalPlan, indexName string, disableIndex bool) (RenderedPhysicalPlan, error) { return renderSemanticGroupProjectionPreview(plan, indexName, disableIndex, false) } -// RenderPhysicalPlanWithSemanticGroupProjectionAssessment renders a bounded -// metadata-only sample of complete Coding projections for runtime admission. func RenderPhysicalPlanWithSemanticGroupProjectionAssessment(plan ir.PhysicalPlan, indexName string, disableIndex bool) (RenderedPhysicalPlan, error) { return renderSemanticGroupProjectionPreview(plan, indexName, disableIndex, true) } -// ValidateSemanticGroupProjectionPreviewPlan reports whether plan is within -// the exact direct-source Group shape shared by the factorized and assessment -// renderers. func ValidateSemanticGroupProjectionPreviewPlan(plan ir.PhysicalPlan) error { _, _, _, err := semanticGroupProjectionPreviewShape(plan) return err } -// RenderPhysicalRootScopeCount renders an exact count of a direct source -// plan's canonical project, generation, and authorization scope. It drops -// source projections and is only valid when those are the plan's sole row -// filters. func RenderPhysicalRootScopeCount(plan ir.PhysicalPlan) (RenderedPhysicalPlan, error) { if plan.Engine == ir.PhysicalEngineClickHouse || plan.StageSequence == nil || len(plan.Operations) < 5 { return RenderedPhysicalPlan{}, fmt.Errorf("root scope count requires a staged AQL source plan") @@ -131,7 +102,6 @@ func RenderPhysicalRootScopeCount(plan ir.PhysicalPlan) (RenderedPhysicalPlan, e } lines = append(lines, rendered...) case ir.PhysicalExpressionLetOp: - // Source projection calculations do not affect row membership. case ir.PhysicalReturnOp: if index != len(plan.Operations)-2 || operation.Return == nil { return RenderedPhysicalPlan{}, fmt.Errorf("root scope count has an unexpected return operation") @@ -150,10 +120,6 @@ func RenderPhysicalRootScopeCount(plan ir.PhysicalPlan) (RenderedPhysicalPlan, e return RenderedPhysicalPlan{Query: query, BindVars: pruneUnusedRuntimeBindVars(renderer.bindVars, query)}, nil } -// RenderPhysicalPlanWithRelatedExpandPreviewLimit bounds a terminal -// RELATED_EXPAND's per-parent candidate rows while retaining the complete -// final output ordering. The optimization is ignored unless the plan ends at -// a related expansion whose row identity is the final row identity. func RenderPhysicalPlanWithRelatedExpandPreviewLimit(plan ir.PhysicalPlan, limit int) (RenderedPhysicalPlan, error) { if limit < 1 { return RenderedPhysicalPlan{}, fmt.Errorf("related expansion preview limit must be positive") @@ -161,10 +127,6 @@ func RenderPhysicalPlanWithRelatedExpandPreviewLimit(plan ir.PhysicalPlan, limit return renderPhysicalPlanWithOptions(plan, physicalRenderOptions{relatedExpandPreviewLimit: limit}) } -// RenderPhysicalPlanWithRootPageKeys resolves a sorted, unique page of exact -// root keys by document lookup before running the unchanged scoped plan. -// relatedExpandPreviewLimit is zero when no terminal related-expansion cap is -// required. func RenderPhysicalPlanWithRootPageKeys(plan ir.PhysicalPlan, bindKey string, relatedExpandPreviewLimit int) (RenderedPhysicalPlan, error) { if !previewSourcePathSegmentPattern.MatchString(bindKey) { return RenderedPhysicalPlan{}, fmt.Errorf("root page key bind %q is not a safe bind name", bindKey) @@ -188,10 +150,6 @@ func RenderPhysicalPlanWithRootPageKeys(plan ir.PhysicalPlan, bindKey string, re }) } -// RenderPhysicalPlanWithCategoryScanPresenceMarker keeps the selected value -// projection ordinary and carries its property-presence result in a separate -// internal field. Category scans consume this marker after their source plan -// has finished filtering rows. func RenderPhysicalPlanWithCategoryScanPresenceMarker(plan ir.PhysicalPlan, columnName, markerColumn string) (RenderedPhysicalPlan, error) { if strings.TrimSpace(columnName) == "" || strings.TrimSpace(markerColumn) == "" { return RenderedPhysicalPlan{}, fmt.Errorf("category scan presence requires a selected column and marker") @@ -202,24 +160,14 @@ func RenderPhysicalPlanWithCategoryScanPresenceMarker(plan ir.PhysicalPlan, colu }) } -// RenderPhysicalPlanWithTerminalProjection returns one public column from a -// validated terminal construction stage while preserving every intermediate -// stage projection needed to evaluate the plan. func RenderPhysicalPlanWithTerminalProjection(plan ir.PhysicalPlan, columnName string) (RenderedPhysicalPlan, error) { return renderPhysicalPlanWithTerminalProjection(plan, columnName, false) } -// RenderPhysicalPlanWithUnorderedTerminalProjection returns one public column -// from a validated terminal construction stage without sorting the terminal -// rows by identity. The caller must impose any ordering its result contract -// requires. func RenderPhysicalPlanWithUnorderedTerminalProjection(plan ir.PhysicalPlan, columnName string) (RenderedPhysicalPlan, error) { return renderPhysicalPlanWithTerminalProjection(plan, columnName, true) } -// RenderPhysicalPlanWithUnorderedTerminalProjectionPresenceMarker renders a -// category scan projection while carrying property presence separately from -// its nullable value through construction stages. func RenderPhysicalPlanWithUnorderedTerminalProjectionPresenceMarker(plan ir.PhysicalPlan, columnName, markerColumn string) (RenderedPhysicalPlan, error) { if strings.TrimSpace(markerColumn) == "" { return RenderedPhysicalPlan{}, fmt.Errorf("category scan presence marker is required") @@ -286,9 +234,6 @@ func physicalStageColumnByID(columns []ir.PhysicalStageColumn, id string) (ir.Ph return ir.PhysicalStageColumn{}, false } -// RenderPhysicalPlanWithDynamicCategoryPivotPreview renders a terminal -// nonunique Pivot preview with category specifications supplied as one bind -// array. Full execution continues to use the canonical static renderer. func RenderPhysicalPlanWithDynamicCategoryPivotPreview(plan ir.PhysicalPlan) (RenderedPhysicalPlan, error) { sequence := plan.StageSequence if sequence == nil || !sequence.PreviewTerminalPivotWindow || sequence.PreviewLimitBindKey == "" || @@ -304,18 +249,10 @@ func RenderPhysicalPlanWithDynamicCategoryPivotPreview(plan ir.PhysicalPlan) (Re return renderPhysicalPlanWithOptions(plan, physicalRenderOptions{dynamicPivotPreview: true}) } -// RenderPhysicalPlanWithStreamingMissingCategoryPivotPreview renders a -// direct-root terminal Pivot preview with per-category COLLECT aggregations. -// It avoids retaining every projected source row in a group array while -// preserving the full source scan and exact presence-aware category matches. func RenderPhysicalPlanWithStreamingMissingCategoryPivotPreview(plan ir.PhysicalPlan) (RenderedPhysicalPlan, error) { return renderStreamingMissingCategoryPivotPreview(plan, "", false) } -// RenderPhysicalPlanWithStreamingMissingCategoryPivotPreviewAndRootIndexHint -// applies a non-forcing root index hint and renders the hidden missing -// category marker directly from its stored parent object. Callers must supply -// a compiler-owned stored-values index matching the source projections. func RenderPhysicalPlanWithStreamingMissingCategoryPivotPreviewAndRootIndexHint(plan ir.PhysicalPlan, indexHint string) (RenderedPhysicalPlan, error) { if !rootIndexHintPattern.MatchString(indexHint) { return RenderedPhysicalPlan{}, fmt.Errorf("root index hint %q is not a safe index name", indexHint) @@ -345,10 +282,6 @@ func renderStreamingMissingCategoryPivotPreview(plan ir.PhysicalPlan, indexHint }) } -// RenderPhysicalPlanWithTwoScanPivotPreview renders a terminal Pivot preview -// by selecting complete group tuples before reading their contributing source -// rows. The caller must prove the source projection is a direct, terminal -// Pivot projection before enabling this path. func RenderPhysicalPlanWithTwoScanPivotPreview(plan ir.PhysicalPlan, indexHint string, groupKeySourcePaths [][]string) (RenderedPhysicalPlan, error) { if !rootIndexHintPattern.MatchString(indexHint) { return RenderedPhysicalPlan{}, fmt.Errorf("root index hint %q is not a safe index name", indexHint) @@ -475,10 +408,6 @@ func renderPhysicalPlanWithOptions(plan ir.PhysicalPlan, options physicalRenderO return renderGraphPhysicalPlan(plan, collectionKeys) } } - // This renderer deliberately supports only BuildGenericPhysicalPlan's - // navigation contract. Validate its required project/auth windows again at - // the executable boundary so a manually assembled plan cannot render an - // unscoped resource scan. if err := ir.ValidateGenericPhysicalPlanScope(plan); err != nil { return RenderedPhysicalPlan{}, fmt.Errorf("verify renderable generic physical plan scope: %w", err) } @@ -755,11 +684,6 @@ func (r *physicalPlanRenderer) renderRootScan(root ir.PhysicalRootScan) ([]strin return lines, nil } -// pruneUnusedRuntimeBindVars is required after physical rewrites. Traversal -// sharing can remove a typed edge predicate while retaining its original -// logical bind in the cloned plan. Arango rejects undeclared bind variables, -// so only values referenced by the final rendered AQL may cross the execution -// boundary. func pruneUnusedRuntimeBindVars(bindVars map[string]any, query string) map[string]any { pruned := make(map[string]any, len(bindVars)) for key, value := range bindVars { @@ -856,10 +780,6 @@ func (r *physicalPlanRenderer) renderScopeOperation(operation ir.PhysicalOperati } } -// physicalNavigationRenderLayout is the intentionally narrow executable shape -// produced by BuildGenericPhysicalPlan. Post-window operations retain physical -// plan order because sets, traversals, and expression LETs may depend on values -// introduced by any preceding operation. type physicalNavigationRenderLayout struct { root ir.PhysicalRootScan rootScope []ir.PhysicalOperation diff --git a/internal/dataframe/compiler/render/aql/row_lineage.go b/internal/dataframe/compiler/render/aql/row_lineage.go index ba163d8c5..a21863334 100644 --- a/internal/dataframe/compiler/render/aql/row_lineage.go +++ b/internal/dataframe/compiler/render/aql/row_lineage.go @@ -357,9 +357,6 @@ func (r *physicalPlanRenderer) renderSemanticGroupRowLineage( } semantic := group.SemanticKey lineageSemantic := *semantic - // The direct Group input row carries `_key`, but the row-lineage source - // projection exposes that same root identity under the hidden occurrence - // key column. Reuse the projected identity when rehydrating the resource. lineageSemantic.SourceIdentityColumn = terminal.OccurrenceKeyColumn input := r.newInternalVariable("row_lineage_semantic_group_input") selected := r.newInternalVariable("row_lineage_semantic_selected_group") diff --git a/internal/dataframe/compiler/render/aql/row_lineage_coded_group.go b/internal/dataframe/compiler/render/aql/row_lineage_coded_group.go index 3dbd101c5..0dea58bfe 100644 --- a/internal/dataframe/compiler/render/aql/row_lineage_coded_group.go +++ b/internal/dataframe/compiler/render/aql/row_lineage_coded_group.go @@ -78,9 +78,6 @@ func (r *physicalPlanRenderer) renderCodedGroupRowLineage( return RenderedPhysicalPlan{Query: query, BindVars: pruneUnusedRuntimeBindVars(r.bindVars, query)}, nil } -// codedGroupTupleRows emits one row per source record and exact coding tuple. -// Its COLLECT has no INTO clause, so duplicate Coding entries from one root -// collapse without materializing contributor lists. func (r *physicalPlanRenderer) codedGroupTupleRows( stage ir.PhysicalConstructionStage, sourceQuery string, diff --git a/internal/dataframe/compiler/render/aql/row_lineage_composed.go b/internal/dataframe/compiler/render/aql/row_lineage_composed.go index 595769b9a..3ae9e7f71 100644 --- a/internal/dataframe/compiler/render/aql/row_lineage_composed.go +++ b/internal/dataframe/compiler/render/aql/row_lineage_composed.go @@ -7,9 +7,6 @@ import ( "github.com/calypr/loom/internal/dataframe/compiler/ir" ) -// renderComposedRelatedRowLineage replays the exact compiler-decoded identity -// through the canonical construction stages. It carries one scalar witness per -// authored RELATED_EXPAND stage, never a route or contributor frontier. func (r *physicalPlanRenderer) renderComposedRelatedRowLineage( sourceQuery string, stages []ir.PhysicalConstructionStage, @@ -63,8 +60,6 @@ func (r *physicalPlanRenderer) renderComposedRelatedRowLineage( lines := []string{ "LET " + states + " = (", " FOR " + rootSource + " IN (\n" + indentQuery(sourceQuery, " ") + "\n )", - // The compiler also places this same key at the indexed root scan before - // projection. This equality keeps the private replay fail-closed. " FILTER " + rootSource + "[@" + rootKeyColumnBind + "] == @" + trace.RootKeyBindKey, " RETURN {row: " + rootSource + ", root: {resourceType: @" + rootTypeBind + ", resourceId: " + rootSource + "[@" + rootIDColumnBind + "], occurrenceKey: " + rootSource + "[@" + rootKeyColumnBind + "]}}", ")", @@ -216,9 +211,6 @@ func (r *physicalPlanRenderer) renderComposedRelatedLineageStage( return "", nil, fmt.Errorf("related stage is missing identity variables") } - // The exact terminal predicate follows the complete authorized route. The - // bounded subplan returns at most one terminal witness, even for duplicate - // edge paths to the same canonical target. subplan := related.RelatedRecords subplan.Sort, subplan.Unique, subplan.DistinctBy = nil, false, nil if match.RelatedRowKind == "RELATED" { diff --git a/internal/dataframe/compiler/render/aql/row_lineage_group_pivot.go b/internal/dataframe/compiler/render/aql/row_lineage_group_pivot.go index 8037635ed..5570b32ef 100644 --- a/internal/dataframe/compiler/render/aql/row_lineage_group_pivot.go +++ b/internal/dataframe/compiler/render/aql/row_lineage_group_pivot.go @@ -7,11 +7,6 @@ import ( "github.com/calypr/loom/internal/dataframe/compiler/ir" ) -// renderConstructionCountRowsGroupPivotRowLineage reverses a constrained -// Group(COUNT_ROWS) -> Pivot sequence. Validation streams grouped tuples into -// the existing full-scope Pivot policy checker; the requested row's source -// documents are then streamed directly from the original scoped source. -// Neither pass collects source records into arrays. func (r *physicalPlanRenderer) renderConstructionCountRowsGroupPivotRowLineage( sourceQuery string, groupStage ir.PhysicalConstructionStage, @@ -102,9 +97,6 @@ func (r *physicalPlanRenderer) renderConstructionCountRowsGroupPivotRowLineage( return RenderedPhysicalPlan{Query: query, BindVars: pruneUnusedRuntimeBindVars(r.bindVars, query)}, nil } -// renderConstructionCountRowsGroupedSource reproduces the authored Group's -// scalar-key and missing-key semantics while returning only its key tuple and -// COUNT_ROWS value. WITH COUNT INTO is a scalar count, not a retained row set. func (r *physicalPlanRenderer) renderConstructionCountRowsGroupedSource( sourceQuery string, stage ir.PhysicalConstructionStage, diff --git a/internal/dataframe/compiler/render/aql/row_lineage_group_rows.go b/internal/dataframe/compiler/render/aql/row_lineage_group_rows.go index ca7e29542..6c362885a 100644 --- a/internal/dataframe/compiler/render/aql/row_lineage_group_rows.go +++ b/internal/dataframe/compiler/render/aql/row_lineage_group_rows.go @@ -7,9 +7,6 @@ import ( "github.com/calypr/loom/internal/dataframe/compiler/ir" ) -// RenderPhysicalGroupRowsLineage returns a bounded page of members for one -// exact row from the standalone explicit-group source. Requested identity -// fields are supplied separately after compiler-side canonical JSON parsing. func RenderPhysicalGroupRowsLineage( plan ir.PhysicalPlan, requestedRevisionID, requestedGroupID string, @@ -28,9 +25,6 @@ func RenderPhysicalGroupRowsLineage( return renderPhysicalGroupRowsLineage(plan, *plan.Operations[0].GroupRows, nil, requestedRevisionID, requestedGroupID, offset, limit) } -// RenderPhysicalCohortGroupRowsLineage traces a composed cohort row through its -// identity-preserving scalar FILTER suffix, then pages pinned membership -// occurrences only when that exact final row still exists. func RenderPhysicalCohortGroupRowsLineage( plan ir.PhysicalPlan, groupStage ir.PhysicalConstructionStage, diff --git a/internal/dataframe/compiler/render/aql/row_lineage_pivot.go b/internal/dataframe/compiler/render/aql/row_lineage_pivot.go index 36716e0e7..3ddd13751 100644 --- a/internal/dataframe/compiler/render/aql/row_lineage_pivot.go +++ b/internal/dataframe/compiler/render/aql/row_lineage_pivot.go @@ -7,9 +7,6 @@ import ( "github.com/calypr/loom/internal/dataframe/compiler/ir" ) -// renderConstructionPivotRowLineage validates Pivot policies over the full -// scoped input, then pages the exact typed-key preimage without a contributor -// array. func (r *physicalPlanRenderer) renderConstructionPivotRowLineage( sourceQuery string, stage ir.PhysicalConstructionStage, @@ -110,10 +107,6 @@ type constructionPivotLineageVariables struct { Source string } -// renderConstructionPivotLineageValidation matches grouped-Pivot execution -// semantics across every source group. Group aggregates retain counts only; -// the final scalar aggregate keeps selected-row discovery from constructing a -// result array proportional to the number of Pivot groups. func (r *physicalPlanRenderer) renderConstructionPivotLineageValidation( sourceQuery string, stage ir.PhysicalConstructionStage, diff --git a/internal/dataframe/compiler/render/aql/selector.go b/internal/dataframe/compiler/render/aql/selector.go index 832ddc48e..d4dab8a06 100644 --- a/internal/dataframe/compiler/render/aql/selector.go +++ b/internal/dataframe/compiler/render/aql/selector.go @@ -6,9 +6,6 @@ import ( "github.com/calypr/loom/internal/dataframe/spec" ) -// selectorHasNoArrays and the helpers below are shared by the physical -// renderer and typed-filter compiler. They deliberately render only validated -// Selector values and do not know anything about storage-plan representation. func selectorHasNoArrays(sel spec.Selector) bool { for _, step := range sel.Steps { if step.Iterate || step.Index != nil { diff --git a/internal/dataframe/compiler/render/aql/selectors.go b/internal/dataframe/compiler/render/aql/selectors.go index 89b2828f1..99a4d3aac 100644 --- a/internal/dataframe/compiler/render/aql/selectors.go +++ b/internal/dataframe/compiler/render/aql/selectors.go @@ -38,11 +38,6 @@ func (r *physicalPlanRenderer) renderExtract(expression ir.PhysicalExpression) ( return "", err } setSource := extract.Source.Variable != "" && r.setVariables[extract.Source.Variable] != "" - // A set-valued source is a collection of resource documents. Its - // PhysicalValue may carry the document payload path from semantic lowering, - // but that path cannot be applied to the collection itself (`set.payload` - // is null in AQL). The selector renderer already projects each set item - // through `item.payload`, so pass the collection variable as the loop source. if setSource { source = extract.Source.Variable } @@ -161,10 +156,6 @@ func rebindPhysicalExtractSource(extract *ir.PhysicalExtract, source ir.Physical } } -// renderUnitNormalizedExtract converts each contributing measurement before -// the owning aggregate reduces it. The source system/code selectors are -// matched exactly against compiler-pinned rule identities; display labels are -// deliberately not part of this expression. func (r *physicalPlanRenderer) renderUnitNormalizedExtract(expression ir.PhysicalExpression) (string, error) { extract := expression.Extract if extract == nil || extract.UnitNormalization == nil { diff --git a/internal/dataframe/compiler/render/aql/source_group_preview.go b/internal/dataframe/compiler/render/aql/source_group_preview.go index 069c17e0c..c0e950816 100644 --- a/internal/dataframe/compiler/render/aql/source_group_preview.go +++ b/internal/dataframe/compiler/render/aql/source_group_preview.go @@ -24,11 +24,6 @@ type sourceGroupPreviewFrontier struct { rootKeysVariable string } -// terminalSourceGroupPreviewWindowFor proves the source-Group shape whose -// first preview groups can be selected before expanding related rows. The -// frontier retains the exact root filters and key values, then checks each -// group for at least one scoped route witness. The canonical source and Group -// still compute all selected contributors, including duplicate edges/items. func terminalSourceGroupPreviewWindowFor( plan, sourcePlan ir.PhysicalPlan, sequence *ir.PhysicalStageSequence, diff --git a/internal/dataframe/compiler/render/aql/stage_sequence.go b/internal/dataframe/compiler/render/aql/stage_sequence.go index ff7a4540a..9093bf070 100644 --- a/internal/dataframe/compiler/render/aql/stage_sequence.go +++ b/internal/dataframe/compiler/render/aql/stage_sequence.go @@ -54,8 +54,6 @@ func renderPhysicalStageSequence(plan ir.PhysicalPlan, options physicalRenderOpt plan, sourcePlan, sequence, options, hasRelatedGroupPreview, ) if hasGroupRowValueWindow { - // The root-key frontier must run before per-root row-value projections; - // do not inline this source directly into the Group stage. inlineGroupSourceRow = "" } sourceGroupWindowPlan := ir.PhysicalPlan{} @@ -361,8 +359,6 @@ func renderPhysicalStageSequence(plan ir.PhysicalPlan, options physicalRenderOpt if stage.GroupedPivot == nil { return RenderedPhysicalPlan{}, fmt.Errorf("stage %q is missing grouped pivot payload", stage.ID) } - // The final construction query sorts on row identity after all stages. - // A one-input-per-group pivot needs no intermediate group-key sort. previewLimitBindKey := "" if sequence.PreviewTerminalPivotWindow && index == len(stages)-1 && !options.twoScanPivotPreview { previewLimitBindKey = sequence.PreviewLimitBindKey @@ -533,10 +529,6 @@ func renderPhysicalStageSequence(plan ir.PhysicalPlan, options physicalRenderOpt }, nil } -// groupSourceRowVariable returns the first Group input row variable when a -// terminal COUNT_ROWS Group can consume its source in scope. The generic -// source RETURN is rendered as a LET before the Group COLLECT, so source rows -// are never collected into an intermediate array. func groupSourceRowVariable(sourcePlan ir.PhysicalPlan, sequence *ir.PhysicalStageSequence, stages []ir.PhysicalConstructionStage, options physicalRenderOptions) string { if sequence == nil || len(stages) != 1 || sequence.RowLineageReturn != nil || sequence.CellTraceReturn != nil || sequence.PreviewSourceWindowByRootID || sequence.PreviewTerminalPivotWindow || sequence.OutputAuthResourcePathBindKey != "" || @@ -600,10 +592,6 @@ func groupSourceRowVariable(sourcePlan ir.PhysicalPlan, sequence *ir.PhysicalSta return stage.InputRowVariable } -// codedGroupSourceRootVariable returns the physical root variable only when -// the first construction stage can consume the source query's rows directly. -// The source query still renders every root filter, auth predicate, population -// selection, and execution window; only its terminal projection is omitted. func codedGroupSourceRootVariable(sourcePlan ir.PhysicalPlan, sequence *ir.PhysicalStageSequence, stages []ir.PhysicalConstructionStage, options physicalRenderOptions) string { if sequence == nil || len(stages) == 0 || sequence.RowLineageReturn != nil || sequence.CellTraceReturn != nil || options.terminalProjectionColumn != "" || options.projectionPresenceMarkerColumn != "" || @@ -627,7 +615,6 @@ func codedGroupSourceRootVariable(sourcePlan ir.PhysicalPlan, sequence *ir.Physi } case ir.PhysicalFilterOp, ir.PhysicalDerivedLetOp, ir.PhysicalExpressionLetOp, ir.PhysicalSetOp, ir.PhysicalSortOp, ir.PhysicalLimitOp: - // These source operations preserve one row per selected root. case ir.PhysicalReturnOp: returns++ if operation.Return == nil || index != len(sourcePlan.Operations)-1 { @@ -643,10 +630,6 @@ func codedGroupSourceRootVariable(sourcePlan ir.PhysicalPlan, sequence *ir.Physi return root.Variable } -// codedPivotSourceRootVariable returns the exact direct root scan variable only -// when CODED_PIVOT is the first source stage. The renderer embeds that source -// query into the Pivot subquery and evaluates each root's coded cells before -// returning it, so a full payload array is never materialized between stages. func codedPivotSourceRootVariable(sourcePlan ir.PhysicalPlan, sequence *ir.PhysicalStageSequence, stages []ir.PhysicalConstructionStage, options physicalRenderOptions) string { if sequence == nil || len(stages) == 0 || sequence.RowLineageReturn != nil || sequence.CellTraceReturn != nil || options.terminalProjectionColumn != "" || options.projectionPresenceMarkerColumn != "" || @@ -672,8 +655,6 @@ func codedPivotSourceRootVariable(sourcePlan ir.PhysicalPlan, sequence *ir.Physi } case ir.PhysicalFilterOp, ir.PhysicalDerivedLetOp, ir.PhysicalExpressionLetOp, ir.PhysicalSetOp, ir.PhysicalSortOp, ir.PhysicalLimitOp: - // Source filters, authorization, and row-preserving LETs remain in - // scope while each direct root is pivoted. case ir.PhysicalReturnOp: returns++ if operation.Return == nil || index != len(sourcePlan.Operations)-1 { @@ -689,11 +670,6 @@ func codedPivotSourceRootVariable(sourcePlan ir.PhysicalPlan, sequence *ir.Physi return root.Variable } -// streamingMissingCategoryPivotSourceRootVariable proves the narrow root scan -// shape that can be consumed directly by a streaming terminal Pivot. Every -// row filter remains in the rendered source operations; this helper only -// decides whether the source RETURN can be omitted and its projections read -// directly from the root document. func streamingMissingCategoryPivotSourceRootVariable(sourcePlan ir.PhysicalPlan, sequence *ir.PhysicalStageSequence, stages []ir.PhysicalConstructionStage, options physicalRenderOptions) string { if sequence == nil || !options.streamingMissingCategoryPivotPreview || len(stages) != 1 || sequence.RowLineageReturn != nil || sequence.CellTraceReturn != nil || sequence.PopulationMappingReturn != nil || @@ -905,9 +881,6 @@ func streamingPivotInputsAreSourceRows(pivot ir.PhysicalGroupedPivot, inputRowVa return true } -// rebindStreamingPivotSourceProjections replaces the Pivot's row passthrough -// selectors with the matching direct source selectors. The stage slice is a -// shallow copy, so clone its Pivot and projection slice before changing it. func rebindStreamingPivotSourceProjections(stages []ir.PhysicalConstructionStage, sourcePlan ir.PhysicalPlan) error { if len(stages) != 1 || stages[0].GroupedPivot == nil { return fmt.Errorf("streaming missing-category Pivot source rebinding requires one Pivot stage") @@ -990,10 +963,6 @@ func pruneUnusedSourceGroupProjections(plan *ir.PhysicalPlan, sequence *ir.Physi pruneUnusedSourceGroupOperations(plan) } -// pruneUnusedSourceGroupOperations removes root-row LETs and sets that cannot -// affect a source Group's retained projection, filters, or execution window. -// It only handles the row-preserving root-scan shape; plans with any other -// top-level operation keep their original source query. func pruneUnusedSourceGroupOperations(plan *ir.PhysicalPlan) { if plan == nil || len(plan.Operations) < 2 || plan.Operations[0].Kind != ir.PhysicalRootScanOp || plan.Operations[0].RootScan == nil || @@ -1041,10 +1010,6 @@ func pruneUnusedSourceGroupOperations(plan *ir.PhysicalPlan) { if operation.Return == nil || index != len(plan.Operations)-1 { return } - // Source-group liveness follows typed references from every retained - // projection below. Restricting this to direct root selectors keeps - // unrelated optional graph sets alive when a retained projection is - // a Pivot or another typed expression. default: return } @@ -1250,8 +1215,6 @@ func terminalKeylessCountRowsOnly(sequence *ir.PhysicalStageSequence) bool { constructionGroupCountsOnlyRows(stage.Group) } -// pruneUnusedRelatedOutputsForCountRows keeps row-changing stages intact and -// drops only related values that later supported stages cannot read. func pruneUnusedRelatedOutputsForCountRows(sequence *ir.PhysicalStageSequence) []ir.PhysicalConstructionStage { stages := append([]ir.PhysicalConstructionStage(nil), sequence.Stages...) if sequence.CellTraceReturn != nil || len(stages) < 2 { diff --git a/internal/dataframe/compiler/render/aql/table_reshape_dynamic.go b/internal/dataframe/compiler/render/aql/table_reshape_dynamic.go index 74cb9312e..388a75219 100644 --- a/internal/dataframe/compiler/render/aql/table_reshape_dynamic.go +++ b/internal/dataframe/compiler/render/aql/table_reshape_dynamic.go @@ -6,9 +6,6 @@ import ( "github.com/calypr/loom/internal/dataframe/compiler/ir" ) -// renderGroupedTablePivotDynamicPreview keeps category-dependent work in one -// bound category loop. It is used only by explicitly selected terminal -// previews; publication and full execution retain the static renderer. func (r *physicalPlanRenderer) renderGroupedTablePivotDynamicPreview( pivot ir.PhysicalGroupedPivot, lines []string, @@ -114,9 +111,6 @@ func (r *physicalPlanRenderer) renderGroupedTablePivotDynamicPreview( unlistedVariable, itemVariable, pivot.GroupRowsVariable, listed, )) if pivot.UnlistedCategoryPolicy == "ERROR" { - // Keep the category loop as a dependency of the unlisted assertion. - // This preserves duplicate and missing-cell error precedence even if - // AQL moves calculations while optimizing the query. lines = append(lines, fmt.Sprintf( ` FILTER (IS_OBJECT(%s) ? ASSERT(%s == 0, "TABLE_PIVOT_UNLISTED_CATEGORY") : false)`, categoryCellsVariable, unlistedVariable, diff --git a/internal/dataframe/compiler/render/aql/table_reshape_streaming.go b/internal/dataframe/compiler/render/aql/table_reshape_streaming.go index 1b08e4190..d69d21cff 100644 --- a/internal/dataframe/compiler/render/aql/table_reshape_streaming.go +++ b/internal/dataframe/compiler/render/aql/table_reshape_streaming.go @@ -365,8 +365,6 @@ func (r *physicalPlanRenderer) renderGroupedTablePivotStreamingMissingPreview( ) reducer := pivot.DuplicatePolicy if reducer == "ERROR" { - // Once the full-group cardinality assertion passes, MAX returns the - // sole value (or NULL) without retaining a per-group value array. reducer = "MAX" } if reducer != "SUM" && reducer != "MIN" && reducer != "MAX" { diff --git a/internal/dataframe/compiler/render/aql/table_reshape_streaming_scalar_test.go b/internal/dataframe/compiler/render/aql/table_reshape_streaming_scalar_test.go index e2952697d..4136365d0 100644 --- a/internal/dataframe/compiler/render/aql/table_reshape_streaming_scalar_test.go +++ b/internal/dataframe/compiler/render/aql/table_reshape_streaming_scalar_test.go @@ -60,9 +60,6 @@ func TestStreamingMissingCategoryPivotUsesDirectScalarInputs(t *testing.T) { } query, renderer := render("SUM", "ERROR", "EXCLUDE_WITH_EVIDENCE") - // Reusing FIRST([root.payload.valueQuantity]) as a LET preserves the existing direct - // selector when that parent is missing, null, or a scalar: the field read - // remains the same unguarded AQL property access on its single result. for _, fragment := range []string{ "LET __loom_physical_reshape_stream_source_parent = FIRST([root.payload.valueQuantity])", "LET __loom_physical_reshape_stream_input_0 = __loom_physical_reshape_stream_source_parent.group_key", diff --git a/internal/dataframe/compiler/render/aql/values.go b/internal/dataframe/compiler/render/aql/values.go index 34ebb8fa6..3f43562a7 100644 --- a/internal/dataframe/compiler/render/aql/values.go +++ b/internal/dataframe/compiler/render/aql/values.go @@ -71,8 +71,6 @@ func (r *physicalPlanRenderer) renderRelatedField(expression ir.PhysicalExpressi if err != nil { return "", fmt.Errorf("related field document ID: %w", err) } - // The FOR variable is scoped to this expression's subquery, so a stable - // compiler-owned name is safe across projections and construction stages. document := "related_field_document" collection := "[DOCUMENT(" + documentID + ")]" field := document + ".payload" @@ -94,9 +92,6 @@ func (r *physicalPlanRenderer) renderRelatedField(expression ir.PhysicalExpressi if err != nil { return "", fmt.Errorf("related field presence: %w", err) } - // The value projection returns null when there is no visible terminal - // document. Keep that row present in the category domain; only an - // existing, exactly scoped terminal can contribute MISSING. rows := fmt.Sprintf("(FOR %s IN %s FILTER %s != null AND %s.project == @project AND %s.dataset_generation == @dataset_generation AND %s.resourceType == @%s AND (@auth_resource_paths_unrestricted == true OR %s.auth_resource_path IN @auth_resource_paths) RETURN %s)", document, collection, document, document, document, document, resourceTypeBind, document, present) return "FIRST(APPEND(" + rows + ", [true]))", nil @@ -156,10 +151,6 @@ func (r *physicalPlanRenderer) renderKeyedMap(expression ir.PhysicalExpression) if len(valueExpressions) > 1 { valueExpression = "FIRST(FOR __loom_keyed_candidate IN [" + strings.Join(valueExpressions, ", ") + "] FILTER __loom_keyed_candidate != null RETURN __loom_keyed_candidate)" } - // Selector extraction returns a subquery whose rows can themselves be - // arrays. Flatten that result directly so a keyed family iterates source - // items. Wrapping the subquery in another array leaves one array layer - // behind and makes item key/value selectors read from the array itself. sourceLoop := source if keyed.FlattenSource { sourceLoop = "FLATTEN(" + source + ")" diff --git a/internal/dataframe/compiler/render/clickhouse/combine.go b/internal/dataframe/compiler/render/clickhouse/combine.go index 4853727bb..3d9f7ed91 100644 --- a/internal/dataframe/compiler/render/clickhouse/combine.go +++ b/internal/dataframe/compiler/render/clickhouse/combine.go @@ -18,23 +18,14 @@ type RenderedCombine struct { Columns []string } -// RenderCombine turns a typed compiler plan and its exact, authorized table -// resolutions into one ClickHouse SELECT. It never accepts SQL fragments from -// recipe input and never consults a moving publication pointer. func RenderCombine(plan ir.PhysicalClickHouseCombine, inputs []ir.ResolvedClickHouseTable, project string) (RenderedCombine, error) { return RenderCombineWithLimit(plan, inputs, project, 0) } -// RenderCombineWithLimit applies a bounded preview limit to the typed -// ClickHouse plan. A zero limit streams the complete result. func RenderCombineWithLimit(plan ir.PhysicalClickHouseCombine, inputs []ir.ResolvedClickHouseTable, project string, limit int) (RenderedCombine, error) { return renderCombineWithResolvedInputs(plan, inputs, project, limit, nil, false) } -// RenderCompositeClickHouseCombine renders a validated terminal Combine that -// consumes one exact private AQL-stage artifact plus exact published inputs. -// expected must come from the compiled AQL stream identity used to create the -// artifact; the complete manifest identity is checked before SQL is emitted. func RenderCompositeClickHouseCombine(plan ir.PhysicalClickHouseCombine, prefix ir.PhysicalClickHousePrefix, published []ir.ResolvedClickHouseTable, artifact ir.ResolvedClickHousePrivateArtifact, expected ir.ClickHouseArtifactIdentity, project string, limit int) (RenderedCombine, error) { if limit < 0 { return RenderedCombine{}, fmt.Errorf("ClickHouse combine limit cannot be negative") @@ -60,7 +51,7 @@ func RenderCompositeClickHouseCombine(plan ir.PhysicalClickHouseCombine, prefix continue } if privateRefFound || ref.PrivateStageID != prefix.StageID { - return RenderedCombine{}, fmt.Errorf("private ClickHouse Combine input does not match the exact AQL prefix stage") + return RenderedCombine{}, fmt.Errorf("private ClickHouse Combine input does not match the exact prefix stage") } privateRefFound = true } diff --git a/internal/dataframe/compiler/render/clickhouse/combine_test.go b/internal/dataframe/compiler/render/clickhouse/combine_test.go index 50411fa77..e469fbeb2 100644 --- a/internal/dataframe/compiler/render/clickhouse/combine_test.go +++ b/internal/dataframe/compiler/render/clickhouse/combine_test.go @@ -403,7 +403,8 @@ func TestRenderCompositeCombineConsumesExactPrivatePrefixArtifact(t *testing.T) Outputs: []ir.PhysicalCombineOutputColumn{{ID: "status", Name: "status", LogicalType: "string", ClickHouseType: "String"}}, } prefix := ir.PhysicalClickHousePrefix{ - StageID: "derive_status", AuthScopeMode: "restricted", AuthResourcePaths: []string{"/programs/p1"}, + SourceBackend: ir.PhysicalPrefixSourceAQL, + StageID: "derive_status", AuthScopeMode: "restricted", AuthResourcePaths: []string{"/programs/p1"}, IncludeAuthResourcePath: true, AuthResourcePathBindKey: "construction_private_auth_resource_path", } expected := ir.ClickHouseArtifactIdentity{ diff --git a/internal/dataframe/compiler/render/clickhouse/membership_nullable_integration_test.go b/internal/dataframe/compiler/render/clickhouse/membership_nullable_integration_test.go index d472ccea5..9a5bd1ed6 100644 --- a/internal/dataframe/compiler/render/clickhouse/membership_nullable_integration_test.go +++ b/internal/dataframe/compiler/render/clickhouse/membership_nullable_integration_test.go @@ -21,10 +21,6 @@ import ( "github.com/google/uuid" ) -// Run with LOOM_CLICKHOUSE_URL and optional LOOM_CLICKHOUSE_DATABASE/ -// LOOM_CLICKHOUSE_USERNAME/LOOM_CLICKHOUSE_PASSWORD, or with -// LOOM_CLICKHOUSE_CONTAINER to use clickhouse-client in an owned container. -// The ordinary unit suite stays hermetic. func TestRenderNullableCompositeMembershipClickHouseSemantics(t *testing.T) { client := newNullableMembershipClickHouse(t) defer func() { @@ -78,7 +74,7 @@ func TestRenderNullableCompositeMembershipClickHouseSemantics(t *testing.T) { } if err := client.InsertRows(ctx, rightTable, columns, []map[string]any{ {"__loom_row_id": "a", "k1": "alpha", "k2": "one"}, - {"__loom_row_id": "b", "k1": "alpha", "k2": "one"}, // duplicate match must not multiply the left row + {"__loom_row_id": "b", "k1": "alpha", "k2": "one"}, {"__loom_row_id": "c", "k1": nil, "k2": "two"}, {"__loom_row_id": "d", "k1": "beta", "k2": nil}, {"__loom_row_id": "e", "k1": nil, "k2": nil}, diff --git a/internal/dataframe/compiler/render/clickhouse/workspace_combine.go b/internal/dataframe/compiler/render/clickhouse/workspace_combine.go index e647bac06..091e3091f 100644 --- a/internal/dataframe/compiler/render/clickhouse/workspace_combine.go +++ b/internal/dataframe/compiler/render/clickhouse/workspace_combine.go @@ -7,18 +7,12 @@ import ( "github.com/calypr/loom/internal/dataframe/compiler/ir" ) -// WorkspaceArtifact is one exact compiler-resolved same-bundle output capture. -// ExpectedIdentity is derived from the source OutputStream and compiler schema, -// not from manifest values supplied by the caller. type WorkspaceArtifact struct { OutputID string ExpectedIdentity ir.ClickHouseArtifactIdentity Artifact ir.ResolvedClickHousePrivateArtifact } -// RenderWorkspaceClickHouseCombine resolves a terminal workspace Combine in -// authored input order. Private captures are keyed by stable output ID; -// published tables are supplied only for the exact published refs in plan. func RenderWorkspaceClickHouseCombine( plan ir.PhysicalClickHouseCombine, workspace []WorkspaceArtifact, @@ -149,8 +143,6 @@ func validateWorkspaceResolvedInputs(plan ir.PhysicalClickHouseCombine, inputs [ } scopeMode := input.ScopeMode if scopeMode == "" && ref.WorkspaceOutputID == "" { - // Older pinned TABLE_REVISION records predate explicit artifact - // scope modes and always retained row-level authorization paths. scopeMode = ir.ClickHouseArtifactScopeRows } switch scopeMode { diff --git a/internal/dataframe/compiler/render/postgres/calls.go b/internal/dataframe/compiler/render/postgres/calls.go new file mode 100644 index 000000000..285137796 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/calls.go @@ -0,0 +1,890 @@ +package postgres + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const maxCallExpressionDepth = 64 + +const maxCallFragmentBytes = 1 << 20 + +func checkCallFragment(fragment string) error { + if len(fragment) > maxCallFragmentBytes { + return fmt.Errorf("%w: call fragment of %d bytes exceeds the %d-byte registered bound (operand replication under nesting)", + ErrUnsupportedPostgresPlan, len(fragment), maxCallFragmentBytes) + } + return nil +} + +var registeredCallNames = map[string]bool{ + "add": true, "subtract": true, "multiply": true, "divide": true, + "cast": true, "coalesce": true, "fallback": true, + "if": true, "case": true, "not": true, "and": true, "or": true, + "eq": true, "neq": true, "gt": true, "gte": true, "lt": true, "lte": true, + "assert": true, "distinct_present_one": true, +} + +type callLiteralKind uint8 + +const ( + callLiteralNot callLiteralKind = iota + callLiteralNull + callLiteralNumber + callLiteralString + callLiteralBoolean + callLiteralArray +) + +type callOperand struct { + sql string + kind callLiteralKind +} + +type callResultFacts struct { + staticKind callLiteralKind +} + +func (r *planRenderer) renderCallProjection(plan ir.PhysicalPlan, projection ir.PhysicalProjection) (string, error) { + fragment, _, err := r.renderCallExpression(plan, *projection.Expression, 0) + if err != nil { + return "", fmt.Errorf("projection %q: %w", projection.Name, err) + } + if err := checkCallFragment(fragment); err != nil { + return "", fmt.Errorf("projection %q: %w", projection.Name, err) + } + return "(" + fragment + ")", nil +} + +func validateCallProjection(projection ir.PhysicalProjection) error { + expression := projection.Expression + if expression == nil { + return fmt.Errorf("%w: projection %q: CALL expression is missing its payload", + ErrUnsupportedPostgresPlan, projection.Name) + } + call := expression.Call + if call == nil { + return fmt.Errorf("%w: projection %q: CALL expression is missing its payload", + ErrUnsupportedPostgresPlan, projection.Name) + } + if expression.Cardinality != ir.PhysicalScalarCardinality || expression.NullBehavior != ir.PhysicalPreserveNull { + return fmt.Errorf("%w: projection %q: CALL is registered for SCALAR-cardinality PRESERVE_NULL expressions; array and object call projections belong to a later capability", + ErrUnsupportedPostgresPlan, projection.Name) + } + if projection.Presence != nil { + return fmt.Errorf("%w: projection %q carries presence metadata on a CALL expression; presence proofs pair with selector extractions only", + ErrUnsupportedPostgresPlan, projection.Name) + } + return validateCallShape(projection.Name, call, 0) +} + +func validateCallShape(projectionName string, call *ir.PhysicalCall, depth int) error { + if depth > maxCallExpressionDepth { + return fmt.Errorf("%w: projection %q: call nesting exceeds the %d-level registered bound", + ErrUnsupportedPostgresPlan, projectionName, maxCallExpressionDepth) + } + name := strings.ToLower(strings.TrimSpace(call.Name)) + if !registeredCallNames[name] { + return fmt.Errorf("%w: call %q is not part of the v1 PostgreSQL registered call surface (unregistered: array reducers, string pipelines, path utilities, uuid markers, and subplan-valued construction calls belong to later capabilities)", + ErrUnsupportedPostgresPlan, call.Name) + } + switch name { + case "add", "subtract", "multiply", "divide", "eq", "neq", "gt", "gte", "lt", "lte": + if len(call.Args) != 2 { + return fmt.Errorf("%w: call %q requires 2 arguments, got %d", ErrUnsupportedPostgresPlan, name, len(call.Args)) + } + case "not": + if len(call.Args) != 1 { + return fmt.Errorf("%w: call %q requires 1 argument, got %d", ErrUnsupportedPostgresPlan, name, len(call.Args)) + } + case "and", "or": + if len(call.Args) < 2 { + return fmt.Errorf("%w: call %q requires at least 2 arguments, got %d", ErrUnsupportedPostgresPlan, name, len(call.Args)) + } + case "if": + if len(call.Args) != 3 { + return fmt.Errorf("%w: call %q requires 3 arguments, got %d", ErrUnsupportedPostgresPlan, name, len(call.Args)) + } + case "case": + if len(call.Args) < 2 { + return fmt.Errorf("%w: call %q requires at least one condition/result pair", ErrUnsupportedPostgresPlan, name) + } + case "coalesce", "fallback": + if len(call.Args) == 0 { + return fmt.Errorf("%w: call %q requires at least one argument", ErrUnsupportedPostgresPlan, name) + } + case "assert": + if len(call.Args) != 2 { + return fmt.Errorf("%w: call %q requires a condition and a message", ErrUnsupportedPostgresPlan, name) + } + case "cast": + if len(call.Args) != 1 { + return fmt.Errorf("%w: cast requires 1 argument, got %d", ErrUnsupportedPostgresPlan, len(call.Args)) + } + if !validCallCastTarget(call.TargetKind) { + return fmt.Errorf("%w: cast target kind %q is not registered for PostgreSQL (date/date_time coercions are gaps)", + ErrUnsupportedPostgresPlan, call.TargetKind) + } + case "distinct_present_one": + if len(call.Args) != 1 || + (call.TargetKind != "STRING" && call.TargetKind != "INTEGER" && call.TargetKind != "DECIMAL" && call.TargetKind != "BOOLEAN") { + return fmt.Errorf("%w: distinct_present_one requires one array argument and a STRING/INTEGER/DECIMAL/BOOLEAN target kind (the aqlTableScalarType domain, render/aql/table_reshape.go:693-704)", + ErrUnsupportedPostgresPlan) + } + default: + return fmt.Errorf("%w: call %q is not part of the v1 PostgreSQL registered call surface", + ErrUnsupportedPostgresPlan, call.Name) + } + if name != "cast" && name != "distinct_present_one" && strings.TrimSpace(call.TargetKind) != "" { + return fmt.Errorf("%w: call %q must not carry target kind %q (only cast and distinct_present_one name a target kind)", + ErrUnsupportedPostgresPlan, name, call.TargetKind) + } + for index := range call.Args { + arg := &call.Args[index] + if arg.Call != nil && arg.Kind == ir.PhysicalCallExpression { + if err := validateCallShape(projectionName, arg.Call, depth+1); err != nil { + return err + } + } + } + return nil +} + +func validCallCastTarget(target string) bool { + switch strings.ToLower(strings.TrimSpace(target)) { + case "string", "code", "uuid", "integer", "decimal", "boolean": + return true + default: + return false + } +} + +func forEachCallOperandRowVariable(expression ir.PhysicalExpression, visit func(variable string, extract *ir.PhysicalExtract) error) error { + call := expression.Call + if call == nil { + return fmt.Errorf("%w: CALL expression is missing its payload", ErrUnsupportedPostgresPlan) + } + for index := range call.Args { + arg := &call.Args[index] + switch arg.Kind { + case ir.PhysicalValueExpression: + if arg.Value == nil { + return fmt.Errorf("%w: VALUE call operand is missing its payload", ErrUnsupportedPostgresPlan) + } + if err := visit(arg.Value.Variable, nil); err != nil { + return fmt.Errorf("argument %d: %w", index, err) + } + case ir.PhysicalExtractExpression: + if arg.Extract == nil { + return fmt.Errorf("%w: EXTRACT call operand is missing its payload", ErrUnsupportedPostgresPlan) + } + if err := visit(arg.Extract.Source.Variable, arg.Extract); err != nil { + return fmt.Errorf("argument %d: %w", index, err) + } + case ir.PhysicalCallExpression: + if err := forEachCallOperandRowVariable(*arg, visit); err != nil { + return fmt.Errorf("argument %d: %w", index, err) + } + } + } + return nil +} + +func (r *planRenderer) renderCallExpression(plan ir.PhysicalPlan, expression ir.PhysicalExpression, depth int) (string, callResultFacts, error) { + if depth > maxCallExpressionDepth { + return "", callResultFacts{}, fmt.Errorf("%w: call nesting exceeds the %d-level registered bound", + ErrUnsupportedPostgresPlan, maxCallExpressionDepth) + } + if expression.Kind != ir.PhysicalCallExpression || expression.Call == nil { + return "", callResultFacts{}, fmt.Errorf("%w: CALL expression payload is missing", ErrUnsupportedPostgresPlan) + } + call := expression.Call + if expression.Cardinality != ir.PhysicalScalarCardinality || expression.NullBehavior != ir.PhysicalPreserveNull { + return "", callResultFacts{}, fmt.Errorf("%w: CALL is registered for SCALAR-cardinality PRESERVE_NULL expressions", + ErrUnsupportedPostgresPlan) + } + if err := validateCallShape("", call, depth); err != nil { + return "", callResultFacts{}, err + } + name := strings.ToLower(strings.TrimSpace(call.Name)) + switch name { + case "add", "subtract", "multiply", "divide": + sql, err := r.renderCallArithmetic(plan, name, call, depth) + return sql, callResultFacts{staticKind: callLiteralNumber}, err + case "cast": + sql, kind, err := r.renderCallCast(plan, call, depth) + return sql, callResultFacts{staticKind: kind}, err + case "coalesce", "fallback": + sql, err := r.renderCallCoalesce(plan, call, depth) + return sql, callResultFacts{}, err + case "if": + sql, err := r.renderCallIf(plan, call, depth) + return sql, callResultFacts{}, err + case "case": + sql, err := r.renderCallCase(plan, call, depth) + return sql, callResultFacts{}, err + case "and", "or": + sql, err := r.renderCallLogical(plan, name, call, depth) + return sql, callResultFacts{}, err + case "not": + operand, err := r.renderCallOperand(plan, call.Args[0], depth) + if err != nil { + return "", callResultFacts{}, err + } + return "to_jsonb(NOT " + callTruthyTest(operand.sql) + ")", callResultFacts{staticKind: callLiteralBoolean}, nil + case "eq", "neq": + left, right, err := r.renderCallPair(plan, call, depth) + if err != nil { + return "", callResultFacts{}, err + } + equal := "COALESCE(" + left.sql + ", 'null'::jsonb) = COALESCE(" + right.sql + ", 'null'::jsonb)" + if name == "neq" { + equal = "NOT (" + equal + ")" + } + return "to_jsonb(" + equal + ")", callResultFacts{staticKind: callLiteralBoolean}, nil + case "gt", "gte", "lt", "lte": + left, right, err := r.renderCallPair(plan, call, depth) + if err != nil { + return "", callResultFacts{}, err + } + return "to_jsonb(" + callOrderTest(left.sql, sqlCallComparison(name), right.sql) + ")", + callResultFacts{staticKind: callLiteralBoolean}, nil + case "assert": + sql, err := r.renderCallAssert(plan, call, depth) + return sql, callResultFacts{staticKind: callLiteralBoolean}, err + case "distinct_present_one": + sql, err := r.renderCallDistinctPresentOne(plan, call, depth) + return sql, callResultFacts{}, err + default: + return "", callResultFacts{}, fmt.Errorf("%w: call %q is not part of the v1 PostgreSQL registered call surface", + ErrUnsupportedPostgresPlan, call.Name) + } +} + +func (r *planRenderer) renderCallOperand(plan ir.PhysicalPlan, expression ir.PhysicalExpression, depth int) (callOperand, error) { + switch expression.Kind { + case ir.PhysicalLiteralExpression: + return r.renderCallLiteral(plan, expression) + case ir.PhysicalExtractExpression: + return r.renderCallExtractOperand(expression) + case ir.PhysicalValueExpression: + return r.renderCallValueOperand(expression) + case ir.PhysicalCallExpression: + sql, _, err := r.renderCallExpression(plan, expression, depth+1) + if err != nil { + return callOperand{}, err + } + sql = "(" + sql + ")" + if err := checkCallFragment(sql); err != nil { + return callOperand{}, err + } + return callOperand{sql: sql}, nil + default: + return callOperand{}, fmt.Errorf("%w: call operand kind %q has no registered PostgreSQL rendering (subplan, aggregate, pivot, object, and related-field operands are capability gaps)", + ErrUnsupportedPostgresPlan, string(expression.Kind)) + } +} + +func (r *planRenderer) renderCallLiteral(plan ir.PhysicalPlan, expression ir.PhysicalExpression) (callOperand, error) { + if expression.Literal == nil || expression.Literal.BindKey == "" { + return callOperand{}, fmt.Errorf("%w: LITERAL call operand is missing its bind key", ErrUnsupportedPostgresPlan) + } + value, ok := plan.BindVars[expression.Literal.BindKey] + if !ok { + return callOperand{}, fmt.Errorf("%w: call literal bind %q is not defined", ErrUnsupportedPostgresPlan, expression.Literal.BindKey) + } + switch typed := value.(type) { + case nil: + return callOperand{sql: "$" + strconv.Itoa(r.jsonbParameter(json.RawMessage("null"))) + "::jsonb", kind: callLiteralNull}, nil + case bool: + return r.bindCallNumberOrScalar(typed, callLiteralBoolean) + case int64: + return r.bindCallNumberOrScalar(typed, callLiteralNumber) + case float64: + return r.bindCallNumberOrScalar(typed, callLiteralNumber) + case string: + return r.bindCallNumberOrScalar(typed, callLiteralString) + case json.RawMessage: + var probe any + if err := json.Unmarshal(typed, &probe); err != nil { + return callOperand{}, fmt.Errorf("%w: call literal bind %q is not valid JSON: %v", + ErrUnsupportedPostgresPlan, expression.Literal.BindKey, err) + } + if _, isArray := probe.([]any); !isArray { + return callOperand{}, fmt.Errorf("%w: call literal bind %q has stored kind %s; only a JSON array binds raw (the distinct_present_one collection form) — the unknown call/type pair gate", + ErrUnsupportedPostgresPlan, expression.Literal.BindKey, jsonProbeKind(probe)) + } + raw := append(json.RawMessage(nil), typed...) + return callOperand{sql: "$" + strconv.Itoa(r.jsonbParameter(raw)) + "::jsonb", kind: callLiteralArray}, nil + default: + return callOperand{}, fmt.Errorf("%w: call literal bind %q has Go type %T; the registered literal bind types are int64, float64, string, bool, nil, and a json.RawMessage array — the unknown call/type pair gate", + ErrUnsupportedPostgresPlan, expression.Literal.BindKey, value) + } +} + +func (r *planRenderer) bindCallNumberOrScalar(value any, kind callLiteralKind) (callOperand, error) { + raw, err := json.Marshal(value) + if err != nil { + return callOperand{}, fmt.Errorf("%w: call literal bind value %T is not JSON-encodable: %v", + ErrUnsupportedPostgresPlan, value, err) + } + return callOperand{sql: "$" + strconv.Itoa(r.jsonbParameter(raw)) + "::jsonb", kind: kind}, nil +} + +func jsonProbeKind(probe any) string { + switch probe.(type) { + case nil: + return "JSON null" + case bool: + return "boolean" + case float64: + return "number" + case string: + return "string" + case map[string]any: + return "object" + } + return "unknown" +} + +func (r *planRenderer) renderCallExtractOperand(expression ir.PhysicalExpression) (callOperand, error) { + if expression.Cardinality != ir.PhysicalScalarCardinality { + return callOperand{}, fmt.Errorf("%w: CALL scalar operands accept SCALAR-cardinality EXTRACT expressions only", + ErrUnsupportedPostgresPlan) + } + path, err := validateSelectorExtraction(ir.PhysicalProjection{Name: "call operand", Expression: &expression}) + if err != nil { + return callOperand{}, err + } + scope, ok := r.rows[expression.Extract.Source.Variable] + if !ok || scope.alias == "" || scope.derivedValue != "" || scope.derivedJSON { + return callOperand{}, fmt.Errorf("%w: call operand extracts from variable %q, which does not bind a readable stored row", + ErrUnsupportedPostgresPlan, expression.Extract.Source.Variable) + } + if expression.Extract.ResourceType != scope.resourceType { + return callOperand{}, fmt.Errorf("%w: call operand validates selector against resource type %q, but the row is scanned as %q", + ErrUnsupportedPostgresPlan, expression.Extract.ResourceType, scope.resourceType) + } + sql, err := r.scalarExtractValueExpression(scope.alias, path, expression.NullBehavior) + if err != nil { + return callOperand{}, err + } + return callOperand{sql: sql}, nil +} + +func (r *planRenderer) renderCallValueOperand(expression ir.PhysicalExpression) (callOperand, error) { + value := expression.Value + if value == nil || value.BindKey != "" { + return callOperand{}, fmt.Errorf("%w: VALUE call operands must read a row path, not a bind value", ErrUnsupportedPostgresPlan) + } + if _, ok := r.rows[value.Variable]; !ok { + return callOperand{}, fmt.Errorf("%w: call operand reads variable %q, which does not bind a readable row", + ErrUnsupportedPostgresPlan, value.Variable) + } + if scope := r.rows[value.Variable]; scope.derivedValue != "" && !scope.derivedJSON { + return callOperand{}, fmt.Errorf("%w: call operand reads derived variable %q, which carries a computed non-JSON value (ordinality and has-item columns are not registered call operands)", + ErrUnsupportedPostgresPlan, value.Variable) + } + if len(value.Path) == 1 && resourceFilterColumns[canonicalKeyColumn(value.Path[0])] { + return callOperand{}, fmt.Errorf("%w: call operand %v addresses the typed storage column %q; registered call operands read JSON documents, not storage columns", + ErrUnsupportedPostgresPlan, value.Path, value.Path[0]) + } + sql, err := r.resolveValue(*value) + if err != nil { + return callOperand{}, err + } + return callOperand{sql: "(" + sql + ")"}, nil +} + +func (r *planRenderer) renderCallPair(plan ir.PhysicalPlan, call *ir.PhysicalCall, depth int) (callOperand, callOperand, error) { + left, err := r.renderCallOperand(plan, call.Args[0], depth) + if err != nil { + return callOperand{}, callOperand{}, fmt.Errorf("argument 0: %w", err) + } + right, err := r.renderCallOperand(plan, call.Args[1], depth) + if err != nil { + return callOperand{}, callOperand{}, fmt.Errorf("argument 1: %w", err) + } + return left, right, nil +} + +func (r *planRenderer) renderCallArithmetic(plan ir.PhysicalPlan, name string, call *ir.PhysicalCall, depth int) (string, error) { + operator, ok := map[string]string{"add": "+", "subtract": "-", "multiply": "*", "divide": "/"}[name] + if !ok { + return "", fmt.Errorf("%w: arithmetic call %q is not registered for PostgreSQL", + ErrUnsupportedPostgresPlan, name) + } + left, err := r.renderCallNumericOperand(plan, call.Args[0], depth) + if err != nil { + return "", fmt.Errorf("argument 0: %w", err) + } + right, err := r.renderCallNumericOperand(plan, call.Args[1], depth) + if err != nil { + return "", fmt.Errorf("argument 1: %w", err) + } + clampedLeft, clampedRight := callBandOperandClamp(left), callBandOperandClamp(right) + switch operator { + case "+", "-": + magnitude := "abs((" + clampedLeft + " " + operator + " " + clampedRight + "))" + return "to_jsonb(CASE WHEN " + magnitude + " < '" + callDoubleAddPassUpper + "'::numeric" + + " THEN " + callDoubleOperand(left) + " " + operator + " " + callDoubleOperand(right) + + " ELSE NULL END)", nil + case "*": + magnitude := "abs((" + clampedLeft + " * " + clampedRight + "))" + return "to_jsonb(CASE WHEN " + magnitude + " < '" + callDoubleMulDivPassUpper + "'::numeric" + + " AND " + magnitude + " >= '" + callDoubleSubnormalFloor + "'::numeric" + + " THEN " + callDoubleOperand(left) + " * " + callDoubleOperand(right) + + " WHEN " + magnitude + " < '" + callDoubleSubnormalFloor + "'::numeric" + + " THEN 0::double precision ELSE NULL END)", nil + default: + quotient := "((" + clampedLeft + " + " + callDividendScaleBias + ") / NULLIF(" + clampedRight + ", 0))" + magnitude := "abs(" + quotient + ")" + return "to_jsonb(CASE WHEN " + magnitude + " < '" + callDoubleMulDivPassUpper + "'::numeric" + + " AND " + magnitude + " >= '" + callDoubleSubnormalFloor + "'::numeric" + + " THEN " + callDoubleOperand(left) + " / " + callDoubleDivisor(right) + + " WHEN " + magnitude + " < '" + callDoubleSubnormalFloor + "'::numeric" + + " THEN 0::double precision ELSE NULL END)", nil + } +} + +const ( + callDoubleOverflowBound = "179769313486231580793728971405303415079934132710037826936173778980444968292764750946649017977587207096330286416692887910946555547851940402630657488671505820681908902000708383676273854845817711531764475730270069855571366959622842914819860834936475292719074168444365510704342711559699508093042880177904174497792" + callDoubleAddPassUpper = "179769313486231560835325876058105298516207002341652166261661174625869553267292326574530099287946549246750631490335877017522087105926987962906277604735569213290190919152394180476217125334960946356387261286640198029037799514183602981511756283727771403830521483963923935633133642802139091669457927874464075218944" + callDoubleMulDivPassUpper = "179769313486231500960116590016510948825025611236495184238123361562143308190875053458173343219024575698011666711264844337248681780152130643733137952927759391115036970607451570876046936802390650830255617955750582549437097177865883181587442630101659737164863430522599210419506436529457842398703070964143777382400" + callDoubleUnderflowFloor = "2.5E-324" + callDoubleSubnormalFloor = "4.9406564584124655E-324" + callDoubleNormalFloor = "2.225073858507201383090232717332404064219215980462331830553327416887204434813918195854283159012511020564067339731035811005152434161553460108856012385377718821130777993532002330479610147442583636071921565046942503734208375250806650616658158948720491179968591639648500635908770118304874799780887753749949451580451605050915399856582470818645113537935804992115981085766051992433352114352390148795699609591288891602992641511063466313393663477586513029371762047325631781485664350872122828637642044846811407613911477062801689853244110024161447421618567166150540154285084716752901903161322778896729707373123334086988983175067838846926092773977972858659654941091369095406136467568702398678315290680984617210924625396728515625E-308" + callDividendScaleBias = "0E-400" + callBandClampCap = "1e400" +) + +func callBandOperandClamp(operand string) string { + return "(CASE" + + " WHEN " + operand + " > '" + callBandClampCap + "'::numeric THEN '" + callBandClampCap + "'::numeric" + + " WHEN " + operand + " < '-" + callBandClampCap + "'::numeric THEN '-" + callBandClampCap + "'::numeric" + + " ELSE " + operand + " END)" +} + +func callDoubleOperand(numericOperand string) string { + return "(CASE WHEN abs(" + numericOperand + ") < '" + callDoubleOverflowBound + "'::numeric" + + " THEN CASE WHEN abs(" + numericOperand + ") >= '" + callDoubleUnderflowFloor + "'::numeric" + + " THEN (" + numericOperand + ")::double precision ELSE 0::double precision END" + + " ELSE NULL END)" +} + +func callDoubleDivisor(numericOperand string) string { + return "(CASE WHEN abs(" + numericOperand + ") < '" + callDoubleOverflowBound + "'::numeric" + + " THEN CASE WHEN abs(" + numericOperand + ") >= '" + callDoubleNormalFloor + "'::numeric" + + " THEN (" + numericOperand + ")::double precision ELSE NULL END" + + " ELSE NULL END)" +} + +func callNumericToJSONB(expression string) string { + return "to_jsonb(CASE WHEN abs(" + expression + ") < '" + callDoubleOverflowBound + "'::numeric" + + " THEN CASE WHEN abs(" + expression + ") >= '" + callDoubleUnderflowFloor + "'::numeric" + + " THEN (" + expression + ")::double precision ELSE 0::double precision END END)" +} + +func (r *planRenderer) renderCallNumericOperand(plan ir.PhysicalPlan, expression ir.PhysicalExpression, depth int) (string, error) { + if expression.Kind == ir.PhysicalLiteralExpression { + literal, err := r.renderCallLiteral(plan, expression) + if err != nil { + return "", err + } + if literal.kind != callLiteralNumber { + return "", fmt.Errorf("%w: arithmetic operand literal has stored kind %s; arithmetic accepts number binds (dynamic operands carry the registered coercion table)", + ErrUnsupportedPostgresPlan, callLiteralKindName(literal.kind)) + } + return "((" + literal.sql + ") #>> '{}')::numeric", nil + } + if expression.Kind == ir.PhysicalCallExpression { + sql, facts, err := r.renderCallExpression(plan, expression, depth) + if err != nil { + return "", err + } + if err := checkCallFragment(sql); err != nil { + return "", err + } + if facts.staticKind == callLiteralNumber { + return "((" + sql + ") #>> '{}')::numeric", nil + } + return callNumericCoercion("(" + sql + ")"), nil + } + operand, err := r.renderCallOperand(plan, expression, depth) + if err != nil { + return "", err + } + return callNumericCoercion(operand.sql), nil +} + +func callNumericCoercion(operand string) string { + return "(CASE" + + " WHEN " + operand + " IS NULL OR jsonb_typeof(" + operand + ") = 'null' THEN NULL" + + " WHEN jsonb_typeof(" + operand + ") = 'number' THEN (" + operand + " #>> '{}')::numeric" + + callDecimalTierArms(operand) + + " ELSE NULL END)" +} + +const callDecimalWhitespace = " \t\n\v\f\r" + +const ( + callDecimalUnderflowIntegerDigits = 99 + callDecimalUnderflowMinExponentDigits = 5 +) + +var callDecimalZeroPattern = `^` + callWhitespaceRegexClass() + `*[+-]?` + + `(0+(\.0*)?|\.[0]+)([eE][+-]?[0-9]+)?` + callWhitespaceRegexClass() + `*$` + +var callDecimalUnderflowPattern = `^` + callWhitespaceRegexClass() + `*[+-]?` + + "[0-9]{1," + strconv.Itoa(callDecimalUnderflowIntegerDigits) + `}` + + `(\.[0-9]*)?[eE]-` + + "[1-9][0-9]{" + strconv.Itoa(callDecimalUnderflowMinExponentDigits-1) + ",}" + + callWhitespaceRegexClass() + `*$` + +var callDecimalStringPattern = `^` + callWhitespaceRegexClass() + `*[+-]?([0-9]+(\.[0-9]*)?|\.[0-9]+)([eE][+-]?[0-9]+)?` + callWhitespaceRegexClass() + `*$` + +func callDecimalTierArms(operand string) string { + text := "(" + operand + ` #>> '{}')` + btrim := "btrim(" + text + ", " + callDecimalTrimCharacters + ")" + castArm := "(" + btrim + " || COALESCE(current_setting(" + quoteSQLLiteral(guardBarrierGUC) + ", true), ''))::numeric" + return " WHEN jsonb_typeof(" + operand + ") = 'string' AND " + text + " ~ " + quoteSQLLiteral(callDecimalStringPattern) + + " THEN CASE WHEN pg_input_is_valid(" + btrim + ", 'numeric') THEN " + castArm + + " WHEN (" + text + " ~ " + quoteSQLLiteral(callDecimalZeroPattern) + + " OR " + text + " ~ " + quoteSQLLiteral(callDecimalUnderflowPattern) + ") THEN 0" + + " ELSE NULL END" +} + +var callDecimalTrimCharacters = quoteSQLLiteral(callDecimalWhitespace) + +func callWhitespaceRegexClass() string { + var class strings.Builder + class.WriteByte('[') + for _, r := range callDecimalWhitespace { + switch r { + case '\\', ']', '^', '-': + class.WriteByte('\\') + } + class.WriteRune(r) + } + class.WriteByte(']') + return class.String() +} + +func (r *planRenderer) renderCallCast(plan ir.PhysicalPlan, call *ir.PhysicalCall, depth int) (string, callLiteralKind, error) { + operand, err := r.renderCallOperand(plan, call.Args[0], depth) + if err != nil { + return "", callLiteralNot, err + } + switch strings.ToLower(strings.TrimSpace(call.TargetKind)) { + case "string", "code", "uuid": + return "CASE" + + " WHEN " + operand.sql + " IS NULL OR jsonb_typeof(" + operand.sql + ") = 'null' THEN '\"\"'::jsonb" + + " WHEN jsonb_typeof(" + operand.sql + ") IN ('string', 'boolean', 'number')" + + " THEN to_jsonb((" + operand.sql + ") #>> '{}')" + + " ELSE NULL END", callLiteralString, nil + case "integer", "decimal": + return callToNumberExpression(operand.sql), callLiteralNumber, nil + case "boolean": + return "CASE" + + " WHEN " + operand.sql + " IS NULL OR jsonb_typeof(" + operand.sql + ") = 'null' THEN 'false'::jsonb" + + " WHEN jsonb_typeof(" + operand.sql + ") = 'boolean' THEN " + operand.sql + + " WHEN jsonb_typeof(" + operand.sql + ") = 'number' THEN CASE WHEN " + operand.sql + + " = '0'::jsonb THEN 'false'::jsonb ELSE 'true'::jsonb END" + + " WHEN jsonb_typeof(" + operand.sql + ") = 'string' THEN CASE WHEN " + operand.sql + + " = '\"\"'::jsonb THEN 'false'::jsonb ELSE 'true'::jsonb END" + + " ELSE 'true'::jsonb END", callLiteralBoolean, nil + default: + return "", callLiteralNot, fmt.Errorf("%w: cast target kind %q is not registered for PostgreSQL", + ErrUnsupportedPostgresPlan, call.TargetKind) + } +} + +const callToNumberMaxRecursion = 3 + +func callToNumberExpression(operand string) string { + return callNumericToJSONB(callToNumberDouble(operand, 0)) +} + +func callToNumberDouble(operand string, depth int) string { + if depth >= callToNumberMaxRecursion { + return "0" + } + element := callToNumberDouble("(("+operand+") -> 0)", depth+1) + return "CASE" + + " WHEN " + operand + " IS NULL OR jsonb_typeof(" + operand + ") = 'null' THEN 0" + + " WHEN jsonb_typeof(" + operand + ") = 'boolean' THEN CASE WHEN " + operand + + " = 'true'::jsonb THEN 1 ELSE 0 END" + + " WHEN jsonb_typeof(" + operand + ") = 'number' THEN (" + operand + " #>> '{}')::numeric" + + callDecimalTierArms(operand) + + " WHEN jsonb_typeof(" + operand + ") = 'string' THEN 0" + + " WHEN jsonb_typeof(" + operand + ") = 'array' THEN CASE" + + " WHEN jsonb_array_length(" + operand + ") = 0 THEN 0" + + " WHEN jsonb_array_length(" + operand + ") = 1 THEN " + element + + " ELSE 0 END" + + " ELSE 0 END" +} + +func (r *planRenderer) renderCallCoalesce(plan ir.PhysicalPlan, call *ir.PhysicalCall, depth int) (string, error) { + parts := make([]string, 0, len(call.Args)) + for index := range call.Args { + operand, err := r.renderCallOperand(plan, call.Args[index], depth) + if err != nil { + return "", fmt.Errorf("argument %d: %w", index, err) + } + parts = append(parts, "NULLIF("+operand.sql+", 'null'::jsonb)") + } + return "COALESCE(" + strings.Join(parts, ", ") + ")", nil +} + +func (r *planRenderer) renderCallIf(plan ir.PhysicalPlan, call *ir.PhysicalCall, depth int) (string, error) { + condition, err := r.renderCallOperand(plan, call.Args[0], depth) + if err != nil { + return "", fmt.Errorf("condition: %w", err) + } + thenOperand, err := r.renderCallOperand(plan, call.Args[1], depth) + if err != nil { + return "", fmt.Errorf("then arm: %w", err) + } + elseOperand, err := r.renderCallOperand(plan, call.Args[2], depth) + if err != nil { + return "", fmt.Errorf("else arm: %w", err) + } + return "CASE WHEN " + callTruthyTest(condition.sql) + " THEN " + thenOperand.sql + " ELSE " + elseOperand.sql + " END", nil +} + +func (r *planRenderer) renderCallCase(plan ir.PhysicalPlan, call *ir.PhysicalCall, depth int) (string, error) { + operands := make([]callOperand, len(call.Args)) + for index := range call.Args { + operand, err := r.renderCallOperand(plan, call.Args[index], depth) + if err != nil { + return "", fmt.Errorf("argument %d: %w", index, err) + } + operands[index] = operand + } + end := len(operands) + result := "NULL" + if end%2 == 1 { + result = operands[end-1].sql + end-- + } + for index := end - 2; index >= 0; index -= 2 { + result = "CASE WHEN " + callTruthyTest(operands[index].sql) + " THEN " + operands[index+1].sql + " ELSE " + result + " END" + } + return result, nil +} + +func (r *planRenderer) renderCallLogical(plan ir.PhysicalPlan, name string, call *ir.PhysicalCall, depth int) (string, error) { + operands := make([]callOperand, len(call.Args)) + for index := range call.Args { + operand, err := r.renderCallOperand(plan, call.Args[index], depth) + if err != nil { + return "", fmt.Errorf("argument %d: %w", index, err) + } + operands[index] = operand + } + sql := operands[len(operands)-1].sql + for index := len(operands) - 2; index >= 0; index-- { + test := callTruthyTest(operands[index].sql) + if name == "and" { + sql = "CASE WHEN " + test + " THEN " + sql + " ELSE " + operands[index].sql + " END" + } else { + sql = "CASE WHEN " + test + " THEN " + operands[index].sql + " ELSE " + sql + " END" + } + } + return sql, nil +} + +func callTruthyTest(operand string) string { + return "NOT (" + operand + " IS NULL" + + " OR jsonb_typeof(" + operand + ") = 'null'" + + " OR " + operand + " = 'false'::jsonb" + + " OR (jsonb_typeof(" + operand + ") = 'number' AND " + operand + " = '0'::jsonb)" + + " OR " + operand + " = '\"\"'::jsonb)" +} + +func sqlCallComparison(name string) string { + switch name { + case "gt": + return ">" + case "gte": + return ">=" + case "lt": + return "<" + case "lte": + return "<=" + default: + return "" + } +} + +func callOrderTest(left, operator, right string) string { + leftRank := callValueRank(left) + rightRank := callValueRank(right) + return "CASE" + + " WHEN " + leftRank + " >= 4 OR " + rightRank + " >= 4 THEN NULL" + + " WHEN " + leftRank + " <> " + rightRank + " THEN " + leftRank + " " + operator + " " + rightRank + + " ELSE (" + callNullNormalized(left) + ") " + operator + " (" + callNullNormalized(right) + ") END" +} + +func callNullNormalized(operand string) string { + return "COALESCE(" + operand + ", 'null'::jsonb)" +} + +func callValueRank(operand string) string { + return "(CASE" + + " WHEN " + operand + " IS NULL OR jsonb_typeof(" + operand + ") = 'null' THEN 0" + + " WHEN jsonb_typeof(" + operand + ") = 'boolean' THEN 1" + + " WHEN jsonb_typeof(" + operand + ") = 'number' THEN 2" + + " WHEN jsonb_typeof(" + operand + ") = 'string' THEN 3" + + " ELSE 4 END)" +} + +func (r *planRenderer) renderCallAssert(plan ir.PhysicalPlan, call *ir.PhysicalCall, depth int) (string, error) { + condition, err := r.renderCallOperand(plan, call.Args[0], depth) + if err != nil { + return "", fmt.Errorf("condition: %w", err) + } + message := call.Args[1] + if message.Kind != ir.PhysicalLiteralExpression || message.Literal == nil || message.Literal.BindKey == "" { + return "", fmt.Errorf("%w: assert message must be one bind-backed string literal", ErrUnsupportedPostgresPlan) + } + value, ok := plan.BindVars[message.Literal.BindKey] + if !ok { + return "", fmt.Errorf("%w: assert message bind %q is not defined", ErrUnsupportedPostgresPlan, message.Literal.BindKey) + } + text, ok := value.(string) + if !ok { + return "", fmt.Errorf("%w: assert message bind %q has type %T; the ASSERT message must be a string (the unknown call/type pair gate)", + ErrUnsupportedPostgresPlan, message.Literal.BindKey, value) + } + index := r.textParameter(text, "") + raise := "to_jsonb(CAST('ASSERT:' || $" + strconv.Itoa(index) + + " || COALESCE(current_setting(" + quoteSQLLiteral(guardBarrierGUC) + ", true), '') AS integer))" + return "CASE WHEN " + callTruthyTest(condition.sql) + " THEN 'true'::jsonb ELSE " + raise + " END", nil +} + +const ( + distinctPresentGuardTypeMismatch = "CONSTRUCTION_ROW_VALUE_TYPE_MISMATCH" + distinctPresentGuardMultipleValues = "CONSTRUCTION_ROW_VALUE_MULTIPLE_VALUES" +) + +func (r *planRenderer) renderCallDistinctPresentOne(plan ir.PhysicalPlan, call *ir.PhysicalCall, depth int) (string, error) { + var expected string + switch call.TargetKind { + case "STRING": + expected = "string" + case "INTEGER", "DECIMAL": + expected = "number" + case "BOOLEAN": + expected = "boolean" + default: + return "", fmt.Errorf("%w: distinct_present_one target kind %q is not registered", ErrUnsupportedPostgresPlan, call.TargetKind) + } + base, err := r.renderCallArrayOperand(plan, call.Args[0], depth) + if err != nil { + return "", err + } + guarded := "CASE WHEN " + base + " IS NULL OR jsonb_typeof(" + base + ") <> 'array' THEN NULL::jsonb ELSE " + base + " END" + return "(SELECT CASE" + + " WHEN bool_or(jsonb_typeof(loom_dp_item.item) <> " + quoteSQLLiteral(expected) + ") THEN to_jsonb(" + guardCast(distinctPresentGuardTypeMismatch) + ")" + + " WHEN COUNT(*) = 0 THEN NULL" + + " WHEN COUNT(DISTINCT loom_dp_item.item) > 1 THEN to_jsonb(" + guardCast(distinctPresentGuardMultipleValues) + ")" + + " ELSE min(loom_dp_item.item::text)::jsonb" + + " END FROM jsonb_array_elements(" + guarded + ") AS loom_dp_outer(value)" + + " LEFT JOIN LATERAL jsonb_array_elements(" + + "CASE WHEN jsonb_typeof(loom_dp_outer.value) = 'array' THEN loom_dp_outer.value ELSE NULL::jsonb END" + + ") AS loom_dp_flattened(value) ON TRUE" + + " LEFT JOIN LATERAL (SELECT CASE" + + " WHEN jsonb_typeof(loom_dp_outer.value) = 'array' THEN loom_dp_flattened.value" + + " ELSE loom_dp_outer.value END AS item) AS loom_dp_item ON TRUE" + + " WHERE loom_dp_item.item IS NOT NULL AND loom_dp_item.item <> 'null'::jsonb)", nil +} + +func (r *planRenderer) renderCallArrayOperand(plan ir.PhysicalPlan, expression ir.PhysicalExpression, depth int) (string, error) { + switch expression.Kind { + case ir.PhysicalLiteralExpression: + literal, err := r.renderCallLiteral(plan, expression) + if err != nil { + return "", err + } + if literal.kind != callLiteralArray { + return "", fmt.Errorf("%w: distinct_present_one literal must bind a JSON array (the unknown call/type pair gate); got stored kind %s", + ErrUnsupportedPostgresPlan, callLiteralKindName(literal.kind)) + } + return literal.sql, nil + case ir.PhysicalExtractExpression: + if expression.Cardinality != ir.PhysicalArrayCardinality { + return "", fmt.Errorf("%w: distinct_present_one EXTRACT argument must be ARRAY-cardinality", ErrUnsupportedPostgresPlan) + } + extract := expression.Extract + if extract == nil || len(extract.Fallbacks) != 0 || extract.Prepared != nil || + extract.UnitNormalization != nil || extract.Distinct || extract.Selector.Filter != nil { + return "", fmt.Errorf("%w: distinct_present_one accepts only the closed direct-property EXTRACT; fallbacks, prepared sets, unit normalization, DISTINCT, and selector filters are gaps", + ErrUnsupportedPostgresPlan) + } + switch extract.ExecutionMode { + case ir.PhysicalSelectorGeneric, ir.PhysicalSelectorDirectScalar, "": + default: + return "", fmt.Errorf("%w: distinct_present_one extraction mode %q has no registered form", + ErrUnsupportedPostgresPlan, string(extract.ExecutionMode)) + } + switch expression.NullBehavior { + case ir.PhysicalPreserveNull, ir.PhysicalEmptyOnNull: + default: + return "", fmt.Errorf("%w: distinct_present_one extraction null behavior %q has no registered form", + ErrUnsupportedPostgresPlan, string(expression.NullBehavior)) + } + source := extract.Source + if source.BindKey != "" || source.Variable == "" || len(source.Path) != 1 || source.Path[0] != "payload" { + return "", fmt.Errorf("%w: distinct_present_one source must be one readable row's document payload", ErrUnsupportedPostgresPlan) + } + scope, ok := r.rows[source.Variable] + if !ok || scope.alias == "" || scope.derivedValue != "" || scope.derivedJSON { + return "", fmt.Errorf("%w: distinct_present_one source variable %q does not bind a readable stored row", + ErrUnsupportedPostgresPlan, source.Variable) + } + if strings.TrimSpace(extract.ResourceType) == "" || extract.ResourceType != scope.resourceType { + return "", fmt.Errorf("%w: distinct_present_one resource type %q contradicts the scanned row type %q", + ErrUnsupportedPostgresPlan, extract.ResourceType, scope.resourceType) + } + path := make([]string, 0, len(extract.Selector.Steps)) + for _, step := range extract.Selector.Steps { + if step.Field == "" || step.Iterate || step.Index != nil { + return "", fmt.Errorf("%w: distinct_present_one accepts only direct property paths; repeated, indexed, and filtered steps are gaps", + ErrUnsupportedPostgresPlan) + } + if !payloadPathSegmentPattern.MatchString(step.Field) { + return "", fmt.Errorf("%w: distinct_present_one path segment %q is not renderer-safe", ErrUnsupportedPostgresPlan, step.Field) + } + path = append(path, step.Field) + } + if len(path) == 0 { + return "", fmt.Errorf("%w: distinct_present_one selector must have at least one property", ErrUnsupportedPostgresPlan) + } + return r.scalarExtractValueExpression(scope.alias, path, expression.NullBehavior) + default: + return "", fmt.Errorf("%w: distinct_present_one argument kind %q is not registered (subplan arguments stay unregistered in this wave; their registration is a coordinator decision)", + ErrUnsupportedPostgresPlan, string(expression.Kind)) + } +} + +func callLiteralKindName(kind callLiteralKind) string { + switch kind { + case callLiteralNull: + return "JSON null" + case callLiteralNumber: + return "number" + case callLiteralString: + return "string" + case callLiteralBoolean: + return "boolean" + case callLiteralArray: + return "array" + default: + return "unknown" + } +} diff --git a/internal/dataframe/compiler/render/postgres/calls_live_test.go b/internal/dataframe/compiler/render/postgres/calls_live_test.go new file mode 100644 index 000000000..a1fd51836 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/calls_live_test.go @@ -0,0 +1,669 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "regexp" + "strconv" + "strings" + "testing" + "time" + + control "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +var sql10LiveNamePattern = regexp.MustCompile(`^verify_sql10v7_[0-9]+_[0-9a-f]+$`) + +type sql10Service struct { + *pgsq01Service +} + +func newSQL10Service(t *testing.T) *sql10Service { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v string) string { + s := v + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + text := err.Error() + for _, secret := range secrets { + if secret != "" { + text = strings.ReplaceAll(text, secret, "") + } + } + t.Fatalf("%s: %s", step, text) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := sql10ResourceName(t) + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err.Error())) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := sql10ResourceName(t) + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err.Error())) + } + }) + + client, err := control.OpenPool(ctx, control.ClientConfig{ + DSN: dsn, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 2 * time.Second, + }) + if err != nil { + fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + t.Cleanup(client.Close) + if _, err := control.MigrateSchema(ctx, client, schema); err != nil { + fail("migrate owned schema", err) + } + return &sql10Service{pgsq01Service: &pgsq01Service{ + ctx: ctx, iso: iso, client: client, schema: schema, scrub: scrub, fail: fail, + }} +} + +func sql10ResourceName(t *testing.T) string { + t.Helper() + name := "verify_sql10v7_" + strconv.FormatInt(time.Now().UnixNano(), 10) + "_" + randomHex(t) + if !sql10LiveNamePattern.MatchString(name) { + t.Fatalf("owned resource name %q does not match the required pattern", name) + } + return name +} + +func sql10DriverArgs(rendered command.RenderedPostgresPhysicalPlan) []any { + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + return args +} + +func (s *sql10Service) renderLive(t *testing.T, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, s.schema)) + if err != nil { + s.fail("render plan", err) + } + return rendered +} + +func (s *sql10Service) explain(t *testing.T, rendered command.RenderedPostgresPhysicalPlan) string { + t.Helper() + var text strings.Builder + for _, row := range s.run(t, "EXPLAIN (FORMAT TEXT) "+rendered.SQL, sql10DriverArgs(rendered)...) { + text.WriteString(strings.Join(row, "\n")) + text.WriteString("\n") + } + if text.Len() == 0 { + s.fail("explain produced no plan", errors.New(rendered.SQL)) + } + return text.String() +} + +func (s *sql10Service) expectRaise(t *testing.T, rendered command.RenderedPostgresPhysicalPlan, wantCode string, wantSubstrings ...string) { + t.Helper() + _, err := s.iso.Exec(s.ctx, rendered.SQL, sql10DriverArgs(rendered)...) + if err == nil { + t.Fatalf("statement executed clean, want %s raising %v: %s", wantCode, wantSubstrings, rendered.SQL) + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + s.fail("expect pg error", err) + } + if pgErr.Code != wantCode { + s.fail("statement raised ("+pgErr.Code+") want ("+wantCode+")", err) + } + for _, want := range wantSubstrings { + if !strings.Contains(pgErr.Message, want) { + t.Fatalf("error message %q does not name %q\nSQL: %s", pgErr.Message, want, rendered.SQL) + } + } +} + +func (s *sql10Service) runOne(t *testing.T, rendered command.RenderedPostgresPhysicalPlan, want ...string) { + t.Helper() + rows := s.runRendered(t, rendered) + if len(rows) != 1 { + t.Fatalf("rows = %v, want exactly one row %v", rows, want) + } + if strings.Join(rows[0], "|") != strings.Join(want, "|") { + t.Fatalf("row = %v, want %v\nSQL: %s", rows[0], want, rendered.SQL) + } +} + +func TestSQL10LiveArithmeticCastCoalesceReadBack(t *testing.T) { + svc := newSQL10Service(t) + svc.insertResource(t, sql10Project, sql10Gen, "Patient", "p-calc", sql10Path, + map[string]any{"multipleBirthInteger": 5, "active": true, + "birthDate": " \t42\n", + "deceasedDateTime": " 42 ", + "text": map[string]any{"status": "\n42.5\r", "div": "\u300042"}, + "deceasedBoolean": "\u2009 42", + "communication": map[string]any{"language": "\u300042"}, + "maritalStatus": "\u00a042", + "managingOrganization": "\u008542", + "meta": map[string]any{"versionId": "9007199254740993", "source": "5e-324", "lastUpdated": "2.4703282292062327e-324"}, + }) + svc.insertResource(t, sql10Project, sql10Gen, "Observation", "p-calc", sql10Path, nil) + + binds := map[string]any{ + "ten": int64(10), + "thirtytwo": int64(32), + "two": int64(2), + "one": int64(1), + "zero": int64(0), + "three": int64(3), + "huge": float64(1e308), + "tiny": float64(1e-308), + "tenth": float64(0.1), + "tenth3": float64(0.3), + "d02": float64(0.2), + "half": float64(0.5), + "eleven": int64(11), + "lakhth": int64(100000), + "numstr": "42", + "flagstr": "true", + "fb": "fallback", + } + plan := sql10Plan("Patient", binds, + sql10CallProjection("sum", sql10Call("add", "", sql10Literal("ten"), sql10Literal("thirtytwo"))), + sql10CallProjection("quot", sql10Call("divide", "", + sql10Extract(t, "Patient", "multipleBirthInteger", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("two"))), + sql10CallProjection("zero", sql10Call("divide", "", sql10Literal("one"), sql10Literal("zero"))), + sql10CallProjection("huge", sql10Call("multiply", "", sql10Literal("huge"), sql10Literal("ten"))), + sql10CallProjection("small", sql10Call("divide", "", sql10Literal("tiny"), sql10Literal("huge"))), + sql10CallProjection("ieeeAdd", sql10Call("add", "", sql10Literal("tenth"), sql10Literal("d02"))), + sql10CallProjection("ieeeDiv", sql10Call("divide", "", sql10Literal("tenth"), sql10Literal("tenth3"))), + sql10CallProjection("ieeeDiv11", sql10Call("divide", "", sql10Literal("eleven"), sql10Literal("lakhth"))), + sql10CallProjection("ieeeDiv3", sql10Call("divide", "", sql10Literal("one"), sql10Literal("three"))), + sql10CallProjection("ieeeParse", sql10Call("add", "", + sql10Extract(t, "Patient", "meta.versionId", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("zero"))), + sql10CallProjection("addOver", sql10Call("add", "", sql10Literal("huge"), sql10Literal("huge"))), + sql10CallProjection("divOver", sql10Call("divide", "", sql10Literal("huge"), sql10Literal("half"))), + sql10CallProjection("mulUnder", sql10Call("multiply", "", sql10Literal("tiny"), sql10Literal("tiny"))), + sql10CallProjection("divMinSub", sql10Call("divide", "", sql10Literal("one"), + sql10Extract(t, "Patient", "meta.source", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull))), + sql10CallProjection("tieAdd", sql10Call("add", "", + sql10Extract(t, "Patient", "meta.lastUpdated", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("zero"))), + sql10CallProjection("num", sql10Call("cast", "INTEGER", sql10Literal("numstr"))), + sql10CallProjection("flag", sql10Call("cast", "BOOLEAN", sql10Literal("flagstr"))), + sql10CallProjection("pick", sql10Call("coalesce", "", + sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("fb"))), + sql10CallProjection("padmul", sql10Call("multiply", "", + sql10Extract(t, "Patient", "birthDate", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("one"))), + sql10CallProjection("padint", sql10Call("cast", "INTEGER", + sql10Extract(t, "Patient", "deceasedDateTime", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull))), + sql10CallProjection("paddec", sql10Call("cast", "DECIMAL", + sql10Extract(t, "Patient", "text.status", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull))), + sql10CallProjection("thinCast", sql10Call("cast", "INTEGER", + sql10Extract(t, "Patient", "deceasedBoolean", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull))), + sql10CallProjection("thinAdd", sql10Call("add", "", + sql10Extract(t, "Patient", "deceasedBoolean", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("zero"))), + sql10CallProjection("ideoCast", sql10Call("cast", "INTEGER", + sql10Extract(t, "Patient", "text.div", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull))), + sql10CallProjection("nbspCast", sql10Call("cast", "INTEGER", + sql10Extract(t, "Patient", "maritalStatus", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull))), + sql10CallProjection("nelCast", sql10Call("cast", "INTEGER", + sql10Extract(t, "Patient", "managingOrganization", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull))), + ) + svc.runOne(t, svc.renderLive(t, plan), + "42", "2.5", "", "", "0", + "0.30000000000000004", "0.33333333333333337", "0.00011", "0.3333333333333333", + "9.007199254740992e+15", + "", "", "0", "", "0", + "42", "true", "fallback", + "42", "42", "42.5", + "0", "", "0", "0", "0") + + witness := svc.run(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource"`+ + ` WHERE project=$1 AND dataset_generation=$2 AND resource_type='Observation' AND logical_id='p-calc'`, + sql10Project, sql10Gen) + if len(witness) != 1 || witness[0][0] != "1" { + t.Fatalf("decoy must exist: %v", witness) + } +} + +func TestSQL10LiveDecimalTierReadBack(t *testing.T) { + svc := newSQL10Service(t) + svc.insertResource(t, sql10Project, sql10Gen, "Patient", "p-tier", sql10Path, + map[string]any{ + "meta": map[string]any{ + "versionId": "1e999999999999", + "source": "1e999999999998", + "lastUpdated": "1e-999999999999", + }, + "deceasedBoolean": "-1e999999999999", + "birthDate": " 1e2147483648 ", + "deceasedDateTime": "0." + strings.Repeat("0", 20000), + "text": map[string]any{ + "status": "1" + strings.Repeat("0", 9999) + "e-9999", + "div": "1" + strings.Repeat("0", 99) + "e-10000", + }, + "maritalStatus": "1" + strings.Repeat("0", 100) + "e-10000", + "managingOrganization": "0." + strings.Repeat("9", 7000) + "e-9000", + "gender": strings.Repeat("9", 32768), + }) + binds := map[string]any{"zero": int64(0)} + str := func(path ...string) ir.PhysicalExpression { + return sql10Extract(t, "Patient", strings.Join(path, "."), ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + } + castOf := func(name string, path ...string) ir.PhysicalProjection { + return sql10CallProjection(name, sql10Call("cast", "INTEGER", str(path...))) + } + addOf := func(name string, path ...string) ir.PhysicalProjection { + return sql10CallProjection(name, sql10Call("add", "", str(path...), sql10Literal("zero"))) + } + plan := sql10Plan("Patient", binds, + castOf("b1", "meta", "versionId"), + castOf("b2", "meta", "source"), + castOf("b3", "meta", "lastUpdated"), + castOf("b4", "deceasedBoolean"), + castOf("b5", "birthDate"), + addOf("b1a", "meta", "versionId"), + addOf("b3a", "meta", "lastUpdated"), + addOf("b5a", "birthDate"), + castOf("u1zero", "deceasedDateTime"), + castOf("safeLegit", "text", "status"), + castOf("u2tiny", "text", "div"), + addOf("u2add", "text", "div"), + castOf("mantissaS1", "maritalStatus"), + castOf("mantissaS2", "managingOrganization"), + castOf("safeCap", "gender"), + ) + svc.runOne(t, svc.renderLive(t, plan), + "", "", "0", "", "", + "", "0", "", + "0", "1", "0", "0", + "0", "0", "") +} + +func TestSQL10LiveDecimalTierRejectedReads(t *testing.T) { + svc := newSQL10Service(t) + svc.insertResource(t, sql10Project, sql10Gen, "Patient", "p-oracle", sql10Path, + map[string]any{ + "meta": map[string]any{ + "versionId": "0." + strings.Repeat("9", 18000), + "source": "0." + strings.Repeat("9", 18000) + "e-00001", + "lastUpdated": strings.Repeat("9", 99) + "." + strings.Repeat("9", 20000) + "e-00001", + }, + "gender": "1." + strings.Repeat("9", 20000) + "e-00001", + "text": map[string]any{ + "div": "0." + strings.Repeat("0", 18000) + "e99999", + "status": "1e-00001", + }, + }) + frac := sql10Extract(t, "Patient", "meta.versionId", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + zeroScaled := sql10Extract(t, "Patient", "text.div", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + spoofTiny := sql10Extract(t, "Patient", "meta.source", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + spoofHuge := sql10Extract(t, "Patient", "meta.lastUpdated", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + spoofMid := sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + plan := sql10Plan("Patient", map[string]any{"zero": int64(0)}, + sql10CallProjection("fracCast", sql10Call("cast", "INTEGER", frac)), + sql10CallProjection("fracAdd", sql10Call("add", "", frac, sql10Literal("zero"))), + sql10CallProjection("zeroCast", sql10Call("cast", "INTEGER", zeroScaled)), + sql10CallProjection("spoofTiny", sql10Call("cast", "INTEGER", spoofTiny)), + sql10CallProjection("spoofMid", sql10Call("cast", "INTEGER", spoofMid)), + sql10CallProjection("spoofHuge", sql10Call("cast", "INTEGER", spoofHuge)), + ) + svc.runOne(t, svc.renderLive(t, plan), "", "", "0", "", "", "") +} + +func TestSQL10LiveDecimalTierLeadingZeroExponent(t *testing.T) { + svc := newSQL10Service(t) + svc.insertResource(t, sql10Project, sql10Gen, "Patient", "p-lzero", sql10Path, + map[string]any{"text": map[string]any{"status": "1e-00001"}}) + status := sql10Extract(t, "Patient", "text.status", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + plan := sql10Plan("Patient", nil, + sql10CallProjection("toNum", sql10Call("cast", "DECIMAL", status)), + ) + svc.runOne(t, svc.renderLive(t, plan), "0.1") +} + +func TestSQL10LiveDivisorBandReadBack(t *testing.T) { + svc := newSQL10Service(t) + svc.insertResource(t, sql10Project, sql10Gen, "Patient", "p-drift", sql10Path, + map[string]any{ + "multipleBirthInteger": json.RawMessage(`1.25e-15`), + "deceasedBoolean": json.RawMessage(`7.4e-324`), + "deceasedDateTime": json.RawMessage(`1e-16`), + "gender": json.RawMessage(`4.9406564584124654e-324`), + "maritalStatus": json.RawMessage(`10`), + "managingOrganization": json.RawMessage(`4`), + "active": json.RawMessage(`8.9e-16`), + "birthDate": json.RawMessage(`5e-324`), + "meta.source": json.RawMessage(`8.88178419700125430e-16`), + "meta.versionId": json.RawMessage(`4.94065645841247e-324`), + }) + num := func(path string) ir.PhysicalExpression { + return sql10Extract(t, "Patient", path, ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + } + plan := sql10Plan("Patient", nil, + sql10CallProjection("driftDiv", sql10Call("divide", "", + num("multipleBirthInteger"), num("deceasedBoolean"))), + sql10CallProjection("minSubDiv", sql10Call("divide", "", + num("deceasedDateTime"), num("gender"))), + sql10CallProjection("sanityDiv", sql10Call("divide", "", + num("maritalStatus"), num("managingOrganization"))), + sql10CallProjection("canonDiv", sql10Call("divide", "", + num("active"), num("birthDate"))), + sql10CallProjection("witnessDiv", sql10Call("divide", "", + num("meta.source"), num("meta.versionId"))), + ) + svc.runOne(t, svc.renderLive(t, plan), "", "", "2.5", "", "") +} + +func TestSQL10LiveDecimalTierLiteralBind(t *testing.T) { + svc := newSQL10Service(t) + svc.insertResource(t, sql10Project, sql10Gen, "Patient", "p-bind", sql10Path, + map[string]any{"multipleBirthInteger": 1}) + cells := []struct { + name string + bind string + kind string + want string + }{ + {"intPlain", "42", "INTEGER", "42"}, + {"intRejected", "1e999999999999", "INTEGER", ""}, + {"intZeroScale", "0." + strings.Repeat("0", 20000), "INTEGER", "0"}, + {"decLeadingZero", "1e-00001", "DECIMAL", "0.1"}, + {"decNaN", "NaN", "DECIMAL", "0"}, + } + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + conn, err := svc.iso.Acquire(svc.ctx) + if err != nil { + svc.fail("acquire pinned connection", err) + } + if _, err := conn.Exec(svc.ctx, "SET plan_cache_mode = "+mode); err != nil { + conn.Release() + svc.fail("pin plan_cache_mode "+mode, err) + } + for _, cell := range cells { + plan := sql10Plan("Patient", map[string]any{"operand": cell.bind}, + sql10CallProjection(cell.name, sql10Call("cast", cell.kind, sql10Literal("operand"))), + ) + rendered := svc.renderLive(t, plan) + args := sql10DriverArgs(rendered) + if _, err := conn.Exec(svc.ctx, "EXPLAIN "+rendered.SQL, args...); err != nil { + conn.Release() + svc.fail("plan-time EXPLAIN raised under "+mode+" on "+cell.name, err) + } + rows, err := conn.Query(svc.ctx, rendered.SQL, args...) + if err != nil { + conn.Release() + svc.fail("execute under "+mode+" on "+cell.name, err) + } + var got []string + for rows.Next() { + values, err := rows.Values() + if err != nil { + rows.Close() + conn.Release() + svc.fail("read row under "+mode+" on "+cell.name, err) + } + for _, value := range values { + got = append(got, stringifyCell(value)) + } + } + err = rows.Err() + rows.Close() + if err != nil { + conn.Release() + svc.fail("finish statement under "+mode+" on "+cell.name, err) + } + if strings.Join(got, "|") != cell.want { + conn.Release() + svc.fail("cell "+cell.name+" under "+mode, + fmt.Errorf("row = %v, want %s\nSQL: %s", got, cell.want, rendered.SQL)) + } + } + conn.Release() + } +} + +func TestSQL10LiveDecimalTierStoredNumbers(t *testing.T) { + svc := newSQL10Service(t) + svc.insertResource(t, sql10Project, sql10Gen, "Patient", "p-num", sql10Path, + map[string]any{ + "multipleBirthInteger": json.RawMessage(`1e100000`), + "deceasedBoolean": json.RawMessage(`1` + strings.Repeat("0", 65535)), + }) + plan := sql10Plan("Patient", nil, + sql10CallProjection("bigNum", sql10Call("cast", "INTEGER", + sql10Extract(t, "Patient", "multipleBirthInteger", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull))), + sql10CallProjection("longNum", sql10Call("cast", "INTEGER", + sql10Extract(t, "Patient", "deceasedBoolean", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull))), + ) + svc.runOne(t, svc.renderLive(t, plan), "", "") +} + +func TestSQL10LiveArithmeticBandClampReadBack(t *testing.T) { + svc := newSQL10Service(t) + svc.insertResource(t, sql10Project, sql10Gen, "Patient", "p-band", sql10Path, + map[string]any{ + "multipleBirthInteger": json.RawMessage(`2`), + "active": json.RawMessage(`3`), + "deceasedBoolean": json.RawMessage(`1e98303`), + "deceasedDateTime": json.RawMessage(`1e98303`), + "text": map[string]any{ + "div": json.RawMessage(`1e131071`), + }, + "maritalStatus": json.RawMessage(`1e-16383`), + "birthDate": json.RawMessage(`9.9e131071`), + "gender": json.RawMessage(`9.9e131071`), + }) + num := func(path string) ir.PhysicalExpression { + return sql10Extract(t, "Patient", path, ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + } + plan := sql10Plan("Patient", nil, + sql10CallProjection("mulSanity", sql10Call("multiply", "", + num("multipleBirthInteger"), num("active"))), + sql10CallProjection("mulExt", sql10Call("multiply", "", + num("deceasedBoolean"), num("deceasedDateTime"))), + sql10CallProjection("divExt", sql10Call("divide", "", + num("text.div"), num("maritalStatus"))), + sql10CallProjection("addExt", sql10Call("add", "", + num("birthDate"), num("gender"))), + sql10CallProjection("divRecip", sql10Call("divide", "", + num("maritalStatus"), num("text.div"))), + ) + svc.runOne(t, svc.renderLive(t, plan), "6", "", "", "", "0") +} + +func TestSQL10LiveComparisonCrossTypeReadBack(t *testing.T) { + svc := newSQL10Service(t) + svc.insertResource(t, sql10Project, sql10Gen, "Patient", "p-cmp", sql10Path, + map[string]any{ + "multipleBirthInteger": 65, "gender": "65", "active": false, + "deceasedBoolean": 1, + "text": map[string]any{"status": "", "div": []any{float64(1)}}, + "managingOrganization": map[string]any{}, + "maritalStatus": map[string]any{"a": nil}, + }) + svc.insertResource(t, sql10Project, sql10Gen, "Observation", "p-cmp", sql10Path, nil) + + binds := map[string]any{ + "str65": "65", "sixtyfive": int64(65), "zero": int64(0), + "onepointzero": float64(1), "nul": nil, + "arrpad": json.RawMessage(`[1,null]`), "arrplain": json.RawMessage(`[1]`), + } + num := sql10Extract(t, "Patient", "multipleBirthInteger", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + numstr := sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + flag := sql10Extract(t, "Patient", "active", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + empty := sql10Extract(t, "Patient", "text.status", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + one := sql10Extract(t, "Patient", "deceasedBoolean", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + arr := sql10Extract(t, "Patient", "text.div", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + obj1 := sql10Extract(t, "Patient", "managingOrganization", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + obj2 := sql10Extract(t, "Patient", "maritalStatus", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + absent := sql10Extract(t, "Patient", "deceasedDateTime", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + + plan := sql10Plan("Patient", binds, + sql10CallProjection("eq65str", sql10Call("eq", "", num, sql10Literal("str65"))), + sql10CallProjection("eq65num", sql10Call("eq", "", num, sql10Literal("sixtyfive"))), + sql10CallProjection("neq65str", sql10Call("neq", "", num, sql10Literal("str65"))), + sql10CallProjection("eqFalse0", sql10Call("eq", "", flag, sql10Literal("zero"))), + sql10CallProjection("eqEmpty0", sql10Call("eq", "", empty, sql10Literal("zero"))), + sql10CallProjection("eqAbsent0", sql10Call("eq", "", absent, sql10Literal("zero"))), + sql10CallProjection("eqAbsentNull", sql10Call("eq", "", absent, sql10Literal("nul"))), + sql10CallProjection("eq1eq10", sql10Call("eq", "", one, sql10Literal("onepointzero"))), + sql10CallProjection("gtStrNum", sql10Call("gt", "", numstr, num)), + sql10CallProjection("ltNumStr", sql10Call("lt", "", num, numstr)), + sql10CallProjection("ltFalseNum", sql10Call("lt", "", flag, num)), + sql10CallProjection("ltNullNum", sql10Call("lt", "", absent, num)), + sql10CallProjection("eqArrPad", sql10Call("eq", "", arr, sql10Literal("arrpad"))), + sql10CallProjection("eqArrSame", sql10Call("eq", "", arr, sql10Literal("arrplain"))), + sql10CallProjection("eqObjNullAttr", sql10Call("eq", "", obj1, obj2)), + sql10CallProjection("gtArrNum", sql10Call("gt", "", arr, num)), + ) + svc.runOne(t, svc.renderLive(t, plan), + "false", "true", "true", "false", "false", "false", "true", "true", + "true", "true", "true", "true", + "false", "true", "false", "") + + witness := svc.run(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource"`+ + ` WHERE project=$1 AND dataset_generation=$2 AND resource_type='Observation' AND logical_id='p-cmp'`, + sql10Project, sql10Gen) + if len(witness) != 1 || witness[0][0] != "1" { + t.Fatalf("decoy must exist: %v", witness) + } +} + +func TestSQL10LiveAssertBarrier(t *testing.T) { + svc := newSQL10Service(t) + const message = "gender must be a non-empty string" + svc.insertResource(t, "sql10-ok", sql10Gen, "Patient", "p-ok", sql10Path, + map[string]any{"gender": "female"}) + svc.insertResource(t, "sql10-bad", sql10Gen, "Patient", "p-bad", sql10Path, + map[string]any{"gender": false}) + svc.insertResource(t, "sql10-lazy", sql10Gen, "Patient", "p-lazy", sql10Path, + map[string]any{"gender": false}) + + falsyCond := func() ir.PhysicalExpression { + return sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + } + + okPlan := sql10Plan("Patient", map[string]any{ + "project": "sql10-ok", "amsg": message, + }, sql10CallProjection("checked", sql10Call("assert", "", falsyCond(), sql10Literal("amsg")))) + okRendered := svc.renderLive(t, okPlan) + svc.explain(t, okRendered) + svc.runOne(t, okRendered, "true") + + badPlan := sql10Plan("Patient", map[string]any{ + "project": "sql10-bad", "amsg": message, + }, sql10CallProjection("checked", sql10Call("assert", "", falsyCond(), sql10Literal("amsg")))) + badRendered := svc.renderLive(t, badPlan) + svc.explain(t, badRendered) + svc.expectRaise(t, badRendered, "22P02", "ASSERT:", message) + + lazyPlan := sql10Plan("Patient", map[string]any{ + "project": "sql10-lazy", "amsg": message, "fl": false, + }, sql10CallProjection("guarded", sql10Call("and", "", + sql10Literal("fl"), + sql10Nested(sql10Call("assert", "", falsyCond(), sql10Literal("amsg")))))) + svc.runOne(t, svc.renderLive(t, lazyPlan), "false") +} + +func TestSQL10LiveDistinctPresentOneGuards(t *testing.T) { + svc := newSQL10Service(t) + scenarios := []struct { + project string + logicalID string + gender any + want string + wantTag string + }{ + {"sql10-dp-single", "d-1", []string{"female"}, "female", ""}, + {"sql10-dp-nested", "d-1", [][]string{{"male"}}, "male", ""}, + {"sql10-dp-empty", "d-1", []string{}, "", ""}, + {"sql10-dp-nulls", "d-1", []any{nil, "x"}, "x", ""}, + {"sql10-dp-scalar", "d-1", "scalar", "", ""}, + {"sql10-dp-multi", "d-1", []string{"female", "male"}, "", distinctPresentGuardMultipleValues}, + {"sql10-dp-typemis", "d-1", []any{"a", 1}, "", distinctPresentGuardTypeMismatch}, + } + for _, scenario := range scenarios { + svc.insertResource(t, scenario.project, sql10Gen, "Patient", scenario.logicalID, sql10Path, + map[string]any{"gender": scenario.gender}) + plan := sql10Plan("Patient", map[string]any{"project": scenario.project}, + sql10CallProjection("one", sql10Call("distinct_present_one", "STRING", + sql10Extract(t, "Patient", "gender", ir.PhysicalArrayCardinality, ir.PhysicalEmptyOnNull)))) + rendered := svc.renderLive(t, plan) + if scenario.wantTag == "" { + svc.runOne(t, rendered, scenario.want) + continue + } + planText := svc.explain(t, rendered) + if !strings.Contains(planText, "SubPlan") && !strings.Contains(planText, "Function Scan") { + t.Fatalf("%s: guard subplan missing from plan:\n%s", scenario.project, planText) + } + svc.expectRaise(t, rendered, "22P02", scenario.wantTag) + } +} diff --git a/internal/dataframe/compiler/render/postgres/calls_test.go b/internal/dataframe/compiler/render/postgres/calls_test.go new file mode 100644 index 000000000..7e842041e --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/calls_test.go @@ -0,0 +1,1233 @@ +package postgres + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "math/big" + "reflect" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" +) + +const ( + sql10Project = "sql10-proj" + sql10Gen = "gen-1" + sql10Path = "FHIR/Patient" +) + +func sql10Plan(resourceType string, extraBinds map[string]any, projections ...ir.PhysicalProjection) ir.PhysicalPlan { + binds := map[string]any{ + "root_collection": resourceType, + "project": sql10Project, + "dataset_generation": sql10Gen, + "auth_resource_paths": []string{sql10Path}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + } + for key, value := range extraBinds { + binds[key] = value + } + return ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: binds, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}}, + }, + } +} + +func sql10CallProjection(name string, expression *ir.PhysicalExpression) ir.PhysicalProjection { + return ir.PhysicalProjection{Name: name, Expression: expression} +} + +func sql10Call(name, targetKind string, args ...ir.PhysicalExpression) *ir.PhysicalExpression { + return &ir.PhysicalExpression{ + Kind: ir.PhysicalCallExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Call: &ir.PhysicalCall{Name: name, TargetKind: targetKind, Args: args}, + } +} + +func sql10Nested(call *ir.PhysicalExpression) ir.PhysicalExpression { return *call } + +func sql10Literal(bindKey string) ir.PhysicalExpression { + return ir.PhysicalExpression{ + Kind: ir.PhysicalLiteralExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Literal: &ir.PhysicalLiteral{BindKey: bindKey}, + } +} + +func sql10ArrayLiteral(bindKey string) ir.PhysicalExpression { + return ir.PhysicalExpression{ + Kind: ir.PhysicalLiteralExpression, + Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Literal: &ir.PhysicalLiteral{BindKey: bindKey}, + } +} + +func sql10Value(path ...string) ir.PhysicalExpression { + return ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{Variable: "root", Path: path}, + } +} + +func sql10Aggregate(cardinality ir.PhysicalCardinality) ir.PhysicalExpression { + behavior := ir.PhysicalPreserveNull + if cardinality == ir.PhysicalArrayCardinality { + behavior = ir.PhysicalEmptyOnNull + } + return ir.PhysicalExpression{ + Kind: ir.PhysicalAggregateExpression, + Cardinality: cardinality, + NullBehavior: behavior, + Aggregate: &ir.PhysicalAggregate{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + Operation: ir.PhysicalCountAggregate, + }, + } +} + +func sql10Selector(t *testing.T, path string) spec.Selector { + t.Helper() + selector, err := spec.ParseSelector(path) + if err != nil { + t.Fatalf("ParseSelector(%q): %v", path, err) + } + return selector +} + +func sql10Extract(t *testing.T, resourceType, path string, + cardinality ir.PhysicalCardinality, behavior ir.PhysicalNullBehavior) ir.PhysicalExpression { + t.Helper() + return ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: cardinality, + NullBehavior: behavior, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: resourceType, + Selector: sql10Selector(t, path), + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + } +} + +func sql10RenderOK(t *testing.T, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + for index := 1; index <= len(rendered.Args); index++ { + if !strings.Contains(rendered.SQL, fmt.Sprintf("$%d", index)) { + t.Fatalf("parameter $%d is never referenced: %s", index, rendered.SQL) + } + } + return rendered +} + +func sql10ScopeArgs() []command.PostgreSQLParameter { + return []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: sql10Project}, + {Type: command.PostgreSQLParameterText, Value: sql10Gen}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["FHIR/Patient"]`)}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + } +} + +func sql10Statement(selects string) string { + return `SELECT ` + selects + + ` FROM "pgir02_it_unit"."fhir_resource" AS r0` + + ` WHERE r0."project" = $1` + + ` AND r0."dataset_generation" = $2` + + ` AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND r0."resource_type" = $4` + + ` ORDER BY r0."resource_seq" ASC` +} + +func sql10Reject(t *testing.T, plan ir.PhysicalPlan, context string, want ...string) { + t.Helper() + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("%s: error = %v (%T), want ErrUnsupportedPostgresPlan", context, err, err) + } + for _, fragment := range want { + if !strings.Contains(err.Error(), fragment) { + t.Fatalf("%s: error %q lacks %q", context, err.Error(), fragment) + } + } +} + +func sql10RejectAny(t *testing.T, plan ir.PhysicalPlan, context string, want ...string) { + t.Helper() + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil { + t.Fatalf("%s: render succeeded, want refusal", context) + } + for _, fragment := range want { + if !strings.Contains(err.Error(), fragment) { + t.Fatalf("%s: error %q lacks %q", context, err.Error(), fragment) + } + } +} + +func sql10NumericText(parameter string) string { + return "((" + parameter + ") #>> '{}')::numeric" +} + +const ( + sql10WantOverflowBound = "179769313486231580793728971405303415079934132710037826936173778980444968292764750946649017977587207096330286416692887910946555547851940402630657488671505820681908902000708383676273854845817711531764475730270069855571366959622842914819860834936475292719074168444365510704342711559699508093042880177904174497792" + sql10WantAddPassUpper = "179769313486231560835325876058105298516207002341652166261661174625869553267292326574530099287946549246750631490335877017522087105926987962906277604735569213290190919152394180476217125334960946356387261286640198029037799514183602981511756283727771403830521483963923935633133642802139091669457927874464075218944" + sql10WantMulDivPassUpper = "179769313486231500960116590016510948825025611236495184238123361562143308190875053458173343219024575698011666711264844337248681780152130643733137952927759391115036970607451570876046936802390650830255617955750582549437097177865883181587442630101659737164863430522599210419506436529457842398703070964143777382400" + sql10WantUnderflowFloor = "2.5E-324" + sql10WantSubnormalFloor = "4.9406564584124655E-324" + sql10WantNormalFloor = "2.225073858507201383090232717332404064219215980462331830553327416887204434813918195854283159012511020564067339731035811005152434161553460108856012385377718821130777993532002330479610147442583636071921565046942503734208375250806650616658158948720491179968591639648500635908770118304874799780887753749949451580451605050915399856582470818645113537935804992115981085766051992433352114352390148795699609591288891602992641511063466313393663477586513029371762047325631781485664350872122828637642044846811407613911477062801689853244110024161447421618567166150540154285084716752901903161322778896729707373123334086988983175067838846926092773977972858659654941091369095406136467568702398678315290680984617210924625396728515625E-308" + sql10WantBandClampCap = "1e400" +) + +func sql10WantNumericToJSONB(expression string) string { + return `to_jsonb(CASE WHEN abs(` + expression + `) < '` + sql10WantOverflowBound + `'::numeric` + + ` THEN CASE WHEN abs(` + expression + `) >= '` + sql10WantUnderflowFloor + `'::numeric` + + ` THEN (` + expression + `)::double precision ELSE 0::double precision END END)` +} + +func sql10WantDoubleOperand(expression string) string { + return `(CASE WHEN abs(` + expression + `) < '` + sql10WantOverflowBound + `'::numeric` + + ` THEN CASE WHEN abs(` + expression + `) >= '` + sql10WantUnderflowFloor + `'::numeric` + + ` THEN (` + expression + `)::double precision ELSE 0::double precision END` + + ` ELSE NULL END)` +} + +func sql10WantDoubleDivisor(expression string) string { + return `(CASE WHEN abs(` + expression + `) < '` + sql10WantOverflowBound + `'::numeric` + + ` THEN CASE WHEN abs(` + expression + `) >= '` + sql10WantNormalFloor + `'::numeric` + + ` THEN (` + expression + `)::double precision ELSE NULL END` + + ` ELSE NULL END)` +} + +func sql10WantBandOperandClamp(operand string) string { + return "(CASE" + + " WHEN " + operand + " > '" + sql10WantBandClampCap + "'::numeric THEN '" + sql10WantBandClampCap + "'::numeric" + + " WHEN " + operand + " < '-" + sql10WantBandClampCap + "'::numeric THEN '-" + sql10WantBandClampCap + "'::numeric" + + " ELSE " + operand + " END)" +} + +func sql10WantArithmetic(operator, left, right string) string { + clampedLeft, clampedRight := sql10WantBandOperandClamp(left), sql10WantBandOperandClamp(right) + switch operator { + case "+", "-": + return `to_jsonb(CASE WHEN abs((` + clampedLeft + ` ` + operator + ` ` + clampedRight + `)) < '` + sql10WantAddPassUpper + `'::numeric` + + ` THEN ` + sql10WantDoubleOperand(left) + ` ` + operator + ` ` + sql10WantDoubleOperand(right) + + ` ELSE NULL END)` + case "*": + magnitude := `abs((` + clampedLeft + ` * ` + clampedRight + `))` + return `to_jsonb(CASE WHEN ` + magnitude + ` < '` + sql10WantMulDivPassUpper + `'::numeric` + + ` AND ` + magnitude + ` >= '` + sql10WantSubnormalFloor + `'::numeric` + + ` THEN ` + sql10WantDoubleOperand(left) + ` * ` + sql10WantDoubleOperand(right) + + ` WHEN ` + magnitude + ` < '` + sql10WantSubnormalFloor + `'::numeric` + + ` THEN 0::double precision ELSE NULL END)` + default: + quotient := `((` + clampedLeft + ` + 0E-400) / NULLIF(` + clampedRight + `, 0))` + magnitude := `abs(` + quotient + `)` + return `to_jsonb(CASE WHEN ` + magnitude + ` < '` + sql10WantMulDivPassUpper + `'::numeric` + + ` AND ` + magnitude + ` >= '` + sql10WantSubnormalFloor + `'::numeric` + + ` THEN ` + sql10WantDoubleOperand(left) + ` / ` + sql10WantDoubleDivisor(right) + + ` WHEN ` + magnitude + ` < '` + sql10WantSubnormalFloor + `'::numeric` + + ` THEN 0::double precision ELSE NULL END)` + } +} + +func TestCallsWhitespaceSingleSource(t *testing.T) { + if callDecimalWhitespace != " \t\n\v\f\r" { + t.Fatalf("whitespace source = %q, want the explicit ASCII isspace set", callDecimalWhitespace) + } + wantClass := `[ ` + "\t\n\v\f\r" + `]` + if got := callWhitespaceRegexClass(); got != wantClass { + t.Fatalf("regex class = %q, want %q", got, wantClass) + } + if got := strings.Count(callDecimalStringPattern, wantClass); got != 2 { + t.Fatalf("grammar admits the derived class %d times, want 2: %q", got, callDecimalStringPattern) + } + for _, forbidden := range []string{"[[:space:]]", "[[:blank:]]", `\s`, `\S`} { + if strings.Contains(callDecimalStringPattern, forbidden) { + t.Fatalf("grammar carries %q instead of the derived class: %q", forbidden, callDecimalStringPattern) + } + } + if got, want := callDecimalTrimCharacters, quoteSQLLiteral(callDecimalWhitespace); got != want { + t.Fatalf("btrim argument = %q, want the derived %q", got, want) + } + for _, pattern := range []string{ + callDecimalZeroPattern, callDecimalUnderflowPattern, + } { + for _, forbidden := range []string{"[[:space:]]", "[[:blank:]]", `\s`, `\S`} { + if strings.Contains(pattern, forbidden) { + t.Fatalf("tier grammar carries %q instead of the derived class: %q", forbidden, pattern) + } + } + } + for name, fragment := range map[string]string{ + "coercion": callNumericCoercion("o"), + "to_number": callToNumberExpression("o"), + } { + if !strings.Contains(fragment, quoteSQLLiteral(callDecimalStringPattern)) || + !strings.Contains(fragment, callDecimalTrimCharacters) { + t.Fatalf("%s guard arm does not embed the shared whitespace construction: %s", name, fragment) + } + } +} + +func TestCallsDecimalTierSingleSource(t *testing.T) { + tenTo := func(n int) int { + v := 1 + for range n { + v *= 10 + } + return v + } + if callDecimalUnderflowIntegerDigits != 99 || callDecimalUnderflowMinExponentDigits != 5 { + t.Fatalf("underflow constants drifted from the registered values: %d %d", + callDecimalUnderflowIntegerDigits, callDecimalUnderflowMinExponentDigits) + } + if margin := tenTo(callDecimalUnderflowMinExponentDigits-1) - callDecimalUnderflowIntegerDigits; margin <= 324 { + t.Fatalf("underflow tiers are not provably IEEE-zero: 10^(%d-%d) = 10^-%d is not below the subnormal floor", + callDecimalUnderflowMinExponentDigits-1, callDecimalUnderflowIntegerDigits, margin) + } + derived := map[string]bool{ + "[0-9]{1," + strconv.Itoa(callDecimalUnderflowIntegerDigits) + "}": true, + "[0-9]{" + strconv.Itoa(callDecimalUnderflowMinExponentDigits-1) + ",}": true, + } + if !strings.Contains(callDecimalUnderflowPattern, `[eE]-[1-9]`) { + t.Fatalf("underflow exponent lost its nonzero leading digit: %q", callDecimalUnderflowPattern) + } + quantifier := regexp.MustCompile(`\[(?:0-9|0)\]\{(\d+)(?:,(\d*))?\}`) + tiers := map[string]string{ + "underflow-zero": callDecimalZeroPattern, + "underflow-tiny": callDecimalUnderflowPattern, + "unbounded": callDecimalStringPattern, + } + for name, pattern := range tiers { + for _, run := range quantifier.FindAllStringSubmatch(pattern, -1) { + if !derived[run[0]] { + t.Fatalf("%s tier carries undervived digit run %q: %q", name, run[0], pattern) + } + for _, digits := range run[1:] { + if digits == "" { + continue + } + count, err := strconv.Atoi(digits) + if err != nil || count > 255 { + t.Fatalf("%s tier repetition count %q exceeds the PG18-measured 255 limit: %q", name, digits, pattern) + } + } + } + } + wide := regexp.MustCompile(callDecimalStringPattern) + zero := regexp.MustCompile(callDecimalZeroPattern) + tiny := regexp.MustCompile(callDecimalUnderflowPattern) + for _, text := range []string{ + "0", "-0", "0.", "0.0", ".0", "00", " 0.000\t", "0e999999999999", + "1e-999999999999", "-99.9e-10000", "1" + strings.Repeat("0", 98) + "e-99999", + } { + if (zero.MatchString(text) || tiny.MatchString(text)) && !wide.MatchString(text) { + t.Fatalf("registered-0 text %q falls outside the registered shape", text) + } + } + if !tiny.MatchString("1" + strings.Repeat("0", callDecimalUnderflowIntegerDigits-1) + "e-99999") { + t.Fatal("tiny tier must admit a mantissa of exactly the registered digit bound") + } + if tiny.MatchString("1" + strings.Repeat("0", callDecimalUnderflowIntegerDigits) + "e-99999") { + t.Fatal("tiny tier admits a mantissa beyond the registered digit bound") + } + if tiny.MatchString("1e-9999") { + t.Fatal("tiny tier admits an exponent below the registered digit bound") + } + for _, spoof := range []string{ + "1e-00001", + "0." + strings.Repeat("9", 18000) + "e-00001", + "1." + strings.Repeat("9", 20000) + "e-00001", + strings.Repeat("9", 99) + "." + strings.Repeat("9", 20000) + "e-00001", + "9.9e-09999", + } { + if tiny.MatchString(spoof) { + t.Fatalf("provable-underflow arm admits leading-zero exponent %q", spoof) + } + } + if !wide.MatchString("1e-00001") { + t.Fatal(`"1e-00001" must stay inside the registered shape (the cast arm answers it)`) + } + for name, cell := range map[string]struct { + fragment string + wantOracle int + }{ + "coercion": {callNumericCoercion("o"), 1}, + "to_number": {callToNumberExpression("o"), 9}, + } { + fragment := cell.fragment + btrim := "btrim((o #>> '{}'), " + callDecimalTrimCharacters + ")" + ordered := []string{ + quoteSQLLiteral(callDecimalStringPattern) + " THEN CASE WHEN ", + "pg_input_is_valid(" + btrim + ", 'numeric') THEN (" + btrim + + " || COALESCE(current_setting(" + quoteSQLLiteral(guardBarrierGUC) + ", true), ''))::numeric", + quoteSQLLiteral(callDecimalZeroPattern), + quoteSQLLiteral(callDecimalUnderflowPattern) + ") THEN 0", + " ELSE NULL END", + } + last := -1 + for _, marker := range ordered { + at := strings.Index(fragment, marker) + if at <= last { + t.Fatalf("%s arms missing or out of registered order at %q: %s", name, marker, fragment) + } + last = at + } + if got := strings.Count(fragment, "pg_input_is_valid("); got != cell.wantOracle { + t.Fatalf("%s embeds the input oracle %d times, want %d: %s", name, got, cell.wantOracle, fragment) + } + } +} + +func TestCallsBandTestClamps(t *testing.T) { + if callBandClampCap != sql10WantBandClampCap { + t.Fatalf("emitter cap %s diverged from the mirror %s", callBandClampCap, sql10WantBandClampCap) + } + if callBandClampCap != "1e400" { + t.Fatalf("clamp cap drifted from the registered value: %s", callBandClampCap) + } + parse := func(s string) *big.Float { + v, _, err := big.ParseFloat(s, 10, 256, big.ToNearestEven) + if err != nil { + t.Fatalf("parse %q: %v", s, err) + } + return v + } + capV := parse(callBandClampCap) + for _, upper := range []string{callDoubleAddPassUpper, callDoubleMulDivPassUpper, callDoubleOverflowBound} { + if capV.Cmp(parse(upper)) <= 0 { + t.Fatalf("clamp cap %s does not exceed the boundary %s", callBandClampCap, upper) + } + } + ceiling := parse("1e131071") + if product := new(big.Float).Mul(capV, capV); product.Cmp(ceiling) >= 0 { + t.Fatalf("worst capped product %s reaches the measured numeric ceiling", product.Text('g', 6)) + } + if quotient := new(big.Float).Quo(capV, parse("1e-16383")); quotient.Cmp(ceiling) >= 0 { + t.Fatalf("worst capped quotient %s reaches the measured numeric ceiling", quotient.Text('g', 6)) + } + if sum := new(big.Float).Mul(capV, big.NewFloat(2)); sum.Cmp(ceiling) >= 0 { + t.Fatalf("worst capped sum %s reaches the measured numeric ceiling", sum.Text('g', 6)) + } + clamped := callBandOperandClamp("o") + for _, forbidden := range []string{" + ", " - ", " * ", " / ", "::double", "::float"} { + if strings.Contains(clamped, forbidden) { + t.Fatalf("clamp performs %q: %s", forbidden, clamped) + } + } + if !strings.HasSuffix(clamped, " ELSE o END)") { + t.Fatalf("clamp does not pass zero and NULL through unchanged: %s", clamped) + } + if got := strings.Count(clamped, "'"+callBandClampCap+"'::numeric"); got != 2 { + t.Fatalf("clamp embeds the cap literal %d times, want 2: %s", got, clamped) + } + if got := strings.Count(clamped, "'-"+callBandClampCap+"'::numeric"); got != 2 { + t.Fatalf("clamp embeds the negated cap literal %d times, want 2: %s", got, clamped) + } +} + +func TestCallsDivisorNormalBand(t *testing.T) { + if callDoubleNormalFloor != sql10WantNormalFloor { + t.Fatalf("emitter normal floor %s diverged from the mirror %s", callDoubleNormalFloor, sql10WantNormalFloor) + } + bits := math.Float64bits(func() float64 { + v, err := strconv.ParseFloat(callDoubleNormalFloor, 64) + if err != nil { + t.Fatalf("parse normal floor %q: %v", callDoubleNormalFloor, err) + } + return v + }()) + if bits != 0x0010000000000000 { + t.Fatalf("normal floor %s is not DBL_MIN (bits %#x)", callDoubleNormalFloor, bits) + } + parse := func(s string) *big.Float { + v, _, err := big.ParseFloat(s, 10, 256, big.ToNearestEven) + if err != nil { + t.Fatalf("parse %q: %v", s, err) + } + return v + } + if parse(callDoubleSubnormalFloor).Cmp(parse(callDoubleNormalFloor)) >= 0 { + t.Fatal("divisor normal floor must sit above the operation subnormal floor") + } + if parse(callDoubleNormalFloor).Cmp(parse(callDoubleMulDivPassUpper)) >= 0 { + t.Fatal("divisor normal floor must sit below the pass upper") + } + divisor := callDoubleDivisor("o") + if strings.Contains(divisor, ")::double precision)::numeric = ") { + t.Fatalf("divisor arm resurrected the tautological round-trip conjunct: %s", divisor) + } + if got := strings.Count(divisor, "(o)::double precision"); got != 1 { + t.Fatalf("divisor arm casts %d times, want exactly 1: %s", got, divisor) + } + if got := strings.Count(divisor, "'"+callDoubleNormalFloor+"'::numeric"); got != 1 { + t.Fatalf("divisor arm embeds the normal floor %d times, want 1: %s", got, divisor) + } + if got := strings.Count(divisor, "'"+callDoubleOverflowBound+"'::numeric"); got != 1 { + t.Fatalf("divisor arm embeds the overflow bound %d times, want 1: %s", got, divisor) + } + if !strings.Contains(divisor, "abs(o) >= '"+callDoubleNormalFloor+"'::numeric") { + t.Fatalf("divisor gate does not refuse the whole subnormal band by magnitude: %s", divisor) + } +} + +func TestCallsGoldenArithmeticAddDivide(t *testing.T) { + left, right := sql10NumericText("$5::jsonb"), sql10NumericText("$6::jsonb") + plan := sql10Plan("Patient", map[string]any{"a": int64(2), "b": float64(1.5)}, + sql10CallProjection("sum", sql10Call("add", "", sql10Literal("a"), sql10Literal("b"))), + ) + rendered := sql10RenderOK(t, plan) + wantSQL := sql10Statement(`(` + sql10WantArithmetic("+", left, right) + `) AS "c0"`) + if rendered.SQL != wantSQL { + t.Fatalf("add SQL =\n%s\nwant\n%s", rendered.SQL, wantSQL) + } + wantArgs := append(sql10ScopeArgs(), + command.PostgreSQLParameter{Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`2`)}, + command.PostgreSQLParameter{Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`1.5`)}, + ) + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("add args = %#v, want %#v", rendered.Args, wantArgs) + } + if !reflect.DeepEqual(rendered.OutputColumns, []string{"sum"}) { + t.Fatalf("add output columns = %v", rendered.OutputColumns) + } + + dividePlan := sql10Plan("Patient", map[string]any{"a": int64(2), "b": float64(1.5)}, + sql10CallProjection("ratio", sql10Call("divide", "", sql10Literal("a"), sql10Literal("b"))), + ) + divideRendered := sql10RenderOK(t, dividePlan) + wantDivide := sql10Statement(`(` + sql10WantArithmetic("/", left, right) + `) AS "c0"`) + if divideRendered.SQL != wantDivide { + t.Fatalf("divide SQL =\n%s\nwant\n%s", divideRendered.SQL, wantDivide) + } +} + +func TestCallsGoldenArithmeticDynamicOperandCoercion(t *testing.T) { + number := sql10NumericText("$6::jsonb") + o := `(r0."payload" #> $5::text[])` + coercion := `(CASE WHEN ` + o + ` IS NULL OR jsonb_typeof(` + o + `) = 'null' THEN NULL` + + ` WHEN jsonb_typeof(` + o + `) = 'number' THEN (` + o + ` #>> '{}')::numeric` + + sql10WantDecimalTierArms(o) + + ` ELSE NULL END)` + plan := sql10Plan("Patient", map[string]any{"a": int64(3)}, + sql10CallProjection("scaled", sql10Call("multiply", "", + sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("a"))), + ) + rendered := sql10RenderOK(t, plan) + wantSQL := sql10Statement(`(` + sql10WantArithmetic("*", coercion, number) + `) AS "c0"`) + if rendered.SQL != wantSQL { + t.Fatalf("multiply SQL =\n%s\nwant\n%s", rendered.SQL, wantSQL) + } + wantArgs := append(sql10ScopeArgs(), + command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: "{gender}"}, + command.PostgreSQLParameter{Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`3`)}, + ) + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("multiply args = %#v, want %#v", rendered.Args, wantArgs) + } +} + +func sql10WantDecimalTierArms(operand string) string { + text := "(" + operand + ` #>> '{}')` + btrim := "btrim(" + text + ", " + callDecimalTrimCharacters + ")" + castArm := "(" + btrim + " || COALESCE(current_setting(" + quoteSQLLiteral(guardBarrierGUC) + ", true), ''))::numeric" + return " WHEN jsonb_typeof(" + operand + ") = 'string' AND " + text + " ~ '" + callDecimalStringPattern + + "' THEN CASE WHEN pg_input_is_valid(" + btrim + ", 'numeric') THEN " + castArm + + " WHEN (" + text + " ~ '" + callDecimalZeroPattern + "' OR " + text + " ~ '" + callDecimalUnderflowPattern + "') THEN 0" + + " ELSE NULL END" +} + +func sql10WantToNumberDouble(operand string, depth int) string { + if depth >= callToNumberMaxRecursion { + return "0" + } + element := sql10WantToNumberDouble("(("+operand+") -> 0)", depth+1) + return "CASE" + + " WHEN " + operand + " IS NULL OR jsonb_typeof(" + operand + ") = 'null' THEN 0" + + " WHEN jsonb_typeof(" + operand + ") = 'boolean' THEN CASE WHEN " + operand + " = 'true'::jsonb THEN 1 ELSE 0 END" + + " WHEN jsonb_typeof(" + operand + ") = 'number' THEN (" + operand + " #>> '{}')::numeric" + + sql10WantDecimalTierArms(operand) + + " WHEN jsonb_typeof(" + operand + ") = 'string' THEN 0" + + " WHEN jsonb_typeof(" + operand + ") = 'array' THEN CASE" + + " WHEN jsonb_array_length(" + operand + ") = 0 THEN 0" + + " WHEN jsonb_array_length(" + operand + ") = 1 THEN " + element + + " ELSE 0 END" + + " ELSE 0 END" +} + +func TestCallsGoldenCastTables(t *testing.T) { + stringPlan := sql10Plan("Patient", nil, + sql10CallProjection("label", sql10Call("cast", "string", + sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull))), + ) + o := `(r0."payload" #> $5::text[])` + stringRendered := sql10RenderOK(t, stringPlan) + wantString := sql10Statement(`(CASE` + + ` WHEN ` + o + ` IS NULL OR jsonb_typeof(` + o + `) = 'null' THEN '""'::jsonb` + + ` WHEN jsonb_typeof(` + o + `) IN ('string', 'boolean', 'number')` + + ` THEN to_jsonb((` + o + `) #>> '{}')` + + ` ELSE NULL END) AS "c0"`) + if stringRendered.SQL != wantString { + t.Fatalf("cast string SQL =\n%s\nwant\n%s", stringRendered.SQL, wantString) + } + + numberPlan := sql10Plan("Patient", map[string]any{"v": "42"}, + sql10CallProjection("count", sql10Call("cast", "integer", sql10Literal("v"))), + ) + numberRendered := sql10RenderOK(t, numberPlan) + numberDouble := sql10WantToNumberDouble("$5::jsonb", 0) + wantNumber := sql10Statement(`(` + sql10WantNumericToJSONB(numberDouble) + `) AS "c0"`) + if numberRendered.SQL != wantNumber { + t.Fatalf("cast integer SQL =\n%s\nwant\n%s", numberRendered.SQL, wantNumber) + } + + booleanPlan := sql10Plan("Patient", nil, + sql10CallProjection("flag", sql10Call("cast", "boolean", + sql10Extract(t, "Patient", "active", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull))), + ) + booleanRendered := sql10RenderOK(t, booleanPlan) + a := `(r0."payload" #> $5::text[])` + wantBoolean := sql10Statement(`(CASE` + + ` WHEN ` + a + ` IS NULL OR jsonb_typeof(` + a + `) = 'null' THEN 'false'::jsonb` + + ` WHEN jsonb_typeof(` + a + `) = 'boolean' THEN ` + a + + ` WHEN jsonb_typeof(` + a + `) = 'number' THEN CASE WHEN ` + a + ` = '0'::jsonb THEN 'false'::jsonb ELSE 'true'::jsonb END` + + ` WHEN jsonb_typeof(` + a + `) = 'string' THEN CASE WHEN ` + a + ` = '""'::jsonb THEN 'false'::jsonb ELSE 'true'::jsonb END` + + ` ELSE 'true'::jsonb END) AS "c0"`) + if booleanRendered.SQL != wantBoolean { + t.Fatalf("cast boolean SQL =\n%s\nwant\n%s", booleanRendered.SQL, wantBoolean) + } +} + +func TestCallsGoldenCoalesceConditionalsLogical(t *testing.T) { + coalescePlan := sql10Plan("Patient", map[string]any{"s": "x", "n": nil}, + sql10CallProjection("pick", sql10Call("coalesce", "", + sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("s"), sql10Literal("n"))), + ) + coalesceRendered := sql10RenderOK(t, coalescePlan) + wantCoalesce := sql10Statement(`(COALESCE(NULLIF((r0."payload" #> $5::text[]), 'null'::jsonb), NULLIF($6::jsonb, 'null'::jsonb), NULLIF($7::jsonb, 'null'::jsonb))) AS "c0"`) + if coalesceRendered.SQL != wantCoalesce { + t.Fatalf("coalesce SQL =\n%s\nwant\n%s", coalesceRendered.SQL, wantCoalesce) + } + if len(coalesceRendered.Args) != 7 { + t.Fatalf("coalesce args = %d, want 7", len(coalesceRendered.Args)) + } + + ifPlan := sql10Plan("Patient", map[string]any{"t": true, "f": false}, + sql10CallProjection("choice", sql10Call("if", "", + sql10Extract(t, "Patient", "active", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("t"), sql10Literal("f"))), + ) + ifRendered := sql10RenderOK(t, ifPlan) + a := `(r0."payload" #> $5::text[])` + truthy := `NOT (` + a + ` IS NULL OR jsonb_typeof(` + a + `) = 'null' OR ` + a + + ` = 'false'::jsonb OR (jsonb_typeof(` + a + `) = 'number' AND ` + a + ` = '0'::jsonb) OR ` + a + ` = '""'::jsonb)` + wantIf := sql10Statement(`(CASE WHEN ` + truthy + ` THEN $6::jsonb ELSE $7::jsonb END) AS "c0"`) + if ifRendered.SQL != wantIf { + t.Fatalf("if SQL =\n%s\nwant\n%s", ifRendered.SQL, wantIf) + } + + casePlan := sql10Plan("Patient", map[string]any{"t": true}, + sql10CallProjection("recoded", sql10Call("case", "", + sql10Literal("t"), sql10Literal("t"))), + ) + caseRendered := sql10RenderOK(t, casePlan) + caseTruthy := `NOT ($5::jsonb IS NULL OR jsonb_typeof($5::jsonb) = 'null' OR $5::jsonb = 'false'::jsonb` + + ` OR (jsonb_typeof($5::jsonb) = 'number' AND $5::jsonb = '0'::jsonb) OR $5::jsonb = '""'::jsonb)` + wantCase := sql10Statement(`(CASE WHEN ` + caseTruthy + ` THEN $6::jsonb ELSE NULL END) AS "c0"`) + if caseRendered.SQL != wantCase { + t.Fatalf("case SQL =\n%s\nwant\n%s", caseRendered.SQL, wantCase) + } + if len(caseRendered.Args) != 6 { + t.Fatalf("case args = %d, want 6 (no stranded else parameter)", len(caseRendered.Args)) + } + + andPlan := sql10Plan("Patient", map[string]any{"a": int64(0), "b": "x"}, + sql10CallProjection("fallback_pair", sql10Call("and", "", sql10Literal("a"), sql10Literal("b"))), + ) + andRendered := sql10RenderOK(t, andPlan) + andTruthy := `NOT ($5::jsonb IS NULL OR jsonb_typeof($5::jsonb) = 'null' OR $5::jsonb = 'false'::jsonb` + + ` OR (jsonb_typeof($5::jsonb) = 'number' AND $5::jsonb = '0'::jsonb) OR $5::jsonb = '""'::jsonb)` + wantAnd := sql10Statement(`(CASE WHEN ` + andTruthy + ` THEN $6::jsonb ELSE $5::jsonb END) AS "c0"`) + if andRendered.SQL != wantAnd { + t.Fatalf("and SQL =\n%s\nwant\n%s", andRendered.SQL, wantAnd) + } + + notPlan := sql10Plan("Patient", nil, + sql10CallProjection("inverted", sql10Call("not", "", + sql10Extract(t, "Patient", "active", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull))), + ) + notRendered := sql10RenderOK(t, notPlan) + wantNot := sql10Statement(`(to_jsonb(NOT ` + truthy + `)) AS "c0"`) + if notRendered.SQL != wantNot { + t.Fatalf("not SQL =\n%s\nwant\n%s", notRendered.SQL, wantNot) + } +} + +func TestCallsGoldenComparisons(t *testing.T) { + eqPlan := sql10Plan("Patient", map[string]any{"s": "female"}, + sql10CallProjection("is_female", sql10Call("eq", "", + sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("s"))), + ) + eqRendered := sql10RenderOK(t, eqPlan) + wantEq := sql10Statement(`(to_jsonb(COALESCE((r0."payload" #> $5::text[]), 'null'::jsonb) = COALESCE($6::jsonb, 'null'::jsonb))) AS "c0"`) + if eqRendered.SQL != wantEq { + t.Fatalf("eq SQL =\n%s\nwant\n%s", eqRendered.SQL, wantEq) + } + + neqPlan := sql10Plan("Patient", map[string]any{"s": "female"}, + sql10CallProjection("other", sql10Call("neq", "", + sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("s"))), + ) + neqRendered := sql10RenderOK(t, neqPlan) + wantNeq := sql10Statement(`(to_jsonb(NOT (COALESCE((r0."payload" #> $5::text[]), 'null'::jsonb) = COALESCE($6::jsonb, 'null'::jsonb)))) AS "c0"`) + if neqRendered.SQL != wantNeq { + t.Fatalf("neq SQL =\n%s\nwant\n%s", neqRendered.SQL, wantNeq) + } + + gtPlan := sql10Plan("Patient", map[string]any{"n": float64(1)}, + sql10CallProjection("over_one", sql10Call("gt", "", + sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("n"))), + ) + gtRendered := sql10RenderOK(t, gtPlan) + lhs := `(r0."payload" #> $5::text[])` + rank := func(operand string) string { + return `(CASE WHEN ` + operand + ` IS NULL OR jsonb_typeof(` + operand + `) = 'null' THEN 0` + + ` WHEN jsonb_typeof(` + operand + `) = 'boolean' THEN 1` + + ` WHEN jsonb_typeof(` + operand + `) = 'number' THEN 2` + + ` WHEN jsonb_typeof(` + operand + `) = 'string' THEN 3` + + ` ELSE 4 END)` + } + rankLeft, rankRight := rank(lhs), rank("$6::jsonb") + wantGt := sql10Statement(`(to_jsonb(CASE` + + ` WHEN ` + rankLeft + ` >= 4 OR ` + rankRight + ` >= 4 THEN NULL` + + ` WHEN ` + rankLeft + ` <> ` + rankRight + ` THEN ` + rankLeft + ` > ` + rankRight + + ` ELSE (COALESCE(` + lhs + `, 'null'::jsonb)) > (COALESCE($6::jsonb, 'null'::jsonb)) END)) AS "c0"`) + if gtRendered.SQL != wantGt { + t.Fatalf("gt SQL =\n%s\nwant\n%s", gtRendered.SQL, wantGt) + } +} + +func TestCallsGoldenAssertBarrier(t *testing.T) { + plan := sql10Plan("Patient", map[string]any{"msg": "MISSING_GENDER"}, + sql10CallProjection("checked", sql10Call("assert", "", + sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("msg"))), + ) + rendered := sql10RenderOK(t, plan) + a := `(r0."payload" #> $5::text[])` + truthy := `NOT (` + a + ` IS NULL OR jsonb_typeof(` + a + `) = 'null' OR ` + a + + ` = 'false'::jsonb OR (jsonb_typeof(` + a + `) = 'number' AND ` + a + ` = '0'::jsonb) OR ` + a + ` = '""'::jsonb)` + want := sql10Statement(`(CASE WHEN ` + truthy + ` THEN 'true'::jsonb ELSE ` + + `to_jsonb(CAST('ASSERT:' || $6 || COALESCE(current_setting('loom.explorer_guard_barrier', true), '') AS integer)) END) AS "c0"`) + if rendered.SQL != want { + t.Fatalf("assert SQL =\n%s\nwant\n%s", rendered.SQL, want) + } + wantArgs := append(sql10ScopeArgs(), + command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: "{gender}"}, + command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: "MISSING_GENDER"}, + ) + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("assert args = %#v, want %#v", rendered.Args, wantArgs) + } +} + +func TestCallsGoldenDistinctPresentOne(t *testing.T) { + plan := sql10Plan("Patient", nil, + sql10CallProjection("one_system", sql10Call("distinct_present_one", "STRING", + sql10Extract(t, "Patient", "gender", ir.PhysicalArrayCardinality, ir.PhysicalEmptyOnNull))), + ) + rendered := sql10RenderOK(t, plan) + base := `COALESCE(NULLIF((r0."payload" #> $5::text[]), 'null'::jsonb), '""'::jsonb)` + want := sql10Statement(`((SELECT CASE` + + ` WHEN bool_or(jsonb_typeof(loom_dp_item.item) <> 'string') THEN to_jsonb(CAST('CONSTRUCTION_ROW_VALUE_TYPE_MISMATCH' || COALESCE(current_setting('loom.explorer_guard_barrier', true), '') AS integer))` + + ` WHEN COUNT(*) = 0 THEN NULL` + + ` WHEN COUNT(DISTINCT loom_dp_item.item) > 1 THEN to_jsonb(CAST('CONSTRUCTION_ROW_VALUE_MULTIPLE_VALUES' || COALESCE(current_setting('loom.explorer_guard_barrier', true), '') AS integer))` + + ` ELSE min(loom_dp_item.item::text)::jsonb` + + ` END FROM jsonb_array_elements(` + + `CASE WHEN ` + base + ` IS NULL OR jsonb_typeof(` + base + `) <> 'array' THEN NULL::jsonb ELSE ` + base + ` END` + + `) AS loom_dp_outer(value)` + + ` LEFT JOIN LATERAL jsonb_array_elements(` + + `CASE WHEN jsonb_typeof(loom_dp_outer.value) = 'array' THEN loom_dp_outer.value ELSE NULL::jsonb END` + + `) AS loom_dp_flattened(value) ON TRUE` + + ` LEFT JOIN LATERAL (SELECT CASE` + + ` WHEN jsonb_typeof(loom_dp_outer.value) = 'array' THEN loom_dp_flattened.value` + + ` ELSE loom_dp_outer.value END AS item) AS loom_dp_item ON TRUE` + + ` WHERE loom_dp_item.item IS NOT NULL AND loom_dp_item.item <> 'null'::jsonb)) AS "c0"`) + if rendered.SQL != want { + t.Fatalf("distinct_present_one SQL =\n%s\nwant\n%s", rendered.SQL, want) + } + wantArgs := append(sql10ScopeArgs(), + command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: "{gender}"}, + ) + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("distinct_present_one args = %#v, want %#v", rendered.Args, wantArgs) + } +} + +func TestCallsGoldenDistinctPresentOneLiteralArray(t *testing.T) { + plan := sql10Plan("Patient", map[string]any{"items": json.RawMessage(`["a","b"]`)}, + sql10CallProjection("one", sql10Call("distinct_present_one", "STRING", sql10ArrayLiteral("items"))), + ) + rendered := sql10RenderOK(t, plan) + if !strings.Contains(rendered.SQL, `jsonb_array_elements(CASE WHEN $5::jsonb IS NULL OR jsonb_typeof($5::jsonb) <> 'array' THEN NULL::jsonb ELSE $5::jsonb END)`) { + t.Fatalf("literal-array distinct form missing: %s", rendered.SQL) + } + if !reflect.DeepEqual(rendered.Args[4], command.PostgreSQLParameter{ + Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["a","b"]`)}) { + t.Fatalf("literal-array arg = %#v", rendered.Args[4]) + } +} + +func TestCallsGoldenNestedCall(t *testing.T) { + plan := sql10Plan("Patient", map[string]any{"a": int64(1), "b": float64(1.0), "c": "z"}, + sql10CallProjection("pick", sql10Call("coalesce", "", + sql10Nested(sql10Call("eq", "", sql10Literal("a"), sql10Literal("b"))), + sql10Literal("c"))), + ) + rendered := sql10RenderOK(t, plan) + if !strings.Contains(rendered.SQL, + `(COALESCE(NULLIF((to_jsonb(COALESCE($5::jsonb, 'null'::jsonb) = COALESCE($6::jsonb, 'null'::jsonb))), 'null'::jsonb), NULLIF($7::jsonb, 'null'::jsonb)))`) { + t.Fatalf("nested call fragment missing: %s", rendered.SQL) + } + if len(rendered.Args) != 7 { + t.Fatalf("nested args = %d, want 7", len(rendered.Args)) + } +} + +func TestCallsRejectUnregisteredNames(t *testing.T) { + for _, name := range []string{ + "first", "all", "distinct", "length", "coalesce_string", "concat", "join", + "contains", "canonical_json", "reference_id", "path_segment", "basename", + "last_segment", "sanitize_name", "sanitize_graphql_name", "uuid3", "uuid5", + } { + plan := sql10Plan("Patient", map[string]any{"a": int64(1)}, + sql10CallProjection("x", sql10Call(name, "", sql10Literal("a")))) + sql10Reject(t, plan, "call "+name, "not part of the v1 PostgreSQL registered call surface") + } + for _, name := range []string{"drop_table", ""} { + plan := sql10Plan("Patient", map[string]any{"a": int64(1)}, + sql10CallProjection("x", sql10Call(name, "", sql10Literal("a")))) + sql10RejectAny(t, plan, "ir-known refusal "+name, "call") + } +} + +func TestCallsRejectUnknownLiteralTypes(t *testing.T) { + for name, bind := range map[string]any{ + "int": int(1), + "uint64": uint64(1), + "time": time.Now(), + "map": map[string]any{"k": 1}, + "slice": []string{"a"}, + "object": json.RawMessage(`{"k":1}`), + "null-raw": json.RawMessage(`null`), + "number-raw": json.RawMessage(`1`), + } { + plan := sql10Plan("Patient", map[string]any{"a": bind}, + sql10CallProjection("x", sql10Call("coalesce", "", sql10Literal("a")))) + sql10Reject(t, plan, "literal "+name, "unknown call/type pair") + } + plan := sql10Plan("Patient", map[string]any{"a": math.Inf(1)}, + sql10CallProjection("x", sql10Call("coalesce", "", sql10Literal("a")))) + sql10Reject(t, plan, "infinite float", "not JSON-encodable") + stringArith := sql10Plan("Patient", map[string]any{"a": "x", "b": int64(1)}, + sql10CallProjection("x", sql10Call("add", "", sql10Literal("a"), sql10Literal("b")))) + sql10Reject(t, stringArith, "string arithmetic literal", "arithmetic accepts number binds") + missing := sql10Plan("Patient", nil, + sql10CallProjection("x", sql10Call("coalesce", "", sql10Literal("ghost")))) + sql10RejectAny(t, missing, "missing bind", "ghost") +} + +func TestCallsRejectPlanLevelShapes(t *testing.T) { + literal := func(key string) ir.PhysicalExpression { return sql10Literal(key) } + binds := map[string]any{"a": int64(1), "b": int64(2), "m": int64(1), "t": true, + "items": json.RawMessage(`["a"]`), "words": "not-an-array"} + + cases := []struct { + context string + call *ir.PhysicalExpression + want string + }{ + {"not arity", sql10Call("not", "", literal("a"), literal("b")), "requires 1 argument"}, + {"and arity", sql10Call("and", "", literal("a")), "requires at least 2 arguments"}, + {"or arity", sql10Call("or", "", literal("a")), "requires at least 2 arguments"}, + {"if arity", sql10Call("if", "", literal("a"), literal("b")), "requires 3 arguments"}, + {"case arity", sql10Call("case", "", literal("a")), "requires at least one condition/result pair"}, + {"coalesce arity", sql10Call("coalesce", ""), "requires at least one argument"}, + {"cast date gap", sql10Call("cast", "date", literal("a")), "date/date_time coercions are gaps"}, + {"cast date_time gap", sql10Call("cast", "date_time", literal("a")), "date/date_time coercions are gaps"}, + {"assert number message", sql10Call("assert", "", literal("t"), literal("m")), "must be a string"}, + {"assert row-path message", sql10Call("assert", "", literal("t"), sql10Value("payload", "gender")), + "bind-backed string literal"}, + {"aggregate operand", sql10Call("coalesce", "", sql10Aggregate(ir.PhysicalScalarCardinality)), "capability gaps"}, + {"typed column operand", sql10Call("coalesce", "", sql10Value("logical_id")), "not storage columns"}, + {"distinct VALUE argument", sql10Call("distinct_present_one", "STRING", func() ir.PhysicalExpression { + value := sql10Value("payload", "identifier") + value.Cardinality = ir.PhysicalArrayCardinality + return value + }()), "is not registered"}, + {"distinct aggregate argument", sql10Call("distinct_present_one", "STRING", + sql10Aggregate(ir.PhysicalArrayCardinality)), "is not registered"}, + {"distinct string literal", sql10Call("distinct_present_one", "STRING", sql10ArrayLiteral("words")), + "must bind a JSON array"}, + } + for _, cell := range cases { + plan := sql10Plan("Patient", binds, sql10CallProjection("x", cell.call)) + sql10Reject(t, plan, cell.context, cell.want) + } + + irCells := []struct { + context string + call *ir.PhysicalExpression + want string + }{ + {"add arity", sql10Call("add", "", literal("a")), "two arguments"}, + {"assert arity", sql10Call("assert", "", literal("a")), "condition and message"}, + {"cast arity", sql10Call("cast", "string", literal("a"), literal("b")), "one argument"}, + {"distinct arity", sql10Call("distinct_present_one", "STRING", literal("items"), literal("a")), "array argument"}, + {"distinct bad kind", sql10Call("distinct_present_one", "OBSERVATION", sql10ArrayLiteral("items")), "target kind"}, + {"add with target kind", sql10Call("add", "STRING", literal("a"), literal("b")), "target kind"}, + {"cast unknown target", sql10Call("cast", "observation", literal("a")), "unsupported"}, + {"cast missing target", sql10Call("cast", "", literal("a")), "requires one argument"}, + } + for _, cell := range irCells { + plan := sql10Plan("Patient", binds, sql10CallProjection("x", cell.call)) + sql10RejectAny(t, plan, cell.context, cell.want) + } +} + +func TestCallsValueOperandRefusesNonJSONDerivedScope(t *testing.T) { + r := &planRenderer{rows: map[string]rowScope{ + "position": {derivedValue: "(lateral.rn - 1)"}, + "has_item": {derivedValue: "(lateral.rn IS NOT NULL)"}, + "item": {derivedValue: "(lateral.item.value)", derivedJSON: true}, + }} + value := func(variable string) ir.PhysicalExpression { + return ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{Variable: variable}, + } + } + for _, variable := range []string{"position", "has_item"} { + _, err := r.renderCallValueOperand(value(variable)) + if err == nil || !strings.Contains(err.Error(), "ordinality and has-item") { + t.Fatalf("derived non-JSON operand %q: error %v lacks the typed refusal", variable, err) + } + } + operand, err := r.renderCallValueOperand(value("item")) + if err != nil { + t.Fatalf("JSON-derived row must stay accepted: %v", err) + } + if !strings.Contains(operand.sql, "lateral.item.value") { + t.Fatalf("JSON-derived operand fragment = %q", operand.sql) + } +} + +func TestCallsRejectFragmentByteBudget(t *testing.T) { + nest := func(levels int) *ir.PhysicalExpression { + expression := sql10Literal("one") + for range levels { + expression = sql10Nested(sql10Call("add", "", expression, sql10Literal("one"))) + } + return &expression + } + sql10Reject(t, sql10Plan("Patient", map[string]any{"one": int64(1)}, + sql10CallProjection("x", nest(40))), "fragment budget", "exceeds the") + shallow := sql10Plan("Patient", map[string]any{"one": int64(1)}, + sql10CallProjection("x", nest(4))) + rendered := sql10RenderOK(t, shallow) + if len(rendered.SQL) >= maxCallFragmentBytes { + t.Fatalf("legitimate 8-level chain produced %d bytes", len(rendered.SQL)) + } +} + +func TestCallsRejectOperandContractViolations(t *testing.T) { + fallback := sql10Plan("Patient", map[string]any{"a": int64(1)}, + sql10CallProjection("x", sql10Call("coalesce", "", func() ir.PhysicalExpression { + expression := sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + expression.Extract.ExecutionMode = ir.PhysicalSelectorGeneric + expression.Extract.Fallbacks = []ir.PhysicalSelectorFallback{{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Patient", + Selector: sql10Selector(t, "active"), + }} + return expression + }()))) + sql10Reject(t, fallback, "extract fallbacks", "fallback selectors are a separate capability") + + repeated := sql10Plan("Patient", map[string]any{"a": int64(1)}, + sql10CallProjection("x", sql10Call("coalesce", "", func() ir.PhysicalExpression { + expression := sql10Extract(t, "Patient", "identifier.system", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + expression.Extract.ExecutionMode = ir.PhysicalSelectorGeneric + expression.Extract.Selector.Steps[0].Iterate = true + return expression + }()))) + sql10Reject(t, repeated, "repeated step", "repeated") + + wrongType := sql10Plan("Patient", map[string]any{"a": int64(1)}, + sql10CallProjection("x", sql10Call("coalesce", "", + sql10Extract(t, "Observation", "status", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull)))) + sql10Reject(t, wrongType, "resource-type contradiction", "scanned as") + + arrayCall := sql10Plan("Patient", map[string]any{"a": int64(1)}, + ir.PhysicalProjection{Name: "x", Expression: func() *ir.PhysicalExpression { + call := sql10Call("coalesce", "", sql10Literal("a")) + call.Cardinality = ir.PhysicalArrayCardinality + return call + }()}) + sql10Reject(t, arrayCall, "array-cardinality CALL", "SCALAR-cardinality") + omitCall := sql10Plan("Patient", map[string]any{"a": int64(1)}, + ir.PhysicalProjection{Name: "x", Expression: func() *ir.PhysicalExpression { + call := sql10Call("coalesce", "", sql10Literal("a")) + call.NullBehavior = ir.PhysicalOmitNulls + return call + }()}) + sql10Reject(t, omitCall, "omit-nulls CALL", "SCALAR-cardinality") + + presenceCell := sql10Plan("Patient", map[string]any{"a": int64(1)}, + ir.PhysicalProjection{Name: "x", Presence: &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + Paths: [][]string{{"gender"}}, + }, Expression: sql10Call("coalesce", "", sql10Literal("a"))}) + sql10Reject(t, presenceCell, "presence on CALL", "presence proofs pair with selector extractions only") + hiddenCell := sql10Plan("Patient", map[string]any{"a": int64(1)}, + func() ir.PhysicalProjection { + projection := sql10CallProjection("x", sql10Call("coalesce", "", sql10Literal("a"))) + projection.Hidden = true + return projection + }()) + sql10Reject(t, hiddenCell, "hidden CALL", "hidden executor-side metadata") + + binds := map[string]any{"a": int64(1)} + var inner ir.PhysicalExpression = sql10Literal("a") + for index := 0; index <= maxCallExpressionDepth+1; index++ { + inner = sql10Nested(sql10Call("coalesce", "", inner)) + } + deep := sql10Plan("Patient", binds, sql10CallProjection("x", &inner)) + sql10Reject(t, deep, "deep nesting", "registered bound") +} + +func TestHiddenAuthResourcePathArmShapeLaw(t *testing.T) { + registered := func(mutate func(*ir.PhysicalProjection)) ir.PhysicalProjection { + projection := ir.PhysicalProjection{Name: "auth_resource_path", Hidden: true, + Value: ir.PhysicalValue{Variable: "root", Path: []string{"auth_resource_path"}}} + if mutate != nil { + mutate(&projection) + } + return projection + } + foreign := []struct { + context string + mutate func(*ir.PhysicalProjection) + irOnly bool + }{ + {"multi-segment payload path", func(p *ir.PhysicalProjection) { + p.Value.Path = []string{"payload", "auth_resource_path"} + }, false}, + {"aliased column path", func(p *ir.PhysicalProjection) { + p.Value.Path = []string{"auth_scope_alias"} + }, false}, + {"bind value", func(p *ir.PhysicalProjection) { + p.Value = ir.PhysicalValue{BindKey: "a"} + }, false}, + {"bind value with path", func(p *ir.PhysicalProjection) { + p.Value.Variable = "" + p.Value.BindKey = "scope_path" + }, true}, + {"expression carrier", func(p *ir.PhysicalProjection) { + p.Value = ir.PhysicalValue{} + p.Expression = sql10Call("coalesce", "", sql10Literal("a")) + }, false}, + {"presence output", func(p *ir.PhysicalProjection) { + p.PresenceOutput = true + }, true}, + {"no row variable", func(p *ir.PhysicalProjection) { + p.Value.Variable = "" + }, true}, + } + for _, cell := range foreign { + if cell.irOnly { + sql10RejectAny(t, sql10Plan("Patient", map[string]any{"a": int64(1)}, registered(cell.mutate)), + "hidden auth_resource_path "+cell.context, "auth_resource_path") + continue + } + sql10Reject(t, sql10Plan("Patient", map[string]any{"a": int64(1)}, registered(cell.mutate)), + "hidden auth_resource_path "+cell.context, "hidden executor-side metadata") + } + rendered := sql10RenderOK(t, sql10Plan("Patient", map[string]any{"a": int64(1)}, registered(nil))) + if !strings.Contains(rendered.SQL, `r0."auth_resource_path" AS "c0"`) { + t.Fatalf("registered hidden auth_resource_path did not render the typed scope column:\n%s", rendered.SQL) + } + precedent := sql10RenderOK(t, sql10Plan("Patient", map[string]any{"a": int64(1)}, + ir.PhysicalProjection{Name: ir.PreviewSourceResourceIDColumn, Hidden: true, + Value: ir.PhysicalValue{Variable: "root", Path: []string{"_key"}}})) + if !strings.Contains(precedent.SQL, `r0."logical_id" AS "c0"`) { + t.Fatalf("preview source identity precedent arm drifted:\n%s", precedent.SQL) + } +} + +func TestCallsDefenseInDepthShapeWalk(t *testing.T) { + literal := func(key string) ir.PhysicalExpression { return sql10Literal(key) } + shape := func(name, target string, args ...ir.PhysicalExpression) *ir.PhysicalCall { + return &ir.PhysicalCall{Name: name, TargetKind: target, Args: args} + } + cells := []struct { + context string + call *ir.PhysicalCall + want string + }{ + {"add 3-arg", shape("add", "", literal("a"), literal("a"), literal("a")), "requires 2 arguments"}, + {"eq 1-arg", shape("eq", "", literal("a")), "requires 2 arguments"}, + {"gt 3-arg", shape("gt", "", literal("a"), literal("a"), literal("a")), "requires 2 arguments"}, + {"if 2-arg", shape("if", "", literal("a"), literal("a")), "requires 3 arguments"}, + {"case 1-arg", shape("case", "", literal("a")), "at least one condition/result pair"}, + {"assert 1-arg", shape("assert", "", literal("a")), "condition and a message"}, + {"distinct 2-arg", shape("distinct_present_one", "STRING", literal("a"), literal("a")), "one array argument"}, + {"cast unknown", shape("cast", "instant", literal("a")), "not registered for PostgreSQL"}, + {"add target kind", shape("add", "STRING", literal("a"), literal("a")), "must not carry target kind"}, + {"unknown name", shape("drop_table", "", literal("a")), "not part of the v1 PostgreSQL registered call surface"}, + } + for _, cell := range cells { + if err := validateCallShape("probe", cell.call, 0); !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("%s: validateCallShape error = %v, want ErrUnsupportedPostgresPlan", cell.context, err) + } else if !strings.Contains(err.Error(), cell.want) { + t.Fatalf("%s: error %q lacks %q", cell.context, err.Error(), cell.want) + } + } +} + +func TestCallsSelectorSliceStaysClosed(t *testing.T) { + plan := sql10Plan("Patient", map[string]any{"a": int64(1)}, + sql10CallProjection("x", sql10Call("coalesce", "", sql10Literal("a")))) + _, err := RenderSelectorExtraction(plan, mustBinding(t)) + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("selector-slice CALL error = %v, want ErrUnsupportedPostgresPlan", err) + } + if !strings.Contains(err.Error(), "full read spine") { + t.Fatalf("selector-slice CALL error names no spine: %q", err.Error()) + } +} + +func TestCallsDeterministic(t *testing.T) { + plan := sql10Plan("Patient", map[string]any{"a": int64(2), "b": "x"}, + sql10CallProjection("sum", sql10Call("add", "", sql10Literal("a"), sql10Literal("a"))), + sql10CallProjection("pick", sql10Call("fallback", "", + sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("b"))), + ) + first := sql10RenderOK(t, plan) + second, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("second render: %v", err) + } + if first.SQL != second.SQL { + t.Fatalf("SQL drifted between renders:\n%s\n%s", first.SQL, second.SQL) + } + if !reflect.DeepEqual(first.Args, second.Args) { + t.Fatalf("args drifted between renders: %#v vs %#v", first.Args, second.Args) + } + for _, forbidden := range []string{"@", "FOR ", "RETURN ", "FILTER ", sql10Project, sql10Gen} { + if strings.Contains(first.SQL, forbidden) { + t.Fatalf("SQL leaked %q: %s", forbidden, first.SQL) + } + } +} + +func TestCallsRowVariableWalkVisitsOperands(t *testing.T) { + visited := 0 + err := forEachCallOperandRowVariable(*sql10Call("coalesce", "", + sql10Value("payload", "gender"), + sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Nested(sql10Call("not", "", sql10Value("payload", "active")))), + func(variable string, extract *ir.PhysicalExtract) error { + visited++ + if variable != "root" { + t.Fatalf("visited variable %q, want root", variable) + } + if extract != nil && extract.ResourceType != "Patient" { + t.Fatalf("visited extract resource type %q", extract.ResourceType) + } + return nil + }) + if err != nil { + t.Fatalf("walk: %v", err) + } + if visited != 3 { + t.Fatalf("visited %d operands, want 3", visited) + } +} + +func TestCallsRankComparatorIsTotalOverNulls(t *testing.T) { + plan := sql10Plan("Patient", map[string]any{"s": "female"}, + sql10CallProjection("ordered", sql10Call("lte", "", + sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("s"))), + ) + rendered := sql10RenderOK(t, plan) + rankArms := []string{"THEN 0", "THEN 1", "THEN 2", "THEN 3", "ELSE 4 END"} + for _, arm := range rankArms { + if !strings.Contains(rendered.SQL, arm) { + t.Fatalf("rank arm %q missing: %s", arm, rendered.SQL) + } + } + if !strings.Contains(rendered.SQL, `>= 4 OR `) || !strings.Contains(rendered.SQL, `>= 4 THEN NULL`) { + t.Fatalf("rank-4 fail-closed arm missing: %s", rendered.SQL) + } +} + +var sql10ParameterPattern = regexp.MustCompile(`\$(\d+)`) + +func TestCallsEveryParameterReferencedExactlyInOrder(t *testing.T) { + plan := sql10Plan("Patient", map[string]any{"a": int64(1), "msg": "TAG"}, + sql10CallProjection("checked", sql10Call("assert", "", + sql10Extract(t, "Patient", "gender", ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull), + sql10Literal("msg"))), + sql10CallProjection("flag", sql10Call("cast", "boolean", sql10Literal("a"))), + ) + rendered := sql10RenderOK(t, plan) + if len(rendered.Args) != 7 { + t.Fatalf("args = %d, want 7", len(rendered.Args)) + } + seen := map[int]bool{} + for _, match := range sql10ParameterPattern.FindAllStringSubmatch(rendered.SQL, -1) { + var index int + fmt.Sscanf(match[0], "$%d", &index) + seen[index] = true + } + for index := 1; index <= len(rendered.Args); index++ { + if !seen[index] { + t.Fatalf("$%d unreferenced (constructor should have refused): %s", index, rendered.SQL) + } + } +} diff --git a/internal/dataframe/compiler/render/postgres/category_scan.go b/internal/dataframe/compiler/render/postgres/category_scan.go new file mode 100644 index 000000000..934a5b6c8 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/category_scan.go @@ -0,0 +1,287 @@ +package postgres + +import ( + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const ( + CategoryScanPresentColumn = "present" + CategoryScanValueColumn = "value" + + maxCategoryScanValues = 256 + + categorySourceCTE = "__loom_category_source" + categoryRowsCTE = "__loom_category_rows" + categoryDistinctCTE = "__loom_category_distinct" + categoryRowAlias = "__loom_category_row" +) + +type RenderedCategoryScan struct { + Values command.RenderedPostgresPhysicalPlan + Witness command.RenderedPostgresPhysicalPlan + PresentColumn string + ValueColumn string + PresenceMarkerColumn string +} + +func RenderCategoryScan(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding, columnName, presenceMarkerColumn string, maxValues int) (RenderedCategoryScan, error) { + if err := storage.Validate(); err != nil { + return RenderedCategoryScan{}, err + } + if strings.TrimSpace(columnName) == "" { + return RenderedCategoryScan{}, fmt.Errorf("%w: a category scan names the column whose distinct values it enumerates", ErrUnsupportedPostgresPlan) + } + if maxValues < 1 || maxValues > maxCategoryScanValues { + return RenderedCategoryScan{}, fmt.Errorf("%w: category scan maxValues %d is outside the registered 1..%d bound", + ErrUnsupportedPostgresPlan, maxValues, maxCategoryScanValues) + } + narrowed, err := categoryScanInnerPlan(plan, columnName, presenceMarkerColumn) + if err != nil { + return RenderedCategoryScan{}, err + } + inner, err := RenderPhysicalPlan(narrowed, storage) + if err != nil { + return RenderedCategoryScan{}, err + } + valueIndex, err := categoryOutputIndex(inner.OutputColumns, columnName) + if err != nil { + return RenderedCategoryScan{}, err + } + presenceIndex, err := categoryOutputIndex(inner.OutputColumns, presenceMarkerColumn) + if err != nil { + return RenderedCategoryScan{}, err + } + values, err := composeCategoryStatement(inner, valueIndex, presenceIndex, maxValues+1, true) + if err != nil { + return RenderedCategoryScan{}, err + } + witness, err := composeCategoryStatement(inner, valueIndex, presenceIndex, maxValues*4+1, false) + if err != nil { + return RenderedCategoryScan{}, err + } + return RenderedCategoryScan{ + Values: values, Witness: witness, + PresentColumn: CategoryScanPresentColumn, ValueColumn: CategoryScanValueColumn, + PresenceMarkerColumn: presenceMarkerColumn, + }, nil +} + +func categoryScanInnerPlan(plan ir.PhysicalPlan, columnName, presenceMarkerColumn string) (ir.PhysicalPlan, error) { + if plan.StageSequence != nil { + return ir.PhysicalPlan{}, fmt.Errorf("%w: category scans over composed construction stages are not registered for PostgreSQL; only the read-spine shape has a rendered category terminal (pg-sql-25)", + ErrUnsupportedPostgresPlan) + } + if len(plan.Operations) == 0 { + return ir.PhysicalPlan{}, fmt.Errorf("%w: a category scan requires a root scan and a terminal RETURN", ErrUnsupportedPostgresPlan) + } + returnIndex := -1 + sortCount := 0 + for index, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalRootScanOp: + if index != 0 || operation.RootScan == nil { + return ir.PhysicalPlan{}, fmt.Errorf("%w: a category scan requires exactly one leading ROOT_SCAN with its payload set", ErrUnsupportedPostgresPlan) + } + case ir.PhysicalFilterOp: + if operation.Filter == nil { + return ir.PhysicalPlan{}, fmt.Errorf("%w: operation %d: filter payload is missing", ErrUnsupportedPostgresPlan, index) + } + case ir.PhysicalDerivedLetOp: + if operation.DerivedLet == nil { + return ir.PhysicalPlan{}, fmt.Errorf("%w: operation %d: derived LET payload is missing", ErrUnsupportedPostgresPlan, index) + } + case ir.PhysicalSortOp: + if operation.Sort == nil { + return ir.PhysicalPlan{}, fmt.Errorf("%w: operation %d: sort payload is missing", ErrUnsupportedPostgresPlan, index) + } + sortCount++ + if sortCount > 1 { + return ir.PhysicalPlan{}, fmt.Errorf("%w: at most one SORT is accepted; the category page order is owned by the distinct layer", ErrUnsupportedPostgresPlan) + } + case ir.PhysicalReturnOp: + if operation.Return == nil || index != len(plan.Operations)-1 { + return ir.PhysicalPlan{}, fmt.Errorf("%w: a category scan requires exactly one terminal RETURN", ErrUnsupportedPostgresPlan) + } + returnIndex = index + case ir.PhysicalLimitOp: + return ir.PhysicalPlan{}, fmt.Errorf("%w: a plan-carried LIMIT would bound the page before the distinct, which is not the category contract; category pages are bounded by maxValues+1", + ErrUnsupportedPostgresPlan) + default: + return ir.PhysicalPlan{}, fmt.Errorf("%w: operation %q is outside the registered PostgreSQL category surface (root scan, filters, canonical auth LET, sort, return)", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + if returnIndex < 0 { + return ir.PhysicalPlan{}, fmt.Errorf("%w: a category scan requires a terminal RETURN", ErrUnsupportedPostgresPlan) + } + + projections := plan.Operations[returnIndex].Return.Projections + if strings.TrimSpace(presenceMarkerColumn) == "" || presenceMarkerColumn == columnName { + return ir.PhysicalPlan{}, fmt.Errorf("%w: the presence marker column must be a distinct non-blank name", ErrUnsupportedPostgresPlan) + } + var category *ir.PhysicalProjection + for index := range projections { + projection := &projections[index] + if projection.Name == presenceMarkerColumn { + return ir.PhysicalPlan{}, fmt.Errorf("%w: projection %q collides with the renderer-owned presence marker column", + ErrUnsupportedPostgresPlan, presenceMarkerColumn) + } + if projection.Name != columnName { + continue + } + if category != nil { + return ir.PhysicalPlan{}, fmt.Errorf("%w: %d projections are named %q; a category column must be projected exactly once", + ErrUnsupportedPostgresPlan, 2, columnName) + } + if projection.Hidden { + return ir.PhysicalPlan{}, fmt.Errorf("%w: category column %q is hidden executor-side metadata and is not a public category", + ErrUnsupportedPostgresPlan, columnName) + } + category = projection + } + if category == nil { + return ir.PhysicalPlan{}, fmt.Errorf("%w: the terminal RETURN does not project the category column %q", ErrUnsupportedPostgresPlan, columnName) + } + path, rowVariable, err := categorySelectorEvidence(*category) + if err != nil { + return ir.PhysicalPlan{}, err + } + + out := ir.ClonePhysicalPlan(plan) + operations := make([]ir.PhysicalOperation, 0, len(out.Operations)) + for _, operation := range out.Operations { + if operation.Kind == ir.PhysicalSortOp { + continue + } + operations = append(operations, operation) + } + returnIndex = len(operations) - 1 + returnOp := *operations[returnIndex].Return + returnOp.Projections = []ir.PhysicalProjection{ + *category, + { + Name: presenceMarkerColumn, + PresenceOutput: true, + Presence: &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: rowVariable, Path: []string{"payload"}}, + Paths: [][]string{append([]string(nil), path...)}, + }, + }, + } + operations[returnIndex].Return = &returnOp + out.Operations = operations + return out, nil +} + +func categorySelectorEvidence(projection ir.PhysicalProjection) ([]string, string, error) { + if projection.Expression == nil || projection.Expression.Kind != ir.PhysicalExtractExpression || + projection.Expression.Cardinality != ir.PhysicalScalarCardinality || projection.Expression.Extract == nil { + return nil, "", fmt.Errorf("%w: category column %q must read the document through a scalar-cardinality EXTRACT selector; derived CALL, OBJECT, SLICE, unit-normalized, and typed-column reads have no registered category reading", + ErrUnsupportedPostgresPlan, projection.Name) + } + extract := projection.Expression.Extract + if extract.ExecutionMode != ir.PhysicalSelectorDirectScalar { + return nil, "", fmt.Errorf("%w: category column %q uses selector execution mode %q; only the registered DIRECT_SCALAR selector has a category reading", + ErrUnsupportedPostgresPlan, projection.Name, string(extract.ExecutionMode)) + } + if len(extract.Fallbacks) != 0 || extract.Prepared != nil || extract.Distinct || extract.UnitNormalization != nil || + extract.Selector.Filter != nil { + return nil, "", fmt.Errorf("%w: category column %q uses a selector shape outside the registered single-path direct scalar form", + ErrUnsupportedPostgresPlan, projection.Name) + } + if extract.Source.BindKey != "" || extract.Source.Variable == "" || + len(extract.Source.Path) != 1 || extract.Source.Path[0] != "payload" { + return nil, "", fmt.Errorf(`%w: category column %q must extract from the row document at ["payload"], the exact source shape lowering emits; got %v`, + ErrUnsupportedPostgresPlan, projection.Name, extract.Source.Path) + } + if len(extract.Selector.Steps) == 0 { + return nil, "", fmt.Errorf("%w: category column %q has an empty selector", ErrUnsupportedPostgresPlan, projection.Name) + } + path := make([]string, 0, len(extract.Selector.Steps)) + for index, step := range extract.Selector.Steps { + if step.Field == "" || step.Iterate || step.Index != nil { + return nil, "", fmt.Errorf("%w: category column %q: selector step %d is not a plain property; repeated and indexed steps have no category reading", + ErrUnsupportedPostgresPlan, projection.Name, index) + } + if !payloadPathSegmentPattern.MatchString(step.Field) { + return nil, "", fmt.Errorf("%w: category column %q: selector step %q is not renderer-safe", + ErrUnsupportedPostgresPlan, projection.Name, step.Field) + } + path = append(path, step.Field) + } + if presence := projection.Presence; presence != nil { + if presence.Source.Variable != extract.Source.Variable || presence.Source.BindKey != "" || + len(presence.Source.Path) != 1 || presence.Source.Path[0] != "payload" || + len(presence.Paths) != 1 || strings.Join(presence.Paths[0], "\x1f") != strings.Join(path, "\x1f") { + return nil, "", fmt.Errorf("%w: category column %q: presence metadata must prove the selector's own path", + ErrUnsupportedPostgresPlan, projection.Name) + } + } + return path, extract.Source.Variable, nil +} + +func categoryOutputIndex(declared []string, name string) (int, error) { + found := -1 + for index, candidate := range declared { + if candidate == name { + if found >= 0 { + return 0, fmt.Errorf("%w: the inner read declares %q twice; a category column must be declared exactly once", + ErrUnsupportedPostgresPlan, name) + } + found = index + } + } + if found < 0 { + return 0, fmt.Errorf("%w: the inner read did not declare %q; the category statement cannot be composed", + ErrUnsupportedPostgresPlan, name) + } + return found, nil +} + +func composeCategoryStatement(inner command.RenderedPostgresPhysicalPlan, valueIndex, presenceIndex, bound int, ordered bool) (command.RenderedPostgresPhysicalPlan, error) { + if bound < 1 { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("%w: category bound %d is not positive", ErrUnsupportedPostgresPlan, bound) + } + valueAlias := `"` + innerColumnAlias(valueIndex) + `"` + presenceAlias := `"` + innerColumnAlias(presenceIndex) + `"` + presentColumn := `"` + CategoryScanPresentColumn + `"` + valueColumn := `"` + CategoryScanValueColumn + `"` + + args := append([]command.PostgreSQLParameter(nil), inner.Args...) + args = append(args, command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(bound)}) + boundParameter := "$" + strconv.Itoa(len(args)) + + statement := "WITH " + categorySourceCTE + " AS (" + inner.SQL + "), " + + categoryRowsCTE + " AS (SELECT " + categoryRowAlias + "." + presenceAlias + " AS " + presentColumn + + ", CASE WHEN " + categoryRowAlias + "." + presenceAlias + " THEN " + categoryRowAlias + "." + valueAlias + + " END AS " + valueColumn + " FROM " + categorySourceCTE + " AS " + categoryRowAlias + "), " + + categoryDistinctCTE + " AS (SELECT " + presentColumn + ", " + valueColumn + " FROM " + categoryRowsCTE + + " GROUP BY " + presentColumn + ", " + valueColumn + ") " + + "SELECT " + presentColumn + " AS \"c0\", " + valueColumn + " AS \"c1\" FROM " + categoryDistinctCTE + if ordered { + statement += " ORDER BY " + presentColumn + " ASC, " + categoryValueRankExpression + " ASC, " + + categoryStringValueKeyExpression + " ASC, " + valueColumn + " ASC" + } + statement += " LIMIT " + boundParameter + result, err := command.NewRenderedPostgresPhysicalPlan(statement, args, + []string{CategoryScanPresentColumn, CategoryScanValueColumn}) + if err != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("compose category statement: %w", err) + } + return result, nil +} + +func innerColumnAlias(index int) string { + return "c" + strconv.Itoa(index) +} + +const categoryValueRankExpression = `CASE WHEN "value" IS NULL THEN 2 ELSE CASE COALESCE(jsonb_typeof("value"), 'null')` + + ` WHEN 'array' THEN 0 WHEN 'boolean' THEN 1 WHEN 'null' THEN 2 WHEN 'number' THEN 3` + + ` WHEN 'object' THEN 4 WHEN 'string' THEN 5 ELSE 6 END END` + +const categoryStringValueKeyExpression = `CASE WHEN jsonb_typeof("value") = 'string' THEN "value" #>> '{}' END COLLATE "C"` diff --git a/internal/dataframe/compiler/render/postgres/category_scan_test.go b/internal/dataframe/compiler/render/postgres/category_scan_test.go new file mode 100644 index 000000000..155585e7d --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/category_scan_test.go @@ -0,0 +1,653 @@ +package postgres + +import ( + "errors" + "fmt" + "reflect" + "sort" + "strings" + "sync" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" +) + +const ( + sql25Marker = "__loom_category_scan_presence" + sql25CategoryColumn = "gender" + sql25OtherColumn = "status" + sql25OfflineSchema = "verify_sql25v1_offline" +) + +func sql25CategoryProjection(name, field string, behavior ir.PhysicalNullBehavior, override func(*ir.PhysicalExtract, *ir.PhysicalProjection)) ir.PhysicalProjection { + source := ir.PhysicalValue{Variable: "root", Path: []string{"payload"}} + expression := &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: behavior, + Extract: &ir.PhysicalExtract{ + Source: source, + ResourceType: "Patient", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: field}}}, + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + } + projection := ir.PhysicalProjection{ + Name: name, + Expression: expression, + Presence: &ir.PhysicalProjectionPresence{Source: source, Paths: [][]string{{field}}}, + } + if override != nil { + override(expression.Extract, &projection) + } + return projection +} + +func sql25Plan(projections ...ir.PhysicalProjection) ir.PhysicalPlan { + return withReturn(scopedPlan(itProject, itGeneration, nil, true), projections...) +} + +func sql25Render(t *testing.T, plan ir.PhysicalPlan, column, marker string, maxValues int) RenderedCategoryScan { + t.Helper() + rendered, err := RenderCategoryScan(plan, bindingFor(t, sql25OfflineSchema), column, marker, maxValues) + if err != nil { + t.Fatalf("RenderCategoryScan(%q): %v", column, err) + } + return rendered +} + +func TestRenderCategoryScanComposesBoundedDistinctPage(t *testing.T) { + plan := sql25Plan( + sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil), + sql25CategoryProjection(sql25OtherColumn, "status", ir.PhysicalPreserveNull, nil), + ) + rendered := sql25Render(t, plan, sql25CategoryColumn, sql25Marker, 8) + + want := `WITH __loom_category_source AS (SELECT (r0."payload" #> $4::text[]) AS "c0", ` + + `COALESCE(jsonb_typeof(r0."payload") = 'object' AND jsonb_exists(r0."payload", $5), false) AS "c1" ` + + `FROM "` + sql25OfflineSchema + `"."fhir_resource" AS r0 ` + + `WHERE r0."project" = $1 AND r0."dataset_generation" = $2 AND r0."resource_type" = $3 ` + + `ORDER BY r0."resource_seq" ASC), ` + + `__loom_category_rows AS (SELECT __loom_category_row."c1" AS "present", ` + + `CASE WHEN __loom_category_row."c1" THEN __loom_category_row."c0" END AS "value" ` + + `FROM __loom_category_source AS __loom_category_row), ` + + `__loom_category_distinct AS (SELECT "present", "value" FROM __loom_category_rows ` + + `GROUP BY "present", "value") ` + + `SELECT "present" AS "c0", "value" AS "c1" FROM __loom_category_distinct ` + + `ORDER BY "present" ASC, CASE WHEN "value" IS NULL THEN 2 ELSE CASE COALESCE(jsonb_typeof("value"), 'null')` + + ` WHEN 'array' THEN 0 WHEN 'boolean' THEN 1 WHEN 'null' THEN 2 WHEN 'number' THEN 3` + + ` WHEN 'object' THEN 4 WHEN 'string' THEN 5 ELSE 6 END END ASC, ` + + `CASE WHEN jsonb_typeof("value") = 'string' THEN "value" #>> '{}' END COLLATE "C" ASC, ` + + `"value" ASC LIMIT $6` + if rendered.Values.SQL != want { + t.Fatalf("category page SQL\ngot: %s\nwant: %s", rendered.Values.SQL, want) + } + wantArgs := []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: itProject}, + {Type: command.PostgreSQLParameterText, Value: itGeneration}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + {Type: command.PostgreSQLParameterText, Value: "{gender}"}, + {Type: command.PostgreSQLParameterText, Value: "gender"}, + {Type: command.PostgreSQLParameterBigint, Value: int64(9)}, + } + if !reflect.DeepEqual(rendered.Values.Args, wantArgs) { + t.Fatalf("category page args = %#v, want %#v", rendered.Values.Args, wantArgs) + } + if got := rendered.Values.OutputColumns; !reflect.DeepEqual(got, []string{"present", "value"}) { + t.Fatalf("declared output columns = %v, want [present value]", got) + } + if rendered.PresentColumn != CategoryScanPresentColumn || rendered.ValueColumn != CategoryScanValueColumn { + t.Fatalf("column contract = %q/%q", rendered.PresentColumn, rendered.ValueColumn) + } + if rendered.PresenceMarkerColumn != sql25Marker { + t.Fatalf("presence marker echoed = %q", rendered.PresenceMarkerColumn) + } + if strings.Contains(rendered.Values.SQL, "@") { + t.Fatalf("AQL bind syntax reached the PostgreSQL category statement: %s", rendered.Values.SQL) + } + if !strings.HasSuffix(rendered.Values.SQL, "LIMIT $6") { + t.Fatalf("page bound is not the last placeholder: %s", rendered.Values.SQL) + } + if strings.Contains(rendered.Values.SQL, `"`+sql25CategoryColumn+`"`) || strings.Contains(rendered.Values.SQL, sql25OtherColumn) { + t.Fatalf("a plan column name leaked into SQL text: %s", rendered.Values.SQL) + } +} + +func TestRenderCategoryScanWitnessSharesTheDistinctRelation(t *testing.T) { + plan := sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil)) + rendered := sql25Render(t, plan, sql25CategoryColumn, sql25Marker, 8) + witness := rendered.Witness + if outer := witness.SQL[strings.Index(witness.SQL, categoryDistinctCTE):]; strings.Contains(outer, "ORDER BY") { + t.Fatalf("the bounded confirmation must not sort its page: %s", witness.SQL) + } + if !strings.Contains(witness.SQL, `ORDER BY r0."resource_seq" ASC`) { + t.Fatalf("the inner deterministic spine order is missing: %s", witness.SQL) + } + if !strings.HasSuffix(witness.SQL, "LIMIT $6") { + t.Fatalf("witness bound is not the last placeholder: %s", witness.SQL) + } + last := witness.Args[len(witness.Args)-1] + if want := (command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(8*4 + 1)}); last != want { + t.Fatalf("witness bound = %#v, want %#v", last, want) + } + if !reflect.DeepEqual(witness.OutputColumns, []string{"present", "value"}) { + t.Fatalf("witness declares %v", witness.OutputColumns) + } + page := rendered.Values.SQL + pageHead, witnessHead := page[:strings.Index(page, " SELECT ")], witness.SQL[:strings.Index(witness.SQL, " SELECT ")] + if pageHead != witnessHead { + t.Fatalf("witness distinct relation differs from the page:\n page: %s\nwitness: %s", pageHead, witnessHead) + } +} + +func TestRenderCategoryScanIsDeterministic(t *testing.T) { + plan := sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil)) + binding := bindingFor(t, sql25OfflineSchema) + first, err := RenderCategoryScan(plan, binding, sql25CategoryColumn, sql25Marker, 64) + if err != nil { + t.Fatal(err) + } + for attempt := 0; attempt < 3; attempt++ { + again, err := RenderCategoryScan(plan, binding, sql25CategoryColumn, sql25Marker, 64) + if err != nil { + t.Fatal(err) + } + if again.Values.SQL != first.Values.SQL || again.Witness.SQL != first.Witness.SQL { + t.Fatalf("render %d was not byte-identical", attempt) + } + if !reflect.DeepEqual(again.Values.Args, first.Values.Args) || !reflect.DeepEqual(again.Witness.Args, first.Witness.Args) { + t.Fatalf("render %d was not argument-identical", attempt) + } + } + if plan.Operations[len(plan.Operations)-1].Return.Projections[0].Name != sql25CategoryColumn || + len(plan.Operations[len(plan.Operations)-1].Return.Projections) != 1 { + t.Fatalf("the caller's plan was mutated: %v", plan.Operations[len(plan.Operations)-1].Return.Projections) + } +} + +func TestRenderCategoryScanPreservesBehaviorFormsBesideATotalPresenceBit(t *testing.T) { + cases := []struct { + name string + behavior ir.PhysicalNullBehavior + wantValue string + rejectValue string + }{ + {"preserve null", ir.PhysicalPreserveNull, `(r0."payload" #> $4::text[]) AS "c0"`, `NULLIF(`}, + {"omit nulls", ir.PhysicalOmitNulls, `NULLIF((r0."payload" #> $4::text[]), 'null'::jsonb) AS "c0"`, ``}, + {"empty on null", ir.PhysicalEmptyOnNull, + `COALESCE(NULLIF((r0."payload" #> $4::text[]), 'null'::jsonb), '""'::jsonb) AS "c0"`, ``}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + rendered := sql25Render(t, sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", test.behavior, nil)), + sql25CategoryColumn, sql25Marker, 4) + sql := rendered.Values.SQL + if !strings.Contains(sql, test.wantValue) { + t.Fatalf("value form %q missing:\n%s", test.wantValue, sql) + } + if test.rejectValue != "" && strings.Contains(sql, test.rejectValue) { + t.Fatalf("unexpected value form %q:\n%s", test.rejectValue, sql) + } + const presence = `COALESCE(jsonb_typeof(r0."payload") = 'object' AND jsonb_exists(r0."payload", $5), false) AS "c1"` + if !strings.Contains(sql, presence) { + t.Fatalf("presence proof form missing:\n%s", sql) + } + if !strings.Contains(sql, `CASE WHEN __loom_category_row."c1" THEN __loom_category_row."c0" END AS "value"`) { + t.Fatalf("value is not gated by presence:\n%s", sql) + } + }) + } +} + +func TestRenderCategoryScanRefusals(t *testing.T) { + withOperation := func(plan ir.PhysicalPlan, operation ir.PhysicalOperation) ir.PhysicalPlan { + operations := append([]ir.PhysicalOperation(nil), plan.Operations...) + operations = append(operations[:len(operations)-1], operation, operations[len(operations)-1]) + plan.Operations = operations + return plan + } + planWithLimit := withOperation(sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil)), + ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "limit"}}) + planWithUnnest := withOperation(sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil)), + ir.PhysicalOperation{Kind: ir.PhysicalUnnestOp, Unnest: &ir.PhysicalUnnest{ + OutputVariable: "item", HasItemVariable: "item_has", + EmptyPolicy: ir.PhysicalUnnestPreserveParent, + }}) + planWithTraversal := withOperation(sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil)), + ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "obs", EdgeVariable: "edge", + Direction: ir.PhysicalOutbound, EdgeCollectionBindKey: "edges", EdgeLabelBindKey: "edge_type", + }}) + planWithTwoSorts := sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil)) + sortOp := ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{ + Keys: []ir.PhysicalValue{{Variable: "root", Path: []string{"_key"}}}}} + planWithTwoSorts.Operations = append(planWithTwoSorts.Operations[:1], append([]ir.PhysicalOperation{sortOp, sortOp}, planWithTwoSorts.Operations[1:]...)...) + planWithStageSequence := sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil)) + planWithStageSequence.StageSequence = &ir.PhysicalStageSequence{SourceStageID: "source_projection"} + planWithDuplicateCategory := sql25Plan( + sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil), + sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil)) + hiddenCategory := sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil) + hiddenCategory.Hidden = true + planWithHiddenCategory := sql25Plan(hiddenCategory) + planWithMarkerCollision := sql25Plan( + sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil), + sql25CategoryProjection(sql25Marker, "status", ir.PhysicalPreserveNull, nil)) + planWithoutCategory := sql25Plan(sql25CategoryProjection(sql25OtherColumn, "status", ir.PhysicalPreserveNull, nil)) + planWithoutReturn := scopedPlan(itProject, itGeneration, nil, true) + planWithReturnlessTail := sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil)) + planWithReturnlessTail.Operations = planWithReturnlessTail.Operations[:len(planWithReturnlessTail.Operations)-1] + foreignPresence := sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil) + foreignPresence.Presence = &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, Paths: [][]string{{"maritalStatus"}}} + planWithForeignPresence := sql25Plan(foreignPresence) + multiPathPresence := sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil) + multiPathPresence.Presence = &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, Paths: [][]string{{"gender"}, {"maritalStatus"}}} + planWithMultiPathPresence := sql25Plan(multiPathPresence) + nested := sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, func(extract *ir.PhysicalExtract, _ *ir.PhysicalProjection) { + extract.Selector = spec.Selector{Steps: []spec.SelectorStep{{Field: "maritalStatus"}, {Field: "coding"}}} + extract.Prepared = nil + }) + nested.Presence = &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, Paths: [][]string{{"maritalStatus", "coding"}}} + planWithNestedSelector := sql25Plan(nested) + + cases := []struct { + name string + plan ir.PhysicalPlan + column string + marker string + bound int + denials []string + }{ + {"empty column", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil)), + " ", sql25Marker, 8, []string{"names the column"}}, + {"column equals marker", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil)), + sql25Marker, sql25Marker, 8, []string{"distinct non-blank name"}}, + {"blank marker", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil)), + sql25CategoryColumn, " ", 8, []string{"distinct non-blank name"}}, + {"zero bound", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil)), + sql25CategoryColumn, sql25Marker, 0, []string{"outside the registered 1..256 bound"}}, + {"oversized bound", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil)), + sql25CategoryColumn, sql25Marker, 257, []string{"outside the registered 1..256 bound"}}, + {"missing category", planWithoutCategory, sql25CategoryColumn, sql25Marker, 8, + []string{"does not project the category column"}}, + {"duplicate category", planWithDuplicateCategory, sql25CategoryColumn, sql25Marker, 8, + []string{"projected exactly once"}}, + {"hidden category", planWithHiddenCategory, sql25CategoryColumn, sql25Marker, 8, + []string{"hidden executor-side metadata"}}, + {"marker collision", planWithMarkerCollision, sql25CategoryColumn, sql25Marker, 8, + []string{"collides with the renderer-owned presence marker"}}, + {"no return", planWithoutReturn, sql25CategoryColumn, sql25Marker, 8, + []string{"requires a terminal RETURN"}}, + {"return not terminal", planWithReturnlessTail, sql25CategoryColumn, sql25Marker, 8, + []string{"terminal RETURN"}}, + {"composed stage sequence", planWithStageSequence, sql25CategoryColumn, sql25Marker, 8, + []string{"composed construction stages are not registered"}}, + {"plan-carried limit", planWithLimit, sql25CategoryColumn, sql25Marker, 8, + []string{"would bound the page before the distinct"}}, + {"unnest", planWithUnnest, sql25CategoryColumn, sql25Marker, 8, + []string{"outside the registered PostgreSQL category surface"}}, + {"traversal", planWithTraversal, sql25CategoryColumn, sql25Marker, 8, + []string{"outside the registered PostgreSQL category surface"}}, + {"two sorts", planWithTwoSorts, sql25CategoryColumn, sql25Marker, 8, + []string{"at most one SORT"}}, + {"foreign presence claim", planWithForeignPresence, sql25CategoryColumn, sql25Marker, 8, + []string{"presence metadata must prove the selector's own path"}}, + {"multi-path presence claim", planWithMultiPathPresence, sql25CategoryColumn, sql25Marker, 8, + []string{"presence metadata must prove the selector's own path"}}, + {"direct value read", sql25Plan(ir.PhysicalProjection{ + Name: sql25CategoryColumn, Value: ir.PhysicalValue{Variable: "root", Path: []string{"gender"}}}), + sql25CategoryColumn, sql25Marker, 8, + []string{"scalar-cardinality EXTRACT selector"}}, + {"generic selector mode", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, + func(extract *ir.PhysicalExtract, _ *ir.PhysicalProjection) { + extract.ExecutionMode = ir.PhysicalSelectorGeneric + })), sql25CategoryColumn, sql25Marker, 8, + []string{"execution mode"}}, + {"conditional-array selector mode", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, + func(extract *ir.PhysicalExtract, _ *ir.PhysicalProjection) { + extract.ExecutionMode = ir.PhysicalSelectorConditionalArray + })), sql25CategoryColumn, sql25Marker, 8, + []string{"execution mode"}}, + {"array cardinality", sql25Plan(func() ir.PhysicalProjection { + projection := sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil) + projection.Expression.Cardinality = ir.PhysicalArrayCardinality + return projection + }()), sql25CategoryColumn, sql25Marker, 8, []string{"scalar-cardinality EXTRACT selector"}}, + {"selector fallback", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, + func(extract *ir.PhysicalExtract, _ *ir.PhysicalProjection) { + extract.Fallbacks = []ir.PhysicalSelectorFallback{{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Patient", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: "maritalStatus"}}}, + }} + })), sql25CategoryColumn, sql25Marker, 8, + []string{"outside the registered single-path direct scalar form"}}, + {"distinct selector", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, + func(extract *ir.PhysicalExtract, _ *ir.PhysicalProjection) { extract.Distinct = true })), + sql25CategoryColumn, sql25Marker, 8, []string{"outside the registered single-path direct scalar form"}}, + {"selector filter", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, + func(extract *ir.PhysicalExtract, _ *ir.PhysicalProjection) { + extract.Selector = spec.Selector{Steps: []spec.SelectorStep{{Field: "gender"}}, + Filter: &spec.ContainsFilter{Field: "system", Needle: "urn:loinc"}} + })), sql25CategoryColumn, sql25Marker, 8, + []string{"outside the registered single-path direct scalar form"}}, + {"indexed step", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, + func(extract *ir.PhysicalExtract, _ *ir.PhysicalProjection) { + index := 0 + extract.Selector = spec.Selector{Steps: []spec.SelectorStep{{Field: "gender", Index: &index}}} + extract.ExecutionMode = ir.PhysicalSelectorGeneric + extract.Prepared = nil + })), sql25CategoryColumn, sql25Marker, 8, + []string{"execution mode"}}, + {"iterated step", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, + func(extract *ir.PhysicalExtract, _ *ir.PhysicalProjection) { + extract.Selector = spec.Selector{Steps: []spec.SelectorStep{{Field: "gender", Iterate: true}}} + extract.ExecutionMode = ir.PhysicalSelectorGeneric + })), sql25CategoryColumn, sql25Marker, 8, + []string{"execution mode"}}, + {"empty selector", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, + func(extract *ir.PhysicalExtract, _ *ir.PhysicalProjection) { + extract.Selector = spec.Selector{} + })), sql25CategoryColumn, sql25Marker, 8, + []string{"empty selector"}}, + {"bind-backed selector source", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, + func(extract *ir.PhysicalExtract, _ *ir.PhysicalProjection) { + extract.Source = ir.PhysicalValue{BindKey: "document"} + })), sql25CategoryColumn, sql25Marker, 8, + []string{`must extract from the row document at ["payload"]`}}, + {"nested document source", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, + func(extract *ir.PhysicalExtract, _ *ir.PhysicalProjection) { + extract.Source = ir.PhysicalValue{Variable: "root", Path: []string{"payload", "meta"}} + })), sql25CategoryColumn, sql25Marker, 8, + []string{`must extract from the row document at ["payload"]`}}, + {"nested selector path", planWithNestedSelector, sql25CategoryColumn, sql25Marker, 8, + []string{"crosses repeated field"}}, + {"foreign selector resource type", sql25Plan(sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, + func(extract *ir.PhysicalExtract, _ *ir.PhysicalProjection) { extract.ResourceType = "Observation" })), + sql25CategoryColumn, sql25Marker, 8, []string{"is not in the active FHIR schema"}}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + _, err := RenderCategoryScan(test.plan, bindingFor(t, sql25OfflineSchema), test.column, test.marker, test.bound) + if err == nil { + t.Fatalf("expected a refusal") + } + if !errors.Is(err, ErrUnsupportedPostgresPlan) && !strings.Contains(err.Error(), "validate physical plan") { + t.Fatalf("refusal must carry the typed unsupported-plan error or the spine validator's typed plan error: %v", err) + } + if strings.Contains(strings.ToLower(err.Error()), "aql") { + t.Fatalf("refusal must not offer an AQL fallback: %v", err) + } + for _, denial := range test.denials { + if !strings.Contains(err.Error(), denial) { + t.Fatalf("refusal must name %q: %v", denial, err) + } + } + }) + } +} + +func TestRenderCategoryScanGuardsAreLoadBearing(t *testing.T) { + category := sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil) + binding := bindingFor(t, sql25OfflineSchema) + + t.Run("a sorted plan loses its inner order but keeps a total page order", func(t *testing.T) { + sorted := sql25Plan(category) + sorted.Operations = append(sorted.Operations[:1], append([]ir.PhysicalOperation{{ + Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{Keys: []ir.PhysicalValue{{Variable: "root", Path: []string{"logical_id"}}}}, + }}, sorted.Operations[1:]...)...) + rendered := sql25Render(t, sorted, sql25CategoryColumn, sql25Marker, 8) + if strings.Contains(rendered.Values.SQL, `ORDER BY r0."logical_id"`) { + t.Fatalf("the plan sort survived into the inner read:\n%s", rendered.Values.SQL) + } + for _, key := range []string{ + `ORDER BY "present" ASC`, `END END ASC,`, + `CASE WHEN jsonb_typeof("value") = 'string' THEN "value" #>> '{}' END COLLATE "C" ASC, "value" ASC`, + } { + if !strings.Contains(rendered.Values.SQL, key) { + t.Fatalf("page order key %q missing:\n%s", key, rendered.Values.SQL) + } + } + }) + + t.Run("a marker collision refuses while a free marker renders", func(t *testing.T) { + colliding := sql25Plan(category, sql25CategoryProjection(sql25Marker, "status", ir.PhysicalPreserveNull, nil)) + if _, err := RenderCategoryScan(colliding, binding, sql25CategoryColumn, sql25Marker, 8); err == nil { + t.Fatal("a projection named like the presence marker must refuse") + } + free := sql25Plan(category, sql25CategoryProjection("status", "status", ir.PhysicalPreserveNull, nil)) + if _, err := RenderCategoryScan(free, binding, sql25CategoryColumn, sql25Marker, 8); err != nil { + t.Fatalf("an uncollided marker must render: %v", err) + } + }) + + t.Run("presence metadata is checked, not ignored", func(t *testing.T) { + agreeing := sql25Plan(category) + if _, err := RenderCategoryScan(agreeing, binding, sql25CategoryColumn, sql25Marker, 8); err != nil { + t.Fatalf("the lowered shape must render: %v", err) + } + mismatched := category + mismatched.Presence = &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, Paths: [][]string{{"activationDate"}}} + if _, err := RenderCategoryScan(sql25Plan(mismatched), binding, sql25CategoryColumn, sql25Marker, 8); err == nil { + t.Fatal("a presence claim for another path must refuse") + } + }) + + t.Run("the presence column is a real proof, not a constant", func(t *testing.T) { + rendered := sql25Render(t, sql25Plan(category), sql25CategoryColumn, sql25Marker, 8) + for _, forbidden := range []string{`true AS "c1"`, `false AS "c1"`, `NULL AS "c1"`} { + if strings.Contains(rendered.Values.SQL, forbidden) { + t.Fatalf("presence column degenerated to %q:\n%s", forbidden, rendered.Values.SQL) + } + } + if !strings.Contains(rendered.Values.SQL, "jsonb_exists(r0.\"payload\", $5)") { + t.Fatalf("presence proof is not the bound-key descent:\n%s", rendered.Values.SQL) + } + if rendered.Values.Args[4] != (command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: "gender"}) { + t.Fatalf("presence key argument = %#v", rendered.Values.Args[4]) + } + }) + + t.Run("an untrusted binding fails before any plan work", func(t *testing.T) { + if _, err := RenderCategoryScan(sql25Plan(category), ir.PostgresStorageBinding{}, sql25CategoryColumn, sql25Marker, 8); err == nil { + t.Fatal("the zero storage binding must refuse") + } else if !strings.Contains(err.Error(), "postgres") { + t.Fatalf("binding refusal = %v", err) + } + }) +} + +var ( + sql25FixturesMu sync.Mutex + sql25TrackedFixtures = map[string]bool{} +) + +func sql25RunDeltaAudit(inner func() int) int { + return runVerifyNamespaceDeltaAudit("verify_sql25v1_", &sql25FixturesMu, sql25TrackedFixtures, inner) +} + +func sortRows(rows [][]string) { + sort.Slice(rows, func(left, right int) bool { + for column := range rows[left] { + if rows[left][column] != rows[right][column] { + return rows[left][column] < rows[right][column] + } + } + return false + }) +} + +func sql25LiveService(t *testing.T) (*pgir02Service, ir.PostgresStorageBinding) { + t.Helper() + testPostgresURL(t) + svc := newOwnedService(t, 0, "verify_sql25v1") + return svc, bindingFor(t, svc.schema) +} + +func sql25PatientPlan(authPaths ...string) ir.PhysicalPlan { + return withReturn(scopedPlan(itProject, itGeneration, authPaths, false), + sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalPreserveNull, nil)) +} + +func sql25RenderLive(t *testing.T, plan ir.PhysicalPlan, binding ir.PostgresStorageBinding, maxValues int) RenderedCategoryScan { + t.Helper() + rendered, err := RenderCategoryScan(plan, binding, sql25CategoryColumn, sql25Marker, maxValues) + if err != nil { + t.Fatalf("live RenderCategoryScan: %v", err) + } + return rendered +} + +func TestCategoryScanLiveExactRowsScopeAndThreeStates(t *testing.T) { + svc, binding := sql25LiveService(t) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-female-1", "FHIR/Patient", map[string]any{"gender": "female"}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-female-2", "FHIR/Patient", map[string]any{"gender": "female"}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-male", "FHIR/Patient", map[string]any{"gender": "male"}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-absent", "FHIR/Patient", map[string]any{"active": true}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-null", "FHIR/Patient", map[string]any{"gender": nil}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-empty", "FHIR/Patient", map[string]any{"gender": ""}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-false", "FHIR/Patient", map[string]any{"gender": false}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-zero", "FHIR/Patient", map[string]any{"gender": 0}) + svc.insertResource(t, svc.schema, "sql25-foreign-project", itGeneration, "Patient", "p-decoy-project", "FHIR/Patient", map[string]any{"gender": "decoy-project"}) + svc.insertResource(t, svc.schema, itProject, "sql25-foreign-generation", "Patient", "p-decoy-generation", "FHIR/Patient", map[string]any{"gender": "decoy-generation"}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-decoy-auth", "FHIR/Encounter", map[string]any{"gender": "decoy-auth"}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Observation", "o-decoy-type", "FHIR/Observation", map[string]any{"gender": "decoy-type"}) + + rendered := sql25RenderLive(t, sql25PatientPlan("FHIR/Patient"), binding, 8) + got := svc.runRendered(t, rendered.Values) + want := [][]string{ + {"false", ""}, + {"true", "false"}, + {"true", ""}, + {"true", "0"}, + {"true", ""}, + {"true", "female"}, + {"true", "male"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("category page rows\ngot: %v\nwant: %v\nsql: %s", got, want, rendered.Values.SQL) + } + for _, forbidden := range []string{"decoy-project", "decoy-generation", "decoy-auth", "decoy-type"} { + for _, row := range got { + if strings.Contains(row[1], forbidden) { + t.Fatalf("foreign-scope data leaked into the page: %v", row) + } + } + } + + direct := svc.witness(t, ` + SELECT present::text, + CASE WHEN value IS NULL THEN '' + WHEN jsonb_typeof(value) = 'null' THEN '' + ELSE value #>> '{}' END + FROM ( + SELECT COALESCE(jsonb_typeof(payload) = 'object' AND jsonb_exists(payload, 'gender'), false) AS present, + CASE WHEN COALESCE(jsonb_typeof(payload) = 'object' AND jsonb_exists(payload, 'gender'), false) + THEN payload #> ARRAY['gender'] END AS value + FROM "`+svc.schema+`"."fhir_resource" + WHERE project = $1 AND dataset_generation = $2 AND resource_type = 'Patient' + AND auth_resource_path = ANY($3::text[]) + ) grouped GROUP BY present, value`, + itProject, itGeneration, []string{"FHIR/Patient"}) + wantDirect := [][]string{ + {"false", ""}, + {"true", "false"}, {"true", ""}, {"true", "0"}, + {"true", ""}, {"true", "female"}, {"true", "male"}, + } + sortedDirect := append([][]string(nil), direct...) + sortRows(sortedDirect) + sortRows(wantDirect) + if !reflect.DeepEqual(sortedDirect, wantDirect) { + t.Fatalf("direct-SQL witness\ngot: %v\nwant: %v", sortedDirect, wantDirect) + } + + confirmation := svc.runRendered(t, rendered.Witness) + if len(confirmation) != len(want) { + t.Fatalf("witness row count = %d, want %d: %v", len(confirmation), len(want), confirmation) + } + again := svc.runRendered(t, rendered.Values) + if !reflect.DeepEqual(again, got) { + t.Fatalf("page order was not deterministic:\nfirst: %v\nsecond: %v", got, again) + } +} + +func TestCategoryScanLiveEmptyPopulationHasNoFabricatedAggregateRow(t *testing.T) { + svc, binding := sql25LiveService(t) + svc.insertResource(t, svc.schema, "sql25-empty-observer", itGeneration, "Patient", "p-visible", "FHIR/Patient", map[string]any{"gender": "female"}) + + rendered := sql25RenderLive(t, sql25PatientPlan("FHIR/Patient"), binding, 8) + if got := svc.runRendered(t, rendered.Values); len(got) != 0 { + t.Fatalf("an empty population must return no rows, never a fabricated aggregate row: %v", got) + } + if got := svc.runRendered(t, rendered.Witness); len(got) != 0 { + t.Fatalf("the bounded confirmation must also be empty: %v", got) + } + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-in-scope", "FHIR/Patient", map[string]any{"gender": "female"}) + if got := svc.runRendered(t, rendered.Values); !reflect.DeepEqual(got, [][]string{{"true", "female"}}) { + t.Fatalf("in-scope population produced %v", got) + } +} + +func TestCategoryScanLiveBoundedPageDetectsTruncation(t *testing.T) { + svc, binding := sql25LiveService(t) + for index, gender := range []string{"a", "b", "c", "d"} { + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", fmt.Sprintf("p-%d", index), "FHIR/Patient", map[string]any{"gender": gender}) + } + rendered := sql25RenderLive(t, sql25PatientPlan("FHIR/Patient"), binding, 2) + rows := svc.runRendered(t, rendered.Values) + if len(rows) != 3 { + t.Fatalf("truncated page = %d rows, want maxValues+1=3: %v", len(rows), rows) + } + untruncated := sql25RenderLive(t, sql25PatientPlan("FHIR/Patient"), binding, 4) + if rows := svc.runRendered(t, untruncated.Values); len(rows) != 4 { + t.Fatalf("untruncated page = %d rows, want 4: %v", len(rows), rows) + } +} + +func TestCategoryScanLiveOmitNullsReadsJSONNullAsSQLNull(t *testing.T) { + svc, binding := sql25LiveService(t) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-null", "FHIR/Patient", map[string]any{"gender": nil}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-absent", "FHIR/Patient", map[string]any{"active": true}) + plan := withReturn(scopedPlan(itProject, itGeneration, []string{"FHIR/Patient"}, false), + sql25CategoryProjection(sql25CategoryColumn, "gender", ir.PhysicalOmitNulls, nil)) + rendered := sql25RenderLive(t, plan, binding, 8) + got := svc.runRendered(t, rendered.Values) + want := [][]string{{"false", ""}, {"true", ""}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("omit-nulls readings\ngot: %v\nwant: %v", got, want) + } +} + +func TestCategoryScanLivePageOrderIsBytePinnedNotCollationDependent(t *testing.T) { + svc, binding := sql25LiveService(t) + for _, gender := range []string{"zeta", "apple", "Äpfel", "Zebra"} { + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-"+gender, "FHIR/Patient", map[string]any{"gender": gender}) + } + collation := svc.witness(t, `SELECT datcollate FROM pg_database WHERE datname = current_database()`)[0][0] + + rendered := sql25RenderLive(t, sql25PatientPlan("FHIR/Patient"), binding, 8) + got := svc.runRendered(t, rendered.Values) + want := [][]string{{"true", "Zebra"}, {"true", "apple"}, {"true", "zeta"}, {"true", "Äpfel"}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("string page order is not byte-pinned (cluster datcollate = %s)\ngot: %v\nwant: %v\nsql: %s", collation, got, want, rendered.Values.SQL) + } + if collation != "C" && collation != "POSIX" && collation != "C.UTF-8" { + unpinned := svc.witness(t, `SELECT payload #>> '{gender}' FROM "`+svc.schema+`"."fhir_resource" `+ + `WHERE project = $1 AND dataset_generation = $2 AND resource_type = 'Patient' `+ + `AND auth_resource_path = ANY($3::text[]) GROUP BY 1 ORDER BY 1 ASC`, + itProject, itGeneration, []string{"FHIR/Patient"}) + plain := make([]string, len(unpinned)) + for index, row := range unpinned { + plain[index] = row[0] + } + if reflect.DeepEqual(plain, []string{"Zebra", "apple", "zeta", "Äpfel"}) { + t.Fatalf("the cluster collation orders like raw bytes; this cell cannot discriminate (datcollate = %s)", collation) + } + } +} diff --git a/internal/dataframe/compiler/render/postgres/cell_trace.go b/internal/dataframe/compiler/render/postgres/cell_trace.go new file mode 100644 index 000000000..72d5a4d9e --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/cell_trace.go @@ -0,0 +1,562 @@ +package postgres + +import ( + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const ( + cellTraceStatusCandidateLimit = 2 + cellTracePageAlias = "capped" +) + +func cellTraceTerminalMentions(plan ir.PhysicalPlan) bool { + if plan.StageSequence != nil { + return false + } + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalCellTraceReturnOp && operation.CellTraceReturn != nil { + return true + } + } + return false +} + +func renderCellTracePhysicalPlan(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + if err := storage.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := validateCellTracePlanSurface(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := plan.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("validate physical plan: %w", err) + } + if plan.Engine != "" && plan.Engine != ir.PhysicalEngineAQL { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: engine %q has no PostgreSQL source; the source graph IR is backend-neutral and ClickHouse plans are not PostgreSQL-rendered", + ErrUnsupportedPostgresPlan, string(plan.Engine)) + } + if plan.ClickHouseCombine != nil || plan.ClickHousePrefix != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: ClickHouse combine/prefix payloads have no PostgreSQL rendering", ErrUnsupportedPostgresPlan) + } + if len(plan.DeferredExpressionLets) != 0 { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: completed plans must not carry deferred expression lets", ErrUnsupportedPostgresPlan) + } + renderer := &planRenderer{ + storage: storage, + rows: map[string]rowScope{}, + parameters: map[string]int{}, + canonicalAuthLets: map[string]string{}, + } + if err := renderer.loadScope(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + sql, err := renderer.renderCellTraceStatement(plan) + if err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return command.NewRenderedPostgresPhysicalPlan(sql, renderer.args, renderer.outputColumns) +} + +func validateCellTracePlanSurface(plan ir.PhysicalPlan) error { + if plan.StageSequence != nil { + return fmt.Errorf("%w: cell-trace dispatch requires an inline plan; composed-construction traces keep the composition slice's registered voice", + ErrUnsupportedPostgresPlan) + } + if len(plan.Operations) == 0 { + return fmt.Errorf("%w: cell-trace dispatch requires at least one operation", ErrUnsupportedPostgresPlan) + } + terminal := (*ir.PhysicalCellTraceReturn)(nil) + terminalIndex := -1 + for index, operation := range plan.Operations { + if operation.Kind == ir.PhysicalCellTraceReturnOp { + if operation.CellTraceReturn == nil { + return fmt.Errorf("%w: operation %d: cell-trace terminal payload is missing", ErrUnsupportedPostgresPlan, index) + } + if terminal != nil { + return fmt.Errorf("%w: operation %d: a plan carries at most one cell-trace terminal", ErrUnsupportedPostgresPlan, index) + } + terminal = operation.CellTraceReturn + terminalIndex = index + continue + } + switch operation.Kind { + case ir.PhysicalPopulationMappingReturnOp, ir.PhysicalTableShapeExclusionReturnOp: + return fmt.Errorf("%w: operation %d: the %q companion terminal keeps its registered surface voice", + ErrUnsupportedPostgresPlan, index, string(operation.Kind)) + } + switch operation.Kind { + case ir.PhysicalRootScanOp, ir.PhysicalFilterOp, ir.PhysicalDerivedLetOp, + ir.PhysicalSetOp, ir.PhysicalSortOp, ir.PhysicalLimitOp: + default: + return fmt.Errorf("%w: operation %q is not part of the cell-trace render surface", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + if terminal == nil { + return fmt.Errorf("%w: cell-trace dispatch requires the evidence terminal", ErrUnsupportedPostgresPlan) + } + if terminalIndex != len(plan.Operations)-1 { + return fmt.Errorf("%w: the cell-trace terminal must be the final operation", ErrUnsupportedPostgresPlan) + } + if terminal.OmissionCode != "" { + return fmt.Errorf("%w: the inline main-path terminal mints no omission code; omission-bearing lineages refuse by name", + ErrUnsupportedPostgresPlan) + } + if terminal.Construction != nil { + return fmt.Errorf("%w: construction cell lineage registers with the composition slice's terminal wave", ErrUnsupportedPostgresPlan) + } + if terminal.Reshape != nil { + return fmt.Errorf("%w: table-shape (pivot/unpivot) cell lineage registers with the pivot and unpivot terminal surfaces", ErrUnsupportedPostgresPlan) + } + if err := validateCellTraceIdentity(terminal); err != nil { + return err + } + if err := validateCellTraceBinds(plan, terminal); err != nil { + return err + } + if terminal.Contribution != nil { + return validateCellTraceContribution(plan, terminal) + } + if terminal.Value.Kind != ir.PhysicalExtractExpression || terminal.Value.Extract == nil || + terminal.Value.Extract.Prepared != nil { + return fmt.Errorf("%w: contributor evidence registers only for the reduced-set contribution and the direct root selector extract; aggregate, owner-records, call-recode, and prepared-extract values register with their terminal surfaces", + ErrUnsupportedPostgresPlan) + } + if len(plan.Operations) < 2 || plan.Operations[0].Kind != ir.PhysicalRootScanOp || plan.Operations[0].RootScan == nil { + return fmt.Errorf("%w: cell-trace dispatch requires one leading direct ROOT_SCAN", ErrUnsupportedPostgresPlan) + } + if terminal.Value.Extract.Source.Variable != plan.Operations[0].RootScan.Variable { + return fmt.Errorf("%w: extract contributor %q must read the root row %q; set-sourced extract contributors have no registered item relation yet", + ErrUnsupportedPostgresPlan, terminal.Value.Extract.Source.Variable, plan.Operations[0].RootScan.Variable) + } + if terminal.Value.Cardinality != ir.PhysicalScalarCardinality { + return fmt.Errorf("%w: selector extraction is registered for SCALAR cardinality only; array and object selector projections belong to a later capability", + ErrUnsupportedPostgresPlan) + } + return nil +} + +func validateCellTraceIdentity(terminal *ir.PhysicalCellTraceReturn) error { + if len(terminal.IdentityParts) == 0 { + return fmt.Errorf("%w: the terminal carries no ordered identity parts; the mint mints them first and the IR validator never admits a parts-less terminal", ErrUnsupportedPostgresPlan) + } + for _, part := range terminal.IdentityParts { + if part.Name == "" { + return fmt.Errorf("%w: identity part without a name", ErrUnsupportedPostgresPlan) + } + if err := validateCellTraceIdentityExpression(part.Expression, "identity part "+strconv.Quote(part.Name)); err != nil { + return err + } + } + if terminal.ExplicitIdentity != nil { + return validateCellTraceIdentityExpression(*terminal.ExplicitIdentity, "explicit identity") + } + return nil +} + +func validateCellTraceIdentityExpression(expression ir.PhysicalExpression, label string) error { + switch expression.Kind { + case ir.PhysicalValueExpression: + if expression.Value == nil || (expression.Value.BindKey == "" && expression.Value.Variable == "") { + return fmt.Errorf("%w: %s is a VALUE without a bind key or row variable", ErrUnsupportedPostgresPlan, label) + } + return nil + case ir.PhysicalExtractExpression: + if expression.Extract == nil || expression.Extract.Prepared != nil { + return fmt.Errorf("%w: %s is a prepared extract, which has no registered PostgreSQL rendering", ErrUnsupportedPostgresPlan, label) + } + if expression.Extract.Source.Variable == "" { + return fmt.Errorf("%w: %s reads from no row variable", ErrUnsupportedPostgresPlan, label) + } + return nil + default: + return fmt.Errorf("%w: %s kind %q has no registered cell-trace rendering", ErrUnsupportedPostgresPlan, label, string(expression.Kind)) + } +} + +func validateCellTraceBinds(plan ir.PhysicalPlan, terminal *ir.PhysicalCellTraceReturn) error { + if terminal.OffsetBindKey == "" || terminal.LimitBindKey == "" || terminal.FetchLimitBindKey == "" { + return fmt.Errorf("%w: the terminal must carry the offset, limit, and fetch-limit bind keys", ErrUnsupportedPostgresPlan) + } + offset, ok := plan.BindVars[terminal.OffsetBindKey].(int) + if !ok || offset < 0 { + return fmt.Errorf("%w: cell-trace offset bind %q must be a non-negative int", ErrUnsupportedPostgresPlan, terminal.OffsetBindKey) + } + limit, ok := plan.BindVars[terminal.LimitBindKey].(int) + if !ok || limit < 1 { + return fmt.Errorf("%w: cell-trace limit bind %q must be an int of at least 1", ErrUnsupportedPostgresPlan, terminal.LimitBindKey) + } + fetch, ok := plan.BindVars[terminal.FetchLimitBindKey].(int) + if !ok || fetch != limit+1 { + return fmt.Errorf("%w: cell-trace fetch-limit bind %q must be the registered limit+1 lookahead", ErrUnsupportedPostgresPlan, terminal.FetchLimitBindKey) + } + return nil +} + +func validateCellTraceContribution(plan ir.PhysicalPlan, terminal *ir.PhysicalCellTraceReturn) error { + contribution := terminal.Contribution + if contribution.SetVariable == "" || contribution.ValueField == "" { + return fmt.Errorf("%w: the contribution must name its source set and value field", ErrUnsupportedPostgresPlan) + } + if terminal.Value.Kind != ir.PhysicalValueExpression || terminal.Value.Value == nil || + terminal.Value.Value.Variable == "" || len(terminal.Value.Value.Path) != 1 || + terminal.Value.Value.BindKey != "" { + return fmt.Errorf("%w: the reduced-set terminal value must read one field of the reduction variable", ErrUnsupportedPostgresPlan) + } + for _, operation := range plan.Operations { + if operation.Kind != ir.PhysicalSetOp || operation.Set == nil || operation.Set.Reduction == nil { + continue + } + set := operation.Set + if set.Reduction.Variable != terminal.Value.Value.Variable { + continue + } + if set.Variable != contribution.SetVariable || set.Reduction.SourceSetVariable != contribution.SetVariable { + return fmt.Errorf("%w: the reduced-set terminal reads reduction %q of set %q, whose contribution names set %q; only the self-sourced child-set reduction shape is registered", + ErrUnsupportedPostgresPlan, set.Reduction.Variable, set.Variable, contribution.SetVariable) + } + matched := false + for _, field := range set.Reduction.Fields { + if field.Name != terminal.Value.Value.Path[0] { + continue + } + if field.SourceField != contribution.ValueField { + return fmt.Errorf("%w: contribution value field %q disagrees with reduction field %q source slot %q", + ErrUnsupportedPostgresPlan, contribution.ValueField, field.Name, field.SourceField) + } + if lossy := field.Mode == ir.PhysicalSetReductionFirst; lossy != contribution.Lossy { + return fmt.Errorf("%w: contribution lossy=%v disagrees with reduction field %q mode %q", + ErrUnsupportedPostgresPlan, contribution.Lossy, field.Name, string(field.Mode)) + } + matched = true + } + if !matched { + return fmt.Errorf("%w: reduction field %q of set %q is unknown", ErrUnsupportedPostgresPlan, terminal.Value.Value.Path[0], set.Variable) + } + return nil + } + return fmt.Errorf("%w: the contribution names set %q, which carries no reduction over the terminal value", ErrUnsupportedPostgresPlan, contribution.SetVariable) +} + +func (r *planRenderer) renderCellTraceStatement(plan ir.PhysicalPlan) (string, error) { + dollar := func(index int) string { return "$" + strconv.Itoa(index) } + terminal := plan.Operations[len(plan.Operations)-1].CellTraceReturn + phase := 0 + sawSort, sawLimit := false, false + rootVariable := "" + for index, operation := range plan.Operations { + last := index == len(plan.Operations)-1 + switch operation.Kind { + case ir.PhysicalRootScanOp: + if phase != 0 || index != 0 { + return "", fmt.Errorf("%w: exactly one leading ROOT_SCAN is required", ErrUnsupportedPostgresPlan) + } + scan := operation.RootScan + if scan.Population != nil || scan.CohortSource != nil || scan.PageCandidateSeed != nil { + return "", fmt.Errorf("%w: membership and candidate-seed roots keep the population surfaces' registered contracts; the cell-trace arms correlate direct stored rows", + ErrUnsupportedPostgresPlan) + } + if err := r.renderRootScan(plan, *scan); err != nil { + return "", err + } + rootVariable = scan.Variable + case ir.PhysicalFilterOp: + if phase != 0 { + return "", fmt.Errorf("%w: filters must precede the execution window", ErrUnsupportedPostgresPlan) + } + canonical, err := r.acceptCanonicalScopeFilter(operation.Filter) + if err != nil { + return "", err + } + if canonical { + continue + } + clause, err := r.renderFilter(plan, *operation.Filter) + if err != nil { + return "", err + } + r.where = append(r.where, clause) + case ir.PhysicalDerivedLetOp: + if phase != 0 { + return "", fmt.Errorf("%w: derived lets must precede the execution window", ErrUnsupportedPostgresPlan) + } + canonical, err := r.acceptCanonicalAuthLet(operation.DerivedLet) + if err != nil { + return "", err + } + if canonical { + continue + } + return "", fmt.Errorf("%w: operation %q is not part of the cell-trace render surface", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + case ir.PhysicalSetOp: + if phase > 1 || (phase == 1 && sawLimit) { + return "", fmt.Errorf("%w: sets must precede the execution window's LIMIT", ErrUnsupportedPostgresPlan) + } + if operation.Set == nil { + return "", fmt.Errorf("%w: operation %d: set payload is missing", ErrUnsupportedPostgresPlan, index) + } + if err := r.renderSet(plan, *operation.Set); err != nil { + return "", err + } + case ir.PhysicalSortOp: + if sawSort || phase > 1 || (phase == 1 && sawLimit) { + return "", fmt.Errorf("%w: at most one SORT, before LIMIT", ErrUnsupportedPostgresPlan) + } + if err := r.renderWindowSort(plan, *operation.Sort); err != nil { + return "", err + } + sawSort, phase = true, 1 + case ir.PhysicalLimitOp: + if sawLimit || phase > 1 { + return "", fmt.Errorf("%w: at most one LIMIT", ErrUnsupportedPostgresPlan) + } + if err := r.renderWindowLimit(plan, *operation.Limit); err != nil { + return "", err + } + sawLimit, phase = true, 1 + case ir.PhysicalCellTraceReturnOp: + if !last || phase > 1 { + return "", fmt.Errorf("%w: the cell-trace terminal must be the final operation", ErrUnsupportedPostgresPlan) + } + default: + return "", fmt.Errorf("%w: operation %q is not part of the cell-trace render surface", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + if rootVariable == "" || r.rootVariable != rootVariable { + return "", fmt.Errorf("%w: the cell-trace terminal requires the direct root row", ErrUnsupportedPostgresPlan) + } + return r.renderCellTraceColumns(plan, terminal, rootVariable, dollar) +} + +func (r *planRenderer) renderCellTraceColumns(plan ir.PhysicalPlan, terminal *ir.PhysicalCellTraceReturn, rootVariable string, dollar func(int) string) (string, error) { + valueText, err := r.renderCellTraceExpression(plan, "trace value", terminal.Value) + if err != nil { + return "", err + } + + var contributors, status, hasMore string + if terminal.Contribution != nil { + contributors, status, hasMore, err = r.renderCellTraceSetArms(plan, terminal, valueText, dollar) + } else { + contributors, status, hasMore, err = r.renderCellTraceRootArms(plan, terminal, rootVariable, valueText, dollar) + } + if err != nil { + return "", err + } + if terminal.RowExists { + status = "CASE WHEN " + cellTraceIsNullForm(valueText) + " THEN 'RECORDED_NULL' ELSE 'VALUE' END" + } + + identityText := "" + identityColumn := ir.PhysicalCellTraceIdentityPartsField + if terminal.ExplicitIdentity != nil { + rendered, renderErr := r.renderCellTraceExpression(plan, "explicit identity", *terminal.ExplicitIdentity) + if renderErr != nil { + return "", renderErr + } + identityText = rendered + identityColumn = ir.PhysicalCellTraceExplicitIdentityField + } else { + parts := make([]string, 0, len(terminal.IdentityParts)) + for _, part := range terminal.IdentityParts { + rendered, renderErr := r.renderCellTraceExpression(plan, "identity part "+strconv.Quote(part.Name), part.Expression) + if renderErr != nil { + return "", renderErr + } + parts = append(parts, rendered) + } + identityText = "jsonb_build_array(" + strings.Join(parts, ", ") + ")" + } + + omission := dollar(r.textParameter("", "cell-trace-omission")) + "::text" + + r.selects = []string{ + valueText + ` AS "` + ir.PhysicalCellTraceValueField + `"`, + contributors + ` AS "` + ir.PhysicalCellTraceContributionsField + `"`, + identityText + ` AS "` + identityColumn + `"`, + status + ` AS "` + ir.PhysicalCellTraceStatusField + `"`, + hasMore + ` AS "` + ir.PhysicalCellTraceHasMoreField + `"`, + omission + ` AS "` + ir.PhysicalCellTraceOmissionField + `"`, + } + r.outputColumns = []string{ + ir.PhysicalCellTraceValueField, ir.PhysicalCellTraceContributionsField, identityColumn, + ir.PhysicalCellTraceStatusField, ir.PhysicalCellTraceHasMoreField, ir.PhysicalCellTraceOmissionField, + } + return r.assemble(), nil +} + +func (r *planRenderer) renderCellTraceSetArms(plan ir.PhysicalPlan, terminal *ir.PhysicalCellTraceReturn, valueText string, dollar func(int) string) (contributors, status, hasMore string, err error) { + contribution := terminal.Contribution + corr, ok := r.setBodies[contribution.SetVariable] + if !ok { + return "", "", "", fmt.Errorf("%w: contribution set %q did not register a correlated item relation", ErrUnsupportedPostgresPlan, contribution.SetVariable) + } + fields := r.setProjections[contribution.SetVariable] + field := ir.PhysicalSetProjectionField{} + found := false + for _, candidate := range fields { + if candidate.Name == contribution.ValueField { + field, found = candidate, true + } + } + if !found { + return "", "", "", fmt.Errorf("%w: contribution value field %q is not a projection slot of set %q", + ErrUnsupportedPostgresPlan, contribution.ValueField, contribution.SetVariable) + } + + seq := r.nextTraversalSequence() + itemAlias := "i" + strconv.Itoa(seq) + elementAlias := "s" + strconv.Itoa(seq) + logicalIDColumn := r.storage.Resource.MustColumn("logical_id").SQLText() + resourceTypeColumn := r.storage.Resource.MustColumn("resource_type").SQLText() + payloadColumn := r.storage.Resource.MustColumn("payload").SQLText() + items := "(SELECT DISTINCT " + corr.targetAlias + "." + logicalIDColumn + " AS " + logicalIDColumn + + ", " + corr.targetAlias + "." + resourceTypeColumn + " AS " + resourceTypeColumn + + ", " + corr.targetAlias + "." + payloadColumn + " AS " + payloadColumn + + " " + corr.from + " WHERE " + corr.where + ")" + slot, err := r.setProjectionSlotExpression(field, corr, itemAlias) + if err != nil { + return "", "", "", fmt.Errorf("contributor slot %q: %w", contribution.ValueField, err) + } + lateral := " CROSS JOIN LATERAL jsonb_array_elements(" + slot + ") WITH ORDINALITY AS " + + elementAlias + "(v, ord)" + fromClause := items + " AS " + itemAlias + lateral + itemOrder := itemAlias + "." + logicalIDColumn + elementOrder := elementAlias + ".ord" + totalOrder := itemOrder + ", " + elementOrder + + offsetIndex := r.integerParameter(plan.BindVars[terminal.OffsetBindKey].(int)) + fetchIndex := r.integerParameter(plan.BindVars[terminal.FetchLimitBindKey].(int)) + limitIndex := r.integerParameter(plan.BindVars[terminal.LimitBindKey].(int)) + + page := "(SELECT " + itemAlias + "." + logicalIDColumn + " AS c_lid, " + + itemAlias + "." + resourceTypeColumn + " AS c_rt, " + + elementAlias + ".v AS c_v, " + elementAlias + ".ord AS c_ord FROM " + fromClause + + " ORDER BY " + totalOrder + + " OFFSET " + dollar(offsetIndex) + " LIMIT " + dollar(fetchIndex) + ")" + contributors = "COALESCE((SELECT jsonb_agg(jsonb_build_object(" + + "'resourceType', " + cellTracePageAlias + ".c_rt, " + + "'resourceId', " + cellTracePageAlias + ".c_lid, " + + "'value', " + cellTracePageAlias + ".c_v" + + ") ORDER BY " + cellTracePageAlias + ".c_lid, " + cellTracePageAlias + ".c_ord" + + ") FROM (SELECT * FROM " + page + " AS " + cellTracePageAlias + + " ORDER BY " + cellTracePageAlias + ".c_lid, " + cellTracePageAlias + ".c_ord" + + " LIMIT " + dollar(limitIndex) + ") AS " + cellTracePageAlias + "), '[]'::jsonb)" + candidateCount := "(SELECT COUNT(*) FROM (SELECT 1 FROM " + fromClause + + " ORDER BY " + totalOrder + " LIMIT " + strconv.Itoa(cellTraceStatusCandidateLimit) + ") AS __loom_t_status)" + hasMore = "(SELECT COUNT(*) > " + dollar(limitIndex) + " FROM (SELECT 1 FROM " + fromClause + + " ORDER BY " + totalOrder + " OFFSET " + dollar(offsetIndex) + " LIMIT " + dollar(fetchIndex) + ") AS __loom_t_more)" + status = cellTraceStatusCase(candidateCount, valueText, contribution.Lossy && !terminal.RowExists) + return contributors, status, hasMore, nil +} + +func (r *planRenderer) renderCellTraceRootArms(plan ir.PhysicalPlan, terminal *ir.PhysicalCellTraceReturn, rootVariable string, valueText string, dollar func(int) string) (contributors, status, hasMore string, err error) { + scope, ok := r.rows[rootVariable] + if !ok || scope.derivedValue != "" { + return "", "", "", fmt.Errorf("%w: cell-trace root %q does not bind a stored row", ErrUnsupportedPostgresPlan, rootVariable) + } + logicalIDColumn := r.storage.Resource.MustColumn("logical_id").SQLText() + resourceTypeColumn := r.storage.Resource.MustColumn("resource_type").SQLText() + + itemExists := "NOT " + cellTraceIsNullForm(valueText) + record := "jsonb_build_object('resourceType', " + r.rootAlias + "." + resourceTypeColumn + + ", 'resourceId', " + r.rootAlias + "." + logicalIDColumn + + ", 'value', " + valueText + ")" + offsetIndex := r.integerParameter(plan.BindVars[terminal.OffsetBindKey].(int)) + contributors = "CASE WHEN " + itemExists + " AND " + dollar(offsetIndex) + " = 0 THEN jsonb_build_array(" + record + ") ELSE '[]'::jsonb END" + candidateCount := "(CASE WHEN " + itemExists + " THEN 1 ELSE 0 END)" + status = cellTraceStatusCase(candidateCount, valueText, false) + hasMore = "false" + return contributors, status, hasMore, nil +} + +func cellTraceStatusCase(candidateCount, valueText string, lossy bool) string { + var builder strings.Builder + builder.WriteString("CASE WHEN " + candidateCount + " = 0 THEN 'NO_MATCH'") + builder.WriteString(" WHEN " + cellTraceIsNullForm(valueText) + " THEN 'RECORDED_NULL'") + if lossy { + builder.WriteString(" WHEN " + candidateCount + " > 1 THEN 'AMBIGUOUS'") + } + builder.WriteString(" ELSE 'VALUE' END") + return builder.String() +} + +func cellTraceIsNullForm(valueText string) string { + return "(((" + valueText + ") IS NULL) OR jsonb_typeof((" + valueText + ")) = 'null')" +} + +func (r *planRenderer) renderCellTraceExpression(plan ir.PhysicalPlan, label string, expression ir.PhysicalExpression) (string, error) { + switch expression.Kind { + case ir.PhysicalValueExpression: + if expression.Value == nil { + return "", fmt.Errorf("%w: %s VALUE has no payload", ErrUnsupportedPostgresPlan, label) + } + if expression.Value.BindKey != "" { + raw, ok := plan.BindVars[expression.Value.BindKey].(string) + if !ok { + return "", fmt.Errorf("%w: %s bind %q must be a non-empty string identity bind", ErrUnsupportedPostgresPlan, label, expression.Value.BindKey) + } + index := r.textParameter(raw, "text:"+raw) + return "to_jsonb($" + strconv.Itoa(index) + "::text)", nil + } + resolved, err := r.resolveValue(*expression.Value) + if err != nil { + return "", fmt.Errorf("%w: %s: %w", ErrUnsupportedPostgresPlan, label, err) + } + if isPayloadExpression(resolved) { + resolved = "(" + resolved + ")" + } + return resolved, nil + case ir.PhysicalExtractExpression: + if expression.Extract == nil || expression.Extract.Prepared != nil { + return "", fmt.Errorf("%w: %s is a prepared extract, which has no registered PostgreSQL rendering", ErrUnsupportedPostgresPlan, label) + } + classified, err := classifyExtractionProjections([]ir.PhysicalProjection{{Name: label, Expression: &expression}}, func(variable string) (string, bool) { + rowScope, ok := r.rows[variable] + return rowScope.resourceType, ok + }) + if err != nil { + return "", err + } + if len(classified) != 1 { + return "", fmt.Errorf("%w: %s did not classify as one projection", ErrUnsupportedPostgresPlan, label) + } + switch classified[0].kind { + case extractionDirectValue, extractionValueExpression: + value := classified[0].projection.Value + if classified[0].kind == extractionValueExpression { + value = *classified[0].projection.Expression.Value + } + resolved, err := r.resolveValue(value) + if err != nil { + return "", fmt.Errorf("%w: %s: %w", ErrUnsupportedPostgresPlan, label, err) + } + if isPayloadExpression(resolved) { + resolved = "(" + resolved + ")" + } + return resolved, nil + case extractionSelector: + scope := r.rows[classified[0].expression.Extract.Source.Variable] + value, err := r.scalarExtractValueExpression(scope.alias, classified[0].path, classified[0].expression.NullBehavior) + if err != nil { + return "", fmt.Errorf("%w: %s: %w", ErrUnsupportedPostgresPlan, label, err) + } + return value, nil + default: + return "", fmt.Errorf("%w: %s kind %q has no registered cell-trace rendering", + ErrUnsupportedPostgresPlan, label, string(expression.Kind)) + } + default: + return "", fmt.Errorf("%w: %s kind %q has no registered cell-trace rendering", ErrUnsupportedPostgresPlan, label, string(expression.Kind)) + } +} diff --git a/internal/dataframe/compiler/render/postgres/cell_trace_test.go b/internal/dataframe/compiler/render/postgres/cell_trace_test.go new file mode 100644 index 000000000..cbf43385d --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/cell_trace_test.go @@ -0,0 +1,1002 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "regexp" + "strconv" + "strings" + "sync" + "testing" + "time" + + control "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" + "github.com/jackc/pgx/v5/pgxpool" +) + +func sql24ArmBOutput() recipe.Output { + return recipe.Output{ + Name: "Patients", RootResourceType: "Patient", RowGrain: "patient", + RootColumnNaming: recipe.RootColumnNamingExact, + Fields: []recipe.Field{ + {Name: "patient_id", Expr: recipe.Expression{Select: "root.id"}}, + {Name: "gender", Expr: recipe.Expression{Select: "root.gender"}}, + }, + } +} + +func sql24ArmAOutput() recipe.Output { + return recipe.Output{ + Name: "Patients", RootResourceType: "Patient", RowGrain: "patient", + RootColumnNaming: recipe.RootColumnNamingExact, TraversalColumnNaming: recipe.TraversalColumnNamingExact, + Fields: []recipe.Field{{Name: "patient_id", Expr: recipe.Expression{Select: "root.id"}}}, + Traversals: []recipe.Traversal{{ + Name: "subject_Patient", ToResourceType: "Condition", Alias: "condition", MatchMode: recipe.MatchOptional, + Fields: []recipe.Field{{Name: "condition_id", Expr: recipe.Expression{Select: "id"}, ValueMode: recipe.ValueModeFirst}}, + }}, + } +} + +func sql24ArmAAllOutput() recipe.Output { + output := sql24ArmAOutput() + output.Traversals[0].Fields[0].ValueMode = recipe.ValueModeAll + return output +} + +func sql24Lower(t *testing.T, output recipe.Output) lower.CompiledRecipeOutput { + t.Helper() + bindings := recipe.RuntimeBindings{Project: group13Project, DatasetGeneration: group13Generation, AuthResourcePaths: []string{group13AuthPath}} + bundle := recipe.Bundle{RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: output.Name, TranslationVersion: "test", Outputs: []recipe.Output{output}} + if err := bundle.Validate(); err != nil { + t.Fatalf("bundle validate: %v", err) + } + plan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy(). + WithRule(ir.PhysicalOptimizationRuleEndpointTraversal, false)) + if err != nil { + t.Fatalf("lower.CompileResolvedRecipePlan: %v", err) + } + for _, candidate := range compiled.Outputs { + if candidate.Name == output.Name { + return candidate + } + } + t.Fatalf("lowered outputs do not carry %q", output.Name) + return lower.CompiledRecipeOutput{} +} + +func sql24MirrorProjectionExpression(projection ir.PhysicalProjection) (ir.PhysicalExpression, error) { + if projection.Expression != nil { + return ir.ClonePhysicalExpression(*projection.Expression), nil + } + if projection.Value.Variable == "" && projection.Value.BindKey == "" { + return ir.PhysicalExpression{}, fmt.Errorf("final return projection has no typed expression or value") + } + return ir.PhysicalExpression{Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, NullBehavior: ir.PhysicalPreserveNull, Value: &ir.PhysicalValue{Variable: projection.Value.Variable, BindKey: projection.Value.BindKey, Path: append([]string(nil), projection.Value.Path...)}}, nil +} + +func sql24ApplyGenericWindow(t *testing.T, plan ir.PhysicalPlan) ir.PhysicalPlan { + t.Helper() + root := plan.Operations[0].RootScan.Variable + insertAt := ir.PhysicalScopeWindowEnd(plan.Operations, 1) + if insertAt <= 1 || insertAt >= len(plan.Operations) { + t.Fatalf("fixture plan does not carry a scoped root followed by RETURN or traversal") + } + window := ir.PhysicalOperation{ + Kind: ir.PhysicalSortOp, + Source: ir.PhysicalSource{SemanticNode: plan.Source.SemanticNode, ResourceType: plan.Source.ResourceType, SemanticField: "_key"}, + Sort: &ir.PhysicalSort{Keys: []ir.PhysicalValue{{Variable: root, Path: []string{"_key"}}}}, + } + operations := make([]ir.PhysicalOperation, 0, len(plan.Operations)+1) + operations = append(operations, plan.Operations[:insertAt]...) + operations = append(operations, window) + operations = append(operations, plan.Operations[insertAt:]...) + plan.Operations = operations + return plan +} + +func sql24AttachTrace(t *testing.T, output lower.CompiledRecipeOutput, column string, offset, limit int) ir.PhysicalPlan { + t.Helper() + plan := sql24ApplyGenericWindow(t, ir.ClonePhysicalPlan(output.Plan)) + terminalIndex := -1 + for index, operation := range plan.Operations { + if operation.Kind == ir.PhysicalReturnOp && operation.Return != nil { + terminalIndex = index + } + } + if terminalIndex < 0 { + t.Fatalf("fixture plan has no RETURN terminal") + } + projections := map[string]ir.PhysicalProjection{} + for _, projection := range plan.Operations[terminalIndex].Return.Projections { + projections[projection.Name] = projection + } + identityParts := make([]ir.PhysicalPopulationMappingIdentityPart, 0, len(output.RowIdentity.Fields)) + for _, field := range output.RowIdentity.Fields { + if projection, ok := projections[field]; ok { + expression, err := sql24MirrorProjectionExpression(projection) + if err != nil { + t.Fatalf("identity field %q: %v", field, err) + } + identityParts = append(identityParts, ir.PhysicalPopulationMappingIdentityPart{Name: field, Expression: expression}) + continue + } + if _, bound := plan.BindVars[field]; bound { + identityParts = append(identityParts, ir.PhysicalPopulationMappingIdentityPart{Name: field, + Expression: ir.PhysicalExpression{Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, NullBehavior: ir.PhysicalPreserveNull, Value: &ir.PhysicalValue{BindKey: field}}}) + continue + } + t.Fatalf("identity field %q is not present in the final return projection or compiler bind vars", field) + } + var explicit *ir.PhysicalExpression + if projection, ok := projections["__loom_row_id"]; ok { + expression, err := sql24MirrorProjectionExpression(projection) + if err != nil { + t.Fatalf("explicit row identity: %v", err) + } + explicit = &expression + } + wanted, ok := projections[column] + if !ok || wanted.Hidden { + t.Fatalf("column %q is not a public output projection", column) + } + value, err := sql24MirrorProjectionExpression(wanted) + if err != nil { + t.Fatalf("trace column %q: %v", column, err) + } + var contribution *ir.PhysicalCellTraceContribution + if value.Kind == ir.PhysicalValueExpression && value.Value != nil && value.Value.Variable != "" && len(value.Value.Path) == 1 { + for _, operation := range plan.Operations { + if operation.Kind != ir.PhysicalSetOp || operation.Set == nil || operation.Set.Reduction == nil || + operation.Set.Reduction.Variable != value.Value.Variable { + continue + } + for _, field := range operation.Set.Reduction.Fields { + if field.Name != value.Value.Path[0] { + continue + } + contribution = &ir.PhysicalCellTraceContribution{ + SetVariable: operation.Set.Reduction.SourceSetVariable, + ValueField: field.SourceField, + Lossy: field.Mode == ir.PhysicalSetReductionFirst, + } + } + } + } + plan.BindVars["cell_trace_offset"] = offset + plan.BindVars["cell_trace_limit"] = limit + plan.BindVars["cell_trace_fetch_limit"] = limit + 1 + plan.Operations[terminalIndex] = ir.PhysicalOperation{ + Kind: ir.PhysicalCellTraceReturnOp, Source: plan.Operations[terminalIndex].Source, + CellTraceReturn: &ir.PhysicalCellTraceReturn{ + Value: value, Contribution: contribution, IdentityParts: identityParts, ExplicitIdentity: explicit, + OffsetBindKey: "cell_trace_offset", LimitBindKey: "cell_trace_limit", FetchLimitBindKey: "cell_trace_fetch_limit", + }, + } + if err := plan.Validate(); err != nil { + t.Fatalf("plan with the cell-trace terminal must validate: %v", err) + } + return plan +} + +func sql24ArmBPlan(t *testing.T, column string, offset, limit int) ir.PhysicalPlan { + t.Helper() + output := sql24Lower(t, sql24ArmBOutput()) + return sql24AttachTrace(t, output, column, offset, limit) +} + +func sql24ArmBIdentityPlan(t *testing.T, column string, offset, limit int) ir.PhysicalPlan { + t.Helper() + output := sql24ArmBOutput() + output.Identity = &recipe.Identity{Name: "row", Expr: recipe.Expression{Select: "root.id"}} + lowered := sql24Lower(t, output) + plan := sql24AttachTrace(t, lowered, column, offset, limit) + sawRowID := false + for _, operation := range lowered.Plan.Operations { + if operation.Return == nil { + continue + } + for _, projection := range operation.Return.Projections { + if projection.Name == "__loom_row_id" { + sawRowID = true + } + } + } + if !sawRowID { + t.Fatal("authored Identity did not project __loom_row_id into the final RETURN") + } + return plan +} + +func sql24ArmAPlan(t *testing.T, all bool, offset, limit int) ir.PhysicalPlan { + t.Helper() + output := sql24Lower(t, sql24ArmAOutput()) + if all { + output = sql24Lower(t, sql24ArmAAllOutput()) + } + return sql24AttachTrace(t, output, "condition_id", offset, limit) +} + +func sql24Render(t *testing.T, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + return rendered +} + +func sql24WantColumns(identityColumn string) []string { + return []string{ + ir.PhysicalCellTraceValueField, ir.PhysicalCellTraceContributionsField, identityColumn, + ir.PhysicalCellTraceStatusField, ir.PhysicalCellTraceHasMoreField, ir.PhysicalCellTraceOmissionField, + } +} + +func sql24ContractCells(t *testing.T, rendered command.RenderedPostgresPhysicalPlan) map[string]string { + t.Helper() + cells := map[string]string{} + for _, column := range rendered.OutputColumns { + marker := ` AS "` + column + `"` + if !strings.Contains(rendered.SQL, marker) { + t.Fatalf("statement does not publish %q as a named column:\n%s", column, rendered.SQL) + } + } + return cells +} + +func TestCellTracePostgresRootExtractContract(t *testing.T) { + plan := sql24ArmBPlan(t, "gender", 2, 5) + rendered := sql24Render(t, plan) + want := sql24WantColumns(ir.PhysicalCellTraceIdentityPartsField) + if strings.Join(rendered.OutputColumns, ",") != strings.Join(want, ",") { + t.Fatalf("output columns = %v, want %v", rendered.OutputColumns, want) + } + sql := rendered.SQL + for _, fragment := range []string{ + `#> $5::text[]) AS "__loom_trace_value"`, + `jsonb_build_object('resourceType',`, `'resourceId',`, + `ELSE '[]'::jsonb END AS "__loom_trace_contributions"`, + `jsonb_build_array(to_jsonb($1::text), r0."logical_id") AS "__loom_trace_identity_parts"`, + `= 0 THEN 'NO_MATCH' WHEN`, `THEN 'RECORDED_NULL' ELSE 'VALUE' END AS "__loom_trace_status"`, + `false AS "__loom_trace_has_more"`, + `$7::text AS "__loom_trace_omission"`, + } { + if !strings.Contains(sql, fragment) { + t.Fatalf("arm B statement missing %q:\n%s", fragment, sql) + } + } + gate := regexp.MustCompile(`AND \$(\d+) = 0 THEN jsonb_build_array\(`).FindStringSubmatch(sql) + if gate == nil { + t.Fatalf("arm B must gate the record on the sealed offset bind:\n%s", sql) + } + gateIndex, convErr := strconv.Atoi(gate[1]) + if convErr != nil || gateIndex < 1 || gateIndex > len(rendered.Args) { + t.Fatalf("offset gate parameter $%s is not a registered arg: %v", gate[1], convErr) + } + if fmt.Sprint(rendered.Args[gateIndex-1].Value) != "2" { + t.Fatalf("offset gate arg = %v, want the sealed offset 2", rendered.Args[gateIndex-1].Value) + } + if strings.Contains(sql, "OFFSET") || strings.Contains(sql, "LIMIT") { + t.Fatalf("arm B has provable zero-or-one bounds (D-24b): no OFFSET/LIMIT page clause may appear — the offset law rides the sealed bind gate:\n%s", sql) + } + if strings.Contains(sql, "'AMBIGUOUS'") { + t.Fatalf("arm B is never lossy; no AMBIGUOUS branch may appear:\n%s", sql) + } + if !strings.Contains(sql, `ORDER BY r0."logical_id" ASC, r0."resource_seq" ASC`) { + t.Fatalf("arm B must keep the stable execution-window order:\n%s", sql) + } + sql24ContractCells(t, rendered) +} + +func TestCellTracePostgresExplicitIdentityContract(t *testing.T) { + plan := sql24ArmBPlan(t, "gender", 0, 5) + terminal := sql24Terminal(t, plan) + if terminal.ExplicitIdentity != nil || len(terminal.IdentityParts) == 0 { + t.Fatal("fixture must mint the identity-parts spelling only") + } + explicit := ir.ClonePhysicalExpression(terminal.Value) + terminal.ExplicitIdentity = &explicit + rendered := sql24Render(t, plan) + want := sql24WantColumns(ir.PhysicalCellTraceExplicitIdentityField) + if strings.Join(rendered.OutputColumns, ",") != strings.Join(want, ",") { + t.Fatalf("output columns = %v, want %v", rendered.OutputColumns, want) + } + sql := rendered.SQL + if !strings.Contains(sql, ` AS "`+ir.PhysicalCellTraceExplicitIdentityField+`"`) { + t.Fatalf("the explicit identity must be published as its contract column in the select list:\n%s", sql) + } + if strings.Contains(sql, ir.PhysicalCellTraceIdentityPartsField) || strings.Contains(sql, "jsonb_build_array(to_jsonb") { + t.Fatalf("the identity-parts spelling must stay unpublished while the explicit identity is the contract column:\n%s", sql) + } + sql24ContractCells(t, rendered) +} + +func TestCellTracePostgresRefusesIdentityWithoutParts(t *testing.T) { + plan := sql24ArmBPlan(t, "gender", 0, 5) + terminal := sql24Terminal(t, plan) + explicit := ir.ClonePhysicalExpression(terminal.Value) + terminal.ExplicitIdentity = &explicit + terminal.IdentityParts = nil + sql24Refuse(t, plan, "no ordered identity parts") +} + +func TestCellTracePostgresAuthoredIdentityContract(t *testing.T) { + rendered := sql24Render(t, sql24ArmBIdentityPlan(t, "gender", 0, 5)) + want := sql24WantColumns(ir.PhysicalCellTraceExplicitIdentityField) + if strings.Join(rendered.OutputColumns, ",") != strings.Join(want, ",") { + t.Fatalf("output columns = %v, want %v", rendered.OutputColumns, want) + } + if !strings.Contains(rendered.SQL, ` AS "`+ir.PhysicalCellTraceExplicitIdentityField+`"`) { + t.Fatalf("authored Identity must publish the explicit contract column:\n%s", rendered.SQL) + } + if strings.Contains(rendered.SQL, ir.PhysicalCellTraceIdentityPartsField) { + t.Fatalf("identity-parts spelling must stay unpublished:\n%s", rendered.SQL) + } + sql24ContractCells(t, rendered) +} + +func sql24Terminal(t *testing.T, plan ir.PhysicalPlan) *ir.PhysicalCellTraceReturn { + t.Helper() + for index := range plan.Operations { + if terminal := plan.Operations[index].CellTraceReturn; terminal != nil { + return terminal + } + } + t.Fatal("fixture lost its cell-trace terminal") + return nil +} + +func TestCellTracePostgresReducedSetContract(t *testing.T) { + plan := sql24ArmAPlan(t, false, 5, 10) + rendered := sql24Render(t, plan) + want := sql24WantColumns(ir.PhysicalCellTraceIdentityPartsField) + if strings.Join(rendered.OutputColumns, ",") != strings.Join(want, ",") { + t.Fatalf("output columns = %v, want %v", rendered.OutputColumns, want) + } + sql := rendered.SQL + for _, fragment := range []string{ + `AS "__loom_trace_value"`, + `jsonb_agg(jsonb_build_object('resourceType', capped.c_rt, 'resourceId', capped.c_lid, 'value', capped.c_v) ORDER BY capped.c_lid, capped.c_ord)`, + `OFFSET $10 LIMIT $11) AS capped`, + `ORDER BY capped.c_lid, capped.c_ord LIMIT $12) AS capped), '[]'::jsonb) AS "__loom_trace_contributions"`, + `COUNT(*) > $12 FROM`, `AS __loom_t_more) AS "__loom_trace_has_more"`, + `LIMIT 2) AS __loom_t_status) = 0 THEN 'NO_MATCH'`, + `> 1 THEN 'AMBIGUOUS'`, + `ELSE 'VALUE' END AS "__loom_trace_status"`, + } { + if !strings.Contains(sql, fragment) { + t.Fatalf("arm A statement missing %q:\n%s", fragment, sql) + } + } + if got := fmt.Sprint(rendered.Args[9].Value, rendered.Args[10].Value, rendered.Args[11].Value); got != "5 11 10" { + t.Fatalf("page bind args = %s, want \"5 11 10\": %v", got, rendered.Args) + } + if !strings.Contains(sql, `WITH ORDINALITY`) || !strings.Contains(sql, `SELECT DISTINCT`) { + t.Fatalf("arm A must reuse the spine's registered DISTINCT-item + WITH ORDINALITY relation:\n%s", sql) + } +} + +func TestCellTracePostgresReducedSetIsSpineTwin(t *testing.T) { + plan := sql24ArmAPlan(t, true, 0, 3) + rendered := sql24Render(t, plan) + sql := rendered.SQL + valueEnd := strings.Index(sql, ` AS "__loom_trace_value"`) + if valueEnd < 0 { + t.Fatalf("no value column:\n%s", sql) + } + valueText := sql[:valueEnd] + for _, fragment := range []string{ + `"fhir_edge_occurrence" AS e1`, `WITH ORDINALITY AS s`, + `ORDER BY i`, `.ord`, `SELECT DISTINCT`, + } { + if !strings.Contains(valueText, fragment) || !strings.Contains(sql[valueEnd:], fragment) { + t.Fatalf("twin fragment %q must appear in both the value column and the contributor page:\n%s", fragment, sql) + } + } +} + +func TestCellTracePostgresEntryEqualsDispatch(t *testing.T) { + for _, plan := range []ir.PhysicalPlan{sql24ArmBPlan(t, "gender", 2, 5), sql24ArmAPlan(t, false, 5, 10)} { + throughEntry := sql24Render(t, ir.ClonePhysicalPlan(plan)) + direct, err := renderCellTracePhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if err != nil { + t.Fatalf("direct door: %v", err) + } + if direct.SQL != throughEntry.SQL || fmt.Sprint(direct.Args) != fmt.Sprint(throughEntry.Args) { + t.Fatalf("door drifted from the entry render") + } + } +} + +func TestCellTracePostgresHeadDoorClaimsEveryTerminalBearingPlan(t *testing.T) { + plan := sql24ArmBPlan(t, "gender", 0, 5) + last := len(plan.Operations) - 1 + plan.Operations = append(plan.Operations[:last], + ir.PhysicalOperation{Kind: ir.PhysicalGraphReturnOp}, plan.Operations[last]) + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil || !strings.Contains(err.Error(), "not part of the cell-trace render surface") { + t.Fatalf("terminal-carrying plan must refuse in the cell-trace slice, got: %v", err) + } + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("refusal must be the typed unsupported-plan error: %v", err) + } +} + +func sql24Refuse(t *testing.T, plan ir.PhysicalPlan, want string) { + t.Helper() + _, err := renderCellTracePhysicalPlan(plan, mustBinding(t)) + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("refusal = %v, want error naming %q", err, want) + } + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("refusal %v is not the typed unsupported-plan error", err) + } +} + +func TestCellTracePostgresSurfaceRefusals(t *testing.T) { + base := sql24ArmBPlan(t, "gender", 2, 5) + armA := sql24ArmAPlan(t, false, 5, 10) + + composed := ir.ClonePhysicalPlan(base) + composed.StageSequence = &ir.PhysicalStageSequence{SourceStageID: "s"} + sql24Refuse(t, composed, "requires an inline plan") + + construction := ir.ClonePhysicalPlan(base) + construction.Operations[len(construction.Operations)-1].CellTraceReturn.Construction = &ir.PhysicalCellTraceConstruction{} + sql24Refuse(t, construction, "composition slice's terminal wave") + + reshape := ir.ClonePhysicalPlan(base) + reshape.Operations[len(reshape.Operations)-1].CellTraceReturn.Reshape = &ir.PhysicalCellTraceReshape{} + sql24Refuse(t, reshape, "pivot and unpivot terminal surfaces") + + omission := ir.ClonePhysicalPlan(base) + omission.Operations[len(omission.Operations)-1].CellTraceReturn.OmissionCode = "TABLE_SHAPE_OMITTED_ROW" + sql24Refuse(t, omission, "mints no omission code") + + neither := ir.ClonePhysicalPlan(base) + neither.Operations[len(neither.Operations)-1].CellTraceReturn.IdentityParts = nil + sql24Refuse(t, neither, "no ordered identity parts") + + badBind := ir.ClonePhysicalPlan(base) + badBind.BindVars["cell_trace_offset"] = "5" + sql24Refuse(t, badBind, "offset bind") + + badLookahead := ir.ClonePhysicalPlan(base) + badLookahead.BindVars["cell_trace_fetch_limit"] = 7 + sql24Refuse(t, badLookahead, "limit+1 lookahead") + + negativeOffset := ir.ClonePhysicalPlan(base) + negativeOffset.BindVars["cell_trace_offset"] = -1 + sql24Refuse(t, negativeOffset, "non-negative int") + + missingBindKeys := ir.ClonePhysicalPlan(base) + missingBindKeys.Operations[len(missingBindKeys.Operations)-1].CellTraceReturn.LimitBindKey = "" + sql24Refuse(t, missingBindKeys, "offset, limit, and fetch-limit bind keys") + + aggregateValue := ir.ClonePhysicalPlan(base) + aggregateValue.Operations[len(aggregateValue.Operations)-1].CellTraceReturn.Value = ir.PhysicalExpression{Kind: ir.PhysicalAggregateExpression, Cardinality: ir.PhysicalScalarCardinality} + sql24Refuse(t, aggregateValue, "aggregate, owner-records, call-recode, and prepared-extract values") + + prepared := ir.ClonePhysicalPlan(base) + prepared.Operations[len(prepared.Operations)-1].CellTraceReturn.Value.Extract.Prepared = &ir.PhysicalPreparedReference{SetVariable: "s", Field: "f"} + sql24Refuse(t, prepared, "prepared-extract values register with their terminal surfaces") + + arrayCardinality := ir.ClonePhysicalPlan(base) + arrayCardinality.Operations[len(arrayCardinality.Operations)-1].CellTraceReturn.Value.Cardinality = ir.PhysicalArrayCardinality + sql24Refuse(t, arrayCardinality, "SCALAR cardinality only") + + foreignExtract := ir.ClonePhysicalPlan(base) + foreignExtract.Operations[len(foreignExtract.Operations)-1].CellTraceReturn.Value.Extract.Source.Variable = "somebody_elses_row" + sql24Refuse(t, foreignExtract, "must read the root row") + + setMismatch := ir.ClonePhysicalPlan(armA) + setMismatch.Operations[len(setMismatch.Operations)-1].CellTraceReturn.Contribution.SetVariable = "unregistered_set" + sql24Refuse(t, setMismatch, "self-sourced child-set reduction shape") + + fieldMismatch := ir.ClonePhysicalPlan(armA) + fieldMismatch.Operations[len(fieldMismatch.Operations)-1].CellTraceReturn.Contribution.ValueField = "not_a_slot" + sql24Refuse(t, fieldMismatch, "disagrees with reduction field") + + lossyMismatch := ir.ClonePhysicalPlan(armA) + lossyMismatch.Operations[len(lossyMismatch.Operations)-1].CellTraceReturn.Contribution.Lossy = !lossyMismatch.Operations[len(lossyMismatch.Operations)-1].CellTraceReturn.Contribution.Lossy + sql24Refuse(t, lossyMismatch, "lossy=") + + valueNotReductionRead := ir.ClonePhysicalPlan(armA) + valueNotReductionRead.Operations[len(valueNotReductionRead.Operations)-1].CellTraceReturn.Value.Value.Path = []string{"deeper", "path"} + sql24Refuse(t, valueNotReductionRead, "one field of the reduction variable") + + secondTerminal := ir.ClonePhysicalPlan(base) + secondTerminal.Operations = append(secondTerminal.Operations[:1], append([]ir.PhysicalOperation{ir.ClonePhysicalOperation(secondTerminal.Operations[len(secondTerminal.Operations)-1])}, secondTerminal.Operations[1:]...)...) + sql24Refuse(t, secondTerminal, "at most one cell-trace terminal") + + survivingReturn := sql24ArmBPlan(t, "gender", 2, 5) + survivingReturn.Operations = append(survivingReturn.Operations[:len(survivingReturn.Operations)-1], + ir.PhysicalOperation{Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{}}, + survivingReturn.Operations[len(survivingReturn.Operations)-1]) + sql24Refuse(t, survivingReturn, "not part of the cell-trace render surface") + + traversalOp := ir.ClonePhysicalPlan(armA) + traversalOp.Operations = append(traversalOp.Operations[:1], append([]ir.PhysicalOperation{ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{}}}, traversalOp.Operations[1:]...)...) + sql24Refuse(t, traversalOp, "not part of the cell-trace render surface") + + companion := ir.ClonePhysicalPlan(base) + companion.Operations = append(companion.Operations, ir.PhysicalOperation{Kind: ir.PhysicalPopulationMappingReturnOp, PopulationMappingReturn: &ir.PhysicalPopulationMappingReturn{}}) + sql24Refuse(t, companion, "companion terminal keeps its registered surface voice") + + terminalNotLast := ir.ClonePhysicalPlan(base) + terminalNotLast.Operations = append(terminalNotLast.Operations, ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{}}) + sql24Refuse(t, terminalNotLast, "final operation") +} + +func TestCellTracePostgresDoorClaimsNothingElse(t *testing.T) { + if cellTraceTerminalMentions(sql24ArmBPlan(t, "gender", 0, 5)) != true { + t.Fatal("a terminal-carrying inline plan must be claimed") + } + withoutTerminal := ir.ClonePhysicalPlan(sql24Lower(t, sql24ArmBOutput()).Plan) + if cellTraceTerminalMentions(withoutTerminal) { + t.Fatal("a plan without the terminal op must not be claimed") + } + rendered, err := RenderPhysicalPlan(withoutTerminal, mustBinding(t)) + if err != nil { + t.Fatalf("a RETURN-terminal plan must keep its pre-slice render path: %v", err) + } + for _, column := range rendered.OutputColumns { + if strings.HasPrefix(column, "__loom_trace_") { + t.Fatalf("unclaimed plan leaked a trace column %q", column) + } + } +} + +var ( + sql24FixturesMu sync.Mutex + sql24TrackedFixtures = map[string]bool{} +) + +func sql24TrackFixtures(t *testing.T, svc *pgir02Service) { + t.Helper() + var database string + if err := svc.iso.QueryRow(svc.ctx, "SELECT current_database()").Scan(&database); err != nil { + svc.fail("track owned database", err) + } + sql24FixturesMu.Lock() + defer sql24FixturesMu.Unlock() + sql24TrackedFixtures[database] = true + sql24TrackedFixtures[svc.schema] = true +} + +func sql24SnapshotOwnedResources(ctx context.Context, raw string) ([]string, error) { + pool, err := pgxpool.New(ctx, raw) + if err != nil { + return nil, fmt.Errorf("connect: %w", err) + } + defer pool.Close() + rows, err := pool.Query(ctx, + `SELECT datname FROM pg_database WHERE datname LIKE 'verify_sql24v1_%'`+ + ` UNION ALL SELECT nspname FROM pg_namespace WHERE nspname LIKE 'verify_sql24v1_%'`) + if err != nil { + return nil, fmt.Errorf("snapshot: %w", err) + } + defer rows.Close() + var names []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("scan: %v", err) + } + names = append(names, name) + } + return names, rows.Err() +} + +func sql24RunDeltaAudit(inner func() int) int { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return inner() + } + beforeList, err := sql24SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-24 delta audit: snapshot before run: %v\n", err) + return inner() + } + if len(beforeList) > 0 { + fmt.Fprintf(os.Stderr, "pg-sql-24 delta audit: pre-snapshot found %d surviving verify_sql24v1_* resources before this run: %v\n", len(beforeList), beforeList) + return 1 + } + before := make(map[string]bool, len(beforeList)) + for _, name := range beforeList { + before[name] = true + } + code := inner() + after, err := sql24SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-24 delta audit: snapshot after run: %v\n", err) + code = 1 + } + sql24FixturesMu.Lock() + defer sql24FixturesMu.Unlock() + for _, name := range after { + if before[name] { + continue + } + if sql24TrackedFixtures[name] { + fmt.Fprintf(os.Stderr, "pg-sql-24 delta audit: %q was created and tracked by this run and still survives\n", name) + code = 1 + continue + } + fmt.Fprintf(os.Stderr, "pg-sql-24 delta audit: note — %q was born during this run but is not owned by it (sibling lane?)\n", name) + } + var survivors []string + for _, name := range after { + if !before[name] && sql24TrackedFixtures[name] { + survivors = append(survivors, name) + } + } + if len(survivors) > 0 { + fmt.Fprintf(os.Stderr, "pg-sql-24 delta audit: %d verify_sql24v1_* resources survive: %v\n", len(survivors), survivors) + code = 1 + } + return code +} + +func newSQL24Service(t *testing.T) *pgir02Service { + t.Helper() + return newOwnedService(t, 0, "verify_sql24v1") +} + +func sql24Insert(t *testing.T, svc *pgir02Service, resourceType, logicalID string, payload map[string]any) { + t.Helper() + svc.insertResource(t, svc.schema, group13Project, group13Generation, resourceType, logicalID, group13AuthPath, payload) +} + +func sql24Edge(t *testing.T, svc *pgir02Service, conditionID, patientID, sourcePath string) { + t.Helper() + svc.insertEdge(t, control.EdgeOccurrenceRow{ + Project: group13Project, DatasetGeneration: group13Generation, AuthResourcePath: group13AuthPath, + Label: "subject_Patient", SourceResourceType: "Condition", SourceLogicalID: conditionID, + TargetResourceType: "Patient", TargetLogicalID: patientID, SourcePath: sourcePath, + }) +} + +type sql24PageRow struct { + value, contributions, identity, status, hasMore, omission string +} + +func sql24RunPage(t *testing.T, svc *pgir02Service, plan ir.PhysicalPlan) []sql24PageRow { + t.Helper() + rendered := group13RenderSvc(t, svc, plan) + if len(rendered.OutputColumns) != 6 { + t.Fatalf("contract columns = %v, want the six registered names", rendered.OutputColumns) + } + identityColumn := rendered.OutputColumns[2] + if identityColumn != ir.PhysicalCellTraceIdentityPartsField && identityColumn != ir.PhysicalCellTraceExplicitIdentityField { + t.Fatalf("third contract column = %q, want an identity spelling", identityColumn) + } + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + rows, err := svc.iso.Query(svc.ctx, + `SELECT w."`+ir.PhysicalCellTraceValueField+`"::text, w."`+ir.PhysicalCellTraceContributionsField+`"::text,`+ + ` w."`+identityColumn+`"::text, w."`+ir.PhysicalCellTraceStatusField+`"::text,`+ + ` w."`+ir.PhysicalCellTraceHasMoreField+`"::text, w."`+ir.PhysicalCellTraceOmissionField+`"::text FROM (`+rendered.SQL+`) AS w`, args...) + if err != nil { + t.Fatalf("execute contributor page: %v", err) + } + defer rows.Close() + var out []sql24PageRow + for rows.Next() { + values, err := rows.Values() + if err != nil { + t.Fatalf("read contributor page row: %v", err) + } + if len(values) != 6 { + t.Fatalf("contributor page row must carry the six contract columns: %v", values) + } + out = append(out, sql24PageRow{ + value: stringifyCell(values[0]), contributions: stringifyCell(values[1]), + identity: stringifyCell(values[2]), status: stringifyCell(values[3]), + hasMore: stringifyCell(values[4]), omission: stringifyCell(values[5]), + }) + } + if err := rows.Err(); err != nil { + t.Fatalf("finish contributor page: %v", err) + } + return out +} + +func sql24RunTextColumns(t *testing.T, svc *pgir02Service, rendered command.RenderedPostgresPhysicalPlan, columns int) [][]string { + t.Helper() + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + selected := make([]string, columns) + for index := range selected { + selected[index] = fmt.Sprintf(`w."c%d"::text`, index) + } + rows, err := svc.iso.Query(svc.ctx, `SELECT `+strings.Join(selected, ", ")+` FROM (`+rendered.SQL+`) AS w`, args...) + if err != nil { + t.Fatalf("execute rendered statement (text cast): %v", err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + t.Fatalf("read row: %v", err) + } + row := make([]string, len(values)) + for index, value := range values { + row[index] = stringifyCell(value) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + t.Fatalf("finish rows: %v", err) + } + return out +} + +func sql24ContributorMembers(t *testing.T, cell string) []string { + t.Helper() + var members []struct { + ResourceType string `json:"resourceType"` + ResourceID json.RawMessage `json:"resourceId"` + Value json.RawMessage `json:"value"` + } + if err := json.Unmarshal([]byte(cell), &members); err != nil { + t.Fatalf("decode contributors %q: %v", cell, err) + } + out := make([]string, len(members)) + for i, member := range members { + out[i] = member.ResourceType + "/" + string(member.ResourceID) + "=" + string(member.Value) + } + return out +} + +func sql24RowBylogicalID(t *testing.T, rows []sql24PageRow, logicalID string) sql24PageRow { + t.Helper() + for _, row := range rows { + if strings.Contains(row.identity, `"`+logicalID+`"`) { + return row + } + } + t.Fatalf("no traced row for %q in %v", logicalID, rows) + return sql24PageRow{} +} + +func TestCellTracePostgresLiveRootExtract(t *testing.T) { + svc := newSQL24Service(t) + sql24TrackFixtures(t, svc) + sql24Insert(t, svc, "Patient", "p-1", map[string]any{"id": "p-1", "gender": "male"}) + sql24Insert(t, svc, "Patient", "p-2", map[string]any{"id": "p-2", "gender": nil}) + sql24Insert(t, svc, "Patient", "p-3", map[string]any{"id": "p-3"}) + svc.insertResource(t, svc.schema, "other-project", group13Generation, "Patient", "p-9", group13AuthPath, + map[string]any{"id": "p-9", "gender": "male"}) + + rows := sql24RunPage(t, svc, sql24ArmBPlan(t, "gender", 0, 5)) + if len(rows) != 3 { + t.Fatalf("exactly the three scoped patients are traced: %v", rows) + } + male := sql24RowBylogicalID(t, rows, "p-1") + if male.status != "VALUE" || male.value != `"male"` || male.hasMore != "false" || male.omission != "" { + t.Fatalf("p-1 = %+v", male) + } + if got := sql24ContributorMembers(t, male.contributions); len(got) != 1 || got[0] != `Patient/"p-1"="male"` { + t.Fatalf("p-1 contributors = %v, want the traced row itself", got) + } + if !strings.Contains(male.identity, `"`+group13Project+`"`) || !strings.Contains(male.identity, `"p-1"`) { + t.Fatalf("p-1 identity = %s", male.identity) + } + recordedNull := sql24RowBylogicalID(t, rows, "p-2") + if recordedNull.status != "NO_MATCH" || recordedNull.value != "null" { + t.Fatalf("p-2 (JSON-null gender) = %+v", recordedNull) + } + if recordedNull.contributions != "[]" || recordedNull.hasMore != "false" { + t.Fatalf("p-2 must publish zero contributors and no more pages: %+v", recordedNull) + } + missing := sql24RowBylogicalID(t, rows, "p-3") + if missing.status != "NO_MATCH" || missing.value != "" { + t.Fatalf("p-3 (absent gender: SQL NULL, distinct from the recorded JSON null) = %+v", missing) + } +} + +func TestCellTracePostgresLiveRootExtractOffsetGate(t *testing.T) { + svc := newSQL24Service(t) + sql24TrackFixtures(t, svc) + sql24Insert(t, svc, "Patient", "p-1", map[string]any{"id": "p-1", "gender": "male"}) + + for _, offset := range []int{1, 2} { + rows := sql24RunPage(t, svc, sql24ArmBPlan(t, "gender", offset, 5)) + male := sql24RowBylogicalID(t, rows, "p-1") + if male.status != "VALUE" || male.value != `"male"` || male.hasMore != "false" { + t.Fatalf("offset %d: candidateCount/status stay offset-free, the LIMIT-2 twin law: %+v", offset, male) + } + if male.contributions != "[]" { + t.Fatalf("offset %d must skip the sole candidate to the empty page (AQL LIMIT @offset twin): %+v", offset, male) + } + } + baseline := sql24RunPage(t, svc, sql24ArmBPlan(t, "gender", 0, 5)) + male := sql24RowBylogicalID(t, baseline, "p-1") + if got := sql24ContributorMembers(t, male.contributions); len(got) != 1 || got[0] != `Patient/"p-1"="male"` { + t.Fatalf("offset 0 must publish the self contributor (the gate is not always-empty): %v", got) + } +} + +func TestCellTracePostgresLiveAuthoredIdentityPage(t *testing.T) { + svc := newSQL24Service(t) + sql24TrackFixtures(t, svc) + sql24Insert(t, svc, "Patient", "p-1", map[string]any{"id": "p-1", "gender": "male"}) + sql24Insert(t, svc, "Patient", "p-2", map[string]any{"id": "p-2", "gender": "female"}) + + rows := sql24RunPage(t, svc, sql24ArmBIdentityPlan(t, "gender", 0, 5)) + if len(rows) != 2 { + t.Fatalf("two scoped patients: %v", rows) + } + first := sql24RowBylogicalID(t, rows, "p-1") + if first.identity != `"p-1"` { + t.Fatalf("p-1 must publish the authored explicit identity through the dispatcher: %+v", first) + } + if first.status != "VALUE" || first.value != `"male"` { + t.Fatalf("p-1 evidence = %+v", first) + } + if got := sql24ContributorMembers(t, first.contributions); len(got) != 1 || got[0] != `Patient/"p-1"="male"` { + t.Fatalf("p-1 contributors = %v", got) + } +} + +func TestCellTracePostgresLivePreviewRoundTrip(t *testing.T) { + svc := newSQL24Service(t) + sql24TrackFixtures(t, svc) + sql24Insert(t, svc, "Patient", "p-1", map[string]any{"id": "p-1", "gender": "male"}) + sql24Insert(t, svc, "Patient", "p-2", map[string]any{"id": "p-2", "gender": "female"}) + + preview := ir.ClonePhysicalPlan(sql24Lower(t, sql24ArmBOutput()).Plan) + rendered := group13RenderSvc(t, svc, preview) + previewRows := sql24RunTextColumns(t, svc, rendered, len(rendered.OutputColumns)) + if len(previewRows) != 2 { + t.Fatalf("preview must publish both scoped patients: %v", previewRows) + } + previewGender := map[string]string{} + for _, row := range previewRows { + previewGender[row[0]] = row[len(row)-1] + } + rows := sql24RunPage(t, svc, sql24ArmBPlan(t, "gender", 0, 5)) + for _, row := range rows { + var identity []string + if err := json.Unmarshal([]byte(row.identity), &identity); err != nil { + t.Fatalf("decode identity %q: %v", row.identity, err) + } + logicalID := identity[len(identity)-1] + want, ok := previewGender[logicalID] + if !ok { + t.Fatalf("traced row %q is not in the preview %v", logicalID, previewRows) + } + if row.value != want { + t.Fatalf("trace value %q drifted from the published preview value %q for %q", row.value, want, logicalID) + } + } +} + +func TestCellTracePostgresLiveReducedSetPages(t *testing.T) { + svc := newSQL24Service(t) + sql24TrackFixtures(t, svc) + sql24Insert(t, svc, "Patient", "p-1", map[string]any{"id": "p-1"}) + sql24Insert(t, svc, "Patient", "p-none", map[string]any{"id": "p-none"}) + for _, condition := range []string{"c-1", "c-2", "c-3"} { + sql24Insert(t, svc, "Condition", condition, map[string]any{"id": condition}) + sql24Edge(t, svc, condition, "p-1", "subject") + } + + all := sql24RunPage(t, svc, sql24ArmAPlan(t, true, 0, 5)) + row := sql24RowBylogicalID(t, all, "p-1") + var reduced []string + if err := json.Unmarshal([]byte(row.value), &reduced); err != nil { + t.Fatalf("ALL-mode value column %q is not the published reduced array: %v", row.value, err) + } + if len(reduced) != 3 { + t.Fatalf("reduced column = %v", reduced) + } + if row.status != "VALUE" || row.hasMore != "false" { + t.Fatalf("p-1 all-mode page = %+v", row) + } + members := sql24ContributorMembers(t, row.contributions) + if len(members) != 3 { + t.Fatalf("contributors = %v, want the full reduced set %v", members, reduced) + } + for i, member := range members { + if want := `Condition/"` + reduced[i] + `"="` + reduced[i] + `"`; member != want { + t.Fatalf("contributor %d = %q, want %q (order must follow the published reduced column)", i, member, want) + } + } + empty := sql24RowBylogicalID(t, all, "p-none") + if empty.status != "NO_MATCH" || empty.contributions != "[]" || empty.hasMore != "false" { + t.Fatalf("p-none (no conditions) = %+v", empty) + } + + first := sql24RunPage(t, svc, sql24ArmAPlan(t, false, 0, 2)) + row = sql24RowBylogicalID(t, first, "p-1") + if row.status != "AMBIGUOUS" { + t.Fatalf("FIRST over three candidates must read AMBIGUOUS: %+v", row) + } + if want := `"` + reduced[0] + `"`; row.value != want { + t.Fatalf("FIRST value %q must be the first published item %q", row.value, want) + } + if got := sql24ContributorMembers(t, row.contributions); len(got) != 2 || got[0] != members[0] { + t.Fatalf("FIRST page cap = %v, want the first two items of %v", got, members) + } + if row.hasMore != "true" { + t.Fatalf("three candidates with limit 2 must report hasMore: %+v", row) + } + + last := sql24RunPage(t, svc, sql24ArmAPlan(t, true, 2, 2)) + row = sql24RowBylogicalID(t, last, "p-1") + if got := sql24ContributorMembers(t, row.contributions); len(got) != 1 || got[0] != members[2] { + t.Fatalf("offset page = %v, want only %v", got, members[2]) + } + if row.hasMore != "false" { + t.Fatalf("the bounded lookahead must report no more pages: %+v", row) + } +} + +func TestCellTracePostgresLiveScopeSabotage(t *testing.T) { + svc := newSQL24Service(t) + sql24TrackFixtures(t, svc) + sql24Insert(t, svc, "Patient", "p-1", map[string]any{"id": "p-1"}) + sql24Insert(t, svc, "Condition", "c-1", map[string]any{"id": "c-1"}) + sql24Edge(t, svc, "c-1", "p-1", "subject") + + before := sql24RunPage(t, svc, sql24ArmAPlan(t, true, 0, 5)) + if got := sql24ContributorMembers(t, sql24RowBylogicalID(t, before, "p-1").contributions); len(got) != 1 { + t.Fatalf("baseline contributor = %v", got) + } + + scratch := fmt.Sprintf("verify_sql24v1_scratch_%d", time.Now().UnixNano()) + if _, err := svc.iso.Exec(svc.ctx, `CREATE SCHEMA "`+scratch+`"`); err != nil { + svc.fail("create scratch schema", err) + } + sql24FixturesMu.Lock() + sql24TrackedFixtures[scratch] = true + sql24FixturesMu.Unlock() + t.Cleanup(func() { + svc.iso.Exec(context.WithoutCancel(svc.ctx), `DROP SCHEMA IF EXISTS "`+scratch+`" CASCADE`) + }) + if _, err := control.MigrateSchema(svc.ctx, svc.client, scratch); err != nil { + svc.fail("migrate scratch schema", err) + } + svc.insertResource(t, scratch, group13Project, group13Generation, "Condition", "c-shadow", group13AuthPath, + map[string]any{"id": "c-shadow"}) + + after := sql24RunPage(t, svc, sql24ArmAPlan(t, true, 0, 5)) + if got := sql24ContributorMembers(t, sql24RowBylogicalID(t, after, "p-1").contributions); len(got) != 1 || got[0] != `Condition/"c-1"="c-1"` { + t.Fatalf("scratch-schema shadow leaked into the contributor page: %v", got) + } + + sql24Insert(t, svc, "Condition", "c-2", map[string]any{"id": "c-2"}) + sql24Edge(t, svc, "c-2", "p-1", "subject") + grown := sql24RunPage(t, svc, sql24ArmAPlan(t, true, 0, 5)) + if got := sql24ContributorMembers(t, sql24RowBylogicalID(t, grown, "p-1").contributions); len(got) != 2 { + t.Fatalf("the page must follow the bound store: %v", got) + } +} + +func cloneForPreview(t *testing.T, output lower.CompiledRecipeOutput) ir.PhysicalPlan { + t.Helper() + return ir.ClonePhysicalPlan(output.Plan) +} diff --git a/internal/dataframe/compiler/render/postgres/coded_group.go b/internal/dataframe/compiler/render/postgres/coded_group.go new file mode 100644 index 000000000..44d2cd596 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/coded_group.go @@ -0,0 +1,625 @@ +package postgres + +import ( + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const codedGroupRowIDColumn = "__loom_row_id" + +const codedGroupMissingKeyTag = "CONSTRUCTION_CODED_GROUP_MISSING_KEY" + +const codedGroupTrimCharacters = " " + +func (r *planRenderer) codedGroupTrimmedNonBlank(expr string) string { + trimChars := r.textParameter(codedGroupTrimCharacters, "text:"+codedGroupTrimCharacters) + dollar := func(index int) string { return "$" + strconv.Itoa(index) } + return "btrim(" + expr + ", " + dollar(trimChars) + "::text) <> ''" +} + +func codedGroupPlanMentionsStage(plan ir.PhysicalPlan) bool { + if plan.StageSequence == nil { + return false + } + for _, stage := range plan.StageSequence.Stages { + if stage.Kind == ir.PhysicalStageCodedGroupOp { + return true + } + } + return false +} + +func renderCodedGroupPhysicalPlan(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + if err := storage.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := validateCodedGroupPlanSurface(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := plan.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("validate physical plan: %w", err) + } + if plan.Engine != "" && plan.Engine != ir.PhysicalEngineAQL { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: engine %q has no PostgreSQL source; the source graph IR is backend-neutral and ClickHouse plans are not PostgreSQL-rendered", + ErrUnsupportedPostgresPlan, string(plan.Engine)) + } + if plan.ClickHouseCombine != nil || plan.ClickHousePrefix != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: ClickHouse combine/prefix payloads have no PostgreSQL rendering", ErrUnsupportedPostgresPlan) + } + if len(plan.DeferredExpressionLets) != 0 { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: completed plans must not carry deferred expression lets", ErrUnsupportedPostgresPlan) + } + + renderer := &planRenderer{ + storage: storage, + rows: map[string]rowScope{}, + parameters: map[string]int{}, + canonicalAuthLets: map[string]string{}, + } + if err := renderer.loadScope(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + sql, err := renderer.renderCodedGroupStatement(plan) + if err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return command.NewRenderedPostgresPhysicalPlan(sql, renderer.args, renderer.outputColumns) +} + +func validateCodedGroupPlanSurface(plan ir.PhysicalPlan) error { + if plan.StageSequence == nil { + return fmt.Errorf("%w: coded-group dispatch requires a stage sequence", ErrUnsupportedPostgresPlan) + } + sequence := plan.StageSequence + if len(sequence.Stages) != 1 { + return fmt.Errorf("%w: the registered coded-group slice admits exactly one CODED_GROUP construction stage; multi-stage composition is a pg-sql-16 capability gap", + ErrUnsupportedPostgresPlan) + } + stage := sequence.Stages[0] + if stage.Kind != ir.PhysicalStageCodedGroupOp || stage.CodedGroup == nil { + return fmt.Errorf("%w: operation %q is not part of the registered coded-group slice", + ErrUnsupportedPostgresPlan, string(stage.Kind)) + } + if stage.Group != nil || stage.CohortGroup != nil || stage.Filter != nil || stage.Expand != nil || + stage.GroupedPivot != nil || stage.Unpivot != nil || stage.RelatedSource != nil || + stage.RelatedExpand != nil || stage.RelatedField != nil || len(stage.DerivedLets) != 0 { + return fmt.Errorf("%w: CODED_GROUP requires only a coded-group payload (ir/physical_stage_validation.go:187 re-read at the door)", + ErrUnsupportedPostgresPlan) + } + if stage.InputStageID != sequence.SourceStageID || stage.ID != sequence.FinalStageID { + return fmt.Errorf("%w: the registered slice admits only the inline first-stage CODED_GROUP (InputStageID must equal %q); the DOCUMENT re-query shape is production-unreachable (recipe/construction.go:665-668)", + ErrUnsupportedPostgresPlan, sequence.SourceStageID) + } + if sequence.CellTraceReturn != nil || sequence.RowLineageReturn != nil || sequence.PopulationMappingReturn != nil { + return fmt.Errorf("%w: row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route (compiler/row_lineage.go:655, cell_trace.go:54, population_mapping.go:38)", + ErrUnsupportedPostgresPlan) + } + if plan.PreviewSourceWindowByRootID || sequence.PreviewSourceWindowByRootID || sequence.PreviewTerminalPivotWindow { + return fmt.Errorf("%w: preview source windowing and the terminal Pivot window require Pivot terminals and are not registered for CODED_GROUP", ErrUnsupportedPostgresPlan) + } + if sequence.OutputAuthResourcePathBindKey != "" { + return fmt.Errorf("%w: the composite Combine prefix authorization path is not registered for CODED_GROUP", ErrUnsupportedPostgresPlan) + } + if sequence.PreviewLimitBindKey != "" { + if value, ok := plan.BindVars[sequence.PreviewLimitBindKey].(int); !ok || value <= 0 { + return fmt.Errorf("%w: coded-group preview limit bind %q must be a positive int", + ErrUnsupportedPostgresPlan, sequence.PreviewLimitBindKey) + } + } + if err := validateCodedGroupPayload(stage, plan.BindVars); err != nil { + return err + } + return validateCodedGroupOperations(plan) +} + +func validateCodedGroupPayload(stage ir.PhysicalConstructionStage, bindVars map[string]any) error { + coded := stage.CodedGroup + for _, key := range []string{coded.RootCollectionBindKey, coded.ConstructionIDBindKey, coded.OccurrenceIDBindKey, coded.CodingPathBindKey} { + if value, ok := bindVars[key].(string); !ok || strings.TrimSpace(value) == "" { + return fmt.Errorf("%w: coded-group bind %q must be a non-empty string", ErrUnsupportedPostgresPlan, key) + } + } + if collection, _ := bindVars[coded.RootCollectionBindKey].(string); collection != coded.ResourceType || strings.TrimSpace(coded.ResourceType) == "" { + return fmt.Errorf("%w: root collection bind does not match the exact root resource type", ErrUnsupportedPostgresPlan) + } + if occurrence, _ := bindVars[coded.OccurrenceIDBindKey].(string); occurrence != "base" { + return fmt.Errorf("%w: the registered inline slice admits only the base root occurrence (ir/physical_stage_validation.go:2274)", + ErrUnsupportedPostgresPlan) + } + if codingPath, _ := bindVars[coded.CodingPathBindKey].(string); codingPath != coded.CodingPath { + return fmt.Errorf("%w: the Coding path bind differs from the compiled path", ErrUnsupportedPostgresPlan) + } + if !coded.SourceRowsUnique || coded.SourceIdentityColumn != "_key" { + return fmt.Errorf("%w: CODED_GROUP requires the compiler proof of one source row per root _key (ir/physical_stage_validation.go:2270-2274)", + ErrUnsupportedPostgresPlan) + } + if err := validateCodedGroupRowValues(coded); err != nil { + return err + } + switch coded.MissingKeyPolicy { + case ir.PhysicalStageGroupMissingKeyGroup, ir.PhysicalStageGroupMissingKeyExclude, ir.PhysicalStageGroupMissingKeyError: + default: + return fmt.Errorf("%w: missing-key policy %q is unsupported", ErrUnsupportedPostgresPlan, string(coded.MissingKeyPolicy)) + } + if len(coded.PathSegments) == 0 || !coded.PathSegments[len(coded.PathSegments)-1].Repeated { + return fmt.Errorf("%w: the Coding path must end in a repeated Coding array", ErrUnsupportedPostgresPlan) + } + var canonical strings.Builder + for index, segment := range coded.PathSegments { + if !payloadPathSegmentPattern.MatchString(segment.Name) { + return fmt.Errorf("%w: coded-group path segment %q is not renderer-safe", ErrUnsupportedPostgresPlan, segment.Name) + } + if index > 0 { + canonical.WriteByte('.') + } + canonical.WriteString(segment.Name) + if segment.Repeated { + canonical.WriteString("[]") + } + } + if canonical.String() != coded.CodingPath { + return fmt.Errorf("%w: the Coding path must be the canonical join of its segments", ErrUnsupportedPostgresPlan) + } + for _, variable := range []string{coded.SystemVariable, coded.VersionVariable, coded.CodeVariable, coded.CountVariable, coded.IdentityVariable, coded.GroupRowsVariable} { + if variable == "" { + return fmt.Errorf("%w: coded-group plan variables must be compiled and non-empty", ErrUnsupportedPostgresPlan) + } + } + return nil +} + +func codedGroupSanitizeRowValueProjections(projections []ir.PhysicalProjection) []ir.PhysicalProjection { + out := make([]ir.PhysicalProjection, len(projections)) + for i, projection := range projections { + projection.Hidden = false + out[i] = projection + } + return out +} + +func validateCodedGroupRowValues(coded *ir.PhysicalStageCodedGroup) error { + if len(coded.RowValues) == 0 && len(coded.RowValueProjections) == 0 { + return nil + } + inputs := make(map[string]bool, len(coded.RowValues)) + for _, rowValue := range coded.RowValues { + if rowValue.InputColumn == "" || rowValue.Output == "" || rowValue.Variable == "" { + return fmt.Errorf("%w: coded-group row value columns and variables must be compiled and non-empty", + ErrUnsupportedPostgresPlan) + } + if inputs[rowValue.InputColumn] { + return fmt.Errorf("%w: coded-group row value input column %q is declared twice (construction_coded_group.go:92-94 parity)", + ErrUnsupportedPostgresPlan, rowValue.InputColumn) + } + inputs[rowValue.InputColumn] = true + switch rowValue.Policy { + case "ALL", "ONE": + default: + return fmt.Errorf("%w: row value %q has unsupported policy %q (row_values.go:53 parity)", + ErrUnsupportedPostgresPlan, rowValue.Output, rowValue.Policy) + } + switch rowValue.InputKind { + case "STRING", "INTEGER", "DECIMAL", "BOOLEAN": + default: + return fmt.Errorf("%w: row value %q has unsupported input kind %q", + ErrUnsupportedPostgresPlan, rowValue.Output, rowValue.InputKind) + } + if rowValue.InputMany { + return fmt.Errorf("%w: coded-group many-cardinality (local-array-expansion) row values are a wave-2 pg-sql-13 gap (the shared pool emitter groupManyRowValuePoolFor renders from g_rows; a CODED_GROUP pool over cg_tuples is wave-2) and are refused, never partially rendered", + ErrUnsupportedPostgresPlan) + } + } + provided := make(map[string]int, len(coded.RowValueProjections)) + classified, err := classifyExtractionProjections(codedGroupSanitizeRowValueProjections(coded.RowValueProjections), func(string) (string, bool) { + return coded.ResourceType, true + }) + if err != nil { + return err + } + for _, projection := range classified { + provided[projection.projection.Name]++ + switch projection.kind { + case extractionDirectValue, extractionValueExpression, extractionSelector: + value := projection.projection.Value + if projection.kind == extractionValueExpression { + value = *projection.projection.Expression.Value + } + if projection.kind != extractionSelector && len(value.Path) == 0 { + return fmt.Errorf("%w: coded-group row value projection %q is not a registered scalar projection (whole-document read)", + ErrUnsupportedPostgresPlan, projection.projection.Name) + } + default: + return fmt.Errorf("%w: coded-group row value projection %q is not a direct payload extraction (only the source-clone scalar shapes — direct value, value expression, selector — are registered, group-arm parity)", + ErrUnsupportedPostgresPlan, projection.projection.Name) + } + } + for name := range inputs { + if provided[name] != 1 { + return fmt.Errorf("%w: coded-group row value input %q needs exactly one source projection (found %d)", + ErrUnsupportedPostgresPlan, name, provided[name]) + } + } + for name := range provided { + if !inputs[name] { + return fmt.Errorf("%w: coded-group row value projection %q addresses no row value input column", + ErrUnsupportedPostgresPlan, name) + } + } + return nil +} + +func validateCodedGroupOperations(plan ir.PhysicalPlan) error { + if len(plan.Operations) == 0 { + return fmt.Errorf("%w: plan has no operations", ErrUnsupportedPostgresPlan) + } + first := plan.Operations[0] + if first.Kind != ir.PhysicalRootScanOp || first.RootScan == nil { + return fmt.Errorf("%w: the coded-group source requires exactly one leading ROOT_SCAN", ErrUnsupportedPostgresPlan) + } + scan := first.RootScan + if scan.Population != nil || scan.CohortSource != nil || scan.PageCandidateSeed != nil { + return fmt.Errorf("%w: population, cohort, and candidate-seed roots are bound only by RenderPopulationRootScan (population.go); the plain spine refuses them", + ErrUnsupportedPostgresPlan) + } + sawSort, sawLimit, sawReturn := false, false, false + for index, operation := range plan.Operations { + last := index == len(plan.Operations)-1 + switch operation.Kind { + case ir.PhysicalRootScanOp: + if index != 0 { + return fmt.Errorf("%w: exactly one leading ROOT_SCAN is required", ErrUnsupportedPostgresPlan) + } + case ir.PhysicalFilterOp, ir.PhysicalDerivedLetOp: + if sawSort || sawLimit || sawReturn { + return fmt.Errorf("%w: source operations must precede the execution window and the terminal RETURN", + ErrUnsupportedPostgresPlan) + } + if operation.Kind == ir.PhysicalFilterOp && operation.Filter == nil { + return fmt.Errorf("%w: operation %d: filter payload is missing", ErrUnsupportedPostgresPlan, index) + } + if operation.Kind == ir.PhysicalDerivedLetOp && operation.DerivedLet == nil { + return fmt.Errorf("%w: operation %d: derived-let payload is missing", ErrUnsupportedPostgresPlan, index) + } + case ir.PhysicalSetOp: + return fmt.Errorf("%w: coded-group source-region SET is a pg-sql-16 capability gap (no PostgreSQL contract for lowering-emitted child-set subplans; probe-attested L-19.4) and is refused, never partially rendered", + ErrUnsupportedPostgresPlan) + case ir.PhysicalSortOp: + if sawSort || sawLimit || sawReturn { + return fmt.Errorf("%w: at most one SORT, before LIMIT", ErrUnsupportedPostgresPlan) + } + if operation.Sort == nil { + return fmt.Errorf("%w: operation %d: sort payload is missing", ErrUnsupportedPostgresPlan, index) + } + sawSort = true + case ir.PhysicalLimitOp: + if sawLimit || sawReturn { + return fmt.Errorf("%w: at most one LIMIT", ErrUnsupportedPostgresPlan) + } + if operation.Limit == nil { + return fmt.Errorf("%w: operation %d: limit payload is missing", ErrUnsupportedPostgresPlan, index) + } + sawLimit = true + case ir.PhysicalReturnOp: + if !last || operation.Return == nil || len(operation.Return.Projections) == 0 { + return fmt.Errorf("%w: the coded-group source requires a terminal RETURN with its projection list (lower/recipe_construction_coded_group.go:180-184)", + ErrUnsupportedPostgresPlan) + } + sawReturn = true + default: + return fmt.Errorf("%w: operation %q is not part of the registered coded-group source surface", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + if !sawReturn { + return fmt.Errorf("%w: the coded-group source requires a terminal RETURN", ErrUnsupportedPostgresPlan) + } + return nil +} + +func (r *planRenderer) renderCodedGroupStatement(plan ir.PhysicalPlan) (string, error) { + coded := plan.StageSequence.Stages[0].CodedGroup + + if err := r.renderRootScan(plan, *plan.Operations[0].RootScan); err != nil { + return "", err + } + for _, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalFilterOp: + canonical, err := r.acceptCanonicalScopeFilter(operation.Filter) + if err != nil { + return "", err + } + if canonical { + continue + } + clause, err := r.renderFilter(plan, *operation.Filter) + if err != nil { + return "", err + } + r.where = append(r.where, clause) + case ir.PhysicalDerivedLetOp: + canonical, err := r.acceptCanonicalAuthLet(operation.DerivedLet) + if err != nil { + return "", err + } + if !canonical { + return "", fmt.Errorf("%w: operation %q is not part of the registered coded-group source surface", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + } + for _, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalSortOp: + if err := r.renderWindowSort(plan, *operation.Sort); err != nil { + return "", err + } + case ir.PhysicalLimitOp: + if err := r.renderWindowLimit(plan, *operation.Limit); err != nil { + return "", err + } + } + } + + type codedGroupRowValue struct { + rowValue ir.PhysicalStageRowValue + column *groupColumn + } + var codedRowValues []codedGroupRowValue + var rvSuffix, rvCarryS, rvCarryC string + if len(coded.RowValues) > 0 { + classified, err := classifyExtractionProjections(codedGroupSanitizeRowValueProjections(coded.RowValueProjections), func(variable string) (string, bool) { + scope, ok := r.rows[variable] + if !ok { + return "", false + } + if scope.resourceType != "" { + return scope.resourceType, true + } + return coded.ResourceType, true + }) + if err != nil { + return "", err + } + byName := map[string]classifiedExtractionProjection{} + for _, projection := range classified { + byName[projection.projection.Name] = projection + } + for i := range coded.RowValues { + rowValue := coded.RowValues[i] + projection, ok := byName[rowValue.InputColumn] + if !ok { + return "", fmt.Errorf("%w: coded-group row value input %q has no source projection", + ErrUnsupportedPostgresPlan, rowValue.InputColumn) + } + var resolved string + switch projection.kind { + case extractionSelector: + extracted, err := r.scalarExtractValueExpression(r.rootAlias, projection.path, projection.expression.NullBehavior) + if err != nil { + return "", fmt.Errorf("coded-group row value %q: %w", rowValue.Output, err) + } + resolved = extracted + case extractionDirectValue, extractionValueExpression: + value := projection.projection.Value + if projection.kind == extractionValueExpression { + value = *projection.projection.Expression.Value + } + if len(value.Path) == 0 { + return "", fmt.Errorf("%w: coded-group row value projection %q is not a registered scalar projection (whole-document read)", + ErrUnsupportedPostgresPlan, rowValue.InputColumn) + } + renderedValue, err := r.resolveValue(value) + if err != nil { + return "", fmt.Errorf("coded-group row value %q: %w", rowValue.Output, err) + } + resolved = renderedValue + default: + return "", fmt.Errorf("%w: coded-group row value projection %q is not a registered scalar arm", + ErrUnsupportedPostgresPlan, rowValue.InputColumn) + } + if isPayloadExpression(resolved) { + resolved = "(" + resolved + ")" + } + alias := "cg_rv" + strconv.Itoa(i) + codedRowValues = append(codedRowValues, codedGroupRowValue{ + rowValue: rowValue, + column: &groupColumn{name: rowValue.InputColumn, alias: alias, expr: resolved, ref: `"` + alias + `"`, jsonb: true}, + }) + rvSuffix += ", " + resolved + ` AS ` + alias + rvCarryS += ", s." + alias + rvCarryC += ", c." + alias + } + } + + resource := r.storage.Resource + source := "SELECT " + r.rootAlias + "." + resource.MustColumn("logical_id").SQLText() + + ` AS cg_root_key, ` + r.rootAlias + "." + resource.MustColumn("payload").SQLText() + ` AS cg_payload` + rvSuffix + + " FROM " + resource.Identifier().SQLText() + " AS " + r.rootAlias + for _, join := range r.joins { + source += " " + join + } + for _, lateral := range r.laterals { + source += " " + lateral + } + if len(r.where) > 0 { + source += " WHERE " + strings.Join(r.where, " AND ") + } + source += " ORDER BY " + strings.Join(r.windowOrder(), ", ") + r.window.limitClause + + lateralSQL, itemExpression, err := r.codedGroupExpansion(coded.PathSegments) + if err != nil { + return "", err + } + + systemKey := r.textParameter("system", "text:system") + versionKey := r.textParameter("version", "text:version") + codeKey := r.textParameter("code", "text:code") + trimChars := r.textParameter(codedGroupTrimCharacters, "text:"+codedGroupTrimCharacters) + dollar := func(index int) string { return "$" + strconv.Itoa(index) } + + itemSystemText := itemExpression + " ->> " + dollar(systemKey) + "::text" + itemCodeText := itemExpression + " ->> " + dollar(codeKey) + "::text" + + valid := "COALESCE(jsonb_typeof(" + itemExpression + ") = 'object'" + + " AND jsonb_typeof(" + itemExpression + " -> " + dollar(systemKey) + "::text) = 'string'" + + " AND btrim(" + itemSystemText + ", " + dollar(trimChars) + "::text) <> ''" + + " AND jsonb_typeof(" + itemExpression + " -> " + dollar(codeKey) + "::text) = 'string'" + + " AND btrim(" + itemCodeText + ", " + dollar(trimChars) + "::text) <> '', false)" + + candidates := "SELECT s.cg_root_key" + rvCarryS + ", " + itemExpression + " AS cg_item, " + valid + " AS cg_valid" + + " FROM cg_src AS s" + lateralSQL + + tupleSystemText := "c.cg_item ->> " + dollar(systemKey) + "::text" + tupleCodeText := "c.cg_item ->> " + dollar(codeKey) + "::text" + tupleVersionText := "c.cg_item ->> " + dollar(versionKey) + "::text" + + var tuplesConjuncts []string + seenTuplesConjunct := map[string]bool{} + for i := range codedRowValues { + _, witness, err := r.groupColumnRead(codedRowValues[i].column, codedRowValues[i].rowValue.InputKind, + groupValueTypeTag, dollar) + if err != nil { + return "", fmt.Errorf("coded-group row value %q: %w", codedRowValues[i].rowValue.Output, err) + } + if witness != "" && !seenTuplesConjunct[witness] { + seenTuplesConjunct[witness] = true + tuplesConjuncts = append(tuplesConjuncts, witness) + } + } + switch coded.MissingKeyPolicy { + case ir.PhysicalStageGroupMissingKeyExclude: + tuplesConjuncts = append(tuplesConjuncts, "c.cg_valid") + case ir.PhysicalStageGroupMissingKeyError: + tuplesConjuncts = append(tuplesConjuncts, + "(CASE WHEN c.cg_valid THEN 1 ELSE "+guardCast(codedGroupMissingKeyTag)+" END) > 0") + } + tuplesWhere := "" + if len(tuplesConjuncts) > 0 { + tuplesWhere = " WHERE " + strings.Join(tuplesConjuncts, " AND ") + } + tuples := "SELECT DISTINCT c.cg_root_key" + rvCarryC + "," + + " CASE WHEN c.cg_valid THEN " + tupleSystemText + " END AS cg_system," + + " CASE WHEN c.cg_valid AND jsonb_typeof(c.cg_item -> " + dollar(versionKey) + "::text) = 'string'" + + " AND btrim(" + tupleVersionText + ", " + dollar(trimChars) + "::text) <> '' THEN " + tupleVersionText + " END AS cg_version," + + " CASE WHEN c.cg_valid THEN " + tupleCodeText + " END AS cg_code" + + " FROM cg_candidates AS c" + tuplesWhere + + groupedSelects := "SELECT cg_system, cg_version, cg_code, COUNT(*) AS cg_count" + for i := range codedRowValues { + expression, err := groupRowValueExpressionFor(codedRowValues[i].rowValue.Policy, + codedRowValues[i].rowValue.InputKind, false, codedRowValues[i].column, dollar) + if err != nil { + return "", fmt.Errorf("coded-group row value %q: %w", codedRowValues[i].rowValue.Output, err) + } + groupedSelects += ", " + expression + " AS cg_g" + strconv.Itoa(i) + } + grouped := groupedSelects + " FROM cg_tuples GROUP BY 1, 2, 3" + + constructionID := r.textParameter(bindStringVar(plan.BindVars, coded.ConstructionIDBindKey), "") + occurrenceID := r.textParameter(bindStringVar(plan.BindVars, coded.OccurrenceIDBindKey), "") + codingPath := r.textParameter(bindStringVar(plan.BindVars, coded.CodingPathBindKey), "") + identity := "'[' || " + + quoteSQLLiteral(`"construction"`) + " || ',' || to_jsonb(" + dollar(constructionID) + "::text)" + + " || ',' || " + quoteSQLLiteral(`"operation"`) + " || ',' || " + quoteSQLLiteral(`"CODED_GROUP"`) + + " || ',' || " + quoteSQLLiteral(`"occurrence"`) + " || ',' || to_jsonb(" + dollar(occurrenceID) + "::text)" + + " || ',' || " + quoteSQLLiteral(`"path"`) + " || ',' || to_jsonb(" + dollar(codingPath) + "::text)" + + " || ',' || " + quoteSQLLiteral(`"system"`) + " || ',' || COALESCE(to_jsonb(g.cg_system)::text, 'null')" + + " || ',' || " + quoteSQLLiteral(`"version"`) + " || ',' || COALESCE(to_jsonb(g.cg_version)::text, 'null')" + + " || ',' || " + quoteSQLLiteral(`"code"`) + " || ',' || COALESCE(to_jsonb(g.cg_code)::text, 'null')" + + " || ']'" + + expressions := map[string]string{ + coded.SystemOutputColumn: "g.cg_system", + coded.VersionOutputColumn: "g.cg_version", + coded.CodeOutputColumn: "g.cg_code", + coded.CountOutputColumn: "g.cg_count", + codedGroupRowIDColumn: identity, + } + for i := range codedRowValues { + expressions[codedRowValues[i].rowValue.Output] = "g.cg_g" + strconv.Itoa(i) + } + selects := make([]string, 0, len(plan.StageSequence.FinalColumns)) + names := make([]string, 0, len(plan.StageSequence.FinalColumns)) + for _, column := range plan.StageSequence.FinalColumns { + expression, ok := expressions[column.Name] + if !ok { + return "", fmt.Errorf("%w: coded-group final column %q is not a registered tuple member", + ErrUnsupportedPostgresPlan, column.Name) + } + for _, seen := range names { + if seen == column.Name { + return "", fmt.Errorf("%w: coded-group final column %q is declared twice", + ErrUnsupportedPostgresPlan, column.Name) + } + } + selects = append(selects, expression+` AS "c`+strconv.Itoa(len(selects))+`"`) + names = append(names, column.Name) + } + if len(names) != 5+len(codedRowValues) { + return "", fmt.Errorf("%w: the registered slice emits the three tuple keys, the root count, the hidden stable identity, and one column per registered row value", + ErrUnsupportedPostgresPlan) + } + r.outputColumns = names + + final := "SELECT " + strings.Join(selects, ", ") + " FROM cg_grouped AS g" + + " ORDER BY " + identity + ` COLLATE "C" ASC` + if plan.StageSequence.PreviewLimitBindKey != "" { + limit := plan.BindVars[plan.StageSequence.PreviewLimitBindKey].(int) + index := r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(limit)}, "") + final += " LIMIT $" + strconv.Itoa(index) + } + + sql := "WITH cg_src AS (" + source + + "), cg_candidates AS (" + candidates + + "), cg_tuples AS (" + tuples + + "), cg_grouped AS (" + grouped + ") " + final + return sql, nil +} + +func (r *planRenderer) codedGroupExpansion(segments []ir.PhysicalCodedGroupPathSegment) (string, string, error) { + current := "s.cg_payload" + var expansions []string + var lastAlias string + var pending []string + sequence := 0 + for _, segment := range segments { + pending = append(pending, segment.Name) + if !segment.Repeated { + continue + } + extraction, err := r.documentPathFromExpression(current, pending) + if err != nil { + return "", "", err + } + current = extraction + pending = nil + sequence++ + alias := "cgx" + strconv.Itoa(sequence) + guard := "CASE WHEN jsonb_typeof(" + current + ") = 'array' THEN " + current + " ELSE '[]'::jsonb END" + expansions = append(expansions, "jsonb_array_elements("+guard+") AS "+alias+"(elem)") + lastAlias = alias + current = alias + ".elem" + } + if len(pending) != 0 || sequence == 0 { + return "", "", fmt.Errorf("%w: the Coding path must end in a repeated Coding array", ErrUnsupportedPostgresPlan) + } + if sequence == 1 { + return " LEFT JOIN LATERAL " + expansions[0] + " ON true", "cgx1.elem", nil + } + chain := " LEFT JOIN LATERAL (SELECT " + lastAlias + ".elem FROM " + + strings.Join(expansions, " CROSS JOIN LATERAL ") + ") AS cgexp(elem) ON true" + return chain, "cgexp.elem", nil +} + +func bindStringVar(bindVars map[string]any, key string) string { + value, _ := bindVars[key].(string) + return value +} diff --git a/internal/dataframe/compiler/render/postgres/coded_group_test.go b/internal/dataframe/compiler/render/postgres/coded_group_test.go new file mode 100644 index 000000000..90efdf2b2 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/coded_group_test.go @@ -0,0 +1,1616 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + coded19CodingPath = "component[].code.coding[]" + coded19StepID = "group_codes" +) + +func coded19Output(policy recipe.ConstructionGroupMissingKeyPolicy, project, generation string) (recipe.Output, recipe.RuntimeBindings) { + columns := []recipe.StageColumn{ + {ID: "system_id", Name: "code_system", Type: "string", Nullable: true}, + {ID: "version_id", Name: "code_version", Type: "string", Nullable: true}, + {ID: "code_id", Name: "code", Type: "string", Nullable: true}, + {ID: "count_id", Name: "source_records", Type: "integer"}, + } + output := recipe.Output{ + Name: "coded_group_rows", RootResourceType: "Observation", RowGrain: "resource", + Fields: []recipe.Field{{Name: "resource_id", ColumnID: "resource_id", Expr: recipe.Expression{Select: "root.id"}}}, + Construction: &recipe.Construction{ + Version: 1, SourceColumns: []recipe.StageColumn{{ID: "resource_id", Name: "resource_id"}}, + Steps: []recipe.ConstructionStep{{ + ID: coded19StepID, Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionCodedGroupOp, CodedGroup: &recipe.ConstructionCodedGroup{ + ConstructionID: coded19StepID, + Source: recipe.ConstructionCodedGroupSource{ + OccurrenceID: "base", ResourceType: "Observation", CodingPath: coded19CodingPath, + FHIRType: "Coding", Cardinality: "MANY", Shape: "ARRAY", Route: []recipe.ConstructionRelatedRouteStep{}, + }, + MissingKeyPolicy: policy, + SystemOutputColumnID: "system_id", VersionOutputColumnID: "version_id", + CodeOutputColumnID: "code_id", DistinctSourceCountOutputColumnID: "count_id", + }}, Outputs: columns, + }}, + }, + } + return output, recipe.RuntimeBindings{Project: project, DatasetGeneration: generation, AuthResourcePaths: []string{"FHIR/Observation"}} +} + +func coded19LoweredPlan(t *testing.T, policy recipe.ConstructionGroupMissingKeyPolicy, project, generation string) ir.PhysicalPlan { + t.Helper() + output, bindings := coded19Output(policy, project, generation) + bundle := recipe.Bundle{RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: output.Name, TranslationVersion: "test", Outputs: []recipe.Output{output}} + plan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("lower.CompileResolvedRecipePlan: %v", err) + } + if compiled.Outputs[0].Plan.StageSequence == nil || len(compiled.Outputs[0].Plan.StageSequence.Stages) != 1 || + compiled.Outputs[0].Plan.StageSequence.Stages[0].Kind != ir.PhysicalStageCodedGroupOp { + t.Fatalf("lowered plan is not the inline single-stage CODED_GROUP shape: %#v", compiled.Outputs[0].Plan.StageSequence) + } + return ir.ClonePhysicalPlan(compiled.Outputs[0].Plan) +} + +func coded19Identity(t *testing.T, system, version, code *string) string { + t.Helper() + member := func(value *string) string { + if value == nil { + return "null" + } + raw, err := json.Marshal(*value) + if err != nil { + t.Fatalf("marshal identity member: %v", err) + } + return string(raw) + } + parts := []string{ + `"construction"`, `"group_codes"`, `"operation"`, `"CODED_GROUP"`, + `"occurrence"`, `"base"`, `"path"`, `"component[].code.coding[]"`, + `"system"`, member(system), `"version"`, member(version), `"code"`, member(code), + } + return "[" + strings.Join(parts, ",") + "]" +} + +type coded19Row struct { + system, version, code *string + count string + identity string +} + +func coded19Rows(t *testing.T, rows [][]string) []coded19Row { + t.Helper() + optional := func(cell string) *string { + if cell == "" { + return nil + } + value := cell + return &value + } + out := make([]coded19Row, 0, len(rows)) + for _, row := range rows { + if len(row) != 5 { + t.Fatalf("coded-group row = %#v, want 5 cells", row) + } + out = append(out, coded19Row{ + system: optional(row[0]), version: optional(row[1]), code: optional(row[2]), + count: row[3], identity: row[4], + }) + } + return out +} + +func coded19Text(value string) *string { return &value } + +func TestCodedGroupRenderedSliceContract(t *testing.T) { + plan := coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, "proj-a", "gen-1") + rendered := renderOK(t, plan) + sql := rendered.SQL + + for _, want := range []string{ + `WITH cg_src AS (`, + `LEFT JOIN LATERAL (SELECT cgx2.elem FROM jsonb_array_elements(CASE WHEN jsonb_typeof(`, + `jsonb_typeof(s.cg_payload #> $`, + `AS cgx1(elem) CROSS JOIN LATERAL jsonb_array_elements(CASE WHEN jsonb_typeof(`, + `) AS cgx2(elem)) AS cgexp(elem) ON true`, + `cgexp.elem ->> $`, + `SELECT DISTINCT c.cg_root_key,`, + `COUNT(*) AS cg_count FROM cg_tuples GROUP BY 1, 2, 3`, + `COLLATE "C" ASC`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("statement is missing %q:\n%s", want, sql) + } + } + if got := strings.Count(sql, "LEFT JOIN LATERAL"); got != 1 { + t.Fatalf("wrapping LEFT JOIN LATERAL count = %d, want exactly 1:\n%s", got, sql) + } + if strings.Contains(sql, " WHERE c.cg_valid") { + t.Fatalf("GROUP policy must not filter candidates:\n%s", sql) + } + if strings.Contains(sql, "@") { + t.Fatalf("statement carries AQL bind syntax:\n%s", sql) + } + for _, leaked := range []string{"proj-a", "gen-1", "FHIR/Observation", "Observation", "component", "coding", "group_codes", coded19CodingPath, "CONSTRUCTION_CODED_GROUP_MISSING_KEY"} { + if strings.Contains(sql, leaked) { + t.Fatalf("value %q leaked into statement text:\n%s", leaked, sql) + } + } + if !strings.Contains(sql, `#> $`) { + t.Fatalf("payload paths must bind through #> text[] parameters:\n%s", sql) + } + if got := strings.Count(sql, "btrim("); got != 3 { + t.Fatalf("btrim gates = %d, want 3 (system+code validity, version key):\n%s", got, sql) + } + if got := strings.Count(sql, ` COLLATE "C"`); got != 1 { + t.Fatalf(`identity COLLATE "C" terms = %d, want 1:\n%s`, got, sql) + } + + want := []string{"code_system", "code_version", "code", "source_records", "__loom_row_id"} + if !reflect.DeepEqual(rendered.OutputColumns, want) { + t.Fatalf("output columns = %#v, want %#v", rendered.OutputColumns, want) + } + + second, err := RenderPhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if err != nil || second.SQL != rendered.SQL || !reflect.DeepEqual(second.Args, rendered.Args) { + t.Fatalf("coded-group render is not deterministic: %v", err) + } +} + +func TestCodedGroupPoliciesRender(t *testing.T) { + group := renderOK(t, coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, "proj-a", "gen-1")) + if strings.Contains(group.SQL, " WHERE c.cg_valid") || strings.Contains(group.SQL, coded19MissingKeyTagSQLProbe) { + t.Fatalf("GROUP policy must neither filter nor guard candidates:\n%s", group.SQL) + } + + exclude := renderOK(t, coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyExclude, "proj-a", "gen-1")) + if !strings.Contains(exclude.SQL, " WHERE c.cg_valid") { + t.Fatalf("EXCLUDE policy must filter candidates:\n%s", exclude.SQL) + } + if strings.Contains(exclude.SQL, guardCast(codedGroupMissingKeyTag)) { + t.Fatalf("EXCLUDE policy must not carry the ERROR guard:\n%s", exclude.SQL) + } + + errorPlan := coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyError, "proj-a", "gen-1") + errorRendered := renderOK(t, errorPlan) + for _, want := range []string{ + " WHERE (CASE WHEN c.cg_valid THEN 1 ELSE CAST('" + codedGroupMissingKeyTag + "'", + guardBarrierGUC, + " END) > 0", + } { + if !strings.Contains(errorRendered.SQL, want) { + t.Fatalf("ERROR guard is missing %q:\n%s", want, errorRendered.SQL) + } + } +} + +const coded19MissingKeyTagSQLProbe = "CASE WHEN c.cg_valid THEN 1" + +func TestCodedGroupIdentityOrderAndBound(t *testing.T) { + plan := coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, "proj-a", "gen-1") + rendered := renderOK(t, plan) + orderIndex := strings.LastIndex(rendered.SQL, " ORDER BY ") + if orderIndex < 0 { + t.Fatalf("final statement has no ORDER BY:\n%s", rendered.SQL) + } + final := rendered.SQL[orderIndex:] + if !strings.Contains(final, `'[' || '"construction"'`) || !strings.Contains(final, `COALESCE(to_jsonb(g.cg_system)::text, 'null')`) { + t.Fatalf("final ORDER BY must be the stable identity concatenation:\n%s", final) + } + if strings.Contains(final, "cg_system ASC") || strings.Contains(final, "cg_version ASC") { + t.Fatalf("the unobservable AQL stage-internal sort must not be re-rendered:\n%s", final) + } + + limited := ir.ClonePhysicalPlan(plan) + limited.StageSequence.PreviewLimitBindKey = "limit" + limited.BindVars["limit"] = 25 + limitedRendered := renderOK(t, limited) + if m, _ := regexp.MatchString(` COLLATE "C" ASC LIMIT \$[0-9]+$`, limitedRendered.SQL); !m { + t.Fatalf("the bound preview LIMIT must follow the identity COLLATE \"C\" order:\n%s", limitedRendered.SQL) + } + if !strings.HasSuffix(rendered.SQL, ` COLLATE "C" ASC`) { + t.Fatalf("the unwindowed statement ends in the identity order:\n%s", rendered.SQL) + } +} + +func TestCodedGroupWindowedSourceRegion(t *testing.T) { + plan := coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, "proj-a", "gen-1") + sorted := ir.ClonePhysicalPlan(plan) + last := len(sorted.Operations) - 1 + sorted.Operations = append(sorted.Operations[:last], + ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{Keys: []ir.PhysicalValue{{Variable: "root", Path: []string{"logical_id"}}}}}, + ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "srclimit"}}, + sorted.Operations[last]) + sorted.BindVars["srclimit"] = 3 + rendered := renderOK(t, sorted) + srcEnd := strings.Index(rendered.SQL, "), cg_candidates AS (") + if srcEnd < 0 { + t.Fatalf("cg_src boundary missing:\n%s", rendered.SQL) + } + source := rendered.SQL[:srcEnd] + if !strings.Contains(source, `r0."logical_id" ASC`) || !strings.Contains(source, `"resource_seq" ASC`) || !strings.Contains(source, "LIMIT $") { + t.Fatalf("the W1 window must be rendered inside cg_src:\n%s", source) + } +} + +func TestCodedGroupSingleSegmentKeepsFlatLateral(t *testing.T) { + plan := coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, "proj-a", "gen-1") + coded := plan.StageSequence.Stages[0].CodedGroup + coded.PathSegments = []ir.PhysicalCodedGroupPathSegment{{Name: "coding", Repeated: true}} + coded.CodingPath = "coding[]" + plan.BindVars[coded.CodingPathBindKey] = coded.CodingPath + sql := renderOK(t, plan).SQL + for _, want := range []string{ + `LEFT JOIN LATERAL jsonb_array_elements(CASE WHEN jsonb_typeof(`, + `AS cgx1(elem) ON true`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("single-segment statement is missing %q:\n%s", want, sql) + } + } + if strings.Contains(sql, "CROSS JOIN LATERAL") || strings.Contains(sql, "cgexp") { + t.Fatalf("single-segment statement must keep the flat form:\n%s", sql) + } +} + +func TestCodedGroupRefusalRegister(t *testing.T) { + base := coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, "proj-a", "gen-1") + + if _, err := RenderPhysicalPlan(ir.ClonePhysicalPlan(base), mustBinding(t)); err != nil { + t.Fatalf("alive control (base plan) must render: %v", err) + } + + cases := []struct { + name string + mutate func(t *testing.T, plan *ir.PhysicalPlan) + pin string + door bool + message string + }{ + { + name: "second_stage_pg_sql_16_gap", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + second := plan.StageSequence.Stages[0] + second.ID = "second" + second.InputStageID = coded19StepID + plan.StageSequence.Stages = append(plan.StageSequence.Stages, second) + }, + pin: "exactly one CODED_GROUP construction stage; multi-stage composition is a pg-sql-16 capability gap", + message: "", + }, + { + name: "stage_kind_not_coded_group", + door: true, + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Kind = ir.PhysicalStageGroupOp + }, + pin: `operation "GROUP" is not part of the registered coded-group slice`, + }, + { + name: "coded_group_payload_nil", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].CodedGroup = nil + }, + pin: "is not part of the registered coded-group slice", + }, + { + name: "foreign_stage_payload_present", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group = &ir.PhysicalStageGroup{} + }, + pin: "CODED_GROUP requires only a coded-group payload", + }, + { + name: "document_requery_shape_production_unreachable", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].InputStageID = "not_the_source_projection" + }, + pin: "the DOCUMENT re-query shape is production-unreachable (recipe/construction.go:665-668)", + }, + { + name: "stage_id_differs_from_final", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].ID = "other" + }, + pin: "the DOCUMENT re-query shape is production-unreachable (recipe/construction.go:665-668)", + }, + { + name: "row_value_input_many_wave2_gap", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + coded19AttachIRRowValuesPolicy(t, plan, "ONE", true) + }, + pin: coded19ManyRowValuesRefusalPin, + }, + { + name: "row_value_input_without_projection", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + coded := plan.StageSequence.Stages[0].CodedGroup + coded.RowValues = []ir.PhysicalStageRowValue{{ + InputColumn: "resource_id", InputKind: "STRING", Output: "row_value_1", Policy: "ONE", Variable: "cg19_rv_var", + }} + }, + pin: "needs exactly one source projection", + }, + { + name: "row_value_projections_without_row_values", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + coded := plan.StageSequence.Stages[0].CodedGroup + coded.RowValueProjections = []ir.PhysicalProjection{{Name: "resource_id", Hidden: true, Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}} + }, + pin: "addresses no row value input column", + }, + { + name: "source_region_set_pg_sql_16_gap", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + last := len(plan.Operations) - 1 + plan.Operations = append(plan.Operations[:last], + ir.PhysicalOperation{Kind: ir.PhysicalSetOp}, + plan.Operations[last]) + }, + pin: "coded-group source-region SET is a pg-sql-16 capability gap (no PostgreSQL contract for lowering-emitted child-set subplans; probe-attested L-19.4) and is refused, never partially rendered", + }, + { + name: "occurrence_must_be_base", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.BindVars[plan.StageSequence.Stages[0].CodedGroup.OccurrenceIDBindKey] = "v2" + }, + pin: "the registered inline slice admits only the base root occurrence (ir/physical_stage_validation.go:2274)", + }, + { + name: "coding_path_bind_mismatch", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.BindVars[plan.StageSequence.Stages[0].CodedGroup.CodingPathBindKey] = "component.code.coding" + }, + pin: "the Coding path bind differs from the compiled path", + }, + { + name: "collection_bind_mismatch", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.BindVars[plan.StageSequence.Stages[0].CodedGroup.RootCollectionBindKey] = "Patient" + }, + pin: "root collection bind does not match the exact root resource type", + }, + { + name: "uniqueness_proof_required", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].CodedGroup.SourceRowsUnique = false + }, + pin: "CODED_GROUP requires the compiler proof of one source row per root _key", + }, + { + name: "identity_column_must_be_root_key", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].CodedGroup.SourceIdentityColumn = "id" + }, + pin: "CODED_GROUP requires the compiler proof of one source row per root _key", + }, + { + name: "unknown_missing_key_policy", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].CodedGroup.MissingKeyPolicy = "SOMETHING" + }, + pin: `missing-key policy "SOMETHING" is unsupported`, + }, + { + name: "path_must_end_repeated", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + coded := plan.StageSequence.Stages[0].CodedGroup + coded.PathSegments = []ir.PhysicalCodedGroupPathSegment{{Name: "component", Repeated: true}, {Name: "code"}} + coded.CodingPath = "component[].code" + plan.BindVars[coded.CodingPathBindKey] = coded.CodingPath + }, + pin: "the Coding path must end in a repeated Coding array", + }, + { + name: "canonical_segment_join_required", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + coded := plan.StageSequence.Stages[0].CodedGroup + coded.PathSegments = []ir.PhysicalCodedGroupPathSegment{{Name: "component"}, {Name: "code"}, {Name: "coding", Repeated: true}} + }, + pin: "the Coding path must be the canonical join of its segments", + }, + { + name: "unsafe_segment_name_refused", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + coded := plan.StageSequence.Stages[0].CodedGroup + coded.PathSegments = []ir.PhysicalCodedGroupPathSegment{{Name: "component", Repeated: true}, {Name: "code"}, {Name: "bad-name", Repeated: true}} + coded.CodingPath = "component[].code.bad-name[]" + plan.BindVars[coded.CodingPathBindKey] = coded.CodingPath + }, + pin: `coded-group path segment "bad-name" is not renderer-safe`, + }, + { + name: "preview_limit_must_be_positive_int", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.PreviewLimitBindKey = "limit" + plan.BindVars["limit"] = 0 + }, + pin: "coded-group preview limit bind \"limit\" must be a positive int", + }, + { + name: "preview_limit_non_int", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.PreviewLimitBindKey = "limit" + plan.BindVars["limit"] = "ten" + }, + pin: "coded-group preview limit bind \"limit\" must be a positive int", + }, + { + name: "traversal_outside_the_source_surface", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + last := len(plan.Operations) - 1 + plan.Operations = append(plan.Operations[:last], + ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp}, + plan.Operations[last]) + }, + pin: `operation "TRAVERSAL" is not part of the registered coded-group source surface`, + }, + { + name: "unnest_outside_the_source_surface", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + last := len(plan.Operations) - 1 + plan.Operations = append(plan.Operations[:last], + ir.PhysicalOperation{Kind: ir.PhysicalUnnestOp, Unnest: &ir.PhysicalUnnest{}}, + plan.Operations[last]) + }, + pin: `operation "UNNEST" is not part of the registered coded-group source surface`, + }, + { + name: "expression_let_outside_the_source_surface", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + last := len(plan.Operations) - 1 + plan.Operations = append(plan.Operations[:last], + ir.PhysicalOperation{Kind: ir.PhysicalExpressionLetOp}, + plan.Operations[last]) + }, + pin: `operation "EXPRESSION_LET" is not part of the registered coded-group source surface`, + }, + { + name: "population_root_population_binding_gap", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.Operations[0].RootScan.Population = &ir.PhysicalPopulationRootSource{} + }, + pin: "bound only by RenderPopulationRootScan (population.go)", + }, + { + name: "diagnostic_returns_have_no_pg_route", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.RowLineageReturn = &ir.PhysicalRowLineageReturn{} + }, + pin: "row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route", + }, + { + name: "cell_trace_return_refused", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.CellTraceReturn = &ir.PhysicalCellTraceReturn{} + }, + pin: "row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route", + }, + { + name: "preview_source_window_requires_pivot", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.PreviewSourceWindowByRootID = true + }, + pin: "preview source windowing and the terminal Pivot window require Pivot terminals", + }, + { + name: "plan_level_preview_source_window_refused", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.PreviewSourceWindowByRootID = true + }, + pin: "preview source windowing and the terminal Pivot window require Pivot terminals", + }, + { + name: "output_auth_path_composite_gap", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.OutputAuthResourcePathBindKey = "output_auth" + }, + pin: "the composite Combine prefix authorization path is not registered for CODED_GROUP", + }, + { + name: "terminal_return_required", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + last := len(plan.Operations) - 1 + plan.Operations = plan.Operations[:last] + }, + pin: "the coded-group source requires a terminal RETURN", + }, + { + name: "second_root_scan_refused", + mutate: func(t *testing.T, plan *ir.PhysicalPlan) { + plan.Operations = append([]ir.PhysicalOperation{{Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root2", CollectionBindKey: "root_collection"}}}, plan.Operations...) + }, + pin: "exactly one leading ROOT_SCAN is required", + }, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + plan := ir.ClonePhysicalPlan(base) + testCase.mutate(t, &plan) + var err error + if testCase.door { + _, err = renderCodedGroupPhysicalPlan(plan, mustBinding(t)) + } else { + _, err = RenderPhysicalPlan(plan, mustBinding(t)) + } + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("error = %v, want ErrUnsupportedPostgresPlan", err) + } + if !strings.Contains(err.Error(), testCase.pin) { + t.Fatalf("refusal message must pin %q verbatim, got: %v", testCase.pin, err) + } + }) + } +} + +func TestCodedGroupEntryDispatchEquality(t *testing.T) { + plan := coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyError, "proj-a", "gen-1") + fromEntry, err := RenderPhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if err != nil { + t.Fatalf("entry render: %v", err) + } + fromDoor, err := renderCodedGroupPhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if err != nil || fromDoor.SQL != fromEntry.SQL || !reflect.DeepEqual(fromDoor.Args, fromEntry.Args) { + t.Fatalf("entry and direct door diverge: %v", err) + } + + broken := ir.ClonePhysicalPlan(plan) + broken.StageSequence.Stages[0].CodedGroup.SourceRowsUnique = false + entryErr := errorOf(RenderPhysicalPlan(broken, mustBinding(t))) + doorErr := errorOf(renderCodedGroupPhysicalPlan(broken, mustBinding(t))) + if entryErr == nil || doorErr == nil || entryErr.Error() != doorErr.Error() { + t.Fatalf("doors must refuse with one message: entry=%v door=%v", entryErr, doorErr) + } +} + +func errorOf(_ any, err error) error { return err } + +const coded19ManyRowValuesRefusalPin = "coded-group many-cardinality (local-array-expansion) row values are a wave-2 pg-sql-13 gap" + +func coded19AttachIRRowValues(t *testing.T, plan *ir.PhysicalPlan) { + t.Helper() + coded19AttachIRRowValuesPolicy(t, plan, "ONE", false) +} + +func coded19AttachIRRowValuesPolicy(t *testing.T, plan *ir.PhysicalPlan, policy string, many bool) { + t.Helper() + sourceReturn := plan.Operations[len(plan.Operations)-1].Return + var resourceID ir.PhysicalProjection + found := false + for _, projection := range sourceReturn.Projections { + if projection.Name == "resource_id" { + resourceID, found = projection, true + } + } + if !found { + t.Fatal("lowered source Return lost the resource_id projection") + } + resourceID.Hidden = true + coded := plan.StageSequence.Stages[0].CodedGroup + coded.RowValues = []ir.PhysicalStageRowValue{{ + InputColumn: "resource_id", InputKind: "STRING", InputMany: many, + Output: "row_value_1", Policy: policy, Variable: "cg19_rv_var", + }} + coded.RowValueProjections = []ir.PhysicalProjection{resourceID} + stage := &plan.StageSequence.Stages[0] + stage.OutputProjections = append(stage.OutputProjections, ir.PhysicalProjection{Name: "row_value_1", Value: ir.PhysicalValue{Variable: "cg19_rv_var"}}) + cardinality := "many" + nullable := false + if policy == "ONE" { + cardinality = "optional_one" + nullable = true + } + column := ir.PhysicalStageColumn{ID: "rv_id", Name: "row_value_1", Label: "row_value_1", Kind: "string", Cardinality: cardinality, Nullable: nullable} + stage.OutputColumns = append(stage.OutputColumns, column) + plan.StageSequence.FinalColumns = append(plan.StageSequence.FinalColumns, column) +} + +func TestCodedGroupIRRowValuesAliveControl(t *testing.T) { + plan := coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, "proj-a", "gen-1") + coded19AttachIRRowValues(t, &plan) + if err := plan.Validate(); err != nil { + t.Fatalf("the producer channel must accept this shape (probe-banked ACCEPT+EMIT): %v", err) + } + shared, err := groupRowValueExpressionFor("ONE", "STRING", false, + &groupColumn{name: "resource_id", alias: "cg_rv0", ref: `"cg_rv0"`, jsonb: true}, + func(index int) string { return "$" + strconv.Itoa(index) }) + if err != nil { + t.Fatalf("shared emitter: %v", err) + } + normalize := regexp.MustCompile(`\$[0-9]+`).ReplaceAllString(shared, "$N") + for label, render := range map[string]func(ir.PhysicalPlan, ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error){ + "entry": RenderPhysicalPlan, + "door": renderCodedGroupPhysicalPlan, + } { + rendered, err := render(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if err != nil { + t.Fatalf("%s must render the registered row-values arm (pg-sql-13 Q4): %v", label, err) + } + for _, want := range []string{ + ` AS cg_payload, `, ` AS cg_rv0`, `, s.cg_rv0`, `, c.cg_rv0`, + `AS cg_g0`, `g.cg_g0 AS "c5"`, + } { + if !strings.Contains(rendered.SQL, want) { + t.Fatalf("%s arm is missing %q:\n%s", label, want, rendered.SQL) + } + } + if normalized := regexp.MustCompile(`\$[0-9]+`).ReplaceAllString(rendered.SQL, "$N"); !strings.Contains(normalized, normalize+` AS cg_g0`) { + t.Fatalf("%s must emit the SHARED ONE fragment byte-identical (same-emitter law, Q4):\n%s", label, rendered.SQL) + } + if len(rendered.OutputColumns) != 6 { + t.Fatalf("%s outputs = %d, want five tuple members plus the row value", label, len(rendered.OutputColumns)) + } + } +} + +func TestCodedGroupRowValuesSharedEmitterLaw(t *testing.T) { + source, err := os.ReadFile("coded_group.go") + if err != nil { + t.Fatalf("read coded_group.go: %v", err) + } + var code []string + for _, line := range strings.Split(string(source), "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "//") || strings.HasPrefix(trimmed, "/*") || strings.HasPrefix(trimmed, "*") { + continue + } + code = append(code, line) + } + text := strings.Join(code, "\n") + for _, want := range []string{`groupRowValueExpressionFor(`, `groupColumnRead(`} { + if !strings.Contains(text, want) { + t.Fatalf("the coded-group row-values arm must dispatch through the shared emitter %q", want) + } + } + for _, forbidden := range []string{`ARRAY_AGG(DISTINCT `, `CONSTRUCTION_ROW_VALUE_MULTIPLE_VALUES`, `CONSTRUCTION_GROUP_VALUE_TYPE_MISMATCH`} { + if strings.Contains(text, forbidden) { + t.Fatalf("coded_group.go carries an inlined emitter literal %q — one emitter only (Q4)", forbidden) + } + } +} + +func TestCodedGroupIRRowValuesLiveZeroSideEffects(t *testing.T) { + svc := newCoded19Service(t) + coded19TrackFixtures(t, svc) + coded19Insert(t, svc, "rvw-1", coded19Project, coded19Generation, "FHIR/Observation", codingPayload(coding("sys-rv", "", "kept"))) + + plan := coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, coded19Project, coded19Generation) + coded19AttachIRRowValuesPolicy(t, &plan, "ONE", true) + + before := coded19Inventory(t, svc) + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err == nil || !strings.Contains(err.Error(), coded19ManyRowValuesRefusalPin) { + t.Fatalf("refusal required before any SQL; got rendered=%v err=%v", rendered.SQL, err) + } + if rendered.SQL != "" || len(rendered.Args) != 0 || len(rendered.OutputColumns) != 0 { + t.Fatalf("a refused plan must carry zero SQL, zero parameters, zero columns: %#v", rendered) + } + if after := coded19Inventory(t, svc); after != before { + t.Fatalf("the refused many row-values plan changed service state:\n before %q\n after %q", before, after) + } + if rows := svc.witness(t, `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" WHERE project = $1`, coded19Project); rows[0][0] != "1" { + t.Fatalf("alive witness: %v, want the single inserted row", rows) + } +} + +func TestCodedGroupRowValuesLiveRender(t *testing.T) { + svc := newCoded19Service(t) + coded19TrackFixtures(t, svc) + onePayload := codingPayload(coding("sys-rv", "", "kept")) + onePayload["id"] = "rv-a" + coded19Insert(t, svc, "rvw-a", coded19Project, coded19Generation, "FHIR/Observation", onePayload) + + onePlan := coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, coded19Project, coded19Generation) + coded19AttachIRRowValuesPolicy(t, &onePlan, "ONE", false) + oneRendered, err := RenderPhysicalPlan(onePlan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("registered row-values arm must render: %v", err) + } + rows := svc.runRendered(t, oneRendered) + if len(rows) != 1 || rows[0][0] != "sys-rv" || rows[0][2] != "kept" || rows[0][3] != "1" || rows[0][5] != "rv-a" { + t.Fatalf("ONE over one contributing root: %v", rows) + } + + allPlan := coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, coded19Project, coded19Generation) + coded19AttachIRRowValuesPolicy(t, &allPlan, "ALL", false) + allRendered, err := RenderPhysicalPlan(allPlan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("registered row-values arm must render (ALL): %v", err) + } + + secondPayload := codingPayload(coding("sys-rv", "", "kept")) + secondPayload["id"] = "rv-b" + coded19Insert(t, svc, "rvw-b", coded19Project, coded19Generation, "FHIR/Observation", secondPayload) + allRows := svc.runRendered(t, allRendered) + if len(allRows) != 1 || allRows[0][3] != "2" || allRows[0][5] != `[rv-a rv-b]` { + t.Fatalf("ALL over two contributing roots: %v", allRows) + } + group13ExpectRaise(t, svc, oneRendered, groupRowValueMultipleTag) + + svc.witness(t, `DELETE FROM "`+svc.schema+`"."fhir_resource" WHERE logical_id = 'rvw-b'`) + if again := svc.runRendered(t, oneRendered); len(again) != 1 || again[0][5] != "rv-a" { + t.Fatalf("ONE after removing the second root: %v", again) + } +} + +func coded19Inventory(t *testing.T, svc *pgir02Service) string { + t.Helper() + tables := svc.witness(t, + `SELECT c.relname, c.reltuples::bigint FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace`+ + ` WHERE n.nspname = $1 AND c.relkind = 'r' ORDER BY 1`, svc.schema) + resourceRows := svc.witness(t, + `SELECT count(*), coalesce(md5(string_agg(payload::text, ',' ORDER BY resource_seq)), '') FROM "`+svc.schema+`"."fhir_resource"`) + return strings.Join([]string{fmt.Sprint(tables), fmt.Sprint(resourceRows)}, "|") +} + +const coded19SetRefusalMessage = "postgres renderer: unsupported physical plan: coded-group source-region SET is a pg-sql-16 capability gap (no PostgreSQL contract for lowering-emitted child-set subplans; probe-attested L-19.4) and is refused, never partially rendered" + +func coded19SetInSourceRegionPlan(t *testing.T, required bool) ir.PhysicalPlan { + t.Helper() + match := "" + if required { + match = `"matchMode":"REQUIRED",` + } + doc := `{"recipeSchemaVersion":1,"name":"coded_group_rows","translationVersion":"test","outputs":[{ + "name":"coded_group_rows","rootResourceType":"Observation","rowGrain":"resource", + "fields":[{"name":"resource_id","columnId":"resource_id","expr":{"select":"root.id"}}], + "traversals":[{"name":"subject_Patient","alias":"occ_parent","toResourceType":"Patient",` + match + `"fields":[{"name":"subject_gender","columnId":"subject_gender","expr":{"select":"occ_parent.gender"}}]}], + "construction":{"version":1, + "sourceColumns":[{"id":"resource_id","name":"resource_id"},{"id":"occ_parent__subject_gender","name":"occ_parent__subject_gender"}], + "steps":[{"id":"group_codes","inputs":[{"kind":"SOURCE_PROJECTION"}], + "operation":{"kind":"CODED_GROUP","codedGroup":{ + "constructionId":"group_codes", + "source":{"occurrenceId":"base","resourceType":"Observation","codingPath":"component[].code.coding[]","fhirType":"Coding","cardinality":"MANY","shape":"ARRAY","route":[]}, + "missingKeyPolicy":"GROUP", + "systemOutputColumnId":"system_id","versionOutputColumnId":"version_id","codeOutputColumnId":"code_id","distinctSourceCountOutputColumnId":"count_id"}}, + "outputs":[ + {"id":"system_id","name":"code_system","type":"string","nullable":true}, + {"id":"version_id","name":"code_version","type":"string","nullable":true}, + {"id":"code_id","name":"code","type":"string","nullable":true}, + {"id":"count_id","name":"source_records","type":"integer"}]}]}}]}` + parsed, err := recipe.Parse([]byte(doc)) + if err != nil { + t.Fatalf("recipe.Parse (probe-banked document): %v", err) + } + bindings := recipe.RuntimeBindings{Project: coded19Project, DatasetGeneration: coded19Generation, AuthResourcePaths: []string{"FHIR/Observation"}} + plan, err := semantic.BuildRecipePlan(parsed, bindings) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("lower.CompileResolvedRecipePlan (the whitelist emits SET): %v", err) + } + lowered := compiled.Outputs[0].Plan + setOps := 0 + for _, operation := range lowered.Operations { + if operation.Kind == ir.PhysicalSetOp { + setOps++ + } + } + if setOps != 1 { + t.Fatalf("the probe-banked document must lower exactly one source-region SET, got %d", setOps) + } + return ir.ClonePhysicalPlan(lowered) +} + +func TestCodedGroupIRSetAliveControl(t *testing.T) { + for _, required := range []bool{false, true} { + plan := coded19SetInSourceRegionPlan(t, required) + if err := plan.Validate(); err != nil { + t.Fatalf("required=%v: the producer channel must accept this shape (probe-banked EMIT): %v", required, err) + } + strategies := map[ir.PhysicalTraversalStrategy]bool{} + for _, operation := range plan.Operations { + if operation.Kind != ir.PhysicalSetOp || operation.Set == nil { + continue + } + for _, sub := range operation.Set.Subplan.Operations { + if sub.Kind == ir.PhysicalTraversalOp && sub.Traversal != nil { + strategies[sub.Traversal.Strategy] = true + } + } + } + if !strategies[ir.PhysicalTraversalEndpointLookup] { + t.Fatalf("required=%v: the probe pinned the default policy to stamp ENDPOINT_LOOKUP on the child-set subplan, got %v", required, strategies) + } + for label, render := range map[string]func(ir.PhysicalPlan, ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error){ + "entry": RenderPhysicalPlan, + "door": renderCodedGroupPhysicalPlan, + } { + rendered, err := render(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if err == nil { + t.Fatalf("required=%v %s rendered the refused SET shape — the fail-closed arm is not load-bearing:\n%s", required, label, rendered.SQL) + } + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("required=%v %s error = %v (%T), want ErrUnsupportedPostgresPlan", required, label, err, err) + } + if err.Error() != coded19SetRefusalMessage { + t.Fatalf("required=%v %s refusal message must be cell-verbatim:\n got %q\nwant %q", required, label, err.Error(), coded19SetRefusalMessage) + } + } + } +} + +func TestCodedGroupIRSetLiveZeroSideEffects(t *testing.T) { + svc := newCoded19Service(t) + coded19TrackFixtures(t, svc) + coded19Insert(t, svc, "stw-1", coded19Project, coded19Generation, "FHIR/Observation", codingPayload(coding("sys-stw", "", "kept"))) + + plan := coded19SetInSourceRegionPlan(t, false) + + before := coded19Inventory(t, svc) + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err == nil || err.Error() != coded19SetRefusalMessage { + t.Fatalf("refusal required before any SQL; got rendered=%v err=%v", rendered.SQL, err) + } + if rendered.SQL != "" || len(rendered.Args) != 0 || len(rendered.OutputColumns) != 0 { + t.Fatalf("a refused plan must carry zero SQL, zero parameters, zero columns: %#v", rendered) + } + if after := coded19Inventory(t, svc); after != before { + t.Fatalf("the refused SET plan changed service state:\n before %q\n after %q", before, after) + } + if rows := svc.witness(t, `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" WHERE project = $1`, coded19Project); rows[0][0] != "1" { + t.Fatalf("alive witness: %v, want the single inserted row", rows) + } +} + +func TestCodedGroupTrimSetStructuralLaw(t *testing.T) { + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read package: %v", err) + } + constDeclaration := `const codedGroupTrimCharacters = " "` + const bindingMarker = `, " + dollar(trimChars) + "::text)` + declarations := 0 + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".go") { + continue + } + source, err := os.ReadFile(name) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + text := string(source) + if strings.HasSuffix(name, "_test.go") { + continue + } + if strings.Contains(text, constDeclaration) { + declarations++ + if name != "coded_group.go" { + t.Fatalf("trim-set constant declared outside coded_group.go: %s", name) + } + } + if name != "coded_group.go" { + for _, line := range strings.Split(text, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "//") || strings.HasPrefix(trimmed, "/*") || strings.HasPrefix(trimmed, "*") { + continue + } + if strings.Contains(line, "btrim(") { + if name != "calls.go" { + t.Fatalf("%s introduces a btrim call outside the registered coded-group gate file: %s", name, trimmed) + } + } + if strings.Contains(line, "codedGroupTrimCharacters") { + t.Fatalf("%s uses the trim-set constant outside the registered coded-group gate file: %s", name, trimmed) + } + } + } + if name != "coded_group.go" { + continue + } + lines := strings.Split(text, "\n") + for lineOffset, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "//") || strings.HasPrefix(trimmed, "/*") || strings.HasPrefix(trimmed, "*") { + continue + } + if !strings.Contains(line, "btrim(") { + continue + } + if !strings.Contains(line, bindingMarker) { + t.Fatalf("coded_group.go line %d has a btrim call that does not bind the trim set through the single constant:\n%s", lineOffset+1, line) + } + } + } + if declarations != 1 { + t.Fatalf("trim-set constant declarations = %d, want exactly 1", declarations) + } +} + +func newCoded19Service(t *testing.T) *pgir02Service { + t.Helper() + return newOwnedService(t, 0, "coded19_it") +} + +func coded19RenderSvc(t *testing.T, svc *pgir02Service, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan(%s): %v", svc.schema, err) + } + return rendered +} + +var ( + coded19FixturesMu sync.Mutex + coded19TrackedFixtures = map[string]bool{} +) + +func coded19TrackFixtures(t *testing.T, svc *pgir02Service) { + t.Helper() + var database string + if err := svc.iso.QueryRow(svc.ctx, "SELECT current_database()").Scan(&database); err != nil { + svc.fail("track owned database", err) + } + coded19FixturesMu.Lock() + defer coded19FixturesMu.Unlock() + coded19TrackedFixtures[database] = true + coded19TrackedFixtures[svc.schema] = true +} + +func coded19SnapshotOwnedResources(ctx context.Context, raw string) ([]string, error) { + pool, err := pgxpool.New(ctx, raw) + if err != nil { + return nil, fmt.Errorf("connect: %w", err) + } + defer pool.Close() + rows, err := pool.Query(ctx, + `SELECT datname FROM pg_database WHERE datname LIKE 'coded19_it_%'`+ + ` UNION ALL SELECT nspname FROM pg_namespace WHERE nspname LIKE 'coded19_it_%'`) + if err != nil { + return nil, fmt.Errorf("snapshot: %w", err) + } + defer rows.Close() + var names []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("scan: %w", err) + } + names = append(names, name) + } + return names, rows.Err() +} + +func coded19RunDeltaAudit(inner func() int) int { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return inner() + } + beforeList, err := coded19SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-19 delta audit: snapshot before run: %v\n", err) + return inner() + } + if len(beforeList) > 0 { + fmt.Fprintf(os.Stderr, "pg-sql-19 delta audit: pre-snapshot found %d surviving coded19_it_* resources before this run: %v\n", len(beforeList), beforeList) + return 1 + } + before := make(map[string]bool, len(beforeList)) + for _, name := range beforeList { + before[name] = true + } + code := inner() + after, err := coded19SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-19 delta audit: snapshot after run: %v\n", err) + code = 1 + } + coded19FixturesMu.Lock() + defer coded19FixturesMu.Unlock() + for _, name := range after { + if before[name] { + continue + } + if coded19TrackedFixtures[name] { + fmt.Fprintf(os.Stderr, "pg-sql-19 delta audit: %q was created and tracked by this run and still survives\n", name) + code = 1 + continue + } + fmt.Fprintf(os.Stderr, "pg-sql-19 delta audit: note — %q was born during this run but is not owned by it (sibling lane?)\n", name) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + survivors, err := coded19SnapshotOwnedResources(ctx, raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-19 delta audit: survivor scan: %v\n", err) + code = 1 + } else if len(survivors) > 0 { + fmt.Fprintf(os.Stderr, "pg-sql-19 delta audit: %d coded19_it_* resources survive: %v\n", len(survivors), survivors) + code = 1 + } + return code +} + +const ( + coded19Project = "p19-proj" + coded19Generation = "p19-gen" +) + +func coding(system, version, code string) map[string]any { + value := map[string]any{} + if system != "" { + value["system"] = system + } + if version != "" { + value["version"] = version + } + if code != "" { + value["code"] = code + } + return value +} + +func codingPayload(codings ...map[string]any) map[string]any { + return map[string]any{"component": []any{map[string]any{"code": map[string]any{"coding": codings}}}} +} + +func coded19Insert(t *testing.T, svc *pgir02Service, logicalID, project, generation, authPath string, payload map[string]any) { + t.Helper() + svc.insertResource(t, svc.schema, project, generation, "Observation", logicalID, authPath, payload) +} + +func TestCodedGroupLiveExactTupleMultiset(t *testing.T) { + svc := newCoded19Service(t) + coded19TrackFixtures(t, svc) + + coded19Insert(t, svc, "obs-a", coded19Project, coded19Generation, "FHIR/Observation", codingPayload( + coding("system-a", "", "shared"), + coding("system-a", "", "shared"), + coding("system-b", "", "shared"), + coding("system-a", "", "only-a"), + )) + coded19Insert(t, svc, "obs-b", coded19Project, coded19Generation, "FHIR/Observation", codingPayload( + coding("system-a", "", "shared"), + coding("system-a", "v1", "shared"), + coding("system-a", " v1 ", "shared"), + )) + coded19Insert(t, svc, "obs-missing", coded19Project, coded19Generation, "FHIR/Observation", nil) + coded19Insert(t, svc, "obs-invalid", coded19Project, coded19Generation, "FHIR/Observation", codingPayload( + coding(" ", "", "shared"), coding("", "", "shared"), coding("system-a", "", ""), + )) + svc.insertResource(t, svc.schema, coded19Project, coded19Generation, "Observation", "obs-nonarray", "FHIR/Observation", + map[string]any{"component": []any{map[string]any{"code": map[string]any{"coding": coding("system-x", "", "y")}}}}) + svc.insertResource(t, svc.schema, coded19Project, coded19Generation, "Observation", "obs-nullitem", "FHIR/Observation", + map[string]any{"component": []any{map[string]any{"code": map[string]any{"coding": []any{nil}}}}}) + + coded19Insert(t, svc, "obs-denied", coded19Project, coded19Generation, "FHIR/Hidden", codingPayload(coding("system-a", "", "denied"))) + coded19Insert(t, svc, "obs-oldgen", coded19Project, "p19-old-gen", "FHIR/Observation", codingPayload(coding("system-a", "", "oldgen"))) + coded19Insert(t, svc, "obs-foreign", "p19-foreign", coded19Generation, "FHIR/Observation", codingPayload(coding("system-a", "", "foreign"))) + + if rows := svc.witness(t, `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" WHERE project = $1`, coded19Project); rows[0][0] != "8" { + t.Fatalf("scope witness: p19-proj row count = %v, want 8", rows) + } + + plan := coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, coded19Project, coded19Generation) + rendered := coded19RenderSvc(t, svc, plan) + for _, leaked := range []string{coded19Project, coded19Generation, "system-a", "shared", "FHIR/Observation"} { + if strings.Contains(rendered.SQL, leaked) { + t.Fatalf("value %q leaked into statement text: %s", leaked, rendered.SQL) + } + } + rows := coded19Rows(t, svc.runRendered(t, rendered)) + + type want struct { + system, version, code *string + count string + } + expected := []want{ + {coded19Text("system-a"), nil, coded19Text("shared"), "2"}, + {coded19Text("system-b"), nil, coded19Text("shared"), "1"}, + {coded19Text("system-a"), nil, coded19Text("only-a"), "1"}, + {coded19Text("system-a"), coded19Text("v1"), coded19Text("shared"), "1"}, + {coded19Text("system-a"), coded19Text(" v1 "), coded19Text("shared"), "1"}, + {nil, nil, nil, "4"}, + } + if len(rows) != len(expected) { + t.Fatalf("grouped tuples = %v, want %d rows", rows, len(expected)) + } + byTuple := map[string]coded19Row{} + for _, row := range rows { + key := fmt.Sprintf("%v|%v|%v", deref(row.system), deref(row.version), deref(row.code)) + if _, clash := byTuple[key]; clash { + t.Fatalf("duplicate tuple row %q: %v", key, rows) + } + byTuple[key] = row + } + for _, want := range expected { + key := fmt.Sprintf("%v|%v|%v", deref(want.system), deref(want.version), deref(want.code)) + row, ok := byTuple[key] + if !ok { + t.Fatalf("missing tuple row %q in %v", key, rows) + } + if row.count != want.count { + t.Fatalf("tuple %q count = %q, want %q", key, row.count, want.count) + } + } + for _, forbidden := range []string{"denied", "oldgen", "foreign", "system-x", "y"} { + for _, row := range rows { + if deref(row.code) == forbidden { + t.Fatalf("out-of-scope coding %q grouped: %v", forbidden, rows) + } + } + } + + identities := make([]string, len(rows)) + for index, row := range rows { + identities[index] = row.identity + if want := coded19Identity(t, row.system, row.version, row.code); row.identity != want { + t.Fatalf("row identity = %q, want %q", row.identity, want) + } + } + sorted := append([]string(nil), identities...) + sort.Strings(sorted) + if !reflect.DeepEqual(identities, sorted) { + t.Fatalf("rows are not ordered by the identity COLLATE \"C\" bytes:\n got %v\nwant %v", identities, sorted) + } + seen := map[string]bool{} + for _, identity := range identities { + if seen[identity] { + t.Fatalf("duplicate row identity %q", identity) + } + seen[identity] = true + } + + repeated := coded19Rows(t, svc.runRendered(t, rendered)) + if !reflect.DeepEqual(rows, repeated) { + t.Fatalf("grouping changed between executions:\n%v\n%v", rows, repeated) + } + + excludeRendered := coded19RenderSvc(t, svc, coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyExclude, coded19Project, coded19Generation)) + excluded := coded19Rows(t, svc.runRendered(t, excludeRendered)) + if len(excluded) != len(expected)-1 { + t.Fatalf("EXCLUDE rows = %v, want %d (missing tuple excluded)", excluded, len(expected)-1) + } + for _, row := range excluded { + if row.system == nil && row.version == nil && row.code == nil { + t.Fatalf("EXCLUDE kept the missing tuple: %v", excluded) + } + } + + errorRendered := coded19RenderSvc(t, svc, coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyError, coded19Project, coded19Generation)) + if err := coded19ExecErr(svc, errorRendered); err == nil { + t.Fatalf("ERROR policy must raise on the invalid candidates") + } else if !strings.Contains(svc.scrub(err), codedGroupMissingKeyTag) { + t.Fatalf("ERROR policy error must carry the canonical tag: %v", svc.scrub(err)) + } else if !strings.Contains(svc.scrub(err), "22P02") { + t.Fatalf("ERROR guard must raise through the barrier cast (22P02): %v", svc.scrub(err)) + } + + coded19Insert(t, svc, "obs-c", coded19Project, coded19Generation, "FHIR/Observation", codingPayload(coding("system-a", "", "shared"))) + mutated := coded19Rows(t, svc.runRendered(t, rendered)) + for _, row := range mutated { + if deref(row.system) == "system-a" && row.version == nil && deref(row.code) == "shared" && row.count == "3" { + return + } + } + t.Fatalf("mutation probe: (system-a,null,shared) count never became 3: %v", mutated) +} + +func deref(value *string) string { + if value == nil { + return "" + } + return *value +} + +func coded19QueryArgs(rendered command.RenderedPostgresPhysicalPlan) []any { + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + return args +} + +func coded19ExecErr(svc *pgir02Service, rendered command.RenderedPostgresPhysicalPlan) error { + rows, err := svc.iso.Query(svc.ctx, rendered.SQL, coded19QueryArgs(rendered)...) + if err != nil { + return err + } + for rows.Next() { + } + closeErr := rows.Err() + rows.Close() + return closeErr +} + +func TestCodedGroupLiveErrorPolicyBarrierMatrix(t *testing.T) { + svc := newCoded19Service(t) + coded19TrackFixtures(t, svc) + + const allInvalidProject = "p19-all-invalid" + coded19Insert(t, svc, "inv-1", allInvalidProject, coded19Generation, "FHIR/Observation", codingPayload(coding("", "", "shared"))) + coded19Insert(t, svc, "inv-2", allInvalidProject, coded19Generation, "FHIR/Observation", nil) + + exclude := coded19RenderSvc(t, svc, coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyExclude, allInvalidProject, coded19Generation)) + if rows := svc.runRendered(t, exclude); len(rows) != 0 { + t.Fatalf("EXCLUDE over all-invalid data must group nothing: %v", rows) + } + grouped := coded19Rows(t, svc.runRendered(t, coded19RenderSvc(t, svc, coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, allInvalidProject, coded19Generation)))) + if len(grouped) != 1 || grouped[0].count != "2" || grouped[0].system != nil { + t.Fatalf("GROUP sabotage control: %v, want one missing tuple with count 2", grouped) + } + errorRendered := coded19RenderSvc(t, svc, coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyError, allInvalidProject, coded19Generation)) + if err := coded19ExecErr(svc, errorRendered); err == nil || + !strings.Contains(svc.scrub(err), codedGroupMissingKeyTag) { + t.Fatalf("ERROR must raise at zero grouped rows: %v", svc.scrub(fmt.Sprint(err))) + } + empty := coded19RenderSvc(t, svc, coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyError, "p19-no-rows", coded19Generation)) + if rows := svc.runRendered(t, empty); len(rows) != 0 { + t.Fatalf("zero-source ERROR must return zero rows: %v", rows) + } + const healthyProject = "p19-healthy" + coded19Insert(t, svc, "ok-1", healthyProject, coded19Generation, "FHIR/Observation", codingPayload(coding("sys-h", "", "c1"))) + healthy := coded19Rows(t, svc.runRendered(t, coded19RenderSvc(t, svc, coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyError, healthyProject, coded19Generation)))) + if len(healthy) != 1 || deref(healthy[0].system) != "sys-h" || healthy[0].count != "1" { + t.Fatalf("healthy ERROR control: %v", healthy) + } + + planRows, err := svc.iso.Query(svc.ctx, "EXPLAIN (FORMAT JSON) "+errorRendered.SQL, coded19QueryArgs(errorRendered)...) + if err != nil { + svc.fail("explain ERROR statement", err) + } + var planText strings.Builder + for planRows.Next() { + var value []byte + if err := planRows.Scan(&value); err != nil { + svc.fail("scan explain row", err) + } + planText.Write(value) + } + planRows.Close() + text := planText.String() + for _, want := range []string{codedGroupMissingKeyTag, guardBarrierGUC} { + if !strings.Contains(text, want) { + t.Fatalf("EXPLAIN plan lost the guard (%q); the barrier was folded:\n%s", want, text) + } + } +} + +func TestCodedGroupLiveWhitespaceUnicodeAndByteOrder(t *testing.T) { + svc := newCoded19Service(t) + coded19TrackFixtures(t, svc) + const project = "p19-ws" + + coded19Insert(t, svc, "ws-1", project, coded19Generation, "FHIR/Observation", codingPayload( + coding("sys-w", "", "\tT"), coding("sys-w", "", "\nN"), coding("sys-w", "", " U"), + )) + coded19Insert(t, svc, "ws-2", project, coded19Generation, "FHIR/Observation", codingPayload( + coding(" ", "", "ws-invalid"), coding("\t", "", "tab-system"), + )) + coded19Insert(t, svc, "ws-3", project, coded19Generation, "FHIR/Observation", codingPayload( + coding("sys-w", "", "null"), + )) + coded19Insert(t, svc, "ws-4", project, coded19Generation, "FHIR/Observation", codingPayload( + coding("sys-u", "", "é"), coding("sys-u", "", "Ω"), coding("sys-u", "", "日本"), coding("sys-u", "", "a"), + )) + + rendered := coded19RenderSvc(t, svc, coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, project, coded19Generation)) + rows := coded19Rows(t, svc.runRendered(t, rendered)) + + type tuple struct{ system, version, code string } + counts := map[tuple]string{} + for _, row := range rows { + counts[tuple{deref(row.system), deref(row.version), deref(row.code)}] = row.count + } + for _, want := range []tuple{ + {"sys-w", "", "\tT"}, {"sys-w", "", "\nN"}, {"sys-w", "", " U"}, + {"\t", "", "tab-system"}, {"sys-w", "", "null"}, + {"sys-u", "", "é"}, {"sys-u", "", "Ω"}, {"sys-u", "", "日本"}, {"sys-u", "", "a"}, + } { + if counts[want] != "1" { + t.Fatalf("tuple %#v count = %q, want 1; all rows: %v", want, counts[want], rows) + } + } + if counts[tuple{"", "", ""}] != "1" { + t.Fatalf("missing tuple must carry exactly the blank-system coding's root: %v", rows) + } + if _, leaked := counts[tuple{" ", "", "ws-invalid"}]; leaked { + t.Fatalf("space-blank system must not survive validity: %v", rows) + } + + identities := make([]string, 0, len(rows)) + for _, row := range rows { + identities = append(identities, row.identity) + } + sorted := append([]string(nil), identities...) + sort.Strings(sorted) + if !reflect.DeepEqual(identities, sorted) { + t.Fatalf("D-19a byte order violated; got %v want %v", identities, sorted) + } + var asciiBeforeNonASCII []string + for _, row := range rows { + if deref(row.system) == "sys-u" { + asciiBeforeNonASCII = append(asciiBeforeNonASCII, deref(row.code)) + } + } + if want := []string{"a", "é", "Ω", "日本"}; !reflect.DeepEqual(asciiBeforeNonASCII, want) { + t.Fatalf("D-19a: sys-u codes ordered %v, want UTF-8 byte order %v", asciiBeforeNonASCII, want) + } +} + +const coded19ProductionDocument = `{ + "recipeSchemaVersion": 1, + "name": "coded_group_rows", + "translationVersion": "test", + "outputs": [ + { + "name": "coded_group_rows", + "rootResourceType": "Observation", + "rowGrain": "resource", + "fields": [ + {"name": "resource_id", "columnId": "resource_id", "expr": {"select": "root.id"}} + ], + "construction": { + "version": 1, + "sourceColumns": [{"id": "resource_id", "name": "resource_id"}], + "steps": [ + { + "id": "group_codes", + "inputs": [{"kind": "SOURCE_PROJECTION"}], + "operation": { + "kind": "CODED_GROUP", + "codedGroup": { + "constructionId": "group_codes", + "source": { + "occurrenceId": "base", + "resourceType": "Observation", + "codingPath": "component[].code.coding[]", + "fhirType": "Coding", + "cardinality": "MANY", + "shape": "ARRAY", + "route": [] + }, + "missingKeyPolicy": "%s", + "systemOutputColumnId": "system_id", + "versionOutputColumnId": "version_id", + "codeOutputColumnId": "code_id", + "distinctSourceCountOutputColumnId": "count_id" + } + }, + "outputs": [ + {"id": "system_id", "name": "code_system", "type": "string", "nullable": true}, + {"id": "version_id", "name": "code_version", "type": "string", "nullable": true}, + {"id": "code_id", "name": "code", "type": "string", "nullable": true}, + {"id": "count_id", "name": "source_records", "type": "integer"} + ] + } + ] + } + } + ] +}` + +func coded19ProductionCompiledPlan(t *testing.T, policy recipe.ConstructionGroupMissingKeyPolicy, project, generation string) ir.PhysicalPlan { + t.Helper() + document := fmt.Sprintf(coded19ProductionDocument, string(policy)) + parsed, err := recipe.Parse([]byte(document)) + if err != nil { + t.Fatalf("recipe.Parse (production document entry): %v", err) + } + bindings := recipe.RuntimeBindings{Project: project, DatasetGeneration: generation, AuthResourcePaths: []string{"FHIR/Observation"}} + plan, err := semantic.BuildRecipePlan(parsed, bindings) + if err != nil { + t.Fatalf("semantic.BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, project, generation) + if err != nil { + t.Fatalf("semantic.ResolveRecipePlan: %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("lower.CompileResolvedRecipePlan: %v", err) + } + sequence := compiled.Outputs[0].Plan.StageSequence + if sequence == nil || len(sequence.Stages) != 1 || sequence.Stages[0].Kind != ir.PhysicalStageCodedGroupOp { + t.Fatalf("production lowering did not emit the inline single-stage CODED_GROUP: %#v", sequence) + } + return ir.ClonePhysicalPlan(compiled.Outputs[0].Plan) +} + +func TestCodedGroupProductionCompiledEqualsStructFixture(t *testing.T) { + for _, policy := range []recipe.ConstructionGroupMissingKeyPolicy{ + recipe.ConstructionGroupMissingKeyGroup, + recipe.ConstructionGroupMissingKeyExclude, + recipe.ConstructionGroupMissingKeyError, + } { + fromDocument := coded19ProductionCompiledPlan(t, policy, "proj-a", "gen-1") + fromStruct := coded19LoweredPlan(t, policy, "proj-a", "gen-1") + if !reflect.DeepEqual(fromDocument, fromStruct) { + t.Fatalf("policy %s: production-document plan differs from the struct fixture", policy) + } + } +} + +func TestCodedGroupLiveProductionCompiledGoldenDedupTieBreak(t *testing.T) { + svc := newCoded19Service(t) + coded19TrackFixtures(t, svc) + const project = "p19-prod" + + coded19Insert(t, svc, "prod-a", project, coded19Generation, "FHIR/Observation", codingPayload( + coding("sys-one", "", "shared"), + coding("sys-one", "", "shared"), + coding("sys-two", "", "shared"), + )) + coded19Insert(t, svc, "prod-b", project, coded19Generation, "FHIR/Observation", codingPayload( + coding("sys-one", "v1", "shared"), + )) + coded19Insert(t, svc, "prod-missing", project, coded19Generation, "FHIR/Observation", nil) + + plan := coded19ProductionCompiledPlan(t, recipe.ConstructionGroupMissingKeyGroup, project, coded19Generation) + rendered := coded19RenderSvc(t, svc, plan) + rows := coded19Rows(t, svc.runRendered(t, rendered)) + + type want struct { + system, version, code string + count string + } + expected := []want{ + {"sys-one", "", "shared", "1"}, + {"sys-two", "", "shared", "1"}, + {"sys-one", "v1", "shared", "1"}, + {"", "", "", "1"}, + } + if len(rows) != len(expected) { + t.Fatalf("production-compiled golden multiset = %v, want %d rows", rows, len(expected)) + } + seen := map[string]string{} + for _, row := range rows { + seen[fmt.Sprintf("%v|%v|%v", deref(row.system), deref(row.version), deref(row.code))] = row.count + } + for _, want := range expected { + key := want.system + "|" + want.version + "|" + want.code + if got, ok := seen[key]; !ok { + t.Fatalf("production-compiled golden row %q missing: %v", key, rows) + } else if got != want.count { + t.Fatalf("production-compiled golden row %q count = %q, want %q", key, got, want.count) + } + } + if seen["sys-one||shared"] != "1" { + t.Fatalf("dedup leg: duplicate identical coding counted %q roots, want 1", seen["sys-one||shared"]) + } + identities := make([]string, len(rows)) + for index, row := range rows { + identities[index] = row.identity + if want := coded19Identity(t, row.system, row.version, row.code); row.identity != want { + t.Fatalf("production-compiled row identity = %q, want %q", row.identity, want) + } + } + sorted := append([]string(nil), identities...) + sort.Strings(sorted) + if !reflect.DeepEqual(identities, sorted) { + t.Fatalf("production-compiled tie-break order violated:\n got %v\nwant %v", identities, sorted) + } + if repeat := coded19Rows(t, svc.runRendered(t, rendered)); !reflect.DeepEqual(rows, repeat) { + t.Fatalf("production-compiled grouping changed between executions:\n%v\n%v", rows, repeat) + } +} + +func TestCodedGroupLiveMultiComponentBranchPruning(t *testing.T) { + svc := newCoded19Service(t) + coded19TrackFixtures(t, svc) + const project = "p19-branch" + const allEmptyProject = "p19-branch-zero" + + component := func(codings any) map[string]any { + if codings == nil { + return map[string]any{"code": map[string]any{}} + } + return map[string]any{"code": map[string]any{"coding": codings}} + } + twoComponents := func(first, second map[string]any) map[string]any { + return map[string]any{"component": []any{first, second}} + } + kept := []any{coding("sys-br", "", "kept")} + coded19Insert(t, svc, "br-empty-second", project, coded19Generation, "FHIR/Observation", + twoComponents(component(kept), component([]any{}))) + coded19Insert(t, svc, "br-absent-second", project, coded19Generation, "FHIR/Observation", + twoComponents(component(kept), component(nil))) + coded19Insert(t, svc, "br-nonarray-second", project, coded19Generation, "FHIR/Observation", + twoComponents(component(kept), component(coding("sys-br", "", "ghost")))) + coded19Insert(t, svc, "br-empty-first", project, coded19Generation, "FHIR/Observation", + twoComponents(component([]any{}), component(kept))) + coded19Insert(t, svc, "br-zero", allEmptyProject, coded19Generation, "FHIR/Observation", + twoComponents(component([]any{}), component(nil))) + + grouped := coded19Rows(t, svc.runRendered(t, coded19RenderSvc(t, svc, + coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, project, coded19Generation)))) + if len(grouped) != 1 || deref(grouped[0].system) != "sys-br" || grouped[0].version != nil || + deref(grouped[0].code) != "kept" || grouped[0].count != "4" { + t.Fatalf("mixed-branch GROUP multiset = %v, want ONLY the (sys-br,null,kept) tuple with count 4 and no null tuple", grouped) + } + + if err := coded19ExecErr(svc, coded19RenderSvc(t, svc, + coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyError, project, coded19Generation))); err != nil { + t.Fatalf("ERROR must stay silent when every root has at least one item: %v", svc.scrub(err)) + } + + zero := coded19Rows(t, svc.runRendered(t, coded19RenderSvc(t, svc, + coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, allEmptyProject, coded19Generation)))) + if len(zero) != 1 || zero[0].system != nil || zero[0].version != nil || zero[0].code != nil || zero[0].count != "1" { + t.Fatalf("whole-expansion-empty root must contribute exactly the missing tuple with count 1: %v", zero) + } + if err := coded19ExecErr(svc, coded19RenderSvc(t, svc, + coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyError, allEmptyProject, coded19Generation))); err == nil || + !strings.Contains(svc.scrub(err), codedGroupMissingKeyTag) { + t.Fatalf("ERROR must raise on the wholly-empty expansion candidate: %v", svc.scrub(fmt.Sprint(err))) + } + + if rows := svc.witness(t, `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" WHERE project = $1`, project); rows[0][0] != "4" { + t.Fatalf("branch witness: %v, want 4 roots", rows) + } + coded19Insert(t, svc, "br-none", project, coded19Generation, "FHIR/Observation", nil) + mutated := coded19Rows(t, svc.runRendered(t, coded19RenderSvc(t, svc, + coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, project, coded19Generation)))) + if len(mutated) != 2 || mutated[1].system != nil || mutated[1].count != "1" { + t.Fatalf("alive control: coding-less insertion must add the missing tuple: %v", mutated) + } +} + +func TestCodedGroupLiveWindowedSource(t *testing.T) { + svc := newCoded19Service(t) + coded19TrackFixtures(t, svc) + const project = "p19-window" + + for index, id := range []string{"w-1", "w-2", "w-3", "w-4"} { + _ = index + coded19Insert(t, svc, id, project, coded19Generation, "FHIR/Observation", codingPayload(coding("sys-window", "", "shared"))) + } + if rows := svc.witness(t, `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" WHERE project = $1`, project); rows[0][0] != "4" { + t.Fatalf("window witness: %v, want 4 rows", rows) + } + + plan := coded19LoweredPlan(t, recipe.ConstructionGroupMissingKeyGroup, project, coded19Generation) + last := len(plan.Operations) - 1 + plan.Operations = append(plan.Operations[:last], + ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{Keys: []ir.PhysicalValue{{Variable: "root", Path: []string{"logical_id"}}}}}, + ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "srclimit"}}, + plan.Operations[last]) + plan.BindVars["srclimit"] = 2 + rendered := coded19RenderSvc(t, svc, plan) + rows := coded19Rows(t, svc.runRendered(t, rendered)) + if len(rows) != 1 || rows[0].count != "2" || deref(rows[0].system) != "sys-window" || rows[0].version != nil || deref(rows[0].code) != "shared" { + t.Fatalf("windowed grouping = %v, want the (sys-window,null,shared) tuple with count 2 (not the missing tuple)", rows) + } + repeat := coded19Rows(t, svc.runRendered(t, rendered)) + if !reflect.DeepEqual(rows, repeat) { + t.Fatalf("the source window is not deterministic across executions: %v vs %v", rows, repeat) + } +} diff --git a/internal/dataframe/compiler/render/postgres/coded_pivot_test.go b/internal/dataframe/compiler/render/postgres/coded_pivot_test.go new file mode 100644 index 000000000..eca57630d --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/coded_pivot_test.go @@ -0,0 +1,1187 @@ +package postgres + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/recipe" + fhirschema "github.com/calypr/loom/internal/fhir/schema" + spec "github.com/calypr/loom/internal/fhir/schema" +) + +func clonePlanForPivot14(plan ir.PhysicalPlan) ir.PhysicalPlan { + clone := plan + clone.BindVars = map[string]any{} + for key, value := range plan.BindVars { + clone.BindVars[key] = value + } + clone.Operations = append([]ir.PhysicalOperation(nil), plan.Operations...) + if plan.StageSequence != nil { + sequence := *plan.StageSequence + sequence.Stages = append([]ir.PhysicalConstructionStage(nil), plan.StageSequence.Stages...) + if sequence.Stages[0].GroupedPivot != nil { + pivot := *sequence.Stages[0].GroupedPivot + if pivot.CodedCorrelation != nil { + correlation := *pivot.CodedCorrelation + pivot.CodedCorrelation = &correlation + } + pivot.CodedCategories = append([]ir.PhysicalGroupedCodedPivotCategory(nil), pivot.CodedCategories...) + pivot.GroupKeys = append([]ir.PhysicalGroupedPivotKey(nil), pivot.GroupKeys...) + pivot.InputProjections = append([]ir.PhysicalProjection(nil), pivot.InputProjections...) + sequence.Stages[0].GroupedPivot = &pivot + } + clone.StageSequence = &sequence + } + return clone +} + +func pivot14CodedBundle(name string, categories []recipe.ConstructionCodedPivotCategory, + duplicate recipe.PivotDuplicatePolicy, missing recipe.PivotMissingCellPolicy, + valuePath string, fallbacks []string, blank bool, arms []string, logical string) recipe.Bundle { + if valuePath == "" { + valuePath = "valueQuantity.value" + } + if logical == "" { + logical = "decimal" + } + if len(arms) == 0 { + arms = []string{"valueQuantity"} + } + if len(categories) == 0 { + categories = []recipe.ConstructionCodedPivotCategory{ + {System: "http://loinc.org", Code: "8480-6", OutputColumnID: "systolic_id"}, + {System: "http://loinc.org", Code: "8462-4", OutputColumnID: "diastolic_id"}, + } + } + outputs := make([]recipe.StageColumn, 0, len(categories)) + for _, category := range categories { + outputs = append(outputs, recipe.StageColumn{ + ID: category.OutputColumnID, Name: strings.TrimSuffix(category.OutputColumnID, "_id"), + Type: logical, Nullable: true}) + } + source := "Observation.component[]." + valuePath + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: name, TranslationVersion: "test", + Outputs: []recipe.Output{{ + Name: name, RootResourceType: "Observation", RowGrain: "resource", + Fields: []recipe.Field{{Name: "resource_id", ColumnID: "resource_id", Expr: recipe.Expression{Select: "root.id"}}}, + Construction: &recipe.Construction{ + Version: 1, + SourceColumns: []recipe.StageColumn{{ID: "resource_id", Name: "resource_id"}}, + Steps: []recipe.ConstructionStep{{ + ID: name, Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{ + Kind: recipe.ConstructionCodedPivotOp, + CodedPivot: &recipe.ConstructionCodedPivot{ + ConstructionID: name, + Source: recipe.ConstructionCodedPivotSource{ + BindingID: name + "-binding", ResourceType: "Observation", + SourcePath: source, SourceCanonical: "http://loinc.org", + OwningScope: "component[]", KeyPath: "component[].code.coding[]", ValuePath: valuePath, + ValueFallbacks: fallbacks, FallbackOnBlankString: blank, + ChoiceArms: arms, LogicalType: logical, + RuleVersion: "semantic-rule-v1", SchemaVersion: 1, + CandidateID: name + "-candidate", NodeID: name + "-node", FieldPath: source, + Route: []recipe.ConstructionRelatedRouteStep{}, + }, + Categories: categories, + DuplicatePolicy: duplicate, + MissingCellPolicy: missing, + }, + }, + Outputs: outputs, + }}, + }, + }}, + } +} + +func pivot14CompileCodedBundle(t *testing.T, name string) ir.PhysicalPlan { + t.Helper() + return pivot14CompileBundle(t, pivot14CodedBundle(name, nil, + recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, "", nil, false, nil, ""), + []string{"FHIR/Observation"}) +} + +func pivot14CodedPivot(t *testing.T, plan ir.PhysicalPlan) *ir.PhysicalGroupedPivot { + t.Helper() + if plan.StageSequence == nil || len(plan.StageSequence.Stages) != 1 || + plan.StageSequence.Stages[0].Kind != ir.PhysicalStagePivotOp || plan.StageSequence.Stages[0].GroupedPivot == nil { + t.Fatalf("want one inline PIVOT stage, got %+v", plan.StageSequence) + } + return plan.StageSequence.Stages[0].GroupedPivot +} + +func pivot14CodedRefused(t *testing.T, plan ir.PhysicalPlan, label string, wantFragment string) { + t.Helper() + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil { + t.Fatalf("%s: expected typed refusal, got success", label) + } + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("%s: refusal not ErrUnsupportedPostgresPlan: %v", label, err) + } + if wantFragment != "" && !strings.Contains(err.Error(), wantFragment) { + t.Fatalf("%s: refusal %q lacks %q", label, err.Error(), wantFragment) + } +} + +func TestCodedPivot14ProductionRenders(t *testing.T) { + plan := pivot14CompileCodedBundle(t, "p14_coded_render") + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("production coded pivot must render through the registered door: %v", err) + } + sql := rendered.SQL + for _, want := range []string{ + "WITH cp_src AS (SELECT", + "cp_owner AS (SELECT s.cp_key,", + "cp_cat_0 AS (", + "cp_elem_0 AS (", + "cp_agg_0 AS (", + "EXISTS (SELECT 1 FROM", + "jsonb_typeof(", + ") = 'string'", + "::text <> ''", + "LEFT JOIN cp_agg_0 AS a0 ON a0.cp_key = s.cp_key", + "WHERE t.cp_m)", + "BOOL_OR(cp_bad) AS cp_bad", + "WHEN COALESCE(a0.cp_bad, false) THEN CAST('INVALID_CHOICE_ARM'", + "jsonb_agg(s.cp_e ORDER BY f.ord, s.ord1)", + "ELSE jsonb_build_array(f.x) END) WITH ORDINALITY AS s(cp_e, ord1)", + "(ARRAY_AGG(e.cp_e ORDER BY e.ord))[1] AS cp_first", + `ORDER BY '[' || '"GROUPED_PIVOT"'`, + `COLLATE "C" ASC`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("coded statement lost %q:\n%s", want, sql) + } + } + multi := pivot14CompileCodedBundle(t, "p14_coded_multi_candidate") + multiCodedPivot := pivot14CodedPivot(t, multi) + multiCodedPivot.CodedCorrelation.ValueFallbacks = []spec.Selector{specSelector(t, "valueString")} + multiRendered, err := RenderPhysicalPlan(multi, mustBinding(t)) + if err != nil { + t.Fatalf("render multi-candidate plan: %v", err) + } + for _, want := range []string{ + "AS c(ord, cand) WHERE ", + "ORDER BY c.ord LIMIT 1", + } { + if !strings.Contains(multiRendered.SQL, want) { + t.Fatalf("multi-candidate pick lost %q:\n%s", want, multiRendered.SQL) + } + } + + for _, forbidden := range []string{"8480-6", "8462-4", "http://loinc.org", "proj-a", "gen-1", "FHIR/Observation"} { + if strings.Contains(sql, forbidden) { + t.Fatalf("coded plan data leaked into SQL text (%q):\n%s", forbidden, sql) + } + } + carried := 0 + for _, parameter := range rendered.Args { + var text string + switch value := parameter.Value.(type) { + case string: + text = value + case []byte: + text = string(value) + } + if strings.Contains(text, "8480-6") || strings.Contains(text, "8462-4") || strings.Contains(text, "loinc") { + carried++ + } + } + if carried < 3 { + t.Fatalf("system/code pairs must ride parameters, carried=%d\n%v", carried, rendered.Args) + } +} + +func TestCodedPivot14GuardOrderAndMissingBranches(t *testing.T) { + plan := pivot14CompileCodedBundle(t, "p14_coded_missing_guard") + plan.StageSequence.Stages[0].GroupedPivot.MissingCellPolicy = "ERROR" + plan.StageSequence.Stages[0].GroupedPivot.DuplicatePolicy = "SUM" + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("render: %v", err) + } + sql := rendered.SQL + sumErrorChainSQL := sql + if !strings.Contains(sql, "COALESCE(a0.cp_tcnt, 0) = 0 THEN CAST('TABLE_PIVOT_CELL_MISSING'") { + t.Fatalf("coded reducer MISSING guard must read the TYPED count (:267):\n%s", sql) + } + if strings.Contains(sql, "COALESCE(a0.cp_cnt, 0) = 0 THEN CAST('TABLE_PIVOT_CELL_MISSING'") { + t.Fatalf("coded reducer branch must NOT read the raw count for MISSING (:267 vs :230):\n%s", sql) + } + + errPlan := pivot14CompileCodedBundle(t, "p14_coded_error_branch") + errPlan.StageSequence.Stages[0].GroupedPivot.DuplicatePolicy = "ERROR" + rendered, err = RenderPhysicalPlan(errPlan, mustBinding(t)) + if err != nil { + t.Fatalf("render: %v", err) + } + sql = rendered.SQL + if !strings.Contains(sql, "COALESCE(a0.cp_cnt, 0) > 1 THEN CAST('TABLE_PIVOT_CELL_CARDINALITY'") { + t.Fatalf("coded ERROR branch cardinality guard must read the RAW count:\n%s", sql) + } + if strings.Contains(sql, "ELSE to_jsonb(a0.cp_sum)") { + t.Fatalf("coded ERROR branch must not enumerate typed reducers:\n%s", sql) + } + + armRendered, err := RenderPhysicalPlan(pivot14CompileCodedBundle(t, "p14_coded_arm_guard"), mustBinding(t)) + if err != nil { + t.Fatalf("render arm guard plan: %v", err) + } + if strings.Contains(armRendered.SQL, "CASE WHEN t.cp_m AND t.cp_bad THEN CAST('INVALID_CHOICE_ARM'") { + t.Fatalf("choice-arm raise must not live in the per-category CTE (unstable cross-CTE order):\n%s", armRendered.SQL) + } + if !strings.Contains(armRendered.SQL, "BOOL_OR(cp_bad) AS cp_bad") || + !strings.Contains(armRendered.SQL, "WHEN COALESCE(a0.cp_bad, false) THEN CAST('INVALID_CHOICE_ARM'") || + !strings.Contains(armRendered.SQL, "WHEN COALESCE(a1.cp_bad, false) THEN CAST('INVALID_CHOICE_ARM'") { + t.Fatalf("choice-arm guard must ride the outer chain per category:\n%s", armRendered.SQL) + } + type term struct { + frag string + label string + } + chain := []term{ + {"WHEN COALESCE(a0.cp_bad, false) THEN CAST('INVALID_CHOICE_ARM'", "cat0 choice-arm"}, + {"WHEN COALESCE(a0.cp_tcnt, 0) = 0 THEN CAST('TABLE_PIVOT_CELL_MISSING'", "cat0 missing"}, + {"a0.cp_mm, false) THEN CAST('TABLE_PIVOT_VALUE_TYPE_MISMATCH'", "cat0 value-type"}, + {"WHEN COALESCE(a1.cp_bad, false) THEN CAST('INVALID_CHOICE_ARM'", "cat1 choice-arm"}, + {"WHEN COALESCE(a1.cp_tcnt, 0) = 0 THEN CAST('TABLE_PIVOT_CELL_MISSING'", "cat1 missing"}, + {"a1.cp_mm, false) THEN CAST('TABLE_PIVOT_VALUE_TYPE_MISMATCH'", "cat1 value-type"}, + } + previous := -1 + for _, entry := range chain { + at := strings.Index(sumErrorChainSQL, entry.frag) + if at < 0 { + t.Fatalf("guard chain lost %s:\n%s", entry.label, sumErrorChainSQL) + } + if at < previous { + t.Fatalf("guard chain order drifted at %s (AQL lazy-LET order):\n%s", entry.label, sumErrorChainSQL) + } + previous = at + } +} + +func TestCodedPivot14IgnoredFieldsByteIdentical(t *testing.T) { + base := pivot14CompileCodedBundle(t, "p14_coded_ignored") + rendered, err := RenderPhysicalPlan(base, mustBinding(t)) + if err != nil { + t.Fatalf("baseline render: %v", err) + } + type variant struct { + name string + drift func(*ir.PhysicalCorrelation, map[string]any) + } + variants := []variant{ + {"value-scope", func(c *ir.PhysicalCorrelation, _ map[string]any) { c.ValueScope = fhirschema.CorrelatedValueScopeOwner }}, + {"selection-scope", func(c *ir.PhysicalCorrelation, _ map[string]any) { + c.SelectionScope = fhirschema.CorrelatedSelectionScopeExact + }}, + {"unit-selector", func(c *ir.PhysicalCorrelation, _ map[string]any) { + c.UnitSelector = &spec.Selector{Steps: []spec.SelectorStep{{Field: "valueQuantity"}, {Field: "unit"}}} + }}, + } + for _, test := range variants { + clone := clonePlanForPivot14(base) + test.drift(clone.StageSequence.Stages[0].GroupedPivot.CodedCorrelation, clone.BindVars) + renderedVariant, err := RenderPhysicalPlan(clone, mustBinding(t)) + if err != nil { + t.Fatalf("variant %s: ir or renderer rejected the ignored field: %v", test.name, err) + } + if renderedVariant.SQL != rendered.SQL { + t.Fatalf("ignored field %s changed the rendering (R2 violation — the ground truth must read byte-identically):\nbase: %s\nvariant: %s", test.name, rendered.SQL, renderedVariant.SQL) + } + } + + carrier := clonePlanForPivot14(base) + carrier.StageSequence.Stages[0].GroupedPivot.CodedCorrelation.VersionSelector = + &spec.Selector{Steps: []spec.SelectorStep{{Field: "code"}, {Field: "coding"}, {Field: "version"}}} + carrier.StageSequence.Stages[0].GroupedPivot.CodedCorrelation.VersionBindKey = "p14_version_bind" + carrier.BindVars["p14_version_bind"] = "3" + if _, err := RenderPhysicalPlan(carrier, mustBinding(t)); err == nil || + !strings.Contains(err.Error(), "version") { + t.Fatalf("ir must refuse the unreachable hand-carried version selector state: %v", err) + } + codingScope := clonePlanForPivot14(base) + codingScope.StageSequence.Stages[0].GroupedPivot.CodedCorrelation.ValueScope = fhirschema.CorrelatedValueScopeCoding + if _, err := RenderPhysicalPlan(codingScope, mustBinding(t)); err == nil || + !strings.Contains(err.Error(), "CODING correlation value scope does not support owner choice arms") { + t.Fatalf("ir must refuse CODING value scope on the pivot's owner choice arms: %v", err) + } +} + +func TestCodedPivot14RefusalRegister(t *testing.T) { + cases := []struct { + name string + mutate func(*ir.PhysicalPlan) + wantFragment string + }{ + {"OneInputRowPerGroup false", func(p *ir.PhysicalPlan) { + pivot14CodedPivot(t, *p).OneInputRowPerGroup = false + }, "OneInputRowPerGroup"}, + {"non-IGNORE unlisted policy", func(p *ir.PhysicalPlan) { + pivot14CodedPivot(t, *p).UnlistedCategoryPolicy = "ERROR" + }, "IGNORE"}, + {"row values (pg-sql-13 divergence)", func(p *ir.PhysicalPlan) { + pivot14CodedPivot(t, *p).RowValues = []ir.PhysicalStageRowValue{{InputColumn: "resource_id", Output: "rv", Variable: "rv0", Policy: "ALL", InputKind: "STRING"}} + }, "pg-sql-13"}, + {"extension URL selectors (C10)", func(p *ir.PhysicalPlan) { + pivot14CodedPivot(t, *p).CodedCorrelation.ExtensionURLSelectors = []spec.Selector{{Steps: []spec.SelectorStep{{Field: "url"}}}} + }, "extension-URL"}, + {"multi-step system selector", func(p *ir.PhysicalPlan) { + correlation := pivot14CodedPivot(t, *p).CodedCorrelation + correlation.SystemSelector.Steps = append([]spec.SelectorStep(nil), correlation.SystemSelector.Steps...) + correlation.SystemSelector.Steps = append(correlation.SystemSelector.Steps, spec.SelectorStep{Field: "extension"}) + }, "system selector must be one plain property step"}, + {"indexed code selector", func(p *ir.PhysicalPlan) { + correlation := pivot14CodedPivot(t, *p).CodedCorrelation + zero := 0 + correlation.CodeSelector.Steps = append([]spec.SelectorStep(nil), correlation.CodeSelector.Steps...) + correlation.CodeSelector.Steps[0].Index = &zero + }, "code selector must be one plain property step"}, + {"CONTAINS-filtered system selector", func(p *ir.PhysicalPlan) { + correlation := pivot14CodedPivot(t, *p).CodedCorrelation + correlation.SystemSelector.Filter = &spec.ContainsFilter{Field: "system", Needle: "loinc"} + }, "system selector must be one plain property step"}, + {"duplicate system/code pair", func(p *ir.PhysicalPlan) { + categories := pivot14CodedPivot(t, *p).CodedCategories + categories[1].CodeBindKey = categories[0].CodeBindKey + }, "duplicates a bound system/code pair"}, + {"blank category bind", func(p *ir.PhysicalPlan) { + categories := pivot14CodedPivot(t, *p).CodedCategories + p.BindVars[categories[0].CodeBindKey] = " " + }, "exact non-empty string"}, + {"group key not the hidden root _key", func(p *ir.PhysicalPlan) { + pivot14CodedPivot(t, *p).GroupKeys[0].Kind = "INTEGER" + }, "hidden root _key"}, + {"input projection pair drift", func(p *ir.PhysicalPlan) { + pivot := pivot14CodedPivot(t, *p) + pivot.InputProjections = pivot.InputProjections[:1] + }, "only the direct root key and payload"}, + {"correlation source drift", func(p *ir.PhysicalPlan) { + pivot14CodedPivot(t, *p).CodedCorrelation.Source.Path = []string{"other"} + }, "staged payload"}, + {"terminal Pivot window", func(p *ir.PhysicalPlan) { + p.StageSequence.PreviewTerminalPivotWindow = true + }, "terminal Pivot window"}, + {"plan-level preview window", func(p *ir.PhysicalPlan) { + p.PreviewSourceWindowByRootID = true + }, "plan-level preview source window"}, + {"active root-id window plus limit (R3)", func(p *ir.PhysicalPlan) { + p.StageSequence.PreviewSourceWindowByRootID = true + p.StageSequence.PreviewLimitBindKey = "p14_preview_limit" + p.BindVars["p14_preview_limit"] = 10 + }, "refused, never silently rendered"}, + {"non-positive preview limit", func(p *ir.PhysicalPlan) { + p.StageSequence.PreviewSourceWindowByRootID = false + p.StageSequence.PreviewLimitBindKey = "p14_preview_limit" + p.BindVars["p14_preview_limit"] = 0 + }, "positive int"}, + {"stage carries a second payload", func(p *ir.PhysicalPlan) { + p.StageSequence.Stages[0].Group = &ir.PhysicalStageGroup{} + }, "second payload"}, + {"stage not final", func(p *ir.PhysicalPlan) { + p.StageSequence.FinalStageID = "someone-else" + }, "must be final"}, + {"cell-trace return", func(p *ir.PhysicalPlan) { + p.StageSequence.CellTraceReturn = &ir.PhysicalCellTraceReturn{} + }, "pg-sql-23 through pg-sql-26"}, + {"source-region SET refused (C6)", func(p *ir.PhysicalPlan) { + p.Operations = append(p.Operations[:1], + ir.PhysicalOperation{Kind: ir.PhysicalSetOp, Set: &ir.PhysicalSet{Variable: "s"}}, + p.Operations[1]) + }, "SET is a pg-sql-16 capability gap"}, + {"source-region EXPRESSION_LET refused (C6)", func(p *ir.PhysicalPlan) { + p.Operations = append(p.Operations[:1], + ir.PhysicalOperation{Kind: ir.PhysicalExpressionLetOp, ExpressionLet: &ir.PhysicalExpressionLet{Variable: "e"}}, + p.Operations[1]) + }, "EXPRESSION_LET is a pg-sql-16 capability gap"}, + {"coexisting traversal refused", func(p *ir.PhysicalPlan) { + p.Operations = append(p.Operations[:1], + ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "t", Direction: ir.PhysicalOutbound, EdgeLabelBindKey: "edge"}}, + p.Operations[1]) + }, "not part of the registered coded-pivot source surface"}, + } + for index, test := range cases { + t.Run(test.name, func(t *testing.T) { + control := pivot14CompileCodedBundle(t, fmt.Sprintf("p14_coded_control_%02d", index)) + if _, err := RenderPhysicalPlan(control, mustBinding(t)); err != nil { + t.Fatalf("ALIVE control refused before any mutation: %v", err) + } + plan := pivot14CompileCodedBundle(t, fmt.Sprintf("p14_coded_refused_%02d", index)) + test.mutate(&plan) + pivot14CodedRefused(t, plan, test.name, test.wantFragment) + }) + } +} + +func pivot14SeedObservation(t *testing.T, svc *pgir02Service, logicalID string, payload any) { + t.Helper() + svc.insertResource(t, svc.schema, pivot14Project, pivot14Generation, "Observation", logicalID, "FHIR/Observation", payload) +} + +func pivot14Coding(system, code, version string) map[string]any { + coding := map[string]any{"system": system, "code": code} + if version != "" { + coding["version"] = version + } + return coding +} + +func pivot14Component(codings []map[string]any, value any, valueString *string) map[string]any { + component := map[string]any{ + "code": map[string]any{"coding": codings}, + } + if value != nil { + component["valueQuantity"] = map[string]any{"value": value} + } + if valueString != nil { + component["valueString"] = *valueString + } + return component +} + +func TestCodedPivot14LiveSystemSeparationAndVersionDuplication(t *testing.T) { + svc := pivot14LiveService(t) + hostileCode := `8462-4",["EVIL",1]` + pivot14SeedObservation(t, svc, "r1", map[string]any{ + "component": []map[string]any{pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "")}, 120, nil)}, + }) + pivot14SeedObservation(t, svc, "r2", map[string]any{ + "component": []map[string]any{pivot14Component([]map[string]any{pivot14Coding("http://snomed.info/sct", "8480-6", "")}, 999, nil)}, + }) + pivot14SeedObservation(t, svc, "r3", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "1")}, 10, nil), + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "2")}, 5, nil), + }, + }) + pivot14SeedObservation(t, svc, "r4", map[string]any{ + "component": []map[string]any{pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", hostileCode, "")}, 7, nil)}, + }) + rows := svc.witness(t, `SELECT logical_id FROM "`+svc.schema+`"."fhir_resource`+ + `" WHERE project = $1 AND dataset_generation = $2 AND logical_id = 'r3'`+ + ` AND payload @> '{"component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","version":"1"}]},"valueQuantity":{"value":10}}]}'::jsonb`+ + ` AND payload @> '{"component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6","version":"2"}]},"valueQuantity":{"value":5}}]}'::jsonb`, + pivot14Project, pivot14Generation) + if len(rows) != 1 { + t.Fatalf("version-varied fixture is vacuous (both version-bearing components must be stored): %v", rows) + } + + plan := pivot14CompileBundle(t, pivot14CodedBundle("p14_live_coded", []recipe.ConstructionCodedPivotCategory{ + {System: "http://loinc.org", Code: "8480-6", OutputColumnID: "systolic_id"}, + {System: "http://loinc.org", Code: hostileCode, OutputColumnID: "hostile_id"}, + }, recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, "", nil, false, nil, ""), []string{"FHIR/Observation"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render live coded plan: %v", err) + } + out := map[string][]string{} + for _, row := range svc.runRendered(t, rendered) { + out[row[0]] = row + } + systolic, hostile := -1, -1 + for index, name := range rendered.OutputColumns { + switch name { + case "systolic": + systolic = index + case "hostile": + hostile = index + } + } + if systolic < 0 || hostile < 0 { + t.Fatalf("category columns missing: %v", rendered.OutputColumns) + } + if got := out["r2"][systolic]; got != "" && got != "null" { + t.Fatalf("same code across systems must not match; r2 systolic = %q", got) + } + if got := out["r3"][systolic]; got != "15" { + t.Fatalf("version-varied codings must be duplicate observations (SUM 15), got %q — silent version filtering would return 10 or 5", got) + } + if got := out["r4"][hostile]; got != "7" { + t.Fatalf("hostile category pair must match byte-exactly, got %q", got) + } + if got := out["r1"][systolic]; got != "120" { + t.Fatalf("r1 systolic = %q, want 120", got) + } + + for _, forbidden := range []string{hostileCode, "EVIL"} { + if strings.Contains(rendered.SQL, forbidden) { + t.Fatalf("hostile category pair leaked into SQL text:\n%s", rendered.SQL) + } + } +} + +func TestCodedPivot14LiveInvalidChoiceArmRaise(t *testing.T) { + svc := pivot14LiveService(t) + pivot14SeedObservation(t, svc, "c1", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://snomed.info/sct", "8480-6", "")}, nil, strPtr("bad-arm-unmatched")), + }, + }) + plan := pivot14CompileBundle(t, pivot14CodedBundle("p14_live_arm_control", nil, + recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, "valueQuantity.value", nil, false, []string{"valueQuantity"}, "decimal"), + []string{"FHIR/Observation"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render control: %v", err) + } + if rows := svc.runRendered(t, rendered); len(rows) == 0 { + t.Fatal("control root must survive") + } + + pivot14SeedObservation(t, svc, "c2", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "")}, nil, strPtr("bad-arm-matched")), + }, + }) + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + conn, err := svc.iso.Acquire(svc.ctx) + if err != nil { + svc.fail("acquire pinned connection", err) + } + if _, err := conn.Exec(svc.ctx, "SET plan_cache_mode = "+mode); err != nil { + conn.Release() + svc.fail("pin plan_cache_mode "+mode, err) + } + if _, err := conn.Exec(svc.ctx, "EXPLAIN "+rendered.SQL, pivot14Args(rendered)...); err != nil { + conn.Release() + svc.fail("plan-time EXPLAIN raised under "+mode, err) + } + _, err = conn.Exec(svc.ctx, rendered.SQL, pivot14Args(rendered)...) + conn.Release() + if err == nil { + t.Fatalf("matched owner with an unsupported choice arm must raise under %s", mode) + } + if !strings.Contains(err.Error(), "INVALID_CHOICE_ARM") || !strings.Contains(err.Error(), "22P02") { + t.Fatalf("choice-arm raise drifted under %s: %v", mode, err) + } + } +} + +func TestCodedPivot14LiveValueFallbackAndTrim(t *testing.T) { + svc := pivot14LiveService(t) + primary := pivot14CodedBundle("p14_live_fallback", []recipe.ConstructionCodedPivotCategory{ + {System: "http://loinc.org", Code: "8480-6", OutputColumnID: "systolic_id"}, + }, recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, + "valueQuantity.value", nil, false, []string{"valueQuantity"}, "decimal") + plan := pivot14CompileBundle(t, primary, []string{"FHIR/Observation"}) + correlation := pivot14CodedPivot(t, plan).CodedCorrelation + correlation.ValueFallbacks = []spec.Selector{specSelector(t, "valueString")} + correlation.ChoiceArms = []string{"valueQuantity", "valueString"} + correlation.FallbackOnBlankString = true + systolicIndex := -1 + probeRendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("probe render: %v", err) + } + for index, name := range probeRendered.OutputColumns { + if name == "systolic" { + systolicIndex = index + } + } + if systolicIndex < 0 { + t.Fatalf("systolic column missing: %v", probeRendered.OutputColumns) + } + + pivot14SeedObservation(t, svc, "f1", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "")}, nil, nil), + }, + }) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render fallback plan: %v", err) + } + rows := svc.runRendered(t, rendered) + if len(rows) != 1 { + t.Fatalf("fallback rows = %v", rows) + } + if got := rows[0][systolicIndex]; got != "" && got != "null" { + t.Fatalf("owner with no candidates must read the null cell, got %q", got) + } + + pivot14SeedObservation(t, svc, "f2", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "")}, nil, strPtr("12.5")), + }, + }) + _, err = svc.iso.Exec(svc.ctx, rendered.SQL, pivot14Args(rendered)...) + if err == nil { + t.Fatal("string value under DECIMAL must raise the value-type guard") + } + if !strings.Contains(err.Error(), "TABLE_PIVOT_VALUE_TYPE_MISMATCH") { + t.Fatalf("value-type raise drifted: %v", err) + } + + blankSvc := pivot14LiveService(t) + pivot14SeedObservation(t, blankSvc, "b1", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "")}, nil, strPtr(" ")), + }, + }) + blankRendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, blankSvc)) + if err != nil { + t.Fatalf("render blank plan: %v", err) + } + blankRows := blankSvc.runRendered(t, blankRendered) + if len(blankRows) != 1 || (blankRows[0][systolicIndex] != "" && blankRows[0][systolicIndex] != "null") { + t.Fatalf("all-blank fallback must be ineligible and read null, got %v", blankRows) + } + if !strings.Contains(blankRendered.SQL, "btrim(") || + strings.Contains(blankRendered.SQL, "btrim(b.x #>> '{}', ' '") { + t.Fatalf("the trim must render with the ONE set as a bound parameter:\n%s", blankRendered.SQL) + } + carriedTrim := false + for _, parameter := range blankRendered.Args { + if text, ok := parameter.Value.(string); ok && text == " " { + carriedTrim = true + } + } + if !carriedTrim { + t.Fatalf("the ONE trim set must ride as a bound parameter:\n%v", blankRendered.Args) + } +} + +func TestCodedPivot14LiveReducerMissingTypedCount(t *testing.T) { + svc := pivot14LiveService(t) + pivot14SeedObservation(t, svc, "m2", json.RawMessage(`{"component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6"}]},"valueQuantity":{"value":null}}]}`)) + codedCategories := []recipe.ConstructionCodedPivotCategory{ + {System: "http://loinc.org", Code: "8480-6", OutputColumnID: "systolic_id"}, + } + plan := pivot14CompileBundle(t, pivot14CodedBundle("p14_live_missing_typed", codedCategories, + recipe.PivotDuplicateSum, recipe.PivotMissingCellError, "", nil, false, nil, ""), + []string{"FHIR/Observation"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render missing plan: %v", err) + } + _, err = svc.iso.Exec(svc.ctx, rendered.SQL, pivot14Args(rendered)...) + t.Logf("raise-phase err: %v\nSQL: %s", err, rendered.SQL) + if err == nil { + t.Fatalf("typed count 0 with raw count > 0 must raise the MISSING guard at the coded branch (:267):\n%s", rendered.SQL) + } + if !strings.Contains(err.Error(), "TABLE_PIVOT_CELL_MISSING") || !strings.Contains(err.Error(), "22P02") { + t.Fatalf("coded MISSING raise drifted: %v", err) + } + nullPlan := pivot14CompileBundle(t, pivot14CodedBundle("p14_live_missing_null", codedCategories, + recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, "", nil, false, nil, ""), + []string{"FHIR/Observation"}) + nullRendered, err := RenderPhysicalPlan(nullPlan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render null-missing plan: %v", err) + } + rows := svc.runRendered(t, nullRendered) + if len(rows) != 1 || rows[0][0] != "m2" { + t.Fatalf("the raw-count-bearing root must survive under NULL missing, got %v", rows) + } + if rows[0][1] != "" && rows[0][1] != "null" { + t.Fatalf("null-only candidate cells must read JSON null, got %v", rows) + } +} + +func TestCodedPivot14LiveFlattenChosenCandidate(t *testing.T) { + svc := pivot14LiveService(t) + pivot14SeedObservation(t, svc, "fl1", json.RawMessage( + `{"component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6"}]},`+ + `"valueQuantity":{"value":[7,3]}}]}`)) + + cardPlan := pivot14CompileBundle(t, pivot14CodedBundle("p14_live_flatten_card", + []recipe.ConstructionCodedPivotCategory{ + {System: "http://loinc.org", Code: "8480-6", OutputColumnID: "systolic_id"}, + }, recipe.PivotDuplicateError, recipe.PivotMissingCellNull, "", nil, false, nil, ""), + []string{"FHIR/Observation"}) + cardRendered, err := RenderPhysicalPlan(cardPlan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render flatten cardinality plan: %v", err) + } + if _, err := svc.iso.Exec(svc.ctx, cardRendered.SQL, pivot14Args(cardRendered)...); err == nil || + !strings.Contains(err.Error(), "TABLE_PIVOT_CELL_CARDINALITY") || !strings.Contains(err.Error(), "22P02") { + t.Fatalf("spliced array member must reach the cardinality guard like AQL: %v", err) + } + + sumBundle := pivot14CodedBundle("p14_live_flatten_sum", + []recipe.ConstructionCodedPivotCategory{ + {System: "http://loinc.org", Code: "8480-6", OutputColumnID: "systolic_id"}, + }, recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, "", nil, false, nil, "") + sumPlan := pivot14CompileBundle(t, sumBundle, []string{"FHIR/Observation"}) + probeRendered, err := RenderPhysicalPlan(sumPlan, mustBinding(t)) + if err != nil { + t.Fatalf("probe flatten SUM plan: %v", err) + } + systolicIndex := -1 + for index, name := range probeRendered.OutputColumns { + if name == "systolic" { + systolicIndex = index + } + } + if systolicIndex < 0 { + t.Fatalf("systolic column missing: %v", probeRendered.OutputColumns) + } + sumRendered, err := RenderPhysicalPlan(sumPlan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render flatten SUM plan: %v", err) + } + rows := svc.runRendered(t, sumRendered) + if len(rows) != 1 || rows[0][systolicIndex] != "10" { + t.Fatalf("array member must FLATTEN into the reducer (SUM 7+3=10), got %v", rows) + } +} + +func TestCodedPivot14LiveFallbackOrdinalPrecedence(t *testing.T) { + svc := pivot14LiveService(t) + bundle := pivot14CodedBundle("p14_live_fallback_precedence", + []recipe.ConstructionCodedPivotCategory{ + {System: "http://loinc.org", Code: "8480-6", OutputColumnID: "systolic_id"}, + }, recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, "valueQuantity.value", nil, false, []string{"valueQuantity"}, "decimal") + plan := pivot14CompileBundle(t, bundle, []string{"FHIR/Observation"}) + correlation := pivot14CodedPivot(t, plan).CodedCorrelation + correlation.ValueFallbacks = []spec.Selector{specSelector(t, "valueString")} + correlation.ChoiceArms = []string{"valueQuantity", "valueString"} + pivot14SeedObservation(t, svc, "fp1", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "")}, 120, strPtr("999")), + }, + }) + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("probe precedence plan: %v", err) + } + systolicIndex := -1 + for index, name := range rendered.OutputColumns { + if name == "systolic" { + systolicIndex = index + } + } + if systolicIndex < 0 { + t.Fatalf("systolic column missing: %v", rendered.OutputColumns) + } + liveRendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render precedence plan: %v", err) + } + rows := svc.runRendered(t, liveRendered) + if len(rows) != 1 { + t.Fatalf("precedence rows = %v", rows) + } + if rows[0][systolicIndex] != "120" { + t.Fatalf("declared-order FIRST must pick the primary value 120, got %q", rows[0][systolicIndex]) + } +} + +func TestCodedPivot14LiveCompetingGuardsCategoryOrder(t *testing.T) { + svc := pivot14LiveService(t) + pivot14SeedObservation(t, svc, "g0", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "")}, nil, nil), + }, + }) + pivot14SeedObservation(t, svc, "g1", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8462-4", "")}, nil, strPtr("bad-arm")), + }, + }) + plan := pivot14CompileBundle(t, pivot14CodedBundle("p14_live_guard_order", nil, + recipe.PivotDuplicateError, recipe.PivotMissingCellError, "", nil, false, nil, ""), + []string{"FHIR/Observation"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render competing-guards plan: %v", err) + } + if _, err := svc.iso.Exec(svc.ctx, rendered.SQL, pivot14Args(rendered)...); err == nil || + !strings.Contains(err.Error(), "TABLE_PIVOT_CELL_MISSING") || + strings.Contains(err.Error(), "INVALID_CHOICE_ARM") { + t.Fatalf("the declared-category-order chain must pick TABLE_PIVOT_CELL_MISSING over the later category's choice arm: %v", err) + } +} + +func strPtr(value string) *string { return &value } + +func TestPivot14ProductionCodedLive(t *testing.T) { + svc := pivot14LiveService(t) + pivot14SeedObservation(t, svc, "q1", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "")}, 120, nil), + }, + }) + pivot14SeedObservation(t, svc, "q2", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://snomed.info/sct", "8480-6", "")}, 999, nil), + pivot14Component([]map[string]any{pivot14Coding("http://snomed.info/sct", "8462-4", "")}, 998, nil), + }, + }) + pivot14SeedObservation(t, svc, "q3", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "1")}, 10, nil), + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "2")}, 5, nil), + }, + }) + pivot14SeedObservation(t, svc, "q4", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8462-4", "")}, 3, nil), + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8462-4", "")}, 4, nil), + }, + }) + pivot14SeedObservation(t, svc, "q5", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://example.org", "9999-9", "")}, 1, nil), + }, + }) + bundle := pivot14CodedBundle("p14_prod_coded", nil, + recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, "", nil, false, nil, "") + plan := pivot14CompileBundle(t, bundle, []string{"FHIR/Observation"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render production coded plan: %v", err) + } + if strings.Contains(rendered.SQL, "8480-6") || strings.Contains(rendered.SQL, "http://loinc.org") { + t.Fatalf("category literals leaked into SQL text:\n%s", rendered.SQL) + } + carried := 0 + for _, parameter := range rendered.Args { + if text := fmt.Sprintf("%s", parameter.Value); strings.Contains(text, "8480-6") || strings.Contains(text, "8462-4") { + carried++ + } + } + if carried < 2 { + t.Fatalf("category pairs must ride parameters: %v", rendered.Args) + } + keyIndex := pivot14ColIndex(t, rendered, "_key") + systolic := pivot14ColIndex(t, rendered, "systolic") + diastolic := pivot14ColIndex(t, rendered, "diastolic") + identityIndex := pivot14ColIndex(t, rendered, "__loom_row_id") + rows := svc.runRendered(t, rendered) + byRoot := map[string][]string{} + for _, row := range rows { + byRoot[row[keyIndex]] = row + } + if len(byRoot) != 5 { + t.Fatalf("production coded roots = %v, want all five", rows) + } + expectations := []struct { + root string + systolic, diastolic string + }{ + {"q1", "120", ""}, + {"q2", "", ""}, + {"q3", "15", ""}, + {"q4", "", "7"}, + {"q5", "", ""}, + } + for _, want := range expectations { + row, ok := byRoot[want.root] + if !ok { + t.Fatalf("root %s missing: %v", want.root, rows) + } + if row[systolic] != want.systolic && !(want.systolic == "" && pivot14CellIsNull(row[systolic])) { + t.Fatalf("%s systolic = %q, want %s", want.root, row[systolic], want.systolic) + } + if row[diastolic] != want.diastolic && !(want.diastolic == "" && pivot14CellIsNull(row[diastolic])) { + t.Fatalf("%s diastolic = %q, want %s", want.root, row[diastolic], want.diastolic) + } + wantIdentity := `["GROUPED_PIVOT","p14_prod_coded",["STRING","` + want.root + `"]]` + if got := row[identityIndex]; got != wantIdentity { + t.Fatalf("%s identity = %q, want %s", want.root, got, wantIdentity) + } + } +} + +func TestCodedPivot14SameCodeDistinctSystemAliveControl(t *testing.T) { + svc := pivot14LiveService(t) + pivot14SeedObservation(t, svc, "sd1", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "")}, 120, nil), + }, + }) + pivot14SeedObservation(t, svc, "sd2", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://snomed.info/sct", "8480-6", "")}, 999, nil), + pivot14Component([]map[string]any{pivot14Coding("http://snomed.info/sct", "8462-4", "")}, 998, nil), + }, + }) + plan := pivot14CompileBundle(t, pivot14CodedBundle("p14_sd_system", nil, + recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, "", nil, false, nil, ""), + []string{"FHIR/Observation"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render system-separation plan: %v", err) + } + rows := svc.runRendered(t, rendered) + byRoot := map[string][]string{} + keyIndex := pivot14ColIndex(t, rendered, "_key") + for _, row := range rows { + byRoot[row[keyIndex]] = row + } + if len(byRoot) != 2 { + t.Fatalf("system-separation roots = %v", rows) + } + if got := byRoot["sd2"][pivot14ColIndex(t, rendered, "systolic")]; !pivot14CellIsNull(got) { + t.Fatalf("foreign-system 8480-6 must never match (got %q — a system-drop would read 999)", got) + } + if got := byRoot["sd2"][pivot14ColIndex(t, rendered, "diastolic")]; !pivot14CellIsNull(got) { + t.Fatalf("foreign-system 8462-4 must never match (got %q — a code-only match would read 998)", got) + } + if got := byRoot["sd1"][pivot14ColIndex(t, rendered, "systolic")]; got != "120" { + t.Fatalf("loinc 8480-6 must match (got %q)", got) + } +} + +func TestCodedPivot14DuplicateObservationAliveControl(t *testing.T) { + bundleSum := pivot14CodedBundle("p14_dup_sum", + []recipe.ConstructionCodedPivotCategory{ + {System: "http://loinc.org", Code: "8480-6", OutputColumnID: "systolic_id"}, + }, recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, "", nil, false, nil, "") + dupPayload := map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "1")}, 7, nil), + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "2")}, 7, nil), + }, + } + sumSvc := pivot14LiveService(t) + pivot14SeedObservation(t, sumSvc, "dup1", dupPayload) + sumPlan := pivot14CompileBundle(t, bundleSum, []string{"FHIR/Observation"}) + sumRendered, err := RenderPhysicalPlan(sumPlan, pivot14Binding(t, sumSvc)) + if err != nil { + t.Fatalf("render duplicate-observation SUM plan: %v", err) + } + sumRows := sumSvc.runRendered(t, sumRendered) + if len(sumRows) != 1 { + t.Fatalf("duplicate-observation SUM rows = %v", sumRows) + } + if got := sumRows[0][pivot14ColIndex(t, sumRendered, "systolic")]; got != "14" { + t.Fatalf("duplicate observations must double-count (SUM 7+7=14), got %q (DISTINCT collapse reads 7)", got) + } + + errSvc := pivot14LiveService(t) + pivot14SeedObservation(t, errSvc, "dup1", dupPayload) + bundleError := pivot14CodedBundle("p14_dup_card", + []recipe.ConstructionCodedPivotCategory{ + {System: "http://loinc.org", Code: "8480-6", OutputColumnID: "systolic_id"}, + }, recipe.PivotDuplicateError, recipe.PivotMissingCellNull, "", nil, false, nil, "") + cardPlan := pivot14CompileBundle(t, bundleError, []string{"FHIR/Observation"}) + cardRendered, err := RenderPhysicalPlan(cardPlan, pivot14Binding(t, errSvc)) + if err != nil { + t.Fatalf("render duplicate-observation cardinality plan: %v", err) + } + pivot14ExpectGuardRaise(t, errSvc, cardRendered, pivotCardinalityTag) +} + +func TestCodedPivot14OwnerZeroItemAliveControl(t *testing.T) { + mixed := map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "")}, nil, nil), + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "")}, 5, nil), + }, + } + zeroRoot := map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://example.org", "9999-9", "")}, 1, nil), + }, + } + single := []recipe.ConstructionCodedPivotCategory{ + {System: "http://loinc.org", Code: "8480-6", OutputColumnID: "systolic_id"}, + } + + zeroSvc := pivot14LiveService(t) + pivot14SeedObservation(t, zeroSvc, "z1", mixed) + zeroPlan := pivot14CompileBundle(t, pivot14CodedBundle("p14_zero_item", single, + recipe.PivotDuplicateError, recipe.PivotMissingCellNull, "", nil, false, nil, ""), + []string{"FHIR/Observation"}) + zeroRendered, err := RenderPhysicalPlan(zeroPlan, pivot14Binding(t, zeroSvc)) + if err != nil { + t.Fatalf("render zero-item plan: %v", err) + } + zeroRows := zeroSvc.runRendered(t, zeroRendered) + if len(zeroRows) != 1 || zeroRows[0][pivot14ColIndex(t, zeroRendered, "systolic")] != "5" { + t.Fatalf("the empty-candidate owner must contribute nothing (cell 5, no cardinality raise), got %v", zeroRows) + } + + nullSvc := pivot14LiveService(t) + pivot14SeedObservation(t, nullSvc, "z2", zeroRoot) + nullPlan := pivot14CompileBundle(t, pivot14CodedBundle("p14_zero_null", single, + recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, "", nil, false, nil, ""), + []string{"FHIR/Observation"}) + nullRendered, err := RenderPhysicalPlan(nullPlan, pivot14Binding(t, nullSvc)) + if err != nil { + t.Fatalf("render zero-coding NULL plan: %v", err) + } + nullRows := nullSvc.runRendered(t, nullRendered) + if len(nullRows) != 1 || !pivot14CellIsNull(nullRows[0][pivot14ColIndex(t, nullRendered, "systolic")]) { + t.Fatalf("zero-coding root must read the NULL-policy cell, got %v", nullRows) + } + + raiseSvc := pivot14LiveService(t) + pivot14SeedObservation(t, raiseSvc, "z2", zeroRoot) + raisePlan := pivot14CompileBundle(t, pivot14CodedBundle("p14_zero_raise", single, + recipe.PivotDuplicateSum, recipe.PivotMissingCellError, "", nil, false, nil, ""), + []string{"FHIR/Observation"}) + raiseRendered, err := RenderPhysicalPlan(raisePlan, pivot14Binding(t, raiseSvc)) + if err != nil { + t.Fatalf("render zero-coding ERROR plan: %v", err) + } + pivot14ExpectGuardRaise(t, raiseSvc, raiseRendered, pivotCellMissingTag) +} + +func TestCodedPivot14FallbackBlankAliveControl(t *testing.T) { + blankBundle := pivot14CodedBundle("p14_fb_blank", + []recipe.ConstructionCodedPivotCategory{ + {System: "http://loinc.org", Code: "8480-6", OutputColumnID: "systolic_id"}, + }, recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, "valueQuantity.value", nil, false, []string{"valueQuantity"}, "decimal") + blankPlan := pivot14CompileBundle(t, blankBundle, []string{"FHIR/Observation"}) + correlation := pivot14CodedPivot(t, blankPlan).CodedCorrelation + correlation.ValueFallbacks = []spec.Selector{specSelector(t, "valueString")} + correlation.ChoiceArms = []string{"valueQuantity", "valueString"} + correlation.FallbackOnBlankString = true + + chosenSvc := pivot14LiveService(t) + pivot14SeedObservation(t, chosenSvc, "fb1", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "")}, nil, strPtr("8.5")), + }, + }) + chosenRendered, err := RenderPhysicalPlan(blankPlan, pivot14Binding(t, chosenSvc)) + if err != nil { + t.Fatalf("render fallback-chosen plan: %v", err) + } + pivot14ExpectGuardRaise(t, chosenSvc, chosenRendered, pivotValueTypeTag) + + nullSvc := pivot14LiveService(t) + pivot14SeedObservation(t, nullSvc, "fb2", map[string]any{ + "component": []map[string]any{ + pivot14Component([]map[string]any{pivot14Coding("http://loinc.org", "8480-6", "")}, nil, strPtr(" ")), + }, + }) + nullRendered, err := RenderPhysicalPlan(blankPlan, pivot14Binding(t, nullSvc)) + if err != nil { + t.Fatalf("render fallback-blank plan: %v", err) + } + rows := nullSvc.runRendered(t, nullRendered) + if len(rows) != 1 || !pivot14CellIsNull(rows[0][pivot14ColIndex(t, nullRendered, "systolic")]) { + t.Fatalf("the all-blank fallback must be ineligible and read null without a raise, got %v", rows) + } +} + +func TestCodedPivot14TypedCountMissingRaiseControl(t *testing.T) { + single := []recipe.ConstructionCodedPivotCategory{ + {System: "http://loinc.org", Code: "8480-6", OutputColumnID: "systolic_id"}, + } + nullSplice := json.RawMessage(`{"component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6"}]},"valueQuantity":{"value":[null]}}]}`) + + raiseSvc := pivot14LiveService(t) + pivot14SeedObservation(t, raiseSvc, "tc1", nullSplice) + raisePlan := pivot14CompileBundle(t, pivot14CodedBundle("p14_typed_raise", single, + recipe.PivotDuplicateSum, recipe.PivotMissingCellError, "", nil, false, nil, ""), + []string{"FHIR/Observation"}) + raiseRendered, err := RenderPhysicalPlan(raisePlan, pivot14Binding(t, raiseSvc)) + if err != nil { + t.Fatalf("render typed-count raise plan: %v", err) + } + pivot14ExpectGuardRaise(t, raiseSvc, raiseRendered, pivotCellMissingTag) + + nullSvc := pivot14LiveService(t) + pivot14SeedObservation(t, nullSvc, "tc1", nullSplice) + nullPlan := pivot14CompileBundle(t, pivot14CodedBundle("p14_typed_null", single, + recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, "", nil, false, nil, ""), + []string{"FHIR/Observation"}) + nullRendered, err := RenderPhysicalPlan(nullPlan, pivot14Binding(t, nullSvc)) + if err != nil { + t.Fatalf("render typed-count NULL plan: %v", err) + } + rows := nullSvc.runRendered(t, nullRendered) + if len(rows) != 1 || !pivot14CellIsNull(rows[0][pivot14ColIndex(t, nullRendered, "systolic")]) { + t.Fatalf("typed-count 0 under NULL policy must read the null cell (raw count 1 must not raise), got %v", rows) + } +} + +func TestCodedPivot14LiveValueTypeRaise(t *testing.T) { + for _, value := range []string{"abc", "12.5"} { + t.Run("value_"+value, func(t *testing.T) { + svc := pivot14LiveService(t) + pivot14SeedObservation(t, svc, "cv1", json.RawMessage( + `{"component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6"}]},"valueQuantity":{"value":"`+value+`"}}]}`)) + plan := pivot14CompileBundle(t, pivot14CodedBundle("p14_coded_valuetype", + []recipe.ConstructionCodedPivotCategory{ + {System: "http://loinc.org", Code: "8480-6", OutputColumnID: "systolic_id"}, + }, recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, "", nil, false, nil, ""), + []string{"FHIR/Observation"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render coded value-type plan: %v", err) + } + pivot14ExpectGuardRaise(t, svc, rendered, pivotValueTypeTag) + }) + } + + cleanSvc := pivot14LiveService(t) + pivot14SeedObservation(t, cleanSvc, "cv0", json.RawMessage( + `{"component":[{"code":{"coding":[{"system":"http://loinc.org","code":"8480-6"}]},"valueQuantity":{"value":6}}]}`)) + cleanPlan := pivot14CompileBundle(t, pivot14CodedBundle("p14_coded_clean", + []recipe.ConstructionCodedPivotCategory{ + {System: "http://loinc.org", Code: "8480-6", OutputColumnID: "systolic_id"}, + }, recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, "", nil, false, nil, ""), + []string{"FHIR/Observation"}) + cleanRendered, err := RenderPhysicalPlan(cleanPlan, pivot14Binding(t, cleanSvc)) + if err != nil { + t.Fatalf("render coded clean twin: %v", err) + } + rows := cleanSvc.runRendered(t, cleanRendered) + if len(rows) != 1 || rows[0][pivot14ColIndex(t, cleanRendered, "systolic")] != "6" { + t.Fatalf("clean coded data must read SUM 6 without a raise, got %v", rows) + } +} + +func TestCodedPivotIRRowValuesAliveControl(t *testing.T) { + base := pivot14CompileCodedBundle(t, "p14_ir_rowvalues") + clone := clonePlanForPivot14(base) + pivot14CodedPivot(t, clone).RowValues = []ir.PhysicalStageRowValue{ + {InputColumn: "resource_id", Output: "rv", Variable: "rv0", Policy: "ALL", InputKind: "STRING"}, + } + pivot14CodedRefused(t, clone, "RowValues (ir-accepted, PG-refused) names pg-sql-13", "pg-sql-13") +} + +func TestCodedPivotIRSetAliveControl(t *testing.T) { + coded := pivot14CompileCodedBundle(t, "p14_ir_set") + clone := clonePlanForPivot14(coded) + operations := make([]ir.PhysicalOperation, 0, len(clone.Operations)+1) + operations = append(operations, clone.Operations[0]) + operations = append(operations, ir.PhysicalOperation{Kind: ir.PhysicalSetOp, Set: &ir.PhysicalSet{Variable: "s"}}) + operations = append(operations, clone.Operations[1:]...) + clone.Operations = operations + pivot14CodedRefused(t, clone, "source-region SET (ir-accepted, PG-refused)", "SET is a pg-sql-16 capability gap") + + scalar := pivot14ScalarPlan(t, pivot14ScalarConfig{}) + scalarOps := make([]ir.PhysicalOperation, 0, len(scalar.Operations)+1) + scalarOps = append(scalarOps, scalar.Operations[0]) + scalarOps = append(scalarOps, ir.PhysicalOperation{Kind: ir.PhysicalSetOp, Set: &ir.PhysicalSet{Variable: "s"}}) + scalarOps = append(scalarOps, scalar.Operations[1:]...) + scalar.Operations = scalarOps + pivot14Refused(t, scalar, "scalar-door SET coexistence", "cannot coexist with GROUPED_PIVOT") +} + +func TestCodedPivotIRExpressionLetAliveControl(t *testing.T) { + coded := pivot14CompileCodedBundle(t, "p14_ir_let") + clone := clonePlanForPivot14(coded) + operations := make([]ir.PhysicalOperation, 0, len(clone.Operations)+1) + operations = append(operations, clone.Operations[0]) + operations = append(operations, ir.PhysicalOperation{Kind: ir.PhysicalExpressionLetOp, + ExpressionLet: &ir.PhysicalExpressionLet{Variable: "e"}}) + operations = append(operations, clone.Operations[1:]...) + clone.Operations = operations + pivot14CodedRefused(t, clone, "source-region EXPRESSION_LET (ir-accepted, PG-refused)", "EXPRESSION_LET is a pg-sql-16 capability gap") + + scalar := pivot14ScalarPlan(t, pivot14ScalarConfig{}) + scalarOps := make([]ir.PhysicalOperation, 0, len(scalar.Operations)+1) + scalarOps = append(scalarOps, scalar.Operations[0]) + scalarOps = append(scalarOps, ir.PhysicalOperation{Kind: ir.PhysicalExpressionLetOp, + ExpressionLet: &ir.PhysicalExpressionLet{Variable: "e"}}) + scalarOps = append(scalarOps, scalar.Operations[1:]...) + scalar.Operations = scalarOps + pivot14Refused(t, scalar, "scalar-door EXPRESSION_LET coexistence", "cannot coexist with GROUPED_PIVOT") +} diff --git a/internal/dataframe/compiler/render/postgres/graph.go b/internal/dataframe/compiler/render/postgres/graph.go new file mode 100644 index 000000000..ef3939636 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/graph.go @@ -0,0 +1,497 @@ +package postgres + +import ( + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const graphOutputColumn = "paths" + +func RenderGraphPhysicalPlan(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + return renderGraphPhysicalPlan(plan, storage) +} + +func graphPlanMentionsPathOperations(plan ir.PhysicalPlan) bool { + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalPathSeedOp || operation.Kind == ir.PhysicalPathExtendOp || operation.Kind == ir.PhysicalGraphReturnOp { + return true + } + } + return false +} + +func renderGraphPhysicalPlan(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + if err := storage.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := validateGraphPlanSurface(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := plan.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("validate physical plan: %w", err) + } + if plan.Engine != "" && plan.Engine != ir.PhysicalEngineAQL { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: engine %q has no PostgreSQL source; the source graph IR is backend-neutral and ClickHouse plans are not PostgreSQL-rendered", + ErrUnsupportedPostgresPlan, string(plan.Engine)) + } + if plan.ClickHouseCombine != nil || plan.ClickHousePrefix != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: ClickHouse combine/prefix payloads have no PostgreSQL rendering", ErrUnsupportedPostgresPlan) + } + if plan.StageSequence != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: staged row-operator sequences are not yet implemented for PostgreSQL", ErrUnsupportedPostgresPlan) + } + if plan.PreviewSourceWindowByRootID { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: preview source windowing is not yet implemented for PostgreSQL", ErrUnsupportedPostgresPlan) + } + if len(plan.DeferredExpressionLets) != 0 { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: completed plans must not carry deferred expression lets", ErrUnsupportedPostgresPlan) + } + renderer := &planRenderer{ + storage: storage, + rows: map[string]rowScope{}, + parameters: map[string]int{}, + canonicalAuthLets: map[string]string{}, + } + if err := renderer.loadScope(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + sql, outputColumns, err := renderer.renderGraphStatement(plan) + if err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return command.NewRenderedPostgresPhysicalPlan(sql, renderer.args, outputColumns) +} + +func validateGraphPlanSurface(plan ir.PhysicalPlan) error { + if len(plan.Operations) < 3 { + return fmt.Errorf("%w: graph plans require at least ROOT_SCAN, PATH_SEED, and a final GRAPH_RETURN", ErrUnsupportedPostgresPlan) + } + first := plan.Operations[0] + if first.Kind != ir.PhysicalRootScanOp || first.RootScan == nil { + return fmt.Errorf("%w: graph plans require one leading ROOT_SCAN payload (lower/graph_plan.go:51-55)", ErrUnsupportedPostgresPlan) + } + last := plan.Operations[len(plan.Operations)-1] + if last.Kind != ir.PhysicalGraphReturnOp || last.GraphReturn == nil { + return fmt.Errorf("%w: graph plans require GRAPH_RETURN as the final operation (lower/graph_plan.go:139)", ErrUnsupportedPostgresPlan) + } + seedIndex := -1 + definedSets := map[string]bool{} + var lastRouteOrder int + var haveRouteOrder bool + for index := 1; index < len(plan.Operations)-1; index++ { + operation := plan.Operations[index] + switch operation.Kind { + case ir.PhysicalFilterOp: + if seedIndex != -1 { + return fmt.Errorf("%w: graph plan operation %d: a FILTER after PATH_SEED must live inside a PATH_EXTEND Scope (the graph plan is flat; lower/graph_plan.go:113-119)", + ErrUnsupportedPostgresPlan, index) + } + if operation.Filter == nil { + return fmt.Errorf("%w: graph plan operation %d: filter payload is missing", ErrUnsupportedPostgresPlan, index) + } + case ir.PhysicalDerivedLetOp: + if seedIndex != -1 { + return fmt.Errorf("%w: graph plan operation %d: a DERIVED_LET after PATH_SEED must live inside a PATH_EXTEND Scope (the graph plan is flat)", + ErrUnsupportedPostgresPlan, index) + } + if operation.DerivedLet == nil { + return fmt.Errorf("%w: graph plan operation %d: derived let payload is missing", ErrUnsupportedPostgresPlan, index) + } + case ir.PhysicalPathSeedOp: + if seedIndex != -1 { + return fmt.Errorf("%w: graph plan operation %d: a second PATH_SEED is outside the closed graph slice", ErrUnsupportedPostgresPlan, index) + } + seed := operation.PathSeed + if seed == nil { + return fmt.Errorf("%w: graph plan operation %d: path seed payload is missing", ErrUnsupportedPostgresPlan, index) + } + if seed.RouteOrder != 0 { + return fmt.Errorf("%w: graph plan operation %d: seed route order %d is not 0 (lower/graph_plan.go:70 always emits 0; the PathSets ascending check treats the seed set as its zero anchor, so a divergence must fail closed rather than let the check and the bound value drift)", + ErrUnsupportedPostgresPlan, index, seed.RouteOrder) + } + if seed.Variable == "" { + return fmt.Errorf("%w: graph plan operation %d: path seed must name its set variable", ErrUnsupportedPostgresPlan, index) + } + if value := seed.Node.Value; value.Variable != first.RootScan.Variable || len(value.Path) != 0 || value.BindKey != "" { + return fmt.Errorf("%w: graph plan operation %d: the seed node must read the root row itself (lower/graph_plan.go:70-72; the AQL seed renders the one root value, render/aql/graph_render.go:80-88)", + ErrUnsupportedPostgresPlan, index) + } + seedIndex = index + definedSets[seed.Variable] = true + case ir.PhysicalPathExtendOp: + if seedIndex == -1 { + return fmt.Errorf("%w: graph plan operation %d: PATH_EXTEND before PATH_SEED is outside the closed graph slice", ErrUnsupportedPostgresPlan, index) + } + extend := operation.PathExtend + if extend == nil { + return fmt.Errorf("%w: graph plan operation %d: path extend payload is missing", ErrUnsupportedPostgresPlan, index) + } + if extend.Variable == "" { + return fmt.Errorf("%w: graph plan operation %d: path extend must name its set variable", ErrUnsupportedPostgresPlan, index) + } + if !definedSets[extend.SourceVariable] { + return fmt.Errorf("%w: graph plan operation %d: source path set %q is not a set defined by this plan", + ErrUnsupportedPostgresPlan, index, extend.SourceVariable) + } + if extend.Traversal.Direction != ir.PhysicalInbound && + extend.Traversal.Direction != ir.PhysicalOutbound { + return fmt.Errorf("%w: graph plan operation %d: %s-direction traversal has no proven PostgreSQL contract (the spine registers INBOUND/OUTBOUND only, lower/traversal.go; ANY is a registered capability gap)", + ErrUnsupportedPostgresPlan, index, string(extend.Traversal.Direction)) + } + if extend.Relationship.LabelBindKey != extend.Traversal.EdgeLabelBindKey { + return fmt.Errorf("%w: graph plan operation %d: relationship label bind %q does not equal the traversal edge label bind %q (lower/graph_plan.go:126 always sets one key for both; the AQL twin filters edges on EdgeLabelBindKey, render/aql/graph_render.go:119/:128, while rendering the public label and dedupe identity from Relationship.LabelBindKey, :144-145 — the renderer reads one bind, so a divergence must fail closed, mirroring the node-type/target-type pin)", + ErrUnsupportedPostgresPlan, index, extend.Relationship.LabelBindKey, extend.Traversal.EdgeLabelBindKey) + } + if len(extend.SourcePath) != 0 { + return fmt.Errorf("%w: graph plan operation %d: a non-empty SourcePath has no registered PostgreSQL spelling (the planner always emits nil, lower/graph_plan.go:124; the AQL nested-terminal read render/aql/graph_render.go:101-107 stays a typed gap)", + ErrUnsupportedPostgresPlan, index) + } + if extend.Traversal.Strategy == ir.PhysicalTraversalEndpointLookup && + (strings.TrimSpace(extend.Traversal.EndpointField) == "" || strings.TrimSpace(extend.Traversal.EndpointJoinField) == "") { + return fmt.Errorf("%w: graph plan operation %d: endpoint-lookup traversal payload is incomplete (the planner always names EndpointField and EndpointJoinField, lower/traversal.go:69-70)", + ErrUnsupportedPostgresPlan, index) + } + if value := extend.Node.Value; value.Variable != extend.Traversal.TargetVariable || len(value.Path) != 0 || value.BindKey != "" { + return fmt.Errorf("%w: graph plan operation %d: the extend node must read the reached target row itself (lower/graph_plan.go:125)", + ErrUnsupportedPostgresPlan, index) + } + blockEnd := 0 + if err := matchCanonicalScopeBlock(extend.Scope, &blockEnd, []string{extend.Traversal.EdgeVariable, extend.Traversal.TargetVariable}); err != nil { + return fmt.Errorf("%w: graph plan operation %d extend scope %d: canonical hop scope block: %s", + ErrUnsupportedPostgresPlan, index, blockEnd, err.Error()) + } + for scopeIndex := blockEnd; scopeIndex < len(extend.Scope); scopeIndex++ { + scopeOperation := extend.Scope[scopeIndex] + switch scopeOperation.Kind { + case ir.PhysicalFilterOp: + if scopeOperation.Filter == nil { + return fmt.Errorf("%w: graph plan operation %d extend scope %d: filter payload is missing", ErrUnsupportedPostgresPlan, index, scopeIndex) + } + if err := validateGraphExtendFilter(index, scopeIndex, extend, scopeOperation.Filter); err != nil { + return err + } + case ir.PhysicalDerivedLetOp: + if scopeOperation.DerivedLet == nil || scopeOperation.DerivedLet.Operator != canonicalAuthLetOperator { + return fmt.Errorf("%w: graph plan operation %d extend scope %d: only the canonical AUTH_RESOURCE_PATH_ALLOWED LET is registered", + ErrUnsupportedPostgresPlan, index, scopeIndex) + } + default: + return fmt.Errorf("%w: graph plan operation %d extend scope %d: operation %q is outside the closed graph slice", + ErrUnsupportedPostgresPlan, index, scopeIndex, string(scopeOperation.Kind)) + } + } + definedSets[extend.Variable] = true + if haveRouteOrder && extend.RouteOrder <= lastRouteOrder { + return fmt.Errorf("%w: graph plan operation %d: route order %d does not ascend after %d (walk/graph_plan.go:85-90 always emits DFS-ascending orders; the PathSets flatten order depends on it)", + ErrUnsupportedPostgresPlan, index, extend.RouteOrder, lastRouteOrder) + } + lastRouteOrder, haveRouteOrder = extend.RouteOrder, true + default: + return fmt.Errorf("%w: graph plan operation %d (%q) is not part of the closed graph slice", + ErrUnsupportedPostgresPlan, index, string(operation.Kind)) + } + } + if seedIndex == -1 { + return fmt.Errorf("%w: graph plans require exactly one PATH_SEED (lower/graph_plan.go:67-74)", ErrUnsupportedPostgresPlan) + } + seenSets := map[string]bool{} + previousRouteOrder := 0 + for setIndex, setName := range last.GraphReturn.PathSets { + if !definedSets[setName] { + return fmt.Errorf("%w: graph return path set %q is not a set defined by this plan", ErrUnsupportedPostgresPlan, setName) + } + if seenSets[setName] { + return fmt.Errorf("%w: graph return path set %q is listed twice", ErrUnsupportedPostgresPlan, setName) + } + seenSets[setName] = true + routeOrder := 0 + if setName != plan.Operations[seedIndex].PathSeed.Variable { + routeOrder = -1 + for index := seedIndex + 1; index < len(plan.Operations)-1; index++ { + if extend := plan.Operations[index].PathExtend; extend != nil && extend.Variable == setName { + routeOrder = extend.RouteOrder + break + } + } + if routeOrder < 0 { + return fmt.Errorf("%w: graph return path set %q is not route-ordered", ErrUnsupportedPostgresPlan, setName) + } + } + if setIndex > 0 && routeOrder <= previousRouteOrder { + return fmt.Errorf("%w: graph return path sets are not route-order ascending at %q (%d after %d); the planner's flatten order is part of the dedupe contract", + ErrUnsupportedPostgresPlan, setName, routeOrder, previousRouteOrder) + } + previousRouteOrder = routeOrder + } + return nil +} + +func validateGraphExtendFilter(index, scopeIndex int, extend *ir.PhysicalPathExtend, filter *ir.PhysicalFilter) error { + refuse := func(reason string) error { + return fmt.Errorf("%w: graph plan operation %d extend scope %d: %s", ErrUnsupportedPostgresPlan, index, scopeIndex, reason) + } + if filter.Expression == nil || filter.Expression.Kind != ir.PhysicalComparisonPredicate || filter.Expression.Comparison == nil { + return refuse("extend-scope FILTERs must be the planner's comparison-expression form (lower/graph_plan.go:277/:313)") + } + comparison := filter.Expression.Comparison + if comparison.Correlation != nil { + return refuse("correlated CODE path filters are a registered capability gap (lower/graph_plan.go:259-278; correlated predicates have no registered PostgreSQL contract)") + } + if comparison.LeftExpression == nil || comparison.LeftExpression.Extract == nil { + return refuse("the compared value must be the planner's selector extraction (lower/graph_plan.go:284-286)") + } + extract := comparison.LeftExpression.Extract + if extract.Source.Variable != extend.Traversal.TargetVariable || len(extract.Source.Path) != 1 || extract.Source.Path[0] != "payload" { + return refuse("extend-scope FILTERs may only read the reached target's payload (lower/graph_plan.go:285-286; the edge row is never readable)") + } + return nil +} + +type graphPathSet struct { + cteAlias string + terminalType string +} + +func (r *planRenderer) renderGraphStatement(plan ir.PhysicalPlan) (string, []string, error) { + rootScan := plan.Operations[0].RootScan + if err := r.renderRootScan(plan, *rootScan); err != nil { + return "", nil, err + } + seedConjuncts := r.where + r.where = nil + + seedIndex := 1 + for index := 1; index < len(plan.Operations)-1; index++ { + operation := plan.Operations[index] + if operation.Kind == ir.PhysicalPathSeedOp { + seedIndex = index + break + } + switch operation.Kind { + case ir.PhysicalFilterOp: + canonical, err := r.acceptCanonicalScopeFilter(operation.Filter) + if err != nil { + return "", nil, err + } + if canonical { + continue + } + renderPlan := graph20NormalizeRequiredMatchSubplan(plan, index) + clause, err := r.renderFilter(renderPlan, *renderPlan.Operations[index].Filter) + if err != nil { + return "", nil, fmt.Errorf("graph root filter %d: %w", index, err) + } + seedConjuncts = append(seedConjuncts, clause) + case ir.PhysicalDerivedLetOp: + canonical, err := r.acceptCanonicalAuthLet(operation.DerivedLet) + if err != nil { + return "", nil, err + } + if canonical { + continue + } + return "", nil, fmt.Errorf("%w: graph root operation %d is not a canonical scope operation", ErrUnsupportedPostgresPlan, index) + } + } + + resourceTable := r.storage.Resource.Identifier().SQLText() + edgeTable := r.storage.Edge.Identifier().SQLText() + resourceTypeColumn := r.storage.Resource.MustColumn("resource_type").SQLText() + logicalIDColumn := r.storage.Resource.MustColumn("logical_id").SQLText() + payloadColumn := r.storage.Resource.MustColumn("payload").SQLText() + edgeSeqColumn := r.storage.Edge.MustColumn("edge_seq").SQLText() + + rootRow := r.rows[rootScan.Variable] + seed := plan.Operations[seedIndex].PathSeed + + seedNode := "jsonb_build_object('alias', $" + strconv.Itoa(r.textParameter(seed.Node.Alias, "text:"+seed.Node.Alias)) + "::text" + + ", 'resourceType', $" + strconv.Itoa(r.textParameter(seed.Node.ResourceType, "text:"+seed.Node.ResourceType)) + "::text" + + ", 'id', " + rootRow.alias + "." + logicalIDColumn + + ", 'key', " + rootRow.alias + "." + logicalIDColumn + + ", 'payload', " + rootRow.alias + "." + payloadColumn + ")" + seedSelect := "SELECT jsonb_build_array(" + rootRow.alias + "." + resourceTypeColumn + ", " + rootRow.alias + "." + logicalIDColumn + ") AS \"path_identity\"" + + ", $" + strconv.Itoa(r.graphBigintParameter(seed.RouteOrder)) + "::bigint AS \"route_order\"" + + ", $" + strconv.Itoa(r.textParameter(seed.Node.Alias, "text:"+seed.Node.Alias)) + "::text AS \"terminal_alias\"" + + ", jsonb_build_array(" + seedNode + ") AS \"nodes\"" + + ", '[]'::jsonb AS \"relationships\", '[]'::jsonb AS \"occurrence_trail\"" + + ", " + rootRow.alias + "." + resourceTypeColumn + " AS \"resource_type\"" + + ", " + rootRow.alias + "." + logicalIDColumn + " AS \"logical_id\"" + + " FROM " + resourceTable + " AS " + rootRow.alias + if len(seedConjuncts) > 0 { + seedSelect += " WHERE " + strings.Join(seedConjuncts, " AND ") + } + + sets := map[string]graphPathSet{seed.Variable: {cteAlias: "path_0", terminalType: seed.Node.ResourceType}} + ctes := []string{"path_0 AS (" + seedSelect + ")"} + terminalTypeMismatch := func(index int, setName, want, got string) error { + return fmt.Errorf("%w: graph plan operation %d: set %q terminal type %q does not continue the authored chain from %q", + ErrUnsupportedPostgresPlan, index, setName, got, want) + } + + for index := seedIndex + 1; index < len(plan.Operations)-1; index++ { + extend := plan.Operations[index].PathExtend + parent, ok := sets[extend.SourceVariable] + if !ok { + return "", nil, fmt.Errorf("%w: graph plan operation %d: source path set %q is not a set defined by this plan", + ErrUnsupportedPostgresPlan, index, extend.SourceVariable) + } + if parent.terminalType != extend.Relationship.FromResourceType { + return "", nil, terminalTypeMismatch(index, extend.SourceVariable, parent.terminalType, extend.Relationship.FromResourceType) + } + const parentAlias = "p" + verified, err := r.validateTraversalContract(plan, graph20NativeStrategyTraversal(extend.Traversal), rowScope{alias: parentAlias, resourceType: extend.Relationship.FromResourceType}) + if err != nil { + return "", nil, fmt.Errorf("graph plan operation %d traversal: %w", index, err) + } + if extend.Node.ResourceType != verified.targetType { + return "", nil, fmt.Errorf("%w: graph plan operation %d: node resource type %q does not match the traversal target type bind %q (lower/graph_plan.go:125-126 always agrees; a divergence must fail closed)", + ErrUnsupportedPostgresPlan, index, extend.Node.ResourceType, verified.targetType) + } + seq := r.nextTraversalSequence() + edgeAlias := "e" + strconv.Itoa(seq) + targetAlias := "x" + strconv.Itoa(seq) + edgeWhere, targetOn := r.edgeOccurrencePredicates(&verified, edgeAlias, targetAlias, parentAlias) + + localRows := map[string]rowScope{extend.Traversal.TargetVariable: {alias: targetAlias, resourceType: verified.targetType}} + var extendConjuncts []string + scopeIndex := 0 + if err := matchCanonicalScopeBlock(extend.Scope, &scopeIndex, []string{extend.Traversal.EdgeVariable, extend.Traversal.TargetVariable}); err != nil { + return "", nil, fmt.Errorf("%w: graph plan operation %d extend scope %d: canonical hop scope block: %s", + ErrUnsupportedPostgresPlan, index, scopeIndex, err.Error()) + } + for ; scopeIndex < len(extend.Scope); scopeIndex++ { + operation := extend.Scope[scopeIndex] + if operation.Kind != ir.PhysicalFilterOp || operation.Filter == nil { + return "", nil, fmt.Errorf("%w: graph plan operation %d extend scope %d: only FILTER operations follow the canonical scope block", + ErrUnsupportedPostgresPlan, index, scopeIndex) + } + fragment, err := r.renderPredicateFragmentScoped(plan, *operation.Filter.Expression, localRows, true) + if err != nil { + return "", nil, fmt.Errorf("%w: graph plan operation %d extend scope %d: %w", ErrUnsupportedPostgresPlan, index, scopeIndex, err) + } + extendConjuncts = append(extendConjuncts, fragment.sql) + } + + labelRef := "$" + strconv.Itoa(verified.labelIndex) + nodeObject := "jsonb_build_object('alias', $" + strconv.Itoa(r.textParameter(extend.Node.Alias, "text:"+extend.Node.Alias)) + "::text" + + ", 'resourceType', $" + strconv.Itoa(r.textParameter(extend.Node.ResourceType, "text:"+extend.Node.ResourceType)) + "::text" + + ", 'id', " + targetAlias + "." + logicalIDColumn + + ", 'key', " + targetAlias + "." + logicalIDColumn + + ", 'payload', " + targetAlias + "." + payloadColumn + ")" + relationshipObject := "jsonb_build_object('alias', $" + strconv.Itoa(r.textParameter(extend.Relationship.Alias, "text:"+extend.Relationship.Alias)) + "::text" + + ", 'label', " + labelRef + "::text" + + ", 'fromResourceType', $" + strconv.Itoa(r.textParameter(extend.Relationship.FromResourceType, "text:"+extend.Relationship.FromResourceType)) + "::text" + + ", 'toResourceType', $" + strconv.Itoa(r.textParameter(extend.Relationship.ToResourceType, "text:"+extend.Relationship.ToResourceType)) + "::text" + ")" + + cteAlias := "path_" + strconv.Itoa(len(ctes)) + selectText := "SELECT p.\"path_identity\" || jsonb_build_array(" + targetAlias + "." + resourceTypeColumn + ", " + targetAlias + "." + logicalIDColumn + ", " + labelRef + "::text) AS \"path_identity\"" + + ", $" + strconv.Itoa(r.graphBigintParameter(extend.RouteOrder)) + "::bigint AS \"route_order\"" + + ", $" + strconv.Itoa(r.textParameter(extend.Node.Alias, "text:"+extend.Node.Alias)) + "::text AS \"terminal_alias\"" + + ", p.\"nodes\" || jsonb_build_array(" + nodeObject + ") AS \"nodes\"" + + ", p.\"relationships\" || jsonb_build_array(" + relationshipObject + ") AS \"relationships\"" + + ", p.\"occurrence_trail\" || jsonb_build_array(" + edgeAlias + "." + edgeSeqColumn + ") AS \"occurrence_trail\"" + + ", " + targetAlias + "." + resourceTypeColumn + " AS \"resource_type\"" + + ", " + targetAlias + "." + logicalIDColumn + " AS \"logical_id\"" + + " FROM " + parent.cteAlias + " AS p" + + " INNER JOIN " + edgeTable + " AS " + edgeAlias + " ON " + edgeWhere + + " INNER JOIN " + resourceTable + " AS " + targetAlias + " ON " + targetOn + if len(extendConjuncts) > 0 { + selectText += " WHERE " + strings.Join(extendConjuncts, " AND ") + } + ctes = append(ctes, cteAlias+" AS ("+selectText+")") + sets[extend.Variable] = graphPathSet{cteAlias: cteAlias, terminalType: extend.Node.ResourceType} + } + + graphReturn := plan.Operations[len(plan.Operations)-1].GraphReturn + arms := make([]string, 0, len(graphReturn.PathSets)) + for _, setName := range graphReturn.PathSets { + pathSet, ok := sets[setName] + if !ok { + return "", nil, fmt.Errorf("%w: graph return path set %q is not a set defined by this plan", ErrUnsupportedPostgresPlan, setName) + } + arms = append(arms, "SELECT \"path_identity\", \"route_order\", \"terminal_alias\", \"nodes\", \"relationships\", \"occurrence_trail\" FROM "+pathSet.cteAlias) + } + + limit, ok := plan.BindVars[graphReturn.LimitBindKey].(int) + if !ok || limit <= 0 { + return "", nil, fmt.Errorf("%w: graph limit bind %q must be a positive int (lower/graph_plan.go:21-23/:49)", + ErrUnsupportedPostgresPlan, graphReturn.LimitBindKey) + } + limitIndex := r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(limit)}, "") + + sql := "WITH " + strings.Join(ctes, ", ") + + ", graph_paths AS (SELECT *, ROW_NUMBER() OVER (PARTITION BY path_identity ORDER BY route_order ASC, occurrence_trail ASC) AS rn" + + " FROM (" + strings.Join(arms, " UNION ALL ") + ") AS g_union)" + + " SELECT jsonb_build_object('terminalAlias', terminal_alias, 'nodes', nodes, 'relationships', relationships) AS \"c0\"" + + " FROM graph_paths AS g_row WHERE g_row.rn = 1" + + " ORDER BY g_row.route_order ASC, g_row.path_identity::text COLLATE \"C\" ASC" + + " LIMIT $" + strconv.Itoa(limitIndex) + return sql, []string{graphOutputColumn}, nil +} + +func graph20NormalizeRequiredMatchSubplan(plan ir.PhysicalPlan, index int) ir.PhysicalPlan { + operation := plan.Operations[index] + if operation.Filter == nil || operation.Filter.Expression == nil || + operation.Filter.Expression.Kind != ir.PhysicalExistsPredicate || operation.Filter.Expression.Exists == nil || + validateRequiredMatchReturn(plan, *operation.Filter.Expression.Exists) != nil { + return plan + } + operations := operation.Filter.Expression.Exists.Operations + kept := make([]ir.PhysicalOperation, 0, len(operations)) + consumed := false + var blockVariables []string + for step := 0; step < len(operations); step++ { + candidate := operations[step] + if candidate.Kind == ir.PhysicalTraversalOp && candidate.Traversal != nil { + blockVariables = []string{candidate.Traversal.EdgeVariable, candidate.Traversal.TargetVariable} + kept = append(kept, candidate) + continue + } + if blockVariables != nil { + cursor := step + if err := matchCanonicalScopeBlock(operations, &cursor, blockVariables); err == nil && cursor > step { + step = cursor - 1 + blockVariables = nil + consumed = true + continue + } + } + kept = append(kept, candidate) + } + if !consumed { + return plan + } + normalized := plan + normalizedOperations := append([]ir.PhysicalOperation(nil), plan.Operations...) + operationCopy := normalizedOperations[index] + filterCopy := *operationCopy.Filter + expressionCopy := *filterCopy.Expression + existsCopy := *expressionCopy.Exists + existsCopy.Operations = kept + expressionCopy.Exists = &existsCopy + filterCopy.Expression = &expressionCopy + operationCopy.Filter = &filterCopy + normalizedOperations[index] = operationCopy + normalized.Operations = normalizedOperations + return normalized +} + +func (r *planRenderer) graphBigintParameter(value int) int { + return r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(value)}, "") +} + +func graph20NativeStrategyTraversal(traversal ir.PhysicalTraversal) ir.PhysicalTraversal { + if traversal.Strategy == ir.PhysicalTraversalEndpointLookup { + traversal.Strategy = ir.PhysicalTraversalNative + } + return traversal +} diff --git a/internal/dataframe/compiler/render/postgres/graph_test.go b/internal/dataframe/compiler/render/postgres/graph_test.go new file mode 100644 index 000000000..09a524286 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/graph_test.go @@ -0,0 +1,1070 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "reflect" + "regexp" + "strconv" + "strings" + "sync" + "testing" + "time" + + pgstore "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/semantic" + "github.com/calypr/loom/internal/dataframe/spec" + fhirschema "github.com/calypr/loom/internal/fhir/schema" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + graph20Project = "pgsq20-proj" + graph20Generation = "gen-1" + graph20AuthPath = "FHIR/Patient" +) + +func graph20Context() semantic.ExecutionContext { + return semantic.ExecutionContext{ + Project: graph20Project, + DatasetGeneration: graph20Generation, + AuthResourcePaths: []string{graph20AuthPath}, + } +} + +func graph20PlanFor(t *testing.T, root semantic.SemanticNode, limit int) ir.PhysicalPlan { + t.Helper() + plan, err := lower.BuildGraphPhysicalPlan(semantic.OutputPlan{Root: root}, graph20Context(), limit, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("BuildGraphPhysicalPlan: %v", err) + } + return plan +} + +func graph20Root(children ...semantic.SemanticNode) semantic.SemanticNode { + return semantic.SemanticNode{Alias: "root", ResourceType: "Patient", Children: children} +} + +func graph20Condition(alias string, mode spec.TraversalMatchMode, children ...semantic.SemanticNode) semantic.SemanticNode { + return semantic.SemanticNode{Alias: alias, ResourceType: "Condition", EdgeLabel: "subject_Patient", MatchMode: mode, Children: children} +} + +func graph20Observation(alias string, mode spec.TraversalMatchMode) semantic.SemanticNode { + return semantic.SemanticNode{Alias: alias, ResourceType: "Observation", EdgeLabel: "subject_Patient", MatchMode: mode} +} + +func graph20Procedure(alias string, mode spec.TraversalMatchMode) semantic.SemanticNode { + return semantic.SemanticNode{Alias: alias, ResourceType: "Procedure", EdgeLabel: "subject_Patient", MatchMode: mode} +} + +func graph20FocusedObservation(alias string, mode spec.TraversalMatchMode) semantic.SemanticNode { + return semantic.SemanticNode{Alias: alias, ResourceType: "Observation", EdgeLabel: "focus_Condition", MatchMode: mode} +} + +func graph20Binding(t *testing.T) ir.PostgresStorageBinding { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("loom") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + return binding +} + +func graph20Render(t *testing.T, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderGraphPhysicalPlan(plan, graph20Binding(t)) + if err != nil { + t.Fatalf("RenderGraphPhysicalPlan: %v", err) + } + return rendered +} + +func graph20ArgsValues(rendered command.RenderedPostgresPhysicalPlan) []any { + values := make([]any, 0, len(rendered.Args)) + for _, parameter := range rendered.Args { + switch value := parameter.Value.(type) { + case json.RawMessage: + values = append(values, string(value)) + default: + values = append(values, value) + } + } + return values +} + +func graph20ParamTypes(rendered command.RenderedPostgresPhysicalPlan) []command.PostgreSQLParameterType { + types := make([]command.PostgreSQLParameterType, 0, len(rendered.Args)) + for _, parameter := range rendered.Args { + types = append(types, parameter.Type) + } + return types +} + +func graph20ClonePlan(plan ir.PhysicalPlan) ir.PhysicalPlan { + clone := plan + clone.Operations = append([]ir.PhysicalOperation(nil), plan.Operations...) + clone.BindVars = map[string]any{} + for key, value := range plan.BindVars { + clone.BindVars[key] = value + } + return clone +} + +func TestGraph20RenderRootOnlyOptionalGolden(t *testing.T) { + plan := graph20PlanFor(t, graph20Root(graph20Procedure("proc", spec.TraversalMatchOptional)), 3) + if last := plan.Operations[len(plan.Operations)-1].GraphReturn; len(last.PathSets) != 2 || last.PathSets[0] != "path_root" || last.PathSets[1] != "path_1" { + t.Fatalf("planner PathSets = %v, want [path_root path_1]", last.PathSets) + } + rendered := graph20Render(t, plan) + + if !reflect.DeepEqual(rendered.OutputColumns, []string{"paths"}) { + t.Fatalf("output columns = %v, want [paths]", rendered.OutputColumns) + } + wantTypes := []command.PostgreSQLParameterType{ + command.PostgreSQLParameterText, command.PostgreSQLParameterText, command.PostgreSQLParameterJSONB, + command.PostgreSQLParameterText, command.PostgreSQLParameterText, + command.PostgreSQLParameterBigint, + command.PostgreSQLParameterText, command.PostgreSQLParameterText, command.PostgreSQLParameterText, + command.PostgreSQLParameterBigint, + command.PostgreSQLParameterBigint, + } + if got := graph20ParamTypes(rendered); !reflect.DeepEqual(got, wantTypes) { + t.Fatalf("argument types =\n%v\nwant\n%v", got, wantTypes) + } + wantValues := []any{ + graph20Project, graph20Generation, `["FHIR/Patient"]`, + "Patient", "root", + int64(0), + "subject_Patient", "Procedure", "proc", + int64(1), + int64(4), + } + if got := graph20ArgsValues(rendered); !reflect.DeepEqual(got, wantValues) { + t.Fatalf("argument values =\n%v\nwant\n%v", got, wantValues) + } + + const wantSQL = `WITH path_0 AS (SELECT jsonb_build_array(r0."resource_type", r0."logical_id") AS "path_identity", $6::bigint AS "route_order", $5::text AS "terminal_alias", jsonb_build_array(jsonb_build_object('alias', $5::text, 'resourceType', $4::text, 'id', r0."logical_id", 'key', r0."logical_id", 'payload', r0."payload")) AS "nodes", '[]'::jsonb AS "relationships", '[]'::jsonb AS "occurrence_trail", r0."resource_type" AS "resource_type", r0."logical_id" AS "logical_id" FROM "loom"."fhir_resource" AS r0 WHERE r0."project" = $1 AND r0."dataset_generation" = $2 AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb)) AND r0."resource_type" = $4), path_1 AS (SELECT p."path_identity" || jsonb_build_array(x1."resource_type", x1."logical_id", $7::text) AS "path_identity", $10::bigint AS "route_order", $9::text AS "terminal_alias", p."nodes" || jsonb_build_array(jsonb_build_object('alias', $9::text, 'resourceType', $8::text, 'id', x1."logical_id", 'key', x1."logical_id", 'payload', x1."payload")) AS "nodes", p."relationships" || jsonb_build_array(jsonb_build_object('alias', $9::text, 'label', $7::text, 'fromResourceType', $4::text, 'toResourceType', $8::text)) AS "relationships", p."occurrence_trail" || jsonb_build_array(e1."edge_seq") AS "occurrence_trail", x1."resource_type" AS "resource_type", x1."logical_id" AS "logical_id" FROM path_0 AS p INNER JOIN "loom"."fhir_edge_occurrence" AS e1 ON e1."project" = $1 AND e1."dataset_generation" = $2 AND e1."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb)) AND e1."label" = $7 AND e1."target_resource_type" = $4 AND e1."target_logical_id" = p."logical_id" AND e1."source_resource_type" = $8 INNER JOIN "loom"."fhir_resource" AS x1 ON x1."project" = $1 AND x1."dataset_generation" = $2 AND x1."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb)) AND x1."resource_type" = $8 AND x1."logical_id" = e1."source_logical_id"), graph_paths AS (SELECT *, ROW_NUMBER() OVER (PARTITION BY path_identity ORDER BY route_order ASC, occurrence_trail ASC) AS rn FROM (SELECT "path_identity", "route_order", "terminal_alias", "nodes", "relationships", "occurrence_trail" FROM path_0 UNION ALL SELECT "path_identity", "route_order", "terminal_alias", "nodes", "relationships", "occurrence_trail" FROM path_1) AS g_union) SELECT jsonb_build_object('terminalAlias', terminal_alias, 'nodes', nodes, 'relationships', relationships) AS "c0" FROM graph_paths AS g_row WHERE g_row.rn = 1 ORDER BY g_row.route_order ASC, g_row.path_identity::text COLLATE "C" ASC LIMIT $11` + if rendered.SQL != wantSQL { + t.Fatalf("golden SQL drift:\n got %s\nwant %s", rendered.SQL, wantSQL) + } + + for _, forbidden := range []string{"Recursive", "__loom_", "recursive"} { + if strings.Contains(rendered.SQL, forbidden) { + t.Fatalf("statement must never contain %q: %s", forbidden, rendered.SQL) + } + } + limitIndex := strings.Index(rendered.SQL, " LIMIT $") + dedupeIndex := strings.Index(rendered.SQL, "ROW_NUMBER() OVER (PARTITION BY path_identity") + if !(dedupeIndex >= 0 && limitIndex > dedupeIndex) { + t.Fatal("bound LIMIT must sit after the dedupe window (render/aql/graph_render.go:164-170 order)") + } +} + +func TestGraph20RenderRequiredOneHop(t *testing.T) { + plan := graph20PlanFor(t, graph20Root(graph20Condition("cond", spec.TraversalMatchRequired)), 100) + last := plan.Operations[len(plan.Operations)-1].GraphReturn + if len(last.PathSets) != 1 || last.PathSets[0] != "path_1" { + t.Fatalf("required tree must drop the root-only set, PathSets = %v", last.PathSets) + } + rendered := graph20Render(t, plan) + + seedBody := rendered.SQL + end := strings.Index(seedBody, "), path_1 AS (") + if end < 0 { + t.Fatalf("seed CTE boundary missing:\n%s", rendered.SQL) + } + seedBody = seedBody[:end] + if !strings.Contains(seedBody, "EXISTS (SELECT 1 FROM") { + t.Fatalf("REQUIRED-match EXISTS semi-join missing from the seed CTE:\n%s", seedBody) + } + if strings.Index(rendered.SQL, "EXISTS (SELECT 1 FROM") > strings.Index(rendered.SQL, "), path_1 AS (") { + t.Fatal("the semi-join must gate root membership inside path_0, never the expanded rows") + } + if values := graph20ArgsValues(rendered); values[len(values)-1] != int64(101) { + t.Fatalf("lookahead argument must be limit+1 (101), got %v", values[len(values)-1]) + } + if count := strings.Count(rendered.SQL, `= ANY (SELECT jsonb_array_elements_text($3::jsonb))`); count != 5 { + t.Fatalf("scope conjuncts = %d, want 5 (root, hop edge, hop target, semi-join edge, semi-join target)", count) + } +} + +func TestGraph20RenderCompositeRouteOrderAndUnionArms(t *testing.T) { + plan := graph20PlanFor(t, graph20Root( + graph20Condition("cond", spec.TraversalMatchRequired, graph20FocusedObservation("obs2", spec.TraversalMatchRequired)), + graph20Observation("obs", spec.TraversalMatchOptional), + ), 5) + last := plan.Operations[len(plan.Operations)-1].GraphReturn + if !reflect.DeepEqual(last.PathSets, []string{"path_1", "path_2", "path_3"}) { + t.Fatalf("PathSets = %v", last.PathSets) + } + rendered := graph20Render(t, plan) + + unionStart := strings.Index(rendered.SQL, "FROM (SELECT \"path_identity\"") + unionEnd := strings.Index(rendered.SQL, ") AS g_union") + if unionStart < 0 || unionEnd < unionStart { + t.Fatalf("union block missing:\n%s", rendered.SQL) + } + arms := strings.Split(rendered.SQL[unionStart:unionEnd], " UNION ALL ") + if len(arms) != 3 { + t.Fatalf("union arms = %d, want 3", len(arms)) + } + for index, wantCTE := range []string{"FROM path_1", "FROM path_2", "FROM path_3"} { + if !strings.HasSuffix(arms[index], wantCTE) { + t.Fatalf("arm %d must read %s: %s", index, wantCTE, arms[index]) + } + } + + var orders []int64 + for _, parameter := range rendered.Args { + if parameter.Type == command.PostgreSQLParameterBigint && parameter.Value != int64(6) { + orders = append(orders, parameter.Value.(int64)) + } + } + if !reflect.DeepEqual(orders, []int64{0, 1, 2, 3}) { + t.Fatalf("bound bigints (route orders) = %v, want [0 1 2 3]", orders) + } + if strings.Contains(rendered.SQL, "path_2 AS") != strings.Contains(rendered.SQL, "path_3 AS") { + t.Fatal("every declared extend set must materialize as its own CTE") + } + setNumber := map[string]int{} + for _, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalPathSeedOp: + setNumber[operation.PathSeed.Variable] = 0 + case ir.PhysicalPathExtendOp: + setNumber[operation.PathExtend.Variable] = len(setNumber) + } + } + nestedSet, nestedParent := "", -1 + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalPathExtendOp && operation.PathExtend.Relationship.FromResourceType == "Condition" { + nestedSet = operation.PathExtend.Variable + nestedParent = setNumber[operation.PathExtend.SourceVariable] + } + } + if nestedSet == "" || nestedParent < 0 { + t.Fatal("composite fixture lost its nested focus extend") + } + nestedNumber := setNumber[nestedSet] + nested := rendered.SQL[strings.Index(rendered.SQL, "path_"+strconv.Itoa(nestedNumber)+" AS ("):] + if end := strings.Index(nested, "), path_"+strconv.Itoa(nestedNumber+1)+" AS ("); end >= 0 { + nested = nested[:end] + } else if end := strings.Index(nested, "), graph_paths AS ("); end >= 0 { + nested = nested[:end] + } + if wantParent := " FROM path_" + strconv.Itoa(nestedParent) + " AS p"; !strings.Contains(nested, wantParent) { + t.Fatalf("nested extend must correlate to its authored parent set path_%d:\n%s", nestedParent, nested) + } + if strings.Count(nested, `= ANY (SELECT jsonb_array_elements_text($3::jsonb))`) != 2 { + t.Fatalf("nested extend must carry the edge and target scope chains:\n%s", nested) + } +} + +func TestGraph20RenderDeterminismAndHostileParameterization(t *testing.T) { + root := graph20Root(graph20Condition("cond", spec.TraversalMatchRequired)) + first := graph20Render(t, graph20PlanFor(t, root, 7)) + second := graph20Render(t, graph20PlanFor(t, root, 7)) + if first.SQL != second.SQL { + t.Fatal("rendering is not byte-stable across runs") + } + if !reflect.DeepEqual(graph20ArgsValues(first), graph20ArgsValues(second)) { + t.Fatal("arguments are not value-stable across runs") + } + + hostileContext := semantic.ExecutionContext{ + Project: `p'gsq20"proj; DROP`, + DatasetGeneration: "gen-$1--x", + AuthResourcePaths: []string{`FHIR/Weird "Path" /*`, "FHIR/日本語"}, + } + plan, err := lower.BuildGraphPhysicalPlan(semantic.OutputPlan{Root: root}, hostileContext, 2, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("hostile BuildGraphPhysicalPlan: %v", err) + } + rendered := graph20Render(t, plan) + for _, hostile := range []string{`p'gsq20"proj; DROP`, "gen-$1--x", `FHIR/Weird "Path" /*`, "FHIR/日本語"} { + if strings.Contains(rendered.SQL, hostile) { + t.Fatalf("hostile value %q leaked into SQL text", hostile) + } + } + found := map[string]bool{} + for _, parameter := range rendered.Args { + if text, ok := parameter.Value.(string); ok { + found[text] = true + } + if raw, ok := parameter.Value.(json.RawMessage); ok && strings.Contains(string(raw), "日本語") { + found["jsonb-unicode"] = true + } + } + for _, hostile := range []string{`p'gsq20"proj; DROP`, "gen-$1--x", "jsonb-unicode"} { + if !found[hostile] { + t.Fatalf("hostile value %q must arrive as an ordered argument", hostile) + } + } + if regexp.MustCompile(`@[A-Za-z_]+`).MatchString(rendered.SQL) { + t.Fatalf("AQL bind syntax in statement: %s", rendered.SQL) + } +} + +func TestGraph20CanonicalDenyReadsZeroRows(t *testing.T) { + plan := graph20PlanFor(t, graph20Root(graph20Observation("obs", spec.TraversalMatchOptional)), 4) + plan.BindVars["scope_allowed"] = false + rendered := graph20Render(t, plan) + if count := strings.Count(rendered.SQL, " AND FALSE"); count == 0 { + t.Fatalf("canonical deny must close the scope chains with AND FALSE:\n%s", rendered.SQL) + } +} + +func graph20Refusal(t *testing.T, plan ir.PhysicalPlan, wantContains string) { + t.Helper() + _, entryErr := RenderGraphPhysicalPlan(plan, graph20Binding(t)) + _, dispatchErr := RenderPhysicalPlan(plan, graph20Binding(t)) + if entryErr == nil { + t.Fatalf("plan accepted, want refusal containing %q", wantContains) + } + if dispatchErr == nil || dispatchErr.Error() != entryErr.Error() { + t.Fatalf("entry==dispatch equality broken:\nentry: %v\ndispatch: %v", entryErr, dispatchErr) + } + if !strings.Contains(entryErr.Error(), wantContains) { + t.Fatalf("refusal %q does not contain %q", entryErr.Error(), wantContains) + } +} + +func graph20FindOp(t *testing.T, plan ir.PhysicalPlan, kind ir.PhysicalOperationKind) int { + t.Helper() + for index, operation := range plan.Operations { + if operation.Kind == kind { + return index + } + } + t.Fatalf("plan has no %s operation", kind) + return -1 +} + +func graph20RequiredOneHopPlan(t *testing.T) ir.PhysicalPlan { + return graph20PlanFor(t, graph20Root(graph20Condition("cond", spec.TraversalMatchRequired)), 10) +} + +func TestGraph20RefusalMatrix(t *testing.T) { + base := graph20RequiredOneHopPlan(t) + + t.Run("SecondPathSeed", func(t *testing.T) { + plan := graph20ClonePlan(base) + seed := plan.Operations[graph20FindOp(t, plan, ir.PhysicalPathSeedOp)] + ops := append(append([]ir.PhysicalOperation{}, plan.Operations[:len(plan.Operations)-1]...), seed) + plan.Operations = append(ops, plan.Operations[len(plan.Operations)-1]) + graph20Refusal(t, plan, "second PATH_SEED") + }) + t.Run("PathExtendBeforeSeed", func(t *testing.T) { + plan := graph20ClonePlan(base) + seedIdx, extendIdx := graph20FindOp(t, plan, ir.PhysicalPathSeedOp), graph20FindOp(t, plan, ir.PhysicalPathExtendOp) + ops := plan.Operations + ops[seedIdx], ops[extendIdx] = ops[extendIdx], ops[seedIdx] + graph20Refusal(t, plan, "PATH_EXTEND before PATH_SEED") + }) + t.Run("NonEmptySourcePath", func(t *testing.T) { + plan := graph20ClonePlan(base) + extendIdx := graph20FindOp(t, plan, ir.PhysicalPathExtendOp) + extend := *plan.Operations[extendIdx].PathExtend + extend.SourcePath = []string{"terminal"} + plan.Operations[extendIdx].PathExtend = &extend + graph20Refusal(t, plan, "non-empty SourcePath") + }) + t.Run("ExtendNodeMustReadTarget", func(t *testing.T) { + plan := graph20ClonePlan(base) + extendIdx := graph20FindOp(t, plan, ir.PhysicalPathExtendOp) + extend := *plan.Operations[extendIdx].PathExtend + extend.Node.Value = ir.PhysicalValue{Variable: extend.Traversal.TargetVariable, Path: []string{"payload"}} + plan.Operations[extendIdx].PathExtend = &extend + graph20Refusal(t, plan, "must read the reached target row itself") + }) + t.Run("SeedNodeMustReadRoot", func(t *testing.T) { + plan := graph20ClonePlan(base) + seedIdx := graph20FindOp(t, plan, ir.PhysicalPathSeedOp) + seed := *plan.Operations[seedIdx].PathSeed + seed.Node.Value = ir.PhysicalValue{BindKey: "sneaky"} + plan.Operations[seedIdx].PathSeed = &seed + graph20Refusal(t, plan, "seed node must read the root row itself") + }) + t.Run("ExtendFilterOnEdgeVariable", func(t *testing.T) { + plan := graph20ClonePlan(base) + extendIdx := graph20FindOp(t, plan, ir.PhysicalPathExtendOp) + extend := *plan.Operations[extendIdx].PathExtend + scope := append(append([]ir.PhysicalOperation(nil), extend.Scope...), + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Expression: &ir.PhysicalPredicateExpression{Kind: ir.PhysicalComparisonPredicate, Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", + LeftExpression: &ir.PhysicalExpression{Kind: ir.PhysicalExtractExpression, Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: extend.Traversal.EdgeVariable, Path: []string{"payload"}}, + }}, + Right: &ir.PhysicalValue{BindKey: "sneaky"}, + }}, + }}) + extend.Scope = scope + plan.Operations[extendIdx].PathExtend = &extend + graph20Refusal(t, plan, "only read the reached target's payload") + }) + t.Run("ExtendFilterBarePredicateForm", func(t *testing.T) { + plan := graph20ClonePlan(base) + extendIdx := graph20FindOp(t, plan, ir.PhysicalPathExtendOp) + extend := *plan.Operations[extendIdx].PathExtend + scope := append(append([]ir.PhysicalOperation(nil), extend.Scope...), + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: extend.Traversal.TargetVariable, Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "sneaky"}, + }}}) + extend.Scope = scope + plan.Operations[extendIdx].PathExtend = &extend + graph20Refusal(t, plan, "comparison-expression form") + }) + t.Run("CorrelatedPathFilter", func(t *testing.T) { + valueKind := spec.FilterCode + child := graph20Procedure("proc", spec.TraversalMatchOptional) + child.Filters = []spec.TypedFilter{{ + FieldRef: "Condition.code", Selector: "code.coding[].code", FieldKind: valueKind, + Repeated: true, Quantifier: spec.QuantifierAny, Operator: spec.FilterEquals, + Values: []spec.FilterValue{{Kind: valueKind, Code: &spec.CodeValue{System: "http://loinc.org", Code: "systolic"}}}, + Correlation: &fhirschema.CorrelatedBinding{ + KeyPath: "code.coding[]", SystemPath: "system", CodePath: "code", + ValueScope: fhirschema.CorrelatedValueScopeCoding, + SelectionScope: fhirschema.CorrelatedSelectionScopeExact, + ValuePath: "display", LogicalType: "string", + }, + }} + _, err := RenderGraphPhysicalPlan(graph20PlanFor(t, graph20Root(child), 5), graph20Binding(t)) + if err == nil || !strings.Contains(err.Error(), "correlated CODE path filters") { + t.Fatalf("correlated extend filter must fail closed, got %v", err) + } + }) + t.Run("ExtraDerivedLetInExtendScope", func(t *testing.T) { + plan := graph20ClonePlan(base) + extendIdx := graph20FindOp(t, plan, ir.PhysicalPathExtendOp) + extend := *plan.Operations[extendIdx].PathExtend + extend.Scope = append(append([]ir.PhysicalOperation(nil), extend.Scope...), ir.PhysicalOperation{ + Kind: ir.PhysicalDerivedLetOp, + DerivedLet: &ir.PhysicalDerivedLet{Variable: "sneaky", Operator: "LENGTH", Inputs: []ir.PhysicalValue{{Variable: extend.Traversal.TargetVariable}}}, + }) + plan.Operations[extendIdx].PathExtend = &extend + graph20Refusal(t, plan, "only the canonical AUTH_RESOURCE_PATH_ALLOWED LET") + }) + t.Run("TraversalInsideGraphSlice", func(t *testing.T) { + plan := graph20ClonePlan(base) + traversal := plan.Operations[graph20FindOp(t, plan, ir.PhysicalPathExtendOp)].PathExtend.Traversal + ops := append([]ir.PhysicalOperation{}, plan.Operations[:len(plan.Operations)-1]...) + ops = append(ops, ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &traversal}) + plan.Operations = append(ops, plan.Operations[len(plan.Operations)-1]) + graph20Refusal(t, plan, "is not part of the closed graph slice") + }) + t.Run("ExtendOnUnknownSourceSet", func(t *testing.T) { + plan := graph20ClonePlan(base) + extendIdx := graph20FindOp(t, plan, ir.PhysicalPathExtendOp) + extend := *plan.Operations[extendIdx].PathExtend + extend.SourceVariable = "path_99" + plan.Operations[extendIdx].PathExtend = &extend + graph20Refusal(t, plan, "not a set defined by this plan") + }) + t.Run("PathSetsNotAscending", func(t *testing.T) { + plan := graph20PlanFor(t, graph20Root( + graph20Condition("cond", spec.TraversalMatchRequired), + graph20Observation("obs", spec.TraversalMatchRequired), + ), 5) + last := *plan.Operations[len(plan.Operations)-1].GraphReturn + last.PathSets = []string{"path_2", "path_1"} + plan.Operations[len(plan.Operations)-1].GraphReturn = &last + graph20Refusal(t, plan, "not route-order ascending") + }) + t.Run("PathSetsUnknownSet", func(t *testing.T) { + plan := graph20ClonePlan(base) + last := *plan.Operations[len(plan.Operations)-1].GraphReturn + last.PathSets = []string{"path_1", "path_7"} + plan.Operations[len(plan.Operations)-1].GraphReturn = &last + graph20Refusal(t, plan, "not a set defined by this plan") + }) + t.Run("GraphReturnNotFinal", func(t *testing.T) { + plan := graph20ClonePlan(base) + plan.Operations = append(plan.Operations, ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS"}}}) + graph20Refusal(t, plan, "GRAPH_RETURN as the final operation") + }) + t.Run("SecondGraphReturn", func(t *testing.T) { + plan := graph20ClonePlan(base) + last := plan.Operations[len(plan.Operations)-1] + plan.Operations = append(plan.Operations[:len(plan.Operations)-1], last, last) + graph20Refusal(t, plan, "is not part of the closed graph slice") + }) + t.Run("AnyDirection", func(t *testing.T) { + plan := graph20ClonePlan(base) + extendIdx := graph20FindOp(t, plan, ir.PhysicalPathExtendOp) + extend := *plan.Operations[extendIdx].PathExtend + extend.Traversal.Direction = ir.PhysicalAny + plan.Operations[extendIdx].PathExtend = &extend + graph20Refusal(t, plan, "ANY-direction traversal has no proven PostgreSQL contract") + }) + t.Run("EndpointLookupStrategyNormalizesToRegisteredJoin", func(t *testing.T) { + plan := graph20ClonePlan(base) + extendIdx := graph20FindOp(t, plan, ir.PhysicalPathExtendOp) + extend := *plan.Operations[extendIdx].PathExtend + extend.Traversal.Strategy = ir.PhysicalTraversalEndpointLookup + extend.Traversal.EndpointField = "subject" + extend.Traversal.EndpointJoinField = "subject" + extend.Traversal.EndpointIndexFields = []string{"subject"} + plan.Operations[extendIdx].PathExtend = &extend + native := graph20Render(t, base).SQL + labelled := graph20Render(t, plan).SQL + if native != labelled { + t.Fatalf("endpoint-lookup strategy must not alter the registered join:\nnative: %s\nlabelled: %s", native, labelled) + } + }) + t.Run("EndpointLookupTruncatedPayloadRefuses", func(t *testing.T) { + plan := graph20ClonePlan(base) + extendIdx := graph20FindOp(t, plan, ir.PhysicalPathExtendOp) + extend := *plan.Operations[extendIdx].PathExtend + extend.Traversal.Strategy = ir.PhysicalTraversalEndpointLookup + extend.Traversal.EndpointField = "" + plan.Operations[extendIdx].PathExtend = &extend + graph20Refusal(t, plan, "endpoint-lookup traversal payload is incomplete") + }) + t.Run("UnknownStrategyStillRefused", func(t *testing.T) { + plan := graph20ClonePlan(base) + extendIdx := graph20FindOp(t, plan, ir.PhysicalPathExtendOp) + extend := *plan.Operations[extendIdx].PathExtend + extend.Traversal.Strategy = ir.PhysicalTraversalStrategy("wild") + plan.Operations[extendIdx].PathExtend = &extend + graph20Refusal(t, plan, "unsupported traversal strategy") + }) + t.Run("NodeTypeMismatchFailsClosed", func(t *testing.T) { + plan := graph20ClonePlan(base) + extendIdx := graph20FindOp(t, plan, ir.PhysicalPathExtendOp) + extend := *plan.Operations[extendIdx].PathExtend + extend.Node.ResourceType = "Observation" + plan.Operations[extendIdx].PathExtend = &extend + graph20Refusal(t, plan, "does not match the traversal target type bind") + }) + t.Run("RelationshipLabelBindMismatchFailsClosed", func(t *testing.T) { + plan := graph20ClonePlan(base) + extendIdx := graph20FindOp(t, plan, ir.PhysicalPathExtendOp) + extend := *plan.Operations[extendIdx].PathExtend + relationship := extend.Relationship + relationship.LabelBindKey = "sneaky_label_bind" + plan.BindVars["sneaky_label_bind"] = "totally_different" + extend.Relationship = relationship + plan.Operations[extendIdx].PathExtend = &extend + graph20Refusal(t, plan, "does not equal the traversal edge label bind") + }) + t.Run("SeedRouteOrderNotZero", func(t *testing.T) { + plan := graph20ClonePlan(base) + seedIdx := graph20FindOp(t, plan, ir.PhysicalPathSeedOp) + seed := *plan.Operations[seedIdx].PathSeed + seed.RouteOrder = 7 + plan.Operations[seedIdx].PathSeed = &seed + graph20Refusal(t, plan, "seed route order 7 is not 0") + }) + t.Run("BrokenChainType", func(t *testing.T) { + plan := graph20ClonePlan(base) + extendIdx := graph20FindOp(t, plan, ir.PhysicalPathExtendOp) + extend := *plan.Operations[extendIdx].PathExtend + extend.Relationship.FromResourceType = "AllergyIntolerance" + plan.Operations[extendIdx].PathExtend = &extend + graph20Refusal(t, plan, "does not continue the authored chain") + }) + t.Run("MissingAuthModeBind", func(t *testing.T) { + plan := graph20ClonePlan(base) + delete(plan.BindVars, "auth_resource_paths_unrestricted") + graph20Refusal(t, plan, "auth_resource_paths_unrestricted") + }) + t.Run("ClickHouseEngine", func(t *testing.T) { + plan := graph20ClonePlan(base) + plan.Engine = ir.PhysicalEngineClickHouse + graph20Refusal(t, plan, "ClickHouse") + }) + t.Run("StageSequence", func(t *testing.T) { + plan := graph20ClonePlan(base) + plan.StageSequence = &ir.PhysicalStageSequence{} + graph20Refusal(t, plan, "stage") + }) + t.Run("DeferredLets", func(t *testing.T) { + plan := graph20ClonePlan(base) + plan.DeferredExpressionLets = []ir.PhysicalOperation{{Kind: ir.PhysicalExpressionLetOp, ExpressionLet: &ir.PhysicalExpressionLet{Variable: "later"}}} + graph20Refusal(t, plan, "deferred expression lets") + }) + t.Run("RootRegionNonCanonicalLet", func(t *testing.T) { + plan := graph20ClonePlan(base) + ops := append([]ir.PhysicalOperation{}, plan.Operations[:1]...) + ops = append(ops, ir.PhysicalOperation{Kind: ir.PhysicalDerivedLetOp, DerivedLet: &ir.PhysicalDerivedLet{Variable: "root_hack", Operator: "LENGTH", Inputs: []ir.PhysicalValue{{Variable: "root"}}}}) + ops = append(ops, plan.Operations[1:]...) + plan.Operations = ops + graph20Refusal(t, plan, "not a canonical scope operation") + }) +} + +func TestGraph20SpineDispatchUntouched(t *testing.T) { + plan := ir.PhysicalPlan{ + Version: 1, + BindVars: map[string]any{ + "project": "p", "dataset_generation": "gen-1", + "auth_resource_paths": []string{"FHIR/Patient"}, + "auth_resource_paths_unrestricted": false, "scope_allowed": true, + "root_collection": "fhir_resource", + }, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalDerivedLetOp, DerivedLet: &ir.PhysicalDerivedLet{Variable: "root_len", Operator: "LENGTH", Inputs: []ir.PhysicalValue{{Variable: "root", Path: []string{"payload"}}}}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{{Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}}}}, + }, + } + _, err := RenderPhysicalPlan(plan, graph20Binding(t)) + if !errors.Is(err, ErrUnsupportedPostgresPlan) || !strings.Contains(err.Error(), `operation "DERIVED_LET" is not part of the v1 PostgreSQL render surface`) { + t.Fatalf("spine default-arm refusal drifted: %v", err) + } +} + +var ( + graph20FixturesMu sync.Mutex + graph20TrackedFixtures = map[string]bool{} +) + +func graph20LiveService(t *testing.T) *pgir02Service { + t.Helper() + svc := newOwnedService(t, 0, "verify_sql20_it") + graph20TrackFixtures(t, svc) + return svc +} + +func graph20TrackFixtures(t *testing.T, svc *pgir02Service) { + t.Helper() + var database string + if err := svc.iso.QueryRow(svc.ctx, "SELECT current_database()").Scan(&database); err != nil { + svc.fail("track owned database", err) + } + graph20FixturesMu.Lock() + defer graph20FixturesMu.Unlock() + graph20TrackedFixtures[database] = true + graph20TrackedFixtures[svc.schema] = true +} + +var graph20OwnedNamePattern = regexp.MustCompile(`^verify_sql20_it_[0-9]+_[0-9a-f]+$`) + +func graph20RunDeltaAudit(inner func() int) int { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return inner() + } + beforeList, err := graph20SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-20 delta audit: snapshot before run: %v\n", err) + return 1 + } + before := make(map[string]bool, len(beforeList)) + for _, name := range beforeList { + before[name] = true + } + code := inner() + after, err := graph20SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-20 delta audit: snapshot after run: %v\n", err) + code = 1 + } + graph20FixturesMu.Lock() + defer graph20FixturesMu.Unlock() + for _, name := range after { + if before[name] { + continue + } + if graph20TrackedFixtures[name] { + fmt.Fprintf(os.Stderr, "pg-sql-20 delta audit: %q was created and tracked by this run and still survives\n", name) + code = 1 + continue + } + fmt.Fprintf(os.Stderr, "pg-sql-20 delta audit: note — %q was born during this run but is not owned by it (sibling lane?)\n", name) + } + return code +} + +func graph20SnapshotOwnedResources(ctx context.Context, raw string) ([]string, error) { + pool, err := pgxpool.New(ctx, raw) + if err != nil { + return nil, err + } + defer pool.Close() + queryCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + var names []string + for _, query := range []string{ + `SELECT datname FROM pg_database WHERE datname LIKE 'verify\_sql20\_%' ESCAPE '\'`, + `SELECT nspname FROM pg_namespace WHERE nspname LIKE 'verify\_sql20\_%' ESCAPE '\'`, + } { + rows, err := pool.Query(queryCtx, query) + if err != nil { + return nil, err + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + return nil, err + } + names = append(names, name) + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, err + } + rows.Close() + } + return names, nil +} + +func graph20SeedResource(t *testing.T, svc *pgir02Service, project, generation, resourceType, logicalID, authPath string, payload json.RawMessage) { + t.Helper() + repo, err := pgstore.NewResourceRepository(svc.client, svc.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + if _, err := repo.InsertResource(svc.ctx, source.Resource{ + Project: project, DatasetGeneration: generation, ResourceType: resourceType, + LogicalID: logicalID, AuthResourcePath: authPath, Payload: payload, + }); err != nil { + svc.fail("insert resource "+resourceType+"/"+logicalID, err) + } +} + +func graph20SeedEdge(t *testing.T, svc *pgir02Service, project, generation, edgeAuth, label, sourceType, sourceID, targetType, targetID string, ordinalOverrides ...int64) int64 { + t.Helper() + repo, err := pgstore.NewEdgeRepository(svc.client, svc.schema) + if err != nil { + t.Fatalf("edge repository: %v", err) + } + ordinal := int64(0) + if len(ordinalOverrides) > 0 { + ordinal = ordinalOverrides[0] + } + seq, err := repo.InsertEdge(svc.ctx, pgstore.EdgeOccurrenceRow{ + Project: project, DatasetGeneration: generation, AuthResourcePath: edgeAuth, + Label: label, SourceResourceType: sourceType, SourceLogicalID: sourceID, + TargetResourceType: targetType, TargetLogicalID: targetID, + SourcePath: "graph20", SourceOrdinal: ordinal, + }) + if err != nil { + svc.fail("insert edge "+sourceID+"->"+targetID+" @"+label, err) + } + return seq +} + +func graph20RenderIn(t *testing.T, svc *pgir02Service, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderGraphPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + svc.fail("RenderGraphPhysicalPlan", err) + } + return rendered +} + +func graph20ExecPaths(t *testing.T, svc *pgir02Service, rendered command.RenderedPostgresPhysicalPlan) []map[string]any { + t.Helper() + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + rows, err := svc.iso.Query(svc.ctx, rendered.SQL, args...) + if err != nil { + svc.fail("execute graph statement "+svc.scrub(rendered.SQL), err) + } + defer rows.Close() + var paths []map[string]any + for rows.Next() { + var raw []byte + if err := rows.Scan(&raw); err != nil { + svc.fail("scan path object", err) + } + var object map[string]any + if err := json.Unmarshal(raw, &object); err != nil { + svc.fail("decode path object "+svc.scrub(string(raw)), err) + } + paths = append(paths, object) + } + if err := rows.Err(); err != nil { + svc.fail("drain path rows", err) + } + return paths +} + +func graph20Path(terminal string, nodes []map[string]any, relationships []map[string]any) map[string]any { + object := map[string]any{"terminalAlias": terminal, "nodes": nodes, "relationships": relationships} + if relationships == nil { + object["relationships"] = []map[string]any{} + } + return object +} + +func graph20Node(alias, resourceType, id, key string, payload string) map[string]any { + var decoded any + if err := json.Unmarshal([]byte(payload), &decoded); err != nil { + panic(err) + } + return map[string]any{"alias": alias, "resourceType": resourceType, "id": id, "key": key, "payload": decoded} +} + +func graph20Relationship(alias, label, fromType, toType string) map[string]any { + return map[string]any{"alias": alias, "label": label, "fromResourceType": fromType, "toResourceType": toType} +} + +func graph20AssertPaths(t *testing.T, got []map[string]any, want []map[string]any) { + t.Helper() + if len(got) != len(want) { + gotJSON, _ := json.Marshal(got) + wantJSON, _ := json.Marshal(want) + t.Fatalf("path sequence mismatch\n got %d rows: %s\nwant %d rows: %s", len(got), gotJSON, len(want), wantJSON) + } + normalizedWant := make([]map[string]any, len(want)) + { + encoded, err := json.Marshal(want) + if err != nil { + t.Fatalf("encode expected paths: %v", err) + } + if err := json.Unmarshal(encoded, &normalizedWant); err != nil { + t.Fatalf("decode expected paths: %v", err) + } + } + for index := range normalizedWant { + if !reflect.DeepEqual(got[index], normalizedWant[index]) { + gotJSON, _ := json.Marshal(got[index]) + wantJSON, _ := json.Marshal(normalizedWant[index]) + t.Fatalf("path %d mismatch\n got %s\nwant %s", index, gotJSON, wantJSON) + } + } +} + +func TestGraph20LivePathUnionOracle(t *testing.T) { + svc := graph20LiveService(t) + ctx := svc.ctx + + graph20SeedResource(t, svc, graph20Project, graph20Generation, "Patient", "p1", graph20AuthPath, json.RawMessage(`{"resourceType":"Patient","id":"p1","gender":"female"}`)) + graph20SeedResource(t, svc, graph20Project, graph20Generation, "Patient", "p2", graph20AuthPath, json.RawMessage(`{"resourceType":"Patient","id":"p2"}`)) + graph20SeedResource(t, svc, graph20Project, graph20Generation, "Patient", "p3", graph20AuthPath, json.RawMessage(`{"resourceType":"Patient","id":"p3"}`)) + graph20SeedResource(t, svc, graph20Project, graph20Generation, "Condition", "c1", "FHIR/Condition", json.RawMessage(`{"resourceType":"Condition","id":"c1","clinicalStatus":"active"}`)) + graph20SeedResource(t, svc, graph20Project, graph20Generation, "Observation", "o2", "FHIR/Observation", json.RawMessage(`{"resourceType":"Observation","id":"o2","status":"final"}`)) + graph20SeedResource(t, svc, "pgsq20-proj-b", graph20Generation, "Patient", "p1", "FHIR/Patient", json.RawMessage(`{"resourceType":"Patient","id":"p1","poison":"cross-project"}`)) + graph20SeedResource(t, svc, "pgsq20-proj-b", graph20Generation, "Condition", "cb", "FHIR/Condition", json.RawMessage(`{"resourceType":"Condition","id":"cb","poison":"cross-project-child"}`)) + graph20SeedResource(t, svc, graph20Project, "gen-2", "Condition", "cg", "FHIR/Condition", json.RawMessage(`{"resourceType":"Condition","id":"cg","poison":"cross-generation"}`)) + graph20SeedResource(t, svc, graph20Project, graph20Generation, "Condition", "c2", "FHIR/Confidential", json.RawMessage(`{"resourceType":"Condition","id":"c2","secret":true}`)) + + duplicateFirst := graph20SeedEdge(t, svc, graph20Project, graph20Generation, "FHIR/Condition", "subject_Patient", "Condition", "c1", "Patient", "p1") + duplicateSecond := graph20SeedEdge(t, svc, graph20Project, graph20Generation, "FHIR/Condition", "subject_Patient", "Condition", "c1", "Patient", "p1", 1) + if duplicateFirst == duplicateSecond { + t.Fatalf("parallel edges must be distinct occurrences, both got seq %d", duplicateFirst) + } + graph20SeedEdge(t, svc, graph20Project, graph20Generation, "FHIR/Observation", "focus_Condition", "Observation", "o2", "Condition", "c1") + graph20SeedEdge(t, svc, "pgsq20-proj-b", graph20Generation, "FHIR/Condition", "subject_Patient", "Condition", "cb", "Patient", "p1") + graph20SeedResource(t, svc, graph20Project, "gen-2", "Patient", "p1", graph20AuthPath, json.RawMessage(`{"resourceType":"Patient","id":"p1"}`)) + graph20SeedEdge(t, svc, graph20Project, "gen-2", "FHIR/Condition", "subject_Patient", "Condition", "cg", "Patient", "p1") + graph20SeedEdge(t, svc, graph20Project, graph20Generation, "FHIR/Confidential", "subject_Patient", "Condition", "c2", "Patient", "p1") + + var occurrences int + if err := svc.iso.QueryRow(ctx, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_edge_occurrence" WHERE label = 'subject_Patient' AND source_resource_type = 'Condition' AND source_logical_id = 'c1' AND target_resource_type = 'Patient' AND target_logical_id = 'p1' AND project = $1 AND dataset_generation = $2`, + graph20Project, graph20Generation).Scan(&occurrences); err != nil { + svc.fail("count parallel edges", err) + } + if occurrences != 2 { + t.Fatalf("dedupe fixture precondition: expected 2 parallel occurrences, stored %d", occurrences) + } + + plan := graph20PlanFor(t, graph20Root( + graph20Condition("cond", spec.TraversalMatchRequired, graph20FocusedObservation("obs2", spec.TraversalMatchRequired)), + graph20Observation("obs", spec.TraversalMatchOptional), + ), 10) + plan.BindVars["auth_resource_paths"] = []string{"FHIR/Patient", "FHIR/Condition", "FHIR/Observation"} + rendered := graph20RenderIn(t, svc, plan) + got := graph20ExecPaths(t, svc, rendered) + want := []map[string]any{ + graph20Path("cond", []map[string]any{ + graph20Node("root", "Patient", "p1", "p1", `{"resourceType":"Patient","id":"p1","gender":"female"}`), + graph20Node("cond", "Condition", "c1", "c1", `{"resourceType":"Condition","id":"c1","clinicalStatus":"active"}`), + }, []map[string]any{graph20Relationship("cond", "subject_Patient", "Patient", "Condition")}), + graph20Path("obs2", []map[string]any{ + graph20Node("root", "Patient", "p1", "p1", `{"resourceType":"Patient","id":"p1","gender":"female"}`), + graph20Node("cond", "Condition", "c1", "c1", `{"resourceType":"Condition","id":"c1","clinicalStatus":"active"}`), + graph20Node("obs2", "Observation", "o2", "o2", `{"resourceType":"Observation","id":"o2","status":"final"}`), + }, []map[string]any{ + graph20Relationship("cond", "subject_Patient", "Patient", "Condition"), + graph20Relationship("obs2", "focus_Condition", "Condition", "Observation"), + }), + } + graph20AssertPaths(t, got, want) + if len(got) != 2 { + t.Fatal("alive control: the parallel-edge duplicate must dedupe to exactly one cond path, and the optional miss contribute nothing") + } + + plan = graph20PlanFor(t, graph20Root( + graph20Condition("cond", spec.TraversalMatchRequired, graph20FocusedObservation("obs2", spec.TraversalMatchRequired)), + graph20Observation("obs", spec.TraversalMatchOptional), + ), 1) + plan.BindVars["auth_resource_paths"] = []string{"FHIR/Patient", "FHIR/Condition", "FHIR/Observation"} + lookahead := graph20ExecPaths(t, svc, graph20RenderIn(t, svc, plan)) + if len(lookahead) != 2 { + lookaheadJSON, _ := json.Marshal(lookahead) + t.Fatalf("limit+1 lookahead over deduped paths must return 2 rows (dedupe precedes the bound), got %s", lookaheadJSON) + } + if got := graph20ArgsValues(graph20RenderIn(t, svc, plan)); got[len(got)-1] != int64(2) { + t.Fatalf("lookahead bound argument must be limit+1=2, got %v", got[len(got)-1]) + } + + plan = graph20PlanFor(t, graph20Root(graph20Observation("obs", spec.TraversalMatchOptional)), 10) + plan.BindVars["auth_resource_paths"] = []string{"FHIR/Patient"} + roots := graph20ExecPaths(t, svc, graph20RenderIn(t, svc, plan)) + wantRoots := []map[string]any{ + graph20Path("root", []map[string]any{graph20Node("root", "Patient", "p1", "p1", `{"resourceType":"Patient","id":"p1","gender":"female"}`)}, nil), + graph20Path("root", []map[string]any{graph20Node("root", "Patient", "p2", "p2", `{"resourceType":"Patient","id":"p2"}`)}, nil), + graph20Path("root", []map[string]any{graph20Node("root", "Patient", "p3", "p3", `{"resourceType":"Patient","id":"p3"}`)}, nil), + } + graph20AssertPaths(t, roots, wantRoots) + + graph20SeedResource(t, svc, graph20Project, graph20Generation, "Procedure", "pr3", "FHIR/Procedure", json.RawMessage(`{"resourceType":"Procedure","id":"pr3","status":"completed"}`)) + graph20SeedEdge(t, svc, graph20Project, graph20Generation, "FHIR/Procedure", "subject_Patient", "Procedure", "pr3", "Patient", "p3") + plan = graph20PlanFor(t, graph20Root( + graph20Condition("cond", spec.TraversalMatchOptional), + graph20Procedure("proc", spec.TraversalMatchOptional), + ), 10) + plan.BindVars["auth_resource_paths"] = []string{"FHIR/Patient", "FHIR/Condition", "FHIR/Procedure"} + siblings := graph20ExecPaths(t, svc, graph20RenderIn(t, svc, plan)) + wantSiblings := []map[string]any{ + graph20Path("root", []map[string]any{graph20Node("root", "Patient", "p1", "p1", `{"resourceType":"Patient","id":"p1","gender":"female"}`)}, nil), + graph20Path("root", []map[string]any{graph20Node("root", "Patient", "p2", "p2", `{"resourceType":"Patient","id":"p2"}`)}, nil), + graph20Path("root", []map[string]any{graph20Node("root", "Patient", "p3", "p3", `{"resourceType":"Patient","id":"p3"}`)}, nil), + graph20Path("cond", []map[string]any{ + graph20Node("root", "Patient", "p1", "p1", `{"resourceType":"Patient","id":"p1","gender":"female"}`), + graph20Node("cond", "Condition", "c1", "c1", `{"resourceType":"Condition","id":"c1","clinicalStatus":"active"}`), + }, []map[string]any{graph20Relationship("cond", "subject_Patient", "Patient", "Condition")}), + graph20Path("proc", []map[string]any{ + graph20Node("root", "Patient", "p3", "p3", `{"resourceType":"Patient","id":"p3"}`), + graph20Node("proc", "Procedure", "pr3", "pr3", `{"resourceType":"Procedure","id":"pr3","status":"completed"}`), + }, []map[string]any{graph20Relationship("proc", "subject_Patient", "Patient", "Procedure")}), + } + graph20AssertPaths(t, siblings, wantSiblings) + + plan = graph20PlanFor(t, graph20Root(graph20Condition("cond", spec.TraversalMatchOptional)), 10) + plan.BindVars["auth_resource_paths"] = []string{"FHIR/Patient", "FHIR/Condition"} + authDenied := graph20ExecPaths(t, svc, graph20RenderIn(t, svc, plan)) + for _, path := range authDenied { + if terminal, _ := path["terminalAlias"].(string); terminal == "cond" { + nodesJSON, _ := json.Marshal(path["nodes"]) + if strings.Contains(string(nodesJSON), `"c2"`) || strings.Contains(string(nodesJSON), "Confidential") { + t.Fatalf("auth-denied condition leaked: %s", nodesJSON) + } + } + } + if len(authDenied) != 4 { + authJSON, _ := json.Marshal(authDenied) + t.Fatalf("auth-denial control expects 4 paths (3 roots + 1 deduped cond), got %s", authJSON) + } + + var poison int + if err := svc.iso.QueryRow(ctx, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" WHERE project = $1 OR dataset_generation = $2`, + "pgsq20-proj-b", "gen-2").Scan(&poison); err != nil { + svc.fail("count poison rows", err) + } + if poison < 2 { + t.Fatalf("scope-control precondition: expected cross-scope poison rows, found %d", poison) + } +} + +func TestGraph20LiveExplainProofs(t *testing.T) { + svc := graph20LiveService(t) + graph20SeedResource(t, svc, graph20Project, graph20Generation, "Patient", "p1", graph20AuthPath, json.RawMessage(`{"resourceType":"Patient","id":"p1"}`)) + graph20SeedResource(t, svc, graph20Project, graph20Generation, "Condition", "c1", "FHIR/Condition", json.RawMessage(`{"resourceType":"Condition","id":"c1"}`)) + graph20SeedResource(t, svc, graph20Project, graph20Generation, "Observation", "o2", "FHIR/Observation", json.RawMessage(`{"resourceType":"Observation","id":"o2"}`)) + graph20SeedEdge(t, svc, graph20Project, graph20Generation, "FHIR/Condition", "subject_Patient", "Condition", "c1", "Patient", "p1") + graph20SeedEdge(t, svc, graph20Project, graph20Generation, "FHIR/Observation", "focus_Condition", "Observation", "o2", "Condition", "c1") + + plan := graph20PlanFor(t, graph20Root(graph20Condition("cond", spec.TraversalMatchRequired, graph20FocusedObservation("obs2", spec.TraversalMatchRequired))), 4) + rendered := graph20RenderIn(t, svc, plan) + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + + if _, err := svc.iso.Exec(svc.ctx, "EXPLAIN "+rendered.SQL, args...); err != nil { + t.Fatalf("plain EXPLAIN must plan: %s", svc.scrub(err.Error())) + } + + var raw []byte + if err := svc.iso.QueryRow(svc.ctx, "EXPLAIN (FORMAT JSON) "+rendered.SQL, args...).Scan(&raw); err != nil { + svc.fail("EXPLAIN FORMAT JSON", err) + } + var decoded []struct { + Plan map[string]any `json:"Plan"` + } + if err := json.Unmarshal(raw, &decoded); err != nil || len(decoded) == 0 || decoded[0].Plan == nil { + t.Fatalf("EXPLAIN JSON decode: %v — %s", err, svc.scrub(string(raw))) + } + tree := decoded[0].Plan + + for _, node := range graph20WalkPlanNodes(tree) { + if nodeType, _ := node["Node Type"].(string); strings.Contains(strings.ToLower(nodeType), "recursive") { + t.Fatalf("recursive node in plan: %v", nodeType) + } + if cteName, _ := node["CTE Name"].(string); cteName != "" && strings.Contains(strings.ToLower(cteName), "recursive") { + t.Fatalf("recursive CTE in plan: %s", cteName) + } + } + + rootType, _ := tree["Node Type"].(string) + if rootType != "Limit" { + t.Fatalf("root plan node = %q, want Limit above everything", rootType) + } + if graph20FindNode(tree, func(node map[string]any) bool { + nodeType, _ := node["Node Type"].(string) + return nodeType == "WindowAgg" + }) == nil { + t.Fatalf("dedupe WindowAgg missing from plan:\n%s", svc.scrub(string(raw))) + } + + if graph20FindNode(tree, func(node map[string]any) bool { + nodeType, _ := node["Node Type"].(string) + joinType, _ := node["Join Type"].(string) + if strings.Contains(nodeType, "Semi") || joinType == "Semi" { + return true + } + for _, key := range []string{"Filter", "Index Cond", "Join Filter", "Subplan Name", "One-Time Filter"} { + if text, _ := node[key].(string); strings.Contains(text, "SubPlan") { + return true + } + } + return false + }) == nil { + t.Fatalf("REQUIRED-match semi-join missing from plan:\n%s", svc.scrub(string(raw))) + } + if graph20FindNode(tree, func(node map[string]any) bool { + relation, _ := node["Relation Name"].(string) + return relation == "fhir_edge_occurrence" + }) == nil { + t.Fatalf("edge occurrence scan missing from plan:\n%s", svc.scrub(string(raw))) + } +} + +func graph20WalkPlanNodes(node map[string]any) []map[string]any { + nodes := []map[string]any{node} + if children, ok := node["Plans"].([]any); ok { + for _, child := range children { + if childMap, ok := child.(map[string]any); ok { + nodes = append(nodes, graph20WalkPlanNodes(childMap)...) + } + } + } + return nodes +} + +func graph20FindNode(node map[string]any, want func(map[string]any) bool) map[string]any { + for _, candidate := range graph20WalkPlanNodes(node) { + if want(candidate) { + return candidate + } + } + return nil +} diff --git a/internal/dataframe/compiler/render/postgres/group.go b/internal/dataframe/compiler/render/postgres/group.go new file mode 100644 index 000000000..1b16793e2 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/group.go @@ -0,0 +1,1107 @@ +package postgres + +import ( + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const groupRowIDColumn = "__loom_row_id" + +const ( + groupMissingKeyTag = "CONSTRUCTION_GROUP_MISSING_KEY" + groupKeyTypeTag = "CONSTRUCTION_GROUP_KEY_TYPE_MISMATCH" + groupValueTypeTag = "CONSTRUCTION_GROUP_VALUE_TYPE_MISMATCH" + groupRowValueMultipleTag = "CONSTRUCTION_ROW_VALUE_MULTIPLE_VALUES" +) + +const groupTrimCharacters = " " + +func groupPlanMentionsStage(plan ir.PhysicalPlan) bool { + if plan.StageSequence == nil { + return false + } + for _, stage := range plan.StageSequence.Stages { + if stage.Kind == ir.PhysicalStageGroupOp { + return true + } + } + return false +} + +func renderGroupPhysicalPlan(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + if err := storage.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := validateGroupPlanSurface(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := plan.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("validate physical plan: %w", err) + } + if plan.Engine != "" && plan.Engine != ir.PhysicalEngineAQL { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: engine %q has no PostgreSQL source; the source graph IR is backend-neutral and ClickHouse plans are not PostgreSQL-rendered", + ErrUnsupportedPostgresPlan, string(plan.Engine)) + } + if plan.ClickHouseCombine != nil || plan.ClickHousePrefix != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: ClickHouse combine/prefix payloads have no PostgreSQL rendering", ErrUnsupportedPostgresPlan) + } + if len(plan.DeferredExpressionLets) != 0 { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: completed plans must not carry deferred expression lets", ErrUnsupportedPostgresPlan) + } + renderer := &planRenderer{ + storage: storage, + rows: map[string]rowScope{}, + parameters: map[string]int{}, + canonicalAuthLets: map[string]string{}, + } + if err := renderer.loadScope(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + sql, err := renderer.renderGroupStatement(plan) + if err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return command.NewRenderedPostgresPhysicalPlan(sql, renderer.args, renderer.outputColumns) +} + +func validateGroupPlanSurface(plan ir.PhysicalPlan) error { + if plan.StageSequence == nil { + return fmt.Errorf("%w: group dispatch requires a stage sequence", ErrUnsupportedPostgresPlan) + } + sequence := plan.StageSequence + if len(sequence.Stages) != 1 { + return fmt.Errorf("%w: the registered group slice admits exactly one GROUP construction stage; multi-stage composition is a pg-sql-16 capability gap (L-13.1)", + ErrUnsupportedPostgresPlan) + } + stage := sequence.Stages[0] + if stage.Kind != ir.PhysicalStageGroupOp || stage.Group == nil { + return fmt.Errorf("%w: operation %q is not part of the registered group slice", + ErrUnsupportedPostgresPlan, string(stage.Kind)) + } + if stage.CodedGroup != nil || stage.CohortGroup != nil || stage.Filter != nil || stage.Expand != nil || + stage.GroupedPivot != nil || stage.Unpivot != nil || stage.RelatedSource != nil || + stage.RelatedExpand != nil || stage.RelatedField != nil || len(stage.DerivedLets) != 0 { + return fmt.Errorf("%w: GROUP requires only a group payload (ir/physical_stage_validation.go:187 re-read at the door)", + ErrUnsupportedPostgresPlan) + } + if stage.InputStageID != sequence.SourceStageID || stage.ID != sequence.FinalStageID { + return fmt.Errorf("%w: the registered slice admits only the inline first-stage GROUP (InputStageID must equal %q); composed sequences are a pg-sql-16 capability gap (L-13.1)", + ErrUnsupportedPostgresPlan, sequence.SourceStageID) + } + if sequence.CellTraceReturn != nil || sequence.RowLineageReturn != nil || sequence.PopulationMappingReturn != nil { + return fmt.Errorf("%w: row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route (compiler/row_lineage.go:655, cell_trace.go:54, population_mapping.go:38)", + ErrUnsupportedPostgresPlan) + } + if plan.PreviewSourceWindowByRootID || sequence.PreviewSourceWindowByRootID || sequence.PreviewTerminalPivotWindow { + return fmt.Errorf("%w: preview source windowing and the terminal Pivot window require Pivot terminals and are not registered for GROUP", ErrUnsupportedPostgresPlan) + } + if sequence.OutputAuthResourcePathBindKey != "" { + return fmt.Errorf("%w: the composite Combine prefix authorization path is not registered for GROUP", ErrUnsupportedPostgresPlan) + } + if sequence.PreviewLimitBindKey != "" { + if value, ok := plan.BindVars[sequence.PreviewLimitBindKey].(int); !ok || value <= 0 { + return fmt.Errorf("%w: group preview limit bind %q must be a positive int", + ErrUnsupportedPostgresPlan, sequence.PreviewLimitBindKey) + } + } + returnNames, err := validateGroupOperations(plan) + if err != nil { + return err + } + if err := validateGroupPayload(stage, plan.BindVars, returnNames); err != nil { + return err + } + return validateGroupSemanticKey(plan, stage) +} + +func validateGroupPayload(stage ir.PhysicalConstructionStage, bindVars map[string]any, returnNames map[string]bool) error { + group := stage.Group + if value, ok := bindVars[group.ConstructionIDBindKey].(string); !ok || strings.TrimSpace(value) == "" { + return fmt.Errorf("%w: group construction identity bind %q must be a non-empty string", + ErrUnsupportedPostgresPlan, group.ConstructionIDBindKey) + } + switch group.MissingKeyPolicy { + case ir.PhysicalStageGroupMissingKeyGroup, ir.PhysicalStageGroupMissingKeyExclude, ir.PhysicalStageGroupMissingKeyError: + default: + return fmt.Errorf("%w: unsupported missing-key policy %q (construction_group.go:131 parity)", + ErrUnsupportedPostgresPlan, string(group.MissingKeyPolicy)) + } + if group.SemanticKey != nil && group.CodingKey != nil { + return fmt.Errorf("%w: Group supports at most one coded identity key (ir/physical_stage_validation.go:1475 re-read at the door)", + ErrUnsupportedPostgresPlan) + } + if group.RelatedOccurrence != nil { + return fmt.Errorf("%w: the related-occurrence Group adapter is a pg-sql-16 capability gap (L-13.5; correlated-hop child-set contracts) and is refused, never partially rendered", + ErrUnsupportedPostgresPlan) + } + if group.RootContributorInputColumn != "" { + return fmt.Errorf("%w: the root-contributor set is production-unreachable through the PostgreSQL compile target (L-13.6) and is refused, never partially rendered", + ErrUnsupportedPostgresPlan) + } + if group.GroupRowsVariable == "" || group.IdentityVariable == "" { + return fmt.Errorf("%w: group plan variables must be compiled and non-empty", ErrUnsupportedPostgresPlan) + } + for index, key := range group.Keys { + switch key.Kind { + case "STRING", "INTEGER", "DECIMAL", "BOOLEAN": + default: + return fmt.Errorf("%w: group key %d has unsupported scalar kind %q", + ErrUnsupportedPostgresPlan, index, key.Kind) + } + if key.InputColumn == "" || key.OutputColumn == "" || key.Variable == "" { + return fmt.Errorf("%w: group key %d must be fully compiled (input, output, variable)", + ErrUnsupportedPostgresPlan, index) + } + if key.InputInternal || key.FieldSource != nil { + return fmt.Errorf("%w: group key %d uses a hidden or field-sourced input surface, which the public-scalar lowering gate never emits (L-13.11)", + ErrUnsupportedPostgresPlan, index) + } + if !returnNames[key.InputColumn] { + return fmt.Errorf("%w: group key %d: group input column %q is not among the source projection names (L-13.11)", + ErrUnsupportedPostgresPlan, index, key.InputColumn) + } + } + keyless := len(group.Keys) == 0 && group.SemanticKey == nil + for index, aggregate := range group.Aggregates { + switch aggregate.Operation { + case "COUNT_ROWS", "COUNT_NON_NULL", "COUNT_DISTINCT", "MIN", "MAX", "SUM", "MEAN": + default: + return fmt.Errorf("%w: aggregate %d: unsupported aggregate operation %q (construction_group.go:807-808 parity)", + ErrUnsupportedPostgresPlan, index, string(aggregate.Operation)) + } + if aggregate.Operation == "COUNT_ROWS" { + if aggregate.InputColumn != "" { + return fmt.Errorf("%w: aggregate %d COUNT_ROWS must not have an input column", ErrUnsupportedPostgresPlan, index) + } + } else if aggregate.InputColumn == "" { + return fmt.Errorf("%w: aggregate %d %s requires an input column", + ErrUnsupportedPostgresPlan, index, string(aggregate.Operation)) + } + if aggregate.Output == "" || aggregate.Variable == "" { + return fmt.Errorf("%w: aggregate %d must be fully compiled (output, variable)", ErrUnsupportedPostgresPlan, index) + } + if aggregate.InputColumn != "" && !returnNames[aggregate.InputColumn] { + return fmt.Errorf("%w: aggregate %d: group input column %q is not among the source projection names (L-13.11)", + ErrUnsupportedPostgresPlan, index, aggregate.InputColumn) + } + } + if keyless { + if len(group.Aggregates) != 1 || group.Aggregates[0].Operation != "COUNT_ROWS" { + return fmt.Errorf("%w: only keyless COUNT_ROWS groups can consume the source scope (construction_group.go:44-52 re-read at the door)", + ErrUnsupportedPostgresPlan) + } + } + for index, rowValue := range group.RowValues { + switch rowValue.Policy { + case "ALL", "ONE": + default: + return fmt.Errorf("%w: row value %q has unsupported policy %q", + ErrUnsupportedPostgresPlan, rowValue.Output, rowValue.Policy) + } + if rowValue.InputColumn == "" || rowValue.Output == "" || rowValue.Variable == "" { + return fmt.Errorf("%w: row value %d must be fully compiled (input, output, variable)", + ErrUnsupportedPostgresPlan, index) + } + if rowValue.FieldSource != nil { + return fmt.Errorf("%w: row value %d uses a field-sourced input surface, which the public-scalar lowering gate never emits (L-13.11)", + ErrUnsupportedPostgresPlan, index) + } + if !returnNames[rowValue.InputColumn] { + return fmt.Errorf("%w: row value %d: group input column %q is not among the source projection names (L-13.11)", + ErrUnsupportedPostgresPlan, index, rowValue.InputColumn) + } + } + if group.CodingKey != nil { + return fmt.Errorf("%w: the CODING_IDENTITY Group key is refused, never partially rendered (L-13.4: route-sourced keys name pg-sql-16; the local-array-expansion key is wave 2 of this ticket)", + ErrUnsupportedPostgresPlan) + } + return nil +} + +func validateGroupOperations(plan ir.PhysicalPlan) (map[string]bool, error) { + if len(plan.Operations) == 0 { + return nil, fmt.Errorf("%w: plan has no operations", ErrUnsupportedPostgresPlan) + } + first := plan.Operations[0] + if first.Kind != ir.PhysicalRootScanOp || first.RootScan == nil { + return nil, fmt.Errorf("%w: the group source requires exactly one leading ROOT_SCAN", ErrUnsupportedPostgresPlan) + } + scan := first.RootScan + if scan.Population != nil || scan.CohortSource != nil || scan.PageCandidateSeed != nil { + return nil, fmt.Errorf("%w: population, cohort, and candidate-seed roots are bound only by RenderPopulationRootScan (population.go); the plain spine refuses them", + ErrUnsupportedPostgresPlan) + } + sawSort, sawLimit, sawReturn := false, false, false + var returnNames map[string]bool + for index, operation := range plan.Operations { + last := index == len(plan.Operations)-1 + switch operation.Kind { + case ir.PhysicalRootScanOp: + if index != 0 { + return nil, fmt.Errorf("%w: exactly one leading ROOT_SCAN is required", ErrUnsupportedPostgresPlan) + } + case ir.PhysicalFilterOp, ir.PhysicalDerivedLetOp: + if sawSort || sawLimit || sawReturn { + return nil, fmt.Errorf("%w: source operations must precede the execution window and the terminal RETURN", + ErrUnsupportedPostgresPlan) + } + if operation.Kind == ir.PhysicalFilterOp && operation.Filter == nil { + return nil, fmt.Errorf("%w: operation %d: filter payload is missing", ErrUnsupportedPostgresPlan, index) + } + if operation.Kind == ir.PhysicalDerivedLetOp && operation.DerivedLet == nil { + return nil, fmt.Errorf("%w: operation %d: derived-let payload is missing", ErrUnsupportedPostgresPlan, index) + } + case ir.PhysicalUnnestOp: + return nil, fmt.Errorf("%w: operation %q is not part of the registered group source surface (L-13.9)", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + case ir.PhysicalSetOp: + return nil, fmt.Errorf("%w: group source-region SET is a pg-sql-16 capability gap (no PostgreSQL contract for lowering-emitted child-set subplans; L-13.2) and is refused, never partially rendered", + ErrUnsupportedPostgresPlan) + case ir.PhysicalSortOp: + if sawSort || sawLimit || sawReturn { + return nil, fmt.Errorf("%w: at most one SORT, before LIMIT", ErrUnsupportedPostgresPlan) + } + if operation.Sort == nil { + return nil, fmt.Errorf("%w: operation %d: sort payload is missing", ErrUnsupportedPostgresPlan, index) + } + sawSort = true + case ir.PhysicalLimitOp: + if sawLimit || sawReturn { + return nil, fmt.Errorf("%w: at most one LIMIT", ErrUnsupportedPostgresPlan) + } + if operation.Limit == nil { + return nil, fmt.Errorf("%w: operation %d: limit payload is missing", ErrUnsupportedPostgresPlan, index) + } + sawLimit = true + case ir.PhysicalReturnOp: + if !last || operation.Return == nil || len(operation.Return.Projections) == 0 { + return nil, fmt.Errorf("%w: the group source requires a terminal RETURN with its projection list", + ErrUnsupportedPostgresPlan) + } + sawReturn = true + returnNames = map[string]bool{} + for _, projection := range operation.Return.Projections { + if projection.Name == groupSemanticCarrierName { + return nil, fmt.Errorf("%w: source projection name %q is reserved for the renderer's semantic carrier column", + ErrUnsupportedPostgresPlan, projection.Name) + } + returnNames[projection.Name] = true + } + default: + return nil, fmt.Errorf("%w: operation %q is not part of the registered group source surface", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + if !sawReturn { + return nil, fmt.Errorf("%w: the group source requires a terminal RETURN", ErrUnsupportedPostgresPlan) + } + return returnNames, nil +} + +type groupColumn struct { + name string + alias string + expr string + ref string + jsonb bool +} + +func (r *planRenderer) renderGroupStatement(plan ir.PhysicalPlan) (string, error) { + stage := plan.StageSequence.Stages[0] + + if err := r.renderRootScan(plan, *plan.Operations[0].RootScan); err != nil { + return "", err + } + for _, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalFilterOp: + canonical, err := r.acceptCanonicalScopeFilter(operation.Filter) + if err != nil { + return "", err + } + if canonical { + continue + } + clause, err := r.renderFilter(plan, *operation.Filter) + if err != nil { + return "", err + } + r.where = append(r.where, clause) + case ir.PhysicalDerivedLetOp: + canonical, err := r.acceptCanonicalAuthLet(operation.DerivedLet) + if err != nil { + return "", err + } + if !canonical { + return "", fmt.Errorf("%w: operation %q is not part of the registered group source surface", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + } + for _, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalSortOp: + if err := r.renderWindowSort(plan, *operation.Sort); err != nil { + return "", err + } + case ir.PhysicalLimitOp: + if err := r.renderWindowLimit(plan, *operation.Limit); err != nil { + return "", err + } + } + } + + var returnOp *ir.PhysicalReturn + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalReturnOp { + returnOp = operation.Return + } + } + columns, err := r.materializeGroupSource(plan, returnOp) + if err != nil { + return "", err + } + if stage.Group.SemanticKey != nil { + carrier, err := r.groupSemanticCarrierColumn(stage.Group.SemanticKey) + if err != nil { + return "", err + } + columns = append(columns, *carrier) + } + resource := r.storage.Resource + selectList := make([]string, 0, len(columns)) + for i := range columns { + selectList = append(selectList, columns[i].expr+` AS "`+columns[i].alias+`"`) + } + source := "SELECT " + strings.Join(selectList, ", ") + + " FROM " + resource.Identifier().SQLText() + " AS " + r.rootAlias + for _, join := range r.joins { + source += " " + join + } + for _, lateral := range r.laterals { + source += " " + lateral + } + if len(r.where) > 0 { + source += " WHERE " + strings.Join(r.where, " AND ") + } + source += " ORDER BY " + strings.Join(r.windowOrder(), ", ") + r.window.limitClause + + region, err := r.renderGroupAggregateRegion(plan, stage, columns, source, "g_", + plan.StageSequence.FinalColumns, groupRowIDColumn) + if err != nil { + return "", err + } + r.outputColumns = region.names + final := region.final + " ORDER BY " + strings.Join(region.order, ", ") + if plan.StageSequence.PreviewLimitBindKey != "" { + limit := plan.BindVars[plan.StageSequence.PreviewLimitBindKey].(int) + index := r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(limit)}, "") + final += " LIMIT $" + strconv.Itoa(index) + } + return "WITH " + strings.Join(region.ctes, ", ") + " " + final, nil +} + +type groupRegionResult struct { + ctes []string + final string + order []string + names []string + keyCollates []string +} + +func (r *planRenderer) renderGroupAggregateRegion( + plan ir.PhysicalPlan, + stage ir.PhysicalConstructionStage, + columns []groupColumn, + sourceText string, + prefix string, + outputColumns []ir.PhysicalStageColumn, + identityColumn string, +) (groupRegionResult, error) { + group := stage.Group + nameToColumn := map[string]*groupColumn{} + var semanticCarrier *groupColumn + for i := range columns { + if columns[i].name == groupSemanticCarrierName { + if semanticCarrier != nil { + return groupRegionResult{}, fmt.Errorf("%w: the semantic carrier column is declared twice", + ErrUnsupportedPostgresPlan) + } + semanticCarrier = &columns[i] + continue + } + nameToColumn[columns[i].name] = &columns[i] + } + + keyColumns := make([]*groupColumn, len(group.Keys)) + for i, key := range group.Keys { + column, err := groupResolveColumn(nameToColumn, key.InputColumn) + if err != nil { + return groupRegionResult{}, fmt.Errorf("group key %q: %w", key.OutputColumn, err) + } + keyColumns[i] = column + } + type groupedAggregate struct { + aggregate ir.PhysicalStageGroupAggregate + column *groupColumn + alias string + } + aggregates := make([]groupedAggregate, 0, len(group.Aggregates)) + for i, aggregate := range group.Aggregates { + var column *groupColumn + if aggregate.InputColumn != "" { + resolved, err := groupResolveColumn(nameToColumn, aggregate.InputColumn) + if err != nil { + return groupRegionResult{}, fmt.Errorf("group aggregate %q: %w", aggregate.Output, err) + } + column = resolved + } + aggregates = append(aggregates, groupedAggregate{ + aggregate: aggregate, column: column, alias: prefix + "a" + strconv.Itoa(i), + }) + } + type groupedRowValue struct { + rowValue ir.PhysicalStageRowValue + column *groupColumn + alias string + } + rowValues := make([]groupedRowValue, 0, len(group.RowValues)) + for i, rowValue := range group.RowValues { + column, err := groupResolveColumn(nameToColumn, rowValue.InputColumn) + if err != nil { + return groupRegionResult{}, fmt.Errorf("group row value %q: %w", rowValue.Output, err) + } + rowValues = append(rowValues, groupedRowValue{ + rowValue: rowValue, column: column, alias: prefix + "rv" + strconv.Itoa(i), + }) + } + + var contributorColumn *groupColumn + contributorAlias := "" + if group.RootContributorInputColumn != "" { + if r.mapping == nil || !r.mapping.witness { + return groupRegionResult{}, fmt.Errorf("%w: the root-contributor set is production-unreachable through the PostgreSQL compile target (L-13.6) and is refused, never partially rendered", + ErrUnsupportedPostgresPlan) + } + if group.RootContributorOutputColumn != populationMappingContributorColumn || group.RootContributorVariable == "" { + return groupRegionResult{}, fmt.Errorf("%w: the composed root-contributor set must materialize the compiler-owned mapping contributor column %q with its compiled variable", + ErrUnsupportedPostgresPlan, populationMappingContributorColumn) + } + resolved, err := groupResolveColumn(nameToColumn, group.RootContributorInputColumn) + if err != nil { + return groupRegionResult{}, fmt.Errorf("group root contributors: %w", err) + } + contributorColumn = resolved + contributorAlias = prefix + "rc" + } + + dollar := func(index int) string { return "$" + strconv.Itoa(index) } + + var semantic *groupSemanticRegion + if group.SemanticKey != nil { + region, err := r.renderGroupSemanticRegion(group.SemanticKey, semanticCarrier, stage, prefix, dollar) + if err != nil { + return groupRegionResult{}, fmt.Errorf("group semantic key %q: %w", group.SemanticKey.OutputColumn, err) + } + semantic = ®ion + } else if semanticCarrier != nil { + return groupRegionResult{}, fmt.Errorf("%w: the semantic carrier column is minted only beside a semantic family key", + ErrUnsupportedPostgresPlan) + } + + var conjuncts []string + keyReads := make([]string, len(group.Keys)) + keyCollates := make([]string, len(group.Keys)) + for i, key := range group.Keys { + typedRead, witness, err := r.groupColumnRead(keyColumns[i], key.Kind, groupKeyTypeTag, dollar) + if err != nil { + return groupRegionResult{}, fmt.Errorf("group key %q: %w", key.OutputColumn, err) + } + keyReads[i] = typedRead + if key.Kind == "STRING" { + keyCollates[i] = ` COLLATE "C"` + } + if witness != "" { + conjuncts = append(conjuncts, witness) + } + switch group.MissingKeyPolicy { + case ir.PhysicalStageGroupMissingKeyExclude: + conjuncts = append(conjuncts, typedRead+" IS NOT NULL") + case ir.PhysicalStageGroupMissingKeyError: + conjuncts = append(conjuncts, + "(CASE WHEN "+typedRead+" IS NOT NULL THEN 1 ELSE "+guardCast(groupMissingKeyTag)+" END) > 0") + } + } + baseKeyReads := append([]string(nil), keyReads...) + for _, item := range aggregates { + if item.column == nil { + continue + } + _, witness, err := r.groupColumnRead(item.column, item.aggregate.InputKind, groupValueTypeTag, dollar) + if err != nil { + return groupRegionResult{}, fmt.Errorf("group aggregate %q: %w", item.aggregate.Output, err) + } + if witness != "" { + conjuncts = append(conjuncts, witness) + } + } + for _, item := range rowValues { + if !item.rowValue.InputMany { + _, witness, err := r.groupColumnRead(item.column, item.rowValue.InputKind, groupValueTypeTag, dollar) + if err != nil { + return groupRegionResult{}, fmt.Errorf("group row value %q: %w", item.rowValue.Output, err) + } + if witness != "" { + conjuncts = append(conjuncts, witness) + } + continue + } + nullRead := groupNullRead(item.column) + conjuncts = append(conjuncts, "(CASE WHEN COALESCE("+nullRead+ + " IS NULL OR jsonb_typeof("+nullRead+") = 'array', false) THEN 1 ELSE "+ + guardCast(groupValueTypeTag)+" END) > 0") + } + + if semantic != nil { + switch group.MissingKeyPolicy { + case ir.PhysicalStageGroupMissingKeyExclude: + conjuncts = append(conjuncts, semantic.codeRead+" IS NOT NULL") + case ir.PhysicalStageGroupMissingKeyError: + conjuncts = append(conjuncts, + "(CASE WHEN "+semantic.codeRead+" IS NOT NULL THEN 1 ELSE "+guardCast(groupMissingKeyTag)+" END) > 0") + } + } + + deduped := make([]string, 0, len(conjuncts)) + seenConjunct := map[string]bool{} + for _, conjunct := range conjuncts { + if seenConjunct[conjunct] { + continue + } + seenConjunct[conjunct] = true + deduped = append(deduped, conjunct) + } + rows := "SELECT * FROM " + prefix + "src" + if semantic != nil { + rows += semantic.lateral + } + if len(deduped) > 0 { + rows += " WHERE " + strings.Join(deduped, " AND ") + } + + grpSelects := make([]string, 0, len(keyReads)+1+len(aggregates)+len(rowValues)) + keyAliases := make([]string, 0, len(keyReads)) + for i := range keyReads { + alias := prefix + "k" + strconv.Itoa(i) + grpSelects = append(grpSelects, keyReads[i]+" AS "+alias) + keyAliases = append(keyAliases, alias) + } + if semantic != nil { + for i, read := range semantic.tupleReads { + alias := semantic.tupleAliases[i] + keyReads = append(keyReads, read) + keyCollates = append(keyCollates, ` COLLATE "C"`) + grpSelects = append(grpSelects, read+" AS "+alias) + keyAliases = append(keyAliases, alias) + } + grpSelects = append(grpSelects, semantic.label+" AS "+prefix+"slb") + } + for _, item := range aggregates { + expression, err := r.groupAggregateExpression(item.aggregate, item.column, dollar) + if err != nil { + return groupRegionResult{}, fmt.Errorf("group aggregate %q: %w", item.aggregate.Output, err) + } + grpSelects = append(grpSelects, expression+" AS "+item.alias) + } + var manyCTEs, manyJoins []string + for i, item := range rowValues { + if item.rowValue.InputMany { + name := prefix + "m" + strconv.Itoa(i) + selectText, singleValueRef, err := r.groupManyRowValuePoolFor(item.rowValue.Policy, item.rowValue.InputKind, + item.column, keyReads, name, prefix+"rows", dollar) + if err != nil { + return groupRegionResult{}, fmt.Errorf("group row value %q: %w", item.rowValue.Output, err) + } + manyCTEs = append(manyCTEs, name+" AS ("+selectText+")") + grpSelects = append(grpSelects, singleValueRef+" AS "+item.alias) + conditions := make([]string, 0, len(keyReads)) + for j := range keyReads { + conditions = append(conditions, keyReads[j]+" IS NOT DISTINCT FROM "+name+".m_k"+strconv.Itoa(j)) + } + on := "TRUE" + if len(conditions) > 0 { + on = strings.Join(conditions, " AND ") + } + manyJoins = append(manyJoins, "LEFT JOIN "+name+" ON "+on) + continue + } + expression, err := groupRowValueExpressionFor(item.rowValue.Policy, item.rowValue.InputKind, + item.rowValue.InputMany, item.column, dollar) + if err != nil { + return groupRegionResult{}, fmt.Errorf("group row value %q: %w", item.rowValue.Output, err) + } + grpSelects = append(grpSelects, expression+" AS "+item.alias) + } + if contributorColumn != nil { + pool := prefix + "rcpool" + from := prefix + "rows" + memberRead := "" + if group.RootContributorInputMany { + if !contributorColumn.jsonb { + return groupRegionResult{}, fmt.Errorf("%w: a many-cardinality root-contributor input requires a jsonb source column", ErrUnsupportedPostgresPlan) + } + nullRead := groupNullRead(contributorColumn) + guard := "CASE WHEN " + nullRead + " IS NULL OR jsonb_typeof(" + nullRead + ") <> 'array' THEN '[]'::jsonb ELSE " + nullRead + " END" + from = prefix + "rows LEFT JOIN LATERAL jsonb_array_elements_text(" + guard + ") AS rcx(elem) ON true" + memberRead = "rcx.elem" + } else { + typedRead, _, err := r.groupColumnRead(contributorColumn, "STRING", groupValueTypeTag, dollar) + if err != nil { + return groupRegionResult{}, fmt.Errorf("group root contributors: %w", err) + } + memberRead = typedRead + } + poolSelects := make([]string, 0, len(baseKeyReads)+1) + for i, keyRead := range baseKeyReads { + poolSelects = append(poolSelects, keyRead+" AS m_k"+strconv.Itoa(i)) + } + distinct := memberRead + ` COLLATE "C"` + poolSelects = append(poolSelects, "to_jsonb(COALESCE(ARRAY_AGG(DISTINCT "+distinct+" ORDER BY "+distinct+"), '{}'::text[])) AS m_pool") + poolSelect := "SELECT " + strings.Join(poolSelects, ", ") + " FROM " + from + if len(baseKeyReads) > 0 { + positions := make([]string, len(baseKeyReads)) + for i := range positions { + positions[i] = strconv.Itoa(i + 1) + } + poolSelect += " GROUP BY " + strings.Join(positions, ", ") + } + manyCTEs = append(manyCTEs, pool+" AS ("+poolSelect+")") + conditions := make([]string, 0, len(baseKeyReads)) + for j := range baseKeyReads { + conditions = append(conditions, baseKeyReads[j]+" IS NOT DISTINCT FROM "+pool+".m_k"+strconv.Itoa(j)) + } + on := "TRUE" + if len(conditions) > 0 { + on = strings.Join(conditions, " AND ") + } + manyJoins = append(manyJoins, "LEFT JOIN "+pool+" ON "+on) + grpSelects = append(grpSelects, "COALESCE(MAX("+pool+".m_pool::text)::jsonb, '[]'::jsonb) AS "+contributorAlias) + } + + grouped := "SELECT " + strings.Join(grpSelects, ", ") + " FROM " + prefix + "rows" + if len(manyJoins) > 0 { + grouped += " " + strings.Join(manyJoins, " ") + } + if len(keyAliases) > 0 { + positions := make([]string, len(keyAliases)) + for i := range positions { + positions[i] = strconv.Itoa(i + 1) + } + grouped += " GROUP BY " + strings.Join(positions, ", ") + } + + constructionID := r.textParameter(bindStringVar(plan.BindVars, group.ConstructionIDBindKey), "") + outputIDs := map[string]string{} + for _, column := range stage.OutputColumns { + if _, seen := outputIDs[column.Name]; !seen { + outputIDs[column.Name] = column.ID + } + } + identityParts := []string{ + `'["construction",' || to_jsonb(` + dollar(constructionID) + "::text) || ']'", + `'["operation","GROUP"]'`, + } + appendKeyPart := func(columnName string, alias string) error { + columnID, ok := outputIDs[columnName] + if !ok { + return fmt.Errorf("%w: group key output %q has no stable column ID (construction_group.go:136-142 parity)", + ErrUnsupportedPostgresPlan, columnName) + } + idBind := r.textParameter(columnID, "") + identityParts = append(identityParts, + "'[' || to_jsonb("+dollar(idBind)+"::text) || ',' || COALESCE(to_jsonb(g."+alias+")::text, 'null') || ']'") + return nil + } + for i, key := range group.Keys { + if err := appendKeyPart(key.OutputColumn, prefix+"k"+strconv.Itoa(i)); err != nil { + return groupRegionResult{}, err + } + } + if semantic != nil { + identityParts = append(identityParts, semantic.identityPart) + } + identity := "'[' || " + strings.Join(identityParts, " || ',' || ") + " || ']'" + + expressions := map[string]string{} + for i, key := range group.Keys { + expressions[key.OutputColumn] = "g." + prefix + "k" + strconv.Itoa(i) + } + for _, item := range aggregates { + expressions[item.aggregate.Output] = "g." + item.alias + } + for _, item := range rowValues { + expressions[item.rowValue.Output] = "g." + item.alias + } + if semantic != nil { + expressions[group.SemanticKey.OutputColumn] = "g." + prefix + "slb" + } + if contributorColumn != nil { + expressions[group.RootContributorOutputColumn] = "g." + contributorAlias + } + expressions[identityColumn] = identity + selects := make([]string, 0, len(outputColumns)) + names := make([]string, 0, len(outputColumns)) + for _, column := range outputColumns { + expression, ok := expressions[column.Name] + if !ok { + return groupRegionResult{}, fmt.Errorf("%w: group final column %q is not a registered key, reducer, row value, or the stable identity", + ErrUnsupportedPostgresPlan, column.Name) + } + for _, seen := range names { + if seen == column.Name { + return groupRegionResult{}, fmt.Errorf("%w: group final column %q is declared twice", ErrUnsupportedPostgresPlan, column.Name) + } + } + selects = append(selects, expression+` AS "c`+strconv.Itoa(len(selects))+`"`) + names = append(names, column.Name) + } + emitted := len(group.Keys) + len(aggregates) + len(rowValues) + 1 + if semantic != nil { + emitted++ + } + if contributorColumn != nil { + emitted++ + } + if len(names) != emitted { + return groupRegionResult{}, fmt.Errorf("%w: the registered slice emits every key, reducer, and row value plus the hidden stable identity", + ErrUnsupportedPostgresPlan) + } + + final := "SELECT " + strings.Join(selects, ", ") + " FROM " + prefix + "grp AS g" + order := make([]string, 0, len(keyAliases)+1) + for i, alias := range keyAliases { + order = append(order, "g."+alias+keyCollates[i]+" ASC NULLS FIRST") + } + order = append(order, identity+` COLLATE "C" ASC`) + + ctes := []string{ + prefix + "src AS (" + sourceText + ")", + prefix + "rows AS (" + rows + ")", + } + ctes = append(ctes, manyCTEs...) + ctes = append(ctes, prefix+"grp AS ("+grouped+")") + return groupRegionResult{ctes: ctes, final: final, order: order, names: names, keyCollates: keyCollates}, nil +} + +func (r *planRenderer) materializeGroupSource(plan ir.PhysicalPlan, ret *ir.PhysicalReturn) ([]groupColumn, error) { + classified, err := classifyExtractionProjections(ret.Projections, func(variable string) (string, bool) { + scope, ok := r.rows[variable] + return scope.resourceType, ok + }) + if err != nil { + return nil, err + } + columns := make([]groupColumn, 0, len(classified)) + seen := map[string]bool{} + for i, projection := range classified { + if seen[projection.projection.Name] { + return nil, fmt.Errorf("%w: group source projection name %q is declared twice (L-13.11)", + ErrUnsupportedPostgresPlan, projection.projection.Name) + } + seen[projection.projection.Name] = true + var expression string + isJSONB := false + switch projection.kind { + case extractionDirectValue, extractionValueExpression: + value := projection.projection.Value + if projection.kind == extractionValueExpression { + value = *projection.projection.Expression.Value + } + if len(value.Path) == 0 { + return nil, fmt.Errorf("%w: group source projection %q is not a registered scalar projection (whole-document read)", + ErrUnsupportedPostgresPlan, projection.projection.Name) + } + resolved, err := r.resolveValue(value) + if err != nil { + return nil, fmt.Errorf("group source projection %q: %w", projection.projection.Name, err) + } + expression = resolved + if len(value.Path) > 0 { + isJSONB = true + } + if isPayloadExpression(expression) { + expression = "(" + expression + ")" + } + case extractionSelector: + scope := r.rows[projection.expression.Extract.Source.Variable] + value, err := r.scalarExtractValueExpression(scope.alias, projection.path, projection.expression.NullBehavior) + if err != nil { + return nil, fmt.Errorf("group source projection %q: %w", projection.projection.Name, err) + } + expression = value + isJSONB = true + case extractionPresenceSidecar: + return nil, fmt.Errorf("%w: group source projection %q is a presence sidecar and cannot be addressed by key, reducer, or row value", + ErrUnsupportedPostgresPlan, projection.projection.Name) + case extractionCallExpression: + value, err := r.renderCallProjection(plan, projection.projection) + if err != nil { + return nil, err + } + expression = value + isJSONB = true + case extractionObject, extractionSlice: + return nil, fmt.Errorf("%w: group source projection %q is a composite and cannot be addressed by key, reducer, or row value (lowering admits public scalars only)", + ErrUnsupportedPostgresPlan, projection.projection.Name) + case extractionUnitNormalized: + return nil, fmt.Errorf("%w: group source projection %q is a unit-normalized extraction; unit normalization belongs to the read spine registration (unit_normalization.go), not the closed GROUP slice", + ErrUnsupportedPostgresPlan, projection.projection.Name) + default: + return nil, fmt.Errorf("%w: group source projection %q is not a registered projection arm", + ErrUnsupportedPostgresPlan, projection.projection.Name) + } + alias := "c" + strconv.Itoa(i) + columns = append(columns, groupColumn{ + name: projection.projection.Name, alias: alias, expr: expression, + ref: `"` + alias + `"`, jsonb: isJSONB, + }) + } + return columns, nil +} + +func groupResolveColumn(nameToColumn map[string]*groupColumn, name string) (*groupColumn, error) { + column, ok := nameToColumn[name] + if !ok { + return nil, fmt.Errorf("%w: group input column %q is not among the source projection names (L-13.11)", + ErrUnsupportedPostgresPlan, name) + } + return column, nil +} + +func groupNullRead(column *groupColumn) string { + if !column.jsonb { + return column.ref + } + return "NULLIF(" + column.ref + ", 'null'::jsonb)" +} + +func (r *planRenderer) groupColumnRead(column *groupColumn, kind string, tag string, dollar func(int) string) (string, string, error) { + nullRead := groupNullRead(column) + var typeofTag string + switch kind { + case "STRING": + typeofTag = "string" + case "INTEGER", "DECIMAL": + typeofTag = "number" + case "BOOLEAN": + typeofTag = "boolean" + default: + return "", "", fmt.Errorf("%w: group column kind %q is unsupported", ErrUnsupportedPostgresPlan, kind) + } + witness := "" + if column.jsonb { + typeBind := r.textParameter(typeofTag, "text:typeof:"+typeofTag) + witness = "(CASE WHEN COALESCE(" + nullRead + " IS NULL OR jsonb_typeof(" + nullRead + ") = " + + dollar(typeBind) + "::text, false) THEN 1 ELSE " + guardCast(tag) + " END) > 0" + } + switch kind { + case "STRING": + if !column.jsonb { + return column.ref, "", nil + } + return "((" + nullRead + ") #>> '{}')", witness, nil + case "INTEGER", "DECIMAL": + if !column.jsonb { + return "(" + column.ref + ")::numeric", "", nil + } + return "((" + nullRead + ") #>> '{}')::numeric", witness, nil + case "BOOLEAN": + if !column.jsonb { + return "(" + column.ref + ")::boolean", "", nil + } + return "((" + nullRead + ") #>> '{}')::boolean", witness, nil + } + return "", "", fmt.Errorf("%w: group column kind %q is unsupported", ErrUnsupportedPostgresPlan, kind) +} + +func (r *planRenderer) groupAggregateExpression(aggregate ir.PhysicalStageGroupAggregate, column *groupColumn, dollar func(int) string) (string, error) { + if aggregate.Operation == "COUNT_ROWS" { + return "COUNT(*)", nil + } + typedRead, _, err := r.groupColumnRead(column, aggregate.InputKind, groupValueTypeTag, dollar) + if err != nil { + return "", err + } + collate := "" + if aggregate.InputKind == "STRING" { + collate = ` COLLATE "C"` + } + switch aggregate.Operation { + case "COUNT_NON_NULL": + return "COUNT(" + typedRead + ")", nil + case "COUNT_DISTINCT": + return "COUNT(DISTINCT " + typedRead + collate + ")", nil + case "MIN": + return "MIN(" + typedRead + collate + ")", nil + case "MAX": + return "MAX(" + typedRead + collate + ")", nil + case "SUM": + return groupBoundedDoubleSum(column, aggregate.InputKind), nil + case "MEAN": + sum := groupBoundedDoubleSum(column, aggregate.InputKind) + poolCount := "COUNT(" + groupNumericRead(column, aggregate.InputKind) + ")" + return "CASE WHEN " + poolCount + " = 0 THEN NULL ELSE " + + sum + " / (" + poolCount + ")::double precision END", nil + } + return "", fmt.Errorf("%w: aggregate %q is not a registered reducer", ErrUnsupportedPostgresPlan, string(aggregate.Operation)) +} + +func groupDoubleBandCast(numericFragment string) string { + return "(CASE WHEN " + numericFragment + " IS NULL THEN NULL" + + " WHEN abs(" + numericFragment + ") < '" + callDoubleOverflowBound + "'::numeric" + + " THEN CASE WHEN abs(" + numericFragment + ") >= '" + callDoubleUnderflowFloor + "'::numeric" + + " THEN (" + numericFragment + ")::double precision ELSE 0::double precision END" + + " ELSE 0::double precision END)" +} + +func groupBoundedDoubleSum(column *groupColumn, kind string) string { + return groupDoubleBandCast("SUM(" + callBandOperandClamp(groupNumericRead(column, kind)) + ")") +} + +func groupNumericRead(column *groupColumn, kind string) string { + if kind == "INTEGER" || kind == "DECIMAL" { + if !column.jsonb { + return "(" + column.ref + ")::numeric" + } + return "((" + groupNullRead(column) + ") #>> '{}')::numeric" + } + return "" +} + +func groupRowValueMemberRead(column *groupColumn, kind string) func(source string) string { + return func(source string) string { + nullRead := source + if column.jsonb { + nullRead = "NULLIF(" + source + ", 'null'::jsonb)" + } + switch kind { + case "STRING": + if column.jsonb { + return "((" + nullRead + ") #>> '{}')" + } + return "(" + nullRead + ")" + case "INTEGER", "DECIMAL": + if column.jsonb { + return "((" + nullRead + ") #>> '{}')::numeric" + } + return "(" + nullRead + ")::numeric" + default: + if column.jsonb { + return "((" + nullRead + ") #>> '{}')::boolean" + } + return "(" + nullRead + ")::boolean" + } + } +} + +func groupRowValueCollateAndArrayType(kind string) (collate, arrayType string, ok bool) { + switch kind { + case "STRING": + return ` COLLATE "C"`, "text[]", true + case "INTEGER", "DECIMAL": + return "", "numeric[]", true + case "BOOLEAN": + return "", "boolean[]", true + } + return "", "", false +} + +func (r *planRenderer) groupManyRowValuePoolFor(policy, kind string, column *groupColumn, keyReads []string, cteName string, rowsRelation string, dollar func(int) string) (selectText, singleValueRef string, err error) { + switch policy { + case "ALL", "ONE": + default: + return "", "", fmt.Errorf("%w: row value policy %q is unsupported", ErrUnsupportedPostgresPlan, policy) + } + collate, arrayType, ok := groupRowValueCollateAndArrayType(kind) + if !ok { + return "", "", fmt.Errorf("%w: row value input kind %q is unsupported", ErrUnsupportedPostgresPlan, kind) + } + if !column.jsonb { + return "", "", fmt.Errorf("%w: a many-cardinality row value requires a jsonb source column", ErrUnsupportedPostgresPlan) + } + member := groupRowValueMemberRead(column, kind) + memberRead := member("(_rv.item)") + selects := make([]string, 0, len(keyReads)+1) + for i, keyRead := range keyReads { + selects = append(selects, keyRead+" AS m_k"+strconv.Itoa(i)) + } + if policy == "ALL" { + distinct := memberRead + collate + selects = append(selects, "to_jsonb(COALESCE(ARRAY_AGG(DISTINCT "+distinct+" ORDER BY "+distinct+ + "), '{}'::"+arrayType+")) AS m_pool") + } else { + selects = append(selects, "(CASE WHEN COUNT(DISTINCT "+memberRead+collate+") <= 1 THEN MIN("+memberRead+collate+ + ") ELSE "+groupRowValueRaiseArm(kind)+" END) AS m_pool") + } + typeofTag := map[string]string{"STRING": "string", "INTEGER": "number", "DECIMAL": "number", "BOOLEAN": "boolean"}[kind] + itemNull := "NULLIF((_rv.item), 'null'::jsonb)" + typeBind := r.textParameter(typeofTag, "text:typeof:"+typeofTag) + selectText = "SELECT " + strings.Join(selects, ", ") + + " FROM " + rowsRelation + ", LATERAL jsonb_array_elements(" + groupNullRead(column) + ") _rv(item)" + + " WHERE " + itemNull + " IS NOT NULL AND (CASE WHEN COALESCE(" + itemNull + + " IS NULL OR jsonb_typeof(" + itemNull + ") = " + dollar(typeBind) + "::text, false) THEN 1 ELSE " + + guardCast(groupValueTypeTag) + " END) > 0" + if len(keyReads) > 0 { + positions := make([]string, len(keyReads)) + for i := range positions { + positions[i] = strconv.Itoa(i + 1) + } + selectText += " GROUP BY " + strings.Join(positions, ", ") + } + if policy == "ALL" { + return selectText, "MAX(" + cteName + ".m_pool::text)::jsonb", nil + } + return selectText, "MAX(" + cteName + ".m_pool)", nil +} + +func groupRowValueExpressionFor(policy, kind string, many bool, column *groupColumn, dollar func(int) string) (string, error) { + switch policy { + case "ALL", "ONE": + default: + return "", fmt.Errorf("%w: row value policy %q is unsupported", ErrUnsupportedPostgresPlan, policy) + } + if many { + return "", fmt.Errorf("%w: many-cardinality row values render through groupManyRowValuePoolFor", ErrUnsupportedPostgresPlan) + } + collate, arrayType, ok := groupRowValueCollateAndArrayType(kind) + if !ok { + return "", fmt.Errorf("%w: row value input kind %q is unsupported", ErrUnsupportedPostgresPlan, kind) + } + member := groupRowValueMemberRead(column, kind) + read := member(column.ref) + if policy == "ALL" { + distinct := read + collate + return "to_jsonb(COALESCE(ARRAY_AGG(DISTINCT " + distinct + " ORDER BY " + distinct + + ") FILTER (WHERE " + read + " IS NOT NULL), '{}'" + "::" + arrayType + "))", nil + } + return "(CASE WHEN COUNT(DISTINCT " + read + collate + ") <= 1 THEN MIN(" + read + collate + + ") ELSE " + groupRowValueRaiseArm(kind) + " END)", nil +} + +func groupRowValueRaiseArm(kind string) string { + guard := guardCast(groupRowValueMultipleTag) + switch kind { + case "STRING": + return "CAST(" + guard + " AS text)" + case "BOOLEAN": + return "CAST(" + guard + " AS boolean)" + } + return guard +} diff --git a/internal/dataframe/compiler/render/postgres/group_test.go b/internal/dataframe/compiler/render/postgres/group_test.go new file mode 100644 index 000000000..2360e9595 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/group_test.go @@ -0,0 +1,1564 @@ +package postgres + +import ( + "context" + "encoding/json" + "fmt" + "os" + "reflect" + "regexp" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + group13StepID = "pg13_group" + group13Project = "p13-proj" + group13Generation = "p13-gen" + group13AuthPath = "FHIR/Observation" +) + +type group13Fixture struct { + policy recipe.ConstructionGroupMissingKeyPolicy + keys []recipe.ConstructionGroupKey + aggregates []recipe.ConstructionGroupAggregate + rowValues []recipe.ConstructionRowValue + outputs []recipe.StageColumn + sourceOrder []string +} + +func group13DefaultOutput() recipe.Output { + return recipe.Output{ + Name: "pg13_group_rows", RootResourceType: "Observation", RowGrain: "resource", + Fields: []recipe.Field{ + {Name: "resource_id", ColumnID: "resource_id", Expr: recipe.Expression{Select: "root.id"}}, + {Name: "status", ColumnID: "status", Expr: recipe.Expression{Select: "root.status"}}, + {Name: "value", ColumnID: "value", Expr: recipe.Expression{Select: "root.valueQuantity.value"}}, + }, + Construction: &recipe.Construction{ + Version: 1, + SourceColumns: []recipe.StageColumn{ + {ID: "resource_id", Name: "resource_id"}, + {ID: "status", Name: "status"}, + {ID: "value", Name: "value"}, + }, + }, + } +} + +func (f group13Fixture) output() recipe.Output { + output := group13DefaultOutput() + if len(f.sourceOrder) > 0 { + byName := map[string]recipe.Field{} + for _, field := range output.Fields { + byName[field.Name] = field + } + columns := map[string]recipe.StageColumn{} + for _, column := range output.Construction.SourceColumns { + columns[column.Name] = column + } + output.Fields = nil + output.Construction.SourceColumns = nil + for _, name := range f.sourceOrder { + output.Fields = append(output.Fields, byName[name]) + output.Construction.SourceColumns = append(output.Construction.SourceColumns, columns[name]) + } + } + output.Construction.Steps = []recipe.ConstructionStep{{ + ID: group13StepID, Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionGroupOp, Group: &recipe.ConstructionGroup{ + ConstructionID: group13StepID, MissingKeyPolicy: f.policy, + Keys: f.keys, Aggregates: f.aggregates, + }}, + Outputs: f.outputs, + RowValues: f.rowValues, + }} + return output +} + +func group13LoweredPlan(t *testing.T, fixture group13Fixture) ir.PhysicalPlan { + t.Helper() + output := fixture.output() + bindings := recipe.RuntimeBindings{Project: group13Project, DatasetGeneration: group13Generation, AuthResourcePaths: []string{group13AuthPath}} + bundle := recipe.Bundle{RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: output.Name, TranslationVersion: "test", Outputs: []recipe.Output{output}} + if err := bundle.Validate(); err != nil { + t.Fatalf("bundle validate: %v", err) + } + plan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("lower.CompileResolvedRecipePlan: %v", err) + } + lowered := compiled.Outputs[0].Plan + if lowered.StageSequence == nil || len(lowered.StageSequence.Stages) != 1 || + lowered.StageSequence.Stages[0].Kind != ir.PhysicalStageGroupOp || + lowered.StageSequence.Stages[0].InputStageID != lowered.StageSequence.SourceStageID { + t.Fatalf("lowered plan is not the inline single-stage GROUP shape: %#v", lowered.StageSequence) + } + return ir.ClonePhysicalPlan(lowered) +} + +func group13KeyedPlan(t *testing.T, policy recipe.ConstructionGroupMissingKeyPolicy) ir.PhysicalPlan { + t.Helper() + return group13LoweredPlan(t, group13Fixture{ + policy: policy, + keys: []recipe.ConstructionGroupKey{ + {InputColumnID: "status", OutputColumnID: "status_group"}, + {InputColumnID: "value", OutputColumnID: "value_group"}, + }, + aggregates: []recipe.ConstructionGroupAggregate{ + {Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}, + {Operation: recipe.ConstructionGroupCountNonNull, InputColumnID: "value", OutputColumnID: "non_null_id"}, + {Operation: recipe.ConstructionGroupCountDistinct, InputColumnID: "status", OutputColumnID: "distinct_id"}, + {Operation: recipe.ConstructionGroupSum, InputColumnID: "value", OutputColumnID: "sum_id"}, + {Operation: recipe.ConstructionGroupMean, InputColumnID: "value", OutputColumnID: "mean_id"}, + {Operation: recipe.ConstructionGroupMin, InputColumnID: "value", OutputColumnID: "min_id"}, + {Operation: recipe.ConstructionGroupMax, InputColumnID: "value", OutputColumnID: "max_id"}, + }, + outputs: []recipe.StageColumn{ + {ID: "status_group", Name: "status_group", Type: "string", Nullable: true}, + {ID: "value_group", Name: "value_group", Type: "decimal", Nullable: true}, + {ID: "rows_id", Name: "rows", Type: "integer"}, + {ID: "non_null_id", Name: "non_null", Type: "integer"}, + {ID: "distinct_id", Name: "distinct", Type: "integer"}, + {ID: "sum_id", Name: "total", Type: "decimal", Nullable: true}, + {ID: "mean_id", Name: "mean", Type: "decimal", Nullable: true}, + {ID: "min_id", Name: "minimum", Type: "decimal", Nullable: true}, + {ID: "max_id", Name: "maximum", Type: "decimal", Nullable: true}, + }, + }) +} + +func group13CountRowsPlan(t *testing.T, policy recipe.ConstructionGroupMissingKeyPolicy) ir.PhysicalPlan { + t.Helper() + return group13LoweredPlan(t, group13Fixture{ + policy: policy, + keys: []recipe.ConstructionGroupKey{{InputColumnID: "status", OutputColumnID: "status_group"}}, + aggregates: []recipe.ConstructionGroupAggregate{{Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}}, + outputs: []recipe.StageColumn{ + {ID: "status_group", Name: "status_group", Type: "string", Nullable: true}, + {ID: "rows_id", Name: "rows", Type: "integer"}, + }, + }) +} + +func group13KeylessPlan(t *testing.T) ir.PhysicalPlan { + t.Helper() + return group13LoweredPlan(t, group13Fixture{ + policy: recipe.ConstructionGroupMissingKeyGroup, + aggregates: []recipe.ConstructionGroupAggregate{{Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}}, + rowValues: []recipe.ConstructionRowValue{ + {InputColumnID: "status", OutputColumnID: "status_values", Policy: recipe.ConstructionRowValueAll}, + {InputColumnID: "value", OutputColumnID: "value_one", Policy: recipe.ConstructionRowValueOne}, + }, + outputs: []recipe.StageColumn{ + {ID: "rows_id", Name: "rows", Type: "integer"}, + {ID: "status_values", Name: "status_values", Type: "array"}, + {ID: "value_one", Name: "value_one", Type: "decimal", Nullable: true}, + }, + }) +} + +func group13RowValuePlan(t *testing.T, policy recipe.ConstructionRowValuePolicy, valueType string) ir.PhysicalPlan { + t.Helper() + return group13LoweredPlan(t, group13Fixture{ + policy: recipe.ConstructionGroupMissingKeyGroup, + keys: []recipe.ConstructionGroupKey{{InputColumnID: "status", OutputColumnID: "status_group"}}, + aggregates: []recipe.ConstructionGroupAggregate{{Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}}, + rowValues: []recipe.ConstructionRowValue{ + {InputColumnID: "value", OutputColumnID: "value_values", Policy: policy}, + }, + outputs: []recipe.StageColumn{ + {ID: "status_group", Name: "status_group", Type: "string", Nullable: true}, + {ID: "rows_id", Name: "rows", Type: "integer"}, + {ID: "value_values", Name: "value_values", Type: valueType, Nullable: true}, + }, + }) +} + +func group13Identity(t *testing.T, plan ir.PhysicalPlan, keys []group13Member) string { + t.Helper() + stage := plan.StageSequence.Stages[0] + outputIDs := map[string]string{} + for _, column := range stage.OutputColumns { + if _, seen := outputIDs[column.Name]; !seen { + outputIDs[column.Name] = column.ID + } + } + member := func(value group13Member) string { + if value.nilValue { + return "null" + } + if value.numeric { + return value.text + } + raw, err := json.Marshal(value.text) + if err != nil { + t.Fatalf("marshal identity member: %v", err) + } + return string(raw) + } + parts := []string{ + `["construction","` + group13StepID + `"]`, `["operation","GROUP"]`, + } + for _, key := range keys { + columnID, ok := outputIDs[key.outputColumn] + if !ok { + t.Fatalf("group key output %q has no stable column ID", key.outputColumn) + } + parts = append(parts, fmt.Sprintf("[%q,%s]", columnID, member(key))) + } + return "[" + strings.Join(parts, ",") + "]" +} + +type group13Member struct { + outputColumn string + text string + numeric bool + nilValue bool +} + +func group13TextMember(outputColumn, text string) group13Member { + return group13Member{outputColumn: outputColumn, text: text} +} + +func group13NumericMember(outputColumn, text string) group13Member { + return group13Member{outputColumn: outputColumn, text: text, numeric: true} +} + +func group13NullMember(outputColumn string) group13Member { + return group13Member{outputColumn: outputColumn, nilValue: true} +} + +func TestGroupRenderedSliceContract(t *testing.T) { + plan := group13KeyedPlan(t, recipe.ConstructionGroupMissingKeyGroup) + rendered := renderOK(t, plan) + sql := rendered.SQL + + for _, want := range []string{ + `WITH g_src AS (`, + `), g_rows AS (SELECT * FROM g_src`, + `), g_grp AS (SELECT`, + `FROM g_rows GROUP BY 1, 2`, + `COUNT(*) AS g_a0`, + `FROM g_grp AS g ORDER BY g.g_k0 COLLATE "C" ASC NULLS FIRST, g.g_k1 ASC NULLS FIRST, `, + `COLLATE "C" ASC`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("statement is missing %q:\n%s", want, sql) + } + } + if got := strings.Count(sql, groupMissingKeyTag); got != 0 { + t.Fatalf("GROUP policy must not carry the missing-key guard (found %d):\n%s", got, sql) + } + if got := strings.Count(sql, groupKeyTypeTag); got != 2 { + t.Fatalf("key-type witnesses = %d, want 2 (status+value):\n%s", got, sql) + } + if got := strings.Count(sql, groupValueTypeTag); got != 2 { + t.Fatalf("value-pool witnesses = %d, want 2 after dedupe (status+value):\n%s", got, sql) + } + if strings.Contains(sql, "@") { + t.Fatalf("statement carries AQL bind syntax:\n%s", sql) + } + for _, leaked := range []string{group13Project, group13Generation, group13AuthPath, "Observation", "status", "value", "valueQuantity", group13StepID} { + if strings.Contains(sql, leaked) { + t.Fatalf("value %q leaked into statement text:\n%s", leaked, sql) + } + } + if !strings.Contains(sql, `#> $`) { + t.Fatalf("payload paths must bind through #> text[] parameters:\n%s", sql) + } + + want := []string{"status_group", "value_group", "rows", "non_null", "distinct", "total", "mean", "minimum", "maximum", "__loom_row_id"} + if !reflect.DeepEqual(rendered.OutputColumns, want) { + t.Fatalf("output columns = %#v, want %#v", rendered.OutputColumns, want) + } + + second, err := RenderPhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if err != nil || second.SQL != rendered.SQL || !reflect.DeepEqual(second.Args, rendered.Args) { + t.Fatalf("group render is not deterministic: %v", err) + } +} + +func TestGroupPoliciesRender(t *testing.T) { + group := renderOK(t, group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup)) + if strings.Contains(group.SQL, " IS NOT NULL") || strings.Contains(group.SQL, guardCast(groupMissingKeyTag)) { + t.Fatalf("GROUP policy must neither filter nor guard key rows:\n%s", group.SQL) + } + + exclude := renderOK(t, group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyExclude)) + if !strings.Contains(exclude.SQL, " IS NOT NULL") { + t.Fatalf("EXCLUDE policy must conjunct the null-key drop:\n%s", exclude.SQL) + } + if strings.Contains(exclude.SQL, guardCast(groupMissingKeyTag)) { + t.Fatalf("EXCLUDE policy must not carry the ERROR guard:\n%s", exclude.SQL) + } + + errorRendered := renderOK(t, group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyError)) + for _, want := range []string{ + "(CASE WHEN ", " IS NOT NULL THEN 1 ELSE CAST('" + groupMissingKeyTag + "'", + guardBarrierGUC, + " END) > 0", + } { + if !strings.Contains(errorRendered.SQL, want) { + t.Fatalf("ERROR guard is missing %q:\n%s", want, errorRendered.SQL) + } + } +} + +func TestGroupReducerTableRender(t *testing.T) { + sql := renderOK(t, group13KeyedPlan(t, recipe.ConstructionGroupMissingKeyGroup)).SQL + for _, want := range []string{ + `COUNT(*) AS g_a0`, + `COUNT(`, + `COUNT(DISTINCT `, + `MIN(`, + `MAX(`, + `SUM((CASE WHEN `, + ` > '1e400'::numeric THEN '1e400'::numeric`, + ` IS NULL THEN NULL WHEN abs(SUM((CASE WHEN `, + `::double precision ELSE 0::double precision END ELSE 0::double precision END)`, + ` = 0 THEN NULL ELSE (CASE WHEN SUM((CASE WHEN `, + `) / (COUNT(`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("reducer table is missing %q:\n%s", want, sql) + } + } + for _, forbidden := range []string{`'Infinity'::double precision`, `groupDoubleNotFinite`} { + if strings.Contains(sql, forbidden) { + t.Fatalf("reducer table reintroduced the deleted gate fragment %q:\n%s", forbidden, sql) + } + } + if !strings.Contains(sql, ` COLLATE "C")`) && !strings.Contains(sql, ` COLLATE "C"))`) && !strings.Contains(sql, ` COLLATE "C") AS`) { + t.Fatalf("string reducer must COLLATE \"C\":\n%s", sql) + } +} + +func TestGroupKeyKindsRender(t *testing.T) { + for _, cell := range []struct { + kind string + read string + collate bool + }{ + {kind: "STRING", read: ` #>> '{}')`, collate: true}, + {kind: "INTEGER", read: ` #>> '{}')::numeric`, collate: false}, + {kind: "DECIMAL", read: ` #>> '{}')::numeric`, collate: false}, + {kind: "BOOLEAN", read: ` #>> '{}')::boolean`, collate: false}, + } { + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + stage := &plan.StageSequence.Stages[0] + stage.Group.Keys[0].Kind = cell.kind + kindLower := strings.ToLower(cell.kind) + for _, columns := range [][]ir.PhysicalStageColumn{plan.StageSequence.SourceColumns, stage.InputColumns, stage.OutputColumns, plan.StageSequence.FinalColumns} { + for i := range columns { + if columns[i].Name == stage.Group.Keys[0].InputColumn || columns[i].Name == stage.Group.Keys[0].OutputColumn { + columns[i].Kind = kindLower + } + } + } + sql := renderOK(t, plan).SQL + if !strings.Contains(sql, cell.read) { + t.Fatalf("kind %s key must read %q:\n%s", cell.kind, cell.read, sql) + } + orderIndex := strings.LastIndex(sql, " ORDER BY ") + order := sql[orderIndex:] + if got := strings.Contains(order, `g.g_k0 COLLATE "C"`); got != cell.collate { + t.Fatalf("kind %s order collate = %v, want %v:\n%s", cell.kind, got, cell.collate, order) + } + } +} + +func TestGroupIdentityOrderAndBound(t *testing.T) { + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + rendered := renderOK(t, plan) + orderIndex := strings.LastIndex(rendered.SQL, " ORDER BY ") + if orderIndex < 0 { + t.Fatalf("final statement has no ORDER BY:\n%s", rendered.SQL) + } + final := rendered.SQL[orderIndex:] + if !strings.HasPrefix(final, ` ORDER BY g.g_k0 COLLATE "C" ASC NULLS FIRST, `) { + t.Fatalf("final order must start with the key sort:\n%s", final) + } + if !strings.Contains(final, `'[' || '["construction",' || to_jsonb($`) || + !strings.Contains(final, `'["operation","GROUP"]'`) || + !strings.Contains(final, `COALESCE(to_jsonb(g.g_k0)::text, 'null')`) { + t.Fatalf("final ORDER BY must end in the stable identity concatenation:\n%s", final) + } + if !strings.HasSuffix(final, ` COLLATE "C" ASC`) { + t.Fatalf("the unwindowed statement ends in the identity order:\n%s", final) + } + + want := group13Identity(t, plan, []group13Member{group13TextMember("status_group", "final")}) + if !strings.Contains(want, `["construction","`+group13StepID+`"]`) || !strings.Contains(want, `["operation","GROUP"]`) { + t.Fatalf("independent identity builder is off contract: %s", want) + } + + limited := ir.ClonePhysicalPlan(plan) + limited.StageSequence.PreviewLimitBindKey = "limit" + limited.BindVars["limit"] = 25 + limitedRendered := renderOK(t, limited) + if m, _ := regexp.MatchString(` COLLATE "C" ASC LIMIT \$[0-9]+$`, limitedRendered.SQL); !m { + t.Fatalf("the bound preview LIMIT must follow the identity order:\n%s", limitedRendered.SQL) + } + if len(limitedRendered.Args) != len(rendered.Args)+1 { + t.Fatalf("preview limit must bind as one extra parameter") + } + last := limitedRendered.Args[len(limitedRendered.Args)-1] + if last.Type != command.PostgreSQLParameterBigint || last.Value != int64(25) { + t.Fatalf("preview limit bind = %#v, want bigint 25", last) + } +} + +func TestGroupWindowedSourceRegion(t *testing.T) { + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + sorted := ir.ClonePhysicalPlan(plan) + last := len(sorted.Operations) - 1 + sorted.Operations = append(sorted.Operations[:last], + ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{Keys: []ir.PhysicalValue{{Variable: "root", Path: []string{"logical_id"}}}}}, + ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "srclimit"}}, + sorted.Operations[last]) + sorted.BindVars["srclimit"] = 3 + rendered := renderOK(t, sorted) + srcEnd := strings.Index(rendered.SQL, "), g_rows AS (") + if srcEnd < 0 { + t.Fatalf("g_src boundary missing:\n%s", rendered.SQL) + } + source := rendered.SQL[:srcEnd] + if !strings.Contains(source, `r0."logical_id" ASC`) || !strings.Contains(source, `"resource_seq" ASC`) || !strings.Contains(source, "LIMIT $") { + t.Fatalf("the W1 window must be rendered inside g_src:\n%s", source) + } +} + +func TestGroupRowValuesRender(t *testing.T) { + all := renderOK(t, group13RowValuePlan(t, recipe.ConstructionRowValueAll, "array")).SQL + for _, want := range []string{ + `to_jsonb(COALESCE(ARRAY_AGG(DISTINCT `, + `) FILTER (WHERE `, + ` IS NOT NULL), '{}'::numeric[]`, + } { + if !strings.Contains(all, want) { + t.Fatalf("ALL emitter is missing %q:\n%s", want, all) + } + } + one := renderOK(t, group13RowValuePlan(t, recipe.ConstructionRowValueOne, "decimal")).SQL + for _, want := range []string{ + `CASE WHEN COUNT(DISTINCT `, + ` <= 1 THEN MIN(`, + ` ELSE `, + ` END) AS g_rv0`, + } { + if !strings.Contains(one, want) { + t.Fatalf("ONE emitter is missing %q:\n%s", want, one) + } + } + if !strings.Contains(one, guardCast(groupRowValueMultipleTag)) { + t.Fatalf("ONE must raise the canonical multi-value tag through the guard:\n%s", one) + } +} + +func TestGroupInputManyRender(t *testing.T) { + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + appendGroup13ManyRowValue(t, &plan, "ALL", "STRING") + sql := renderOK(t, plan).SQL + for _, want := range []string{ + ` AS (SELECT `, + ` FROM g_rows, LATERAL jsonb_array_elements(`, + `) _rv(item) WHERE `, + ` WHERE NULLIF((_rv.item), 'null'::jsonb) IS NOT NULL AND (CASE WHEN COALESCE(NULLIF((_rv.item), 'null'::jsonb) IS NULL OR jsonb_typeof(NULLIF((_rv.item), 'null'::jsonb)) =`, + ` THEN 1 ELSE CAST('` + groupValueTypeTag + `'`, + ` GROUP BY 1`, + `AS m_pool`, + `ARRAY_AGG(DISTINCT `, + `'{}'::text[]`, + `LEFT JOIN g_m0 ON `, + ` IS NOT DISTINCT FROM g_m0.m_k0`, + `MAX(g_m0.m_pool::text)::jsonb AS g_rv0`, + `= 'array', false) THEN 1 ELSE CAST('` + groupValueTypeTag + `'`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("InputMany ALL emitter is missing %q:\n%s", want, sql) + } + } + onePlan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + appendGroup13ManyRowValue(t, &onePlan, "ONE", "STRING") + oneSQL := renderOK(t, onePlan).SQL + for _, want := range []string{ + `COUNT(DISTINCT `, + `<= 1 THEN MIN(`, + ` ELSE ` + groupRowValueRaiseArm("STRING"), + `AS m_pool`, + `MAX(g_m0.m_pool) AS g_rv0`, + } { + if !strings.Contains(oneSQL, want) { + t.Fatalf("InputMany ONE emitter is missing %q:\n%s", want, oneSQL) + } + } +} + +func appendGroup13ManyRowValue(t *testing.T, plan *ir.PhysicalPlan, policy, inputKind string) { + t.Helper() + returnIndex := -1 + for i, operation := range plan.Operations { + if operation.Kind == ir.PhysicalReturnOp { + returnIndex = i + } + } + if returnIndex < 0 { + t.Fatal("plan has no terminal RETURN") + } + arrayValue := ir.PhysicalValue{Variable: "root", Path: []string{"values"}} + plan.Operations[returnIndex].Return.Projections = append(plan.Operations[returnIndex].Return.Projections, + ir.PhysicalProjection{Name: "values", Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalPreserveNull, Value: &arrayValue, + }}) + columnKind := map[string]string{ + "STRING": "string", "INTEGER": "integer", "DECIMAL": "decimal", "BOOLEAN": "boolean", + }[inputKind] + if columnKind == "" { + t.Fatalf("unknown row-value input kind %q", inputKind) + } + stage := &plan.StageSequence.Stages[0] + stage.Group.RowValues = []ir.PhysicalStageRowValue{{ + InputColumn: "values", InputKind: inputKind, InputMany: true, + Output: "values_set", Policy: policy, Variable: "values_set", + }} + sourceColumn := ir.PhysicalStageColumn{ID: "values-source", Name: "values", Kind: columnKind, Cardinality: "many"} + plan.StageSequence.SourceColumns = append(plan.StageSequence.SourceColumns, sourceColumn) + stage.InputColumns = append(stage.InputColumns, sourceColumn) + stage.InputProjections = append(stage.InputProjections, + ir.PhysicalProjection{Name: "values", Value: ir.PhysicalValue{Variable: stage.InputRowVariable, Path: []string{"values"}}}) + cardinality := "many" + if policy == "ONE" { + cardinality = "optional_one" + } + column := ir.PhysicalStageColumn{ID: "values-set-output", Name: "values_set", Kind: columnKind, Cardinality: cardinality, Nullable: policy == "ONE"} + stage.OutputColumns = append(stage.OutputColumns, column) + plan.StageSequence.FinalColumns = append(plan.StageSequence.FinalColumns, column) + stage.OutputProjections = append(stage.OutputProjections, + ir.PhysicalProjection{Name: "values_set", Value: ir.PhysicalValue{Variable: "values_set"}}) +} + +func TestGroupKeylessGrandTotal(t *testing.T) { + sql := renderOK(t, group13KeylessPlan(t)).SQL + if strings.Contains(sql, "GROUP BY") { + t.Fatalf("the keyless grand total renders without GROUP BY:\n%s", sql) + } + for _, want := range []string{`COUNT(*) AS g_a0`, `to_jsonb(COALESCE(ARRAY_AGG(DISTINCT `, `CASE WHEN COUNT(DISTINCT `} { + if !strings.Contains(sql, want) { + t.Fatalf("keyless grand total is missing %q:\n%s", want, sql) + } + } + keylessSum := group13KeylessPlan(t) + stage := &keylessSum.StageSequence.Stages[0] + stage.Group.Aggregates = []ir.PhysicalStageGroupAggregate{ + {Operation: "COUNT_ROWS", Output: "rows", Variable: stage.Group.Aggregates[0].Variable, OutputKind: "integer"}, + {Operation: "SUM", InputColumn: "value", Output: "total", Variable: "g_sum", InputKind: "DECIMAL", OutputKind: "decimal"}, + } + keylessSum.StageSequence.FinalColumns = append(keylessSum.StageSequence.FinalColumns, + ir.PhysicalStageColumn{ID: "sum_id", Name: "total", Kind: "decimal", Cardinality: "optional_one", Nullable: true}) + stage.OutputColumns = append(stage.OutputColumns, + ir.PhysicalStageColumn{ID: "sum_id", Name: "total", Kind: "decimal", Cardinality: "optional_one", Nullable: true}) + stage.OutputProjections = append(stage.OutputProjections, + ir.PhysicalProjection{Name: "total", Value: ir.PhysicalValue{Variable: "g_sum"}}) + requireGroup13Refusal(t, keylessSum, "only keyless COUNT_ROWS groups can consume the source scope") +} + +func TestGroupNameMapFollowsSourceOrder(t *testing.T) { + swappedFixture := group13Fixture{ + policy: recipe.ConstructionGroupMissingKeyGroup, + keys: []recipe.ConstructionGroupKey{ + {InputColumnID: "status", OutputColumnID: "status_group"}, + {InputColumnID: "value", OutputColumnID: "value_group"}, + }, + aggregates: []recipe.ConstructionGroupAggregate{ + {Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}, + {Operation: recipe.ConstructionGroupCountNonNull, InputColumnID: "value", OutputColumnID: "non_null_id"}, + {Operation: recipe.ConstructionGroupCountDistinct, InputColumnID: "status", OutputColumnID: "distinct_id"}, + {Operation: recipe.ConstructionGroupSum, InputColumnID: "value", OutputColumnID: "sum_id"}, + {Operation: recipe.ConstructionGroupMean, InputColumnID: "value", OutputColumnID: "mean_id"}, + {Operation: recipe.ConstructionGroupMin, InputColumnID: "value", OutputColumnID: "min_id"}, + {Operation: recipe.ConstructionGroupMax, InputColumnID: "value", OutputColumnID: "max_id"}, + }, + outputs: []recipe.StageColumn{ + {ID: "status_group", Name: "status_group", Type: "string", Nullable: true}, + {ID: "value_group", Name: "value_group", Type: "decimal", Nullable: true}, + {ID: "rows_id", Name: "rows", Type: "integer"}, + {ID: "non_null_id", Name: "non_null", Type: "integer"}, + {ID: "distinct_id", Name: "distinct", Type: "integer"}, + {ID: "sum_id", Name: "total", Type: "decimal", Nullable: true}, + {ID: "mean_id", Name: "mean", Type: "decimal", Nullable: true}, + {ID: "min_id", Name: "minimum", Type: "decimal", Nullable: true}, + {ID: "max_id", Name: "maximum", Type: "decimal", Nullable: true}, + }, + sourceOrder: []string{"resource_id", "value", "status"}, + } + baseSameShape := group13LoweredPlan(t, func(f group13Fixture) group13Fixture { f.sourceOrder = nil; return f }(swappedFixture)) + swapped := group13LoweredPlan(t, swappedFixture) + baseSQL := renderOK(t, baseSameShape).SQL + swappedSQL := renderOK(t, swapped).SQL + grp := func(sql string) string { + start := strings.Index(sql, "), g_grp AS (SELECT ") + if start < 0 { + t.Fatalf("g_grp boundary missing:\n%s", sql) + } + end := strings.Index(sql[start:], " FROM g_rows") + if end < 0 { + t.Fatalf("g_rows boundary missing:\n%s", sql) + } + region := regexp.MustCompile(`\$[0-9]+`).ReplaceAllString(sql[start:start+end], "$N") + seen := map[string]string{} + return regexp.MustCompile(`"c[0-9]+"`).ReplaceAllStringFunc(region, func(match string) string { + if canonical, ok := seen[match]; ok { + return canonical + } + canonical := `"k` + strconv.Itoa(len(seen)) + `"` + seen[match] = canonical + return canonical + }) + } + if grp(baseSQL) != grp(swappedSQL) { + t.Fatalf("g_grp must not depend on source projection order:\nbase: %s\nswapped: %s", grp(baseSQL), grp(swappedSQL)) + } +} + +func requireGroup13Refusal(t *testing.T, plan ir.PhysicalPlan, want string) { + t.Helper() + _, err := renderGroupPhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if err == nil { + t.Fatalf("plan must be refused (want %q)", want) + } + if !strings.Contains(err.Error(), want) { + t.Fatalf("refusal = %q, want substring %q", err, want) + } + if !strings.Contains(err.Error(), ErrUnsupportedPostgresPlan.Error()) && + !strings.HasPrefix(err.Error(), "validate physical plan:") { + t.Fatalf("refusal is neither a door refusal nor an ir plan refusal: %q", err) + } + if groupPlanMentionsStage(plan) && !stageSequenceCompositionMentions(plan) { + _, entryErr := RenderPhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if entryErr == nil || entryErr.Error() != err.Error() { + t.Fatalf("entry door refused differently: entry=%v dispatch=%v", entryErr, err) + } + } +} + +func group13AliveControl(t *testing.T) { + t.Helper() + if _, err := renderGroupPhysicalPlan(group13KeyedPlan(t, recipe.ConstructionGroupMissingKeyGroup), mustBinding(t)); err != nil { + t.Fatalf("alive control: the healthy base plan must render: %v", err) + } +} + +func TestGroupRefusalRegister(t *testing.T) { + base := group13KeyedPlan(t, recipe.ConstructionGroupMissingKeyGroup) + + cases := []struct { + name string + mutate func(t *testing.T, plan *ir.PhysicalPlan) + want string + }{ + {"L-13.1-multi-stage", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages = append(plan.StageSequence.Stages, ir.ClonePhysicalPlan(*plan).StageSequence.Stages[0]) + }, "exactly one GROUP construction stage"}, + {"L-13.1-foreign-stage-kind", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Kind = ir.PhysicalStageUnpivotOp + }, "operation \"UNPIVOT\" is not part of the registered group slice"}, + {"L-13.1-foreign-payload", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Filter = &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: "root", Path: []string{"project"}}, Right: &ir.PhysicalValue{BindKey: "project"}}} + }, "GROUP requires only a group payload"}, + {"L-13.1-not-inline-first-stage", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].InputStageID = "somewhere-else" + }, "admits only the inline first-stage GROUP"}, + {"L-13.10-cell-trace-return", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.CellTraceReturn = &ir.PhysicalCellTraceReturn{} + }, "row-lineage, cell-trace, and population-mapping returns"}, + {"L-13.10-preview-window", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.PreviewSourceWindowByRootID = true + }, "preview source windowing"}, + {"L-13.10-auth-resource-path-bind", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.OutputAuthResourcePathBindKey = "output_auth_resource_path" + }, "composite Combine prefix authorization path"}, + {"L-13.13-preview-limit-zero", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.PreviewLimitBindKey = "limit" + plan.BindVars["limit"] = 0 + }, "must be a positive int"}, + {"L-13.13-preview-limit-type", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.PreviewLimitBindKey = "limit" + plan.BindVars["limit"] = "25" + }, "must be a positive int"}, + {"L-13.13-construction-bind-blank", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.BindVars[plan.StageSequence.Stages[0].Group.ConstructionIDBindKey] = " " + }, "must be a non-empty string"}, + {"L-13.12-missing-key-policy", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.MissingKeyPolicy = "DROP" + }, "unsupported missing-key policy"}, + {"L-13.12-reducer-operation", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.Aggregates[0].Operation = "MEDIAN" + }, "unsupported aggregate operation"}, + {"L-13.12-row-value-policy", func(t *testing.T, plan *ir.PhysicalPlan) { + appendGroup13ManyRowValue(t, plan, "MANY", "STRING") + }, "has unsupported policy"}, + {"L-13.3-semantic-key", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.SemanticKey = &ir.PhysicalStageGroupSemanticKey{ + OutputColumn: "category", TupleVariable: "sem_tuple", LabelVariable: "sem_label", + SourceRootVariable: "root", SourceIdentityColumn: "_key", RootCollectionBindKey: "root_collection", + ResourceType: "Observation", CodingPath: "code.coding[]", + } + }, "semantic family Group key is a registered named gap"}, + {"L-13.4-coding-key", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.CodingKey = &ir.PhysicalStageGroupCodingKey{ + OutputColumn: "code", InputTupleVariable: "code_tuple", TupleVariable: "code_item", + LabelVariable: "code_label", MembershipColumn: "codes", + } + }, "CODING_IDENTITY Group key is refused"}, + {"L-13.6-root-contributor", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.RootContributorInputColumn = "contributors" + }, "root-contributor set is production-unreachable"}, + {"L-13.8-population-root", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.Operations[0].RootScan.Population = &ir.PhysicalPopulationRootSource{} + }, "population, cohort, and candidate-seed roots"}, + {"L-13.9-unnest-operation", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.Operations = append(plan.Operations[:1], append([]ir.PhysicalOperation{{Kind: ir.PhysicalUnnestOp, + Unnest: &ir.PhysicalUnnest{}}}, plan.Operations[1:]...)...) + }, "not part of the registered group source surface (L-13.9)"}, + {"L-13.2-set-operation", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.Operations = append(plan.Operations[:1], append([]ir.PhysicalOperation{{Kind: ir.PhysicalSetOp}}, plan.Operations[1:]...)...) + }, "group source-region SET is a pg-sql-16 capability gap"}, + {"L-13.9-missing-return", func(t *testing.T, plan *ir.PhysicalPlan) { + last := len(plan.Operations) - 1 + plan.Operations = append(plan.Operations[:last], plan.Operations[last+1:]...) + }, "requires a terminal RETURN"}, + {"L-13.11-key-kind", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.Keys[0].Kind = "ARRAY" + }, "unsupported scalar kind"}, + {"L-13.11-key-hidden", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.Keys[0].InputInternal = true + }, "hidden or field-sourced input surface"}, + {"L-13.11-key-field-source", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.Keys[0].FieldSource = &ir.PhysicalStageGroupFieldSource{ + Kind: ir.PhysicalStageGroupFieldSourceCurrentStage, + } + }, "hidden or field-sourced input surface"}, + {"L-13.11-row-value-field-source", func(t *testing.T, plan *ir.PhysicalPlan) { + appendGroup13ManyRowValue(t, plan, "ALL", "STRING") + plan.StageSequence.Stages[0].Group.RowValues[0].FieldSource = &ir.PhysicalStageGroupFieldSource{ + Kind: ir.PhysicalStageGroupFieldSourceCurrentStage, + } + }, "field-sourced input surface"}, + {"L-13.11-key-unknown-input", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.Keys[0].InputColumn = "no_such_column" + }, "not among the source projection names (L-13.11)"}, + {"L-13.11-aggregate-unknown-input", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.Aggregates[1].InputColumn = "no_such_column" + }, "not among the source projection names (L-13.11)"}, + {"L-13.13-count-rows-with-input", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.Aggregates[0].InputColumn = "status" + }, "COUNT_ROWS must not have an input column"}, + {"L-13.13-sum-without-input", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.Aggregates[3].InputColumn = "" + }, "requires an input column"}, + } + for _, cell := range cases { + t.Run(cell.name, func(t *testing.T) { + plan := ir.ClonePhysicalPlan(base) + cell.mutate(t, &plan) + requireGroup13Refusal(t, plan, cell.want) + }) + } + t.Run("L-13.5-related-occurrence-routes-to-composition", func(t *testing.T) { + plan := ir.ClonePhysicalPlan(base) + plan.StageSequence.Stages[0].Group.RelatedOccurrence = &ir.PhysicalStageGroupRelatedOccurrenceAdapter{RouteID: "r"} + if !stageSequenceRelatedOccurrenceMentions(plan) { + t.Fatal("the entry claim must route the adapter shape to the composition slice") + } + _, entryErr := RenderPhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if entryErr == nil || !strings.Contains(entryErr.Error(), "unreferenced related route adapter") { + t.Fatalf("the entry must reach the composition door and the ir bound-field gate: %v", entryErr) + } + if strings.Contains(entryErr.Error(), "registered group slice") || + strings.Contains(entryErr.Error(), "L-13.5; correlated-hop") { + t.Fatalf("the group door must not answer an adapter-bearing sequence: %v", entryErr) + } + _, directErr := renderGroupPhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if directErr == nil || !strings.Contains(directErr.Error(), + "related-occurrence Group adapter is a pg-sql-16 capability gap") { + t.Fatalf("the direct door must keep the registered L-13.5 defense-in-depth refusal: %v", directErr) + } + }) + + group13AliveControl(t) +} + +func TestGroupDuplicateNameRefusals(t *testing.T) { + t.Run("duplicate-projection-names", func(t *testing.T) { + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + last := len(plan.Operations) - 1 + dup := plan.Operations[last].Return.Projections[0] + plan.Operations[last].Return.Projections = append(plan.Operations[last].Return.Projections, dup) + requireGroup13Refusal(t, plan, "is empty or duplicated") + }) + t.Run("duplicate-final-column", func(t *testing.T) { + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + plan.StageSequence.FinalColumns = append(plan.StageSequence.FinalColumns, + plan.StageSequence.FinalColumns[0]) + requireGroup13Refusal(t, plan, "final output schema differs from the final stage") + }) + t.Run("unregistered-final-column", func(t *testing.T) { + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + plan.StageSequence.FinalColumns = append(plan.StageSequence.FinalColumns, + ir.PhysicalStageColumn{ID: "ghost", Name: "ghost", Kind: "string", Cardinality: "optional_one", Nullable: true}) + requireGroup13Refusal(t, plan, "final output schema differs from the final stage") + }) + t.Run("key-output-without-stable-id", func(t *testing.T) { + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + for i := range plan.StageSequence.Stages[0].OutputColumns { + if plan.StageSequence.Stages[0].OutputColumns[i].Name == "status_group" { + plan.StageSequence.Stages[0].OutputColumns[i].ID = "" + } + } + _, err := renderGroupPhysicalPlan(plan, mustBinding(t)) + if err == nil { + t.Fatal("must refuse a key output without a stable column ID") + } + }) + t.Run("whole-document-direct-value", func(t *testing.T) { + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + last := len(plan.Operations) - 1 + plan.Operations[last].Return.Projections = append(plan.Operations[last].Return.Projections, + ir.PhysicalProjection{Name: "doc", Value: ir.PhysicalValue{Variable: "root"}}) + requireGroup13Refusal(t, plan, "scalar projection") + }) + t.Run("composite-input-refused", func(t *testing.T) { + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + last := len(plan.Operations) - 1 + objExpr := ir.PhysicalExpression{ + Kind: ir.PhysicalObjectExpression, Cardinality: ir.PhysicalObjectCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Object: &ir.PhysicalObject{Fields: []ir.PhysicalExpressionProjection{{ + Name: "gender", + Expression: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{Variable: "root", Path: []string{"payload", "gender"}}, + }, + }}}, + } + plan.Operations[last].Return.Projections = append(plan.Operations[last].Return.Projections, + ir.PhysicalProjection{Name: "obj", Expression: &objExpr}) + stage := &plan.StageSequence.Stages[0] + stage.Group.Aggregates[0] = ir.PhysicalStageGroupAggregate{ + Operation: "COUNT_DISTINCT", InputColumn: "obj", Output: "rows", Variable: stage.Group.Aggregates[0].Variable, + InputKind: "STRING", OutputKind: "integer", + } + requireGroup13Refusal(t, plan, "is not a public scalar input") + }) + t.Run("unit-normalized-refused", func(t *testing.T) { + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + last := len(plan.Operations) - 1 + plan.Operations[last].Return.Projections = append(plan.Operations[last].Return.Projections, + sql21UnitProjection(t, nil)) + requireGroup13Refusal(t, plan, "unit normalization belongs to the read spine registration (unit_normalization.go), not the closed GROUP slice") + }) +} + +func TestGroupEntryDispatchEquality(t *testing.T) { + plan := group13KeyedPlan(t, recipe.ConstructionGroupMissingKeyGroup) + plan.StageSequence.Stages[0].Group.MissingKeyPolicy = "NOPE" + _, entryErr := RenderPhysicalPlan(plan, mustBinding(t)) + _, dispatchErr := renderGroupPhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if entryErr == nil || dispatchErr == nil || entryErr.Error() != dispatchErr.Error() { + t.Fatalf("entry/dispatch refusal mismatch: entry=%v dispatch=%v", entryErr, dispatchErr) + } + spine := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + spine.StageSequence.Stages[0].Kind = ir.PhysicalStagePivotOp + spine.StageSequence.Stages[0].Group = nil + if groupPlanMentionsStage(spine) { + t.Fatal("a PIVOT-kind sequence must not trigger the group door") + } + _, err := RenderPhysicalPlan(spine, mustBinding(t)) + if err == nil { + t.Fatal("the non-group sequence must be refused") + } + if strings.Contains(err.Error(), "registered group slice") { + t.Fatalf("the group door must not answer a non-GROUP sequence: %v", err) + } +} + +func TestGroupStructuralLaws(t *testing.T) { + source := readGroup13Source(t, "group.go") + + for _, line := range strings.Split(source, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "//") || strings.HasPrefix(trimmed, "/*") || strings.HasPrefix(trimmed, "*") { + continue + } + if strings.Contains(line, "btrim(") { + t.Fatalf(`group.go must not bind btrim in wave 1 (wave 2 pre-approval covers the ONE groupTrimCharacters gate only): %s`, trimmed) + } + if strings.Contains(line, "codedGroupTrimCharacters") { + t.Fatalf("group.go must never bind the coded-group trim constant: %s", trimmed) + } + } + if got := strings.Count(source, "const groupTrimCharacters"); got != 1 { + t.Fatalf("groupTrimCharacters declarations = %d, want 1", got) + } + + if got := strings.Count(source, "render"+"Unnest"); got != 0 { + t.Fatalf("group.go lateral-expansion emitter references = %d, want 0 (L-13.9 refusal is the single UNNEST gate)", got) + } + + start := strings.Index(source, "func (r *planRenderer) materializeGroupSource") + end := strings.Index(source[start:], "\nfunc groupResolveColumn") + if start < 0 || end < 0 { + t.Fatal("materializeGroupSource boundary not found for the switch-exhaustiveness law") + } + switchBody := source[start : start+end] + for _, required := range []string{ + "case extractionUnitNormalized:", + "return nil, fmt.Errorf(\"%w: group source projection %q is a unit-normalized extraction", + "default:", + "return nil, fmt.Errorf(\"%w: group source projection %q is not a registered projection arm", + } { + if !strings.Contains(switchBody, required) { + t.Fatalf("materializeGroupSource switch is missing the required arm %q", required) + } + } + + if got := strings.Count(source, "func groupRowValueExpressionFor"); got != 1 { + t.Fatalf("ALL/ONE emitter declarations = %d, want 1 (the shared Q4 emitter)", got) + } + + for _, file := range []string{"render.go", "coded_group.go", "graph.go", "unnest.go", "unpivot.go"} { + other := readGroup13Source(t, file) + if strings.Contains(other, "renderGroupPhysicalPlan") && file != "render.go" { + t.Fatalf("%s must not dispatch the group slice", file) + } + } + if got := strings.Count(readGroup13Source(t, "render.go"), "renderGroupPhysicalPlan("); got != 1 { + t.Fatalf("render.go group door calls = %d, want 1", got) + } +} + +func readGroup13Source(t *testing.T, name string) string { + t.Helper() + raw, err := os.ReadFile(name) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + return string(raw) +} + +func newGroup13Service(t *testing.T) *pgir02Service { + t.Helper() + return newOwnedService(t, 0, "verify_sql13v1") +} + +func group13RenderSvc(t *testing.T, svc *pgir02Service, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan(%s): %v", svc.schema, err) + } + return rendered +} + +var ( + group13FixturesMu sync.Mutex + group13TrackedFixtures = map[string]bool{} +) + +func group13TrackFixtures(t *testing.T, svc *pgir02Service) { + t.Helper() + var database string + if err := svc.iso.QueryRow(svc.ctx, "SELECT current_database()").Scan(&database); err != nil { + svc.fail("track owned database", err) + } + group13FixturesMu.Lock() + defer group13FixturesMu.Unlock() + group13TrackedFixtures[database] = true + group13TrackedFixtures[svc.schema] = true +} + +func group13SnapshotOwnedResources(ctx context.Context, raw string) ([]string, error) { + pool, err := pgxpool.New(ctx, raw) + if err != nil { + return nil, fmt.Errorf("connect: %w", err) + } + defer pool.Close() + rows, err := pool.Query(ctx, + `SELECT datname FROM pg_database WHERE datname LIKE 'verify_sql13v1_%'`+ + ` UNION ALL SELECT nspname FROM pg_namespace WHERE nspname LIKE 'verify_sql13v1_%'`) + if err != nil { + return nil, fmt.Errorf("snapshot: %w", err) + } + defer rows.Close() + var names []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("scan: %v", err) + } + names = append(names, name) + } + return names, rows.Err() +} + +func group13RunDeltaAudit(inner func() int) int { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return inner() + } + beforeList, err := group13SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-13 delta audit: snapshot before run: %v\n", err) + return inner() + } + if len(beforeList) > 0 { + fmt.Fprintf(os.Stderr, "pg-sql-13 delta audit: pre-snapshot found %d surviving verify_sql13v1_* resources before this run: %v\n", len(beforeList), beforeList) + return 1 + } + before := make(map[string]bool, len(beforeList)) + for _, name := range beforeList { + before[name] = true + } + code := inner() + after, err := group13SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-13 delta audit: snapshot after run: %v\n", err) + code = 1 + } + group13FixturesMu.Lock() + defer group13FixturesMu.Unlock() + for _, name := range after { + if before[name] { + continue + } + if group13TrackedFixtures[name] { + fmt.Fprintf(os.Stderr, "pg-sql-13 delta audit: %q was created and tracked by this run and still survives\n", name) + code = 1 + continue + } + fmt.Fprintf(os.Stderr, "pg-sql-13 delta audit: note — %q was born during this run but is not owned by it (sibling lane?)\n", name) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + survivors, err := group13SnapshotOwnedResources(ctx, raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-13 delta audit: survivor scan: %v\n", err) + code = 1 + } else if len(survivors) > 0 { + fmt.Fprintf(os.Stderr, "pg-sql-13 delta audit: %d verify_sql13v1_* resources survive: %v\n", len(survivors), survivors) + code = 1 + } + return code +} + +func group13Insert(t *testing.T, svc *pgir02Service, logicalID, project, generation, authPath string, payload map[string]any) { + t.Helper() + svc.insertResource(t, svc.schema, project, generation, "Observation", logicalID, authPath, payload) +} + +func group13StatusRow(status string) map[string]any { + return map[string]any{"status": status} +} + +func group13ValueOnlyRow(v any) map[string]any { + return map[string]any{"valueQuantity": map[string]any{"value": v}} +} + +func group13ValueRow(status string, v any) map[string]any { + return map[string]any{"status": status, "valueQuantity": map[string]any{"value": v}} +} + +func group13MainFixture(t *testing.T, svc *pgir02Service) { + t.Helper() + group13Insert(t, svc, "o-1", group13Project, group13Generation, group13AuthPath, group13ValueRow("final", 2)) + group13Insert(t, svc, "o-2", group13Project, group13Generation, group13AuthPath, group13ValueRow("final", 2)) + group13Insert(t, svc, "o-3", group13Project, group13Generation, group13AuthPath, group13ValueRow("final", 2.5)) + group13Insert(t, svc, "o-4", group13Project, group13Generation, group13AuthPath, group13ValueRow("amended", 3)) + group13Insert(t, svc, "o-5", group13Project, group13Generation, group13AuthPath, group13StatusRow("final")) + group13Insert(t, svc, "o-6", group13Project, group13Generation, group13AuthPath, group13ValueOnlyRow(5)) + group13Insert(t, svc, "o-7", group13Project, group13Generation, group13AuthPath, group13ValueRow("final", nil)) + group13Insert(t, svc, "w-denied", group13Project, group13Generation, "FHIR/Hidden", group13ValueRow("final", 2)) + group13Insert(t, svc, "w-oldgen", group13Project, "p13-old-gen", group13AuthPath, group13ValueRow("final", 2)) + group13Insert(t, svc, "w-foreign", "p13-foreign", group13Generation, group13AuthPath, group13ValueRow("final", 2)) + if rows := svc.witness(t, `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" WHERE project = $1 AND dataset_generation = $2 AND auth_resource_path = $3`, + group13Project, group13Generation, group13AuthPath); rows[0][0] != "7" { + t.Fatalf("scope witness: acceptance row count = %v, want 7", rows) + } +} + +func group13KeyedRows(t *testing.T, plan ir.PhysicalPlan) [][]string { + t.Helper() + null := "" + return [][]string{ + {null, "5", "1", "1", "0", "5", "5", "5", "5", + group13Identity(t, plan, []group13Member{group13NullMember("status_group"), group13NumericMember("value_group", "5")})}, + {"amended", "3", "1", "1", "1", "3", "3", "3", "3", + group13Identity(t, plan, []group13Member{group13TextMember("status_group", "amended"), group13NumericMember("value_group", "3")})}, + {"final", null, "2", "0", "1", null, null, null, null, + group13Identity(t, plan, []group13Member{group13TextMember("status_group", "final"), group13NullMember("value_group")})}, + {"final", "2", "2", "2", "1", "4", "2", "2", "2", + group13Identity(t, plan, []group13Member{group13TextMember("status_group", "final"), group13NumericMember("value_group", "2")})}, + {"final", "2.5", "1", "1", "1", "2.5", "2.5", "2.5", "2.5", + group13Identity(t, plan, []group13Member{group13TextMember("status_group", "final"), group13NumericMember("value_group", "2.5")})}, + } +} + +func TestGroupLiveExactGroupsMultiset(t *testing.T) { + svc := newGroup13Service(t) + group13TrackFixtures(t, svc) + group13MainFixture(t, svc) + + plan := group13KeyedPlan(t, recipe.ConstructionGroupMissingKeyGroup) + rendered := group13RenderSvc(t, svc, plan) + for _, leaked := range []string{group13Project, group13Generation, group13AuthPath, "final", "amended", "Observation"} { + if strings.Contains(rendered.SQL, leaked) { + t.Fatalf("value %q leaked into statement text: %s", leaked, rendered.SQL) + } + } + + group13ExplainBothModes(t, svc, rendered) + + rows := svc.runRendered(t, rendered) + if !reflect.DeepEqual(rows, group13KeyedRows(t, plan)) { + t.Fatalf("grouped rows mismatch:\n got %v\nwant %v", rows, group13KeyedRows(t, plan)) + } + + if repeated := svc.runRendered(t, rendered); !reflect.DeepEqual(repeated, rows) { + t.Fatalf("grouping changed between executions:\n%v\n%v", rows, repeated) + } + + seen := map[string]bool{} + for _, row := range rows { + if seen[row[9]] { + t.Fatalf("duplicate row identity %q", row[9]) + } + seen[row[9]] = true + } + + excludePlan := group13KeyedPlan(t, recipe.ConstructionGroupMissingKeyExclude) + excluded := svc.runRendered(t, group13RenderSvc(t, svc, excludePlan)) + if len(excluded) != 3 { + t.Fatalf("EXCLUDE rows = %v, want the three full-key groups", excluded) + } + for _, row := range excluded { + if row[0] == "" || row[1] == "" { + t.Fatalf("EXCLUDE kept a null-key group: %v", row) + } + } + + errorPlan := group13KeyedPlan(t, recipe.ConstructionGroupMissingKeyError) + errorRendered := group13RenderSvc(t, svc, errorPlan) + group13ExpectRaise(t, svc, errorRendered, groupMissingKeyTag) + + limited := group13KeyedPlan(t, recipe.ConstructionGroupMissingKeyGroup) + limited.StageSequence.PreviewLimitBindKey = "limit" + limited.BindVars["limit"] = 2 + limitedRows := svc.runRendered(t, group13RenderSvc(t, svc, limited)) + if len(limitedRows) != 2 || limitedRows[0][0] != "" || limitedRows[1][0] != "amended" { + t.Fatalf("preview limit must keep the registered order's first two groups: %v", limitedRows) + } + + group13Insert(t, svc, "o-8", group13Project, group13Generation, group13AuthPath, group13ValueRow("final", 2)) + mutated := svc.runRendered(t, rendered) + if mutated[3][2] != "3" { + t.Fatalf("mutation probe: (final,2) rows never became 3: %v", mutated) + } +} + +func group13Args(rendered command.RenderedPostgresPhysicalPlan) []any { + args := make([]any, len(rendered.Args)) + for i, parameter := range rendered.Args { + args[i] = parameter.Value + } + return args +} + +func group13ExplainBothModes(t *testing.T, svc *pgir02Service, rendered command.RenderedPostgresPhysicalPlan) { + t.Helper() + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + conn, err := svc.iso.Acquire(svc.ctx) + if err != nil { + t.Fatalf("acquire pinned connection: %v", err) + } + if _, err := conn.Exec(svc.ctx, "SET plan_cache_mode = "+mode); err != nil { + conn.Release() + t.Fatalf("pin plan_cache_mode %s: %v", mode, err) + } + args := make([]any, len(rendered.Args)) + for i, parameter := range rendered.Args { + args[i] = parameter.Value + } + if _, err := conn.Exec(svc.ctx, "EXPLAIN "+rendered.SQL, args...); err != nil { + conn.Release() + t.Fatalf("plan-time EXPLAIN raised under %s: %v", mode, err) + } + conn.Release() + } +} + +func group13ExpectRaise(t *testing.T, svc *pgir02Service, rendered command.RenderedPostgresPhysicalPlan, tag string) { + t.Helper() + rows, err := svc.iso.Query(svc.ctx, rendered.SQL, group13Args(rendered)...) + if err == nil { + for rows.Next() { + } + err = rows.Err() + rows.Close() + } + if err == nil { + t.Fatalf("statement must raise %q through the barrier", tag) + } + scrubbed := svc.scrub(err) + if !strings.Contains(scrubbed, tag) || !strings.Contains(scrubbed, "22P02") { + t.Fatalf("must raise %q through the barrier cast: %v", tag, scrubbed) + } +} + +func TestGroupLiveValueTypeWitnesses(t *testing.T) { + svc := newGroup13Service(t) + group13TrackFixtures(t, svc) + group13MainFixture(t, svc) + + keyPlan := group13LoweredPlan(t, group13Fixture{ + policy: recipe.ConstructionGroupMissingKeyGroup, + keys: []recipe.ConstructionGroupKey{ + {InputColumnID: "status", OutputColumnID: "status_group"}, + {InputColumnID: "value", OutputColumnID: "value_group"}, + }, + aggregates: []recipe.ConstructionGroupAggregate{ + {Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}, + }, + outputs: []recipe.StageColumn{ + {ID: "status_group", Name: "status_group", Type: "string", Nullable: true}, + {ID: "value_group", Name: "value_group", Type: "decimal", Nullable: true}, + {ID: "rows_id", Name: "rows", Type: "integer"}, + }, + }) + rendered := group13RenderSvc(t, svc, keyPlan) + group13ExplainBothModes(t, svc, rendered) + if healthy := svc.runRendered(t, rendered); len(healthy) != 5 { + t.Fatalf("key-witness alive control: healthy rows = %v, want the five groups", healthy) + } + group13Insert(t, svc, "p-key", group13Project, group13Generation, group13AuthPath, + map[string]any{"status": "poisoned", "valueQuantity": map[string]any{"value": "abc"}}) + group13ExpectRaise(t, svc, rendered, groupKeyTypeTag) + + valuePlan := group13LoweredPlan(t, group13Fixture{ + policy: recipe.ConstructionGroupMissingKeyGroup, + keys: []recipe.ConstructionGroupKey{{InputColumnID: "status", OutputColumnID: "status_group"}}, + aggregates: []recipe.ConstructionGroupAggregate{ + {Operation: recipe.ConstructionGroupSum, InputColumnID: "value", OutputColumnID: "sum_id"}, + }, + outputs: []recipe.StageColumn{ + {ID: "status_group", Name: "status_group", Type: "string", Nullable: true}, + {ID: "sum_id", Name: "total", Type: "decimal", Nullable: true}, + }, + }) + valueRendered := group13RenderSvc(t, svc, valuePlan) + group13ExplainBothModes(t, svc, valueRendered) + group13ExpectRaise(t, svc, valueRendered, groupValueTypeTag) +} + +func TestGroupLiveRowValues(t *testing.T) { + svc := newGroup13Service(t) + group13TrackFixtures(t, svc) + group13Insert(t, svc, "rv-a", group13Project, group13Generation, group13AuthPath, group13ValueRow("rv", 3)) + group13Insert(t, svc, "rv-b", group13Project, group13Generation, group13AuthPath, group13ValueRow("rv", 1)) + group13Insert(t, svc, "rv-c", group13Project, group13Generation, group13AuthPath, group13ValueRow("rv", 3)) + group13Insert(t, svc, "rv-d", group13Project, group13Generation, group13AuthPath, group13StatusRow("rv")) + group13Insert(t, svc, "rv-e", group13Project, group13Generation, group13AuthPath, group13ValueRow("rv", nil)) + group13Insert(t, svc, "rv-f", group13Project, group13Generation, group13AuthPath, group13StatusRow("rvnull")) + + allPlan := group13RowValuePlan(t, recipe.ConstructionRowValueAll, "array") + allRendered := group13RenderSvc(t, svc, allPlan) + group13ExplainBothModes(t, svc, allRendered) + rows := svc.runRendered(t, allRendered) + if len(rows) != 2 { + t.Fatalf("ALL rows = %v, want two groups", rows) + } + if rows[0][0] != "rv" || rows[0][2] != `[1 3]` { + t.Fatalf("ALL distinct-sorted null-free array mismatch: %v", rows[0]) + } + if rows[1][0] != "rvnull" || rows[1][2] != `[]` { + t.Fatalf("all-missing group must read the empty array: %v", rows[1]) + } +} + +func TestGroupLiveRowValueOne(t *testing.T) { + svc := newGroup13Service(t) + group13TrackFixtures(t, svc) + group13Insert(t, svc, "ro-a", group13Project, group13Generation, group13AuthPath, group13ValueRow("same", 7)) + group13Insert(t, svc, "ro-b", group13Project, group13Generation, group13AuthPath, group13ValueRow("same", 7)) + group13Insert(t, svc, "ro-c", group13Project, group13Generation, group13AuthPath, group13StatusRow("onlynull")) + + plan := group13RowValuePlan(t, recipe.ConstructionRowValueOne, "decimal") + rendered := group13RenderSvc(t, svc, plan) + group13ExplainBothModes(t, svc, rendered) + rows := svc.runRendered(t, rendered) + if len(rows) != 2 || rows[0][0] != "onlynull" || rows[0][2] != "" || rows[1][0] != "same" || rows[1][2] != "7" { + t.Fatalf("ONE rows = %v, want (onlynull,) then (same,7)", rows) + } + + group13Insert(t, svc, "ro-d", group13Project, group13Generation, group13AuthPath, group13ValueRow("same", 8)) + group13ExpectRaise(t, svc, rendered, groupRowValueMultipleTag) +} + +func TestGroupLiveRowValueOneString(t *testing.T) { + svc := newGroup13Service(t) + group13TrackFixtures(t, svc) + group13Insert(t, svc, "rs-a", group13Project, group13Generation, group13AuthPath, + map[string]any{"status": "same", "id": "s-1"}) + plan := group13LoweredPlan(t, group13Fixture{ + policy: recipe.ConstructionGroupMissingKeyGroup, + keys: []recipe.ConstructionGroupKey{{InputColumnID: "status", OutputColumnID: "status_group"}}, + aggregates: []recipe.ConstructionGroupAggregate{ + {Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}, + }, + rowValues: []recipe.ConstructionRowValue{ + {InputColumnID: "resource_id", OutputColumnID: "id_one", Policy: recipe.ConstructionRowValueOne}, + }, + outputs: []recipe.StageColumn{ + {ID: "status_group", Name: "status_group", Type: "string", Nullable: true}, + {ID: "rows_id", Name: "rows", Type: "integer"}, + {ID: "id_one", Name: "id_one", Type: "string", Nullable: true}, + }, + }) + rendered := group13RenderSvc(t, svc, plan) + group13ExplainBothModes(t, svc, rendered) + if rows := svc.runRendered(t, rendered); len(rows) != 1 || rows[0][2] != "s-1" { + t.Fatalf("ONE over one STRING member: %v, want the single id", rows) + } + group13Insert(t, svc, "rs-b", group13Project, group13Generation, group13AuthPath, + map[string]any{"status": "same", "id": "s-2"}) + group13ExpectRaise(t, svc, rendered, groupRowValueMultipleTag) +} + +func TestGroupLiveRowValueInputMany(t *testing.T) { + svc := newGroup13Service(t) + group13TrackFixtures(t, svc) + group13Insert(t, svc, "mn-a", group13Project, group13Generation, group13AuthPath, map[string]any{"status": "many", "values": []any{"b", "a", "b"}}) + group13Insert(t, svc, "mn-b", group13Project, group13Generation, group13AuthPath, map[string]any{"status": "many", "values": []any{"c"}}) + + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + appendGroup13ManyRowValue(t, &plan, "ALL", "STRING") + rendered := group13RenderSvc(t, svc, plan) + group13ExplainBothModes(t, svc, rendered) + rows := svc.runRendered(t, rendered) + if len(rows) != 1 || rows[0][0] != "many" || rows[0][3] != `[a b c]` { + t.Fatalf("InputMany ALL must flatten one layer and dedupe across rows: %v", rows) + } + + group13Insert(t, svc, "mn-c", group13Project, group13Generation, group13AuthPath, map[string]any{"status": "many", "values": "oops"}) + group13ExpectRaise(t, svc, rendered, groupValueTypeTag) +} + +func TestGroupLiveRowValueInputManyDecimal(t *testing.T) { + svc := newGroup13Service(t) + group13TrackFixtures(t, svc) + group13Insert(t, svc, "dm-a", group13Project, group13Generation, group13AuthPath, + map[string]any{"status": "pool", "values": []any{1, 2, 2}}) + group13Insert(t, svc, "dm-b", group13Project, group13Generation, group13AuthPath, + map[string]any{"status": "pool", "values": []any{3}}) + + allPlan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + appendGroup13ManyRowValue(t, &allPlan, "ALL", "DECIMAL") + allRendered := group13RenderSvc(t, svc, allPlan) + group13ExplainBothModes(t, svc, allRendered) + rows := svc.runRendered(t, allRendered) + if len(rows) != 1 || rows[0][3] != `[1 2 3]` { + t.Fatalf("DECIMAL ALL pool must dedupe and sort numerically: %v", rows) + } + + group13Insert(t, svc, "dm-c", group13Project, group13Generation, group13AuthPath, + map[string]any{"status": "pool", "values": []any{1, "oops"}}) + group13ExpectRaise(t, svc, group13RenderSvc(t, svc, allPlan), groupValueTypeTag) + + oneSvc := newGroup13Service(t) + group13TrackFixtures(t, oneSvc) + group13Insert(t, oneSvc, "dn-a", group13Project, group13Generation, group13AuthPath, + map[string]any{"status": "pool", "values": []any{7}}) + onePlan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + appendGroup13ManyRowValue(t, &onePlan, "ONE", "DECIMAL") + oneRendered := group13RenderSvc(t, oneSvc, onePlan) + group13ExplainBothModes(t, oneSvc, oneRendered) + oneRows := oneSvc.runRendered(t, oneRendered) + if len(oneRows) != 1 || oneRows[0][3] != "7" { + t.Fatalf("DECIMAL ONE pool must read the single numeric value: %v", oneRows) + } +} + +func TestGroupLiveDoubleSumOverflow(t *testing.T) { + svc := newGroup13Service(t) + group13TrackFixtures(t, svc) + + conn, err := svc.iso.Acquire(svc.ctx) + if err != nil { + t.Fatalf("acquire pinned connection: %v", err) + } + _, err = conn.Exec(svc.ctx, + `SELECT SUM(x) FROM (VALUES ('1.7e308'::double precision), ('1.7e308'::double precision)) v(x)`) + if err == nil || !strings.Contains(err.Error(), "22003") { + conn.Release() + t.Fatalf("fault control: float8 SUM overflow must raise 22003 on PG18.6, got %v", err) + } + conn.Release() + + fixture := group13Fixture{ + policy: recipe.ConstructionGroupMissingKeyGroup, + keys: []recipe.ConstructionGroupKey{ + {InputColumnID: "status", OutputColumnID: "status_group"}, + }, + aggregates: []recipe.ConstructionGroupAggregate{ + {Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}, + {Operation: recipe.ConstructionGroupSum, InputColumnID: "value", OutputColumnID: "sum_id"}, + {Operation: recipe.ConstructionGroupMean, InputColumnID: "value", OutputColumnID: "mean_id"}, + }, + outputs: []recipe.StageColumn{ + {ID: "status_group", Name: "status_group", Type: "string", Nullable: true}, + {ID: "rows_id", Name: "rows", Type: "integer"}, + {ID: "sum_id", Name: "total", Type: "decimal", Nullable: true}, + {ID: "mean_id", Name: "mean", Type: "decimal", Nullable: true}, + }, + } + plan := group13LoweredPlan(t, fixture) + for i := 0; i < 2; i++ { + group13Insert(t, svc, fmt.Sprintf("ovf-p%d", i), group13Project, group13Generation, group13AuthPath, group13ValueRow("pair", 1.7e308)) + } + for i := 0; i < 20; i++ { + group13Insert(t, svc, fmt.Sprintf("ovf-c%d", i), group13Project, group13Generation, group13AuthPath, group13ValueRow("crowd", 1e307)) + } + for i := 0; i < 2; i++ { + group13Insert(t, svc, fmt.Sprintf("ovf-k%d", i), group13Project, group13Generation, group13AuthPath, group13ValueRow("kept", 1e307)) + } + for i := 0; i < 17; i++ { + group13Insert(t, svc, fmt.Sprintf("ovf-s%d", i), group13Project, group13Generation, group13AuthPath, group13ValueRow("seventeen", 1e307)) + } + + rendered := group13RenderSvc(t, svc, plan) + + group13ExplainBothModes(t, svc, rendered) + + rows := svc.runRendered(t, rendered) + if len(rows) != 4 { + t.Fatalf("four groups expected: %v", rows) + } + sevSum, sevErr := strconv.ParseFloat("1.7e308", 64) + if sevErr != nil { + t.Fatalf("parse 1.7e308: %v", sevErr) + } + want := map[string]struct { + count string + sum, mean string + sumVal, meanVal float64 + exact bool + }{ + "crowd": {count: "20", sum: "0", mean: "0"}, + "pair": {count: "2", sum: "0", mean: "0"}, + "kept": {count: "2", sumVal: 2e307, meanVal: 1e307, exact: true}, + "seventeen": {count: "17", sumVal: sevSum, meanVal: sevSum / 17, exact: true}, + } + for _, row := range rows { + w, ok := want[row[0]] + if !ok { + t.Fatalf("unexpected group %q", row[0]) + } + if row[1] != w.count { + t.Fatalf("group %s count = %q, want %q: %v", row[0], row[1], w.count, row) + } + if !w.exact { + if row[2] != w.sum || row[3] != w.mean { + t.Fatalf("overflow-band group %s = %v, want [%s 0 0]", row[0], row, w.count) + } + continue + } + sumVal, sumErr := strconv.ParseFloat(row[2], 64) + meanVal, meanErr := strconv.ParseFloat(row[3], 64) + if sumErr != nil || meanErr != nil || sumVal != w.sumVal || meanVal != w.meanVal { + t.Fatalf("representable group %s = %v, want exact %v/%v", row[0], row, w.sumVal, w.meanVal) + } + } +} + +func TestGroupLiveKeylessGrandTotal(t *testing.T) { + svc := newGroup13Service(t) + group13TrackFixtures(t, svc) + group13Insert(t, svc, "w-only", "p13-foreign", group13Generation, group13AuthPath, group13ValueRow("final", 1)) + + for _, policy := range []ir.PhysicalStageGroupMissingKeyPolicy{ + ir.PhysicalStageGroupMissingKeyGroup, ir.PhysicalStageGroupMissingKeyError, + } { + plan := group13KeylessPlan(t) + plan.StageSequence.Stages[0].Group.MissingKeyPolicy = policy + keylessRendered := group13RenderSvc(t, svc, plan) + group13ExplainBothModes(t, svc, keylessRendered) + rows := svc.runRendered(t, keylessRendered) + if len(rows) != 1 || rows[0][0] != "0" || rows[0][1] != "[]" || rows[0][2] != "" { + t.Fatalf("keyless grand total over zero rows under %s = %v, want one row [0,[],null]", policy, rows) + } + } +} + +func TestGroupLiveSourceOrderSwap(t *testing.T) { + svc := newGroup13Service(t) + group13TrackFixtures(t, svc) + group13MainFixture(t, svc) + + fixture := group13Fixture{ + policy: recipe.ConstructionGroupMissingKeyGroup, + keys: []recipe.ConstructionGroupKey{ + {InputColumnID: "status", OutputColumnID: "status_group"}, + {InputColumnID: "value", OutputColumnID: "value_group"}, + }, + aggregates: []recipe.ConstructionGroupAggregate{ + {Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}, + {Operation: recipe.ConstructionGroupSum, InputColumnID: "value", OutputColumnID: "sum_id"}, + }, + outputs: []recipe.StageColumn{ + {ID: "status_group", Name: "status_group", Type: "string", Nullable: true}, + {ID: "value_group", Name: "value_group", Type: "decimal", Nullable: true}, + {ID: "rows_id", Name: "rows", Type: "integer"}, + {ID: "sum_id", Name: "total", Type: "decimal", Nullable: true}, + }, + } + planA := group13LoweredPlan(t, fixture) + swapped := fixture + swapped.sourceOrder = []string{"resource_id", "value", "status"} + planB := group13LoweredPlan(t, swapped) + renderedA := group13RenderSvc(t, svc, planA) + renderedB := group13RenderSvc(t, svc, planB) + group13ExplainBothModes(t, svc, renderedA) + group13ExplainBothModes(t, svc, renderedB) + rowsA := svc.runRendered(t, renderedA) + rowsB := svc.runRendered(t, renderedB) + if !reflect.DeepEqual(rowsA, rowsB) { + t.Fatalf("swapping the source projection order must not change results (the name→column map must follow names):\nA %v\nB %v", rowsA, rowsB) + } + if len(rowsA) != 5 { + t.Fatalf("expected five groups, got %v", rowsA) + } + if rowsB[3][0] != "final" || rowsB[3][1] != "2" || rowsB[3][3] != "4" { + t.Fatalf("value key misread after swap: %v", rowsB) + } +} diff --git a/internal/dataframe/compiler/render/postgres/main_test.go b/internal/dataframe/compiler/render/postgres/main_test.go new file mode 100644 index 000000000..b82f3146e --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/main_test.go @@ -0,0 +1,80 @@ +package postgres + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestMain(m *testing.M) { + code := pwin01RunDeltaAudit(func() int { + return sql24RunDeltaAudit(func() int { + return sql22RunDeltaAudit(func() int { + return sql23RunDeltaAudit(func() int { + return excl26RunDeltaAudit(func() int { + return sql25RunDeltaAudit(func() int { + return sql16RunDeltaAudit(func() int { + return pivot14RunDeltaAudit(func() int { + return group13RunDeltaAudit(func() int { + return sql21RunDeltaAudit(func() int { + return coded19RunDeltaAudit(func() int { + return obj11RunDeltaAudit(func() int { + return graph20RunDeltaAudit(func() int { return unpRunDeltaAudit(func() int { return sets08RunDeltaAudit(m) }) }) + }) + }) + }) + }) + }) + }) + }) + }) + }) + }) + }) + }) + for _, prefix := range []string{"verify_sql05_", "verify_sql10_", "verify_sql10v7_", "verify_sql21v1_", "verify_sql13v1_", "verify_sql14v1_", "verify_sql16v1_", "verify_sql25v1_", "verify_sql26v1_", "verify_sql23v1_", "verify_sql22v1_", "verify_sql24v1_"} { + if err := auditNoLeftoverVerifyNamespace(prefix); err != nil { + fmt.Fprintf(os.Stderr, "zero-leftover audit FAILED: %v\n", err) + if code == 0 { + code = 1 + } + } + } + os.Exit(code) +} + +func auditNoLeftoverVerifyNamespace(prefix string) error { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + return fmt.Errorf("connect: %w", err) + } + defer admin.Close() + var databases int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE $1", prefix+"%").Scan(&databases); err != nil { + return fmt.Errorf("count databases: %w", err) + } + if databases != 0 { + return fmt.Errorf("%d %s* databases still exist", databases, prefix) + } + var schemas int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_namespace WHERE nspname LIKE $1", prefix+"%").Scan(&schemas); err != nil { + return fmt.Errorf("count schemas: %w", err) + } + if schemas != 0 { + return fmt.Errorf("%d %s* schemas still exist in the service database", schemas, prefix) + } + return nil +} diff --git a/internal/dataframe/compiler/render/postgres/null_semantics.go b/internal/dataframe/compiler/render/postgres/null_semantics.go new file mode 100644 index 000000000..e3df13f0f --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/null_semantics.go @@ -0,0 +1,142 @@ +package postgres + +import ( + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +func (r *planRenderer) documentColumn(alias string) string { + return alias + "." + r.storage.Resource.MustColumn("payload").SQLText() +} + +func (r *planRenderer) documentPathExpression(alias string, path []string) (string, error) { + return r.documentPathFromExpression(r.documentColumn(alias), path) +} + +func (r *planRenderer) documentPathFromExpression(base string, path []string) (string, error) { + if len(path) == 0 { + return "", fmt.Errorf("%w: a document path must have at least one segment", ErrUnsupportedPostgresPlan) + } + for _, segment := range path { + if !payloadPathSegmentPattern.MatchString(segment) { + return "", fmt.Errorf("%w: payload path segment %q is not renderer-safe", ErrUnsupportedPostgresPlan, segment) + } + } + elements := make([]string, len(path)) + for index, segment := range path { + elements[index] = arrayLiteralElement(segment) + } + pathLiteral := "{" + strings.Join(elements, ",") + "}" + return base + " #> $" + strconv.Itoa(r.textParameter(pathLiteral, "path:"+pathLiteral)) + "::text[]", nil +} + +type documentPathStep struct { + field string + index int + indexed bool +} + +func (r *planRenderer) documentPathStepsExpression(alias string, steps []documentPathStep) (string, error) { + return r.documentPathStepsFromExpression(r.documentColumn(alias), steps) +} + +func (r *planRenderer) documentPathStepsFromExpression(base string, steps []documentPathStep) (string, error) { + if len(steps) == 0 { + return "", fmt.Errorf("%w: a document path must have at least one segment", ErrUnsupportedPostgresPlan) + } + current := base + for start := 0; start < len(steps); { + end := start + for end < len(steps) && !(steps[end].indexed && steps[end].index < 0) { + end++ + } + if end > start { + elements := make([]string, 0, end-start) + for _, step := range steps[start:end] { + if !payloadPathSegmentPattern.MatchString(step.field) { + return "", fmt.Errorf("%w: payload path segment %q is not renderer-safe", ErrUnsupportedPostgresPlan, step.field) + } + if step.indexed { + if step.index < 0 { + return "", fmt.Errorf("%w: negative document path index %d under segment %q reached the non-negative chunk", ErrUnsupportedPostgresPlan, step.index, step.field) + } + elements = append(elements, arrayLiteralElement(step.field)) + elements = append(elements, arrayLiteralElement(strconv.Itoa(step.index))) + continue + } + elements = append(elements, arrayLiteralElement(step.field)) + } + pathLiteral := "{" + strings.Join(elements, ",") + "}" + current = current + " #> $" + strconv.Itoa(r.textParameter(pathLiteral, "path:"+pathLiteral)) + "::text[]" + } + if end >= len(steps) { + break + } + step := steps[end] + if !payloadPathSegmentPattern.MatchString(step.field) { + return "", fmt.Errorf("%w: payload path segment %q is not renderer-safe", ErrUnsupportedPostgresPlan, step.field) + } + arrLiteral := "{" + arrayLiteralElement(step.field) + "}" + arr := current + " #> $" + strconv.Itoa(r.textParameter(arrLiteral, "path:"+arrLiteral)) + "::text[]" + indexParameter := strconv.Itoa(r.addParameter( + command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(step.index)}, + "tailidx:"+strconv.Itoa(step.index))) + propertyLiteral := "{" + arrayLiteralElement(strconv.Itoa(step.index)) + "}" + property := arr + " #> $" + strconv.Itoa(r.textParameter(propertyLiteral, "path:"+propertyLiteral)) + "::text[]" + current = "(CASE WHEN jsonb_typeof(" + arr + ") = 'array' THEN (CASE WHEN $" + indexParameter + + " < 0 THEN " + arr + " -> (jsonb_array_length(" + arr + ") + $" + indexParameter + ") END) ELSE " + + property + " END)" + start = end + 1 + } + return current, nil +} + +func arrayLiteralElement(segment string) string { + if strings.EqualFold(segment, "null") { + return `"` + segment + `"` + } + return segment +} + +func (r *planRenderer) scalarExtractValueExpression(alias string, path []string, behavior ir.PhysicalNullBehavior) (string, error) { + extraction, err := r.documentPathExpression(alias, path) + if err != nil { + return "", err + } + switch behavior { + case ir.PhysicalPreserveNull: + return "(" + extraction + ")", nil + case ir.PhysicalOmitNulls: + return "NULLIF((" + extraction + "), 'null'::jsonb)", nil + case ir.PhysicalEmptyOnNull: + return "COALESCE(NULLIF((" + extraction + "), 'null'::jsonb), '\"\"'::jsonb)", nil + default: + return "", fmt.Errorf("%w: null behavior %q has no registered PostgreSQL extraction form", + ErrUnsupportedPostgresPlan, string(behavior)) + } +} + +func (r *planRenderer) propertyPresenceExpression(alias string, path []string) (string, error) { + if len(path) == 0 { + return "", fmt.Errorf("%w: a presence proof needs at least one property", ErrUnsupportedPostgresPlan) + } + for _, segment := range path { + if !payloadPathSegmentPattern.MatchString(segment) { + return "", fmt.Errorf("%w: presence path segment %q is not renderer-safe", ErrUnsupportedPostgresPlan, segment) + } + } + parent := r.documentColumn(alias) + if len(path) > 1 { + prefix, err := r.documentPathExpression(alias, path[:len(path)-1]) + if err != nil { + return "", err + } + parent = prefix + } + keyParameter := r.textParameter(path[len(path)-1], "text:"+path[len(path)-1]) + return "COALESCE(jsonb_typeof(" + parent + ") = 'object' AND jsonb_exists(" + parent + ", $" + strconv.Itoa(keyParameter) + "), false)", nil +} diff --git a/internal/dataframe/compiler/render/postgres/null_semantics_test.go b/internal/dataframe/compiler/render/postgres/null_semantics_test.go new file mode 100644 index 000000000..6987c9e74 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/null_semantics_test.go @@ -0,0 +1,472 @@ +package postgres + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" +) + +const ( + pgsq03Project = "pgsq03-proj" + pgsq03Other = "pgsq03-other" + pgsq03Gen = "gen-1" + pgsq03Path = "FHIR/Patient" +) + +func pgsq03DocumentSource() ir.PhysicalValue { + return ir.PhysicalValue{Variable: "root", Path: []string{"payload"}} +} + +func pgsq03SelectorPlan(resourceType string, projections ...ir.PhysicalProjection) ir.PhysicalPlan { + return ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": resourceType, + "project": pgsq03Project, + "dataset_generation": pgsq03Gen, + "auth_resource_paths": []string{pgsq03Path}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + }, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}}, + }, + } +} + +func pgsq03ExtractProjection(t *testing.T, name, resourceType, selectorPath string, + behavior ir.PhysicalNullBehavior, mode ir.PhysicalSelectorExecutionMode) ir.PhysicalProjection { + t.Helper() + selector, err := spec.ParseSelector(selectorPath) + if err != nil { + t.Fatalf("ParseSelector(%q): %v", selectorPath, err) + } + source := pgsq03DocumentSource() + return ir.PhysicalProjection{ + Name: name, + Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: behavior, + Extract: &ir.PhysicalExtract{ + Source: source, + ResourceType: resourceType, + Selector: selector, + ExecutionMode: mode, + }, + }, + Presence: &ir.PhysicalProjectionPresence{ + Source: source, + Paths: [][]string{pgsq03SelectorPath(t, selectorPath)}, + }, + } +} + +func pgsq03SelectorPath(t *testing.T, selectorPath string) []string { + t.Helper() + selector, err := spec.ParseSelector(selectorPath) + if err != nil { + t.Fatalf("ParseSelector(%q): %v", selectorPath, err) + } + path := make([]string, 0, len(selector.Steps)) + for _, step := range selector.Steps { + path = append(path, step.Field) + } + return path +} + +func pgsq03PresenceSidecar(name string, path ...string) ir.PhysicalProjection { + return ir.PhysicalProjection{ + Name: name, + Presence: &ir.PhysicalProjectionPresence{ + Source: pgsq03DocumentSource(), + Paths: [][]string{path}, + }, + PresenceOutput: true, + } +} + +func pgsq03RenderOK(t *testing.T, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderSelectorExtraction(plan, mustBinding(t)) + if err != nil { + t.Fatalf("RenderSelectorExtraction: %v", err) + } + return rendered +} + +const pgsq03GoldenTail = ` FROM "pgir02_it_unit"."fhir_resource" AS r0` + + ` WHERE r0."project" = $1` + + ` AND r0."dataset_generation" = $2` + + ` AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND r0."resource_type" = $4` + + ` ORDER BY r0."resource_seq" ASC` + +func pgsq03GoldenSQL(selects string) string { + return `SELECT ` + selects + pgsq03GoldenTail +} + +func pgsq03ScopeArgs() []command.PostgreSQLParameter { + return []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: pgsq03Project}, + {Type: command.PostgreSQLParameterText, Value: pgsq03Gen}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["FHIR/Patient"]`)}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + } +} + +func TestNullSemanticsGoldenPreserveNullValueAndPresence(t *testing.T) { + plan := pgsq03SelectorPlan("Patient", + ir.PhysicalProjection{Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + pgsq03ExtractProjection(t, "gender", "Patient", "gender", ir.PhysicalPreserveNull, ir.PhysicalSelectorDirectScalar), + pgsq03PresenceSidecar("gender_present", "gender"), + ) + rendered := pgsq03RenderOK(t, plan) + + wantSQL := pgsq03GoldenSQL(`r0."logical_id" AS "c0", (r0."payload" #> $5::text[]) AS "c1", ` + + `COALESCE(jsonb_typeof(r0."payload") = 'object' AND jsonb_exists(r0."payload", $6), false) AS "c2"`) + if rendered.SQL != wantSQL { + t.Fatalf("preserve-null SQL =\n%s\nwant\n%s", rendered.SQL, wantSQL) + } + wantArgs := append(pgsq03ScopeArgs(), + command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: "{gender}"}, + command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: "gender"}, + ) + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("preserve-null args = %#v, want %#v", rendered.Args, wantArgs) + } + if !reflect.DeepEqual(rendered.OutputColumns, []string{"logical_id", "gender", "gender_present"}) { + t.Fatalf("output columns = %v", rendered.OutputColumns) + } + for _, forbidden := range []string{"@", "gender", "Patient", pgsq03Project, pgsq03Gen, pgsq03Path} { + if strings.Contains(rendered.SQL, forbidden) { + t.Fatalf("SQL leaked %q: %s", forbidden, rendered.SQL) + } + } +} + +func TestNullSemanticsGoldenOmitNullsCollapsesOnlyJSONNull(t *testing.T) { + plan := pgsq03SelectorPlan("Patient", + pgsq03ExtractProjection(t, "gender", "Patient", "gender", ir.PhysicalOmitNulls, ir.PhysicalSelectorGeneric)) + rendered := pgsq03RenderOK(t, plan) + + wantSQL := pgsq03GoldenSQL(`NULLIF((r0."payload" #> $5::text[]), 'null'::jsonb) AS "c0"`) + if rendered.SQL != wantSQL { + t.Fatalf("omit-nulls SQL =\n%s\nwant\n%s", rendered.SQL, wantSQL) + } + wantArgs := append(pgsq03ScopeArgs(), + command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: "{gender}"}, + ) + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("omit-nulls args = %#v", rendered.Args) + } +} + +func TestNullSemanticsGoldenEmptyOnNullCoercesOnlyNullStates(t *testing.T) { + plan := pgsq03SelectorPlan("Patient", + pgsq03ExtractProjection(t, "gender", "Patient", "gender", ir.PhysicalEmptyOnNull, "")) + rendered := pgsq03RenderOK(t, plan) + + wantSQL := pgsq03GoldenSQL(`COALESCE(NULLIF((r0."payload" #> $5::text[]), 'null'::jsonb), '""'::jsonb) AS "c0"`) + if rendered.SQL != wantSQL { + t.Fatalf("empty-on-null SQL =\n%s\nwant\n%s", rendered.SQL, wantSQL) + } + if _, err := json.Marshal(rendered.Args); err != nil { + t.Fatalf("args not encodable: %v", err) + } +} + +func TestNullSemanticsGoldenNestedPathBindsPrefixAndTerminalKey(t *testing.T) { + plan := pgsq03SelectorPlan("Patient", + pgsq03ExtractProjection(t, "meta_source", "Patient", "meta.source", ir.PhysicalPreserveNull, ir.PhysicalSelectorDirectScalar), + pgsq03PresenceSidecar("meta_source_present", "meta", "source"), + ) + rendered := pgsq03RenderOK(t, plan) + + wantSQL := pgsq03GoldenSQL(`(r0."payload" #> $5::text[]) AS "c0", ` + + `COALESCE(jsonb_typeof(r0."payload" #> $6::text[]) = 'object' AND jsonb_exists(r0."payload" #> $6::text[], $7), false) AS "c1"`) + if rendered.SQL != wantSQL { + t.Fatalf("nested SQL =\n%s\nwant\n%s", rendered.SQL, wantSQL) + } + wantArgs := append(pgsq03ScopeArgs(), + command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: "{meta,source}"}, + command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: "{meta}"}, + command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: "source"}, + ) + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("nested args = %#v, want %#v", rendered.Args, wantArgs) + } + projections := rendered.SQL[:strings.Index(rendered.SQL, " FROM ")] + for _, forbidden := range []string{"meta", "source"} { + if strings.Contains(projections, forbidden) { + t.Fatalf("path segment %q leaked into SQL projections: %s", forbidden, projections) + } + } +} + +func TestNullSemanticsNullNamedSegmentsBindQuotedArrayElements(t *testing.T) { + r := &planRenderer{storage: mustBinding(t), parameters: map[string]int{}, rows: map[string]rowScope{}} + + pathArgument := func(t *testing.T, expression string) string { + t.Helper() + var index int + if _, err := fmt.Sscanf(expression, `r0."payload" #> $%d::text[]`, &index); err != nil || index < 1 || index > len(r.args) { + t.Fatalf("unexpected path expression %q (err %v, %d args)", expression, err, len(r.args)) + } + text, ok := r.args[index-1].Value.(string) + if !ok { + t.Fatalf("path argument %d is %#v, want string", index, r.args[index-1]) + } + return text + } + + if got, err := r.documentPathExpression("r0", []string{"gender"}); err != nil || pathArgument(t, got) != "{gender}" { + t.Fatalf("plain segment must stay unquoted: %q %v", got, err) + } + for _, keyword := range []string{"null", "NULL", "nUlL"} { + got, err := r.documentPathExpression("r0", []string{keyword}) + if err != nil { + t.Fatalf("keyword %q: %v", keyword, err) + } + if want := `{"` + keyword + `"}`; pathArgument(t, got) != want { + t.Fatalf("keyword %q bound as %q, want %q", keyword, pathArgument(t, got), want) + } + } + if got, err := r.documentPathExpression("r0", []string{"nullable"}); err != nil || pathArgument(t, got) != "{nullable}" { + t.Fatalf("near-miss segment must stay unquoted: %q %v", got, err) + } + if got, err := r.documentPathExpression("r0", []string{"null", "x"}); err != nil || pathArgument(t, got) != `{"null",x}` { + t.Fatalf("nested keyword path = %q %v", got, err) + } + + presence, err := r.propertyPresenceExpression("r0", []string{"null", "x"}) + if err != nil { + t.Fatalf("nested presence: %v", err) + } + var prefixIndex, prefixAgain, keyIndex int + if _, err := fmt.Sscanf(presence, + `COALESCE(jsonb_typeof(r0."payload" #> $%d::text[]) = 'object' AND jsonb_exists(r0."payload" #> $%d::text[], $%d), false)`, + &prefixIndex, &prefixAgain, &keyIndex); err != nil || prefixIndex != prefixAgain { + t.Fatalf("nested presence shape unparseable: %q (err %v)", presence, err) + } + if prefixIndex < 1 || prefixIndex > len(r.args) { + t.Fatalf("nested presence prefix index %d out of range", prefixIndex) + } + if text, ok := r.args[prefixIndex-1].Value.(string); !ok || text != `{"null"}` { + t.Fatalf("presence prefix = %#v, want {\"null\"}", r.args[prefixIndex-1]) + } + if keyIndex < 1 || keyIndex > len(r.args) { + t.Fatalf("nested presence key index %d out of range", keyIndex) + } + if text, ok := r.args[keyIndex-1].Value.(string); !ok || text != "x" { + t.Fatalf("presence terminal key = %#v, want plain x", r.args[keyIndex-1]) + } + + single, err := r.propertyPresenceExpression("r0", []string{"null"}) + if err != nil { + t.Fatalf("single presence: %v", err) + } + if _, err := fmt.Sscanf(single, + `COALESCE(jsonb_typeof(r0."payload") = 'object' AND jsonb_exists(r0."payload", $%d), false)`, + &keyIndex); err != nil || keyIndex < 1 || keyIndex > len(r.args) { + t.Fatalf("single presence shape unparseable: %q (err %v)", single, err) + } + if text, ok := r.args[keyIndex-1].Value.(string); !ok || text != "null" { + t.Fatalf("single presence key = %#v, want plain null (scalar keys are not array elements)", r.args[keyIndex-1]) + } + + for _, argument := range r.args { + if text, ok := argument.Value.(string); ok && text == "{null}" { + t.Fatalf("unquoted null array element reached the driver: %s", text) + } + } +} + +func TestNullSemanticsFilterPayloadPathsShareQuotedEncoding(t *testing.T) { + plan := basePlan() + plan.BindVars["null_value"] = "present" + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"null"}}, + Right: &ir.PhysicalValue{BindKey: "null_value"}, + }}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"null", "x"}}, + Right: &ir.PhysicalValue{BindKey: "null_value"}, + }}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + } + rendered := renderOK(t, plan) + if text, ok := rendered.Args[4].Value.(string); !ok || text != `{"null"}` { + t.Fatalf("filter payload path arg = %#v, want {\"null\"}", rendered.Args[4]) + } + if text, ok := rendered.Args[6].Value.(string); !ok || text != `{"null",x}` { + t.Fatalf("nested filter payload path arg = %#v, want {\"null\",x}", rendered.Args[6]) + } + for _, argument := range rendered.Args { + if text, ok := argument.Value.(string); ok && text == "{null}" { + t.Fatalf("unquoted null array element reached the driver: %s", text) + } + } +} + +func TestNullSemanticsPresenceIsTotalNeverNull(t *testing.T) { + plan := pgsq03SelectorPlan("Patient", + pgsq03ExtractProjection(t, "gender", "Patient", "gender", ir.PhysicalPreserveNull, ir.PhysicalSelectorGeneric), + pgsq03PresenceSidecar("gender_present", "gender"), + ) + sql := pgsq03RenderOK(t, plan).SQL + if !strings.Contains(sql, `COALESCE(jsonb_typeof(r0."payload") = 'object' AND jsonb_exists(r0."payload", $6), false)`) { + t.Fatalf("presence proof must be a COALESCE-total two-valued predicate:\n%s", sql) + } + unguarded := strings.Replace(sql, `COALESCE(jsonb_typeof(r0."payload") = 'object' AND jsonb_exists(r0."payload", $6), false)`, + `jsonb_typeof(r0."payload") = 'object' AND jsonb_exists(r0."payload", $6)`, 1) + if unguarded == sql { + t.Fatalf("unguarded drop-probe did not change the statement") + } + if strings.Contains(unguarded, ", false)") { + t.Fatalf("drop probe still guarded") + } +} + +func TestNullSemanticsBehaviorFormsAreDistinctNotDecorated(t *testing.T) { + forms := map[ir.PhysicalNullBehavior]string{} + for _, behavior := range []ir.PhysicalNullBehavior{ + ir.PhysicalPreserveNull, ir.PhysicalOmitNulls, ir.PhysicalEmptyOnNull, + } { + plan := pgsq03SelectorPlan("Patient", + pgsq03ExtractProjection(t, "gender", "Patient", "gender", behavior, ir.PhysicalSelectorGeneric)) + forms[behavior] = pgsq03RenderOK(t, plan).SQL + } + if forms[ir.PhysicalPreserveNull] == forms[ir.PhysicalOmitNulls] || + forms[ir.PhysicalOmitNulls] == forms[ir.PhysicalEmptyOnNull] || + forms[ir.PhysicalPreserveNull] == forms[ir.PhysicalEmptyOnNull] { + t.Fatalf("null behaviors rendered identically: %v", forms) + } + if strings.Contains(forms[ir.PhysicalPreserveNull], "NULLIF") { + t.Fatalf("preserve-null must never coerce the stored JSON null:\n%s", forms[ir.PhysicalPreserveNull]) + } + if strings.Contains(forms[ir.PhysicalOmitNulls], "COALESCE") { + t.Fatalf("omit-nulls must not fabricate an empty value:\n%s", forms[ir.PhysicalOmitNulls]) + } + if !strings.Contains(forms[ir.PhysicalEmptyOnNull], "NULLIF") || !strings.Contains(forms[ir.PhysicalEmptyOnNull], "COALESCE") { + t.Fatalf("empty-on-null must coerce both JSON null and SQL NULL:\n%s", forms[ir.PhysicalEmptyOnNull]) + } +} + +func TestNullSemanticsSamePathSharesOneParameter(t *testing.T) { + plan := pgsq03SelectorPlan("Patient", + pgsq03ExtractProjection(t, "a", "Patient", "meta.source", ir.PhysicalPreserveNull, ir.PhysicalSelectorGeneric), + pgsq03ExtractProjection(t, "b", "Patient", "meta.source", ir.PhysicalOmitNulls, ir.PhysicalSelectorGeneric), + pgsq03PresenceSidecar("c", "meta", "source"), + ) + rendered := pgsq03RenderOK(t, plan) + if len(rendered.Args) != 7 { + t.Fatalf("deduplicated args = %d (%#v), want 7", len(rendered.Args), rendered.Args) + } + if !strings.Contains(rendered.SQL, `$5`) || !strings.Contains(rendered.SQL, `$6`) || !strings.Contains(rendered.SQL, `$7`) { + t.Fatalf("expected exactly $1..$7 in:\n%s", rendered.SQL) + } + if strings.Contains(rendered.SQL, "$8") { + t.Fatalf("a path parameter was duplicated:\n%s", rendered.SQL) + } +} + +func TestNullSemanticsHostileSegmentsFailClosedBeforeSQL(t *testing.T) { + for _, segment := range []string{"", "bad-name", "a.b", `"x"`, "drop;--", "1x", "ſexp"} { + projection := ir.PhysicalProjection{ + Name: "evil", + Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Extract: &ir.PhysicalExtract{ + Source: pgsq03DocumentSource(), + ResourceType: "Patient", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: segment}}}, + }, + }, + } + plan := pgsq03SelectorPlan("Patient", projection) + rendered, err := RenderSelectorExtraction(plan, mustBinding(t)) + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("segment %q error = %v, want ErrUnsupportedPostgresPlan", segment, err) + } + if rendered.SQL != "" || rendered.Args != nil || rendered.OutputColumns != nil { + t.Fatalf("rejected segment %q produced output: %#v", segment, rendered) + } + wantIn := strconv.Quote(segment) + if segment == "" { + wantIn = "has no property" + } + if !strings.Contains(err.Error(), wantIn) { + t.Fatalf("refusal for %q must contain %q: %v", segment, wantIn, err) + } + } +} + +func TestNullSemanticsDeterministicAndNonAliasing(t *testing.T) { + plan := pgsq03SelectorPlan("Patient", + pgsq03ExtractProjection(t, "gender", "Patient", "gender", ir.PhysicalEmptyOnNull, ir.PhysicalSelectorGeneric), + pgsq03PresenceSidecar("gender_present", "gender"), + ) + first := pgsq03RenderOK(t, plan) + second := pgsq03RenderOK(t, plan) + if first.SQL != second.SQL || !reflect.DeepEqual(first.Args, second.Args) { + t.Fatalf("repeated render drifted") + } + capturedSQL, capturedArgs := first.SQL, append([]command.PostgreSQLParameter(nil), first.Args...) + + plan.BindVars["project"] = "mutated" + plan.BindVars["auth_resource_paths"].([]string)[0] = "MUTATED" + third, err := RenderSelectorExtraction(plan, mustBinding(t)) + if err != nil { + t.Fatalf("render after mutation: %v", err) + } + if first.SQL != capturedSQL || !reflect.DeepEqual(first.Args, capturedArgs) { + t.Fatalf("rendered result aliased plan data") + } + if reflect.DeepEqual(third.Args, capturedArgs) { + t.Fatalf("mutated plan re-bound the frozen arguments") + } +} + +func TestNullSemanticsUntrustedBindingAndScopeFailuresProduceNothing(t *testing.T) { + plan := pgsq03SelectorPlan("Patient", + pgsq03ExtractProjection(t, "gender", "Patient", "gender", ir.PhysicalPreserveNull, ir.PhysicalSelectorGeneric), + pgsq03PresenceSidecar("gender_present", "gender"), + ) + if _, err := RenderSelectorExtraction(plan, ir.PostgresStorageBinding{}); !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + t.Fatalf("zero binding error = %v, want ErrUnregisteredPostgresIdentifier", err) + } + for _, mutate := range []func(map[string]any){ + func(b map[string]any) { delete(b, "project") }, + func(b map[string]any) { delete(b, "scope_allowed") }, + func(b map[string]any) { b["scope_allowed"] = "yes" }, + func(b map[string]any) { b["auth_resource_paths_unrestricted"] = true }, + } { + mutated := pgsq03SelectorPlan("Patient", + pgsq03ExtractProjection(t, "gender", "Patient", "gender", ir.PhysicalPreserveNull, ir.PhysicalSelectorGeneric)) + mutate(mutated.BindVars) + rendered, err := RenderSelectorExtraction(mutated, mustBinding(t)) + if !errors.Is(err, ErrInvalidPostgresPlanScope) { + t.Fatalf("scope defect error = %v, want ErrInvalidPostgresPlanScope", err) + } + if rendered.SQL != "" || rendered.Args != nil || rendered.OutputColumns != nil { + t.Fatalf("rejected scope produced output: %#v", rendered) + } + } +} diff --git a/internal/dataframe/compiler/render/postgres/obj11_production_test.go b/internal/dataframe/compiler/render/postgres/obj11_production_test.go new file mode 100644 index 000000000..0c859b465 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/obj11_production_test.go @@ -0,0 +1,280 @@ +package postgres + +import ( + "strings" + "testing" + + pgstore "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" +) + +const ( + obj11ProdProject = "proj-a" + obj11ProdGeneration = "gen-1" + obj11ProdEdgeLabel = "subject_Patient" +) + +func obj11WitnessBundle(name string, limit int, whereNeedle string) recipe.Bundle { + var where *recipe.Filter + if whereNeedle != "" { + needle := whereNeedle + where = &recipe.Filter{Select: "occ_patient.gender", Operator: recipe.FilterEquals, + Values: []recipe.FilterValue{{Kind: recipe.FilterString, String: &needle}}} + } + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: name, + TranslationVersion: "test", + Outputs: []recipe.Output{{ + Name: "observations", RootResourceType: "Observation", RowGrain: "resource", + Fields: []recipe.Field{{Name: "status", Expr: recipe.Expression{Select: "root.status"}}}, + Traversals: []recipe.Traversal{{ + Name: obj11ProdEdgeLabel, Alias: "occ_patient", ToResourceType: "Patient", MatchMode: recipe.MatchOptional, + Slices: []recipe.RepresentativeSlice{{ + Name: "representatives", Limit: limit, Where: where, + Fields: []recipe.Field{ + {Name: "gender", Expr: recipe.Expression{Select: "occ_patient.gender"}}, + {Name: "active", Expr: recipe.Expression{Select: "occ_patient.active"}}, + }, + }}, + }}, + }}, + } +} + +func obj11RegisteredSurfacePolicy() ir.PhysicalOptimizationPolicy { + return ir.DefaultPhysicalOptimizationPolicy().WithRule(ir.PhysicalOptimizationRuleEndpointTraversal, false) +} + +func obj11CompileProductionWithPolicy(t *testing.T, bundle recipe.Bundle, policy ir.PhysicalOptimizationPolicy) ir.PhysicalPlan { + t.Helper() + bindings := recipe.RuntimeBindings{ + Project: obj11ProdProject, DatasetGeneration: obj11ProdGeneration, + AuthResourcePaths: []string{"FHIR/Observation", "FHIR/Patient"}, + } + plan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, policy) + if err != nil { + t.Fatalf("lower.CompileResolvedRecipePlan: %v", err) + } + if len(compiled.Outputs) != 1 { + t.Fatalf("compiled outputs = %d, want 1", len(compiled.Outputs)) + } + output := compiled.Outputs[0] + if output.OptimizedPlan != nil { + return *output.OptimizedPlan + } + return output.Plan +} + +func obj11CompileProduction(t *testing.T, bundle recipe.Bundle) ir.PhysicalPlan { + t.Helper() + return obj11CompileProductionWithPolicy(t, bundle, obj11RegisteredSurfacePolicy()) +} + +func obj11ProductionSlice(t *testing.T, plan ir.PhysicalPlan) (*ir.PhysicalSlice, *ir.PhysicalSet) { + t.Helper() + var set *ir.PhysicalSet + for index := range plan.Operations { + operation := plan.Operations[index] + if operation.Kind == ir.PhysicalSetOp && operation.Set != nil && + operation.Set.Projection != nil && len(operation.Set.Projection.Fields) > 0 { + set = operation.Set + } + } + if set == nil { + t.Fatalf("production plan carries no compact-projection child set with registered slots:\n%s", obj11PlanShape(plan)) + } + for index := range plan.Operations { + operation := plan.Operations[index] + if operation.Kind != ir.PhysicalReturnOp || operation.Return == nil { + continue + } + for projectionIndex := range operation.Return.Projections { + expression := operation.Return.Projections[projectionIndex].Expression + if expression != nil && expression.Kind == ir.PhysicalSliceExpression && expression.Slice != nil { + if expression.Slice.Source.Variable != set.Variable { + t.Fatalf("production SLICE reads %q, want the child set %q", expression.Slice.Source.Variable, set.Variable) + } + return expression.Slice, set + } + } + } + t.Fatalf("production plan carries no RETURN-projection SLICE:\n%s", obj11PlanShape(plan)) + return nil, nil +} + +func obj11PlanShape(plan ir.PhysicalPlan) string { + var b strings.Builder + for index, operation := range plan.Operations { + b.WriteString(string(operation.Kind)) + if operation.Set != nil { + b.WriteString("(" + operation.Set.Variable + ")") + } + if index+1 < len(plan.Operations) { + b.WriteString(" -> ") + } + } + return b.String() +} + +func TestObject11ProductionPlanProducerWitnessRenders(t *testing.T) { + plan := obj11CompileProduction(t, obj11WitnessBundle("obj11_producer_witness", 10, "")) + slice, set := obj11ProductionSlice(t, plan) + + defaultPlan := obj11CompileProductionWithPolicy(t, obj11WitnessBundle("obj11_default_policy_premise", 10, ""), ir.DefaultPhysicalOptimizationPolicy()) + defaultSlice, defaultSet := obj11ProductionSlice(t, defaultPlan) + for _, projection := range defaultSlice.Projections { + if projection.Expression.Extract == nil || projection.Expression.Extract.Prepared == nil { + t.Fatalf("default-policy planner did NOT annotate slice projection %q (Option A premise broken — investigate the lowering): %+v", projection.Name, projection.Expression) + } + if projection.Expression.Extract.Prepared.SetVariable != defaultSet.Variable { + t.Fatalf("default-policy annotation names %q, want set %q", projection.Expression.Extract.Prepared.SetVariable, defaultSet.Variable) + } + } + + registeredSlots := map[string]bool{} + for _, field := range set.Projection.Fields { + registeredSlots[field.Name] = true + } + for _, projection := range slice.Projections { + extract := projection.Expression.Extract + if extract == nil || extract.Prepared == nil { + t.Fatalf("production planner did NOT annotate slice projection %q (Option A premise broken — investigate the lowering):\n%+v", projection.Name, projection.Expression) + } + if extract.Prepared.SetVariable != set.Variable || !registeredSlots[extract.Prepared.Field] { + t.Fatalf("production annotation names %s.%v, want a slot of set %q (registered: %v)", + extract.Prepared.SetVariable, extract.Prepared.Field, set.Variable, set.Projection.Fields) + } + } + + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("production compact-projection slice plan must render after Option A: %v", err) + } + for _, want := range []string{"(SELECT DISTINCT ", "COALESCE(jsonb_agg(", `item_key ASC LIMIT $`} { + if !strings.Contains(rendered.SQL, want) { + t.Fatalf("rendered production statement lost %q:\n%s", want, rendered.SQL) + } + } + for _, forbidden := range []string{"prepared", "__loom_projection", "__loom_reduced"} { + if strings.Contains(rendered.SQL, forbidden) { + t.Fatalf("%q leaked into rendered statement text:\n%s", forbidden, rendered.SQL) + } + } +} + +func TestObject11ProductionPlanPredicateProducerWitnessRenders(t *testing.T) { + plan := obj11CompileProduction(t, obj11WitnessBundle("obj11_predicate_witness", 10, "male")) + slice, set := obj11ProductionSlice(t, plan) + if slice.Predicate == nil || slice.Predicate.Comparison == nil || + slice.Predicate.Comparison.LeftExpression == nil || + slice.Predicate.Comparison.LeftExpression.Extract == nil { + t.Fatalf("production slice Where did not lower to the registered predicate shape: %+v", slice.Predicate) + } + prepared := slice.Predicate.Comparison.LeftExpression.Extract.Prepared + if prepared == nil || prepared.SetVariable != set.Variable { + t.Fatalf("production planner did NOT annotate the predicate extraction (Option A premise broken — investigate the lowering): %+v", slice.Predicate.Comparison.LeftExpression.Extract) + } + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); err != nil { + t.Fatalf("production predicate-arm plan must render after Option A: %v", err) + } +} + +func TestObject11LiveProductionPlanSliceLeg(t *testing.T) { + svc := newOwnedService(t, 0, "obj11_it") + insert := func(resourceType, logicalID, authPath string, payload map[string]any) { + svc.insertResource(t, svc.schema, obj11ProdProject, obj11ProdGeneration, resourceType, logicalID, authPath, payload) + } + insert("Observation", "o-1", "FHIR/Observation", map[string]any{"status": "final"}) + insert("Observation", "o-2", "FHIR/Observation", map[string]any{"status": "final"}) + insert("Patient", "p-a", "FHIR/Patient", map[string]any{"gender": "female", "active": true}) + insert("Patient", "p-b", "FHIR/Patient", map[string]any{"gender": "female", "active": true}) + insert("Patient", "p-c", "FHIR/Patient", map[string]any{"gender": "male", "active": true}) + insert("Patient", "p-denied", "FHIR/Denied", map[string]any{"gender": "denied", "active": true}) + edge := func(from, to, sourcePath string, ordinal int64, authPath string) { + svc.insertEdge(t, pgstore.EdgeOccurrenceRow{ + Project: obj11ProdProject, DatasetGeneration: obj11ProdGeneration, AuthResourcePath: authPath, + Label: obj11ProdEdgeLabel, SourceResourceType: "Observation", SourceLogicalID: from, + TargetResourceType: "Patient", TargetLogicalID: to, SourcePath: sourcePath, SourceOrdinal: ordinal, + }) + } + edge("o-1", "p-a", "subject[0]", 0, "FHIR/Observation") + edge("o-1", "p-b", "subject[0]", 0, "FHIR/Observation") + edge("o-1", "p-b", "subject[1]", 1, "FHIR/Observation") + edge("o-1", "p-c", "subject[0]", 0, "FHIR/Observation") + edge("o-1", "p-denied", "subject[0]", 0, "FHIR/Denied") + + plan := obj11CompileProduction(t, obj11WitnessBundle("obj11_live_witness", 10, "")) + slice, _ := obj11ProductionSlice(t, plan) + _ = slice + returnOp := plan.Operations[len(plan.Operations)-1] + sliceIndex := -1 + for index, projection := range returnOp.Return.Projections { + if projection.Expression != nil && projection.Expression.Kind == ir.PhysicalSliceExpression { + sliceIndex = index + } + } + if sliceIndex < 0 { + t.Fatalf("slice projection not found in RETURN") + } + + rows := map[string][]string{} + for _, row := range svc.runRendered(t, obj11RenderOK(t, svc, plan)) { + rows[row[0]] = row + } + if len(rows) != 2 { + t.Fatalf("both root rows must survive (parent grain), got %v", rows) + } + const wantItems = "[map[active:true gender:female] map[active:true gender:female] map[active:true gender:male]]" + if got := rows["o-1"][sliceIndex]; got != wantItems { + t.Fatalf("production slice items = %q, want %q (dedup + _key identity order + auth scope)", got, wantItems) + } + if got := rows["o-2"][sliceIndex]; got != "[]" { + t.Fatalf("childless root must render the EMPTY_ON_NULL array, got %q", got) + } + + direct := svc.witness(t, `SELECT DISTINCT x.logical_id + FROM "`+svc.schema+`"."fhir_edge_occurrence" e + INNER JOIN "`+svc.schema+`"."fhir_resource" x + ON x.project = $1 AND x.dataset_generation = $2 + AND x.auth_resource_path IN ('FHIR/Observation','FHIR/Patient') + AND x.resource_type = 'Patient' AND x.logical_id = e.target_logical_id + WHERE e.project = $1 AND e.dataset_generation = $2 + AND e.auth_resource_path IN ('FHIR/Observation','FHIR/Patient') + AND e.label = $3 AND e.source_resource_type = 'Observation' + AND e.source_logical_id = 'o-1' AND e.target_resource_type = 'Patient' + ORDER BY x.logical_id ASC`, obj11ProdProject, obj11ProdGeneration, obj11ProdEdgeLabel) + if len(direct) != 3 || direct[0][0] != "p-a" || direct[1][0] != "p-b" || direct[2][0] != "p-c" { + t.Fatalf("direct witness order drifted (dedup?): %v", direct) + } + limitedPlan := obj11CompileProduction(t, obj11WitnessBundle("obj11_live_limited", 2, "")) + limitedRows := map[string][]string{} + for _, row := range svc.runRendered(t, obj11RenderOK(t, svc, limitedPlan)) { + limitedRows[row[0]] = row + } + const wantPrefix = "[map[active:true gender:female] map[active:true gender:female]]" + if got := limitedRows["o-1"][sliceIndex]; got != wantPrefix { + t.Fatalf("LIMIT 2 must render the witness prefix, got %q", got) + } + + predicatePlan := obj11CompileProduction(t, obj11WitnessBundle("obj11_live_predicate", 10, "male")) + predicateRows := map[string][]string{} + for _, row := range svc.runRendered(t, obj11RenderOK(t, svc, predicatePlan)) { + predicateRows[row[0]] = row + } + const wantMale = "[map[active:true gender:male]]" + if got := predicateRows["o-1"][sliceIndex]; got != wantMale { + t.Fatalf("production predicate arm = %q, want %q", got, wantMale) + } +} diff --git a/internal/dataframe/compiler/render/postgres/objects.go b/internal/dataframe/compiler/render/postgres/objects.go new file mode 100644 index 000000000..8a799a0db --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/objects.go @@ -0,0 +1,516 @@ +package postgres + +import ( + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const maxCompositeDepth = 4 + +func sliceConsumesSetVariable(plan ir.PhysicalPlan, setVariable string) bool { + for _, operation := range plan.Operations { + if operation.Kind != ir.PhysicalReturnOp || operation.Return == nil { + continue + } + for index := range operation.Return.Projections { + if operation.Return.Projections[index].Expression != nil && + expressionConsumesSetVariable(*operation.Return.Projections[index].Expression, setVariable) { + return true + } + } + } + return false +} + +func expressionConsumesSetVariable(expression ir.PhysicalExpression, setVariable string) bool { + if slice := expression.Slice; slice != nil { + if slice.Source.Variable == setVariable && slice.Source.BindKey == "" { + return true + } + for index := range slice.Projections { + if expressionConsumesSetVariable(slice.Projections[index].Expression, setVariable) { + return true + } + } + } + if object := expression.Object; object != nil { + for index := range object.Fields { + if expressionConsumesSetVariable(object.Fields[index].Expression, setVariable) { + return true + } + } + } + return false +} + +const sliceItemVariable = "__loom_slice_item" + +func rebindSliceOperand(expression ir.PhysicalExpression, sourceVariable string) ir.PhysicalExpression { + rewritten := expression + if extract := rewritten.Extract; extract != nil && extract.Source.Variable == sourceVariable { + extractCopy := *extract + extractCopy.Source = ir.PhysicalValue{Variable: sliceItemVariable, Path: extractCopy.Source.Path} + rewritten.Extract = &extractCopy + } + if value := rewritten.Value; value != nil && value.Variable == sourceVariable { + valueCopy := *value + valueCopy.Variable = sliceItemVariable + rewritten.Value = &valueCopy + } + if object := rewritten.Object; object != nil { + fields := make([]ir.PhysicalExpressionProjection, len(object.Fields)) + for index, field := range object.Fields { + fields[index] = field + fields[index].Expression = rebindSliceOperand(field.Expression, sourceVariable) + } + objectCopy := *object + objectCopy.Fields = fields + rewritten.Object = &objectCopy + } + return rewritten +} + +func (r *planRenderer) resolveSlicePreparedExtract(setVariable string, original, rewritten ir.PhysicalExpression, context string) error { + extract := rewritten.Extract + if extract == nil { + return nil + } + if extract.Prepared != nil { + reference := *extract.Prepared + originalSourceVariable := "" + if original.Extract != nil { + originalSourceVariable = original.Extract.Source.Variable + } + verbatim := fmt.Errorf("%w: composite EXTRACT operand: prepared-set selector references are not implemented for PostgreSQL", + ErrUnsupportedPostgresPlan) + if setVariable == "" || reference.SetVariable != setVariable || originalSourceVariable != setVariable { + return verbatim + } + var slot *ir.PhysicalSetProjectionField + for index := range r.setProjections[setVariable] { + if r.setProjections[setVariable][index].Name == reference.Field { + slot = &r.setProjections[setVariable][index] + break + } + } + if slot == nil { + return verbatim + } + switch slot.Demand { + case ir.PhysicalSelectorAllValues, ir.PhysicalSelectorFirstValue: + default: + return fmt.Errorf("%w: slice %s: prepared slot %q carries unsupported value demand %q", + ErrUnsupportedPostgresPlan, context, reference.Field, string(slot.Demand)) + } + switch slot.ExecutionMode { + case "", ir.PhysicalSelectorGeneric, ir.PhysicalSelectorDirectScalar: + case ir.PhysicalSelectorConditionalArray: + return fmt.Errorf("%w: slice %s: prepared slot %q is CONDITIONAL_ARRAY, whose repeated-step rendering is not registered for PostgreSQL", + ErrUnsupportedPostgresPlan, context, reference.Field) + default: + return fmt.Errorf("%w: slice %s: prepared slot %q has unknown selector execution mode %q", + ErrUnsupportedPostgresPlan, context, reference.Field, string(slot.ExecutionMode)) + } + if slot.ResourceType != extract.ResourceType { + return fmt.Errorf("%w: slice %s: prepared slot %q projects resource type %q but the operand validates against %q", + ErrUnsupportedPostgresPlan, context, reference.Field, slot.ResourceType, extract.ResourceType) + } + if !reflect.DeepEqual(extract.Selector, slot.Selector) { + return fmt.Errorf("%w: slice %s: prepared slot %q names a different selector than the operand carries; the annotation must re-read the registered slot exactly", + ErrUnsupportedPostgresPlan, context, reference.Field) + } + if slot.Selector.Filter != nil || len(slot.Selector.Steps) == 0 { + return fmt.Errorf("%w: slice %s: prepared slot %q is not a direct property path; prepared resolution is registered for direct-path slots only", + ErrUnsupportedPostgresPlan, context, reference.Field) + } + for index, step := range slot.Selector.Steps { + if step.Field == "" || step.Iterate || step.Index != nil { + return fmt.Errorf("%w: slice %s: prepared slot %q step %d is not a direct property step; prepared resolution is registered for direct-path slots only", + ErrUnsupportedPostgresPlan, context, reference.Field, index) + } + } + extract.Prepared = nil + } + return nil +} + +func (r *planRenderer) objectFieldJSON(plan ir.PhysicalPlan, rows map[string]rowScope, expression ir.PhysicalExpression, depth int) (string, error) { + switch expression.Kind { + case ir.PhysicalValueExpression: + return r.compositeValueJSON(rows, expression) + case ir.PhysicalExtractExpression: + return r.compositeExtractJSON(rows, expression) + case ir.PhysicalObjectExpression: + return r.renderObjectValue(plan, rows, expression, depth+1) + case ir.PhysicalSliceExpression: + return r.renderSliceValue(plan, rows, expression, depth+1) + default: + return "", fmt.Errorf("%w: composite operand kind %q has no registered PostgreSQL rendering (literal, aggregate, pivot, owner-records, object-lookup, keyed-map, object-keys, key-set, subplan, call, and related-field forms are separate capabilities)", + ErrUnsupportedPostgresPlan, string(expression.Kind)) + } +} + +func (r *planRenderer) compositeValueJSON(rows map[string]rowScope, expression ir.PhysicalExpression) (string, error) { + value := expression.Value + if value == nil { + return "", fmt.Errorf("%w: composite VALUE operand has no value payload", ErrUnsupportedPostgresPlan) + } + if value.BindKey != "" { + return "", fmt.Errorf("%w: composite VALUE operand %v must read a row value, not a bind value", ErrUnsupportedPostgresPlan, value.Path) + } + scope, ok := rows[value.Variable] + if !ok || scope.derivedValue != "" || scope.alias == "" { + return "", fmt.Errorf("%w: composite VALUE operand reads variable %q, which does not bind a readable stored row", + ErrUnsupportedPostgresPlan, value.Variable) + } + if len(value.Path) == 0 { + return "", fmt.Errorf("%w: composite VALUE operand must name a document path or a typed column; the whole-document reference is spelled [\"payload\"] (lower/recipe_expression.go:128-146)", + ErrUnsupportedPostgresPlan) + } + column := func(name string) string { + return "to_jsonb(" + scope.alias + "." + r.storage.Resource.MustColumn(name).SQLText() + ")" + } + if len(value.Path) == 1 { + segment := value.Path[0] + switch { + case segment == "payload": + if expression.Cardinality != ir.PhysicalObjectCardinality { + return "", fmt.Errorf(`%w: composite VALUE operand ["payload"] must carry OBJECT cardinality (lower/recipe_expression.go:133-137)`, ErrUnsupportedPostgresPlan) + } + return r.documentColumn(scope.alias), nil + case segment == "resourceType": + return column("resource_type"), nil + case segment == "_key" || segment == "id": + return column(canonicalKeyColumn(segment)), nil + case resourceFilterColumns[segment]: + return column(segment), nil + } + } + return r.documentPathExpression(scope.alias, value.Path) +} + +func (r *planRenderer) compositeExtractJSON(rows map[string]rowScope, expression ir.PhysicalExpression) (string, error) { + extract := expression.Extract + if extract == nil { + return "", fmt.Errorf("%w: composite EXTRACT operand has no payload", ErrUnsupportedPostgresPlan) + } + if expression.Cardinality != ir.PhysicalScalarCardinality && expression.Cardinality != ir.PhysicalArrayCardinality { + return "", fmt.Errorf("%w: composite EXTRACT operand is registered for SCALAR and ARRAY cardinality only", ErrUnsupportedPostgresPlan) + } + switch expression.NullBehavior { + case ir.PhysicalPreserveNull, ir.PhysicalOmitNulls, ir.PhysicalEmptyOnNull: + default: + return "", fmt.Errorf("%w: composite EXTRACT operand: null behavior %q has no registered PostgreSQL extraction form", + ErrUnsupportedPostgresPlan, string(expression.NullBehavior)) + } + switch extract.ExecutionMode { + case "", ir.PhysicalSelectorGeneric, ir.PhysicalSelectorDirectScalar: + case ir.PhysicalSelectorConditionalArray: + return "", fmt.Errorf("%w: composite EXTRACT operand: CONDITIONAL_ARRAY selector mode requires repeated-step rendering, which is not registered for PostgreSQL", + ErrUnsupportedPostgresPlan) + default: + return "", fmt.Errorf("%w: composite EXTRACT operand: unknown selector execution mode %q", ErrUnsupportedPostgresPlan, string(extract.ExecutionMode)) + } + if extract.Prepared != nil { + return "", fmt.Errorf("%w: composite EXTRACT operand: prepared-set selector references are not implemented for PostgreSQL", + ErrUnsupportedPostgresPlan) + } + if extract.Distinct { + return "", fmt.Errorf("%w: composite EXTRACT operand: DISTINCT projection is meaningful only for array-valued selectors registered with their own reduction", + ErrUnsupportedPostgresPlan) + } + if len(extract.Fallbacks) != 0 { + return "", fmt.Errorf("%w: composite EXTRACT operand: ordered fallback selectors are a separate capability", ErrUnsupportedPostgresPlan) + } + if extract.UnitNormalization != nil { + return "", fmt.Errorf("%w: composite EXTRACT operand: unit-normalized extraction is not implemented for PostgreSQL", ErrUnsupportedPostgresPlan) + } + if extract.Selector.Filter != nil { + return "", fmt.Errorf("%w: composite EXTRACT operand: selector CONTAINS filters are not implemented for PostgreSQL", ErrUnsupportedPostgresPlan) + } + if len(extract.Selector.Steps) == 0 { + return "", fmt.Errorf("%w: composite EXTRACT operand: the selector is empty", ErrUnsupportedPostgresPlan) + } + path := make([]string, 0, len(extract.Selector.Steps)) + for index, step := range extract.Selector.Steps { + switch { + case step.Field == "": + return "", fmt.Errorf("%w: composite EXTRACT operand: selector step %d has no property", ErrUnsupportedPostgresPlan, index) + case step.Iterate: + return "", fmt.Errorf("%w: composite EXTRACT operand: repeated ([]) selector step %d requires array rendering, which is not registered", + ErrUnsupportedPostgresPlan, index) + case step.Index != nil: + return "", fmt.Errorf("%w: composite EXTRACT operand: indexed selector step %d requires array rendering, which is not registered", + ErrUnsupportedPostgresPlan, index) + } + if !payloadPathSegmentPattern.MatchString(step.Field) { + return "", fmt.Errorf("%w: composite EXTRACT operand: selector step %q is not renderer-safe", ErrUnsupportedPostgresPlan, step.Field) + } + path = append(path, step.Field) + } + if extract.Source.BindKey != "" { + return "", fmt.Errorf("%w: composite EXTRACT operand: the selector source must be a row document, not a bind value", ErrUnsupportedPostgresPlan) + } + if len(extract.Source.Path) != 0 && (len(extract.Source.Path) != 1 || extract.Source.Path[0] != "payload") { + return "", fmt.Errorf(`%w: composite EXTRACT operand: the selector source must be the row document, spelled ["payload"] or as the bare row (empty path, the slice lowerer's leftSource at lower/root_projection.go:402); got %v`, + ErrUnsupportedPostgresPlan, extract.Source.Path) + } + scope, ok := rows[extract.Source.Variable] + if !ok || scope.derivedValue != "" || scope.alias == "" { + return "", fmt.Errorf("%w: composite EXTRACT operand reads variable %q, which does not bind a readable stored row", + ErrUnsupportedPostgresPlan, extract.Source.Variable) + } + if strings.TrimSpace(extract.ResourceType) == "" || extract.ResourceType != scope.resourceType { + return "", fmt.Errorf("%w: composite EXTRACT operand validates selector against resource type %q, but the row is scanned as %q", + ErrUnsupportedPostgresPlan, extract.ResourceType, scope.resourceType) + } + return r.scalarExtractValueExpression(scope.alias, path, expression.NullBehavior) +} + +func (r *planRenderer) renderObjectValue(plan ir.PhysicalPlan, rows map[string]rowScope, expression ir.PhysicalExpression, depth int) (string, error) { + object := expression.Object + if object == nil { + return "", fmt.Errorf("%w: OBJECT expression is missing payload", ErrUnsupportedPostgresPlan) + } + if r.membership != nil { + return "", fmt.Errorf("%w: OBJECT projection is registered on the plain spine only; membership-root routes keep the population.go contract and refuse composite projections", ErrUnsupportedPostgresPlan) + } + if depth > maxCompositeDepth { + return "", fmt.Errorf("%w: OBJECT nesting exceeds the registered maximum depth %d (defensive closed-surface bound; production lowering derives at most depth 1 — SLICE projections are EXTRACT-only and the document envelope's fields are VALUE-only — and AQL's renderExpression recursion is unbounded, cited per ruling)", + ErrUnsupportedPostgresPlan, maxCompositeDepth) + } + switch expression.Cardinality { + case ir.PhysicalObjectCardinality: + default: + return "", fmt.Errorf("%w: OBJECT expression must carry OBJECT cardinality, got %q", ErrUnsupportedPostgresPlan, string(expression.Cardinality)) + } + switch expression.NullBehavior { + case ir.PhysicalPreserveNull, ir.PhysicalOmitNulls, ir.PhysicalEmptyOnNull: + default: + return "", fmt.Errorf("%w: OBJECT expression null behavior %q is not a registered form", ErrUnsupportedPostgresPlan, string(expression.NullBehavior)) + } + if len(object.Fields) == 0 { + return "", fmt.Errorf("%w: OBJECT expression requires at least one field (ir/physical_validation_shapes.go:39-54)", ErrUnsupportedPostgresPlan) + } + seen := make(map[string]bool, len(object.Fields)) + parts := make([]string, 0, len(object.Fields)) + anyOmit := false + for index, field := range object.Fields { + if strings.TrimSpace(field.Name) == "" || seen[field.Name] { + return "", fmt.Errorf("%w: OBJECT field %d name %q is empty or duplicated (ir/physical_validation_shapes.go:45-47)", + ErrUnsupportedPostgresPlan, index, field.Name) + } + seen[field.Name] = true + value, err := r.objectFieldJSON(plan, rows, field.Expression, depth) + if err != nil { + return "", fmt.Errorf("object field %q: %w", field.Name, err) + } + nameIndex := r.textParameter(field.Name, "text:"+field.Name) + pair := "jsonb_build_object($" + strconv.Itoa(nameIndex) + "::text, " + value + ")" + if field.Expression.NullBehavior == ir.PhysicalOmitNulls { + anyOmit = true + pair = "CASE WHEN (" + value + ") IS NULL OR (" + value + ") = 'null'::jsonb THEN '{}'::jsonb ELSE " + pair + " END" + } + parts = append(parts, pair) + } + if !anyOmit { + return strings.Join(parts, " || "), nil + } + return "(" + strings.Join(parts, " || ") + ")", nil +} + +func (r *planRenderer) renderSliceValue(plan ir.PhysicalPlan, rows map[string]rowScope, expression ir.PhysicalExpression, depth int) (string, error) { + slice := expression.Slice + if slice == nil { + return "", fmt.Errorf("%w: SLICE expression is missing payload", ErrUnsupportedPostgresPlan) + } + if r.membership != nil { + return "", fmt.Errorf("%w: SLICE projection is registered on the plain spine only; membership-root routes keep the population.go contract and refuse composite projections", ErrUnsupportedPostgresPlan) + } + if depth > maxCompositeDepth { + return "", fmt.Errorf("%w: SLICE nesting exceeds the registered maximum depth %d (defensive closed-surface bound; production SLICE projections are EXTRACT-only, lower/root_projection.go:459-463, and AQL recursion is unbounded, cited per ruling)", + ErrUnsupportedPostgresPlan, maxCompositeDepth) + } + if expression.Cardinality != ir.PhysicalArrayCardinality { + return "", fmt.Errorf("%w: SLICE expression must carry ARRAY cardinality, got %q", ErrUnsupportedPostgresPlan, string(expression.Cardinality)) + } + if expression.NullBehavior != ir.PhysicalEmptyOnNull { + return "", fmt.Errorf("%w: SLICE expression null behavior %q is not the registered EMPTY_ON_NULL array reading (lower/root_projection.go:465)", + ErrUnsupportedPostgresPlan, string(expression.NullBehavior)) + } + if len(slice.Projections) == 0 { + return "", fmt.Errorf("%w: SLICE requires at least one projection (ir/physical_validation_shapes.go:39-54)", ErrUnsupportedPostgresPlan) + } + + source := slice.Source + if source.BindKey != "" { + return "", fmt.Errorf("%w: SLICE source must be a row variable, not a bind value", ErrUnsupportedPostgresPlan) + } + if len(source.Path) != 0 { + return "", fmt.Errorf("%w: SLICE source must be the row itself, the exact form lower/root_projection.go:402-408 passes (no document path); got %v", + ErrUnsupportedPostgresPlan, source.Path) + } + if body, ok := r.setBodies[source.Variable]; ok { + return r.renderSliceOverChildSet(plan, slice, body, source.Variable, depth) + } + scope, ok := rows[source.Variable] + if !ok || scope.derivedValue != "" || scope.alias == "" { + return "", fmt.Errorf("%w: SLICE source variable %q does not bind a readable stored row", ErrUnsupportedPostgresPlan, source.Variable) + } + if source.Variable != r.rootVariable { + return "", fmt.Errorf("%w: SLICE source variable %q is not a registered item source: the registered sources are the ROOT_SCAN row (singleton) and a child set whose reduction contract was validated by sets.go; traversal targets, derived values, prepared variables, and melt outputs have no registered slice contract", + ErrUnsupportedPostgresPlan, source.Variable) + } + return r.assembleSliceSublink(plan, slice, map[string]rowScope{sliceItemVariable: scope}, scope, "", "", depth) +} + +func (r *planRenderer) renderSliceOverChildSet(plan ir.PhysicalPlan, slice *ir.PhysicalSlice, body correlatedSetBody, setVariable string, depth int) (string, error) { + r.sliceSeq++ + itemAlias := "slx" + strconv.Itoa(r.sliceSeq) + logicalIDColumn := r.storage.Resource.MustColumn("logical_id").SQLText() + resourceTypeColumn := r.storage.Resource.MustColumn("resource_type").SQLText() + payloadColumn := r.storage.Resource.MustColumn("payload").SQLText() + items := "(SELECT DISTINCT " + body.targetAlias + "." + logicalIDColumn + " AS " + logicalIDColumn + + ", " + body.targetAlias + "." + resourceTypeColumn + " AS " + resourceTypeColumn + + ", " + body.targetAlias + "." + payloadColumn + " AS " + payloadColumn + + " " + body.from + " WHERE " + body.where + ")" + itemScope := rowScope{alias: itemAlias, resourceType: body.targetType} + return r.assembleSliceSublink(plan, slice, + map[string]rowScope{sliceItemVariable: itemScope}, itemScope, + " FROM "+items+" AS "+itemAlias, setVariable, depth) +} + +func (r *planRenderer) assembleSliceSublink(plan ir.PhysicalPlan, slice *ir.PhysicalSlice, itemRows map[string]rowScope, itemScope rowScope, fromClause string, setVariable string, depth int) (string, error) { + itemObject, err := r.sliceItemObject(plan, slice, itemRows, setVariable, depth) + if err != nil { + return "", err + } + sortExpr, err := r.renderSliceSort(slice, itemRows, itemScope, setVariable) + if err != nil { + return "", err + } + innerSelect := "SELECT " + itemObject + " AS \"item\", " + + sortExpr + " AS \"item_sort\", " + + itemScope.alias + "." + r.storage.Resource.MustColumn("logical_id").SQLText() + " AS \"item_key\"" + fromClause + if slice.Predicate != nil { + predicate, err := r.renderSlicePredicate(plan, slice, itemRows, setVariable) + if err != nil { + return "", err + } + innerSelect += " WHERE " + predicate + } + limitValue, ok := plan.BindVars[slice.LimitBindKey].(int) + if !ok || limitValue <= 0 { + return "", fmt.Errorf("%w: slice limit bind %q must be a positive int (ir/physical_validation_shapes.go:28-31)", + ErrUnsupportedPostgresPlan, slice.LimitBindKey) + } + limitIndex := r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(limitValue)}, "") + innerSelect += " ORDER BY " + sortExpr + " ASC NULLS FIRST, item_key ASC LIMIT $" + strconv.Itoa(limitIndex) + + r.sliceSeq++ + sliceAlias := "sl" + strconv.Itoa(r.sliceSeq) + return "(SELECT COALESCE(jsonb_agg(" + sliceAlias + ".\"item\" ORDER BY " + sliceAlias + ".\"item_sort\" ASC NULLS FIRST, " + + sliceAlias + ".\"item_key\"), '[]'::jsonb) FROM (" + innerSelect + ") " + sliceAlias + ")", nil +} + +func (r *planRenderer) sliceItemObject(plan ir.PhysicalPlan, slice *ir.PhysicalSlice, itemRows map[string]rowScope, setVariable string, depth int) (string, error) { + seen := make(map[string]bool, len(slice.Projections)) + args := make([]string, 0, 2*len(slice.Projections)) + for index, projection := range slice.Projections { + if strings.TrimSpace(projection.Name) == "" || seen[projection.Name] { + return "", fmt.Errorf("%w: slice projection %d name %q is empty or duplicated (ir/physical_validation_shapes.go:45-47)", + ErrUnsupportedPostgresPlan, index, projection.Name) + } + seen[projection.Name] = true + operand := rebindSliceOperand(projection.Expression, slice.Source.Variable) + if err := r.resolveSlicePreparedExtract(setVariable, projection.Expression, operand, + "projection "+strconv.Itoa(index)+" ("+projection.Name+")"); err != nil { + return "", fmt.Errorf("slice projection %d (%s): %w", index, projection.Name, err) + } + value, err := r.objectFieldJSON(plan, itemRows, operand, depth) + if err != nil { + return "", fmt.Errorf("slice projection %d (%s): %w", index, projection.Name, err) + } + nameIndex := r.textParameter(projection.Name, "text:"+projection.Name) + args = append(args, "$"+strconv.Itoa(nameIndex)+"::text", value) + } + return "jsonb_build_object(" + strings.Join(args, ", ") + ")", nil +} + +func (r *planRenderer) renderSlicePredicate(plan ir.PhysicalPlan, slice *ir.PhysicalSlice, itemRows map[string]rowScope, setVariable string) (string, error) { + predicate := *slice.Predicate + if predicate.Kind != ir.PhysicalComparisonPredicate || predicate.Comparison == nil { + return "", fmt.Errorf("%w: slice predicate must be a comparison (the AQL contract, render/aql/collections.go:93-95)", ErrUnsupportedPostgresPlan) + } + comparison := *predicate.Comparison + if comparison.LeftExpression == nil || comparison.LeftExpression.Extract == nil { + return "", fmt.Errorf("%w: slice predicate must carry the selector-extraction left side both slice producers emit (lower/typed_predicate.go:30-44, lower/root_projection.go:417-430); a direct column comparison is not a slice lowering output", + ErrUnsupportedPostgresPlan) + } + if comparison.Left.Variable != "" || comparison.Left.BindKey != "" { + return "", fmt.Errorf("%w: slice predicate must not also carry a direct left value", ErrUnsupportedPostgresPlan) + } + left := *comparison.LeftExpression + extract := *left.Extract + if extract.Source.Variable != slice.Source.Variable { + return "", fmt.Errorf("%w: slice predicate extraction must read the slice source row (rebind target), got variable %q", + ErrUnsupportedPostgresPlan, extract.Source.Variable) + } + extract.Source = ir.PhysicalValue{Variable: sliceItemVariable, Path: []string{"payload"}} + left.Extract = &extract + if err := r.resolveSlicePreparedExtract(setVariable, *comparison.LeftExpression, left, "predicate"); err != nil { + return "", fmt.Errorf("slice predicate: %w", err) + } + comparison.LeftExpression = &left + fragment, err := r.renderPredicateFragmentScoped(plan, ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalComparisonPredicate, + Comparison: &comparison, + }, itemRows, true) + if err != nil { + return "", fmt.Errorf("slice predicate: %w", err) + } + return fragment.sql, nil +} + +func (r *planRenderer) renderSliceSort(slice *ir.PhysicalSlice, itemRows map[string]rowScope, scope rowScope, setVariable string) (string, error) { + if slice.Sort == nil { + return "", fmt.Errorf("%w: slice requires sort expression (ir/physical_validation_shapes.go:19-24)", ErrUnsupportedPostgresPlan) + } + sortExpression := *slice.Sort + if sortExpression.Kind == ir.PhysicalValueExpression && sortExpression.Value != nil { + value := *sortExpression.Value + if value.Variable != slice.Source.Variable || value.BindKey != "" || len(value.Path) != 1 || + (value.Path[0] != "_key" && value.Path[0] != "id") { + return "", fmt.Errorf("%w: slice VALUE sort must be the lowering identity key over the slice source variable (lower/root_projection.go:406: VALUE %v); got variable %q path %v", + ErrUnsupportedPostgresPlan, []string{"_key"}, value.Variable, value.Path) + } + return scope.alias + "." + r.storage.Resource.MustColumn(canonicalKeyColumn(value.Path[0])).SQLText(), nil + } + if sortExpression.Kind == ir.PhysicalExtractExpression { + if sortExpression.Extract == nil { + return "", fmt.Errorf("%w: slice EXTRACT sort has no extraction payload", ErrUnsupportedPostgresPlan) + } + if sortExpression.Cardinality != ir.PhysicalScalarCardinality { + return "", fmt.Errorf("%w: slice EXTRACT sort must be scalar-cardinality; array sort values are a separate capability", ErrUnsupportedPostgresPlan) + } + extract := *sortExpression.Extract + if extract.Source.Variable != slice.Source.Variable { + return "", fmt.Errorf("%w: slice sort extraction must read the slice source row, got variable %q", + ErrUnsupportedPostgresPlan, extract.Source.Variable) + } + extract.Source = ir.PhysicalValue{Variable: sliceItemVariable, Path: []string{"payload"}} + sortCopy := sortExpression + sortCopy.Extract = &extract + if err := r.resolveSlicePreparedExtract(setVariable, sortExpression, sortCopy, "sort"); err != nil { + return "", fmt.Errorf("slice sort: %w", err) + } + return r.compositeExtractJSON(itemRows, sortCopy) + } + return "", fmt.Errorf("%w: slice sort kind %q is outside the registered forms (VALUE identity key or scalar EXTRACT)", + ErrUnsupportedPostgresPlan, string(sortExpression.Kind)) +} diff --git a/internal/dataframe/compiler/render/postgres/objects_test.go b/internal/dataframe/compiler/render/postgres/objects_test.go new file mode 100644 index 000000000..25fa4ba19 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/objects_test.go @@ -0,0 +1,1321 @@ +package postgres + +import ( + "context" + "encoding/json" + "fmt" + "os" + "reflect" + "strings" + "testing" + "time" + + pgstore "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + objLimitBind = "obj11_limit" + objPredicate = "obj11_predicate_value" + objSlicePrefix = "child_set_0" +) + +func objValueExpr(variable string, path []string, cardinality ir.PhysicalCardinality, behavior ir.PhysicalNullBehavior) ir.PhysicalExpression { + return ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: cardinality, NullBehavior: behavior, + Value: &ir.PhysicalValue{Variable: variable, Path: path}, + } +} + +func objExtractExpr(variable, resourceType string, steps []spec.SelectorStep, cardinality ir.PhysicalCardinality, behavior ir.PhysicalNullBehavior) ir.PhysicalExpression { + return ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, Cardinality: cardinality, NullBehavior: behavior, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: variable, Path: []string{"payload"}}, + ResourceType: resourceType, + Selector: spec.Selector{Steps: steps}, + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + } +} + +func objGenderExtract(variable, resourceType string) ir.PhysicalExpression { + return objExtractExpr(variable, resourceType, []spec.SelectorStep{{Field: "gender"}}, + ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) +} + +func objObjectExpr(fields ...ir.PhysicalExpressionProjection) ir.PhysicalExpression { + return ir.PhysicalExpression{ + Kind: ir.PhysicalObjectExpression, Cardinality: ir.PhysicalObjectCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Object: &ir.PhysicalObject{Fields: fields}, + } +} + +func objField(name string, expression ir.PhysicalExpression) ir.PhysicalExpressionProjection { + return ir.PhysicalExpressionProjection{Name: name, Expression: expression} +} + +func objSliceExpr(sourceVariable, limitBind string, projections ...ir.PhysicalExpressionProjection) ir.PhysicalExpression { + return ir.PhysicalExpression{ + Kind: ir.PhysicalSliceExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Slice: &ir.PhysicalSlice{ + Source: ir.PhysicalValue{Variable: sourceVariable}, + Sort: &ir.PhysicalExpression{Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, NullBehavior: ir.PhysicalPreserveNull, Value: &ir.PhysicalValue{Variable: sourceVariable, Path: []string{"_key"}}}, + LimitBindKey: limitBind, + Projections: projections, + }, + } +} + +func objExpressionProjection(name string, expression ir.PhysicalExpression) ir.PhysicalProjection { + return ir.PhysicalProjection{Name: name, Expression: &expression} +} + +func objPlan() ir.PhysicalPlan { + plan := setsPlan() + plan.BindVars[objLimitBind] = 2 + plan.Operations[len(plan.Operations)-1].Return.Projections = []ir.PhysicalProjection{ + {Name: "patient_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + objExpressionProjection("envelope", objObjectExpr( + objField("key", objValueExpr("root", []string{"_key"}, ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull)), + objField("status", objValueExpr("root", []string{"status"}, ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull)), + objField("gender", objGenderExtract("root", "Patient")), + )), + objExpressionProjection("items", objSliceExpr("root", objLimitBind, + objField("value", objExtractExpr("root", "Patient", []spec.SelectorStep{{Field: "active"}}, + ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull)), + )), + } + return plan +} + +func TestObjectProjectionBindsFieldNamesAsParameters(t *testing.T) { + rendered := renderOK(t, objPlan()) + sql := rendered.SQL + if got := strings.Count(sql, "jsonb_build_object($"); got != 4 { + t.Fatalf("jsonb_build_object pairs = %d, want 4 (three envelope fields + one slice item field): %s", got, sql) + } + for _, want := range []string{ + `::text, to_jsonb(r0."logical_id"))`, + `::text, (r0."payload" #> $`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("object rendering missing %q:\n%s", want, sql) + } + } + bound := map[string]bool{} + for _, arg := range rendered.Args { + if text, ok := arg.Value.(string); ok { + bound[text] = true + } + } + for _, want := range []string{"key", "status", "gender", "value", "{status}", "{gender}", "{active}"} { + if !bound[want] { + t.Fatalf("field/path %q never bound: %v", want, rendered.Args) + } + } + hostile := objPlan() + hostile.Operations[len(hostile.Operations)-1].Return.Projections[1]. + Expression.Object.Fields[0].Name = `evil"; DROP TABLE fhir_resource; --` + hostileRendered := renderOK(t, hostile) + if strings.Contains(hostileRendered.SQL, "DROP TABLE") { + t.Fatalf("hostile field name leaked into SQL text: %s", hostileRendered.SQL) + } + hostileBound := false + for _, arg := range hostileRendered.Args { + if text, ok := arg.Value.(string); ok && text == `evil"; DROP TABLE fhir_resource; --` { + hostileBound = true + } + } + if !hostileBound { + t.Fatalf("hostile field name must cross the $n boundary: %v", hostileRendered.Args) + } + again := renderOK(t, objPlan()) + if again.SQL != sql || !reflect.DeepEqual(again.Args, rendered.Args) { + t.Fatalf("object rendering is not deterministic") + } +} + +func TestObjectOmitFieldDropsBothNullStates(t *testing.T) { + plan := objPlan() + envelope := plan.Operations[len(plan.Operations)-1].Return.Projections[1].Expression.Object + envelope.Fields[1].Expression.NullBehavior = ir.PhysicalOmitNulls + rendered := renderOK(t, plan) + sql := rendered.SQL + if !strings.Contains(sql, `) IS NULL OR (`) || + !strings.Contains(sql, `= 'null'::jsonb THEN '{}'::jsonb ELSE jsonb_build_object(`) { + t.Fatalf("omission CASE missing the both-null-states reading:\n%s", sql) + } + if !strings.Contains(sql, `CASE WHEN (r0."payload" #> $`) { + t.Fatalf("omission CASE must guard the rendered value expression:\n%s", sql) + } + if got := strings.Count(sql, "CASE WHEN"); got != 1 { + t.Fatalf("CASE-wrapped fields = %d, want exactly the one omit field: %s", got, sql) + } + if !strings.Contains(sql, "(jsonb_build_object($") { + t.Fatalf("mixed omit/preserve chain must be parenthesized: %s", sql) + } +} + +func TestSliceSingletonGolden(t *testing.T) { + rendered := renderOK(t, objPlan()) + sql := rendered.SQL + if !strings.Contains(sql, `COALESCE(jsonb_agg(sl1."item" ORDER BY sl1."item_sort" ASC NULLS FIRST, sl1."item_key"), '[]'::jsonb) FROM (SELECT`) { + t.Fatalf("slice sublink spelling wrong:\n%s", sql) + } + if !strings.Contains(sql, `ORDER BY r0."logical_id" ASC NULLS FIRST, item_key ASC LIMIT $`) { + t.Fatalf("slice total-order SORT missing:\n%s", sql) + } + limitFound := false + for _, arg := range rendered.Args { + if arg.Type == command.PostgreSQLParameterBigint && reflect.DeepEqual(arg.Value, int64(2)) { + limitFound = true + } + } + if !limitFound { + t.Fatalf("slice limit must cross as a Bigint int64 parameter: %v", rendered.Args) + } + if strings.Contains(sql, "LIMIT 2") { + t.Fatalf("limit must never be interpolated: %s", sql) + } +} + +func TestSlicePredicateSortAndLimitBind(t *testing.T) { + plan := objPlan() + slice := plan.Operations[len(plan.Operations)-1].Return.Projections[2].Expression.Slice + slice.Predicate = &ir.PhysicalPredicateExpression{Kind: ir.PhysicalComparisonPredicate, Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", ValueKind: spec.FilterString, + LeftExpression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Patient", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: "gender"}}}, + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + }, + Right: &ir.PhysicalValue{BindKey: objPredicate}, + }} + slice.Sort = &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Patient", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: "gender"}}}, + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + } + plan.BindVars[objPredicate] = "female" + rendered := renderOK(t, plan) + sql := rendered.SQL + if !strings.Contains(sql, " WHERE ") || !strings.Contains(sql, `#> $`) { + t.Fatalf("slice predicate missing:\n%s", sql) + } + if strings.Contains(sql, "female") { + t.Fatalf("predicate value must bind, not inline: %s", sql) + } + if !strings.Contains(sql, `ORDER BY (r0."payload" #> $`) { + t.Fatalf("EXTRACT sort must render over the item row scope:\n%s", sql) + } + if got := strings.Count(sql, `r0."payload" #> $`); got < 3 { + t.Fatalf("predicate + item projection + sort must all read the item document (count %d): %s", got, sql) + } +} + +func objChildSetSlicePlan() ir.PhysicalPlan { + set := setsChildSet(ir.PhysicalSetReductionAll) + set.Reduction = nil + plan := setsPlan() + plan.BindVars[objLimitBind] = 2 + plan.Operations = plan.Operations[:len(plan.Operations)-1] + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalSetOp, Set: &set}, + ir.PhysicalOperation{Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "patient_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + objExpressionProjection("items", objSliceExpr(setsPrefix, objLimitBind, + objField("value", objExtractExpr(setsPrefix, "Observation", []spec.SelectorStep{{Field: "status"}}, + ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull)), + )), + }}}) + return plan +} + +func TestChildSetSliceSharesTheReductionItemRelation(t *testing.T) { + reductionRendered := renderOK(t, setsPlanWithSet(setsChildSet(ir.PhysicalSetReductionAll), "__loom_reduced_0")) + reductionSQL := reductionRendered.SQL + start := strings.Index(reductionSQL, `(SELECT DISTINCT `) + if start < 0 { + t.Fatalf("reduction item relation missing from %s", reductionSQL) + } + end := strings.Index(reductionSQL[start:], `) AS i`) + if end < 0 { + t.Fatalf("reduction item relation does not close: %s", reductionSQL) + } + items := reductionSQL[start : start+end] + + sliceRendered := renderOK(t, objChildSetSlicePlan()) + sql := sliceRendered.SQL + if !strings.Contains(sql, items+`) AS slx`) { + t.Fatalf("slice item relation must be byte-identical to the reduction relation\n want substring: %s\n got: %s", items, sql) + } + if !strings.Contains(sql, `COALESCE(jsonb_agg(sl`) { + t.Fatalf("child-set slice must still be a COALESCE'd jsonb_agg: %s", sql) + } + for _, forbidden := range []string{"LEFT JOIN", "LATERAL (SELECT"} { + if strings.Contains(sql, forbidden) { + t.Fatalf("slice must not add outer joins (%q present): %s", forbidden, sql) + } + } + if !strings.Contains(sql, `= r0."logical_id"`) { + t.Fatalf("child-set slice must correlate on the outer row identity: %s", sql) + } +} + +func TestNilReductionWithoutSliceConsumerKeepsTheSetsRefusal(t *testing.T) { + set := setsChildSet(ir.PhysicalSetReductionAll) + set.Reduction = nil + plan := setsPlan() + plan.Operations = plan.Operations[:len(plan.Operations)-1] + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalSetOp, Set: &set}, + setsReturn()) + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + wantUnsupported(t, err, "nil-Reduction set without a slice consumer") + want := `postgres renderer: unsupported physical plan: set "child_set_0" has no reduction contract; the registered surface renders sets only through their FIRST/ALL/DISTINCT selector reductions, SLICE consumers, and SUM/COUNT/MIN/MAX aggregate reducers over the registered projection slots, because no other consumer of the item array is registered` + if err.Error() != want { + t.Fatalf("no-consumer refusal drifted:\n got: %s\nwant: %s", err.Error(), want) + } +} + +func TestPreparedBearingSliceStaysRefusedVerbatim(t *testing.T) { + plan := objPlan() + r := &planRenderer{ + storage: mustBinding(t), + parameters: map[string]int{}, + rows: map[string]rowScope{"root": {alias: "r0", resourceType: "Patient"}}, + rootVariable: "root", + } + sliceExpression := objSliceExpr("root", objLimitBind, + objField("value", objExtractExpr("root", "Patient", []spec.SelectorStep{{Field: "active"}}, + ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull))) + sliceExpression.Slice.Projections[0].Expression.Extract.Prepared = &ir.PhysicalPreparedReference{ + SetVariable: "child_set_0_prepared", Field: "__loom_projection_0", + } + _, err := r.renderSliceValue(plan, r.rows, sliceExpression, 1) + wantUnsupported(t, err, "Prepared-bearing slice projection") + want := `postgres renderer: unsupported physical plan: composite EXTRACT operand: prepared-set selector references are not implemented for PostgreSQL` + if !strings.HasSuffix(err.Error(), want) { + t.Fatalf("Prepared refusal drifted from the D3 cell:\n got: %s\nwant suffix: %s", err.Error(), want) + } +} + +func objPreparedReference() *ir.PhysicalPreparedReference { + return &ir.PhysicalPreparedReference{SetVariable: setsPrefix, Field: "__loom_projection_0"} +} + +func objChildSliceOf(plan ir.PhysicalPlan) *ir.PhysicalSlice { + return plan.Operations[len(plan.Operations)-1].Return.Projections[1].Expression.Slice +} + +func obj11SlicePredicate(plan ir.PhysicalPlan) { + plan.BindVars[objPredicate] = "same" + objChildSliceOf(plan).Predicate = &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalComparisonPredicate, Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", ValueKind: spec.FilterString, + LeftExpression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: setsPrefix, Path: []string{"payload"}}, + ResourceType: "Observation", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: "status"}}}, + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + }, + Right: &ir.PhysicalValue{BindKey: objPredicate}, + }} +} + +func TestPreparedResolutionRendersByteIdenticallyToUnannotated(t *testing.T) { + arms := []struct { + name string + build func(ir.PhysicalPlan) + annotate func(ir.PhysicalPlan) + }{ + {"projection", nil, func(plan ir.PhysicalPlan) { + objChildSliceOf(plan).Projections[0].Expression.Extract.Prepared = objPreparedReference() + }}, + {"sort", obj11StatusSort, func(plan ir.PhysicalPlan) { + objChildSliceOf(plan).Sort.Extract.Prepared = objPreparedReference() + }}, + {"predicate", obj11SlicePredicate, func(plan ir.PhysicalPlan) { + objChildSliceOf(plan).Predicate.Comparison.LeftExpression.Extract.Prepared = objPreparedReference() + }}, + {"bare-row-source", func(plan ir.PhysicalPlan) { + objChildSliceOf(plan).Projections[0].Expression.Extract.Source.Path = nil + }, func(plan ir.PhysicalPlan) { + objChildSliceOf(plan).Projections[0].Expression.Extract.Prepared = objPreparedReference() + }}, + } + for _, arm := range arms { + t.Run(arm.name, func(t *testing.T) { + plain, annotated := objChildSetSlicePlan(), objChildSetSlicePlan() + if arm.build != nil { + arm.build(plain) + arm.build(annotated) + } + arm.annotate(annotated) + plainRendered, annotatedRendered := renderOK(t, plain), renderOK(t, annotated) + probe := objChildSliceOf(annotated) + carries := func() bool { + switch arm.name { + case "sort": + return probe.Sort.Extract != nil && probe.Sort.Extract.Prepared != nil + case "predicate": + return probe.Predicate != nil && probe.Predicate.Comparison != nil && + probe.Predicate.Comparison.LeftExpression != nil && + probe.Predicate.Comparison.LeftExpression.Extract != nil && + probe.Predicate.Comparison.LeftExpression.Extract.Prepared != nil + default: + return probe.Projections[0].Expression.Extract != nil && + probe.Projections[0].Expression.Extract.Prepared != nil + } + } + if !carries() { + t.Fatalf("rendering mutated the plan: %s-arm Extract.Prepared was stripped from the IR", arm.name) + } + if annotatedRendered.SQL != plainRendered.SQL { + t.Fatalf("annotated SQL drifted from the unannotated slot text:\nplain: %s\nannotated: %s", plainRendered.SQL, annotatedRendered.SQL) + } + if !reflect.DeepEqual(annotatedRendered.Args, plainRendered.Args) { + t.Fatalf("annotated Args drifted from the unannotated slot text:\nplain: %v\nannotated: %v", plainRendered.Args, annotatedRendered.Args) + } + for _, leak := range []string{"__loom_projection", "prepared", "child_set"} { + if strings.Contains(annotatedRendered.SQL, leak) { + t.Fatalf("%q leaked into resolved statement text: %s", leak, annotatedRendered.SQL) + } + } + }) + } +} + +func TestPreparedRegisterSlotTableRefusals(t *testing.T) { + const verbatim = `postgres renderer: unsupported physical plan: composite EXTRACT operand: prepared-set selector references are not implemented for PostgreSQL` + directStatus := spec.Selector{Steps: []spec.SelectorStep{{Field: "status"}}} + iteratingCategory := spec.Selector{Steps: []spec.SelectorStep{{Field: "category", Iterate: true}}} + operand := func(selector spec.Selector) ir.PhysicalExpression { + return objExtractExpr(setsPrefix, "Observation", selector.Steps, + ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + } + for _, testCase := range []struct { + name string + slot *ir.PhysicalSetProjectionField + extract ir.PhysicalExpression + want string + }{ + { + name: "referenceNamesAnotherSet", + slot: &ir.PhysicalSetProjectionField{Name: "__loom_projection_0", ResourceType: "Observation", Selector: directStatus, ExecutionMode: ir.PhysicalSelectorDirectScalar}, + extract: operand(directStatus), + want: verbatim, + }, + { + name: "referenceNamesAnUnregisteredSlot", + slot: &ir.PhysicalSetProjectionField{Name: "__loom_projection_0", ResourceType: "Observation", Selector: directStatus, ExecutionMode: ir.PhysicalSelectorDirectScalar}, + extract: operand(directStatus), + want: verbatim, + }, + { + name: "slotNamesADifferentSelector", + slot: &ir.PhysicalSetProjectionField{Name: "__loom_projection_0", ResourceType: "Observation", Selector: directStatus, ExecutionMode: ir.PhysicalSelectorDirectScalar}, + extract: operand(spec.Selector{Steps: []spec.SelectorStep{{Field: "note"}}}), + want: "names a different selector than the operand carries", + }, + { + name: "slotResourceTypeDisagrees", + slot: &ir.PhysicalSetProjectionField{Name: "__loom_projection_0", ResourceType: "Patient", Selector: directStatus, ExecutionMode: ir.PhysicalSelectorDirectScalar}, + extract: operand(directStatus), + want: "projects resource type \"Patient\" but the operand validates against \"Observation\"", + }, + { + name: "slotSelectorIsNotADirectPath", + slot: &ir.PhysicalSetProjectionField{Name: "__loom_projection_0", ResourceType: "Observation", Selector: iteratingCategory, ExecutionMode: ir.PhysicalSelectorDirectScalar}, + extract: operand(iteratingCategory), + want: "not a direct property step", + }, + { + name: "slotIsConditionalArray", + slot: &ir.PhysicalSetProjectionField{Name: "__loom_projection_0", ResourceType: "Observation", Selector: directStatus, ExecutionMode: ir.PhysicalSelectorConditionalArray}, + extract: operand(directStatus), + want: "is CONDITIONAL_ARRAY, whose repeated-step rendering is not registered", + }, + { + name: "slotDemandIsUnregistered", + slot: &ir.PhysicalSetProjectionField{Name: "__loom_projection_0", ResourceType: "Observation", Selector: directStatus, ExecutionMode: ir.PhysicalSelectorDirectScalar, Demand: ir.PhysicalSelectorValueDemand("BOGUS")}, + extract: operand(directStatus), + want: "carries unsupported value demand \"BOGUS\"", + }, + { + name: "slotModeIsUnknown", + slot: &ir.PhysicalSetProjectionField{Name: "__loom_projection_0", ResourceType: "Observation", Selector: directStatus, ExecutionMode: ir.PhysicalSelectorExecutionMode("MYSTERY")}, + extract: operand(directStatus), + want: "unknown selector execution mode \"MYSTERY\"", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + extract := testCase.extract + reference := objPreparedReference() + switch testCase.name { + case "referenceNamesAnotherSet": + reference.SetVariable = "child_set_9" + case "referenceNamesAnUnregisteredSlot": + reference.Field = "__loom_projection_7" + } + extract.Extract.Prepared = reference + r := &planRenderer{ + storage: mustBinding(t), + parameters: map[string]int{}, + rows: map[string]rowScope{"root": {alias: "r0", resourceType: "Patient"}}, + rootVariable: "root", + setBodies: map[string]correlatedSetBody{setsPrefix: {from: "FROM fhir_resource x1", where: `x1.project = $1`, targetAlias: "x1", targetType: "Observation", targetVar: "x1"}}, + setProjections: map[string][]ir.PhysicalSetProjectionField{setsPrefix: {*testCase.slot}}, + } + plan := setsPlan() + plan.BindVars[objLimitBind] = 2 + sliceExpression := objSliceExpr(setsPrefix, objLimitBind, objField("value", extract)) + _, err := r.renderSliceValue(plan, r.rows, sliceExpression, 1) + wantUnsupported(t, err, "Prepared register: "+testCase.name) + if !strings.Contains(err.Error(), testCase.want) { + t.Fatalf("refusal %q drifted:\n got: %s\nwant substring: %s", testCase.name, err.Error(), testCase.want) + } + }) + } +} + +func TestPreparedInsideSliceItemObjectKeepsVerbatimRefusal(t *testing.T) { + plan := objChildSetSlicePlan() + field := objField("v", objExtractExpr(setsPrefix, "Observation", []spec.SelectorStep{{Field: "status"}}, + ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull)) + field.Expression.Extract.Prepared = objPreparedReference() + objChildSliceOf(plan).Projections[0].Expression = objObjectExpr(field) + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + wantUnsupported(t, err, "Prepared inside a slice item OBJECT") + const verbatim = `postgres renderer: unsupported physical plan: composite EXTRACT operand: prepared-set selector references are not implemented for PostgreSQL` + if !strings.HasSuffix(err.Error(), verbatim) { + t.Fatalf("OBJECT-arm Prepared refusal drifted:\n got: %s\nwant suffix: %s", err.Error(), verbatim) + } +} + +func objNestedObjects(depth int) ir.PhysicalExpression { + expression := objGenderExtract("root", "Patient") + for level := 0; level < depth; level++ { + expression = objObjectExpr(objField("level", expression)) + } + return expression +} + +func TestCompositeDepthCapIsDefensive(t *testing.T) { + plan := objPlan() + plan.Operations[len(plan.Operations)-1].Return.Projections[1]. + Expression = &ir.PhysicalExpression{} + *plan.Operations[len(plan.Operations)-1].Return.Projections[1].Expression = objNestedObjects(4) + renderOK(t, plan) + + deep := objPlan() + deepExpression := objNestedObjects(5) + deep.Operations[len(deep.Operations)-1].Return.Projections[1]. + Expression = &deepExpression + _, err := RenderPhysicalPlan(deep, mustBinding(t)) + wantUnsupported(t, err, "depth-5 composite") + if !strings.Contains(err.Error(), "nesting exceeds the registered maximum depth 4") { + t.Fatalf("depth refusal missing the cap sentence: %v", err) + } + if !strings.Contains(err.Error(), "unbounded") { + t.Fatalf("depth refusal must cite the AQL-unbounded contrast: %v", err) + } +} + +func objBatteryReturn(plan *ir.PhysicalPlan) []ir.PhysicalProjection { + return plan.Operations[len(plan.Operations)-1].Return.Projections +} + +func TestCompositeSurfaceRefusals(t *testing.T) { + tests := []struct { + name string + want string + rendererLevel bool + mutate func(*ir.PhysicalPlan) + }{ + { + name: "object cardinality dropped", + want: "must carry OBJECT cardinality", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[1].Expression.Cardinality = ir.PhysicalScalarCardinality + }, + }, + { + name: "slice cardinality dropped", + want: "must carry ARRAY cardinality", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[2].Expression.Cardinality = ir.PhysicalScalarCardinality + }, + }, + { + name: "slice null behavior dropped", + want: "EMPTY_ON_NULL", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[2].Expression.NullBehavior = ir.PhysicalPreserveNull + }, + }, + { + name: "slice source is a bind value", + want: "not a bind value", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + plan.BindVars["obj11_bind_source"] = "root" + objBatteryReturn(plan)[2].Expression.Slice.Source = ir.PhysicalValue{BindKey: "obj11_bind_source"} + }, + }, + { + name: "slice source carries a document path", + want: "must be the row itself", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[2].Expression.Slice.Source.Path = []string{"payload"} + }, + }, + { + name: "slice source is an unregistered variable", + want: "not a registered item source", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + traversal := setsTraversal() + traversal.TargetVariable = "spine_node" + plan.Operations = append(plan.Operations[:len(plan.Operations)-1], + ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &traversal}, + plan.Operations[len(plan.Operations)-1]) + objBatteryReturn(plan)[2].Expression.Slice.Source.Variable = "spine_node" + sliceSort := objBatteryReturn(plan)[2].Expression.Slice.Sort + sliceSort.Value.Variable = "spine_node" + }, + }, + { + name: "slice without a projection", + want: "at least one projection", + rendererLevel: false, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[2].Expression.Slice.Projections = nil + }, + }, + { + name: "slice without sort", + want: "stable sort", + rendererLevel: false, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[2].Expression.Slice.Sort = nil + }, + }, + { + name: "slice VALUE sort off the identity key", + want: "identity key", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[2].Expression.Slice.Sort.Value.Path = []string{"gender"} + }, + }, + { + name: "slice sort of a foreign kind", + want: "outside the registered forms", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + literal := ir.PhysicalExpression{Kind: ir.PhysicalLiteralExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, Literal: &ir.PhysicalLiteral{BindKey: objLimitBind}} + objBatteryReturn(plan)[2].Expression.Slice.Sort = &literal + }, + }, + { + name: "slice predicate extraction on another row", + want: "must read the slice source row", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + traversal := setsTraversal() + traversal.TargetVariable = "spine_node" + plan.Operations = append(plan.Operations[:len(plan.Operations)-1], + ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &traversal}, + plan.Operations[len(plan.Operations)-1]) + predicate := objBatteryReturn(plan)[2].Expression.Slice.Predicate + predicate.Comparison.LeftExpression.Extract.Source.Variable = "spine_node" + }, + }, + { + name: "slice predicate of a foreign kind", + want: "must be a comparison", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + slice := objBatteryReturn(plan)[2].Expression.Slice + comparison := slice.Predicate.Comparison + slice.Predicate = &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalAllPredicate, + Children: []ir.PhysicalPredicateExpression{{Kind: ir.PhysicalComparisonPredicate, Comparison: comparison}}, + } + }, + }, + { + name: "slice limit bind zero", + want: "positive int", + rendererLevel: false, + mutate: func(plan *ir.PhysicalPlan) { + plan.BindVars[objLimitBind] = 0 + }, + }, + { + name: "composite literal operand", + want: "no registered PostgreSQL rendering", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + literal := ir.PhysicalExpression{Kind: ir.PhysicalLiteralExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, Literal: &ir.PhysicalLiteral{BindKey: "project"}} + objBatteryReturn(plan)[1].Expression.Object.Fields[0].Expression = literal + }, + }, + { + name: "composite VALUE operand reads a bind", + want: "not a bind value", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[1].Expression.Object.Fields[1].Expression.Value = &ir.PhysicalValue{BindKey: "project"} + }, + }, + { + name: "composite VALUE operand without a path", + want: "must name a document path", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[1].Expression.Object.Fields[1].Expression.Value.Path = nil + }, + }, + { + name: "composite extraction over a foreign source path", + want: `spelled ["payload"] or as the bare row`, + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[1].Expression.Object.Fields[2].Expression.Extract.Source.Path = []string{"gender"} + }, + }, + { + name: "composite extraction DISTINCT", + want: "DISTINCT projection is meaningful only", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[1].Expression.Object.Fields[2].Expression.Extract.Distinct = true + }, + }, + { + name: "composite extraction iterate step", + want: "direct scalar selector mode requires", + rendererLevel: false, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[1].Expression.Object.Fields[2].Expression.Extract.Selector.Steps[0].Iterate = true + }, + }, + { + name: "composite extraction conditional array mode", + want: "conditional array selector mode requires", + rendererLevel: false, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[1].Expression.Object.Fields[2].Expression.Extract.ExecutionMode = ir.PhysicalSelectorConditionalArray + }, + }, + { + name: "composite extraction resource-type mismatch", + want: "scanned as", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + plan.BindVars["root_collection"] = "Observation" + }, + }, + { + name: "empty object", + want: "at least one projection", + rendererLevel: false, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[1].Expression.Object.Fields = nil + }, + }, + { + name: "duplicate object field name", + want: "empty or duplicated", + rendererLevel: false, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[1].Expression.Object.Fields[1].Name = "key" + }, + }, + { + name: "duplicate slice projection name", + want: "empty or duplicated", + rendererLevel: false, + mutate: func(plan *ir.PhysicalPlan) { + slice := objBatteryReturn(plan)[2].Expression.Slice + slice.Projections = append(slice.Projections, slice.Projections[0]) + }, + }, + { + name: "presence metadata on an OBJECT projection", + want: "presence proofs pair with selector extractions only", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[1].Presence = &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + Paths: [][]string{{"gender"}}, + } + }, + }, + { + name: "object expression without payload", + want: "exactly one payload", + rendererLevel: false, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[1].Expression.Object = nil + }, + }, + { + name: "slice expression without payload", + want: "exactly one payload", + rendererLevel: false, + mutate: func(plan *ir.PhysicalPlan) { + objBatteryReturn(plan)[2].Expression.Slice = nil + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + plan := objPlan() + if test.want == "must read the slice source row" || test.want == "must be a comparison" { + slice := plan.Operations[len(plan.Operations)-1].Return.Projections[2].Expression.Slice + slice.Predicate = &ir.PhysicalPredicateExpression{Kind: ir.PhysicalComparisonPredicate, Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", ValueKind: spec.FilterString, + LeftExpression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Patient", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: "gender"}}}, + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + }, + Right: &ir.PhysicalValue{BindKey: objPredicate}, + }} + plan.BindVars[objPredicate] = "female" + } + test.mutate(&plan) + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil { + t.Fatalf("%s: expected refusal, render succeeded", test.name) + } + if !strings.Contains(err.Error(), test.want) { + t.Fatalf("%s: error = %v, want substring %q", test.name, err, test.want) + } + if test.rendererLevel { + wantUnsupported(t, err, test.name) + } + }) + } +} + +func TestClosedSurfacesKeepTheirCompositeReceipts(t *testing.T) { + plan := basePlan() + plan.BindVars[objLimitBind] = 2 + plan.Operations[len(plan.Operations)-1].Return.Projections = []ir.PhysicalProjection{ + objExpressionProjection("envelope", objObjectExpr( + objField("gender", objGenderExtract("root", "Patient")), + )), + } + _, err := RenderSelectorExtraction(plan, mustBinding(t)) + wantUnsupported(t, err, "selector-extraction composite receipt") + if !strings.Contains(err.Error(), "composites belong to the read spine registration (objects.go)") { + t.Fatalf("selector-extraction receipt drifted: %v", err) + } + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); err != nil { + t.Fatalf("spine must render the composite projection: %v", err) + } +} + +func TestMeltInputKeepsItsCompositeReceipt(t *testing.T) { + plan := unp15Plan(t, unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + for index := range plan.Operations { + if plan.Operations[index].Kind == ir.PhysicalUnpivotOp { + plan.Operations[index].Unpivot.InputProjections[1].Expression = &ir.PhysicalExpression{} + *plan.Operations[index].Unpivot.InputProjections[1].Expression = objObjectExpr( + objField("gender", objGenderExtract("root", "Patient"))) + plan.Operations[index].Unpivot.InputProjections[1].Value = ir.PhysicalValue{} + } + } + }}) + _, err := RenderUnpivot(plan, mustBinding(t)) + if err == nil { + t.Fatalf("melt input composite must be refused") + } + if !strings.Contains(err.Error(), "melt inputs are the spine's scalar/selector projections") { + t.Fatalf("melt-input receipt drifted: %v", err) + } + wantUnsupported(t, err, "melt-input composite receipt") +} + +func TestMembershipRootRefusesCompositeProjections(t *testing.T) { + plan := objPlan() + plan.BindVars["obj11_membership_probe"] = true + r := &planRenderer{ + storage: mustBinding(t), + parameters: map[string]int{}, + rows: map[string]rowScope{"root": {alias: "r0", resourceType: "Patient"}}, + membership: &PostgresPopulationRootBinding{}, + } + _, err := r.renderObjectValue(plan, r.rows, *objBatteryReturn(&plan)[1].Expression, 1) + wantUnsupported(t, err, "OBJECT on a membership root") + if !strings.Contains(err.Error(), "membership-root routes") { + t.Fatalf("membership gate drifted: %v", err) + } + _, err = r.renderSliceValue(plan, r.rows, *objBatteryReturn(&plan)[2].Expression, 1) + wantUnsupported(t, err, "SLICE on a membership root") + if !strings.Contains(err.Error(), "membership-root routes") { + t.Fatalf("membership gate drifted: %v", err) + } +} + +func TestSpineStatementsStayByteIdentical(t *testing.T) { + const witnessSQL = `SELECT r0."logical_id" AS "c0", r0."project" AS "c1", (r0."payload" #> $5::text[]) AS "c2" FROM "pgir02_it_unit"."fhir_resource" AS r0 WHERE r0."project" = $1 AND r0."dataset_generation" = $2 AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb)) AND r0."resource_type" = $4 ORDER BY r0."resource_seq" ASC` + plan := basePlan() + plan.Operations[len(plan.Operations)-1].Return.Projections = []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + {Name: "project", Value: ir.PhysicalValue{Variable: "root", Path: []string{"project"}}}, + objExpressionProjection("gender", objGenderExtract("root", "Patient")), + } + rendered := renderOK(t, plan) + if rendered.SQL != witnessSQL { + t.Fatalf("spine statement drifted from the f1389b45 witness:\n got: %s\nwant: %s", rendered.SQL, witnessSQL) + } + wantArgs := []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "proj-a"}, + {Type: command.PostgreSQLParameterText, Value: "gen-1"}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["FHIR/Patient","FHIR/Observation"]`)}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + {Type: command.PostgreSQLParameterText, Value: "{gender}"}, + } + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("spine arguments drifted from the f1389b45 witness:\n got: %v\nwant: %v", rendered.Args, wantArgs) + } +} + +func obj11RenderOK(t *testing.T, svc *pgir02Service, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + return rendered +} + +func obj11ObjectPlan(order []string) ir.PhysicalPlan { + plan := basePlan() + fields := make([]ir.PhysicalExpressionProjection, 0, len(order)) + for _, name := range order { + switch name { + case "gender": + fields = append(fields, objField("gender", objGenderExtract("root", "Patient"))) + case "active": + fields = append(fields, objField("active", objExtractExpr("root", "Patient", + []spec.SelectorStep{{Field: "active"}}, ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull))) + case "dropped": + fields = append(fields, objField("dropped", objExtractExpr("root", "Patient", + []spec.SelectorStep{{Field: "gender"}}, ir.PhysicalScalarCardinality, ir.PhysicalOmitNulls))) + case "kept": + fields = append(fields, objField("kept", objGenderExtract("root", "Patient"))) + default: + panic("unknown object field " + name) + } + } + projections := []ir.PhysicalProjection{ + {Name: "pid", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + {Name: "kept", Expression: &ir.PhysicalExpression{}}, + } + *projections[1].Expression = objGenderExtract("root", "Patient") + if len(order) > 0 { + projections = append([]ir.PhysicalProjection{ + objExpressionProjection("envelope", objObjectExpr(fields...)), + }, projections...) + } + plan.Operations[len(plan.Operations)-1].Return.Projections = projections + return plan +} + +func TestObject11LiveObjectNullStatesAndOmission(t *testing.T) { + svc := newOwnedService(t, 0, "obj11_it") + svc.insertResource(t, svc.schema, "proj-a", "gen-1", "Patient", "p-absent", "FHIR/Patient", map[string]any{"note": "x"}) + svc.insertResource(t, svc.schema, "proj-a", "gen-1", "Patient", "p-null", "FHIR/Patient", map[string]any{"gender": nil}) + svc.insertResource(t, svc.schema, "proj-a", "gen-1", "Patient", "p-present", "FHIR/Patient", map[string]any{"gender": "female"}) + + rendered := obj11RenderOK(t, svc, obj11ObjectPlan([]string{"kept", "dropped"})) + if strings.Contains(rendered.SQL, "female") || strings.Contains(rendered.SQL, "dropped") { + t.Fatalf("field names and values must stay below the $n boundary: %s", rendered.SQL) + } + rows := svc.runRendered(t, rendered) + if len(rows) != 3 { + t.Fatalf("rows = %d, want 3 (parent grain): %v", len(rows), rows) + } + byID := map[string][]string{} + for _, row := range rows { + byID[row[1]] = row + } + if got := byID["p-present"][0]; got != "map[dropped:female kept:female]" { + t.Fatalf("present envelope = %q", got) + } + for _, id := range []string{"p-null", "p-absent"} { + if got := byID[id][0]; got != "map[kept:]" { + t.Fatalf("%s envelope = %q, want the kept key alone with a JSON null", id, got) + } + } + witness := svc.witness(t, `SELECT logical_id, + ((payload #> '{gender}') IS NULL)::text, + jsonb_typeof(payload #> '{gender}')::text + FROM "`+svc.schema+`"."fhir_resource" + WHERE project = 'proj-a' AND dataset_generation = 'gen-1' AND resource_type = 'Patient' + ORDER BY logical_id`) + if len(witness) != 3 { + t.Fatalf("witness rows = %d: %v", len(witness), witness) + } + states := map[string][2]string{ + "p-absent": {"true", ""}, + "p-null": {"false", "null"}, + "p-present": {"false", "string"}, + } + for _, row := range witness { + wantState, ok := states[row[0]] + if !ok || row[1] != wantState[0] || row[2] != wantState[1] { + t.Fatalf("witness state for %s = (%s, %s), want %v — the store must keep the two null states distinct", row[0], row[1], row[2], wantState) + } + omitted := row[1] == "true" || row[2] == "null" + hasDropped := strings.Contains(byID[row[0]][0], "dropped:") + if hasDropped == omitted { + t.Fatalf("rendered omission disagrees with the SQL witness for %s (dropped key=%v, witness omitted=%v)", row[0], hasDropped, omitted) + } + } + counts := svc.witness(t, `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" + WHERE project='proj-a' AND dataset_generation='gen-1' AND logical_id='p-absent' + AND jsonb_exists(payload, 'gender')`) + if len(counts) != 1 || counts[0][0] != "0" { + t.Fatalf("p-absent must store no gender key: %v", counts) + } +} + +func TestObject11LiveKeyOrderIrrelevance(t *testing.T) { + svc := newOwnedService(t, 0, "obj11_it") + svc.insertResource(t, svc.schema, "proj-a", "gen-1", "Patient", "p-1", "FHIR/Patient", + map[string]any{"gender": "male", "active": true}) + + first := obj11RenderOK(t, svc, obj11ObjectPlan([]string{"gender", "active"})) + second := obj11RenderOK(t, svc, obj11ObjectPlan([]string{"active", "gender"})) + if first.SQL == second.SQL { + t.Fatalf("declaration orders must reach different statement shapes (pair order): %s", first.SQL) + } + rowsFirst := svc.runRendered(t, first) + rowsSecond := svc.runRendered(t, second) + if len(rowsFirst) != 1 || len(rowsSecond) != 1 { + t.Fatalf("one row expected: %v / %v", rowsFirst, rowsSecond) + } + wantCell := "map[active:true gender:male]" + if rowsFirst[0][0] != wantCell || rowsSecond[0][0] != wantCell { + t.Fatalf("decoded object drifted by declaration order:\n first: %q\n second: %q\n want: %q", + rowsFirst[0][0], rowsSecond[0][0], wantCell) + } + witness := svc.witness(t, `SELECT jsonb_build_object('active', true, 'gender', 'male'::text) + UNION ALL SELECT jsonb_build_object('gender', 'male'::text, 'active', true)`) + if len(witness) != 2 || witness[0][0] != wantCell || witness[1][0] != wantCell { + t.Fatalf("witness key-order irrelevance failed: %v", witness) + } +} + +func obj11StatusSort(plan ir.PhysicalPlan) { + slice := plan.Operations[len(plan.Operations)-1].Return.Projections[1].Expression.Slice + slice.Sort = &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: setsPrefix, Path: []string{"payload"}}, + ResourceType: "Observation", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: "status"}}}, + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + } +} + +func TestObject11LiveChildSetSliceDedupTieBreakLimitAndPredicate(t *testing.T) { + svc := newOwnedService(t, 0, "obj11_it") + svc.insertResource(t, svc.schema, "proj-a", "gen-1", "Patient", "p-1", "FHIR/Patient", map[string]any{}) + svc.insertResource(t, svc.schema, "proj-a", "gen-1", "Patient", "p-2", "FHIR/Patient", map[string]any{}) + for _, pair := range [][2]string{{"obs-a", "same"}, {"obs-b", "same"}, {"obs-c", "early"}} { + svc.insertResource(t, svc.schema, "proj-a", "gen-1", "Observation", pair[0], "FHIR/Observation", map[string]any{"status": pair[1]}) + } + svc.insertResource(t, svc.schema, "proj-a", "gen-1", "Observation", "obs-blocked", "FHIR/Blocked", map[string]any{"status": "sneaky"}) + insert := func(from, to, sourcePath string) { + svc.insertEdge(t, pgstore.EdgeOccurrenceRow{ + Project: "proj-a", DatasetGeneration: "gen-1", AuthResourcePath: "FHIR/Patient", + Label: "has-observation", SourceResourceType: "Patient", SourceLogicalID: from, + TargetResourceType: "Observation", TargetLogicalID: to, SourcePath: sourcePath, SourceOrdinal: 0, + }) + } + insert("p-1", "obs-a", "observation[0]") + insert("p-1", "obs-b", "observation[0]") + insert("p-1", "obs-b", "observation[1]") + insert("p-1", "obs-c", "observation[2]") + insert("p-1", "obs-blocked", "observation[9]") + + plan := objChildSetSlicePlan() + plan.BindVars[objLimitBind] = 10 + obj11StatusSort(plan) + full := svc.runRendered(t, obj11RenderOK(t, svc, plan)) + if len(full) != 2 { + t.Fatalf("both parents must survive (empty-array grain): %v", full) + } + if full[0][0] != "p-1" || full[1][0] != "p-2" { + t.Fatalf("unexpected row order: %v", full) + } + const fullItems = "[map[value:early] map[value:same] map[value:same]]" + if full[0][1] != fullItems { + t.Fatalf("items = %q, want %q (two occurrences of obs-b dedup to one; status ASC, logical_id tie-break)", full[0][1], fullItems) + } + if full[1][1] != "[]" { + t.Fatalf("zero-child parent must decode to the empty array, got %q", full[1][1]) + } + if strings.Contains(full[0][1], "sneaky") { + t.Fatalf("denied-auth child leaked into the slice: %s", full[0][1]) + } + witness := svc.witness(t, `SELECT DISTINCT x.logical_id, x.payload #> '{status}' + FROM "`+svc.schema+`"."fhir_edge_occurrence" e + INNER JOIN "`+svc.schema+`"."fhir_resource" x + ON x.project = 'proj-a' AND x.dataset_generation = 'gen-1' + AND x.auth_resource_path IN ('FHIR/Patient','FHIR/Observation') + AND x.resource_type = 'Observation' AND x.logical_id = e.target_logical_id + WHERE e.project = 'proj-a' AND e.dataset_generation = 'gen-1' + AND e.auth_resource_path IN ('FHIR/Patient','FHIR/Observation') + AND e.label = 'has-observation' AND e.source_resource_type = 'Patient' + AND e.source_logical_id = 'p-1' AND e.target_resource_type = 'Observation' + ORDER BY (x.payload #> '{status}') ASC NULLS FIRST, x.logical_id ASC`) + if len(witness) != 3 || witness[0][0] != "obs-c" || witness[1][0] != "obs-a" || witness[2][0] != "obs-b" { + t.Fatalf("witness order drifted: %v", witness) + } + if witness[0][1] != "early" || witness[1][1] != "same" || witness[2][1] != "same" { + t.Fatalf("witness values drifted: %v", witness) + } + + limited := objChildSetSlicePlan() + limited.BindVars[objLimitBind] = 2 + obj11StatusSort(limited) + limitedRows := svc.runRendered(t, obj11RenderOK(t, svc, limited)) + if limitedRows[0][1] != "[map[value:early] map[value:same]]" { + t.Fatalf("LIMIT 2 must render the witness prefix, got %q", limitedRows[0][1]) + } + + predicated := objChildSetSlicePlan() + predicated.BindVars[objLimitBind] = 10 + predicated.BindVars[objPredicate] = "same" + predicated.Operations[len(predicated.Operations)-1].Return.Projections[1].Expression.Slice.Predicate = + &ir.PhysicalPredicateExpression{Kind: ir.PhysicalComparisonPredicate, Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", ValueKind: spec.FilterString, + LeftExpression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: setsPrefix, Path: []string{"payload"}}, + ResourceType: "Observation", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: "status"}}}, + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + }, + Right: &ir.PhysicalValue{BindKey: objPredicate}, + }} + predicatedRows := svc.runRendered(t, obj11RenderOK(t, svc, predicated)) + if predicatedRows[0][1] != "[map[value:same] map[value:same]]" { + t.Fatalf("predicate must keep exactly the two \"same\" children, got %q", predicatedRows[0][1]) + } +} + +func TestObject11LiveSortHomogeneousTypeClasses(t *testing.T) { + svc := newOwnedService(t, 0, "obj11_it") + svc.insertResource(t, svc.schema, "proj-a", "gen-1", "Patient", "p-1", "FHIR/Patient", map[string]any{}) + for _, fixture := range []struct { + logicalID string + payload map[string]any + }{ + {"obs-1", map[string]any{"valueInteger": 10}}, + {"obs-2", map[string]any{"valueInteger": 2}}, + {"obs-3", map[string]any{"valueInteger": 1}}, + {"obs-4", map[string]any{}}, + } { + svc.insertResource(t, svc.schema, "proj-a", "gen-1", "Observation", fixture.logicalID, "FHIR/Observation", fixture.payload) + } + for ordinal, target := range []string{"obs-1", "obs-2", "obs-3", "obs-4"} { + svc.insertEdge(t, pgstore.EdgeOccurrenceRow{ + Project: "proj-a", DatasetGeneration: "gen-1", AuthResourcePath: "FHIR/Patient", + Label: "has-observation", SourceResourceType: "Patient", SourceLogicalID: "p-1", + TargetResourceType: "Observation", TargetLogicalID: target, SourcePath: "observation[0]", SourceOrdinal: int64(ordinal), + }) + } + + sortOn := func(plan ir.PhysicalPlan, property string, steps []spec.SelectorStep) { + slice := objChildSliceOf(plan) + slice.Projections[0].Expression = objExtractExpr(setsPrefix, "Observation", steps, + ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull) + slice.Sort = &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: setsPrefix}, + ResourceType: "Observation", + Selector: spec.Selector{Steps: steps}, + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + } + } + + numeric := objChildSetSlicePlan() + numeric.BindVars[objLimitBind] = 10 + sortOn(numeric, "valueInteger", []spec.SelectorStep{{Field: "valueInteger"}}) + byParent := map[string]string{} + for _, row := range svc.runRendered(t, obj11RenderOK(t, svc, numeric)) { + byParent[row[0]] = row[1] + } + const wantNumeric = "[map[value:] map[value:1] map[value:2] map[value:10]]" + if got := byParent["p-1"]; got != wantNumeric { + t.Fatalf("numeric sort = %q, want %q (numeric order, null first — AQL total order within the class)", got, wantNumeric) + } + + svc.insertResource(t, svc.schema, "proj-a", "gen-1", "Patient", "p-2", "FHIR/Patient", map[string]any{}) + for _, fixture := range []struct { + logicalID string + payload map[string]any + }{ + {"bool-1", map[string]any{"valueBoolean": true}}, + {"bool-2", map[string]any{"valueBoolean": false}}, + {"bool-3", map[string]any{}}, + } { + svc.insertResource(t, svc.schema, "proj-a", "gen-1", "Observation", fixture.logicalID, "FHIR/Observation", fixture.payload) + svc.insertEdge(t, pgstore.EdgeOccurrenceRow{ + Project: "proj-a", DatasetGeneration: "gen-1", AuthResourcePath: "FHIR/Patient", + Label: "has-observation", SourceResourceType: "Patient", SourceLogicalID: "p-2", + TargetResourceType: "Observation", TargetLogicalID: fixture.logicalID, SourcePath: "observation[9]", SourceOrdinal: 9, + }) + } + boolean := objChildSetSlicePlan() + boolean.BindVars[objLimitBind] = 10 + sortOn(boolean, "valueBoolean", []spec.SelectorStep{{Field: "valueBoolean"}}) + byParent = map[string]string{} + for _, row := range svc.runRendered(t, obj11RenderOK(t, svc, boolean)) { + byParent[row[0]] = row[1] + } + const wantBoolean = "[map[value:] map[value:false] map[value:true]]" + if got := byParent["p-2"]; got != wantBoolean { + t.Fatalf("boolean sort = %q, want %q (missing < false < true — AQL total order within the class)", got, wantBoolean) + } +} + +func TestCanonicalKeyReferencesResolveToTheIdentityColumnOnBothLegs(t *testing.T) { + plan := basePlan() + plan.BindVars["id_value"] = "p-1" + plan.Operations = plan.Operations[:len(plan.Operations)-1] + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: "root", Path: []string{"id"}}, + Right: &ir.PhysicalValue{BindKey: "id_value"}, + }}}, + ir.PhysicalOperation{Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "key", Value: ir.PhysicalValue{Variable: "root", Path: []string{"_key"}}}, + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}) + rendered := renderOK(t, plan) + for _, want := range []string{`r0."logical_id" AS "c0"`, `r0."logical_id" AS "c1"`, `r0."logical_id" = $`} { + if !strings.Contains(rendered.SQL, want) { + t.Fatalf("canonical-key leg lost %q: %s", want, rendered.SQL) + } + } + for _, forbidden := range []string{"_key", "id", "_id"} { + for _, arg := range rendered.Args { + if text, ok := arg.Value.(string); ok && text == forbidden { + t.Fatalf("identity spelling %q reached Args: %v", forbidden, rendered.Args) + } + } + } +} + +func obj11SnapshotOwnedResources(ctx context.Context, raw string) ([]string, error) { + pool, err := pgxpool.New(ctx, raw) + if err != nil { + return nil, err + } + defer pool.Close() + queryCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + var names []string + for _, query := range []string{ + `SELECT datname FROM pg_database WHERE datname LIKE 'obj11\_it\_%' ESCAPE '\'`, + `SELECT nspname FROM pg_namespace WHERE nspname LIKE 'obj11\_it\_%' ESCAPE '\'`, + } { + rows, err := pool.Query(queryCtx, query) + if err != nil { + return nil, err + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + return nil, err + } + names = append(names, name) + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, err + } + rows.Close() + } + return names, nil +} + +func obj11RunDeltaAudit(inner func() int) int { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return inner() + } + code := 0 + before, err := obj11SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-11 delta audit: snapshot before run: %v\n", err) + code = 1 + } else if len(before) != 0 { + fmt.Fprintf(os.Stderr, "pg-sql-11 delta audit: %d obj11_it_* resources exist before the run: %v\n", len(before), before) + code = 1 + } + if innerCode := inner(); innerCode != 0 { + code = innerCode + } + survivors, err := obj11SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-11 delta audit: survivor scan: %v\n", err) + code = 1 + } else if len(survivors) != 0 { + fmt.Fprintf(os.Stderr, "pg-sql-11 delta audit: %d obj11_it_* resources survive: %v\n", len(survivors), survivors) + code = 1 + } + return code +} diff --git a/internal/dataframe/compiler/render/postgres/pivot.go b/internal/dataframe/compiler/render/postgres/pivot.go new file mode 100644 index 000000000..55234862a --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/pivot.go @@ -0,0 +1,1583 @@ +package postgres + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + spec "github.com/calypr/loom/internal/fhir/schema" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const ( + pivotValueTypeTag = "TABLE_PIVOT_VALUE_TYPE_MISMATCH" + pivotCardinalityTag = "TABLE_PIVOT_CELL_CARDINALITY" + pivotCellMissingTag = "TABLE_PIVOT_CELL_MISSING" + pivotUnlistedTag = "TABLE_PIVOT_UNLISTED_CATEGORY" + pivotChoiceArmTag = "INVALID_CHOICE_ARM" + pivotRowIDColumn = "__loom_row_id" + pivotIdentityKind = "GROUPED_PIVOT" +) + +var pivotValueTypeOf = map[string]string{ + "STRING": "string", + "INTEGER": "number", + "DECIMAL": "number", + "BOOLEAN": "boolean", +} + +var pivotDuplicatePolicies = map[string]bool{"ERROR": true, "SUM": true, "MIN": true, "MAX": true} +var pivotMissingPolicies = map[string]bool{"NULL": true, "ERROR": true} + +func RenderGroupedPivot(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + if err := validateGroupedPivotPlanSurface(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return renderGroupedPivotPhysicalPlan(plan, storage) +} + +func RenderGroupedCodedPivot(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + return renderGroupedCodedPivotPhysicalPlan(plan, storage) +} + +func groupedPivotStagePlanMentionsSurface(plan ir.PhysicalPlan) bool { + if plan.StageSequence == nil { + return false + } + for _, stage := range plan.StageSequence.Stages { + if stage.Kind == ir.PhysicalStagePivotOp && stage.GroupedPivot != nil && stage.GroupedPivot.CodedCorrelation != nil { + return true + } + } + return false +} + +func pivotDollar(index int) string { return "$" + strconv.Itoa(index) } + +func pivotAQLNotNull(expr string) string { + return "(" + expr + " IS NOT NULL AND " + expr + " <> 'null'::jsonb)" +} + +func pivotScalarText(expr string) string { + return "(" + expr + " #>> '{}')" +} + +func pivotSelectorOK(selector spec.Selector) error { + if len(selector.Steps) == 0 { + return fmt.Errorf("%w: selector is empty", ErrUnsupportedPostgresPlan) + } + for index, step := range selector.Steps { + if step.Field == "" || !payloadPathSegmentPattern.MatchString(step.Field) { + return fmt.Errorf("%w: selector step %d %q is not renderer-safe", ErrUnsupportedPostgresPlan, index, step.Field) + } + if step.Iterate && step.Index != nil { + return fmt.Errorf("%w: selector step %d cannot both iterate and index", ErrUnsupportedPostgresPlan, index) + } + if step.Index != nil && *step.Index < 0 { + return fmt.Errorf("%w: selector step %d indexes below zero", ErrUnsupportedPostgresPlan, index) + } + } + if selector.Filter != nil { + if selector.Filter.Field == "" || !payloadPathSegmentPattern.MatchString(selector.Filter.Field) { + return fmt.Errorf("%w: selector CONTAINS filter property is not renderer-safe", ErrUnsupportedPostgresPlan) + } + } + return nil +} + +func (r *planRenderer) pivotSelectorChain(base string, selector spec.Selector, name string) (from string, item string, ords []string, where []string, err error) { + if err = pivotSelectorOK(selector); err != nil { + return "", "", nil, nil, err + } + steps := selector.Steps + root := name + "0" + from = " CROSS JOIN LATERAL (VALUES (" + base + ")) AS " + root + "(v)" + current := root + ".v" + where = make([]string, 0, 1) + memberParam := func(field string) string { + return pivotDollar(r.textParameter(field, "pivot:prop:"+field)) + } + for _, step := range steps[:len(steps)-1] { + property := memberParam(step.Field) + isObject := "jsonb_typeof(" + current + ") = 'object'" + has := "jsonb_exists(" + current + ", " + property + "::text)" + switch { + case step.Index != nil: + indexParameter := pivotDollar(r.addParameter( + command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(*step.Index)}, "")) + expr := "CASE WHEN " + isObject + " AND " + has + + " AND jsonb_typeof(" + current + " -> " + property + "::text) = 'array'" + + " THEN (" + current + " -> " + property + "::text) -> " + indexParameter + "::int END" + alias := name + "_s" + from += " JOIN LATERAL (SELECT " + expr + " AS v) AS " + alias + "(v) ON " + pivotAQLNotNull(alias+".v") + current = alias + ".v" + case step.Iterate: + guard := "CASE WHEN " + isObject + " AND " + has + + " AND jsonb_typeof(" + current + " -> " + property + "::text) = 'array'" + + " THEN " + current + " -> " + property + "::text ELSE '[]'::jsonb END" + alias := name + "_s" + from += " CROSS JOIN LATERAL jsonb_array_elements(" + guard + ") WITH ORDINALITY AS " + alias + "(v, ord)" + current = alias + ".v" + ords = append(ords, alias+".ord") + default: + expr := "CASE WHEN " + isObject + " AND " + has + + " THEN " + current + " -> " + property + "::text END" + alias := name + "_s" + from += " JOIN LATERAL (SELECT " + expr + " AS v) AS " + alias + "(v) ON " + pivotAQLNotNull(alias+".v") + current = alias + ".v" + } + name += "x" + } + if selector.Filter != nil { + property := memberParam(selector.Filter.Field) + needle := pivotDollar(r.textParameter(selector.Filter.Needle, "pivot:needle:"+selector.Filter.Needle)) + hay := "CASE WHEN jsonb_typeof(" + current + ") = 'object'" + + " AND jsonb_typeof(" + current + " -> " + property + "::text) = 'string'" + + " THEN " + current + " ->> " + property + "::text ELSE '' END" + where = append(where, "(length("+needle+") = 0 OR strpos("+hay+", "+needle+") > 0)") + } + last := steps[len(steps)-1] + lastProperty := memberParam(last.Field) + lastIsObject := "jsonb_typeof(" + current + ") = 'object'" + lastHas := "jsonb_exists(" + current + ", " + lastProperty + "::text)" + alias := name + "_f" + switch { + case last.Iterate: + guard := "CASE WHEN " + lastIsObject + " AND " + lastHas + + " AND jsonb_typeof(" + current + " -> " + lastProperty + "::text) = 'array'" + + " THEN " + current + " -> " + lastProperty + "::text ELSE '[]'::jsonb END" + from += " CROSS JOIN LATERAL jsonb_array_elements(" + guard + ") WITH ORDINALITY AS " + alias + "(v, ord)" + item = alias + ".v" + ords = append(ords, alias+".ord") + case last.Index != nil: + indexParameter := pivotDollar(r.addParameter( + command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(*last.Index)}, "")) + expr := "CASE WHEN " + lastIsObject + " AND " + lastHas + + " AND jsonb_typeof(" + current + " -> " + lastProperty + "::text) = 'array'" + + " THEN (" + current + " -> " + lastProperty + "::text) -> " + indexParameter + "::int END" + from += " JOIN LATERAL (SELECT " + expr + " AS v) AS " + alias + "(v) ON " + pivotAQLNotNull(alias+".v") + item = alias + ".v" + default: + expr := "CASE WHEN " + lastIsObject + " AND " + lastHas + + " THEN " + current + " -> " + lastProperty + "::text END" + from += " JOIN LATERAL (SELECT " + expr + " AS v) AS " + alias + "(v) ON " + pivotAQLNotNull(alias+".v") + item = alias + ".v" + } + return from, item, ords, where, nil +} + +func pivotWhereJoin(where []string) string { + if len(where) == 0 { + return "" + } + return " WHERE " + strings.Join(where, " AND ") +} + +func (r *planRenderer) pivotSelectorArray(base string, selector spec.Selector, name string) (string, error) { + from, item, ords, where, err := r.pivotSelectorChain(base, selector, name) + if err != nil { + return "", err + } + orderBy := "" + if len(ords) != 0 { + orderBy = " ORDER BY " + strings.Join(ords, ", ") + } + return "(SELECT COALESCE(jsonb_agg(" + item + orderBy + "), '[]'::jsonb)" + + " FROM (SELECT 1) AS " + name + "_hub" + from + pivotWhereJoin(where) + ")", nil +} + +func (r *planRenderer) pivotIdentitySQL(constructionID string, kinds []string, keyExprs []string) string { + identity := "'[' || " + quoteSQLLiteral(`"`+pivotIdentityKind+`"`) + + " || ',' || to_jsonb(" + pivotDollar(r.textParameter(constructionID, "")) + "::text)" + for index, key := range keyExprs { + kind := kinds[index] + identity += " || ',' || '[' || to_jsonb(" + pivotDollar(r.textParameter(kind, "pivot:kind:"+kind)) + "::text)" + + " || ',' || COALESCE(" + key + "::text, 'null') || ']'" + } + return identity + " || ']'" +} + +func validateGroupedPivotPlanSurface(plan ir.PhysicalPlan) error { + if len(plan.Operations) == 0 { + return fmt.Errorf("%w: plan has no operations", ErrUnsupportedPostgresPlan) + } + if plan.Operations[0].Kind != ir.PhysicalRootScanOp || plan.Operations[0].RootScan == nil { + return fmt.Errorf("%w: the pivot slice leads with exactly one ROOT_SCAN at position 0", ErrUnsupportedPostgresPlan) + } + if plan.StageSequence != nil { + return fmt.Errorf("%w: a pivot combined with a stage sequence has no PostgreSQL rendering (staged compositions are pg-sql-16)", ErrUnsupportedPostgresPlan) + } + if plan.PreviewSourceWindowByRootID { + return fmt.Errorf("%w: the plan-level preview source window requires Pivot-terminal windowing contracts that are not registered for the scalar pivot (pg-sql-16)", ErrUnsupportedPostgresPlan) + } + var pivot *ir.PhysicalGroupedPivot + pivotSeen, sortSeen, limitSeen, returnSeen := false, false, false, false + for index, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalRootScanOp: + if index != 0 { + return fmt.Errorf("%w: the pivot slice leads with exactly one ROOT_SCAN at position 0, got one at %d", + ErrUnsupportedPostgresPlan, index) + } + if operation.RootScan == nil { + return fmt.Errorf("%w: ROOT_SCAN is missing its payload", ErrUnsupportedPostgresPlan) + } + if err := validatePivotRootScan(plan, *operation.RootScan); err != nil { + return err + } + case ir.PhysicalFilterOp: + if pivotSeen { + return fmt.Errorf("%w: FILTER must precede the pivot; filtering reshaped cells is a registered pg-sql-16 composition", ErrUnsupportedPostgresPlan) + } + if operation.Filter == nil { + return fmt.Errorf("%w: FILTER is missing its payload", ErrUnsupportedPostgresPlan) + } + case ir.PhysicalDerivedLetOp: + if pivotSeen { + return fmt.Errorf("%w: DERIVED_LET must precede the pivot", ErrUnsupportedPostgresPlan) + } + if operation.DerivedLet == nil { + return fmt.Errorf("%w: DERIVED_LET is missing its payload", ErrUnsupportedPostgresPlan) + } + case ir.PhysicalGroupedPivotOp: + if pivotSeen { + return fmt.Errorf("%w: the pivot slice registers exactly one reshape; a second pivot has no PostgreSQL rendering (multiple reshapes are construction stage sequences, which stay refused)", ErrUnsupportedPostgresPlan) + } + if operation.GroupedPivot == nil { + return fmt.Errorf("%w: GROUPED_PIVOT is missing its payload", ErrUnsupportedPostgresPlan) + } + pivot = operation.GroupedPivot + pivotSeen = true + case ir.PhysicalSortOp: + if !pivotSeen || sortSeen || limitSeen || returnSeen { + return fmt.Errorf("%w: the reshaped execution SORT must be at most one SORT directly after the pivot and before any LIMIT or RETURN", ErrUnsupportedPostgresPlan) + } + if operation.Sort == nil { + return fmt.Errorf("%w: SORT is missing its payload", ErrUnsupportedPostgresPlan) + } + if err := validatePivotSortKeys(*operation.Sort, *pivot); err != nil { + return err + } + sortSeen = true + case ir.PhysicalLimitOp: + if !pivotSeen || limitSeen || returnSeen { + return fmt.Errorf("%w: the reshaped execution LIMIT must be at most one LIMIT after the pivot's SORT and before RETURN", ErrUnsupportedPostgresPlan) + } + if !sortSeen { + return fmt.Errorf("%w: a LIMIT after GROUPED_PIVOT requires the stable row identity SORT first (the ir reshaped-window guard's contract, re-read at the renderer)", ErrUnsupportedPostgresPlan) + } + if operation.Limit == nil { + return fmt.Errorf("%w: LIMIT is missing its payload", ErrUnsupportedPostgresPlan) + } + if value, ok := plan.BindVars[operation.Limit.BindKey].(int); !ok || value <= 0 { + return fmt.Errorf("%w: reshaped LIMIT bind %q must be a positive int", ErrUnsupportedPostgresPlan, operation.Limit.BindKey) + } + limitSeen = true + case ir.PhysicalReturnOp: + if !pivotSeen { + return fmt.Errorf("%w: RETURN cannot run before the pivot it projects", ErrUnsupportedPostgresPlan) + } + if returnSeen { + return fmt.Errorf("%w: the pivot slice registers exactly one terminal RETURN", ErrUnsupportedPostgresPlan) + } + if index != len(plan.Operations)-1 { + return fmt.Errorf("%w: the pivot slice requires a terminal RETURN after its pivot", ErrUnsupportedPostgresPlan) + } + if operation.Return == nil { + return fmt.Errorf("%w: RETURN is missing its payload", ErrUnsupportedPostgresPlan) + } + if err := validatePivotReturn(*operation.Return, *pivot); err != nil { + return err + } + returnSeen = true + default: + return fmt.Errorf("%w: operation %q cannot coexist with GROUPED_PIVOT in the registered slice; filters beyond the canonical scope, traversals, unnests, sets, subplans, and every other shape belong to the read spine or the pg-sql-16 composition capability", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + if !pivotSeen { + return fmt.Errorf("%w: the pivot slice requires exactly one GROUPED_PIVOT between ROOT_SCAN and RETURN", ErrUnsupportedPostgresPlan) + } + if !returnSeen { + return fmt.Errorf("%w: the pivot slice requires a terminal RETURN after its pivot", ErrUnsupportedPostgresPlan) + } + return validateGroupedPivotPayload(plan, *pivot, plan.Operations[0].RootScan.Variable) +} + +func validatePivotRootScan(plan ir.PhysicalPlan, scan ir.PhysicalRootScan) error { + if scan.Population != nil || scan.CohortSource != nil || scan.PageCandidateSeed != nil { + return fmt.Errorf("%w: population, cohort, and candidate-seed roots are not registered for PostgreSQL; only the direct root collection scan has a rendered contract", + ErrUnsupportedPostgresPlan) + } + if strings.TrimSpace(scan.Variable) == "" { + return fmt.Errorf("%w: ROOT_SCAN must name its row variable", ErrUnsupportedPostgresPlan) + } + if strings.TrimSpace(scan.CollectionBindKey) == "" { + return fmt.Errorf("%w: ROOT_SCAN must name the bind carrying the single resource type", ErrUnsupportedPostgresPlan) + } + if resourceType, ok := plan.BindVars[scan.CollectionBindKey].(string); !ok || strings.TrimSpace(resourceType) == "" { + return fmt.Errorf("%w: root collection bind %q must name one resource type as a non-empty string", + ErrUnsupportedPostgresPlan, scan.CollectionBindKey) + } + return nil +} + +func validatePivotSortKeys(sort ir.PhysicalSort, pivot ir.PhysicalGroupedPivot) error { + if len(sort.Keys) != 1 || sort.Keys[0].BindKey != "" || + sort.Keys[0].Variable != pivot.OutputRowVariable || + len(sort.Keys[0].Path) != 1 || sort.Keys[0].Path[0] != pivotRowIDColumn { + return fmt.Errorf("%w: the reshaped pivot SORT must sort on %s.%s exactly", + ErrUnsupportedPostgresPlan, pivot.OutputRowVariable, pivotRowIDColumn) + } + return nil +} + +func pivotLegalOutputNames(pivot ir.PhysicalGroupedPivot) map[string]bool { + legal := map[string]bool{pivotRowIDColumn: true} + for _, key := range pivot.GroupKeys { + name := key.Output + if name == "" { + name = key.Column + } + legal[name] = true + } + for _, category := range pivot.Categories { + legal[category.Output] = true + } + if pivot.UnlistedCategoryPolicy == "EXCLUDE_WITH_EVIDENCE" { + legal[pivot.UnlistedEvidenceColumn] = true + } + return legal +} + +func validatePivotReturn(ret ir.PhysicalReturn, pivot ir.PhysicalGroupedPivot) error { + if len(ret.Projections) == 0 { + return fmt.Errorf("%w: the pivot slice requires a terminal RETURN with at least one projection", ErrUnsupportedPostgresPlan) + } + legal := pivotLegalOutputNames(pivot) + seen := map[string]bool{} + for _, projection := range ret.Projections { + if !legal[projection.Name] { + return fmt.Errorf("%w: pivot RETURN projection %q is not a registered pivot output", + ErrUnsupportedPostgresPlan, projection.Name) + } + if seen[projection.Name] { + return fmt.Errorf("%w: pivot RETURN projects %q twice", ErrUnsupportedPostgresPlan, projection.Name) + } + seen[projection.Name] = true + if projection.Expression != nil || projection.Presence != nil || projection.PresenceOutput { + return fmt.Errorf("%w: pivot RETURN projection %q must be a direct value read", + ErrUnsupportedPostgresPlan, projection.Name) + } + if projection.Value.Variable != pivot.OutputRowVariable { + return fmt.Errorf("%w: pivot RETURN projection %q must read the pivot output row %q, got %q", + ErrUnsupportedPostgresPlan, projection.Name, pivot.OutputRowVariable, projection.Value.Variable) + } + if len(projection.Value.Path) != 1 || projection.Value.Path[0] != projection.Name { + return fmt.Errorf("%w: pivot RETURN projection %q must be the single-name read %s.%s", + ErrUnsupportedPostgresPlan, projection.Name, pivot.OutputRowVariable, projection.Name) + } + if projection.Value.BindKey != "" { + return fmt.Errorf("%w: pivot RETURN projection %q must be a row read, not a bind", + ErrUnsupportedPostgresPlan, projection.Name) + } + } + return nil +} + +func validateGroupedPivotPayload(plan ir.PhysicalPlan, pivot ir.PhysicalGroupedPivot, rootVariable string) error { + if pivot.CodedCorrelation != nil || len(pivot.CodedCategories) != 0 || strings.TrimSpace(pivot.CodedSourceVariable) != "" { + return fmt.Errorf("%w: coded Pivot fields on a spine operation have no rendering (the AQL producer census keeps them on the construction stage; ir/physical_table_shape.go:36-38)", + ErrUnsupportedPostgresPlan) + } + if len(pivot.RowValues) != 0 { + return fmt.Errorf("%w: pivot row values have no PostgreSQL rendering (row values are the pg-sql-13 capability; the spine producer lower/recipe_table_reshape.go never emits them)", + ErrUnsupportedPostgresPlan) + } + if pivot.RootContributorInputColumn != "" || pivot.RootContributorOutputColumn != "" || pivot.RootContributorVariable != "" { + return fmt.Errorf("%w: pivot root-contributor columns have no PostgreSQL rendering (the construction-only producer lower/recipe_construction.go:1291-1308)", + ErrUnsupportedPostgresPlan) + } + if len(pivot.Categories) == 0 { + return fmt.Errorf("%w: the registered slice requires at least one frozen pivot category", ErrUnsupportedPostgresPlan) + } + if !pivotDuplicatePolicies[pivot.DuplicatePolicy] { + return fmt.Errorf("%w: pivot duplicate policy %q is not registered", ErrUnsupportedPostgresPlan, pivot.DuplicatePolicy) + } + if !pivotMissingPolicies[pivot.MissingCellPolicy] { + return fmt.Errorf("%w: pivot missing-cell policy %q is not registered", ErrUnsupportedPostgresPlan, pivot.MissingCellPolicy) + } + if pivot.UnlistedCategoryPolicy != "ERROR" && pivot.UnlistedCategoryPolicy != "EXCLUDE_WITH_EVIDENCE" { + return fmt.Errorf("%w: pivot unlisted-category policy %q is not registered for the ordinary pivot (IGNORE is coded-only, ir/physical_table_shape.go:65-73)", + ErrUnsupportedPostgresPlan, pivot.UnlistedCategoryPolicy) + } + if _, ok := pivotValueTypeOf[pivot.CategoryType]; !ok { + return fmt.Errorf("%w: pivot category type %q is not a registered table scalar kind", ErrUnsupportedPostgresPlan, pivot.CategoryType) + } + if _, ok := pivotValueTypeOf[pivot.ValueType]; !ok { + return fmt.Errorf("%w: pivot value type %q is not a registered table scalar kind", ErrUnsupportedPostgresPlan, pivot.ValueType) + } + if pivot.DuplicatePolicy != "ERROR" && pivot.ValueType != "INTEGER" && pivot.ValueType != "DECIMAL" { + return fmt.Errorf("%w: pivot reducer %q requires a numeric value column (the producer gate, lower/recipe_table_reshape.go:126-130)", + ErrUnsupportedPostgresPlan, pivot.DuplicatePolicy) + } + if (pivot.UnlistedCategoryPolicy == "EXCLUDE_WITH_EVIDENCE") != (pivot.UnlistedEvidenceColumn != "") { + return fmt.Errorf("%w: the unlisted evidence column exists exactly under EXCLUDE_WITH_EVIDENCE (ir/physical_table_shape.go:74-80)", + ErrUnsupportedPostgresPlan) + } + if value, ok := plan.BindVars[pivot.ConstructionIDBindKey].(string); !ok || value == "" { + return fmt.Errorf("%w: pivot construction ID bind %q must be a non-empty string", + ErrUnsupportedPostgresPlan, pivot.ConstructionIDBindKey) + } + missingCategory := false + outputs := map[string]bool{} + for index, category := range pivot.Categories { + if outputs[category.Output] { + return fmt.Errorf("%w: pivot category %d repeats output %q", ErrUnsupportedPostgresPlan, index, category.Output) + } + outputs[category.Output] = true + switch category.MatchKind { + case ir.PhysicalPivotCategoryValueMatch: + if category.ValueBindKey == "" { + return fmt.Errorf("%w: VALUE category %d requires a frozen value bind", ErrUnsupportedPostgresPlan, index) + } + typeOf, ok := pivotValueTypeOf[category.ValueKind] + if !ok { + return fmt.Errorf("%w: VALUE category %d kind %q is not a registered table scalar kind", + ErrUnsupportedPostgresPlan, index, category.ValueKind) + } + if want := pivotValueTypeOf[pivot.CategoryType]; typeOf != want { + return fmt.Errorf("%w: VALUE category %d kind %q disagrees with the frozen category type %q", + ErrUnsupportedPostgresPlan, index, category.ValueKind, pivot.CategoryType) + } + if err := validatePivotCategoryBind(plan.BindVars, category); err != nil { + return err + } + case ir.PhysicalPivotCategoryNullMatch: + if category.ValueBindKey != "" { + return fmt.Errorf("%w: NULL category %d must not carry a value bind", ErrUnsupportedPostgresPlan, index) + } + case ir.PhysicalPivotCategoryMissingMatch: + missingCategory = true + if category.ValueBindKey != "" { + return fmt.Errorf("%w: MISSING category %d must not carry a value bind", ErrUnsupportedPostgresPlan, index) + } + default: + return fmt.Errorf("%w: pivot category %d match kind %q is not registered", + ErrUnsupportedPostgresPlan, index, string(category.MatchKind)) + } + } + if missingCategory && pivot.CategoryPresence == nil && !pivot.CategoryPresenceFromInput { + return fmt.Errorf("%w: a MISSING pivot category requires a preserved category presence contract (ir/physical_table_shape.go, re-read)", ErrUnsupportedPostgresPlan) + } + if pivot.CategoryPresence != nil && pivot.CategoryPresenceFromInput { + return fmt.Errorf("%w: category presence cannot be both contract-computed and input-companion", ErrUnsupportedPostgresPlan) + } + if pivot.CategoryPresence != nil { + if len(pivot.CategoryPresence.Paths) == 0 { + return fmt.Errorf("%w: the category presence contract must prove at least one path", ErrUnsupportedPostgresPlan) + } + source := pivot.CategoryPresence.Source + if source.Variable != "" && source.Variable != rootVariable { + return fmt.Errorf("%w: the category presence contract must prove over the pivot's scanned root row %q, got %q", + ErrUnsupportedPostgresPlan, rootVariable, source.Variable) + } + if len(source.Path) != 0 && !(len(source.Path) == 1 && source.Path[0] == "payload") { + return fmt.Errorf("%w: the category presence contract source must be the root document, got path %v", + ErrUnsupportedPostgresPlan, source.Path) + } + } + if pivot.CategoryPresenceFromInput { + companion := false + for _, projection := range pivot.InputProjections { + if projection.Name == pivot.CategoryPresenceColumn { + companion = true + } + } + if pivot.CategoryPresenceColumn == "" || !companion { + return fmt.Errorf("%w: CategoryPresenceFromInput requires the companion column %q among the input projections", + ErrUnsupportedPostgresPlan, pivot.CategoryPresenceColumn) + } + } + for _, key := range pivot.GroupKeys { + if _, ok := pivotValueTypeOf[key.Kind]; !ok { + return fmt.Errorf("%w: pivot group key %q kind %q is not a registered table scalar kind", + ErrUnsupportedPostgresPlan, key.Column, key.Kind) + } + } + return validatePivotInputProjections(pivot, rootVariable) +} + +func validatePivotCategoryBind(bindVars map[string]any, category ir.PhysicalGroupedPivotCategory) error { + raw, ok := bindVars[category.ValueBindKey] + if !ok { + return fmt.Errorf("%w: pivot category value bind %q is missing", ErrUnsupportedPostgresPlan, category.ValueBindKey) + } + switch category.ValueKind { + case "STRING": + if _, ok := raw.(string); !ok { + return fmt.Errorf("%w: pivot category value bind %q must be a string for kind STRING", + ErrUnsupportedPostgresPlan, category.ValueBindKey) + } + case "INTEGER": + switch raw.(type) { + case int64, int: + default: + return fmt.Errorf("%w: pivot category value bind %q must be an integer for kind INTEGER", + ErrUnsupportedPostgresPlan, category.ValueBindKey) + } + case "DECIMAL": + switch raw.(type) { + case float64, int64, int: + default: + return fmt.Errorf("%w: pivot category value bind %q must be numeric for kind DECIMAL", + ErrUnsupportedPostgresPlan, category.ValueBindKey) + } + case "BOOLEAN": + if _, ok := raw.(bool); !ok { + return fmt.Errorf("%w: pivot category value bind %q must be a boolean for kind BOOLEAN", + ErrUnsupportedPostgresPlan, category.ValueBindKey) + } + } + return nil +} + +func pivotConsumedInputNames(pivot ir.PhysicalGroupedPivot) map[string]bool { + consumed := map[string]bool{pivot.CategoryColumn: true, pivot.ValueColumn: true} + for _, key := range pivot.GroupKeys { + consumed[key.Column] = true + } + if pivot.CategoryPresenceFromInput { + consumed[pivot.CategoryPresenceColumn] = true + } + return consumed +} + +func pivotInputReadShape(projection ir.PhysicalProjection, rootVariable string) (path []string, behavior ir.PhysicalNullBehavior, err error) { + if projection.Expression != nil { + path, err := validateSelectorExtraction(projection) + if err != nil { + return nil, "", err + } + switch projection.Expression.NullBehavior { + case ir.PhysicalPreserveNull, ir.PhysicalOmitNulls, ir.PhysicalEmptyOnNull: + default: + return nil, "", fmt.Errorf("%w: pivot input projection %q null behavior %q has no registered extraction form", + ErrUnsupportedPostgresPlan, projection.Name, string(projection.Expression.NullBehavior)) + } + if projection.Expression.Extract != nil && projection.Expression.Extract.Source.Variable != rootVariable { + return nil, "", fmt.Errorf("%w: pivot input projection %q must extract from the scanned root row %q, got %q", + ErrUnsupportedPostgresPlan, projection.Name, rootVariable, projection.Expression.Extract.Source.Variable) + } + return path, projection.Expression.NullBehavior, nil + } + if projection.Value.Variable != rootVariable || len(projection.Value.Path) == 0 || projection.Value.BindKey != "" { + return nil, "", fmt.Errorf("%w: pivot input projection %q must be a direct document read of %q or a registered selector extraction", + ErrUnsupportedPostgresPlan, projection.Name, rootVariable) + } + return projection.Value.Path, ir.PhysicalPreserveNull, nil +} + +func validatePivotInputProjections(pivot ir.PhysicalGroupedPivot, rootVariable string) error { + consumed := pivotConsumedInputNames(pivot) + found := map[string]bool{} + for _, projection := range pivot.InputProjections { + if !consumed[projection.Name] { + continue + } + if found[projection.Name] { + return fmt.Errorf("%w: pivot input column %q is projected twice", ErrUnsupportedPostgresPlan, projection.Name) + } + found[projection.Name] = true + if _, _, err := pivotInputReadShape(projection, rootVariable); err != nil { + return err + } + } + for name := range consumed { + if !found[name] { + return fmt.Errorf("%w: the pivot reads input column %q but no input projection carries it", + ErrUnsupportedPostgresPlan, name) + } + } + return nil +} + +func renderGroupedPivotPhysicalPlan(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + if err := storage.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := validateGroupedPivotPlanSurface(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := plan.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("validate physical plan: %w", err) + } + if plan.Engine != "" && plan.Engine != ir.PhysicalEngineAQL { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: engine %q has no PostgreSQL source; the source graph IR is backend-neutral and ClickHouse plans are not PostgreSQL-rendered", + ErrUnsupportedPostgresPlan, string(plan.Engine)) + } + if plan.ClickHouseCombine != nil || plan.ClickHousePrefix != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: ClickHouse combine/prefix payloads have no PostgreSQL rendering", ErrUnsupportedPostgresPlan) + } + if len(plan.DeferredExpressionLets) != 0 { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: completed plans must not carry deferred expression lets", ErrUnsupportedPostgresPlan) + } + renderer := &planRenderer{ + storage: storage, + rows: map[string]rowScope{}, + parameters: map[string]int{}, + canonicalAuthLets: map[string]string{}, + } + if err := renderer.loadScope(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + sql, err := renderer.renderGroupedPivotStatement(plan) + if err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return command.NewRenderedPostgresPhysicalPlan(sql, renderer.args, renderer.outputColumns) +} + +func (r *planRenderer) renderGroupedPivotStatement(plan ir.PhysicalPlan) (string, error) { + var pivot *ir.PhysicalGroupedPivot + sortSeen := false + limitBind := "" + for _, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalGroupedPivotOp: + pivot = operation.GroupedPivot + case ir.PhysicalSortOp: + sortSeen = true + case ir.PhysicalLimitOp: + limitBind = operation.Limit.BindKey + } + } + rootVariable := plan.Operations[0].RootScan.Variable + + if err := r.renderRootScan(plan, *plan.Operations[0].RootScan); err != nil { + return "", err + } + for _, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalFilterOp: + canonical, err := r.acceptCanonicalScopeFilter(operation.Filter) + if err != nil { + return "", err + } + if !canonical { + return "", fmt.Errorf("%w: a non-canonical FILTER cannot coexist with the registered pivot (probe-attested renders-at-AQL divergence, pg-sql-16)", + ErrUnsupportedPostgresPlan) + } + case ir.PhysicalDerivedLetOp: + canonical, err := r.acceptCanonicalAuthLet(operation.DerivedLet) + if err != nil { + return "", err + } + if !canonical { + return "", fmt.Errorf("%w: operation %q is not part of the registered pivot source surface", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + } + + fromText := r.storage.Resource.Identifier().SQLText() + " AS " + r.rootAlias + for _, join := range r.joins { + fromText += " " + join + } + for _, lateral := range r.laterals { + fromText += " " + lateral + } + if len(r.where) > 0 { + fromText += " WHERE " + strings.Join(r.where, " AND ") + } + var returnOp *ir.PhysicalReturn + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalReturnOp { + returnOp = operation.Return + } + } + projectionNames := make([]string, 0, len(returnOp.Projections)) + for _, projection := range returnOp.Projections { + projectionNames = append(projectionNames, projection.Name) + } + region, err := r.renderGroupedPivotRegion(plan, pivot, "pv_", projectionNames, pivotRowIDColumn, + fromText, + func(projection ir.PhysicalProjection) (string, ir.PhysicalNullBehavior, error) { + path, behavior, err := pivotInputReadShape(projection, rootVariable) + if err != nil { + return "", "", err + } + if projection.Expression != nil { + extraction, err := r.documentPathExpression(r.rootAlias, path) + if err != nil { + return "", "", err + } + switch behavior { + case ir.PhysicalOmitNulls: + return "NULLIF((" + extraction + "), 'null'::jsonb)", behavior, nil + case ir.PhysicalEmptyOnNull: + return "COALESCE(NULLIF((" + extraction + "), 'null'::jsonb), '\"\"'::jsonb)", behavior, nil + default: + return "(" + extraction + ")", behavior, nil + } + } + extraction, err := r.documentPathFromExpression(r.documentColumn(r.rootAlias), path) + if err != nil { + return "", "", err + } + return "(" + extraction + ")", behavior, nil + }, + func(paths [][]string) (string, error) { + proofs := make([]string, 0, len(paths)) + for _, path := range paths { + proof, err := r.propertyPresenceExpression(r.rootAlias, path) + if err != nil { + return "", err + } + proofs = append(proofs, proof) + } + return "CASE WHEN " + strings.Join(proofs, " OR ") + " THEN 'true'::jsonb ELSE 'false'::jsonb END", nil + }) + if err != nil { + return "", err + } + r.outputColumns = region.names + statement := "WITH " + strings.Join(region.ctes, ", ") + " " + region.final + statement += " ORDER BY " + pivotWindowOrder(region.keySlots, region.identity, sortSeen) + if limitBind != "" { + limit := plan.BindVars[limitBind].(int) + index := r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(limit)}, "") + statement += " LIMIT " + pivotDollar(index) + } + return statement, nil +} + +type pivotRegionResult struct { + ctes []string + final string + keySlots []string + identity string + names []string +} + +func (r *planRenderer) renderGroupedPivotRegion( + plan ir.PhysicalPlan, + pivot *ir.PhysicalGroupedPivot, + prefix string, + projectionNames []string, + identityColumn string, + fromText string, + readFor func(projection ir.PhysicalProjection) (string, ir.PhysicalNullBehavior, error), + presenceProof func(paths [][]string) (string, error), +) (pivotRegionResult, error) { + consumed := pivotConsumedInputNames(*pivot) + reads := map[string]string{} + slotExpressions := make([]string, 0, len(pivot.InputProjections)) + addSlot := func(expression string) string { + slot := prefix + strconv.Itoa(len(slotExpressions)) + slotExpressions = append(slotExpressions, expression+" AS "+slot) + return "p." + slot + } + categoryBehavior := ir.PhysicalPreserveNull + for _, projection := range pivot.InputProjections { + if !consumed[projection.Name] { + continue + } + expression, behavior, err := readFor(projection) + if err != nil { + return pivotRegionResult{}, err + } + reads[projection.Name] = addSlot(expression) + if projection.Name == pivot.CategoryColumn { + categoryBehavior = behavior + } + } + pvCat, err := pivotSlotRead(reads, pivot.CategoryColumn) + if err != nil { + return pivotRegionResult{}, err + } + pvVal, err := pivotSlotRead(reads, pivot.ValueColumn) + if err != nil { + return pivotRegionResult{}, err + } + + present := "" + switch { + case pivot.CategoryPresence != nil: + if presenceProof == nil { + return pivotRegionResult{}, fmt.Errorf("%w: a document-path category presence needs the source document; a staged pivot reads typed intermediate columns, so the presence must ride a materialized presence companion (CategoryPresenceFromInput)", + ErrUnsupportedPostgresPlan) + } + presence, err := presenceProof(pivot.CategoryPresence.Paths) + if err != nil { + return pivotRegionResult{}, err + } + present = addSlot(presence) + " = 'true'::jsonb" + case pivot.CategoryPresenceFromInput: + companion, err := pivotSlotRead(reads, pivot.CategoryPresenceColumn) + if err != nil { + return pivotRegionResult{}, err + } + present = "COALESCE(" + companion + " = 'true'::jsonb, false)" + default: + switch categoryBehavior { + case ir.PhysicalOmitNulls: + present = pvCat + " IS NOT NULL" + default: + present = "true" + } + } + + valueTypeOf := quoteSQLLiteral(pivotValueTypeOf[pivot.ValueType]) + typedConjunct := pvVal + " IS NOT NULL AND " + pvVal + " <> 'null'::jsonb" + + matchPredicates := make([]string, 0, len(pivot.Categories)) + for _, category := range pivot.Categories { + switch category.MatchKind { + case ir.PhysicalPivotCategoryValueMatch: + raw, err := pivotCategoryRaw(plan.BindVars, category) + if err != nil { + return pivotRegionResult{}, err + } + valueParameter := pivotDollar(r.jsonbParameter(raw)) + matchPredicates = append(matchPredicates, + "("+present+" AND jsonb_typeof("+pvCat+") = "+quoteSQLLiteral(pivotValueTypeOf[category.ValueKind])+ + " AND "+pvCat+" = "+valueParameter+"::jsonb)") + case ir.PhysicalPivotCategoryNullMatch: + nullRead := "(" + pvCat + " IS NULL OR " + pvCat + " = 'null'::jsonb)" + if categoryBehavior == ir.PhysicalOmitNulls { + nullRead = "(" + pvCat + " IS NOT NULL AND " + pvCat + " = 'null'::jsonb)" + } + matchPredicates = append(matchPredicates, "("+present+" AND "+nullRead+")") + case ir.PhysicalPivotCategoryMissingMatch: + matchPredicates = append(matchPredicates, "NOT ("+present+")") + } + } + listed := "(" + strings.Join(matchPredicates, " OR ") + ")" + + keySlots := make([]string, 0, len(pivot.GroupKeys)) + kinds := make([]string, 0, len(pivot.GroupKeys)) + keyNames := make([]string, 0, len(pivot.GroupKeys)) + for _, key := range pivot.GroupKeys { + expr, err := pivotSlotRead(reads, key.Column) + if err != nil { + return pivotRegionResult{}, err + } + keySlots = append(keySlots, expr) + kinds = append(kinds, key.Kind) + name := key.Output + if name == "" { + name = key.Column + } + keyNames = append(keyNames, name) + } + + cellSelects := make([]string, 0, len(pivot.Categories)) + guardTerms := make([]string, 0, 2*len(pivot.Categories)) + if pivot.UnlistedCategoryPolicy == "ERROR" { + guardTerms = append(guardTerms, + "WHEN COUNT(*) FILTER (WHERE NOT COALESCE("+listed+", false)) > 0 THEN "+guardCast(pivotUnlistedTag)) + } + for _, match := range matchPredicates { + rawCount := "COUNT(*) FILTER (WHERE " + match + ")" + typedCount := "COUNT(*) FILTER (WHERE " + match + " AND " + typedConjunct + ")" + mismatchCount := "COUNT(*) FILTER (WHERE " + match + " AND " + typedConjunct + " AND jsonb_typeof(" + pvVal + ") <> " + valueTypeOf + ")" + cell := "" + switch pivot.DuplicatePolicy { + case "ERROR": + firstRaw := "(ARRAY_AGG(" + pvVal + ") FILTER (WHERE " + match + "))[1]" + cell = "COALESCE(" + firstRaw + ", 'null'::jsonb)" + if pivot.MissingCellPolicy == "ERROR" { + guardTerms = append(guardTerms, "WHEN "+rawCount+" = 0 THEN "+guardCast(pivotCellMissingTag)) + } + guardTerms = append(guardTerms, "WHEN "+rawCount+" > 1 THEN "+guardCast(pivotCardinalityTag)) + default: + safeCast := "CASE WHEN jsonb_typeof(" + pvVal + ") = " + valueTypeOf + " THEN " + + pivotScalarText(pvVal) + "::double precision END" + reducer := pivot.DuplicatePolicy + "(" + safeCast + ") FILTER (WHERE " + match + " AND " + typedConjunct + ")" + cell = "(CASE WHEN " + typedCount + " = 0 THEN 'null'::jsonb ELSE to_jsonb(" + reducer + ") END)" + if pivot.MissingCellPolicy == "ERROR" { + guardTerms = append(guardTerms, "WHEN "+rawCount+" = 0 THEN "+guardCast(pivotCellMissingTag)) + } + guardTerms = append(guardTerms, "WHEN "+mismatchCount+" > 0 THEN "+guardCast(pivotValueTypeTag)) + } + cellSelects = append(cellSelects, cell) + } + + identity := r.pivotIdentitySQL(bindStringVar(plan.BindVars, pivot.ConstructionIDBindKey), kinds, keySlots) + + expressions := map[string]string{} + for index, name := range keyNames { + expressions[name] = keySlots[index] + } + for index, category := range pivot.Categories { + expressions[category.Output] = cellSelects[index] + } + if pivot.UnlistedCategoryPolicy == "EXCLUDE_WITH_EVIDENCE" { + expressions[pivot.UnlistedEvidenceColumn] = + "to_jsonb(COUNT(*) FILTER (WHERE NOT COALESCE(" + listed + ", false)))" + } + expressions[identityColumn] = identity + selects := make([]string, 0, len(projectionNames)) + names := make([]string, 0, len(projectionNames)) + for _, name := range projectionNames { + expression, ok := expressions[name] + if !ok { + return pivotRegionResult{}, fmt.Errorf("%w: pivot output projection %q has no rendered expression", + ErrUnsupportedPostgresPlan, name) + } + selects = append(selects, expression+` AS "c`+strconv.Itoa(len(selects))+`"`) + names = append(names, name) + } + + sourceText := "SELECT " + strings.Join(slotExpressions, ", ") + " FROM " + fromText + final := "SELECT " + strings.Join(selects, ", ") + " FROM " + prefix + "src AS p" + if len(keySlots) > 0 { + final += " GROUP BY " + strings.Join(keySlots, ", ") + } else { + final += " GROUP BY ()" + } + if len(guardTerms) > 0 { + final += " HAVING (CASE " + strings.Join(guardTerms, " ") + " ELSE 1 END) = 1" + } + return pivotRegionResult{ + ctes: []string{prefix + "src AS (" + sourceText + ")"}, + final: final, + keySlots: keySlots, + identity: identity, + names: names, + }, nil +} + +func pivotSlotRead(reads map[string]string, name string) (string, error) { + expr, ok := reads[name] + if !ok { + return "", fmt.Errorf("%w: pivot input column %q has no rendered read", ErrUnsupportedPostgresPlan, name) + } + return expr, nil +} + +func pivotCategoryRaw(bindVars map[string]any, category ir.PhysicalGroupedPivotCategory) (json.RawMessage, error) { + raw, ok := bindVars[category.ValueBindKey] + if !ok { + return nil, fmt.Errorf("%w: pivot category value bind %q is missing", ErrUnsupportedPostgresPlan, category.ValueBindKey) + } + encoded, err := json.Marshal(raw) + if err != nil { + return nil, fmt.Errorf("%w: pivot category value bind %q is not JSON-encodable: %v", + ErrUnsupportedPostgresPlan, category.ValueBindKey, err) + } + return json.RawMessage(encoded), nil +} + +func pivotWindowOrder(keySlots []string, identity string, sortSeen bool) string { + if sortSeen { + return identity + ` COLLATE "C" ASC` + } + terms := make([]string, 0, 4*len(keySlots)+1) + for _, key := range keySlots { + terms = append(terms, + "(CASE WHEN "+key+" IS NULL THEN 0 WHEN jsonb_typeof("+key+") = 'boolean' THEN 1"+ + " WHEN jsonb_typeof("+key+") = 'number' THEN 2 WHEN jsonb_typeof("+key+") = 'string' THEN 3 ELSE 4 END) ASC", + "(CASE WHEN jsonb_typeof("+key+") = 'boolean' THEN "+key+"::text END) ASC", + "((CASE WHEN jsonb_typeof("+key+") = 'number' THEN "+key+" #>> '{}' END)::double precision) ASC", + "((CASE WHEN jsonb_typeof("+key+") = 'string' THEN "+key+" #>> '{}' END) COLLATE \"C\") ASC") + } + terms = append(terms, identity+` COLLATE "C" ASC`) + return strings.Join(terms, ", ") +} + +func validateGroupedCodedPivotPlanSurface(plan ir.PhysicalPlan) error { + sequence := plan.StageSequence + if sequence == nil || len(sequence.Stages) == 0 { + return fmt.Errorf("%w: the coded-pivot slice requires exactly one PIVOT stage", ErrUnsupportedPostgresPlan) + } + if len(sequence.Stages) != 1 { + return fmt.Errorf("%w: multi-stage construction sequences are the pg-sql-16 capability; the registered slice admits only the inline single-stage CODED_PIVOT", + ErrUnsupportedPostgresPlan) + } + stage := sequence.Stages[0] + if stage.Kind != ir.PhysicalStagePivotOp { + return fmt.Errorf("%w: the registered slice admits only a PIVOT stage", ErrUnsupportedPostgresPlan) + } + pivot := stage.GroupedPivot + if pivot == nil { + return fmt.Errorf("%w: PIVOT is missing its grouped-pivot payload (ir/physical_stage_validation.go re-read at the door)", ErrUnsupportedPostgresPlan) + } + if stage.Filter != nil || stage.Group != nil || stage.CodedGroup != nil || stage.CohortGroup != nil || + stage.Expand != nil || stage.Unpivot != nil || stage.RelatedSource != nil || + stage.RelatedExpand != nil || stage.RelatedField != nil || len(stage.DerivedLets) != 0 { + return fmt.Errorf("%w: the coded Pivot stage carries a second payload (ir/physical_stage_validation.go:99 re-read at the door)", ErrUnsupportedPostgresPlan) + } + if pivot.CodedCorrelation == nil || len(pivot.CodedCategories) == 0 { + return fmt.Errorf("%w: the coded-pivot door admits only coded categories", ErrUnsupportedPostgresPlan) + } + if stage.InputStageID != sequence.SourceStageID || stage.ID != sequence.FinalStageID { + return fmt.Errorf("%w: the registered slice admits only the inline first-stage CODED_PIVOT (InputStageID must equal %q and the stage must be final); the DOCUMENT re-query shape is production-unreachable", + ErrUnsupportedPostgresPlan, sequence.SourceStageID) + } + if sequence.CellTraceReturn != nil || sequence.RowLineageReturn != nil || sequence.PopulationMappingReturn != nil { + return fmt.Errorf("%w: row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route (compiler/row_lineage.go:655, cell_trace.go:54, population_mapping.go:38); pivot terminals belong to pg-sql-23 through pg-sql-26", + ErrUnsupportedPostgresPlan) + } + if plan.PreviewSourceWindowByRootID || sequence.PreviewTerminalPivotWindow { + return fmt.Errorf("%w: the plan-level preview source window and the terminal Pivot window require Pivot-terminal windowing contracts that are not registered for the coded pivot", ErrUnsupportedPostgresPlan) + } + if sequence.PreviewSourceWindowByRootID && sequence.PreviewLimitBindKey != "" { + return fmt.Errorf("%w: the active preview root-id source window combined with a preview limit is a pg-sql-16 capability and is refused, never silently rendered (probe P-14.4)", + ErrUnsupportedPostgresPlan) + } + if sequence.OutputAuthResourcePathBindKey != "" { + return fmt.Errorf("%w: the composite Combine prefix authorization path is not registered for the coded pivot (ir/physical_stage_sequence.go:25-28)", ErrUnsupportedPostgresPlan) + } + if sequence.PreviewLimitBindKey != "" { + if value, ok := plan.BindVars[sequence.PreviewLimitBindKey].(int); !ok || value <= 0 { + return fmt.Errorf("%w: coded-pivot preview limit bind %q must be a positive int", + ErrUnsupportedPostgresPlan, sequence.PreviewLimitBindKey) + } + } + if err := validateCodedPivotOperations(plan); err != nil { + return err + } + return validateCodedPivotPayload(plan, stage, pivot) +} + +func validateCodedPivotOperations(plan ir.PhysicalPlan) error { + if len(plan.Operations) == 0 { + return fmt.Errorf("%w: plan has no operations", ErrUnsupportedPostgresPlan) + } + if plan.Operations[0].Kind != ir.PhysicalRootScanOp || plan.Operations[0].RootScan == nil { + return fmt.Errorf("%w: the coded-pivot source requires exactly one leading ROOT_SCAN", ErrUnsupportedPostgresPlan) + } + sawSort, sawLimit, sawReturn := false, false, false + for index, operation := range plan.Operations { + last := index == len(plan.Operations)-1 + switch operation.Kind { + case ir.PhysicalRootScanOp: + if index != 0 { + return fmt.Errorf("%w: exactly one leading ROOT_SCAN is required", ErrUnsupportedPostgresPlan) + } + case ir.PhysicalFilterOp, ir.PhysicalDerivedLetOp: + if sawSort || sawLimit || sawReturn { + return fmt.Errorf("%w: source operations must precede the execution window and the terminal RETURN", ErrUnsupportedPostgresPlan) + } + if operation.Kind == ir.PhysicalFilterOp && operation.Filter == nil { + return fmt.Errorf("%w: operation %d: filter payload is missing", ErrUnsupportedPostgresPlan, index) + } + if operation.Kind == ir.PhysicalDerivedLetOp && operation.DerivedLet == nil { + return fmt.Errorf("%w: operation %d: derived-let payload is missing", ErrUnsupportedPostgresPlan, index) + } + case ir.PhysicalSetOp: + return fmt.Errorf("%w: coded-pivot source-region SET is a pg-sql-16 capability gap (probe-attested production-unreachable: construction outputs reject the legacy shape operations that mint it, probe P-14.3) and is refused, never partially rendered", + ErrUnsupportedPostgresPlan) + case ir.PhysicalExpressionLetOp: + return fmt.Errorf("%w: coded-pivot source-region EXPRESSION_LET is a pg-sql-16 capability gap (probe-attested production-unreachable, probe P-14.3) and is refused, never partially rendered", + ErrUnsupportedPostgresPlan) + case ir.PhysicalSortOp: + if sawSort || sawLimit || sawReturn { + return fmt.Errorf("%w: at most one SORT, before LIMIT", ErrUnsupportedPostgresPlan) + } + if operation.Sort == nil { + return fmt.Errorf("%w: operation %d: sort payload is missing", ErrUnsupportedPostgresPlan, index) + } + sawSort = true + case ir.PhysicalLimitOp: + if sawLimit || sawReturn { + return fmt.Errorf("%w: at most one LIMIT", ErrUnsupportedPostgresPlan) + } + if operation.Limit == nil { + return fmt.Errorf("%w: operation %d: limit payload is missing", ErrUnsupportedPostgresPlan, index) + } + sawLimit = true + case ir.PhysicalReturnOp: + if !last || operation.Return == nil || len(operation.Return.Projections) == 0 { + return fmt.Errorf("%w: the coded-pivot source requires a terminal RETURN with its projection list", ErrUnsupportedPostgresPlan) + } + sawReturn = true + default: + return fmt.Errorf("%w: operation %q is not part of the registered coded-pivot source surface (coexisting shapes are pg-sql-16)", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + return nil +} + +func validateCodedPivotPayload(plan ir.PhysicalPlan, stage ir.PhysicalConstructionStage, pivot *ir.PhysicalGroupedPivot) error { + if !pivot.OneInputRowPerGroup { + return fmt.Errorf("%w: the registered CODED_PIVOT requires the producer-proven OneInputRowPerGroup (lower/recipe_construction_coded_pivot.go:73-74, recipe_construction.go:476-478)", ErrUnsupportedPostgresPlan) + } + if pivot.UnlistedCategoryPolicy != "IGNORE" { + return fmt.Errorf("%w: the producer-stamped coded Pivot unlisted-category policy is IGNORE (ir/physical_table_shape.go); %q has no rendering", + ErrUnsupportedPostgresPlan, pivot.UnlistedCategoryPolicy) + } + if !pivotDuplicatePolicies[pivot.DuplicatePolicy] { + return fmt.Errorf("%w: coded pivot duplicate policy %q is not registered", ErrUnsupportedPostgresPlan, pivot.DuplicatePolicy) + } + if !pivotMissingPolicies[pivot.MissingCellPolicy] { + return fmt.Errorf("%w: coded pivot missing-cell policy %q is not registered", ErrUnsupportedPostgresPlan, pivot.MissingCellPolicy) + } + if _, ok := pivotValueTypeOf[pivot.ValueType]; !ok { + return fmt.Errorf("%w: coded pivot value type %q is not a registered table scalar kind", ErrUnsupportedPostgresPlan, pivot.ValueType) + } + if len(pivot.RowValues) != 0 { + return fmt.Errorf("%w: coded pivot row values have no PostgreSQL rendering (registered named divergence: accepted at recipe level, emitted by the producer at lower/recipe_construction_coded_pivot.go:80-91, refused fail-closed here; row values are the pg-sql-13 capability)", + ErrUnsupportedPostgresPlan) + } + if pivot.RootContributorInputColumn != "" || pivot.RootContributorOutputColumn != "" || pivot.RootContributorVariable != "" { + return fmt.Errorf("%w: coded pivot root-contributor columns have no rendering and the sole producer never writes them (C5)", ErrUnsupportedPostgresPlan) + } + if pivot.DuplicatePolicy != "ERROR" && pivot.ValueType != "INTEGER" && pivot.ValueType != "DECIMAL" { + return fmt.Errorf("%w: coded pivot reducer %q requires a numeric value type", ErrUnsupportedPostgresPlan, pivot.DuplicatePolicy) + } + if pivot.UnlistedEvidenceColumn != "" { + return fmt.Errorf("%w: the coded pivot (IGNORE policy) carries an evidence column", ErrUnsupportedPostgresPlan) + } + if len(pivot.GroupKeys) != 1 || pivot.GroupKeys[0].Column != "_key" || !pivot.GroupKeys[0].Hidden || + pivot.GroupKeys[0].Kind != "STRING" { + return fmt.Errorf("%w: the coded Pivot must group by the retained hidden root _key (ir/physical_table_shape.go:238-241)", ErrUnsupportedPostgresPlan) + } + if value, ok := plan.BindVars[pivot.ConstructionIDBindKey].(string); !ok || value == "" { + return fmt.Errorf("%w: coded pivot construction ID bind %q must be a non-empty string", + ErrUnsupportedPostgresPlan, pivot.ConstructionIDBindKey) + } + rootVariable := plan.Operations[0].RootScan.Variable + if pivot.CodedSourceVariable != rootVariable { + return fmt.Errorf("%w: the coded pivot must stream the scanned root variable %q, got %q", + ErrUnsupportedPostgresPlan, rootVariable, pivot.CodedSourceVariable) + } + if len(pivot.InputProjections) != 2 { + return fmt.Errorf("%w: the coded pivot source projection carries only the direct root key and payload (ir/physical_table_shape.go:246-258)", ErrUnsupportedPostgresPlan) + } + rootKey, payload := false, false + for _, projection := range pivot.InputProjections { + switch projection.Name { + case "_key": + rootKey = projection.Hidden && projection.Value.Variable == pivot.CodedSourceVariable && + len(projection.Value.Path) == 1 && projection.Value.Path[0] == "_key" + case ir.PhysicalCodedPivotSourcePayloadColumn: + payload = projection.Hidden && projection.Value.Variable == pivot.CodedSourceVariable && + len(projection.Value.Path) == 1 && projection.Value.Path[0] == "payload" + default: + return fmt.Errorf("%w: coded pivot input projection %q is outside the registered pair", + ErrUnsupportedPostgresPlan, projection.Name) + } + } + if !rootKey || !payload { + return fmt.Errorf("%w: the coded pivot _key and payload projections must read the scanned root variable (ir/physical_table_shape.go:247-258)", ErrUnsupportedPostgresPlan) + } + correlation := pivot.CodedCorrelation + if correlation.Source.Variable != pivot.InputRowVariable || + len(correlation.Source.Path) != 1 || correlation.Source.Path[0] != ir.PhysicalCodedPivotSourcePayloadColumn { + return fmt.Errorf("%w: the coded pivot correlation must read the staged payload from its input row (ir/physical_table_shape.go:259-262)", ErrUnsupportedPostgresPlan) + } + if len(correlation.ExtensionURLSelectors) != 0 || len(correlation.ExtensionURLBindKeys) != 0 { + return fmt.Errorf("%w: extension-URL correlations are never lowered onto the coded pivot (lower/correlated_binding.go:690-706; probe P-14.5) and have no rendering (C10)", + ErrUnsupportedPostgresPlan) + } + for name, selector := range map[string]*spec.Selector{ + "owner": &correlation.OwnerSelector, + "key": &correlation.KeySelector, + "system": &correlation.SystemSelector, + "code": &correlation.CodeSelector, + "value": &correlation.ValueSelector, + } { + if err := pivotSelectorOK(*selector); err != nil { + return fmt.Errorf("%w: coded pivot correlation %s selector: %w", ErrUnsupportedPostgresPlan, name, err) + } + } + for _, entry := range []struct { + name string + selector *spec.Selector + }{ + {"system", &correlation.SystemSelector}, + {"code", &correlation.CodeSelector}, + } { + selector := entry.selector + if len(selector.Steps) != 1 || selector.Filter != nil || + selector.Steps[0].Iterate || selector.Steps[0].Index != nil { + return fmt.Errorf("%w: coded pivot correlation %s selector must be one plain property step (the rendered match is a single property test on the coding item; multi-step, indexed, iterated, and filtered forms have no registered rendering, pg-sql-16)", + ErrUnsupportedPostgresPlan, entry.name) + } + } + for index, fallback := range correlation.ValueFallbacks { + if err := pivotSelectorOK(fallback); err != nil { + return fmt.Errorf("%w: coded pivot value fallback %d: %w", ErrUnsupportedPostgresPlan, index, err) + } + } + for index, choice := range correlation.ChoiceSelectors { + if err := pivotSelectorOK(choice); err != nil { + return fmt.Errorf("%w: coded pivot choice selector %d: %w", ErrUnsupportedPostgresPlan, index, err) + } + } + if correlation.VersionSelector != nil && strings.TrimSpace(correlation.VersionBindKey) == "" { + return fmt.Errorf("%w: a hand-carried correlation version selector requires its bind (ir re-read; production never carries version on the pivot — probe P-14.5)", ErrUnsupportedPostgresPlan) + } + if len(pivot.CodedCategories) == 0 || len(pivot.CodedCategories) > 50 { + return fmt.Errorf("%w: the coded pivot registers 1..50 frozen categories", ErrUnsupportedPostgresPlan) + } + outputs, identities := map[string]bool{}, map[string]bool{} + for index, category := range pivot.CodedCategories { + if category.Output == "" || outputs[category.Output] { + return fmt.Errorf("%w: coded pivot category %d output is empty or duplicated", ErrUnsupportedPostgresPlan, index) + } + outputs[category.Output] = true + if category.Output == "_key" || category.Output == pivotRowIDColumn { + return fmt.Errorf("%w: coded pivot category %d collides with the root identity", ErrUnsupportedPostgresPlan, index) + } + for _, bindKey := range []string{category.SystemBindKey, category.CodeBindKey} { + value, ok := plan.BindVars[bindKey].(string) + if !ok || strings.TrimSpace(value) == "" || value != strings.TrimSpace(value) { + return fmt.Errorf("%w: coded pivot category %d bind %q must be an exact non-empty string", + ErrUnsupportedPostgresPlan, index, bindKey) + } + } + system, _ := plan.BindVars[category.SystemBindKey].(string) + code, _ := plan.BindVars[category.CodeBindKey].(string) + pair := system + "\x00" + code + if identities[pair] { + return fmt.Errorf("%w: coded pivot category %d duplicates a bound system/code pair", ErrUnsupportedPostgresPlan, index) + } + identities[pair] = true + } + if correlation.SystemBindKey != pivot.CodedCategories[0].SystemBindKey || + correlation.CodeBindKey != pivot.CodedCategories[0].CodeBindKey { + return fmt.Errorf("%w: the shared correlation must bind the first category (ir/physical_table_shape.go:260-262)", ErrUnsupportedPostgresPlan) + } + names := map[string]bool{"_key": true, pivotRowIDColumn: true} + seen := map[string]bool{} + for _, column := range plan.StageSequence.FinalColumns { + if seen[column.Name] { + return fmt.Errorf("%w: coded pivot final column %q is declared twice", ErrUnsupportedPostgresPlan, column.Name) + } + seen[column.Name] = true + if !(names[column.Name] || outputs[column.Name]) { + return fmt.Errorf("%w: coded pivot final column %q is not a registered tuple member", + ErrUnsupportedPostgresPlan, column.Name) + } + } + if !seen["_key"] || !seen[pivotRowIDColumn] || len(seen) != len(pivot.CodedCategories)+2 { + return fmt.Errorf("%w: the registered slice emits the hidden root identity, every category column, and the hidden stable identity", ErrUnsupportedPostgresPlan) + } + if stage.OutputRowVariable == "" || stage.RowIdentityColumn == "" { + return fmt.Errorf("%w: the coded pivot stage must name its output row and identity columns", ErrUnsupportedPostgresPlan) + } + return nil +} + +func renderGroupedCodedPivotPhysicalPlan(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + if err := storage.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := validateGroupedCodedPivotPlanSurface(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := plan.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("validate physical plan: %w", err) + } + if plan.Engine != "" && plan.Engine != ir.PhysicalEngineAQL { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: engine %q has no PostgreSQL source; the source graph IR is backend-neutral and ClickHouse plans are not PostgreSQL-rendered", + ErrUnsupportedPostgresPlan, string(plan.Engine)) + } + if plan.ClickHouseCombine != nil || plan.ClickHousePrefix != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: ClickHouse combine/prefix payloads have no PostgreSQL rendering", ErrUnsupportedPostgresPlan) + } + if len(plan.DeferredExpressionLets) != 0 { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: completed plans must not carry deferred expression lets", ErrUnsupportedPostgresPlan) + } + renderer := &planRenderer{ + storage: storage, + rows: map[string]rowScope{}, + parameters: map[string]int{}, + canonicalAuthLets: map[string]string{}, + } + if err := renderer.loadScope(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + sql, err := renderer.renderGroupedCodedPivotStatement(plan) + if err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return command.NewRenderedPostgresPhysicalPlan(sql, renderer.args, renderer.outputColumns) +} + +func (r *planRenderer) renderGroupedCodedPivotStatement(plan ir.PhysicalPlan) (string, error) { + sequence := plan.StageSequence + stage := sequence.Stages[0] + pivot := stage.GroupedPivot + correlation := pivot.CodedCorrelation + + if err := r.renderRootScan(plan, *plan.Operations[0].RootScan); err != nil { + return "", err + } + for _, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalFilterOp: + canonical, err := r.acceptCanonicalScopeFilter(operation.Filter) + if err != nil { + return "", err + } + if !canonical { + return "", fmt.Errorf("%w: a non-canonical FILTER cannot coexist with the registered coded pivot (renders-at-AQL divergence, pg-sql-16)", + ErrUnsupportedPostgresPlan) + } + case ir.PhysicalDerivedLetOp: + canonical, err := r.acceptCanonicalAuthLet(operation.DerivedLet) + if err != nil { + return "", err + } + if !canonical { + return "", fmt.Errorf("%w: operation %q is not part of the registered coded-pivot source surface", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + case ir.PhysicalSortOp: + if err := r.renderWindowSort(plan, *operation.Sort); err != nil { + return "", err + } + case ir.PhysicalLimitOp: + if err := r.renderWindowLimit(plan, *operation.Limit); err != nil { + return "", err + } + } + } + + resource := r.storage.Resource + source := "SELECT " + r.rootAlias + "." + resource.MustColumn("logical_id").SQLText() + + ` AS cp_key, ` + r.rootAlias + "." + resource.MustColumn("payload").SQLText() + ` AS cp_payload` + + " FROM " + resource.Identifier().SQLText() + " AS " + r.rootAlias + for _, join := range r.joins { + source += " " + join + } + for _, lateral := range r.laterals { + source += " " + lateral + } + if len(r.where) > 0 { + source += " WHERE " + strings.Join(r.where, " AND ") + } + source += " ORDER BY " + strings.Join(r.windowOrder(), ", ") + r.window.limitClause + + valueTypeOf := quoteSQLLiteral(pivotValueTypeOf[pivot.ValueType]) + + ownerFrom := "" + ownerItem := "s.cp_payload" + ownerWhere := "" + if correlation.OwnerSelector.CanonicalPath() != "" { + from, item, _, where, err := r.pivotSelectorChain("s.cp_payload", correlation.OwnerSelector, "ow") + if err != nil { + return "", err + } + ownerFrom, ownerItem, ownerWhere = from, item, pivotWhereJoin(where) + } + owners := "SELECT s.cp_key, " + ownerItem + " AS cp_owner FROM cp_src AS s" + ownerFrom + ownerWhere + + ctes := []string{"cp_src AS (" + source + ")", "cp_owner AS (" + owners + ")"} + aggJoins := make([]string, 0, len(pivot.CodedCategories)) + cellExpressions := make([]string, 0, len(pivot.CodedCategories)) + guardTerms := make([]string, 0, 2*len(pivot.CodedCategories)) + + for index, category := range pivot.CodedCategories { + tag := strconv.Itoa(index) + systemValue := bindStringVar(plan.BindVars, category.SystemBindKey) + codeValue := bindStringVar(plan.BindVars, category.CodeBindKey) + systemParameter := pivotDollar(r.textParameter(systemValue, "pivot:coded-system:"+systemValue)) + codeParameter := pivotDollar(r.textParameter(codeValue, "pivot:coded-code:"+codeValue)) + systemPath := correlation.SystemSelector.Steps[0].Field + codePath := correlation.CodeSelector.Steps[0].Field + systemKey := pivotDollar(r.textParameter(systemPath, "pivot:prop:"+systemPath)) + codeKey := pivotDollar(r.textParameter(codePath, "pivot:prop:"+codePath)) + + codingFrom, codingItem, _, codingWhere, err := r.pivotSelectorChain("o.cp_owner", correlation.KeySelector, "cd"+tag) + if err != nil { + return "", err + } + exists := "EXISTS (SELECT 1 FROM (SELECT 1) AS " + "cd" + tag + "_hub" + codingFrom + + " WHERE jsonb_typeof(" + codingItem + " -> " + systemKey + "::text) = 'string'" + + " AND " + codingItem + " ->> " + systemKey + "::text <> ''" + + " AND " + codingItem + " ->> " + systemKey + "::text = " + systemParameter + + " AND jsonb_typeof(" + codingItem + " -> " + codeKey + "::text) = 'string'" + + " AND " + codingItem + " ->> " + codeKey + "::text <> ''" + + " AND " + codingItem + " ->> " + codeKey + "::text = " + codeParameter + + pivotCodingExtraWhere(codingWhere) + ")" + + unsupported, err := r.pivotUnsupportedChoiceValues(correlation, "o.cp_owner", "cs"+tag) + if err != nil { + return "", err + } + + values, err := r.pivotCodedValues(correlation, "o.cp_owner", "cv"+tag) + if err != nil { + return "", err + } + + matchCTE := "cp_cat_" + tag + " AS (" + + "SELECT * FROM (SELECT o.cp_key, " + exists + " AS cp_m, " + + "(jsonb_array_length(" + unsupported + ")) > 0 AS cp_bad, " + values + " AS cp_vals" + + " FROM cp_owner AS o) t" + + " WHERE t.cp_m)" + elemCTE := "cp_elem_" + tag + " AS (" + + "SELECT m.cp_key, e.cp_e, e.ord FROM cp_cat_" + tag + " AS m" + + " CROSS JOIN LATERAL jsonb_array_elements(m.cp_vals) WITH ORDINALITY AS e(cp_e, ord))" + aggCTE := "cp_agg_" + tag + " AS (" + + "SELECT k.cp_key, k.cp_cnt," + + " COALESCE(x.cp_tcnt, 0) AS cp_tcnt, COALESCE(x.cp_mm, false) AS cp_mm," + + " x.cp_first, x.cp_sum, x.cp_min, x.cp_max, k.cp_bad" + + " FROM (SELECT cp_key, SUM(jsonb_array_length(cp_vals)) AS cp_cnt," + + " BOOL_OR(cp_bad) AS cp_bad FROM cp_cat_" + tag + " GROUP BY cp_key) AS k" + + " LEFT JOIN (SELECT cp_key," + + " COUNT(*) FILTER (WHERE e.cp_e <> 'null'::jsonb) AS cp_tcnt," + + " BOOL_OR(e.cp_e <> 'null'::jsonb AND jsonb_typeof(e.cp_e) <> " + valueTypeOf + ") AS cp_mm," + + " (ARRAY_AGG(e.cp_e ORDER BY e.ord))[1] AS cp_first," + + " SUM(CASE WHEN jsonb_typeof(e.cp_e) = " + valueTypeOf + " THEN " + pivotScalarText("e.cp_e") + "::double precision END) FILTER (WHERE e.cp_e <> 'null'::jsonb) AS cp_sum," + + " MIN(CASE WHEN jsonb_typeof(e.cp_e) = " + valueTypeOf + " THEN " + pivotScalarText("e.cp_e") + "::double precision END) FILTER (WHERE e.cp_e <> 'null'::jsonb) AS cp_min," + + " MAX(CASE WHEN jsonb_typeof(e.cp_e) = " + valueTypeOf + " THEN " + pivotScalarText("e.cp_e") + "::double precision END) FILTER (WHERE e.cp_e <> 'null'::jsonb) AS cp_max" + + " FROM cp_elem_" + tag + " AS e GROUP BY cp_key) AS x ON x.cp_key = k.cp_key)" + ctes = append(ctes, matchCTE, elemCTE, aggCTE) + + alias := "a" + tag + aggJoins = append(aggJoins, " LEFT JOIN cp_agg_"+tag+" AS "+alias+" ON "+alias+".cp_key = s.cp_key") + + guardTerms = append(guardTerms, "WHEN COALESCE("+alias+".cp_bad, false) THEN "+guardCast(pivotChoiceArmTag)) + cnt := "COALESCE(" + alias + ".cp_cnt, 0)" + typedCnt := "COALESCE(" + alias + ".cp_tcnt, 0)" + cell := "" + switch pivot.DuplicatePolicy { + case "ERROR": + cell = "COALESCE(" + alias + ".cp_first, 'null'::jsonb)" + if pivot.MissingCellPolicy == "ERROR" { + guardTerms = append(guardTerms, "WHEN "+cnt+" = 0 THEN "+guardCast(pivotCellMissingTag)) + } + guardTerms = append(guardTerms, "WHEN "+cnt+" > 1 THEN "+guardCast(pivotCardinalityTag)) + default: + reducerColumn := map[string]string{"SUM": "cp_sum", "MIN": "cp_min", "MAX": "cp_max"}[pivot.DuplicatePolicy] + cell = "(CASE WHEN " + typedCnt + " = 0 THEN 'null'::jsonb ELSE to_jsonb(" + alias + "." + reducerColumn + ") END)" + if pivot.MissingCellPolicy == "ERROR" { + guardTerms = append(guardTerms, "WHEN "+typedCnt+" = 0 THEN "+guardCast(pivotCellMissingTag)) + } + guardTerms = append(guardTerms, "WHEN COALESCE("+alias+".cp_mm, false) THEN "+guardCast(pivotValueTypeTag)) + } + cellExpressions = append(cellExpressions, cell) + } + + keyExpr := "to_jsonb(s.cp_key::text)" + identity := r.pivotIdentitySQL(bindStringVar(plan.BindVars, pivot.ConstructionIDBindKey), + []string{pivot.GroupKeys[0].Kind}, []string{keyExpr}) + + expressions := map[string]string{"_key": keyExpr, pivotRowIDColumn: identity} + for index, category := range pivot.CodedCategories { + expressions[category.Output] = cellExpressions[index] + } + selects := make([]string, 0, len(sequence.FinalColumns)) + names := make([]string, 0, len(sequence.FinalColumns)) + for _, column := range sequence.FinalColumns { + expression, ok := expressions[column.Name] + if !ok { + return "", fmt.Errorf("%w: coded pivot final column %q has no rendered expression", + ErrUnsupportedPostgresPlan, column.Name) + } + selects = append(selects, expression+` AS "c`+strconv.Itoa(len(selects))+`"`) + names = append(names, column.Name) + } + r.outputColumns = names + + statement := "WITH " + strings.Join(ctes, ", ") + + " SELECT " + strings.Join(selects, ", ") + " FROM cp_src AS s" + + strings.Join(aggJoins, "") + if len(guardTerms) > 0 { + statement += " WHERE (CASE " + strings.Join(guardTerms, " ") + " ELSE 1 END) = 1" + } + statement += ` ORDER BY ` + identity + ` COLLATE "C" ASC` + if sequence.PreviewLimitBindKey != "" { + limit := plan.BindVars[sequence.PreviewLimitBindKey].(int) + limitIndex := r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(limit)}, "") + statement += " LIMIT " + pivotDollar(limitIndex) + } + return statement, nil +} + +func pivotCodingExtraWhere(where []string) string { + if len(where) == 0 { + return "" + } + return " AND " + strings.Join(where, " AND ") +} + +func (r *planRenderer) pivotUnsupportedChoiceValues(correlation *ir.PhysicalCorrelation, base string, name string) (string, error) { + if len(correlation.ChoiceSelectors) == 0 || len(correlation.ChoiceArms) == 0 { + return "'[]'::jsonb", nil + } + parts := make([]string, 0, len(correlation.ChoiceSelectors)) + for index, selector := range correlation.ChoiceSelectors { + if len(selector.Steps) == 0 || pivotChoiceArmAllowed(selector.Steps[0].Field, correlation.ChoiceArms) { + continue + } + array, err := r.pivotSelectorArray(base, selector, name+strconv.Itoa(index)) + if err != nil { + return "", err + } + parts = append(parts, array) + } + if len(parts) == 0 { + return "'[]'::jsonb", nil + } + if len(parts) == 1 { + return parts[0], nil + } + elements := make([]string, 0, len(parts)) + for _, part := range parts { + elements = append(elements, + "SELECT u.x FROM jsonb_array_elements(CASE WHEN jsonb_typeof("+part+") = 'array' THEN "+part+" ELSE '[]'::jsonb END) AS u(x)") + } + return "(SELECT COALESCE(jsonb_agg(w.x), '[]'::jsonb) FROM (" + strings.Join(elements, " UNION ALL ") + ") AS w)", nil +} + +func pivotChoiceArmAllowed(arm string, allowed []string) bool { + for _, candidate := range allowed { + if strings.TrimSpace(candidate) == strings.TrimSpace(arm) { + return true + } + } + return false +} + +func (r *planRenderer) pivotCodedValues(correlation *ir.PhysicalCorrelation, base string, name string) (string, error) { + candidates := make([]spec.Selector, 0, 1+len(correlation.ValueFallbacks)) + candidates = append(candidates, correlation.ValueSelector) + candidates = append(candidates, correlation.ValueFallbacks...) + arrays := make([]string, 0, len(candidates)) + for index, selector := range candidates { + array, err := r.pivotSelectorArray(base, selector, name+strconv.Itoa(index)) + if err != nil { + return "", err + } + arrays = append(arrays, array) + } + chosen := arrays[0] + if len(arrays) > 1 { + eligible := "(jsonb_typeof(c.cand) = 'array' AND jsonb_array_length(c.cand) > 0)" + if correlation.FallbackOnBlankString { + eligible = "(jsonb_typeof(c.cand) = 'array' AND EXISTS (" + + "SELECT 1 FROM jsonb_array_elements(c.cand) AS b(x)" + + " WHERE jsonb_typeof(b.x) = 'string'" + + " AND " + r.codedGroupTrimmedNonBlank("b.x #>> '{}'") + "))" + } + rows := make([]string, 0, len(arrays)) + for index, array := range arrays { + rows = append(rows, "("+strconv.Itoa(index+1)+",("+array+"))") + } + chosen = "COALESCE((SELECT c.cand FROM (VALUES " + strings.Join(rows, ",") + + ") AS c(ord, cand) WHERE " + eligible + " ORDER BY c.ord LIMIT 1), '[]'::jsonb)" + } + return "(SELECT COALESCE(jsonb_agg(s.cp_e ORDER BY f.ord, s.ord1), '[]'::jsonb)" + + " FROM jsonb_array_elements(CASE WHEN jsonb_typeof(" + chosen + ") = 'array' THEN " + chosen + " ELSE '[]'::jsonb END)" + + " WITH ORDINALITY AS f(x, ord)" + + " CROSS JOIN LATERAL jsonb_array_elements(CASE WHEN jsonb_typeof(f.x) = 'array' THEN f.x" + + " ELSE jsonb_build_array(f.x) END) WITH ORDINALITY AS s(cp_e, ord1))", nil +} diff --git a/internal/dataframe/compiler/render/postgres/pivot14_noncoded_test.go b/internal/dataframe/compiler/render/postgres/pivot14_noncoded_test.go new file mode 100644 index 000000000..f4a4a86c9 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/pivot14_noncoded_test.go @@ -0,0 +1,49 @@ +package postgres + +import ( + "testing" + + "github.com/calypr/loom/internal/dataframe/recipe" +) + +func pivot14NonCodedPivotBundle(t *testing.T, name string) recipe.Bundle { + t.Helper() + alpha := "alpha" + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: name, TranslationVersion: "test", + Outputs: []recipe.Output{{ + Name: name, RootResourceType: "Observation", RowGrain: "resource", + Fields: []recipe.Field{ + {Name: "grp", ColumnID: "grp_id", Expr: recipe.Expression{Select: "root.status"}}, + {Name: "category", ColumnID: "cat_id", Expr: recipe.Expression{Select: "root.status"}}, + {Name: "amount", ColumnID: "val_id", Expr: recipe.Expression{Select: "root.valueQuantity.value"}}, + }, + Construction: &recipe.Construction{ + Version: 1, + SourceColumns: []recipe.StageColumn{ + {ID: "grp_id", Name: "grp"}, {ID: "cat_id", Name: "category"}, {ID: "val_id", Name: "amount"}, + }, + Steps: []recipe.ConstructionStep{{ + ID: name + "_step", + Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{ + Kind: recipe.ConstructionPivotOp, + Pivot: &recipe.ConstructionPivot{ + ConstructionID: name, + GroupKeyIDs: []string{"grp_id"}, + CategoryColumnID: "cat_id", ValueColumnID: "val_id", + Categories: []recipe.ConstructionPivotCategory{ + {Key: recipe.TableScalar{Kind: recipe.TableScalarString, String: &alpha}, OutputColumnID: "alpha_id"}, + }, + DuplicatePolicy: recipe.PivotDuplicateError, MissingCellPolicy: recipe.PivotMissingCellNull, + UnlistedCategoryPolicy: recipe.PivotUnlistedCategoryError, + }, + }, + Outputs: []recipe.StageColumn{ + {ID: "grp_id", Name: "grp"}, {ID: "alpha_id", Name: "alpha", Type: "decimal", Nullable: true}, + }, + }}, + }, + }}, + } +} diff --git a/internal/dataframe/compiler/render/postgres/pivot_test.go b/internal/dataframe/compiler/render/postgres/pivot_test.go new file mode 100644 index 000000000..4adb80da6 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/pivot_test.go @@ -0,0 +1,1540 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" + spec "github.com/calypr/loom/internal/fhir/schema" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + pivot14Project = "proj-a" + pivot14Generation = "gen-1" + pivot14AuthPath = "FHIR/Patient" + pivot14Prefix = "verify_sql14v1_" +) + +func pivot14CompileBundle(t *testing.T, bundle recipe.Bundle, authPaths []string) ir.PhysicalPlan { + t.Helper() + bindings := recipe.RuntimeBindings{ + Project: pivot14Project, DatasetGeneration: pivot14Generation, + AuthResourcePaths: authPaths, + } + plan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("lower.CompileResolvedRecipePlan: %v", err) + } + if len(compiled.Outputs) != 1 { + t.Fatalf("compiled outputs = %d, want 1", len(compiled.Outputs)) + } + output := compiled.Outputs[0] + if output.OptimizedPlan != nil { + return *output.OptimizedPlan + } + return output.Plan +} + +func pivot14ScalarBundle(name string, filters []recipe.Filter, duplicate recipe.PivotDuplicatePolicy, + missing recipe.PivotMissingCellPolicy, unlisted recipe.PivotUnlistedCategoryPolicy) recipe.Bundle { + zero := int64(0) + one := int64(1) + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: name, TranslationVersion: "test", + Outputs: []recipe.Output{{ + Name: "patients", RootResourceType: "Patient", RowGrain: "patient", Filters: filters, + Fields: []recipe.Field{ + {Name: "grp", Expr: recipe.Expression{Select: "gender"}}, + {Name: "category", Expr: recipe.Expression{Select: "multipleBirthInteger"}}, + {Name: "value", Expr: recipe.Expression{Select: "multipleBirthInteger"}}, + }, + TableReshape: &recipe.TableReshape{ + Kind: recipe.TableReshapeGroupedPivot, + GroupedPivot: &recipe.GroupedPivot{ + ConstructionID: name, GroupKeys: []string{"grp"}, + CategoryColumn: "category", ValueColumn: "value", + Categories: []recipe.GroupedPivotCategory{ + {Key: recipe.TableScalar{Kind: recipe.TableScalarInteger, Integer: &zero}, Output: "birth_zero", Label: "Zero"}, + {Key: recipe.TableScalar{Kind: recipe.TableScalarInteger, Integer: &one}, Output: "birth_one", Label: "One"}, + }, + DuplicatePolicy: duplicate, MissingCellPolicy: missing, + UnlistedCategoryPolicy: unlisted, + }, + }, + }}, + } +} + +func pivot14FemaleFilter() []recipe.Filter { + female := "female" + return []recipe.Filter{{Select: "gender", Operator: recipe.FilterEquals, + Values: []recipe.FilterValue{{Kind: recipe.FilterString, String: &female}}}} +} + +type pivot14ScalarConfig struct { + keyless bool + duplicate string + missing string + unlisted string + window bool + presence string + categoryOmit bool + extractorValue bool + categories []ir.PhysicalGroupedPivotCategory + categoryType string + valueType string + constructionID string + categoryBinds map[string]any + returnDriftName string +} + +func pivot14ScalarPlan(t *testing.T, cfg pivot14ScalarConfig) ir.PhysicalPlan { + t.Helper() + duplicate := cfg.duplicate + if duplicate == "" { + duplicate = "ERROR" + } + missing := cfg.missing + if missing == "" { + missing = "NULL" + } + unlisted := cfg.unlisted + if unlisted == "" { + unlisted = "ERROR" + } + categories := cfg.categories + if categories == nil { + categories = []ir.PhysicalGroupedPivotCategory{ + {Output: "cell_alpha", MatchKind: ir.PhysicalPivotCategoryValueMatch, ValueBindKey: "cat_alpha", ValueKind: "STRING"}, + {Output: "cell_beta", MatchKind: ir.PhysicalPivotCategoryValueMatch, ValueBindKey: "cat_beta", ValueKind: "STRING"}, + {Output: "cell_null", MatchKind: ir.PhysicalPivotCategoryNullMatch}, + } + if cfg.presence != "" { + categories = append(categories, + ir.PhysicalGroupedPivotCategory{Output: "cell_missing", MatchKind: ir.PhysicalPivotCategoryMissingMatch}) + } + } + categoryType := cfg.categoryType + if categoryType == "" { + categoryType = "STRING" + } + valueType := cfg.valueType + if valueType == "" { + valueType = "DECIMAL" + } + constructionID := cfg.constructionID + if constructionID == "" { + constructionID = "p14-hand-construct" + } + bindVars := map[string]any{ + "root_collection": "Patient", + "project": pivot14Project, + "dataset_generation": pivot14Generation, + "auth_resource_paths": []string{pivot14AuthPath}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + "p14_construction_id": constructionID, + "cat_alpha": "alpha", + "cat_beta": "beta", + "limit": 2, + } + for key, value := range cfg.categoryBinds { + bindVars[key] = value + } + presence := &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + Paths: [][]string{{"cProp"}}, + } + categoryProjection := ir.PhysicalProjection{ + Name: "ccol", + Value: ir.PhysicalValue{Variable: "root", Path: []string{"cProp"}}, + } + categoryPresenceColumn := "" + switch cfg.presence { + case "contract": + categoryProjection.Presence = presence + categoryPresenceColumn = "cProp_present" + case "from-input": + } + valueProjection := ir.PhysicalProjection{ + Name: "vcol", + Value: ir.PhysicalValue{Variable: "root", Path: []string{"vProp"}}, + } + if cfg.extractorValue { + valueProjection = ir.PhysicalProjection{ + Name: "vcol", + Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Patient", + Selector: specSelector(t, "multipleBirthInteger"), + }, + }, + } + } + pivot := &ir.PhysicalGroupedPivot{ + ConstructionID: constructionID, + ConstructionIDBindKey: "p14_construction_id", + InputRowVariable: "irow", + GroupRowsVariable: "grow", + OutputRowVariable: "prow", + InputProjections: []ir.PhysicalProjection{ + {Name: "gcol", Value: ir.PhysicalValue{Variable: "root", Path: []string{"gProp"}}}, + categoryProjection, + valueProjection, + }, + CategoryColumn: "ccol", + CategoryType: categoryType, + ValueColumn: "vcol", + ValueType: valueType, + Categories: categories, + DuplicatePolicy: duplicate, + MissingCellPolicy: missing, + UnlistedCategoryPolicy: unlisted, + } + if cfg.presence == "contract" { + pivot.CategoryPresence = presence + pivot.CategoryPresenceColumn = categoryPresenceColumn + } + if cfg.presence == "from-input" { + pivot.CategoryPresenceFromInput = true + pivot.CategoryPresenceColumn = "cpresent" + pivot.InputProjections = append(pivot.InputProjections, ir.PhysicalProjection{ + Name: "cpresent", + Value: ir.PhysicalValue{Variable: "root", Path: []string{"cPresent"}}, + }) + } + if cfg.categoryOmit { + pivot.InputProjections[1].Expression = &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalOmitNulls, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Patient", + Selector: specSelector(t, "gender"), + }, + } + pivot.InputProjections[1].Value = ir.PhysicalValue{} + pivot.InputProjections[1].Presence = nil + } + if unlisted == "EXCLUDE_WITH_EVIDENCE" { + pivot.UnlistedEvidenceColumn = "unlisted_n" + } + var groupKeys []ir.PhysicalGroupedPivotKey + if !cfg.keyless { + groupKeys = append(groupKeys, ir.PhysicalGroupedPivotKey{Column: "gcol", Variable: "g0", Kind: "STRING", Output: "grp"}) + } + pivot.GroupKeys = groupKeys + + returnNames := []string{} + if !cfg.keyless { + returnNames = append(returnNames, "grp") + } + for _, category := range categories { + returnNames = append(returnNames, category.Output) + } + if unlisted == "EXCLUDE_WITH_EVIDENCE" { + returnNames = append(returnNames, "unlisted_n") + } + returnNames = append(returnNames, "__loom_row_id") + projections := make([]ir.PhysicalProjection, 0, len(returnNames)) + for _, name := range returnNames { + if cfg.returnDriftName != "" && name == cfg.returnDriftName { + projections = append(projections, ir.PhysicalProjection{ + Name: name, + Value: ir.PhysicalValue{Variable: "prow", Path: []string{"drifted_" + name}}, + }) + continue + } + projections = append(projections, ir.PhysicalProjection{ + Name: name, + Value: ir.PhysicalValue{Variable: "prow", Path: []string{name}}, + }) + } + + operations := []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalGroupedPivotOp, GroupedPivot: pivot}, + } + if cfg.window { + operations = append(operations, + ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{ + Keys: []ir.PhysicalValue{{Variable: "prow", Path: []string{"__loom_row_id"}}}}}, + ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "limit"}}) + } + operations = append(operations, ir.PhysicalOperation{Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}}) + return ir.PhysicalPlan{Version: 1, Engine: ir.PhysicalEngineAQL, BindVars: bindVars, Operations: operations} +} + +func pivot14Render(t *testing.T, plan ir.PhysicalPlan) string { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + return rendered.SQL +} + +func pivot14Refused(t *testing.T, plan ir.PhysicalPlan, label string, wantFragment string) { + t.Helper() + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil { + t.Fatalf("%s: expected typed refusal, got success", label) + } + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("%s: refusal not ErrUnsupportedPostgresPlan: %v", label, err) + } + if wantFragment != "" && !strings.Contains(err.Error(), wantFragment) { + t.Fatalf("%s: refusal %q lacks %q", label, err.Error(), wantFragment) + } +} + +func TestPivot14ProductionScalarPivotRenders(t *testing.T) { + plan := pivot14CompileBundle(t, pivot14ScalarBundle("p14_prod_render", nil, + recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, recipe.PivotUnlistedCategoryExcludeWithEvidence), + []string{pivot14AuthPath}) + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("production scalar pivot must render through the registered door: %v", err) + } + sql := rendered.SQL + for _, want := range []string{ + "WITH pv_src AS (SELECT", + "FROM pv_src AS p", + "GROUP BY", + `ORDER BY`, + `COLLATE "C" ASC`, + "jsonb_typeof(", + "::double precision)", + } { + if !strings.Contains(sql, want) { + t.Fatalf("production statement lost %q:\n%s", want, sql) + } + } + for _, forbidden := range []string{"proj-a", "gen-1", "FHIR/Patient", "female"} { + if strings.Contains(sql, forbidden) { + t.Fatalf("plan data leaked into statement text (%q):\n%s", forbidden, sql) + } + } + if len(rendered.Args) == 0 { + t.Fatal("production pivot statement carries no parameters") + } +} + +func TestPivot14GoldenScalarStatement(t *testing.T) { + sql := pivot14Render(t, pivot14ScalarPlan(t, pivot14ScalarConfig{ + duplicate: "SUM", missing: "NULL", unlisted: "ERROR", presence: "contract", + })) + unlistedFirst := strings.Index(sql, `CAST('TABLE_PIVOT_UNLISTED_CATEGORY'`) + cardinalityAfter := strings.Index(sql, `CAST('TABLE_PIVOT_VALUE_TYPE_MISMATCH'`) + if unlistedFirst < 0 || cardinalityAfter < unlistedFirst { + t.Fatalf("guard chain order drifted (unlisted must precede per-category guards):\n%s", sql) + } + for _, want := range []string{ + "COUNT(*) FILTER (WHERE NOT COALESCE((", + "THEN CAST('TABLE_PIVOT_UNLISTED_CATEGORY'", + "CASE WHEN COUNT(*) FILTER (WHERE", + "THEN 'null'::jsonb ELSE to_jsonb(SUM(CASE WHEN jsonb_typeof(", + ") = 'number' THEN ", + "::double precision END) FILTER (WHERE", + "jsonb_typeof(", + "THEN CAST('TABLE_PIVOT_VALUE_TYPE_MISMATCH'", + "jsonb_typeof(", + "= 'string'", + "::jsonb)", + `'[' || '"GROUPED_PIVOT"' || ',' || to_jsonb(`, + `|| ',' || '[' || to_jsonb(`, + `|| ']' || ']'`, + "GROUP BY p.", + `COLLATE "C" ASC`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("golden statement lost %q:\n%s", want, sql) + } + } + if strings.Contains(sql, "TABLE_PIVOT_CELL_MISSING") { + t.Fatalf("NULL missing policy must not emit the missing guard:\n%s", sql) + } + if !strings.Contains(sql, "NOT (p.") { + t.Fatalf("MISSING category must render NOT (present):\n%s", sql) + } +} + +func TestPivot14GoldenErrorBranch(t *testing.T) { + sql := pivot14Render(t, pivot14ScalarPlan(t, pivot14ScalarConfig{ + duplicate: "ERROR", missing: "ERROR", unlisted: "ERROR", + })) + for _, want := range []string{ + "(ARRAY_AGG(p.", + ") FILTER (WHERE", + ")[1]", + "COALESCE(", + "THEN CAST('TABLE_PIVOT_CELL_MISSING'", + "THEN CAST('TABLE_PIVOT_CELL_CARDINALITY'", + } { + if !strings.Contains(sql, want) { + t.Fatalf("ERROR-branch golden lost %q:\n%s", want, sql) + } + } + if strings.Contains(sql, "ELSE to_jsonb(") { + t.Fatalf("ERROR branch must not enumerate typed reducer values:\n%s", sql) + } +} + +func TestPivot14GoldenWindowClauses(t *testing.T) { + sql := pivot14Render(t, pivot14ScalarPlan(t, pivot14ScalarConfig{ + duplicate: "MAX", window: true, + })) + if !strings.Contains(sql, `COLLATE "C" ASC LIMIT $`) { + t.Fatalf("window leg lost the identity order + bound LIMIT:\n%s", sql) + } + if strings.Contains(sql, "LIMIT 2") { + t.Fatalf("LIMIT must stay parameterized:\n%s", sql) + } +} + +func TestPivot14GoldenNoWindowPerKeyOrder(t *testing.T) { + sql := pivot14Render(t, pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "MIN"})) + for _, want := range []string{ + "CASE WHEN p.", + "= 'boolean' THEN 1", + "= 'number' THEN 2", + "= 'string' THEN 3", + "ELSE 4 END) ASC", + "::double precision) ASC", + `COLLATE "C") ASC`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("per-key order lost %q:\n%s", want, sql) + } + } +} + +func TestPivot14GoldenPresenceForms(t *testing.T) { + sql := pivot14Render(t, pivot14ScalarPlan(t, pivot14ScalarConfig{ + duplicate: "ERROR", presence: "contract", + categories: []ir.PhysicalGroupedPivotCategory{ + {Output: "cell_alpha", MatchKind: ir.PhysicalPivotCategoryValueMatch, ValueBindKey: "cat_alpha", ValueKind: "STRING"}, + {Output: "cell_missing", MatchKind: ir.PhysicalPivotCategoryMissingMatch}, + }, + })) + if !strings.Contains(sql, "THEN 'true'::jsonb ELSE 'false'::jsonb END") { + t.Fatalf("presence contract lost the tri-state proof read:\n%s", sql) + } + if !strings.Contains(sql, " = 'true'::jsonb") { + t.Fatalf("presence comparison drifted:\n%s", sql) + } + + sql = pivot14Render(t, pivot14ScalarPlan(t, pivot14ScalarConfig{ + duplicate: "ERROR", presence: "from-input", + categories: []ir.PhysicalGroupedPivotCategory{ + {Output: "cell_alpha", MatchKind: ir.PhysicalPivotCategoryValueMatch, ValueBindKey: "cat_alpha", ValueKind: "STRING"}, + {Output: "cell_missing", MatchKind: ir.PhysicalPivotCategoryMissingMatch}, + }, + })) + if strings.Contains(sql, "THEN 'true'::jsonb ELSE 'false'::jsonb END") { + t.Fatalf("input-carried presence must not recompute the proof:\n%s", sql) + } + if !strings.Contains(sql, "COALESCE(") || !strings.Contains(sql, " = 'true'::jsonb, false)") { + t.Fatalf("input-carried presence must compare the companion bit two-valued:\n%s", sql) + } + + sql = pivot14Render(t, pivot14ScalarPlan(t, pivot14ScalarConfig{ + duplicate: "ERROR", categoryOmit: true, + categories: []ir.PhysicalGroupedPivotCategory{ + {Output: "cell_null", MatchKind: ir.PhysicalPivotCategoryNullMatch}, + }, + })) + if !strings.Contains(sql, "IS NOT NULL") || !strings.Contains(sql, "= 'null'::jsonb") { + t.Fatalf("null-omission category read drifted:\n%s", sql) + } +} + +func TestPivot14HiddenAuthGroupKeyHandIR(t *testing.T) { + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "ERROR"}) + pivot := plan.Operations[1].GroupedPivot + pivot.GroupKeys = append(pivot.GroupKeys, ir.PhysicalGroupedPivotKey{ + Column: "authcol", Variable: "authg", Kind: "STRING", Output: "authcol", Hidden: true}) + pivot.InputProjections = append(pivot.InputProjections, ir.PhysicalProjection{ + Name: "authcol", + Hidden: true, + Value: ir.PhysicalValue{Variable: "root", Path: []string{"auth_resource_path"}}, + }) + plan.Operations[1].GroupedPivot = pivot + returnOp := plan.Operations[len(plan.Operations)-1] + returnOp.Return.Projections = append(returnOp.Return.Projections, ir.PhysicalProjection{ + Name: "authcol", + Value: ir.PhysicalValue{Variable: "prow", Path: []string{"authcol"}}, + }) + plan.Operations[len(plan.Operations)-1] = returnOp + sql := pivot14Render(t, plan) + if !strings.Contains(sql, "auth_resource_path") { + t.Fatalf("hidden auth group key lost its read:\n%s", sql) + } + if strings.Contains(sql, "GROUP BY p.") && !strings.Contains(sql, "auth") { + t.Fatalf("hidden auth key missing from the group:\n%s", sql) + } +} + +func TestPivot14HostileCategoryValuesStayParameterized(t *testing.T) { + hostile := map[string]any{ + "cat_alpha": "a\"b',c]}}\"{[ -- /* x' || $$ $$", + } + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{ + duplicate: "ERROR", + categories: []ir.PhysicalGroupedPivotCategory{{Output: "cell_alpha", MatchKind: ir.PhysicalPivotCategoryValueMatch, ValueBindKey: "cat_alpha", ValueKind: "STRING"}}, + constructionID: `p14",["EVIL",1]`, + categoryBinds: hostile, + }) + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("hostile category render: %v", err) + } + sql := rendered.SQL + for _, forbidden := range []string{"a\"b'", "EVIL", "--", "$$"} { + if strings.Contains(sql, forbidden) { + t.Fatalf("hostile data leaked into SQL text (%q):\n%s", forbidden, sql) + } + } + carried := 0 + for _, parameter := range rendered.Args { + text := fmt.Sprintf("%s", parameter.Value) + if strings.Contains(text, "EVIL") || strings.Contains(text, "$$") { + carried++ + } + } + if carried < 2 { + t.Fatalf("hostile construction ID/category must ride two parameters, got %d:\n%v", carried, rendered.Args) + } +} + +func TestPivot14SpinePathUntouched(t *testing.T) { + if sql := pivot14Render(t, pivot14ScalarPlan(t, pivot14ScalarConfig{})); sql == "" { + t.Fatal("canonical pivot rendered empty") + } + nonCoded := pivot14CompileBundle(t, pivot14NonCodedPivotBundle(t, "p14_spine_noncoded"), []string{"FHIR/Observation"}) + if nonCoded.StageSequence == nil || nonCoded.StageSequence.Stages[0].GroupedPivot == nil || + nonCoded.StageSequence.Stages[0].GroupedPivot.CodedCorrelation != nil { + t.Fatalf("non-coded construction pivot shape drifted: %+v", nonCoded.StageSequence) + } + rendered, err := RenderPhysicalPlan(nonCoded, mustBinding(t)) + if err != nil { + t.Fatalf("staged non-coded pivot must render through the composition door (G4): %v", err) + } + if !strings.Contains(rendered.SQL, "WITH ") || !strings.Contains(rendered.SQL, "__loom_") { + t.Fatalf("staged non-coded pivot did not render a composed CTE chain:\n%s", rendered.SQL) + } + windowed := basePlan() + windowed.PreviewSourceWindowByRootID = true + if _, err := RenderPhysicalPlan(windowed, mustBinding(t)); err == nil || + !strings.Contains(err.Error(), "preview source windowing is not yet implemented for PostgreSQL") { + t.Fatalf("plan-level window refusal drifted: %v", err) + } +} + +func TestPivot14ScalarRefusalRegister(t *testing.T) { + cases := []struct { + name string + mutate func(*ir.PhysicalPlan) + wantFragment string + }{ + {"stage sequence coexistence refused by the group door", func(p *ir.PhysicalPlan) { + p.StageSequence = &ir.PhysicalStageSequence{Stages: []ir.PhysicalConstructionStage{{Kind: ir.PhysicalStageGroupOp}}} + }, "registered group slice"}, + {"stage sequence coexistence refused by the pivot slice", func(p *ir.PhysicalPlan) { + p.StageSequence = &ir.PhysicalStageSequence{Stages: []ir.PhysicalConstructionStage{{Kind: ir.PhysicalStageFilterOp}}} + }, "stage sequence"}, + {"plan-level preview window refused at the door", func(p *ir.PhysicalPlan) { + p.PreviewSourceWindowByRootID = true + }, "pg-sql-16"}, + {"second root scan", func(p *ir.PhysicalPlan) { + p.Operations = append(p.Operations[:1], + ir.PhysicalOperation{Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + p.Operations[1]) + }, "ROOT_SCAN"}, + {"traversal coexistence", func(p *ir.PhysicalPlan) { + p.Operations = append(p.Operations[:1], + ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "t", Direction: ir.PhysicalOutbound, EdgeLabelBindKey: "edge"}}, + p.Operations[1]) + }, "cannot coexist with GROUPED_PIVOT"}, + {"filter after pivot", func(p *ir.PhysicalPlan) { + p.Operations = append(p.Operations[:len(p.Operations)-1], + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "IS_NULL"}}}, + p.Operations[len(p.Operations)-1]) + }, "FILTER must precede the pivot"}, + {"second pivot", func(p *ir.PhysicalPlan) { + clone := *p.Operations[1].GroupedPivot + p.Operations = append(p.Operations[:2], + ir.PhysicalOperation{Kind: ir.PhysicalGroupedPivotOp, GroupedPivot: &clone}, + p.Operations[2]) + }, "exactly one reshape"}, + {"window sort drift", func(p *ir.PhysicalPlan) { + p.Operations = append(p.Operations[:2], + ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{ + Keys: []ir.PhysicalValue{{Variable: "prow", Path: []string{"other"}}}}}, + p.Operations[2]) + }, "must sort on prow.__loom_row_id exactly"}, + {"limit without sort", func(p *ir.PhysicalPlan) { + p.Operations = append(p.Operations[:2], + ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "limit"}}, + p.Operations[2]) + }, "requires the stable row identity SORT"}, + {"non-positive limit", func(p *ir.PhysicalPlan) { + p.Operations = append(p.Operations[:2], + ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{ + Keys: []ir.PhysicalValue{{Variable: "prow", Path: []string{"__loom_row_id"}}}}}, + ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "bad_limit"}}, + p.Operations[2]) + p.BindVars["bad_limit"] = 0 + }, "positive int"}, + {"missing leading ROOT_SCAN", func(p *ir.PhysicalPlan) { + p.Operations = p.Operations[1:] + }, "ROOT_SCAN"}, + {"ROOT_SCAN not first", func(p *ir.PhysicalPlan) { + p.Operations = append([]ir.PhysicalOperation{{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Predicate: ir.PhysicalPredicate{Operator: "IS_NULL"}}}}, p.Operations...) + }, "ROOT_SCAN"}, + {"population root", func(p *ir.PhysicalPlan) { + p.Operations[0].RootScan.Population = &ir.PhysicalPopulationRootSource{} + }, "population"}, + {"coded fields on the spine op", func(p *ir.PhysicalPlan) { + p.Operations[1].GroupedPivot.CodedCorrelation = &ir.PhysicalCorrelation{} + }, "coded Pivot fields"}, + {"row values", func(p *ir.PhysicalPlan) { + p.Operations[1].GroupedPivot.RowValues = []ir.PhysicalStageRowValue{{InputColumn: "gcol", Output: "rv", Variable: "rv0", Policy: "ALL", InputKind: "STRING"}} + }, "pg-sql-13"}, + {"root contributor columns", func(p *ir.PhysicalPlan) { + p.Operations[1].GroupedPivot.RootContributorVariable = "rc" + }, "root-contributor"}, + {"unregistered duplicate policy", func(p *ir.PhysicalPlan) { + p.Operations[1].GroupedPivot.DuplicatePolicy = "MEAN" + }, "duplicate policy"}, + {"IGNORE unlisted on the ordinary pivot", func(p *ir.PhysicalPlan) { + p.Operations[1].GroupedPivot.UnlistedCategoryPolicy = "IGNORE" + p.Operations[1].GroupedPivot.UnlistedEvidenceColumn = "" + }, "IGNORE is coded-only"}, + {"category kind disagreement", func(p *ir.PhysicalPlan) { + p.Operations[1].GroupedPivot.Categories[0].ValueKind = "INTEGER" + }, "disagrees with the frozen category type"}, + {"category bind wrong type", func(p *ir.PhysicalPlan) { + p.BindVars["cat_alpha"] = 7 + }, "must be a string"}, + {"missing category value bind", func(p *ir.PhysicalPlan) { + delete(p.BindVars, "cat_alpha") + }, "missing"}, + {"evidence column without EXCLUDE", func(p *ir.PhysicalPlan) { + p.Operations[1].GroupedPivot.UnlistedEvidenceColumn = "evidence_x" + }, "EXCLUDE_WITH_EVIDENCE"}, + {"consumed projection missing", func(p *ir.PhysicalPlan) { + p.Operations[1].GroupedPivot.InputProjections = p.Operations[1].GroupedPivot.InputProjections[:1] + }, "no input projection carries"}, + {"consumed projection twice", func(p *ir.PhysicalPlan) { + dup := p.Operations[1].GroupedPivot.InputProjections[1] + p.Operations[1].GroupedPivot.InputProjections = append(p.Operations[1].GroupedPivot.InputProjections, dup) + }, "projected twice"}, + {"return outside the registered names", func(p *ir.PhysicalPlan) { + returnOp := p.Operations[len(p.Operations)-1] + returnOp.Return.Projections = append(returnOp.Return.Projections, ir.PhysicalProjection{ + Name: "not_a_pivot_output", Value: ir.PhysicalValue{Variable: "prow", Path: []string{"not_a_pivot_output"}}}) + p.Operations[len(p.Operations)-1] = returnOp + }, "not a registered pivot output"}, + {"return path drift", func(p *ir.PhysicalPlan) { + returnOp := p.Operations[len(p.Operations)-1] + returnOp.Return.Projections[0].Value.Path = []string{"drifted"} + p.Operations[len(p.Operations)-1] = returnOp + }, "single-name read"}, + {"hostile category type", func(p *ir.PhysicalPlan) { + p.Operations[1].GroupedPivot.ValueType = "OBJECT" + }, "table scalar kind"}, + {"reducer on non-numeric value type", func(p *ir.PhysicalPlan) { + p.Operations[1].GroupedPivot.DuplicatePolicy = "SUM" + p.Operations[1].GroupedPivot.ValueType = "STRING" + }, "numeric value column"}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + control := pivot14ScalarPlan(t, pivot14ScalarConfig{}) + if _, err := RenderPhysicalPlan(control, mustBinding(t)); err != nil { + t.Fatalf("ALIVE control refused before any mutation: %v", err) + } + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{}) + test.mutate(&plan) + pivot14Refused(t, plan, test.name, test.wantFragment) + }) + } +} + +func TestPivot14ProductionFilterRendersAtAQLDivergence(t *testing.T) { + control := pivot14CompileBundle(t, pivot14ScalarBundle("p14_filter_control", nil, + recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, recipe.PivotUnlistedCategoryError), []string{pivot14AuthPath}) + if _, err := RenderPhysicalPlan(control, mustBinding(t)); err != nil { + t.Fatalf("ALIVE control (no filter) refused: %v", err) + } + plan := pivot14CompileBundle(t, pivot14ScalarBundle("p14_filter_divergence", pivot14FemaleFilter(), + recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, recipe.PivotUnlistedCategoryError), []string{pivot14AuthPath}) + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil { + t.Fatal("authored FILTER must refuse (probe P-14.2 renders-at-AQL divergence)") + } + if !errors.Is(err, ErrUnsupportedPostgresPlan) || !strings.Contains(err.Error(), "pg-sql-16") { + t.Fatalf("FILTER divergence refusal drifted: %v", err) + } +} + +var ( + pivot14FixturesMu sync.Mutex + pivot14TrackedFixtures = map[string]bool{} +) + +func pivot14LiveService(t *testing.T) *pgir02Service { + t.Helper() + svc := newOwnedService(t, 0, "verify_sql14v1") + var database string + if err := svc.iso.QueryRow(svc.ctx, "SELECT current_database()").Scan(&database); err != nil { + svc.fail("track owned database", err) + } + pivot14FixturesMu.Lock() + defer pivot14FixturesMu.Unlock() + pivot14TrackedFixtures[database] = true + pivot14TrackedFixtures[svc.schema] = true + return svc +} + +func pivot14Binding(t *testing.T, svc *pgir02Service) ir.PostgresStorageBinding { + t.Helper() + binding, err := ir.NewPostgresStorageBinding(svc.schema) + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + return binding +} + +func pivot14Seed(t *testing.T, svc *pgir02Service, logicalID string, payload any) { + t.Helper() + svc.insertResource(t, svc.schema, pivot14Project, pivot14Generation, "Patient", logicalID, pivot14AuthPath, payload) +} + +func pivot14SnapshotOwnedResources(ctx context.Context, raw string) ([]string, error) { + pool, err := pgxpool.New(ctx, raw) + if err != nil { + return nil, fmt.Errorf("connect: %w", err) + } + defer pool.Close() + rows, err := pool.Query(ctx, + `SELECT datname FROM pg_database WHERE datname LIKE 'verify_sql14v1_%'`+ + ` UNION ALL SELECT nspname FROM pg_namespace WHERE nspname LIKE 'verify_sql14v1_%'`) + if err != nil { + return nil, fmt.Errorf("snapshot: %w", err) + } + defer rows.Close() + var names []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("scan: %v", err) + } + names = append(names, name) + } + return names, rows.Err() +} + +func pivot14RunDeltaAudit(inner func() int) int { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return inner() + } + beforeList, err := pivot14SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-14 delta audit: snapshot before run: %v\n", err) + return inner() + } + if len(beforeList) > 0 { + fmt.Fprintf(os.Stderr, "pg-sql-14 delta audit: pre-snapshot found %d surviving verify_sql14v1_* resources before this run: %v\n", len(beforeList), beforeList) + return 1 + } + before := make(map[string]bool, len(beforeList)) + for _, name := range beforeList { + before[name] = true + } + code := inner() + after, err := pivot14SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-14 delta audit: snapshot after run: %v\n", err) + code = 1 + } + pivot14FixturesMu.Lock() + defer pivot14FixturesMu.Unlock() + for _, name := range after { + if before[name] { + continue + } + if pivot14TrackedFixtures[name] { + fmt.Fprintf(os.Stderr, "pg-sql-14 delta audit: %q was created and tracked by this run and still survives\n", name) + code = 1 + continue + } + fmt.Fprintf(os.Stderr, "pg-sql-14 delta audit: note — %q was born during this run but is not owned by it (sibling lane?)\n", name) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + survivors, err := pivot14SnapshotOwnedResources(ctx, raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-14 delta audit: survivor scan: %v\n", err) + code = 1 + } else if len(survivors) > 0 { + fmt.Fprintf(os.Stderr, "pg-sql-14 delta audit: %d verify_sql14v1_* resources survive: %v\n", len(survivors), survivors) + code = 1 + } + return code +} + +func TestZZNoLeftoverPivot14Resources(t *testing.T) { + if err := auditNoLeftoverVerifyNamespace(pivot14Prefix); err != nil { + t.Fatalf("zero-leftover census: %v", err) + } +} + +func pivot14SeedOracle(t *testing.T, svc *pgir02Service) { + t.Helper() + pivot14Seed(t, svc, "o1", json.RawMessage(`{"gender":"female","multipleBirthInteger":0}`)) + pivot14Seed(t, svc, "o2", json.RawMessage(`{"gender":"female","multipleBirthInteger":1}`)) + pivot14Seed(t, svc, "o3", json.RawMessage(`{"gender":"female","multipleBirthInteger":0}`)) + pivot14Seed(t, svc, "o4", json.RawMessage(`{"gender":"female","multipleBirthInteger":2}`)) + pivot14Seed(t, svc, "o5", json.RawMessage(`{"gender":"female"}`)) + pivot14Seed(t, svc, "o6", json.RawMessage(`{"gender":"female","multipleBirthInteger":null}`)) + pivot14Seed(t, svc, "o7", json.RawMessage(`{"gender":"male","multipleBirthInteger":1}`)) + for _, state := range []struct{ id, payload string }{ + {"o4", `{"gender":"female","multipleBirthInteger":2}`}, + {"o5", `{"gender":"female"}`}, + {"o6", `{"gender":"female","multipleBirthInteger":null}`}, + } { + rows := svc.witness(t, fmt.Sprintf( + `SELECT logical_id FROM %s.fhir_resource WHERE logical_id = $1 AND project = $2 AND dataset_generation = $3 AND payload = $4::jsonb`, + `"`+svc.schema+`"`), state.id, pivot14Project, pivot14Generation, state.payload) + if len(rows) != 1 { + t.Fatalf("stored state %s is not exactly %s — the fixture must not be vacuous", state.id, state.payload) + } + } +} + +func TestPivot14LiveScalarOracleMatrix(t *testing.T) { + svc := pivot14LiveService(t) + pivot14SeedOracle(t, svc) + + plan := pivot14CompileBundle(t, pivot14ScalarBundle("p14_live_sum", nil, + recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, recipe.PivotUnlistedCategoryExcludeWithEvidence), + []string{pivot14AuthPath}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render live SUM plan: %v", err) + } + rows := svc.runRendered(t, rendered) + if len(rows) != 2 { + t.Fatalf("live rows = %v, want the female and male groups", rows) + } + var groupIndex int + for index, name := range rendered.OutputColumns { + if name == "grp" { + groupIndex = index + } + } + byGroup := map[string][]string{} + for _, row := range rows { + byGroup[row[groupIndex]] = row + } + if _, ok := byGroup["female"]; !ok { + t.Fatalf("female group missing (identity/group key drifted): %v", rows) + } + var birthZero, birthOne int + for index, name := range rendered.OutputColumns { + switch name { + case "birth_zero": + birthZero = index + case "birth_one": + birthOne = index + } + } + if got := byGroup["female"][birthZero]; got != "0" { + t.Fatalf("SUM over the two category-0 rows = %q, want 0", got) + } + if got := byGroup["female"][birthOne]; got != "1" { + t.Fatalf("SUM over the single category-1 row = %q, want 1", got) + } + if got := byGroup["male"][birthZero]; got != "" && got != "null" { + t.Fatalf("male group has no category-0 rows; the cell must read JSON null, got %q", got) + } +} + +func TestPivot14LiveScalarReducerMissingRawCount(t *testing.T) { + svc := pivot14LiveService(t) + pivot14SeedOracle(t, svc) + plan := pivot14CompileBundle(t, pivot14ScalarBundle("p14_live_dup", nil, + recipe.PivotDuplicateError, recipe.PivotMissingCellNull, recipe.PivotUnlistedCategoryExcludeWithEvidence), + []string{pivot14AuthPath}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render live duplicate plan: %v", err) + } + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + conn, err := svc.iso.Acquire(svc.ctx) + if err != nil { + svc.fail("acquire pinned connection", err) + } + if _, err := conn.Exec(svc.ctx, "SET plan_cache_mode = "+mode); err != nil { + conn.Release() + svc.fail("pin plan_cache_mode "+mode, err) + } + if _, err := conn.Exec(svc.ctx, "EXPLAIN "+rendered.SQL, pivot14Args(rendered)...); err != nil { + conn.Release() + svc.fail("plan-time EXPLAIN raised under "+mode, err) + } + _, err = conn.Exec(svc.ctx, rendered.SQL, pivot14Args(rendered)...) + conn.Release() + if err == nil { + t.Fatalf("duplicate category-0 rows must raise under "+mode+", got success\n%s", rendered.SQL) + } + if !strings.Contains(err.Error(), "TABLE_PIVOT_CELL_CARDINALITY") || !strings.Contains(err.Error(), "22P02") { + t.Fatalf("duplicate raise drifted under %s: %v", mode, err) + } + } +} + +func TestPivot14LiveScalarMissingRawCountNoRaise(t *testing.T) { + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "s1", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":null}`)) + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{ + duplicate: "SUM", missing: "ERROR", unlisted: "EXCLUDE_WITH_EVIDENCE", + categories: []ir.PhysicalGroupedPivotCategory{ + {Output: "cell_alpha", MatchKind: ir.PhysicalPivotCategoryValueMatch, ValueBindKey: "cat_alpha", ValueKind: "STRING"}, + }, + }) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render scalar missing control: %v", err) + } + rows := svc.runRendered(t, rendered) + if len(rows) != 1 { + t.Fatalf("rows = %v, want the single group", rows) + } + if rows[0][1] != "" && rows[0][1] != "null" { + t.Fatalf("raw>0/typed=0 must read the null cell without raising, got %v", rows[0]) + } + + missingSvc := pivot14LiveService(t) + pivot14Seed(t, missingSvc, "s2", json.RawMessage(`{"gProp":"g2","cProp":"beta","vProp":1}`)) + missingPlan := pivot14ScalarPlan(t, pivot14ScalarConfig{ + duplicate: "SUM", missing: "ERROR", unlisted: "EXCLUDE_WITH_EVIDENCE", + categories: []ir.PhysicalGroupedPivotCategory{ + {Output: "cell_alpha", MatchKind: ir.PhysicalPivotCategoryValueMatch, ValueBindKey: "cat_alpha", ValueKind: "STRING"}, + }, + }) + missingRendered, err := RenderPhysicalPlan(missingPlan, pivot14Binding(t, missingSvc)) + if err != nil { + t.Fatalf("render scalar missing raise: %v", err) + } + _, err = missingSvc.iso.Exec(missingSvc.ctx, missingRendered.SQL, pivot14Args(missingRendered)...) + if err == nil { + t.Fatalf("raw count 0 must raise the scalar MISSING guard (:230):\n%s", missingRendered.SQL) + } + if !strings.Contains(err.Error(), "TABLE_PIVOT_CELL_MISSING") || !strings.Contains(err.Error(), "22P02") { + t.Fatalf("scalar MISSING raise drifted: %v", err) + } +} + +func TestPivot14LiveWindowOrderAndLimit(t *testing.T) { + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "w1", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":1}`)) + pivot14Seed(t, svc, "w2", json.RawMessage(`{"gProp":"g1","cProp":null,"vProp":2}`)) + pivot14Seed(t, svc, "w3", json.RawMessage(`{"gProp":"g1","vProp":3}`)) + pivot14Seed(t, svc, "w4", json.RawMessage(`{"gProp":"G2","cProp":"alpha","vProp":10}`)) + + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "ERROR", window: true, presence: "contract"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render windowed plan live: %v", err) + } + rows := svc.runRendered(t, rendered) + if len(rows) != 2 { + t.Fatalf("windowed rows = %v, want both groups under LIMIT 2", rows) + } + groupIndex := -1 + for index, name := range rendered.OutputColumns { + if name == "grp" { + groupIndex = index + } + } + if groupIndex < 0 { + t.Fatalf("group column missing from output columns: %v", rendered.OutputColumns) + } + if rows[0][groupIndex] != "G2" || rows[1][groupIndex] != "g1" { + t.Fatalf("identity COLLATE \"C\" order drifted (byte order puts G2 first): %v", rows) + } + cellIndex := map[string]int{} + for index, name := range rendered.OutputColumns { + cellIndex[name] = index + } + if got := rows[1][cellIndex["cell_alpha"]]; got != "1" { + t.Fatalf("g1 alpha cell = %q, want 1", got) + } + if got := rows[1][cellIndex["cell_null"]]; got != "2" { + t.Fatalf("g1 null cell = %q, want 2", got) + } + if got := rows[1][cellIndex["cell_missing"]]; got != "3" { + t.Fatalf("g1 missing cell = %q, want 3 (the presence contract proves cProp per row; the producer always carries it for MISSING kinds, ir/physical_table_shape.go re-read)", got) + } + + limited := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "ERROR", window: true, presence: "contract"}) + limited.BindVars["limit"] = 1 + limitedRendered, err := RenderPhysicalPlan(limited, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render limited plan live: %v", err) + } + limitedRows := svc.runRendered(t, limitedRendered) + if len(limitedRows) != 1 || limitedRows[0][groupIndex] != "G2" { + t.Fatalf("LIMIT prefix drifted: %v", limitedRows) + } +} + +func pivot14Args(rendered command.RenderedPostgresPhysicalPlan) []any { + args := make([]any, 0, len(rendered.Args)) + for _, parameter := range rendered.Args { + args = append(args, parameter.Value) + } + return args +} + +func specSelector(t *testing.T, fields ...string) spec.Selector { + t.Helper() + steps := make([]spec.SelectorStep, 0, len(fields)) + for _, field := range fields { + steps = append(steps, spec.SelectorStep{Field: field}) + } + return spec.Selector{Steps: steps} +} + +var _ = sort.Strings + +func pivot14ColIndex(t *testing.T, rendered command.RenderedPostgresPhysicalPlan, name string) int { + t.Helper() + for index, candidate := range rendered.OutputColumns { + if candidate == name { + return index + } + } + t.Fatalf("output column %q missing: %v", name, rendered.OutputColumns) + return -1 +} + +func pivot14ExpectGuardRaise(t *testing.T, svc *pgir02Service, rendered command.RenderedPostgresPhysicalPlan, wantTag string) { + t.Helper() + const sentinel = "PIVOT14FOLD42" + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + conn, err := svc.iso.Acquire(svc.ctx) + if err != nil { + svc.fail("acquire pinned connection", err) + } + if _, err := conn.Exec(svc.ctx, "SET plan_cache_mode = "+mode); err != nil { + conn.Release() + svc.fail("pin plan_cache_mode "+mode, err) + } + if _, err := conn.Exec(svc.ctx, "EXPLAIN "+rendered.SQL, pivot14Args(rendered)...); err != nil { + conn.Release() + svc.fail("plan-time EXPLAIN raised under "+mode, err) + } + if _, err := conn.Exec(svc.ctx, "SET loom.explorer_guard_barrier = "+quoteSQLLiteral(sentinel)); err != nil { + conn.Release() + svc.fail("set guard barrier under "+mode, err) + } + _, err = conn.Exec(svc.ctx, rendered.SQL, pivot14Args(rendered)...) + if _, resetErr := conn.Exec(svc.ctx, "RESET loom.explorer_guard_barrier"); resetErr != nil { + conn.Release() + svc.fail("reset guard barrier", resetErr) + } + conn.Release() + if err == nil { + t.Fatalf("guard %s must raise under %s:\n%s", wantTag, mode, rendered.SQL) + } + if message := err.Error(); !strings.Contains(message, "22P02") || + !strings.Contains(message, wantTag) || !strings.Contains(message, sentinel) { + t.Fatalf("guard %s raise drifted under %s: %v", wantTag, mode, err) + } + } +} + +func pivot14CellIsNull(value string) bool { + return value == "" || value == "null" +} + +func TestPivot14ProductionScalarLive(t *testing.T) { + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "s1", json.RawMessage(`{"gender":"female","multipleBirthInteger":0}`)) + pivot14Seed(t, svc, "s2", json.RawMessage(`{"gender":"female","multipleBirthInteger":0}`)) + pivot14Seed(t, svc, "s3", json.RawMessage(`{"gender":"female","multipleBirthInteger":1}`)) + pivot14Seed(t, svc, "s4", json.RawMessage(`{"gender":"female","multipleBirthInteger":1}`)) + pivot14Seed(t, svc, "s5", json.RawMessage(`{"gender":"female","multipleBirthInteger":null}`)) + pivot14Seed(t, svc, "s6", json.RawMessage(`{"gender":"female"}`)) + pivot14Seed(t, svc, "s7", json.RawMessage(`{"gender":"female","multipleBirthInteger":2}`)) + pivot14Seed(t, svc, "s8", json.RawMessage(`{"gender":"male","multipleBirthInteger":1}`)) + + cases := []struct { + name string + duplicate recipe.PivotDuplicatePolicy + birthZero string + birthOne string + }{ + {"sum", recipe.PivotDuplicateSum, "0", "2"}, + {"min", recipe.PivotDuplicateMin, "0", "1"}, + {"max", recipe.PivotDuplicateMax, "0", "1"}, + } + for _, testCase := range cases { + bundle := pivot14ScalarBundle("p14_prod_scalar_"+testCase.name, nil, + testCase.duplicate, recipe.PivotMissingCellNull, recipe.PivotUnlistedCategoryExcludeWithEvidence) + plan := pivot14CompileBundle(t, bundle, []string{pivot14AuthPath}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render %s plan: %v", testCase.name, err) + } + wantColumns := []string{"grp", "birth_zero", "birth_one", "__loom_reshape_unlisted_count", "__loom_row_id"} + if len(rendered.OutputColumns) != len(wantColumns) { + t.Fatalf("%s output columns = %v, want %v", testCase.name, rendered.OutputColumns, wantColumns) + } + for index, name := range wantColumns { + if rendered.OutputColumns[index] != name { + t.Fatalf("%s output column %d = %q, want %q", testCase.name, index, rendered.OutputColumns[index], name) + } + } + rows := svc.runRendered(t, rendered) + if len(rows) != 2 { + t.Fatalf("%s rows = %v, want the female and male groups", testCase.name, rows) + } + byGroup := map[string][]string{} + for _, row := range rows { + byGroup[row[pivot14ColIndex(t, rendered, "grp")]] = row + } + female, ok := byGroup["female"] + if !ok { + t.Fatalf("%s female group missing: %v", testCase.name, rows) + } + if got := female[pivot14ColIndex(t, rendered, "birth_zero")]; got != testCase.birthZero { + t.Fatalf("%s female birth_zero = %q, want %s", testCase.name, got, testCase.birthZero) + } + if got := female[pivot14ColIndex(t, rendered, "birth_one")]; got != testCase.birthOne { + t.Fatalf("%s female birth_one = %q, want %s", testCase.name, got, testCase.birthOne) + } + if got := female[pivot14ColIndex(t, rendered, "__loom_reshape_unlisted_count")]; got != "3" { + t.Fatalf("%s female unlisted evidence = %q, want 3", testCase.name, got) + } + male, ok := byGroup["male"] + if !ok { + t.Fatalf("%s male group missing: %v", testCase.name, rows) + } + if got := male[pivot14ColIndex(t, rendered, "birth_one")]; got != "1" { + t.Fatalf("%s male birth_one = %q, want 1", testCase.name, got) + } + if got := male[pivot14ColIndex(t, rendered, "birth_zero")]; !pivot14CellIsNull(got) { + t.Fatalf("%s male birth_zero = %q, want the null cell", testCase.name, got) + } + if got := male[pivot14ColIndex(t, rendered, "__loom_reshape_unlisted_count")]; got != "0" { + t.Fatalf("%s male unlisted evidence = %q, want 0", testCase.name, got) + } + for _, row := range rows { + group := row[pivot14ColIndex(t, rendered, "grp")] + want := `["GROUPED_PIVOT","p14_prod_scalar_` + testCase.name + `",["STRING","` + group + `"]]` + if got := row[pivot14ColIndex(t, rendered, "__loom_row_id")]; got != want { + t.Fatalf("%s identity = %q, want %s", testCase.name, got, want) + } + } + + foreign := pivot14CompileBundle(t, bundle, []string{"FHIR/Encounter"}) + foreignRendered, err := RenderPhysicalPlan(foreign, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render foreign-auth %s plan: %v", testCase.name, err) + } + if rows := svc.runRendered(t, foreignRendered); len(rows) != 0 { + t.Fatalf("%s foreign auth must yield zero rows, got %v", testCase.name, rows) + } + } + + bundle := pivot14ScalarBundle("p14_prod_scalar_window", nil, + recipe.PivotDuplicateSum, recipe.PivotMissingCellNull, recipe.PivotUnlistedCategoryExcludeWithEvidence) + plan := pivot14CompileBundle(t, bundle, []string{pivot14AuthPath}) + outputRowVariable := "" + for _, operation := range plan.Operations { + if operation.GroupedPivot != nil { + outputRowVariable = operation.GroupedPivot.OutputRowVariable + } + } + if outputRowVariable == "" { + t.Fatalf("production plan lost its GROUPED_PIVOT operation") + } + operations := make([]ir.PhysicalOperation, 0, len(plan.Operations)+2) + returnIndex := len(plan.Operations) - 1 + operations = append(operations, plan.Operations[:returnIndex]...) + operations = append(operations, + ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{ + Keys: []ir.PhysicalValue{{Variable: outputRowVariable, + Path: []string{"__loom_row_id"}}}}}, + ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "p14_prod_limit"}}) + operations = append(operations, plan.Operations[returnIndex]) + plan.Operations = operations + plan.BindVars["p14_prod_limit"] = 1 + windowRendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render windowed production plan: %v", err) + } + windowRows := svc.runRendered(t, windowRendered) + if len(windowRows) != 1 || windowRows[0][pivot14ColIndex(t, windowRendered, "grp")] != "female" { + t.Fatalf("production LIMIT prefix drifted: %v", windowRows) + } +} + +func TestPivot14LiveValueTypeRaise(t *testing.T) { + for _, value := range []string{"abc", "12.5"} { + t.Run("value_"+value, func(t *testing.T) { + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "vt1", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":"`+value+`"}`)) + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "SUM", missing: "NULL"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render value-type plan: %v", err) + } + pivot14ExpectGuardRaise(t, svc, rendered, pivotValueTypeTag) + }) + } + + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "vt0", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":4}`)) + clean := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "SUM", missing: "NULL"}) + cleanRendered, err := RenderPhysicalPlan(clean, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render clean value-type twin: %v", err) + } + rows := svc.runRendered(t, cleanRendered) + if len(rows) != 1 || rows[0][pivot14ColIndex(t, cleanRendered, "cell_alpha")] != "4" { + t.Fatalf("clean numeric data must read SUM 4 without a raise, got %v", rows) + } +} + +func TestPivot14LiveErrorBranchRawCountParity(t *testing.T) { + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "eb1", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":null}`)) + pivot14Seed(t, svc, "eb2", json.RawMessage(`{"gProp":"g1","cProp":"beta","vProp":7}`)) + pivot14Seed(t, svc, "eb3", json.RawMessage(`{"gProp":"g1","cProp":null,"vProp":4}`)) + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "ERROR", missing: "ERROR"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render ERROR-branch raw-count parity plan: %v", err) + } + rows := svc.runRendered(t, rendered) + if len(rows) != 1 { + t.Fatalf("parity rows = %v, want the single g1 group", rows) + } + row := rows[0] + if got := row[pivot14ColIndex(t, rendered, "cell_alpha")]; !pivot14CellIsNull(got) { + t.Fatalf("alpha cell = %q, want JSON null (rawCount=1, null-preserving FIRST over the raw list — no CELL_MISSING raise, no cell collapse)", got) + } + if got := row[pivot14ColIndex(t, rendered, "cell_beta")]; got != "7" { + t.Fatalf("beta cell = %q, want 7 (typed row reads normally)", got) + } + if got := row[pivot14ColIndex(t, rendered, "cell_null")]; got != "4" { + t.Fatalf("null-kind cell = %q, want 4", got) + } +} + +func TestPivot14LiveErrorBranchRawCountCardinalityAlive(t *testing.T) { + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "ec1", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":3}`)) + pivot14Seed(t, svc, "ec2", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":null}`)) + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "ERROR", missing: "ERROR"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render ERROR-branch cardinality ALIVE plan: %v", err) + } + pivot14ExpectGuardRaise(t, svc, rendered, pivotCardinalityTag) +} + +func TestPivot14LiveDuplicateCardinalityRaise(t *testing.T) { + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "dc1", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":3}`)) + pivot14Seed(t, svc, "dc2", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":4}`)) + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "ERROR", missing: "NULL"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render cardinality plan: %v", err) + } + pivot14ExpectGuardRaise(t, svc, rendered, pivotCardinalityTag) +} + +func TestPivot14LiveMissingCellRaise(t *testing.T) { + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "mc1", json.RawMessage(`{"gProp":"g1","cProp":"beta","vProp":5}`)) + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "SUM", missing: "ERROR"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render missing plan: %v", err) + } + pivot14ExpectGuardRaise(t, svc, rendered, pivotCellMissingTag) +} + +func TestPivot14LiveUnlistedRaise(t *testing.T) { + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "ul1", json.RawMessage(`{"gProp":"g1","cProp":"zeta","vProp":5}`)) + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "SUM", missing: "NULL", unlisted: "ERROR"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render unlisted plan: %v", err) + } + pivot14ExpectGuardRaise(t, svc, rendered, pivotUnlistedTag) +} + +func TestPivot14LiveUnlistedEvidenceRowsExact(t *testing.T) { + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "ev1", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":1}`)) + pivot14Seed(t, svc, "ev2", json.RawMessage(`{"gProp":"g1","cProp":"beta","vProp":2}`)) + pivot14Seed(t, svc, "ev3", json.RawMessage(`{"gProp":"g1","cProp":"zeta","vProp":9}`)) + pivot14Seed(t, svc, "ev4", json.RawMessage(`{"gProp":"g2","cProp":"alpha","vProp":7}`)) + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "SUM", missing: "NULL", unlisted: "EXCLUDE_WITH_EVIDENCE"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render evidence plan: %v", err) + } + rows := svc.runRendered(t, rendered) + if len(rows) != 2 { + t.Fatalf("evidence rows = %v", rows) + } + byGroup := map[string][]string{} + for _, row := range rows { + byGroup[row[pivot14ColIndex(t, rendered, "grp")]] = row + } + g1, ok := byGroup["g1"] + if !ok { + t.Fatalf("g1 group missing: %v", rows) + } + if got := g1[pivot14ColIndex(t, rendered, "cell_alpha")]; got != "1" { + t.Fatalf("g1 alpha cell = %q, want 1", got) + } + if got := g1[pivot14ColIndex(t, rendered, "cell_beta")]; got != "2" { + t.Fatalf("g1 beta cell = %q, want 2", got) + } + if got := g1[pivot14ColIndex(t, rendered, "unlisted_n")]; got != "1" { + t.Fatalf("g1 evidence = %q, want the kept unlisted count 1", got) + } + g2, ok := byGroup["g2"] + if !ok { + t.Fatalf("g2 group missing: %v", rows) + } + if got := g2[pivot14ColIndex(t, rendered, "unlisted_n")]; got != "0" { + t.Fatalf("g2 evidence = %q, want 0", got) + } + if got := g2[pivot14ColIndex(t, rendered, "cell_beta")]; !pivot14CellIsNull(got) { + t.Fatalf("g2 beta cell = %q, want the null cell", got) + } +} + +func TestPivot14LiveAbsentCompanionMissingParity(t *testing.T) { + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "ac1", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":5}`)) + pivot14Seed(t, svc, "ac2", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":1,"cPresent":true}`)) + pivot14Seed(t, svc, "ac3", json.RawMessage(`{"gProp":"g1","cProp":"beta","vProp":2,"cPresent":true}`)) + pivot14Seed(t, svc, "ac4", json.RawMessage(`{"gProp":"g1","cProp":null,"vProp":3,"cPresent":true}`)) + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "ERROR", missing: "ERROR", unlisted: "ERROR", presence: "from-input"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render absent-companion parity plan: %v", err) + } + rows := svc.runRendered(t, rendered) + if len(rows) != 1 { + t.Fatalf("parity rows = %v", rows) + } + row := rows[0] + for name, want := range map[string]string{ + "cell_alpha": "1", + "cell_beta": "2", + "cell_null": "3", + "cell_missing": "5", + } { + if got := row[pivot14ColIndex(t, rendered, name)]; got != want { + t.Fatalf("%s = %q, want %s (companion-absent row must land in MISSING, present-true rows keep their kinds)", name, got, want) + } + } +} + +func TestPivot14LiveAbsentCompanionEvidenceAndAliveControls(t *testing.T) { + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "ae1", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":5}`)) + pivot14Seed(t, svc, "ae2", json.RawMessage(`{"gProp":"g2","vProp":7,"cPresent":true}`)) + cats := []ir.PhysicalGroupedPivotCategory{ + {Output: "cell_alpha", MatchKind: ir.PhysicalPivotCategoryValueMatch, ValueBindKey: "cat_alpha", ValueKind: "STRING"}, + {Output: "cell_missing", MatchKind: ir.PhysicalPivotCategoryMissingMatch}, + } + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{ + duplicate: "SUM", missing: "NULL", unlisted: "EXCLUDE_WITH_EVIDENCE", + presence: "from-input", categories: cats, + }) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render evidence plan: %v", err) + } + rows := svc.runRendered(t, rendered) + byGroup := map[string][]string{} + for _, row := range rows { + byGroup[row[pivot14ColIndex(t, rendered, "grp")]] = row + } + g1, ok := byGroup["g1"] + if !ok { + t.Fatalf("g1 missing: %v", rows) + } + if got := g1[pivot14ColIndex(t, rendered, "unlisted_n")]; got != "0" { + t.Fatalf("g1 evidence = %q, want 0 (absent-companion row is LISTED as MISSING per AQL)", got) + } + if got := g1[pivot14ColIndex(t, rendered, "cell_missing")]; got != "5" { + t.Fatalf("g1 missing cell = %q, want 5 (the row must count in rawCount, not be dropped)", got) + } + g2, ok := byGroup["g2"] + if !ok { + t.Fatalf("g2 missing: %v", rows) + } + if got := g2[pivot14ColIndex(t, rendered, "unlisted_n")]; got != "1" { + t.Fatalf("g2 evidence = %q, want the kept unlisted count 1", got) + } + + svc2 := pivot14LiveService(t) + pivot14Seed(t, svc2, "al1", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":1,"cPresent":true}`)) + pivot14Seed(t, svc2, "al2", json.RawMessage(`{"gProp":"g1","cProp":"zeta","vProp":9,"cPresent":true}`)) + raisePlan := pivot14ScalarPlan(t, pivot14ScalarConfig{ + duplicate: "ERROR", missing: "NULL", unlisted: "ERROR", + presence: "from-input", categories: cats, + }) + rendered2, err := RenderPhysicalPlan(raisePlan, pivot14Binding(t, svc2)) + if err != nil { + t.Fatalf("render unlisted ALIVE plan: %v", err) + } + pivot14ExpectGuardRaise(t, svc2, rendered2, pivotUnlistedTag) + + svc3 := pivot14LiveService(t) + pivot14Seed(t, svc3, "am1", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":1,"cPresent":true}`)) + missingPlan := pivot14ScalarPlan(t, pivot14ScalarConfig{ + duplicate: "ERROR", missing: "ERROR", unlisted: "ERROR", + presence: "from-input", categories: cats, + }) + rendered3, err := RenderPhysicalPlan(missingPlan, pivot14Binding(t, svc3)) + if err != nil { + t.Fatalf("render cell-missing ALIVE plan: %v", err) + } + pivot14ExpectGuardRaise(t, svc3, rendered3, pivotCellMissingTag) +} + +func TestPivot14AbsentVersusNullAliveControl(t *testing.T) { + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "an1", json.RawMessage(`{"gProp":"g1","cProp":null,"vProp":9}`)) + pivot14Seed(t, svc, "an2", json.RawMessage(`{"gProp":"g1","vProp":8}`)) + pivot14Seed(t, svc, "an3", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":1}`)) + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "SUM", missing: "NULL", presence: "contract"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render absent-versus-null plan: %v", err) + } + rows := svc.runRendered(t, rendered) + if len(rows) != 1 { + t.Fatalf("absent-versus-null rows = %v, want one group", rows) + } + row := rows[0] + if got := row[pivot14ColIndex(t, rendered, "cell_null")]; got != "9" { + t.Fatalf("stored-null row must land in the NULL cell, got %q (collapse to MISSING is the plan §7 failure)", got) + } + if got := row[pivot14ColIndex(t, rendered, "cell_missing")]; got != "8" { + t.Fatalf("absent row must land in the MISSING cell, got %q", got) + } + if got := row[pivot14ColIndex(t, rendered, "cell_alpha")]; got != "1" { + t.Fatalf("alpha cell = %q, want 1", got) + } +} + +func TestPivot14UnlistedEvidenceAliveControl(t *testing.T) { + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "ue1", json.RawMessage(`{"gProp":"g_ok","cProp":"alpha","vProp":1}`)) + pivot14Seed(t, svc, "ue2", json.RawMessage(`{"gProp":"g_mix","cProp":"alpha","vProp":2}`)) + pivot14Seed(t, svc, "ue3", json.RawMessage(`{"gProp":"g_mix","cProp":"zeta","vProp":5}`)) + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "SUM", missing: "NULL", unlisted: "EXCLUDE_WITH_EVIDENCE"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render evidence control plan: %v", err) + } + rows := svc.runRendered(t, rendered) + if len(rows) != 2 { + t.Fatalf("evidence control rows = %v", rows) + } + byGroup := map[string][]string{} + for _, row := range rows { + byGroup[row[pivot14ColIndex(t, rendered, "grp")]] = row + } + okRow, ok := byGroup["g_ok"] + mixRow, mix := byGroup["g_mix"] + if !ok || !mix { + t.Fatalf("evidence control groups missing: %v", rows) + } + if got := okRow[pivot14ColIndex(t, rendered, "unlisted_n")]; got != "0" { + t.Fatalf("all-listed group evidence = %q, want 0", got) + } + if got := mixRow[pivot14ColIndex(t, rendered, "unlisted_n")]; got != "1" { + t.Fatalf("mixed group evidence = %q, want 1 (0≠1 is the non-vacuity differentiator)", got) + } +} + +func TestPivot14ReducerTypedOnlyAliveControl(t *testing.T) { + svc := pivot14LiveService(t) + pivot14Seed(t, svc, "rt1", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":1}`)) + pivot14Seed(t, svc, "rt2", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":null}`)) + pivot14Seed(t, svc, "rt3", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":2}`)) + pivot14Seed(t, svc, "rt4", json.RawMessage(`{"gProp":"g1","cProp":"beta","vProp":5}`)) + pivot14Seed(t, svc, "rt5", json.RawMessage(`{"gProp":"g1","cProp":null,"vProp":6}`)) + plan := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "SUM", missing: "ERROR"}) + rendered, err := RenderPhysicalPlan(plan, pivot14Binding(t, svc)) + if err != nil { + t.Fatalf("render typed-only plan: %v", err) + } + rows := svc.runRendered(t, rendered) + if len(rows) != 1 { + t.Fatalf("typed-only rows = %v, want one group", rows) + } + if got := rows[0][pivot14ColIndex(t, rendered, "cell_alpha")]; got != "3" { + t.Fatalf("alpha reducer must skip JSON null (SUM 1+2=3), got %q", got) + } + + badSvc := pivot14LiveService(t) + pivot14Seed(t, badSvc, "rb1", json.RawMessage(`{"gProp":"g1","cProp":"alpha","vProp":1}`)) + pivot14Seed(t, badSvc, "rb2", json.RawMessage(`{"gProp":"g1","cProp":"beta","vProp":"abc"}`)) + pivot14Seed(t, badSvc, "rb3", json.RawMessage(`{"gProp":"g1","cProp":null,"vProp":6}`)) + badPlan := pivot14ScalarPlan(t, pivot14ScalarConfig{duplicate: "SUM", missing: "ERROR"}) + badRendered, err := RenderPhysicalPlan(badPlan, pivot14Binding(t, badSvc)) + if err != nil { + t.Fatalf("render typed-guard-live plan: %v", err) + } + pivot14ExpectGuardRaise(t, badSvc, badRendered, pivotValueTypeTag) +} diff --git a/internal/dataframe/compiler/render/postgres/population.go b/internal/dataframe/compiler/render/postgres/population.go new file mode 100644 index 000000000..ea7d20cb6 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/population.go @@ -0,0 +1,802 @@ +package postgres + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +type postgresMembershipTableKind string + +const ( + membershipSelectionMembers postgresMembershipTableKind = "selection_members" + membershipSelectionRevisions postgresMembershipTableKind = "selection_revisions" + membershipGroupRevisions postgresMembershipTableKind = "explorer_explicit_group_revisions" + membershipGroupMemberships postgresMembershipTableKind = "explorer_explicit_group_memberships" +) + +var canonicalMembershipCollections = map[string]postgresMembershipTableKind{ + "loom_explorer_selection_members": membershipSelectionMembers, + "loom_explorer_selections": membershipSelectionRevisions, + "loom_explorer_explicit_group_revisions": membershipGroupRevisions, + "loom_explorer_explicit_group_memberships": membershipGroupMemberships, +} + +type postgresMembershipTableSpec struct { + physicalTable string + columns map[string]struct{} +} + +func membershipColumnSet(names ...string) map[string]struct{} { + set := make(map[string]struct{}, len(names)) + for _, name := range names { + set[name] = struct{}{} + } + return set +} + +var postgresMembershipTables = map[postgresMembershipTableKind]postgresMembershipTableSpec{ + membershipSelectionMembers: { + physicalTable: "selection_members", + columns: membershipColumnSet( + "selection_id", "member_key", "project", "generation", + "resource_type", "id", "ordinal", + ), + }, + membershipSelectionRevisions: { + physicalTable: "selection_revisions", + columns: membershipColumnSet( + "id", "project", "generation", "resource_type", + "scope_digest", "membership_digest", "complete", + ), + }, + membershipGroupRevisions: { + physicalTable: "explorer_explicit_group_revisions", + columns: membershipColumnSet( + "id", "project", "generation", "resource_type", "scope_digest", + "source_selection_revision_id", "source_membership_digest", + "membership_digest", "state", + ), + }, + membershipGroupMemberships: { + physicalTable: "explorer_explicit_group_memberships", + columns: membershipColumnSet( + "revision_id", "group_id", "project", "generation", + "resource_type", "resource_id", + ), + }, +} + +var membershipSchemaPattern = regexp.MustCompile(`^[a-z_][a-z0-9_]{0,62}$`) + +func quoteMembershipIdentifier(name string) string { + out := make([]byte, 0, len(name)+2) + out = append(out, '"') + for index := 0; index < len(name); index++ { + if name[index] == '"' { + out = append(out, '"') + } + out = append(out, name[index]) + } + out = append(out, '"') + return string(out) +} + +type PostgresPopulationRootBinding struct { + schema string +} + +func NewPostgresPopulationRootBinding(schema string) (PostgresPopulationRootBinding, error) { + if !membershipSchemaPattern.MatchString(schema) { + return PostgresPopulationRootBinding{}, fmt.Errorf("%w: membership schema name %q must match [a-z_][a-z0-9_]{0,62}", + ir.ErrUnregisteredPostgresIdentifier, schema) + } + return PostgresPopulationRootBinding{schema: schema}, nil +} + +func (b PostgresPopulationRootBinding) Validate() error { + if !membershipSchemaPattern.MatchString(b.schema) { + return fmt.Errorf("%w: membership binding schema %q is not registered", + ir.ErrUnregisteredPostgresIdentifier, b.schema) + } + return nil +} + +func (b PostgresPopulationRootBinding) tableSQL(kind postgresMembershipTableKind) string { + spec, ok := postgresMembershipTables[kind] + if !ok { + panic(fmt.Sprintf("population registry: unknown membership table kind %q", string(kind))) + } + return quoteMembershipIdentifier(b.schema) + "." + quoteMembershipIdentifier(spec.physicalTable) +} + +func (b PostgresPopulationRootBinding) columnSQL(kind postgresMembershipTableKind, column string) string { + spec, ok := postgresMembershipTables[kind] + if !ok { + panic(fmt.Sprintf("population registry: unknown membership table kind %q", string(kind))) + } + if _, allowed := spec.columns[column]; !allowed { + panic(fmt.Sprintf("unregistered postgres identifier: column %q is not a column of membership relation %q", + column, string(kind))) + } + return quoteMembershipIdentifier(column) +} + +const ( + cohortGuardRevisionInvalid = "EXPLICIT_GROUP_REVISION_INVALID" + cohortGuardSourceStale = "EXPLICIT_GROUP_SOURCE_STALE" + cohortGuardUnassignedMember = "EXPLICIT_GROUP_UNASSIGNED_MEMBER" + + cohortStateComplete = "COMPLETE" + cohortPolicyError = "ERROR" +) + +var canonicalCohortPolicies = map[string]bool{ + "ERROR": true, "EXCLUDE": true, "GROUP_AS_UNASSIGNED": true, +} + +var populationMemberFilterOrder = []struct { + memberColumn string + planPath string + canonicalBind string + nonEmpty bool +}{ + {"selection_id", "selectionId", "", true}, + {"project", "project", "", true}, + {"generation", "generation", datasetGenerationBind, false}, + {"resource_type", "resourceType", "", true}, +} + +func ValidatePopulationRootSource(plan ir.PhysicalPlan, binding PostgresPopulationRootBinding) error { + if err := binding.Validate(); err != nil { + return err + } + if err := plan.Validate(); err != nil { + return fmt.Errorf("validate physical plan: %w", err) + } + if len(plan.Operations) == 0 { + return fmt.Errorf("%w: plan has no operations", ErrUnsupportedPostgresPlan) + } + root := plan.Operations[0] + if root.Kind != ir.PhysicalRootScanOp || root.RootScan == nil { + return fmt.Errorf("%w: the leading operation must be ROOT_SCAN with its payload set", ErrUnsupportedPostgresPlan) + } + scan := *root.RootScan + if scan.Population == nil && scan.CohortSource == nil { + return fmt.Errorf("%w: the plan carries no membership root source; the direct root scan is registered by root_scan.go, not this slice", + ErrUnsupportedPostgresPlan) + } + if scan.Population != nil && scan.CohortSource != nil { + return fmt.Errorf("%w: root scan cannot combine cohort and population sources", ErrUnsupportedPostgresPlan) + } + if scan.PageCandidateSeed != nil { + return fmt.Errorf("%w: membership roots cannot combine with the candidate seed, which stays unregistered", + ErrUnsupportedPostgresPlan) + } + if err := requireMembershipRootBinds(scan); err != nil { + return err + } + if scan.Population != nil { + _, err := validatePopulationPayload(plan, scan, false) + return err + } + return validateCohortPayload(plan, scan) +} + +func requireMembershipRootBinds(scan ir.PhysicalRootScan) error { + if strings.TrimSpace(scan.Variable) == "" { + return fmt.Errorf("%w: ROOT_SCAN must name its row variable", ErrUnsupportedPostgresPlan) + } + if strings.TrimSpace(scan.CollectionBindKey) == "" { + return fmt.Errorf("%w: ROOT_SCAN must name the bind carrying the single resource type", ErrUnsupportedPostgresPlan) + } + return nil +} + +func RenderPopulationRootScan( + plan ir.PhysicalPlan, + storage ir.PostgresStorageBinding, + binding PostgresPopulationRootBinding, +) (command.RenderedPostgresPhysicalPlan, error) { + if err := storage.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := ValidatePopulationRootSource(plan, binding); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalUnnestOp { + if err := validateUnnestPlanSurface(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + break + } + } + if plan.Engine != "" && plan.Engine != ir.PhysicalEngineAQL { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: engine %q has no PostgreSQL source; the source graph IR is backend-neutral and ClickHouse plans are not PostgreSQL-rendered", + ErrUnsupportedPostgresPlan, string(plan.Engine)) + } + if plan.ClickHouseCombine != nil || plan.ClickHousePrefix != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: ClickHouse combine/prefix payloads have no PostgreSQL rendering", ErrUnsupportedPostgresPlan) + } + if plan.StageSequence != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: staged row-operator sequences are not yet implemented for PostgreSQL", ErrUnsupportedPostgresPlan) + } + if plan.PreviewSourceWindowByRootID { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: preview source windowing is not yet implemented for PostgreSQL", ErrUnsupportedPostgresPlan) + } + if len(plan.DeferredExpressionLets) != 0 { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: completed plans must not carry deferred expression lets", ErrUnsupportedPostgresPlan) + } + bound := binding + renderer := &planRenderer{ + storage: storage, + rows: map[string]rowScope{}, + parameters: map[string]int{}, + membership: &bound, + } + if err := renderer.loadScope(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := renderer.renderOperations(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return command.NewRenderedPostgresPhysicalPlan(renderer.assemble(), renderer.args, renderer.outputColumns) +} + +func (r *planRenderer) renderRootSources(plan ir.PhysicalPlan, scan ir.PhysicalRootScan) error { + if scan.Population != nil && scan.CohortSource != nil { + return fmt.Errorf("%w: root scan cannot combine cohort and population sources", ErrUnsupportedPostgresPlan) + } + if scan.PageCandidateSeed != nil { + return fmt.Errorf("%w: membership roots cannot combine with the candidate seed, which stays unregistered", + ErrUnsupportedPostgresPlan) + } + if r.membership == nil { + return fmt.Errorf("%w: population and cohort roots require the membership binding registered by RenderPopulationRootScan; the plain spine keeps them a capability gap", + ErrUnsupportedPostgresPlan) + } + if err := r.membership.Validate(); err != nil { + return err + } + resourceType, ok := plan.BindVars[scan.CollectionBindKey].(string) + if !ok || strings.TrimSpace(resourceType) == "" { + return fmt.Errorf("%w: root collection bind %q must name one resource type as a non-empty string", + ErrUnsupportedPostgresPlan, scan.CollectionBindKey) + } + r.rootAlias = "r0" + r.rows[scan.Variable] = rowScope{alias: r.rootAlias, resourceType: resourceType} + denyConjunct := " AND FALSE" + if scan.Population == nil { + denyConjunct = "" + } + r.where = append(r.where, r.scopedResourcePredicateDenyConjunct(r.rootAlias, + r.textParameter(resourceType, "text:"+resourceType), denyConjunct)) + if scan.Population != nil { + return r.renderPopulationRootSource(plan, scan) + } + return r.renderCohortRootSource(plan, scan) +} + +type populationRootShape struct { + memberVariable string + sourceVariable string + sourceType string + terminalVariable string + memberBindKeys [4]string +} + +func validatePopulationPayload(plan ir.PhysicalPlan, scan ir.PhysicalRootScan, witness bool) (populationRootShape, error) { + var shape populationRootShape + population := scan.Population + if !witness && population.CollectMembersVariable != "" { + return shape, fmt.Errorf("%w: population member witnesses (CollectMembersVariable %q) belong to the pg-sql-22 population terminal and are not registered by pg-sql-18 (D2)", + ErrUnsupportedPostgresPlan, population.CollectMembersVariable) + } + if witness && population.CollectMembersVariable != ir.PopulationMappingMembersVariable { + return shape, fmt.Errorf("%w: the member witness is registered only for CollectMembersVariable %q, got %q", + ErrUnsupportedPostgresPlan, ir.PopulationMappingMembersVariable, population.CollectMembersVariable) + } + shape.memberVariable = population.MemberScan.Variable + if shape.memberVariable == "" { + return shape, fmt.Errorf("%w: population member scan must name its variable", ErrUnsupportedPostgresPlan) + } + if err := requireMembershipCollection(plan, population.MemberScan.CollectionBindKey, membershipSelectionMembers); err != nil { + return shape, err + } + if len(population.MemberFilters) != len(populationMemberFilterOrder) { + return shape, fmt.Errorf("%w: population member scan must carry exactly the %d canonical member filters of lower/population.go, got %d", + ErrUnsupportedPostgresPlan, len(populationMemberFilterOrder), len(population.MemberFilters)) + } + if err := matchMemberFilters(plan, population, &shape); err != nil { + return shape, err + } + + operations := population.ResourceOperations + if len(operations) == 0 || operations[0].Kind != ir.PhysicalCollectionScanOp || operations[0].CollectionScan == nil { + return shape, fmt.Errorf("%w: population resource operations must begin with the single source COLLECTION_SCAN of lower/population.go:59", + ErrUnsupportedPostgresPlan) + } + sourceScan := operations[0].CollectionScan + shape.sourceVariable = sourceScan.Variable + if shape.sourceVariable == "" || shape.sourceVariable == shape.memberVariable { + return shape, fmt.Errorf("%w: population source scan must name its own variable", ErrUnsupportedPostgresPlan) + } + sourceType, ok := plan.BindVars[sourceScan.CollectionBindKey].(string) + if !ok || strings.TrimSpace(sourceType) == "" { + return shape, fmt.Errorf("%w: population source collection bind %q must name one resource type as a non-empty string", + ErrUnsupportedPostgresPlan, sourceScan.CollectionBindKey) + } + shape.sourceType = sourceType + + index := 1 + if err := matchCanonicalScopeBlock(operations, &index, []string{shape.sourceVariable}); err != nil { + return shape, fmt.Errorf("population source scope block: %w: %w", ErrUnsupportedPostgresPlan, err) + } + if index >= len(operations) || operations[index].Kind != ir.PhysicalFilterOp || operations[index].Filter == nil { + return shape, fmt.Errorf("%w: population resource operations must carry the source-id/member-id join FILTER after the source scope block", + ErrUnsupportedPostgresPlan) + } + memberID := population.MemberID + if memberID.Variable != shape.memberVariable || len(memberID.Path) != 1 || memberID.Path[0] != "id" || memberID.BindKey != "" { + return shape, fmt.Errorf("%w: population MemberID must be the member-variable logical id", ErrUnsupportedPostgresPlan) + } + if operations[index].Filter.Expression != nil || + !sameCanonicalPredicate(operations[index].Filter.Predicate, + ir.PhysicalValue{Variable: shape.sourceVariable, Path: []string{"id"}}, + ir.PhysicalValue{Variable: memberID.Variable, Path: []string{"id"}}) { + return shape, fmt.Errorf("%w: population source-id/member-id join must be exactly %s.id == %s.id (EQUALS, no quantifier or correlation)", + ErrUnsupportedPostgresPlan, shape.sourceVariable, shape.memberVariable) + } + index++ + + terminalVariable := shape.sourceVariable + terminalType := sourceType + for step := 0; index < len(operations); step++ { + operation := operations[index] + if operation.Kind != ir.PhysicalTraversalOp || operation.Traversal == nil { + return shape, fmt.Errorf("%w: population resource operation %d has kind %q; the route is a TRAVERSAL followed by its canonical scope block", + ErrUnsupportedPostgresPlan, index, string(operation.Kind)) + } + traversal := *operation.Traversal + if traversal.SourceVariable != terminalVariable { + return shape, fmt.Errorf("%w: population route step %d must anchor on %q (the source scan or the previous step's reached row), got %q", + ErrUnsupportedPostgresPlan, index, terminalVariable, traversal.SourceVariable) + } + verified, err := probeTraversalContract(plan, traversal, rowScope{resourceType: terminalType}) + if err != nil { + return shape, fmt.Errorf("population route step %d: %w: %w", index, ErrUnsupportedPostgresPlan, err) + } + index++ + if err := matchCanonicalScopeBlock(operations, &index, []string{traversal.EdgeVariable, traversal.TargetVariable}); err != nil { + return shape, fmt.Errorf("population route step %d scope block: %w: %w", index, ErrUnsupportedPostgresPlan, err) + } + terminalVariable = traversal.TargetVariable + terminalType = verified.targetType + } + if population.RootKey.Variable != terminalVariable || + len(population.RootKey.Path) != 1 || population.RootKey.Path[0] != "_key" || + population.RootKey.BindKey != "" { + return shape, fmt.Errorf("%w: population RootKey must be the final route row's storage identity (the terminal _key of lower/population.go:102), got variable %q path %v", + ErrUnsupportedPostgresPlan, population.RootKey.Variable, population.RootKey.Path) + } + if rootType, ok := plan.BindVars[scan.CollectionBindKey].(string); !ok || strings.TrimSpace(rootType) == "" { + return shape, fmt.Errorf("%w: root collection bind %q must name one resource type as a non-empty string", + ErrUnsupportedPostgresPlan, scan.CollectionBindKey) + } else if rootType != terminalType { + return shape, fmt.Errorf("%w: population root collection type %q must match the final route row type %q", + ErrUnsupportedPostgresPlan, rootType, terminalType) + } + shape.terminalVariable = terminalVariable + return shape, nil +} + +func probeTraversalContract(plan ir.PhysicalPlan, traversal ir.PhysicalTraversal, source rowScope) (validatedTraversal, error) { + probe := &planRenderer{parameters: map[string]int{}} + return probe.validateTraversalContract(plan, traversal, source) +} + +func matchMemberFilters(plan ir.PhysicalPlan, population *ir.PhysicalPopulationRootSource, shape *populationRootShape) error { + for position, expected := range populationMemberFilterOrder { + filter := population.MemberFilters[position] + if filter.Expression != nil { + return fmt.Errorf("%w: member filter %d must use the canonical Predicate form of lower/population.go", ErrUnsupportedPostgresPlan, position) + } + predicate := filter.Predicate + if !strings.EqualFold(strings.TrimSpace(predicate.Operator), "EQUALS") || + predicate.Left.Variable != population.MemberScan.Variable || + len(predicate.Left.Path) != 1 || predicate.Left.Path[0] != expected.planPath || + predicate.Left.BindKey != "" || + predicate.Right == nil || predicate.Right.BindKey == "" || + predicate.Right.Variable != "" || len(predicate.Right.Path) != 0 || + predicate.Quantifier != "" || predicate.Correlation != nil || predicate.LeftExpression != nil { + return fmt.Errorf("%w: member filter %d must be exactly %s.%s == @bind (EQUALS, member-variable path, bind-backed right side)", + ErrUnsupportedPostgresPlan, position, population.MemberScan.Variable, expected.planPath) + } + value, ok := plan.BindVars[predicate.Right.BindKey].(string) + if !ok { + return fmt.Errorf("%w: member filter %d right bind %q must be a string", ErrUnsupportedPostgresPlan, position, predicate.Right.BindKey) + } + if expected.nonEmpty && strings.TrimSpace(value) == "" { + return fmt.Errorf("%w: member filter %d bind %q must be a non-empty string", ErrUnsupportedPostgresPlan, position, predicate.Right.BindKey) + } + if expected.canonicalBind != "" { + if predicate.Right.BindKey != expected.canonicalBind { + return fmt.Errorf("%w: member filter %d must bind the plan-canonical %q key (D4: the pinned member set is generation-scoped to the plan)", + ErrUnsupportedPostgresPlan, position, expected.canonicalBind) + } + planGeneration, ok := plan.BindVars[datasetGenerationBind].(string) + if !ok || planGeneration != value { + return fmt.Errorf("%w: member filter %d must carry the plan dataset_generation value exactly", ErrUnsupportedPostgresPlan, position) + } + } + shape.memberBindKeys[position] = predicate.Right.BindKey + } + return nil +} + +func matchCanonicalScopeBlock(operations []ir.PhysicalOperation, index *int, variables []string) error { + expectedLength := 2*len(variables) + 2 + if *index+expectedLength > len(operations) { + return fmt.Errorf("requires %d operations for the project, dataset_generation, and auth legs of %s", + expectedLength, strings.Join(variables, ", ")) + } + for position, variable := range variables { + operation := operations[*index+position] + if operation.Kind != ir.PhysicalFilterOp || operation.Filter == nil || + !sameCanonicalPredicate(operation.Filter.Predicate, + ir.PhysicalValue{Variable: variable, Path: []string{"project"}}, + ir.PhysicalValue{BindKey: projectBindKey}) { + return fmt.Errorf("project scope must be %s.project == @%s", variable, projectBindKey) + } + } + for position, variable := range variables { + operation := operations[*index+len(variables)+position] + if operation.Kind != ir.PhysicalFilterOp || operation.Filter == nil || + !sameCanonicalPredicate(operation.Filter.Predicate, + ir.PhysicalValue{Variable: variable, Path: []string{"dataset_generation"}}, + ir.PhysicalValue{BindKey: datasetGenerationBind}) { + return fmt.Errorf("dataset generation scope must be %s.dataset_generation == @%s", variable, datasetGenerationBind) + } + } + authLet := operations[*index+2*len(variables)] + if authLet.Kind != ir.PhysicalDerivedLetOp || authLet.DerivedLet == nil || + strings.ToUpper(strings.TrimSpace(authLet.DerivedLet.Operator)) != "AUTH_RESOURCE_PATH_ALLOWED" || + authLet.DerivedLet.Variable == "" { + return fmt.Errorf("scope block requires the AUTH_RESOURCE_PATH_ALLOWED LET after the dataset generation filters") + } + expectedInputs := make([]ir.PhysicalValue, 0, len(variables)+2) + for _, variable := range variables { + expectedInputs = append(expectedInputs, ir.PhysicalValue{Variable: variable, Path: []string{"auth_resource_path"}}) + } + expectedInputs = append(expectedInputs, + ir.PhysicalValue{BindKey: authPathsBindKey}, + ir.PhysicalValue{BindKey: authUnrestrictedBind}) + if len(authLet.DerivedLet.Inputs) != len(expectedInputs) { + return fmt.Errorf("AUTH_RESOURCE_PATH_ALLOWED requires the exact canonical auth scope inputs") + } + for position, wanted := range expectedInputs { + if !sameCanonicalPhysicalValue(authLet.DerivedLet.Inputs[position], wanted) { + return fmt.Errorf("AUTH_RESOURCE_PATH_ALLOWED input %d must be %v", position, wanted) + } + } + authEquality := operations[*index+2*len(variables)+1] + if authEquality.Kind != ir.PhysicalFilterOp || authEquality.Filter == nil || + !sameCanonicalPredicate(authEquality.Filter.Predicate, + ir.PhysicalValue{Variable: authLet.DerivedLet.Variable}, + ir.PhysicalValue{BindKey: scopeAllowedBind}) { + return fmt.Errorf("scope block must close with FILTER %s == @%s", authLet.DerivedLet.Variable, scopeAllowedBind) + } + *index += expectedLength + return nil +} + +func sameCanonicalPredicate(predicate ir.PhysicalPredicate, left, right ir.PhysicalValue) bool { + if !strings.EqualFold(strings.TrimSpace(predicate.Operator), "EQUALS") { + return false + } + if predicate.Quantifier != "" || predicate.Correlation != nil || predicate.LeftExpression != nil { + return false + } + if !sameCanonicalPhysicalValue(predicate.Left, left) { + return false + } + return predicate.Right != nil && sameCanonicalPhysicalValue(*predicate.Right, right) +} + +func sameCanonicalPhysicalValue(got, wanted ir.PhysicalValue) bool { + if got.Variable != wanted.Variable || got.BindKey != wanted.BindKey || len(got.Path) != len(wanted.Path) { + return false + } + for index := range wanted.Path { + if got.Path[index] != wanted.Path[index] { + return false + } + } + return true +} + +func (r *planRenderer) renderPopulationRootSource(plan ir.PhysicalPlan, scan ir.PhysicalRootScan) error { + shape, err := validatePopulationPayload(plan, scan, r.mapping != nil && r.mapping.witness) + if err != nil { + return err + } + memberResourceType, ok := plan.BindVars[shape.memberBindKeys[3]].(string) + if !ok || memberResourceType != shape.sourceType { + return fmt.Errorf("%w: population member resource type must equal the source collection type %q (lower/population.go binds one value to both)", + ErrUnsupportedPostgresPlan, shape.sourceType) + } + + memberAlias := "pm" + memberTable := r.membership.tableSQL(membershipSelectionMembers) + predicates := make([]string, 0, 4) + for position, expected := range populationMemberFilterOrder { + value, _ := plan.BindVars[shape.memberBindKeys[position]].(string) + predicates = append(predicates, + memberAlias+"."+r.membership.columnSQL(membershipSelectionMembers, expected.memberColumn)+ + " = $"+strconv.Itoa(r.textParameter(value, "pop-member:"+expected.memberColumn+":"+value))) + } + + resource := r.storage.Resource + sourceAlias := "x0" + sourcePredicate := r.scopedResourcePredicate(sourceAlias, + r.textParameter(shape.sourceType, "text:"+shape.sourceType)) + sourcePredicate += " AND " + sourceAlias + "." + resource.MustColumn("logical_id").SQLText() + + " = " + memberAlias + "." + r.membership.columnSQL(membershipSelectionMembers, "id") + fromParts := []string{ + memberTable + " AS " + memberAlias, + "INNER JOIN " + resource.Identifier().SQLText() + " AS " + sourceAlias + " ON " + sourcePredicate, + } + + operations := scan.Population.ResourceOperations + index := 1 + (2*1 + 2) + 1 + currentAlias := sourceAlias + currentType := shape.sourceType + for index < len(operations) { + traversal := operations[index].Traversal + sourceScope := rowScope{alias: currentAlias, resourceType: currentType} + verified, err := r.validateTraversalContract(plan, *traversal, sourceScope) + if err != nil { + return fmt.Errorf("population route step: %w", err) + } + seq := r.nextTraversalSequence() + edgeAlias := "e" + strconv.Itoa(seq) + targetAlias := "x" + strconv.Itoa(seq) + edgeWhere, targetOn := r.edgeOccurrencePredicates(&verified, edgeAlias, targetAlias, sourceScope.alias) + fromParts = append(fromParts, + "INNER JOIN "+r.storage.Edge.Identifier().SQLText()+" AS "+edgeAlias+" ON "+edgeWhere, + "INNER JOIN "+resource.Identifier().SQLText()+" AS "+targetAlias+" ON "+targetOn) + currentAlias = targetAlias + currentType = verified.targetType + index += 1 + (2*2 + 2) + } + + terminalAlias := currentAlias + identityColumns := []string{"project", "dataset_generation", "resource_type", "logical_id"} + projection := make([]string, 0, 4) + join := make([]string, 0, 4) + for _, column := range identityColumns { + projection = append(projection, terminalAlias+"."+resource.MustColumn(column).SQLText()) + join = append(join, "pop."+resource.MustColumn(column).SQLText()+" = "+r.rootAlias+"."+resource.MustColumn(column).SQLText()) + } + candidate := "INNER JOIN (" + if r.mapping != nil && r.mapping.witness { + memberRead := memberAlias + "." + r.membership.columnSQL(membershipSelectionMembers, "id") + ` COLLATE "C"` + candidate += "SELECT " + strings.Join(projection, ", ") + + ", to_jsonb(ARRAY_AGG(DISTINCT (" + memberRead + ") ORDER BY (" + memberRead + `))) AS "` + + populationMappingMembersColumnAlias + `"` + + " FROM " + strings.Join(fromParts, " ") + + " WHERE " + strings.Join(predicates, " AND ") + + " GROUP BY " + strings.Join(projection, ", ") + + ") AS pop ON " + strings.Join(join, " AND ") + r.mapping.membersColumnRef = "pop." + quoteMembershipIdentifier(populationMappingMembersColumnAlias) + } else { + candidate += "SELECT DISTINCT " + strings.Join(projection, ", ") + + " FROM " + strings.Join(fromParts, " ") + + " WHERE " + strings.Join(predicates, " AND ") + + ") AS pop ON " + strings.Join(join, " AND ") + } + r.joins = append(r.joins, candidate) + return nil +} + +func validateCohortPayload(plan ir.PhysicalPlan, scan ir.PhysicalRootScan) error { + cohort := scan.CohortSource + if err := requireMembershipCollection(plan, cohort.RevisionCollectionBindKey, membershipGroupRevisions); err != nil { + return err + } + if err := requireMembershipCollection(plan, cohort.SelectionCollectionBindKey, membershipSelectionRevisions); err != nil { + return err + } + if err := requireMembershipCollection(plan, cohort.SelectionMembersCollectionBindKey, membershipSelectionMembers); err != nil { + return err + } + if err := requireMembershipCollection(plan, cohort.MembershipsCollectionBindKey, membershipGroupMemberships); err != nil { + return err + } + if cohort.RootIdentityColumn != "_key" { + return fmt.Errorf("%w: cohort RootIdentityColumn must be the lowered \"_key\" identity (lower/recipe_construction.go:403), got %q", + ErrUnsupportedPostgresPlan, cohort.RootIdentityColumn) + } + if strings.TrimSpace(cohort.CohortStageID) == "" || strings.TrimSpace(cohort.CohortInputStageID) == "" { + return fmt.Errorf("%w: cohort root source must name its cohort stage and input stage", ErrUnsupportedPostgresPlan) + } + if rootType, ok := plan.BindVars[scan.CollectionBindKey].(string); !ok || strings.TrimSpace(rootType) == "" || cohort.RootResourceType != rootType { + return fmt.Errorf("%w: cohort RootResourceType must equal the root collection resource type", ErrUnsupportedPostgresPlan) + } + read := func(key, role string, nonEmpty bool) error { + value, ok := plan.BindVars[key].(string) + if !ok { + return fmt.Errorf("%w: cohort %s bind %q must be a string", ErrUnsupportedPostgresPlan, role, key) + } + if nonEmpty && strings.TrimSpace(value) == "" { + return fmt.Errorf("%w: cohort %s bind %q must be a non-empty string", ErrUnsupportedPostgresPlan, role, key) + } + return nil + } + for _, check := range []struct { + key string + role string + nonEmpty bool + }{ + {cohort.RevisionIDBindKey, "revision id", true}, + {cohort.ProjectBindKey, "selection project", true}, + {cohort.DatasetGenerationBindKey, "dataset generation", true}, + {cohort.ResourceTypeBindKey, "resource type", true}, + {cohort.ResourceProjectBindKey, "resource-storage project", true}, + {cohort.PolicyBindKey, "unassigned member policy", true}, + } { + if err := read(check.key, check.role, check.nonEmpty); err != nil { + return err + } + } + policy, _ := plan.BindVars[cohort.PolicyBindKey].(string) + if !canonicalCohortPolicies[policy] { + return fmt.Errorf("%w: cohort policy bind %q carries %q; the closed domain is ERROR, EXCLUDE, GROUP_AS_UNASSIGNED", + ErrUnsupportedPostgresPlan, cohort.PolicyBindKey, policy) + } + return nil +} + +func (r *planRenderer) renderCohortRootSource(plan ir.PhysicalPlan, scan ir.PhysicalRootScan) error { + if err := validateCohortPayload(plan, scan); err != nil { + return err + } + cohort := scan.CohortSource + binding := r.membership + revisionID, _ := plan.BindVars[cohort.RevisionIDBindKey].(string) + project, _ := plan.BindVars[cohort.ProjectBindKey].(string) + generation, _ := plan.BindVars[cohort.DatasetGenerationBindKey].(string) + resourceType, _ := plan.BindVars[cohort.ResourceTypeBindKey].(string) + resourceProject, _ := plan.BindVars[cohort.ResourceProjectBindKey].(string) + policy, _ := plan.BindVars[cohort.PolicyBindKey].(string) + + revisionIndex := r.textParameter(revisionID, "cohort-rev:"+revisionID) + projectIndex := r.textParameter(project, "cohort-proj:"+project) + generationIndex := r.textParameter(generation, "cohort-gen:"+generation) + resourceTypeIndex := r.textParameter(resourceType, "cohort-type:"+resourceType) + resourceProjectIndex := r.textParameter(resourceProject, "text:"+resourceProject) + policyIndex := r.textParameter(policy, "cohort-policy:"+policy) + + groupRevisions := binding.tableSQL(membershipGroupRevisions) + selectionRevisions := binding.tableSQL(membershipSelectionRevisions) + selectionMembers := binding.tableSQL(membershipSelectionMembers) + groupMemberships := binding.tableSQL(membershipGroupMemberships) + col := func(kind postgresMembershipTableKind, column string) string { + return binding.columnSQL(kind, column) + } + + r.where = append(r.where, + r.rootAlias+"."+r.storage.Resource.MustColumn("project").SQLText()+" = $"+strconv.Itoa(resourceProjectIndex), + r.rootAlias+"."+r.storage.Resource.MustColumn("dataset_generation").SQLText()+" = $"+strconv.Itoa(generationIndex), + r.rootAlias+"."+r.storage.Resource.MustColumn("resource_type").SQLText()+" = $"+strconv.Itoa(resourceTypeIndex), + ) + + revisionGuard := "EXISTS (SELECT 1 FROM " + groupRevisions + " gr0 WHERE " + + "gr0." + col(membershipGroupRevisions, "id") + " = $" + strconv.Itoa(revisionIndex) + + " AND gr0." + col(membershipGroupRevisions, "state") + " = " + quoteSQLLiteral(cohortStateComplete) + + " AND gr0." + col(membershipGroupRevisions, "project") + " = $" + strconv.Itoa(projectIndex) + + " AND gr0." + col(membershipGroupRevisions, "generation") + " = $" + strconv.Itoa(generationIndex) + + " AND gr0." + col(membershipGroupRevisions, "resource_type") + " = $" + strconv.Itoa(resourceTypeIndex) + ")" + + selectionGuard := "EXISTS (SELECT 1 FROM " + groupRevisions + " gr1 INNER JOIN " + selectionRevisions + + " sv0 ON sv0." + col(membershipSelectionRevisions, "id") + " = gr1." + col(membershipGroupRevisions, "source_selection_revision_id") + + " WHERE gr1." + col(membershipGroupRevisions, "id") + " = $" + strconv.Itoa(revisionIndex) + + " AND sv0." + col(membershipSelectionRevisions, "complete") + " = TRUE" + + " AND sv0." + col(membershipSelectionRevisions, "project") + " = $" + strconv.Itoa(projectIndex) + + " AND sv0." + col(membershipSelectionRevisions, "generation") + " = $" + strconv.Itoa(generationIndex) + + " AND sv0." + col(membershipSelectionRevisions, "resource_type") + " = $" + strconv.Itoa(resourceTypeIndex) + + " AND sv0." + col(membershipSelectionRevisions, "scope_digest") + " = gr1." + col(membershipGroupRevisions, "scope_digest") + + " AND sv0." + col(membershipSelectionRevisions, "membership_digest") + " = gr1." + col(membershipGroupRevisions, "source_membership_digest") + ")" + + pinnedSelectionID := "(SELECT gr2." + col(membershipGroupRevisions, "source_selection_revision_id") + + " FROM " + groupRevisions + " gr2 WHERE gr2." + col(membershipGroupRevisions, "id") + + " = $" + strconv.Itoa(revisionIndex) + ")" + unassignedGuard := "EXISTS (SELECT 1 FROM " + selectionMembers + " sm0 WHERE " + + "sm0." + col(membershipSelectionMembers, "selection_id") + " = " + pinnedSelectionID + + " AND sm0." + col(membershipSelectionMembers, "project") + " = $" + strconv.Itoa(projectIndex) + + " AND sm0." + col(membershipSelectionMembers, "generation") + " = $" + strconv.Itoa(generationIndex) + + " AND sm0." + col(membershipSelectionMembers, "resource_type") + " = $" + strconv.Itoa(resourceTypeIndex) + + " AND NOT EXISTS (SELECT 1 FROM " + groupMemberships + " mm0 WHERE " + + "mm0." + col(membershipGroupMemberships, "revision_id") + " = $" + strconv.Itoa(revisionIndex) + + " AND mm0." + col(membershipGroupMemberships, "project") + " = sm0." + col(membershipSelectionMembers, "project") + + " AND mm0." + col(membershipGroupMemberships, "generation") + " = sm0." + col(membershipSelectionMembers, "generation") + + " AND mm0." + col(membershipGroupMemberships, "resource_type") + " = sm0." + col(membershipSelectionMembers, "resource_type") + + " AND mm0." + col(membershipGroupMemberships, "resource_id") + " = sm0." + col(membershipSelectionMembers, "id") + "))" + + guard := "(SELECT CASE WHEN NOT (" + revisionGuard + ") THEN " + guardCast(cohortGuardRevisionInvalid) + + " WHEN NOT (" + selectionGuard + ") THEN " + guardCast(cohortGuardSourceStale) + + " WHEN $" + strconv.Itoa(policyIndex) + " = " + quoteSQLLiteral(cohortPolicyError) + " AND (" + unassignedGuard + ")" + + " THEN " + guardCast(cohortGuardUnassignedMember) + + " ELSE 1 END)" + if r.scope.denied { + r.where = append(r.where, "("+guard+cohortDenyGuardSuffix+")") + } else { + r.where = append(r.where, guard+" = 1") + } + + selectionCandidates := "SELECT sm1." + col(membershipSelectionMembers, "id") + + " AS " + quoteMembershipIdentifier("candidate_id") + " FROM " + selectionMembers + " sm1 WHERE " + + "sm1." + col(membershipSelectionMembers, "selection_id") + " = " + pinnedSelectionID + + " AND sm1." + col(membershipSelectionMembers, "project") + " = $" + strconv.Itoa(projectIndex) + + " AND sm1." + col(membershipSelectionMembers, "generation") + " = $" + strconv.Itoa(generationIndex) + + " AND sm1." + col(membershipSelectionMembers, "resource_type") + " = $" + strconv.Itoa(resourceTypeIndex) + membershipCandidates := "SELECT mm1." + col(membershipGroupMemberships, "resource_id") + " FROM " + groupMemberships + " mm1 WHERE " + + "mm1." + col(membershipGroupMemberships, "revision_id") + " = $" + strconv.Itoa(revisionIndex) + + " AND mm1." + col(membershipGroupMemberships, "project") + " = $" + strconv.Itoa(projectIndex) + + " AND mm1." + col(membershipGroupMemberships, "generation") + " = $" + strconv.Itoa(generationIndex) + + " AND mm1." + col(membershipGroupMemberships, "resource_type") + " = $" + strconv.Itoa(resourceTypeIndex) + candidate := "INNER JOIN (" + selectionCandidates + " UNION " + membershipCandidates + + ") AS cand ON cand." + quoteMembershipIdentifier("candidate_id") + " = " + r.rootAlias + "." + + r.storage.Resource.MustColumn("logical_id").SQLText() + r.joins = append(r.joins, candidate) + return nil +} + +const guardBarrierGUC = "loom.explorer_guard_barrier" + +const cohortDenyGuardSuffix = " * 0 = 1" + +func guardCast(tag string) string { + return "CAST(" + quoteSQLLiteral(tag) + + " || COALESCE(current_setting(" + quoteSQLLiteral(guardBarrierGUC) + ", true), '') AS integer)" +} + +func quoteSQLLiteral(value string) string { + return "'" + strings.ReplaceAll(value, "'", "''") + "'" +} + +func requireMembershipCollection(plan ir.PhysicalPlan, bindKey string, kind postgresMembershipTableKind) error { + bound, ok := plan.BindVars[bindKey].(string) + if !ok { + return fmt.Errorf("%w: membership collection bind %q must be a string", ErrUnsupportedPostgresPlan, bindKey) + } + if expected, registered := canonicalMembershipCollections[bound]; !registered || expected != kind { + return fmt.Errorf("%w: membership collection bind %q must be bound to %q (the only registered relation for this payload role; lower/group_rows.go:33-37 pins the name)", + ErrUnsupportedPostgresPlan, bindKey, canonicalCollectionName(kind)) + } + return nil +} + +func canonicalCollectionName(kind postgresMembershipTableKind) string { + for _, name := range []string{ + "loom_explorer_selection_members", + "loom_explorer_selections", + "loom_explorer_explicit_group_revisions", + "loom_explorer_explicit_group_memberships", + } { + if canonicalMembershipCollections[name] == kind { + return name + } + } + return string(kind) +} diff --git a/internal/dataframe/compiler/render/postgres/population_live_test.go b/internal/dataframe/compiler/render/postgres/population_live_test.go new file mode 100644 index 000000000..c0a55dbf2 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/population_live_test.go @@ -0,0 +1,754 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + control "github.com/calypr/loom/internal/backends/postgres/control" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + explorer "github.com/calypr/loom/internal/explorer" +) + +const ( + pop18Project = "pop18-proj" + pop18Foreign = "pop18-proj-x" + pop18Generation = "gen-1" + pop18RouteLabel = "encounter-diagnosed_condition" + pop18PatientAuth = "FHIR/Patient" + pop18ObsAuth = "FHIR/Observation" + pop18SecretAuth = "FHIR/Secret" +) + +type pop18Service struct { + *pgir02Service + store *control.ExplorerStore + groups *control.ExplicitGroupStore + binding ir.PostgresStorageBinding + members PostgresPopulationRootBinding +} + +func newPop18Service(t *testing.T) *pop18Service { + t.Helper() + svc := newOwnedService(t, 0, "verify_pop18") + applyExplorerMembershipDDL(t, svc) + store, err := control.NewExplorerStore(svc.client, svc.schema) + if err != nil { + svc.fail("open explorer store", err) + } + groups, err := control.NewExplicitGroupStore(svc.client, svc.schema) + if err != nil { + svc.fail("open explicit-group store", err) + } + binding, err := ir.NewPostgresStorageBinding(svc.schema) + if err != nil { + svc.fail("storage binding", err) + } + members, err := NewPostgresPopulationRootBinding(svc.schema) + if err != nil { + svc.fail("membership binding", err) + } + return &pop18Service{pgir02Service: svc, store: store, groups: groups, binding: binding, members: members} +} + +func applyExplorerMembershipDDL(t *testing.T, svc *pgir02Service) { + t.Helper() + migrations, err := control.OwnerMigrations() + if err != nil { + svc.fail("load owner migrations", err) + } + conn, err := svc.iso.Acquire(svc.ctx) + if err != nil { + svc.fail("acquire DDL connection", err) + } + defer conn.Release() + if _, err := conn.Exec(svc.ctx, `SET search_path TO "`+svc.schema+`"`); err != nil { + svc.fail("pin search_path for membership DDL", err) + } + applied := 0 + for _, migration := range migrations { + if migration.Version == 1 { + continue + } + if _, err := conn.Exec(svc.ctx, migration.SQL); err != nil { + svc.fail(fmt.Sprintf("apply explorer DDL %d_%s", migration.Version, migration.Name), err) + } + applied++ + } + if applied != len(migrations)-1 { + t.Fatalf("applied %d explorer migrations, want %d", applied, len(migrations)-1) + } +} + +func (s *pop18Service) render(t *testing.T, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderPopulationRootScan(plan, s.binding, s.members) + if err != nil { + s.fail("render membership plan", err) + } + return rendered +} + +func (s *pop18Service) runErr(t *testing.T, rendered command.RenderedPostgresPhysicalPlan) ([][]string, error) { + t.Helper() + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + rows, err := s.iso.Query(s.ctx, rendered.SQL, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + return nil, err + } + row := make([]string, len(values)) + for index, value := range values { + row[index] = stringifyCell(value) + } + out = append(out, row) + } + return out, rows.Err() +} + +func wantGuardTag(t *testing.T, err error, tag string) { + t.Helper() + if err == nil { + t.Fatalf("guard %s accepted instead of failing", tag) + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + t.Fatalf("guard %s error is not a PgError: %v", tag, err) + } + if pgErr.Code != "22P02" || !strings.Contains(pgErr.Message, tag) { + t.Fatalf("guard %s error = %s: %s, want 22P02 containing %q", tag, pgErr.Code, pgErr.Message, tag) + } +} + +func singleColumnRows(rows [][]string) []string { + var out []string + for _, row := range rows { + if len(row) != 1 { + out = append(out, strings.Join(row, "|")) + continue + } + out = append(out, row[0]) + } + return out +} + +func (s *pop18Service) seedSelection(t *testing.T, id, project, resourceType string, memberIDs []string, writer string) explorer.SelectionRevision { + t.Helper() + now := time.Now().UTC() + header, err := s.store.BeginSelection(s.ctx, explorer.SelectionRevision{ + ID: id, Project: project, Generation: pop18Generation, ResourceType: resourceType, + Rule: explorer.SelectionRule{Kind: explorer.SelectionRuleExplicit}, + Source: explorer.SelectionSource{Kind: explorer.SelectionSourceExplicit}, + ScopeDigest: "scope-18", RuleDigest: "rule-18", CreatedAt: now, + }, writer) + if err != nil { + s.fail("begin selection "+id, err) + } + members := make([]explorer.SelectionMember, 0, len(memberIDs)) + for _, memberID := range memberIDs { + members = append(members, explorer.SelectionMember{Ref: explorer.ResourceRef{ + Project: project, Generation: pop18Generation, ResourceType: resourceType, ID: memberID}}) + } + if _, err := s.store.AppendSelectionMembers(s.ctx, header.ID, writer, members); err != nil { + s.fail("append selection members "+id, err) + } + digest, count, bytes, err := s.store.DigestSelectionMembers(s.ctx, project, header.ID) + if err != nil { + s.fail("digest selection "+id, err) + } + completed, err := s.store.CompleteSelection(s.ctx, header.ID, writer, digest, count, bytes, time.Now().UTC()) + if err != nil { + s.fail("complete selection "+id, err) + } + return *completed +} + +func (s *pop18Service) seedGroup(t *testing.T, key string, source explorer.SelectionRevision, definitions []explorer.ExplicitGroupDefinition, memberships []explorer.ExplicitGroupMembership, writer string) explorer.ExplicitGroupRevision { + t.Helper() + revision := explorer.ExplicitGroupRevision{ + ID: explorer.ExplicitGroupRevisionIDFor(source.Project, key), + Project: source.Project, Generation: source.Generation, ScopeDigest: source.ScopeDigest, + ResourceType: source.ResourceType, SourceSelectionRevisionID: source.ID, + SourceMembershipDigest: source.MembershipDigest, + State: explorer.ExplicitGroupRevisionStaging, + IdempotencyKey: key, CreatedAt: time.Now().UTC(), + } + revision, err := s.seedGroupFrom(t, revision, source, definitions, memberships, writer) + if err != nil { + s.fail("seed group "+key, err) + } + return revision +} + +func (s *pop18Service) seedGroupFrom(t *testing.T, revision explorer.ExplicitGroupRevision, source explorer.SelectionRevision, definitions []explorer.ExplicitGroupDefinition, memberships []explorer.ExplicitGroupMembership, writer string) (explorer.ExplicitGroupRevision, error) { + t.Helper() + if _, err := s.groups.BeginExplicitGroupRevision(s.ctx, revision, source, writer); err != nil { + return revision, err + } + if err := s.groups.PutExplicitGroupDefinitions(s.ctx, revision.ID, writer, definitions); err != nil { + return revision, err + } + if _, err := s.groups.AppendExplicitGroupMemberships(s.ctx, revision.ID, writer, memberships); err != nil { + return revision, err + } + definitionDigest, membershipDigest, groupCount, memberCount, err := s.groups.DigestExplicitGroupRevision(s.ctx, revision.Project, revision.ID) + if err != nil { + return revision, err + } + if _, err := s.groups.CompleteExplicitGroupRevision(s.ctx, revision.ID, writer, definitionDigest, membershipDigest, groupCount, memberCount, time.Now().UTC()); err != nil { + return revision, err + } + return revision, nil +} + +func pop18Patient(t *testing.T, s *pop18Service, project, logicalID, authPath string) { + t.Helper() + s.insertResource(t, s.schema, project, pop18Generation, "Patient", logicalID, authPath, + map[string]any{"resourceType": "Patient", "id": logicalID}) +} + +func livePopulationPlan(selectionID string, paths []string, unrestricted, allowed bool) ir.PhysicalPlan { + plan := populationPlan(1) + plan.BindVars["project"] = pop18Project + plan.BindVars["dataset_generation"] = pop18Generation + plan.BindVars["population_project"] = pop18Project + plan.BindVars["population_selection_id"] = selectionID + plan.BindVars["population_resource_type"] = "Observation" + plan.BindVars["population_source_collection"] = "Observation" + plan.BindVars["root_collection"] = "Patient" + plan.BindVars["population_root_route_0_target_type"] = "Patient" + plan.BindVars["population_root_route_0_edge_label"] = pop18RouteLabel + plan.BindVars["auth_resource_paths"] = paths + plan.BindVars["auth_resource_paths_unrestricted"] = unrestricted + plan.BindVars["scope_allowed"] = allowed + return plan +} + +func seedPopulationFixture(t *testing.T, s *pop18Service) { + t.Helper() + s.insertResource(t, s.schema, pop18Project, pop18Generation, "Observation", "O1", pop18ObsAuth, + map[string]any{"resourceType": "Observation", "id": "O1"}) + s.insertResource(t, s.schema, pop18Project, pop18Generation, "Observation", "O2", pop18ObsAuth, + map[string]any{"resourceType": "Observation", "id": "O2"}) + s.insertResource(t, s.schema, pop18Project, pop18Generation, "Observation", "O3", pop18ObsAuth, + map[string]any{"resourceType": "Observation", "id": "O3"}) + s.seedSelection(t, "sel18_obs", pop18Project, "Observation", []string{"O1"}, "pop18-writer") + s.seedSelection(t, "sel18_obs_x", pop18Foreign, "Observation", []string{"O3"}, "pop18-writer-x") + pop18Patient(t, s, pop18Project, "P1", pop18PatientAuth) + s.insertEdge(t, control.EdgeOccurrenceRow{ + Project: pop18Project, DatasetGeneration: pop18Generation, AuthResourcePath: pop18PatientAuth, + Label: pop18RouteLabel, SourceResourceType: "Patient", SourceLogicalID: "P1", + TargetResourceType: "Observation", TargetLogicalID: "O1", SourcePath: "focus[0]", SourceOrdinal: 0, + }) + pop18Patient(t, s, pop18Project, "P2", pop18PatientAuth) + s.insertEdge(t, control.EdgeOccurrenceRow{ + Project: pop18Project, DatasetGeneration: pop18Generation, AuthResourcePath: pop18PatientAuth, + Label: pop18RouteLabel, SourceResourceType: "Patient", SourceLogicalID: "P2", + TargetResourceType: "Observation", TargetLogicalID: "O2", SourcePath: "focus[0]", SourceOrdinal: 0, + }) + pop18Patient(t, s, pop18Project, "P3", pop18PatientAuth) + s.insertEdge(t, control.EdgeOccurrenceRow{ + Project: pop18Project, DatasetGeneration: pop18Generation, AuthResourcePath: pop18PatientAuth, + Label: "provenance-entity", SourceResourceType: "Patient", SourceLogicalID: "P3", + TargetResourceType: "Observation", TargetLogicalID: "O3", SourcePath: "focus[0]", SourceOrdinal: 0, + }) + pop18Patient(t, s, pop18Project, "P4", pop18PatientAuth) + s.insertResource(t, s.schema, pop18Project, "gen-2", "Patient", "P4", pop18PatientAuth, + map[string]any{"resourceType": "Patient", "id": "P4"}) + s.insertResource(t, s.schema, pop18Project, "gen-2", "Observation", "O3", pop18ObsAuth, + map[string]any{"resourceType": "Observation", "id": "O3"}) + s.insertEdge(t, control.EdgeOccurrenceRow{ + Project: pop18Project, DatasetGeneration: "gen-2", AuthResourcePath: pop18PatientAuth, + Label: pop18RouteLabel, SourceResourceType: "Patient", SourceLogicalID: "P4", + TargetResourceType: "Observation", TargetLogicalID: "O3", SourcePath: "focus[0]", SourceOrdinal: 0, + }) + pop18Patient(t, s, pop18Project, "P5", pop18SecretAuth) + s.insertEdge(t, control.EdgeOccurrenceRow{ + Project: pop18Project, DatasetGeneration: pop18Generation, AuthResourcePath: pop18PatientAuth, + Label: pop18RouteLabel, SourceResourceType: "Patient", SourceLogicalID: "P5", + TargetResourceType: "Observation", TargetLogicalID: "O1", SourcePath: "focus[0]", SourceOrdinal: 0, + }) + pop18Patient(t, s, pop18Project, "P6", pop18PatientAuth) + pop18Patient(t, s, pop18Foreign, "P6", pop18PatientAuth) + s.insertEdge(t, control.EdgeOccurrenceRow{ + Project: pop18Foreign, DatasetGeneration: pop18Generation, AuthResourcePath: pop18PatientAuth, + Label: pop18RouteLabel, SourceResourceType: "Patient", SourceLogicalID: "P6", + TargetResourceType: "Observation", TargetLogicalID: "O1", SourcePath: "focus[0]", SourceOrdinal: 0, + }) + pop18Patient(t, s, pop18Project, "P7", pop18PatientAuth) + s.insertEdge(t, control.EdgeOccurrenceRow{ + Project: pop18Project, DatasetGeneration: pop18Generation, AuthResourcePath: pop18PatientAuth, + Label: pop18RouteLabel, SourceResourceType: "Patient", SourceLogicalID: "P7", + TargetResourceType: "Observation", TargetLogicalID: "O3", SourcePath: "focus[0]", SourceOrdinal: 0, + }) + s.insertEdge(t, control.EdgeOccurrenceRow{ + Project: pop18Project, DatasetGeneration: pop18Generation, AuthResourcePath: pop18PatientAuth, + Label: pop18RouteLabel, SourceResourceType: "Patient", SourceLogicalID: "P1", + TargetResourceType: "Observation", TargetLogicalID: "O1", SourcePath: "focus[1]", SourceOrdinal: 1, + }) +} + +func TestPopulationLiveSelectedRowsOnly(t *testing.T) { + s := newPop18Service(t) + seedPopulationFixture(t, s) + rendered := s.render(t, livePopulationPlan("sel18_obs", + []string{pop18PatientAuth, pop18ObsAuth}, false, true)) + rows, err := s.runErr(t, rendered) + if err != nil { + s.fail("execute population root", err) + } + got := singleColumnRows(rows) + want := []string{"P1"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("population rows = %v, want %v", got, want) + } + if len(rows) != 1 { + t.Fatalf("duplicate route edge leaked a second row: %v", rows) + } + edgeCount := s.witness(t, `SELECT count(*) FROM "`+s.schema+`".fhir_edge_occurrence + WHERE project = $1 AND dataset_generation = $2 AND label = $3 + AND source_resource_type = 'Patient' AND source_logical_id = 'P1'`, + pop18Project, pop18Generation, pop18RouteLabel) + if len(edgeCount) != 1 || edgeCount[0][0] != "2" { + t.Fatalf("duplicate route edge insertion not present: %v", edgeCount) + } + candDupes := s.witness(t, `SELECT count(*) FROM ( + SELECT x1."project", x1."dataset_generation", x1."resource_type", x1."logical_id" + FROM "`+s.schema+`"."selection_members" pm + INNER JOIN "`+s.schema+`"."fhir_resource" x0 ON x0."project" = $1 AND x0."dataset_generation" = $2 + AND x0."resource_type" = 'Observation' AND x0."logical_id" = pm."id" + INNER JOIN "`+s.schema+`"."fhir_edge_occurrence" e1 ON e1."project" = $1 AND e1."dataset_generation" = $2 + AND e1."label" = $3 AND e1."target_resource_type" = 'Observation' + AND e1."target_logical_id" = x0."logical_id" AND e1."source_resource_type" = 'Patient' + INNER JOIN "`+s.schema+`"."fhir_resource" x1 ON x1."project" = $1 AND x1."dataset_generation" = $2 + AND x1."resource_type" = 'Patient' AND x1."logical_id" = e1."source_logical_id" + WHERE pm."selection_id" = 'sel18_obs' AND pm."project" = $1 + AND pm."generation" = $2 AND pm."resource_type" = 'Observation') t + WHERE t."logical_id" = 'P1'`, + pop18Project, pop18Generation, pop18RouteLabel) + if len(candDupes) != 1 || candDupes[0][0] != "2" { + t.Fatalf("undistinct candidate join must yield the P1 identity tuple twice, got %v", candDupes) + } + dupes := s.witness(t, `SELECT count(*) FROM ( + SELECT project, dataset_generation, resource_type, logical_id + FROM "`+s.schema+`"."fhir_resource" + GROUP BY 1, 2, 3, 4 HAVING count(*) > 1) d`) + if len(dupes) != 1 || dupes[0][0] != "0" { + t.Fatalf("duplicate identity witness = %v, want [[0]]", dupes) + } +} + +func TestPopulationLiveMembershipScopingAlive(t *testing.T) { + s := newPop18Service(t) + seedPopulationFixture(t, s) + s.seedSelection(t, "sel18_obs_two", pop18Project, "Observation", []string{"O1", "O3"}, "pop18-writer-2") + rows := s.runRendered(t, s.render(t, livePopulationPlan("sel18_obs_two", + []string{pop18PatientAuth, pop18ObsAuth}, false, true))) + got := strings.Join(singleColumnRows(rows), ",") + if got != "P1,P7" { + t.Fatalf("promoted-membership rows = %q, want P1,P7", got) + } +} + +func TestPopulationLiveCanonicalDenyAndUnrestricted(t *testing.T) { + s := newPop18Service(t) + seedPopulationFixture(t, s) + denied := s.runRendered(t, s.render(t, livePopulationPlan("sel18_obs", + []string{pop18PatientAuth, pop18ObsAuth}, false, false))) + if len(denied) != 0 { + t.Fatalf("canonical deny must read nothing, got %v", denied) + } + allowed := s.runRendered(t, s.render(t, livePopulationPlan("sel18_obs", + []string{pop18PatientAuth, pop18ObsAuth}, false, true))) + if strings.Join(singleColumnRows(allowed), ",") != "P1" { + t.Fatalf("allowed control rows = %v, want [P1]", allowed) + } + unrestricted := s.runRendered(t, s.render(t, livePopulationPlan("sel18_obs", + nil, true, true))) + if got := strings.Join(singleColumnRows(unrestricted), ","); got != "P1,P5" { + t.Fatalf("unrestricted rows = %q, want P1,P5", got) + } +} + +func liveCohortPlan(revisionID string, paths []string, unrestricted, allowed bool, policy string) ir.PhysicalPlan { + plan := cohortPlan() + plan.BindVars["project"] = pop18Project + plan.BindVars["dataset_generation"] = pop18Generation + plan.BindVars["cohort_project"] = pop18Project + plan.BindVars["cohort_resource_project"] = pop18Project + plan.BindVars["cohort_dataset_generation"] = pop18Generation + plan.BindVars["cohort_revision_id"] = revisionID + plan.BindVars["cohort_policy"] = policy + plan.BindVars["auth_resource_paths"] = paths + plan.BindVars["auth_resource_paths_unrestricted"] = unrestricted + plan.BindVars["scope_allowed"] = allowed + return plan +} + +type pop18CohortFixture struct { + source explorer.SelectionRevision + group explorer.ExplicitGroupRevision +} + +func seedCohortFixture(t *testing.T, s *pop18Service) pop18CohortFixture { + t.Helper() + for _, id := range []string{"R1", "R2", "R3"} { + pop18Patient(t, s, pop18Project, id, pop18PatientAuth) + } + pop18Patient(t, s, pop18Project, "R4", pop18SecretAuth) + pop18Patient(t, s, pop18Project, "R9", pop18PatientAuth) + source := s.seedSelection(t, "sel18_pat", pop18Project, "Patient", + []string{"R1", "R2", "R3", "R4"}, "pop18-cwriter") + definitions := []explorer.ExplicitGroupDefinition{ + {ID: "group-alpha", Label: "Alpha", Ordinal: 0}, + {ID: "group-beta", Label: "Beta", Ordinal: 1}, + } + var memberships []explorer.ExplicitGroupMembership + for _, group := range []string{"group-alpha", "group-beta"} { + memberships = append(memberships, explorer.ExplicitGroupMembership{GroupID: explorer.ExplicitGroupID(group), Ref: explorer.ResourceRef{ + Project: pop18Project, Generation: pop18Generation, ResourceType: "Patient", ID: "R1"}}) + } + group := s.seedGroup(t, "pop18-group-a", source, definitions, memberships, "pop18-gwriter") + return pop18CohortFixture{source: source, group: group} +} + +func TestCohortLiveAssignedUnassignedAndScoping(t *testing.T) { + s := newPop18Service(t) + fixture := seedCohortFixture(t, s) + plan := liveCohortPlan(string(fixture.group.ID), []string{pop18PatientAuth}, false, true, "EXCLUDE") + first := s.runRendered(t, s.render(t, plan)) + if got := strings.Join(singleColumnRows(first), ","); got != "R1,R2,R3" { + t.Fatalf("cohort rows = %q, want R1,R2,R3", got) + } + unionDupes := s.witness(t, `SELECT count(*) FROM ( + SELECT sm."id" FROM "`+s.schema+`"."selection_members" sm + WHERE sm."selection_id" = 'sel18_pat' AND sm."project" = $1 AND sm."generation" = $2 AND sm."resource_type" = 'Patient' + UNION ALL + SELECT mm."resource_id" FROM "`+s.schema+`"."explorer_explicit_group_memberships" mm + WHERE mm."revision_id" = $3 AND mm."project" = $1 AND mm."generation" = $2 AND mm."resource_type" = 'Patient') u + WHERE u."id" = 'R1'`, pop18Project, pop18Generation, string(fixture.group.ID)) + if len(unionDupes) != 1 || unionDupes[0][0] != "3" { + t.Fatalf("undeduplicated candidates must carry R1 three times, got %v", unionDupes) + } + second, err := s.runErr(t, s.render(t, plan)) + if err != nil || !reflectDeepEqualStrings(singleColumnRows(first), singleColumnRows(second)) { + t.Fatalf("cohort rows are not deterministic: %v vs %v (%v)", singleColumnRows(first), second, err) + } + policy := liveCohortPlan(string(fixture.group.ID), []string{pop18PatientAuth}, false, true, "GROUP_AS_UNASSIGNED") + if got := strings.Join(singleColumnRows(s.runRendered(t, s.render(t, policy))), ","); got != "R1,R2,R3" { + t.Fatalf("GROUP_AS_UNASSIGNED rows = %q, want R1,R2,R3", got) + } + unrestricted := liveCohortPlan(string(fixture.group.ID), nil, true, true, "EXCLUDE") + if got := strings.Join(singleColumnRows(s.runRendered(t, s.render(t, unrestricted))), ","); got != "R1,R2,R3,R4" { + t.Fatalf("unrestricted cohort rows = %q, want R1,R2,R3,R4", got) + } + denied := liveCohortPlan(string(fixture.group.ID), []string{pop18PatientAuth}, false, false, "EXCLUDE") + if rows := s.runRendered(t, s.render(t, denied)); len(rows) != 0 { + t.Fatalf("cohort deny must read nothing, got %v", rows) + } +} + +func TestCohortLiveGuardIsUncorrelatedInitPlan(t *testing.T) { + s := newPop18Service(t) + fixture := seedCohortFixture(t, s) + rendered := s.render(t, liveCohortPlan(string(fixture.group.ID), []string{pop18PatientAuth}, false, true, "EXCLUDE")) + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + planRows := s.witness(t, "EXPLAIN (FORMAT TEXT) "+rendered.SQL, args...) + var text strings.Builder + for _, row := range planRows { + text.WriteString(strings.Join(row, "\n")) + text.WriteString("\n") + } + planText := text.String() + if !strings.Contains(planText, "InitPlan") { + t.Fatalf("guard sublinks must execute as InitPlans, plan:\n%s", planText) + } + if strings.Contains(planText, "SubPlan") { + t.Fatalf("guard must stay uncorrelated (no per-row SubPlans), plan:\n%s", planText) + } + t.Logf("plan:\n%s", planText) +} + +func TestCohortLiveUnassignedErrorPolicy(t *testing.T) { + s := newPop18Service(t) + fixture := seedCohortFixture(t, s) + _, err := s.runErr(t, s.render(t, + liveCohortPlan(string(fixture.group.ID), []string{pop18PatientAuth}, false, true, "ERROR"))) + wantGuardTag(t, err, "EXPLICIT_GROUP_UNASSIGNED_MEMBER") + definitions := []explorer.ExplicitGroupDefinition{{ID: "group-all", Label: "All", Ordinal: 0}} + var memberships []explorer.ExplicitGroupMembership + for _, id := range []string{"R1", "R2", "R3", "R4"} { + memberships = append(memberships, explorer.ExplicitGroupMembership{GroupID: "group-all", Ref: explorer.ResourceRef{ + Project: pop18Project, Generation: pop18Generation, ResourceType: "Patient", ID: id}}) + } + assigned := s.seedGroup(t, "pop18-group-all", fixture.source, definitions, memberships, "pop18-gwriter-all") + rows := s.runRendered(t, s.render(t, + liveCohortPlan(string(assigned.ID), []string{pop18PatientAuth}, false, true, "ERROR"))) + if got := strings.Join(singleColumnRows(rows), ","); got != "R1,R2,R3" { + t.Fatalf("fully assigned ERROR rows = %q, want R1,R2,R3", got) + } +} + +func TestCohortLiveNullMembershipSubstitutedControls(t *testing.T) { + s := newPop18Service(t) + fixture := seedCohortFixture(t, s) + definitionsC := []explorer.ExplicitGroupDefinition{{ID: "group-d", Label: "D", Ordinal: 0}} + var membershipsC []explorer.ExplicitGroupMembership + for _, id := range []string{"R1", "R2", "R3", "R4"} { + membershipsC = append(membershipsC, explorer.ExplicitGroupMembership{GroupID: "group-d", Ref: explorer.ResourceRef{ + Project: pop18Project, Generation: pop18Generation, ResourceType: "Patient", ID: id}}) + } + danglingSource := s.seedSelection(t, "sel18_pat_d", pop18Project, "Patient", + []string{"R1", "R2", "R3", "R4", "R8"}, "pop18-cwriter-d") + revisionC := s.seedGroup(t, "pop18-group-d", danglingSource, definitionsC, membershipsC, "pop18-gwriter-d") + _, err := s.runErr(t, s.render(t, + liveCohortPlan(string(revisionC.ID), []string{pop18PatientAuth}, false, true, "ERROR"))) + wantGuardTag(t, err, "EXPLICIT_GROUP_UNASSIGNED_MEMBER") + if got := strings.Join(singleColumnRows(s.runRendered(t, s.render(t, + liveCohortPlan(string(revisionC.ID), []string{pop18PatientAuth}, false, true, "EXCLUDE")))), ","); got != "R1,R2,R3" { + t.Fatalf("dangling-reference EXCLUDE rows = %q, want R1,R2,R3", got) + } + definitionsE := []explorer.ExplicitGroupDefinition{{ID: "group-e", Label: "E", Ordinal: 0}} + var membershipsE []explorer.ExplicitGroupMembership + for _, id := range []string{"R2", "R3"} { + membershipsE = append(membershipsE, explorer.ExplicitGroupMembership{GroupID: "group-e", Ref: explorer.ResourceRef{ + Project: pop18Project, Generation: pop18Generation, ResourceType: "Patient", ID: id}}) + } + revisionE := s.seedGroup(t, "pop18-group-e", fixture.source, definitionsE, membershipsE, "pop18-gwriter-e") + _, err = s.runErr(t, s.render(t, + liveCohortPlan(string(fixture.group.ID), []string{pop18PatientAuth}, false, true, "ERROR"))) + wantGuardTag(t, err, "EXPLICIT_GROUP_UNASSIGNED_MEMBER") + if got := strings.Join(singleColumnRows(s.runRendered(t, s.render(t, + liveCohortPlan(string(revisionE.ID), []string{pop18PatientAuth}, false, true, "EXCLUDE")))), ","); got != "R1,R2,R3" { + t.Fatalf("foreign-scope control rows = %q, want R1,R2,R3", got) + } +} + +func TestCohortLiveRevisionInvalidFiresAtZeroRows(t *testing.T) { + s := newPop18Service(t) + fixture := seedCohortFixture(t, s) + staging := explorer.ExplicitGroupRevision{ + ID: explorer.ExplicitGroupRevisionIDFor(pop18Project, "pop18-group-staging"), + Project: pop18Project, Generation: pop18Generation, ScopeDigest: fixture.source.ScopeDigest, + ResourceType: fixture.source.ResourceType, SourceSelectionRevisionID: fixture.source.ID, + SourceMembershipDigest: fixture.source.MembershipDigest, + State: explorer.ExplicitGroupRevisionStaging, + IdempotencyKey: "pop18-group-staging", CreatedAt: time.Now().UTC(), + } + if _, err := s.groups.BeginExplicitGroupRevision(s.ctx, staging, fixture.source, "pop18-gwriter-staging"); err != nil { + s.fail("begin staging group", err) + } + _, err := s.runErr(t, s.render(t, + liveCohortPlan(string(staging.ID), []string{pop18PatientAuth}, false, true, "EXCLUDE"))) + wantGuardTag(t, err, "EXPLICIT_GROUP_REVISION_INVALID") + _, err = s.runErr(t, s.render(t, + liveCohortPlan(string(staging.ID), []string{"FHIR/Nothing"}, false, true, "EXCLUDE"))) + wantGuardTag(t, err, "EXPLICIT_GROUP_REVISION_INVALID") + rows := s.runRendered(t, s.render(t, + liveCohortPlan(string(fixture.group.ID), []string{"FHIR/Nothing"}, false, true, "EXCLUDE"))) + if len(rows) != 0 { + t.Fatalf("impossible-auth cohort rows = %v, want none", rows) + } +} + +func TestCohortLiveGuardFiresAtZeroRowsUnderExplainAnalyze(t *testing.T) { + s := newPop18Service(t) + fixture := seedCohortFixture(t, s) + staging := explorer.ExplicitGroupRevision{ + ID: explorer.ExplicitGroupRevisionIDFor(pop18Project, "pop18-group-analyze"), + Project: pop18Project, Generation: pop18Generation, ScopeDigest: fixture.source.ScopeDigest, + ResourceType: fixture.source.ResourceType, SourceSelectionRevisionID: fixture.source.ID, + SourceMembershipDigest: fixture.source.MembershipDigest, + State: explorer.ExplicitGroupRevisionStaging, + IdempotencyKey: "pop18-group-analyze", CreatedAt: time.Now().UTC(), + } + if _, err := s.groups.BeginExplicitGroupRevision(s.ctx, staging, fixture.source, "pop18-gwriter-analyze"); err != nil { + s.fail("begin staging group for analyze", err) + } + rendered := s.render(t, liveCohortPlan(string(staging.ID), []string{"FHIR/Nothing"}, false, true, "EXCLUDE")) + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + ctx, cancel := context.WithTimeout(s.ctx, 60*time.Second) + defer cancel() + _, err := s.iso.Exec(ctx, "EXPLAIN (ANALYZE, TIMING OFF, SUMMARY OFF) "+rendered.SQL, args...) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "22P02" || !strings.Contains(pgErr.Message, "EXPLICIT_GROUP_REVISION_INVALID") { + t.Fatalf("EXPLAIN ANALYZE zero-row guard raise = %v, want 22P02 naming EXPLICIT_GROUP_REVISION_INVALID", err) + } + planRows := s.witness(t, "EXPLAIN "+rendered.SQL, args...) + if len(planRows) == 0 { + t.Fatalf("plain EXPLAIN produced no plan under the violating guard") + } +} + +func seedStaleGroup(t *testing.T, s *pop18Service, fixture pop18CohortFixture) explorer.ExplicitGroupRevisionID { + t.Helper() + key := "pop18-group-stale" + staleID := explorer.ExplicitGroupRevisionIDFor(pop18Project, key) + writer := "pop18-gwriter-stale" + now := time.Now().UTC() + s.witness(t, `INSERT INTO "`+s.schema+`".explorer_explicit_group_revisions + (id, project, generation, scope_digest, resource_type, source_selection_revision_id, + source_membership_digest, definition_digest, membership_digest, group_count, member_count, + state, idempotency_key, created_at, completed_at, writer_token, writer_expires_at) + VALUES ($1, $2, $3, 'scope-forged', $4, $5, $6, '', '', 0, 0, 'STAGING', $7, $8, NULL, $9, $10)`, + string(staleID), pop18Project, pop18Generation, "Patient", fixture.source.ID, + fixture.source.MembershipDigest, key, now, writer, now.Add(time.Hour)) + s.witness(t, `INSERT INTO "`+s.schema+`".explorer_explicit_group_definitions + (revision_id, group_id, label, ordinal) VALUES ($1, 'group-stale', 'Stale', 0)`, string(staleID)) + s.witness(t, `INSERT INTO "`+s.schema+`".explorer_explicit_group_memberships + (revision_id, group_id, project, generation, resource_type, resource_id) + VALUES ($1, 'group-stale', $2, $3, 'Patient', 'R1')`, string(staleID), pop18Project, pop18Generation) + conn, err := s.iso.Acquire(s.ctx) + if err != nil { + s.fail("acquire completion channel", err) + } + defer conn.Release() + if _, err := conn.Exec(s.ctx, `SELECT set_config('loom.explorer_explicit_group_writer_token', $1, false)`, writer); err != nil { + s.fail("set writer-token guard channel", err) + } + if _, err := conn.Exec(s.ctx, `UPDATE "`+s.schema+`".explorer_explicit_group_revisions + SET state = 'COMPLETE', definition_digest = 'def-18', membership_digest = 'mem-18', + group_count = 1, member_count = 1, completed_at = $2 + WHERE id = $1`, string(staleID), time.Now().UTC()); err != nil { + s.fail("complete stale header through the guarded channel", err) + } + return staleID +} + +func TestCohortLiveDenyStillRaisesGuards(t *testing.T) { + s := newPop18Service(t) + fixture := seedCohortFixture(t, s) + staging := explorer.ExplicitGroupRevision{ + ID: explorer.ExplicitGroupRevisionIDFor(pop18Project, "pop18-group-deny-staging"), + Project: pop18Project, Generation: pop18Generation, ScopeDigest: fixture.source.ScopeDigest, + ResourceType: fixture.source.ResourceType, SourceSelectionRevisionID: fixture.source.ID, + SourceMembershipDigest: fixture.source.MembershipDigest, + State: explorer.ExplicitGroupRevisionStaging, + IdempotencyKey: "pop18-group-deny-staging", CreatedAt: time.Now().UTC(), + } + if _, err := s.groups.BeginExplicitGroupRevision(s.ctx, staging, fixture.source, "pop18-gwriter-deny-staging"); err != nil { + s.fail("begin staging group for deny", err) + } + staleID := seedStaleGroup(t, s, fixture) + arms := []struct { + name string + revision string + tag string + }{ + {"revision-invalid", string(staging.ID), "EXPLICIT_GROUP_REVISION_INVALID"}, + {"source-stale", string(staleID), "EXPLICIT_GROUP_SOURCE_STALE"}, + {"unassigned-member", string(fixture.group.ID), "EXPLICIT_GROUP_UNASSIGNED_MEMBER"}, + } + for _, arm := range arms { + t.Run(arm.name, func(t *testing.T) { + _, err := s.runErr(t, s.render(t, + liveCohortPlan(arm.revision, []string{pop18PatientAuth}, false, false, "ERROR"))) + wantGuardTag(t, err, arm.tag) + }) + } + if rows := s.runRendered(t, s.render(t, + liveCohortPlan(string(fixture.group.ID), []string{pop18PatientAuth}, false, false, "EXCLUDE"))); len(rows) != 0 { + t.Fatalf("healthy denied cohort must read nothing, got %v", rows) + } + rendered := s.render(t, liveCohortPlan(string(staging.ID), []string{pop18PatientAuth}, false, false, "ERROR")) + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + planRows := s.witness(t, "EXPLAIN (FORMAT TEXT) "+rendered.SQL, args...) + var text strings.Builder + for _, row := range planRows { + text.WriteString(strings.Join(row, "\n")) + text.WriteString("\n") + } + planText := text.String() + if !strings.Contains(planText, "InitPlan") { + t.Fatalf("guard InitPlan was pruned under canonical deny, plan:\n%s", planText) + } + if strings.Contains(planText, "One-Time Filter: false") { + t.Fatalf("denied cohort folded to a constant-FALSE one-time filter, pruning the guard, plan:\n%s", planText) + } + t.Logf("denied cohort plan:\n%s", planText) +} + +func TestCohortLiveSourceStale(t *testing.T) { + s := newPop18Service(t) + fixture := seedCohortFixture(t, s) + staleID := seedStaleGroup(t, s, fixture) + _, err := s.runErr(t, s.render(t, + liveCohortPlan(string(staleID), []string{pop18PatientAuth}, false, true, "EXCLUDE"))) + wantGuardTag(t, err, "EXPLICIT_GROUP_SOURCE_STALE") + rows := s.runRendered(t, s.render(t, + liveCohortPlan(string(fixture.group.ID), []string{pop18PatientAuth}, false, true, "EXCLUDE"))) + if got := strings.Join(singleColumnRows(rows), ","); got != "R1,R2,R3" { + t.Fatalf("unaffected cohort rows = %q, want R1,R2,R3", got) + } +} + +func reflectDeepEqualStrings(left, right []string) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if left[index] != right[index] { + return false + } + } + return true +} + +func TestZZNoLeftoverPop18Resources(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return + } + if _, err := os.Stat("/proc/self/"); err != nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer admin.Close() + var databases int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify_pop18_%'").Scan(&databases); err != nil { + t.Fatalf("count databases: %v", err) + } + if databases != 0 { + t.Fatalf("%d verify_pop18_* databases still exist", databases) + } + var schemas int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'verify_pop18_%'").Scan(&schemas); err != nil { + t.Fatalf("count schemas: %v", err) + } + if schemas != 0 { + t.Fatalf("%d verify_pop18_* schemas still exist in the service database", schemas) + } +} diff --git a/internal/dataframe/compiler/render/postgres/population_mapping.go b/internal/dataframe/compiler/render/postgres/population_mapping.go new file mode 100644 index 000000000..9259a6f9a --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/population_mapping.go @@ -0,0 +1,469 @@ +package postgres + +import ( + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const populationMappingMembersColumnAlias = "__loom_population_members" + +const populationMappingContributorColumn = "__loom_root_contributor_keys" + +type populationMappingRender struct { + witness bool + construction bool + terminal *ir.PhysicalStagePopulationMappingReturn + membersColumnRef string + memberOrderTerm string +} + +func ValidatePopulationMappingSource(plan ir.PhysicalPlan, binding PostgresPopulationRootBinding) error { + if err := validatePopulationMappingPlanSurface(plan, binding); err != nil { + return err + } + if err := plan.Validate(); err != nil { + return fmt.Errorf("validate physical plan: %w", err) + } + return nil +} + +func RenderPopulationMappingPlan( + plan ir.PhysicalPlan, + storage ir.PostgresStorageBinding, + binding PostgresPopulationRootBinding, +) (command.RenderedPostgresPhysicalPlan, error) { + if err := storage.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := validatePopulationMappingPlanSurface(plan, binding); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if plan.StageSequence != nil { + return renderPopulationMappingConstruction(plan, storage, binding) + } + return renderPopulationMappingFlat(plan, storage, binding) +} + +func validatePopulationMappingPlanSurface(plan ir.PhysicalPlan, binding PostgresPopulationRootBinding) error { + if err := binding.Validate(); err != nil { + return err + } + if len(plan.Operations) == 0 { + return fmt.Errorf("%w: population mapping plan has no operations", ErrUnsupportedPostgresPlan) + } + if plan.Engine != "" && plan.Engine != ir.PhysicalEngineAQL { + return fmt.Errorf( + "%w: engine %q has no PostgreSQL source; the source graph IR is backend-neutral and ClickHouse plans are not PostgreSQL-rendered", + ErrUnsupportedPostgresPlan, string(plan.Engine)) + } + if plan.ClickHouseCombine != nil || plan.ClickHousePrefix != nil { + return fmt.Errorf("%w: ClickHouse combine/prefix payloads have no PostgreSQL rendering", ErrUnsupportedPostgresPlan) + } + if len(plan.DeferredExpressionLets) != 0 { + return fmt.Errorf("%w: completed plans must not carry deferred expression lets", ErrUnsupportedPostgresPlan) + } + if plan.PreviewSourceWindowByRootID { + return fmt.Errorf("%w: preview source windowing is not implemented for the PostgreSQL population mapping terminals", + ErrUnsupportedPostgresPlan) + } + root := plan.Operations[0] + if root.Kind != ir.PhysicalRootScanOp || root.RootScan == nil { + return fmt.Errorf("%w: the leading operation must be ROOT_SCAN with its payload set", ErrUnsupportedPostgresPlan) + } + scan := *root.RootScan + if scan.Population == nil { + return fmt.Errorf("%w: population mapping requires a membership-driven population root source", ErrUnsupportedPostgresPlan) + } + if scan.CohortSource != nil { + return fmt.Errorf("%w: population mapping cannot combine cohort and population sources", ErrUnsupportedPostgresPlan) + } + if scan.PageCandidateSeed != nil { + return fmt.Errorf("%w: population mapping cannot combine with the candidate seed, which stays unregistered", + ErrUnsupportedPostgresPlan) + } + if err := requireMembershipRootBinds(scan); err != nil { + return err + } + if _, err := validatePopulationPayload(plan, scan, true); err != nil { + return err + } + if plan.StageSequence != nil { + if err := validateStageSequencePlanSurface(plan, true); err != nil { + return err + } + return validatePopulationMappingConstructionSurface(plan) + } + return validatePopulationMappingFlatSurface(plan) +} + +func validatePopulationMappingConstructionSurface(plan ir.PhysicalPlan) error { + found := false + for _, operation := range plan.Operations { + if operation.Kind != ir.PhysicalReturnOp || operation.Return == nil { + continue + } + for _, projection := range operation.Return.Projections { + if projection.Name != ir.PhysicalPopulationMappingMembersColumn { + continue + } + if !projection.Hidden || projection.Expression != nil || + projection.Value.Variable != ir.PopulationMappingMembersVariable || len(projection.Value.Path) != 0 { + return fmt.Errorf("%w: source projection %q is not the collected selected member IDs", + ErrUnsupportedPostgresPlan, projection.Name) + } + found = true + } + } + if !found { + return fmt.Errorf("%w: the composition mapping source RETURN omits the compiler-owned member witness projection %q (compiler/population_mapping.go adds it)", + ErrUnsupportedPostgresPlan, ir.PhysicalPopulationMappingMembersColumn) + } + if key := plan.StageSequence.PreviewLimitBindKey; key != "" { + if limit, ok := plan.BindVars[key].(int); !ok || limit <= 0 { + return fmt.Errorf("%w: composition preview limit bind %q must be a positive int (render/aql/stage_sequence.go:452-453 twin)", + ErrUnsupportedPostgresPlan, key) + } + } + return nil +} + +func validatePopulationMappingFlatSurface(plan ir.PhysicalPlan) error { + sawTerminal := false + for index, operation := range plan.Operations { + last := index == len(plan.Operations)-1 + switch operation.Kind { + case ir.PhysicalRootScanOp: + if index != 0 { + return fmt.Errorf("%w: exactly one leading ROOT_SCAN is required", ErrUnsupportedPostgresPlan) + } + case ir.PhysicalPopulationMappingReturnOp: + if !last || sawTerminal { + return fmt.Errorf("%w: POPULATION_MAPPING_RETURN must be the single final operation", ErrUnsupportedPostgresPlan) + } + sawTerminal = true + if operation.PopulationMappingReturn == nil { + return fmt.Errorf("%w: operation %d: population-mapping payload is missing", ErrUnsupportedPostgresPlan, index) + } + if err := validatePopulationMappingReturnShape(*operation.PopulationMappingReturn); err != nil { + return fmt.Errorf("operation %d: %w", index, err) + } + case ir.PhysicalUnnestOp, ir.PhysicalSetOp: + return fmt.Errorf("%w: operation %q on a member-collecting mapping root is a named capability gap (unnest.go/sets.go bind their own root contracts); the shape is refused, never partially rendered", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + case ir.PhysicalReturnOp: + return fmt.Errorf("%w: the population mapping terminal replaces the RETURN projection list; a plan carrying both is not the registered shape", + ErrUnsupportedPostgresPlan) + } + } + if !sawTerminal { + return fmt.Errorf("%w: the flat population mapping requires a terminal POPULATION_MAPPING_RETURN", ErrUnsupportedPostgresPlan) + } + return nil +} + +func validatePopulationMappingReturnShape(terminal ir.PhysicalPopulationMappingReturn) error { + if terminal.Members.Kind != ir.PhysicalValueExpression || + terminal.Members.Value == nil || terminal.Members.Value.Variable != ir.PopulationMappingMembersVariable || + len(terminal.Members.Value.Path) != 0 || terminal.Members.Value.BindKey != "" { + return fmt.Errorf("%w: mapping Members must be the collected-member value over %q", + ErrUnsupportedPostgresPlan, ir.PopulationMappingMembersVariable) + } + if terminal.ExplicitIdentity == nil && len(terminal.IdentityParts) == 0 { + return fmt.Errorf("%w: mapping identity has no renderable value", ErrUnsupportedPostgresPlan) + } + for index, part := range terminal.IdentityParts { + if strings.TrimSpace(part.Name) == "" { + return fmt.Errorf("%w: mapping identity part %d has no name", ErrUnsupportedPostgresPlan, index) + } + } + return nil +} + +func renderPopulationMappingFlat( + plan ir.PhysicalPlan, + storage ir.PostgresStorageBinding, + binding PostgresPopulationRootBinding, +) (command.RenderedPostgresPhysicalPlan, error) { + bound := binding + renderer := &planRenderer{ + storage: storage, + rows: map[string]rowScope{}, + parameters: map[string]int{}, + canonicalAuthLets: map[string]string{}, + membership: &bound, + mapping: &populationMappingRender{witness: true}, + } + if err := renderer.loadScope(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := plan.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("validate physical plan: %w", err) + } + if err := renderer.renderOperations(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return command.NewRenderedPostgresPhysicalPlan( + renderer.assemblePopulationMapping(), renderer.args, renderer.outputColumns) +} + +func renderPopulationMappingConstruction( + plan ir.PhysicalPlan, + storage ir.PostgresStorageBinding, + binding PostgresPopulationRootBinding, +) (command.RenderedPostgresPhysicalPlan, error) { + bound := binding + renderer := &planRenderer{ + storage: storage, + rows: map[string]rowScope{}, + parameters: map[string]int{}, + canonicalAuthLets: map[string]string{}, + stageRows: map[string]stageRowScope{}, + membership: &bound, + mapping: &populationMappingRender{ + witness: true, + construction: true, + terminal: plan.StageSequence.PopulationMappingReturn, + }, + } + if err := renderer.loadScope(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := plan.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("validate physical plan: %w", err) + } + sql, err := renderer.renderStageSequenceMappingStatement(plan) + if err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return command.NewRenderedPostgresPhysicalPlan(sql, renderer.args, renderer.outputColumns) +} + +func (r *planRenderer) renderPopulationMappingTerminal(plan ir.PhysicalPlan, terminal ir.PhysicalPopulationMappingReturn) error { + if r.mapping == nil || !r.mapping.witness { + return fmt.Errorf("%w: operation %q is not part of the v1 PostgreSQL render surface", + ErrUnsupportedPostgresPlan, string(ir.PhysicalPopulationMappingReturnOp)) + } + if r.mapping.membersColumnRef == "" { + return fmt.Errorf("%w: the member witness never rendered — the population root did not take the witness posture", + ErrUnsupportedPostgresPlan) + } + alias := "w0" + guard := "CASE WHEN " + r.mapping.membersColumnRef + " IS NULL OR jsonb_typeof(" + + r.mapping.membersColumnRef + ") <> 'array' THEN '[]'::jsonb ELSE " + r.mapping.membersColumnRef + " END" + lateral := "CROSS JOIN LATERAL jsonb_array_elements_text(" + guard + ") AS " + alias + "(member)" + r.laterals = append(r.laterals, lateral) + memberSQL := alias + ".member" + identitySQL, identityColumn, err := r.renderMappingIdentityValue(plan, terminal) + if err != nil { + return err + } + r.selects = append(r.selects, memberSQL+` AS "c0"`, identitySQL+` AS "c1"`) + r.outputColumns = []string{ir.PhysicalPopulationMappingMemberField, identityColumn} + r.mapping.memberOrderTerm = memberSQL + ` COLLATE "C" ASC` + return nil +} + +func (r *planRenderer) assemblePopulationMapping() string { + var builder strings.Builder + builder.WriteString("SELECT ") + builder.WriteString(strings.Join(r.selects, ", ")) + builder.WriteString(" FROM " + r.storage.Resource.Identifier().SQLText() + " AS " + r.rootAlias) + for _, join := range r.joins { + builder.WriteString(" " + join) + } + for _, lateral := range r.laterals { + builder.WriteString(" " + lateral) + } + if len(r.where) > 0 { + builder.WriteString(" WHERE " + strings.Join(r.where, " AND ")) + } + order := append(r.windowOrder(), r.mapping.memberOrderTerm) + builder.WriteString(" ORDER BY " + strings.Join(order, ", ")) + builder.WriteString(r.window.limitClause) + return builder.String() +} + +func (r *planRenderer) renderMappingIdentityValue(plan ir.PhysicalPlan, terminal ir.PhysicalPopulationMappingReturn) (string, string, error) { + if terminal.ExplicitIdentity != nil { + value, err := r.renderMappingIdentityExpression(plan, *terminal.ExplicitIdentity) + if err != nil { + return "", "", fmt.Errorf("mapping explicit identity: %w", err) + } + return value, ir.PhysicalPopulationMappingExplicitIdentityField, nil + } + if len(terminal.IdentityParts) == 0 { + return "", "", fmt.Errorf("%w: mapping identity has no renderable value", ErrUnsupportedPostgresPlan) + } + parts := make([]string, 0, len(terminal.IdentityParts)) + for index, part := range terminal.IdentityParts { + value, err := r.renderMappingIdentityExpression(plan, part.Expression) + if err != nil { + return "", "", fmt.Errorf("mapping identity part %d (%s): %w", index, part.Name, err) + } + parts = append(parts, value) + } + return "jsonb_build_array(" + strings.Join(parts, ", ") + ")", ir.PhysicalPopulationMappingIdentityPartsField, nil +} + +func (r *planRenderer) renderMappingIdentityExpression(plan ir.PhysicalPlan, expression ir.PhysicalExpression) (string, error) { + switch expression.Kind { + case ir.PhysicalValueExpression: + if expression.Value == nil { + return "", fmt.Errorf("%w: mapping identity VALUE expression has no value payload", ErrUnsupportedPostgresPlan) + } + return r.renderMappingValue(plan, *expression.Value) + case ir.PhysicalExtractExpression: + extract := expression + classified, err := classifyExtractionProjections([]ir.PhysicalProjection{ + {Name: "__loom_mapping_identity", Expression: &extract}, + }, func(variable string) (string, bool) { + scope, ok := r.rows[variable] + return scope.resourceType, ok + }) + if err != nil { + return "", err + } + if len(classified) != 1 || classified[0].kind != extractionSelector { + return "", fmt.Errorf("%w: mapping identity EXTRACT expression is not a registered scalar selector reading", + ErrUnsupportedPostgresPlan) + } + scope, ok := r.rows[classified[0].expression.Extract.Source.Variable] + if !ok || scope.derivedValue != "" { + return "", fmt.Errorf("%w: mapping identity selector reads a variable that binds no stored row", + ErrUnsupportedPostgresPlan) + } + return r.scalarExtractValueExpression(scope.alias, classified[0].path, expression.NullBehavior) + default: + return "", fmt.Errorf("%w: mapping identity expression kind %q is a named capability gap and is refused, never partially rendered", + ErrUnsupportedPostgresPlan, string(expression.Kind)) + } +} + +func (r *planRenderer) renderMappingValue(plan ir.PhysicalPlan, value ir.PhysicalValue) (string, error) { + if value.BindKey != "" { + raw, ok := plan.BindVars[value.BindKey] + if !ok { + return "", fmt.Errorf("%w: mapping identity bind %q is not in the plan bind vars", ErrUnsupportedPostgresPlan, value.BindKey) + } + return r.mappingBindParameter(raw, value.BindKey) + } + return r.resolveValue(value) +} + +func (r *planRenderer) mappingBindParameter(raw any, bindKey string) (string, error) { + var parameter command.PostgreSQLParameter + var cast string + switch value := raw.(type) { + case nil: + parameter = command.PostgreSQLParameter{Type: command.PostgreSQLParameterNull, Value: nil} + cast = "::text" + case string: + parameter = command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: value} + cast = "::text" + case int: + parameter = command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(value)} + cast = "::bigint" + case int64: + parameter = command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: value} + cast = "::bigint" + case float64: + parameter = command.PostgreSQLParameter{Type: command.PostgreSQLParameterDoublePrecision, Value: value} + cast = "::double precision" + case bool: + parameter = command.PostgreSQLParameter{Type: command.PostgreSQLParameterBoolean, Value: value} + cast = "::boolean" + default: + return "", fmt.Errorf("%w: mapping identity bind %q carries %T; the typed band is string, int, int64, float64, bool, or nil", + ErrUnsupportedPostgresPlan, bindKey, raw) + } + index := r.addParameter(parameter, "mapping-bind:"+bindKey) + return "$" + strconv.Itoa(index) + cast, nil +} + +func (r *planRenderer) renderStageSequenceMappingStatement(plan ir.PhysicalPlan) (string, error) { + sequence := plan.StageSequence + terminal := sequence.PopulationMappingReturn + if terminal == nil { + return "", fmt.Errorf("%w: the composition mapping terminal requires the PopulationMappingReturn payload", + ErrUnsupportedPostgresPlan) + } + if terminal.SourceMemberIDsColumn != ir.PhysicalPopulationMappingMembersColumn || + terminal.SourceRootKeyColumn != "_key" || terminal.FinalRootContributorColumn == "" || + terminal.RowIdentityColumn == "" { + return "", fmt.Errorf("%w: composition mapping terminal columns are not the compiler-owned mapping fields", + ErrUnsupportedPostgresPlan) + } + chain, err := r.renderStageSequenceCTEChain(plan) + if err != nil { + return "", err + } + sourceKey, ok := chain.sourceFacts[terminal.SourceRootKeyColumn] + if !ok || sourceKey.jsonb { + return "", fmt.Errorf("%w: the composition source relation does not carry the materialized text %q root-key fact", + ErrUnsupportedPostgresPlan, terminal.SourceRootKeyColumn) + } + sourceMembers, ok := chain.sourceFacts[terminal.SourceMemberIDsColumn] + if !ok || !sourceMembers.jsonb { + return "", fmt.Errorf("%w: the composition source relation does not carry the member witness fact", + ErrUnsupportedPostgresPlan) + } + contributors, ok := chain.facts[terminal.FinalRootContributorColumn] + if !ok { + return "", fmt.Errorf("%w: final contributor column %q is not a column of the last stage", + ErrUnsupportedPostgresPlan, terminal.FinalRootContributorColumn) + } + identity, ok := chain.facts[terminal.RowIdentityColumn] + if !ok { + return "", fmt.Errorf("%w: row identity column %q is not a column of the last stage", + ErrUnsupportedPostgresPlan, terminal.RowIdentityColumn) + } + + contribRef := stageTerminalAlias + `."` + contributors.alias + `"` + var contributorGuard string + if terminal.FinalRootContributorsMany { + if !contributors.jsonb { + return "", fmt.Errorf("%w: many-cardinality contributors column %q is a SQL scalar and carries no array", + ErrUnsupportedPostgresPlan, terminal.FinalRootContributorColumn) + } + contributorGuard = "CASE WHEN " + contribRef + " IS NULL OR jsonb_typeof(" + contribRef + + ") <> 'array' THEN '[]'::jsonb ELSE " + contribRef + " END" + } else { + if contributors.jsonb { + return "", fmt.Errorf("%w: scalar contributors column %q is jsonb and has no registered singleton reading", + ErrUnsupportedPostgresPlan, terminal.FinalRootContributorColumn) + } + contributorGuard = "CASE WHEN " + contribRef + " IS NULL THEN '[]'::jsonb ELSE jsonb_build_array(" + contribRef + ") END" + } + memberRef := "ps." + `"` + sourceMembers.alias + `"` + memberGuard := "CASE WHEN " + memberRef + " IS NULL OR jsonb_typeof(" + memberRef + ") <> 'array' THEN '[]'::jsonb ELSE " + memberRef + " END" + identityRef := stageTerminalAlias + `."` + identity.alias + `"` + identityOrder := identityRef + ` COLLATE "C" ASC` + if identity.jsonb { + identityOrder = identityRef + `::text COLLATE "C" ASC` + } + + sql := "WITH " + strings.Join(chain.ctes, ", ") + + ` SELECT mn0.member AS "c0", jsonb_build_array(` + identityRef + `) AS "c1"` + + " FROM " + chain.relation + " AS " + stageTerminalAlias + + " CROSS JOIN LATERAL jsonb_array_elements_text(" + contributorGuard + `) AS cn0(contributor)` + + ` INNER JOIN ` + stageSourceRelation + ` AS ps ON ps."` + sourceKey.alias + `" = cn0.contributor` + + " CROSS JOIN LATERAL jsonb_array_elements_text(" + memberGuard + `) AS mn0(member)` + + " GROUP BY " + identityRef + ", mn0.member" + + " ORDER BY " + identityOrder + `, mn0.member COLLATE "C" ASC` + if sequence.PreviewLimitBindKey != "" { + limit, ok := plan.BindVars[sequence.PreviewLimitBindKey].(int) + if !ok || limit <= 0 { + return "", fmt.Errorf("%w: composition preview limit bind %q must be a positive int (render/aql/stage_sequence.go:452-453 twin)", + ErrUnsupportedPostgresPlan, sequence.PreviewLimitBindKey) + } + index := r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(limit)}, "") + sql += " LIMIT $" + strconv.Itoa(index) + } + identityColumn := ir.PhysicalPopulationMappingIdentityPartsField + r.outputColumns = []string{ir.PhysicalPopulationMappingMemberField, identityColumn} + return sql, nil +} diff --git a/internal/dataframe/compiler/render/postgres/population_mapping_test.go b/internal/dataframe/compiler/render/postgres/population_mapping_test.go new file mode 100644 index 000000000..37b355690 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/population_mapping_test.go @@ -0,0 +1,1003 @@ +package postgres + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "sync" + "testing" + "time" + + control "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" + explorer "github.com/calypr/loom/internal/explorer" +) + +var ( + sql22FixturesMu sync.Mutex + sql22TrackedFixtures = map[string]bool{} +) + +func sql22RunDeltaAudit(inner func() int) int { + return runVerifyNamespaceDeltaAudit("verify_sql22v1_", &sql22FixturesMu, sql22TrackedFixtures, inner) +} + +func sql22TrackResource(t *testing.T, name string) { + t.Helper() + sql22FixturesMu.Lock() + defer sql22FixturesMu.Unlock() + sql22TrackedFixtures[name] = true +} + +func map22Bindings() recipe.RuntimeBindings { + return recipe.RuntimeBindings{ + Project: "fixture_program-project-a", SelectionProject: "fixture_program/project-a", + DatasetGeneration: "generation-a", SelectionMembersCollection: "loom_explorer_selection_members", + } +} + +func map22Compile(t *testing.T, output recipe.Output) ir.PhysicalPlan { + t.Helper() + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "population-mapping-render-test", TranslationVersion: "test", + Outputs: []recipe.Output{output}, + } + if err := bundle.Validate(); err != nil { + t.Fatalf("bundle validate: %v", err) + } + plan, err := semantic.BuildRecipePlan(bundle, map22Bindings()) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, "scope-a", "generation-a") + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("lower: %v", err) + } + if len(compiled.Outputs) != 1 { + t.Fatalf("compiled outputs = %d, want 1", len(compiled.Outputs)) + } + return compiled.Outputs[0].Plan +} + +func map22Population(resourceType string) *recipe.PopulationConstraint { + return &recipe.PopulationConstraint{ + SelectionRevisionID: "selrev_22", MembershipDigest: "sha256:members-22", MemberCount: 3, + ResourceType: resourceType, + } +} + +func map22SpecimenOutput() recipe.Output { + return recipe.Output{ + Name: "Specimens", RootResourceType: "Specimen", RowGrain: "specimen", + Fields: []recipe.Field{{Name: "id", Expr: recipe.Expression{Select: "root.id"}}}, + Population: map22Population("Specimen"), + } +} + +func map22StampFlat(t *testing.T, plan ir.PhysicalPlan) ir.PhysicalPlan { + t.Helper() + plan = ir.ClonePhysicalPlan(plan) + scan := plan.Operations[0].RootScan + scan.Population.CollectMembersVariable = ir.PopulationMappingMembersVariable + returnIndex := -1 + for index, operation := range plan.Operations { + if operation.Kind == ir.PhysicalReturnOp && operation.Return != nil { + returnIndex = index + } + } + if returnIndex < 0 { + t.Fatal("flat mapping plan has no RETURN operation") + } + returnOp := plan.Operations[returnIndex].Return + projections := map[string]ir.PhysicalProjection{} + for _, projection := range returnOp.Projections { + projections[projection.Name] = projection + } + parts := make([]ir.PhysicalPopulationMappingIdentityPart, 0, 2) + for _, field := range []string{"project", "_key"} { + if projection, ok := projections[field]; ok { + expression := ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, Value: &projection.Value, + } + if projection.Expression != nil { + expression = ir.ClonePhysicalExpression(*projection.Expression) + } + parts = append(parts, ir.PhysicalPopulationMappingIdentityPart{Name: field, Expression: expression}) + continue + } + if _, bound := plan.BindVars[field]; bound { + parts = append(parts, ir.PhysicalPopulationMappingIdentityPart{ + Name: field, + Expression: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{BindKey: field}, + }, + }) + continue + } + t.Fatalf("identity field %q is neither projected nor bound", field) + } + members := ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Value: &ir.PhysicalValue{Variable: ir.PopulationMappingMembersVariable}, + } + plan.Operations[returnIndex] = ir.PhysicalOperation{ + Kind: ir.PhysicalPopulationMappingReturnOp, + Source: plan.Operations[returnIndex].Source, + PopulationMappingReturn: &ir.PhysicalPopulationMappingReturn{ + Members: members, IdentityParts: parts, + }, + } + if err := plan.Validate(); err != nil { + t.Fatalf("stamp flat validate: %v", err) + } + return plan +} + +func map22SummaryOutput(keyed bool) recipe.Output { + output := recipe.Output{ + Name: "construction_summary_mapping", RootResourceType: "Observation", RowGrain: "observation", + Fields: []recipe.Field{ + {Name: "status", ColumnID: "status_id", Expr: recipe.Expression{Select: "root.status"}}, + {Name: "amount", ColumnID: "amount_id", Expr: recipe.Expression{Select: "root.valueQuantity.value"}}, + }, + Construction: &recipe.Construction{ + Version: 1, + SourceColumns: []recipe.StageColumn{{ID: "status_id", Name: "status"}, {ID: "amount_id", Name: "amount"}}, + Steps: []recipe.ConstructionStep{{ + ID: "summary", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionGroupOp, Group: &recipe.ConstructionGroup{ + ConstructionID: "whole_table_summary", + Aggregates: []recipe.ConstructionGroupAggregate{ + {Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}, + {Operation: recipe.ConstructionGroupCountNonNull, InputColumnID: "status_id", OutputColumnID: "status_count_id"}, + }, + }}, + Outputs: []recipe.StageColumn{ + {ID: "rows_id", Name: "rows"}, {ID: "status_count_id", Name: "status_count"}, + }, + }}, + }, + Population: map22Population("Observation"), + } + if keyed { + group := output.Construction.Steps[0].Operation.Group + group.Keys = []recipe.ConstructionGroupKey{{InputColumnID: "status_id", OutputColumnID: "status_key_id"}} + output.Construction.Steps[0].Outputs = append( + []recipe.StageColumn{{ID: "status_key_id", Name: "status_key"}}, + output.Construction.Steps[0].Outputs...) + } + return output +} + +func map22StampConstruction(t *testing.T, plan ir.PhysicalPlan) ir.PhysicalPlan { + t.Helper() + plan = ir.ClonePhysicalPlan(plan) + plan.Operations[0].RootScan.Population.CollectMembersVariable = ir.PopulationMappingMembersVariable + rootKeyFound := false + for _, column := range plan.StageSequence.SourceColumns { + if column.Name == "_key" && column.Internal && + column.RootContributorResourceType == plan.Source.ResourceType && + column.Kind == "string" && column.Cardinality == "required_one" { + rootKeyFound = true + } + } + if !rootKeyFound { + t.Fatal("construction mapping source does not retain the exact root storage key") + } + stamped := false + for index := range plan.Operations { + operation := &plan.Operations[index] + if operation.Kind != ir.PhysicalReturnOp || operation.Return == nil { + continue + } + operation.Return.Projections = append(operation.Return.Projections, ir.PhysicalProjection{ + Name: ir.PhysicalPopulationMappingMembersColumn, Hidden: true, + Value: ir.PhysicalValue{Variable: ir.PopulationMappingMembersVariable}, + }) + stamped = true + break + } + if !stamped { + t.Fatal("construction mapping source has no terminal source RETURN") + } + contributorsMany := false + contributorFound := false + for _, column := range plan.StageSequence.FinalColumns { + if column.Internal && column.Name == "__loom_root_contributor_keys" && + column.RootContributorResourceType == plan.Source.ResourceType && column.Kind == "string" { + contributorsMany = column.Cardinality == "many" + contributorFound = true + } + } + if !contributorFound { + t.Fatal("construction mapping requires compiler-proven root contributor identities in its final stage") + } + plan.StageSequence.PopulationMappingReturn = &ir.PhysicalStagePopulationMappingReturn{ + SourceRootKeyColumn: "_key", SourceMemberIDsColumn: ir.PhysicalPopulationMappingMembersColumn, + FinalRootContributorColumn: "__loom_root_contributor_keys", FinalRootContributorsMany: contributorsMany, + RowIdentityColumn: plan.StageSequence.FinalRowIdentity, + } + if err := plan.Validate(); err != nil { + t.Fatalf("stamp construction validate: %v", err) + } + return plan +} + +func map22Render(t *testing.T, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderPopulationMappingPlan(plan, mustBinding(t), populationBindingOK(t)) + if err != nil { + t.Fatalf("RenderPopulationMappingPlan: %v\n%s", err, map22Dump(plan)) + } + return rendered +} + +func map22Dump(plan ir.PhysicalPlan) string { + var builder strings.Builder + for index, operation := range plan.Operations { + if index > 12 { + break + } + builder.WriteString(string(operation.Kind) + " ") + } + return builder.String() +} + +func map22WantRefusal(t *testing.T, plan ir.PhysicalPlan, want string) { + t.Helper() + binding := populationBindingOK(t) + _, renderErr := RenderPopulationMappingPlan(plan, mustBinding(t), binding) + validateErr := ValidatePopulationMappingSource(plan, binding) + if renderErr == nil { + t.Fatalf("render accepted a plan that must refuse %q:\n%s", want, map22Dump(plan)) + } + if !strings.Contains(renderErr.Error(), want) { + t.Fatalf("refusal for %q says %q", want, renderErr) + } + if validateErr == nil || validateErr.Error() != renderErr.Error() { + t.Fatalf("entry==dispatch broken for %q: render=%v validate=%v", want, renderErr, validateErr) + } + if !strings.Contains(renderErr.Error(), ErrUnsupportedPostgresPlan.Error()) && + !strings.HasPrefix(renderErr.Error(), "validate physical plan:") { + t.Fatalf("refusal for %q is neither typed nor the registered ir gate: %v", want, renderErr) + } +} + +func map22WantSQL(t *testing.T, sql string, wants ...string) { + t.Helper() + for _, want := range wants { + if !strings.Contains(sql, want) { + t.Fatalf("mapping SQL is missing %q:\n%s", want, sql) + } + } +} + +func TestPopulationMappingFlatWitnessSQLIsExactAndParameterized(t *testing.T) { + plan := map22StampFlat(t, map22Compile(t, map22SpecimenOutput())) + rendered := map22Render(t, plan) + if rendered.SQL == "" { + t.Fatal("empty SQL") + } + if want := []string{ir.PhysicalPopulationMappingMemberField, ir.PhysicalPopulationMappingIdentityPartsField}; !reflect.DeepEqual(rendered.OutputColumns, want) { + t.Fatalf("output columns = %#v, want %#v", rendered.OutputColumns, want) + } + memberTable := `FROM "` + testSchema + `"."selection_members" AS pm` + map22WantSQL(t, rendered.SQL, + memberTable, + `to_jsonb(ARRAY_AGG(DISTINCT (pm."id" COLLATE "C") ORDER BY (pm."id" COLLATE "C"))) AS "__loom_population_members"`, + `CROSS JOIN LATERAL jsonb_array_elements_text(CASE WHEN pop."__loom_population_members" IS NULL OR jsonb_typeof(pop."__loom_population_members") <> 'array' THEN '[]'::jsonb ELSE pop."__loom_population_members" END) AS w0(member)`, + `SELECT w0.member AS "c0", jsonb_build_array(`, + `w0.member COLLATE "C" ASC`, + ) + if strings.Contains(rendered.SQL, "@") { + t.Fatalf("mapping SQL carries AQL bind syntax:\n%s", rendered.SQL) + } + if strings.Contains(rendered.SQL, "selrev_22") || strings.Contains(rendered.SQL, "fixture_program") { + t.Fatalf("mapping SQL inlines plan data:\n%s", rendered.SQL) + } + foundSelection := false + for _, parameter := range rendered.Args { + if text, ok := parameter.Value.(string); ok && text == "selrev_22" { + foundSelection = true + } + } + if !foundSelection { + t.Fatalf("selection revision bind never parameterized: %#v", rendered.Args) + } +} + +func TestPopulationMappingFlatDeterministic(t *testing.T) { + plan := map22StampFlat(t, map22Compile(t, map22SpecimenOutput())) + first := map22Render(t, plan) + second := map22Render(t, plan) + if first.SQL != second.SQL { + t.Fatal("rendering the same plan twice produced different SQL") + } + if len(first.Args) != len(second.Args) { + t.Fatalf("argument count drifted: %d vs %d", len(first.Args), len(second.Args)) + } + for index := range first.Args { + if first.Args[index].Type != second.Args[index].Type || !reflect.DeepEqual(first.Args[index].Value, second.Args[index].Value) { + t.Fatalf("argument %d drifted: %#v vs %#v", index, first.Args[index], second.Args[index]) + } + } +} + +func TestPopulationMappingFlatExplicitIdentityPreservesCompiledExpression(t *testing.T) { + output := map22SpecimenOutput() + output.Identity = &recipe.Identity{Name: "row", Expr: recipe.Expression{Select: "root.id"}} + plan := map22Compile(t, output) + plan = ir.ClonePhysicalPlan(plan) + plan.Operations[0].RootScan.Population.CollectMembersVariable = ir.PopulationMappingMembersVariable + returnIndex := -1 + for index, operation := range plan.Operations { + if operation.Kind == ir.PhysicalReturnOp && operation.Return != nil { + returnIndex = index + } + } + var explicit *ir.PhysicalExpression + for _, projection := range plan.Operations[returnIndex].Return.Projections { + if projection.Name == "__loom_row_id" && projection.Expression != nil { + expression := ir.ClonePhysicalExpression(*projection.Expression) + explicit = &expression + } + } + if explicit == nil { + t.Fatal("compiled explicit identity projection is missing") + } + parts := []ir.PhysicalPopulationMappingIdentityPart{ + {Name: "project", Expression: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, Value: &ir.PhysicalValue{BindKey: "project"}, + }}, + {Name: "_key", Expression: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, Value: &ir.PhysicalValue{Variable: plan.Operations[0].RootScan.Variable, Path: []string{"_key"}}, + }}, + } + plan.Operations[returnIndex] = ir.PhysicalOperation{ + Kind: ir.PhysicalPopulationMappingReturnOp, + Source: plan.Operations[returnIndex].Source, + PopulationMappingReturn: &ir.PhysicalPopulationMappingReturn{ + Members: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Value: &ir.PhysicalValue{Variable: ir.PopulationMappingMembersVariable}, + }, + IdentityParts: parts, + ExplicitIdentity: explicit, + }, + } + if err := plan.Validate(); err != nil { + t.Fatalf("explicit stamp validate: %v", err) + } + rendered := map22Render(t, plan) + if want := []string{ir.PhysicalPopulationMappingMemberField, ir.PhysicalPopulationMappingExplicitIdentityField}; !reflect.DeepEqual(rendered.OutputColumns, want) { + t.Fatalf("output columns = %#v, want %#v", rendered.OutputColumns, want) + } + map22WantSQL(t, rendered.SQL, `#> $`, `AS "c1"`) + if strings.Contains(rendered.SQL, "jsonb_build_array(") { + t.Fatalf("explicit identity re-spelled as a parts array:\n%s", rendered.SQL) + } + if strings.Contains(rendered.SQL, "root.payload.id") || strings.Contains(rendered.SQL, "__loom_row_id") { + t.Fatalf("explicit identity leaked a plan string into SQL:\n%s", rendered.SQL) + } +} + +func TestPopulationMappingConstructionWitnessSQL(t *testing.T) { + for _, keyed := range []bool{false, true} { + name := "grand-total" + if keyed { + name = "keyed" + } + t.Run(name, func(t *testing.T) { + plan := map22StampConstruction(t, map22Compile(t, map22SummaryOutput(keyed))) + rendered := map22Render(t, plan) + map22WantSQL(t, rendered.SQL, + "WITH __loom_src AS (", + `FROM "`+testSchema+`"."selection_members" AS pm`, + `pop."__loom_population_members" AS "c3"`, + "rcpool AS (", + "ARRAY_AGG(DISTINCT", + "INNER JOIN __loom_src AS ps ON", + "CROSS JOIN LATERAL jsonb_array_elements_text(", + `GROUP BY s."`, + `COLLATE "C" ASC, mn0.member COLLATE "C" ASC`, + ) + if strings.Contains(rendered.SQL, "@") { + t.Fatalf("construction mapping SQL carries AQL bind syntax:\n%s", rendered.SQL) + } + if want := []string{ir.PhysicalPopulationMappingMemberField, ir.PhysicalPopulationMappingIdentityPartsField}; !reflect.DeepEqual(rendered.OutputColumns, want) { + t.Fatalf("output columns = %#v, want %#v", rendered.OutputColumns, want) + } + }) + } +} + +func TestPopulationMappingConstructionPreviewLimitIsBound(t *testing.T) { + plan := map22StampConstruction(t, map22Compile(t, map22SummaryOutput(true))) + plan.StageSequence.PreviewLimitBindKey = "preview_limit" + plan.BindVars["preview_limit"] = 7 + rendered := map22Render(t, plan) + if !strings.HasSuffix(rendered.SQL, " LIMIT $13") && !strings.Contains(rendered.SQL, " LIMIT $") { + t.Fatalf("preview limit is not a bound parameter:\n%s", rendered.SQL) + } + found := false + for _, parameter := range rendered.Args { + if value, ok := parameter.Value.(int64); ok && value == 7 && parameter.Type == command.PostgreSQLParameterBigint { + found = true + } + } + if !found { + t.Fatalf("preview limit bind is not a typed bigint 7: %#v", rendered.Args) + } + broken := ir.ClonePhysicalPlan(plan) + broken.BindVars["preview_limit"] = 0 + map22WantRefusal(t, broken, "preview limit bind") +} + +func TestPopulationMappingRefusals(t *testing.T) { + flat := func() ir.PhysicalPlan { + return map22StampFlat(t, map22Compile(t, map22SpecimenOutput())) + } + construction := func() ir.PhysicalPlan { + return map22StampConstruction(t, map22Compile(t, map22SummaryOutput(true))) + } + + t.Run("wrong-collect-members-variable", func(t *testing.T) { + plan := flat() + plan.Operations[0].RootScan.Population.CollectMembersVariable = "someone_elses_members" + map22WantRefusal(t, plan, "CollectMembersVariable") + }) + t.Run("plain-population-root-unstamped", func(t *testing.T) { + plan := map22Compile(t, map22SpecimenOutput()) + map22WantRefusal(t, plan, "CollectMembersVariable") + }) + t.Run("members-variable-mismatch", func(t *testing.T) { + plan := flat() + terminal := plan.Operations[len(plan.Operations)-1].PopulationMappingReturn + terminal.Members.Value.Variable = "wrong_members" + map22WantRefusal(t, plan, "collected-member value") + }) + t.Run("identity-less-terminal", func(t *testing.T) { + plan := flat() + terminal := plan.Operations[len(plan.Operations)-1].PopulationMappingReturn + terminal.IdentityParts = nil + map22WantRefusal(t, plan, "no renderable value") + }) + t.Run("unnamed-identity-part", func(t *testing.T) { + plan := flat() + terminal := plan.Operations[len(plan.Operations)-1].PopulationMappingReturn + terminal.IdentityParts[0].Name = "" + map22WantRefusal(t, plan, "has no name") + }) + t.Run("flat-return-alongside-terminal", func(t *testing.T) { + plan := flat() + terminal := *plan.Operations[len(plan.Operations)-1].PopulationMappingReturn + plan.Operations = plan.Operations[:len(plan.Operations)-1] + plan.Operations = append(plan.Operations, ir.PhysicalOperation{ + Kind: ir.PhysicalReturnOp, + Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "x", Value: ir.PhysicalValue{BindKey: "project"}}, + }}, + }) + plan.Operations = append(plan.Operations, ir.PhysicalOperation{ + Kind: ir.PhysicalPopulationMappingReturnOp, PopulationMappingReturn: &terminal, + }) + map22WantRefusal(t, plan, "replaces the RETURN projection list") + }) + t.Run("flat-terminal-not-last", func(t *testing.T) { + plan := flat() + terminal := *plan.Operations[len(plan.Operations)-1].PopulationMappingReturn + operations := []ir.PhysicalOperation{ + plan.Operations[0], + {Kind: ir.PhysicalPopulationMappingReturnOp, PopulationMappingReturn: &terminal}, + } + operations = append(operations, plan.Operations[1:len(plan.Operations)-1]...) + plan.Operations = operations + map22WantRefusal(t, plan, "single final operation") + }) + t.Run("unnest-on-mapping-root", func(t *testing.T) { + plan := flat() + terminal := *plan.Operations[len(plan.Operations)-1].PopulationMappingReturn + plan.Operations = plan.Operations[:len(plan.Operations)-1] + plan.Operations = append(plan.Operations, ir.PhysicalOperation{ + Kind: ir.PhysicalUnnestOp, + Unnest: &ir.PhysicalUnnest{OutputVariable: "item"}, + }) + plan.Operations = append(plan.Operations, ir.PhysicalOperation{ + Kind: ir.PhysicalPopulationMappingReturnOp, PopulationMappingReturn: &terminal, + }) + map22WantRefusal(t, plan, "named capability gap") + }) + t.Run("construction-missing-member-projection", func(t *testing.T) { + plan := map22Compile(t, map22SummaryOutput(true)) + plan = ir.ClonePhysicalPlan(plan) + plan.Operations[0].RootScan.Population.CollectMembersVariable = ir.PopulationMappingMembersVariable + plan.StageSequence.PopulationMappingReturn = &ir.PhysicalStagePopulationMappingReturn{ + SourceRootKeyColumn: "_key", SourceMemberIDsColumn: ir.PhysicalPopulationMappingMembersColumn, + FinalRootContributorColumn: "__loom_root_contributor_keys", FinalRootContributorsMany: false, + RowIdentityColumn: plan.StageSequence.FinalRowIdentity, + } + map22WantRefusal(t, plan, "omits the compiler-owned member witness projection") + }) + t.Run("construction-foreign-contributor-column", func(t *testing.T) { + plan := construction() + plan.StageSequence.PopulationMappingReturn.FinalRootContributorColumn = "someone_elses_keys" + map22WantRefusal(t, plan, "not compiler-owned provenance") + }) + t.Run("construction-missing-terminal-payload", func(t *testing.T) { + plan := construction() + plan.StageSequence.PopulationMappingReturn = nil + map22WantRefusal(t, plan, "PopulationMappingReturn payload") + }) + t.Run("construction-foreign-terminal-columns", func(t *testing.T) { + plan := construction() + plan.StageSequence.PopulationMappingReturn.RowIdentityColumn = "" + map22WantRefusal(t, plan, "row identity column does not match") + }) + t.Run("preview-source-window-flag", func(t *testing.T) { + plan := flat() + plan.PreviewSourceWindowByRootID = true + map22WantRefusal(t, plan, "preview source windowing") + }) + t.Run("foreign-engine", func(t *testing.T) { + plan := flat() + plan.Engine = ir.PhysicalEngineClickHouse + map22WantRefusal(t, plan, "has no PostgreSQL source") + }) + t.Run("no-population-root", func(t *testing.T) { + plan := map22StampConstruction(t, map22Compile(t, map22SummaryOutput(false))) + plan.Operations[0].RootScan.Population = nil + map22WantRefusal(t, plan, "membership-driven population root source") + }) + for _, kind := range []struct { + name string + opKind ir.PhysicalOperationKind + opValue ir.PhysicalOperation + want string + }{ + {"nil-sort-payload", ir.PhysicalSortOp, ir.PhysicalOperation{Kind: ir.PhysicalSortOp}, "validate physical plan: operation 5 (SORT)"}, + {"nil-limit-payload", ir.PhysicalLimitOp, ir.PhysicalOperation{Kind: ir.PhysicalLimitOp}, "validate physical plan: operation 5 (LIMIT)"}, + {"nil-filter-payload", ir.PhysicalFilterOp, ir.PhysicalOperation{Kind: ir.PhysicalFilterOp}, "validate physical plan: operation 5 (FILTER)"}, + } { + kind := kind + t.Run(kind.name, func(t *testing.T) { + plan := flat() + terminal := plan.Operations[len(plan.Operations)-1] + plan.Operations = append(plan.Operations[:len(plan.Operations)-1], kind.opValue, terminal) + map22WantRefusal(t, plan, kind.want) + }) + } +} + +func TestPopulationMappingPlainEntriesKeepTheDoorsShut(t *testing.T) { + plan := map22StampFlat(t, map22Compile(t, map22SpecimenOutput())) + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); err == nil || + !strings.Contains(err.Error(), ErrUnsupportedPostgresPlan.Error()) { + t.Fatalf("plain spine accepted a mapping plan: %v", err) + } + if _, err := RenderPopulationRootScan(plan, mustBinding(t), populationBindingOK(t)); err == nil || + !strings.Contains(err.Error(), "pg-sql-22 population terminal") { + t.Fatalf("RenderPopulationRootScan accepted a member witness root: %v", err) + } + construction := map22StampConstruction(t, map22Compile(t, map22SummaryOutput(true))) + if _, err := RenderPhysicalPlan(construction, mustBinding(t)); err == nil || + !strings.Contains(err.Error(), ErrUnsupportedPostgresPlan.Error()) { + t.Fatalf("composition door accepted a mapping terminal: %v", err) + } + if _, err := renderStageSequencePhysicalPlan(construction, mustBinding(t)); err == nil || + !strings.Contains(err.Error(), "population-mapping returns have no PostgreSQL compiled route") { + t.Fatalf("composition door surface spoke a different voice: %v", err) + } +} + +const ( + map22Project = "sql22-proj" + map22Foreign = "sql22-proj-x" + map22Generation = "gen-1" + map22ObsAuth = "FHIR/Observation" + map22PatAuth = "FHIR/Patient" + map22SecretAuth = "FHIR/Secret" + map22RouteLabel = "encounter-diagnosed_condition" +) + +type map22Service struct { + *pgir02Service + store *control.ExplorerStore + binding ir.PostgresStorageBinding + members PostgresPopulationRootBinding +} + +func newMap22Service(t *testing.T) *map22Service { + t.Helper() + svc := newOwnedService(t, 0, "verify_sql22v1") + var database string + if err := svc.iso.QueryRow(svc.ctx, "SELECT current_database()").Scan(&database); err != nil { + svc.fail("track owned database", err) + } + sql22TrackResource(t, database) + sql22TrackResource(t, svc.schema) + applyExplorerMembershipDDL(t, svc) + store, err := control.NewExplorerStore(svc.client, svc.schema) + if err != nil { + svc.fail("open explorer store", err) + } + binding, err := ir.NewPostgresStorageBinding(svc.schema) + if err != nil { + svc.fail("storage binding", err) + } + members, err := NewPostgresPopulationRootBinding(svc.schema) + if err != nil { + svc.fail("membership binding", err) + } + return &map22Service{pgir02Service: svc, store: store, binding: binding, members: members} +} + +func (s *map22Service) render(t *testing.T, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderPopulationMappingPlan(plan, s.binding, s.members) + if err != nil { + s.fail("render mapping plan", err) + } + return rendered +} + +func (s *map22Service) seedSelection(t *testing.T, id, project, resourceType string, memberIDs []string) { + t.Helper() + now := time.Now().UTC() + header, err := s.store.BeginSelection(s.ctx, explorer.SelectionRevision{ + ID: id, Project: project, Generation: map22Generation, ResourceType: resourceType, + Rule: explorer.SelectionRule{Kind: explorer.SelectionRuleExplicit}, + Source: explorer.SelectionSource{Kind: explorer.SelectionSourceExplicit}, + ScopeDigest: "scope-22", RuleDigest: "rule-22", CreatedAt: now, + }, "sql22-writer") + if err != nil { + s.fail("begin selection "+id, err) + } + members := make([]explorer.SelectionMember, 0, len(memberIDs)) + for _, memberID := range memberIDs { + members = append(members, explorer.SelectionMember{Ref: explorer.ResourceRef{ + Project: project, Generation: map22Generation, ResourceType: resourceType, ID: memberID}}) + } + if _, err := s.store.AppendSelectionMembers(s.ctx, header.ID, "sql22-writer", members); err != nil { + s.fail("append selection members "+id, err) + } + digest, count, bytes, err := s.store.DigestSelectionMembers(s.ctx, project, header.ID) + if err != nil { + s.fail("digest selection "+id, err) + } + if _, err := s.store.CompleteSelection(s.ctx, header.ID, "sql22-writer", digest, count, bytes, time.Now().UTC()); err != nil { + s.fail("complete selection "+id, err) + } +} + +func map22Observation(t *testing.T, s *map22Service, logicalID, status string, value float64, authPath string) { + t.Helper() + s.insertResource(t, s.schema, map22Project, map22Generation, "Observation", logicalID, authPath, + map[string]any{ + "resourceType": "Observation", "id": logicalID, "status": status, + "valueQuantity": map[string]any{"value": value}, + }) +} + +func map22Rebind(t *testing.T, plan ir.PhysicalPlan, selectionID string) ir.PhysicalPlan { + t.Helper() + plan = ir.ClonePhysicalPlan(plan) + seen := map[string]bool{} + for key, value := range plan.BindVars { + text, ok := value.(string) + if !ok { + continue + } + switch text { + case "fixture_program-project-a", "fixture_program/project-a": + plan.BindVars[key] = map22Project + seen["project"] = true + case "generation-a": + plan.BindVars[key] = map22Generation + seen["generation"] = true + case "selrev_22": + plan.BindVars[key] = selectionID + seen["selection"] = true + } + } + for _, want := range []string{"project", "generation", "selection"} { + if !seen[want] { + t.Fatalf("rebind never saw the offline %s constant", want) + } + } + plan.BindVars["auth_resource_paths"] = []string{map22ObsAuth, map22PatAuth} + plan.BindVars["auth_resource_paths_unrestricted"] = false + plan.BindVars["scope_allowed"] = true + return plan +} + +func map22LiveFlatPlan(hops int, selectionID string, authPaths []string, unrestricted, allowed bool) ir.PhysicalPlan { + plan := populationPlan(hops) + plan.BindVars["project"] = map22Project + plan.BindVars["dataset_generation"] = map22Generation + plan.BindVars["population_project"] = map22Project + plan.BindVars["population_selection_id"] = selectionID + plan.BindVars["population_resource_type"] = "Observation" + plan.BindVars["population_source_collection"] = "Observation" + plan.BindVars["root_collection"] = "Observation" + if hops > 0 { + plan.BindVars["root_collection"] = "Patient" + plan.BindVars["population_root_route_0_target_type"] = "Patient" + plan.BindVars["population_root_route_0_edge_label"] = map22RouteLabel + } + plan.BindVars["auth_resource_paths"] = authPaths + plan.BindVars["auth_resource_paths_unrestricted"] = unrestricted + plan.BindVars["scope_allowed"] = allowed + plan.Operations[0].RootScan.Population.CollectMembersVariable = ir.PopulationMappingMembersVariable + returnIndex := -1 + for index, operation := range plan.Operations { + if operation.Kind == ir.PhysicalReturnOp && operation.Return != nil { + returnIndex = index + } + } + plan.Operations[returnIndex] = ir.PhysicalOperation{ + Kind: ir.PhysicalPopulationMappingReturnOp, + Source: plan.Operations[returnIndex].Source, + PopulationMappingReturn: &ir.PhysicalPopulationMappingReturn{ + Members: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Value: &ir.PhysicalValue{Variable: ir.PopulationMappingMembersVariable}, + }, + IdentityParts: []ir.PhysicalPopulationMappingIdentityPart{ + {Name: "project", Expression: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, Value: &ir.PhysicalValue{BindKey: "project"}, + }}, + {Name: "_key", Expression: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{Variable: plan.Operations[0].RootScan.Variable, Path: []string{"_key"}}, + }}, + }, + }, + } + return plan +} + +func (s *map22Service) runRows(t *testing.T, rendered command.RenderedPostgresPhysicalPlan) [][]string { + t.Helper() + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + rows, err := s.iso.Query(s.ctx, rendered.SQL, args...) + if err != nil { + s.fail("execute mapping statement", err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + s.fail("read mapping row", err) + } + row := make([]string, len(values)) + for index, value := range values { + row[index] = map22Cell(value) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + s.fail("mapping row iteration", err) + } + return out +} + +func map22Cell(value any) string { + switch value.(type) { + case []any, map[string]any: + text, err := json.Marshal(value) + if err != nil { + return fmt.Sprintf("", err) + } + return string(text) + default: + return stringifyCell(value) + } +} + +func map22JoinRows(rows [][]string) string { + parts := make([]string, 0, len(rows)) + for _, row := range rows { + parts = append(parts, strings.Join(row, "|")) + } + return strings.Join(parts, " ; ") +} + +func TestPopulationMappingLiveExactRowsAndScope(t *testing.T) { + s := newMap22Service(t) + map22Observation(t, s, "O1", "final", 1, map22ObsAuth) + map22Observation(t, s, "O2", "final", 2, map22ObsAuth) + map22Observation(t, s, "O3", "final", 3, map22ObsAuth) + s.seedSelection(t, "sel22_obs", map22Project, "Observation", []string{"O1", "O2"}) + s.seedSelection(t, "sel22_obs_x", map22Foreign, "Observation", []string{"O3"}) + plan := map22LiveFlatPlan(0, "sel22_obs", []string{map22ObsAuth}, false, true) + rows := s.runRows(t, s.render(t, plan)) + want := `O1|["` + map22Project + `","O1"] ; O2|["` + map22Project + `","O2"]` + if got := map22JoinRows(rows); got != want { + t.Fatalf("witness rows = %q, want %q", got, want) + } +} + +func TestPopulationMappingLiveRouteMembersDedupAndOrder(t *testing.T) { + s := newMap22Service(t) + map22Observation(t, s, "O1", "final", 1, map22ObsAuth) + map22Observation(t, s, "O2", "final", 2, map22ObsAuth) + map22Observation(t, s, "O3", "final", 3, map22ObsAuth) + s.seedSelection(t, "sel22_obs", map22Project, "Observation", []string{"O1", "O2"}) + s.insertResource(t, s.schema, map22Project, map22Generation, "Patient", "P1", map22PatAuth, + map[string]any{"resourceType": "Patient", "id": "P1"}) + s.insertResource(t, s.schema, map22Project, map22Generation, "Patient", "P2", map22PatAuth, + map[string]any{"resourceType": "Patient", "id": "P2"}) + for _, pair := range []struct{ target, path string }{{"O1", "focus[0]"}, {"O1", "focus[1]"}, {"O2", "focus[2]"}} { + s.insertEdge(t, control.EdgeOccurrenceRow{ + Project: map22Project, DatasetGeneration: map22Generation, AuthResourcePath: map22PatAuth, + Label: map22RouteLabel, SourceResourceType: "Patient", SourceLogicalID: "P1", + TargetResourceType: "Observation", TargetLogicalID: pair.target, SourcePath: pair.path, + }) + } + s.insertEdge(t, control.EdgeOccurrenceRow{ + Project: map22Project, DatasetGeneration: map22Generation, AuthResourcePath: map22PatAuth, + Label: map22RouteLabel, SourceResourceType: "Patient", SourceLogicalID: "P2", + TargetResourceType: "Observation", TargetLogicalID: "O3", SourcePath: "focus[0]", + }) + plan := map22LiveFlatPlan(1, "sel22_obs", []string{map22PatAuth, map22ObsAuth}, false, true) + rendered := s.render(t, plan) + rows := s.runRows(t, rendered) + want := `O1|["` + map22Project + `","P1"] ; O2|["` + map22Project + `","P1"]` + if got := map22JoinRows(rows); got != want { + t.Fatalf("route witness rows = %q, want %q\nSQL: %s", got, want, rendered.SQL) + } + raw := s.witness(t, `SELECT count(*) FROM "`+s.schema+`".selection_members pm + INNER JOIN "`+s.schema+`".fhir_resource x0 ON x0.project = $1 AND x0.dataset_generation = $2 + AND x0.resource_type = 'Observation' AND x0.logical_id = pm.id + INNER JOIN "`+s.schema+`".fhir_edge_occurrence e1 ON e1.project = $1 AND e1.dataset_generation = $2 + AND e1.label = $3 AND e1.target_resource_type = 'Observation' AND e1.target_logical_id = x0.logical_id + AND e1.source_resource_type = 'Patient' AND e1.source_logical_id = 'P1' + WHERE pm.selection_id = $4 AND pm.project = $1`, + map22Project, map22Generation, map22RouteLabel, "sel22_obs") + if len(raw) != 1 || raw[0][0] != "3" { + t.Fatalf("duplicate-edge fixture not present (raw aggregation = %v, want 3 rows)", raw) + } +} + +func TestPopulationMappingLiveEmptyPopulationNoRows(t *testing.T) { + s := newMap22Service(t) + map22Observation(t, s, "O1", "final", 1, map22ObsAuth) + s.seedSelection(t, "sel22_empty", map22Project, "Observation", []string{"O9"}) + plan := map22LiveFlatPlan(0, "sel22_empty", []string{map22ObsAuth}, false, true) + rows := s.runRows(t, s.render(t, plan)) + if len(rows) != 0 { + t.Fatalf("empty population produced witness rows: %v", map22JoinRows(rows)) + } + members := s.witness(t, `SELECT count(*) FROM "`+s.schema+`".selection_members + WHERE selection_id = $1 AND project = $2`, "sel22_empty", map22Project) + if len(members) != 1 || members[0][0] != "1" { + t.Fatalf("empty-population fixture missing: %v", members) + } +} + +func TestPopulationMappingLiveConstructionAndPreviewLimit(t *testing.T) { + s := newMap22Service(t) + map22Observation(t, s, "O1", "final", 1, map22ObsAuth) + map22Observation(t, s, "O2", "final", 2, map22ObsAuth) + map22Observation(t, s, "O3", "amended", 3, map22ObsAuth) + s.seedSelection(t, "sel22_obs", map22Project, "Observation", []string{"O1", "O2"}) + plan := map22Rebind(t, map22StampConstruction(t, map22CompileLive(t, map22SummaryOutput(true))), "sel22_obs") + rendered := s.render(t, plan) + rows := s.runRows(t, rendered) + if len(rows) != 2 || rows[0][0] != "O1" || rows[1][0] != "O2" { + t.Fatalf("construction witness rows = %v", map22JoinRows(rows)) + } + if rows[0][1] != rows[1][1] { + t.Fatalf("both members must share the one group identity: %v", map22JoinRows(rows)) + } + limited := ir.ClonePhysicalPlan(plan) + limited.StageSequence.PreviewLimitBindKey = "preview_limit" + limited.BindVars["preview_limit"] = 1 + limitedRendered := s.render(t, limited) + if !strings.Contains(limitedRendered.SQL, " LIMIT $") { + t.Fatalf("preview limit not parameterized:\n%s", limitedRendered.SQL) + } + limitedRows := s.runRows(t, limitedRendered) + if len(limitedRows) != 1 || limitedRows[0][0] != rows[0][0] || limitedRows[0][1] != rows[0][1] { + t.Fatalf("preview prefix must equal the full-run first row: %v vs %v", + map22JoinRows(limitedRows), map22JoinRows(rows)) + } +} + +func map22CompileLive(t *testing.T, output recipe.Output) ir.PhysicalPlan { + t.Helper() + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "population-mapping-live", TranslationVersion: "test", + Outputs: []recipe.Output{output}, + } + if err := bundle.Validate(); err != nil { + t.Fatalf("bundle validate: %v", err) + } + plan, err := semantic.BuildRecipePlan(bundle, map22Bindings()) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, map22Project, map22Generation) + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("lower: %v", err) + } + if len(compiled.Outputs) != 1 { + t.Fatalf("compiled outputs = %d, want 1", len(compiled.Outputs)) + } + return compiled.Outputs[0].Plan +} + +func TestPopulationMappingLiveNullIdentityPartSurvives(t *testing.T) { + s := newMap22Service(t) + map22Observation(t, s, "O1", "final", 1, map22ObsAuth) + s.seedSelection(t, "sel22_obs", map22Project, "Observation", []string{"O1"}) + plan := map22LiveFlatPlan(0, "sel22_obs", []string{map22ObsAuth}, false, true) + terminal := plan.Operations[len(plan.Operations)-1].PopulationMappingReturn + terminal.IdentityParts = []ir.PhysicalPopulationMappingIdentityPart{ + {Name: "nullable", Expression: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, Value: &ir.PhysicalValue{BindKey: "nullable_part"}, + }}, + {Name: "_key", Expression: terminal.IdentityParts[1].Expression}, + } + plan.BindVars["nullable_part"] = nil + rows := s.runRows(t, s.render(t, plan)) + want := `O1|[null,"O1"]` + if got := map22JoinRows(rows); got != want { + t.Fatalf("null-part witness rows = %q, want %q", got, want) + } +} + +func TestPopulationMappingLiveStableOrderAndDeny(t *testing.T) { + s := newMap22Service(t) + map22Observation(t, s, "O1", "final", 1, map22ObsAuth) + map22Observation(t, s, "O2", "final", 2, map22ObsAuth) + s.seedSelection(t, "sel22_obs", map22Project, "Observation", []string{"O1", "O2"}) + plan := map22LiveFlatPlan(0, "sel22_obs", []string{map22ObsAuth}, false, true) + rendered := s.render(t, plan) + first := map22JoinRows(s.runRows(t, rendered)) + second := map22JoinRows(s.runRows(t, rendered)) + if first != second { + t.Fatalf("repeat execution diverged:\n%s\n%s", first, second) + } + denied := ir.ClonePhysicalPlan(plan) + denied.BindVars["scope_allowed"] = false + denyRows := s.runRows(t, s.render(t, denied)) + if len(denyRows) != 0 { + t.Fatalf("canonical deny read rows: %v", map22JoinRows(denyRows)) + } + unauth := ir.ClonePhysicalPlan(plan) + unauth.BindVars["auth_resource_paths"] = []string{map22SecretAuth} + if len(s.runRows(t, s.render(t, unauth))) != 0 { + t.Fatal("auth exclusion read rows") + } +} diff --git a/internal/dataframe/compiler/render/postgres/population_test.go b/internal/dataframe/compiler/render/postgres/population_test.go new file mode 100644 index 000000000..41fb29f57 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/population_test.go @@ -0,0 +1,653 @@ +package postgres + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const ( + popTestMembersCollection = "loom_explorer_selection_members" + popTestSelectionCollection = "loom_explorer_selections" + popTestRevisionsCollection = "loom_explorer_explicit_group_revisions" + popTestGroupCollection = "loom_explorer_explicit_group_memberships" +) + +func populationBindingOK(t *testing.T) PostgresPopulationRootBinding { + t.Helper() + binding, err := NewPostgresPopulationRootBinding(testSchema) + if err != nil { + t.Fatalf("NewPostgresPopulationRootBinding: %v", err) + } + return binding +} + +func renderPopulationOK(t *testing.T, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderPopulationRootScan(plan, mustBinding(t), populationBindingOK(t)) + if err != nil { + t.Fatalf("RenderPopulationRootScan: %v", err) + } + return rendered +} + +func populationPlan(hops int) ir.PhysicalPlan { + plan := basePlan() + scan := plan.Operations[0].RootScan + memberVariable := "population_member" + sourceVariable := "population_source" + memberID := ir.PhysicalValue{Variable: memberVariable, Path: []string{"id"}} + memberBind := func(field string) *ir.PhysicalValue { + return &ir.PhysicalValue{BindKey: "population_" + field} + } + source := &ir.PhysicalPopulationRootSource{ + MemberScan: ir.PhysicalCollectionScan{Variable: memberVariable, CollectionBindKey: "population_members_collection"}, + MemberID: memberID, + MemberFilters: []ir.PhysicalFilter{ + canonicalMemberFilter(memberVariable, "selectionId", memberBind("selection_id")), + canonicalMemberFilter(memberVariable, "project", memberBind("project")), + canonicalMemberFilter(memberVariable, "generation", &ir.PhysicalValue{BindKey: "dataset_generation"}), + canonicalMemberFilter(memberVariable, "resourceType", memberBind("resource_type")), + }, + RootKey: ir.PhysicalValue{Variable: "population_root_node_0", Path: []string{"_key"}}, + } + if hops == 0 { + source.RootKey = ir.PhysicalValue{Variable: sourceVariable, Path: []string{"_key"}} + } + source.ResourceOperations = append(source.ResourceOperations, + ir.PhysicalOperation{Kind: ir.PhysicalCollectionScanOp, CollectionScan: &ir.PhysicalCollectionScan{ + Variable: sourceVariable, CollectionBindKey: "population_source_collection"}}) + source.ResourceOperations = appendScopeBlock(source.ResourceOperations, []string{sourceVariable}, "population_source_scope_allowed") + source.ResourceOperations = append(source.ResourceOperations, ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Predicate: ir.PhysicalPredicate{Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: sourceVariable, Path: []string{"id"}}, + Right: &memberID}}}) + terminal := sourceVariable + for hop := 0; hop < hops; hop++ { + target := fmt.Sprintf("population_root_node_%d", hop) + edge := fmt.Sprintf("population_root_edge_%d", hop) + prefix := fmt.Sprintf("population_root_route_%d", hop) + source.ResourceOperations = append(source.ResourceOperations, ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: terminal, TargetVariable: target, EdgeVariable: edge, + Direction: ir.PhysicalInbound, EdgeCollectionBindKey: prefix + "_edge_collection", + EdgeLabelBindKey: prefix + "_edge_label", TargetTypeBindKey: prefix + "_target_type", + EdgeTargetTypeField: "from_type", + }}) + source.ResourceOperations = appendScopeBlock(source.ResourceOperations, []string{edge, target}, prefix+"_scope_allowed") + terminal = target + } + scan.Population = source + plan.BindVars["population_selection_id"] = "selrev_7" + plan.BindVars["population_project"] = "proj-a" + plan.BindVars["population_resource_type"] = "Observation" + plan.BindVars["population_members_collection"] = popTestMembersCollection + plan.BindVars["population_source_collection"] = "Observation" + for hop := 0; hop < hops; hop++ { + prefix := fmt.Sprintf("population_root_route_%d", hop) + plan.BindVars[prefix+"_edge_collection"] = "fhir_edge" + plan.BindVars[prefix+"_edge_label"] = "encounter-diagnosed_condition" + plan.BindVars[prefix+"_target_type"] = "Patient" + } + return plan +} + +func canonicalMemberFilter(memberVariable, path string, right *ir.PhysicalValue) ir.PhysicalFilter { + return ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: memberVariable, Path: []string{path}}, + Right: right, + }} +} + +func appendScopeBlock(operations []ir.PhysicalOperation, variables []string, resultVariable string) []ir.PhysicalOperation { + projectRight := ir.PhysicalValue{BindKey: "project"} + generationRight := ir.PhysicalValue{BindKey: "dataset_generation"} + scopeRight := ir.PhysicalValue{BindKey: "scope_allowed"} + for _, variable := range variables { + operations = append(operations, ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Predicate: ir.PhysicalPredicate{Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: variable, Path: []string{"project"}}, + Right: &projectRight}}}) + } + for _, variable := range variables { + operations = append(operations, ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Predicate: ir.PhysicalPredicate{Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: variable, Path: []string{"dataset_generation"}}, + Right: &generationRight}}}) + } + inputs := make([]ir.PhysicalValue, 0, len(variables)+2) + for _, variable := range variables { + inputs = append(inputs, ir.PhysicalValue{Variable: variable, Path: []string{"auth_resource_path"}}) + } + inputs = append(inputs, + ir.PhysicalValue{BindKey: "auth_resource_paths"}, + ir.PhysicalValue{BindKey: "auth_resource_paths_unrestricted"}) + operations = append(operations, ir.PhysicalOperation{Kind: ir.PhysicalDerivedLetOp, DerivedLet: &ir.PhysicalDerivedLet{ + Variable: resultVariable, Operator: "AUTH_RESOURCE_PATH_ALLOWED", Inputs: inputs}}) + operations = append(operations, ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Predicate: ir.PhysicalPredicate{Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: resultVariable}, + Right: &scopeRight}}}) + return operations +} + +func cohortPlan() ir.PhysicalPlan { + plan := basePlan() + plan.Source = ir.PhysicalSource{ResourceType: "Patient"} + plan.Operations[0].RootScan.CohortSource = &ir.PhysicalCohortRootSource{ + CohortStageID: "group_rows", CohortInputStageID: "source-stage", + RootIdentityColumn: "_key", RootResourceType: "Patient", + RevisionCollectionBindKey: "cohort_revision_collection", + SelectionCollectionBindKey: "cohort_selection_collection", + SelectionMembersCollectionBindKey: "cohort_selection_members_collection", + MembershipsCollectionBindKey: "cohort_memberships_collection", + RevisionIDBindKey: "cohort_revision_id", + ProjectBindKey: "cohort_project", + ResourceProjectBindKey: "cohort_resource_project", + DatasetGenerationBindKey: "cohort_dataset_generation", + ResourceTypeBindKey: "cohort_resource_type", + PolicyBindKey: "cohort_policy", + } + plan.BindVars["cohort_revision_collection"] = popTestRevisionsCollection + plan.BindVars["cohort_selection_collection"] = popTestSelectionCollection + plan.BindVars["cohort_selection_members_collection"] = popTestMembersCollection + plan.BindVars["cohort_memberships_collection"] = popTestGroupCollection + plan.BindVars["cohort_revision_id"] = "grouprev_1" + plan.BindVars["cohort_project"] = "proj-a" + plan.BindVars["cohort_resource_project"] = "proj-a" + plan.BindVars["cohort_dataset_generation"] = "gen-1" + plan.BindVars["cohort_resource_type"] = "Patient" + plan.BindVars["cohort_policy"] = "EXCLUDE" + return plan +} + +func TestMembershipBindingTrust(t *testing.T) { + if _, err := NewPostgresPopulationRootBinding(""); !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + t.Fatalf("empty schema error = %v, want ErrUnregisteredPostgresIdentifier", err) + } + for _, schema := range []string{"Public", `x"; DROP`, "0abc", strings.Repeat("a", 64)} { + if _, err := NewPostgresPopulationRootBinding(schema); !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + t.Errorf("schema %q error = %v, want ErrUnregisteredPostgresIdentifier", schema, err) + } + } + if err := (PostgresPopulationRootBinding{}).Validate(); !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + t.Fatalf("zero binding error = %v, want ErrUnregisteredPostgresIdentifier", err) + } + if _, err := RenderPopulationRootScan(populationPlan(1), mustBinding(t), PostgresPopulationRootBinding{}); !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + t.Fatalf("zero membership binding error = %v, want ErrUnregisteredPostgresIdentifier", err) + } + binding := populationBindingOK(t) + if binding.schema != testSchema { + t.Fatalf("constructed binding schema = %q", binding.schema) + } + if got, want := binding.tableSQL(membershipSelectionMembers), `"pgir02_it_unit"."selection_members"`; got != want { + t.Fatalf("member table SQL = %q, want %q", got, want) + } +} + +func TestMembershipAllowlistsMatchMigrationsLiterally(t *testing.T) { + cases := []struct { + migration string + table postgresMembershipTableKind + want []string + }{ + {"000008_selection_revisions.sql", membershipSelectionMembers, + []string{"generation", "id", "member_key", "ordinal", "project", "resource_type", "selection_id"}}, + {"000008_selection_revisions.sql", membershipSelectionRevisions, + []string{"complete", "generation", "id", "membership_digest", "project", "resource_type", "scope_digest"}}, + {"000009_explorer_explicit_groups.sql", membershipGroupRevisions, + []string{"generation", "id", "membership_digest", "project", "resource_type", "scope_digest", "source_membership_digest", "source_selection_revision_id", "state"}}, + {"000010_explorer_explicit_group_memberships.sql", membershipGroupMemberships, + []string{"generation", "group_id", "project", "resource_id", "resource_type", "revision_id"}}, + } + for _, tc := range cases { + spec, ok := postgresMembershipTables[tc.table] + if !ok { + t.Fatalf("registry missing relation %q", string(tc.table)) + } + if spec.physicalTable != string(tc.table) { + t.Fatalf("relation %q physical table %q drift", string(tc.table), spec.physicalTable) + } + var got []string + for name := range spec.columns { + got = append(got, name) + } + sortStrings(got) + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("relation %q columns = %v, want %v", string(tc.table), got, tc.want) + } + raw, err := os.ReadFile(filepath.Join("..", "..", "..", "..", "backends", "postgres", "internal", "explorer", "migrations", tc.migration)) + if err != nil { + t.Fatalf("read %s: %v", tc.migration, err) + } + text := string(raw) + if !strings.Contains(text, "CREATE TABLE IF NOT EXISTS "+spec.physicalTable+" (") { + t.Fatalf("%s: table %q creation not found literally", tc.migration, spec.physicalTable) + } + for _, column := range tc.want { + pattern := regexp.MustCompile(`(?m)^ ` + regexp.QuoteMeta(column) + ` +(text|bigint|boolean) +NOT NULL`) + if !pattern.MatchString(text) { + t.Fatalf("%s: column %q of %q is not a literal NOT NULL column", tc.migration, column, spec.physicalTable) + } + } + } + for name, kind := range canonicalMembershipCollections { + switch kind { + case membershipSelectionMembers, membershipSelectionRevisions, membershipGroupRevisions, membershipGroupMemberships: + default: + t.Fatalf("collection %q maps to unregistered kind %q", name, string(kind)) + } + } +} + +func sortStrings(values []string) { + for i := 1; i < len(values); i++ { + for j := i; j > 0 && values[j] < values[j-1]; j-- { + values[j], values[j-1] = values[j-1], values[j] + } + } +} + +func goldenString(v string) string { return v } + +func argumentText(value any) string { + if raw, ok := value.(json.RawMessage); ok { + return string(raw) + } + if raw, ok := value.([]byte); ok { + return string(raw) + } + return fmt.Sprintf("%v", value) +} + +func TestPopulationSingleHopSQLIsExactAndParameterized(t *testing.T) { + plan := populationPlan(1) + plan.Operations = append(plan.Operations[:1], + ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{ + Keys: []ir.PhysicalValue{{Variable: "root", Path: []string{"logical_id"}}}}}, + plan.Operations[1]) + rendered := renderPopulationOK(t, plan) + wantSQL := goldenString(`SELECT r0."logical_id" AS "c0"` + + ` FROM "pgir02_it_unit"."fhir_resource" AS r0` + + ` INNER JOIN (SELECT DISTINCT x1."project", x1."dataset_generation", x1."resource_type", x1."logical_id"` + + ` FROM "pgir02_it_unit"."selection_members" AS pm` + + ` INNER JOIN "pgir02_it_unit"."fhir_resource" AS x0 ON x0."project" = $1 AND x0."dataset_generation" = $2` + + ` AND x0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb)) AND x0."resource_type" = $9` + + ` AND x0."logical_id" = pm."id"` + + ` INNER JOIN "pgir02_it_unit"."fhir_edge_occurrence" AS e1 ON e1."project" = $1 AND e1."dataset_generation" = $2` + + ` AND e1."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND e1."label" = $10 AND e1."target_resource_type" = $9 AND e1."target_logical_id" = x0."logical_id"` + + ` AND e1."source_resource_type" = $4` + + ` INNER JOIN "pgir02_it_unit"."fhir_resource" AS x1 ON x1."project" = $1 AND x1."dataset_generation" = $2` + + ` AND x1."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb)) AND x1."resource_type" = $4` + + ` AND x1."logical_id" = e1."source_logical_id"` + + ` WHERE pm."selection_id" = $5 AND pm."project" = $6 AND pm."generation" = $7 AND pm."resource_type" = $8)` + + ` AS pop ON pop."project" = r0."project" AND pop."dataset_generation" = r0."dataset_generation"` + + ` AND pop."resource_type" = r0."resource_type" AND pop."logical_id" = r0."logical_id"` + + ` WHERE r0."project" = $1 AND r0."dataset_generation" = $2` + + ` AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb)) AND r0."resource_type" = $4` + + ` ORDER BY r0."logical_id" ASC, r0."resource_seq" ASC`) + if rendered.SQL != wantSQL { + t.Fatalf("population SQL drift:\n got: %s\nwant: %s", rendered.SQL, wantSQL) + } + wantArgs := []struct { + value any + }{ + {"proj-a"}, {"gen-1"}, {`["FHIR/Patient","FHIR/Observation"]`}, {"Patient"}, {"selrev_7"}, + {"proj-a"}, {"gen-1"}, {"Observation"}, {"Observation"}, {"encounter-diagnosed_condition"}, + } + if len(rendered.Args) != len(wantArgs) { + t.Fatalf("argument count = %d, want %d: %#v", len(rendered.Args), len(wantArgs), rendered.Args) + } + for index, want := range wantArgs { + got := argumentText(rendered.Args[index].Value) + if got != fmt.Sprintf("%v", want.value) { + t.Fatalf("argument %d = %v, want %v", index+1, got, want.value) + } + } + second, err := RenderPopulationRootScan(plan, mustBinding(t), populationBindingOK(t)) + if err != nil || second.SQL != rendered.SQL || !reflect.DeepEqual(second.Args, rendered.Args) { + t.Fatalf("population render is not deterministic: %v", err) + } +} + +func TestPopulationDirectSourceSQL(t *testing.T) { + plan := populationPlan(0) + plan.BindVars["root_collection"] = "Observation" + rendered := renderPopulationOK(t, plan) + if !strings.Contains(rendered.SQL, `SELECT DISTINCT x0."project", x0."dataset_generation", x0."resource_type", x0."logical_id"`) { + t.Fatalf("zero-hop candidate must dedup the source identity:\n%s", rendered.SQL) + } + if strings.Contains(rendered.SQL, "fhir_edge_occurrence") { + t.Fatalf("zero-hop population root must not join the edge relation:\n%s", rendered.SQL) + } +} + +func TestPopulationCanonicalDenyReadsNothing(t *testing.T) { + plan := populationPlan(1) + plan.BindVars["scope_allowed"] = false + rendered := renderPopulationOK(t, plan) + for _, fragment := range []string{ + `x0."resource_type" = $9 AND FALSE`, + `x1."resource_type" = $4 AND FALSE`, + `e1."source_resource_type" = $4`, + `r0."resource_type" = $4 AND FALSE`, + } { + if !strings.Contains(rendered.SQL, fragment) { + t.Fatalf("canonical deny must close every scoped row chain (%q):\n%s", fragment, rendered.SQL) + } + } + for _, argument := range rendered.Args { + if fmt.Sprintf("%v", argument.Value) == "false" { + t.Fatalf("deny must never bind a boolean parameter: %#v", rendered.Args) + } + } +} + +func TestPopulationUnrestrictedAddsNoAuthPredicate(t *testing.T) { + plan := populationPlan(1) + plan.BindVars["auth_resource_paths_unrestricted"] = true + plan.BindVars["auth_resource_paths"] = []string(nil) + rendered := renderPopulationOK(t, plan) + if strings.Contains(rendered.SQL, "auth_resource_path") { + t.Fatalf("explicit unrestricted mode must drop every auth predicate:\n%s", rendered.SQL) + } +} + +func TestMembershipRootRefusals(t *testing.T) { + cases := []struct { + name string + builder func() ir.PhysicalPlan + }{ + {"collect-members variable is deferred to pg-sql-22", func() ir.PhysicalPlan { + plan := populationPlan(1) + plan.Operations[0].RootScan.Population.CollectMembersVariable = "population_members" + return plan + }}, + {"member filters out of canonical order", func() ir.PhysicalPlan { + plan := populationPlan(1) + f := plan.Operations[0].RootScan.Population.MemberFilters + f[0], f[1] = f[1], f[0] + return plan + }}, + {"member filter missing", func() ir.PhysicalPlan { + plan := populationPlan(1) + plan.Operations[0].RootScan.Population.MemberFilters = + plan.Operations[0].RootScan.Population.MemberFilters[:3] + return plan + }}, + {"member filter path altered", func() ir.PhysicalPlan { + plan := populationPlan(1) + plan.Operations[0].RootScan.Population.MemberFilters[0].Predicate.Left.Path = []string{"member_key"} + return plan + }}, + {"member filter binds its left side", func() ir.PhysicalPlan { + plan := populationPlan(1) + plan.Operations[0].RootScan.Population.MemberFilters[0].Predicate.Left = + ir.PhysicalValue{BindKey: "population_selection_id"} + return plan + }}, + {"member filter compares non-string bind", func() ir.PhysicalPlan { + plan := populationPlan(1) + plan.BindVars["population_selection_id"] = 42 + return plan + }}, + {"member generation must bind the canonical dataset_generation", func() ir.PhysicalPlan { + plan := populationPlan(1) + plan.BindVars["population_generation"] = "gen-1" + plan.Operations[0].RootScan.Population.MemberFilters[2].Predicate.Right = + &ir.PhysicalValue{BindKey: "population_generation"} + return plan + }}, + {"member collection must be a registered canonical relation", func() ir.PhysicalPlan { + plan := populationPlan(1) + plan.BindVars["population_members_collection"] = "selection_members" + return plan + }}, + {"member collection must be a known collection", func() ir.PhysicalPlan { + plan := populationPlan(1) + plan.BindVars["population_members_collection"] = "not_a_loom_collection" + return plan + }}, + {"member join predicate may not flip", func() ir.PhysicalPlan { + plan := populationPlan(1) + ops := plan.Operations[0].RootScan.Population.ResourceOperations + filter := ops[5].Filter.Predicate + filter.Left, *filter.Right = *filter.Right, filter.Left + return plan + }}, + {"source scan must read the member type", func() ir.PhysicalPlan { + plan := populationPlan(1) + plan.BindVars["population_source_collection"] = "Condition" + return plan + }}, + {"scope block may rebind the project leg", func() ir.PhysicalPlan { + plan := populationPlan(1) + ops := plan.Operations[0].RootScan.Population.ResourceOperations + *ops[1].Filter.Predicate.Right = ir.PhysicalValue{BindKey: "population_project"} + return plan + }}, + {"auth block may drop an input", func() ir.PhysicalPlan { + plan := populationPlan(1) + ops := plan.Operations[0].RootScan.Population.ResourceOperations + ops[3].DerivedLet.Inputs = ops[3].DerivedLet.Inputs[:2] + return plan + }}, + {"auth result may filter a row instead of the LET result", func() ir.PhysicalPlan { + plan := populationPlan(1) + ops := plan.Operations[0].RootScan.Population.ResourceOperations + ops[4].Filter.Predicate.Left = ir.PhysicalValue{Variable: "population_source", Path: []string{"project"}} + return plan + }}, + {"route step may anchor on the member row", func() ir.PhysicalPlan { + plan := populationPlan(1) + ops := plan.Operations[0].RootScan.Population.ResourceOperations + ops[6].Traversal.SourceVariable = "population_member" + return plan + }}, + {"route step may not widen direction", func() ir.PhysicalPlan { + plan := populationPlan(1) + ops := plan.Operations[0].RootScan.Population.ResourceOperations + ops[6].Traversal.Direction = ir.PhysicalAny + return plan + }}, + {"route step may not use endpoint lookup", func() ir.PhysicalPlan { + plan := populationPlan(1) + ops := plan.Operations[0].RootScan.Population.ResourceOperations + ops[6].Traversal.Strategy = ir.PhysicalTraversalEndpointLookup + ops[6].Traversal.EndpointIndexFields = []string{"_from"} + return plan + }}, + {"route step edge relation must be the canonical bind name", func() ir.PhysicalPlan { + plan := populationPlan(1) + plan.BindVars["population_root_route_0_edge_collection"] = "fhir_edge_occurrence" + return plan + }}, + {"route target type must match the root type", func() ir.PhysicalPlan { + plan := populationPlan(1) + plan.BindVars["population_root_route_0_target_type"] = "Encounter" + return plan + }}, + {"root key must name the terminal row", func() ir.PhysicalPlan { + plan := populationPlan(1) + plan.Operations[0].RootScan.Population.RootKey.Variable = "population_root_node_1" + return plan + }}, + {"root key must be _key", func() ir.PhysicalPlan { + plan := populationPlan(1) + plan.Operations[0].RootScan.Population.RootKey.Path = []string{"id"} + return plan + }}, + {"population payload may not carry a stray operation", func() ir.PhysicalPlan { + plan := populationPlan(1) + ops := plan.Operations[0].RootScan.Population.ResourceOperations + plan.Operations[0].RootScan.Population.ResourceOperations = append(ops, + ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "limit"}}) + return plan + }}, + {"cohort collections must be registered canonical relations", func() ir.PhysicalPlan { + plan := cohortPlan() + plan.BindVars["cohort_selection_members_collection"] = "selection_members" + return plan + }}, + {"cohort collections must be known collections", func() ir.PhysicalPlan { + plan := cohortPlan() + plan.BindVars["cohort_memberships_collection"] = "random_table" + return plan + }}, + {"cohort policy must stay in the lowered domain", func() ir.PhysicalPlan { + plan := cohortPlan() + plan.BindVars["cohort_policy"] = "ERRORS" + return plan + }}, + {"cohort policy bind must exist", func() ir.PhysicalPlan { + plan := cohortPlan() + delete(plan.BindVars, "cohort_policy") + return plan + }}, + {"cohort revision id must be a nonblank string", func() ir.PhysicalPlan { + plan := cohortPlan() + plan.BindVars["cohort_revision_id"] = " " + return plan + }}, + } + for _, tc := range cases { + plan := tc.builder() + _, err := RenderPopulationRootScan(plan, mustBinding(t), populationBindingOK(t)) + if err == nil { + t.Fatalf("%s: mutation accepted, want typed refusal", tc.name) + } + if !errors.Is(err, ErrUnsupportedPostgresPlan) && !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + if !strings.Contains(err.Error(), "validate physical plan") { + t.Fatalf("%s: error = %v (%T), want a typed gate", tc.name, err, err) + } + } + } + if _, err := RenderPopulationRootScan(populationPlan(1), mustBinding(t), populationBindingOK(t)); err != nil { + t.Fatalf("alive control (population) refused: %v", err) + } + if _, err := RenderPopulationRootScan(cohortPlan(), mustBinding(t), populationBindingOK(t)); err != nil { + t.Fatalf("alive control (cohort) refused: %v", err) + } +} + +func TestPlainSpineKeepsTheBindingGate(t *testing.T) { + if _, err := RenderPopulationRootScan(basePlan(), mustBinding(t), populationBindingOK(t)); !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("plain spine via membership entry error = %v, want ErrUnsupportedPostgresPlan", err) + } + plan := populationPlan(1) + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("population without binding error = %v, want ErrUnsupportedPostgresPlan", err) + } + seeded := basePlan() + seeded.BindVars["seed_collection"] = "Observation" + seedValue := ir.PhysicalValue{Variable: "seed_scan", Path: []string{"logical_id"}} + seeded.Operations[0].RootScan.PageCandidateSeed = &ir.PhysicalRootPageCandidateSeed{ + Subplan: ir.PhysicalSubplan{ + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalCollectionScanOp, CollectionScan: &ir.PhysicalCollectionScan{ + Variable: "seed_scan", CollectionBindKey: "seed_collection"}}, + }, + Return: ir.PhysicalExpression{Kind: ir.PhysicalValueExpression, + Cardinality: ir.PhysicalScalarCardinality, NullBehavior: ir.PhysicalPreserveNull, + Value: &seedValue}, + }, + } + if _, err := RenderPhysicalPlan(seeded, mustBinding(t)); !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("candidate seed via spine error = %v, want ErrUnsupportedPostgresPlan (render gate, not ir)", err) + } + if _, err := RenderPopulationRootScan(seeded, mustBinding(t), populationBindingOK(t)); !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("candidate seed via membership entry error = %v, want ErrUnsupportedPostgresPlan (render gate, not ir)", err) + } +} + +func TestCohortRootSourceSQLIsExactAndParameterized(t *testing.T) { + rendered := renderPopulationOK(t, cohortPlan()) + if !strings.Contains(rendered.SQL, + `INNER JOIN (SELECT sm1."id" AS "candidate_id" FROM "pgir02_it_unit"."selection_members" sm1 WHERE sm1."selection_id" = (SELECT gr2."source_selection_revision_id" FROM "pgir02_it_unit"."explorer_explicit_group_revisions" gr2 WHERE gr2."id" = $5)`) { + t.Fatalf("cohort candidate selection side drifted:\n%s", rendered.SQL) + } + if !strings.Contains(rendered.SQL, + `UNION SELECT mm1."resource_id" FROM "pgir02_it_unit"."explorer_explicit_group_memberships" mm1 WHERE mm1."revision_id" = $5`) { + t.Fatalf("cohort candidate membership side drifted:\n%s", rendered.SQL) + } + if !strings.Contains(rendered.SQL, + `(SELECT CASE WHEN NOT (EXISTS (SELECT 1 FROM "pgir02_it_unit"."explorer_explicit_group_revisions" gr0 WHERE gr0."id" = $5 AND gr0."state" = 'COMPLETE'`) { + t.Fatalf("cohort revision guard drifted:\n%s", rendered.SQL) + } + for _, fragment := range []string{ + `CAST('EXPLICIT_GROUP_REVISION_INVALID' || COALESCE(current_setting('loom.explorer_guard_barrier', true), '') AS integer)`, + `CAST('EXPLICIT_GROUP_SOURCE_STALE' || COALESCE(current_setting('loom.explorer_guard_barrier', true), '') AS integer)`, + `CAST('EXPLICIT_GROUP_UNASSIGNED_MEMBER' || COALESCE(current_setting('loom.explorer_guard_barrier', true), '') AS integer)`, + `AND NOT EXISTS (SELECT 1 FROM "pgir02_it_unit"."explorer_explicit_group_memberships" mm0 WHERE mm0."revision_id" = $5`, + `$9 = 'ERROR'`, + `) = 1`, + `r0."project" = $1 AND r0."dataset_generation" = $7 AND r0."resource_type" = $8`, + `cand."candidate_id" = r0."logical_id"`, + `ORDER BY r0."resource_seq" ASC`, + } { + if !strings.Contains(rendered.SQL, fragment) { + t.Fatalf("cohort SQL missing fragment %q:\n%s", fragment, rendered.SQL) + } + } + if strings.Contains(rendered.SQL, "@") { + t.Fatalf("cohort SQL carries AQL bind syntax:\n%s", rendered.SQL) + } + if strings.Contains(rendered.SQL, "r0.\"logical_id\") THEN") { + t.Fatalf("guard must stay uncorrelated:\n%s", rendered.SQL) + } + wantArgs := []string{"proj-a", "gen-1", `["FHIR/Patient","FHIR/Observation"]`, "Patient", + "grouprev_1", "proj-a", "gen-1", "Patient", "EXCLUDE"} + if len(rendered.Args) != len(wantArgs) { + t.Fatalf("cohort argument count = %d, want %d: %#v", len(rendered.Args), len(wantArgs), rendered.Args) + } + for index, want := range wantArgs { + if got := argumentText(rendered.Args[index].Value); got != want { + t.Fatalf("cohort argument %d = %q, want %q", index+1, got, want) + } + } + if strings.Contains(rendered.SQL, "$10") { + t.Fatalf("cohort render must reuse parameters, not mint new ones:\n%s", rendered.SQL) + } + second, err := RenderPopulationRootScan(cohortPlan(), mustBinding(t), populationBindingOK(t)) + if err != nil || second.SQL != rendered.SQL || !reflect.DeepEqual(second.Args, rendered.Args) { + t.Fatalf("cohort render is not deterministic: %v", err) + } +} + +func TestCohortDenyAndUnrestrictedScope(t *testing.T) { + denied := cohortPlan() + denied.BindVars["scope_allowed"] = false + rendered := renderPopulationOK(t, denied) + if !strings.Contains(rendered.SQL, `ELSE 1 END) * 0 = 1)`) { + t.Fatalf("cohort deny must merge into the always-false guard clause:\n%s", rendered.SQL) + } + if strings.Contains(rendered.SQL, " AND FALSE") { + t.Fatalf("cohort deny must never fold to a literal FALSE conjunct:\n%s", rendered.SQL) + } + if strings.Contains(rendered.SQL, " = 1 AND") || strings.Contains(rendered.SQL, " = 1 WHERE") || + strings.HasSuffix(rendered.SQL, " = 1") || strings.Contains(rendered.SQL, " = 1 ORDER BY") { + t.Fatalf("denied cohort must not keep a separate guard-equals-one conjunct:\n%s", rendered.SQL) + } + for _, argument := range rendered.Args { + if fmt.Sprintf("%v", argument.Value) == "false" { + t.Fatalf("cohort deny must never bind a boolean parameter: %#v", rendered.Args) + } + } + unrestricted := cohortPlan() + unrestricted.BindVars["auth_resource_paths_unrestricted"] = true + unrestricted.BindVars["auth_resource_paths"] = []string(nil) + rendered = renderPopulationOK(t, unrestricted) + if strings.Contains(rendered.SQL, "auth_resource_path") { + t.Fatalf("explicit unrestricted cohort root must not add auth predicates:\n%s", rendered.SQL) + } +} diff --git a/internal/dataframe/compiler/render/postgres/post_window.go b/internal/dataframe/compiler/render/postgres/post_window.go new file mode 100644 index 000000000..490ccaf23 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/post_window.go @@ -0,0 +1,544 @@ +package postgres + +import ( + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +func (r *planRenderer) acceptPostWindowTraversal(plan ir.PhysicalPlan, index int) (handled bool, next int, err error) { + _ = r + traversal := plan.Operations[index].Traversal + source, ok := r.rows[traversal.SourceVariable] + if !ok || source.derivedValue != "" { + return false, 0, nil + } + if shapeErr := validateDeadPostWindowTraversal(plan, *traversal, source); shapeErr != nil { + return false, 0, nil + } + after := index + 1 + if blockErr := matchCanonicalScopeBlock(plan.Operations, &after, + []string{traversal.EdgeVariable, traversal.TargetVariable}); blockErr != nil { + return false, 0, nil + } + variables := map[string]bool{traversal.EdgeVariable: true, traversal.TargetVariable: true} + for scan := after; scan < len(plan.Operations); scan++ { + if operationReferencesVariables(plan.Operations[scan], variables) { + return false, 0, fmt.Errorf("%w: operation %d reads %q or %q, so the post-window traversal is not dead; only traversals whose LET has no consumer in the AQL reading are registered after the execution window", + ErrUnsupportedPostgresPlan, scan, traversal.EdgeVariable, traversal.TargetVariable) + } + } + return true, after, nil +} + +func acceptPostWindowSet(plan ir.PhysicalPlan, set ir.PhysicalSet) bool { + return set.Reduction != nil || sliceConsumesSetVariable(plan, set.Variable) || + aggregateConsumesSetVariable(plan, set.Variable) +} + +func aggregateConsumesSetVariable(plan ir.PhysicalPlan, setVariable string) bool { + for _, operation := range plan.Operations { + if operation.Kind != ir.PhysicalReturnOp || operation.Return == nil { + continue + } + for index := range operation.Return.Projections { + expression := operation.Return.Projections[index].Expression + if expression != nil && aggregateReadsSet(expression.Aggregate, setVariable) { + return true + } + } + } + return false +} + +func aggregateReadsSet(aggregate *ir.PhysicalAggregate, setVariable string) bool { + if aggregate == nil || setVariable == "" { + return false + } + switch aggregate.Operation { + case ir.PhysicalSumAggregate, ir.PhysicalCountAggregate, ir.PhysicalMinAggregate, ir.PhysicalMaxAggregate: + default: + return false + } + return aggregate.Source.Variable == setVariable && aggregate.Source.BindKey == "" && len(aggregate.Source.Path) == 0 +} + +func operationReferencesVariables(operation ir.PhysicalOperation, variables map[string]bool) bool { + switch operation.Kind { + case ir.PhysicalFilterOp: + if operation.Filter == nil { + return true + } + if operation.Filter.Expression != nil { + return predicateReferencesVariables(*operation.Filter.Expression, variables) + } + return comparisonReferencesVariables(operation.Filter.Predicate, variables) + case ir.PhysicalDerivedLetOp: + if operation.DerivedLet == nil { + return true + } + if variables[operation.DerivedLet.Variable] { + return true + } + for _, input := range operation.DerivedLet.Inputs { + if valueReferencesVariables(input, variables) { + return true + } + } + return false + case ir.PhysicalTraversalOp: + if operation.Traversal == nil { + return true + } + return variables[operation.Traversal.SourceVariable] || + variables[operation.Traversal.EdgeVariable] || + variables[operation.Traversal.TargetVariable] + case ir.PhysicalSetOp: + if operation.Set == nil { + return true + } + set := operation.Set + if variables[set.Variable] || variables[set.SourceSetVariable] || variables[set.ItemVariable] { + return true + } + return subplanReferencesVariables(set.Subplan, variables) + case ir.PhysicalUnnestOp: + if operation.Unnest == nil { + return true + } + unnest := operation.Unnest + if variables[unnest.OutputVariable] || variables[unnest.Ordinality] || variables[unnest.HasItemVariable] { + return true + } + if expressionReferencesVariables(unnest.Expression, variables) { + return true + } + for _, ancestor := range unnest.Ancestors { + if variables[ancestor.Variable] { + return true + } + } + return false + case ir.PhysicalSortOp: + if operation.Sort == nil { + return true + } + for _, key := range operation.Sort.Keys { + if valueReferencesVariables(key, variables) { + return true + } + } + return false + case ir.PhysicalRootScanOp, ir.PhysicalLimitOp: + return false + case ir.PhysicalReturnOp: + if operation.Return == nil { + return true + } + for index := range operation.Return.Projections { + if projectionReferencesVariables(operation.Return.Projections[index], variables) { + return true + } + } + return false + default: + return true + } +} + +func subplanReferencesVariables(subplan ir.PhysicalSubplan, variables map[string]bool) bool { + for _, capture := range subplan.Captures { + if variables[capture] { + return true + } + } + for _, operation := range subplan.Operations { + if operationReferencesVariables(operation, variables) { + return true + } + } + if subplan.Sort != nil && valueReferencesVariables(*subplan.Sort, variables) { + return true + } + if subplan.DistinctBy != nil && valueReferencesVariables(*subplan.DistinctBy, variables) { + return true + } + return expressionReferencesVariables(subplan.Return, variables) +} + +func projectionReferencesVariables(projection ir.PhysicalProjection, variables map[string]bool) bool { + if valueReferencesVariables(projection.Value, variables) { + return true + } + if projection.Expression != nil && expressionReferencesVariables(*projection.Expression, variables) { + return true + } + if projection.Presence != nil && valueReferencesVariables(projection.Presence.Source, variables) { + return true + } + return false +} + +func expressionReferencesVariables(expression ir.PhysicalExpression, variables map[string]bool) bool { + switch expression.Kind { + case ir.PhysicalValueExpression: + return expression.Value != nil && valueReferencesVariables(*expression.Value, variables) + case ir.PhysicalLiteralExpression: + return false + case ir.PhysicalExtractExpression: + return expression.Extract != nil && valueReferencesVariables(expression.Extract.Source, variables) + case ir.PhysicalAggregateExpression: + aggregate := expression.Aggregate + if aggregate == nil { + return true + } + if valueReferencesVariables(aggregate.Source, variables) { + return true + } + if aggregate.Value != nil && expressionReferencesVariables(*aggregate.Value, variables) { + return true + } + if aggregate.Predicate != nil && predicateReferencesVariables(*aggregate.Predicate, variables) { + return true + } + if aggregate.ContributorWindow != nil && + (expressionReferencesVariables(aggregate.ContributorWindow.Timestamp, variables) || + expressionReferencesVariables(aggregate.ContributorWindow.Anchor, variables)) { + return true + } + if aggregate.Ordering != nil && expressionReferencesVariables(aggregate.Ordering.Timestamp, variables) { + return true + } + return false + case ir.PhysicalSliceExpression: + slice := expression.Slice + if slice == nil { + return true + } + if valueReferencesVariables(slice.Source, variables) { + return true + } + if slice.Predicate != nil && predicateReferencesVariables(*slice.Predicate, variables) { + return true + } + if slice.Sort != nil && expressionReferencesVariables(*slice.Sort, variables) { + return true + } + for index := range slice.Projections { + if expressionReferencesVariables(slice.Projections[index].Expression, variables) { + return true + } + } + return false + case ir.PhysicalObjectExpression: + object := expression.Object + if object == nil { + return true + } + for index := range object.Fields { + if expressionReferencesVariables(object.Fields[index].Expression, variables) { + return true + } + } + return false + case ir.PhysicalSubplanExpression: + if expression.Subplan == nil { + return true + } + return subplanReferencesVariables(*expression.Subplan, variables) + default: + return true + } +} + +func predicateReferencesVariables(predicate ir.PhysicalPredicateExpression, variables map[string]bool) bool { + switch predicate.Kind { + case ir.PhysicalComparisonPredicate: + if predicate.Comparison == nil { + return true + } + return comparisonReferencesVariables(*predicate.Comparison, variables) + case ir.PhysicalAllPredicate, ir.PhysicalAnyPredicate, ir.PhysicalNotPredicate: + for _, child := range predicate.Children { + if predicateReferencesVariables(child, variables) { + return true + } + } + return false + case ir.PhysicalExistsPredicate: + if predicate.Exists == nil { + return true + } + return subplanReferencesVariables(*predicate.Exists, variables) + default: + return true + } +} + +func comparisonReferencesVariables(predicate ir.PhysicalPredicate, variables map[string]bool) bool { + if valueReferencesVariables(predicate.Left, variables) { + return true + } + if predicate.Right != nil && valueReferencesVariables(*predicate.Right, variables) { + return true + } + if predicate.LeftExpression != nil && expressionReferencesVariables(*predicate.LeftExpression, variables) { + return true + } + return false +} + +func valueReferencesVariables(value ir.PhysicalValue, variables map[string]bool) bool { + return value.Variable != "" && variables[value.Variable] +} + +func (r *planRenderer) renderSetAggregateExpression(plan ir.PhysicalPlan, projection ir.PhysicalProjection) (string, error) { + aggregate := projection.Expression.Aggregate + if aggregate == nil { + return "", fmt.Errorf("%w: projection %q: AGGREGATE expression is missing its payload", + ErrUnsupportedPostgresPlan, projection.Name) + } + if projection.Expression.Cardinality != ir.PhysicalScalarCardinality { + return "", fmt.Errorf("%w: projection %q: the registered aggregate surface emits SCALAR-cardinality aggregates, the form the lowering emits for one reduction per row", + ErrUnsupportedPostgresPlan, projection.Name) + } + if projection.Expression.NullBehavior != ir.PhysicalEmptyOnNull { + return "", fmt.Errorf("%w: projection %q: the registered aggregate surface emits EMPTY_ON_NULL aggregates; got %q", + ErrUnsupportedPostgresPlan, projection.Name, string(projection.Expression.NullBehavior)) + } + switch aggregate.Operation { + case ir.PhysicalSumAggregate, ir.PhysicalCountAggregate, ir.PhysicalMinAggregate, ir.PhysicalMaxAggregate: + default: + return "", fmt.Errorf("%w: projection %q: aggregate operation %q has no registered PostgreSQL reducer; the correlated-sublink surface covers SUM, COUNT, MIN, MAX", + ErrUnsupportedPostgresPlan, projection.Name, string(aggregate.Operation)) + } + if aggregate.ContributorWindow != nil || aggregate.Ordering != nil || aggregate.RequiredValuesBindKey != "" { + return "", fmt.Errorf("%w: projection %q: contributor windows, temporal ordering, and required-value comparisons have no registered PostgreSQL reducer", + ErrUnsupportedPostgresPlan, projection.Name) + } + setVariable := aggregate.Source.Variable + if setVariable == "" || aggregate.Source.BindKey != "" || len(aggregate.Source.Path) != 0 { + return "", fmt.Errorf("%w: projection %q: the aggregate source must be a registered set variable", + ErrUnsupportedPostgresPlan, projection.Name) + } + corr, ok := r.setBodies[setVariable] + if !ok { + return "", fmt.Errorf("%w: projection %q: aggregate source %q is not a registered child set", + ErrUnsupportedPostgresPlan, projection.Name, setVariable) + } + fields := make(map[string]ir.PhysicalSetProjectionField, len(r.setProjections[setVariable])) + for _, field := range r.setProjections[setVariable] { + fields[field.Name] = field + } + + seq := r.nextTraversalSequence() + itemAlias := "a" + strconv.Itoa(seq) + elementAlias := "v" + strconv.Itoa(seq) + logicalIDColumn := r.storage.Resource.MustColumn("logical_id").SQLText() + resourceTypeColumn := r.storage.Resource.MustColumn("resource_type").SQLText() + payloadColumn := r.storage.Resource.MustColumn("payload").SQLText() + + whereParts := []string{corr.where} + if aggregate.Predicate != nil { + clause, clauseErr := r.aggregatePredicateFragment(plan, setVariable, fields, corr, corr.targetAlias, aggregate.Predicate) + if clauseErr != nil { + return "", fmt.Errorf("projection %q: aggregate predicate: %w", projection.Name, clauseErr) + } + whereParts = append(whereParts, clause) + } + items := "(SELECT DISTINCT " + corr.targetAlias + "." + logicalIDColumn + " AS " + logicalIDColumn + + ", " + corr.targetAlias + "." + resourceTypeColumn + " AS " + resourceTypeColumn + + ", " + corr.targetAlias + "." + payloadColumn + " AS " + payloadColumn + + " " + corr.from + " WHERE " + strings.Join(whereParts, " AND ") + ")" + + if aggregate.Operation == ir.PhysicalCountAggregate && aggregate.Value == nil { + return "(SELECT to_jsonb(COUNT(*)) FROM " + items + " AS " + itemAlias + ")", nil + } + + valueArray, valueErr := r.aggregateSlotValues(setVariable, fields, corr, itemAlias, aggregate.Value, "value") + if valueErr != nil { + return "", fmt.Errorf("projection %q: aggregate value: %w", projection.Name, valueErr) + } + if valueArray == "" { + return "", fmt.Errorf("%w: projection %q: this aggregate operation requires a value expression", + ErrUnsupportedPostgresPlan, projection.Name) + } + elements := "FROM " + items + " AS " + itemAlias + + " CROSS JOIN LATERAL jsonb_array_elements(" + valueArray + ") AS " + elementAlias + "(v)" + + " WHERE " + elementAlias + ".v <> 'null'::jsonb" + + switch aggregate.Operation { + case ir.PhysicalCountAggregate: + return "(SELECT to_jsonb(COUNT(*)) " + elements + ")", nil + case ir.PhysicalSumAggregate: + messageIndex := r.textParameter("NUMERIC_AGGREGATE_NON_NUMERIC", "") + raise := "to_jsonb(CAST('ASSERT:' || $" + strconv.Itoa(messageIndex) + + " || COALESCE(current_setting(" + quoteSQLLiteral(guardBarrierGUC) + ", true), '') AS integer))" + numericSum := groupDoubleBandCast("SUM(" + callBandOperandClamp( + "(("+elementAlias+".v #>> '{}')::numeric)") + ")") + return "(SELECT CASE WHEN COUNT(*) = 0 THEN NULL::jsonb" + + " WHEN bool_and(jsonb_typeof(" + elementAlias + ".v) = 'number')" + + " THEN to_jsonb(" + numericSum + ") ELSE " + raise + " END " + elements + ")", nil + case ir.PhysicalMinAggregate, ir.PhysicalMaxAggregate: + direction := "ASC" + if aggregate.Operation == ir.PhysicalMaxAggregate { + direction = "DESC" + } + numberKey := "CASE WHEN jsonb_typeof(" + elementAlias + ".v) = 'number' THEN (" + elementAlias + ".v #>> '{}')::numeric END" + booleanKey := "CASE WHEN jsonb_typeof(" + elementAlias + ".v) = 'boolean' THEN (" + elementAlias + ".v #>> '{}') END COLLATE \"C\"" + stringKey := "CASE WHEN jsonb_typeof(" + elementAlias + ".v) = 'string' THEN (" + elementAlias + ".v #>> '{}') END COLLATE \"C\"" + rank := "CASE jsonb_typeof(" + elementAlias + ".v) WHEN 'boolean' THEN 1 WHEN 'number' THEN 2 WHEN 'string' THEN 3 WHEN 'array' THEN 4 ELSE 5 END" + orderBy := rank + " " + direction + " NULLS LAST, " + + numberKey + " " + direction + " NULLS LAST, " + + booleanKey + " " + direction + " NULLS LAST, " + + stringKey + " " + direction + " NULLS LAST, " + + elementAlias + ".v " + direction + " NULLS LAST" + return "(SELECT " + elementAlias + ".v " + elements + " ORDER BY " + orderBy + " LIMIT 1)", nil + } + return "", fmt.Errorf("%w: projection %q: aggregate operation %q is not registered", + ErrUnsupportedPostgresPlan, projection.Name, string(aggregate.Operation)) +} + +func (r *planRenderer) aggregatePredicateFragment( + plan ir.PhysicalPlan, + setVariable string, + fields map[string]ir.PhysicalSetProjectionField, + corr correlatedSetBody, + baseAlias string, + predicate *ir.PhysicalPredicateExpression, +) (string, error) { + if predicate.Kind != ir.PhysicalComparisonPredicate || predicate.Comparison == nil { + return "", fmt.Errorf("%w: only a comparison predicate is registered on a set aggregate (the AQL reading at render/aql/collections.go renderAggregateItems refuses other forms)", + ErrUnsupportedPostgresPlan) + } + values, err := r.aggregateSlotValues(setVariable, fields, corr, baseAlias, predicate.Comparison.LeftExpression, "predicate") + if err != nil { + return "", err + } + fragment, err := r.quantifiedMatchFragment(plan, *predicate.Comparison, values) + if err != nil { + return "", err + } + return fragment.sql, nil +} + +func (r *planRenderer) aggregateSlotValues( + setVariable string, + fields map[string]ir.PhysicalSetProjectionField, + corr correlatedSetBody, + baseAlias string, + expression *ir.PhysicalExpression, + role string, +) (string, error) { + if expression == nil { + return "", nil + } + if expression.Kind != ir.PhysicalExtractExpression || expression.Extract == nil { + return "", fmt.Errorf("%w: aggregate %s must be an EXTRACT selector expression, the form lower/child_sets.go emits", + ErrUnsupportedPostgresPlan, role) + } + extract := expression.Extract + if expression.Cardinality != ir.PhysicalArrayCardinality { + return "", fmt.Errorf("%w: aggregate %s must be ARRAY-cardinality; got %q", + ErrUnsupportedPostgresPlan, role, string(expression.Cardinality)) + } + if expression.NullBehavior != ir.PhysicalEmptyOnNull { + return "", fmt.Errorf("%w: aggregate %s must carry EMPTY_ON_NULL null behavior; got %q", + ErrUnsupportedPostgresPlan, role, string(expression.NullBehavior)) + } + if extract.Prepared == nil { + return "", fmt.Errorf("%w: aggregate %s must carry the prepared projection reference lower/child_sets.go computes", + ErrUnsupportedPostgresPlan, role) + } + if extract.Prepared.SetVariable != setVariable { + return "", fmt.Errorf("%w: aggregate %s reads a prepared slot of set %q, not the aggregate source set %q", + ErrUnsupportedPostgresPlan, role, extract.Prepared.SetVariable, setVariable) + } + field, ok := fields[extract.Prepared.Field] + if !ok { + return "", fmt.Errorf("%w: aggregate %s reads projection slot %q, which set %q does not declare", + ErrUnsupportedPostgresPlan, role, extract.Prepared.Field, setVariable) + } + if extract.Source.Variable != setVariable { + return "", fmt.Errorf("%w: aggregate %s extracts from variable %q, not the aggregate source set %q", + ErrUnsupportedPostgresPlan, role, extract.Source.Variable, setVariable) + } + switch extract.ExecutionMode { + case "", ir.PhysicalSelectorGeneric, ir.PhysicalSelectorDirectScalar, ir.PhysicalSelectorConditionalArray: + default: + return "", fmt.Errorf("%w: aggregate %s selector execution mode %q is unknown", + ErrUnsupportedPostgresPlan, role, string(extract.ExecutionMode)) + } + if extract.Distinct { + return "", fmt.Errorf("%w: aggregate %s DISTINCT changes element multiplicity the quantified reducer does not model (the AQL predicate reading consumes the raw value list)", + ErrUnsupportedPostgresPlan, role) + } + if len(extract.Fallbacks) != 0 { + return "", fmt.Errorf("%w: aggregate %s selector fallbacks are a separate capability", ErrUnsupportedPostgresPlan, role) + } + if extract.UnitNormalization != nil { + return "", fmt.Errorf("%w: aggregate %s unit normalization is not implemented for PostgreSQL", ErrUnsupportedPostgresPlan, role) + } + if strings.TrimSpace(field.ResourceType) == "" || field.ResourceType != corr.targetType { + return "", fmt.Errorf("%w: aggregate %s slot validates against resource type %q, but the set reaches %q", + ErrUnsupportedPostgresPlan, role, field.ResourceType, corr.targetType) + } + if strings.TrimSpace(extract.ResourceType) != field.ResourceType { + return "", fmt.Errorf("%w: aggregate %s validates its selector against %q, but the slot declares %q", + ErrUnsupportedPostgresPlan, role, extract.ResourceType, field.ResourceType) + } + base := baseAlias + "." + r.storage.Resource.MustColumn("payload").SQLText() + name := "ags" + strconv.Itoa(r.nextTraversalSequence()) + return r.pivotSelectorArray(base, field.Selector, name) +} + +func validateDeadPostWindowTraversal(plan ir.PhysicalPlan, traversal ir.PhysicalTraversal, source rowScope) error { + switch traversal.Direction { + case ir.PhysicalOutbound, ir.PhysicalInbound: + case ir.PhysicalAny: + return fmt.Errorf("%w: ANY-direction traversal has no proven PostgreSQL contract (render/aql/navigation_layout.go:394-396 refuses it too)", ErrUnsupportedPostgresPlan) + default: + return fmt.Errorf("%w: unknown traversal direction %q", ErrUnsupportedPostgresPlan, string(traversal.Direction)) + } + switch traversal.Strategy { + case "", ir.PhysicalTraversalNative, ir.PhysicalTraversalEndpointLookup: + default: + return fmt.Errorf("%w: unknown traversal strategy %q", ErrUnsupportedPostgresPlan, string(traversal.Strategy)) + } + if traversal.Strategy == ir.PhysicalTraversalEndpointLookup && + (strings.TrimSpace(traversal.EndpointField) == "" || strings.TrimSpace(traversal.EndpointJoinField) == "") { + return fmt.Errorf("%w: ENDPOINT_LOOKUP traversal must name its endpoint fields (the shared lowering contract always does)", ErrUnsupportedPostgresPlan) + } + expectedField := "to_type" + if traversal.Direction == ir.PhysicalInbound { + expectedField = "from_type" + } + if traversal.EdgeTargetTypeField != expectedField { + return fmt.Errorf("%w: %s traversal must carry EdgeTargetTypeField %q, got %q", + ErrUnsupportedPostgresPlan, string(traversal.Direction), expectedField, traversal.EdgeTargetTypeField) + } + if traversal.SourceVariable == "" || traversal.EdgeVariable == "" || traversal.TargetVariable == "" { + return fmt.Errorf("%w: traversal must name its source, edge, and target variables (the shared lowering contract always does)", ErrUnsupportedPostgresPlan) + } + if traversal.EdgeCollectionBindKey == "" { + return fmt.Errorf("%w: traversal must name the edge collection bind", ErrUnsupportedPostgresPlan) + } + if collection, ok := plan.BindVars[traversal.EdgeCollectionBindKey].(string); !ok || collection != "fhir_edge" { + return fmt.Errorf("%w: traversal edge collection bind %q must be bound to \"fhir_edge\" (the only registered edge relation)", + ErrUnsupportedPostgresPlan, traversal.EdgeCollectionBindKey) + } + for _, bindKey := range []string{traversal.EdgeLabelBindKey, traversal.TargetTypeBindKey} { + if value, ok := plan.BindVars[bindKey].(string); !ok || strings.TrimSpace(value) == "" { + return fmt.Errorf("%w: traversal bind %q must be a non-empty string", ErrUnsupportedPostgresPlan, bindKey) + } + } + if traversal.SourceTypeBindKey != "" { + if bound, ok := plan.BindVars[traversal.SourceTypeBindKey].(string); !ok || bound != source.resourceType { + return fmt.Errorf("%w: traversal source type bind does not match the current row resource type %q", + ErrUnsupportedPostgresPlan, source.resourceType) + } + } + return nil +} diff --git a/internal/dataframe/compiler/render/postgres/post_window_live_test.go b/internal/dataframe/compiler/render/postgres/post_window_live_test.go new file mode 100644 index 000000000..3cd26fc99 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/post_window_live_test.go @@ -0,0 +1,155 @@ +package postgres + +import ( + "context" + "fmt" + "os" + "strings" + "sync" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func newPWin01Service(t *testing.T) *pgir02Service { + t.Helper() + return newOwnedService(t, 0, "pwin01_it") +} + +var ( + pwin01FixturesMu sync.Mutex + pwin01TrackedFixtures = map[string]bool{} +) + +func pwin01TrackFixtures(t *testing.T, svc *pgir02Service) { + t.Helper() + var database string + if err := svc.iso.QueryRow(svc.ctx, "SELECT current_database()").Scan(&database); err != nil { + svc.fail("track owned database", err) + } + pwin01FixturesMu.Lock() + defer pwin01FixturesMu.Unlock() + pwin01TrackedFixtures[database] = true + pwin01TrackedFixtures[svc.schema] = true +} + +func pwin01SnapshotOwnedResources(ctx context.Context, raw string) ([]string, error) { + pool, err := pgxpool.New(ctx, raw) + if err != nil { + return nil, fmt.Errorf("connect: %w", err) + } + defer pool.Close() + rows, err := pool.Query(ctx, + `SELECT datname FROM pg_database WHERE datname LIKE 'pwin01_it_%'`+ + ` UNION ALL SELECT nspname FROM pg_namespace WHERE nspname LIKE 'pwin01_it_%'`) + if err != nil { + return nil, fmt.Errorf("snapshot: %w", err) + } + defer rows.Close() + var names []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("scan: %w", err) + } + names = append(names, name) + } + return names, rows.Err() +} + +func pwin01RunDeltaAudit(inner func() int) int { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return inner() + } + beforeList, err := pwin01SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pwin-01 delta audit: snapshot before run: %v\n", err) + return inner() + } + before := make(map[string]bool, len(beforeList)) + for _, name := range beforeList { + before[name] = true + } + code := inner() + after, err := pwin01SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pwin-01 delta audit: survivor scan: %v\n", err) + return 1 + } + var survivors []string + for _, name := range after { + if !before[name] { + survivors = append(survivors, name) + } + } + if len(survivors) > 0 { + fmt.Fprintf(os.Stderr, "pwin-01 delta audit: %d pwin01_it_* resources survive: %v\n", len(survivors), survivors) + code = 1 + } + return code +} + +func TestPostWindowLiveReducersExecuteWithPredicates(t *testing.T) { + svc := newPWin01Service(t) + pwin01TrackFixtures(t, svc) + + setsInsertPatient(t, svc, "p-1") + setsInsertPatient(t, svc, "p-2") + setsInsertObservation(t, svc, "obs-a", "FHIR/Observation", map[string]any{ + "code": map[string]any{"coding": []any{map[string]any{"code": "asthma"}}}, + "valueQuantity": map[string]any{"value": 10}, + }) + setsInsertObservation(t, svc, "obs-b", "FHIR/Observation", map[string]any{ + "code": map[string]any{"coding": []any{map[string]any{"code": "asthma"}}}, + "valueQuantity": map[string]any{"value": 30}, + }) + setsInsertObservation(t, svc, "obs-c", "FHIR/Observation", map[string]any{ + "code": map[string]any{"coding": []any{map[string]any{"code": "diabetes"}}}, + "valueQuantity": map[string]any{"value": 99}, + }) + setsInsertObservation(t, svc, "obs-d", "FHIR/Observation", map[string]any{ + "valueQuantity": map[string]any{"value": 5}, + }) + + setsInsertRouteEdge(t, svc, "p-1", "obs-a", "observation[0]") + setsInsertRouteEdge(t, svc, "p-1", "obs-a", "observation[1]") + setsInsertRouteEdge(t, svc, "p-1", "obs-b", "observation[2]") + setsInsertRouteEdge(t, svc, "p-1", "obs-c", "observation[3]") + setsInsertRouteEdge(t, svc, "p-1", "obs-d", "observation[4]") + setsInsertRouteEdge(t, svc, "p-2", "obs-c", "observation[0]") + + plan := postWindowAggregatePlan(postWindowAggregateSet(false), + postWindowAggregateProjection("sum_age", ir.PhysicalSumAggregate, true, true), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, true), + postWindowAggregateProjection("min_age", ir.PhysicalMinAggregate, true, true), + postWindowAggregateProjection("max_age", ir.PhysicalMaxAggregate, true, true)) + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + got := svc.runRendered(t, rendered) + + want := [][]string{ + {"p-1", "40", "2", "10", "30"}, + {"p-2", "", "0", "", ""}, + } + if len(got) != len(want) { + t.Fatalf("rows = %v, want %v", got, want) + } + for row := range want { + if strings.Join(got[row], "|") != strings.Join(want[row], "|") { + t.Fatalf("row %d = %v, want %v", row, got[row], want[row]) + } + } + + counts := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_edge_occurrence"`+ + ` WHERE project=$1 AND dataset_generation=$2 AND source_logical_id='p-1' AND target_logical_id='obs-a'`, + setsProj, setsGen) + if len(counts) != 1 || counts[0][0] != "2" { + t.Fatalf("duplicate occurrence must be stored for the per-occurrence grain: %v", counts) + } +} diff --git a/internal/dataframe/compiler/render/postgres/post_window_test.go b/internal/dataframe/compiler/render/postgres/post_window_test.go new file mode 100644 index 000000000..dcb38d15f --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/post_window_test.go @@ -0,0 +1,502 @@ +package postgres + +import ( + "regexp" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" +) + +func postWindowSetField(name string, steps ...spec.SelectorStep) ir.PhysicalSetProjectionField { + return ir.PhysicalSetProjectionField{ + Name: name, ResourceType: "Observation", + Selector: spec.Selector{Steps: steps}, + ExecutionMode: ir.PhysicalSelectorConditionalArray, + } +} + +func postWindowCodeSteps() []spec.SelectorStep { + return []spec.SelectorStep{{Field: "code"}, {Field: "coding", Iterate: true}, {Field: "code"}} +} + +func postWindowValueSteps() []spec.SelectorStep { + return []spec.SelectorStep{{Field: "valueQuantity"}, {Field: "value"}} +} + +func postWindowAggregateSet(endpointStrategy bool) ir.PhysicalSet { + traversal := setsTraversal() + if endpointStrategy { + traversal.Strategy = ir.PhysicalTraversalEndpointLookup + traversal.EndpointField, traversal.EndpointJoinField = "_from", "_to" + traversal.EndpointIndexFields = []string{"_from", "project", "dataset_generation", "label", "to_type"} + } + operations := []ir.PhysicalOperation{{Kind: ir.PhysicalTraversalOp, Traversal: &traversal}} + operations = appendScopeBlock(operations, []string{setsEdgeVariable, setsTargetVariable}, setsPrefix+"_scope_auth") + return ir.PhysicalSet{ + Variable: setsPrefix, + Subplan: ir.PhysicalSubplan{ + Captures: []string{"root"}, + Operations: operations, + Return: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalObjectCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{Variable: setsTargetVariable}, + }, + }, + Unique: true, + SortByKey: true, + Projection: &ir.PhysicalSetProjection{Fields: []ir.PhysicalSetProjectionField{ + postWindowSetField("__loom_projection_0", postWindowCodeSteps()...), + (func() ir.PhysicalSetProjectionField { + field := postWindowSetField("__loom_projection_1", postWindowValueSteps()...) + field.ExecutionMode = ir.PhysicalSelectorDirectScalar + return field + })(), + }}, + } +} + +func postWindowSlotExpression(field string, mode ir.PhysicalSelectorExecutionMode, steps []spec.SelectorStep) *ir.PhysicalExpression { + return &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: setsPrefix, Path: []string{"payload"}}, + ResourceType: "Observation", + Selector: spec.Selector{Steps: steps}, + ExecutionMode: mode, + Prepared: &ir.PhysicalPreparedReference{SetVariable: setsPrefix, Field: field}, + }, + } +} + +func postWindowAggregateProjection(name string, operation ir.PhysicalAggregateOperation, withValue, withPredicate bool) ir.PhysicalProjection { + aggregate := &ir.PhysicalAggregate{ + Source: ir.PhysicalValue{Variable: setsPrefix}, + Operation: operation, + } + if withValue { + aggregate.Value = postWindowSlotExpression("__loom_projection_1", ir.PhysicalSelectorDirectScalar, postWindowValueSteps()) + } + if withPredicate { + aggregate.Predicate = &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalComparisonPredicate, + Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", + LeftExpression: postWindowSlotExpression("__loom_projection_0", ir.PhysicalSelectorConditionalArray, postWindowCodeSteps()), + Right: &ir.PhysicalValue{BindKey: setsPrefix + "_predicate_value"}, + Quantifier: spec.QuantifierAny, + ValueKind: "CODE", + }, + } + } + return ir.PhysicalProjection{ + Name: name, + Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalAggregateExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, Aggregate: aggregate, + }, + } +} + +func postWindowAggregatePlan(set ir.PhysicalSet, projections ...ir.PhysicalProjection) ir.PhysicalPlan { + plan := setsPlan() + plan.BindVars[setsPrefix+"_predicate_value"] = "asthma" + sortOp := ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{ + Keys: []ir.PhysicalValue{{Variable: "root", Path: []string{"logical_id"}}}, + }} + rootScan := plan.Operations[0] + all := append([]ir.PhysicalProjection{{Name: "patient_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}}, projections...) + ret := ir.PhysicalOperation{Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: all}} + plan.Operations = []ir.PhysicalOperation{rootScan, sortOp, {Kind: ir.PhysicalSetOp, Set: &set}, ret} + return plan +} + +func TestPostWindowAggregateReducersRenderAsCorrelatedSublinks(t *testing.T) { + plan := postWindowAggregatePlan(postWindowAggregateSet(false), + postWindowAggregateProjection("sum_age", ir.PhysicalSumAggregate, true, true), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, true), + postWindowAggregateProjection("min_age", ir.PhysicalMinAggregate, true, true), + postWindowAggregateProjection("max_age", ir.PhysicalMaxAggregate, true, true), + ) + rendered := renderOK(t, plan) + sql := rendered.SQL + + itemsRegion := regexp.MustCompile(`(?s)\(SELECT DISTINCT x1\."logical_id".*?\) AS a\d+`) + regions := itemsRegion.FindAllString(sql, -1) + if len(regions) != 4 { + t.Fatalf("every aggregate needs its own DISTINCT items subselect, got %d:\n%s", len(regions), sql) + } + for _, region := range regions { + if regexp.MustCompile(`\ba\d+\."payload"`).MatchString(region) { + t.Fatalf("items subselect references its own alias (unexecutable plan analysis):\n%s", region) + } + if !strings.Contains(region, `x1."payload"`) { + t.Fatalf("items subselect predicate must read the reached target's payload column:\n%s", region) + } + } + + if strings.Contains(sql, "LEFT JOIN") { + t.Fatalf("aggregate sublinks must not add outer joins:\n%s", sql) + } + for _, want := range []string{ + "CASE WHEN COUNT(*) = 0 THEN NULL::jsonb", + "bool_and(jsonb_typeof(", + "SUM(", + "ASSERT:", + "to_jsonb(COUNT(*))", + "WHEN 'boolean' THEN 1 WHEN 'number' THEN 2 WHEN 'string' THEN 3 WHEN 'array' THEN 4", + "ORDER BY", + "LIMIT 1", + "END COLLATE \"C\"", + "EXISTS (SELECT 1 FROM jsonb_array_elements(", + "fhir_edge_occurrence", + } { + if !strings.Contains(sql, want) { + t.Fatalf("aggregate statement is missing %q:\n%s", want, sql) + } + } + if got := strings.Count(sql, `END COLLATE "C"`); got != 4 { + t.Fatalf("MIN and MAX must carry exactly the two C-collated text keys each, got %d:\n%s", got, sql) + } +} + +func TestPostWindowCountWithValueCountsSlotsNotRows(t *testing.T) { + bare := renderOK(t, postWindowAggregatePlan(postWindowAggregateSet(false), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, false))) + withValue := renderOK(t, postWindowAggregatePlan(postWindowAggregateSet(false), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, true, false))) + if strings.Contains(bare.SQL, "CROSS JOIN LATERAL jsonb_array_elements") { + t.Fatalf("Value-less COUNT must count the set array without a slot unnest:\n%s", bare.SQL) + } + if !strings.Contains(withValue.SQL, "CROSS JOIN LATERAL jsonb_array_elements") || + !strings.Contains(withValue.SQL, "to_jsonb(COUNT(*))") { + t.Fatalf("COUNT with a Value must count the unnested non-null slots:\n%s", withValue.SQL) + } +} + +func TestPostWindowAggregateSurfaceRefusals(t *testing.T) { + tests := []struct { + name string + mutate func(*ir.PhysicalPlan) + want string + }{ + {"mean has no registered reducer", func(plan *ir.PhysicalPlan) { + plan.Operations[3].Return.Projections[2].Expression.Aggregate.Operation = ir.PhysicalMeanAggregate + }, "no registered PostgreSQL reducer"}, + {"scalar cardinality is the registered form", func(plan *ir.PhysicalPlan) { + plan.Operations[3].Return.Projections[2].Expression.Cardinality = ir.PhysicalArrayCardinality + }, "SCALAR-cardinality aggregates"}, + {"preserve-null behavior is outside the registered form", func(plan *ir.PhysicalPlan) { + plan.Operations[3].Return.Projections[2].Expression.NullBehavior = ir.PhysicalPreserveNull + }, "EMPTY_ON_NULL aggregates"}, + {"value must name a declared projection slot", func(plan *ir.PhysicalPlan) { + plan.Operations[3].Return.Projections[2].Expression.Aggregate.Value.Extract.Prepared.Field = "__loom_projection_9" + }, "does not declare"}, + {"distinct slots change element multiplicity", func(plan *ir.PhysicalPlan) { + plan.Operations[3].Return.Projections[2].Expression.Aggregate.Value.Extract.Distinct = true + }, "element multiplicity"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + plan := postWindowAggregatePlan(postWindowAggregateSet(false), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, false), + postWindowAggregateProjection("sum_age", ir.PhysicalSumAggregate, true, false)) + test.mutate(&plan) + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + wantUnsupported(t, err, test.name) + if !strings.Contains(err.Error(), string(ir.PhysicalAggregateExpression)) || + !strings.Contains(err.Error(), "no registered PostgreSQL rendering") { + t.Fatalf("%s: aggregate gate text drifted: %v", test.name, err) + } + if !strings.Contains(err.Error(), test.want) { + t.Fatalf("%s: error = %v, want substring %q", test.name, err, test.want) + } + }) + } +} + +func TestPostWindowSetWithoutRegisteredConsumerKeepsTheWindowGuard(t *testing.T) { + set := postWindowAggregateSet(false) + plan := postWindowAggregatePlan(set, + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, false)) + plan.Operations[3].Return.Projections = plan.Operations[3].Return.Projections[:1] + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + wantUnsupported(t, err, "unconsumed post-window set") + if !strings.Contains(err.Error(), "sets must precede the execution window") { + t.Fatalf("the window guard must stay fail-closed: %v", err) + } +} + +func TestPostWindowEndpointLookupIsAdmittedOnlyForAggregateConsumers(t *testing.T) { + rendered := renderOK(t, postWindowAggregatePlan(postWindowAggregateSet(true), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, false))) + aggregateSQL := rendered.SQL + + native := postWindowAggregatePlan(postWindowAggregateSet(false), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, false)) + nativeSQL := renderOK(t, native).SQL + if aggregateSQL != nativeSQL { + t.Fatalf("endpoint-lookup must normalize to the registered join byte-identically:\n%s\n---\n%s", aggregateSQL, nativeSQL) + } + + reductionPlan := setsPlanWithSet(setsChildSet(ir.PhysicalSetReductionFirst), "__loom_reduced_0") + for index := range reductionPlan.Operations { + if reductionPlan.Operations[index].Kind == ir.PhysicalSetOp { + traversal := reductionPlan.Operations[index].Set.Subplan.Operations[0].Traversal + traversal.Strategy = ir.PhysicalTraversalEndpointLookup + traversal.EndpointField, traversal.EndpointJoinField = "_from", "_to" + traversal.EndpointIndexFields = []string{"_from", "project", "dataset_generation", "label", "to_type"} + } + } + _, err := RenderPhysicalPlan(reductionPlan, mustBinding(t)) + wantUnsupported(t, err, "endpoint-lookup on the reduction surface") + if !strings.Contains(err.Error(), "no proven PostgreSQL contract for this consumer") { + t.Fatalf("the reduction surface must keep the fail-closed refusal: %v", err) + } +} + +func TestPostWindowEndpointLookupRefusesFieldsThatDisagreeWithDirection(t *testing.T) { + withTraversal := func(mutate func(*ir.PhysicalTraversal)) ir.PhysicalPlan { + set := postWindowAggregateSet(true) + mutate(set.Subplan.Operations[0].Traversal) + return postWindowAggregatePlan(set, + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, false)) + } + + inboundPairOnOutbound := withTraversal(func(traversal *ir.PhysicalTraversal) { + traversal.EndpointField, traversal.EndpointJoinField = "_to", "_from" + traversal.EndpointIndexFields = []string{"_to", "project", "dataset_generation", "label", "from_type"} + }) + _, err := RenderPhysicalPlan(inboundPairOnOutbound, mustBinding(t)) + wantUnsupported(t, err, "endpoint fields paired for the opposite direction") + if !strings.Contains(err.Error(), "must fail closed") { + t.Fatalf("the direction mismatch must fail closed, not normalize: %v", err) + } + + swapOnlyJoinField := withTraversal(func(traversal *ir.PhysicalTraversal) { + traversal.EndpointJoinField = "_from" + }) + _, err = RenderPhysicalPlan(swapOnlyJoinField, mustBinding(t)) + wantUnsupported(t, err, "endpoint join field swapped against the direction") + + unknownDirection := withTraversal(func(traversal *ir.PhysicalTraversal) { + traversal.Direction = ir.PhysicalTraversalDirection("SELF") + }) + _, err = RenderPhysicalPlan(unknownDirection, mustBinding(t)) + if err == nil || !strings.Contains(err.Error(), "invalid traversal direction") { + t.Fatalf("an undirected endpoint lookup must be refused, got: %v", err) + } +} + +func TestPostWindowDeadTraversalRendersNothingAndLocatedConsumersRefuse(t *testing.T) { + base := postWindowAggregatePlan(postWindowAggregateSet(false), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, false)) + baseline := renderOK(t, base).SQL + + dead := postWindowAggregatePlan(postWindowAggregateSet(false), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, false)) + deadTraversal := setsTraversal() + deadEdge, deadTarget := "route_edge_9", "route_node_9" + deadTraversal.EdgeVariable, deadTraversal.TargetVariable = deadEdge, deadTarget + deadOps := appendScopeBlock([]ir.PhysicalOperation{{Kind: ir.PhysicalTraversalOp, Traversal: &deadTraversal}}, + []string{deadEdge, deadTarget}, "route_9_scope_auth") + deadOps = append([]ir.PhysicalOperation{dead.Operations[0], dead.Operations[1], dead.Operations[2]}, deadOps...) + deadOps = append(deadOps, dead.Operations[3]) + dead.Operations = deadOps + rendered := renderOK(t, dead).SQL + if rendered != baseline { + t.Fatalf("a dead post-window traversal must not change the statement:\n%s\n---\n%s", rendered, baseline) + } + + live := postWindowAggregatePlan(postWindowAggregateSet(false), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, false)) + liveTraversal := setsTraversal() + liveTraversal.EdgeVariable, liveTraversal.TargetVariable = deadEdge, deadTarget + liveRet := live.Operations[3] + liveRet.Return.Projections = append(liveRet.Return.Projections, ir.PhysicalProjection{ + Name: "leak", Value: ir.PhysicalValue{Variable: deadTarget, Path: []string{"logical_id"}}}) + liveOps := appendScopeBlock([]ir.PhysicalOperation{{Kind: ir.PhysicalTraversalOp, Traversal: &liveTraversal}}, + []string{deadEdge, deadTarget}, "route_9_scope_auth") + liveOps = append([]ir.PhysicalOperation{live.Operations[0], live.Operations[1], live.Operations[2]}, liveOps...) + liveOps = append(liveOps, liveRet) + live.Operations = liveOps + _, err := RenderPhysicalPlan(live, mustBinding(t)) + wantUnsupported(t, err, "post-window traversal with a live consumer") + if !strings.Contains(err.Error(), deadTarget) || !strings.Contains(err.Error(), "not dead") { + t.Fatalf("the refusal must name the read variable and the dead-LET rule: %v", err) + } +} + +func TestPostWindowAggregateOverNonSetSourceRefused(t *testing.T) { + plan := setsPlanWithSet(setsChildSet(ir.PhysicalSetReductionFirst), "__loom_reduced_0") + plan.BindVars[setsPrefix+"_predicate_value"] = "asthma" + returnOp := plan.Operations[len(plan.Operations)-1] + returnOp.Return.Projections = append(returnOp.Return.Projections, + postWindowAggregateProjection("count_all", ir.PhysicalCountAggregate, false, false)) + plan.Operations[len(plan.Operations)-1].Return.Projections[len(plan.Operations[len(plan.Operations)-1].Return.Projections)-1]. + Expression.Aggregate.Source = ir.PhysicalValue{Variable: "root"} + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + wantUnsupported(t, err, "aggregate over a non-set source") + if !strings.Contains(err.Error(), "not a registered child set") { + t.Fatalf("the gate must name the unregistered source: %v", err) + } +} + +func deadTraversalOps(plan ir.PhysicalPlan, edgeVariable, targetVariable string) ir.PhysicalPlan { + traversal := setsTraversal() + traversal.EdgeVariable, traversal.TargetVariable = edgeVariable, targetVariable + block := appendScopeBlock([]ir.PhysicalOperation{{Kind: ir.PhysicalTraversalOp, Traversal: &traversal}}, + []string{edgeVariable, targetVariable}, "dead_scope_auth") + ops := []ir.PhysicalOperation{plan.Operations[0], plan.Operations[1], plan.Operations[2]} + ops = append(ops, block...) + ops = append(ops, plan.Operations[3]) + plan.Operations = ops + return plan +} + +func TestPostWindowDeadTraversalNearMisses(t *testing.T) { + const deadEdge, deadTarget = "dead_edge", "dead_node" + + t.Run("consumer in a later subplan block", func(t *testing.T) { + base := postWindowAggregatePlan(postWindowAggregateSet(false), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, false)) + plan := deadTraversalOps(base, deadEdge, deadTarget) + poison := postWindowAggregateSet(false) + poison.Subplan.Captures = []string{"root", deadTarget} + poison.Variable = "child_set_1" + last := len(plan.Operations) - 1 + plan.Operations = append(plan.Operations[:last], + append([]ir.PhysicalOperation{{Kind: ir.PhysicalSetOp, Set: &poison}}, plan.Operations[last:]...)...) + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + wantUnsupported(t, err, "dead hop read from a later subplan") + if !strings.Contains(err.Error(), deadTarget) || !strings.Contains(err.Error(), "not dead") { + t.Fatalf("refusal must name the read variable and the dead-LET rule: %v", err) + } + }) + + t.Run("reducer references the dead target", func(t *testing.T) { + base := postWindowAggregatePlan(postWindowAggregateSet(false), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, false)) + plan := deadTraversalOps(base, deadEdge, deadTarget) + projections := plan.Operations[len(plan.Operations)-1].Return.Projections + projections[1].Expression.Aggregate.Value = postWindowSlotExpression("__loom_projection_1", + ir.PhysicalSelectorDirectScalar, postWindowValueSteps()) + projections[1].Expression.Aggregate.Value.Extract.Source.Variable = deadTarget + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + wantUnsupported(t, err, "reducer reading the dead target") + if !strings.Contains(err.Error(), deadTarget) || !strings.Contains(err.Error(), "not dead") { + t.Fatalf("refusal must name the read variable and the dead-LET rule: %v", err) + } + }) + + t.Run("subplan sort key references the dead target", func(t *testing.T) { + base := postWindowAggregatePlan(postWindowAggregateSet(false), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, false)) + plan := deadTraversalOps(base, deadEdge, deadTarget) + poison := postWindowAggregateSet(false) + poison.Subplan.Captures = []string{"root", deadTarget} + poison.Subplan.Sort = &ir.PhysicalValue{Variable: deadTarget, Path: []string{"payload", "status"}} + poison.Variable = "child_set_1" + last := len(plan.Operations) - 1 + plan.Operations = append(plan.Operations[:last], + append([]ir.PhysicalOperation{{Kind: ir.PhysicalSetOp, Set: &poison}}, plan.Operations[last:]...)...) + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + wantUnsupported(t, err, "subplan sort key reading the dead target") + if !strings.Contains(err.Error(), deadTarget) || !strings.Contains(err.Error(), "not dead") { + t.Fatalf("refusal must name the read variable and the dead-LET rule: %v", err) + } + }) +} + +func TestPostWindowUnrimmedAggregateOperationsRefuse(t *testing.T) { + for _, operation := range []ir.PhysicalAggregateOperation{ + ir.PhysicalMeanAggregate, ir.PhysicalExistsAggregate, ir.PhysicalCountDistinctAggregate, + ir.PhysicalDistinctValuesAggregate, ir.PhysicalFirstAggregate, ir.PhysicalCollectAggregate, + ir.PhysicalRequireOneAggregate, ir.PhysicalContainsAllAggregate, ir.PhysicalFirstOrderedAggregate, + } { + t.Run(string(operation), func(t *testing.T) { + plan := postWindowAggregatePlan(postWindowAggregateSet(false), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, false), + postWindowAggregateProjection("sum_age", ir.PhysicalSumAggregate, true, false)) + aggregate := plan.Operations[3].Return.Projections[2].Expression.Aggregate + aggregate.Operation = operation + switch operation { + case ir.PhysicalExistsAggregate: + aggregate.Value = nil + case ir.PhysicalContainsAllAggregate: + plan.BindVars["required_values"] = []string{"asthma"} + aggregate.RequiredValuesBindKey = "required_values" + case ir.PhysicalFirstOrderedAggregate: + stamp := postWindowSlotExpression("__loom_projection_1", ir.PhysicalSelectorDirectScalar, postWindowValueSteps()) + aggregate.ContributorWindow = &ir.PhysicalContributorWindow{ + Timestamp: *stamp, Anchor: *stamp, Precision: "INSTANT", + } + aggregate.Ordering = &ir.PhysicalTemporalOrdering{Timestamp: *stamp, Direction: "ASC", TiePolicy: "RESOURCE_KEY"} + } + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + wantUnsupported(t, err, "unrimmed aggregate operation "+string(operation)) + if !strings.Contains(err.Error(), string(ir.PhysicalAggregateExpression)) || + !strings.Contains(err.Error(), "no registered PostgreSQL rendering") { + t.Fatalf("aggregate gate text drifted: %v", err) + } + }) + } +} + +func TestPostWindowReducersPreserveRowCounts(t *testing.T) { + const tailMarker = `."fhir_resource" AS r0 WHERE r0."project"` + + baseline := postWindowAggregatePlan(postWindowAggregateSet(false)) + baseline.Operations = []ir.PhysicalOperation{baseline.Operations[0], baseline.Operations[1], baseline.Operations[3]} + baseline.Operations[2].Return.Projections = baseline.Operations[2].Return.Projections[:1] + baselineSQL := renderOK(t, baseline).SQL + + full := postWindowAggregatePlan(postWindowAggregateSet(false), + postWindowAggregateProjection("sum_age", ir.PhysicalSumAggregate, true, true), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, true), + postWindowAggregateProjection("min_age", ir.PhysicalMinAggregate, true, true), + postWindowAggregateProjection("max_age", ir.PhysicalMaxAggregate, true, true)) + fullSQL := renderOK(t, full).SQL + + baselineTail := baselineSQL[strings.Index(baselineSQL, tailMarker):] + fullTail := fullSQL[strings.Index(fullSQL, tailMarker):] + if baselineTail != fullTail { + t.Fatalf("post-window reducers changed the outer row spine:\nbaseline: %s\nfull: %s", baselineTail, fullTail) + } + if strings.Count(fullSQL, "AS r0") != 1 || strings.Contains(fullSQL, "LEFT JOIN") { + t.Fatalf("reducers must stay scalar sublinks over one root scan:\n%s", fullSQL) + } +} + +func TestPostWindowReducersUnderPreviewLimit(t *testing.T) { + plan := postWindowAggregatePlan(postWindowAggregateSet(false), + postWindowAggregateProjection("count_age", ir.PhysicalCountAggregate, false, false)) + plan.BindVars["preview_limit"] = 25 + limitOp := ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "preview_limit"}} + ops := append([]ir.PhysicalOperation{}, plan.Operations[:2]...) + ops = append(ops, limitOp, plan.Operations[2], plan.Operations[3]) + plan.Operations = ops + + rendered := renderOK(t, plan) + if !strings.Contains(rendered.SQL, "LIMIT $") { + t.Fatalf("preview LIMIT must stay on the execution window:\n%s", rendered.SQL) + } + for _, want := range []string{"to_jsonb(COUNT(*))", `AS "c1"`} { + if !strings.Contains(rendered.SQL, want) { + t.Fatalf("preview-path statement is missing %q:\n%s", want, rendered.SQL) + } + } + sawLimitParameter := false + for _, parameter := range rendered.Args { + if parameter.Type == command.PostgreSQLParameterBigint && parameter.Value == int64(25) { + sawLimitParameter = true + } + } + if !sawLimitParameter { + t.Fatalf("preview LIMIT parameter bigint 25 missing: %#v", rendered.Args) + } +} diff --git a/internal/dataframe/compiler/render/postgres/predicates.go b/internal/dataframe/compiler/render/postgres/predicates.go new file mode 100644 index 000000000..ce889196a --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/predicates.go @@ -0,0 +1,605 @@ +package postgres + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" +) + +func (r *planRenderer) renderFilter(plan ir.PhysicalPlan, filter ir.PhysicalFilter) (string, error) { + if err := r.rejectOptionalRowReferences(filter); err != nil { + return "", err + } + if filter.Expression != nil { + if filter.Expression.Kind == ir.PhysicalExistsPredicate { + if isRelatedExistsSubplan(filter.Expression.Exists) { + return r.renderRelatedExists(plan, *filter.Expression.Exists) + } + fragment, err := r.renderRequiredMatchExists(plan, *filter.Expression) + if err != nil { + return "", err + } + return fragment.sql, nil + } + if filter.Expression.Kind == ir.PhysicalNotPredicate && len(filter.Expression.Children) == 1 && + filter.Expression.Children[0].Kind == ir.PhysicalExistsPredicate && + isRelatedExistsSubplan(filter.Expression.Children[0].Exists) { + clause, err := r.renderRelatedExists(plan, *filter.Expression.Children[0].Exists) + if err != nil { + return "", err + } + return "NOT (" + clause + ")", nil + } + return r.renderPredicateExpression(plan, *filter.Expression) + } + return r.renderComparison(plan, filter.Predicate) +} + +func (r *planRenderer) renderPredicateExpression(plan ir.PhysicalPlan, predicate ir.PhysicalPredicateExpression) (string, error) { + fragment, err := r.renderPredicateFragment(plan, predicate) + if err != nil { + return "", err + } + return fragment.sql, nil +} + +type predicateFragment struct { + sql string + mayYieldNull bool +} + +func (r *planRenderer) renderPredicateFragment(plan ir.PhysicalPlan, predicate ir.PhysicalPredicateExpression) (predicateFragment, error) { + switch predicate.Kind { + case ir.PhysicalComparisonPredicate: + if predicate.Comparison == nil { + return predicateFragment{}, fmt.Errorf("%w: COMPARISON predicate has no payload", ErrUnsupportedPostgresPlan) + } + return r.renderComparisonFragment(plan, *predicate.Comparison) + case ir.PhysicalNotPredicate: + if len(predicate.Children) != 1 { + return predicateFragment{}, fmt.Errorf("%w: NOT requires exactly one child", ErrUnsupportedPostgresPlan) + } + child, err := r.renderPredicateFragment(plan, predicate.Children[0]) + if err != nil { + return predicateFragment{}, err + } + if child.mayYieldNull { + return predicateFragment{sql: "COALESCE(NOT (" + child.sql + "), true)"}, nil + } + return predicateFragment{sql: "NOT (" + child.sql + ")"}, nil + case ir.PhysicalAllPredicate: + if len(predicate.Children) == 0 { + return predicateFragment{}, fmt.Errorf("%w: ALL requires at least one child", ErrUnsupportedPostgresPlan) + } + parts := make([]string, 0, len(predicate.Children)) + mayYieldNull := false + for _, child := range predicate.Children { + part, err := r.renderPredicateFragment(plan, child) + if err != nil { + return predicateFragment{}, err + } + mayYieldNull = mayYieldNull || part.mayYieldNull + parts = append(parts, "("+part.sql+")") + } + return predicateFragment{sql: strings.Join(parts, " AND "), mayYieldNull: mayYieldNull}, nil + case ir.PhysicalAnyPredicate: + if len(predicate.Children) == 0 { + return predicateFragment{}, fmt.Errorf("%w: ANY requires at least one child", ErrUnsupportedPostgresPlan) + } + parts := make([]string, 0, len(predicate.Children)) + mayYieldNull := false + for _, child := range predicate.Children { + part, err := r.renderPredicateFragment(plan, child) + if err != nil { + return predicateFragment{}, err + } + mayYieldNull = mayYieldNull || part.mayYieldNull + parts = append(parts, "("+part.sql+")") + } + return predicateFragment{sql: strings.Join(parts, " OR "), mayYieldNull: mayYieldNull}, nil + default: + return predicateFragment{}, fmt.Errorf("%w: predicate kind %q is not yet implemented for PostgreSQL", + ErrUnsupportedPostgresPlan, string(predicate.Kind)) + } +} + +func (r *planRenderer) renderComparison(plan ir.PhysicalPlan, predicate ir.PhysicalPredicate) (string, error) { + fragment, err := r.renderComparisonFragment(plan, predicate) + if err != nil { + return "", err + } + return fragment.sql, nil +} + +func (r *planRenderer) renderComparisonFragment(plan ir.PhysicalPlan, predicate ir.PhysicalPredicate) (predicateFragment, error) { + if predicate.Correlation != nil { + return predicateFragment{}, fmt.Errorf("%w: correlated predicates are not yet implemented for PostgreSQL", + ErrUnsupportedPostgresPlan) + } + if predicate.Left.BindKey != "" { + return predicateFragment{}, fmt.Errorf("%w: filter left side must be a row path, not a bind value", ErrUnsupportedPostgresPlan) + } + if predicate.LeftExpression != nil { + return r.renderSelectorComparisonFragment(plan, predicate) + } + if predicate.Quantifier != "" { + return predicateFragment{}, fmt.Errorf("%w: array quantifier %q is not yet implemented for PostgreSQL", + ErrUnsupportedPostgresPlan, string(predicate.Quantifier)) + } + left, err := r.resolveValue(predicate.Left) + if err != nil { + return predicateFragment{}, err + } + operator := strings.ToUpper(strings.TrimSpace(predicate.Operator)) + if err := validateDirectOperatorKind(operator, predicate.ValueKind); err != nil { + return predicateFragment{}, err + } + switch operator { + case "EXISTS", "MISSING": + if predicate.Right != nil { + return predicateFragment{}, fmt.Errorf("%w: %s must not carry a right value", ErrUnsupportedPostgresPlan, operator) + } + if isPayloadExpression(left) { + presence := "COALESCE(jsonb_typeof(" + left + "), 'null')" + if operator == "EXISTS" { + return predicateFragment{sql: presence + " <> 'null'"}, nil + } + return predicateFragment{sql: presence + " = 'null'"}, nil + } + if operator == "EXISTS" { + return predicateFragment{sql: left + " IS NOT NULL"}, nil + } + return predicateFragment{sql: left + " IS NULL"}, nil + case "EQUALS", "NOT_EQUALS", "GT", "GTE", "LT", "LTE", "IN": + default: + return predicateFragment{}, fmt.Errorf("%w: filter operator %q is not part of the v1 PostgreSQL surface", + ErrUnsupportedPostgresPlan, predicate.Operator) + } + if predicate.Right == nil || predicate.Right.BindKey == "" { + return predicateFragment{}, fmt.Errorf("%w: %s requires a bind-backed right value", ErrUnsupportedPostgresPlan, operator) + } + value, ok := plan.BindVars[predicate.Right.BindKey] + if !ok { + return predicateFragment{}, fmt.Errorf("%w: filter value bind %q is not defined", ErrUnsupportedPostgresPlan, predicate.Right.BindKey) + } + if !isPayloadExpression(left) { + return r.renderColumnComparison(left, operator, predicate.ValueKind, value) + } + return r.renderPayloadComparison(left, operator, predicate.ValueKind, value) +} + +func validateDirectOperatorKind(operator string, kind spec.FilterValueKind) error { + switch operator { + case "GT", "GTE", "LT", "LTE": + if kind != "" && !spec.OperatorSupportsKind(spec.FilterOperator(operator), kind) { + return fmt.Errorf("%w: operator %s is not compatible with value kind %q", + ErrUnsupportedPostgresPlan, operator, string(kind)) + } + } + return nil +} + +func isPayloadExpression(expression string) bool { return strings.Contains(expression, " #> $") } + +func (r *planRenderer) renderColumnComparison(left, operator string, kind spec.FilterValueKind, value any) (predicateFragment, error) { + isRecordedAt := strings.HasSuffix(left, "."+r.storage.Resource.MustColumn("recorded_at").SQLText()) + if operator == "IN" { + paths, ok := value.([]string) + if !ok { + return predicateFragment{}, fmt.Errorf("%w: column IN requires a []string bind value", ErrUnsupportedPostgresPlan) + } + raw, err := json.Marshal(append([]string(nil), paths...)) + if err != nil { + return predicateFragment{}, fmt.Errorf("%w: column IN values are not JSON-encodable: %v", ErrUnsupportedPostgresPlan, err) + } + if string(raw) == "null" { + raw = []byte("[]") + } + return predicateFragment{sql: left + " = ANY (SELECT jsonb_array_elements_text($" + strconv.Itoa(r.jsonbParameter(raw)) + "::jsonb))"}, nil + } + if isRecordedAt { + if kind != "" && kind != spec.FilterDate && kind != spec.FilterDateTime { + return predicateFragment{}, fmt.Errorf("%w: value kind %q has no registered recorded-at comparison; that column admits only temporal comparisons", + ErrUnsupportedPostgresPlan, string(kind)) + } + timestamp, err := temporalComparisonValue(kind, value) + if err != nil { + return predicateFragment{}, err + } + index := r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterTimestamptz, Value: timestamp}, "") + return predicateFragment{sql: left + " " + sqlComparison(operator) + " $" + strconv.Itoa(index)}, nil + } + if kind != "" && kind != spec.FilterString && kind != spec.FilterCode { + return predicateFragment{}, fmt.Errorf("%w: value kind %q has no registered typed-column comparison; text columns accept STRING/CODE", + ErrUnsupportedPostgresPlan, string(kind)) + } + text, ok := value.(string) + if !ok { + return predicateFragment{}, fmt.Errorf("%w: typed column comparisons require a string bind value, got %T", + ErrUnsupportedPostgresPlan, value) + } + return predicateFragment{sql: left + " " + sqlComparison(operator) + " $" + strconv.Itoa(r.textParameter(text, "text:"+text))}, nil +} + +func temporalComparisonValue(kind spec.FilterValueKind, value any) (time.Time, error) { + switch typed := value.(type) { + case time.Time: + if kind == "" || kind == spec.FilterDate || kind == spec.FilterDateTime { + return typed.UTC(), nil + } + return time.Time{}, fmt.Errorf("%w: value kind %q cannot bind a timestamp", ErrUnsupportedPostgresPlan, string(kind)) + case string: + if strings.Contains(typed, ",") { + return time.Time{}, fmt.Errorf("%w: comparison literal %q uses a comma decimal, which the PostgreSQL renderer refuses (AQL DATE_TIMESTAMP folds comma fractions to NaN and never matches)", + ErrUnsupportedPostgresPlan, typed) + } + if kind == spec.FilterDate { + parsed, err := time.Parse("2006-01-02", typed) + if err != nil { + return time.Time{}, fmt.Errorf("%w: DATE comparison literal %q is not a full-precision YYYY-MM-DD date", + ErrUnsupportedPostgresPlan, typed) + } + return parsed.UTC(), nil + } + if kind == spec.FilterDateTime || kind == "" { + parsed, err := time.Parse(time.RFC3339Nano, typed) + if err != nil { + return time.Time{}, fmt.Errorf("%w: DATE_TIME comparison literal %q is not a full RFC3339 timestamp", + ErrUnsupportedPostgresPlan, typed) + } + return parsed.UTC(), nil + } + return time.Time{}, fmt.Errorf("%w: value kind %q cannot bind a timestamp", ErrUnsupportedPostgresPlan, string(kind)) + default: + return time.Time{}, fmt.Errorf("%w: recorded_at comparisons require a time.Time or full-precision temporal string bind value, got %T", + ErrUnsupportedPostgresPlan, value) + } +} + +func (r *planRenderer) renderPayloadComparison(left, operator string, kind spec.FilterValueKind, value any) (predicateFragment, error) { + if operator == "IN" { + return predicateFragment{}, fmt.Errorf("%w: payload-path IN is not yet implemented for PostgreSQL", ErrUnsupportedPostgresPlan) + } + if err := validatePayloadBindType(kind, value); err != nil { + return predicateFragment{}, err + } + raw, err := json.Marshal(value) + if err != nil { + return predicateFragment{}, fmt.Errorf("%w: filter value %T is not JSON-encodable: %v", ErrUnsupportedPostgresPlan, value, err) + } + if string(raw) == "null" { + presence := "COALESCE(jsonb_typeof(" + left + "), 'null')" + switch operator { + case "EQUALS", "LTE": + return predicateFragment{sql: presence + " = 'null'"}, nil + case "NOT_EQUALS", "GT": + return predicateFragment{sql: presence + " <> 'null'"}, nil + case "LT", "GTE": + return predicateFragment{}, fmt.Errorf("%w: payload ordering %s against a JSON null literal is not part of the v1 PostgreSQL surface", + ErrUnsupportedPostgresPlan, operator) + default: + return predicateFragment{}, fmt.Errorf("%w: payload operator %q against a JSON null literal is not part of the v1 PostgreSQL surface", + ErrUnsupportedPostgresPlan, operator) + } + } + if kind == spec.FilterDate || kind == spec.FilterDateTime { + switch operator { + case "GT", "GTE", "LT", "LTE": + return r.renderTemporalPayloadComparison(left, operator, kind, value) + } + } + if isNumericOrdering(operator, kind) { + index := r.jsonbParameter(raw) + return predicateFragment{sql: "(jsonb_typeof((" + left + ")) = 'number' AND (" + left + ") " + + sqlComparison(operator) + " ($" + strconv.Itoa(index) + "::jsonb))", mayYieldNull: true}, nil + } + index := r.jsonbParameter(raw) + if operator == "NOT_EQUALS" { + return predicateFragment{sql: "(" + left + ") IS DISTINCT FROM ($" + strconv.Itoa(index) + "::jsonb)"}, nil + } + return predicateFragment{sql: "(" + left + ") " + sqlComparison(operator) + " ($" + strconv.Itoa(index) + "::jsonb)", mayYieldNull: true}, nil +} + +func validatePayloadBindType(kind spec.FilterValueKind, value any) error { + if kind == "" { + switch value.(type) { + case string, bool, int, int64, float64, json.RawMessage: + return nil + default: + return fmt.Errorf("%w: payload comparisons accept string, bool, number, or raw JSON values, got %T", + ErrUnsupportedPostgresPlan, value) + } + } + switch kind { + case spec.FilterString, spec.FilterCode, spec.FilterDate, spec.FilterDateTime: + if _, ok := value.(string); !ok { + return fmt.Errorf("%w: value kind %q requires a string bind value, got %T", ErrUnsupportedPostgresPlan, string(kind), value) + } + return nil + case spec.FilterBoolean: + if _, ok := value.(bool); !ok { + return fmt.Errorf("%w: value kind BOOLEAN requires a bool bind value, got %T", ErrUnsupportedPostgresPlan, value) + } + return nil + case spec.FilterInteger: + if _, ok := value.(int64); !ok { + return fmt.Errorf("%w: value kind INTEGER requires an int64 bind value, got %T", ErrUnsupportedPostgresPlan, value) + } + return nil + case spec.FilterDecimal: + if _, ok := value.(float64); !ok { + return fmt.Errorf("%w: value kind DECIMAL requires a float64 bind value, got %T", ErrUnsupportedPostgresPlan, value) + } + return nil + default: + return fmt.Errorf("%w: value kind %q is not registered for PostgreSQL", ErrUnsupportedPostgresPlan, string(kind)) + } +} + +const ( + temporalLeapTensOnesPattern = `(0[48]|[2468][048]|[13579][26])` + temporalStoredYearPattern = `[1-9][0-9]{3}` + temporalStoredDateCorePattern = `([1-9][0-9]{3}-(0[13578]|1[02])-(0[1-9]|[12][0-9]|3[01])|` + temporalStoredYearPattern + `-(0[469]|11)-(0[1-9]|[12][0-9]|30)|` + temporalStoredYearPattern + `-02-(0[1-9]|1[0-9]|2[0-8])|` + temporalLeapTensOnesPattern + `00-02-29|[0-9]{2}` + temporalLeapTensOnesPattern + `-02-29)` + temporalStoredTimePattern = `(0[0-9]|1[0-9]|2[0-3]):[0-5][0-9](:[0-5][0-9]([.][0-9]+)?)?` + temporalStoredOffsetPattern = `(Z|[+-]((0[0-9]|1[0-4]):[0-5][0-9]|15:[0-5][0-9]))` + temporalStoredDatePattern = `^` + temporalStoredDateCorePattern + `$` + temporalStoredInstantPattern = `^` + temporalStoredDateCorePattern + `T` + temporalStoredTimePattern + temporalStoredOffsetPattern + `$` + temporalStoredNoOffsetPattern = `^` + temporalStoredDateCorePattern + `T` + temporalStoredTimePattern + `$` + temporalStoredComparisonSuffix = `T00:00:00+00:00` +) + +func temporalStoredExpression(extraction string) string { + text := "((" + extraction + ") #>> '{}')" + return "CASE" + + " WHEN jsonb_typeof(" + extraction + ") = 'string' AND " + text + " ~ '" + temporalStoredDatePattern + "'" + + " THEN (" + text + " || '" + temporalStoredComparisonSuffix + "')::timestamptz" + + " WHEN jsonb_typeof(" + extraction + ") = 'string' AND " + text + " ~ '" + temporalStoredInstantPattern + "'" + + " THEN " + text + "::timestamptz" + + " WHEN jsonb_typeof(" + extraction + ") = 'string' AND " + text + " ~ '" + temporalStoredNoOffsetPattern + "'" + + " THEN (" + text + " || '+00:00')::timestamptz" + + " END" +} + +func (r *planRenderer) renderTemporalPayloadComparison(left, operator string, kind spec.FilterValueKind, value any) (predicateFragment, error) { + timestamp, err := temporalComparisonValue(kind, value) + if err != nil { + return predicateFragment{}, err + } + index := r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterTimestamptz, Value: timestamp}, "") + return predicateFragment{ + sql: "(" + temporalStoredExpression(left) + ") " + sqlComparison(operator) + " $" + strconv.Itoa(index), + mayYieldNull: true, + }, nil +} + +func sqlComparison(operator string) string { + switch operator { + case "EQUALS": + return "=" + case "NOT_EQUALS": + return "IS DISTINCT FROM" + case "GT": + return ">" + case "GTE": + return ">=" + case "LT": + return "<" + case "LTE": + return "<=" + default: + return "" + } +} + +func (r *planRenderer) renderSelectorComparisonFragment(plan ir.PhysicalPlan, predicate ir.PhysicalPredicate) (predicateFragment, error) { + expression := predicate.LeftExpression + if expression.Kind != ir.PhysicalExtractExpression || expression.Cardinality != ir.PhysicalArrayCardinality { + return predicateFragment{}, fmt.Errorf("%w: selector filter left side must be an ARRAY-cardinality EXTRACT expression", + ErrUnsupportedPostgresPlan) + } + if expression.NullBehavior != ir.PhysicalEmptyOnNull { + return predicateFragment{}, fmt.Errorf("%w: selector filter extraction null behavior %q has no registered comparison form", + ErrUnsupportedPostgresPlan, string(expression.NullBehavior)) + } + extract := expression.Extract + if extract == nil || len(extract.Fallbacks) != 0 || extract.Prepared != nil || + extract.UnitNormalization != nil || extract.Distinct || extract.Selector.Filter != nil { + return predicateFragment{}, fmt.Errorf("%w: selector filter extraction carries unregistered payload (fallbacks, prepared sets, unit normalization, DISTINCT, or selector filters are gaps)", + ErrUnsupportedPostgresPlan) + } + switch extract.ExecutionMode { + case ir.PhysicalSelectorGeneric, ir.PhysicalSelectorDirectScalar, "": + default: + return predicateFragment{}, fmt.Errorf("%w: selector filter execution mode %q has no registered comparison form", + ErrUnsupportedPostgresPlan, string(extract.ExecutionMode)) + } + source := extract.Source + if source.BindKey != "" || source.Variable == "" || len(source.Path) != 1 || source.Path[0] != "payload" { + return predicateFragment{}, fmt.Errorf("%w: selector filter source must be one readable row's document payload", + ErrUnsupportedPostgresPlan) + } + scope, ok := r.rows[source.Variable] + if !ok { + return predicateFragment{}, fmt.Errorf("%w: selector filter source variable %q does not bind a readable row", + ErrUnsupportedPostgresPlan, source.Variable) + } + if strings.TrimSpace(extract.ResourceType) == "" || extract.ResourceType != scope.resourceType { + return predicateFragment{}, fmt.Errorf("%w: selector filter resource type %q contradicts the scanned row type %q", + ErrUnsupportedPostgresPlan, extract.ResourceType, scope.resourceType) + } + if predicate.Left.Variable != "" || len(predicate.Left.Path) != 0 { + return predicateFragment{}, fmt.Errorf("%w: selector filter predicate cannot also declare an ordinary left value", + ErrUnsupportedPostgresPlan) + } + if predicate.Right != nil && predicate.Right.BindKey == "" { + return predicateFragment{}, fmt.Errorf("%w: selector filter right value must be bind-backed", ErrUnsupportedPostgresPlan) + } + if predicate.Right != nil && (predicate.Right.Variable != "" || len(predicate.Right.Path) != 0) { + return predicateFragment{}, fmt.Errorf("%w: selector filter right value must be a bind, not a row path", ErrUnsupportedPostgresPlan) + } + + path := make([]string, 0, len(extract.Selector.Steps)) + for _, step := range extract.Selector.Steps { + if step.Field == "" || step.Iterate || step.Index != nil { + return predicateFragment{}, fmt.Errorf("%w: selector filter accepts only direct property paths; repeated, indexed, and filtered steps are gaps", + ErrUnsupportedPostgresPlan) + } + path = append(path, step.Field) + } + if len(path) == 0 { + return predicateFragment{}, fmt.Errorf("%w: selector filter selector must have at least one property", ErrUnsupportedPostgresPlan) + } + operator := strings.ToUpper(strings.TrimSpace(predicate.Operator)) + if err := validateDirectOperatorKind(operator, predicate.ValueKind); err != nil { + return predicateFragment{}, err + } + + extraction, err := r.documentPathExpression(scope.alias, path) + if err != nil { + return predicateFragment{}, err + } + presence := "COALESCE(jsonb_typeof(" + extraction + "), 'null') <> 'null'" + + switch operator { + case "EXISTS": + if predicate.Right != nil { + return predicateFragment{}, fmt.Errorf("%w: EXISTS must not carry a right value", ErrUnsupportedPostgresPlan) + } + return predicateFragment{sql: presence}, nil + case "MISSING": + if predicate.Right != nil { + return predicateFragment{}, fmt.Errorf("%w: MISSING must not carry a right value", ErrUnsupportedPostgresPlan) + } + return predicateFragment{sql: "NOT (" + presence + ")"}, nil + } + if predicate.Right == nil || predicate.Right.BindKey == "" { + return predicateFragment{}, fmt.Errorf("%w: %s requires a bind-backed right value", ErrUnsupportedPostgresPlan, operator) + } + if !predicate.ValueKind.Valid() { + return predicateFragment{}, fmt.Errorf("%w: selector filter value kind %q is not registered", + ErrUnsupportedPostgresPlan, string(predicate.ValueKind)) + } + value, ok := plan.BindVars[predicate.Right.BindKey] + if !ok { + return predicateFragment{}, fmt.Errorf("%w: filter value bind %q is not defined", ErrUnsupportedPostgresPlan, predicate.Right.BindKey) + } + match, err := r.selectorMatchExpression(extraction, operator, predicate.ValueKind, value) + if err != nil { + return predicateFragment{}, err + } + any_ := "(" + presence + " AND " + match + ")" + switch predicate.Quantifier { + case "", spec.QuantifierAny: + return predicateFragment{sql: any_}, nil + case spec.QuantifierNone: + return predicateFragment{sql: "NOT (" + any_ + ")"}, nil + case spec.QuantifierAll: + return predicateFragment{sql: any_}, nil + default: + return predicateFragment{}, fmt.Errorf("%w: selector filter quantifier %q is not registered", + ErrUnsupportedPostgresPlan, string(predicate.Quantifier)) + } +} + +func (r *planRenderer) selectorMatchExpression(extraction, operator string, kind spec.FilterValueKind, value any) (string, error) { + if operator == "CONTAINS_TEXT" { + return "", fmt.Errorf("%w: selector CONTAINS_TEXT is not part of the registered PostgreSQL surface", + ErrUnsupportedPostgresPlan) + } + if operator == "IN" { + if err := validateSelectorInList(kind, value); err != nil { + return "", err + } + raw, err := json.Marshal(value) + if err != nil { + return "", fmt.Errorf("%w: selector IN values are not JSON-encodable: %v", ErrUnsupportedPostgresPlan, err) + } + if string(raw) == "null" { + raw = []byte("[]") + } + index := r.jsonbParameter(raw) + return "EXISTS (SELECT 1 FROM jsonb_array_elements($" + strconv.Itoa(index) + "::jsonb) AS loom_selector_item WHERE loom_selector_item = (" + extraction + "))", nil + } + if err := validatePayloadBindType(kind, value); err != nil { + return "", err + } + if kind == spec.FilterDate || kind == spec.FilterDateTime { + switch operator { + case "GT", "GTE", "LT", "LTE": + timestamp, err := temporalComparisonValue(kind, value) + if err != nil { + return "", err + } + index := r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterTimestamptz, Value: timestamp}, "") + return "COALESCE((" + temporalStoredExpression(extraction) + ") " + sqlComparison(operator) + + " $" + strconv.Itoa(index) + ", false)", nil + } + } + raw, err := json.Marshal(value) + if err != nil { + return "", fmt.Errorf("%w: filter value %T is not JSON-encodable: %v", ErrUnsupportedPostgresPlan, value, err) + } + if string(raw) == "null" { + return "", fmt.Errorf("%w: selector comparison against a null literal is not part of the registered PostgreSQL surface", + ErrUnsupportedPostgresPlan) + } + comparison := sqlComparison(operator) + if operator == "NOT_EQUALS" { + comparison = "<>" + } + if isNumericOrdering(operator, kind) { + return "COALESCE(jsonb_typeof(" + extraction + ") = 'number' AND (" + extraction + ") " + + comparison + " ($" + strconv.Itoa(r.jsonbParameter(raw)) + "::jsonb), false)", nil + } + return "(" + extraction + ") " + comparison + " ($" + strconv.Itoa(r.jsonbParameter(raw)) + "::jsonb)", nil +} + +func isNumericOrdering(operator string, kind spec.FilterValueKind) bool { + switch operator { + case "GT", "GTE", "LT", "LTE": + return kind == spec.FilterInteger || kind == spec.FilterDecimal + } + return false +} + +func validateSelectorInList(kind spec.FilterValueKind, value any) error { + var elements []any + switch list := value.(type) { + case []string: + if kind != spec.FilterString && kind != spec.FilterCode { + return fmt.Errorf("%w: selector IN with a []string bind is admitted only for STRING and CODE kinds, got kind %q", + ErrUnsupportedPostgresPlan, string(kind)) + } + return nil + case []any: + elements = list + default: + return fmt.Errorf("%w: selector IN requires a []any or []string bind value, got %T", ErrUnsupportedPostgresPlan, value) + } + for _, element := range elements { + var ok bool + switch kind { + case spec.FilterString, spec.FilterCode, spec.FilterDate, spec.FilterDateTime: + _, ok = element.(string) + case spec.FilterBoolean: + _, ok = element.(bool) + case spec.FilterInteger: + _, ok = element.(int64) + case spec.FilterDecimal: + _, ok = element.(float64) + default: + return fmt.Errorf("%w: selector IN value kind %q is not registered", ErrUnsupportedPostgresPlan, string(kind)) + } + if !ok { + return fmt.Errorf("%w: selector IN element %T contradicts the declared value kind %q", + ErrUnsupportedPostgresPlan, element, string(kind)) + } + } + return nil +} diff --git a/internal/dataframe/compiler/render/postgres/predicates_pg18_test.go b/internal/dataframe/compiler/render/postgres/predicates_pg18_test.go new file mode 100644 index 000000000..3dbdc8567 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/predicates_pg18_test.go @@ -0,0 +1,638 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "os" + "reflect" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + sql05Prefix = "verify_sql05_it" + sql05Project = "verify_sql05-proj" + sql05Gen = "gen-1" + sql05RowPrefix = "verify_sql05-" +) + +func newSQL05Service(t *testing.T) *pgir02Service { + t.Helper() + return newOwnedService(t, 0, sql05Prefix) +} + +func sql05Plan(resourceType string) ir.PhysicalPlan { + return ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": resourceType, + "project": sql05Project, + "dataset_generation": sql05Gen, + "auth_resource_paths": []string{"FHIR/" + resourceType}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + }, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + }, + } +} + +func sql05Insert(t *testing.T, svc *pgir02Service, resourceType, logicalID string, payload any) { + t.Helper() + svc.insertResource(t, svc.schema, sql05Project, sql05Gen, resourceType, sql05RowPrefix+logicalID, "FHIR/"+resourceType, payload) +} + +func sql05Rows(logicalIDs ...string) [][]string { + rows := make([][]string, 0, len(logicalIDs)) + for _, id := range logicalIDs { + rows = append(rows, []string{sql05RowPrefix + id}) + } + return rows +} + +func sql05Compare(t *testing.T, label string, got [][]string, want [][]string) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Fatalf("%s rows = %v, want %v", label, got, want) + } +} + +func sql05DirectLeaf(field, operator, bindKey string, kind spec.FilterValueKind, withRight bool) ir.PhysicalPredicate { + predicate := ir.PhysicalPredicate{ + Operator: operator, + ValueKind: kind, + Left: ir.PhysicalValue{Variable: "root", Path: []string{field}}, + } + if withRight { + predicate.Right = &ir.PhysicalValue{BindKey: bindKey} + } + return predicate +} + +func sql05RenderedSelect(t *testing.T, svc *pgir02Service, plan ir.PhysicalPlan) [][]string { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + if strings.Contains(rendered.SQL, sql05Project) || strings.Contains(rendered.SQL, sql05RowPrefix) { + t.Fatalf("plan data leaked into SQL: %s", rendered.SQL) + } + return svc.runRendered(t, rendered) +} + +func sql05WitnessRows(t *testing.T, svc *pgir02Service, resourceType, predicate string) [][]string { + t.Helper() + return svc.witness(t, + `SELECT logical_id FROM "`+svc.schema+`"."fhir_resource"`+ + ` WHERE project=$1 AND dataset_generation=$2 AND resource_type=$3 AND (`+predicate+`)`+ + ` ORDER BY logical_id, resource_seq`, sql05Project, sql05Gen, resourceType) +} + +func TestPG18ScalarFilterIndependentRows(t *testing.T) { + svc := newSQL05Service(t) + + sql05Insert(t, svc, "Patient", "str-amended", map[string]any{"status": "amended"}) + sql05Insert(t, svc, "Patient", "str-final", map[string]any{"status": "final"}) + sql05Insert(t, svc, "Patient", "str-missing", map[string]any{"unrelated": 1}) + sql05Insert(t, svc, "Patient", "str-null", map[string]any{"status": nil}) + sql05Insert(t, svc, "Observation", "score-decimal", map[string]any{"score": 2.5}) + sql05Insert(t, svc, "Observation", "score-missing", map[string]any{"unrelated": 2}) + sql05Insert(t, svc, "Observation", "score-seven", map[string]any{"score": 7}) + sql05Insert(t, svc, "Observation", "score-string", map[string]any{"score": "seven"}) + sql05Insert(t, svc, "Condition", "date-feb29-leap", map[string]any{"observed": "2024-02-29"}) + sql05Insert(t, svc, "Condition", "date-feb29-nonleap", map[string]any{"observed": "2023-02-29"}) + sql05Insert(t, svc, "Condition", "date-invalid-day", map[string]any{"observed": "2024-02-30"}) + sql05Insert(t, svc, "Condition", "date-invalid-month", map[string]any{"observed": "2024-13-01"}) + sql05Insert(t, svc, "Condition", "date-january", map[string]any{"observed": "2024-01-01"}) + sql05Insert(t, svc, "Condition", "date-march", map[string]any{"observed": "2024-03-15"}) + sql05Insert(t, svc, "Condition", "date-missing", map[string]any{"unrelated": 3}) + sql05Insert(t, svc, "Condition", "date-partial", map[string]any{"observed": "2024"}) + sql05Insert(t, svc, "Condition", "date-string", map[string]any{"observed": "not-a-date"}) + sql05Insert(t, svc, "Encounter", "instant-bad-offset", map[string]any{"recorded": "2024-03-15T10:20:30+99:99"}) + sql05Insert(t, svc, "Encounter", "instant-bad-time", map[string]any{"recorded": "2024-01-01T25:70:99Z"}) + sql05Insert(t, svc, "Encounter", "instant-early", map[string]any{"recorded": "2024-03-15T09:00:00+02:00"}) + sql05Insert(t, svc, "Encounter", "instant-late", map[string]any{"recorded": "2024-03-15T10:20:30Z"}) + sql05Insert(t, svc, "Encounter", "instant-offset-twin", map[string]any{"recorded": "2024-03-15T12:20:30+02:00"}) + + storedDateWitness := func(field string) string { + extraction := `(payload #> '{` + field + `}'::text[])` + text := `((` + extraction + `) #>> '{}')` + return "CASE" + + ` WHEN jsonb_typeof(` + extraction + `) = 'string' AND ` + text + ` ~ '^([1-9][0-9]{3}-(0[13578]|1[02])-(0[1-9]|[12][0-9]|3[01])|[1-9][0-9]{3}-(0[469]|11)-(0[1-9]|[12][0-9]|30)|[1-9][0-9]{3}-02-(0[1-9]|1[0-9]|2[0-8])|(0[48]|[2468][048]|[13579][26])00-02-29|[0-9]{2}(0[48]|[2468][048]|[13579][26])-02-29)$'` + + ` THEN ((` + text + ` || 'T00:00:00+00:00')::timestamptz)` + + ` WHEN jsonb_typeof(` + extraction + `) = 'string' AND ` + text + ` ~ '^([1-9][0-9]{3}-(0[13578]|1[02])-(0[1-9]|[12][0-9]|3[01])|[1-9][0-9]{3}-(0[469]|11)-(0[1-9]|[12][0-9]|30)|[1-9][0-9]{3}-02-(0[1-9]|1[0-9]|2[0-8])|(0[48]|[2468][048]|[13579][26])00-02-29|[0-9]{2}(0[48]|[2468][048]|[13579][26])-02-29)T(0[0-9]|1[0-9]|2[0-3]):[0-5][0-9](:[0-5][0-9]([.][0-9]+)?)?(Z|[+-]((0[0-9]|1[0-4]):[0-5][0-9]|15:[0-5][0-9]))$'` + + ` THEN (` + text + `)::timestamptz` + + ` WHEN jsonb_typeof(` + extraction + `) = 'string' AND ` + text + ` ~ '^([1-9][0-9]{3}-(0[13578]|1[02])-(0[1-9]|[12][0-9]|3[01])|[1-9][0-9]{3}-(0[469]|11)-(0[1-9]|[12][0-9]|30)|[1-9][0-9]{3}-02-(0[1-9]|1[0-9]|2[0-8])|(0[48]|[2468][048]|[13579][26])00-02-29|[0-9]{2}(0[48]|[2468][048]|[13579][26])-02-29)T(0[0-9]|1[0-9]|2[0-3]):[0-5][0-9](:[0-5][0-9]([.][0-9]+)?)?$'` + + ` THEN ((` + text + ` || '+00:00')::timestamptz)` + + ` END` + } + + cases := []struct { + label string + resourceType string + operator string + field string + kind spec.FilterValueKind + bindKey string + bind any + want [][]string + witness string + }{ + { + label: "string equality", resourceType: "Patient", operator: "EQUALS", field: "status", + kind: spec.FilterString, bindKey: "status_value", bind: "final", + want: sql05Rows("str-final"), + witness: `payload #> '{status}'::text[] = '"final"'::jsonb`, + }, + { + label: "string inequality keeps null and missing", resourceType: "Patient", operator: "NOT_EQUALS", field: "status", + kind: spec.FilterString, bindKey: "status_value", bind: "final", + want: sql05Rows("str-amended", "str-missing", "str-null"), + witness: `payload #> '{status}'::text[] IS DISTINCT FROM '"final"'::jsonb`, + }, + { + label: "explicit null equality is null-or-missing", resourceType: "Patient", operator: "EQUALS", field: "status", + bindKey: "null_value", bind: json.RawMessage("null"), + want: sql05Rows("str-missing", "str-null"), + witness: `payload #> '{status}'::text[] IS NULL OR payload #> '{status}'::text[] = 'null'::jsonb`, + }, + { + label: "code equality matches the stored code", resourceType: "Patient", operator: "EQUALS", field: "status", + kind: spec.FilterCode, bindKey: "status_value", bind: "amended", + want: sql05Rows("str-amended"), + witness: `payload #> '{status}'::text[] = '"amended"'::jsonb`, + }, + { + label: "missing treats stored JSON null as absent", resourceType: "Patient", operator: "MISSING", field: "status", + want: sql05Rows("str-missing", "str-null"), + witness: `payload #> '{status}'::text[] IS NULL OR payload #> '{status}'::text[] = 'null'::jsonb`, + }, + { + label: "exists keeps present values only", resourceType: "Patient", operator: "EXISTS", field: "status", + want: sql05Rows("str-amended", "str-final"), + witness: `COALESCE(jsonb_typeof(payload #> '{status}'::text[]), 'null') <> 'null'`, + }, + { + label: "decimal equality", resourceType: "Observation", operator: "EQUALS", field: "score", + kind: spec.FilterDecimal, bindKey: "score_value", bind: 2.5, + want: sql05Rows("score-decimal"), + witness: `payload #> '{score}'::text[] = '2.5'::jsonb`, + }, + { + label: "integer range lower bound", resourceType: "Observation", operator: "GTE", field: "score", + kind: spec.FilterInteger, bindKey: "score_value", bind: int64(5), + want: sql05Rows("score-seven"), + witness: `jsonb_typeof((payload #> '{score}'::text[])) = 'number' AND ((payload #> '{score}'::text[])) >= '5'::jsonb`, + }, + { + label: "decimal range upper bound inclusive", resourceType: "Observation", operator: "LTE", field: "score", + kind: spec.FilterDecimal, bindKey: "score_value", bind: 7.0, + want: sql05Rows("score-decimal", "score-seven"), + witness: `jsonb_typeof((payload #> '{score}'::text[])) = 'number' AND ((payload #> '{score}'::text[])) <= '7'::jsonb`, + }, + { + label: "date range orders full and leap dates and excludes never-comparable partials and invalid calendars", resourceType: "Condition", operator: "GTE", field: "observed", + kind: spec.FilterDate, bindKey: "date_value", bind: "2024-02-01", + want: sql05Rows("date-feb29-leap", "date-march"), + witness: storedDateWitness("observed") + ` >= '2024-02-01T00:00:00+00:00'::timestamptz`, + }, + { + label: "payload-path date ordering upper bound excludes never-comparable strings", resourceType: "Condition", operator: "LTE", field: "observed", + kind: spec.FilterDate, bindKey: "date_value", bind: "2024-02-01", + want: sql05Rows("date-january"), + witness: storedDateWitness("observed") + ` <= '2024-02-01T00:00:00+00:00'::timestamptz`, + }, + { + label: "date-time equality stays raw text (AQL parity)", resourceType: "Encounter", operator: "EQUALS", field: "recorded", + kind: spec.FilterDateTime, bindKey: "when_value", bind: "2024-03-15T10:20:30Z", + want: sql05Rows("instant-late"), + witness: `payload #> '{recorded}'::text[] = '"2024-03-15T10:20:30Z"'::jsonb`, + }, + { + label: "instant range orders across offsets", resourceType: "Encounter", operator: "GTE", field: "recorded", + kind: spec.FilterDateTime, bindKey: "when_value", bind: "2024-03-15T09:00:00Z", + want: sql05Rows("instant-late", "instant-offset-twin"), + witness: storedDateWitness("recorded") + ` >= '2024-03-15T09:00:00+00:00'::timestamptz`, + }, + } + + for _, testCase := range cases { + t.Run(testCase.label, func(t *testing.T) { + plan := sql05Plan(testCase.resourceType) + if testCase.bind != nil { + plan.BindVars[testCase.bindKey] = testCase.bind + } + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate( + sql05DirectLeaf(testCase.field, testCase.operator, testCase.bindKey, testCase.kind, testCase.bind != nil))) + sql05Compare(t, testCase.label, sql05RenderedSelect(t, svc, plan), testCase.want) + sql05Compare(t, testCase.label+" witness", sql05WitnessRows(t, svc, testCase.resourceType, testCase.witness), testCase.want) + }) + } +} + +func TestPG18SelectorQuantifierTruthTables(t *testing.T) { + svc := newSQL05Service(t) + sql05Insert(t, svc, "Patient", "g-array-ab", map[string]any{"gender": []any{"a", "b"}}) + sql05Insert(t, svc, "Patient", "g-array-null-a", map[string]any{"gender": []any{nil, "a"}}) + sql05Insert(t, svc, "Patient", "g-female", map[string]any{"gender": "female"}) + sql05Insert(t, svc, "Patient", "g-male", map[string]any{"gender": "male"}) + sql05Insert(t, svc, "Patient", "g-missing", map[string]any{"unrelated": 1}) + sql05Insert(t, svc, "Patient", "g-null", map[string]any{"gender": nil}) + + allRows := sql05Rows("g-array-ab", "g-array-null-a", "g-female", "g-male", "g-missing", "g-null") + + selectorFilter := func(selectorPath, operator, bindKey string, quantifier spec.ArrayQuantifier, withRight bool) ir.PhysicalOperation { + return pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", selectorPath, operator, quantifier, spec.FilterString, bindKey, withRight)) + } + + cases := []struct { + label string + selector string + operator string + quant spec.ArrayQuantifier + bindKey string + bind any + want [][]string + }{ + {label: "ANY matches the present item", selector: "gender", operator: "EQUALS", + quant: spec.QuantifierAny, bindKey: "g", bind: "female", want: sql05Rows("g-female")}, + {label: "NONE keeps male, JSON null, and missing alike", selector: "gender", operator: "EQUALS", + quant: spec.QuantifierNone, bindKey: "g", bind: "female", + want: sql05Rows("g-array-ab", "g-array-null-a", "g-male", "g-missing", "g-null")}, + {label: "ALL requires the matching item", selector: "gender", operator: "EQUALS", + quant: spec.QuantifierAll, bindKey: "g", bind: "male", want: sql05Rows("g-male")}, + {label: "ANY equality ignores JSON null as absent", selector: "gender", operator: "EQUALS", + quant: spec.QuantifierAny, bindKey: "g", bind: "nope", want: nil}, + {label: "array elements are never quantified individually", selector: "gender", operator: "EQUALS", + quant: spec.QuantifierAny, bindKey: "t", bind: "a", want: nil}, + {label: "NONE over arrays proves the whole item is compared", selector: "gender", operator: "EQUALS", + quant: spec.QuantifierNone, bindKey: "t", bind: "a", want: allRows}, + {label: "IN binds top-level list membership", selector: "gender", operator: "IN", + quant: spec.QuantifierAny, bindKey: "gs", bind: []any{"male", "female"}, want: sql05Rows("g-female", "g-male")}, + {label: "empty IN list selects nobody", selector: "gender", operator: "IN", + quant: spec.QuantifierAny, bindKey: "gs", bind: []any{}, want: nil}, + } + + for _, testCase := range cases { + t.Run(testCase.label, func(t *testing.T) { + plan := sql05Plan("Patient") + if testCase.bind != nil { + plan.BindVars[testCase.bindKey] = testCase.bind + } + plan = pgsq05AddFilter(plan, selectorFilter(testCase.selector, testCase.operator, testCase.bindKey, testCase.quant, testCase.bind != nil)) + sql05Compare(t, testCase.label, sql05RenderedSelect(t, svc, plan), testCase.want) + }) + } + + existsPlan := sql05Plan("Patient") + existsPlan = pgsq05AddFilter(existsPlan, selectorFilter("gender", "EXISTS", "", "", false)) + sql05Compare(t, "selector EXISTS", sql05RenderedSelect(t, svc, existsPlan), sql05Rows("g-array-ab", "g-array-null-a", "g-female", "g-male")) + + missingPlan := sql05Plan("Patient") + missingPlan = pgsq05AddFilter(missingPlan, selectorFilter("gender", "MISSING", "", "", false)) + sql05Compare(t, "selector MISSING", sql05RenderedSelect(t, svc, missingPlan), sql05Rows("g-missing", "g-null")) +} + +func TestPG18BooleanPredicateTruthTable(t *testing.T) { + svc := newSQL05Service(t) + sql05Insert(t, svc, "Patient", "tt-ff", map[string]any{"alpha": false, "beta": false}) + sql05Insert(t, svc, "Patient", "tt-ft", map[string]any{"alpha": false, "beta": true}) + sql05Insert(t, svc, "Patient", "tt-missing", map[string]any{"unrelated": 1}) + sql05Insert(t, svc, "Patient", "tt-null", map[string]any{"alpha": nil, "beta": true}) + sql05Insert(t, svc, "Patient", "tt-tf", map[string]any{"alpha": true, "beta": false}) + sql05Insert(t, svc, "Patient", "tt-tt", map[string]any{"alpha": true, "beta": true}) + + leaf := func(field, bindKey string) ir.PhysicalPredicateExpression { + return pgsq05ComparisonLeaf(ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: "root", Path: []string{field}}, + Right: &ir.PhysicalValue{BindKey: bindKey}, + }) + } + all := func(children ...ir.PhysicalPredicateExpression) ir.PhysicalPredicateExpression { + return ir.PhysicalPredicateExpression{Kind: ir.PhysicalAllPredicate, Children: children} + } + any := func(children ...ir.PhysicalPredicateExpression) ir.PhysicalPredicateExpression { + return ir.PhysicalPredicateExpression{Kind: ir.PhysicalAnyPredicate, Children: children} + } + not := func(child ir.PhysicalPredicateExpression) ir.PhysicalPredicateExpression { + return ir.PhysicalPredicateExpression{Kind: ir.PhysicalNotPredicate, Children: []ir.PhysicalPredicateExpression{child}} + } + + const alphaW = `COALESCE(jsonb_typeof(payload #> '{alpha}'::text[]), 'null') <> 'null' AND (payload #> '{alpha}'::text[]) = 'true'::jsonb` + const betaW = `COALESCE(jsonb_typeof(payload #> '{beta}'::text[]), 'null') <> 'null' AND (payload #> '{beta}'::text[]) = 'true'::jsonb` + + cases := []struct { + label string + build func(a, b ir.PhysicalPredicateExpression) ir.PhysicalPredicateExpression + witness string + want [][]string + }{ + {label: "AND keeps only the true-true row", + build: func(a, b ir.PhysicalPredicateExpression) ir.PhysicalPredicateExpression { return all(a, b) }, + witness: `(` + alphaW + `) AND (` + betaW + `)`, + want: sql05Rows("tt-tt")}, + {label: "OR selects every row with a true side", + build: func(a, b ir.PhysicalPredicateExpression) ir.PhysicalPredicateExpression { return any(a, b) }, + witness: `(` + alphaW + `) OR (` + betaW + `)`, + want: sql05Rows("tt-ft", "tt-null", "tt-tf", "tt-tt")}, + {label: "NOR excludes every row with a true side", + build: func(a, b ir.PhysicalPredicateExpression) ir.PhysicalPredicateExpression { + return not(any(a, b)) + }, + witness: `NOT ((` + alphaW + `) OR (` + betaW + `))`, + want: sql05Rows("tt-ff", "tt-missing")}, + {label: "AND with negated beta", + build: func(a, b ir.PhysicalPredicateExpression) ir.PhysicalPredicateExpression { + return all(a, not(b)) + }, + witness: `(` + alphaW + `) AND (NOT (` + betaW + `))`, + want: sql05Rows("tt-tf")}, + {label: "exclusive-or pattern", + build: func(a, b ir.PhysicalPredicateExpression) ir.PhysicalPredicateExpression { + return any(all(a, not(b)), all(not(a), b)) + }, + witness: `((` + alphaW + `) AND (NOT (` + betaW + `))) OR ((NOT (` + alphaW + `)) AND (` + betaW + `))`, + want: sql05Rows("tt-ft", "tt-null", "tt-tf")}, + } + + for _, testCase := range cases { + t.Run(testCase.label, func(t *testing.T) { + plan := sql05Plan("Patient") + plan.BindVars["alpha_value"] = true + plan.BindVars["beta_value"] = true + plan = pgsq05AddFilter(plan, pgsq05FilterByExpression(testCase.build(leaf("alpha", "alpha_value"), leaf("beta", "beta_value")))) + sql05Compare(t, testCase.label, sql05RenderedSelect(t, svc, plan), testCase.want) + sql05Compare(t, testCase.label+" witness", sql05WitnessRows(t, svc, "Patient", testCase.witness), testCase.want) + }) + } + + notPlan := sql05Plan("Patient") + notPlan.BindVars["alpha_value"] = true + notPlan = pgsq05AddFilter(notPlan, pgsq05FilterByExpression(not(leaf("alpha", "alpha_value")))) + sql05Compare(t, "NOT keeps null and missing rows", sql05RenderedSelect(t, svc, notPlan), + sql05Rows("tt-ff", "tt-ft", "tt-missing", "tt-null")) +} + +func TestPG18SelectorHostileValuesAreInert(t *testing.T) { + svc := newSQL05Service(t) + hostile := `x'; DROP TABLE fhir_resource; --` + sql05Insert(t, svc, "Patient", "h-plain", map[string]any{"gender": "plain"}) + sql05Insert(t, svc, "Patient", "h-value", map[string]any{"gender": hostile}) + + plan := sql05Plan("Patient") + plan.BindVars["gender_value"] = hostile + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "gender", "EQUALS", spec.QuantifierAny, spec.FilterString, "gender_value", true))) + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + if strings.Contains(rendered.SQL, "DROP") || strings.Contains(rendered.SQL, hostile) { + t.Fatalf("hostile value reached statement text: %s", rendered.SQL) + } + sql05Compare(t, "hostile selector equality", svc.runRendered(t, rendered), sql05Rows("h-value")) + for _, table := range []string{"fhir_resource", "fhir_edge_occurrence"} { + survived := svc.witness(t, `SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1 AND c.relname=$2`, svc.schema, table) + if len(survived) != 1 || survived[0][0] != "1" { + t.Fatalf("%s table was affected by a bound selector filter value", table) + } + } +} + +func TestPG18RecordedAtTypedDateLiteral(t *testing.T) { + svc := newSQL05Service(t) + sql05Insert(t, svc, "Patient", "recent", map[string]any{"resourceType": "Patient"}) + + recorded := func(operator, literal string) ir.PhysicalPlan { + plan := sql05Plan("Patient") + plan.BindVars["at_value"] = literal + return pgsq05AddFilter(plan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: operator, + ValueKind: spec.FilterDate, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"recorded_at"}}, + Right: &ir.PhysicalValue{BindKey: "at_value"}, + })) + } + + if got := sql05RenderedSelect(t, svc, recorded("GTE", "2020-01-01")); !reflect.DeepEqual(got, sql05Rows("recent")) { + t.Fatalf("recorded_at GTE 2020 rows = %v", got) + } + if got := sql05RenderedSelect(t, svc, recorded("LT", "2099-01-01")); !reflect.DeepEqual(got, sql05Rows("recent")) { + t.Fatalf("recorded_at LT 2099 rows = %v", got) + } + if got := sql05RenderedSelect(t, svc, recorded("GTE", "2099-01-01")); len(got) != 0 { + t.Fatalf("recorded_at GTE 2099 rows = %v, want none", got) + } +} + +const sql05MaxAdmittedOffsetSeconds = 15*3600 + 59*60 + +func TestPG18TemporalGuardExcludedSideProbes(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("LOOM_TEST_POSTGRES_URL not set") + } + pool, err := pgxpool.New(context.Background(), raw) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer pool.Close() + probes := []struct { + value string + expect string + }{ + {"2024-03-15T10:20:30+16:00", "22009"}, + {"2024-03-15T10:20:30-16:00", "22009"}, + {"2024-03-15T10:20:30+23:59", "22009"}, + {"2024-03-15T10:20:30-23:59", "22009"}, + {"2024-03-15T10:20:30,5", "22007"}, + {"2024-03-15T10:20:30,5Z", "22007"}, + {"2024-03-15T10:20:30+15:59", ""}, + {"2024-03-15T10:20:30-15:59", ""}, + {"2024-03-15T10:20:30+14:00", ""}, + {"2024-03-15T10:20:30.5Z", ""}, + } + for _, p := range probes { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + _, err := pool.Exec(ctx, "SELECT $1::text::timestamptz", p.value) + cancel() + if p.expect == "" { + if err != nil { + t.Errorf("control cast of %q failed: %v", p.value, err) + } + continue + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + if err == nil { + t.Errorf("cast of excluded shape %q succeeded: widen the guard pattern, do not relax this assertion", p.value) + } else { + t.Errorf("cast of %q: want SQLSTATE %s, got error %v", p.value, p.expect, err) + } + continue + } + if pgErr.Code != p.expect { + t.Errorf("cast of %q: want SQLSTATE %s, got %s (%s)", p.value, p.expect, pgErr.Code, pgErr.Message) + } + } +} + +func TestPG18FixedZoneInstantBindReturnsNormalizedMatches(t *testing.T) { + svc := newSQL05Service(t) + sql05Insert(t, svc, "Encounter", "fz-a", map[string]any{"recorded": "2024-01-01T00:00:00Z"}) + sql05Insert(t, svc, "Encounter", "fz-b", map[string]any{"recorded": "2024-01-02T00:00:00Z"}) + + recordedGTE := func(bind time.Time) ir.PhysicalPlan { + plan := sql05Plan("Encounter") + plan.BindVars["when_value"] = bind + return pgsq05AddFilter(plan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "GTE", + ValueKind: spec.FilterDateTime, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"recorded_at"}}, + Right: &ir.PhysicalValue{BindKey: "when_value"}, + })) + } + eastSixteen := func(t time.Time) *time.Location { + return time.FixedZone("Etc/GMT-16", 16*3600) + } + if got := sql05RenderedSelect(t, svc, recordedGTE(time.Now().Add(-time.Hour).In(eastSixteen(time.Now())))); !reflect.DeepEqual(got, sql05Rows("fz-a", "fz-b")) { + t.Fatalf("fixed-zone GTE (now-1h) rows = %v, want both", got) + } + if got := sql05RenderedSelect(t, svc, recordedGTE(time.Now().Add(time.Hour).In(eastSixteen(time.Now())))); len(got) != 0 { + t.Fatalf("fixed-zone GTE (now+1h) rows = %v, want none", got) + } +} + +func sql05ArmOrderableAtOrAfter2020(value string) bool { + bound := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + if strings.Contains(value, ",") { + return false + } + if day, err := time.Parse("2006-01-02", value); err == nil { + return !day.UTC().Before(bound) + } + for _, layout := range []string{ + time.RFC3339Nano, + "2006-01-02T15:04Z07:00", + "2006-01-02T15:04:05.999999999", + "2006-01-02T15:04", + } { + instant, err := time.Parse(layout, value) + if err != nil { + continue + } + _, offsetSeconds := instant.Zone() + if offsetSeconds > sql05MaxAdmittedOffsetSeconds || + offsetSeconds < -sql05MaxAdmittedOffsetSeconds { + return false + } + return !instant.UTC().Before(bound) + } + return false +} + +func TestPG18TemporalGuardBoundarySweep(t *testing.T) { + svc := newSQL05Service(t) + + arms := []struct { + id string + value string + rule string + }{ + {"b-1400", "2024-03-15T10:20:30+14:00", "offset inside admitted range"}, + {"b-1559", "2024-03-15T10:20:30+15:59", "offset admitted upper bound"}, + {"b-1600", "2024-03-15T10:20:30+16:00", "offset beyond 15:59 (22009)"}, + {"b-1900feb29", "1900-02-29", "century non-leap"}, + {"b-2000feb29", "2000-02-29", "century leap, before the filter bound"}, + {"b-2359", "2024-03-15T10:20:30+23:59", "offset beyond 15:59 (22009)"}, + {"b-2400feb29", "2400-02-29", "century leap orders"}, + {"b-apr31", "2024-04-31", "30-day month"}, + {"b-badtime", "2024-01-01T25:70:99Z", "time fields out of range"}, + {"b-comma-sec", "2024-03-15T10:20:30,5", "comma decimal (22007)"}, + {"b-comma-z", "2024-03-15T10:20:30,5Z", "comma decimal (22007)"}, + {"b-dec31", "2024-12-31", "31-day month"}, + {"b-dot-off", "2024-03-15T10:20:30.5+02:00", "dot fraction with offset"}, + {"b-dot-z", "2024-03-15T10:20:30.5Z", "dot fraction"}, + {"b-feb29leap", "2024-02-29", "leap day orders"}, + {"b-h24", "2024-03-15T24:00:00Z", "hour out of range"}, + {"b-jun30", "2024-06-30", "30-day month ok"}, + {"b-m1600", "2024-03-15T10:20:30-16:00", "negative offset beyond (22009)"}, + {"b-m2359", "2024-03-15T10:20:30-23:59", "negative offset beyond (22009)"}, + {"b-may31", "2024-05-31", "31-day month"}, + {"b-min", "2024-03-15T10:20Z", "minute precision"}, + {"b-nooff", "2024-03-15T10:20", "minute precision, no offset"}, + {"b-nov30", "2024-11-30", "30-day month ok"}, + {"b-sec60", "2024-03-15T10:20:60Z", "leap second excluded"}, + {"b-sep31", "2024-09-31", "30-day month"}, + {"b-yr0", "0000-01-01", "year 0000 never cast (also before bound)"}, + } + var want [][]string + for _, a := range arms { + sql05Insert(t, svc, "Encounter", a.id, map[string]any{"recorded": a.value}) + if sql05ArmOrderableAtOrAfter2020(a.value) { + want = append(want, []string{sql05RowPrefix + a.id}) + } + } + + plan := sql05Plan("Encounter") + plan.BindVars["when_value"] = "2020-01-01T00:00:00Z" + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "GTE", + ValueKind: spec.FilterDateTime, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"recorded"}}, + Right: &ir.PhysicalValue{BindKey: "when_value"}, + })) + sql05Compare(t, "temporal guard boundary sweep", sql05RenderedSelect(t, svc, plan), want) +} + +func TestSQL05TemporalGuardOracleSelfCheck(t *testing.T) { + keep := map[string]bool{ + "2024-03-15T10:20:30+14:00": true, + "2024-03-15T10:20:30+15:59": true, + "2024-03-15T10:20:30.5+02:00": true, + "2024-03-15T10:20:30.5Z": true, + "2024-03-15T10:20Z": true, + "2024-03-15T10:20": true, + "2400-02-29": true, + "2024-02-29": true, + "2024-12-31": true, + "2024-05-31": true, + "2024-06-30": true, + "2024-11-30": true, + "2024-03-15T10:20:30+16:00": false, + "2024-03-15T10:20:30-16:00": false, + "2024-03-15T10:20:30+23:59": false, + "2024-03-15T10:20:30-23:59": false, + "2024-03-15T10:20:30,5": false, + "2024-03-15T10:20:30,5Z": false, + "2024-01-01T25:70:99Z": false, + "2024-03-15T24:00:00Z": false, + "2024-03-15T10:20:60Z": false, + "1900-02-29": false, + "2000-02-29": false, + "2024-04-31": false, + "2024-09-31": false, + "0000-01-01": false, + "2024": false, + } + for value, want := range keep { + if got := sql05ArmOrderableAtOrAfter2020(value); got != want { + t.Errorf("oracle(%q) = %v, want %v", value, got, want) + } + } +} diff --git a/internal/dataframe/compiler/render/postgres/predicates_test.go b/internal/dataframe/compiler/render/postgres/predicates_test.go new file mode 100644 index 000000000..02afb9e96 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/predicates_test.go @@ -0,0 +1,1061 @@ +package postgres + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "regexp" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" +) + +const ( + pgsq05Project = "pgsq05-proj" + pgsq05Gen = "gen-1" + pgsq05Path = "FHIR/Patient" +) + +func pgsq05Plan(resourceType string) ir.PhysicalPlan { + return ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": resourceType, + "project": pgsq05Project, + "dataset_generation": pgsq05Gen, + "auth_resource_paths": []string{pgsq05Path}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + }, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + }, + } +} + +func pgsq05AddFilter(plan ir.PhysicalPlan, operation ir.PhysicalOperation) ir.PhysicalPlan { + operations := make([]ir.PhysicalOperation, 0, len(plan.Operations)+1) + operations = append(operations, plan.Operations[0]) + operations = append(operations, operation) + operations = append(operations, plan.Operations[1:]...) + plan.Operations = operations + return plan +} + +func pgsq05FilterByPredicate(predicate ir.PhysicalPredicate) ir.PhysicalOperation { + return ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: predicate}} +} + +func pgsq05FilterByExpression(expression ir.PhysicalPredicateExpression) ir.PhysicalOperation { + return ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Expression: &expression}} +} + +func pgsq05ComparisonLeaf(predicate ir.PhysicalPredicate) ir.PhysicalPredicateExpression { + return ir.PhysicalPredicateExpression{Kind: ir.PhysicalComparisonPredicate, Comparison: &predicate} +} + +func pgsq05SelectorPredicate(resourceType, selectorPath, operator string, + quantifier spec.ArrayQuantifier, kind spec.FilterValueKind, bindKey string, withRight bool) ir.PhysicalPredicate { + predicate := ir.PhysicalPredicate{ + Operator: operator, + Quantifier: quantifier, + ValueKind: kind, + LeftExpression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: resourceType, + Selector: pgsq05SelectorNoT(selectorPath), + }, + }, + } + if withRight { + predicate.Right = &ir.PhysicalValue{BindKey: bindKey} + } + return predicate +} + +func pgsq05SelectorNoT(path string) spec.Selector { + selector, err := spec.ParseSelector(path) + if err != nil { + panic("fixture selector " + path + ": " + err.Error()) + } + return selector +} + +func pgsq05WhereClause(t *testing.T, rendered command.RenderedPostgresPhysicalPlan) string { + t.Helper() + marker := " AND r0.\"resource_type\" = $4" + start := strings.Index(rendered.SQL, marker) + if start < 0 { + t.Fatalf("statement does not carry the canonical scope frame: %s", rendered.SQL) + } + rest := rendered.SQL[start+len(marker):] + end := strings.Index(rest, " ORDER BY ") + if end < 0 { + t.Fatalf("statement has no ORDER BY terminal: %s", rendered.SQL) + } + clause := rest[:end] + if !strings.HasPrefix(clause, " AND ") { + t.Fatalf("filter clause does not extend the WHERE list: %q", clause) + } + return strings.TrimPrefix(clause, " AND ") +} + +func pgsq05ScopeArgs(resourceType string) []command.PostgreSQLParameter { + return []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: pgsq05Project}, + {Type: command.PostgreSQLParameterText, Value: pgsq05Gen}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["FHIR/Patient"]`)}, + {Type: command.PostgreSQLParameterText, Value: resourceType}, + } +} + +func pgsq05Reject(t *testing.T, plan ir.PhysicalPlan, context string) { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil { + t.Fatalf("%s: render succeeded, want fail-closed rejection\nSQL: %s", context, rendered.SQL) + } + if rendered.SQL != "" || rendered.Args != nil || rendered.OutputColumns != nil { + t.Fatalf("%s: rejection leaked a rendered plan: %#v", context, rendered) + } + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("%s: error = %v (%T), want ErrUnsupportedPostgresPlan", context, err, err) + } +} + +func pgsq05RejectAny(t *testing.T, plan ir.PhysicalPlan, context string) { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil { + t.Fatalf("%s: render succeeded, want rejection\nSQL: %s", context, rendered.SQL) + } + if rendered.SQL != "" || rendered.Args != nil || rendered.OutputColumns != nil { + t.Fatalf("%s: rejection leaked a rendered plan: %#v", context, rendered) + } +} + +func pgsq05JSONRaw(t *testing.T, value any) json.RawMessage { + t.Helper() + raw, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal fixture value %#v: %v", value, err) + } + return raw +} + +func TestRenderSelectorComparisonGoldenAnyEquality(t *testing.T) { + plan := pgsq05Plan("Patient") + plan.BindVars["gender_value"] = "female" + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "gender", "EQUALS", spec.QuantifierAny, spec.FilterCode, "gender_value", true))) + rendered := renderOK(t, plan) + + wantClause := `(COALESCE(jsonb_typeof(r0."payload" #> $5::text[]), 'null') <> 'null'` + + ` AND (r0."payload" #> $5::text[]) = ($6::jsonb))` + if got := pgsq05WhereClause(t, rendered); got != wantClause { + t.Fatalf("ANY equality clause =\n%s\nwant\n%s", got, wantClause) + } + wantArgs := append(pgsq05ScopeArgs("Patient"), + command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: "{gender}"}, + command.PostgreSQLParameter{Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`"female"`)}, + ) + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("ANY equality args = %#v, want %#v", rendered.Args, wantArgs) + } +} + +func TestRenderSelectorComparisonEmptyQuantifierIsAny(t *testing.T) { + plan := pgsq05Plan("Patient") + plan.BindVars["gender_value"] = "female" + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "gender", "EQUALS", "", spec.FilterString, "gender_value", true))) + anyPlan := pgsq05Plan("Patient") + anyPlan.BindVars["gender_value"] = "female" + anyPlan = pgsq05AddFilter(anyPlan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "gender", "EQUALS", spec.QuantifierAny, spec.FilterString, "gender_value", true))) + if a, b := renderOK(t, plan).SQL, renderOK(t, anyPlan).SQL; a != b { + t.Fatalf("empty quantifier must equal ANY:\n%s\n%s", a, b) + } +} + +func TestRenderSelectorComparisonNoneIsNegatedAny(t *testing.T) { + plan := pgsq05Plan("Patient") + plan.BindVars["gender_value"] = "female" + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "gender", "EQUALS", spec.QuantifierNone, spec.FilterCode, "gender_value", true))) + rendered := renderOK(t, plan) + wantClause := `NOT ((COALESCE(jsonb_typeof(r0."payload" #> $5::text[]), 'null') <> 'null'` + + ` AND (r0."payload" #> $5::text[]) = ($6::jsonb)))` + if got := pgsq05WhereClause(t, rendered); got != wantClause { + t.Fatalf("NONE clause =\n%s\nwant\n%s", got, wantClause) + } +} + +func TestRenderSelectorComparisonAllRequiresTheItem(t *testing.T) { + plan := pgsq05Plan("Patient") + plan.BindVars["active_value"] = true + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "active", "EQUALS", spec.QuantifierAll, spec.FilterBoolean, "active_value", true))) + rendered := renderOK(t, plan) + wantClause := `(COALESCE(jsonb_typeof(r0."payload" #> $5::text[]), 'null') <> 'null'` + + ` AND (r0."payload" #> $5::text[]) = ($6::jsonb))` + if got := pgsq05WhereClause(t, rendered); got != wantClause { + t.Fatalf("ALL clause =\n%s\nwant\n%s", got, wantClause) + } + if got := rendered.Args[5]; got.Type != command.PostgreSQLParameterJSONB || string(got.Value.(json.RawMessage)) != "true" { + t.Fatalf("BOOLEAN equality arg = %#v", got) + } + negated := pgsq05Plan("Patient") + negated.BindVars["active_value"] = true + negated = pgsq05AddFilter(negated, pgsq05FilterByExpression(ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalNotPredicate, + Children: []ir.PhysicalPredicateExpression{pgsq05ComparisonLeaf(pgsq05SelectorPredicate( + "Patient", "active", "EQUALS", spec.QuantifierAll, spec.FilterBoolean, "active_value", true))}, + })) + negatedRendered := renderOK(t, negated) + if !strings.HasPrefix(pgsq05WhereClause(t, negatedRendered), "NOT (") { + t.Fatalf("NOT over a total quantifier comparison must stay plain:\n%s", negatedRendered.SQL) + } + if strings.Contains(pgsq05WhereClause(t, negatedRendered), "COALESCE(NOT") { + t.Fatalf("total quantifier comparison must not get the null guard:\n%s", negatedRendered.SQL) + } +} + +func TestRenderSelectorComparisonNotEqualsUsesTotalDestruction(t *testing.T) { + plan := pgsq05Plan("Patient") + plan.BindVars["gender_value"] = "male" + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "gender", "NOT_EQUALS", spec.QuantifierAny, spec.FilterCode, "gender_value", true))) + rendered := renderOK(t, plan) + wantClause := `(COALESCE(jsonb_typeof(r0."payload" #> $5::text[]), 'null') <> 'null'` + + ` AND (r0."payload" #> $5::text[]) <> ($6::jsonb))` + if got := pgsq05WhereClause(t, rendered); got != wantClause { + t.Fatalf("NOT_EQUALS clause =\n%s\nwant\n%s", got, wantClause) + } +} + +func TestRenderSelectorComparisonExistenceKeepsMissingDistinct(t *testing.T) { + plan := pgsq05Plan("Patient") + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "birthDate", "EXISTS", "", "", "unused", false))) + rendered := renderOK(t, plan) + if got := pgsq05WhereClause(t, rendered); got != `COALESCE(jsonb_typeof(r0."payload" #> $5::text[]), 'null') <> 'null'` { + t.Fatalf("selector EXISTS clause = %s", got) + } + missingPlan := pgsq05Plan("Patient") + missingPlan = pgsq05AddFilter(missingPlan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "birthDate", "MISSING", "", "", "unused", false))) + missingRendered := renderOK(t, missingPlan) + if got := pgsq05WhereClause(t, missingRendered); got != `NOT (COALESCE(jsonb_typeof(r0."payload" #> $5::text[]), 'null') <> 'null')` { + t.Fatalf("selector MISSING clause = %s", got) + } + if len(rendered.Args) != 5 || len(missingRendered.Args) != 5 { + t.Fatalf("existence comparisons bind scope plus one path parameter only: %#v", rendered.Args) + } +} + +func TestRenderSelectorComparisonInBindsTopLevelMembership(t *testing.T) { + plan := pgsq05Plan("Patient") + plan.BindVars["gender_values"] = []any{"male", "female"} + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "gender", "IN", spec.QuantifierAny, spec.FilterCode, "gender_values", true))) + rendered := renderOK(t, plan) + wantClause := `(COALESCE(jsonb_typeof(r0."payload" #> $5::text[]), 'null') <> 'null'` + + ` AND EXISTS (SELECT 1 FROM jsonb_array_elements($6::jsonb) AS loom_selector_item WHERE loom_selector_item = (r0."payload" #> $5::text[])))` + if got := pgsq05WhereClause(t, rendered); got != wantClause { + t.Fatalf("selector IN clause =\n%s\nwant\n%s", got, wantClause) + } + if got := rendered.Args[5]; got.Type != command.PostgreSQLParameterJSONB || + string(got.Value.(json.RawMessage)) != `["male","female"]` { + t.Fatalf("selector IN arg = %#v", got) + } + + empty := pgsq05Plan("Patient") + empty.BindVars["gender_values"] = []any{} + empty = pgsq05AddFilter(empty, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "gender", "IN", spec.QuantifierAny, spec.FilterCode, "gender_values", true))) + emptyRendered := renderOK(t, empty) + if got := emptyRendered.Args[5]; got.Type != command.PostgreSQLParameterJSONB || + string(got.Value.(json.RawMessage)) != `[]` { + t.Fatalf("empty selector IN arg = %#v, want []", got) + } +} + +func TestRenderSelectorComparisonTemporalOrdersByGuardedInstant(t *testing.T) { + plan := pgsq05Plan("Patient") + plan.BindVars["birth_value"] = "2024-03-15" + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "birthDate", "GT", spec.QuantifierAny, spec.FilterDate, "birth_value", true))) + rendered := renderOK(t, plan) + extraction := `r0."payload" #> $5::text[]` + text := `((` + extraction + `) #>> '{}')` + wantClause := `(COALESCE(jsonb_typeof(` + extraction + `), 'null') <> 'null'` + + ` AND COALESCE((CASE` + + ` WHEN jsonb_typeof(` + extraction + `) = 'string' AND ` + text + ` ~ '^([1-9][0-9]{3}-(0[13578]|1[02])-(0[1-9]|[12][0-9]|3[01])|[1-9][0-9]{3}-(0[469]|11)-(0[1-9]|[12][0-9]|30)|[1-9][0-9]{3}-02-(0[1-9]|1[0-9]|2[0-8])|(0[48]|[2468][048]|[13579][26])00-02-29|[0-9]{2}(0[48]|[2468][048]|[13579][26])-02-29)$'` + + ` THEN (` + text + ` || 'T00:00:00+00:00')::timestamptz` + + ` WHEN jsonb_typeof(` + extraction + `) = 'string' AND ` + text + ` ~ '^([1-9][0-9]{3}-(0[13578]|1[02])-(0[1-9]|[12][0-9]|3[01])|[1-9][0-9]{3}-(0[469]|11)-(0[1-9]|[12][0-9]|30)|[1-9][0-9]{3}-02-(0[1-9]|1[0-9]|2[0-8])|(0[48]|[2468][048]|[13579][26])00-02-29|[0-9]{2}(0[48]|[2468][048]|[13579][26])-02-29)T(0[0-9]|1[0-9]|2[0-3]):[0-5][0-9](:[0-5][0-9]([.][0-9]+)?)?(Z|[+-]((0[0-9]|1[0-4]):[0-5][0-9]|15:[0-5][0-9]))$'` + + ` THEN ` + text + `::timestamptz` + + ` WHEN jsonb_typeof(` + extraction + `) = 'string' AND ` + text + ` ~ '^([1-9][0-9]{3}-(0[13578]|1[02])-(0[1-9]|[12][0-9]|3[01])|[1-9][0-9]{3}-(0[469]|11)-(0[1-9]|[12][0-9]|30)|[1-9][0-9]{3}-02-(0[1-9]|1[0-9]|2[0-8])|(0[48]|[2468][048]|[13579][26])00-02-29|[0-9]{2}(0[48]|[2468][048]|[13579][26])-02-29)T(0[0-9]|1[0-9]|2[0-3]):[0-5][0-9](:[0-5][0-9]([.][0-9]+)?)?$'` + + ` THEN (` + text + ` || '+00:00')::timestamptz` + + ` END) > $6, false))` + if got := pgsq05WhereClause(t, rendered); got != wantClause { + t.Fatalf("DATE GT clause =\n%s\nwant\n%s", got, wantClause) + } + arg := rendered.Args[5] + if arg.Type != command.PostgreSQLParameterTimestamptz { + t.Fatalf("DATE literal arg type = %q, want timestamptz", arg.Type) + } + if stamp, ok := arg.Value.(time.Time); !ok || !stamp.Equal(time.Date(2024, 3, 15, 0, 0, 0, 0, time.UTC)) { + t.Fatalf("DATE literal arg = %#v, want 2024-03-15T00:00:00Z", arg.Value) + } +} + +func TestRenderSelectorComparisonDateTimeLiteralMustBeFullRFC3339(t *testing.T) { + good := pgsq05Plan("Patient") + good.BindVars["when_value"] = "2024-03-15T10:20:30+02:00" + good = pgsq05AddFilter(good, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "birthDate", "GTE", spec.QuantifierAny, spec.FilterDateTime, "when_value", true))) + rendered := renderOK(t, good) + arg := rendered.Args[5] + if arg.Type != command.PostgreSQLParameterTimestamptz { + t.Fatalf("DATE_TIME arg type = %q", arg.Type) + } + want := time.Date(2024, 3, 15, 8, 20, 30, 0, time.UTC) + if stamp, ok := arg.Value.(time.Time); !ok || !stamp.Equal(want) { + t.Fatalf("DATE_TIME arg = %#v, want %s", arg.Value, want) + } + + for _, literal := range []string{"2024", "2024-03", "2024-03-15", "not-a-date"} { + plan := pgsq05Plan("Patient") + plan.BindVars["when_value"] = literal + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "birthDate", "LT", spec.QuantifierAny, spec.FilterDateTime, "when_value", true))) + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); err == nil { + t.Fatalf("DATE_TIME literal %q accepted, want fail-closed rejection", literal) + } + } +} + +func TestRenderSelectorComparisonIntegerAndDecimalBindTypedScalars(t *testing.T) { + plan := pgsq05Plan("Patient") + plan.BindVars["score_value"] = int64(7) + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "gender", "GTE", spec.QuantifierAny, spec.FilterInteger, "score_value", true))) + rendered := renderOK(t, plan) + if got := rendered.Args[5]; got.Type != command.PostgreSQLParameterJSONB || + string(got.Value.(json.RawMessage)) != "7" { + t.Fatalf("INTEGER arg = %#v", got) + } + for _, mismatched := range []any{float64(7), "7", 7} { + plan := pgsq05Plan("Patient") + plan.BindVars["score_value"] = mismatched + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "gender", "GTE", spec.QuantifierAny, spec.FilterInteger, "score_value", true))) + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); err == nil { + t.Fatalf("INTEGER bind %#v accepted, want rejection", mismatched) + } + } + + decimalPlan := pgsq05Plan("Patient") + decimalPlan.BindVars["score_value"] = 2.5 + decimalPlan = pgsq05AddFilter(decimalPlan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "gender", "LTE", spec.QuantifierAny, spec.FilterDecimal, "score_value", true))) + decimalRendered := renderOK(t, decimalPlan) + if got := decimalRendered.Args[5]; got.Type != command.PostgreSQLParameterJSONB || + string(got.Value.(json.RawMessage)) != "2.5" { + t.Fatalf("DECIMAL arg = %#v", got) + } +} + +func TestRenderSelectorComparisonFailsClosedOutsideClosedShape(t *testing.T) { + iterPlan := pgsq05Plan("Patient") + iterPlan.BindVars["v"] = "x" + iterPlan = pgsq05AddFilter(iterPlan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "link[].other.reference", "EQUALS", spec.QuantifierAny, spec.FilterString, "v", true))) + pgsq05Reject(t, iterPlan, "repeated selector step") + + fallback := pgsq05SelectorPredicate("Patient", "gender", "EQUALS", spec.QuantifierAny, spec.FilterString, "v", true) + fallback.LeftExpression.Extract.Fallbacks = []ir.PhysicalSelectorFallback{{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Patient", + Selector: pgsq05SelectorNoT("active"), + }} + fallbackPlan := pgsq05Plan("Patient") + fallbackPlan.BindVars["v"] = "x" + fallbackPlan = pgsq05AddFilter(fallbackPlan, pgsq05FilterByPredicate(fallback)) + pgsq05Reject(t, fallbackPlan, "selector fallback") + + distinct := pgsq05SelectorPredicate("Patient", "gender", "EQUALS", spec.QuantifierAny, spec.FilterString, "v", true) + distinct.LeftExpression.Extract.Distinct = true + distinctPlan := pgsq05Plan("Patient") + distinctPlan.BindVars["v"] = "x" + distinctPlan = pgsq05AddFilter(distinctPlan, pgsq05FilterByPredicate(distinct)) + pgsq05Reject(t, distinctPlan, "selector DISTINCT") + + behavior := pgsq05SelectorPredicate("Patient", "gender", "EQUALS", spec.QuantifierAny, spec.FilterString, "v", true) + behavior.LeftExpression.NullBehavior = ir.PhysicalPreserveNull + behaviorPlan := pgsq05Plan("Patient") + behaviorPlan.BindVars["v"] = "x" + behaviorPlan = pgsq05AddFilter(behaviorPlan, pgsq05FilterByPredicate(behavior)) + pgsq05Reject(t, behaviorPlan, "selector null behavior outside registered table") + + contradict := pgsq05SelectorPredicate("Observation", "status", "EQUALS", spec.QuantifierAny, spec.FilterString, "v", true) + contradictPlan := pgsq05Plan("Patient") + contradictPlan.BindVars["v"] = "x" + contradictPlan = pgsq05AddFilter(contradictPlan, pgsq05FilterByPredicate(contradict)) + pgsq05Reject(t, contradictPlan, "selector resource type contradiction") + + source := pgsq05SelectorPredicate("Patient", "gender", "EQUALS", spec.QuantifierAny, spec.FilterString, "v", true) + source.LeftExpression.Extract.Source = ir.PhysicalValue{Variable: "root"} + sourcePlan := pgsq05Plan("Patient") + sourcePlan.BindVars["v"] = "x" + sourcePlan = pgsq05AddFilter(sourcePlan, pgsq05FilterByPredicate(source)) + pgsq05Reject(t, sourcePlan, "selector source without payload anchor") + + nullRight := pgsq05SelectorPredicate("Patient", "gender", "EQUALS", spec.QuantifierAny, spec.FilterString, "v", true) + nullRightPlan := pgsq05Plan("Patient") + nullRightPlan.BindVars["v"] = json.RawMessage("null") + nullRightPlan = pgsq05AddFilter(nullRightPlan, pgsq05FilterByPredicate(nullRight)) + pgsq05Reject(t, nullRightPlan, "selector comparison against a null literal") + + ordering := pgsq05SelectorPredicate("Patient", "gender", "GT", spec.QuantifierAny, spec.FilterString, "v", true) + orderingPlan := pgsq05Plan("Patient") + orderingPlan.BindVars["v"] = "a" + orderingPlan = pgsq05AddFilter(orderingPlan, pgsq05FilterByPredicate(ordering)) + pgsq05Reject(t, orderingPlan, "GT over STRING kind in selector form") + + existing := pgsq05SelectorPredicate("Patient", "gender", "EXISTS", "", "", "v", true) + existingPlan := pgsq05Plan("Patient") + existingPlan.BindVars["v"] = "a" + existingPlan = pgsq05AddFilter(existingPlan, pgsq05FilterByPredicate(existing)) + pgsq05RejectAny(t, existingPlan, "selector EXISTS carrying a right value") + + inBind := pgsq05SelectorPredicate("Patient", "gender", "IN", spec.QuantifierAny, spec.FilterString, "v", true) + inPlan := pgsq05Plan("Patient") + inPlan.BindVars["v"] = "a" + inPlan = pgsq05AddFilter(inPlan, pgsq05FilterByPredicate(inBind)) + pgsq05Reject(t, inPlan, "selector IN with a non-list bind") + + contains := pgsq05SelectorPredicate("Patient", "gender", "CONTAINS_TEXT", spec.QuantifierAny, spec.FilterString, "v", true) + containsPlan := pgsq05Plan("Patient") + containsPlan.BindVars["v"] = "a" + containsPlan = pgsq05AddFilter(containsPlan, pgsq05FilterByPredicate(contains)) + pgsq05Reject(t, containsPlan, "selector CONTAINS_TEXT stays a gap") + + quantified := ir.PhysicalPredicate{ + Operator: "EQUALS", + Quantifier: spec.QuantifierAny, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "v"}, + } + quantifiedPlan := pgsq05Plan("Patient") + quantifiedPlan.BindVars["v"] = "x" + quantifiedPlan = pgsq05AddFilter(quantifiedPlan, pgsq05FilterByPredicate(quantified)) + pgsq05Reject(t, quantifiedPlan, "quantifier on the direct value form") + + correlated := pgsq05Plan("Patient") + correlated = pgsq05AddFilter(correlated, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "EXISTS", + Correlation: &ir.PhysicalCorrelation{ResourceType: "Patient"}, + })) + pgsq05RejectAny(t, correlated, "correlated predicate") + + unknown := pgsq05Plan("Patient") + unknown.BindVars["v"] = "x" + unknown = pgsq05AddFilter(unknown, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "LIKE", + Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "v"}, + })) + pgsq05RejectAny(t, unknown, "LIKE operator") +} + +func TestRenderDirectTypedColumnTemporalLiterals(t *testing.T) { + plan := pgsq05Plan("Patient") + plan.BindVars["at_value"] = "2024-03-15" + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "GTE", + ValueKind: spec.FilterDate, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"recorded_at"}}, + Right: &ir.PhysicalValue{BindKey: "at_value"}, + })) + rendered := renderOK(t, plan) + if got := pgsq05WhereClause(t, rendered); got != `r0."recorded_at" >= $5` { + t.Fatalf("recorded_at GTE clause = %s", got) + } + arg := rendered.Args[4] + if arg.Type != command.PostgreSQLParameterTimestamptz { + t.Fatalf("recorded_at literal arg type = %q", arg.Type) + } + if stamp, ok := arg.Value.(time.Time); !ok || !stamp.Equal(time.Date(2024, 3, 15, 0, 0, 0, 0, time.UTC)) { + t.Fatalf("recorded_at literal arg = %#v", arg.Value) + } + + for _, bad := range []struct { + kind spec.FilterValueKind + value any + }{ + {spec.FilterDate, "2024-03"}, + {spec.FilterDate, "15/03/2024"}, + {spec.FilterDateTime, "2024-03-15"}, + {spec.FilterDateTime, int64(5)}, + {spec.FilterString, "2024-03-15T10:20:30Z"}, + } { + plan := pgsq05Plan("Patient") + plan.BindVars["at_value"] = bad.value + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "GTE", + ValueKind: bad.kind, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"recorded_at"}}, + Right: &ir.PhysicalValue{BindKey: "at_value"}, + })) + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); err == nil { + t.Fatalf("recorded_at literal %v kind %q accepted, want rejection", bad.value, bad.kind) + } + } + + timePlan := pgsq05Plan("Patient") + timePlan.BindVars["at_value"] = time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC) + timePlan = pgsq05AddFilter(timePlan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: "root", Path: []string{"recorded_at"}}, + Right: &ir.PhysicalValue{BindKey: "at_value"}, + })) + if got := pgsq05WhereClause(t, renderOK(t, timePlan)); got != `r0."recorded_at" = $5` { + t.Fatalf("recorded_at time.Time clause = %s", got) + } +} + +func TestRenderDirectTypedColumnsRejectNontextKinds(t *testing.T) { + plan := pgsq05Plan("Patient") + plan.BindVars["v"] = int64(7) + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "EQUALS", + ValueKind: spec.FilterInteger, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + Right: &ir.PhysicalValue{BindKey: "v"}, + })) + pgsq05Reject(t, plan, "INTEGER kind over a text column") +} + +func TestRenderDirectTypedPayloadKinds(t *testing.T) { + cases := []struct { + name string + kind spec.FilterValueKind + value any + raw string + }{ + {"string", spec.FilterString, "active", `"active"`}, + {"code", spec.FilterCode, "final", `"final"`}, + {"boolean", spec.FilterBoolean, true, "true"}, + {"integer", spec.FilterInteger, int64(7), "7"}, + {"decimal", spec.FilterDecimal, 2.5, "2.5"}, + {"date equality stays raw text", spec.FilterDate, "2024-03-15", `"2024-03-15"`}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + plan := pgsq05Plan("Patient") + plan.BindVars["v"] = testCase.value + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "EQUALS", + ValueKind: testCase.kind, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"note"}}, + Right: &ir.PhysicalValue{BindKey: "v"}, + })) + rendered := renderOK(t, plan) + wantClause := `(r0."payload" #> $5::text[]) = ($6::jsonb)` + if got := pgsq05WhereClause(t, rendered); got != wantClause { + t.Fatalf("clause = %s, want %s", got, wantClause) + } + if got := rendered.Args[5]; got.Type != command.PostgreSQLParameterJSONB || + string(got.Value.(json.RawMessage)) != testCase.raw { + t.Fatalf("arg = %#v, want jsonb %s", got, testCase.raw) + } + nilPlan := pgsq05Plan("Patient") + nilPlan.BindVars["v"] = nil + nilPlan = pgsq05AddFilter(nilPlan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "EQUALS", + ValueKind: testCase.kind, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"note"}}, + Right: &ir.PhysicalValue{BindKey: "v"}, + })) + if _, err := RenderPhysicalPlan(nilPlan, mustBinding(t)); err == nil { + t.Fatalf("Go nil bind accepted for kind %q, want rejection", testCase.kind) + } + }) + } +} + +func TestRenderDirectTypedPayloadKindMismatchesRejected(t *testing.T) { + mismatches := []struct { + kind spec.FilterValueKind + value any + }{ + {spec.FilterString, int64(1)}, + {spec.FilterBoolean, "true"}, + {spec.FilterInteger, float64(1)}, + {spec.FilterDecimal, "1.5"}, + {spec.FilterDate, int64(1)}, + } + for _, mismatch := range mismatches { + plan := pgsq05Plan("Patient") + plan.BindVars["v"] = mismatch.value + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "EQUALS", + ValueKind: mismatch.kind, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"note"}}, + Right: &ir.PhysicalValue{BindKey: "v"}, + })) + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); err == nil { + t.Fatalf("kind %q with bind %#v accepted, want rejection", mismatch.kind, mismatch.value) + } + } +} + +func TestRenderDirectTypedPayloadRangeUsesTemporalGuards(t *testing.T) { + plan := pgsq05Plan("Patient") + plan.BindVars["v"] = "2024-03-15T10:20:30Z" + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "LT", + ValueKind: spec.FilterDateTime, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"note"}}, + Right: &ir.PhysicalValue{BindKey: "v"}, + })) + rendered := renderOK(t, plan) + clause := pgsq05WhereClause(t, rendered) + if !strings.Contains(clause, "::timestamptz") || !strings.Contains(clause, "END) < $6") { + t.Fatalf("DATE_TIME payload LT must compare guarded instants:\n%s", clause) + } + if rendered.Args[5].Type != command.PostgreSQLParameterTimestamptz { + t.Fatalf("DATE_TIME payload arg = %#v", rendered.Args[5]) + } + negated := pgsq05Plan("Patient") + negated.BindVars["v"] = "2024-03-15T10:20:30Z" + negated = pgsq05AddFilter(negated, pgsq05FilterByExpression(ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalNotPredicate, + Children: []ir.PhysicalPredicateExpression{pgsq05ComparisonLeaf(ir.PhysicalPredicate{ + Operator: "LT", + ValueKind: spec.FilterDateTime, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"note"}}, + Right: &ir.PhysicalValue{BindKey: "v"}, + })}, + })) + negatedRendered := renderOK(t, negated) + if !strings.HasPrefix(pgsq05WhereClause(t, negatedRendered), "COALESCE(NOT (") { + t.Fatalf("NOT over a three-valued temporal comparison must be guarded:\n%s", negatedRendered.SQL) + } + + stringOrder := pgsq05Plan("Patient") + stringOrder.BindVars["v"] = "b" + stringOrder = pgsq05AddFilter(stringOrder, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "GT", + ValueKind: spec.FilterString, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"note"}}, + Right: &ir.PhysicalValue{BindKey: "v"}, + })) + pgsq05Reject(t, stringOrder, "GT over STRING kind") +} + +func pgsq05PayloadLeaf(bindKey string) ir.PhysicalPredicate { + return ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: bindKey}, + } +} + +func pgsq05ColumnLeaf(bindKey string) ir.PhysicalPredicate { + return ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + Right: &ir.PhysicalValue{BindKey: bindKey}, + } +} + +func TestRenderOrPredicateGoldenAndNullGuardPlacement(t *testing.T) { + plan := pgsq05Plan("Patient") + plan.BindVars["a"] = "x" + plan.BindVars["b"] = "y" + plan = pgsq05AddFilter(plan, pgsq05FilterByExpression(ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalAnyPredicate, + Children: []ir.PhysicalPredicateExpression{ + pgsq05ComparisonLeaf(pgsq05PayloadLeaf("a")), + pgsq05ComparisonLeaf(pgsq05PayloadLeaf("b")), + }, + })) + rendered := renderOK(t, plan) + wantClause := `((r0."payload" #> $5::text[]) = ($6::jsonb)) OR ((r0."payload" #> $5::text[]) = ($7::jsonb))` + if got := pgsq05WhereClause(t, rendered); got != wantClause { + t.Fatalf("OR clause =\n%s\nwant\n%s", got, wantClause) + } + + twoValued := pgsq05Plan("Patient") + twoValued.BindVars["a"] = "x" + twoValued.BindVars["b"] = "y" + twoValued = pgsq05AddFilter(twoValued, pgsq05FilterByExpression(ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalNotPredicate, + Children: []ir.PhysicalPredicateExpression{{ + Kind: ir.PhysicalAnyPredicate, + Children: []ir.PhysicalPredicateExpression{ + pgsq05ComparisonLeaf(pgsq05ColumnLeaf("a")), + pgsq05ComparisonLeaf(pgsq05ColumnLeaf("b")), + }, + }}, + })) + if got := pgsq05WhereClause(t, renderOK(t, twoValued)); !strings.HasPrefix(got, "NOT ((") { + t.Fatalf("NOT over a two-valued OR must stay plain:\n%s", got) + } + + mixed := pgsq05Plan("Patient") + mixed.BindVars["a"] = "x" + mixed.BindVars["b"] = "y" + mixed = pgsq05AddFilter(mixed, pgsq05FilterByExpression(ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalNotPredicate, + Children: []ir.PhysicalPredicateExpression{{ + Kind: ir.PhysicalAnyPredicate, + Children: []ir.PhysicalPredicateExpression{ + pgsq05ComparisonLeaf(pgsq05ColumnLeaf("a")), + pgsq05ComparisonLeaf(pgsq05PayloadLeaf("b")), + }, + }}, + })) + if got := pgsq05WhereClause(t, renderOK(t, mixed)); !strings.HasPrefix(got, "COALESCE(NOT (") { + t.Fatalf("NOT over a mixed OR must be guarded:\n%s", got) + } +} + +func TestRenderNestedTruthTableShapes(t *testing.T) { + plan := pgsq05Plan("Patient") + plan.BindVars["a"] = "x" + plan.BindVars["b"] = "y" + plan = pgsq05AddFilter(plan, pgsq05FilterByExpression(ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalNotPredicate, + Children: []ir.PhysicalPredicateExpression{{ + Kind: ir.PhysicalAllPredicate, + Children: []ir.PhysicalPredicateExpression{ + pgsq05ComparisonLeaf(pgsq05PayloadLeaf("a")), + pgsq05ComparisonLeaf(pgsq05ColumnLeaf("b")), + }, + }}, + })) + got := pgsq05WhereClause(t, renderOK(t, plan)) + want := `COALESCE(NOT (((r0."payload" #> $5::text[]) = ($6::jsonb)) AND (r0."logical_id" = $7)), true)` + if got != want { + t.Fatalf("nested clause =\n%s\nwant\n%s", got, want) + } + + deep := pgsq05Plan("Patient") + deep.BindVars["a"] = "x" + deep.BindVars["b"] = "y" + deep.BindVars["c"] = "z" + deep = pgsq05AddFilter(deep, pgsq05FilterByExpression(ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalAllPredicate, + Children: []ir.PhysicalPredicateExpression{ + { + Kind: ir.PhysicalAnyPredicate, + Children: []ir.PhysicalPredicateExpression{ + pgsq05ComparisonLeaf(pgsq05ColumnLeaf("a")), + pgsq05ComparisonLeaf(pgsq05ColumnLeaf("b")), + }, + }, + {Kind: ir.PhysicalNotPredicate, Children: []ir.PhysicalPredicateExpression{ + pgsq05ComparisonLeaf(pgsq05PayloadLeaf("c")), + }}, + }, + })) + deepRendered := renderOK(t, deep) + clause := pgsq05WhereClause(t, deepRendered) + wantDeep := `((r0."logical_id" = $5) OR (r0."logical_id" = $6))` + + ` AND (COALESCE(NOT ((r0."payload" #> $7::text[]) = ($8::jsonb)), true))` + if clause != wantDeep { + t.Fatalf("deep clause =\n%s\nwant\n%s", clause, wantDeep) + } +} + +func TestRenderBooleanConnectiveShapeRejections(t *testing.T) { + emptyAny := pgsq05Plan("Patient") + emptyAny = pgsq05AddFilter(emptyAny, pgsq05FilterByExpression(ir.PhysicalPredicateExpression{Kind: ir.PhysicalAnyPredicate})) + pgsq05RejectAny(t, emptyAny, "ANY without children") + + emptyAll := pgsq05Plan("Patient") + emptyAll = pgsq05AddFilter(emptyAll, pgsq05FilterByExpression(ir.PhysicalPredicateExpression{Kind: ir.PhysicalAllPredicate})) + pgsq05RejectAny(t, emptyAll, "ALL without children") + + twoChildNot := pgsq05Plan("Patient") + twoChildNot.BindVars["a"] = "x" + twoChildNot.BindVars["b"] = "y" + twoChildNot = pgsq05AddFilter(twoChildNot, pgsq05FilterByExpression(ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalNotPredicate, + Children: []ir.PhysicalPredicateExpression{ + pgsq05ComparisonLeaf(pgsq05ColumnLeaf("a")), + pgsq05ComparisonLeaf(pgsq05ColumnLeaf("b")), + }, + })) + pgsq05RejectAny(t, twoChildNot, "NOT with two children") + + kindlessComparison := pgsq05Plan("Patient") + kindlessComparison = pgsq05AddFilter(kindlessComparison, pgsq05FilterByExpression(ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalComparisonPredicate, + })) + pgsq05RejectAny(t, kindlessComparison, "COMPARISON without payload") +} + +func TestRenderWidenedSurfaceIsDeterministicAndLeakFree(t *testing.T) { + build := func() ir.PhysicalPlan { + plan := pgsq05Plan("Patient") + plan.BindVars["gender_value"] = "female" + plan.BindVars["birth_value"] = "2024-03-15" + plan.BindVars["id_value"] = "p-1" + plan = pgsq05AddFilter(plan, pgsq05FilterByExpression(ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalAnyPredicate, + Children: []ir.PhysicalPredicateExpression{ + pgsq05ComparisonLeaf(pgsq05SelectorPredicate( + "Patient", "gender", "EQUALS", spec.QuantifierAny, spec.FilterCode, "gender_value", true)), + {Kind: ir.PhysicalNotPredicate, Children: []ir.PhysicalPredicateExpression{ + pgsq05ComparisonLeaf(pgsq05SelectorPredicate( + "Patient", "birthDate", "GTE", spec.QuantifierAny, spec.FilterDate, "birth_value", true)), + }}, + pgsq05ComparisonLeaf(pgsq05ColumnLeaf("id_value")), + }, + })) + return plan + } + first := renderOK(t, build()) + second := renderOK(t, build()) + if first.SQL != second.SQL { + t.Fatalf("render is not deterministic:\n%s\n%s", first.SQL, second.SQL) + } + if !reflect.DeepEqual(first.Args, second.Args) { + t.Fatalf("arguments are not deterministic:\n%#v\n%#v", first.Args, second.Args) + } + for _, leaked := range []string{"female", "2024-03-15", "p-1", pgsq05Project} { + if strings.Contains(first.SQL, leaked) { + t.Fatalf("plan data %q leaked into SQL: %s", leaked, first.SQL) + } + } +} + +func TestRenderNumericOrderingRequiresStoredNumber(t *testing.T) { + selectorPlan := pgsq05Plan("Patient") + selectorPlan.BindVars["score_value"] = int64(5) + selectorPlan = pgsq05AddFilter(selectorPlan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "gender", "GTE", spec.QuantifierAny, spec.FilterInteger, "score_value", true))) + selectorClause := pgsq05WhereClause(t, renderOK(t, selectorPlan)) + wantSelector := `COALESCE(jsonb_typeof(r0."payload" #> $5::text[]), 'null') <> 'null' AND ` + + `COALESCE(jsonb_typeof(r0."payload" #> $5::text[]) = 'number' AND ` + + `(r0."payload" #> $5::text[]) >= ($6::jsonb), false)` + if !strings.Contains(selectorClause, wantSelector) { + t.Fatalf("numeric selector ordering clause =\n%s\nwant substring\n%s", selectorClause, wantSelector) + } + + directPlan := pgsq05Plan("Patient") + directPlan.BindVars["score_value"] = 2.5 + directPlan = pgsq05AddFilter(directPlan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "LTE", + ValueKind: spec.FilterDecimal, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"payload", "score"}}, + Right: &ir.PhysicalValue{BindKey: "score_value"}, + })) + directClause := pgsq05WhereClause(t, renderOK(t, directPlan)) + wantDirect := `(jsonb_typeof((r0."payload" #> $5::text[])) = 'number' AND ` + + `(r0."payload" #> $5::text[]) <= ($6::jsonb))` + if !strings.Contains(directClause, wantDirect) { + t.Fatalf("numeric direct ordering clause =\n%s\nwant substring\n%s", directClause, wantDirect) + } +} + +func TestRenderRejectsKindMismatchedSelectorInList(t *testing.T) { + inPlan := func(kind spec.FilterValueKind, bind any) ir.PhysicalPlan { + plan := pgsq05Plan("Patient") + plan.BindVars["list"] = bind + return pgsq05AddFilter(plan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "gender", "IN", spec.QuantifierAny, kind, "list", true))) + } + pgsq05Reject(t, inPlan(spec.FilterInteger, []string{"1"}), "INTEGER kind IN with a []string bind") + pgsq05Reject(t, inPlan(spec.FilterBoolean, []any{"true"}), "BOOLEAN kind IN with string elements") + pgsq05Reject(t, inPlan(spec.FilterDecimal, []any{int64(1)}), "DECIMAL kind IN with int64 elements") + pgsq05Reject(t, inPlan(spec.FilterString, []any{int64(1)}), "STRING kind IN with int64 elements") + + plan := inPlan(spec.FilterInteger, []any{int64(1), int64(2)}) + rendered := renderOK(t, plan) + if got := rendered.Args[5]; got.Type != command.PostgreSQLParameterJSONB || + string(got.Value.(json.RawMessage)) != "[1,2]" { + t.Fatalf("INTEGER IN arg = %#v", got) + } +} + +func TestRenderRejectsStringKindOrderingInAllThreeForms(t *testing.T) { + columnPlan := pgsq05Plan("Patient") + columnPlan.BindVars["v"] = "Patient" + columnPlan = pgsq05AddFilter(columnPlan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "GT", + ValueKind: spec.FilterString, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"resource_type"}}, + Right: &ir.PhysicalValue{BindKey: "v"}, + })) + pgsq05Reject(t, columnPlan, "GT over STRING kind on a column") + + directPlan := pgsq05Plan("Patient") + directPlan.BindVars["v"] = "a" + directPlan = pgsq05AddFilter(directPlan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "GTE", + ValueKind: spec.FilterString, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"payload", "status"}}, + Right: &ir.PhysicalValue{BindKey: "v"}, + })) + pgsq05Reject(t, directPlan, "GTE over STRING kind on a payload path") + + selectorPlan := pgsq05Plan("Patient") + selectorPlan.BindVars["v"] = "a" + selectorPlan = pgsq05AddFilter(selectorPlan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "gender", "LT", spec.QuantifierAny, spec.FilterString, "v", true))) + pgsq05Reject(t, selectorPlan, "LT over STRING kind in selector form") +} + +func TestRenderRejectsCommaDecimalLiteralsInAllThreeForms(t *testing.T) { + for _, value := range []string{ + "2024-03-15T10:20:30,5Z", + "2024-03-15T10:20:30,5+02:00", + "2024-03-15T10:20:30,5", + } { + columnPlan := pgsq05Plan("Patient") + columnPlan.BindVars["at_value"] = value + columnPlan = pgsq05AddFilter(columnPlan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "GTE", + ValueKind: spec.FilterDateTime, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"recorded_at"}}, + Right: &ir.PhysicalValue{BindKey: "at_value"}, + })) + pgsq05Reject(t, columnPlan, "DATE_TIME column comparison against comma decimal "+value) + + directPlan := pgsq05Plan("Patient") + directPlan.BindVars["at_value"] = value + directPlan = pgsq05AddFilter(directPlan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "LT", + ValueKind: spec.FilterDateTime, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"payload", "meta", "lastUpdated"}}, + Right: &ir.PhysicalValue{BindKey: "at_value"}, + })) + pgsq05Reject(t, directPlan, "DATE_TIME payload comparison against comma decimal "+value) + + selectorPlan := pgsq05Plan("Patient") + selectorPlan.BindVars["at_value"] = value + selectorPlan = pgsq05AddFilter(selectorPlan, pgsq05FilterByPredicate(pgsq05SelectorPredicate( + "Patient", "gender", "LTE", spec.QuantifierAll, spec.FilterDateTime, "at_value", true))) + pgsq05Reject(t, selectorPlan, "DATE_TIME selector comparison against comma decimal "+value) + } +} + +func TestRenderNormalizesFixedZoneTimeTimeBind(t *testing.T) { + bind := time.Date(2024, 3, 14, 18, 20, 30, 0, time.FixedZone("Etc/GMT-16", 16*3600)) + plan := pgsq05Plan("Patient") + plan.BindVars["at_value"] = bind + plan = pgsq05AddFilter(plan, pgsq05FilterByPredicate(ir.PhysicalPredicate{ + Operator: "GTE", + ValueKind: spec.FilterDateTime, + Left: ir.PhysicalValue{Variable: "root", Path: []string{"recorded_at"}}, + Right: &ir.PhysicalValue{BindKey: "at_value"}, + })) + rendered := renderOK(t, plan) + arg := rendered.Args[4] + if arg.Type != command.PostgreSQLParameterTimestamptz { + t.Fatalf("bind arg type = %q, want timestamptz", arg.Type) + } + stamp, ok := arg.Value.(time.Time) + if !ok { + t.Fatalf("bind arg value = %#v, want time.Time", arg.Value) + } + if !stamp.Equal(bind) { + t.Fatalf("bind changed the instant: %v vs %v", stamp, bind) + } + if stamp.Location() != time.UTC { + t.Fatalf("bind location = %v, want UTC", stamp.Location()) + } + if got, want := stamp.Format(time.RFC3339Nano), bind.UTC().Format(time.RFC3339Nano); got != want { + t.Fatalf("bind text encoding = %q, want %q", got, want) + } +} + +func TestRenderTemporalGuardPatternUnitArms(t *testing.T) { + date := regexp.MustCompile(temporalStoredDatePattern) + instant := regexp.MustCompile(temporalStoredInstantPattern) + noOffset := regexp.MustCompile(temporalStoredNoOffsetPattern) + + type arm struct { + value string + date bool + inst bool + noOff bool + } + arms := []arm{ + arm{"2000-02-29", true, false, false}, {"2400-02-29", true, false, false}, + arm{"1600-02-29", true, false, false}, {"1900-02-29", false, false, false}, + arm{"2100-02-29", false, false, false}, {"0000-02-29", false, false, false}, + arm{"0000-01-01", false, false, false}, + arm{"2024-02-29", true, false, false}, {"2023-02-29", false, false, false}, + arm{"2024-02-28", true, false, false}, {"2024-02-30", false, false, false}, + } + maxDay := map[string]int{"01": 31, "02": 28, "03": 31, "04": 30, "05": 31, "06": 30, + "07": 31, "08": 31, "09": 30, "10": 31, "11": 30, "12": 31} + for month, max := range maxDay { + arms = append(arms, + arm{"2023-" + month + "-01", true, false, false}, + arm{fmt.Sprintf("2023-%s-%02d", month, max), true, false, false}, + arm{fmt.Sprintf("2023-%s-%02d", month, max+1), false, false, false}, + arm{"2023-" + month + "-00", false, false, false}, + arm{"2023-" + month + "-32", false, false, false}, + ) + } + arms = append(arms, + arm{"2024-03-15T23:59:59Z", false, true, false}, + arm{"2024-03-15T24:00:00Z", false, false, false}, + arm{"2024-03-15T23:60:00Z", false, false, false}, + arm{"2024-03-15T23:59:60Z", false, false, false}, + arm{"2024-03-15T10:20Z", false, true, false}, + arm{"2024-03-15T10:20:30+00:00", false, true, false}, + arm{"2024-03-15T10:20:30+14:00", false, true, false}, + arm{"2024-03-15T10:20:30+15:59", false, true, false}, + arm{"2024-03-15T10:20:30-15:59", false, true, false}, + arm{"2024-03-15T10:20:30+16:00", false, false, false}, + arm{"2024-03-15T10:20:30-16:00", false, false, false}, + arm{"2024-03-15T10:20:30+23:59", false, false, false}, + arm{"2024-03-15T10:20:30-23:59", false, false, false}, + arm{"2024-03-15T10:20:30+15:60", false, false, false}, + arm{"2024-03-15T10:20:30.5Z", false, true, false}, + arm{"2024-03-15T10:20:30,5Z", false, false, false}, + arm{"2024-03-15T10:20:30,5", false, false, false}, + arm{"2024-03-15T10:20:30.5+02:00", false, true, false}, + arm{"2024-03-15T10:20:30,5+02:00", false, false, false}, + arm{"2024-03-15T10:20:30", false, false, true}, + arm{"2024-03-15T10:20", false, false, true}, + arm{"2024-03-15T24:00:00", false, false, false}, + arm{"2024", false, false, false}, + arm{"not-a-date", false, false, false}, + arm{"2024-03-15 10:20:30Z", false, false, false}, + ) + + for _, a := range arms { + if got := date.MatchString(a.value); got != a.date { + t.Errorf("date pattern %q = %v, want %v", a.value, got, a.date) + } + if got := instant.MatchString(a.value); got != a.inst { + t.Errorf("instant pattern %q = %v, want %v", a.value, got, a.inst) + } + if got := noOffset.MatchString(a.value); got != a.noOff { + t.Errorf("no-offset pattern %q = %v, want %v", a.value, got, a.noOff) + } + } +} diff --git a/internal/dataframe/compiler/render/postgres/render.go b/internal/dataframe/compiler/render/postgres/render.go new file mode 100644 index 000000000..8909bac3b --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/render.go @@ -0,0 +1,532 @@ +package postgres + +import ( + "encoding/json" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +var ErrUnsupportedPostgresPlan = errors.New("postgres renderer: unsupported physical plan") + +var ErrInvalidPostgresPlanScope = errors.New("postgres renderer: invalid plan scope") + +func canonicalKeyColumn(path string) string { + switch path { + case "_key", "id": + return "logical_id" + default: + return path + } +} + +var resourceFilterColumns = map[string]bool{ + "project": true, + "dataset_generation": true, + "resource_type": true, + "logical_id": true, + "auth_resource_path": true, + "recorded_at": true, +} + +func RenderPhysicalPlan(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + if err := storage.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if cellTraceTerminalMentions(plan) { + return renderCellTracePhysicalPlan(plan, storage) + } + if graphPlanMentionsPathOperations(plan) { + return renderGraphPhysicalPlan(plan, storage) + } + if rowLineageTerminalMentions(plan) { + return renderRowLineagePhysicalPlan(plan, storage) + } + if stageSequenceCompositionMentions(plan) { + return renderStageSequencePhysicalPlan(plan, storage) + } + if codedGroupPlanMentionsStage(plan) { + return renderCodedGroupPhysicalPlan(plan, storage) + } + if stageSequenceRelatedOccurrenceMentions(plan) { + return renderStageSequencePhysicalPlan(plan, storage) + } + if groupPlanMentionsStage(plan) { + return renderGroupPhysicalPlan(plan, storage) + } + if groupedPivotStagePlanMentionsSurface(plan) { + return renderGroupedCodedPivotPhysicalPlan(plan, storage) + } + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalUnnestOp { + if err := validateUnnestPlanSurface(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + break + } + } + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalUnpivotOp { + if err := validateUnpivotPlanSurface(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + break + } + } + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalGroupedPivotOp { + return renderGroupedPivotPhysicalPlan(plan, storage) + } + } + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalTableShapeExclusionReturnOp { + return renderTableShapeExclusionPhysicalPlan(plan, storage) + } + } + if plan.StageSequence != nil { + return renderStageSequencePhysicalPlan(plan, storage) + } + if err := plan.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("validate physical plan: %w", err) + } + if plan.Engine != "" && plan.Engine != ir.PhysicalEngineAQL { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: engine %q has no PostgreSQL source; the source graph IR is backend-neutral and ClickHouse plans are not PostgreSQL-rendered", + ErrUnsupportedPostgresPlan, string(plan.Engine)) + } + if plan.ClickHouseCombine != nil || plan.ClickHousePrefix != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: ClickHouse combine/prefix payloads have no PostgreSQL rendering", ErrUnsupportedPostgresPlan) + } + if plan.PreviewSourceWindowByRootID { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: preview source windowing is not yet implemented for PostgreSQL", ErrUnsupportedPostgresPlan) + } + if len(plan.DeferredExpressionLets) != 0 { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: completed plans must not carry deferred expression lets", ErrUnsupportedPostgresPlan) + } + + renderer := &planRenderer{ + storage: storage, + rows: map[string]rowScope{}, + parameters: map[string]int{}, + canonicalAuthLets: map[string]string{}, + } + if err := renderer.loadScope(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := renderer.renderOperations(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + sql := renderer.assemble() + return command.NewRenderedPostgresPhysicalPlan(sql, renderer.args, renderer.outputColumns) +} + +type rowScope struct { + alias string + resourceType string + derivedValue string + derivedJSON bool + optional bool +} + +type planRenderer struct { + storage ir.PostgresStorageBinding + args []command.PostgreSQLParameter + parameters map[string]int + rows map[string]rowScope + canonicalAuthLets map[string]string + + stageRows map[string]stageRowScope + + scope planScope + rootAlias string + edgeAliases []string + traversalSeq int + where []string + joins []string + selects []string + outputColumns []string + + window executionWindow + + laterals []string + lateralAliases []string + unnestSeq int + + unpivotSeq int + unpivotOutputs map[string]map[string]string + + membership *PostgresPopulationRootBinding + + mapping *populationMappingRender + + rootVariable string + sliceSeq int + setBodies map[string]correlatedSetBody + setProjections map[string][]ir.PhysicalSetProjectionField +} + +func (r *planRenderer) addParameter(parameter command.PostgreSQLParameter, cacheKey string) int { + if cacheKey != "" { + if index, ok := r.parameters[cacheKey]; ok { + return index + } + } + r.args = append(r.args, parameter) + index := len(r.args) + if cacheKey != "" { + r.parameters[cacheKey] = index + } + return index +} + +func (r *planRenderer) textParameter(value, cacheKey string) int { + return r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: value}, cacheKey) +} + +func (r *planRenderer) jsonbParameter(raw json.RawMessage) int { + return r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterJSONB, Value: append(json.RawMessage(nil), raw...)}, "") +} + +func (r *planRenderer) renderOperations(plan ir.PhysicalPlan) error { + if len(plan.Operations) == 0 { + return fmt.Errorf("%w: plan has no operations", ErrUnsupportedPostgresPlan) + } + phase := 0 + sawSort, sawLimit := false, false + for index := 0; index < len(plan.Operations); index++ { + operation := plan.Operations[index] + last := index == len(plan.Operations)-1 + switch operation.Kind { + case ir.PhysicalRootScanOp: + if phase != 0 || index != 0 { + return fmt.Errorf("%w: exactly one leading ROOT_SCAN is required", ErrUnsupportedPostgresPlan) + } + if err := r.renderRootScan(plan, *operation.RootScan); err != nil { + return err + } + case ir.PhysicalFilterOp: + if phase != 0 { + return fmt.Errorf("%w: filters must precede the execution window", ErrUnsupportedPostgresPlan) + } + canonical, err := r.acceptCanonicalScopeFilter(operation.Filter) + if err != nil { + return err + } + if canonical { + continue + } + clause, err := r.renderFilter(plan, *operation.Filter) + if err != nil { + return err + } + r.where = append(r.where, clause) + case ir.PhysicalDerivedLetOp: + if phase != 0 { + return fmt.Errorf("%w: derived lets must precede the execution window", ErrUnsupportedPostgresPlan) + } + canonical, err := r.acceptCanonicalAuthLet(operation.DerivedLet) + if err != nil { + return err + } + if canonical { + continue + } + return fmt.Errorf("%w: operation %q is not part of the v1 PostgreSQL render surface", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + case ir.PhysicalTraversalOp: + if phase != 0 { + if operation.Traversal == nil { + return fmt.Errorf("%w: operation %d: traversal payload is missing", ErrUnsupportedPostgresPlan, index) + } + if handled, next, acceptErr := r.acceptPostWindowTraversal(plan, index); acceptErr != nil { + return acceptErr + } else if handled { + index = next - 1 + continue + } + return fmt.Errorf("%w: traversals must precede the execution window", ErrUnsupportedPostgresPlan) + } + if err := r.renderTraversal(plan, *operation.Traversal, false); err != nil { + return err + } + case ir.PhysicalUnnestOp: + if phase != 0 { + return fmt.Errorf("%w: unnest must precede the execution window", ErrUnsupportedPostgresPlan) + } + if operation.Unnest == nil { + return fmt.Errorf("%w: operation %d: unnest payload is missing", ErrUnsupportedPostgresPlan, index) + } + if err := r.renderUnnest(plan, *operation.Unnest); err != nil { + return err + } + case ir.PhysicalSetOp: + if phase != 0 { + if operation.Set == nil { + return fmt.Errorf("%w: operation %d: set payload is missing", ErrUnsupportedPostgresPlan, index) + } + if !acceptPostWindowSet(plan, *operation.Set) { + return fmt.Errorf("%w: sets must precede the execution window", ErrUnsupportedPostgresPlan) + } + } + if operation.Set == nil { + return fmt.Errorf("%w: operation %d: set payload is missing", ErrUnsupportedPostgresPlan, index) + } + if err := r.renderSet(plan, *operation.Set); err != nil { + return err + } + case ir.PhysicalUnpivotOp: + if phase != 0 { + return fmt.Errorf("%w: unpivot must precede the execution window", ErrUnsupportedPostgresPlan) + } + if operation.Unpivot == nil { + return fmt.Errorf("%w: operation %d: unpivot payload is missing", ErrUnsupportedPostgresPlan, index) + } + if err := r.renderUnpivot(plan, *operation.Unpivot); err != nil { + return err + } + case ir.PhysicalSortOp: + if sawSort || phase > 1 || (phase == 1 && sawLimit) { + return fmt.Errorf("%w: at most one SORT, before LIMIT", ErrUnsupportedPostgresPlan) + } + if err := r.renderWindowSort(plan, *operation.Sort); err != nil { + return err + } + sawSort, phase = true, 1 + case ir.PhysicalLimitOp: + if sawLimit || phase > 1 { + return fmt.Errorf("%w: at most one LIMIT", ErrUnsupportedPostgresPlan) + } + if err := r.renderWindowLimit(plan, *operation.Limit); err != nil { + return err + } + sawLimit, phase = true, 1 + case ir.PhysicalReturnOp: + if !last || phase > 1 { + return fmt.Errorf("%w: RETURN must be the final operation", ErrUnsupportedPostgresPlan) + } + if err := r.renderReturn(plan, *operation.Return); err != nil { + return err + } + phase = 2 + case ir.PhysicalPopulationMappingReturnOp: + if r.mapping == nil { + return fmt.Errorf("%w: operation %q is not part of the v1 PostgreSQL render surface", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + if !last || phase > 1 { + return fmt.Errorf("%w: POPULATION_MAPPING_RETURN must be the single final operation", ErrUnsupportedPostgresPlan) + } + if operation.PopulationMappingReturn == nil { + return fmt.Errorf("%w: operation %d: population-mapping payload is missing", ErrUnsupportedPostgresPlan, index) + } + if err := r.renderPopulationMappingTerminal(plan, *operation.PopulationMappingReturn); err != nil { + return err + } + phase = 2 + default: + return fmt.Errorf("%w: operation %q is not part of the v1 PostgreSQL render surface", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + if phase != 2 { + return fmt.Errorf("%w: plan requires a RETURN terminal", ErrUnsupportedPostgresPlan) + } + return nil +} + +func (r *planRenderer) renderRootScan(plan ir.PhysicalPlan, scan ir.PhysicalRootScan) error { + if scan.Population != nil || scan.CohortSource != nil { + return r.renderRootSources(plan, scan) + } + if scan.PageCandidateSeed != nil { + return fmt.Errorf("%w: population, cohort, and candidate-seed roots are not yet implemented for PostgreSQL", + ErrUnsupportedPostgresPlan) + } + resourceType, ok := plan.BindVars[scan.CollectionBindKey].(string) + if !ok || strings.TrimSpace(resourceType) == "" { + return fmt.Errorf("%w: root collection bind %q must name one resource type as a non-empty string", + ErrUnsupportedPostgresPlan, scan.CollectionBindKey) + } + r.rootAlias = "r0" + r.rootVariable = scan.Variable + r.rows[scan.Variable] = rowScope{alias: r.rootAlias, resourceType: resourceType} + r.where = append(r.where, r.scopedResourcePredicate(r.rootAlias, + r.textParameter(resourceType, "text:"+resourceType))) + return nil +} + +func (r *planRenderer) resolveValue(value ir.PhysicalValue) (string, error) { + if scope, ok := r.stageRows[value.Variable]; ok { + return r.resolveStageRowValue(scope, value) + } + if columns, ok := r.unpivotOutputs[value.Variable]; ok { + return r.resolveUnpivotOutputValue(columns, value) + } + if scope, ok := r.rows[value.Variable]; ok && scope.derivedValue != "" { + return r.resolveDerivedValue(value, scope) + } + return r.resolveValueInRows(r.rows, value) +} + +func (r *planRenderer) rejectOptionalRowReferences(filter ir.PhysicalFilter) error { + if filter.Expression != nil { + return rejectOptionalInPredicate(*filter.Expression, r.rows) + } + return rejectOptionalInPredicate(ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalComparisonPredicate, + Comparison: &filter.Predicate, + }, r.rows) +} + +func rejectOptionalInPredicate(predicate ir.PhysicalPredicateExpression, rows map[string]rowScope) error { + switch predicate.Kind { + case ir.PhysicalComparisonPredicate: + if predicate.Comparison == nil { + return nil + } + if predicate.Comparison.LeftExpression != nil && + predicate.Comparison.LeftExpression.Extract != nil { + if scope, ok := rows[predicate.Comparison.LeftExpression.Extract.Source.Variable]; ok && scope.optional { + return optionalRowRefError(predicate.Comparison.LeftExpression.Extract.Source.Variable) + } + } + if predicate.Comparison.Left.BindKey == "" && predicate.Comparison.Left.Variable != "" { + if scope, ok := rows[predicate.Comparison.Left.Variable]; ok && scope.optional { + return optionalRowRefError(predicate.Comparison.Left.Variable) + } + } + return nil + case ir.PhysicalAllPredicate, ir.PhysicalAnyPredicate: + for _, child := range predicate.Children { + if err := rejectOptionalInPredicate(child, rows); err != nil { + return err + } + } + return nil + case ir.PhysicalNotPredicate: + for _, child := range predicate.Children { + if err := rejectOptionalInPredicate(child, rows); err != nil { + return err + } + } + return nil + default: + return nil + } +} + +func optionalRowRefError(variable string) error { + return fmt.Errorf("%w: FILTER references %q, a row materialized by an OPTIONAL traversal LEFT JOIN LATERAL; a WHERE conjunct over null-extended child rows would re-add the inner-join exclusion the AQL LET-set never applies (render/aql/navigation_render.go renderTraversalSet). Express required matching as the typed EXISTS subplan FILTER form", + ErrUnsupportedPostgresPlan, variable) +} + +var payloadPathSegmentPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +func (r *planRenderer) renderReturn(plan ir.PhysicalPlan, ret ir.PhysicalReturn) error { + if r.unpivotOutputs != nil { + return r.renderUnpivotReturn(ret) + } + if len(ret.Projections) == 0 { + return fmt.Errorf("%w: RETURN requires at least one projection", ErrUnsupportedPostgresPlan) + } + classified, err := classifyExtractionProjections(ret.Projections, func(variable string) (string, bool) { + scope, ok := r.rows[variable] + return scope.resourceType, ok + }) + if err != nil { + return err + } + for _, projection := range classified { + var expression string + switch projection.kind { + case extractionDirectValue, extractionValueExpression: + value := projection.projection.Value + if projection.kind == extractionValueExpression { + value = *projection.projection.Expression.Value + } + resolved, err := r.resolveValue(value) + if err != nil { + return fmt.Errorf("projection %q: %w", projection.projection.Name, err) + } + expression = resolved + if isPayloadExpression(expression) { + expression = "(" + expression + ")" + } + case extractionSelector: + scope := r.rows[projection.expression.Extract.Source.Variable] + value, err := r.scalarExtractValueExpression(scope.alias, projection.path, projection.expression.NullBehavior) + if err != nil { + return fmt.Errorf("projection %q: %w", projection.projection.Name, err) + } + expression = value + case extractionPresenceSidecar: + scope := r.rows[projection.presence.Source.Variable] + presence, err := r.propertyPresenceExpression(scope.alias, projection.presencePath) + if err != nil { + return fmt.Errorf("presence projection %q: %w", projection.projection.Name, err) + } + expression = presence + case extractionCallExpression: + value, err := r.renderCallProjection(plan, projection.projection) + if err != nil { + return err + } + expression = value + case extractionObject: + value, err := r.renderObjectValue(plan, r.rows, *projection.projection.Expression, 1) + if err != nil { + return fmt.Errorf("projection %q: %w", projection.projection.Name, err) + } + expression = value + case extractionSlice: + value, err := r.renderSliceValue(plan, r.rows, *projection.projection.Expression, 1) + if err != nil { + return fmt.Errorf("projection %q: %w", projection.projection.Name, err) + } + expression = value + case extractionUnitNormalized: + value, err := r.renderUnitNormalizedProjection(projection) + if err != nil { + return err + } + expression = value + case extractionAggregate: + value, err := r.renderSetAggregateExpression(plan, projection.projection) + if err != nil { + return fmt.Errorf("%w: %s expression has no registered PostgreSQL rendering for this shape: %s", + ErrUnsupportedPostgresPlan, ir.PhysicalAggregateExpression, err.Error()) + } + expression = value + } + r.selects = append(r.selects, expression+` AS "c`+strconv.Itoa(len(r.selects))+`"`) + r.outputColumns = append(r.outputColumns, projection.projection.Name) + } + return nil +} + +func (r *planRenderer) assemble() string { + var builder strings.Builder + builder.WriteString("SELECT ") + builder.WriteString(strings.Join(r.selects, ", ")) + builder.WriteString(" FROM " + r.storage.Resource.Identifier().SQLText() + " AS " + r.rootAlias) + for _, join := range r.joins { + builder.WriteString(" " + join) + } + for _, lateral := range r.laterals { + builder.WriteString(" " + lateral) + } + if len(r.where) > 0 { + builder.WriteString(" WHERE " + strings.Join(r.where, " AND ")) + } + builder.WriteString(" ORDER BY " + strings.Join(r.windowOrder(), ", ")) + builder.WriteString(r.window.limitClause) + return builder.String() +} diff --git a/internal/dataframe/compiler/render/postgres/render_pg18_test.go b/internal/dataframe/compiler/render/postgres/render_pg18_test.go new file mode 100644 index 000000000..5007f65a9 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/render_pg18_test.go @@ -0,0 +1,855 @@ +package postgres + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + control "github.com/calypr/loom/internal/backends/postgres/control" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" +) + +var pgir02ResourceNamePattern = regexp.MustCompile(`^pgir02_it_[0-9]+_[0-9a-f]+$`) + +func testPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 renderer integration tests") + } + return raw +} + +func randomHex(t *testing.T) string { + t.Helper() + buf := make([]byte, 6) + if _, err := rand.Read(buf); err != nil { + t.Fatalf("read random bytes: %v", err) + } + return hex.EncodeToString(buf) +} + +type pgir02Service struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + client *control.Client + schema string + extraSchemas []string + resources *control.ResourceRepository + edges *control.EdgeRepository + scrub func(any) string + fail func(step string, err error) +} + +func newPGIR02Service(t *testing.T, extraSchemaCount int) *pgir02Service { + t.Helper() + return newOwnedService(t, extraSchemaCount, "pgir02_it") +} + +func newOwnedService(t *testing.T, extraSchemaCount int, prefix string) *pgir02Service { + t.Helper() + raw := testPostgresURL(t) + namePattern := regexp.MustCompile(`^` + regexp.QuoteMeta(prefix) + `_[0-9]+_[0-9a-f]+$`) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf(prefix+"_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !namePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(prefix+"_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !namePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + client, err := control.OpenPool(ctx, control.ClientConfig{ + DSN: dsn, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 2 * time.Second, + }) + if err != nil { + fail("open pg-store client (requires a PostgreSQL 18 service)", err) + } + t.Cleanup(client.Close) + if _, err := control.MigrateSchema(ctx, client, schema); err != nil { + fail("migrate owned schema", err) + } + resources, err := control.NewResourceRepository(client, schema) + if err != nil { + fail("build resource repository", err) + } + edges, err := control.NewEdgeRepository(client, schema) + if err != nil { + fail("build edge repository", err) + } + svc := &pgir02Service{ + ctx: ctx, admin: admin, iso: iso, client: client, schema: schema, + resources: resources, edges: edges, scrub: scrub, fail: fail, + } + for index := 0; index < extraSchemaCount; index++ { + extra := fmt.Sprintf(prefix+"_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !namePattern.MatchString(extra) { + t.Fatalf("extra owned schema name %q does not match the required pattern", extra) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+extra+`"`); err != nil { + fail("create extra schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+extra+`" CASCADE`); err != nil { + t.Errorf("drop extra schema: %s", scrub(err)) + } + }) + if _, err := control.MigrateSchema(ctx, client, extra); err != nil { + fail("migrate extra schema", err) + } + svc.extraSchemas = append(svc.extraSchemas, extra) + } + return svc +} + +func (s *pgir02Service) insertResource(t *testing.T, schema string, project, generation, resourceType, logicalID, authPath string, payload any) int64 { + t.Helper() + var payloadValue json.RawMessage + if payload != nil { + raw, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + payloadValue = raw + } + repo := s.resources + if schema != s.schema { + var err error + repo, err = control.NewResourceRepository(s.client, schema) + if err != nil { + t.Fatalf("build repository for schema %s: %v", schema, err) + } + } + seq, err := repo.InsertResource(s.ctx, source.Resource{ + Project: project, + DatasetGeneration: generation, + ResourceType: resourceType, + LogicalID: logicalID, + AuthResourcePath: authPath, + Payload: payloadValue, + }) + if err != nil { + s.fail("insert resource "+resourceType+"/"+logicalID, err) + } + return seq +} + +func (s *pgir02Service) insertEdge(t *testing.T, rec control.EdgeOccurrenceRow) int64 { + t.Helper() + seq, err := s.edges.InsertEdge(s.ctx, rec) + if err != nil { + s.fail("insert edge "+rec.Label+" "+rec.SourceLogicalID+"->"+rec.TargetLogicalID, err) + } + return seq +} + +func (s *pgir02Service) runRendered(t *testing.T, rendered command.RenderedPostgresPhysicalPlan) [][]string { + t.Helper() + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + rows, err := s.iso.Query(s.ctx, rendered.SQL, args...) + if err != nil { + s.fail("execute rendered statement", err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + s.fail("read rendered row", err) + } + row := make([]string, len(values)) + for index, value := range values { + row[index] = stringifyCell(value) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + s.fail("finish rendered statement", err) + } + return out +} + +func (s *pgir02Service) witness(t *testing.T, sql string, args ...any) [][]string { + t.Helper() + rows, err := s.iso.Query(s.ctx, sql, args...) + if err != nil { + s.fail("execute direct-SQL witness", err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + s.fail("read witness row", err) + } + row := make([]string, len(values)) + for index, value := range values { + row[index] = stringifyCell(value) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + s.fail("finish witness", err) + } + return out +} + +func stringifyCell(value any) string { + switch typed := value.(type) { + case nil: + return "" + case string: + return typed + case []byte: + return string(typed) + case pgtype.Numeric: + if !typed.Valid { + return "" + } + text, err := typed.MarshalJSON() + if err != nil { + return fmt.Sprintf("%v", typed) + } + return string(text) + default: + return fmt.Sprintf("%v", typed) + } +} + +func scopedPlan(project, generation string, paths []string, unrestricted bool) ir.PhysicalPlan { + return ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": "Patient", + "project": project, + "dataset_generation": generation, + "auth_resource_paths": paths, + "auth_resource_paths_unrestricted": unrestricted, + "scope_allowed": true, + }, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + }, + } +} + +func withReturn(plan ir.PhysicalPlan, projections ...ir.PhysicalProjection) ir.PhysicalPlan { + plan.Operations = append(plan.Operations, ir.PhysicalOperation{ + Kind: ir.PhysicalReturnOp, + Return: &ir.PhysicalReturn{Projections: projections}, + }) + return plan +} + +const ( + itProject = "pgir02-proj" + itGeneration = "gen-1" +) + +func bindingFor(t *testing.T, schema string) ir.PostgresStorageBinding { + t.Helper() + binding, err := ir.NewPostgresStorageBinding(schema) + if err != nil { + t.Fatalf("NewPostgresStorageBinding(%q): %v", schema, err) + } + return binding +} + +func TestPG18ScopedResourceReadMatchesLiteralRowsAndDirectSQLWitness(t *testing.T) { + svc := newPGIR02Service(t, 0) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-final", "FHIR/Patient", map[string]any{"status": "final", "score": 3}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-prelim", "FHIR/Patient", map[string]any{"status": "prelim"}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-other-path", "FHIR/Encounter", map[string]any{"status": "final"}) + svc.insertResource(t, svc.schema, itProject, "gen-2", "Patient", "p-other-gen", "FHIR/Patient", map[string]any{"status": "final"}) + svc.insertResource(t, svc.schema, "pgir02-other", itGeneration, "Patient", "p-other-proj", "FHIR/Patient", map[string]any{"status": "final"}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Observation", "o-1", "FHIR/Observation", map[string]any{"status": "final"}) + + plan := scopedPlan(itProject, itGeneration, []string{"FHIR/Patient", "FHIR/Encounter"}, false) + plan.BindVars["status_value"] = "final" + plan.BindVars["limit"] = 5 + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "status_value"}, + }}}, + ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{ + Keys: []ir.PhysicalValue{{Variable: "root", Path: []string{"logical_id"}}}, + }}, + ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "limit"}}, + ) + plan = withReturn(plan, + ir.PhysicalProjection{Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + ir.PhysicalProjection{Name: "auth", Value: ir.PhysicalValue{Variable: "root", Path: []string{"auth_resource_path"}}}, + ) + + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + for _, data := range []string{"final", itProject, itGeneration, "FHIR/Patient", "p-final"} { + if strings.Contains(rendered.SQL, data) { + t.Fatalf("value %q leaked into rendered SQL: %s", data, rendered.SQL) + } + } + got := svc.runRendered(t, rendered) + want := [][]string{ + {"p-final", "FHIR/Patient"}, + {"p-other-path", "FHIR/Encounter"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("rendered rows = %v, want %v", got, want) + } + + witness := svc.witness(t, + `SELECT logical_id, auth_resource_path FROM "`+svc.schema+`"."fhir_resource"`+ + ` WHERE project = 'pgir02-proj' AND dataset_generation = 'gen-1'`+ + ` AND resource_type = 'Patient'`+ + ` AND auth_resource_path IN ('FHIR/Patient','FHIR/Encounter')`+ + ` AND payload #> '{status}'::text[] = '"final"'::jsonb`+ + ` ORDER BY logical_id, resource_seq LIMIT 5`) + if !reflect.DeepEqual(witness, want) { + t.Fatalf("witness rows = %v, want %v", witness, want) + } +} + +func TestPG18RestrictedEmptyAuthReadsNothingWitnessShowsRows(t *testing.T) { + svc := newPGIR02Service(t, 0) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-1", "FHIR/Patient", map[string]any{"status": "final"}) + + plan := withReturn(scopedPlan(itProject, itGeneration, []string{}, false), + ir.PhysicalProjection{Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}) + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + if got := svc.runRendered(t, rendered); len(got) != 0 { + t.Fatalf("restricted-empty auth must read nothing, got %v", got) + } + if !strings.Contains(rendered.SQL, "$3") || !strings.Contains(rendered.SQL, "auth_resource_path") { + t.Fatalf("restricted mode must keep the auth predicate: %s", rendered.SQL) + } + witness := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" WHERE project = $1 AND dataset_generation = $2 AND resource_type = 'Patient'`, + itProject, itGeneration) + if len(witness) != 1 || witness[0][0] != "1" { + t.Fatalf("witness must show the row exists for the restricted-empty read to mean anything: %v", witness) + } +} + +func TestPG18ExplicitUnrestrictedSeesAllPaths(t *testing.T) { + svc := newPGIR02Service(t, 0) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-1", "FHIR/Patient", nil) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-2", "", nil) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-3", "FHIR/Anything", nil) + + plan := withReturn(scopedPlan(itProject, itGeneration, nil, true), + ir.PhysicalProjection{Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}) + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + if strings.Contains(rendered.SQL, "auth_resource_path") { + t.Fatalf("explicit unrestricted mode must add no auth predicate: %s", rendered.SQL) + } + got := svc.runRendered(t, rendered) + want := [][]string{{"p-1"}, {"p-2"}, {"p-3"}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("rows = %v, want %v", got, want) + } +} + +func TestPG18EdgeTraversalPreservesMultiplicityAndExcludesUnresolvedTargets(t *testing.T) { + svc := newPGIR02Service(t, 0) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "sp-1", "FHIR/Patient", nil) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Specimen", "st-1", "FHIR/Specimen", nil) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Specimen", "st-2", "FHIR/Specimen", nil) + + svc.insertEdge(t, control.EdgeOccurrenceRow{ + Project: itProject, DatasetGeneration: itGeneration, AuthResourcePath: "FHIR/Patient", + Label: "has-specimen", SourceResourceType: "Patient", SourceLogicalID: "sp-1", + TargetResourceType: "Specimen", TargetLogicalID: "st-1", SourcePath: "specimen[0]", SourceOrdinal: 0, + }) + svc.insertEdge(t, control.EdgeOccurrenceRow{ + Project: itProject, DatasetGeneration: itGeneration, AuthResourcePath: "FHIR/Patient", + Label: "has-specimen", SourceResourceType: "Patient", SourceLogicalID: "sp-1", + TargetResourceType: "Specimen", TargetLogicalID: "st-1", SourcePath: "specimen[1]", SourceOrdinal: 0, + }) + svc.insertEdge(t, control.EdgeOccurrenceRow{ + Project: itProject, DatasetGeneration: itGeneration, AuthResourcePath: "FHIR/Patient", + Label: "has-specimen", SourceResourceType: "Patient", SourceLogicalID: "sp-1", + TargetResourceType: "Specimen", TargetLogicalID: "st-2", SourcePath: "contained[0].specimen", SourceOrdinal: 0, + }) + svc.insertEdge(t, control.EdgeOccurrenceRow{ + Project: itProject, DatasetGeneration: itGeneration, AuthResourcePath: "FHIR/Patient", + Label: "has-specimen", SourceResourceType: "Patient", SourceLogicalID: "sp-1", + TargetResourceType: "Specimen", TargetLogicalID: "st-ghost", SourcePath: "specimen[9]", SourceOrdinal: 0, + }) + + plan := scopedPlan(itProject, itGeneration, []string{"FHIR/Patient", "FHIR/Specimen"}, false) + plan.BindVars["edge_collection"] = "fhir_edge" + plan.BindVars["edge_label"] = "has-specimen" + plan.BindVars["target_type"] = "Specimen" + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "target", EdgeVariable: "edge", + Direction: ir.PhysicalOutbound, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: "edge_label", TargetTypeBindKey: "target_type", + EdgeTargetTypeField: "to_type", Strategy: ir.PhysicalTraversalNative, + }}, + ) + plan = withReturn(plan, + ir.PhysicalProjection{Name: "patient_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + ir.PhysicalProjection{Name: "specimen_id", Value: ir.PhysicalValue{Variable: "target", Path: []string{"logical_id"}}}, + ) + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + for _, data := range []string{"has-specimen", itProject, "Specimen"} { + if strings.Contains(rendered.SQL, data) { + t.Fatalf("value %q leaked into traversal SQL: %s", data, rendered.SQL) + } + } + + got := svc.runRendered(t, rendered) + want := [][]string{{"sp-1", "st-1"}, {"sp-1", "st-1"}, {"sp-1", "st-2"}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("traversal rows = %v, want repeated-owner multiplicity %v", got, want) + } + + stored := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_edge_occurrence" WHERE project=$1 AND dataset_generation=$2 AND target_logical_id='st-ghost'`, + itProject, itGeneration) + if len(stored) != 1 || stored[0][0] != "1" { + t.Fatalf("witness must show the unresolved occurrence is stored: %v", stored) + } + witness := svc.witness(t, + `SELECT r0.logical_id, r1.logical_id FROM "`+svc.schema+`"."fhir_resource" r0`+ + ` JOIN "`+svc.schema+`"."fhir_edge_occurrence" e ON e.project=r0.project AND e.dataset_generation=r0.dataset_generation`+ + ` AND e.auth_resource_path='FHIR/Patient' AND e.label='has-specimen'`+ + ` AND e.source_resource_type='Patient' AND e.source_logical_id=r0.logical_id`+ + ` AND e.target_resource_type='Specimen'`+ + ` JOIN "`+svc.schema+`"."fhir_resource" r1 ON r1.project=e.project AND r1.dataset_generation=e.dataset_generation`+ + ` AND r1.auth_resource_path='FHIR/Specimen' AND r1.resource_type='Specimen' AND r1.logical_id=e.target_logical_id`+ + ` WHERE r0.project=$1 AND r0.dataset_generation=$2 AND r0.resource_type='Patient' AND r0.auth_resource_path='FHIR/Patient'`+ + ` ORDER BY r0.resource_seq, e.edge_seq`, + itProject, itGeneration) + if !reflect.DeepEqual(witness, want) { + t.Fatalf("witness rows = %v, want %v", witness, want) + } +} + +func TestPG18InboundTraversalBindsReverseIdentity(t *testing.T) { + svc := newPGIR02Service(t, 0) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Specimen", "st-1", "FHIR/Specimen", nil) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Observation", "o-1", "FHIR/Observation", nil) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Observation", "o-2", "FHIR/Observation", nil) + svc.insertEdge(t, control.EdgeOccurrenceRow{ + Project: itProject, DatasetGeneration: itGeneration, AuthResourcePath: "FHIR/Observation", + Label: "focus-specimen", SourceResourceType: "Observation", SourceLogicalID: "o-1", + TargetResourceType: "Specimen", TargetLogicalID: "st-1", SourcePath: "focus[0]", SourceOrdinal: 0, + }) + + plan := ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": "Specimen", + "project": itProject, + "dataset_generation": itGeneration, + "auth_resource_paths": []string{"FHIR/Specimen", "FHIR/Observation"}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + "edge_collection": "fhir_edge", + "edge_label": "focus-specimen", + "target_type": "Observation", + }, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "target", EdgeVariable: "edge", + Direction: ir.PhysicalInbound, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: "edge_label", TargetTypeBindKey: "target_type", + EdgeTargetTypeField: "from_type", Strategy: ir.PhysicalTraversalNative, + }}, + }, + } + plan = withReturn(plan, + ir.PhysicalProjection{Name: "specimen_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + ir.PhysicalProjection{Name: "observation_id", Value: ir.PhysicalValue{Variable: "target", Path: []string{"logical_id"}}}, + ) + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + got := svc.runRendered(t, rendered) + want := [][]string{{"st-1", "o-1"}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("inbound rows = %v, want %v", got, want) + } + witness := svc.witness(t, + `SELECT 1 FROM "`+svc.schema+`"."fhir_edge_occurrence" WHERE project=$1 AND dataset_generation=$2 AND label='focus-specimen' AND target_logical_id='st-1' AND source_logical_id='o-1'`, + itProject, itGeneration) + if len(witness) != 1 { + t.Fatalf("witness occurrence missing: %v", witness) + } +} + +func TestPG18HostileFilterValuesAreInert(t *testing.T) { + svc := newPGIR02Service(t, 0) + hostile := `x'; DROP TABLE fhir_resource; --` + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-hostile", "FHIR/Patient", map[string]any{"note": hostile}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-plain", "FHIR/Patient", map[string]any{"note": "plain"}) + + plan := scopedPlan(itProject, itGeneration, []string{"FHIR/Patient"}, false) + plan.BindVars["note_value"] = hostile + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"note"}}, + Right: &ir.PhysicalValue{BindKey: "note_value"}, + }}}, + ) + plan = withReturn(plan, + ir.PhysicalProjection{Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}) + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + if strings.Contains(rendered.SQL, "DROP") || strings.Contains(rendered.SQL, hostile) { + t.Fatalf("hostile value reached statement text: %s", rendered.SQL) + } + got := svc.runRendered(t, rendered) + if !reflect.DeepEqual(got, [][]string{{"p-hostile"}}) { + t.Fatalf("hostile equality rows = %v, want the stored hostile row only", got) + } + for _, table := range []string{"fhir_resource", "fhir_edge_occurrence"} { + survived := svc.witness(t, `SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1 AND c.relname=$2`, svc.schema, table) + if len(survived) != 1 || survived[0][0] != "1" { + t.Fatalf("%s table was affected by a bound value", table) + } + } + resourcesRemaining := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" WHERE project=$1 AND dataset_generation=$2`, itProject, itGeneration) + if len(resourcesRemaining) != 1 || resourcesRemaining[0][0] != "2" { + t.Fatalf("fhir_resource rows were affected by a bound value: %v", resourcesRemaining) + } + witness := svc.witness(t, + `SELECT logical_id FROM "`+svc.schema+`"."fhir_resource" WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' AND payload #> '{note}'::text[] = $3::jsonb`, + itProject, itGeneration, func() json.RawMessage { + raw, _ := json.Marshal(hostile) + return raw + }()) + if !reflect.DeepEqual(witness, [][]string{{"p-hostile"}}) { + t.Fatalf("witness rows = %v", witness) + } +} + +func TestPG18BindingSelectsOwnedSchemaOnly(t *testing.T) { + svc := newPGIR02Service(t, 1) + other := svc.extraSchemas[0] + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-in-owned", "FHIR/Patient", nil) + shadowRepo, err := control.NewResourceRepository(svc.client, other) + if err != nil { + t.Fatalf("shadow repository: %v", err) + } + if _, err := shadowRepo.InsertResource(svc.ctx, source.Resource{ + Project: itProject, DatasetGeneration: itGeneration, ResourceType: "Patient", + LogicalID: "p-in-shadow", AuthResourcePath: "FHIR/Patient", + }); err != nil { + svc.fail("insert into shadow schema", err) + } + + plan := withReturn(scopedPlan(itProject, itGeneration, []string{"FHIR/Patient"}, false), + ir.PhysicalProjection{Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}) + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + if !strings.Contains(rendered.SQL, `"`+svc.schema+`"."fhir_resource"`) { + t.Fatalf("statement does not name the bound schema: %s", rendered.SQL) + } + got := svc.runRendered(t, rendered) + if !reflect.DeepEqual(got, [][]string{{"p-in-owned"}}) { + t.Fatalf("rows = %v, want only the owned-schema row", got) + } + shadow := svc.witness(t, + `SELECT logical_id FROM "`+other+`"."fhir_resource" WHERE project=$1 AND dataset_generation=$2`, + itProject, itGeneration) + if !reflect.DeepEqual(shadow, [][]string{{"p-in-shadow"}}) { + t.Fatalf("shadow schema rows = %v, want the shadow row to exist", shadow) + } +} + +func TestPG18PredicateParityRepairs(t *testing.T) { + svc := newPGIR02Service(t, 0) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-active", "FHIR/Patient", map[string]any{"status": "active"}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", "p-nostatus", "FHIR/Patient", map[string]any{"note": "x"}) + + logicalID := ir.PhysicalProjection{Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}} + + notPlan := scopedPlan(itProject, itGeneration, []string{"FHIR/Patient"}, false) + notPlan.BindVars["status_value"] = "active" + notPlan.Operations = append(notPlan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Expression: &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalNotPredicate, + Children: []ir.PhysicalPredicateExpression{ + {Kind: ir.PhysicalComparisonPredicate, Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "status_value"}, + }}, + }, + }}}, + ) + notPlan = withReturn(notPlan, logicalID) + notRendered, err := RenderPhysicalPlan(notPlan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("NOT render: %v", err) + } + notGot := svc.runRendered(t, notRendered) + if !reflect.DeepEqual(notGot, [][]string{{"p-nostatus"}}) { + t.Fatalf("NOT over missing-key rows = %v, want only the missing-key row", notGot) + } + witness := svc.witness(t, + `SELECT logical_id FROM "`+svc.schema+`"."fhir_resource" WHERE project=$1 AND dataset_generation=$2 AND payload #> '{status}'::text[] IS NULL`, + itProject, itGeneration) + if !reflect.DeepEqual(witness, [][]string{{"p-nostatus"}}) { + t.Fatalf("missing-key witness = %v", witness) + } + + inPlan := scopedPlan(itProject, itGeneration, []string{"FHIR/Patient"}, false) + inPlan.BindVars["in_values"] = []string{} + inPlan.Operations = append(inPlan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "IN", Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + Right: &ir.PhysicalValue{BindKey: "in_values"}, + }}}, + ) + inPlan = withReturn(inPlan, logicalID) + inRendered, err := RenderPhysicalPlan(inPlan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("empty IN render: %v", err) + } + if got := svc.runRendered(t, inRendered); len(got) != 0 { + t.Fatalf("empty IN rows = %v, want none", got) + } + + existsPlan := scopedPlan(itProject, itGeneration, []string{"FHIR/Patient"}, false) + existsPlan.Operations = append(existsPlan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EXISTS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + }}}, + ) + existsPlan = withReturn(existsPlan, logicalID) + existsRendered, err := RenderPhysicalPlan(existsPlan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("EXISTS render: %v", err) + } + existsRows := svc.runRendered(t, existsRendered) + if len(existsRows) != 2 { + t.Fatalf("column EXISTS rows = %v, want both rows", existsRows) + } + missingPlan := scopedPlan(itProject, itGeneration, []string{"FHIR/Patient"}, false) + missingPlan.Operations = append(missingPlan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "MISSING", Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + }}}, + ) + missingPlan = withReturn(missingPlan, logicalID) + missingRendered, err := RenderPhysicalPlan(missingPlan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("MISSING render: %v", err) + } + if got := svc.runRendered(t, missingRendered); len(got) != 0 { + t.Fatalf("column MISSING rows = %v, want none", got) + } + + neqPlan := scopedPlan(itProject, itGeneration, []string{"FHIR/Patient"}, false) + neqPlan.BindVars["id_value"] = "p-active" + neqPlan.Operations = append(neqPlan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "NOT_EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + Right: &ir.PhysicalValue{BindKey: "id_value"}, + }}}, + ) + neqPlan = withReturn(neqPlan, logicalID) + neqRendered, err := RenderPhysicalPlan(neqPlan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("column NOT_EQUALS render: %v", err) + } + if got := svc.runRendered(t, neqRendered); !reflect.DeepEqual(got, [][]string{{"p-nostatus"}}) { + t.Fatalf("column NOT_EQUALS rows = %v, want the other row", got) + } + + for _, tc := range []struct { + operator string + want [][]string + }{ + {"LTE", [][]string{{"p-nostatus"}}}, + {"GT", [][]string{{"p-active"}}}, + {"EQUALS", [][]string{{"p-nostatus"}}}, + {"NOT_EQUALS", [][]string{{"p-active"}}}, + } { + nullPlan := scopedPlan(itProject, itGeneration, []string{"FHIR/Patient"}, false) + nullPlan.BindVars["null_value"] = json.RawMessage("null") + nullPlan.Operations = append(nullPlan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: tc.operator, Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "null_value"}, + }}}, + ) + nullPlan = withReturn(nullPlan, logicalID) + nullRendered, err := RenderPhysicalPlan(nullPlan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("%s null literal render: %v", tc.operator, err) + } + got := svc.runRendered(t, nullRendered) + if len(got) != len(tc.want) { + t.Fatalf("%s null literal rows = %v, want %v", tc.operator, got, tc.want) + } + seen := map[string]bool{} + for _, row := range got { + seen[row[0]] = true + } + for _, row := range tc.want { + if !seen[row[0]] { + t.Fatalf("%s null literal rows = %v, want %v", tc.operator, got, tc.want) + } + } + } + for _, operator := range []string{"LT", "GTE"} { + rejectPlan := scopedPlan(itProject, itGeneration, []string{"FHIR/Patient"}, false) + rejectPlan.BindVars["null_value"] = json.RawMessage("null") + rejectPlan.Operations = append(rejectPlan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: operator, Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "null_value"}, + }}}, + ) + rejectPlan = withReturn(rejectPlan, logicalID) + if _, err := RenderPhysicalPlan(rejectPlan, bindingFor(t, svc.schema)); !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("%s against null literal error = %v, want ErrUnsupportedPostgresPlan", operator, err) + } + } +} + +func TestZZNoLeftoverPgIR02Resources(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer admin.Close() + var databases int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'pgir02_it_%'").Scan(&databases); err != nil { + t.Fatalf("count databases: %v", err) + } + if databases != 0 { + t.Fatalf("%d pgir02_it_* databases still exist", databases) + } + var schemas int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgir02_it_%'").Scan(&schemas); err != nil { + t.Fatalf("count schemas: %v", err) + } + if schemas != 0 { + t.Fatalf("%d pgir02_it_* schemas still exist in the service database", schemas) + } +} diff --git a/internal/dataframe/compiler/render/postgres/render_test.go b/internal/dataframe/compiler/render/postgres/render_test.go new file mode 100644 index 000000000..6182fb062 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/render_test.go @@ -0,0 +1,701 @@ +package postgres + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const testSchema = "pgir02_it_unit" + +func mustBinding(t *testing.T) ir.PostgresStorageBinding { + t.Helper() + binding, err := ir.NewPostgresStorageBinding(testSchema) + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + return binding +} + +func basePlan() ir.PhysicalPlan { + return ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": "Patient", + "project": "proj-a", + "dataset_generation": "gen-1", + "auth_resource_paths": []string{"FHIR/Patient", "FHIR/Observation"}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + "limit": 10, + }, + Operations: []ir.PhysicalOperation{ + { + Kind: ir.PhysicalRootScanOp, + RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}, + }, + { + Kind: ir.PhysicalReturnOp, + Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}, + }, + }, + } +} + +func renderOK(t *testing.T, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + return rendered +} + +func wantUnsupported(t *testing.T, err error, context string) { + t.Helper() + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("%s: error = %v (%T), want ErrUnsupportedPostgresPlan", context, err, err) + } +} + +func TestStorageBindingAcceptsOnlyRegisteredIdentifiers(t *testing.T) { + binding := mustBinding(t) + if err := binding.Validate(); err != nil { + t.Fatalf("constructed binding rejected: %v", err) + } + if got, want := binding.Resource.Identifier().SQLText(), `"pgir02_it_unit"."fhir_resource"`; got != want { + t.Fatalf("resource SQLText = %q, want %q", got, want) + } + if got, want := binding.Edge.Identifier().SQLText(), `"pgir02_it_unit"."fhir_edge_occurrence"`; got != want { + t.Fatalf("edge SQLText = %q, want %q", got, want) + } + if _, err := ir.NewPostgresStorageBinding(`public"; DROP TABLE fhir_resource; --`); !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + t.Fatalf("injection schema error = %v, want ErrUnregisteredPostgresIdentifier", err) + } + for _, schema := range []string{"", " ", "Public", "public name", "0abc", strings.Repeat("a", 64)} { + if _, err := ir.NewPostgresStorageBinding(schema); !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + t.Errorf("schema %q error = %v, want ErrUnregisteredPostgresIdentifier", schema, err) + } + } + if _, err := ir.NewPostgresTableIdentifier("public", ir.PostgresLogicalTableKind("clickhouse_rows")); !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + t.Fatalf("unknown logical kind error = %v, want ErrUnregisteredPostgresIdentifier", err) + } +} + +func TestStorageBindingZeroValueFailsRenderer(t *testing.T) { + if err := (ir.PostgresStorageBinding{}).Validate(); !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + t.Fatalf("zero binding error = %v, want ErrUnregisteredPostgresIdentifier", err) + } + if _, err := RenderPhysicalPlan(basePlan(), ir.PostgresStorageBinding{}); !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + t.Fatalf("zero binding render error = %v, want ErrUnregisteredPostgresIdentifier", err) + } +} + +func TestColumnAllowlistsMatchMigrationsLiterally(t *testing.T) { + resourceColumns, err := ir.RegisteredPostgresColumnNames(ir.PostgresResourceLogicalTable) + if err != nil { + t.Fatalf("resource columns: %v", err) + } + wantResource := []string{"auth_resource_path", "dataset_generation", "logical_id", "payload", "project", "recorded_at", "resource_seq", "resource_type"} + if !reflect.DeepEqual(resourceColumns, wantResource) { + t.Fatalf("resource columns = %v, want %v", resourceColumns, wantResource) + } + edgeColumns, err := ir.RegisteredPostgresColumnNames(ir.PostgresEdgeOccurrenceLogicalTable) + if err != nil { + t.Fatalf("edge columns: %v", err) + } + wantEdge := []string{ + "auth_resource_path", "dataset_generation", "edge_seq", "label", "logical_edge_key", + "project", "recorded_at", "source_logical_id", "source_ordinal", "source_path", + "source_resource_type", "target_logical_id", "target_resource_type", + } + if !reflect.DeepEqual(edgeColumns, wantEdge) { + t.Fatalf("edge columns = %v, want %v", edgeColumns, wantEdge) + } + binding := mustBinding(t) + if _, err := binding.Resource.Column("secret_column"); !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + t.Fatalf("unregistered column error = %v, want ErrUnregisteredPostgresIdentifier", err) + } + if _, err := (ir.PostgresLogicalTable{}).Column("project"); !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + t.Fatalf("zero logical table column error = %v, want ErrUnregisteredPostgresIdentifier", err) + } +} + +func TestRenderScopedRootScanIsExactAndParameterized(t *testing.T) { + plan := basePlan() + plan.Operations = plan.Operations[:1] + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{ + Keys: []ir.PhysicalValue{{Variable: "root", Path: []string{"logical_id"}}}, + }}, + ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "limit"}}, + ir.PhysicalOperation{Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + {Name: "status", Value: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}}, + }}}, + ) + rendered := renderOK(t, plan) + + wantSQL := `SELECT r0."logical_id" AS "c0", (r0."payload" #> $6::text[]) AS "c1"` + + ` FROM "pgir02_it_unit"."fhir_resource" AS r0` + + ` WHERE r0."project" = $1 AND r0."dataset_generation" = $2` + + ` AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND r0."resource_type" = $4` + + ` ORDER BY r0."logical_id" ASC, r0."resource_seq" ASC LIMIT $5` + if rendered.SQL != wantSQL { + t.Fatalf("SQL mismatch:\n got %s\nwant %s", rendered.SQL, wantSQL) + } + wantArgs := []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "proj-a"}, + {Type: command.PostgreSQLParameterText, Value: "gen-1"}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["FHIR/Patient","FHIR/Observation"]`)}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + {Type: command.PostgreSQLParameterBigint, Value: int64(10)}, + {Type: command.PostgreSQLParameterText, Value: "{status}"}, + } + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("args mismatch:\n got %#v\nwant %#v", rendered.Args, wantArgs) + } + if !reflect.DeepEqual(rendered.OutputColumns, []string{"logical_id", "status"}) { + t.Fatalf("output columns = %v", rendered.OutputColumns) + } + if strings.Contains(rendered.SQL, "@") { + t.Fatalf("rendered PostgreSQL SQL carries AQL bind syntax: %s", rendered.SQL) + } +} + +func TestRenderFiltersBindValuesAndPaths(t *testing.T) { + binding := mustBinding(t) + plan := basePlan() + plan.BindVars["status_value"] = "final" + plan.BindVars["count_value"] = float64(3) + plan.BindVars["in_values"] = []string{"a", "b"} + plan.BindVars["since"] = time.Date(2026, 11, 9, 12, 0, 0, 0, time.UTC) + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "status_value"}, + }}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "GT", Left: ir.PhysicalValue{Variable: "root", Path: []string{"repeats"}}, + Right: &ir.PhysicalValue{BindKey: "count_value"}, + }}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "IN", Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + Right: &ir.PhysicalValue{BindKey: "in_values"}, + }}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "GTE", Left: ir.PhysicalValue{Variable: "root", Path: []string{"recorded_at"}}, + Right: &ir.PhysicalValue{BindKey: "since"}, + }}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EXISTS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"subject"}}, + }}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "MISSING", Left: ir.PhysicalValue{Variable: "root", Path: []string{"deceased"}}, + }}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + } + _ = binding + rendered := renderOK(t, plan) + for _, want := range []string{ + `(r0."payload" #> $5::text[]) = ($6::jsonb)`, + `(r0."payload" #> $7::text[]) > ($8::jsonb)`, + `r0."logical_id" = ANY (SELECT jsonb_array_elements_text($9::jsonb))`, + `r0."recorded_at" >= $10`, + `COALESCE(jsonb_typeof(r0."payload" #> $11::text[]), 'null') <> 'null'`, + `COALESCE(jsonb_typeof(r0."payload" #> $12::text[]), 'null') = 'null'`, + } { + if !strings.Contains(rendered.SQL, want) { + t.Fatalf("SQL missing %q:\n%s", want, rendered.SQL) + } + } + for _, leaked := range []string{"final", "\"a\"", "2026-11-09", "subject", "status"} { + if strings.Contains(rendered.SQL, leaked) { + t.Fatalf("filter data %q leaked into SQL:\n%s", leaked, rendered.SQL) + } + } + if got := rendered.Args[5]; got.Type != command.PostgreSQLParameterJSONB || string(got.Value.(json.RawMessage)) != `"final"` { + t.Fatalf("status value arg = %#v", got) + } + if got := rendered.Args[7]; string(got.Value.(json.RawMessage)) != `3` { + t.Fatalf("numeric value arg = %#v", got) + } + if got := rendered.Args[8]; string(got.Value.(json.RawMessage)) != `["a","b"]` { + t.Fatalf("IN value arg = %#v", got) + } +} + +func TestRenderNotEqualsUsesDistinctFromForMissingParity(t *testing.T) { + plan := basePlan() + plan.BindVars["status_value"] = "final" + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "NOT_EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "status_value"}, + }}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + } + rendered := renderOK(t, plan) + if !strings.Contains(rendered.SQL, `(r0."payload" #> $5::text[]) IS DISTINCT FROM ($6::jsonb)`) { + t.Fatalf("NOT_EQUALS must be distinct-from semantics:\n%s", rendered.SQL) + } +} + +func TestRenderCompoundPredicateExpression(t *testing.T) { + plan := basePlan() + plan.BindVars["status_value"] = "final" + plan.BindVars["gender_value"] = "female" + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Expression: &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalAllPredicate, + Children: []ir.PhysicalPredicateExpression{ + {Kind: ir.PhysicalComparisonPredicate, Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"gender"}}, + Right: &ir.PhysicalValue{BindKey: "gender_value"}, + }}, + {Kind: ir.PhysicalNotPredicate, Children: []ir.PhysicalPredicateExpression{ + {Kind: ir.PhysicalComparisonPredicate, Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "status_value"}, + }}, + }}, + }, + }}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + } + rendered := renderOK(t, plan) + if !strings.Contains(rendered.SQL, `COALESCE(NOT ((r0."payload" #> $7::text[]) = ($8::jsonb)), true)`) { + t.Fatalf("compound predicate SQL:\n%s", rendered.SQL) + } +} + +func TestPlanStringsNeverBecomeSQLText(t *testing.T) { + plan := basePlan() + evil := `x"; DROP TABLE fhir_resource; --` + evilPath := "evil_payload_field" + plan.BindVars["project"] = evil + plan.BindVars["dataset_generation"] = `g"; DELETE FROM fhir_edge_occurrence WHERE 1=1 --` + plan.BindVars["auth_resource_paths"] = []string{evil} + plan.BindVars["root_collection"] = evil + plan.BindVars["status_value"] = `s' OR '1'='1` + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{evilPath}}, + Right: &ir.PhysicalValue{BindKey: "status_value"}, + }}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: evil, Value: ir.PhysicalValue{Variable: "root", Path: []string{evilPath}}}, + }}}, + } + rendered := renderOK(t, plan) + for _, leaked := range []string{"DROP", "DELETE", evil, `s' OR '1'='1`} { + if strings.Contains(rendered.SQL, leaked) { + t.Fatalf("plan text %q leaked into SQL:\n%s", leaked, rendered.SQL) + } + } + if rendered.OutputColumns[0] != evil { + t.Fatalf("output name not carried as data: %v", rendered.OutputColumns) + } + var sawProject, sawGeneration, sawAuth, sawValue bool + for _, arg := range rendered.Args { + switch value := arg.Value.(type) { + case string: + sawProject = sawProject || value == evil + sawGeneration = sawGeneration || strings.HasPrefix(value, `g"; DELETE`) + case json.RawMessage: + sawAuth = sawAuth || strings.Contains(string(value), "DROP") + sawValue = sawValue || string(value) == `"s' OR '1'='1"` + } + } + if !sawProject || !sawGeneration || !sawAuth || !sawValue { + t.Fatalf("hostile values missing from ordered args (project=%t generation=%t auth=%t value=%t): %#v", + sawProject, sawGeneration, sawAuth, sawValue, rendered.Args) + } +} + +func TestRenderFailsClosedForScopeDefects(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(map[string]any) + }{ + {"missing project", func(b map[string]any) { delete(b, "project") }}, + {"blank project", func(b map[string]any) { b["project"] = " " }}, + {"missing generation", func(b map[string]any) { delete(b, "dataset_generation") }}, + {"missing auth mode", func(b map[string]any) { delete(b, "auth_resource_paths_unrestricted") }}, + {"missing auth paths", func(b map[string]any) { delete(b, "auth_resource_paths") }}, + {"unrestricted with paths", func(b map[string]any) { b["auth_resource_paths_unrestricted"] = true }}, + } { + plan := basePlan() + tc.mutate(plan.BindVars) + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if !errors.Is(err, ErrInvalidPostgresPlanScope) { + t.Errorf("%s: error = %v, want ErrInvalidPostgresPlanScope", tc.name, err) + } + } +} + +func TestRenderRestrictedEmptyAuthPathsBindsEmptySet(t *testing.T) { + plan := basePlan() + plan.BindVars["auth_resource_paths"] = []string{} + rendered := renderOK(t, plan) + if !strings.Contains(rendered.SQL, `= ANY (SELECT jsonb_array_elements_text($3::jsonb))`) { + t.Fatalf("restricted mode must bind the (empty) path set:\n%s", rendered.SQL) + } + if string(rendered.Args[2].Value.(json.RawMessage)) != "[]" { + t.Fatalf("empty auth path set must bind as [], got %#v", rendered.Args[2]) + } +} + +func TestRenderExplicitUnrestrictedDropsAuthPredicate(t *testing.T) { + plan := basePlan() + plan.BindVars["auth_resource_paths_unrestricted"] = true + plan.BindVars["auth_resource_paths"] = []string(nil) + rendered := renderOK(t, plan) + if strings.Contains(rendered.SQL, "auth_resource_path") { + t.Fatalf("explicit unrestricted mode must not add an auth predicate:\n%s", rendered.SQL) + } +} + +func TestRenderRejectsUnsupportedSurfaceWithTypedErrors(t *testing.T) { + cases := []struct { + name string + gate bool + mutate func(*ir.PhysicalPlan) + }{ + {"clickhouse engine", false, func(p *ir.PhysicalPlan) { p.Engine = ir.PhysicalEngineClickHouse }}, + {"population root", false, func(p *ir.PhysicalPlan) { + p.Operations[0].RootScan.Population = &ir.PhysicalPopulationRootSource{} + }}, + {"cohort root", false, func(p *ir.PhysicalPlan) { + p.Operations[0].RootScan.CohortSource = &ir.PhysicalCohortRootSource{} + }}, + {"candidate seed", false, func(p *ir.PhysicalPlan) { + p.Operations[0].RootScan.PageCandidateSeed = &ir.PhysicalRootPageCandidateSeed{} + }}, + {"unnest", false, func(p *ir.PhysicalPlan) { + p.Operations = append(p.Operations[:1], + ir.PhysicalOperation{Kind: ir.PhysicalUnnestOp, Unnest: &ir.PhysicalUnnest{}}, p.Operations[1]) + }}, + {"collection scan", false, func(p *ir.PhysicalPlan) { + p.Operations = append(p.Operations[:1], + ir.PhysicalOperation{Kind: ir.PhysicalCollectionScanOp, CollectionScan: &ir.PhysicalCollectionScan{}}, p.Operations[1]) + }}, + {"grouped pivot", true, func(p *ir.PhysicalPlan) { + p.Operations = append(p.Operations[:1], + ir.PhysicalOperation{Kind: ir.PhysicalGroupedPivotOp, GroupedPivot: &ir.PhysicalGroupedPivot{}}, p.Operations[1]) + }}, + {"missing return", false, func(p *ir.PhysicalPlan) { p.Operations = p.Operations[:1] }}, + {"any direction traversal", true, func(p *ir.PhysicalPlan) { + addTraversal(p, ir.PhysicalAny, "to_type", "", nil) + }}, + {"endpoint lookup strategy", true, func(p *ir.PhysicalPlan) { + addTraversal(p, ir.PhysicalOutbound, "to_type", ir.PhysicalTraversalEndpointLookup, []string{"_from"}) + }}, + {"unproven discriminator", true, func(p *ir.PhysicalPlan) { + addTraversal(p, ir.PhysicalOutbound, "from_type", "", nil) + }}, + {"contains_text filter", true, func(p *ir.PhysicalPlan) { + p.BindVars["needle"] = "abc" + p.Operations = append(p.Operations[:1], ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Predicate: ir.PhysicalPredicate{ + Operator: "CONTAINS_TEXT", Left: ir.PhysicalValue{Variable: "root", Path: []string{"name"}}, + Right: &ir.PhysicalValue{BindKey: "needle"}, + }}}, p.Operations[1]) + }}, + {"column IN with non-slice", true, func(p *ir.PhysicalPlan) { + p.BindVars["bad_in"] = "not-a-list" + p.Operations = append(p.Operations[:1], ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Predicate: ir.PhysicalPredicate{ + Operator: "IN", Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + Right: &ir.PhysicalValue{BindKey: "bad_in"}, + }}}, p.Operations[1]) + }}, + {"payload IN", true, func(p *ir.PhysicalPlan) { + p.BindVars["bad_in"] = []string{"a"} + p.Operations = append(p.Operations[:1], ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Predicate: ir.PhysicalPredicate{ + Operator: "IN", Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "bad_in"}, + }}}, p.Operations[1]) + }}, + {"payload sort", true, func(p *ir.PhysicalPlan) { + p.Operations = append(p.Operations[:1], ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{ + Keys: []ir.PhysicalValue{{Variable: "root", Path: []string{"status"}}}}}, p.Operations[1]) + }}, + {"hidden projection", true, func(p *ir.PhysicalPlan) { + p.Operations[1].Return.Projections[0].Hidden = true + }}, + {"bind-backed projection", true, func(p *ir.PhysicalPlan) { + p.BindVars["const"] = "x" + p.Operations[1].Return.Projections[0].Value = ir.PhysicalValue{BindKey: "const"} + }}, + } + for _, tc := range cases { + plan := basePlan() + tc.mutate(&plan) + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil { + t.Errorf("%s: render succeeded, want fail-closed rejection", tc.name) + continue + } + if tc.gate && !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Errorf("%s: error = %v, want ErrUnsupportedPostgresPlan", tc.name, err) + } + } +} + +func addTraversal(p *ir.PhysicalPlan, direction ir.PhysicalTraversalDirection, field string, strategy ir.PhysicalTraversalStrategy, indexFields []string) { + p.BindVars["edge_collection"] = "fhir_edge" + p.BindVars["edge_label"] = "l" + p.BindVars["target_type"] = "Specimen" + p.Operations = []ir.PhysicalOperation{p.Operations[0], + {Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "target", Direction: direction, + EdgeCollectionBindKey: "edge_collection", EdgeLabelBindKey: "edge_label", + TargetTypeBindKey: "target_type", EdgeTargetTypeField: field, + Strategy: strategy, EndpointIndexFields: indexFields, + }}, + p.Operations[1]} + if strategy == ir.PhysicalTraversalEndpointLookup { + traversal := p.Operations[1].Traversal + traversal.EndpointField, traversal.EndpointJoinField = "_from", "_to" + } +} + +func TestRenderRejectsInvalidPlanStructurally(t *testing.T) { + plan := basePlan() + plan.Version = 0 + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); err == nil || !strings.Contains(err.Error(), "validate physical plan") { + t.Fatalf("invalid plan accepted: %v", err) + } +} + +func filteredPlan() ir.PhysicalPlan { + plan := basePlan() + plan.BindVars["status_value"] = "final" + plan.Operations = append(plan.Operations[:1], + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "status_value"}, + }}}, + ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "limit"}}, + basePlan().Operations[1]) + return plan +} + +func TestRenderIsDeterministicAndDoesNotAliasPlanData(t *testing.T) { + plan := filteredPlan() + first := renderOK(t, plan) + second := renderOK(t, plan) + if first.SQL != second.SQL { + t.Fatalf("repeated render changed SQL:\n%s\n%s", first.SQL, second.SQL) + } + if !reflect.DeepEqual(first.Args, second.Args) { + t.Fatalf("repeated render changed args:\n%#v\n%#v", first.Args, second.Args) + } + + capturedSQL, capturedArgs := first.SQL, first.Args + capturedColumns := first.OutputColumns + + plan.BindVars["project"] = "mutated" + plan.BindVars["dataset_generation"] = "mutated" + plan.BindVars["auth_resource_paths"].([]string)[0] = "MUTATED" + plan.BindVars["root_collection"] = "Observation" + plan.BindVars["limit"] = 999 + plan.BindVars["status_value"] = "mutated" + + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); err != nil { + t.Fatalf("render after mutation: %v", err) + } + if first.SQL != capturedSQL || !reflect.DeepEqual(first.Args, capturedArgs) || !reflect.DeepEqual(first.OutputColumns, capturedColumns) { + t.Fatalf("rendered result aliased input plan data") + } + + third := renderOK(t, filteredPlan()) + if third.SQL != capturedSQL || !reflect.DeepEqual(third.Args, capturedArgs) { + t.Fatalf("identical re-plan did not re-render identically:\n first %#v\n third %#v", first, third) + } + + first.Args[0].Value = "tampered" + first.OutputColumns[0] = "tampered" + fourth := renderOK(t, filteredPlan()) + if fourth.SQL != second.SQL || !reflect.DeepEqual(fourth.Args, second.Args) { + t.Fatalf("result mutation leaked into the renderer") + } +} + +func TestRenderedResultCopiesConstructorSlices(t *testing.T) { + args := []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterText, Value: "x"}} + columns := []string{"a"} + rendered, err := command.NewRenderedPostgresPhysicalPlan(`SELECT $1`, args, columns) + if err != nil { + t.Fatalf("construct: %v", err) + } + args[0].Value = "mutated" + columns[0] = "mutated" + if rendered.Args[0].Value != "x" || rendered.OutputColumns[0] != "a" { + t.Fatalf("constructor aliased caller slices: %#v %v", rendered.Args, rendered.OutputColumns) + } +} + +func TestRenderNotEqualsOnTypedColumnIsTwoValued(t *testing.T) { + plan := basePlan() + plan.BindVars["id_value"] = "x" + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "NOT_EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + Right: &ir.PhysicalValue{BindKey: "id_value"}, + }}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + } + rendered := renderOK(t, plan) + if !strings.Contains(rendered.SQL, `r0."logical_id" IS DISTINCT FROM $5`) { + t.Fatalf("NOT_EQUALS over a typed column must render IS DISTINCT FROM:\n%s", rendered.SQL) + } + if strings.Contains(rendered.SQL, `"x"`) { + t.Fatalf("bind value leaked into SQL:\n%s", rendered.SQL) + } + if got := rendered.Args[4]; got.Type != command.PostgreSQLParameterText || got.Value != "x" { + t.Fatalf("NOT_EQUALS column value arg = %#v", got) + } +} + +func TestRenderEmptyColumnINBindsEmptyArray(t *testing.T) { + plan := basePlan() + plan.BindVars["in_values"] = []string{} + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "IN", Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + Right: &ir.PhysicalValue{BindKey: "in_values"}, + }}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + } + rendered := renderOK(t, plan) + if !strings.Contains(rendered.SQL, `r0."logical_id" = ANY (SELECT jsonb_array_elements_text($5::jsonb))`) { + t.Fatalf("empty IN SQL:\n%s", rendered.SQL) + } + if got := rendered.Args[4]; got.Type != command.PostgreSQLParameterJSONB || string(got.Value.(json.RawMessage)) != `[]` { + t.Fatalf("empty IN arg = %#v, want JSONB []", got) + } +} + +func TestRenderExistsMissingOverTypedColumnsUseNullTests(t *testing.T) { + plan := basePlan() + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EXISTS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + }}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "MISSING", Left: ir.PhysicalValue{Variable: "root", Path: []string{"recorded_at"}}, + }}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + } + rendered := renderOK(t, plan) + if !strings.Contains(rendered.SQL, `r0."logical_id" IS NOT NULL`) || !strings.Contains(rendered.SQL, `r0."recorded_at" IS NULL`) { + t.Fatalf("column EXISTS/MISSING must be null tests:\n%s", rendered.SQL) + } + if strings.Contains(rendered.SQL, "jsonb_typeof") { + t.Fatalf("jsonb_typeof over a typed column cannot execute:\n%s", rendered.SQL) + } +} + +func TestRenderPayloadNullLiteralComparisons(t *testing.T) { + cases := []struct { + operator string + want string + }{ + {"EQUALS", `COALESCE(jsonb_typeof(r0."payload" #> $5::text[]), 'null') = 'null'`}, + {"LTE", `COALESCE(jsonb_typeof(r0."payload" #> $5::text[]), 'null') = 'null'`}, + {"NOT_EQUALS", `COALESCE(jsonb_typeof(r0."payload" #> $5::text[]), 'null') <> 'null'`}, + {"GT", `COALESCE(jsonb_typeof(r0."payload" #> $5::text[]), 'null') <> 'null'`}, + } + for _, tc := range cases { + t.Run(tc.operator, func(t *testing.T) { + rendered := renderOK(t, nullLiteralPlan(tc.operator)) + if !strings.Contains(rendered.SQL, tc.want) { + t.Fatalf("%s against null literal missing %q:\n%s", tc.operator, tc.want, rendered.SQL) + } + if len(rendered.Args) != 5 { + t.Fatalf("%s null literal bound extra args: %#v", tc.operator, rendered.Args) + } + }) + } + for _, operator := range []string{"LT", "GTE"} { + t.Run(operator, func(t *testing.T) { + _, err := RenderPhysicalPlan(nullLiteralPlan(operator), mustBinding(t)) + wantUnsupported(t, err, operator+" against JSON null literal") + }) + } +} + +func nullLiteralPlan(operator string) ir.PhysicalPlan { + plan := basePlan() + plan.BindVars["null_value"] = json.RawMessage("null") + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: operator, Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "null_value"}, + }}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + } + return plan +} + +func TestRenderNotOverColumnComparisonStaysPlain(t *testing.T) { + plan := basePlan() + plan.BindVars["id_value"] = "x" + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Expression: &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalNotPredicate, + Children: []ir.PhysicalPredicateExpression{ + {Kind: ir.PhysicalComparisonPredicate, Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + Right: &ir.PhysicalValue{BindKey: "id_value"}, + }}, + }, + }}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + } + rendered := renderOK(t, plan) + if !strings.Contains(rendered.SQL, `NOT (r0."logical_id" = $5)`) { + t.Fatalf("NOT over a two-valued column child must stay plain:\n%s", rendered.SQL) + } + if strings.Contains(rendered.SQL, "COALESCE") { + t.Fatalf("two-valued child must not be COALESCE-wrapped:\n%s", rendered.SQL) + } +} diff --git a/internal/dataframe/compiler/render/postgres/root_scan.go b/internal/dataframe/compiler/render/postgres/root_scan.go new file mode 100644 index 000000000..72175cca2 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/root_scan.go @@ -0,0 +1,79 @@ +package postgres + +import ( + "fmt" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +func RenderRootScan(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + if err := validateRootScanOperation(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return RenderPhysicalPlan(plan, storage) +} + +func validateRootScanOperation(plan ir.PhysicalPlan) error { + if len(plan.Operations) != 2 { + return fmt.Errorf( + "%w: the root-scan slice is exactly one ROOT_SCAN followed by one RETURN, got %d operation(s); filters, sorts, limits, traversals, and every other operation belong to the full read spine or to a later capability, not to RenderRootScan", + ErrUnsupportedPostgresPlan, len(plan.Operations)) + } + rootOperation, returnOperation := plan.Operations[0], plan.Operations[1] + if rootOperation.Kind != ir.PhysicalRootScanOp || rootOperation.RootScan == nil { + return fmt.Errorf("%w: the leading operation must be ROOT_SCAN with its payload set; the root-scan slice never renders a traversal, a collection scan, or a payload-less ROOT_SCAN", + ErrUnsupportedPostgresPlan) + } + if returnOperation.Kind != ir.PhysicalReturnOp || returnOperation.Return == nil { + return fmt.Errorf("%w: the terminal operation must be RETURN with its payload set", ErrUnsupportedPostgresPlan) + } + + scan := rootOperation.RootScan + if scan.Population != nil || scan.CohortSource != nil || scan.PageCandidateSeed != nil { + return fmt.Errorf("%w: population, cohort, and candidate-seed roots are not registered for PostgreSQL; only the direct root collection scan has a rendered contract", + ErrUnsupportedPostgresPlan) + } + if scan.Variable == "" { + return fmt.Errorf("%w: ROOT_SCAN must name its row variable", ErrUnsupportedPostgresPlan) + } + if scan.CollectionBindKey == "" { + return fmt.Errorf("%w: ROOT_SCAN must name the bind carrying the single resource type", ErrUnsupportedPostgresPlan) + } + if resourceType, ok := plan.BindVars[scan.CollectionBindKey].(string); !ok || strings.TrimSpace(resourceType) == "" { + return fmt.Errorf("%w: root collection bind %q must name one resource type as a non-empty string", + ErrUnsupportedPostgresPlan, scan.CollectionBindKey) + } + + if len(returnOperation.Return.Projections) == 0 { + return fmt.Errorf("%w: the root-scan RETURN must project at least one typed scope column", ErrUnsupportedPostgresPlan) + } + for _, projection := range returnOperation.Return.Projections { + if strings.TrimSpace(projection.Name) == "" { + return fmt.Errorf("%w: projection names are required output metadata (never SQL text) and must be set in the root-scan slice", + ErrUnsupportedPostgresPlan) + } + if projection.Expression != nil { + return fmt.Errorf("%w: projection %q uses an expression form; selector and expression projections belong to the extraction capability, not to the root-scan slice", + ErrUnsupportedPostgresPlan, projection.Name) + } + if projection.Presence != nil || projection.PresenceOutput || projection.Hidden { + return fmt.Errorf("%w: projection %q carries presence or hidden metadata; sidecars are outside the root-scan slice", + ErrUnsupportedPostgresPlan, projection.Name) + } + value := projection.Value + if value.BindKey != "" { + return fmt.Errorf("%w: projection %q must read a row column, not a bind value", ErrUnsupportedPostgresPlan, projection.Name) + } + if value.Variable != scan.Variable { + return fmt.Errorf("%w: projection %q reads variable %q; the root-scan slice may only read the root row variable %q", + ErrUnsupportedPostgresPlan, projection.Name, value.Variable, scan.Variable) + } + if len(value.Path) != 1 || !resourceFilterColumns[value.Path[0]] { + return fmt.Errorf("%w: projection %q must read exactly one typed scope column of the root row; payload-path extraction is a separate capability", + ErrUnsupportedPostgresPlan, projection.Name) + } + } + return nil +} diff --git a/internal/dataframe/compiler/render/postgres/root_scan_test.go b/internal/dataframe/compiler/render/postgres/root_scan_test.go new file mode 100644 index 000000000..aafe5287d --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/root_scan_test.go @@ -0,0 +1,792 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + control "github.com/calypr/loom/internal/backends/postgres/control" + "net/url" + "os" + "reflect" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + pgsq01Project = "pgsq01-proj" + pgsq01Other = "pgsq01-other" + pgsq01Gen = "gen-1" + pgsq01GenTwo = "gen-2" + pgsq01Path = "FHIR/Patient" + pgsq01ConfPath = "FHIR/Confidential" +) + +var pgsq01ResourceNamePattern = regexp.MustCompile(`^pgsq01_it_[0-9]+_[0-9a-f]+$`) + +func pgsq01Plan(project, generation string, paths []string, unrestricted bool, resourceType string, columns ...string) ir.PhysicalPlan { + projections := make([]ir.PhysicalProjection, 0, len(columns)) + for _, column := range columns { + projections = append(projections, ir.PhysicalProjection{ + Name: column, + Value: ir.PhysicalValue{Variable: "root", Path: []string{column}}, + }) + } + return ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": resourceType, + "project": project, + "dataset_generation": generation, + "auth_resource_paths": paths, + "auth_resource_paths_unrestricted": unrestricted, + "scope_allowed": true, + }, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}}, + }, + } +} + +func TestRootScanGoldenRestrictedSQLAndArguments(t *testing.T) { + plan := pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path, "FHIR/Encounter"}, false, "Patient", "logical_id", "resource_type") + binding := pgsq01UnitBinding(t) + + rendered, err := RenderRootScan(plan, binding) + if err != nil { + t.Fatalf("RenderRootScan: %v", err) + } + const wantSQL = `SELECT r0."logical_id" AS "c0", r0."resource_type" AS "c1"` + + ` FROM "pgsq01_it_unit"."fhir_resource" AS r0` + + ` WHERE r0."project" = $1` + + ` AND r0."dataset_generation" = $2` + + ` AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND r0."resource_type" = $4` + + ` ORDER BY r0."resource_seq" ASC` + if rendered.SQL != wantSQL { + t.Fatalf("restricted SQL =\n%s\nwant\n%s", rendered.SQL, wantSQL) + } + wantArgs := []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: pgsq01Project}, + {Type: command.PostgreSQLParameterText, Value: pgsq01Gen}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["FHIR/Patient","FHIR/Encounter"]`)}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + } + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("restricted args = %#v, want %#v", rendered.Args, wantArgs) + } + if !reflect.DeepEqual(rendered.OutputColumns, []string{"logical_id", "resource_type"}) { + t.Fatalf("output columns = %v", rendered.OutputColumns) + } + + for _, forbidden := range []string{"@", "DISTINCT", "GROUP BY", "JOIN", "LIMIT", "OFFSET", + pgsq01Project, pgsq01Gen, pgsq01Path, "Patient", "FHIR/Encounter"} { + if strings.Contains(rendered.SQL, forbidden) { + t.Fatalf("restricted SQL leaked %q: %s", forbidden, rendered.SQL) + } + } + + second, err := RenderRootScan(plan, binding) + if err != nil || second.SQL != rendered.SQL || !reflect.DeepEqual(second.Args, rendered.Args) { + t.Fatalf("second render drifted: %v %#v", err, second) + } + + mutable := plan.BindVars["auth_resource_paths"].([]string) + mutable[0] = "FHIR/Stolen" + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("earlier rendered args were aliased to plan data: %#v", rendered.Args) + } + third, err := RenderRootScan(plan, binding) + if err != nil { + t.Fatalf("third render after mutation: %v", err) + } + if !reflect.DeepEqual(third.Args[2].Value, json.RawMessage(`["FHIR/Stolen","FHIR/Encounter"]`)) { + t.Fatalf("plan-mutated args = %#v", third.Args[2].Value) + } +} + +func TestRootScanGoldenUnrestrictedSQLAndArguments(t *testing.T) { + plan := pgsq01Plan(pgsq01Project, "", nil, true, "Observation", "logical_id", "auth_resource_path") + rendered, err := RenderRootScan(plan, pgsq01UnitBinding(t)) + if err != nil { + t.Fatalf("RenderRootScan: %v", err) + } + const wantSQL = `SELECT r0."logical_id" AS "c0", r0."auth_resource_path" AS "c1"` + + ` FROM "pgsq01_it_unit"."fhir_resource" AS r0` + + ` WHERE r0."project" = $1` + + ` AND r0."dataset_generation" = $2` + + ` AND r0."resource_type" = $3` + + ` ORDER BY r0."resource_seq" ASC` + if rendered.SQL != wantSQL { + t.Fatalf("unrestricted SQL =\n%s\nwant\n%s", rendered.SQL, wantSQL) + } + wantArgs := []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: pgsq01Project}, + {Type: command.PostgreSQLParameterText, Value: ""}, + {Type: command.PostgreSQLParameterText, Value: "Observation"}, + } + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("unrestricted args = %#v, want %#v", rendered.Args, wantArgs) + } + if strings.Contains(rendered.SQL, "auth_resource_path =") { + t.Fatalf("explicit unrestricted mode must add no auth predicate: %s", rendered.SQL) + } +} + +func TestRootScanEmptyRestrictedPathSetRendersEmptyArrayParameter(t *testing.T) { + for _, paths := range [][]string{{}, nil} { + plan := pgsq01Plan(pgsq01Project, pgsq01Gen, paths, false, "Patient", "logical_id") + rendered, err := RenderRootScan(plan, pgsq01UnitBinding(t)) + if err != nil { + t.Fatalf("RenderRootScan: %v", err) + } + if got, ok := rendered.Args[2].Value.(json.RawMessage); !ok || string(got) != "[]" { + t.Fatalf("empty restricted set arg = %#v, want JSON empty array", rendered.Args[2].Value) + } + if rendered.Args[2].Type != command.PostgreSQLParameterJSONB { + t.Fatalf("empty restricted set type = %q", rendered.Args[2].Type) + } + } +} + +func pgsq01UnitBinding(t *testing.T) ir.PostgresStorageBinding { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("pgsq01_it_unit") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + return binding +} + +func pgsq01PlanWithOperations(operations ...ir.PhysicalOperation) ir.PhysicalPlan { + plan := pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path}, false, "Patient", "logical_id") + plan.Operations = operations + return plan +} + +func pgsq01ReturnOp(columns ...string) ir.PhysicalOperation { + projections := make([]ir.PhysicalProjection, 0, len(columns)) + for _, column := range columns { + projections = append(projections, ir.PhysicalProjection{ + Name: column, + Value: ir.PhysicalValue{Variable: "root", Path: []string{column}}, + }) + } + return ir.PhysicalOperation{Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}} +} + +func pgsq01RootScanOp() ir.PhysicalOperation { + return ir.PhysicalOperation{Kind: ir.PhysicalRootScanOp, + RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}} +} + +func pgsq01Projections(projections ...ir.PhysicalProjection) ir.PhysicalPlan { + plan := pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path}, false, "Patient") + plan.Operations[1].Return.Projections = projections + return plan +} + +func TestRootScanRejectsEveryShapeOutsideTheClosedSlice(t *testing.T) { + binding := pgsq01UnitBinding(t) + cases := []struct { + name string + plan ir.PhysicalPlan + wantIn string + }{ + {"no operations", pgsq01PlanWithOperations(), "exactly one ROOT_SCAN"}, + {"scan without return", pgsq01PlanWithOperations(pgsq01RootScanOp()), "exactly one ROOT_SCAN"}, + {"filter inserted", pgsq01PlanWithOperations(pgsq01RootScanOp(), + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + Right: &ir.PhysicalValue{BindKey: "value"}}}}, + pgsq01ReturnOp("logical_id")), "exactly one ROOT_SCAN"}, + {"traversal inserted", pgsq01PlanWithOperations(pgsq01RootScanOp(), + ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "t", Direction: ir.PhysicalOutbound, + EdgeLabelBindKey: "edge_label", TargetTypeBindKey: "target_type", + EdgeTargetTypeField: "to_type", Strategy: ir.PhysicalTraversalNative}}, + pgsq01ReturnOp("logical_id")), "exactly one ROOT_SCAN"}, + {"sort inserted", pgsq01PlanWithOperations(pgsq01RootScanOp(), + ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{ + Keys: []ir.PhysicalValue{{Variable: "root", Path: []string{"logical_id"}}}}}, + pgsq01ReturnOp("logical_id")), "exactly one ROOT_SCAN"}, + {"limit inserted", pgsq01PlanWithOperations(pgsq01RootScanOp(), + ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "limit"}}, + pgsq01ReturnOp("logical_id")), "exactly one ROOT_SCAN"}, + {"return before scan", pgsq01PlanWithOperations(pgsq01ReturnOp("logical_id"), pgsq01RootScanOp()), + "leading operation must be ROOT_SCAN"}, + {"second scan instead of return", pgsq01PlanWithOperations(pgsq01RootScanOp(), pgsq01RootScanOp()), + "terminal operation must be RETURN"}, + {"root scan without payload", pgsq01PlanWithOperations( + ir.PhysicalOperation{Kind: ir.PhysicalRootScanOp}, pgsq01ReturnOp("logical_id")), + "payload set"}, + {"return without payload", pgsq01PlanWithOperations(pgsq01RootScanOp(), + ir.PhysicalOperation{Kind: ir.PhysicalReturnOp}), "RETURN with its payload set"}, + {"unknown operation", pgsq01PlanWithOperations(pgsq01RootScanOp(), + ir.PhysicalOperation{Kind: ir.PhysicalOperationKind("NOPE")}), "terminal operation must be RETURN"}, + {"population root", func() ir.PhysicalPlan { + plan := pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path}, false, "Patient", "logical_id") + plan.Operations[0].RootScan.Population = &ir.PhysicalPopulationRootSource{} + return plan + }(), "not registered for PostgreSQL"}, + {"cohort root", func() ir.PhysicalPlan { + plan := pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path}, false, "Patient", "logical_id") + plan.Operations[0].RootScan.CohortSource = &ir.PhysicalCohortRootSource{} + return plan + }(), "not registered for PostgreSQL"}, + {"candidate seed root", func() ir.PhysicalPlan { + plan := pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path}, false, "Patient", "logical_id") + plan.Operations[0].RootScan.PageCandidateSeed = &ir.PhysicalRootPageCandidateSeed{} + return plan + }(), "not registered for PostgreSQL"}, + {"unnamed root variable", func() ir.PhysicalPlan { + plan := pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path}, false, "Patient", "logical_id") + plan.Operations[0].RootScan.Variable = "" + return plan + }(), "must name its row variable"}, + {"missing collection bind key", func() ir.PhysicalPlan { + plan := pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path}, false, "Patient", "logical_id") + plan.Operations[0].RootScan.CollectionBindKey = "" + return plan + }(), "bind carrying the single resource type"}, + {"collection bind not a string", func() ir.PhysicalPlan { + plan := pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path}, false, "Patient", "logical_id") + plan.BindVars["root_collection"] = 7 + return plan + }(), "root collection bind"}, + {"empty return", func() ir.PhysicalPlan { + return pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path}, false, "Patient") + }(), "at least one typed scope column"}, + {"blank projection name", pgsq01Projections(ir.PhysicalProjection{ + Name: " ", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}), + "projection names are required"}, + {"expression projection", pgsq01Projections(ir.PhysicalProjection{ + Name: "logical_id", + Expression: &ir.PhysicalExpression{Kind: ir.PhysicalValueExpression, + Value: &ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}}), + "expression form"}, + {"hidden projection", pgsq01Projections(ir.PhysicalProjection{ + Name: "logical_id", Hidden: true, + Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}), + "presence or hidden"}, + {"bind-value projection", pgsq01Projections(ir.PhysicalProjection{ + Name: "v", Value: ir.PhysicalValue{BindKey: "value"}}), + "not a bind value"}, + {"foreign variable projection", pgsq01Projections(ir.PhysicalProjection{ + Name: "v", Value: ir.PhysicalValue{Variable: "other", Path: []string{"logical_id"}}}), + "root row variable"}, + {"payload projection", pgsq01Projections(ir.PhysicalProjection{ + Name: "status", Value: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}}), + "typed scope column"}, + {"multi-segment projection", pgsq01Projections(ir.PhysicalProjection{ + Name: "nested", Value: ir.PhysicalValue{Variable: "root", Path: []string{"meta", "tag"}}}), + "typed scope column"}, + {"storage sequence projection", pgsq01Projections(ir.PhysicalProjection{ + Name: "seq", Value: ir.PhysicalValue{Variable: "root", Path: []string{"resource_seq"}}}), + "typed scope column"}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + rendered, err := RenderRootScan(testCase.plan, binding) + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("error = %v, want ErrUnsupportedPostgresPlan", err) + } + if rendered.SQL != "" || rendered.Args != nil || rendered.OutputColumns != nil { + t.Fatalf("rejected plan produced output: %#v", rendered) + } + if !strings.Contains(err.Error(), testCase.wantIn) { + t.Fatalf("error %q must contain %q", err.Error(), testCase.wantIn) + } + if strings.Contains(strings.ToUpper(err.Error()), "AQL") { + t.Fatalf("refusal mentions AQL: %v", err) + } + }) + } +} + +func TestRootScanRejectsMissingOrContradictoryScope(t *testing.T) { + binding := pgsq01UnitBinding(t) + cases := []struct { + name string + mutate func(plan ir.PhysicalPlan) + wantIn string + }{ + {"project missing", func(p ir.PhysicalPlan) { delete(p.BindVars, "project") }, `"project"`}, + {"project blank", func(p ir.PhysicalPlan) { p.BindVars["project"] = " " }, `"project"`}, + {"project wrong type", func(p ir.PhysicalPlan) { p.BindVars["project"] = 9 }, `"project"`}, + {"generation missing", func(p ir.PhysicalPlan) { delete(p.BindVars, "dataset_generation") }, `"dataset_generation"`}, + {"generation wrong type", func(p ir.PhysicalPlan) { p.BindVars["dataset_generation"] = 1 }, `"dataset_generation"`}, + {"auth mode missing", func(p ir.PhysicalPlan) { delete(p.BindVars, "auth_resource_paths_unrestricted") }, + `"auth_resource_paths_unrestricted"`}, + {"auth mode wrong type", func(p ir.PhysicalPlan) { p.BindVars["auth_resource_paths_unrestricted"] = "yes" }, + `"auth_resource_paths_unrestricted"`}, + {"unrestricted with paths", func(p ir.PhysicalPlan) { + p.BindVars["auth_resource_paths_unrestricted"] = true + p.BindVars["auth_resource_paths"] = []string{pgsq01Path} + }, "unrestricted mode must not carry auth resource paths"}, + {"paths wrong type", func(p ir.PhysicalPlan) { p.BindVars["auth_resource_paths"] = "FHIR/Patient" }, + `"auth_resource_paths"`}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + plan := pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path}, false, "Patient", "logical_id") + testCase.mutate(plan) + rendered, err := RenderRootScan(plan, binding) + if !errors.Is(err, ErrInvalidPostgresPlanScope) { + t.Fatalf("error = %v, want ErrInvalidPostgresPlanScope", err) + } + if rendered.SQL != "" || rendered.Args != nil || rendered.OutputColumns != nil { + t.Fatalf("rejected scope produced output: %#v", rendered) + } + if !strings.Contains(err.Error(), testCase.wantIn) { + t.Fatalf("error %q must name %q", err.Error(), testCase.wantIn) + } + }) + } +} + +func TestRootScanRejectsUntrustedStorageBinding(t *testing.T) { + plan := pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path}, false, "Patient", "logical_id") + if _, err := RenderRootScan(plan, ir.PostgresStorageBinding{}); !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + t.Fatalf("zero binding error = %v, want ErrUnregisteredPostgresIdentifier", err) + } + if _, err := ir.NewPostgresStorageBinding("PGSQ01"); !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + t.Fatalf("uppercase schema error = %v, want ErrUnregisteredPostgresIdentifier", err) + } +} + +type pgsq01Service struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + client *control.Client + schema string + scrub func(string) string + fail func(step string, err error) +} + +func newPGSQ01Service(t *testing.T) *pgsq01Service { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v string) string { + s := v + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + text := err.Error() + for _, secret := range secrets { + if secret != "" { + text = strings.ReplaceAll(text, secret, "") + } + } + t.Fatalf("%s: %s", step, text) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := pgsq01ResourceName(t) + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err.Error())) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := pgsq01ResourceName(t) + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err.Error())) + } + }) + + client, err := control.OpenPool(ctx, control.ClientConfig{ + DSN: dsn, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 2 * time.Second, + }) + if err != nil { + fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + t.Cleanup(client.Close) + if _, err := control.MigrateSchema(ctx, client, schema); err != nil { + fail("migrate owned schema", err) + } + return &pgsq01Service{ctx: ctx, admin: admin, iso: iso, client: client, schema: schema, scrub: scrub, fail: fail} +} + +func pgsq01ResourceName(t *testing.T) string { + t.Helper() + name := "pgsq01_it_" + strconv.FormatInt(time.Now().UnixNano(), 10) + "_" + randomHex(t) + if !pgsq01ResourceNamePattern.MatchString(name) { + t.Fatalf("owned resource name %q does not match the required pattern", name) + } + return name +} + +func (s *pgsq01Service) insertResource(t *testing.T, project, generation, resourceType, logicalID, authPath string, payload map[string]any) int64 { + t.Helper() + repo, err := control.NewResourceRepository(s.client, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + var raw json.RawMessage + if payload != nil { + encoded, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + raw = encoded + } + seq, err := repo.InsertResource(s.ctx, source.Resource{ + Project: project, + DatasetGeneration: generation, + ResourceType: resourceType, + LogicalID: logicalID, + AuthResourcePath: authPath, + Payload: raw, + }) + if err != nil { + s.fail("insert resource "+resourceType+"/"+logicalID, err) + } + return seq +} + +func (s *pgsq01Service) run(t *testing.T, sql string, args ...any) [][]string { + t.Helper() + rows, err := s.iso.Query(s.ctx, sql, args...) + if err != nil { + s.fail("execute statement "+sql, err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + s.fail("read row", err) + } + row := make([]string, len(values)) + for index, value := range values { + row[index] = stringifyCell(value) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + s.fail("finish statement", err) + } + return out +} + +func (s *pgsq01Service) runRendered(t *testing.T, rendered command.RenderedPostgresPhysicalPlan) [][]string { + t.Helper() + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + return s.run(t, rendered.SQL, args...) +} + +func (s *pgsq01Service) expectRefused(t *testing.T, sql string, wantCode, wantConstraint string, args ...any) { + t.Helper() + _, err := s.iso.Exec(s.ctx, sql, args...) + if err == nil { + t.Fatalf("statement was accepted, want refusal by %s: %s", wantConstraint, sql) + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + s.fail("expect pg error for "+sql, err) + } + if pgErr.Code != wantCode || pgErr.ConstraintName != wantConstraint { + s.fail("statement refused by ("+pgErr.Code+","+pgErr.ConstraintName+ + ") want ("+wantCode+","+wantConstraint+")", err) + } +} + +func TestPGSQ01RootScanReturnsOnlySelectedProjectGenerationType(t *testing.T) { + svc := newPGSQ01Service(t) + svc.insertResource(t, pgsq01Project, pgsq01Gen, "Patient", "p-1", pgsq01Path, map[string]any{"code": "shared"}) + svc.insertResource(t, pgsq01Project, pgsq01Gen, "Patient", "p-dup-a", pgsq01Path, map[string]any{"code": "shared"}) + svc.insertResource(t, pgsq01Project, pgsq01Gen, "Patient", "p-dup-b", pgsq01Path, map[string]any{"code": "shared"}) + svc.insertResource(t, pgsq01Project, pgsq01Gen, "Patient", "p-plain", pgsq01Path, map[string]any{"code": "other"}) + svc.insertResource(t, pgsq01Project, pgsq01Gen, "Observation", "p-1", pgsq01Path, nil) + svc.insertResource(t, pgsq01Project, pgsq01GenTwo, "Patient", "p-1", pgsq01Path, nil) + svc.insertResource(t, pgsq01Other, pgsq01Gen, "Patient", "p-1", pgsq01Path, nil) + svc.insertResource(t, pgsq01Project, pgsq01Gen, "Patient", "p-hidden", pgsq01ConfPath, nil) + + plan := pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path}, false, "Patient", "logical_id", "resource_type") + rendered, err := RenderRootScan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderRootScan: %v", err) + } + + wantArgs := []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: pgsq01Project}, + {Type: command.PostgreSQLParameterText, Value: pgsq01Gen}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["FHIR/Patient"]`)}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + } + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("args = %#v, want %#v", rendered.Args, wantArgs) + } + + want := [][]string{ + {"p-1", "Patient"}, + {"p-dup-a", "Patient"}, + {"p-dup-b", "Patient"}, + {"p-plain", "Patient"}, + } + got := svc.runRendered(t, rendered) + if !reflect.DeepEqual(got, want) { + t.Fatalf("root scan rows = %v, want %v", got, want) + } + + witness := svc.run(t, + `SELECT logical_id, resource_type FROM "`+svc.schema+`"."fhir_resource"`+ + ` WHERE project = 'pgsq01-proj' AND dataset_generation = 'gen-1'`+ + ` AND auth_resource_path = 'FHIR/Patient' AND resource_type = 'Patient'`+ + ` ORDER BY resource_seq`) + if !reflect.DeepEqual(witness, want) { + t.Fatalf("witness rows = %v, want %v", witness, want) + } + + for _, probe := range []struct { + desc, sql string + args []any + }{ + {"observation decoy", `SELECT count(*) FROM "` + svc.schema + `".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Observation' AND logical_id='p-1'`, []any{pgsq01Project, pgsq01Gen}}, + {"generation decoy", `SELECT count(*) FROM "` + svc.schema + `".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' AND logical_id='p-1'`, []any{pgsq01Project, pgsq01GenTwo}}, + {"project decoy", `SELECT count(*) FROM "` + svc.schema + `".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' AND logical_id='p-1'`, []any{pgsq01Other, pgsq01Gen}}, + {"auth decoy", `SELECT count(*) FROM "` + svc.schema + `".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' AND logical_id='p-hidden'`, []any{pgsq01Project, pgsq01Gen}}, + } { + rows := svc.run(t, probe.sql, probe.args...) + if len(rows) != 1 || rows[0][0] != "1" { + t.Fatalf("%s must exist: %v", probe.desc, rows) + } + } + + type probe struct { + desc string + sql string + args []any + wantRows [][]string + } + base := `SELECT logical_id FROM "` + svc.schema + `".fhir_resource WHERE ` + const orderTail = ` ORDER BY resource_seq` + for _, probe := range []probe{ + {"full chain (control)", base + `project=$1 AND dataset_generation=$2 AND auth_resource_path = ANY (SELECT jsonb_array_elements_text($3::jsonb)) AND resource_type=$4` + orderTail, + []any{pgsq01Project, pgsq01Gen, json.RawMessage(`["FHIR/Patient"]`), "Patient"}, + [][]string{{"p-1"}, {"p-dup-a"}, {"p-dup-b"}, {"p-plain"}}}, + {"without project", base + `dataset_generation=$1 AND auth_resource_path = ANY (SELECT jsonb_array_elements_text($2::jsonb)) AND resource_type=$3` + orderTail, + []any{pgsq01Gen, json.RawMessage(`["FHIR/Patient"]`), "Patient"}, + [][]string{{"p-1"}, {"p-dup-a"}, {"p-dup-b"}, {"p-plain"}, {"p-1"}}}, + {"without generation", base + `project=$1 AND auth_resource_path = ANY (SELECT jsonb_array_elements_text($2::jsonb)) AND resource_type=$3` + orderTail, + []any{pgsq01Project, json.RawMessage(`["FHIR/Patient"]`), "Patient"}, + [][]string{{"p-1"}, {"p-dup-a"}, {"p-dup-b"}, {"p-plain"}, {"p-1"}}}, + {"without auth", base + `project=$1 AND dataset_generation=$2 AND resource_type=$3` + orderTail, + []any{pgsq01Project, pgsq01Gen, "Patient"}, + [][]string{{"p-1"}, {"p-dup-a"}, {"p-dup-b"}, {"p-plain"}, {"p-hidden"}}}, + {"without resource type", base + `project=$1 AND dataset_generation=$2 AND auth_resource_path = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + orderTail, + []any{pgsq01Project, pgsq01Gen, json.RawMessage(`["FHIR/Patient"]`)}, + [][]string{{"p-1"}, {"p-dup-a"}, {"p-dup-b"}, {"p-plain"}, {"p-1"}}}, + } { + rows := svc.run(t, probe.sql, probe.args...) + if len(rows) != len(probe.wantRows) { + t.Fatalf("guard-drop %s rows = %v, want %v", probe.desc, rows, probe.wantRows) + } + for index := range rows { + if rows[index][0] != probe.wantRows[index][0] { + t.Fatalf("guard-drop %s rows = %v, want %v", probe.desc, rows, probe.wantRows) + } + } + } +} + +func TestPGSQ01AuthModeIsLoadBearingAndStableIdentityIsPreserved(t *testing.T) { + svc := newPGSQ01Service(t) + svc.insertResource(t, pgsq01Project, pgsq01Gen, "Patient", "p-visible", pgsq01Path, nil) + svc.insertResource(t, pgsq01Project, pgsq01Gen, "Patient", "p-conf", pgsq01ConfPath, nil) + svc.insertResource(t, pgsq01Project, pgsq01Gen, "Patient", "p-legacy", "", nil) + + binding := bindingFor(t, svc.schema) + restricted, err := RenderRootScan( + pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path}, false, "Patient", "logical_id"), binding) + if err != nil { + t.Fatalf("restricted render: %v", err) + } + if got := svc.runRendered(t, restricted); !reflect.DeepEqual(got, [][]string{{"p-visible"}}) { + t.Fatalf("restricted rows = %v, want only the authorized path", got) + } + unrestricted, err := RenderRootScan( + pgsq01Plan(pgsq01Project, pgsq01Gen, nil, true, "Patient", "logical_id", "auth_resource_path"), binding) + if err != nil { + t.Fatalf("unrestricted render: %v", err) + } + want := [][]string{{"p-visible", "FHIR/Patient"}, {"p-conf", "FHIR/Confidential"}, {"p-legacy", ""}} + if got := svc.runRendered(t, unrestricted); !reflect.DeepEqual(got, want) { + t.Fatalf("unrestricted rows = %v, want %v", got, want) + } + + orderSQL := `SELECT logical_id, resource_seq FROM "` + svc.schema + `".fhir_resource` + + ` WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' ORDER BY resource_seq` + first := svc.run(t, orderSQL, pgsq01Project, pgsq01Gen) + second := svc.run(t, orderSQL, pgsq01Project, pgsq01Gen) + if !reflect.DeepEqual(first, second) || len(first) != 3 { + t.Fatalf("storage identity order unstable: %v vs %v", first, second) + } + if first[0][0] != "p-visible" || first[1][0] != "p-conf" || first[2][0] != "p-legacy" { + t.Fatalf("unexpected insertion order: %v", first) + } + if !(first[0][1] < first[1][1] && first[1][1] < first[2][1]) { + t.Fatalf("resource_seq must be strictly increasing: %v", first) + } +} + +func TestPGSQ01DuplicatePhysicalDocumentsAreNotCollapsed(t *testing.T) { + svc := newPGSQ01Service(t) + duplicate := map[string]any{"code": "same", "meta": map[string]any{"tag": "dup"}} + svc.insertResource(t, pgsq01Project, pgsq01Gen, "Patient", "d-1", pgsq01Path, duplicate) + svc.insertResource(t, pgsq01Project, pgsq01Gen, "Patient", "d-2", pgsq01Path, duplicate) + svc.insertResource(t, pgsq01Project, pgsq01Gen, "Patient", "d-3", pgsq01Path, duplicate) + + binding := bindingFor(t, svc.schema) + + constantRendered, err := RenderRootScan( + pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path}, false, "Patient", "resource_type"), binding) + if err != nil { + t.Fatalf("constant-projection render: %v", err) + } + for _, forbidden := range []string{"DISTINCT", "GROUP BY"} { + if strings.Contains(strings.ToUpper(constantRendered.SQL), forbidden) { + t.Fatalf("root scan collapses rows with %s: %s", forbidden, constantRendered.SQL) + } + } + if got := svc.runRendered(t, constantRendered); !reflect.DeepEqual(got, [][]string{{"Patient"}, {"Patient"}, {"Patient"}}) { + t.Fatalf("duplicate physical documents collapsed to %v, want three rows", got) + } + + identityRendered, err := RenderRootScan( + pgsq01Plan(pgsq01Project, pgsq01Gen, []string{pgsq01Path}, false, "Patient", "logical_id"), binding) + if err != nil { + t.Fatalf("identity-projection render: %v", err) + } + if got := svc.runRendered(t, identityRendered); !reflect.DeepEqual(got, [][]string{{"d-1"}, {"d-2"}, {"d-3"}}) { + t.Fatalf("stable logical ids = %v", got) + } + + for _, logicalID := range []string{"d-1", "d-2", "d-3"} { + rows := svc.run(t, `SELECT count(*) FROM "`+svc.schema+`".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' AND logical_id=$3`, + pgsq01Project, pgsq01Gen, logicalID) + if len(rows) != 1 || rows[0][0] != "1" { + t.Fatalf("physical row count for %s = %v, want exactly 1", logicalID, rows) + } + } + duplicateInsert := `INSERT INTO "` + svc.schema + `".fhir_resource (project, dataset_generation, resource_type, logical_id, auth_resource_path) VALUES ($1,$2,$3,$4,$5)` + svc.expectRefused(t, duplicateInsert, "23505", "fhir_resource_natural_identity_ux", + pgsq01Project, pgsq01Gen, "Patient", "d-1", pgsq01Path) + + svc.expectRefused(t, duplicateInsert, "23514", "fhir_resource_project_present", + "", pgsq01Gen, "Patient", "ck-project", pgsq01Path) + svc.expectRefused(t, duplicateInsert, "23514", "fhir_resource_resource_type_present", + pgsq01Project, pgsq01Gen, "", "ck-type", pgsq01Path) + svc.expectRefused(t, duplicateInsert, "23514", "fhir_resource_logical_id_present", + pgsq01Project, pgsq01Gen, "Patient", "", pgsq01Path) + if _, err := svc.iso.Exec(svc.ctx, duplicateInsert, pgsq01Project, pgsq01Gen, "Patient", "ck-emptypath", ""); err != nil { + t.Fatalf("empty auth path is the legacy namespace and must insert: %v", err) + } + + repo, err := control.NewResourceRepository(svc.client, svc.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + if _, err := repo.InsertResource(svc.ctx, source.Resource{ + Project: pgsq01Project, DatasetGeneration: pgsq01Gen, ResourceType: "Patient", + LogicalID: "d-1", AuthResourcePath: pgsq01Path, + }); !errors.Is(err, control.ErrResourceConflict) { + t.Fatalf("duplicate repository insert = %v, want ErrResourceConflict", err) + } +} + +func TestZZNoLeftoverPgSQ01Resources(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer admin.Close() + var databases int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'pgsq01_it_%'").Scan(&databases); err != nil { + t.Fatalf("count databases: %v", err) + } + if databases != 0 { + t.Fatalf("%d pgsq01_it_* databases still exist", databases) + } + var schemas int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgsq01_it_%'").Scan(&schemas); err != nil { + t.Fatalf("count schemas: %v", err) + } + if schemas != 0 { + t.Fatalf("%d pgsq01_it_* schemas still exist in the service database", schemas) + } +} diff --git a/internal/dataframe/compiler/render/postgres/row_lineage.go b/internal/dataframe/compiler/render/postgres/row_lineage.go new file mode 100644 index 000000000..15f1fe69e --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/row_lineage.go @@ -0,0 +1,547 @@ +package postgres + +import ( + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const ( + rowLineageSourceRelation = "__loom_l_src" + rowLineageRowsRelation = "__loom_l_rows" + rowLineageGroupRelation = "__loom_l_grp" + rowLineageSelectedRow = "__loom_l_sel" + rowLineagePageRelation = "__loom_l_page" +) + +const rowLineageContributorAlias = "p" + +const ( + rowLineageResourceIDColumn = "__loom_row_lineage_resource_id" + rowLineageOccurrenceKeyColumn = "__loom_row_lineage_occurrence_key" +) + +func rowLineageHiddenRead(projection ir.PhysicalProjection, name string, rootVariable string) bool { + if !projection.Hidden || projection.Name != name || projection.Expression != nil || + projection.PresenceOutput || projection.Presence != nil || + projection.Value.Variable != rootVariable || projection.Value.BindKey != "" { + return false + } + switch name { + case rowLineageResourceIDColumn: + return len(projection.Value.Path) == 2 && + projection.Value.Path[0] == "payload" && projection.Value.Path[1] == "id" + case rowLineageOccurrenceKeyColumn: + return len(projection.Value.Path) == 1 && projection.Value.Path[0] == "_key" + default: + return false + } +} + +func rowLineageTerminalMentions(plan ir.PhysicalPlan) bool { + if plan.StageSequence == nil { + return false + } + sequence := plan.StageSequence + terminal := sequence.RowLineageReturn + if terminal == nil || sequence.CellTraceReturn != nil || sequence.PopulationMappingReturn != nil { + return false + } + if terminal.Trace != nil || terminal.RowIDBindKey == "" || terminal.OffsetBindKey == "" || + terminal.LimitBindKey == "" || terminal.FetchLimitBindKey == "" || + terminal.ResourceType == "" || + terminal.ResourceIDColumn != rowLineageResourceIDColumn || + terminal.OccurrenceKeyColumn != rowLineageOccurrenceKeyColumn { + return false + } + if len(sequence.Stages) != 1 { + return false + } + stage := sequence.Stages[0] + return stage.Kind == ir.PhysicalStageGroupOp && stage.Group != nil +} + +func renderRowLineagePhysicalPlan(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + if err := storage.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := validateRowLineagePlanSurface(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := plan.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("validate physical plan: %w", err) + } + if plan.Engine != "" && plan.Engine != ir.PhysicalEngineAQL { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: engine %q has no PostgreSQL source; the source graph IR is backend-neutral and ClickHouse plans are not PostgreSQL-rendered", + ErrUnsupportedPostgresPlan, string(plan.Engine)) + } + if plan.ClickHouseCombine != nil || plan.ClickHousePrefix != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: ClickHouse combine/prefix payloads have no PostgreSQL rendering", ErrUnsupportedPostgresPlan) + } + if len(plan.DeferredExpressionLets) != 0 { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: completed plans must not carry deferred expression lets", ErrUnsupportedPostgresPlan) + } + renderer := &planRenderer{ + storage: storage, + rows: map[string]rowScope{}, + parameters: map[string]int{}, + canonicalAuthLets: map[string]string{}, + } + if err := renderer.loadScope(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + sql, err := renderer.renderRowLineageStatement(plan) + if err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return command.NewRenderedPostgresPhysicalPlan(sql, renderer.args, renderer.outputColumns) +} + +func validateRowLineagePlanSurface(plan ir.PhysicalPlan) error { + if plan.StageSequence == nil { + return fmt.Errorf("%w: row-lineage dispatch requires a stage sequence", ErrUnsupportedPostgresPlan) + } + sequence := plan.StageSequence + terminal := sequence.RowLineageReturn + if terminal == nil { + return fmt.Errorf("%w: row-lineage dispatch requires the bounded contributor-page terminal", ErrUnsupportedPostgresPlan) + } + if sequence.CellTraceReturn != nil { + return fmt.Errorf("%w: cell-trace returns have no PostgreSQL compiled route (cell_trace.go:54)", ErrUnsupportedPostgresPlan) + } + if sequence.PopulationMappingReturn != nil { + return fmt.Errorf("%w: population-mapping returns have no PostgreSQL compiled route (population_mapping.go:38)", ErrUnsupportedPostgresPlan) + } + if len(sequence.Stages) != 1 { + return fmt.Errorf("%w: the registered contributor-page slice admits exactly one GROUP construction stage; a composed lineage sequence (filter suffix, Pivot preimage) is a separate terminal registration (render/aql/row_lineage.go:532-600, :189+)", + ErrUnsupportedPostgresPlan, + ) + } + if terminal.Trace != nil { + return fmt.Errorf("%w: a traced row-lineage terminal (RELATED_EXPAND owner chain or Pivot preimage identities) has no PostgreSQL compiled route; it needs the Pivot and related-record terminal surfaces (render/aql/row_lineage_pivot.go, row_lineage_group_pivot.go, row_lineage_composed.go)", + ErrUnsupportedPostgresPlan) + } + if terminal.RowIDBindKey == "" || terminal.OffsetBindKey == "" || terminal.LimitBindKey == "" || terminal.FetchLimitBindKey == "" { + return fmt.Errorf("%w: the contributor-page terminal requires its row-id, offset, limit, and fetch-limit bind keys", ErrUnsupportedPostgresPlan) + } + if strings.TrimSpace(terminal.ResourceType) == "" { + return fmt.Errorf("%w: the contributor-page terminal requires its resource type", ErrUnsupportedPostgresPlan) + } + if terminal.ResourceIDColumn != rowLineageResourceIDColumn || terminal.OccurrenceKeyColumn != rowLineageOccurrenceKeyColumn { + return fmt.Errorf("%w: the contributor page addresses the compiler-minted identity columns %q and %q (compiler/row_lineage.go:23-24); a page addressing other columns has no registered rendering", + ErrUnsupportedPostgresPlan, rowLineageResourceIDColumn, rowLineageOccurrenceKeyColumn) + } + if _, ok := plan.BindVars[terminal.RowIDBindKey].(string); !ok { + return fmt.Errorf("%w: contributor-page row id bind %q must be a string", ErrUnsupportedPostgresPlan, terminal.RowIDBindKey) + } + for _, bind := range []struct { + key string + label string + }{ + {terminal.OffsetBindKey, "offset"}, + {terminal.LimitBindKey, "limit"}, + {terminal.FetchLimitBindKey, "fetch limit"}, + } { + if value, ok := plan.BindVars[bind.key].(int); !ok || value < 0 { + return fmt.Errorf("%w: contributor-page %s bind %q must be a non-negative int", ErrUnsupportedPostgresPlan, bind.label, bind.key) + } + } + limit := plan.BindVars[terminal.LimitBindKey].(int) + if limit < 1 { + return fmt.Errorf("%w: contributor-page limit bind %q must be at least 1", ErrUnsupportedPostgresPlan, terminal.LimitBindKey) + } + if plan.BindVars[terminal.FetchLimitBindKey].(int) <= limit { + return fmt.Errorf("%w: contributor-page fetch limit bind %q must exceed the limit (the hasMore lookahead)", ErrUnsupportedPostgresPlan, terminal.FetchLimitBindKey) + } + stage := sequence.Stages[0] + if stage.Kind != ir.PhysicalStageGroupOp || stage.Group == nil { + return fmt.Errorf("%w: operation %q has no PostgreSQL contributor-page rendering; only the ordinary GROUP terminal page is registered", + ErrUnsupportedPostgresPlan, string(stage.Kind)) + } + if stage.CodedGroup != nil || stage.CohortGroup != nil || stage.Filter != nil || stage.Expand != nil || + stage.GroupedPivot != nil || stage.Unpivot != nil || stage.RelatedSource != nil || + stage.RelatedExpand != nil || stage.RelatedField != nil || len(stage.DerivedLets) != 0 { + return fmt.Errorf("%w: GROUP requires only a group payload (ir/physical_stage_validation.go:187 re-read at the door)", + ErrUnsupportedPostgresPlan) + } + if stage.InputStageID != sequence.SourceStageID || stage.ID != sequence.FinalStageID { + return fmt.Errorf("%w: the registered slice admits only the inline first-stage GROUP (InputStageID must equal %q)", + ErrUnsupportedPostgresPlan, sequence.SourceStageID) + } + if stage.RowIdentityColumn == "" || stage.RowIdentityColumn != sequence.FinalRowIdentity { + return fmt.Errorf("%w: the contributor page is addressed by the final row identity, which stage %q does not carry", + ErrUnsupportedPostgresPlan, stage.ID) + } + group := stage.Group + if group.SemanticKey != nil { + return fmt.Errorf("%w: the semantic-family contributor page needs the semantic Group lineage renderer (render/aql/row_lineage.go:349+) and is refused, never partially rendered", + ErrUnsupportedPostgresPlan) + } + if group.CodingKey != nil { + return fmt.Errorf("%w: the CodingKey contributor page needs the coding-identity lineage renderer (render/aql/row_lineage_coded_group.go, row_lineage_related_coding.go) and is refused, never partially rendered", + ErrUnsupportedPostgresPlan) + } + if group.RelatedOccurrence != nil { + return fmt.Errorf("%w: the related-occurrence contributor page needs the correlated-hop contracts (L-13.5) and is refused, never partially rendered", + ErrUnsupportedPostgresPlan) + } + if group.RootContributorInputColumn != "" { + return fmt.Errorf("%w: the root-contributor set is production-unreachable through the PostgreSQL compile target (L-13.6) and is refused, never partially rendered", + ErrUnsupportedPostgresPlan) + } + if group.GroupRowsVariable == "" || group.IdentityVariable == "" { + return fmt.Errorf("%w: group plan variables must be compiled and non-empty", ErrUnsupportedPostgresPlan) + } + if value, ok := plan.BindVars[group.ConstructionIDBindKey].(string); !ok || strings.TrimSpace(value) == "" { + return fmt.Errorf("%w: group construction identity bind %q must be a non-empty string", + ErrUnsupportedPostgresPlan, group.ConstructionIDBindKey) + } + switch group.MissingKeyPolicy { + case ir.PhysicalStageGroupMissingKeyGroup, ir.PhysicalStageGroupMissingKeyExclude, ir.PhysicalStageGroupMissingKeyError: + default: + return fmt.Errorf("%w: unsupported missing-key policy %q (construction_group.go:131 parity)", + ErrUnsupportedPostgresPlan, string(group.MissingKeyPolicy)) + } + for index, key := range group.Keys { + switch key.Kind { + case "STRING", "INTEGER", "DECIMAL", "BOOLEAN": + default: + return fmt.Errorf("%w: group key %d has unsupported scalar kind %q", + ErrUnsupportedPostgresPlan, index, key.Kind) + } + if key.InputColumn == "" || key.OutputColumn == "" || key.Variable == "" { + return fmt.Errorf("%w: group key %d must be fully compiled (input, output, variable)", + ErrUnsupportedPostgresPlan, index) + } + if key.InputInternal || key.FieldSource != nil { + return fmt.Errorf("%w: group key %d uses a hidden or field-sourced input surface, which the public-scalar lowering gate never emits (L-13.11)", + ErrUnsupportedPostgresPlan, index) + } + } + if plan.PreviewSourceWindowByRootID || sequence.PreviewSourceWindowByRootID || sequence.PreviewTerminalPivotWindow { + return fmt.Errorf("%w: preview source windowing and the terminal Pivot window are terminal windowing contracts and are refused, never silently rendered", + ErrUnsupportedPostgresPlan) + } + if sequence.OutputAuthResourcePathBindKey != "" { + return fmt.Errorf("%w: the composite Combine prefix authorization path is not registered for the contributor page (ir/physical_stage_sequence.go:25-28; the auth path column registers with pg-ir-03)", + ErrUnsupportedPostgresPlan) + } + if sequence.PreviewLimitBindKey != "" { + if value, ok := plan.BindVars[sequence.PreviewLimitBindKey].(int); !ok || value <= 0 { + return fmt.Errorf("%w: contributor-page preview limit bind %q must be a positive int", + ErrUnsupportedPostgresPlan, sequence.PreviewLimitBindKey) + } + } + returnNames, err := validateGroupOperations(plan) + if err != nil { + return err + } + if err := validateGroupPayload(stage, plan.BindVars, returnNames); err != nil { + return err + } + rootVariable := plan.Operations[0].RootScan.Variable + hiddenSeen := map[string]bool{} + for _, operation := range plan.Operations { + if operation.Kind != ir.PhysicalReturnOp || operation.Return == nil { + continue + } + for _, projection := range operation.Return.Projections { + switch projection.Name { + case rowLineageResourceIDColumn, rowLineageOccurrenceKeyColumn: + if !rowLineageHiddenRead(projection, projection.Name, rootVariable) { + return fmt.Errorf("%w: contributor identity column %q is not the compiler-minted hidden root read (compiler/row_lineage.go:939-956)", + ErrUnsupportedPostgresPlan, projection.Name) + } + hiddenSeen[projection.Name] = true + } + } + } + for _, column := range []struct{ name, label string }{ + {rowLineageResourceIDColumn, "resource-id"}, + {rowLineageOccurrenceKeyColumn, "occurrence-key"}, + } { + if !hiddenSeen[column.name] || !returnNames[column.name] { + return fmt.Errorf("%w: contributor-page %s column %q is not among the source projections (compiler/row_lineage.go:939-956 mints it)", + ErrUnsupportedPostgresPlan, column.label, column.name) + } + } + return nil +} + +func (r *planRenderer) renderRowLineageStatement(plan ir.PhysicalPlan) (string, error) { + sequence := plan.StageSequence + terminal := *sequence.RowLineageReturn + stage := sequence.Stages[0] + group := stage.Group + dollar := func(index int) string { return "$" + strconv.Itoa(index) } + + if err := r.renderRootScan(plan, *plan.Operations[0].RootScan); err != nil { + return "", err + } + for _, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalFilterOp: + canonical, err := r.acceptCanonicalScopeFilter(operation.Filter) + if err != nil { + return "", err + } + if canonical { + continue + } + clause, err := r.renderFilter(plan, *operation.Filter) + if err != nil { + return "", err + } + r.where = append(r.where, clause) + case ir.PhysicalDerivedLetOp: + canonical, err := r.acceptCanonicalAuthLet(operation.DerivedLet) + if err != nil { + return "", err + } + if !canonical { + return "", fmt.Errorf("%w: operation %q is not part of the registered contributor-page source surface", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + } + for _, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalSortOp: + if err := r.renderWindowSort(plan, *operation.Sort); err != nil { + return "", err + } + case ir.PhysicalLimitOp: + if err := r.renderWindowLimit(plan, *operation.Limit); err != nil { + return "", err + } + } + } + var returnOp *ir.PhysicalReturn + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalReturnOp { + returnOp = operation.Return + } + } + kept := make([]ir.PhysicalProjection, 0, len(returnOp.Projections)) + for _, projection := range returnOp.Projections { + if projection.Name == rowLineageResourceIDColumn || projection.Name == rowLineageOccurrenceKeyColumn { + continue + } + kept = append(kept, projection) + } + filteredReturn := *returnOp + filteredReturn.Projections = kept + columns, err := r.materializeGroupSource(plan, &filteredReturn) + if err != nil { + return "", err + } + rootScope, ok := r.rows[plan.Operations[0].RootScan.Variable] + if !ok || rootScope.derivedValue != "" { + return "", fmt.Errorf("%w: the contributor page root row does not bind a readable relation", ErrUnsupportedPostgresPlan) + } + resourceExpr, err := r.documentPathExpression(rootScope.alias, []string{"id"}) + if err != nil { + return "", fmt.Errorf("contributor resource-id column: %w", err) + } + occurrenceExpr := rootScope.alias + "." + r.storage.Resource.MustColumn("logical_id").SQLText() + base := len(columns) + for i, minted := range []struct { + name string + expr string + jsonb bool + }{ + {rowLineageResourceIDColumn, resourceExpr, true}, + {rowLineageOccurrenceKeyColumn, occurrenceExpr, false}, + } { + alias := "c" + strconv.Itoa(base+i) + columns = append(columns, groupColumn{ + name: minted.name, alias: alias, expr: minted.expr, + ref: `"` + alias + `"`, jsonb: minted.jsonb, + }) + } + nameToColumn := map[string]*groupColumn{} + for i := range columns { + nameToColumn[columns[i].name] = &columns[i] + } + resourceIDColumn, err := groupResolveColumn(nameToColumn, terminal.ResourceIDColumn) + if err != nil { + return "", fmt.Errorf("contributor-page resource-id column: %w", err) + } + occurrenceColumn, err := groupResolveColumn(nameToColumn, terminal.OccurrenceKeyColumn) + if err != nil { + return "", fmt.Errorf("contributor-page occurrence-key column: %w", err) + } + + selectList := make([]string, 0, len(columns)) + for i := range columns { + selectList = append(selectList, columns[i].expr+` AS "`+columns[i].alias+`"`) + } + source := "SELECT " + strings.Join(selectList, ", ") + + " FROM " + r.storage.Resource.Identifier().SQLText() + " AS " + r.rootAlias + for _, join := range r.joins { + source += " " + join + } + for _, lateral := range r.laterals { + source += " " + lateral + } + if len(r.where) > 0 { + source += " WHERE " + strings.Join(r.where, " AND ") + } + source += " ORDER BY " + strings.Join(r.windowOrder(), ", ") + r.window.limitClause + + keyReads := make([]string, len(group.Keys)) + keyCollates := make([]string, len(group.Keys)) + var conjuncts []string + seenConjunct := map[string]bool{} + addConjunct := func(conjunct string) { + if seenConjunct[conjunct] { + return + } + seenConjunct[conjunct] = true + conjuncts = append(conjuncts, conjunct) + } + for i, key := range group.Keys { + column, err := groupResolveColumn(nameToColumn, key.InputColumn) + if err != nil { + return "", fmt.Errorf("group key %q: %w", key.OutputColumn, err) + } + typedRead, witness, err := r.groupColumnRead(column, key.Kind, groupKeyTypeTag, dollar) + if err != nil { + return "", fmt.Errorf("group key %q: %w", key.OutputColumn, err) + } + keyReads[i] = typedRead + if key.Kind == "STRING" { + keyCollates[i] = ` COLLATE "C"` + } + if witness != "" { + addConjunct(witness) + } + switch group.MissingKeyPolicy { + case ir.PhysicalStageGroupMissingKeyExclude: + addConjunct(typedRead + " IS NOT NULL") + case ir.PhysicalStageGroupMissingKeyError: + addConjunct("(CASE WHEN " + typedRead + " IS NOT NULL THEN 1 ELSE " + + guardCast(groupMissingKeyTag) + " END) > 0") + } + } + rows := "SELECT * FROM " + rowLineageSourceRelation + if len(conjuncts) > 0 { + rows += " WHERE " + strings.Join(conjuncts, " AND ") + } + + constructionID := r.textParameter(bindStringVar(plan.BindVars, group.ConstructionIDBindKey), "") + identityParts := []string{ + `'["construction",' || to_jsonb(` + dollar(constructionID) + "::text) || ']'", + `'["operation","GROUP"]'`, + } + outputIDs := map[string]string{} + for _, column := range stage.OutputColumns { + if _, seen := outputIDs[column.Name]; !seen { + outputIDs[column.Name] = column.ID + } + } + grpSelects := make([]string, 0, len(keyReads)+1) + keyAliases := make([]string, 0, len(keyReads)) + for i, key := range group.Keys { + columnID, ok := outputIDs[key.OutputColumn] + if !ok { + return "", fmt.Errorf("%w: group key output %q has no stable column ID (construction_group.go:136-142 parity)", + ErrUnsupportedPostgresPlan, key.OutputColumn) + } + idBind := r.textParameter(columnID, "") + alias := "l_k" + strconv.Itoa(i) + grpSelects = append(grpSelects, keyReads[i]+" AS "+alias) + keyAliases = append(keyAliases, alias) + identityParts = append(identityParts, + "'[' || to_jsonb("+dollar(idBind)+"::text) || ',' || COALESCE(to_jsonb("+keyReads[i]+")::text, 'null') || ']'") + } + identity := "'[' || " + strings.Join(identityParts, " || ',' || ") + " || ']'" + grp := "SELECT " + strings.Join(append(grpSelects, identity+" AS l_id"), ", ") + + " FROM " + rowLineageRowsRelation + if len(keyAliases) > 0 { + positions := make([]string, len(keyAliases)) + for i := range positions { + positions[i] = strconv.Itoa(i + 1) + } + grp += " GROUP BY " + strings.Join(positions, ", ") + } else { + grp += " HAVING COUNT(*) > 0" + } + + selected := "SELECT * FROM " + rowLineageGroupRelation + " WHERE l_id = " + + dollar(r.textParameter(bindStringVar(plan.BindVars, terminal.RowIDBindKey), "")) + order := make([]string, 0, len(keyAliases)+1) + for i, alias := range keyAliases { + order = append(order, alias+keyCollates[i]+" ASC NULLS FIRST") + } + order = append(order, `l_id COLLATE "C" ASC`) + selected += " ORDER BY " + strings.Join(order, ", ") + " LIMIT 1" + + contributorMatch := make([]string, 0, len(keyReads)) + for i, key := range group.Keys { + column, err := groupResolveColumn(nameToColumn, key.InputColumn) + if err != nil { + return "", fmt.Errorf("contributor key %q: %w", key.OutputColumn, err) + } + repointed := *column + repointed.ref = rowLineageContributorAlias + "." + column.ref + typedRead, _, err := r.groupColumnRead(&repointed, key.Kind, groupKeyTypeTag, dollar) + if err != nil { + return "", fmt.Errorf("contributor key %q: %w", key.OutputColumn, err) + } + contributorMatch = append(contributorMatch, + typedRead+" IS NOT DISTINCT FROM "+keyAliases[i]) + } + qualifiedOccurrence := rowLineageContributorAlias + "." + occurrenceColumn.ref + qualifiedResourceID := rowLineageContributorAlias + "." + resourceIDColumn.ref + resourceTypeBind := r.textParameter(terminal.ResourceType, "") + pageSelects := []string{ + dollar(resourceTypeBind) + "::text AS l_resource_type", + qualifiedResourceID + " AS l_resource_id", + qualifiedOccurrence + " AS l_occurrence_key", + } + page := "SELECT " + strings.Join(pageSelects, ", ") + " FROM " + + rowLineageRowsRelation + " AS " + rowLineageContributorAlias + ", " + rowLineageSelectedRow + if len(contributorMatch) > 0 { + page += " WHERE " + strings.Join(contributorMatch, " AND ") + } + page += " ORDER BY " + qualifiedOccurrence + ` COLLATE "C" ASC` + + " OFFSET " + dollar(r.integerParameter(plan.BindVars[terminal.OffsetBindKey].(int))) + + " LIMIT " + dollar(r.integerParameter(plan.BindVars[terminal.FetchLimitBindKey].(int))) + + limitBind := r.integerParameter(plan.BindVars[terminal.LimitBindKey].(int)) + contributors := "COALESCE((SELECT jsonb_agg(jsonb_build_object(" + + "'resourceType', capped.l_resource_type, " + + "'resourceId', capped.l_resource_id, " + + "'occurrenceKey', capped.l_occurrence_key" + + ") ORDER BY capped.l_occurrence_key" + ` COLLATE "C" ASC` + + ") FROM (SELECT * FROM " + rowLineagePageRelation + + " ORDER BY l_occurrence_key" + ` COLLATE "C" ASC` + + " LIMIT " + dollar(limitBind) + ") AS capped), '[]'::jsonb)" + found := "(SELECT COUNT(*) > 0 FROM " + rowLineageSelectedRow + ")" + hasMore := "(SELECT COUNT(*) > " + dollar(limitBind) + " FROM " + rowLineagePageRelation + ")" + r.outputColumns = []string{"found", "contributors", "hasMore"} + + ctes := []string{ + rowLineageSourceRelation + " AS (" + source + ")", + rowLineageRowsRelation + " AS (" + rows + ")", + rowLineageGroupRelation + " AS (" + grp + ")", + rowLineageSelectedRow + " AS (" + selected + ")", + rowLineagePageRelation + " AS (" + page + ")", + } + return "WITH " + strings.Join(ctes, ", ") + " SELECT " + + found + ` AS "found", ` + contributors + ` AS "contributors", ` + hasMore + ` AS "hasMore"`, nil +} + +func (r *planRenderer) integerParameter(value int) int { + return r.addParameter(command.PostgreSQLParameter{ + Type: command.PostgreSQLParameterBigint, + Value: int64(value), + }, "") +} diff --git a/internal/dataframe/compiler/render/postgres/row_lineage_test.go b/internal/dataframe/compiler/render/postgres/row_lineage_test.go new file mode 100644 index 000000000..920e2ff23 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/row_lineage_test.go @@ -0,0 +1,770 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "sync" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/jackc/pgx/v5/pgxpool" +) + +func sql23AttachTerminal(t *testing.T, plan ir.PhysicalPlan, rowID string, offset, limit int) ir.PhysicalPlan { + t.Helper() + plan = ir.ClonePhysicalPlan(plan) + if plan.StageSequence == nil || len(plan.StageSequence.Stages) != 1 || plan.StageSequence.Stages[0].Group == nil { + t.Fatalf("fixture is not the inline single-stage GROUP shape") + } + root := plan.Operations[0].RootScan.Variable + returns := 0 + for index := range plan.Operations { + operation := &plan.Operations[index] + if operation.Kind != ir.PhysicalReturnOp || operation.Return == nil { + continue + } + returns++ + projections := append([]ir.PhysicalProjection{}, operation.Return.Projections...) + projections = append(projections, + ir.PhysicalProjection{Name: rowLineageResourceIDColumn, Hidden: true, + Value: ir.PhysicalValue{Variable: root, Path: []string{"payload", "id"}}}, + ir.PhysicalProjection{Name: rowLineageOccurrenceKeyColumn, Hidden: true, + Value: ir.PhysicalValue{Variable: root, Path: []string{"_key"}}}, + ) + operation.Return.Projections = projections + } + if returns != 1 { + t.Fatalf("fixture has %d terminal projections, want 1", returns) + } + plan.BindVars["row_lineage_row_id"] = rowID + plan.BindVars["row_lineage_offset"] = offset + plan.BindVars["row_lineage_limit"] = limit + plan.BindVars["row_lineage_fetch_limit"] = limit + 1 + plan.StageSequence.RowLineageReturn = &ir.PhysicalRowLineageReturn{ + RowIDBindKey: "row_lineage_row_id", OffsetBindKey: "row_lineage_offset", + LimitBindKey: "row_lineage_limit", FetchLimitBindKey: "row_lineage_fetch_limit", + ResourceType: "Observation", ResourceIDColumn: rowLineageResourceIDColumn, + OccurrenceKeyColumn: rowLineageOccurrenceKeyColumn, + } + if err := plan.Validate(); err != nil { + t.Fatalf("plan with the lineage terminal must validate: %v", err) + } + return plan +} + +func sql23LineagePlan(t *testing.T, policy recipe.ConstructionGroupMissingKeyPolicy, rowID string, offset, limit int) ir.PhysicalPlan { + t.Helper() + return sql23AttachTerminal(t, group13CountRowsPlan(t, policy), rowID, offset, limit) +} + +func sql23Render(t *testing.T, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + return rendered +} + +func sql23ContributorCell(resourceType, resourceID, occurrenceKey string) string { + return fmt.Sprintf(`{"occurrenceKey":%q,"resourceId":%s,"resourceType":%q}`, + occurrenceKey, resourceID, resourceType) +} + +func sql23Contributors(t *testing.T, cell string) []string { + t.Helper() + var members []struct { + ResourceType string `json:"resourceType"` + ResourceID json.RawMessage `json:"resourceId"` + OccurrenceKey string `json:"occurrenceKey"` + } + if err := json.Unmarshal([]byte(cell), &members); err != nil { + t.Fatalf("decode contributors %q: %v", cell, err) + } + out := make([]string, len(members)) + for i, member := range members { + out[i] = sql23ContributorCell(member.ResourceType, string(member.ResourceID), member.OccurrenceKey) + } + return out +} + +func TestRowLineageOfflineStatementShape(t *testing.T) { + plan := sql23LineagePlan(t, recipe.ConstructionGroupMissingKeyGroup, "opaque-row-id", 25, 10) + rendered := sql23Render(t, plan) + sqlText := rendered.SQL + for _, want := range []string{ + "WITH __loom_l_src AS (", "__loom_l_rows AS (", "__loom_l_grp AS (", + "__loom_l_sel AS (", "__loom_l_page AS (", + `IS NOT DISTINCT FROM`, + `COLLATE "C" ASC`, + `"found"`, `"contributors"`, `"hasMore"`, + "'resourceType'", "'resourceId'", "'occurrenceKey'", + `["operation","GROUP"]`, `'["construction",'`, + `OFFSET $`, `LIMIT $`, + "ORDER BY l_occurrence_key", "ASC NULLS FIRST", + } { + if !strings.Contains(sqlText, want) { + t.Errorf("statement is missing %q:\n%s", want, sqlText) + } + } + for _, forbidden := range []string{"COLLECT", "@row_lineage", "AQL", "OFFSET @", "LIMIT @", " FOR "} { + if strings.Contains(sqlText, forbidden) { + t.Errorf("statement carries %q (AQL surface leaked):\n%s", forbidden, sqlText) + } + } + var sawIDPath bool + for _, parameter := range rendered.Args { + if text, ok := parameter.Value.(string); ok && text == "{id}" { + sawIDPath = true + } + } + if !sawIDPath { + t.Errorf("the resource-id read must bind the payload-relative {id} path:\n%s", sqlText) + } + if !strings.Contains(sqlText, `"logical_id" AS "c2"`) && !strings.Contains(sqlText, `"logical_id" AS "c1"`) { + if !strings.Contains(sqlText, `"logical_id"`) { + t.Errorf("the occurrence key must read the typed logical_id column:\n%s", sqlText) + } + } + var bigints []int64 + for _, parameter := range rendered.Args { + if parameter.Type == command.PostgreSQLParameterBigint { + bigints = append(bigints, parameter.Value.(int64)) + } + } + if len(bigints) != 3 || bigints[0] != 25 || bigints[1] != 11 || bigints[2] != 10 { + t.Fatalf("page coordinate parameters = %v, want [25 11 10]", bigints) + } + if strings.Join(rendered.OutputColumns, ",") != "found,contributors,hasMore" { + t.Fatalf("output columns = %v", rendered.OutputColumns) + } + second := sql23Render(t, plan) + if second.SQL != sqlText { + t.Fatalf("re-render is not byte-identical") + } + if fmt.Sprint(second.Args) != fmt.Sprint(rendered.Args) { + t.Fatalf("re-render arguments drifted") + } +} + +func TestRowLineageRefusalRegister(t *testing.T) { + cases := []struct { + name string + mutate func(t *testing.T, plan *ir.PhysicalPlan) + want string + }{ + {"row-id-not-string", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.BindVars["row_lineage_row_id"] = 7 + }, "row id bind"}, + {"negative-offset", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.BindVars["row_lineage_offset"] = -1 + }, "offset bind"}, + {"zero-limit", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.BindVars["row_lineage_limit"] = 0 + plan.BindVars["row_lineage_fetch_limit"] = 1 + }, "limit bind"}, + {"fetch-not-ahead", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.BindVars["row_lineage_limit"] = 5 + plan.BindVars["row_lineage_fetch_limit"] = 5 + }, "fetch limit bind"}, + {"resource-type-blank", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.RowLineageReturn.ResourceType = " " + }, "requires its resource type"}, + {"semantic-key", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.SemanticKey = &ir.PhysicalStageGroupSemanticKey{} + }, "semantic-family contributor page"}, + {"coding-key", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.CodingKey = &ir.PhysicalStageGroupCodingKey{} + }, "CodingKey contributor page"}, + {"related-occurrence", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.RelatedOccurrence = &ir.PhysicalStageGroupRelatedOccurrenceAdapter{} + }, "related-occurrence contributor page"}, + {"root-contributor", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.RootContributorInputColumn = "keys" + }, "root-contributor set"}, + {"missing-key-policy", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.MissingKeyPolicy = "DROP" + }, "unsupported missing-key policy"}, + {"key-kind", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.Keys[0].Kind = "ARRAY" + }, "unsupported scalar kind"}, + {"key-uncompiled", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.Keys[0].InputColumn = "" + }, "must be fully compiled"}, + {"key-hidden", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.Keys[0].InputInternal = true + }, "hidden or field-sourced input surface"}, + {"preview-window-plan", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.PreviewSourceWindowByRootID = true + }, "preview source windowing"}, + {"preview-window-sequence", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.PreviewSourceWindowByRootID = true + }, "preview source windowing"}, + {"auth-path-bind", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.OutputAuthResourcePathBindKey = "output_auth_resource_path" + }, "composite Combine prefix authorization path"}, + {"preview-limit-zero", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.PreviewLimitBindKey = "limit" + plan.BindVars["limit"] = 0 + }, "preview limit bind"}, + {"construction-blank", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.BindVars[plan.StageSequence.Stages[0].Group.ConstructionIDBindKey] = " " + }, "construction identity bind"}, + {"row-identity-drift", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].RowIdentityColumn = "somewhere-else" + }, "addressed by the final row identity"}, + {"hidden-column-missing", func(t *testing.T, plan *ir.PhysicalPlan) { + for index := range plan.Operations { + operation := &plan.Operations[index] + if operation.Kind == ir.PhysicalReturnOp && operation.Return != nil { + kept := operation.Return.Projections[:0:0] + for _, projection := range operation.Return.Projections { + if projection.Name != rowLineageOccurrenceKeyColumn { + kept = append(kept, projection) + } + } + operation.Return.Projections = kept + } + } + }, "occurrence-key column"}, + {"hidden-column-wrong-shape", func(t *testing.T, plan *ir.PhysicalPlan) { + for index := range plan.Operations { + operation := &plan.Operations[index] + if operation.Kind == ir.PhysicalReturnOp && operation.Return != nil { + for projectionIndex := range operation.Return.Projections { + projection := &operation.Return.Projections[projectionIndex] + if projection.Name == rowLineageResourceIDColumn { + projection.Value.Path = []string{"id"} + } + } + } + } + }, "not the compiler-minted hidden root read"}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + plan := sql23LineagePlan(t, recipe.ConstructionGroupMissingKeyGroup, "opaque-row-id", 0, 5) + testCase.mutate(t, &plan) + _, entryErr := RenderPhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + _, dispatchErr := renderRowLineagePhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + for label, err := range map[string]error{"entry": entryErr, "dispatch": dispatchErr} { + if err == nil { + t.Fatalf("%s must refuse", label) + } + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("%s refusal must be typed: %v", label, err) + } + if !strings.Contains(err.Error(), testCase.want) { + t.Fatalf("%s refusal %q lacks %q", label, err, testCase.want) + } + } + if entryErr.Error() != dispatchErr.Error() { + t.Fatalf("entry/dispatch voices differ:\nentry=%v\ndispatch=%v", entryErr, dispatchErr) + } + }) + } +} + +func TestRowLineageUnclaimedShapesDualVoices(t *testing.T) { + cases := []struct { + name string + mutate func(t *testing.T, plan *ir.PhysicalPlan) + entryWant string + dispatchWant string + }{ + {"stages-composed", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages = append(plan.StageSequence.Stages, ir.ClonePhysicalPlan(*plan).StageSequence.Stages[0]) + }, "row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route", + "admits exactly one GROUP construction stage"}, + {"traced-terminal", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.RowLineageReturn.Trace = &ir.PhysicalRowLineageTrace{RootKeyBindKey: "row_lineage_root_key"} + }, "row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route", + "traced row-lineage terminal"}, + {"missing-bind-keys", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.RowLineageReturn.OffsetBindKey = "" + }, "row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route", + "requires its row-id, offset, limit, and fetch-limit bind keys"}, + {"identity-columns-renamed", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.RowLineageReturn.ResourceIDColumn = "mine" + }, "row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route", + "compiler-minted identity columns"}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + plan := sql23LineagePlan(t, recipe.ConstructionGroupMissingKeyGroup, "opaque-row-id", 0, 5) + testCase.mutate(t, &plan) + _, entryErr := RenderPhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if entryErr == nil || !errors.Is(entryErr, ErrUnsupportedPostgresPlan) || + !strings.Contains(entryErr.Error(), testCase.entryWant) { + t.Fatalf("entry must keep the merged voice %q: %v", testCase.entryWant, entryErr) + } + _, dispatchErr := renderRowLineagePhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if dispatchErr == nil || !errors.Is(dispatchErr, ErrUnsupportedPostgresPlan) || + !strings.Contains(dispatchErr.Error(), testCase.dispatchWant) { + t.Fatalf("direct door call must speak the slice contract %q: %v", testCase.dispatchWant, dispatchErr) + } + }) + } +} + +func TestRowLineageDoorClaimsNothingElse(t *testing.T) { + t.Run("cell-trace-companion", func(t *testing.T) { + plan := sql23LineagePlan(t, recipe.ConstructionGroupMissingKeyGroup, "opaque-row-id", 0, 5) + plan.StageSequence.CellTraceReturn = &ir.PhysicalCellTraceReturn{} + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil || !strings.Contains(err.Error(), "row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route") { + t.Fatalf("companion cell-trace must keep the merged door voice: %v", err) + } + _, dispatchErr := renderRowLineagePhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if dispatchErr == nil || !strings.Contains(dispatchErr.Error(), "cell-trace returns have no PostgreSQL compiled route") { + t.Fatalf("direct call must name the companion contract: %v", dispatchErr) + } + }) + t.Run("population-mapping-companion", func(t *testing.T) { + plan := sql23LineagePlan(t, recipe.ConstructionGroupMissingKeyGroup, "opaque-row-id", 0, 5) + plan.StageSequence.PopulationMappingReturn = &ir.PhysicalStagePopulationMappingReturn{} + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil || !strings.Contains(err.Error(), "row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route") { + t.Fatalf("companion population-mapping must keep the merged door voice: %v", err) + } + _, dispatchErr := renderRowLineagePhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if dispatchErr == nil || !strings.Contains(dispatchErr.Error(), "population-mapping returns have no PostgreSQL compiled route") { + t.Fatalf("direct call must name the companion contract: %v", dispatchErr) + } + }) + t.Run("traced-falls-to-registered-voice", func(t *testing.T) { + plan := sql23LineagePlan(t, recipe.ConstructionGroupMissingKeyGroup, "opaque-row-id", 0, 5) + plan.StageSequence.RowLineageReturn.Trace = &ir.PhysicalRowLineageTrace{RootKeyBindKey: "row_lineage_root_key"} + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil || !strings.Contains(err.Error(), "row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route") { + t.Fatalf("a traced page must fall through to the registered refusal: %v", err) + } + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("fall-through refusal must stay typed: %v", err) + } + }) + t.Run("hand-assembled-empty-terminal-falls-through", func(t *testing.T) { + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + plan.StageSequence.RowLineageReturn = &ir.PhysicalRowLineageReturn{} + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil || !strings.Contains(err.Error(), "row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route") { + t.Fatalf("an unclaimed terminal must keep the merged voice: %v", err) + } + }) +} + +func TestRowLineagePolicyAndOrderArms(t *testing.T) { + t.Run("exclude-drops-null-key-rows", func(t *testing.T) { + rendered := sql23Render(t, sql23LineagePlan(t, recipe.ConstructionGroupMissingKeyExclude, "opaque-row-id", 0, 5)) + if !strings.Contains(rendered.SQL, `IS NOT NULL`) { + t.Fatalf("EXCLUDE must drop null-key rows with a typed-read IS NOT NULL conjunct:\n%s", rendered.SQL) + } + }) + t.Run("error-policy-raises-through-the-guard", func(t *testing.T) { + rendered := sql23Render(t, sql23LineagePlan(t, recipe.ConstructionGroupMissingKeyError, "opaque-row-id", 0, 5)) + if !strings.Contains(rendered.SQL, groupMissingKeyTag) || !strings.Contains(rendered.SQL, "guard_barrier") { + t.Fatalf("ERROR must raise %q through the guard barrier:\n%s", groupMissingKeyTag, rendered.SQL) + } + }) + t.Run("key-type-witness-present", func(t *testing.T) { + rendered := sql23Render(t, sql23LineagePlan(t, recipe.ConstructionGroupMissingKeyGroup, "opaque-row-id", 0, 5)) + if !strings.Contains(rendered.SQL, groupKeyTypeTag) { + t.Fatalf("the STRING key must carry the registered type witness:\n%s", rendered.SQL) + } + }) + t.Run("keyless-group-exists-only-with-rows", func(t *testing.T) { + plan := sql23AttachTerminal(t, group13KeylessPlan(t), "opaque-row-id", 0, 5) + rendered := sql23Render(t, plan) + if !strings.Contains(rendered.SQL, "HAVING COUNT(*) > 0") { + t.Fatalf("the keyless group must exist only when at least one source row does (AQL COLLECT WITH COUNT INTO shape):\n%s", rendered.SQL) + } + }) + t.Run("page-capped-at-limit-not-fetch", func(t *testing.T) { + rendered := sql23Render(t, sql23LineagePlan(t, recipe.ConstructionGroupMissingKeyGroup, "opaque-row-id", 3, 7)) + if strings.Count(rendered.SQL, "LIMIT $") != 2 || !strings.Contains(rendered.SQL, "LIMIT 1") { + t.Fatalf("statement must bound the page at the fetch limit, cap the published array at the limit, and select one group:\n%s", rendered.SQL) + } + var bigints []int64 + for _, parameter := range rendered.Args { + if parameter.Type == command.PostgreSQLParameterBigint { + bigints = append(bigints, parameter.Value.(int64)) + } + } + if len(bigints) != 3 || bigints[0] != 3 || bigints[1] != 8 || bigints[2] != 7 { + t.Fatalf("page coordinates = %v, want [3 8 7]", bigints) + } + }) +} + +const sql23Namespace = "verify_sql23v1_" + +var ( + sql23FixturesMu sync.Mutex + sql23TrackedFixtures = map[string]bool{} +) + +func sql23TrackFixtures(t *testing.T, svc *pgir02Service) { + t.Helper() + var database string + if err := svc.iso.QueryRow(svc.ctx, "SELECT current_database()").Scan(&database); err != nil { + svc.fail("track owned database", err) + } + sql23FixturesMu.Lock() + defer sql23FixturesMu.Unlock() + sql23TrackedFixtures[database] = true + sql23TrackedFixtures[svc.schema] = true +} + +func sql23SnapshotOwnedResources(ctx context.Context, raw string) ([]string, error) { + pool, err := pgxpool.New(ctx, raw) + if err != nil { + return nil, fmt.Errorf("connect: %w", err) + } + defer pool.Close() + rows, err := pool.Query(ctx, + `SELECT datname FROM pg_database WHERE datname LIKE 'verify_sql23v1_%'`+ + ` UNION ALL SELECT nspname FROM pg_namespace WHERE nspname LIKE 'verify_sql23v1_%'`) + if err != nil { + return nil, fmt.Errorf("snapshot: %w", err) + } + defer rows.Close() + var names []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("scan: %v", err) + } + names = append(names, name) + } + return names, rows.Err() +} + +func sql23RunDeltaAudit(inner func() int) int { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return inner() + } + beforeList, err := sql23SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-23 delta audit: snapshot before run: %v\n", err) + return inner() + } + if len(beforeList) > 0 { + fmt.Fprintf(os.Stderr, "pg-sql-23 delta audit: pre-snapshot found %d surviving verify_sql23v1_* resources before this run: %v\n", len(beforeList), beforeList) + return 1 + } + before := make(map[string]bool, len(beforeList)) + for _, name := range beforeList { + before[name] = true + } + code := inner() + after, err := sql23SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-23 delta audit: snapshot after run: %v\n", err) + code = 1 + } + sql23FixturesMu.Lock() + defer sql23FixturesMu.Unlock() + for _, name := range after { + if before[name] { + continue + } + if sql23TrackedFixtures[name] { + fmt.Fprintf(os.Stderr, "pg-sql-23 delta audit: %q was created and tracked by this run and still survives\n", name) + code = 1 + continue + } + fmt.Fprintf(os.Stderr, "pg-sql-23 delta audit: note — %q was born during this run but is not owned by it (sibling lane?)\n", name) + } + var survivors []string + for _, name := range after { + if !before[name] && sql23TrackedFixtures[name] { + survivors = append(survivors, name) + } + } + if len(survivors) > 0 { + fmt.Fprintf(os.Stderr, "pg-sql-23 delta audit: %d verify_sql23v1_* resources survive: %v\n", len(survivors), survivors) + code = 1 + } + return code +} + +func newSQL23Service(t *testing.T) *pgir02Service { + t.Helper() + return newOwnedService(t, 0, "verify_sql23v1") +} + +func sql23Insert(t *testing.T, svc *pgir02Service, logicalID string, payload map[string]any) { + t.Helper() + svc.insertResource(t, svc.schema, group13Project, group13Generation, "Observation", logicalID, group13AuthPath, payload) +} + +func sql23Payload(id, status string) map[string]any { + row := map[string]any{} + if id != "" { + row["id"] = id + } + if status != "" { + row["status"] = status + } + return row +} + +func sql23PreviewIdentity(t *testing.T, svc *pgir02Service, plan ir.PhysicalPlan, statusKey string) string { + t.Helper() + preview := ir.ClonePhysicalPlan(plan) + preview.StageSequence.RowLineageReturn = nil + rendered := group13RenderSvc(t, svc, preview) + rows := svc.runRendered(t, rendered) + for _, row := range rows { + cell := row[0] + if statusKey == "" && cell != "" { + continue + } + if statusKey != "" && cell != statusKey { + continue + } + return row[len(row)-1] + } + t.Fatalf("preview published no group with key %q: %v", statusKey, rows) + return "" +} + +func sql23RunPage(t *testing.T, svc *pgir02Service, plan ir.PhysicalPlan) (found, hasMore string, contributors []string) { + t.Helper() + rendered := group13RenderSvc(t, svc, plan) + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + rows, err := svc.iso.Query(svc.ctx, + `SELECT w."found"::text, w."contributors"::text, w."hasMore"::text FROM (`+rendered.SQL+`) AS w`, args...) + if err != nil { + t.Fatalf("execute contributor page: %v", err) + } + defer rows.Close() + var foundCells, contributorCells, hasMoreCells []string + for rows.Next() { + values, err := rows.Values() + if err != nil { + t.Fatalf("read contributor page row: %v", err) + } + if len(values) != 3 { + t.Fatalf("contributor page row must carry found/contributors/hasMore: %v", values) + } + foundCells = append(foundCells, stringifyCell(values[0])) + contributorCells = append(contributorCells, stringifyCell(values[1])) + hasMoreCells = append(hasMoreCells, stringifyCell(values[2])) + } + if err := rows.Err(); err != nil { + t.Fatalf("finish contributor page: %v", err) + } + if len(foundCells) != 1 { + t.Fatalf("the contributor page must return exactly one found/contributors/hasMore row: %d", len(foundCells)) + } + return foundCells[0], hasMoreCells[0], sql23Contributors(t, contributorCells[0]) +} + +func TestRowLineageLivePreviewRoundTrip(t *testing.T) { + svc := newSQL23Service(t) + sql23TrackFixtures(t, svc) + sql23Insert(t, svc, "obs-a", sql23Payload("obs-a", "final")) + sql23Insert(t, svc, "obs-b", sql23Payload("obs-b", "final")) + sql23Insert(t, svc, "obs-c", sql23Payload("obs-c", "amended")) + sql23Insert(t, svc, "obs-d", map[string]any{"id": nil, "status": "final"}) + + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + identity := sql23PreviewIdentity(t, svc, plan, "final") + if want := group13Identity(t, plan, []group13Member{group13TextMember("status_group", "final")}); identity != want { + t.Fatalf("preview identity %q, want %q", identity, want) + } + + page := sql23AttachTerminal(t, plan, identity, 0, 10) + found, hasMore, contributors := sql23RunPage(t, svc, page) + if found != "true" || hasMore != "false" { + t.Fatalf("found/hasMore = %s/%s, want true/false", found, hasMore) + } + want := []string{ + sql23ContributorCell("Observation", `"obs-a"`, "obs-a"), + sql23ContributorCell("Observation", `"obs-b"`, "obs-b"), + sql23ContributorCell("Observation", `null`, "obs-d"), + } + if strings.Join(contributors, "|") != strings.Join(want, "|") { + t.Fatalf("contributors mismatch:\n got %v\nwant %v", contributors, want) + } + + if again, _, againMembers := sql23RunPage(t, svc, page); again != "true" || strings.Join(againMembers, "|") != strings.Join(want, "|") { + t.Fatalf("page drifted between executions: %v", againMembers) + } + + rendered := group13RenderSvc(t, svc, page) + for _, leaked := range []string{group13Project, group13Generation, group13AuthPath, "final", "amended", identity} { + if strings.Contains(rendered.SQL, leaked) { + t.Fatalf("value %q leaked into statement text: %s", leaked, rendered.SQL) + } + } + + missing := sql23AttachTerminal(t, plan, `[["construction","pg13_group"],["operation","GROUP"],["status_group","cancelled"]]`, 0, 10) + found, hasMore, contributors = sql23RunPage(t, svc, missing) + if found != "false" || hasMore != "false" || strings.Join(contributors, "|") != "" { + t.Fatalf("missing row must read found=false with an empty array: %s %s %v", found, hasMore, contributors) + } +} + +func TestRowLineageLiveNullKeyGroup(t *testing.T) { + svc := newSQL23Service(t) + sql23TrackFixtures(t, svc) + sql23Insert(t, svc, "obs-n1", sql23Payload("obs-n1", "")) + sql23Insert(t, svc, "obs-n2", map[string]any{"id": "obs-n2", "status": nil}) + sql23Insert(t, svc, "obs-k", sql23Payload("obs-k", "final")) + + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + identity := sql23PreviewIdentity(t, svc, plan, "") + if want := group13Identity(t, plan, []group13Member{group13NullMember("status_group")}); identity != want { + t.Fatalf("null-key identity %q, want %q", identity, want) + } + found, hasMore, contributors := sql23RunPage(t, svc, sql23AttachTerminal(t, plan, identity, 0, 10)) + if found != "true" || hasMore != "false" { + t.Fatalf("the null-key group must resolve: %s/%s", found, hasMore) + } + want := []string{ + sql23ContributorCell("Observation", `"obs-n1"`, "obs-n1"), + sql23ContributorCell("Observation", `"obs-n2"`, "obs-n2"), + } + if strings.Join(contributors, "|") != strings.Join(want, "|") { + t.Fatalf("null-key contributors (AQL == is null-equal; SQL = would drop them):\n got %v\nwant %v", contributors, want) + } + + found, hasMore, contributors = sql23RunPage(t, svc, sql23AttachTerminal(t, + group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyExclude), identity, 0, 10)) + if found != "false" || hasMore != "false" || strings.Join(contributors, "|") != "" { + t.Fatalf("EXCLUDE must erase the null-key group: %s %s %v", found, hasMore, contributors) + } +} + +func TestRowLineageLivePaging(t *testing.T) { + svc := newSQL23Service(t) + sql23TrackFixtures(t, svc) + for _, id := range []string{"p-a", "p-b", "p-c", "p-d", "p-e"} { + sql23Insert(t, svc, id, sql23Payload(id, "final")) + } + svc.insertResource(t, svc.schema, group13Project, group13Generation, "Observation", "w-denied", "FHIR/Hidden", sql23Payload("w-denied", "final")) + svc.insertResource(t, svc.schema, group13Project, "p23-old-gen", "Observation", "w-oldgen", group13AuthPath, sql23Payload("w-oldgen", "final")) + svc.insertResource(t, svc.schema, "p23-foreign", group13Generation, "Observation", "w-foreign", group13AuthPath, sql23Payload("w-foreign", "final")) + svc.insertResource(t, svc.schema, group13Project, group13Generation, "Patient", "w-patient", group13AuthPath, sql23Payload("w-patient", "final")) + + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + identity := sql23PreviewIdentity(t, svc, plan, "final") + + cells := func(ids ...string) string { + parts := make([]string, len(ids)) + for i, id := range ids { + parts[i] = sql23ContributorCell("Observation", `"`+id+`"`, id) + } + return strings.Join(parts, "|") + } + cases := []struct { + offset, limit int + want string + hasMore string + }{ + {0, 2, cells("p-a", "p-b"), "true"}, + {2, 2, cells("p-c", "p-d"), "true"}, + {4, 2, cells("p-e"), "false"}, + {5, 2, "", "false"}, + {0, 4, cells("p-a", "p-b", "p-c", "p-d"), "true"}, + {0, 5, cells("p-a", "p-b", "p-c", "p-d", "p-e"), "false"}, + {0, 100, cells("p-a", "p-b", "p-c", "p-d", "p-e"), "false"}, + } + for _, testCase := range cases { + found, hasMore, contributors := sql23RunPage(t, svc, sql23AttachTerminal(t, plan, identity, testCase.offset, testCase.limit)) + if found != "true" { + t.Fatalf("offset %d limit %d: found = %s, want true", testCase.offset, testCase.limit, found) + } + if hasMore != testCase.hasMore { + t.Fatalf("offset %d limit %d: hasMore = %s, want %s", testCase.offset, testCase.limit, hasMore, testCase.hasMore) + } + if got := strings.Join(contributors, "|"); got != testCase.want { + t.Fatalf("offset %d limit %d: contributors\n got %q\nwant %q", testCase.offset, testCase.limit, got, testCase.want) + } + if len(contributors) > testCase.limit { + t.Fatalf("page exceeded the limit: %d > %d", len(contributors), testCase.limit) + } + } + + if _, _, all := sql23RunPage(t, svc, sql23AttachTerminal(t, + group13KeylessPlan(t), group13Identity(t, group13KeylessPlan(t), nil), 0, 100)); len(all) != 5 { + t.Fatalf("keyless page must hold exactly the in-scope rows: %v", all) + } +} + +func TestRowLineageLiveCollationOrder(t *testing.T) { + svc := newSQL23Service(t) + sql23TrackFixtures(t, svc) + for _, id := range []string{"k-B", "k-C", "k-a", "k-b"} { + sql23Insert(t, svc, id, sql23Payload(id, "case")) + } + + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + identity := sql23PreviewIdentity(t, svc, plan, "case") + + cases := []struct { + offset, limit int + want []string + hasMore string + }{ + {0, 1, []string{"k-B"}, "true"}, + {0, 2, []string{"k-B", "k-C"}, "true"}, + {1, 1, []string{"k-C"}, "true"}, + {0, 4, []string{"k-B", "k-C", "k-a", "k-b"}, "false"}, + } + for _, testCase := range cases { + found, hasMore, contributors := sql23RunPage(t, svc, + sql23AttachTerminal(t, plan, identity, testCase.offset, testCase.limit)) + if found != "true" { + t.Fatalf("collation page not found: offset %d limit %d", testCase.offset, testCase.limit) + } + if hasMore != testCase.hasMore { + t.Fatalf("offset %d limit %d: hasMore = %s, want %s", testCase.offset, testCase.limit, hasMore, testCase.hasMore) + } + if got := strings.Join(contributors, "|"); got != strings.Join(cellsOf(testCase.want), "|") { + t.Fatalf("offset %d limit %d: collation order\n got %v\nwant %v", testCase.offset, testCase.limit, contributors, testCase.want) + } + } +} + +func cellsOf(ids []string) []string { + out := make([]string, len(ids)) + for i, id := range ids { + out[i] = sql23ContributorCell("Observation", `"`+id+`"`, id) + } + return out +} + +func TestRowLineageLiveGuardRaises(t *testing.T) { + svc := newSQL23Service(t) + sql23TrackFixtures(t, svc) + sql23Insert(t, svc, "ok-1", sql23Payload("ok-1", "final")) + sql23Insert(t, svc, "ok-2", sql23Payload("ok-2", "final")) + + base := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + identity := sql23PreviewIdentity(t, svc, base, "final") + + if found, _, members := sql23RunPage(t, svc, sql23AttachTerminal(t, base, identity, 0, 10)); found != "true" || len(members) != 2 { + t.Fatalf("healthy page: %s %v", found, members) + } + + sql23Insert(t, svc, "poison-null", sql23Payload("poison-null", "")) + errorPlan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyError) + group13ExpectRaise(t, svc, group13RenderSvc(t, svc, sql23AttachTerminal(t, errorPlan, identity, 0, 10)), groupMissingKeyTag) + + sql23Insert(t, svc, "poison-type", map[string]any{"id": "poison-type", "status": 7}) + group13ExpectRaise(t, svc, group13RenderSvc(t, svc, sql23AttachTerminal(t, + group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup), identity, 0, 10)), groupKeyTypeTag) +} diff --git a/internal/dataframe/compiler/render/postgres/scope.go b/internal/dataframe/compiler/render/postgres/scope.go new file mode 100644 index 000000000..433e9d7d0 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/scope.go @@ -0,0 +1,108 @@ +package postgres + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const ( + projectBindKey = "project" + datasetGenerationBind = "dataset_generation" + authPathsBindKey = "auth_resource_paths" + authUnrestrictedBind = "auth_resource_paths_unrestricted" + scopeAllowedBind = "scope_allowed" +) + +type planScope struct { + projectIndex int + generationIndex int + authPathsIndex int + denied bool +} + +func (r *planRenderer) loadScope(plan ir.PhysicalPlan) error { + project, ok := plan.BindVars[projectBindKey].(string) + if !ok || strings.TrimSpace(project) == "" { + return fmt.Errorf("%w: %q bind must be a non-empty string", ErrInvalidPostgresPlanScope, projectBindKey) + } + generation, ok := plan.BindVars[datasetGenerationBind].(string) + if !ok { + return fmt.Errorf("%w: %q bind must be a string (the empty string is the legacy namespace)", + ErrInvalidPostgresPlanScope, datasetGenerationBind) + } + unrestricted, ok := plan.BindVars[authUnrestrictedBind].(bool) + if !ok { + return fmt.Errorf("%w: %q bind must be set explicitly; there is no implicit auth scope", + ErrInvalidPostgresPlanScope, authUnrestrictedBind) + } + paths, ok := plan.BindVars[authPathsBindKey].([]string) + if !ok { + return fmt.Errorf("%w: %q bind must be a []string", ErrInvalidPostgresPlanScope, authPathsBindKey) + } + if unrestricted && len(paths) != 0 { + return fmt.Errorf("%w: unrestricted mode must not carry auth resource paths (%d given)", + ErrInvalidPostgresPlanScope, len(paths)) + } + allowed, ok := plan.BindVars[scopeAllowedBind].(bool) + if !ok { + return fmt.Errorf("%w: %q bind must be set explicitly to a boolean; false is the canonical deny state (zero rows) and absence or a mistyped value is a plan defect", + ErrInvalidPostgresPlanScope, scopeAllowedBind) + } + + r.scope = planScope{ + projectIndex: r.textParameter(project, "text:"+project), + generationIndex: r.textParameter(generation, "text:"+generation), + denied: !allowed, + } + if !unrestricted { + copied := append([]string(nil), paths...) + raw, err := json.Marshal(copied) + if err != nil { + return fmt.Errorf("%w: auth resource paths are not JSON-encodable: %v", ErrInvalidPostgresPlanScope, err) + } + if string(raw) == "null" { + raw = json.RawMessage("[]") + } + r.scope.authPathsIndex = r.jsonbParameter(raw) + } + return nil +} + +func (r *planRenderer) scopedResourcePredicate(alias string, typeIndex int) string { + return r.scopedResourcePredicateDenyConjunct(alias, typeIndex, " AND FALSE") +} + +func (r *planRenderer) scopedResourcePredicateDenyConjunct(alias string, typeIndex int, denyConjunct string) string { + var builder strings.Builder + builder.WriteString(alias + "." + r.storage.Resource.MustColumn("project").SQLText() + " = $" + strconv.Itoa(r.scope.projectIndex)) + builder.WriteString(" AND " + alias + "." + r.storage.Resource.MustColumn("dataset_generation").SQLText() + " = $" + strconv.Itoa(r.scope.generationIndex)) + if r.scope.authPathsIndex != 0 { + builder.WriteString(" AND " + alias + "." + r.storage.Resource.MustColumn("auth_resource_path").SQLText()) + builder.WriteString(" = ANY (SELECT jsonb_array_elements_text($" + strconv.Itoa(r.scope.authPathsIndex) + "::jsonb))") + } + if typeIndex != 0 { + builder.WriteString(" AND " + alias + "." + r.storage.Resource.MustColumn("resource_type").SQLText() + " = $" + strconv.Itoa(typeIndex)) + } + if r.scope.denied { + builder.WriteString(denyConjunct) + } + return builder.String() +} + +func (r *planRenderer) scopedEdgePredicate(alias string) string { + columns := r.storage.Edge + predicate := alias + "." + columns.MustColumn("project").SQLText() + " = $" + strconv.Itoa(r.scope.projectIndex) + predicate += " AND " + alias + "." + columns.MustColumn("dataset_generation").SQLText() + " = $" + strconv.Itoa(r.scope.generationIndex) + if r.scope.authPathsIndex != 0 { + predicate += " AND " + alias + "." + columns.MustColumn("auth_resource_path").SQLText() + + " = ANY (SELECT jsonb_array_elements_text($" + strconv.Itoa(r.scope.authPathsIndex) + "::jsonb))" + } + if r.scope.denied { + predicate += " AND FALSE" + } + return predicate +} diff --git a/internal/dataframe/compiler/render/postgres/scope_ops.go b/internal/dataframe/compiler/render/postgres/scope_ops.go new file mode 100644 index 000000000..c5ebb9691 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/scope_ops.go @@ -0,0 +1,99 @@ +package postgres + +import ( + "fmt" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const ( + scopeFieldNameProject = "project" + scopeFieldNameGeneration = "dataset_generation" + scopeFieldNameAuthPath = "auth_resource_path" + canonicalAuthLetOperator = "AUTH_RESOURCE_PATH_ALLOWED" + scopeComparisonOperator = "EQUALS" +) + +func (r *planRenderer) acceptCanonicalScopeFilter(filter *ir.PhysicalFilter) (bool, error) { + if filter == nil || filter.Expression != nil { + return false, nil + } + predicate := filter.Predicate + if predicate.Operator != scopeComparisonOperator || predicate.LeftExpression != nil || predicate.Right == nil { + return false, nil + } + if predicate.Right.Variable != "" || len(predicate.Right.Path) != 0 { + return false, nil + } + switch predicate.Right.BindKey { + case projectBindKey: + return r.acceptCanonicalRowField(predicate, scopeFieldNameProject, projectBindKey) + case datasetGenerationBind: + return r.acceptCanonicalRowField(predicate, scopeFieldNameGeneration, datasetGenerationBind) + case scopeAllowedBind: + if predicate.Left.Variable == "" || len(predicate.Left.Path) != 0 { + return false, fmt.Errorf("%w: scope_allowed equality must bind a canonical auth LET variable directly", ErrInvalidPostgresPlanScope) + } + if _, ok := r.canonicalAuthLets[predicate.Left.Variable]; !ok { + return false, fmt.Errorf("%w: scope_allowed equality names %q, which no canonical AUTH_RESOURCE_PATH_ALLOWED LET established", + ErrInvalidPostgresPlanScope, predicate.Left.Variable) + } + return true, nil + default: + return false, nil + } +} + +func (r *planRenderer) acceptCanonicalRowField(predicate ir.PhysicalPredicate, fieldName, bindKey string) (bool, error) { + if _, ok := r.rows[predicate.Left.Variable]; !ok { + return false, nil + } + if len(predicate.Left.Path) != 1 || predicate.Left.Path[0] != fieldName { + return false, fmt.Errorf("%w: %q scope filter must compare %s.%s", + ErrInvalidPostgresPlanScope, bindKey, predicate.Left.Variable, fieldName) + } + return true, nil +} + +func (r *planRenderer) acceptCanonicalAuthLet(derivedLet *ir.PhysicalDerivedLet) (bool, error) { + if derivedLet == nil || derivedLet.Operator != canonicalAuthLetOperator { + return false, nil + } + refuse := func(reason string) (bool, error) { + return false, fmt.Errorf("%w: canonical AUTH_RESOURCE_PATH_ALLOWED LET %s", ErrInvalidPostgresPlanScope, reason) + } + if derivedLet.Variable == "" { + return refuse("must name a variable") + } + if _, taken := r.rows[derivedLet.Variable]; taken { + return refuse("shadows a scope row variable") + } + if _, claimed := r.canonicalAuthLets[derivedLet.Variable]; claimed { + return refuse("redefines an established auth variable") + } + var overRow string + sawPaths, sawUnrestricted := false, false + for _, input := range derivedLet.Inputs { + switch { + case input.BindKey == authPathsBindKey && input.Variable == "" && len(input.Path) == 0: + sawPaths = true + case input.BindKey == authUnrestrictedBind && input.Variable == "" && len(input.Path) == 0: + sawUnrestricted = true + case input.BindKey == "": + if _, ok := r.rows[input.Variable]; !ok { + return refuse(fmt.Sprintf("inputs %q, which is not a registered scope row", input.Variable)) + } + if len(input.Path) != 1 || input.Path[0] != scopeFieldNameAuthPath { + return refuse("must read .auth_resource_path from its registered scope row") + } + overRow = input.Variable + default: + return refuse("carries an unknown input") + } + } + if overRow == "" || !sawPaths || !sawUnrestricted { + return refuse("must read .auth_resource_path, @auth_resource_paths, and @auth_resource_paths_unrestricted") + } + r.canonicalAuthLets[derivedLet.Variable] = overRow + return true, nil +} diff --git a/internal/dataframe/compiler/render/postgres/scope_ops_witness_test.go b/internal/dataframe/compiler/render/postgres/scope_ops_witness_test.go new file mode 100644 index 000000000..acd6bd7e1 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/scope_ops_witness_test.go @@ -0,0 +1,192 @@ +package postgres + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +func scopeBlockOperations() []ir.PhysicalOperation { + return []ir.PhysicalOperation{ + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: "root", Path: []string{"project"}}, + Right: &ir.PhysicalValue{BindKey: "project"}, + }}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: "root", Path: []string{"dataset_generation"}}, + Right: &ir.PhysicalValue{BindKey: "dataset_generation"}, + }}}, + {Kind: ir.PhysicalDerivedLetOp, DerivedLet: &ir.PhysicalDerivedLet{ + Variable: "root_auth_allowed", + Operator: "AUTH_RESOURCE_PATH_ALLOWED", + Inputs: []ir.PhysicalValue{ + {Variable: "root", Path: []string{"auth_resource_path"}}, + {BindKey: "auth_resource_paths"}, + {BindKey: "auth_resource_paths_unrestricted"}, + }, + }}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: "root_auth_allowed"}, + Right: &ir.PhysicalValue{BindKey: "scope_allowed"}, + }}}, + } +} + +func planWithScopeBlock(mutate func(ops []ir.PhysicalOperation)) ir.PhysicalPlan { + plan := basePlan() + block := scopeBlockOperations() + if mutate != nil { + mutate(block) + } + operations := []ir.PhysicalOperation{plan.Operations[0]} + operations = append(operations, block...) + return ir.PhysicalPlan{ + Version: plan.Version, + Engine: plan.Engine, + BindVars: plan.BindVars, + Operations: append(operations, plan.Operations[1]), + } +} + +func TestCanonicalScopeBlockIsVerifiedThenDiscarded(t *testing.T) { + rendered := renderOK(t, planWithScopeBlock(nil)) + for _, forbidden := range []string{"root_auth_allowed", "@", "auth_resource_paths ="} { + if strings.Contains(rendered.SQL, forbidden) { + t.Fatalf("plan-authored scope text %q survived into SQL: %s", forbidden, rendered.SQL) + } + } + for _, required := range []string{"project", "dataset_generation", "auth_resource_path", "resource_type"} { + if !strings.Contains(rendered.SQL, required) { + t.Fatalf("renderer-owned scope predicate %q missing from SQL: %s", required, rendered.SQL) + } + } + withBlock, withoutBlock := rendered.SQL, renderOK(t, basePlan()).SQL + if withBlock != withoutBlock { + t.Fatalf("dropping the canonical block changed the statement:\nwith %s\nwithout %s", withBlock, withoutBlock) + } +} + +func TestCanonicalScopeNearMissesAreScopeDefects(t *testing.T) { + cases := []struct { + name string + mutate func(ops []ir.PhysicalOperation) + }{ + {"project filter compares the wrong field", func(ops []ir.PhysicalOperation) { + ops[0].Filter.Predicate.Left = ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}} + }}, + {"project filter compares a nested path", func(ops []ir.PhysicalOperation) { + ops[0].Filter.Predicate.Left = ir.PhysicalValue{Variable: "root", Path: []string{"payload", "project"}} + }}, + {"generation filter compares the wrong field", func(ops []ir.PhysicalOperation) { + ops[1].Filter.Predicate.Left = ir.PhysicalValue{Variable: "root", Path: []string{"generation"}} + }}, + {"scope_allowed equality names an unestablished variable", func(ops []ir.PhysicalOperation) { + ops[3].Filter.Predicate.Left = ir.PhysicalValue{Variable: "rogue_auth_allowed"} + }}, + {"scope_allowed equality carries a path", func(ops []ir.PhysicalOperation) { + ops[3].Filter.Predicate.Left = ir.PhysicalValue{Variable: "root_auth_allowed", Path: []string{"allowed"}} + }}, + {"scope_allowed equality binds the wrong side", func(ops []ir.PhysicalOperation) { + ops[3].Filter.Predicate.Right = &ir.PhysicalValue{Variable: "root", Path: []string{"project"}} + }}, + {"auth LET drops the unrestricted input", func(ops []ir.PhysicalOperation) { + ops[2].DerivedLet.Inputs = ops[2].DerivedLet.Inputs[:2] + }}, + {"auth LET reads the wrong row field", func(ops []ir.PhysicalOperation) { + ops[2].DerivedLet.Inputs[0] = ir.PhysicalValue{Variable: "root", Path: []string{"id"}} + }}, + {"auth LET reads a nested field", func(ops []ir.PhysicalOperation) { + ops[2].DerivedLet.Inputs[0] = ir.PhysicalValue{Variable: "root", Path: []string{"payload", "auth_resource_path"}} + }}, + {"auth LET reads an unregistered row", func(ops []ir.PhysicalOperation) { + ops[2].DerivedLet.Inputs[0] = ir.PhysicalValue{Variable: "ghost", Path: []string{"auth_resource_path"}} + }}, + {"auth LET shadows a scope row variable", func(ops []ir.PhysicalOperation) { + ops[2].DerivedLet.Variable = "root" + }}, + {"auth LET carries an unknown input", func(ops []ir.PhysicalOperation) { + ops[2].DerivedLet.Inputs = append(ops[2].DerivedLet.Inputs, + ir.PhysicalValue{BindKey: "auth_resource_paths", Path: []string{"extra"}}) + }}, + } + earlyReject := map[string]bool{ + "scope_allowed equality names an unestablished variable": true, + "auth LET reads an unregistered row": true, + "auth LET shadows a scope row variable": true, + "auth LET carries an unknown input": true, + } + renderSurface := map[string]bool{ + "scope_allowed equality binds the wrong side": true, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := RenderPhysicalPlan(planWithScopeBlock(tc.mutate), mustBinding(t)) + if err == nil { + t.Fatal("near-miss scope operation must never render") + } + switch { + case earlyReject[tc.name]: + if errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("near-miss must be refused by the IR scope validator, not the unsupported surface: %v", err) + } + case renderSurface[tc.name]: + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("non-scope predicate shape must die on the closed render surface: %v", err) + } + default: + if !errors.Is(err, ErrInvalidPostgresPlanScope) { + t.Fatalf("near-miss must fail closed with the scope-defect sentinel, got %v (%T)", err, err) + } + } + }) + } +} + +func TestAuthLETDuplicateRegistrationIsRejected(t *testing.T) { + plan := planWithScopeBlock(func(ops []ir.PhysicalOperation) {}) + duplicate := *plan.Operations[3].DerivedLet + operations := append([]ir.PhysicalOperation{}, plan.Operations...) + operations = append(operations[:5], append([]ir.PhysicalOperation{{Kind: ir.PhysicalDerivedLetOp, DerivedLet: &duplicate}}, operations[5:]...)...) + plan.Operations = operations + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil || errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("redefining an established auth variable must fail closed, got %v", err) + } +} + +func TestNonCanonicalFilterIsNotConsumedByTheScopeSkip(t *testing.T) { + plan := planWithScopeBlock(nil) + plan.BindVars["status_value"] = "final" + custom := ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "status_value"}, + }}} + operations := []ir.PhysicalOperation{plan.Operations[0], custom} + operations = append(operations, plan.Operations[1:]...) + plan.Operations = operations + rendered := renderOK(t, plan) + var sawPathArg, sawValueArg bool + for _, arg := range rendered.Args { + text := fmt.Sprintf("%v", arg.Value) + if raw, ok := arg.Value.(json.RawMessage); ok { + text = string(raw) + } + if text == "{status}" { + sawPathArg = true + } + if strings.Contains(text, "final") { + sawValueArg = true + } + } + if !sawPathArg || !sawValueArg { + t.Fatalf("non-canonical FILTER was consumed by the scope skip: sql=%s args=%v", rendered.SQL, rendered.Args) + } +} diff --git a/internal/dataframe/compiler/render/postgres/scope_test.go b/internal/dataframe/compiler/render/postgres/scope_test.go new file mode 100644 index 000000000..e4353f702 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/scope_test.go @@ -0,0 +1,1135 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + control "github.com/calypr/loom/internal/backends/postgres/control" + "net/url" + "os" + "reflect" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +func pgsq02ScopePlan(project, generation string, paths []string, unrestricted bool) ir.PhysicalPlan { + return ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": "Patient", + projectBindKey: project, + datasetGenerationBind: generation, + authPathsBindKey: paths, + authUnrestrictedBind: unrestricted, + scopeAllowedBind: true, + }, + } +} + +func pgsq02ScopeRenderer(t *testing.T, plan ir.PhysicalPlan) *planRenderer { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("pgsq02_it_unit") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + r := &planRenderer{storage: binding, parameters: map[string]int{}, rows: map[string]rowScope{}} + if err := r.loadScope(plan); err != nil { + t.Fatalf("loadScope: %v", err) + } + return r +} + +func TestScopeResourcePredicateGoldenRestricted(t *testing.T) { + r := pgsq02ScopeRenderer(t, pgsq02ScopePlan("pgsq02-proj-a", "gen-1", + []string{"FHIR/Patient", "FHIR/Specimen"}, false)) + typeIndex := r.textParameter("Patient", "text:Patient") + + const want = `r0."project" = $1` + + ` AND r0."dataset_generation" = $2` + + ` AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND r0."resource_type" = $4` + if got := r.scopedResourcePredicate("r0", typeIndex); got != want { + t.Fatalf("restricted resource predicate =\n%s\nwant\n%s", got, want) + } + if got := r.scopedResourcePredicate("r0", 0); got != strings.TrimSuffix(want, ` AND r0."resource_type" = $4`) { + t.Fatalf("type-less resource predicate =\n%s", got) + } + wantArgs := []struct { + Type string + Value any + }{ + {"text", "pgsq02-proj-a"}, + {"text", "gen-1"}, + {"jsonb", json.RawMessage(`["FHIR/Patient","FHIR/Specimen"]`)}, + {"text", "Patient"}, + } + if len(r.args) != len(wantArgs) { + t.Fatalf("args = %d, want %d", len(r.args), len(wantArgs)) + } + for index, wantArg := range wantArgs { + if string(r.args[index].Type) != wantArg.Type || !reflect.DeepEqual(r.args[index].Value, wantArg.Value) { + t.Fatalf("arg %d = %#v (%s), want %#v (%s)", index, r.args[index].Value, r.args[index].Type, wantArg.Value, wantArg.Type) + } + } +} + +func TestScopeResourcePredicateGoldenUnrestricted(t *testing.T) { + r := pgsq02ScopeRenderer(t, pgsq02ScopePlan("pgsq02-proj-a", "", nil, true)) + typeIndex := r.textParameter("Patient", "text:Patient") + + const want = `r0."project" = $1 AND r0."dataset_generation" = $2 AND r0."resource_type" = $3` + got := r.scopedResourcePredicate("r0", typeIndex) + if got != want { + t.Fatalf("unrestricted resource predicate =\n%s\nwant\n%s", got, want) + } + if strings.Contains(got, "auth_resource_path") { + t.Fatalf("explicit unrestricted mode must add no auth predicate: %s", got) + } + if len(r.args) != 3 { + t.Fatalf("unrestricted scope must bind exactly project, generation, type; args = %#v", r.args) + } + for _, arg := range r.args { + if string(arg.Type) == "jsonb" { + t.Fatalf("unrestricted scope bound an auth parameter anyway: %#v", r.args) + } + } +} + +func TestScopeDenyAllKeepsAuthClauseWithEmptyJSONArray(t *testing.T) { + for name, paths := range map[string][]string{"empty": {}, "nil": nil} { + r := pgsq02ScopeRenderer(t, pgsq02ScopePlan("pgsq02-proj-a", "gen-1", paths, false)) + const want = `e1."project" = $1 AND e1."dataset_generation" = $2` + + ` AND e1."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + if got := r.scopedEdgePredicate("e1"); got != want { + t.Fatalf("%s deny-all edge predicate =\n%s\nwant\n%s", name, got, want) + } + if len(r.args) != 3 { + t.Fatalf("%s deny-all args = %#v, want project, generation, empty array", name, r.args) + } + auth := r.args[2] + if string(auth.Type) != "jsonb" { + t.Fatalf("%s deny-all auth type = %s, want jsonb", name, auth.Type) + } + if raw, ok := auth.Value.(json.RawMessage); !ok || string(raw) != "[]" { + t.Fatalf("%s deny-all auth value = %#v, want JSON empty array", name, auth.Value) + } + } +} + +func TestScopeEdgePredicateGoldenRestrictedAndUnrestricted(t *testing.T) { + restricted := pgsq02ScopeRenderer(t, pgsq02ScopePlan("pgsq02-proj-a", "gen-1", []string{"FHIR/Patient"}, false)) + const wantRestricted = `e1."project" = $1` + + ` AND e1."dataset_generation" = $2` + + ` AND e1."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + if got := restricted.scopedEdgePredicate("e1"); got != wantRestricted { + t.Fatalf("restricted edge predicate =\n%s\nwant\n%s", got, wantRestricted) + } + for _, leaked := range []string{"label", "source_logical_id", "target_logical_id", "resource_type"} { + if strings.Contains(restricted.scopedEdgePredicate("e1"), leaked) { + t.Fatalf("canonical edge predicate must not carry traversal payload %q: %s", leaked, wantRestricted) + } + } + unrestricted := pgsq02ScopeRenderer(t, pgsq02ScopePlan("pgsq02-proj-a", "", nil, true)) + const wantUnrestricted = `e1."project" = $1 AND e1."dataset_generation" = $2` + if got := unrestricted.scopedEdgePredicate("e1"); got != wantUnrestricted { + t.Fatalf("unrestricted edge predicate =\n%s\nwant\n%s", got, wantUnrestricted) + } +} + +func pgsq02DeniedPlan(plan ir.PhysicalPlan) ir.PhysicalPlan { + plan.BindVars[scopeAllowedBind] = false + return plan +} + +func TestScopeAllowedFalseMapsToLiteralZeroRowPredicate(t *testing.T) { + restricted := pgsq02ScopeRenderer(t, pgsq02DeniedPlan( + pgsq02ScopePlan("pgsq02-proj-a", "gen-1", []string{"FHIR/Patient", "FHIR/Specimen"}, false))) + typeIndex := restricted.textParameter("Patient", "text:Patient") + const wantDeniedResource = `r0."project" = $1` + + ` AND r0."dataset_generation" = $2` + + ` AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND r0."resource_type" = $4 AND FALSE` + if got := restricted.scopedResourcePredicate("r0", typeIndex); got != wantDeniedResource { + t.Fatalf("denied restricted resource predicate =\n%s\nwant\n%s", got, wantDeniedResource) + } + const wantDeniedEdge = `e1."project" = $1` + + ` AND e1."dataset_generation" = $2` + + ` AND e1."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb)) AND FALSE` + if got := restricted.scopedEdgePredicate("e1"); got != wantDeniedEdge { + t.Fatalf("denied restricted edge predicate =\n%s\nwant\n%s", got, wantDeniedEdge) + } + if len(restricted.args) != 4 { + t.Fatalf("denied scope must bind exactly the allowed scope's parameters; args = %#v", restricted.args) + } + for _, arg := range restricted.args { + if string(arg.Type) == "boolean" { + t.Fatalf("the deny became a parameter instead of a predicate shape: %#v", restricted.args) + } + } + if raw, ok := restricted.args[2].Value.(json.RawMessage); !ok || string(raw) != `["FHIR/Patient","FHIR/Specimen"]` { + t.Fatalf("denied auth array = %#v, want the exact authorized set kept", restricted.args[2]) + } + + free := pgsq02ScopeRenderer(t, pgsq02DeniedPlan(pgsq02ScopePlan("pgsq02-proj-a", "", nil, true))) + freeType := free.textParameter("Patient", "text:Patient") + const wantDeniedFree = `r0."project" = $1 AND r0."dataset_generation" = $2` + + ` AND r0."resource_type" = $3 AND FALSE` + if got := free.scopedResourcePredicate("r0", freeType); got != wantDeniedFree { + t.Fatalf("denied unrestricted resource predicate =\n%s\nwant\n%s", got, wantDeniedFree) + } + if strings.Contains(free.scopedEdgePredicate("e1"), "auth_resource_path") { + t.Fatalf("denied unrestricted mode must stay auth-predicate-free: %s", free.scopedEdgePredicate("e1")) + } + + denyAll := pgsq02ScopeRenderer(t, pgsq02DeniedPlan(pgsq02ScopePlan("pgsq02-proj-a", "gen-1", nil, false))) + if got := denyAll.scopedEdgePredicate("e1"); got != wantDeniedEdge { + t.Fatalf("denied deny-all edge predicate =\n%s\nwant\n%s", got, wantDeniedEdge) + } + if raw, ok := denyAll.args[2].Value.(json.RawMessage); !ok || string(raw) != "[]" { + t.Fatalf("denied deny-all auth value = %#v, want JSON empty array kept", denyAll.args[2]) + } + + allowed := pgsq02ScopeRenderer(t, pgsq02ScopePlan("pgsq02-proj-a", "gen-1", []string{"FHIR/Patient"}, false)) + if strings.Contains(allowed.scopedResourcePredicate("r0", typeIndex), "FALSE") || + strings.Contains(allowed.scopedEdgePredicate("e1"), "FALSE") { + t.Fatalf("allowed scope must not emit a deny token") + } +} + +func TestScopeAllowedFalseIsWiredIntoEveryRowSource(t *testing.T) { + plan := pgsq02TraversalPlanForScope("pgsq02-proj-a", "gen-1", []string{"FHIR/Patient", "FHIR/Specimen"}, false, + "Patient", "has-specimen", "Specimen") + allowed, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("allowed traversal render: %v", err) + } + if strings.Contains(allowed.SQL, "FALSE") { + t.Fatalf("allowed statement carries a deny token: %s", allowed.SQL) + } + denied, err := RenderPhysicalPlan(pgsq02DeniedPlan(plan), mustBinding(t)) + if err != nil { + t.Fatalf("denied traversal render: %v", err) + } + if !reflect.DeepEqual(denied.Args, allowed.Args) { + t.Fatalf("denied args drifted from the allowed render: %#v vs %#v", denied.Args, allowed.Args) + } + for _, wantFragment := range []string{ + ` WHERE r0."project" = $1 AND r0."dataset_generation" = $2` + + ` AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND r0."resource_type" = $4 AND FALSE`, + ` WHERE e1."project" = $1 AND e1."dataset_generation" = $2` + + ` AND e1."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND FALSE AND e1."label" = $5`, + ` ON x1."project" = $1 AND x1."dataset_generation" = $2` + + ` AND x1."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND x1."resource_type" = $6 AND FALSE AND x1."logical_id" = e1."target_logical_id"`, + } { + if !strings.Contains(denied.SQL, wantFragment) { + t.Fatalf("denied statement missing %q:\n%s", wantFragment, denied.SQL) + } + } + if count := strings.Count(denied.SQL, " AND FALSE"); count != 3 { + t.Fatalf("deny conjunct count = %d, want 3 (root, edge, target):\n%s", count, denied.SQL) + } + + root := pgsq02RootPlan("pgsq02-proj-a", "gen-1", []string{"FHIR/Patient"}, false, "Patient", "logical_id") + deniedRoot, err := RenderRootScan(pgsq02DeniedPlan(root), mustBinding(t)) + if err != nil { + t.Fatalf("denied root scan render: %v", err) + } + if !strings.Contains(deniedRoot.SQL, ` WHERE r0."project" = $1 AND r0."dataset_generation" = $2`+ + ` AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))`+ + ` AND r0."resource_type" = $4 AND FALSE`) { + t.Fatalf("denied root-scan statement missing its deny conjunct:\n%s", deniedRoot.SQL) + } +} + +func TestScopeLoadRefusesEveryDefectiveBindBeforeBindingAnything(t *testing.T) { + for _, probe := range []struct { + desc string + mutate func(map[string]any) + }{ + {"project missing", func(b map[string]any) { delete(b, projectBindKey) }}, + {"project empty", func(b map[string]any) { b[projectBindKey] = "" }}, + {"project blank", func(b map[string]any) { b[projectBindKey] = " " }}, + {"project not a string", func(b map[string]any) { b[projectBindKey] = 42 }}, + {"generation missing", func(b map[string]any) { delete(b, datasetGenerationBind) }}, + {"generation not a string", func(b map[string]any) { b[datasetGenerationBind] = 7 }}, + {"auth mode missing (no implicit scope)", func(b map[string]any) { delete(b, authUnrestrictedBind) }}, + {"auth mode not a bool", func(b map[string]any) { b[authUnrestrictedBind] = "true" }}, + {"auth paths missing", func(b map[string]any) { delete(b, authPathsBindKey) }}, + {"auth paths not []string", func(b map[string]any) { b[authPathsBindKey] = []any{"FHIR/Patient"} }}, + {"unrestricted carries paths", func(b map[string]any) { + b[authUnrestrictedBind] = true + b[authPathsBindKey] = []string{"FHIR/Patient"} + }}, + {"scope_allowed missing (fifth canonical bind, no implicit allow)", func(b map[string]any) { delete(b, scopeAllowedBind) }}, + {"scope_allowed not a bool", func(b map[string]any) { b[scopeAllowedBind] = "true" }}, + } { + plan := pgsq02ScopePlan("pgsq02-proj-a", "gen-1", []string{"FHIR/Patient"}, false) + probe.mutate(plan.BindVars) + binding, err := ir.NewPostgresStorageBinding("pgsq02_it_unit") + if err != nil { + t.Fatalf("binding: %v", err) + } + r := &planRenderer{storage: binding, parameters: map[string]int{}, rows: map[string]rowScope{}} + err = r.loadScope(plan) + if !errors.Is(err, ErrInvalidPostgresPlanScope) { + t.Fatalf("%s: error = %v, want ErrInvalidPostgresPlanScope", probe.desc, err) + } + if len(r.args) != 0 { + t.Fatalf("%s: refused scope still bound %d parameter(s): %#v", probe.desc, len(r.args), r.args) + } + if r.scope != (planScope{}) { + t.Fatalf("%s: refused scope populated %+v", probe.desc, r.scope) + } + } +} + +func TestScopeRendererEntryRefusesDefectiveScopeWithNoSQL(t *testing.T) { + plan := pgsq02ScopePlan("pgsq02-proj-a", "gen-1", []string{"FHIR/Patient"}, false) + delete(plan.BindVars, authUnrestrictedBind) + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + } + if rendered, err := RenderRootScan(plan, mustBinding(t)); !errors.Is(err, ErrInvalidPostgresPlanScope) { + t.Fatalf("RenderRootScan defective scope: err = %v, want ErrInvalidPostgresPlanScope", err) + } else if rendered.SQL != "" || len(rendered.Args) != 0 { + t.Fatalf("refused plan produced SQL/args: %q %#v", rendered.SQL, rendered.Args) + } + if rendered, err := RenderPhysicalPlan(plan, mustBinding(t)); !errors.Is(err, ErrInvalidPostgresPlanScope) { + t.Fatalf("RenderPhysicalPlan defective scope: err = %v, want ErrInvalidPostgresPlanScope", err) + } else if rendered.SQL != "" || len(rendered.Args) != 0 { + t.Fatalf("refused plan produced SQL/args: %q %#v", rendered.SQL, rendered.Args) + } +} + +func TestScopeValuesNeverBecomeSQLTextAndRenderingIsDeterministic(t *testing.T) { + hostileProject := `pg'; DROP TABLE fhir_resource; -- "$1` + hostileGeneration := "gen-\"2\"\\$2" + hostilePaths := []string{`FHIR/*' OR 1=1 --`, `FHIR/"injected"; $3`} + + plan := pgsq02TraversalPlanForScope(hostileProject, hostileGeneration, hostilePaths, false, + "Patient", "has-specimen", "Specimen") + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + for _, leaked := range append([]string{hostileProject, hostileGeneration}, hostilePaths...) { + if strings.Contains(rendered.SQL, leaked) { + t.Fatalf("scope value %q leaked into SQL text: %s", leaked, rendered.SQL) + } + } + if len(rendered.Args) < 3 { + t.Fatalf("scope arguments missing: %#v", rendered.Args) + } + if got := rendered.Args[0]; string(got.Type) != "text" || got.Value != hostileProject { + t.Fatalf("project argument = %#v", got) + } + if got := rendered.Args[1]; string(got.Type) != "text" || got.Value != hostileGeneration { + t.Fatalf("generation argument = %#v", got) + } + if got := rendered.Args[2]; string(got.Type) != "jsonb" || + string(got.Value.(json.RawMessage)) != `["FHIR/*' OR 1=1 --","FHIR/\"injected\"; $3"]` { + t.Fatalf("auth argument = %#v", got) + } + + second, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil || second.SQL != rendered.SQL || !reflect.DeepEqual(second.Args, rendered.Args) { + t.Fatalf("second render drifted: %v", err) + } +} + +func TestScopeFragmentsAreWiredIntoAssembledStatements(t *testing.T) { + plan := pgsq02TraversalPlanForScope("pgsq02-proj-a", "gen-1", []string{"FHIR/Patient", "FHIR/Specimen"}, false, + "Patient", "has-specimen", "Specimen") + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + for _, wantFragment := range []string{ + ` WHERE e1."project" = $1 AND e1."dataset_generation" = $2` + + ` AND e1."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND e1."label" = $5`, + ` ON x1."project" = $1 AND x1."dataset_generation" = $2` + + ` AND x1."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND x1."resource_type" = $6 AND x1."logical_id" = e1."target_logical_id"`, + ` WHERE r0."project" = $1 AND r0."dataset_generation" = $2` + + ` AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND r0."resource_type" = $4`, + } { + if !strings.Contains(rendered.SQL, wantFragment) { + t.Fatalf("assembled SQL missing canonical fragment %q:\n%s", wantFragment, rendered.SQL) + } + } +} + +func pgsq02TraversalPlanForScope(project, generation string, paths []string, unrestricted bool, + rootType, label, targetType string) ir.PhysicalPlan { + plan := pgsq02ScopePlan(project, generation, paths, unrestricted) + plan.BindVars["edge_collection"] = "fhir_edge" + plan.BindVars["edge_label"] = label + plan.BindVars["target_type"] = targetType + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "target", EdgeVariable: "edge", + Direction: ir.PhysicalOutbound, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: "edge_label", TargetTypeBindKey: "target_type", + EdgeTargetTypeField: "to_type", Strategy: ir.PhysicalTraversalNative, + }}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "root_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + {Name: "target_id", Value: ir.PhysicalValue{Variable: "target", Path: []string{"logical_id"}}}, + }}}, + } + return plan +} + +const ( + pgsq02ProjectA = "pgsq02-proj-a" + pgsq02ProjectB = "pgsq02-proj-b" + pgsq02GenOne = "gen-1" + pgsq02GenTwo = "gen-2" + pgsq02PathPatient = "FHIR/Patient" + pgsq02PathSpecimen = "FHIR/Specimen" + pgsq02PathConf = "FHIR/Confidential" + pgsq02Label = "has-specimen" +) + +var pgsq02ResourceNamePattern = regexp.MustCompile(`^pgsq02_it_[0-9]+_[0-9a-f]+$`) + +type pgsq02Service struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + client *control.Client + schema string + fail func(step string, err error) + scrub func(string) string +} + +func newPGSQ02Service(t *testing.T) *pgsq02Service { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v string) string { + s := v + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err.Error())) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := "pgsq02_it_" + strconv.FormatInt(time.Now().UnixNano(), 10) + "_" + randomHex(t) + if !pgsq02ResourceNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err.Error())) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := "pgsq02_it_" + strconv.FormatInt(time.Now().UnixNano(), 10) + "_" + randomHex(t) + if !pgsq02ResourceNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err.Error())) + } + }) + + client, err := control.OpenPool(ctx, control.ClientConfig{ + DSN: dsn, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 2 * time.Second, + }) + if err != nil { + fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + t.Cleanup(client.Close) + if _, err := control.MigrateSchema(ctx, client, schema); err != nil { + fail("migrate owned schema", err) + } + return &pgsq02Service{ctx: ctx, admin: admin, iso: iso, client: client, schema: schema, fail: fail, scrub: scrub} +} + +func (s *pgsq02Service) insertResource(t *testing.T, project, generation, resourceType, logicalID, authPath string) int64 { + t.Helper() + repo, err := control.NewResourceRepository(s.client, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + seq, err := repo.InsertResource(s.ctx, source.Resource{ + Project: project, + DatasetGeneration: generation, + ResourceType: resourceType, + LogicalID: logicalID, + AuthResourcePath: authPath, + }) + if err != nil { + s.fail("insert resource "+resourceType+"/"+logicalID, err) + } + return seq +} + +func (s *pgsq02Service) insertEdge(t *testing.T, project, generation, authPath, sourceType, sourceID, targetType, targetID, sourcePath string, ordinal int64) int64 { + t.Helper() + repo, err := control.NewEdgeRepository(s.client, s.schema) + if err != nil { + t.Fatalf("edge repository: %v", err) + } + seq, err := repo.InsertEdge(s.ctx, control.EdgeOccurrenceRow{ + Project: project, DatasetGeneration: generation, AuthResourcePath: authPath, + Label: pgsq02Label, SourceResourceType: sourceType, SourceLogicalID: sourceID, + TargetResourceType: targetType, TargetLogicalID: targetID, + SourcePath: sourcePath, SourceOrdinal: ordinal, + }) + if err != nil { + s.fail("insert edge "+sourceID+"->"+targetID+" @"+project+"/"+generation, err) + } + return seq +} + +func (s *pgsq02Service) run(t *testing.T, sql string, args ...any) [][]string { + t.Helper() + rows, err := s.iso.Query(s.ctx, sql, args...) + if err != nil { + s.fail("execute statement "+sql, err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + s.fail("read row", err) + } + row := make([]string, len(values)) + for index, value := range values { + row[index] = stringifyCell(value) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + s.fail("finish statement", err) + } + return out +} + +func (s *pgsq02Service) runRendered(t *testing.T, rendered command.RenderedPostgresPhysicalPlan) [][]string { + t.Helper() + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + return s.run(t, rendered.SQL, args...) +} + +func (s *pgsq02Service) runOnTwoDistinctConnections(t *testing.T, rendered command.RenderedPostgresPhysicalPlan) ([][]string, [][]string, int32, int32) { + t.Helper() + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + read := func(conn *pgxpool.Conn) ([][]string, int32) { + var pid int32 + if err := conn.QueryRow(s.ctx, `SELECT pg_backend_pid()`).Scan(&pid); err != nil { + s.fail("read pg_backend_pid", err) + } + rows, err := conn.Query(s.ctx, rendered.SQL, args...) + if err != nil { + s.fail("execute rendered statement on dedicated connection", err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + s.fail("read connection row", err) + } + row := make([]string, len(values)) + for index, value := range values { + row[index] = stringifyCell(value) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + s.fail("finish connection statement", err) + } + return out, pid + } + first, err := s.iso.Acquire(s.ctx) + if err != nil { + s.fail("acquire first dedicated connection", err) + } + defer first.Release() + second, err := s.iso.Acquire(s.ctx) + if err != nil { + s.fail("acquire second dedicated connection", err) + } + defer second.Release() + firstRows, firstPID := read(first) + secondRows, secondPID := read(second) + return firstRows, secondRows, firstPID, secondPID +} + +func (s *pgsq02Service) expectRefused(t *testing.T, sql string, wantCode, wantConstraint string, args ...any) { + t.Helper() + _, err := s.iso.Exec(s.ctx, sql, args...) + if err == nil { + t.Fatalf("statement was accepted, want refusal by %s: %s", wantConstraint, sql) + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + s.fail("expect pg error for "+sql, err) + } + if pgErr.Code != wantCode || pgErr.ConstraintName != wantConstraint { + s.fail("statement refused by ("+pgErr.Code+","+pgErr.ConstraintName+ + ") want ("+wantCode+","+wantConstraint+")", err) + } +} + +func pgsq02RootPlan(project, generation string, paths []string, unrestricted bool, resourceType string, columns ...string) ir.PhysicalPlan { + projections := make([]ir.PhysicalProjection, 0, len(columns)) + for _, column := range columns { + projections = append(projections, ir.PhysicalProjection{ + Name: column, + Value: ir.PhysicalValue{Variable: "root", Path: []string{column}}, + }) + } + plan := pgsq02ScopePlan(project, generation, paths, unrestricted) + plan.BindVars["root_collection"] = resourceType + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}}, + } + return plan +} + +func TestPGSQ02SameResourceIDTwoProjectsGenerationsReturnsAuthorizedTuple(t *testing.T) { + svc := newPGSQ02Service(t) + svc.insertResource(t, pgsq02ProjectA, pgsq02GenOne, "Patient", "p-shared", pgsq02PathPatient) + svc.insertResource(t, pgsq02ProjectB, pgsq02GenOne, "Patient", "p-shared", pgsq02PathPatient) + svc.insertResource(t, pgsq02ProjectA, pgsq02GenTwo, "Patient", "p-shared", pgsq02PathPatient) + svc.insertResource(t, pgsq02ProjectB, pgsq02GenTwo, "Patient", "p-shared", pgsq02PathPatient) + svc.insertResource(t, pgsq02ProjectA, pgsq02GenOne, "Observation", "p-shared", pgsq02PathPatient) + svc.insertResource(t, pgsq02ProjectA, pgsq02GenOne, "Patient", "p-hidden", pgsq02PathConf) + + plan := pgsq02RootPlan(pgsq02ProjectA, pgsq02GenOne, []string{pgsq02PathPatient}, false, + "Patient", "project", "dataset_generation", "resource_type", "logical_id", "auth_resource_path") + rendered, err := RenderRootScan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderRootScan: %v", err) + } + + wantArgs := []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: pgsq02ProjectA}, + {Type: command.PostgreSQLParameterText, Value: pgsq02GenOne}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["FHIR/Patient"]`)}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + } + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("args = %#v, want %#v", rendered.Args, wantArgs) + } + + want := [][]string{{pgsq02ProjectA, pgsq02GenOne, "Patient", "p-shared", pgsq02PathPatient}} + if got := svc.runRendered(t, rendered); !reflect.DeepEqual(got, want) { + t.Fatalf("scoped scan rows = %v, want exactly the authorized tuple %v", got, want) + } + + witness := svc.run(t, + `SELECT project, dataset_generation, resource_type, logical_id, auth_resource_path`+ + ` FROM "`+svc.schema+`"."fhir_resource"`+ + ` WHERE project = 'pgsq02-proj-a' AND dataset_generation = 'gen-1'`+ + ` AND auth_resource_path = 'FHIR/Patient' AND resource_type = 'Patient'`+ + ` ORDER BY resource_seq`) + if !reflect.DeepEqual(witness, want) { + t.Fatalf("witness rows = %v, want %v", witness, want) + } + + for _, probe := range []struct { + desc, sql string + args []any + }{ + {"project decoy", `SELECT count(*) FROM "` + svc.schema + `".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' AND logical_id='p-shared'`, []any{pgsq02ProjectB, pgsq02GenOne}}, + {"generation decoy", `SELECT count(*) FROM "` + svc.schema + `".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' AND logical_id='p-shared'`, []any{pgsq02ProjectA, pgsq02GenTwo}}, + {"project+generation decoy", `SELECT count(*) FROM "` + svc.schema + `".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' AND logical_id='p-shared'`, []any{pgsq02ProjectB, pgsq02GenTwo}}, + {"type decoy", `SELECT count(*) FROM "` + svc.schema + `".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Observation' AND logical_id='p-shared'`, []any{pgsq02ProjectA, pgsq02GenOne}}, + {"auth decoy", `SELECT count(*) FROM "` + svc.schema + `".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' AND logical_id='p-hidden'`, []any{pgsq02ProjectA, pgsq02GenOne}}, + } { + rows := svc.run(t, probe.sql, probe.args...) + if len(rows) != 1 || rows[0][0] != "1" { + t.Fatalf("%s must exist: %v", probe.desc, rows) + } + } + + type probe struct { + desc string + sql string + args []any + wantRows [][]string + } + base := `SELECT logical_id FROM "` + svc.schema + `".fhir_resource WHERE ` + const orderTail = ` ORDER BY resource_seq` + for _, p := range []probe{ + {"full chain (control)", base + + `project=$1 AND dataset_generation=$2 AND auth_resource_path = ANY (SELECT jsonb_array_elements_text($3::jsonb)) AND resource_type=$4` + orderTail, + []any{pgsq02ProjectA, pgsq02GenOne, json.RawMessage(`["FHIR/Patient"]`), "Patient"}, + [][]string{{"p-shared"}}}, + {"without project", base + + `dataset_generation=$1 AND auth_resource_path = ANY (SELECT jsonb_array_elements_text($2::jsonb)) AND resource_type=$3` + orderTail, + []any{pgsq02GenOne, json.RawMessage(`["FHIR/Patient"]`), "Patient"}, + [][]string{{"p-shared"}, {"p-shared"}}}, + {"without generation", base + + `project=$1 AND auth_resource_path = ANY (SELECT jsonb_array_elements_text($2::jsonb)) AND resource_type=$3` + orderTail, + []any{pgsq02ProjectA, json.RawMessage(`["FHIR/Patient"]`), "Patient"}, + [][]string{{"p-shared"}, {"p-shared"}}}, + {"without auth", base + + `project=$1 AND dataset_generation=$2 AND resource_type=$3` + orderTail, + []any{pgsq02ProjectA, pgsq02GenOne, "Patient"}, + [][]string{{"p-shared"}, {"p-hidden"}}}, + {"without resource type", base + + `project=$1 AND dataset_generation=$2 AND auth_resource_path = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + orderTail, + []any{pgsq02ProjectA, pgsq02GenOne, json.RawMessage(`["FHIR/Patient"]`)}, + [][]string{{"p-shared"}, {"p-shared"}}}, + } { + rows := svc.run(t, p.sql, p.args...) + if !reflect.DeepEqual(rows, p.wantRows) { + t.Fatalf("guard-drop %s rows = %v, want %v", p.desc, rows, p.wantRows) + } + } + + duplicateInsert := `INSERT INTO "` + svc.schema + `".fhir_resource (project, dataset_generation, resource_type, logical_id, auth_resource_path) VALUES ($1,$2,$3,$4,$5)` + svc.expectRefused(t, duplicateInsert, "23505", "fhir_resource_natural_identity_ux", + pgsq02ProjectA, pgsq02GenOne, "Patient", "p-shared", pgsq02PathPatient) + svc.expectRefused(t, duplicateInsert, "23514", "fhir_resource_project_present", + "", pgsq02GenOne, "Patient", "ck-project", pgsq02PathPatient) +} + +func TestPGSQ02RestrictedPathsExactAndDenyAllZeroRowsRootAndEdge(t *testing.T) { + svc := newPGSQ02Service(t) + svc.insertResource(t, pgsq02ProjectA, pgsq02GenOne, "Patient", "sp-1", pgsq02PathPatient) + svc.insertResource(t, pgsq02ProjectA, pgsq02GenOne, "Specimen", "st-allow", pgsq02PathSpecimen) + svc.insertResource(t, pgsq02ProjectA, pgsq02GenOne, "Specimen", "st-hidden", pgsq02PathConf) + svc.insertResource(t, pgsq02ProjectA, pgsq02GenOne, "Specimen", "st-off", pgsq02PathSpecimen) + svc.insertEdge(t, pgsq02ProjectA, pgsq02GenOne, pgsq02PathPatient, "Patient", "sp-1", "Specimen", "st-allow", "specimen[0]", 0) + svc.insertEdge(t, pgsq02ProjectA, pgsq02GenOne, pgsq02PathPatient, "Patient", "sp-1", "Specimen", "st-hidden", "specimen[1]", 0) + svc.insertEdge(t, pgsq02ProjectA, pgsq02GenOne, pgsq02PathConf, "Patient", "sp-1", "Specimen", "st-off", "specimen[2]", 0) + + binding := bindingFor(t, svc.schema) + allowed := []string{pgsq02PathPatient, pgsq02PathSpecimen} + + patients, err := RenderRootScan(pgsq02RootPlan(pgsq02ProjectA, pgsq02GenOne, allowed, false, "Patient", "logical_id"), binding) + if err != nil { + t.Fatalf("restricted patient scan: %v", err) + } + if got := svc.runRendered(t, patients); !reflect.DeepEqual(got, [][]string{{"sp-1"}}) { + t.Fatalf("restricted patient rows = %v, want exactly [sp-1]", got) + } + specimens, err := RenderRootScan(pgsq02RootPlan(pgsq02ProjectA, pgsq02GenOne, allowed, false, "Specimen", "logical_id"), binding) + if err != nil { + t.Fatalf("restricted specimen scan: %v", err) + } + wantVisible := [][]string{{"st-allow"}, {"st-off"}} + if got := svc.runRendered(t, specimens); !reflect.DeepEqual(got, wantVisible) { + t.Fatalf("restricted specimen rows = %v, want exactly %v", got, wantVisible) + } + + unrestricted, err := RenderRootScan(pgsq02RootPlan(pgsq02ProjectA, pgsq02GenOne, nil, true, "Specimen", "logical_id"), binding) + if err != nil { + t.Fatalf("unrestricted specimen scan: %v", err) + } + if got := svc.runRendered(t, unrestricted); !reflect.DeepEqual(got, [][]string{{"st-allow"}, {"st-hidden"}, {"st-off"}}) { + t.Fatalf("unrestricted specimen rows = %v, want every stored path", got) + } + + for _, resourceType := range []string{"Patient", "Specimen"} { + denyAll, err := RenderRootScan(pgsq02RootPlan(pgsq02ProjectA, pgsq02GenOne, []string{}, false, resourceType, "logical_id"), binding) + if err != nil { + t.Fatalf("deny-all %s scan: %v", resourceType, err) + } + if raw, ok := denyAll.Args[2].Value.(json.RawMessage); !ok || string(raw) != "[]" || + denyAll.Args[2].Type != command.PostgreSQLParameterJSONB { + t.Fatalf("deny-all auth argument = %#v, want JSON empty array", denyAll.Args[2]) + } + if got := svc.runRendered(t, denyAll); len(got) != 0 { + t.Fatalf("deny-all %s scan read %v, want zero rows", resourceType, got) + } + stored := svc.run(t, `SELECT count(*) FROM "`+svc.schema+`".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type=$3`, + pgsq02ProjectA, pgsq02GenOne, resourceType) + if len(stored) != 1 || stored[0][0] == "0" { + t.Fatalf("deny-all %s witness must show stored rows: %v", resourceType, stored) + } + } + + plan := pgsq02TraversalPlanForScope(pgsq02ProjectA, pgsq02GenOne, allowed, false, "Patient", pgsq02Label, "Specimen") + rendered, err := RenderPhysicalPlan(plan, binding) + if err != nil { + t.Fatalf("restricted traversal: %v", err) + } + wantArgs := []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: pgsq02ProjectA}, + {Type: command.PostgreSQLParameterText, Value: pgsq02GenOne}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["FHIR/Patient","FHIR/Specimen"]`)}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + {Type: command.PostgreSQLParameterText, Value: pgsq02Label}, + {Type: command.PostgreSQLParameterText, Value: "Specimen"}, + } + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("traversal args = %#v, want %#v", rendered.Args, wantArgs) + } + wantRoute := [][]string{{"sp-1", "st-allow"}} + if got := svc.runRendered(t, rendered); !reflect.DeepEqual(got, wantRoute) { + t.Fatalf("restricted traversal rows = %v, want exactly %v", got, wantRoute) + } + + denyEdge := pgsq02TraversalPlanForScope(pgsq02ProjectA, pgsq02GenOne, []string{}, false, "Patient", pgsq02Label, "Specimen") + denyRendered, err := RenderPhysicalPlan(denyEdge, binding) + if err != nil { + t.Fatalf("deny-all traversal: %v", err) + } + if raw, ok := denyRendered.Args[2].Value.(json.RawMessage); !ok || string(raw) != "[]" { + t.Fatalf("deny-all traversal auth = %#v, want JSON empty array", denyRendered.Args[2]) + } + if got := svc.runRendered(t, denyRendered); len(got) != 0 { + t.Fatalf("deny-all traversal read %v, want zero rows", got) + } + storedEdges := svc.run(t, `SELECT count(*) FROM "`+svc.schema+`".fhir_edge_occurrence WHERE project=$1 AND dataset_generation=$2 AND label=$3`, + pgsq02ProjectA, pgsq02GenOne, pgsq02Label) + if len(storedEdges) != 1 || storedEdges[0][0] != "3" { + t.Fatalf("edge witness must show all three occurrences stored: %v", storedEdges) + } + + edgeJoin := `SELECT r0.logical_id, r1.logical_id FROM "` + svc.schema + `".fhir_resource r0` + + ` JOIN "` + svc.schema + `".fhir_edge_occurrence e ON ` + edgeScope := `e.project=$1 AND e.dataset_generation=$2 AND e.auth_resource_path = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND e.label=$5 AND e.source_resource_type=$4 AND e.source_logical_id=r0.logical_id AND e.target_resource_type=$6` + targetJoin := ` JOIN "` + svc.schema + `".fhir_resource r1 ON r1.project=$1 AND r1.dataset_generation=$2` + + ` AND r1.auth_resource_path = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND r1.resource_type=$6 AND r1.logical_id=e.target_logical_id` + where := ` WHERE r0.project=$1 AND r0.dataset_generation=$2` + + ` AND r0.auth_resource_path = ANY (SELECT jsonb_array_elements_text($3::jsonb)) AND r0.resource_type=$4` + order := ` ORDER BY r0.resource_seq, e.edge_seq` + scopeArgs := []any{pgsq02ProjectA, pgsq02GenOne, json.RawMessage(`["FHIR/Patient","FHIR/Specimen"]`), "Patient", pgsq02Label, "Specimen"} + for _, p := range []struct { + desc, sql string + wantRows [][]string + }{ + {"full chain (control)", edgeJoin + edgeScope + targetJoin + where + order, wantRoute}, + {"without edge auth", edgeJoin + + `e.project=$1 AND e.dataset_generation=$2 AND e.label=$5 AND e.source_resource_type=$4` + + ` AND e.source_logical_id=r0.logical_id AND e.target_resource_type=$6` + targetJoin + where + order, + [][]string{{"sp-1", "st-allow"}, {"sp-1", "st-off"}}}, + {"without target auth", edgeJoin + edgeScope + + ` JOIN "` + svc.schema + `".fhir_resource r1 ON r1.project=$1 AND r1.dataset_generation=$2` + + ` AND r1.resource_type=$6 AND r1.logical_id=e.target_logical_id` + where + order, + [][]string{{"sp-1", "st-allow"}, {"sp-1", "st-hidden"}}}, + } { + rows := svc.run(t, p.sql, scopeArgs...) + if !reflect.DeepEqual(rows, p.wantRows) { + t.Fatalf("guard-drop %s rows = %v, want %v", p.desc, rows, p.wantRows) + } + } + + first, second, pid1, pid2 := svc.runOnTwoDistinctConnections(t, rendered) + if pid1 == pid2 { + t.Fatalf("the two connections were the same server process: pg_backend_pid %d", pid1) + } + if !reflect.DeepEqual(first, wantRoute) || !reflect.DeepEqual(second, wantRoute) { + t.Fatalf("connection-scoped rows diverged: %v vs %v vs want %v", first, second, wantRoute) + } + denyFirst, denySecond, denyPID1, denyPID2 := svc.runOnTwoDistinctConnections(t, denyRendered) + if denyPID1 == denyPID2 { + t.Fatalf("the two deny-all connections were the same server process: pg_backend_pid %d", denyPID1) + } + if len(denyFirst) != 0 || len(denySecond) != 0 { + t.Fatalf("deny-all read rows on a dedicated connection: %v / %v", denyFirst, denySecond) + } + + duplicateEdge := `INSERT INTO "` + svc.schema + `".fhir_edge_occurrence (project, dataset_generation, auth_resource_path, label,` + + ` source_resource_type, source_logical_id, target_resource_type, target_logical_id, source_path, source_ordinal)` + + ` VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)` + svc.expectRefused(t, duplicateEdge, "23505", "fhir_edge_occurrence_identity_ux", + pgsq02ProjectA, pgsq02GenOne, pgsq02PathPatient, pgsq02Label, "Patient", "sp-1", "Specimen", "st-allow", "specimen[0]", int64(0)) + svc.expectRefused(t, duplicateEdge, "23514", "fhir_edge_occurrence_project_present", + "", pgsq02GenOne, pgsq02PathPatient, pgsq02Label, "Patient", "sp-1", "Specimen", "st-allow", "specimen[1]", int64(0)) +} + +func TestPGSQ02EdgeIdentityAcrossScopesStaysIsolated(t *testing.T) { + svc := newPGSQ02Service(t) + type scope struct{ project, generation string } + scopes := []scope{ + {pgsq02ProjectA, pgsq02GenOne}, + {pgsq02ProjectB, pgsq02GenOne}, + {pgsq02ProjectA, pgsq02GenTwo}, + {pgsq02ProjectB, pgsq02GenTwo}, + } + for _, s := range scopes { + svc.insertResource(t, s.project, s.generation, "Patient", "p-src", pgsq02PathPatient) + svc.insertResource(t, s.project, s.generation, "Specimen", "p-tgt", pgsq02PathSpecimen) + } + for _, s := range scopes { + svc.insertEdge(t, s.project, s.generation, pgsq02PathPatient, "Patient", "p-src", "Specimen", "p-tgt", "specimen[0]", 0) + } + + for _, pair := range []struct { + sql string + args []any + wanted string + }{ + {`SELECT count(*) FROM "` + svc.schema + `".fhir_resource WHERE resource_type='Patient' AND logical_id='p-src'`, nil, "4"}, + {`SELECT count(*) FROM "` + svc.schema + `".fhir_resource WHERE resource_type='Specimen' AND logical_id='p-tgt'`, nil, "4"}, + {`SELECT count(*) FROM "` + svc.schema + `".fhir_edge_occurrence WHERE label=$1 AND source_logical_id='p-src' AND target_logical_id='p-tgt'`, []any{pgsq02Label}, "4"}, + } { + rows := svc.run(t, pair.sql, pair.args...) + if len(rows) != 1 || rows[0][0] != pair.wanted { + t.Fatalf("witness %s = %v, want %s stored", pair.sql, rows, pair.wanted) + } + } + + plan := pgsq02TraversalPlanForScope(pgsq02ProjectA, pgsq02GenOne, + []string{pgsq02PathPatient, pgsq02PathSpecimen}, false, "Patient", pgsq02Label, "Specimen") + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + wantArgs := []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: pgsq02ProjectA}, + {Type: command.PostgreSQLParameterText, Value: pgsq02GenOne}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["FHIR/Patient","FHIR/Specimen"]`)}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + {Type: command.PostgreSQLParameterText, Value: pgsq02Label}, + {Type: command.PostgreSQLParameterText, Value: "Specimen"}, + } + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("args = %#v, want %#v", rendered.Args, wantArgs) + } + want := [][]string{{"p-src", "p-tgt"}} + if got := svc.runRendered(t, rendered); !reflect.DeepEqual(got, want) { + t.Fatalf("scoped traversal over four live scopes = %v, want exactly the authorized occurrence %v", got, want) + } + + edgeJoin := `SELECT r0.logical_id, r0.project, r1.project, r1.logical_id FROM "` + svc.schema + `".fhir_resource r0` + + ` JOIN "` + svc.schema + `".fhir_edge_occurrence e ON ` + targetPrefix := ` JOIN "` + svc.schema + `".fhir_resource r1 ON ` + rootWhere := ` WHERE r0.project=$1 AND r0.dataset_generation=$2` + + ` AND r0.auth_resource_path = ANY (SELECT jsonb_array_elements_text($3::jsonb)) AND r0.resource_type=$4` + order := ` ORDER BY r0.resource_seq, e.edge_seq, r1.resource_seq` + scopeArgs := []any{pgsq02ProjectA, pgsq02GenOne, json.RawMessage(`["FHIR/Patient","FHIR/Specimen"]`), "Patient", pgsq02Label, "Specimen"} + edgeFull := `e.project=$1 AND e.dataset_generation=$2 AND e.auth_resource_path = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND e.label=$5 AND e.source_resource_type=$4 AND e.source_logical_id=r0.logical_id AND e.target_resource_type=$6` + targetFull := `r1.project=$1 AND r1.dataset_generation=$2 AND r1.auth_resource_path = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND r1.resource_type=$6 AND r1.logical_id=e.target_logical_id` + authorizedRow := []string{"p-src", pgsq02ProjectA, pgsq02ProjectA, "p-tgt"} + for _, p := range []struct { + desc, sql string + wantRows [][]string + }{ + {"full chain (control)", edgeJoin + edgeFull + targetPrefix + targetFull + rootWhere + order, + [][]string{authorizedRow}}, + {"without edge project", edgeJoin + + strings.Replace(edgeFull, `e.project=$1 AND `, "", 1) + targetPrefix + targetFull + rootWhere + order, + [][]string{authorizedRow, authorizedRow}}, + {"without edge dataset_generation", edgeJoin + + strings.Replace(edgeFull, `e.dataset_generation=$2 AND `, "", 1) + targetPrefix + targetFull + rootWhere + order, + [][]string{authorizedRow, authorizedRow}}, + {"without root project", edgeJoin + edgeFull + targetPrefix + targetFull + + strings.Replace(rootWhere, `r0.project=$1 AND `, "", 1) + order, + [][]string{authorizedRow, {"p-src", pgsq02ProjectB, pgsq02ProjectA, "p-tgt"}}}, + {"without target project", edgeJoin + edgeFull + targetPrefix + + strings.Replace(targetFull, `r1.project=$1 AND `, "", 1) + rootWhere + order, + [][]string{authorizedRow, {"p-src", pgsq02ProjectA, pgsq02ProjectB, "p-tgt"}}}, + } { + rows := svc.run(t, p.sql, scopeArgs...) + if !reflect.DeepEqual(rows, p.wantRows) { + t.Fatalf("guard-drop %s rows = %v, want %v", p.desc, rows, p.wantRows) + } + } + + duplicateEdge := `INSERT INTO "` + svc.schema + `".fhir_edge_occurrence (project, dataset_generation, auth_resource_path, label,` + + ` source_resource_type, source_logical_id, target_resource_type, target_logical_id, source_path, source_ordinal)` + + ` VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)` + svc.expectRefused(t, duplicateEdge, "23505", "fhir_edge_occurrence_identity_ux", + pgsq02ProjectA, pgsq02GenOne, pgsq02PathPatient, pgsq02Label, "Patient", "p-src", "Specimen", "p-tgt", "specimen[0]", int64(0)) + svc.expectRefused(t, duplicateEdge, "23514", "fhir_edge_occurrence_project_present", + "", pgsq02GenOne, pgsq02PathPatient, pgsq02Label, "Patient", "p-src", "Specimen", "p-tgt", "specimen[9]", int64(0)) + duplicateResource := `INSERT INTO "` + svc.schema + `".fhir_resource (project, dataset_generation, resource_type, logical_id, auth_resource_path) VALUES ($1,$2,$3,$4,$5)` + svc.expectRefused(t, duplicateResource, "23514", "fhir_resource_project_present", + "", pgsq02GenOne, "Patient", "ck-project", pgsq02PathPatient) +} + +func TestPGSQ02ScopeAllowedFalseReadsZeroRowsRootAndEdge(t *testing.T) { + svc := newPGSQ02Service(t) + svc.insertResource(t, pgsq02ProjectA, pgsq02GenOne, "Patient", "sp-1", pgsq02PathPatient) + svc.insertResource(t, pgsq02ProjectA, pgsq02GenOne, "Specimen", "st-allow", pgsq02PathSpecimen) + svc.insertResource(t, pgsq02ProjectA, pgsq02GenOne, "Specimen", "st-hidden", pgsq02PathConf) + svc.insertEdge(t, pgsq02ProjectA, pgsq02GenOne, pgsq02PathPatient, "Patient", "sp-1", "Specimen", "st-allow", "specimen[0]", 0) + + binding := bindingFor(t, svc.schema) + authorized := []string{pgsq02PathPatient, pgsq02PathSpecimen} + + rootAllowed, err := RenderRootScan(pgsq02RootPlan(pgsq02ProjectA, pgsq02GenOne, authorized, false, + "Patient", "logical_id"), binding) + if err != nil { + t.Fatalf("allowed root scan: %v", err) + } + if got := svc.runRendered(t, rootAllowed); !reflect.DeepEqual(got, [][]string{{"sp-1"}}) { + t.Fatalf("allowed root scan rows = %v, want [sp-1]", got) + } + edgeAllowed, err := RenderPhysicalPlan( + pgsq02TraversalPlanForScope(pgsq02ProjectA, pgsq02GenOne, authorized, false, "Patient", pgsq02Label, "Specimen"), + binding) + if err != nil { + t.Fatalf("allowed traversal: %v", err) + } + wantRoute := [][]string{{"sp-1", "st-allow"}} + if got := svc.runRendered(t, edgeAllowed); !reflect.DeepEqual(got, wantRoute) { + t.Fatalf("allowed traversal rows = %v, want %v", got, wantRoute) + } + + deniedRoot, err := RenderRootScan(pgsq02DeniedPlan(pgsq02RootPlan(pgsq02ProjectA, pgsq02GenOne, authorized, false, + "Patient", "logical_id")), binding) + if err != nil { + t.Fatalf("denied root scan: %v", err) + } + if !reflect.DeepEqual(deniedRoot.Args, rootAllowed.Args) { + t.Fatalf("denied root args drifted: %#v vs %#v", deniedRoot.Args, rootAllowed.Args) + } + if got := svc.runRendered(t, deniedRoot); len(got) != 0 { + t.Fatalf("denied root scan read %v, want zero rows", got) + } + + deniedEdge, err := RenderPhysicalPlan(pgsq02DeniedPlan( + pgsq02TraversalPlanForScope(pgsq02ProjectA, pgsq02GenOne, authorized, false, "Patient", pgsq02Label, "Specimen")), + binding) + if err != nil { + t.Fatalf("denied traversal: %v", err) + } + if !reflect.DeepEqual(deniedEdge.Args, edgeAllowed.Args) { + t.Fatalf("denied traversal args drifted: %#v vs %#v", deniedEdge.Args, edgeAllowed.Args) + } + if got := svc.runRendered(t, deniedEdge); len(got) != 0 { + t.Fatalf("denied traversal read %v, want zero rows", got) + } + + for _, probe := range []struct { + desc, sql string + args []any + wanted string + }{ + {"root row", `SELECT count(*) FROM "` + svc.schema + `".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' AND logical_id='sp-1'`, + []any{pgsq02ProjectA, pgsq02GenOne}, "1"}, + {"target row", `SELECT count(*) FROM "` + svc.schema + `".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Specimen' AND logical_id='st-allow'`, + []any{pgsq02ProjectA, pgsq02GenOne}, "1"}, + {"auth decoy", `SELECT count(*) FROM "` + svc.schema + `".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Specimen' AND logical_id='st-hidden'`, + []any{pgsq02ProjectA, pgsq02GenOne}, "1"}, + {"edge occurrence", `SELECT count(*) FROM "` + svc.schema + `".fhir_edge_occurrence WHERE project=$1 AND dataset_generation=$2 AND label=$3`, + []any{pgsq02ProjectA, pgsq02GenOne, pgsq02Label}, "1"}, + } { + rows := svc.run(t, probe.sql, probe.args...) + if len(rows) != 1 || rows[0][0] != probe.wanted { + t.Fatalf("%s must be stored (%s): %v", probe.desc, probe.wanted, rows) + } + } + + first, second, pid1, pid2 := svc.runOnTwoDistinctConnections(t, deniedEdge) + if pid1 == pid2 { + t.Fatalf("the two deny connections were the same server process: pg_backend_pid %d", pid1) + } + if len(first) != 0 || len(second) != 0 { + t.Fatalf("denied traversal read rows on dedicated connections: %v / %v", first, second) + } +} + +func TestZZNoLeftoverPgSQ02Resources(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer admin.Close() + var databases int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'pgsq02_it_%'").Scan(&databases); err != nil { + t.Fatalf("count databases: %v", err) + } + if databases != 0 { + t.Fatalf("%d pgsq02_it_* databases still exist", databases) + } + var schemas int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgsq02_it_%'").Scan(&schemas); err != nil { + t.Fatalf("count schemas: %v", err) + } + if schemas != 0 { + t.Fatalf("%d pgsq02_it_* schemas still exist in the service database", schemas) + } +} diff --git a/internal/dataframe/compiler/render/postgres/selectors.go b/internal/dataframe/compiler/render/postgres/selectors.go new file mode 100644 index 000000000..c98e03f17 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/selectors.go @@ -0,0 +1,443 @@ +package postgres + +import ( + "fmt" + "reflect" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +func RenderSelectorExtraction(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + if err := validateSelectorExtractionPlan(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return RenderPhysicalPlan(plan, storage) +} + +type extractionProjectionKind uint8 + +const ( + extractionDirectValue extractionProjectionKind = iota + extractionValueExpression + extractionSelector + extractionPresenceSidecar + extractionCallExpression + extractionObject + extractionSlice + extractionUnitNormalized + extractionAggregate +) + +type classifiedExtractionProjection struct { + kind extractionProjectionKind + projection ir.PhysicalProjection + expression *ir.PhysicalExpression + path []string + presence *ir.PhysicalProjectionPresence + presencePath []string + unit *unitProjectionFacts +} + +type extractionEvidence struct { + variable string + path string +} + +func pathEvidenceKey(path []string) string { + return strings.Join(path, "\x1f") +} + +func classifyExtractionProjections(projections []ir.PhysicalProjection, + rowType func(variable string) (resourceType string, ok bool)) ([]classifiedExtractionProjection, error) { + classified := make([]classifiedExtractionProjection, 0, len(projections)) + extracted := make(map[extractionEvidence]bool) + for index := range projections { + projection := projections[index] + c, err := classifyExtractionProjection(projection) + if err != nil { + return nil, err + } + switch c.kind { + case extractionDirectValue: + if _, ok := rowType(c.projection.Value.Variable); !ok { + return nil, fmt.Errorf("%w: projection %q reads variable %q, which does not bind a readable row", + ErrUnsupportedPostgresPlan, projection.Name, c.projection.Value.Variable) + } + case extractionValueExpression: + if _, ok := rowType(c.projection.Expression.Value.Variable); !ok { + return nil, fmt.Errorf("%w: projection %q reads variable %q, which does not bind a readable row", + ErrUnsupportedPostgresPlan, projection.Name, c.projection.Expression.Value.Variable) + } + case extractionSelector: + source := c.expression.Extract.Source + resourceType, ok := rowType(source.Variable) + if !ok { + return nil, fmt.Errorf("%w: projection %q extracts from variable %q, which does not bind a readable row", + ErrUnsupportedPostgresPlan, projection.Name, source.Variable) + } + if c.expression.Extract.ResourceType != resourceType { + return nil, fmt.Errorf("%w: projection %q validates selector against resource type %q, but the row is scanned as %q", + ErrUnsupportedPostgresPlan, projection.Name, c.expression.Extract.ResourceType, resourceType) + } + extracted[extractionEvidence{variable: source.Variable, path: pathEvidenceKey(c.path)}] = true + case extractionUnitNormalized: + resourceType, ok := rowType(c.unit.sourceVariable) + if !ok { + return nil, fmt.Errorf("%w: projection %q extracts from variable %q, which does not bind a readable row", + ErrUnsupportedPostgresPlan, projection.Name, c.unit.sourceVariable) + } + if c.unit.resourceType != resourceType { + return nil, fmt.Errorf("%w: projection %q validates unit selectors against resource type %q, but the row is scanned as %q", + ErrUnsupportedPostgresPlan, projection.Name, c.unit.resourceType, resourceType) + } + case extractionCallExpression: + if err := forEachCallOperandRowVariable(*projection.Expression, + func(variable string, extract *ir.PhysicalExtract) error { + resourceType, ok := rowType(variable) + if !ok { + return fmt.Errorf("%w: projection %q reads variable %q, which does not bind a readable row", + ErrUnsupportedPostgresPlan, projection.Name, variable) + } + if extract != nil && extract.ResourceType != resourceType { + return fmt.Errorf("%w: projection %q validates selector against resource type %q, but the row is scanned as %q", + ErrUnsupportedPostgresPlan, projection.Name, extract.ResourceType, resourceType) + } + return nil + }); err != nil { + return nil, err + } + case extractionPresenceSidecar: + if _, ok := rowType(c.presence.Source.Variable); !ok { + return nil, fmt.Errorf("%w: presence projection %q proves variable %q, which does not bind a readable row", + ErrUnsupportedPostgresPlan, projection.Name, c.presence.Source.Variable) + } + } + classified = append(classified, c) + } + for _, c := range classified { + if c.kind != extractionPresenceSidecar { + continue + } + if !extracted[extractionEvidence{variable: c.presence.Source.Variable, path: pathEvidenceKey(c.presencePath)}] { + return nil, fmt.Errorf("%w: presence projection %q proves path %v of variable %q, which no selector in this RETURN extracts: presence metadata must agree with a rendered projection", + ErrUnsupportedPostgresPlan, c.projection.Name, c.presencePath, c.presence.Source.Variable) + } + } + return classified, nil +} + +func hiddenRootKeyRead(projection ir.PhysicalProjection) bool { + return projection.Expression == nil && !projection.PresenceOutput && projection.Presence == nil && + projection.Value.Variable != "" && projection.Value.BindKey == "" && + len(projection.Value.Path) == 1 && (projection.Value.Path[0] == "id" || projection.Value.Path[0] == "_key") +} + +func hiddenAuthResourcePathRead(projection ir.PhysicalProjection) bool { + return projection.Expression == nil && !projection.PresenceOutput && projection.Presence == nil && + projection.Value.Variable != "" && projection.Value.BindKey == "" && + len(projection.Value.Path) == 1 && projection.Value.Path[0] == "auth_resource_path" +} + +func classifyExtractionProjection(projection ir.PhysicalProjection) (classifiedExtractionProjection, error) { + if strings.TrimSpace(projection.Name) == "" { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection names are required output metadata (never SQL text) and must be set", + ErrUnsupportedPostgresPlan) + } + if projection.Hidden { + if projection.Name == ir.PreviewSourceResourceIDColumn && hiddenRootKeyRead(projection) { + return classifiedExtractionProjection{kind: extractionDirectValue, projection: projection}, nil + } + if projection.Name == "auth_resource_path" && hiddenAuthResourcePathRead(projection) { + return classifiedExtractionProjection{kind: extractionDirectValue, projection: projection}, nil + } + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q is hidden executor-side metadata with no registered PostgreSQL rendering", + ErrUnsupportedPostgresPlan, projection.Name) + } + hasValue := projection.Value.Variable != "" || projection.Value.BindKey != "" || len(projection.Value.Path) != 0 + if projection.PresenceOutput { + if projection.Presence == nil { + return classifiedExtractionProjection{}, fmt.Errorf("%w: presence projection %q has no presence proof to emit", + ErrUnsupportedPostgresPlan, projection.Name) + } + if hasValue || projection.Expression != nil { + return classifiedExtractionProjection{}, fmt.Errorf("%w: presence projection %q must carry only its typed presence proof", + ErrUnsupportedPostgresPlan, projection.Name) + } + presence := projection.Presence + if presence.Source.BindKey != "" { + return classifiedExtractionProjection{}, fmt.Errorf("%w: presence projection %q must prove a row path, not a bind value", + ErrUnsupportedPostgresPlan, projection.Name) + } + if len(presence.Source.Path) != 1 || presence.Source.Path[0] != "payload" { + return classifiedExtractionProjection{}, fmt.Errorf(`%w: presence projection %q must prove a path of the row document at ["payload"], the exact source form lowering emits; got %v`, + ErrUnsupportedPostgresPlan, projection.Name, presence.Source.Path) + } + if len(presence.Paths) != 1 { + return classifiedExtractionProjection{}, fmt.Errorf("%w: presence projection %q carries %d candidate paths; multi-path presence proofs depend on selector fallbacks, which are a separate capability", + ErrUnsupportedPostgresPlan, projection.Name, len(presence.Paths)) + } + return classifiedExtractionProjection{ + kind: extractionPresenceSidecar, + projection: projection, + presence: presence, + presencePath: append([]string(nil), presence.Paths[0]...), + }, nil + } + if projection.Expression != nil { + if hasValue { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q sets both a direct value and an expression; exactly one is allowed", + ErrUnsupportedPostgresPlan, projection.Name) + } + expression := projection.Expression + switch expression.Kind { + case ir.PhysicalValueExpression: + if expression.Value == nil { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q uses a VALUE expression with no value", + ErrUnsupportedPostgresPlan, projection.Name) + } + if projection.Presence != nil { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q carries presence metadata on a value expression; presence proofs pair with selector extractions only", + ErrUnsupportedPostgresPlan, projection.Name) + } + return classifiedExtractionProjection{kind: extractionValueExpression, projection: projection}, nil + case ir.PhysicalExtractExpression: + if expression.Extract != nil && expression.Extract.UnitNormalization != nil { + return classifyUnitNormalizedProjection(projection) + } + path, err := validateSelectorExtraction(projection) + if err != nil { + return classifiedExtractionProjection{}, err + } + return classifiedExtractionProjection{ + kind: extractionSelector, + projection: projection, + expression: expression, + path: path, + }, nil + case ir.PhysicalCallExpression: + if err := validateCallProjection(projection); err != nil { + return classifiedExtractionProjection{}, err + } + return classifiedExtractionProjection{ + kind: extractionCallExpression, + projection: projection, + expression: expression, + }, nil + case ir.PhysicalAggregateExpression: + if projection.Presence != nil { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q carries presence metadata on an AGGREGATE expression; presence proofs pair with selector extractions only", + ErrUnsupportedPostgresPlan, projection.Name) + } + return classifiedExtractionProjection{ + kind: extractionAggregate, + projection: projection, + expression: expression, + }, nil + case ir.PhysicalObjectExpression, ir.PhysicalSliceExpression: + if projection.Presence != nil { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q carries presence metadata on a %s expression; presence proofs pair with selector extractions only", + ErrUnsupportedPostgresPlan, projection.Name, string(expression.Kind)) + } + if expression.Kind == ir.PhysicalObjectExpression { + if expression.Object == nil { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q uses an OBJECT expression with no payload", + ErrUnsupportedPostgresPlan, projection.Name) + } + return classifiedExtractionProjection{kind: extractionObject, projection: projection}, nil + } + if expression.Slice == nil { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q uses a SLICE expression with no payload", + ErrUnsupportedPostgresPlan, projection.Name) + } + return classifiedExtractionProjection{kind: extractionSlice, projection: projection}, nil + default: + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q uses expression kind %q, which has no registered PostgreSQL rendering", + ErrUnsupportedPostgresPlan, projection.Name, string(expression.Kind)) + } + } + if !hasValue { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q must read exactly one typed row value", + ErrUnsupportedPostgresPlan, projection.Name) + } + if projection.Value.BindKey != "" { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q must read a row value, not a bind value", + ErrUnsupportedPostgresPlan, projection.Name) + } + if projection.Presence != nil { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q carries presence metadata on a direct value; presence proofs pair with selector extractions only", + ErrUnsupportedPostgresPlan, projection.Name) + } + return classifiedExtractionProjection{kind: extractionDirectValue, projection: projection}, nil +} + +func validateSelectorExtraction(projection ir.PhysicalProjection) ([]string, error) { + expression := projection.Expression + extract := expression.Extract + if extract == nil { + return nil, fmt.Errorf("%w: projection %q: EXTRACT expression is missing its payload", + ErrUnsupportedPostgresPlan, projection.Name) + } + if expression.Cardinality != ir.PhysicalScalarCardinality { + return nil, fmt.Errorf("%w: projection %q: selector extraction is registered for SCALAR cardinality only; array and object selector projections belong to a later capability", + ErrUnsupportedPostgresPlan, projection.Name) + } + switch expression.NullBehavior { + case ir.PhysicalPreserveNull, ir.PhysicalOmitNulls, ir.PhysicalEmptyOnNull: + default: + return nil, fmt.Errorf("%w: projection %q: null behavior %q has no registered PostgreSQL extraction form", + ErrUnsupportedPostgresPlan, projection.Name, string(expression.NullBehavior)) + } + switch extract.ExecutionMode { + case "", ir.PhysicalSelectorGeneric, ir.PhysicalSelectorDirectScalar: + case ir.PhysicalSelectorConditionalArray: + return nil, fmt.Errorf("%w: projection %q: CONDITIONAL_ARRAY selector mode requires repeated-step rendering, which is not registered for PostgreSQL", + ErrUnsupportedPostgresPlan, projection.Name) + default: + return nil, fmt.Errorf("%w: projection %q: unknown selector execution mode %q", + ErrUnsupportedPostgresPlan, projection.Name, string(extract.ExecutionMode)) + } + if extract.Distinct { + return nil, fmt.Errorf("%w: projection %q: DISTINCT projection is meaningful only for array-valued selectors", + ErrUnsupportedPostgresPlan, projection.Name) + } + if len(extract.Fallbacks) != 0 { + return nil, fmt.Errorf("%w: projection %q: ordered fallback selectors are a separate capability", + ErrUnsupportedPostgresPlan, projection.Name) + } + if extract.Prepared != nil { + return nil, fmt.Errorf("%w: projection %q: prepared-set selector references are not implemented for PostgreSQL", + ErrUnsupportedPostgresPlan, projection.Name) + } + if extract.UnitNormalization != nil { + return nil, fmt.Errorf("%w: projection %q: unit-normalized extraction is not implemented for PostgreSQL", + ErrUnsupportedPostgresPlan, projection.Name) + } + if extract.Selector.Filter != nil { + return nil, fmt.Errorf("%w: projection %q: selector CONTAINS filters are not implemented for PostgreSQL", + ErrUnsupportedPostgresPlan, projection.Name) + } + if len(extract.Selector.Steps) == 0 { + return nil, fmt.Errorf("%w: projection %q: the selector is empty", ErrUnsupportedPostgresPlan, projection.Name) + } + path := make([]string, 0, len(extract.Selector.Steps)) + for index, step := range extract.Selector.Steps { + switch { + case step.Field == "": + return nil, fmt.Errorf("%w: projection %q: selector step %d has no property", + ErrUnsupportedPostgresPlan, projection.Name, index) + case step.Iterate: + return nil, fmt.Errorf("%w: projection %q: repeated ([]) selector step %d requires array rendering, which is not registered", + ErrUnsupportedPostgresPlan, projection.Name, index) + case step.Index != nil: + return nil, fmt.Errorf("%w: projection %q: indexed selector step %d requires array rendering, which is not registered", + ErrUnsupportedPostgresPlan, projection.Name, index) + } + if !payloadPathSegmentPattern.MatchString(step.Field) { + return nil, fmt.Errorf("%w: projection %q: selector step %q is not renderer-safe", + ErrUnsupportedPostgresPlan, projection.Name, step.Field) + } + path = append(path, step.Field) + } + if extract.Source.BindKey != "" { + return nil, fmt.Errorf("%w: projection %q: the selector source must be a row document, not a bind value", + ErrUnsupportedPostgresPlan, projection.Name) + } + if len(extract.Source.Path) != 1 || extract.Source.Path[0] != "payload" { + return nil, fmt.Errorf(`%w: projection %q: the selector source must be the row document at ["payload"], the exact source shape lowering emits (lower/root_projection.go); got %v`, + ErrUnsupportedPostgresPlan, projection.Name, extract.Source.Path) + } + if strings.TrimSpace(extract.ResourceType) == "" { + return nil, fmt.Errorf("%w: projection %q: selector extraction must name the schema resource type it was validated against", + ErrUnsupportedPostgresPlan, projection.Name) + } + if projection.Presence != nil { + presence := projection.Presence + if presence.Source.Variable != extract.Source.Variable || presence.Source.BindKey != "" || + !reflect.DeepEqual(presence.Source.Path, extract.Source.Path) { + return nil, fmt.Errorf("%w: projection %q: presence metadata must prove the selector's own document source", + ErrUnsupportedPostgresPlan, projection.Name) + } + if len(presence.Paths) != 1 || !reflect.DeepEqual(presence.Paths[0], path) { + return nil, fmt.Errorf("%w: projection %q: presence metadata must be exactly the selector path; multi-path presence claims belong to the selector-fallback capability", + ErrUnsupportedPostgresPlan, projection.Name) + } + } + return path, nil +} + +func validateSelectorExtractionPlan(plan ir.PhysicalPlan) error { + if len(plan.Operations) != 2 { + return fmt.Errorf( + "%w: the selector-extraction slice is exactly one ROOT_SCAN followed by one RETURN, got %d operation(s); filters, sorts, limits, traversals, and every other operation belong to the full read spine or to a later capability, not to RenderSelectorExtraction", + ErrUnsupportedPostgresPlan, len(plan.Operations)) + } + rootOperation, returnOperation := plan.Operations[0], plan.Operations[1] + if rootOperation.Kind != ir.PhysicalRootScanOp || rootOperation.RootScan == nil { + return fmt.Errorf("%w: the leading operation must be ROOT_SCAN with its payload set; the selector-extraction slice never renders a traversal, a collection scan, or a payload-less ROOT_SCAN", + ErrUnsupportedPostgresPlan) + } + if returnOperation.Kind != ir.PhysicalReturnOp || returnOperation.Return == nil { + return fmt.Errorf("%w: the terminal operation must be RETURN with its payload set", ErrUnsupportedPostgresPlan) + } + + scan := rootOperation.RootScan + if scan.Population != nil || scan.CohortSource != nil || scan.PageCandidateSeed != nil { + return fmt.Errorf("%w: population, cohort, and candidate-seed roots are not registered for PostgreSQL; only the direct root collection scan has a rendered contract", + ErrUnsupportedPostgresPlan) + } + if scan.Variable == "" { + return fmt.Errorf("%w: ROOT_SCAN must name its row variable", ErrUnsupportedPostgresPlan) + } + if scan.CollectionBindKey == "" { + return fmt.Errorf("%w: ROOT_SCAN must name the bind carrying the single resource type", ErrUnsupportedPostgresPlan) + } + resourceType, ok := plan.BindVars[scan.CollectionBindKey].(string) + if !ok || strings.TrimSpace(resourceType) == "" { + return fmt.Errorf("%w: root collection bind %q must name one resource type as a non-empty string", + ErrUnsupportedPostgresPlan, scan.CollectionBindKey) + } + if len(returnOperation.Return.Projections) == 0 { + return fmt.Errorf("%w: the selector-extraction RETURN must project at least one selector extraction or typed scope column", + ErrUnsupportedPostgresPlan) + } + + classified, err := classifyExtractionProjections(returnOperation.Return.Projections, + func(variable string) (string, bool) { + if variable == scan.Variable { + return resourceType, true + } + return "", false + }) + if err != nil { + return err + } + for _, c := range classified { + switch c.kind { + case extractionDirectValue: + value := c.projection.Value + if len(value.Path) != 1 || !resourceFilterColumns[canonicalKeyColumn(value.Path[0])] { + return fmt.Errorf("%w: projection %q must read exactly one typed scope column of the root row; payload values reach PostgreSQL through EXTRACT selectors, not direct values", + ErrUnsupportedPostgresPlan, c.projection.Name) + } + case extractionValueExpression: + return fmt.Errorf("%w: projection %q uses a VALUE expression; the selector-extraction slice emits typed scope columns, EXTRACT selector values, and presence sidecars", + ErrUnsupportedPostgresPlan, c.projection.Name) + case extractionCallExpression: + return fmt.Errorf("%w: projection %q uses a CALL expression; derived scalar calls register on the full read spine (RenderPhysicalPlan), not the closed selector-extraction slice", + ErrUnsupportedPostgresPlan, c.projection.Name) + case extractionUnitNormalized: + return fmt.Errorf("%w: projection %q uses a unit-normalized extraction; the selector-extraction slice emits typed scope columns, EXTRACT selector values, and presence sidecars — unit normalization belongs to the read spine registration (unit_normalization.go)", + ErrUnsupportedPostgresPlan, c.projection.Name) + case extractionObject, extractionSlice: + return fmt.Errorf("%w: projection %q uses a %s expression; the selector-extraction slice emits typed scope columns, EXTRACT selector values, and presence sidecars — composites belong to the read spine registration (objects.go)", + ErrUnsupportedPostgresPlan, c.projection.Name, string(c.projection.Expression.Kind)) + case extractionAggregate: + return fmt.Errorf("%w: projection %q uses an AGGREGATE expression; the selector-extraction slice emits typed scope columns, EXTRACT selector values, and presence sidecars — set reducers belong to the read spine registration (post_window.go)", + ErrUnsupportedPostgresPlan, c.projection.Name) + case extractionSelector, extractionPresenceSidecar: + } + } + return nil +} diff --git a/internal/dataframe/compiler/render/postgres/selectors_test.go b/internal/dataframe/compiler/render/postgres/selectors_test.go new file mode 100644 index 000000000..be73ebec6 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/selectors_test.go @@ -0,0 +1,1092 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + control "github.com/calypr/loom/internal/backends/postgres/control" + "net/url" + "os" + "reflect" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +var pgsq03ResourceNamePattern = regexp.MustCompile(`^pgsq03_it_[0-9]+_[0-9a-f]+$`) + +func pgsq03RawProjection(name string, expression *ir.PhysicalExpression) ir.PhysicalProjection { + return ir.PhysicalProjection{Name: name, Expression: expression} +} + +func pgsq03ExtractExpression(override func(*ir.PhysicalExtract, *ir.PhysicalExpression)) *ir.PhysicalExpression { + selector := spec.Selector{Steps: []spec.SelectorStep{{Field: "gender"}}} + extract := &ir.PhysicalExtract{ + Source: pgsq03DocumentSource(), + ResourceType: "Patient", + Selector: selector, + ExecutionMode: ir.PhysicalSelectorGeneric, + } + expression := &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Extract: extract, + } + if override != nil { + override(extract, expression) + } + return expression +} + +func pgsq03ActiveSelector() spec.Selector { + return spec.Selector{Steps: []spec.SelectorStep{{Field: "active"}}} +} + +func TestSelectorExtractionRejectsEveryShapeOutsideTheClosedSlice(t *testing.T) { + binding := mustBinding(t) + indexZero := 0 + cases := []struct { + name string + plan ir.PhysicalPlan + wantIn string + }{ + {"no operations", func() ir.PhysicalPlan { + plan := pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", pgsq03ExtractExpression(nil))) + plan.Operations = nil + return plan + }(), "exactly one ROOT_SCAN"}, + {"scan without return", func() ir.PhysicalPlan { + plan := pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", pgsq03ExtractExpression(nil))) + plan.Operations = plan.Operations[:1] + return plan + }(), "exactly one ROOT_SCAN"}, + {"return before scan", func() ir.PhysicalPlan { + plan := pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", pgsq03ExtractExpression(nil))) + plan.Operations = []ir.PhysicalOperation{plan.Operations[1], plan.Operations[0]} + return plan + }(), "leading operation must be ROOT_SCAN"}, + {"scan without payload", func() ir.PhysicalPlan { + plan := pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", pgsq03ExtractExpression(nil))) + plan.Operations[0].RootScan = nil + return plan + }(), "payload set"}, + {"return without payload", func() ir.PhysicalPlan { + plan := pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", pgsq03ExtractExpression(nil))) + plan.Operations[1].Return = nil + return plan + }(), "RETURN with its payload set"}, + {"population root", func() ir.PhysicalPlan { + plan := pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", pgsq03ExtractExpression(nil))) + plan.Operations[0].RootScan.Population = &ir.PhysicalPopulationRootSource{} + return plan + }(), "not registered for PostgreSQL"}, + {"cohort root", func() ir.PhysicalPlan { + plan := pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", pgsq03ExtractExpression(nil))) + plan.Operations[0].RootScan.CohortSource = &ir.PhysicalCohortRootSource{} + return plan + }(), "not registered for PostgreSQL"}, + {"candidate seed root", func() ir.PhysicalPlan { + plan := pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", pgsq03ExtractExpression(nil))) + plan.Operations[0].RootScan.PageCandidateSeed = &ir.PhysicalRootPageCandidateSeed{} + return plan + }(), "not registered for PostgreSQL"}, + {"unnamed root variable", func() ir.PhysicalPlan { + plan := pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", pgsq03ExtractExpression(nil))) + plan.Operations[0].RootScan.Variable = "" + return plan + }(), "must name its row variable"}, + {"missing collection bind key", func() ir.PhysicalPlan { + plan := pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", pgsq03ExtractExpression(nil))) + plan.Operations[0].RootScan.CollectionBindKey = "" + return plan + }(), "bind carrying the single resource type"}, + {"collection bind not a string", func() ir.PhysicalPlan { + plan := pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", pgsq03ExtractExpression(nil))) + plan.BindVars["root_collection"] = 7 + return plan + }(), "root collection bind"}, + {"empty return", pgsq03SelectorPlan("Patient"), "at least one selector extraction"}, + {"blank projection name", pgsq03SelectorPlan("Patient", + pgsq03RawProjection(" ", pgsq03ExtractExpression(nil))), "projection names are required"}, + {"hidden projection", func() ir.PhysicalPlan { + projection := pgsq03RawProjection("g", pgsq03ExtractExpression(nil)) + projection.Hidden = true + return pgsq03SelectorPlan("Patient", projection) + }(), "hidden executor-side metadata"}, + {"value-expression projection", pgsq03SelectorPlan("Patient", pgsq03RawProjection("v", + &ir.PhysicalExpression{Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, Value: &ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}})), + "VALUE expression"}, + {"direct value with payload path", pgsq03SelectorPlan("Patient", ir.PhysicalProjection{ + Name: "status", Value: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}}), + "typed scope column"}, + {"direct value multi-segment", pgsq03SelectorPlan("Patient", ir.PhysicalProjection{ + Name: "nested", Value: ir.PhysicalValue{Variable: "root", Path: []string{"meta", "source"}}}), + "typed scope column"}, + {"direct value foreign variable", pgsq03SelectorPlan("Patient", ir.PhysicalProjection{ + Name: "other", Value: ir.PhysicalValue{Variable: "other", Path: []string{"logical_id"}}}), + "does not bind a readable row"}, + {"direct value bind-backed", pgsq03SelectorPlan("Patient", ir.PhysicalProjection{ + Name: "const", Value: ir.PhysicalValue{BindKey: "const"}}), + "not a bind value"}, + {"direct value with presence metadata", pgsq03SelectorPlan("Patient", ir.PhysicalProjection{ + Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + Presence: &ir.PhysicalProjectionPresence{Source: pgsq03DocumentSource(), Paths: [][]string{{"gender"}}}}), + "presence proofs pair with selector extractions only"}, + {"extract without payload", pgsq03SelectorPlan("Patient", + pgsq03RawProjection("g", &ir.PhysicalExpression{Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalScalarCardinality, NullBehavior: ir.PhysicalPreserveNull})), + "missing its payload"}, + {"extract array cardinality", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(_ *ir.PhysicalExtract, e *ir.PhysicalExpression) { e.Cardinality = ir.PhysicalArrayCardinality }))), + "SCALAR cardinality only"}, + {"extract object cardinality", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(_ *ir.PhysicalExtract, e *ir.PhysicalExpression) { e.Cardinality = ir.PhysicalObjectCardinality }))), + "SCALAR cardinality only"}, + {"extract invalid null behavior", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(_ *ir.PhysicalExtract, e *ir.PhysicalExpression) { e.NullBehavior = "NOPE" }))), + "null behavior"}, + {"extract conditional array mode", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { + x.ExecutionMode = ir.PhysicalSelectorConditionalArray + }))), + "CONDITIONAL_ARRAY"}, + {"extract unknown execution mode", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { x.ExecutionMode = "TELEPORT" }))), + "unknown selector execution mode"}, + {"extract distinct", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { x.Distinct = true }))), + "DISTINCT"}, + {"extract fallbacks", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { + x.Fallbacks = []ir.PhysicalSelectorFallback{{Source: pgsq03DocumentSource(), ResourceType: "Patient", Selector: pgsq03ActiveSelector()}} + }))), + "fallback selectors"}, + {"extract prepared reference", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { + x.Prepared = &ir.PhysicalPreparedReference{SetVariable: "prepared", Field: "value"} + }))), + "prepared-set"}, + {"extract unit normalization", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { + x.UnitNormalization = &ir.PhysicalUnitNormalization{} + }))), + "unit-normalized"}, + {"extract selector filter", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { + x.Selector.Filter = &spec.ContainsFilter{Field: "gender", Needle: "x"} + }))), + "CONTAINS filters"}, + {"extract empty selector", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { x.Selector = spec.Selector{} }))), + "selector is empty"}, + {"extract iterate step", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { + x.Selector.Steps = []spec.SelectorStep{{Field: "gender", Iterate: true}} + }))), + "repeated ([]) selector step"}, + {"extract index step", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { + x.Selector.Steps = []spec.SelectorStep{{Field: "gender", Index: &indexZero}} + }))), + "indexed selector step"}, + {"extract step without field", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { + x.Selector.Steps = []spec.SelectorStep{{Field: ""}} + }))), + "has no property"}, + {"extract bind source", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { + x.Source = ir.PhysicalValue{BindKey: "doc"} + }))), + "not a bind value"}, + {"extract nested document source", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { + x.Source = ir.PhysicalValue{Variable: "root", Path: []string{"payload", "contained"}} + }))), + `["payload"]`}, + {"extract empty source path", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { + x.Source = ir.PhysicalValue{Variable: "root"} + }))), + `["payload"]`}, + {"extract foreign source variable", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { + x.Source = ir.PhysicalValue{Variable: "other", Path: []string{"payload"}} + }))), + "does not bind a readable row"}, + {"extract missing resource type", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { x.ResourceType = "" }))), + "schema resource type"}, + {"extract contradicts scanned type", pgsq03SelectorPlan("Patient", pgsq03RawProjection("g", + pgsq03ExtractExpression(func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression) { x.ResourceType = "Observation" }))), + "but the row is scanned as"}, + {"extract presence metadata path mismatch", func() ir.PhysicalPlan { + projection := pgsq03RawProjection("g", pgsq03ExtractExpression(nil)) + projection.Presence = &ir.PhysicalProjectionPresence{Source: pgsq03DocumentSource(), Paths: [][]string{{"active"}}} + return pgsq03SelectorPlan("Patient", projection) + }(), "exactly the selector path"}, + {"extract presence metadata foreign source", func() ir.PhysicalPlan { + projection := pgsq03RawProjection("g", pgsq03ExtractExpression(nil)) + projection.Presence = &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "other", Path: []string{"payload"}}, Paths: [][]string{{"gender"}}} + return pgsq03SelectorPlan("Patient", projection) + }(), "selector's own document source"}, + {"sidecar without proof", pgsq03SelectorPlan("Patient", ir.PhysicalProjection{ + Name: "p", PresenceOutput: true}), "no presence proof"}, + {"sidecar with value", pgsq03SelectorPlan("Patient", ir.PhysicalProjection{ + Name: "p", PresenceOutput: true, Presence: &ir.PhysicalProjectionPresence{Source: pgsq03DocumentSource(), Paths: [][]string{{"gender"}}}, + Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}), "only its typed presence proof"}, + {"sidecar with bind source", pgsq03SelectorPlan("Patient", ir.PhysicalProjection{ + Name: "p", PresenceOutput: true, Presence: &ir.PhysicalProjectionPresence{Source: ir.PhysicalValue{BindKey: "doc"}, Paths: [][]string{{"gender"}}}}), + "not a bind value"}, + {"sidecar with nested source", pgsq03SelectorPlan("Patient", ir.PhysicalProjection{ + Name: "p", PresenceOutput: true, Presence: &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload", "meta"}}, Paths: [][]string{{"gender"}}}}), + "row document at"}, + {"sidecar with multiple candidate paths", pgsq03SelectorPlan("Patient", ir.PhysicalProjection{ + Name: "p", PresenceOutput: true, Presence: &ir.PhysicalProjectionPresence{ + Source: pgsq03DocumentSource(), Paths: [][]string{{"gender"}, {"active"}}}}), + "candidate paths"}, + {"sidecar proves no rendered selector", pgsq03SelectorPlan("Patient", + pgsq03RawProjection("g", pgsq03ExtractExpression(nil)), + pgsq03PresenceSidecar("p", "maritalStatus", "text")), "which no selector in this RETURN extracts"}, + {"sidecar proves unreadable variable", pgsq03SelectorPlan("Patient", + pgsq03RawProjection("g", pgsq03ExtractExpression(nil)), + ir.PhysicalProjection{Name: "p", PresenceOutput: true, Presence: &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "other", Path: []string{"payload"}}, Paths: [][]string{{"gender"}}}}), + "does not bind a readable row"}, + {"expression and direct value together", pgsq03SelectorPlan("Patient", ir.PhysicalProjection{ + Name: "g", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + Expression: pgsq03ExtractExpression(nil)}), "both a direct value and an expression"}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + rendered, err := RenderSelectorExtraction(testCase.plan, binding) + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("error = %v, want ErrUnsupportedPostgresPlan", err) + } + if rendered.SQL != "" || rendered.Args != nil || rendered.OutputColumns != nil { + t.Fatalf("rejected plan produced output: %#v", rendered) + } + if !strings.Contains(err.Error(), testCase.wantIn) { + t.Fatalf("error %q must contain %q", err.Error(), testCase.wantIn) + } + if strings.Contains(strings.ToUpper(err.Error()), "AQL") { + t.Fatalf("refusal mentions AQL: %v", err) + } + }) + } +} + +func TestSelectorExtractionGoldenUnrestrictedMode(t *testing.T) { + plan := pgsq03SelectorPlan("Patient", + pgsq03RawProjection("g", pgsq03ExtractExpression(nil)), + pgsq03PresenceSidecar("g_present", "gender")) + plan.BindVars["auth_resource_paths_unrestricted"] = true + plan.BindVars["auth_resource_paths"] = []string(nil) + rendered, err := RenderSelectorExtraction(plan, mustBinding(t)) + if err != nil { + t.Fatalf("RenderSelectorExtraction: %v", err) + } + wantSQL := `SELECT (r0."payload" #> $4::text[]) AS "c0", ` + + `COALESCE(jsonb_typeof(r0."payload") = 'object' AND jsonb_exists(r0."payload", $5), false) AS "c1"` + + ` FROM "pgir02_it_unit"."fhir_resource" AS r0` + + ` WHERE r0."project" = $1` + + ` AND r0."dataset_generation" = $2` + + ` AND r0."resource_type" = $3` + + ` ORDER BY r0."resource_seq" ASC` + if rendered.SQL != wantSQL { + t.Fatalf("unrestricted SQL =\n%s\nwant\n%s", rendered.SQL, wantSQL) + } + if strings.Contains(rendered.SQL, "auth_resource_path") { + t.Fatalf("explicit unrestricted mode must add no auth predicate: %s", rendered.SQL) + } +} + +type pgsq03Service struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + client *control.Client + schema string + scrub func(any) string + fail func(step string, err error) +} + +func newPGSQ03Service(t *testing.T) *pgsq03Service { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { t.Fatalf("%s: %s", step, scrub(err)) } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := "pgsq03_it_" + strconv.FormatInt(time.Now().UnixNano(), 10) + "_" + randomHex(t) + if !pgsq03ResourceNamePattern.MatchString(dbName) { + t.Fatalf("owned database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := "pgsq03_it_" + strconv.FormatInt(time.Now().UnixNano(), 10) + "_" + randomHex(t) + if !pgsq03ResourceNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + client, err := control.OpenPool(ctx, control.ClientConfig{ + DSN: dsn, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 2 * time.Second, + }) + if err != nil { + fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + t.Cleanup(client.Close) + if _, err := control.MigrateSchema(ctx, client, schema); err != nil { + fail("migrate owned schema", err) + } + return &pgsq03Service{ctx: ctx, admin: admin, iso: iso, client: client, schema: schema, scrub: scrub, fail: fail} +} + +func (s *pgsq03Service) insertResource(t *testing.T, resourceType, logicalID string, payload any) int64 { + t.Helper() + var raw json.RawMessage + if payload != nil { + encoded, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + raw = encoded + } + repo, err := control.NewResourceRepository(s.client, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + seq, err := repo.InsertResource(s.ctx, source.Resource{ + Project: pgsq03Project, + DatasetGeneration: pgsq03Gen, + ResourceType: resourceType, + LogicalID: logicalID, + AuthResourcePath: pgsq03Path, + Payload: raw, + }) + if err != nil { + s.fail("insert resource "+resourceType+"/"+logicalID, err) + } + return seq +} + +func (s *pgsq03Service) runRendered(t *testing.T, rendered command.RenderedPostgresPhysicalPlan) [][]string { + t.Helper() + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + rows, err := s.iso.Query(s.ctx, rendered.SQL, args...) + if err != nil { + s.fail("execute rendered statement "+rendered.SQL, err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + s.fail("read row", err) + } + row := make([]string, len(values)) + for index, value := range values { + row[index] = stringifyCell(value) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + s.fail("finish statement", err) + } + return out +} + +func (s *pgsq03Service) witness(t *testing.T, sql string, args ...any) [][]string { + t.Helper() + rows, err := s.iso.Query(s.ctx, sql, args...) + if err != nil { + s.fail("execute direct-SQL witness "+sql, err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + s.fail("read witness row", err) + } + row := make([]string, len(values)) + for index, value := range values { + row[index] = stringifyCell(value) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + s.fail("finish witness", err) + } + return out +} + +func (s *pgsq03Service) expectStored(t *testing.T, logicalID, payloadText string) { + t.Helper() + rows := s.witness(t, `SELECT coalesce(payload::text, '') FROM "`+s.schema+`".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND logical_id=$3`, + pgsq03Project, pgsq03Gen, logicalID) + if len(rows) != 1 || rows[0][0] != payloadText { + t.Fatalf("decoy %s must be stored with payload %s, got %v", logicalID, payloadText, rows) + } +} + +func pgsq03LivePlan(paths []string, unrestricted, scopeAllowed bool, projections ...ir.PhysicalProjection) ir.PhysicalPlan { + return ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": "Patient", + "project": pgsq03Project, + "dataset_generation": pgsq03Gen, + "auth_resource_paths": paths, + "auth_resource_paths_unrestricted": unrestricted, + "scope_allowed": scopeAllowed, + }, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}}, + }, + } +} + +func pgsq03GenderPlan(t *testing.T, behavior ir.PhysicalNullBehavior, paths []string, unrestricted, scopeAllowed bool) ir.PhysicalPlan { + return pgsq03LivePlan(paths, unrestricted, scopeAllowed, + ir.PhysicalProjection{Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + pgsq03ExtractProjection(t, "gender", "Patient", "gender", behavior, ir.PhysicalSelectorDirectScalar), + pgsq03PresenceSidecar("gender_present", "gender")) +} + +func TestPGSQ03ScalarNullBehaviorMatrixLive(t *testing.T) { + svc := newPGSQ03Service(t) + states := []struct { + logicalID string + payload any + payloadText string + }{ + {"m-absent", map[string]any{"active": true}, `{"active": true}`}, + {"m-null", json.RawMessage(`{"gender": null}`), `{"gender": null}`}, + {"m-emptyarray", json.RawMessage(`{"gender": []}`), `{"gender": []}`}, + {"m-arraynull", json.RawMessage(`{"gender": [null]}`), `{"gender": [null]}`}, + {"m-emptystring", json.RawMessage(`{"gender": ""}`), `{"gender": ""}`}, + {"m-false", json.RawMessage(`{"gender": false}`), `{"gender": false}`}, + {"m-zero", json.RawMessage(`{"gender": 0}`), `{"gender": 0}`}, + {"m-nopayload", nil, ""}, + {"m-male", json.RawMessage(`{"gender": "male"}`), `{"gender": "male"}`}, + } + for _, state := range states { + svc.insertResource(t, "Patient", state.logicalID, state.payload) + } + for _, state := range states { + svc.expectStored(t, state.logicalID, state.payloadText) + } + + order := make([]string, 0, len(states)) + for _, state := range states { + order = append(order, state.logicalID) + } + + cells := map[ir.PhysicalNullBehavior]map[string]struct { + value string + present string + }{ + ir.PhysicalPreserveNull: { + "m-absent": {"", "false"}, + "m-null": {"", "true"}, + "m-emptyarray": {"[]", "true"}, + "m-arraynull": {"[]", "true"}, + "m-emptystring": {"", "true"}, + "m-false": {"false", "true"}, + "m-zero": {"0", "true"}, + "m-nopayload": {"", "false"}, + "m-male": {"male", "true"}, + }, + ir.PhysicalOmitNulls: { + "m-absent": {"", "false"}, + "m-null": {"", "true"}, + "m-emptyarray": {"[]", "true"}, + "m-arraynull": {"[]", "true"}, + "m-emptystring": {"", "true"}, + "m-false": {"false", "true"}, + "m-zero": {"0", "true"}, + "m-nopayload": {"", "false"}, + "m-male": {"male", "true"}, + }, + ir.PhysicalEmptyOnNull: { + "m-absent": {"", "false"}, + "m-null": {"", "true"}, + "m-emptyarray": {"[]", "true"}, + "m-arraynull": {"[]", "true"}, + "m-emptystring": {"", "true"}, + "m-false": {"false", "true"}, + "m-zero": {"0", "true"}, + "m-nopayload": {"", "false"}, + "m-male": {"male", "true"}, + }, + } + + for _, behavior := range []ir.PhysicalNullBehavior{ + ir.PhysicalPreserveNull, ir.PhysicalOmitNulls, ir.PhysicalEmptyOnNull, + } { + plan := pgsq03GenderPlan(t, behavior, []string{pgsq03Path}, false, true) + rendered, err := RenderSelectorExtraction(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("%s render: %v", behavior, err) + } + wantArgs := []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: pgsq03Project}, + {Type: command.PostgreSQLParameterText, Value: pgsq03Gen}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["FHIR/Patient"]`)}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + {Type: command.PostgreSQLParameterText, Value: "{gender}"}, + {Type: command.PostgreSQLParameterText, Value: "gender"}, + } + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("%s args = %#v, want %#v", behavior, rendered.Args, wantArgs) + } + got := svc.runRendered(t, rendered) + if len(got) != len(order) { + t.Fatalf("%s rows = %v, want %d rows", behavior, got, len(order)) + } + for index, logicalID := range order { + cell := cells[behavior][logicalID] + if got[index][0] != logicalID || got[index][1] != cell.value || got[index][2] != cell.present { + t.Fatalf("%s row %d = %v, want [%s %s %s]", behavior, index, got[index], logicalID, cell.value, cell.present) + } + } + + var probe string + switch behavior { + case ir.PhysicalPreserveNull: + probe = `((payload #> '{gender}'::text[]) IS NULL) + <> NOT (COALESCE(jsonb_typeof(payload) = 'object' AND jsonb_exists(payload, 'gender'), false))` + case ir.PhysicalOmitNulls: + probe = `(NULLIF((payload #> '{gender}'::text[]), 'null'::jsonb) IS NULL) + <> (NOT (COALESCE(jsonb_typeof(payload) = 'object' AND jsonb_exists(payload, 'gender'), false)) + OR jsonb_typeof(payload #> '{gender}'::text[]) = 'null')` + case ir.PhysicalEmptyOnNull: + probe = `(COALESCE(NULLIF((payload #> '{gender}'::text[]), 'null'::jsonb), '""'::jsonb) = '""'::jsonb) + <> (NOT (COALESCE(jsonb_typeof(payload) = 'object' AND jsonb_exists(payload, 'gender'), false)) + OR jsonb_typeof(payload #> '{gender}'::text[]) = 'null' + OR (payload #> '{gender}'::text[]) = '""'::jsonb)` + } + disagree := svc.witness(t, `SELECT count(*) FROM "`+svc.schema+`".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' AND `+probe, + pgsq03Project, pgsq03Gen) + if len(disagree) != 1 || disagree[0][0] != "0" { + t.Fatalf("%s presence/value disagreement rows = %v, want none", behavior, disagree) + } + } +} + +func TestPGSQ03NestedPresenceDistinguishesParentsAndGuardsAreLoadBearing(t *testing.T) { + svc := newPGSQ03Service(t) + states := []struct { + logicalID string + payload any + payloadText string + value string + present string + }{ + {"n-present", json.RawMessage(`{"meta": {"source": "urn:oid:1"}}`), `{"meta": {"source": "urn:oid:1"}}`, "urn:oid:1", "true"}, + {"n-nullterminal", json.RawMessage(`{"meta": {"source": null}}`), `{"meta": {"source": null}}`, "", "true"}, + {"n-parentnull", json.RawMessage(`{"meta": null}`), `{"meta": null}`, "", "false"}, + {"n-parentscalar", json.RawMessage(`{"meta": 5}`), `{"meta": 5}`, "", "false"}, + {"n-parentarray", json.RawMessage(`{"meta": ["source"]}`), `{"meta": ["source"]}`, "", "false"}, + {"n-absent", map[string]any{"active": true}, `{"active": true}`, "", "false"}, + } + for _, state := range states { + svc.insertResource(t, "Patient", state.logicalID, state.payload) + } + for _, state := range states { + svc.expectStored(t, state.logicalID, state.payloadText) + } + + plan := pgsq03LivePlan([]string{pgsq03Path}, false, true, + ir.PhysicalProjection{Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + pgsq03ExtractProjection(t, "meta_source", "Patient", "meta.source", ir.PhysicalPreserveNull, ir.PhysicalSelectorDirectScalar), + pgsq03PresenceSidecar("meta_source_present", "meta", "source")) + rendered, err := RenderSelectorExtraction(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("nested render: %v", err) + } + wantArgs := []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: pgsq03Project}, + {Type: command.PostgreSQLParameterText, Value: pgsq03Gen}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["FHIR/Patient"]`)}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + {Type: command.PostgreSQLParameterText, Value: "{meta,source}"}, + {Type: command.PostgreSQLParameterText, Value: "{meta}"}, + {Type: command.PostgreSQLParameterText, Value: "source"}, + } + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("nested args = %#v, want %#v", rendered.Args, wantArgs) + } + got := svc.runRendered(t, rendered) + if len(got) != len(states) { + t.Fatalf("nested rows = %v, want %d rows", got, len(states)) + } + for index, state := range states { + if got[index][0] != state.logicalID || got[index][1] != state.value || got[index][2] != state.present { + t.Fatalf("nested row %d = %v, want [%s %s %s]", index, got[index], state.logicalID, state.value, state.present) + } + } + + gated := svc.witness(t, `SELECT count(*) FROM "`+svc.schema+`".fhir_resource + WHERE project=$1 AND dataset_generation=$2 AND logical_id='n-parentarray' + AND COALESCE(jsonb_typeof(payload #> '{meta}'::text[]) = 'object' AND jsonb_exists(payload #> '{meta}'::text[], 'source'), false)`, + pgsq03Project, pgsq03Gen) + if gated[0][0] != "0" { + t.Fatalf("gated presence must reject the array-contains decoy, got %v", gated) + } + ungated := svc.witness(t, `SELECT count(*) FROM "`+svc.schema+`".fhir_resource + WHERE project=$1 AND dataset_generation=$2 AND logical_id='n-parentarray' + AND COALESCE(jsonb_exists(payload #> '{meta}'::text[], 'source'), false)`, + pgsq03Project, pgsq03Gen) + if ungated[0][0] != "1" { + t.Fatalf("the typeof gate is not load-bearing: without it the decoy still reads false (%v)", ungated) + } + + svc.insertResource(t, "Patient", "g-nodoc", nil) + totally := svc.witness(t, `SELECT count(*) FROM "`+svc.schema+`".fhir_resource + WHERE project=$1 AND dataset_generation=$2 AND logical_id='g-nodoc' + AND (jsonb_typeof(payload #> '{meta}'::text[]) = 'object' AND jsonb_exists(payload #> '{meta}'::text[], 'source')) IS NULL`, + pgsq03Project, pgsq03Gen) + if totally[0][0] != "1" { + t.Fatalf("without COALESCE the presence bit must go NULL for an absent document, got %v", totally) + } + withCOALESCE := svc.witness(t, `SELECT count(*) FROM "`+svc.schema+`".fhir_resource + WHERE project=$1 AND dataset_generation=$2 AND logical_id='g-nodoc' + AND COALESCE(jsonb_typeof(payload #> '{meta}'::text[]) = 'object' AND jsonb_exists(payload #> '{meta}'::text[], 'source'), false) IS NULL`, + pgsq03Project, pgsq03Gen) + if withCOALESCE[0][0] != "0" { + t.Fatalf("with COALESCE the presence bit must never be NULL, got %v", withCOALESCE) + } + + collapsed := svc.witness(t, `SELECT payload #>> '{meta,source}'::text[] FROM "`+svc.schema+`".fhir_resource + WHERE project=$1 AND dataset_generation=$2 AND logical_id='n-nullterminal'`, pgsq03Project, pgsq03Gen) + if len(collapsed) != 1 || collapsed[0][0] != "" { + t.Fatalf("baseline: #>> is expected to collapse the JSON null (proving the choice of #> is load-bearing), got %v", collapsed) + } + kept := svc.witness(t, `SELECT (payload #> '{meta,source}'::text[])::text FROM "`+svc.schema+`".fhir_resource + WHERE project=$1 AND dataset_generation=$2 AND logical_id='n-nullterminal'`, pgsq03Project, pgsq03Gen) + if len(kept) != 1 || kept[0][0] != "null" { + t.Fatalf("the rendered jsonb form must keep the stored JSON null distinct, got %v", kept) + } +} + +func TestPGSQ03NullKeywordPayloadPathsReadStoredValuesLive(t *testing.T) { + svc := newPGSQ03Service(t) + for _, state := range []struct{ logicalID, payload string }{ + {"k-value", `{"null": "stored-value"}`}, + {"k-jsonnull", `{"null": null}`}, + {"k-absent", `{"gender": "female"}`}, + {"k-nested", `{"null": {"x": 7}}`}, + {"k-nestedarray", `{"null": ["x"]}`}, + {"k-upper", `{"NULL": "upper-value"}`}, + } { + svc.insertResource(t, "Patient", state.logicalID, json.RawMessage(state.payload)) + svc.expectStored(t, state.logicalID, state.payload) + } + + filterPlan := func(path []string, bindKey string, value any) ir.PhysicalPlan { + plan := pgsq03LivePlan([]string{pgsq03Path}, false, true, + ir.PhysicalProjection{Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}) + plan.BindVars[bindKey] = value + predicate := ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: path}, + Right: &ir.PhysicalValue{BindKey: bindKey}, + }}} + plan.Operations = append(plan.Operations[:1], append([]ir.PhysicalOperation{predicate}, plan.Operations[1:]...)...) + return plan + } + runWithNaivePath := func(t *testing.T, rendered command.RenderedPostgresPhysicalPlan, pathIndex int) [][]string { + t.Helper() + naive := append([]command.PostgreSQLParameter(nil), rendered.Args...) + text, ok := naive[pathIndex].Value.(string) + if !ok || text[0] != '{' { + t.Fatalf("argument %d is not an array literal: %#v", pathIndex, naive[pathIndex]) + } + naive[pathIndex].Value = strings.ReplaceAll(strings.ReplaceAll(text, `"null"`, "null"), `"NULL"`, "NULL") + return svc.runRendered(t, command.RenderedPostgresPhysicalPlan{SQL: rendered.SQL, Args: naive}) + } + + binding := bindingFor(t, svc.schema) + + rendered, err := RenderPhysicalPlan(filterPlan([]string{"null"}, "null_value", "stored-value"), binding) + if err != nil { + t.Fatalf("null-key filter render: %v", err) + } + if text, ok := rendered.Args[4].Value.(string); !ok || text != `{"null"}` { + t.Fatalf("null-key path arg = %#v, want {\"null\"}", rendered.Args[4]) + } + if got := svc.runRendered(t, rendered); !reflect.DeepEqual(got, [][]string{{"k-value"}}) { + t.Fatalf("null-key filter rows = %v, want [k-value]", got) + } + if got := runWithNaivePath(t, rendered, 4); len(got) != 0 { + t.Fatalf("the same statement with the naive unquoted {null} element must read nothing, got %v", got) + } + + nested, err := RenderPhysicalPlan(filterPlan([]string{"null", "x"}, "seven_value", float64(7)), binding) + if err != nil { + t.Fatalf("nested null-key filter render: %v", err) + } + if text, ok := nested.Args[4].Value.(string); !ok || text != `{"null",x}` { + t.Fatalf("nested null-key path arg = %#v, want {\"null\",x}", nested.Args[4]) + } + if got := svc.runRendered(t, nested); !reflect.DeepEqual(got, [][]string{{"k-nested"}}) { + t.Fatalf("nested null-key filter rows = %v, want [k-nested]", got) + } + if got := runWithNaivePath(t, nested, 4); len(got) != 0 { + t.Fatalf("the same nested statement with a naive unquoted prefix must read nothing, got %v", got) + } + + upper, err := RenderPhysicalPlan(filterPlan([]string{"NULL"}, "upper_value", "upper-value"), binding) + if err != nil { + t.Fatalf("upper null-key filter render: %v", err) + } + if text, ok := upper.Args[4].Value.(string); !ok || text != `{"NULL"}` { + t.Fatalf("upper null-key path arg = %#v, want {\"NULL\"}", upper.Args[4]) + } + if got := svc.runRendered(t, upper); !reflect.DeepEqual(got, [][]string{{"k-upper"}}) { + t.Fatalf("upper null-key filter rows = %v, want [k-upper]", got) + } + if got := runWithNaivePath(t, upper, 4); len(got) != 0 { + t.Fatalf("the same upper-case statement with the naive unquoted NULL element must read nothing, got %v", got) + } + storedUpper := svc.witness(t, `SELECT payload #> '{"NULL"}'::text[] FROM "`+svc.schema+`".fhir_resource + WHERE project=$1 AND dataset_generation=$2 AND logical_id='k-upper'`, pgsq03Project, pgsq03Gen) + if len(storedUpper) != 1 || storedUpper[0][0] != "upper-value" { + t.Fatalf("the stored upper-case null-named key must read its value, got %v", storedUpper) + } + + r := &planRenderer{storage: binding, parameters: map[string]int{}, rows: map[string]rowScope{}} + presence, err := r.propertyPresenceExpression("r0", []string{"null", "x"}) + if err != nil { + t.Fatalf("null-key presence render: %v", err) + } + if len(r.args) != 2 { + t.Fatalf("presence bound %d args, want quoted prefix + key", len(r.args)) + } + if text, ok := r.args[0].Value.(string); !ok || text != `{"null"}` { + t.Fatalf("presence prefix arg = %#v, want {\"null\"}", r.args[0]) + } + if text, ok := r.args[1].Value.(string); !ok || text != "x" { + t.Fatalf("presence key arg = %#v, want x", r.args[1]) + } + presenceSQL := `SELECT logical_id, ` + presence + ` FROM "` + svc.schema + `".fhir_resource AS r0` + + ` WHERE r0."project"=$3 AND r0."dataset_generation"=$4 AND r0."resource_type"='Patient' ORDER BY r0."resource_seq"` + got := svc.witness(t, presenceSQL, r.args[0].Value, r.args[1].Value, pgsq03Project, pgsq03Gen) + want := [][]string{ + {"k-value", "false"}, {"k-jsonnull", "false"}, {"k-absent", "false"}, + {"k-nested", "true"}, {"k-nestedarray", "false"}, {"k-upper", "false"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("null-key presence rows = %v, want %v", got, want) + } + naive := svc.witness(t, presenceSQL, "{null}", r.args[1].Value, pgsq03Project, pgsq03Gen) + for _, row := range naive { + if row[1] != "false" { + t.Fatalf("with the naive unquoted prefix no row may prove presence, got %v", naive) + } + } +} + +func TestPGSQ03DenyPathsReadZeroRowsWithStoredDecoys(t *testing.T) { + svc := newPGSQ03Service(t) + svc.insertResource(t, "Patient", "d-visible", json.RawMessage(`{"gender": "female"}`)) + svc.insertResource(t, "Patient", "d-null", json.RawMessage(`{"gender": null}`)) + svc.expectStored(t, "d-visible", `{"gender": "female"}`) + svc.expectStored(t, "d-null", `{"gender": null}`) + + binding := bindingFor(t, svc.schema) + extractor := func(paths []string, unrestricted, allowed bool) command.RenderedPostgresPhysicalPlan { + plan := pgsq03GenderPlan(t, ir.PhysicalPreserveNull, paths, unrestricted, allowed) + rendered, err := RenderSelectorExtraction(plan, binding) + if err != nil { + t.Fatalf("render: %v", err) + } + return rendered + } + + denied := extractor([]string{pgsq03Path}, false, false) + if !strings.Contains(denied.SQL, " AND FALSE") { + t.Fatalf("canonical deny must render the literal zero-row conjunct inherited from scope.go: %s", denied.SQL) + } + if got := svc.runRendered(t, denied); len(got) != 0 { + t.Fatalf("canonical scope_allowed=false deny read %v, want zero rows", got) + } + empty := extractor([]string{}, false, true) + if got := svc.runRendered(t, empty); len(got) != 0 { + t.Fatalf("restricted empty auth read %v, want zero rows", got) + } + visible := extractor(nil, true, true) + want := [][]string{{"d-visible", "female", "true"}, {"d-null", "", "true"}} + if got := svc.runRendered(t, visible); !reflect.DeepEqual(got, want) { + t.Fatalf("unrestricted control rows = %v, want %v", got, want) + } +} + +func TestPGSQ03ExtractionIsConnectionIndependent(t *testing.T) { + svc := newPGSQ03Service(t) + svc.insertResource(t, "Patient", "c-female", json.RawMessage(`{"gender": "female"}`)) + svc.insertResource(t, "Patient", "c-missing", map[string]any{"active": true}) + + rendered, err := RenderSelectorExtraction(pgsq03GenderPlan(t, ir.PhysicalPreserveNull, []string{pgsq03Path}, false, true), bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("render: %v", err) + } + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + + connA, err := svc.iso.Acquire(svc.ctx) + if err != nil { + svc.fail("acquire connection A", err) + } + defer connA.Release() + connB, err := svc.iso.Acquire(svc.ctx) + if err != nil { + svc.fail("acquire connection B", err) + } + defer connB.Release() + + var pidA, pidB int + if err := connA.QueryRow(svc.ctx, `SELECT pg_backend_pid()`).Scan(&pidA); err != nil { + svc.fail("backend pid A", err) + } + if err := connB.QueryRow(svc.ctx, `SELECT pg_backend_pid()`).Scan(&pidB); err != nil { + svc.fail("backend pid B", err) + } + if pidA == pidB || pidA == 0 || pidB == 0 { + t.Fatalf("the two executions must run on distinct real backends, got %d and %d", pidA, pidB) + } + + readRows := func(conn *pgxpool.Conn) [][]string { + rows, err := conn.Query(svc.ctx, rendered.SQL, args...) + if err != nil { + svc.fail("execute on backend "+strconv.Itoa(pidA)+"/"+strconv.Itoa(pidB), err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + svc.fail("read row", err) + } + row := make([]string, len(values)) + for index, value := range values { + row[index] = stringifyCell(value) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + svc.fail("finish statement", err) + } + return out + } + want := [][]string{{"c-female", "female", "true"}, {"c-missing", "", "false"}} + rowsA := readRows(connA) + rowsB := readRows(connB) + if !reflect.DeepEqual(rowsA, want) || !reflect.DeepEqual(rowsB, want) { + t.Fatalf("connection-dependent rows: A=%v B=%v want %v", rowsA, rowsB, want) + } +} + +func TestPGSQ03LoweredShapeEndToEnd(t *testing.T) { + svc := newPGSQ03Service(t) + svc.insertResource(t, "Patient", "l-full", json.RawMessage(`{"gender": "female", "meta": {"source": "urn:x"}}`)) + svc.insertResource(t, "Patient", "l-empty", map[string]any{"active": true}) + + documentSource := pgsq03DocumentSource() + genderSelector, err := spec.ParseSelector("gender") + if err != nil { + t.Fatalf("ParseSelector: %v", err) + } + metaSelector, err := spec.ParseSelector("meta.source") + if err != nil { + t.Fatalf("ParseSelector: %v", err) + } + plan := pgsq03LivePlan([]string{pgsq03Path}, false, true, + ir.PhysicalProjection{ + Name: "gender", + Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Extract: &ir.PhysicalExtract{ + Source: documentSource, + ResourceType: "Patient", + Selector: genderSelector, + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + }, + Presence: &ir.PhysicalProjectionPresence{Source: documentSource, Paths: [][]string{{"gender"}}}, + }, + ir.PhysicalProjection{Name: "gender_present", Presence: &ir.PhysicalProjectionPresence{Source: documentSource, Paths: [][]string{{"gender"}}}, PresenceOutput: true}, + ir.PhysicalProjection{ + Name: "meta_source", + Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Extract: &ir.PhysicalExtract{ + Source: documentSource, + ResourceType: "Patient", + Selector: metaSelector, + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + }, + Presence: &ir.PhysicalProjectionPresence{Source: documentSource, Paths: [][]string{{"meta", "source"}}}, + }, + ir.PhysicalProjection{Name: "meta_source_present", Presence: &ir.PhysicalProjectionPresence{Source: documentSource, Paths: [][]string{{"meta", "source"}}}, PresenceOutput: true}) + + rendered, err := RenderSelectorExtraction(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("lowered-shape render: %v", err) + } + if !reflect.DeepEqual(rendered.OutputColumns, []string{"gender", "gender_present", "meta_source", "meta_source_present"}) { + t.Fatalf("output columns = %v", rendered.OutputColumns) + } + selectPrefix := rendered.SQL[:strings.Index(rendered.SQL, " FROM ")] + for _, forbidden := range []string{"@", "gender", "meta", "urn", "Patient", pgsq03Project, pgsq03Gen, pgsq03Path} { + if strings.Contains(selectPrefix, forbidden) { + t.Fatalf("lowered-shape SQL leaked %q: %s", forbidden, rendered.SQL) + } + } + got := svc.runRendered(t, rendered) + wantRows := [][]string{ + {"female", "true", "urn:x", "true"}, + {"", "false", "", "false"}, + } + if len(got) != len(wantRows) { + t.Fatalf("lowered-shape rows = %v, want %v", got, wantRows) + } + for index, row := range wantRows { + if got[index][0] != row[0] || got[index][1] != row[1] || got[index][2] != row[2] || got[index][3] != row[3] { + t.Fatalf("lowered-shape row %d = %v, want %v", index, got[index], row) + } + } + witness := svc.witness(t, `SELECT payload #> '{gender}'::text[], + COALESCE(jsonb_typeof(payload) = 'object' AND jsonb_exists(payload, 'gender'), false), + payload #> '{meta,source}'::text[], + COALESCE(jsonb_typeof(payload #> '{meta}'::text[]) = 'object' AND jsonb_exists(payload #> '{meta}'::text[], 'source'), false) + FROM "`+svc.schema+`".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' ORDER BY resource_seq`, + pgsq03Project, pgsq03Gen) + if len(witness) != len(wantRows) { + t.Fatalf("witness rows = %v", witness) + } + for index, row := range wantRows { + if witness[index][0] != row[0] || witness[index][1] != row[1] || witness[index][2] != row[2] || witness[index][3] != row[3] { + t.Fatalf("witness row %d = %v, want %v", index, witness[index], row) + } + } +} + +func TestZZNoLeftoverPgSQ03Resources(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer admin.Close() + var databases int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'pgsq03_it_%'").Scan(&databases); err != nil { + t.Fatalf("count databases: %v", err) + } + if databases != 0 { + t.Fatalf("%d pgsq03_it_* databases still exist", databases) + } + var schemas int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgsq03_it_%'").Scan(&schemas); err != nil { + t.Fatalf("count schemas: %v", err) + } + if schemas != 0 { + t.Fatalf("%d pgsq03_it_* schemas still exist in the service database", schemas) + } +} diff --git a/internal/dataframe/compiler/render/postgres/sets.go b/internal/dataframe/compiler/render/postgres/sets.go new file mode 100644 index 000000000..409595472 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/sets.go @@ -0,0 +1,351 @@ +package postgres + +import ( + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +type correlatedSetBody struct { + from string + where string + targetAlias string + targetType string + targetVar string +} + +func (r *planRenderer) renderCorrelatedChildSteps( + plan ir.PhysicalPlan, + subplan ir.PhysicalSubplan, + allowOptionalCapture bool, + multipleTraversals bool, + allowEndpointStrategy bool, +) (correlatedSetBody, error) { + if len(subplan.Captures) != 1 { + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan must capture exactly one correlation anchor, got %d captures", + ErrUnsupportedPostgresPlan, len(subplan.Captures)) + } + captureVariable := subplan.Captures[0] + capture, ok := r.rows[captureVariable] + if !ok || capture.derivedValue != "" { + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan capture %q does not bind a readable row (edge variables, set variables, computed values, and unreached variables cannot anchor a correlation)", + ErrUnsupportedPostgresPlan, captureVariable) + } + if capture.optional && !allowOptionalCapture { + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan capture %q is a row materialized by an OPTIONAL traversal LEFT JOIN LATERAL; a NULL-extended anchor column would let the semi-join compare absent rows (the traversal.go:350-354 required-match rule)", + ErrUnsupportedPostgresPlan, captureVariable) + } + + rows := map[string]rowScope{captureVariable: capture} + fromParts := make([]string, 0, 4) + whereParts := make([]string, 0, 4) + sawTraversal := false + nextSource := capture + nextSourceVariable := captureVariable + for index := 0; index < len(subplan.Operations); { + operation := subplan.Operations[index] + switch operation.Kind { + case ir.PhysicalTraversalOp: + if operation.Traversal == nil { + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan operation %d: TRAVERSAL payload is missing", + ErrUnsupportedPostgresPlan, index) + } + if sawTraversal && !multipleTraversals { + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan operation %d: the registered child set carries exactly one depth-one traversal hop (lower/child_sets.go buildOptionalChildPhysicalSet); chained routes belong to the related-existence surface", + ErrUnsupportedPostgresPlan, index) + } + traversal := *operation.Traversal + if traversal.Strategy == ir.PhysicalTraversalEndpointLookup { + if !allowEndpointStrategy { + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan operation %d: endpoint-lookup strategy has no proven PostgreSQL contract for this consumer; the registered child-set surface accepts it only for sets reduced by a registered SUM/COUNT/MIN/MAX aggregate expression", + ErrUnsupportedPostgresPlan, index) + } + if strings.TrimSpace(traversal.EndpointField) == "" || strings.TrimSpace(traversal.EndpointJoinField) == "" { + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan operation %d: endpoint-lookup traversal payload is incomplete (the planner always names EndpointField and EndpointJoinField, lower/traversal.go:143-147)", + ErrUnsupportedPostgresPlan, index) + } + expectedField, expectedJoinField := "_from", "_to" + switch traversal.Direction { + case ir.PhysicalInbound: + expectedField, expectedJoinField = "_to", "_from" + case ir.PhysicalOutbound: + default: + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan operation %d: endpoint-lookup direction %q carries no direction-derived join contract to check the named fields against", + ErrUnsupportedPostgresPlan, index, string(traversal.Direction)) + } + if traversal.EndpointField != expectedField || traversal.EndpointJoinField != expectedJoinField { + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan operation %d: endpoint-lookup names (%s, %s) but direction %q derives (%s, %s) (lower/storage_route.go:30-39 always agrees; a divergence must fail closed)", + ErrUnsupportedPostgresPlan, index, traversal.EndpointField, traversal.EndpointJoinField, + string(traversal.Direction), expectedField, expectedJoinField) + } + traversal.Strategy = ir.PhysicalTraversalNative + } + if traversal.SourceVariable != nextSourceVariable { + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan operation %d: traversal must anchor on %q (the capture for the first hop, the previous hop's reached row thereafter), got source %q", + ErrUnsupportedPostgresPlan, index, nextSourceVariable, traversal.SourceVariable) + } + verified, err := r.validateTraversalContract(plan, traversal, nextSource) + if err != nil { + return correlatedSetBody{}, fmt.Errorf("correlated subplan operation %d: %w", index, err) + } + seq := r.nextTraversalSequence() + edgeAlias := "e" + strconv.Itoa(seq) + targetAlias := "x" + strconv.Itoa(seq) + edgeWhere, targetOn := r.edgeOccurrencePredicates(&verified, edgeAlias, targetAlias, nextSource.alias) + if !sawTraversal { + fromParts = append(fromParts, "FROM "+r.storage.Edge.Identifier().SQLText()+" AS "+edgeAlias) + whereParts = append(whereParts, edgeWhere) + } else { + fromParts = append(fromParts, + "INNER JOIN "+r.storage.Edge.Identifier().SQLText()+" AS "+edgeAlias+" ON "+edgeWhere) + } + fromParts = append(fromParts, + "INNER JOIN "+r.storage.Resource.Identifier().SQLText()+" AS "+targetAlias+" ON "+targetOn) + rows[traversal.TargetVariable] = rowScope{alias: targetAlias, resourceType: verified.targetType} + nextSource = rows[traversal.TargetVariable] + nextSourceVariable = traversal.TargetVariable + sawTraversal = true + index++ + if err := matchCanonicalScopeBlock(subplan.Operations, &index, + []string{traversal.EdgeVariable, traversal.TargetVariable}); err != nil { + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan operation %d: canonical subplan scope block: %s", + ErrUnsupportedPostgresPlan, index, err.Error()) + } + case ir.PhysicalFilterOp: + if operation.Filter == nil { + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan operation %d: FILTER payload is missing", + ErrUnsupportedPostgresPlan, index) + } + if !sawTraversal { + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan operation %d: a FILTER may only follow a traversal step (the reduction is over reached child rows, and subplan scope is structural)", + ErrUnsupportedPostgresPlan, index) + } + if operation.Filter.Expression == nil { + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan operation %d: subplan filters must use the typed Expression form the lowering emits", + ErrUnsupportedPostgresPlan, index) + } + clause, err := r.renderPredicateFragmentScoped(plan, *operation.Filter.Expression, rows, true) + if err != nil { + return correlatedSetBody{}, fmt.Errorf("correlated subplan operation %d: %w", index, err) + } + whereParts = append(whereParts, clause.sql) + index++ + default: + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan operation %d has kind %q; only TRAVERSAL hops (each closing with its canonical scope block) and typed FILTER operations are registered — shared subsets, nested sets and unnests, collection scans, keyed lookups, document lookups, and non-canonical scope operations are capability gaps", + ErrUnsupportedPostgresPlan, index, string(operation.Kind)) + } + } + if !sawTraversal { + return correlatedSetBody{}, fmt.Errorf("%w: correlated subplan must contain at least one traversal hop", ErrUnsupportedPostgresPlan) + } + return correlatedSetBody{ + from: strings.Join(fromParts, " "), + where: strings.Join(whereParts, " AND "), + targetAlias: nextSource.alias, + targetType: nextSource.resourceType, + targetVar: nextSourceVariable, + }, nil +} + +func (r *planRenderer) renderSet(plan ir.PhysicalPlan, set ir.PhysicalSet) error { + if set.SourceSetVariable != "" { + return fmt.Errorf("%w: shared subsets over a materialized set (source set %q) have no registered PostgreSQL contract (the AQL counterpart is render/aql/navigation_render.go renderSharedSubset)", + ErrUnsupportedPostgresPlan, set.SourceSetVariable) + } + if set.Prepared != nil { + return fmt.Errorf("%w: prepared sets are not implemented for PostgreSQL", ErrUnsupportedPostgresPlan) + } + switch set.Kind { + case "", ir.PhysicalNodeSetKind: + case ir.PhysicalNodePathSetKind: + return fmt.Errorf("%w: NODE_PATH sets have no registered PostgreSQL contract (the graph-return surface is a separate capability)", ErrUnsupportedPostgresPlan) + default: + return fmt.Errorf("%w: unknown set kind %q", ErrUnsupportedPostgresPlan, string(set.Kind)) + } + if set.Reduction == nil && !sliceConsumesSetVariable(plan, set.Variable) && !aggregateConsumesSetVariable(plan, set.Variable) { + return fmt.Errorf("%w: set %q has no reduction contract; the registered surface renders sets only through their FIRST/ALL/DISTINCT selector reductions, SLICE consumers, and SUM/COUNT/MIN/MAX aggregate reducers over the registered projection slots, because no other consumer of the item array is registered", + ErrUnsupportedPostgresPlan, set.Variable) + } + if set.Reduction != nil && set.Projection == nil { + return fmt.Errorf("%w: set %q reduction requires the traversal-time selector projection lower/child_sets.go computes in the child subquery", + ErrUnsupportedPostgresPlan, set.Variable) + } + if !set.Unique || !set.SortByKey { + return fmt.Errorf("%w: set %q must carry Unique and SortByKey (the child_sets.go:88 contract the (logical_id, element-ordinality) render order implements)", + ErrUnsupportedPostgresPlan, set.Variable) + } + if set.Subplan.Sort != nil || set.Subplan.Unique || set.Subplan.DistinctBy != nil { + return fmt.Errorf("%w: set %q subplan cannot carry projection-only modifiers (the set-level SortByKey/Unique contract owns the ordering)", + ErrUnsupportedPostgresPlan, set.Variable) + } + + corr, err := r.renderCorrelatedChildSteps(plan, set.Subplan, true, false, aggregateConsumesSetVariable(plan, set.Variable)) + if err != nil { + return fmt.Errorf("set %q: %w", set.Variable, err) + } + + ret := set.Subplan.Return + if ret.Kind != ir.PhysicalValueExpression || ret.Cardinality != ir.PhysicalObjectCardinality || + ret.NullBehavior != ir.PhysicalPreserveNull || ret.Value == nil || + ret.Value.BindKey != "" || ret.Value.Variable != corr.targetVar || len(ret.Value.Path) != 0 { + return fmt.Errorf("%w: set %q subplan return must be the canonical whole-document VALUE expression over the reached target row (lower/child_sets.go buildOptionalChildPhysicalSet)", + ErrUnsupportedPostgresPlan, set.Variable) + } + + if r.setBodies == nil { + r.setBodies = make(map[string]correlatedSetBody) + } + r.setBodies[set.Variable] = corr + if set.Projection != nil && len(set.Projection.Fields) > 0 { + if r.setProjections == nil { + r.setProjections = make(map[string][]ir.PhysicalSetProjectionField) + } + r.setProjections[set.Variable] = append([]ir.PhysicalSetProjectionField(nil), set.Projection.Fields...) + } + if set.Reduction == nil { + return nil + } + + projectionFields := make(map[string]ir.PhysicalSetProjectionField, len(set.Projection.Fields)) + for _, field := range set.Projection.Fields { + if _, duplicate := projectionFields[field.Name]; duplicate { + return fmt.Errorf("%w: set %q projection field %q is duplicated", ErrUnsupportedPostgresPlan, set.Variable, field.Name) + } + projectionFields[field.Name] = field + } + + objectArgs := make([]string, 0, len(set.Reduction.Fields)) + for _, field := range set.Reduction.Fields { + projectionField, ok := projectionFields[field.SourceField] + if !ok { + return fmt.Errorf("%w: set %q reduction field %q reads projection slot %q, which the set projection does not declare", + ErrUnsupportedPostgresPlan, set.Variable, field.Name, field.SourceField) + } + keyIndex := r.textParameter(field.Name, "text:"+field.Name) + value, err := r.setReductionSublink(set.Variable, field.Mode, projectionField, corr) + if err != nil { + return fmt.Errorf("set %q reduction field %q: %w", set.Variable, field.Name, err) + } + objectArgs = append(objectArgs, "$"+strconv.Itoa(keyIndex)+"::text, "+value) + } + object := "jsonb_build_object(" + strings.Join(objectArgs, ", ") + ")" + return r.registerDerivedValue(set.Reduction.Variable, object, true) +} + +func (r *planRenderer) setProjectionSlotExpression( + field ir.PhysicalSetProjectionField, + corr correlatedSetBody, + itemAlias string, +) (string, error) { + rows := map[string]rowScope{corr.targetVar: {alias: itemAlias, resourceType: corr.targetType}} + expression := ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: corr.targetVar, Path: []string{"payload"}}, + ResourceType: field.ResourceType, + Selector: field.Selector, + ExecutionMode: field.ExecutionMode, + }, + } + slot, err := r.selectorValuesExpression(expression, rows) + if err != nil { + return "", fmt.Errorf("projected slot %q: %w", field.Name, err) + } + return slot, nil +} + +func (r *planRenderer) setReductionSublink( + setVariable string, + mode ir.PhysicalSetReductionMode, + field ir.PhysicalSetProjectionField, + corr correlatedSetBody, +) (string, error) { + seq := r.nextTraversalSequence() + itemAlias := "i" + strconv.Itoa(seq) + elementAlias := "s" + strconv.Itoa(seq) + distinctAlias := "u" + strconv.Itoa(seq) + logicalIDColumn := r.storage.Resource.MustColumn("logical_id").SQLText() + resourceTypeColumn := r.storage.Resource.MustColumn("resource_type").SQLText() + payloadColumn := r.storage.Resource.MustColumn("payload").SQLText() + + items := "(SELECT DISTINCT " + corr.targetAlias + "." + logicalIDColumn + " AS " + logicalIDColumn + + ", " + corr.targetAlias + "." + resourceTypeColumn + " AS " + resourceTypeColumn + + ", " + corr.targetAlias + "." + payloadColumn + " AS " + payloadColumn + + " " + corr.from + " WHERE " + corr.where + ")" + slot, err := r.setProjectionSlotExpression(field, corr, itemAlias) + if err != nil { + return "", err + } + lateral := " CROSS JOIN LATERAL jsonb_array_elements(" + slot + ") WITH ORDINALITY AS " + + elementAlias + "(v, ord)" + itemOrder := itemAlias + "." + logicalIDColumn + elementOrder := elementAlias + ".ord" + + switch mode { + case ir.PhysicalSetReductionFirst: + return "(SELECT " + elementAlias + ".v FROM " + items + " AS " + itemAlias + lateral + + " ORDER BY " + itemOrder + ", " + elementOrder + " LIMIT 1)", nil + case ir.PhysicalSetReductionAll: + return "(SELECT COALESCE(jsonb_agg(" + elementAlias + ".v ORDER BY " + itemOrder + ", " + elementOrder + + "), '[]'::jsonb) FROM " + items + " AS " + itemAlias + lateral + ")", nil + case ir.PhysicalSetReductionDistinct: + return "(SELECT COALESCE(jsonb_agg(" + distinctAlias + ".v ORDER BY " + + distinctAlias + "." + logicalIDColumn + ", " + distinctAlias + ".ord), '[]'::jsonb) FROM " + + "(SELECT DISTINCT ON (" + itemOrder + ", " + elementAlias + ".v) " + + itemOrder + " AS " + logicalIDColumn + ", " + elementAlias + ".v AS v, " + elementAlias + ".ord AS ord FROM " + + items + " AS " + itemAlias + lateral + + " ORDER BY " + itemOrder + ", " + elementAlias + ".v, " + elementOrder + ") " + distinctAlias + ")", nil + default: + return "", fmt.Errorf("%w: set %q reduction field %q has unsupported mode %q", + ErrUnsupportedPostgresPlan, setVariable, field.Name, string(mode)) + } +} + +func isRelatedExistsSubplan(subplan *ir.PhysicalSubplan) bool { + if subplan == nil || len(subplan.Captures) != 1 { + return false + } + if subplan.Sort != nil || subplan.Unique || subplan.DistinctBy != nil { + return false + } + lastTarget := "" + for _, operation := range subplan.Operations { + if operation.Kind == ir.PhysicalTraversalOp && operation.Traversal != nil { + lastTarget = operation.Traversal.TargetVariable + } + } + if lastTarget == "" { + return false + } + ret := subplan.Return + return ret.Kind == ir.PhysicalValueExpression && + ret.Cardinality == ir.PhysicalScalarCardinality && + ret.NullBehavior == ir.PhysicalPreserveNull && + ret.Value != nil && ret.Value.BindKey == "" && + ret.Value.Variable == lastTarget && + len(ret.Value.Path) == 1 && ret.Value.Path[0] == "_id" +} + +func (r *planRenderer) renderRelatedExists(plan ir.PhysicalPlan, subplan ir.PhysicalSubplan) (string, error) { + if subplan.Sort != nil || subplan.Unique || subplan.DistinctBy != nil { + return "", fmt.Errorf("%w: related-existence subplan cannot carry projection-only sort or unique modifiers (render/aql/predicates.go:315-317 refuses them for bounded EXISTS too; count-style eligibility is a separate capability)", + ErrUnsupportedPostgresPlan) + } + corr, err := r.renderCorrelatedChildSteps(plan, subplan, false, true, false) + if err != nil { + return "", fmt.Errorf("related-existence subplan: %w", err) + } + ret := subplan.Return + if ret.Kind != ir.PhysicalValueExpression || ret.Cardinality != ir.PhysicalScalarCardinality || + ret.NullBehavior != ir.PhysicalPreserveNull || ret.Value == nil || + ret.Value.BindKey != "" || ret.Value.Variable != corr.targetVar || + len(ret.Value.Path) != 1 || ret.Value.Path[0] != "_id" { + return "", fmt.Errorf("%w: related-existence subplan return must be the canonical scalar {_id} VALUE expression over the last hop's target row (lower/recipe_construction_related_eligibility.go physicalValueExpression form)", + ErrUnsupportedPostgresPlan) + } + return "EXISTS (SELECT 1 " + corr.from + " WHERE " + corr.where + ")", nil +} diff --git a/internal/dataframe/compiler/render/postgres/sets_live_test.go b/internal/dataframe/compiler/render/postgres/sets_live_test.go new file mode 100644 index 000000000..a377fb57c --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/sets_live_test.go @@ -0,0 +1,331 @@ +package postgres + +import ( + "context" + "fmt" + "os" + "strings" + "sync" + "testing" + "time" + + pgstore "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/jackc/pgx/v5/pgxpool" +) + +func newSets08Service(t *testing.T) *pgir02Service { + t.Helper() + return newOwnedService(t, 0, "sets08_it") +} + +var ( + sets08FixturesMu sync.Mutex + sets08TrackedFixtures = map[string]bool{} +) + +func sets08TrackFixtures(t *testing.T, svc *pgir02Service) { + t.Helper() + var database string + if err := svc.iso.QueryRow(svc.ctx, "SELECT current_database()").Scan(&database); err != nil { + svc.fail("track owned database", err) + } + sets08FixturesMu.Lock() + defer sets08FixturesMu.Unlock() + sets08TrackedFixtures[database] = true + sets08TrackedFixtures[svc.schema] = true +} + +func sets08SnapshotOwnedResources(ctx context.Context, raw string) ([]string, error) { + pool, err := pgxpool.New(ctx, raw) + if err != nil { + return nil, fmt.Errorf("connect: %w", err) + } + defer pool.Close() + rows, err := pool.Query(ctx, + `SELECT datname FROM pg_database WHERE datname LIKE 'sets08_it_%'`+ + ` UNION ALL SELECT nspname FROM pg_namespace WHERE nspname LIKE 'sets08_it_%'`) + if err != nil { + return nil, fmt.Errorf("snapshot: %w", err) + } + defer rows.Close() + var names []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("scan: %w", err) + } + names = append(names, name) + } + return names, rows.Err() +} + +func sets08RunDeltaAudit(m *testing.M) int { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return trvRunDeltaAudit(m) + } + beforeList, err := sets08SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-08 delta audit: snapshot before run: %v\n", err) + return trvRunDeltaAudit(m) + } + before := make(map[string]bool, len(beforeList)) + for _, name := range beforeList { + before[name] = true + } + code := trvRunDeltaAudit(m) + after, err := sets08SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-08 delta audit: snapshot after run: %v\n", err) + code = 1 + } + sets08FixturesMu.Lock() + defer sets08FixturesMu.Unlock() + for _, name := range after { + if before[name] { + continue + } + if sets08TrackedFixtures[name] { + fmt.Fprintf(os.Stderr, "pg-sql-08 delta audit: %q was created and tracked by this run and still survives\n", name) + code = 1 + continue + } + fmt.Fprintf(os.Stderr, "pg-sql-08 delta audit: note — %q was born during this run but is not owned by it (sibling lane?)\n", name) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + survivors, err := sets08SnapshotOwnedResources(ctx, raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-08 delta audit: survivor scan: %v\n", err) + code = 1 + } else if len(survivors) > 0 { + fmt.Fprintf(os.Stderr, "pg-sql-08 delta audit: %d sets08_it_* resources survive: %v\n", len(survivors), survivors) + code = 1 + } + return code +} + +const ( + setsProj = "proj-a" + setsGen = "gen-1" + setsEdgeLabel = "has-observation" +) + +func setsInsertPatient(t *testing.T, svc *pgir02Service, logicalID string) { + t.Helper() + svc.insertResource(t, svc.schema, setsProj, setsGen, "Patient", logicalID, "FHIR/Patient", nil) +} + +func setsInsertObservation(t *testing.T, svc *pgir02Service, logicalID, authPath string, payload map[string]any) { + t.Helper() + svc.insertResource(t, svc.schema, setsProj, setsGen, "Observation", logicalID, authPath, payload) +} + +func setsInsertRouteEdge(t *testing.T, svc *pgir02Service, from, to, sourcePath string) { + t.Helper() + svc.insertEdge(t, pgstore.EdgeOccurrenceRow{ + Project: setsProj, DatasetGeneration: setsGen, AuthResourcePath: "FHIR/Patient", + Label: setsEdgeLabel, SourceResourceType: "Patient", SourceLogicalID: from, + TargetResourceType: "Observation", TargetLogicalID: to, SourcePath: sourcePath, SourceOrdinal: 0, + }) +} + +func jsonMin(value string) string { + return strings.ReplaceAll(value, " ", "") +} + +func TestSetLiveReductionsMatchLiteralRowsAndWitness(t *testing.T) { + svc := newSets08Service(t) + sets08TrackFixtures(t, svc) + + setsInsertPatient(t, svc, "p-1") + setsInsertPatient(t, svc, "p-2") + setsInsertPatient(t, svc, "p-3") + setsInsertPatient(t, svc, "p-4") + setsInsertObservation(t, svc, "obs-final", "FHIR/Observation", map[string]any{"status": "final"}) + setsInsertObservation(t, svc, "obs-prelim", "FHIR/Observation", map[string]any{"status": "prelim"}) + setsInsertObservation(t, svc, "obs-null", "FHIR/Observation", map[string]any{"status": nil}) + setsInsertObservation(t, svc, "obs-absent", "FHIR/Observation", map[string]any{"note": "x"}) + setsInsertObservation(t, svc, "obs-c1", "FHIR/Observation", map[string]any{"status": "dup"}) + setsInsertObservation(t, svc, "obs-c2", "FHIR/Observation", map[string]any{"status": "dup"}) + setsInsertObservation(t, svc, "obs-blocked", "FHIR/Blocked", map[string]any{"status": "sneaky"}) + + setsInsertRouteEdge(t, svc, "p-1", "obs-final", "observation[0]") + setsInsertRouteEdge(t, svc, "p-1", "obs-final", "observation[1]") + setsInsertRouteEdge(t, svc, "p-1", "obs-prelim", "observation[2]") + setsInsertRouteEdge(t, svc, "p-1", "obs-blocked", "observation[9]") + setsInsertRouteEdge(t, svc, "p-2", "obs-null", "observation[0]") + setsInsertRouteEdge(t, svc, "p-2", "obs-absent", "observation[1]") + setsInsertRouteEdge(t, svc, "p-4", "obs-c1", "observation[0]") + setsInsertRouteEdge(t, svc, "p-4", "obs-c2", "observation[1]") + + set := setsChildSet(ir.PhysicalSetReductionFirst, ir.PhysicalSetReductionAll, ir.PhysicalSetReductionDistinct) + rendered, err := RenderPhysicalPlan(setsPlanWithSet(set, + "__loom_reduced_0", "__loom_reduced_1", "__loom_reduced_2"), bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + for _, leaked := range []string{"p-1", "obs-final", "final", setsProj, setsGen, setsEdgeLabel, "FHIR/Patient"} { + if strings.Contains(rendered.SQL, leaked) { + t.Fatalf("value %q leaked into rendered SQL: %s", leaked, rendered.SQL) + } + } + got := svc.runRendered(t, rendered) + + if len(got) != 4 { + t.Fatalf("parent grain broken: %v", got) + } + want := [][]string{ + {"p-1", `final`, `[final prelim]`, `[final prelim]`}, + {"p-2", "", `[]`, `[]`}, + {"p-3", "", `[]`, `[]`}, + {"p-4", `dup`, `[dup dup]`, `[dup dup]`}, + } + if len(got) != len(want) { + t.Fatalf("rows = %v, want %v", got, want) + } + for row := range want { + if got[row][0] != want[row][0] || got[row][1] != want[row][1] || + jsonMin(got[row][2]) != jsonMin(want[row][2]) || jsonMin(got[row][3]) != jsonMin(want[row][3]) { + t.Fatalf("row %d = %v, want %v", row, got[row], want[row]) + } + } + + witness := svc.witness(t, + `SELECT 'p-1'::text, '"final"'::jsonb, '["final","prelim"]'::jsonb, '["final","prelim"]'::jsonb`+ + ` UNION ALL SELECT 'p-2', NULL, '[]'::jsonb, '[]'::jsonb`+ + ` UNION ALL SELECT 'p-3', NULL, '[]'::jsonb, '[]'::jsonb`+ + ` UNION ALL SELECT 'p-4', '"dup"', '["dup","dup"]'::jsonb, '["dup","dup"]'::jsonb`+ + ` ORDER BY 1`) + if len(witness) != 4 { + t.Fatalf("witness shape: %v", witness) + } + for row := range want { + if witness[row][0] != want[row][0] || witness[row][1] != want[row][1] || + jsonMin(witness[row][2]) != jsonMin(want[row][2]) || jsonMin(witness[row][3]) != jsonMin(want[row][3]) { + t.Fatalf("witness row %d = %v, want %v", row, witness[row], want[row]) + } + } + + counts := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_edge_occurrence"`+ + ` WHERE project=$1 AND dataset_generation=$2 AND source_logical_id='p-1' AND target_logical_id='obs-final'`, + setsProj, setsGen) + if len(counts) != 1 || counts[0][0] != "2" { + t.Fatalf("duplicate occurrence must be stored (UNIQUE parity needs it): %v", counts) + } + blocked := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource"`+ + ` WHERE project=$1 AND dataset_generation=$2 AND logical_id='obs-blocked' AND payload #> '{status}'::text[] = '"sneaky"'::jsonb`, + setsProj, setsGen) + if len(blocked) != 1 || blocked[0][0] != "1" { + t.Fatalf("denied-path child must be stored for the E4 exclusion to mean anything: %v", blocked) + } + for _, row := range got { + for _, cell := range row { + if strings.Contains(cell, "sneaky") { + t.Fatalf("denied-path value leaked into results: %v", got) + } + } + } + + if _, err := svc.iso.Exec(svc.ctx, + `UPDATE "`+svc.schema+`"."fhir_resource" SET payload = '{"status":"withdrawn"}'::jsonb`+ + ` WHERE project=$1 AND dataset_generation=$2 AND logical_id='obs-prelim'`, + setsProj, setsGen); err != nil { + svc.fail("mutation probe", err) + } + mutated := svc.runRendered(t, rendered) + if len(mutated) != 4 || jsonMin(mutated[0][2]) != jsonMin(`[final withdrawn]`) { + t.Fatalf("mutation probe did not move the aggregate: %v", mutated) + } +} + +func TestSetLiveRelatedExistsAndNegationAreSemiJoins(t *testing.T) { + svc := newSets08Service(t) + sets08TrackFixtures(t, svc) + + setsInsertPatient(t, svc, "p-has") + setsInsertPatient(t, svc, "p-none") + setsInsertPatient(t, svc, "p-denied") + setsInsertObservation(t, svc, "obs-live", "FHIR/Observation", map[string]any{"status": "final"}) + setsInsertObservation(t, svc, "obs-hidden", "FHIR/Blocked", map[string]any{"status": "final"}) + setsInsertRouteEdge(t, svc, "p-has", "obs-live", "observation[0]") + setsInsertRouteEdge(t, svc, "p-denied", "obs-hidden", "observation[0]") + + positive := svc.runRendered(t, mustRender(t, setsRelatedExistsPlan(false), svc.schema)) + if len(positive) != 1 || positive[0][0] != "p-has" { + t.Fatalf("EXISTS form must keep exactly the parent with a reachable child: %v", positive) + } + negative := svc.runRendered(t, mustRender(t, setsRelatedExistsPlan(true), svc.schema)) + if len(negative) != 2 { + t.Fatalf("NOT(EXISTS) must keep the non-matching parents (p-none and p-denied): %v", negative) + } + if negative[0][0] != "p-none" || negative[1][0] != "p-denied" { + t.Fatalf("NOT(EXISTS) rows = %v, want p-none then p-denied", negative) + } + + stored := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_edge_occurrence" WHERE project=$1 AND dataset_generation=$2`, + setsProj, setsGen) + if len(stored) != 1 || stored[0][0] != "2" { + t.Fatalf("both route occurrences must be stored: %v", stored) + } + hidden := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" WHERE project=$1 AND dataset_generation=$2 AND logical_id='obs-hidden'`, + setsProj, setsGen) + if len(hidden) != 1 || hidden[0][0] != "1" { + t.Fatalf("denied-path child must exist for the E4 leg to mean anything: %v", hidden) + } +} + +func mustRender(t *testing.T, plan ir.PhysicalPlan, schema string) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + return rendered +} + +func TestSetLiveRelatedExistsChainedRouteSemiJoin(t *testing.T) { + svc := newSets08Service(t) + sets08TrackFixtures(t, svc) + + setsInsertPatient(t, svc, "p-chain") + setsInsertPatient(t, svc, "p-deadend") + svc.insertResource(t, svc.schema, setsProj, setsGen, "Observation", "obs-mid", "FHIR/Observation", nil) + svc.insertResource(t, svc.schema, setsProj, setsGen, "Observation", "obs-alt", "FHIR/Observation", nil) + svc.insertResource(t, svc.schema, setsProj, setsGen, "Specimen", "spec-1", "FHIR/Specimen", nil) + setsInsertRouteEdge(t, svc, "p-chain", "obs-mid", "observation[0]") + setsInsertRouteEdge(t, svc, "p-deadend", "obs-alt", "observation[0]") + svc.insertEdge(t, pgstore.EdgeOccurrenceRow{ + Project: setsProj, DatasetGeneration: setsGen, AuthResourcePath: "FHIR/Observation", + Label: "has-specimen", SourceResourceType: "Observation", SourceLogicalID: "obs-mid", + TargetResourceType: "Specimen", TargetLogicalID: "spec-1", SourcePath: "specimen[0]", SourceOrdinal: 0, + }) + + plan := setsRelatedExistsTwoHopPlan() + plan.BindVars["auth_resource_paths"] = []string{"FHIR/Patient", "FHIR/Observation", "FHIR/Specimen"} + positive := svc.runRendered(t, mustRender(t, plan, svc.schema)) + if len(positive) != 1 || positive[0][0] != "p-chain" { + t.Fatalf("chained EXISTS must keep only the parent whose second hop resolves: %v", positive) + } + + firstHop := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_edge_occurrence"`+ + ` WHERE project=$1 AND dataset_generation=$2 AND label=$3 AND source_logical_id='p-deadend'`, + setsProj, setsGen, setsEdgeLabel) + if len(firstHop) != 1 || firstHop[0][0] != "1" { + t.Fatalf("dead-end parent must store its first hop for the exclusion to mean anything: %v", firstHop) + } + secondHop := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_edge_occurrence"`+ + ` WHERE project=$1 AND dataset_generation=$2 AND label='has-specimen'`, + setsProj, setsGen) + if len(secondHop) != 1 || secondHop[0][0] != "1" { + t.Fatalf("the chained occurrence must be stored exactly once: %v", secondHop) + } +} diff --git a/internal/dataframe/compiler/render/postgres/sets_test.go b/internal/dataframe/compiler/render/postgres/sets_test.go new file mode 100644 index 000000000..414320b1c --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/sets_test.go @@ -0,0 +1,797 @@ +package postgres + +import ( + "fmt" + "reflect" + "strconv" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" +) + +const ( + setsPrefix = "child_set_0" + setsEdgeVariable = "child_set_0_edge" + setsTargetVariable = "child_set_0_node" + setsReducedVariable = "child_set_0_reduced" +) + +func setsTraversal() ir.PhysicalTraversal { + return ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: setsTargetVariable, EdgeVariable: setsEdgeVariable, + Direction: ir.PhysicalOutbound, + EdgeCollectionBindKey: setsPrefix + "_edge_collection", + EdgeLabelBindKey: setsPrefix + "_edge_label", + TargetTypeBindKey: setsPrefix + "_target_type", + EdgeTargetTypeField: "to_type", Strategy: ir.PhysicalTraversalNative, + } +} + +func setsPlan() ir.PhysicalPlan { + plan := basePlan() + plan.BindVars[setsPrefix+"_edge_collection"] = "fhir_edge" + plan.BindVars[setsPrefix+"_edge_label"] = "has-observation" + plan.BindVars[setsPrefix+"_target_type"] = "Observation" + return plan +} + +func setsProjectionField(name, property string) ir.PhysicalSetProjectionField { + return ir.PhysicalSetProjectionField{ + Name: name, ResourceType: "Observation", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: property}}}, + ExecutionMode: ir.PhysicalSelectorDirectScalar, + } +} + +func setsChildSet(modes ...ir.PhysicalSetReductionMode) ir.PhysicalSet { + traversal := setsTraversal() + operations := []ir.PhysicalOperation{{Kind: ir.PhysicalTraversalOp, Traversal: &traversal}} + operations = appendScopeBlock(operations, []string{setsEdgeVariable, setsTargetVariable}, setsPrefix+"_scope_auth") + fields := make([]ir.PhysicalSetProjectionField, 0, len(modes)) + reductionFields := make([]ir.PhysicalSetReductionField, 0, len(modes)) + for index, mode := range modes { + slot := "__loom_projection_" + strconv.Itoa(index) + fields = append(fields, setsProjectionField(slot, "status")) + reductionFields = append(reductionFields, ir.PhysicalSetReductionField{ + Name: "__loom_reduced_" + strconv.Itoa(index), SourceField: slot, Mode: mode, + }) + } + return ir.PhysicalSet{ + Variable: setsPrefix, + Subplan: ir.PhysicalSubplan{ + Captures: []string{"root"}, + Operations: operations, + Return: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalObjectCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{Variable: setsTargetVariable}, + }, + }, + Unique: true, + SortByKey: true, + Projection: &ir.PhysicalSetProjection{ + Fields: fields, + }, + Reduction: &ir.PhysicalSetReduction{ + Variable: setsReducedVariable, SourceSetVariable: setsPrefix, Fields: reductionFields, + }, + } +} + +func setsReturn(names ...string) ir.PhysicalOperation { + projections := make([]ir.PhysicalProjection, 0, len(names)+1) + projections = append(projections, ir.PhysicalProjection{ + Name: "patient_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}) + for _, name := range names { + projections = append(projections, ir.PhysicalProjection{ + Name: name, Value: ir.PhysicalValue{Variable: setsReducedVariable, Path: []string{name}}}) + } + return ir.PhysicalOperation{Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}} +} + +func setsPlanWithSet(set ir.PhysicalSet, names ...string) ir.PhysicalPlan { + plan := setsPlan() + plan.Operations = plan.Operations[:len(plan.Operations)-1] + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalSetOp, Set: &set}, + setsReturn(names...)) + return plan +} + +func TestSetChildSetReductionsRenderAsCorrelatedScalarSublinks(t *testing.T) { + set := setsChildSet(ir.PhysicalSetReductionFirst, ir.PhysicalSetReductionAll, ir.PhysicalSetReductionDistinct) + rendered := renderOK(t, setsPlanWithSet(set, + "__loom_reduced_0", "__loom_reduced_1", "__loom_reduced_2")) + + sql := rendered.SQL + for _, forbidden := range []string{"LEFT JOIN", "LATERAL (SELECT"} { + if strings.Contains(sql, forbidden) { + t.Fatalf("set must not add outer joins (%q present): %s", forbidden, sql) + } + } + for _, want := range []string{ + `ORDER BY i2."logical_id", s2.ord LIMIT 1`, + `jsonb_agg(s3.v ORDER BY i3."logical_id", s3.ord)`, + `DISTINCT ON (i4."logical_id", s4.v)`, + } { + if got := strings.Count(sql, want); got != 3 { + t.Fatalf("occurrences of %q = %d, want 3 (one per reduced-variable projection): %s", want, got, sql) + } + } + if got := strings.Count(sql, `= r0."logical_id"`); got != 9 { + t.Fatalf("correlated item scans = %d, want 9 (3 fields x 3 readers): %s", got, sql) + } + if got := strings.Count(sql, `SELECT DISTINCT x1."logical_id"`); got != 9 { + t.Fatalf("item dedup count = %d, want 9: %s", got, sql) + } + if !strings.Contains(sql, "jsonb_build_object($") || !strings.Contains(sql, "::text, (SELECT") { + t.Fatalf("reduced object must bind text-cast keys over sublink values: %s", sql) + } + for _, leaked := range []string{"has-observation", "Observation", "proj-a", "gen-1", "FHIR/Patient", "__loom_reduced_0"} { + if strings.Contains(sql, leaked) { + t.Fatalf("value %q leaked into statement text: %s", leaked, sql) + } + } + if !strings.Contains(sql, "$") { + t.Fatalf("expected bound parameters: %s", sql) + } + for _, want := range []string{"e1.", "auth_resource_path"} { + if !strings.Contains(sql, want) { + t.Fatalf("sublinks must carry the structural edge scope (%q missing): %s", want, sql) + } + } + + again := renderOK(t, setsPlanWithSet(set, + "__loom_reduced_0", "__loom_reduced_1", "__loom_reduced_2")) + if again.SQL != sql || !reflect.DeepEqual(again.Args, rendered.Args) { + t.Fatalf("rendering is not deterministic") + } + if got, want := len(rendered.OutputColumns), 4; got != want { + t.Fatalf("output columns = %d, want %d", got, want) + } +} + +func setsRelatedExistsExpression(negate bool) ir.PhysicalPredicateExpression { + traversal := setsTraversal() + operations := []ir.PhysicalOperation{{Kind: ir.PhysicalTraversalOp, Traversal: &traversal}} + operations = appendScopeBlock(operations, []string{setsEdgeVariable, setsTargetVariable}, setsPrefix+"_scope_auth") + subplan := ir.PhysicalSubplan{ + Captures: []string{"root"}, + Operations: operations, + Return: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{Variable: setsTargetVariable, Path: []string{"_id"}}, + }, + } + exists := ir.PhysicalPredicateExpression{Kind: ir.PhysicalExistsPredicate, Exists: &subplan} + if !negate { + return exists + } + return ir.PhysicalPredicateExpression{Kind: ir.PhysicalNotPredicate, Children: []ir.PhysicalPredicateExpression{exists}} +} + +func setsRelatedExistsPlan(negate bool) ir.PhysicalPlan { + plan := setsPlan() + plan.Operations = plan.Operations[:len(plan.Operations)-1] + expression := setsRelatedExistsExpression(negate) + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Expression: &expression}}, + setsReturn()) + return plan +} + +func TestRelatedExistsFilterRendersBoundedSemiJoin(t *testing.T) { + rendered := renderOK(t, setsRelatedExistsPlan(false)) + sql := rendered.SQL + if !strings.Contains(sql, "AND EXISTS (SELECT 1 FROM ") { + t.Fatalf("related existence filter must render the bounded semi-join:\n%s", sql) + } + if strings.Contains(sql, "LIMIT") { + t.Fatalf("EXISTS makes the AQL LIMIT 1 witness unnecessary (no LIMIT allowed): %s", sql) + } + if !strings.Contains(sql, `= r0."logical_id"`) { + t.Fatalf("semi-join must correlate on the outer row identity: %s", sql) + } + for _, want := range []string{`e1."auth_resource_path"`, `x1."auth_resource_path"`} { + if !strings.Contains(sql, want) { + t.Fatalf("semi-join must carry the canonical auth scope (%q missing): %s", want, sql) + } + } + if strings.Contains(sql, "LEFT JOIN") { + t.Fatalf("semi-join must not add a row-multiplying join: %s", sql) + } +} + +func TestRelatedExistsNotFormIsUnguardedTotalNot(t *testing.T) { + rendered := renderOK(t, setsRelatedExistsPlan(true)) + sql := rendered.SQL + if !strings.Contains(sql, "AND NOT (EXISTS (SELECT 1 FROM ") { + t.Fatalf("registered top-level NOT(EXISTS) form missing:\n%s", sql) + } + if strings.Contains(sql, "COALESCE") { + t.Fatalf("NOT over the total semi-join must stay unguarded: %s", sql) + } +} + +func TestRequiredShapeAndUnknownReturnsKeepTheTraversalPath(t *testing.T) { + plan := setsPlan() + plan.Operations = plan.Operations[:len(plan.Operations)-1] + plan.BindVars["required_0_result"] = 1 + traversal := setsTraversal() + operations := []ir.PhysicalOperation{{Kind: ir.PhysicalTraversalOp, Traversal: &traversal}} + subplan := ir.PhysicalSubplan{ + Captures: []string{"root"}, + Operations: operations, + Return: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{BindKey: "required_0_result"}, + }, + } + exists := ir.PhysicalPredicateExpression{Kind: ir.PhysicalExistsPredicate, Exists: &subplan} + notExpression := ir.PhysicalPredicateExpression{Kind: ir.PhysicalNotPredicate, Children: []ir.PhysicalPredicateExpression{exists}} + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Expression: ¬Expression}}, + setsReturn()) + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + wantUnsupported(t, err, "NOT over the REQUIRED constant-1 EXISTS") + + plan = setsPlan() + plan.Operations = plan.Operations[:len(plan.Operations)-1] + traversal = setsTraversal() + bad := ir.PhysicalSubplan{ + Captures: []string{"root"}, + Operations: []ir.PhysicalOperation{{Kind: ir.PhysicalTraversalOp, Traversal: &traversal}}, + Return: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalObjectCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{Variable: setsTargetVariable}, + }, + } + expression := ir.PhysicalPredicateExpression{Kind: ir.PhysicalExistsPredicate, Exists: &bad} + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Expression: &expression}}, + setsReturn()) + _, err = RenderPhysicalPlan(plan, mustBinding(t)) + wantUnsupported(t, err, "EXISTS with a non-canonical return") + if !strings.Contains(err.Error(), "required-match subplan return") { + t.Fatalf("unknown EXISTS shapes must keep the traversal.go refusal, got: %v", err) + } +} + +func TestSetSubplanFilterRendersSharedSelectorMatch(t *testing.T) { + set := setsChildSet(ir.PhysicalSetReductionAll) + filterValue := "final" + set.Subplan.Operations = append(set.Subplan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Expression: &ir.PhysicalPredicateExpression{Kind: ir.PhysicalComparisonPredicate, Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", ValueKind: spec.FilterString, + LeftExpression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: setsTargetVariable, Path: []string{"payload"}}, + ResourceType: "Observation", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: "status"}}}, + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + }, + Right: &ir.PhysicalValue{BindKey: "child_filter_value"}, + }}}}) + plan := setsPlanWithSet(set, "__loom_reduced_0") + plan.BindVars["child_filter_value"] = filterValue + + rendered := renderOK(t, plan) + sql := rendered.SQL + if !strings.Contains(sql, "x1.") || strings.Contains(sql, filterValue) { + t.Fatalf("child filter value must bind, not inline: %s", sql) + } + if got := strings.Count(sql, `x1."payload" #> $`); got < 2 { + t.Fatalf("child filter must participate in the correlated scan (payload path count %d): %s", got, sql) + } + boundFilter := false + for _, arg := range rendered.Args { + if strings.Contains(fmt.Sprintf("%s", arg.Value), strconv.Quote(filterValue)) { + boundFilter = true + } + } + if !boundFilter { + t.Fatalf("filter value never reached the bind list: %v", rendered.Args) + } +} + +func setsSpineTraversal() ir.PhysicalOperation { + traversal := setsTraversal() + return ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &traversal} +} + +func TestSetCaptureAllowsOptionalAnchorExistanceCaptureRefuses(t *testing.T) { + optional := setsTraversal() + optional.TargetVariable = "spine_node" + plan := setsPlan() + plan.Operations = plan.Operations[:len(plan.Operations)-1] + set := setsChildSet(ir.PhysicalSetReductionFirst) + set.Subplan.Captures = []string{"spine_node"} + set.Subplan.Operations[0].Traversal.SourceVariable = "spine_node" + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &optional}, + ir.PhysicalOperation{Kind: ir.PhysicalSetOp, Set: &set}, + setsReturn("__loom_reduced_0")) + rendered := renderOK(t, plan) + if !strings.Contains(rendered.SQL, `= r1."logical_id"`) { + t.Fatalf("SET sublink must correlate on the OPTIONAL lateral row: %s", rendered.SQL) + } + + expression := setsRelatedExistsExpression(false) + expression.Exists.Captures = []string{"spine_node"} + expression.Exists.Operations[0].Traversal.SourceVariable = "spine_node" + plan = setsPlan() + plan.Operations = plan.Operations[:len(plan.Operations)-1] + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &optional}, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Expression: &expression}}, + setsReturn()) + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + wantUnsupported(t, err, "EXISTS anchored on an OPTIONAL row") + if !strings.Contains(err.Error(), "NULL-extended anchor") { + t.Fatalf("refusal must name the anchor rule, got: %v", err) + } +} + +func setsBatteryCase(t *testing.T, name, want string, rendererLevel bool, mutate func(*ir.PhysicalPlan)) { + t.Helper() + set := setsChildSet(ir.PhysicalSetReductionFirst) + plan := setsPlanWithSet(set, "__loom_reduced_0") + mutate(&plan) + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil { + t.Fatalf("%s: expected refusal, render succeeded", name) + } + if !strings.Contains(err.Error(), want) { + t.Fatalf("%s: error = %v, want substring %q", name, err, want) + } + if rendererLevel { + wantUnsupported(t, err, name) + } +} + +func setsBatterySet(plan *ir.PhysicalPlan) *ir.PhysicalSet { + for index := range plan.Operations { + if plan.Operations[index].Kind == ir.PhysicalSetOp { + return plan.Operations[index].Set + } + } + panic("battery plan has no SET operation") +} + +func TestSetSurfaceRefusals(t *testing.T) { + tests := []struct { + name string + want string + rendererLevel bool + mutate func(*ir.PhysicalPlan) + }{ + { + name: "missing reduction", + want: "no reduction contract", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + set := setsBatterySet(plan) + set.Reduction = nil + plan.Operations[len(plan.Operations)-1] = setsReturn() + }, + }, + { + name: "missing projection", + want: "reduction requires a selector projection", + mutate: func(plan *ir.PhysicalPlan) { + setsBatterySet(plan).Projection = nil + }, + }, + { + name: "unique flag dropped", + want: "Unique and SortByKey", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { setsBatterySet(plan).Unique = false }, + }, + { + name: "sort-by-key flag dropped", + want: "Unique and SortByKey", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { setsBatterySet(plan).SortByKey = false }, + }, + { + name: "node-path kind", + want: "NODE_PATH", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { setsBatterySet(plan).Kind = ir.PhysicalNodePathSetKind }, + }, + { + name: "unknown kind", + want: "unknown set kind", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { setsBatterySet(plan).Kind = "GRAPH_WALK" }, + }, + { + name: "shared subset", + want: "shared subsets", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + set := setsBatterySet(plan) + set.SourceSetVariable = "root" + set.ItemVariable = "item" + filter := ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: "root", Path: []string{"project"}}, + Right: &ir.PhysicalValue{BindKey: "project"}, + }} + set.Subplan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalFilterOp, Filter: &filter}} + set.Subplan.Return.Value = &ir.PhysicalValue{Variable: "item"} + }, + }, + { + name: "prepared set", + want: "prepared sets are not implemented", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + setsBatterySet(plan).Prepared = &ir.PhysicalPreparedSet{ + Variable: "child_set_0_prepared", SourceSetVariable: setsPrefix, + Fields: []ir.PhysicalPreparedField{{ + Name: "__loom_prepared_0", ResourceType: "Observation", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: "status"}}}, + }}, + } + }, + }, + { + name: "subplan sort modifier", + want: "projection-only modifiers", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + setsBatterySet(plan).Subplan.Sort = &ir.PhysicalValue{ + Variable: setsTargetVariable, Path: []string{"logical_id"}} + }, + }, + { + name: "chained hops in a set", + want: "exactly one depth-one traversal hop", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + set := setsBatterySet(plan) + second := setsTraversal() + second.SourceVariable = setsTargetVariable + second.TargetVariable = setsPrefix + "_node2" + second.EdgeVariable = setsPrefix + "_edge2" + set.Subplan.Operations = append(set.Subplan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &second}) + }, + }, + { + name: "nested set in a subplan", + want: "has kind \"SET\"", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + set := setsBatterySet(plan) + traversal := setsTraversal() + set.Subplan.Operations = append(set.Subplan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalSetOp, Set: &ir.PhysicalSet{ + Variable: "inner_set", + Subplan: ir.PhysicalSubplan{ + Captures: []string{"root"}, + Operations: []ir.PhysicalOperation{{Kind: ir.PhysicalTraversalOp, Traversal: &traversal}}, + Return: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalObjectCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{Variable: setsTargetVariable}, + }, + }, + }}) + }, + }, + { + name: "collection scan in a subplan", + want: "has kind \"COLLECTION_SCAN\"", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + plan.BindVars["sets_cs_collection"] = "Observation" + set := setsBatterySet(plan) + set.Subplan.Operations = append(set.Subplan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalCollectionScanOp, CollectionScan: &ir.PhysicalCollectionScan{ + Variable: "cs", CollectionBindKey: "sets_cs_collection", + }}) + }, + }, + { + name: "filter before the hop", + want: "may only follow a traversal step", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + set := setsBatterySet(plan) + filter := ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: "root", Path: []string{"project"}}, + Right: &ir.PhysicalValue{BindKey: "project"}, + }} + set.Subplan.Operations = append([]ir.PhysicalOperation{ + {Kind: ir.PhysicalFilterOp, Filter: &filter}}, set.Subplan.Operations...) + }, + }, + { + name: "non-canonical scope block", + want: "canonical subplan scope block", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + set := setsBatterySet(plan) + set.Subplan.Operations[1].Filter.Predicate.Left.Path = []string{"projekt"} + }, + }, + { + name: "direct read of the set variable", + want: "does not bind a readable row", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + ret := plan.Operations[len(plan.Operations)-1].Return + ret.Projections[1].Value = ir.PhysicalValue{Variable: setsPrefix, Path: []string{"payload"}} + }, + }, + { + name: "set after the execution window without a registered consumer", + want: "sets must precede the execution window", + rendererLevel: true, + mutate: func(plan *ir.PhysicalPlan) { + sortOp := ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{ + Keys: []ir.PhysicalValue{{Variable: "root", Path: []string{"logical_id"}}}, + }} + rootScan, setOp, ret := plan.Operations[0], plan.Operations[1], plan.Operations[2] + setOp.Set.Reduction = nil + ret.Return.Projections = ret.Return.Projections[:1] + plan.Operations = []ir.PhysicalOperation{rootScan, sortOp, setOp, ret} + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + setsBatteryCase(t, test.name, test.want, test.rendererLevel, test.mutate) + }) + } +} + +func TestRelatedExistsSurfaceRefusals(t *testing.T) { + tests := []struct { + name string + want string + rendererLevel bool + mutate func(*ir.PhysicalSubplan, *ir.PhysicalPlan) + }{ + { + name: "projection modifiers on an EXISTS subplan", + want: "projection-only sort or unique modifiers", + mutate: func(subplan *ir.PhysicalSubplan, _ *ir.PhysicalPlan) { + subplan.Sort = &ir.PhysicalValue{Variable: setsTargetVariable, Path: []string{"_id"}} + subplan.Unique = true + }, + }, + { + name: "missing scope block on a hop", + want: "canonical subplan scope block", + rendererLevel: true, + mutate: func(subplan *ir.PhysicalSubplan, _ *ir.PhysicalPlan) { + subplan.Operations = subplan.Operations[:len(subplan.Operations)-1] + }, + }, + { + name: "chained hop without its own block", + want: "canonical subplan scope block", + rendererLevel: true, + mutate: func(subplan *ir.PhysicalSubplan, _ *ir.PhysicalPlan) { + second := setsTraversal() + second.SourceVariable = setsTargetVariable + second.TargetVariable = setsPrefix + "_node2" + second.EdgeVariable = setsPrefix + "_edge2" + subplan.Operations = append(subplan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &second}) + subplan.Return.Value.Variable = setsPrefix + "_node2" + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + expression := setsRelatedExistsExpression(false) + plan := setsPlan() + plan.Operations = plan.Operations[:len(plan.Operations)-1] + test.mutate(expression.Exists, &plan) + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Expression: &expression}}, + setsReturn()) + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil { + t.Fatalf("%s: expected refusal, render succeeded", test.name) + } + if !strings.Contains(err.Error(), test.want) { + t.Fatalf("%s: error = %v, want substring %q", test.name, err, test.want) + } + if test.rendererLevel { + wantUnsupported(t, err, test.name) + } + }) + } +} + +func TestTwoSetsRenderIndependentReducedObjects(t *testing.T) { + first := setsChildSet(ir.PhysicalSetReductionFirst) + second := setsChildSet(ir.PhysicalSetReductionDistinct) + second.Variable = "child_set_1" + second.Subplan.Captures = []string{"root"} + second.Reduction.Variable = "child_set_1_reduced" + second.Reduction.SourceSetVariable = "child_set_1" + second.Reduction.Fields[0].Name = "__loom_reduced_0" + second.Projection.Fields[0].Name = "__loom_projection_0" + second.Projection.Fields[0].Selector = spec.Selector{Steps: []spec.SelectorStep{{Field: "code"}}} + + plan := setsPlan() + plan.Operations = plan.Operations[:len(plan.Operations)-1] + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalSetOp, Set: &first}, + ir.PhysicalOperation{Kind: ir.PhysicalSetOp, Set: &second}, + ir.PhysicalOperation{Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "patient_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + {Name: "first_status", Value: ir.PhysicalValue{Variable: "child_set_0_reduced", Path: []string{"__loom_reduced_0"}}}, + {Name: "distinct_codes", Value: ir.PhysicalValue{Variable: "child_set_1_reduced", Path: []string{"__loom_reduced_0"}}}, + }}}) + rendered := renderOK(t, plan) + if got := strings.Count(rendered.SQL, "jsonb_build_object("); got != 2 { + t.Fatalf("reduced objects = %d, want one per set: %s", got, rendered.SQL) + } + bound := map[string]bool{} + for _, arg := range rendered.Args { + if text, ok := arg.Value.(string); ok { + bound[text] = true + } + } + for _, want := range []string{"{status}", "{code}"} { + if !bound[want] { + t.Fatalf("selector path %q never bound: %v", want, rendered.Args) + } + } +} + +func setsAnchorPrefixedPlan(t *testing.T, anchorKind string) ir.PhysicalPlan { + t.Helper() + traversal := setsTraversal() + anchorVariable := "related_eligibility_0_anchor" + operations := make([]ir.PhysicalOperation, 0, 10) + switch anchorKind { + case "scan": + operations = append(operations, + ir.PhysicalOperation{Kind: ir.PhysicalCollectionScanOp, CollectionScan: &ir.PhysicalCollectionScan{ + Variable: anchorVariable, CollectionBindKey: "root_collection", + }}, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: anchorVariable, Path: []string{"_key"}}, + Right: &ir.PhysicalValue{Variable: "root", Path: []string{"_key"}}, + }}}) + case "lookup": + operations = append(operations, + ir.PhysicalOperation{Kind: ir.PhysicalDocumentLookupOp, DocumentLookup: &ir.PhysicalDocumentLookup{ + Variable: anchorVariable, CollectionBindKey: "related_eligibility_anchor_collection", + ExactID: ir.PhysicalValue{Variable: "root", Path: []string{"focus_id"}}, + }}) + default: + t.Fatalf("unknown anchor kind %q", anchorKind) + } + operations = appendScopeBlock(operations, []string{anchorVariable}, "related_eligibility_0_anchor_scope_allowed") + operations = append(operations, ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &traversal}) + operations = appendScopeBlock(operations, []string{setsEdgeVariable, setsTargetVariable}, setsPrefix+"_scope_auth") + subplan := ir.PhysicalSubplan{ + Captures: []string{"root"}, + Operations: operations, + Return: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{Variable: setsTargetVariable, Path: []string{"_id"}}, + }, + } + expression := ir.PhysicalPredicateExpression{Kind: ir.PhysicalExistsPredicate, Exists: &subplan} + plan := setsPlan() + plan.BindVars["related_eligibility_anchor_collection"] = "Observation" + plan.Operations = plan.Operations[:len(plan.Operations)-1] + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Expression: &expression}}, + setsReturn()) + return plan +} + +func TestRelatedExistsAnchorPrefixedProductionShapeStaysRefused(t *testing.T) { + for _, anchorKind := range []string{"scan", "lookup"} { + plan := setsAnchorPrefixedPlan(t, anchorKind) + _, err := RenderPhysicalPlan(plan, mustBinding(t)) + wantUnsupported(t, err, "anchor-prefixed ("+anchorKind+") eligibility subplan") + if !strings.Contains(err.Error(), "has kind") { + t.Fatalf("%s anchor refusal must name the anchor operation kind: %v", anchorKind, err) + } + } +} + +func setsRelatedExistsTwoHopPlan() ir.PhysicalPlan { + hopOne := setsTraversal() + hopTwo := setsTraversal() + hopTwo.SourceVariable = setsTargetVariable + hopTwo.TargetVariable = setsPrefix + "_node2" + hopTwo.EdgeVariable = setsPrefix + "_edge2" + hopTwo.EdgeCollectionBindKey = setsPrefix + "_edge_collection2" + hopTwo.EdgeLabelBindKey = setsPrefix + "_edge_label2" + hopTwo.TargetTypeBindKey = setsPrefix + "_target_type2" + operations := []ir.PhysicalOperation{{Kind: ir.PhysicalTraversalOp, Traversal: &hopOne}} + operations = appendScopeBlock(operations, []string{setsEdgeVariable, setsTargetVariable}, setsPrefix+"_scope_auth") + operations = append(operations, ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &hopTwo}) + operations = appendScopeBlock(operations, + []string{setsPrefix + "_edge2", setsPrefix + "_node2"}, setsPrefix+"_scope_auth2") + subplan := ir.PhysicalSubplan{ + Captures: []string{"root"}, + Operations: operations, + Return: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{Variable: setsPrefix + "_node2", Path: []string{"_id"}}, + }, + } + expression := ir.PhysicalPredicateExpression{Kind: ir.PhysicalExistsPredicate, Exists: &subplan} + plan := setsPlan() + plan.BindVars[setsPrefix+"_edge_collection2"] = "fhir_edge" + plan.BindVars[setsPrefix+"_edge_label2"] = "has-specimen" + plan.BindVars[setsPrefix+"_target_type2"] = "Specimen" + plan.Operations = plan.Operations[:len(plan.Operations)-1] + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Expression: &expression}}, + setsReturn()) + return plan +} + +func TestRelatedExistsChainedRouteGoldenAssemblesBothHops(t *testing.T) { + rendered := renderOK(t, setsRelatedExistsTwoHopPlan()) + sql := rendered.SQL + for _, want := range []string{ + `FROM "pgir02_it_unit"."fhir_edge_occurrence" AS e1`, + `INNER JOIN "pgir02_it_unit"."fhir_resource" AS x1 ON`, + `INNER JOIN "pgir02_it_unit"."fhir_edge_occurrence" AS e2 ON`, + `INNER JOIN "pgir02_it_unit"."fhir_resource" AS x2 ON`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("chained FROM missing %q:\n%s", want, sql) + } + } + if !strings.Contains(sql, `e1."source_logical_id" = r0."logical_id"`) { + t.Fatalf("first hop must correlate on the outer row: %s", sql) + } + if !strings.Contains(sql, `e2."source_logical_id" = x1."logical_id"`) { + t.Fatalf("second hop must anchor on x1 with its full predicate chain: %s", sql) + } + if !strings.Contains(sql, `e2."label" = $`) || !strings.Contains(sql, `e2."target_resource_type" = $`) { + t.Fatalf("second hop must bind its label and reached type: %s", sql) + } + for _, want := range []string{`e1."auth_resource_path"`, `x1."auth_resource_path"`, `e2."auth_resource_path"`, `x2."auth_resource_path"`} { + if !strings.Contains(sql, want) { + t.Fatalf("chained semi-join must carry the canonical auth scope (%q missing): %s", want, sql) + } + } + if strings.Contains(sql, "LIMIT") { + t.Fatalf("EXISTS stays limit-free: %s", sql) + } + if got := len(rendered.Args); got != 8 { + t.Fatalf("chained parameter count = %d, want 8 (deduped discriminators): %v", got, rendered.Args) + } + for _, leaked := range []string{"has-specimen", "Specimen", "has-observation"} { + if strings.Contains(sql, leaked) { + t.Fatalf("value %q leaked into statement text: %s", leaked, sql) + } + } + again := renderOK(t, setsRelatedExistsTwoHopPlan()) + if again.SQL != sql || !reflect.DeepEqual(again.Args, rendered.Args) { + t.Fatalf("chained rendering is not deterministic") + } +} diff --git a/internal/dataframe/compiler/render/postgres/stage_related_occurrence.go b/internal/dataframe/compiler/render/postgres/stage_related_occurrence.go new file mode 100644 index 000000000..780cc5008 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/stage_related_occurrence.go @@ -0,0 +1,651 @@ +package postgres + +import ( + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const ( + stageRelatedCodingOrdinalSemantics = "ZERO_BASED_POSITION_IN_FLATTENED_EXPANSION_ARRAY" + + stageRelatedTerminalField = "terminal_id" + stageRelatedResourceField = "resource_id" +) + +type stageRelatedOccurrenceRegion struct { + sourceText string + columns map[string]groupColumn + orderTerms []string +} + +func (r *planRenderer) renderStageGroupRelatedOccurrence( + plan ir.PhysicalPlan, + stage ir.PhysicalConstructionStage, + index int, + relation string, + facts map[string]stageColumnFact, +) (stageRelatedOccurrenceRegion, error) { + adapter := stage.Group.RelatedOccurrence + if adapter == nil { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: the related-occurrence adapter arm requires a Group payload that names one", + ErrUnsupportedPostgresPlan) + } + if err := validateStageRelatedOccurrenceAdapter(plan, stage); err != nil { + return stageRelatedOccurrenceRegion{}, err + } + related := adapter.RelatedRecords + + keyFact, ok := facts[related.AnchorColumnID] + if !ok { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: the related route anchor column %q is not among the prior stage's registered columns (L-16.1)", + ErrUnsupportedPostgresPlan, related.AnchorColumnID) + } + if keyFact.jsonb { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: the related route anchor column %q is jsonb, not the materialized root-key scalar", + ErrUnsupportedPostgresPlan, related.AnchorColumnID) + } + anchorRead := stageTerminalAlias + "." + `"` + keyFact.alias + `"` + + route, err := r.renderStageRelatedRoute(plan, stage, anchorRead) + if err != nil { + return stageRelatedOccurrenceRegion{}, err + } + + docBase := route.alias + ".doc" + + fields := related.RelatedRecords.Return.Object.Fields + scalarFields := make([]ir.PhysicalProjection, 0, len(fields)) + var codingProjection *ir.PhysicalProjection + for fieldIndex := range fields { + field := fields[fieldIndex] + if adapter.CodingUnnest != nil && field.Name == adapter.CodingArrayColumn { + expression := field.Expression + codingProjection = &ir.PhysicalProjection{Name: field.Name, Expression: &expression} + continue + } + expression := field.Expression + scalarFields = append(scalarFields, ir.PhysicalProjection{Name: field.Name, Expression: &expression}) + } + classifiedFields, err := classifyExtractionProjections(scalarFields, + func(variable string) (string, bool) { + if variable == route.variable { + return route.targetType, true + } + return "", false + }) + if err != nil { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("related route target projection: %w", err) + } + classified := make(map[string]classifiedExtractionProjection, len(classifiedFields)) + for _, field := range classifiedFields { + classified[field.projection.Name] = field + } + for _, name := range []string{stageRelatedTerminalField, stageRelatedResourceField} { + field, present := classified[name] + if !present { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: the related route target projection lacks its %s identity field (recipe_construction_related_expand.go:218-225 parity)", + ErrUnsupportedPostgresPlan, name) + } + value := identityFieldValue(field) + if value == nil || value.BindKey != "" || value.Variable != route.variable || len(value.Path) != 1 { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: the related route %s field must read the reached target row directly", + ErrUnsupportedPostgresPlan, name) + } + want := "_id" + if name == stageRelatedResourceField { + want = "id" + } + if value.Path[0] != want { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: the related route %s field must read the target's {%s} (logical_id is its registered counterpart, graph.go:99-107)", + ErrUnsupportedPostgresPlan, name, want) + } + } + + prefix := "__loom_r" + strconv.Itoa(index) + "_" + columns := map[string]groupColumn{} + adapterSelects := make([]string, 0, len(adapter.ScalarProjections)+6) + + for _, projection := range adapter.ScalarProjections { + field, present := classified[projection.ResultField] + if !present { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: the related route scalar projection %q reads target field %q, which the target projection does not declare", + ErrUnsupportedPostgresPlan, projection.InputColumn, projection.ResultField) + } + if field.kind != extractionSelector || field.expression == nil || + field.expression.Cardinality != ir.PhysicalScalarCardinality || field.expression.Extract == nil { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: the related route scalar projection %q is not a scalar selector extraction of the reached target document", + ErrUnsupportedPostgresPlan, projection.InputColumn) + } + if field.expression.Extract.ResourceType != projection.OwnerResourceType { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: the related route scalar projection %q validates its selector against %q, not the owner type %q", + ErrUnsupportedPostgresPlan, projection.InputColumn, field.expression.Extract.ResourceType, projection.OwnerResourceType) + } + extraction, err := r.documentPathFromExpression(docBase, field.path) + if err != nil { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("related route scalar projection %q: %w", projection.InputColumn, err) + } + value := "(" + extraction + ")" + switch field.expression.NullBehavior { + case ir.PhysicalPreserveNull: + case ir.PhysicalOmitNulls: + value = "NULLIF(" + extraction + ", 'null'::jsonb)" + default: + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: the related route scalar projection %q has null behavior %q; the selected scalar reads are PRESERVE_NULL (the OmitNulls twin is registered, anything else is a capability gap)", + ErrUnsupportedPostgresPlan, projection.InputColumn, string(field.expression.NullBehavior)) + } + alias := prefix + "s" + strconv.Itoa(len(columns)) + adapterSelects = append(adapterSelects, value+` AS "`+alias+`"`) + columns[projection.InputColumn] = groupColumn{ + name: projection.InputColumn, alias: alias, expr: value, ref: `"` + alias + `"`, jsonb: true, + } + } + + codingAlias := prefix + "cod" + codingSelect := "" + if adapter.CodingUnnest != nil { + if codingProjection == nil { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: the related route Coding array column %q is not declared by the target projection", + ErrUnsupportedPostgresPlan, adapter.CodingArrayColumn) + } + expression := codingProjection.Expression + if expression == nil || expression.Kind != ir.PhysicalExtractExpression || + expression.Cardinality != ir.PhysicalArrayCardinality || + expression.NullBehavior != ir.PhysicalEmptyOnNull || expression.Extract == nil { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: the related route Coding column %q is not an EMPTY_ON_NULL array selector extraction of the reached target document", + ErrUnsupportedPostgresPlan, adapter.CodingArrayColumn) + } + if expression.Extract.Source.Variable != route.variable { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: the related route Coding column %q must read the reached target row directly", + ErrUnsupportedPostgresPlan, adapter.CodingArrayColumn) + } + if expression.Extract.ResourceType != related.TargetResourceType { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: the related route Coding selector validates against %q, not the target type %q", + ErrUnsupportedPostgresPlan, expression.Extract.ResourceType, related.TargetResourceType) + } + arrayForm, err := r.stageRelatedCodingArrayExpression(docBase, adapter.CodingArrayColumn, expression.Extract) + if err != nil { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("related route Coding array: %w", err) + } + elementAlias := codingAlias + "_e" + codingSelect = "(SELECT " + elementAlias + ".item AS item, ROW_NUMBER() OVER (ORDER BY " + + elementAlias + ".ord) AS rn FROM jsonb_array_elements(" + arrayForm + + ") WITH ORDINALITY AS " + elementAlias + "(item, ord) WHERE jsonb_typeof(" + elementAlias + + ".item) <> 'null')" + membershipArgs := make([]string, 0, 4) + for _, member := range []string{"system", "version", "code", "display"} { + memberRead, memberErr := r.documentPathFromExpression(codingAlias+".item", []string{member}) + if memberErr != nil { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("related route Coding membership: %w", memberErr) + } + bind := r.textParameter(member, "text:coding:"+member) + membershipArgs = append(membershipArgs, "$"+strconv.Itoa(bind)+"::text, "+memberRead) + } + membershipSelect := "jsonb_build_object(" + strings.Join(membershipArgs, ", ") + ")" + membershipAlias := prefix + "m" + adapterSelects = append(adapterSelects, membershipSelect+` AS "`+membershipAlias+`"`) + columns[adapter.CodingMembershipColumn] = groupColumn{ + name: adapter.CodingMembershipColumn, alias: membershipAlias, expr: membershipSelect, + ref: `"` + membershipAlias + `"`, jsonb: true, + } + } else if _, declared := facts[adapter.CodingMembershipColumn]; declared { + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: the related route declares a Coding membership column without a Coding selector", + ErrUnsupportedPostgresPlan) + } + + routeHasItem := route.alias + ".term_id IS NOT NULL" + itemPresent := "false" + ordinalRead := "NULL::int" + codingHasItem := "" + if adapter.CodingUnnest != nil { + codingHasItem = codingAlias + ".rn IS NOT NULL" + itemPresent = "(" + routeHasItem + " AND " + codingHasItem + ")" + ordinalRead = "(" + codingAlias + ".rn - 1)::int" + } + keyArgs := make([]string, 0, 4) + for _, member := range []struct { + name string + value string + }{ + {"root_id", anchorRead}, + {"terminal_id", route.alias + ".term_id"}, + {"item_present", itemPresent}, + {"ordinal", ordinalRead}, + } { + bind := r.textParameter(member.name, "text:occurrence:"+member.name) + keyArgs = append(keyArgs, "$"+strconv.Itoa(bind)+"::text, "+toRelatedJSONBMember(member.value)) + } + occurrenceSelect := "jsonb_build_object(" + strings.Join(keyArgs, ", ") + ")" + occurrenceAlias := prefix + "k" + adapterSelects = append(adapterSelects, occurrenceSelect+` AS "`+occurrenceAlias+`"`) + columns[adapter.OccurrenceKeyColumn] = groupColumn{ + name: adapter.OccurrenceKeyColumn, alias: occurrenceAlias, expr: occurrenceSelect, + ref: `"` + occurrenceAlias + `"`, jsonb: true, + } + + hasItemAlias := prefix + "h" + adapterSelects = append(adapterSelects, + "(CASE WHEN "+itemPresent+" THEN true ELSE false END) AS "+`"`+hasItemAlias+`"`) + routeHasItemAlias := prefix + "r" + adapterSelects = append(adapterSelects, + "(CASE WHEN "+routeHasItem+" THEN true ELSE false END) AS "+`"`+routeHasItemAlias+`"`) + rankOrder := anchorRead + ` COLLATE "C" ASC NULLS FIRST, ` + route.alias + `.term_id COLLATE "C" ASC NULLS FIRST` + if adapter.CodingUnnest != nil { + rankOrder += ", " + codingAlias + ".rn ASC NULLS FIRST" + } + rankAlias := prefix + "n" + adapterSelects = append(adapterSelects, + "ROW_NUMBER() OVER (ORDER BY "+rankOrder+") - 1 AS "+`"`+rankAlias+`"`) + + where := []string{} + if adapter.RelatedRecords.EmptyPolicy == ir.PhysicalUnnestError { + where = append(where, "(CASE WHEN "+routeHasItem+" THEN 1 ELSE "+guardCast(stageExpandEmptyTag)+" END) > 0") + } + if adapter.CodingUnnest != nil { + switch adapter.CodingEmptyPolicy { + case ir.PhysicalUnnestPreserveParent: + case ir.PhysicalUnnestExclude: + where = append(where, "(CASE WHEN NOT ("+routeHasItem+") OR "+codingHasItem+" THEN 1 ELSE 0 END) > 0") + case ir.PhysicalUnnestError: + where = append(where, "(CASE WHEN NOT ("+routeHasItem+") OR "+codingHasItem+" THEN 1 ELSE "+ + guardCast(stageExpandEmptyTag)+" END) > 0") + default: + return stageRelatedOccurrenceRegion{}, fmt.Errorf("%w: related Coding empty policy %q has no registered PostgreSQL form", + ErrUnsupportedPostgresPlan, string(adapter.CodingEmptyPolicy)) + } + } + + innerSelect := "SELECT " + strings.Join(adapterSelects, ", ") + " FROM (SELECT 1) AS " + unitAlias + + " " + route.joinKeyword + " (" + route.selectText + ") AS " + route.alias + " ON TRUE" + if adapter.CodingUnnest != nil { + innerSelect += " LEFT JOIN LATERAL " + codingSelect + " AS " + codingAlias + " ON TRUE" + } + if len(where) > 0 { + innerSelect += " WHERE " + strings.Join(where, " AND ") + } + outerJoin := "LEFT JOIN LATERAL (" + innerSelect + ") AS " + relAlias + " ON TRUE" + if adapter.RelatedRecords.EmptyPolicy == ir.PhysicalUnnestExclude { + outerJoin = "INNER JOIN LATERAL (" + innerSelect + ") AS " + relAlias + " ON TRUE" + } + + sourceText := "SELECT " + stageTerminalAlias + ".*, " + relAlias + ".* FROM " + + stageRelationSource(relation) + " " + outerJoin + + " ORDER BY " + anchorRead + " ASC, " + relAlias + `."` + rankAlias + `" ASC` + return stageRelatedOccurrenceRegion{ + sourceText: sourceText, + columns: columns, + orderTerms: []string{anchorRead + ` ASC`, stageTerminalAlias + `."` + rankAlias + `" ASC`}, + }, nil +} + +type stageRelatedRoute struct { + selectText string + joinKeyword string + alias string + variable string + targetType string +} + +func (r *planRenderer) renderStageRelatedRoute( + plan ir.PhysicalPlan, + stage ir.PhysicalConstructionStage, + anchorRead string, +) (stageRelatedRoute, error) { + adapter := stage.Group.RelatedOccurrence + related := adapter.RelatedRecords + subplan := related.RelatedRecords + if len(subplan.Captures) != 1 || subplan.Captures[0] != stage.InputRowVariable { + return stageRelatedRoute{}, fmt.Errorf("%w: the related route subplan must capture exactly the Group stage's input row %q, got %v", + ErrUnsupportedPostgresPlan, stage.InputRowVariable, subplan.Captures) + } + if subplan.Sort == nil || subplan.Sort.Variable == "" || len(subplan.Sort.Path) != 1 || + subplan.Sort.Path[0] != "_id" || !subplan.Unique { + return stageRelatedRoute{}, fmt.Errorf("%w: the related route subplan must be UNIQUE and sorted by the reached target's {_id} (recipe_construction_related_expand.go:215-217)", + ErrUnsupportedPostgresPlan) + } + if subplan.DistinctBy != nil { + return stageRelatedRoute{}, fmt.Errorf("%w: the related route subplan cannot carry a DISTINCT BY modifier", ErrUnsupportedPostgresPlan) + } + if len(subplan.Operations) == 0 { + return stageRelatedRoute{}, fmt.Errorf("%w: the related route subplan has no operations", ErrUnsupportedPostgresPlan) + } + + anchorOp := subplan.Operations[0] + if anchorOp.Kind != ir.PhysicalCollectionScanOp || anchorOp.CollectionScan == nil { + return stageRelatedRoute{}, fmt.Errorf("%w: the related route subplan must open with the root-collection anchor scan the lowering emits, got kind %q", + ErrUnsupportedPostgresPlan, string(anchorOp.Kind)) + } + scan := anchorOp.CollectionScan + if scan.Variable == "" || scan.CollectionBindKey == "" { + return stageRelatedRoute{}, fmt.Errorf("%w: the related route anchor scan must name its variable and root collection bind", + ErrUnsupportedPostgresPlan) + } + if bound, ok := plan.BindVars[scan.CollectionBindKey].(string); !ok || bound != related.AnchorResourceType { + return stageRelatedRoute{}, fmt.Errorf("%w: the related route anchor collection bind %q must be bound to the anchor resource type %q", + ErrUnsupportedPostgresPlan, scan.CollectionBindKey, related.AnchorResourceType) + } + index := 1 + if index >= len(subplan.Operations) { + return stageRelatedRoute{}, fmt.Errorf("%w: the related route anchor scan must be followed by its root-key equality filter", + ErrUnsupportedPostgresPlan) + } + anchorFilter := subplan.Operations[index] + if anchorFilter.Kind != ir.PhysicalFilterOp || anchorFilter.Filter == nil || + anchorFilter.Filter.Expression != nil || anchorFilter.Filter.Predicate.Right == nil { + return stageRelatedRoute{}, fmt.Errorf("%w: the related route anchor filter must be the typed Predicate form the lowering emits", + ErrUnsupportedPostgresPlan) + } + predicate := anchorFilter.Filter.Predicate + if predicate.Operator != "EQUALS" || predicate.Quantifier != "" || predicate.Right == nil || + predicate.Left.Variable != scan.Variable || len(predicate.Left.Path) != 1 || predicate.Left.Path[0] != "_key" || + predicate.Right.Variable != stage.InputRowVariable || len(predicate.Right.Path) != 1 || + predicate.Right.Path[0] != related.AnchorColumnID { + return stageRelatedRoute{}, fmt.Errorf("%w: the related route anchor filter must be the exact root-key equality %s._key == inputRow[%q] the lowering emits", + ErrUnsupportedPostgresPlan, scan.Variable, related.AnchorColumnID) + } + index++ + + resource := r.storage.Resource + anchorAlias := "a" + strconv.Itoa(r.nextTraversalSequence()) + anchorTypeIndex := r.textParameter(related.AnchorResourceType, "text:"+related.AnchorResourceType) + anchorWhere := r.scopedResourcePredicate(anchorAlias, anchorTypeIndex) + rows := map[string]rowScope{scan.Variable: {alias: anchorAlias, resourceType: related.AnchorResourceType}} + + if err := matchCanonicalScopeBlock(subplan.Operations, &index, []string{scan.Variable}); err != nil { + return stageRelatedRoute{}, fmt.Errorf("%w: related route anchor scope block: %s", ErrUnsupportedPostgresPlan, err.Error()) + } + + fromParts := []string{"FROM " + resource.Identifier().SQLText() + " AS " + anchorAlias} + var conjuncts []string + currentVar, currentAlias, currentType := scan.Variable, anchorAlias, related.AnchorResourceType + for hopIndex, hop := range related.Route { + if index >= len(subplan.Operations) { + return stageRelatedRoute{}, fmt.Errorf("%w: related route hop %d is missing its operation", ErrUnsupportedPostgresPlan, hopIndex) + } + operation := subplan.Operations[index] + if operation.Kind != ir.PhysicalTraversalOp || operation.Traversal == nil { + return stageRelatedRoute{}, fmt.Errorf("%w: related route hop %d has kind %q; the route is exactly its canonical hops", + ErrUnsupportedPostgresPlan, hopIndex, string(operation.Kind)) + } + traversal := *operation.Traversal + if traversal.Strategy == ir.PhysicalTraversalEndpointLookup { + if strings.TrimSpace(traversal.EndpointField) == "" || strings.TrimSpace(traversal.EndpointJoinField) == "" { + return stageRelatedRoute{}, fmt.Errorf("%w: related route hop %d endpoint-lookup traversal payload is incomplete (the planner always names EndpointField and EndpointJoinField, lower/traversal.go:143-147)", + ErrUnsupportedPostgresPlan, hopIndex) + } + traversal.Strategy = ir.PhysicalTraversalNative + } + if traversal.SourceVariable != currentVar { + return stageRelatedRoute{}, fmt.Errorf("%w: related route hop %d must anchor on %q, got source %q", + ErrUnsupportedPostgresPlan, hopIndex, currentVar, traversal.SourceVariable) + } + source, ok := rows[currentVar] + if !ok { + return stageRelatedRoute{}, fmt.Errorf("%w: related route hop %d anchors on %q, which binds no readable row", + ErrUnsupportedPostgresPlan, hopIndex, currentVar) + } + verified, err := r.validateTraversalContract(plan, traversal, source) + if err != nil { + return stageRelatedRoute{}, fmt.Errorf("related route hop %d: %w", hopIndex, err) + } + if hop.Relationship != verified.label || hop.ToResourceType != verified.targetType { + return stageRelatedRoute{}, fmt.Errorf("%w: related route hop %d names relationship %q and type %q, but its traversal payload carries label %q and type %q", + ErrUnsupportedPostgresPlan, hopIndex, hop.Relationship, hop.ToResourceType, verified.label, verified.targetType) + } + if strings.ToUpper(string(traversal.Direction)) != hop.StorageDirection { + return stageRelatedRoute{}, fmt.Errorf("%w: related route hop %d storage direction changed", ErrUnsupportedPostgresPlan, hopIndex) + } + seq := r.nextTraversalSequence() + edgeAlias := "e" + strconv.Itoa(seq) + targetAlias := "x" + strconv.Itoa(seq) + edgeWhere, targetOn := r.edgeOccurrencePredicates(&verified, edgeAlias, targetAlias, currentAlias) + fromParts = append(fromParts, + "INNER JOIN "+r.storage.Edge.Identifier().SQLText()+" AS "+edgeAlias+" ON "+edgeWhere, + "INNER JOIN "+resource.Identifier().SQLText()+" AS "+targetAlias+" ON "+targetOn) + rows[traversal.TargetVariable] = rowScope{alias: targetAlias, resourceType: verified.targetType} + currentVar, currentAlias, currentType = traversal.TargetVariable, targetAlias, verified.targetType + index++ + if err := matchCanonicalScopeBlock(subplan.Operations, &index, + []string{traversal.EdgeVariable, traversal.TargetVariable}); err != nil { + return stageRelatedRoute{}, fmt.Errorf("%w: related route hop %d scope block: %s", ErrUnsupportedPostgresPlan, hopIndex, err.Error()) + } + } + if currentType != related.TargetResourceType { + return stageRelatedRoute{}, fmt.Errorf("%w: the related route reaches %q, not its declared target type %q", + ErrUnsupportedPostgresPlan, currentType, related.TargetResourceType) + } + + for ; index < len(subplan.Operations); index++ { + operation := subplan.Operations[index] + if operation.Kind != ir.PhysicalFilterOp || operation.Filter == nil || operation.Filter.Expression == nil { + return stageRelatedRoute{}, fmt.Errorf("%w: related route operation %d has kind %q; after its hops the route carries only typed contributor FILTERs (shared subsets, nested sets and unnests, keyed lookups, document lookups, and non-canonical scope operations are capability gaps)", + ErrUnsupportedPostgresPlan, index, string(operation.Kind)) + } + fragment, err := r.renderPredicateFragmentScoped(plan, *operation.Filter.Expression, rows, true) + if err != nil { + return stageRelatedRoute{}, fmt.Errorf("related route contributor filter: %w", err) + } + conjuncts = append(conjuncts, fragment.sql) + } + + logicalID := resource.MustColumn("logical_id").SQLText() + payload := resource.MustColumn("payload").SQLText() + terminalRead := currentAlias + "." + logicalID + selectText := "SELECT DISTINCT ON (" + terminalRead + ` COLLATE "C") ` + + terminalRead + ` AS term_id, ` + currentAlias + "." + payload + ` AS doc` + + " " + strings.Join(fromParts, " ") + anchorPredicate := anchorWhere + " AND " + anchorAlias + "." + logicalID + " = " + anchorCorrelationPlaceholder + if !strings.Contains(anchorPredicate, anchorCorrelationPlaceholder) { + return stageRelatedRoute{}, fmt.Errorf("%w: internal error: the related route anchor equality lost its correlation marker", + ErrUnsupportedPostgresPlan) + } + anchorPredicate = strings.ReplaceAll(anchorPredicate, anchorCorrelationPlaceholder, anchorRead) + routeWhere := append([]string{anchorPredicate}, conjuncts...) + selectText += " WHERE " + strings.Join(routeWhere, " AND ") + + ` ORDER BY ` + terminalRead + ` COLLATE "C"` + + joinKeyword := "LEFT JOIN LATERAL" + if adapter.RelatedRecords.EmptyPolicy == ir.PhysicalUnnestExclude { + joinKeyword = "INNER JOIN LATERAL" + } + return stageRelatedRoute{ + selectText: selectText, + joinKeyword: joinKeyword, + alias: routeAlias, + variable: currentVar, + targetType: currentType, + }, nil +} + +func (r *planRenderer) stageRelatedCodingArrayExpression( + base string, + name string, + extract *ir.PhysicalExtract, +) (string, error) { + if extract.Selector.Filter != nil || len(extract.Fallbacks) != 0 || extract.Distinct || + extract.Prepared != nil || extract.UnitNormalization != nil { + return "", fmt.Errorf("%w: Coding column %q uses a richer selector form (filter, fallbacks, DISTINCT, prepared, or unit normalization) that is not registered", + ErrUnsupportedPostgresPlan, name) + } + if len(extract.Source.Path) != 1 || extract.Source.Path[0] != "payload" { + return "", fmt.Errorf("%w: Coding column %q must read the reached document at [\"payload\"], got %v", + ErrUnsupportedPostgresPlan, name, extract.Source.Path) + } + if extract.ExecutionMode != "" && extract.ExecutionMode != ir.PhysicalSelectorGeneric && + extract.ExecutionMode != ir.PhysicalSelectorDirectScalar && + extract.ExecutionMode != ir.PhysicalSelectorConditionalArray { + return "", fmt.Errorf("%w: Coding column %q selector execution mode %q is unknown", + ErrUnsupportedPostgresPlan, name, string(extract.ExecutionMode)) + } + iterate := -1 + for stepIndex, step := range extract.Selector.Steps { + if step.Index != nil { + return "", fmt.Errorf("%w: Coding column %q: indexed steps are the unit slice's registered reading, not the Coding gathering", + ErrUnsupportedPostgresPlan, name) + } + if !payloadPathSegmentPattern.MatchString(step.Field) { + return "", fmt.Errorf("%w: Coding column %q: selector step %q is not renderer-safe", + ErrUnsupportedPostgresPlan, name, step.Field) + } + if step.Iterate { + if iterate >= 0 { + return "", fmt.Errorf("%w: Coding column %q: repeated steps beyond one iterate step are the later array-rendering capability", + ErrUnsupportedPostgresPlan, name) + } + iterate = stepIndex + } + } + if iterate < 0 { + return "", fmt.Errorf("%w: Coding column %q has no iterated selector step; a Coding array needs exactly one [] step", + ErrUnsupportedPostgresPlan, name) + } + prefixSteps := make([]string, 0, iterate+1) + for _, step := range extract.Selector.Steps[:iterate+1] { + prefixSteps = append(prefixSteps, step.Field) + } + tailSteps := make([]string, 0, len(extract.Selector.Steps)-iterate-1) + for _, step := range extract.Selector.Steps[iterate+1:] { + tailSteps = append(tailSteps, step.Field) + } + arr := base + if len(prefixSteps) > 0 { + var err error + arr, err = r.documentPathFromExpression(arr, prefixSteps) + if err != nil { + return "", fmt.Errorf("Coding column %q: %w", name, err) + } + } + leafTail := "{}" + if len(tailSteps) > 0 { + leafTail = "{" + strings.Join(mapSlice(tailSteps, arrayLiteralElement), ",") + "}" + } + leafTailParameter := r.textParameter(leafTail, "path:"+leafTail) + leaf := "e #> $" + strconv.Itoa(leafTailParameter) + "::text[]" + return "(CASE WHEN jsonb_typeof(" + arr + ") = 'array' THEN COALESCE((SELECT jsonb_agg(" + leaf + + ") FROM jsonb_array_elements(" + arr + ") AS e WHERE " + leaf + " IS NOT NULL), '[]'::jsonb) ELSE '[]'::jsonb END)", nil +} + +func validateStageRelatedOccurrenceAdapter(plan ir.PhysicalPlan, stage ir.PhysicalConstructionStage) error { + adapter := stage.Group.RelatedOccurrence + related := adapter.RelatedRecords + if related.AnchorKind != "root" { + return fmt.Errorf("%w: the related route anchor kind %q has no registered PostgreSQL contract (the root-contributor set is production-unreachable, L-13.6; an active related-record anchor is a related-route capability)", + ErrUnsupportedPostgresPlan, related.AnchorKind) + } + if related.AnchorColumnID != "_key" || related.ParentIdentityColumn != "_key" || related.RootContributorColumn != "" { + return fmt.Errorf("%w: the related route must anchor on the materialized root _key (recipe_construction_related_expand.go:48-54), got anchor %q with contributor set %q", + ErrUnsupportedPostgresPlan, related.AnchorColumnID, related.RootContributorColumn) + } + if related.Multiplicity != "DISTINCT_TARGET_PER_INPUT" { + return fmt.Errorf("%w: the related route multiplicity %q has no registered PostgreSQL contract (only one DISTINCT target per input row renders)", + ErrUnsupportedPostgresPlan, related.Multiplicity) + } + if adapter.RouteID == "" || related.RouteID != adapter.RouteID { + return fmt.Errorf("%w: the related route adapter and its expansion must name the same non-empty route", ErrUnsupportedPostgresPlan) + } + if adapter.RouteHasItemVariable == "" || adapter.OccurrenceKeyColumn == "" { + return fmt.Errorf("%w: the related route adapter lacks its has-item witness or its occurrence key column", ErrUnsupportedPostgresPlan) + } + if len(related.Route) == 0 { + return fmt.Errorf("%w: the related route must carry at least one hop", ErrUnsupportedPostgresPlan) + } + if bindStringVar(plan.BindVars, related.ConstructionIDBindKey) == "" { + return fmt.Errorf("%w: the related route expansion construction ID bind %q must be a non-empty string", + ErrUnsupportedPostgresPlan, related.ConstructionIDBindKey) + } + if related.AnchorNodeID == "" || related.AnchorResourceType == "" || + related.TargetNodeID == "" || related.TargetResourceType == "" { + return fmt.Errorf("%w: the related route must name its anchor and target nodes and resource types", ErrUnsupportedPostgresPlan) + } + if related.RelatedRecordsVariable == "" || related.IndexVariable == "" || + related.ItemVariable == "" || related.IdentityVariable == "" { + return fmt.Errorf("%w: the related route expansion must name its subplan, index, item, and identity variables", + ErrUnsupportedPostgresPlan) + } + if related.EmptyPolicy != ir.PhysicalUnnestPreserveParent && related.EmptyPolicy != ir.PhysicalUnnestExclude && + related.EmptyPolicy != ir.PhysicalUnnestError { + return fmt.Errorf("%w: related route empty policy %q has no registered PostgreSQL form", + ErrUnsupportedPostgresPlan, string(related.EmptyPolicy)) + } + if related.RelatedRecords.Return.Kind != ir.PhysicalObjectExpression || + related.RelatedRecords.Return.Object == nil || len(related.RelatedRecords.Return.Object.Fields) == 0 { + return fmt.Errorf("%w: the related route target projection must be a typed OBJECT expression", ErrUnsupportedPostgresPlan) + } + for hopIndex, hop := range related.Route { + if hop.StepIndex != hopIndex || hop.EdgeID == "" || hop.Relationship == "" || + hop.FromNodeID == "" || hop.FromResourceType == "" || hop.ToNodeID == "" || hop.ToResourceType == "" || + (hop.StorageDirection != "INBOUND" && hop.StorageDirection != "OUTBOUND") || + (hop.MatchMode != "OPTIONAL" && hop.MatchMode != "REQUIRED") { + return fmt.Errorf("%w: related route hop %d is incomplete or discontinuous", ErrUnsupportedPostgresPlan, hopIndex) + } + if hopIndex == 0 { + if hop.FromNodeID != related.AnchorNodeID || hop.FromResourceType != related.AnchorResourceType { + return fmt.Errorf("%w: related route hop 0 does not start at the anchor node", ErrUnsupportedPostgresPlan) + } + continue + } + prior := related.Route[hopIndex-1] + if hop.FromNodeID != prior.ToNodeID || hop.FromResourceType != prior.ToResourceType { + return fmt.Errorf("%w: related route hop %d is discontinuous", ErrUnsupportedPostgresPlan, hopIndex) + } + } + last := related.Route[len(related.Route)-1] + if last.ToNodeID != related.TargetNodeID || last.ToResourceType != related.TargetResourceType { + return fmt.Errorf("%w: the related route does not end at its declared target", ErrUnsupportedPostgresPlan) + } + if adapter.CodingSelector == "" { + if adapter.CodingUnnest != nil || adapter.CodingArrayColumn != "" || adapter.CodingMembershipColumn != "" || + adapter.OrdinalSemantics != "" || adapter.CodingEmptyPolicy != "" { + return fmt.Errorf("%w: the related route declares Coding details without a Coding selector", ErrUnsupportedPostgresPlan) + } + return nil + } + if adapter.OrdinalSemantics != stageRelatedCodingOrdinalSemantics { + return fmt.Errorf("%w: related route Coding ordinal semantics %q is not the registered zero-based flattened position", + ErrUnsupportedPostgresPlan, adapter.OrdinalSemantics) + } + if adapter.CodingUnnest == nil || adapter.CodingArrayColumn == "" || adapter.CodingMembershipColumn == "" { + return fmt.Errorf("%w: the related route Coding selector has no matching array column, membership column, or ordinal operation", + ErrUnsupportedPostgresPlan) + } + coding := adapter.CodingUnnest + if coding.OutputVariable == "" || coding.HasItemVariable == "" || coding.Ordinality == "" || + len(coding.Ancestors) != 0 || coding.EmptyPolicy != adapter.CodingEmptyPolicy || + coding.Expression.Kind != ir.PhysicalValueExpression || coding.Expression.Value == nil || + coding.Expression.Value.Variable != related.ItemVariable || + len(coding.Expression.Value.Path) != 1 || coding.Expression.Value.Path[0] != adapter.CodingArrayColumn { + return fmt.Errorf("%w: the related route Coding expansion is not the exact ordinal operation over its own array column (the AQL renderer-shape twin, construction_group.go:386-401)", + ErrUnsupportedPostgresPlan) + } + switch adapter.CodingEmptyPolicy { + case ir.PhysicalUnnestPreserveParent, ir.PhysicalUnnestExclude, ir.PhysicalUnnestError: + return nil + default: + return fmt.Errorf("%w: related Coding empty policy %q has no registered PostgreSQL form", + ErrUnsupportedPostgresPlan, string(adapter.CodingEmptyPolicy)) + } +} + +func identityFieldValue(field classifiedExtractionProjection) *ir.PhysicalValue { + if field.kind == extractionValueExpression && field.projection.Expression != nil { + return field.projection.Expression.Value + } + if field.kind == extractionDirectValue { + return &field.projection.Value + } + return nil +} + +func toRelatedJSONBMember(value string) string { + return "to_jsonb((" + value + "))" +} + +const ( + relAlias = "__loom_rel" + routeAlias = "__loom_route" + unitAlias = "__loom_one" +) + +const anchorCorrelationPlaceholder = "__loom_anchor_key" diff --git a/internal/dataframe/compiler/render/postgres/stage_related_occurrence_p4_test.go b/internal/dataframe/compiler/render/postgres/stage_related_occurrence_p4_test.go new file mode 100644 index 000000000..44f674a92 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/stage_related_occurrence_p4_test.go @@ -0,0 +1,387 @@ +package postgres + +import ( + "errors" + "sort" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" + + pgstore "github.com/calypr/loom/internal/backends/postgres/control" +) + +const ( + sql16roP4Project = "sql16ro-proj" + sql16roP4Generation = "gen-1" + sql16roP4CodingSelector = "valueCodeableConcept.coding[]" + sql16roP4CodingRefusal = "CodingKey group keys are refused at the composition payload gate until their registered arm lands (group.go:444 twin, wave law)" +) + +type sql16roP4Authoring struct { + base recipe.Output + binding recipe.RuntimeBindings + context *lower.CompiledGroupFieldSourceContext +} + +func sql16roP4BaseOutput(project, generation string) (recipe.Output, recipe.RuntimeBindings) { + bindings := recipe.RuntimeBindings{ + Project: project, DatasetGeneration: generation, + AuthResourcePaths: []string{"FHIR/Patient", "FHIR/Observation"}, + } + base := recipe.Output{ + Name: "patient_related_coding_group", RootResourceType: "Patient", + RootOccurrenceID: "patient-root", RowGrain: "patient", + Fields: []recipe.Field{{Name: "patient_id", ColumnID: "patient_id", Expr: recipe.Expression{Select: "root.id"}}}, + } + return base, bindings +} + +func sql16roP4AuthoringFor(t *testing.T, project, generation string) sql16roP4Authoring { + t.Helper() + base, bindings := sql16roP4BaseOutput(project, generation) + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: base.Name, + TranslationVersion: "test", Outputs: []recipe.Output{base}, + } + plan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + t.Fatalf("BuildRecipePlan (base): %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan (base): %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("CompileResolvedRecipePlan (base): %v", err) + } + descriptor, err := lower.DescribeConstructionSourceStage( + compiled.Outputs[0].OutputSchema, base.RootResourceType, &compiled.Outputs[0].Plan) + if err != nil || descriptor.GroupFieldSourceContext == nil { + t.Fatalf("describe Patient source context: descriptor=%#v err=%v", descriptor, err) + } + return sql16roP4Authoring{base: base, binding: bindings, context: descriptor.GroupFieldSourceContext} +} + +func (a sql16roP4Authoring) sql16roP4Source(t *testing.T, valueKind, resourceType, candidateID, canonicalPath, codingSelector string) recipe.ConstructionGroupFieldSource { + t.Helper() + related := &recipe.ConstructionGroupRelatedOccurrenceFieldSource{ + AnchorNodeID: "patient-node", AnchorResourceType: "Patient", + Route: []recipe.ConstructionGroupRelatedRouteStep{{ + StepIndex: 0, FromNodeID: "patient-node", FromResourceType: "Patient", + EdgeID: "patient-observation-edge", Relationship: "subject_Patient", StorageDirection: "INBOUND", MatchMode: "OPTIONAL", + ToNodeID: "observation-node", ToResourceType: "Observation", + }}, + TargetNodeID: "observation-node", TargetResourceType: "Observation", + ResourceType: resourceType, CandidateID: candidateID, CanonicalPath: canonicalPath, + EmptyPolicy: recipe.ExpansionPreserveParent, + Multiplicity: recipe.ConstructionGroupRelatedMultiplicityDistinctTargetPerInput, + } + if valueKind == "CODING_IDENTITY" { + related.ResourceType = "Coding" + related.CodingSelector = codingSelector + related.CodingEmptyPolicy = recipe.ConstructionGroupCodingEmptyPolicyGroup + related.OrdinalSemantics = recipe.ConstructionGroupRelatedOrdinalSemanticsZeroBasedPosition + } + source := recipe.ConstructionGroupFieldSource{ + Kind: "BINDING", ValueKind: valueKind, + Input: &recipe.ConstructionGroupFieldSourceInput{Kind: "RELATED_OCCURRENCE", RelatedOccurrence: related}, + } + bound, err := lower.BindGroupRelatedOccurrenceSource(a.context, source) + if err != nil { + t.Fatalf("bind %s related Group source: %v", valueKind, err) + } + return bound +} + +func (a sql16roP4Authoring) sql16roP4Compiled(t *testing.T, withCoding bool) lower.CompiledRecipeOutput { + t.Helper() + status := a.sql16roP4Source(t, "SCALAR", "string", "observation-status", "status", "") + keys := []recipe.ConstructionGroupKey{ + {OutputColumnID: "status_group_id", FieldSource: &status}, + } + outputs := []recipe.StageColumn{ + {ID: "status_group_id", Name: "status_group", Type: "string"}, + } + if withCoding { + coding := a.sql16roP4Source(t, "CODING_IDENTITY", "Coding", "", "", sql16roP4CodingSelector) + keys = append(keys, recipe.ConstructionGroupKey{OutputColumnID: "coding_group_id", FieldSource: &coding}) + outputs = append(outputs, recipe.StageColumn{ID: "coding_group_id", Name: "coding_group", Type: "string"}) + } + outputs = append(outputs, recipe.StageColumn{ID: "row_count_id", Name: "row_count", Type: "integer"}) + output := a.base + output.Name = "patient_related_coding_group_p4" + output.Construction = &recipe.Construction{ + Version: 1, SourceColumns: []recipe.StageColumn{{ID: "patient_id", Name: "patient_id"}}, + Steps: []recipe.ConstructionStep{{ + ID: "group_related", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionGroupOp, Group: &recipe.ConstructionGroup{ + ConstructionID: "group_related_id", MissingKeyPolicy: recipe.ConstructionGroupMissingKeyGroup, + Keys: keys, + Aggregates: []recipe.ConstructionGroupAggregate{{Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "row_count_id"}}, + }}, + Outputs: outputs, + }}, + } + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: output.Name, + TranslationVersion: "test", Outputs: []recipe.Output{output}, + } + plan, err := semantic.BuildRecipePlan(bundle, a.binding) + if err != nil { + t.Fatalf("BuildRecipePlan (group): %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, a.binding.Project, a.binding.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan (group): %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("CompileResolvedRecipePlan (group): %v", err) + } + return compiled.Outputs[0] +} + +func TestSQL16RelatedOccurrenceP4CompiledChannel(t *testing.T) { + output := sql16roP4AuthoringFor(t, sql16roP4Project, sql16roP4Generation).sql16roP4Compiled(t, true) + if err := output.Plan.Validate(); err != nil { + t.Fatalf("compiled plan Validate: %v", err) + } + sequence := output.Plan.StageSequence + if sequence == nil || len(sequence.Stages) != 1 { + t.Fatalf("compiled plan is not the single-stage GROUP shape: %#v", sequence) + } + stage := sequence.Stages[0] + if stage.Kind != ir.PhysicalStageGroupOp || stage.Group == nil || stage.Group.RelatedOccurrence == nil { + t.Fatalf("compiled stage is not a RelatedOccurrence GROUP: kind=%s", stage.Kind) + } + adapter := stage.Group.RelatedOccurrence + if adapter.RouteID == "" || adapter.OccurrenceKeyColumn == "" || adapter.CodingMembershipColumn == "" { + t.Fatalf("compiler identity columns missing: %#v", adapter) + } + if adapter.CodingSelector != sql16roP4CodingSelector || + adapter.CodingEmptyPolicy != ir.PhysicalUnnestPreserveParent || + adapter.OrdinalSemantics != stageRelatedCodingOrdinalSemantics { + t.Fatalf("coding legs drifted: selector=%q empty=%v ordinal=%q", + adapter.CodingSelector, adapter.CodingEmptyPolicy, adapter.OrdinalSemantics) + } + if len(adapter.ScalarProjections) != 1 || adapter.ScalarProjections[0].InputColumn == "" { + t.Fatalf("scalar projection set = %#v, want exactly the status leg", adapter.ScalarProjections) + } + if len(adapter.RelatedRecords.Route) != 1 || + adapter.RelatedRecords.Route[0].Relationship != "subject_Patient" || + adapter.RelatedRecords.Route[0].MatchMode != "OPTIONAL" { + t.Fatalf("route facts drifted: %#v", adapter.RelatedRecords.Route) + } + prior := sequence.SourceColumns + extra := stage.InputColumns[len(prior):] + if len(stage.InputColumns) != len(prior)+3 { + t.Fatalf("extension width = %d over prior %d", len(stage.InputColumns), len(prior)) + } + allowed := map[string]bool{ + adapter.OccurrenceKeyColumn: true, + adapter.CodingMembershipColumn: true, + adapter.ScalarProjections[0].InputColumn: true, + } + for _, column := range extra { + if !column.Internal || column.ID != column.Name || !allowed[column.Name] { + t.Fatalf("extension column %q is not a compiler-minted adapter column", column.Name) + } + delete(allowed, column.Name) + } + if len(allowed) != 0 { + t.Fatalf("adapter column %v absent from the stage input extension", allowed) + } + bound := 0 + for _, key := range stage.Group.Keys { + if key.FieldSource != nil && key.FieldSource.Kind == ir.PhysicalStageGroupFieldSourceRelatedOccurrence && + key.FieldSource.RelatedOccurrence != nil && key.FieldSource.RelatedOccurrence.RouteID != "" && + key.FieldSource.RelatedOccurrence.SourceShapeID != "" { + bound++ + } + } + if codingKey := stage.Group.CodingKey; codingKey != nil && codingKey.RelatedSource != nil && + codingKey.RelatedSource.RouteID != "" && codingKey.RelatedSource.SourceShapeID != "" { + bound++ + } + if bound != 2 { + t.Fatalf("bound related field sources = %d, want 2 (status key + coding key)", bound) + } +} + +func TestSQL16RelatedOccurrenceP4DoorRender(t *testing.T) { + output := sql16roP4AuthoringFor(t, sql16roP4Project, sql16roP4Generation).sql16roP4Compiled(t, false) + rendered, err := RenderPhysicalPlan(ir.ClonePhysicalPlan(output.Plan), mustBinding(t)) + if err != nil { + t.Fatalf("door render of the compiled plan: %v", err) + } + for _, want := range []string{relAlias, routeAlias, "LEFT JOIN", `ORDER BY`} { + if !strings.Contains(rendered.SQL, want) { + t.Fatalf("door statement is missing %q:\n%s", want, rendered.SQL) + } + } + if strings.Contains(rendered.SQL, "__loom_group_related_coding_membership") { + t.Fatalf("a SCALAR-only plan must not grow the coding membership leg:\n%s", rendered.SQL) + } + for _, leaked := range []string{sql16roP4Project, sql16roP4Generation, "subject_Patient", "observation-node", "group_related_id"} { + if strings.Contains(rendered.SQL, leaked) { + t.Fatalf("door statement leaked plan data %q:\n%s", leaked, rendered.SQL) + } + } +} + +func TestSQL16RelatedOccurrenceP4CodingKeyGateRefusal(t *testing.T) { + output := sql16roP4AuthoringFor(t, sql16roP4Project, sql16roP4Generation).sql16roP4Compiled(t, true) + rendered, err := RenderPhysicalPlan(ir.ClonePhysicalPlan(output.Plan), mustBinding(t)) + if err == nil { + t.Fatalf("the composition gate must refuse the route-sourced CodingKey, got:\n%s", rendered.SQL) + } + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("the refusal must be the typed door refusal, got: %v", err) + } + if !strings.Contains(err.Error(), sql16roP4CodingRefusal) { + t.Fatalf("refusal drifted from the registered twin text: %v", err) + } + if rendered.SQL != "" || len(rendered.Args) != 0 { + t.Fatalf("a refused render must produce no statement: sql=%q args=%d", rendered.SQL, len(rendered.Args)) + } + if _, err := RenderPhysicalPlan(ir.ClonePhysicalPlan( + sql16roP4AuthoringFor(t, sql16roP4Project, sql16roP4Generation).sql16roP4Compiled(t, false).Plan), mustBinding(t)); err != nil { + t.Fatalf("the registered SCALAR shape must render while the CodingKey refuses: %v", err) + } +} + +func sql16roP4Seed(t *testing.T, svc *pgir02Service, project, label string, statuses []string) { + t.Helper() + tag := p4Tag(project) + svc.insertResource(t, svc.schema, project, sql16roP4Generation, "Patient", tag+"1", "FHIR/Patient", map[string]any{"gender": "female"}) + svc.insertResource(t, svc.schema, project, sql16roP4Generation, "Patient", tag+"2", "FHIR/Patient", map[string]any{"gender": "male"}) + for index, status := range statuses { + observation := tag + "o" + string(rune('1'+index)) + svc.insertResource(t, svc.schema, project, sql16roP4Generation, "Observation", observation, "FHIR/Observation", map[string]any{"status": status}) + svc.insertEdge(t, pgstore.EdgeOccurrenceRow{ + Project: project, DatasetGeneration: sql16roP4Generation, AuthResourcePath: "FHIR/Observation", + Label: label, SourceResourceType: "Observation", SourceLogicalID: observation, + TargetResourceType: "Patient", TargetLogicalID: tag + "1", SourcePath: "subject[0]", SourceOrdinal: int64(index), + }) + } +} + +func p4Tag(project string) string { + if strings.HasSuffix(project, "-b") { + return "b-" + } + return "a-" +} + +func sql16roP4LivePlan(t *testing.T, project string) (lower.CompiledRecipeOutput, string) { + t.Helper() + output := sql16roP4AuthoringFor(t, project, sql16roP4Generation).sql16roP4Compiled(t, false) + label, ok := output.Plan.BindVars["related_expand_0_hop_1_label"].(string) + if !ok || label == "" { + t.Fatalf("compiled plan lost its route label bind: %#v", output.Plan.BindVars) + } + return output, label +} + +func TestSQL16RelatedOccurrenceP4LiveRows(t *testing.T) { + svc := newOwnedService(t, 0, "verify_sql16_g3_p4") + output, label := sql16roP4LivePlan(t, sql16roP4Project) + sql16roP4Seed(t, svc, sql16roP4Project, label, []string{"final", "final", "preliminary"}) + rendered := coded19RenderSvc(t, svc, ir.ClonePhysicalPlan(output.Plan)) + rows := svc.runRendered(t, rendered) + + type tuple struct{ status, count string } + got := make([]tuple, 0, len(rows)) + for _, row := range rows { + if len(row) != 3 { + t.Fatalf("row = %#v, want 3 cells (group columns + __loom_row_id identity)", row) + } + keyJSON := "null" + if row[0] != "" { + keyJSON = sql16JSONString(row[0]) + } + if want := sql16GroupIdentity("group_related_id", "status_group_id", keyJSON); row[2] != want { + t.Fatalf("runtime-key identity = %q, want %q", row[2], want) + } + got = append(got, tuple{row[0], row[1]}) + } + sort.Slice(got, func(i, j int) bool { return got[i].status < got[j].status }) + if len(got) != 3 { + t.Fatalf("row multiset = %v, want 3 rows", got) + } + var pad string + counts := map[string]string{} + distinct := map[string]bool{} + for _, row := range got { + if row.status == "" { + pad = row.count + continue + } + counts[row.status] = row.count + distinct[row.status] = true + } + if pad != "1" { + t.Fatalf("the preserved-parent grand-total group must count 1, got %q", pad) + } + if len(counts) != 2 { + t.Fatalf("the three observations must group to two distinct status identities: %v", got) + } + sum := 0 + for _, count := range counts { + if count != "1" && count != "2" { + t.Fatalf("unexpected group count %q in %v", count, got) + } + sum += int(count[0] - '0') + } + if sum != 3 { + t.Fatalf("group counts must partition the three reachable observations: %v", got) + } +} + +func TestSQL16RelatedOccurrenceP4LiveShareLaw(t *testing.T) { + svc := newOwnedService(t, 0, "verify_sql16_g3_p4") + const projectA, projectB = "sql16ro-share-a", "sql16ro-share-b" + planA, labelA := sql16roP4LivePlan(t, projectA) + planB, labelB := sql16roP4LivePlan(t, projectB) + sql16roP4Seed(t, svc, projectA, labelA, []string{"final", "final"}) + sql16roP4Seed(t, svc, projectB, labelB, []string{"preliminary", "amended"}) + + render := func(output lower.CompiledRecipeOutput) [][]string { + return svc.runRendered(t, coded19RenderSvc(t, svc, ir.ClonePhysicalPlan(output.Plan))) + } + rowsA, rowsB := render(planA), render(planB) + identitiesOf := func(rows [][]string) []string { + var out []string + for _, row := range rows { + if row[0] != "" { + out = append(out, row[0]) + } + } + sort.Strings(out) + return out + } + idsA, idsB := identitiesOf(rowsA), identitiesOf(rowsB) + if len(idsA) != 1 || len(idsB) != 2 { + t.Fatalf("each partition must group only its own route: A=%v B=%v (A=%v B=%v)", idsA, idsB, rowsA, rowsB) + } + var countsA []string + for _, row := range rowsA { + countsA = append(countsA, row[1]) + } + sort.Strings(countsA) + if strings.Join(countsA, ",") != "1,2" { + t.Fatalf("A's multiset = %v, want the final group (2) plus the preserved pad (1)", countsA) + } + for _, identity := range idsA { + for _, other := range idsB { + if identity == other { + t.Fatalf("partitions borrowed one expansion (share violation): %s", identity) + } + } + } +} diff --git a/internal/dataframe/compiler/render/postgres/stage_related_occurrence_production_test.go b/internal/dataframe/compiler/render/postgres/stage_related_occurrence_production_test.go new file mode 100644 index 000000000..a9f68ecec --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/stage_related_occurrence_production_test.go @@ -0,0 +1,476 @@ +package postgres + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + pgstore "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const ( + sql16roLivePriorRelation = "__loom_stage_0" + sql16roDeniedAuth = "FHIR/Denied" + sql16roLiveLabel = "subject_Reference" +) + +type sql16roLiveFixture struct { + svc *pgir02Service +} + +func newSQL16roLiveFixture(t *testing.T) sql16roLiveFixture { + t.Helper() + svc := newOwnedService(t, 0, "verify_sql16v1") + insert := func(resourceType, logicalID, authPath string, payload map[string]any) { + svc.insertResource(t, svc.schema, sql16roProject, sql16roGeneration, resourceType, logicalID, authPath, payload) + } + edge := func(observation, patient, authPath string, ordinal int64, sourcePath string) { + svc.insertEdge(t, pgstore.EdgeOccurrenceRow{ + Project: sql16roProject, DatasetGeneration: sql16roGeneration, AuthResourcePath: authPath, + Label: sql16roLiveLabel, SourceResourceType: "Observation", SourceLogicalID: observation, + TargetResourceType: "Patient", TargetLogicalID: patient, SourcePath: sourcePath, SourceOrdinal: ordinal, + }) + } + + insert("Patient", "p-1", "FHIR/Patient", map[string]any{"gender": "female"}) + insert("Patient", "p-2", "FHIR/Patient", map[string]any{"gender": "male"}) + insert("Patient", "p-denied", sql16roDeniedAuth, map[string]any{"gender": "denied"}) + insert("Observation", "o-1", "FHIR/Observation", map[string]any{ + "status": "final", + "valueCodeableConcept": map[string]any{ + "coding": []any{ + map[string]any{"system": "urn:sys", "code": "c1", "display": "One"}, + map[string]any{"system": "urn:sys", "code": "c2"}, + }, + }, + }) + insert("Observation", "o-3", "FHIR/Observation", map[string]any{"status": "amended"}) + insert("Observation", "o-4", "FHIR/Observation", map[string]any{ + "status": "final", + "valueCodeableConcept": map[string]any{ + "coding": []any{map[string]any{"system": "urn:other", "code": "solo", "display": "Solo"}}, + }, + }) + insert("Observation", "o-denied", sql16roDeniedAuth, map[string]any{ + "status": "final", + "valueCodeableConcept": map[string]any{ + "coding": []any{map[string]any{"system": "urn:denied", "code": "d"}}, + }, + }) + edge("o-1", "p-1", "FHIR/Observation", 0, "subject[0]") + edge("o-1", "p-1", "FHIR/Observation", 1, "subject[1]") + edge("o-3", "p-1", "FHIR/Observation", 0, "subject[0]") + edge("o-4", "p-1", "FHIR/Observation", 0, "subject[0]") + edge("o-denied", "p-1", sql16roDeniedAuth, 0, "subject[0]") + insert("Observation", "o-guard", "FHIR/Observation", map[string]any{ + "status": "final", + "valueCodeableConcept": map[string]any{ + "coding": []any{map[string]any{"system": "urn:guard", "code": "g"}}, + }, + }) + edge("o-guard", "p-denied", "FHIR/Observation", 0, "subject[0]") + return sql16roLiveFixture{svc: svc} +} + +func (f sql16roLiveFixture) priorView(t *testing.T, anchors ...string) { + t.Helper() + quoted := make([]string, 0, len(anchors)) + for _, anchor := range anchors { + quoted = append(quoted, "'"+anchor+"'") + } + svc := f.svc + create := fmt.Sprintf(`CREATE VIEW "%s"."%s" AS SELECT logical_id AS "c0" FROM "%s"."fhir_resource" + WHERE project = '%s' AND dataset_generation = '%s' AND resource_type = 'Patient' + AND logical_id IN (%s)`, + svc.schema, sql16roLivePriorRelation, svc.schema, sql16roProject, sql16roGeneration, + strings.Join(quoted, ",")) + if _, err := svc.iso.Exec(svc.ctx, create); err != nil { + svc.fail("create prior relation stand-in", err) + } + t.Cleanup(func() { + if _, err := svc.iso.Exec(context.WithoutCancel(svc.ctx), + `DROP VIEW IF EXISTS "`+svc.schema+`"."`+sql16roLivePriorRelation+`"`); err != nil { + t.Errorf("drop prior relation stand-in: %v", err) + } + }) +} + +func (f sql16roLiveFixture) renderLive(t *testing.T, plan ir.PhysicalPlan, stage ir.PhysicalConstructionStage, + facts map[string]stageColumnFact, +) (stageRelatedOccurrenceRegion, *planRenderer) { + t.Helper() + binding, err := ir.NewPostgresStorageBinding(f.svc.schema) + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + r := &planRenderer{ + storage: binding, rows: map[string]rowScope{}, + parameters: map[string]int{}, canonicalAuthLets: map[string]string{}, + } + if err := r.loadScope(plan); err != nil { + t.Fatalf("loadScope: %v", err) + } + region, err := r.renderStageGroupRelatedOccurrence(plan, stage, 1, sql16roLivePriorRelation, facts) + if err != nil { + t.Fatalf("renderStageGroupRelatedOccurrence: %v", err) + } + return region, r +} + +func (f sql16roLiveFixture) qualifiedRegionText(t *testing.T, region stageRelatedOccurrenceRegion) string { + t.Helper() + from := "FROM " + sql16roLivePriorRelation + " AS " + stageTerminalAlias + if !strings.Contains(region.sourceText, from) { + t.Fatalf("region does not read the prior relation as %q:\n%s", from, region.sourceText) + } + return strings.Replace(region.sourceText, from, + `FROM "`+f.svc.schema+`"."`+sql16roLivePriorRelation+`" AS `+stageTerminalAlias, 1) +} + +func (f sql16roLiveFixture) regionArgs(r *planRenderer) []any { + args := make([]any, len(r.args)) + for index, parameter := range r.args { + args[index] = parameter.Value + } + return args +} + +func (f sql16roLiveFixture) queryRegion(t *testing.T, region stageRelatedOccurrenceRegion, + r *planRenderer, selectList string, +) [][]string { + t.Helper() + sql := "SELECT " + selectList + " FROM (" + f.qualifiedRegionText(t, region) + ") AS q" + + sql16roOrder(t, region) + return f.svc.witness(t, sql, f.regionArgs(r)...) +} + +func sql16roOrder(t *testing.T, region stageRelatedOccurrenceRegion) string { + t.Helper() + order := make([]string, 0, len(region.orderTerms)) + for _, term := range region.orderTerms { + qualified := strings.Replace(term, stageTerminalAlias+".", "q.", 1) + if qualified == term { + t.Fatalf("order term %q is not a terminal-alias read", term) + } + order = append(order, qualified) + } + return " ORDER BY " + strings.Join(order, ", ") +} + +type sql16roKey struct { + RootID *string `json:"root_id"` + TerminalID *string `json:"terminal_id"` + ItemPresent bool `json:"item_present"` + Ordinal *int `json:"ordinal"` +} + +type sql16roMembership struct { + System *string `json:"system"` + Version *string `json:"version"` + Code *string `json:"code"` + Display *string `json:"display"` +} + +func sql16roParse[T any](t *testing.T, cell string) T { + t.Helper() + var out T + if cell == "" { + return out + } + if err := json.Unmarshal([]byte(cell), &out); err != nil { + t.Fatalf("parse jsonb cell %q: %v", cell, err) + } + return out +} + +func sql16roText(v any) *string { + text, _ := v.(string) + return &text +} + +func sql16roInt(value int) *int { return &value } + +func sql16roShow[T any](value *T) string { + if value == nil { + return "" + } + return fmt.Sprint(*value) +} + +func (k sql16roKey) String() string { + return fmt.Sprintf("root=%s terminal=%s item_present=%v ordinal=%s", + sql16roShow(k.RootID), sql16roShow(k.TerminalID), k.ItemPresent, sql16roShow(k.Ordinal)) +} + +func (m sql16roMembership) String() string { + return fmt.Sprintf("system=%s version=%s code=%s display=%s", + sql16roShow(m.System), sql16roShow(m.Version), sql16roShow(m.Code), sql16roShow(m.Display)) +} + +type wantRow struct { + key sql16roKey + membership sql16roMembership + hasItem string + routeItem string + rank string + status string +} + +func (w wantRow) String() string { + return fmt.Sprintf("{%s | %s | has=%s route=%s rank=%s status=%s}", + w.key, w.membership, w.hasItem, w.routeItem, w.rank, w.status) +} + +func TestSQL16RelatedOccurrenceLiveRouteSourcedCodingKey(t *testing.T) { + fixture := newSQL16roLiveFixture(t) + fixture.priorView(t, "p-1", "p-2", "p-denied") + plan, stage, facts := sql16roFixture(t) + if got := plan.BindVars["ro_label"]; got != sql16roLiveLabel { + t.Fatalf("the seeded edge label %q drifted from the plan's ro_label bind %v", sql16roLiveLabel, got) + } + + region, r := fixture.renderLive(t, plan, stage, facts) + + const selectList = `q."__loom_r1_k"::text, q."__loom_r1_m"::text, q."__loom_r1_h"::text, ` + + `q."__loom_r1_r"::text, q."__loom_r1_n", q."__loom_r1_s0"::text` + rows := fixture.queryRegion(t, region, r, selectList) + + want := []wantRow{ + { + key: sql16roKey{RootID: sql16roText("p-1"), TerminalID: sql16roText("o-1"), ItemPresent: true, Ordinal: sql16roInt(0)}, + membership: sql16roMembership{System: sql16roText("urn:sys"), Code: sql16roText("c1"), Display: sql16roText("One")}, + hasItem: "true", routeItem: "true", rank: "0", status: `"final"`, + }, + { + key: sql16roKey{RootID: sql16roText("p-1"), TerminalID: sql16roText("o-1"), ItemPresent: true, Ordinal: sql16roInt(1)}, + membership: sql16roMembership{System: sql16roText("urn:sys"), Code: sql16roText("c2")}, + hasItem: "true", routeItem: "true", rank: "1", status: `"final"`, + }, + { + key: sql16roKey{RootID: sql16roText("p-1"), TerminalID: sql16roText("o-4"), ItemPresent: true, Ordinal: sql16roInt(0)}, + membership: sql16roMembership{System: sql16roText("urn:other"), Code: sql16roText("solo"), Display: sql16roText("Solo")}, + hasItem: "true", routeItem: "true", rank: "2", status: `"final"`, + }, + { + key: sql16roKey{RootID: sql16roText("p-2"), ItemPresent: false}, + membership: sql16roMembership{}, + hasItem: "false", routeItem: "false", rank: "0", status: "", + }, + { + key: sql16roKey{RootID: sql16roText("p-denied"), ItemPresent: false}, + membership: sql16roMembership{}, + hasItem: "false", routeItem: "false", rank: "0", status: "", + }, + } + if len(rows) != len(want) { + t.Fatalf("row count = %d, want %d:\n%v", len(rows), len(want), rows) + } + for index, row := range rows { + got := wantRow{ + key: sql16roParse[sql16roKey](t, row[0]), + membership: sql16roParse[sql16roMembership](t, row[1]), + hasItem: row[2], routeItem: row[3], rank: row[4], status: row[5], + } + if got.String() != want[index].String() { + t.Fatalf("row %d = %v, want %v", index, got, want[index]) + } + } + + for index, row := range rows { + key := sql16roParse[sql16roKey](t, row[0]) + if key.TerminalID != nil && (*key.TerminalID == "o-denied" || *key.TerminalID == "o-guard") { + t.Fatalf("row %d reached a scope-excluded target %q:\n%s", index, *key.TerminalID, row[0]) + } + if strings.Contains(row[1], "urn:denied") || strings.Contains(row[1], "urn:guard") { + t.Fatalf("row %d carries Coding from a scope-excluded target:\n%s", index, row[1]) + } + } + + o1Items := 0 + for _, row := range rows { + if strings.Contains(row[0], `"terminal_id": "o-1"`) { + o1Items++ + } + } + if o1Items != 2 { + t.Fatalf("two edge occurrences to one target must dedup to one reached target with two Coding items, got %d rows", o1Items) + } + + typed := fixture.queryRegion(t, region, r, + `pg_typeof(q."__loom_r1_k"), pg_typeof(q."__loom_r1_m"), pg_typeof(q."__loom_r1_h"), `+ + `pg_typeof(q."__loom_r1_n"), pg_typeof(q."__loom_r1_s0")`) + const wantTypes = "jsonb jsonb boolean bigint jsonb" + if got := strings.Join(typed[0], " "); got != wantTypes { + t.Fatalf("adapter column types = %q, want %q", got, wantTypes) + } + sql16roCheckDriverTypes(t, fixture, region, r) + + witness := fixture.svc.witness(t, ` + WITH anchors AS ( + SELECT logical_id AS root, auth_resource_path AS root_auth FROM "`+fixture.svc.schema+`"."fhir_resource" + WHERE project = $1 AND dataset_generation = $2 AND resource_type = 'Patient' + AND logical_id IN ('p-1','p-2','p-denied')), + reached AS ( + SELECT DISTINCT ON (x.logical_id COLLATE "C") a.root AS root, x.logical_id AS term, x.payload AS doc + FROM anchors a + INNER JOIN "`+fixture.svc.schema+`"."fhir_edge_occurrence" e + ON e.project = $1 AND e.dataset_generation = $2 + AND e.auth_resource_path = ANY($3::text[]) + AND e.label = $4 AND e.target_resource_type = 'Patient' + AND e.target_logical_id = a.root AND e.source_resource_type = 'Observation' + INNER JOIN "`+fixture.svc.schema+`"."fhir_resource" x + ON x.project = $1 AND x.dataset_generation = $2 + AND x.auth_resource_path = ANY($3::text[]) + AND x.resource_type = 'Observation' AND x.logical_id = e.source_logical_id + -- the anchor's own auth legs: a denied anchor reaches nothing, but + -- it is a prior-grain row, so the LEFT JOIN keeps its pad row: the + -- pad branch must NOT re-filter auth, or the guard cell would be + -- comparing different grains. + WHERE a.root_auth = ANY($3::text[]) + ORDER BY x.logical_id COLLATE "C"), + items AS ( + SELECT reached.root AS root, reached.term AS term, (c.ordinal_row - 1) AS ordinal + FROM reached CROSS JOIN LATERAL jsonb_array_elements( + CASE WHEN jsonb_typeof(reached.doc #> '{valueCodeableConcept,coding}'::text[]) = 'array' + THEN reached.doc #> '{valueCodeableConcept,coding}'::text[] ELSE '[]'::jsonb END + ) WITH ORDINALITY AS c(item, ordinal_row) + WHERE jsonb_typeof(c.item) <> 'null') + SELECT coalesce(root,'') || '/' || coalesce(term,'') || '/' || coalesce(ordinal::text,'-') + FROM (SELECT root, term, ordinal FROM items + UNION ALL + SELECT a.root, NULL, NULL FROM anchors a + WHERE NOT EXISTS (SELECT 1 FROM reached WHERE reached.root = a.root)) u + ORDER BY root NULLS FIRST, term NULLS FIRST, ordinal NULLS FIRST`, + sql16roProject, sql16roGeneration, []string{"FHIR/Patient", "FHIR/Observation"}, sql16roLiveLabel) + var gotTriples []string + for _, row := range rows { + key := sql16roParse[sql16roKey](t, row[0]) + root, term, ordinal := "", "", "-" + if key.RootID != nil { + root = *key.RootID + } + if key.TerminalID != nil { + term = *key.TerminalID + } + if key.Ordinal != nil { + ordinal = fmt.Sprint(*key.Ordinal) + } + gotTriples = append(gotTriples, root+"/"+term+"/"+ordinal) + } + var wantTriples []string + for _, row := range witness { + wantTriples = append(wantTriples, row[0]) + } + if fmt.Sprint(gotTriples) != fmt.Sprint(wantTriples) { + t.Fatalf("row identity drifted from the independent witness:\n got %v\nwant %v", gotTriples, wantTriples) + } + + t.Run("preserve-parent-coding-keeps-the-route-with-no-items", func(t *testing.T) { + policyPlan, policyStage, policyFacts := sql16roFixture(t) + adapter := policyStage.Group.RelatedOccurrence + adapter.CodingEmptyPolicy = ir.PhysicalUnnestPreserveParent + adapter.CodingUnnest.EmptyPolicy = ir.PhysicalUnnestPreserveParent + policyRegion, policyRenderer := fixture.renderLive(t, policyPlan, policyStage, policyFacts) + policyRows := fixture.queryRegion(t, policyRegion, policyRenderer, selectList) + if len(policyRows) != len(want)+1 { + t.Fatalf("PRESERVE_PARENT must add the itemless route row, got %d rows:\n%v", len(policyRows), policyRows) + } + var itemless []string + for _, row := range policyRows { + if strings.Contains(row[0], `"terminal_id": "o-3"`) { + itemless = row + } + } + if itemless == nil { + t.Fatalf("the itemless route row is missing:\n%v", policyRows) + } + key := sql16roParse[sql16roKey](t, itemless[0]) + if key.RootID == nil || *key.RootID != "p-1" || key.TerminalID == nil || *key.TerminalID != "o-3" || + key.ItemPresent || key.Ordinal != nil { + t.Fatalf("itemless route key = %#v, want root p-1, terminal o-3, item_present false, null ordinal", key) + } + if itemless[2] != "false" || itemless[3] != "true" { + t.Fatalf("itemless route witnesses = %q/%q, want false/true", itemless[2], itemless[3]) + } + }) + + t.Run("route-error-policy-raises-the-registered-guard-tag", func(t *testing.T) { + errorPlan, errorStage, errorFacts := sql16roFixture(t) + errorStage.Group.RelatedOccurrence.RelatedRecords.EmptyPolicy = ir.PhysicalUnnestError + errorRegion, errorRenderer := fixture.renderLive(t, errorPlan, errorStage, errorFacts) + rows, err := fixture.svc.iso.Query(fixture.svc.ctx, + fixture.qualifiedRegionText(t, errorRegion), fixture.regionArgs(errorRenderer)...) + if err == nil { + for rows.Next() { + } + err = rows.Err() + rows.Close() + } + if err == nil { + t.Fatalf("the route ERROR policy must raise on the unmatched anchor") + } + if !strings.Contains(err.Error(), stageExpandEmptyTag) { + t.Fatalf("the guard must carry the registered tag %q, got: %v", stageExpandEmptyTag, err) + } + }) + + t.Run("two-relations-never-share-an-expansion", func(t *testing.T) { + other := newSQL16roLiveFixture(t) + other.priorView(t, "p-2") + regionA, rendererA := fixture.renderLive(t, plan, stage, facts) + regionB, rendererB := other.renderLive(t, plan, stage, facts) + rowsA := fixture.queryRegion(t, regionA, rendererA, `q."__loom_r1_k"::text`) + rowsB := other.queryRegion(t, regionB, rendererB, `q."__loom_r1_k"::text`) + if !strings.Contains(strings.Join(flattenSQL16ro(rowsA), " "), "p-1") { + t.Fatalf("relation A must carry its own reachable anchor p-1: %v", rowsA) + } + for _, cell := range flattenSQL16ro(rowsB) { + if strings.Contains(cell, "p-1") { + t.Fatalf("relation B borrowed relation A's expansion (share violation): %s", cell) + } + } + if len(rowsB) != 1 { + t.Fatalf("relation B is its own grain: one pad row for p-2, got %v", rowsB) + } + }) +} + +func sql16roCheckDriverTypes(t *testing.T, fixture sql16roLiveFixture, + region stageRelatedOccurrenceRegion, r *planRenderer, +) { + t.Helper() + sql := `SELECT q."__loom_r1_n", q."__loom_r1_h" FROM (` + fixture.qualifiedRegionText(t, region) + ") AS q" + + sql16roOrder(t, region) + " LIMIT 1" + rows, err := fixture.svc.iso.Query(fixture.svc.ctx, sql, fixture.regionArgs(r)...) + if err != nil { + fixture.svc.fail("execute driver-type probe", err) + } + defer rows.Close() + if !rows.Next() { + t.Fatal("driver-type probe returned no rows") + } + values, err := rows.Values() + if err != nil { + fixture.svc.fail("read driver-type probe", err) + } + if _, ok := values[0].(int64); !ok { + t.Fatalf("the rank must arrive as a driver-side bigint, got %T", values[0]) + } + if _, ok := values[1].(bool); !ok { + t.Fatalf("the has-item witness must arrive as a driver-side bool, got %T", values[1]) + } + if err := rows.Err(); err != nil { + fixture.svc.fail("finish driver-type probe", err) + } +} + +func flattenSQL16ro(rows [][]string) []string { + var out []string + for _, row := range rows { + out = append(out, row...) + } + return out +} diff --git a/internal/dataframe/compiler/render/postgres/stage_related_occurrence_test.go b/internal/dataframe/compiler/render/postgres/stage_related_occurrence_test.go new file mode 100644 index 000000000..a00bab6de --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/stage_related_occurrence_test.go @@ -0,0 +1,797 @@ +package postgres + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" +) + +const ( + sql16roProject = "sql16ro-proj" + sql16roGeneration = "gen-1" + sql16roAuthPath = "FHIR/Patient" +) + +func sql16roScopeBlock(variables []string, authVariable string) []ir.PhysicalOperation { + operations := make([]ir.PhysicalOperation, 0, 2*len(variables)+2) + for _, field := range []struct { + path string + bind string + }{{"project", "project"}, {"dataset_generation", "dataset_generation"}} { + for _, variable := range variables { + operations = append(operations, ir.PhysicalOperation{ + Kind: ir.PhysicalFilterOp, + Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: variable, Path: []string{field.path}}, + Right: &ir.PhysicalValue{BindKey: field.bind}, + }}, + }) + } + } + authInputs := make([]ir.PhysicalValue, 0, len(variables)+2) + for _, variable := range variables { + authInputs = append(authInputs, ir.PhysicalValue{Variable: variable, Path: []string{"auth_resource_path"}}) + } + authInputs = append(authInputs, + ir.PhysicalValue{BindKey: "auth_resource_paths"}, + ir.PhysicalValue{BindKey: "auth_resource_paths_unrestricted"}) + operations = append(operations, + ir.PhysicalOperation{Kind: ir.PhysicalDerivedLetOp, DerivedLet: &ir.PhysicalDerivedLet{ + Variable: authVariable, Operator: "AUTH_RESOURCE_PATH_ALLOWED", Inputs: authInputs, + }}, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: authVariable}, + Right: &ir.PhysicalValue{BindKey: "scope_allowed"}, + }}}, + ) + return operations +} + +func sql16roValueExpression(variable string, path ...string) ir.PhysicalExpression { + return ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, Value: &ir.PhysicalValue{Variable: variable, Path: path}, + } +} + +func sql16roExtract(variable, resourceType, selector string, cardinality ir.PhysicalCardinality, + nullBehavior ir.PhysicalNullBehavior, +) ir.PhysicalExpression { + return ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, Cardinality: cardinality, NullBehavior: nullBehavior, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: variable, Path: []string{"payload"}}, + ResourceType: resourceType, + Selector: sql16roSelectorNameOnly(selector), + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + } +} + +func sql16roSelectorNameOnly(text string) spec.Selector { + selector, err := spec.ParseSelector(text) + if err != nil { + panic("sql16ro fixture selector " + text + ": " + err.Error()) + } + return selector +} + +func sql16roFixture(t *testing.T) (ir.PhysicalPlan, ir.PhysicalConstructionStage, map[string]stageColumnFact) { + t.Helper() + plan := ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "project": sql16roProject, + "dataset_generation": sql16roGeneration, + "auth_resource_paths": []string{sql16roAuthPath, "FHIR/Observation"}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + "ro_anchor_collection": "Patient", + "ro_edge_collection": "fhir_edge", + "ro_label": "subject_Reference", + "ro_source_type": "Patient", + "ro_target_type": "Observation", + "ro_cid": "group_related_coding", + "ro_contributor_value": "final", + }, + } + route := []ir.PhysicalStageRelatedRouteStep{{ + StepIndex: 0, FromOccurrenceID: "patient-occ", FromNodeID: "patient-node", FromResourceType: "Patient", + EdgeID: "subject-edge", Relationship: "subject_Reference", StorageDirection: "INBOUND", MatchMode: "REQUIRED", + ToOccurrenceID: "observation-occ", ToNodeID: "observation-node", ToResourceType: "Observation", + }} + subplan := ir.PhysicalSubplan{ + Captures: []string{"inputrow"}, + Unique: true, + Sort: &ir.PhysicalValue{Variable: "observation_doc", Path: []string{"_id"}}, + Return: ir.PhysicalExpression{ + Kind: ir.PhysicalObjectExpression, Cardinality: ir.PhysicalObjectCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Object: &ir.PhysicalObject{Fields: []ir.PhysicalExpressionProjection{ + {Name: "terminal_id", Expression: sql16roValueExpression("observation_doc", "_id")}, + {Name: "resource_id", Expression: sql16roValueExpression("observation_doc", "id")}, + {Name: "status", Expression: sql16roExtract("observation_doc", "Observation", "status", + ir.PhysicalScalarCardinality, ir.PhysicalPreserveNull)}, + {Name: "__loom_group_related_codings", Expression: sql16roExtract("observation_doc", "Observation", + "valueCodeableConcept.coding[]", ir.PhysicalArrayCardinality, ir.PhysicalEmptyOnNull)}, + }}}, + } + subplan.Operations = append(subplan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalCollectionScanOp, + CollectionScan: &ir.PhysicalCollectionScan{Variable: "anchor_doc", CollectionBindKey: "ro_anchor_collection"}}, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: "anchor_doc", Path: []string{"_key"}}, + Right: &ir.PhysicalValue{Variable: "inputrow", Path: []string{"_key"}}, + }}}, + ) + subplan.Operations = append(subplan.Operations, sql16roScopeBlock([]string{"anchor_doc"}, "anchor_auth")...) + subplan.Operations = append(subplan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: "anchor_doc", TargetVariable: "observation_doc", EdgeVariable: "subject_edge", + Direction: ir.PhysicalInbound, EdgeCollectionBindKey: "ro_edge_collection", + EdgeLabelBindKey: "ro_label", SourceTypeBindKey: "ro_source_type", TargetTypeBindKey: "ro_target_type", + EdgeTargetTypeField: "from_type", Strategy: ir.PhysicalTraversalNative, + }}, + ) + subplan.Operations = append(subplan.Operations, sql16roScopeBlock([]string{"subject_edge", "observation_doc"}, "hop_auth")...) + + adapter := &ir.PhysicalStageGroupRelatedOccurrenceAdapter{ + RouteID: "related-route-1", + RelatedRecords: ir.PhysicalStageRelatedExpand{ + RouteID: "related-route-1", Multiplicity: "DISTINCT_TARGET_PER_INPUT", + AnchorColumnID: "_key", AnchorKind: "root", AnchorOccurrenceID: "patient-occ", + AnchorNodeID: "patient-node", AnchorResourceType: "Patient", + TargetNodeID: "observation-node", TargetResourceType: "Observation", + ParentIdentityColumn: "_key", + RelatedRecordsVariable: "related_records", IndexVariable: "related_index", + ItemVariable: "related_item", IdentityVariable: "related_identity", + ConstructionIDBindKey: "ro_cid", EmptyPolicy: ir.PhysicalUnnestPreserveParent, + RelatedRecords: subplan, + Route: route, + }, + ScalarProjections: []ir.PhysicalStageGroupRelatedProjection{{ + InputColumn: "related_status", ResultField: "status", + OwnerResourceType: "Observation", ValueResourceType: "string", + }}, + CodingSelector: "valueCodeableConcept.coding[]", + CodingArrayColumn: "__loom_group_related_codings", + CodingEmptyPolicy: ir.PhysicalUnnestExclude, + OrdinalSemantics: stageRelatedCodingOrdinalSemantics, + CodingUnnest: &ir.PhysicalUnnest{ + OutputVariable: "coding_item", Ordinality: "coding_ordinal", HasItemVariable: "coding_has_item", + EmptyPolicy: ir.PhysicalUnnestExclude, + Expression: ir.PhysicalExpression{Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Value: &ir.PhysicalValue{Variable: "related_item", Path: []string{"__loom_group_related_codings"}}}, + }, + CodingMembershipColumn: "membership", + OccurrenceKeyColumn: "occurrence", + RouteHasItemVariable: "route_has_item", + } + stage := ir.PhysicalConstructionStage{ + ID: "group_related", InputStageID: "source_projection", Kind: ir.PhysicalStageGroupOp, + InputRowVariable: "inputrow", + OutputRowVariable: "group_row", + InputColumns: []ir.PhysicalStageColumn{ + {ID: "key_id", Name: "_key", Kind: "string", Cardinality: "required_one"}, + {ID: "related_status_id", Name: "related_status", Kind: "string", Cardinality: "optional_one", Internal: true}, + {ID: "membership_id", Name: "membership", Kind: "object", Cardinality: "optional_one", Internal: true}, + {ID: "occurrence_id", Name: "occurrence", Kind: "object", Cardinality: "required_one", Internal: true}, + }, + OutputColumns: []ir.PhysicalStageColumn{{ID: "occurrence_id", Name: "occurrence", Kind: "object", Cardinality: "required_one", Internal: true}}, + RowIdentityColumn: "occurrence", + Group: &ir.PhysicalStageGroup{ + ConstructionIDBindKey: "ro_cid", + MissingKeyPolicy: ir.PhysicalStageGroupMissingKeyExclude, + RelatedOccurrence: adapter, + }, + } + facts := map[string]stageColumnFact{ + "_key": {alias: "c0"}, + "related_status": {alias: "c1"}, + "membership": {alias: "c2", jsonb: true}, + "occurrence": {alias: "c3", jsonb: true}, + } + return plan, stage, facts +} + +func sql16roRender(t *testing.T, plan ir.PhysicalPlan, stage ir.PhysicalConstructionStage, + relation string, facts map[string]stageColumnFact, +) (stageRelatedOccurrenceRegion, *planRenderer) { + t.Helper() + r := &planRenderer{ + storage: mustBinding(t), rows: map[string]rowScope{}, + parameters: map[string]int{}, canonicalAuthLets: map[string]string{}, + } + if err := r.loadScope(plan); err != nil { + t.Fatalf("loadScope: %v", err) + } + region, err := r.renderStageGroupRelatedOccurrence(plan, stage, 1, relation, facts) + if err != nil { + t.Fatalf("renderStageGroupRelatedOccurrence: %v", err) + } + return region, r +} + +func sql16roBoundTexts(r *planRenderer) map[string]bool { + bound := map[string]bool{} + for _, parameter := range r.args { + if text, ok := parameter.Value.(string); ok { + bound[text] = true + } + } + return bound +} + +func TestSQL16RelatedOccurrenceGoldenOneHop(t *testing.T) { + plan, stage, facts := sql16roFixture(t) + region, r := sql16roRender(t, plan, stage, "__loom_stage_0", facts) + sql := region.sourceText + + for _, want := range []string{ + "SELECT s.*, __loom_rel.* FROM __loom_stage_0 AS s LEFT JOIN LATERAL", + "FROM (SELECT 1) AS __loom_one LEFT JOIN LATERAL (", + ") AS __loom_route ON TRUE", + "LEFT JOIN LATERAL (SELECT __loom_r1_cod_e.item AS item, ROW_NUMBER() OVER (ORDER BY __loom_r1_cod_e.ord) AS rn FROM jsonb_array_elements(", + ") WITH ORDINALITY AS __loom_r1_cod_e(item, ord) WHERE jsonb_typeof(__loom_r1_cod_e.item) <> 'null') AS __loom_r1_cod ON TRUE", + } { + if !strings.Contains(sql, want) { + t.Fatalf("source text missing spine fragment:\n want: %s\n got: %s", want, sql) + } + } + + for _, want := range []string{ + `SELECT DISTINCT ON (`, ` COLLATE "C") `, ` AS term_id, `, ` AS doc`, + ` ORDER BY `, ` COLLATE "C"`, + `"logical_id" = s."c0"`, + `target_logical_id" = a1.`, + `"source_logical_id"`, + ` INNER JOIN `, + } { + if !strings.Contains(sql, want) { + t.Fatalf("source text missing route fragment:\n want: %s\n got: %s", want, sql) + } + } + routeEnd := strings.Index(sql, "AS __loom_route ON TRUE") + relStart := strings.Index(sql, "AS __loom_rel ON TRUE") + if routeEnd < 0 || relStart < 0 || routeEnd > relStart { + t.Fatalf("the route/coding lateral structure drifted:\n%s", sql) + } + for _, inside := range []string{`a1."logical_id" = s."c0"`, `WHERE a1."project" = $`, + `a1."auth_resource_path" = ANY`} { + at := strings.Index(sql, inside) + if at < 0 || at > routeEnd { + t.Fatalf("conjunct %q is not composed inside the route subquery:\n%s", inside, sql) + } + } + if outer := sql[relStart:]; strings.Contains(outer, "a1.") || strings.Contains(outer, "x2.") || strings.Contains(outer, "e2.") { + t.Fatalf("a hop-alias conjunct escaped into the outer lateral:\n%s", outer) + } + + for _, want := range []string{ + `ROW_NUMBER() OVER (ORDER BY s."c0" COLLATE "C" ASC NULLS FIRST, ` + + `__loom_route.term_id COLLATE "C" ASC NULLS FIRST, __loom_r1_cod.rn ASC NULLS FIRST) - 1`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("source text lost the total order:\n want: %s\n got: %s", want, sql) + } + } + + if got := r.args[7]; string(got.Type) != "text" || got.Value != "{valueCodeableConcept,coding}" { + t.Fatalf("the Coding array must bind the path through the iterated step, got %#v", got) + } + + if got := strings.Count(sql, "resource_type\" = $"); got < 2 { + t.Fatalf("anchor and target scope predicates missing (resource_type conjuncts = %d):\n%s", got, sql) + } + + if !strings.Contains(sql, "__loom_route.doc #> $") { + t.Fatalf("scalar projection must read the materialized route document:\n%s", sql) + } + if _, ok := region.columns["related_status"]; !ok { + t.Fatalf("adapter column related_status missing: %v", region.columns) + } + + if !strings.Contains(sql, `(__loom_r1_cod.rn - 1)::int`) { + t.Fatalf("Coding ordinal must be the dense position minus one:\n%s", sql) + } + + for _, want := range []string{ + `(CASE WHEN (__loom_route.term_id IS NOT NULL AND __loom_r1_cod.rn IS NOT NULL) THEN true ELSE false END) AS "__loom_r1_h"`, + `(CASE WHEN __loom_route.term_id IS NOT NULL THEN true ELSE false END) AS "__loom_r1_r"`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("has-item witness drifted:\n want: %s\n got: %s", want, sql) + } + } + + for _, want := range []string{ + "jsonb_build_object(", "to_jsonb((s.\"c0\"))", "to_jsonb((__loom_route.term_id))", + `"__loom_r1_k"`, `"__loom_r1_h"`, `"__loom_r1_r"`, `"__loom_r1_n"`, + "__loom_route.term_id IS NOT NULL", + "__loom_r1_cod.rn IS NOT NULL", + } { + if !strings.Contains(sql, want) { + t.Fatalf("source text missing witness fragment:\n want: %s\n got: %s", want, sql) + } + } + + if !strings.Contains(sql, "CASE WHEN NOT (__loom_route.term_id IS NOT NULL) OR __loom_r1_cod.rn IS NOT NULL THEN 1 ELSE 0 END") { + t.Fatalf("Coding EXCLUDE conjunct drifted:\n%s", sql) + } + + if !strings.HasSuffix(sql, `ORDER BY s."c0" ASC, __loom_rel."__loom_r1_n" ASC`) { + t.Fatalf("source text must end in the parent-then-rank order:\n%s", sql) + } + if len(region.orderTerms) != 2 || region.orderTerms[0] != `s."c0" ASC` || + region.orderTerms[1] != `s."__loom_r1_n" ASC` { + t.Fatalf("order terms = %v, want [s.\"c0\" ASC s.\"__loom_r1_n\" ASC]", region.orderTerms) + } + + for name, column := range region.columns { + if !strings.HasPrefix(column.ref, `"`) || strings.Contains(column.ref, ".") { + t.Fatalf("adapter column %q ref %q is not the materialized unqualified read", name, column.ref) + } + } + + for _, forbidden := range []string{"'Patient'", "'Observation'", "'subject_Reference'", + "'valueCodeableConcept'", "'coding'", "'final'", "'inputrow'", "'observation_doc'", "'fhir_edge'"} { + if strings.Contains(sql, forbidden) { + t.Fatalf("plan data reached SQL text (%s):\n%s", forbidden, sql) + } + } + bound := sql16roBoundTexts(r) + for _, want := range []string{"Patient", "Observation", "subject_Reference", + "{valueCodeableConcept,coding}", "{}", "root_id", "terminal_id", "item_present", "ordinal", + "system", "version", "code", "display", sql16roProject, sql16roGeneration} { + if !bound[want] { + t.Fatalf("plan value %q was never bound as a parameter; args=%v", want, r.args) + } + } +} + +func TestSQL16RelatedOccurrenceShareLaw(t *testing.T) { + plan, stage, facts := sql16roFixture(t) + + regionA, _ := sql16roRender(t, plan, stage, "__loom_stage_0", facts) + otherFacts := map[string]stageColumnFact{"_key": {alias: "c7"}} + cloneStage := stage + regionB, _ := sql16roRender(t, plan, cloneStage, "__loom_stage_7", otherFacts) + + if !strings.Contains(regionA.sourceText, "__loom_stage_0 AS s") || + !strings.Contains(regionA.sourceText, `s."c0"`) { + t.Fatalf("expansion A must bind relation 0 with its own key column:\n%s", regionA.sourceText) + } + if !strings.Contains(regionB.sourceText, "__loom_stage_7 AS s") || + !strings.Contains(regionB.sourceText, `s."c7"`) { + t.Fatalf("expansion B must bind relation 7 with its own key column:\n%s", regionB.sourceText) + } + if strings.Contains(regionA.sourceText, "__loom_stage_7") || strings.Contains(regionA.sourceText, `s."c7"`) { + t.Fatalf("expansion A borrowed the second relation:\n%s", regionA.sourceText) + } + if strings.Contains(regionB.sourceText, "__loom_stage_0") || strings.Contains(regionB.sourceText, `s."c0"`) { + t.Fatalf("expansion B borrowed the first relation (share violation):\n%s", regionB.sourceText) + } + + regionC, _ := sql16roRender(t, plan, stage, "__loom_stage_0", facts) + if regionC.sourceText != regionA.sourceText { + t.Fatalf("rendering is not deterministic:\nA: %s\nC: %s", regionA.sourceText, regionC.sourceText) + } +} + +func TestSQL16RelatedOccurrencePolicies(t *testing.T) { + cases := []struct { + name string + routePolicy ir.PhysicalUnnestEmptyPolicy + codingPolicy ir.PhysicalUnnestEmptyPolicy + want []string + deny []string + }{ + {"preserve-exclude", ir.PhysicalUnnestPreserveParent, ir.PhysicalUnnestExclude, []string{ + "FROM (SELECT 1) AS __loom_one", + "LEFT JOIN LATERAL (SELECT", + "(CASE WHEN NOT (__loom_route.term_id IS NOT NULL) OR __loom_r1_cod.rn IS NOT NULL THEN 1 ELSE 0 END) > 0", + }, []string{"CONSTRUCTION_EXPANSION_EMPTY", "INNER JOIN LATERAL"}}, + {"exclude-route", ir.PhysicalUnnestExclude, ir.PhysicalUnnestPreserveParent, []string{ + "INNER JOIN LATERAL (SELECT DISTINCT ON", + ") AS __loom_route ON TRUE", + "INNER JOIN LATERAL (SELECT (__loom_route.doc", + }, []string{"CONSTRUCTION_EXPANSION_EMPTY"}}, + {"error-route", ir.PhysicalUnnestError, ir.PhysicalUnnestPreserveParent, []string{ + "FROM (SELECT 1) AS __loom_one", + "CONSTRUCTION_EXPANSION_EMPTY", + "(CASE WHEN __loom_route.term_id IS NOT NULL THEN 1 ELSE CAST('CONSTRUCTION_EXPANSION_EMPTY'", + }, nil}, + {"error-coding", ir.PhysicalUnnestPreserveParent, ir.PhysicalUnnestError, []string{ + "(CASE WHEN NOT (__loom_route.term_id IS NOT NULL) OR __loom_r1_cod.rn IS NOT NULL THEN 1 ELSE CAST('CONSTRUCTION_EXPANSION_EMPTY'", + "CONSTRUCTION_EXPANSION_EMPTY", + }, []string{"THEN 1 ELSE 0 END) > 0"}}, + } + for _, cell := range cases { + t.Run(cell.name, func(t *testing.T) { + plan, stage, facts := sql16roFixture(t) + adapter := stage.Group.RelatedOccurrence + adapter.RelatedRecords.EmptyPolicy = cell.routePolicy + if adapter.CodingUnnest != nil { + adapter.CodingUnnest.EmptyPolicy = cell.codingPolicy + } + adapter.CodingEmptyPolicy = cell.codingPolicy + region, _ := sql16roRender(t, plan, stage, "__loom_stage_0", facts) + for _, want := range cell.want { + if !strings.Contains(region.sourceText, want) { + t.Fatalf("missing policy fragment %q in:\n%s", want, region.sourceText) + } + } + for _, denied := range cell.deny { + if strings.Contains(region.sourceText, denied) { + t.Fatalf("policy must not emit %q in:\n%s", denied, region.sourceText) + } + } + }) + } +} + +func TestSQL16RelatedOccurrenceGoldenNoCoding(t *testing.T) { + plan, stage, facts := sql16roFixture(t) + adapter := stage.Group.RelatedOccurrence + adapter.CodingSelector = "" + adapter.CodingUnnest = nil + adapter.CodingArrayColumn = "" + adapter.CodingMembershipColumn = "" + adapter.CodingEmptyPolicy = "" + adapter.OrdinalSemantics = "" + delete(facts, "membership") + projections := stage.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Return.Object.Fields + stage.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Return.Object.Fields = projections[:3] + region, _ := sql16roRender(t, plan, stage, "__loom_stage_0", facts) + sql := region.sourceText + if strings.Contains(sql, "jsonb_array_elements") && strings.Contains(sql, "WITH ORDINALITY") { + t.Fatalf("no-Coding adapter must not emit a Coding lateral:\n%s", sql) + } + if strings.Contains(sql, "__loom_r1_cod") { + t.Fatalf("no-Coding adapter must not reference the Coding lateral:\n%s", sql) + } + if !strings.Contains(sql, "to_jsonb((false))") || !strings.Contains(sql, "to_jsonb((NULL::int))") { + t.Fatalf("AQL parity: item_present false and ordinal null without a Coding (construction_group.go:590-593):\n%s", sql) + } + if strings.Contains(sql, "__loom_r1_cod.rn ASC") { + t.Fatalf("rank must not order by the absent Coding:\n%s", sql) + } + if _, ok := region.columns["occurrence"]; !ok { + t.Fatalf("occurrence key column must remain registered") + } +} + +func TestSQL16RelatedOccurrenceRefusals(t *testing.T) { + type cell struct { + name string + mutate func(*ir.PhysicalConstructionStage, map[string]stageColumnFact, *ir.PhysicalPlan) + want string + } + cells := []cell{ + {"no-adapter", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence = nil + }, "names one"}, + {"anchor-kind", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.AnchorKind = "relatedRecord" + }, "anchor kind"}, + {"anchor-column", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.AnchorColumnID = "_id" + }, "materialized root _key"}, + {"contributor-set-drift", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RootContributorColumn = "contributors" + }, "materialized root _key"}, + {"multiplicity", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.Multiplicity = "ALL_TARGETS" + }, "multiplicity"}, + {"route-id-mismatch", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RouteID = "other-route" + }, "same non-empty route"}, + {"cid-bind-missing", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + delete(p.BindVars, "ro_cid") + }, "construction ID bind"}, + {"hop-discontinuous", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.Route[0].ToNodeID = "other-node" + }, "does not end at its declared target"}, + {"hop-match-mode", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.Route[0].MatchMode = "MAYBE" + }, "incomplete or discontinuous"}, + {"ordinal-semantics", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.OrdinalSemantics = "ONE_BASED" + }, "zero-based flattened position"}, + {"coding-details-without-selector", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.CodingSelector = "" + }, "without a Coding selector"}, + {"coding-expression-drift", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.CodingUnnest.Expression.Value.Variable = "someone_elses_item" + }, "exact ordinal operation"}, + {"coding-policy-unknown", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.CodingEmptyPolicy = "SILENT" + s.Group.RelatedOccurrence.CodingUnnest.EmptyPolicy = "SILENT" + }, "related Coding empty policy"}, + {"route-policy-unknown", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.EmptyPolicy = "SILENT" + }, "route empty policy"}, + {"captures-drift", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Captures = []string{"someone_else"} + }, "capture exactly"}, + {"sort-not-terminal-id", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Sort.Path = []string{"_key"} + }, "sorted by the reached target's {_id}"}, + {"not-unique", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Unique = false + }, "UNIQUE and sorted"}, + {"anchor-scan-missing", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + ops := s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations = ops[1:] + }, "open with the root-collection anchor scan"}, + {"anchor-collection-bind-drift", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + p.BindVars["ro_anchor_collection"] = "Observation" + }, "bound to the anchor resource type"}, + {"anchor-equality-drift", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations[1].Filter.Predicate.Left.Path = []string{"_id"} + }, "exact root-key equality"}, + {"anchor-equality-operator", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations[1].Filter.Predicate.Operator = "IN" + }, "exact root-key equality"}, + {"anchor-scope-block-defect", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + ops := s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations = append(ops[:3], ops[4:]...) + }, "anchor scope block"}, + {"traversal-any-direction", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations[6].Traversal.Direction = ir.PhysicalAny + }, "ANY-direction"}, + {"traversal-endpoint-lookup", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations[6].Traversal.Strategy = ir.PhysicalTraversalEndpointLookup + }, "endpoint-lookup traversal payload is incomplete"}, + {"endpoint-lookup-join-field-missing", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + traversal := s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations[6].Traversal + traversal.Strategy = ir.PhysicalTraversalEndpointLookup + traversal.EndpointField = "subject" + }, "payload is incomplete"}, + {"edge-bind-drift", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + p.BindVars["ro_edge_collection"] = "fhir_edge_two" + }, "must be bound to \"fhir_edge\""}, + {"hop-label-mismatch", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.Route[0].Relationship = "other_Reference" + }, "names relationship"}, + {"hop-storage-direction-drift", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.Route[0].StorageDirection = "OUTBOUND" + }, "storage direction changed"}, + {"trailing-operation-shape", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations = append( + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalDerivedLetOp, DerivedLet: &ir.PhysicalDerivedLet{ + Variable: "extra", Operator: "UNIQUE", + Inputs: []ir.PhysicalValue{{Variable: "observation_doc", Path: []string{"payload"}}}, + }}) + }, "only typed contributor FILTERs"}, + {"terminal-id-field-missing", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + fields := s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Return.Object.Fields + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Return.Object.Fields = fields[1:] + }, "lacks its terminal_id"}, + {"terminal-id-wrong-path", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + field := &s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Return.Object.Fields[0] + field.Expression.Value.Path = []string{"_key"} + }, "must read the target's {_id}"}, + {"scalar-unknown-field", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.ScalarProjections[0].ResultField = "no_such_field" + }, "which the target projection does not declare"}, + {"scalar-owner-drift", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.ScalarProjections[0].OwnerResourceType = "Condition" + }, "validates its selector against"}, + {"coding-column-unnest-drift", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.CodingArrayColumn = "status" + }, "exact ordinal operation"}, + {"coding-column-undeclared", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + fields := s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Return.Object.Fields + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Return.Object.Fields = fields[:3] + }, "is not declared by the target projection"}, + {"undeclared-array-field", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + adapter := s.Group.RelatedOccurrence + adapter.CodingSelector = "" + adapter.CodingUnnest = nil + adapter.CodingArrayColumn = "" + adapter.CodingMembershipColumn = "" + adapter.CodingEmptyPolicy = "" + adapter.OrdinalSemantics = "" + }, "SCALAR cardinality only"}, + {"anchor-fact-missing", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + delete(f, "_key") + }, "not among the prior stage's registered columns"}, + {"anchor-fact-jsonb", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + f["_key"] = stageColumnFact{alias: "c0", jsonb: true} + }, "not the materialized root-key scalar"}, + {"identity-field-arity", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + field := &s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Return.Object.Fields[0] + field.Expression.Value.Path = []string{"_id", "extra"} + }, "field must read the reached target row directly"}, + {"coding-column-source-drift", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + coding := s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Return.Object.Fields[3] + coding.Expression.Extract.Source.Variable = "anchor_doc" + }, "Coding column \"__loom_group_related_codings\" must read the reached target row directly"}, + {"coding-resource-type-drift", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + coding := s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Return.Object.Fields[3] + coding.Expression.Extract.ResourceType = "Patient" + }, "not the target type"}, + {"subplan-distinct-by", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.DistinctBy = + &ir.PhysicalValue{Variable: "observation_doc", Path: []string{"_id"}} + }, "DISTINCT BY modifier"}, + {"subplan-empty-operations", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations = nil + }, "no operations"}, + {"anchor-scan-unnamed", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations[0].CollectionScan.Variable = "" + }, "name its variable and root collection bind"}, + {"anchor-filter-missing", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations = + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations[:1] + }, "followed by its root-key equality filter"}, + {"hop-operation-missing", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations = + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations[:6] + }, "is missing its operation"}, + {"hop-operation-kind", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations[6] = + ir.PhysicalOperation{Kind: ir.PhysicalDerivedLetOp, DerivedLet: &ir.PhysicalDerivedLet{ + Variable: "extra", Operator: "UNIQUE", + Inputs: []ir.PhysicalValue{{Variable: "observation_doc", Path: []string{"payload"}}}, + }} + }, "exactly its canonical hops"}, + {"hop-source-variable", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations[6].Traversal.SourceVariable = "anchor_other" + }, "must anchor on"}, + {"hop-scope-block-defect", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + ops := s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations + s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations = append(ops[:9], ops[10:]...) + }, "hop 0 scope block"}, + {"coding-source-path-drift", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + coding := s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Return.Object.Fields[3] + coding.Expression.Extract.Source.Path = []string{"document"} + }, "reached document at"}, + {"coding-selector-indexed-step", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + coding := s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Return.Object.Fields[3] + index := 0 + coding.Expression.Extract.Selector.Steps[0].Index = &index + }, "indexed steps"}, + {"coding-selector-unsafe-step", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + coding := s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Return.Object.Fields[3] + coding.Expression.Extract.Selector.Steps[1].Field = "bad field[]" + }, "Coding column \"__loom_group_related_codings\": selector step \"bad field[]\" is not renderer-safe"}, + {"coding-selector-two-iterates", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + coding := s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Return.Object.Fields[3] + coding.Expression.Extract.Selector.Steps = append( + coding.Expression.Extract.Selector.Steps, spec.SelectorStep{Field: "tail", Iterate: true}) + }, "repeated steps beyond one iterate step"}, + {"coding-selector-no-iterate", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + coding := s.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Return.Object.Fields[3] + coding.Expression.Extract.Selector.Steps = coding.Expression.Extract.Selector.Steps[:1] + }, "no iterated selector step"}, + {"adapter-witness-missing", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RouteHasItemVariable = "" + }, "lacks its has-item witness"}, + {"route-empty", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.Route = nil + }, "at least one hop"}, + {"hop0-not-at-anchor", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.RelatedRecords.Route[0].FromNodeID = "wrong-node" + }, "does not start at the anchor node"}, + {"hop1-discontinuous", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + route := s.Group.RelatedOccurrence.RelatedRecords.Route + s.Group.RelatedOccurrence.RelatedRecords.Route = append(route, ir.PhysicalStageRelatedRouteStep{ + StepIndex: 1, FromOccurrenceID: "other-occ", FromNodeID: "wrong-node", FromResourceType: "Patient", + EdgeID: "level-edge", Relationship: "subject_Reference", StorageDirection: "INBOUND", MatchMode: "REQUIRED", + ToOccurrenceID: "observation-occ", ToNodeID: "observation-node", ToResourceType: "Observation", + }) + }, "is discontinuous"}, + {"coding-membership-missing", func(s *ir.PhysicalConstructionStage, f map[string]stageColumnFact, p *ir.PhysicalPlan) { + s.Group.RelatedOccurrence.CodingMembershipColumn = "" + }, "no matching array column"}, + } + { + plan, stage, facts := sql16roFixture(t) + sql16roRender(t, plan, stage, "__loom_stage_0", facts) + } + for _, item := range cells { + t.Run(item.name, func(t *testing.T) { + plan, stage, facts := sql16roFixture(t) + item.mutate(&stage, facts, &plan) + r := &planRenderer{ + storage: mustBinding(t), rows: map[string]rowScope{}, + parameters: map[string]int{}, canonicalAuthLets: map[string]string{}, + } + if err := r.loadScope(plan); err != nil { + t.Fatalf("loadScope: %v", err) + } + _, err := r.renderStageGroupRelatedOccurrence(plan, stage, 1, "__loom_stage_0", facts) + wantUnsupported(t, err, item.name) + if !strings.Contains(err.Error(), item.want) { + t.Fatalf("%s: refusal must cite %q, got: %v", item.name, item.want, err) + } + }) + } +} + +func TestSQL16RelatedOccurrenceContributorFilter(t *testing.T) { + plan, stage, facts := sql16roFixture(t) + subplan := &stage.Group.RelatedOccurrence.RelatedRecords.RelatedRecords + subplan.Operations = append(subplan.Operations, ir.PhysicalOperation{ + Kind: ir.PhysicalFilterOp, + Filter: &ir.PhysicalFilter{Expression: &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalComparisonPredicate, + Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", + LeftExpression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "observation_doc", Path: []string{"payload"}}, + ResourceType: "Observation", + Selector: sql16roSelectorNameOnly("category.coding.system"), + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + }, + Right: &ir.PhysicalValue{BindKey: "ro_contributor_value"}, + ValueKind: spec.FilterString, + }, + }}, + }) + region, renderer := sql16roRender(t, plan, stage, "__loom_stage_0", facts) + want := `AND (EXISTS (SELECT 1 FROM jsonb_array_elements((CASE WHEN (x2."payload" #> $7::text[]) IS NULL ` + + `OR jsonb_typeof((x2."payload" #> $7::text[])) = 'null' THEN '[]'::jsonb ` + + `ELSE jsonb_build_array((x2."payload" #> $7::text[])) END)) AS __sv(v) WHERE (__sv.v) = ($8::jsonb)))` + if !strings.Contains(region.sourceText, want) { + t.Fatalf("contributor conjunct missing %q:\n%s", want, region.sourceText) + } + conjunct := strings.Index(region.sourceText, "AS __sv(v)") + routeEnd := strings.Index(region.sourceText, "AS __loom_route ON TRUE") + if conjunct < 0 || routeEnd < 0 || conjunct > routeEnd { + t.Fatalf("contributor conjunct escaped the route subquery:\n%s", region.sourceText) + } + if strings.Contains(region.sourceText, "'final'") { + t.Fatalf("contributor value reached SQL text:\n%s", region.sourceText) + } + if len(renderer.args) < 9 { + t.Fatalf("args = %d, want the contributor path and value parameters", len(renderer.args)) + } + if got := renderer.args[6]; string(got.Type) != "text" || got.Value != "{category,coding,system}" { + t.Fatalf("contributor path argument = %#v, want text {category,coding,system}", got) + } + if raw, ok := renderer.args[7].Value.(json.RawMessage); !ok || string(raw) != `"final"` { + t.Fatalf("contributor value argument = %#v, want jsonb \"final\"", renderer.args[7].Value) + } +} + +func TestSQL16RelatedOccurrenceEndpointLookupEquivalence(t *testing.T) { + plan, stage, facts := sql16roFixture(t) + baseline, baselineRenderer := sql16roRender(t, plan, stage, "__loom_stage_0", facts) + traversal := stage.Group.RelatedOccurrence.RelatedRecords.RelatedRecords.Operations[6].Traversal + if traversal.Strategy != ir.PhysicalTraversalNative { + t.Fatalf("fixture hop is not native: %q", traversal.Strategy) + } + traversal.Strategy = ir.PhysicalTraversalEndpointLookup + traversal.EndpointField = "subject" + traversal.EndpointJoinField = "logical_id" + traversal.EndpointIndexFields = []string{"project", "dataset_generation"} + endpoint, endpointRenderer := sql16roRender(t, plan, stage, "__loom_stage_0", facts) + if endpoint.sourceText != baseline.sourceText { + t.Fatalf("endpoint-lookup normalization changed the statement:\n%s\n---\n%s", endpoint.sourceText, baseline.sourceText) + } + if len(endpoint.columns) != len(baseline.columns) || len(endpointRenderer.args) != len(baselineRenderer.args) { + t.Fatalf("endpoint-lookup normalization changed the column/arg shape: %d/%d vs %d/%d", + len(endpoint.columns), len(endpointRenderer.args), len(baseline.columns), len(baselineRenderer.args)) + } + for name, column := range baseline.columns { + if other, ok := endpoint.columns[name]; !ok || other.expr != column.expr { + t.Fatalf("adapter column %q drifted under normalization", name) + } + } +} diff --git a/internal/dataframe/compiler/render/postgres/stage_semantic.go b/internal/dataframe/compiler/render/postgres/stage_semantic.go new file mode 100644 index 000000000..f3922c8f3 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/stage_semantic.go @@ -0,0 +1,267 @@ +package postgres + +import ( + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const groupSemanticCarrierName = "__loom_semantic_raw_codings" + +const groupSemanticSourceIdentity = "_key" + +const groupSemanticSourceStageID = "source_projection" + +func validateGroupSemanticKey(plan ir.PhysicalPlan, stage ir.PhysicalConstructionStage) error { + semantic := stage.Group.SemanticKey + if semantic == nil { + return nil + } + for _, field := range []struct{ name, value string }{ + {"family binding", semantic.BindingID}, + {"family rule version", semantic.RuleVersion}, + {"family ID", semantic.FamilyID}, + {"family system", semantic.System}, + } { + if strings.TrimSpace(field.value) == "" { + return fmt.Errorf("%w: the unresolved semantic family Group key is a registered named gap (L-13.3; the server-resolved authoring channel stays refused at recipe.Bundle.Validate; missing %s) and is refused, never partially rendered", + ErrUnsupportedPostgresPlan, field.name) + } + } + for _, field := range []struct { + name, value string + }{ + {"output column", semantic.OutputColumn}, + {"tuple variable", semantic.TupleVariable}, + {"label variable", semantic.LabelVariable}, + {"source root variable", semantic.SourceRootVariable}, + {"root collection bind", semantic.RootCollectionBindKey}, + {"resource type", semantic.ResourceType}, + {"coding path", semantic.CodingPath}, + } { + if strings.TrimSpace(field.value) == "" { + return fmt.Errorf("%w: the semantic family key is missing its %s (ir/physical_stage_validation.go:1478-1486 re-read at the door)", + ErrUnsupportedPostgresPlan, field.name) + } + } + if stage.InputStageID != groupSemanticSourceStageID { + return fmt.Errorf("%w: the semantic family key requires the direct-root %q input stage (ir/physical_stage_validation.go:1480)", + ErrUnsupportedPostgresPlan, groupSemanticSourceStageID) + } + if semantic.SourceIdentityColumn != groupSemanticSourceIdentity { + return fmt.Errorf("%w: the semantic family key requires source rows identified by %q (lower/recipe_construction_reshape.go:69)", + ErrUnsupportedPostgresPlan, groupSemanticSourceIdentity) + } + if collection, ok := plan.BindVars[semantic.RootCollectionBindKey].(string); !ok || collection != semantic.ResourceType { + return fmt.Errorf("%w: the semantic family root collection bind %q does not carry the exact root resource type %q", + ErrUnsupportedPostgresPlan, semantic.RootCollectionBindKey, semantic.ResourceType) + } + if len(plan.Operations) == 0 || plan.Operations[0].RootScan == nil { + return fmt.Errorf("%w: the semantic family key requires one leading ROOT_SCAN", ErrUnsupportedPostgresPlan) + } + scan := plan.Operations[0].RootScan + if scan.CollectionBindKey != semantic.RootCollectionBindKey { + return fmt.Errorf("%w: the semantic family root collection bind %q must be the root scan's own collection bind %q", + ErrUnsupportedPostgresPlan, semantic.RootCollectionBindKey, scan.CollectionBindKey) + } + if scan.Variable != semantic.SourceRootVariable { + return fmt.Errorf("%w: the semantic family source root variable %q must be the root scan variable %q", + ErrUnsupportedPostgresPlan, semantic.SourceRootVariable, scan.Variable) + } + if len(semantic.PathSegments) == 0 || !semantic.PathSegments[len(semantic.PathSegments)-1].Repeated { + return fmt.Errorf("%w: the semantic Coding path must end in a repeated Coding array (lower/recipe_construction_reshape.go:85-105 demands a generated repeated Coding path)", + ErrUnsupportedPostgresPlan) + } + var canonical strings.Builder + for index, segment := range semantic.PathSegments { + if !payloadPathSegmentPattern.MatchString(segment.Name) { + return fmt.Errorf("%w: semantic Coding path segment %q is not renderer-safe", + ErrUnsupportedPostgresPlan, segment.Name) + } + if index > 0 { + canonical.WriteByte('.') + } + canonical.WriteString(segment.Name) + if segment.Repeated { + canonical.WriteString("[]") + } + } + if canonical.String() != semantic.CodingPath { + return fmt.Errorf("%w: the semantic Coding path must be the canonical join of its segments", ErrUnsupportedPostgresPlan) + } + for _, name := range groupRegisteredOutputNames(stage.Group) { + if name == semantic.OutputColumn { + return fmt.Errorf("%w: the semantic family output %q is already a registered key, reducer, or row-value output", + ErrUnsupportedPostgresPlan, semantic.OutputColumn) + } + } + return nil +} + +func groupRegisteredOutputNames(group *ir.PhysicalStageGroup) []string { + names := make([]string, 0, len(group.Keys)+len(group.Aggregates)+len(group.RowValues)) + for _, key := range group.Keys { + names = append(names, key.OutputColumn) + } + for _, aggregate := range group.Aggregates { + names = append(names, aggregate.Output) + } + for _, rowValue := range group.RowValues { + names = append(names, rowValue.Output) + } + return names +} + +func (r *planRenderer) groupSemanticCarrierColumn(semantic *ir.PhysicalStageGroupSemanticKey) (*groupColumn, error) { + current := r.documentColumn(r.rootAlias) + var steps []string + var pending []string + lastAlias := "" + sequence := 0 + for _, segment := range semantic.PathSegments { + pending = append(pending, segment.Name) + if !segment.Repeated { + continue + } + extraction, err := r.documentPathFromExpression(current, pending) + if err != nil { + return nil, fmt.Errorf("semantic Coding path: %w", err) + } + current = extraction + pending = nil + sequence++ + alias := "semx" + strconv.Itoa(sequence) + guard := "CASE WHEN jsonb_typeof(" + current + ") = 'array' THEN " + current + " ELSE '[]'::jsonb END" + steps = append(steps, "jsonb_array_elements("+guard+") AS "+alias+"(item)") + current = alias + ".item" + lastAlias = alias + } + if len(pending) != 0 || lastAlias == "" { + return nil, fmt.Errorf("%w: the semantic Coding path must end in a repeated Coding array", ErrUnsupportedPostgresPlan) + } + expr := "COALESCE((SELECT jsonb_agg(" + lastAlias + ".item) FROM " + + strings.Join(steps, " CROSS JOIN LATERAL ") + "), '[]'::jsonb)" + return &groupColumn{ + name: groupSemanticCarrierName, alias: "csem", expr: expr, ref: `"csem"`, jsonb: true, + }, nil +} + +type groupSemanticRegion struct { + lateral string + tupleReads []string + tupleAliases []string + label string + identityPart string + codeRead string +} + +func (r *planRenderer) renderGroupSemanticRegion( + semantic *ir.PhysicalStageGroupSemanticKey, + carrier *groupColumn, + stage ir.PhysicalConstructionStage, + prefix string, + dollar func(int) string, +) (groupSemanticRegion, error) { + if carrier == nil || carrier.name != groupSemanticCarrierName { + return groupSemanticRegion{}, fmt.Errorf("%w: the semantic family key requires the inline door's root-payload carrier; a composed sequence has none and stays refused at the composition door (stage_sequence.go validateStageKindPayload GROUP-arm gate)", + ErrUnsupportedPostgresPlan) + } + + systemKey := r.textParameter("system", "text:system") + versionKey := r.textParameter("version", "text:version") + codeKey := r.textParameter("code", "text:code") + displayKey := r.textParameter("display", "text:display") + nonBlank := r.codedGroupTrimmedNonBlank + familySystem := r.textParameter(semantic.System, "") + familyVersion := r.textParameter(semantic.Version, "") + + matchFor := func(item string) string { + systemJSON := item + " -> " + dollar(systemKey) + "::text" + systemText := item + " ->> " + dollar(systemKey) + "::text" + versionJSON := item + " -> " + dollar(versionKey) + "::text" + versionText := item + " ->> " + dollar(versionKey) + "::text" + versionRead := "CASE WHEN jsonb_typeof(" + versionJSON + ") = 'string'" + + " AND " + nonBlank(versionText) + " THEN " + versionText + " END" + versionPredicate := versionRead + " IS NULL" + if semantic.Version != "" { + versionPredicate = versionRead + " = " + dollar(familyVersion) + "::text" + } + return "jsonb_typeof(" + item + ") = 'object'" + + " AND jsonb_typeof(" + systemJSON + ") = 'string'" + + " AND " + systemText + " = " + dollar(familySystem) + "::text" + + " AND " + versionPredicate + } + blankGated := func(item string, key int) string { + valueJSON := item + " -> " + dollar(key) + "::text" + valueText := item + " ->> " + dollar(key) + "::text" + return "CASE WHEN jsonb_typeof(" + valueJSON + ") = 'string'" + + " AND " + nonBlank(valueText) + " THEN " + valueText + " END" + } + + aggCode := prefix + "sc0" + aggDisplay := prefix + "sc1" + dedup := "SELECT " + blankGated("_mc.coding", codeKey) + " AS " + aggCode + + ", MIN(" + blankGated("_mc.coding", displayKey) + ` COLLATE "C") AS ` + aggDisplay + + " FROM jsonb_array_elements(" + carrier.ref + ") AS _mc(coding)" + + " WHERE " + matchFor("_mc.coding") + + " GROUP BY 1" + + versionMember := dollar(familyVersion) + "::text" + if semantic.Version == "" { + versionMember = "NULL::text" + } + matched := "SELECT CASE WHEN " + aggCode + " IS NOT NULL THEN " + dollar(familySystem) + "::text END," + + " CASE WHEN " + aggCode + " IS NOT NULL THEN " + versionMember + " END," + + aggCode + ", " + aggDisplay + + " FROM (" + dedup + ") AS " + prefix + "sd" + fallback := "SELECT NULL::text, NULL::text, NULL::text, NULL::text" + + " WHERE NOT EXISTS (SELECT 1 FROM jsonb_array_elements(" + carrier.ref + ") AS _me(coding)" + + " WHERE " + matchFor("_me.coding") + ")" + + tupleAliases := []string{prefix + "ssy", prefix + "svr", prefix + "sco", prefix + "sdi"} + lateral := " CROSS JOIN LATERAL (" + matched + " UNION ALL " + fallback + ") AS " + prefix + "sm(" + + strings.Join(tupleAliases, ", ") + ")" + + minDisplay := "MIN(" + tupleAliases[3] + ` COLLATE "C")` + label := "CASE WHEN " + minDisplay + " IS NULL THEN " + tupleAliases[2] + " ELSE " + minDisplay + " END" + + semanticOutputID := "" + for _, column := range stage.OutputColumns { + if column.Name == semantic.OutputColumn { + semanticOutputID = column.ID + break + } + } + if semanticOutputID == "" { + return groupSemanticRegion{}, fmt.Errorf("%w: semantic Group output %q has no stable column ID (construction_group.go:167-169 parity)", + ErrUnsupportedPostgresPlan, semantic.OutputColumn) + } + outputBind := r.textParameter(semanticOutputID, "") + bindingBind := r.textParameter(semantic.BindingID, "") + ruleBind := r.textParameter(semantic.RuleVersion, "") + familyBind := r.textParameter(semantic.FamilyID, "") + pathBind := r.textParameter(semantic.CodingPath, "") + tupleText := "'[' || COALESCE(to_jsonb(g." + tupleAliases[0] + ")::text, 'null')" + + " || ',' || COALESCE(to_jsonb(g." + tupleAliases[1] + ")::text, 'null')" + + " || ',' || COALESCE(to_jsonb(g." + tupleAliases[2] + ")::text, 'null') || ']'" + identityPart := "'[' || to_jsonb(" + dollar(outputBind) + "::text)" + + " || ',' || " + quoteSQLLiteral(`"semantic_family"`) + + " || ',' || to_jsonb(" + dollar(bindingBind) + "::text)" + + " || ',' || to_jsonb(" + dollar(ruleBind) + "::text)" + + " || ',' || to_jsonb(" + dollar(familyBind) + "::text)" + + " || ',' || to_jsonb(" + dollar(familySystem) + "::text)" + + " || ',' || to_jsonb(" + dollar(familyVersion) + "::text)" + + " || ',' || to_jsonb(" + dollar(pathBind) + "::text)" + + " || ',' || " + tupleText + " || ']'" + + return groupSemanticRegion{ + lateral: lateral, + tupleReads: []string{tupleAliases[0], tupleAliases[1], tupleAliases[2]}, + tupleAliases: tupleAliases, + label: label, + identityPart: identityPart, + codeRead: tupleAliases[2], + }, nil +} diff --git a/internal/dataframe/compiler/render/postgres/stage_semantic_production_test.go b/internal/dataframe/compiler/render/postgres/stage_semantic_production_test.go new file mode 100644 index 000000000..b4af47a71 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/stage_semantic_production_test.go @@ -0,0 +1,247 @@ +package postgres + +import ( + "encoding/json" + "reflect" + "strconv" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/recipe" +) + +func newSQL16SemanticService(t *testing.T) *pgir02Service { + t.Helper() + return newOwnedService(t, 0, "verify_sql16v1") +} + +func sql16SemanticTrackFixtures(t *testing.T, svc *pgir02Service) { + t.Helper() + var database string + if err := svc.iso.QueryRow(svc.ctx, "SELECT current_database()").Scan(&database); err != nil { + svc.fail("track owned database", err) + } + sql16FixturesMu.Lock() + defer sql16FixturesMu.Unlock() + sql16TrackedFixtures[database] = true + sql16TrackedFixtures[svc.schema] = true +} + +func sql16Coding(system, version, code, display any) map[string]any { + coding := map[string]any{} + if system != nil { + coding["system"] = system + } + if version != nil { + coding["version"] = version + } + if code != nil { + coding["code"] = code + } + if display != nil { + coding["display"] = display + } + return coding +} + +func sql16SemanticRow(t *testing.T, svc *pgir02Service, logicalID, project, generation, authPath, status string, codings ...map[string]any) { + t.Helper() + payload := map[string]any{"status": status} + if codings != nil { + payload["code"] = map[string]any{"coding": codings} + } + svc.insertResource(t, svc.schema, project, generation, "Observation", logicalID, authPath, payload) +} + +func sql16SemanticFixture(t *testing.T, svc *pgir02Service) { + t.Helper() + sql16SemanticRow(t, svc, "s-1", group13Project, group13Generation, group13AuthPath, "final", + sql16Coding(sql16SemanticSystem, sql16SemanticIdentity, "a", "zulu"), + sql16Coding(sql16SemanticSystem, sql16SemanticIdentity, "a", "alpha"), + sql16Coding(sql16SemanticSystem, sql16SemanticIdentity, "b", "bravo"), + ) + sql16SemanticRow(t, svc, "s-2", group13Project, group13Generation, group13AuthPath, "final", + sql16Coding(sql16SemanticSystem, sql16SemanticIdentity, "a", "mike")) + sql16SemanticRow(t, svc, "s-3", group13Project, group13Generation, group13AuthPath, "final", + sql16Coding(sql16SemanticSystem, sql16SemanticIdentity, "c", "")) + sql16SemanticRow(t, svc, "s-4", group13Project, group13Generation, group13AuthPath, "final", + sql16Coding("http://loom.test/fhir/CodeSystem/foreign", sql16SemanticIdentity, "x", "xray")) + sql16SemanticRow(t, svc, "s-5", group13Project, group13Generation, group13AuthPath, "final", + sql16Coding(sql16SemanticSystem, sql16SemanticIdentity, nil, "ghost")) + sql16SemanticRow(t, svc, "s-6", group13Project, group13Generation, group13AuthPath, "final", + sql16Coding(sql16SemanticSystem, "9.9.9", "a", "nope")) + sql16SemanticRow(t, svc, "s-7", group13Project, group13Generation, group13AuthPath, "final", + sql16Coding(sql16SemanticSystem, sql16SemanticIdentity, "a", nil)) + sql16SemanticRow(t, svc, "s-9", group13Project, group13Generation, group13AuthPath, "final", + sql16Coding(json.Number("123"), sql16SemanticIdentity, "a", "num")) + sql16SemanticRow(t, svc, "s-10", group13Project, group13Generation, group13AuthPath, "final", + map[string]any{"display": "keyless"}) + sql16SemanticRow(t, svc, "w-out", group13Project, group13Generation, "FHIR/Hidden", "final", + sql16Coding(sql16SemanticSystem, sql16SemanticIdentity, "secret", "out")) + if rows := svc.witness(t, `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" WHERE project = $1 AND dataset_generation = $2 AND auth_resource_path = $3`, + group13Project, group13Generation, group13AuthPath); rows[0][0] != "9" { + t.Fatalf("scope witness: acceptance row count = %v, want 9", rows) + } +} + +func sql16SemanticIdentityTag(t *testing.T, plan ir.PhysicalPlan, version *string, tuple [3]*string) string { + t.Helper() + _ = plan + quote := func(value *string) string { + if value == nil { + return "null" + } + raw, err := json.Marshal(*value) + if err != nil { + t.Fatalf("marshal identity member: %v", err) + } + return string(raw) + } + str := func(value string) *string { return &value } + parts := []string{ + `["construction","` + group13StepID + `"]`, `["operation","GROUP"]`, + `["status_group","final"]`, + `["category","semantic_family",` + quote(str(sql16SemanticBinding)) + `,` + + quote(str(sql16SemanticRule)) + `,` + quote(str(sql16SemanticFamily)) + `,` + + quote(str(sql16SemanticSystem)) + `,` + quote(version) + `,` + + quote(str(sql16SemanticPath)) + `,[` + quote(tuple[0]) + `,` + quote(tuple[1]) + `,` + quote(tuple[2]) + `]]`, + } + return "[" + strings.Join(parts, ",") + "]" +} + +func sql16Str(value string) *string { return &value } + +func sql16SemanticColumnTypes(t *testing.T, svc *pgir02Service, rendered command.RenderedPostgresPhysicalPlan) []string { + t.Helper() + conn, err := svc.iso.Acquire(svc.ctx) + if err != nil { + svc.fail("acquire for column types", err) + } + defer conn.Release() + pgxConn := conn.Conn() + sd, err := pgxConn.Prepare(svc.ctx, "sql16_semantic_describe", rendered.SQL) + if err != nil { + svc.fail("prepare for column types", err) + } + defer pgxConn.Deallocate(svc.ctx, "sql16_semantic_describe") + var oids []string + for _, field := range sd.Fields { + oids = append(oids, strconv.FormatUint(uint64(field.DataTypeOID), 10)) + } + return oids +} + +func sql16SemanticRows(t *testing.T, plan ir.PhysicalPlan) [][]string { + t.Helper() + familySystem, familyVersion := sql16Str(sql16SemanticSystem), sql16Str(sql16SemanticIdentity) + return [][]string{ + {"final", "5", "ghost", sql16SemanticIdentityTag(t, plan, sql16Str(sql16SemanticIdentity), [3]*string{nil, nil, nil})}, + {"final", "3", "alpha", sql16SemanticIdentityTag(t, plan, sql16Str(sql16SemanticIdentity), [3]*string{familySystem, familyVersion, sql16Str("a")})}, + {"final", "1", "bravo", sql16SemanticIdentityTag(t, plan, sql16Str(sql16SemanticIdentity), [3]*string{familySystem, familyVersion, sql16Str("b")})}, + {"final", "1", "c", sql16SemanticIdentityTag(t, plan, sql16Str(sql16SemanticIdentity), [3]*string{familySystem, familyVersion, sql16Str("c")})}, + } +} + +func TestSQL16SemanticLiveMatrix(t *testing.T) { + svc := newSQL16SemanticService(t) + sql16SemanticTrackFixtures(t, svc) + sql16SemanticFixture(t, svc) + + plan := sql16SemanticPlan(t, recipe.ConstructionGroupMissingKeyGroup, sql16SemanticIdentity) + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("render: %v", err) + } + for _, leaked := range []string{sql16SemanticSystem, sql16SemanticBinding, sql16SemanticRule, + sql16SemanticFamily, sql16SemanticIdentity, group13Project, group13Generation, "final", "alpha"} { + if strings.Contains(rendered.SQL, leaked) { + t.Fatalf("value %q leaked into statement text", leaked) + } + } + + rows := svc.runRendered(t, rendered) + if !reflect.DeepEqual(rows, sql16SemanticRows(t, plan)) { + t.Fatalf("semantic groups mismatch:\n got %q\nwant %q", rows, sql16SemanticRows(t, plan)) + } + if repeated := svc.runRendered(t, rendered); !reflect.DeepEqual(repeated, rows) { + t.Fatalf("grouping changed between executions:\n%v\n%v", rows, repeated) + } + seen := map[string]bool{} + for _, row := range rows { + if seen[row[3]] { + t.Fatalf("duplicate row identity %q", row[3]) + } + seen[row[3]] = true + } + + columnTypes := sql16SemanticColumnTypes(t, svc, rendered) + for index, want := range []string{"25", "20", "25", "25"} { + if columnTypes[index] != want { + t.Fatalf("column %d type = %q, want %q", index, columnTypes[index], want) + } + } + + sql16SemanticRow(t, svc, "s-11", group13Project, group13Generation, group13AuthPath, "final", + sql16Coding(sql16SemanticSystem, sql16SemanticIdentity, "a", "aardvark")) + mutated := svc.runRendered(t, rendered) + if mutated[1][2] != "aardvark" { + t.Fatalf("mutation probe: (S,V,a) label never became aardvark: %v", mutated) + } + if mutated[1][1] != "4" { + t.Fatalf("mutation probe: (S,V,a) rows never became 4: %v", mutated) + } +} + +func TestSQL16SemanticLiveVersionlessFamily(t *testing.T) { + svc := newSQL16SemanticService(t) + sql16SemanticTrackFixtures(t, svc) + sql16SemanticFixture(t, svc) + sql16SemanticRow(t, svc, "s-12", group13Project, group13Generation, group13AuthPath, "final", + sql16Coding(sql16SemanticSystem, nil, "d", "delta")) + + plan := sql16SemanticPlan(t, recipe.ConstructionGroupMissingKeyGroup, "") + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("render: %v", err) + } + rows := svc.runRendered(t, rendered) + null := "" + familySystem := sql16Str(sql16SemanticSystem) + want := [][]string{ + {"final", "9", null, sql16SemanticIdentityTag(t, plan, sql16Str(""), [3]*string{nil, nil, nil})}, + {"final", "1", "delta", sql16SemanticIdentityTag(t, plan, sql16Str(""), [3]*string{familySystem, nil, sql16Str("d")})}, + } + if !reflect.DeepEqual(rows, want) { + t.Fatalf("versionless family mismatch:\n got %q\nwant %q", rows, want) + } +} + +func TestSQL16SemanticLiveMissingKeyPolicy(t *testing.T) { + svc := newSQL16SemanticService(t) + sql16SemanticTrackFixtures(t, svc) + sql16SemanticFixture(t, svc) + + excludePlan := sql16SemanticPlan(t, recipe.ConstructionGroupMissingKeyExclude, sql16SemanticIdentity) + excludeRendered, err := RenderPhysicalPlan(excludePlan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("render exclude: %v", err) + } + excluded := svc.runRendered(t, excludeRendered) + if len(excluded) != 3 { + t.Fatalf("EXCLUDE rows = %v, want the three family-coded groups", excluded) + } + for _, row := range excluded { + if row[2] == "" { + t.Fatalf("EXCLUDE kept a null-code group: %v", row) + } + } + + errorPlan := sql16SemanticPlan(t, recipe.ConstructionGroupMissingKeyError, sql16SemanticIdentity) + errorRendered, err := RenderPhysicalPlan(errorPlan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("render error: %v", err) + } + group13ExpectRaise(t, svc, errorRendered, groupMissingKeyTag) +} diff --git a/internal/dataframe/compiler/render/postgres/stage_semantic_test.go b/internal/dataframe/compiler/render/postgres/stage_semantic_test.go new file mode 100644 index 000000000..ea6abf477 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/stage_semantic_test.go @@ -0,0 +1,433 @@ +package postgres + +import ( + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/recipe" +) + +const ( + sql16SemanticOutput = "category" + sql16SemanticSystem = "http://loom.test/fhir/CodeSystem/finding" + sql16SemanticPath = "code.coding[]" + sql16SemanticBinding = "binding-observation-finding" + sql16SemanticRule = "rule-v3" + sql16SemanticFamily = "family-observation-finding" + sql16SemanticIdentity = "1.4.2" +) + +func sql16SemanticAttach(t *testing.T, plan *ir.PhysicalPlan, version string) { + t.Helper() + if plan.StageSequence == nil || len(plan.StageSequence.Stages) != 1 { + t.Fatalf("expected the inline single-stage group shape") + } + stage := &plan.StageSequence.Stages[0] + group := stage.Group + if group == nil { + t.Fatalf("group payload missing") + } + if len(plan.Operations) == 0 || plan.Operations[0].RootScan == nil { + t.Fatalf("root scan missing") + } + scan := plan.Operations[0].RootScan + resourceType, ok := plan.BindVars[scan.CollectionBindKey].(string) + if !ok || resourceType == "" { + t.Fatalf("root collection bind %q carries no resource type", scan.CollectionBindKey) + } + template := ir.PhysicalStageColumn{} + for _, column := range stage.OutputColumns { + if column.Kind == "string" { + template = column + break + } + } + if template.Name == "" { + t.Fatalf("the base plan carries no string output to clone") + } + label := template + label.ID = sql16SemanticOutput + label.Name = sql16SemanticOutput + label.Internal = false + insertBefore := func(columns []ir.PhysicalStageColumn) []ir.PhysicalStageColumn { + for index := range columns { + if columns[index].Name == groupRowIDColumn { + columns = append(columns, ir.PhysicalStageColumn{}) + copy(columns[index+1:], columns[index:]) + columns[index] = label + return columns + } + } + return append(columns, label) + } + stage.OutputColumns = insertBefore(stage.OutputColumns) + plan.StageSequence.FinalColumns = insertBefore(plan.StageSequence.FinalColumns) + projection := ir.PhysicalProjection{Name: sql16SemanticOutput, + Value: ir.PhysicalValue{Variable: "group_semantic_label_9"}} + inserted := false + for index := range stage.OutputProjections { + if stage.OutputProjections[index].Name == groupRowIDColumn { + stage.OutputProjections = append(stage.OutputProjections, ir.PhysicalProjection{}) + copy(stage.OutputProjections[index+1:], stage.OutputProjections[index:]) + stage.OutputProjections[index] = projection + inserted = true + break + } + } + if !inserted { + stage.OutputProjections = append(stage.OutputProjections, projection) + } + group.SemanticKey = &ir.PhysicalStageGroupSemanticKey{ + OutputColumn: sql16SemanticOutput, + TupleVariable: "group_semantic_tuple_9", LabelVariable: "group_semantic_label_9", + SourceRootVariable: scan.Variable, SourceIdentityColumn: groupSemanticSourceIdentity, + RootCollectionBindKey: scan.CollectionBindKey, ResourceType: resourceType, + CodingPath: sql16SemanticPath, + PathSegments: []ir.PhysicalCodedGroupPathSegment{ + {Name: "code"}, {Name: "coding", Repeated: true}, + }, + BindingID: sql16SemanticBinding, RuleVersion: sql16SemanticRule, + FamilyID: sql16SemanticFamily, System: sql16SemanticSystem, Version: version, + } +} + +func sql16SemanticPlan(t *testing.T, policy recipe.ConstructionGroupMissingKeyPolicy, version string) ir.PhysicalPlan { + t.Helper() + plan := group13CountRowsPlan(t, policy) + sql16SemanticAttach(t, &plan, version) + return plan +} + +func sql16SemanticRender(t *testing.T, plan ir.PhysicalPlan) (string, []string) { + t.Helper() + rendered, err := renderGroupPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("the registered semantic arm must render: %v", err) + } + return rendered.SQL, rendered.OutputColumns +} + +func sql16RegionRenderer(t *testing.T, plan ir.PhysicalPlan) *planRenderer { + t.Helper() + renderer := &planRenderer{ + storage: mustBinding(t), parameters: map[string]int{}, + rows: map[string]rowScope{}, canonicalAuthLets: map[string]string{}, rootAlias: "r0", + } + renderer.rows[plan.Operations[0].RootScan.Variable] = rowScope{alias: renderer.rootAlias, resourceType: "Observation"} + return renderer +} + +func group13ReturnOp(t *testing.T, plan ir.PhysicalPlan) *ir.PhysicalReturn { + t.Helper() + var ret *ir.PhysicalReturn + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalReturnOp { + ret = operation.Return + } + } + if ret == nil { + t.Fatalf("the lowered plan carries no RETURN operation") + } + return ret +} + +func sql16SemanticRefusal(t *testing.T, plan ir.PhysicalPlan, want string) { + t.Helper() + _, err := renderGroupPhysicalPlan(plan, mustBinding(t)) + if err == nil { + t.Fatalf("refusal required, want %q", want) + } + if !strings.Contains(err.Error(), want) { + t.Fatalf("refusal = %q, want substring %q", err.Error(), want) + } +} + +func TestSQL16SemanticGolden(t *testing.T) { + plan := sql16SemanticPlan(t, recipe.ConstructionGroupMissingKeyGroup, sql16SemanticIdentity) + sql, names := sql16SemanticRender(t, plan) + + for _, want := range []string{ + `COALESCE((SELECT jsonb_agg(semx1.item) FROM jsonb_array_elements(CASE WHEN jsonb_typeof(`, + `) = 'array' THEN `, ` ELSE '[]'::jsonb END) AS semx1(item)), '[]'::jsonb)`, + } { + if !strings.Contains(sql, want) { + t.Errorf("walk fragment missing: %s\n%s", want, sql) + } + } + + for _, want := range []string{ + `jsonb_typeof(_mc.coding) = 'object'`, + `_mc.coding -> $`, + `= 'string'`, + `_mc.coding ->> $`, + `IS NULL`, + } { + if !strings.Contains(sql, want) { + t.Errorf("matcher fragment missing: %s", want) + } + } + for _, want := range []string{ + `MIN(`, ` COLLATE "C") AS g_sc1`, + `FROM jsonb_array_elements("csem") AS _mc(coding)`, + `GROUP BY 1`, + `UNION ALL SELECT NULL::text, NULL::text, NULL::text, NULL::text WHERE NOT EXISTS (SELECT 1 FROM jsonb_array_elements("csem") AS _me(coding)`, + `CROSS JOIN LATERAL (`, `) AS g_sm(g_ssy, g_svr, g_sco, g_sdi)`, + } { + if !strings.Contains(sql, want) { + t.Errorf("dedup/fallback fragment missing: %s\n%s", want, sql) + } + } + for _, want := range []string{ + `g_ssy AS g_ssy`, `g_svr AS g_svr`, `g_sco AS g_sco`, + `CASE WHEN MIN(g_sdi COLLATE "C") IS NULL THEN g_sco ELSE MIN(g_sdi COLLATE "C") END AS g_slb`, + `GROUP BY 1, 2, 3, 4`, + } { + if !strings.Contains(sql, want) { + t.Errorf("grouping fragment missing: %s\n%s", want, sql) + } + } + for _, want := range []string{ + `"semantic_family"`, + `to_jsonb($`, + `COALESCE(to_jsonb(g.g_ssy)::text, 'null')`, + `COALESCE(to_jsonb(g.g_svr)::text, 'null')`, + `COALESCE(to_jsonb(g.g_sco)::text, 'null')`, + `g.g_ssy COLLATE "C" ASC NULLS FIRST`, + `g.g_svr COLLATE "C" ASC NULLS FIRST`, + `g.g_sco COLLATE "C" ASC NULLS FIRST`, + ` COLLATE "C" ASC`, + } { + if !strings.Contains(sql, want) { + t.Errorf("identity/order fragment missing: %s\n%s", want, sql) + } + } + for _, secret := range []string{sql16SemanticSystem, sql16SemanticBinding, sql16SemanticRule, + sql16SemanticFamily, sql16SemanticIdentity, sql16SemanticPath, "coding", "code"} { + if strings.Contains(sql, "'"+secret+"'") { + t.Errorf("family string %q reached SQL text", secret) + } + } + if len(names) != 4 || names[2] != sql16SemanticOutput || names[3] != groupRowIDColumn { + t.Fatalf("output columns = %v", names) + } +} + +func TestSQL16SemanticWalkChain(t *testing.T) { + plan := sql16SemanticPlan(t, recipe.ConstructionGroupMissingKeyGroup, sql16SemanticIdentity) + key := plan.StageSequence.Stages[0].Group.SemanticKey + key.PathSegments = []ir.PhysicalCodedGroupPathSegment{ + {Name: "extension", Repeated: true}, {Name: "coding", Repeated: true}, + } + key.CodingPath = "extension[].coding[]" + sql, _ := sql16SemanticRender(t, plan) + for _, want := range []string{ + `COALESCE((SELECT jsonb_agg(semx2.item) FROM jsonb_array_elements(CASE WHEN jsonb_typeof(`, + `) = 'array' THEN `, ` ELSE '[]'::jsonb END) AS semx1(item) CROSS JOIN LATERAL jsonb_array_elements(CASE WHEN jsonb_typeof(semx1.item #> `, + `::text[]) = 'array' THEN `, ` ELSE '[]'::jsonb END) AS semx2(item)), '[]'::jsonb)`, + } { + if !strings.Contains(sql, want) { + t.Errorf("chain fragment missing: %s", want) + } + } +} + +func TestSQL16SemanticVersionBranches(t *testing.T) { + t.Run("versioned-family", func(t *testing.T) { + sql, _ := sql16SemanticRender(t, sql16SemanticPlan(t, recipe.ConstructionGroupMissingKeyGroup, sql16SemanticIdentity)) + if !strings.Contains(sql, "END = $") { + t.Fatalf("versioned family must compare the gated version read against the bind:\n%s", sql) + } + if strings.Contains(sql, "END IS NULL") { + t.Fatalf("a versioned family must not use the null twin") + } + }) + t.Run("versionless-family", func(t *testing.T) { + sql, _ := sql16SemanticRender(t, sql16SemanticPlan(t, recipe.ConstructionGroupMissingKeyGroup, "")) + if !strings.Contains(sql, "END IS NULL") { + t.Fatalf("a blank-version family must demand the null twin:\n%s", sql) + } + }) +} + +func TestSQL16SemanticMissingKeyPolicy(t *testing.T) { + t.Run("group-keeps-null-code", func(t *testing.T) { + sql, _ := sql16SemanticRender(t, sql16SemanticPlan(t, recipe.ConstructionGroupMissingKeyGroup, sql16SemanticIdentity)) + if strings.Contains(sql, "g_sco IS NOT NULL") { + t.Fatalf("GROUP must keep the null-code membership row:\n%s", sql) + } + }) + t.Run("exclude", func(t *testing.T) { + sql, _ := sql16SemanticRender(t, sql16SemanticPlan(t, recipe.ConstructionGroupMissingKeyExclude, sql16SemanticIdentity)) + if !strings.Contains(sql, "g_sco IS NOT NULL") { + t.Fatalf("EXCLUDE must conjunct the membership code:\n%s", sql) + } + }) + t.Run("error", func(t *testing.T) { + sql, _ := sql16SemanticRender(t, sql16SemanticPlan(t, recipe.ConstructionGroupMissingKeyError, sql16SemanticIdentity)) + if !strings.Contains(sql, `(CASE WHEN g_sco IS NOT NULL THEN 1 ELSE CAST('CONSTRUCTION_GROUP_MISSING_KEY'`) { + t.Fatalf("ERROR must raise the canonical tag over the membership code:\n%s", sql) + } + }) + group13AliveControl(t) +} + +func TestSQL16SemanticKeylessGrandTotal(t *testing.T) { + plan := group13KeylessPlan(t) + sql16SemanticAttach(t, &plan, sql16SemanticIdentity) + stage := &plan.StageSequence.Stages[0] + stage.Group.Aggregates = append(stage.Group.Aggregates, ir.PhysicalStageGroupAggregate{ + Operation: "MIN", InputColumn: "value", Output: "semantic_min", Variable: "group_semantic_min_9", + InputKind: "DECIMAL", OutputKind: "decimal", + }) + column := ir.PhysicalStageColumn{ID: "semantic_min", Name: "semantic_min", Kind: "decimal", Cardinality: "optional_one", Nullable: true} + insert := func(columns []ir.PhysicalStageColumn) []ir.PhysicalStageColumn { + for index := range columns { + if columns[index].Name == groupRowIDColumn { + columns = append(columns, ir.PhysicalStageColumn{}) + copy(columns[index+1:], columns[index:]) + columns[index] = column + return columns + } + } + return append(columns, column) + } + stage.OutputColumns = insert(stage.OutputColumns) + plan.StageSequence.FinalColumns = insert(plan.StageSequence.FinalColumns) + minProjection := ir.PhysicalProjection{Name: "semantic_min", + Value: ir.PhysicalValue{Variable: "group_semantic_min_9"}} + if stage.OutputProjections[len(stage.OutputProjections)-1].Name == groupRowIDColumn { + last := len(stage.OutputProjections) - 1 + stage.OutputProjections = append(stage.OutputProjections, ir.PhysicalProjection{}) + copy(stage.OutputProjections[last+1:], stage.OutputProjections[last:]) + stage.OutputProjections[last] = minProjection + } else { + stage.OutputProjections = append(stage.OutputProjections, minProjection) + } + if _, err := renderGroupPhysicalPlan(plan, mustBinding(t)); err != nil { + t.Fatalf("a keyless Group carrying a semantic family key must render: %v", err) + } + + without := group13KeylessPlan(t) + withoutStage := &without.StageSequence.Stages[0] + withoutStage.Group.Aggregates = append(withoutStage.Group.Aggregates, ir.PhysicalStageGroupAggregate{ + Operation: "MIN", InputColumn: "value", Output: "semantic_min", Variable: "group_semantic_min_9", + InputKind: "DECIMAL", OutputKind: "decimal", + }) + _, err := renderGroupPhysicalPlan(without, mustBinding(t)) + if err == nil || !strings.Contains(err.Error(), "only keyless COUNT_ROWS groups can consume the source scope") { + t.Fatalf("keyless-without-family refusal = %v", err) + } +} + +func TestSQL16SemanticDoorRefusals(t *testing.T) { + cases := []struct { + name string + mutate func(t *testing.T, plan *ir.PhysicalPlan) + want string + }{ + {"missing-binding", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.SemanticKey.BindingID = "" + }, "the unresolved semantic family Group key is a registered named gap"}, + {"missing-system", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.SemanticKey.System = " " + }, "the unresolved semantic family Group key is a registered named gap"}, + {"missing-output", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.SemanticKey.OutputColumn = "" + }, "missing its output column"}, + {"foreign-input-stage", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.SourceStageID = "not_the_source_projection" + plan.StageSequence.Stages[0].InputStageID = "not_the_source_projection" + }, "direct-root"}, + {"identity-not-key", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.SemanticKey.SourceIdentityColumn = "id" + }, `identified by "_key"`}, + {"collection-bind-mismatch", func(t *testing.T, plan *ir.PhysicalPlan) { + key := plan.StageSequence.Stages[0].Group.SemanticKey + plan.BindVars[key.RootCollectionBindKey] = "ObservationKind" + }, "does not carry the exact root resource type"}, + {"foreign-collection-bind", func(t *testing.T, plan *ir.PhysicalPlan) { + key := plan.StageSequence.Stages[0].Group.SemanticKey + plan.BindVars["other_collection"] = key.ResourceType + key.RootCollectionBindKey = "other_collection" + }, "must be the root scan's own collection bind"}, + {"foreign-root-variable", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.SemanticKey.SourceRootVariable = "not_the_root" + }, "must be the root scan variable"}, + {"path-not-canonical", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.SemanticKey.CodingPath = "code.codings[]" + }, "canonical join of its segments"}, + {"path-not-repeated-tail", func(t *testing.T, plan *ir.PhysicalPlan) { + key := plan.StageSequence.Stages[0].Group.SemanticKey + key.PathSegments = []ir.PhysicalCodedGroupPathSegment{{Name: "code"}, {Name: "coding"}} + }, "must end in a repeated Coding array"}, + {"path-segment-unsafe", func(t *testing.T, plan *ir.PhysicalPlan) { + key := plan.StageSequence.Stages[0].Group.SemanticKey + key.PathSegments = []ir.PhysicalCodedGroupPathSegment{{Name: "code"}, {Name: "coding[]", Repeated: true}} + key.CodingPath = "code.coding[][]" + }, "not renderer-safe"}, + {"output-addressed-twice", func(t *testing.T, plan *ir.PhysicalPlan) { + plan.StageSequence.Stages[0].Group.SemanticKey.OutputColumn = "status_group" + }, "already a registered key, reducer, or row-value output"}, + } + for _, cell := range cases { + t.Run(cell.name, func(t *testing.T) { + plan := sql16SemanticPlan(t, recipe.ConstructionGroupMissingKeyGroup, sql16SemanticIdentity) + cell.mutate(t, &plan) + sql16SemanticRefusal(t, plan, cell.want) + }) + } + t.Run("carrier-name-is-reserved", func(t *testing.T) { + plan := sql16SemanticPlan(t, recipe.ConstructionGroupMissingKeyGroup, sql16SemanticIdentity) + last := len(plan.Operations) - 1 + plan.Operations[last].Return.Projections = append(plan.Operations[last].Return.Projections, + ir.PhysicalProjection{Name: groupSemanticCarrierName, + Value: ir.PhysicalValue{Variable: "root", Path: []string{"payload", "status"}}}) + sql16SemanticRefusal(t, plan, "reserved for the renderer's semantic carrier column") + }) + t.Run("family-key-mints-no-carrier-at-composition", func(t *testing.T) { + plan := sql16SemanticPlan(t, recipe.ConstructionGroupMissingKeyGroup, sql16SemanticIdentity) + stage := plan.StageSequence.Stages[0] + renderer := sql16RegionRenderer(t, plan) + columns, err := renderer.materializeGroupSource(plan, group13ReturnOp(t, plan)) + if err != nil { + t.Fatalf("materialize: %v", err) + } + if _, err := renderer.renderGroupAggregateRegion(plan, stage, columns, "SELECT 1", "g_", + plan.StageSequence.FinalColumns, groupRowIDColumn); err == nil || + !strings.Contains(err.Error(), "requires the inline door's root-payload carrier") { + t.Fatalf("carrier-less family refusal = %v", err) + } + }) + t.Run("carrier-without-family", func(t *testing.T) { + plan := group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup) + stage := plan.StageSequence.Stages[0] + renderer := sql16RegionRenderer(t, plan) + columns, err := renderer.materializeGroupSource(plan, group13ReturnOp(t, plan)) + if err != nil { + t.Fatalf("materialize: %v", err) + } + carrier, err := renderer.groupSemanticCarrierColumn( + &ir.PhysicalStageGroupSemanticKey{PathSegments: []ir.PhysicalCodedGroupPathSegment{{Name: "code"}, {Name: "coding", Repeated: true}}}) + if err != nil { + t.Fatalf("carrier: %v", err) + } + if _, err := renderer.renderGroupAggregateRegion(plan, stage, append(columns, *carrier), "SELECT 1", "g_", + plan.StageSequence.FinalColumns, groupRowIDColumn); err == nil || + !strings.Contains(err.Error(), "minted only beside a semantic family key") { + t.Fatalf("carrier-without-family refusal = %v", err) + } + }) + group13AliveControl(t) +} + +func TestSQL16SemanticNonSemanticDriftIsZero(t *testing.T) { + sql, _ := sql16SemanticRender(t, group13CountRowsPlan(t, recipe.ConstructionGroupMissingKeyGroup)) + for _, forbidden := range []string{ + "_sm(", "_mc(", "_me(", "csem", "semantic_family", "semx1", "COLLECT", + } { + if strings.Contains(sql, forbidden) { + t.Fatalf("a plan without a semantic family key must not carry %q:\n%s", forbidden, sql) + } + } +} diff --git a/internal/dataframe/compiler/render/postgres/stage_sequence.go b/internal/dataframe/compiler/render/postgres/stage_sequence.go new file mode 100644 index 000000000..8dc31ebd9 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/stage_sequence.go @@ -0,0 +1,1339 @@ +package postgres + +import ( + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const ( + stageExpandArrayTypeTag = "CONSTRUCTION_EXPAND_ARRAY_TYPE_MISMATCH" + stageExpandItemTypeTag = "CONSTRUCTION_EXPAND_ITEM_TYPE_MISMATCH" + stageExpandEmptyTag = "CONSTRUCTION_EXPANSION_EMPTY" +) + +const stageSequenceCTEPrefix = "__loom_" + +func stageSequenceCompositionMentions(plan ir.PhysicalPlan) bool { + return plan.StageSequence != nil && len(plan.StageSequence.Stages) >= 2 +} + +func stageSequenceRelatedOccurrenceMentions(plan ir.PhysicalPlan) bool { + if plan.StageSequence == nil { + return false + } + for _, stage := range plan.StageSequence.Stages { + if stage.Group != nil && stage.Group.RelatedOccurrence != nil { + return true + } + } + return false +} + +func renderStageSequencePhysicalPlan(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + if err := storage.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := validateStageSequencePlanSurface(plan, false); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := plan.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("validate physical plan: %w", err) + } + if plan.Engine != "" && plan.Engine != ir.PhysicalEngineAQL { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: engine %q has no PostgreSQL source; the source graph IR is backend-neutral and ClickHouse plans are not PostgreSQL-rendered", + ErrUnsupportedPostgresPlan, string(plan.Engine)) + } + if plan.ClickHouseCombine != nil || plan.ClickHousePrefix != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: ClickHouse combine/prefix payloads have no PostgreSQL rendering", ErrUnsupportedPostgresPlan) + } + if len(plan.DeferredExpressionLets) != 0 { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: completed plans must not carry deferred expression lets", ErrUnsupportedPostgresPlan) + } + renderer := &planRenderer{ + storage: storage, + rows: map[string]rowScope{}, + parameters: map[string]int{}, + canonicalAuthLets: map[string]string{}, + } + if err := renderer.loadScope(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + sql, err := renderer.renderStageSequenceStatement(plan) + if err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return command.NewRenderedPostgresPhysicalPlan(sql, renderer.args, renderer.outputColumns) +} + +func validateStageSequencePlanSurface(plan ir.PhysicalPlan, mappingTerminal bool) error { + if plan.StageSequence == nil { + return fmt.Errorf("%w: composition dispatch requires a stage sequence", ErrUnsupportedPostgresPlan) + } + sequence := plan.StageSequence + if len(sequence.Stages) == 0 { + return fmt.Errorf("%w: stage %d ID is empty, untrimmed, or duplicated — the composition door requires at least one construction stage (ir/physical_stage_validation.go:15 re-read at the door)", + ErrUnsupportedPostgresPlan, 0) + } + if sequence.CellTraceReturn != nil || sequence.RowLineageReturn != nil || + (sequence.PopulationMappingReturn != nil && !mappingTerminal) { + return fmt.Errorf("%w: row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route (compiler/row_lineage.go:655, cell_trace.go:54, population_mapping.go:38)", + ErrUnsupportedPostgresPlan) + } + if mappingTerminal && sequence.PopulationMappingReturn == nil { + return fmt.Errorf("%w: the population-mapping surface requires the stage sequence's PopulationMappingReturn payload", + ErrUnsupportedPostgresPlan) + } + if plan.PreviewSourceWindowByRootID || sequence.PreviewSourceWindowByRootID || sequence.PreviewTerminalPivotWindow { + return fmt.Errorf("%w: preview source windowing and the terminal Pivot window are pg-sql-22/23 terminal windowing contracts and are refused, never silently rendered", + ErrUnsupportedPostgresPlan) + } + if sequence.OutputAuthResourcePathBindKey != "" { + return fmt.Errorf("%w: the composite Combine prefix authorization path is not registered for composed sequences (ir/physical_stage_sequence.go:25-28; the auth path column registers with pg-ir-03)", + ErrUnsupportedPostgresPlan) + } + if sequence.PreviewLimitBindKey != "" { + if value, ok := plan.BindVars[sequence.PreviewLimitBindKey].(int); !ok || value <= 0 { + return fmt.Errorf("%w: composition preview limit bind %q must be a positive int", + ErrUnsupportedPostgresPlan, sequence.PreviewLimitBindKey) + } + } + for index, stage := range sequence.Stages { + switch { + case stage.Kind == ir.PhysicalStageCodedGroupOp || stage.CodedGroup != nil: + return fmt.Errorf("%w: the registered coded-group slice admits exactly one CODED_GROUP construction stage; multi-stage composition is a pg-sql-16 capability gap", + ErrUnsupportedPostgresPlan) + case stage.Kind == ir.PhysicalStageCohortGroupOp: + return fmt.Errorf("%w: the COHORT_GROUP stage (index %d) is not part of the registered composition surface: the explicit-cohort boundary has no PostgreSQL rendering and its lineage surfaces refuse at their own doors (compiler/row_lineage.go:655)", + ErrUnsupportedPostgresPlan, index) + case stage.GroupedPivot != nil && stage.GroupedPivot.CodedCorrelation != nil: + if index == 0 { + return fmt.Errorf("%w: multi-stage construction sequences are the pg-sql-16 capability; the registered slice admits only the inline single-stage CODED_PIVOT", + ErrUnsupportedPostgresPlan) + } + return fmt.Errorf("%w: a coded Pivot at stage index %d violates the ir rule that the coded Pivot must be the first stage over the direct source projection (ir/physical_stage_validation.go:106-108); coded-pivot composition beyond stage 1 stays the pg-sql-22/23 adjacency", + ErrUnsupportedPostgresPlan, index) + } + } + if err := validateStageSequenceOperations(plan, mappingTerminal); err != nil { + return err + } + return validateStageSequenceChain(plan, mappingTerminal) +} + +func validateStageSequenceOperations(plan ir.PhysicalPlan, mappingTerminal bool) error { + if len(plan.Operations) == 0 { + return fmt.Errorf("%w: plan has no operations", ErrUnsupportedPostgresPlan) + } + first := plan.Operations[0] + if first.Kind != ir.PhysicalRootScanOp || first.RootScan == nil { + return fmt.Errorf("%w: the composition source requires exactly one leading ROOT_SCAN", ErrUnsupportedPostgresPlan) + } + scan := first.RootScan + membershipRoot := scan.Population != nil || scan.CohortSource != nil || scan.PageCandidateSeed != nil + if membershipRoot && !(mappingTerminal && scan.Population != nil && scan.CohortSource == nil && scan.PageCandidateSeed == nil) { + return fmt.Errorf("%w: population, cohort, and candidate-seed roots are bound only by RenderPopulationRootScan (population.go); the composition door refuses them", + ErrUnsupportedPostgresPlan) + } + if mappingTerminal && scan.Population != nil && scan.Population.CollectMembersVariable != ir.PopulationMappingMembersVariable { + return fmt.Errorf("%w: the composition member-collecting root is registered only for CollectMembersVariable %q (compiler/population_mapping.go); got %q", + ErrUnsupportedPostgresPlan, ir.PopulationMappingMembersVariable, scan.Population.CollectMembersVariable) + } + sawSort, sawLimit, sawReturn := false, false, false + for index, operation := range plan.Operations { + last := index == len(plan.Operations)-1 + switch operation.Kind { + case ir.PhysicalRootScanOp: + if index != 0 { + return fmt.Errorf("%w: exactly one leading ROOT_SCAN is required", ErrUnsupportedPostgresPlan) + } + case ir.PhysicalFilterOp, ir.PhysicalDerivedLetOp: + if sawSort || sawLimit || sawReturn { + return fmt.Errorf("%w: source operations must precede the execution window and the terminal RETURN", + ErrUnsupportedPostgresPlan) + } + if operation.Kind == ir.PhysicalFilterOp && operation.Filter == nil { + return fmt.Errorf("%w: operation %d: filter payload is missing", ErrUnsupportedPostgresPlan, index) + } + if operation.Kind == ir.PhysicalDerivedLetOp && operation.DerivedLet == nil { + return fmt.Errorf("%w: operation %d: derived-let payload is missing", ErrUnsupportedPostgresPlan, index) + } + case ir.PhysicalUnnestOp: + if sawSort || sawLimit || sawReturn { + return fmt.Errorf("%w: source expansions must precede the execution window and the terminal RETURN", + ErrUnsupportedPostgresPlan) + } + if operation.Unnest == nil { + return fmt.Errorf("%w: operation %d: unnest payload is missing", ErrUnsupportedPostgresPlan, index) + } + case ir.PhysicalSetOp: + if sawSort || sawLimit || sawReturn { + return fmt.Errorf("%w: source child sets must precede the execution window and the terminal RETURN", + ErrUnsupportedPostgresPlan) + } + if operation.Set == nil { + return fmt.Errorf("%w: operation %d: set payload is missing", ErrUnsupportedPostgresPlan, index) + } + case ir.PhysicalExpressionLetOp: + return fmt.Errorf("%w: source-region EXPRESSION_LET has no dispatch arm in the composition builder and is refused fail-closed (L-16.5, probe P-14.3 twin)", + ErrUnsupportedPostgresPlan) + case ir.PhysicalSortOp: + if sawSort || sawLimit || sawReturn { + return fmt.Errorf("%w: at most one SORT, before LIMIT", ErrUnsupportedPostgresPlan) + } + if operation.Sort == nil { + return fmt.Errorf("%w: operation %d: sort payload is missing", ErrUnsupportedPostgresPlan, index) + } + sawSort = true + case ir.PhysicalLimitOp: + if sawLimit || sawReturn { + return fmt.Errorf("%w: at most one LIMIT", ErrUnsupportedPostgresPlan) + } + if operation.Limit == nil { + return fmt.Errorf("%w: operation %d: limit payload is missing", ErrUnsupportedPostgresPlan, index) + } + sawLimit = true + case ir.PhysicalReturnOp: + if !last || operation.Return == nil || len(operation.Return.Projections) == 0 { + return fmt.Errorf("%w: the composition source requires a terminal RETURN with its projection list", + ErrUnsupportedPostgresPlan) + } + sawReturn = true + default: + return fmt.Errorf("%w: operation %q is not part of the registered composition source surface", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + if !sawReturn { + return fmt.Errorf("%w: the composition source requires a terminal RETURN", ErrUnsupportedPostgresPlan) + } + return nil +} + +func validateStageSequenceChain(plan ir.PhysicalPlan, mappingTerminal bool) error { + sequence := plan.StageSequence + if sequence.SourceStageID == "" || sequence.SourceRowIdentity == "" || sequence.FinalStageID == "" || sequence.FinalRowIdentity == "" { + return fmt.Errorf("%w: source/final stage IDs and row identity columns are required (ir/physical_stage_validation.go:12 re-read at the door)", + ErrUnsupportedPostgresPlan) + } + projectionNames := map[string]bool{} + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalGroupedPivotOp || operation.Kind == ir.PhysicalUnpivotOp { + return fmt.Errorf("%w: source projection cannot contain a terminal table reshape (ir/physical_stage_validation.go:33 re-read at the door)", + ErrUnsupportedPostgresPlan) + } + if operation.Kind == ir.PhysicalReturnOp && operation.Return != nil { + for _, projection := range operation.Return.Projections { + projectionNames[projection.Name] = true + } + } + } + if err := validateStageColumns(sequence.SourceColumns, sequence.SourceRowIdentity); err != nil { + return fmt.Errorf("%w: composition source projection schema: %v", ErrUnsupportedPostgresPlan, err) + } + for _, column := range sequence.SourceColumns { + if !projectionNames[column.Name] { + return fmt.Errorf("%w: source projection column %q is missing from the typed source RETURN (ir/physical_stage_validation.go:46 re-read at the door)", + ErrUnsupportedPostgresPlan, column.Name) + } + } + priorStageID := sequence.SourceStageID + priorColumns := sequence.SourceColumns + priorRowIdentity := sequence.SourceRowIdentity + seenIDs := map[string]bool{sequence.SourceStageID: true} + for index, stage := range sequence.Stages { + path := "stage[" + strconv.Itoa(index) + "]" + if strings.TrimSpace(stage.ID) == "" || stage.ID != strings.TrimSpace(stage.ID) || seenIDs[stage.ID] { + return fmt.Errorf("%w: %s ID is empty, untrimmed, or duplicated (L-16.1)", ErrUnsupportedPostgresPlan, path) + } + seenIDs[stage.ID] = true + if stage.InputStageID != priorStageID { + return fmt.Errorf("%w: %s input stage %q does not reference prior stage %q — the CTE chain reads exactly the prior relation (L-16.1)", + ErrUnsupportedPostgresPlan, path, stage.InputStageID, priorStageID) + } + if err := validateStageColumns(stage.InputColumns, priorRowIdentity); err != nil { + return fmt.Errorf("%w: %s input schema: %v", ErrUnsupportedPostgresPlan, path, err) + } + if !sameStageColumns(stage.InputColumns, priorColumns) && !groupInputColumnsExtendSource(stage, priorColumns) { + return fmt.Errorf("%w: %s input schema differs from preceding stage output (ir/physical_stage_validation.go:66 re-read at the door)", + ErrUnsupportedPostgresPlan, path) + } + if err := validateStageColumns(stage.OutputColumns, stage.RowIdentityColumn); err != nil { + return fmt.Errorf("%w: %s output schema: %v", ErrUnsupportedPostgresPlan, path, err) + } + if err := validateStageKindPayload(stage, mappingTerminal); err != nil { + return fmt.Errorf("%w: %s: %v", ErrUnsupportedPostgresPlan, path, err) + } + priorStageID = stage.ID + priorColumns = stage.OutputColumns + priorRowIdentity = stage.RowIdentityColumn + } + last := sequence.Stages[len(sequence.Stages)-1] + if sequence.FinalStageID != last.ID { + return fmt.Errorf("%w: final stage ID %q does not reference the last stage %q (L-16.1)", + ErrUnsupportedPostgresPlan, sequence.FinalStageID, last.ID) + } + if sequence.FinalRowIdentity != last.RowIdentityColumn { + return fmt.Errorf("%w: final row identity %q drifts from the last stage identity %q (L-16.1)", + ErrUnsupportedPostgresPlan, sequence.FinalRowIdentity, last.RowIdentityColumn) + } + if !sameStageColumns(sequence.FinalColumns, last.OutputColumns) { + return fmt.Errorf("%w: final columns differ from the last stage output schema (ir/physical_stage_validation.go:286 re-read at the door)", + ErrUnsupportedPostgresPlan) + } + return nil +} + +func validateStageColumns(columns []ir.PhysicalStageColumn, identityColumn string) error { + if len(columns) == 0 { + return fmt.Errorf("a stage schema must declare at least one column") + } + seen := map[string]bool{} + for _, column := range columns { + if strings.TrimSpace(column.Name) == "" { + return fmt.Errorf("column ID %q has an empty name", column.ID) + } + if seen[column.Name] { + return fmt.Errorf("column name %q is declared twice", column.Name) + } + seen[column.Name] = true + } + if identityColumn != "" { + for _, column := range columns { + if column.Name != identityColumn { + continue + } + if !column.Internal || !column.Identity { + return fmt.Errorf("row identity column %q must be internal and identity-marked (ir/physical_stage_validation.go:2634 re-read at the door)", identityColumn) + } + return nil + } + return fmt.Errorf("row identity column %q is not declared in the stage schema", identityColumn) + } + return nil +} + +func sameStageColumns(left, right []ir.PhysicalStageColumn) bool { + if len(left) != len(right) { + return false + } + for i := range left { + a, b := left[i], right[i] + if a.ID != b.ID || a.Name != b.Name || a.Label != b.Label || a.Kind != b.Kind || + a.Cardinality != b.Cardinality || a.Nullable != b.Nullable || a.Internal != b.Internal || + a.Identity != b.Identity || a.RootContributorResourceType != b.RootContributorResourceType || + !reflect.DeepEqual(a.RelatedRecordAnchor, b.RelatedRecordAnchor) { + return false + } + } + return true +} + +func groupInputColumnsExtendSource(stage ir.PhysicalConstructionStage, prior []ir.PhysicalStageColumn) bool { + if stage.Kind != ir.PhysicalStageGroupOp || stage.Group == nil || stage.Group.RelatedOccurrence == nil || + len(stage.InputColumns) <= len(prior) || + !sameStageColumns(stage.InputColumns[:len(prior)], prior) { + return false + } + adapter := stage.Group.RelatedOccurrence + expected := make(map[string]bool, len(adapter.ScalarProjections)+2) + for _, projection := range adapter.ScalarProjections { + if projection.InputColumn == "" || expected[projection.InputColumn] { + return false + } + expected[projection.InputColumn] = true + } + if adapter.CodingMembershipColumn != "" { + if expected[adapter.CodingMembershipColumn] { + return false + } + expected[adapter.CodingMembershipColumn] = true + } + if adapter.OccurrenceKeyColumn != "" { + if expected[adapter.OccurrenceKeyColumn] { + return false + } + expected[adapter.OccurrenceKeyColumn] = true + } + if len(stage.InputColumns)-len(prior) != len(expected) { + return false + } + for _, column := range stage.InputColumns[len(prior):] { + if !expected[column.Name] || !column.Internal || column.ID != column.Name { + return false + } + delete(expected, column.Name) + } + return len(expected) == 0 +} + +func validateStageKindPayload(stage ir.PhysicalConstructionStage, mappingTerminal bool) error { + switch stage.Kind { + case ir.PhysicalStageDeriveOp: + if stage.Filter != nil || stage.Group != nil || stage.Expand != nil || stage.GroupedPivot != nil || + stage.Unpivot != nil || stage.RelatedField != nil || stage.RelatedSource != nil || stage.RelatedExpand != nil { + return fmt.Errorf("DERIVE has mismatched operation payloads (ir/physical_stage_validation.go:82 re-read at the door)") + } + if len(stage.DerivedLets) == 0 { + return fmt.Errorf("DERIVE requires at least one typed expression LET (ir/physical_stage_validation.go:86 re-read at the door)") + } + for index, operation := range stage.DerivedLets { + if operation.Kind != ir.PhysicalExpressionLetOp || operation.ExpressionLet == nil { + return fmt.Errorf("DERIVE derived operation %d has kind %q (only typed expression LETs are registered)", + index, string(operation.Kind)) + } + } + case ir.PhysicalStageFilterOp: + if stage.Filter == nil || stage.Group != nil || stage.Expand != nil || stage.GroupedPivot != nil || + stage.Unpivot != nil || stage.RelatedField != nil || stage.RelatedSource != nil || stage.RelatedExpand != nil || + len(stage.DerivedLets) != 0 { + return fmt.Errorf("FILTER requires only a filter payload (ir/physical_stage_validation.go:92 re-read at the door)") + } + case ir.PhysicalStageExpandOp: + if stage.Expand == nil || stage.Filter != nil || stage.Group != nil || stage.GroupedPivot != nil || + stage.Unpivot != nil || stage.RelatedField != nil || stage.RelatedSource != nil || stage.RelatedExpand != nil || + len(stage.DerivedLets) != 0 { + return fmt.Errorf("EXPAND requires only an expand payload (ir/physical_stage_validation.go:82 shape re-read at the door)") + } + case ir.PhysicalStageUnpivotOp: + if stage.Unpivot == nil || stage.Filter != nil || stage.Group != nil || stage.Expand != nil || + stage.GroupedPivot != nil || stage.RelatedField != nil || stage.RelatedSource != nil || + stage.RelatedExpand != nil || len(stage.DerivedLets) != 0 { + return fmt.Errorf("UNPIVOT requires only an unpivot payload (ir/physical_stage_validation.go:141 re-read at the door)") + } + if stage.OutputRowVariable != stage.Unpivot.OutputRowVariable { + return fmt.Errorf("output row variable does not match unpivot (ir/physical_stage_validation.go:145 re-read at the door)") + } + case ir.PhysicalStageGroupOp: + if stage.Group == nil || stage.Filter != nil || stage.Expand != nil || stage.GroupedPivot != nil || + stage.Unpivot != nil || stage.RelatedField != nil || stage.RelatedSource != nil || + stage.RelatedExpand != nil || len(stage.DerivedLets) != 0 { + return fmt.Errorf("GROUP requires only a group payload (ir/physical_stage_validation.go:155 re-read at the door)") + } + if stage.Group.SemanticKey != nil { + return fmt.Errorf("the semantic family Group key is a registered named gap (L-13.3; enabling contract: the semantic membership walk and matcher) and is refused, never partially rendered") + } + if stage.Group.CodingKey != nil { + return fmt.Errorf("CodingKey group keys are refused at the composition payload gate until their registered arm lands (group.go:444 twin, wave law)") + } + if stage.Group.RootContributorInputColumn != "" && !mappingTerminal { + return fmt.Errorf("the root-contributor set is production-unreachable through the PostgreSQL compile target (L-13.6) and is refused, never partially rendered") + } + case ir.PhysicalStagePivotOp: + if stage.GroupedPivot == nil || stage.Filter != nil || stage.Group != nil || stage.Expand != nil || + stage.Unpivot != nil || stage.RelatedField != nil || stage.RelatedSource != nil || + stage.RelatedExpand != nil || len(stage.DerivedLets) != 0 { + return fmt.Errorf("PIVOT requires only a grouped-pivot payload (ir/physical_stage_validation.go:99 re-read at the door)") + } + if stage.OutputRowVariable != stage.GroupedPivot.OutputRowVariable { + return fmt.Errorf("output row variable does not match grouped pivot (ir/physical_stage_validation.go:102 re-read at the door)") + } + case ir.PhysicalStageRelatedSourceOp, ir.PhysicalStageRelatedExpandOp, + ir.PhysicalStageRelatedEligibilityOp, ir.PhysicalStageRelatedFieldOp: + return fmt.Errorf("the %s stage is a related-route capability and is refused in composition, never partially rendered (the correlated-hop child-set contracts; sets.go:82-95 hop finding cited)", + string(stage.Kind)) + default: + return fmt.Errorf("stage kind %q has no registered composition rendering", string(stage.Kind)) + } + return nil +} + +const ( + stageSourceRelation = stageSequenceCTEPrefix + "src" + stageRelationPrefix = stageSequenceCTEPrefix + "stage_" + stageTerminalAlias = "s" +) + +type stageColumnFact struct { + alias string + jsonb bool +} + +type stageColumnExpr struct { + expr string + jsonb bool +} + +type stageRowScope struct { + qualification string + columns map[string]stageColumnFact +} + +func (r *planRenderer) resolveStageRowValue(scope stageRowScope, value ir.PhysicalValue) (string, error) { + if value.BindKey != "" { + return "", fmt.Errorf("%w: a stage row read must name an intermediate column, not a bind value", + ErrUnsupportedPostgresPlan) + } + if len(value.Path) == 0 { + return "", fmt.Errorf("%w: a stage row read must name at least one intermediate column; the whole intermediate row is a composite, not a registered scalar", + ErrUnsupportedPostgresPlan) + } + fact, ok := scope.columns[value.Path[0]] + if !ok { + return "", fmt.Errorf("%w: stage column %q is not among the registered intermediate columns of the prior stage (L-16.1)", + ErrUnsupportedPostgresPlan, value.Path[0]) + } + base := scope.qualification + `"` + fact.alias + `"` + if len(value.Path) == 1 { + if !fact.jsonb { + return base, nil + } + return r.stageJSONBRead(base), nil + } + if !fact.jsonb { + return "", fmt.Errorf("%w: stage column %q is a SQL scalar and carries no document path %v", + ErrUnsupportedPostgresPlan, value.Path[0], value.Path[1:]) + } + return r.documentPathFromExpression(base, value.Path[1:]) +} + +func (r *planRenderer) stageJSONBRead(base string) string { + return base + " #> $" + strconv.Itoa(r.textParameter("{}", "path:{}")) + "::text[]" +} + +func guardCastJSONB(tag string) string { + return "CAST(" + guardCast(tag) + "::text AS jsonb)" +} + +type stageSequenceChain struct { + ctes []string + relation string + facts map[string]stageColumnFact + sourceFacts map[string]stageColumnFact + last *ir.PhysicalConstructionStage +} + +func (r *planRenderer) renderStageSequenceCTEChain(plan ir.PhysicalPlan) (stageSequenceChain, error) { + sequence := plan.StageSequence + source, sourceFacts, err := r.renderStageSequenceSource(plan) + if err != nil { + return stageSequenceChain{}, err + } + ctes := []string{stageSourceRelation + " AS (" + source + ")"} + relation := stageSourceRelation + facts := sourceFacts + var last *ir.PhysicalConstructionStage + for index := range sequence.Stages { + stage := sequence.Stages[index] + body, stageCTEs, stageFacts, err := r.renderStageCTE(plan, stage, index, relation, facts) + if err != nil { + return stageSequenceChain{}, err + } + ctes = append(ctes, stageCTEs...) + relation = stageRelationPrefix + strconv.Itoa(index) + ctes = append(ctes, relation+" AS ("+body+")") + facts = stageFacts + last = &sequence.Stages[index] + } + if last == nil { + return stageSequenceChain{}, fmt.Errorf("%w: the composition builder requires at least one stage", ErrUnsupportedPostgresPlan) + } + return stageSequenceChain{ctes: ctes, relation: relation, facts: facts, sourceFacts: sourceFacts, last: last}, nil +} + +func (r *planRenderer) renderStageSequenceStatement(plan ir.PhysicalPlan) (string, error) { + sequence := plan.StageSequence + chain, err := r.renderStageSequenceCTEChain(plan) + if err != nil { + return "", err + } + ctes, relation, facts, last := chain.ctes, chain.relation, chain.facts, chain.last + + selects := make([]string, 0, len(sequence.FinalColumns)) + names := make([]string, 0, len(sequence.FinalColumns)) + for index, column := range sequence.FinalColumns { + fact, ok := facts[column.Name] + if !ok { + return "", fmt.Errorf("%w: final column %q is not a column of the last stage (ir/physical_stage_validation.go:286 re-read at the builder)", + ErrUnsupportedPostgresPlan, column.Name) + } + selects = append(selects, stageTerminalAlias+`.`+`"`+fact.alias+`"`+` AS "c`+strconv.Itoa(index)+`"`) + names = append(names, column.Name) + } + final := "SELECT " + strings.Join(selects, ", ") + " FROM " + relation + " AS " + stageTerminalAlias + // D-16a attribution (coordinator ruling 2026-11-11): CTE terminal re-sort over GROUP-last per coordinator ruling 2026-11-11; SQL CTE order unspecified; refines registered group total order. + final += " ORDER BY " + strings.Join(stageTerminalOrder(*last, facts), ", ") + if sequence.PreviewLimitBindKey != "" { + limit, ok := plan.BindVars[sequence.PreviewLimitBindKey].(int) + if !ok || limit <= 0 { + return "", fmt.Errorf("%w: composition preview limit bind %q must be a positive int (render/aql/stage_sequence.go:452-453 twin)", + ErrUnsupportedPostgresPlan, sequence.PreviewLimitBindKey) + } + index := r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(limit)}, "") + final += " LIMIT $" + strconv.Itoa(index) + } + r.outputColumns = names + return "WITH " + strings.Join(ctes, ", ") + " " + final, nil +} + +func stageTerminalOrder(stage ir.PhysicalConstructionStage, facts map[string]stageColumnFact) []string { + read := func(name string) string { + return stageTerminalAlias + "." + `"` + facts[name].alias + `"` + } + if stage.Kind == ir.PhysicalStageGroupOp && stage.Group != nil { + terms := make([]string, 0, len(stage.Group.Keys)+1) + for _, key := range stage.Group.Keys { + collate := "" + if key.Kind == "STRING" { + collate = ` COLLATE "C"` + } + terms = append(terms, read(key.OutputColumn)+collate+" ASC NULLS FIRST") + } + terms = append(terms, read(stage.RowIdentityColumn)+` COLLATE "C" ASC`) + return terms + } + return []string{read(stage.RowIdentityColumn) + ` COLLATE "C" ASC`} +} + +func (r *planRenderer) renderStageSequenceSource(plan ir.PhysicalPlan) (string, map[string]stageColumnFact, error) { + if err := r.renderRootScan(plan, *plan.Operations[0].RootScan); err != nil { + return "", nil, err + } + var returnOp *ir.PhysicalReturn + for _, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalFilterOp: + canonical, err := r.acceptCanonicalScopeFilter(operation.Filter) + if err != nil { + return "", nil, err + } + if canonical { + continue + } + clause, err := r.renderFilter(plan, *operation.Filter) + if err != nil { + return "", nil, err + } + r.where = append(r.where, clause) + case ir.PhysicalDerivedLetOp: + canonical, err := r.acceptCanonicalAuthLet(operation.DerivedLet) + if err != nil { + return "", nil, err + } + if !canonical { + return "", nil, fmt.Errorf("%w: a non-canonical DERIVED_LET has no dispatch arm in the composition source region (L-16.6)", + ErrUnsupportedPostgresPlan) + } + case ir.PhysicalUnnestOp: + if err := r.renderUnnest(plan, *operation.Unnest); err != nil { + return "", nil, err + } + case ir.PhysicalSetOp: + if err := r.renderSet(plan, *operation.Set); err != nil { + return "", nil, err + } + case ir.PhysicalSortOp: + if err := r.renderWindowSort(plan, *operation.Sort); err != nil { + return "", nil, err + } + case ir.PhysicalLimitOp: + if err := r.renderWindowLimit(plan, *operation.Limit); err != nil { + return "", nil, err + } + case ir.PhysicalReturnOp: + returnOp = operation.Return + } + } + addressed := map[string]bool{} + for _, column := range plan.StageSequence.SourceColumns { + addressed[column.Name] = true + } + companionExprs := map[string]string{} + declaredCardinality := map[string]string{} + for _, column := range plan.StageSequence.SourceColumns { + declaredCardinality[column.Name] = strings.ToUpper(strings.TrimSpace(column.Cardinality)) + } + arrayExprs := map[string]string{} + keyExprs := map[string]string{} + setExprs := map[string]string{} + memberWitnessExpr := "" + memberWitnessSeen := false + filtered := ir.PhysicalReturn{} + for _, projection := range returnOp.Projections { + if projection.Name == ir.PhysicalPopulationMappingMembersColumn && r.mapping != nil && r.mapping.witness { + if memberWitnessSeen { + return "", nil, fmt.Errorf("%w: the source RETURN projects the member witness column twice", + ErrUnsupportedPostgresPlan) + } + if !projection.Hidden || projection.Expression != nil || + projection.Value.Variable != ir.PopulationMappingMembersVariable || len(projection.Value.Path) != 0 { + return "", nil, fmt.Errorf("%w: source projection %q is not the collected selected member IDs", + ErrUnsupportedPostgresPlan, projection.Name) + } + if r.mapping.membersColumnRef == "" { + return "", nil, fmt.Errorf("%w: the member witness projection reads a member variable the member-collecting population root did not bind", + ErrUnsupportedPostgresPlan) + } + memberWitnessExpr = r.mapping.membersColumnRef + memberWitnessSeen = true + continue + } + if !addressed[projection.Name] { + continue + } + if projection.Hidden && projection.PresenceOutput && projection.Presence != nil && + len(projection.Presence.Paths) == 1 && len(projection.Presence.Paths[0]) > 0 { + scope, ok := r.rows[projection.Presence.Source.Variable] + if !ok || scope.alias != r.rootAlias { + return "", nil, fmt.Errorf("%w: presence companion %q reads a variable the composition source region has not registered (L-16.6)", + ErrUnsupportedPostgresPlan, projection.Name) + } + presence, err := r.propertyPresenceExpression(r.rootAlias, projection.Presence.Paths[0]) + if err != nil { + return "", nil, err + } + companionExprs[projection.Name] = "(" + presence + ")" + continue + } + if projection.Expression != nil && projection.Expression.Kind == ir.PhysicalSliceExpression && + projection.Expression.Slice != nil { + if _, registered := r.setBodies[projection.Expression.Slice.Source.Variable]; registered { + expression, err := r.renderSliceValue(plan, r.rows, *projection.Expression, 0) + if err != nil { + return "", nil, fmt.Errorf("slice column %q: %w", projection.Name, err) + } + setExprs[projection.Name] = expression + continue + } + } + if projection.Hidden && hiddenRootKeyRead(projection) { + scope, ok := r.rows[projection.Value.Variable] + if !ok || scope.alias != r.rootAlias { + return "", nil, fmt.Errorf("%w: key identity projection %q reads a variable the composition source region has not registered (L-16.6)", + ErrUnsupportedPostgresPlan, projection.Name) + } + keyExprs[projection.Name] = r.rootAlias + "." + r.storage.Resource.MustColumn("logical_id").SQLText() + continue + } + if projection.Expression != nil && projection.Expression.Kind == ir.PhysicalExtractExpression && + projection.Expression.Cardinality != ir.PhysicalArrayCardinality { + if extract := projection.Expression.Extract; extract != nil && + len(extract.Selector.Steps) == 1 && !extract.Selector.Steps[0].Iterate && + extract.Selector.Steps[0].Index == nil && extract.Selector.Filter == nil && + len(extract.Fallbacks) == 0 && !extract.Distinct && + extract.Prepared == nil && extract.UnitNormalization == nil && + len(extract.Source.Path) == 1 && extract.Source.Path[0] == "payload" { + posture := projection.Expression.NullBehavior + if column := canonicalKeyColumn(extract.Selector.Steps[0].Field); resourceFilterColumns[column] && + (posture == ir.PhysicalPreserveNull || posture == ir.PhysicalOmitNulls || + posture == ir.PhysicalEmptyOnNull) { + scope, ok := r.rows[extract.Source.Variable] + if !ok || scope.alias != r.rootAlias { + return "", nil, fmt.Errorf("%w: key selector %q reads a variable the composition source region has not registered (L-16.6)", + ErrUnsupportedPostgresPlan, projection.Name) + } + if extract.ResourceType != scope.resourceType { + return "", nil, fmt.Errorf("%w: key selector %q validates against resource type %q, but the row is scanned as %q", + ErrUnsupportedPostgresPlan, projection.Name, extract.ResourceType, scope.resourceType) + } + keyExprs[projection.Name] = r.rootAlias + "." + + r.storage.Resource.MustColumn(column).SQLText() + continue + } + } + } + if projection.Expression != nil && projection.Expression.Kind == ir.PhysicalExtractExpression && + projection.Expression.Cardinality == ir.PhysicalArrayCardinality { + if declaredCardinality[projection.Name] != "MANY" { + return "", nil, fmt.Errorf("%w: source projection %q is an array EXTRACT but its declared column cardinality is %q (the many storage class is the only registered array reading)", + ErrUnsupportedPostgresPlan, projection.Name, declaredCardinality[projection.Name]) + } + expression, err := r.stageArraySelectorExpression(projection) + if err != nil { + return "", nil, err + } + arrayExprs[projection.Name] = expression + continue + } + filtered.Projections = append(filtered.Projections, projection) + } + columns, err := r.materializeGroupSource(plan, &filtered) + if err != nil { + return "", nil, err + } + byName := map[string]groupColumn{} + for i := range columns { + byName[columns[i].name] = columns[i] + } + selects := make([]string, 0, len(plan.StageSequence.SourceColumns)) + facts := map[string]stageColumnFact{} + for index, column := range plan.StageSequence.SourceColumns { + alias := "c" + strconv.Itoa(index) + if presence, ok := companionExprs[column.Name]; ok { + if !strings.EqualFold(strings.TrimSpace(column.Kind), "boolean") || column.Identity { + return "", nil, fmt.Errorf("%w: source column %q is rendered as a presence companion but is declared kind %q identity=%v", + ErrUnsupportedPostgresPlan, column.Name, column.Kind, column.Identity) + } + selects = append(selects, presence+` AS "`+alias+`"`) + facts[column.Name] = stageColumnFact{alias: alias, jsonb: false} + continue + } + if composite, ok := setExprs[column.Name]; ok { + if err := r.validateStageColumnClass(column, true); err != nil { + return "", nil, fmt.Errorf("%w: source column %q: %v", ErrUnsupportedPostgresPlan, column.Name, err) + } + selects = append(selects, composite+` AS "`+alias+`"`) + facts[column.Name] = stageColumnFact{alias: alias, jsonb: true} + continue + } + if key, ok := keyExprs[column.Name]; ok { + if err := r.validateStageColumnClass(column, false); err != nil { + return "", nil, fmt.Errorf("%w: source column %q: %v", ErrUnsupportedPostgresPlan, column.Name, err) + } + selects = append(selects, key+` AS "`+alias+`"`) + facts[column.Name] = stageColumnFact{alias: alias, jsonb: false} + continue + } + if array, ok := arrayExprs[column.Name]; ok { + if !strings.EqualFold(strings.TrimSpace(column.Cardinality), "many") { + return "", nil, fmt.Errorf("%w: source column %q rendered an array reading under cardinality %q", + ErrUnsupportedPostgresPlan, column.Name, column.Cardinality) + } + selects = append(selects, array+` AS "`+alias+`"`) + facts[column.Name] = stageColumnFact{alias: alias, jsonb: true} + continue + } + source, ok := byName[column.Name] + if !ok { + return "", nil, fmt.Errorf("%w: source projection column %q is not among the typed source RETURN projections (ir/physical_stage_validation.go:46 re-read at the builder)", + ErrUnsupportedPostgresPlan, column.Name) + } + selects = append(selects, source.expr+` AS "`+alias+`"`) + class := isPayloadExpression(source.expr) + if err := r.validateStageColumnClass(column, class); err != nil { + return "", nil, fmt.Errorf("%w: source column %q: %v", ErrUnsupportedPostgresPlan, column.Name, err) + } + facts[column.Name] = stageColumnFact{alias: alias, jsonb: class} + } + if r.mapping != nil && r.mapping.witness { + if !memberWitnessSeen { + return "", nil, fmt.Errorf("%w: the composition mapping source RETURN omits the compiler-owned member witness projection %q (compiler/population_mapping.go adds it)", + ErrUnsupportedPostgresPlan, ir.PhysicalPopulationMappingMembersColumn) + } + memberAlias := "c" + strconv.Itoa(len(plan.StageSequence.SourceColumns)) + selects = append(selects, memberWitnessExpr+` AS "`+memberAlias+`"`) + facts[ir.PhysicalPopulationMappingMembersColumn] = stageColumnFact{alias: memberAlias, jsonb: true} + } + resource := r.storage.Resource + sql := "SELECT " + strings.Join(selects, ", ") + " FROM " + resource.Identifier().SQLText() + " AS " + r.rootAlias + for _, join := range r.joins { + sql += " " + join + } + for _, lateral := range r.laterals { + sql += " " + lateral + } + if len(r.where) > 0 { + sql += " WHERE " + strings.Join(r.where, " AND ") + } + sql += " ORDER BY " + strings.Join(r.windowOrder(), ", ") + r.window.limitClause + return sql, facts, nil +} + +func (r *planRenderer) stageArraySelectorExpression(projection ir.PhysicalProjection) (string, error) { + extract := projection.Expression.Extract + if extract == nil { + return "", fmt.Errorf("%w: array source projection %q: EXTRACT expression is missing its payload", + ErrUnsupportedPostgresPlan, projection.Name) + } + if projection.Expression.NullBehavior != ir.PhysicalEmptyOnNull { + return "", fmt.Errorf("%w: array source projection %q: null behavior %q has no registered array reading (the empty-array posture is EMPTY_ON_NULL)", + ErrUnsupportedPostgresPlan, projection.Name, string(projection.Expression.NullBehavior)) + } + if extract.Selector.Filter != nil || len(extract.Fallbacks) != 0 || extract.Distinct || + extract.Prepared != nil || extract.UnitNormalization != nil { + return "", fmt.Errorf("%w: array source projection %q uses a richer selector form (filter, fallbacks, DISTINCT, prepared, or unit normalization) that is not registered", + ErrUnsupportedPostgresPlan, projection.Name) + } + if len(extract.Source.Path) != 1 || extract.Source.Path[0] != "payload" { + return "", fmt.Errorf("%w: array source projection %q must read the row document at [\"payload\"], got %v", + ErrUnsupportedPostgresPlan, projection.Name, extract.Source.Path) + } + scope, ok := r.rows[extract.Source.Variable] + if !ok || scope.alias != r.rootAlias { + return "", fmt.Errorf("%w: array source projection %q reads a variable the composition source region has not registered (L-16.6)", + ErrUnsupportedPostgresPlan, projection.Name) + } + iterate := -1 + for index, step := range extract.Selector.Steps { + if step.Index != nil { + return "", fmt.Errorf("%w: array source projection %q: indexed steps are the unit slice's registered reading, not the composition array gathering", + ErrUnsupportedPostgresPlan, projection.Name) + } + if !payloadPathSegmentPattern.MatchString(step.Field) { + return "", fmt.Errorf("%w: array source projection %q: selector step %q is not renderer-safe", + ErrUnsupportedPostgresPlan, projection.Name, step.Field) + } + if step.Iterate { + if iterate >= 0 { + return "", fmt.Errorf("%w: array source projection %q: repeated steps beyond one iterate step are the later array-rendering capability", + ErrUnsupportedPostgresPlan, projection.Name) + } + iterate = index + } + } + if iterate < 0 { + return "", fmt.Errorf("%w: array source projection %q has no iterated selector step; a many column needs exactly one [] step", + ErrUnsupportedPostgresPlan, projection.Name) + } + prefixSteps := make([]string, 0, iterate+1) + for _, step := range extract.Selector.Steps[:iterate+1] { + prefixSteps = append(prefixSteps, step.Field) + } + tailSteps := make([]string, 0, len(extract.Selector.Steps)-iterate-1) + for _, step := range extract.Selector.Steps[iterate+1:] { + tailSteps = append(tailSteps, step.Field) + } + arr := r.documentColumn(r.rootAlias) + if len(prefixSteps) > 0 { + var err error + arr, err = r.documentPathFromExpression(arr, prefixSteps) + if err != nil { + return "", fmt.Errorf("array source projection %q: %w", projection.Name, err) + } + } + var leafTail string + if len(tailSteps) == 0 { + leafTail = "{}" + } else { + leafTail = "{" + strings.Join(mapSlice(tailSteps, arrayLiteralElement), ",") + "}" + } + leafTailParameter := r.textParameter(leafTail, "path:"+leafTail) + leaf := "e #> $" + strconv.Itoa(leafTailParameter) + "::text[]" + return "(CASE WHEN jsonb_typeof(" + arr + ") = 'array' THEN COALESCE((SELECT jsonb_agg(" + leaf + + ") FROM jsonb_array_elements(" + arr + ") AS e WHERE " + leaf + " IS NOT NULL), '[]'::jsonb) ELSE '[]'::jsonb END)", nil +} + +func mapSlice(input []string, mapFn func(string) string) []string { + out := make([]string, 0, len(input)) + for _, value := range input { + out = append(out, mapFn(value)) + } + return out +} + +func (r *planRenderer) validateStageColumnClass(column ir.PhysicalStageColumn, jsonb bool) error { + cardinality := strings.ToUpper(strings.TrimSpace(column.Cardinality)) + if cardinality == "MANY" && !jsonb { + return fmt.Errorf("cardinality %q needs the jsonb storage class but the emitter rendered a SQL scalar", column.Cardinality) + } + if column.RootContributorResourceType != "" { + if r.mapping != nil && r.mapping.witness && + column.Internal && column.Name == populationMappingContributorColumn && + strings.EqualFold(strings.TrimSpace(column.Kind), "string") && cardinality == "MANY" { + return nil + } + if !(column.Internal && column.Name == "_key" && + strings.EqualFold(strings.TrimSpace(column.Kind), "string") && + (strings.EqualFold(strings.TrimSpace(column.Cardinality), "required_one") || + strings.EqualFold(strings.TrimSpace(column.Cardinality), "optional_one"))) { + return fmt.Errorf("root-contributor columns are bound only by RenderPopulationRootScan (population.go) and are outside the registered composition surface") + } + } + return nil +} + +func (r *planRenderer) renderStageCTE( + plan ir.PhysicalPlan, + stage ir.PhysicalConstructionStage, + index int, + relation string, + facts map[string]stageColumnFact, +) (body string, ctes []string, out map[string]stageColumnFact, err error) { + r.stageRows = map[string]stageRowScope{} + r.unpivotOutputs = nil + scope := stageRowScope{qualification: stageTerminalAlias + ".", columns: facts} + switch stage.Kind { + case ir.PhysicalStageFilterOp, ir.PhysicalStageDeriveOp: + body, out, err = r.renderPassThroughStage(plan, stage, relation, scope) + case ir.PhysicalStageExpandOp: + body, out, err = r.renderExpandStage(plan, stage, index, relation, scope) + case ir.PhysicalStageUnpivotOp: + body, out, err = r.renderStageMelt(plan, stage, relation, scope) + case ir.PhysicalStageGroupOp: + body, ctes, out, err = r.renderStageGroup(plan, stage, index, relation, facts) + case ir.PhysicalStagePivotOp: + body, ctes, out, err = r.renderStagePivot(plan, stage, index, relation, facts) + default: + return "", nil, nil, fmt.Errorf("%w: stage kind %q has no registered composition rendering", + ErrUnsupportedPostgresPlan, string(stage.Kind)) + } + if err != nil { + return "", nil, nil, err + } + if _, ok := out[stage.RowIdentityColumn]; !ok { + return "", nil, nil, fmt.Errorf("%w: stage %q does not emit its registered row identity column %q (stage identity contract)", + ErrUnsupportedPostgresPlan, stage.ID, stage.RowIdentityColumn) + } + return body, ctes, out, nil +} + +func (r *planRenderer) renderPassThroughStage(plan ir.PhysicalPlan, stage ir.PhysicalConstructionStage, relation string, scope stageRowScope) (string, map[string]stageColumnFact, error) { + r.stageRows[stage.InputRowVariable] = scope + r.stageRows[stage.OutputRowVariable] = scope + direct := map[string]stageColumnExpr{} + for _, operation := range stage.DerivedLets { + derived := operation.ExpressionLet + if derived.Expression.Kind != ir.PhysicalValueExpression || derived.Expression.Value == nil { + return "", nil, fmt.Errorf("%w: a composition DERIVE LET of expression kind %q has no registered rendering; CALL, selector, and unit-normalization expressions belong to their own slice registrations (calls.go, selectors.go, unit_normalization.go) and are refused here (L-16.6)", + ErrUnsupportedPostgresPlan, string(derived.Expression.Kind)) + } + expression, err := r.resolveValue(*derived.Expression.Value) + if err != nil { + return "", nil, fmt.Errorf("derive LET %q: %w", derived.Variable, err) + } + jsonb := isPayloadExpression(expression) + r.rows[derived.Variable] = rowScope{derivedValue: expression, derivedJSON: jsonb} + direct[derived.Variable] = stageColumnExpr{expr: expression, jsonb: jsonb} + } + var where []string + if stage.Filter != nil { + if stage.Filter.Expression != nil && stage.Filter.Expression.Kind == ir.PhysicalExistsPredicate { + return "", nil, fmt.Errorf("%w: an EXISTS filter inside a composition stage is a related-route contract (sets.go renderRelatedExists) and is refused over intermediate rows (L-16.6)", + ErrUnsupportedPostgresPlan) + } + clause, err := r.renderFilter(plan, *stage.Filter) + if err != nil { + return "", nil, err + } + where = append(where, clause) + } + selects, facts, err := r.renderStageOutputSelects(plan, stage, direct, scope) + if err != nil { + return "", nil, err + } + sql := "SELECT " + strings.Join(selects, ", ") + " FROM " + stageRelationSource(relation) + if len(where) > 0 { + sql += " WHERE " + strings.Join(where, " AND ") + } + return sql, facts, nil +} + +func stageRelationSource(relation string) string { + return relation + " AS " + stageTerminalAlias +} + +func (r *planRenderer) renderExpandStage(plan ir.PhysicalPlan, stage ir.PhysicalConstructionStage, index int, relation string, scope stageRowScope) (string, map[string]stageColumnFact, error) { + expand := stage.Expand + if _, ok := plan.BindVars[expand.ConstructionIDBindKey]; !ok { + return "", nil, fmt.Errorf("%w: expand construction ID bind %q is missing", ErrUnsupportedPostgresPlan, expand.ConstructionIDBindKey) + } + input, ok := scope.columns[expand.InputColumn] + if !ok { + return "", nil, fmt.Errorf("%w: expand input column %q is not among the prior stage's registered columns (L-16.1)", + ErrUnsupportedPostgresPlan, expand.InputColumn) + } + if !input.jsonb { + return "", nil, fmt.Errorf("%w: expand input column %q is a SQL scalar and cannot hold an array", ErrUnsupportedPostgresPlan, expand.InputColumn) + } + identityColumn := "" + for _, column := range stage.InputColumns { + if column.Identity { + identityColumn = column.Name + break + } + } + if identityColumn == "" { + return "", nil, fmt.Errorf("%w: expand input has no row identity column (render/aql/construction_expand.go:26 twin)", ErrUnsupportedPostgresPlan) + } + identity, ok := scope.columns[identityColumn] + if !ok { + return "", nil, fmt.Errorf("%w: expand input identity column %q is missing from the prior relation (L-16.1)", + ErrUnsupportedPostgresPlan, identityColumn) + } + typeofTag := r.textParameter("array", "text:typeof:array") + itemTypeOf, err := stageScalarTypeOf(expand.InputKind) + if err != nil { + return "", nil, fmt.Errorf("expand item: %w", err) + } + itemTypeBind := r.textParameter(itemTypeOf, "text:typeof:"+itemTypeOf) + + raw := stageTerminalAlias + "." + `"` + input.alias + `"` + items := "COALESCE(NULLIF(" + raw + ", 'null'::jsonb), '[]'::jsonb)" + guarded := "CASE WHEN NOT (" + raw + " IS NULL OR " + raw + " = 'null'::jsonb OR jsonb_typeof(" + raw + ") = $" + + strconv.Itoa(typeofTag) + "::text) THEN " + guardCastJSONB(stageExpandArrayTypeTag) + if expand.EmptyPolicy == ir.PhysicalUnnestError { + guarded += " WHEN jsonb_array_length(" + items + ") = 0 THEN " + guardCastJSONB(stageExpandEmptyTag) + } + guarded += " ELSE " + items + " END" + + expansion := "jsonb_array_elements(" + guarded + ") WITH ORDINALITY" + join := "CROSS JOIN LATERAL " + expansion + " AS " + expandRelation(index) + "(item, ord)" + if expand.EmptyPolicy == ir.PhysicalUnnestPreserveParent { + join = "LEFT JOIN LATERAL " + expansion + " AS " + expandRelation(index) + "(item, ord) ON TRUE" + } + item := expandRelation(index) + ".item" + ordinal := expandRelation(index) + ".ord - 1" + + itemGuard := "(CASE WHEN " + item + " IS NULL OR " + item + " = 'null'::jsonb OR jsonb_typeof(" + item + + ") = $" + strconv.Itoa(itemTypeBind) + "::text THEN 1 ELSE " + guardCast(stageExpandItemTypeTag) + " END) > 0" + + identityRead := stageTerminalAlias + "." + `"` + identity.alias + `"` + constructionBind := r.textParameter(bindStringVar(plan.BindVars, expand.ConstructionIDBindKey), "") + identityParts := []string{ + `'["input",' || COALESCE(to_jsonb(` + identityRead + `)::text, 'null') || ']'`, + `'["construction",' || to_jsonb($` + strconv.Itoa(constructionBind) + `::text) || ']'`, + `'["ordinal",' || COALESCE((` + expandRelation(index) + `.ord - 1)::text, 'null') || ']'`, + } + identityText := "'[' || " + strings.Join(identityParts, " || ',' || ") + " || ']'" + + direct := map[string]stageColumnExpr{ + expand.OutputColumn: {expr: item, jsonb: true}, + expand.OrdinalColumn: {expr: "(" + ordinal + ")", jsonb: false}, + stage.RowIdentityColumn: {expr: identityText, jsonb: false}, + } + r.stageRows[stage.InputRowVariable] = scope + r.stageRows[stage.OutputRowVariable] = scope + selects, facts, err := r.renderStageOutputSelects(plan, stage, direct, scope) + if err != nil { + return "", nil, err + } + sql := "SELECT " + strings.Join(selects, ", ") + " FROM " + stageRelationSource(relation) + " " + join + + " WHERE " + itemGuard + return sql, facts, nil +} + +func expandRelation(index int) string { return stageSequenceCTEPrefix + "ex" + strconv.Itoa(index) } + +func stageScalarTypeOf(kind string) (string, error) { + switch kind { + case "STRING", "CODE": + return "string", nil + case "INTEGER", "DECIMAL": + return "number", nil + case "BOOLEAN": + return "boolean", nil + default: + return "", fmt.Errorf("%w: expand item kind %q is unsupported", ErrUnsupportedPostgresPlan, kind) + } +} + +func (r *planRenderer) renderStageMelt(plan ir.PhysicalPlan, stage ir.PhysicalConstructionStage, relation string, scope stageRowScope) (string, map[string]stageColumnFact, error) { + unpivot := stage.Unpivot + inputExprs := make(map[string]string, len(unpivot.InputProjections)) + for _, projection := range unpivot.InputProjections { + expression, err := r.resolveStageColumnRead(plan, scope, projection) + if err != nil { + return "", nil, fmt.Errorf("unpivot input %q: %w", projection.Name, err) + } + inputExprs[projection.Name] = expression + } + r.unpivotSeq++ + alias := "x" + strconv.Itoa(r.unpivotSeq) + region, err := r.renderMeltRegion(plan, *unpivot, inputExprs, alias) + if err != nil { + return "", nil, err + } + r.unpivotOutputs = map[string]map[string]string{unpivot.OutputRowVariable: region.columns} + direct := map[string]stageColumnExpr{} + for name, expression := range region.columns { + direct[name] = stageColumnExpr{expr: expression, jsonb: name == unpivot.KeyOutput || name == unpivot.ValueOutput} + } + r.stageRows[stage.InputRowVariable] = scope + selects, facts, err := r.renderStageOutputSelects(plan, stage, direct, scope) + if err != nil { + return "", nil, err + } + sql := "SELECT " + strings.Join(selects, ", ") + " FROM " + stageRelationSource(relation) + " " + region.lateral + if len(region.conjuncts) > 0 { + sql += " WHERE " + strings.Join(region.conjuncts, " AND ") + } + return sql, facts, nil +} + +func (r *planRenderer) resolveStageColumnRead(plan ir.PhysicalPlan, scope stageRowScope, projection ir.PhysicalProjection) (string, error) { + if projection.Expression != nil { + if projection.Expression.Kind == ir.PhysicalObjectLookupExpression && projection.Expression.ObjectLookup != nil && + projection.Expression.Cardinality == ir.PhysicalScalarCardinality && + projection.Expression.NullBehavior == ir.PhysicalPreserveNull { + lookup := projection.Expression.ObjectLookup + if lookup.ObjectVariable == "" || lookup.KeyBindKey == "" { + return "", fmt.Errorf("%w: stage input projection %q: the carry-over object lookup needs its object variable and key bind", + ErrUnsupportedPostgresPlan, projection.Name) + } + if _, registered := r.stageRows[lookup.ObjectVariable]; !registered { + return "", fmt.Errorf("%w: stage input projection %q reads object variable %q, which no stage row registers (L-16.1)", + ErrUnsupportedPostgresPlan, projection.Name, lookup.ObjectVariable) + } + column, ok := plan.BindVars[lookup.KeyBindKey].(string) + if !ok || strings.TrimSpace(column) == "" { + return "", fmt.Errorf("%w: stage input projection %q: carry-over key bind %q is not a non-blank string", + ErrUnsupportedPostgresPlan, projection.Name, lookup.KeyBindKey) + } + return r.resolveStageRowValue(scope, ir.PhysicalValue{Variable: lookup.ObjectVariable, Path: []string{column}}) + } + return "", fmt.Errorf("%w: stage input projection %q uses expression kind %q; staged inputs are named intermediate columns and selector/call forms register with their own slices (L-16.6)", + ErrUnsupportedPostgresPlan, projection.Name, string(projection.Expression.Kind)) + } + if projection.Value.Variable == "" || len(projection.Value.Path) == 0 { + return "", fmt.Errorf("%w: stage input projection %q is not a named intermediate column read", ErrUnsupportedPostgresPlan, projection.Name) + } + return r.resolveStageRowValue(scope, projection.Value) +} + +func (r *planRenderer) renderStageGroup( + plan ir.PhysicalPlan, + stage ir.PhysicalConstructionStage, + index int, + relation string, + facts map[string]stageColumnFact, +) (string, []string, map[string]stageColumnFact, error) { + var relRegion *stageRelatedOccurrenceRegion + source := "SELECT * FROM " + relation + if stage.Group.RelatedOccurrence != nil { + relRendered, relErr := r.renderStageGroupRelatedOccurrence(plan, stage, index, relation, facts) + if relErr != nil { + return "", nil, nil, relErr + } + relRegion = &relRendered + source = relRegion.sourceText + } + columns := make([]groupColumn, 0, len(stage.InputColumns)) + for _, column := range stage.InputColumns { + if relRegion != nil { + if adapted, ok := relRegion.columns[column.Name]; ok { + columns = append(columns, adapted) + continue + } + } + fact, ok := facts[column.Name] + if !ok { + return "", nil, nil, fmt.Errorf("%w: group input column %q is not among the prior stage's registered columns (L-16.1)", + ErrUnsupportedPostgresPlan, column.Name) + } + columns = append(columns, groupColumn{ + name: column.Name, alias: fact.alias, ref: `"` + fact.alias + `"`, jsonb: fact.jsonb, + }) + } + jsonbOutputs := map[string]bool{} + for _, rowValue := range stage.Group.RowValues { + jsonbOutputs[rowValue.Output] = rowValue.Policy == "ALL" + } + if stage.Group.RootContributorOutputColumn != "" { + jsonbOutputs[stage.Group.RootContributorOutputColumn] = true + } + region, err := r.renderGroupAggregateRegion(plan, stage, columns, + source, "__loom_g"+strconv.Itoa(index)+"_", + stage.OutputColumns, stage.RowIdentityColumn) + if err != nil { + return "", nil, nil, err + } + out := factsFromNames(region.names, func(name string) bool { return jsonbOutputs[name] }) + return region.final, region.ctes, out, nil +} + +func (r *planRenderer) renderStagePivot( + plan ir.PhysicalPlan, + stage ir.PhysicalConstructionStage, + index int, + relation string, + facts map[string]stageColumnFact, +) (string, []string, map[string]stageColumnFact, error) { + pivot := stage.GroupedPivot + scope := stageRowScope{qualification: stageTerminalAlias + ".", columns: facts} + projectionNames := make([]string, 0, len(stage.OutputColumns)) + for _, column := range stage.OutputColumns { + projectionNames = append(projectionNames, column.Name) + } + region, err := r.renderGroupedPivotRegion(plan, pivot, "__loom_pv"+strconv.Itoa(index)+"_", + projectionNames, stage.RowIdentityColumn, relation+" AS "+stageTerminalAlias, + func(projection ir.PhysicalProjection) (string, ir.PhysicalNullBehavior, error) { + expression, err := r.resolveStageColumnRead(plan, scope, projection) + if err != nil { + return "", "", err + } + return "to_jsonb(" + expression + ")", ir.PhysicalPreserveNull, nil + }, nil) + if err != nil { + return "", nil, nil, err + } + out := factsFromNames(region.names, func(name string) bool { return name != stage.RowIdentityColumn }) + return region.final, region.ctes, out, nil +} + +func factsFromNames(names []string, jsonb func(string) bool) map[string]stageColumnFact { + facts := make(map[string]stageColumnFact, len(names)) + for index, name := range names { + facts[name] = stageColumnFact{alias: "c" + strconv.Itoa(index), jsonb: jsonb(name)} + } + return facts +} + +func (r *planRenderer) renderStageOutputSelects( + plan ir.PhysicalPlan, + stage ir.PhysicalConstructionStage, + direct map[string]stageColumnExpr, + scope stageRowScope, +) ([]string, map[string]stageColumnFact, error) { + byName := map[string]ir.PhysicalProjection{} + for _, projection := range stage.OutputProjections { + if _, seen := byName[projection.Name]; seen { + return nil, nil, fmt.Errorf("%w: output projection %q is declared twice (stage schema parity)", + ErrUnsupportedPostgresPlan, projection.Name) + } + byName[projection.Name] = projection + } + selects := make([]string, 0, len(stage.OutputColumns)) + facts := make(map[string]stageColumnFact, len(stage.OutputColumns)) + for index, column := range stage.OutputColumns { + var expression string + isJSONB := false + if rendered, ok := direct[column.Name]; ok { + expression, isJSONB = rendered.expr, rendered.jsonb + } else { + projection, ok := byName[column.Name] + if !ok { + return nil, nil, fmt.Errorf("%w: output column %q has no output projection and no registered reshape expression (stage output/input parity)", + ErrUnsupportedPostgresPlan, column.Name) + } + if projection.Presence != nil && projection.Value.Variable == "" { + proof, err := r.stagePresenceProof(scope, projection) + if err != nil { + return nil, nil, err + } + expression, isJSONB = proof, true + } else { + read, err := r.resolveStageColumnRead(plan, scope, projection) + if err != nil { + return nil, nil, fmt.Errorf("output column %q: %w", column.Name, err) + } + expression, isJSONB = read, isPayloadExpression(read) + } + } + if err := r.validateStageColumnClass(column, isJSONB); err != nil { + return nil, nil, fmt.Errorf("%w: stage output column %q: %v", ErrUnsupportedPostgresPlan, column.Name, err) + } + alias := "c" + strconv.Itoa(index) + selects = append(selects, expression+` AS "`+alias+`"`) + facts[column.Name] = stageColumnFact{alias: alias, jsonb: isJSONB} + } + return selects, facts, nil +} + +func (r *planRenderer) stagePresenceProof(scope stageRowScope, projection ir.PhysicalProjection) (string, error) { + if len(projection.Presence.Paths) != 1 || len(projection.Presence.Paths[0]) == 0 { + return "", fmt.Errorf("%w: a stage presence companion needs exactly one column path (null_semantics.go contract)", + ErrUnsupportedPostgresPlan) + } + name := projection.Presence.Paths[0][0] + fact, ok := scope.columns[name] + if !ok { + return "", fmt.Errorf("%w: presence companion %q names no registered intermediate column (L-16.1)", + ErrUnsupportedPostgresPlan, projection.Name) + } + if len(projection.Presence.Paths[0]) > 1 { + return "", fmt.Errorf("%w: a stage presence companion cannot address a document tail path over an intermediate column", + ErrUnsupportedPostgresPlan) + } + base := scope.qualification + `"` + fact.alias + `"` + if !fact.jsonb { + return "(CASE WHEN " + base + " IS NULL THEN 'false'::jsonb ELSE 'true'::jsonb END)", nil + } + return "(CASE WHEN COALESCE(" + base + " IS NULL OR " + base + " = 'null'::jsonb, false) THEN 'false'::jsonb ELSE 'true'::jsonb END)", nil +} diff --git a/internal/dataframe/compiler/render/postgres/stage_sequence_production_test.go b/internal/dataframe/compiler/render/postgres/stage_sequence_production_test.go new file mode 100644 index 000000000..58cfd7ac7 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/stage_sequence_production_test.go @@ -0,0 +1,425 @@ +package postgres + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + control "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" + "github.com/jackc/pgx/v5/pgconn" +) + +func sql16LiveService(t *testing.T) *pgir02Service { + t.Helper() + svc := newOwnedService(t, 0, "verify_sql16v1") + var database string + if err := svc.iso.QueryRow(svc.ctx, "SELECT current_database()").Scan(&database); err != nil { + svc.fail("track owned database", err) + } + sql16TrackResource(t, database) + sql16TrackResource(t, svc.schema) + return svc +} + +func sql16SeedObservation(t *testing.T, svc *pgir02Service, logicalID string, payload map[string]any) { + t.Helper() + svc.insertResource(t, svc.schema, group13Project, group13Generation, "Observation", logicalID, group13AuthPath, payload) +} + +func sql16DriverArgs(rendered command.RenderedPostgresPhysicalPlan) []any { + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + return args +} + +func sql16ThroughStage(t *testing.T, plan ir.PhysicalPlan, through int) ir.PhysicalPlan { + t.Helper() + clone := ir.ClonePhysicalPlan(plan) + if clone.StageSequence == nil || through >= len(clone.StageSequence.Stages) { + t.Fatalf("prefix %d out of range", through) + } + stages := clone.StageSequence.Stages[:through+1] + last := stages[len(stages)-1] + clone.StageSequence.Stages = stages + clone.StageSequence.FinalColumns = append([]ir.PhysicalStageColumn(nil), last.OutputColumns...) + clone.StageSequence.FinalStageID = last.ID + clone.StageSequence.FinalRowIdentity = last.RowIdentityColumn + return clone +} + +func sql16GroupIdentity(constructionID, keyColumnID, keyJSON string) string { + return fmt.Sprintf(`[["construction",%q],["operation","GROUP"],[%q,%s]]`, + constructionID, keyColumnID, keyJSON) +} + +func sql16ExpandIdentity(constructionID, inputIdentityJSON, ordinal string) string { + return fmt.Sprintf(`[["input",%s],["construction",%q],["ordinal",%s]]`, + inputIdentityJSON, constructionID, ordinal) +} + +func sql16PivotIdentity(constructionID, kind, keyJSON string) string { + return fmt.Sprintf(`["GROUPED_PIVOT",%q,[%q,%s]]`, constructionID, kind, keyJSON) +} + +func sql16JSONString(value string) string { + encoded, err := json.Marshal(value) + if err != nil { + panic(err) + } + return string(encoded) +} + +func sql16ExpectRows(t *testing.T, svc *pgir02Service, plan ir.PhysicalPlan, want [][]string) { + t.Helper() + rendered, err := renderPhysicalPlanForSchema(plan, svc.schema) + if err != nil { + svc.fail("render plan", err) + } + got := svc.runRendered(t, rendered) + if len(got) != len(want) { + t.Fatalf("rows = %d %v, want %d %v\nSQL: %s", len(got), got, len(want), want, rendered.SQL) + } + for index := range want { + if strings.Join(got[index], "|") != strings.Join(want[index], "|") { + t.Fatalf("row %d = %v, want %v\nSQL: %s", index, got[index], want[index], rendered.SQL) + } + } +} + +func renderPhysicalPlanForSchema(plan ir.PhysicalPlan, schema string) (command.RenderedPostgresPhysicalPlan, error) { + binding, err := ir.NewPostgresStorageBinding(schema) + if err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return RenderPhysicalPlan(plan, binding) +} + +func sql16ProbeTypes(t *testing.T, svc *pgir02Service, plan ir.PhysicalPlan) []string { + t.Helper() + rendered, err := renderPhysicalPlanForSchema(plan, svc.schema) + if err != nil { + svc.fail("render plan", err) + } + cells := make([]string, 0, len(rendered.OutputColumns)) + for index := range rendered.OutputColumns { + cells = append(cells, fmt.Sprintf(`pg_typeof("c%d")`, index)) + } + probe := "SELECT " + strings.Join(cells, ", ") + " FROM (" + rendered.SQL + ") AS __probe LIMIT 1" + rows := svc.witness(t, probe, sql16DriverArgs(rendered)...) + if len(rows) != 1 { + t.Fatalf("type probe returned %d rows: %v", len(rows), rows) + } + return rows[0] +} + +func TestSQL16LiveExpandGroupAfterEveryStage(t *testing.T) { + svc := sql16LiveService(t) + sql16SeedObservation(t, svc, "obs-1", map[string]any{"status": "final", "note": []any{ + map[string]any{"text": "a"}, map[string]any{"text": "b"}, + }}) + sql16SeedObservation(t, svc, "obs-2", map[string]any{"status": "entered-in-errors", "note": []any{ + map[string]any{"text": "b"}, + }}) + sql16SeedObservation(t, svc, "obs-3", map[string]any{"status": "final", "note": []any{}}) + sql16SeedObservation(t, svc, "obs-4", map[string]any{"status": "final"}) + plan := sql16ExpandGroupPlan(t) + + expand := sql16ThroughStage(t, plan, 0) + sql16ExpectRows(t, svc, expand, [][]string{ + {"final", "a", "0", "obs-1", sql16ExpandIdentity("expand_tag_values", sql16JSONString("obs-1"), "0")}, + {"final", "b", "1", "obs-1", sql16ExpandIdentity("expand_tag_values", sql16JSONString("obs-1"), "1")}, + {"entered-in-errors", "b", "0", "obs-2", sql16ExpandIdentity("expand_tag_values", sql16JSONString("obs-2"), "0")}, + {"final", "", "", "obs-3", sql16ExpandIdentity("expand_tag_values", sql16JSONString("obs-3"), "null")}, + {"final", "", "", "obs-4", sql16ExpandIdentity("expand_tag_values", sql16JSONString("obs-4"), "null")}, + }) + if got := sql16ProbeTypes(t, svc, expand); strings.Join(got, "|") != "jsonb|jsonb|bigint|text|text" { + t.Fatalf("expand stage SQL types = %v, want jsonb|jsonb|bigint|text|text", got) + } + + sql16ExpectRows(t, svc, plan, [][]string{ + {"", "2", "2", "1", sql16GroupIdentity("group_tag_values", "grouped_tag_id", "null")}, + {"a", "1", "1", "1", sql16GroupIdentity("group_tag_values", "grouped_tag_id", `"a"`)}, + {"b", "2", "2", "2", sql16GroupIdentity("group_tag_values", "grouped_tag_id", `"b"`)}, + }) + if got := sql16ProbeTypes(t, svc, plan); strings.Join(got, "|") != "text|bigint|bigint|bigint|text" { + t.Fatalf("group stage SQL types = %v, want text|bigint|bigint|bigint|text", got) + } +} + +func TestSQL16LiveThreeStageAfterEveryStage(t *testing.T) { + svc := sql16LiveService(t) + sql16SeedObservation(t, svc, "obs-1", map[string]any{"status": "final", "note": []any{ + map[string]any{"text": "a"}, map[string]any{"text": "b"}, + }}) + sql16SeedObservation(t, svc, "obs-2", map[string]any{"status": "entered-in-errors", "note": []any{ + map[string]any{"text": "b"}, + }}) + sql16SeedObservation(t, svc, "obs-3", map[string]any{"status": "final", "note": []any{}}) + sql16SeedObservation(t, svc, "obs-4", map[string]any{"status": "final"}) + plans := sql16ThreeStagePlans(t) + + filtered := [][]string{ + {"final", "[a b]", "obs-1"}, + {"final", "[]", "obs-3"}, + {"final", "[]", "obs-4"}, + } + + t1 := plans["group-last"] + sql16ExpectRows(t, svc, sql16ThroughStage(t, t1, 0), filtered) + t1Expand := [][]string{ + {"final", "a", "0", "obs-1", sql16ExpandIdentity("expand_tag_values", sql16JSONString("obs-1"), "0")}, + {"final", "b", "1", "obs-1", sql16ExpandIdentity("expand_tag_values", sql16JSONString("obs-1"), "1")}, + {"final", "", "", "obs-3", sql16ExpandIdentity("expand_tag_values", sql16JSONString("obs-3"), "null")}, + {"final", "", "", "obs-4", sql16ExpandIdentity("expand_tag_values", sql16JSONString("obs-4"), "null")}, + } + sql16ExpectRows(t, svc, sql16ThroughStage(t, t1, 1), t1Expand) + sql16ExpectRows(t, svc, t1, [][]string{ + {"", "2", "2", "1", sql16GroupIdentity("group_tag_values", "grouped_tag_id", "null")}, + {"a", "1", "1", "1", sql16GroupIdentity("group_tag_values", "grouped_tag_id", `"a"`)}, + {"b", "1", "1", "1", sql16GroupIdentity("group_tag_values", "grouped_tag_id", `"b"`)}, + }) + + t2 := plans["pivot-last"] + sql16ExpectRows(t, svc, sql16ThroughStage(t, t2, 0), filtered) + groupIdentity := sql16GroupIdentity("group_status_values", "status_group_id", `"final"`) + sql16ExpectRows(t, svc, sql16ThroughStage(t, t2, 1), [][]string{ + {"final", "final", "3", groupIdentity}, + }) + sql16ExpectRows(t, svc, t2, [][]string{ + {"final", "3", sql16PivotIdentity("tier_pivot", "STRING", `"final"`)}, + }) + + t3 := plans["expand-last"] + sql16ExpectRows(t, svc, sql16ThroughStage(t, t3, 0), filtered) + sql16ExpectRows(t, svc, sql16ThroughStage(t, t3, 1), [][]string{ + {"final", "[final]", "3", groupIdentity}, + }) + sql16ExpectRows(t, svc, t3, [][]string{ + {"final", "3", "final", "0", + sql16ExpandIdentity("expand_status_values", sql16JSONString(groupIdentity), "0")}, + }) +} + +func TestSQL16LiveStagedPivotCoexistence(t *testing.T) { + svc := sql16LiveService(t) + sql16SeedObservation(t, svc, "obs-1", map[string]any{"status": "final", "valueString": "x"}) + sql16SeedObservation(t, svc, "obs-3", map[string]any{"status": "final", "valueString": "y"}) + sql16SeedObservation(t, svc, "obs-4", map[string]any{"status": "final", "valueString": "z"}) + + final := "final" + single := sql16Compile(t, recipe.Output{ + Name: "sql16_single_pivot", RootResourceType: "Observation", RowGrain: "observation", + Fields: []recipe.Field{ + {Name: "status", ColumnID: "status_id", Expr: recipe.Expression{Select: "root.status"}}, + {Name: "class", ColumnID: "class_id", Expr: recipe.Expression{Select: "root.valueString"}}, + {Name: "rid", ColumnID: "rid_id", Expr: recipe.Expression{Select: "root.id"}}, + }, + Construction: &recipe.Construction{Version: 1, + SourceColumns: []recipe.StageColumn{ + {ID: "status_id", Name: "status"}, {ID: "class_id", Name: "class"}, {ID: "rid_id", Name: "rid"}, + }, + Steps: []recipe.ConstructionStep{{ + ID: "pivot_classes", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionPivotOp, Pivot: &recipe.ConstructionPivot{ + ConstructionID: "class_pivot", GroupKeyIDs: []string{"class_id"}, + CategoryColumnID: "status_id", ValueColumnID: "rid_id", + Categories: []recipe.ConstructionPivotCategory{ + {Key: recipe.TableScalar{Kind: recipe.TableScalarString, String: &final}, OutputColumnID: "final_id"}, + }, + DuplicatePolicy: recipe.PivotDuplicateError, MissingCellPolicy: recipe.PivotMissingCellNull, + UnlistedCategoryPolicy: recipe.PivotUnlistedCategoryError, + }}, + Outputs: []recipe.StageColumn{ + {ID: "class_id", Name: "class", Type: "string", Nullable: true}, + {ID: "final_id", Name: "final_cell", Type: "string", Nullable: true}, + }, + }}}, + }) + if single.StageSequence == nil || len(single.StageSequence.Stages) != 1 { + t.Fatalf("single-step pivot compiled without a one-stage sequence: %#v", single.StageSequence) + } + singleRendered := mustRender(t, single, svc.schema) + if !strings.Contains(singleRendered.SQL, "__loom_src AS") { + t.Fatalf("single-stage pivot claim did not render the composition spine:\n%s", singleRendered.SQL) + } + sql16ExpectRows(t, svc, single, [][]string{ + {"x", "obs-1", sql16PivotIdentity("class_pivot", "STRING", `"x"`)}, + {"y", "obs-3", sql16PivotIdentity("class_pivot", "STRING", `"y"`)}, + {"z", "obs-4", sql16PivotIdentity("class_pivot", "STRING", `"z"`)}, + }) + + t2 := sql16ThreeStagePlans(t)["pivot-last"] + sql16SeedObservation(t, svc, "obs-2", map[string]any{"status": "entered-in-errors", "note": []any{}}) + sql16ExpectRows(t, svc, t2, [][]string{ + {"final", "3", sql16PivotIdentity("tier_pivot", "STRING", `"final"`)}, + }) +} + +func TestSQL16LiveSourceRegionSET(t *testing.T) { + svc := sql16LiveService(t) + sql16SeedObservation(t, svc, "obs-1", map[string]any{"status": "final"}) + sql16SeedObservation(t, svc, "obs-2", map[string]any{"status": "entered-in-errors"}) + svc.insertResource(t, svc.schema, group13Project, group13Generation, "Patient", "pat-1", "FHIR/Patient", + map[string]any{"gender": "female", "active": true}) + svc.insertEdge(t, control.EdgeOccurrenceRow{ + Project: group13Project, DatasetGeneration: group13Generation, AuthResourcePath: group13AuthPath, + Label: "subject_Patient", SourceResourceType: "Observation", SourceLogicalID: "obs-1", + TargetResourceType: "Patient", TargetLogicalID: "pat-1", SourcePath: "subject", SourceOrdinal: 0, + }) + + bundle := sql16SETCompositionBundle() + + plan, err := compileWithRegisteredSurfacePolicy(t, bundle) + if err != nil { + t.Fatalf("production compile: %v", err) + } + if plan.StageSequence == nil || len(plan.StageSequence.Stages) != 1 { + t.Fatalf("expected a one-stage composition over the SET source, got %#v", plan.StageSequence) + } + sawSet, windowOpened := false, false + for _, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalSetOp: + if windowOpened { + t.Fatalf("SET must precede the execution window (door law): %#v", plan.Operations) + } + sawSet = true + case ir.PhysicalSortOp, ir.PhysicalLimitOp, ir.PhysicalReturnOp: + windowOpened = true + } + } + if !sawSet { + t.Fatalf("production lowering emitted no child-set SET: %#v", plan.Operations) + } + rendered, err := renderPhysicalPlanForSchema(plan, svc.schema) + if err != nil { + svc.fail("render SET composition", err) + } + rows := svc.runRendered(t, rendered) + if len(rows) != 1 { + t.Fatalf("rows = %v, want exactly the kept final row\nSQL: %s", rows, rendered.SQL) + } + if rows[0][0] != "final" { + t.Fatalf("status cell = %q, want final", rows[0][0]) + } + if !strings.Contains(rows[0][1], "female") || !strings.Contains(rows[0][1], "gender") { + t.Fatalf("slice column %q lost the child-set witness (gender female)\nSQL: %s", rows[0][1], rendered.SQL) + } + if rows[0][len(rows[0])-1] != "obs-1" { + t.Fatalf("pass-through identity = %q, want obs-1", rows[0][len(rows[0])-1]) + } +} + +func compileWithRegisteredSurfacePolicy(t *testing.T, bundle recipe.Bundle) (ir.PhysicalPlan, error) { + t.Helper() + bindings := recipe.RuntimeBindings{ + Project: group13Project, DatasetGeneration: group13Generation, + AuthResourcePaths: []string{group13AuthPath, "FHIR/Patient"}, + } + if err := bundle.Validate(); err != nil { + return ir.PhysicalPlan{}, err + } + semPlan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + return ir.PhysicalPlan{}, err + } + resolved, err := semantic.ResolveRecipePlan(semPlan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + return ir.PhysicalPlan{}, err + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy(). + WithRule(ir.PhysicalOptimizationRuleEndpointTraversal, false)) + if err != nil { + return ir.PhysicalPlan{}, err + } + if len(compiled.Outputs) != 1 { + return ir.PhysicalPlan{}, fmt.Errorf("compiled outputs = %d, want 1", len(compiled.Outputs)) + } + output := compiled.Outputs[0] + if output.OptimizedPlan != nil { + return *output.OptimizedPlan, nil + } + return output.Plan, nil +} + +func TestSQL16LiveTailIndexParity(t *testing.T) { + svc := sql16LiveService(t) + payload := map[string]any{"status": "final", "component": []any{ + map[string]any{"valueQuantity": map[string]any{"value": 7, "system": "urn:not-ucum", "code": "cm"}}, + map[string]any{"valueQuantity": map[string]any{"value": 9, "system": sql21UCUM, "code": "cm"}}, + }} + svc.insertResource(t, svc.schema, "sql16-tail", sql21Gen, "Observation", "sql16-tail", sql21Path, payload) + + tail := sql21IndexedComponentProjection(t, -1) + plan := sql21LivePlan("sql16-tail", "root", tail) + rows := svc.runRendered(t, mustRender(t, plan, svc.schema)) + if len(rows) != 1 || rows[0][0] != "[9]" { + t.Fatalf("tail read = %v, want [9] (o1 receipt: bound negative index reads the last array item)", rows) + } + rendered := mustRender(t, plan, svc.schema) + for _, fragment := range []string{"jsonb_array_length(", "< 0 THEN", "-> ("} { + if !strings.Contains(rendered.SQL, fragment) { + t.Fatalf("tail-arm SQL lost fragment %q:\n%s", fragment, rendered.SQL) + } + } +} + +func TestSQL16LivePreviewLimitAndGuardRaises(t *testing.T) { + svc := sql16LiveService(t) + + sql16SeedObservation(t, svc, "obs-1", map[string]any{"status": "final", "note": []any{ + map[string]any{"text": "a"}, map[string]any{"text": "b"}, + }}) + limited := sql16ExpandGroupPlan(t) + limited.StageSequence.PreviewLimitBindKey = "preview_limit" + limited.BindVars["preview_limit"] = 2 + rendered := mustRender(t, limited, svc.schema) + rows := svc.runRendered(t, rendered) + if len(rows) != 2 { + t.Fatalf("preview limit rows = %v, want the first 2 of the total order", rows) + } + bad := ir.ClonePhysicalPlan(limited) + bad.BindVars["preview_limit"] = 0 + if _, err := renderPhysicalPlanForSchema(bad, svc.schema); err == nil || + !strings.Contains(err.Error(), "preview limit bind") { + t.Fatalf("non-positive preview bind refused with %v, want the registered refusal", err) + } + + sql16SeedObservation(t, svc, "obs-bad-item", map[string]any{"status": "final", "note": []any{ + map[string]any{"text": 42}, + }}) + badItem := sql16ExpandGroupPlan(t) + badItem.BindVars["project"] = group13Project + guarded := mustRender(t, badItem, svc.schema) + planText := strings.Builder{} + for _, row := range svc.witness(t, "EXPLAIN (FORMAT TEXT) "+guarded.SQL, sql16DriverArgs(guarded)...) { + planText.WriteString(strings.Join(row, "\n")) + } + if !strings.Contains(planText.String(), "SubPlan") && !strings.Contains(planText.String(), "Function Scan") { + t.Fatalf("guard subplan missing from plan (barrier folded it):\n%s", planText.String()) + } + _, err := svc.iso.Exec(svc.ctx, guarded.SQL, sql16DriverArgs(guarded)...) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "22P02" || + !strings.Contains(pgErr.Message, "CONSTRUCTION_EXPAND_ITEM_TYPE_MISMATCH") { + t.Fatalf("numeric expand item = %v, want 22P02 naming CONSTRUCTION_EXPAND_ITEM_TYPE_MISMATCH", err) + } + + sql16SeedObservation(t, svc, "obs-bad-key", map[string]any{"status": 42}) + badKey := sql16ThreeStagePlans(t)["pivot-last"] + guardedKey := mustRender(t, badKey, svc.schema) + _, err = svc.iso.Exec(svc.ctx, guardedKey.SQL, sql16DriverArgs(guardedKey)...) + if !errors.As(err, &pgErr) || pgErr.Code != "22P02" || + !strings.Contains(pgErr.Message, "CONSTRUCTION_GROUP_KEY_TYPE_MISMATCH") { + t.Fatalf("numeric group key = %v, want 22P02 naming CONSTRUCTION_GROUP_KEY_TYPE_MISMATCH", err) + } +} diff --git a/internal/dataframe/compiler/render/postgres/stage_sequence_test.go b/internal/dataframe/compiler/render/postgres/stage_sequence_test.go new file mode 100644 index 000000000..fd20bed06 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/stage_sequence_test.go @@ -0,0 +1,1287 @@ +package postgres + +// D-16a attribution (mandatory, coordinator ruling 2026-11-11): CTE terminal re-sort over GROUP-last per coordinator ruling 2026-11-11; SQL CTE order unspecified; refines registered group total order. + +import ( + "os" + "regexp" + "strconv" + "strings" + "sync" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" + "github.com/calypr/loom/internal/dataframe/spec" + fhirschema "github.com/calypr/loom/internal/fhir/schema" +) + +var ( + sql16FixturesMu sync.Mutex + sql16TrackedFixtures = map[string]bool{} +) + +func sql16RunDeltaAudit(inner func() int) int { + return runVerifyNamespaceDeltaAudit("verify_sql16v1_", &sql16FixturesMu, sql16TrackedFixtures, inner) +} + +func sql16TrackResource(t *testing.T, name string) { + t.Helper() + sql16FixturesMu.Lock() + defer sql16FixturesMu.Unlock() + sql16TrackedFixtures[name] = true +} + +func sql16Compile(t *testing.T, output recipe.Output) ir.PhysicalPlan { + t.Helper() + bindings := recipe.RuntimeBindings{ + Project: group13Project, DatasetGeneration: group13Generation, + AuthResourcePaths: []string{group13AuthPath}, + } + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: output.Name, TranslationVersion: "test", + Outputs: []recipe.Output{output}, + } + if err := bundle.Validate(); err != nil { + t.Fatalf("bundle validate: %v", err) + } + plan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("lower.CompileResolvedRecipePlan: %v", err) + } + if len(compiled.Outputs) != 1 { + t.Fatalf("compiled outputs = %d, want 1", len(compiled.Outputs)) + } + return compiled.Outputs[0].Plan +} + +func sql16ExpandGroupPlan(t *testing.T) ir.PhysicalPlan { + t.Helper() + output := recipe.Output{ + Name: "sql16_expand_group", RootResourceType: "Observation", RowGrain: "observation", + Fields: []recipe.Field{ + {Name: "status", ColumnID: "status_id", Expr: recipe.Expression{Select: "root.status"}}, + {Name: "tags", ColumnID: "tags_id", Expr: recipe.Expression{Select: "root.note[].text"}, ValueMode: recipe.ValueModeAll}, + }, + Construction: &recipe.Construction{ + Version: 1, + SourceColumns: []recipe.StageColumn{ + {ID: "status_id", Name: "status"}, {ID: "tags_id", Name: "tags"}, + }, + Steps: []recipe.ConstructionStep{ + { + ID: "expand_tags", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionExpandOp, Expand: &recipe.ConstructionExpand{ + ConstructionID: "expand_tag_values", InputColumnID: "tags_id", OutputColumnID: "tag_id", + OrdinalColumnID: "ordinal_id", EmptyPolicy: recipe.ExpansionPreserveParent, + }}, + Outputs: []recipe.StageColumn{ + {ID: "status_id", Name: "status"}, {ID: "tag_id", Name: "tag"}, {ID: "ordinal_id", Name: "ordinal"}, + }, + }, + { + ID: "group_tags", + Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionStepOutputInput, StepID: "expand_tags"}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionGroupOp, Group: &recipe.ConstructionGroup{ + ConstructionID: "group_tag_values", + Keys: []recipe.ConstructionGroupKey{{InputColumnID: "tag_id", OutputColumnID: "grouped_tag_id"}}, + Aggregates: []recipe.ConstructionGroupAggregate{ + {Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}, + {Operation: recipe.ConstructionGroupCountNonNull, InputColumnID: "status_id", OutputColumnID: "status_count_id"}, + {Operation: recipe.ConstructionGroupCountDistinct, InputColumnID: "status_id", OutputColumnID: "status_distinct_id"}, + }, + }}, + Outputs: []recipe.StageColumn{ + {ID: "grouped_tag_id", Name: "tag"}, {ID: "rows_id", Name: "rows"}, + {ID: "status_count_id", Name: "status_count"}, {ID: "status_distinct_id", Name: "status_distinct"}, + }, + }, + }, + }, + } + plan := sql16Compile(t, output) + sequence := plan.StageSequence + if sequence == nil || len(sequence.Stages) != 2 || + sequence.Stages[0].Kind != ir.PhysicalStageExpandOp || sequence.Stages[1].Kind != ir.PhysicalStageGroupOp { + t.Fatalf("compiled plan is not the EXPAND→GROUP composition: %#v", sequence) + } + return ir.ClonePhysicalPlan(plan) +} + +func sql16FilterStep(id string, inputKind recipe.ConstructionInputKind, inputStep, columnID string) recipe.ConstructionStep { + not := "entered-in-errors" + return recipe.ConstructionStep{ + ID: id, Inputs: []recipe.ConstructionInputRef{{Kind: inputKind, StepID: inputStep}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionFilterOp, Filter: &recipe.ConstructionFilter{ + ColumnID: columnID, Operator: recipe.FilterNotEquals, + Values: []recipe.FilterValue{{Kind: recipe.FilterString, String: ¬}}, + }}, + } +} + +func sql16ExpandStep(id string) recipe.ConstructionStep { + return recipe.ConstructionStep{ + ID: id, Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionStepOutputInput, StepID: "keep_rows"}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionExpandOp, Expand: &recipe.ConstructionExpand{ + ConstructionID: "expand_tag_values", InputColumnID: "tags_id", OutputColumnID: "tag_id", + OrdinalColumnID: "ordinal_id", EmptyPolicy: recipe.ExpansionPreserveParent, + }}, + Outputs: []recipe.StageColumn{ + {ID: "status_id", Name: "status"}, {ID: "tag_id", Name: "tag"}, {ID: "ordinal_id", Name: "ordinal"}, + }, + } +} + +func sql16GroupStep(id, inputStep string) recipe.ConstructionStep { + return recipe.ConstructionStep{ + ID: id, Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionStepOutputInput, StepID: inputStep}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionGroupOp, Group: &recipe.ConstructionGroup{ + ConstructionID: "group_tag_values", + Keys: []recipe.ConstructionGroupKey{{InputColumnID: "tag_id", OutputColumnID: "grouped_tag_id"}}, + Aggregates: []recipe.ConstructionGroupAggregate{ + {Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}, + {Operation: recipe.ConstructionGroupCountNonNull, InputColumnID: "status_id", OutputColumnID: "status_count_id"}, + {Operation: recipe.ConstructionGroupCountDistinct, InputColumnID: "status_id", OutputColumnID: "status_distinct_id"}, + }, + }}, + Outputs: []recipe.StageColumn{ + {ID: "grouped_tag_id", Name: "tag"}, {ID: "rows_id", Name: "rows"}, + {ID: "status_count_id", Name: "status_count"}, {ID: "status_distinct_id", Name: "status_distinct"}, + }, + } +} + +func sql16SourceFields() []recipe.Field { + return []recipe.Field{ + {Name: "status", ColumnID: "status_id", Expr: recipe.Expression{Select: "root.status"}}, + {Name: "tags", ColumnID: "tags_id", Expr: recipe.Expression{Select: "root.note[].text"}, ValueMode: recipe.ValueModeAll}, + } +} + +func sql16SourceColumns() []recipe.StageColumn { + return []recipe.StageColumn{ + {ID: "status_id", Name: "status"}, {ID: "tags_id", Name: "tags"}, + } +} + +func sql16ThreeStagePlans(t *testing.T) map[string]ir.PhysicalPlan { + t.Helper() + plans := map[string]ir.PhysicalPlan{} + + keep := sql16FilterStep("keep_rows", recipe.ConstructionSourceProjectionInput, "", "status_id") + keep.Outputs = []recipe.StageColumn{{ID: "status_id", Name: "status"}, {ID: "tags_id", Name: "tags"}} + plans["group-last"] = sql16Compile(t, recipe.Output{ + Name: "sql16_filter_expand_group", RootResourceType: "Observation", RowGrain: "observation", + Fields: sql16SourceFields(), + Construction: &recipe.Construction{Version: 1, SourceColumns: sql16SourceColumns(), + Steps: []recipe.ConstructionStep{keep, sql16ExpandStep("expand_tags"), sql16GroupStep("group_tags", "expand_tags")}}, + }) + + group := recipe.ConstructionStep{ + ID: "by_status", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionStepOutputInput, StepID: "keep_rows"}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionGroupOp, Group: &recipe.ConstructionGroup{ + ConstructionID: "group_status_values", + Keys: []recipe.ConstructionGroupKey{{InputColumnID: "status_id", OutputColumnID: "status_group_id"}}, + Aggregates: []recipe.ConstructionGroupAggregate{{Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}}, + }}, + RowValues: []recipe.ConstructionRowValue{{InputColumnID: "status_id", OutputColumnID: "tier_id", Policy: recipe.ConstructionRowValueOne}}, + Outputs: []recipe.StageColumn{ + {ID: "status_group_id", Name: "status_group", Type: "string", Nullable: true}, + {ID: "tier_id", Name: "tier", Type: "string", Nullable: true}, + {ID: "rows_id", Name: "rows", Type: "integer"}, + }, + } + final := "final" + pivot := recipe.ConstructionStep{ + ID: "pivot_tiers", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionStepOutputInput, StepID: "by_status"}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionPivotOp, Pivot: &recipe.ConstructionPivot{ + ConstructionID: "tier_pivot", GroupKeyIDs: []string{"status_group_id"}, + CategoryColumnID: "tier_id", ValueColumnID: "rows_id", + Categories: []recipe.ConstructionPivotCategory{{ + Key: recipe.TableScalar{Kind: recipe.TableScalarString, String: &final}, OutputColumnID: "final_rows_id", + }}, + DuplicatePolicy: recipe.PivotDuplicateError, MissingCellPolicy: recipe.PivotMissingCellNull, + UnlistedCategoryPolicy: recipe.PivotUnlistedCategoryError, + }}, + Outputs: []recipe.StageColumn{ + {ID: "status_group_id", Name: "status_group", Type: "string", Nullable: true}, + {ID: "final_rows_id", Name: "final_rows", Type: "integer", Nullable: true}, + }, + } + keepT2 := sql16FilterStep("keep_rows", recipe.ConstructionSourceProjectionInput, "", "status_id") + keepT2.Outputs = []recipe.StageColumn{{ID: "status_id", Name: "status"}, {ID: "tags_id", Name: "tags"}} + plans["pivot-last"] = sql16Compile(t, recipe.Output{ + Name: "sql16_filter_group_pivot", RootResourceType: "Observation", RowGrain: "observation", + Fields: sql16SourceFields(), + Construction: &recipe.Construction{Version: 1, SourceColumns: sql16SourceColumns(), + Steps: []recipe.ConstructionStep{keepT2, group, pivot}}, + }) + + groupT3 := recipe.ConstructionStep{ + ID: "by_status", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionStepOutputInput, StepID: "keep_rows"}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionGroupOp, Group: &recipe.ConstructionGroup{ + ConstructionID: "group_status_values", + Keys: []recipe.ConstructionGroupKey{{InputColumnID: "status_id", OutputColumnID: "status_group_id"}}, + Aggregates: []recipe.ConstructionGroupAggregate{{Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}}, + }}, + RowValues: []recipe.ConstructionRowValue{{InputColumnID: "status_id", OutputColumnID: "status_values_id", Policy: recipe.ConstructionRowValueAll}}, + Outputs: []recipe.StageColumn{ + {ID: "status_group_id", Name: "status_group", Type: "string", Nullable: true}, + {ID: "status_values_id", Name: "status_values", Type: "array"}, + {ID: "rows_id", Name: "rows", Type: "integer"}, + }, + } + expandT3 := recipe.ConstructionStep{ + ID: "expand_values", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionStepOutputInput, StepID: "by_status"}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionExpandOp, Expand: &recipe.ConstructionExpand{ + ConstructionID: "expand_status_values", InputColumnID: "status_values_id", OutputColumnID: "item_id", + OrdinalColumnID: "ordinal_id", EmptyPolicy: recipe.ExpansionPreserveParent, + }}, + Outputs: []recipe.StageColumn{ + {ID: "status_group_id", Name: "status_group", Type: "string", Nullable: true}, + {ID: "rows_id", Name: "rows", Type: "integer"}, + {ID: "item_id", Name: "item"}, {ID: "ordinal_id", Name: "ordinal"}, + }, + } + plans["expand-last"] = sql16Compile(t, recipe.Output{ + Name: "sql16_filter_group_expand", RootResourceType: "Observation", RowGrain: "observation", + Fields: sql16SourceFields(), + Construction: &recipe.Construction{Version: 1, SourceColumns: sql16SourceColumns(), + Steps: []recipe.ConstructionStep{keepT2, groupT3, expandT3}}, + }) + for name, plan := range plans { + if plan.StageSequence == nil || len(plan.StageSequence.Stages) != 3 { + t.Fatalf("%s: compiled plan is not a three-stage composition: %#v", name, plan.StageSequence) + } + } + return plans +} + +func TestSQL16ThreeStageComposedRenders(t *testing.T) { + for _, cell := range []struct { + name string + lastKind ir.PhysicalStageOperationKind + terminalBy string + }{ + {"group-last", ir.PhysicalStageGroupOp, `ORDER BY s."c0" COLLATE "C" ASC NULLS FIRST, s."c4" COLLATE "C" ASC`}, + {"pivot-last", ir.PhysicalStagePivotOp, `COLLATE "C" ASC`}, + {"expand-last", ir.PhysicalStageExpandOp, `ORDER BY s."c4" COLLATE "C" ASC`}, + } { + t.Run(cell.name, func(t *testing.T) { + plans := sql16ThreeStagePlans(t) + plan, ok := plans[cell.name] + if !ok { + t.Fatalf("no %s triple", cell.name) + } + last := plan.StageSequence.Stages[2] + if last.Kind != cell.lastKind { + t.Fatalf("%s: last stage kind = %q, want %q", cell.name, last.Kind, cell.lastKind) + } + rendered := sql16Render(t, plan) + sql := rendered.SQL + for _, fragment := range []string{ + "WITH __loom_src AS (", + "__loom_stage_0 AS (", "__loom_stage_1 AS (", "__loom_stage_2 AS (", + "FROM __loom_src AS", "FROM __loom_stage_2 AS s", + cell.terminalBy, + } { + if !strings.Contains(sql, fragment) { + t.Fatalf("%s: composed SQL lost fragment %q:\n%s", cell.name, fragment, sql) + } + } + for _, relation := range []string{"__loom_src", "__loom_stage_0", "__loom_stage_1", "__loom_stage_2"} { + if strings.Count(sql, "FROM "+relation) < 1 { + t.Fatalf("%s: relation %s is never read:\n%s", cell.name, relation, sql) + } + } + if strings.Contains(sql, "__loom_stage_3") { + t.Fatalf("%s: phantom stage relation:\n%s", cell.name, sql) + } + if strings.Join(rendered.OutputColumns, ",") != strings.Join(finalColumnNames(plan), ",") { + t.Fatalf("%s: OutputColumns %v != FinalColumns %v", cell.name, + rendered.OutputColumns, finalColumnNames(plan)) + } + }) + } +} + +func finalColumnNames(plan ir.PhysicalPlan) []string { + var names []string + for _, column := range plan.StageSequence.FinalColumns { + names = append(names, column.Name) + } + return names +} + +func TestSQL16StageSchemaContract(t *testing.T) { + plans := sql16ThreeStagePlans(t) + for name, base := range plans { + t.Run(name, func(t *testing.T) { + plan := ir.ClonePhysicalPlan(base) + sequence := plan.StageSequence + for index, stage := range sequence.Stages { + wantInput := sequence.SourceStageID + if index > 0 { + wantInput = sequence.Stages[index-1].ID + } + if stage.InputStageID != wantInput { + t.Fatalf("stage %d: InputStageID = %q, want %q (adjacent chain)", index, stage.InputStageID, wantInput) + } + identityHidden := false + for _, projection := range stage.OutputProjections { + if projection.Name == stage.RowIdentityColumn { + identityHidden = projection.Hidden + } + } + if !identityHidden { + t.Fatalf("stage %d: row identity column %q is not the hidden projection", index, stage.RowIdentityColumn) + } + } + for index := 1; index < len(sequence.Stages); index++ { + in := map[string]ir.PhysicalStageColumn{} + for _, column := range sequence.Stages[index].InputColumns { + in[column.Name] = column + } + for _, column := range publicColumnNames(sequence.Stages[index-1].OutputColumns) { + if _, ok := in[column]; !ok { + t.Fatalf("stage %d: prior public column %q is not an input", index, column) + } + } + for _, column := range sequence.Stages[index].InputColumns { + if !column.Internal && column.Name != "_key" { + if nameIn(column.Name, publicColumnNames(sequence.Stages[index-1].OutputColumns)) { + continue + } + t.Fatalf("stage %d: input column %q is neither prior-public nor internal", index, column.Name) + } + } + } + skewed := ir.ClonePhysicalPlan(base) + if skewed.StageSequence.Stages[1].InputColumns[0].Kind != "" { + skewed.StageSequence.Stages[1].InputColumns[0].Kind = "INTEGER" + } else { + skewed.StageSequence.Stages[1].InputColumns[0].Kind = "__INVENTED__" + } + sql16RequireRefusal(t, skewed, "input schema differs") + }) + } +} + +func nameIn(name string, names []string) bool { + for _, candidate := range names { + if candidate == name { + return true + } + } + return false +} + +func publicColumnNames(columns []ir.PhysicalStageColumn) []string { + var names []string + for _, column := range columns { + if !column.Internal { + names = append(names, column.Name) + } + } + return names +} + +func columnNames(columns []ir.PhysicalStageColumn) []string { + var names []string + for _, column := range columns { + names = append(names, column.Name) + } + return names +} + +func sql16Render(t *testing.T, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + if !stageSequenceCompositionMentions(plan) { + t.Fatalf("plan does not mention the composition surface: %#v", plan.StageSequence) + } + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("composition render: %v", err) + } + return rendered +} + +func sql16RequireRefusal(t *testing.T, plan ir.PhysicalPlan, want string) { + t.Helper() + _, entryErr := RenderPhysicalPlan(plan, mustBinding(t)) + if entryErr == nil || (!strings.Contains(entryErr.Error(), ErrUnsupportedPostgresPlan.Error()) && + !strings.HasPrefix(entryErr.Error(), "validate physical plan:")) || + !strings.Contains(entryErr.Error(), want) { + t.Fatalf("entry refusal = %v, want typed refusal containing %q", entryErr, want) + } + _, doorErr := renderStageSequencePhysicalPlan(plan, mustBinding(t)) + if doorErr == nil || doorErr.Error() != entryErr.Error() { + t.Fatalf("door refused differently: entry=%v door=%v", entryErr, doorErr) + } +} + +func TestSQL16TwoStageComposedRender(t *testing.T) { + rendered := sql16Render(t, sql16ExpandGroupPlan(t)) + sql := rendered.SQL + for _, fragment := range []string{ + "WITH __loom_src AS (", + "__loom_stage_0 AS (", + "__loom_stage_1 AS (", + "FROM __loom_src AS", + "FROM __loom_stage_0)", + "FROM __loom_stage_1 AS s", + `AS "c0"`, + `AS "c1"`, + `AS "c2"`, + `AS "c3"`, + "jsonb_array_elements(", + `jsonb_typeof(r0."payload" #> $`, + `jsonb_array_elements(r0."payload" #> $`, + "WITH ORDINALITY", + "ORDER BY", + `COLLATE "C"`, + } { + if !strings.Contains(sql, fragment) { + t.Fatalf("composed SQL lost fragment %q:\n%s", fragment, sql) + } + } + if strings.Count(sql, "FROM __loom_src") != 1 || + strings.Count(sql, "FROM __loom_stage_0") != 1 || + strings.Count(sql, "FROM __loom_stage_1") != 1 { + t.Fatalf("CTE chain relations wrong (each must be read exactly once):\n%s", sql) + } + if got, want := rendered.OutputColumns, + []string{"tag", "rows", "status_count", "status_distinct", "__loom_row_id"}; len(got) != len(want) { + t.Fatalf("output columns = %v, want %v", got, want) + } else { + for i := range want { + if got[i] != want[i] { + t.Fatalf("output column %d = %q, want %q", i, got[i], want[i]) + } + } + } + if strings.Contains(sql, "__loom_root_contributor") { + t.Fatalf("population-bound contributor column leaked into the composition:\n%s", sql) + } +} + +func TestSQL16ChainRefusals(t *testing.T) { + base := sql16ExpandGroupPlan(t) + if _, err := RenderPhysicalPlan(ir.ClonePhysicalPlan(base), mustBinding(t)); err != nil { + t.Fatalf("ALIVE control: the canonical composition must render: %v", err) + } + for _, cell := range []struct { + name string + mutate func(*ir.PhysicalPlan) + want string + }{ + {"zero-stages", func(p *ir.PhysicalPlan) { p.StageSequence.Stages = nil }, + "at least one construction stage"}, + {"stage-id-empty", func(p *ir.PhysicalPlan) { p.StageSequence.Stages[0].ID = "" }, + "ID is empty, untrimmed, or duplicated (L-16.1)"}, + {"non-adjacent-input", func(p *ir.PhysicalPlan) { + p.StageSequence.Stages[1].InputStageID = p.StageSequence.SourceStageID + }, "does not reference prior stage"}, + {"self-input", func(p *ir.PhysicalPlan) { + p.StageSequence.Stages[1].InputStageID = p.StageSequence.Stages[1].ID + }, "does not reference prior stage"}, + {"final-stage-wrong", func(p *ir.PhysicalPlan) { + p.StageSequence.FinalStageID = p.StageSequence.Stages[0].ID + }, "final stage ID"}, + {"final-identity-drift", func(p *ir.PhysicalPlan) { + p.StageSequence.FinalRowIdentity = "__loom_other_id" + }, "final row identity"}, + {"group-input-unknown-column", func(p *ir.PhysicalPlan) { + p.StageSequence.Stages[1].Group.Keys[0].InputColumn = "no_such_column" + }, "is not a public scalar input"}, + {"row-identity-not-identity-marked", func(p *ir.PhysicalPlan) { + last := &p.StageSequence.Stages[len(p.StageSequence.Stages)-1] + for i := range last.OutputColumns { + if last.OutputColumns[i].Name == last.RowIdentityColumn { + last.OutputColumns[i].Identity = false + } + } + }, "must be internal and identity-marked"}, + {"row-identity-not-internal", func(p *ir.PhysicalPlan) { + last := &p.StageSequence.Stages[len(p.StageSequence.Stages)-1] + for i := range last.OutputColumns { + if last.OutputColumns[i].Name == last.RowIdentityColumn { + last.OutputColumns[i].Internal = false + } + } + }, "must be internal and identity-marked"}, + } { + t.Run(cell.name, func(t *testing.T) { + plan := ir.ClonePhysicalPlan(base) + cell.mutate(&plan) + sql16RequireRefusal(t, plan, cell.want) + }) + } +} + +func TestSQL16RefusalBattery(t *testing.T) { + base := sql16ExpandGroupPlan(t) + if _, err := RenderPhysicalPlan(ir.ClonePhysicalPlan(base), mustBinding(t)); err != nil { + t.Fatalf("ALIVE control: the canonical composition must render: %v", err) + } + for _, cell := range []struct { + name string + mutate func(*ir.PhysicalPlan) + want string + }{ + {"C-10 cohort-group", func(p *ir.PhysicalPlan) { + p.StageSequence.Stages[1].Kind = ir.PhysicalStageCohortGroupOp + }, "COHORT_GROUP stage (index 1) is not part of the registered composition surface"}, + {"C-11 related-source", func(p *ir.PhysicalPlan) { + p.StageSequence.Stages[1].Kind = ir.PhysicalStageRelatedSourceOp + }, "the RELATED_SOURCE stage is a related-route capability and is refused in composition"}, + {"C-13 coded-group voice", func(p *ir.PhysicalPlan) { + p.StageSequence.Stages[1].Kind = ir.PhysicalStageCodedGroupOp + }, "the registered coded-group slice admits exactly one CODED_GROUP construction stage; multi-stage composition is a pg-sql-16 capability gap"}, + {"C-14 coded pivot trailing voice", func(p *ir.PhysicalPlan) { + p.StageSequence.Stages[1].GroupedPivot = &ir.PhysicalGroupedPivot{ + CodedCorrelation: &ir.PhysicalCorrelation{}, + } + }, "violates the ir rule that the coded Pivot must be the first stage over the direct source projection"}, + {"C-16 lineage return", func(p *ir.PhysicalPlan) { + p.StageSequence.RowLineageReturn = &ir.PhysicalRowLineageReturn{} + }, "row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route"}, + {"C-17 sequence window", func(p *ir.PhysicalPlan) { + p.StageSequence.PreviewSourceWindowByRootID = true + }, "preview source windowing and the terminal Pivot window are pg-sql-22/23 terminal windowing contracts"}, + {"C-17 plan window", func(p *ir.PhysicalPlan) { + p.PreviewSourceWindowByRootID = true + }, "preview source windowing and the terminal Pivot window are pg-sql-22/23 terminal windowing contracts"}, + {"C-18 kind payload mismatch", func(p *ir.PhysicalPlan) { + p.StageSequence.Stages[1].Group = nil + }, "re-read at the door"}, + {"preview limit non-positive", func(p *ir.PhysicalPlan) { + p.StageSequence.PreviewLimitBindKey = "limit" + p.BindVars["limit"] = 0 + }, "composition preview limit bind"}, + {"group-arm-semantic", func(p *ir.PhysicalPlan) { + p.StageSequence.Stages[1].Group.SemanticKey = &ir.PhysicalStageGroupSemanticKey{} + }, "the semantic family Group key is a registered named gap (L-13.3"}, + {"group-arm-related-occurrence", func(p *ir.PhysicalPlan) { + p.StageSequence.Stages[1].Group.RelatedOccurrence = &ir.PhysicalStageGroupRelatedOccurrenceAdapter{} + }, "unreferenced related route adapter"}, + {"group-arm-coding-key", func(p *ir.PhysicalPlan) { + p.StageSequence.Stages[1].Group.CodingKey = &ir.PhysicalStageGroupCodingKey{} + }, "CodingKey group keys are refused at the composition payload gate"}, + {"group-arm-root-contributor", func(p *ir.PhysicalPlan) { + p.StageSequence.Stages[1].Group.RootContributorInputColumn = "status" + }, "the root-contributor set is production-unreachable"}, + {"D-16f correlated filter predicate", func(p *ir.PhysicalPlan) { + last := len(p.Operations) - 1 + filterOp := ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Predicate: ir.PhysicalPredicate{ + Operator: "EXISTS", + Correlation: &ir.PhysicalCorrelation{ + Source: ir.PhysicalValue{Variable: p.Operations[0].RootScan.Variable, Path: []string{"payload"}}, + ResourceType: "Observation", + OwnerResource: "CodeableConcept", + KeyResource: "Coding", + KeyIsArray: true, + KeySelector: spec.Selector{Steps: []spec.SelectorStep{{Field: "coding", Iterate: true}}}, + ValueScope: fhirschema.CorrelatedValueScopeCoding, + SystemSelector: spec.Selector{Steps: []spec.SelectorStep{{Field: "system"}}}, + CodeSelector: spec.Selector{Steps: []spec.SelectorStep{{Field: "code"}}}, + ValueSelector: spec.Selector{Steps: []spec.SelectorStep{{Field: "display"}}}, + LogicalType: "string", + SystemBindKey: "corrsys", + CodeBindKey: "corrcode", + }, + }, + }} + ops := append([]ir.PhysicalOperation{}, p.Operations[:last]...) + p.Operations = append(append(ops, filterOp), p.Operations[last]) + p.BindVars["corrsys"] = "http://loinc.org" + p.BindVars["corrcode"] = "systolic" + }, "correlated predicate"}, + {"D-16f version selector without bind", func(p *ir.PhysicalPlan) { + last := len(p.Operations) - 1 + filterOp := ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Predicate: ir.PhysicalPredicate{ + Operator: "EXISTS", + Correlation: &ir.PhysicalCorrelation{ + Source: ir.PhysicalValue{Variable: p.Operations[0].RootScan.Variable, Path: []string{"payload"}}, + ResourceType: "Observation", + OwnerResource: "CodeableConcept", + KeyResource: "Coding", + KeyIsArray: true, + KeySelector: spec.Selector{Steps: []spec.SelectorStep{{Field: "coding", Iterate: true}}}, + ValueScope: fhirschema.CorrelatedValueScopeCoding, + SystemSelector: spec.Selector{Steps: []spec.SelectorStep{{Field: "system"}}}, + CodeSelector: spec.Selector{Steps: []spec.SelectorStep{{Field: "code"}}}, + ValueSelector: spec.Selector{Steps: []spec.SelectorStep{{Field: "display"}}}, + LogicalType: "string", + VersionSelector: &spec.Selector{Steps: []spec.SelectorStep{{Field: "version"}}}, + }, + }, + }} + ops := append([]ir.PhysicalOperation{}, p.Operations[:last]...) + p.Operations = append(append(ops, filterOp), p.Operations[last]) + }, "correlation version bind key is required"}, + {"auth path bind", func(p *ir.PhysicalPlan) { + p.StageSequence.OutputAuthResourcePathBindKey = "auth_path" + }, "composite Combine prefix authorization path is not registered"}, + } { + t.Run(cell.name, func(t *testing.T) { + plan := ir.ClonePhysicalPlan(base) + cell.mutate(&plan) + sql16RequireRefusal(t, plan, cell.want) + }) + } +} + +func TestSQL16CodedVoicesByteEqual(t *testing.T) { + doorPlan := sql16ExpandGroupPlan(t) + doorPlan.StageSequence.Stages[0].Kind = ir.PhysicalStageCodedGroupOp + _, doorErr := renderCodedGroupPhysicalPlan(doorPlan, mustBinding(t)) + if doorErr == nil || !strings.Contains(doorErr.Error(), + "the registered coded-group slice admits exactly one CODED_GROUP construction stage") { + t.Fatalf("coded-group door voice drifted: %v", doorErr) + } + composed := sql16ExpandGroupPlan(t) + composed.StageSequence.Stages[1].Kind = ir.PhysicalStageCodedGroupOp + _, compositionErr := renderStageSequencePhysicalPlan(composed, mustBinding(t)) + if compositionErr == nil { + t.Fatal("composition must refuse a CODED_GROUP stage") + } + if !strings.Contains(compositionErr.Error(), + "the registered coded-group slice admits exactly one CODED_GROUP construction stage; multi-stage composition is a pg-sql-16 capability gap") { + t.Fatalf("composition voice drifted from the coded-group door:\ncoded=%q\ncomposition=%q", doorErr, compositionErr) + } +} + +func TestSQL16LawCells(t *testing.T) { + source, err := os.ReadFile("stage_sequence.go") + if err != nil { + t.Fatalf("read stage_sequence.go: %v", err) + } + const attribution = "CTE terminal re-sort over GROUP-last per coordinator ruling 2026-11-11; SQL CTE order unspecified; refines registered group total order" + if !strings.Contains(string(source), attribution) { + t.Fatal("stage_sequence.go lost the mandatory D-16a attribution line") + } + ownSource, err := os.ReadFile("stage_sequence_test.go") + if err != nil { + t.Fatalf("read stage_sequence_test.go: %v", err) + } + if !strings.Contains(string(ownSource), "// D-16a attribution (mandatory, coordinator ruling 2026-11-11): "+attribution) { + t.Fatal("the terminal-order test register lost the verbatim D-16a attribution comment line") + } + rendered := sql16Render(t, sql16ExpandGroupPlan(t)) + for _, forbidden := range []string{"; ", "DROP ", "DELETE ", "INSERT ", "UPDATE ", "SET search_path"} { + if strings.Contains(strings.ToUpper(rendered.SQL), strings.ToUpper(forbidden)) { + t.Fatalf("forbidden fragment %q in composed SQL:\n%s", forbidden, rendered.SQL) + } + } + plan := sql16ExpandGroupPlan(t) + plan.StageSequence.PreviewLimitBindKey = "limit" + plan.BindVars = map[string]any{"limit": "7"} + _, err = RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil || !strings.Contains(err.Error(), "must be a positive int") { + t.Fatalf("string-bound preview limit must be a typed refusal, got %v", err) + } + if plan.BindVars == nil { + plan.BindVars = map[string]any{} + } + plan.BindVars["limit"] = 0 + if _, err = RenderPhysicalPlan(plan, mustBinding(t)); err == nil || + !strings.Contains(err.Error(), "must be a positive int") { + t.Fatalf("zero preview limit must be a typed refusal, got %v", err) + } +} + +func TestSQL16KeySelectorSidecar(t *testing.T) { + final := "final" + plan := sql16Compile(t, recipe.Output{ + Name: "sql16_key_sidecar", RootResourceType: "Observation", RowGrain: "observation", + Fields: []recipe.Field{ + {Name: "status", ColumnID: "status_id", Expr: recipe.Expression{Select: "root.status"}}, + {Name: "rid", ColumnID: "rid_id", Expr: recipe.Expression{Select: "root.id"}}, + {Name: "val", ColumnID: "val_id", Expr: recipe.Expression{Select: "root.valueString"}}, + }, + Construction: &recipe.Construction{Version: 1, + SourceColumns: []recipe.StageColumn{ + {ID: "status_id", Name: "status"}, {ID: "rid_id", Name: "rid"}, {ID: "val_id", Name: "val"}, + }, + Steps: []recipe.ConstructionStep{{ + ID: "pivot_status", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionPivotOp, Pivot: &recipe.ConstructionPivot{ + ConstructionID: "status_pivot", GroupKeyIDs: []string{"val_id"}, + CategoryColumnID: "status_id", ValueColumnID: "rid_id", + Categories: []recipe.ConstructionPivotCategory{ + {Key: recipe.TableScalar{Kind: recipe.TableScalarString, String: &final}, OutputColumnID: "final_id"}, + }, + DuplicatePolicy: recipe.PivotDuplicateError, MissingCellPolicy: recipe.PivotMissingCellNull, + UnlistedCategoryPolicy: recipe.PivotUnlistedCategoryError, + }}, + Outputs: []recipe.StageColumn{ + {ID: "val_id", Name: "val", Type: "string", Nullable: true}, + {ID: "final_id", Name: "final_cell", Type: "string", Nullable: true}, + }, + }}}, + }) + if plan.StageSequence == nil || len(plan.StageSequence.Stages) != 1 { + t.Fatalf("plan is not a one-stage composition: %#v", plan.StageSequence) + } + rendered, err := RenderPhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if err != nil { + t.Fatalf("entry render: %v", err) + } + sql := rendered.SQL + if !strings.Contains(sql, `r0."logical_id" AS "c1"`) { + t.Fatalf("key selector did not render the logical_id sidecar read:\n%s", sql) + } + if !strings.Contains(sql, `to_jsonb(s."c1")`) { + t.Fatalf("scalar key fact was re-read as a jsonb path:\n%s", sql) + } + door, err := renderStageSequencePhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if err != nil { + t.Fatalf("direct door: %v", err) + } + if door.SQL != sql { + t.Fatalf("door SQL drifted from the entry render") + } +} + +func sql16SETCompositionBundle() recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "sql16_source_region_set", TranslationVersion: "test", + Outputs: []recipe.Output{{ + Name: "set_composition", RootResourceType: "Observation", RowGrain: "resource", + Fields: []recipe.Field{{Name: "status", ColumnID: "status_id", Expr: recipe.Expression{Select: "root.status"}}}, + Traversals: []recipe.Traversal{{ + Name: "subject_Patient", Alias: "occ_patient", ToResourceType: "Patient", MatchMode: recipe.MatchOptional, + Slices: []recipe.RepresentativeSlice{{ + Name: "representatives", Limit: 1, + Fields: []recipe.Field{ + {Name: "gender", Expr: recipe.Expression{Select: "occ_patient.gender"}}, + {Name: "active", Expr: recipe.Expression{Select: "occ_patient.active"}}, + }, + }}, + }}, + Construction: &recipe.Construction{Version: 1, + SourceColumns: []recipe.StageColumn{ + {ID: "status_id", Name: "status"}, + {ID: "reps_id", Name: "occ_patient__representatives", Type: "array"}, + }, + Steps: []recipe.ConstructionStep{{ + ID: "keep_status", + Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionFilterOp, Filter: &recipe.ConstructionFilter{ + ColumnID: "status_id", Operator: recipe.FilterEquals, + Values: []recipe.FilterValue{{Kind: recipe.FilterString, String: func() *string { v := "final"; return &v }()}}, + }}, + Outputs: []recipe.StageColumn{ + {ID: "status_id", Name: "status"}, + {ID: "reps_id", Name: "occ_patient__representatives", Type: "array", Nullable: true}, + }, + }}}, + }}, + } +} + +func TestSQL16SETSliceSourceColumn(t *testing.T) { + plan, err := compileWithRegisteredSurfacePolicy(t, sql16SETCompositionBundle()) + if err != nil { + t.Fatalf("production compile: %v", err) + } + if plan.StageSequence == nil || len(plan.StageSequence.Stages) != 1 { + t.Fatalf("not a one-stage composition: %#v", plan.StageSequence) + } + rendered, err := RenderPhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if err != nil { + t.Fatalf("entry render: %v", err) + } + sql := rendered.SQL + for _, fragment := range []string{ + `fhir_edge_occurrence" AS e1`, + `e1."source_logical_id" = r0."logical_id"`, + `jsonb_build_object(`, + `COALESCE(jsonb_agg(sl2."item" ORDER BY`, + `AS "c1"`, + } { + if !strings.Contains(sql, fragment) { + t.Fatalf("SET slice column lost fragment %q:\n%s", fragment, sql) + } + } + door, err := renderStageSequencePhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if err != nil { + t.Fatalf("direct door: %v", err) + } + if door.SQL != sql { + t.Fatalf("door SQL drifted from the entry render") + } +} + +func TestSQL16ArraySelectorIteratePathBind(t *testing.T) { + rendered := sql16Render(t, sql16ExpandGroupPlan(t)) + sql := rendered.SQL + guard := regexp.MustCompile(`jsonb_typeof\(r0\."payload" #> \$(\d+)::text\[\]\) = 'array'`).FindStringSubmatch(sql) + gather := regexp.MustCompile(`FROM jsonb_array_elements\(r0\."payload" #> \$(\d+)::text\[\]\) AS e`).FindStringSubmatch(sql) + if guard == nil || gather == nil { + t.Fatalf("the guarded gather lost its path-bound payload read:\n%s", sql) + } + if guard[1] != gather[1] { + t.Fatalf("the typeof guard and the elements gather read different paths (guard $%s, gather $%s):\n%s", guard[1], gather[1], sql) + } + pathIndex, err := strconv.Atoi(guard[1]) + if err != nil || pathIndex < 1 || pathIndex > len(rendered.Args) { + t.Fatalf("gather path parameter $%s is not a minted bind", guard[1]) + } + if bound, ok := rendered.Args[pathIndex-1].Value.(string); !ok || bound != "{note}" { + t.Fatalf("the gather path binds %v, want {note} (the iterated step's own field, inclusive)", rendered.Args[pathIndex-1].Value) + } + tail := regexp.MustCompile(`SELECT jsonb_agg\(e #> \$(\d+)::text\[\]\)`).FindStringSubmatch(sql) + if tail == nil { + t.Fatalf("the gather lost its leaf tail:\n%s", sql) + } + tailIndex, err := strconv.Atoi(tail[1]) + if err != nil || tailIndex < 1 || tailIndex > len(rendered.Args) { + t.Fatalf("leaf tail parameter $%s is not a minted bind", tail[1]) + } + if bound, ok := rendered.Args[tailIndex-1].Value.(string); !ok || bound != "{text}" { + t.Fatalf("the leaf tail binds %v, want {text}", rendered.Args[tailIndex-1].Value) + } + if strings.Contains(sql, `jsonb_typeof(r0."payload") = 'array'`) { + t.Fatalf("the gather regressed to the pathless raw-payload read:\n%s", sql) + } +} + +func TestSQL16RowValuePolicyStorageClasses(t *testing.T) { + keepPolicy := sql16FilterStep("keep_rows", recipe.ConstructionSourceProjectionInput, "", "status_id") + keepPolicy.Outputs = []recipe.StageColumn{{ID: "status_id", Name: "status"}, {ID: "tags_id", Name: "tags"}} + groupOutputs := []recipe.StageColumn{ + {ID: "status_key_id", Name: "status_key"}, {ID: "rows_id", Name: "rows"}, + {ID: "tags_all_id", Name: "tags_all"}, {ID: "status_one_id", Name: "status_one"}, + } + filterOne := sql16FilterStep("filter_one", recipe.ConstructionStepOutputInput, "group_status", "status_one_id") + filterOne.Outputs = groupOutputs + plan := sql16Compile(t, recipe.Output{ + Name: "sql16_policy_storage", RootResourceType: "Observation", RowGrain: "observation", + Fields: sql16SourceFields(), + Construction: &recipe.Construction{Version: 1, SourceColumns: sql16SourceColumns(), + Steps: []recipe.ConstructionStep{ + keepPolicy, + { + ID: "group_status", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionStepOutputInput, StepID: "keep_rows"}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionGroupOp, Group: &recipe.ConstructionGroup{ + ConstructionID: "group_status_values", + Keys: []recipe.ConstructionGroupKey{{InputColumnID: "status_id", OutputColumnID: "status_key_id"}}, + Aggregates: []recipe.ConstructionGroupAggregate{ + {Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}, + }, + }}, + RowValues: []recipe.ConstructionRowValue{ + {InputColumnID: "status_id", OutputColumnID: "tags_all_id", Policy: recipe.ConstructionRowValueAll}, + {InputColumnID: "status_id", OutputColumnID: "status_one_id", Policy: recipe.ConstructionRowValueOne}, + }, + Outputs: []recipe.StageColumn{ + {ID: "status_key_id", Name: "status_key"}, {ID: "rows_id", Name: "rows"}, + {ID: "tags_all_id", Name: "tags_all"}, {ID: "status_one_id", Name: "status_one"}, + }, + }, + filterOne, + }, + }, + }) + rendered := sql16Render(t, plan) + sql := rendered.SQL + alias := func(marker string) string { + match := regexp.MustCompile(regexp.QuoteMeta(marker) + ` AS "(c\d+)"`).FindStringSubmatch(sql) + if match == nil { + t.Fatalf("%s never materialized into a stage column:\n%s", marker, sql) + } + return match[1] + } + allAlias := alias("__loom_g1_rv0") + oneAlias := alias("__loom_g1_rv1") + for _, fragment := range []string{ + "to_jsonb(COALESCE(ARRAY_AGG(DISTINCT", + "(CASE WHEN COUNT(DISTINCT", + "THEN MIN(", + } { + if !strings.Contains(sql, fragment) { + t.Fatalf("the group region lost the row-value policy fragment %q:\n%s", fragment, sql) + } + } + if !strings.Contains(sql, `s."`+allAlias+`" #> $`) { + t.Fatalf("the ALL row value fact lost its jsonb downstream read:\n%s", sql) + } + if strings.Contains(sql, `s."`+oneAlias+`" #> `) { + t.Fatalf("the ONE row value fact was marked jsonb (the 42883 class):\n%s", sql) + } + if !strings.Contains(sql, `s."`+oneAlias+`" AS "`) { + t.Fatalf("the ONE row value fact was not bare-passthrough:\n%s", sql) + } + if !strings.Contains(sql, `WHERE s."`+oneAlias+`" IS DISTINCT FROM $`) || + strings.Contains(sql, `"`+oneAlias+`" IS DISTINCT FROM (`) { + t.Fatalf("the downstream filter did not compare the ONE column as a scalar:\n%s", sql) + } + door, err := renderStageSequencePhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if err != nil { + t.Fatalf("direct door: %v", err) + } + if door.SQL != sql { + t.Fatalf("door SQL drifted from the entry render") + } +} + +func TestSQL16SETSliceSilentDropCounterfactual(t *testing.T) { + plan, err := compileWithRegisteredSurfacePolicy(t, sql16SETCompositionBundle()) + if err != nil { + t.Fatalf("production compile: %v", err) + } + last := len(plan.Operations) - 1 + mutated := false + for i := range plan.Operations[last].Return.Projections { + projection := &plan.Operations[last].Return.Projections[i] + if projection.Expression != nil && projection.Expression.Kind == ir.PhysicalSliceExpression && + projection.Expression.Slice != nil { + projection.Expression.Slice.Source.Variable = "__loom_unregistered_set_var" + mutated = true + } + } + if !mutated { + t.Fatal("the bundle carries no slice projection to probe") + } + sql16RequireRefusal(t, plan, "is out of scope") +} + +func TestSQL16RelatedOccurrenceDoorExtend(t *testing.T) { + prior := []ir.PhysicalStageColumn{ + {ID: "key_id", Name: "_key", Kind: "string", Cardinality: "required_one", Internal: true, Identity: true}, + {ID: "status_id", Name: "status", Kind: "string", Cardinality: "optional_one"}, + } + adapter := &ir.PhysicalStageGroupRelatedOccurrenceAdapter{ + ScalarProjections: []ir.PhysicalStageGroupRelatedProjection{{InputColumn: "related_status", ResultField: "status"}}, + CodingMembershipColumn: "membership", + OccurrenceKeyColumn: "occurrence", + } + extension := func() []ir.PhysicalStageColumn { + return append(append([]ir.PhysicalStageColumn{}, prior...), + ir.PhysicalStageColumn{ID: "related_status", Name: "related_status", Kind: "string", Cardinality: "optional_one", Internal: true}, + ir.PhysicalStageColumn{ID: "membership", Name: "membership", Kind: "object", Cardinality: "optional_one", Internal: true}, + ir.PhysicalStageColumn{ID: "occurrence", Name: "occurrence", Kind: "object", Cardinality: "required_one", Internal: true}, + ) + } + output := []ir.PhysicalStageColumn{ + {ID: "key_id", Name: "_key", Kind: "string", Cardinality: "required_one", Internal: true, Identity: true}, + {ID: "status_id", Name: "status", Kind: "string", Cardinality: "optional_one"}, + {ID: "occurrence", Name: "occurrence", Kind: "object", Cardinality: "required_one", Internal: true, Identity: true}, + } + chain := func(inputColumns []ir.PhysicalStageColumn, withAdapter bool) ir.PhysicalPlan { + group := &ir.PhysicalStageGroup{ConstructionIDBindKey: "ro_cid"} + if withAdapter { + group.RelatedOccurrence = adapter + } + stage := ir.PhysicalConstructionStage{ + ID: "group_related", InputStageID: "source_projection", Kind: ir.PhysicalStageGroupOp, + InputRowVariable: "inputrow", OutputRowVariable: "group_row", + InputColumns: inputColumns, OutputColumns: output, RowIdentityColumn: "occurrence", + Group: group, + } + return ir.PhysicalPlan{ + Operations: []ir.PhysicalOperation{{Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{ + Projections: []ir.PhysicalProjection{{Name: "_key"}, {Name: "status"}}, + }}}, + StageSequence: &ir.PhysicalStageSequence{ + SourceStageID: "source_projection", SourceRowIdentity: "_key", SourceColumns: prior, + Stages: []ir.PhysicalConstructionStage{stage}, + FinalStageID: "group_related", + FinalRowIdentity: "occurrence", + FinalColumns: output, + }, + } + } + t.Run("extend-shape-passes", func(t *testing.T) { + if err := validateStageSequenceChain(chain(extension(), true), false); err != nil { + t.Fatalf("the production extension shape must pass the door mirror (ir :2659 twin): %v", err) + } + }) + t.Run("renamed-extension-refuses", func(t *testing.T) { + columns := extension() + columns[2].Name = "membershp" + if err := validateStageSequenceChain(chain(columns, true), false); err == nil || + !strings.Contains(err.Error(), "input schema differs from preceding stage output") { + t.Fatalf("renamed adapter column must die at the door: %v", err) + } + }) + t.Run("fourth-extension-refuses", func(t *testing.T) { + columns := append(extension(), ir.PhysicalStageColumn{ID: "extra", Name: "extra", Kind: "object", Cardinality: "optional_one", Internal: true}) + if err := validateStageSequenceChain(chain(columns, true), false); err == nil || + !strings.Contains(err.Error(), "input schema differs from preceding stage output") { + t.Fatalf("a fourth extension column proves the arm is exact-set, not superset: %v", err) + } + }) + t.Run("non-internal-extension-refuses", func(t *testing.T) { + columns := extension() + columns[3].Internal = false + if err := validateStageSequenceChain(chain(columns, true), false); err == nil || + !strings.Contains(err.Error(), "input schema differs from preceding stage output") { + t.Fatalf("an adapter column that is not Internal must die at the door: %v", err) + } + }) + t.Run("id-drift-extension-refuses", func(t *testing.T) { + columns := extension() + columns[2].ID = "membership_alias" + if err := validateStageSequenceChain(chain(columns, true), false); err == nil || + !strings.Contains(err.Error(), "input schema differs from preceding stage output") { + t.Fatalf("extension columns carry ID==Name (sidecar law); drift must die at the door: %v", err) + } + }) + t.Run("prefix-drift-refuses", func(t *testing.T) { + columns := extension() + columns[1].Kind = "required_one" + if err := validateStageSequenceChain(chain(columns, true), false); err == nil || + !strings.Contains(err.Error(), "input schema differs from preceding stage output") { + t.Fatalf("the prior prefix must be schema-identical: %v", err) + } + }) + t.Run("plain-group-unknown-column-refuses", func(t *testing.T) { + columns := append(append([]ir.PhysicalStageColumn{}, prior...), + ir.PhysicalStageColumn{ID: "ghost", Name: "ghost", Kind: "object", Cardinality: "optional_one", Internal: true}) + if err := validateStageSequenceChain(chain(columns, false), false); err == nil || + !strings.Contains(err.Error(), "input schema differs from preceding stage output") { + t.Fatalf("plain GROUP may not extend the prior schema: %v", err) + } + }) + t.Run("plain-group-reordered-refuses", func(t *testing.T) { + reordered := []ir.PhysicalStageColumn{prior[1], prior[0]} + if err := validateStageSequenceChain(chain(reordered, false), false); err == nil || + !strings.Contains(err.Error(), "input schema differs from preceding stage output") { + t.Fatalf("a reordered plain GROUP input must die at the door (ir :65 is positional): %v", err) + } + }) + t.Run("anchor-drift-input-refuses", func(t *testing.T) { + columns := append([]ir.PhysicalStageColumn{}, prior...) + columns[0].RelatedRecordAnchor = &ir.PhysicalStageRelatedRecordAnchor{ResourceType: "Patient"} + if err := validateStageSequenceChain(chain(columns, false), false); err == nil || + !strings.Contains(err.Error(), "input schema differs from preceding stage output") { + t.Fatalf("RelatedRecordAnchor drift must die at the door (ir DeepEqual leg): %v", err) + } + }) + t.Run("presence-companion-drift-tolerated", func(t *testing.T) { + columns := append([]ir.PhysicalStageColumn{}, prior...) + columns[1].PresenceCompanionName = "status_present" + if err := validateStageSequenceChain(chain(columns, false), false); err != nil { + t.Fatalf("PresenceCompanionName drift must pass (ir does not compare it): %v", err) + } + }) + t.Run("empty-projection-input-column-refuses", func(t *testing.T) { + broken := &ir.PhysicalStageGroupRelatedOccurrenceAdapter{ + ScalarProjections: []ir.PhysicalStageGroupRelatedProjection{{InputColumn: "", ResultField: "status"}}, + OccurrenceKeyColumn: "occurrence", + } + group := &ir.PhysicalStageGroup{ConstructionIDBindKey: "ro_cid", RelatedOccurrence: broken} + stages := chain(extension(), true).StageSequence + stages.Stages[0].Group = group + if err := validateStageSequenceChain(ir.PhysicalPlan{ + Operations: chain(extension(), true).Operations, + StageSequence: stages, + }, false); err == nil || !strings.Contains(err.Error(), "input schema differs from preceding stage output") { + t.Fatalf("empty projection InputColumn must refuse (ir rejects, not skips): %v", err) + } + }) + t.Run("empty-coding-names-tolerated", func(t *testing.T) { + noCoding := &ir.PhysicalStageGroupRelatedOccurrenceAdapter{ + ScalarProjections: []ir.PhysicalStageGroupRelatedProjection{{InputColumn: "related_status", ResultField: "status"}}, + } + columns := append(append([]ir.PhysicalStageColumn{}, prior...), + ir.PhysicalStageColumn{ID: "related_status", Name: "related_status", Kind: "string", Cardinality: "optional_one", Internal: true}) + group := &ir.PhysicalStageGroup{ConstructionIDBindKey: "ro_cid", RelatedOccurrence: noCoding} + base := chain(columns, true) + base.StageSequence.Stages[0].Group = group + if err := validateStageSequenceChain(base, false); err != nil { + t.Fatalf("empty Coding column names must be treated as absent (ir skip arms): %v", err) + } + }) +} + +func sql16SweepOutput(t *testing.T, kind ir.PhysicalStageOperationKind) recipe.Output { + t.Helper() + base := func(name string, fields []recipe.Field, columns []recipe.StageColumn) recipe.Output { + return recipe.Output{ + Name: "sql16_sweep_" + name, RootResourceType: "Observation", RowGrain: "observation", + Fields: fields, + Construction: &recipe.Construction{Version: 1, SourceColumns: columns}, + } + } + numeric := func() ([]recipe.Field, []recipe.StageColumn) { + return []recipe.Field{ + {Name: "status", ColumnID: "status_id", Expr: recipe.Expression{Select: "root.status"}}, + {Name: "score", ColumnID: "score_id", Expr: recipe.Expression{Select: "root.valueInteger"}}, + {Name: "amount", ColumnID: "amount_id", Expr: recipe.Expression{Select: "root.valueQuantity.value"}}, + }, []recipe.StageColumn{ + {ID: "status_id", Name: "status"}, {ID: "score_id", Name: "score"}, {ID: "amount_id", Name: "amount"}, + } + } + switch kind { + case ir.PhysicalStageFilterOp: + output := base("filter", sql16SourceFields(), sql16SourceColumns()) + step := sql16FilterStep("keep_rows", recipe.ConstructionSourceProjectionInput, "", "status_id") + step.Outputs = []recipe.StageColumn{{ID: "status_id", Name: "status"}, {ID: "tags_id", Name: "tags"}} + output.Construction.Steps = []recipe.ConstructionStep{step} + return output + case ir.PhysicalStageGroupOp: + groupFields, groupColumns := numeric() + _ = groupFields + output := base("group", groupFields, groupColumns) + output.Construction.Steps = []recipe.ConstructionStep{{ + ID: "group_status", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionGroupOp, Group: &recipe.ConstructionGroup{ + ConstructionID: "group_status_values", + Keys: []recipe.ConstructionGroupKey{{InputColumnID: "status_id", OutputColumnID: "status_key_id"}}, + Aggregates: []recipe.ConstructionGroupAggregate{ + {Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}, + }, + }}, + Outputs: []recipe.StageColumn{ + {ID: "status_key_id", Name: "status_key"}, {ID: "rows_id", Name: "rows"}, + }, + }} + return output + case ir.PhysicalStageExpandOp: + output := base("expand", sql16SourceFields(), sql16SourceColumns()) + output.Construction.Steps = []recipe.ConstructionStep{{ + ID: "expand_tags", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionExpandOp, Expand: &recipe.ConstructionExpand{ + ConstructionID: "expand_tag_values", InputColumnID: "tags_id", + OutputColumnID: "tag_id", OrdinalColumnID: "ordinal_id", + EmptyPolicy: recipe.ExpansionPreserveParent, + }}, + Outputs: []recipe.StageColumn{ + {ID: "status_id", Name: "status"}, {ID: "tag_id", Name: "tag"}, {ID: "ordinal_id", Name: "ordinal"}, + }, + }} + return output + case ir.PhysicalStageDeriveOp: + fields, columns := numeric() + output := base("derive", fields, columns) + output.Construction.Steps = []recipe.ConstructionStep{{ + ID: "derive_total", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionDeriveOp, Derive: &recipe.ConstructionDerive{ + ConstructionID: "derive_total_sum", OutputColumnID: "total_id", Operation: recipe.DerivedAdd, + Left: recipe.ConstructionOperand{Kind: recipe.DerivedColumnOperand, ColumnID: "score_id"}, + Right: recipe.ConstructionOperand{Kind: recipe.DerivedColumnOperand, ColumnID: "amount_id"}, + MissingInputPolicy: recipe.MissingInputPropagateNull, + }}, + Outputs: []recipe.StageColumn{ + {ID: "status_id", Name: "status"}, {ID: "score_id", Name: "score"}, + {ID: "amount_id", Name: "amount"}, {ID: "total_id", Name: "total"}, + }, + }} + return output + case ir.PhysicalStageUnpivotOp: + fields, columns := numeric() + output := base("unpivot", fields, columns) + keyScore, keyAmount := "score", "amount" + output.Construction.Steps = []recipe.ConstructionStep{{ + ID: "unpivot_measures", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionUnpivotOp, Unpivot: &recipe.ConstructionUnpivot{ + ConstructionID: "unpivot_measures_values", + Inputs: []recipe.ConstructionUnpivotInput{ + {ColumnID: "score_id", Key: recipe.TableScalar{Kind: recipe.TableScalarString, String: &keyScore}}, + {ColumnID: "amount_id", Key: recipe.TableScalar{Kind: recipe.TableScalarString, String: &keyAmount}}, + }, + KeyOutputColumnID: "measure_id", ValueOutputColumnID: "measure_value_id", + NullRowPolicy: recipe.UnpivotNullDrop, + }}, + Outputs: []recipe.StageColumn{ + {ID: "status_id", Name: "status"}, {ID: "measure_id", Name: "measure"}, + {ID: "measure_value_id", Name: "measure_value"}, + }, + }} + return output + case ir.PhysicalStagePivotOp: + final := "final" + return recipe.Output{ + Name: "sql16_sweep_pivot", RootResourceType: "Observation", RowGrain: "observation", + Fields: []recipe.Field{ + {Name: "status", ColumnID: "status_id", Expr: recipe.Expression{Select: "root.status"}}, + {Name: "rid", ColumnID: "rid_id", Expr: recipe.Expression{Select: "root.id"}}, + {Name: "val", ColumnID: "val_id", Expr: recipe.Expression{Select: "root.valueString"}}, + }, + Construction: &recipe.Construction{Version: 1, + SourceColumns: []recipe.StageColumn{ + {ID: "status_id", Name: "status"}, {ID: "rid_id", Name: "rid"}, {ID: "val_id", Name: "val"}, + }, + Steps: []recipe.ConstructionStep{{ + ID: "pivot_status", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionPivotOp, Pivot: &recipe.ConstructionPivot{ + ConstructionID: "status_pivot", GroupKeyIDs: []string{"val_id"}, + CategoryColumnID: "status_id", ValueColumnID: "rid_id", + Categories: []recipe.ConstructionPivotCategory{ + {Key: recipe.TableScalar{Kind: recipe.TableScalarString, String: &final}, OutputColumnID: "final_id"}, + }, + DuplicatePolicy: recipe.PivotDuplicateError, MissingCellPolicy: recipe.PivotMissingCellNull, + UnlistedCategoryPolicy: recipe.PivotUnlistedCategoryError, + }}, + Outputs: []recipe.StageColumn{ + {ID: "val_id", Name: "val", Type: "string", Nullable: true}, + {ID: "final_id", Name: "final_cell", Type: "string", Nullable: true}, + }, + }}, + }, + } + } + t.Fatalf("unregistered sweep kind %q", string(kind)) + return recipe.Output{} +} + +func TestSQL16CompositionKindSweep(t *testing.T) { + for _, kind := range []ir.PhysicalStageOperationKind{ + ir.PhysicalStageFilterOp, ir.PhysicalStageDeriveOp, ir.PhysicalStageExpandOp, + ir.PhysicalStageUnpivotOp, ir.PhysicalStageGroupOp, ir.PhysicalStagePivotOp, + } { + t.Run(string(kind), func(t *testing.T) { + plan := sql16Compile(t, sql16SweepOutput(t, kind)) + sequence := plan.StageSequence + if sequence == nil || len(sequence.Stages) != 1 { + t.Fatalf("the recipe must lower to one composed stage, got %#v", sequence) + } + if got := sequence.Stages[0].Kind; got != kind { + t.Fatalf("production lowering emitted kind %q, want %q (the sweep cell must exercise its kind)", got, string(kind)) + } + if kind == ir.PhysicalStageDeriveOp { + sql16RequireRefusal(t, ir.ClonePhysicalPlan(plan), + `composition DERIVE LET of expression kind "CALL" has no registered rendering`) + return + } + if err := validateStageSequenceChain(ir.ClonePhysicalPlan(plan), false); err != nil { + t.Fatalf("door refuses the production-lowered %s plan (door/ir drift class): %v", string(kind), err) + } + rendered, err := RenderPhysicalPlan(ir.ClonePhysicalPlan(plan), mustBinding(t)) + if err != nil { + t.Fatalf("entry refuses the production-lowered %s plan the door accepted: %v", string(kind), err) + } + switch kind { + case ir.PhysicalStageUnpivotOp: + for _, want := range []string{ + "CROSS JOIN LATERAL (VALUES", + `AS x1(slot_key, slot_value, slot_ord)`, + `"TABLE_UNPIVOT"`, + } { + if !strings.Contains(rendered.SQL, want) { + t.Fatalf("composition melt stage missing fragment %q:\n%s", want, rendered.SQL) + } + } + } + }) + } +} diff --git a/internal/dataframe/compiler/render/postgres/table_shape_exclusions.go b/internal/dataframe/compiler/render/postgres/table_shape_exclusions.go new file mode 100644 index 000000000..12d5dab47 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/table_shape_exclusions.go @@ -0,0 +1,457 @@ +package postgres + +import ( + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const exclusionSlotPrefix = "px_" + +func tableShapeExclusionPlanMentionsSurface(plan ir.PhysicalPlan) bool { + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalTableShapeExclusionReturnOp { + return true + } + } + return false +} + +func RenderTableShapeExclusions(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + return renderTableShapeExclusionPhysicalPlan(plan, storage) +} + +func renderTableShapeExclusionPhysicalPlan(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + if err := storage.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := validateTableShapeExclusionPlanSurface(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + if err := plan.Validate(); err != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("validate physical plan: %w", err) + } + if plan.Engine != "" && plan.Engine != ir.PhysicalEngineAQL { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: engine %q has no PostgreSQL source; the source graph IR is backend-neutral and ClickHouse plans are not PostgreSQL-rendered", + ErrUnsupportedPostgresPlan, string(plan.Engine)) + } + if plan.ClickHouseCombine != nil || plan.ClickHousePrefix != nil { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: ClickHouse combine/prefix payloads have no PostgreSQL rendering", ErrUnsupportedPostgresPlan) + } + if len(plan.DeferredExpressionLets) != 0 { + return command.RenderedPostgresPhysicalPlan{}, fmt.Errorf( + "%w: completed plans must not carry deferred expression lets", ErrUnsupportedPostgresPlan) + } + renderer := &planRenderer{ + storage: storage, + rows: map[string]rowScope{}, + parameters: map[string]int{}, + canonicalAuthLets: map[string]string{}, + } + if err := renderer.loadScope(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + sql, err := renderer.renderTableShapeExclusionStatement(plan) + if err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return command.NewRenderedPostgresPhysicalPlan(sql, renderer.args, renderer.outputColumns) +} + +func validateTableShapeExclusionPlanSurface(plan ir.PhysicalPlan) error { + if len(plan.Operations) == 0 { + return fmt.Errorf("%w: plan has no operations", ErrUnsupportedPostgresPlan) + } + if plan.Operations[0].Kind != ir.PhysicalRootScanOp || plan.Operations[0].RootScan == nil { + return fmt.Errorf("%w: the exclusion slice leads with exactly one ROOT_SCAN at position 0", ErrUnsupportedPostgresPlan) + } + if plan.StageSequence != nil { + return fmt.Errorf("%w: an exclusion terminal combined with a stage sequence has no PostgreSQL rendering (staged compositions are pg-sql-16)", ErrUnsupportedPostgresPlan) + } + if plan.PreviewSourceWindowByRootID { + return fmt.Errorf("%w: the plan-level preview source window requires terminal windowing contracts that are not registered for the exclusion terminal (pg-sql-16)", ErrUnsupportedPostgresPlan) + } + var terminal *ir.PhysicalTableShapeExclusionReturn + for index, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalRootScanOp: + if index != 0 { + return fmt.Errorf("%w: the exclusion slice leads with exactly one ROOT_SCAN at position 0, got one at %d", + ErrUnsupportedPostgresPlan, index) + } + if operation.RootScan == nil { + return fmt.Errorf("%w: ROOT_SCAN is missing its payload", ErrUnsupportedPostgresPlan) + } + if err := validatePivotRootScan(plan, *operation.RootScan); err != nil { + return err + } + case ir.PhysicalFilterOp: + if operation.Filter == nil { + return fmt.Errorf("%w: FILTER is missing its payload", ErrUnsupportedPostgresPlan) + } + case ir.PhysicalDerivedLetOp: + if operation.DerivedLet == nil { + return fmt.Errorf("%w: DERIVED_LET is missing its payload", ErrUnsupportedPostgresPlan) + } + case ir.PhysicalTableShapeExclusionReturnOp: + if index != len(plan.Operations)-1 { + return fmt.Errorf("%w: the evidence terminal must be the final operation, got position %d of %d", + ErrUnsupportedPostgresPlan, index, len(plan.Operations)) + } + if operation.TableShapeExclusionReturn == nil { + return fmt.Errorf("%w: the evidence terminal is missing its payload", ErrUnsupportedPostgresPlan) + } + terminal = operation.TableShapeExclusionReturn + default: + return fmt.Errorf("%w: operation %q cannot coexist with the evidence terminal in the registered slice; the terminal owns its own SORT and OFFSET/LIMIT window, and every other shape belongs to the read spine or the pg-sql-16 composition capability", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + if terminal == nil { + return fmt.Errorf("%w: the exclusion slice requires a terminal evidence operation after ROOT_SCAN", ErrUnsupportedPostgresPlan) + } + rootVariable := plan.Operations[0].RootScan.Variable + if err := validateTableShapeExclusionBinds(plan, *terminal); err != nil { + return err + } + pivot := terminal.Pivot + if pivot.UnlistedCategoryPolicy != string(ir.PhysicalPivotUnlistedCategoryExcludeWithEvidence) { + return fmt.Errorf("%w: the evidence terminal exists exactly under the EXCLUDE_WITH_EVIDENCE policy, got %q (ir/physical_validation.go re-read; the compiler entry refuses any other policy with TABLE_SHAPE_EXCLUSION_NO_POLICY)", + ErrUnsupportedPostgresPlan, pivot.UnlistedCategoryPolicy) + } + if err := validateGroupedPivotPayload(plan, pivot, rootVariable); err != nil { + return err + } + if _, err := exclusionStableRowColumn(pivot); err != nil { + return err + } + return nil +} + +func validateTableShapeExclusionBinds(plan ir.PhysicalPlan, terminal ir.PhysicalTableShapeExclusionReturn) error { + bind := func(key, field string) (int, error) { + if strings.TrimSpace(key) == "" { + return 0, fmt.Errorf("%w: the evidence terminal has no %s bind key", ErrUnsupportedPostgresPlan, field) + } + value, ok := plan.BindVars[key].(int) + if !ok { + return 0, fmt.Errorf("%w: evidence %s bind %q must be an int", ErrUnsupportedPostgresPlan, field, key) + } + return value, nil + } + offset, err := bind(terminal.OffsetBindKey, "offset") + if err != nil { + return err + } + limit, err := bind(terminal.LimitBindKey, "limit") + if err != nil { + return err + } + fetchLimit, err := bind(terminal.FetchLimitBindKey, "fetch limit") + if err != nil { + return err + } + if offset < 0 { + return fmt.Errorf("%w: evidence offset must be non-negative, got %d", ErrUnsupportedPostgresPlan, offset) + } + if limit <= 0 { + return fmt.Errorf("%w: evidence limit must be positive, got %d", ErrUnsupportedPostgresPlan, limit) + } + if fetchLimit != limit+1 { + return fmt.Errorf("%w: evidence fetch limit must be limit+1 (%d), got %d", + ErrUnsupportedPostgresPlan, limit+1, fetchLimit) + } + return nil +} + +func exclusionStableRowColumn(pivot ir.PhysicalGroupedPivot) (string, error) { + stable := "" + for _, projection := range pivot.InputProjections { + if projection.Name == pivotRowIDColumn { + return pivotRowIDColumn, nil + } + if projection.Name == "_key" { + stable = projection.Name + } + } + if stable == "" { + return "", fmt.Errorf("%w: table-shape exclusion input has no stable row tie-breaker (%s or _key)", + ErrUnsupportedPostgresPlan, pivotRowIDColumn) + } + return stable, nil +} + +func exclusionConsumedInputNames(pivot ir.PhysicalGroupedPivot, stableRowColumn string) map[string]bool { + consumed := map[string]bool{pivot.CategoryColumn: true, stableRowColumn: true} + for _, key := range pivot.GroupKeys { + consumed[key.Column] = true + } + if pivot.CategoryPresenceFromInput { + consumed[pivot.CategoryPresenceColumn] = true + } + return consumed +} + +func (r *planRenderer) renderTableShapeExclusionStatement(plan ir.PhysicalPlan) (string, error) { + var terminal *ir.PhysicalTableShapeExclusionReturn + for index := range plan.Operations { + if plan.Operations[index].Kind == ir.PhysicalTableShapeExclusionReturnOp { + terminal = plan.Operations[index].TableShapeExclusionReturn + } + } + if terminal == nil { + return "", fmt.Errorf("%w: the exclusion slice requires a terminal evidence operation", ErrUnsupportedPostgresPlan) + } + pivot := terminal.Pivot + rootVariable := plan.Operations[0].RootScan.Variable + + if err := r.renderRootScan(plan, *plan.Operations[0].RootScan); err != nil { + return "", err + } + for _, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalFilterOp: + canonical, err := r.acceptCanonicalScopeFilter(operation.Filter) + if err != nil { + return "", err + } + if !canonical { + return "", fmt.Errorf("%w: a non-canonical FILTER cannot coexist with the registered evidence terminal (probe-attested renders-at-AQL divergence, pg-sql-16)", + ErrUnsupportedPostgresPlan) + } + case ir.PhysicalDerivedLetOp: + canonical, err := r.acceptCanonicalAuthLet(operation.DerivedLet) + if err != nil { + return "", err + } + if !canonical { + return "", fmt.Errorf("%w: operation %q is not part of the registered evidence terminal source surface", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + } + fromText := r.storage.Resource.Identifier().SQLText() + " AS " + r.rootAlias + for _, join := range r.joins { + fromText += " " + join + } + for _, lateral := range r.laterals { + fromText += " " + lateral + } + if len(r.where) > 0 { + fromText += " WHERE " + strings.Join(r.where, " AND ") + } + + stableRowColumn, err := exclusionStableRowColumn(pivot) + if err != nil { + return "", err + } + consumed := exclusionConsumedInputNames(pivot, stableRowColumn) + reads := map[string]string{} + slotExpressions := make([]string, 0, len(consumed)) + addSlot := func(expression string) string { + slot := exclusionSlotPrefix + strconv.Itoa(len(slotExpressions)) + slotExpressions = append(slotExpressions, expression+" AS "+slot) + return "p." + slot + } + bareReads := map[string]string{} + categoryBehavior := ir.PhysicalPreserveNull + for _, projection := range pivot.InputProjections { + if !consumed[projection.Name] { + continue + } + if _, rendered := reads[projection.Name]; rendered { + continue + } + expression, behavior, err := r.exclusionInputRead(projection, rootVariable) + if err != nil { + return "", err + } + slot := exclusionSlotPrefix + strconv.Itoa(len(slotExpressions)) + reads[projection.Name] = addSlot(expression) + bareReads[projection.Name] = slot + if projection.Name == pivot.CategoryColumn { + categoryBehavior = behavior + } + } + pvCat, err := pivotSlotRead(reads, pivot.CategoryColumn) + if err != nil { + return "", err + } + stableRead, err := pivotSlotRead(reads, stableRowColumn) + if err != nil { + return "", err + } + + present := "" + switch { + case pivot.CategoryPresence != nil: + proofs := make([]string, 0, len(pivot.CategoryPresence.Paths)) + for _, path := range pivot.CategoryPresence.Paths { + proof, err := r.propertyPresenceExpression(r.rootAlias, path) + if err != nil { + return "", err + } + proofs = append(proofs, proof) + } + present = addSlot("CASE WHEN " + strings.Join(proofs, " OR ") + " THEN true ELSE false END") + case pivot.CategoryPresenceFromInput: + if _, err := pivotSlotRead(reads, pivot.CategoryPresenceColumn); err != nil { + return "", err + } + present = addSlot("COALESCE(" + bareReads[pivot.CategoryPresenceColumn] + " = 'true'::jsonb, false)") + default: + switch categoryBehavior { + case ir.PhysicalOmitNulls: + present = addSlot(bareReads[pivot.CategoryColumn] + " IS NOT NULL") + default: + present = addSlot("true") + } + } + + matchPredicates := make([]string, 0, len(pivot.Categories)) + for _, category := range pivot.Categories { + switch category.MatchKind { + case ir.PhysicalPivotCategoryValueMatch: + raw, err := pivotCategoryRaw(plan.BindVars, category) + if err != nil { + return "", err + } + valueParameter := pivotDollar(r.jsonbParameter(raw)) + matchPredicates = append(matchPredicates, + "("+present+" AND jsonb_typeof("+pvCat+") = "+quoteSQLLiteral(pivotValueTypeOf[category.ValueKind])+ + " AND "+pvCat+" = "+valueParameter+"::jsonb)") + case ir.PhysicalPivotCategoryNullMatch: + nullRead := "(" + pvCat + " IS NULL OR " + pvCat + " = 'null'::jsonb)" + if categoryBehavior == ir.PhysicalOmitNulls { + nullRead = "(" + pvCat + " IS NOT NULL AND " + pvCat + " = 'null'::jsonb)" + } + matchPredicates = append(matchPredicates, "("+present+" AND "+nullRead+")") + case ir.PhysicalPivotCategoryMissingMatch: + matchPredicates = append(matchPredicates, "NOT ("+present+")") + default: + return "", fmt.Errorf("%w: pivot category match kind %q is not registered", + ErrUnsupportedPostgresPlan, string(category.MatchKind)) + } + } + listed := "(" + strings.Join(matchPredicates, " OR ") + ")" + + keySlots := make([]string, 0, len(pivot.GroupKeys)) + kinds := make([]string, 0, len(pivot.GroupKeys)) + for _, key := range pivot.GroupKeys { + expr, err := pivotSlotRead(reads, key.Column) + if err != nil { + return "", err + } + keySlots = append(keySlots, expr) + kinds = append(kinds, key.Kind) + } + identity := r.pivotIdentitySQL(bindStringVar(plan.BindVars, pivot.ConstructionIDBindKey), kinds, keySlots) + + document := addSlot(r.documentColumn(r.rootAlias)) + sourceDocument := r.documentColumn(r.rootAlias) + identityAvailable := addSlot("COALESCE(jsonb_typeof(" + sourceDocument + ") = 'object' AND " + + "jsonb_typeof(" + sourceDocument + " -> 'resourceType') = 'string' AND " + + exclusionTextMember(sourceDocument, "resourceType") + " <> '' AND " + + "jsonb_typeof(" + sourceDocument + " -> 'id') = 'string' AND " + + exclusionTextMember(sourceDocument, "id") + " <> '', false)") + + evidenceValue := "CASE WHEN " + present + " THEN " + pvCat + " END" + textParameter := func(value, key string) string { + return pivotDollar(r.textParameter(value, "exclusion:"+key)) + } + selects := []string{ + "(CASE WHEN " + identityAvailable + " THEN " + document + " -> 'resourceType' END)" + + ` AS "c0"`, + "(CASE WHEN " + identityAvailable + " THEN " + document + " -> 'id' END)" + + ` AS "c1"`, + "(CASE WHEN " + identityAvailable + " THEN " + textParameter(ir.PhysicalTableShapeExclusionIdentityExact, "exact") + + " ELSE " + textParameter(ir.PhysicalTableShapeExclusionIdentityUnavailable, "unavailable") + " END)" + + ` AS "c2"`, + evidenceValue + ` AS "c3"`, + "to_jsonb(" + present + ")" + ` AS "c4"`, + textParameter(pivot.CategoryType, "category-type") + ` AS "c5"`, + identity + ` AS "c6"`, + textParameter(ir.PhysicalTableShapeExclusionReasonUnlistedCategory, "reason") + ` AS "c7"`, + "(CASE WHEN " + identityAvailable + " THEN NULL ELSE " + + textParameter(ir.PhysicalTableShapeExclusionSourceIdentityUnavailable, "omission") + " END)" + ` AS "c8"`, + } + r.outputColumns = []string{ + ir.PhysicalTableShapeExclusionResourceTypeField, + ir.PhysicalTableShapeExclusionResourceIDField, + ir.PhysicalTableShapeExclusionIdentityStatusField, + ir.PhysicalTableShapeExclusionCategoryValueField, + ir.PhysicalTableShapeExclusionCategoryPresentField, + ir.PhysicalTableShapeExclusionCategoryTypeField, + ir.PhysicalTableShapeExclusionOutputRowIDField, + ir.PhysicalTableShapeExclusionReasonField, + ir.PhysicalTableShapeExclusionOmissionField, + } + + order := []string{identity + ` COLLATE "C" ASC`, identityAvailable + " DESC"} + order = append(order, exclusionTypeRankTerms(document+" -> 'resourceType'")...) + order = append(order, exclusionTypeRankTerms(document+" -> 'id'")...) + order = append(order, present+" ASC") + order = append(order, exclusionTypeRankTerms("CASE WHEN "+present+" THEN "+pvCat+" END")...) + order = append(order, exclusionTypeRankTerms(stableRead)...) + + statement := "WITH " + exclusionSlotPrefix + "src AS (SELECT " + + strings.Join(slotExpressions, ", ") + " FROM " + fromText + ") " + + "SELECT " + strings.Join(selects, ", ") + " FROM " + exclusionSlotPrefix + "src AS p " + + "WHERE NOT COALESCE(" + listed + ", false) " + + "ORDER BY " + strings.Join(order, ", ") + offset := plan.BindVars[terminal.OffsetBindKey].(int) + fetchLimit := plan.BindVars[terminal.FetchLimitBindKey].(int) + offsetIndex := r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(offset)}, "") + limitIndex := r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(fetchLimit)}, "") + statement += " OFFSET " + pivotDollar(offsetIndex) + " LIMIT " + pivotDollar(limitIndex) + return statement, nil +} + +func (r *planRenderer) exclusionInputRead(projection ir.PhysicalProjection, rootVariable string) (string, ir.PhysicalNullBehavior, error) { + path, behavior, err := pivotInputReadShape(projection, rootVariable) + if err != nil { + return "", "", err + } + if projection.Expression != nil { + extraction, err := r.documentPathExpression(r.rootAlias, path) + if err != nil { + return "", "", err + } + switch behavior { + case ir.PhysicalOmitNulls: + return "NULLIF((" + extraction + "), 'null'::jsonb)", behavior, nil + case ir.PhysicalEmptyOnNull: + return "COALESCE(NULLIF((" + extraction + "), 'null'::jsonb), '\"\"'::jsonb)", behavior, nil + default: + return "(" + extraction + ")", behavior, nil + } + } + if len(path) == 1 { + if column := canonicalKeyColumn(path[0]); resourceFilterColumns[column] { + return "to_jsonb(" + r.rootAlias + "." + r.storage.Resource.MustColumn(column).SQLText() + "::text)", behavior, nil + } + } + extraction, err := r.documentPathFromExpression(r.documentColumn(r.rootAlias), path) + if err != nil { + return "", "", err + } + return "(" + extraction + ")", behavior, nil +} +func exclusionTextMember(document, member string) string { + return "(" + document + " -> '" + member + "' #>> '{}')" +} + +func exclusionTypeRankTerms(expr string) []string { + return []string{ + "(CASE WHEN " + expr + " IS NULL THEN 0 WHEN jsonb_typeof(" + expr + ") = 'boolean' THEN 1" + + " WHEN jsonb_typeof(" + expr + ") = 'number' THEN 2 WHEN jsonb_typeof(" + expr + ") = 'string' THEN 3 ELSE 4 END) ASC", + "(CASE WHEN jsonb_typeof(" + expr + ") = 'boolean' THEN " + expr + "::text END) ASC", + "((CASE WHEN jsonb_typeof(" + expr + ") = 'number' THEN " + expr + " #>> '{}' END)::double precision) ASC", + "((CASE WHEN jsonb_typeof(" + expr + ") = 'string' THEN " + expr + " #>> '{}' END) COLLATE \"C\") ASC", + } +} diff --git a/internal/dataframe/compiler/render/postgres/table_shape_exclusions_live_test.go b/internal/dataframe/compiler/render/postgres/table_shape_exclusions_live_test.go new file mode 100644 index 000000000..f05c873a8 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/table_shape_exclusions_live_test.go @@ -0,0 +1,500 @@ +package postgres + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +var ( + excl26FixturesMu sync.Mutex + excl26TrackedFixtures = map[string]bool{} +) + +func excl26LiveService(t *testing.T) *pgir02Service { + t.Helper() + svc := newOwnedService(t, 0, "verify_sql26v1") + var database string + if err := svc.iso.QueryRow(svc.ctx, "SELECT current_database()").Scan(&database); err != nil { + svc.fail("track owned database", err) + } + excl26FixturesMu.Lock() + defer excl26FixturesMu.Unlock() + excl26TrackedFixtures[database] = true + excl26TrackedFixtures[svc.schema] = true + return svc +} + +func excl26Binding(t *testing.T, svc *pgir02Service) ir.PostgresStorageBinding { + t.Helper() + binding, err := ir.NewPostgresStorageBinding(svc.schema) + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + return binding +} + +func excl26Seed(t *testing.T, svc *pgir02Service, logicalID string, payload any) { + t.Helper() + svc.insertResource(t, svc.schema, excl26Project, excl26Generation, "Patient", logicalID, excl26AuthPath, payload) +} + +func excl26Doc(logicalID, fields string) json.RawMessage { + return json.RawMessage(`{"resourceType":"Patient","id":"` + logicalID + `"` + fields + `}`) +} + +func excl26StringCategories() []ir.PhysicalGroupedPivotCategory { + return []ir.PhysicalGroupedPivotCategory{ + {Output: "cell_zero", MatchKind: ir.PhysicalPivotCategoryValueMatch, ValueBindKey: "cat_zero", ValueKind: "STRING"}, + {Output: "cell_one", MatchKind: ir.PhysicalPivotCategoryValueMatch, ValueBindKey: "cat_one", ValueKind: "STRING"}, + } +} + +func excl26EvidencePlan(t *testing.T, offset, limit int) ir.PhysicalPlan { + t.Helper() + plan := excl26Plan(t, excl26Config{ + categories: excl26StringCategories(), + offset: offset, + limit: limit, + zeroOffset: offset == 0, + }) + plan.BindVars["cat_zero"] = "alpha-zero" + plan.BindVars["cat_one"] = "alpha-one" + return plan +} + +func excl26RenderLive(t *testing.T, svc *pgir02Service, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderTableShapeExclusions(plan, excl26Binding(t, svc)) + if err != nil { + t.Fatalf("render evidence terminal live: %v", err) + } + want := []string{ + ir.PhysicalTableShapeExclusionResourceTypeField, + ir.PhysicalTableShapeExclusionResourceIDField, + ir.PhysicalTableShapeExclusionIdentityStatusField, + ir.PhysicalTableShapeExclusionCategoryValueField, + ir.PhysicalTableShapeExclusionCategoryPresentField, + ir.PhysicalTableShapeExclusionCategoryTypeField, + ir.PhysicalTableShapeExclusionOutputRowIDField, + ir.PhysicalTableShapeExclusionReasonField, + ir.PhysicalTableShapeExclusionOmissionField, + } + if strings.Join(rendered.OutputColumns, ",") != strings.Join(want, ",") { + t.Fatalf("evidence output columns = %v, want the nine frozen fields", rendered.OutputColumns) + } + return rendered +} + +func excl26ColIndex(t *testing.T, columns []string, field string) int { + t.Helper() + for index, name := range columns { + if name == field { + return index + } + } + t.Fatalf("output columns %v lack %q", columns, field) + return -1 +} + +func TestExclusion26LiveEvidenceRows(t *testing.T) { + svc := excl26LiveService(t) + excl26Seed(t, svc, "e1", excl26Doc("e1", `,"gProp":"G1","cProp":"alpha-zero","vProp":1`)) + excl26Seed(t, svc, "e2", excl26Doc("e2", `,"gProp":"G1","cProp":"alpha-one","vProp":1`)) + excl26Seed(t, svc, "e3", excl26Doc("e3", `,"gProp":"G1","cProp":"zzz","vProp":1`)) + excl26Seed(t, svc, "e4", excl26Doc("e4", `,"gProp":"G1","cProp":"zzz","vProp":2`)) + excl26Seed(t, svc, "e5", excl26Doc("e5", `,"gProp":"G1","vProp":1`)) + excl26Seed(t, svc, "e6", excl26Doc("e6", `,"gProp":"G1","cProp":null,"vProp":1`)) + excl26Seed(t, svc, "e7", excl26Doc("e7", `,"gProp":"G2","cProp":"alpha-zero","vProp":1`)) + svc.insertResource(t, svc.schema, "other-project", excl26Generation, "Patient", "x1", excl26AuthPath, + excl26Doc("x-scope", `,"gProp":"G9","cProp":"nope"`)) + svc.insertResource(t, svc.schema, excl26Project, "other-gen", "Patient", "x2", excl26AuthPath, + excl26Doc("x-scope", `,"gProp":"G9","cProp":"nope"`)) + svc.insertResource(t, svc.schema, excl26Project, excl26Generation, "Patient", "x3", "FHIR/Observation", + excl26Doc("x-scope", `,"gProp":"G9","cProp":"nope"`)) + + plan := excl26Plan(t, excl26Config{ + categories: excl26StringCategories(), + presence: "contract", + offset: 0, + zeroOffset: true, + limit: 100, + }) + plan.BindVars["cat_zero"] = "alpha-zero" + plan.BindVars["cat_one"] = "alpha-one" + out := excl26RenderLive(t, svc, plan) + rows := svc.runRendered(t, out) + + idCol := excl26ColIndex(t, out.OutputColumns, ir.PhysicalTableShapeExclusionOutputRowIDField) + categoryCol := excl26ColIndex(t, out.OutputColumns, ir.PhysicalTableShapeExclusionCategoryValueField) + presentCol := excl26ColIndex(t, out.OutputColumns, ir.PhysicalTableShapeExclusionCategoryPresentField) + statusCol := excl26ColIndex(t, out.OutputColumns, ir.PhysicalTableShapeExclusionIdentityStatusField) + reasonCol := excl26ColIndex(t, out.OutputColumns, ir.PhysicalTableShapeExclusionReasonField) + + if len(rows) != 4 { + t.Fatalf("evidence rows = %d, want the four unlisted G1 contributors:\n%v", len(rows), rows) + } + for _, row := range rows { + if row[reasonCol] != ir.PhysicalTableShapeExclusionReasonUnlistedCategory { + t.Fatalf("reason = %q, want %q", row[reasonCol], ir.PhysicalTableShapeExclusionReasonUnlistedCategory) + } + if row[statusCol] != ir.PhysicalTableShapeExclusionIdentityExact { + t.Fatalf("identity status = %q, want %q (AQL :75 identity-available branch)", row[statusCol], ir.PhysicalTableShapeExclusionIdentityExact) + } + if !strings.HasPrefix(row[idCol], `["GROUPED_PIVOT","p26-hand-construct",["STRING","G1"]]`) { + t.Fatalf("identity = %q, want the G1 pivot identity (AQL :137-150)", row[idCol]) + } + } + var zzz, absent, storedNull int + for _, row := range rows { + switch { + case row[categoryCol] == "zzz" && row[presentCol] == "true": + zzz++ + case row[categoryCol] == "" && row[presentCol] == "false": + absent++ + case (row[categoryCol] == "null" || row[categoryCol] == "") && row[presentCol] == "true": + storedNull++ + } + } + if zzz != 2 || absent != 1 || storedNull != 1 { + t.Fatalf("tri-state split = zzz:%d absent:%d null:%d, want 2/1/1:\n%v", zzz, absent, storedNull, rows) + } + + var witness int + if err := svc.iso.QueryRow(svc.ctx, + `SELECT count(*) FROM `+svc.schema+`.fhir_resource + WHERE project = $1 AND dataset_generation = $2 AND auth_resource_path = $3 AND resource_type = 'Patient' + AND payload #> '{gProp}' IS NOT NULL + AND NOT COALESCE(jsonb_typeof(payload #> '{cProp}') = 'string' + AND (payload #> '{cProp}' = '"alpha-zero"'::jsonb OR payload #> '{cProp}' = '"alpha-one"'::jsonb), false)`, + excl26Project, excl26Generation, excl26AuthPath).Scan(&witness); err != nil { + svc.fail("direct-SQL witness", err) + } + if witness != 4 { + t.Fatalf("direct-SQL witness = %d, want 4", witness) + } + + empty := excl26EvidencePlan(t, 0, 100) + empty.BindVars["auth_resource_paths"] = []string{"FHIR/Encounter"} + emptyOut := excl26RenderLive(t, svc, empty) + if rows := svc.runRendered(t, emptyOut); len(rows) != 0 { + t.Fatalf("foreign auth scope returned %d evidence rows, want zero (no phantom row):\n%v", len(rows), rows) + } +} + +func TestExclusion26LiveIdentityUnavailable(t *testing.T) { + svc := excl26LiveService(t) + excl26Seed(t, svc, "d1", excl26Doc("d1", `,"gProp":"G1","cProp":"alpha-zero"`)) + excl26Seed(t, svc, "d2", nil) + + out := excl26RenderLive(t, svc, excl26EvidencePlan(t, 0, 100)) + rows := svc.runRendered(t, out) + if len(rows) != 1 { + t.Fatalf("rows = %d, want only the null-document contributor:\n%v", len(rows), rows) + } + row := rows[0] + statusCol := excl26ColIndex(t, out.OutputColumns, ir.PhysicalTableShapeExclusionIdentityStatusField) + typeCol := excl26ColIndex(t, out.OutputColumns, ir.PhysicalTableShapeExclusionResourceTypeField) + idCol := excl26ColIndex(t, out.OutputColumns, ir.PhysicalTableShapeExclusionResourceIDField) + omissionCol := excl26ColIndex(t, out.OutputColumns, ir.PhysicalTableShapeExclusionOmissionField) + if row[statusCol] != ir.PhysicalTableShapeExclusionIdentityUnavailable { + t.Fatalf("identity status = %q, want %q", row[statusCol], ir.PhysicalTableShapeExclusionIdentityUnavailable) + } + if row[typeCol] != "" || row[idCol] != "" { + t.Fatalf("null-document row leaked document fields: %v", row) + } + if row[omissionCol] != ir.PhysicalTableShapeExclusionSourceIdentityUnavailable { + t.Fatalf("omission = %q, want %q (AQL :132 omission rides only the unavailable branch)", + row[omissionCol], ir.PhysicalTableShapeExclusionSourceIdentityUnavailable) + } +} + +func TestExclusion26LiveMatchesPivotUnlistedCount(t *testing.T) { + svc := excl26LiveService(t) + seeds := []struct { + id string + fields string + }{ + {"m1", `,"gProp":"GA","cProp":"alpha-zero","vProp":1`}, + {"m2", `,"gProp":"GA","cProp":"beta","vProp":1`}, + {"m3", `,"gProp":"GA","cProp":"beta","vProp":2`}, + {"m4", `,"gProp":"GA","vProp":1`}, + {"m5", `,"gProp":"GB","cProp":"alpha-one","vProp":1`}, + {"m6", `,"gProp":"GB","cProp":"gamma","vProp":1`}, + {"m7", `,"gProp":"GB","cProp":null,"vProp":1`}, + {"m8", `,"gProp":"GC","cProp":"alpha-zero","vProp":1`}, + } + for _, seed := range seeds { + excl26Seed(t, svc, seed.id, excl26Doc(seed.id, seed.fields)) + } + + pivotPlan := excl26EvidencePlan(t, 0, 100) + pivot := pivotPlan.Operations[1].TableShapeExclusionReturn.Pivot + pivot.UnlistedEvidenceColumn = "__loom_reshape_unlisted_count" + pivotPlan.Operations[1] = ir.PhysicalOperation{Kind: ir.PhysicalGroupedPivotOp, GroupedPivot: &pivot} + pivotPlan.Operations = append(pivotPlan.Operations, ir.PhysicalOperation{ + Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: pivotReturnProjections(pivot)}, + }) + pivotRendered, err := RenderPhysicalPlan(pivotPlan, excl26Binding(t, svc)) + if err != nil { + t.Fatalf("render pivot leg live: %v", err) + } + grpCol := excl26ColIndex(t, pivotRendered.OutputColumns, "grp") + countCol := excl26ColIndex(t, pivotRendered.OutputColumns, "__loom_reshape_unlisted_count") + pivotRows := svc.runRendered(t, pivotRendered) + wantPerGroup := map[string]string{} + for _, row := range pivotRows { + wantPerGroup[row[grpCol]] = row[countCol] + } + if wantPerGroup["GA"] != "3" || wantPerGroup["GB"] != "2" || wantPerGroup["GC"] != "0" { + t.Fatalf("pivot unlisted counts drifted from the fixture (GA/GB/GC = %v/%v/%v):\n%v", + wantPerGroup["GA"], wantPerGroup["GB"], wantPerGroup["GC"], pivotRows) + } + + out := excl26RenderLive(t, svc, excl26EvidencePlan(t, 0, 100)) + rows := svc.runRendered(t, out) + idCol := excl26ColIndex(t, out.OutputColumns, ir.PhysicalTableShapeExclusionOutputRowIDField) + gotPerGroup := map[string]int{} + for _, row := range rows { + for _, group := range []string{"GA", "GB", "GC"} { + if strings.Contains(row[idCol], `,"`+group+`"]]`) { + gotPerGroup[group]++ + } + } + } + for group, want := range wantPerGroup { + if fmt.Sprint(gotPerGroup[group]) != want { + t.Fatalf("group %s: evidence rows %d, pivot unlisted count %s — predicate re-port diverged:\n%v", + group, gotPerGroup[group], want, rows) + } + } + + if _, err := svc.iso.Exec(svc.ctx, + `UPDATE `+svc.schema+`.fhir_resource SET payload = payload || '{"vProp":42}'::jsonb + WHERE project = $1 AND dataset_generation = $2`, excl26Project, excl26Generation); err != nil { + svc.fail("perturb value column", err) + } + after := svc.runRendered(t, out) + if strings.Join(flattenRows(after), "|") != strings.Join(flattenRows(rows), "|") { + t.Fatalf("value-column perturbation perturbed the evidence rows:\nbefore %v\nafter %v", rows, after) + } +} + +func pivotReturnProjections(pivot ir.PhysicalGroupedPivot) []ir.PhysicalProjection { + names := []string{pivotRowIDColumn} + for _, key := range pivot.GroupKeys { + name := key.Output + if name == "" { + name = key.Column + } + names = append(names, name) + } + for _, category := range pivot.Categories { + names = append(names, category.Output) + } + if pivot.UnlistedCategoryPolicy == "EXCLUDE_WITH_EVIDENCE" { + names = append(names, pivot.UnlistedEvidenceColumn) + } + projections := make([]ir.PhysicalProjection, 0, len(names)) + for _, name := range names { + projections = append(projections, ir.PhysicalProjection{ + Name: name, + Value: ir.PhysicalValue{Variable: pivot.OutputRowVariable, Path: []string{name}}, + }) + } + return projections +} + +func flattenRows(rows [][]string) []string { + var out []string + for _, row := range rows { + out = append(out, strings.Join(row, "\x1f")) + } + return out +} + +func TestExclusion26LivePaging(t *testing.T) { + svc := excl26LiveService(t) + for index := 0; index < 5; index++ { + excl26Seed(t, svc, fmt.Sprintf("p%d", index), + excl26Doc(fmt.Sprintf("p%d", index), fmt.Sprintf(`,"gProp":"G1","cProp":"unlisted-%d"`, index))) + } + full := svc.runRendered(t, excl26RenderLive(t, svc, excl26EvidencePlan(t, 0, 100))) + if len(full) != 5 { + t.Fatalf("unpaginated evidence = %d, want 5:\n%v", len(full), full) + } + page1 := svc.runRendered(t, excl26RenderLive(t, svc, excl26EvidencePlan(t, 0, 2))) + page2 := svc.runRendered(t, excl26RenderLive(t, svc, excl26EvidencePlan(t, 2, 2))) + if len(page1) != 3 || len(page2) != 3 { + t.Fatalf("probe pages = %d/%d, want the limit+1 fetch of 2/2 (3 each):\n%v\n%v", len(page1), len(page2), page1, page2) + } + recombined := append(append([][]string{}, page1[:2]...), page2[:2]...) + for index := range recombined { + if strings.Join(recombined[index], "\x1f") != strings.Join(full[index], "\x1f") { + t.Fatalf("page1++page2 diverges from the unpaginated list at %d:\n%v\n%v", index, recombined, full) + } + } + tail := svc.runRendered(t, excl26RenderLive(t, svc, excl26EvidencePlan(t, 4, 2))) + if len(tail) != 1 { + t.Fatalf("tail page = %d rows, want 1 (no phantom probe row):\n%v", len(tail), tail) + } + probe := svc.runRendered(t, excl26RenderLive(t, svc, excl26EvidencePlan(t, 0, 4))) + if len(probe) != 5 { + t.Fatalf("limit-4 probe returned %d rows, want the fifth row to ride the limit+1 fetch:\n%v", len(probe), probe) + } + again := svc.runRendered(t, excl26RenderLive(t, svc, excl26EvidencePlan(t, 0, 2))) + if strings.Join(flattenRows(again), "|") != strings.Join(flattenRows(page1), "|") { + t.Fatalf("page order is not deterministic:\n%v\n%v", page1, again) + } +} + +func TestExclusion26LiveHostileValues(t *testing.T) { + svc := excl26LiveService(t) + hostile := []string{ + `a"b`, + `a'b`, + `a\b`, + `'; DROP TABLE fhir_resource;--`, + `é中`, + `123`, + } + for index, value := range hostile { + encoded, err := json.Marshal(map[string]string{ + "resourceType": "Patient", + "id": fmt.Sprintf("h%d", index), + "gProp": value, + "cProp": fmt.Sprintf(`un"listed\%c`, 'A'+rune(index)), + }) + if err != nil { + t.Fatalf("marshal hostile fixture: %v", err) + } + excl26Seed(t, svc, fmt.Sprintf("h%d", index), json.RawMessage(encoded)) + } + out := excl26RenderLive(t, svc, excl26EvidencePlan(t, 0, 100)) + rows := svc.runRendered(t, out) + if len(rows) != len(hostile) { + t.Fatalf("rows = %d, want %d", len(rows), len(hostile)) + } + idCol := excl26ColIndex(t, out.OutputColumns, ir.PhysicalTableShapeExclusionOutputRowIDField) + var got, want []string + for _, row := range rows { + got = append(got, row[idCol]) + } + for _, value := range hostile { + encoded, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal identity member: %v", err) + } + want = append(want, `["GROUPED_PIVOT","p26-hand-construct",["STRING",`+string(encoded)+`]]`) + } + sortStrings(want) + sortStrings(got) + for index := range want { + if got[index] != want[index] { + t.Fatalf("hostile identity %d = %q, want %q", index, got[index], want[index]) + } + } + for _, value := range hostile { + if strings.Contains(out.SQL, value) { + t.Fatalf("hostile value %q reached the statement text", value) + } + } + var survivors int + if err := svc.iso.QueryRow(svc.ctx, + `SELECT count(*) FROM `+svc.schema+`.fhir_resource WHERE project = $1`, excl26Project).Scan(&survivors); err != nil { + svc.fail("count survivors", err) + } + if survivors != len(hostile) { + t.Fatalf("surviving rows = %d, want %d (parameters are inert)", survivors, len(hostile)) + } +} + +func excl26SnapshotOwnedResources(ctx context.Context, raw string) ([]string, error) { + pool, err := pgxpool.New(ctx, raw) + if err != nil { + return nil, fmt.Errorf("connect: %w", err) + } + defer pool.Close() + rows, err := pool.Query(ctx, + `SELECT datname FROM pg_database WHERE datname LIKE 'verify_sql26v1_%'`+ + ` UNION ALL SELECT nspname FROM pg_namespace WHERE nspname LIKE 'verify_sql26v1_%'`) + if err != nil { + return nil, fmt.Errorf("snapshot: %w", err) + } + defer rows.Close() + var names []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("scan: %v", err) + } + names = append(names, name) + } + return names, rows.Err() +} + +func excl26RunDeltaAudit(inner func() int) int { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return inner() + } + beforeList, err := excl26SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-26 delta audit: snapshot before run: %v\n", err) + return inner() + } + if len(beforeList) > 0 { + fmt.Fprintf(os.Stderr, "pg-sql-26 delta audit: pre-snapshot found %d surviving verify_sql26v1_* resources before this run: %v\n", len(beforeList), beforeList) + return 1 + } + before := make(map[string]bool, len(beforeList)) + for _, name := range beforeList { + before[name] = true + } + code := inner() + after, err := excl26SnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-26 delta audit: snapshot after run: %v\n", err) + code = 1 + } + excl26FixturesMu.Lock() + defer excl26FixturesMu.Unlock() + for _, name := range after { + if before[name] { + continue + } + if excl26TrackedFixtures[name] { + fmt.Fprintf(os.Stderr, "pg-sql-26 delta audit: %q was created and tracked by this run and still survives\n", name) + code = 1 + continue + } + fmt.Fprintf(os.Stderr, "pg-sql-26 delta audit: note — %q was born during this run but is not owned by it (sibling lane?)\n", name) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + survivors, err := excl26SnapshotOwnedResources(ctx, raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-26 delta audit: survivor scan: %v\n", err) + code = 1 + } else if len(survivors) > 0 { + fmt.Fprintf(os.Stderr, "pg-sql-26 delta audit: %d verify_sql26v1_* resources survive: %v\n", len(survivors), survivors) + code = 1 + } + return code +} + +func TestZZNoLeftoverExcl26Resources(t *testing.T) { + if err := auditNoLeftoverVerifyNamespace(excl26Prefix); err != nil { + t.Fatalf("zero-leftover census: %v", err) + } +} diff --git a/internal/dataframe/compiler/render/postgres/table_shape_exclusions_test.go b/internal/dataframe/compiler/render/postgres/table_shape_exclusions_test.go new file mode 100644 index 000000000..0dee350c1 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/table_shape_exclusions_test.go @@ -0,0 +1,669 @@ +package postgres + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/recipe" +) + +const ( + excl26Project = "proj-a" + excl26Generation = "gen-1" + excl26AuthPath = "FHIR/Patient" + excl26Prefix = "verify_sql26v1_" +) + +const ( + excl26OffsetBind = "table_shape_exclusion_offset" + excl26LimitBind = "table_shape_exclusion_limit" + excl26FetchLimitBind = "table_shape_exclusion_fetch_limit" +) + +type excl26Config struct { + categories []ir.PhysicalGroupedPivotCategory + categoryType string + valueType string + keyless bool + presence string + omitNulls bool + stable string + offset int + zeroOffset bool + limit int + duplicate string + missing string + unlisted string + bindDrop string + policyDrift string +} + +func excl26DefaultCategories() []ir.PhysicalGroupedPivotCategory { + return []ir.PhysicalGroupedPivotCategory{ + {Output: "cell_zero", MatchKind: ir.PhysicalPivotCategoryValueMatch, ValueBindKey: "cat_zero", ValueKind: "INTEGER"}, + {Output: "cell_one", MatchKind: ir.PhysicalPivotCategoryValueMatch, ValueBindKey: "cat_one", ValueKind: "INTEGER"}, + } +} + +func excl26Plan(t *testing.T, cfg excl26Config) ir.PhysicalPlan { + t.Helper() + categories := cfg.categories + if categories == nil { + categories = excl26DefaultCategories() + } + categoryType := cfg.categoryType + if categoryType == "" { + categoryType = "STRING" + for _, category := range categories { + if category.MatchKind == ir.PhysicalPivotCategoryValueMatch && category.ValueKind != "" { + categoryType = category.ValueKind + break + } + } + } + valueType := cfg.valueType + if valueType == "" { + valueType = "DECIMAL" + } + duplicate := cfg.duplicate + if duplicate == "" { + duplicate = "ERROR" + } + missing := cfg.missing + if missing == "" { + missing = "NULL" + } + unlisted := cfg.unlisted + if unlisted == "" { + unlisted = string(ir.PhysicalPivotUnlistedCategoryExcludeWithEvidence) + } + if cfg.policyDrift != "" { + unlisted = cfg.policyDrift + } + offset, limit := cfg.offset, cfg.limit + if limit == 0 { + limit = 5 + } + if offset == 0 && !cfg.zeroOffset { + offset = 2 + } + bindVars := map[string]any{ + "root_collection": "Patient", + "project": excl26Project, + "dataset_generation": excl26Generation, + "auth_resource_paths": []string{excl26AuthPath}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + "p26_construction_id": "p26-hand-construct", + "cat_zero": int64(0), + "cat_one": int64(1), + excl26OffsetBind: offset, + excl26LimitBind: limit, + excl26FetchLimitBind: limit + 1, + } + for _, category := range categories { + if category.MatchKind != ir.PhysicalPivotCategoryValueMatch || category.ValueBindKey == "" { + continue + } + if _, exists := bindVars[category.ValueBindKey]; exists { + continue + } + switch category.ValueKind { + case "STRING": + bindVars[category.ValueBindKey] = "alpha-" + category.ValueBindKey + case "INTEGER": + bindVars[category.ValueBindKey] = int64(0) + case "DECIMAL": + bindVars[category.ValueBindKey] = 0.5 + case "BOOLEAN": + bindVars[category.ValueBindKey] = true + } + } + for _, drop := range strings.Split(cfg.bindDrop, ",") { + if drop != "" { + delete(bindVars, drop) + } + } + + inputs := []ir.PhysicalProjection{ + {Name: "_key", Hidden: true, Value: ir.PhysicalValue{Variable: "root", Path: []string{"_key"}}}, + {Name: "gcol", Value: ir.PhysicalValue{Variable: "root", Path: []string{"gProp"}}}, + {Name: "ccol", Value: ir.PhysicalValue{Variable: "root", Path: []string{"cProp"}}}, + {Name: "vcol", Value: ir.PhysicalValue{Variable: "root", Path: []string{"vProp"}}}, + } + pivot := &ir.PhysicalGroupedPivot{ + ConstructionID: "p26-hand-construct", + ConstructionIDBindKey: "p26_construction_id", + InputRowVariable: "irow", + GroupRowsVariable: "grow", + OutputRowVariable: "prow", + InputProjections: inputs, + CategoryColumn: "ccol", + CategoryType: categoryType, + ValueColumn: "vcol", + ValueType: valueType, + Categories: categories, + DuplicatePolicy: duplicate, + MissingCellPolicy: missing, + UnlistedCategoryPolicy: unlisted, + } + if unlisted == string(ir.PhysicalPivotUnlistedCategoryExcludeWithEvidence) { + pivot.UnlistedEvidenceColumn = "unlisted_n" + } + switch cfg.presence { + case "contract": + presence := &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + Paths: [][]string{{"cProp"}}, + } + pivot.CategoryPresence = presence + pivot.CategoryPresenceColumn = "cProp_present" + for index := range pivot.InputProjections { + if pivot.InputProjections[index].Name == pivot.CategoryColumn { + pivot.InputProjections[index].Presence = presence + } + } + case "from-input": + pivot.CategoryPresenceFromInput = true + pivot.CategoryPresenceColumn = "cpresent" + pivot.InputProjections = append(pivot.InputProjections, ir.PhysicalProjection{ + Name: "cpresent", + Value: ir.PhysicalValue{Variable: "root", Path: []string{"cPresent"}}, + }) + } + if cfg.omitNulls { + for index := range pivot.InputProjections { + if pivot.InputProjections[index].Name != pivot.CategoryColumn { + continue + } + pivot.InputProjections[index] = ir.PhysicalProjection{ + Name: pivot.CategoryColumn, + Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalOmitNulls, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Patient", + Selector: specSelector(t, "gender"), + }, + }, + } + } + } + switch cfg.stable { + case "rowid": + pivot.InputProjections = append(pivot.InputProjections, ir.PhysicalProjection{ + Name: pivotRowIDColumn, Hidden: true, + Value: ir.PhysicalValue{Variable: "root", Path: []string{"rowId"}}, + }) + case "none": + trimmed := pivot.InputProjections[:0] + for _, projection := range pivot.InputProjections { + if projection.Name == "_key" || projection.Name == pivotRowIDColumn { + continue + } + trimmed = append(trimmed, projection) + } + pivot.InputProjections = trimmed + } + if !cfg.keyless { + pivot.GroupKeys = []ir.PhysicalGroupedPivotKey{ + {Column: "gcol", Variable: "g0", Kind: "STRING", Output: "grp"}, + } + } + + operations := []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalTableShapeExclusionReturnOp, TableShapeExclusionReturn: &ir.PhysicalTableShapeExclusionReturn{ + Pivot: *pivot, OffsetBindKey: excl26OffsetBind, LimitBindKey: excl26LimitBind, + FetchLimitBindKey: excl26FetchLimitBind, + }}, + } + return ir.PhysicalPlan{Version: 1, Engine: ir.PhysicalEngineAQL, BindVars: bindVars, Operations: operations} +} + +func excl26ProductionTerminalPlan(t *testing.T, bundle recipe.Bundle, offset, limit int) ir.PhysicalPlan { + t.Helper() + plan := pivot14CompileBundle(t, bundle, []string{excl26AuthPath}) + pivotIndex := -1 + var pivot *ir.PhysicalGroupedPivot + for index := range plan.Operations { + if plan.Operations[index].Kind == ir.PhysicalGroupedPivotOp { + pivotIndex, pivot = index, plan.Operations[index].GroupedPivot + } + } + if pivot == nil { + t.Fatalf("production plan carries no GROUPED_PIVOT: %#v", plan.Operations) + } + for _, bind := range []string{excl26OffsetBind, excl26LimitBind, excl26FetchLimitBind} { + if _, exists := plan.BindVars[bind]; exists { + t.Fatalf("bind %q is already defined", bind) + } + } + plan.BindVars[excl26OffsetBind] = offset + plan.BindVars[excl26LimitBind] = limit + plan.BindVars[excl26FetchLimitBind] = limit + 1 + plan.Operations = append(plan.Operations[:pivotIndex], ir.PhysicalOperation{ + Kind: ir.PhysicalTableShapeExclusionReturnOp, + Source: plan.Operations[pivotIndex].Source, + TableShapeExclusionReturn: &ir.PhysicalTableShapeExclusionReturn{ + Pivot: *pivot, OffsetBindKey: excl26OffsetBind, LimitBindKey: excl26LimitBind, + FetchLimitBindKey: excl26FetchLimitBind, + }, + }) + if err := plan.Validate(); err != nil { + t.Fatalf("validate minted terminal plan: %v", err) + } + return plan +} + +func excl26Bundle(name string, categories []recipe.GroupedPivotCategory, filters []recipe.Filter) recipe.Bundle { + zero := int64(0) + one := int64(1) + if categories == nil { + categories = []recipe.GroupedPivotCategory{ + {Key: recipe.TableScalar{Kind: recipe.TableScalarInteger, Integer: &zero}, Output: "birth_zero", Label: "Zero"}, + {Key: recipe.TableScalar{Kind: recipe.TableScalarInteger, Integer: &one}, Output: "birth_one", Label: "One"}, + } + } + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: name, TranslationVersion: "test", + Outputs: []recipe.Output{{ + Name: "patients", RootResourceType: "Patient", RowGrain: "patient", Filters: filters, + Fields: []recipe.Field{ + {Name: "grp", Expr: recipe.Expression{Select: "gender"}}, + {Name: "category", Expr: recipe.Expression{Select: "multipleBirthInteger"}}, + {Name: "value", Expr: recipe.Expression{Select: "multipleBirthInteger"}}, + }, + TableReshape: &recipe.TableReshape{ + Kind: recipe.TableReshapeGroupedPivot, + GroupedPivot: &recipe.GroupedPivot{ + ConstructionID: name, GroupKeys: []string{"grp"}, + CategoryColumn: "category", ValueColumn: "value", + Categories: categories, + DuplicatePolicy: recipe.PivotDuplicateSum, + MissingCellPolicy: recipe.PivotMissingCellNull, + UnlistedCategoryPolicy: recipe.PivotUnlistedCategoryExcludeWithEvidence, + }, + }, + }}, + } +} + +func excl26Render(t *testing.T, plan ir.PhysicalPlan) string { + t.Helper() + rendered, err := RenderTableShapeExclusions(plan, mustBinding(t)) + if err != nil { + t.Fatalf("RenderTableShapeExclusions: %v", err) + } + return rendered.SQL +} + +func excl26Refused(t *testing.T, plan ir.PhysicalPlan, label, wantFragment string) { + t.Helper() + _, doorErr := RenderTableShapeExclusions(plan, mustBinding(t)) + if doorErr == nil { + t.Fatalf("%s: expected typed refusal, got success", label) + } + if !errors.Is(doorErr, ErrUnsupportedPostgresPlan) { + t.Fatalf("%s: refusal is not ErrUnsupportedPostgresPlan: %v", label, doorErr) + } + if wantFragment != "" && !strings.Contains(doorErr.Error(), wantFragment) { + t.Fatalf("%s: refusal %q lacks %q", label, doorErr.Error(), wantFragment) + } + _, dispatchErr := RenderPhysicalPlan(plan, mustBinding(t)) + if dispatchErr == nil || dispatchErr.Error() != doorErr.Error() { + t.Fatalf("%s: door and dispatch differ:\n door %v\n dispatch %v", label, doorErr, dispatchErr) + } +} + +func TestExclusion26GoldenCanonicalStatement(t *testing.T) { + sql := excl26Render(t, excl26Plan(t, excl26Config{presence: "contract", stable: "key"})) + for _, want := range []string{ + "WITH px_src AS (SELECT ", + `FROM "` + testSchema + `"."fhir_resource" AS r0`, + "FROM px_src AS p", + "WHERE NOT COALESCE((", + "COALESCE(jsonb_typeof(", + "= 'object'", + "#>> '{}') <> ''", + "AS \"c0\"", "AS \"c1\"", "AS \"c2\"", "AS \"c3\"", "AS \"c4\"", + "AS \"c5\"", "AS \"c6\"", "AS \"c7\"", "AS \"c8\"", + "ELSE $", + "THEN NULL ELSE $", + `'[' || '"GROUPED_PIVOT"' || ',' || to_jsonb($`, + "OFFSET $", " LIMIT $", + `COLLATE "C" ASC`, + " DESC", + "jsonb_typeof(p.", + "::jsonb)", + } { + if !strings.Contains(sql, want) { + t.Fatalf("golden statement lost %q:\n%s", want, sql) + } + } + for _, forbidden := range []string{"vProp", "p26-hand-construct", "FHIR/Patient", excl26Project, excl26Generation} { + if strings.Contains(sql, forbidden) { + t.Fatalf("statement text carries data that must be a parameter (%q):\n%s", forbidden, sql) + } + } + for _, forbidden := range []string{ + ir.PhysicalTableShapeExclusionResourceTypeField, + ir.PhysicalTableShapeExclusionOutputRowIDField, + ir.PhysicalTableShapeExclusionReasonUnlistedCategory, + ir.PhysicalTableShapeExclusionIdentityExact, + ir.PhysicalTableShapeExclusionIdentityUnavailable, + ir.PhysicalTableShapeExclusionSourceIdentityUnavailable, + } { + if strings.Contains(sql, forbidden) { + t.Fatalf("evidence label %q was interpolated into statement text:\n%s", forbidden, sql) + } + } + if strings.Contains(sql, "= 0::") || strings.Contains(sql, "= 1::") { + t.Fatalf("frozen category value was interpolated, not parameterized:\n%s", sql) + } +} + +func TestExclusion26GoldenColumnOrderAndParameterLaw(t *testing.T) { + plan := excl26Plan(t, excl26Config{presence: "contract", stable: "key"}) + rendered, err := RenderTableShapeExclusions(plan, mustBinding(t)) + if err != nil { + t.Fatalf("render: %v", err) + } + want := []string{ + ir.PhysicalTableShapeExclusionResourceTypeField, + ir.PhysicalTableShapeExclusionResourceIDField, + ir.PhysicalTableShapeExclusionIdentityStatusField, + ir.PhysicalTableShapeExclusionCategoryValueField, + ir.PhysicalTableShapeExclusionCategoryPresentField, + ir.PhysicalTableShapeExclusionCategoryTypeField, + ir.PhysicalTableShapeExclusionOutputRowIDField, + ir.PhysicalTableShapeExclusionReasonField, + ir.PhysicalTableShapeExclusionOmissionField, + } + if strings.Join(rendered.OutputColumns, ",") != strings.Join(want, ",") { + t.Fatalf("output columns = %v, want the AQL RETURN order %v", rendered.OutputColumns, want) + } + if len(rendered.Args) < 10 { + t.Fatalf("statement carries only %d parameters", len(rendered.Args)) + } + last := len(rendered.Args) - 1 + if rendered.Args[last].Type != "bigint" || rendered.Args[last].Value.(int64) != 6 { + t.Fatalf("final argument must be the fetch limit $%d=6, got %#v", last+1, rendered.Args[last]) + } + if rendered.Args[last-1].Type != "bigint" || rendered.Args[last-1].Value.(int64) != 2 { + t.Fatalf("penultimate argument must be the offset $%d=2, got %#v", last, rendered.Args[last-1]) + } + if !strings.HasSuffix(rendered.SQL, "OFFSET $"+fmt.Sprint(last)+" LIMIT $"+fmt.Sprint(last+1)) { + t.Fatalf("window is not the trailing ordered pair:\n%s", rendered.SQL) + } + for index := 1; index <= len(rendered.Args); index++ { + needle := "$" + fmt.Sprint(index) + if !strings.Contains(rendered.SQL, needle) && !strings.Contains(rendered.SQL, needle+" ") { + t.Fatalf("parameter $%d (%#v) is never referenced", index, rendered.Args[index-1].Value) + } + } + for _, parameter := range rendered.Args { + if parameter.Value == nil { + t.Fatal("a renderer argument is nil") + } + } +} + +func TestExclusion26GoldenCategoryMatchKinds(t *testing.T) { + sql := excl26Render(t, excl26Plan(t, excl26Config{ + presence: "contract", + categories: []ir.PhysicalGroupedPivotCategory{ + {Output: "cell_alpha", MatchKind: ir.PhysicalPivotCategoryValueMatch, ValueBindKey: "cat_alpha_s", ValueKind: "STRING"}, + {Output: "cell_null", MatchKind: ir.PhysicalPivotCategoryNullMatch}, + {Output: "cell_missing", MatchKind: ir.PhysicalPivotCategoryMissingMatch}, + }, + categoryType: "STRING", + })) + if !strings.Contains(sql, `jsonb_typeof(p.`) || !strings.Contains(sql, `= 'string'`) { + t.Fatalf("VALUE category lost its typeof conjunct:\n%s", sql) + } + if !strings.Contains(sql, "= 'null'::jsonb") { + t.Fatalf("NULL category lost its stored-null disjunct:\n%s", sql) + } + if !strings.Contains(sql, "NOT (") { + t.Fatalf("MISSING category lost its NOT(present) arm:\n%s", sql) + } + omit := excl26Render(t, excl26Plan(t, excl26Config{ + omitNulls: true, + categories: []ir.PhysicalGroupedPivotCategory{ + {Output: "cell_null", MatchKind: ir.PhysicalPivotCategoryNullMatch}, + }, + categoryType: "STRING", + })) + if !strings.Contains(omit, " IS NOT NULL AND ") || strings.Contains(omit, "IS NULL OR ") { + t.Fatalf("OMIT_NULLS NULL arm did not take the registered branch:\n%s", omit) + } + if strings.Contains(sql, "IS NOT NULL AND p.") { + t.Fatalf("the preserved-null plan took the OMIT_NULLS NULL arm:\n%s", sql) + } +} + +func TestExclusion26GoldenPresenceForms(t *testing.T) { + contract := excl26Render(t, excl26Plan(t, excl26Config{presence: "contract"})) + if !strings.Contains(contract, "jsonb_exists(r0.\"payload\", $") || !strings.Contains(contract, " THEN true ELSE false END") { + t.Fatalf("presence contract lost its CASE proof:\n%s", contract) + } + companion := excl26Render(t, excl26Plan(t, excl26Config{presence: "from-input"})) + if !strings.Contains(companion, "COALESCE(p.") || !strings.Contains(companion, " = 'true'::jsonb, false)") { + t.Fatalf("input companion lost its two-valued COALESCE:\n%s", companion) + } + plain := excl26Render(t, excl26Plan(t, excl26Config{})) + if !strings.Contains(plain, "true AS px_") || + !strings.Contains(plain, "to_jsonb(p.px_") || !strings.Contains(plain, ") AS \"c4\"") { + t.Fatalf("PRESERVE_NULL presence must render always-present through a slot, exactly as AQL's HAS on an always-populated projected column does:\n%s", plain) + } + if strings.Contains(plain, "ORDER BY") && strings.Contains(plain[strings.Index(plain, "ORDER BY"):], "true ASC") { + t.Fatalf("a bare constant reached the ORDER BY (SQLSTATE 42601):\n%s", plain) + } + if strings.Contains(plain, "'true'::jsonb ELSE") { + t.Fatalf("a plan with no presence contract invented a document-path proof:\n%s", plain) + } +} + +func TestExclusion26KeylessTerminalIsIrIllegal(t *testing.T) { + _, err := RenderTableShapeExclusions(excl26Plan(t, excl26Config{keyless: true}), mustBinding(t)) + if err == nil || !strings.Contains(err.Error(), "group keys") { + t.Fatalf("keyless terminal must fail closed on the ir pivot contract, got %v", err) + } +} + +func TestExclusion26GoldenIdentityAndTieBreaker(t *testing.T) { + keyed := excl26Render(t, excl26Plan(t, excl26Config{})) + if strings.Contains(keyed, `"STRING"`) { + t.Fatalf("a group-key kind label was interpolated:\n%s", keyed) + } + if !strings.Contains(keyed, `to_jsonb(r0."logical_id"::text)`) { + t.Fatalf("_key did not read the canonical identity column:\n%s", keyed) + } + if strings.Contains(keyed, `'{_key}'`) { + t.Fatalf("_key was bound as a document path:\n%s", keyed) + } + rowid, err := RenderTableShapeExclusions(excl26Plan(t, excl26Config{stable: "rowid"}), mustBinding(t)) + if err != nil { + t.Fatalf("rowid tie-breaker render: %v", err) + } + if strings.Contains(rowid.SQL, `to_jsonb(r0."logical_id"::text)`) { + t.Fatalf("_key was read although __loom_row_id wins the tie-breaker:\n%s", rowid.SQL) + } + bindsRowID := false + for _, arg := range rowid.Args { + if value, ok := arg.Value.(string); ok && value == "{rowId}" { + bindsRowID = true + } + } + if !bindsRowID { + t.Fatalf("the row-id tie-breaker path is not a bound text[] parameter:\n%s", rowid.SQL) + } +} + +func TestExclusion26RefusalRegister(t *testing.T) { + base := func() ir.PhysicalPlan { return excl26Plan(t, excl26Config{stable: "key"}) } + + sortAfter := base() + sortAfter.Operations = append(sortAfter.Operations, ir.PhysicalOperation{ + Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{Keys: []ir.PhysicalValue{{Variable: "prow", Path: []string{"__loom_row_id"}}}}, + }) + excl26Refused(t, sortAfter, "X1 SORT after the terminal", "must be the final operation") + + insert := func(plan ir.PhysicalPlan, operation ir.PhysicalOperation) ir.PhysicalPlan { + operations := []ir.PhysicalOperation{plan.Operations[0]} + operations = append(operations, operation) + return ir.PhysicalPlan{Version: plan.Version, Engine: plan.Engine, + BindVars: plan.BindVars, Operations: append(operations, plan.Operations[1])} + } + sortInSource := insert(base(), ir.PhysicalOperation{ + Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{Keys: []ir.PhysicalValue{{Variable: "prow", Path: []string{"__loom_row_id"}}}}, + }) + excl26Refused(t, sortInSource, "X1 SORT in the source region", "cannot coexist with the evidence terminal") + limitInSource := insert(base(), ir.PhysicalOperation{ + Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: excl26FetchLimitBind}, + }) + excl26Refused(t, limitInSource, "X1 LIMIT in the source region", "cannot coexist with the evidence terminal") + + notLast := base() + operations := []ir.PhysicalOperation{notLast.Operations[0], notLast.Operations[1]} + operations = append(operations, ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, + Filter: &ir.PhysicalFilter{}}) + notLast.Operations = operations + excl26Refused(t, notLast, "X1 terminal not final", "must be the final operation") + + missingTerminal := base() + missingTerminal.Operations = missingTerminal.Operations[:1] + if _, err := RenderTableShapeExclusions(missingTerminal, mustBinding(t)); err == nil || + !errors.Is(err, ErrUnsupportedPostgresPlan) || + !strings.Contains(err.Error(), "requires a terminal evidence operation") { + t.Fatalf("X1 no terminal: entry must refuse typed: %v", err) + } + if _, err := RenderPhysicalPlan(missingTerminal, mustBinding(t)); err == nil { + t.Fatal("X1 no terminal: dispatch must refuse") + } + + nilPayload := base() + nilPayload.Operations[1].TableShapeExclusionReturn = nil + excl26Refused(t, nilPayload, "X1 terminal payload nil", "missing its payload") + + preview := base() + preview.PreviewSourceWindowByRootID = true + excl26Refused(t, preview, "X2 plan-level preview window", "preview source window") + + drift := excl26Plan(t, excl26Config{stable: "key", policyDrift: "ERROR"}) + excl26Refused(t, drift, "X3 policy drift", "EXCLUDE_WITH_EVIDENCE") + + for _, mutation := range []struct { + label, bind, fragment string + value any + }{ + {"X4 negative offset", excl26OffsetBind, "offset must be non-negative", -1}, + {"X4 zero limit", excl26LimitBind, "limit must be positive", 0}, + {"X4 probe drift", excl26FetchLimitBind, "must be limit+1", 99}, + {"X4 non-int bind", excl26OffsetBind, "must be an int", "2"}, + } { + plan := base() + plan.BindVars[mutation.bind] = mutation.value + excl26Refused(t, plan, mutation.label, mutation.fragment) + } + missingBind := base() + delete(missingBind.BindVars, excl26LimitBind) + excl26Refused(t, missingBind, "X4 missing limit bind", "evidence limit bind") + + excl26Refused(t, excl26Plan(t, excl26Config{stable: "none"}), "X5 no tie-breaker", "no stable row tie-breaker") + + badRead := base() + for index := range badRead.Operations[1].TableShapeExclusionReturn.Pivot.InputProjections { + if badRead.Operations[1].TableShapeExclusionReturn.Pivot.InputProjections[index].Name == "ccol" { + badRead.Operations[1].TableShapeExclusionReturn.Pivot.InputProjections[index].Value.Variable = "elsewhere" + } + } + excl26Refused(t, badRead, "X5 category read is not a root read", "direct document read") + + populationRoot := base() + populationRoot.Operations[0].RootScan.Population = &ir.PhysicalPopulationRootSource{} + excl26Refused(t, populationRoot, "X6 population root", "population, cohort, and candidate-seed roots") + + noRoot := base() + noRoot.Operations = noRoot.Operations[1:] + excl26Refused(t, noRoot, "X1 no leading root scan", "leads with exactly one ROOT_SCAN") +} + +func TestExclusion26AliveControlsRender(t *testing.T) { + for _, caseName := range []string{"canonical", "contract", "companion", "rowid", "zero-page"} { + cfg := excl26Config{} + switch caseName { + case "contract": + cfg.presence = "contract" + case "zero-page": + cfg.zeroOffset = true + case "companion": + cfg.presence = "from-input" + case "rowid": + cfg.stable = "rowid" + } + if sql := excl26Render(t, excl26Plan(t, cfg)); !strings.Contains(sql, "OFFSET $") { + t.Fatalf("%s: control did not render a windowed statement:\n%s", caseName, sql) + } + } + plan := excl26ProductionTerminalPlan(t, excl26Bundle("p26_alive_control", nil, nil), 0, 3) + rendered, err := RenderTableShapeExclusions(plan, mustBinding(t)) + if err != nil { + t.Fatalf("production-compiled terminal must render: %v", err) + } + if !strings.Contains(rendered.SQL, "WHERE NOT COALESCE(") { + t.Fatalf("production statement lost the unlisted filter:\n%s", rendered.SQL) + } +} + +func TestExclusion26DoorEqualsDispatch(t *testing.T) { + plan := excl26Plan(t, excl26Config{stable: "key"}) + direct, directErr := RenderTableShapeExclusions(plan, mustBinding(t)) + viaDispatch, dispatchErr := RenderPhysicalPlan(plan, mustBinding(t)) + if directErr != nil || dispatchErr != nil { + t.Fatalf("both doors must render: %v / %v", directErr, dispatchErr) + } + if direct.SQL != viaDispatch.SQL { + t.Fatalf("door and dispatch disagree:\n%s\n%s", direct.SQL, viaDispatch.SQL) + } + if len(direct.Args) != len(viaDispatch.Args) { + t.Fatalf("door and dispatch disagree on arguments: %d vs %d", len(direct.Args), len(viaDispatch.Args)) + } + plainPlan := withReturn(scopedPlan(excl26Project, excl26Generation, []string{excl26AuthPath}, false), + ir.PhysicalProjection{Name: "k", Value: ir.PhysicalValue{Variable: "root", Path: []string{"_key"}}}) + plain, err := RenderPhysicalPlan(plainPlan, mustBinding(t)) + if err != nil { + t.Fatalf("plain scoped read must still render: %v", err) + } + if strings.Contains(plain.SQL, "px_src") { + t.Fatalf("the exclusion arm claimed a plan that does not carry the terminal:\n%s", plain.SQL) + } +} + +func TestExclusion26SurfaceStableUnderRepeatedRender(t *testing.T) { + plan := excl26Plan(t, excl26Config{presence: "contract", stable: "key"}) + first, err := RenderTableShapeExclusions(plan, mustBinding(t)) + if err != nil { + t.Fatalf("first render: %v", err) + } + second, err := RenderTableShapeExclusions(plan, mustBinding(t)) + if err != nil { + t.Fatalf("second render: %v", err) + } + if first.SQL != second.SQL { + t.Fatal("repeated render is not byte-identical") + } + if fmt.Sprint(first.Args) != fmt.Sprint(second.Args) { + t.Fatalf("repeated render re-ordered arguments:\n%v\n%v", first.Args, second.Args) + } + third, err := RenderTableShapeExclusions(plan, mustBinding(t)) + if err != nil || third.SQL != first.SQL { + t.Fatalf("rendering perturbed the plan: %v", err) + } +} diff --git a/internal/dataframe/compiler/render/postgres/traversal.go b/internal/dataframe/compiler/render/postgres/traversal.go new file mode 100644 index 000000000..6c2ca36ff --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/traversal.go @@ -0,0 +1,598 @@ +package postgres + +import ( + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" +) + +var traversalResourceColumns = []string{ + "auth_resource_path", "dataset_generation", "logical_id", "payload", + "project", "recorded_at", "resource_seq", "resource_type", +} + +const requiredMatchResultValue = 1 + +type validatedTraversal struct { + traversal ir.PhysicalTraversal + label string + labelIndex int + sourceType string + targetType string + anchoredType string + anchoredID string + reachedType string + reachedID string + anchoredIndex int + reachedIndex int +} + +func (r *planRenderer) validateTraversalContract( + plan ir.PhysicalPlan, + traversal ir.PhysicalTraversal, + source rowScope, +) (validatedTraversal, error) { + switch traversal.Direction { + case ir.PhysicalOutbound, ir.PhysicalInbound: + case ir.PhysicalAny: + return validatedTraversal{}, fmt.Errorf("%w: ANY-direction traversal has no proven PostgreSQL contract (the AQL generic navigation surface refuses it too, render/aql/navigation_layout.go:394-396)", ErrUnsupportedPostgresPlan) + default: + return validatedTraversal{}, fmt.Errorf("%w: unknown traversal direction %q", ErrUnsupportedPostgresPlan, string(traversal.Direction)) + } + if traversal.Strategy == ir.PhysicalTraversalEndpointLookup { + return validatedTraversal{}, fmt.Errorf("%w: endpoint-lookup strategy has no proven PostgreSQL contract yet; native typed-identity joins are the only registered shape", ErrUnsupportedPostgresPlan) + } + if traversal.Strategy != "" && traversal.Strategy != ir.PhysicalTraversalNative { + return validatedTraversal{}, fmt.Errorf("%w: unknown traversal strategy %q", ErrUnsupportedPostgresPlan, string(traversal.Strategy)) + } + expectedField := "to_type" + if traversal.Direction == ir.PhysicalInbound { + expectedField = "from_type" + } + if traversal.EdgeTargetTypeField != expectedField { + return validatedTraversal{}, fmt.Errorf("%w: %s traversal must carry EdgeTargetTypeField %q, got %q", + ErrUnsupportedPostgresPlan, string(traversal.Direction), expectedField, traversal.EdgeTargetTypeField) + } + if traversal.EdgeVariable == "" { + return validatedTraversal{}, fmt.Errorf("%w: traversal must name its edge variable (the shared lowering contract always does; render/aql/navigation_layout.go:404)", ErrUnsupportedPostgresPlan) + } + if traversal.EdgeCollectionBindKey == "" { + return validatedTraversal{}, fmt.Errorf("%w: traversal must name the edge collection bind", ErrUnsupportedPostgresPlan) + } + if collection, ok := plan.BindVars[traversal.EdgeCollectionBindKey].(string); !ok || collection != "fhir_edge" { + return validatedTraversal{}, fmt.Errorf("%w: traversal edge collection bind %q must be bound to \"fhir_edge\" (the only registered edge relation; render/aql/navigation_layout.go:407-409)", + ErrUnsupportedPostgresPlan, traversal.EdgeCollectionBindKey) + } + label, ok := plan.BindVars[traversal.EdgeLabelBindKey].(string) + if !ok || strings.TrimSpace(label) == "" { + return validatedTraversal{}, fmt.Errorf("%w: edge label bind %q must be a non-empty string", ErrUnsupportedPostgresPlan, traversal.EdgeLabelBindKey) + } + targetType, ok := plan.BindVars[traversal.TargetTypeBindKey].(string) + if !ok || strings.TrimSpace(targetType) == "" { + return validatedTraversal{}, fmt.Errorf("%w: traversal target type bind %q must be a non-empty string", + ErrUnsupportedPostgresPlan, traversal.TargetTypeBindKey) + } + if traversal.SourceTypeBindKey != "" { + if bound, ok := plan.BindVars[traversal.SourceTypeBindKey].(string); !ok || bound != source.resourceType { + return validatedTraversal{}, fmt.Errorf("%w: traversal source type bind does not match the current row resource type %q", + ErrUnsupportedPostgresPlan, source.resourceType) + } + } + + verified := validatedTraversal{traversal: traversal, label: label, sourceType: source.resourceType, targetType: targetType} + if traversal.Direction == ir.PhysicalOutbound { + verified.anchoredType, verified.anchoredID = "source_resource_type", "source_logical_id" + verified.reachedType, verified.reachedID = "target_resource_type", "target_logical_id" + } else { + verified.anchoredType, verified.anchoredID = "target_resource_type", "target_logical_id" + verified.reachedType, verified.reachedID = "source_resource_type", "source_logical_id" + } + return verified, nil +} + +func (r *planRenderer) edgeOccurrencePredicates( + verified *validatedTraversal, + edgeAlias, targetAlias, sourceAlias string, +) (edgeWhere string, targetOn string) { + if verified.labelIndex == 0 { + verified.labelIndex = r.textParameter(verified.label, "text:"+verified.label) + } + if verified.anchoredIndex == 0 { + verified.anchoredIndex = r.textParameter(verified.sourceType, "text:"+verified.sourceType) + } + if verified.reachedIndex == 0 { + verified.reachedIndex = r.textParameter(verified.targetType, "text:"+verified.targetType) + } + edgeColumns := r.storage.Edge + resourceColumns := r.storage.Resource + + edgeWhere = r.scopedEdgePredicate(edgeAlias) + edgeWhere += " AND " + edgeAlias + "." + edgeColumns.MustColumn("label").SQLText() + + " = $" + strconv.Itoa(verified.labelIndex) + edgeWhere += " AND " + edgeAlias + "." + edgeColumns.MustColumn(verified.anchoredType).SQLText() + + " = $" + strconv.Itoa(verified.anchoredIndex) + edgeWhere += " AND " + edgeAlias + "." + edgeColumns.MustColumn(verified.anchoredID).SQLText() + + " = " + sourceAlias + "." + resourceColumns.MustColumn("logical_id").SQLText() + edgeWhere += " AND " + edgeAlias + "." + edgeColumns.MustColumn(verified.reachedType).SQLText() + + " = $" + strconv.Itoa(verified.reachedIndex) + + targetOn = r.scopedResourcePredicate(targetAlias, verified.reachedIndex) + + " AND " + targetAlias + "." + resourceColumns.MustColumn("logical_id").SQLText() + + " = " + edgeAlias + "." + edgeColumns.MustColumn(verified.reachedID).SQLText() + return edgeWhere, targetOn +} + +func (r *planRenderer) renderTraversal(plan ir.PhysicalPlan, traversal ir.PhysicalTraversal, routeInner bool) error { + source, ok := r.rows[traversal.SourceVariable] + if !ok || source.derivedValue != "" { + return fmt.Errorf("%w: traversal source variable %q does not bind a readable row (edge variables, set variables, and computed unnest values are not readable)", + ErrUnsupportedPostgresPlan, traversal.SourceVariable) + } + verified, err := r.validateTraversalContract(plan, traversal, source) + if err != nil { + return err + } + + seq := r.nextTraversalSequence() + edgeAlias := "e" + strconv.Itoa(seq) + innerAlias := "x" + strconv.Itoa(seq) + lateralAlias := "r" + strconv.Itoa(seq) + edgeWhere, targetOn := r.edgeOccurrencePredicates(&verified, edgeAlias, innerAlias, source.alias) + + if routeInner { + join := "INNER JOIN " + r.storage.Edge.Identifier().SQLText() + " AS " + edgeAlias + + " ON " + edgeWhere + + " INNER JOIN " + r.storage.Resource.Identifier().SQLText() + " AS " + innerAlias + + " ON " + targetOn + r.joins = append(r.joins, join) + r.edgeAliases = append(r.edgeAliases, edgeAlias) + r.rows[traversal.TargetVariable] = rowScope{alias: innerAlias, resourceType: verified.targetType} + return r.window.rememberEdge(traversal.EdgeVariable, edgeAlias) + } + + projection := make([]string, 0, len(traversalResourceColumns)+1) + for _, column := range traversalResourceColumns { + name := r.storage.Resource.MustColumn(column).SQLText() + projection = append(projection, innerAlias+"."+name+" AS "+name) + } + projection = append(projection, edgeAlias+"."+r.storage.Edge.MustColumn("edge_seq").SQLText()+` AS "edge_seq"`) + + join := "LEFT JOIN LATERAL (SELECT " + strings.Join(projection, ", ") + + " FROM " + r.storage.Edge.Identifier().SQLText() + " AS " + edgeAlias + + " INNER JOIN " + r.storage.Resource.Identifier().SQLText() + " AS " + innerAlias + + " ON " + targetOn + + " WHERE " + edgeWhere + + ") AS " + lateralAlias + " ON TRUE" + r.joins = append(r.joins, join) + r.edgeAliases = append(r.edgeAliases, lateralAlias) + r.rows[traversal.TargetVariable] = rowScope{alias: lateralAlias, resourceType: verified.targetType, optional: true} + return r.window.rememberEdge(traversal.EdgeVariable, lateralAlias) +} + +func (r *planRenderer) renderRequiredMatchExists( + plan ir.PhysicalPlan, + predicate ir.PhysicalPredicateExpression, +) (predicateFragment, error) { + if predicate.Exists == nil { + return predicateFragment{}, fmt.Errorf("%w: EXISTS predicate has no subplan payload", ErrUnsupportedPostgresPlan) + } + subplan := *predicate.Exists + if len(subplan.Captures) != 1 { + return predicateFragment{}, fmt.Errorf("%w: required-match subplan must capture exactly one correlation anchor, got %d captures", + ErrUnsupportedPostgresPlan, len(subplan.Captures)) + } + anchorVariable := subplan.Captures[0] + anchor, ok := r.rows[anchorVariable] + if !ok || anchor.derivedValue != "" || anchor.optional { + return predicateFragment{}, fmt.Errorf("%w: required-match subplan capture %q does not bind a readable row (edge variables, computed unnest values, and OPTIONAL LEFT JOIN LATERAL rows cannot anchor a correlation; lower/required_match.go always captures the root row, and NULL-extended anchor columns would let subplan FILTERs compare absent rows)", + ErrUnsupportedPostgresPlan, anchorVariable) + } + if err := validateRequiredMatchReturn(plan, subplan); err != nil { + return predicateFragment{}, err + } + + rows := map[string]rowScope{anchorVariable: anchor} + fromParts := make([]string, 0, 4) + whereParts := make([]string, 0, 4) + sawTraversal := false + nextSource := rowScope{alias: anchor.alias, resourceType: anchor.resourceType} + nextSourceVariable := anchorVariable + for index, operation := range subplan.Operations { + switch operation.Kind { + case ir.PhysicalTraversalOp: + if operation.Traversal == nil { + return predicateFragment{}, fmt.Errorf("%w: required-match subplan operation %d: TRAVERSAL payload is missing", + ErrUnsupportedPostgresPlan, index) + } + traversal := *operation.Traversal + if traversal.SourceVariable != nextSourceVariable { + return predicateFragment{}, fmt.Errorf("%w: required-match subplan operation %d: traversal must anchor on %q (the capture for the first step, the previous step's reached row thereafter), got source %q", + ErrUnsupportedPostgresPlan, index, nextSourceVariable, traversal.SourceVariable) + } + verified, err := r.validateTraversalContract(plan, traversal, nextSource) + if err != nil { + return predicateFragment{}, fmt.Errorf("required-match subplan operation %d: %w", index, err) + } + seq := r.nextTraversalSequence() + edgeAlias := "e" + strconv.Itoa(seq) + targetAlias := "x" + strconv.Itoa(seq) + edgeWhere, targetOn := r.edgeOccurrencePredicates(&verified, edgeAlias, targetAlias, nextSource.alias) + if !sawTraversal { + fromParts = append(fromParts, "FROM "+r.storage.Edge.Identifier().SQLText()+" AS "+edgeAlias) + whereParts = append(whereParts, edgeWhere) + } else { + fromParts = append(fromParts, + "INNER JOIN "+r.storage.Edge.Identifier().SQLText()+" AS "+edgeAlias+" ON "+edgeWhere) + } + fromParts = append(fromParts, + "INNER JOIN "+r.storage.Resource.Identifier().SQLText()+" AS "+targetAlias+" ON "+targetOn) + rows[traversal.TargetVariable] = rowScope{alias: targetAlias, resourceType: verified.targetType} + nextSource = rows[traversal.TargetVariable] + nextSourceVariable = traversal.TargetVariable + sawTraversal = true + case ir.PhysicalFilterOp: + if operation.Filter == nil { + return predicateFragment{}, fmt.Errorf("%w: required-match subplan operation %d: FILTER payload is missing", + ErrUnsupportedPostgresPlan, index) + } + if !sawTraversal { + return predicateFragment{}, fmt.Errorf("%w: required-match subplan operation %d: a FILTER may only follow a traversal step (the match is over reached child rows, and subplan scope is structural)", + ErrUnsupportedPostgresPlan, index) + } + if operation.Filter.Expression == nil { + return predicateFragment{}, fmt.Errorf("%w: required-match subplan operation %d: typed match filters must use the Expression form", + ErrUnsupportedPostgresPlan, index) + } + clause, err := r.renderPredicateFragmentScoped(plan, *operation.Filter.Expression, rows, true) + if err != nil { + return predicateFragment{}, fmt.Errorf("required-match subplan operation %d: %w", index, err) + } + whereParts = append(whereParts, clause.sql) + default: + return predicateFragment{}, fmt.Errorf("%w: required-match subplan operation %d has kind %q; only chained TRAVERSAL steps and typed FILTER operations are registered, and plan-authored scope operations (including the canonical project/generation/auth block) are refused because scope is established structurally from the re-read binds", + ErrUnsupportedPostgresPlan, index, string(operation.Kind)) + } + } + if !sawTraversal { + return predicateFragment{}, fmt.Errorf("%w: required-match subplan must contain at least one traversal step", ErrUnsupportedPostgresPlan) + } + body := "EXISTS (SELECT 1 " + strings.Join(fromParts, " ") + " WHERE " + strings.Join(whereParts, " AND ") + ")" + return predicateFragment{sql: body}, nil +} + +func validateRequiredMatchReturn(plan ir.PhysicalPlan, subplan ir.PhysicalSubplan) error { + expression := subplan.Return + if expression.Kind != ir.PhysicalValueExpression || expression.Value == nil || + expression.Value.BindKey == "" || expression.Value.Variable != "" || len(expression.Value.Path) != 0 { + return fmt.Errorf("%w: required-match subplan return must be the canonical scalar VALUE expression over one bind (the lower/required_match.go constant-1 form)", + ErrUnsupportedPostgresPlan) + } + if expression.Cardinality != ir.PhysicalScalarCardinality || expression.NullBehavior != ir.PhysicalPreserveNull { + return fmt.Errorf("%w: required-match subplan return must be SCALAR cardinality with PRESERVE_NULL null behavior (the lower/required_match.go constant-1 form)", + ErrUnsupportedPostgresPlan) + } + if subplan.Sort != nil || subplan.Unique || subplan.DistinctBy != nil { + return fmt.Errorf("%w: required-match subplan cannot carry projection modifiers", ErrUnsupportedPostgresPlan) + } + if value, ok := plan.BindVars[expression.Value.BindKey].(int); !ok || value != requiredMatchResultValue { + return fmt.Errorf("%w: required-match subplan return bind %q must be bound to the integer constant %d", + ErrUnsupportedPostgresPlan, expression.Value.BindKey, requiredMatchResultValue) + } + return nil +} + +func (r *planRenderer) nextTraversalSequence() int { + r.traversalSeq++ + return r.traversalSeq +} + +func (r *planRenderer) resolveValueInRows(rows map[string]rowScope, value ir.PhysicalValue) (string, error) { + scope, ok := rows[value.Variable] + if !ok || scope.derivedValue != "" { + return "", fmt.Errorf("%w: variable %q does not bind a readable row", ErrUnsupportedPostgresPlan, value.Variable) + } + if len(value.Path) == 1 { + if column := canonicalKeyColumn(value.Path[0]); resourceFilterColumns[column] { + return scope.alias + "." + r.storage.Resource.MustColumn(column).SQLText(), nil + } + } + return r.documentPathExpression(scope.alias, value.Path) +} + +func (r *planRenderer) renderPredicateFragmentScoped( + plan ir.PhysicalPlan, + predicate ir.PhysicalPredicateExpression, + rows map[string]rowScope, + allowSelectorExtract bool, +) (predicateFragment, error) { + switch predicate.Kind { + case ir.PhysicalComparisonPredicate: + if predicate.Comparison == nil { + return predicateFragment{}, fmt.Errorf("%w: COMPARISON predicate has no payload", ErrUnsupportedPostgresPlan) + } + return r.renderComparisonFragmentScoped(plan, *predicate.Comparison, rows, allowSelectorExtract) + case ir.PhysicalNotPredicate: + if len(predicate.Children) != 1 { + return predicateFragment{}, fmt.Errorf("%w: NOT requires exactly one child", ErrUnsupportedPostgresPlan) + } + child, err := r.renderPredicateFragmentScoped(plan, predicate.Children[0], rows, allowSelectorExtract) + if err != nil { + return predicateFragment{}, err + } + if child.mayYieldNull { + return predicateFragment{sql: "COALESCE(NOT (" + child.sql + "), true)"}, nil + } + return predicateFragment{sql: "NOT (" + child.sql + ")"}, nil + case ir.PhysicalAllPredicate: + if len(predicate.Children) == 0 { + return predicateFragment{}, fmt.Errorf("%w: ALL requires at least one child", ErrUnsupportedPostgresPlan) + } + parts := make([]string, 0, len(predicate.Children)) + mayYieldNull := false + for _, child := range predicate.Children { + part, err := r.renderPredicateFragmentScoped(plan, child, rows, allowSelectorExtract) + if err != nil { + return predicateFragment{}, err + } + mayYieldNull = mayYieldNull || part.mayYieldNull + parts = append(parts, "("+part.sql+")") + } + return predicateFragment{sql: strings.Join(parts, " AND "), mayYieldNull: mayYieldNull}, nil + default: + return predicateFragment{}, fmt.Errorf("%w: predicate kind %q is not part of the registered traversal predicate surface", + ErrUnsupportedPostgresPlan, string(predicate.Kind)) + } +} + +func (r *planRenderer) renderComparisonFragmentScoped( + plan ir.PhysicalPlan, + predicate ir.PhysicalPredicate, + rows map[string]rowScope, + allowSelectorExtract bool, +) (predicateFragment, error) { + if predicate.Correlation != nil { + return predicateFragment{}, fmt.Errorf("%w: correlated predicates are not yet implemented for PostgreSQL", + ErrUnsupportedPostgresPlan) + } + if predicate.LeftExpression != nil { + if !allowSelectorExtract { + return predicateFragment{}, fmt.Errorf("%w: selector-expression and correlated predicates are not yet implemented for PostgreSQL", + ErrUnsupportedPostgresPlan) + } + return r.renderSelectorMatchFragment(plan, predicate, rows) + } + if predicate.Quantifier != "" { + return predicateFragment{}, fmt.Errorf("%w: array quantifier %q is not yet implemented for PostgreSQL", + ErrUnsupportedPostgresPlan, string(predicate.Quantifier)) + } + if predicate.Left.BindKey != "" { + return predicateFragment{}, fmt.Errorf("%w: filter left side must be a row path, not a bind value", ErrUnsupportedPostgresPlan) + } + var left string + var err error + switch { + case predicate.Left.Variable != "": + left, err = r.resolveValueInRows(rows, predicate.Left) + if err != nil { + return predicateFragment{}, err + } + default: + return predicateFragment{}, fmt.Errorf("%w: filter must read a row path or a registered selector expression", + ErrUnsupportedPostgresPlan) + } + operator := strings.ToUpper(strings.TrimSpace(predicate.Operator)) + if err := validateDirectOperatorKind(operator, predicate.ValueKind); err != nil { + return predicateFragment{}, err + } + switch operator { + case "EXISTS", "MISSING": + if predicate.Right != nil { + return predicateFragment{}, fmt.Errorf("%w: %s must not carry a right value", ErrUnsupportedPostgresPlan, operator) + } + if isPayloadExpression(left) { + presence := "COALESCE(jsonb_typeof(" + left + "), 'null')" + if operator == "EXISTS" { + return predicateFragment{sql: presence + " <> 'null'"}, nil + } + return predicateFragment{sql: presence + " = 'null'"}, nil + } + if operator == "EXISTS" { + return predicateFragment{sql: left + " IS NOT NULL"}, nil + } + return predicateFragment{sql: left + " IS NULL"}, nil + case "EQUALS", "NOT_EQUALS", "GT", "GTE", "LT", "LTE", "IN": + default: + return predicateFragment{}, fmt.Errorf("%w: filter operator %q is not part of the v1 PostgreSQL surface", + ErrUnsupportedPostgresPlan, predicate.Operator) + } + if predicate.Right == nil || predicate.Right.BindKey == "" { + return predicateFragment{}, fmt.Errorf("%w: %s requires a bind-backed right value", ErrUnsupportedPostgresPlan, operator) + } + value, ok := plan.BindVars[predicate.Right.BindKey] + if !ok { + return predicateFragment{}, fmt.Errorf("%w: filter value bind %q is not defined", ErrUnsupportedPostgresPlan, predicate.Right.BindKey) + } + if !isPayloadExpression(left) { + return r.renderColumnComparison(left, operator, predicate.ValueKind, value) + } + return r.renderPayloadComparison(left, operator, predicate.ValueKind, value) +} + +func (r *planRenderer) renderSelectorMatchFragment( + plan ir.PhysicalPlan, + predicate ir.PhysicalPredicate, + rows map[string]rowScope, +) (predicateFragment, error) { + if predicate.Correlation != nil { + return predicateFragment{}, fmt.Errorf("%w: correlated predicates are not yet implemented for PostgreSQL", + ErrUnsupportedPostgresPlan) + } + if predicate.Quantifier != "" && !predicate.Quantifier.Valid() { + return predicateFragment{}, fmt.Errorf("%w: array quantifier %q is not part of the registered traversal predicate surface", + ErrUnsupportedPostgresPlan, string(predicate.Quantifier)) + } + values, err := r.selectorValuesExpression(*predicate.LeftExpression, rows) + if err != nil { + return predicateFragment{}, err + } + return r.quantifiedMatchFragment(plan, predicate, values) +} + +func (r *planRenderer) quantifiedMatchFragment( + plan ir.PhysicalPlan, + predicate ir.PhysicalPredicate, + values string, +) (predicateFragment, error) { + if predicate.Correlation != nil { + return predicateFragment{}, fmt.Errorf("%w: correlated predicates are not yet implemented for PostgreSQL", + ErrUnsupportedPostgresPlan) + } + if predicate.Quantifier != "" && !predicate.Quantifier.Valid() { + return predicateFragment{}, fmt.Errorf("%w: array quantifier %q is not part of the registered predicate surface", + ErrUnsupportedPostgresPlan, string(predicate.Quantifier)) + } + quantifier := predicate.Quantifier + if quantifier == "" { + quantifier = spec.QuantifierAny + } + operator := strings.ToUpper(strings.TrimSpace(predicate.Operator)) + if err := validateDirectOperatorKind(operator, predicate.ValueKind); err != nil { + return predicateFragment{}, err + } + length := "jsonb_array_length(" + values + ")" + switch operator { + case "EXISTS": + if predicate.Right != nil { + return predicateFragment{}, fmt.Errorf("%w: EXISTS must not carry a right value", ErrUnsupportedPostgresPlan) + } + return predicateFragment{sql: "(" + length + " > 0)"}, nil + case "MISSING": + if predicate.Right != nil { + return predicateFragment{}, fmt.Errorf("%w: MISSING must not carry a right value", ErrUnsupportedPostgresPlan) + } + return predicateFragment{sql: "(" + length + " = 0)"}, nil + case "EQUALS", "NOT_EQUALS", "IN", "GT", "GTE", "LT", "LTE": + case "CONTAINS_TEXT": + return predicateFragment{}, fmt.Errorf("%w: selector CONTAINS_TEXT match needs the AQL TO_STRING/CONTAINS reading (render/aql/predicates.go:243) and is not registered for PostgreSQL", + ErrUnsupportedPostgresPlan) + default: + return predicateFragment{}, fmt.Errorf("%w: selector filter operator %q is not part of the v1 PostgreSQL surface", + ErrUnsupportedPostgresPlan, predicate.Operator) + } + if !predicate.ValueKind.Valid() { + return predicateFragment{}, fmt.Errorf("%w: selector filter value kind %q is invalid", ErrUnsupportedPostgresPlan, string(predicate.ValueKind)) + } + if predicate.Right == nil || predicate.Right.BindKey == "" { + return predicateFragment{}, fmt.Errorf("%w: %s requires a bind-backed right value", ErrUnsupportedPostgresPlan, operator) + } + value, ok := plan.BindVars[predicate.Right.BindKey] + if !ok { + return predicateFragment{}, fmt.Errorf("%w: filter value bind %q is not defined", ErrUnsupportedPostgresPlan, predicate.Right.BindKey) + } + match, err := r.selectorMatchExpression("__sv.v", operator, predicate.ValueKind, value) + if err != nil { + return predicateFragment{}, err + } + anyMatch := "EXISTS (SELECT 1 FROM jsonb_array_elements(" + values + ") AS __sv(v) WHERE " + match + ")" + switch quantifier { + case spec.QuantifierAny: + return predicateFragment{sql: "(" + anyMatch + ")"}, nil + case spec.QuantifierNone: + return predicateFragment{sql: "(NOT " + anyMatch + ")"}, nil + default: + mismatch := "EXISTS (SELECT 1 FROM jsonb_array_elements(" + values + ") AS __sv(v) WHERE NOT (" + match + "))" + return predicateFragment{sql: "(" + length + " > 0 AND NOT " + mismatch + ")"}, nil + } +} + +func (r *planRenderer) selectorValuesExpression( + expression ir.PhysicalExpression, + rows map[string]rowScope, +) (string, error) { + if expression.Kind != ir.PhysicalExtractExpression { + return "", fmt.Errorf("%w: required-match filter left side must be an EXTRACT selector expression, got kind %q", + ErrUnsupportedPostgresPlan, string(expression.Kind)) + } + extract := expression.Extract + if extract == nil { + return "", fmt.Errorf("%w: required-match filter selector is missing its EXTRACT payload", ErrUnsupportedPostgresPlan) + } + if expression.Cardinality != ir.PhysicalArrayCardinality { + return "", fmt.Errorf("%w: required-match filter selector must be the ARRAY-cardinality form lower/required_match.go emits; got %q", + ErrUnsupportedPostgresPlan, string(expression.Cardinality)) + } + if expression.NullBehavior != ir.PhysicalEmptyOnNull { + return "", fmt.Errorf("%w: required-match filter selector must carry the EMPTY_ON_NULL null behavior lower/required_match.go emits; got %q", + ErrUnsupportedPostgresPlan, string(expression.NullBehavior)) + } + switch extract.ExecutionMode { + case "", ir.PhysicalSelectorGeneric, ir.PhysicalSelectorDirectScalar: + case ir.PhysicalSelectorConditionalArray: + return "", fmt.Errorf("%w: required-match filter selector mode CONDITIONAL_ARRAY requires repeated-step rendering, which is not registered for PostgreSQL", ErrUnsupportedPostgresPlan) + default: + return "", fmt.Errorf("%w: required-match filter selector execution mode %q is unknown", + ErrUnsupportedPostgresPlan, string(extract.ExecutionMode)) + } + if extract.Distinct { + return "", fmt.Errorf("%w: required-match filter selector DISTINCT changes element multiplicity the quantified match does not model (render/aql/predicates.go reads the raw value list)", ErrUnsupportedPostgresPlan) + } + if len(extract.Fallbacks) != 0 { + return "", fmt.Errorf("%w: required-match filter selector fallbacks are a separate capability", ErrUnsupportedPostgresPlan) + } + if extract.Prepared != nil { + return "", fmt.Errorf("%w: required-match filter selector prepared-set references are not implemented for PostgreSQL", ErrUnsupportedPostgresPlan) + } + if extract.UnitNormalization != nil { + return "", fmt.Errorf("%w: required-match filter selector unit normalization is not implemented for PostgreSQL", ErrUnsupportedPostgresPlan) + } + if extract.Selector.Filter != nil { + return "", fmt.Errorf("%w: required-match filter selector CONTAINS filters are not implemented for PostgreSQL", ErrUnsupportedPostgresPlan) + } + if len(extract.Selector.Steps) == 0 { + return "", fmt.Errorf("%w: required-match filter selector is empty", ErrUnsupportedPostgresPlan) + } + path := make([]string, 0, len(extract.Selector.Steps)) + for index, step := range extract.Selector.Steps { + switch { + case step.Field == "": + return "", fmt.Errorf("%w: required-match filter selector step %d has no property", ErrUnsupportedPostgresPlan, index) + case step.Iterate: + return "", fmt.Errorf("%w: required-match filter selector repeated ([]) step %d requires array rendering, which is not registered", ErrUnsupportedPostgresPlan, index) + case step.Index != nil: + return "", fmt.Errorf("%w: required-match filter selector indexed step %d requires array rendering, which is not registered", ErrUnsupportedPostgresPlan, index) + } + if !payloadPathSegmentPattern.MatchString(step.Field) { + return "", fmt.Errorf("%w: required-match filter selector step %q is not renderer-safe", ErrUnsupportedPostgresPlan, step.Field) + } + path = append(path, step.Field) + } + if extract.Source.BindKey != "" { + return "", fmt.Errorf("%w: required-match filter selector source must be a row document, not a bind value", ErrUnsupportedPostgresPlan) + } + if len(extract.Source.Path) != 1 || extract.Source.Path[0] != "payload" { + return "", fmt.Errorf(`%w: required-match filter selector source must be the row document at ["payload"], the exact form lower/required_match.go emits; got %v`, + ErrUnsupportedPostgresPlan, extract.Source.Path) + } + scope, ok := rows[extract.Source.Variable] + if !ok { + return "", fmt.Errorf("%w: required-match filter selector variable %q does not bind a readable subplan row", + ErrUnsupportedPostgresPlan, extract.Source.Variable) + } + if strings.TrimSpace(extract.ResourceType) == "" { + return "", fmt.Errorf("%w: required-match filter selector must name the schema resource type it was validated against", ErrUnsupportedPostgresPlan) + } + if extract.ResourceType != scope.resourceType { + return "", fmt.Errorf("%w: required-match filter selector validates against resource type %q, but the row is joined as %q", + ErrUnsupportedPostgresPlan, extract.ResourceType, scope.resourceType) + } + extract_, err := r.documentPathExpression(scope.alias, path) + if err != nil { + return "", err + } + document := "(" + extract_ + ")" + return "(CASE WHEN " + document + " IS NULL OR jsonb_typeof(" + document + ") = 'null' THEN '[]'::jsonb" + + " ELSE jsonb_build_array(" + document + ") END)", nil +} diff --git a/internal/dataframe/compiler/render/postgres/traversal_test.go b/internal/dataframe/compiler/render/postgres/traversal_test.go new file mode 100644 index 000000000..05e4cc109 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/traversal_test.go @@ -0,0 +1,1359 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + control "github.com/calypr/loom/internal/backends/postgres/control" + "os" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/jackc/pgx/v5/pgxpool" +) + +func pgstoreEdgeRow(project, generation, label, patient, specimen, path string) control.EdgeOccurrenceRow { + return control.EdgeOccurrenceRow{ + Project: project, DatasetGeneration: generation, AuthResourcePath: "FHIR/Patient", + Label: label, SourceResourceType: "Patient", SourceLogicalID: patient, + TargetResourceType: "Specimen", TargetLogicalID: specimen, + SourcePath: path, SourceOrdinal: 0, + } +} + +func trvInsertDanglingSourceEdge(t *testing.T, svc *pgir02Service, label, sourceType, sourceID, targetType, targetID, authPath, path string) { + t.Helper() + _, err := svc.iso.Exec(svc.ctx, + `INSERT INTO "`+svc.schema+`"."fhir_edge_occurrence"`+ + ` (project, dataset_generation, auth_resource_path, label,`+ + ` source_resource_type, source_logical_id, target_resource_type, target_logical_id,`+ + ` source_path, source_ordinal)`+ + ` VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,0)`, + itProject, itGeneration, authPath, label, sourceType, sourceID, targetType, targetID, path) + if err != nil { + svc.fail("insert dangling-source edge", err) + } +} + +func trvTraversalPlan(direction ir.PhysicalTraversalDirection, edgeTargetTypeField, label, targetType string) ir.PhysicalPlan { + plan := basePlan() + plan.BindVars["edge_collection"] = "fhir_edge" + plan.BindVars["edge_label"] = label + plan.BindVars["target_type"] = targetType + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "target", EdgeVariable: "edge", + Direction: direction, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: "edge_label", TargetTypeBindKey: "target_type", + EdgeTargetTypeField: edgeTargetTypeField, Strategy: ir.PhysicalTraversalNative, + }}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "patient_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + {Name: "specimen_id", Value: ir.PhysicalValue{Variable: "target", Path: []string{"logical_id"}}}, + }}}, + } + return plan +} + +func TestRenderOutboundTraversalSQLIsExact(t *testing.T) { + plan := trvTraversalPlan(ir.PhysicalOutbound, "to_type", "has-specimen", "Specimen") + rendered := renderOK(t, plan) + wantSQL := `SELECT r0."logical_id" AS "c0", r1."logical_id" AS "c1"` + + ` FROM "pgir02_it_unit"."fhir_resource" AS r0` + + ` LEFT JOIN LATERAL (SELECT x1."auth_resource_path" AS "auth_resource_path",` + + ` x1."dataset_generation" AS "dataset_generation",` + + ` x1."logical_id" AS "logical_id", x1."payload" AS "payload",` + + ` x1."project" AS "project", x1."recorded_at" AS "recorded_at",` + + ` x1."resource_seq" AS "resource_seq", x1."resource_type" AS "resource_type",` + + ` e1."edge_seq" AS "edge_seq"` + + ` FROM "pgir02_it_unit"."fhir_edge_occurrence" AS e1` + + ` INNER JOIN "pgir02_it_unit"."fhir_resource" AS x1` + + ` ON x1."project" = $1 AND x1."dataset_generation" = $2` + + ` AND x1."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND x1."resource_type" = $6 AND x1."logical_id" = e1."target_logical_id"` + + ` WHERE e1."project" = $1 AND e1."dataset_generation" = $2` + + ` AND e1."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND e1."label" = $5 AND e1."source_resource_type" = $4` + + ` AND e1."source_logical_id" = r0."logical_id" AND e1."target_resource_type" = $6)` + + ` AS r1 ON TRUE` + + ` WHERE r0."project" = $1 AND r0."dataset_generation" = $2` + + ` AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND r0."resource_type" = $4` + + ` ORDER BY r0."resource_seq" ASC, r1."edge_seq" ASC` + if rendered.SQL != wantSQL { + t.Fatalf("SQL mismatch:\n got %s\nwant %s", rendered.SQL, wantSQL) + } + if len(rendered.Args) != 6 { + t.Fatalf("args = %d, want 6 (project, generation, auth paths, root type, label, target type)", len(rendered.Args)) + } + for index, want := range []struct { + typ, value string + }{ + {"text", "proj-a"}, {"text", "gen-1"}, {"jsonb", ""}, + {"text", "Patient"}, {"text", "has-specimen"}, {"text", "Specimen"}, + } { + got := rendered.Args[index] + if string(got.Type) != want.typ { + t.Fatalf("arg %d type = %q, want %q", index, string(got.Type), want.typ) + } + if want.value != "" && got.Value != want.value { + t.Fatalf("arg %d value = %#v, want %q", index, got.Value, want.value) + } + } + if raw, ok := rendered.Args[2].Value.(json.RawMessage); !ok || + string(raw) != `["FHIR/Patient","FHIR/Observation"]` { + t.Fatalf("auth argument = %#v", rendered.Args[2].Value) + } + for _, leaked := range []string{"fhir_edge\"", "\"edge\"", "\"target\"", "\"root\""} { + if strings.Contains(rendered.SQL, leaked) { + t.Fatalf("plan data %q leaked into SQL: %s", leaked, rendered.SQL) + } + } +} + +func TestRenderInboundTraversalBindsReverseEndpointColumns(t *testing.T) { + plan := trvTraversalPlan(ir.PhysicalInbound, "from_type", "has-specimen", "Observation") + rendered := renderOK(t, plan) + for _, want := range []string{ + `e1."target_resource_type" = $4`, + `e1."target_logical_id" = r0."logical_id"`, + `e1."source_resource_type" = $6`, + `x1."resource_type" = $6`, + `x1."logical_id" = e1."source_logical_id"`, + `x1."resource_type" AS "resource_type"`, + } { + if !strings.Contains(rendered.SQL, want) { + t.Fatalf("inbound SQL missing %q: %s", want, rendered.SQL) + } + } + if strings.Contains(rendered.SQL, `e1."source_logical_id" = r0."logical_id"`) { + t.Fatalf("inbound traversal must not anchor on the stored source side: %s", rendered.SQL) + } +} + +func TestRenderTraversalOrderByUsesEdgeWitness(t *testing.T) { + plan := basePlan() + plan.BindVars["edge_collection"] = "fhir_edge" + plan.BindVars["edge_label"] = "has-specimen" + plan.BindVars["target_type"] = "Specimen" + plan.BindVars["edge_label_2"] = "has-observation" + plan.BindVars["target_type_2"] = "Observation" + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "specimen", EdgeVariable: "edge", + Direction: ir.PhysicalOutbound, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: "edge_label", TargetTypeBindKey: "target_type", + EdgeTargetTypeField: "to_type", Strategy: ir.PhysicalTraversalNative, + }}, + {Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: "specimen", TargetVariable: "observation", EdgeVariable: "edge2", + Direction: ir.PhysicalOutbound, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: "edge_label_2", TargetTypeBindKey: "target_type_2", + EdgeTargetTypeField: "to_type", Strategy: ir.PhysicalTraversalNative, + }}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "patient_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + {Name: "observation_id", Value: ir.PhysicalValue{Variable: "observation", Path: []string{"logical_id"}}}, + }}}, + } + rendered := renderOK(t, plan) + for _, want := range []string{ + `ORDER BY r0."resource_seq" ASC, r1."edge_seq" ASC, r2."edge_seq" ASC`, + `"fhir_edge_occurrence" AS e1`, `"fhir_edge_occurrence" AS e2`, + `AS r1 ON TRUE`, `AS r2 ON TRUE`, + `e2."source_logical_id" = r1."logical_id"`, + `x2."logical_id" = e2."target_logical_id"`, + } { + if !strings.Contains(rendered.SQL, want) { + t.Fatalf("chained traversal SQL missing %q: %s", want, rendered.SQL) + } + } +} + +func trvRequiredSubplan(operations ...ir.PhysicalOperation) ir.PhysicalSubplan { + return ir.PhysicalSubplan{ + Captures: []string{"root"}, + Operations: operations, + Return: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{BindKey: "required_match_result"}, + }, + } +} + +func trvSubplanTraversal(sourceVariable, targetVariable, labelKey, targetTypeKey string) ir.PhysicalOperation { + return ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: sourceVariable, TargetVariable: targetVariable, + EdgeVariable: targetVariable + "_edge", + Direction: ir.PhysicalOutbound, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: labelKey, TargetTypeBindKey: targetTypeKey, + EdgeTargetTypeField: "to_type", Strategy: ir.PhysicalTraversalNative, + }} +} + +func trvPayloadMatchPredicate(variable, resourceType, operator string, valueKind spec.FilterValueKind, steps ...string) *ir.PhysicalPredicateExpression { + selectorSteps := make([]spec.SelectorStep, 0, len(steps)) + for _, field := range steps { + selectorSteps = append(selectorSteps, spec.SelectorStep{Field: field}) + } + predicate := &ir.PhysicalPredicate{ + Operator: operator, + ValueKind: valueKind, + LeftExpression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: variable, Path: []string{"payload"}}, + ResourceType: resourceType, + Selector: spec.Selector{Steps: selectorSteps}, + }, + }, + } + if operator != "EXISTS" && operator != "MISSING" { + predicate.Right = &ir.PhysicalValue{BindKey: "status_value"} + } + return &ir.PhysicalPredicateExpression{Kind: ir.PhysicalComparisonPredicate, Comparison: predicate} +} + +func trvRequiredPlan(subplan ir.PhysicalSubplan) ir.PhysicalPlan { + plan := basePlan() + plan.BindVars["edge_collection"] = "fhir_edge" + plan.BindVars["edge_label"] = "has-specimen" + plan.BindVars["target_type"] = "Specimen" + plan.BindVars["required_match_result"] = 1 + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Expression: &ir.PhysicalPredicateExpression{Kind: ir.PhysicalExistsPredicate, Exists: &subplan}, + }}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + } + return plan +} + +func TestRenderRequiredMatchExistsIsExactSemiJoin(t *testing.T) { + subplan := trvRequiredSubplan(trvSubplanTraversal("root", "child", "edge_label", "target_type")) + rendered := renderOK(t, trvRequiredPlan(subplan)) + wantSQL := `SELECT r0."logical_id" AS "c0"` + + ` FROM "pgir02_it_unit"."fhir_resource" AS r0` + + ` WHERE r0."project" = $1 AND r0."dataset_generation" = $2` + + ` AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND r0."resource_type" = $4` + + ` AND EXISTS (SELECT 1 FROM "pgir02_it_unit"."fhir_edge_occurrence" AS e1` + + ` INNER JOIN "pgir02_it_unit"."fhir_resource" AS x1` + + ` ON x1."project" = $1 AND x1."dataset_generation" = $2` + + ` AND x1."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND x1."resource_type" = $6 AND x1."logical_id" = e1."target_logical_id"` + + ` WHERE e1."project" = $1 AND e1."dataset_generation" = $2` + + ` AND e1."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND e1."label" = $5 AND e1."source_resource_type" = $4` + + ` AND e1."source_logical_id" = r0."logical_id" AND e1."target_resource_type" = $6)` + + ` ORDER BY r0."resource_seq" ASC` + if rendered.SQL != wantSQL { + t.Fatalf("SQL mismatch:\n got %s\nwant %s", rendered.SQL, wantSQL) + } + if len(rendered.Args) != 6 { + t.Fatalf("args = %d, want 6 (scope and traversal parameters deduped with the root scan)", len(rendered.Args)) + } + if strings.Contains(rendered.SQL, "LIMIT") { + t.Fatalf("EXISTS makes the witness LIMIT 1 unnecessary; none may appear: %s", rendered.SQL) + } + if strings.Contains(rendered.SQL, "LEFT JOIN") { + t.Fatalf("a required match must exclude no-match parents, never null-extend: %s", rendered.SQL) + } + if strings.Contains(rendered.SQL, "COALESCE") { + t.Fatalf("EXISTS is total and needs no null guard: %s", rendered.SQL) + } + if strings.Index(rendered.SQL, "EXISTS") > strings.Index(rendered.SQL, "ORDER BY") { + t.Fatalf("required match must precede the ordering window: %s", rendered.SQL) + } +} + +func TestRenderRequiredMatchInboundCorrelatesReversed(t *testing.T) { + step := trvSubplanTraversal("root", "child", "edge_label", "target_type") + step.Traversal.Direction = ir.PhysicalInbound + step.Traversal.EdgeTargetTypeField = "from_type" + rendered := renderOK(t, trvRequiredPlan(trvRequiredSubplan(step))) + for _, want := range []string{ + `e1."target_resource_type" = $4`, + `e1."target_logical_id" = r0."logical_id"`, + `e1."source_resource_type" = $6`, + `x1."logical_id" = e1."source_logical_id"`, + } { + if !strings.Contains(rendered.SQL, want) { + t.Fatalf("inbound required-match SQL missing %q: %s", want, rendered.SQL) + } + } +} + +func TestRenderRequiredMatchTypedSubplanFilters(t *testing.T) { + t.Run("payload selector comparison", func(t *testing.T) { + plan := trvRequiredPlan(trvRequiredSubplan( + trvSubplanTraversal("root", "child", "edge_label", "target_type"), + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Expression: trvPayloadMatchPredicate("child", "Specimen", "EQUALS", spec.FilterString, "status"), + }}, + )) + plan.BindVars["status_value"] = "final" + rendered := renderOK(t, plan) + document := `(x1."payload" #> $7::text[])` + values := "(CASE WHEN " + document + " IS NULL OR jsonb_typeof(" + document + ") = 'null' THEN '[]'::jsonb" + + " ELSE jsonb_build_array(" + document + ") END)" + want := `AND (EXISTS (SELECT 1 FROM jsonb_array_elements(` + values + `) AS __sv(v) WHERE (__sv.v) = ($8::jsonb)))` + if !strings.Contains(rendered.SQL, want) { + t.Fatalf("typed payload filter missing %q:\n%s", want, rendered.SQL) + } + if len(rendered.Args) != 8 { + t.Fatalf("args = %d, want 8", len(rendered.Args)) + } + if got := rendered.Args[6]; string(got.Type) != "text" || got.Value != "{status}" { + t.Fatalf("path argument = %#v, want text {status}", got) + } + if raw, ok := rendered.Args[7].Value.(json.RawMessage); !ok || string(raw) != `"final"` { + t.Fatalf("value argument = %#v, want jsonb \"final\"", rendered.Args[7].Value) + } + }) + t.Run("column IN over reached row", func(t *testing.T) { + plan := trvRequiredPlan(trvRequiredSubplan( + trvSubplanTraversal("root", "child", "edge_label", "target_type"), + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Expression: &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalComparisonPredicate, + Comparison: &ir.PhysicalPredicate{ + Operator: "IN", + Left: ir.PhysicalValue{Variable: "child", Path: []string{"resource_type"}}, + Right: &ir.PhysicalValue{BindKey: "type_values"}, + }, + }, + }}, + )) + plan.BindVars["type_values"] = []string{"Specimen"} + rendered := renderOK(t, plan) + if !strings.Contains(rendered.SQL, + `AND x1."resource_type" = ANY (SELECT jsonb_array_elements_text($7::jsonb))`) { + t.Fatalf("typed column IN filter missing its canonical form: %s", rendered.SQL) + } + }) +} + +func TestRenderRequiredMatchChainedSteps(t *testing.T) { + plan := trvRequiredPlan(ir.PhysicalSubplan{ + Captures: []string{"root"}, + Operations: []ir.PhysicalOperation{ + trvSubplanTraversal("root", "specimen", "edge_label", "target_type"), + trvSubplanTraversal("specimen", "observation", "edge_label_2", "target_type_2"), + }, + Return: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{BindKey: "required_match_result"}, + }, + }) + plan.BindVars["edge_label_2"] = "has-observation" + plan.BindVars["target_type_2"] = "Observation" + rendered := renderOK(t, plan) + for _, want := range []string{ + `EXISTS (SELECT 1 FROM "pgir02_it_unit"."fhir_edge_occurrence" AS e1`, + `INNER JOIN "pgir02_it_unit"."fhir_resource" AS x1 ON x1."project" = $1`, + `INNER JOIN "pgir02_it_unit"."fhir_edge_occurrence" AS e2 ON e2."project" = $1` + + ` AND e2."dataset_generation" = $2` + + ` AND e2."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND e2."label" = $7 AND e2."source_resource_type" = $6` + + ` AND e2."source_logical_id" = x1."logical_id" AND e2."target_resource_type" = $8`, + `INNER JOIN "pgir02_it_unit"."fhir_resource" AS x2 ON x2."project" = $1`, + `AND x2."resource_type" = $8 AND x2."logical_id" = e2."target_logical_id"`, + `AND e1."source_logical_id" = r0."logical_id"`, + } { + if !strings.Contains(rendered.SQL, want) { + t.Fatalf("chained required-match SQL missing %q: %s", want, rendered.SQL) + } + } + if len(rendered.Args) != 8 { + t.Fatalf("args = %d, want 8 (project, generation, auth, Patient, label1, Specimen, label2, Observation)", len(rendered.Args)) + } +} + +func trvOptionalWithFilter(filter ir.PhysicalFilter) ir.PhysicalPlan { + plan := trvTraversalPlan(ir.PhysicalOutbound, "to_type", "has-specimen", "Specimen") + returnOp := plan.Operations[len(plan.Operations)-1] + plan.Operations = append(plan.Operations[:2], ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &filter}, returnOp) + return plan +} + +func TestMainSequenceRejectsOptionalRowReferences(t *testing.T) { + optional := func(variable string) ir.PhysicalValue { + return ir.PhysicalValue{Variable: variable, Path: []string{"logical_id"}} + } + comparison := func(variable string) ir.PhysicalPredicateExpression { + return ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalComparisonPredicate, + Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", Left: optional(variable), + Right: &ir.PhysicalValue{BindKey: "status_value"}, + }, + } + } + cases := []struct { + name string + filter ir.PhysicalFilter + }{ + {"predicate form over optional row", + ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: optional("target"), + Right: &ir.PhysicalValue{BindKey: "status_value"}, + }}}, + {"comparison leaf over optional row", + ir.PhysicalFilter{Expression: &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalComparisonPredicate, Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", Left: optional("target"), + Right: &ir.PhysicalValue{BindKey: "status_value"}, + }, + }}}, + {"NOT-composed leaf over optional row", + ir.PhysicalFilter{Expression: &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalNotPredicate, + Children: []ir.PhysicalPredicateExpression{comparison("target")}, + }}}, + {"ALL-composed leaf over optional row", + ir.PhysicalFilter{Expression: &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalAllPredicate, + Children: []ir.PhysicalPredicateExpression{comparison("root"), comparison("target")}, + }}}, + {"selector expression over optional row", + *func() *ir.PhysicalFilter { + filter := ir.PhysicalFilter{Expression: trvPayloadMatchPredicate("target", "Specimen", "EQUALS", spec.FilterString, "status")} + return &filter + }()}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + plan := trvOptionalWithFilter(testCase.filter) + plan.BindVars["status_value"] = "final" + trvWantRefusal(t, plan, "FILTER references \"target\"") + }) + } + + plan := trvOptionalWithFilter(ir.PhysicalFilter{Expression: &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalNotPredicate, + Children: []ir.PhysicalPredicateExpression{comparison("root")}, + }}) + plan.BindVars["status_value"] = "final" + rendered := renderOK(t, plan) + if !strings.Contains(rendered.SQL, `NOT (r0."logical_id" = $7)`) { + t.Fatalf("root-row NOT filter degraded: %s", rendered.SQL) + } +} + +func TestMainSequenceRejectsSortOnOptionalRow(t *testing.T) { + plan := trvTraversalPlan(ir.PhysicalOutbound, "to_type", "has-specimen", "Specimen") + returnOp := plan.Operations[len(plan.Operations)-1] + plan.Operations = append(plan.Operations[:2], ir.PhysicalOperation{ + Kind: ir.PhysicalSortOp, + Sort: &ir.PhysicalSort{Keys: []ir.PhysicalValue{{Variable: "target", Path: []string{"logical_id"}}}}, + }, returnOp) + trvWantRefusal(t, plan, "OPTIONAL traversal LEFT JOIN LATERAL") + + plan.Operations[2].Sort = &ir.PhysicalSort{Keys: []ir.PhysicalValue{{Variable: "root", Path: []string{"logical_id"}}}} + rendered := renderOK(t, plan) + if !strings.Contains(rendered.SQL, `ORDER BY r0."logical_id" ASC, r0."resource_seq" ASC, r1."edge_seq" ASC`) { + t.Fatalf("root sort degraded: %s", rendered.SQL) + } +} + +func TestRequiredMatchAcceptedAlongsideOptionalTraversal(t *testing.T) { + plan := trvTraversalPlan(ir.PhysicalOutbound, "to_type", "has-specimen", "Specimen") + subplan := trvRequiredSubplan(trvSubplanTraversal("root", "child", "edge_label", "target_type")) + returnOp := plan.Operations[len(plan.Operations)-1] + plan.Operations = append(plan.Operations[:2], ir.PhysicalOperation{ + Kind: ir.PhysicalFilterOp, + Filter: &ir.PhysicalFilter{Expression: &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalExistsPredicate, Exists: &subplan, + }}, + }, returnOp) + plan.BindVars["required_match_result"] = 1 + rendered := renderOK(t, plan) + if !strings.Contains(rendered.SQL, "EXISTS (SELECT 1 FROM") || + !strings.Contains(rendered.SQL, "LEFT JOIN LATERAL") { + t.Fatalf("combined OPTIONAL+REQUIRED plan missing a shape: %s", rendered.SQL) + } +} + +func trvWantRefusal(t *testing.T, plan ir.PhysicalPlan, wantIn string) { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + wantUnsupported(t, err, "traversal refusal") + if err == nil || !strings.Contains(err.Error(), wantIn) { + t.Fatalf("error = %v, want substring %q", err, wantIn) + } + if rendered.SQL != "" || len(rendered.Args) != 0 { + t.Fatalf("refusal produced SQL or args: %s %#v", rendered.SQL, rendered.Args) + } +} + +func TestTraversalRefusalMatrix(t *testing.T) { + mutate := func(plan ir.PhysicalPlan, edit func(*ir.PhysicalTraversal)) ir.PhysicalPlan { + tr := *plan.Operations[1].Traversal + edit(&tr) + plan.Operations[1] = ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &tr} + return plan + } + badSubplan := func(edit func(*ir.PhysicalSubplan)) ir.PhysicalPlan { + subplan := trvRequiredSubplan(trvSubplanTraversal("root", "child", "edge_label", "target_type")) + edit(&subplan) + return trvRequiredPlan(subplan) + } + badFirstStep := func(edit func(*ir.PhysicalOperation)) ir.PhysicalPlan { + step := trvSubplanTraversal("root", "child", "edge_label", "target_type") + edit(&step) + return trvRequiredPlan(trvRequiredSubplan(step)) + } + + cases := []struct { + name string + plan ir.PhysicalPlan + wantIn string + }{ + {"any direction", mutate(trvTraversalPlan(ir.PhysicalOutbound, "to_type", "l", "Specimen"), + func(tr *ir.PhysicalTraversal) { tr.Direction = ir.PhysicalAny }), + "ANY-direction traversal has no proven PostgreSQL contract"}, + {"outbound with from_type", mutate(trvTraversalPlan(ir.PhysicalOutbound, "to_type", "l", "Specimen"), + func(tr *ir.PhysicalTraversal) { tr.EdgeTargetTypeField = "from_type" }), + `must carry EdgeTargetTypeField "to_type"`}, + {"inbound with to_type", mutate(trvTraversalPlan(ir.PhysicalInbound, "from_type", "l", "Observation"), + func(tr *ir.PhysicalTraversal) { tr.EdgeTargetTypeField = "to_type" }), + `must carry EdgeTargetTypeField "from_type"`}, + {"missing edge variable", mutate(trvTraversalPlan(ir.PhysicalOutbound, "to_type", "l", "Specimen"), + func(tr *ir.PhysicalTraversal) { tr.EdgeVariable = "" }), + "must name its edge variable"}, + {"edge collection not fhir_edge", func() ir.PhysicalPlan { + plan := trvTraversalPlan(ir.PhysicalOutbound, "to_type", "l", "Specimen") + plan.BindVars["edge_collection"] = "fhir_edge_old" + return plan + }(), `must be bound to "fhir_edge"`}, + {"empty label bind", func() ir.PhysicalPlan { + plan := trvTraversalPlan(ir.PhysicalOutbound, "to_type", "l", "Specimen") + plan.BindVars["edge_label"] = " " + return plan + }(), "edge label bind \"edge_label\" must be a non-empty string"}, + {"non-string label bind", func() ir.PhysicalPlan { + plan := trvTraversalPlan(ir.PhysicalOutbound, "to_type", "l", "Specimen") + plan.BindVars["edge_label"] = 42 + return plan + }(), "edge label bind \"edge_label\" must be a non-empty string"}, + {"mismatched source type bind", func() ir.PhysicalPlan { + plan := trvTraversalPlan(ir.PhysicalOutbound, "to_type", "l", "Specimen") + plan.Operations[1].Traversal.SourceTypeBindKey = "source_type" + plan.BindVars["source_type"] = "Observation" + return plan + }(), "traversal source type bind does not match the current row resource type"}, + + {"exists return reads a row", badSubplan(func(sp *ir.PhysicalSubplan) { + sp.Return.Value = &ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}} + }), "canonical scalar VALUE expression"}, + {"exists return array cardinality", badSubplan(func(sp *ir.PhysicalSubplan) { + sp.Return.Cardinality = ir.PhysicalArrayCardinality + }), "SCALAR cardinality with PRESERVE_NULL"}, + {"exists return omit-nulls", badSubplan(func(sp *ir.PhysicalSubplan) { + sp.Return.NullBehavior = ir.PhysicalOmitNulls + }), "SCALAR cardinality with PRESERVE_NULL"}, + {"exists return constant not one", func() ir.PhysicalPlan { + plan := trvRequiredPlan(trvRequiredSubplan(trvSubplanTraversal("root", "child", "edge_label", "target_type"))) + plan.BindVars["required_match_result"] = 2 + return plan + }(), "must be bound to the integer constant 1"}, + {"exists filter before traversal", badSubplan(func(sp *ir.PhysicalSubplan) { + sp.Operations = []ir.PhysicalOperation{{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Expression: &ir.PhysicalPredicateExpression{Kind: ir.PhysicalComparisonPredicate, Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + Right: &ir.PhysicalValue{BindKey: "required_match_result"}, + }}}, + }} + }), "a FILTER may only follow a traversal step"}, + {"exists derived-let scope block", badSubplan(func(sp *ir.PhysicalSubplan) { + sp.Operations = append(sp.Operations, ir.PhysicalOperation{ + Kind: ir.PhysicalDerivedLetOp, + DerivedLet: &ir.PhysicalDerivedLet{Variable: "scope", Operator: "UNIQUE"}, + }) + }), "plan-authored scope operations"}, + {"exists filter predicate form", badSubplan(func(sp *ir.PhysicalSubplan) { + sp.Operations = append(sp.Operations, ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, + Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "child", Path: []string{"logical_id"}}, + Right: &ir.PhysicalValue{BindKey: "required_match_result"}, + }}}) + }), "must use the Expression form"}, + {"nested exists in subplan filter", badSubplan(func(sp *ir.PhysicalSubplan) { + nested := ir.PhysicalSubplan{ + Captures: []string{"child"}, + Operations: []ir.PhysicalOperation{ + trvSubplanTraversal("child", "grand", "edge_label", "target_type"), + }, + Return: ir.PhysicalExpression{Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, Value: &ir.PhysicalValue{BindKey: "required_match_result"}}, + } + sp.Operations = append(sp.Operations, ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, + Filter: &ir.PhysicalFilter{Expression: &ir.PhysicalPredicateExpression{Kind: ir.PhysicalExistsPredicate, Exists: &nested}}}) + }), "not part of the registered traversal predicate surface"}, + {"any predicate", trvRequiredPlan(ir.PhysicalSubplan{ + Captures: []string{"root"}, + Operations: []ir.PhysicalOperation{ + trvSubplanTraversal("root", "child", "edge_label", "target_type"), + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Expression: &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalAnyPredicate, + Children: []ir.PhysicalPredicateExpression{{Kind: ir.PhysicalComparisonPredicate, Comparison: &ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + Right: &ir.PhysicalValue{BindKey: "required_match_result"}, + }}}, + }}}, + }, + Return: ir.PhysicalExpression{Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, Value: &ir.PhysicalValue{BindKey: "required_match_result"}}, + }), "not part of the registered traversal predicate surface"}, + {"subplan traversal discriminator drift", badFirstStep(func(op *ir.PhysicalOperation) { + op.Traversal.EdgeTargetTypeField = "from_type" + }), `must carry EdgeTargetTypeField "to_type"`}, + {"subplan traversal bad edge bind", func() ir.PhysicalPlan { + plan := trvRequiredPlan(trvRequiredSubplan(trvSubplanTraversal("root", "child", "edge_label", "target_type"))) + plan.BindVars["edge_collection"] = "fhir_edge_old" + return plan + }(), `must be bound to "fhir_edge"`}, + {"subplan step 2 not anchored on step 1", func() ir.PhysicalPlan { + plan := trvRequiredPlan(trvRequiredSubplan( + trvSubplanTraversal("root", "specimen", "edge_label", "target_type"), + trvSubplanTraversal("specimen", "observation", "edge_label_2", "target_type_2"), + )) + plan.BindVars["edge_label_2"] = "has-observation" + plan.BindVars["target_type_2"] = "Observation" + plan.Operations[1].Filter.Expression.Exists.Operations[1].Traversal.SourceVariable = "root" + return plan + }(), "must anchor on"}, + {"exists anchored on optional row", func() ir.PhysicalPlan { + plan := trvTraversalPlan(ir.PhysicalOutbound, "to_type", "has-specimen", "Specimen") + sub := trvRequiredSubplan(trvSubplanTraversal("target", "deep", "edge_label", "target_type")) + sub.Captures = []string{"target"} + plan.BindVars["required_match_result"] = 1 + filterOp := ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Expression: &ir.PhysicalPredicateExpression{Kind: ir.PhysicalExistsPredicate, Exists: &sub}, + }} + plan.Operations = append(plan.Operations[:2], append([]ir.PhysicalOperation{filterOp}, plan.Operations[2:]...)...) + return plan + }(), "cannot anchor a correlation"}, + } + + irCases := []struct { + name string + plan ir.PhysicalPlan + wantIn string + }{ + {"unknown direction", mutate(trvTraversalPlan(ir.PhysicalOutbound, "to_type", "l", "Specimen"), + func(tr *ir.PhysicalTraversal) { tr.Direction = ir.PhysicalTraversalDirection("SIDEWAYS") }), + `invalid traversal direction "SIDEWAYS"`}, + {"endpoint lookup strategy", mutate(trvTraversalPlan(ir.PhysicalOutbound, "to_type", "l", "Specimen"), + func(tr *ir.PhysicalTraversal) { tr.Strategy = ir.PhysicalTraversalEndpointLookup }), + "endpoint lookup requires safe endpoint and join fields"}, + {"unknown strategy", mutate(trvTraversalPlan(ir.PhysicalOutbound, "to_type", "l", "Specimen"), + func(tr *ir.PhysicalTraversal) { tr.Strategy = ir.PhysicalTraversalStrategy("HASH_JOIN") }), + `unsupported traversal strategy "HASH_JOIN"`}, + {"missing edge collection bind key", mutate(trvTraversalPlan(ir.PhysicalOutbound, "to_type", "l", "Specimen"), + func(tr *ir.PhysicalTraversal) { tr.EdgeCollectionBindKey = "" }), + "traversal edge collection bind key is required"}, + {"edge collection bind missing", func() ir.PhysicalPlan { + plan := trvTraversalPlan(ir.PhysicalOutbound, "to_type", "l", "Specimen") + delete(plan.BindVars, "edge_collection") + return plan + }(), `bind key "edge_collection" is not defined`}, + {"missing target type bind", func() ir.PhysicalPlan { + plan := trvTraversalPlan(ir.PhysicalOutbound, "to_type", "l", "Specimen") + delete(plan.BindVars, "target_type") + return plan + }(), `bind key "target_type" is not defined`}, + {"source variable unbound", mutate(trvTraversalPlan(ir.PhysicalOutbound, "to_type", "l", "Specimen"), + func(tr *ir.PhysicalTraversal) { tr.SourceVariable = "ghost" }), + `traversal source variable "ghost" is out of scope`}, + {"exists without subplan payload", func() ir.PhysicalPlan { + plan := basePlan() + plan.Operations = []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{ + Expression: &ir.PhysicalPredicateExpression{Kind: ir.PhysicalExistsPredicate}, + }}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}, + } + return plan + }(), "EXISTS predicate requires exactly one subplan"}, + {"exists no captures", badSubplan(func(sp *ir.PhysicalSubplan) { sp.Captures = nil }), + "at least one explicit capture"}, + {"exists two captures", badSubplan(func(sp *ir.PhysicalSubplan) { sp.Captures = []string{"root", "child"} }), + `capture "child" is out of scope`}, + {"exists unknown capture", badSubplan(func(sp *ir.PhysicalSubplan) { sp.Captures = []string{"ghost"} }), + `capture "ghost" is out of scope`}, + {"exists return is not a VALUE expression", trvRequiredPlan(ir.PhysicalSubplan{ + Captures: []string{"root"}, + Operations: []ir.PhysicalOperation{ + trvSubplanTraversal("root", "child", "edge_label", "target_type"), + }, + Return: ir.PhysicalExpression{Kind: ir.PhysicalCallExpression}, + }), "expression has invalid cardinality"}, + {"exists return sort modifier", badSubplan(func(sp *ir.PhysicalSubplan) { + sp.Sort = &ir.PhysicalValue{Variable: "child", Path: []string{"logical_id"}} + }), "projection-only sort or unique modifiers"}, + {"exists return bind undefined", func() ir.PhysicalPlan { + plan := trvRequiredPlan(trvRequiredSubplan(trvSubplanTraversal("root", "child", "edge_label", "target_type"))) + delete(plan.BindVars, "required_match_result") + return plan + }(), `bind key "required_match_result" is not defined`}, + {"exists no operations", badSubplan(func(sp *ir.PhysicalSubplan) { sp.Operations = nil }), + "subplan requires at least one operation"}, + {"exists expression-let", badSubplan(func(sp *ir.PhysicalSubplan) { + sp.Operations = append(sp.Operations, ir.PhysicalOperation{ + Kind: ir.PhysicalExpressionLetOp, + ExpressionLet: &ir.PhysicalExpressionLet{Variable: "x", Expression: ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{BindKey: "required_match_result"}, + }}, + }) + }), `subplan operation 1 has unsupported kind "EXPRESSION_LET"`}, + {"exists sort op", badSubplan(func(sp *ir.PhysicalSubplan) { + sp.Operations = append(sp.Operations, ir.PhysicalOperation{ + Kind: ir.PhysicalSortOp, + Sort: &ir.PhysicalSort{Keys: []ir.PhysicalValue{{Variable: "child", Path: []string{"logical_id"}}}}, + }) + }), "subplan operation 1 cannot be SORT"}, + {"exists missing traversal payload", badFirstStep(func(op *ir.PhysicalOperation) { op.Traversal = nil }), + "operation must contain exactly one payload"}, + {"exists traversal not anchored on capture", badFirstStep(func(op *ir.PhysicalOperation) { + op.Traversal.SourceVariable = "someone-else" + }), `traversal source variable "someone-else" is out of scope`}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { trvWantRefusal(t, testCase.plan, testCase.wantIn) }) + } + for _, testCase := range irCases { + t.Run("ir-gate/"+testCase.name, func(t *testing.T) { trvWantClosedRejection(t, testCase.plan, testCase.wantIn) }) + } +} + +func trvWantClosedRejection(t *testing.T, plan ir.PhysicalPlan, wantIn string) { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err == nil || !strings.Contains(err.Error(), wantIn) { + t.Fatalf("error = %v, want closed rejection %q", err, wantIn) + } + if rendered.SQL != "" || len(rendered.Args) != 0 { + t.Fatalf("rejection produced SQL or args: %s %#v", rendered.SQL, rendered.Args) + } +} + +func trvSelectorProbe(t *testing.T, edit func(*ir.PhysicalExtract, *ir.PhysicalExpression, *ir.PhysicalPredicate)) (string, error) { + t.Helper() + extract := &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "child", Path: []string{"payload"}}, + ResourceType: "Specimen", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: "status"}}}, + } + expression := &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Extract: extract, + } + predicate := ir.PhysicalPredicate{ + Operator: "EQUALS", ValueKind: spec.FilterString, + LeftExpression: expression, + Right: &ir.PhysicalValue{BindKey: "status_value"}, + } + if edit != nil { + edit(extract, expression, &predicate) + } + plan := basePlan() + plan.BindVars["status_value"] = "final" + plan.BindVars["array_value"] = []any{"final", "prelim"} + plan.BindVars["date_value"] = "2024-01-01T00:00:00Z" + plan.BindVars["count_value"] = int64(7) + r := &planRenderer{ + storage: mustBinding(t), + parameters: map[string]int{}, + rows: map[string]rowScope{ + "root": {alias: "r0", resourceType: "Patient"}, + "child": {alias: "x1", resourceType: "Specimen"}, + }, + } + fragment, err := r.renderSelectorMatchFragment(plan, predicate, r.rows) + if err != nil { + return "", err + } + return fragment.sql, nil +} + +func trvSelectorRefusal(t *testing.T, name string, edit func(*ir.PhysicalExtract, *ir.PhysicalExpression, *ir.PhysicalPredicate), wantIn string) { + t.Helper() + t.Run(name, func(t *testing.T) { + sql, err := trvSelectorProbe(t, edit) + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("error = %v (%T), want ErrUnsupportedPostgresPlan", err, err) + } + if err == nil || !strings.Contains(err.Error(), wantIn) { + t.Fatalf("error = %v, want substring %q", err, wantIn) + } + if sql != "" { + t.Fatalf("rejection emitted SQL: %s", sql) + } + }) +} + +func TestRequiredMatchSelectorFilterRefusals(t *testing.T) { + indexZero := 0 + trvSelectorRefusal(t, "wrong expression kind", func(_ *ir.PhysicalExtract, e *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + e.Kind = ir.PhysicalCallExpression + }, "must be an EXTRACT selector expression") + trvSelectorRefusal(t, "missing extract payload", func(x *ir.PhysicalExtract, e *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + e.Extract = nil + _ = x + }, "missing its EXTRACT payload") + trvSelectorRefusal(t, "scalar cardinality", func(_ *ir.PhysicalExtract, e *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + e.Cardinality = ir.PhysicalScalarCardinality + }, "must be the ARRAY-cardinality form") + trvSelectorRefusal(t, "preserve-null behavior", func(_ *ir.PhysicalExtract, e *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + e.NullBehavior = ir.PhysicalPreserveNull + }, "must carry the EMPTY_ON_NULL null behavior") + trvSelectorRefusal(t, "unknown null behavior", func(_ *ir.PhysicalExtract, e *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + e.NullBehavior = ir.PhysicalNullBehavior("SILENT") + }, "must carry the EMPTY_ON_NULL null behavior") + trvSelectorRefusal(t, "conditional array mode", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.ExecutionMode = ir.PhysicalSelectorConditionalArray + }, "CONDITIONAL_ARRAY") + trvSelectorRefusal(t, "unknown execution mode", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.ExecutionMode = ir.PhysicalSelectorExecutionMode("RAW_SQL") + }, "execution mode") + trvSelectorRefusal(t, "distinct", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.Distinct = true + }, "DISTINCT changes element multiplicity") + trvSelectorRefusal(t, "fallbacks", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.Fallbacks = []ir.PhysicalSelectorFallback{{Source: x.Source, Selector: x.Selector}} + }, "fallbacks are a separate capability") + trvSelectorRefusal(t, "prepared reference", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.Prepared = &ir.PhysicalPreparedReference{SetVariable: "s", Field: "f"} + }, "prepared-set references") + trvSelectorRefusal(t, "unit normalization", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.UnitNormalization = &ir.PhysicalUnitNormalization{} + }, "unit normalization is not implemented") + trvSelectorRefusal(t, "selector CONTAINS filter", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.Selector.Filter = &spec.ContainsFilter{} + }, "CONTAINS filters are not implemented") + trvSelectorRefusal(t, "empty selector", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.Selector = spec.Selector{} + }, "selector is empty") + trvSelectorRefusal(t, "step without property", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.Selector = spec.Selector{Steps: []spec.SelectorStep{{Field: ""}}} + }, "has no property") + trvSelectorRefusal(t, "iterate step", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.Selector = spec.Selector{Steps: []spec.SelectorStep{{Field: "status", Iterate: true}}} + }, "repeated ([]) step") + trvSelectorRefusal(t, "indexed step", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.Selector = spec.Selector{Steps: []spec.SelectorStep{{Field: "status", Index: &indexZero}}} + }, "indexed step") + trvSelectorRefusal(t, "unsafe field", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.Selector = spec.Selector{Steps: []spec.SelectorStep{{Field: "bad field"}}} + }, "not renderer-safe") + trvSelectorRefusal(t, "source bind", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.Source = ir.PhysicalValue{BindKey: "status_value"} + }, "source must be a row document, not a bind value") + trvSelectorRefusal(t, "source path not payload", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.Source = ir.PhysicalValue{Variable: "child", Path: []string{"payload", "status"}} + }, `source must be the row document at ["payload"]`) + trvSelectorRefusal(t, "unknown variable", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.Source = ir.PhysicalValue{Variable: "ghost", Path: []string{"payload"}} + }, "does not bind a readable subplan row") + trvSelectorRefusal(t, "missing resource type", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.ResourceType = " " + }, "must name the schema resource type") + trvSelectorRefusal(t, "resource type mismatch", func(x *ir.PhysicalExtract, _ *ir.PhysicalExpression, _ *ir.PhysicalPredicate) { + x.ResourceType = "Observation" + }, "but the row is joined as") + + trvSelectorRefusal(t, "contains_text gap", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Operator = "CONTAINS_TEXT" + }, "CONTAINS_TEXT") + trvSelectorRefusal(t, "unknown operator", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Operator = "REGEX" + }, "not part of the v1 PostgreSQL surface") + trvSelectorRefusal(t, "LTE against STRING is not spec-orderable", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Operator = "LTE" + }, "not compatible with value kind") + trvSelectorRefusal(t, "invalid value kind", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.ValueKind = spec.FilterValueKind("JSON") + }, "value kind") + trvSelectorRefusal(t, "unknown quantifier", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Quantifier = spec.ArrayQuantifier("MANY") + }, "array quantifier") + trvSelectorRefusal(t, "scalar bind for IN", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Operator = "IN" + }, "requires a []any or []string bind value") + trvSelectorRefusal(t, "array bind for EQUALS", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Right = &ir.PhysicalValue{BindKey: "array_value"} + }, "requires a string bind value") + trvSelectorRefusal(t, "missing right value", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Right = nil + }, "requires a bind-backed right value") + trvSelectorRefusal(t, "undefined right bind", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Right = &ir.PhysicalValue{BindKey: "not-defined"} + }, "is not defined") + trvSelectorRefusal(t, "exists with right value", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Operator = "EXISTS" + }, "EXISTS must not carry a right value") + trvSelectorRefusal(t, "correlated left expression", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Correlation = &ir.PhysicalCorrelation{ResourceType: "Specimen"} + }, "correlated predicates") +} + +func TestRequiredMatchSelectorFilterRegisteredForms(t *testing.T) { + const valuesPin = `(CASE WHEN (x1."payload" #> $1::text[]) IS NULL OR jsonb_typeof((x1."payload" #> $1::text[])) = 'null' THEN '[]'::jsonb ELSE jsonb_build_array((x1."payload" #> $1::text[])) END)` + cases := []struct { + name string + edit func(*ir.PhysicalExtract, *ir.PhysicalExpression, *ir.PhysicalPredicate) + want string + }{ + {"any default quantifier", nil, "(EXISTS (SELECT 1 FROM jsonb_array_elements(" + valuesPin + ") AS __sv(v) WHERE (__sv.v) = ($2::jsonb)))"}, + {"explicit ANY", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Quantifier = spec.QuantifierAny + }, "(EXISTS (SELECT 1 FROM jsonb_array_elements(" + valuesPin + ") AS __sv(v) WHERE (__sv.v) = ($2::jsonb)))"}, + {"NONE quantifier", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Quantifier = spec.QuantifierNone + }, "(NOT EXISTS (SELECT 1 FROM jsonb_array_elements(" + valuesPin + ") AS __sv(v) WHERE (__sv.v) = ($2::jsonb)))"}, + {"ALL quantifier", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Quantifier = spec.QuantifierAll + }, "(jsonb_array_length(" + valuesPin + ") > 0 AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(" + valuesPin + ") AS __sv(v) WHERE NOT ((__sv.v) = ($2::jsonb))))"}, + {"NOT_EQUALS", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Operator = "NOT_EQUALS" + }, "(__sv.v) <> ($2::jsonb)"}, + {"IN membership", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Operator = "IN" + p.Right = &ir.PhysicalValue{BindKey: "array_value"} + }, "EXISTS (SELECT 1 FROM jsonb_array_elements($2::jsonb) AS loom_selector_item WHERE loom_selector_item = (__sv.v))"}, + {"LTE ordering (INTEGER, numeric guard)", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Operator = "LTE" + p.ValueKind = spec.FilterInteger + p.Right = &ir.PhysicalValue{BindKey: "count_value"} + }, "COALESCE(jsonb_typeof(__sv.v) = 'number' AND (__sv.v) <= ($2::jsonb), false)"}, + {"DATE equality", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.ValueKind = spec.FilterDate + p.Right = &ir.PhysicalValue{BindKey: "date_value"} + }, "(__sv.v) = ($2::jsonb)"}, + {"DATE_TIME ordering takes the temporal form", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Operator = "LTE" + p.ValueKind = spec.FilterDateTime + p.Right = &ir.PhysicalValue{BindKey: "date_value"} + }, ") <= $2, false)"}, + {"EXISTS emptiness", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Operator = "EXISTS" + p.Right = nil + }, "(jsonb_array_length(" + valuesPin + ") > 0)"}, + {"MISSING emptiness", func(_ *ir.PhysicalExtract, _ *ir.PhysicalExpression, p *ir.PhysicalPredicate) { + p.Operator = "MISSING" + p.Right = nil + }, "(jsonb_array_length(" + valuesPin + ") = 0)"}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + sql, err := trvSelectorProbe(t, testCase.edit) + if err != nil { + t.Fatalf("registered form refused: %v", err) + } + if !strings.Contains(sql, testCase.want) { + t.Fatalf("fragment missing %q:\n%s", testCase.want, sql) + } + }) + } +} + +func TestTraversalRenderingIsDeterministicAndPlanDataInert(t *testing.T) { + plan := trvTraversalPlan(ir.PhysicalOutbound, "to_type", `x'; DROP TABLE fhir_resource; --`, `"injected";$3`) + first := renderOK(t, plan) + second := renderOK(t, plan) + if first.SQL != second.SQL || !reflect.DeepEqual(first.Args, second.Args) { + t.Fatalf("renders drifted:\n%s\n%s", first.SQL, second.SQL) + } + for _, hostile := range []string{"DROP TABLE", `injected`, "\"injected\""} { + if strings.Contains(first.SQL, hostile) { + t.Fatalf("hostile plan data %q reached SQL text: %s", hostile, first.SQL) + } + } + foundLabel := false + for _, parameter := range first.Args { + if text, ok := parameter.Value.(string); ok && strings.Contains(text, "DROP TABLE") { + foundLabel = true + } + } + if !foundLabel { + t.Fatal("hostile label never reached the parameter list; it was dropped, not bound") + } +} + +func trvRunDeltaAudit(m *testing.M) int { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return m.Run() + } + beforeList, err := trvSnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-06 delta audit: snapshot before run: %v\n", err) + return 1 + } + before := make(map[string]bool, len(beforeList)) + for _, name := range beforeList { + before[name] = true + } + code := m.Run() + after, err := trvSnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-06 delta audit: snapshot after run: %v\n", err) + code = 1 + } + trvFixturesMu.Lock() + defer trvFixturesMu.Unlock() + for _, name := range after { + if before[name] { + continue + } + if trvTrackedFixtures[name] { + fmt.Fprintf(os.Stderr, "pg-sql-06 delta audit: %q was created and tracked by this run and still survives\n", name) + code = 1 + continue + } + fmt.Fprintf(os.Stderr, "pg-sql-06 delta audit: note — %q was born during this run but is not owned by it (sibling lane?)\n", name) + } + return code +} + +var ( + trvFixturesMu sync.Mutex + trvTrackedFixtures = map[string]bool{} +) + +func trvTrackFixtures(t *testing.T, svc *pgir02Service) { + t.Helper() + var database string + if err := svc.iso.QueryRow(svc.ctx, "SELECT current_database()").Scan(&database); err != nil { + svc.fail("track owned database", err) + } + trvFixturesMu.Lock() + defer trvFixturesMu.Unlock() + trvTrackedFixtures[database] = true + trvTrackedFixtures[svc.schema] = true +} + +func trvSnapshotOwnedResources(ctx context.Context, raw string) ([]string, error) { + pool, err := pgxpool.New(ctx, raw) + if err != nil { + return nil, err + } + defer pool.Close() + queryCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + var names []string + for _, query := range []string{ + `SELECT datname FROM pg_database WHERE datname LIKE 'pgir02\_it\_%' ESCAPE '\'`, + `SELECT nspname FROM pg_namespace WHERE nspname LIKE 'pgir02\_it\_%' ESCAPE '\'`, + } { + rows, err := pool.Query(queryCtx, query) + if err != nil { + return nil, err + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + return nil, err + } + names = append(names, name) + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, err + } + rows.Close() + } + return names, nil +} + +func trvLiveTraversalPlan(rootType string, direction ir.PhysicalTraversalDirection, edgeTargetTypeField, label, targetType string, paths []string) ir.PhysicalPlan { + plan := scopedPlan(itProject, itGeneration, paths, false) + plan.BindVars["root_collection"] = rootType + plan.BindVars["edge_collection"] = "fhir_edge" + plan.BindVars["edge_label"] = label + plan.BindVars["target_type"] = targetType + plan.Operations = append(plan.Operations, ir.PhysicalOperation{ + Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "target", EdgeVariable: "edge", + Direction: direction, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: "edge_label", TargetTypeBindKey: "target_type", + EdgeTargetTypeField: edgeTargetTypeField, Strategy: ir.PhysicalTraversalNative, + }, + }) + return withReturn(plan, + ir.PhysicalProjection{Name: "parent_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + ir.PhysicalProjection{Name: "child_id", Value: ir.PhysicalValue{Variable: "target", Path: []string{"logical_id"}}}, + ) +} + +func TestPG18OptionalTraversalPreservesParentRows(t *testing.T) { + svc := newPGIR02Service(t, 0) + trvTrackFixtures(t, svc) + for _, id := range []string{"pa", "pb", "pc", "pd", "pe"} { + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", id, "FHIR/Patient", nil) + } + svc.insertResource(t, svc.schema, itProject, itGeneration, "Specimen", "st-1", "FHIR/Specimen", nil) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Specimen", "st-2", "FHIR/Specimen", nil) + edge := func(label, source, target, path string) { + svc.insertEdge(t, pgstoreEdgeRow(itProject, itGeneration, label, source, target, path)) + } + edge("has-specimen", "pa", "st-1", "specimen[0]") + edge("has-specimen", "pa", "st-1", "specimen[1]") + edge("has-specimen", "pa", "st-2", "contained[0].specimen") + edge("has-specimen", "pa", "st-ghost", "specimen[9]") + edge("has-specimen", "pb", "st-2", "specimen[0]") + edge("has-specimen", "pc", "st-ghost", "specimen[0]") + edge("other-label", "pe", "st-1", "specimen[0]") + + plan := trvLiveTraversalPlan("Patient", ir.PhysicalOutbound, "to_type", "has-specimen", "Specimen", + []string{"FHIR/Patient", "FHIR/Specimen"}) + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + got := svc.runRendered(t, rendered) + want := [][]string{ + {"pa", "st-1"}, {"pa", "st-1"}, {"pa", "st-2"}, + {"pb", "st-2"}, + {"pc", ""}, + {"pd", ""}, + {"pe", ""}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("OPTIONAL traversal rows = %v, want parent-preserving matrix %v", got, want) + } + + stored := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_edge_occurrence" WHERE project=$1 AND dataset_generation=$2 AND label='has-specimen'`, + itProject, itGeneration) + if len(stored) != 1 || stored[0][0] != "6" { + t.Fatalf("witness must show all six stored occurrences (two ghosts included): %v", stored) + } + nullParents := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" r0`+ + ` WHERE r0.project=$1 AND r0.dataset_generation=$2 AND r0.resource_type='Patient' AND r0.auth_resource_path='FHIR/Patient'`+ + ` AND NOT EXISTS (SELECT 1 FROM "`+svc.schema+`"."fhir_edge_occurrence" e1`+ + ` INNER JOIN "`+svc.schema+`"."fhir_resource" x1 ON x1.project=e1.project AND x1.dataset_generation=e1.dataset_generation`+ + ` AND x1.auth_resource_path='FHIR/Specimen' AND x1.resource_type='Specimen' AND x1.logical_id=e1.target_logical_id`+ + ` WHERE e1.project=$1 AND e1.dataset_generation=$2 AND e1.auth_resource_path='FHIR/Patient'`+ + ` AND e1.label='has-specimen' AND e1.source_resource_type='Patient' AND e1.source_logical_id=r0.logical_id`+ + ` AND e1.target_resource_type='Specimen')`, + itProject, itGeneration) + if len(nullParents) != 1 || nullParents[0][0] != "3" { + t.Fatalf("witness parent count with zero surviving occurrences = %v, want 3 (pc, pd, pe)", nullParents) + } +} + +func trvLiveRequiredPlan(rootType, label, targetType string, paths []string, direction ir.PhysicalTraversalDirection, edgeTargetTypeField string, subplanFilter *ir.PhysicalPredicateExpression) ir.PhysicalPlan { + plan := scopedPlan(itProject, itGeneration, paths, false) + plan.BindVars["root_collection"] = rootType + plan.BindVars["edge_collection"] = "fhir_edge" + plan.BindVars["edge_label"] = label + plan.BindVars["target_type"] = targetType + plan.BindVars["required_match_result"] = 1 + step := trvSubplanTraversal("root", "child", "edge_label", "target_type") + step.Traversal.Direction = direction + step.Traversal.EdgeTargetTypeField = edgeTargetTypeField + operations := []ir.PhysicalOperation{step} + if subplanFilter != nil { + operations = append(operations, ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, + Filter: &ir.PhysicalFilter{Expression: subplanFilter}}) + } + subplan := trvRequiredSubplan(operations...) + plan.Operations = append(plan.Operations, ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, + Filter: &ir.PhysicalFilter{Expression: &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalExistsPredicate, Exists: &subplan, + }}}) + return withReturn(plan, + ir.PhysicalProjection{Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}) +} + +func TestPG18RequiredMatchExcludesUnmatchedParents(t *testing.T) { + svc := newPGIR02Service(t, 0) + trvTrackFixtures(t, svc) + for _, id := range []string{"p-a", "p-b", "p-c", "p-d"} { + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", id, "FHIR/Patient", nil) + } + svc.insertResource(t, svc.schema, itProject, itGeneration, "Specimen", "st-1", "FHIR/Specimen", nil) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Specimen", "st-9", "FHIR/Specimen", nil) + svc.insertEdge(t, pgstoreEdgeRow(itProject, itGeneration, "has-specimen", "p-a", "st-1", "specimen[0]")) + svc.insertEdge(t, pgstoreEdgeRow(itProject, itGeneration, "wrong-label", "p-c", "st-9", "specimen[0]")) + svc.insertEdge(t, pgstoreEdgeRow(itProject, itGeneration, "has-specimen", "p-d", "st-ghost", "specimen[0]")) + + plan := trvLiveRequiredPlan("Patient", "has-specimen", "Specimen", + []string{"FHIR/Patient", "FHIR/Specimen"}, ir.PhysicalOutbound, "to_type", nil) + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + got := svc.runRendered(t, rendered) + if want := [][]string{{"p-a"}}; !reflect.DeepEqual(got, want) { + t.Fatalf("REQUIRED rows = %v, want %v", got, want) + } + + stored := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_edge_occurrence" WHERE project=$1 AND dataset_generation=$2 AND label='wrong-label' AND source_logical_id='p-c'`, + itProject, itGeneration) + if len(stored) != 1 || stored[0][0] != "1" { + t.Fatalf("wrong-label witness occurrence must be stored: %v", stored) + } + ghost := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_edge_occurrence" WHERE project=$1 AND dataset_generation=$2 AND label='has-specimen' AND source_logical_id='p-d' AND target_logical_id='st-ghost'`, + itProject, itGeneration) + if len(ghost) != 1 || ghost[0][0] != "1" { + t.Fatalf("dangling witness occurrence must be stored: %v", ghost) + } + + optionalPlan := trvLiveTraversalPlan("Patient", ir.PhysicalOutbound, "to_type", "has-specimen", "Specimen", + []string{"FHIR/Patient", "FHIR/Specimen"}) + subplan := trvRequiredSubplan(trvSubplanTraversal("root", "child", "edge_label", "target_type")) + returnOp := optionalPlan.Operations[len(optionalPlan.Operations)-1] + optionalPlan.BindVars["required_match_result"] = 1 + optionalPlan.Operations = append(optionalPlan.Operations[:2], ir.PhysicalOperation{ + Kind: ir.PhysicalFilterOp, + Filter: &ir.PhysicalFilter{Expression: &ir.PhysicalPredicateExpression{ + Kind: ir.PhysicalExistsPredicate, Exists: &subplan, + }}, + }, returnOp) + combined, err := RenderPhysicalPlan(optionalPlan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("combined RenderPhysicalPlan: %v", err) + } + got = svc.runRendered(t, combined) + if want := [][]string{{"p-a", "st-1"}}; !reflect.DeepEqual(got, want) { + t.Fatalf("combined OPTIONAL+REQUIRED rows = %v, want %v", got, want) + } +} + +func TestPG18RequiredMatchWithTypedPayloadFilter(t *testing.T) { + svc := newPGIR02Service(t, 0) + trvTrackFixtures(t, svc) + for _, id := range []string{"p-a", "p-b", "p-c", "p-d", "p-e"} { + svc.insertResource(t, svc.schema, itProject, itGeneration, "Patient", id, "FHIR/Patient", nil) + } + svc.insertResource(t, svc.schema, itProject, itGeneration, "Specimen", "st-f", "FHIR/Specimen", map[string]any{"status": "final"}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Specimen", "st-p", "FHIR/Specimen", map[string]any{"status": "prelim"}) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Specimen", "st-m", "FHIR/Specimen", nil) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Specimen", "st-a", "FHIR/Specimen", map[string]any{"status": []any{"final"}}) + svc.insertEdge(t, pgstoreEdgeRow(itProject, itGeneration, "has-specimen", "p-a", "st-f", "specimen[0]")) + svc.insertEdge(t, pgstoreEdgeRow(itProject, itGeneration, "has-specimen", "p-b", "st-p", "specimen[0]")) + svc.insertEdge(t, pgstoreEdgeRow(itProject, itGeneration, "has-specimen", "p-c", "st-m", "specimen[0]")) + svc.insertEdge(t, pgstoreEdgeRow(itProject, itGeneration, "has-specimen", "p-d", "st-f", "specimen[0]")) + svc.insertEdge(t, pgstoreEdgeRow(itProject, itGeneration, "has-specimen", "p-d", "st-p", "specimen[1]")) + svc.insertEdge(t, pgstoreEdgeRow(itProject, itGeneration, "has-specimen", "p-e", "st-a", "specimen[0]")) + + payloadFilter := trvPayloadMatchPredicate("child", "Specimen", "EQUALS", spec.FilterString, "status") + plan := trvLiveRequiredPlan("Patient", "has-specimen", "Specimen", + []string{"FHIR/Patient", "FHIR/Specimen"}, ir.PhysicalOutbound, "to_type", payloadFilter) + plan.BindVars["status_value"] = "final" + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + got := svc.runRendered(t, rendered) + if want := [][]string{{"p-a"}, {"p-d"}}; !reflect.DeepEqual(got, want) { + t.Fatalf("typed payload REQUIRED rows = %v, want %v", got, want) + } + + witness := svc.witness(t, + `SELECT r0.logical_id FROM "`+svc.schema+`"."fhir_resource" r0`+ + ` WHERE r0.project=$1 AND r0.dataset_generation=$2 AND r0.resource_type='Patient' AND r0.auth_resource_path='FHIR/Patient'`+ + ` AND EXISTS (SELECT 1 FROM "`+svc.schema+`"."fhir_edge_occurrence" e1`+ + ` INNER JOIN "`+svc.schema+`"."fhir_resource" x1 ON x1.project=e1.project AND x1.dataset_generation=e1.dataset_generation`+ + ` AND x1.auth_resource_path='FHIR/Specimen' AND x1.resource_type='Specimen' AND x1.logical_id=e1.target_logical_id`+ + ` WHERE e1.project=$1 AND e1.dataset_generation=$2 AND e1.auth_resource_path='FHIR/Patient'`+ + ` AND e1.label='has-specimen' AND e1.source_resource_type='Patient' AND e1.source_logical_id=r0.logical_id`+ + ` AND e1.target_resource_type='Specimen' AND x1.payload->>'status'='final')`+ + ` ORDER BY r0.resource_seq`, + itProject, itGeneration) + if !reflect.DeepEqual(witness, [][]string{{"p-a"}, {"p-d"}}) { + t.Fatalf("typed payload witness = %v", witness) + } +} + +func TestPG18InboundTraversalOptionalAndRequired(t *testing.T) { + svc := newPGIR02Service(t, 0) + trvTrackFixtures(t, svc) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Specimen", "st-1", "FHIR/Specimen", nil) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Specimen", "st-2", "FHIR/Specimen", nil) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Specimen", "st-3", "FHIR/Specimen", nil) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Specimen", "st-4", "FHIR/Specimen", nil) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Observation", "o-1", "FHIR/Observation", nil) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Observation", "o-2", "FHIR/Observation", nil) + svc.insertResource(t, svc.schema, itProject, itGeneration, "Observation", "o-3", "FHIR/Observation", nil) + inbound := func(label, observation, specimen, path string) { + svc.insertEdge(t, control.EdgeOccurrenceRow{ + Project: itProject, DatasetGeneration: itGeneration, AuthResourcePath: "FHIR/Observation", + Label: label, SourceResourceType: "Observation", SourceLogicalID: observation, + TargetResourceType: "Specimen", TargetLogicalID: specimen, SourcePath: path, SourceOrdinal: 0, + }) + } + inbound("focus", "o-1", "st-1", "focus[0]") + inbound("focus", "o-2", "st-1", "focus[0]") + inbound("focus", "o-3", "st-2", "focus[0]") + trvInsertDanglingSourceEdge(t, svc, "focus", "Observation", "o-ghost", "Specimen", "st-4", "FHIR/Observation", "focus[0]") + inbound("other", "o-1", "st-3", "focus[1]") + + paths := []string{"FHIR/Specimen", "FHIR/Observation"} + optionalPlan := trvLiveTraversalPlan("Specimen", ir.PhysicalInbound, "from_type", "focus", "Observation", paths) + rendered, err := RenderPhysicalPlan(optionalPlan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("inbound OPTIONAL render: %v", err) + } + got := svc.runRendered(t, rendered) + want := [][]string{ + {"st-1", "o-1"}, {"st-1", "o-2"}, + {"st-2", "o-3"}, + {"st-3", ""}, + {"st-4", ""}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("inbound OPTIONAL rows = %v, want %v", got, want) + } + + requiredPlan := trvLiveRequiredPlan("Specimen", "focus", "Observation", paths, ir.PhysicalInbound, "from_type", nil) + rendered, err = RenderPhysicalPlan(requiredPlan, bindingFor(t, svc.schema)) + if err != nil { + t.Fatalf("inbound REQUIRED render: %v", err) + } + got = svc.runRendered(t, rendered) + if want := [][]string{{"st-1"}, {"st-2"}}; !reflect.DeepEqual(got, want) { + t.Fatalf("inbound REQUIRED rows = %v, want %v", got, want) + } + + stored := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_edge_occurrence" WHERE project=$1 AND dataset_generation=$2 AND label='focus' AND target_logical_id='st-4'`, + itProject, itGeneration) + if len(stored) != 1 || stored[0][0] != "1" { + t.Fatalf("dangling inbound witness occurrence must be stored: %v", stored) + } +} diff --git a/internal/dataframe/compiler/render/postgres/unit_normalization.go b/internal/dataframe/compiler/render/postgres/unit_normalization.go new file mode 100644 index 000000000..cc252c60b --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/unit_normalization.go @@ -0,0 +1,320 @@ +package postgres + +import ( + "encoding/json" + "fmt" + "math" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/calypr/loom/internal/dataframe/unit" + + fhirschema "github.com/calypr/loom/internal/fhir/schema" +) + +const ( + unitGuardIdentityUnknown = "UNIT_IDENTITY_UNKNOWN" + unitGuardRuleAmbiguous = "UNIT_RULE_AMBIGUOUS" +) + +const unitAddPassUpper = "179769313486231251480077898176534491778436481631674425806715807129950620372469748806686859598516352578265980131802208169442826256090225147178389403728551798718562185003524030875337817916681086138040437410377184717767504109875384015236135739992861126057954874517079522029393077059952637103891167171142536396800" + +type unitProjectionFacts struct { + sourceVariable string + resourceType string + valueSteps []documentPathStep + systemSteps []documentPathStep + codeSteps []documentPathStep + rules []unit.UnitConversionRule + target unit.UnitIdentity + dimension unit.UnitDimension +} + +func classifyUnitNormalizedProjection(projection ir.PhysicalProjection) (classifiedExtractionProjection, error) { + expression := projection.Expression + extract := expression.Extract + if expression.Cardinality != ir.PhysicalArrayCardinality || expression.NullBehavior != ir.PhysicalEmptyOnNull { + return classifiedExtractionProjection{}, fmt.Errorf( + "%w: projection %q: unit-normalized extraction is registered for the exact ARRAY-cardinality EMPTY_ON_NULL posture lower/root_projection.go:334 emits; got cardinality %q, null behavior %q", + ErrUnsupportedPostgresPlan, projection.Name, string(expression.Cardinality), string(expression.NullBehavior)) + } + if projection.Presence != nil { + return classifiedExtractionProjection{}, fmt.Errorf( + "%w: projection %q carries presence metadata on a unit-normalized expression; presence proofs pair with selector extractions only", + ErrUnsupportedPostgresPlan, projection.Name) + } + if len(extract.Fallbacks) != 0 { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: ordered fallback selectors are a separate capability", ErrUnsupportedPostgresPlan, projection.Name) + } + if extract.Prepared != nil { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: prepared-set selector references are not implemented for PostgreSQL", ErrUnsupportedPostgresPlan, projection.Name) + } + if extract.Distinct { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: DISTINCT unit-normalized projection has no producer (the lowering never sets it) and no registered SORTED_UNIQUE reading", ErrUnsupportedPostgresPlan, projection.Name) + } + if extract.Selector.Filter != nil { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: selector CONTAINS filters are not implemented for PostgreSQL", ErrUnsupportedPostgresPlan, projection.Name) + } + switch extract.ExecutionMode { + case "", ir.PhysicalSelectorGeneric, ir.PhysicalSelectorDirectScalar: + case ir.PhysicalSelectorConditionalArray: + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: CONDITIONAL_ARRAY selector mode requires repeated-step rendering, which is not registered for PostgreSQL", ErrUnsupportedPostgresPlan, projection.Name) + default: + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: unknown selector execution mode %q", ErrUnsupportedPostgresPlan, projection.Name, string(extract.ExecutionMode)) + } + if extract.Source.BindKey != "" { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: the selector source must be a row document, not a bind value", ErrUnsupportedPostgresPlan, projection.Name) + } + if len(extract.Source.Path) != 1 || extract.Source.Path[0] != "payload" { + return classifiedExtractionProjection{}, fmt.Errorf(`%w: projection %q: the selector source must be the row document at ["payload"], the exact source shape lowering emits (lower/root_projection.go); got %v`, + ErrUnsupportedPostgresPlan, projection.Name, extract.Source.Path) + } + if strings.TrimSpace(extract.ResourceType) == "" { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: unit-normalized extraction must name the schema resource type it was validated against", ErrUnsupportedPostgresPlan, projection.Name) + } + + normalization := extract.UnitNormalization + if !normalization.Target.Valid() || len(normalization.Rules) == 0 { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: unit normalization target and approved rules are required", ErrUnsupportedPostgresPlan, projection.Name) + } + if normalization.OriginalValue.CanonicalPath() != extract.Selector.CanonicalPath() { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: unit normalization original value must match extract selector", ErrUnsupportedPostgresPlan, projection.Name) + } + valuePath := normalization.OriginalValue.CanonicalPath() + if !strings.HasSuffix(valuePath, ".value") { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: unit normalization original value must select a FHIR Quantity value", ErrUnsupportedPostgresPlan, projection.Name) + } + quantityPath := strings.TrimSuffix(valuePath, ".value") + if unitSelectorIterates(normalization.OriginalValue) || unitSelectorIterates(normalization.SourceSystem) || unitSelectorIterates(normalization.SourceCode) { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: unit normalization requires one scalar or indexed Quantity", ErrUnsupportedPostgresPlan, projection.Name) + } + if normalization.SourceSystem.CanonicalPath() != quantityPath+".system" || normalization.SourceCode.CanonicalPath() != quantityPath+".code" { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: unit normalization system and code selectors must be siblings of the original Quantity value", ErrUnsupportedPostgresPlan, projection.Name) + } + quantity, ok := fhirschema.ResolveFieldSemantics(extract.ResourceType, quantityPath) + if !ok || quantity.Kind != fhirschema.FieldKindObject || quantity.Reference != "Quantity" { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: unit normalization original value must be owned by a FHIR Quantity", ErrUnsupportedPostgresPlan, projection.Name) + } + for index, rule := range normalization.Rules { + if err := rule.Validate(normalization.Target, normalization.Dimension); err != nil { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: unit normalization rule %d: %w", ErrUnsupportedPostgresPlan, projection.Name, index, err) + } + } + references := make([]unit.UnitRuleReference, 0, len(normalization.Rules)) + for _, rule := range normalization.Rules { + references = append(references, unit.UnitRuleReference{ID: rule.ID, Version: rule.Version}) + } + resolvedDimension, resolvedRules, err := unit.ResolveApprovedUnitRules(references, normalization.Target) + if err != nil { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: unit normalization rules: %w", ErrUnsupportedPostgresPlan, projection.Name, err) + } + if !resolvedDimension.Equal(normalization.Dimension) { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: unit normalization dimension %v does not match the pinned registry dimension %v for target %s/%s: %w", + ErrUnsupportedPostgresPlan, projection.Name, normalization.Dimension, resolvedDimension, normalization.Target.System, normalization.Target.Code, unit.ErrUnitDimensionIncompatible) + } + if len(resolvedRules) != len(normalization.Rules) { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: unit normalization rule count %d does not match the resolved registry count %d", + ErrUnsupportedPostgresPlan, projection.Name, len(normalization.Rules), len(resolvedRules)) + } + for index := range normalization.Rules { + if normalization.Rules[index] != resolvedRules[index] { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: unit normalization rule %d (%q@%s) does not match the pinned registry entry (tampered coefficient, kind, identity, or dimension)", + ErrUnsupportedPostgresPlan, projection.Name, index, normalization.Rules[index].ID, normalization.Rules[index].Version) + } + } + if err := unitRulesPairAmbiguity(normalization.Rules); err != nil { + return classifiedExtractionProjection{}, fmt.Errorf("%w: projection %q: %w", ErrUnsupportedPostgresPlan, projection.Name, err) + } + + valueSteps, err := unitProjectionSelectorSteps(projection.Name, "original value", normalization.OriginalValue) + if err != nil { + return classifiedExtractionProjection{}, err + } + systemSteps, err := unitProjectionSelectorSteps(projection.Name, "system", normalization.SourceSystem) + if err != nil { + return classifiedExtractionProjection{}, err + } + codeSteps, err := unitProjectionSelectorSteps(projection.Name, "code", normalization.SourceCode) + if err != nil { + return classifiedExtractionProjection{}, err + } + return classifiedExtractionProjection{ + kind: extractionUnitNormalized, + projection: projection, + unit: &unitProjectionFacts{ + sourceVariable: extract.Source.Variable, + resourceType: extract.ResourceType, + valueSteps: valueSteps, + systemSteps: systemSteps, + codeSteps: codeSteps, + rules: append([]unit.UnitConversionRule(nil), normalization.Rules...), + target: normalization.Target, + dimension: normalization.Dimension, + }, + }, nil +} + +func unitSelectorIterates(selector spec.Selector) bool { + for _, step := range selector.Steps { + if step.Iterate { + return true + } + } + return false +} + +func unitProjectionSelectorSteps(projectionName, role string, selector spec.Selector) ([]documentPathStep, error) { + if len(selector.Steps) == 0 { + return nil, fmt.Errorf("%w: projection %q: the unit %s selector is empty", ErrUnsupportedPostgresPlan, projectionName, role) + } + steps := make([]documentPathStep, 0, len(selector.Steps)) + for index, step := range selector.Steps { + switch { + case step.Field == "": + return nil, fmt.Errorf("%w: projection %q: unit %s selector step %d has no property", ErrUnsupportedPostgresPlan, projectionName, role, index) + case step.Iterate: + return nil, fmt.Errorf("%w: projection %q: unit %s selector repeated ([]) step %d requires array rendering, which is not registered (the enabling capability is pg-sql-16)", ErrUnsupportedPostgresPlan, projectionName, role, index) + } + if !payloadPathSegmentPattern.MatchString(step.Field) { + return nil, fmt.Errorf("%w: projection %q: unit %s selector step %q is not renderer-safe", ErrUnsupportedPostgresPlan, projectionName, role, step.Field) + } + projectionStep := documentPathStep{field: step.Field} + if step.Index != nil { + projectionStep.indexed = true + projectionStep.index = *step.Index + } + steps = append(steps, projectionStep) + } + return steps, nil +} + +func unitRulesPairAmbiguity(rules []unit.UnitConversionRule) error { + seen := make(map[string]struct{}, len(rules)) + for _, rule := range rules { + key := rule.Source.System + "\x00" + rule.Source.Code + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("unit rule set carries two rules for source %s/%s; the registered identity match admits at most one rule per exact source pair (ambiguity must never resolve by bind-array order)", + rule.Source.System, rule.Source.Code) + } + seen[key] = struct{}{} + } + return nil +} + +type unitRulesBinding struct { + ID string `json:"id"` + Version string `json:"version"` + Kind string `json:"kind"` + SourceSystem string `json:"source_system"` + SourceCode string `json:"source_code"` + TargetSystem string `json:"target_system"` + TargetCode string `json:"target_code"` + Scale float64 `json:"scale"` + Offset float64 `json:"offset"` +} + +func unitRulesBindingJSON(rules []unit.UnitConversionRule) (json.RawMessage, error) { + bindings := make([]unitRulesBinding, 0, len(rules)) + for _, rule := range rules { + if math.IsNaN(rule.Scale) || math.IsInf(rule.Scale, 0) || math.IsNaN(rule.Offset) || math.IsInf(rule.Offset, 0) { + return nil, fmt.Errorf("%w: unit rule %q carries a non-finite scale or offset (the unknown call/type pair gate before execution)", ErrUnsupportedPostgresPlan, rule.ID) + } + bindings = append(bindings, unitRulesBinding{ + ID: rule.ID, Version: rule.Version, Kind: string(rule.Kind), + SourceSystem: rule.Source.System, SourceCode: rule.Source.Code, + TargetSystem: rule.Target.System, TargetCode: rule.Target.Code, + Scale: rule.Scale, Offset: rule.Offset, + }) + } + raw, err := json.Marshal(bindings) + if err != nil { + return nil, fmt.Errorf("%w: unit rule set is not JSON-encodable: %v", ErrUnsupportedPostgresPlan, err) + } + return json.RawMessage(raw), nil +} + +func unitValueListExpression(document string) string { + return "(CASE WHEN " + document + " IS NULL OR jsonb_typeof(" + document + ") = 'null' THEN '[]'::jsonb" + + " WHEN jsonb_typeof(" + document + ") = 'array' THEN " + document + + " ELSE jsonb_build_array(" + document + ") END)" +} + +func (r *planRenderer) renderUnitNormalizedProjection(projection classifiedExtractionProjection) (string, error) { + facts := projection.unit + scope, ok := r.rows[facts.sourceVariable] + if !ok || scope.alias == "" || scope.derivedValue != "" || scope.derivedJSON { + return "", fmt.Errorf("%w: projection %q: unit normalization extracts from variable %q, which does not bind a readable stored row", + ErrUnsupportedPostgresPlan, projection.projection.Name, facts.sourceVariable) + } + if facts.resourceType != scope.resourceType { + return "", fmt.Errorf("%w: projection %q: unit normalization validates selectors against resource type %q, but the row is scanned as %q", + ErrUnsupportedPostgresPlan, projection.projection.Name, facts.resourceType, scope.resourceType) + } + rulesRaw, err := unitRulesBindingJSON(facts.rules) + if err != nil { + return "", fmt.Errorf("projection %q: %w", projection.projection.Name, err) + } + rulesIndex := r.addParameter(command.PostgreSQLParameter{ + Type: command.PostgreSQLParameterJSONB, + Value: append(json.RawMessage(nil), rulesRaw...), + }, "unit_rules:"+string(rulesRaw)) + + valueDocument, err := r.documentPathStepsExpression(scope.alias, facts.valueSteps) + if err != nil { + return "", fmt.Errorf("projection %q: unit original value selector: %w", projection.projection.Name, err) + } + systemDocument, err := r.documentPathStepsExpression(scope.alias, facts.systemSteps) + if err != nil { + return "", fmt.Errorf("projection %q: unit system selector: %w", projection.projection.Name, err) + } + codeDocument, err := r.documentPathStepsExpression(scope.alias, facts.codeSteps) + if err != nil { + return "", fmt.Errorf("projection %q: unit code selector: %w", projection.projection.Name, err) + } + + valueList := unitValueListExpression(valueDocument) + systemFirst := "(" + unitValueListExpression(systemDocument) + ") -> 0" + codeFirst := "(" + unitValueListExpression(codeDocument) + ") -> 0" + + value := "loom_un_value.v" + coerced := callNumericCoercion(value) + clamped := callBandOperandClamp(coerced) + scaleNumeric := "((loom_un_rule.r ->> 'scale')::numeric)" + offsetNumeric := "((loom_un_rule.r ->> 'offset')::numeric)" + scaleDouble := "((loom_un_rule.r ->> 'scale')::double precision)" + offsetDouble := "((loom_un_rule.r ->> 'offset')::double precision)" + productMagnitude := "abs((" + callBandOperandClamp(coerced) + " * " + callBandOperandClamp(scaleNumeric) + "))" + sumMagnitude := "abs((" + callBandOperandClamp(coerced) + " * " + callBandOperandClamp(scaleNumeric) + " + " + callBandOperandClamp(offsetNumeric) + "))" + + converted := "(CASE" + + " WHEN " + coerced + " IS NULL THEN 'null'::jsonb" + + " WHEN abs(" + clamped + ") >= '" + callDoubleOverflowBound + "'::numeric THEN 'null'::jsonb" + + " WHEN " + productMagnitude + " < '" + callDoubleMulDivPassUpper + "'::numeric" + + " AND " + sumMagnitude + " < '" + unitAddPassUpper + "'::numeric" + + " THEN to_jsonb(" + callDoubleOperand(coerced) + " * " + scaleDouble + " + " + offsetDouble + ")" + + " WHEN " + productMagnitude + " < '" + callDoubleSubnormalFloor + "'::numeric" + + " THEN to_jsonb(0::double precision + " + offsetDouble + ")" + + " ELSE 'null'::jsonb END)" + + fragment := "(SELECT COALESCE(jsonb_agg(" + converted + " ORDER BY loom_un_value.v_ord), '[]'::jsonb)" + + " FROM (SELECT loom_un_item.v, loom_un_item.ord AS v_ord" + + " FROM jsonb_array_elements(" + valueList + ") WITH ORDINALITY AS loom_un_item(v, ord)" + + " WHERE loom_un_item.v IS NOT NULL AND loom_un_item.v <> 'null'::jsonb) AS loom_un_value" + + " CROSS JOIN LATERAL (SELECT" + + " CASE WHEN jsonb_typeof(" + systemFirst + ") = 'string' THEN " + systemFirst + " #>> '{}' END AS system_text," + + " CASE WHEN jsonb_typeof(" + codeFirst + ") = 'string' THEN " + codeFirst + " #>> '{}' END AS code_text) AS loom_un_identity" + + " CROSS JOIN LATERAL (SELECT CASE" + + " WHEN COUNT(*) = 0 THEN to_jsonb(" + guardCast(unitGuardIdentityUnknown) + ")" + + " WHEN COUNT(*) > 1 THEN to_jsonb(" + guardCast(unitGuardRuleAmbiguous) + ")" + + " ELSE MIN(loom_un_candidate.r::text)::jsonb END AS r" + + " FROM jsonb_array_elements($" + strconv.Itoa(rulesIndex) + "::jsonb) AS loom_un_candidate(r)" + + " WHERE loom_un_candidate.r ->> 'source_system' = loom_un_identity.system_text" + + " AND loom_un_candidate.r ->> 'source_code' = loom_un_identity.code_text) AS loom_un_rule)" + if err := checkCallFragment(fragment); err != nil { + return "", fmt.Errorf("projection %q: %w", projection.projection.Name, err) + } + return "(" + fragment + ")", nil +} diff --git a/internal/dataframe/compiler/render/postgres/unit_normalization_production_test.go b/internal/dataframe/compiler/render/postgres/unit_normalization_production_test.go new file mode 100644 index 000000000..1ea7c107c --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/unit_normalization_production_test.go @@ -0,0 +1,665 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "math/big" + "net/url" + "regexp" + "strconv" + "strings" + "testing" + "time" + + control "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/calypr/loom/internal/dataframe/unit" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + sql21UCUM = "http://unitsofmeasure.org" + sql21Gen = "gen-1" + sql21Path = "FHIR/Observation" + sql21Prod = "proj-a" +) + +var sql21LiveNamePattern = regexp.MustCompile(`^verify_sql21v1_[0-9]+_[0-9a-f]+$`) + +type sql21Service struct { + *pgsq01Service +} + +func newSQL21Service(t *testing.T) *sql21Service { + t.Helper() + raw := testPostgresURL(t) + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v string) string { + s := v + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err.Error())) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := sql21ResourceName(t) + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + sql21TrackResource(t, dbName) + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err.Error())) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := sql21ResourceName(t) + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + sql21TrackResource(t, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err.Error())) + } + }) + + client, err := control.OpenPool(ctx, control.ClientConfig{ + DSN: dsn, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 2 * time.Second, + }) + if err != nil { + fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + t.Cleanup(client.Close) + if _, err := control.MigrateSchema(ctx, client, schema); err != nil { + fail("migrate owned schema", err) + } + return &sql21Service{pgsq01Service: &pgsq01Service{ + ctx: ctx, iso: iso, client: client, schema: schema, scrub: scrub, fail: fail, + }} +} + +func sql21ResourceName(t *testing.T) string { + t.Helper() + name := "verify_sql21v1_" + strconv.FormatInt(time.Now().UnixNano(), 10) + "_" + randomHex(t) + if !sql21LiveNamePattern.MatchString(name) { + t.Fatalf("owned resource name %q does not match the required pattern", name) + } + return name +} + +func sql21TrackResource(t *testing.T, name string) { + t.Helper() + sql21FixturesMu.Lock() + defer sql21FixturesMu.Unlock() + sql21TrackedFixtures[name] = true +} + +func sql21DriverArgs(rendered command.RenderedPostgresPhysicalPlan) []any { + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + return args +} + +func (s *sql21Service) renderLive(t *testing.T, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, bindingFor(t, s.schema)) + if err != nil { + s.fail("render plan", err) + } + return rendered +} + +func (s *sql21Service) explain(t *testing.T, rendered command.RenderedPostgresPhysicalPlan) string { + t.Helper() + var text strings.Builder + for _, row := range s.run(t, "EXPLAIN (FORMAT TEXT) "+rendered.SQL, sql21DriverArgs(rendered)...) { + text.WriteString(strings.Join(row, "\n")) + text.WriteString("\n") + } + if text.Len() == 0 { + s.fail("explain produced no plan", errors.New(rendered.SQL)) + } + return text.String() +} + +func (s *sql21Service) expectRaise(t *testing.T, rendered command.RenderedPostgresPhysicalPlan, wantCode string, wantSubstrings ...string) { + t.Helper() + _, err := s.iso.Exec(s.ctx, rendered.SQL, sql21DriverArgs(rendered)...) + if err == nil { + t.Fatalf("statement executed clean, want %s raising %v: %s", wantCode, wantSubstrings, rendered.SQL) + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + s.fail("expect pg error", err) + } + if pgErr.Code != wantCode { + s.fail("statement raised ("+pgErr.Code+") want ("+wantCode+")", err) + } + for _, want := range wantSubstrings { + if !strings.Contains(pgErr.Message, want) { + t.Fatalf("error message %q does not name %q\nSQL: %s", pgErr.Message, want, rendered.SQL) + } + } +} + +func (s *sql21Service) runOne(t *testing.T, rendered command.RenderedPostgresPhysicalPlan, want ...string) { + t.Helper() + rows := s.runRendered(t, rendered) + if len(rows) != 1 { + t.Fatalf("rows = %v, want exactly one row %v", rows, want) + } + if strings.Join(rows[0], "|") != strings.Join(want, "|") { + t.Fatalf("row = %v, want %v\nSQL: %s", rows[0], want, rendered.SQL) + } +} + +func sql21Observation(value any, system any, code any) map[string]any { + quantity := map[string]any{} + if value != nil { + quantity["value"] = value + } + if system != nil { + quantity["system"] = system + } + if code != nil { + quantity["code"] = code + } + return map[string]any{"status": "final", "valueQuantity": quantity} +} + +var sql21JSONNull = json.RawMessage("null") + +func sql21LivePlan(project string, scanVariable string, projections ...ir.PhysicalProjection) ir.PhysicalPlan { + return ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": "Observation", + "project": project, + "dataset_generation": sql21Gen, + "auth_resource_paths": []string{sql21Path}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + }, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: scanVariable, CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}}, + }, + } +} + +func sql21FahrenheitProjection(t *testing.T) ir.PhysicalProjection { + t.Helper() + target, dimension, rules := sql21FahrenheitFacts(t) + valueSelector := sql21Steps("valueQuantity", "value") + extract := &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Observation", + Selector: valueSelector, + ExecutionMode: ir.PhysicalSelectorGeneric, + UnitNormalization: &ir.PhysicalUnitNormalization{ + OriginalValue: valueSelector, + SourceSystem: sql21Steps("valueQuantity", "system"), + SourceCode: sql21Steps("valueQuantity", "code"), + Target: target, + Dimension: dimension, + Rules: rules, + }, + } + return ir.PhysicalProjection{Name: "temp_f", Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, Extract: extract, + }} +} + +func sql21IndexedComponentProjection(t *testing.T, index int) ir.PhysicalProjection { + t.Helper() + target, dimension, rules := sql21CentimeterFacts(t) + valueSelector := spec.Selector{Steps: []spec.SelectorStep{ + {Field: "component", Index: &index}, {Field: "valueQuantity"}, {Field: "value"}, + }} + systemSelector := spec.Selector{Steps: []spec.SelectorStep{ + {Field: "component", Index: &index}, {Field: "valueQuantity"}, {Field: "system"}, + }} + codeSelector := spec.Selector{Steps: []spec.SelectorStep{ + {Field: "component", Index: &index}, {Field: "valueQuantity"}, {Field: "code"}, + }} + extract := &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Observation", + Selector: valueSelector, + ExecutionMode: ir.PhysicalSelectorGeneric, + UnitNormalization: &ir.PhysicalUnitNormalization{ + OriginalValue: valueSelector, + SourceSystem: systemSelector, + SourceCode: codeSelector, + Target: target, + Dimension: dimension, + Rules: rules, + }, + } + return ir.PhysicalProjection{Name: "component_cm", Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, Extract: extract, + }} +} + +type sql21Cell struct { + project string + payload map[string]any + want string + wantTag string +} + +func sql21RunCells(t *testing.T, svc *sql21Service, projection ir.PhysicalProjection, cells []sql21Cell) { + t.Helper() + for _, cell := range cells { + svc.insertResource(t, cell.project, sql21Gen, "Observation", cell.project, sql21Path, cell.payload) + rendered := svc.renderLive(t, sql21LivePlan(cell.project, "root", projection)) + if cell.wantTag == "" { + svc.runOne(t, rendered, cell.want) + continue + } + planText := svc.explain(t, rendered) + if !strings.Contains(planText, "SubPlan") && !strings.Contains(planText, "Function Scan") { + t.Fatalf("%s: guard subplan missing from plan:\n%s", cell.project, planText) + } + svc.expectRaise(t, rendered, "22P02", cell.wantTag) + } +} + +func TestSQL21LiveLinearAndIdentityConversionsReadExact(t *testing.T) { + svc := newSQL21Service(t) + cm := sql21UnitProjection(t, nil) + sql21RunCells(t, svc, cm, []sql21Cell{ + {"sql21-exact-cm", sql21Observation(5, sql21UCUM, "cm"), "[5]", ""}, + {"sql21-exact-m", sql21Observation(2, sql21UCUM, "m"), "[200]", ""}, + }) + fahrenheit := sql21FahrenheitProjection(t) + sql21RunCells(t, svc, fahrenheit, []sql21Cell{ + {"sql21-cel", sql21Observation(20, sql21UCUM, "Cel"), "[68]", ""}, + {"sql21-degf-identity", sql21Observation(98.6, sql21UCUM, "[degF]"), "[98.6]", ""}, + }) + svc.insertResource(t, "sql21-exact-cm", sql21Gen, "Patient", "sql21-exact-cm", sql21Path, + map[string]any{"gender": "female"}) + svc.runOne(t, svc.renderLive(t, sql21LivePlan("sql21-exact-cm", "root", cm)), "[5]") +} + +func TestSQL21LiveNullValuesSkipBeforeIdentityAssert(t *testing.T) { + svc := newSQL21Service(t) + cm := sql21UnitProjection(t, nil) + sql21RunCells(t, svc, cm, []sql21Cell{ + {"sql21-null-value", sql21Observation(sql21JSONNull, sql21UCUM, "cm"), "[]", ""}, + {"sql21-null-value-bad-identity", sql21Observation(sql21JSONNull, "urn:example:bad", "furlong"), "[]", ""}, + {"sql21-array-nulls", sql21Observation([]any{2, nil, 4}, sql21UCUM, "cm"), "[2 4]", ""}, + {"sql21-null-quantity", map[string]any{"status": "final", "valueQuantity": sql21JSONNull}, "[]", ""}, + }) +} + +func TestSQL21LiveEmptyValueListReadsEmptyArray(t *testing.T) { + svc := newSQL21Service(t) + cm := sql21UnitProjection(t, nil) + sql21RunCells(t, svc, cm, []sql21Cell{ + {"sql21-missing-quantity", map[string]any{"status": "final"}, "[]", ""}, + {"sql21-empty-array", sql21Observation([]any{}, sql21UCUM, "cm"), "[]", ""}, + {"sql21-quantity-no-value", map[string]any{"status": "final", + "valueQuantity": map[string]any{"system": sql21UCUM, "code": "cm"}}, "[]", ""}, + }) +} + +func TestSQL21LiveDisplayLabelDoesNotSelectRule(t *testing.T) { + svc := newSQL21Service(t) + cm := sql21UnitProjection(t, nil) + displayLabel := map[string]any{"status": "final", "valueQuantity": map[string]any{ + "value": 5, "system": sql21UCUM, "code": "centimeter", "unit": "cm", + }} + sql21RunCells(t, svc, cm, []sql21Cell{ + {"sql21-display-label", displayLabel, "", unitGuardIdentityUnknown}, + {"sql21-missing-system", map[string]any{"status": "final", + "valueQuantity": map[string]any{"value": 5, "code": "cm"}}, "", unitGuardIdentityUnknown}, + }) +} + +func TestSQL21LiveExactIdentityPair(t *testing.T) { + svc := newSQL21Service(t) + cm := sql21UnitProjection(t, nil) + sql21RunCells(t, svc, cm, []sql21Cell{ + {"sql21-code-leading-space", sql21Observation(5, sql21UCUM, " cm"), "", unitGuardIdentityUnknown}, + {"sql21-code-case", sql21Observation(5, sql21UCUM, "CM"), "", unitGuardIdentityUnknown}, + {"sql21-system-space", sql21Observation(5, sql21UCUM+" ", "cm"), "", unitGuardIdentityUnknown}, + {"sql21-unknown-code", sql21Observation(5, sql21UCUM, "furlong"), "", unitGuardIdentityUnknown}, + {"sql21-code-number", sql21Observation(5, sql21UCUM, 5), "", unitGuardIdentityUnknown}, + }) +} + +func TestSQL21LiveRepeatedSourceSiblingIdentity(t *testing.T) { + svc := newSQL21Service(t) + cm := sql21UnitProjection(t, nil) + firstWins := map[string]any{"status": "final", "valueQuantity": map[string]any{ + "value": 4, "system": []any{sql21UCUM, sql21UCUM}, "code": []any{"cm", "m"}, + }} + nullFirst := map[string]any{"status": "final", "valueQuantity": map[string]any{ + "value": 4, "system": []any{nil, sql21UCUM}, "code": []any{nil, "cm"}, + }} + sql21RunCells(t, svc, cm, []sql21Cell{ + {"sql21-first-sibling", firstWins, "[4]", ""}, + {"sql21-single-item-array", map[string]any{"status": "final", "valueQuantity": map[string]any{ + "value": 6, "system": []any{sql21UCUM}, "code": []any{"cm"}, + }}, "[6]", ""}, + {"sql21-null-first-sibling", nullFirst, "", unitGuardIdentityUnknown}, + }) +} + +func TestSQL21LiveIndexedQuantityReadsInRangeAndOutOfRange(t *testing.T) { + svc := newSQL21Service(t) + payload := map[string]any{"status": "final", "component": []any{ + map[string]any{"valueQuantity": map[string]any{"value": 7, "system": sql21UCUM, "code": "cm"}}, + }} + sql21RunCells(t, svc, sql21IndexedComponentProjection(t, 0), []sql21Cell{ + {"sql21-indexed-in-range", payload, "[7]", ""}, + }) + sql21RunCells(t, svc, sql21IndexedComponentProjection(t, 3), []sql21Cell{ + {"sql21-indexed-out-of-range", payload, "[]", ""}, + }) +} + +func TestSQL21LiveDecimalStringAndNonNumericReadBack(t *testing.T) { + svc := newSQL21Service(t) + cm := sql21UnitProjection(t, nil) + sql21RunCells(t, svc, cm, []sql21Cell{ + {"sql21-decimal-string", sql21Observation("2.50", sql21UCUM, "cm"), "[2.5]", ""}, + {"sql21-boolean-value", sql21Observation(true, sql21UCUM, "cm"), "[]", ""}, + {"sql21-object-value", sql21Observation(map[string]any{"a": 1}, sql21UCUM, "cm"), "[]", ""}, + {"sql21-word-value", sql21Observation("abc", sql21UCUM, "cm"), "[]", ""}, + {"sql21-huge-exponent", sql21Observation("1e999999999999", sql21UCUM, "cm"), "[]", ""}, + {"sql21-null-then-value", sql21Observation([]any{nil, "3"}, sql21UCUM, "cm"), "[3]", ""}, + }) +} + +func TestSQL21LiveOverflowReadsNull(t *testing.T) { + svc := newSQL21Service(t) + cm := sql21UnitProjection(t, nil) + sql21RunCells(t, svc, cm, []sql21Cell{ + {"sql21-multiply-overflow", sql21Observation("1e307", sql21UCUM, "m"), "[]", ""}, + {"sql21-operand-overflow", sql21Observation("1.8e308", sql21UCUM, "cm"), "[]", ""}, + }) +} + +func TestSQL21LiveAdditionNearTieReadsNull(t *testing.T) { + svc := newSQL21Service(t) + upper := new(big.Int).Mul( + new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 53), big.NewInt(1)), + new(big.Int).Lsh(big.NewInt(1), 971)) + upper.Sub(upper, new(big.Int).Lsh(big.NewInt(1), 975)) + valueInt := new(big.Int).Quo(new(big.Int).Mul(upper, big.NewInt(10)), big.NewInt(18)) + productTenths := new(big.Int).Mul(valueInt, big.NewInt(18)) + lowerTenths := new(big.Int).Sub(new(big.Int).Mul(upper, big.NewInt(10)), big.NewInt(18)) + if productTenths.Cmp(lowerTenths) < 0 || productTenths.Cmp(new(big.Int).Mul(upper, big.NewInt(10))) >= 0 { + t.Fatalf("constructed near-tie product left [upper-1.8, upper)") + } + if new(big.Int).Add(productTenths, big.NewInt(320)).Cmp(new(big.Int).Mul(upper, big.NewInt(10))) <= 0 { + t.Fatalf("near-tie sum must sit above the addition pass boundary") + } + mulDiv := new(big.Int) + if _, ok := mulDiv.SetString(callDoubleMulDivPassUpper, 10); !ok { + t.Fatalf("callDoubleMulDivPassUpper is not integral decimal text") + } + if mulDiv.Cmp(upper) <= 0 { + t.Fatalf("multiply bound must sit above the addition bound") + } + operand := new(big.Int) + if _, ok := operand.SetString(callDoubleOverflowBound, 10); !ok { + t.Fatalf("callDoubleOverflowBound is not integral decimal text") + } + if valueInt.Cmp(operand) >= 0 { + t.Fatalf("near-tie operand must stay below the operand overflow band") + } + + fahrenheit := sql21FahrenheitProjection(t) + sql21RunCells(t, svc, fahrenheit, []sql21Cell{ + {"sql21-addition-near-tie", sql21Observation(valueInt.String(), sql21UCUM, "Cel"), "[]", ""}, + }) +} + +func sql21UnitNormalizationBundle(name string, target unit.UnitIdentity, rules []unit.UnitRuleReference) recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: name, + TranslationVersion: "test", + Outputs: []recipe.Output{{ + Name: "observations", RootResourceType: "Observation", RowGrain: "resource", + Fields: []recipe.Field{{Name: "status", Expr: recipe.Expression{Select: "root.status"}}}, + Aggregates: []recipe.Aggregate{{ + Name: "height_cm", Operation: recipe.AggregateMax, + Expr: &recipe.Expression{Select: "root.valueQuantity.value"}, + UnitNormalization: &recipe.UnitNormalizationPolicy{ + SystemPath: "valueQuantity.system", CodePath: "valueQuantity.code", + Target: target, + Rules: rules, + }, + }}, + }}, + } +} + +func sql21CompileProduction(t *testing.T, bundle recipe.Bundle) ir.PhysicalPlan { + t.Helper() + bindings := recipe.RuntimeBindings{ + Project: sql21Prod, DatasetGeneration: sql21Gen, + AuthResourcePaths: []string{sql21Path}, + } + plan, err := semantic.BuildRecipePlan(bundle, bindings) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("lower.CompileResolvedRecipePlan: %v", err) + } + if len(compiled.Outputs) != 1 { + t.Fatalf("compiled outputs = %d, want 1", len(compiled.Outputs)) + } + output := compiled.Outputs[0] + if output.OptimizedPlan != nil { + return *output.OptimizedPlan + } + return output.Plan +} + +func sql21CentimeterPolicyRefs(t *testing.T) (unit.UnitIdentity, []unit.UnitRuleReference) { + t.Helper() + policy, err := unit.ResolveApprovedUnitPolicy("to-centimeters", "1") + if err != nil { + t.Fatalf("to-centimeters policy must resolve: %v", err) + } + references := make([]unit.UnitRuleReference, 0, len(policy.Rules)) + for _, rule := range policy.Rules { + references = append(references, unit.UnitRuleReference{ID: rule.ID, Version: rule.Version}) + } + return policy.Target, references +} + +func sql21FindAggregateNormalization(t *testing.T, plan ir.PhysicalPlan) *ir.PhysicalAggregate { + t.Helper() + for _, operation := range plan.Operations { + if operation.Kind != ir.PhysicalReturnOp || operation.Return == nil { + continue + } + for _, projection := range operation.Return.Projections { + expression := projection.Expression + if expression == nil || expression.Kind != ir.PhysicalAggregateExpression || expression.Aggregate == nil { + continue + } + value := expression.Aggregate.Value + if value != nil && value.Extract != nil && value.Extract.UnitNormalization != nil { + return expression.Aggregate + } + } + } + t.Fatalf("production plan carries no unit-normalized aggregate projection") + return nil +} + +func TestSQL21ProductionRecipeRefusedAtAggregateGate(t *testing.T) { + target, references := sql21CentimeterPolicyRefs(t) + plan := sql21CompileProduction(t, sql21UnitNormalizationBundle("sql21_p1_aggregate_gate", target, references)) + aggregate := sql21FindAggregateNormalization(t, plan) + if aggregate.Value.Extract.UnitNormalization == nil { + t.Fatal("witness lost its normalization") + } + binding, err := ir.NewPostgresStorageBinding("loom_offline") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + _, err = RenderPhysicalPlan(plan, binding) + if err == nil { + t.Fatal("production aggregate plan rendered; the aggregate gate must refuse it (pg-sql-13 owns that surface)") + } + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("aggregate gate error = %v, want ErrUnsupportedPostgresPlan", err) + } + if !strings.Contains(err.Error(), string(ir.PhysicalAggregateExpression)) || + !strings.Contains(err.Error(), "no registered PostgreSQL rendering") { + t.Fatalf("aggregate gate refusal text drifted: %v", err) + } + t.Logf("P-1 verbatim aggregate-gate refusal (banked): %v", err) +} + +func TestSQL21ProductionLoweredExtractHoistsByteIdentically(t *testing.T) { + target, references := sql21CentimeterPolicyRefs(t) + plan := sql21CompileProduction(t, sql21UnitNormalizationBundle("sql21_p2_hoist_identity", target, references)) + aggregate := sql21FindAggregateNormalization(t, plan) + production := aggregate.Value + + productionRaw, err := json.Marshal(production) + if err != nil { + t.Fatalf("marshal production expression: %v", err) + } + var hoisted ir.PhysicalExpression + if err := json.Unmarshal(productionRaw, &hoisted); err != nil { + t.Fatalf("unmarshal hoisted expression: %v", err) + } + hoistedRaw, err := json.Marshal(&hoisted) + if err != nil { + t.Fatalf("marshal hoisted expression: %v", err) + } + t.Logf("P-2 lowering output : %s", productionRaw) + t.Logf("P-2 hoisted EXTRACT : %s", hoistedRaw) + if string(productionRaw) != string(hoistedRaw) { + t.Fatalf("hoisted expression is not byte-identical to the lowering output") + } + if hoisted.Extract == nil || hoisted.Extract.UnitNormalization == nil || + hoisted.Kind != ir.PhysicalExtractExpression || + hoisted.Cardinality != ir.PhysicalArrayCardinality || + hoisted.NullBehavior != ir.PhysicalEmptyOnNull { + t.Fatalf("hoisted expression left the registered posture: %+v", hoisted) + } + + projection := ir.PhysicalProjection{Name: "length_cm", Expression: &hoisted} + offlineBinding, err := ir.NewPostgresStorageBinding("loom_offline") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + livePlan := func(project string) ir.PhysicalPlan { + return sql21LivePlan(project, hoisted.Extract.Source.Variable, projection) + } + if _, err := RenderPhysicalPlan(livePlan("sql21_p2_offline"), offlineBinding); err != nil { + t.Fatalf("renderer refused the hoisted production EXTRACT: %v", err) + } + + svc := newSQL21Service(t) + sql21RunCells(t, svc, projection, []sql21Cell{ + {"sql21_p2_cm", sql21Observation(5, sql21UCUM, "cm"), "[5]", ""}, + {"sql21_p2_m", sql21Observation(2, sql21UCUM, "m"), "[200]", ""}, + {"sql21_p2_null", sql21Observation(nil, sql21UCUM, "cm"), "[]", ""}, + }) +} + +func TestSQL21TamperedRuleSetFailsClosed(t *testing.T) { + tamperedScale := sql21UnitProjection(t, func(x *ir.PhysicalExtract) { + x.UnitNormalization.Rules[0].Scale = 101 + }) + err := sql21ContractError(t, tamperedScale, nil) + if !strings.Contains(err.Error(), "does not match the pinned registry entry") { + t.Fatalf("tampered scale refused with drifted text: %v", err) + } + + unknown := sql21UnitProjection(t, func(x *ir.PhysicalExtract) { + x.UnitNormalization.Rules = []unit.UnitConversionRule{{ + ID: "invented", Version: "1", Kind: unit.UnitConversionLinear, + Source: unit.UnitIdentity{System: sql21UCUM, Code: "furlong"}, + Target: unit.UnitIdentity{System: sql21UCUM, Code: "cm"}, + Dimension: unit.UnitDimension{Length: 1}, Scale: 20116.8, + }} + }) + err = sql21ContractError(t, unknown, nil) + if !errors.Is(err, unit.ErrUnitIdentityUnknown) { + t.Fatalf("unapproved rule material = %v, want unit.ErrUnitIdentityUnknown sentinel", err) + } + + driftedDimension := sql21UnitProjection(t, func(x *ir.PhysicalExtract) { + x.UnitNormalization.Dimension = unit.UnitDimension{Time: 1} + }) + err = sql21ContractError(t, driftedDimension, nil) + if !errors.Is(err, unit.ErrUnitDimensionIncompatible) { + t.Fatalf("drifted dimension = %v, want unit.ErrUnitDimensionIncompatible sentinel", err) + } +} diff --git a/internal/dataframe/compiler/render/postgres/unit_normalization_test.go b/internal/dataframe/compiler/render/postgres/unit_normalization_test.go new file mode 100644 index 000000000..a27774b54 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/unit_normalization_test.go @@ -0,0 +1,832 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "math/big" + "os" + "reflect" + "regexp" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/calypr/loom/internal/dataframe/unit" +) + +func sql21CentimeterFacts(t *testing.T) (unit.UnitIdentity, unit.UnitDimension, []unit.UnitConversionRule) { + t.Helper() + policy, err := unit.ResolveApprovedUnitPolicy("to-centimeters", "1") + if err != nil { + t.Fatalf("ResolveApprovedUnitPolicy(to-centimeters): %v", err) + } + dimension, rules, err := unit.ResolveApprovedUnitRules(policy.Rules, policy.Target) + if err != nil { + t.Fatalf("ResolveApprovedUnitRules(to-centimeters): %v", err) + } + return policy.Target, dimension, rules +} + +func sql21FahrenheitFacts(t *testing.T) (unit.UnitIdentity, unit.UnitDimension, []unit.UnitConversionRule) { + t.Helper() + policy, err := unit.ResolveApprovedUnitPolicy("to-fahrenheit", "1") + if err != nil { + t.Fatalf("ResolveApprovedUnitPolicy(to-fahrenheit): %v", err) + } + dimension, rules, err := unit.ResolveApprovedUnitRules(policy.Rules, policy.Target) + if err != nil { + t.Fatalf("ResolveApprovedUnitRules(to-fahrenheit): %v", err) + } + return policy.Target, dimension, rules +} + +func sql21Steps(fields ...string) spec.Selector { + steps := make([]spec.SelectorStep, 0, len(fields)) + for _, field := range fields { + steps = append(steps, spec.SelectorStep{Field: field}) + } + return spec.Selector{Steps: steps} +} + +func sql21UnitExtract(t *testing.T, mutate func(*ir.PhysicalExtract)) *ir.PhysicalExtract { + t.Helper() + target, dimension, rules := sql21CentimeterFacts(t) + valueSelector := sql21Steps("valueQuantity", "value") + extract := &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Observation", + Selector: valueSelector, + ExecutionMode: ir.PhysicalSelectorGeneric, + UnitNormalization: &ir.PhysicalUnitNormalization{ + OriginalValue: valueSelector, + SourceSystem: sql21Steps("valueQuantity", "system"), + SourceCode: sql21Steps("valueQuantity", "code"), + Target: target, + Dimension: dimension, + Rules: rules, + }, + } + if mutate != nil { + mutate(extract) + } + return extract +} + +func sql21UnitProjection(t *testing.T, mutate func(*ir.PhysicalExtract)) ir.PhysicalProjection { + t.Helper() + expression := &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Extract: sql21UnitExtract(t, mutate), + } + return ir.PhysicalProjection{Name: "length_cm", Expression: expression} +} + +func sql21Plan(projections ...ir.PhysicalProjection) ir.PhysicalPlan { + return ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": "Observation", + "project": "sql21-offline", + "dataset_generation": "gen-1", + "auth_resource_paths": []string{"FHIR/Observation"}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + }, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}}, + }, + } +} + +func sql21Render(t *testing.T, plan ir.PhysicalPlan) string { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("loom_offline") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + rendered, err := RenderPhysicalPlan(plan, binding) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + if strings.Contains(rendered.SQL, "@") { + t.Fatalf("unit projection SQL carries AQL bind syntax: %s", rendered.SQL) + } + return rendered.SQL +} + +func sql21RenderError(t *testing.T, projection ir.PhysicalProjection) error { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("loom_offline") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + _, err = RenderPhysicalPlan(sql21Plan(projection), binding) + if err == nil { + t.Fatalf("unit projection accepted, want refusal") + } + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("refusal error = %v, want ErrUnsupportedPostgresPlan", err) + } + return err +} + +func sql21RulesParameters(t *testing.T, args []command.PostgreSQLParameter) [][]map[string]any { + t.Helper() + var all [][]map[string]any + for _, argument := range args { + raw, ok := argument.Value.(json.RawMessage) + if !ok { + continue + } + if argument.Type != command.PostgreSQLParameterJSONB || !strings.Contains(string(raw), "source_system") { + continue + } + var rules []map[string]any + if err := json.Unmarshal(raw, &rules); err != nil { + t.Fatalf("rules parameter is not valid JSON: %v", err) + } + all = append(all, rules) + } + return all +} + +func sql21RulesParameter(t *testing.T, args []command.PostgreSQLParameter) []map[string]any { + t.Helper() + all := sql21RulesParameters(t, args) + if len(all) != 1 { + t.Fatalf("rules-shaped jsonb binds = %d, want exactly 1 (one renderer-minted shared bind per distinct rule set)", len(all)) + } + return all[0] +} + +func sql21RulesBindIndexes(t *testing.T, sql string) []string { + t.Helper() + var indexes []string + for _, match := range sql21RulesBindPattern.FindAllStringSubmatch(sql, -1) { + indexes = append(indexes, match[1]) + } + return indexes +} + +var sql21RulesBindPattern = regexp.MustCompile(`jsonb_array_elements\(\$(\d+)::jsonb\)`) + +func TestSQL21IdenticalRuleSetsShareOneBind(t *testing.T) { + first := sql21UnitProjection(t, nil) + first.Name = "length_cm" + second := sql21UnitProjection(t, nil) + second.Name = "width_cm" + binding, err := ir.NewPostgresStorageBinding("loom_offline") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + rendered, err := RenderPhysicalPlan(sql21Plan(first, second), binding) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + if all := sql21RulesParameters(t, rendered.Args); len(all) != 1 { + t.Fatalf("identical rule sets minted %d rules binds, want one shared bind", len(all)) + } + indexes := sql21RulesBindIndexes(t, rendered.SQL) + if len(indexes) != 2 { + t.Fatalf("rules-lateral references = %d, want one per projection:\n%s", len(indexes), rendered.SQL) + } + if indexes[0] != indexes[1] { + t.Fatalf("identical rule sets read separate binds $%s and $%s:\n%s", indexes[0], indexes[1], rendered.SQL) + } + position, err := strconv.Atoi(indexes[0]) + if err != nil || position < 1 || position > len(rendered.Args) || + rendered.Args[position-1].Type != command.PostgreSQLParameterJSONB { + t.Fatalf("shared rules bind $%s does not name a jsonb argument", indexes[0]) + } +} + +func TestSQL21DistinctRuleSetsGetSeparateBinds(t *testing.T) { + binding, err := ir.NewPostgresStorageBinding("loom_offline") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + rendered, err := RenderPhysicalPlan(sql21Plan(sql21UnitProjection(t, nil), sql21FahrenheitProjection(t)), binding) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + if all := sql21RulesParameters(t, rendered.Args); len(all) != 2 { + t.Fatalf("distinct rule sets minted %d binds, want two", len(all)) + } + indexes := sql21RulesBindIndexes(t, rendered.SQL) + if len(indexes) != 2 || indexes[0] == indexes[1] { + t.Fatalf("distinct rule sets must read two separate binds, got %v:\n%s", indexes, rendered.SQL) + } +} + +func TestSQL21UnitProjectionRendersRegisteredShape(t *testing.T) { + binding, err := ir.NewPostgresStorageBinding("loom_offline") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + rendered, err := RenderPhysicalPlan(sql21Plan(sql21UnitProjection(t, nil)), binding) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + sql := rendered.SQL + for _, want := range []string{ + "jsonb_array_elements(", "WITH ORDINALITY AS loom_un_item(v, ord)", + "WHERE loom_un_item.v IS NOT NULL AND loom_un_item.v <> 'null'::jsonb", + `loom_un_candidate.r ->> 'source_system' = loom_un_identity.system_text`, + `loom_un_candidate.r ->> 'source_code' = loom_un_identity.code_text`, + "UNIT_IDENTITY_UNKNOWN", "UNIT_RULE_AMBIGUOUS", + "COALESCE(jsonb_agg(", "ORDER BY loom_un_value.v_ord), '[]'::jsonb)", + "#> $", "::text[]", + } { + if !strings.Contains(sql, want) { + t.Fatalf("SQL missing %q:\n%s", want, sql) + } + } + dropIndex := strings.Index(sql, "loom_un_item.v IS NOT NULL") + guardIndex := strings.Index(sql, "UNIT_IDENTITY_UNKNOWN") + if dropIndex < 0 || guardIndex < 0 || dropIndex > guardIndex { + t.Fatalf("null drop must precede the identity guard:\n%s", sql) + } + if strings.Contains(sql, `->> 'unit'`) || strings.Contains(sql, `#>> '{unit}`) { + t.Fatalf("display label consulted:\n%s", sql) + } + for _, leaked := range []string{"valueQuantity", "value", "system", "code"} { + if strings.Contains(sql, "'"+leaked+"'") { + t.Fatalf("selector segment %q leaked into SQL text:\n%s", leaked, sql) + } + } + rules := sql21RulesParameter(t, rendered.Args) + if len(rules) != 2 { + t.Fatalf("rules binding = %d entries, want 2 (identity + ucum:m-to-cm): %#v", len(rules), rules) + } + if rules[0]["id"] != "ucum:m-to-cm" || rules[0]["scale"] != float64(100) || rules[0]["offset"] != float64(0) || + rules[0]["source_system"] != "http://unitsofmeasure.org" || rules[0]["source_code"] != "m" { + t.Fatalf("linear rule binding = %#v", rules[0]) + } + if rules[1]["id"] != "identity-v1" || rules[1]["scale"] != float64(1) || rules[1]["offset"] != float64(0) || + rules[1]["source_system"] != "http://unitsofmeasure.org" || rules[1]["source_code"] != "cm" { + t.Fatalf("identity rule binding = %#v", rules[1]) + } + for _, arm := range []string{"COUNT(*) > 1", "UNIT_RULE_AMBIGUOUS", callDoubleMulDivPassUpper[:24], unitAddPassUpper[:24]} { + if !strings.Contains(sql, arm) { + t.Fatalf("fragment missing registered arm %q", arm) + } + } + second, err := RenderPhysicalPlan(sql21Plan(sql21UnitProjection(t, nil)), binding) + if err != nil || second.SQL != rendered.SQL || !reflect.DeepEqual(second.Args, rendered.Args) { + t.Fatalf("unit projection render is not deterministic") + } +} + +func TestSQL21AffineProjectionRegistersOffset(t *testing.T) { + target, dimension, rules := sql21FahrenheitFacts(t) + valueSelector := sql21Steps("valueQuantity", "value") + expression := &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Observation", + Selector: valueSelector, + ExecutionMode: ir.PhysicalSelectorGeneric, + UnitNormalization: &ir.PhysicalUnitNormalization{ + OriginalValue: valueSelector, + SourceSystem: sql21Steps("valueQuantity", "system"), + SourceCode: sql21Steps("valueQuantity", "code"), + Target: target, Dimension: dimension, Rules: rules, + }, + }, + } + binding, err := ir.NewPostgresStorageBinding("loom_offline") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + rendered, err := RenderPhysicalPlan(sql21Plan(ir.PhysicalProjection{Name: "temp_f", Expression: expression}), binding) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + bound := sql21RulesParameter(t, rendered.Args) + var cToF map[string]any + for _, rule := range bound { + if rule["id"] == "ucum:c-to-f" { + cToF = rule + } + } + if cToF == nil { + t.Fatalf("ucum:c-to-f missing from binding %#v", bound) + } + if cToF["scale"] != 1.8 || cToF["offset"] != float64(32) || cToF["kind"] != "AFFINE" || + cToF["source_code"] != "Cel" || cToF["target_code"] != "[degF]" { + t.Fatalf("affine binding = %#v", cToF) + } +} + +func TestSQL21IndexedQuantityRegistersBoundIndex(t *testing.T) { + projection := sql21UnitProjection(t, func(x *ir.PhysicalExtract) { + index := 2 + valueSelector := spec.Selector{Steps: []spec.SelectorStep{ + {Field: "valueQuantity", Index: &index}, {Field: "value"}, + }} + systemSelector := spec.Selector{Steps: []spec.SelectorStep{ + {Field: "valueQuantity", Index: &index}, {Field: "system"}, + }} + codeSelector := spec.Selector{Steps: []spec.SelectorStep{ + {Field: "valueQuantity", Index: &index}, {Field: "code"}, + }} + x.Selector = valueSelector + x.UnitNormalization.OriginalValue = valueSelector + x.UnitNormalization.SourceSystem = systemSelector + x.UnitNormalization.SourceCode = codeSelector + }) + binding, err := ir.NewPostgresStorageBinding("loom_offline") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + rendered, err := RenderPhysicalPlan(sql21Plan(projection), binding) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + var boundIndexPath bool + for _, argument := range rendered.Args { + if argument.Type != command.PostgreSQLParameterText { + continue + } + if text, ok := argument.Value.(string); ok && strings.HasPrefix(text, "{valueQuantity,2,") { + boundIndexPath = true + } + } + if !boundIndexPath { + t.Fatalf("indexed step did not cross as a bound text[] path:\n%s", rendered.SQL) + } + if strings.Contains(rendered.SQL, "[2]") || strings.Contains(rendered.SQL, "-> 2") || + strings.Contains(rendered.SQL, "valueQuantity,2") { + t.Fatalf("index leaked into SQL text:\n%s", rendered.SQL) + } +} + +func TestSQL21NegativeIndexTailArm(t *testing.T) { + projection := sql21UnitProjection(t, func(x *ir.PhysicalExtract) { + index := -1 + valueSelector := spec.Selector{Steps: []spec.SelectorStep{ + {Field: "valueQuantity", Index: &index}, {Field: "value"}, + }} + systemSelector := spec.Selector{Steps: []spec.SelectorStep{ + {Field: "valueQuantity", Index: &index}, {Field: "system"}, + }} + codeSelector := spec.Selector{Steps: []spec.SelectorStep{ + {Field: "valueQuantity", Index: &index}, {Field: "code"}, + }} + x.Selector = valueSelector + x.UnitNormalization.OriginalValue = valueSelector + x.UnitNormalization.SourceSystem = systemSelector + x.UnitNormalization.SourceCode = codeSelector + }) + binding, err := ir.NewPostgresStorageBinding("loom_offline") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + rendered, err := RenderPhysicalPlan(sql21Plan(projection), binding) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + sql := rendered.SQL + for _, fragment := range []string{ + "CASE WHEN jsonb_typeof(", + "THEN (CASE WHEN $", + "< 0 THEN ", + "-> (jsonb_array_length(", + ") + $", + ") END) ELSE ", + "#> $", + } { + if !strings.Contains(sql, fragment) { + t.Fatalf("tail-arm SQL lost fragment %q:\n%s", fragment, sql) + } + } + var boundIndex bool + for _, argument := range rendered.Args { + if argument.Type == command.PostgreSQLParameterBigint { + if value, ok := argument.Value.(int64); ok && value == -1 { + boundIndex = true + } + } + } + if !boundIndex { + t.Fatalf("negative index did not cross as a bound bigint parameter:\n%s", sql) + } + if strings.Contains(sql, "[-1]") || strings.Contains(sql, "- 1)") || strings.Contains(sql, ",-1}") || + strings.Contains(sql, "{-1}") { + t.Fatalf("negative index leaked into SQL text:\n%s", sql) + } +} + +func TestSQL21RefusalTable(t *testing.T) { + for _, caseItem := range []struct { + name string + mutate func(*ir.PhysicalExtract) + mutate2 func(*ir.PhysicalProjection) + want string + }{ + {"scalar posture", func(x *ir.PhysicalExtract) {}, func(p *ir.PhysicalProjection) { + p.Expression.Cardinality = ir.PhysicalScalarCardinality + }, "ARRAY-cardinality EMPTY_ON_NULL"}, + {"preserve-null posture", func(x *ir.PhysicalExtract) {}, func(p *ir.PhysicalProjection) { + p.Expression.NullBehavior = ir.PhysicalPreserveNull + }, "ARRAY-cardinality EMPTY_ON_NULL"}, + {"distinct", func(x *ir.PhysicalExtract) { x.Distinct = true }, nil, "DISTINCT"}, + {"fallbacks", func(x *ir.PhysicalExtract) { + x.Fallbacks = []ir.PhysicalSelectorFallback{{Source: x.Source, ResourceType: "Observation", Selector: sql21Steps("status")}} + }, nil, "fallback selectors"}, + {"prepared", func(x *ir.PhysicalExtract) { + x.Prepared = &ir.PhysicalPreparedReference{SetVariable: "prepared", Field: "value"} + }, nil, "prepared-set"}, + {"selector filter", func(x *ir.PhysicalExtract) { + x.Selector.Filter = &spec.ContainsFilter{Field: "status", Needle: "x"} + }, nil, "CONTAINS filters"}, + {"conditional array mode", func(x *ir.PhysicalExtract) { + x.ExecutionMode = ir.PhysicalSelectorConditionalArray + }, nil, "CONDITIONAL_ARRAY"}, + {"unknown mode", func(x *ir.PhysicalExtract) { x.ExecutionMode = "TELEPORT" }, nil, "unknown selector execution mode"}, + {"source bind", func(x *ir.PhysicalExtract) { x.Source = ir.PhysicalValue{BindKey: "doc"} }, nil, "not a bind value"}, + {"source path", func(x *ir.PhysicalExtract) { x.Source.Path = []string{"data"} }, nil, `["payload"]`}, + {"empty resource type", func(x *ir.PhysicalExtract) { x.ResourceType = "" }, nil, "schema resource type"}, + {"original mismatch", func(x *ir.PhysicalExtract) { + x.UnitNormalization.OriginalValue = sql21Steps("valueQuantity", "comparison") + }, nil, "original value must match extract selector"}, + {"original not value", func(x *ir.PhysicalExtract) { + x.Selector = sql21Steps("valueQuantity", "unit") + x.UnitNormalization.OriginalValue = sql21Steps("valueQuantity", "unit") + }, nil, "FHIR Quantity value"}, + {"system not sibling", func(x *ir.PhysicalExtract) { + x.UnitNormalization.SourceSystem = sql21Steps("status") + }, nil, "must be siblings"}, + {"code not sibling", func(x *ir.PhysicalExtract) { + x.UnitNormalization.SourceCode = sql21Steps("status") + }, nil, "must be siblings"}, + {"iterated value selector", func(x *ir.PhysicalExtract) { + steps := []spec.SelectorStep{{Field: "valueQuantity", Iterate: true}, {Field: "value"}} + x.Selector = spec.Selector{Steps: steps} + x.UnitNormalization.OriginalValue = spec.Selector{Steps: steps} + }, nil, "scalar or indexed Quantity"}, + {"iterated value step", func(x *ir.PhysicalExtract) { + steps := []spec.SelectorStep{{Field: "component", Iterate: true}, {Field: "valueQuantity"}, {Field: "value"}} + x.Selector = spec.Selector{Steps: steps} + x.UnitNormalization.OriginalValue = spec.Selector{Steps: steps} + x.UnitNormalization.SourceSystem = spec.Selector{Steps: []spec.SelectorStep{{Field: "component", Iterate: true}, {Field: "valueQuantity"}, {Field: "system"}}} + x.UnitNormalization.SourceCode = spec.Selector{Steps: []spec.SelectorStep{{Field: "component", Iterate: true}, {Field: "valueQuantity"}, {Field: "code"}}} + }, nil, "scalar or indexed Quantity"}, + {"iterated system step", func(x *ir.PhysicalExtract) { + x.UnitNormalization.SourceSystem = spec.Selector{Steps: []spec.SelectorStep{{Field: "valueQuantity"}, {Field: "system", Iterate: true}}} + }, nil, "scalar or indexed Quantity"}, + {"empty selector", func(x *ir.PhysicalExtract) { x.Selector = spec.Selector{} }, nil, "original value must match"}, + {"empty value steps", func(x *ir.PhysicalExtract) { + x.Selector = sql21Steps("valueQuantity", "value") + x.UnitNormalization.OriginalValue = spec.Selector{} + }, nil, "original value must match"}, + {"invalid target", func(x *ir.PhysicalExtract) { + x.UnitNormalization.Target = unit.UnitIdentity{System: "http://unitsofmeasure.org"} + }, nil, "target and approved rules are required"}, + {"empty rules", func(x *ir.PhysicalExtract) { x.UnitNormalization.Rules = nil }, nil, "target and approved rules are required"}, + {"not a quantity owner", func(x *ir.PhysicalExtract) { x.ResourceType = "Patient" }, nil, "owned by a FHIR Quantity"}, + {"unknown rule material", func(x *ir.PhysicalExtract) { + x.UnitNormalization.Rules[0] = unit.UnitConversionRule{ID: "ucum:not-approved", Version: "1", + Kind: unit.UnitConversionLinear, Source: unit.UnitIdentity{System: "http://unitsofmeasure.org", Code: "m"}, + Target: unit.UnitIdentity{System: "http://unitsofmeasure.org", Code: "cm"}, + Dimension: unit.UnitDimension{Length: 1}, Scale: 100} + }, nil, "is not approved"}, + {"tampered scale", func(x *ir.PhysicalExtract) { + x.UnitNormalization.Rules[0].Scale = 101 + }, nil, "does not match the pinned registry entry"}, + {"tampered kind", func(x *ir.PhysicalExtract) { + x.UnitNormalization.Rules[0].Kind = unit.UnitConversionAffine + }, nil, "does not match the pinned registry entry"}, + {"tampered dimension", func(x *ir.PhysicalExtract) { + x.UnitNormalization.Rules[0].Dimension = unit.UnitDimension{Mass: 1} + }, nil, "UNIT_DIMENSION_INCOMPATIBLE"}, + {"tampered policy dimension", func(x *ir.PhysicalExtract) { + x.UnitNormalization.Dimension = unit.UnitDimension{Mass: 1} + }, nil, "UNIT_DIMENSION_INCOMPATIBLE"}, + {"duplicate rule reference", func(x *ir.PhysicalExtract) { + x.UnitNormalization.Rules = append(x.UnitNormalization.Rules, x.UnitNormalization.Rules[1]) + }, nil, "duplicates rule reference"}, + } { + t.Run(caseItem.name, func(t *testing.T) { + projection := sql21UnitProjection(t, caseItem.mutate) + if caseItem.mutate2 != nil { + caseItem.mutate2(&projection) + } + err := sql21ContractError(t, projection, nil) + if !strings.Contains(err.Error(), caseItem.want) { + t.Fatalf("refusal %q does not name %q: %v", err.Error(), caseItem.want, err) + } + if caseItem.name == "unknown rule material" && !errors.Is(err, unit.ErrUnitIdentityUnknown) { + t.Fatalf("unapproved rule material must keep the UNIT_IDENTITY_UNKNOWN sentinel: %v", err) + } + if strings.Contains(caseItem.want, "UNIT_DIMENSION_INCOMPATIBLE") && !errors.Is(err, unit.ErrUnitDimensionIncompatible) { + t.Fatalf("dimension refusal must keep the UNIT_DIMENSION_INCOMPATIBLE sentinel: %v", err) + } + }) + } + for _, caseItem := range []struct { + name string + mutate func(*ir.PhysicalExtract) + mutate2 func(*ir.PhysicalProjection) + want string + }{ + {"scalar posture", func(x *ir.PhysicalExtract) {}, func(p *ir.PhysicalProjection) { + p.Expression.Cardinality = ir.PhysicalScalarCardinality + }, "ARRAY-cardinality EMPTY_ON_NULL"}, + {"preserve-null posture", func(x *ir.PhysicalExtract) {}, func(p *ir.PhysicalProjection) { + p.Expression.NullBehavior = ir.PhysicalPreserveNull + }, "ARRAY-cardinality EMPTY_ON_NULL"}, + {"distinct", func(x *ir.PhysicalExtract) { x.Distinct = true }, nil, "DISTINCT"}, + } { + t.Run("entrypoint/"+caseItem.name, func(t *testing.T) { + projection := sql21UnitProjection(t, caseItem.mutate) + if caseItem.mutate2 != nil { + caseItem.mutate2(&projection) + } + err := sql21RenderError(t, projection) + if !strings.Contains(err.Error(), caseItem.want) { + t.Fatalf("entry refusal %q does not name %q: %v", err.Error(), caseItem.want, err) + } + }) + } + t.Run("unknown source variable", func(t *testing.T) { + projection := sql21UnitProjection(t, nil) + projection.Expression.Extract.Source.Variable = "elsewhere" + projection.Expression.Extract.UnitNormalization.OriginalValue.Steps[0].Field = "valueQuantity" + err := sql21ContractError(t, projection, nil) + if !strings.Contains(err.Error(), "does not bind a readable row") { + t.Fatalf("unknown-variable refusal misdiagnosed: %v", err) + } + }) + t.Run("derived row source", func(t *testing.T) { + err := sql21ContractError(t, sql21UnitProjection(t, nil), &rowScope{derivedValue: "x", derivedJSON: true}) + if !strings.Contains(err.Error(), "does not bind a readable stored row") { + t.Fatalf("derived-row refusal misdiagnosed: %v", err) + } + }) + t.Run("resource type contradiction", func(t *testing.T) { + err := sql21ContractError(t, sql21UnitProjection(t, nil), &rowScope{alias: "r0", resourceType: "Patient"}) + if !strings.Contains(err.Error(), "validates unit selectors against resource type") { + t.Fatalf("resource-type refusal misdiagnosed: %v", err) + } + }) +} + +func sql21ContractError(t *testing.T, projection ir.PhysicalProjection, override *rowScope) error { + t.Helper() + resourceType := "Observation" + if override != nil && override.resourceType != "" { + resourceType = override.resourceType + } + classified, err := classifyExtractionProjections([]ir.PhysicalProjection{projection}, + func(variable string) (string, bool) { + if variable == "root" { + return resourceType, true + } + return "", false + }) + if err != nil { + return err + } + binding, err := ir.NewPostgresStorageBinding("loom_offline") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + scope := rowScope{alias: "r0", resourceType: "Observation"} + if override != nil { + scope = *override + if scope.resourceType == "" { + scope.resourceType = "Observation" + } + } + r := &planRenderer{storage: binding, parameters: map[string]int{}, + rows: map[string]rowScope{"root": scope}} + if _, err := r.renderUnitNormalizedProjection(classified[0]); err != nil { + return err + } + t.Fatalf("unit projection accepted, want refusal") + return nil +} + +func TestSQL21PresenceRefusedOnUnitProjection(t *testing.T) { + projection := sql21UnitProjection(t, nil) + projection.Presence = &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + Paths: [][]string{{"valueQuantity", "value"}}, + } + err := sql21RenderError(t, projection) + if !strings.Contains(err.Error(), "presence metadata") { + t.Fatalf("presence on unit projection accepted or misdiagnosed: %v", err) + } +} + +func TestSQL21ClosedSliceRefusesUnitProjection(t *testing.T) { + binding, err := ir.NewPostgresStorageBinding("loom_offline") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + _, err = RenderSelectorExtraction(sql21Plan(sql21UnitProjection(t, nil)), binding) + if err == nil || !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("closed slice accepted a unit projection: %v", err) + } + if !strings.Contains(err.Error(), "read spine registration (unit_normalization.go)") { + t.Fatalf("closed slice refusal misdiagnosed: %v", err) + } +} + +func TestSQL21OperandPathsKeepLegacyRefusal(t *testing.T) { + unitExtract := sql21UnitExtract(t, nil) + call := &ir.PhysicalCall{Name: "distinct_present_one", TargetKind: "DECIMAL", Args: []ir.PhysicalExpression{{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalEmptyOnNull, + Extract: unitExtract, + }}} + projection := ir.PhysicalProjection{Name: "one_value", Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalCallExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, Call: call, + }} + err := sql21RenderError(t, projection) + if !strings.Contains(err.Error(), "unit normalization") || !strings.Contains(err.Error(), "distinct_present_one") { + t.Fatalf("distinct_present_one unit refusal drifted: %v", err) + } + + scalarUnit := sql21UnitExtract(t, nil) + scalarUnit.Selector = sql21Steps("valueQuantity", "value") + callOperand := &ir.PhysicalCall{Name: "add", Args: []ir.PhysicalExpression{ + {Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, NullBehavior: ir.PhysicalPreserveNull, + Value: &ir.PhysicalValue{Variable: "root", Path: []string{"meta", "versionId"}}}, + {Kind: ir.PhysicalExtractExpression, Cardinality: ir.PhysicalScalarCardinality, NullBehavior: ir.PhysicalPreserveNull, + Extract: scalarUnit}, + }} + operandProjection := ir.PhysicalProjection{Name: "sum", Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalCallExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, Call: callOperand, + }} + const legacy = "unit-normalized extraction is not implemented for PostgreSQL" + err = sql21RenderError(t, operandProjection) + if !strings.Contains(err.Error(), legacy) { + t.Fatalf("operand-path unit refusal must stay byte-identical to %q, got: %v", legacy, err) + } +} + +func TestSQL21UnitAddPassUpperDoubleDerivation(t *testing.T) { + dblMax := new(big.Float).SetPrec(2048) + twoPow971 := new(big.Float).SetPrec(2048).SetInt(new(big.Int).Lsh(big.NewInt(1), 971)) + mantissa := new(big.Int).Lsh(big.NewInt(1), 53) + mantissa.Sub(mantissa, big.NewInt(1)) + dblMax.SetInt(mantissa) + dblMax.Mul(dblMax, twoPow971) + power975 := new(big.Float).SetPrec(2048).SetInt(new(big.Int).Lsh(big.NewInt(1), 975)) + want := new(big.Float).SetPrec(2048).Sub(dblMax, power975) + if want.Text('f', 0) != unitAddPassUpper { + t.Fatalf("unitAddPassUpper drifted from the big.Float DBL_MAX - 2^975 expansion") + } + got, _, err := big.ParseFloat(unitAddPassUpper, 10, 2048, big.ToNearestEven) + if err != nil { + t.Fatalf("parse: %v", err) + } + overBound, _, _ := big.ParseFloat(callDoubleOverflowBound, 10, 2048, big.ToNearestEven) + if got.Cmp(overBound) >= 0 { + t.Fatalf("unitAddPassUpper must sit below the overflow tie") + } + higher := new(big.Float).SetPrec(2048).Sub(dblMax, new(big.Float).SetPrec(2048).SetInt(new(big.Int).Lsh(big.NewInt(1), 974))) + lower := new(big.Float).SetPrec(2048).Sub(dblMax, new(big.Float).SetPrec(2048).SetInt(new(big.Int).Lsh(big.NewInt(1), 976))) + if got.Cmp(higher) >= 0 || got.Cmp(lower) <= 0 { + t.Fatalf("unitAddPassUpper outside the registered margin band") + } +} + +func TestSQL21SelectorStepValidatorDirect(t *testing.T) { + if _, err := unitProjectionSelectorSteps("p", "code", + spec.Selector{Steps: []spec.SelectorStep{{Field: "valueQuantity"}, {Field: "c de"}}}); err == nil || + !strings.Contains(err.Error(), "not renderer-safe") { + t.Fatalf("unsafe segment accepted: %v", err) + } + if _, err := unitProjectionSelectorSteps("p", "code", spec.Selector{}); err == nil || + !strings.Contains(err.Error(), "selector is empty") { + t.Fatalf("empty selector accepted: %v", err) + } + if _, err := unitProjectionSelectorSteps("p", "code", + spec.Selector{Steps: []spec.SelectorStep{{Field: "valueQuantity"}, {Field: "code"}}}); err != nil { + t.Fatalf("legal steps refused: %v", err) + } +} + +func TestSQL21RulesBindingDeterminismAndFinitenessGate(t *testing.T) { + _, _, rules := sql21CentimeterFacts(t) + first, err := unitRulesBindingJSON(rules) + if err != nil { + t.Fatalf("marshal: %v", err) + } + second, err := unitRulesBindingJSON(rules) + if err != nil || string(first) != string(second) { + t.Fatalf("rule binding is not deterministic") + } + tampered := append([]unit.UnitConversionRule(nil), rules...) + tampered[0].Scale = math.Inf(1) + _, tamperErr := unitRulesBindingJSON(tampered) + if tamperErr == nil || !strings.Contains(tamperErr.Error(), "non-finite scale or offset") { + t.Fatalf("non-finite coefficient accepted: %v", tamperErr) + } + if !errors.Is(tamperErr, ErrUnsupportedPostgresPlan) { + t.Fatalf("non-finite refusal must be typed: %v", tamperErr) + } +} + +var ( + sql21FixturesMu sync.Mutex + sql21TrackedFixtures = map[string]bool{} +) + +func runVerifyNamespaceDeltaAudit(prefix string, mu *sync.Mutex, tracked map[string]bool, inner func() int) int { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return inner() + } + escaped := strings.ReplaceAll(prefix, `_`, `\_`) + like := "'" + escaped + "%' ESCAPE '\\'" + snapshot := func(ctx context.Context) ([]string, error) { + pool, err := pgxpool.New(ctx, raw) + if err != nil { + return nil, err + } + defer pool.Close() + queryCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + var names []string + for _, query := range []string{ + `SELECT datname FROM pg_database WHERE datname LIKE ` + like, + `SELECT nspname FROM pg_namespace WHERE nspname LIKE ` + like, + } { + rows, err := pool.Query(queryCtx, query) + if err != nil { + return nil, err + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + return nil, err + } + names = append(names, name) + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, err + } + rows.Close() + } + return names, nil + } + beforeList, err := snapshot(context.Background()) + if err != nil { + fmt.Fprintf(os.Stderr, "%s delta audit: snapshot before run: %v\n", prefix, err) + return 1 + } + before := make(map[string]bool, len(beforeList)) + for _, name := range beforeList { + before[name] = true + } + code := inner() + after, err := snapshot(context.Background()) + if err != nil { + fmt.Fprintf(os.Stderr, "%s delta audit: snapshot after run: %v\n", prefix, err) + code = 1 + } + mu.Lock() + defer mu.Unlock() + for _, name := range after { + if before[name] { + continue + } + if tracked[name] { + fmt.Fprintf(os.Stderr, "%s delta audit: %q was created and tracked by this run and still survives\n", prefix, name) + code = 1 + continue + } + fmt.Fprintf(os.Stderr, "%s delta audit: note — %q was born during this run but is not owned by it (sibling lane?)\n", prefix, name) + } + return code +} + +func sql21RunDeltaAudit(inner func() int) int { + return runVerifyNamespaceDeltaAudit("verify_sql21_", &sql21FixturesMu, sql21TrackedFixtures, inner) +} diff --git a/internal/dataframe/compiler/render/postgres/unit_normalization_tie_test.go b/internal/dataframe/compiler/render/postgres/unit_normalization_tie_test.go new file mode 100644 index 000000000..b7baa9445 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/unit_normalization_tie_test.go @@ -0,0 +1,62 @@ +package postgres + +import ( + "errors" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/unit" +) + +func sql21ScratchAmbiguousRules() []unit.UnitConversionRule { + ucum := "http://unitsofmeasure.org" + target := unit.UnitIdentity{System: ucum, Code: "cm"} + dimension := unit.UnitDimension{Length: 1} + source := unit.UnitIdentity{System: ucum, Code: "inch"} + return []unit.UnitConversionRule{ + {ID: "scratch:pair-a", Version: "1", Kind: unit.UnitConversionLinear, + Source: source, Target: target, Dimension: dimension, Scale: 2.54, Offset: 0}, + {ID: "scratch:pair-b", Version: "1", Kind: unit.UnitConversionLinear, + Source: source, Target: target, Dimension: dimension, Scale: 2.541, Offset: 0}, + } +} + +func TestSQL21ScratchTieCellCraftedFixture(t *testing.T) { + rules := sql21ScratchAmbiguousRules() + for index, rule := range rules { + if err := rule.Validate(rules[0].Target, rules[0].Dimension); err != nil { + t.Fatalf("crafted rule %d must individually validate: %v", index, err) + } + } + err := unitRulesPairAmbiguity(rules) + if err == nil { + t.Fatal("crafted ambiguous pair accepted: the tie cell must refuse before any SQL exists") + } + for _, want := range []string{"two rules for source http://unitsofmeasure.org/inch", "at most one rule per exact source pair", "bind-array order"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("tie refusal %q does not name %q", err.Error(), want) + } + } + _, _, legal := sql21CentimeterFacts(t) + if err := unitRulesPairAmbiguity(legal); err != nil { + t.Fatalf("legal rule set refused: %v", err) + } + if err := unitRulesPairAmbiguity(rules[:1]); err != nil { + t.Fatalf("single crafted rule refused: %v", err) + } +} + +func TestSQL21ScratchTieCellRendererReachabilityBoundary(t *testing.T) { + rules := sql21ScratchAmbiguousRules() + projection := sql21UnitProjection(t, func(x *ir.PhysicalExtract) { + x.UnitNormalization.Rules = rules + }) + err := sql21ContractError(t, projection, nil) + if !errors.Is(err, unit.ErrUnitIdentityUnknown) { + t.Fatalf("crafted ambiguous pair through the renderer = %v, want the registry gate (ErrUnitIdentityUnknown) to answer first", err) + } + if !strings.Contains(err.Error(), "is not approved") { + t.Fatalf("registry gate text drifted: %v", err) + } +} diff --git a/internal/dataframe/compiler/render/postgres/unnest.go b/internal/dataframe/compiler/render/postgres/unnest.go new file mode 100644 index 000000000..64e6b76d0 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/unnest.go @@ -0,0 +1,490 @@ +package postgres + +import ( + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +func RenderUnnest(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + if err := validateUnnestPlanSurface(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return RenderPhysicalPlan(plan, storage) +} + +func validateUnnestPlanSurface(plan ir.PhysicalPlan) error { + sawUnnest, sawReturn := false, false + sawSort, sawLimit := false, false + for index, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalRootScanOp: + if index != 0 { + return fmt.Errorf("%w: the unnest slice leads with exactly one ROOT_SCAN at position 0, got one at %d", ErrUnsupportedPostgresPlan, index) + } + if operation.RootScan == nil { + return fmt.Errorf("%w: operation %d: ROOT_SCAN payload is missing", ErrUnsupportedPostgresPlan, index) + } + if err := validateUnnestRootScan(plan, *operation.RootScan); err != nil { + return err + } + case ir.PhysicalUnnestOp: + sawUnnest = true + if index == 0 { + return fmt.Errorf("%w: operation %d: UNNEST cannot run before the rows it expands", ErrUnsupportedPostgresPlan, index) + } + if sawSort || sawLimit { + return fmt.Errorf("%w: operation %d: UNNEST must precede the execution window; expansion after SORT/LIMIT would window the pre-expansion rows the window contract orders", ErrUnsupportedPostgresPlan, index) + } + if operation.Unnest == nil { + return fmt.Errorf("%w: operation %d: UNNEST payload is missing", ErrUnsupportedPostgresPlan, index) + } + if err := validateUnnestOperationShape(plan, *operation.Unnest); err != nil { + return fmt.Errorf("operation %d: %w", index, err) + } + case ir.PhysicalSortOp: + if operation.Sort == nil { + return fmt.Errorf("%w: operation %d: SORT payload is missing", ErrUnsupportedPostgresPlan, index) + } + if !sawUnnest { + return fmt.Errorf("%w: operation %d: SORT must follow every UNNEST; the window orders final expanded rows, not pre-expansion sources", ErrUnsupportedPostgresPlan, index) + } + if sawSort || sawLimit { + return fmt.Errorf("%w: operation %d: the unnest window admits at most one SORT, before LIMIT", ErrUnsupportedPostgresPlan, index) + } + sawSort = true + case ir.PhysicalLimitOp: + if operation.Limit == nil { + return fmt.Errorf("%w: operation %d: LIMIT payload is missing", ErrUnsupportedPostgresPlan, index) + } + if !sawUnnest { + return fmt.Errorf("%w: operation %d: LIMIT must follow every UNNEST; the window bounds final expanded rows, not pre-expansion sources", ErrUnsupportedPostgresPlan, index) + } + if sawLimit { + return fmt.Errorf("%w: operation %d: the unnest window admits at most one LIMIT", ErrUnsupportedPostgresPlan, index) + } + if !sawSort { + return fmt.Errorf("%w: operation %d: a LIMIT after UNNEST requires the complete stable row identity SORT first", ErrUnsupportedPostgresPlan, index) + } + sawLimit = true + case ir.PhysicalReturnOp: + sawReturn = true + if index != len(plan.Operations)-1 { + return fmt.Errorf("%w: RETURN must be the final operation of the unnest slice", ErrUnsupportedPostgresPlan) + } + if operation.Return == nil { + return fmt.Errorf("%w: the terminal operation must be RETURN with its payload set", ErrUnsupportedPostgresPlan) + } + if err := validateUnnestReturn(*operation.Return); err != nil { + return err + } + default: + return fmt.Errorf("%w: operation %q cannot coexist with UNNEST in the registered slice; filters, traversal operations, and every other shape beyond the SORT/LIMIT window belong to the read spine or a later capability", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + if !sawUnnest { + return fmt.Errorf("%w: the unnest slice requires at least one UNNEST operation between ROOT_SCAN and RETURN", ErrUnsupportedPostgresPlan) + } + if !sawReturn { + return fmt.Errorf("%w: the unnest slice requires a terminal RETURN after its expansions", ErrUnsupportedPostgresPlan) + } + bound := map[string]string{plan.Operations[0].RootScan.Variable: "root scan row"} + for index, operation := range plan.Operations { + if operation.Kind != ir.PhysicalUnnestOp { + continue + } + unnest := operation.Unnest + outputs := []struct { + role, variable string + }{ + {"item", unnest.OutputVariable}, + {"item-discriminator", unnest.HasItemVariable}, + {"ordinality", unnest.Ordinality}, + } + for _, ancestor := range unnest.Ancestors { + outputs = append(outputs, struct { + role, variable string + }{"ancestor", ancestor.Variable}) + } + for _, output := range outputs { + if output.variable == "" { + continue + } + if ownerRole, taken := bound[output.variable]; taken { + return fmt.Errorf("%w: operation %d binds %s variable %q, already bound as the %s; expansion outputs must be unique", + ErrUnsupportedPostgresPlan, index, output.role, output.variable, ownerRole) + } + bound[output.variable] = "operation " + strconv.Itoa(index) + " " + output.role + } + } + return nil +} + +func validateUnnestRootScan(plan ir.PhysicalPlan, scan ir.PhysicalRootScan) error { + if scan.Population != nil || scan.CohortSource != nil || scan.PageCandidateSeed != nil { + return fmt.Errorf("%w: population, cohort, and candidate-seed roots are not registered for PostgreSQL; only the direct root collection scan has a rendered contract", + ErrUnsupportedPostgresPlan) + } + if scan.Variable == "" { + return fmt.Errorf("%w: ROOT_SCAN must name its row variable", ErrUnsupportedPostgresPlan) + } + if scan.CollectionBindKey == "" { + return fmt.Errorf("%w: ROOT_SCAN must name the bind carrying the single resource type", ErrUnsupportedPostgresPlan) + } + if resourceType, ok := plan.BindVars[scan.CollectionBindKey].(string); !ok || strings.TrimSpace(resourceType) == "" { + return fmt.Errorf("%w: root collection bind %q must name one resource type as a non-empty string", + ErrUnsupportedPostgresPlan, scan.CollectionBindKey) + } + return nil +} + +func validateUnnestReturn(ret ir.PhysicalReturn) error { + if len(ret.Projections) == 0 { + return fmt.Errorf("%w: the unnest slice RETURN must project at least one value", ErrUnsupportedPostgresPlan) + } + for _, projection := range ret.Projections { + if strings.TrimSpace(projection.Name) == "" { + return fmt.Errorf("%w: projection names are required output metadata (never SQL text) and must be set", ErrUnsupportedPostgresPlan) + } + if projection.Hidden { + return fmt.Errorf("%w: projection %q is hidden executor-side metadata with no registered PostgreSQL rendering", + ErrUnsupportedPostgresPlan, projection.Name) + } + if projection.Presence != nil || projection.PresenceOutput { + return fmt.Errorf("%w: projection %q carries presence metadata; presence proofs pair with selector extractions, which are outside the unnest slice", + ErrUnsupportedPostgresPlan, projection.Name) + } + if projection.Expression != nil { + if projection.Expression.Kind != ir.PhysicalValueExpression || projection.Expression.Value == nil { + return fmt.Errorf("%w: projection %q uses expression kind %q; the unnest slice emits direct values and VALUE expressions only, and selector projections over expanded items belong to a later capability", + ErrUnsupportedPostgresPlan, projection.Name, string(projection.Expression.Kind)) + } + if projection.Expression.Value.BindKey != "" { + return fmt.Errorf("%w: projection %q must read a row or derived value, not a bind value", ErrUnsupportedPostgresPlan, projection.Name) + } + continue + } + if projection.Value.BindKey != "" { + return fmt.Errorf("%w: projection %q must read a row or derived value, not a bind value", ErrUnsupportedPostgresPlan, projection.Name) + } + if projection.Value.Variable == "" { + return fmt.Errorf("%w: projection %q must read exactly one typed row or derived value", ErrUnsupportedPostgresPlan, projection.Name) + } + } + return nil +} + +func validateUnnestOperationShape(plan ir.PhysicalPlan, unnest ir.PhysicalUnnest) error { + owner := unnest.Owner + if owner.RootVariable == "" || owner.OwnerVariable == "" || unnest.OutputVariable == "" || unnest.HasItemVariable == "" { + return fmt.Errorf("%w: unnest requires root, owner, output, and item-discriminator variables", ErrUnsupportedPostgresPlan) + } + switch unnest.EmptyPolicy { + case ir.PhysicalUnnestExclude, ir.PhysicalUnnestPreserveParent: + case ir.PhysicalUnnestError: + return fmt.Errorf("%w: ERROR unnest policy has no proven PostgreSQL contract yet (AQL's ASSERT-style typed raise has no pure-SQL primitive); it is registered as a capability gap rather than approximated", + ErrUnsupportedPostgresPlan) + default: + return fmt.Errorf("%w: unknown unnest empty policy %q", ErrUnsupportedPostgresPlan, string(unnest.EmptyPolicy)) + } + + previous := owner.RootVariable + for stepIndex, step := range owner.Route { + if strings.TrimSpace(step.OccurrenceID) == "" { + return fmt.Errorf("%w: unnest owner route step %d is missing an occurrence ID", ErrUnsupportedPostgresPlan, stepIndex) + } + if len(step.Scope) != 0 { + return fmt.Errorf("%w: unnest owner route step %d carries %d scope operations, which have no registered PostgreSQL rendering", + ErrUnsupportedPostgresPlan, stepIndex, len(step.Scope)) + } + switch step.Traversal.Direction { + case ir.PhysicalOutbound, ir.PhysicalInbound: + default: + return fmt.Errorf("%w: unnest owner route step %d has direction %q, which has no proven PostgreSQL contract", + ErrUnsupportedPostgresPlan, stepIndex, string(step.Traversal.Direction)) + } + if step.Traversal.Strategy != "" && step.Traversal.Strategy != ir.PhysicalTraversalNative { + return fmt.Errorf("%w: unnest owner route step %d has strategy %q; native typed-identity joins are the only registered shape", + ErrUnsupportedPostgresPlan, stepIndex, string(step.Traversal.Strategy)) + } + if step.Traversal.SourceVariable != previous { + return fmt.Errorf("%w: unnest owner route step %d starts at %q, want %q", + ErrUnsupportedPostgresPlan, stepIndex, step.Traversal.SourceVariable, previous) + } + previous = step.Traversal.TargetVariable + } + if previous != owner.OwnerVariable { + return fmt.Errorf("%w: unnest route terminates at %q, not owner %q", ErrUnsupportedPostgresPlan, previous, owner.OwnerVariable) + } + + expression := unnest.Expression + if expression.Kind != ir.PhysicalExtractExpression || expression.Extract == nil { + return fmt.Errorf("%w: unnest source must be an EXTRACT selector over the owner row document; value and other expression sources belong to a later capability", + ErrUnsupportedPostgresPlan) + } + if expression.Cardinality != ir.PhysicalArrayCardinality { + return fmt.Errorf("%w: unnest source expression must be array-valued, got %q", ErrUnsupportedPostgresPlan, string(expression.Cardinality)) + } + if expression.NullBehavior != ir.PhysicalPreserveNull { + return fmt.Errorf("%w: unnest source null behavior %q has no registered PostgreSQL form; the expansion normalizes null-or-missing sources per the AQL contract and scalar null behaviors belong to value projections", + ErrUnsupportedPostgresPlan, string(expression.NullBehavior)) + } + extract := expression.Extract + switch extract.ExecutionMode { + case "", ir.PhysicalSelectorGeneric: + case ir.PhysicalSelectorDirectScalar: + return fmt.Errorf("%w: DIRECT_SCALAR selector mode contradicts a repeated terminal step", ErrUnsupportedPostgresPlan) + case ir.PhysicalSelectorConditionalArray: + return fmt.Errorf("%w: CONDITIONAL_ARRAY selector mode requires conditional-array rendering, which is not registered for PostgreSQL", + ErrUnsupportedPostgresPlan) + default: + return fmt.Errorf("%w: unknown selector execution mode %q", ErrUnsupportedPostgresPlan, string(extract.ExecutionMode)) + } + if extract.Prepared != nil { + return fmt.Errorf("%w: prepared-set selector references are not implemented for PostgreSQL", ErrUnsupportedPostgresPlan) + } + if len(extract.Fallbacks) != 0 { + return fmt.Errorf("%w: ordered fallback selectors are a separate capability", ErrUnsupportedPostgresPlan) + } + if extract.UnitNormalization != nil { + return fmt.Errorf("%w: unit-normalized extraction is not implemented for PostgreSQL", ErrUnsupportedPostgresPlan) + } + if extract.Distinct { + return fmt.Errorf("%w: DISTINCT expansion is a separate capability: deduplication would erase the duplicate-item preservation this operation exists to provide", + ErrUnsupportedPostgresPlan) + } + if extract.Selector.Filter != nil { + return fmt.Errorf("%w: selector CONTAINS filters are not implemented for PostgreSQL", ErrUnsupportedPostgresPlan) + } + if extract.Source.BindKey != "" { + return fmt.Errorf("%w: the unnest source must be a row document, not a bind value", ErrUnsupportedPostgresPlan) + } + if len(extract.Source.Path) != 1 || extract.Source.Path[0] != "payload" { + return fmt.Errorf("%w: the unnest source must be the owner row document at [\"payload\"], the exact source shape lowering emits (lower/row_expansion.go); got %v", + ErrUnsupportedPostgresPlan, extract.Source.Path) + } + if extract.Source.Variable != owner.OwnerVariable { + return fmt.Errorf("%w: unnest source reads variable %q, want the exact owner occurrence %q", + ErrUnsupportedPostgresPlan, extract.Source.Variable, owner.OwnerVariable) + } + if strings.TrimSpace(extract.ResourceType) == "" || extract.ResourceType != owner.ResourceType { + return fmt.Errorf("%w: unnest selector schema type %q must name the owner resource type %q", + ErrUnsupportedPostgresPlan, extract.ResourceType, owner.ResourceType) + } + + steps := extract.Selector.Steps + if len(steps) == 0 { + return fmt.Errorf("%w: the unnest selector is empty", ErrUnsupportedPostgresPlan) + } + for stepIndex, step := range steps { + if step.Field == "" { + return fmt.Errorf("%w: unnest selector step %d has no property", ErrUnsupportedPostgresPlan, stepIndex) + } + if step.Index != nil { + return fmt.Errorf("%w: unnest selector step %d is indexed; positional selection belongs to a later capability", + ErrUnsupportedPostgresPlan, stepIndex) + } + if !payloadPathSegmentPattern.MatchString(step.Field) { + return fmt.Errorf("%w: unnest selector step %q is not renderer-safe", ErrUnsupportedPostgresPlan, step.Field) + } + } + if !steps[len(steps)-1].Iterate { + return fmt.Errorf("%w: the unnest selector must end with a repeated ([]) step, so expansion has a defined terminal cardinality", + ErrUnsupportedPostgresPlan) + } + seenAncestorSteps := make(map[int]bool, len(unnest.Ancestors)) + for ancestorIndex, ancestor := range unnest.Ancestors { + if ancestor.StepIndex < 0 || ancestor.StepIndex >= len(steps)-1 || !steps[ancestor.StepIndex].Iterate { + return fmt.Errorf("%w: unnest ancestor %d does not identify a repeated prefix selector step", + ErrUnsupportedPostgresPlan, ancestorIndex) + } + if seenAncestorSteps[ancestor.StepIndex] { + return fmt.Errorf("%w: unnest repeats ancestor selector step %d", ErrUnsupportedPostgresPlan, ancestor.StepIndex) + } + seenAncestorSteps[ancestor.StepIndex] = true + } + return nil +} + +func (r *planRenderer) renderUnnest(plan ir.PhysicalPlan, unnest ir.PhysicalUnnest) error { + if r.unnestSeq == 0 { + if err := validateUnnestPlanSurface(plan); err != nil { + return err + } + } + if err := validateUnnestOperationShape(plan, unnest); err != nil { + return err + } + + for stepIndex, step := range unnest.Owner.Route { + if err := r.renderTraversal(plan, step.Traversal, true); err != nil { + return fmt.Errorf("%w: unnest owner route step %d: %v", ErrUnsupportedPostgresPlan, stepIndex, err) + } + } + ownerScope, ok := r.rows[unnest.Owner.OwnerVariable] + if !ok || ownerScope.derivedValue != "" { + return fmt.Errorf("%w: unnest owner %q does not bind a readable stored row", ErrUnsupportedPostgresPlan, unnest.Owner.OwnerVariable) + } + if ownerScope.resourceType != unnest.Owner.ResourceType { + return fmt.Errorf("%w: unnest owner %q is scanned as %q, not the declared owner type %q", + ErrUnsupportedPostgresPlan, unnest.Owner.OwnerVariable, ownerScope.resourceType, unnest.Owner.ResourceType) + } + + type expansion struct { + path []string + iterateStep int + expansionAlias string + valueColumn string + } + var expansions []expansion + current := make([]string, 0, 4) + for stepIndex, step := range unnest.Expression.Extract.Selector.Steps { + current = append(current, step.Field) + if step.Iterate { + expansions = append(expansions, expansion{ + path: append([]string(nil), current...), + iterateStep: stepIndex, + }) + current = nil + } + } + + r.unnestSeq++ + lateralAlias := "u" + strconv.Itoa(r.unnestSeq) + + var from strings.Builder + orderByOrds := make([]string, 0, len(expansions)) + ancestorColumns := make([]string, len(unnest.Ancestors)) + base := r.documentColumn(ownerScope.alias) + terminal := len(expansions) - 1 + for index, stepExpansion := range expansions { + extraction, err := r.documentPathFromExpression(base, stepExpansion.path) + if err != nil { + return fmt.Errorf("unnest expansion %d: %w", index, err) + } + expansionAlias := lateralAlias + "_e" + strconv.Itoa(index) + valueColumn := "value" + var sourceForm string + if index == terminal && len(unnest.Ancestors) == 0 { + valueColumn = "item" + sourceForm = unnestSingleLayerForm(extraction) + } else { + if index == terminal { + valueColumn = "item" + } + sourceForm = unnestNullToEmptyForm(extraction) + } + fragment := "jsonb_array_elements(" + sourceForm + ") WITH ORDINALITY AS " + + expansionAlias + "(" + valueColumn + ", ord)" + if index == 0 { + from.WriteString("FROM " + fragment) + } else { + from.WriteString(" CROSS JOIN LATERAL " + fragment) + } + expansions[index].expansionAlias = expansionAlias + expansions[index].valueColumn = valueColumn + orderByOrds = append(orderByOrds, expansionAlias+".ord") + base = expansionAlias + ".value" + } + for ancestorIndex, ancestor := range unnest.Ancestors { + var captured string + for _, stepExpansion := range expansions { + if stepExpansion.iterateStep == ancestor.StepIndex { + captured = stepExpansion.expansionAlias + "." + stepExpansion.valueColumn + break + } + } + if captured == "" { + return fmt.Errorf("%w: unnest ancestor %d was not captured by an expansion", ErrUnsupportedPostgresPlan, ancestorIndex) + } + ancestorColumns[ancestorIndex] = captured + " AS ancestor_" + strconv.Itoa(ancestorIndex) + } + + rowNumber := "ROW_NUMBER() OVER (ORDER BY " + strings.Join(orderByOrds, ", ") + ") AS rn" + selectList := expansions[terminal].expansionAlias + "." + expansions[terminal].valueColumn + " AS item" + for _, column := range ancestorColumns { + if column != "" { + selectList += ", " + column + } + } + selectList += ", " + rowNumber + terminalItem := expansions[terminal].expansionAlias + "." + expansions[terminal].valueColumn + subquery := "SELECT " + selectList + " " + from.String() + + " WHERE jsonb_typeof(" + terminalItem + ") <> 'null'" + + join := "CROSS JOIN LATERAL (" + subquery + ") AS " + lateralAlias + if unnest.EmptyPolicy == ir.PhysicalUnnestPreserveParent { + join = "LEFT JOIN LATERAL (" + subquery + ") AS " + lateralAlias + " ON true" + } + r.laterals = append(r.laterals, join) + r.lateralAliases = append(r.lateralAliases, lateralAlias) + r.window.hasUnnest = true + + if err := r.registerDerivedValue(unnest.OutputVariable, lateralAlias+".item", true); err != nil { + return err + } + if unnest.Ordinality != "" { + if err := r.registerDerivedValue(unnest.Ordinality, lateralAlias+".rn - 1", false); err != nil { + return err + } + } + if err := r.registerDerivedValue(unnest.HasItemVariable, lateralAlias+".rn IS NOT NULL", false); err != nil { + return err + } + for ancestorIndex, ancestor := range unnest.Ancestors { + if err := r.registerDerivedValue(ancestor.Variable, lateralAlias+".ancestor_"+strconv.Itoa(ancestorIndex), true); err != nil { + return err + } + } + return nil +} + +func (r *planRenderer) registerDerivedValue(variable, expression string, jsonValue bool) error { + if _, taken := r.rows[variable]; taken { + return fmt.Errorf("%w: unnest output variable %q is already bound; expansion output variables must be unique because they name distinct SQL outputs", + ErrUnsupportedPostgresPlan, variable) + } + r.rows[variable] = rowScope{derivedValue: expression, derivedJSON: jsonValue} + return nil +} + +func (r *planRenderer) resolveDerivedValue(value ir.PhysicalValue, scope rowScope) (string, error) { + if len(value.Path) == 0 { + return "(" + scope.derivedValue + ")", nil + } + if !scope.derivedJSON { + return "", fmt.Errorf("%w: value %q reads path %v of computed variable %q, which is not a JSON document", + ErrUnsupportedPostgresPlan, value.Variable, value.Path, value.Variable) + } + expression, err := r.documentPathFromExpression(scope.derivedValue, value.Path) + if err != nil { + return "", err + } + return "(" + expression + ")", nil +} + +const unnestFalsyArms = " WHEN (%s) = 'false'::jsonb OR (%s) = '0'::jsonb OR (%s) = '\"\"'::jsonb THEN '[]'::jsonb" + +func unnestNullToEmptyForm(extraction string) string { + return "CASE" + + " WHEN (" + extraction + ") IS NULL OR jsonb_typeof((" + extraction + ")) = 'null' THEN '[]'::jsonb" + + " WHEN jsonb_typeof((" + extraction + ")) = 'array' THEN (" + extraction + ")" + + unnestFalsyArmsWith(extraction) + + " ELSE (" + extraction + ") END" +} + +func unnestSingleLayerForm(extraction string) string { + return "CASE" + + " WHEN (" + extraction + ") IS NULL OR jsonb_typeof((" + extraction + ")) = 'null' THEN '[]'::jsonb" + + " WHEN jsonb_typeof((" + extraction + ")) = 'array' THEN (" + extraction + ")" + + unnestFalsyArmsWith(extraction) + + " ELSE jsonb_build_array((" + extraction + ")) END" +} + +func unnestFalsyArmsWith(extraction string) string { + return fmt.Sprintf(unnestFalsyArms, extraction, extraction, extraction) +} diff --git a/internal/dataframe/compiler/render/postgres/unnest_test.go b/internal/dataframe/compiler/render/postgres/unnest_test.go new file mode 100644 index 000000000..e7e279a4b --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/unnest_test.go @@ -0,0 +1,1626 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + control "github.com/calypr/loom/internal/backends/postgres/control" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/render/aql" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +func arrayExtract(variable, resourceType string, steps ...spec.SelectorStep) ir.PhysicalExpression { + return ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: variable, Path: []string{"payload"}}, + ResourceType: resourceType, + Selector: spec.Selector{Steps: steps}, + }, + } +} + +func identifierExpansion(policy ir.PhysicalUnnestEmptyPolicy) ir.PhysicalUnnest { + return ir.PhysicalUnnest{ + Owner: ir.PhysicalUnnestOwner{OccurrenceID: "root", ResourceType: "Patient", RootVariable: "root", OwnerVariable: "root"}, + OutputVariable: "item", Ordinality: "position", HasItemVariable: "has_item", + Expression: arrayExtract("root", "Patient", spec.SelectorStep{Field: "identifier", Iterate: true}), + EmptyPolicy: policy, + } +} + +func codingExpansion(policy ir.PhysicalUnnestEmptyPolicy) ir.PhysicalUnnest { + return ir.PhysicalUnnest{ + Owner: ir.PhysicalUnnestOwner{OccurrenceID: "obs-owner", ResourceType: "Observation", RootVariable: "root", OwnerVariable: "root"}, + OutputVariable: "item", Ordinality: "position", HasItemVariable: "has_item", + Expression: arrayExtract("root", "Observation", + spec.SelectorStep{Field: "component", Iterate: true}, + spec.SelectorStep{Field: "code"}, + spec.SelectorStep{Field: "coding", Iterate: true}), + Ancestors: []ir.PhysicalUnnestAncestor{{StepIndex: 0, Variable: "component_item"}}, + EmptyPolicy: policy, + } +} + +func unnestOp(unnest ir.PhysicalUnnest) ir.PhysicalOperation { + return ir.PhysicalOperation{Kind: ir.PhysicalUnnestOp, Unnest: &unnest} +} + +func valueProjection(name, variable string, path ...string) ir.PhysicalProjection { + return ir.PhysicalProjection{Name: name, Value: ir.PhysicalValue{Variable: variable, Path: path}} +} + +func expressionProjection(name, variable string, path ...string) ir.PhysicalProjection { + value := ir.PhysicalValue{Variable: variable, Path: path} + return ir.PhysicalProjection{Name: name, Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, Value: &value, + }} +} + +func unnestPlan(projections ...ir.PhysicalProjection) ir.PhysicalPlan { + plan := basePlan() + unnest := identifierExpansion(ir.PhysicalUnnestExclude) + plan.Operations = []ir.PhysicalOperation{plan.Operations[0], unnestOp(unnest), + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}}} + return plan +} + +func expansionProjections() []ir.PhysicalProjection { + return []ir.PhysicalProjection{ + valueProjection("logical_id", "root", "logical_id"), + valueProjection("item", "item"), + valueProjection("position", "position"), + valueProjection("has_item", "has_item"), + } +} + +func TestUnnestInnerRendersCrossJoinLateralWithZeroBasedStableOrdinality(t *testing.T) { + rendered := renderOK(t, unnestPlan(expansionProjections()...)) + sql := rendered.SQL + + for _, want := range []string{ + `CROSS JOIN LATERAL (SELECT u1_e0.item AS item, ROW_NUMBER() OVER (ORDER BY u1_e0.ord) AS rn FROM jsonb_array_elements(`, + `) WITH ORDINALITY AS u1_e0(item, ord) WHERE jsonb_typeof(u1_e0.item) <> 'null') AS u1`, + `ORDER BY r0."resource_seq" ASC, u1.rn ASC`, + `(u1.item) AS "c1"`, + `(u1.rn - 1) AS "c2"`, + `(u1.rn IS NOT NULL) AS "c3"`, + `IS NULL OR jsonb_typeof((r0."payload" #> $5::text[])) = 'null' THEN '[]'::jsonb`, + `= 'array' THEN (r0."payload" #> $5::text[])`, + `WHEN (r0."payload" #> $5::text[]) = 'false'::jsonb OR (r0."payload" #> $5::text[]) = '0'::jsonb OR (r0."payload" #> $5::text[]) = '""'::jsonb THEN '[]'::jsonb`, + `ELSE jsonb_build_array((r0."payload" #> $5::text[])) END`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("SQL missing %q:\n%s", want, sql) + } + } + if strings.Contains(sql, "LEFT JOIN") { + t.Fatalf("INNER rendered OUTER padding:\n%s", sql) + } + if strings.Contains(sql, "identifier") { + t.Fatalf("selector field leaked into SQL text: %s", sql) + } + if _, err := command.NewRenderedPostgresPhysicalPlan(sql, rendered.Args, rendered.OutputColumns); err != nil { + t.Fatalf("rendered plan contract violated: %v", err) + } + if !reflect.DeepEqual(rendered.OutputColumns, []string{"logical_id", "item", "position", "has_item"}) { + t.Fatalf("output columns = %v", rendered.OutputColumns) + } + + if len(rendered.Args) != 5 { + t.Fatalf("args = %v, want four scope args plus one path", rendered.Args) + } + if got := rendered.Args[4]; got.Type != command.PostgreSQLParameterText || got.Value != "{identifier}" { + t.Fatalf("path argument = %#v, want text {identifier}", got) + } + + viaEntry, err := RenderUnnest(unnestPlan(expansionProjections()...), mustBinding(t)) + if err != nil { + t.Fatalf("RenderUnnest: %v", err) + } + if viaEntry.SQL != rendered.SQL || !reflect.DeepEqual(viaEntry.Args, rendered.Args) { + t.Fatalf("entry and dispatch disagree:\n%s\n%v\nvs\n%s\n%v", viaEntry.SQL, viaEntry.Args, rendered.SQL, rendered.Args) + } +} + +func TestUnnestOuterPadsWithLeftJoinLateralOnTrue(t *testing.T) { + plan := basePlan() + unnest := identifierExpansion(ir.PhysicalUnnestPreserveParent) + plan.Operations = []ir.PhysicalOperation{plan.Operations[0], unnestOp(unnest), + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: expansionProjections()}}} + rendered := renderOK(t, plan) + sql := rendered.SQL + if !strings.Contains(sql, `LEFT JOIN LATERAL (SELECT u1_e0.item AS item, ROW_NUMBER() OVER (ORDER BY u1_e0.ord) AS rn FROM jsonb_array_elements(`) || + !strings.Contains(sql, `WHERE jsonb_typeof(u1_e0.item) <> 'null') AS u1 ON true`) { + t.Fatalf("OUTER did not render LEFT JOIN LATERAL ... ON true with the terminal null-item filter:\n%s", sql) + } + if strings.Contains(sql, "CROSS JOIN LATERAL") { + t.Fatalf("OUTER must not use CROSS JOIN LATERAL for its outer boundary:\n%s", sql) + } +} + +func TestUnnestNestedRepeatedArraysChainExpansionsAndCaptureAncestors(t *testing.T) { + plan := basePlan() + plan.BindVars["root_collection"] = "Observation" + unnest := codingExpansion(ir.PhysicalUnnestExclude) + projections := append(expansionProjections(), valueProjection("component", "component_item")) + plan.Operations = []ir.PhysicalOperation{plan.Operations[0], unnestOp(unnest), + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}}} + rendered := renderOK(t, plan) + sql := rendered.SQL + + for _, want := range []string{ + `FROM jsonb_array_elements(CASE WHEN (r0."payload" #> $5::text[]) IS NULL OR jsonb_typeof((r0."payload" #> $5::text[])) = 'null' THEN '[]'::jsonb WHEN jsonb_typeof((r0."payload" #> $5::text[])) = 'array' THEN (r0."payload" #> $5::text[]) WHEN (r0."payload" #> $5::text[]) = 'false'::jsonb OR (r0."payload" #> $5::text[]) = '0'::jsonb OR (r0."payload" #> $5::text[]) = '""'::jsonb THEN '[]'::jsonb ELSE (r0."payload" #> $5::text[]) END) WITH ORDINALITY AS u1_e0(value, ord)`, + `CROSS JOIN LATERAL jsonb_array_elements(CASE WHEN (u1_e0.value #> $6::text[]) IS NULL OR jsonb_typeof((u1_e0.value #> $6::text[])) = 'null' THEN '[]'::jsonb WHEN jsonb_typeof((u1_e0.value #> $6::text[])) = 'array' THEN (u1_e0.value #> $6::text[]) WHEN (u1_e0.value #> $6::text[]) = 'false'::jsonb OR (u1_e0.value #> $6::text[]) = '0'::jsonb OR (u1_e0.value #> $6::text[]) = '""'::jsonb THEN '[]'::jsonb ELSE (u1_e0.value #> $6::text[]) END) WITH ORDINALITY AS u1_e1(item, ord)`, + `WHERE jsonb_typeof(u1_e1.item) <> 'null'`, + `SELECT u1_e1.item AS item, u1_e0.value AS ancestor_0, ROW_NUMBER() OVER (ORDER BY u1_e0.ord, u1_e1.ord) AS rn`, + `ORDER BY r0."resource_seq" ASC, u1.rn ASC`, + `(u1.ancestor_0) AS "c4"`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("SQL missing %q:\n%s", want, sql) + } + } + var pathCount int + for _, argument := range rendered.Args { + if text, ok := argument.Value.(string); ok && (text == "{component}" || text == "{code,coding}") { + pathCount++ + } + } + if pathCount != 2 { + t.Fatalf("path parameters = %v, want exactly {component} and {code,coding}: %#v", pathCount, rendered.Args) + } + for _, leaked := range []string{"component", "coding"} { + if strings.Contains(sql, leaked) { + t.Fatalf("selector field %q leaked into SQL text: %s", leaked, sql) + } + } +} + +func TestUnnestDerivedItemPathAndValueExpressionProjectAsBoundPaths(t *testing.T) { + plan := basePlan() + unnest := identifierExpansion(ir.PhysicalUnnestExclude) + plan.Operations = []ir.PhysicalOperation{plan.Operations[0], unnestOp(unnest), + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + valueProjection("logical_id", "root", "logical_id"), + valueProjection("item_system", "item", "system"), + expressionProjection("item_value", "item", "value"), + valueProjection("position", "position"), + valueProjection("has_item", "has_item"), + }}}} + rendered := renderOK(t, plan) + sql := rendered.SQL + + for _, want := range []string{ + `((u1.item #> $6::text[])) AS "c1"`, + `((u1.item #> $7::text[])) AS "c2"`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("SQL missing %q:\n%s", want, sql) + } + } + for _, leaked := range []string{`"system"`, `"value"`, "identifier"} { + if strings.Contains(sql, leaked) { + t.Fatalf("selector field %q leaked into SQL text: %s", leaked, sql) + } + } + if len(rendered.Args) != 7 { + t.Fatalf("args = %v, want four scope args plus {identifier}, {system}, {value}", rendered.Args) + } + for index, want := range []string{"{identifier}", "{system}", "{value}"} { + if got := rendered.Args[4+index]; got.Type != command.PostgreSQLParameterText || got.Value != want { + t.Fatalf("path argument %d = %#v, want text %s", 4+index, got, want) + } + } + viaEntry, err := RenderUnnest(plan, mustBinding(t)) + if err != nil { + t.Fatalf("RenderUnnest: %v", err) + } + if viaEntry.SQL != rendered.SQL || !reflect.DeepEqual(viaEntry.Args, rendered.Args) { + t.Fatalf("entry and dispatch disagree:\n%s\nvs\n%s", viaEntry.SQL, rendered.SQL) + } +} + +func TestUnnestTwoExpansionsAssembleInPlanOrder(t *testing.T) { + plan := basePlan() + first := identifierExpansion(ir.PhysicalUnnestExclude) + second := ir.PhysicalUnnest{ + Owner: ir.PhysicalUnnestOwner{OccurrenceID: "root", ResourceType: "Patient", + RootVariable: "root", OwnerVariable: "root"}, + OutputVariable: "telecom_item", Ordinality: "telecom_position", HasItemVariable: "telecom_has_item", + Expression: arrayExtract("root", "Patient", spec.SelectorStep{Field: "telecom", Iterate: true}), + EmptyPolicy: ir.PhysicalUnnestExclude, + } + plan.Operations = []ir.PhysicalOperation{plan.Operations[0], unnestOp(first), unnestOp(second), + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + valueProjection("logical_id", "root", "logical_id"), + valueProjection("identifier_value", "item", "value"), + valueProjection("telecom_system", "telecom_item", "system"), + expressionProjection("telecom_value", "telecom_item", "value"), + valueProjection("position", "position"), + valueProjection("telecom_position", "telecom_position"), + }}}} + rendered := renderOK(t, plan) + sql := rendered.SQL + + for _, want := range []string{ + `) AS u1 CROSS JOIN LATERAL (SELECT u2_e0.item AS item, ROW_NUMBER() OVER (ORDER BY u2_e0.ord) AS rn FROM jsonb_array_elements(`, + `WHERE jsonb_typeof(u1_e0.item) <> 'null'`, + `WHERE jsonb_typeof(u2_e0.item) <> 'null'`, + `ORDER BY r0."resource_seq" ASC, u1.rn ASC, u2.rn ASC`, + `((u1.item #> $7::text[])) AS "c1"`, + `((u2.item #> $8::text[])) AS "c2"`, + `((u2.item #> $7::text[])) AS "c3"`, + } { + if !strings.Contains(sql, want) { + t.Fatalf("SQL missing %q:\n%s", want, sql) + } + } + for _, leaked := range []string{"identifier", "telecom", `"system"`, `"value"`} { + if strings.Contains(sql, leaked) { + t.Fatalf("selector field or path %q leaked into SQL text: %s", leaked, sql) + } + } + if len(rendered.Args) != 8 { + t.Fatalf("args = %v, want four scope args plus {identifier}, {telecom}, {value}, {system}", rendered.Args) + } + for index, want := range []string{"{identifier}", "{telecom}", "{value}", "{system}"} { + if got := rendered.Args[4+index]; got.Type != command.PostgreSQLParameterText || got.Value != want { + t.Fatalf("path argument %d = %#v, want text %s", 4+index, got, want) + } + } + if strings.Contains(sql, "LEFT JOIN") { + t.Fatalf("INNER multi-expansion rendered OUTER padding:\n%s", sql) + } +} + +func TestUnnestOwnerRouteRendersTypedIdentityJoinBeforeLaterals(t *testing.T) { + plan := basePlan() + plan.BindVars["edge_collection"] = "fhir_edge" + plan.BindVars["edge_label"] = "has-observation" + plan.BindVars["target_type"] = "Observation" + unnest := codingExpansion(ir.PhysicalUnnestPreserveParent) + unnest.Owner = ir.PhysicalUnnestOwner{ + OccurrenceID: "obs-owner", ResourceType: "Observation", RootVariable: "root", OwnerVariable: "obs", + Route: []ir.PhysicalUnnestRouteStep{{OccurrenceID: "obs-owner", Traversal: ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "obs", EdgeVariable: "obs_edge", + Direction: ir.PhysicalOutbound, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: "edge_label", TargetTypeBindKey: "target_type", + EdgeTargetTypeField: "to_type", Strategy: ir.PhysicalTraversalNative, + }}}, + } + unnest.Expression = arrayExtract("obs", "Observation", + spec.SelectorStep{Field: "component", Iterate: true}, + spec.SelectorStep{Field: "code"}, + spec.SelectorStep{Field: "coding", Iterate: true}) + projections := []ir.PhysicalProjection{ + valueProjection("patient_id", "root", "logical_id"), + valueProjection("observation_id", "obs", "logical_id"), + valueProjection("item", "item"), + valueProjection("position", "position"), + valueProjection("has_item", "has_item"), + valueProjection("component", "component_item"), + } + plan.Operations = []ir.PhysicalOperation{plan.Operations[0], unnestOp(unnest), + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}}} + rendered := renderOK(t, plan) + sql := rendered.SQL + + routeAt := strings.Index(sql, "INNER JOIN") + lateralAt := strings.Index(sql, "JOIN LATERAL") + orderAt := strings.LastIndex(sql, "ORDER BY") + if routeAt < 0 || lateralAt < 0 || !(routeAt < lateralAt && lateralAt < orderAt) { + t.Fatalf("storage join and lateral must sit between FROM and ORDER BY in that order:\n%s", sql) + } + if strings.Contains(sql, ") AS r1 ON TRUE") { + t.Fatalf("owner route must not render the OPTIONAL parent-preserving lateral:\n%s", sql) + } + if !strings.Contains(sql, `ORDER BY r0."resource_seq" ASC, e1."edge_seq" ASC, u1.rn ASC`) { + t.Fatalf("route edge_seq and expansion rn must both extend the total order:\n%s", sql) + } + if !strings.Contains(sql, `(x1."payload" #>`) { + t.Fatalf("owner-route expansion must extract from the joined owner row:\n%s", sql) + } + for _, leaked := range []string{"has-observation"} { + if strings.Contains(sql, leaked) { + t.Fatalf("route bind value %q leaked into SQL text: %s", leaked, sql) + } + } +} + +func TestUnnestRenderIsDeterministicAndDoesNotAliasPlanData(t *testing.T) { + plan := unnestPlan(expansionProjections()...) + first := renderOK(t, plan) + second := renderOK(t, plan) + if first.SQL != second.SQL || !reflect.DeepEqual(first.Args, second.Args) || !reflect.DeepEqual(first.OutputColumns, second.OutputColumns) { + t.Fatalf("repeated render changed the statement") + } + capturedSQL, capturedArgs := first.SQL, first.Args + plan.BindVars["project"] = "mutated" + plan.BindVars["dataset_generation"] = "mutated" + plan.BindVars["auth_resource_paths"].([]string)[0] = "MUTATED" + plan.BindVars["root_collection"] = "Observation" + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); err == nil { + t.Fatalf("render after mutation accepted a resource-type contradiction") + } + if first.SQL != capturedSQL || !reflect.DeepEqual(first.Args, capturedArgs) { + t.Fatalf("render aliased plan data") + } +} + +func aqlScopeOperations(variable string) []ir.PhysicalOperation { + return []ir.PhysicalOperation{ + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: variable, Path: []string{"project"}}, + Right: &ir.PhysicalValue{BindKey: "project"}}}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: variable, Path: []string{"dataset_generation"}}, + Right: &ir.PhysicalValue{BindKey: "dataset_generation"}}}}, + {Kind: ir.PhysicalDerivedLetOp, DerivedLet: &ir.PhysicalDerivedLet{ + Variable: "scope_auth", Operator: "AUTH_RESOURCE_PATH_ALLOWED", + Inputs: []ir.PhysicalValue{ + {Variable: variable, Path: []string{"auth_resource_path"}}, + {BindKey: "auth_resource_paths"}, + {BindKey: "auth_resource_paths_unrestricted"}, + }}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "scope_auth"}, + Right: &ir.PhysicalValue{BindKey: "scope_allowed"}}}}, + } +} + +func TestUnnestCommandDispatchStaysFailClosedForPostgreSQL(t *testing.T) { + plan := unnestPlan(expansionProjections()...) + unnestOpCopy := plan.Operations[1] + unnestOpCopy.Source = ir.PhysicalSource{ResourceType: "Patient", SemanticField: "expand"} + returnOperation := plan.Operations[2] + operations := append([]ir.PhysicalOperation{plan.Operations[0]}, aqlScopeOperations("root")...) + operations = append(operations, unnestOpCopy, returnOperation) + plan.Operations = operations + rendered, err := aql.RenderPhysicalPlan(plan) + if err != nil { + t.Fatalf("aql render: %v", err) + } + if _, err := command.NewAQLCommand(rendered.Query, rendered.BindVars); err != nil { + t.Fatalf("AQL command dispatch: %v", err) + } + unregistered := command.CompiledCommand{} + if _, _, err := unregistered.PostgreSQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("unregistered command dispatch error = %v, want not-registered while the planner gate stays fail-closed", err) + } + if _, _, err := unregistered.AQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("unregistered command dispatch error = %v, want not-registered on the AQL side too", err) + } + + if _, err := RenderUnnest(unnestPlan(expansionProjections()...), mustBinding(t)); err != nil { + t.Fatalf("RenderUnnest on the registered shape: %v", err) + } +} + +func TestUnnestZeroValueBindingFailsLikeSiblings(t *testing.T) { + if _, err := RenderUnnest(unnestPlan(expansionProjections()...), ir.PostgresStorageBinding{}); !errors.Is(err, ir.ErrUnregisteredPostgresIdentifier) { + t.Fatalf("zero binding error = %v, want ErrUnregisteredPostgresIdentifier", err) + } +} + +func unnestSurfaceRejectCases() []struct { + name string + mutate func(*ir.PhysicalPlan) +} { + mutate := func(edit func(*ir.PhysicalPlan, *ir.PhysicalUnnest)) func(*ir.PhysicalPlan) { + return func(plan *ir.PhysicalPlan) { + edit(plan, plan.Operations[1].Unnest) + } + } + return []struct { + name string + mutate func(*ir.PhysicalPlan) + }{ + {"no unnest", func(p *ir.PhysicalPlan) { p.Operations = []ir.PhysicalOperation{p.Operations[0], p.Operations[2]} }}, + {"unnest before root scan", func(p *ir.PhysicalPlan) { p.Operations = []ir.PhysicalOperation{p.Operations[1], p.Operations[2]} }}, + {"return not last", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + p.Operations = []ir.PhysicalOperation{p.Operations[0], p.Operations[1], p.Operations[2], p.Operations[2]} + })}, + {"missing return", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { p.Operations = p.Operations[:2] })}, + {"second root scan", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + p.Operations = []ir.PhysicalOperation{p.Operations[0], p.Operations[0], p.Operations[1], p.Operations[2]} + })}, + {"filter coexists with unnest", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + p.BindVars["status_value"] = "final" + p.Operations = []ir.PhysicalOperation{p.Operations[0], + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "status_value"}}}}, + p.Operations[1], p.Operations[2]} + })}, + {"sort coexists with unnest", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + p.Operations = []ir.PhysicalOperation{p.Operations[0], p.Operations[1], + {Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{Keys: []ir.PhysicalValue{{Variable: "root", Path: []string{"logical_id"}}}}}, + p.Operations[2]} + })}, + {"limit coexists with unnest", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + p.Operations = []ir.PhysicalOperation{p.Operations[0], p.Operations[1], + {Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "limit"}}, p.Operations[2]} + })}, + {"traversal op coexists with unnest", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + addTraversal(p, ir.PhysicalOutbound, "to_type", "", nil) + p.Operations = []ir.PhysicalOperation{p.Operations[0], p.Operations[1], p.Operations[1], p.Operations[2]} + })}, + {"population root", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + p.Operations[0].RootScan.Population = &ir.PhysicalPopulationRootSource{} + })}, + {"error policy is a capability gap", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { u.EmptyPolicy = ir.PhysicalUnnestError })}, + {"unknown policy", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { u.EmptyPolicy = "SOMETIMES" })}, + {"missing output variables", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { u.OutputVariable = "" })}, + {"missing has-item variable", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { u.HasItemVariable = "" })}, + {"value-expression source", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + value := ir.PhysicalValue{Variable: "root", Path: []string{"payload"}} + u.Expression = ir.PhysicalExpression{Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalArrayCardinality, + NullBehavior: ir.PhysicalPreserveNull, Value: &value} + })}, + {"scalar cardinality source", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Expression.Cardinality = ir.PhysicalScalarCardinality + })}, + {"omit-nulls source behavior", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { u.Expression.NullBehavior = ir.PhysicalOmitNulls })}, + {"extract payload missing", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { u.Expression.Extract = nil })}, + {"conditional-array mode", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Expression.Extract.ExecutionMode = ir.PhysicalSelectorConditionalArray + })}, + {"direct-scalar mode", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Expression.Extract.ExecutionMode = ir.PhysicalSelectorDirectScalar + })}, + {"prepared reference", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Expression.Extract.Prepared = &ir.PhysicalPreparedReference{SetVariable: "set_1", Field: "field"} + })}, + {"fallback selectors", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Expression.Extract.Fallbacks = []ir.PhysicalSelectorFallback{{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, ResourceType: "Patient", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: "name", Iterate: true}}}, + }} + })}, + {"unit normalization", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Expression.Extract.UnitNormalization = &ir.PhysicalUnitNormalization{} + })}, + {"distinct expansion", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { u.Expression.Extract.Distinct = true })}, + {"selector filter", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Expression.Extract.Selector.Filter = &spec.ContainsFilter{Field: "system", Needle: "urn:x"} + })}, + {"empty selector", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { u.Expression.Extract.Selector.Steps = nil })}, + {"non-iterated terminal step", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Expression.Extract.Selector.Steps[0].Iterate = false + })}, + {"indexed step", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + zero := 0 + u.Expression.Extract.Selector.Steps[0].Index = &zero + })}, + {"property-less step", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Expression.Extract.Selector.Steps[0].Field = "" + })}, + {"unsafe path segment", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Expression.Extract.Selector.Steps[0].Field = "bad-name" + })}, + {"source not payload", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Expression.Extract.Source.Path = []string{"logical_id"} + })}, + {"source is bind", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + p := u + p.Expression.Extract.Source.BindKey = "doc" + })}, + {"source variable not owner", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Expression.Extract.Source.Variable = "elsewhere" + })}, + {"resource type mismatch", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Expression.Extract.ResourceType = "Observation" + })}, + {"resource type blank", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Expression.Extract.ResourceType = "" + })}, + {"route scope operations", mutate(func(p *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + p.BindVars["edge_collection"] = "fhir_edge" + p.BindVars["edge_label"] = "has-observation" + p.BindVars["target_type"] = "Observation" + u.Owner = ir.PhysicalUnnestOwner{OccurrenceID: "obs-owner", ResourceType: "Patient", RootVariable: "root", OwnerVariable: "obs", + Route: []ir.PhysicalUnnestRouteStep{{OccurrenceID: "obs-owner", Traversal: ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "obs", EdgeVariable: "obs_edge", + Direction: ir.PhysicalOutbound, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: "edge_label", TargetTypeBindKey: "target_type", + EdgeTargetTypeField: "to_type", Strategy: ir.PhysicalTraversalNative, + }, Scope: []ir.PhysicalOperation{{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EXISTS", Left: ir.PhysicalValue{Variable: "obs", Path: []string{"status"}}}}}}}}} + u.Expression = arrayExtract("obs", "Patient", spec.SelectorStep{Field: "identifier", Iterate: true}) + })}, + {"route any direction", mutate(func(p *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + planRouteTraversal(p, u, ir.PhysicalAny, "") + })}, + {"route endpoint lookup", mutate(func(p *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + planRouteTraversal(p, u, ir.PhysicalOutbound, ir.PhysicalTraversalEndpointLookup) + })}, + {"route chain break", mutate(func(p *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + planRouteTraversal(p, u, ir.PhysicalOutbound, "") + u.Owner.Route[0].Traversal.SourceVariable = "not-root" + })}, + {"route does not end at owner", mutate(func(p *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + planRouteTraversal(p, u, ir.PhysicalOutbound, "") + u.Owner.OwnerVariable = "unreached" + })}, + {"ancestor indexes a non-iterated prefix step", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Ancestors = []ir.PhysicalUnnestAncestor{{StepIndex: 1, Variable: "code_item"}} + })}, + {"ancestor indexes the terminal step", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Ancestors = []ir.PhysicalUnnestAncestor{{StepIndex: 2, Variable: "coding_item"}} + })}, + {"ancestor out of range", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Ancestors = []ir.PhysicalUnnestAncestor{{StepIndex: 3, Variable: "component_item"}} + })}, + {"ancestor repeats a step", mutate(func(_ *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.Ancestors = []ir.PhysicalUnnestAncestor{ + {StepIndex: 0, Variable: "component_item"}, {StepIndex: 0, Variable: "component_item_2"}} + })}, + {"zero projections", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + p.Operations[2].Return.Projections = nil + })}, + {"unnamed projection", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + p.Operations[2].Return.Projections[0].Name = "" + })}, + {"hidden projection", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + p.Operations[2].Return.Projections[0].Hidden = true + })}, + {"presence sidecar", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + p.Operations[2].Return.Projections = append(p.Operations[2].Return.Projections, ir.PhysicalProjection{ + Name: "has_identifier", PresenceOutput: true, + Presence: &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + Paths: [][]string{{"identifier"}}, + }, + }) + })}, + {"selector projection", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + projection := ir.PhysicalProjection{Name: "identifier_system"} + source := arrayExtract("root", "Patient", spec.SelectorStep{Field: "identifier"}) + source.Cardinality = ir.PhysicalScalarCardinality + projection.Expression = &source + p.Operations[2].Return.Projections = append(p.Operations[2].Return.Projections, projection) + })}, + {"bind-backed projection", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + p.BindVars["const"] = "x" + p.Operations[2].Return.Projections[0] = ir.PhysicalProjection{Name: "c", Value: ir.PhysicalValue{BindKey: "const"}} + })}, + {"bind-backed expression projection", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + p.BindVars["const"] = "x" + value := ir.PhysicalValue{BindKey: "const"} + p.Operations[2].Return.Projections[0] = ir.PhysicalProjection{Name: "c", Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalValueExpression, Cardinality: ir.PhysicalScalarCardinality, Value: &value}} + })}, + {"expression kind outside the slice", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + p.Operations[2].Return.Projections[0] = ir.PhysicalProjection{Name: "c", Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalLiteralExpression, Cardinality: ir.PhysicalScalarCardinality}} + })}, + {"derived output variable collides with a row", mutate(func(p *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + u.OutputVariable = "root" + })}, + {"duplicate expansion output variables", mutate(func(p *ir.PhysicalPlan, u *ir.PhysicalUnnest) { + p.Operations = []ir.PhysicalOperation{p.Operations[0], unnestOp(*u), unnestOp(*u), p.Operations[2]} + })}, + {"path over non-jsonb derived value", mutate(func(p *ir.PhysicalPlan, _ *ir.PhysicalUnnest) { + p.Operations[2].Return.Projections[2] = valueProjection("position_value", "position", "x") + })}, + } +} + +func TestUnnestRejectsSurfaceWithTypedErrors(t *testing.T) { + for _, tc := range unnestSurfaceRejectCases() { + plan := unnestPlan(expansionProjections()...) + tc.mutate(&plan) + if _, err := RenderUnnest(plan, mustBinding(t)); !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Errorf("%s: RenderUnnest error = %v, want ErrUnsupportedPostgresPlan", tc.name, err) + } + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); err == nil { + t.Errorf("%s: core dispatch accepted an unregistered shape", tc.name) + } + } +} + +func TestUnnestDispatchGateIsFullBackstop(t *testing.T) { + for _, tc := range unnestSurfaceRejectCases() { + plan := unnestPlan(expansionProjections()...) + tc.mutate(&plan) + _, entryErr := RenderUnnest(plan, mustBinding(t)) + if !errors.Is(entryErr, ErrUnsupportedPostgresPlan) { + t.Fatalf("%s: RenderUnnest error = %v, want ErrUnsupportedPostgresPlan", tc.name, entryErr) + } + hasUnnest := false + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalUnnestOp { + hasUnnest = true + } + } + if !hasUnnest { + continue + } + _, dispatchErr := RenderPhysicalPlan(plan, mustBinding(t)) + if !errors.Is(dispatchErr, ErrUnsupportedPostgresPlan) { + t.Errorf("%s: core dispatch error = %v, want the surface contract's ErrUnsupportedPostgresPlan", tc.name, dispatchErr) + continue + } + if entryErr.Error() != dispatchErr.Error() { + t.Errorf("%s: dispatch error %q diverged from entry error %q", tc.name, dispatchErr, entryErr) + } + } +} + +func planRouteTraversal(p *ir.PhysicalPlan, u *ir.PhysicalUnnest, direction ir.PhysicalTraversalDirection, strategy ir.PhysicalTraversalStrategy) { + p.BindVars["edge_collection"] = "fhir_edge" + p.BindVars["edge_label"] = "has-observation" + p.BindVars["target_type"] = "Patient" + u.Owner = ir.PhysicalUnnestOwner{OccurrenceID: "obs-owner", ResourceType: "Patient", RootVariable: "root", OwnerVariable: "obs", + Route: []ir.PhysicalUnnestRouteStep{{OccurrenceID: "obs-owner", Traversal: ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "obs", EdgeVariable: "obs_edge", + Direction: direction, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: "edge_label", TargetTypeBindKey: "target_type", + EdgeTargetTypeField: func() string { + if direction == ir.PhysicalInbound { + return "from_type" + } + return "to_type" + }(), + Strategy: strategy, EndpointField: "_from", EndpointJoinField: "_to", + }}}} + u.Expression = arrayExtract("obs", "Patient", spec.SelectorStep{Field: "identifier", Iterate: true}) +} + +func TestUnnestRejectsOwnerTypeContradictionAgainstScannedRows(t *testing.T) { + plan := unnestPlan(expansionProjections()...) + unnest := plan.Operations[1].Unnest + unnest.Owner.ResourceType = "Observation" + unnest.Expression.Extract.ResourceType = "Observation" + if _, err := RenderUnnest(plan, mustBinding(t)); !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("owner/row type contradiction error = %v, want ErrUnsupportedPostgresPlan", err) + } +} + +var sql09ResourceNamePattern = regexp.MustCompile(`^verify_sql09_[0-9]+_[0-9a-f]+$`) + +type sql09Service struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + client *control.Client + schema string + resources *control.ResourceRepository + fail func(step string, err error) +} + +func sql09ServiceURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 unnest acceptance tests") + } + return raw +} + +func newSQL09Service(t *testing.T) *sql09Service { + t.Helper() + raw := sql09ServiceURL(t) + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %v", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("verify_sql09_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !sql09ResourceNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + t.Fatalf("create isolated database: %v", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %v", err) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect to isolated database: %v", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("verify_sql09_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !sql09ResourceNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + t.Fatalf("create owned schema: %v", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %v", err) + } + }) + + client, err := control.OpenPool(ctx, control.ClientConfig{DSN: dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 2 * time.Second}) + if err != nil { + t.Fatalf("open pg-store client (requires a PostgreSQL 18 service): %v", err) + } + t.Cleanup(client.Close) + if _, err := control.MigrateSchema(ctx, client, schema); err != nil { + t.Fatalf("migrate owned schema: %v", err) + } + resources, err := control.NewResourceRepository(client, schema) + if err != nil { + t.Fatalf("build resource repository: %v", err) + } + return &sql09Service{ctx: ctx, admin: admin, iso: iso, client: client, schema: schema, resources: resources, + fail: func(step string, err error) { t.Fatalf("%s: %v", step, err) }} +} + +func (s *sql09Service) insertResource(t *testing.T, project, generation, resourceType, logicalID, authPath string, payload json.RawMessage) { + t.Helper() + if _, err := s.resources.InsertResource(s.ctx, source.Resource{ + Project: project, DatasetGeneration: generation, ResourceType: resourceType, + LogicalID: logicalID, AuthResourcePath: authPath, Payload: payload, + }); err != nil { + s.fail("insert resource "+resourceType+"/"+logicalID, err) + } +} + +func (s *sql09Service) insertEdge(t *testing.T, rec control.EdgeOccurrenceRow) { + t.Helper() + edges, err := control.NewEdgeRepository(s.client, s.schema) + if err != nil { + s.fail("build edge repository", err) + } + if _, err := edges.InsertEdge(s.ctx, rec); err != nil { + s.fail("insert edge "+rec.Label+" "+rec.SourceLogicalID+"->"+rec.TargetLogicalID, err) + } +} + +func (s *sql09Service) runRendered(t *testing.T, rendered command.RenderedPostgresPhysicalPlan) [][]string { + t.Helper() + return s.queryRows(t, rendered.SQL, parameterValues(rendered.Args)...) +} + +func sql09StringifyCell(value any) string { + switch typed := value.(type) { + case nil: + return "" + case string: + return typed + case []byte: + return string(typed) + case map[string]any, []any, json.RawMessage: + if out, err := json.Marshal(typed); err == nil { + return string(out) + } + } + return fmt.Sprintf("%v", value) +} + +func (s *sql09Service) queryRows(t *testing.T, sql string, args ...any) [][]string { + t.Helper() + rows, err := s.iso.Query(s.ctx, sql, args...) + if err != nil { + s.fail("execute statement: "+sql, err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + s.fail("read row", err) + } + row := make([]string, len(values)) + for index, value := range values { + row[index] = sql09StringifyCell(value) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + s.fail("finish statement", err) + } + return out +} + +func (s *sql09Service) witness(t *testing.T, sql string, args ...any) [][]string { + t.Helper() + return s.queryRows(t, sql, args...) +} + +func (s *sql09Service) sql09IsNullProbe(t *testing.T, rendered command.RenderedPostgresPhysicalPlan, columns ...string) [][]string { + t.Helper() + selects := make([]string, 0, len(columns)) + for _, column := range columns { + selects = append(selects, column+" IS NULL") + } + sql := "SELECT " + strings.Join(selects, ", ") + " FROM (" + rendered.SQL + ") AS sql09_probe" + return s.queryRows(t, sql, parameterValues(rendered.Args)...) +} + +func parameterValues(args []command.PostgreSQLParameter) []any { + values := make([]any, len(args)) + for index, parameter := range args { + values[index] = parameter.Value + } + return values +} + +func sql09CanonicalCell(cell string) string { + var value any + if err := json.Unmarshal([]byte(cell), &value); err != nil { + return cell + } + out, err := json.Marshal(value) + if err != nil { + return cell + } + return string(out) +} + +func sql09CanonicalRows(rows [][]string) [][]string { + out := make([][]string, len(rows)) + for index, row := range rows { + cells := make([]string, len(row)) + for cellIndex, cell := range row { + cells[cellIndex] = sql09CanonicalCell(cell) + } + out[index] = cells + } + return out +} + +const ( + sql09Project = "sql09-proj" + sql09Generation = "gen-1" +) + +func sql09JSON(t *testing.T, document string) json.RawMessage { + t.Helper() + if !json.Valid([]byte(document)) { + t.Fatalf("fixture document is not valid JSON: %s", document) + } + return json.RawMessage(document) +} + +func sql09CanonicalObject(t *testing.T, document string) string { + t.Helper() + return sql09CanonicalCell(string(sql09JSON(t, document))) +} + +func sql09PatientUnnestPlan(policy ir.PhysicalUnnestEmptyPolicy) ir.PhysicalPlan { + plan := scopedPlan(sql09Project, sql09Generation, []string{"FHIR/Patient", "FHIR/Observation"}, false) + plan.Operations = append(plan.Operations, unnestOp(identifierExpansion(policy))) + return withReturn(plan, expansionProjections()...) +} + +func sql09ObservationUnnestPlan(policy ir.PhysicalUnnestEmptyPolicy) ir.PhysicalPlan { + plan := scopedPlan(sql09Project, sql09Generation, []string{"FHIR/Patient", "FHIR/Observation"}, false) + plan.BindVars["root_collection"] = "Observation" + plan.Operations = append(plan.Operations, unnestOp(codingExpansion(policy))) + return withReturn(plan, + valueProjection("logical_id", "root", "logical_id"), + valueProjection("component", "component_item"), + valueProjection("item", "item"), + valueProjection("position", "position"), + valueProjection("has_item", "has_item"), + ) +} + +func sql09Binding(t *testing.T, svc *sql09Service) ir.PostgresStorageBinding { + t.Helper() + binding, err := ir.NewPostgresStorageBinding(svc.schema) + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + return binding +} + +func TestPG18UnnestInnerEmitsLiteralRowsAndPreservesDuplicates(t *testing.T) { + svc := newSQL09Service(t) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-missing", "FHIR/Patient", sql09JSON(t, `{}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-null", "FHIR/Patient", sql09JSON(t, `{"identifier": null}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-empty", "FHIR/Patient", sql09JSON(t, `{"identifier": []}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-nopayload", "FHIR/Patient", nil) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-nullitem", "FHIR/Patient", sql09JSON(t, `{"identifier": [null]}`)) + dupA := `{"system": "urn:x", "value": "a"}` + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-dups", "FHIR/Patient", + sql09JSON(t, `{"identifier": [`+dupA+`, `+dupA+`, {"system": "urn:x", "value": "b"}, `+dupA+`]}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-dup-scalars", "FHIR/Patient", + sql09JSON(t, `{"identifier": ["a", "a", "b", "a"]}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-mixed", "FHIR/Patient", + sql09JSON(t, `{"identifier": ["a", null, "b"]}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-falsy-zero", "FHIR/Patient", sql09JSON(t, `{"identifier": 0}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-truthy-scalar", "FHIR/Patient", sql09JSON(t, `{"identifier": 5}`)) + svc.insertResource(t, "sql09-other-project", sql09Generation, "Patient", "p-scope", "FHIR/Patient", + sql09JSON(t, `{"identifier": [{"value": "out-of-scope"}]}`)) + + rendered, err := RenderUnnest(sql09PatientUnnestPlan(ir.PhysicalUnnestExclude), sql09Binding(t, svc)) + if err != nil { + t.Fatalf("RenderUnnest: %v", err) + } + got := sql09CanonicalRows(svc.runRendered(t, rendered)) + + want := [][]string{ + {"p-dups", sql09CanonicalObject(t, dupA), "0", "true"}, + {"p-dups", sql09CanonicalObject(t, dupA), "1", "true"}, + {"p-dups", sql09CanonicalObject(t, `{"system": "urn:x", "value": "b"}`), "2", "true"}, + {"p-dups", sql09CanonicalObject(t, dupA), "3", "true"}, + {"p-dup-scalars", "a", "0", "true"}, + {"p-dup-scalars", "a", "1", "true"}, + {"p-dup-scalars", "b", "2", "true"}, + {"p-dup-scalars", "a", "3", "true"}, + {"p-mixed", "a", "0", "true"}, + {"p-mixed", "b", "1", "true"}, + {"p-truthy-scalar", "5", "0", "true"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("INNER rows = %v, want %v", got, want) + } + itemNull := svc.sql09IsNullProbe(t, rendered, "c1") + if len(itemNull) != len(want) { + t.Fatalf("item IS NULL probe = %v, want one boolean per emitted row", itemNull) + } + for index, row := range itemNull { + if row[0] != "false" { + t.Fatalf("INNER row %d item cell IS NULL = %s, want false", index, row[0]) + } + } + + for _, absent := range []string{"p-missing", "p-null", "p-empty", "p-nopayload", "p-nullitem", "p-falsy-zero", "p-scope"} { + for _, row := range got { + if row[0] == absent { + t.Fatalf("INNER emitted empty/out-of-scope source %q: %v", absent, row) + } + } + } + for _, row := range got { + if strings.Contains(strings.Join(row, "|"), "out-of-scope") { + t.Fatalf("INNER emitted out-of-scope payload: %v", row) + } + } + dupCount := svc.witness(t, + `SELECT count(*), count(DISTINCT e ->> 'value') FROM "`+svc.schema+`"."fhir_resource" r,`+ + ` jsonb_array_elements(r.payload -> 'identifier') e`+ + ` WHERE r.project=$1 AND r.dataset_generation=$2 AND r.logical_id='p-dups'`, + sql09Project, sql09Generation) + if len(dupCount) != 1 || dupCount[0][0] != "4" || dupCount[0][1] != "2" { + t.Fatalf("duplicate witness = %v, want 4 elements / 2 distinct values", dupCount) + } + nullSurvivors := svc.witness(t, + `SELECT logical_id, count(e) FILTER (WHERE jsonb_typeof(e) <> 'null') FROM "`+svc.schema+`"."fhir_resource" r,`+ + ` jsonb_array_elements(r.payload -> 'identifier') e`+ + ` WHERE r.project=$1 AND r.dataset_generation=$2 AND r.logical_id IN ('p-nullitem','p-mixed')`+ + ` GROUP BY logical_id ORDER BY min(resource_seq)`, + sql09Project, sql09Generation) + if len(nullSurvivors) != 2 || nullSurvivors[0][0] != "p-nullitem" || nullSurvivors[0][1] != "0" || + nullSurvivors[1][0] != "p-mixed" || nullSurvivors[1][1] != "2" { + t.Fatalf("null-survivor witness = %v, want p-nullitem=0 survivors and p-mixed=2", nullSurvivors) + } + emptyForms := svc.witness(t, + `SELECT logical_id, CASE`+ + ` WHEN payload -> 'identifier' IS NULL OR jsonb_typeof(payload -> 'identifier') IN ('null','false') THEN 0`+ + ` WHEN jsonb_typeof(payload -> 'identifier') = 'array'`+ + ` THEN (SELECT count(*) FROM jsonb_array_elements(payload -> 'identifier') e WHERE jsonb_typeof(e) <> 'null')`+ + ` WHEN payload -> 'identifier' = '0'::jsonb OR payload -> 'identifier' = '""'::jsonb THEN 0`+ + ` ELSE -1 END FROM "`+svc.schema+`"."fhir_resource"`+ + ` WHERE project=$1 AND dataset_generation=$2`+ + ` AND logical_id IN ('p-missing','p-null','p-empty','p-nopayload','p-nullitem','p-falsy-zero') ORDER BY resource_seq`, + sql09Project, sql09Generation) + if len(emptyForms) != 6 { + t.Fatalf("empty-source witness rows = %v, want six zero-row sources", emptyForms) + } + for _, row := range emptyForms { + if row[1] != "0" { + t.Fatalf("empty-source %s has %s surviving items, want 0", row[0], row[1]) + } + } +} + +func TestPG18UnnestOuterEmitsOneNullItemRowPerEmptySource(t *testing.T) { + svc := newSQL09Service(t) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-missing", "FHIR/Patient", sql09JSON(t, `{}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-null", "FHIR/Patient", sql09JSON(t, `{"identifier": null}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-empty", "FHIR/Patient", sql09JSON(t, `{"identifier": []}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-nopayload", "FHIR/Patient", nil) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-nullitem", "FHIR/Patient", sql09JSON(t, `{"identifier": [null]}`)) + dupA := `{"system": "urn:x", "value": "a"}` + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-dups", "FHIR/Patient", + sql09JSON(t, `{"identifier": [`+dupA+`, `+dupA+`, {"system": "urn:x", "value": "b"}, `+dupA+`]}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-dup-scalars", "FHIR/Patient", + sql09JSON(t, `{"identifier": ["a", "a", "b", "a"]}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-mixed", "FHIR/Patient", + sql09JSON(t, `{"identifier": ["a", null, "b"]}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-falsy-zero", "FHIR/Patient", sql09JSON(t, `{"identifier": 0}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-truthy-scalar", "FHIR/Patient", sql09JSON(t, `{"identifier": 5}`)) + + rendered, err := RenderUnnest(sql09PatientUnnestPlan(ir.PhysicalUnnestPreserveParent), sql09Binding(t, svc)) + if err != nil { + t.Fatalf("RenderUnnest: %v", err) + } + got := sql09CanonicalRows(svc.runRendered(t, rendered)) + + pad := func(id string) []string { return []string{id, "", "", "false"} } + want := [][]string{ + pad("p-missing"), pad("p-null"), pad("p-empty"), pad("p-nopayload"), pad("p-nullitem"), + {"p-dups", sql09CanonicalObject(t, dupA), "0", "true"}, + {"p-dups", sql09CanonicalObject(t, dupA), "1", "true"}, + {"p-dups", sql09CanonicalObject(t, `{"system": "urn:x", "value": "b"}`), "2", "true"}, + {"p-dups", sql09CanonicalObject(t, dupA), "3", "true"}, + {"p-dup-scalars", "a", "0", "true"}, + {"p-dup-scalars", "a", "1", "true"}, + {"p-dup-scalars", "b", "2", "true"}, + {"p-dup-scalars", "a", "3", "true"}, + {"p-mixed", "a", "0", "true"}, + {"p-mixed", "b", "1", "true"}, + pad("p-falsy-zero"), + {"p-truthy-scalar", "5", "0", "true"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("OUTER rows = %v, want %v", got, want) + } + + dataRows := svc.witness(t, + `SELECT r.logical_id FROM "`+svc.schema+`"."fhir_resource" r,`+ + ` jsonb_array_elements(CASE`+ + ` WHEN r.payload -> 'identifier' IS NULL THEN '[]'::jsonb`+ + ` WHEN jsonb_typeof(r.payload -> 'identifier') = 'array' THEN r.payload -> 'identifier'`+ + ` WHEN r.payload -> 'identifier' IN ('null'::jsonb, 'false'::jsonb, '0'::jsonb, '""'::jsonb) THEN '[]'::jsonb`+ + ` ELSE jsonb_build_array(r.payload -> 'identifier') END) e`+ + ` WHERE jsonb_typeof(e) <> 'null'`+ + ` AND r.project=$1 AND r.dataset_generation=$2 AND r.auth_resource_path = ANY ('{FHIR/Patient,FHIR/Observation}'::text[])`+ + ` ORDER BY r.resource_seq`, + sql09Project, sql09Generation) + if len(dataRows) != 11 { + t.Fatalf("witness item rows = %d rows %v, want the eleven non-pad rows", len(dataRows), dataRows) + } + nullProbe := svc.sql09IsNullProbe(t, rendered, "c1", "c2", "c3") + if len(nullProbe) != len(want) { + t.Fatalf("pad probe = %v, want one row per OUTER row", nullProbe) + } + pads := 0 + for index, row := range nullProbe { + isPad := want[index][3] == "false" + wantItemNull, wantOrdinalityNull := "false", "false" + if isPad { + wantItemNull, wantOrdinalityNull = "true", "true" + pads++ + if row[0] != "true" || row[1] != "true" { + t.Fatalf("OUTER pad row %d (logical_id %s): item/ordinality IS NULL = %s/%s, want true/true", + index, got[index][0], row[0], row[1]) + } + } else if row[0] != wantItemNull || row[1] != wantOrdinalityNull { + t.Fatalf("OUTER item row %d (logical_id %s): item/ordinality IS NULL = %s/%s, want false/false", + index, got[index][0], row[0], row[1]) + } + if row[2] != "false" { + t.Fatalf("OUTER row %d: has_item is SQL NULL; it is a total boolean, never SQL NULL", index) + } + } + if pads != 6 { + t.Fatalf("pad rows = %d, want exactly one per empty source (6)", pads) + } +} + +func TestPG18UnnestNestedRepeatedArraysPreserveOrderAndAncestry(t *testing.T) { + svc := newSQL09Service(t) + componentZero := `{"code": {"coding": [{"system": "urn:sys1", "code": "c1"}, {"system": "urn:sys1", "code": "c2"}]}}` + componentOne := `{"code": {"coding": [{"system": "urn:sys2", "code": "c9"}]}}` + svc.insertResource(t, sql09Project, sql09Generation, "Observation", "o-nested", "FHIR/Observation", + sql09JSON(t, `{"component": [`+componentZero+`, `+componentOne+`]}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Observation", "o-empty-components", "FHIR/Observation", sql09JSON(t, `{"component": []}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Observation", "o-null-components", "FHIR/Observation", sql09JSON(t, `{"component": null}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Observation", "o-missing", "FHIR/Observation", sql09JSON(t, `{}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Observation", "o-falsy-zero", "FHIR/Observation", sql09JSON(t, `{"component": 0}`)) + + inner, err := RenderUnnest(sql09ObservationUnnestPlan(ir.PhysicalUnnestExclude), sql09Binding(t, svc)) + if err != nil { + t.Fatalf("RenderUnnest INNER: %v", err) + } + got := sql09CanonicalRows(svc.runRendered(t, inner)) + + want := [][]string{ + {"o-nested", sql09CanonicalObject(t, componentZero), sql09CanonicalObject(t, `{"system": "urn:sys1", "code": "c1"}`), "0", "true"}, + {"o-nested", sql09CanonicalObject(t, componentZero), sql09CanonicalObject(t, `{"system": "urn:sys1", "code": "c2"}`), "1", "true"}, + {"o-nested", sql09CanonicalObject(t, componentOne), sql09CanonicalObject(t, `{"system": "urn:sys2", "code": "c9"}`), "2", "true"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("nested INNER rows = %v, want row-major carrier order %v", got, want) + } + + stored := svc.witness(t, + `SELECT payload -> 'component' -> 0, payload -> 'component' -> 1 FROM "`+svc.schema+`"."fhir_resource"`+ + ` WHERE project=$1 AND dataset_generation=$2 AND logical_id='o-nested'`, + sql09Project, sql09Generation) + if len(stored) != 1 { + t.Fatalf("stored components witness = %v", stored) + } + enclosing := []string{sql09CanonicalCell(stored[0][0]), sql09CanonicalCell(stored[0][1])} + for index, row := range want { + expectedEnclosing := enclosing[0] + if index == 2 { + expectedEnclosing = enclosing[1] + } + if row[1] != expectedEnclosing { + t.Fatalf("row %d ancestor = %v, want the exact enclosing component %v", index, row[1], expectedEnclosing) + } + } + + outer, err := RenderUnnest(sql09ObservationUnnestPlan(ir.PhysicalUnnestPreserveParent), sql09Binding(t, svc)) + if err != nil { + t.Fatalf("RenderUnnest OUTER: %v", err) + } + gotOuter := sql09CanonicalRows(svc.runRendered(t, outer)) + if len(gotOuter) != 7 { + t.Fatalf("nested OUTER rows = %v, want three items plus four pads", gotOuter) + } + for index, row := range gotOuter[3:] { + wantID := []string{"o-empty-components", "o-null-components", "o-missing", "o-falsy-zero"}[index] + if !reflect.DeepEqual(row, []string{wantID, "", "", "", "false"}) { + t.Fatalf("nested OUTER pad %d = %v, want one null-item row for %s", index, row, wantID) + } + } + + carriers := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" r,`+ + ` jsonb_array_elements(CASE WHEN r.payload -> 'component' IS NULL`+ + ` OR jsonb_typeof(r.payload -> 'component') <> 'array' THEN '[]'::jsonb ELSE r.payload -> 'component' END) c,`+ + ` jsonb_array_elements(CASE WHEN c -> 'code' -> 'coding' IS NULL`+ + ` OR jsonb_typeof(c -> 'code' -> 'coding') <> 'array' THEN '[]'::jsonb ELSE c -> 'code' -> 'coding' END) e`+ + ` WHERE r.project=$1 AND r.dataset_generation=$2`, + sql09Project, sql09Generation) + if len(carriers) != 1 || carriers[0][0] != "3" { + t.Fatalf("carrier witness = %v, want 3", carriers) + } +} + +func TestPG18UnnestPrefixTruthyScalarRaisesLikeAQL(t *testing.T) { + svc := newSQL09Service(t) + svc.insertResource(t, sql09Project, sql09Generation, "Observation", "o-truthy-prefix", "FHIR/Observation", + sql09JSON(t, `{"component": 5}`)) + outer, err := RenderUnnest(sql09ObservationUnnestPlan(ir.PhysicalUnnestPreserveParent), sql09Binding(t, svc)) + if err != nil { + t.Fatalf("RenderUnnest OUTER: %v", err) + } + rows, err := svc.iso.Query(svc.ctx, outer.SQL, parameterValues(outer.Args)...) + if err == nil { + for rows.Next() { + } + err = rows.Err() + rows.Close() + } + if err == nil { + t.Fatalf("truthy scalar prefix expanded without error; AQL parity requires an execution error") + } + svc2 := newSQL09Service(t) + svc2.insertResource(t, sql09Project, sql09Generation, "Observation", "o-falsy-false", "FHIR/Observation", + sql09JSON(t, `{"component": false}`)) + svc2.insertResource(t, sql09Project, sql09Generation, "Observation", "o-falsy-string", "FHIR/Observation", + sql09JSON(t, `{"component": ""}`)) + inner, err := RenderUnnest(sql09ObservationUnnestPlan(ir.PhysicalUnnestExclude), sql09Binding(t, svc2)) + if err != nil { + t.Fatalf("RenderUnnest INNER: %v", err) + } + if rows := svc2.runRendered(t, inner); len(rows) != 0 { + t.Fatalf("falsy prefix rows = %v, want none (AQL ternary falsy set)", rows) + } +} + +func TestPG18UnnestAncestorTerminalTruthyScalarRaisesLikeAQL(t *testing.T) { + svc := newSQL09Service(t) + svc.insertResource(t, sql09Project, sql09Generation, "Observation", "o-terminal-truthy", "FHIR/Observation", + sql09JSON(t, `{"component": [{"code": {"coding": 5}}]}`)) + outer, err := RenderUnnest(sql09ObservationUnnestPlan(ir.PhysicalUnnestPreserveParent), sql09Binding(t, svc)) + if err != nil { + t.Fatalf("RenderUnnest OUTER: %v", err) + } + rows, err := svc.iso.Query(svc.ctx, outer.SQL, parameterValues(outer.Args)...) + if err == nil { + for rows.Next() { + } + err = rows.Err() + rows.Close() + } + if err == nil { + t.Fatalf("truthy scalar at an ancestor-bearing terminal step expanded without error; AQL lineage parity requires an execution error") + } + svc2 := newSQL09Service(t) + svc2.insertResource(t, sql09Project, sql09Generation, "Observation", "o-terminal-falsy", "FHIR/Observation", + sql09JSON(t, `{"component": [{"code": {"coding": 0}}]}`)) + inner, err := RenderUnnest(sql09ObservationUnnestPlan(ir.PhysicalUnnestExclude), sql09Binding(t, svc2)) + if err != nil { + t.Fatalf("RenderUnnest INNER: %v", err) + } + if rows := svc2.runRendered(t, inner); len(rows) != 0 { + t.Fatalf("falsy ancestor-terminal rows = %v, want none (AQL ternary falsy set)", rows) + } +} + +func TestPG18UnnestOwnerRouteKeepsParentIdentityAndEdgeMultiplicity(t *testing.T) { + svc := newSQL09Service(t) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-route", "FHIR/Patient", sql09JSON(t, `{"gender": "female"}`)) + codingSet := func(system, code string) string { + return `{"component": [{"code": {"coding": [{"system": "` + system + `", "code": "` + code + `"}]}}]}` + } + svc.insertResource(t, sql09Project, sql09Generation, "Observation", "o-obs1", "FHIR/Observation", sql09JSON(t, codingSet("urn:obs1", "o1"))) + svc.insertResource(t, sql09Project, sql09Generation, "Observation", "o-obs2", "FHIR/Observation", sql09JSON(t, codingSet("urn:obs2", "o2"))) + for index, target := range []string{"o-obs1", "o-obs1", "o-obs2"} { + svc.insertEdge(t, control.EdgeOccurrenceRow{ + Project: sql09Project, DatasetGeneration: sql09Generation, AuthResourcePath: "FHIR/Patient", + Label: "has-observation", SourceResourceType: "Patient", SourceLogicalID: "p-route", + TargetResourceType: "Observation", TargetLogicalID: target, + SourcePath: fmt.Sprintf("observation[%d]", index), SourceOrdinal: int64(index), + }) + } + + plan := scopedPlan(sql09Project, sql09Generation, []string{"FHIR/Patient", "FHIR/Observation"}, false) + plan.BindVars["edge_collection"] = "fhir_edge" + plan.BindVars["edge_label"] = "has-observation" + plan.BindVars["target_type"] = "Observation" + unnest := codingExpansion(ir.PhysicalUnnestPreserveParent) + unnest.Owner = ir.PhysicalUnnestOwner{ + OccurrenceID: "obs-owner", ResourceType: "Observation", RootVariable: "root", OwnerVariable: "obs", + Route: []ir.PhysicalUnnestRouteStep{{OccurrenceID: "obs-owner", Traversal: ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "obs", EdgeVariable: "obs_edge", + Direction: ir.PhysicalOutbound, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: "edge_label", TargetTypeBindKey: "target_type", + EdgeTargetTypeField: "to_type", Strategy: ir.PhysicalTraversalNative, + }}}, + } + unnest.Expression = arrayExtract("obs", "Observation", + spec.SelectorStep{Field: "component", Iterate: true}, + spec.SelectorStep{Field: "code"}, + spec.SelectorStep{Field: "coding", Iterate: true}) + plan.Operations = append(plan.Operations, unnestOp(unnest)) + plan = withReturn(plan, + valueProjection("patient_id", "root", "logical_id"), + valueProjection("observation_id", "obs", "logical_id"), + valueProjection("item", "item"), + valueProjection("position", "position"), + valueProjection("has_item", "has_item"), + ) + rendered, err := RenderUnnest(plan, sql09Binding(t, svc)) + if err != nil { + t.Fatalf("RenderUnnest: %v", err) + } + got := sql09CanonicalRows(svc.runRendered(t, rendered)) + + want := [][]string{ + {"p-route", "o-obs1", sql09CanonicalObject(t, `{"system": "urn:obs1", "code": "o1"}`), "0", "true"}, + {"p-route", "o-obs1", sql09CanonicalObject(t, `{"system": "urn:obs1", "code": "o1"}`), "0", "true"}, + {"p-route", "o-obs2", sql09CanonicalObject(t, `{"system": "urn:obs2", "code": "o2"}`), "0", "true"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("owner-route rows = %v, want the duplicate occurrence expanded per edge %v", got, want) + } + + storedEdges := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_edge_occurrence" WHERE project=$1 AND dataset_generation=$2 AND label='has-observation'`, + sql09Project, sql09Generation) + if len(storedEdges) != 1 || storedEdges[0][0] != "3" { + t.Fatalf("stored edge occurrences = %v, want 3", storedEdges) + } + witnessRows := svc.witness(t, + `SELECT r0.logical_id, r1.logical_id, e.item ->> 'code' FROM "`+svc.schema+`"."fhir_resource" r0`+ + ` JOIN "`+svc.schema+`"."fhir_edge_occurrence" e0 ON e0.project=r0.project AND e0.dataset_generation=r0.dataset_generation`+ + ` AND e0.label='has-observation' AND e0.source_resource_type='Patient' AND e0.source_logical_id=r0.logical_id`+ + ` JOIN "`+svc.schema+`"."fhir_resource" r1 ON r1.project=e0.project AND r1.dataset_generation=e0.dataset_generation`+ + ` AND r1.resource_type='Observation' AND r1.logical_id=e0.target_logical_id`+ + ` CROSS JOIN LATERAL jsonb_array_elements(r1.payload -> 'component') c`+ + ` CROSS JOIN LATERAL jsonb_array_elements(c -> 'code' -> 'coding') e(item)`+ + ` WHERE r0.project=$1 AND r0.dataset_generation=$2 AND r0.logical_id='p-route'`+ + ` ORDER BY r0.resource_seq, e0.edge_seq`, + sql09Project, sql09Generation) + if len(witnessRows) != 3 { + t.Fatalf("witness rows = %v, want 3", witnessRows) + } + for index, row := range witnessRows { + if row[0] != want[index][0] || row[1] != want[index][1] { + t.Fatalf("witness row %d identity = %v, want %v", index, row, want[index]) + } + } +} + +func TestPG18UnnestOwnerRouteExcludesZeroRouteParents(t *testing.T) { + svc := newSQL09Service(t) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-route", "FHIR/Patient", sql09JSON(t, `{"gender": "female"}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-zero", "FHIR/Patient", sql09JSON(t, `{"gender": "male"}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Observation", "o-obs1", "FHIR/Observation", + sql09JSON(t, `{"component": [{"code": {"coding": [{"system": "urn:z", "code": "z1"}]}}]}`)) + svc.insertEdge(t, control.EdgeOccurrenceRow{ + Project: sql09Project, DatasetGeneration: sql09Generation, AuthResourcePath: "FHIR/Patient", + Label: "has-observation", SourceResourceType: "Patient", SourceLogicalID: "p-route", + TargetResourceType: "Observation", TargetLogicalID: "o-obs1", + SourcePath: "observation[0]", SourceOrdinal: 0, + }) + + for _, policy := range []ir.PhysicalUnnestEmptyPolicy{ir.PhysicalUnnestExclude, ir.PhysicalUnnestPreserveParent} { + plan := scopedPlan(sql09Project, sql09Generation, []string{"FHIR/Patient", "FHIR/Observation"}, false) + plan.BindVars["edge_collection"] = "fhir_edge" + plan.BindVars["edge_label"] = "has-observation" + plan.BindVars["target_type"] = "Observation" + unnest := codingExpansion(policy) + unnest.Owner = ir.PhysicalUnnestOwner{ + OccurrenceID: "obs-owner", ResourceType: "Observation", RootVariable: "root", OwnerVariable: "obs", + Route: []ir.PhysicalUnnestRouteStep{{OccurrenceID: "obs-owner", Traversal: ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "obs", EdgeVariable: "obs_edge", + Direction: ir.PhysicalOutbound, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: "edge_label", TargetTypeBindKey: "target_type", + EdgeTargetTypeField: "to_type", Strategy: ir.PhysicalTraversalNative, + }}}, + } + unnest.Expression = arrayExtract("obs", "Observation", + spec.SelectorStep{Field: "component", Iterate: true}, + spec.SelectorStep{Field: "code"}, + spec.SelectorStep{Field: "coding", Iterate: true}) + plan.Operations = append(plan.Operations, unnestOp(unnest)) + plan = withReturn(plan, + valueProjection("patient_id", "root", "logical_id"), + valueProjection("observation_id", "obs", "logical_id"), + valueProjection("item", "item"), + valueProjection("position", "position"), + valueProjection("has_item", "has_item"), + ) + rendered, err := RenderUnnest(plan, sql09Binding(t, svc)) + if err != nil { + t.Fatalf("RenderUnnest (%s): %v", policy, err) + } + got := sql09CanonicalRows(svc.runRendered(t, rendered)) + want := [][]string{ + {"p-route", "o-obs1", sql09CanonicalObject(t, `{"system": "urn:z", "code": "z1"}`), "0", "true"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("owner-route rows under %s = %v, want only the routed parent's row (p-zero has zero route occurrences and must emit no rows; its pad row would be indistinguishable from an owner-present-empty-source pad)", policy, got) + } + } + + stored := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" WHERE project=$1 AND dataset_generation=$2 AND logical_id='p-zero'`, + sql09Project, sql09Generation) + if len(stored) != 1 || stored[0][0] != "1" { + t.Fatalf("p-zero must be stored for the exclusion to be meaningful: %v", stored) + } +} + +func TestPG18UnnestDerivedItemPathsAndValueExpressionProjectLive(t *testing.T) { + svc := newSQL09Service(t) + dupA := `{"system": "urn:x", "value": "a"}` + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-dups", "FHIR/Patient", + sql09JSON(t, `{"identifier": [`+dupA+`, `+dupA+`, {"system": "urn:x", "value": "b"}, `+dupA+`]}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-scalars", "FHIR/Patient", + sql09JSON(t, `{"identifier": ["s1", "s2"]}`)) + svc.insertResource(t, "sql09-other-project", sql09Generation, "Patient", "p-scope", "FHIR/Patient", + sql09JSON(t, `{"identifier": [{"system": "urn:out", "value": "out-of-scope"}]}`)) + + plan := scopedPlan(sql09Project, sql09Generation, []string{"FHIR/Patient", "FHIR/Observation"}, false) + plan.Operations = append(plan.Operations, unnestOp(identifierExpansion(ir.PhysicalUnnestExclude))) + plan = withReturn(plan, + valueProjection("logical_id", "root", "logical_id"), + valueProjection("item_system", "item", "system"), + expressionProjection("item_value", "item", "value"), + valueProjection("position", "position"), + valueProjection("has_item", "has_item"), + ) + rendered, err := RenderUnnest(plan, sql09Binding(t, svc)) + if err != nil { + t.Fatalf("RenderUnnest: %v", err) + } + got := sql09CanonicalRows(svc.runRendered(t, rendered)) + want := [][]string{ + {"p-dups", "urn:x", "a", "0", "true"}, + {"p-dups", "urn:x", "a", "1", "true"}, + {"p-dups", "urn:x", "b", "2", "true"}, + {"p-dups", "urn:x", "a", "3", "true"}, + {"p-scalars", "", "", "0", "true"}, + {"p-scalars", "", "", "1", "true"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("derived-path rows = %v, want %v", got, want) + } + + probe := svc.sql09IsNullProbe(t, rendered, "c1", "c2") + for index, row := range probe { + wantNull := "false" + if index >= 4 { + wantNull = "true" + } + if row[0] != wantNull || row[1] != wantNull { + t.Fatalf("row %d IS NULL cells = %v, want both %s", index, row, wantNull) + } + } + + stored := svc.witness(t, + `SELECT e ->> 'system', e ->> 'value' FROM "`+svc.schema+`"."fhir_resource" r,`+ + ` jsonb_array_elements(r.payload -> 'identifier') e`+ + ` WHERE r.project=$1 AND r.dataset_generation=$2 AND r.logical_id='p-dups'`, + sql09Project, sql09Generation) + for index, row := range stored { + if row[0] != want[index][1] || row[1] != want[index][2] { + t.Fatalf("stored extraction %d = %v, want %v/%v", index, row, want[index][1], want[index][2]) + } + } + for _, row := range got { + if row[0] == "p-scope" || strings.Contains(strings.Join(row, "|"), "out-of-scope") { + t.Fatalf("derived-path projection leaked out-of-scope row: %v", row) + } + } +} + +func TestPG18UnnestTwoExpansionsFormCartesianProductPerParentRow(t *testing.T) { + svc := newSQL09Service(t) + telecom := `[{"system": "urn:t", "value": "t1"}, {"system": "urn:t", "value": "t2"}]` + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-cart", "FHIR/Patient", + sql09JSON(t, `{"identifier": [{"value": "i1"}, {"value": "i2"}], "telecom": `+telecom+`}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-single", "FHIR/Patient", + sql09JSON(t, `{"identifier": [{"value": "only"}], "telecom": `+telecom+`}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-noident", "FHIR/Patient", + sql09JSON(t, `{"telecom": [{"system": "urn:t", "value": "t1"}]}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-notel", "FHIR/Patient", + sql09JSON(t, `{"identifier": [{"value": "i1"}]}`)) + svc.insertResource(t, "sql09-other-project", sql09Generation, "Patient", "p-scope", "FHIR/Patient", + sql09JSON(t, `{"identifier": [{"value": "out-of-scope"}], "telecom": `+telecom+`}`)) + + first := identifierExpansion(ir.PhysicalUnnestExclude) + second := ir.PhysicalUnnest{ + Owner: ir.PhysicalUnnestOwner{OccurrenceID: "root", ResourceType: "Patient", + RootVariable: "root", OwnerVariable: "root"}, + OutputVariable: "telecom_item", Ordinality: "telecom_position", HasItemVariable: "telecom_has_item", + Expression: arrayExtract("root", "Patient", spec.SelectorStep{Field: "telecom", Iterate: true}), + EmptyPolicy: ir.PhysicalUnnestExclude, + } + plan := scopedPlan(sql09Project, sql09Generation, []string{"FHIR/Patient", "FHIR/Observation"}, false) + plan.Operations = append(plan.Operations, unnestOp(first), unnestOp(second)) + plan = withReturn(plan, + valueProjection("logical_id", "root", "logical_id"), + valueProjection("identifier_value", "item", "value"), + expressionProjection("telecom_value", "telecom_item", "value"), + valueProjection("telecom_system", "telecom_item", "system"), + valueProjection("position", "position"), + valueProjection("telecom_position", "telecom_position"), + ) + rendered, err := RenderUnnest(plan, sql09Binding(t, svc)) + if err != nil { + t.Fatalf("RenderUnnest: %v", err) + } + if !strings.Contains(rendered.SQL, "ORDER BY r0.\"resource_seq\" ASC, u1.rn ASC, u2.rn ASC") { + t.Fatalf("multi-expansion ORDER BY missing either rn key:\n%s", rendered.SQL) + } + got := sql09CanonicalRows(svc.runRendered(t, rendered)) + want := [][]string{ + {"p-cart", "i1", "t1", "urn:t", "0", "0"}, + {"p-cart", "i1", "t2", "urn:t", "0", "1"}, + {"p-cart", "i2", "t1", "urn:t", "1", "0"}, + {"p-cart", "i2", "t2", "urn:t", "1", "1"}, + {"p-single", "only", "t1", "urn:t", "0", "0"}, + {"p-single", "only", "t2", "urn:t", "0", "1"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("cartesian rows = %v, want %v", got, want) + } + + loop := svc.witness(t, + `SELECT r.logical_id, a.value ->> 'value', b.value ->> 'value', b.value ->> 'system',`+ + ` a.ord - 1, b.ord - 1`+ + ` FROM "`+svc.schema+`"."fhir_resource" r`+ + ` CROSS JOIN LATERAL jsonb_array_elements(r.payload -> 'identifier') WITH ORDINALITY AS a(value, ord)`+ + ` CROSS JOIN LATERAL jsonb_array_elements(r.payload -> 'telecom') WITH ORDINALITY AS b(value, ord)`+ + ` WHERE r.project=$1 AND r.dataset_generation=$2 AND r.resource_type='Patient'`+ + ` ORDER BY r.resource_seq, a.ord, b.ord`, + sql09Project, sql09Generation) + if !reflect.DeepEqual(loop, want) { + t.Fatalf("nested-loop witness = %v, want %v", loop, want) + } + + probe := svc.sql09IsNullProbe(t, rendered, "c1", "c2", "c3", "c4", "c5") + if len(probe) != len(want) { + t.Fatalf("IS NULL probe rows = %d, want %d", len(probe), len(want)) + } + for index, row := range probe { + for column, cell := range row { + if cell != "false" { + t.Fatalf("cartesian row %d column %d IS NULL = %s, want false", index, column, cell) + } + } + } + for _, row := range got { + if row[0] == "p-noident" || row[0] == "p-notel" || row[0] == "p-scope" { + t.Fatalf("INNER multi-expansion emitted an empty-source or out-of-scope parent: %v", row) + } + } +} + +func TestPG18UnnestScopeFencesExpandedRows(t *testing.T) { + svc := newSQL09Service(t) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-in", "FHIR/Patient", sql09JSON(t, `{"identifier": [{"value": "seen"}]}`)) + svc.insertResource(t, sql09Project, sql09Generation, "Patient", "p-secret", "FHIR/Secret", sql09JSON(t, `{"identifier": [{"value": "leak"}]}`)) + svc.insertResource(t, sql09Project, "gen-2", "Patient", "p-gen2", "FHIR/Patient", sql09JSON(t, `{"identifier": [{"value": "leak2"}]}`)) + + for _, policy := range []ir.PhysicalUnnestEmptyPolicy{ir.PhysicalUnnestExclude, ir.PhysicalUnnestPreserveParent} { + rendered, err := RenderUnnest(sql09PatientUnnestPlan(policy), sql09Binding(t, svc)) + if err != nil { + t.Fatalf("RenderUnnest(%s): %v", policy, err) + } + for _, row := range svc.runRendered(t, rendered) { + joined := strings.Join(row, "|") + if strings.Contains(joined, "leak") || row[0] == "p-secret" || row[0] == "p-gen2" { + t.Fatalf("%s expansion leaked a row outside scope: %v", policy, row) + } + } + } + stored := svc.witness(t, + `SELECT count(*) FROM "`+svc.schema+`"."fhir_resource" WHERE project=$1 AND logical_id IN ('p-secret','p-gen2')`, + sql09Project) + if len(stored) != 1 || stored[0][0] != "2" { + t.Fatalf("witness must show both out-of-scope rows are stored: %v", stored) + } + rendered, err := RenderUnnest(sql09PatientUnnestPlan(ir.PhysicalUnnestExclude), sql09Binding(t, svc)) + if err != nil { + t.Fatalf("RenderUnnest: %v", err) + } + if strings.Contains(rendered.SQL, "identifier") { + t.Fatalf("selector path leaked into SQL text: %s", rendered.SQL) + } +} + +func TestZZNoLeftoverSQL09Resources(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer admin.Close() + var databases int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify_sql09_%'").Scan(&databases); err != nil { + t.Fatalf("count databases: %v", err) + } + if databases != 0 { + t.Fatalf("%d verify_sql09_* databases still exist", databases) + } + var schemas int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'verify_sql09_%'").Scan(&schemas); err != nil { + t.Fatalf("count schemas: %v", err) + } + if schemas != 0 { + t.Fatalf("%d verify_sql09_* schemas still exist in the service database", schemas) + } +} diff --git a/internal/dataframe/compiler/render/postgres/unpivot.go b/internal/dataframe/compiler/render/postgres/unpivot.go new file mode 100644 index 000000000..cbd40b68e --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/unpivot.go @@ -0,0 +1,633 @@ +package postgres + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +const unpivotValueTypeTag = "TABLE_UNPIVOT_VALUE_TYPE_MISMATCH" + +const unpivotRowIDColumn = "__loom_row_id" + +var unpivotValueTypeOf = map[string]string{ + "STRING": "string", + "INTEGER": "number", + "DECIMAL": "number", + "BOOLEAN": "boolean", +} + +func RenderUnpivot(plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) (command.RenderedPostgresPhysicalPlan, error) { + if err := validateUnpivotPlanSurface(plan); err != nil { + return command.RenderedPostgresPhysicalPlan{}, err + } + return RenderPhysicalPlan(plan, storage) +} + +func validateUnpivotPlanSurface(plan ir.PhysicalPlan) error { + if len(plan.Operations) == 0 { + return fmt.Errorf("%w: plan has no operations", ErrUnsupportedPostgresPlan) + } + sawUnpivot, sawReturn := false, false + sawSort, sawLimit := false, false + var melt *ir.PhysicalUnpivot + for index, operation := range plan.Operations { + switch operation.Kind { + case ir.PhysicalRootScanOp: + if index != 0 { + return fmt.Errorf("%w: the unpivot slice leads with exactly one ROOT_SCAN at position 0, got one at %d", ErrUnsupportedPostgresPlan, index) + } + if operation.RootScan == nil { + return fmt.Errorf("%w: operation %d: ROOT_SCAN payload is missing", ErrUnsupportedPostgresPlan, index) + } + if err := validateUnpivotRootScan(plan, *operation.RootScan); err != nil { + return err + } + case ir.PhysicalUnpivotOp: + if sawUnpivot { + return fmt.Errorf("%w: operation %d: the unpivot slice registers exactly one melt; a second table reshape has no PostgreSQL rendering (multiple melts are construction stage sequences, which stay refused)", ErrUnsupportedPostgresPlan, index) + } + sawUnpivot = true + if index == 0 { + return fmt.Errorf("%w: operation %d: UNPIVOT cannot run before the rows it melts", ErrUnsupportedPostgresPlan, index) + } + if sawSort || sawLimit { + return fmt.Errorf("%w: operation %d: UNPIVOT must precede the execution window; melting after SORT/LIMIT would window the pre-melt rows the window contract orders", ErrUnsupportedPostgresPlan, index) + } + if operation.Unpivot == nil { + return fmt.Errorf("%w: operation %d: UNPIVOT payload is missing", ErrUnsupportedPostgresPlan, index) + } + if err := validateUnpivotOperationShape(plan, *operation.Unpivot); err != nil { + return fmt.Errorf("operation %d: %w", index, err) + } + melt = operation.Unpivot + case ir.PhysicalSortOp: + if operation.Sort == nil { + return fmt.Errorf("%w: operation %d: SORT payload is missing", ErrUnsupportedPostgresPlan, index) + } + if !sawUnpivot { + return fmt.Errorf("%w: operation %d: SORT must follow the melt; the reshaped window orders melted rows, not source rows", ErrUnsupportedPostgresPlan, index) + } + if sawSort || sawLimit { + return fmt.Errorf("%w: operation %d: the unpivot window admits at most one SORT, before LIMIT", ErrUnsupportedPostgresPlan, index) + } + if err := validateUnpivotSortKeys(*operation.Sort, *melt); err != nil { + return err + } + sawSort = true + case ir.PhysicalLimitOp: + if operation.Limit == nil { + return fmt.Errorf("%w: operation %d: LIMIT payload is missing", ErrUnsupportedPostgresPlan, index) + } + if !sawUnpivot { + return fmt.Errorf("%w: operation %d: LIMIT must follow the melt; the window bounds melted rows, not source rows", ErrUnsupportedPostgresPlan, index) + } + if sawLimit { + return fmt.Errorf("%w: operation %d: the unpivot window admits at most one LIMIT", ErrUnsupportedPostgresPlan, index) + } + if !sawSort { + return fmt.Errorf("%w: operation %d: a LIMIT after UNPIVOT requires the stable row identity SORT first (the ir reshaped-window guard's contract, re-read at the renderer)", ErrUnsupportedPostgresPlan, index) + } + sawLimit = true + case ir.PhysicalReturnOp: + sawReturn = true + if index != len(plan.Operations)-1 { + return fmt.Errorf("%w: RETURN must be the final operation of the unpivot slice", ErrUnsupportedPostgresPlan) + } + if operation.Return == nil { + return fmt.Errorf("%w: the terminal operation must be RETURN with its payload set", ErrUnsupportedPostgresPlan) + } + if melt == nil { + return fmt.Errorf("%w: RETURN at %d precedes the melt; the registered slice reads melt outputs after UNPIVOT", ErrUnsupportedPostgresPlan, index) + } + if err := validateUnpivotReturn(*operation.Return, *melt); err != nil { + return err + } + default: + return fmt.Errorf("%w: operation %q cannot coexist with UNPIVOT in the registered slice; filters, traversals, unnests, sets, subplans, expression lets, cell trace, and every other shape belong to the read spine or a later capability", + ErrUnsupportedPostgresPlan, string(operation.Kind)) + } + } + if !sawUnpivot { + return fmt.Errorf("%w: the unpivot slice requires exactly one UNPIVOT between ROOT_SCAN and RETURN", ErrUnsupportedPostgresPlan) + } + if !sawReturn { + return fmt.Errorf("%w: the unpivot slice requires a terminal RETURN after its melt", ErrUnsupportedPostgresPlan) + } + return nil +} + +func validateUnpivotRootScan(plan ir.PhysicalPlan, scan ir.PhysicalRootScan) error { + if scan.Population != nil || scan.CohortSource != nil || scan.PageCandidateSeed != nil { + return fmt.Errorf("%w: population, cohort, and candidate-seed roots are not registered for PostgreSQL; only the direct root collection scan has a rendered contract", + ErrUnsupportedPostgresPlan) + } + if strings.TrimSpace(scan.Variable) == "" { + return fmt.Errorf("%w: ROOT_SCAN must name its row variable", ErrUnsupportedPostgresPlan) + } + if strings.TrimSpace(scan.CollectionBindKey) == "" { + return fmt.Errorf("%w: ROOT_SCAN must name the bind carrying the single resource type", ErrUnsupportedPostgresPlan) + } + if resourceType, ok := plan.BindVars[scan.CollectionBindKey].(string); !ok || strings.TrimSpace(resourceType) == "" { + return fmt.Errorf("%w: root collection bind %q must name one resource type as a non-empty string", + ErrUnsupportedPostgresPlan, scan.CollectionBindKey) + } + return nil +} + +func validateUnpivotOperationShape(plan ir.PhysicalPlan, unpivot ir.PhysicalUnpivot) error { + if unpivot.InputRowVariable == "" || unpivot.SlotVariable == "" || unpivot.OutputRowVariable == "" { + return fmt.Errorf("%w: unpivot requires input-row, slot, and output-row variables", ErrUnsupportedPostgresPlan) + } + if unpivot.InputRowVariable == unpivot.OutputRowVariable || unpivot.SlotVariable == unpivot.OutputRowVariable || unpivot.SlotVariable == unpivot.InputRowVariable { + return fmt.Errorf("%w: unpivot variables must be distinct; they name distinct SQL placements", ErrUnsupportedPostgresPlan) + } + if len(unpivot.InputProjections) == 0 || len(unpivot.Inputs) == 0 || len(unpivot.IdentityParts) == 0 { + return fmt.Errorf("%w: unpivot requires input projections, at least one selected input, and identity parts", ErrUnsupportedPostgresPlan) + } + if strings.TrimSpace(unpivot.ConstructionID) == "" { + return fmt.Errorf("%w: unpivot construction ID is required", ErrUnsupportedPostgresPlan) + } + if _, ok := plan.BindVars[unpivot.ConstructionIDBindKey].(string); !ok { + return fmt.Errorf("%w: unpivot construction ID bind %q must carry a string", ErrUnsupportedPostgresPlan, unpivot.ConstructionIDBindKey) + } + if plan.BindVars[unpivot.ConstructionIDBindKey].(string) != unpivot.ConstructionID { + return fmt.Errorf("%w: unpivot construction ID bind does not match construction metadata", ErrUnsupportedPostgresPlan) + } + if unpivot.KeyOutput == "" || unpivot.ValueOutput == "" || unpivot.KeyOutput == unpivot.ValueOutput { + return fmt.Errorf("%w: unpivot key and value outputs must be non-empty and distinct", ErrUnsupportedPostgresPlan) + } + if _, ok := unpivotValueTypeOf[unpivot.KeyType]; !ok { + return fmt.Errorf("%w: unpivot key type %q is outside the closed scalar registry", ErrUnsupportedPostgresPlan, unpivot.KeyType) + } + if _, ok := unpivotValueTypeOf[unpivot.ValueType]; !ok { + return fmt.Errorf("%w: unpivot value type %q is outside the closed scalar registry", ErrUnsupportedPostgresPlan, unpivot.ValueType) + } + if unpivot.NullRowPolicy != "DROP" && unpivot.NullRowPolicy != "PRESERVE" { + return fmt.Errorf("%w: unsupported null-row policy %q", ErrUnsupportedPostgresPlan, unpivot.NullRowPolicy) + } + projectionNames := make(map[string]bool, len(unpivot.InputProjections)) + for _, projection := range unpivot.InputProjections { + if strings.TrimSpace(projection.Name) == "" { + return fmt.Errorf("%w: unpivot input projection names are required", ErrUnsupportedPostgresPlan) + } + if projectionNames[projection.Name] { + return fmt.Errorf("%w: unpivot input projection %q is duplicated", ErrUnsupportedPostgresPlan, projection.Name) + } + projectionNames[projection.Name] = true + } + if projectionNames[unpivot.KeyOutput] || projectionNames[unpivot.ValueOutput] { + return fmt.Errorf("%w: unpivot key or value output collides with an input projection", ErrUnsupportedPostgresPlan) + } + selected := make(map[string]bool, len(unpivot.Inputs)) + for _, input := range unpivot.Inputs { + if input.Column == "" || !projectionNames[input.Column] { + return fmt.Errorf("%w: unpivot input column %q is not an input projection", ErrUnsupportedPostgresPlan, input.Column) + } + if selected[input.Column] { + return fmt.Errorf("%w: unpivot input column %q is duplicated", ErrUnsupportedPostgresPlan, input.Column) + } + selected[input.Column] = true + if input.KeyBindKey == "" { + return fmt.Errorf("%w: unpivot input %q requires a key bind", ErrUnsupportedPostgresPlan, input.Column) + } + if input.KeyKind != unpivot.KeyType { + return fmt.Errorf("%w: unpivot input %q key kind %q does not match the canonical key type %q", + ErrUnsupportedPostgresPlan, input.Column, input.KeyKind, unpivot.KeyType) + } + if !unpivotValueKindCompatible(unpivot.ValueType, input.ValueKind) { + return fmt.Errorf("%w: unpivot input %q value kind %q is incompatible with the output type %q", + ErrUnsupportedPostgresPlan, input.Column, input.ValueKind, unpivot.ValueType) + } + } + preservedInputs := make(map[string]bool, len(unpivot.PreservedOutputs)) + preservedOutputs := make(map[string]bool, len(unpivot.PreservedOutputs)) + for _, output := range unpivot.PreservedOutputs { + if output.InputColumn == "" || output.OutputColumn == "" { + return fmt.Errorf("%w: unpivot preserved output mapping is incomplete", ErrUnsupportedPostgresPlan) + } + if !selected[output.InputColumn] && !projectionNames[output.InputColumn] { + return fmt.Errorf("%w: unpivot preserved input %q is not an input projection", ErrUnsupportedPostgresPlan, output.InputColumn) + } + if preservedInputs[output.InputColumn] || preservedOutputs[output.OutputColumn] { + return fmt.Errorf("%w: unpivot preserved output mapping duplicates %q", ErrUnsupportedPostgresPlan, output.InputColumn) + } + if output.OutputColumn == unpivot.KeyOutput || output.OutputColumn == unpivot.ValueOutput || + output.OutputColumn == unpivotRowIDColumn { + return fmt.Errorf("%w: unpivot preserved output %q collides with the key, value, or stable row identity output", + ErrUnsupportedPostgresPlan, output.OutputColumn) + } + preservedInputs[output.InputColumn], preservedOutputs[output.OutputColumn] = true, true + } + identityNames := make(map[string]bool, len(unpivot.IdentityParts)) + for _, part := range unpivot.IdentityParts { + if part.Name == "" || identityNames[part.Name] { + return fmt.Errorf("%w: unpivot identity part name is empty or duplicated", ErrUnsupportedPostgresPlan) + } + identityNames[part.Name] = true + if part.Value.Variable == unpivot.InputRowVariable { + if len(part.Value.Path) != 1 || !projectionNames[part.Value.Path[0]] { + return fmt.Errorf("%w: unpivot identity part %q references an input column that is not an input projection", + ErrUnsupportedPostgresPlan, part.Name) + } + continue + } + if part.Value.Variable != "" || len(part.Value.Path) != 0 || part.Value.BindKey == "" { + return fmt.Errorf("%w: unpivot identity part %q must reference the input row by one column or a scalar bind", + ErrUnsupportedPostgresPlan, part.Name) + } + } + return nil +} + +func unpivotValueKindCompatible(output, input string) bool { + return output == input || output == "DECIMAL" && input == "INTEGER" +} + +func validateUnpivotSortKeys(sort ir.PhysicalSort, unpivot ir.PhysicalUnpivot) error { + if len(sort.Keys) != 1 { + return fmt.Errorf("%w: the reshaped execution SORT must carry exactly the stable row identity key", ErrUnsupportedPostgresPlan) + } + key := sort.Keys[0] + if key.Variable != unpivot.OutputRowVariable || key.BindKey != "" || len(key.Path) != 1 || key.Path[0] != unpivotRowIDColumn { + return fmt.Errorf("%w: the reshaped execution SORT must sort on %s.%s exactly", + ErrUnsupportedPostgresPlan, unpivot.OutputRowVariable, unpivotRowIDColumn) + } + return nil +} + +func validateUnpivotReturn(ret ir.PhysicalReturn, unpivot ir.PhysicalUnpivot) error { + if len(ret.Projections) == 0 { + return fmt.Errorf("%w: the unpivot slice RETURN must project at least one value", ErrUnsupportedPostgresPlan) + } + legal := unpivotLegalOutputNames(unpivot) + for _, projection := range ret.Projections { + if strings.TrimSpace(projection.Name) == "" { + return fmt.Errorf("%w: projection names are required output metadata (never SQL text) and must be set", ErrUnsupportedPostgresPlan) + } + if projection.Expression != nil || projection.Presence != nil || projection.PresenceOutput { + return fmt.Errorf("%w: melt projection %q uses %s; the melt RETURN reads only registered output columns by direct value", + ErrUnsupportedPostgresPlan, projection.Name, unpivotProjectionForm(projection)) + } + if projection.Value.BindKey != "" { + return fmt.Errorf("%w: melt projection %q must read an output column, not a bind value", ErrUnsupportedPostgresPlan, projection.Name) + } + if projection.Value.Variable != unpivot.OutputRowVariable { + return fmt.Errorf("%w: melt projection %q reads variable %q; the melt RETURN reads only the melt output variable %q", + ErrUnsupportedPostgresPlan, projection.Name, projection.Value.Variable, unpivot.OutputRowVariable) + } + if len(projection.Value.Path) != 1 { + return fmt.Errorf("%w: melt projection %q must read the output column by exactly one name", ErrUnsupportedPostgresPlan, projection.Name) + } + if !legal[projection.Value.Path[0]] { + return fmt.Errorf("%w: melt projection %q reads %q, which is not a registered melt output", + ErrUnsupportedPostgresPlan, projection.Name, projection.Value.Path[0]) + } + } + return nil +} + +func unpivotProjectionForm(projection ir.PhysicalProjection) string { + switch { + case projection.PresenceOutput: + return "presence-sidecar metadata" + case projection.Presence != nil: + return "presence metadata" + default: + return fmt.Sprintf("expression kind %q", string(projection.Expression.Kind)) + } +} + +func unpivotLegalOutputNames(unpivot ir.PhysicalUnpivot) map[string]bool { + legal := map[string]bool{unpivot.KeyOutput: true, unpivot.ValueOutput: true, unpivotRowIDColumn: true} + for _, output := range unpivot.PreservedOutputs { + legal[output.OutputColumn] = true + } + return legal +} + +func (r *planRenderer) renderUnpivot(plan ir.PhysicalPlan, unpivot ir.PhysicalUnpivot) error { + if r.unpivotSeq == 0 { + if err := validateUnpivotPlanSurface(plan); err != nil { + return err + } + } + if err := validateUnpivotOperationShape(plan, unpivot); err != nil { + return err + } + if _, taken := r.unpivotOutputs[unpivot.OutputRowVariable]; taken { + return fmt.Errorf("%w: melt output variable %q is already bound; melt outputs name distinct SQL placements", + ErrUnsupportedPostgresPlan, unpivot.OutputRowVariable) + } + if _, taken := r.rows[unpivot.OutputRowVariable]; taken { + return fmt.Errorf("%w: melt output variable %q collides with a stored row variable", + ErrUnsupportedPostgresPlan, unpivot.OutputRowVariable) + } + + classified, err := classifyExtractionProjections(unpivot.InputProjections, func(variable string) (string, bool) { + scope, ok := r.rows[variable] + return scope.resourceType, ok + }) + if err != nil { + return fmt.Errorf("unpivot input projections: %w", err) + } + inputExprs := make(map[string]string, len(classified)) + for _, projection := range classified { + var expression string + switch projection.kind { + case extractionDirectValue, extractionValueExpression: + value := projection.projection.Value + if projection.kind == extractionValueExpression { + value = *projection.projection.Expression.Value + } + resolved, err := r.resolveValue(value) + if err != nil { + return fmt.Errorf("unpivot input %q: %w", projection.projection.Name, err) + } + expression = resolved + if isPayloadExpression(expression) { + expression = "(" + expression + ")" + } + case extractionSelector: + scope := r.rows[projection.expression.Extract.Source.Variable] + value, err := r.scalarExtractValueExpression(scope.alias, projection.path, projection.expression.NullBehavior) + if err != nil { + return fmt.Errorf("unpivot input %q: %w", projection.projection.Name, err) + } + expression = value + case extractionPresenceSidecar: + scope := r.rows[projection.presence.Source.Variable] + presence, err := r.propertyPresenceExpression(scope.alias, projection.presencePath) + if err != nil { + return fmt.Errorf("unpivot input %q: %w", projection.projection.Name, err) + } + expression = presence + case extractionObject, extractionSlice: + return fmt.Errorf("%w: unpivot input %q uses a %s expression; melt inputs are the spine's scalar/selector projections, composites belong to the read spine registration (objects.go)", + ErrUnsupportedPostgresPlan, projection.projection.Name, string(projection.projection.Expression.Kind)) + } + inputExprs[projection.projection.Name] = expression + } + + r.unpivotSeq++ + alias := "x" + strconv.Itoa(r.unpivotSeq) + + region, err := r.renderMeltRegion(plan, unpivot, inputExprs, alias) + if err != nil { + return err + } + r.laterals = append(r.laterals, region.lateral) + r.where = append(r.where, region.conjuncts...) + if r.unpivotOutputs == nil { + r.unpivotOutputs = map[string]map[string]string{} + } + r.unpivotOutputs[unpivot.OutputRowVariable] = region.columns + + r.window.hasUnpivot = true + r.window.unpivotOrdinals = append(r.window.unpivotOrdinals, region.ordinal) + return nil +} + +type meltRegionResult struct { + lateral string + conjuncts []string + columns map[string]string + ordinal string +} + +func (r *planRenderer) renderMeltRegion(plan ir.PhysicalPlan, unpivot ir.PhysicalUnpivot, inputExprs map[string]string, alias string) (meltRegionResult, error) { + rows := make([]string, 0, len(unpivot.Inputs)) + for _, input := range unpivot.Inputs { + raw, err := unpivotKeyJSON(plan.BindVars[input.KeyBindKey], input.KeyKind) + if err != nil { + return meltRegionResult{}, fmt.Errorf("unpivot input %q: %w", input.Column, err) + } + keyIndex := r.jsonbParameter(raw) + expression, ok := inputExprs[input.Column] + if !ok { + return meltRegionResult{}, fmt.Errorf("%w: unpivot input %q has no rendered projection expression", ErrUnsupportedPostgresPlan, input.Column) + } + rows = append(rows, fmt.Sprintf("($%d::jsonb, to_jsonb(%s), %d)", keyIndex, expression, len(rows)+1)) + } + lateral := "CROSS JOIN LATERAL (VALUES " + strings.Join(rows, ", ") + ") AS " + + alias + "(slot_key, slot_value, slot_ord)" + + slotValue := alias + ".slot_value" + valueTypeOf := unpivotValueTypeOf[unpivot.ValueType] + bad := "(" + slotValue + " IS NOT NULL AND " + slotValue + " <> 'null'::jsonb AND jsonb_typeof(" + slotValue + ") <> " + quoteSQLLiteral(valueTypeOf) + ")" + conjuncts := []string{"(CASE WHEN " + bad + " THEN " + guardCast(unpivotValueTypeTag) + " ELSE 1 END) = 1"} + if unpivot.NullRowPolicy == "DROP" { + conjuncts = append(conjuncts, "("+slotValue+" IS NOT NULL AND "+slotValue+" <> 'null'::jsonb)") + } + + columns := map[string]string{ + unpivot.KeyOutput: alias + ".slot_key", + unpivot.ValueOutput: slotValue, + } + preservedNames := make(map[string]string, len(unpivot.PreservedOutputs)) + for _, output := range unpivot.PreservedOutputs { + preservedNames[output.InputColumn] = output.OutputColumn + } + selectedColumns := make(map[string]bool, len(unpivot.Inputs)) + for _, input := range unpivot.Inputs { + selectedColumns[input.Column] = true + } + for _, projection := range unpivot.InputProjections { + if selectedColumns[projection.Name] { + continue + } + if projection.Name == unpivotRowIDColumn { + continue + } + outputName := projection.Name + if renamed, ok := preservedNames[projection.Name]; ok { + outputName = renamed + } + expression, ok := inputExprs[projection.Name] + if !ok { + return meltRegionResult{}, fmt.Errorf("%w: unpivot preserved column %q has no rendered projection expression", ErrUnsupportedPostgresPlan, projection.Name) + } + if _, taken := columns[outputName]; taken { + return meltRegionResult{}, fmt.Errorf("%w: melt output column %q is emitted twice", ErrUnsupportedPostgresPlan, outputName) + } + columns[outputName] = expression + } + columns[unpivotRowIDColumn] = r.unpivotIdentityExpression(plan, unpivot, inputExprs, alias) + if columns[unpivotRowIDColumn] == "" { + return meltRegionResult{}, fmt.Errorf("%w: unpivot identity expression failed", ErrUnsupportedPostgresPlan) + } + return meltRegionResult{lateral: lateral, conjuncts: conjuncts, columns: columns, ordinal: alias + ".slot_ord"}, nil +} + +func unpivotKeyJSON(value any, kind string) (json.RawMessage, error) { + switch kind { + case "STRING": + text, ok := value.(string) + if !ok { + return nil, fmt.Errorf("%w: unpivot key bind carries %T under STRING", ErrUnsupportedPostgresPlan, value) + } + return marshalUnpivotKey(text) + case "INTEGER": + var integer int64 + switch typed := value.(type) { + case int: + integer = int64(typed) + case int8: + integer = int64(typed) + case int16: + integer = int64(typed) + case int32: + integer = int64(typed) + case int64: + integer = typed + case uint: + integer = int64(typed) + case uint8: + integer = int64(typed) + case uint16: + integer = int64(typed) + case uint32: + integer = int64(typed) + case uint64: + if typed > 1<<53 { + return nil, fmt.Errorf("%w: unpivot INTEGER key %d exceeds the 2^53 double round-trip bound", ErrUnsupportedPostgresPlan, typed) + } + integer = int64(typed) + default: + return nil, fmt.Errorf("%w: unpivot key bind carries %T under INTEGER", ErrUnsupportedPostgresPlan, value) + } + if integer > 1<<53 || integer < -(1<<53) { + return nil, fmt.Errorf("%w: unpivot INTEGER key %d exceeds the 2^53 double round-trip bound", ErrUnsupportedPostgresPlan, integer) + } + return marshalUnpivotKey(integer) + case "DECIMAL": + var number float64 + switch typed := value.(type) { + case float32: + number = float64(typed) + case float64: + number = typed + default: + return nil, fmt.Errorf("%w: unpivot key bind carries %T under DECIMAL", ErrUnsupportedPostgresPlan, value) + } + if !(number == number) || number > 1.7976931348623157e308 || number < -1.7976931348623157e308 { + return nil, fmt.Errorf("%w: unpivot DECIMAL key bind must be finite", ErrUnsupportedPostgresPlan) + } + return marshalUnpivotKey(number) + case "BOOLEAN": + boolean, ok := value.(bool) + if !ok { + return nil, fmt.Errorf("%w: unpivot key bind carries %T under BOOLEAN", ErrUnsupportedPostgresPlan, value) + } + return marshalUnpivotKey(boolean) + default: + return nil, fmt.Errorf("%w: unpivot key kind %q is outside the closed registry", ErrUnsupportedPostgresPlan, kind) + } +} + +func marshalUnpivotKey(value any) (json.RawMessage, error) { + raw, err := json.Marshal(value) + if err != nil || string(raw) == "null" { + return nil, fmt.Errorf("%w: unpivot key is not JSON-encodable", ErrUnsupportedPostgresPlan) + } + return raw, nil +} + +func (r *planRenderer) unpivotIdentityExpression(plan ir.PhysicalPlan, unpivot ir.PhysicalUnpivot, inputExprs map[string]string, alias string) string { + pairs := make([]string, 0, len(unpivot.IdentityParts)+2) + for _, part := range unpivot.IdentityParts { + nameIndex := r.textParameter(part.Name, "unpivot-identity-name:"+part.Name) + var valueText string + if part.Value.Variable == unpivot.InputRowVariable { + expression, ok := inputExprs[part.Value.Path[0]] + if !ok { + return "" + } + valueText = "COALESCE(to_jsonb(" + expression + "), 'null'::jsonb)::text" + } else { + bound, ok := plan.BindVars[part.Value.BindKey] + if !ok { + return "" + } + text, ok := bound.(string) + if !ok { + return "" + } + bindIndex := r.textParameter(text, "unpivot-identity-bind:"+part.Value.BindKey+":"+text) + valueText = "to_jsonb($" + strconv.Itoa(bindIndex) + "::text)::text" + } + pairs = append(pairs, "'[' || to_jsonb($"+strconv.Itoa(nameIndex)+"::text) || ',' || "+valueText+" || ']'") + } + constructionIndex := r.textParameter(unpivot.ConstructionID, "unpivot-construction-id:"+unpivot.ConstructionID) + pairs = append(pairs, "'[' || to_jsonb($"+strconv.Itoa(constructionIndex)+"::text) || ',' || "+quoteSQLLiteral(`"TABLE_UNPIVOT"`)+" || ']'") + keyKind := unpivot.Inputs[0].KeyKind + pairs = append(pairs, "'[' || "+quoteSQLLiteral(`"`+keyKind+`"`)+" || ',' || "+alias+".slot_key::text || ']'") + return "'[' || " + strings.Join(pairs, " || ',' || ") + " || ']'" +} + +func (r *planRenderer) resolveUnpivotOutputValue(columns map[string]string, value ir.PhysicalValue) (string, error) { + if value.BindKey != "" { + return "", fmt.Errorf("%w: melt reads must name an output column, not a bind value", ErrUnsupportedPostgresPlan) + } + if len(value.Path) != 1 { + return "", fmt.Errorf("%w: melt reads must name exactly one output column, got %v", ErrUnsupportedPostgresPlan, value.Path) + } + expression, ok := columns[value.Path[0]] + if !ok { + return "", fmt.Errorf("%w: %q is not a registered melt output", ErrUnsupportedPostgresPlan, value.Path[0]) + } + return "(" + expression + ")", nil +} + +func (r *planRenderer) renderUnpivotReturn(ret ir.PhysicalReturn) error { + if len(ret.Projections) == 0 { + return fmt.Errorf("%w: the unpivot slice RETURN must project at least one value", ErrUnsupportedPostgresPlan) + } + for _, projection := range ret.Projections { + if strings.TrimSpace(projection.Name) == "" { + return fmt.Errorf("%w: projection names are required output metadata (never SQL text) and must be set", ErrUnsupportedPostgresPlan) + } + if projection.Expression != nil || projection.Presence != nil || projection.PresenceOutput { + return fmt.Errorf("%w: melt projection %q carries %s", ErrUnsupportedPostgresPlan, projection.Name, unpivotProjectionForm(projection)) + } + columns, ok := r.unpivotOutputs[projection.Value.Variable] + if !ok { + return fmt.Errorf("%w: melt projection %q reads variable %q, which never bound a melt", + ErrUnsupportedPostgresPlan, projection.Name, projection.Value.Variable) + } + expression, err := r.resolveUnpivotOutputValue(columns, projection.Value) + if err != nil { + return fmt.Errorf("projection %q: %w", projection.Name, err) + } + r.selects = append(r.selects, expression+` AS "c`+strconv.Itoa(len(r.selects))+`"`) + r.outputColumns = append(r.outputColumns, projection.Name) + } + return nil +} + +func (r *planRenderer) renderFinalWindowSortUnpivot(sort ir.PhysicalSort) error { + if r.window.sortSeen { + return fmt.Errorf("%w: at most one SORT", ErrUnsupportedPostgresPlan) + } + if len(r.window.unpivotOrdinals) == 0 { + return fmt.Errorf("%w: the melt window orders rows before the melt bound", ErrUnsupportedPostgresPlan) + } + if len(sort.Keys) != 1 { + return fmt.Errorf("%w: the reshaped execution SORT must carry exactly the stable row identity key", ErrUnsupportedPostgresPlan) + } + key := sort.Keys[0] + columns, ok := r.unpivotOutputs[key.Variable] + if !ok || key.BindKey != "" || len(key.Path) != 1 || key.Path[0] != unpivotRowIDColumn { + return fmt.Errorf("%w: the reshaped execution SORT must sort on the melt stable row identity column", ErrUnsupportedPostgresPlan) + } + identity, ok := columns[unpivotRowIDColumn] + if !ok { + return fmt.Errorf("%w: the melt stable row identity column is not registered", ErrUnsupportedPostgresPlan) + } + r.window.keys = append(r.window.keys, "("+identity+") ASC") + return nil +} diff --git a/internal/dataframe/compiler/render/postgres/unpivot_test.go b/internal/dataframe/compiler/render/postgres/unpivot_test.go new file mode 100644 index 000000000..32bc9379d --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/unpivot_test.go @@ -0,0 +1,933 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "reflect" + "regexp" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + unp15Project = "sql15-proj" + unp15Foreign = "sql15-proj-x" + unp15Generation = "gen-15" + unp15Auth = "FHIR/Patient" + unp15SecretAuth = "FHIR/Secret" +) + +type unp15Fixture struct { + KeyType string + ValueType string + Keys []any + Columns []string + Policy string + Window bool + Limit int + Mutate func(plan *ir.PhysicalPlan) +} + +func (f unp15Fixture) with(fallback unp15Fixture) unp15Fixture { + if f.KeyType == "" { + f.KeyType = fallback.KeyType + } + if f.ValueType == "" { + f.ValueType = fallback.ValueType + } + if f.Keys == nil { + f.Keys = fallback.Keys + } + if f.Columns == nil { + f.Columns = fallback.Columns + } + if f.Policy == "" { + f.Policy = fallback.Policy + } + return f +} + +var unp15Canonical = unp15Fixture{ + KeyType: "STRING", + ValueType: "STRING", + Keys: []any{"first", "second"}, + Columns: []string{"a", "b"}, + Policy: "PRESERVE", +} + +func unp15Plan(t *testing.T, fixture unp15Fixture) ir.PhysicalPlan { + t.Helper() + fixture = fixture.with(unp15Canonical) + if len(fixture.Keys) != len(fixture.Columns) { + t.Fatalf("fixture keys %d != columns %d", len(fixture.Keys), len(fixture.Columns)) + } + inputs := make([]ir.PhysicalUnpivotInput, 0, len(fixture.Columns)) + for index, column := range fixture.Columns { + inputs = append(inputs, ir.PhysicalUnpivotInput{ + Column: column, + KeyBindKey: fmt.Sprintf("reshape_unpivot_key_%d", index), + KeyKind: fixture.KeyType, + ValueKind: fixture.ValueType, + }) + } + projections := []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + {Name: "a", Value: ir.PhysicalValue{Variable: "root", Path: []string{"a"}}}, + {Name: "b", Value: ir.PhysicalValue{Variable: "root", Path: []string{"b"}}}, + } + preserved := make([]ir.PhysicalUnpivotOutput, 0, 1) + for _, projection := range projections { + drop := projection.Name == unpivotRowIDColumn + for _, input := range inputs { + if input.Column == projection.Name { + drop = true + } + } + if !drop { + preserved = append(preserved, ir.PhysicalUnpivotOutput{InputColumn: projection.Name, OutputColumn: projection.Name}) + } + } + unpivot := &ir.PhysicalUnpivot{ + ConstructionID: "cfg-15", + InputRowVariable: "__loom_reshape_unpivot_input_0", + SlotVariable: "__loom_reshape_unpivot_slot_0", + OutputRowVariable: "__loom_reshape_unpivot_output_0", + InputProjections: projections, + Inputs: inputs, + PreservedOutputs: preserved, + KeyOutput: "measure", + KeyType: fixture.KeyType, + ValueOutput: "amount", + ValueType: fixture.ValueType, + IdentityParts: []ir.PhysicalUnpivotIdentityPart{ + {Name: unpivotRowIDColumn, Value: ir.PhysicalValue{Variable: "__loom_reshape_unpivot_input_0", Path: []string{"logical_id"}}}, + {Name: "project", Value: ir.PhysicalValue{BindKey: "project"}}, + }, + ConstructionIDBindKey: "reshape_construction", + NullRowPolicy: fixture.Policy, + } + outputVariable := unpivot.OutputRowVariable + returns := []ir.PhysicalProjection{ + {Name: "measure", Value: ir.PhysicalValue{Variable: outputVariable, Path: []string{"measure"}}}, + {Name: "amount", Value: ir.PhysicalValue{Variable: outputVariable, Path: []string{"amount"}}}, + {Name: "logical_id", Value: ir.PhysicalValue{Variable: outputVariable, Path: []string{"logical_id"}}}, + {Name: unpivotRowIDColumn, Hidden: true, Value: ir.PhysicalValue{Variable: outputVariable, Path: []string{unpivotRowIDColumn}}}, + } + operations := []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalUnpivotOp, Unpivot: unpivot}, + } + bindVars := map[string]any{ + "root_collection": "Patient", + "project": unp15Project, + "dataset_generation": unp15Generation, + "auth_resource_paths": []string{unp15Auth}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + "reshape_construction": "cfg-15", + } + for index, key := range fixture.Keys { + bindVars[fmt.Sprintf("reshape_unpivot_key_%d", index)] = key + } + if fixture.Window { + operations = append(operations, + ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{Keys: []ir.PhysicalValue{ + {Variable: outputVariable, Path: []string{unpivotRowIDColumn}}, + }}}, + ) + if fixture.Limit > 0 { + bindVars["unpivot_limit"] = fixture.Limit + operations = append(operations, ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "unpivot_limit"}}) + } + } + operations = append(operations, ir.PhysicalOperation{Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: returns}}) + plan := ir.PhysicalPlan{Version: 1, Engine: ir.PhysicalEngineAQL, BindVars: bindVars, Operations: operations} + if fixture.Mutate != nil { + fixture.Mutate(&plan) + } + return plan +} + +func unp15Binding(t *testing.T, svc *pgir02Service) ir.PostgresStorageBinding { + t.Helper() + binding, err := ir.NewPostgresStorageBinding(svc.schema) + if err != nil { + svc.fail("storage binding", err) + } + return binding +} + +func unp15Render(t *testing.T, plan ir.PhysicalPlan, storage ir.PostgresStorageBinding) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderUnpivot(plan, storage) + if err != nil { + t.Fatalf("RenderUnpivot: %v", err) + } + return rendered +} + +func unp15Seed(t *testing.T, svc *pgir02Service, project, logicalID string, payload any) { + t.Helper() + svc.insertResource(t, svc.schema, project, unp15Generation, "Patient", logicalID, unp15Auth, payload) +} + +func unp15Rows(t *testing.T, svc *pgir02Service, plan ir.PhysicalPlan) [][]string { + t.Helper() + rendered := unp15Render(t, plan, unp15Binding(t, svc)) + return svc.runRendered(t, rendered) +} + +func unp15Multiset(rows [][]string) []string { + out := make([]string, 0, len(rows)) + for _, row := range rows { + out = append(out, strings.Join(row, "|")) + } + sort.Strings(out) + return out +} + +func unp15ExpectMultiset(t *testing.T, got [][]string, want []string) { + t.Helper() + gotSet := unp15Multiset(got) + wantSet := append([]string(nil), want...) + sort.Strings(wantSet) + if !reflect.DeepEqual(gotSet, wantSet) { + t.Fatalf("row multiset mismatch\n got %d rows %v\nwant %d rows %v", len(gotSet), gotSet, len(wantSet), wantSet) + } +} + +func unp15ExecError(svc *pgir02Service, rendered command.RenderedPostgresPhysicalPlan) error { + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + rows, err := svc.iso.Query(svc.ctx, rendered.SQL, args...) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + } + return rows.Err() +} + +func unp15ExecConnError(t *testing.T, svc *pgir02Service, conn *pgxpool.Conn, rendered command.RenderedPostgresPhysicalPlan) error { + t.Helper() + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + rows, err := conn.Query(svc.ctx, rendered.SQL, args...) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + } + return rows.Err() +} + +func unp15StateProbe(t *testing.T, svc *pgir02Service, plan ir.PhysicalPlan, amountColumn string) [][3]string { + t.Helper() + rendered := unp15Render(t, plan, unp15Binding(t, svc)) + amountIndex := -1 + for index, name := range rendered.OutputColumns { + if name == amountColumn { + amountIndex = index + } + } + if amountIndex < 0 { + t.Fatalf("output column %q not rendered", amountColumn) + } + keyIndex := -1 + for index, name := range rendered.OutputColumns { + if name == "measure" { + keyIndex = index + } + } + if keyIndex < 0 { + t.Fatalf("output column measure not rendered") + } + value := fmt.Sprintf("q.\"c%d\"", amountIndex) + sql := fmt.Sprintf( + "SELECT q.\"c%d\"::text, "+ + "CASE WHEN %s IS NULL THEN 'SQL_NULL' WHEN %s = 'null'::jsonb THEN 'JSON_NULL' ELSE 'VALUE' END, "+ + "COALESCE(%s::text, '') FROM ("+rendered.SQL+") AS q", + keyIndex, value, value, value) + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + rows := svc.witness(t, sql, args...) + out := make([][3]string, 0, len(rows)) + for _, row := range rows { + out = append(out, [3]string{row[0], row[1], row[2]}) + } + return out +} + +func TestUnpivot15GoldenStatement(t *testing.T) { + plan := unp15Plan(t, unp15Fixture{}) + if err := plan.Validate(); err != nil { + t.Fatalf("canonical melt plan invalid: %v", err) + } + binding := unp15OfflineBinding(t) + first := unp15Render(t, plan, binding) + second := unp15Render(t, plan, binding) + if first.SQL != second.SQL || !reflect.DeepEqual(first.Args, second.Args) { + t.Fatal("melt render is not deterministic") + } + if strings.Contains(first.SQL, "@") { + t.Fatalf("melt SQL carries AQL bind syntax: %s", first.SQL) + } + wantSQL := `SELECT (x1.slot_key) AS "c0", (x1.slot_value) AS "c1", (r0."logical_id") AS "c2", ` + + `('[' || '[' || to_jsonb($9::text) || ',' || COALESCE(to_jsonb(r0."logical_id"), 'null'::jsonb)::text || ']' || ',' || ` + + `'[' || to_jsonb($10::text) || ',' || to_jsonb($11::text)::text || ']' || ',' || ` + + `'[' || to_jsonb($12::text) || ',' || '"TABLE_UNPIVOT"' || ']' || ',' || ` + + `'[' || '"STRING"' || ',' || x1.slot_key::text || ']' || ']') AS "c3" ` + + `FROM "sql15_it_binding"."fhir_resource" AS r0 ` + + `CROSS JOIN LATERAL (VALUES ($7::jsonb, to_jsonb((r0."payload" #> $5::text[])), 1), ($8::jsonb, to_jsonb((r0."payload" #> $6::text[])), 2)) AS x1(slot_key, slot_value, slot_ord) ` + + `WHERE r0."project" = $1 AND r0."dataset_generation" = $2 AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb)) ` + + `AND r0."resource_type" = $4 AND (CASE WHEN (x1.slot_value IS NOT NULL AND x1.slot_value <> 'null'::jsonb AND jsonb_typeof(x1.slot_value) <> 'string') ` + + `THEN CAST('TABLE_UNPIVOT_VALUE_TYPE_MISMATCH' || COALESCE(current_setting('loom.explorer_guard_barrier', true), '') AS integer) ELSE 1 END) = 1 ` + + `ORDER BY r0."resource_seq" ASC, x1.slot_ord ASC` + if first.SQL != wantSQL { + t.Fatalf("golden SQL drift:\n got %s\nwant %s", first.SQL, wantSQL) + } + wantArgs := []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: unp15Project}, + {Type: command.PostgreSQLParameterText, Value: unp15Generation}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["FHIR/Patient"]`)}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + {Type: command.PostgreSQLParameterText, Value: "{a}"}, + {Type: command.PostgreSQLParameterText, Value: "{b}"}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`"first"`)}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`"second"`)}, + {Type: command.PostgreSQLParameterText, Value: "__loom_row_id"}, + {Type: command.PostgreSQLParameterText, Value: "project"}, + {Type: command.PostgreSQLParameterText, Value: unp15Project}, + {Type: command.PostgreSQLParameterText, Value: "cfg-15"}, + } + if !reflect.DeepEqual(first.Args, wantArgs) { + t.Fatalf("golden args = %#v, want %#v", first.Args, wantArgs) + } + wantColumns := []string{"measure", "amount", "logical_id", "__loom_row_id"} + if !reflect.DeepEqual(first.OutputColumns, wantColumns) { + t.Fatalf("output columns = %v, want %v", first.OutputColumns, wantColumns) + } +} + +func unp15OfflineBinding(t *testing.T) ir.PostgresStorageBinding { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("sql15_it_binding") + if err != nil { + t.Fatalf("storage binding: %v", err) + } + return binding +} + +func TestUnpivot15SurfaceRefusalsAreFullBackstop(t *testing.T) { + cases := []struct { + name string + fixture unp15Fixture + needle string + rawValidate bool + siblingGate bool + }{ + {"missing-payload", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { plan.Operations[1].Unpivot = nil }}, "payload is missing", false, false}, + {"second-melt", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + melt := *plan.Operations[1].Unpivot + melt.OutputRowVariable = "second_output" + plan.Operations = append(plan.Operations[:2], append([]ir.PhysicalOperation{{Kind: ir.PhysicalUnpivotOp, Unpivot: &melt}}, plan.Operations[2:]...)...) + }}, "second table reshape|multiple table reshapes", false, false}, + {"coexisting-filter", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + filter := &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", + Left: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + Right: &ir.PhysicalValue{BindKey: "compare_value"}, + ValueKind: spec.FilterString, + }} + plan.Operations = append(plan.Operations[:1], append([]ir.PhysicalOperation{{Kind: ir.PhysicalFilterOp, Filter: filter}}, plan.Operations[1:]...)...) + plan.BindVars["compare_value"] = "x" + }}, "cannot coexist with UNPIVOT", false, false}, + {"coexisting-unnest", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + plan.Operations = append(plan.Operations[:1], append([]ir.PhysicalOperation{{Kind: ir.PhysicalUnnestOp, Unnest: &ir.PhysicalUnnest{}}}, plan.Operations[1:]...)...) + }}, "cannot coexist with UNPIVOT", false, true}, + {"melt-before-scan", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + plan.Operations[0], plan.Operations[1] = plan.Operations[1], plan.Operations[0] + }}, "cannot run before the rows it melts", false, false}, + {"melt-after-window", unp15Fixture{Window: true, Limit: 3, Mutate: func(plan *ir.PhysicalPlan) { + melt := plan.Operations[1] + plan.Operations = append(plan.Operations[:1], plan.Operations[2:]...) + plan.Operations = append(plan.Operations[:len(plan.Operations)-1], append([]ir.PhysicalOperation{melt}, plan.Operations[len(plan.Operations)-1])...) + }}, "SORT must follow the melt", false, false}, + {"sort-wrong-key", unp15Fixture{Window: true, Mutate: func(plan *ir.PhysicalPlan) { + plan.Operations[2].Sort.Keys[0] = ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}} + }}, "must sort on __loom_reshape_unpivot_output_0.__loom_row_id exactly", false, false}, + {"sort-two-keys", unp15Fixture{Window: true, Mutate: func(plan *ir.PhysicalPlan) { + out := plan.Operations[1].Unpivot.OutputRowVariable + plan.Operations[2].Sort.Keys = append(plan.Operations[2].Sort.Keys, ir.PhysicalValue{Variable: out, Path: []string{unpivotRowIDColumn}}) + }}, "exactly the stable row identity key", false, false}, + {"limit-without-sort", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + plan.BindVars["unpivot_limit"] = 3 + limit := ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: "unpivot_limit"}} + returnOp := plan.Operations[len(plan.Operations)-1] + plan.Operations = append(plan.Operations[:2], append([]ir.PhysicalOperation{limit, returnOp}, plan.Operations[2:len(plan.Operations)-1]...)...) + }}, "requires the stable row identity SORT|LIMIT requires|precede the execution window", false, false}, + {"unknown-policy", unp15Fixture{Policy: "SOMETHING"}, "unsupported null-row policy", false, false}, + {"unknown-value-type", unp15Fixture{ValueType: "BLOB", Keys: []any{"first", "second"}}, "outside the closed scalar registry", false, false}, + {"key-kind-mismatch", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + plan.Operations[1].Unpivot.Inputs[0].KeyKind = "INTEGER" + }}, "canonical key type", false, false}, + {"construction-mismatch", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + plan.BindVars["reshape_construction"] = "other" + }}, "does not match construction metadata", false, false}, + {"identity-missing-column", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + plan.Operations[1].Unpivot.IdentityParts[0].Value.Path = []string{"nope"} + }}, "not an input projection|missing", false, false}, + {"identity-unknown-bind", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + plan.Operations[1].Unpivot.IdentityParts[1].Value.BindKey = "no_such_bind" + }}, "is not defined", true, false}, + {"return-unknown-name", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + out := plan.Operations[len(plan.Operations)-1].Return + out.Projections[2].Value.Path = []string{"nope"} + }}, "not a registered melt output", false, false}, + {"return-other-variable", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + out := plan.Operations[len(plan.Operations)-1].Return + out.Projections[2].Value.Variable = "root" + }}, "reads only the melt output variable", false, false}, + {"return-multi-path", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + out := plan.Operations[len(plan.Operations)-1].Return + out.Projections[0].Value.Path = []string{"measure", "nested"} + }}, "exactly one name|single name", false, false}, + {"return-binding-value", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + out := plan.Operations[len(plan.Operations)-1].Return + out.Projections[0].Value = ir.PhysicalValue{BindKey: "reshape_construction"} + }}, "not a bind value|bind value", false, false}, + {"return-expression-form", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + out := plan.Operations[len(plan.Operations)-1].Return + value := ir.PhysicalValue{Variable: "__loom_reshape_unpivot_output_0", Path: []string{"measure"}} + out.Projections[0].Expression = &ir.PhysicalExpression{Kind: ir.PhysicalValueExpression, Value: &value} + out.Projections[0].Value = ir.PhysicalValue{} + }}, "direct value|expression kind", false, false}, + {"population-root", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + plan.Operations[0].RootScan.Population = &ir.PhysicalPopulationRootSource{} + }}, "population, cohort, and candidate-seed roots", false, false}, + {"preserved-rowid-collision", unp15Fixture{Mutate: func(plan *ir.PhysicalPlan) { + melt := plan.Operations[1].Unpivot + melt.PreservedOutputs[0].OutputColumn = unpivotRowIDColumn + ret := plan.Operations[len(plan.Operations)-1].Return + ret.Projections = append(ret.Projections[:2], ir.PhysicalProjection{ + Name: unpivotRowIDColumn, Value: ir.PhysicalValue{Variable: melt.OutputRowVariable, Path: []string{unpivotRowIDColumn}}, + }) + if err := plan.Validate(); err != nil { + t.Fatalf("the reviewer repro must stay IR-legal (physical_table_shape.go checks preserved names only against key/value/preserved): %v", err) + } + }}, "collides with the key, value, or stable row identity output", false, false}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + plan := unp15Plan(t, testCase.fixture) + _, entryErr := RenderUnpivot(plan, unp15OfflineBinding(t)) + if entryErr == nil { + t.Fatalf("RenderUnpivot accepted a %s plan", testCase.name) + } + if !testCase.rawValidate && !errors.Is(entryErr, ErrUnsupportedPostgresPlan) { + t.Fatalf("RenderUnpivot %s error = %v, want ErrUnsupportedPostgresPlan", testCase.name, entryErr) + } + _, dispatchErr := RenderPhysicalPlan(plan, unp15OfflineBinding(t)) + if dispatchErr == nil { + t.Fatalf("RenderPhysicalPlan accepted a %s plan", testCase.name) + } + if !testCase.rawValidate && !errors.Is(dispatchErr, ErrUnsupportedPostgresPlan) { + t.Fatalf("RenderPhysicalPlan %s error = %v, want ErrUnsupportedPostgresPlan", testCase.name, dispatchErr) + } + if !testCase.siblingGate && entryErr.Error() != dispatchErr.Error() { + t.Fatalf("backstop message drift:\n entry %v\n dispatch %v", entryErr, dispatchErr) + } + t.Logf("refusal: %v", entryErr) + if !regexNeedle(entryErr.Error(), testCase.needle) { + t.Fatalf("%s refusal = %q, want needle %q", testCase.name, entryErr, testCase.needle) + } + }) + } +} + +func regexNeedle(haystack, needle string) bool { + for _, alternative := range strings.Split(needle, "|") { + if strings.Contains(haystack, alternative) { + return true + } + } + return false +} + +func TestUnpivot15GoldenWindowAndPolicyClauses(t *testing.T) { + binding := unp15OfflineBinding(t) + + windowed := unp15Render(t, unp15Plan(t, unp15Fixture{Window: true, Limit: 4}), binding) + const marker = " ORDER BY " + orderIndex := strings.Index(windowed.SQL, marker) + if orderIndex < 0 { + t.Fatalf("windowed statement carries no ORDER BY:\n%s", windowed.SQL) + } + order := windowed.SQL[orderIndex+len(marker):] + if !strings.Contains(order, ") ASC, r0.\"resource_seq\" ASC, x1.slot_ord ASC LIMIT $13") { + t.Fatalf("windowed ORDER BY/LIMIT = %q, want identity then resource_seq then slot_ord then LIMIT $13", order) + } + + dropped := unp15Render(t, unp15Plan(t, unp15Fixture{Policy: "DROP"}), binding) + if !strings.Contains(dropped.SQL, "WHERE") || !strings.Contains(dropped.SQL, + "AND (x1.slot_value IS NOT NULL AND x1.slot_value <> 'null'::jsonb)") { + t.Fatalf("DROP conjunct missing:\n%s", dropped.SQL) + } + if !strings.Contains(dropped.SQL, "THEN CAST('TABLE_UNPIVOT_VALUE_TYPE_MISMATCH' || COALESCE(current_setting('loom.explorer_guard_barrier', true), '') AS integer) ELSE 1 END) = 1") { + t.Fatalf("value-type witness missing or unfolded (guard barrier required):\n%s", dropped.SQL) + } + if strings.Contains(dropped.SQL, "loom.explorer_guard_barrier\") = ") { + t.Fatalf("barrier GUC leaked as a filter conjunct:\n%s", dropped.SQL) + } +} + +func TestUnpivot15ValueKindCompatibilityMirrorsIR(t *testing.T) { + for _, testCase := range []struct { + output, input string + want bool + }{ + {"STRING", "STRING", true}, + {"INTEGER", "INTEGER", true}, + {"DECIMAL", "DECIMAL", true}, + {"BOOLEAN", "BOOLEAN", true}, + {"DECIMAL", "INTEGER", true}, + {"INTEGER", "DECIMAL", false}, + {"STRING", "INTEGER", false}, + {"INTEGER", "STRING", false}, + {"BOOLEAN", "STRING", false}, + {"DECIMAL", "STRING", false}, + } { + if got := unpivotValueKindCompatible(testCase.output, testCase.input); got != testCase.want { + t.Errorf("unpivotValueKindCompatible(%q, %q) = %v, want %v (ir rule: same-kind or DECIMAL from INTEGER only)", + testCase.output, testCase.input, got, testCase.want) + } + } +} + +func TestUnpivot15KeyKindEncodingsAndBounds(t *testing.T) { + binding := unp15OfflineBinding(t) + kindFixture := func(kind string, keys ...any) unp15Fixture { + return unp15Fixture{KeyType: kind, ValueType: "STRING", Keys: keys} + } + for _, testCase := range []struct { + name string + fix unp15Fixture + want string + }{ + {"string", kindFixture("STRING", "first", "second"), `"first"`}, + {"integer", kindFixture("INTEGER", int64(7), int64(8)), "7"}, + {"boolean", kindFixture("BOOLEAN", true, false), "true"}, + {"decimal", kindFixture("DECIMAL", 1.5, 2.25), "1.5"}, + } { + t.Run(testCase.name, func(t *testing.T) { + rendered := unp15Render(t, unp15Plan(t, testCase.fix), binding) + found := false + for _, argument := range rendered.Args { + if argument.Type == command.PostgreSQLParameterJSONB && strings.Contains(string(argument.Value.(json.RawMessage)), testCase.want) { + found = true + } + } + if !found { + t.Fatalf("key JSON %q absent from %#v", testCase.want, rendered.Args) + } + }) + } + for _, testCase := range []struct { + name string + fix unp15Fixture + }{ + {"integer-above-2^53", unp15Fixture{KeyType: "INTEGER", ValueType: "STRING", Keys: []any{int64(9007199254740993), int64(8)}}}, + {"integer-below-2^53", unp15Fixture{KeyType: "INTEGER", ValueType: "STRING", Keys: []any{int64(-9007199254740993), int64(8)}}}, + } { + t.Run(testCase.name, func(t *testing.T) { + plan := unp15Plan(t, testCase.fix) + if err := plan.Validate(); err != nil { + t.Fatalf("fixture must reach the renderer: %v", err) + } + _, err := RenderUnpivot(plan, binding) + if !errors.Is(err, ErrUnsupportedPostgresPlan) || !strings.Contains(err.Error(), "round-trip bound") { + t.Fatalf("bound refusal = %v, want typed round-trip bound refusal", err) + } + }) + } +} + +func TestUnpivot15HostilePlanDataStaysParameterized(t *testing.T) { + hostile := "x'); DROP TABLE t; --\"'@bind #> {evil}" + plan := unp15Plan(t, unp15Fixture{Keys: []any{hostile, "second"}}) + if err := plan.Validate(); err != nil { + t.Fatalf("hostile fixture must be structurally valid: %v", err) + } + rendered := unp15Render(t, plan, unp15OfflineBinding(t)) + if strings.Contains(rendered.SQL, "DROP TABLE") || strings.Contains(rendered.SQL, "evil") { + t.Fatalf("hostile plan data reached SQL text:\n%s", rendered.SQL) + } + carried := false + for _, argument := range rendered.Args { + switch value := argument.Value.(type) { + case string: + if strings.Contains(value, "DROP TABLE") { + carried = true + } + case json.RawMessage: + if strings.Contains(string(value), "DROP TABLE") { + carried = true + } + } + } + if !carried { + t.Fatal("hostile key never reached the parameter list; it was dropped, not bound") + } +} + +func TestUnpivot15SpinePathUntouched(t *testing.T) { + plan := scopedPlan(unp15Project, unp15Generation, []string{unp15Auth}, false) + plan.Operations = append(plan.Operations, ir.PhysicalOperation{Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: []ir.PhysicalProjection{ + {Name: "logical_id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + }}}) + rendered, err := RenderPhysicalPlan(plan, unp15OfflineBinding(t)) + if err != nil { + t.Fatalf("spine render: %v", err) + } + if !strings.Contains(rendered.SQL, `AS "c0"`) || strings.Contains(rendered.SQL, "CROSS JOIN LATERAL (VALUES") { + t.Fatalf("spine statement mutated by the melt slice:\n%s", rendered.SQL) + } +} + +func unp15LiveService(t *testing.T) *pgir02Service { + t.Helper() + svc := newOwnedService(t, 0, "verify_sql15") + unpTrackFixtures(t, svc) + return svc +} + +func unp15SeedMeltStates(t *testing.T, svc *pgir02Service) { + t.Helper() + unp15Seed(t, svc, unp15Project, "p1", json.RawMessage(`{"a": "alpha", "b": "beta"}`)) + unp15Seed(t, svc, unp15Project, "p2", json.RawMessage(`{"b": null}`)) + unp15Seed(t, svc, unp15Project, "p3", json.RawMessage(`{"a": "gamma"}`)) + unp15Seed(t, svc, unp15Project, "p4", json.RawMessage(`{"a": "null", "b": "null"}`)) + for _, state := range []struct{ id, payload string }{ + {"p1", `{"a": "alpha", "b": "beta"}`}, + {"p2", `{"b": null}`}, + {"p3", `{"a": "gamma"}`}, + {"p4", `{"a": "null", "b": "null"}`}, + } { + rows := svc.witness(t, fmt.Sprintf( + `SELECT payload::text FROM %s.fhir_resource WHERE logical_id = $1 AND project = $2 AND dataset_generation = $3`, + `"`+svc.schema+`"`), state.id, unp15Project, unp15Generation) + if len(rows) != 1 || rows[0][0] != state.payload { + t.Fatalf("stored state %s = %v, want exactly %s — the fixture must not be vacuous", state.id, rows, state.payload) + } + } +} + +func TestUnpivot15LiveNullPolicyMatrix(t *testing.T) { + svc := unp15LiveService(t) + unp15SeedMeltStates(t, svc) + + preserved := unp15StateProbe(t, svc, unp15Plan(t, unp15Fixture{Policy: "PRESERVE"}), "amount") + wantPreserved := [][3]string{ + {`"first"`, "SQL_NULL", ""}, + {`"first"`, "VALUE", `"alpha"`}, + {`"first"`, "VALUE", `"gamma"`}, + {`"first"`, "VALUE", `"null"`}, + {`"second"`, "SQL_NULL", ""}, + {`"second"`, "JSON_NULL", "null"}, + {`"second"`, "VALUE", `"beta"`}, + {`"second"`, "VALUE", `"null"`}, + } + sortProbes := func(rows [][3]string) { + sort.Slice(rows, func(i, j int) bool { return rows[i][0]+rows[i][2]+rows[i][1] < rows[j][0]+rows[j][2]+rows[j][1] }) + } + sortProbes(preserved) + sortProbes(wantPreserved) + if !reflect.DeepEqual(preserved, wantPreserved) { + t.Fatalf("PRESERVE probe multiset = %v, want %v", preserved, wantPreserved) + } + + dropped := unp15StateProbe(t, svc, unp15Plan(t, unp15Fixture{Policy: "DROP"}), "amount") + if len(dropped) != 5 { + t.Fatalf("DROP rows = %v, want the 5 non-null slots (string \"null\" included)", dropped) + } + for _, row := range dropped { + if row[1] != "VALUE" { + t.Fatalf("DROP kept a %s slot: %v", row[1], row) + } + } +} + +func TestUnpivot15LiveValueTypeRaise(t *testing.T) { + svc := unp15LiveService(t) + unp15Seed(t, svc, unp15Project, "ok1", json.RawMessage(`{"a": "alpha", "b": "beta"}`)) + unp15Seed(t, svc, unp15Project, "bad1", json.RawMessage(`{"a": 42, "b": "beta"}`)) + rendered := unp15Render(t, unp15Plan(t, unp15Fixture{Policy: "DROP"}), unp15Binding(t, svc)) + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + if _, err := svc.iso.Exec(svc.ctx, "EXPLAIN "+rendered.SQL, args...); err != nil { + t.Fatalf("plain EXPLAIN must plan without executing the witness: %v", err) + } + _, err := svc.iso.Exec(svc.ctx, "EXPLAIN (ANALYZE, TIMING OFF, SUMMARY OFF) "+rendered.SQL, args...) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "22P02" || !strings.Contains(pgErr.Message, unpivotValueTypeTag) { + t.Fatalf("EXPLAIN ANALYZE raise = %v, want 22P02 naming %s", err, unpivotValueTypeTag) + } + t.Logf("raise under EXPLAIN ANALYZE: %s %s", pgErr.Code, pgErr.Message) + execErr := unp15ExecError(svc, rendered) + if execErr == nil { + t.Fatal("wrong-typed slot executed instead of raising") + } + conn, err := svc.iso.Acquire(svc.ctx) + if err != nil { + svc.fail("acquire barrier connection", err) + } + defer conn.Release() + barrier := fmt.Sprintf("foldproof-%d", time.Now().UnixNano()) + if _, err := conn.Exec(svc.ctx, "SET loom.explorer_guard_barrier = "+quoteSQLLiteral(barrier)); err != nil { + t.Fatalf("set barrier GUC: %v", err) + } + var barrierErr *pgconn.PgError + drainErr := unp15ExecConnError(t, svc, conn, rendered) + if !errors.As(drainErr, &barrierErr) || barrierErr.Code != "22P02" || + !strings.Contains(barrierErr.Message, unpivotValueTypeTag) || !strings.Contains(barrierErr.Message, barrier) { + t.Fatalf("barrier raise = %v, want 22P02 naming %s and the injected barrier %q", drainErr, unpivotValueTypeTag, barrier) + } + t.Logf("fold-control raise with injected barrier: %s %s", barrierErr.Code, barrierErr.Message) + unp15Seed(t, svc, unp15Foreign, "bad-other", json.RawMessage(`{"a": 42}`)) + svc.witness(t, `DELETE FROM "`+svc.schema+`".fhir_resource WHERE logical_id = 'bad1'`) + rows := svc.runRendered(t, rendered) + if len(rows) != 2 { + t.Fatalf("control rows after removing the violation = %v, want the two ordinary slots", rows) + } +} + +func TestUnpivot15LiveKeyTypingAndAuthoredOrder(t *testing.T) { + svc := unp15LiveService(t) + unp15Seed(t, svc, unp15Project, "p1", json.RawMessage(`{"a": "alpha", "b": "beta"}`)) + fixture := unp15Fixture{KeyType: "INTEGER", ValueType: "STRING", Keys: []any{int64(101), int64(202)}, Window: true} + rows := unp15Rows(t, svc, unp15Plan(t, fixture)) + want := []string{ + `101|alpha|p1|[["__loom_row_id","p1"],["project","sql15-proj"],["cfg-15","TABLE_UNPIVOT"],["INTEGER",101]]`, + `202|beta|p1|[["__loom_row_id","p1"],["project","sql15-proj"],["cfg-15","TABLE_UNPIVOT"],["INTEGER",202]]`, + } + unp15ExpectMultiset(t, rows, want) + + swapped := unp15Fixture{KeyType: "INTEGER", ValueType: "STRING", Keys: []any{int64(202), int64(101)}, Window: true} + swappedRows := unp15Rows(t, svc, unp15Plan(t, swapped)) + swappedWant := []string{ + `101|beta|p1|[["__loom_row_id","p1"],["project","sql15-proj"],["cfg-15","TABLE_UNPIVOT"],["INTEGER",101]]`, + `202|alpha|p1|[["__loom_row_id","p1"],["project","sql15-proj"],["cfg-15","TABLE_UNPIVOT"],["INTEGER",202]]`, + } + unp15ExpectMultiset(t, swappedRows, swappedWant) + if reflect.DeepEqual(rows, swappedRows) { + t.Fatal("authored order is not observable: permutation changed nothing") + } +} + +func TestUnpivot15LiveWindowBoundsMeltedRows(t *testing.T) { + svc := unp15LiveService(t) + unp15Seed(t, svc, unp15Project, "p1", json.RawMessage(`{"a": "alpha", "b": "beta"}`)) + control := unp15Rows(t, svc, unp15Plan(t, unp15Fixture{Window: true})) + if len(control) != 2 { + t.Fatalf("unbounded window rows = %v, want 2 melted rows", control) + } + bounded := unp15Rows(t, svc, unp15Plan(t, unp15Fixture{Window: true, Limit: 1})) + if len(bounded) != 1 { + t.Fatalf("bounded window rows = %v, want exactly 1", bounded) + } + repeat := unp15Rows(t, svc, unp15Plan(t, unp15Fixture{Window: true, Limit: 1})) + if !reflect.DeepEqual(bounded, repeat) { + t.Fatalf("bounded window is not deterministic across executions:\n%v\n%v", bounded, repeat) + } + rendered := unp15Render(t, unp15Plan(t, unp15Fixture{Window: true, Limit: 1}), unp15Binding(t, svc)) + args := make([]any, len(rendered.Args)) + for index, parameter := range rendered.Args { + args[index] = parameter.Value + } + planRows := svc.witness(t, "EXPLAIN (ANALYZE, FORMAT JSON) "+rendered.SQL, args...) + if len(planRows) == 0 { + t.Fatalf("EXPLAIN ANALYZE returned no plan rows:\n%s", planRows) + } + dump := planRows[0][0] + limitAt, valuesAt := strings.Index(dump, "Node Type:Limit"), strings.Index(dump, "Node Type:Values Scan") + if limitAt < 0 || valuesAt < 0 || limitAt > valuesAt { + t.Fatalf("EXPLAIN ANALYZE plan lacks a Limit node above the VALUES expansion (limitAt=%d valuesAt=%d):\n%s", limitAt, valuesAt, dump) + } +} + +func TestUnpivot15LivePreservedAndIdentityStability(t *testing.T) { + svc := unp15LiveService(t) + unp15Seed(t, svc, unp15Project, "p1", json.RawMessage(`{"a": "alpha", "b": "beta"}`)) + plan := unp15Plan(t, unp15Fixture{}) + rendered := unp15Render(t, plan, unp15Binding(t, svc)) + rows := svc.runRendered(t, rendered) + want := []string{ + `first|alpha|p1|[["__loom_row_id","p1"],["project","sql15-proj"],["cfg-15","TABLE_UNPIVOT"],["STRING","first"]]`, + `second|beta|p1|[["__loom_row_id","p1"],["project","sql15-proj"],["cfg-15","TABLE_UNPIVOT"],["STRING","second"]]`, + } + unp15ExpectMultiset(t, rows, want) + again := svc.runRendered(t, rendered) + if !reflect.DeepEqual(rows, again) { + t.Fatalf("stable row identity drifted:\n%v\n%v", rows, again) + } + probe := svc.witness(t, + `SELECT ('[' || '[' || '"__loom_row_id"' || ',' || '"p1"' || ']' || ',' || '[' || '"project"' || ',' || '"sql15-proj"' || ']' || ',' || `+ + `'[' || '"cfg-15"' || ',' || '"TABLE_UNPIVOT"' || ']' || ',' || '[' || '"STRING"' || ',' || '"first"' || ']' || ']')::text`) + renderedIdentity := "" + for _, row := range rows { + if row[0] == "first" { + renderedIdentity = row[3] + } + } + if renderedIdentity == "" { + t.Fatalf("first-slot row missing from %v", rows) + } + if probe[0][0] != renderedIdentity { + t.Fatalf("identity witness drift:\n direct SQL %s\n rendered %s", probe[0][0], renderedIdentity) + } +} + +func TestUnpivot15LiveScopeControls(t *testing.T) { + svc := unp15LiveService(t) + unp15Seed(t, svc, unp15Project, "p1", json.RawMessage(`{"a": "alpha", "b": "beta"}`)) + plan := unp15Plan(t, unp15Fixture{}) + plan.BindVars["auth_resource_paths"] = []string{unp15SecretAuth} + if rows := svc.runRendered(t, unp15Render(t, plan, unp15Binding(t, svc))); len(rows) != 0 { + t.Fatalf("unauthorized auth path returned %v, want zero rows", rows) + } + if rows := unp15Rows(t, svc, unp15Plan(t, unp15Fixture{})); len(rows) != 2 { + t.Fatalf("authorized control rows = %v, want 2", rows) + } + denied := unp15Plan(t, unp15Fixture{}) + denied.BindVars["scope_allowed"] = false + if rows := svc.runRendered(t, unp15Render(t, denied, unp15Binding(t, svc))); len(rows) != 0 { + t.Fatalf("canonical deny returned %v, want zero rows", rows) + } +} + +func TestUnpivot15LiveArgumentsPinned(t *testing.T) { + svc := unp15LiveService(t) + unp15Seed(t, svc, unp15Project, "p1", json.RawMessage(`{"a": "alpha", "b": "beta"}`)) + rendered := unp15Render(t, unp15Plan(t, unp15Fixture{Window: true, Limit: 1}), unp15Binding(t, svc)) + if len(rendered.Args) != 13 { + t.Fatalf("melt argument count = %d, want the 12 canonical plus the bound limit: %#v", len(rendered.Args), rendered.Args) + } + if last := rendered.Args[12]; last.Type != command.PostgreSQLParameterBigint || last.Value != int64(1) { + t.Fatalf("limit argument = %#v, want bigint 1", last) + } + rows := svc.runRendered(t, rendered) + if len(rows) != 1 || rows[0][0] != "first" { + t.Fatalf("deterministic first row = %v, want the authored-first slot", rows) + } +} + +var ( + unpFixturesMu sync.Mutex + unpTrackedFixtures = map[string]bool{} +) + +func unpTrackFixtures(t *testing.T, svc *pgir02Service) { + t.Helper() + var database string + if err := svc.iso.QueryRow(svc.ctx, "SELECT current_database()").Scan(&database); err != nil { + svc.fail("track owned database", err) + } + unpFixturesMu.Lock() + defer unpFixturesMu.Unlock() + unpTrackedFixtures[database] = true + unpTrackedFixtures[svc.schema] = true +} + +func unpRunDeltaAudit(inner func() int) int { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return inner() + } + beforeList, err := unpSnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-15 delta audit: snapshot before run: %v\n", err) + return 1 + } + before := make(map[string]bool, len(beforeList)) + for _, name := range beforeList { + before[name] = true + } + code := inner() + after, err := unpSnapshotOwnedResources(context.Background(), raw) + if err != nil { + fmt.Fprintf(os.Stderr, "pg-sql-15 delta audit: snapshot after run: %v\n", err) + code = 1 + } + unpFixturesMu.Lock() + defer unpFixturesMu.Unlock() + for _, name := range after { + if before[name] { + continue + } + if unpTrackedFixtures[name] { + fmt.Fprintf(os.Stderr, "pg-sql-15 delta audit: %q was created and tracked by this run and still survives\n", name) + code = 1 + continue + } + fmt.Fprintf(os.Stderr, "pg-sql-15 delta audit: note — %q was born during this run but is not owned by it (sibling lane?)\n", name) + } + return code +} + +func unpSnapshotOwnedResources(ctx context.Context, raw string) ([]string, error) { + pool, err := pgxpool.New(ctx, raw) + if err != nil { + return nil, err + } + defer pool.Close() + queryCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + var names []string + for _, query := range []string{ + `SELECT datname FROM pg_database WHERE datname LIKE 'verify\_sql15\_%' ESCAPE '\'`, + `SELECT nspname FROM pg_namespace WHERE nspname LIKE 'verify\_sql15\_%' ESCAPE '\'`, + } { + rows, err := pool.Query(queryCtx, query) + if err != nil { + return nil, err + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + return nil, err + } + names = append(names, name) + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, err + } + rows.Close() + } + return names, nil +} + +var _ = regexp.MustCompile diff --git a/internal/dataframe/compiler/render/postgres/window.go b/internal/dataframe/compiler/render/postgres/window.go new file mode 100644 index 000000000..2fbd92684 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/window.go @@ -0,0 +1,236 @@ +package postgres + +import ( + "fmt" + "strconv" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +type executionWindow struct { + keys []string + limitClause string + sortSeen bool + limitSeen bool + hasUnnest bool + hasUnpivot bool + unpivotOrdinals []string + edgeVars map[string]string +} + +func (w *executionWindow) rememberEdge(edgeVariable, edgeAlias string) error { + if edgeVariable == "" { + return nil + } + if w.edgeVars == nil { + w.edgeVars = map[string]string{} + } + if previous, taken := w.edgeVars[edgeVariable]; taken { + return fmt.Errorf("%w: edge variable %q was already joined as %q; a stable occurrence witness needs one join per edge variable", + ErrUnsupportedPostgresPlan, edgeVariable, previous) + } + w.edgeVars[edgeVariable] = edgeAlias + return nil +} + +func (r *planRenderer) renderWindowSort(plan ir.PhysicalPlan, sort ir.PhysicalSort) error { + if r.window.sortSeen { + return fmt.Errorf("%w: at most one SORT", ErrUnsupportedPostgresPlan) + } + if r.window.hasUnnest { + if err := r.renderFinalWindowSort(plan, sort); err != nil { + return err + } + } else if r.window.hasUnpivot { + if err := r.renderFinalWindowSortUnpivot(sort); err != nil { + return err + } + } else { + for _, key := range sort.Keys { + scope, ok := r.rows[key.Variable] + if !ok { + return fmt.Errorf("%w: sort variable %q does not bind a readable row", ErrUnsupportedPostgresPlan, key.Variable) + } + if scope.optional { + return fmt.Errorf("%w: sort key on %q, a row materialized by an OPTIONAL traversal LEFT JOIN LATERAL, is not registered; sorting expanded child rows is a later capability", + ErrUnsupportedPostgresPlan, key.Variable) + } + if key.BindKey != "" || len(key.Path) != 1 { + return fmt.Errorf("%w: sort keys must be single typed columns; payload-path sorting is not yet implemented", + ErrUnsupportedPostgresPlan) + } + column := canonicalKeyColumn(key.Path[0]) + if !resourceFilterColumns[column] { + return fmt.Errorf("%w: sort keys must be single typed columns; payload-path sorting is not yet implemented", + ErrUnsupportedPostgresPlan) + } + r.window.keys = append(r.window.keys, scope.alias+"."+r.storage.Resource.MustColumn(column).SQLText()+" ASC") + } + } + r.window.sortSeen = true + return nil +} + +func (r *planRenderer) renderFinalWindowSort(plan ir.PhysicalPlan, sort ir.PhysicalSort) error { + position := 0 + expansion := 0 + for _, operation := range plan.Operations { + if operation.Kind != ir.PhysicalUnnestOp { + continue + } + unnest := operation.Unnest + if unnest == nil { + return fmt.Errorf("%w: UNNEST payload is missing while the window orders the expansion set", ErrUnsupportedPostgresPlan) + } + lateral, ok := r.lateralAliasAt(expansion) + if !ok { + return fmt.Errorf("%w: stable row identity references expansion %d before its lateral bound", ErrUnsupportedPostgresPlan, expansion+1) + } + expansion++ + consumed, err := r.renderIdentitySegment(*unnest, lateral, sort.Keys, position) + if err != nil { + return err + } + position += consumed + } + if position != len(sort.Keys) { + return fmt.Errorf("%w: SORT after UNNEST must carry the complete stable row identity tuple and nothing else: %d keys given for %d expected", + ErrUnsupportedPostgresPlan, len(sort.Keys), position) + } + return nil +} + +func (r *planRenderer) renderIdentitySegment(unnest ir.PhysicalUnnest, lateral string, keys []ir.PhysicalValue, position int) (int, error) { + want := ir.PhysicalUnnestSortKeys(unnest) + if position+len(want) > len(keys) { + return 0, fmt.Errorf("%w: SORT after UNNEST must carry the complete stable row identity tuple and nothing else: %d keys given for %d expected", + ErrUnsupportedPostgresPlan, len(keys), position+len(want)) + } + consumed := 0 + check := func(wanted ir.PhysicalValue) error { + got := keys[position+consumed] + if !sameWindowIdentityValue(got, wanted) { + return fmt.Errorf("%w: SORT key %d does not match the stable row identity tuple", ErrUnsupportedPostgresPlan, position+consumed) + } + consumed++ + return nil + } + rootTerm, err := r.windowRowIdentityTerm(unnest.Owner.RootVariable, "root") + if err != nil { + return 0, err + } + if err := check(want[consumed]); err != nil { + return 0, err + } + r.window.keys = append(r.window.keys, rootTerm) + for _, step := range unnest.Owner.Route { + if step.Traversal.EdgeVariable == "" { + continue + } + if err := check(want[consumed]); err != nil { + return 0, err + } + edgeAlias, ok := r.window.edgeVars[step.Traversal.EdgeVariable] + if !ok { + return 0, fmt.Errorf("%w: stable row identity edge %q was not joined", ErrUnsupportedPostgresPlan, step.Traversal.EdgeVariable) + } + r.window.keys = append(r.window.keys, edgeAlias+"."+r.storage.Edge.MustColumn("edge_seq").SQLText()+" ASC") + } + if unnest.Owner.OwnerVariable != unnest.Owner.RootVariable { + ownerTerm, err := r.windowRowIdentityTerm(unnest.Owner.OwnerVariable, "owner") + if err != nil { + return 0, err + } + if err := check(want[consumed]); err != nil { + return 0, err + } + r.window.keys = append(r.window.keys, ownerTerm) + } + if err := check(want[consumed]); err != nil { + return 0, err + } + r.window.keys = append(r.window.keys, "("+lateral+".rn IS NOT NULL) ASC") + if unnest.Ordinality != "" { + if err := check(want[consumed]); err != nil { + return 0, err + } + r.window.keys = append(r.window.keys, lateral+".rn ASC") + } + return consumed, nil +} + +func (r *planRenderer) windowRowIdentityTerm(variable, role string) (string, error) { + scope, ok := r.rows[variable] + if !ok || scope.derivedValue != "" || scope.alias == "" { + return "", fmt.Errorf("%w: stable row identity %s variable %q does not bind a readable stored row", + ErrUnsupportedPostgresPlan, role, variable) + } + return scope.alias + "." + r.storage.Resource.MustColumn("logical_id").SQLText() + " ASC", nil +} + +func (r *planRenderer) lateralAliasAt(expansionIndex int) (string, bool) { + if expansionIndex < 0 || expansionIndex >= len(r.lateralAliases) { + return "", false + } + return r.lateralAliases[expansionIndex], true +} + +func sameWindowIdentityValue(got, wanted ir.PhysicalValue) bool { + if got.Variable != wanted.Variable || got.BindKey != wanted.BindKey || len(got.Path) != len(wanted.Path) { + return false + } + for index := range wanted.Path { + if got.Path[index] != wanted.Path[index] { + return false + } + } + return true +} + +func (r *planRenderer) renderWindowLimit(plan ir.PhysicalPlan, limit ir.PhysicalLimit) error { + if r.window.limitSeen { + return fmt.Errorf("%w: at most one LIMIT", ErrUnsupportedPostgresPlan) + } + if r.window.hasUnnest && !r.window.sortSeen { + return fmt.Errorf("%w: a LIMIT after UNNEST requires the complete stable row identity SORT first", ErrUnsupportedPostgresPlan) + } + value, ok := plan.BindVars[limit.BindKey].(int) + if !ok || value <= 0 { + return fmt.Errorf("%w: limit bind %q must be a positive int", ErrUnsupportedPostgresPlan, limit.BindKey) + } + index := r.addParameter(command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(value)}, "") + r.window.limitClause = " LIMIT $" + strconv.Itoa(index) + r.window.limitSeen = true + return nil +} + +func (r *planRenderer) windowOrder() []string { + terms := make([]string, 0, len(r.window.keys)+2+len(r.edgeAliases)+2*len(r.lateralAliases)) + seen := make(map[string]bool, cap(terms)) + dedupe := len(r.lateralAliases) > 0 + add := func(term string) { + if dedupe && seen[term] { + return + } + seen[term] = true + terms = append(terms, term) + } + for _, key := range r.window.keys { + add(key) + } + add(r.rootAlias + "." + r.storage.Resource.MustColumn("resource_seq").SQLText() + " ASC") + for _, edgeAlias := range r.edgeAliases { + add(edgeAlias + "." + r.storage.Edge.MustColumn("edge_seq").SQLText() + " ASC") + } + for _, lateral := range r.lateralAliases { + if r.window.sortSeen { + add("(" + lateral + ".rn IS NOT NULL) ASC") + } + add(lateral + ".rn ASC") + } + for _, ordinal := range r.window.unpivotOrdinals { + terms = append(terms, ordinal+" ASC") + } + return terms +} diff --git a/internal/dataframe/compiler/render/postgres/window_test.go b/internal/dataframe/compiler/render/postgres/window_test.go new file mode 100644 index 000000000..925805343 --- /dev/null +++ b/internal/dataframe/compiler/render/postgres/window_test.go @@ -0,0 +1,1048 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + control "github.com/calypr/loom/internal/backends/postgres/control" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +func windowTuple(unnest ir.PhysicalUnnest) []ir.PhysicalValue { + return ir.PhysicalUnnestSortKeys(unnest) +} + +func windowSortOp(keys ...ir.PhysicalValue) ir.PhysicalOperation { + return ir.PhysicalOperation{Kind: ir.PhysicalSortOp, Sort: &ir.PhysicalSort{Keys: keys}} +} + +func windowLimitOp(bind string) ir.PhysicalOperation { + return ir.PhysicalOperation{Kind: ir.PhysicalLimitOp, Limit: &ir.PhysicalLimit{BindKey: bind}} +} + +func windowReturnOp(projections ...ir.PhysicalProjection) ir.PhysicalOperation { + return ir.PhysicalOperation{Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}} +} + +func windowedUnnestPlan(unnest ir.PhysicalUnnest, limit int, projections ...ir.PhysicalProjection) ir.PhysicalPlan { + plan := basePlan() + plan.BindVars["limit"] = limit + plan.Operations = []ir.PhysicalOperation{ + plan.Operations[0], + unnestOp(unnest), + windowSortOp(windowTuple(unnest)...), + windowLimitOp("limit"), + windowReturnOp(projections...), + } + return plan +} + +func windowOrderByClause(t *testing.T, sql string) string { + t.Helper() + const marker = " ORDER BY " + start := strings.Index(sql, marker) + if start < 0 { + t.Fatalf("statement carries no ORDER BY:\n%s", sql) + } + rest := sql[start+len(marker):] + if limit := strings.Index(rest, " LIMIT $"); limit >= 0 { + rest = rest[:limit] + } + return rest +} + +func requireWindowRefusal(t *testing.T, plan ir.PhysicalPlan, context string) { + t.Helper() + _, entryErr := RenderUnnest(plan, mustBinding(t)) + if !errors.Is(entryErr, ErrUnsupportedPostgresPlan) { + t.Fatalf("%s: RenderUnnest error = %v, want ErrUnsupportedPostgresPlan", context, entryErr) + } + sawUnnest := false + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalUnnestOp { + sawUnnest = true + } + } + if !sawUnnest { + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); err == nil { + t.Fatalf("%s: spine dispatch accepted a shape that must fail closed", context) + } + return + } + _, dispatchErr := RenderPhysicalPlan(plan, mustBinding(t)) + if !errors.Is(dispatchErr, ErrUnsupportedPostgresPlan) { + t.Fatalf("%s: core dispatch error = %v, want ErrUnsupportedPostgresPlan", context, dispatchErr) + } + if entryErr.Error() != dispatchErr.Error() { + t.Fatalf("%s: dispatch error %q diverged from entry error %q", context, dispatchErr, entryErr) + } +} + +func TestWindowSpineSortLimitSQLIsByteIdentical(t *testing.T) { + plan := basePlan() + plan.Operations = []ir.PhysicalOperation{ + plan.Operations[0], + windowSortOp(ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}), + windowLimitOp("limit"), + windowReturnOp( + valueProjection("logical_id", "root", "logical_id"), + valueProjection("status", "root", "status"), + ), + } + rendered := renderOK(t, plan) + wantSQL := `SELECT r0."logical_id" AS "c0", (r0."payload" #> $6::text[]) AS "c1"` + + ` FROM "pgir02_it_unit"."fhir_resource" AS r0` + + ` WHERE r0."project" = $1 AND r0."dataset_generation" = $2` + + ` AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb))` + + ` AND r0."resource_type" = $4` + + ` ORDER BY r0."logical_id" ASC, r0."resource_seq" ASC LIMIT $5` + if rendered.SQL != wantSQL { + t.Fatalf("window.go drifted the spine golden:\n got %s\nwant %s", rendered.SQL, wantSQL) + } + if len(rendered.Args) != 6 || rendered.Args[4].Type != command.PostgreSQLParameterBigint || rendered.Args[4].Value != int64(10) { + t.Fatalf("window args drifted: %#v", rendered.Args) + } +} + +func TestWindowSpineLimitWithoutSortStaysDeterministic(t *testing.T) { + plan := basePlan() + plan.Operations = []ir.PhysicalOperation{ + plan.Operations[0], + windowLimitOp("limit"), + windowReturnOp(valueProjection("logical_id", "root", "logical_id")), + } + rendered := renderOK(t, plan) + if got := windowOrderByClause(t, rendered.SQL); got != `r0."resource_seq" ASC` { + t.Fatalf("bare-LIMIT ORDER BY = %q, want the resource_seq witness", got) + } + if !strings.HasSuffix(rendered.SQL, " LIMIT $5") { + t.Fatalf("bare-LIMIT statement lost its bound limit:\n%s", rendered.SQL) + } +} + +func TestWindowSpineDuplicateSortKeysAreEmittedVerbatim(t *testing.T) { + plan := basePlan() + plan.Operations = []ir.PhysicalOperation{ + plan.Operations[0], + windowSortOp( + ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + ), + windowReturnOp(valueProjection("logical_id", "root", "logical_id")), + } + rendered := renderOK(t, plan) + if got, want := windowOrderByClause(t, rendered.SQL), + `r0."logical_id" ASC, r0."logical_id" ASC, r0."resource_seq" ASC`; got != want { + t.Fatalf("duplicate-key spine ORDER BY =\n got %q\nwant %q", got, want) + } +} + +func TestWindowEdgeVariableRebindIsRefusedAtRegistration(t *testing.T) { + t.Run("plan surface refuses duplicate edge variable in both contexts", func(t *testing.T) { + spine := basePlan() + spine.BindVars["edge_collection"] = "fhir_edge" + spine.BindVars["edge_label"] = "has-specimen" + spine.BindVars["target_type"] = "Specimen" + traversal := func(target string) ir.PhysicalOperation { + return ir.PhysicalOperation{Kind: ir.PhysicalTraversalOp, Traversal: &ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: target, EdgeVariable: "edge", + Direction: ir.PhysicalOutbound, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: "edge_label", TargetTypeBindKey: "target_type", + EdgeTargetTypeField: "to_type", Strategy: ir.PhysicalTraversalNative, + }} + } + spine.Operations = []ir.PhysicalOperation{ + spine.Operations[0], traversal("target_a"), traversal("target_b"), + windowReturnOp(valueProjection("logical_id", "root", "logical_id")), + } + requireRebindRefusal(t, spine, `variable "edge" is already defined`) + + routeTraversal := func(occurrence, target string) ir.PhysicalUnnestRouteStep { + return ir.PhysicalUnnestRouteStep{OccurrenceID: occurrence, Traversal: ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: target, EdgeVariable: "obs_edge", + Direction: ir.PhysicalOutbound, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: "edge_label", TargetTypeBindKey: "target_type", + EdgeTargetTypeField: "to_type", Strategy: ir.PhysicalTraversalNative, + }} + } + routed := func(occurrence, owner, output, ordinality, hasItem, ancestor string) ir.PhysicalUnnest { + unnest := codingExpansion(ir.PhysicalUnnestExclude) + unnest.Owner = ir.PhysicalUnnestOwner{OccurrenceID: occurrence, ResourceType: "Observation", + RootVariable: "root", OwnerVariable: owner, Route: []ir.PhysicalUnnestRouteStep{routeTraversal(occurrence, owner)}} + unnest.Expression = arrayExtract(owner, "Observation", + spec.SelectorStep{Field: "component", Iterate: true}, + spec.SelectorStep{Field: "code"}, + spec.SelectorStep{Field: "coding", Iterate: true}) + unnest.OutputVariable, unnest.Ordinality, unnest.HasItemVariable = output, ordinality, hasItem + unnest.Ancestors = []ir.PhysicalUnnestAncestor{{StepIndex: 0, Variable: ancestor}} + return unnest + } + windowless := basePlan() + windowless.BindVars["edge_collection"] = "fhir_edge" + windowless.BindVars["edge_label"] = "has-observation" + windowless.BindVars["target_type"] = "Observation" + windowless.Operations = []ir.PhysicalOperation{ + windowless.Operations[0], + unnestOp(routed("obs-a-owner", "obs_a", "item_a", "position_a", "has_item_a", "component_item_a")), + unnestOp(routed("obs-b-owner", "obs_b", "item_b", "position_b", "has_item_b", "component_item_b")), + windowReturnOp(valueProjection("logical_id", "root", "logical_id")), + } + requireRebindRefusal(t, windowless, `variable "obs_edge" is already defined`) + }) + + t.Run("registration arm refuses a rebind and names the first join", func(t *testing.T) { + for _, context := range []struct{ name, variable, first, second string }{ + {"spine join rebind", "edge", "e1", "e3"}, + {"unnest route rebind", "obs_edge", "e2", "e4"}, + } { + window := &executionWindow{} + if err := window.rememberEdge(context.variable, context.first); err != nil { + t.Fatalf("%s: first registration: %v", context.name, err) + } + err := window.rememberEdge(context.variable, context.second) + if !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("%s: rebind error = %v, want ErrUnsupportedPostgresPlan", context.name, err) + } + if want := `edge variable "` + context.variable + `"`; !strings.Contains(err.Error(), want) { + t.Fatalf("%s: rebind error %q must name the variable", context.name, err) + } + if want := `already joined as "` + context.first + `"`; !strings.Contains(err.Error(), want) { + t.Fatalf("%s: rebind error %q must name the first join alias", context.name, err) + } + } + window := &executionWindow{} + if err := window.rememberEdge("", "e9"); err != nil { + t.Fatalf("untyped (edgeless) traversal must be a no-op: %v", err) + } + if len(window.edgeVars) != 0 { + t.Fatalf("edgeless traversal registered %v", window.edgeVars) + } + }) +} + +func requireRebindRefusal(t *testing.T, plan ir.PhysicalPlan, wantFragment string) { + t.Helper() + sawUnnest := false + for _, operation := range plan.Operations { + if operation.Kind == ir.PhysicalUnnestOp { + sawUnnest = true + } + } + check := func(name string, err error) { + if err == nil || !strings.Contains(err.Error(), wantFragment) { + t.Fatalf("%s: %s error = %v, want plan-surface refusal containing %q", t.Name(), name, err, wantFragment) + } + } + if sawUnnest { + _, entryErr := RenderUnnest(plan, mustBinding(t)) + check("entry", entryErr) + } + _, dispatchErr := RenderPhysicalPlan(plan, mustBinding(t)) + check("dispatch", dispatchErr) +} + +func TestWindowSpineSortsRootKeyByLogicalIdAndRejectsPayloadSorts(t *testing.T) { + t.Run("RootKeySortRendersLogicalIdOrdering", func(t *testing.T) { + plan := basePlan() + plan.Operations = []ir.PhysicalOperation{ + plan.Operations[0], windowSortOp(ir.PhysicalValue{Variable: "root", Path: []string{"_key"}}), windowLimitOp("limit"), + windowReturnOp(valueProjection("logical_id", "root", "logical_id")), + } + rendered, err := RenderPhysicalPlan(plan, mustBinding(t)) + if err != nil { + t.Fatalf("root _key spine sort must render through the canonical identity mapping: %v", err) + } + if got := windowOrderByClause(t, rendered.SQL); got != `r0."logical_id" ASC, r0."resource_seq" ASC` { + t.Fatalf("root _key spine sort must order by the logical_id storage column, got %q", got) + } + }) + for _, tc := range []struct { + name string + key ir.PhysicalValue + }{ + {"payload path sort", ir.PhysicalValue{Variable: "root", Path: []string{"status"}}}, + {"multi-segment path", ir.PhysicalValue{Variable: "root", Path: []string{"payload", "status"}}}, + } { + plan := basePlan() + plan.Operations = []ir.PhysicalOperation{ + plan.Operations[0], windowSortOp(tc.key), windowLimitOp("limit"), + windowReturnOp(valueProjection("logical_id", "root", "logical_id")), + } + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Errorf("%s: error = %v, want ErrUnsupportedPostgresPlan", tc.name, err) + } + } +} + +func TestWindowFinalSortLimitOverExpansionIsExact(t *testing.T) { + unnest := identifierExpansion(ir.PhysicalUnnestExclude) + plan := windowedUnnestPlan(unnest, 3, + valueProjection("logical_id", "root", "logical_id"), + valueProjection("position", "position"), + valueProjection("has_item", "has_item"), + ) + rendered := renderOK(t, plan) + + if got, want := windowOrderByClause(t, rendered.SQL), + `r0."logical_id" ASC, (u1.rn IS NOT NULL) ASC, u1.rn ASC, r0."resource_seq" ASC`; got != want { + t.Fatalf("final-window ORDER BY =\n got %q\nwant %q", got, want) + } + if !strings.HasSuffix(rendered.SQL, " LIMIT $6") { + t.Fatalf("final window must end in the bound LIMIT:\n%s", rendered.SQL) + } + if strings.Contains(rendered.SQL, "@") { + t.Fatalf("windowed SQL carries AQL bind syntax:\n%s", rendered.SQL) + } + wantArgs := []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "proj-a"}, + {Type: command.PostgreSQLParameterText, Value: "gen-1"}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["FHIR/Patient","FHIR/Observation"]`)}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + {Type: command.PostgreSQLParameterText, Value: "{identifier}"}, + {Type: command.PostgreSQLParameterBigint, Value: int64(3)}, + } + if !reflect.DeepEqual(rendered.Args, wantArgs) { + t.Fatalf("window args =\n got %#v\nwant %#v", rendered.Args, wantArgs) + } +} + +func TestWindowFinalSortLimitOverOuterExpansionOrdersHasItemWitness(t *testing.T) { + unnest := identifierExpansion(ir.PhysicalUnnestPreserveParent) + plan := windowedUnnestPlan(unnest, 2, + valueProjection("logical_id", "root", "logical_id"), + valueProjection("item", "item"), + valueProjection("position", "position"), + valueProjection("has_item", "has_item"), + ) + rendered := renderOK(t, plan) + if got, want := windowOrderByClause(t, rendered.SQL), + `r0."logical_id" ASC, (u1.rn IS NOT NULL) ASC, u1.rn ASC, r0."resource_seq" ASC`; got != want { + t.Fatalf("OUTER final-window ORDER BY =\n got %q\nwant %q", got, want) + } + if !strings.Contains(rendered.SQL, "LEFT JOIN LATERAL") { + t.Fatalf("OUTER expansion lost its pad join:\n%s", rendered.SQL) + } +} + +func TestWindowFinalSortLimitOverOwnerRouteIncludesOccurrenceWitness(t *testing.T) { + unnest := identifierExpansion(ir.PhysicalUnnestExclude) + plan := basePlan() + plan.BindVars["limit"] = 2 + planRouteTraversal(&plan, &unnest, ir.PhysicalOutbound, "") + plan.Operations = []ir.PhysicalOperation{ + plan.Operations[0], + unnestOp(unnest), + windowSortOp(windowTuple(unnest)...), + windowLimitOp("limit"), + windowReturnOp( + valueProjection("logical_id", "root", "logical_id"), + valueProjection("owner_logical_id", "obs", "logical_id"), + valueProjection("position", "position"), + ), + } + rendered := renderOK(t, plan) + if got, want := windowOrderByClause(t, rendered.SQL), + `r0."logical_id" ASC, e1."edge_seq" ASC, x1."logical_id" ASC, (u1.rn IS NOT NULL) ASC, u1.rn ASC, r0."resource_seq" ASC`; got != want { + t.Fatalf("owner-route final-window ORDER BY =\n got %q\nwant %q", got, want) + } +} + +func TestWindowFinalSortLimitOverTwoExpansionsOrdersEveryWitness(t *testing.T) { + first := identifierExpansion(ir.PhysicalUnnestExclude) + second := ir.PhysicalUnnest{ + Owner: ir.PhysicalUnnestOwner{OccurrenceID: "root", ResourceType: "Patient", + RootVariable: "root", OwnerVariable: "root"}, + OutputVariable: "telecom_item", Ordinality: "telecom_position", HasItemVariable: "telecom_has_item", + Expression: arrayExtract("root", "Patient", spec.SelectorStep{Field: "telecom", Iterate: true}), + EmptyPolicy: ir.PhysicalUnnestExclude, + } + plan := basePlan() + plan.BindVars["limit"] = 4 + keys := append(windowTuple(first), windowTuple(second)...) + plan.Operations = []ir.PhysicalOperation{ + plan.Operations[0], unnestOp(first), unnestOp(second), + windowSortOp(keys...), windowLimitOp("limit"), + windowReturnOp( + valueProjection("logical_id", "root", "logical_id"), + valueProjection("position", "position"), + valueProjection("telecom_position", "telecom_position"), + ), + } + rendered := renderOK(t, plan) + if got, want := windowOrderByClause(t, rendered.SQL), + `r0."logical_id" ASC, (u1.rn IS NOT NULL) ASC, u1.rn ASC, (u2.rn IS NOT NULL) ASC, u2.rn ASC, r0."resource_seq" ASC`; got != want { + t.Fatalf("two-expansion final-window ORDER BY =\n got %q\nwant %q", got, want) + } +} + +const mergedNoWindowUnnestGolden = `SELECT r0."logical_id" AS "c0", (u1.item) AS "c1", (u1.rn - 1) AS "c2", (u1.rn IS NOT NULL) AS "c3" FROM "pgir02_it_unit"."fhir_resource" AS r0 CROSS JOIN LATERAL (SELECT u1_e0.item AS item, ROW_NUMBER() OVER (ORDER BY u1_e0.ord) AS rn FROM jsonb_array_elements(CASE WHEN (r0."payload" #> $5::text[]) IS NULL OR jsonb_typeof((r0."payload" #> $5::text[])) = 'null' THEN '[]'::jsonb WHEN jsonb_typeof((r0."payload" #> $5::text[])) = 'array' THEN (r0."payload" #> $5::text[]) WHEN (r0."payload" #> $5::text[]) = 'false'::jsonb OR (r0."payload" #> $5::text[]) = '0'::jsonb OR (r0."payload" #> $5::text[]) = '""'::jsonb THEN '[]'::jsonb ELSE jsonb_build_array((r0."payload" #> $5::text[])) END) WITH ORDINALITY AS u1_e0(item, ord) WHERE jsonb_typeof(u1_e0.item) <> 'null') AS u1 WHERE r0."project" = $1 AND r0."dataset_generation" = $2 AND r0."auth_resource_path" = ANY (SELECT jsonb_array_elements_text($3::jsonb)) AND r0."resource_type" = $4 ORDER BY r0."resource_seq" ASC, u1.rn ASC` + +func TestWindowWithoutWindowKeepsMergedUnnestSQLByteIdentical(t *testing.T) { + rendered := renderOK(t, unnestPlan(expansionProjections()...)) + if rendered.SQL != mergedNoWindowUnnestGolden { + t.Fatalf("no-window statement drifted from the merged pg-sql-09 golden:\n got %q\nwant %q", rendered.SQL, mergedNoWindowUnnestGolden) + } + if got, want := windowOrderByClause(t, rendered.SQL), `r0."resource_seq" ASC, u1.rn ASC`; got != want { + t.Fatalf("no-window ORDER BY drifted:\n got %q\nwant %q", got, want) + } + if strings.Contains(rendered.SQL, "IS NOT NULL) ASC") { + t.Fatalf("no-window statement grew a has-item witness:\n%s", rendered.SQL) + } +} + +func TestWindowRenderIsDeterministicAndDoesNotAliasPlanData(t *testing.T) { + unnest := identifierExpansion(ir.PhysicalUnnestPreserveParent) + plan := windowedUnnestPlan(unnest, 3, + valueProjection("logical_id", "root", "logical_id"), + valueProjection("item", "item"), + valueProjection("position", "position"), + ) + first := renderOK(t, plan) + second := renderOK(t, plan) + if first.SQL != second.SQL || !reflect.DeepEqual(first.Args, second.Args) { + t.Fatalf("window render is not deterministic") + } + plan.BindVars["limit"] = 99 + third := renderOK(t, plan) + if third.SQL != first.SQL { + t.Fatalf("statement text followed a mutated plan bind:\n got %s\nwant %s", third.SQL, first.SQL) + } + for _, parameter := range first.Args { + if bigint, ok := parameter.Value.(int64); ok && bigint == 99 { + t.Fatalf("rendered arguments aliased a later plan mutation: %#v", first.Args) + } + } +} + +func TestWindowFinalSortRejectsEveryTupleDeviation(t *testing.T) { + type mutation func(unnest *ir.PhysicalUnnest, keys *[]ir.PhysicalValue) + cases := []struct { + name string + mutate mutation + projections []ir.PhysicalProjection + }{ + {"drop the ordinality key", func(_ *ir.PhysicalUnnest, keys *[]ir.PhysicalValue) { *keys = (*keys)[:2] }, nil}, + {"drop the has-item key", func(_ *ir.PhysicalUnnest, keys *[]ir.PhysicalValue) { + *keys = []ir.PhysicalValue{(*keys)[0], (*keys)[2]} + }, nil}, + {"drop the root key", func(_ *ir.PhysicalUnnest, keys *[]ir.PhysicalValue) { *keys = (*keys)[1:] }, nil}, + {"root key uses logical_id instead of _key", func(_ *ir.PhysicalUnnest, keys *[]ir.PhysicalValue) { + (*keys)[0] = ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}} + }, nil}, + {"root key uses a payload path", func(_ *ir.PhysicalUnnest, keys *[]ir.PhysicalValue) { + (*keys)[0] = ir.PhysicalValue{Variable: "root", Path: []string{"payload", "name"}} + }, nil}, + {"swap has-item and ordinality", func(_ *ir.PhysicalUnnest, keys *[]ir.PhysicalValue) { + (*keys)[1], (*keys)[2] = (*keys)[2], (*keys)[1] + }, nil}, + {"append a duplicate root key", func(_ *ir.PhysicalUnnest, keys *[]ir.PhysicalValue) { + *keys = append(*keys, ir.PhysicalValue{Variable: "root", Path: []string{"_key"}}) + }, nil}, + {"sort on the expansion item", func(_ *ir.PhysicalUnnest, keys *[]ir.PhysicalValue) { + (*keys)[1] = ir.PhysicalValue{Variable: "item"} + }, nil}, + {"sort on the item with a path", func(_ *ir.PhysicalUnnest, keys *[]ir.PhysicalValue) { + (*keys)[1] = ir.PhysicalValue{Variable: "item", Path: []string{"value"}} + }, nil}, + {"has-item key carries a path", func(_ *ir.PhysicalUnnest, keys *[]ir.PhysicalValue) { + (*keys)[1] = ir.PhysicalValue{Variable: "has_item", Path: []string{"_key"}} + }, nil}, + {"key is a bind value", func(_ *ir.PhysicalUnnest, keys *[]ir.PhysicalValue) { + (*keys)[1] = ir.PhysicalValue{BindKey: "limit"} + }, nil}, + {"ordinality removed from the expansion while a key remains", func(unnest *ir.PhysicalUnnest, keys *[]ir.PhysicalValue) { + unnest.Ordinality = "" + *keys = []ir.PhysicalValue{ + ir.PhysicalValue{Variable: "root", Path: []string{"_key"}}, + ir.PhysicalValue{Variable: "has_item"}, + ir.PhysicalValue{Variable: "root", Path: []string{"_key"}}, + } + }, []ir.PhysicalProjection{ + valueProjection("logical_id", "root", "logical_id"), + valueProjection("item", "item"), + }}, + } + for _, tc := range cases { + unnest := identifierExpansion(ir.PhysicalUnnestExclude) + keys := windowTuple(unnest) + tc.mutate(&unnest, &keys) + projections := tc.projections + if projections == nil { + projections = expansionProjections() + } + plan := basePlan() + plan.BindVars["limit"] = 3 + plan.Operations = []ir.PhysicalOperation{ + plan.Operations[0], unnestOp(unnest), windowSortOp(keys...), windowLimitOp("limit"), + windowReturnOp(projections...), + } + requireWindowRefusal(t, plan, tc.name) + } +} + +func TestWindowFinalSortKeepsThePlanRouteOccurrenceProofExact(t *testing.T) { + unnest := identifierExpansion(ir.PhysicalUnnestExclude) + plan := basePlan() + plan.BindVars["limit"] = 2 + planRouteTraversal(&plan, &unnest, ir.PhysicalOutbound, "") + keys := windowTuple(unnest) + for _, drop := range []int{1} { + broken := append([]ir.PhysicalValue(nil), keys[:drop]...) + broken = append(broken, keys[drop+1:]...) + bad := basePlan() + bad.BindVars["limit"] = 2 + planRouteTraversal(&bad, &unnest, ir.PhysicalOutbound, "") + bad.Operations = []ir.PhysicalOperation{ + bad.Operations[0], unnestOp(unnest), windowSortOp(broken...), windowLimitOp("limit"), + windowReturnOp(valueProjection("logical_id", "root", "logical_id")), + } + requireWindowRefusal(t, bad, "drop route edge witness") + } +} + +func TestWindowRejectsWindowPositionViolations(t *testing.T) { + unnest := identifierExpansion(ir.PhysicalUnnestExclude) + validTuple := windowTuple(unnest) + type arms = []ir.PhysicalOperation + cases := []struct { + name string + ops func(root, ret ir.PhysicalOperation, u ir.PhysicalOperation) arms + }{ + {"SORT before the expansion", func(root, ret, u ir.PhysicalOperation) arms { + return arms{root, windowSortOp(validTuple...), u, ret} + }}, + {"LIMIT before the expansion", func(root, ret, u ir.PhysicalOperation) arms { + return arms{root, windowLimitOp("limit"), u, ret} + }}, + {"LIMIT before the SORT", func(root, ret, u ir.PhysicalOperation) arms { + return arms{root, u, windowLimitOp("limit"), windowSortOp(validTuple...), ret} + }}, + {"second SORT after the LIMIT", func(root, ret, u ir.PhysicalOperation) arms { + return arms{root, u, windowSortOp(validTuple...), windowLimitOp("limit"), windowSortOp(validTuple...), ret} + }}, + {"second SORT before the LIMIT", func(root, ret, u ir.PhysicalOperation) arms { + return arms{root, u, windowSortOp(validTuple...), windowSortOp(validTuple...), windowLimitOp("limit"), ret} + }}, + {"second LIMIT", func(root, ret, u ir.PhysicalOperation) arms { + return arms{root, u, windowSortOp(validTuple...), windowLimitOp("limit"), windowLimitOp("limit"), ret} + }}, + {"expansion after the window opens", func(root, ret, u ir.PhysicalOperation) arms { + late := ir.PhysicalUnnest{ + Owner: ir.PhysicalUnnestOwner{OccurrenceID: "root", ResourceType: "Patient", + RootVariable: "root", OwnerVariable: "root"}, + OutputVariable: "telecom_item", Ordinality: "telecom_position", HasItemVariable: "telecom_has_item", + Expression: arrayExtract("root", "Patient", spec.SelectorStep{Field: "telecom", Iterate: true}), + EmptyPolicy: ir.PhysicalUnnestExclude, + } + return arms{root, u, windowSortOp(validTuple...), unnestOp(late), ret} + }}, + {"nil SORT payload", func(root, ret, u ir.PhysicalOperation) arms { + return arms{root, {Kind: ir.PhysicalSortOp}, u, ret} + }}, + {"nil LIMIT payload", func(root, ret, u ir.PhysicalOperation) arms { + return arms{root, u, {Kind: ir.PhysicalLimitOp}, ret} + }}, + } + for _, tc := range cases { + plan := basePlan() + plan.BindVars["limit"] = 3 + root := plan.Operations[0] + u := unnestOp(identifierExpansion(ir.PhysicalUnnestExclude)) + ret := windowReturnOp(expansionProjections()...) + plan.Operations = tc.ops(root, ret, u) + requireWindowRefusal(t, plan, tc.name) + } +} + +func TestWindowRejectsLimitBindDefects(t *testing.T) { + for _, tc := range []struct { + name string + value any + }{ + {"zero limit", 0}, + {"negative limit", -4}, + {"string limit", "10"}, + } { + unnest := identifierExpansion(ir.PhysicalUnnestExclude) + plan := windowedUnnestPlan(unnest, 3, expansionProjections()...) + plan.BindVars["limit"] = tc.value + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); err == nil { + t.Errorf("%s: render succeeded, want fail-closed rejection", tc.name) + } + } + unnest := identifierExpansion(ir.PhysicalUnnestExclude) + plan := windowedUnnestPlan(unnest, 3, expansionProjections()...) + delete(plan.BindVars, "limit") + if _, err := RenderPhysicalPlan(plan, mustBinding(t)); err == nil { + t.Errorf("missing limit bind: render succeeded, want fail-closed rejection") + } +} + +func TestWindowPreviewSourceWindowByRootIDStaysRefused(t *testing.T) { + unnest := identifierExpansion(ir.PhysicalUnnestExclude) + plan := windowedUnnestPlan(unnest, 3, expansionProjections()...) + plan.PreviewSourceWindowByRootID = true + requireWindowRefusal(t, plan, "preview source window by root id") + + spine := basePlan() + spine.Operations = []ir.PhysicalOperation{ + spine.Operations[0], + windowSortOp(ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}), + windowLimitOp("limit"), + windowReturnOp(valueProjection("logical_id", "root", "logical_id")), + } + spine.PreviewSourceWindowByRootID = true + if _, err := RenderPhysicalPlan(spine, mustBinding(t)); !errors.Is(err, ErrUnsupportedPostgresPlan) { + t.Fatalf("spine preview-source-window error = %v, want ErrUnsupportedPostgresPlan", err) + } +} + +const ( + sql12Project = "sql12-proj" + sql12Generation = "gen-1" +) + +var sql12ResourceNamePattern = regexp.MustCompile(`^verify_sql12_[0-9]+_[0-9a-f]+$`) + +type sql12Service struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + client *control.Client + schema string + resources *control.ResourceRepository + fail func(step string, err error) +} + +func newSQL12Service(t *testing.T) *sql12Service { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the pg-sql-12 window acceptance tests") + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %v", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf("verify_sql12_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !sql12ResourceNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + t.Fatalf("create isolated database: %v", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %v", err) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect to isolated database: %v", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf("verify_sql12_%d_%s", time.Now().UnixNano(), randomHex(t)) + if !sql12ResourceNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + t.Fatalf("create owned schema: %v", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %v", err) + } + }) + + client, err := control.OpenPool(ctx, control.ClientConfig{DSN: dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 2 * time.Second}) + if err != nil { + t.Fatalf("open pg-store client (requires a PostgreSQL 18 service): %v", err) + } + t.Cleanup(client.Close) + if _, err := control.MigrateSchema(ctx, client, schema); err != nil { + t.Fatalf("migrate owned schema: %v", err) + } + resources, err := control.NewResourceRepository(client, schema) + if err != nil { + t.Fatalf("build resource repository: %v", err) + } + return &sql12Service{ctx: ctx, admin: admin, iso: iso, client: client, schema: schema, resources: resources, + fail: func(step string, err error) { t.Fatalf("%s: %v", step, err) }} +} + +func (s *sql12Service) insertResource(t *testing.T, resourceType, logicalID, authPath string, payload map[string]any) { + t.Helper() + raw, err := json.Marshal(payload) + if err != nil { + t.Fatalf("encode payload: %v", err) + } + if _, err := s.resources.InsertResource(s.ctx, source.Resource{ + Project: sql12Project, DatasetGeneration: sql12Generation, ResourceType: resourceType, + LogicalID: logicalID, AuthResourcePath: authPath, Payload: raw, + }); err != nil { + s.fail("insert resource "+resourceType+"/"+logicalID, err) + } +} + +func (s *sql12Service) insertEdge(t *testing.T, rec control.EdgeOccurrenceRow) { + t.Helper() + edges, err := control.NewEdgeRepository(s.client, s.schema) + if err != nil { + s.fail("build edge repository", err) + } + if _, err := edges.InsertEdge(s.ctx, rec); err != nil { + s.fail("insert edge "+rec.Label+" "+rec.SourceLogicalID+"->"+rec.TargetLogicalID, err) + } +} + +func (s *sql12Service) runRendered(t *testing.T, rendered command.RenderedPostgresPhysicalPlan) [][]string { + t.Helper() + return s.queryRows(t, rendered.SQL, windowParameterValues(rendered.Args)...) +} + +func (s *sql12Service) queryRows(t *testing.T, sql string, args ...any) [][]string { + t.Helper() + rows, err := s.iso.Query(s.ctx, sql, args...) + if err != nil { + s.fail("execute statement: "+sql, err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + t.Fatalf("read row: %v", err) + } + row := make([]string, len(values)) + for index, value := range values { + row[index] = sql09StringifyCell(value) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + t.Fatalf("finish statement: %v", err) + } + return out +} + +func (s *sql12Service) witness(t *testing.T, sql string, args ...any) [][]string { + t.Helper() + return s.queryRows(t, sql, args...) +} + +func windowParameterValues(args []command.PostgreSQLParameter) []any { + values := make([]any, len(args)) + for index, parameter := range args { + values[index] = parameter.Value + } + return values +} + +func windowRepeat(t *testing.T, svc *sql12Service, rendered command.RenderedPostgresPhysicalPlan, executions int, want [][]string) { + t.Helper() + for run := 1; run <= executions; run++ { + got := svc.runRendered(t, rendered) + if !reflect.DeepEqual(got, want) { + t.Fatalf("execution %d page =\n got %v\nwant %v", run, got, want) + } + } +} + +func sql12Binding(t *testing.T, schema string) ir.PostgresStorageBinding { + t.Helper() + binding, err := ir.NewPostgresStorageBinding(schema) + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + return binding +} + +func (s *sql12Service) render(t *testing.T, plan ir.PhysicalPlan) command.RenderedPostgresPhysicalPlan { + t.Helper() + rendered, err := RenderPhysicalPlan(plan, sql12Binding(t, s.schema)) + if err != nil { + t.Fatalf("RenderPhysicalPlan: %v", err) + } + return rendered +} + +func TestPG18WindowDuplicateRootRowsPageStably(t *testing.T) { + svc := newSQL12Service(t) + for _, id := range []string{"dup-1", "dup-2", "dup-3", "dup-4", "dup-5"} { + svc.insertResource(t, "Patient", id, "FHIR/Patient", map[string]any{"status": "final"}) + } + plan := scopedPlan(sql12Project, sql12Generation, []string{"FHIR/Patient"}, false) + plan.BindVars["limit"] = 3 + plan.Operations = append(plan.Operations, + windowSortOp(ir.PhysicalValue{Variable: "root", Path: []string{"resource_type"}}), + windowLimitOp("limit")) + plan = withReturn(plan, valueProjection("logical_id", "root", "logical_id")) + rendered := svc.render(t, plan) + + want := [][]string{{"dup-1"}, {"dup-2"}, {"dup-3"}} + windowRepeat(t, svc, rendered, 5, want) + + witness := svc.witness(t, `SELECT logical_id FROM "`+svc.schema+`"."fhir_resource"`+ + ` WHERE project = 'sql12-proj' AND dataset_generation = 'gen-1'`+ + ` AND resource_type = 'Patient' AND auth_resource_path = 'FHIR/Patient'`+ + ` ORDER BY resource_type, resource_seq LIMIT 3`) + if !reflect.DeepEqual(witness, want) { + t.Fatalf("witness page = %v, want %v", witness, want) + } +} + +func TestPG18WindowExpandedChildrenPageStablyWithDuplicateItems(t *testing.T) { + svc := newSQL12Service(t) + svc.insertResource(t, "Patient", "p-array", "FHIR/Patient", map[string]any{ + "identifier": []any{ + map[string]any{"value": "a"}, map[string]any{"value": "a"}, + map[string]any{"value": "b"}, map[string]any{"value": "c"}, + }, + }) + unnest := identifierExpansion(ir.PhysicalUnnestExclude) + plan := scopedPlan(sql12Project, sql12Generation, []string{"FHIR/Patient"}, false) + plan.Operations = append(plan.Operations, + unnestOp(unnest), + windowSortOp(windowTuple(unnest)...), + windowLimitOp("limit")) + plan.BindVars["limit"] = 3 + plan = withReturn(plan, + valueProjection("logical_id", "root", "logical_id"), + valueProjection("item_value", "item", "value"), + valueProjection("position", "position")) + rendered := svc.render(t, plan) + + want := [][]string{ + {"p-array", "a", "0"}, + {"p-array", "a", "1"}, + {"p-array", "b", "2"}, + } + windowRepeat(t, svc, rendered, 5, want) + + witness := svc.witness(t, `SELECT r.logical_id, elem.value #> '{value}', elem.ord - 1`+ + ` FROM "`+svc.schema+`"."fhir_resource" AS r`+ + ` CROSS JOIN LATERAL jsonb_array_elements(r.payload #> '{identifier}'::text[]) WITH ORDINALITY AS elem(value, ord)`+ + ` WHERE r.project = 'sql12-proj' AND r.dataset_generation = 'gen-1'`+ + ` AND r.resource_type = 'Patient' AND r.auth_resource_path = 'FHIR/Patient'`+ + ` ORDER BY r.logical_id, r.resource_seq, elem.ord LIMIT 3`) + if !reflect.DeepEqual(witness, want) { + t.Fatalf("witness page = %v, want %v", witness, want) + } +} + +func TestPG18WindowOwnerOccurrencesPageStably(t *testing.T) { + svc := newSQL12Service(t) + svc.insertResource(t, "Patient", "p-own", "FHIR/Patient", map[string]any{}) + svc.insertResource(t, "Patient", "p-ta", "FHIR/Patient", map[string]any{}) + svc.insertResource(t, "Patient", "p-tb", "FHIR/Patient", map[string]any{}) + svc.insertResource(t, "Patient", "q-own", "FHIR/Patient", map[string]any{}) + svc.insertResource(t, "Patient", "q-ta", "FHIR/Patient", map[string]any{}) + for index, target := range []string{"p-ta", "p-tb"} { + svc.insertEdge(t, control.EdgeOccurrenceRow{ + Project: sql12Project, DatasetGeneration: sql12Generation, AuthResourcePath: "FHIR/Patient", + Label: "has-item-owner", SourceResourceType: "Patient", SourceLogicalID: "p-own", + TargetResourceType: "Patient", TargetLogicalID: target, + SourcePath: fmt.Sprintf("item[%d]", index), SourceOrdinal: int64(index), + }) + } + svc.insertEdge(t, control.EdgeOccurrenceRow{ + Project: sql12Project, DatasetGeneration: sql12Generation, AuthResourcePath: "FHIR/Patient", + Label: "has-item-owner", SourceResourceType: "Patient", SourceLogicalID: "q-own", + TargetResourceType: "Patient", TargetLogicalID: "q-ta", + SourcePath: "item[0]", SourceOrdinal: 0, + }) + + unnest := ir.PhysicalUnnest{ + Owner: ir.PhysicalUnnestOwner{OccurrenceID: "owner-route", ResourceType: "Patient", + RootVariable: "root", OwnerVariable: "own", + Route: []ir.PhysicalUnnestRouteStep{{OccurrenceID: "owner-route", Traversal: ir.PhysicalTraversal{ + SourceVariable: "root", TargetVariable: "own", EdgeVariable: "owner_edge", + Direction: ir.PhysicalOutbound, EdgeCollectionBindKey: "edge_collection", + EdgeLabelBindKey: "edge_label", TargetTypeBindKey: "target_type", + EdgeTargetTypeField: "to_type", Strategy: ir.PhysicalTraversalNative, + }}}}, + OutputVariable: "item", Ordinality: "position", HasItemVariable: "has_item", + Expression: arrayExtract("own", "Patient", spec.SelectorStep{Field: "identifier", Iterate: true}), + EmptyPolicy: ir.PhysicalUnnestPreserveParent, + } + plan := scopedPlan(sql12Project, sql12Generation, []string{"FHIR/Patient"}, false) + plan.BindVars["edge_collection"] = "fhir_edge" + plan.BindVars["edge_label"] = "has-item-owner" + plan.BindVars["target_type"] = "Patient" + plan.BindVars["limit"] = 3 + plan.Operations = append(plan.Operations, + unnestOp(unnest), + windowSortOp(windowTuple(unnest)...), + windowLimitOp("limit")) + plan = withReturn(plan, + valueProjection("logical_id", "root", "logical_id"), + valueProjection("owner_logical_id", "own", "logical_id")) + rendered := svc.render(t, plan) + + want := [][]string{ + {"p-own", "p-ta"}, + {"p-own", "p-tb"}, + {"q-own", "q-ta"}, + } + windowRepeat(t, svc, rendered, 5, want) + + witness := svc.witness(t, `SELECT r.logical_id, r1.logical_id`+ + ` FROM "`+svc.schema+`"."fhir_resource" AS r`+ + ` INNER JOIN "`+svc.schema+`"."fhir_edge_occurrence" AS e`+ + ` ON e.project = 'sql12-proj' AND e.dataset_generation = 'gen-1'`+ + ` AND e.auth_resource_path = 'FHIR/Patient' AND e.label = 'has-item-owner'`+ + ` AND e.source_resource_type = 'Patient' AND e.source_logical_id = r.logical_id`+ + ` AND e.target_resource_type = 'Patient'`+ + ` INNER JOIN "`+svc.schema+`"."fhir_resource" AS r1`+ + ` ON r1.project = 'sql12-proj' AND r1.dataset_generation = 'gen-1'`+ + ` AND r1.auth_resource_path = 'FHIR/Patient' AND r1.resource_type = 'Patient'`+ + ` AND r1.logical_id = e.target_logical_id`+ + ` WHERE r.project = 'sql12-proj' AND r.dataset_generation = 'gen-1'`+ + ` AND r.resource_type = 'Patient' AND r.auth_resource_path = 'FHIR/Patient'`+ + ` ORDER BY r.logical_id, e.edge_seq, r1.logical_id LIMIT 3`) + if !reflect.DeepEqual(witness, want) { + t.Fatalf("witness page = %v, want %v", witness, want) + } +} + +func TestPG18WindowOuterPadRowPageContents(t *testing.T) { + svc := newSQL12Service(t) + svc.insertResource(t, "Patient", "a-empty", "FHIR/Patient", map[string]any{}) + svc.insertResource(t, "Patient", "b-full", "FHIR/Patient", map[string]any{ + "identifier": []any{map[string]any{"value": "x"}, map[string]any{"value": "y"}}, + }) + unnest := identifierExpansion(ir.PhysicalUnnestPreserveParent) + plan := scopedPlan(sql12Project, sql12Generation, []string{"FHIR/Patient"}, false) + plan.BindVars["limit"] = 3 + plan.Operations = append(plan.Operations, + unnestOp(unnest), + windowSortOp(windowTuple(unnest)...), + windowLimitOp("limit")) + plan = withReturn(plan, + valueProjection("logical_id", "root", "logical_id"), + valueProjection("item", "item"), + valueProjection("position", "position"), + valueProjection("has_item", "has_item")) + rendered := svc.render(t, plan) + + want := [][]string{ + {"a-empty", "", "", "false"}, + {"b-full", `{"value":"x"}`, "0", "true"}, + {"b-full", `{"value":"y"}`, "1", "true"}, + } + windowRepeat(t, svc, rendered, 5, want) + + witness := svc.witness(t, `SELECT r.logical_id, elem.value, elem.ord - 1`+ + ` FROM "`+svc.schema+`"."fhir_resource" AS r`+ + ` CROSS JOIN LATERAL jsonb_array_elements(r.payload #> '{identifier}'::text[])`+ + ` WITH ORDINALITY AS elem(value, ord)`+ + ` WHERE r.project = 'sql12-proj' AND r.dataset_generation = 'gen-1'`+ + ` AND r.resource_type = 'Patient' AND r.auth_resource_path = 'FHIR/Patient'`+ + ` AND r.logical_id = 'b-full'`+ + ` ORDER BY r.logical_id, r.resource_seq, elem.ord`) + wantNoPad := [][]string{{"b-full", `{"value":"x"}`, "0"}, {"b-full", `{"value":"y"}`, "1"}} + if !reflect.DeepEqual(witness, wantNoPad) { + t.Fatalf("non-pad witness page = %v, want %v", witness, wantNoPad) + } +} + +func TestPG18WindowFilterExecutesBeforeFinalWindow(t *testing.T) { + svc := newSQL12Service(t) + svc.insertResource(t, "Patient", "x-final-1", "FHIR/Patient", map[string]any{"status": "final"}) + svc.insertResource(t, "Patient", "x-final-2", "FHIR/Patient", map[string]any{"status": "final"}) + svc.insertResource(t, "Patient", "x-prelim-0", "FHIR/Patient", map[string]any{"status": "prelim"}) + svc.insertResource(t, "Patient", "x-zz-prelim", "FHIR/Patient", map[string]any{"status": "prelim"}) + + plan := scopedPlan(sql12Project, sql12Generation, []string{"FHIR/Patient"}, false) + plan.BindVars["limit"] = 3 + plan.BindVars["status_value"] = "final" + plan.Operations = append(plan.Operations, + ir.PhysicalOperation{Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{ + Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"status"}}, + Right: &ir.PhysicalValue{BindKey: "status_value"}, + }}}, + windowSortOp(ir.PhysicalValue{Variable: "root", Path: []string{"resource_type"}}), + windowLimitOp("limit")) + plan = withReturn(plan, valueProjection("logical_id", "root", "logical_id")) + rendered := svc.render(t, plan) + + want := [][]string{{"x-final-1"}, {"x-final-2"}} + windowRepeat(t, svc, rendered, 5, want) + order := windowOrderByClause(t, rendered.SQL) + if !strings.HasPrefix(order, `r0."resource_type" ASC`) || !strings.HasSuffix(order, `r0."resource_seq" ASC`) { + t.Fatalf("window did not stay the final clause before LIMIT: %q", order) + } + + witness := svc.witness(t, `SELECT logical_id FROM "`+svc.schema+`"."fhir_resource"`+ + ` WHERE project = 'sql12-proj' AND dataset_generation = 'gen-1'`+ + ` AND resource_type = 'Patient' AND auth_resource_path = 'FHIR/Patient'`+ + ` AND payload #> '{status}'::text[] = '"final"'::jsonb`+ + ` ORDER BY resource_type, resource_seq LIMIT 3`) + if !reflect.DeepEqual(witness, want) { + t.Fatalf("witness page = %v, want %v", witness, want) + } +} + +func TestZZNoLeftoverSQL12Resources(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer admin.Close() + var databases int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify_sql12_%'").Scan(&databases); err != nil { + t.Fatalf("count databases: %v", err) + } + if databases != 0 { + t.Fatalf("%d verify_sql12_* databases still exist", databases) + } + var schemas int + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'verify_sql12_%'").Scan(&schemas); err != nil { + t.Fatalf("count schemas: %v", err) + } + if schemas != 0 { + t.Fatalf("%d verify_sql12_* schemas still exist in the service database", schemas) + } +} diff --git a/internal/dataframe/compiler/root_page_related_seed_arango_test.go b/internal/dataframe/compiler/root_page_related_seed_arango_test.go index 5edada031..abd5795e9 100644 --- a/internal/dataframe/compiler/root_page_related_seed_arango_test.go +++ b/internal/dataframe/compiler/root_page_related_seed_arango_test.go @@ -11,11 +11,11 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -23,12 +23,12 @@ func TestRootPageRelatedSeedConstructionFilterPlacement(t *testing.T) { const project, generation, targetID = "root-page-related-seed-project", "root-page-related-seed-generation", "patient-match" bindings := rootPageRelatedSeedBindings(project, generation) compiled := compileRootPageRelatedSeedOutput(t, rootPageRelatedSeedOutput(targetID), bindings) - page, err := CompileRecipeOutputPageWithPolicy(compiled, bindings, 1, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled, bindings, 1, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile related root page: %v", err) } - query := page.RootKeysQuery + query := testAQLText(t, page.RootKeys) unique := strings.Index(query, "LET child_set_1 = UNIQUE((") childSort := strings.Index(query, "SORT child_set_1_node._key") first := strings.Index(query, "FIRST(FLATTEN(child_set_1[*].__loom_projection_0))") @@ -43,26 +43,26 @@ func TestRootPageRelatedSeedConstructionFilterPlacement(t *testing.T) { unique > childSort || childSort > first || first > filter || filter > filterBind || filterBind > rootSort || rootSort > limit { t.Fatalf("related construction filter must follow UNIQUE, sorted child FIRST, and precede the root-key LIMIT: UNIQUE=%d child-sort=%d FIRST=%d FILTER=%d bind=%d root-sort=%d LIMIT=%d\n%s", unique, childSort, first, filter, filterBind, rootSort, limit, query) } - if got := page.RootKeysBindVars["construction_filter_value"]; got != targetID { + if got := testAQLBinds(t, page.RootKeys)["construction_filter_value"]; got != targetID { t.Fatalf("related root-key construction filter bind = %#v, want %q", got, targetID) } - if got := page.RootKeysBindVars["project"]; got != project { + if got := testAQLBinds(t, page.RootKeys)["project"]; got != project { t.Fatalf("related root-key project bind = %#v, want %q", got, project) } - if got := page.RootKeysBindVars["dataset_generation"]; got != generation { + if got := testAQLBinds(t, page.RootKeys)["dataset_generation"]; got != generation { t.Fatalf("related root-key generation bind = %#v, want %q", got, generation) } - if got := page.RootKeysBindVars["auth_resource_paths_unrestricted"]; got != false { + if got := testAQLBinds(t, page.RootKeys)["auth_resource_paths_unrestricted"]; got != false { t.Fatalf("related root-key auth bypass bind = %#v, want false", got) } - if paths, ok := page.RootKeysBindVars["auth_resource_paths"].([]string); !ok || !reflect.DeepEqual(paths, []string{"/allowed"}) { - t.Fatalf("related root-key auth paths = %#v, want [/allowed]", page.RootKeysBindVars["auth_resource_paths"]) + if paths, ok := testAQLBinds(t, page.RootKeys)["auth_resource_paths"].([]string); !ok || !reflect.DeepEqual(paths, []string{"/allowed"}) { + t.Fatalf("related root-key auth paths = %#v, want [/allowed]", testAQLBinds(t, page.RootKeys)["auth_resource_paths"]) } } func TestRootPageRelatedSeedFalsePositiveDoesNotConsumePageAgainstArango(t *testing.T) { ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Observation"}, {Name: "Patient"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatalf("bootstrap related root-page fixture collections: %v", err) @@ -145,8 +145,6 @@ func TestRootPageRelatedSeedFalsePositiveDoesNotConsumePageAgainstArango(t *test staleTargetRoot := addObservation("027_stale_target", "stale-target", project, generation, "/allowed") foreignTargetRoot := addObservation("028_foreign_target", "foreign-target", project, generation, "/allowed") - // The first root is a seed-only false positive: its later related Patient - // matches, but its sorted-first Patient does not. addEdge("false_first_nonmatch", falsePositiveRoot, nonmatchingFirst, project, generation, "/allowed") addEdge("false_later_match", falsePositiveRoot, target, project, generation, "/allowed") addEdge("first_target", firstEligibleRoot, target, project, generation, "/allowed") @@ -199,7 +197,7 @@ func TestRootPageRelatedSeedFalsePositiveDoesNotConsumePageAgainstArango(t *test bindings := rootPageRelatedSeedBindings(project, generation) compiled := compileRootPageRelatedSeedOutput(t, rootPageRelatedSeedOutput(targetID), bindings) - full, err := CompileRecipeOutputWithPolicy(compiled, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + full, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile full related construction output: %v", err) } @@ -227,11 +225,11 @@ func TestRootPageRelatedSeedFalsePositiveDoesNotConsumePageAgainstArango(t *test t.Fatalf("unpaged related construction has duplicate rows per root: %#v", fullRows) } - page, err := CompileRecipeOutputPageWithPolicy(compiled, bindings, 1, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiled, bindings, 1, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile paged related construction output: %v", err) } - keyBinds := rootPageRelatedSeedCloneBinds(page.RowsBindVars) + keyBinds := rootPageRelatedSeedCloneBinds(testAQLBinds(t, page.Rows)) keyBinds[RootPageKeysBind] = []string{ falsePositiveRoot, firstEligibleRoot, firstEligibleRoot, secondEligibleRoot, deniedRoot, staleRoot, foreignRoot, deniedEdgeRoot, staleEdgeRoot, @@ -239,11 +237,11 @@ func TestRootPageRelatedSeedFalsePositiveDoesNotConsumePageAgainstArango(t *test "missing-root-page-key", } selectedRows := make([]map[string]any, 0, len(fullRows)) - if err := client.QueryRows(ctx, page.RowsQuery, 100, keyBinds, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, page.Rows), 100, keyBinds, func(row map[string]any) error { selectedRows = append(selectedRows, row) return nil }); err != nil { - t.Fatalf("execute selected-root key-set with duplicate, missing, and out-of-scope keys: %v\n%s", err, page.RowsQuery) + t.Fatalf("execute selected-root key-set with duplicate, missing, and out-of-scope keys: %v\n%s", err, testAQLText(t, page.Rows)) } if !reflect.DeepEqual(rootPageRelatedSeedRowSignatures(selectedRows), rootPageRelatedSeedRowSignatures(fullRows)) { t.Fatalf("selected-root key-set changed exact scoped related rows: got %#v, want %#v", selectedRows, fullRows) @@ -295,12 +293,12 @@ type rootPageRelatedSeedPage struct { rows []map[string]any } -func rootPageRelatedSeedReadPage(t *testing.T, ctx context.Context, client *store.Client, page CompiledOutputPage, after string) rootPageRelatedSeedPage { +func rootPageRelatedSeedReadPage(t *testing.T, ctx context.Context, client *arangofactory.Client, page CompiledOutputPage, after string) rootPageRelatedSeedPage { t.Helper() - keyBinds := rootPageRelatedSeedCloneBinds(page.RootKeysBindVars) + keyBinds := rootPageRelatedSeedCloneBinds(testAQLBinds(t, page.RootKeys)) keyBinds[RootPageAfterKeyBind] = after result := rootPageRelatedSeedPage{keys: make([]string, 0, 1)} - if err := client.QueryRows(ctx, page.RootKeysQuery, 100, keyBinds, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, page.RootKeys), 100, keyBinds, func(row map[string]any) error { key, ok := row["_key"].(string) if !ok || key == "" { return fmt.Errorf("root-key query returned invalid _key %#v", row["_key"]) @@ -308,7 +306,7 @@ func rootPageRelatedSeedReadPage(t *testing.T, ctx context.Context, client *stor result.keys = append(result.keys, key) return nil }); err != nil { - t.Fatalf("execute related root-key page after %q: %v\n%s", after, err, page.RootKeysQuery) + t.Fatalf("execute related root-key page after %q: %v\n%s", after, err, testAQLText(t, page.RootKeys)) } if len(result.keys) == 0 { return result @@ -316,13 +314,13 @@ func rootPageRelatedSeedReadPage(t *testing.T, ctx context.Context, client *stor if len(result.keys) != 1 { t.Fatalf("related root-key page after %q returned %#v, want at most one key", after, result.keys) } - rowBinds := rootPageRelatedSeedCloneBinds(page.RowsBindVars) + rowBinds := rootPageRelatedSeedCloneBinds(testAQLBinds(t, page.Rows)) rowBinds[RootPageKeysBind] = result.keys - if err := client.QueryRows(ctx, page.RowsQuery, 100, rowBinds, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, page.Rows), 100, rowBinds, func(row map[string]any) error { result.rows = append(result.rows, row) return nil }); err != nil { - t.Fatalf("execute selected-root rows after %q: %v\n%s", after, err, page.RowsQuery) + t.Fatalf("execute selected-root rows after %q: %v\n%s", after, err, testAQLText(t, page.Rows)) } return result } diff --git a/internal/dataframe/compiler/row_lineage.go b/internal/dataframe/compiler/row_lineage.go index bc079b382..badc384d4 100644 --- a/internal/dataframe/compiler/row_lineage.go +++ b/internal/dataframe/compiler/row_lineage.go @@ -2,15 +2,18 @@ package compiler import ( "encoding/json" + "errors" "fmt" "io" "math" "reflect" "strings" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/compiler/render/aql" + postgresrender "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" ) const ( @@ -44,8 +47,7 @@ func (err *RowLineageUnsupportedError) Error() string { var _ error = (*RowLineageUnsupportedError)(nil) type CompiledRowLineageQuery struct { - Query string - BindVars map[string]any + Command command.CompiledCommand Offset int Limit int FoundColumn string @@ -128,9 +130,6 @@ func RowLineageCapabilityForOutput(output lower.CompiledRecipeOutput) RowLineage return RowLineageCapability{Available: true} } -// rowLineageRelatedCodingGroup admits only the terminal direct-root Group -// shape whose Coding tuple and scalar sources are prepared by one typed route -// adapter. All other related Groups keep the existing unsupported behavior. func rowLineageRelatedCodingGroup(output lower.CompiledRecipeOutput) bool { sequence := output.Plan.StageSequence if sequence == nil || sequence.SourceRowIdentity != "_key" || len(sequence.Stages) != 1 || @@ -355,9 +354,6 @@ func rowLineageExpandedCodingGroup(output lower.CompiledRecipeOutput) bool { sourceUnnest.Expression.Extract != nil && sourceUnnest.Expression.Extract.Selector.CanonicalPath() == group.CodingKey.CodingPath } -// rowLineageConstructionPivotPreimage admits only a direct terminal ordinary -// construction Pivot. Its typed key tuple selects the multi-root preimage; -// mixed-owner sequences remain unsupported until their owner chain is decoded. func rowLineageConstructionPivotPreimage(output lower.CompiledRecipeOutput) bool { sequence := output.Plan.StageSequence if sequence == nil || sequence.SourceRowIdentity != "_key" || len(sequence.Stages) != 1 || @@ -376,11 +372,6 @@ func rowLineageConstructionPivotPreimage(output lower.CompiledRecipeOutput) bool output.RootResourceType != "" && rowLineageHasPhysicalIdentity(output, stage, sequence) } -// rowLineageConstructionGroupPivotPreimage admits the bounded first composed -// construction shape: a direct-source scalar GROUP with COUNT_ROWS only, -// followed by an ordinary Pivot over every Group dimension. The Group is a -// fan-in owner set; the final Pivot's typed key tuple selects the relevant -// Group tuples, whose source preimage is streamed from the original root rows. func rowLineageConstructionGroupPivotPreimage(output lower.CompiledRecipeOutput) bool { sequence := output.Plan.StageSequence if sequence == nil || sequence.SourceRowIdentity != "_key" || len(sequence.Stages) != 2 || @@ -623,7 +614,6 @@ func rowLineageGroupFilterSuffix(sequence *ir.PhysicalStageSequence) bool { } func rowLineageHasPhysicalIdentity(output lower.CompiledRecipeOutput, stage ir.PhysicalConstructionStage, sequence *ir.PhysicalStageSequence) bool { - // Publication-grain metadata need not name a construction stage's physical row ID. return output.RowIdentity != nil && stage.RowIdentityColumn != "" && stage.RowIdentityColumn == sequence.FinalRowIdentity } @@ -644,32 +634,31 @@ func rowLineageHasDirectRootSource(operations []ir.PhysicalOperation) bool { return rootScans == 1 } -func CompileRowLineageOutput(output lower.CompiledRecipeOutput, rowID string, offset, limit int, policy ir.PhysicalOptimizationPolicy) (CompiledRowLineageQuery, error) { - rowID = strings.TrimSpace(rowID) - if rowID == "" { - return CompiledRowLineageQuery{}, fmt.Errorf("row lineage row ID is required") +func CompileRowLineageOutput(target Target, output lower.CompiledRecipeOutput, rowID string, offset, limit int, policy ir.PhysicalOptimizationPolicy) (CompiledRowLineageQuery, error) { + if err := target.validate(); err != nil { + return CompiledRowLineageQuery{}, err } - if offset < 0 { - return CompiledRowLineageQuery{}, fmt.Errorf("row lineage offset cannot be negative") + if target.backend == command.CompiledBackendPostgreSQL { + return compileRowLineagePostgreSQL(target, output, rowID, offset, limit) } - if limit == 0 { - limit = DefaultRowLineageLimit + if err := target.rejectPostgreSQL("row lineage compilation"); err != nil { + return CompiledRowLineageQuery{}, err } - if limit < 1 || limit > MaxRowLineageContributors { - return CompiledRowLineageQuery{}, fmt.Errorf("row lineage limit must be between 1 and %d", MaxRowLineageContributors) + var err error + if rowID, offset, limit, err = normalizeRowLineagePage(rowID, offset, limit); err != nil { + return CompiledRowLineageQuery{}, err } capability := RowLineageCapabilityForOutput(output) if !capability.Available { return CompiledRowLineageQuery{}, &RowLineageUnsupportedError{Capability: capability} } - expandedCodingGroup := rowLineageExpandedCodingGroup(output) - relatedCodingGroup := rowLineageRelatedCodingGroup(output) if output.RootResourceType == "" { return CompiledRowLineageQuery{}, fmt.Errorf("row lineage source resource type is required") } if output.RowIdentity == nil { return CompiledRowLineageQuery{}, fmt.Errorf("row lineage requires stable output row identity") } + _ = policy if rowLineageCohortGroupFilterSuffix(output) { sequence := output.Plan.StageSequence groupStage := sequence.Stages[0] @@ -685,8 +674,12 @@ func CompileRowLineageOutput(output lower.CompiledRecipeOutput, rowID string, of if err != nil { return CompiledRowLineageQuery{}, fmt.Errorf("render composed cohort row lineage: %w", err) } + compiledCommand, commandErr := command.NewAQLCommand(rendered.Query, rendered.BindVars) + if commandErr != nil { + return CompiledRowLineageQuery{}, fmt.Errorf("tag composed cohort row lineage command: %w", commandErr) + } return CompiledRowLineageQuery{ - Query: rendered.Query, BindVars: rendered.BindVars, Offset: offset, Limit: limit, + Command: compiledCommand, Offset: offset, Limit: limit, FoundColumn: "found", ContributorsColumn: "contributors", HasMoreColumn: "hasMore", }, nil } @@ -701,19 +694,94 @@ func CompileRowLineageOutput(output lower.CompiledRecipeOutput, rowID string, of if err != nil { return CompiledRowLineageQuery{}, fmt.Errorf("render explicit group row lineage: %w", err) } + compiledCommand, commandErr := command.NewAQLCommand(rendered.Query, rendered.BindVars) + if commandErr != nil { + return CompiledRowLineageQuery{}, fmt.Errorf("tag explicit group row lineage command: %w", commandErr) + } return CompiledRowLineageQuery{ - Query: rendered.Query, BindVars: rendered.BindVars, Offset: offset, Limit: limit, + Command: compiledCommand, Offset: offset, Limit: limit, FoundColumn: "found", ContributorsColumn: "contributors", HasMoreColumn: "hasMore", }, nil } + physical, err := prepareRowLineagePhysicalPlan(output, rowID, offset, limit) + if err != nil { + return CompiledRowLineageQuery{}, err + } + rendered, err := aql.RenderPhysicalPlan(physical) + if err != nil { + return CompiledRowLineageQuery{}, fmt.Errorf("render row lineage physical plan: %w", err) + } + compiledCommand, commandErr := command.NewAQLCommand(rendered.Query, rendered.BindVars) + if commandErr != nil { + return CompiledRowLineageQuery{}, fmt.Errorf("tag row lineage command: %w", commandErr) + } + return CompiledRowLineageQuery{ + Command: compiledCommand, Offset: offset, Limit: limit, + FoundColumn: "found", ContributorsColumn: "contributors", HasMoreColumn: "hasMore", + }, nil +} + +func compileRowLineagePostgreSQL(target Target, output lower.CompiledRecipeOutput, rowID string, offset, limit int) (CompiledRowLineageQuery, error) { + rowID, offset, limit, err := normalizeRowLineagePage(rowID, offset, limit) + if err != nil { + return CompiledRowLineageQuery{}, err + } + capability := RowLineageCapabilityForOutput(output) + if !capability.Available { + return CompiledRowLineageQuery{}, fmt.Errorf("row lineage compilation is unsupported on the PostgreSQL compile target: %w", errors.Join( + &RowLineageUnsupportedError{Capability: capability}, postgresrender.ErrUnsupportedPostgresPlan)) + } + if rowLineageStandaloneGroupRows(output) { + return CompiledRowLineageQuery{}, fmt.Errorf("explicit group row lineage is unsupported on the PostgreSQL compile target: %w", postgresrender.ErrUnsupportedPostgresPlan) + } + if rowLineageCohortGroupFilterSuffix(output) { + return CompiledRowLineageQuery{}, fmt.Errorf("composed cohort row lineage is unsupported on the PostgreSQL compile target: %w", postgresrender.ErrUnsupportedPostgresPlan) + } + if output.RootResourceType == "" { + return CompiledRowLineageQuery{}, fmt.Errorf("row lineage source resource type is required") + } + if output.RowIdentity == nil { + return CompiledRowLineageQuery{}, fmt.Errorf("row lineage requires stable output row identity") + } + physical, err := prepareRowLineagePhysicalPlan(output, rowID, offset, limit) + if err != nil { + return CompiledRowLineageQuery{}, err + } + compiledCommand, _, err := target.renderPostgreSQLFinalPlan(physical) + if err != nil { + return CompiledRowLineageQuery{}, err + } + return CompiledRowLineageQuery{ + Command: compiledCommand, Offset: offset, Limit: limit, + FoundColumn: "found", ContributorsColumn: "contributors", HasMoreColumn: "hasMore", + }, nil +} + +func normalizeRowLineagePage(rowID string, offset, limit int) (string, int, int, error) { + rowID = strings.TrimSpace(rowID) + if rowID == "" { + return "", 0, 0, fmt.Errorf("row lineage row ID is required") + } + if offset < 0 { + return "", 0, 0, fmt.Errorf("row lineage offset cannot be negative") + } + if limit == 0 { + limit = DefaultRowLineageLimit + } + if limit < 1 || limit > MaxRowLineageContributors { + return "", 0, 0, fmt.Errorf("row lineage limit must be between 1 and %d", MaxRowLineageContributors) + } + return rowID, offset, limit, nil +} + +func prepareRowLineagePhysicalPlan(output lower.CompiledRecipeOutput, rowID string, offset, limit int) (ir.PhysicalPlan, error) { physical := ir.ClonePhysicalPlan(output.Plan) - _ = policy // Row lineage uses the canonical pre-optimization stage semantics. if physical.StageSequence == nil { - return CompiledRowLineageQuery{}, fmt.Errorf("row lineage requires a construction stage sequence") + return ir.PhysicalPlan{}, fmt.Errorf("row lineage requires a construction stage sequence") } physical, err := withGenericPhysicalExecutionWindow(physical, 0) if err != nil { - return CompiledRowLineageQuery{}, fmt.Errorf("apply row lineage execution window: %w", err) + return ir.PhysicalPlan{}, fmt.Errorf("apply row lineage execution window: %w", err) } root := "" for _, operation := range physical.Operations { @@ -723,8 +791,10 @@ func CompileRowLineageOutput(output lower.CompiledRecipeOutput, rowID string, of } } if root == "" { - return CompiledRowLineageQuery{}, fmt.Errorf("row lineage requires a direct root source") + return ir.PhysicalPlan{}, fmt.Errorf("row lineage requires a direct root source") } + expandedCodingGroup := rowLineageExpandedCodingGroup(output) + relatedCodingGroup := rowLineageRelatedCodingGroup(output) var lineageTrace *ir.PhysicalRowLineageTrace if rowLineageHasRelatedExpand(physical.StageSequence) { rootKeyBindKey := "row_lineage_root_key" @@ -732,8 +802,6 @@ func CompileRowLineageOutput(output lower.CompiledRecipeOutput, rowID string, of rootKey, decoded, decodedOK := decodeRelatedRowLineageIdentity(rowID, physical.StageSequence.Stages, physical.BindVars) var rootKeyValue any = rootKey if !decodedOK { - // Null cannot be a stored Arango _key, so malformed, wrong-stage, and - // wrong-owner identities become an unmatchable query. rootKeyValue = nil for stageIndex, stage := range physical.StageSequence.Stages { if stage.Kind != ir.PhysicalStageRelatedExpandOp { @@ -763,14 +831,14 @@ func CompileRowLineageOutput(output lower.CompiledRecipeOutput, rowID string, of }}, } if len(physical.Operations) == 0 || physical.Operations[0].Kind != ir.PhysicalRootScanOp { - return CompiledRowLineageQuery{}, fmt.Errorf("row lineage could not anchor its direct root source") + return ir.PhysicalPlan{}, fmt.Errorf("row lineage could not anchor its direct root source") } if err := ir.ValidateGenericPhysicalPlanScope(physical); err != nil { - return CompiledRowLineageQuery{}, fmt.Errorf("validate row lineage source scopes before anchoring: %w", err) + return ir.PhysicalPlan{}, fmt.Errorf("validate row lineage source scopes before anchoring: %w", err) } anchorIndex := ir.PhysicalScopeWindowEnd(physical.Operations, 1) if anchorIndex <= 1 || anchorIndex >= len(physical.Operations) { - return CompiledRowLineageQuery{}, fmt.Errorf("row lineage source has no validated root scope before its projection") + return ir.PhysicalPlan{}, fmt.Errorf("row lineage source has no validated root scope before its projection") } physical.Operations = append(physical.Operations[:anchorIndex], append([]ir.PhysicalOperation{rootFilter}, physical.Operations[anchorIndex:]...)...) } else if rowLineageConstructionGroupPivotPreimage(output) { @@ -833,7 +901,7 @@ func CompileRowLineageOutput(output lower.CompiledRecipeOutput, rowID string, of } } if !found { - return CompiledRowLineageQuery{}, fmt.Errorf("expanded Group source is missing exact occurrence projection %q", occurrenceKeyColumn) + return ir.PhysicalPlan{}, fmt.Errorf("expanded Group source is missing exact occurrence projection %q", occurrenceKeyColumn) } } else if !relatedCodingGroup { operation.Return.Projections = appendUniqueProjection(operation.Return.Projections, @@ -842,7 +910,7 @@ func CompileRowLineageOutput(output lower.CompiledRecipeOutput, rowID string, of } } if returnCount != 1 { - return CompiledRowLineageQuery{}, fmt.Errorf("row lineage direct source requires one terminal projection") + return ir.PhysicalPlan{}, fmt.Errorf("row lineage direct source requires one terminal projection") } physical.BindVars[rowLineageRowIDBind] = rowID physical.BindVars[rowLineageOffsetBind] = offset @@ -856,19 +924,12 @@ func CompileRowLineageOutput(output lower.CompiledRecipeOutput, rowID string, of OccurrenceKeyColumn: occurrenceKeyColumn, } if err := physical.Validate(); err != nil { - return CompiledRowLineageQuery{}, fmt.Errorf("validate row lineage physical plan: %w", err) + return ir.PhysicalPlan{}, fmt.Errorf("validate row lineage physical plan: %w", err) } if err := ir.ValidateGenericPhysicalPlanScope(physical); err != nil { - return CompiledRowLineageQuery{}, fmt.Errorf("verify row lineage physical scope: %w", err) - } - rendered, err := aql.RenderPhysicalPlan(physical) - if err != nil { - return CompiledRowLineageQuery{}, fmt.Errorf("render row lineage physical plan: %w", err) + return ir.PhysicalPlan{}, fmt.Errorf("verify row lineage physical scope: %w", err) } - return CompiledRowLineageQuery{ - Query: rendered.Query, BindVars: rendered.BindVars, Offset: offset, Limit: limit, - FoundColumn: "found", ContributorsColumn: "contributors", HasMoreColumn: "hasMore", - }, nil + return physical, nil } func decodeConstructionPivotLineageIdentity(rowID string, pivot ir.PhysicalGroupedPivot) ([]any, bool) { diff --git a/internal/dataframe/compiler/row_lineage_cohort_filter_arango_test.go b/internal/dataframe/compiler/row_lineage_cohort_filter_arango_test.go index edccea213..5834be945 100644 --- a/internal/dataframe/compiler/row_lineage_cohort_filter_arango_test.go +++ b/internal/dataframe/compiler/row_lineage_cohort_filter_arango_test.go @@ -9,10 +9,10 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/projectid" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -29,17 +29,15 @@ func TestComposedCohortFilterLineageAgainstArango(t *testing.T) { revisionsCollection, selectionsCollection, definitionsCollection, membershipsCollection, selectionMembersCollection, "Observation", } - specs := make([]store.CollectionSpec, 0, len(collections)) + specs := make([]arangofactory.CollectionSpec, 0, len(collections)) for _, collection := range collections { - specs = append(specs, store.CollectionSpec{Name: collection}) + specs = append(specs, arangofactory.CollectionSpec{Name: collection}) } - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: specs}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: specs}); err != nil { t.Fatal(err) } project := "loom_cohort_filter_lineage_" + uuid.NewString() - // Production selection/group metadata uses canonical project identity; - // resource documents retain the legacy storage alias. selectionProject := projectid.Canonical(project) if selectionProject == project { t.Fatal("fixture must exercise distinct selection and resource project identities") @@ -142,7 +140,7 @@ func TestComposedCohortFilterLineageAgainstArango(t *testing.T) { if capability := RowLineageCapabilityForOutput(compiledOutput); !capability.Available { t.Fatalf("composed cohort FILTER lineage capability = %#v", capability) } - previewQuery, err := CompileRecipeOutputWithPolicy(compiledOutput, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + previewQuery, err := CompileRecipeOutput(ArangoTarget(), compiledOutput, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile filtered cohort preview: %v", err) } @@ -154,7 +152,7 @@ func TestComposedCohortFilterLineageAgainstArango(t *testing.T) { if err != nil { t.Fatal(err) } - lineage, err := CompileRowLineageOutput(compiledOutput, string(rowID), 0, 5, ir.DefaultPhysicalOptimizationPolicy()) + lineage, err := CompileRowLineageOutput(ArangoTarget(), compiledOutput, string(rowID), 0, 5, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile final filtered cohort row lineage: %v", err) } @@ -187,7 +185,7 @@ func TestComposedCohortFilterLineageAgainstArango(t *testing.T) { if err != nil { t.Fatal(err) } - filteredOut, err := CompileRowLineageOutput(compiledOutput, string(filteredOutID), 0, 5, ir.DefaultPhysicalOptimizationPolicy()) + filteredOut, err := CompileRowLineageOutput(ArangoTarget(), compiledOutput, string(filteredOutID), 0, 5, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile filtered-out cohort row lineage: %v", err) } @@ -197,7 +195,7 @@ func TestComposedCohortFilterLineageAgainstArango(t *testing.T) { if err != nil { t.Fatal(err) } - forged, err := CompileRowLineageOutput(compiledOutput, string(forgedID), 0, 5, ir.DefaultPhysicalOptimizationPolicy()) + forged, err := CompileRowLineageOutput(ArangoTarget(), compiledOutput, string(forgedID), 0, 5, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile forged cohort row lineage: %v", err) } diff --git a/internal/dataframe/compiler/row_lineage_composed_arango_test.go b/internal/dataframe/compiler/row_lineage_composed_arango_test.go index 26cec4c8e..66326a1ce 100644 --- a/internal/dataframe/compiler/row_lineage_composed_arango_test.go +++ b/internal/dataframe/compiler/row_lineage_composed_arango_test.go @@ -12,24 +12,19 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/recipe" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) -// This fixture independently enumerates membership roots and each authored -// relationship stage in raw AQL, then asks CompileRowLineageOutput to resolve -// those canonical row IDs. It emits the same Specimen at authored stages one -// and three: repeated paths collapse within each stage, while both authored -// occurrences remain distinct contributors. func TestComposedRelatedExpandLineageMatchesIndependentMembershipOracleAgainstArango(t *testing.T) { if os.Getenv("LOOM_TEST_ARANGO_URL") == "" || os.Getenv("LOOM_TEST_ARANGO_DATABASE") == "" { t.Skip("LOOM_TEST_ARANGO_URL and LOOM_TEST_ARANGO_DATABASE must point to the supplied Docker Arango service") } ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "Specimen"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, {Name: "loom_explorer_selection_members"}, }}); err != nil { @@ -102,9 +97,6 @@ func TestComposedRelatedExpandLineageMatchesIndependentMembershipOracleAgainstAr insertResource("Observation", id, "Observation", "/allowed", project, generation, nil) } - // Membership contains two allowed roots plus one denied root. p3 has a - // complete valid route but is not a member; p_bridge is an allowed route - // bridge but is neither selected nor authored as a terminal source record. members := make([]json.RawMessage, 0, 3) for _, id := range []string{"p1", "p2", "p_denied"} { doc, err := json.Marshal(map[string]any{ @@ -120,9 +112,6 @@ func TestComposedRelatedExpandLineageMatchesIndependentMembershipOracleAgainstAr t.Fatalf("insert selected-root membership: %v", err) } - // Authored stage one uses the generated Patient|subject_Patient|Specimen - // tuple. Duplicate paths to s1 collapse within this stage. The old-generation - // edge is the only route to s_stale and cannot admit that target. insertEdge("p1_s1_a", "Specimen", "s1", "Patient", "p1", "subject_Patient", project, generation, "/allowed") insertEdge("p1_s1_b", "Specimen", "s1", "Patient", "p1", "subject_Patient", project, generation, "/allowed") insertEdge("p1_s_empty", "Specimen", "s_empty", "Patient", "p1", "subject_Patient", project, generation, "/allowed") @@ -130,11 +119,6 @@ func TestComposedRelatedExpandLineageMatchesIndependentMembershipOracleAgainstAr insertEdge("p3_s3", "Specimen", "s3", "Patient", "p3", "subject_Patient", project, generation, "/allowed") insertEdge("p1_s_stale", "Specimen", "s_stale", "Patient", "p1", "subject_Patient", project, "old-generation", "/allowed") - // Authored stage two is the generated two-hop route Specimen -> Patient -> - // Observation. p_bridge is a bridge-only resource; two paths to o1 should - // still yield one authored terminal. The selected p2 row reaches an - // unauthorized bridge with a valid, visible Observation and Specimen beyond - // it; the bridge auth check is what must preserve the Specimen parent as empty. insertEdge("s1_bridge_a", "Specimen", "s1", "Patient", "p_bridge", "subject_Patient", project, generation, "/allowed") insertEdge("s1_bridge_b", "Specimen", "s1", "Patient", "p_bridge", "subject_Patient", project, generation, "/allowed") insertEdge("o1_bridge_a", "Observation", "o1", "Patient", "p_bridge", "subject_Patient", project, generation, "/allowed") @@ -144,9 +128,6 @@ func TestComposedRelatedExpandLineageMatchesIndependentMembershipOracleAgainstAr insertEdge("o3_decoy_bridge", "Observation", "o3", "Patient", "p3", "subject_Patient", project, generation, "/allowed") insertEdge("s3_decoy_bridge", "Specimen", "s3", "Patient", "p3", "subject_Patient", project, generation, "/allowed") - // Authored stage three returns to s1 using the supported Observation to - // Specimen reference. Duplicate edge witnesses again collapse within the - // stage, while s1 remains present a second time after stage one. insertEdge("o1_s1_a", "Observation", "o1", "Specimen", "s1", "specimen_Specimen", project, generation, "/allowed") insertEdge("o1_s1_b", "Observation", "o1", "Specimen", "s1", "specimen_Specimen", project, generation, "/allowed") insertEdge("o2_s2", "Observation", "o2", "Specimen", "s2", "specimen_Specimen", project, generation, "/allowed") @@ -161,7 +142,7 @@ func TestComposedRelatedExpandLineageMatchesIndependentMembershipOracleAgainstAr if capability := RowLineageCapabilityForOutput(output); !capability.Available { t.Fatalf("composed related expansion row lineage capability = %#v", capability) } - previewQuery, err := CompileRecipeOutputWithPolicy(output, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + previewQuery, err := CompileRecipeOutput(ArangoTarget(), output, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -202,9 +183,6 @@ func TestComposedRelatedExpandLineageMatchesIndependentMembershipOracleAgainstAr t.Fatalf("independent oracle rows = %#v, want p1/s1, p1/s_empty, p2/s2, excluding denied and nonmember roots", oracleRows) } - // The full row includes Patient/p1, Specimen/s1, Observation/o1, and the - // same Specimen/s1 emitted again. limit=2 checks stage order, limit+1 - // hasMore, continuation, and past-end behavior. var fullRowID string for rowID, oracle := range oracleByRowID { if oracle["root_id"] == "p1" && oracle["specimen_id"] == "s1" && oracle["observation_id"] == "o1" && oracle["returned_specimen_id"] == "s1" { @@ -216,7 +194,7 @@ func TestComposedRelatedExpandLineageMatchesIndependentMembershipOracleAgainstAr } page := func(offset int) map[string]any { t.Helper() - compiled, compileErr := CompileRowLineageOutput(output, fullRowID, offset, 2, ir.DefaultPhysicalOptimizationPolicy()) + compiled, compileErr := CompileRowLineageOutput(ArangoTarget(), output, fullRowID, offset, 2, ir.DefaultPhysicalOptimizationPolicy()) if compileErr != nil { t.Fatalf("compile composed row lineage at offset %d: %v", offset, compileErr) } @@ -243,8 +221,6 @@ func TestComposedRelatedExpandLineageMatchesIndependentMembershipOracleAgainstAr } } - // A later preserved empty emits no null/bridge contributor. This exercises - // both no matching route and a route whose only bridge is denied by scope. for rowID, oracle := range oracleByRowID { if (oracle["root_id"] != "p1" || oracle["specimen_id"] != "s_empty") && (oracle["root_id"] != "p2" || oracle["specimen_id"] != "s2") { @@ -253,7 +229,7 @@ func TestComposedRelatedExpandLineageMatchesIndependentMembershipOracleAgainstAr if oracle["observation_id"] != nil || oracle["returned_specimen_id"] != nil { t.Fatalf("empty oracle row unexpectedly has related terminals: %#v", oracle) } - compiled, compileErr := CompileRowLineageOutput(output, rowID, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + compiled, compileErr := CompileRowLineageOutput(ArangoTarget(), output, rowID, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) if compileErr != nil { t.Fatalf("compile preserved-empty composed lineage: %v", compileErr) } @@ -266,8 +242,6 @@ func TestComposedRelatedExpandLineageMatchesIndependentMembershipOracleAgainstAr } } - // Validly shaped identities for a nonmember root, a denied bridge terminal, - // a forged stage-two terminal, and malformed JSON all fail closed. stage1ID, stage2ID, stage3ID := stepIDs["expand_specimens"], stepIDs["expand_observations"], stepIDs["return_specimen"] s2Parent := relatedExpandRowID(t, key("p2"), stage1ID, "Specimen/"+key("s2")) forgedRows := []string{ @@ -277,7 +251,7 @@ func TestComposedRelatedExpandLineageMatchesIndependentMembershipOracleAgainstAr "not-json", } for _, forged := range forgedRows { - compiled, compileErr := CompileRowLineageOutput(output, forged, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + compiled, compileErr := CompileRowLineageOutput(ArangoTarget(), output, forged, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) if compileErr != nil { t.Fatalf("compile fail-closed composed identity %q: %v", forged, compileErr) } @@ -377,11 +351,8 @@ func composedRelatedExpandConstructionIDs(t *testing.T, output lower.CompiledRec return ids } -func executeComposedRelatedExpandOracle(t *testing.T, ctx context.Context, client *store.Client, project, generation, selectionID string) []map[string]any { +func executeComposedRelatedExpandOracle(t *testing.T, ctx context.Context, client *arangofactory.Client, project, generation, selectionID string) []map[string]any { t.Helper() - // This is a source-side oracle, independent from construction compilation: - // it spells out membership, every typed route hop, every edge label, and all - // project/generation/auth guards before deduplicating per authored stage. query := ` FOR member IN loom_explorer_selection_members FILTER member.selectionId == @selection_id AND member.project == @project AND member.generation == @generation AND member.resourceType == "Patient" diff --git a/internal/dataframe/compiler/row_lineage_group_pivot_arango_test.go b/internal/dataframe/compiler/row_lineage_group_pivot_arango_test.go index a5bce66ef..41c2f43f2 100644 --- a/internal/dataframe/compiler/row_lineage_group_pivot_arango_test.go +++ b/internal/dataframe/compiler/row_lineage_group_pivot_arango_test.go @@ -41,8 +41,6 @@ func TestConstructionCountRowsGroupPivotRowLineageMatchesScopedSourcePreimageAga {id: "d_explicit_null", patient: "Patient/target", project: project, generation: generation, authPath: "/allowed", status: nil}, {id: "e_missing_status", patient: "Patient/target", project: project, generation: generation, authPath: "/allowed"}, {id: "other_active", patient: "Patient/other", project: project, generation: generation, authPath: "/allowed", status: &active}, - // These match the requested Pivot row keys and category, but each violates - // one source boundary. They must affect neither preview counts nor lineage. {id: "z_denied", patient: "Patient/target", project: project, generation: generation, authPath: "/denied", status: &active}, {id: "z_old_generation", patient: "Patient/target", project: project, generation: "old-generation", authPath: "/allowed", status: &active}, {id: "z_foreign_project", patient: "Patient/target", project: foreignProject, generation: generation, authPath: "/allowed", status: &active}, @@ -87,7 +85,7 @@ func TestConstructionCountRowsGroupPivotRowLineageMatchesScopedSourcePreimageAga if capability := RowLineageCapabilityForOutput(compiledOutput); !capability.Available { t.Fatalf("COUNT_ROWS Group/Pivot row-lineage capability = %#v", capability) } - preview, err := CompileRecipeOutputWithPolicy(compiledOutput, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + preview, err := CompileRecipeOutput(ArangoTarget(), compiledOutput, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -126,7 +124,7 @@ func TestConstructionCountRowsGroupPivotRowLineageMatchesScopedSourcePreimageAga compilePage := func(requestedID string, offset int) CompiledRowLineageQuery { t.Helper() - compiled, compileErr := CompileRowLineageOutput(compiledOutput, requestedID, offset, 2, ir.DefaultPhysicalOptimizationPolicy()) + compiled, compileErr := CompileRowLineageOutput(ArangoTarget(), compiledOutput, requestedID, offset, 2, ir.DefaultPhysicalOptimizationPolicy()) if compileErr != nil { t.Fatalf("compile Group/Pivot lineage page at offset %d: %v", offset, compileErr) } @@ -134,13 +132,13 @@ func TestConstructionCountRowsGroupPivotRowLineageMatchesScopedSourcePreimageAga "root.project == @project", "root.dataset_generation == @dataset_generation", "root.auth_resource_path IN @auth_resource_paths", "LIMIT @row_lineage_offset, @row_lineage_fetch_limit", } { - if !strings.Contains(compiled.Query, want) { - t.Errorf("Group/Pivot row-lineage query lost source scope or bounded page clause %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Errorf("Group/Pivot row-lineage query lost source scope or bounded page clause %q:\n%s", want, testAQLText(t, compiled.Command)) } } - if strings.Contains(compiled.Query, "COLLECT INTO") || strings.Contains(compiled.Query, "SORTED_UNIQUE") || - strings.Contains(compiled.Query, "__loom_root_contributor_keys") { - t.Errorf("Group/Pivot row-lineage query relied on materialized contributor arrays or sidecars:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), "COLLECT INTO") || strings.Contains(testAQLText(t, compiled.Command), "SORTED_UNIQUE") || + strings.Contains(testAQLText(t, compiled.Command), "__loom_root_contributor_keys") { + t.Errorf("Group/Pivot row-lineage query relied on materialized contributor arrays or sidecars:\n%s", testAQLText(t, compiled.Command)) } return compiled } @@ -183,14 +181,10 @@ func TestConstructionCountRowsGroupPivotRowLineageMatchesScopedSourcePreimageAga t.Fatalf("wrong-construction Group/Pivot identity disclosed source rows: %#v", forged) } - // The unlisted category belongs to another Pivot row and sorts after the - // requested contributor page. Full-scope policy validation must still fail. insert([]fixture{{id: "zz_unlisted_other", patient: "Patient/unrelated", project: project, generation: generation, authPath: "/allowed", status: &unlisted}}) lineage := compilePage(rowID, 0) - err = client.QueryRows(ctx, lineage.Query, 32, lineage.BindVars, func(map[string]any) error { return nil }) - if err == nil || !strings.Contains(err.Error(), "TABLE_PIVOT_UNLISTED_CATEGORY") { - t.Fatalf("unlisted category outside the requested row/page produced error %v, want TABLE_PIVOT_UNLISTED_CATEGORY", err) - } + err = client.QueryRows(ctx, testAQLText(t, lineage.Command), 32, testAQLBinds(t, lineage.Command), func(map[string]any) error { return nil }) + requireStableRuntimeCode(t, err, "TABLE_PIVOT_UNLISTED_CATEGORY") } func constructionCountRowsGroupPivotLineageArangoOutput() recipe.Output { diff --git a/internal/dataframe/compiler/row_lineage_group_rows_arango_test.go b/internal/dataframe/compiler/row_lineage_group_rows_arango_test.go index d1bbade33..0affcfa64 100644 --- a/internal/dataframe/compiler/row_lineage_group_rows_arango_test.go +++ b/internal/dataframe/compiler/row_lineage_group_rows_arango_test.go @@ -10,17 +10,13 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/projectid" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) -// This exercises standalone explicit-cohort row lineage against the same -// pinned membership documents used by GROUP_ROWS rendering. It intentionally -// uses a restricted scope so a revoked member cannot be confused with a -// missing source resource. func TestStandaloneExplicitGroupRowLineagePagesAgainstArango(t *testing.T) { ctx, client := openConstructionReshapeArango(t) const ( @@ -34,17 +30,15 @@ func TestStandaloneExplicitGroupRowLineagePagesAgainstArango(t *testing.T) { revisionsCollection, selectionsCollection, definitionsCollection, membershipsCollection, selectionMembersCollection, } - specs := make([]store.CollectionSpec, 0, len(collections)) + specs := make([]arangofactory.CollectionSpec, 0, len(collections)) for _, collection := range collections { - specs = append(specs, store.CollectionSpec{Name: collection}) + specs = append(specs, arangofactory.CollectionSpec{Name: collection}) } - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: specs}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: specs}); err != nil { t.Fatal(err) } project := "loom_cohort_lineage_" + uuid.NewString() - // Production selection/group metadata uses canonical project identity; - // resource documents retain the legacy storage alias. selectionProject := projectid.Canonical(project) if selectionProject == project { t.Fatal("fixture must exercise distinct selection and resource project identities") @@ -119,8 +113,6 @@ func TestStandaloneExplicitGroupRowLineagePagesAgainstArango(t *testing.T) { map[string]any{"_key": project + "_resource_" + revokedID, "id": revokedID, "project": project, "dataset_generation": generation, "resourceType": "Observation", "auth_resource_path": "/revoked", "payload": map[string]any{"id": revokedID}}, map[string]any{"_key": project + "_resource_" + unassignedID, "id": unassignedID, "project": project, "dataset_generation": generation, "resourceType": "Observation", "auth_resource_path": "/allowed", "payload": map[string]any{"id": unassignedID}}, ) - // missing-c has a verified selection and assigned membership but no current - // resource document. GROUP_ROWS preserves that identity with a null payload. bindings := recipe.RuntimeBindings{ Project: project, SelectionProject: selectionProject, DatasetGeneration: generation, @@ -141,7 +133,7 @@ func TestStandaloneExplicitGroupRowLineagePagesAgainstArango(t *testing.T) { if err != nil { t.Fatal(err) } - query, err := CompileRowLineageOutput(compiled, string(rowID), offset, limit, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRowLineageOutput(ArangoTarget(), compiled, string(rowID), offset, limit, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile standalone cohort lineage page for %s at %d: %v", groupID, offset, err) } @@ -185,10 +177,7 @@ func TestStandaloneExplicitGroupRowLineagePagesAgainstArango(t *testing.T) { unassigned := executeRowLineageOracleQuery(t, ctx, client, compilePage("__loom_unassigned__", 0, 2)) assertPage(unassigned, []string{unassignedID}, []string{project + "_selected_" + unassignedID}, true, false) - // Compare these identities with canonical GROUP_ROWS materialization so - // revoked members are filtered and a missing current resource remains an - // identity-only contributor in both renderers. - groupRowsQuery, err := CompileRecipeOutputWithPolicy(compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + groupRowsQuery, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile canonical standalone GROUP_ROWS fixture: %v", err) } @@ -224,14 +213,12 @@ func TestStandaloneExplicitGroupRowLineagePagesAgainstArango(t *testing.T) { if err != nil { t.Fatal(err) } - forged, err := CompileRowLineageOutput(compiled, string(forgedID), 0, 5, ir.DefaultPhysicalOptimizationPolicy()) + forged, err := CompileRowLineageOutput(ArangoTarget(), compiled, string(forgedID), 0, 5, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile forged cross-revision row identity: %v", err) } assertPage(executeRowLineageOracleQuery(t, ctx, client, forged), nil, nil, false, false) - // A stale source membership digest must fail the same guard used by the - // canonical GROUP_ROWS renderer, before any contributor page is returned. insert(revisionsCollection, map[string]any{ "_key": staleRevisionID, "state": "COMPLETE", "project": selectionProject, "generation": generation, "resourceType": "Observation", "sourceSelectionRevisionId": staleSelectionID, @@ -250,11 +237,11 @@ func TestStandaloneExplicitGroupRowLineagePagesAgainstArango(t *testing.T) { if err != nil { t.Fatal(err) } - staleQuery, err := CompileRowLineageOutput(staleCompiled, string(staleRowID), 0, 1, ir.DefaultPhysicalOptimizationPolicy()) + staleQuery, err := CompileRowLineageOutput(ArangoTarget(), staleCompiled, string(staleRowID), 0, 1, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile stale standalone cohort row lineage: %v", err) } - queryErr := client.QueryRows(ctx, staleQuery.Query, 32, staleQuery.BindVars, func(map[string]any) error { return nil }) + queryErr := client.QueryRows(ctx, testAQLText(t, staleQuery.Command), 32, testAQLBinds(t, staleQuery.Command), func(map[string]any) error { return nil }) if queryErr == nil || !strings.Contains(queryErr.Error(), "EXPLICIT_GROUP_SOURCE_STALE") { t.Fatalf("source membership digest mismatch error = %v, want EXPLICIT_GROUP_SOURCE_STALE", queryErr) } diff --git a/internal/dataframe/compiler/row_lineage_test.go b/internal/dataframe/compiler/row_lineage_test.go index d95df66f4..8939f4b2f 100644 --- a/internal/dataframe/compiler/row_lineage_test.go +++ b/internal/dataframe/compiler/row_lineage_test.go @@ -4,15 +4,18 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "reflect" "strings" "testing" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/compiler/render/aql" + renderpostgres "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" "github.com/calypr/loom/internal/dataframe/recipe" ) @@ -49,7 +52,7 @@ func TestSemanticGroupUsesExactFamilyTupleAndReconstructsContributors(t *testing t.Fatalf("display fallback must read the tuple's code slot, not an object attribute:\n%s", groupPlan.Query) } - lineage, err := CompileRowLineageOutput(output, "semantic-row-id", 0, 5, ir.DefaultPhysicalOptimizationPolicy()) + lineage, err := CompileRowLineageOutput(ArangoTarget(), output, "semantic-row-id", 0, 5, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -62,8 +65,8 @@ func TestSemanticGroupUsesExactFamilyTupleAndReconstructsContributors(t *testing "FILTER [__loom_physical_construction_row_lineage_semantic_membership_1.system, __loom_physical_construction_row_lineage_semantic_membership_1.version, __loom_physical_construction_row_lineage_semantic_membership_1.code] == __loom_physical_construction_row_lineage_semantic_selected_group.semanticTuple", "LIMIT @row_lineage_offset, @row_lineage_fetch_limit", } { - if !strings.Contains(lineage.Query, want) { - t.Errorf("semantic Group lineage query is missing %q:\n%s", want, lineage.Query) + if !strings.Contains(testAQLText(t, lineage.Command), want) { + t.Errorf("semantic Group lineage query is missing %q:\n%s", want, testAQLText(t, lineage.Command)) } } } @@ -76,7 +79,7 @@ func TestExpandedCodingGroupLineagePagesExactInputOccurrences(t *testing.T) { if capability := RowLineageCapabilityForOutput(output); !capability.Available { t.Fatalf("expanded Coding Group row lineage is unavailable: %#v", capability) } - compiled, err := CompileRowLineageOutput(output, "opaque-expanded-coded-group-row", 4, 7, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), output, "opaque-expanded-coded-group-row", 4, 7, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile expanded Coding Group lineage: %v", err) } @@ -89,25 +92,25 @@ func TestExpandedCodingGroupLineagePagesExactInputOccurrences(t *testing.T) { "LIMIT @row_lineage_offset, @row_lineage_fetch_limit", "occurrenceKey: __loom_physical_construction_row_lineage_coding_contributor", } { - if !strings.Contains(compiled.Query, want) { - t.Errorf("expanded Coding Group lineage query missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Errorf("expanded Coding Group lineage query missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } - if strings.Count(compiled.Query, "FOR root IN @@root_collection") != 2 || - strings.Count(compiled.Query, "FOR __loom_physical_unnest_index_0") != 2 { - t.Fatalf("lineage must requery the exact scoped root/UNNEST preimage twice:\n%s", compiled.Query) + if strings.Count(testAQLText(t, compiled.Command), "FOR root IN @@root_collection") != 2 || + strings.Count(testAQLText(t, compiled.Command), "FOR __loom_physical_unnest_index_0") != 2 { + t.Fatalf("lineage must requery the exact scoped root/UNNEST preimage twice:\n%s", testAQLText(t, compiled.Command)) } - if strings.Contains(compiled.Query, "UNIQUE(") || strings.Contains(compiled.Query, "SORT __loom_physical_construction_row_lineage_coding_contributor[@__loom_physical_construction_row_lineage_coding_occurrence_key_column] ASC") == false { - t.Fatalf("expanded occurrence lineage must page exact occurrence identities without owner dedup:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), "UNIQUE(") || strings.Contains(testAQLText(t, compiled.Command), "SORT __loom_physical_construction_row_lineage_coding_contributor[@__loom_physical_construction_row_lineage_coding_occurrence_key_column] ASC") == false { + t.Fatalf("expanded occurrence lineage must page exact occurrence identities without owner dedup:\n%s", testAQLText(t, compiled.Command)) } foundExactOccurrence := false - for _, value := range compiled.BindVars { + for _, value := range testAQLBinds(t, compiled.Command) { if value == "__loom_group_source_occurrence_key" { foundExactOccurrence = true } } if !foundExactOccurrence { - t.Fatalf("lineage binds did not pin the full expanded occurrence column: %#v", compiled.BindVars) + t.Fatalf("lineage binds did not pin the full expanded occurrence column: %#v", testAQLBinds(t, compiled.Command)) } } @@ -125,7 +128,7 @@ func TestRelatedCodingGroupLineageReplaysExactRouteAndCodingOrdinal(t *testing.T } adapter := stage.Group.RelatedOccurrence rowID := `[["construction","group_related_id"],["operation","GROUP"],["status_group_id","final"],["coding_group_id","coding_identity",["urn:example:codes","1","same-code"]]]` - compiled, err := CompileRowLineageOutput(output, rowID, 2, 5, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), output, rowID, 2, 5, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile related Coding Group row lineage: %v", err) } @@ -133,16 +136,16 @@ func TestRelatedCodingGroupLineageReplaysExactRouteAndCodingOrdinal(t *testing.T if terminal != nil { t.Fatal("compilation mutated its input output plan") } - if compiled.BindVars[rowLineageRowIDBind] != rowID || compiled.BindVars[rowLineageOffsetBind] != 2 || - compiled.BindVars[rowLineageFetchLimitBind] != 6 || compiled.BindVars[rowLineageLimitBind] != 5 { - t.Fatalf("lineage query has wrong identity/page bindings: %#v", compiled.BindVars) + if testAQLBinds(t, compiled.Command)[rowLineageRowIDBind] != rowID || testAQLBinds(t, compiled.Command)[rowLineageOffsetBind] != 2 || + testAQLBinds(t, compiled.Command)[rowLineageFetchLimitBind] != 6 || testAQLBinds(t, compiled.Command)[rowLineageLimitBind] != 5 { + t.Fatalf("lineage query has wrong identity/page bindings: %#v", testAQLBinds(t, compiled.Command)) } - if !rowLineageHasBindValue(compiled.BindVars, adapter.OccurrenceKeyColumn) || - !rowLineageHasBindValue(compiled.BindVars, rowLineageResourceID) { - t.Fatalf("lineage output does not expose the exact Group occurrence and root resource ID columns: %#v", compiled.BindVars) + if !rowLineageHasBindValue(testAQLBinds(t, compiled.Command), adapter.OccurrenceKeyColumn) || + !rowLineageHasBindValue(testAQLBinds(t, compiled.Command), rowLineageResourceID) { + t.Fatalf("lineage output does not expose the exact Group occurrence and root resource ID columns: %#v", testAQLBinds(t, compiled.Command)) } - if _, ok := compiled.BindVars["@related_expand_0_hop_1_edge_collection"]; !ok { - t.Fatalf("related Group route collection bind is missing: %#v", compiled.BindVars) + if _, ok := testAQLBinds(t, compiled.Command)["@related_expand_0_hop_1_edge_collection"]; !ok { + t.Fatalf("related Group route collection bind is missing: %#v", testAQLBinds(t, compiled.Command)) } for _, want := range []string{ "row_lineage_related_selected", @@ -158,26 +161,26 @@ func TestRelatedCodingGroupLineageReplaysExactRouteAndCodingOrdinal(t *testing.T "LIMIT @row_lineage_offset, @row_lineage_fetch_limit", "occurrenceKey: __loom_physical_construction_row_lineage_related_contributor[@__loom_physical_construction_row_lineage_related_occurrence_key_column]", } { - if !strings.Contains(compiled.Query, want) { - t.Errorf("related Coding Group lineage query is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Errorf("related Coding Group lineage query is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } for _, field := range []string{"root_id", "terminal_id", "item_present", "ordinal"} { - if !rowLineageHasBindValue(compiled.BindVars, field) { + if !rowLineageHasBindValue(testAQLBinds(t, compiled.Command), field) { t.Errorf("lineage occurrence projection does not bind %q", field) } } - if !strings.Contains(compiled.Query, " == 0 ? [null] : RANGE(0, LENGTH(") || - !strings.Contains(compiled.Query, " AND "+adapter.CodingUnnest.HasItemVariable) { - t.Fatalf("lineage replay lost the route/Coding empty sentinels or route-gated Coding presence:\n%s", compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), " == 0 ? [null] : RANGE(0, LENGTH(") || + !strings.Contains(testAQLText(t, compiled.Command), " AND "+adapter.CodingUnnest.HasItemVariable) { + t.Fatalf("lineage replay lost the route/Coding empty sentinels or route-gated Coding presence:\n%s", testAQLText(t, compiled.Command)) } - if strings.Count(compiled.Query, "FOR root IN @@root_collection") != 2 || - strings.Count(compiled.Query, "FOR __loom_construction_related_expand_0_hop_1_edge_1 IN @@related_expand_0_hop_1_edge_collection") != 2 || - strings.Count(compiled.Query, "LET "+adapter.CodingUnnest.OutputVariable+" =") != 2 { - t.Fatalf("lineage must replay the same scoped root, route, and Coding expansion for lookup and paging:\n%s", compiled.Query) + if strings.Count(testAQLText(t, compiled.Command), "FOR root IN @@root_collection") != 2 || + strings.Count(testAQLText(t, compiled.Command), "FOR __loom_construction_related_expand_0_hop_1_edge_1 IN @@related_expand_0_hop_1_edge_collection") != 2 || + strings.Count(testAQLText(t, compiled.Command), "LET "+adapter.CodingUnnest.OutputVariable+" =") != 2 { + t.Fatalf("lineage must replay the same scoped root, route, and Coding expansion for lookup and paging:\n%s", testAQLText(t, compiled.Command)) } - if strings.Contains(compiled.Query, " INTO ") { - t.Fatalf("related Coding Group lineage must preserve Coding ordinals without materializing or deduplicating contributors:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), " INTO ") { + t.Fatalf("related Coding Group lineage must preserve Coding ordinals without materializing or deduplicating contributors:\n%s", testAQLText(t, compiled.Command)) } unsupported := output @@ -343,25 +346,25 @@ func TestExpandedCodingGroupLineageReusesRoutedAncestorOccurrence(t *testing.T) if capability := RowLineageCapabilityForOutput(output); !capability.Available { t.Fatalf("routed expanded Coding Group row lineage is unavailable: %#v", capability) } - compiled, err := CompileRowLineageOutput(output, "routed-expanded-row", 0, 7, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), output, "routed-expanded-row", 0, 7, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile routed expanded Coding Group lineage: %v", err) } - if strings.Count(compiled.Query, "FOR population_member IN @@population_members_collection") != 2 || - strings.Count(compiled.Query, "FOR __loom_physical_unnest_index_0") != 2 { - t.Fatalf("routed lineage must requery the same root/route/UNNEST preimage twice:\n%s", compiled.Query) + if strings.Count(testAQLText(t, compiled.Command), "FOR population_member IN @@population_members_collection") != 2 || + strings.Count(testAQLText(t, compiled.Command), "FOR __loom_physical_unnest_index_0") != 2 { + t.Fatalf("routed lineage must requery the same root/route/UNNEST preimage twice:\n%s", testAQLText(t, compiled.Command)) } for _, want := range []string{ "row_lineage_coding_contributor", "occurrenceKey:", "component", "coding", "ordinal", } { - if !strings.Contains(compiled.Query, want) { - t.Errorf("routed expanded Coding Group lineage lacks %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Errorf("routed expanded Coding Group lineage lacks %q:\n%s", want, testAQLText(t, compiled.Command)) } } - bindText := fmt.Sprint(compiled.BindVars) + bindText := fmt.Sprint(testAQLBinds(t, compiled.Command)) for _, want := range []string{"subject_Patient", "edge_00_id", "owner_id", "item_present"} { if !strings.Contains(bindText, want) { - t.Errorf("lineage binds omitted routed occurrence fact %q: %#v", want, compiled.BindVars) + t.Errorf("lineage binds omitted routed occurrence fact %q: %#v", want, testAQLBinds(t, compiled.Command)) } } } @@ -384,37 +387,37 @@ func TestExpandedCodingGroupLineageCapabilityAcceptsExactPopulationRouteFixture( if capability := RowLineageCapabilityForOutput(output); !capability.Available { t.Fatalf("exact population/route/UNNEST Coding Group lineage capability = %#v", capability) } - groupQuery, err := CompileRecipeOutputWithPolicy(output, recipe.RuntimeBindings{ + groupQuery, err := CompileRecipeOutput(ArangoTarget(), output, recipe.RuntimeBindings{ Project: "row-lineage-project", SelectionProject: "row-lineage-project", SelectionMembersCollection: "loom_explorer_selection_members", DatasetGeneration: "row-lineage-generation", }, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile exact fixture Group query: %v", err) } - if !strings.HasPrefix(groupQuery.Query, "LET __loom_construction_stage_1 = (\n FOR population_member IN @@population_members_collection") || - strings.Count(groupQuery.Query, "FOR population_member IN @@population_members_collection") != 1 || - strings.Count(groupQuery.Query, "FOR __loom_expansion_edge_1 IN @@expansion_route_1_edge_collection") != 1 || - strings.Count(groupQuery.Query, "FOR __loom_physical_unnest_index_0") != 1 || - strings.Contains(groupQuery.Query, "LET __loom_construction_source_projection = (") || - strings.Contains(groupQuery.Query, " INTO __loom_construction_group_rows") || - strings.Contains(groupQuery.Query, "__loom_root_contributor_keys") || - strings.Contains(groupQuery.Query, "__loom_construction_root_contributors") { - t.Fatalf("exact fixture Group did not stream its scoped population/route/UNNEST into COLLECT:\n%s", groupQuery.Query) - } - if !strings.Contains(groupQuery.Query, "COLLECT __loom_construction_group_key_1 =") || - !strings.Contains(groupQuery.Query, " AGGREGATE __loom_physical_construction_construction_group_count_rows = SUM(") || - strings.Contains(groupQuery.Query, " INTO ") { - t.Fatalf("exact fixture Group did not aggregate rows directly without materializing Group rows:\n%s", groupQuery.Query) - } - inputStart := strings.Index(groupQuery.Query, "LET __loom_construction_input_1 = {") + if !strings.HasPrefix(testAQLText(t, groupQuery.Command), "LET __loom_construction_stage_1 = (\n FOR population_member IN @@population_members_collection") || + strings.Count(testAQLText(t, groupQuery.Command), "FOR population_member IN @@population_members_collection") != 1 || + strings.Count(testAQLText(t, groupQuery.Command), "FOR __loom_expansion_edge_1 IN @@expansion_route_1_edge_collection") != 1 || + strings.Count(testAQLText(t, groupQuery.Command), "FOR __loom_physical_unnest_index_0") != 1 || + strings.Contains(testAQLText(t, groupQuery.Command), "LET __loom_construction_source_projection = (") || + strings.Contains(testAQLText(t, groupQuery.Command), " INTO __loom_construction_group_rows") || + strings.Contains(testAQLText(t, groupQuery.Command), "__loom_root_contributor_keys") || + strings.Contains(testAQLText(t, groupQuery.Command), "__loom_construction_root_contributors") { + t.Fatalf("exact fixture Group did not stream its scoped population/route/UNNEST into COLLECT:\n%s", testAQLText(t, groupQuery.Command)) + } + if !strings.Contains(testAQLText(t, groupQuery.Command), "COLLECT __loom_construction_group_key_1 =") || + !strings.Contains(testAQLText(t, groupQuery.Command), " AGGREGATE __loom_physical_construction_construction_group_count_rows = SUM(") || + strings.Contains(testAQLText(t, groupQuery.Command), " INTO ") { + t.Fatalf("exact fixture Group did not aggregate rows directly without materializing Group rows:\n%s", testAQLText(t, groupQuery.Command)) + } + inputStart := strings.Index(testAQLText(t, groupQuery.Command), "LET __loom_construction_input_1 = {") if inputStart < 0 { - t.Fatalf("exact fixture Group has no narrow per-row source projection:\n%s", groupQuery.Query) + t.Fatalf("exact fixture Group has no narrow per-row source projection:\n%s", testAQLText(t, groupQuery.Command)) } - inputEnd := strings.Index(groupQuery.Query[inputStart:], "LET __loom_physical_construction_construction_group_coding_membership =") + inputEnd := strings.Index(testAQLText(t, groupQuery.Command)[inputStart:], "LET __loom_physical_construction_construction_group_coding_membership =") if inputEnd < 0 { - t.Fatalf("exact fixture Group has no narrow per-row source projection:\n%s", groupQuery.Query) + t.Fatalf("exact fixture Group has no narrow per-row source projection:\n%s", testAQLText(t, groupQuery.Command)) } - inputProjection := groupQuery.Query[inputStart : inputStart+inputEnd] + inputProjection := testAQLText(t, groupQuery.Command)[inputStart : inputStart+inputEnd] for _, want := range []string{ "__loom_expansion_node_1.payload.id", "__loom_expansion_node_1.payload.status", "coding_item.code", "coding_item.display", "coding_item.system", "coding_item.version", @@ -424,19 +427,19 @@ func TestExpandedCodingGroupLineageCapabilityAcceptsExactPopulationRouteFixture( t.Errorf("exact fixture Group projection omitted selected scalar/occurrence fact %q:\n%s", want, inputProjection) } } - if strings.Contains(inputProjection, "coding_item,") || !strings.Contains(groupQuery.Query, "construction_group_row_value_arrays_0 = UNIQUE(") { - t.Fatalf("Group input must project compact Coding fields and only the requested ALL RowValue, not whole Coding payloads:\nprojection=%s\nquery=%s", inputProjection, groupQuery.Query) + if strings.Contains(inputProjection, "coding_item,") || !strings.Contains(testAQLText(t, groupQuery.Command), "construction_group_row_value_arrays_0 = UNIQUE(") { + t.Fatalf("Group input must project compact Coding fields and only the requested ALL RowValue, not whole Coding payloads:\nprojection=%s\nquery=%s", inputProjection, testAQLText(t, groupQuery.Command)) } - compiled, err := CompileRowLineageOutput(output, "fixture-expanded-coded-group-row", 0, 20, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), output, "fixture-expanded-coded-group-row", 0, 20, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile exact fixture Coding Group lineage: %v", err) } - if strings.Count(compiled.Query, "FOR population_member IN @@population_members_collection") != 2 || - strings.Count(compiled.Query, "FOR __loom_physical_unnest_index_0") != 2 { - t.Fatalf("fixture lineage must requery the exact population/route/UNNEST preimage twice:\n%s", compiled.Query) + if strings.Count(testAQLText(t, compiled.Command), "FOR population_member IN @@population_members_collection") != 2 || + strings.Count(testAQLText(t, compiled.Command), "FOR __loom_physical_unnest_index_0") != 2 { + t.Fatalf("fixture lineage must requery the exact population/route/UNNEST preimage twice:\n%s", testAQLText(t, compiled.Command)) } for _, want := range []string{"population_selection_id", "subject_Patient", "ordinal", "occurrenceKey:"} { - if !strings.Contains(fmt.Sprint(compiled.BindVars), want) && !strings.Contains(compiled.Query, want) { + if !strings.Contains(fmt.Sprint(testAQLBinds(t, compiled.Command)), want) && !strings.Contains(testAQLText(t, compiled.Command), want) { t.Errorf("fixture lineage omitted exact source fact %q from AQL and binds", want) } } @@ -622,7 +625,7 @@ func TestCompileRowLineageUsesBoundedCanonicalGroupTerminal(t *testing.T) { if !capability.Available { t.Fatalf("unsupported source operation: %s", capability.Operation) } - compiled, err := CompileRowLineageOutput(output, "opaque-group-row", 25, 10, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), output, "opaque-group-row", 25, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -631,16 +634,16 @@ func TestCompileRowLineageUsesBoundedCanonicalGroupTerminal(t *testing.T) { "TO_STRING([[\"construction\"", "SORT __loom_physical_construction_row_lineage_contributor", "LIMIT @row_lineage_offset, @row_lineage_fetch_limit", "hasMore:", "found:", } { - if !strings.Contains(compiled.Query, want) { - t.Errorf("row lineage query is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Errorf("row lineage query is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } - if strings.Contains(compiled.Query, "COLLECT ") && strings.Contains(compiled.Query, " INTO ") { - t.Fatalf("row lineage query materialized unbounded Group contributors:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), "COLLECT ") && strings.Contains(testAQLText(t, compiled.Command), " INTO ") { + t.Fatalf("row lineage query materialized unbounded Group contributors:\n%s", testAQLText(t, compiled.Command)) } - if compiled.BindVars[rowLineageRowIDBind] != "opaque-group-row" || compiled.BindVars[rowLineageOffsetBind] != 25 || - compiled.BindVars[rowLineageLimitBind] != 10 || compiled.BindVars[rowLineageFetchLimitBind] != 11 { - t.Fatalf("row lineage page bindings = %#v", compiled.BindVars) + if testAQLBinds(t, compiled.Command)[rowLineageRowIDBind] != "opaque-group-row" || testAQLBinds(t, compiled.Command)[rowLineageOffsetBind] != 25 || + testAQLBinds(t, compiled.Command)[rowLineageLimitBind] != 10 || testAQLBinds(t, compiled.Command)[rowLineageFetchLimitBind] != 11 { + t.Fatalf("row lineage page bindings = %#v", testAQLBinds(t, compiled.Command)) } } @@ -656,7 +659,7 @@ func TestCompileStandaloneExplicitGroupRowLineageUsesPinnedBoundedMembers(t *tes if err != nil { t.Fatal(err) } - compiled, err := CompileRowLineageOutput(output, string(rowID), 3, 7, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), output, string(rowID), 3, 7, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -671,19 +674,19 @@ func TestCompileStandaloneExplicitGroupRowLineageUsesPinnedBoundedMembers(t *tes "LIMIT @__loom_physical_row_lineage_offset, @__loom_physical_row_lineage_fetch_limit", "contributors: SLICE(page, 0, @__loom_physical_row_lineage_limit)", } { - if !strings.Contains(compiled.Query, want) { - t.Errorf("standalone explicit group row lineage query is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Errorf("standalone explicit group row lineage query is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } - if strings.Contains(compiled.Query, "COLLECT member") || strings.Contains(compiled.Query, "COLLECT selected") || strings.Contains(compiled.Query, "COLLECT resource") { - t.Fatalf("explicit group lineage materialized an unbounded contributor collection:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), "COLLECT member") || strings.Contains(testAQLText(t, compiled.Command), "COLLECT selected") || strings.Contains(testAQLText(t, compiled.Command), "COLLECT resource") { + t.Fatalf("explicit group lineage materialized an unbounded contributor collection:\n%s", testAQLText(t, compiled.Command)) } - if compiled.BindVars["__loom_physical_row_lineage_group_revision_id"] != "grouprev_row_lineage" || - compiled.BindVars["__loom_physical_row_lineage_group_id"] != "cohort-a" || - compiled.BindVars["__loom_physical_row_lineage_offset"] != 3 || - compiled.BindVars["__loom_physical_row_lineage_limit"] != 7 || - compiled.BindVars["__loom_physical_row_lineage_fetch_limit"] != 8 { - t.Fatalf("explicit group row lineage bindings = %#v", compiled.BindVars) + if testAQLBinds(t, compiled.Command)["__loom_physical_row_lineage_group_revision_id"] != "grouprev_row_lineage" || + testAQLBinds(t, compiled.Command)["__loom_physical_row_lineage_group_id"] != "cohort-a" || + testAQLBinds(t, compiled.Command)["__loom_physical_row_lineage_offset"] != 3 || + testAQLBinds(t, compiled.Command)["__loom_physical_row_lineage_limit"] != 7 || + testAQLBinds(t, compiled.Command)["__loom_physical_row_lineage_fetch_limit"] != 8 { + t.Fatalf("explicit group row lineage bindings = %#v", testAQLBinds(t, compiled.Command)) } for _, invalidID := range []string{ @@ -691,12 +694,12 @@ func TestCompileStandaloneExplicitGroupRowLineageUsesPinnedBoundedMembers(t *tes `{"group_id":"cohort-a","group_revision_id":"grouprev_row_lineage","extra":"forged"}`, `{"group_revision_id":"grouprev_row_lineage", "group_id":"cohort-a"}`, } { - invalid, compileErr := CompileRowLineageOutput(output, invalidID, 0, 7, ir.DefaultPhysicalOptimizationPolicy()) + invalid, compileErr := CompileRowLineageOutput(ArangoTarget(), output, invalidID, 0, 7, ir.DefaultPhysicalOptimizationPolicy()) if compileErr != nil { t.Fatalf("compile malformed/cross-revision identity %q: %v", invalidID, compileErr) } - if invalid.BindVars["__loom_physical_row_lineage_group_revision_id"] != "" || invalid.BindVars["__loom_physical_row_lineage_group_id"] != "" { - t.Errorf("invalid identity %q retained requested group binds %#v", invalidID, invalid.BindVars) + if testAQLBinds(t, invalid.Command)["__loom_physical_row_lineage_group_revision_id"] != "" || testAQLBinds(t, invalid.Command)["__loom_physical_row_lineage_group_id"] != "" { + t.Errorf("invalid identity %q retained requested group binds %#v", invalidID, testAQLBinds(t, invalid.Command)) } } } @@ -739,7 +742,7 @@ func TestCompileCohortGroupLineageAppliesFilterBeforePaging(t *testing.T) { if err != nil { t.Fatal(err) } - compiled, err := CompileRowLineageOutput(compiledOutput, string(rowID), 2, 4, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), compiledOutput, string(rowID), 2, 4, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -754,37 +757,37 @@ func TestCompileCohortGroupLineageAppliesFilterBeforePaging(t *testing.T) { "source.auth_resource_path IN @", "LIMIT @__loom_physical_row_lineage_offset, @__loom_physical_row_lineage_fetch_limit", } { - if !strings.Contains(compiled.Query, want) { - t.Errorf("cohort Group lineage query is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Errorf("cohort Group lineage query is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } - if compiled.BindVars["__loom_physical_row_lineage_group_revision_id"] != "grouprev_cohort_row_lineage" || - compiled.BindVars["__loom_physical_row_lineage_group_id"] != "cohort-a" || - compiled.BindVars["__loom_physical_row_lineage_offset"] != 2 || - compiled.BindVars["__loom_physical_row_lineage_fetch_limit"] != 5 { - t.Fatalf("cohort Group lineage bindings = %#v", compiled.BindVars) + if testAQLBinds(t, compiled.Command)["__loom_physical_row_lineage_group_revision_id"] != "grouprev_cohort_row_lineage" || + testAQLBinds(t, compiled.Command)["__loom_physical_row_lineage_group_id"] != "cohort-a" || + testAQLBinds(t, compiled.Command)["__loom_physical_row_lineage_offset"] != 2 || + testAQLBinds(t, compiled.Command)["__loom_physical_row_lineage_fetch_limit"] != 5 { + t.Fatalf("cohort Group lineage bindings = %#v", testAQLBinds(t, compiled.Command)) } for _, invalidID := range []string{ `{"group_revision_id":"grouprev_other","group_id":"cohort-a"}`, `{"group_revision_id":"grouprev_cohort_row_lineage","group_id":"forged"}`, } { - invalid, compileErr := CompileRowLineageOutput(compiledOutput, invalidID, 0, 4, ir.DefaultPhysicalOptimizationPolicy()) + invalid, compileErr := CompileRowLineageOutput(ArangoTarget(), compiledOutput, invalidID, 0, 4, ir.DefaultPhysicalOptimizationPolicy()) if compileErr != nil { t.Fatalf("compile invalid cohort row ID %q: %v", invalidID, compileErr) } - if invalid.BindVars["__loom_physical_row_lineage_group_revision_id"] != "" || - invalid.BindVars["__loom_physical_row_lineage_group_id"] != "" { - t.Errorf("invalid cohort row ID %q retained binds %#v", invalidID, invalid.BindVars) + if testAQLBinds(t, invalid.Command)["__loom_physical_row_lineage_group_revision_id"] != "" || + testAQLBinds(t, invalid.Command)["__loom_physical_row_lineage_group_id"] != "" { + t.Errorf("invalid cohort row ID %q retained binds %#v", invalidID, testAQLBinds(t, invalid.Command)) } } - forged, err := CompileRowLineageOutput(compiledOutput, + forged, err := CompileRowLineageOutput(ArangoTarget(), compiledOutput, `{"group_id":"forged","group_revision_id":"grouprev_cohort_row_lineage"}`, 0, 4, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile canonical but nonexistent cohort row ID: %v", err) } - if forged.BindVars["__loom_physical_row_lineage_group_id"] != "forged" || - !strings.Contains(forged.Query, "FILTER found AND") || !strings.Contains(forged.Query, "LET found =") { - t.Fatalf("nonexistent cohort row is not gated by final row membership: binds=%#v query=%s", forged.BindVars, forged.Query) + if testAQLBinds(t, forged.Command)["__loom_physical_row_lineage_group_id"] != "forged" || + !strings.Contains(testAQLText(t, forged.Command), "FILTER found AND") || !strings.Contains(testAQLText(t, forged.Command), "LET found =") { + t.Fatalf("nonexistent cohort row is not gated by final row membership: binds=%#v query=%s", testAQLBinds(t, forged.Command), testAQLText(t, forged.Command)) } memberFilterOutput := compiledOutput memberFilterOutput.Plan = ir.ClonePhysicalPlan(compiledOutput.Plan) @@ -802,27 +805,27 @@ func TestCompileGroupRowLineageAppliesEveryTrailingFilter(t *testing.T) { if !capability.Available { t.Fatalf("Group with trailing filters has no lineage capability: %#v", capability) } - compiled, err := CompileRowLineageOutput(output, "opaque-filtered-group-row", 25, 10, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), output, "opaque-filtered-group-row", 25, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if strings.Count(compiled.Query, ".rows > @") != 2 { - t.Fatalf("lineage query did not evaluate both typed filters against the candidate Group row:\n%s", compiled.Query) + if strings.Count(testAQLText(t, compiled.Command), ".rows > @") != 2 { + t.Fatalf("lineage query did not evaluate both typed filters against the candidate Group row:\n%s", testAQLText(t, compiled.Command)) } - if !strings.Contains(compiled.Query, "WITH COUNT INTO") || strings.Contains(compiled.Query, "row_lineage_filter_group_inputs") || strings.Contains(compiled.Query, "__loom_construction_group_rows") { - t.Fatalf("COUNT_ROWS lineage buffered Group contributors instead of streaming its candidate count:\n%s", compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), "WITH COUNT INTO") || strings.Contains(testAQLText(t, compiled.Command), "row_lineage_filter_group_inputs") || strings.Contains(testAQLText(t, compiled.Command), "__loom_construction_group_rows") { + t.Fatalf("COUNT_ROWS lineage buffered Group contributors instead of streaming its candidate count:\n%s", testAQLText(t, compiled.Command)) } - if !strings.Contains(compiled.Query, "found: ") || !strings.Contains(compiled.Query, " AND ") || !strings.Contains(compiled.Query, "contributors: SLICE(") { - t.Fatalf("lineage result does not bind contributor lookup to final filter membership:\n%s", compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), "found: ") || !strings.Contains(testAQLText(t, compiled.Command), " AND ") || !strings.Contains(testAQLText(t, compiled.Command), "contributors: SLICE(") { + t.Fatalf("lineage result does not bind contributor lookup to final filter membership:\n%s", testAQLText(t, compiled.Command)) } thresholds := 0 - for _, value := range compiled.BindVars { + for _, value := range testAQLBinds(t, compiled.Command) { if value == int64(104) || value == int64(103) { thresholds++ } } if thresholds != 2 { - t.Fatalf("filter chain lost typed thresholds; bind vars = %#v", compiled.BindVars) + t.Fatalf("filter chain lost typed thresholds; bind vars = %#v", testAQLBinds(t, compiled.Command)) } changedSchema := output @@ -840,18 +843,18 @@ func TestCompileNonCountGroupFilterLineageProjectsSourceRowsInScope(t *testing.T if capability := RowLineageCapabilityForOutput(output); !capability.Available { t.Fatalf("non-count Group with a trailing filter has no lineage capability: %#v", capability) } - compiled, err := CompileRowLineageOutput(output, "opaque-filtered-summary-row", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), output, "opaque-filtered-summary-row", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if !strings.Contains(compiled.Query, "construction_group_projected_input") { - t.Fatalf("non-count candidate did not project source rows in scope:\n%s", compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), "construction_group_projected_input") { + t.Fatalf("non-count candidate did not project source rows in scope:\n%s", testAQLText(t, compiled.Command)) } - if strings.Contains(compiled.Query, "row_lineage_filter_group_inputs") { - t.Fatalf("non-count candidate materialized an extra selected-group input array:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), "row_lineage_filter_group_inputs") { + t.Fatalf("non-count candidate materialized an extra selected-group input array:\n%s", testAQLText(t, compiled.Command)) } - if !strings.Contains(compiled.Query, ".amount_sum > @") { - t.Fatalf("typed filter was not evaluated against the candidate aggregate:\n%s", compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), ".amount_sum > @") { + t.Fatalf("typed filter was not evaluated against the candidate aggregate:\n%s", testAQLText(t, compiled.Command)) } } @@ -887,15 +890,15 @@ func TestCompileKeylessEmptyGroupHasExplicitEmptyContributorPage(t *testing.T) { if capability := RowLineageCapabilityForOutput(output); !capability.Available { t.Fatalf("keyless Group capability = %#v, want available", capability) } - compiled, err := CompileRowLineageOutput(output, "opaque-empty-group-row", 0, 0, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), output, "opaque-empty-group-row", 0, 0, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if !strings.Contains(compiled.Query, "COLLECT WITH COUNT INTO") || !strings.Contains(compiled.Query, "contributors: SLICE") || strings.Contains(compiled.Query, "COLLECT "+"__loom_physical_row_lineage_group_key") { - t.Fatalf("keyless Group lineage terminal does not preserve the empty Group row and bounded empty page:\n%s", compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), "COLLECT WITH COUNT INTO") || !strings.Contains(testAQLText(t, compiled.Command), "contributors: SLICE") || strings.Contains(testAQLText(t, compiled.Command), "COLLECT "+"__loom_physical_row_lineage_group_key") { + t.Fatalf("keyless Group lineage terminal does not preserve the empty Group row and bounded empty page:\n%s", testAQLText(t, compiled.Command)) } - if compiled.Limit != DefaultRowLineageLimit || compiled.BindVars[rowLineageFetchLimitBind] != DefaultRowLineageLimit+1 { - t.Fatalf("default page = limit %d, bindings %#v", compiled.Limit, compiled.BindVars) + if compiled.Limit != DefaultRowLineageLimit || testAQLBinds(t, compiled.Command)[rowLineageFetchLimitBind] != DefaultRowLineageLimit+1 { + t.Fatalf("default page = limit %d, bindings %#v", compiled.Limit, testAQLBinds(t, compiled.Command)) } } @@ -939,7 +942,7 @@ func TestCompileCodedGroupRowLineageResolvesExactTupleAndPagesRootSources(t *tes if capability := RowLineageCapabilityForOutput(output); !capability.Available { t.Fatalf("CODED_GROUP lineage capability = %#v", capability) } - compiled, err := CompileRowLineageOutput(output, "opaque-coded-group-row", 4, 10, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), output, "opaque-coded-group-row", 4, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -955,17 +958,17 @@ func TestCompileCodedGroupRowLineageResolvesExactTupleAndPagesRootSources(t *tes "contributors: SLICE(__loom_physical_construction_row_lineage_coded_group_page, 0, @row_lineage_limit)", "resourceId:", "occurrenceKey:", "hasMore:", } { - if !strings.Contains(compiled.Query, want) { - t.Errorf("CODED_GROUP lineage query is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Errorf("CODED_GROUP lineage query is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } - if strings.Contains(compiled.Query, "COLLECT "+"__loom_physical_row_lineage_coded_group_source_id = ") && - strings.Contains(compiled.Query, " INTO ") { - t.Fatalf("CODED_GROUP lineage materialized an unbounded contributor list:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), "COLLECT "+"__loom_physical_row_lineage_coded_group_source_id = ") && + strings.Contains(testAQLText(t, compiled.Command), " INTO ") { + t.Fatalf("CODED_GROUP lineage materialized an unbounded contributor list:\n%s", testAQLText(t, compiled.Command)) } - if compiled.BindVars[rowLineageRowIDBind] != "opaque-coded-group-row" || compiled.BindVars[rowLineageOffsetBind] != 4 || - compiled.BindVars[rowLineageLimitBind] != 10 || compiled.BindVars[rowLineageFetchLimitBind] != 11 { - t.Fatalf("CODED_GROUP lineage page bindings = %#v", compiled.BindVars) + if testAQLBinds(t, compiled.Command)[rowLineageRowIDBind] != "opaque-coded-group-row" || testAQLBinds(t, compiled.Command)[rowLineageOffsetBind] != 4 || + testAQLBinds(t, compiled.Command)[rowLineageLimitBind] != 10 || testAQLBinds(t, compiled.Command)[rowLineageFetchLimitBind] != 11 { + t.Fatalf("CODED_GROUP lineage page bindings = %#v", testAQLBinds(t, compiled.Command)) } } @@ -1014,7 +1017,7 @@ func TestCompileConstructionPivotRowLineageUsesTypedScopedPreimageAndPagination( if err != nil { t.Fatal(err) } - compiled, err := CompileRowLineageOutput(compiledOutput, string(rowID), 4, 7, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), compiledOutput, string(rowID), 4, 7, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -1030,15 +1033,15 @@ func TestCompileConstructionPivotRowLineageUsesTypedScopedPreimageAndPagination( "SORT __loom_physical_construction_row_lineage_pivot_source", "LIMIT @row_lineage_offset, @row_lineage_fetch_limit", "contributors: SLICE(", "hasMore:", } { - if !strings.Contains(compiled.Query, want) { - t.Errorf("construction Pivot lineage query is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Errorf("construction Pivot lineage query is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } - pageAt := strings.Index(compiled.Query, "LET __loom_physical_construction_row_lineage_pivot_page = (") + pageAt := strings.Index(testAQLText(t, compiled.Command), "LET __loom_physical_construction_row_lineage_pivot_page = (") if pageAt < 0 { - t.Fatalf("Pivot contributor page subquery is missing:\n%s", compiled.Query) + t.Fatalf("Pivot contributor page subquery is missing:\n%s", testAQLText(t, compiled.Command)) } - validationQuery := compiled.Query[:pageAt] + validationQuery := testAQLText(t, compiled.Command)[:pageAt] if !strings.Contains(validationQuery, "TABLE_PIVOT_UNLISTED_CATEGORY") || !strings.Contains(validationQuery, "TABLE_PIVOT_CELL_CARDINALITY") || !strings.Contains(validationQuery, "== @row_lineage_row_id ? 1 : 0") { @@ -1048,26 +1051,26 @@ func TestCompileConstructionPivotRowLineageUsesTypedScopedPreimageAndPagination( strings.Contains(validationQuery, "FILTER __loom_physical_construction_row_lineage_pivot_identity == @row_lineage_row_id") { t.Fatalf("Pivot policy validation was narrowed to the requested key before global checks:\n%s", validationQuery) } - if strings.Contains(compiled.Query, "COLLECT INTO") || strings.Contains(compiled.Query, "sourceRows") || strings.Contains(compiled.Query, "fullsourceRows") { - t.Fatalf("construction Pivot lineage retained an unbounded source/member array:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), "COLLECT INTO") || strings.Contains(testAQLText(t, compiled.Command), "sourceRows") || strings.Contains(testAQLText(t, compiled.Command), "fullsourceRows") { + t.Fatalf("construction Pivot lineage retained an unbounded source/member array:\n%s", testAQLText(t, compiled.Command)) } - if strings.Contains(compiled.Query, "TABLE_PIVOT_VALUE_TYPE_MISMATCH") { - t.Fatalf("ERROR duplicate policy gained a value type check that ordinary Pivot does not apply:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), "TABLE_PIVOT_VALUE_TYPE_MISMATCH") { + t.Fatalf("ERROR duplicate policy gained a value type check that ordinary Pivot does not apply:\n%s", testAQLText(t, compiled.Command)) } - keyValue, keyFound := compiled.BindVars["row_lineage_pivot_group_key_0"] + keyValue, keyFound := testAQLBinds(t, compiled.Command)["row_lineage_pivot_group_key_0"] if !keyFound || keyValue != nil { - t.Fatalf("decoded null Pivot group key = (%#v, %t), want a present null binding; binds=%#v", keyValue, keyFound, compiled.BindVars) + t.Fatalf("decoded null Pivot group key = (%#v, %t), want a present null binding; binds=%#v", keyValue, keyFound, testAQLBinds(t, compiled.Command)) } - if compiled.BindVars[rowLineageRowIDBind] != string(rowID) || compiled.BindVars[rowLineageOffsetBind] != 4 || - compiled.BindVars[rowLineageLimitBind] != 7 || compiled.BindVars[rowLineageFetchLimitBind] != 8 { - t.Fatalf("construction Pivot row lineage identity/page bindings = %#v", compiled.BindVars) + if testAQLBinds(t, compiled.Command)[rowLineageRowIDBind] != string(rowID) || testAQLBinds(t, compiled.Command)[rowLineageOffsetBind] != 4 || + testAQLBinds(t, compiled.Command)[rowLineageLimitBind] != 7 || testAQLBinds(t, compiled.Command)[rowLineageFetchLimitBind] != 8 { + t.Fatalf("construction Pivot row lineage identity/page bindings = %#v", testAQLBinds(t, compiled.Command)) } - if compiled.BindVars["population_selection_id"] != "selection-lineage-revision" || - compiled.BindVars["population_project"] != "lineage-selection-project" || - compiled.BindVars["project"] != "lineage-project" || - compiled.BindVars["dataset_generation"] != "lineage-generation" || - !reflect.DeepEqual(compiled.BindVars["auth_resource_paths"], []string{"/programs/lineage"}) { - t.Fatalf("Pivot lineage lost population or read-scope bindings: %#v", compiled.BindVars) + if testAQLBinds(t, compiled.Command)["population_selection_id"] != "selection-lineage-revision" || + testAQLBinds(t, compiled.Command)["population_project"] != "lineage-selection-project" || + testAQLBinds(t, compiled.Command)["project"] != "lineage-project" || + testAQLBinds(t, compiled.Command)["dataset_generation"] != "lineage-generation" || + !reflect.DeepEqual(testAQLBinds(t, compiled.Command)["auth_resource_paths"], []string{"/programs/lineage"}) { + t.Fatalf("Pivot lineage lost population or read-scope bindings: %#v", testAQLBinds(t, compiled.Command)) } forgedIDs := []string{ @@ -1076,30 +1079,28 @@ func TestCompileConstructionPivotRowLineageUsesTypedScopedPreimageAndPagination( `not-a-pivot-row-id`, } for _, forgedID := range forgedIDs { - forged, compileErr := CompileRowLineageOutput(compiledOutput, forgedID, 0, 7, ir.DefaultPhysicalOptimizationPolicy()) + forged, compileErr := CompileRowLineageOutput(ArangoTarget(), compiledOutput, forgedID, 0, 7, ir.DefaultPhysicalOptimizationPolicy()) if compileErr != nil { t.Fatalf("compile forged Pivot row ID %q: %v", forgedID, compileErr) } - if !strings.Contains(forged.Query, "FILTER __loom_physical_construction_row_lineage_pivot_identity == @row_lineage_row_id") || - forged.BindVars[rowLineageRowIDBind] != forgedID { - t.Errorf("forged Pivot row ID %q did not remain an exact owner-identity comparison: %#v", forgedID, forged.BindVars) + if !strings.Contains(testAQLText(t, forged.Command), "FILTER __loom_physical_construction_row_lineage_pivot_identity == @row_lineage_row_id") || + testAQLBinds(t, forged.Command)[rowLineageRowIDBind] != forgedID { + t.Errorf("forged Pivot row ID %q did not remain an exact owner-identity comparison: %#v", forgedID, testAQLBinds(t, forged.Command)) } - if key, ok := forged.BindVars["row_lineage_pivot_group_key_0"]; !ok || key != nil { - t.Errorf("forged Pivot row ID %q did not fail closed with a null key bind: %#v", forgedID, forged.BindVars) + if key, ok := testAQLBinds(t, forged.Command)["row_lineage_pivot_group_key_0"]; !ok || key != nil { + t.Errorf("forged Pivot row ID %q did not fail closed with a null key bind: %#v", forgedID, testAQLBinds(t, forged.Command)) } } - // Reducers consume only non-null values of the declared type, and an - // ERROR-on-missing Pivot validates that each requested cell has one. compiledOutput.Plan.StageSequence.Stages[0].GroupedPivot.DuplicatePolicy = "SUM" compiledOutput.Plan.StageSequence.Stages[0].GroupedPivot.MissingCellPolicy = "ERROR" - summed, err := CompileRowLineageOutput(compiledOutput, string(rowID), 0, 7, ir.DefaultPhysicalOptimizationPolicy()) + summed, err := CompileRowLineageOutput(ArangoTarget(), compiledOutput, string(rowID), 0, 7, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile SUM Pivot lineage: %v", err) } for _, want := range []string{"TABLE_PIVOT_VALUE_TYPE_MISMATCH", "TABLE_PIVOT_CELL_MISSING"} { - if !strings.Contains(summed.Query, want) { - t.Errorf("SUM Pivot lineage query is missing %q:\n%s", want, summed.Query) + if !strings.Contains(testAQLText(t, summed.Command), want) { + t.Errorf("SUM Pivot lineage query is missing %q:\n%s", want, testAQLText(t, summed.Command)) } } } @@ -1127,7 +1128,7 @@ func TestCompileConstructionCountRowsGroupPivotLineageStreamsScopedRootPreimage( if err != nil { t.Fatal(err) } - compiled, err := CompileRowLineageOutput(compiledOutput, string(rowID), 6, 9, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), compiledOutput, string(rowID), 6, 9, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -1144,38 +1145,38 @@ func TestCompileConstructionCountRowsGroupPivotLineageStreamsScopedRootPreimage( "SORT __loom_physical_construction_row_lineage_group_pivot_source", "LIMIT @row_lineage_offset, @row_lineage_fetch_limit", "contributors: SLICE(", "hasMore:", } { - if !strings.Contains(compiled.Query, want) { - t.Errorf("Group/Pivot lineage query is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Errorf("Group/Pivot lineage query is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } - if strings.Contains(compiled.Query, "COLLECT INTO") || strings.Contains(compiled.Query, "SORTED_UNIQUE") || strings.Contains(compiled.Query, "__loom_root_contributor_keys") { - t.Fatalf("Group/Pivot row lineage used a row array or RootContributor sidecar:\n%s", compiled.Query) + if strings.Contains(testAQLText(t, compiled.Command), "COLLECT INTO") || strings.Contains(testAQLText(t, compiled.Command), "SORTED_UNIQUE") || strings.Contains(testAQLText(t, compiled.Command), "__loom_root_contributor_keys") { + t.Fatalf("Group/Pivot row lineage used a row array or RootContributor sidecar:\n%s", testAQLText(t, compiled.Command)) } - pageAt := strings.Index(compiled.Query, "LET __loom_physical_construction_row_lineage_group_pivot_page = (") + pageAt := strings.Index(testAQLText(t, compiled.Command), "LET __loom_physical_construction_row_lineage_group_pivot_page = (") if pageAt < 0 { - t.Fatalf("Group/Pivot source preimage page is missing:\n%s", compiled.Query) + t.Fatalf("Group/Pivot source preimage page is missing:\n%s", testAQLText(t, compiled.Command)) } - validation := compiled.Query[:pageAt] + validation := testAQLText(t, compiled.Command)[:pageAt] if !strings.Contains(validation, "TABLE_PIVOT_UNLISTED_CATEGORY") || !strings.Contains(validation, "TABLE_PIVOT_CELL_CARDINALITY") || !strings.Contains(validation, "== @row_lineage_row_id ? 1 : 0") { t.Fatalf("Group/Pivot policies were not checked across the full scoped grouped input before paging:\n%s", validation) } for _, key := range []string{"row_lineage_pivot_group_key_0", "row_lineage_row_id", "row_lineage_offset", "row_lineage_limit", "row_lineage_fetch_limit"} { - if _, ok := compiled.BindVars[key]; !ok { - t.Errorf("Group/Pivot lineage omitted typed/page bind %q: %#v", key, compiled.BindVars) + if _, ok := testAQLBinds(t, compiled.Command)[key]; !ok { + t.Errorf("Group/Pivot lineage omitted typed/page bind %q: %#v", key, testAQLBinds(t, compiled.Command)) } } - if compiled.BindVars["row_lineage_pivot_group_key_0"] != "patient-7" || compiled.BindVars[rowLineageRowIDBind] != string(rowID) || - compiled.BindVars[rowLineageOffsetBind] != 6 || compiled.BindVars[rowLineageLimitBind] != 9 || compiled.BindVars[rowLineageFetchLimitBind] != 10 { - t.Errorf("Group/Pivot typed identity/page bindings = %#v", compiled.BindVars) + if testAQLBinds(t, compiled.Command)["row_lineage_pivot_group_key_0"] != "patient-7" || testAQLBinds(t, compiled.Command)[rowLineageRowIDBind] != string(rowID) || + testAQLBinds(t, compiled.Command)[rowLineageOffsetBind] != 6 || testAQLBinds(t, compiled.Command)[rowLineageLimitBind] != 9 || testAQLBinds(t, compiled.Command)[rowLineageFetchLimitBind] != 10 { + t.Errorf("Group/Pivot typed identity/page bindings = %#v", testAQLBinds(t, compiled.Command)) } - if compiled.BindVars["population_selection_id"] != "selection-group-pivot-revision" || - compiled.BindVars["population_project"] != "group-pivot-selection-project" || - compiled.BindVars["project"] != "group-pivot-project" || - compiled.BindVars["dataset_generation"] != "group-pivot-generation" || - !reflect.DeepEqual(compiled.BindVars["auth_resource_paths"], []string{"/programs/group-pivot"}) { - t.Fatalf("Group/Pivot lineage lost source scope bindings: %#v", compiled.BindVars) + if testAQLBinds(t, compiled.Command)["population_selection_id"] != "selection-group-pivot-revision" || + testAQLBinds(t, compiled.Command)["population_project"] != "group-pivot-selection-project" || + testAQLBinds(t, compiled.Command)["project"] != "group-pivot-project" || + testAQLBinds(t, compiled.Command)["dataset_generation"] != "group-pivot-generation" || + !reflect.DeepEqual(testAQLBinds(t, compiled.Command)["auth_resource_paths"], []string{"/programs/group-pivot"}) { + t.Fatalf("Group/Pivot lineage lost source scope bindings: %#v", testAQLBinds(t, compiled.Command)) } for policy, want := range map[ir.PhysicalStageGroupMissingKeyPolicy]string{ ir.PhysicalStageGroupMissingKeyExclude: "FILTER __loom_physical_construction_row_lineage_group_pivot_group_value_0 != null", @@ -1184,12 +1185,12 @@ func TestCompileConstructionCountRowsGroupPivotLineageStreamsScopedRootPreimage( policyOutput := compiledOutput policyOutput.Plan = ir.ClonePhysicalPlan(compiledOutput.Plan) policyOutput.Plan.StageSequence.Stages[0].Group.MissingKeyPolicy = policy - policyCompiled, policyErr := CompileRowLineageOutput(policyOutput, string(rowID), 0, 9, ir.DefaultPhysicalOptimizationPolicy()) + policyCompiled, policyErr := CompileRowLineageOutput(ArangoTarget(), policyOutput, string(rowID), 0, 9, ir.DefaultPhysicalOptimizationPolicy()) if policyErr != nil { t.Fatalf("compile Group/Pivot lineage with Group missing policy %q: %v", policy, policyErr) } - if !strings.Contains(policyCompiled.Query, want) { - t.Errorf("Group/Pivot lineage did not preserve Group missing policy %q (%q):\n%s", policy, want, policyCompiled.Query) + if !strings.Contains(testAQLText(t, policyCompiled.Command), want) { + t.Errorf("Group/Pivot lineage did not preserve Group missing policy %q (%q):\n%s", policy, want, testAQLText(t, policyCompiled.Command)) } } @@ -1364,7 +1365,7 @@ func TestCompileRelatedExpandRowLineageUsesAnchoredCanonicalAuthorizedRoute(t *t stage := output.Plan.StageSequence.Stages[0] constructionID := output.Plan.BindVars[stage.RelatedExpand.ConstructionIDBindKey].(string) rowID := relatedExpandRowID(t, "patient-storage-key", constructionID, "Observation/observation-storage-key") - compiled, err := CompileRowLineageOutput(output, rowID, 0, 1, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), output, rowID, 0, 1, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -1380,17 +1381,17 @@ func TestCompileRelatedExpandRowLineageUsesAnchoredCanonicalAuthorizedRoute(t *t "LIMIT 1", "contributors: SLICE", } { - if !strings.Contains(compiled.Query, want) { - t.Errorf("related row lineage query is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Errorf("related row lineage query is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } - if compiled.BindVars[rowLineageRootKeyBind] != "patient-storage-key" || - compiled.BindVars[rowLineageStage0TerminalIDBind] != "Observation/observation-storage-key" || - compiled.BindVars[rowLineageStage0RowIDBind] != rowID || compiled.BindVars[rowLineageRowIDBind] != rowID { - t.Fatalf("row identity bindings = %#v", compiled.BindVars) + if testAQLBinds(t, compiled.Command)[rowLineageRootKeyBind] != "patient-storage-key" || + testAQLBinds(t, compiled.Command)[rowLineageStage0TerminalIDBind] != "Observation/observation-storage-key" || + testAQLBinds(t, compiled.Command)[rowLineageStage0RowIDBind] != rowID || testAQLBinds(t, compiled.Command)[rowLineageRowIDBind] != rowID { + t.Fatalf("row identity bindings = %#v", testAQLBinds(t, compiled.Command)) } - if !strings.Contains(compiled.Query, "auth_resource_path IN @auth_resource_paths") { - t.Fatalf("related row lineage omitted the restricted route scope:\n%s", compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), "auth_resource_path IN @auth_resource_paths") { + t.Fatalf("related row lineage omitted the restricted route scope:\n%s", testAQLText(t, compiled.Command)) } } @@ -1400,23 +1401,23 @@ func TestCompileRelatedExpandPreserveParentLineageReturnsRootOnlyForEmptyIdentit stage := output.Plan.StageSequence.Stages[0] constructionID := output.Plan.BindVars[stage.RelatedExpand.ConstructionIDBindKey].(string) rowID := relatedExpandEmptyRowID(t, "patient-storage-key", constructionID) - compiled, err := CompileRowLineageOutput(output, rowID, 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), output, rowID, 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if compiled.BindVars[rowLineageRootKeyBind] != "patient-storage-key" || - compiled.BindVars[rowLineageStage0RowIDBind] != rowID { - t.Fatalf("empty related row identity bindings = %#v", compiled.BindVars) + if testAQLBinds(t, compiled.Command)[rowLineageRootKeyBind] != "patient-storage-key" || + testAQLBinds(t, compiled.Command)[rowLineageStage0RowIDBind] != rowID { + t.Fatalf("empty related row identity bindings = %#v", testAQLBinds(t, compiled.Command)) } - if _, exists := compiled.BindVars[rowLineageStage0TerminalIDBind]; exists { - t.Fatalf("empty row retained an unused terminal bind: %#v", compiled.BindVars) + if _, exists := testAQLBinds(t, compiled.Command)[rowLineageStage0TerminalIDBind]; exists { + t.Fatalf("empty row retained an unused terminal bind: %#v", testAQLBinds(t, compiled.Command)) } for _, want := range []string{"FILTER __loom_physical_construction_row_lineage", "[\"empty\"]", "contributors: SLICE", "root:"} { - if !strings.Contains(compiled.Query, want) { - t.Errorf("PRESERVE_PARENT row lineage query is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Errorf("PRESERVE_PARENT row lineage query is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } - for key := range compiled.BindVars { + for key := range testAQLBinds(t, compiled.Command) { if strings.Contains(key, "row_lineage_related_resource_type") { t.Fatalf("PRESERVE_PARENT empty row retained a related contributor bind %q", key) } @@ -1429,30 +1430,30 @@ func TestCompileRelatedExpandRowLineageTreatsForgedIdentityAsUnmatchable(t *test stage := output.Plan.StageSequence.Stages[0] constructionID := output.Plan.BindVars[stage.RelatedExpand.ConstructionIDBindKey].(string) forged := relatedExpandRowID(t, "patient-storage-key", constructionID, "Observation/forged-key") - compiled, err := CompileRowLineageOutput(output, forged, 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), output, forged, 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if compiled.BindVars[rowLineageRootKeyBind] != "patient-storage-key" || - compiled.BindVars[rowLineageStage0TerminalIDBind] != "Observation/forged-key" || - compiled.BindVars[rowLineageStage0RowIDBind] != forged { - t.Fatalf("forged row ID was not bound as an exact candidate: %#v", compiled.BindVars) + if testAQLBinds(t, compiled.Command)[rowLineageRootKeyBind] != "patient-storage-key" || + testAQLBinds(t, compiled.Command)[rowLineageStage0TerminalIDBind] != "Observation/forged-key" || + testAQLBinds(t, compiled.Command)[rowLineageStage0RowIDBind] != forged { + t.Fatalf("forged row ID was not bound as an exact candidate: %#v", testAQLBinds(t, compiled.Command)) } - if !strings.Contains(compiled.Query, "_id == @row_lineage_stage_0_terminal_id") || !strings.Contains(compiled.Query, "== @row_lineage_stage_0_row_id") { - t.Fatalf("forged row ID is not checked against the authorized terminal and canonical identity:\n%s", compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), "_id == @row_lineage_stage_0_terminal_id") || !strings.Contains(testAQLText(t, compiled.Command), "== @row_lineage_stage_0_row_id") { + t.Fatalf("forged row ID is not checked against the authorized terminal and canonical identity:\n%s", testAQLText(t, compiled.Command)) } } func TestCompileRelatedExpandRowLineageBindsMalformedIdentityToNullRoot(t *testing.T) { bindings := recipe.RuntimeBindings{Project: "row-lineage-project", DatasetGeneration: "row-lineage-generation"} output := lowerConstructionOutput(t, directRelatedExpandOutput(recipe.ExpansionExclude), bindings) - compiled, err := CompileRowLineageOutput(output, "not-json", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), output, "not-json", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if compiled.BindVars[rowLineageRootKeyBind] != nil || compiled.BindVars[rowLineageStage0TerminalIDBind] != "" || - compiled.BindVars[rowLineageStage0RowIDBind] != "not-json" { - t.Fatalf("malformed identity was not made unmatchable at the root: %#v", compiled.BindVars) + if testAQLBinds(t, compiled.Command)[rowLineageRootKeyBind] != nil || testAQLBinds(t, compiled.Command)[rowLineageStage0TerminalIDBind] != "" || + testAQLBinds(t, compiled.Command)[rowLineageStage0RowIDBind] != "not-json" { + t.Fatalf("malformed identity was not made unmatchable at the root: %#v", testAQLBinds(t, compiled.Command)) } } @@ -1520,17 +1521,17 @@ func TestCompileComposedRelatedRowLineageBindsEachOwnerAndPagesWitnesses(t *test secondConstructionID := output.Plan.BindVars[stages[1].RelatedExpand.ConstructionIDBindKey] firstRowID := relatedExpandRowID(t, "patient-storage-key", firstConstructionID.(string), "Observation/observation-storage-key") finalRowID := relatedExpandRowID(t, firstRowID, secondConstructionID.(string), "Patient/related-patient-storage-key") - compiled, err := CompileRowLineageOutput(output, finalRowID, 2, 7, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := CompileRowLineageOutput(ArangoTarget(), output, finalRowID, 2, 7, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if compiled.BindVars[rowLineageRootKeyBind] != "patient-storage-key" || - compiled.BindVars[rowLineageStage0RowIDBind] != firstRowID || - compiled.BindVars[rowLineageStage0TerminalIDBind] != "Observation/observation-storage-key" || - compiled.BindVars["row_lineage_stage_1_row_id"] != finalRowID || - compiled.BindVars["row_lineage_stage_1_terminal_id"] != "Patient/related-patient-storage-key" || - compiled.BindVars[rowLineageRowIDBind] != finalRowID { - t.Fatalf("composed owner bindings do not retain exact stage identities: %#v", compiled.BindVars) + if testAQLBinds(t, compiled.Command)[rowLineageRootKeyBind] != "patient-storage-key" || + testAQLBinds(t, compiled.Command)[rowLineageStage0RowIDBind] != firstRowID || + testAQLBinds(t, compiled.Command)[rowLineageStage0TerminalIDBind] != "Observation/observation-storage-key" || + testAQLBinds(t, compiled.Command)["row_lineage_stage_1_row_id"] != finalRowID || + testAQLBinds(t, compiled.Command)["row_lineage_stage_1_terminal_id"] != "Patient/related-patient-storage-key" || + testAQLBinds(t, compiled.Command)[rowLineageRowIDBind] != finalRowID { + t.Fatalf("composed owner bindings do not retain exact stage identities: %#v", testAQLBinds(t, compiled.Command)) } for _, want := range []string{ "root._key == @row_lineage_root_key", @@ -1539,8 +1540,8 @@ func TestCompileComposedRelatedRowLineageBindsEachOwnerAndPagesWitnesses(t *test "LIMIT @row_lineage_offset, @row_lineage_fetch_limit", "contributors: SLICE", "hasMore:", } { - if !strings.Contains(compiled.Query, want) { - t.Errorf("composed row lineage query is missing %q:\n%s", want, compiled.Query) + if !strings.Contains(testAQLText(t, compiled.Command), want) { + t.Errorf("composed row lineage query is missing %q:\n%s", want, testAQLText(t, compiled.Command)) } } } @@ -1672,3 +1673,221 @@ func relatedExpandEmptyRowID(t *testing.T, parentKey, constructionID string) str } return string(encoded) } + +func pg23LineageTarget(t *testing.T) Target { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("loom") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + target, err := PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("PostgreSQLTarget: %v", err) + } + return target +} + +func TestCompileRowLineagePostgreSQLBoundedGroupPage(t *testing.T) { + output := lowerConstructionOutput(t, constructionMissingKeyPolicyOutput(recipe.ConstructionGroupMissingKeyGroup), recipe.RuntimeBindings{ + Project: "row-lineage-project", DatasetGeneration: "row-lineage-generation", + }) + compiled, err := CompileRowLineageOutput(pg23LineageTarget(t), output, "opaque-group-row", 25, 10, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("postgresql lineage compile: %v", err) + } + if compiled.Command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("selected PostgreSQL target produced %q", compiled.Command.Backend()) + } + if _, _, err := compiled.Command.AQLParameters(); err == nil { + t.Fatal("the AQL adapter must refuse the PostgreSQL lineage command") + } + sqlText, args, err := compiled.Command.PostgreSQLParameters() + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "WITH __loom_l_src AS (", "__loom_l_rows AS (", "__loom_l_grp AS (", + "__loom_l_sel AS (", "__loom_l_page AS (", + `IS NOT DISTINCT FROM`, `COLLATE "C" ASC`, + `"found"`, `"contributors"`, `"hasMore"`, + "'resourceType'", "'resourceId'", "'occurrenceKey'", + `["operation","GROUP"]`, + } { + if !strings.Contains(sqlText, want) { + t.Errorf("contributor-page SQL is missing %q:\n%s", want, sqlText) + } + } + for _, forbidden := range []string{"COLLECT", "@row_lineage", "OFFSET @", " FOR "} { + if strings.Contains(sqlText, forbidden) { + t.Errorf("contributor-page SQL carries %q (AQL leaked):\n%s", forbidden, sqlText) + } + } + if compiled.FoundColumn != "found" || compiled.ContributorsColumn != "contributors" || + compiled.HasMoreColumn != "hasMore" || compiled.Offset != 25 || compiled.Limit != 10 { + t.Fatalf("compiled page contract = %#v", compiled) + } + var sawRowID bool + var bigints []int64 + for _, parameter := range args { + switch parameter.Type { + case command.PostgreSQLParameterText: + if parameter.Value == "opaque-group-row" { + sawRowID = true + } + case command.PostgreSQLParameterBigint: + bigints = append(bigints, parameter.Value.(int64)) + } + } + if !sawRowID { + t.Fatalf("row id parameter missing from %v", args) + } + if len(bigints) != 3 || bigints[0] != 25 || bigints[1] != 11 || bigints[2] != 10 { + t.Fatalf("page coordinates = %v, want [25 11 10]", bigints) + } + aqlCompiled, err := CompileRowLineageOutput(ArangoTarget(), output, "opaque-group-row", 25, 10, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("aql twin: %v", err) + } + if aqlCompiled.Command.Backend() != command.CompiledBackendAQL { + t.Fatalf("aql twin produced %q", aqlCompiled.Command.Backend()) + } +} + +func TestCompileRowLineagePostgreSQLPageSeal(t *testing.T) { + output := lowerConstructionOutput(t, constructionMissingKeyPolicyOutput(recipe.ConstructionGroupMissingKeyGroup), recipe.RuntimeBindings{ + Project: "row-lineage-project", DatasetGeneration: "row-lineage-generation", + }) + target := pg23LineageTarget(t) + if _, err := CompileRowLineageOutput(Target{}, output, "row", 0, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil || + !errorsIsTargetNotSelected(err) { + t.Fatalf("unselected target: %v", err) + } + for name, rowID := range map[string]string{"empty": "", "blank": " "} { + if _, err := CompileRowLineageOutput(target, output, rowID, 0, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil || + !strings.Contains(err.Error(), "row lineage row ID is required") { + t.Fatalf("%s row id: %v", name, err) + } + } + if _, err := CompileRowLineageOutput(target, output, "row", -1, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil || + !strings.Contains(err.Error(), "offset cannot be negative") { + t.Fatalf("negative offset: %v", err) + } + if _, err := CompileRowLineageOutput(target, output, "row", 0, MaxRowLineageContributors+1, ir.DefaultPhysicalOptimizationPolicy()); err == nil || + !strings.Contains(err.Error(), "between 1 and") { + t.Fatalf("over-limit: %v", err) + } + compiled, err := CompileRowLineageOutput(target, output, "row", 0, 0, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("default limit: %v", err) + } + if compiled.Limit != DefaultRowLineageLimit { + t.Fatalf("default limit = %d", compiled.Limit) + } + _, args, err := compiled.Command.PostgreSQLParameters() + if err != nil { + t.Fatal(err) + } + var sawFetch bool + for _, parameter := range args { + if parameter.Type == command.PostgreSQLParameterBigint && parameter.Value == int64(DefaultRowLineageLimit+1) { + sawFetch = true + } + } + if !sawFetch { + t.Fatalf("fetch-limit lookahead parameter %d missing", DefaultRowLineageLimit+1) + } +} + +func errorsIsTargetNotSelected(err error) bool { + return errors.Is(err, ErrTargetNotSelected) +} + +func TestCompileRowLineagePostgreSQLRefusals(t *testing.T) { + pipelineBindings := recipe.RuntimeBindings{ + Project: "row-lineage-project", DatasetGeneration: "row-lineage-generation", + } + cohortBindings := recipe.RuntimeBindings{ + Project: "row-lineage-project", SelectionProject: "row-lineage-project", DatasetGeneration: "row-lineage-generation", + AuthScopeMode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{"/programs/p1"}, + } + target := pg23LineageTarget(t) + cases := []struct { + name string + output func(t *testing.T) lower.CompiledRecipeOutput + rowID string + want string + }{ + {"zero-output-names-its-surface", func(t *testing.T) lower.CompiledRecipeOutput { + return lower.CompiledRecipeOutput{} + }, "row", "row lineage compilation is unsupported on the PostgreSQL compile target"}, + {"explicit-group-rows", func(t *testing.T) lower.CompiledRecipeOutput { + return lowerConstructionOutput(t, recipe.Output{ + Name: "NamedCohort", RootResourceType: "Observation", RowGrain: "groups", + GroupRows: &recipe.GroupRows{RevisionID: "grouprev_row_lineage", UnassignedMemberPolicy: "GROUP_AS_UNASSIGNED"}, + }, cohortBindings) + }, `{"group_revision_id":"grouprev_row_lineage","group_id":"cohort-a"}`, "explicit group row lineage is unsupported on the PostgreSQL compile target"}, + {"composed-cohort-membership", func(t *testing.T) lower.CompiledRecipeOutput { + filterValue := "Cohort" + return lowerConstructionOutput(t, recipe.Output{ + Name: "NamedCohort", RootResourceType: "Observation", RowGrain: "groups", + Fields: []recipe.Field{{Name: "specimen_id", ColumnID: "specimen_id", Expr: recipe.Expression{Select: "root.id"}}}, + Construction: &recipe.Construction{Version: 1, SourceColumns: []recipe.StageColumn{{ID: "specimen_id", Name: "specimen_id", Type: "string"}}, + Steps: []recipe.ConstructionStep{{ + ID: "filter_group_rows", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionStepOutputInput, StepID: recipe.ConstructionCohortGroupStageID}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionFilterOp, Filter: &recipe.ConstructionFilter{ + ColumnID: "group_label", Operator: recipe.FilterEquals, + Values: []recipe.FilterValue{{Kind: recipe.FilterString, String: &filterValue}}, + }}, + Outputs: []recipe.StageColumn{ + {ID: "group_id", Name: "group_id", Type: "string"}, + {ID: "group_label", Name: "group_label", Type: "string"}, + {ID: "group_ordinal", Name: "group_ordinal", Type: "integer"}, + {ID: "members", Name: "members", Type: "array"}, + {ID: "specimen_id", Name: "specimen_id", Type: "array"}, + }, + }}}, + GroupRows: &recipe.GroupRows{ + RevisionID: "grouprev_cohort_row_lineage", UnassignedMemberPolicy: "EXCLUDE", + RowValues: []recipe.GroupRowValuePolicy{{ColumnID: "specimen_id", Policy: recipe.ConstructionRowValueAll}}, + }, + }, cohortBindings) + }, `{"group_revision_id":"grouprev_cohort_row_lineage","group_id":"cohort-a"}`, "composed cohort row lineage is unsupported on the PostgreSQL compile target"}, + {"semantic-family", func(t *testing.T) lower.CompiledRecipeOutput { + return lowerConstructionOutput(t, semanticGroupOutputForLineageTest(), pipelineBindings) + }, "semantic-row-id", "semantic-family contributor page"}, + {"coded-group-fixture-uses-the-coded-group-door-voice", func(t *testing.T) lower.CompiledRecipeOutput { + return expandedCodingGroupFixtureLineageOutput(t) + }, "fixture-expanded-coded-group-row", "row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route (compiler/row_lineage.go:655, cell_trace.go:54, population_mapping.go:38)"}, + {"related-coding-traced", func(t *testing.T) lower.CompiledRecipeOutput { + return relatedCodingGroupLineageOutput(t) + }, "opaque-row", "row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route"}, + {"pivot-preimage-traced", func(t *testing.T) lower.CompiledRecipeOutput { + return lowerConstructionOutput(t, constructionPivotLineageOutput(), pipelineBindings) + }, "not-json", "row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route"}, + {"group-filter-suffix", func(t *testing.T) lower.CompiledRecipeOutput { + return lowerConstructionOutput(t, constructionGroupFilterLineageOutput(104, 103), pipelineBindings) + }, "opaque-filtered-group-row", "row-lineage, cell-trace, and population-mapping returns have no PostgreSQL compiled route"}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + output := testCase.output(t) + _, err := CompileRowLineageOutput(target, output, testCase.rowID, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + if err == nil { + t.Fatalf("must refuse with the registered voice %q", testCase.want) + } + if !strings.Contains(err.Error(), testCase.want) { + t.Fatalf("refusal %q lacks %q", err, testCase.want) + } + if !errorsIsUnsupportedPlan(err) { + t.Fatalf("refusal must carry the typed unsupported-plan error: %v", err) + } + if strings.Contains(strings.ToLower(err.Error()), "fallback") { + t.Fatalf("refusal must not offer a fallback: %v", err) + } + }) + } +} + +func errorsIsUnsupportedPlan(err error) bool { + return errors.Is(err, renderpostgres.ErrUnsupportedPostgresPlan) +} diff --git a/internal/dataframe/compiler/selected_source_contract_test.go b/internal/dataframe/compiler/selected_source_contract_test.go new file mode 100644 index 000000000..84f24e01f --- /dev/null +++ b/internal/dataframe/compiler/selected_source_contract_test.go @@ -0,0 +1,222 @@ +package compiler_test + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + renderpostgres "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" + "github.com/calypr/loom/internal/dataframe/recipe" + semantic "github.com/calypr/loom/internal/dataframe/semantic" +) + +func archBackend11PostgreSQLTarget(t *testing.T) compiler.Target { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("loom") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("PostgreSQLTarget: %v", err) + } + return target +} + +func TestSelectedSourceCompilerTargetSeal(t *testing.T) { + zeroOutput := lower.CompiledRecipeOutput{} + zeroBindings := recipe.RuntimeBindings{} + zeroResolved := semantic.ResolvedRecipePlan{} + policy := ir.DefaultPhysicalOptimizationPolicy() + + t.Run("ZeroTargetIsRefusedByEverySelectedEntry", func(t *testing.T) { + entries := []struct { + name string + call func(compiler.Target) error + }{ + {"CompileResolvedRecipePlan", func(target compiler.Target) error { + _, err := compiler.CompileResolvedRecipePlan(target, zeroResolved, 0, policy) + return err + }}, + {"CompileRecipeOutput", func(target compiler.Target) error { + _, err := compiler.CompileRecipeOutput(target, zeroOutput, zeroBindings, 0, policy) + return err + }}, + {"CompileRecipeOutputPage", func(target compiler.Target) error { + _, err := compiler.CompileRecipeOutputPage(target, zeroOutput, zeroBindings, 50, policy) + return err + }}, + {"CompileCategoryScanOutput", func(target compiler.Target) error { + _, err := compiler.CompileCategoryScanOutput(target, zeroOutput, "id", 10, policy) + return err + }}, + {"CompileCategoryScanStage", func(target compiler.Target) error { + _, err := compiler.CompileCategoryScanStage(target, zeroOutput, "stage", "c", "v", 10, policy) + return err + }}, + {"CompileCellTraceOutput", func(target compiler.Target) error { + _, err := compiler.CompileCellTraceOutput(target, zeroOutput, "name", 0, 10, policy) + return err + }}, + {"CompileRowLineageOutput", func(target compiler.Target) error { + _, err := compiler.CompileRowLineageOutput(target, zeroOutput, "row", 0, 10, policy) + return err + }}, + {"CompileResolvedGraphQuery", func(target compiler.Target) error { + _, err := compiler.CompileResolvedGraphQuery(target, zeroResolved, 10, policy) + return err + }}, + {"CompilePopulationMappingOutput", func(target compiler.Target) error { + _, err := compiler.CompilePopulationMappingOutput(target, zeroOutput, zeroBindings, policy) + return err + }}, + {"CompileTableShapeExclusions", func(target compiler.Target) error { + _, err := compiler.CompileTableShapeExclusions(target, zeroOutput, 0, 10, policy) + return err + }}, + } + if len(entries) != 10 { + t.Fatalf("expected the ten selected compile entries, got %d", len(entries)) + } + for _, entry := range entries { + err := entry.call(compiler.Target{}) + if !errors.Is(err, compiler.ErrTargetNotSelected) { + t.Fatalf("%s must refuse an unselected target with the named sentinel: %v", entry.name, err) + } + } + }) + + t.Run("EachAQLOnlySurfaceRefusesThePostgreSQLTargetByName", func(t *testing.T) { + target := archBackend11PostgreSQLTarget(t) + surfaces := []struct { + surface string + call func() error + }{ + {"graph query compilation", func() error { + _, err := compiler.CompileResolvedGraphQuery(target, zeroResolved, 10, policy) + return err + }}, + {"root-key output paging", func() error { + _, err := compiler.CompileRecipeOutputPage(target, zeroOutput, zeroBindings, 50, policy) + return err + }}, + } + for _, surface := range surfaces { + err := surface.call() + if err == nil || !strings.Contains(err.Error(), surface.surface+" is unsupported on the PostgreSQL compile target") { + t.Fatalf("%s must name its PostgreSQL refusal: %v", surface.surface, err) + } + if !errors.Is(err, renderpostgres.ErrUnsupportedPostgresPlan) { + t.Fatalf("%s refusal must carry the typed unsupported-plan error: %v", surface.surface, err) + } + if strings.Contains(strings.ToLower(err.Error()), "aql") { + t.Fatalf("%s refusal must not offer an AQL fallback: %v", surface.surface, err) + } + } + }) + + t.Run("CommandsAreSealedAcrossTheSelectedTarget", func(t *testing.T) { + output, bindings := archBackend11FingerprintFixture() + pgQuery, err := compiler.CompileRecipeOutput(archBackend11PostgreSQLTarget(t), output, bindings, 0, policy) + if err != nil { + t.Fatalf("postgresql compile: %v", err) + } + if pgQuery.Command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("selected PostgreSQL target produced %q", pgQuery.Command.Backend()) + } + if _, _, err := pgQuery.Command.AQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("the AQL adapter must refuse a PostgreSQL command: %v", err) + } + aqlQuery, err := compiler.CompileRecipeOutput(compiler.ArangoTarget(), output, bindings, 0, policy) + if err != nil { + t.Fatalf("aql compile: %v", err) + } + if aqlQuery.Command.Backend() != command.CompiledBackendAQL { + t.Fatalf("AQL control produced %q", aqlQuery.Command.Backend()) + } + if _, _, err := aqlQuery.Command.PostgreSQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("the PostgreSQL adapter must refuse an AQL command: %v", err) + } + query, binds, err := aqlQuery.Command.AQLParameters() + if err != nil { + t.Fatalf("AQL control parameters: %v", err) + } + if !strings.Contains(query, "FOR root IN") || !strings.Contains(query, "@project") { + t.Fatalf("AQL control drifted from canonical AQL rendering: %q", query) + } + if _, ok := binds["project"]; !ok { + t.Fatalf("AQL control lost named binds: %v", binds) + } + }) +} + +func TestCompiledCommandLeafMutationProbes(t *testing.T) { + t.Run("RenderedTextIsRequired", func(t *testing.T) { + if _, err := command.NewAQLCommand(" ", map[string]any{"p": "1"}); err == nil { + t.Fatal("blank AQL query must be refused") + } + if _, err := command.NewPostgreSQLCommand("", nil); err == nil { + t.Fatal("empty statement SQL must be refused") + } + }) + + t.Run("ArgumentsMustBeRegisteredAndTyped", func(t *testing.T) { + if _, err := command.NewPostgreSQLCommand("SELECT 1", []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterType("oid"), Value: "x"}, + }); err == nil { + t.Fatal("unregistered parameter type must be refused") + } + if _, err := command.NewPostgreSQLCommand("SELECT $1", []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: nil}, + }); err == nil { + t.Fatal("nil value outside the SQL-null type must be refused") + } + }) + + t.Run("ZeroValueDispatchesNowhere", func(t *testing.T) { + zero := command.CompiledCommand{} + if _, _, err := zero.AQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("zero tag must refuse the AQL adapter: %v", err) + } + if _, _, err := zero.PostgreSQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("zero tag must refuse the PostgreSQL adapter: %v", err) + } + aqlRan, pgRan := false, false + if err := zero.Dispatch( + func(command.AQLCommand) error { aqlRan = true; return nil }, + func(command.PostgreSQLCommand) error { pgRan = true; return nil }, + ); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("zero tag must refuse dispatch: %v", err) + } + if aqlRan || pgRan { + t.Fatal("zero tag must not invoke any adapter") + } + }) + + t.Run("NamedBindOverridesAreAQLOnly", func(t *testing.T) { + pg, err := command.NewPostgreSQLCommand("SELECT $1", []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "P1"}, + }) + if err != nil { + t.Fatalf("NewPostgreSQLCommand: %v", err) + } + if _, err := pg.WithAQLBinds(map[string]any{"project": "P2"}); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("named bind overrides must not reach a PostgreSQL command: %v", err) + } + aql, err := command.NewAQLCommand("FOR d IN @@col RETURN d", map[string]any{"col": "Patient"}) + if err != nil { + t.Fatalf("NewAQLCommand: %v", err) + } + if _, err := aql.WithAQLBinds(map[string]any{"col": json.RawMessage(`1`)}); err == nil { + t.Fatal("bind override must keep the rendered Go type") + } + if _, err := aql.WithAQLBinds(map[string]any{"missing": "x"}); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("unknown bind keys must be refused: %v", err) + } + }) +} diff --git a/internal/dataframe/compiler/semantic_group_arango_test.go b/internal/dataframe/compiler/semantic_group_arango_test.go index 40971ffbe..d7f60c8b4 100644 --- a/internal/dataframe/compiler/semantic_group_arango_test.go +++ b/internal/dataframe/compiler/semantic_group_arango_test.go @@ -37,12 +37,9 @@ func TestSemanticCategoryGroupPreservesExactMembershipAndRowMultiplicityAgainstA "id": "same-source-row", "status": "active", "component": []any{ semanticGroupComponent(targetSystem, targetVersion, "shared-code", "Z Shared"), - // Same exact tuple, a second representation, and same-row duplicate. semanticGroupComponent(targetSystem, targetVersion, "shared-code", "A Shared"), semanticGroupComponent(targetSystem, targetVersion, "shared-code", "A Shared"), - // A different code with the same label must remain a separate group. semanticGroupComponent(targetSystem, targetVersion, "different-code", "A Shared"), - // With no nonblank display, the public label falls back to code. semanticGroupComponent(targetSystem, targetVersion, "fallback-code", " "), }, }, @@ -73,7 +70,7 @@ func TestSemanticCategoryGroupPreservesExactMembershipAndRowMultiplicityAgainstA compile := func(policy recipe.ConstructionGroupMissingKeyPolicy) CompiledQuery { t.Helper() if policy == recipe.ConstructionGroupMissingKeyGroup { - query, err := CompileRecipeOutputWithPolicy(groupedCompiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), groupedCompiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile semantic Group with %s missing-key policy: %v", policy, err) } @@ -89,7 +86,7 @@ func TestSemanticCategoryGroupPreservesExactMembershipAndRowMultiplicityAgainstA if len(grouped) != 5 { t.Fatalf("semantic Group rows = %#v, want four keyed groups plus one missing-key group", grouped) } - unlimitedQuery, err := CompileRecipeOutputWithPolicy(groupedCompiled, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + unlimitedQuery, err := CompileRecipeOutput(ArangoTarget(), groupedCompiled, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile unlimited semantic Group query: %v", err) } @@ -113,8 +110,6 @@ func TestSemanticCategoryGroupPreservesExactMembershipAndRowMultiplicityAgainstA sharedDisplayIDs = append(sharedDisplayIDs, rowID) } } - // Two distinct semantic tuples intentionally share the same public label. - // Their row identities must still be distinct, so the label cannot own key equality. if len(sharedDisplayIDs) != 2 || sharedDisplayIDs[0] == "" || sharedDisplayIDs[1] == "" || sharedDisplayIDs[0] == sharedDisplayIDs[1] { t.Fatalf("same-display semantic groups have row IDs %#v, want two distinct tuple identities", sharedDisplayIDs) } @@ -138,27 +133,29 @@ func TestSemanticCategoryGroupPreservesExactMembershipAndRowMultiplicityAgainstA t.Fatalf("active shared-code Group row with two contributors is missing: %#v", grouped) } sharedCodeRowID, _ := sharedCodeRow["__loom_row_id"].(string) - lineageQuery, err := CompileRowLineageOutput(groupedCompiled, sharedCodeRowID, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + lineageQuery, err := CompileRowLineageOutput(ArangoTarget(), groupedCompiled, sharedCodeRowID, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile semantic Group row-lineage page: %v", err) } identityProbe := lineageQuery - identityProbe.BindVars = make(map[string]any, len(lineageQuery.BindVars)) - for key, value := range lineageQuery.BindVars { - identityProbe.BindVars[key] = value + probeText := testAQLText(t, lineageQuery.Command) + probeBinds := make(map[string]any, len(testAQLBinds(t, lineageQuery.Command))) + for key, value := range testAQLBinds(t, lineageQuery.Command) { + probeBinds[key] = value } - identityProbe.Query = strings.Replace(identityProbe.Query, + probeText = strings.Replace(probeText, "FILTER __loom_physical_construction_row_lineage_semantic_identity == @row_lineage_row_id", "FILTER __loom_physical_construction_row_lineage_semantic_collected_key_0 == @diagnostic_scalar_key AND __loom_physical_construction_row_lineage_semantic_tuple == @diagnostic_semantic_tuple", 1) - identityProbe.Query = strings.Replace(identityProbe.Query, + probeText = strings.Replace(probeText, "RETURN {key0: __loom_physical_construction_row_lineage_semantic_collected_key_0, semanticTuple: __loom_physical_construction_row_lineage_semantic_tuple}", "RETURN {identity: __loom_physical_construction_row_lineage_semantic_identity, key0: __loom_physical_construction_row_lineage_semantic_collected_key_0, semanticTuple: __loom_physical_construction_row_lineage_semantic_tuple}", 1) - identityProbe.Query = strings.Replace(identityProbe.Query, + probeText = strings.Replace(probeText, "RETURN {found: __loom_physical_construction_row_lineage_semantic_selected_group != null, contributors:", "RETURN {selectedGroup: __loom_physical_construction_row_lineage_semantic_selected_group, found: __loom_physical_construction_row_lineage_semantic_selected_group != null, contributors:", 1) - identityProbe.BindVars["diagnostic_scalar_key"] = "active" - identityProbe.BindVars["diagnostic_semantic_tuple"] = []any{targetSystem, targetVersion, "shared-code"} - delete(identityProbe.BindVars, rowLineageRowIDBind) + probeBinds["diagnostic_scalar_key"] = "active" + probeBinds["diagnostic_semantic_tuple"] = []any{targetSystem, targetVersion, "shared-code"} + delete(probeBinds, rowLineageRowIDBind) + identityProbe.Command = testAQLCommand(probeText, probeBinds) identityResult := executeRowLineageOracleQuery(t, ctx, client, identityProbe) selectedGroup, _ := identityResult["selectedGroup"].(map[string]any) lineageIdentity, _ := selectedGroup["identity"].(string) @@ -169,7 +166,7 @@ func TestSemanticCategoryGroupPreservesExactMembershipAndRowMultiplicityAgainstA contributors, _ := lineage["contributors"].([]any) if lineage["found"] != true || lineage["hasMore"] != false || len(contributors) != 2 { t.Fatalf("semantic Group row lineage = %#v, want the two exact matching source rows; grouped row=%#v executed identity=%#v; AQL:\n%s", - lineage, sharedCodeRow, identityResult, lineageQuery.Query) + lineage, sharedCodeRow, identityResult, testAQLText(t, lineageQuery.Command)) } wantContributorIDs := map[string]bool{"same-source-row": false, "same-code-active-second-row": false} for _, value := range contributors { @@ -191,12 +188,17 @@ func TestSemanticCategoryGroupPreservesExactMembershipAndRowMultiplicityAgainstA bagQuery := compile(recipe.ConstructionGroupMissingKeyGroup) const rootScan = "FOR root IN @@root_collection" - if strings.Count(bagQuery.Query, rootScan) != 1 { - t.Fatalf("semantic Group query has %d replaceable direct-root scans, want one:\n%s", strings.Count(bagQuery.Query, rootScan), bagQuery.Query) + if strings.Count(testAQLText(t, bagQuery.Command), rootScan) != 1 { + t.Fatalf("semantic Group query has %d replaceable direct-root scans, want one:\n%s", strings.Count(testAQLText(t, bagQuery.Command), rootScan), testAQLText(t, bagQuery.Command)) } - bagQuery.Query = strings.Replace(bagQuery.Query, rootScan, + bagText := strings.Replace(testAQLText(t, bagQuery.Command), rootScan, "FOR root IN [DOCUMENT(@@root_collection, @duplicate_root_key), DOCUMENT(@@root_collection, @duplicate_root_key)]", 1) - bagQuery.BindVars["duplicate_root_key"] = project + "_same-source-row" + bagBinds := make(map[string]any, len(testAQLBinds(t, bagQuery.Command))+1) + for key, value := range testAQLBinds(t, bagQuery.Command) { + bagBinds[key] = value + } + bagBinds["duplicate_root_key"] = project + "_same-source-row" + bagQuery.Command = testAQLCommand(bagText, bagBinds) bagRows := executeReshapeOracleQuery(t, ctx, client, bagQuery) bagCounts := make(map[string][]float64, len(bagRows)) for _, row := range bagRows { @@ -254,7 +256,7 @@ func TestSemanticCategoryGroupPreservesExactMembershipAndRowMultiplicityAgainstA } errorQuery := compile(recipe.ConstructionGroupMissingKeyError) - if err := client.QueryRows(ctx, errorQuery.Query, 100, errorQuery.BindVars, func(map[string]any) error { return nil }); err == nil { + if err := client.QueryRows(ctx, testAQLText(t, errorQuery.Command), 100, testAQLBinds(t, errorQuery.Command), func(map[string]any) error { return nil }); err == nil { t.Fatal("ERROR semantic Group accepted missing and out-of-family category rows") } } @@ -349,7 +351,7 @@ func semanticGroupArangoOutput( func compileSemanticCategoryGroupQuery(t *testing.T, bindings recipe.RuntimeBindings, output recipe.Output) CompiledQuery { t.Helper() compiled := lowerConstructionOutput(t, output, bindings) - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile semantic Group query: %v", err) } diff --git a/internal/dataframe/compiler/semantic_group_preview_arango_test.go b/internal/dataframe/compiler/semantic_group_preview_arango_test.go index 38e554bd5..255bd0a8c 100644 --- a/internal/dataframe/compiler/semantic_group_preview_arango_test.go +++ b/internal/dataframe/compiler/semantic_group_preview_arango_test.go @@ -23,17 +23,17 @@ func TestSemanticGroupScalarKeyPreviewFrontierActivatesOnCompiledPlan(t *testing targetSystem, targetVersion, codingPath, ) compiled := lowerConstructionOutput(t, output, bindings) - preview, err := CompileRecipeOutputWithPolicy(compiled, bindings, 2, ir.DefaultPhysicalOptimizationPolicy()) + preview, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 2, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile semantic Group preview: %v", err) } - if strings.Contains(preview.Query, "group_preview_selected_root_keys") { - if strings.Count(preview.Query, "root._key") < 2 || !strings.Contains(preview.Query, "group_preview_selected_root_keys") { - t.Fatalf("frontier did not retain and replay the direct root identity projection:\n%s", preview.Query) + if strings.Contains(testAQLText(t, preview.Command), "group_preview_selected_root_keys") { + if strings.Count(testAQLText(t, preview.Command), "root._key") < 2 || !strings.Contains(testAQLText(t, preview.Command), "group_preview_selected_root_keys") { + t.Fatalf("frontier did not retain and replay the direct root identity projection:\n%s", testAQLText(t, preview.Command)) } return } - t.Fatalf("compiled semantic Group plan did not enter its proven frontier:\n%s", preview.Query) + t.Fatalf("compiled semantic Group plan did not enter its proven frontier:\n%s", testAQLText(t, preview.Command)) } func TestSemanticGroupScalarKeyPreviewMatchesFullQueryAgainstArango(t *testing.T) { @@ -94,27 +94,28 @@ func TestSemanticGroupScalarKeyPreviewMatchesFullQueryAgainstArango(t *testing.T ) compiled := lowerConstructionOutput(t, output, bindings) const previewLimit = 4 - preview, err := CompileRecipeOutputWithPolicy(compiled, bindings, previewLimit, ir.DefaultPhysicalOptimizationPolicy()) + preview, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, previewLimit, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile semantic Group preview: %v", err) } - full, err := CompileRecipeOutputWithPolicy(compiled, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + full, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile full semantic Group query: %v", err) } - if !strings.Contains(preview.Query, "group_preview_selected_root_keys") { - t.Fatalf("semantic Group preview did not activate the direct-root key frontier:\n%s", preview.Query) + if !strings.Contains(testAQLText(t, preview.Command), "group_preview_selected_root_keys") { + t.Fatalf("semantic Group preview did not activate the direct-root key frontier:\n%s", testAQLText(t, preview.Command)) } - if strings.Count(preview.Query, "FILTER root.project == @project") != 2 || - strings.Count(preview.Query, "FILTER root.dataset_generation == @dataset_generation") != 2 || - strings.Count(preview.Query, "root.auth_resource_path IN @auth_resource_paths") != 2 { - t.Fatalf("frontier or selected-root replay dropped a source/auth predicate:\n%s", preview.Query) + if strings.Count(testAQLText(t, preview.Command), "FILTER root.project == @project") != 2 || + strings.Count(testAQLText(t, preview.Command), "FILTER root.dataset_generation == @dataset_generation") != 2 || + strings.Count(testAQLText(t, preview.Command), "root.auth_resource_path IN @auth_resource_paths") != 2 { + t.Fatalf("frontier or selected-root replay dropped a source/auth predicate:\n%s", testAQLText(t, preview.Command)) } if preview.PreviewCoveringIndex != nil { - hinted, _, err := RenderPreviewPhysicalPlanWithRootIndexHint(preview.PhysicalPlan, preview.PreviewCoveringIndex.Name) + hintedCommand, err := RenderPreviewPhysicalPlanWithRootIndexHint(preview.PhysicalPlan, preview.PreviewCoveringIndex.Name) if err != nil { t.Fatalf("render semantic Group frontier with the non-forcing preview index hint: %v", err) } + hinted := testAQLText(t, hintedCommand) if !strings.Contains(hinted, "OPTIONS { indexHint: \""+preview.PreviewCoveringIndex.Name+"\", forceIndexHint: false }") || !strings.Contains(hinted, "group_preview_selected_root_keys") { t.Fatalf("non-forcing preview hint disabled the semantic Group frontier:\n%s", hinted) @@ -153,20 +154,17 @@ func TestSemanticGroupScalarKeyPreviewMatchesFullQueryAgainstArango(t *testing.T t.Fatalf("fixture must have more scalar keys than its preview limit %d: %#v", previewLimit, uniqueScalarKeys) } - // This malformed scalar sorts after the valid string-key page. The frontier - // must preserve the canonical global type error instead of hiding it beyond - // the preview limit. insertConstructionReshapeRows(t, ctx, client, project, generation, []map[string]any{{ "id": "late_malformed_status", "status": map[string]any{"unexpected": "object"}, "auth_resource_path": "/allowed", "component": []any{semanticGroupComponent(targetSystem, targetVersion, "late", "Late")}, }}) for _, limit := range []int{previewLimit, 0} { - query, err := CompileRecipeOutputWithPolicy(compiled, bindings, limit, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled, bindings, limit, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile semantic Group query with malformed scalar key and limit %d: %v", limit, err) } - if err := client.QueryRows(ctx, query.Query, 100, query.BindVars, func(map[string]any) error { return nil }); err == nil { + if err := client.QueryRows(ctx, testAQLText(t, query.Command), 100, testAQLBinds(t, query.Command), func(map[string]any) error { return nil }); err == nil { t.Fatalf("semantic Group with limit %d hid the malformed key outside its selected page", limit) } } diff --git a/internal/dataframe/compiler/semantic_group_projection_factorization_arango_test.go b/internal/dataframe/compiler/semantic_group_projection_factorization_arango_test.go index 49f4f1057..15cc01d43 100644 --- a/internal/dataframe/compiler/semantic_group_projection_factorization_arango_test.go +++ b/internal/dataframe/compiler/semantic_group_projection_factorization_arango_test.go @@ -16,15 +16,16 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/catalog" "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/recipe/exec" "github.com/calypr/loom/internal/dataframe/semantic" - arangostore "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -52,8 +53,6 @@ func TestSemanticGroupProjectionFactorizationMatchesExactArangoOutputs(t *testin })}, )) } - // This otherwise matching projection must be removed by the source scope - // before either the assessment sample or the weighted Group is evaluated. payloads = append(payloads, semanticGroupProjectionRow( "blocked_entropy", "/blocked", []any{semanticGroupProjectionComponent(targetSystem, targetVersion, @@ -85,7 +84,6 @@ func TestSemanticGroupProjectionFactorizationMatchesExactArangoOutputs(t *testin {code: "low-common", display: "Z Common"}, {code: "low-common", display: "A Common"}, {code: "low-common", display: "A Common"}, - // Different tuple identities intentionally share one display. {code: "low-label-one", display: "Shared label"}, {code: "low-label-two", display: "Shared label"}, }, @@ -106,17 +104,12 @@ func TestSemanticGroupProjectionFactorizationMatchesExactArangoOutputs(t *testin })}, )) } - // Both source rows are in scope but have no matching Coding tuple. GROUP - // preserves them as one missing-category group; EXCLUDE drops them and - // ERROR rejects the query. payloads = append(payloads, semanticGroupProjectionRow("zz_empty_coding", "/included", []any{}), semanticGroupProjectionRow("zz_no_matching_coding", "/included", []any{semanticGroupProjectionComponent("urn:loom:other-system", targetVersion, []semanticGroupProjectionCoding{{code: "unmatched", display: "Unmatched"}})}), ) - // This matching projection sorts before the repeated prefix but is outside - // the allowed source scope and must not affect the bounded assessment. payloads = append(payloads, semanticGroupProjectionRow( "000_blocked", "/blocked", []any{sharedProjection}, )) @@ -128,8 +121,6 @@ func TestSemanticGroupProjectionFactorizationMatchesExactArangoOutputs(t *testin result := compileRunAndCaptureSemanticGroupProjectionPreview(t, ctx, client, bindings, groupOutput, 600) assertSemanticGroupProjectionRowsEqual(t, result.ordinaryRows, result.factoredRows) assertSemanticGroupProjectionRowsEqual(t, result.ordinaryRows, result.previewRows) - // The sample is bounded and unordered. The exact preview must still - // include every unique projection after its 512-row assessment window. assertSemanticGroupProjectionAssessment(t, result.assessmentRows, 512, 1, 512) if uniqueTailCount <= 512 { t.Fatal("low-diversity fixture must place more unique projections after the bounded sample than the sample can contain") @@ -151,7 +142,7 @@ func TestSemanticGroupProjectionFactorizationMatchesExactArangoOutputs(t *testin errorQuery := compileSemanticGroupProjectionPolicy(t, ctx, result.engine, bindings, semanticGroupProjectionOutput(t, recipe.ConstructionGroupMissingKeyError, bindingID, familyID, ruleVersion, targetSystem, targetVersion, codingPath), 600) - if err := client.QueryRowsWithMaxRuntime(ctx, errorQuery.Query, 2000, errorQuery.BindVars, 8*time.Second, func(map[string]any) error { return nil }); err == nil { + if err := client.QueryRowsWithMaxRuntime(ctx, testAQLText(t, errorQuery.Command), 2000, testAQLBinds(t, errorQuery.Command), 8*time.Second, func(map[string]any) error { return nil }); err == nil { t.Fatal("ERROR semantic Group accepted in-scope empty and nonmatching Coding arrays") } }) @@ -181,8 +172,6 @@ func TestSemanticGroupProjectionFactorizationSelectsReadOnlyIndexForScopedFixtur rows = append(rows, semanticGroupProjectionDirectCodingRow("blocked", "/blocked", codings)) insertSemanticGroupProjectionRowsForResource(t, ctx, client, resourceType, project, generation, rows) - // The selector is read-only: explicitly provision its prerequisite in this - // test-owned fixture instead of relying on an index from a developer's CDA DB. const fixtureIndex = "loom_pivot_preview_semantic_projection_fixture" indexesBeforeSetup := semanticProjectionIndexSnapshot(t, ctx, resourceType) if err := client.EnsurePreviewCoveringIndex(ctx, resourceType, fixtureIndex, @@ -228,7 +217,6 @@ func TestSemanticGroupProjectionFactorizationSelectsExistingIndexOnFullCDAReadOn if os.Getenv("LOOM_TEST_FULL_CDA_READONLY") != "1" { t.Skip("requires explicit LOOM_TEST_FULL_CDA_READONLY=1 and a preseeded full CDA database with its existing covering index; the bounded scoped fixture runs separately") } - // Do not let arangostore.Open create an empty DB for this read-only probe. semanticProjectionIndexRequest(t, context.Background(), http.MethodGet, "/_api/database/current") ctx, client := openSemanticGroupProjectionArango(t) const resourceType, project, generation, codingPath = "Specimen", "loom_dev_cda_fhir", "cda-fhir-v1", "type.coding[]" @@ -303,12 +291,12 @@ func TestSemanticGroupProjectionFactorizationProjectsSimpleCodingArrayWhole(t *t if err != nil { t.Fatalf("lower simple-path semantic Group plan: %v", err) } - query, err := compiler.CompileRecipeOutputWithPolicy(compiled.Outputs[0], bindings, 32, ir.DefaultPhysicalOptimizationPolicy()) + query, err := compiler.CompileRecipeOutput(compiler.ArangoTarget(), compiled.Outputs[0], bindings, 32, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile simple-path semantic Group preview: %v", err) } if query.PreviewGroupProjection == nil { - t.Fatalf("simple repeated Coding path did not qualify for projection factoring:\n%s", query.Query) + t.Fatalf("simple repeated Coding path did not qualify for projection factoring:\n%s", testAQLText(t, query.Command)) } rootVariable := "" for _, operation := range query.PhysicalPlan.Operations { @@ -323,18 +311,18 @@ func TestSemanticGroupProjectionFactorizationProjectsSimpleCodingArrayWhole(t *t arrayPath := fmt.Sprintf("%s.payload[%q][%q]", rootVariable, "code", "coding") wantArray := fmt.Sprintf("(IS_ARRAY(%s) ? %s : [])", arrayPath, arrayPath) assertSimpleSemanticGroupProjectionArray(t, "assessment", func() string { - text, _, err := compiler.RenderPreviewGroupProjectionAssessment(query.PhysicalPlan, "", true) + assessmentCommand, err := compiler.RenderPreviewGroupProjectionAssessment(query.PhysicalPlan, "", true) if err != nil { t.Fatalf("render simple-path assessment: %v", err) } - return text + return testAQLText(t, assessmentCommand) }(), wantArray) assertSimpleSemanticGroupProjectionArray(t, "factored query", func() string { - text, _, err := compiler.RenderPreviewGroupProjectionFactorization(query.PhysicalPlan, "", true) + factorCommand, err := compiler.RenderPreviewGroupProjectionFactorization(query.PhysicalPlan, "", true) if err != nil { t.Fatalf("render simple-path factored query: %v", err) } - return text + return testAQLText(t, factorCommand) }(), wantArray) } @@ -370,13 +358,13 @@ type semanticGroupProjectionCallbackCounts struct { type semanticGroupProjectionExplainResult struct { query string - result arangostore.ExplainResult + result arangofactory.ExplainResult } func compileRunAndCaptureSemanticGroupProjectionPreview( t *testing.T, ctx context.Context, - client *arangostore.Client, + client *arangofactory.Client, bindings recipe.RuntimeBindings, output recipe.Output, limit int, @@ -385,19 +373,19 @@ func compileRunAndCaptureSemanticGroupProjectionPreview( engine, resolved, ordinaryQuery, callbacks := prepareSemanticGroupProjectionPreview(t, ctx, client, bindings, output, limit) ordinaryRows := querySemanticGroupProjectionRows(t, ctx, client, ordinaryQuery, 2000) - factoredText, factoredBindVars, err := compiler.RenderPreviewGroupProjectionFactorization(ordinaryQuery.PhysicalPlan, "", false) + factoredCommand, err := compiler.RenderPreviewGroupProjectionFactorization(ordinaryQuery.PhysicalPlan, "", false) if err != nil { t.Fatalf("render exact factored semantic Group projection query: %v", err) } factoredRows := querySemanticGroupProjectionRows(t, ctx, client, - compiler.CompiledQuery{Query: factoredText, BindVars: factoredBindVars}, 2000) + compiler.CompiledQuery{Command: factoredCommand}, 2000) - assessmentText, assessmentBindVars, err := compiler.RenderPreviewGroupProjectionAssessment(ordinaryQuery.PhysicalPlan, "", false) + assessmentCommand, err := compiler.RenderPreviewGroupProjectionAssessment(ordinaryQuery.PhysicalPlan, "", false) if err != nil { t.Fatalf("render bounded semantic Group projection assessment: %v", err) } assessmentRows := querySemanticGroupProjectionRows(t, ctx, client, - compiler.CompiledQuery{Query: assessmentText, BindVars: assessmentBindVars}, 10) + compiler.CompiledQuery{Command: assessmentCommand}, 10) var previewRows []map[string]any summary, err := engine.PreviewOutput(ctx, resolved, @@ -423,7 +411,7 @@ func compileRunAndCaptureSemanticGroupProjectionPreview( func prepareSemanticGroupProjectionPreview( t *testing.T, ctx context.Context, - client *arangostore.Client, + client *arangofactory.Client, bindings recipe.RuntimeBindings, output recipe.Output, limit int, @@ -432,14 +420,27 @@ func prepareSemanticGroupProjectionPreview( bundle := semanticGroupProjectionBundle(output) callbacks := &semanticGroupProjectionCallbackCounts{} engine, err := execution.New(execution.Config{ - Registry: semanticGroupProjectionRegistry{bundle: bundle}, - QueryRows: func(ctx context.Context, query string, batchSize int, bindVars map[string]any, visit func(map[string]any) error) error { - return client.QueryRows(ctx, query, batchSize, bindVars, visit) + CompileTarget: compiler.ArangoTarget(), + Registry: semanticGroupProjectionRegistry{bundle: bundle}, + SourceRows: func(ctx context.Context, request execution.RowsRequest, visit execution.RowVisitor) error { + command := request.Command + batchSize := request.BatchSize + query, bindVars, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return client.QueryRows(ctx, query, batchSize, bindVars, arangofactory.RowVisitor(visit)) }, - PreviewQueryRows: func(ctx context.Context, query string, batchSize int, bindVars map[string]any, visit func(map[string]any) error) error { + PreviewSourceRows: func(ctx context.Context, request execution.RowsRequest, visit execution.RowVisitor) error { + command := request.Command + batchSize := request.BatchSize + query, bindVars, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } callbacks.previewQueries = append(callbacks.previewQueries, query) started := time.Now() - err := client.QueryRowsWithMaxRuntime(ctx, query, batchSize, bindVars, 8*time.Second, visit) + err := client.QueryRowsWithMaxRuntime(ctx, query, batchSize, bindVars, 8*time.Second, arangofactory.RowVisitor(visit)) callbacks.previewQueryDurations = append(callbacks.previewQueryDurations, time.Since(started)) return err }, @@ -447,19 +448,33 @@ func prepareSemanticGroupProjectionPreview( callbacks.revisions++ return client.CollectionRevision(ctx, collection) }, - PreviewExplainQuery: func(ctx context.Context, query string, bindVars map[string]any) (arangostore.ExplainResult, error) { + PreviewCollectionScans: func(ctx context.Context, command command.CompiledCommand, collection string) (execution.PreviewCollectionScanEvidence, error) { callbacks.explains++ - result, err := client.Explain(ctx, arangostore.ExplainRequest{Query: query, BindVars: bindVars}) + query, bindVars, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return execution.PreviewCollectionScanEvidence{}, dispatchErr + } + result, err := client.Explain(ctx, arangofactory.ExplainRequest{Query: query, BindVars: bindVars}) callbacks.explainResults = append(callbacks.explainResults, semanticGroupProjectionExplainResult{query: query, result: result}) - return result, err + if err != nil { + return execution.PreviewCollectionScanEvidence{}, err + } + return semanticGroupProjectionScanEvidence(result, collection), nil }, - PreviewCoveringIndexes: func(ctx context.Context, collection string) ([]arangostore.PreviewCoveringIndex, error) { + PreviewCoveringIndexes: func(ctx context.Context, collection string) ([]execution.PreviewCollectionIndex, error) { callbacks.indexLists++ indexes, err := client.ListPreviewCoveringIndexes(ctx, collection) + converted := make([]execution.PreviewCollectionIndex, 0, len(indexes)) for _, index := range indexes { callbacks.listedIndexNames = append(callbacks.listedIndexNames, index.Name) + converted = append(converted, execution.PreviewCollectionIndex{ + Collection: index.Collection, + Name: index.Name, + Fields: index.Fields, + StoredValues: index.StoredValues, + }) } - return indexes, err + return converted, err }, PreviewCollectionCount: func(ctx context.Context, collection string) (int64, error) { callbacks.collectionCount++ @@ -473,14 +488,14 @@ func prepareSemanticGroupProjectionPreview( if err != nil { t.Fatalf("resolve semantic Group projection fixture: %v", err) } - ordinaryQuery, err := compiler.CompileRecipeOutputWithPolicy( + ordinaryQuery, err := compiler.CompileRecipeOutput(compiler.ArangoTarget(), resolved.Compiled.Outputs[0], bindings, limit, ir.DefaultPhysicalOptimizationPolicy(), ) if err != nil { t.Fatalf("compile ordinary semantic Group projection query: %v", err) } if ordinaryQuery.PreviewGroupProjection == nil { - t.Fatalf("direct semantic Group has no exact source-scope projection evidence:\n%s", ordinaryQuery.Query) + t.Fatalf("direct semantic Group has no exact source-scope projection evidence:\n%s", testAQLText(t, ordinaryQuery.Command)) } return engine, resolved, ordinaryQuery, callbacks } @@ -498,7 +513,7 @@ func compileSemanticGroupProjectionPolicy( if err != nil { t.Fatalf("resolve semantic Group %s policy: %v", output.Construction.Steps[0].Operation.Group.MissingKeyPolicy, err) } - query, err := compiler.CompileRecipeOutputWithPolicy(resolved.Compiled.Outputs[0], bindings, limit, ir.DefaultPhysicalOptimizationPolicy()) + query, err := compiler.CompileRecipeOutput(compiler.ArangoTarget(), resolved.Compiled.Outputs[0], bindings, limit, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile semantic Group %s policy: %v", output.Construction.Steps[0].Operation.Group.MissingKeyPolicy, err) } @@ -630,8 +645,6 @@ func semanticGroupProjectionDirectCodingRow(id, authResourcePath string, codings } } -// Only stable index definition fields are compared; read queries may update -// planner estimates, which are not schema mutations. type semanticProjectionIndexDefinition struct { ID string `json:"id"` Name string `json:"name"` @@ -701,7 +714,7 @@ func semanticGroupProjectionCodingValue(system, version, code, display string) m return coding } -func openSemanticGroupProjectionArango(t *testing.T) (context.Context, *arangostore.Client) { +func openSemanticGroupProjectionArango(t *testing.T) (context.Context, *arangofactory.Client) { t.Helper() url, database := os.Getenv("LOOM_TEST_ARANGO_URL"), os.Getenv("LOOM_TEST_ARANGO_DATABASE") if url == "" || database == "" { @@ -709,10 +722,11 @@ func openSemanticGroupProjectionArango(t *testing.T) (context.Context, *arangost } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) t.Cleanup(cancel) - client, err := arangostore.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client t.Cleanup(func() { _ = client.Close(context.Background()) }) return ctx, client } @@ -720,7 +734,7 @@ func openSemanticGroupProjectionArango(t *testing.T) (context.Context, *arangost func insertSemanticGroupProjectionRows( t *testing.T, ctx context.Context, - client *arangostore.Client, + client *arangofactory.Client, project, generation string, payloads []map[string]any, ) { @@ -731,7 +745,7 @@ func insertSemanticGroupProjectionRows( func insertSemanticGroupProjectionRowsForResource( t *testing.T, ctx context.Context, - client *arangostore.Client, + client *arangofactory.Client, resourceType, project, generation string, payloads []map[string]any, ) { @@ -760,12 +774,12 @@ func insertSemanticGroupProjectionRowsForResource( } } -func registerSemanticGroupProjectionCleanup(t *testing.T, client *arangostore.Client, project, generation string) { +func registerSemanticGroupProjectionCleanup(t *testing.T, client *arangofactory.Client, project, generation string) { t.Helper() registerSemanticGroupProjectionCleanupForResource(t, client, "Observation", project, generation) } -func registerSemanticGroupProjectionCleanupForResource(t *testing.T, client *arangostore.Client, resourceType, project, generation string) { +func registerSemanticGroupProjectionCleanupForResource(t *testing.T, client *arangofactory.Client, resourceType, project, generation string) { t.Helper() t.Cleanup(func() { cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -782,17 +796,17 @@ func registerSemanticGroupProjectionCleanupForResource(t *testing.T, client *ara func querySemanticGroupProjectionRows( t *testing.T, ctx context.Context, - client *arangostore.Client, + client *arangofactory.Client, query compiler.CompiledQuery, maxRows int, ) []map[string]any { t.Helper() rows := make([]map[string]any, 0) - if err := client.QueryRowsWithMaxRuntime(ctx, query.Query, maxRows, query.BindVars, 8*time.Second, func(row map[string]any) error { + if err := client.QueryRowsWithMaxRuntime(ctx, testAQLText(t, query.Command), maxRows, testAQLBinds(t, query.Command), 8*time.Second, func(row map[string]any) error { rows = append(rows, row) return nil }); err != nil { - t.Fatalf("execute semantic Group projection query: %v\n%s", err, query.Query) + t.Fatalf("execute semantic Group projection query: %v\n%s", err, testAQLText(t, query.Command)) } return rows } @@ -864,7 +878,7 @@ func assertSemanticGroupProjectionIndexDescriptor(t *testing.T, query compiler.C func assertSemanticGroupProjectionScopeCount( t *testing.T, ctx context.Context, - client *arangostore.Client, + client *arangofactory.Client, query compiler.CompiledQuery, want int, ) { @@ -872,10 +886,7 @@ func assertSemanticGroupProjectionScopeCount( if query.PreviewGroupProjection == nil { t.Fatal("semantic Group query lost its source-scope count evidence") } - countQuery := compiler.CompiledQuery{ - Query: query.PreviewGroupProjection.ScopeCountQuery, - BindVars: query.PreviewGroupProjection.ScopeCountBindVars, - } + countQuery := compiler.CompiledQuery{Command: query.PreviewGroupProjection.ScopeCount} rows := querySemanticGroupProjectionRows(t, ctx, client, countQuery, 10) if len(rows) != 1 || !semanticGroupProjectionNumberEqual(rows[0]["count"], float64(want)) { t.Fatalf("exact source-scope count = %#v, want %d scoped rows", rows, want) @@ -903,7 +914,7 @@ func assertProjectionSelectionCallbacks( } } for _, query := range callbacks.previewQueries { - if query == ordinaryQuery.PreviewGroupProjection.ScopeCountQuery { + if query == testAQLText(t, ordinaryQuery.PreviewGroupProjection.ScopeCount) { scopeCountSeen = true } if strings.Contains(query, "distinctProjections") && strings.Contains(query, "sampleRows") { @@ -938,7 +949,7 @@ func assertProjectionSelectionCallbacks( for index, query := range callbacks.previewQueries { role := "final preview" switch { - case query == ordinaryQuery.PreviewGroupProjection.ScopeCountQuery: + case query == testAQLText(t, ordinaryQuery.PreviewGroupProjection.ScopeCount): role = "exact scope count" case strings.Contains(query, "distinctProjections") && strings.Contains(query, "sampleRows"): role = "projection assessment" @@ -966,7 +977,7 @@ func hasCoveringFactoredIndexExplain(records []semanticGroupProjectionExplainRes if !strings.Contains(record.query, semanticGroupProjectionFactoredRawAlias) || !strings.Contains(record.query, indexName) { continue } - plans := append([]arangostore.ExplainPlan(nil), record.result.Plans...) + plans := append([]arangofactory.ExplainPlan(nil), record.result.Plans...) if record.result.Plan != nil { plans = append(plans, *record.result.Plan) } @@ -988,6 +999,45 @@ func hasCoveringFactoredIndexExplain(records []semanticGroupProjectionExplainRes return false } +func semanticGroupProjectionScanEvidence(result arangofactory.ExplainResult, collection string) execution.PreviewCollectionScanEvidence { + var plan *arangofactory.ExplainPlan + switch { + case result.Plan != nil && len(result.Plans) == 0: + plan = result.Plan + case result.Plan == nil && len(result.Plans) == 1: + plan = &result.Plans[0] + default: + return execution.PreviewCollectionScanEvidence{} + } + evidence := execution.PreviewCollectionScanEvidence{} + for _, node := range plan.Nodes { + if node.Collection != collection { + continue + } + switch node.Type { + case "EnumerateCollectionNode": + evidence.FullCollectionEnumeration = true + case "IndexNode": + nodeEvidence := execution.PreviewIndexNodeEvidence{Coverage: execution.PreviewIndexCoverageUnknown} + if node.IndexCoversProjections != nil && node.IndexCoversOutProjections != nil { + switch { + case *node.IndexCoversProjections && *node.IndexCoversOutProjections: + nodeEvidence.Coverage = execution.PreviewIndexCoverageCovering + case !*node.IndexCoversProjections && !*node.IndexCoversOutProjections: + nodeEvidence.Coverage = execution.PreviewIndexCoverageNotCovering + } + } + for _, index := range node.Indexes { + if index.Collection == "" || index.Collection == collection { + nodeEvidence.IndexNames = append(nodeEvidence.IndexNames, index.Name) + } + } + evidence.IndexNodes = append(evidence.IndexNodes, nodeEvidence) + } + } + return evidence +} + func assertSemanticGroupProjectionExpectedDisplayCounts(t *testing.T, rows []map[string]any, want map[string][]float64) { t.Helper() semanticGroupProjectionRowsByID(t, rows) diff --git a/internal/dataframe/compiler/semantic_group_projection_preview.go b/internal/dataframe/compiler/semantic_group_projection_preview.go index 00a71f5da..8b4f81642 100644 --- a/internal/dataframe/compiler/semantic_group_projection_preview.go +++ b/internal/dataframe/compiler/semantic_group_projection_preview.go @@ -1,30 +1,25 @@ package compiler import ( + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/render/aql" ) -// RenderPreviewGroupProjectionFactorization renders the exact semantic Group -// alternative after the runtime has selected its root scan. An empty indexName -// leaves the planner choice unchanged; disableIndex requests a full scan. -func RenderPreviewGroupProjectionFactorization(plan ir.PhysicalPlan, indexName string, disableIndex bool) (string, map[string]any, error) { +func RenderPreviewGroupProjectionFactorization(plan ir.PhysicalPlan, indexName string, disableIndex bool) (command.CompiledCommand, error) { rendered, err := aql.RenderPhysicalPlanWithSemanticGroupProjectionFactorization(plan, indexName, disableIndex) if err != nil { - return "", nil, err + return command.CompiledCommand{}, err } - return rendered.Query, rendered.BindVars, nil + return command.NewAQLCommand(rendered.Query, rendered.BindVars) } -// RenderPreviewGroupProjectionAssessment returns bounded metadata about the -// exact complete Coding-array projections seen under the plan's source scope. -// Its result contains no source values. -func RenderPreviewGroupProjectionAssessment(plan ir.PhysicalPlan, indexName string, disableIndex bool) (string, map[string]any, error) { +func RenderPreviewGroupProjectionAssessment(plan ir.PhysicalPlan, indexName string, disableIndex bool) (command.CompiledCommand, error) { rendered, err := aql.RenderPhysicalPlanWithSemanticGroupProjectionAssessment(plan, indexName, disableIndex) if err != nil { - return "", nil, err + return command.CompiledCommand{}, err } - return rendered.Query, rendered.BindVars, nil + return command.NewAQLCommand(rendered.Query, rendered.BindVars) } func previewGroupProjectionSpec(plan ir.PhysicalPlan) *PreviewGroupProjectionSpec { @@ -40,9 +35,12 @@ func previewGroupProjectionSpec(plan ir.PhysicalPlan) *PreviewGroupProjectionSpe if err != nil || scopeCount.Query == "" { return nil } - binds := ir.ClonePhysicalPlan(ir.PhysicalPlan{BindVars: scopeCount.BindVars}).BindVars + scopeCountCommand, err := command.NewAQLCommand(scopeCount.Query, scopeCount.BindVars) + if err != nil { + return nil + } return &PreviewGroupProjectionSpec{ - Collection: collection, ScopeCountQuery: scopeCount.Query, ScopeCountBindVars: binds, + Collection: collection, ScopeCount: scopeCountCommand, ReadOnlyIndex: previewGroupProjectionIndexSpec(plan), } } diff --git a/internal/dataframe/compiler/source_group_preview_arango_test.go b/internal/dataframe/compiler/source_group_preview_arango_test.go index 41045cd7e..81574f9d6 100644 --- a/internal/dataframe/compiler/source_group_preview_arango_test.go +++ b/internal/dataframe/compiler/source_group_preview_arango_test.go @@ -10,9 +10,9 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/recipe" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -23,30 +23,30 @@ func TestSourceGroupPreviewMatchesGenericScopedExpandedGroupAgainstArango(t *tes AuthScopeMode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{"/allowed"}, } compiledOutput := lowerConstructionOutput(t, sourceGroupPreviewArangoOutput(), bindings) - preview, err := CompileRecipeOutputWithPolicy(compiledOutput, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + preview, err := CompileRecipeOutput(ArangoTarget(), compiledOutput, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile key-paged source Group preview: %v", err) } - full, err := CompileRecipeOutputWithPolicy(compiledOutput, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + full, err := CompileRecipeOutput(ArangoTarget(), compiledOutput, bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile generic full-source Group: %v", err) } - selectedKeys := strings.Index(preview.Query, "source_group_preview_selected_root_keys") - canonicalGroup := strings.Index(preview.Query, "LET __loom_construction_stage_1 = (") + selectedKeys := strings.Index(testAQLText(t, preview.Command), "source_group_preview_selected_root_keys") + canonicalGroup := strings.Index(testAQLText(t, preview.Command), "LET __loom_construction_stage_1 = (") hasFocusRelationship := false - for _, value := range preview.BindVars { + for _, value := range testAQLBinds(t, preview.Command) { if relationship, ok := value.(string); ok && relationship == "focus_Specimen" { hasFocusRelationship = true break } } if selectedKeys < 0 || canonicalGroup < selectedKeys || !hasFocusRelationship || - !strings.Contains(preview.Query, "WITH COUNT INTO") || !strings.Contains(preview.Query, "__loom_physical_unnest_index_0 IN") { - t.Fatalf("preview did not key-page before the unchanged routed COUNT_ROWS Group:\n%s", preview.Query) + !strings.Contains(testAQLText(t, preview.Command), "WITH COUNT INTO") || !strings.Contains(testAQLText(t, preview.Command), "__loom_physical_unnest_index_0 IN") { + t.Fatalf("preview did not key-page before the unchanged routed COUNT_ROWS Group:\n%s", testAQLText(t, preview.Command)) } ctx, client := openConstructionReshapeArango(t) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{ + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Specimen"}, {Name: "Observation"}, {Name: "fhir_edge", Edge: true}, }}); err != nil { t.Fatal(err) @@ -141,15 +141,11 @@ func TestSourceGroupPreviewMatchesGenericScopedExpandedGroupAgainstArango(t *tes } addValid("missing_key", "observation_missing_key", nil, false, "one") addValid("null_key", "observation_null_key", nil, true, "one", "two") - // This malformed key has no eligible relationship. It must not trigger the - // STRING-key assertion in either the preview frontier or the generic Group. roots = append(roots, resource("malformed_ineligible", "Specimen", project, generation, "/allowed", rootPayload(map[string]any{"unexpected": "object"}, true))) observations = append(observations, resource("observation_malformed_ineligible", "Observation", project, generation, "/allowed", observationPayload())) roots = append(roots, resource("malformed_eligible", "Specimen", project, generation, "/allowed", rootPayload(map[string]any{"unexpected": "eligible-object"}, true))) - // A key without any route is excluded from both the scoped source rows and - // the preview frontier. roots = append(roots, resource("no_eligible_route", "Specimen", project, generation, "/allowed", rootPayload("00-no-route", true))) addInvalidRoute := func(rootID, observationID, relation string, edgeProject, edgeGeneration, edgeAuth, targetProject, targetGeneration, targetAuth string) { @@ -191,8 +187,6 @@ func TestSourceGroupPreviewMatchesGenericScopedExpandedGroupAgainstArango(t *tes t.Fatalf("last generic Group = %#v, want group_26 after the 25-row preview boundary", fullRows[len(fullRows)-1]) } - // Confirm from Arango's own mixed-type sort that an object key lies beyond - // this 25-group page before making it relationship-eligible below. var orderedTypes []string orderQuery := `FOR root IN Specimen FILTER root.project == @project AND root.dataset_generation == @dataset_generation @@ -223,11 +217,11 @@ RETURN {kind: TYPENAME(group_key)}` insert("Observation", []json.RawMessage{eligibleTarget}) eligibleEdge := edge("edge_malformed_eligible", "malformed_eligible", "observation_malformed_eligible", project, generation, "/allowed") insert("fhir_edge", []json.RawMessage{eligibleEdge}) - previewErr := executeSourceGroupPreviewError(ctx, client, preview) + previewErr := executeSourceGroupPreviewError(t, ctx, client, preview) if previewErr == nil || !strings.Contains(previewErr.Error(), "CONSTRUCTION_GROUP_KEY_TYPE_MISMATCH") { t.Fatalf("eligible malformed STRING key preview error = %v, want CONSTRUCTION_GROUP_KEY_TYPE_MISMATCH despite being beyond the first 25 groups", previewErr) } - fullErr := executeSourceGroupPreviewError(ctx, client, full) + fullErr := executeSourceGroupPreviewError(t, ctx, client, full) if fullErr == nil || !strings.Contains(fullErr.Error(), "CONSTRUCTION_GROUP_KEY_TYPE_MISMATCH") { t.Fatalf("generic Group error for eligible malformed key = %v, want CONSTRUCTION_GROUP_KEY_TYPE_MISMATCH", fullErr) } @@ -264,6 +258,6 @@ func sourceGroupPreviewArangoOutput() recipe.Output { } } -func executeSourceGroupPreviewError(ctx context.Context, client *store.Client, query CompiledQuery) error { - return client.QueryRows(ctx, query.Query, 500, query.BindVars, func(map[string]any) error { return nil }) +func executeSourceGroupPreviewError(t *testing.T, ctx context.Context, client *arangofactory.Client, query CompiledQuery) error { + return client.QueryRows(ctx, testAQLText(t, query.Command), 500, testAQLBinds(t, query.Command), func(map[string]any) error { return nil }) } diff --git a/internal/dataframe/compiler/streaming_missing_category_pivot_test.go b/internal/dataframe/compiler/streaming_missing_category_pivot_test.go index 6f351b6ee..1b6eddba5 100644 --- a/internal/dataframe/compiler/streaming_missing_category_pivot_test.go +++ b/internal/dataframe/compiler/streaming_missing_category_pivot_test.go @@ -54,7 +54,7 @@ func TestStreamingMissingCategoryPivotPreviewKeepsFullScopedPopulation(t *testin t.Fatal("full scoped terminal MISSING-category Pivot did not pass the streaming eligibility proof") } - query, err := CompileRecipeOutputWithPolicy(compiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -68,18 +68,18 @@ func TestStreamingMissingCategoryPivotPreviewKeepsFullScopedPopulation(t *testin !reflect.DeepEqual(index.Supersedes.Fields, wantLegacyFields) || !reflect.DeepEqual(index.Supersedes.StoredValues, wantStoredValues) { t.Fatalf("streaming missing-category Pivot did not augment its exact stored-parent index: index=%+v supersedes=%+v wantFields=%v wantLegacy=%v", index, index.Supersedes, wantFields, wantLegacyFields) } - if !strings.Contains(query.Query, "COLLECT ") || !strings.Contains(query.Query, " AGGREGATE ") { - t.Fatalf("query does not stream grouped values through COLLECT AGGREGATE:\n%s", query.Query) + if !strings.Contains(testAQLText(t, query.Command), "COLLECT ") || !strings.Contains(testAQLText(t, query.Command), " AGGREGATE ") { + t.Fatalf("query does not stream grouped values through COLLECT AGGREGATE:\n%s", testAQLText(t, query.Command)) } - if strings.Contains(query.Query, "INTO __loom_reshape_pivot_group_rows_") { - t.Fatalf("query materializes a full group-row array:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "INTO __loom_reshape_pivot_group_rows_") { + t.Fatalf("query materializes a full group-row array:\n%s", testAQLText(t, query.Command)) } - collectAt := strings.Index(query.Query, "COLLECT ") - assertAt := strings.Index(query.Query, "FILTER ASSERT(") - sortAt := strings.Index(query.Query, "SORT ") - limitAt := strings.Index(query.Query, "LIMIT @limit") + collectAt := strings.Index(testAQLText(t, query.Command), "COLLECT ") + assertAt := strings.Index(testAQLText(t, query.Command), "FILTER ASSERT(") + sortAt := strings.Index(testAQLText(t, query.Command), "SORT ") + limitAt := strings.Index(testAQLText(t, query.Command), "LIMIT @limit") if collectAt < 0 || assertAt < collectAt || sortAt < assertAt || limitAt < sortAt { - t.Fatalf("full-population Pivot assertions/window are out of order: collect=%d assert=%d sort=%d limit=%d\n%s", collectAt, assertAt, sortAt, limitAt, query.Query) + t.Fatalf("full-population Pivot assertions/window are out of order: collect=%d assert=%d sort=%d limit=%d\n%s", collectAt, assertAt, sortAt, limitAt, testAQLText(t, query.Command)) } for _, required := range []string{ "FILTER root.project == @project", @@ -87,40 +87,35 @@ func TestStreamingMissingCategoryPivotPreviewKeepsFullScopedPopulation(t *testin "FILTER root_scope_allowed == @scope_allowed", "TABLE_PIVOT_UNLISTED_CATEGORY", } { - if !strings.Contains(query.Query, required) { - t.Fatalf("query lost required full-scope or typed-category clause %q:\n%s", required, query.Query) + if !strings.Contains(testAQLText(t, query.Command), required) { + t.Fatalf("query lost required full-scope or typed-category clause %q:\n%s", required, testAQLText(t, query.Command)) } } pivot := compiled.Outputs[0].Plan.StageSequence.Stages[0].GroupedPivot - if streamingTestBindKey(query.BindVars, "reshape_category_column", pivot.CategoryColumn) != "" || - streamingTestBindKey(query.BindVars, "reshape_category_presence_column", pivot.CategoryPresenceColumn) != "" { - t.Fatalf("direct scalar Pivot retained field-name binds: %#v", query.BindVars) + if streamingTestBindKey(testAQLBinds(t, query.Command), "reshape_category_column", pivot.CategoryColumn) != "" || + streamingTestBindKey(testAQLBinds(t, query.Command), "reshape_category_presence_column", pivot.CategoryPresenceColumn) != "" { + t.Fatalf("direct scalar Pivot retained field-name binds: %#v", testAQLBinds(t, query.Command)) } - categoryValue := streamingTestDirectScalarVariable(query.Query, "__loom_physical_construction_reshape_stream_source_parent.code") - presenceValue := streamingTestDirectScalarVariable(query.Query, "IS_OBJECT(__loom_physical_construction_reshape_stream_source_parent)") + categoryValue := streamingTestDirectScalarVariable(testAQLText(t, query.Command), "__loom_physical_construction_reshape_stream_source_parent.code") + presenceValue := streamingTestDirectScalarVariable(testAQLText(t, query.Command), "IS_OBJECT(__loom_physical_construction_reshape_stream_source_parent)") if categoryValue == "" || presenceValue == "" { - t.Fatalf("direct scalar category or presence projection missing: category=%q presence=%q\n%s", categoryValue, presenceValue, query.Query) + t.Fatalf("direct scalar category or presence projection missing: category=%q presence=%q\n%s", categoryValue, presenceValue, testAQLText(t, query.Command)) } missingMatch := fmt.Sprintf("NOT (%s == true)", presenceValue) nullMatch := fmt.Sprintf("(%s == true AND %s == null)", presenceValue, categoryValue) - if !strings.Contains(query.Query, missingMatch) || !strings.Contains(query.Query, nullMatch) { - t.Fatalf("MISSING and explicit NULL categories are not distinguished by the presence sidecar: missing=%q null=%q\n%s", missingMatch, nullMatch, query.Query) + if !strings.Contains(testAQLText(t, query.Command), missingMatch) || !strings.Contains(testAQLText(t, query.Command), nullMatch) { + t.Fatalf("MISSING and explicit NULL categories are not distinguished by the presence sidecar: missing=%q null=%q\n%s", missingMatch, nullMatch, testAQLText(t, query.Command)) } - if strings.Contains(query.Query, "LET "+pivot.InputRowVariable+" = {") || strings.Contains(query.Query, pivot.InputRowVariable+"[@") { - t.Fatalf("direct scalar Pivot retained its temporary input row object or field reads:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "LET "+pivot.InputRowVariable+" = {") || strings.Contains(testAQLText(t, query.Command), pivot.InputRowVariable+"[@") { + t.Fatalf("direct scalar Pivot retained its temporary input row object or field reads:\n%s", testAQLText(t, query.Command)) } - // The renderer must not rewrite the validated canonical stage while it - // rebinds the clone to direct root projections for this preview query. canonicalPivot := compiled.Outputs[0].Plan.StageSequence.Stages[0].GroupedPivot if canonicalPivot == nil || len(canonicalPivot.InputProjections) == 0 || canonicalPivot.InputProjections[0].Value.Variable != compiled.Outputs[0].Plan.StageSequence.Stages[0].InputRowVariable { t.Fatal("streaming render mutated the canonical Pivot input projections") } - // Construction previews always request source identity. A Pivot has - // composite source metadata and must not gain a single root ID, so this - // production-shaped ERROR case remains eligible for streaming. identityOutput := streamingMissingCategoryPivotTestOutput() id := recipe.StageColumn{ID: "id_id", Name: "id", Type: "string", Nullable: true} identityOutput.Fields = append([]recipe.Field{{Name: "id", ColumnID: id.ID, Expr: recipe.Expression{Select: "root.id"}}}, identityOutput.Fields...) @@ -158,42 +153,42 @@ func TestStreamingMissingCategoryPivotPreviewKeepsFullScopedPopulation(t *testin } identityBindings := bindings identityBindings.IncludeSourceIdentity = true - identityQuery, err := CompileRecipeOutputWithPolicy(identityCompiled.Outputs[0], identityBindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + identityQuery, err := CompileRecipeOutput(ArangoTarget(), identityCompiled.Outputs[0], identityBindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if !strings.Contains(identityQuery.Query, " AGGREGATE ") || strings.Contains(identityQuery.Query, "INTO __loom_reshape_pivot_group_rows_") { - t.Fatalf("composite-source ERROR Pivot did not use full-population streaming aggregation:\n%s", identityQuery.Query) + if !strings.Contains(testAQLText(t, identityQuery.Command), " AGGREGATE ") || strings.Contains(testAQLText(t, identityQuery.Command), "INTO __loom_reshape_pivot_group_rows_") { + t.Fatalf("composite-source ERROR Pivot did not use full-population streaming aggregation:\n%s", testAQLText(t, identityQuery.Command)) } for _, required := range []string{ "FILTER root.project == @project", "FILTER root.dataset_generation == @dataset_generation", "FILTER root_scope_allowed == @scope_allowed", } { - if !strings.Contains(identityQuery.Query, required) { - t.Fatalf("identity-enabled streaming Pivot lost scoped source predicate %q:\n%s", required, identityQuery.Query) + if !strings.Contains(testAQLText(t, identityQuery.Command), required) { + t.Fatalf("identity-enabled streaming Pivot lost scoped source predicate %q:\n%s", required, testAQLText(t, identityQuery.Command)) } } - if strings.Contains(identityQuery.Query, ir.PreviewSourceResourceIDColumn) { - t.Fatalf("composite-source Pivot acquired a single-root source ID:\n%s", identityQuery.Query) + if strings.Contains(testAQLText(t, identityQuery.Command), ir.PreviewSourceResourceIDColumn) { + t.Fatalf("composite-source Pivot acquired a single-root source ID:\n%s", testAQLText(t, identityQuery.Command)) } for _, column := range identityQuery.OutputSchema { if column.Name == ir.PreviewSourceResourceIDColumn { t.Fatalf("composite-source Pivot added internal root-ID schema column: %#v", column) } } - errorLimitAt := strings.Index(identityQuery.Query, "LIMIT @limit") + errorLimitAt := strings.Index(testAQLText(t, identityQuery.Command), "LIMIT @limit") for _, code := range []string{"TABLE_PIVOT_CELL_CARDINALITY", "TABLE_PIVOT_UNLISTED_CATEGORY"} { - assertAt := strings.Index(identityQuery.Query, code) + assertAt := strings.Index(testAQLText(t, identityQuery.Command), code) if assertAt < 0 || errorLimitAt < 0 || assertAt >= errorLimitAt { - t.Fatalf("streaming Pivot lost full-population %s check before the preview window: assertion=%d limit=%d\n%s", code, assertAt, errorLimitAt, identityQuery.Query) + t.Fatalf("streaming Pivot lost full-population %s check before the preview window: assertion=%d limit=%d\n%s", code, assertAt, errorLimitAt, testAQLText(t, identityQuery.Command)) } } - ordinaryIdentityQuery, err := CompileRecipeOutputWithPolicy(identityCompiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + ordinaryIdentityQuery, err := CompileRecipeOutput(ArangoTarget(), identityCompiled.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - if identityQuery.Query != ordinaryIdentityQuery.Query || !reflect.DeepEqual(identityQuery.BindVars, ordinaryIdentityQuery.BindVars) || !reflect.DeepEqual(identityQuery.OutputSchema, ordinaryIdentityQuery.OutputSchema) || !reflect.DeepEqual(identityQuery.RowIdentity, ordinaryIdentityQuery.RowIdentity) { + if testAQLText(t, identityQuery.Command) != testAQLText(t, ordinaryIdentityQuery.Command) || !reflect.DeepEqual(testAQLBinds(t, identityQuery.Command), testAQLBinds(t, ordinaryIdentityQuery.Command)) || !reflect.DeepEqual(identityQuery.OutputSchema, ordinaryIdentityQuery.OutputSchema) || !reflect.DeepEqual(identityQuery.RowIdentity, ordinaryIdentityQuery.RowIdentity) { t.Fatal("requesting composite source metadata changed the Pivot query, schema, binds, or stable group identity") } @@ -219,15 +214,15 @@ func TestStreamingMissingCategoryPivotPreviewKeepsFullScopedPopulation(t *testin if err != nil { t.Fatal(err) } - assertionQuery, err := CompileRecipeOutputWithPolicy(assertionOutputs.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + assertionQuery, err := CompileRecipeOutput(ArangoTarget(), assertionOutputs.Outputs[0], bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - assertionLimit := strings.Index(assertionQuery.Query, "LIMIT @limit") + assertionLimit := strings.Index(testAQLText(t, assertionQuery.Command), "LIMIT @limit") for _, code := range []string{"TABLE_PIVOT_CELL_MISSING", "TABLE_PIVOT_CELL_CARDINALITY", "TABLE_PIVOT_UNLISTED_CATEGORY"} { - assertion := strings.Index(assertionQuery.Query, code) + assertion := strings.Index(testAQLText(t, assertionQuery.Command), code) if assertion < 0 || assertionLimit < 0 || assertion >= assertionLimit { - t.Fatalf("full-population %s assertion does not precede preview LIMIT: assertion=%d limit=%d\n%s", code, assertion, assertionLimit, assertionQuery.Query) + t.Fatalf("full-population %s assertion does not precede preview LIMIT: assertion=%d limit=%d\n%s", code, assertion, assertionLimit, testAQLText(t, assertionQuery.Command)) } } } diff --git a/internal/dataframe/compiler/table_reshape_arango_test.go b/internal/dataframe/compiler/table_reshape_arango_test.go index f6f010085..cfc589a76 100644 --- a/internal/dataframe/compiler/table_reshape_arango_test.go +++ b/internal/dataframe/compiler/table_reshape_arango_test.go @@ -11,12 +11,12 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/compiler/render/aql" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -131,12 +131,13 @@ func TestConstructionTerminalPivotTwoScanPreviewMatchesCanonicalWindowAgainstAra } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Patient"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Patient"}}}); err != nil { t.Fatal(err) } @@ -186,7 +187,7 @@ func TestConstructionTerminalPivotTwoScanPreviewMatchesCanonicalWindowAgainstAra if err != nil { t.Fatalf("resolve two-scan Pivot fixture plan: %v", err) } - queries, err := CompileResolvedRecipePlanWithPolicy(resolved, limit, ir.DefaultPhysicalOptimizationPolicy()) + queries, err := CompileResolvedRecipePlan(ArangoTarget(), resolved, limit, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile two-scan Pivot fixture: %v", err) } @@ -214,12 +215,13 @@ func TestDynamicConstructionPivotPreviewMatchesReducersAndNullCategoriesAgainstA } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Observation"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Observation"}}}); err != nil { t.Fatal(err) } @@ -281,7 +283,7 @@ func TestDynamicConstructionPivotPreviewMatchesReducersAndNullCategoriesAgainstA } compile := func(limit int) CompiledQuery { t.Helper() - queries, compileErr := CompileResolvedRecipePlanWithPolicy(resolved, limit, ir.DefaultPhysicalOptimizationPolicy()) + queries, compileErr := CompileResolvedRecipePlan(ArangoTarget(), resolved, limit, ir.DefaultPhysicalOptimizationPolicy()) if compileErr != nil { t.Fatalf("compile %s Pivot preview: %v", policy, compileErr) } @@ -289,8 +291,8 @@ func TestDynamicConstructionPivotPreviewMatchesReducersAndNullCategoriesAgainstA } staticRows := executeReshapeOracleQuery(t, ctx, client, compile(0)) preview := compile(10) - if !strings.Contains(preview.Query, "reshape_category_specs") { - t.Fatalf("%s preview did not use dynamic category specs:\n%s", policy, preview.Query) + if !strings.Contains(testAQLText(t, preview.Command), "reshape_category_specs") { + t.Fatalf("%s preview did not use dynamic category specs:\n%s", policy, testAQLText(t, preview.Command)) } previewRows := executeReshapeOracleQuery(t, ctx, client, preview) if !reflect.DeepEqual(previewRows, staticRows) { @@ -375,18 +377,18 @@ FOR document IN Observation if pivot == nil { t.Fatal("compiled output has no grouped Pivot") } - if !strings.Contains(query.Query, "COLLECT ") || !strings.Contains(query.Query, " AGGREGATE ") || - !strings.Contains(query.Query, "FIRST([root.payload.valueQuantity])") || - !strings.Contains(query.Query, "IS_OBJECT(__loom_physical_construction_reshape_stream_source_parent)") { - t.Fatalf("eligible direct-scalar Pivot did not emit the shared source-parent projection and streaming aggregate:\n%s", query.Query) + if !strings.Contains(testAQLText(t, query.Command), "COLLECT ") || !strings.Contains(testAQLText(t, query.Command), " AGGREGATE ") || + !strings.Contains(testAQLText(t, query.Command), "FIRST([root.payload.valueQuantity])") || + !strings.Contains(testAQLText(t, query.Command), "IS_OBJECT(__loom_physical_construction_reshape_stream_source_parent)") { + t.Fatalf("eligible direct-scalar Pivot did not emit the shared source-parent projection and streaming aggregate:\n%s", testAQLText(t, query.Command)) } - if strings.Contains(query.Query, "LET "+pivot.InputRowVariable+" = {") || strings.Contains(query.Query, pivot.InputRowVariable+"[@") { - t.Fatalf("eligible direct-scalar Pivot retained its per-row input object or repeated bound reads:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "LET "+pivot.InputRowVariable+" = {") || strings.Contains(testAQLText(t, query.Command), pivot.InputRowVariable+"[@") { + t.Fatalf("eligible direct-scalar Pivot retained its per-row input object or repeated bound reads:\n%s", testAQLText(t, query.Command)) } - categoryValue := streamingTestDirectScalarVariable(query.Query, "__loom_physical_construction_reshape_stream_source_parent.code") - amountValue := streamingTestDirectScalarVariable(query.Query, "__loom_physical_construction_reshape_stream_source_parent.value") + categoryValue := streamingTestDirectScalarVariable(testAQLText(t, query.Command), "__loom_physical_construction_reshape_stream_source_parent.code") + amountValue := streamingTestDirectScalarVariable(testAQLText(t, query.Command), "__loom_physical_construction_reshape_stream_source_parent.value") if categoryValue == "" || amountValue == "" { - t.Fatalf("direct category/value scalar expressions are missing: category=%q amount=%q\n%s", categoryValue, amountValue, query.Query) + t.Fatalf("direct category/value scalar expressions are missing: category=%q amount=%q\n%s", categoryValue, amountValue, testAQLText(t, query.Command)) } rows := executeReshapeOracleQuery(t, ctx, client, query) @@ -432,13 +434,11 @@ FOR document IN Observation errorOutput.Filters = []recipe.Filter{{Select: "root.status", Operator: recipe.FilterIn, Values: []recipe.FilterValue{reshapeOracleStringFilter(test.status)}}} errorOutput.Construction.Steps[0].Operation.Pivot.DuplicatePolicy = test.duplicate errorQuery := compileConstructionOutputQueryWithBindings(t, errorOutput, project, generation, recipe.RuntimeBindings{}) - if !strings.Contains(errorQuery.Query, "FIRST([root.payload.valueQuantity])") || strings.Contains(errorQuery.Query, "LET "+pivot.InputRowVariable+" = {") { - t.Fatalf("error case did not retain the direct-scalar streaming path:\n%s", errorQuery.Query) - } - err := client.QueryRows(ctx, errorQuery.Query, 500, errorQuery.BindVars, func(map[string]any) error { return nil }) - if err == nil || !strings.Contains(err.Error(), test.wantErrorCode) { - t.Fatalf("direct-scalar Pivot error = %v, want %s", err, test.wantErrorCode) + if !strings.Contains(testAQLText(t, errorQuery.Command), "FIRST([root.payload.valueQuantity])") || strings.Contains(testAQLText(t, errorQuery.Command), "LET "+pivot.InputRowVariable+" = {") { + t.Fatalf("error case did not retain the direct-scalar streaming path:\n%s", testAQLText(t, errorQuery.Command)) } + err := client.QueryRows(ctx, testAQLText(t, errorQuery.Command), 500, testAQLBinds(t, errorQuery.Command), func(map[string]any) error { return nil }) + requireStableRuntimeCode(t, err, test.wantErrorCode) }) } } @@ -543,12 +543,13 @@ func TestS04TableReshapeOracleAgainstArango(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Observation"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Observation"}}}); err != nil { t.Fatal(err) } @@ -612,8 +613,8 @@ func TestS04TableReshapeOracleAgainstArango(t *testing.T) { if !directIDFastPath { t.Fatal("direct root-ID pivot did not lower with its one-input-per-group proof") } - if strings.Contains(directIDQuery.Query, "COLLECT") || strings.Contains(directIDQuery.Query, "FOR __loom_physical_reshape_cell_") { - t.Fatalf("direct root-ID pivot retained group materialization or per-category scans:\n%s", directIDQuery.Query) + if strings.Contains(testAQLText(t, directIDQuery.Command), "COLLECT") || strings.Contains(testAQLText(t, directIDQuery.Command), "FOR __loom_physical_reshape_cell_") { + t.Fatalf("direct root-ID pivot retained group materialization or per-category scans:\n%s", testAQLText(t, directIDQuery.Command)) } directIDRows := executeReshapeOracleQuery(t, ctx, client, directIDQuery) if len(directIDRows) != len(reshapeOracleSourceRows) { @@ -735,12 +736,13 @@ func TestS04GroupedPivotMaterializationPreservesAuthResourcePathAgainstArango(t } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Observation"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Observation"}}}); err != nil { t.Fatal(err) } @@ -808,7 +810,7 @@ func TestS04GroupedPivotMaterializationPreservesAuthResourcePathAgainstArango(t t.Fatal(err) } compiledOutput := compiled.Outputs[0] - query, err := CompileRecipeOutputWithPolicy(compiledOutput, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiledOutput, bindings, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile auth-scoped grouped pivot: %v", err) } @@ -819,12 +821,12 @@ func TestS04GroupedPivotMaterializationPreservesAuthResourcePathAgainstArango(t } assertAuthScopedPivotRows(t, executeReshapeOracleQuery(t, ctx, client, query)) - page, err := CompileRecipeOutputPageWithPolicy(compiledOutput, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + page, err := CompileRecipeOutputPage(ArangoTarget(), compiledOutput, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile auth-scoped grouped pivot materialization page: %v", err) } rootKeys := make([]string, 0, len(fixtures)) - if err := client.QueryRows(ctx, page.RootKeysQuery, 500, page.RootKeysBindVars, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, page.RootKeys), 500, testAQLBinds(t, page.RootKeys), func(row map[string]any) error { key, ok := row["_key"].(string) if !ok || key == "" { return fmt.Errorf("root-key page returned invalid _key %v", row["_key"]) @@ -832,22 +834,22 @@ func TestS04GroupedPivotMaterializationPreservesAuthResourcePathAgainstArango(t rootKeys = append(rootKeys, key) return nil }); err != nil { - t.Fatalf("execute auth-scoped root-key page: %v\n%s", err, page.RootKeysQuery) + t.Fatalf("execute auth-scoped root-key page: %v\n%s", err, testAQLText(t, page.RootKeys)) } if len(rootKeys) != len(fixtures) { t.Fatalf("root-key page returned %d keys, want %d", len(rootKeys), len(fixtures)) } - rowBindVars := make(map[string]any, len(page.RowsBindVars)+1) - for key, value := range page.RowsBindVars { + rowBindVars := make(map[string]any, len(testAQLBinds(t, page.Rows))+1) + for key, value := range testAQLBinds(t, page.Rows) { rowBindVars[key] = value } rowBindVars[RootPageKeysBind] = rootKeys rows := make([]map[string]any, 0, 2) - if err := client.QueryRows(ctx, page.RowsQuery, 500, rowBindVars, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, page.Rows), 500, rowBindVars, func(row map[string]any) error { rows = append(rows, row) return nil }); err != nil { - t.Fatalf("execute auth-scoped grouped pivot materialization page: %v\n%s", err, page.RowsQuery) + t.Fatalf("execute auth-scoped grouped pivot materialization page: %v\n%s", err, testAQLText(t, page.Rows)) } assertAuthScopedPivotRows(t, rows) } @@ -892,7 +894,7 @@ func assertAuthScopedPivotRows(t *testing.T, rows []map[string]any) { } } -func assertReshapeOracleExclusions(t *testing.T, ctx context.Context, client *store.Client, project, generation string) { +func assertReshapeOracleExclusions(t *testing.T, ctx context.Context, client *arangofactory.Client, project, generation string) { t.Helper() fixtures := []struct { id string @@ -995,8 +997,8 @@ func assertReshapeOracleExclusions(t *testing.T, ctx context.Context, client *st missingNullFastPath = operation.GroupedPivot.OneInputRowPerGroup } } - if !missingNullFastPath || strings.Contains(missingNullQuery.Query, "COLLECT") { - t.Fatalf("missing/null oracle did not use direct-ID renderer:\n%s", missingNullQuery.Query) + if !missingNullFastPath || strings.Contains(testAQLText(t, missingNullQuery.Command), "COLLECT") { + t.Fatalf("missing/null oracle did not use direct-ID renderer:\n%s", testAQLText(t, missingNullQuery.Command)) } missingNullRows := executeReshapeOracleQuery(t, ctx, client, missingNullQuery) if len(missingNullRows) != 2 { @@ -1086,22 +1088,22 @@ func assertReshapeOracleExclusions(t *testing.T, ctx context.Context, client *st t.Logf("literal exclusions: string=%v; boolean false=%v; integer zero=%v; MISSING present=false vs NULL present=true", summarizeReshapeOracleExclusions(stringRows), summarizeReshapeOracleExclusions(booleanRows), summarizeReshapeOracleExclusions(zeroRows)) } -func runReshapeOracleExclusionQuery(t *testing.T, ctx context.Context, client *store.Client, project, generation string, output recipe.Output) []map[string]any { +func runReshapeOracleExclusionQuery(t *testing.T, ctx context.Context, client *arangofactory.Client, project, generation string, output recipe.Output) []map[string]any { t.Helper() compiled, _, err := compileReshapeOracle(output, project, generation, 100) if err != nil { t.Fatalf("compile exclusion oracle output: %v", err) } - query, err := CompileTableShapeExclusionsWithPolicy(compiled, 0, 100, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileTableShapeExclusions(ArangoTarget(), compiled, 0, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile exact table-shape exclusions: %v", err) } rows := make([]map[string]any, 0) - if err := client.QueryRows(ctx, query.Query, 500, query.BindVars, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, query.Command), 500, testAQLBinds(t, query.Command), func(row map[string]any) error { rows = append(rows, row) return nil }); err != nil { - t.Fatalf("execute exact table-shape exclusions: %v\n%s", err, query.Query) + t.Fatalf("execute exact table-shape exclusions: %v\n%s", err, testAQLText(t, query.Command)) } return rows } @@ -1203,21 +1205,21 @@ func compileReshapeOracle(output recipe.Output, project, generation string, limi if err != nil { return lower.CompiledRecipeOutput{}, CompiledQuery{}, err } - query, err := CompileRecipeOutputWithPolicy(compiled.Outputs[0], bindings, limit, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileRecipeOutput(ArangoTarget(), compiled.Outputs[0], bindings, limit, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { return lower.CompiledRecipeOutput{}, CompiledQuery{}, err } return compiled.Outputs[0], query, nil } -func executeReshapeOracleQuery(t *testing.T, ctx context.Context, client *store.Client, query CompiledQuery) []map[string]any { +func executeReshapeOracleQuery(t *testing.T, ctx context.Context, client *arangofactory.Client, query CompiledQuery) []map[string]any { t.Helper() rows := make([]map[string]any, 0) - if err := client.QueryRows(ctx, query.Query, 500, query.BindVars, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, query.Command), 500, testAQLBinds(t, query.Command), func(row map[string]any) error { rows = append(rows, row) return nil }); err != nil { - t.Fatalf("execute compiler-generated reshape query: %v\n%s", err, query.Query) + t.Fatalf("execute compiler-generated reshape query: %v\n%s", err, testAQLText(t, query.Command)) } return rows } @@ -1231,7 +1233,7 @@ func outputColumn(columns []lower.CompiledOutputColumn, name string) any { return nil } -func assertReshapeOracleRepeatedIdentities(t *testing.T, ctx context.Context, client *store.Client, query CompiledQuery, first []map[string]any) { +func assertReshapeOracleRepeatedIdentities(t *testing.T, ctx context.Context, client *arangofactory.Client, query CompiledQuery, first []map[string]any) { t.Helper() second := executeReshapeOracleQuery(t, ctx, client, query) firstIDs, secondIDs := reshapeOracleIdentities(first), reshapeOracleIdentities(second) @@ -1249,7 +1251,7 @@ func reshapeOracleIdentities(rows []map[string]any) []string { return identities } -func assertReshapeOraclePivotWindow(t *testing.T, ctx context.Context, client *store.Client, output recipe.Output, project, generation string) { +func assertReshapeOraclePivotWindow(t *testing.T, ctx context.Context, client *arangofactory.Client, output recipe.Output, project, generation string) { t.Helper() _, query, err := compileReshapeOracle(output, project, generation, 2) if err != nil { @@ -1274,7 +1276,7 @@ func assertReshapeOraclePivotWindow(t *testing.T, ctx context.Context, client *s } } -func assertReshapeOraclePivotReducers(t *testing.T, ctx context.Context, client *store.Client, project, generation string) { +func assertReshapeOraclePivotReducers(t *testing.T, ctx context.Context, client *arangofactory.Client, project, generation string) { t.Helper() tests := []struct { policy recipe.PivotDuplicatePolicy @@ -1336,7 +1338,7 @@ func assertReshapeOraclePivotReducers(t *testing.T, ctx context.Context, client } } -func assertReshapeOracleNumericCategories(t *testing.T, ctx context.Context, client *store.Client, project, generation string) { +func assertReshapeOracleNumericCategories(t *testing.T, ctx context.Context, client *arangofactory.Client, project, generation string) { t.Helper() output := reshapeOracleOutput("numeric_categories", &recipe.TableReshape{ Kind: recipe.TableReshapeGroupedPivot, @@ -1378,7 +1380,7 @@ func assertReshapeOracleNumericCategories(t *testing.T, ctx context.Context, cli } } -func assertReshapeOracleBooleanValues(t *testing.T, ctx context.Context, client *store.Client, project, generation string) { +func assertReshapeOracleBooleanValues(t *testing.T, ctx context.Context, client *arangofactory.Client, project, generation string) { t.Helper() output := reshapeOracleOutput("boolean_values", &recipe.TableReshape{ Kind: recipe.TableReshapeGroupedPivot, @@ -1420,7 +1422,7 @@ func assertReshapeOracleBooleanValues(t *testing.T, ctx context.Context, client } } -func assertReshapeOracleEmptyStringValue(t *testing.T, ctx context.Context, client *store.Client, project, generation string) { +func assertReshapeOracleEmptyStringValue(t *testing.T, ctx context.Context, client *arangofactory.Client, project, generation string) { t.Helper() output := reshapeOracleOutput("empty_string_value", &recipe.TableReshape{ Kind: recipe.TableReshapeGroupedPivot, @@ -1451,7 +1453,7 @@ func assertReshapeOracleEmptyStringValue(t *testing.T, ctx context.Context, clie } } -func assertReshapeOraclePivotErrors(t *testing.T, ctx context.Context, client *store.Client, project, generation string) { +func assertReshapeOraclePivotErrors(t *testing.T, ctx context.Context, client *arangofactory.Client, project, generation string) { t.Helper() tests := []struct { name string @@ -1500,15 +1502,13 @@ func assertReshapeOraclePivotErrors(t *testing.T, ctx context.Context, client *s if err != nil { t.Fatalf("compile error case: %v", err) } - _, err = queryReshapeOracle(ctx, client, query) - if err == nil || !strings.Contains(err.Error(), test.stableErr) { - t.Fatalf("runtime error = %v, want stable code %s", err, test.stableErr) - } + _, err = queryReshapeOracle(t, ctx, client, query) + requireStableRuntimeCode(t, err, test.stableErr) }) } } -func assertReshapeOracleUnpivot(t *testing.T, ctx context.Context, client *store.Client, project, generation string) { +func assertReshapeOracleUnpivot(t *testing.T, ctx context.Context, client *arangofactory.Client, project, generation string) { t.Helper() output := reshapeOracleUnpivotOutput(recipe.UnpivotNullPreserve, "s04-unpivot") output.Filters = []recipe.Filter{{Select: "root.id", Operator: recipe.FilterIn, Values: []recipe.FilterValue{reshapeOracleStringFilter("alpha-a"), reshapeOracleStringFilter("empty-and-null")}}} @@ -1580,7 +1580,7 @@ func assertReshapeOracleUnpivot(t *testing.T, ctx context.Context, client *store if err != nil { t.Fatalf("render unwindowed compiler plan: %v", err) } - rawRows, err := queryReshapeOracle(ctx, client, CompiledQuery{Query: rendered.Query, BindVars: rendered.BindVars}) + rawRows, err := queryReshapeOracle(t, ctx, client, CompiledQuery{Command: testAQLCommand(rendered.Query, rendered.BindVars)}) if err != nil { t.Fatalf("execute unwindowed compiler plan: %v\n%s", err, rendered.Query) } @@ -1598,7 +1598,7 @@ func assertReshapeOracleUnpivot(t *testing.T, ctx context.Context, client *store } } -func assertReshapeOracleUnpivotCellTrace(t *testing.T, ctx context.Context, client *store.Client, output lower.CompiledRecipeOutput, project string) { +func assertReshapeOracleUnpivotCellTrace(t *testing.T, ctx context.Context, client *arangofactory.Client, output lower.CompiledRecipeOutput, project string) { t.Helper() _, amount := executeReshapeOracleTrace(t, ctx, client, output, "amount") alphaID := reshapeOracleUnpivotID(project, "s04-unpivot", "alpha-a", "a-decimal") @@ -1686,18 +1686,18 @@ func assertReshapeOracleUnpivotRejectsIncompatibleInputs(t *testing.T, project, } } -func queryReshapeOracle(ctx context.Context, client *store.Client, query CompiledQuery) ([]map[string]any, error) { +func queryReshapeOracle(t *testing.T, ctx context.Context, client *arangofactory.Client, query CompiledQuery) ([]map[string]any, error) { rows := make([]map[string]any, 0) - err := client.QueryRows(ctx, query.Query, 500, query.BindVars, func(row map[string]any) error { + err := client.QueryRows(ctx, testAQLText(t, query.Command), 500, testAQLBinds(t, query.Command), func(row map[string]any) error { rows = append(rows, row) return nil }) return rows, err } -func executeReshapeOracleTrace(t *testing.T, ctx context.Context, client *store.Client, output lower.CompiledRecipeOutput, column string) (CompiledCellTraceQuery, map[string]operatorOracleTraceResult) { +func executeReshapeOracleTrace(t *testing.T, ctx context.Context, client *arangofactory.Client, output lower.CompiledRecipeOutput, column string) (CompiledCellTraceQuery, map[string]operatorOracleTraceResult) { t.Helper() - trace, err := CompileCellTraceOutputWithPolicy(output, column, 0, 100, ir.DefaultPhysicalOptimizationPolicy()) + trace, err := CompileCellTraceOutput(ArangoTarget(), output, column, 0, 100, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile %s cell trace: %v", column, err) } @@ -1705,11 +1705,11 @@ func executeReshapeOracleTrace(t *testing.T, ctx context.Context, client *store. t.Fatalf("reshaped %s trace has no explicit output-row identity", column) } rows := make([]map[string]any, 0) - if err := client.QueryRows(ctx, trace.Query, 500, trace.BindVars, func(row map[string]any) error { + if err := client.QueryRows(ctx, testAQLText(t, trace.Command), 500, testAQLBinds(t, trace.Command), func(row map[string]any) error { rows = append(rows, row) return nil }); err != nil { - t.Fatalf("execute %s cell trace: %v\n%s", column, err, trace.Query) + t.Fatalf("execute %s cell trace: %v\n%s", column, err, testAQLText(t, trace.Command)) } results := make(map[string]operatorOracleTraceResult, len(rows)) for _, row := range rows { @@ -1753,7 +1753,7 @@ func reshapeOracleContributors(values ...any) []operatorOracleContributor { return contributors } -func assertReshapeOraclePivotCellTrace(t *testing.T, ctx context.Context, client *store.Client, output lower.CompiledRecipeOutput) { +func assertReshapeOraclePivotCellTrace(t *testing.T, ctx context.Context, client *arangofactory.Client, output lower.CompiledRecipeOutput) { t.Helper() rowID := reshapeOraclePivotSumWant[0].RowID _, alpha := executeReshapeOracleTrace(t, ctx, client, output, "alpha") @@ -1766,7 +1766,7 @@ func assertReshapeOraclePivotCellTrace(t *testing.T, ctx context.Context, client assertReshapeTraceResult(t, group[rowID], "final", "VALUE", reshapeOracleContributors("alpha-a", "final", "alpha-b", "final", "beta", "final", "unlisted", "final", "zero", "final"), "") } -func assertReshapeOraclePivotReducerTrace(t *testing.T, ctx context.Context, client *store.Client, output recipe.Output, project, generation string, policy recipe.PivotDuplicatePolicy) { +func assertReshapeOraclePivotReducerTrace(t *testing.T, ctx context.Context, client *arangofactory.Client, output recipe.Output, project, generation string, policy recipe.PivotDuplicatePolicy) { t.Helper() compiled, _, err := compileReshapeOracle(output, project, generation, 100) if err != nil { @@ -1789,7 +1789,7 @@ func assertReshapeOraclePivotReducerTrace(t *testing.T, ctx context.Context, cli assertReshapeTraceResult(t, cells[rowID], value, "VALUE", contributors, "") } -func assertReshapeOraclePivotErrorTrace(t *testing.T, ctx context.Context, client *store.Client, project, generation string) { +func assertReshapeOraclePivotErrorTrace(t *testing.T, ctx context.Context, client *arangofactory.Client, project, generation string) { t.Helper() output := reshapeOracleOutput("pivot_error_trace", &recipe.TableReshape{ Kind: recipe.TableReshapeGroupedPivot, diff --git a/internal/dataframe/compiler/table_shape_exclusions.go b/internal/dataframe/compiler/table_shape_exclusions.go index 0e4ccc506..861e42ca6 100644 --- a/internal/dataframe/compiler/table_shape_exclusions.go +++ b/internal/dataframe/compiler/table_shape_exclusions.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/compiler/render/aql" @@ -29,12 +30,8 @@ type TableShapeExclusionRefusal struct { func (e *TableShapeExclusionRefusal) Error() string { return string(e.Code) } -// CompiledTableShapeExclusionQuery is an exact, bounded diagnostic query -// derived from one finalized output plan. It has no caller-supplied paths or -// query fragments. type CompiledTableShapeExclusionQuery struct { - Query string - BindVars map[string]any + Command command.CompiledCommand ResourceTypeField string ResourceIDField string IdentityStatusField string @@ -49,7 +46,10 @@ type CompiledTableShapeExclusionQuery struct { Diagnostics ir.CompilerPlanDiagnostics } -func CompileTableShapeExclusionsWithPolicy(output lower.CompiledRecipeOutput, offset, limit int, policy ir.PhysicalOptimizationPolicy) (CompiledTableShapeExclusionQuery, error) { +func CompileTableShapeExclusions(target Target, output lower.CompiledRecipeOutput, offset, limit int, policy ir.PhysicalOptimizationPolicy) (CompiledTableShapeExclusionQuery, error) { + if err := target.validate(); err != nil { + return CompiledTableShapeExclusionQuery{}, err + } if offset < 0 { return CompiledTableShapeExclusionQuery{}, &TableShapeExclusionRefusal{Code: TableShapeExclusionInvalidOffset} } @@ -60,7 +60,7 @@ func CompileTableShapeExclusionsWithPolicy(output lower.CompiledRecipeOutput, of return CompiledTableShapeExclusionQuery{}, &TableShapeExclusionRefusal{Code: TableShapeExclusionInvalidLimit} } - _ = policy // Diagnostics execute against the exact canonical plan, before publication optimizations. + _ = policy physical, err := withGenericPhysicalExecutionWindow(ir.ClonePhysicalPlan(output.Plan), 0) if err != nil { return CompiledTableShapeExclusionQuery{}, fmt.Errorf("prepare table-shape exclusion plan: %w", err) @@ -86,9 +86,6 @@ func CompileTableShapeExclusionsWithPolicy(output lower.CompiledRecipeOutput, of return CompiledTableShapeExclusionQuery{}, &TableShapeExclusionRefusal{Code: TableShapeExclusionNoPolicy} } - // Ordinary Pivot previews omit presence metadata to retain bounded execution. - // Diagnostics must recover missing versus explicit null from the exact direct - // root projection, rather than HAS on its always-populated projected column. if pivot.CategoryPresence == nil && !pivot.CategoryPresenceFromInput && physical.StageSequence == nil && len(physical.Operations) > 0 && physical.Operations[0].RootScan != nil { root := physical.Operations[0].RootScan.Variable @@ -131,15 +128,32 @@ func CompileTableShapeExclusionsWithPolicy(output lower.CompiledRecipeOutput, of if err := ir.ValidateGenericPhysicalPlanScope(physical); err != nil { return CompiledTableShapeExclusionQuery{}, fmt.Errorf("verify table-shape exclusion scope: %w", err) } - rendered, err := aql.RenderPhysicalPlan(physical) - if err != nil { - return CompiledTableShapeExclusionQuery{}, fmt.Errorf("render table-shape exclusion plan: %w", err) - } - if strings.TrimSpace(output.Name) == "" { - return CompiledTableShapeExclusionQuery{}, fmt.Errorf("table-shape exclusion output name is required") + var compiledCommand command.CompiledCommand + if target.backend == command.CompiledBackendPostgreSQL { + postgresCommand, _, postgresErr := target.renderPostgreSQLFinalPlan(physical) + if postgresErr != nil { + return CompiledTableShapeExclusionQuery{}, postgresErr + } + if strings.TrimSpace(output.Name) == "" { + return CompiledTableShapeExclusionQuery{}, fmt.Errorf("table-shape exclusion output name is required") + } + compiledCommand = postgresCommand + } else { + rendered, err := aql.RenderPhysicalPlan(physical) + if err != nil { + return CompiledTableShapeExclusionQuery{}, fmt.Errorf("render table-shape exclusion plan: %w", err) + } + if strings.TrimSpace(output.Name) == "" { + return CompiledTableShapeExclusionQuery{}, fmt.Errorf("table-shape exclusion output name is required") + } + aqlCommand, commandErr := command.NewAQLCommand(rendered.Query, rendered.BindVars) + if commandErr != nil { + return CompiledTableShapeExclusionQuery{}, fmt.Errorf("tag table-shape exclusion command: %w", commandErr) + } + compiledCommand = aqlCommand } return CompiledTableShapeExclusionQuery{ - Query: rendered.Query, BindVars: rendered.BindVars, + Command: compiledCommand, ResourceTypeField: ir.PhysicalTableShapeExclusionResourceTypeField, ResourceIDField: ir.PhysicalTableShapeExclusionResourceIDField, IdentityStatusField: ir.PhysicalTableShapeExclusionIdentityStatusField, diff --git a/internal/dataframe/compiler/table_shape_exclusions_test.go b/internal/dataframe/compiler/table_shape_exclusions_test.go index 76b3ef8dd..9de1ab09f 100644 --- a/internal/dataframe/compiler/table_shape_exclusions_test.go +++ b/internal/dataframe/compiler/table_shape_exclusions_test.go @@ -6,8 +6,10 @@ import ( "strings" "testing" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" + renderpostgres "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" "github.com/calypr/loom/internal/dataframe/recipe" ) @@ -33,7 +35,7 @@ func TestCompileTableShapeExclusionsUsesCanonicalGroupedPivotInputs(t *testing.T t.Fatal(err) } originalPlan := ir.ClonePhysicalPlan(compiledOutput.Plan) - query, err := CompileTableShapeExclusionsWithPolicy(compiledOutput, 7, 12, ir.DefaultPhysicalOptimizationPolicy()) + query, err := CompileTableShapeExclusions(ArangoTarget(), compiledOutput, 7, 12, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -45,24 +47,24 @@ func TestCompileTableShapeExclusionsUsesCanonicalGroupedPivotInputs(t *testing.T "LIMIT @table_shape_exclusion_offset, @table_shape_exclusion_fetch_limit", "__loom_physical_exclusion_output_row_id", "@__loom_physical_exclusion_category_column", } { - if !strings.Contains(query.Query, want) { - t.Fatalf("exclusion query missing %q:\n%s", want, query.Query) + if !strings.Contains(testAQLText(t, query.Command), want) { + t.Fatalf("exclusion query missing %q:\n%s", want, testAQLText(t, query.Command)) } } - if strings.Contains(query.Query, "derived-after-pivot") || strings.Contains(query.Query, "LET after =") { - t.Fatalf("diagnostic terminal evaluated work after the grouped pivot:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), "derived-after-pivot") || strings.Contains(testAQLText(t, query.Command), "LET after =") { + t.Fatalf("diagnostic terminal evaluated work after the grouped pivot:\n%s", testAQLText(t, query.Command)) } - if query.Offset != 7 || query.Limit != 12 || query.BindVars["table_shape_exclusion_offset"] != 7 || query.BindVars["table_shape_exclusion_fetch_limit"] != 13 { - t.Fatalf("unexpected bounded page metadata: %#v %#v", query, query.BindVars) + if query.Offset != 7 || query.Limit != 12 || testAQLBinds(t, query.Command)["table_shape_exclusion_offset"] != 7 || testAQLBinds(t, query.Command)["table_shape_exclusion_fetch_limit"] != 13 { + t.Fatalf("unexpected bounded page metadata: %#v %#v", query, testAQLBinds(t, query.Command)) } - if !containsBindValue(query.BindVars, ir.PhysicalCellTraceSourceDocumentField) || !containsBindValue(query.BindVars, "string_category") || !containsBindValue(query.BindVars, "known") || !containsBindValue(query.BindVars, ir.PhysicalTableShapeExclusionCategoryPresentField) || !containsBindValue(query.BindVars, ir.PhysicalTableShapeExclusionReasonUnlistedCategory) { - t.Fatalf("query does not bind exact source document and canonical category identity: %#v", query.BindVars) + if !containsBindValue(testAQLBinds(t, query.Command), ir.PhysicalCellTraceSourceDocumentField) || !containsBindValue(testAQLBinds(t, query.Command), "string_category") || !containsBindValue(testAQLBinds(t, query.Command), "known") || !containsBindValue(testAQLBinds(t, query.Command), ir.PhysicalTableShapeExclusionCategoryPresentField) || !containsBindValue(testAQLBinds(t, query.Command), ir.PhysicalTableShapeExclusionReasonUnlistedCategory) { + t.Fatalf("query does not bind exact source document and canonical category identity: %#v", testAQLBinds(t, query.Command)) } - if !strings.Contains(query.Query, "HAS(") || !containsBindValue(query.BindVars, "__loom_exclusion_category_present") { - t.Fatalf("exclusion diagnostic lost source presence for ordinary value categories: %s", query.Query) + if !strings.Contains(testAQLText(t, query.Command), "HAS(") || !containsBindValue(testAQLBinds(t, query.Command), "__loom_exclusion_category_present") { + t.Fatalf("exclusion diagnostic lost source presence for ordinary value categories: %s", testAQLText(t, query.Command)) } - if strings.Contains(query.Query, `"known"`) { - t.Fatalf("category value was interpolated rather than retained as a typed canonical bind:\n%s", query.Query) + if strings.Contains(testAQLText(t, query.Command), `"known"`) { + t.Fatalf("category value was interpolated rather than retained as a typed canonical bind:\n%s", testAQLText(t, query.Command)) } } @@ -75,7 +77,7 @@ func TestCompileTableShapeExclusionsRefusesUnsupportedOrUnconfiguredOutputs(t *t KeyOutput: "key", KeyLabel: "Key", ValueOutput: "value", ValueLabel: "Value", NullRowPolicy: recipe.UnpivotNullPreserve, }}, }) - _, err := CompileTableShapeExclusionsWithPolicy(unsupported, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + _, err := CompileTableShapeExclusions(ArangoTarget(), unsupported, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) assertExclusionRefusal(t, err, TableShapeExclusionUnsupported) noPolicy := lowerOutputForExclusionTest(t, recipe.Output{ @@ -88,12 +90,12 @@ func TestCompileTableShapeExclusionsRefusesUnsupportedOrUnconfiguredOutputs(t *t UnlistedCategoryPolicy: recipe.PivotUnlistedCategoryError, }}, }) - _, err = CompileTableShapeExclusionsWithPolicy(noPolicy, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + _, err = CompileTableShapeExclusions(ArangoTarget(), noPolicy, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) assertExclusionRefusal(t, err, TableShapeExclusionNoPolicy) - _, err = CompileTableShapeExclusionsWithPolicy(noPolicy, -1, 10, ir.DefaultPhysicalOptimizationPolicy()) + _, err = CompileTableShapeExclusions(ArangoTarget(), noPolicy, -1, 10, ir.DefaultPhysicalOptimizationPolicy()) assertExclusionRefusal(t, err, TableShapeExclusionInvalidOffset) - _, err = CompileTableShapeExclusionsWithPolicy(noPolicy, 0, MaxTableShapeExclusionLimit+1, ir.DefaultPhysicalOptimizationPolicy()) + _, err = CompileTableShapeExclusions(ArangoTarget(), noPolicy, 0, MaxTableShapeExclusionLimit+1, ir.DefaultPhysicalOptimizationPolicy()) assertExclusionRefusal(t, err, TableShapeExclusionInvalidLimit) } @@ -113,3 +115,126 @@ func assertExclusionRefusal(t *testing.T, err error, want TableShapeExclusionRef t.Fatalf("refusal = %#v, want %s", err, want) } } + +func exclusionPostgreSQLTarget(t *testing.T) Target { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("loom") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + target, err := PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("PostgreSQLTarget: %v", err) + } + return target +} + +func exclusionCanonicalPGOutput(t *testing.T) lower.CompiledRecipeOutput { + t.Helper() + output := reshapeOracleOutput("Observations", &recipe.TableReshape{ + Kind: recipe.TableReshapeGroupedPivot, + GroupedPivot: &recipe.GroupedPivot{ + ConstructionID: "exclusion-pg", GroupKeys: []string{"group_text", "group_number"}, + CategoryColumn: "string_category", ValueColumn: "numeric_value", + Categories: []recipe.GroupedPivotCategory{{Key: reshapeOracleString("known"), Output: "known", Label: "Known"}}, + DuplicatePolicy: recipe.PivotDuplicateSum, MissingCellPolicy: recipe.PivotMissingCellNull, + UnlistedCategoryPolicy: recipe.PivotUnlistedCategoryExcludeWithEvidence, + }, + }) + compiled, _, err := compileReshapeOracle(output, "project", "generation", 10) + if err != nil { + t.Fatal(err) + } + return compiled +} + +func TestCompileTableShapeExclusionsPostgreSQLTargetRendersTheTerminal(t *testing.T) { + compiled := exclusionCanonicalPGOutput(t) + originalPlan := ir.ClonePhysicalPlan(compiled.Plan) + query, err := CompileTableShapeExclusions(exclusionPostgreSQLTarget(t), compiled, 7, 12, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("postgresql exclusion compile: %v", err) + } + if !reflect.DeepEqual(compiled.Plan, originalPlan) { + t.Fatal("postgresql exclusion compilation mutated the canonical published plan") + } + if query.Command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("selected PostgreSQL target produced %q", query.Command.Backend()) + } + if _, _, err := query.Command.AQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("the AQL adapter must refuse the PostgreSQL evidence command: %v", err) + } + sql, args, err := query.Command.PostgreSQLParameters() + if err != nil { + t.Fatalf("postgresql parameters: %v", err) + } + for _, want := range []string{ + "WITH px_src AS", `COLLATE "C" ASC`, "NOT COALESCE(", "OFFSET $", " LIMIT $", + } { + if !strings.Contains(sql, want) { + t.Fatalf("evidence statement missing %q:\n%s", want, sql) + } + } + if len(args) < 2 { + t.Fatalf("statement carries only %d parameters", len(args)) + } + if offset, ok := args[len(args)-2].Value.(int64); !ok || offset != 7 { + t.Fatalf("offset parameter = %#v, want int64 7", args[len(args)-2]) + } + if fetch, ok := args[len(args)-1].Value.(int64); !ok || fetch != 13 { + t.Fatalf("fetch-limit parameter = %#v, want int64 13", args[len(args)-1]) + } + aqlQuery, err := CompileTableShapeExclusions(ArangoTarget(), compiled, 7, 12, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("aql control compile: %v", err) + } + aqlText, _, err := aqlQuery.Command.AQLParameters() + if err != nil { + t.Fatalf("aql control parameters: %v", err) + } + if aqlQuery.Command.Backend() != command.CompiledBackendAQL || !strings.Contains(aqlText, "FILTER NOT (") || + !strings.Contains(aqlText, "LIMIT @table_shape_exclusion_offset, @table_shape_exclusion_fetch_limit") { + t.Fatalf("aql control drifted from canonical AQL rendering:\n%s", aqlText) + } +} + +func TestCompileTableShapeExclusionsPostgreSQLTargetRefusalsStayTyped(t *testing.T) { + unsupported := lowerOutputForExclusionTest(t, recipe.Output{ + Name: "UnpivotPG", RootResourceType: "Observation", RowGrain: "observation", + Fields: []recipe.Field{{Name: "id", Expr: recipe.Expression{Select: "root.id"}}}, + TableReshape: &recipe.TableReshape{Kind: recipe.TableReshapeUnpivot, Unpivot: &recipe.Unpivot{ + ConstructionID: "u", Inputs: []recipe.UnpivotInput{{Column: "id", Key: reshapeOracleString("id")}}, + KeyOutput: "key", KeyLabel: "Key", ValueOutput: "value", ValueLabel: "Value", NullRowPolicy: recipe.UnpivotNullPreserve, + }}, + }) + _, err := CompileTableShapeExclusions(exclusionPostgreSQLTarget(t), unsupported, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + assertExclusionRefusal(t, err, TableShapeExclusionUnsupported) + + filtered := reshapeOracleOutput("ObservationsFiltered", &recipe.TableReshape{ + Kind: recipe.TableReshapeGroupedPivot, + GroupedPivot: &recipe.GroupedPivot{ + ConstructionID: "exclusion-pg-filter", GroupKeys: []string{"group_text"}, + CategoryColumn: "string_category", ValueColumn: "numeric_value", + Categories: []recipe.GroupedPivotCategory{{Key: reshapeOracleString("known"), Output: "known", Label: "Known"}}, + DuplicatePolicy: recipe.PivotDuplicateSum, MissingCellPolicy: recipe.PivotMissingCellNull, + UnlistedCategoryPolicy: recipe.PivotUnlistedCategoryExcludeWithEvidence, + }, + }) + filterStatus := "final" + filtered.Filters = append(filtered.Filters, recipe.Filter{ + Select: "root.status", Operator: recipe.FilterEquals, + Values: []recipe.FilterValue{{Kind: recipe.FilterString, String: &filterStatus}}, + }) + compiled, _, err := compileReshapeOracle(filtered, "project", "generation", 10) + if err != nil { + t.Fatalf("compile filtered output: %v", err) + } + aqlQuery, aqlErr := CompileTableShapeExclusions(ArangoTarget(), compiled, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + if aqlErr != nil || aqlQuery.Command.Backend() != command.CompiledBackendAQL { + t.Fatalf("the filtered shape is registered at AQL and must render: %v", aqlErr) + } + _, err = CompileTableShapeExclusions(exclusionPostgreSQLTarget(t), compiled, 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + if err == nil || !errors.Is(err, renderpostgres.ErrUnsupportedPostgresPlan) { + t.Fatalf("outside-slice shape must fail closed with the typed renderer error, got %v", err) + } +} diff --git a/internal/dataframe/compiler/target.go b/internal/dataframe/compiler/target.go new file mode 100644 index 000000000..e8fc2c581 --- /dev/null +++ b/internal/dataframe/compiler/target.go @@ -0,0 +1,107 @@ +package compiler + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + postgresrender "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" +) + +type Target struct { + backend command.CompiledBackend + postgres ir.PostgresStorageBinding + selected bool +} + +func ArangoTarget() Target { + return Target{backend: command.CompiledBackendAQL, selected: true} +} + +func PostgreSQLTarget(binding ir.PostgresStorageBinding) (Target, error) { + if err := binding.Validate(); err != nil { + return Target{}, fmt.Errorf("postgres compile target: %w", err) + } + return Target{backend: command.CompiledBackendPostgreSQL, postgres: binding, selected: true}, nil +} + +func (t Target) Valid() bool { return t.selected } + +func (t Target) Backend() command.CompiledBackend { return t.backend } + +var ErrTargetNotSelected = fmt.Errorf("compile target is not selected: composition must pass compiler.ArangoTarget or compiler.PostgreSQLTarget") + +func (t Target) validate() error { + if !t.selected { + return ErrTargetNotSelected + } + switch t.backend { + case command.CompiledBackendAQL, command.CompiledBackendPostgreSQL: + return nil + default: + return fmt.Errorf("%w: unknown compile target backend %q", ErrTargetNotSelected, string(t.backend)) + } +} + +func (t Target) rejectPostgreSQL(surface string) error { + if t.backend == command.CompiledBackendPostgreSQL { + return fmt.Errorf("%s is unsupported on the PostgreSQL compile target: %w", surface, postgresrender.ErrUnsupportedPostgresPlan) + } + return nil +} + +func (t Target) renderPostgreSQLFinalPlan(plan ir.PhysicalPlan) (command.CompiledCommand, command.RenderedPostgresPhysicalPlan, error) { + if err := t.validate(); err != nil { + return command.CompiledCommand{}, command.RenderedPostgresPhysicalPlan{}, err + } + if t.backend != command.CompiledBackendPostgreSQL { + return command.CompiledCommand{}, command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("target %s cannot render a PostgreSQL plan", string(t.backend)) + } + rendered, err := postgresrender.RenderPhysicalPlan(plan, t.postgres) + if err != nil { + return command.CompiledCommand{}, command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("render postgresql physical plan: %w", err) + } + compiledCommand, err := command.NewPostgreSQLCommand(rendered.SQL, rendered.Args) + if err != nil { + return command.CompiledCommand{}, command.RenderedPostgresPhysicalPlan{}, fmt.Errorf("tag postgresql command: %w", err) + } + return compiledCommand, rendered, nil +} + +func postgreSQLPlanFingerprint(sqlText string, args []command.PostgreSQLParameter, columns []string) string { + payload, err := json.Marshal(struct { + SQL string + Args []command.PostgreSQLParameter + Columns []string + }{SQL: sqlText, Args: args, Columns: columns}) + if err != nil { + return "" + } + digest := sha256.Sum256(payload) + return hex.EncodeToString(digest[:]) +} + +func postgreSQLPublicColumns(schema []lower.CompiledOutputColumn, declared []string) []string { + if len(schema) == 0 { + return append([]string(nil), declared...) + } + declaredSet := make(map[string]struct{}, len(declared)) + for _, column := range declared { + declaredSet[column] = struct{}{} + } + public := make([]string, 0, len(declared)) + for _, column := range schema { + if column.Internal { + continue + } + if _, ok := declaredSet[column.Name]; !ok { + continue + } + public = append(public, column.Name) + } + return public +} diff --git a/internal/dataframe/compiler/testdata/clickhouse_prefix_aql.binds.json b/internal/dataframe/compiler/testdata/clickhouse_prefix_aql.binds.json new file mode 100644 index 000000000..5ed919579 --- /dev/null +++ b/internal/dataframe/compiler/testdata/clickhouse_prefix_aql.binds.json @@ -0,0 +1,72 @@ +{ + "@root_collection": "Patient", + "__loom_physical_construction_projection_0_name": "group", + "__loom_physical_construction_projection_0_name_1": "group", + "__loom_physical_construction_projection_0_name_2": "group", + "__loom_physical_construction_projection_0_name_3": "group", + "__loom_physical_construction_projection_0_name_4": "group", + "__loom_physical_construction_projection_0_name_5": "group", + "__loom_physical_construction_projection_0_name_6": "group", + "__loom_physical_construction_projection_1_name": "category", + "__loom_physical_construction_projection_1_name_1": "amount_a", + "__loom_physical_construction_projection_1_name_2": "amount_a", + "__loom_physical_construction_projection_1_name_3": "amount_a", + "__loom_physical_construction_projection_1_name_4": "amount_a", + "__loom_physical_construction_projection_1_name_5": "total", + "__loom_physical_construction_projection_1_name_6": "total", + "__loom_physical_construction_projection_2_name": "amount", + "__loom_physical_construction_projection_2_name_1": "amount_b", + "__loom_physical_construction_projection_2_name_2": "amount_b", + "__loom_physical_construction_projection_2_name_3": "amount_b", + "__loom_physical_construction_projection_2_name_4": "amount_b", + "__loom_physical_construction_projection_2_name_5": "measure", + "__loom_physical_construction_projection_2_name_6": "measure", + "__loom_physical_construction_projection_3_name": "_key", + "__loom_physical_construction_projection_3_name_1": "__loom_row_id", + "__loom_physical_construction_projection_3_name_2": "total", + "__loom_physical_construction_projection_3_name_3": "total", + "__loom_physical_construction_projection_3_name_4": "total", + "__loom_physical_construction_projection_3_name_5": "amount", + "__loom_physical_construction_projection_3_name_6": "amount", + "__loom_physical_construction_projection_4_name": "__loom_presence_c4239a7fe09dfa1e", + "__loom_physical_construction_projection_4_name_1": "__loom_row_id", + "__loom_physical_construction_projection_4_name_2": "__loom_row_id", + "__loom_physical_construction_projection_4_name_3": "__loom_row_id", + "__loom_physical_construction_projection_4_name_4": "__loom_row_id", + "__loom_physical_construction_projection_4_name_5": "__loom_row_id", + "__loom_physical_construction_projection_5_name": "auth_resource_path", + "__loom_physical_construction_reshape_category_column": "category", + "__loom_physical_construction_reshape_category_type": "number", + "__loom_physical_construction_reshape_group_column_0": "group", + "__loom_physical_construction_reshape_identity_name_0": "__loom_row_id", + "__loom_physical_construction_reshape_unpivot_column_0": "amount_a", + "__loom_physical_construction_reshape_unpivot_column_1": "amount_b", + "__loom_physical_construction_reshape_unpivot_preserved_column": "group", + "__loom_physical_construction_reshape_unpivot_preserved_column_1": "total", + "__loom_physical_construction_reshape_unpivot_value_type": "number", + "__loom_physical_construction_reshape_value_column": "amount", + "__loom_physical_construction_reshape_value_type": "number", + "__loom_physical_projection_0_name": "_key", + "__loom_physical_projection_1_name": "group", + "__loom_physical_projection_2_name": "category", + "__loom_physical_projection_3_name": "amount", + "__loom_physical_projection_4_name": "__loom_presence_c4239a7fe09dfa1e", + "__loom_physical_reshape_presence_path": "multipleBirthInteger", + "auth_resource_paths": [ + "/programs/p1" + ], + "auth_resource_paths_unrestricted": false, + "construction_filter_value": 5, + "construction_private_auth_resource_path": "/programs/p1", + "dataset_generation": "generation", + "derived_literal_0": 1, + "derived_null_0": null, + "project": "project", + "reshape_category_0": 0, + "reshape_category_1": 1, + "reshape_construction": "pivot_values", + "reshape_construction_1": "unpivot_values", + "reshape_unpivot_key_0": "a", + "reshape_unpivot_key_1": "b", + "scope_allowed": true +} \ No newline at end of file diff --git a/internal/dataframe/compiler/testdata/clickhouse_prefix_aql.query b/internal/dataframe/compiler/testdata/clickhouse_prefix_aql.query new file mode 100644 index 000000000..504fe1bd9 --- /dev/null +++ b/internal/dataframe/compiler/testdata/clickhouse_prefix_aql.query @@ -0,0 +1,50 @@ +LET __loom_construction_source_projection = ( + FOR root IN @@root_collection + FILTER root.project == @project + FILTER root.dataset_generation == @dataset_generation + LET root_scope_allowed = @auth_resource_paths_unrestricted == true OR root.auth_resource_path IN @auth_resource_paths + FILTER root_scope_allowed == @scope_allowed + RETURN { [@__loom_physical_projection_0_name]: root._key, [@__loom_physical_projection_1_name]: root.payload.gender, [@__loom_physical_projection_2_name]: root.payload.multipleBirthInteger, [@__loom_physical_projection_3_name]: root.payload.multipleBirthInteger, [@__loom_physical_projection_4_name]: (IS_OBJECT(root.payload) ? HAS(root.payload, @__loom_physical_reshape_presence_path) : false) } +) +LET __loom_construction_stage_1 = ( + FOR __loom_construction_input_1 IN __loom_construction_source_projection + LET __loom_reshape_pivot_input_0 = { [@__loom_physical_construction_projection_0_name]: __loom_construction_input_1.group, [@__loom_physical_construction_projection_1_name]: __loom_construction_input_1.category, [@__loom_physical_construction_projection_2_name]: __loom_construction_input_1.amount, [@__loom_physical_construction_projection_3_name]: __loom_construction_input_1._key, [@__loom_physical_construction_projection_4_name]: __loom_construction_input_1.__loom_presence_c4239a7fe09dfa1e } + COLLECT __loom_reshape_pivot_group_0 = __loom_reshape_pivot_input_0[@__loom_physical_construction_reshape_group_column_0] INTO __loom_reshape_pivot_group_rows_0 = __loom_reshape_pivot_input_0 + SORT __loom_reshape_pivot_group_0 ASC + LET __loom_physical_construction_reshape_values_0 = (FOR __loom_physical_construction_reshape_cell_0 IN __loom_reshape_pivot_group_rows_0 FILTER (HAS(__loom_physical_construction_reshape_cell_0, @__loom_physical_construction_reshape_category_column) AND TYPENAME(__loom_physical_construction_reshape_cell_0[@__loom_physical_construction_reshape_category_column]) == @__loom_physical_construction_reshape_category_type AND __loom_physical_construction_reshape_cell_0[@__loom_physical_construction_reshape_category_column] == @reshape_category_0) RETURN __loom_physical_construction_reshape_cell_0[@__loom_physical_construction_reshape_value_column]) + LET __loom_physical_construction_reshape_typed_values_0 = (FOR __loom_physical_construction_reshape_cell_value_0 IN __loom_physical_construction_reshape_values_0 FILTER __loom_physical_construction_reshape_cell_value_0 != null FILTER ASSERT(TYPENAME(__loom_physical_construction_reshape_cell_value_0) == @__loom_physical_construction_reshape_value_type, "TABLE_PIVOT_VALUE_TYPE_MISMATCH") RETURN __loom_physical_construction_reshape_cell_value_0) + LET __loom_physical_construction_reshape_cell_result_0 = (ASSERT(true, "TABLE_PIVOT_CELL_MISSING") ? (LENGTH(__loom_physical_construction_reshape_typed_values_0) == 0 ? null : SUM(__loom_physical_construction_reshape_typed_values_0)) : null) + LET __loom_physical_construction_reshape_values_1 = (FOR __loom_physical_construction_reshape_cell_1 IN __loom_reshape_pivot_group_rows_0 FILTER (HAS(__loom_physical_construction_reshape_cell_1, @__loom_physical_construction_reshape_category_column) AND TYPENAME(__loom_physical_construction_reshape_cell_1[@__loom_physical_construction_reshape_category_column]) == @__loom_physical_construction_reshape_category_type AND __loom_physical_construction_reshape_cell_1[@__loom_physical_construction_reshape_category_column] == @reshape_category_1) RETURN __loom_physical_construction_reshape_cell_1[@__loom_physical_construction_reshape_value_column]) + LET __loom_physical_construction_reshape_typed_values_1 = (FOR __loom_physical_construction_reshape_cell_value_1 IN __loom_physical_construction_reshape_values_1 FILTER __loom_physical_construction_reshape_cell_value_1 != null FILTER ASSERT(TYPENAME(__loom_physical_construction_reshape_cell_value_1) == @__loom_physical_construction_reshape_value_type, "TABLE_PIVOT_VALUE_TYPE_MISMATCH") RETURN __loom_physical_construction_reshape_cell_value_1) + LET __loom_physical_construction_reshape_cell_result_1 = (ASSERT(true, "TABLE_PIVOT_CELL_MISSING") ? (LENGTH(__loom_physical_construction_reshape_typed_values_1) == 0 ? null : SUM(__loom_physical_construction_reshape_typed_values_1)) : null) + LET __loom_physical_construction_reshape_unlisted_count = LENGTH((FOR __loom_physical_construction_reshape_unlisted_item IN __loom_reshape_pivot_group_rows_0 FILTER NOT ((HAS(__loom_physical_construction_reshape_unlisted_item, @__loom_physical_construction_reshape_category_column) AND TYPENAME(__loom_physical_construction_reshape_unlisted_item[@__loom_physical_construction_reshape_category_column]) == @__loom_physical_construction_reshape_category_type AND __loom_physical_construction_reshape_unlisted_item[@__loom_physical_construction_reshape_category_column] == @reshape_category_0) OR (HAS(__loom_physical_construction_reshape_unlisted_item, @__loom_physical_construction_reshape_category_column) AND TYPENAME(__loom_physical_construction_reshape_unlisted_item[@__loom_physical_construction_reshape_category_column]) == @__loom_physical_construction_reshape_category_type AND __loom_physical_construction_reshape_unlisted_item[@__loom_physical_construction_reshape_category_column] == @reshape_category_1)) RETURN 1)) + FILTER ASSERT(__loom_physical_construction_reshape_unlisted_count == 0, "TABLE_PIVOT_UNLISTED_CATEGORY") + LET __loom_physical_construction_reshape_identity = TO_STRING(["GROUPED_PIVOT", @reshape_construction, ["STRING", __loom_reshape_pivot_group_0]]) + LET __loom_reshape_pivot_output_0 = { [@__loom_physical_construction_projection_0_name_1]: __loom_reshape_pivot_group_0, [@__loom_physical_construction_projection_1_name_1]: __loom_physical_construction_reshape_cell_result_0, [@__loom_physical_construction_projection_2_name_1]: __loom_physical_construction_reshape_cell_result_1, [@__loom_physical_construction_projection_3_name_1]: __loom_physical_construction_reshape_identity } + RETURN __loom_reshape_pivot_output_0 +) +LET __loom_construction_stage_2 = ( + FOR __loom_construction_input_2 IN __loom_construction_stage_1 + LET __loom_derived_0 = __loom_construction_input_2.amount_a + LET __loom_derived_1 = (((__loom_derived_0 == @derived_null_0) OR (@derived_literal_0 == @derived_null_0)) ? @derived_null_0 : (__loom_derived_0 + @derived_literal_0)) + LET __loom_construction_output_2 = { [@__loom_physical_construction_projection_0_name_2]: __loom_construction_input_2.group, [@__loom_physical_construction_projection_1_name_2]: __loom_construction_input_2.amount_a, [@__loom_physical_construction_projection_2_name_2]: __loom_construction_input_2.amount_b, [@__loom_physical_construction_projection_3_name_2]: __loom_derived_1, [@__loom_physical_construction_projection_4_name_1]: __loom_construction_input_2.__loom_row_id } + RETURN __loom_construction_output_2 +) +LET __loom_construction_stage_3 = ( + FOR __loom_construction_input_3 IN __loom_construction_stage_2 + FILTER __loom_construction_input_3.total >= @construction_filter_value + LET __loom_construction_output_3 = { [@__loom_physical_construction_projection_0_name_3]: __loom_construction_input_3.group, [@__loom_physical_construction_projection_1_name_3]: __loom_construction_input_3.amount_a, [@__loom_physical_construction_projection_2_name_3]: __loom_construction_input_3.amount_b, [@__loom_physical_construction_projection_3_name_3]: __loom_construction_input_3.total, [@__loom_physical_construction_projection_4_name_2]: __loom_construction_input_3.__loom_row_id } + RETURN __loom_construction_output_3 +) +LET __loom_construction_stage_4 = ( + FOR __loom_construction_input_4 IN __loom_construction_stage_3 + LET __loom_reshape_unpivot_input_0 = { [@__loom_physical_construction_projection_0_name_4]: __loom_construction_input_4.group, [@__loom_physical_construction_projection_1_name_4]: __loom_construction_input_4.amount_a, [@__loom_physical_construction_projection_2_name_4]: __loom_construction_input_4.amount_b, [@__loom_physical_construction_projection_3_name_4]: __loom_construction_input_4.total, [@__loom_physical_construction_projection_4_name_3]: __loom_construction_input_4.__loom_row_id } + FOR __loom_reshape_unpivot_slot_0 IN [{key: @reshape_unpivot_key_0, value: __loom_reshape_unpivot_input_0[@__loom_physical_construction_reshape_unpivot_column_0]}, {key: @reshape_unpivot_key_1, value: __loom_reshape_unpivot_input_0[@__loom_physical_construction_reshape_unpivot_column_1]}] + FILTER ASSERT(__loom_reshape_unpivot_slot_0.value == null OR TYPENAME(__loom_reshape_unpivot_slot_0.value) == @__loom_physical_construction_reshape_unpivot_value_type, "TABLE_UNPIVOT_VALUE_TYPE_MISMATCH") + LET __loom_physical_construction_reshape_unpivot_identity = TO_STRING([[@__loom_physical_construction_reshape_identity_name_0, __loom_reshape_unpivot_input_0.__loom_row_id], [@reshape_construction_1, "TABLE_UNPIVOT"], ["STRING", __loom_reshape_unpivot_slot_0.key]]) + LET __loom_reshape_unpivot_output_0 = { [@__loom_physical_construction_projection_0_name_5]: __loom_reshape_unpivot_input_0[@__loom_physical_construction_reshape_unpivot_preserved_column], [@__loom_physical_construction_projection_1_name_5]: __loom_reshape_unpivot_input_0[@__loom_physical_construction_reshape_unpivot_preserved_column_1], [@__loom_physical_construction_projection_2_name_5]: __loom_reshape_unpivot_slot_0.key, [@__loom_physical_construction_projection_3_name_5]: __loom_reshape_unpivot_slot_0.value, [@__loom_physical_construction_projection_4_name_4]: __loom_physical_construction_reshape_unpivot_identity } + RETURN __loom_reshape_unpivot_output_0 +) +FOR __loom_construction_final_row IN __loom_construction_stage_4 +SORT __loom_construction_final_row.__loom_row_id ASC +RETURN { [@__loom_physical_construction_projection_0_name_6]: __loom_construction_final_row.group, [@__loom_physical_construction_projection_1_name_6]: __loom_construction_final_row.total, [@__loom_physical_construction_projection_2_name_6]: __loom_construction_final_row.measure, [@__loom_physical_construction_projection_3_name_6]: __loom_construction_final_row.amount, [@__loom_physical_construction_projection_4_name_5]: __loom_construction_final_row.__loom_row_id, [@__loom_physical_construction_projection_5_name]: @construction_private_auth_resource_path } diff --git a/internal/dataframe/compiler/workspace_output_capture_test.go b/internal/dataframe/compiler/workspace_output_capture_test.go index 25d13332d..75cd668e9 100644 --- a/internal/dataframe/compiler/workspace_output_capture_test.go +++ b/internal/dataframe/compiler/workspace_output_capture_test.go @@ -22,10 +22,10 @@ func TestWorkspaceOutputExecutionRemainsExplicitlyUnsupportedUntilCapture(t *tes WorkspaceOutputSources: []lower.WorkspaceOutputSource{{InputIndex: 0, OutputID: "base", Schema: []lower.CompiledOutputColumn{{ID: "value", Kind: "string"}}}}, } bindings := recipe.RuntimeBindings{Project: "project-a"} - if _, err := CompileRecipeOutputWithPolicy(output, bindings, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil || !strings.Contains(err.Error(), "server-owned workspace capture is not available") { + if _, err := CompileRecipeOutput(ArangoTarget(), output, bindings, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil || !strings.Contains(err.Error(), "server-owned workspace capture is not available") { t.Fatalf("query compile error = %v", err) } - if _, err := CompileRecipeOutputPageWithPolicy(output, bindings, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil || !strings.Contains(err.Error(), "server-owned workspace capture is not available") { + if _, err := CompileRecipeOutputPage(ArangoTarget(), output, bindings, 10, ir.DefaultPhysicalOptimizationPolicy()); err == nil || !strings.Contains(err.Error(), "server-owned workspace capture is not available") { t.Fatalf("page compile error = %v", err) } } @@ -49,7 +49,7 @@ func TestCompiledQueryRetainsFinalizedPlanThatRendersExecutedQuery(t *testing.T) if err != nil { t.Fatalf("ResolveRecipePlan: %v", err) } - queries, err := CompileResolvedRecipePlanWithPolicy(resolved, 7, ir.DefaultPhysicalOptimizationPolicy()) + queries, err := CompileResolvedRecipePlan(ArangoTarget(), resolved, 7, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("CompileResolvedRecipePlanWithPolicy: %v", err) } @@ -60,10 +60,10 @@ func TestCompiledQueryRetainsFinalizedPlanThatRendersExecutedQuery(t *testing.T) if err != nil { t.Fatalf("render retained plan: %v", err) } - if rendered.Query != queries[0].Query || !reflect.DeepEqual(rendered.BindVars, queries[0].BindVars) { - t.Fatalf("retained plan differs from executed query/binds: queryMatch=%t bindMatch=%t", rendered.Query == queries[0].Query, reflect.DeepEqual(rendered.BindVars, queries[0].BindVars)) + if rendered.Query != testAQLText(t, queries[0].Command) || !reflect.DeepEqual(rendered.BindVars, testAQLBinds(t, queries[0].Command)) { + t.Fatalf("retained plan differs from executed query/binds: queryMatch=%t bindMatch=%t", rendered.Query == testAQLText(t, queries[0].Command), reflect.DeepEqual(rendered.BindVars, testAQLBinds(t, queries[0].Command))) } - if got := queries[0].BindVars["limit"]; got != 7 { + if got := testAQLBinds(t, queries[0].Command)["limit"]; got != 7 { t.Fatalf("retained execution plan lost its final preview window, limit bind=%#v", got) } } diff --git a/internal/dataframe/errors/errors.go b/internal/dataframe/errors/errors.go index 1ef1f25b0..4a694d315 100644 --- a/internal/dataframe/errors/errors.go +++ b/internal/dataframe/errors/errors.go @@ -7,8 +7,6 @@ import ( "strings" ) -// ErrorCode is the stable, transport-neutral identifier for a dataframe -// request failure. Clients should branch on this value, not on Error(). type ErrorCode string const ( @@ -81,8 +79,6 @@ const ( const CodeConstructionRowValueMultipleValues ErrorCode = "CONSTRUCTION_ROW_VALUE_MULTIPLE_VALUES" -// IsFeatureResolutionCode identifies data-dependent feature policies that a -// Builder user can resolve without an operator or a retry. func IsFeatureResolutionCode(code string) bool { switch ErrorCode(code) { case CodeRelationshipCardinalityViolation, CodeTemporalAnchorInvalid, CodeTemporalPrecisionUnsupported, CodeTemporalTieAmbiguous, CodeUnitIdentityUnknown, CodeUnitDimensionIncompatible, CodeConstructionExpansionEmpty, CodeExplicitGroupUnassignedMember, CodeConstructionRowValueMultipleValues, CodeTablePivotCellCardinality, CodeTablePivotUnlistedCategory: @@ -92,8 +88,6 @@ func IsFeatureResolutionCode(code string) bool { } } -// UserError is the semantic error contract shared by GraphQL, preview, and -// export adapters. Details are intentionally a safe, copied view. type UserError interface { error Code() string @@ -102,9 +96,6 @@ type UserError interface { Retryable() bool } -// Error is Loom's typed semantic error. The cause remains available to -// server-side logs and errors.Is/errors.As callers, but is never exposed by -// the transport mappers unless it is itself a safe UserError. type Error struct { code ErrorCode message string @@ -121,7 +112,6 @@ type errorOptions struct { cause error } -// ErrorOption configures a typed dataframe Error. type ErrorOption func(*errorOptions) func WithFieldPath(path ...string) ErrorOption { @@ -204,14 +194,11 @@ func (e *Error) Retryable() bool { return e != nil && e.retryable } -// These sentinels let semantic owners wrap backend failures without making -// the GraphQL or HTTP layer inspect driver-specific errors. var ( ErrBackendUnavailable = errors.New("dataframe backend unavailable") ErrClientCanceled = errors.New("dataframe client canceled") ) -// AsUserError extracts a typed semantic error through arbitrary wrapping. func AsUserError(err error) (UserError, bool) { if err == nil { return nil, false @@ -223,16 +210,11 @@ func AsUserError(err error) (UserError, bool) { return nil, false } -// Normalize converts only well-defined transport-neutral conditions. Unknown -// errors intentionally become INTERNAL_ERROR; no English error text is parsed. func Normalize(err error) UserError { if err == nil { return nil } if userErr, ok := AsUserError(err); ok { - // Preserve our concrete type without adding an unnecessary wrapper. An - // adapter-owned implementation is copied through Error so its details - // receive the same redaction and defensive-copy rules. if typed, ok := userErr.(*Error); ok { return typed } @@ -265,10 +247,6 @@ func PublicMessage(err error) string { return defaultMessage(ErrorCode(userErr.Code())) } -// SanitizePersistedFailure converts an opaque durable failure string into the -// stable public contract. Durable records intentionally retain only a legacy -// text field, so unknown text is always treated as an internal failure and is -// never returned to clients. func SanitizePersistedFailure(raw string) (message, code string, retryable bool) { if strings.TrimSpace(raw) == "" { return "", "", false @@ -474,8 +452,6 @@ func cloneSafeValue(value any) (any, bool) { case map[string]any: return cloneSafeDetails(typed), true default: - // Accept only primitive aliases, not arbitrary structs such as Arango - // driver errors or query objects. rv := reflect.ValueOf(value) if rv.IsValid() && rv.Kind() == reflect.String { return rv.String(), true @@ -484,5 +460,4 @@ func cloneSafeValue(value any) (any, bool) { } } -// Ensure the concrete type continues to satisfy the public contract. var _ UserError = (*Error)(nil) diff --git a/internal/dataframe/execution/backend_neutral_diagnostics_test.go b/internal/dataframe/execution/backend_neutral_diagnostics_test.go new file mode 100644 index 000000000..148b84f4d --- /dev/null +++ b/internal/dataframe/execution/backend_neutral_diagnostics_test.go @@ -0,0 +1,458 @@ +package execution + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" +) + +func TestBackendNeutralDiagnosticsAndPlanIdentity(t *testing.T) { + ctx := context.Background() + + arch12PGCompile := func(t *testing.T, resolved Resolved, limit int) compiler.CompiledQuery { + t.Helper() + query, err := compiler.CompileRecipeOutput(exec02PostgreSQLTarget(t), resolved.Compiled.Outputs[0], resolved.Semantic.SemanticPlan.Bindings, limit, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("postgresql reference compile: %v", err) + } + if query.Command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("reference command tag = %q", query.Command.Backend()) + } + return query + } + arch12AQLCompile := func(t *testing.T, resolved Resolved, limit int) compiler.CompiledQuery { + t.Helper() + query, err := compiler.CompileRecipeOutput(compiler.ArangoTarget(), resolved.Compiled.Outputs[0], resolved.Semantic.SemanticPlan.Bindings, limit, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("aql reference compile: %v", err) + } + if query.Command.Backend() != command.CompiledBackendAQL { + t.Fatalf("reference command tag = %q", query.Command.Backend()) + } + return query + } + + t.Run("postgresql diagnostic shape ignores bound values while artifact identity follows them", func(t *testing.T) { + resolved := exec02TestResolved() + baseline := arch12PGCompile(t, resolved, 5) + shape := SourcePlanFingerprint(baseline) + bound := baseline.PlanDiagnostics.Fingerprint + if shape == "" || bound == "" { + t.Fatalf("baseline digests must be present: shape=%q bound=%q", shape, bound) + } + if shape == bound { + t.Fatal("the value-free diagnostic shape must not be the value-bound artifact digest") + } + if again := SourcePlanFingerprint(arch12PGCompile(t, resolved, 5)); again != shape { + t.Fatalf("diagnostic shape is not deterministic: %q vs %q", again, shape) + } + scopeChange := exec02TestResolved() + scopeChange.Compiled.Outputs[0].Plan.BindVars["project"] = "P2" + rebound := arch12PGCompile(t, scopeChange, 5) + if got := SourcePlanFingerprint(rebound); got != shape { + t.Fatalf("bound-value-only change moved the diagnostic shape: %q vs %q", got, shape) + } + if rebound.PlanDiagnostics.Fingerprint == bound { + t.Fatal("bound-value-only change left the value-bound artifact digest unchanged") + } + sqlTextA, _, err := baseline.Command.PostgreSQLParameters() + if err != nil { + t.Fatal(err) + } + remapped, err := command.NewPostgreSQLCommand(sqlTextA+" ", nil) + if err != nil { + t.Fatal(err) + } + shapeQuery := compiler.CompiledQuery{Command: mustPGCommand(t, sqlTextA, []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "abc"}, + {Type: command.PostgreSQLParameterBigint, Value: int64(1)}, + }), Columns: []string{"id", "name"}, PublicColumns: []string{"id", "name"}} + if SourcePlanFingerprint(shapeQuery) == SourcePlanFingerprint(compiler.CompiledQuery{Command: remapped, Columns: []string{"id", "name"}, PublicColumns: []string{"id", "name"}}) { + t.Fatal("statement SQL change did not move the diagnostic shape") + } + valueSwap := compiler.CompiledQuery{Command: mustPGCommand(t, sqlTextA, []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "totally-different"}, + {Type: command.PostgreSQLParameterBigint, Value: int64(999)}, + }), Columns: []string{"id", "name"}, PublicColumns: []string{"id", "name"}} + if SourcePlanFingerprint(valueSwap) != SourcePlanFingerprint(shapeQuery) { + t.Fatal("bound values leaked into the PostgreSQL diagnostic shape") + } + typeSwap := compiler.CompiledQuery{Command: mustPGCommand(t, sqlTextA, []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "abc"}, + {Type: command.PostgreSQLParameterJSONB, Value: jsonRaw(t)}, + }), Columns: []string{"id", "name"}, PublicColumns: []string{"id", "name"}} + if SourcePlanFingerprint(typeSwap) == SourcePlanFingerprint(shapeQuery) { + t.Fatal("ordered parameter type change did not move the diagnostic shape") + } + publicChange := shapeQuery + publicChange.PublicColumns = []string{"id"} + if SourcePlanFingerprint(publicChange) == SourcePlanFingerprint(shapeQuery) { + t.Fatal("public output metadata change did not move the diagnostic shape") + } + pivotChange := shapeQuery + pivotChange.PivotFields = []string{"category"} + if SourcePlanFingerprint(pivotChange) == SourcePlanFingerprint(shapeQuery) { + t.Fatal("pivot metadata change did not move the diagnostic shape") + } + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, exec02Rows()) + limited := exec02TestResolved() + limited.Semantic.SemanticPlan.Bindings.PreviewLimit = 5 + summary, err := engine.PreviewOutput(ctx, limited, PreviewRequest{Output: "resources", Limit: 5}, func(map[string]any) error { return nil }) + if err != nil { + t.Fatalf("postgresql PreviewOutput: %v", err) + } + control := Control{Engine: engine} + explanation, err := control.ExplainResolved(ctx, limited, false) + if err != nil { + t.Fatalf("postgresql ExplainResolved: %v", err) + } + wantShape := SourcePlanFingerprint(arch12PGCompile(t, limited, 5)) + if summary.PlanFingerprint != wantShape { + t.Fatalf("preview summary must report the value-free shape: got %q want %q", summary.PlanFingerprint, wantShape) + } + if summary.PlanFingerprint == bound { + t.Fatal("preview summary leaked the value-bound artifact digest as the public plan fingerprint") + } + if explanation.Outputs[0].PlanFingerprint != summary.PlanFingerprint { + t.Fatalf("Explain and preview diverged for the same postgresql plan: explain=%q preview=%q", explanation.Outputs[0].PlanFingerprint, summary.PlanFingerprint) + } + }) + + t.Run("postgresql pinned diagnostic shape bytes", func(t *testing.T) { + query := compiler.CompiledQuery{Command: mustPGCommand(t, "SELECT id, name FROM loom.fhir_resource WHERE project = $1 AND dataset_generation = $2", []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "P1"}, + {Type: command.PostgreSQLParameterText, Value: "G1"}, + }), Columns: []string{"id", "name"}, PublicColumns: []string{"id", "name"}} + if got := SourcePlanFingerprint(query); got != "86d795c00159019f31b5270e5b4f20b4406e939475e3980646866d513427394d" { + t.Fatalf("postgres diagnostic shape digest drifted from the pinned bytes: got %s", got) + } + }) + + t.Run("aql public bytes are the historical shape and exclude bind values", func(t *testing.T) { + resolved := exec02TestResolved() + baseline := arch12AQLCompile(t, resolved, 5) + shape := SourcePlanFingerprint(baseline) + if shape != CompiledQueryFingerprint(baseline) { + t.Fatal("the AQL public diagnostic fingerprint must remain the historical CompiledQueryFingerprint bytes") + } + bound := baseline.PlanDiagnostics.Fingerprint + if shape == "" || bound == "" || shape == bound { + t.Fatalf("aql shape and bound artifact digest must both exist and differ: shape=%q bound=%q", shape, bound) + } + scopeChange := exec02TestResolved() + scopeChange.Compiled.Outputs[0].Plan.BindVars["project"] = "P2" + rebound := arch12AQLCompile(t, scopeChange, 5) + if got := SourcePlanFingerprint(rebound); got != shape { + t.Fatalf("AQL bound-value-only change moved the public shape: %q vs %q", got, shape) + } + if rebound.PlanDiagnostics.Fingerprint == bound { + t.Fatal("AQL bound-value-only change left the value-bound artifact digest unchanged") + } + metadataChange := baseline + metadataChange.Columns = append([]string(nil), baseline.Columns...) + metadataChange.Columns[0] = "identifier" + if SourcePlanFingerprint(metadataChange) == shape { + t.Fatal("AQL output column change did not move the public shape") + } + spies := &dispatchSpies{} + config := spies.arangoEngine() + engine := config.newEngineForTest(t) + limited := exec02TestResolved() + limited.Semantic.SemanticPlan.Bindings.PreviewLimit = 5 + summary, err := engine.PreviewOutput(ctx, limited, PreviewRequest{Output: "resources", Limit: 5}, func(map[string]any) error { return nil }) + if err != nil { + t.Fatalf("aql PreviewOutput: %v", err) + } + control := Control{Engine: engine} + explanation, err := control.ExplainResolved(ctx, limited, false) + if err != nil { + t.Fatalf("aql ExplainResolved: %v", err) + } + if summary.PlanFingerprint == "" || summary.PlanFingerprint != explanation.Outputs[0].PlanFingerprint { + t.Fatalf("aql Explain and preview diverged: explain=%q preview=%q", explanation.Outputs[0].PlanFingerprint, summary.PlanFingerprint) + } + if summary.PlanFingerprint == bound { + t.Fatal("aql preview summary leaked the value-bound artifact digest as the public plan fingerprint") + } + if want := CompiledQueryFingerprint(arch12AQLCompile(t, limited, 5)); summary.PlanFingerprint != want { + t.Fatalf("aql preview summary must report the historical AQL shape bytes: got %q want %q", summary.PlanFingerprint, want) + } + }) + + t.Run("one cursor interval excludes materialization for both adapters", func(t *testing.T) { + const ( + cursorQuantum = 5 * time.Second + rowMaterialization = 400 * time.Millisecond + arch12RowsPerStream = 2 + ) + for _, lane := range []struct { + name string + tag command.CompiledBackend + ported func(RowsRequest, RowVisitor, *arch12Clock) error + }{ + { + name: "AQL", tag: command.CompiledBackendAQL, + ported: func(request RowsRequest, visit RowVisitor, clock *arch12Clock) error { + if _, _, err := request.Command.AQLParameters(); err != nil { + return err + } + clock.advance(cursorQuantum) + for range arch12RowsPerStream { + if err := visit(map[string]any{"id": "row"}); err != nil { + return err + } + } + return nil + }, + }, + { + name: "PostgreSQL", tag: command.CompiledBackendPostgreSQL, + ported: func(request RowsRequest, visit RowVisitor, clock *arch12Clock) error { + if _, _, err := request.Command.PostgreSQLParameters(); err != nil { + return err + } + clock.advance(cursorQuantum) + for range arch12RowsPerStream { + if err := visit(map[string]any{"id": "row"}); err != nil { + return err + } + } + return nil + }, + }, + } { + t.Run(lane.name, func(t *testing.T) { + clock := newArch12Clock() + restore := useArch12Clock(t, clock) + defer restore() + compiled := compiler.CompiledQuery{ + Columns: []string{"id"}, PublicColumns: []string{"id"}, + Command: arch12TaggedCommand(t, lane.tag), + } + service := NewService(ServiceConfig{SourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + return lane.ported(request, visit, clock) + }}) + streamed, err := service.streamQuery(ctx, compiled, func(map[string]any) error { + clock.advance(rowMaterialization) + return nil + }) + if err != nil { + t.Fatalf("streamQuery: %v", err) + } + if got := streamed.Diagnostics.SourceQuery; got != cursorQuantum { + t.Fatalf("%s source query = %v, want exactly the held cursor quantum %v", lane.name, got, cursorQuantum) + } + if got, want := streamed.Diagnostics.RowMaterialization, time.Duration(arch12RowsPerStream)*rowMaterialization; got != want { + t.Fatalf("%s row materialization = %v, want %v", lane.name, got, want) + } + if got := streamed.Diagnostics.SourceBackend; got != lane.tag { + t.Fatalf("%s source backend tag = %q, want %q", lane.name, got, lane.tag) + } + }) + } + }) + + t.Run("cursor interval survives visitor failure and cancellation accounting", func(t *testing.T) { + const cursorQuantum = 3 * time.Second + visitorFailure := errors.New("visitor rejected the row") + for _, lane := range []struct { + name string + tag command.CompiledBackend + }{ + {name: "AQL", tag: command.CompiledBackendAQL}, + {name: "PostgreSQL", tag: command.CompiledBackendPostgreSQL}, + } { + t.Run(lane.name, func(t *testing.T) { + clock := newArch12Clock() + restore := useArch12Clock(t, clock) + defer restore() + compiled := compiler.CompiledQuery{Columns: []string{"id"}, Command: arch12TaggedCommand(t, lane.tag)} + failing := NewService(ServiceConfig{SourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + clock.advance(cursorQuantum) + if err := visit(map[string]any{"id": "row"}); err != nil { + return err + } + return nil + }}) + _, err := failing.streamQuery(ctx, compiled, func(map[string]any) error { + clock.advance(250 * time.Millisecond) + return visitorFailure + }) + if !errors.Is(err, visitorFailure) { + t.Fatalf("visitor failure must surface unchanged: %v", err) + } + canceled, cancel := context.WithCancel(ctx) + cancel() + blocked := false + cancelling := NewService(ServiceConfig{SourceRows: func(context.Context, RowsRequest, RowVisitor) error { + blocked = true + return nil + }}) + streamed, err := cancelling.streamQuery(canceled, compiled, func(map[string]any) error { return nil }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled run = %v, want context.Canceled", err) + } + if blocked { + t.Fatal("a canceled caller opened a backend cursor") + } + if streamed.Diagnostics.SourceQuery != 0 || streamed.Diagnostics.RowMaterialization != 0 { + t.Fatalf("canceled run must report zero source accounting, got %+v", streamed.Diagnostics) + } + }) + } + }) + + t.Run("preview source duration excludes row delivery for both lanes", func(t *testing.T) { + const ( + cursorQuantum = 3 * time.Second + rowDelivery = 250 * time.Millisecond + ) + for _, lane := range []struct { + name string + pg bool + }{ + {name: "AQL"}, + {name: "PostgreSQL", pg: true}, + } { + t.Run(lane.name, func(t *testing.T) { + clock := newArch12Clock() + restore := useArch12Clock(t, clock) + defer restore() + spies := &dispatchSpies{} + config := spies.arangoEngine() + port := func(_ context.Context, request RowsRequest, visit RowVisitor) error { + if lane.pg { + if _, _, err := request.Command.PostgreSQLParameters(); err != nil { + return err + } + } else if _, _, err := request.Command.AQLParameters(); err != nil { + return err + } + clock.advance(cursorQuantum) + for _, row := range exec02Rows() { + if err := visit(row); err != nil { + return err + } + } + return nil + } + if lane.pg { + config.CompileTarget = exec02PostgreSQLTarget(t) + } + config.SourceRows = port + config.PreviewSourceRows = port + engine := config.newEngineForTest(t) + limited := exec02TestResolved() + limited.Semantic.SemanticPlan.Bindings.PreviewLimit = 10 + summary, err := engine.PreviewOutput(ctx, limited, PreviewRequest{Output: "resources", Limit: 10}, func(map[string]any) error { + clock.advance(rowDelivery) + return nil + }) + if err != nil { + t.Fatalf("%s PreviewOutput: %v", lane.name, err) + } + if summary.RowCount != 2 { + t.Fatalf("%s preview row count = %d, want 2", lane.name, summary.RowCount) + } + if got, want := summary.SourceQueryDuration, cursorQuantum; got != want { + t.Fatalf("%s preview source duration = %v, want the cursor quantum %v with row delivery excluded", lane.name, got, want) + } + }) + } + }) + + t.Run("projection gate refuses an empty public fingerprint", func(t *testing.T) { + if err := requireProjectedPlanFingerprint("resources", "86d795c00159019f31b5270e5b4f20b4406e939475e3980646866d513427394d"); err != nil { + t.Fatalf("a real shape digest must pass the gate: %v", err) + } + if err := requireProjectedPlanFingerprint("resources", "combine-fingerprint"); err != nil { + t.Fatalf("the ClickHouse combine digest must pass the gate: %v", err) + } + for _, empty := range []string{"", " "} { + err := requireProjectedPlanFingerprint("resources", empty) + if err == nil { + t.Fatalf("empty fingerprint %q must fail the projection gate", empty) + } + if !strings.Contains(err.Error(), "resources") || !strings.Contains(err.Error(), "fingerprint") { + t.Fatalf("the gate error must name the output and the field: %v", err) + } + userErr, typed := dataframeerrors.AsUserError(err) + if !typed || userErr.Code() != string(dataframeerrors.CodeInternalError) || userErr.Retryable() { + t.Fatalf("the gate must refuse with typed non-retryable INTERNAL_ERROR, got typed=%v err=%v", typed, err) + } + } + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, exec02Rows()) + limited := exec02TestResolved() + limited.Semantic.SemanticPlan.Bindings.PreviewLimit = 5 + explanation, err := (Control{Engine: engine}).ExplainResolved(ctx, limited, false) + if err != nil { + t.Fatalf("ExplainResolved: %v", err) + } + if got := explanation.Outputs[0].PlanFingerprint; len(got) != 64 || strings.TrimLeft(got, "0123456789abcdef") != "" { + t.Fatalf("Explain must project a real 64-hex shape through the gate, got %q", got) + } + }) + + t.Run("clickhouse combine preview keeps its ClickHouse-owned digest", func(t *testing.T) { + query := compiler.CompiledQuery{PlanMode: "clickhouse", PlanProfile: "pinned_table_combine", Limit: 2, PlanDiagnostics: ir.CompilerPlanDiagnostics{Fingerprint: "combine-fingerprint"}} + stream := OutputStream{physicalEngine: ir.PhysicalEngineClickHouse} + if got := previewPlanFingerprint(query, stream); got != "combine-fingerprint" { + t.Fatalf("clickhouse combine preview fingerprint = %q, want the combine digest", got) + } + }) +} + +type arch12Clock struct { + current time.Time +} + +func newArch12Clock() *arch12Clock { + return &arch12Clock{current: time.Date(2026, time.November, 12, 9, 0, 0, 0, time.UTC)} +} + +func (c *arch12Clock) now() time.Time { return c.current } + +func (c *arch12Clock) advance(d time.Duration) { c.current = c.current.Add(d) } + +func useArch12Clock(t *testing.T, clock *arch12Clock) func() { + t.Helper() + previous := now + now = clock.now + return func() { now = previous } +} + +func arch12TaggedCommand(t *testing.T, tag command.CompiledBackend) command.CompiledCommand { + t.Helper() + switch tag { + case command.CompiledBackendAQL: + return mustAQLCommand("FOR d IN Patient FILTER d.project == @project RETURN d", map[string]any{"project": "P1"}) + case command.CompiledBackendPostgreSQL: + return mustPGCommand(t, "SELECT id FROM loom.fhir_resource WHERE project = $1", []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterText, Value: "P1"}}) + default: + t.Fatalf("unsupported tag %q", tag) + return command.CompiledCommand{} + } +} + +func mustPGCommand(t *testing.T, sqlText string, args []command.PostgreSQLParameter) command.CompiledCommand { + t.Helper() + cmd, err := command.NewPostgreSQLCommand(sqlText, args) + if err != nil { + t.Fatalf("build PostgreSQL command: %v", err) + } + return cmd +} + +func jsonRaw(t *testing.T) []byte { + t.Helper() + return []byte(`{"k":1}`) +} diff --git a/internal/dataframe/execution/boundary_helpers_test.go b/internal/dataframe/execution/boundary_helpers_test.go new file mode 100644 index 000000000..0a9b96483 --- /dev/null +++ b/internal/dataframe/execution/boundary_helpers_test.go @@ -0,0 +1,49 @@ +package execution + +import ( + "context" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" +) + +func aqlView(t *testing.T, command command.CompiledCommand) (string, map[string]any) { + t.Helper() + query, bindVars, err := command.AQLParameters() + if err != nil { + t.Fatalf("dispatch compiled command to AQL view: %v", err) + } + return query, bindVars +} + +func mustAQLCommand(query string, bindVars map[string]any) command.CompiledCommand { + compiledCommand, err := command.NewAQLCommand(query, bindVars) + if err != nil { + panic(err) + } + return compiledCommand +} + +func aqlRows(fake func(context.Context, string, int, map[string]any, func(map[string]any) error) error) SourceRows { + return func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + batch := request.BatchSize + query, bindVars, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return fake(ctx, query, batch, bindVars, visit) + } +} + +func commandText(t *testing.T, command command.CompiledCommand) string { + t.Helper() + query, _ := aqlView(t, command) + return query +} + +func commandBinds(t *testing.T, command command.CompiledCommand) map[string]any { + t.Helper() + _, bindVars := aqlView(t, command) + return bindVars +} diff --git a/internal/dataframe/execution/category_scan.go b/internal/dataframe/execution/category_scan.go index 089b3a6fe..a6cfa8829 100644 --- a/internal/dataframe/execution/category_scan.go +++ b/internal/dataframe/execution/category_scan.go @@ -2,7 +2,9 @@ package execution import ( "context" + "encoding/json" "fmt" + "github.com/calypr/loom/internal/dataframe/compiler/command" "strings" "sync" "time" @@ -54,9 +56,9 @@ func (e *Engine) ScanCategories(ctx context.Context, resolved Resolved, request var compiled compiler.CompiledCategoryScanQuery var err error if request.StageID != "" { - compiled, err = compiler.CompileCategoryScanStageWithPolicy(output, request.StageID, request.ColumnID, request.ValueColumnID, request.MaxValues, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err = compiler.CompileCategoryScanStage(e.compileTarget, output, request.StageID, request.ColumnID, request.ValueColumnID, request.MaxValues, ir.DefaultPhysicalOptimizationPolicy()) } else { - compiled, err = compiler.CompileCategoryScanOutputWithPolicy(output, request.Column, request.MaxValues, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err = compiler.CompileCategoryScanOutput(e.compileTarget, output, request.Column, request.MaxValues, ir.DefaultPhysicalOptimizationPolicy()) } if err != nil { return CategoryScanResult{}, err @@ -66,8 +68,15 @@ func (e *Engine) ScanCategories(ctx context.Context, resolved Resolved, request return CategoryScanResult{}, fmt.Errorf("category scan output %q was not found", request.Output) } +func (e *Engine) previewRowsPort() SourceRows { + if e.previewSourceRows != nil { + return e.previewSourceRows + } + return e.sourceRows +} + func (e *Engine) ScanCategoriesCompiled(ctx context.Context, compiled compiler.CompiledCategoryScanQuery) (CategoryScanResult, error) { - if e == nil || e.queryRows == nil { + if e == nil || e.sourceRows == nil { return CategoryScanResult{}, fmt.Errorf("category scan query executor is required") } if compiled.PresentColumn == "" || compiled.ValueColumn == "" || compiled.Proof.MaxValues < 1 { @@ -76,6 +85,14 @@ func (e *Engine) ScanCategoriesCompiled(ctx context.Context, compiled compiler.C if err := ctx.Err(); err != nil { return CategoryScanResult{}, err } + switch compiled.Command.Backend() { + case command.CompiledBackendPostgreSQL: + return e.scanCategoriesPostgreSQL(ctx, compiled) + case command.CompiledBackendAQL: + default: + return CategoryScanResult{}, fmt.Errorf("%w: category scan command backend %q is not registered", + command.ErrCompiledCommandBackendNotRegistered, compiled.Command.Backend()) + } if compiled.CategoryIndex != nil && e.preparePreviewIndex != nil { _ = e.preparePreviewIndex(ctx, *compiled.CategoryIndex) if err := ctx.Err(); err != nil { @@ -85,16 +102,10 @@ func (e *Engine) ScanCategoriesCompiled(ctx context.Context, compiled compiler.C if compiled.PreviewCoveringIndex != nil { schedulePreviewIndexPrewarm(ctx, e, *compiled.PreviewCoveringIndex) } - values := make([]CategoryValue, 0, compiled.Proof.MaxValues) - overflow := false - queryRows := e.queryRows - if e.previewQueryRows != nil { - queryRows = e.previewQueryRows - } if compiled.OverflowWitness != nil { witnessCount := 0 missing := false - err := queryRows(ctx, compiled.OverflowWitness.Query, e.batchSize, compiled.OverflowWitness.BindVars, func(row map[string]any) error { + err := e.previewRowsPort()(ctx, RowsRequest{Command: compiled.OverflowWitness.Command, BatchSize: e.batchSize}, func(row map[string]any) error { if err := ctx.Err(); err != nil { return err } @@ -122,7 +133,22 @@ func (e *Engine) ScanCategoriesCompiled(ctx context.Context, compiled compiler.C return CategoryScanResult{ConclusiveMissing: true, Proof: compiled.Proof}, nil } } - err := queryRows(ctx, compiled.Query, e.batchSize, compiled.BindVars, func(row map[string]any) error { + return e.collectCategoryResultPage(ctx, compiled, nil, identityCategoryValue) +} + +func (e *Engine) collectCategoryResultPage( + ctx context.Context, + compiled compiler.CompiledCategoryScanQuery, + declaredColumns []string, + decodeValue func(any) (any, error), +) (CategoryScanResult, error) { + values := make([]CategoryValue, 0, compiled.Proof.MaxValues) + overflow := false + err := e.previewRowsPort()(ctx, RowsRequest{ + Command: compiled.Command, + DeclaredColumns: declaredColumns, + BatchSize: e.batchSize, + }, func(row map[string]any) error { if err := ctx.Err(); err != nil { return err } @@ -130,10 +156,14 @@ func (e *Engine) ScanCategoriesCompiled(ctx context.Context, compiled compiler.C if !ok { return fmt.Errorf("category scan row has invalid presence marker") } - value, exists := row[compiled.ValueColumn] + raw, exists := row[compiled.ValueColumn] if !exists { return fmt.Errorf("category scan row is missing value") } + value, err := decodeValue(raw) + if err != nil { + return err + } if len(values) == compiled.Proof.MaxValues { overflow = true return nil @@ -150,6 +180,35 @@ func (e *Engine) ScanCategoriesCompiled(ctx context.Context, compiled compiler.C return CategoryScanResult{Values: values, Complete: !overflow, Overflow: overflow, Proof: compiled.Proof}, nil } +func identityCategoryValue(raw any) (any, error) { + return raw, nil +} + +func (e *Engine) scanCategoriesPostgreSQL(ctx context.Context, compiled compiler.CompiledCategoryScanQuery) (CategoryScanResult, error) { + if compiled.OverflowWitness != nil { + return CategoryScanResult{}, fmt.Errorf("category scan: a PostgreSQL command must not carry an overflow witness; the bounded maxValues+1 page owns truncation detection") + } + return e.collectCategoryResultPage(ctx, compiled, []string{compiled.PresentColumn, compiled.ValueColumn}, decodePostgreSQLCategoryValue) +} + +func decodePostgreSQLCategoryValue(raw any) (any, error) { + switch typed := raw.(type) { + case nil: + return nil, nil + case json.RawMessage: + if len(typed) == 0 || !json.Valid(typed) { + return nil, fmt.Errorf("category scan row has invalid JSONB value") + } + var decoded any + if err := json.Unmarshal(typed, &decoded); err != nil { + return nil, fmt.Errorf("category scan row has invalid JSONB value: %w", err) + } + return decoded, nil + default: + return nil, fmt.Errorf("category scan PostgreSQL row carries %T in the value column; the PostgreSQL adapter delivers JSON values only as raw JSON bytes", raw) + } +} + func schedulePreviewIndexPrewarm(ctx context.Context, engine *Engine, spec compiler.PreviewCoveringIndexSpec) { if engine == nil || engine.preparePreviewIndex == nil || spec.Collection == "" || spec.Name == "" || len(spec.Fields) == 0 { return diff --git a/internal/dataframe/execution/category_scan_postgres_test.go b/internal/dataframe/execution/category_scan_postgres_test.go new file mode 100644 index 000000000..bddeadeac --- /dev/null +++ b/internal/dataframe/execution/category_scan_postgres_test.go @@ -0,0 +1,694 @@ +package execution_test + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + "github.com/calypr/loom/internal/dataframe/recipe" + exec "github.com/calypr/loom/internal/dataframe/recipe/exec" + "github.com/calypr/loom/internal/dataframe/semantic" +) + +type exec06Call struct { + sql string + args []command.PostgreSQLParameter + declared []string + tag command.CompiledBackend +} + +type exec06Port struct { + calls []exec06Call + rows []map[string]any + visitBefore func(index int) error + fail error +} + +func (p *exec06Port) port(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + if p.fail != nil { + return p.fail + } + var call exec06Call + switch request.Command.Backend() { + case command.CompiledBackendPostgreSQL: + sqlText, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + if len(request.DeclaredColumns) != 2 || request.DeclaredColumns[0] != "present" || request.DeclaredColumns[1] != "value" { + return errors.New("fake PostgreSQL port: declared column contract must be exactly [present value]") + } + call = exec06Call{sql: sqlText, args: args, declared: append([]string(nil), request.DeclaredColumns...), tag: request.Command.Backend()} + case command.CompiledBackendAQL: + if _, _, err := request.Command.AQLParameters(); err != nil { + return err + } + if len(request.DeclaredColumns) != 0 { + return errors.New("fake port: the AQL arm must stay contract-free") + } + call = exec06Call{tag: request.Command.Backend()} + default: + return command.ErrCompiledCommandBackendNotRegistered + } + p.calls = append(p.calls, call) + for index, row := range p.rows { + if p.visitBefore != nil { + if err := p.visitBefore(index); err != nil { + return err + } + } + if err := ctx.Err(); err != nil { + return err + } + if err := visit(row); err != nil { + return err + } + } + return nil +} + +func (p *exec06Port) counts() int { return len(p.calls) } + +func exec06JSONB(t *testing.T, text string) json.RawMessage { + t.Helper() + if !json.Valid([]byte(text)) { + t.Fatalf("fixture is not valid JSON: %s", text) + } + return json.RawMessage(text) +} + +func exec06PGCommand(t *testing.T, bound int) command.CompiledCommand { + t.Helper() + sqlText := `WITH __loom_category_source AS (SELECT $1::jsonb AS "c0", $2::jsonb AS "c1"), ` + + `__loom_category_rows AS (SELECT __loom_category_row."c1" AS "present", CASE WHEN __loom_category_row."c1" THEN __loom_category_row."c0" END AS "value" FROM __loom_category_source AS __loom_category_row), ` + + `__loom_category_distinct AS (SELECT "present", "value" FROM __loom_category_rows GROUP BY "present", "value") ` + + `SELECT "present" AS "c0", "value" AS "c1" FROM __loom_category_distinct ORDER BY "present" ASC LIMIT $3` + compiled, err := command.NewPostgreSQLCommand(sqlText, []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterJSONB, Value: `{"gender":"female"}`}, + {Type: command.PostgreSQLParameterBoolean, Value: true}, + {Type: command.PostgreSQLParameterBigint, Value: int64(bound)}, + }) + if err != nil { + t.Fatalf("tag PostgreSQL category command: %v", err) + } + return compiled +} + +func exec06AQLCommand(t *testing.T) command.CompiledCommand { + t.Helper() + compiled, err := command.NewAQLCommand("FOR row IN Patients RETURN { present: true, value: row.gender }", map[string]any{"__loom_category_limit": 9}) + if err != nil { + t.Fatalf("tag AQL category command: %v", err) + } + return compiled +} + +func exec06Compiled(t *testing.T, backend command.CompiledBackend, maxValues int) compiler.CompiledCategoryScanQuery { + t.Helper() + var cmd command.CompiledCommand + switch backend { + case command.CompiledBackendPostgreSQL: + cmd = exec06PGCommand(t, maxValues+1) + case command.CompiledBackendAQL: + cmd = exec06AQLCommand(t) + default: + t.Fatalf("test helper does not build %q commands", backend) + } + return compiler.CompiledCategoryScanQuery{ + Command: cmd, + PresentColumn: "present", ValueColumn: "value", + Proof: compiler.CategoryScanProof{Version: 1, Output: "patients", Column: "gender", MaxValues: maxValues, PresenceTracked: true, Fingerprint: "exec06-proof"}, + } +} + +func exec06Target(t *testing.T, backend command.CompiledBackend) compiler.Target { + t.Helper() + switch backend { + case command.CompiledBackendPostgreSQL: + binding, err := ir.NewPostgresStorageBinding("verify_exec06_offline") + if err != nil { + t.Fatalf("storage binding: %v", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("PostgreSQL target: %v", err) + } + return target + case command.CompiledBackendAQL: + return compiler.ArangoTarget() + default: + t.Fatalf("test helper does not build %q targets", backend) + return compiler.Target{} + } +} + +type exec06RefusingRegistry struct{} + +func (exec06RefusingRegistry) LoadRecipe(context.Context, string) (exec.Entry, error) { + return exec.Entry{}, errors.New("the exec06 offline battery compiles authored bundles only") +} + +func (exec06RefusingRegistry) LoadRecipeVersion(context.Context, string, string) (exec.Entry, error) { + return exec.Entry{}, errors.New("the exec06 offline battery compiles authored bundles only") +} + +func exec06Landmines() dataframeexecution.Config { + return dataframeexecution.Config{ + Registry: exec06RefusingRegistry{}, + SourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, _ dataframeexecution.RowVisitor) error { + if _, _, err := request.Command.AQLParameters(); err != nil { + return err + } + return errors.New("AQL QueryRows must never run for a PostgreSQL-served category scan") + }, + PreviewSourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, _ dataframeexecution.RowVisitor) error { + if _, _, err := request.Command.AQLParameters(); err != nil { + return err + } + return errors.New("AQL PreviewQueryRows must never run for a PostgreSQL-served category scan") + }, + PreparePreviewIndex: func(context.Context, compiler.PreviewCoveringIndexSpec) error { + return errors.New("preparePreviewIndex must never run for a PostgreSQL-served category scan") + }, + PreviewCoveringIndexes: func(context.Context, string) ([]dataframeexecution.PreviewCollectionIndex, error) { + return nil, errors.New("previewCoveringIndexes must never run for a PostgreSQL-served category scan") + }, + PreviewCollectionRevision: func(context.Context, string) (string, error) { + return "", errors.New("previewCollectionRevision must never run for a PostgreSQL-served category scan") + }, + PreviewCollectionScans: func(context.Context, command.CompiledCommand, string) (dataframeexecution.PreviewCollectionScanEvidence, error) { + return dataframeexecution.PreviewCollectionScanEvidence{}, errors.New("previewCollectionScans must never run for a PostgreSQL-served category scan") + }, + PreviewCollectionCount: func(context.Context, string) (int64, error) { + return 0, errors.New("previewCollectionCount must never run for a PostgreSQL-served category scan") + }, + ClickHouseQueryRows: func(context.Context, string, []string, func(map[string]any) error, ...any) error { + return errors.New("ClickHouse callbacks must never run for a PostgreSQL-served category scan") + }, + } +} + +func exec06Engine(t *testing.T, backend command.CompiledBackend, port *exec06Port) *dataframeexecution.Engine { + t.Helper() + config := exec06Landmines() + config.CompileTarget = exec06Target(t, backend) + config.PreviewSourceRows = nil + config.SourceRows = port.port + engine, err := dataframeexecution.New(config) + if err != nil { + t.Fatalf("engine: %v", err) + } + return engine +} + +func TestExec06PGCategoryExactValuesDecode(t *testing.T) { + port := &exec06Port{rows: []map[string]any{ + {"present": false, "value": nil}, + {"present": true, "value": exec06JSONB(t, "false")}, + {"present": true, "value": exec06JSONB(t, "null")}, + {"present": true, "value": exec06JSONB(t, "0")}, + {"present": true, "value": exec06JSONB(t, `""`)}, + {"present": true, "value": exec06JSONB(t, `"female"`)}, + {"present": true, "value": exec06JSONB(t, `"male"`)}, + }} + engine := exec06Engine(t, command.CompiledBackendPostgreSQL, port) + result, err := engine.ScanCategoriesCompiled(context.Background(), exec06Compiled(t, command.CompiledBackendPostgreSQL, 8)) + if err != nil { + t.Fatalf("scan: %v", err) + } + want := []dataframeexecution.CategoryValue{ + {Present: false, Value: nil}, + {Present: true, Value: false}, + {Present: true, Value: nil}, + {Present: true, Value: float64(0)}, + {Present: true, Value: ""}, + {Present: true, Value: "female"}, + {Present: true, Value: "male"}, + } + if len(result.Values) != len(want) { + t.Fatalf("values = %#v, want %#v", result.Values, want) + } + for index := range want { + if result.Values[index] != want[index] { + t.Fatalf("value %d = %#v, want %#v", index, result.Values[index], want[index]) + } + } + if !result.Complete || result.Overflow || result.ConclusiveMissing { + t.Fatalf("status = Complete:%v Overflow:%v ConclusiveMissing:%v, want complete, not overflow, not conclusive-missing", result.Complete, result.Overflow, result.ConclusiveMissing) + } + if result.Proof.Fingerprint != "exec06-proof" { + t.Fatalf("proof not echoed: %#v", result.Proof) + } + if port.counts() != 1 { + t.Fatalf("port calls = %d, want exactly one bounded statement", port.counts()) + } + call := port.calls[0] + if call.tag != command.CompiledBackendPostgreSQL { + t.Fatalf("tag = %q, want POSTGRESQL", call.tag) + } + if len(call.declared) != 2 || call.declared[0] != "present" || call.declared[1] != "value" { + t.Fatalf("declared columns = %v, want [present value]", call.declared) + } + if !strings.Contains(call.sql, "LIMIT $3") || strings.Contains(call.sql, "@") { + t.Fatalf("statement is not the positional bounded page: %s", call.sql) + } + if call.args[len(call.args)-1].Value != int64(9) { + t.Fatalf("bound argument = %+v, want bigint 9 (maxValues+1)", call.args[len(call.args)-1]) + } +} + +func TestExec06PGCategoryOverflowCapsPageAtBound(t *testing.T) { + port := &exec06Port{rows: []map[string]any{ + {"present": true, "value": exec06JSONB(t, `"a"`)}, + {"present": true, "value": exec06JSONB(t, `"b"`)}, + {"present": true, "value": exec06JSONB(t, `"c"`)}, + }} + engine := exec06Engine(t, command.CompiledBackendPostgreSQL, port) + result, err := engine.ScanCategoriesCompiled(context.Background(), exec06Compiled(t, command.CompiledBackendPostgreSQL, 2)) + if err != nil { + t.Fatalf("scan: %v", err) + } + if len(result.Values) != 2 || result.Values[0].Value != "a" || result.Values[1].Value != "b" { + t.Fatalf("values = %#v, want the first two capped", result.Values) + } + if result.Complete || !result.Overflow { + t.Fatalf("status = Complete:%v Overflow:%v, want Complete=false Overflow=true", result.Complete, result.Overflow) + } + if port.counts() != 1 { + t.Fatalf("port calls = %d, want one — truncation is page-owned, no second query", port.counts()) + } +} + +func TestExec06PGCategoryDecodeFailuresFailLoud(t *testing.T) { + cases := []struct { + name string + row map[string]any + contains string + }{ + {"empty JSONB bytes", map[string]any{"present": true, "value": json.RawMessage(nil)}, "invalid JSONB value"}, + {"malformed JSONB bytes", map[string]any{"present": true, "value": json.RawMessage(`{oops`)}, "invalid JSONB value"}, + {"Go-native string value", map[string]any{"present": true, "value": "secret-value"}, "raw JSON bytes"}, + {"Go-native number value", map[string]any{"present": true, "value": int64(3)}, "raw JSON bytes"}, + {"missing value key", map[string]any{"present": true}, "missing value"}, + {"missing presence marker", map[string]any{"value": exec06JSONB(t, `"a"`)}, "invalid presence marker"}, + {"non-bool presence marker", map[string]any{"present": "true", "value": exec06JSONB(t, `"a"`)}, "invalid presence marker"}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + port := &exec06Port{rows: []map[string]any{ + {"present": true, "value": exec06JSONB(t, `"female"`)}, + test.row, + }} + engine := exec06Engine(t, command.CompiledBackendPostgreSQL, port) + result, err := engine.ScanCategoriesCompiled(context.Background(), exec06Compiled(t, command.CompiledBackendPostgreSQL, 8)) + if err == nil { + t.Fatalf("malformed row accepted: result %#v", result) + } + if !strings.Contains(err.Error(), test.contains) { + t.Fatalf("error = %v, want it to mention %q", err, test.contains) + } + if strings.Contains(err.Error(), "secret-value") || strings.Contains(err.Error(), "{oops") { + t.Fatalf("error leaked row value text: %v", err) + } + if result.Complete || result.Overflow || result.ConclusiveMissing || len(result.Values) != 0 { + t.Fatalf("failed scan must carry a zero result, got %#v", result) + } + }) + } +} + +func TestExec06PGCategoryWitnessAndForeignTagRefusedBeforePort(t *testing.T) { + t.Run("hand-attached witness refused", func(t *testing.T) { + port := &exec06Port{} + engine := exec06Engine(t, command.CompiledBackendPostgreSQL, port) + compiled := exec06Compiled(t, command.CompiledBackendPostgreSQL, 8) + witnessCommand, err := command.NewAQLCommand("RETURN {present: false, value: null}", nil) + if err != nil { + t.Fatalf("tag witness: %v", err) + } + compiled.OverflowWitness = &compiler.CategoryOverflowWitness{Command: witnessCommand} + result, err := engine.ScanCategoriesCompiled(context.Background(), compiled) + if err == nil || !strings.Contains(err.Error(), "overflow witness") { + t.Fatalf("witness-carrying PostgreSQL command = (%#v, %v), want a typed witness refusal", result, err) + } + if result.Complete || len(result.Values) != 0 { + t.Fatalf("refused scan must be a zero result, got %#v", result) + } + if port.counts() != 0 { + t.Fatalf("port calls = %d, want zero before any witness dispatch", port.counts()) + } + }) + t.Run("unregistered command tag refused", func(t *testing.T) { + port := &exec06Port{} + engine := exec06Engine(t, command.CompiledBackendPostgreSQL, port) + compiled := exec06Compiled(t, command.CompiledBackendPostgreSQL, 8) + compiled.Command = command.CompiledCommand{} + _, err := engine.ScanCategoriesCompiled(context.Background(), compiled) + if !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("zero-tag command = %v, want ErrCompiledCommandBackendNotRegistered", err) + } + if port.counts() != 0 { + t.Fatalf("port calls = %d, want zero", port.counts()) + } + }) + t.Run("AQL arm stays contract-free", func(t *testing.T) { + port := &exec06Port{rows: []map[string]any{{"present": true, "value": "female"}}} + engine := exec06Engine(t, command.CompiledBackendAQL, port) + result, err := engine.ScanCategoriesCompiled(context.Background(), exec06Compiled(t, command.CompiledBackendAQL, 8)) + if err != nil { + t.Fatalf("AQL control scan: %v", err) + } + if !result.Complete || len(result.Values) != 1 || result.Values[0] != (dataframeexecution.CategoryValue{Present: true, Value: "female"}) { + t.Fatalf("AQL control result = %#v", result) + } + if port.counts() != 1 || port.calls[0].tag != command.CompiledBackendAQL || len(port.calls[0].declared) != 0 { + t.Fatalf("AQL arm gained a declared contract: %#v", port.calls) + } + }) +} + +func TestExec06PGCategoryPortErrorsHonestStatuses(t *testing.T) { + t.Run("generic port failure is a wrapped hard error", func(t *testing.T) { + port := &exec06Port{fail: errors.New("boom")} + engine := exec06Engine(t, command.CompiledBackendPostgreSQL, port) + result, err := engine.ScanCategoriesCompiled(context.Background(), exec06Compiled(t, command.CompiledBackendPostgreSQL, 8)) + if err == nil || !strings.Contains(err.Error(), "execute category scan") || !strings.Contains(err.Error(), "boom") { + t.Fatalf("port failure = (%#v, %v), want a wrapped hard error", result, err) + } + if result.Complete || result.Overflow || len(result.Values) != 0 { + t.Fatalf("failed scan must be zero-result, got %#v", result) + } + }) + t.Run("cancellation never yields a complete result", func(t *testing.T) { + port := &exec06Port{rows: []map[string]any{ + {"present": true, "value": exec06JSONB(t, `"female"`)}, + {"present": true, "value": exec06JSONB(t, `"male"`)}, + }} + engine := exec06Engine(t, command.CompiledBackendPostgreSQL, port) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + port.visitBefore = func(index int) error { + if index == 0 { + cancel() + } + return nil + } + result, err := engine.ScanCategoriesCompiled(ctx, exec06Compiled(t, command.CompiledBackendPostgreSQL, 8)) + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled scan = (%#v, %v), want an errors.Is(context.Canceled) error", result, err) + } + if result.Complete || result.Overflow || len(result.Values) != 0 { + t.Fatalf("cancelled scan must be zero-result and never complete, got %#v", result) + } + port.visitBefore = nil + alive, err := engine.ScanCategoriesCompiled(context.Background(), exec06Compiled(t, command.CompiledBackendPostgreSQL, 8)) + if err != nil { + t.Fatalf("ALIVE control never converged: %v", err) + } + if !alive.Complete || len(alive.Values) != 2 { + t.Fatalf("ALIVE control result = %#v", alive) + } + }) +} + +func TestExec06PGCategoryStatusInvariants(t *testing.T) { + port := &exec06Port{rows: []map[string]any{{"present": false, "value": nil}}} + engine := exec06Engine(t, command.CompiledBackendPostgreSQL, port) + compiled := exec06Compiled(t, command.CompiledBackendPostgreSQL, 8) + compiled.CategoryIndex = &compiler.PreviewCoveringIndexSpec{Collection: "Patient", Name: "exec06_sneak", Fields: []string{"payload.gender"}} + compiled.PreviewCoveringIndex = &compiler.PreviewCoveringIndexSpec{Collection: "Patient", Name: "exec06_sneak_2", Fields: []string{"project", "payload.gender"}} + result, err := engine.ScanCategoriesCompiled(context.Background(), compiled) + if err != nil { + t.Fatalf("scan with hand-attached specs: %v (Arango accelerator callbacks are armed as landmines)", err) + } + if result.ConclusiveMissing { + t.Fatalf("ConclusiveMissing must be unreachable on the PostgreSQL arm") + } + if !result.Complete || result.Overflow || len(result.Values) != 1 || result.Values[0] != (dataframeexecution.CategoryValue{Present: false}) { + t.Fatalf("all-absent page result = %#v, want one complete ABSENT group", result) + } + if port.counts() != 1 { + t.Fatalf("port calls = %d, want one", port.counts()) + } +} + +func TestExec06PGCategoryUsesPreviewPortOverride(t *testing.T) { + preview := &exec06Port{rows: []map[string]any{{"present": true, "value": exec06JSONB(t, `"female"`)}}} + config := exec06Landmines() + config.CompileTarget = exec06Target(t, command.CompiledBackendPostgreSQL) + config.SourceRows = func(context.Context, dataframeexecution.RowsRequest, dataframeexecution.RowVisitor) error { + return errors.New("the canonical port must not serve a category scan when the preview override is installed") + } + config.PreviewSourceRows = preview.port + engine, err := dataframeexecution.New(config) + if err != nil { + t.Fatalf("engine: %v", err) + } + result, err := engine.ScanCategoriesCompiled(context.Background(), exec06Compiled(t, command.CompiledBackendPostgreSQL, 8)) + if err != nil { + t.Fatalf("scan: %v", err) + } + if !result.Complete || len(result.Values) != 1 { + t.Fatalf("result = %#v", result) + } + if preview.counts() != 1 { + t.Fatalf("preview port calls = %d, want one", preview.counts()) + } +} + +func TestExec06PGCategoryPageCollectorDialectParity(t *testing.T) { + logicalValues := []dataframeexecution.CategoryValue{ + {Present: false, Value: nil}, + {Present: true, Value: false}, + {Present: true, Value: nil}, + {Present: true, Value: float64(0)}, + {Present: true, Value: ""}, + {Present: true, Value: "female"}, + } + rowsFor := func(t *testing.T, backend command.CompiledBackend, limit int) []map[string]any { + t.Helper() + rows := make([]map[string]any, 0, limit) + for _, value := range logicalValues[:limit] { + if backend == command.CompiledBackendPostgreSQL && value.Present { + switch typed := value.Value.(type) { + case nil: + rows = append(rows, map[string]any{"present": true, "value": exec06JSONB(t, "null")}) + case bool: + rows = append(rows, map[string]any{"present": true, "value": exec06JSONB(t, "false")}) + case float64: + rows = append(rows, map[string]any{"present": true, "value": exec06JSONB(t, "0")}) + case string: + raw, err := json.Marshal(typed) + if err != nil { + t.Fatalf("marshal fixture: %v", err) + } + rows = append(rows, map[string]any{"present": true, "value": exec06JSONB(t, string(raw))}) + default: + t.Fatalf("fixture value %T not covered", typed) + } + continue + } + rows = append(rows, map[string]any{"present": value.Present, "value": value.Value}) + } + return rows + } + scan := func(t *testing.T, backend command.CompiledBackend, rows []map[string]any, maxValues int) (dataframeexecution.CategoryScanResult, error) { + t.Helper() + port := &exec06Port{rows: rows} + engine := exec06Engine(t, backend, port) + result, err := engine.ScanCategoriesCompiled(context.Background(), exec06Compiled(t, backend, maxValues)) + if port.counts() != 1 || port.calls[0].tag != backend { + t.Fatalf("%s port calls = %#v, want one call tagged %s", backend, port.calls, backend) + } + return result, err + } + t.Run("same logical page yields identical results across dialects", func(t *testing.T) { + aql, aqlErr := scan(t, command.CompiledBackendAQL, rowsFor(t, command.CompiledBackendAQL, len(logicalValues)), 8) + pg, pgErr := scan(t, command.CompiledBackendPostgreSQL, rowsFor(t, command.CompiledBackendPostgreSQL, len(logicalValues)), 8) + if aqlErr != nil || pgErr != nil { + t.Fatalf("aql err = %v, pg err = %v", aqlErr, pgErr) + } + if !reflect.DeepEqual(aql.Values, logicalValues) || !reflect.DeepEqual(pg.Values, logicalValues) { + t.Fatalf("aql values = %#v, pg values = %#v, want %#v", aql.Values, pg.Values, logicalValues) + } + if !reflect.DeepEqual(aql, pg) { + t.Fatalf("dialect results diverge: aql = %#v, pg = %#v", aql, pg) + } + if !aql.Complete || aql.Overflow || aql.ConclusiveMissing { + t.Fatalf("parity page status = Complete:%v Overflow:%v ConclusiveMissing:%v, want complete", aql.Complete, aql.Overflow, aql.ConclusiveMissing) + } + if aql.Proof.Fingerprint != "exec06-proof" { + t.Fatalf("proof not echoed: %#v", aql.Proof) + } + }) + t.Run("overflow caps identically at the bound in both dialects", func(t *testing.T) { + aql, aqlErr := scan(t, command.CompiledBackendAQL, rowsFor(t, command.CompiledBackendAQL, 3), 2) + pg, pgErr := scan(t, command.CompiledBackendPostgreSQL, rowsFor(t, command.CompiledBackendPostgreSQL, 3), 2) + if aqlErr != nil || pgErr != nil { + t.Fatalf("aql err = %v, pg err = %v", aqlErr, pgErr) + } + if !reflect.DeepEqual(aql, pg) { + t.Fatalf("overflow results diverge: aql = %#v, pg = %#v", aql, pg) + } + want := logicalValues[:2] + if !reflect.DeepEqual(aql.Values, want) || aql.Complete || !aql.Overflow { + t.Fatalf("overflow result = %#v, want values %#v with Complete=false Overflow=true", aql, want) + } + }) + t.Run("refusal of malformed raw JSON stays PostgreSQL-only", func(t *testing.T) { + malformed := rowsFor(t, command.CompiledBackendPostgreSQL, len(logicalValues)) + malformed[len(malformed)-1]["value"] = json.RawMessage(`{oops`) + pgResult, pgErr := scan(t, command.CompiledBackendPostgreSQL, malformed, 8) + if pgErr == nil || !strings.Contains(pgErr.Error(), "invalid JSONB value") { + t.Fatalf("malformed raw JSON accepted: result %#v, err %v", pgResult, pgErr) + } + if pgResult.Complete || pgResult.Overflow || pgResult.ConclusiveMissing || len(pgResult.Values) != 0 { + t.Fatalf("refused scan must be a zero result, got %#v", pgResult) + } + aqlResult, aqlErr := scan(t, command.CompiledBackendAQL, rowsFor(t, command.CompiledBackendAQL, len(logicalValues)), 8) + if aqlErr != nil || !aqlResult.Complete || !reflect.DeepEqual(aqlResult.Values, logicalValues) { + t.Fatalf("AQL twin of the logical page failed: result %#v, err %v", aqlResult, aqlErr) + } + }) +} + +func exec06ConstructionBundle() recipe.Bundle { + two := int64(2) + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "exec06_identity", + TranslationVersion: "exec06", + Outputs: []recipe.Output{{ + Name: "trace", RootResourceType: "Observation", RowGrain: "observation", + RootColumnNaming: recipe.RootColumnNamingExact, + Fields: []recipe.Field{ + {Name: "amount", ColumnID: "amount_id", Expr: recipe.Expression{Select: "root.valueInteger"}}, + {Name: "status", ColumnID: "status_id", Expr: recipe.Expression{Select: "root.status"}}, + }, + Construction: &recipe.Construction{ + Version: 1, + SourceColumns: []recipe.StageColumn{ + {ID: "amount_id", Name: "amount"}, {ID: "status_id", Name: "status"}, + }, + Steps: []recipe.ConstructionStep{{ + ID: "derive_total", Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionDeriveOp, Derive: &recipe.ConstructionDerive{ + ConstructionID: "calc_total", OutputColumnID: "total_id", Operation: recipe.DerivedAdd, + Left: recipe.ConstructionOperand{Kind: recipe.DerivedColumnOperand, ColumnID: "amount_id"}, + Right: recipe.ConstructionOperand{Kind: recipe.DerivedLiteralOperand, Literal: &recipe.DerivedLiteral{Kind: recipe.NumericInteger, Integer: &two}}, + MissingInputPolicy: recipe.MissingInputPropagateNull, + }}, + Outputs: []recipe.StageColumn{ + {ID: "amount_id", Name: "amount"}, {ID: "status_id", Name: "status"}, {ID: "total_id", Name: "total"}, + }, + }}, + }, + }}, + } +} + +func exec06Resolve(t *testing.T) dataframeexecution.Resolved { + t.Helper() + bindings := recipe.RuntimeBindings{Project: "exec06-identity", DatasetGeneration: "generation-1"} + plan, err := semantic.BuildRecipePlan(exec06ConstructionBundle(), bindings) + if err != nil { + t.Fatalf("semantic build: %v", err) + } + resolvedPlan, err := semantic.ResolveRecipePlan(plan, bindings.Project, bindings.DatasetGeneration) + if err != nil { + t.Fatalf("semantic resolve: %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolvedPlan, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("lower compile: %v", err) + } + return dataframeexecution.Resolved{Compiled: compiled} +} + +func TestExec06CategoryIdentityRefusalsStayTypedAtTheCompileEntry(t *testing.T) { + resolved := exec06Resolve(t) + if len(resolved.Compiled.Outputs) != 1 || len(resolved.Compiled.Outputs[0].Stages) == 0 { + t.Fatalf("fixture: resolved output carries no stage descriptors: %#v", resolved.Compiled.Outputs) + } + cases := []struct { + name string + request dataframeexecution.CategoryScanRequest + code compiler.CategoryScanRefusalCode + }{ + {"output form names an absent column", + dataframeexecution.CategoryScanRequest{Output: "trace", Column: "nope", MaxValues: 256}, + compiler.CategoryScanColumnUnknown}, + {"stage form names an absent stage", + dataframeexecution.CategoryScanRequest{Output: "trace", StageID: "nope_stage", ColumnID: "status_id", ValueColumnID: "amount_id", MaxValues: 256}, + compiler.CategoryScanStageUnknown}, + {"stage form names an absent category column", + dataframeexecution.CategoryScanRequest{Output: "trace", StageID: recipe.ConstructionSourceProjectionID, ColumnID: "nope_id", ValueColumnID: "amount_id", MaxValues: 256}, + compiler.CategoryScanColumnUnknown}, + {"stage form names an absent value column", + dataframeexecution.CategoryScanRequest{Output: "trace", StageID: recipe.ConstructionSourceProjectionID, ColumnID: "status_id", ValueColumnID: "nope_id", MaxValues: 256}, + compiler.CategoryScanColumnUnknown}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + port := &exec06Port{} + engine := exec06Engine(t, command.CompiledBackendPostgreSQL, port) + result, err := engine.ScanCategories(context.Background(), resolved, test.request) + if err == nil { + t.Fatalf("absent identity accepted: result %#v", result) + } + code, ok := compiler.CategoryScanRefusalCodeOf(err) + if !ok || code != test.code { + t.Fatalf("refusal = %v (code %q, ok %v), want typed %q from the compiler entry", err, code, ok, test.code) + } + if result.Complete || result.Overflow || result.ConclusiveMissing || len(result.Values) != 0 { + t.Fatalf("refusal softened into a result page: %#v", result) + } + if port.counts() != 0 { + t.Fatalf("port calls = %d, want zero — the compile entry refuses before execution", port.counts()) + } + }) + } + t.Run("absent output is the executor layer, not a compiler refusal", func(t *testing.T) { + port := &exec06Port{} + engine := exec06Engine(t, command.CompiledBackendPostgreSQL, port) + _, err := engine.ScanCategories(context.Background(), resolved, dataframeexecution.CategoryScanRequest{Output: "nope", Column: "status", MaxValues: 256}) + if err == nil || !strings.Contains(err.Error(), "was not found") { + t.Fatalf("output-name refusal = %v, want the executor's own not-found error", err) + } + if _, ok := compiler.CategoryScanRefusalCodeOf(err); ok { + t.Fatalf("output-name lookup must stay the executor layer, got a compiler refusal code: %v", err) + } + if port.counts() != 0 { + t.Fatalf("port calls = %d, want zero", port.counts()) + } + }) + t.Run("valid source_projection stage scan proceeds", func(t *testing.T) { + port := &exec06Port{rows: []map[string]any{{"present": true, "value": exec06JSONB(t, `"final"`)}}} + engine := exec06Engine(t, command.CompiledBackendPostgreSQL, port) + result, err := engine.ScanCategories(context.Background(), resolved, dataframeexecution.CategoryScanRequest{ + Output: "trace", StageID: recipe.ConstructionSourceProjectionID, + ColumnID: "status_id", ValueColumnID: "amount_id", MaxValues: 256, + }) + if err != nil { + t.Fatalf("valid stage-form scan: %v", err) + } + if !result.Complete || len(result.Values) != 1 || result.Values[0] != (dataframeexecution.CategoryValue{Present: true, Value: "final"}) { + t.Fatalf("stage-form result = %#v", result) + } + if result.Proof.Version != 2 || result.Proof.StageID != recipe.ConstructionSourceProjectionID || + result.Proof.ColumnID != "status_id" || result.Proof.ValueColumnID != "amount_id" { + t.Fatalf("stage proof not bound: %#v", result.Proof) + } + if port.counts() != 1 || port.calls[0].tag != command.CompiledBackendPostgreSQL { + t.Fatalf("calls = %#v, want one PostgreSQL statement", port.calls) + } + }) +} diff --git a/internal/dataframe/execution/category_scan_test.go b/internal/dataframe/execution/category_scan_test.go index e070e2cb8..d5339391b 100644 --- a/internal/dataframe/execution/category_scan_test.go +++ b/internal/dataframe/execution/category_scan_test.go @@ -20,7 +20,7 @@ import ( func compiledCategoryScan(max int) compiler.CompiledCategoryScanQuery { return compiler.CompiledCategoryScanQuery{ - Query: "scan", BindVars: map[string]any{"column": "category"}, PresentColumn: "present", ValueColumn: "value", + Command: mustAQLCommand("scan", map[string]any{"column": "category"}), PresentColumn: "present", ValueColumn: "value", Proof: compiler.CategoryScanProof{Version: 1, Output: "output", Column: "category", MaxValues: max, Fingerprint: "proof"}, } } @@ -34,7 +34,13 @@ func TestScanCategoriesCompiledReportsCompleteAtBoundAndOverflowAtBoundPlusOne(t {257, 256, false, true}, } { t.Run(fmt.Sprintf("rows_%d", test.rows), func(t *testing.T) { - engine := &Engine{queryRows: func(_ context.Context, query string, _ int, binds map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + query, binds, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if query != "scan" || binds["column"] != "category" { t.Fatalf("wrong invocation: %q %#v", query, binds) } @@ -67,9 +73,15 @@ func TestScanCategoriesCompiledUsesExactOverflowWitnessBeforeFullScan(t *testing } { t.Run(test.name, func(t *testing.T) { compiled := compiledCategoryScan(2) - compiled.OverflowWitness = &compiler.CategoryOverflowWitness{Query: "witness", BindVars: map[string]any{"scope": "same"}} + compiled.OverflowWitness = &compiler.CategoryOverflowWitness{Command: mustAQLCommand("witness", map[string]any{"scope": "same"})} fullScan := false - engine := &Engine{queryRows: func(_ context.Context, query string, _ int, binds map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + query, binds, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if query == "witness" { if binds["scope"] != "same" { t.Fatalf("witness binds = %#v", binds) @@ -114,7 +126,13 @@ func TestScanCategoriesCompiledPreservesOrderedMissingNullAndFalseyValues(t *tes {"present": true, "value": float64(0)}, {"present": true, "value": ""}, } - engine := &Engine{queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + for _, row := range rows { if err := visit(row); err != nil { return err @@ -135,9 +153,15 @@ func TestScanCategoriesCompiledPreservesOrderedMissingNullAndFalseyValues(t *tes func TestScanCategoriesCompiledContinuesAfterMissingWitnessWhenPresenceIsTracked(t *testing.T) { compiled := compiledCategoryScan(4) compiled.Proof.PresenceTracked = true - compiled.OverflowWitness = &compiler.CategoryOverflowWitness{Query: "witness"} + compiled.OverflowWitness = &compiler.CategoryOverflowWitness{Command: mustAQLCommand("witness", nil)} mainScanned := false - engine := &Engine{queryRows: func(_ context.Context, query string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + query, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + switch query { case "witness": return visit(map[string]any{"present": false, "value": nil}) @@ -163,7 +187,13 @@ func TestScanCategoriesCompiledContinuesAfterMissingWitnessWhenPresenceIsTracked func TestScanCategoriesCompiledHonorsContextCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) - engine := &Engine{queryRows: func(ctx context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + cancel() return visit(map[string]any{"present": true, "value": "unreachable"}) }} @@ -175,12 +205,24 @@ func TestScanCategoriesCompiledHonorsContextCancellation(t *testing.T) { func TestScanCategoriesCompiledUsesBoundedPreviewExecutor(t *testing.T) { called := false - engine := &Engine{ - queryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), + sourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + t.Fatal("category discovery used the unbounded execution query path") return nil }, - previewQueryRows: func(_ context.Context, query string, _ int, _ map[string]any, visit func(map[string]any) error) error { + previewSourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + query, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + called = true if query != "scan" { t.Fatalf("query = %q", query) @@ -200,8 +242,14 @@ func TestScanCategoriesCompiledPrewarmsPreviewIndexWithoutBlockingAndDedupes(t * prepareFinished := make(chan struct{}) var prepareCalls atomic.Int32 var queryCalls atomic.Int32 - engine := &Engine{ - queryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), + sourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + t.Fatal("category discovery used the unbounded executor") return nil }, @@ -219,7 +267,13 @@ func TestScanCategoriesCompiledPrewarmsPreviewIndexWithoutBlockingAndDedupes(t * return ctx.Err() } }, - previewQueryRows: func(_ context.Context, query string, _ int, _ map[string]any, visit func(map[string]any) error) error { + previewSourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + query, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if query != "scan" { t.Fatalf("query = %q", query) } @@ -263,7 +317,7 @@ func TestScanCategoriesReturnsTypedCompilerRefusalForUnsupportedColumn(t *testin Name: "structured", Kind: string(expression.KindObject), Cardinality: string(expression.OptionalOne), }}, }}}} - _, err := (&Engine{}).ScanCategories(context.Background(), resolved, CategoryScanRequest{Output: "output", Column: "structured", MaxValues: 256}) + _, err := (&Engine{compileTarget: compiler.ArangoTarget()}).ScanCategories(context.Background(), resolved, CategoryScanRequest{Output: "output", Column: "structured", MaxValues: 256}) code, ok := compiler.CategoryScanRefusalCodeOf(err) if !ok || code != compiler.CategoryScanColumnUnsupported { t.Fatalf("refusal = %q/%t, error = %v", code, ok, err) @@ -291,7 +345,13 @@ func TestScanCategoriesExecutesExactConstructionStagePrefix(t *testing.T) { t.Fatal(err) } var query string - engine := &Engine{queryRows: func(_ context.Context, queryText string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + query = queryText return visit(map[string]any{"present": true, "value": "final"}) }} @@ -312,12 +372,18 @@ func TestScanCategoriesExecutesExactConstructionStagePrefix(t *testing.T) { func TestScanCategoriesPreparesCategoryIndexBeforeQuery(t *testing.T) { prepared := false - engine := &Engine{ + engine := &Engine{compileTarget: compiler.ArangoTarget(), preparePreviewIndex: func(ctx context.Context, spec compiler.PreviewCoveringIndexSpec) error { prepared = true return nil }, - queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if !prepared { t.Fatal("category query ran before its index was prepared") } diff --git a/internal/dataframe/execution/cell_trace.go b/internal/dataframe/execution/cell_trace.go index d37b662a4..b14449a97 100644 --- a/internal/dataframe/execution/cell_trace.go +++ b/internal/dataframe/execution/cell_trace.go @@ -11,15 +11,13 @@ import ( "strings" "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" ) const defaultCellTraceWitnessRows int64 = 1_000_000 -// CellTraceStatus describes what the canonical output plan established for -// one published cell. Incomplete is reserved for a bounded trace that could -// not reach the requested row; it must never be presented as absence. type CellTraceStatus string const ( @@ -71,34 +69,25 @@ type CellTraceResult struct { var errCellTraceFound = errors.New("cell trace row found") var errCellTraceWitnessLimit = errors.New("cell trace witness limit exceeded") -// ErrCellTraceRowNotFound marks a client-supplied row ID that matched no -// published row in an existing output; the authoring transport presents it as -// a typed not-found, and only drift between bundle and compiled outputs can -// keep it from being resolved upstream. var ErrCellTraceRowNotFound = errors.New("published cell trace row was not found") -// CellTrace executes a diagnostic terminal compiled from the same resolved -// output as publication. It locates a published row by its stable opaque ID; -// callers never submit FHIR paths or expressions for interpretation here. func (e *Engine) CellTrace(ctx context.Context, resolved Resolved, request CellTraceRequest) (CellTraceResult, error) { if strings.TrimSpace(request.Output) == "" { return CellTraceResult{}, fmt.Errorf("cell trace output is required") } - compiled, err := compileCellTraceOutput(resolved, request) + compiled, err := compileCellTraceOutput(e.compileTarget, resolved, request) if err != nil { return CellTraceResult{}, err } return e.cellTraceCompiled(ctx, compiled, request) } -// CellTraceCompiled is the narrow execution seam used by focused tests and -// specialized adapters that already hold the compiler result. func (e *Engine) CellTraceCompiled(ctx context.Context, compiled compiler.CompiledCellTraceQuery, request CellTraceRequest) (CellTraceResult, error) { return e.cellTraceCompiled(ctx, compiled, request) } func (e *Engine) cellTraceCompiled(ctx context.Context, compiled compiler.CompiledCellTraceQuery, request CellTraceRequest) (CellTraceResult, error) { - if e == nil || e.queryRows == nil { + if e == nil || e.sourceRows == nil { return CellTraceResult{}, fmt.Errorf("cell trace query executor is required") } rowID := strings.TrimSpace(request.RowID) @@ -122,9 +111,45 @@ func (e *Engine) cellTraceCompiled(ctx context.Context, compiled compiler.Compil return CellTraceResult{}, fmt.Errorf("cell trace query is missing typed result columns") } + dialect := cellTraceAQLDialect + var declaredColumns []string + switch compiled.Command.Backend() { + case command.CompiledBackendPostgreSQL: + identityColumn := compiled.IdentityPartsColumn + if compiled.ExplicitIdentityColumn != "" { + identityColumn = compiled.ExplicitIdentityColumn + } + declaredColumns = []string{compiled.ValueColumn, compiled.ContributionsColumn, identityColumn, + compiled.StatusColumn, compiled.HasMoreColumn, compiled.OmissionColumn} + for i := range declaredColumns { + for j := i + 1; j < len(declaredColumns); j++ { + if declaredColumns[i] == declaredColumns[j] { + return CellTraceResult{}, fmt.Errorf("cell trace query is missing typed result columns") + } + } + } + dialect = cellTracePostgreSQLDialect + case command.CompiledBackendAQL: + default: + return CellTraceResult{}, fmt.Errorf("%w: cell trace command backend %q is not registered", + command.ErrCompiledCommandBackendNotRegistered, compiled.Command.Backend()) + } + return e.scanCellTraceEvidence(ctx, compiled, request, rowID, column, maxWitness, declaredColumns, dialect) +} + +type cellTraceDialect struct { + identity func(compiler.CompiledCellTraceQuery, map[string]any) (string, error) + parse func(compiler.CompiledCellTraceQuery, CellTraceRequest, map[string]any) (CellTraceResult, error) +} + +var cellTraceAQLDialect = cellTraceDialect{identity: cellTraceRowIdentity, parse: parseCellTraceRow} + +var cellTracePostgreSQLDialect = cellTraceDialect{identity: cellTracePostgreSQLIdentity, parse: parseCellTracePostgreSQLRow} + +func (e *Engine) scanCellTraceEvidence(ctx context.Context, compiled compiler.CompiledCellTraceQuery, request CellTraceRequest, rowID, column string, maxWitness int64, declaredColumns []string, dialect cellTraceDialect) (CellTraceResult, error) { var result CellTraceResult witnessRows := int64(0) - err := e.queryRows(ctx, compiled.Query, e.batchSize, compiled.BindVars, func(row map[string]any) error { + err := e.sourceRows(ctx, RowsRequest{Command: compiled.Command, DeclaredColumns: declaredColumns, BatchSize: e.batchSize}, func(row map[string]any) error { if err := ctx.Err(); err != nil { return err } @@ -132,14 +157,14 @@ func (e *Engine) cellTraceCompiled(ctx context.Context, compiled compiler.Compil if witnessRows > maxWitness { return errCellTraceWitnessLimit } - identity, identityErr := cellTraceRowIdentity(compiled, row) + identity, identityErr := dialect.identity(compiled, row) if identityErr != nil { return identityErr } if identity != rowID { return nil } - parsed, parseErr := parseCellTraceRow(compiled, request, row) + parsed, parseErr := dialect.parse(compiled, request, row) if parseErr != nil { return parseErr } @@ -161,10 +186,10 @@ func (e *Engine) cellTraceCompiled(ctx context.Context, compiled compiler.Compil return CellTraceResult{}, dataframeerrors.Wrap(ErrCellTraceRowNotFound, dataframeerrors.CodeInvalidRequest, fmt.Sprintf("published row %q was not found in output %q", rowID, request.Output), dataframeerrors.WithDetails(map[string]any{"rowId": rowID, "output": request.Output})) } -func compileCellTraceOutput(resolved Resolved, request CellTraceRequest) (compiler.CompiledCellTraceQuery, error) { +func compileCellTraceOutput(target compiler.Target, resolved Resolved, request CellTraceRequest) (compiler.CompiledCellTraceQuery, error) { for _, output := range resolved.Compiled.Outputs { if output.Name == request.Output { - return compiler.CompileCellTraceOutputWithPolicy(output, request.Column, request.Offset, request.Limit, ir.DefaultPhysicalOptimizationPolicy()) + return compiler.CompileCellTraceOutput(target, output, request.Column, request.Offset, request.Limit, ir.DefaultPhysicalOptimizationPolicy()) } } return compiler.CompiledCellTraceQuery{}, fmt.Errorf("cell trace output %q was not found", request.Output) @@ -185,7 +210,14 @@ func cellTraceRowIdentity(query compiler.CompiledCellTraceQuery, row map[string] return fmt.Sprint(value), nil } parts, ok := row[query.IdentityPartsColumn].([]any) - if !ok || query.RowIdentity == nil || len(parts) == 0 || len(parts) != len(query.RowIdentity.Fields) { + if !ok { + return "", fmt.Errorf("cell trace row has an invalid default identity shape") + } + return cellTraceIdentityPartsDigest(parts, query) +} + +func cellTraceIdentityPartsDigest(parts []any, query compiler.CompiledCellTraceQuery) (string, error) { + if query.RowIdentity == nil || len(parts) == 0 || len(parts) != len(query.RowIdentity.Fields) { return "", fmt.Errorf("cell trace row has an invalid default identity shape") } for _, part := range parts { @@ -201,9 +233,64 @@ func cellTraceRowIdentity(query compiler.CompiledCellTraceQuery, row map[string] return hex.EncodeToString(digest[:]), nil } -// The compiler publishes a singleton __loom_row_id for composed cohort -// outputs; its cohort stage defines that identity as a required object. Other -// construction identities retain their scalar or identity-parts metadata. +func cellTracePostgreSQLIdentity(query compiler.CompiledCellTraceQuery, row map[string]any) (string, error) { + if query.ExplicitIdentityColumn != "" { + value, ok := row[query.ExplicitIdentityColumn] + if raw, isRaw := value.(json.RawMessage); isRaw { + decoded, err := decodePostgreSQLCellTraceJSON(raw) + if err != nil { + return "", fmt.Errorf("cell trace row has an invalid explicit identity: %w", err) + } + value, ok = decoded, true + } + if !ok || value == nil { + return "", fmt.Errorf("cell trace row has an invalid explicit identity") + } + if query.ExplicitIdentityObject { + return canonicalCellTraceObjectValue(value) + } + if strings.TrimSpace(fmt.Sprint(value)) == "" { + return "", fmt.Errorf("cell trace row has an invalid explicit identity") + } + return fmt.Sprint(value), nil + } + parts, err := cellTracePostgreSQLJSONParts(row[query.IdentityPartsColumn]) + if err != nil { + return "", fmt.Errorf("cell trace row has an invalid default identity shape") + } + return cellTraceIdentityPartsDigest(parts, query) +} + +func cellTracePostgreSQLJSONParts(value any) ([]any, error) { + switch typed := value.(type) { + case []any: + return typed, nil + case json.RawMessage: + decoded, err := decodePostgreSQLCellTraceJSON(typed) + if err != nil { + return nil, err + } + items, ok := decoded.([]any) + if !ok { + return nil, fmt.Errorf("cell trace row carries a non-array JSON value where an array page is published") + } + return items, nil + default: + return nil, fmt.Errorf("cell trace row is missing its published JSON array") + } +} + +func decodePostgreSQLCellTraceJSON(raw json.RawMessage) (any, error) { + if len(raw) == 0 || !json.Valid(raw) { + return nil, fmt.Errorf("cell trace row carries invalid raw JSON bytes") + } + var decoded any + if err := json.Unmarshal(raw, &decoded); err != nil { + return nil, fmt.Errorf("cell trace row carries invalid raw JSON bytes: %w", err) + } + return decoded, nil +} + func canonicalCellTraceObjectValue(value any) (string, error) { encoded, err := json.Marshal(value) if err != nil { @@ -266,12 +353,21 @@ func canonicalCellTraceIdentityJSON(raw string) (string, error) { return string(canonical), nil } -func parseCellTraceRow(query compiler.CompiledCellTraceQuery, request CellTraceRequest, row map[string]any) (CellTraceResult, error) { +type cellTraceDecodedRow struct { + value any + contributions []CellTraceContribution +} + +func cellTraceRowStatus(query compiler.CompiledCellTraceQuery, row map[string]any) (CellTraceStatus, error) { status, ok := row[query.StatusColumn].(string) if !ok || !validCellTraceStatus(CellTraceStatus(status)) { - return CellTraceResult{}, fmt.Errorf("cell trace row has an invalid status") + return "", fmt.Errorf("cell trace row has an invalid status") } - contributions, err := parseCellTraceContributions(row[query.ContributionsColumn]) + return CellTraceStatus(status), nil +} + +func assembleCellTraceResult(query compiler.CompiledCellTraceQuery, request CellTraceRequest, row map[string]any, decoded cellTraceDecodedRow) (CellTraceResult, error) { + status, err := cellTraceRowStatus(query, row) if err != nil { return CellTraceResult{}, err } @@ -279,15 +375,49 @@ func parseCellTraceRow(query compiler.CompiledCellTraceQuery, request CellTraceR omission, _ := row[query.OmissionColumn].(string) nextOffset := 0 if hasMore { - nextOffset = request.Offset + len(contributions) + nextOffset = request.Offset + len(decoded.contributions) } return CellTraceResult{ - RowID: strings.TrimSpace(request.RowID), Column: strings.TrimSpace(request.Column), Value: row[query.ValueColumn], - Status: CellTraceStatus(status), Contributions: contributions, HasMore: hasMore, NextOffset: nextOffset, + RowID: strings.TrimSpace(request.RowID), Column: strings.TrimSpace(request.Column), Value: decoded.value, + Status: status, Contributions: decoded.contributions, HasMore: hasMore, NextOffset: nextOffset, OmissionCode: omission, Complete: true, }, nil } +func parseCellTraceRow(query compiler.CompiledCellTraceQuery, request CellTraceRequest, row map[string]any) (CellTraceResult, error) { + if _, err := cellTraceRowStatus(query, row); err != nil { + return CellTraceResult{}, err + } + contributions, err := parseCellTraceContributions(row[query.ContributionsColumn]) + if err != nil { + return CellTraceResult{}, err + } + return assembleCellTraceResult(query, request, row, cellTraceDecodedRow{value: row[query.ValueColumn], contributions: contributions}) +} + +func parseCellTracePostgreSQLRow(query compiler.CompiledCellTraceQuery, request CellTraceRequest, row map[string]any) (CellTraceResult, error) { + if _, err := cellTraceRowStatus(query, row); err != nil { + return CellTraceResult{}, err + } + items, err := cellTracePostgreSQLJSONParts(row[query.ContributionsColumn]) + if err != nil { + return CellTraceResult{}, fmt.Errorf("cell trace row has invalid contributions: %w", err) + } + contributions, err := cellTraceContributionsFromItems(items) + if err != nil { + return CellTraceResult{}, err + } + value := row[query.ValueColumn] + if raw, isRaw := value.(json.RawMessage); isRaw { + decoded, decodeErr := decodePostgreSQLCellTraceJSON(raw) + if decodeErr != nil { + return CellTraceResult{}, fmt.Errorf("cell trace row has an invalid traced value: %w", decodeErr) + } + value = decoded + } + return assembleCellTraceResult(query, request, row, cellTraceDecodedRow{value: value, contributions: contributions}) +} + func parseCellTraceContributions(value any) ([]CellTraceContribution, error) { if value == nil { return []CellTraceContribution{}, nil @@ -296,6 +426,10 @@ func parseCellTraceContributions(value any) ([]CellTraceContribution, error) { if !ok { return nil, fmt.Errorf("cell trace row has invalid contributions") } + return cellTraceContributionsFromItems(items) +} + +func cellTraceContributionsFromItems(items []any) ([]CellTraceContribution, error) { result := make([]CellTraceContribution, 0, len(items)) for _, item := range items { object, ok := item.(map[string]any) diff --git a/internal/dataframe/execution/cell_trace_postgres_test.go b/internal/dataframe/execution/cell_trace_postgres_test.go new file mode 100644 index 000000000..d2fd7c70b --- /dev/null +++ b/internal/dataframe/execution/cell_trace_postgres_test.go @@ -0,0 +1,505 @@ +package execution + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + "github.com/calypr/loom/internal/dataframe/spec" +) + +func exec09AdapterRow(t *testing.T, identity any, value any, status string, hasMore bool, contributions any) map[string]any { + t.Helper() + return map[string]any{ + ir.PhysicalCellTraceIdentityPartsField: identity, + ir.PhysicalCellTraceValueField: value, + ir.PhysicalCellTraceStatusField: status, + ir.PhysicalCellTraceHasMoreField: hasMore, + ir.PhysicalCellTraceOmissionField: "", + ir.PhysicalCellTraceContributionsField: contributions, + } +} + +func exec09ArmB(t *testing.T, offset, limit int) (compiler.CompiledCellTraceQuery, Resolved) { + t.Helper() + resolved := exec02TestResolved() + compiled, err := compiler.CompileCellTraceOutput(exec02PostgreSQLTarget(t), + resolved.Compiled.Outputs[0], "name", offset, limit, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("registered arm B cell trace must compile on the PostgreSQL target: %v", err) + } + if compiled.Command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("compiled trace must carry the PostgreSQL tag, got %q", compiled.Command.Backend()) + } + return compiled, resolved +} + +func exec09PreviewRowID(t *testing.T, identity *spec.RowIdentity, row map[string]any) string { + t.Helper() + if err := ensureStableRowIdentity(row, identity, nil); err != nil { + t.Fatalf("preview-side identity mint: %v", err) + } + published, ok := row["__loom_row_id"].(string) + if !ok || published == "" { + t.Fatalf("preview-side identity mint produced no published ID: %#v", row) + } + return published +} + +func exec09Request(rowID string) CellTraceRequest { + return CellTraceRequest{Output: "resources", RowID: rowID, Column: "name", Offset: 0, Limit: 2} +} + +func TestPostgresCellTraceOfflineArmCarriesSealedDeclaredContract(t *testing.T) { + compiled, resolved := exec09ArmB(t, 0, 2) + rowID := exec09PreviewRowID(t, resolved.Compiled.Outputs[0].RowIdentity, map[string]any{"id": "patient-one"}) + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, []map[string]any{exec09AdapterRow(t, + json.RawMessage(`["patient-one"]`), json.RawMessage(`"Ada"`), "VALUE", false, + json.RawMessage(`[{"resourceType":"Patient","resourceId":"patient-one","operation":"EXTRACT","value":"female"}]`))}) + result, err := engine.CellTraceCompiled(context.Background(), compiled, exec09Request(rowID)) + if err != nil { + t.Fatalf("arm B trace over adapter-shaped rows: %v", err) + } + if !result.Complete || result.Status != CellTraceValue || result.Value != "Ada" { + t.Fatalf("adapter-shaped VALUE = %#v", result) + } + if len(spies.pgFull) != 1 || spies.pgRequests != 1 { + t.Fatalf("exactly one full-port call expected: full=%d total=%d", len(spies.pgFull), spies.pgRequests) + } + call := spies.pgFull[0] + want := []string{ + ir.PhysicalCellTraceValueField, ir.PhysicalCellTraceContributionsField, + ir.PhysicalCellTraceIdentityPartsField, ir.PhysicalCellTraceStatusField, + ir.PhysicalCellTraceHasMoreField, ir.PhysicalCellTraceOmissionField, + } + if strings.Join(call.Columns, ",") != strings.Join(want, ",") { + t.Fatalf("declared contract = %#v, want the sealed renderer order %#v", call.Columns, want) + } + if len(call.Args) == 0 { + t.Fatal("every scope and page fact must travel as an ordered parameter") + } + for _, leak := range []string{"patient-one", "female", "Ada"} { + if strings.Contains(call.SQL, leak) { + t.Fatalf("traced literal %q leaked into the SQL text:\n%s", leak, call.SQL) + } + } + exec02AssertUntouchedArango(t, spies) +} + +func TestPostgresCellTraceOfflineIdentityPartsDigestConvergesWithPreview(t *testing.T) { + compiled, resolved := exec09ArmB(t, 0, 2) + rowID := exec09PreviewRowID(t, resolved.Compiled.Outputs[0].RowIdentity, map[string]any{"id": "patient-one"}) + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, []map[string]any{ + exec09AdapterRow(t, json.RawMessage(`["patient-two"]`), json.RawMessage(`"Grace"`), "VALUE", false, json.RawMessage(`[]`)), + exec09AdapterRow(t, json.RawMessage(`["patient-one"]`), json.RawMessage(`"Ada"`), "VALUE", true, + json.RawMessage(`[{"resourceType":"Patient","resourceId":"patient-one","inputStageId":"source_projection","inputColumnId":"gender_id","inputColumn":"gender","operation":"EXTRACT","value":"female"},{"resourceType":"Patient","resourceId":"patient-one","operation":"EXTRACT"}]`)), + }) + result, err := engine.CellTraceCompiled(context.Background(), compiled, CellTraceRequest{Output: "resources", RowID: rowID, Column: "name", Offset: 3, Limit: 2}) + if err != nil { + t.Fatalf("identity-matched trace: %v", err) + } + if !result.Complete || result.Status != CellTraceValue || result.Value != "Ada" || !result.HasMore || result.NextOffset != 5 { + t.Fatalf("paged adapter-shaped VALUE = %#v", result) + } + if len(result.Contributions) != 2 || + result.Contributions[0].ResourceType != "Patient" || result.Contributions[0].ResourceID != "patient-one" || + result.Contributions[0].InputStageID != "source_projection" || result.Contributions[0].InputColumnID != "gender_id" || + result.Contributions[0].InputColumn != "gender" || result.Contributions[0].Operation != "EXTRACT" || + result.Contributions[0].Value != "female" || + result.Contributions[1].ResourceID != "patient-one" || result.Contributions[1].Value != nil { + t.Fatalf("raw-JSON contributor page decoded wrong: %#v", result.Contributions) + } + exec02AssertUntouchedArango(t, spies) +} + +func TestPostgresCellTraceOfflineThreeStateValueDecode(t *testing.T) { + compiled, resolved := exec09ArmB(t, 0, 2) + rowID := exec09PreviewRowID(t, resolved.Compiled.Outputs[0].RowIdentity, map[string]any{"id": "patient-one"}) + for _, test := range []struct { + name string + value any + status string + want any + wantKind CellTraceStatus + }{ + {name: "sql-null", value: nil, status: "NO_MATCH", want: nil, wantKind: CellTraceNoMatch}, + {name: "recorded-json-null", value: json.RawMessage(`null`), status: "RECORDED_NULL", want: nil, wantKind: CellTraceRecordedNull}, + {name: "json-number", value: json.RawMessage(`3`), status: "VALUE", want: float64(3), wantKind: CellTraceValue}, + {name: "json-boolean", value: json.RawMessage(`true`), status: "VALUE", want: true, wantKind: CellTraceValue}, + {name: "native-passthrough-convergence", value: "Ada", status: "VALUE", want: "Ada", wantKind: CellTraceValue}, + } { + t.Run(test.name, func(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, []map[string]any{exec09AdapterRow(t, + json.RawMessage(`["patient-one"]`), test.value, test.status, false, json.RawMessage(`[]`))}) + result, err := engine.CellTraceCompiled(context.Background(), compiled, exec09Request(rowID)) + if err != nil { + t.Fatalf("three-state decode (%s): %v", test.name, err) + } + if result.Value != test.want || result.Status != test.wantKind || !result.Complete { + t.Fatalf("three-state decode (%s) = %#v, want value %#v status %s", test.name, result, test.want, test.wantKind) + } + }) + } +} + +func TestPostgresCellTraceOfflinePoisonMatrixRefusesFailClosed(t *testing.T) { + compiled, resolved := exec09ArmB(t, 0, 2) + rowID := exec09PreviewRowID(t, resolved.Compiled.Outputs[0].RowIdentity, map[string]any{"id": "patient-one"}) + poison := func(label string, poisonRow map[string]any) { + t.Helper() + t.Run(label, func(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, []map[string]any{poisonRow}) + result, err := engine.CellTraceCompiled(context.Background(), compiled, exec09Request(rowID)) + if err == nil { + t.Fatalf("poison shape %s was accepted: %#v", label, result) + } + if errors.Is(err, ErrCellTraceRowNotFound) { + t.Fatalf("poison shape %s was skipped as row absence, not refused: %v", label, err) + } + if result.Status != "" || result.Complete || result.Value != nil || result.Contributions != nil { + t.Fatalf("poison shape %s returned a partial result: %#v", label, result) + } + if len(spies.pgFull) != 1 { + t.Fatalf("poison shape %s must fail after the single port call: %d", label, len(spies.pgFull)) + } + }) + } + good := exec09AdapterRow(t, json.RawMessage(`["patient-one"]`), json.RawMessage(`"Ada"`), "VALUE", false, json.RawMessage(`[]`)) + for _, page := range []any{nil, json.RawMessage(``), json.RawMessage(`{bad`), json.RawMessage(`null`), json.RawMessage(`{"a":1}`), "[]", 42} { + bad := map[string]any{} + for k, v := range good { + bad[k] = v + } + bad[ir.PhysicalCellTraceContributionsField] = page + poison("contributions-page", bad) + } + for _, value := range []any{json.RawMessage(``), json.RawMessage(`{bad`)} { + bad := map[string]any{} + for k, v := range good { + bad[k] = v + } + bad[ir.PhysicalCellTraceValueField] = value + poison("traced-value", bad) + } + for _, identity := range []any{nil, json.RawMessage(``), json.RawMessage(`{bad`), json.RawMessage(`null`), + json.RawMessage(`["patient-one","extra"]`), json.RawMessage(`[null]`), json.RawMessage(`"patient-one"`), 7} { + bad := map[string]any{} + for k, v := range good { + bad[k] = v + } + bad[ir.PhysicalCellTraceIdentityPartsField] = identity + poison("identity-parts", bad) + } + for _, status := range []any{nil, "", "MAYBE", json.RawMessage(`"VALUE"`)} { + bad := map[string]any{} + for k, v := range good { + bad[k] = v + } + bad[ir.PhysicalCellTraceStatusField] = status + poison("status", bad) + } + bad := map[string]any{} + for k, v := range good { + bad[k] = v + } + bad[ir.PhysicalCellTraceContributionsField] = json.RawMessage(`["not-an-object"]`) + poison("contribution-member", bad) + bad[ir.PhysicalCellTraceContributionsField] = json.RawMessage(`[42]`) + poison("contribution-member-number", bad) + bad[ir.PhysicalCellTraceContributionsField] = json.RawMessage(`[null]`) + poison("contribution-member-null", bad) +} + +func TestPostgresCellTraceOfflineExplicitIdentityRawShapes(t *testing.T) { + pgCommand := exec02PostgreSQLCommand(t) + newQuery := func(mutate func(*compiler.CompiledCellTraceQuery)) compiler.CompiledCellTraceQuery { + query := compiler.CompiledCellTraceQuery{ + Command: pgCommand, ValueColumn: ir.PhysicalCellTraceValueField, ContributionsColumn: ir.PhysicalCellTraceContributionsField, + StatusColumn: ir.PhysicalCellTraceStatusField, HasMoreColumn: ir.PhysicalCellTraceHasMoreField, + OmissionColumn: ir.PhysicalCellTraceOmissionField, ExplicitIdentityColumn: ir.PhysicalCellTraceExplicitIdentityField, + } + mutate(&query) + return query + } + t.Run("scalar", func(t *testing.T) { + query := newQuery(func(*compiler.CompiledCellTraceQuery) {}) + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, []map[string]any{{ + ir.PhysicalCellTraceExplicitIdentityField: json.RawMessage(`"patient-7"`), + ir.PhysicalCellTraceValueField: json.RawMessage(`"female"`), + ir.PhysicalCellTraceStatusField: "VALUE", ir.PhysicalCellTraceHasMoreField: false, + ir.PhysicalCellTraceOmissionField: "", ir.PhysicalCellTraceContributionsField: json.RawMessage(`[]`), + }}) + result, err := engine.CellTraceCompiled(context.Background(), query, exec09Request("patient-7")) + if err != nil || result.Status != CellTraceValue || result.Value != "female" { + t.Fatalf("raw scalar explicit identity: result=%#v err=%v", result, err) + } + }) + t.Run("object", func(t *testing.T) { + query := newQuery(func(q *compiler.CompiledCellTraceQuery) { q.ExplicitIdentityObject = true }) + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, []map[string]any{{ + ir.PhysicalCellTraceExplicitIdentityField: json.RawMessage(`{"group_id":"cohort-a","group_revision_id":"rev-1"}`), + ir.PhysicalCellTraceValueField: json.RawMessage(`"Pair"`), + ir.PhysicalCellTraceStatusField: "VALUE", ir.PhysicalCellTraceHasMoreField: false, + ir.PhysicalCellTraceOmissionField: "", ir.PhysicalCellTraceContributionsField: json.RawMessage(`[]`), + }}) + result, err := engine.CellTraceCompiled(context.Background(), query, exec09Request(`{"group_id":"cohort-a","group_revision_id":"rev-1"}`)) + if err != nil || result.Status != CellTraceValue || result.Value != "Pair" { + t.Fatalf("raw object explicit identity: result=%#v err=%v", result, err) + } + }) + for _, forged := range []struct { + name string + raw json.RawMessage + rowID string + }{ + {name: "extra-key", raw: json.RawMessage(`{"group_id":"cohort-a","group_revision_id":"rev-1","extra":"forged"}`), rowID: `{"group_id":"cohort-a","group_revision_id":"rev-1"}`}, + {name: "blank-scalar", raw: json.RawMessage(`" "`), rowID: ""}, + {name: "json-null-scalar", raw: json.RawMessage(`null`), rowID: ""}, + } { + t.Run(forged.name, func(t *testing.T) { + query := newQuery(func(q *compiler.CompiledCellTraceQuery) { + q.ExplicitIdentityObject = strings.HasPrefix(forged.name, "extra") + }) + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, []map[string]any{{ + ir.PhysicalCellTraceExplicitIdentityField: forged.raw, + ir.PhysicalCellTraceValueField: json.RawMessage(`"x"`), + ir.PhysicalCellTraceStatusField: "VALUE", ir.PhysicalCellTraceHasMoreField: false, + ir.PhysicalCellTraceOmissionField: "", ir.PhysicalCellTraceContributionsField: json.RawMessage(`[]`), + }}) + if forged.rowID == "" { + if _, err := engine.CellTraceCompiled(context.Background(), query, exec09Request("anything")); err == nil { + t.Fatalf("forged identity %s was accepted", forged.name) + } + return + } + result, err := engine.CellTraceCompiled(context.Background(), query, exec09Request(forged.rowID)) + if err == nil || !strings.Contains(err.Error(), "identity") { + t.Fatalf("forged identity %s: result=%#v err=%v", forged.name, result, err) + } + }) + } +} + +func TestPostgresCellTraceOfflineWitnessCapReportsIncompleteNotAbsence(t *testing.T) { + compiled, resolved := exec09ArmB(t, 0, 2) + target := exec09PreviewRowID(t, resolved.Compiled.Outputs[0].RowIdentity, map[string]any{"id": "patient-three"}) + rows := []map[string]any{ + exec09AdapterRow(t, json.RawMessage(`["patient-one"]`), json.RawMessage(`"Ada"`), "VALUE", false, json.RawMessage(`[]`)), + exec09AdapterRow(t, json.RawMessage(`["patient-two"]`), json.RawMessage(`"Grace"`), "VALUE", false, json.RawMessage(`[]`)), + exec09AdapterRow(t, json.RawMessage(`["patient-three"]`), json.RawMessage(`"Radium"`), "VALUE", false, json.RawMessage(`[]`)), + } + spies := &dispatchSpies{} + capped, _ := exec02pgEngine(t, spies, rows) + result, err := capped.CellTraceCompiled(context.Background(), compiled, CellTraceRequest{Output: "resources", RowID: target, Column: "name", MaxWitnessRows: 2}) + if err != nil { + t.Fatalf("capped witness scan: %v", err) + } + if result.Status != CellTraceIncomplete || result.Complete || result.OmissionCode != "TRACE_WITNESS_INCOMPLETE" { + t.Fatalf("capped scan must report incompleteness, never absence: %#v", result) + } + spies = &dispatchSpies{} + uncapped, _ := exec02pgEngine(t, spies, rows) + result, err = uncapped.CellTraceCompiled(context.Background(), compiled, exec09Request(target)) + if err != nil || result.Status != CellTraceValue || result.Value != "Radium" || !result.Complete { + t.Fatalf("permissive-cap control must read VALUE: result=%#v err=%v", result, err) + } +} + +func TestPostgresCellTraceOfflineCrossTagAndUnregisteredRefusals(t *testing.T) { + t.Run("postgres-tag-into-aql-port", func(t *testing.T) { + compiled, _ := exec09ArmB(t, 0, 2) + calls := 0 + engine := &Engine{compileTarget: exec02PostgreSQLTarget(t), sourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + calls++ + _, _, err := request.Command.AQLParameters() + return err + }} + _, err := engine.CellTraceCompiled(context.Background(), compiled, exec09Request("whatever")) + if !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("postgres-tagged command through the AQL port = %v, want the typed mismatch", err) + } + if calls != 1 { + t.Fatalf("port call count = %d, want 1", calls) + } + }) + t.Run("aql-command-into-postgres-port", func(t *testing.T) { + resolved := exec02TestResolved() + aqlTrace, err := compiler.CompileCellTraceOutput(compiler.ArangoTarget(), + resolved.Compiled.Outputs[0], "name", 0, 2, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("AQL control compile: %v", err) + } + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + _, _, err := request.Command.PostgreSQLParameters() + return err + }} + if _, err := engine.CellTraceCompiled(context.Background(), aqlTrace, exec09Request("whatever")); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("AQL command through the PostgreSQL port = %v, want the typed mismatch", err) + } + }) + t.Run("unregistered-zero-tag", func(t *testing.T) { + calls := 0 + query := compiler.CompiledCellTraceQuery{ + Command: command.CompiledCommand{}, + ValueColumn: ir.PhysicalCellTraceValueField, ContributionsColumn: ir.PhysicalCellTraceContributionsField, + StatusColumn: ir.PhysicalCellTraceStatusField, HasMoreColumn: ir.PhysicalCellTraceHasMoreField, + OmissionColumn: ir.PhysicalCellTraceOmissionField, IdentityPartsColumn: ir.PhysicalCellTraceIdentityPartsField, + RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, + } + engine := &Engine{compileTarget: exec02PostgreSQLTarget(t), sourceRows: func(_ context.Context, _ RowsRequest, _ RowVisitor) error { + calls++ + return nil + }} + if _, err := engine.CellTraceCompiled(context.Background(), query, exec09Request("whatever")); !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("zero tag = %v, want the named not-registered refusal", err) + } + if calls != 0 { + t.Fatalf("zero-tag command reached the port %d times", calls) + } + }) +} + +func TestPostgresCellTraceOfflineSealedContractRefusals(t *testing.T) { + pgCommand := exec02PostgreSQLCommand(t) + base := func() compiler.CompiledCellTraceQuery { + return compiler.CompiledCellTraceQuery{ + Command: pgCommand, ValueColumn: ir.PhysicalCellTraceValueField, ContributionsColumn: ir.PhysicalCellTraceContributionsField, + StatusColumn: ir.PhysicalCellTraceStatusField, HasMoreColumn: ir.PhysicalCellTraceHasMoreField, + OmissionColumn: ir.PhysicalCellTraceOmissionField, IdentityPartsColumn: ir.PhysicalCellTraceIdentityPartsField, + RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, + } + } + t.Run("duplicate-contract-name", func(t *testing.T) { + query := base() + query.HasMoreColumn = query.StatusColumn + calls := 0 + engine := &Engine{compileTarget: exec02PostgreSQLTarget(t), sourceRows: func(_ context.Context, _ RowsRequest, _ RowVisitor) error { + calls++ + return nil + }} + if _, err := engine.CellTraceCompiled(context.Background(), query, exec09Request("whatever")); err == nil || + !strings.Contains(err.Error(), "missing typed result columns") || calls != 0 { + t.Fatalf("duplicate contract name: err=%v calls=%d", err, calls) + } + }) + t.Run("identity-pick-law", func(t *testing.T) { + query := base() + query.ExplicitIdentityColumn = ir.PhysicalCellTraceExplicitIdentityField + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, []map[string]any{{ + ir.PhysicalCellTraceExplicitIdentityField: json.RawMessage(`"row-a"`), + ir.PhysicalCellTraceValueField: json.RawMessage(`"x"`), + ir.PhysicalCellTraceStatusField: "VALUE", ir.PhysicalCellTraceHasMoreField: false, + ir.PhysicalCellTraceOmissionField: "", ir.PhysicalCellTraceContributionsField: json.RawMessage(`[]`), + }}) + if _, err := engine.CellTraceCompiled(context.Background(), query, exec09Request("row-a")); err != nil { + t.Fatalf("both-identity forged query: %v", err) + } + if len(spies.pgFull) != 1 { + t.Fatalf("port calls = %d", len(spies.pgFull)) + } + got := spies.pgFull[0].Columns + if len(got) != 6 || got[2] != ir.PhysicalCellTraceExplicitIdentityField { + t.Fatalf("contract identity slot = %#v, want the explicit pick at index 2", got) + } + }) +} + +func TestPostgresCellTraceOfflineAdmissionRefusalsRunZeroStatements(t *testing.T) { + compiled, _ := exec09ArmB(t, 0, 2) + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, nil) + if _, err := engine.CellTrace(context.Background(), exec02TestResolved(), CellTraceRequest{Output: " ", RowID: "r", Column: "name"}); err == nil || + !strings.Contains(err.Error(), "output is required") { + t.Fatalf("blank output: %v", err) + } + if _, err := engine.CellTraceCompiled(context.Background(), compiled, exec09Request(" ")); err == nil || + !strings.Contains(err.Error(), "row ID and column are required") { + t.Fatalf("blank row ID: %v", err) + } + if _, err := engine.CellTrace(context.Background(), exec02TestResolved(), CellTraceRequest{Output: "no-such-output", RowID: "r", Column: "name"}); err == nil || + !strings.Contains(err.Error(), "was not found") { + t.Fatalf("unknown output: %v", err) + } + if spies.pgRequests != 0 || len(spies.aqlQueries) != 0 { + t.Fatalf("admission refusals touched the ports: pg=%d aql=%v", spies.pgRequests, spies.aqlQueries) + } +} + +func TestPostgresCellTraceOfflineFailureClassificationOnPostgresArm(t *testing.T) { + compiled, _ := exec09ArmB(t, 0, 2) + for _, test := range []struct { + code dataframeerrors.ErrorCode + status CellTraceStatus + }{ + {code: dataframeerrors.CodeTemporalTieAmbiguous, status: CellTraceAmbiguous}, + {code: dataframeerrors.CodeRelationshipCardinalityViolation, status: CellTraceAmbiguous}, + {code: dataframeerrors.CodeUnitIdentityUnknown, status: CellTraceInvalidUnit}, + {code: dataframeerrors.CodeUnitDimensionIncompatible, status: CellTraceInvalidUnit}, + {code: dataframeerrors.CodeInvalidData, status: CellTraceInvalidType}, + {code: dataframeerrors.CodeRecipeContractViolation, status: CellTraceInvalidType}, + } { + t.Run(string(test.code), func(t *testing.T) { + pgFailing := &Engine{compileTarget: exec02PostgreSQLTarget(t), sourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + if _, _, err := request.Command.PostgreSQLParameters(); err != nil { + return err + } + return dataframeerrors.NewError(test.code, "") + }} + result, err := pgFailing.CellTraceCompiled(context.Background(), compiled, exec09Request("row-1")) + if err != nil || !result.Complete || result.Status != test.status || result.OmissionCode != string(test.code) || result.Contributions == nil { + t.Fatalf("result=%#v err=%v", result, err) + } + }) + } +} + +func TestPostgresCellTraceOfflineAQLControlStaysContractFree(t *testing.T) { + resolved := exec02TestResolved() + aqlTrace, err := compiler.CompileCellTraceOutput(compiler.ArangoTarget(), + resolved.Compiled.Outputs[0], "name", 0, 2, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("AQL control compile: %v", err) + } + if aqlTrace.Command.Backend() != command.CompiledBackendAQL { + t.Fatalf("AQL control tag = %q", aqlTrace.Command.Backend()) + } + var captured []string + capturedSet := false + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + if _, _, err := request.Command.AQLParameters(); err != nil { + return err + } + captured = request.DeclaredColumns + capturedSet = request.DeclaredColumns != nil + return visit(map[string]any{ + aqlTrace.IdentityPartsColumn: []any{"patient-one"}, aqlTrace.ValueColumn: "Ada", + aqlTrace.StatusColumn: "VALUE", aqlTrace.HasMoreColumn: false, aqlTrace.OmissionColumn: "", + aqlTrace.ContributionsColumn: nil, + }) + }} + rowID := exec09PreviewRowID(t, resolved.Compiled.Outputs[0].RowIdentity, map[string]any{"id": "patient-one"}) + result, err := engine.CellTraceCompiled(context.Background(), aqlTrace, exec09Request(rowID)) + if err != nil || result.Status != CellTraceValue || result.Value != "Ada" { + t.Fatalf("AQL control trace: result=%#v err=%v", result, err) + } + if capturedSet || captured != nil { + t.Fatalf("AQL arm must travel contract-free: %#v", captured) + } + if len(result.Contributions) != 0 { + t.Fatalf("AQL nil-page leniency changed: %#v", result.Contributions) + } +} diff --git a/internal/dataframe/execution/cell_trace_row_not_found_class_test.go b/internal/dataframe/execution/cell_trace_row_not_found_class_test.go index 0b09cd907..ff59e599d 100644 --- a/internal/dataframe/execution/cell_trace_row_not_found_class_test.go +++ b/internal/dataframe/execution/cell_trace_row_not_found_class_test.go @@ -37,15 +37,29 @@ func assertCellTraceMissingRowUserFault(t *testing.T, err error, wantRowID, want } func TestEngineCellTraceCompiledMissingRowIsUserFault(t *testing.T) { - query := compiler.CompiledCellTraceQuery{Query: "trace", ValueColumn: "value", ContributionsColumn: "contributors", StatusColumn: "status", HasMoreColumn: "hasMore", OmissionColumn: "omission", ExplicitIdentityColumn: "identity"} - engine := &Engine{queryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }} + query := compiler.CompiledCellTraceQuery{Command: mustAQLCommand("trace", nil), ValueColumn: "value", ContributionsColumn: "contributors", StatusColumn: "status", HasMoreColumn: "hasMore", OmissionColumn: "omission", ExplicitIdentityColumn: "identity"} + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }} _, err := engine.CellTraceCompiled(context.Background(), query, CellTraceRequest{Output: "patients", RowID: "row-does-not-exist", Column: "gender"}) assertCellTraceMissingRowUserFault(t, err, "row-does-not-exist", "patients") } func TestEngineCellTraceMissingPublishedRowIsUserFault(t *testing.T) { output := compileComposedCohortIdentityOutput(t) - engine := &Engine{queryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }} + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }} resolved := Resolved{Compiled: lower.CompiledRecipe{Outputs: []lower.CompiledRecipeOutput{output}}} rowID := `{"group_id":"cohort-absent","group_revision_id":"grouprev_absent"}` _, err := engine.CellTrace(context.Background(), resolved, CellTraceRequest{Output: output.Name, RowID: rowID, Column: "group_label", Limit: 10}) @@ -53,8 +67,14 @@ func TestEngineCellTraceMissingPublishedRowIsUserFault(t *testing.T) { } func TestEngineCellTraceKnownRowNeverTakesTheNotFoundArm(t *testing.T) { - query := compiler.CompiledCellTraceQuery{Query: "trace", ValueColumn: "value", ContributionsColumn: "contributors", StatusColumn: "status", HasMoreColumn: "hasMore", OmissionColumn: "omission", ExplicitIdentityColumn: "identity"} - engine := &Engine{queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + query := compiler.CompiledCellTraceQuery{Command: mustAQLCommand("trace", nil), ValueColumn: "value", ContributionsColumn: "contributors", StatusColumn: "status", HasMoreColumn: "hasMore", OmissionColumn: "omission", ExplicitIdentityColumn: "identity"} + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{"identity": "row-7", "value": nil, "status": "NO_MATCH", "contributors": []any{}, "hasMore": false}) }} result, err := engine.CellTraceCompiled(context.Background(), query, CellTraceRequest{Output: "patients", RowID: "row-7", Column: "condition"}) diff --git a/internal/dataframe/execution/cell_trace_test.go b/internal/dataframe/execution/cell_trace_test.go index 3537832ed..14c05b622 100644 --- a/internal/dataframe/execution/cell_trace_test.go +++ b/internal/dataframe/execution/cell_trace_test.go @@ -51,13 +51,13 @@ func TestConstructionPivotCellTraceUsesPreviewLiteralIdentity(t *testing.T) { if !strings.Contains(preview.Query, `TO_STRING(["GROUPED_PIVOT"`) { t.Fatalf("Preview query does not emit the compiler-generated string identity:\n%s", preview.Query) } - trace, err := compiler.CompileCellTraceOutputWithPolicy(output, "alpha", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + trace, err := compiler.CompileCellTraceOutput(compiler.ArangoTarget(), output, "alpha", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } if trace.ExplicitIdentityColumn != ir.PhysicalCellTraceExplicitIdentityField || trace.IdentityPartsColumn != "" || trace.ExplicitIdentityObject || - !strings.Contains(trace.Query, "__loom_construction_final_row.__loom_row_id") { - t.Fatalf("cell trace does not read Preview's explicit identity directly:\n%s\n%#v", trace.Query, trace) + !strings.Contains(commandText(t, trace.Command), "__loom_construction_final_row.__loom_row_id") { + t.Fatalf("cell trace does not read Preview's explicit identity directly:\n%s\n%#v", commandText(t, trace.Command), trace) } previewID := `["GROUPED_PIVOT","pivot_identity",["STRING","final"]]` @@ -133,13 +133,13 @@ func TestComposedCohortCellTraceIdentityMismatchAgainstPreviewObject(t *testing. if !strings.Contains(preview.Query, "__loom_row_id: {group_revision_id: revision._key, group_id:") { t.Fatalf("ordinary Preview lowering does not emit its structured cohort identity:\n%s", preview.Query) } - trace, err := compiler.CompileCellTraceOutputWithPolicy(output, "group_label", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + trace, err := compiler.CompileCellTraceOutput(compiler.ArangoTarget(), output, "group_label", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } if trace.ExplicitIdentityColumn != ir.PhysicalCellTraceExplicitIdentityField || - !strings.Contains(trace.Query, ".__loom_row_id") { - t.Fatalf("ordinary CellTrace lowering does not read the same structured identity:\n%s\n%#v", trace.Query, trace) + !strings.Contains(commandText(t, trace.Command), ".__loom_row_id") { + t.Fatalf("ordinary CellTrace lowering does not read the same structured identity:\n%s\n%#v", commandText(t, trace.Command), trace) } if !trace.ExplicitIdentityObject || trace.RowIdentity == nil || trace.RowIdentity.Grain != spec.RowGrainGroups || len(trace.RowIdentity.Fields) != 1 || trace.RowIdentity.Fields[0] != "__loom_row_id" { @@ -161,12 +161,18 @@ func TestComposedCohortCellTraceIdentityMismatchAgainstPreviewObject(t *testing. if err != nil { t.Fatal(err) } - previewRowID := string(previewIdentityBytes) // Preview GraphQL marshals row maps; the client later JSON.stringify's the parsed object. + previewRowID := string(previewIdentityBytes) for _, rowID := range []string{ previewRowID, `{"group_id":"cohort-a","group_revision_id":"grouprev_cell_trace_identity"}`, } { - engine := &Engine{queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{ trace.ExplicitIdentityColumn: previewRow["__loom_row_id"], trace.ValueColumn: "Pair", @@ -187,7 +193,7 @@ func TestComposedCohortCellTraceIdentityMismatchAgainstPreviewObject(t *testing. func TestComposedCohortCellTraceObjectIdentityRejectsMalformedAndExtraKeys(t *testing.T) { output := compileComposedCohortIdentityOutput(t) - trace, err := compiler.CompileCellTraceOutputWithPolicy(output, "group_label", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + trace, err := compiler.CompileCellTraceOutput(compiler.ArangoTarget(), output, "group_label", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -204,7 +210,13 @@ func TestComposedCohortCellTraceObjectIdentityRejectsMalformedAndExtraKeys(t *te } { t.Run(test.name, func(t *testing.T) { queried := false - engine := &Engine{queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + queried = true return visit(map[string]any{ trace.ExplicitIdentityColumn: map[string]any{ @@ -250,7 +262,7 @@ func TestSourceOnlyGroupRowsCellTraceMatchesPreviewObjectAndReturnsMemberField(t if err != nil { t.Fatal(err) } - trace, err := compiler.CompileCellTraceOutputWithPolicy(compiled.Outputs[0], "resourceType", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + trace, err := compiler.CompileCellTraceOutput(compiler.ArangoTarget(), compiled.Outputs[0], "resourceType", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile ordinary group-row CellTrace: %v", err) } @@ -258,8 +270,14 @@ func TestSourceOnlyGroupRowsCellTraceMatchesPreviewObjectAndReturnsMemberField(t t.Fatalf("group-row CellTrace identity = %#v, want Preview's typed object identity", trace) } rowIdentity := map[string]any{"group_revision_id": "grouprev_cell_trace", "group_id": "cohort-a"} - engine := &Engine{queryRows: func(_ context.Context, query string, _ int, _ map[string]any, visit func(map[string]any) error) error { - if query != trace.Query { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + query, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + + if query != commandText(t, trace.Command) { t.Fatalf("CellTrace executed a query other than its compiled terminal") } return visit(map[string]any{ @@ -291,7 +309,13 @@ func TestSourceOnlyGroupRowsCellTraceMatchesPreviewObjectAndReturnsMemberField(t `["grouprev_cell_trace","cohort-a"]`, } { queried := false - malformedEngine := &Engine{queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + malformedEngine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + queried = true return visit(map[string]any{}) }} @@ -320,7 +344,7 @@ func TestComposedCohortPivotKeepsCompilerProvenScalarIdentity(t *testing.T) { if finalIdentity == nil || finalIdentity.Kind != "string" || finalIdentity.Cardinality != "required_one" || !finalIdentity.Internal || !finalIdentity.Identity { t.Fatalf("cohort Pivot Preview identity schema = %#v, want a required compiler-owned scalar", finalIdentity) } - trace, err := compiler.CompileCellTraceOutputWithPolicy(output, "ordinal_pair", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) + trace, err := compiler.CompileCellTraceOutput(compiler.ArangoTarget(), output, "ordinal_pair", 0, 10, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } @@ -329,7 +353,13 @@ func TestComposedCohortPivotKeepsCompilerProvenScalarIdentity(t *testing.T) { t.Fatalf("cohort Pivot CellTrace identity = %#v, want scalar explicit identity", trace) } rowID := `["GROUPED_PIVOT","cohort_pivot_identity",["STRING","pair"]]` - engine := &Engine{queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{ trace.ExplicitIdentityColumn: rowID, trace.ValueColumn: int64(2), @@ -431,11 +461,17 @@ func TestCellTraceCompiledFindsPublishedDefaultIdentityAndReturnsEvidence(t *tes digest := sha256.Sum256(encoded) rowID := hex.EncodeToString(digest[:]) query := compiler.CompiledCellTraceQuery{ - Query: "trace", BindVars: map[string]any{"project": "p"}, ValueColumn: "value", ContributionsColumn: "contributors", + Command: mustAQLCommand("trace", map[string]any{"project": "p"}), ValueColumn: "value", ContributionsColumn: "contributors", StatusColumn: "status", HasMoreColumn: "hasMore", OmissionColumn: "omission", IdentityPartsColumn: "identity", RowIdentity: &spec.RowIdentity{Fields: []string{"id", "generation"}}, ContributionLimit: 1, } - engine := &Engine{queryRows: func(_ context.Context, queryText string, _ int, bindVars map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + queryText, bindVars, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if queryText != "trace" || bindVars["project"] != "p" { t.Fatalf("unexpected query invocation: %q %#v", queryText, bindVars) } @@ -467,7 +503,7 @@ func TestCellTraceCompiledFindsPublishedDefaultIdentityAndReturnsEvidence(t *tes func TestCellTraceRelatedSourcePreservesDuplicateOccurrencesAndNullValues(t *testing.T) { query := compiler.CompiledCellTraceQuery{ - Query: "trace", ValueColumn: "value", ContributionsColumn: "contributors", + Command: mustAQLCommand("trace", nil), ValueColumn: "value", ContributionsColumn: "contributors", StatusColumn: "status", HasMoreColumn: "hasMore", OmissionColumn: "omission", ExplicitIdentityColumn: "identity", } contributor := func(resourceID string, value any) map[string]any { @@ -479,7 +515,13 @@ func TestCellTraceRelatedSourcePreservesDuplicateOccurrencesAndNullValues(t *tes } } contributors := []any{contributor("observation-1", nil), contributor("observation-1", nil), contributor("observation-2", "final")} - engine := &Engine{queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{ "identity": "patient-key", "value": []any{nil, nil, "final"}, "status": "VALUE", "contributors": contributors, "hasMore": false, "omission": "", @@ -502,8 +544,14 @@ func TestCellTraceRelatedSourcePreservesDuplicateOccurrencesAndNullValues(t *tes } func TestCellTraceCompiledMatchesExplicitPublishedIdentity(t *testing.T) { - query := compiler.CompiledCellTraceQuery{Query: "trace", ValueColumn: "value", ContributionsColumn: "contributors", StatusColumn: "status", HasMoreColumn: "hasMore", OmissionColumn: "omission", ExplicitIdentityColumn: "identity"} - engine := &Engine{queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + query := compiler.CompiledCellTraceQuery{Command: mustAQLCommand("trace", nil), ValueColumn: "value", ContributionsColumn: "contributors", StatusColumn: "status", HasMoreColumn: "hasMore", OmissionColumn: "omission", ExplicitIdentityColumn: "identity"} + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{"identity": "patient-7", "value": nil, "status": "NO_MATCH", "contributors": []any{}, "hasMore": false}) }} result, err := engine.CellTraceCompiled(context.Background(), query, CellTraceRequest{Output: "patients", RowID: "patient-7", Column: "condition", Limit: 10}) @@ -513,8 +561,14 @@ func TestCellTraceCompiledMatchesExplicitPublishedIdentity(t *testing.T) { } func TestCellTraceCompiledReportsWitnessBoundAsIncomplete(t *testing.T) { - query := compiler.CompiledCellTraceQuery{Query: "trace", ValueColumn: "value", ContributionsColumn: "contributors", StatusColumn: "status", HasMoreColumn: "hasMore", OmissionColumn: "omission", ExplicitIdentityColumn: "identity"} - engine := &Engine{queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + query := compiler.CompiledCellTraceQuery{Command: mustAQLCommand("trace", nil), ValueColumn: "value", ContributionsColumn: "contributors", StatusColumn: "status", HasMoreColumn: "hasMore", OmissionColumn: "omission", ExplicitIdentityColumn: "identity"} + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + for _, id := range []string{"one", "two"} { if err := visit(map[string]any{"identity": id}); err != nil { return err @@ -529,7 +583,7 @@ func TestCellTraceCompiledReportsWitnessBoundAsIncomplete(t *testing.T) { } func TestCellTraceCompiledReturnsTypedSemanticFailureStatuses(t *testing.T) { - query := compiler.CompiledCellTraceQuery{Query: "trace", ValueColumn: "value", ContributionsColumn: "contributors", StatusColumn: "status", HasMoreColumn: "hasMore", OmissionColumn: "omission", ExplicitIdentityColumn: "identity"} + query := compiler.CompiledCellTraceQuery{Command: mustAQLCommand("trace", nil), ValueColumn: "value", ContributionsColumn: "contributors", StatusColumn: "status", HasMoreColumn: "hasMore", OmissionColumn: "omission", ExplicitIdentityColumn: "identity"} for _, test := range []struct { code dataframeerrors.ErrorCode status CellTraceStatus @@ -539,7 +593,13 @@ func TestCellTraceCompiledReturnsTypedSemanticFailureStatuses(t *testing.T) { {code: dataframeerrors.CodeInvalidData, status: CellTraceInvalidType}, } { t.Run(string(test.code), func(t *testing.T) { - engine := &Engine{queryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return dataframeerrors.NewError(test.code, "") }} result, err := engine.CellTraceCompiled(context.Background(), query, CellTraceRequest{Output: "patients", RowID: "row-1", Column: "feature"}) @@ -550,10 +610,61 @@ func TestCellTraceCompiledReturnsTypedSemanticFailureStatuses(t *testing.T) { } } +func TestCellTraceCompiledRejectsInvalidStatusBeforeEvidenceDecoding(t *testing.T) { + t.Run("aql", func(t *testing.T) { + query := compiler.CompiledCellTraceQuery{Command: mustAQLCommand("trace", nil), ValueColumn: "value", ContributionsColumn: "contributors", StatusColumn: "status", HasMoreColumn: "hasMore", OmissionColumn: "omission", ExplicitIdentityColumn: "identity"} + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + + return visit(map[string]any{"identity": "row", "status": "MAYBE", "contributors": "not-an-array"}) + }} + _, err := engine.CellTraceCompiled(context.Background(), query, CellTraceRequest{Output: "o", RowID: "row", Column: "c"}) + if err == nil || !strings.Contains(err.Error(), "cell trace row has an invalid status") || strings.Contains(err.Error(), "invalid contributions") { + t.Fatalf("status error = %v, want the status refusal to precede contribution decoding", err) + } + }) + t.Run("postgres", func(t *testing.T) { + query := compiler.CompiledCellTraceQuery{ + Command: exec02PostgreSQLCommand(t), ValueColumn: ir.PhysicalCellTraceValueField, ContributionsColumn: ir.PhysicalCellTraceContributionsField, + StatusColumn: ir.PhysicalCellTraceStatusField, HasMoreColumn: ir.PhysicalCellTraceHasMoreField, + OmissionColumn: ir.PhysicalCellTraceOmissionField, IdentityPartsColumn: ir.PhysicalCellTraceIdentityPartsField, + RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, + } + rowID := exec09PreviewRowID(t, query.RowIdentity, map[string]any{"id": "patient-one"}) + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, []map[string]any{{ + ir.PhysicalCellTraceIdentityPartsField: json.RawMessage(`["patient-one"]`), + ir.PhysicalCellTraceValueField: json.RawMessage(`{bad`), + ir.PhysicalCellTraceStatusField: "MAYBE", + ir.PhysicalCellTraceHasMoreField: false, + ir.PhysicalCellTraceOmissionField: "", + ir.PhysicalCellTraceContributionsField: json.RawMessage(`{bad`), + }}) + result, err := engine.CellTraceCompiled(context.Background(), query, CellTraceRequest{Output: "resources", RowID: rowID, Column: "name", Limit: 2}) + if err == nil || !strings.Contains(err.Error(), "cell trace row has an invalid status") || + strings.Contains(err.Error(), "invalid contributions") || strings.Contains(err.Error(), "traced value") { + t.Fatalf("status error = %v, want the status refusal to precede JSONB decoding: %#v", err, result) + } + if errors.Is(err, ErrCellTraceRowNotFound) { + t.Fatalf("invalid status with malformed JSONB was classified as row absence: %v", err) + } + }) +} + func TestCellTraceCompiledRejectsMalformedEvidenceAndMissingRows(t *testing.T) { - query := compiler.CompiledCellTraceQuery{Query: "trace", ValueColumn: "value", ContributionsColumn: "contributors", StatusColumn: "status", HasMoreColumn: "hasMore", OmissionColumn: "omission", ExplicitIdentityColumn: "identity"} + query := compiler.CompiledCellTraceQuery{Command: mustAQLCommand("trace", nil), ValueColumn: "value", ContributionsColumn: "contributors", StatusColumn: "status", HasMoreColumn: "hasMore", OmissionColumn: "omission", ExplicitIdentityColumn: "identity"} t.Run("malformed", func(t *testing.T) { - engine := &Engine{queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{"identity": "row", "status": "MAYBE", "contributors": []any{}}) }} if _, err := engine.CellTraceCompiled(context.Background(), query, CellTraceRequest{Output: "o", RowID: "row", Column: "c"}); err == nil { @@ -561,7 +672,14 @@ func TestCellTraceCompiledRejectsMalformedEvidenceAndMissingRows(t *testing.T) { } }) t.Run("missing", func(t *testing.T) { - engine := &Engine{queryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }} + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }} _, err := engine.CellTraceCompiled(context.Background(), query, CellTraceRequest{Output: "o", RowID: "row", Column: "c"}) if err == nil || errors.Is(err, errCellTraceFound) { t.Fatalf("missing row error = %v", err) diff --git a/internal/dataframe/execution/chartifact/artifact.go b/internal/dataframe/execution/chartifact/artifact.go index 77e44d895..c4f0e4500 100644 --- a/internal/dataframe/execution/chartifact/artifact.go +++ b/internal/dataframe/execution/chartifact/artifact.go @@ -1,6 +1,3 @@ -// Package chartifact manages private ClickHouse tables used to cross an -// execution boundary. These artifacts are never registered as published -// outputs and never advance a visibility pointer. package chartifact import ( @@ -45,16 +42,10 @@ var ( ) const ( - // ScopeModeRows preserves and validates the authorization path on every row. - ScopeModeRows = "ROW_SCOPED" - // ScopeModeWhole binds the complete relation to one compiler-verified - // authorization scope. Such rows intentionally carry no individual path. + ScopeModeRows = "ROW_SCOPED" ScopeModeWhole = "WHOLE_SCOPE" ) -// Identity binds an intermediate table to the exact source execution and -// compiler result that created it. SchemaDigest and ScopeDigest are computed -// by Begin; supplied values, when present, must match. type Identity struct { ExecutionID string OutputID string @@ -71,8 +62,6 @@ type Identity struct { ScopeEvidenceDigest string } -// Column records a stable compiler ID alongside logical and physical types. -// The physical type is restricted to the supported scalar storage types. type Column struct { ID string `json:"id,omitempty"` Name string `json:"name"` @@ -85,9 +74,6 @@ type Column struct { NormalizedUnit string `json:"normalizedUnit,omitempty"` } -// Manifest is the durable identity and cleanup record for one private table. -// Catalog implementations must treat Identity, Columns, and PhysicalTable as -// immutable after Create. type Manifest struct { ArtifactID string `json:"artifactId"` Identity Identity `json:"identity"` @@ -102,11 +88,6 @@ type Manifest struct { UpdatedAt time.Time `json:"updatedAt"` } -// Progress is the only mutable portion of a manifest. Catalog.Update must -// compare the current lease owner with owner and must not replace immutable -// identity or schema fields. For active states it may extend LeaseUntil but -// must not shorten it; CLEANUP_PENDING may make the lease immediately -// reclaimable after the owner has stopped using the table. type Progress struct { State State RowCount int64 @@ -115,13 +96,6 @@ type Progress struct { UpdatedAt time.Time } -// Catalog is the durable manifest and lease boundary. Create must be -// conditional on ArtifactID being absent. Update and Renew must compare the -// owner and reject expired leases; Renew must not resurrect an expired lease. -// Update must preserve immutable identity, schema, and table fields. ClaimCleanup -// must atomically compare LeaseUntil with expiredBefore, change the owner, and -// set the cleanup lease; only one caller may win. Delete and ReleaseCleanup -// must verify the current owner. type Catalog interface { Create(context.Context, Manifest) error Update(context.Context, string, string, Progress) error @@ -132,10 +106,6 @@ type Catalog interface { ReleaseCleanup(context.Context, string, string, time.Time) error } -// ClickHouse is the narrow private-table contract. The production adapter can -// be the existing typed client; tests use a fake and do not need a server. -// InsertRows must consume its bounded batch before returning, and DropTable -// must be idempotent. type ClickHouse interface { CreateTable(context.Context, string, []clickhouse.Column) error InsertRows(context.Context, string, []clickhouse.Column, []map[string]any) error @@ -190,8 +160,6 @@ func New(cfg Config) (*Manager, error) { }, nil } -// NormalizeSchema appends the Loom-owned identity and scope columns required -// by a private intermediate artifact. Conflicting uses of reserved names fail. func NormalizeSchema(columns []Column) ([]Column, error) { var rowID *Column var authPath *Column @@ -281,9 +249,6 @@ func NormalizeSchema(columns []Column) ([]Column, error) { return result, nil } -// ColumnsFromCompiledOutput converts the compiler's finalized output schema -// into the physical scalar schema used at the private engine boundary. Hidden -// compiler helpers are omitted; the stable row identity is always retained. func ColumnsFromCompiledOutput(schema []lower.CompiledOutputColumn) ([]Column, error) { columns := make([]Column, 0, len(schema)) rowIDSeen := false @@ -297,9 +262,6 @@ func ColumnsFromCompiledOutput(schema []lower.CompiledOutputColumn) ([]Column, e columns = append(columns, Column{ID: compiled.ID, Name: rowIDColumn, SemanticPath: "loom:row_id", LogicalType: "string", Internal: true}) continue } - // Other internal identity helpers (for example, the root FHIR `_key`) - // are not the artifact's stable row identity. They are compiler inputs - // for deriving `__loom_row_id` and must not be exposed as artifact data. if compiled.Internal { continue } @@ -372,8 +334,6 @@ func scopeDigest(identity Identity) string { return hex.EncodeToString(sum[:]) } -// ScopeDigest returns the private artifact scope digest for an exact identity. -// An omitted scope mode retains the legacy row-scoped meaning. func ScopeDigest(identity Identity) string { return scopeDigest(identity) } func schemaDigest(stageID string, columns []Column) (string, error) { @@ -389,7 +349,6 @@ func schemaDigest(stageID string, columns []Column) (string, error) { return hex.EncodeToString(sum[:]), nil } -// SchemaDigest returns the digest of a normalized compiler stage schema. func SchemaDigest(stageID string, columns []Column) (string, error) { normalized, err := NormalizeSchema(columns) if err != nil { @@ -523,8 +482,6 @@ type Writer struct { mu sync.Mutex } -// Write validates and buffers one row. The writer retains a defensive copy; -// the caller may reuse the supplied map as soon as Write returns. func (w *Writer) Write(ctx context.Context, input map[string]any) error { w.mu.Lock() defer w.mu.Unlock() @@ -605,7 +562,6 @@ func (w *Writer) validateRow(input map[string]any) (map[string]any, int, error) return nil, 0, fmt.Errorf("private artifact row authorization path is outside the exact restricted scope") } } - // project_id is Loom-owned metadata; never trust a source row value. row[projectIDColumn] = w.manifest.Identity.Project for _, column := range w.manifest.Columns { value, exists := row[column.Name] @@ -756,8 +712,6 @@ func (w *Writer) flush(ctx context.Context) error { return nil } -// Finalize flushes and verifies the table before making it available to a -// later typed ClickHouse stage. Its lease remains active until Release. func (w *Writer) Finalize(ctx context.Context) (*Artifact, error) { w.mu.Lock() defer w.mu.Unlock() @@ -786,8 +740,6 @@ func (w *Writer) Finalize(ctx context.Context) (*Artifact, error) { return &Artifact{manager: w.manager, manifest: cloneManifest(w.manifest), owner: w.owner, lease: w.lease}, nil } -// Abort removes a partially written artifact. If ClickHouse cannot drop the -// table, the cleanup-pending manifest remains for Reconcile. func (w *Writer) Abort(ctx context.Context) error { w.mu.Lock() defer w.mu.Unlock() @@ -839,8 +791,6 @@ func (a *Artifact) CheckLease() error { return a.lease.check() } -// LeaseContext cancels ctx if the durable cleanup lease is lost. A streaming -// ClickHouse query should use this context until it stops reading the table. func (a *Artifact) LeaseContext(ctx context.Context) (context.Context, context.CancelFunc) { if ctx == nil { ctx = context.Background() @@ -860,8 +810,6 @@ func (a *Artifact) LeaseContext(ctx context.Context) (context.Context, context.C return leaseCtx, cancel } -// Release drops the table and removes its manifest. It is safe to call more -// than once. A failed drop leaves an expired cleanup-pending manifest. func (a *Artifact) Release(ctx context.Context) error { if a == nil { return nil @@ -888,8 +836,6 @@ func (a *Artifact) Release(ctx context.Context) error { return a.releaseErr } -// Reconcile claims and removes artifacts whose execution leases expired. It -// can be run repeatedly and is safe when multiple service replicas race. func (m *Manager) Reconcile(ctx context.Context, now time.Time, limit int) error { if ctx == nil { return fmt.Errorf("private artifact reconciliation context is required") diff --git a/internal/dataframe/execution/chartifact/artifact_test.go b/internal/dataframe/execution/chartifact/artifact_test.go index a4d41a2dd..2903f9d02 100644 --- a/internal/dataframe/execution/chartifact/artifact_test.go +++ b/internal/dataframe/execution/chartifact/artifact_test.go @@ -300,7 +300,7 @@ func TestReconcileClaimsExpiredCreatingArtifact(t *testing.T) { t.Fatal(err) } manifest := writer.manifest - writer.lease.stop() // simulate a crashed process; the durable lease expires below + writer.lease.stop() catalog.expire(manifest.ArtifactID, time.Now().UTC().Add(-time.Minute)) if err := manager.Reconcile(context.Background(), time.Now().UTC(), 20); err != nil { t.Fatal(err) diff --git a/internal/dataframe/execution/clickhouse_artifact.go b/internal/dataframe/execution/clickhouse_artifact.go index 6a9c64b16..5195bc7cc 100644 --- a/internal/dataframe/execution/clickhouse_artifact.go +++ b/internal/dataframe/execution/clickhouse_artifact.go @@ -8,13 +8,11 @@ import ( "strings" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/execution/chartifact" ) -// MaterializeClickHouseArtifact streams a complete AQL output into a private, -// typed ClickHouse artifact. The artifact remains leased after this call and -// must be released after the ClickHouse stage finishes reading it. func (s OutputStream) MaterializeClickHouseArtifact(ctx context.Context, manager *chartifact.Manager, executionID, stageID string, columns []chartifact.Column) (*chartifact.Artifact, error) { capture, err := s.materializeClickHouseArtifact(ctx, manager, executionID, stageID, columns) if err != nil { @@ -30,11 +28,19 @@ func (s OutputStream) materializeClickHouseArtifact(ctx context.Context, manager if s.captureExecutionID != "" && executionID != s.captureExecutionID { return nil, fmt.Errorf("private artifact execution ID differs from its owning workspace capture") } - aqlSource := s.physicalEngine != ir.PhysicalEngineClickHouse && s.clickHouseCombine == nil && s.stream != nil + sourceStream := s.physicalEngine != ir.PhysicalEngineClickHouse && s.clickHouseCombine == nil && s.sourceRows != nil + pgSource := sourceStream && s.command.Backend() == command.CompiledBackendPostgreSQL && + s.expectedBackend == command.CompiledBackendPostgreSQL && len(s.commandColumns) > 0 + aqlSource := sourceStream && !pgSource && s.command.Backend() != command.CompiledBackendPostgreSQL clickHouseSource := s.physicalEngine == ir.PhysicalEngineClickHouse && s.clickHouseCombine != nil && s.clickHouseQueryRows != nil && s.compiledOutput - if !aqlSource && !clickHouseSource { - return nil, fmt.Errorf("private artifact requires a compiler-resolved streamable AQL or ClickHouse output") + if !aqlSource && !pgSource && !clickHouseSource { + return nil, fmt.Errorf("private artifact requires a compiler-resolved streamable AQL, PostgreSQL, or ClickHouse output") } + sourceLabel := "AQL" + if pgSource { + sourceLabel = "PostgreSQL" + } + sourceNeedsEvidence := aqlSource || pgSource if s.queryLimit != 0 { return nil, fmt.Errorf("private AQL artifact cannot be built from a bounded preview") } @@ -57,23 +63,23 @@ func (s OutputStream) materializeClickHouseArtifact(ctx context.Context, manager if err != nil { return nil, fmt.Errorf("private artifact authorization scope: %w", err) } - if aqlSource && (s.scopeEvidence == nil || s.scopeEvidenceIdentity == nil || !s.scopeEvidence.Matches(s.physicalPlan, *s.scopeEvidenceIdentity)) { - return nil, fmt.Errorf("private AQL artifact requires opaque evidence for its exact complete source plan") + if sourceNeedsEvidence && (s.scopeEvidence == nil || s.scopeEvidenceIdentity == nil || !s.scopeEvidence.Matches(s.physicalPlan, *s.scopeEvidenceIdentity)) { + return nil, fmt.Errorf("private %s artifact requires opaque evidence for its exact complete source plan", sourceLabel) } - if aqlSource && s.artifactScopeMode == "" { + if sourceNeedsEvidence && s.artifactScopeMode == "" { s.artifactScopeMode = chartifact.ScopeModeRows } if s.artifactScopeMode != "" && s.artifactScopeMode != chartifact.ScopeModeRows && s.artifactScopeMode != chartifact.ScopeModeWhole { return nil, fmt.Errorf("private artifact requires a supported compiler output scope mode") } - if aqlSource && s.artifactScopeMode == chartifact.ScopeModeWhole && (!s.compiledOutput || strings.TrimSpace(s.artifactScopeEvidence) == "") { + if sourceNeedsEvidence && s.artifactScopeMode == chartifact.ScopeModeWhole && (!s.compiledOutput || strings.TrimSpace(s.artifactScopeEvidence) == "") { return nil, fmt.Errorf("whole-scope private artifact requires compiler-issued scope evidence") } - if aqlSource && s.artifactScopeMode == chartifact.ScopeModeRows && s.artifactScopeEvidence != "" { + if sourceNeedsEvidence && s.artifactScopeMode == chartifact.ScopeModeRows && s.artifactScopeEvidence != "" { return nil, fmt.Errorf("row-scoped private artifact cannot carry whole-scope evidence") } - if aqlSource && authMode == authscope.ReadScopeRestricted && s.artifactScopeMode == chartifact.ScopeModeRows && !s.bindings.IncludeAuthResourcePath { - return nil, fmt.Errorf("restricted AQL artifact requires a row-level authorization path projection") + if sourceNeedsEvidence && authMode == authscope.ReadScopeRestricted && s.artifactScopeMode == chartifact.ScopeModeRows && !s.bindings.IncludeAuthResourcePath { + return nil, fmt.Errorf("restricted %s artifact requires a row-level authorization path projection", sourceLabel) } normalized, err := chartifact.NormalizeSchema(columns) @@ -110,14 +116,14 @@ func (s OutputStream) materializeClickHouseArtifact(ctx context.Context, manager } writer, err := manager.Begin(ctx, identity, normalized) if err != nil { - return nil, fmt.Errorf("begin private AQL artifact: %w", err) + return nil, fmt.Errorf("begin private %s artifact: %w", sourceLabel, err) } streamResult, err := s.Stream(ctx, func(row map[string]any) error { return writer.Write(ctx, row) }) if err != nil { abortErr := writer.Abort(context.WithoutCancel(ctx)) - return nil, errors.Join(fmt.Errorf("stream AQL prefix into private ClickHouse artifact: %w", err), abortErr) + return nil, errors.Join(fmt.Errorf("stream %s prefix into private ClickHouse artifact: %w", sourceLabel, err), abortErr) } artifact, err := writer.Finalize(ctx) if err != nil { @@ -135,9 +141,6 @@ func (s OutputStream) materializeClickHouseArtifact(ctx context.Context, manager }, nil } -// materializePreparedClickHouse resolves every exact input and composes the -// query's scope evidence before opening a writer. Dependency leases remain -// owned by withPreparedClickHouse until the complete source query is consumed. func (s OutputStream) materializePreparedClickHouse(ctx context.Context, manager *chartifact.Manager, executionID, stageID string, columns []chartifact.Column) (*materializedClickHouseCapture, error) { var capture *materializedClickHouseCapture captureStream := s @@ -208,8 +211,6 @@ func (s OutputStream) materializePreparedClickHouse(ctx context.Context, manager return capture, nil } -// PrivateClickHouseArtifactSource identifies one exact resolved workspace -// output to capture before a terminal ClickHouse operation. type PrivateClickHouseArtifactSource struct { OutputID string Stream OutputStream @@ -217,8 +218,6 @@ type PrivateClickHouseArtifactSource struct { ExpectedIdentity ir.ClickHouseArtifactIdentity } -// PrivateClickHouseArtifactInput preserves the caller's source order when the -// completed private captures are handed to the ClickHouse consumer. type PrivateClickHouseArtifactInput struct { OutputID string Artifact *chartifact.Artifact @@ -228,10 +227,6 @@ type PrivateClickHouseArtifactInput struct { ScopeEvidence ir.WholeRelationScopeEvidence } -// WithPrivateClickHouseArtifacts pins any published inputs, captures each -// workspace output in order, invokes consume while every capture lease and -// published pin is live, then releases every capture before returning. A -// zero-length publishedRevisions list is valid for an all-workspace Combine. func WithPrivateClickHouseArtifacts( ctx context.Context, manager *chartifact.Manager, diff --git a/internal/dataframe/execution/clickhouse_artifact_test.go b/internal/dataframe/execution/clickhouse_artifact_test.go index dad24df88..0dcdf801a 100644 --- a/internal/dataframe/execution/clickhouse_artifact_test.go +++ b/internal/dataframe/execution/clickhouse_artifact_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/execution/chartifact" "github.com/calypr/loom/internal/dataframe/recipe" @@ -189,7 +190,13 @@ func TestWithPrivateClickHouseArtifactsPinPublishedInputsThroughAllCaptures(t *t sources := []PrivateClickHouseArtifactSource{privateCaptureSource(t, "patients", "Ada"), privateCaptureSource(t, "observations", "registered")} for index := range sources { outputID := sources[index].OutputID - sources[index].Stream.stream = func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + sources[index].Stream.sourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if !pinned { return fmt.Errorf("workspace output %q streamed outside published pins", outputID) } @@ -218,7 +225,13 @@ func TestWithPrivateClickHouseArtifactsCleansEarlierCaptureAfterSecondFailure(t t.Fatal(err) } sources := []PrivateClickHouseArtifactSource{privateCaptureSource(t, "patients", "Ada"), privateCaptureSource(t, "observations", "registered")} - sources[1].Stream.stream = func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + sources[1].Stream.sourceRows = func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return errors.New("second source failed") } consumeCalled := false @@ -304,7 +317,13 @@ func TestWithPrivateClickHouseArtifactsCleansCapturesAfterCancellationAndConsume ctx, cancel := context.WithCancel(context.Background()) defer cancel() sources := []PrivateClickHouseArtifactSource{privateCaptureSource(t, "patients", "Ada"), privateCaptureSource(t, "observations", "registered")} - sources[1].Stream.stream = func(ctx context.Context, _ string, _ int, _ map[string]any, _ func(map[string]any) error) error { + sources[1].Stream.sourceRows = func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + cancel() <-ctx.Done() return ctx.Err() @@ -346,7 +365,13 @@ func TestWithPrivateClickHouseArtifactsLeaseLossCancelsNextSourceStream(t *testi t.Fatal(err) } sources := []PrivateClickHouseArtifactSource{privateCaptureSource(t, "patients", "Ada"), privateCaptureSource(t, "observations", "registered")} - sources[1].Stream.stream = func(ctx context.Context, _ string, _ int, _ map[string]any, _ func(map[string]any) error) error { + sources[1].Stream.sourceRows = func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + catalog.mu.Lock() if len(catalog.createdIDs) == 0 { catalog.mu.Unlock() @@ -442,8 +467,16 @@ func compiledPrivateCaptureStreamWithBindings(t *testing.T, outputID, columnName Outputs: []recipe.Output{makeOutput("patients"), makeOutput("observations")}, } engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -452,14 +485,20 @@ func compiledPrivateCaptureStreamWithBindings(t *testing.T, outputID, columnName if err != nil { t.Fatalf("compile fixture capture output %q: %v", outputID, err) } - stream, _, err := engine.streamForOutput(resolved, outputID, 0) + stream, _, err := engine.streamForOutput(context.Background(), resolved, outputID, 0) if err != nil { t.Fatalf("build fixture capture stream %q: %v", outputID, err) } if stream.scopeEvidence == nil || stream.scopeEvidenceIdentity == nil || !stream.scopeEvidence.Matches(stream.physicalPlan, *stream.scopeEvidenceIdentity) { t.Fatalf("fixture capture stream %q has no compiler-issued source evidence", outputID) } - stream.stream = func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + stream.sourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(privateCaptureRowFor(columnName, value)) } columns, err := chartifact.ColumnsFromCompiledOutput(stream.outputSchema) diff --git a/internal/dataframe/execution/cohort_preview_paging_arango_test.go b/internal/dataframe/execution/cohort_preview_paging_arango_test.go index 2934b5f83..392568981 100644 --- a/internal/dataframe/execution/cohort_preview_paging_arango_test.go +++ b/internal/dataframe/execution/cohort_preview_paging_arango_test.go @@ -11,11 +11,11 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler" compilerIR "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/projectid" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -26,20 +26,21 @@ func TestCohortPreviewWithSmallerRootPageKeepsAllMemberValuesAgainstArango(t *te } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) t.Cleanup(cancel) - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client t.Cleanup(func() { _ = client.Close(context.Background()) }) collections := []string{ "Patient", "loom_explorer_explicit_group_revisions", "loom_explorer_selections", "loom_explorer_selection_members", "loom_explorer_explicit_group_definitions", "loom_explorer_explicit_group_memberships", } - specs := make([]store.CollectionSpec, 0, len(collections)) + specs := make([]arangofactory.CollectionSpec, 0, len(collections)) for _, name := range collections { - specs = append(specs, store.CollectionSpec{Name: name}) + specs = append(specs, arangofactory.CollectionSpec{Name: name}) } - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: specs}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: specs}); err != nil { t.Fatal(err) } project, generation := "loom_cohort_preview_"+uuid.NewString(), "cohort-preview-generation" @@ -144,8 +145,16 @@ func TestCohortPreviewWithSmallerRootPageKeepsAllMemberValuesAgainstArango(t *te candidateScanBinds := map[string]any{} queryErr := error(nil) engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, ScopeDigest: func(recipe.RuntimeBindings) string { return "scope" }, RootPageRows: 1, - QueryRows: func(queryCtx context.Context, query string, batchSize int, bindVars map[string]any, visit func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, ScopeDigest: func(recipe.RuntimeBindings) string { return "scope" }, RootPageRows: 1, + SourceRows: func(queryCtx context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + batchSize := request.BatchSize + query, bindVars, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if _, paged := bindVars[compiler.RootPageKeysBind]; paged { rootPageQueries++ } @@ -153,7 +162,7 @@ func TestCohortPreviewWithSmallerRootPageKeepsAllMemberValuesAgainstArango(t *te candidateScanQuery = query candidateScanBinds = bindVars } - queryErr = client.QueryRows(queryCtx, query, batchSize, bindVars, visit) + queryErr = client.QueryRows(queryCtx, query, batchSize, bindVars, arangofactory.RowVisitor(visit)) return queryErr }, }) diff --git a/internal/dataframe/execution/command_boundary_contract_test.go b/internal/dataframe/execution/command_boundary_contract_test.go new file mode 100644 index 000000000..6da83116d --- /dev/null +++ b/internal/dataframe/execution/command_boundary_contract_test.go @@ -0,0 +1,365 @@ +package execution + +import ( + "context" + "errors" + "github.com/calypr/loom/internal/dataframe/compiler" + "go/ast" + "go/parser" + "go/token" + "io/fs" + "reflect" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/recipe" +) + +func TestSourceCommandBoundaryContractRefusesMismatchedTagBeforeAdapter(t *testing.T) { + v := &verifyArch01Trace{} + config := verifyArch01Config(v) + config.CompileTarget = verifyArch01PostgreSQLTarget(t) + config.PreviewSourceRows = nil + engine, err := New(config) + if err != nil { + t.Fatalf("New: %v", err) + } + _, streamErr := engine.Run(context.Background(), verifyArch01PostgreSQLResolved("resources", []string{"id"})) + if !errors.Is(streamErr, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("PostgreSQL command on the AQL-only port: err = %v, want %v", streamErr, command.ErrCompiledCommandBackendMismatch) + } + aqlFull, aqlPreview, pgFull, pgPreview, controls, ch := v.snapshot() + if len(aqlFull) != 0 || len(aqlPreview) != 0 || len(pgFull) != 0 || len(pgPreview) != 0 || len(controls) != 0 || ch != 0 { + t.Fatalf("mismatched tag reached an adapter before refusal: aql=%v preview=%v pg=%v pgPreview=%v controls=%v ch=%d", + aqlFull, aqlPreview, pgFull, pgPreview, controls, ch) + } +} + +func TestSourceCommandBoundaryContractZeroTaggedCommandFailsClosed(t *testing.T) { + v := &verifyArch01Trace{} + stream := OutputStream{Name: "resources", command: command.CompiledCommand{}, sourceRows: func(context.Context, RowsRequest, RowVisitor) error { + v.add(&v.aqlFull, "zero-tag adapter ran") + return nil + }} + streamErr := stream.streamRaw(context.Background(), func(map[string]any) error { return nil }) + if !errors.Is(streamErr, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("zero-tagged command: err = %v, want %v", streamErr, command.ErrCompiledCommandBackendNotRegistered) + } + aqlFull, aqlPreview, pgFull, pgPreview, controls, ch := v.snapshot() + if len(aqlFull) != 0 || len(aqlPreview) != 0 || len(pgFull) != 0 || len(pgPreview) != 0 || len(controls) != 0 || ch != 0 { + t.Fatalf("zero-tagged command reached an adapter before refusal: aql=%v preview=%v pg=%v pgPreview=%v controls=%v ch=%d", + aqlFull, aqlPreview, pgFull, pgPreview, controls, ch) + } +} + +func TestSourceCommandBoundaryContractPostgreSQLNeverFallsBackToAQL(t *testing.T) { + v := &verifyArch01Trace{} + if _, err := New(Config{CompileTarget: compiler.ArangoTarget(), Registry: invalidRecipeRegistry{}}); err == nil || !strings.Contains(err.Error(), "source row port is required") { + t.Fatalf("engine without a source row port must be refused at construction: %v", err) + } + compiledCommand := verifyArch01Command(t) + stream := OutputStream{Name: "resources", command: compiledCommand} + if streamErr := stream.streamRaw(context.Background(), func(map[string]any) error { return nil }); !errors.Is(streamErr, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("PostgreSQL command without executor: err = %v, want %v", streamErr, command.ErrCompiledCommandBackendNotRegistered) + } + aqlFull, aqlPreview, pgFull, pgPreview, _, ch := v.snapshot() + if len(pgFull) != 0 || len(pgPreview) != 0 || ch != 0 { + t.Fatalf("missing-executor path reached an executor: pg=%v pgPreview=%v ch=%d", pgFull, pgPreview, ch) + } + if len(aqlFull) != 0 || len(aqlPreview) != 0 { + t.Fatalf("PostgreSQL request fell back to the AQL executor: full=%v preview=%v", aqlFull, aqlPreview) + } +} + +func TestSourceCommandBoundaryContractAQLWithoutAdapterFailsClosed(t *testing.T) { + compiledCommand, err := command.NewAQLCommand("FOR d IN patients RETURN d", map[string]any{"project": "P1"}) + if err != nil { + t.Fatalf("forge AQL command: %v", err) + } + stream := OutputStream{Name: "resources", command: compiledCommand} + if streamErr := stream.streamRaw(context.Background(), func(map[string]any) error { return nil }); !errors.Is(streamErr, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("AQL command without adapter: err = %v, want %v", streamErr, command.ErrCompiledCommandBackendNotRegistered) + } +} + +func TestSourceCommandBoundaryContractAQLExecutorReceivesTypedCommand(t *testing.T) { + v := &verifyArch01Trace{} + config := verifyArch01Config(v) + engine, err := New(config) + if err != nil { + t.Fatalf("New: %v", err) + } + resolved, err := engine.CompileResolvedBundle(context.Background(), testResolvedBundle([]string{}), + recipe.RuntimeBindings{Project: "P1", DatasetGeneration: "G1"}) + if err != nil { + t.Fatalf("compile: %v", err) + } + streams, err := engine.Streams(context.Background(), resolved) + if err != nil { + t.Fatalf("Streams: %v", err) + } + if len(streams) == 0 { + t.Fatal("no streams to exercise the AQL boundary") + } + for _, stream := range streams { + if _, err := stream.Stream(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("Stream: %v", err) + } + } + aqlFull, _, _, _, _, _ := v.snapshot() + if len(aqlFull) != 1 || !strings.Contains(aqlFull[0], "RETURN") { + t.Fatalf("AQL executor did not receive the compiled command text: %v", aqlFull) + } + captured := command.CompiledCommand{} + captureEngine, err := New(Config{CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "capture-scope" }, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + captured = command + return nil + }, + }) + if err != nil { + t.Fatalf("New capture engine: %v", err) + } + captureResolved, err := captureEngine.CompileResolvedBundle(context.Background(), testResolvedBundle([]string{}), + recipe.RuntimeBindings{Project: "P1", DatasetGeneration: "G1"}) + if err != nil { + t.Fatalf("capture compile: %v", err) + } + captureStreams, err := captureEngine.Streams(context.Background(), captureResolved) + if err != nil { + t.Fatalf("capture Streams: %v", err) + } + if len(captureStreams) == 0 { + t.Fatal("no capture streams") + } + for _, stream := range captureStreams { + if _, err := stream.Stream(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("capture Stream: %v", err) + } + } + if captured.Backend() != command.CompiledBackendAQL { + t.Fatalf("adapter received command tagged %q, want the AQL tag", captured.Backend()) + } +} + +func TestSourceCommandBoundaryContractConfigAdaptersAreNamedTypes(t *testing.T) { + fields := parseStructFieldTypes(t, "Config") + allowed := map[string]bool{ + "SourceRows": true, "PreviewSourceRows": true, + "PreviewQueryRows": true, "ClickHouseQueryRows": true, + } + for _, field := range fields { + if !strings.HasSuffix(field.name, "SourceRows") { + continue + } + if !allowed[field.name] { + t.Fatalf("Config has unexpected SourceRows-typed field %q; the executor field set is closed", field.name) + } + if field.inlineFunc { + t.Fatalf("Config field %q uses an inline func type; row executors must be the named typed contracts", field.name) + } + if !namedExecutorTypes[field.typeText] { + t.Fatalf("Config field %q has executor type %q, want one of the named typed contracts", field.name, field.typeText) + } + } +} + +var namedExecutorTypes = map[string]bool{ + "SourceRows": true, + "PostgreSQLQueryRows": true, + "ClickHouseQueryRows": true, +} + +func TestSourceCommandBoundaryContractNoLooseRowAdapterSignatures(t *testing.T) { + fset := token.NewFileSet() + packages, err := parser.ParseDir(fset, ".", func(info fs.FileInfo) bool { + return strings.HasSuffix(info.Name(), ".go") && !strings.HasSuffix(info.Name(), "_test.go") + }, 0) + if err != nil { + t.Fatalf("parse package sources: %v", err) + } + for _, pkg := range packages { + for _, file := range pkg.Files { + ast.Inspect(file, func(node ast.Node) bool { + function, ok := node.(*ast.FuncType) + if !ok || function.Params == nil { + return true + } + if isLooseRowAdapterSignature(function) { + position := fset.Position(function.Pos()) + t.Errorf("%s: loose row-adapter signature (context, string, int, bind map, visitor) reappeared", position) + } + return true + }) + } + } +} + +func isLooseRowAdapterSignature(signature *ast.FuncType) bool { + if signature.Params == nil || len(signature.Params.List) < 4 { + return false + } + var flattened []ast.Expr + for _, field := range signature.Params.List { + for range field.Names { + flattened = append(flattened, field.Type) + } + if len(field.Names) == 0 { + flattened = append(flattened, field.Type) + } + } + if len(flattened) < 5 { + return false + } + if !isQualifiedIdent(flattened[0], "context", "Context") { + return false + } + if identName(flattened[1]) != "string" || identName(flattened[2]) != "int" { + return false + } + bindMap, ok := flattened[3].(*ast.MapType) + if !ok || identName(bindMap.Key) != "string" { + return false + } + switch bindMap.Value.(type) { + case *ast.Ident, *ast.InterfaceType: + default: + return false + } + _, isVisitor := flattened[4].(*ast.FuncType) + return isVisitor +} + +type structFieldInfo struct { + name string + typeText string + inlineFunc bool +} + +func parseStructFieldTypes(t *testing.T, structName string) []structFieldInfo { + t.Helper() + fset := token.NewFileSet() + packages, err := parser.ParseDir(fset, ".", func(info fs.FileInfo) bool { + return strings.HasSuffix(info.Name(), ".go") && !strings.HasSuffix(info.Name(), "_test.go") + }, 0) + if err != nil { + t.Fatalf("parse package sources: %v", err) + } + var fields []structFieldInfo + found := false + for _, pkg := range packages { + for _, file := range pkg.Files { + for _, decl := range file.Decls { + general, ok := decl.(*ast.GenDecl) + if !ok || general.Tok != token.TYPE { + continue + } + for _, spec := range general.Specs { + typeSpec, ok := spec.(*ast.TypeSpec) + if !ok || typeSpec.Name.Name != structName { + continue + } + found = true + subject, ok := typeSpec.Type.(*ast.StructType) + if !ok { + t.Fatalf("%s is not a struct", structName) + } + for _, field := range subject.Fields.List { + typeText := exprText(field.Type) + inline := false + if _, ok := field.Type.(*ast.FuncType); ok { + inline = true + } + if len(field.Names) == 0 { + fields = append(fields, structFieldInfo{typeText: typeText, inlineFunc: inline}) + continue + } + for _, name := range field.Names { + fields = append(fields, structFieldInfo{name: name.Name, typeText: typeText, inlineFunc: inline}) + } + } + } + } + } + } + if !found { + t.Fatalf("type %s not found in package sources", structName) + } + return fields +} + +func exprText(expr ast.Expr) string { + switch typed := expr.(type) { + case *ast.Ident: + return typed.Name + case *ast.SelectorExpr: + return exprText(typed.X) + "." + typed.Sel.Name + case *ast.StarExpr: + return "*" + exprText(typed.X) + case *ast.FuncType: + return "func" + default: + return "?" + } +} + +func identName(expr ast.Expr) string { + if ident, ok := expr.(*ast.Ident); ok { + return ident.Name + } + return "" +} + +func isQualifiedIdent(expr ast.Expr, pkg, name string) bool { + selector, ok := expr.(*ast.SelectorExpr) + if !ok { + return false + } + return identName(selector.X) == pkg && selector.Sel.Name == name +} + +func TestSourceCommandBoundaryContractPostgreSQLServedQueryCarriesDocumentedTag(t *testing.T) { + config := verifyArch01Config(&verifyArch01Trace{}) + config.CompileTarget = verifyArch01PostgreSQLTarget(t) + config.SourceRows = func(context.Context, RowsRequest, RowVisitor) error { + return errors.New("seam inspection must not invoke the source row port") + } + config.PreviewSourceRows = nil + engine, err := New(config) + if err != nil { + t.Fatalf("New: %v", err) + } + _, query, err := engine.streamForOutput(context.Background(), verifyArch01PostgreSQLResolved("resources", []string{"id"}), "resources", 5) + if err != nil { + t.Fatalf("streamForOutput: %v", err) + } + if query.Command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("PostgreSQL-served compiler.CompiledQuery carries backend %q, want %q: the serving seam must hand callers the documented tagged command", + query.Command.Backend(), command.CompiledBackendPostgreSQL) + } + sqlText, args, err := query.Command.PostgreSQLParameters() + if err != nil { + t.Fatalf("served query command must dispatch as PostgreSQL: %v", err) + } + target := verifyArch01PostgreSQLTarget(t) + reference, err := compiler.CompileRecipeOutput(target, verifyArch01PostgreSQLResolved("resources", []string{"id"}).Compiled.Outputs[0], + verifyArch01PostgreSQLResolved("resources", []string{"id"}).Semantic.SemanticPlan.Bindings, 5, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("reference PostgreSQL compile: %v", err) + } + wantSQL, wantArgs, err := reference.Command.PostgreSQLParameters() + if err != nil { + t.Fatalf("reference parameters: %v", err) + } + if sqlText != wantSQL || !reflect.DeepEqual(args, wantArgs) { + t.Fatalf("served query command payload drifted from the compiler-rendered statement: sql=%q args=%v want sql=%q args=%v", sqlText, args, wantSQL, wantArgs) + } + if strings.Contains(sqlText, "@") { + t.Fatalf("served PostgreSQL statement carries AQL bind syntax: %s", sqlText) + } +} diff --git a/internal/dataframe/execution/construction_cell_trace_arango_test.go b/internal/dataframe/execution/construction_cell_trace_arango_test.go index cfef1be9f..3ae4e8101 100644 --- a/internal/dataframe/execution/construction_cell_trace_arango_test.go +++ b/internal/dataframe/execution/construction_cell_trace_arango_test.go @@ -8,12 +8,12 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/semantic" - store "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -24,12 +24,13 @@ func TestConstructionDerivedCellTraceReturnsValueAndStageLineageAgainstArango(t } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := store.Open(ctx, url, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) - if err := client.Bootstrap(ctx, store.BootstrapSpec{Collections: []store.CollectionSpec{{Name: "Observation"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Observation"}}}); err != nil { t.Fatal(err) } @@ -73,16 +74,21 @@ func TestConstructionDerivedCellTraceReturnsValueAndStageLineageAgainstArango(t t.Fatal(err) } compiledOutput := compiled.Outputs[0] - trace, err := compiler.CompileCellTraceOutputWithPolicy(compiledOutput, "total", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) + trace, err := compiler.CompileCellTraceOutput(compiler.ArangoTarget(), compiledOutput, "total", 0, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatal(err) } - engine := &Engine{batchSize: 100, queryRows: func(queryCtx context.Context, query string, batchSize int, bindVars map[string]any, visit func(map[string]any) error) error { - return client.QueryRows(queryCtx, query, batchSize, bindVars, visit) + engine := &Engine{compileTarget: compiler.ArangoTarget(), batchSize: 100, sourceRows: func(queryCtx context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + batchSize := request.BatchSize + query, bindVars, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + + return client.QueryRows(queryCtx, query, batchSize, bindVars, arangofactory.RowVisitor(visit)) }} for _, want := range wants { - // Source identities include both tenant scope and the inserted Arango key. - // Supplying only the first compiled field loses one of those parts. identity := map[string]any{"project": project, "_key": project + "_" + want.id} if err := ensureStableRowIdentity(identity, compiledOutput.RowIdentity, nil); err != nil { t.Fatalf("build expected row identity for %q: %v", want.id, err) diff --git a/internal/dataframe/execution/control.go b/internal/dataframe/execution/control.go index f7677a519..730fdea7d 100644 --- a/internal/dataframe/execution/control.go +++ b/internal/dataframe/execution/control.go @@ -6,20 +6,17 @@ import ( "fmt" "github.com/calypr/loom/internal/dataframe/compiler" - "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/command" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" "github.com/calypr/loom/internal/dataframe/recipe" recipeexec "github.com/calypr/loom/internal/dataframe/recipe/exec" "github.com/calypr/loom/internal/dataframe/semantic" ) -// Control adapts the production recipe engine to the backend-neutral -// recipe control contract consumed by GraphQL and other transport adapters. -// It resolves a recipe exactly once per operation and exposes only sanitized -// physical diagnostics through ExplainPhysical; raw IR and AQL stay private. type Control struct { - Engine *Engine - ExplainConnection func(context.Context, CompiledQuery) (ExplainAssessment, error) + Engine *Engine + ExplainConnection func(context.Context, compiler.CompiledQuery) (ExplainAssessment, error) + ExplainPostgreSQLConnection func(context.Context, compiler.CompiledQuery) (PostgreSQLPlanAssessment, error) } func (c Control) Validate(ctx context.Context, name string, bindings recipe.RuntimeBindings) (Validation, error) { @@ -55,10 +52,6 @@ func (c Control) Explain(ctx context.Context, name string, bindings recipe.Runti return validated.Plan.Explain(), nil } -// ExplainPhysical compiles the same resolved canonical outputs consumed by -// Preview and Materialize. The compiler-only path is always available; live -// Arango assessment is opt-in and requires ExplainConnection. No AQL or bind -// values cross the recipe control boundary. func (c Control) ExplainPhysical(ctx context.Context, name string, bindings recipe.RuntimeBindings, live bool) (PhysicalExplanation, error) { if c.Engine == nil { return PhysicalExplanation{}, fmt.Errorf("recipe engine is required") @@ -67,29 +60,52 @@ func (c Control) ExplainPhysical(ctx context.Context, name string, bindings reci if err != nil { return PhysicalExplanation{}, err } - limit := bindings.PreviewLimit + return c.ExplainResolved(ctx, resolved, live) +} + +func (c Control) ExplainResolved(ctx context.Context, resolved Resolved, live bool) (PhysicalExplanation, error) { + if c.Engine == nil { + return PhysicalExplanation{}, fmt.Errorf("recipe engine is required") + } + limit := resolved.Semantic.SemanticPlan.Bindings.PreviewLimit if limit <= 0 { limit = 25 } result := PhysicalExplanation{Outputs: make([]PhysicalOutputExplanation, 0, len(resolved.Compiled.Outputs))} for _, output := range resolved.Compiled.Outputs { - compiled, err := compiler.CompileRecipeOutputWithPolicy(output, resolved.Semantic.SemanticPlan.Bindings, limit, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := c.explainCompiledForOutput(ctx, resolved, output, limit) if err != nil { return PhysicalExplanation{}, fmt.Errorf("output %q: %w", output.Name, err) } + fingerprint := SourcePlanFingerprint(compiled) + if err := requireProjectedPlanFingerprint(output.Name, fingerprint); err != nil { + return PhysicalExplanation{}, err + } item := PhysicalOutputExplanation{ - Name: output.Name, PlanFingerprint: CompiledQueryFingerprint(compiled), + Name: output.Name, PlanFingerprint: fingerprint, Columns: append([]string(nil), compiled.PublicColumns...), Diagnostics: compiled.PlanDiagnostics, } if live { - if c.ExplainConnection == nil { - return PhysicalExplanation{}, fmt.Errorf("live physical recipe Explain requires an Arango connection") - } - assessment, err := c.ExplainConnection(ctx, compiled) - if err != nil { - return PhysicalExplanation{}, recipeControlBackend(fmt.Errorf("output %q live Explain: %w", output.Name, err)) + switch compiled.Command.Backend() { + case command.CompiledBackendPostgreSQL: + if c.ExplainPostgreSQLConnection == nil { + return PhysicalExplanation{}, fmt.Errorf("live physical recipe Explain requires a PostgreSQL connection") + } + assessment, err := c.ExplainPostgreSQLConnection(ctx, compiled) + if err != nil { + return PhysicalExplanation{}, recipeControlBackend(fmt.Errorf("output %q live Explain: %w", output.Name, err)) + } + item.LivePostgreSQL = &assessment + default: + if c.ExplainConnection == nil { + return PhysicalExplanation{}, fmt.Errorf("live physical recipe Explain requires an Arango connection") + } + assessment, err := c.ExplainConnection(ctx, compiled) + if err != nil { + return PhysicalExplanation{}, recipeControlBackend(fmt.Errorf("output %q live Explain: %w", output.Name, err)) + } + item.Live = &assessment } - item.Live = &assessment } result.Outputs = append(result.Outputs, item) } @@ -123,9 +139,6 @@ func (c Control) Preview(ctx context.Context, name string, bindings recipe.Runti return Preview{Plan: resolved, Outputs: outputRows(full, rows)}, nil } -// Run executes the complete resolved recipe without a preview limit. It is an -// explicit, bounded-by-the-caller's-memory transport operation; production -// bulk workflows should use Materialize instead. func (c Control) Run(ctx context.Context, name string, bindings recipe.RuntimeBindings) (Preview, error) { if c.Engine == nil { return Preview{}, fmt.Errorf("recipe engine is required") diff --git a/internal/dataframe/execution/control_explain_types.go b/internal/dataframe/execution/control_explain_types.go index 2ff047652..726d11a1c 100644 --- a/internal/dataframe/execution/control_explain_types.go +++ b/internal/dataframe/execution/control_explain_types.go @@ -2,13 +2,8 @@ package execution import ( "github.com/calypr/loom/internal/dataframe/compiler/ir" - "github.com/calypr/loom/internal/store/arango" ) -// PhysicalExplanation is the output-scoped physical half of recipe Explain. -// It contains compiler structure and an optional live Arango assessment; it -// never contains AQL text, bind values, auth paths, credentials, or connection -// information. type PhysicalExplanation struct { Outputs []PhysicalOutputExplanation } @@ -19,11 +14,43 @@ type PhysicalOutputExplanation struct { Columns []string Diagnostics ir.CompilerPlanDiagnostics Live *ExplainAssessment + LivePostgreSQL *PostgreSQLPlanAssessment +} + +const ( + PostgreSQLScanSequential = "sequential" + PostgreSQLScanIndex = "index" +) + +type PostgreSQLPlanAssessment struct { + Backend string + Nodes []PostgreSQLPlanNode + FullScans []PostgreSQLScanEvidence + IndexScans []PostgreSQLScanEvidence + RootEstimate PostgreSQLPlanEstimate +} + +type PostgreSQLPlanNode struct { + Ordinal int + ParentOrdinal int + Depth int + NodeType string + Estimate PostgreSQLPlanEstimate +} + +type PostgreSQLPlanEstimate struct { + StartupCost float64 + TotalCost float64 + PlanRows float64 + PlanWidth int +} + +type PostgreSQLScanEvidence struct { + Ordinal int + NodeType string + ScanKind string } -// ExplainAssessment is a transport-neutral, sanitized copy of Arango's -// ExplainAssessment. Keeping this DTO here lets GraphQL and HTTP adapters use -// one assessment shape without parsing Explain responses themselves. type ExplainAssessment struct { Plans []ExplainPlanEstimate FullCollectionScans []ExplainCollectionScan @@ -62,37 +89,3 @@ type ExplainWarning struct { Code int Message string } - -// AssessmentFromArango converts the common store assessment to the sanitized -// control-plane DTO. It copies every slice so callers cannot mutate shared -// assessment state. -func AssessmentFromArango(input arango.ExplainAssessment) ExplainAssessment { - out := ExplainAssessment{ - Plans: make([]ExplainPlanEstimate, 0, len(input.Plans)), - FullCollectionScans: make([]ExplainCollectionScan, 0, len(input.FullCollectionScans)), - Indexes: make([]ExplainIndexSummary, 0, len(input.Indexes)), - Warnings: make([]ExplainWarning, 0, len(input.Warnings)), - AppliedOptimizerRules: append([]string(nil), input.AppliedOptimizerRules...), - } - for _, plan := range input.Plans { - out.Plans = append(out.Plans, ExplainPlanEstimate{Plan: plan.Plan, EstimatedCost: plan.EstimatedCost, EstimatedNrItems: plan.EstimatedNrItems}) - } - for _, scan := range input.FullCollectionScans { - out.FullCollectionScans = append(out.FullCollectionScans, ExplainCollectionScan{Plan: scan.Plan, NodeID: scan.NodeID, Collection: scan.Collection}) - } - for _, index := range input.Indexes { - copyIndex := ExplainIndexSummary{Collection: index.Collection, ID: index.ID, Name: index.Name, Type: index.Type, Fields: append([]string(nil), index.Fields...), Uses: make([]ExplainIndexLocation, 0, len(index.Uses))} - for _, use := range index.Uses { - copyIndex.Uses = append(copyIndex.Uses, ExplainIndexLocation{Plan: use.Plan, NodeID: use.NodeID}) - } - out.Indexes = append(out.Indexes, copyIndex) - } - for _, warning := range input.Warnings { - out.Warnings = append(out.Warnings, ExplainWarning{Code: warning.Code, Message: warning.Message}) - } - return out -} - -// ClonePhysicalExplanation returns an ownership-safe copy suitable for a -// transport adapter. Compiler diagnostics are structural and contain no bind -// values; clone their nested slices before returning them to callers. diff --git a/internal/dataframe/execution/control_types.go b/internal/dataframe/execution/control_types.go index bff98f0b7..355fc2066 100644 --- a/internal/dataframe/execution/control_types.go +++ b/internal/dataframe/execution/control_types.go @@ -1,6 +1,3 @@ -// Package control exposes the backend-neutral control-plane operations -// needed by GraphQL or HTTP adapters. It deliberately contains no transport -// types and no translation-specific output names. package execution import ( @@ -23,8 +20,3 @@ type OutputRows struct { Columns []string Rows []map[string]any } - -// ExplainPhysical returns compiler diagnostics and, when requested by the -// caller, a live Arango assessment for every resolved output. Semantic Explain -// remains available through Explain and is intentionally not replaced by this -// backend-facing view. diff --git a/internal/dataframe/execution/control_validate_digest_test.go b/internal/dataframe/execution/control_validate_digest_test.go index 0737bc0da..ab39a6691 100644 --- a/internal/dataframe/execution/control_validate_digest_test.go +++ b/internal/dataframe/execution/control_validate_digest_test.go @@ -3,6 +3,7 @@ package execution import ( "context" "errors" + "github.com/calypr/loom/internal/dataframe/compiler" "testing" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" @@ -10,13 +11,6 @@ import ( recipeexec "github.com/calypr/loom/internal/dataframe/recipe/exec" ) -// validateDataframeRecipe/explainDataframeRecipe used the named-registry -// lookup unconditionally, ignoring bindings.recipeDigest. Since the named -// registry only holds the startup default recipe, every digest-addressed -// validation of a registered revision returned RECIPE_NOT_FOUND even though -// the digest-addressed readback and preview/run paths resolved it (control -// asymmetry: Validate was the only control entry bypassing Engine.Resolve). - type valNamedAbsent struct{ loads int } func (r *valNamedAbsent) LoadRecipe(context.Context, string) (recipeexec.Entry, error) { @@ -56,9 +50,16 @@ func valControl(t *testing.T, revisions *valRevisions) (*Engine, *valNamedAbsent t.Helper() registry := &valNamedAbsent{} engine, err := New(Config{ - Registry: registry, - Revisions: revisions, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: registry, + Revisions: revisions, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil }, }) @@ -121,9 +122,6 @@ func TestValidateDigestBlankProjectGuardedFirst(t *testing.T) { } } -// Wave-3 boundary extended to validate: now that digest-addressed revisions -// resolve here, structural user faults stay typed INVALID_REQUEST through -// Validate (and therefore Explain, which delegates to it). func TestValidateDigestAddressedStructuralFaultStaysTyped(t *testing.T) { controlEngine, _ := valControl(t, &valRevisions{revision: revisionFor("Patient", "abc")}) control := Control{Engine: controlEngine} @@ -137,11 +135,6 @@ func TestValidateDigestAddressedStructuralFaultStaysTyped(t *testing.T) { } } -// A wrong digest must stay the resolver-classifiable sentinel (mapped to -// RECIPE_NOT_FOUND by recipeGraphQLError), not be wrapped into -// BACKEND_UNAVAILABLE by recipeControlBackend — the sibling preview lane -// (Engine.Resolve, unwrapped) returns the sentinel raw, and both lanes must -// give the boundary the same signal. func TestValidateUnknownDigestStaysResolverClassifiable(t *testing.T) { controlEngine, _ := valControl(t, &valRevisions{revision: revisionFor("patient", "abc")}) control := Control{Engine: controlEngine} diff --git a/internal/dataframe/execution/engine.go b/internal/dataframe/execution/engine.go index 8c9fe88d4..db30d6938 100644 --- a/internal/dataframe/execution/engine.go +++ b/internal/dataframe/execution/engine.go @@ -1,7 +1,3 @@ -// Package execution is Loom's production recipe execution seam. It owns -// recipe resolution, scoped discovery, canonical compiler orchestration, and -// streaming row execution; transport adapters do not interpret recipes or -// construct AQL. package execution import ( @@ -17,6 +13,7 @@ import ( "github.com/calypr/loom/internal/authscope" "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" "github.com/calypr/loom/internal/dataframe/compiler/optimize" @@ -28,15 +25,19 @@ import ( "github.com/calypr/loom/internal/dataframe/semantic" "github.com/calypr/loom/internal/dataframe/spec" "github.com/calypr/loom/internal/projectid" - "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) -type QueryRows func(context.Context, string, int, map[string]any, func(map[string]any) error) error +type RowVisitor func(map[string]any) error + +type RowsRequest struct { + Command command.CompiledCommand + DeclaredColumns []string + BatchSize int +} + +type SourceRows func(context.Context, RowsRequest, RowVisitor) error -// ClickHouseQueryRows executes a typed ClickHouse result query with a fixed -// result column contract and positional values. SQL fragments never come from -// recipe or request data. type ClickHouseQueryRows func(context.Context, string, []string, func(map[string]any) error, ...any) error type ResolveClickHouseInputs func(context.Context, ir.PhysicalClickHouseCombine, recipe.RuntimeBindings) ([]ir.ResolvedClickHouseTable, error) @@ -44,41 +45,30 @@ type ResolveClickHouseInputs func(context.Context, ir.PhysicalClickHouseCombine, type WithExecutionReadPins func(context.Context, []string, func(context.Context) error) error const ( - // DefaultPreviewLimit is used when a preview request omits its limit. DefaultPreviewLimit = 25 - // MaxPreviewLimit bounds rows accumulated by one output preview. - MaxPreviewLimit = 1000 - previewPlanMode = "physical" - previewPlanProfile = "generic_fhir_graph_recipe" + MaxPreviewLimit = 1000 + previewPlanMode = "physical" + previewPlanProfile = "generic_fhir_graph_recipe" ) -// PreviewRequest selects one compiled output and bounds its preview rows. type PreviewRequest struct { Output string Limit int IncludeRowIdentity bool } -// PreviewSummary is safe execution metadata for one preview output. It does -// not contain AQL, bind variables, or physical plan contents. type PreviewSummary struct { - Output string - Columns []string - RowCount int - PlanMode string - PlanProfile string - PlanFingerprint string - TraversalCount int - LoweringDuration time.Duration - QueryDuration time.Duration - // Complete is true only when the execution naturally exhausted before - // reaching its configured row limit. Truncated is explicit when the - // bounded preview stopped at errPreviewLimit; callers must not infer full - // population facts from a bounded result. - Complete bool - Truncated bool - // PartialValidation marks compiler-bounded construction previews that - // validate only the displayed deterministic sample of complete groups. + Output string + Columns []string + RowCount int + PlanMode string + PlanProfile string + PlanFingerprint string + TraversalCount int + LoweringDuration time.Duration + SourceQueryDuration time.Duration + Complete bool + Truncated bool PartialValidation bool RowLineageCapability compiler.RowLineageCapability } @@ -87,12 +77,13 @@ type Config struct { Registry exec.Reader Revisions recipe.RevisionStore ResolveBundle func(context.Context, recipe.Bundle, recipe.RuntimeBindings) (recipe.Bundle, error) - QueryRows QueryRows - PreviewQueryRows QueryRows + CompileTarget compiler.Target + SourceRows SourceRows + PreviewSourceRows SourceRows PreparePreviewIndex func(context.Context, compiler.PreviewCoveringIndexSpec) error - PreviewCoveringIndexes func(context.Context, string) ([]arango.PreviewCoveringIndex, error) + PreviewCoveringIndexes func(context.Context, string) ([]PreviewCollectionIndex, error) PreviewCollectionRevision func(context.Context, string) (string, error) - PreviewExplainQuery func(context.Context, string, map[string]any) (arango.ExplainResult, error) + PreviewCollectionScans PreviewCollectionScans PreviewCollectionCount func(context.Context, string) (int64, error) ClickHouseQueryRows ClickHouseQueryRows ResolveClickHouseInputs ResolveClickHouseInputs @@ -100,21 +91,20 @@ type Config struct { PrivateClickHouseArtifacts *chartifact.Manager ScopeDigest func(recipe.RuntimeBindings) string BatchSize int - // RootPageRows bounds the number of root documents evaluated by one wide - // output query. Zero keeps the legacy single-query execution path. - RootPageRows int + RootPageRows int } type Engine struct { registry exec.Reader revisions recipe.RevisionStore resolveBundle func(context.Context, recipe.Bundle, recipe.RuntimeBindings) (recipe.Bundle, error) - queryRows QueryRows - previewQueryRows QueryRows + compileTarget compiler.Target + sourceRows SourceRows + previewSourceRows SourceRows preparePreviewIndex func(context.Context, compiler.PreviewCoveringIndexSpec) error - previewCoveringIndexes func(context.Context, string) ([]arango.PreviewCoveringIndex, error) + previewCoveringIndexes func(context.Context, string) ([]PreviewCollectionIndex, error) previewCollectionRevision func(context.Context, string) (string, error) - previewExplainQuery func(context.Context, string, map[string]any) (arango.ExplainResult, error) + previewCollectionScans PreviewCollectionScans previewCollectionCount func(context.Context, string) (int64, error) groupPreviewRowsCache groupPreviewRowsCache clickHouseQueryRows ClickHouseQueryRows @@ -126,11 +116,6 @@ type Engine struct { rootPageRows int } -// Resolved contains the immutable recipe after schema discovery, semantic -// discovery provenance, and canonical physical plans per output. Bundle is the -// fully resolved recipe that may be passed back through CompileResolvedBundle -// after a receipt or other durable boundary. Compiled remains request-scoped -// runtime state; callers must not persist it. type Resolved struct { Bundle recipe.Bundle Semantic semantic.ResolvedRecipePlan @@ -139,9 +124,6 @@ type Resolved struct { ResolvedSchemaDigest string } -// ResolutionError marks failures while compiling a recipe before publication -// starts. Transport adapters can expose these as actionable recipe errors; -// publication and backend failures remain opaque internal errors. type ResolutionError struct{ Err error } func (e *ResolutionError) Error() string { @@ -158,19 +140,38 @@ func (e *ResolutionError) Unwrap() error { return e.Err } +var ErrPostgreSQLRootPageSteps = errors.New("postgresql root page steps are invalid") + +var ErrPostgreSQLResultContract = errors.New("postgresql root page result contract violation") + +type PostgreSQLRootPageSteps struct { + RootKeys command.CompiledCommand + Rows command.CompiledCommand + RootKeyColumn string + RowRootKeyColumn string + CursorArgIndex int + PageSizeArgIndex int + PageKeysArgIndex int + PageSize int +} + +const maxPostgreSQLRootPageSize = 1 << 20 + type OutputStream struct { Name string Columns []string sourceIdentityMode string RowIdentity *spec.RowIdentity DynamicChecks map[string]map[string]DynamicColumnCheck - query string - bindVars map[string]any - stream QueryRows + command command.CompiledCommand + expectedBackend command.CompiledBackend + commandColumns []string + sourceRows SourceRows batchSize int rootPageRows int initialRootPageRows int page *compiler.CompiledOutputPage + pgRootPage *PostgreSQLRootPageSteps physicalEngine ir.PhysicalEngine clickHouseCombine *ir.PhysicalClickHouseCombine project string @@ -197,10 +198,6 @@ type OutputStream struct { physicalPlan ir.PhysicalPlan } -// preparedClickHouseQuery is the immutable result of resolving every exact -// published input and capturing every same-workspace dependency. Its identity -// and opaque scope evidence are complete before a private artifact writer is -// opened. type preparedClickHouseQuery struct { Query string Columns []string @@ -211,9 +208,6 @@ type preparedClickHouseQuery struct { ScopeEvidence ir.WholeRelationScopeEvidence } -// materializedClickHouseCapture carries the source proof beside the live -// artifact lease. The proof remains in memory for recursive workspace-output -// composition; only its digest is persisted for whole-scope artifacts. type materializedClickHouseCapture struct { Artifact *chartifact.Artifact Identity ir.ClickHouseArtifactIdentity @@ -239,8 +233,15 @@ func New(cfg Config) (*Engine, error) { if cfg.Registry == nil { return nil, fmt.Errorf("recipe registry is required") } - if cfg.QueryRows == nil { - return nil, fmt.Errorf("recipe query executor is required") + if !cfg.CompileTarget.Valid() { + return nil, fmt.Errorf("compile target is required: %w", compiler.ErrTargetNotSelected) + } + if cfg.SourceRows == nil { + return nil, fmt.Errorf("source row port is required: composition must install the row adapter for the selected compile target") + } + previewRows := cfg.PreviewSourceRows + if previewRows == nil { + previewRows = cfg.SourceRows } batch := cfg.BatchSize if batch <= 0 { @@ -251,20 +252,18 @@ func New(cfg Config) (*Engine, error) { } return &Engine{ registry: cfg.Registry, revisions: cfg.Revisions, resolveBundle: cfg.ResolveBundle, - queryRows: cfg.QueryRows, previewQueryRows: cfg.PreviewQueryRows, preparePreviewIndex: cfg.PreparePreviewIndex, + compileTarget: cfg.CompileTarget, + sourceRows: cfg.SourceRows, previewSourceRows: previewRows, preparePreviewIndex: cfg.PreparePreviewIndex, previewCoveringIndexes: cfg.PreviewCoveringIndexes, - previewCollectionRevision: cfg.PreviewCollectionRevision, previewExplainQuery: cfg.PreviewExplainQuery, - previewCollectionCount: cfg.PreviewCollectionCount, clickHouseQueryRows: cfg.ClickHouseQueryRows, + previewCollectionRevision: cfg.PreviewCollectionRevision, previewCollectionScans: cfg.PreviewCollectionScans, + previewCollectionCount: cfg.PreviewCollectionCount, + clickHouseQueryRows: cfg.ClickHouseQueryRows, resolveClickHouseInputs: cfg.ResolveClickHouseInputs, withExecutionReadPins: cfg.WithExecutionReadPins, privateClickHouseArtifacts: cfg.PrivateClickHouseArtifacts, scopeDigest: cfg.ScopeDigest, batchSize: batch, rootPageRows: cfg.RootPageRows, }, nil } -// entry loads the addressed recipe revision when the bindings carry a -// recipeDigest, otherwise the named default entry. Every control and run -// surface must address recipes through this seam so a digest-addressed -// request behaves identically across preview, run, explain, and validate. func (e *Engine) entry(ctx context.Context, name string, bindings recipe.RuntimeBindings) (exec.Entry, error) { if strings.TrimSpace(bindings.Project) == "" { return exec.Entry{}, dataframeerrors.NewError(dataframeerrors.CodeProjectRequired, "") @@ -287,8 +286,6 @@ func (e *Engine) Resolve(ctx context.Context, name string, bindings recipe.Runti return e.resolveEntry(ctx, entry, bindings) } -// ResolveVersion loads an exact immutable recipe version. New publication -// workflows must use this method; Resolve is the deprecated default alias. func (e *Engine) ResolveVersion(ctx context.Context, name, translationVersion string, bindings recipe.RuntimeBindings) (Resolved, error) { entry, err := e.registry.LoadRecipeVersion(ctx, name, translationVersion) if err != nil { @@ -297,10 +294,6 @@ func (e *Engine) ResolveVersion(ctx context.Context, name, translationVersion st return e.resolveEntry(ctx, entry, bindings) } -// ResolveBundle runs an unregistered immutable bundle through the identical -// schema, semantic, optimization, and lowering pipeline as a registry entry. -// Explorer uses this to avoid creating recipe drafts/revisions for authored UI -// intent. func (e *Engine) ResolveBundle(ctx context.Context, bundle recipe.Bundle, bindings recipe.RuntimeBindings) (Resolved, error) { if strings.TrimSpace(bindings.Project) == "" { return Resolved{}, dataframeerrors.NewError(dataframeerrors.CodeProjectRequired, "") @@ -312,11 +305,11 @@ func (e *Engine) ResolveBundle(ctx context.Context, bundle recipe.Bundle, bindin return e.resolveEntry(ctx, exec.Entry{Bundle: bundle, Digest: digest}, bindings) } -// CompileResolvedBundle validates and compiles a bundle that has already been -// resolved by schema discovery. It never invokes Config.ResolveBundle and -// therefore performs no catalog or schema discovery. Dynamic declarations -// with nil Columns are rejected by the semantic boundary; a non-nil empty -// Columns slice is an explicitly resolved, zero-column family and is valid. +func (e *Engine) CombinePortsConfigured() bool { + return e.clickHouseQueryRows != nil && e.resolveClickHouseInputs != nil && + e.withExecutionReadPins != nil && e.privateClickHouseArtifacts != nil +} + func (e *Engine) CompileResolvedBundle(ctx context.Context, bundle recipe.Bundle, bindings recipe.RuntimeBindings) (Resolved, error) { if strings.TrimSpace(bindings.Project) == "" { return Resolved{}, dataframeerrors.NewError(dataframeerrors.CodeProjectRequired, "") @@ -336,8 +329,6 @@ func (e *Engine) PreviewBundle(ctx context.Context, bundle recipe.Bundle, bindin return e.Preview(ctx, resolved, bindings.PreviewLimit) } -// PreviewResolvedBundle compiles an already schema-resolved bundle without -// catalog discovery, then executes its request-scoped preview. func (e *Engine) PreviewResolvedBundle(ctx context.Context, bundle recipe.Bundle, bindings recipe.RuntimeBindings) (map[string][]map[string]any, error) { resolved, err := e.CompileResolvedBundle(ctx, bundle, bindings) if err != nil { @@ -359,9 +350,6 @@ func (e *Engine) MaterializeBundle(ctx context.Context, bundle recipe.Bundle, bi return resolved, nil } -// MaterializeResolvedBundle compiles an already schema-resolved bundle -// without catalog discovery and hands the resulting request-scoped plan to -// the publisher. func (e *Engine) MaterializeResolvedBundle(ctx context.Context, bundle recipe.Bundle, bindings recipe.RuntimeBindings, publish func(context.Context, Resolved) error) (Resolved, error) { resolved, err := e.CompileResolvedBundle(ctx, bundle, bindings) if err != nil { @@ -393,9 +381,6 @@ func (e *Engine) resolveEntry(ctx context.Context, entry exec.Entry, bindings re return e.compileResolvedBundle(ctx, bundle, bindings, storedRecipeDigest) } -// compileResolvedBundle is the shared post-discovery compiler. Keeping this -// helper separate from resolveEntry makes it impossible for receipt-backed -// callers to accidentally rediscover catalog fields during compilation. func (e *Engine) compileResolvedBundle(ctx context.Context, bundle recipe.Bundle, bindings recipe.RuntimeBindings, storedRecipeDigest string) (Resolved, error) { if strings.TrimSpace(bindings.Project) == "" { return Resolved{}, dataframeerrors.NewError(dataframeerrors.CodeProjectRequired, "") @@ -411,9 +396,6 @@ func (e *Engine) compileResolvedBundle(ctx context.Context, bundle recipe.Bundle if err != nil { return Resolved{}, err } - // The public recipe identity is the registered document digest. The - // resolved schema digest below captures catalog-derived fields and scope so - // materializations cannot collide when one recipe resolves differently. semanticPlan.RecipeDigest = storedRecipeDigest scope := "" if e.scopeDigest != nil { @@ -427,9 +409,6 @@ func (e *Engine) compileResolvedBundle(ctx context.Context, bundle recipe.Bundle if err != nil { return Resolved{}, err } - // Lowering and AQL optimization are request-scoped work. ClickHouse plans - // have no AQL traversal rewrites; validate and cache their exact typed plan. - // Preview/materialization streams then only clone, window, and render it. policy := ir.DefaultPhysicalOptimizationPolicy() for index := range compiled.Outputs { plan := compiled.Outputs[index].Plan @@ -485,8 +464,6 @@ func resolvedBundleSchemaDigest(storedDigest string, bundle recipe.Bundle, bindi return hex.EncodeToString(sum[:]), nil } -// Materialize resolves a recipe once and hands the complete resolved plan to -// the publisher. Resolve and compile therefore share one discovery snapshot. func (e *Engine) Materialize(ctx context.Context, name string, bindings recipe.RuntimeBindings, publish func(context.Context, Resolved) error) (Resolved, error) { resolved, err := e.Resolve(ctx, name, bindings) if err != nil { @@ -517,9 +494,6 @@ func (e *Engine) Streams(ctx context.Context, resolved Resolved) ([]OutputStream return e.streamsWithLimit(ctx, resolved, 0) } -// Run consumes every row from every selected output. Unlike Preview, this -// deliberately compiles without an execution LIMIT and is intended for -// callers that explicitly requested the complete dataframe in memory. func (e *Engine) Run(ctx context.Context, resolved Resolved) (map[string][]map[string]any, error) { streams, err := e.Streams(ctx, resolved) if err != nil { @@ -540,7 +514,7 @@ func (e *Engine) Run(ctx context.Context, resolved Resolved) (map[string][]map[s return result, nil } -func (e *Engine) streamsWithLimit(_ context.Context, resolved Resolved, limit int) ([]OutputStream, error) { +func (e *Engine) streamsWithLimit(ctx context.Context, resolved Resolved, limit int) ([]OutputStream, error) { streams := make([]OutputStream, 0, len(resolved.Compiled.Outputs)) selected := selectedOutputNames(resolved.Semantic.SemanticPlan.Bindings.OutputNames, resolved.Compiled.Outputs) for _, name := range resolved.Semantic.SemanticPlan.Bindings.OutputNames { @@ -552,7 +526,7 @@ func (e *Engine) streamsWithLimit(_ context.Context, resolved Resolved, limit in if selected != nil && !selected[output.Name] { continue } - stream, _, err := e.streamForOutput(resolved, output.Name, limit) + stream, _, err := e.streamForOutput(ctx, resolved, output.Name, limit) if err != nil { return nil, err } @@ -561,21 +535,21 @@ func (e *Engine) streamsWithLimit(_ context.Context, resolved Resolved, limit in return streams, nil } -func (e *Engine) streamForOutput(resolved Resolved, name string, limit int) (OutputStream, compiler.CompiledQuery, error) { +func (e *Engine) streamForOutput(ctx context.Context, resolved Resolved, name string, limit int) (OutputStream, compiler.CompiledQuery, error) { for _, output := range resolved.Compiled.Outputs { if output.Name != name { continue } - return e.streamForCompiledOutput(resolved, output, limit) + return e.streamForCompiledOutput(ctx, resolved, output, limit) } return OutputStream{}, compiler.CompiledQuery{}, previewAdmissionError(dataframeerrors.CodeInvalidRequest, "requested preview output is not available", map[string]any{"output": name}) } -func (e *Engine) streamForCompiledOutput(resolved Resolved, output lower.CompiledRecipeOutput, limit int) (OutputStream, compiler.CompiledQuery, error) { - return e.buildCompiledOutputStream(resolved, output, limit, make(map[string]bool), false) +func (e *Engine) streamForCompiledOutput(ctx context.Context, resolved Resolved, output lower.CompiledRecipeOutput, limit int) (OutputStream, compiler.CompiledQuery, error) { + return e.buildCompiledOutputStream(ctx, resolved, output, limit, make(map[string]bool), false) } -func (e *Engine) buildCompiledOutputStream(resolved Resolved, output lower.CompiledRecipeOutput, limit int, building map[string]bool, privateArtifactCapture bool) (OutputStream, compiler.CompiledQuery, error) { +func (e *Engine) buildCompiledOutputStream(ctx context.Context, resolved Resolved, output lower.CompiledRecipeOutput, limit int, building map[string]bool, privateArtifactCapture bool) (OutputStream, compiler.CompiledQuery, error) { if building[output.Name] { return OutputStream{}, compiler.CompiledQuery{}, fmt.Errorf("workspace output dependency cycle includes %q", output.Name) } @@ -583,30 +557,35 @@ func (e *Engine) buildCompiledOutputStream(resolved Resolved, output lower.Compi defer delete(building, output.Name) bindings := resolved.Semantic.SemanticPlan.Bindings.Clone() if privateArtifactCapture && requiresRestrictedRowAuthPathCapture(output, bindings) { - // Private row-scoped artifacts need the source path on each row so the - // downstream Combine can reapply the exact authorization scope. Keep this - // projection request-local; it does not change preview bindings/schema. bindings.IncludeAuthResourcePath = true } switch output.Plan.Engine { case ir.PhysicalEngineClickHouse: - return e.clickHouseStreamForOutput(resolved, output, limit, building, bindings) + return e.clickHouseStreamForOutput(ctx, resolved, output, limit, building, bindings) case "", ir.PhysicalEngineAQL: default: return OutputStream{}, compiler.CompiledQuery{}, fmt.Errorf("output %q has unsupported physical engine %q", output.Name, output.Plan.Engine) } - query, err := compiler.CompileRecipeOutputWithPolicy(output, bindings, limit, ir.DefaultPhysicalOptimizationPolicy()) + if e.compileTarget.Backend() == command.CompiledBackendPostgreSQL { + if output.Plan.ClickHouseCombine != nil { + return OutputStream{}, compiler.CompiledQuery{}, fmt.Errorf("output %q: PostgreSQL-served output cannot carry a ClickHouse combine payload", output.Name) + } + } + query, err := compiler.CompileRecipeOutput(e.compileTarget, output, bindings, limit, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { return OutputStream{}, compiler.CompiledQuery{}, fmt.Errorf("output %q: %w", output.Name, err) } + if query.Command.Backend() == command.CompiledBackendPostgreSQL { + return e.postgreSQLStreamForOutput(resolved, output, limit, bindings, query, privateArtifactCapture) + } rootPageRows := e.rootPageRows if limit > 0 && limit < rootPageRows { rootPageRows = limit } stream := OutputStream{ Name: output.Name, Columns: append([]string(nil), query.PublicColumns...), sourceIdentityMode: previewSourceIdentityMode(output, query.OutputSchema), RowIdentity: query.RowIdentity.Clone(), - DynamicChecks: dynamicChecks(output.DynamicColumns), query: query.Query, bindVars: query.BindVars, - stream: e.queryRows, batchSize: e.batchSize, rootPageRows: rootPageRows, + DynamicChecks: dynamicChecks(output.DynamicColumns), command: query.Command, expectedBackend: e.compileTarget.Backend(), + sourceRows: e.sourceRows, batchSize: e.batchSize, rootPageRows: rootPageRows, queryLimit: limit, recipeDigest: resolved.StoredRecipeDigest, planFingerprint: query.PlanDiagnostics.Fingerprint, bindings: bindings, stageID: compiledOutputFinalStageID(output), outputSchema: lower.CloneCompiledOutputSchema(output.OutputSchema), compiledOutput: true, @@ -620,7 +599,6 @@ func (e *Engine) buildCompiledOutputStream(resolved Resolved, output lower.Compi if scopeMode == chartifact.ScopeModeWhole && scopeEvidence != nil { stream.artifactScopeEvidence = scopeEvidence.Digest() } - // Aggregation must see the full input before the output row limit applies. wholeInput := len(output.Plan.Operations) == 1 && output.Plan.Operations[0].Kind == ir.PhysicalGroupRowsOp && output.Plan.Operations[0].GroupRows != nil for _, operation := range output.Plan.Operations { @@ -645,7 +623,7 @@ func (e *Engine) buildCompiledOutputStream(resolved Resolved, output lower.Compi if rootPageRows > 0 && !wholeInput { pageBindings := resolved.Semantic.SemanticPlan.Bindings.Clone() pageBindings.PreviewLimit = limit - page, pageErr := compiler.CompileRecipeOutputPageWithPolicy(output, pageBindings, rootPageRows, ir.DefaultPhysicalOptimizationPolicy()) + page, pageErr := compiler.CompileRecipeOutputPage(e.compileTarget, output, pageBindings, rootPageRows, ir.DefaultPhysicalOptimizationPolicy()) if pageErr != nil { return OutputStream{}, compiler.CompiledQuery{}, fmt.Errorf("output %q paging: %w", output.Name, pageErr) } @@ -676,7 +654,7 @@ func compiledFinalStageID(plan ir.PhysicalPlan) string { return recipe.ConstructionSourceProjectionID } -func (e *Engine) clickHouseStreamForOutput(resolved Resolved, output lower.CompiledRecipeOutput, limit int, building map[string]bool, bindings recipe.RuntimeBindings) (OutputStream, compiler.CompiledQuery, error) { +func (e *Engine) clickHouseStreamForOutput(ctx context.Context, resolved Resolved, output lower.CompiledRecipeOutput, limit int, building map[string]bool, bindings recipe.RuntimeBindings) (OutputStream, compiler.CompiledQuery, error) { physicalPlan := output.Plan if output.OptimizedPlan != nil { physicalPlan = ir.ClonePhysicalPlan(*output.OptimizedPlan) @@ -742,7 +720,7 @@ func (e *Engine) clickHouseStreamForOutput(resolved Resolved, output lower.Compi if !ok { return OutputStream{}, compiler.CompiledQuery{}, fmt.Errorf("output %q is missing compiled workspace dependency %q", output.Name, input.WorkspaceOutputID) } - dependencyStream, _, err := e.buildCompiledOutputStream(resolved, dependency, 0, building, true) + dependencyStream, _, err := e.buildCompiledOutputStream(ctx, resolved, dependency, 0, building, true) if err != nil { return OutputStream{}, compiler.CompiledQuery{}, fmt.Errorf("output %q dependency %q: %w", output.Name, input.WorkspaceOutputID, err) } @@ -758,6 +736,48 @@ func (e *Engine) clickHouseStreamForOutput(resolved Resolved, output lower.Compi return stream, query, nil } +func (e *Engine) postgreSQLStreamForOutput(resolved Resolved, output lower.CompiledRecipeOutput, limit int, bindings recipe.RuntimeBindings, query compiler.CompiledQuery, privateArtifactCapture bool) (OutputStream, compiler.CompiledQuery, error) { + if output.Plan.ClickHouseCombine != nil { + return OutputStream{}, compiler.CompiledQuery{}, fmt.Errorf("output %q: PostgreSQL-served output cannot carry a ClickHouse combine payload", output.Name) + } + if e.sourceRows == nil { + return OutputStream{}, compiler.CompiledQuery{}, fmt.Errorf("%w: output %q selected a PostgreSQL command but no source row port serves it; PostgreSQL requests never fall back to AQL", + command.ErrCompiledCommandBackendNotRegistered, output.Name) + } + if len(query.Columns) == 0 { + return OutputStream{}, compiler.CompiledQuery{}, fmt.Errorf("output %q: PostgreSQL command requires a non-empty declared result column contract", output.Name) + } + if len(output.OutputSchema) > 0 && len(query.PublicColumns) == 0 { + return OutputStream{}, compiler.CompiledQuery{}, fmt.Errorf("output %q: every declared PostgreSQL result column is an internal projection; refusing to publish internal columns in the public row", output.Name) + } + stream := OutputStream{ + Name: output.Name, Columns: append([]string(nil), query.PublicColumns...), + sourceIdentityMode: previewSourceIdentityMode(output, query.OutputSchema), RowIdentity: output.RowIdentity.Clone(), + DynamicChecks: dynamicChecks(output.DynamicColumns), + batchSize: e.batchSize, queryLimit: limit, + recipeDigest: resolved.StoredRecipeDigest, planFingerprint: query.PlanDiagnostics.Fingerprint, + bindings: bindings, stageID: compiledOutputFinalStageID(output), + outputSchema: lower.CloneCompiledOutputSchema(query.OutputSchema), + compiledOutput: true, + physicalPlan: query.PhysicalPlan, + sourceRows: e.sourceRows, + command: query.Command, + expectedBackend: command.CompiledBackendPostgreSQL, + commandColumns: append([]string(nil), query.Columns...), + } + if privateArtifactCapture { + scopeMode, scopeEvidence, scopeIdentity, err := compiledArtifactScope(output, bindings, query.PhysicalPlan) + if err != nil { + return OutputStream{}, compiler.CompiledQuery{}, fmt.Errorf("output %q artifact scope: %w", output.Name, err) + } + stream.artifactScopeMode, stream.scopeEvidence, stream.scopeEvidenceIdentity = scopeMode, scopeEvidence, scopeIdentity + if scopeMode == chartifact.ScopeModeWhole && scopeEvidence != nil { + stream.artifactScopeEvidence = scopeEvidence.Digest() + } + } + return stream, query, nil +} + func requiresRestrictedRowAuthPathCapture(output lower.CompiledRecipeOutput, bindings recipe.RuntimeBindings) bool { if outputUsesWholeRelationScope(output.Plan) { return false @@ -793,9 +813,6 @@ func compiledArtifactScope(output lower.CompiledRecipeOutput, bindings recipe.Ru if outputUsesWholeRelationScope(output.Plan) { mode = chartifact.ScopeModeWhole } - // A preview query is intentionally bounded and therefore cannot carry - // whole-relation capture evidence. The complete lowered plan remains - // compiler-attested; private capture recompiles it without this window. return mode, nil, nil, nil } optimizedEvidence, err := ir.NewWholeRelationScopeEvidence(actualPlan, identity) @@ -859,9 +876,6 @@ func sameExecutionStrings(left, right []string) bool { return true } -// streamExecutionAuthScopeMode uses compiler-issued evidence or an exact typed -// plan bind as the authority. An empty runtime mode by itself never grants -// unrestricted access; when paths are present it remains restricted. func streamExecutionAuthScopeMode(stream OutputStream) (authscope.ReadScopeMode, error) { if identity := stream.scopeEvidenceIdentity; identity != nil { if identity.Project != stream.bindings.Project || identity.DatasetGeneration != stream.bindings.DatasetGeneration || @@ -881,8 +895,6 @@ func streamExecutionAuthScopeMode(stream OutputStream) (authscope.ReadScopeMode, return stream.bindings.AuthScopeMode, nil } if len(stream.bindings.AuthResourcePaths) != 0 { - // Legacy callers that supply paths without a mode are restricted; the - // empty path-list case still requires a typed bind or opaque evidence. return authscope.ReadScopeRestricted, nil } if unrestricted, ok := stream.physicalPlan.BindVars["auth_resource_paths_unrestricted"].(bool); ok { @@ -1017,9 +1029,6 @@ func hasCompiledOutput(resolved Resolved, name string) bool { return false } -// PreviewOutput executes exactly one selected output and exposes only the -// public output schema. Physical query text, bind variables, and internal -// projections never cross this boundary. func (e *Engine) PreviewOutput(ctx context.Context, resolved Resolved, request PreviewRequest, visit func(map[string]any) error) (PreviewSummary, error) { if err := contextError(ctx); err != nil { return PreviewSummary{}, err @@ -1035,7 +1044,7 @@ func (e *Engine) PreviewOutput(ctx context.Context, resolved Resolved, request P return PreviewSummary{}, previewAdmissionError(dataframeerrors.CodeInvalidRequest, "preview visitor is required", nil) } loweringStarted := time.Now() - stream, query, err := e.streamForOutput(resolved, request.Output, limit) + stream, query, err := e.streamForOutput(ctx, resolved, request.Output, limit) if err != nil { if _, ok := dataframeerrors.AsUserError(err); ok { return PreviewSummary{}, err @@ -1048,7 +1057,8 @@ func (e *Engine) PreviewOutput(ctx context.Context, resolved Resolved, request P if err := validatePreviewPlan(query, limit, stream.physicalEngine); err != nil { return PreviewSummary{}, err } - if query.PreviewCoveringIndex != nil && !query.PreviewCoveringIndex.PrepareAfterPreview && e.preparePreviewIndex != nil { + postgreSQLPreview := stream.command.Backend() == command.CompiledBackendPostgreSQL + if !postgreSQLPreview && query.PreviewCoveringIndex != nil && !query.PreviewCoveringIndex.PrepareAfterPreview && e.preparePreviewIndex != nil { prepareCtx, cancel := context.WithTimeout(ctx, 4*time.Second) _ = e.preparePreviewIndex(prepareCtx, *query.PreviewCoveringIndex) cancel() @@ -1056,8 +1066,8 @@ func (e *Engine) PreviewOutput(ctx context.Context, resolved Resolved, request P return PreviewSummary{}, err } } - if stream.physicalEngine != ir.PhysicalEngineClickHouse && e.previewQueryRows != nil { - stream.stream = e.previewQueryRows + if stream.physicalEngine != ir.PhysicalEngineClickHouse { + stream.sourceRows = e.previewSourceRows } if err := e.chooseGroupPreviewScan(ctx, &query, &stream); err != nil { return PreviewSummary{}, err @@ -1067,13 +1077,13 @@ func (e *Engine) PreviewOutput(ctx context.Context, resolved Resolved, request P var previewCacheKeyWeight int var previewCacheEnabled bool var previewCacheHitRows []map[string]any - if query.PreviewCoveringIndex != nil && (query.PreviewCoveringIndex.PrepareAfterPreview || query.PreviewGroupScan != nil) && e.previewCollectionRevision != nil { + if !postgreSQLPreview && query.PreviewCoveringIndex != nil && (query.PreviewCoveringIndex.PrepareAfterPreview || query.PreviewGroupScan != nil) && e.previewCollectionRevision != nil { revision, revisionErr := e.previewCollectionRevision(ctx, query.PreviewCoveringIndex.Collection) if err := contextError(ctx); err != nil { return PreviewSummary{}, err } if revisionErr == nil { - key, keyWeight, keyOK := newGroupPreviewRowsKey(query.Query, query.BindVars, query.PreviewCoveringIndex.Collection, revision, limit) + key, keyWeight, keyOK := newGroupPreviewRowsKey(query.Command, query.PreviewCoveringIndex.Collection, revision, limit) if keyOK { previewCacheKey = key previewCacheKeyWeight = keyWeight @@ -1088,16 +1098,29 @@ func (e *Engine) PreviewOutput(ctx context.Context, resolved Resolved, request P } } } - summary := PreviewSummary{Output: stream.Name, Columns: append([]string(nil), stream.Columns...), PlanMode: query.PlanMode, PlanProfile: query.PlanProfile, PlanFingerprint: query.PlanDiagnostics.Fingerprint, TraversalCount: query.TraversalCount, LoweringDuration: loweringDuration, Complete: true, PartialValidation: query.PartialValidation} + planFingerprint := previewPlanFingerprint(query, stream) + if err := requireProjectedPlanFingerprint(stream.Name, planFingerprint); err != nil { + return PreviewSummary{}, err + } + summary := PreviewSummary{Output: stream.Name, Columns: append([]string(nil), stream.Columns...), PlanMode: query.PlanMode, PlanProfile: query.PlanProfile, PlanFingerprint: planFingerprint, TraversalCount: query.TraversalCount, LoweringDuration: loweringDuration, Complete: true, PartialValidation: query.PartialValidation} for _, output := range resolved.Compiled.Outputs { if output.Name == request.Output { summary.RowLineageCapability = compiler.RowLineageCapabilityForOutput(output) break } } + var previewIdentityBinds map[string]any + if query.Command.Backend() == command.CompiledBackendAQL { + _, binds, dispatchErr := query.Command.AQLParameters() + if dispatchErr != nil { + return PreviewSummary{}, dispatchErr + } + previewIdentityBinds = binds + } count := 0 var visitorErr error - queryStarted := time.Now() + var rowMaterialization time.Duration + queryStarted := now() consumeRow := func(row map[string]any) error { if err := contextError(ctx); err != nil { return err @@ -1105,13 +1128,15 @@ func (e *Engine) PreviewOutput(ctx context.Context, resolved Resolved, request P if count >= limit { return errPreviewLimit } + rowStarted := now() + defer func() { rowMaterialization += now().Sub(rowStarted) }() resolvedRow, err := materializePostQueryRowWithChecks(row, stream.DynamicChecks) if err != nil { return err } includeRowIdentity := request.IncludeRowIdentity || resolved.Semantic.SemanticPlan.Bindings.IncludeRowIdentity if includeRowIdentity { - if err := ensureStableRowIdentity(resolvedRow, stream.RowIdentity, query.BindVars); err != nil { + if err := ensureStableRowIdentity(resolvedRow, stream.RowIdentity, previewIdentityBinds); err != nil { return err } } @@ -1155,12 +1180,23 @@ func (e *Engine) PreviewOutput(ctx context.Context, resolved Resolved, request P }) } summary.RowCount = count - summary.QueryDuration = time.Since(queryStarted) + summary.SourceQueryDuration = now().Sub(queryStarted) - rowMaterialization + if summary.SourceQueryDuration < 0 { + summary.SourceQueryDuration = 0 + } if visitorErr != nil { + if postgreSQLPreview { + summary.Complete = false + summary.Truncated = false + } return summary, normalizePreviewError(visitorErr, false) } if queryErr != nil && !errors.Is(queryErr, errPreviewLimit) { e.rejectGroupProjectionDecisionAfterResourceLimit(stream, queryErr) + if postgreSQLPreview { + summary.Complete = false + summary.Truncated = false + } return summary, normalizePreviewError(queryErr, true) } if errors.Is(queryErr, errPreviewLimit) { @@ -1168,6 +1204,10 @@ func (e *Engine) PreviewOutput(ctx context.Context, resolved Resolved, request P summary.Truncated = true } if err := contextError(ctx); err != nil { + if postgreSQLPreview { + summary.Complete = false + summary.Truncated = false + } return summary, err } if err := e.confirmGroupProjectionDecision(ctx, stream); err != nil { @@ -1183,14 +1223,27 @@ func (e *Engine) PreviewOutput(ctx context.Context, resolved Resolved, request P e.groupPreviewRowsCache.put(previewCacheKey, capture.rows) } } - if query.PreviewCoveringIndex != nil && query.PreviewCoveringIndex.PrepareAfterPreview { + if !postgreSQLPreview && query.PreviewCoveringIndex != nil && query.PreviewCoveringIndex.PrepareAfterPreview { schedulePreviewIndexPrewarm(ctx, e, *query.PreviewCoveringIndex) } return summary, nil } +func isQueryResourceLimitError(err error) bool { + userErr, ok := dataframeerrors.AsUserError(err) + if !ok { + return false + } + switch dataframeerrors.ErrorCode(userErr.Code()) { + case dataframeerrors.CodeQueryResourceLimitExceeded, dataframeerrors.CodeQueryMemoryLimitExceeded: + return true + default: + return false + } +} + func (e *Engine) rejectGroupProjectionDecisionAfterResourceLimit(stream OutputStream, queryErr error) { - if queryErr == nil || stream.projectionDecisionKey == nil || stream.projectionDecision == nil || !arango.IsQueryResourceLimitExceeded(queryErr) { + if queryErr == nil || stream.projectionDecisionKey == nil || stream.projectionDecision == nil || !isQueryResourceLimitError(queryErr) { return } decision := *stream.projectionDecision @@ -1335,17 +1388,33 @@ func publicPreviewRow(row map[string]any, columns []string, includeRowIdentity b var errPreviewLimit = fmt.Errorf("preview limit reached") -func (s OutputStream) Stream(ctx context.Context, visit func(map[string]any) error) (StreamResult, error) { +func (s OutputStream) Stream(ctx context.Context, visit RowVisitor) (StreamResult, error) { if visit == nil { return StreamResult{}, fmt.Errorf("row visitor is required") } + var identityBinds map[string]any + switch s.command.Backend() { + case command.CompiledBackendAQL: + _, binds, dispatchErr := s.command.AQLParameters() + if dispatchErr != nil { + return StreamResult{}, dispatchErr + } + identityBinds = binds + case command.CompiledBackendPostgreSQL: + identityBinds = map[string]any{} + for _, bindKey := range []string{"project", "dataset_generation"} { + if value, ok := s.physicalPlan.BindVars[bindKey].(string); ok { + identityBinds[bindKey] = value + } + } + } count := 0 err := s.streamRaw(ctx, func(row map[string]any) error { resolved, err := materializePostQueryRowWithChecks(row, s.DynamicChecks) if err != nil { return err } - if err := ensureStableRowIdentity(resolved, s.RowIdentity, s.bindVars); err != nil { + if err := ensureStableRowIdentity(resolved, s.RowIdentity, identityBinds); err != nil { return err } resolved = publicStreamRow(resolved, s.Columns) @@ -1355,11 +1424,7 @@ func (s OutputStream) Stream(ctx context.Context, visit func(map[string]any) err return StreamResult{Output: s.Name, Columns: append([]string(nil), s.Columns...), RowCount: count}, err } -// streamPreparedClickHouse consumes an already resolved query while its input -// pins and private artifact leases remain owned by the preparation callback. -// It applies the same row identity and post-query projection contract as -// OutputStream.Stream without resolving or acquiring inputs a second time. -func (s OutputStream) streamPreparedClickHouse(ctx context.Context, prepared preparedClickHouseQuery, visit func(map[string]any) error) (StreamResult, error) { +func (s OutputStream) streamPreparedClickHouse(ctx context.Context, prepared preparedClickHouseQuery, visit RowVisitor) (StreamResult, error) { if ctx == nil || visit == nil || s.clickHouseQueryRows == nil || strings.TrimSpace(prepared.Query) == "" { return StreamResult{}, fmt.Errorf("prepared ClickHouse stream requires context, query executor, query, and visitor") } @@ -1372,7 +1437,7 @@ func (s OutputStream) streamPreparedClickHouse(ctx context.Context, prepared pre if err != nil { return err } - if err := ensureStableRowIdentity(resolved, s.RowIdentity, s.bindVars); err != nil { + if err := ensureStableRowIdentity(resolved, s.RowIdentity, nil); err != nil { return err } resolved = publicStreamRow(resolved, s.Columns) @@ -1382,17 +1447,41 @@ func (s OutputStream) streamPreparedClickHouse(ctx context.Context, prepared pre return StreamResult{Output: s.Name, Columns: append([]string(nil), s.Columns...), RowCount: count}, err } -func (s OutputStream) streamRaw(ctx context.Context, visit func(map[string]any) error) error { - if s.queryMemoryLimit > 0 { - ctx = arango.WithQueryMemoryLimit(ctx, s.queryMemoryLimit) - } +func (s OutputStream) streamRaw(ctx context.Context, visit RowVisitor) error { if s.physicalEngine == ir.PhysicalEngineClickHouse { return s.withPreparedClickHouse(ctx, func(preparedCtx context.Context, prepared preparedClickHouseQuery) error { return s.clickHouseQueryRows(preparedCtx, prepared.Query, prepared.Columns, visit, prepared.Args...) }) } - if s.page == nil || s.rootPageRows == 0 { - return s.stream(ctx, s.query, s.batchSize, s.bindVars, visit) + if s.sourceRows == nil { + return fmt.Errorf("%w: stream %q has no source row port", + command.ErrCompiledCommandBackendNotRegistered, s.Name) + } + streamRequest := func(cmd command.CompiledCommand) RowsRequest { + return RowsRequest{Command: cmd, DeclaredColumns: append([]string(nil), s.commandColumns...), BatchSize: s.batchSize} + } + if s.command.Backend() == "" { + return fmt.Errorf("%w: stream %q carries an unregistered backend tag", + command.ErrCompiledCommandBackendNotRegistered, s.Name) + } + if s.expectedBackend != "" && s.command.Backend() != s.expectedBackend { + return fmt.Errorf("%w: stream %q command is tagged %q while the selected target serves %q", + command.ErrCompiledCommandBackendMismatch, s.Name, string(s.command.Backend()), string(s.expectedBackend)) + } + if s.command.Backend() == command.CompiledBackendPostgreSQL && len(s.commandColumns) == 0 { + return fmt.Errorf("stream %q: PostgreSQL command requires a non-empty declared result column contract", s.Name) + } + if s.command.Backend() == command.CompiledBackendPostgreSQL && s.pgRootPage != nil { + return s.streamPostgreSQLRootPages(ctx, visit) + } + if s.command.Backend() != command.CompiledBackendAQL || s.page == nil || s.rootPageRows == 0 { + if s.command.Backend() == command.CompiledBackendAQL && s.queryMemoryLimit > 0 { + ctx = WithQueryMemoryBudget(ctx, s.queryMemoryLimit) + } + return s.sourceRows(ctx, streamRequest(s.command), visit) + } + if s.queryMemoryLimit > 0 { + ctx = WithQueryMemoryBudget(ctx, s.queryMemoryLimit) } after := "" pageRows := s.rootPageRows @@ -1401,10 +1490,14 @@ func (s OutputStream) streamRaw(ctx context.Context, visit func(map[string]any) } for { keys := make([]string, 0, pageRows) - keyBinds := cloneBindVars(s.page.RootKeysBindVars) - keyBinds[compiler.RootPageAfterKeyBind] = after - keyBinds[compiler.RootPageSizeBind] = pageRows - if err := s.stream(ctx, s.page.RootKeysQuery, s.batchSize, keyBinds, func(row map[string]any) error { + keyCommand, keyBindErr := s.page.RootKeys.WithAQLBinds(map[string]any{ + compiler.RootPageAfterKeyBind: after, + compiler.RootPageSizeBind: pageRows, + }) + if keyBindErr != nil { + return keyBindErr + } + if err := s.sourceRows(ctx, streamRequest(keyCommand), func(row map[string]any) error { key, ok := row["_key"].(string) if !ok || key == "" { return fmt.Errorf("root-key page returned an invalid _key") @@ -1420,10 +1513,14 @@ func (s OutputStream) streamRaw(ctx context.Context, visit func(map[string]any) if len(keys) == 0 { return nil } - rowBinds := cloneBindVars(s.page.RowsBindVars) - rowBinds[compiler.RootPageKeysBind] = keys + rowCommand, rowBindErr := s.page.Rows.WithAQLBinds(map[string]any{ + compiler.RootPageKeysBind: keys, + }) + if rowBindErr != nil { + return rowBindErr + } emitted := 0 - if err := s.stream(ctx, s.page.RowsQuery, s.batchSize, rowBinds, func(row map[string]any) error { + if err := s.sourceRows(ctx, streamRequest(rowCommand), func(row map[string]any) error { emitted++ return visit(row) }); err != nil { @@ -1443,6 +1540,161 @@ func (s OutputStream) streamRaw(ctx context.Context, visit func(map[string]any) } } +func (s OutputStream) streamPostgreSQLRootPages(ctx context.Context, visit RowVisitor) error { + steps := s.pgRootPage + if err := s.validatePostgreSQLRootPageSteps(); err != nil { + return err + } + after := "" + for { + if err := ctx.Err(); err != nil { + return err + } + keyCommand, err := withPostgreSQLPageParameters(steps.RootKeys, []postgreSQLPageParameter{ + {index: steps.CursorArgIndex, parameter: command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: after}}, + {index: steps.PageSizeArgIndex, parameter: command.PostgreSQLParameter{Type: command.PostgreSQLParameterBigint, Value: int64(steps.PageSize)}}, + }) + if err != nil { + return err + } + keys := make([]string, 0, steps.PageSize) + if err := s.sourceRows(ctx, RowsRequest{Command: keyCommand, DeclaredColumns: []string{steps.RootKeyColumn}, BatchSize: s.batchSize}, func(row map[string]any) error { + key, ok := row[steps.RootKeyColumn].(string) + if !ok || key == "" { + return fmt.Errorf("%w: root-key page returned an invalid continuation identity in column %q", ErrPostgreSQLResultContract, steps.RootKeyColumn) + } + if after != "" && key <= after { + return fmt.Errorf("%w: root-key page does not advance past the previous page tail %q", ErrPostgreSQLResultContract, after) + } + if len(keys) != 0 && key <= keys[len(keys)-1] { + return fmt.Errorf("%w: root-key page is not strictly ordered", ErrPostgreSQLResultContract) + } + keys = append(keys, key) + return nil + }); err != nil { + return err + } + if len(keys) == 0 { + return nil + } + if len(keys) > steps.PageSize { + return fmt.Errorf("%w: root-key page returned %d keys for a requested page of %d", ErrPostgreSQLResultContract, len(keys), steps.PageSize) + } + if err := ctx.Err(); err != nil { + return err + } + keysJSON, err := json.Marshal(keys) + if err != nil { + return fmt.Errorf("%w: cannot encode the requested key page: %v", ErrPostgreSQLResultContract, err) + } + rowCommand, err := withPostgreSQLPageParameters(steps.Rows, []postgreSQLPageParameter{ + {index: steps.PageKeysArgIndex, parameter: command.PostgreSQLParameter{Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(keysJSON)}}, + }) + if err != nil { + return err + } + requested := make(map[string]struct{}, len(keys)) + for _, key := range keys { + requested[key] = struct{}{} + } + if err := s.sourceRows(ctx, RowsRequest{Command: rowCommand, DeclaredColumns: append([]string(nil), s.commandColumns...), BatchSize: s.batchSize}, func(row map[string]any) error { + root, ok := row[steps.RowRootKeyColumn].(string) + if !ok || root == "" { + return fmt.Errorf("%w: result row is missing its root continuation identity in column %q", ErrPostgreSQLResultContract, steps.RowRootKeyColumn) + } + if _, belongs := requested[root]; !belongs { + return fmt.Errorf("%w: result row carries a root continuation identity outside the requested key page", ErrPostgreSQLResultContract) + } + return visit(row) + }); err != nil { + return err + } + after = keys[len(keys)-1] + if len(keys) < steps.PageSize { + return nil + } + } +} + +func (s OutputStream) validatePostgreSQLRootPageSteps() error { + steps := s.pgRootPage + if steps == nil { + return fmt.Errorf("%w: no steps were supplied", ErrPostgreSQLRootPageSteps) + } + if steps.PageSize < 1 { + return fmt.Errorf("%w: page size must be positive, got %d", ErrPostgreSQLRootPageSteps, steps.PageSize) + } + if steps.PageSize > maxPostgreSQLRootPageSize { + return fmt.Errorf("%w: page size %d exceeds the sanity bound %d", ErrPostgreSQLRootPageSteps, steps.PageSize, maxPostgreSQLRootPageSize) + } + if strings.TrimSpace(steps.RootKeyColumn) == "" { + return fmt.Errorf("%w: the root-key result column must be named", ErrPostgreSQLRootPageSteps) + } + if strings.TrimSpace(steps.RowRootKeyColumn) == "" { + return fmt.Errorf("%w: the row root key column must be named", ErrPostgreSQLRootPageSteps) + } + declared := false + for _, column := range s.commandColumns { + if column == steps.RowRootKeyColumn { + declared = true + break + } + } + if !declared { + return fmt.Errorf("%w: row root key column %q is not part of the declared output contract", ErrPostgreSQLRootPageSteps, steps.RowRootKeyColumn) + } + if steps.CursorArgIndex == steps.PageSizeArgIndex { + return fmt.Errorf("%w: the cursor and page size must occupy distinct key-step parameter positions", ErrPostgreSQLRootPageSteps) + } + if err := checkPostgreSQLPageParameter(steps.RootKeys, steps.CursorArgIndex, command.PostgreSQLParameterText, "key-step cursor"); err != nil { + return err + } + if err := checkPostgreSQLPageParameter(steps.RootKeys, steps.PageSizeArgIndex, command.PostgreSQLParameterBigint, "key-step page size"); err != nil { + return err + } + if err := checkPostgreSQLPageParameter(steps.Rows, steps.PageKeysArgIndex, command.PostgreSQLParameterJSONB, "row-step key page"); err != nil { + return err + } + return nil +} + +type postgreSQLPageParameter struct { + index int + parameter command.PostgreSQLParameter +} + +func checkPostgreSQLPageParameter(base command.CompiledCommand, index int, want command.PostgreSQLParameterType, name string) error { + _, args, err := base.PostgreSQLParameters() + if err != nil { + return err + } + if index < 0 || index >= len(args) { + return fmt.Errorf("%w: %s parameter index %d is outside the %d rendered statement arguments", ErrPostgreSQLRootPageSteps, name, index, len(args)) + } + if args[index].Type != want { + return fmt.Errorf("%w: %s parameter %d is declared %q and cannot be rebound as %q", ErrPostgreSQLRootPageSteps, name, index+1, string(args[index].Type), string(want)) + } + return nil +} + +func withPostgreSQLPageParameters(base command.CompiledCommand, replacements []postgreSQLPageParameter) (command.CompiledCommand, error) { + sqlText, args, err := base.PostgreSQLParameters() + if err != nil { + return command.CompiledCommand{}, err + } + updated := append([]command.PostgreSQLParameter(nil), args...) + for _, replacement := range replacements { + if replacement.index < 0 || replacement.index >= len(args) { + return command.CompiledCommand{}, fmt.Errorf("%w: page parameter index %d is outside the %d rendered statement arguments", ErrPostgreSQLRootPageSteps, replacement.index, len(args)) + } + if args[replacement.index].Type != replacement.parameter.Type { + return command.CompiledCommand{}, fmt.Errorf("%w: page parameter %d is declared %q and cannot be rebound as %q", ErrPostgreSQLRootPageSteps, replacement.index+1, string(args[replacement.index].Type), string(replacement.parameter.Type)) + } + updated[replacement.index] = replacement.parameter + } + return command.NewPostgreSQLCommand(sqlText, updated) +} + func (s OutputStream) withPreparedClickHouse(ctx context.Context, consume func(context.Context, preparedClickHouseQuery) error) error { if ctx == nil || consume == nil || s.clickHouseCombine == nil || s.clickHouseQueryRows == nil { return fmt.Errorf("ClickHouse preparation requires context, typed plan, query executor, and consumer") @@ -1728,19 +1980,6 @@ func compiledOutputSchemaDigest(schema []lower.CompiledOutputColumn) (string, er return hex.EncodeToString(digest[:]), nil } -func cloneBindVars(input map[string]any) map[string]any { - clone := make(map[string]any, len(input)) - for key, value := range input { - clone[key] = value - } - return clone -} - -// publicStreamRow removes compiler-only projections after they have served -// post-query validation and row-identity derivation. auth_resource_path is the -// one hidden projection retained for publication because it preserves the -// source row's authorization scope when a materialization spans multiple -// authorized paths. func publicStreamRow(row map[string]any, columns []string) map[string]any { public := make(map[string]any, len(columns)+2) for _, column := range columns { diff --git a/internal/dataframe/execution/engine_blank_project_review_test.go b/internal/dataframe/execution/engine_blank_project_review_test.go index fd841157f..e7e008069 100644 --- a/internal/dataframe/execution/engine_blank_project_review_test.go +++ b/internal/dataframe/execution/engine_blank_project_review_test.go @@ -2,6 +2,7 @@ package execution import ( "context" + "github.com/calypr/loom/internal/dataframe/compiler" "testing" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" @@ -22,13 +23,18 @@ func (r *reviewBlankRegistry) LoadRecipeVersion(context.Context, string, string) return recipeexec.Entry{}, nil } -// Review attack 1: direct engine entry with a caller-constructed blank -// bindings, bypassing any GraphQL authorizer. func TestEngineResolveBlankProjectDirectClassification(t *testing.T) { registry := &reviewBlankRegistry{} engine, err := New(Config{ - Registry: registry, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: registry, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil }, }) @@ -53,9 +59,6 @@ func TestEngineResolveBlankProjectDirectClassification(t *testing.T) { } } -// coverAllFiveGuards exercises every blank-project guard, including the two -// unexported ones (resolveEntry, compileResolvedBundle) in-package, plus the -// ResolveVersion lane that reaches resolveEntry without a preceding guard. func TestEngineAllFiveBlankProjectGuards(t *testing.T) { bundle := recipe.Bundle{RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: "x", TranslationVersion: "t"} blank := recipe.RuntimeBindings{Project: " \t\n "} @@ -68,8 +71,15 @@ func TestEngineAllFiveBlankProjectGuards(t *testing.T) { } registry := &reviewBlankRegistry{} engine, err := New(Config{ - Registry: registry, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: registry, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil }, }) @@ -89,20 +99,22 @@ func TestEngineAllFiveBlankProjectGuards(t *testing.T) { } else { requireTyped("compileResolvedBundle(unexported)", err) } - // ResolveVersion has no own guard: blank must be typed by resolveEntry. _, err = engine.ResolveVersion(context.Background(), "x", "v1", blank) requireTyped("ResolveVersion->resolveEntry lane", err) if registry.loads == 0 { t.Fatal("ResolveVersion lane must consult registry before the resolveEntry guard") } - // Attack-2 divergence artifact: whitespace guard and Canonical emptiness - // predicates differ. "/" is non-blank per the guards yet Canonical("") — - // it reaches the registry instead of being classified. If this load - // attempt ever becomes a rejection, the two predicates converged. divergeRegistry := &reviewBlankRegistry{} divergeEngine, err := New(Config{ - Registry: divergeRegistry, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: divergeRegistry, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil }, }) diff --git a/internal/dataframe/execution/engine_postgres_capture_pg18_test.go b/internal/dataframe/execution/engine_postgres_capture_pg18_test.go new file mode 100644 index 000000000..a77642862 --- /dev/null +++ b/internal/dataframe/execution/engine_postgres_capture_pg18_test.go @@ -0,0 +1,1078 @@ +package execution_test + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "reflect" + "regexp" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/authscope" + postgres "github.com/calypr/loom/internal/backends/postgres" + storepg "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + "github.com/calypr/loom/internal/dataframe/execution/chartifact" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/recipe/exec" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/calypr/loom/internal/source" + "github.com/calypr/loom/internal/store/clickhouse" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + exec11lvPrefix = "verify_exec11_" + exec11lvNameFmt = exec11lvPrefix + "%d_%s" +) + +var exec11lvNamePattern = regexp.MustCompile(`^verify_exec11_[0-9]+_[0-9a-f]+$`) + +var exec11lvIdentifierRE = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +func clickhouseIdentifier(name string) bool { return exec11lvIdentifierRE.MatchString(name) } + +var ( + exec11lvOwnedMu sync.Mutex + exec11lvOwnedByName = map[string]string{} +) + +func exec11lvRegister(db, schema string) { + exec11lvOwnedMu.Lock() + defer exec11lvOwnedMu.Unlock() + exec11lvOwnedByName[db] = schema +} + +func exec11lvRegistrySnapshot() map[string]string { + exec11lvOwnedMu.Lock() + defer exec11lvOwnedMu.Unlock() + out := make(map[string]string, len(exec11lvOwnedByName)) + for db, schema := range exec11lvOwnedByName { + out[db] = schema + } + return out +} + +var ( + exec11lvBaselineOnce sync.Once + exec11lvBaselineNames []string + exec11lvBaselineErr error +) + +func exec11lvCaptureBaseline(raw string) { + exec11lvBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec11lvBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec11\\_%' ESCAPE '\\'") + if err != nil { + exec11lvBaselineErr = fmt.Errorf("read baseline: %w", err) + return + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + exec11lvBaselineErr = fmt.Errorf("scan baseline: %w", err) + return + } + exec11lvBaselineNames = append(exec11lvBaselineNames, name) + } + }) +} + +type exec11lvService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + schema string + dsn string + scrub func(any) string + fail func(step string, err error) + executor dataframeexecution.SourceRows +} + +func newExec11lvService(t *testing.T) *exec11lvService { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 private-capture acceptance tests") + } + scrub, addSecret := exec03pqScrubberFor(raw) + exec11lvCaptureBaseline(raw) + if exec11lvBaselineErr != nil { + t.Fatalf("capture pg_database baseline: %s", scrub(exec11lvBaselineErr)) + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, "SELECT current_setting('server_version_num')::int8, current_setting('server_version')").Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if err := storepg.CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required PostgreSQL 18 backend: %v", err) + } + + dbName := fmt.Sprintf(exec11lvNameFmt, time.Now().UnixNano(), exec03pqRandomHex(t)) + if !exec11lvNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + var created int64 + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", dbName).Scan(&created); err != nil { + fail("verify isolated database creation", err) + } + if created != 1 { + t.Fatalf("isolated database %q did not verifiably come into existence", dbName) + } + exec11lvRegister(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + u, err := parseExec11lvURL(raw) + if err != nil { + t.Fatal(scrub(err)) + } + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + addSecret(dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(exec11lvNameFmt, time.Now().UnixNano(), exec03pqRandomHex(t)) + if !exec11lvNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + exec11lvRegister(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + executor, err := postgres.NewSourceRows(iso) + if err != nil { + fail("create PostgreSQL row executor", err) + } + return &exec11lvService{ctx: ctx, admin: admin, iso: iso, schema: schema, dsn: dsn, scrub: scrub, fail: fail, executor: executor} +} + +func (s *exec11lvService) compileTarget(t *testing.T) compiler.Target { + t.Helper() + binding, err := ir.NewPostgresStorageBinding(s.schema) + if err != nil { + t.Fatalf("bind PostgreSQL storage placement: %v", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("select PostgreSQL compile target: %v", err) + } + return target +} + +func (s *exec11lvService) migrateAndSeed(t *testing.T) { + t.Helper() + client, err := storepg.OpenPool(s.ctx, storepg.ClientConfig{DSN: s.dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 2 * time.Second}) + if err != nil { + s.fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + defer client.Close() + if _, err := storepg.MigrateSchema(s.ctx, client, s.schema); err != nil { + s.fail("migrate owned schema", err) + } + repo, err := storepg.NewResourceRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + type seedRow struct{ project, generation, kind, logicalID, authPath, payload string } + fixture := []seedRow{ + {"project-a", "generation-2", "Patient", "exec11-ada", "/programs/a", `{"id":"patient-ada"}`}, + {"project-a", "generation-2", "Patient", "exec11-grace", "/programs/a", `{"id":"patient-grace"}`}, + {"project-a", "generation-2", "Patient", "exec11-alan", "/programs/a", `{"id":"patient-alan"}`}, + {"project-a", "generation-2", "Patient", "exec11-hidden", "/programs/b", `{"id":"patient-hidden"}`}, + {"project-b", "generation-2", "Patient", "exec11-intruder", "/programs/a", `{"id":"patient-intruder"}`}, + {"project-a", "generation-other", "Patient", "exec11-stale", "/programs/a", `{"id":"patient-stale"}`}, + } + for _, row := range fixture { + if _, err := repo.InsertResource(s.ctx, source.Resource{ + Project: row.project, DatasetGeneration: row.generation, ResourceType: row.kind, + LogicalID: row.logicalID, AuthResourcePath: row.authPath, Payload: []byte(row.payload), + }); err != nil { + s.fail("insert fixture resource "+row.logicalID, err) + } + } +} + +func (s *exec11lvService) witnessIDs(t *testing.T) []string { + t.Helper() + rows, err := s.iso.Query(s.ctx, fmt.Sprintf( + `SELECT payload->>'id' FROM "%s".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' AND auth_resource_path=$3 ORDER BY 1`, + s.schema), "project-a", "generation-2", "/programs/a") + if err != nil { + s.fail("direct-SQL witness", err) + } + defer rows.Close() + var got []string + for rows.Next() { + var value string + if err := rows.Scan(&value); err != nil { + s.fail("scan witness row", err) + } + got = append(got, value) + } + if err := rows.Err(); err != nil { + s.fail("witness rows", err) + } + return got +} + +func parseExec11lvURL(raw string) (*url.URL, error) { + u, err := url.Parse(raw) + if err != nil { + return nil, err + } + if u.Scheme != "postgres" && u.Scheme != "postgresql" { + return nil, errors.New("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + return u, nil +} + +type exec11lvInvalidRegistry struct{} + +func (exec11lvInvalidRegistry) LoadRecipe(context.Context, string) (exec.Entry, error) { + return exec.Entry{}, errors.New("the exec-11 capture battery compiles authored bundles only") +} + +func (exec11lvInvalidRegistry) LoadRecipeVersion(context.Context, string, string) (exec.Entry, error) { + return exec.Entry{}, errors.New("the exec-11 capture battery compiles authored bundles only") +} + +type exec11lvCatalog struct { + mu sync.Mutex + items map[string]chartifact.Manifest +} + +func (c *exec11lvCatalog) Create(_ context.Context, value chartifact.Manifest) error { + c.mu.Lock() + defer c.mu.Unlock() + c.items[value.ArtifactID] = value + return nil +} + +func (c *exec11lvCatalog) Update(_ context.Context, id, owner string, progress chartifact.Progress) error { + c.mu.Lock() + defer c.mu.Unlock() + value, ok := c.items[id] + if !ok || value.LeaseOwner != owner { + return chartifact.ErrLeaseLost + } + value.State, value.RowCount, value.ByteCount, value.UpdatedAt = progress.State, progress.RowCount, progress.ByteCount, progress.UpdatedAt + if progress.LeaseUntil.After(value.LeaseUntil) || progress.State == chartifact.StateCleanupPending { + value.LeaseUntil = progress.LeaseUntil + } + c.items[id] = value + return nil +} + +func (c *exec11lvCatalog) Renew(_ context.Context, id, owner string, until time.Time) (bool, error) { + c.mu.Lock() + defer c.mu.Unlock() + value, ok := c.items[id] + if !ok || value.LeaseOwner != owner { + return false, nil + } + if until.After(value.LeaseUntil) { + value.LeaseUntil = until + c.items[id] = value + } + return true, nil +} + +func (c *exec11lvCatalog) ListExpired(_ context.Context, before time.Time, limit int) ([]chartifact.Manifest, error) { + c.mu.Lock() + defer c.mu.Unlock() + result := make([]chartifact.Manifest, 0) + for _, item := range c.items { + if !item.LeaseUntil.After(before) { + result = append(result, item) + } + } + if len(result) > limit { + result = result[:limit] + } + return result, nil +} + +func (c *exec11lvCatalog) ClaimCleanup(_ context.Context, id, owner string, expiredBefore, until time.Time) (chartifact.Manifest, bool, error) { + c.mu.Lock() + defer c.mu.Unlock() + value, ok := c.items[id] + if !ok || value.LeaseUntil.After(expiredBefore) { + return chartifact.Manifest{}, false, nil + } + value.LeaseOwner, value.LeaseUntil, value.State = owner, until, chartifact.StateCleanupPending + c.items[id] = value + return value, true, nil +} + +func (c *exec11lvCatalog) Delete(_ context.Context, id, owner string) error { + c.mu.Lock() + defer c.mu.Unlock() + value, ok := c.items[id] + if !ok || value.LeaseOwner != owner { + return chartifact.ErrLeaseLost + } + delete(c.items, id) + return nil +} + +func (c *exec11lvCatalog) ReleaseCleanup(_ context.Context, id, owner string, until time.Time) error { + c.mu.Lock() + defer c.mu.Unlock() + value, ok := c.items[id] + if !ok || value.LeaseOwner != owner { + return chartifact.ErrLeaseLost + } + value.LeaseOwner, value.LeaseUntil = "", until + c.items[id] = value + return nil +} + +func (c *exec11lvCatalog) snapshot() []chartifact.Manifest { + c.mu.Lock() + defer c.mu.Unlock() + out := make([]chartifact.Manifest, 0, len(c.items)) + for _, manifest := range c.items { + out = append(out, manifest) + } + return out +} + +func (c *exec11lvCatalog) count() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.items) +} + +type exec11lvClickHouse struct { + mu sync.Mutex + table map[string][]clickhouse.Column + rows map[string][]map[string]any +} + +func (c *exec11lvClickHouse) CreateTable(_ context.Context, table string, columns []clickhouse.Column) error { + c.mu.Lock() + defer c.mu.Unlock() + c.table[table] = append([]clickhouse.Column(nil), columns...) + c.rows[table] = nil + return nil +} + +func (c *exec11lvClickHouse) InsertRows(_ context.Context, table string, columns []clickhouse.Column, rows []map[string]any) error { + c.mu.Lock() + defer c.mu.Unlock() + if !reflect.DeepEqual(c.table[table], columns) { + return fmt.Errorf("unexpected insert schema for %s", table) + } + for _, row := range rows { + copied := make(map[string]any, len(row)) + for key, value := range row { + copied[key] = value + } + c.rows[table] = append(c.rows[table], copied) + } + return nil +} + +func (c *exec11lvClickHouse) VerifyOutput(_ context.Context, table string, columns []clickhouse.Column, expected int64) error { + c.mu.Lock() + defer c.mu.Unlock() + if !reflect.DeepEqual(c.table[table], columns) || int64(len(c.rows[table])) != expected { + return fmt.Errorf("artifact verification mismatch for %s", table) + } + return nil +} + +func (c *exec11lvClickHouse) DropTable(_ context.Context, table string) error { + c.mu.Lock() + defer c.mu.Unlock() + delete(c.table, table) + delete(c.rows, table) + return nil +} + +func (c *exec11lvClickHouse) rowsSnapshot(table string) []map[string]any { + c.mu.Lock() + defer c.mu.Unlock() + rows := c.rows[table] + out := make([]map[string]any, 0, len(rows)) + for _, row := range rows { + copied := make(map[string]any, len(row)) + for key, value := range row { + copied[key] = value + } + out = append(out, copied) + } + return out +} + +func (c *exec11lvClickHouse) allRows() map[string][]map[string]any { + c.mu.Lock() + defer c.mu.Unlock() + out := make(map[string][]map[string]any, len(c.rows)) + for table, rows := range c.rows { + if len(rows) == 0 { + continue + } + copied := make([]map[string]any, 0, len(rows)) + for _, row := range rows { + item := make(map[string]any, len(row)) + for key, value := range row { + item[key] = value + } + copied = append(copied, item) + } + out[table] = copied + } + return out +} + +type exec11lvPorts struct { + combineQueries []string + combineArgs [][]any + seenDuringRun []chartifact.Manifest + capturedRows map[string][]map[string]any + visibleDuringQ int + pinRevisions [][]string + resolveCalls int +} + +func newExec11lvPorts() *exec11lvPorts { + return &exec11lvPorts{capturedRows: map[string][]map[string]any{}} +} + +func (p *exec11lvPorts) queryRows(catalog *exec11lvCatalog, ch *exec11lvClickHouse, visitRows []map[string]any) dataframeexecution.ClickHouseQueryRows { + return func(_ context.Context, query string, _ []string, visit func(map[string]any) error, args ...any) error { + p.combineQueries = append(p.combineQueries, query) + p.combineArgs = append(p.combineArgs, append([]any(nil), args...)) + manifests := catalog.snapshot() + p.visibleDuringQ = len(manifests) + for _, manifest := range manifests { + p.seenDuringRun = append(p.seenDuringRun, manifest) + if rows := ch.rowsSnapshot(manifest.PhysicalTable); rows != nil { + p.capturedRows[manifest.PhysicalTable] = rows + } + } + for _, row := range visitRows { + if err := visit(row); err != nil { + return err + } + } + return nil + } +} + +func exec11lvBindings() recipe.RuntimeBindings { + return recipe.RuntimeBindings{ + Project: "project-a", DatasetGeneration: "generation-2", AuthScopeMode: authscope.ReadScopeRestricted, + AuthResourcePaths: []string{"/programs/a"}, + } +} + +func exec11lvBundle() recipe.Bundle { + output := func(name string) recipe.Output { + return recipe.Output{ + Name: name, RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{{Name: "value", ColumnID: "value-id", Expr: recipe.Expression{Select: "root.id"}}}, + } + } + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: "exec11-live-combine", TranslationVersion: "test", + Outputs: []recipe.Output{output("left"), output("right")}, + } +} + +func exec11lvCombineOutput(name string, inputs []ir.PhysicalCombineInputRef) lower.CompiledRecipeOutput { + plan := ir.PhysicalClickHouseCombine{ + Kind: ir.PhysicalCombineAppend, Inputs: append([]ir.PhysicalCombineInputRef(nil), inputs...), + Outputs: []ir.PhysicalCombineOutputColumn{{ID: "value-id", Name: "value", LogicalType: "string", ClickHouseType: "Nullable(String)", Nullable: true}}, + } + for index := range inputs { + plan.Projections = append(plan.Projections, ir.PhysicalCombineProjection{OutputColumnID: "value-id", InputIndex: index, InputColumnID: "value-id"}) + } + return lower.CompiledRecipeOutput{ + Name: name, Columns: []string{"value"}, + Stages: []lower.CompiledStageDescriptor{{ID: "combine-final"}}, + Plan: ir.PhysicalPlan{Version: 1, Engine: ir.PhysicalEngineClickHouse, ClickHouseCombine: &plan}, + OutputSchema: []lower.CompiledOutputColumn{ + {ID: "loom:row_id", Name: "__loom_row_id", Kind: "string", Cardinality: "required_one", Internal: true, Identity: true}, + {ID: "value-id", Name: "value", Kind: "string", Cardinality: "optional_one", Nullable: true}, + }, + RowIdentity: &spec.RowIdentity{Fields: []string{"__loom_row_id"}}, + } +} + +func exec11lvResolved(t *testing.T, engine *dataframeexecution.Engine, withPublished bool) dataframeexecution.Resolved { + t.Helper() + resolved, err := engine.CompileResolvedBundle(context.Background(), exec11lvBundle(), exec11lvBindings()) + if err != nil { + t.Fatalf("compile live bundle: %v", err) + } + if len(resolved.Compiled.Outputs) != 2 || resolved.Compiled.Outputs[0].Name != "left" || resolved.Compiled.Outputs[1].Name != "right" { + t.Fatalf("live bundle outputs drifted: %s/%s", resolved.Compiled.Outputs[0].Name, resolved.Compiled.Outputs[1].Name) + } + dependencies := append([]lower.CompiledRecipeOutput(nil), resolved.Compiled.Outputs...) + refs := []ir.PhysicalCombineInputRef{{WorkspaceOutputID: "left"}, {WorkspaceOutputID: "right"}} + if withPublished { + refs = append(refs, ir.PhysicalCombineInputRef{TableID: "exec11_pub_table", RevisionID: "exec11_pub_rev", OutputID: "pub"}) + } + resolved.Compiled.Outputs = []lower.CompiledRecipeOutput{exec11lvCombineOutput("joined", refs)} + resolved.Compiled.WorkspaceDependencies = dependencies + resolved.Semantic.SemanticPlan.Bindings.OutputNames = []string{"joined"} + return resolved +} + +func exec11lvJoin(t *testing.T, engine *dataframeexecution.Engine, resolved dataframeexecution.Resolved) dataframeexecution.OutputStream { + t.Helper() + streams, err := engine.Streams(context.Background(), resolved) + if err != nil { + t.Fatalf("build live workspace Combine streams: %v", err) + } + for _, stream := range streams { + if stream.Name == "joined" { + return stream + } + } + names := make([]string, 0, len(streams)) + for _, stream := range streams { + names = append(names, stream.Name) + } + t.Fatalf("joined stream missing from %v", names) + return dataframeexecution.OutputStream{} +} + +func TestExec11lvLiveRestrictedCaptureFeedsCombine(t *testing.T) { + service := newExec11lvService(t) + service.migrateAndSeed(t) + witness := service.witnessIDs(t) + if strings.Join(witness, ",") != "patient-ada,patient-alan,patient-grace" { + t.Fatalf("witness drifted: %v", witness) + } + + catalog := &exec11lvCatalog{items: map[string]chartifact.Manifest{}} + ch := &exec11lvClickHouse{table: map[string][]clickhouse.Column{}, rows: map[string][]map[string]any{}} + manager, err := chartifact.New(chartifact.Config{Catalog: catalog, ClickHouse: ch, BatchRows: 1}) + if err != nil { + t.Fatal(err) + } + ports := newExec11lvPorts() + bindings := exec11lvBindings() + engine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: service.compileTarget(t), + Registry: exec11lvInvalidRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "exec11-live-scope" }, + SourceRows: service.executor, + PrivateClickHouseArtifacts: manager, + WithExecutionReadPins: func(_ context.Context, revisions []string, visit func(context.Context) error) error { + ports.pinRevisions = append(ports.pinRevisions, append([]string(nil), revisions...)) + return visit(context.Background()) + }, + ResolveClickHouseInputs: func(_ context.Context, plan ir.PhysicalClickHouseCombine, got recipe.RuntimeBindings) ([]ir.ResolvedClickHouseTable, error) { + ports.resolveCalls++ + if got.Project != bindings.Project || got.DatasetGeneration != bindings.DatasetGeneration { + return nil, fmt.Errorf("published resolution lost the exact project/generation") + } + for _, input := range plan.Inputs { + if input.RevisionID == "" { + return nil, fmt.Errorf("published resolver received a non-published reference") + } + } + return []ir.ResolvedClickHouseTable{{ + TableID: "exec11_pub_table", RevisionID: "exec11_pub_rev", OutputID: "pub", + Project: bindings.Project, DatasetGeneration: bindings.DatasetGeneration, + ReceiptID: "receipt-exec11-pub", SchemaDigest: "schema-exec11-pub", ScopeDigest: "scope-exec11-pub", + PhysicalTable: "exec11_published_physical", AuthResourcePaths: []string{"/programs/a"}, ScopeMode: chartifact.ScopeModeRows, + Columns: []ir.ResolvedClickHouseColumn{ + {ID: "loom:row_id", Name: "__loom_row_id", LogicalType: "string", ClickHouseType: "String"}, + {ID: "value-id", Name: "value", LogicalType: "string", ClickHouseType: "Nullable(String)"}, + {ID: "auth-path", Name: "auth_resource_path", LogicalType: "string", ClickHouseType: "String"}, + }, + }}, nil + }, + ClickHouseQueryRows: ports.queryRows(catalog, ch, []map[string]any{ + {"__loom_row_id": "joined-1", "value": "patient-ada", "auth_resource_path": "/programs/a"}, + {"__loom_row_id": "joined-2", "value": "patient-alan", "auth_resource_path": "/programs/a"}, + }), + }) + if err != nil { + t.Fatal(err) + } + resolved := exec11lvResolved(t, engine, true) + stream := exec11lvJoin(t, engine, resolved) + var rows []map[string]any + result, err := stream.Stream(context.Background(), func(row map[string]any) error { + rows = append(rows, row) + return nil + }) + if err != nil { + t.Fatalf("live workspace Combine over PostgreSQL captures: %v", err) + } + if result.RowCount != 2 || len(rows) != 2 || rows[0]["auth_resource_path"] != "/programs/a" { + t.Fatalf("terminal rows drifted: %#v (%#v)", rows, result) + } + + if len(ports.pinRevisions) != 1 || strings.Join(ports.pinRevisions[0], ",") != "exec11_pub_rev" { + t.Fatalf("published pin revisions = %v, want exec11_pub_rev", ports.pinRevisions) + } + if ports.resolveCalls != 1 { + t.Fatalf("published resolution calls = %d, want 1", ports.resolveCalls) + } + if len(ports.combineQueries) != 1 { + t.Fatalf("Combine query count = %d, want 1: %v", len(ports.combineQueries), ports.combineQueries) + } + terminal := ports.combineQueries[0] + if !strings.Contains(terminal, "loom_private_") || !strings.Contains(terminal, "exec11_published_physical") { + t.Fatalf("Combine query did not join both private captures and the published table: %s", terminal) + } + var flatArgs []string + var flattenExec11 func(any) + flattenExec11 = func(value any) { + switch typed := value.(type) { + case []any: + for _, element := range typed { + flattenExec11(element) + } + case []string: + flatArgs = append(flatArgs, typed...) + default: + flatArgs = append(flatArgs, fmt.Sprintf("%v", typed)) + } + } + for _, group := range ports.combineArgs { + for _, arg := range group { + flattenExec11(arg) + } + } + reapplied := false + for _, arg := range flatArgs { + if arg == "/programs/a" { + reapplied = true + } + if arg == "/programs/b" || arg == "patient-hidden" || arg == "patient-intruder" || arg == "patient-stale" { + t.Fatalf("out-of-scope value %q reached the Combine query", arg) + } + } + if !reapplied { + t.Fatalf("restricted scope evidence was not reapplied to the Combine read: args=%v", ports.combineArgs) + } + + if ports.visibleDuringQ != 2 { + t.Fatalf("live captures visible during Combine = %d, want 2", ports.visibleDuringQ) + } + if len(ports.seenDuringRun) != 2 { + t.Fatalf("manifest observations = %d, want 2", len(ports.seenDuringRun)) + } + for _, manifest := range ports.seenDuringRun { + if manifest.Identity.AuthScopeMode != authscope.ReadScopeRestricted || + len(manifest.Identity.AuthResourcePaths) != 1 || manifest.Identity.AuthResourcePaths[0] != "/programs/a" || + manifest.Identity.ScopeMode != chartifact.ScopeModeRows || manifest.Identity.PlanDigest == "" || + manifest.Identity.Project != "project-a" || manifest.Identity.DatasetGeneration != "generation-2" { + t.Fatalf("live capture lost its restricted compiler scope identity: %#v", manifest.Identity) + } + captured := ports.capturedRows[manifest.PhysicalTable] + if int64(len(captured)) != manifest.RowCount || manifest.RowCount != int64(len(witness)) { + t.Fatalf("capture %s row count = %d/%d, want %d", manifest.PhysicalTable, len(captured), manifest.RowCount, len(witness)) + } + var values []string + for _, row := range captured { + if row["auth_resource_path"] != "/programs/a" || row["project_id"] != "project-a" { + t.Fatalf("captured row lost the real authorization path: %v", row) + } + values = append(values, fmt.Sprintf("%v", row["value"])) + } + sort.Strings(values) + if strings.Join(values, ",") != strings.Join(witness, ",") { + t.Fatalf("captured values %v drifted from the direct-SQL witness %v", values, witness) + } + } + for table, inserted := range ch.allRows() { + for _, row := range inserted { + value := fmt.Sprintf("%v", row["value"]) + if value == "patient-hidden" || value == "patient-intruder" || value == "patient-stale" { + t.Fatalf("out-of-scope source row %q was captured into %s", value, table) + } + } + } + if catalog.count() != 0 || len(ch.allRows()) != 0 { + t.Fatalf("live Combine leaked %d manifests and %d tables", catalog.count(), len(ch.allRows())) + } +} + +func TestExec11lvLiveCaptureRejectsForgedAuthPath(t *testing.T) { + service := newExec11lvService(t) + service.migrateAndSeed(t) + catalog := &exec11lvCatalog{items: map[string]chartifact.Manifest{}} + ch := &exec11lvClickHouse{table: map[string][]clickhouse.Column{}, rows: map[string][]map[string]any{}} + manager, err := chartifact.New(chartifact.Config{Catalog: catalog, ClickHouse: ch, BatchRows: 1}) + if err != nil { + t.Fatal(err) + } + ports := newExec11lvPorts() + var portCalls int + tampered := func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + portCalls++ + forge := portCalls == 2 + return service.executor(ctx, request, func(row map[string]any) error { + if forge { + forged := make(map[string]any, len(row)) + for key, value := range row { + forged[key] = value + } + forged["auth_resource_path"] = "/programs/b" + return visit(forged) + } + return visit(row) + }) + } + engine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: service.compileTarget(t), + Registry: exec11lvInvalidRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "exec11-live-scope" }, + SourceRows: tampered, + PrivateClickHouseArtifacts: manager, + WithExecutionReadPins: func(_ context.Context, _ []string, visit func(context.Context) error) error { + return visit(context.Background()) + }, + ClickHouseQueryRows: ports.queryRows(catalog, ch, nil), + }) + if err != nil { + t.Fatal(err) + } + resolved := exec11lvResolved(t, engine, false) + stream := exec11lvJoin(t, engine, resolved) + if _, err := stream.Stream(context.Background(), func(map[string]any) error { return nil }); err == nil || + !strings.Contains(err.Error(), "outside the exact restricted scope") { + t.Fatalf("forged authorization path error = %v, want exact-scope rejection", err) + } + if len(ports.combineQueries) != 0 { + t.Fatalf("Combine ran %d times after a forged scope proof: %v", len(ports.combineQueries), ports.combineQueries) + } + if catalog.count() != 0 || len(ch.allRows()) != 0 { + t.Fatalf("forged capture leaked %d manifests and %d tables", catalog.count(), len(ch.allRows())) + } +} + +func TestExec11lvLiveCaptureCancellationLeavesNothing(t *testing.T) { + service := newExec11lvService(t) + service.migrateAndSeed(t) + catalog := &exec11lvCatalog{items: map[string]chartifact.Manifest{}} + ch := &exec11lvClickHouse{table: map[string][]clickhouse.Column{}, rows: map[string][]map[string]any{}} + manager, err := chartifact.New(chartifact.Config{Catalog: catalog, ClickHouse: ch, BatchRows: 1}) + if err != nil { + t.Fatal(err) + } + ports := newExec11lvPorts() + runCtx, cancelRun := context.WithCancel(context.Background()) + defer cancelRun() + var portCalls int + cancelling := func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + portCalls++ + return service.executor(ctx, request, func(row map[string]any) error { + if err := visit(row); err != nil { + return err + } + if portCalls == 1 { + cancelRun() + } + return nil + }) + } + engine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: service.compileTarget(t), + Registry: exec11lvInvalidRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "exec11-live-scope" }, + SourceRows: cancelling, + PrivateClickHouseArtifacts: manager, + WithExecutionReadPins: func(_ context.Context, _ []string, visit func(context.Context) error) error { + return visit(context.Background()) + }, + ClickHouseQueryRows: ports.queryRows(catalog, ch, nil), + }) + if err != nil { + t.Fatal(err) + } + resolved := exec11lvResolved(t, engine, false) + stream := exec11lvJoin(t, engine, resolved) + if _, err := stream.Stream(runCtx, func(map[string]any) error { return nil }); err == nil { + t.Fatal("cancelled capture reported success") + } else if !errors.Is(err, context.Canceled) && !strings.Contains(err.Error(), "cancel") { + t.Fatalf("cancellation surfaced as an unrelated error: %v", err) + } + if len(ports.combineQueries) != 0 { + t.Fatalf("Combine ran %d times after cancellation: %v", len(ports.combineQueries), ports.combineQueries) + } + if catalog.count() != 0 || len(ch.allRows()) != 0 { + t.Fatalf("cancelled capture leaked %d manifests and %d tables", catalog.count(), len(ch.allRows())) + } +} + +func TestExec11lvLiveEvidenceFreeCaptureRefused(t *testing.T) { + service := newExec11lvService(t) + service.migrateAndSeed(t) + catalog := &exec11lvCatalog{items: map[string]chartifact.Manifest{}} + ch := &exec11lvClickHouse{table: map[string][]clickhouse.Column{}, rows: map[string][]map[string]any{}} + manager, err := chartifact.New(chartifact.Config{Catalog: catalog, ClickHouse: ch, BatchRows: 1}) + if err != nil { + t.Fatal(err) + } + ports := newExec11lvPorts() + engine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: service.compileTarget(t), + Registry: exec11lvInvalidRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "exec11-live-scope" }, + SourceRows: service.executor, + PrivateClickHouseArtifacts: manager, + WithExecutionReadPins: func(_ context.Context, _ []string, visit func(context.Context) error) error { + return visit(context.Background()) + }, + ResolveClickHouseInputs: func(context.Context, ir.PhysicalClickHouseCombine, recipe.RuntimeBindings) ([]ir.ResolvedClickHouseTable, error) { + return nil, nil + }, + ClickHouseQueryRows: ports.queryRows(catalog, ch, nil), + }) + if err != nil { + t.Fatal(err) + } + resolved := exec11lvResolved(t, engine, false) + dependencies := append([]lower.CompiledRecipeOutput(nil), resolved.Compiled.WorkspaceDependencies...) + for index := range dependencies { + dependencies[index].ScopeEvidence = nil + dependencies[index].ScopeEvidenceIdentity = nil + } + resolved.Compiled.WorkspaceDependencies = dependencies + stream := exec11lvJoin(t, engine, resolved) + _, err = stream.Stream(context.Background(), func(map[string]any) error { return nil }) + if err == nil || !strings.Contains(err.Error(), "requires opaque evidence for its exact complete source plan") { + t.Fatalf("evidence-free live capture error = %v, want the opaque-evidence refusal", err) + } + if catalog.count() != 0 || len(ch.allRows()) != 0 || len(ports.combineQueries) != 0 { + t.Fatalf("evidence-free capture leaked %d manifests, %d tables, %d Combine queries", + catalog.count(), len(ch.allRows()), len(ports.combineQueries)) + } +} + +func TestExec11lvLiveCombineExecutesAgainstClickHouse(t *testing.T) { + service := newExec11lvService(t) + service.migrateAndSeed(t) + witness := service.witnessIDs(t) + chURL := strings.TrimSpace(os.Getenv("LOOM_CLICKHOUSE_URL")) + if chURL == "" { + t.Skip("set LOOM_CLICKHOUSE_URL to execute the live workspace Combine against a real ClickHouse") + } + database := strings.TrimSpace(os.Getenv("LOOM_CLICKHOUSE_DATABASE")) + if database == "" { + database = "loom_test" + } + if !clickhouseIdentifier(database) { + t.Fatalf("LOOM_CLICKHOUSE_DATABASE %q is not a plain identifier", database) + } + client, err := clickhouse.New(clickhouse.Options{ + URL: chURL, Database: database, + Username: os.Getenv("LOOM_CLICKHOUSE_USERNAME"), + Password: os.Getenv("LOOM_CLICKHOUSE_PASSWORD"), + Timeout: 30 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = client.Close() }) + ctx := context.Background() + if err := client.EnsureDatabase(ctx); err != nil { + t.Fatal(err) + } + pubColumns := []clickhouse.Column{ + {Name: "__loom_row_id", Type: "String"}, + {Name: "value", Type: "Nullable(String)"}, + {Name: "auth_resource_path", Type: "String"}, + } + if err := client.CreateTable(ctx, "exec11_published_physical", pubColumns); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = client.DropTable(context.Background(), "exec11_published_physical") }) + if err := client.InsertRows(ctx, "exec11_published_physical", pubColumns, []map[string]any{ + {"__loom_row_id": "exec11-pub-1", "value": "patient-published", "auth_resource_path": "/programs/a"}, + }); err != nil { + t.Fatal(err) + } + + catalog := &exec11lvCatalog{items: map[string]chartifact.Manifest{}} + manager, err := chartifact.New(chartifact.Config{Catalog: catalog, ClickHouse: client, BatchRows: 1}) + if err != nil { + t.Fatal(err) + } + bindings := exec11lvBindings() + engine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: service.compileTarget(t), + Registry: exec11lvInvalidRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "exec11-live-scope" }, + SourceRows: service.executor, + PrivateClickHouseArtifacts: manager, + WithExecutionReadPins: func(_ context.Context, revisions []string, visit func(context.Context) error) error { + if strings.Join(revisions, ",") != "exec11_pub_rev" { + return fmt.Errorf("published pins = %v, want exec11_pub_rev", revisions) + } + return visit(context.Background()) + }, + ResolveClickHouseInputs: func(_ context.Context, _ ir.PhysicalClickHouseCombine, got recipe.RuntimeBindings) ([]ir.ResolvedClickHouseTable, error) { + if got.Project != bindings.Project || got.DatasetGeneration != bindings.DatasetGeneration { + return nil, fmt.Errorf("published resolution lost the exact project/generation") + } + return []ir.ResolvedClickHouseTable{{ + TableID: "exec11_pub_table", RevisionID: "exec11_pub_rev", OutputID: "pub", + Project: bindings.Project, DatasetGeneration: bindings.DatasetGeneration, + ReceiptID: "receipt-exec11-pub", SchemaDigest: "schema-exec11-pub", ScopeDigest: "scope-exec11-pub", + PhysicalTable: "exec11_published_physical", AuthResourcePaths: []string{"/programs/a"}, ScopeMode: chartifact.ScopeModeRows, + Columns: []ir.ResolvedClickHouseColumn{ + {ID: "loom:row_id", Name: "__loom_row_id", LogicalType: "string", ClickHouseType: "String"}, + {ID: "value-id", Name: "value", LogicalType: "string", ClickHouseType: "Nullable(String)"}, + {ID: "auth-path", Name: "auth_resource_path", LogicalType: "string", ClickHouseType: "String"}, + }, + }}, nil + }, + ClickHouseQueryRows: client.QueryRowsArgsVisit, + }) + if err != nil { + t.Fatal(err) + } + resolved := exec11lvResolved(t, engine, true) + stream := exec11lvJoin(t, engine, resolved) + var values []string + result, err := stream.Stream(ctx, func(row map[string]any) error { + value := fmt.Sprintf("%v", row["value"]) + if value == "patient-hidden" || value == "patient-intruder" || value == "patient-stale" { + return fmt.Errorf("out-of-scope value %q crossed the real ClickHouse Combine", value) + } + values = append(values, value) + return nil + }) + if err != nil { + t.Fatalf("real ClickHouse Combine: %v", err) + } + sort.Strings(values) + want := append(append(append([]string(nil), witness...), witness...), "patient-published") + sort.Strings(want) + if result.RowCount != len(want) || strings.Join(values, ",") != strings.Join(want, ",") { + t.Fatalf("real ClickHouse Combine rows = %v (%d rows), want %v", values, result.RowCount, want) + } + if catalog.count() != 0 { + t.Fatalf("%d manifests survived the completed real-ClickHouse Combine", catalog.count()) + } + var residue []string + if err := client.QueryRowsArgsVisit(ctx, + "SELECT name FROM system.tables WHERE database = '"+database+"'", []string{"name"}, + func(row map[string]any) error { + residue = append(residue, fmt.Sprintf("%v", row["name"])) + return nil + }); err != nil { + t.Fatal(err) + } + for _, name := range residue { + if strings.HasPrefix(name, "loom_private_") { + t.Fatalf("private artifact table %q survived the completed Combine", name) + } + } +} + +func TestZZPostgresCaptureZeroLeftoverFixtures(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 private-capture acceptance tests") + } + scrub, _ := exec03pqScrubberFor(raw) + exec11lvCaptureBaseline(raw) + if exec11lvBaselineErr != nil { + t.Fatalf("capture pg_database baseline for audit: %s", scrub(exec11lvBaselineErr)) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for audit: %s", scrub(err)) + } + defer admin.Close() + owned := exec11lvRegistrySnapshot() + for db := range owned { + var count int + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Fatalf("audit %s: %s", db, scrub(err)) + } + if count != 0 { + t.Errorf("owned fixture database %s survived", db) + } + } + baseline := map[string]bool{} + for _, name := range exec11lvBaselineNames { + baseline[name] = true + } + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec11\\_%' ESCAPE '\\'") + if err != nil { + t.Fatalf("audit sweep: %s", scrub(err)) + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("audit scan: %s", scrub(err)) + } + if !baseline[name] { + t.Errorf("unregistered survivor under owned prefix: %s", name) + } + } + if err := rows.Err(); err != nil { + t.Fatalf("audit rows: %s", scrub(err)) + } + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatalf("audit timed out") + } +} diff --git a/internal/dataframe/execution/engine_postgres_capture_test.go b/internal/dataframe/execution/engine_postgres_capture_test.go new file mode 100644 index 000000000..e6f423fec --- /dev/null +++ b/internal/dataframe/execution/engine_postgres_capture_test.go @@ -0,0 +1,396 @@ +package execution + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "strings" + "testing" + + "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/execution/chartifact" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/calypr/loom/internal/store/clickhouse" +) + +type exec11CaptureFixture struct { + engine *Engine + spies *dispatchSpies + catalog *testArtifactCatalog + ch *testArtifactClickHouse + manager *chartifact.Manager + resolved Resolved + stream OutputStream + columns []chartifact.Column + declared []string +} + +func exec11RestrictedBindings() recipe.RuntimeBindings { + return recipe.RuntimeBindings{ + Project: "project-a", DatasetGeneration: "generation-2", AuthScopeMode: authscope.ReadScopeRestricted, + AuthResourcePaths: []string{"/programs/a"}, + } +} + +func newExec11Capture(t *testing.T, rows []map[string]any) exec11CaptureFixture { + t.Helper() + makeOutput := func(name string) recipe.Output { + return recipe.Output{ + Name: name, RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{{Name: "value", ColumnID: name + "-value", Expr: recipe.Expression{Select: "root.id"}}}, + } + } + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: "pg-capture-fixture", TranslationVersion: "test", + Outputs: []recipe.Output{makeOutput("patients"), makeOutput("observations")}, + } + spies := &dispatchSpies{} + config := spies.arangoEngine() + config.CompileTarget = exec02PostgreSQLTarget(t) + config.SourceRows = exec02pgPort(spies, &spies.pgFull, rows) + config.PreviewSourceRows = exec02pgPort(spies, &spies.pgPreview, rows) + engine := config.newEngineForTest(t) + resolved, err := engine.CompileResolvedBundle(context.Background(), bundle, exec11RestrictedBindings()) + if err != nil { + t.Fatalf("compile capture bundle: %v", err) + } + var output = resolved.Compiled.Outputs[0] + if output.Name != "patients" { + t.Fatalf("fixture output ordering drifted: %q", output.Name) + } + stream, query, err := engine.buildCompiledOutputStream(context.Background(), resolved, output, 0, map[string]bool{}, true) + if err != nil { + t.Fatalf("build PostgreSQL capture stream: %v", err) + } + if stream.command.Backend() != command.CompiledBackendPostgreSQL || stream.expectedBackend != command.CompiledBackendPostgreSQL { + t.Fatalf("capture stream is not fully PostgreSQL-tagged: command=%q expected=%q", stream.command.Backend(), stream.expectedBackend) + } + columns, err := chartifact.ColumnsFromCompiledOutput(stream.outputSchema) + if err != nil { + t.Fatalf("artifact columns: %v", err) + } + catalog := &testArtifactCatalog{items: map[string]chartifact.Manifest{}} + ch := &testArtifactClickHouse{columns: map[string][]clickhouse.Column{}, rows: map[string][]map[string]any{}} + manager, err := chartifact.New(chartifact.Config{Catalog: catalog, ClickHouse: ch, BatchRows: 1}) + if err != nil { + t.Fatalf("chartifact.New: %v", err) + } + return exec11CaptureFixture{ + engine: engine, spies: spies, catalog: catalog, ch: ch, manager: manager, + resolved: resolved, stream: stream, columns: columns, declared: append([]string(nil), query.Columns...), + } +} + +func exec11PortRows(values ...string) []map[string]any { + rows := make([]map[string]any, 0, len(values)) + for index, value := range values { + rows = append(rows, map[string]any{ + "_key": []string{"generation-2/exec11-row-one", "generation-2/exec11-row-two"}[index], + "value": value, + "auth_resource_path": "/programs/a", + }) + } + return rows +} + +func exec11IdentityDigest(project, key string) string { + encoded, err := json.Marshal([]any{project, key}) + if err != nil { + panic(err) + } + sum := sha256.Sum256(encoded) + return hex.EncodeToString(sum[:]) +} + +func TestPostgreSQLPrivateCaptureCombinePortsPredicate(t *testing.T) { + catalog := &testArtifactCatalog{items: map[string]chartifact.Manifest{}} + ch := &testArtifactClickHouse{columns: map[string][]clickhouse.Column{}, rows: map[string][]map[string]any{}} + manager, err := chartifact.New(chartifact.Config{Catalog: catalog, ClickHouse: ch, BatchRows: 1}) + if err != nil { + t.Fatal(err) + } + build := func(withManager, withQueryRows, withResolve, withPins bool) *Engine { + config := Config{ + CompileTarget: exec02PostgreSQLTarget(t), + Registry: invalidRecipeRegistry{}, + SourceRows: func(context.Context, RowsRequest, RowVisitor) error { return nil }, + } + if withQueryRows { + config.ClickHouseQueryRows = func(context.Context, string, []string, func(map[string]any) error, ...any) error { return nil } + } + if withResolve { + config.ResolveClickHouseInputs = func(context.Context, ir.PhysicalClickHouseCombine, recipe.RuntimeBindings) ([]ir.ResolvedClickHouseTable, error) { + return nil, nil + } + } + if withPins { + config.WithExecutionReadPins = func(_ context.Context, _ []string, visit func(context.Context) error) error { + return visit(context.Background()) + } + } + if withManager { + config.PrivateClickHouseArtifacts = manager + } + engine, err := New(config) + if err != nil { + t.Fatalf("New: %v", err) + } + return engine + } + if !build(true, true, true, true).CombinePortsConfigured() { + t.Fatal("fully wired combine engine must report configured ports") + } + for _, missing := range []struct { + name string + skip int + }{ + {"manager", 0}, {"query-rows", 1}, {"resolve", 2}, {"pins", 3}, + } { + with := []bool{true, true, true, true} + with[missing.skip] = false + if build(with[0], with[1], with[2], with[3]).CombinePortsConfigured() { + t.Fatalf("engine missing the %s port must report unconfigured", missing.name) + } + } +} + +func exec11Refusal(t *testing.T, fixture exec11CaptureFixture, wantPhrase string) { + t.Helper() + _, err := fixture.stream.MaterializeClickHouseArtifact(context.Background(), fixture.manager, "exec11-1", fixture.stream.stageID, fixture.columns) + if err == nil || !strings.Contains(err.Error(), wantPhrase) { + t.Fatalf("refusal drifted: got %v, want %q", err, wantPhrase) + } + if len(fixture.catalog.items) != 0 { + t.Fatalf("refused capture created %d artifacts", len(fixture.catalog.items)) + } +} + +func TestPostgreSQLPrivateCaptureServesRestrictedRows(t *testing.T) { + fixture := newExec11Capture(t, exec11PortRows("Ada", "Grace")) + if fixture.stream.artifactScopeMode != chartifact.ScopeModeRows || fixture.stream.scopeEvidence == nil || fixture.stream.scopeEvidenceIdentity == nil { + t.Fatalf("capture stream lacks compiler scope: mode=%q evidence=%v identity=%v", + fixture.stream.artifactScopeMode, fixture.stream.scopeEvidence != nil, fixture.stream.scopeEvidenceIdentity != nil) + } + artifact, err := fixture.stream.MaterializeClickHouseArtifact(context.Background(), fixture.manager, "exec11-1", fixture.stream.stageID, fixture.columns) + if err != nil { + t.Fatalf("materialize PostgreSQL capture: %v", err) + } + manifest := artifact.Manifest() + if manifest.RowCount != 2 || manifest.Identity.ExecutionID != "exec11-1" || manifest.Identity.OutputID != "patients" || + manifest.Identity.AuthScopeMode != authscope.ReadScopeRestricted || + len(manifest.Identity.AuthResourcePaths) != 1 || manifest.Identity.AuthResourcePaths[0] != "/programs/a" || + manifest.Identity.ScopeMode != chartifact.ScopeModeRows || + manifest.Identity.RecipeDigest != fixture.resolved.StoredRecipeDigest { + t.Fatalf("PostgreSQL capture lost its exact restricted identity: %#v", manifest) + } + if manifest.Identity.PlanDigest == "" { + t.Fatalf("PostgreSQL capture artifact lacks its plan fingerprint") + } + written := fixture.ch.rows[manifest.PhysicalTable] + if len(written) != 2 { + t.Fatalf("artifact rows = %d, want 2", len(written)) + } + for index, row := range written { + identity, _ := row["__loom_row_id"].(string) + if len(identity) != 64 { + t.Fatalf("PostgreSQL capture row %d lacks a derived 64-hex stable identity: %v", index, row) + } + if row["value"] != exec11PortRows("Ada", "Grace")[index]["value"] || + row["auth_resource_path"] != "/programs/a" || row["project_id"] != "project-a" { + t.Fatalf("PostgreSQL capture row %d drifted or lost its authorization path: %v", index, row) + } + } + if written[0]["__loom_row_id"] == written[1]["__loom_row_id"] { + t.Fatalf("distinct _key rows collapsed to one stable identity") + } + if digest := exec11IdentityDigest("project-a", "generation-2/exec11-row-one"); written[0]["__loom_row_id"] != digest { + t.Fatalf("stable identity is not the scope-bound digest of the row key") + } + if fixture.spies.clickHouseQueries != 0 || len(fixture.spies.aqlQueries) != 0 || len(fixture.spies.aqlPreviewQueries) != 0 || len(fixture.spies.controlSnapshot()) != 0 { + t.Fatalf("PostgreSQL capture leaked into AQL/ClickHouse callbacks: clickhouse=%d aql=%v preview=%v controls=%v", + fixture.spies.clickHouseQueries, fixture.spies.aqlQueries, fixture.spies.aqlPreviewQueries, fixture.spies.controlSnapshot()) + } + if len(fixture.spies.pgFull) != 1 { + t.Fatalf("capture port calls = %d, want 1", len(fixture.spies.pgFull)) + } + if strings.Join(fixture.spies.pgFull[0].Columns, ",") != strings.Join(fixture.declared, ",") { + t.Fatalf("capture port received declared columns %v, want the compiled contract %v", fixture.spies.pgFull[0].Columns, fixture.declared) + } + if err := artifact.Release(context.Background()); err != nil { + t.Fatalf("release: %v", err) + } +} + +func exec11IdentityDigestFields(values ...any) string { + encoded, err := json.Marshal(append([]any(nil), values...)) + if err != nil { + panic(err) + } + sum := sha256.Sum256(encoded) + return hex.EncodeToString(sum[:]) +} + +func TestPostgreSQLPrivateCaptureRowIdentityMatchesCommandBinds(t *testing.T) { + t.Run("UntrimmedRawBindingCannotEnterTheDigest", func(t *testing.T) { + fixture := newExec11Capture(t, exec11PortRows("Ada")) + fixture.stream.RowIdentity = &spec.RowIdentity{Fields: []string{"project", "dataset_generation", "_key"}} + fixture.stream.bindings.Project = " project-a " + fixture.stream.bindings.DatasetGeneration = " generation-2 " + sqlText, params, err := fixture.stream.command.PostgreSQLParameters() + if err != nil { + t.Fatal(err) + } + if len(params) < 2 { + t.Fatalf("command carries fewer than two scope parameters: %s", sqlText) + } + boundProject, okProject := params[0].Value.(string) + boundGeneration, okGeneration := params[1].Value.(string) + if !okProject || !okGeneration || boundProject != "project-a" || boundGeneration != "generation-2" { + t.Fatalf("command scope arguments drifted from the trimmed binds: %#v/%#v in %s", + params[0].Value, params[1].Value, sqlText) + } + want := exec11IdentityDigestFields(boundProject, boundGeneration, "generation-2/exec11-row-one") + var got string + if _, err := fixture.stream.Stream(context.Background(), func(row map[string]any) error { + got, _ = row["__loom_row_id"].(string) + return nil + }); err != nil { + t.Fatalf("stream: %v", err) + } + if got != want { + t.Fatalf("row identity did not derive from the values the command bound: got %s, want command-bound %s", got, want) + } + }) + t.Run("MissingScopeBindFailsClosedLikeTheAQLArm", func(t *testing.T) { + fixture := newExec11Capture(t, exec11PortRows("Ada")) + fixture.stream.RowIdentity = &spec.RowIdentity{Fields: []string{"project", "dataset_generation", "_key"}} + fixture.stream.physicalPlan.BindVars["dataset_generation"] = 42 + if _, err := fixture.stream.Stream(context.Background(), func(map[string]any) error { return nil }); err == nil || + !strings.Contains(err.Error(), `stable row identity field "dataset_generation" is missing`) { + t.Fatalf("untyped scope bind error = %v, want typed missing-field refusal", err) + } + }) + t.Run("BlankGenerationCannotBuildACaptureCommand", func(t *testing.T) { + bindings := exec11RestrictedBindings() + bindings.DatasetGeneration = " " + makeOutput := func(name string) recipe.Output { + return recipe.Output{ + Name: name, RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{{Name: "value", ColumnID: name + "-value", Expr: recipe.Expression{Select: "root.id"}}}, + } + } + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: "pg-capture-blank", TranslationVersion: "test", + Outputs: []recipe.Output{makeOutput("patients"), makeOutput("observations")}, + } + spies := &dispatchSpies{} + config := spies.arangoEngine() + config.CompileTarget = exec02PostgreSQLTarget(t) + config.SourceRows = exec02pgPort(spies, &spies.pgFull, exec11PortRows("Ada")) + engine := config.newEngineForTest(t) + resolved, err := engine.CompileResolvedBundle(context.Background(), bundle, bindings) + if err != nil { + t.Fatalf("compile: %v", err) + } + if _, _, err := engine.buildCompiledOutputStream(context.Background(), resolved, resolved.Compiled.Outputs[0], 0, map[string]bool{}, true); err == nil || + !strings.Contains(err.Error(), `"dataset_generation" bind must be a string`) { + t.Fatalf("blank-generation capture build error = %v, want the renderer's typed scope refusal", err) + } + }) +} + +func TestPostgreSQLPrivateCaptureDispatchRefusesIncompleteSourceShapes(t *testing.T) { + t.Run("ForeignExpectedBackend", func(t *testing.T) { + fixture := newExec11Capture(t, exec11PortRows("Ada")) + fixture.stream.expectedBackend = command.CompiledBackendAQL + exec11Refusal(t, fixture, "private artifact requires a compiler-resolved streamable AQL, PostgreSQL, or ClickHouse output") + }) + t.Run("MissingDeclaredContract", func(t *testing.T) { + fixture := newExec11Capture(t, exec11PortRows("Ada")) + fixture.stream.commandColumns = nil + exec11Refusal(t, fixture, "private artifact requires a compiler-resolved streamable AQL, PostgreSQL, or ClickHouse output") + }) + t.Run("BoundedPreview", func(t *testing.T) { + fixture := newExec11Capture(t, exec11PortRows("Ada")) + fixture.stream.queryLimit = 10 + exec11Refusal(t, fixture, "cannot be built from a bounded preview") + }) +} + +func TestPostgreSQLPrivateCaptureRestrictedContractFailClosed(t *testing.T) { + t.Run("RestrictedWithoutRowPathProjection", func(t *testing.T) { + fixture := newExec11Capture(t, exec11PortRows("Ada")) + fixture.stream.bindings.IncludeAuthResourcePath = false + exec11Refusal(t, fixture, "restricted PostgreSQL artifact requires a row-level authorization path projection") + }) + t.Run("MissingStableRowIdentity", func(t *testing.T) { + fixture := newExec11Capture(t, []map[string]any{{"value": "Ada", "auth_resource_path": "/programs/a"}}) + fixture.stream.RowIdentity = nil + _, err := fixture.stream.MaterializeClickHouseArtifact(context.Background(), fixture.manager, "exec11-1", fixture.stream.stageID, fixture.columns) + if err == nil || !strings.Contains(err.Error(), "compiled output is missing a stable row identity") || + !strings.Contains(err.Error(), "stream PostgreSQL prefix into private ClickHouse artifact") { + t.Fatalf("missing row identity must fail through the PostgreSQL-labelled stream error: %v", err) + } + if len(fixture.catalog.items) != 0 { + t.Fatalf("refused capture created %d artifacts", len(fixture.catalog.items)) + } + }) + t.Run("MissingRowKeyColumn", func(t *testing.T) { + fixture := newExec11Capture(t, []map[string]any{{"value": "Ada", "auth_resource_path": "/programs/a"}}) + exec11Refusal(t, fixture, `stable row identity field "_key" is missing`) + }) + t.Run("ScopeEvidenceMissing", func(t *testing.T) { + fixture := newExec11Capture(t, exec11PortRows("Ada")) + fixture.stream.scopeEvidence = nil + exec11Refusal(t, fixture, "private PostgreSQL artifact requires opaque evidence for its exact complete source plan") + }) +} + +func TestPostgreSQLPrivateCaptureScopeModeCoherence(t *testing.T) { + t.Run("WholeScopeWithoutCompilerDigest", func(t *testing.T) { + fixture := newExec11Capture(t, exec11PortRows("Ada")) + fixture.stream.artifactScopeMode = chartifact.ScopeModeWhole + fixture.stream.artifactScopeEvidence = "" + exec11Refusal(t, fixture, "whole-scope private artifact requires compiler-issued scope evidence") + }) + t.Run("RowScopeCarryingWholeDigest", func(t *testing.T) { + fixture := newExec11Capture(t, exec11PortRows("Ada")) + fixture.stream.artifactScopeEvidence = "forged-digest" + exec11Refusal(t, fixture, "row-scoped private artifact cannot carry whole-scope evidence") + }) +} + +func TestPostgreSQLPrivateCaptureSchemaIdentity(t *testing.T) { + t.Run("SchemaDigestDrift", func(t *testing.T) { + fixture := newExec11Capture(t, exec11PortRows("Ada")) + drifted := append([]chartifact.Column(nil), fixture.columns...) + for index := range drifted { + if drifted[index].Name == "value" { + drifted[index].LogicalType = "integer" + drifted[index].ClickHouseType = "" + } + } + _, err := fixture.stream.MaterializeClickHouseArtifact(context.Background(), fixture.manager, "exec11-1", fixture.stream.stageID, drifted) + if err == nil || !strings.Contains(err.Error(), "does not match the compiled output schema") { + t.Fatalf("schema drift must refuse: %v", err) + } + if len(fixture.catalog.items) != 0 { + t.Fatalf("refused capture created %d artifacts", len(fixture.catalog.items)) + } + }) + t.Run("StreamColumnMissingFromSchema", func(t *testing.T) { + fixture := newExec11Capture(t, exec11PortRows("Ada")) + fixture.stream.Columns = nil + _, err := fixture.stream.MaterializeClickHouseArtifact(context.Background(), fixture.manager, "exec11-1", fixture.stream.stageID, fixture.columns) + if err == nil || !strings.Contains(err.Error(), "do not match the private artifact schema") { + t.Fatalf("stream/schema divergence must refuse: %v", err) + } + if len(fixture.catalog.items) != 0 { + t.Fatalf("refused capture created %d artifacts", len(fixture.catalog.items)) + } + }) +} diff --git a/internal/dataframe/execution/engine_postgres_category_pg18_test.go b/internal/dataframe/execution/engine_postgres_category_pg18_test.go new file mode 100644 index 000000000..59c27869f --- /dev/null +++ b/internal/dataframe/execution/engine_postgres_category_pg18_test.go @@ -0,0 +1,780 @@ +package execution_test + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "sync" + "testing" + "time" + + postgres "github.com/calypr/loom/internal/backends/postgres" + storepg "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + pgrender "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + exec06csPrefix = "verify_exec06_cs_" + exec06csNameFmt = exec06csPrefix + "%d_%s" +) + +var exec06csNamePattern = regexp.MustCompile(`^verify_exec06_cs_[0-9]+_[0-9a-f]+$`) + +var ( + exec06csOwnedMu sync.Mutex + exec06csOwnedByName = map[string]string{} +) + +func exec06csRegister(db, schema string) { + exec06csOwnedMu.Lock() + defer exec06csOwnedMu.Unlock() + exec06csOwnedByName[db] = schema +} + +func exec06csRegistrySnapshot() map[string]string { + exec06csOwnedMu.Lock() + defer exec06csOwnedMu.Unlock() + out := make(map[string]string, len(exec06csOwnedByName)) + for db, schema := range exec06csOwnedByName { + out[db] = schema + } + return out +} + +var ( + exec06csBaselineOnce sync.Once + exec06csBaselineNames []string + exec06csBaselineErr error +) + +func exec06csCaptureBaseline(raw string) { + exec06csBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec06csBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec06\\_cs\\_%' ESCAPE '\\'") + if err != nil { + exec06csBaselineErr = fmt.Errorf("read baseline: %w", err) + return + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + exec06csBaselineErr = fmt.Errorf("scan baseline: %w", err) + return + } + exec06csBaselineNames = append(exec06csBaselineNames, name) + } + }) +} + +func exec06csRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +type exec06csService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + schema string + dsn string + target compiler.Target + scrub func(any) string + fail func(step string, err error) + executor dataframeexecution.SourceRows +} + +func newExec06csService(t *testing.T) *exec06csService { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 category scan acceptance tests") + } + scrub, addSecret := exec04pvScrubberFor(raw) + exec06csCaptureBaseline(raw) + if exec06csBaselineErr != nil { + t.Fatalf("capture pg_database baseline: %s", scrub(exec06csBaselineErr)) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, "SELECT current_setting('server_version_num')::int8, current_setting('server_version')").Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if err := storepg.CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required PostgreSQL 18 backend: %v", err) + } + + dbName := fmt.Sprintf(exec06csNameFmt, time.Now().UnixNano(), exec06csRandomHex(t)) + if !exec06csNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + var created int64 + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", dbName).Scan(&created); err != nil { + fail("verify isolated database creation", err) + } + if created != 1 { + t.Fatalf("isolated database %q did not verifiably come into existence", dbName) + } + exec06csRegister(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + addSecret(dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(exec06csNameFmt, time.Now().UnixNano(), exec06csRandomHex(t)) + if !exec06csNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + exec06csRegister(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + executor, err := postgres.NewSourceRows(iso) + if err != nil { + fail("create PostgreSQL row executor", err) + } + binding, err := ir.NewPostgresStorageBinding(schema) + if err != nil { + fail("bind PostgreSQL storage placement", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + fail("select PostgreSQL compile target", err) + } + return &exec06csService{ctx: ctx, admin: admin, iso: iso, schema: schema, dsn: dsn, target: target, scrub: scrub, fail: fail, executor: executor} +} + +func (s *exec06csService) migrate(t *testing.T) { + t.Helper() + client, err := storepg.OpenPool(s.ctx, storepg.ClientConfig{DSN: s.dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 2 * time.Second}) + if err != nil { + s.fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + defer client.Close() + if _, err := storepg.MigrateSchema(s.ctx, client, s.schema); err != nil { + s.fail("migrate owned schema", err) + } +} + +func (s *exec06csService) seed(t *testing.T, rows []exec04pvFixtureRow) { + t.Helper() + repo, err := storepg.NewResourceRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + for _, row := range rows { + if _, err := repo.InsertResource(s.ctx, source.Resource{ + Project: row.project, DatasetGeneration: row.generation, ResourceType: row.resourceType, + LogicalID: row.logicalID, AuthResourcePath: row.authPath, Payload: json.RawMessage(row.payload), + }); err != nil { + s.fail("insert fixture resource "+row.logicalID, err) + } + } +} + +func (s *exec06csService) countResources(t *testing.T) int64 { + t.Helper() + var count int64 + if err := s.iso.QueryRow(s.ctx, `SELECT count(*) FROM "`+s.schema+`"."fhir_resource"`).Scan(&count); err != nil { + s.fail("count fixture rows", err) + } + return count +} + +func (s *exec06csService) witness(t *testing.T, query string, args ...any) [][]string { + t.Helper() + rows, err := s.iso.Query(s.ctx, query, args...) + if err != nil { + s.fail("direct-SQL witness", err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + var left, right any + if err := rows.Scan(&left, &right); err != nil { + s.fail("witness scan", err) + } + line := make([]string, 2) + for index, cell := range []any{left, right} { + if cell == nil { + line[index] = "" + continue + } + line[index] = fmt.Sprintf("%v", cell) + } + out = append(out, line) + } + if err := rows.Err(); err != nil { + s.fail("witness rows", err) + } + return out +} + +type exec06csRecorder struct { + s *exec06csService + calls []exec06csCall + afterRow func(rowsSeen int) + failPort error +} + +type exec06csCall struct { + sql string + args []command.PostgreSQLParameter + declared []string + tag command.CompiledBackend +} + +func (r *exec06csRecorder) port(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + if r.failPort != nil { + return r.failPort + } + sqlText, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + r.calls = append(r.calls, exec06csCall{sql: sqlText, args: args, declared: append([]string(nil), request.DeclaredColumns...), tag: request.Command.Backend()}) + seen := 0 + wrapped := func(row map[string]any) error { + if err := visit(row); err != nil { + return err + } + seen++ + if r.afterRow != nil { + r.afterRow(seen) + } + return nil + } + return r.s.executor(ctx, request, wrapped) +} + +func (r *exec06csRecorder) counts() int { return len(r.calls) } + +func (s *exec06csService) engine(t *testing.T, recorder *exec06csRecorder) *dataframeexecution.Engine { + t.Helper() + config := exec04pvForbidArango() + config.CompileTarget = s.target + config.PreviewSourceRows = nil + config.ScopeDigest = func(recipe.RuntimeBindings) string { return "exec06-cs-scope" } + config.SourceRows = recorder.port + engine, err := dataframeexecution.New(config) + if err != nil { + t.Fatalf("engine: %v", s.scrub(err)) + } + return engine +} + +func exec06csCategoryFixture() []exec04pvFixtureRow { + return []exec04pvFixtureRow{ + {"P1", "gen-a", "Patient", "p-female-1", "FHIR/Patient", `{"gender":"female"}`}, + {"P1", "gen-a", "Patient", "p-female-2", "FHIR/Patient", `{"gender":"female"}`}, + {"P1", "gen-a", "Patient", "p-male", "FHIR/Patient", `{"gender":"male"}`}, + {"P1", "gen-a", "Patient", "p-absent", "FHIR/Patient", `{"active":true}`}, + {"P1", "gen-a", "Patient", "p-null", "FHIR/Patient", `{"gender":null}`}, + {"P1", "gen-a", "Patient", "p-empty", "FHIR/Patient", `{"gender":""}`}, + {"P1", "gen-a", "Patient", "p-false", "FHIR/Patient", `{"gender":false}`}, + {"P1", "gen-a", "Patient", "p-zero", "FHIR/Patient", `{"gender":0}`}, + {"P2", "gen-a", "Patient", "p-decoy-project", "FHIR/Patient", `{"gender":"decoy-project"}`}, + {"P1", "gen-b", "Patient", "p-decoy-generation", "FHIR/Patient", `{"gender":"decoy-generation"}`}, + {"P1", "gen-a", "Patient", "p-decoy-auth", "FHIR/Encounter", `{"gender":"decoy-auth"}`}, + {"P1", "gen-a", "Observation", "o-decoy-type", "FHIR/Observation", `{"gender":"decoy-type"}`}, + } +} + +func exec06csScan(t *testing.T, s *exec06csService, engine *dataframeexecution.Engine, resolved dataframeexecution.Resolved, request dataframeexecution.CategoryScanRequest) (dataframeexecution.CategoryScanResult, error) { + t.Helper() + return engine.ScanCategories(s.ctx, resolved, request) +} + +func exec06csPatientBundle() recipe.Bundle { return exec04pvBundle() } + +func (s *exec06csService) preparePatients(t *testing.T, project, generation string, recorder *exec06csRecorder) (*dataframeexecution.Engine, dataframeexecution.Resolved) { + t.Helper() + s.migrate(t) + s.seed(t, exec06csCategoryFixture()) + engine := s.engine(t, recorder) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec06csPatientBundle(), exec04pvBindings(project, generation)) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + return engine, resolved +} + +func exec06csWantValues() []dataframeexecution.CategoryValue { + return []dataframeexecution.CategoryValue{ + {Present: false, Value: nil}, + {Present: true, Value: false}, + {Present: true, Value: nil}, + {Present: true, Value: float64(0)}, + {Present: true, Value: ""}, + {Present: true, Value: "female"}, + {Present: true, Value: "male"}, + } +} + +func TestPostgresCategoryExactValuesLivePG18(t *testing.T) { + s := newExec06csService(t) + recorder := &exec06csRecorder{s: s} + engine, resolved := s.preparePatients(t, "P1", "gen-a", recorder) + result, err := exec06csScan(t, s, engine, resolved, dataframeexecution.CategoryScanRequest{Output: "patients", Column: "gender", MaxValues: 8}) + if err != nil { + t.Fatalf("scan: %s", s.scrub(err)) + } + want := exec06csWantValues() + if !reflect.DeepEqual(result.Values, want) { + t.Fatalf("values = %#v, want %#v", result.Values, want) + } + if !result.Complete || result.Overflow || result.ConclusiveMissing { + t.Fatalf("status = Complete:%v Overflow:%v ConclusiveMissing:%v", result.Complete, result.Overflow, result.ConclusiveMissing) + } + if result.Proof.Version != 1 || result.Proof.Output != "patients" || result.Proof.Column != "gender" || + result.Proof.MaxValues != 8 || !result.Proof.PresenceTracked || strings.TrimSpace(result.Proof.Fingerprint) == "" { + t.Fatalf("proof not bound to the request: %#v", result.Proof) + } + if recorder.counts() != 1 { + t.Fatalf("port calls = %d, want exactly one bounded page statement", recorder.counts()) + } + call := recorder.calls[0] + if call.tag != command.CompiledBackendPostgreSQL { + t.Fatalf("tag = %q, want POSTGRESQL", call.tag) + } + if len(call.declared) != 2 || call.declared[0] != "present" || call.declared[1] != "value" { + t.Fatalf("declared contract = %v, want [present value]", call.declared) + } + if !strings.Contains(call.sql, "GROUP BY") || !strings.Contains(call.sql, "LIMIT") || strings.Contains(call.sql, "@") { + t.Fatalf("statement is not the bounded distinct page: %s", call.sql) + } + last := call.args[len(call.args)-1] + if last.Type != command.PostgreSQLParameterBigint || last.Value != int64(9) { + t.Fatalf("bound argument = %+v, want bigint 9 (maxValues+1)", last) + } + for _, forbidden := range []string{"decoy-project", "decoy-generation", "decoy-auth", "decoy-type"} { + for _, value := range result.Values { + if text, ok := value.Value.(string); ok && strings.Contains(text, forbidden) { + t.Fatalf("foreign-scope data leaked into the page: %v", value) + } + } + } +} + +func TestPostgresCategoryDeterministicRerunLivePG18(t *testing.T) { + s := newExec06csService(t) + recorder := &exec06csRecorder{s: s} + engine, resolved := s.preparePatients(t, "P1", "gen-a", recorder) + first, err := exec06csScan(t, s, engine, resolved, dataframeexecution.CategoryScanRequest{Output: "patients", Column: "gender", MaxValues: 8}) + if err != nil { + t.Fatalf("first scan: %s", s.scrub(err)) + } + second, err := exec06csScan(t, s, engine, resolved, dataframeexecution.CategoryScanRequest{Output: "patients", Column: "gender", MaxValues: 8}) + if err != nil { + t.Fatalf("second scan: %s", s.scrub(err)) + } + if !reflect.DeepEqual(first.Values, second.Values) { + t.Fatalf("page order was not deterministic:\nfirst: %#v\nsecond: %#v", first.Values, second.Values) + } + direct := s.witness(t, ` + SELECT present::text, + CASE WHEN value IS NULL THEN '' + WHEN jsonb_typeof(value) = 'null' THEN '' + ELSE value #>> '{}' END + FROM ( + SELECT COALESCE(jsonb_typeof(payload) = 'object' AND jsonb_exists(payload, 'gender'), false) AS present, + CASE WHEN COALESCE(jsonb_typeof(payload) = 'object' AND jsonb_exists(payload, 'gender'), false) + THEN payload #> ARRAY['gender'] END AS value + FROM "`+s.schema+`"."fhir_resource" + WHERE project = $1 AND dataset_generation = $2 AND resource_type = 'Patient' + AND auth_resource_path = ANY($3::text[]) + ) grouped GROUP BY present, value`, + "P1", "gen-a", []string{"FHIR/Patient"}) + witnessSet := map[string]int{} + for _, row := range direct { + witnessSet[row[0]+"|"+row[1]]++ + } + scanSet := map[string]int{} + for _, got := range first.Values { + encoded := fmt.Sprintf("%v", got.Value) + if got.Value == nil { + encoded = "" + if got.Present { + encoded = "" + } + } else if text, ok := got.Value.(string); ok { + encoded = text + } + scanSet[fmt.Sprintf("%v", got.Present)+"|"+encoded]++ + } + if !reflect.DeepEqual(witnessSet, scanSet) { + t.Fatalf("decoded page disagrees with the direct-SQL witness:\nscan: %v\nwitness: %v", scanSet, witnessSet) + } + if recorder.counts() != 2 { + t.Fatalf("port calls = %d, want two (one per scan)", recorder.counts()) + } +} + +func TestPostgresCategoryOverflowLivePG18(t *testing.T) { + s := newExec06csService(t) + recorder := &exec06csRecorder{s: s} + s.migrate(t) + s.seed(t, exec06csCategoryFixture()) + engine := s.engine(t, recorder) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec06csPatientBundle(), exec04pvBindings("P1", "gen-a")) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + before := recorder.counts() + result, err := exec06csScan(t, s, engine, resolved, dataframeexecution.CategoryScanRequest{Output: "patients", Column: "gender", MaxValues: 2}) + if err != nil { + t.Fatalf("overflow scan: %s", s.scrub(err)) + } + if recorder.counts()-before != 1 { + t.Fatalf("overflow scan issued %d statements, want exactly one — truncation is page-owned", recorder.counts()-before) + } + if len(result.Values) != 2 { + t.Fatalf("values = %#v, want exactly two capped", result.Values) + } + if result.Complete || !result.Overflow || result.ConclusiveMissing { + t.Fatalf("status = Complete:%v Overflow:%v ConclusiveMissing:%v, want Complete=false Overflow=true", result.Complete, result.Overflow, result.ConclusiveMissing) + } +} + +func TestPostgresCategoryEmptyScopeNoLeakLivePG18(t *testing.T) { + s := newExec06csService(t) + recorder := &exec06csRecorder{s: s} + engine, resolved := s.preparePatients(t, "P-none", "gen-a", recorder) + result, err := exec06csScan(t, s, engine, resolved, dataframeexecution.CategoryScanRequest{Output: "patients", Column: "gender", MaxValues: 8}) + if err != nil { + t.Fatalf("empty-scope scan: %s", s.scrub(err)) + } + if len(result.Values) != 0 || !result.Complete || result.Overflow { + t.Fatalf("empty scope result = %#v, want zero values, complete", result) + } + if total := s.countResources(t); total != 12 { + t.Fatalf("fixture witness: fhir_resource holds %d rows, want 12 (scope-empty, not database-empty)", total) + } + if recorder.counts() != 1 { + t.Fatalf("port calls = %d, want one", recorder.counts()) + } + call := recorder.calls[0] + if len(call.args) < 5 { + t.Fatalf("statement carries %d arguments, want at least the five scope/window arguments: %s", len(call.args), call.sql) + } + if call.args[0].Type != command.PostgreSQLParameterText || call.args[0].Value != "P-none" { + t.Fatalf("project argument = %+v, want text \"P-none\"", call.args[0]) + } + if call.args[1].Type != command.PostgreSQLParameterText || call.args[1].Value != "gen-a" { + t.Fatalf("generation argument = %+v, want text \"gen-a\"", call.args[1]) + } +} + +func TestPostgresCategoryCancellationLivePG18(t *testing.T) { + s := newExec06csService(t) + recorder := &exec06csRecorder{s: s} + engine, resolved := s.preparePatients(t, "P1", "gen-a", recorder) + ctx, cancel := context.WithCancel(s.ctx) + defer cancel() + recorder.afterRow = func(rowsSeen int) { + if rowsSeen == 1 { + cancel() + } + } + result, err := engine.ScanCategories(ctx, resolved, dataframeexecution.CategoryScanRequest{Output: "patients", Column: "gender", MaxValues: 8}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled scan = (%#v, %v), want an errors.Is(context.Canceled) error", result, err) + } + if result.Complete || result.Overflow || len(result.Values) != 0 { + t.Fatalf("cancelled scan must be zero-result and never complete, got %#v", result) + } + recorder.afterRow = nil + var alive dataframeexecution.CategoryScanResult + var aliveErr error + for attempt := 1; attempt <= 6; attempt++ { + alive, aliveErr = engine.ScanCategories(s.ctx, resolved, dataframeexecution.CategoryScanRequest{Output: "patients", Column: "gender", MaxValues: 8}) + if aliveErr == nil { + break + } + } + if aliveErr != nil { + t.Fatalf("ALIVE control never converged: %s", s.scrub(aliveErr)) + } + if !reflect.DeepEqual(alive.Values, exec06csWantValues()) || !alive.Complete { + t.Fatalf("ALIVE control result = %#v", alive) + } +} + +func TestPostgresCategoryAuthScopeLivePG18(t *testing.T) { + s := newExec06csService(t) + recorder := &exec06csRecorder{s: s} + s.migrate(t) + fixture := append(exec06csCategoryFixture(), + exec04pvFixtureRow{"P1", "gen-a", "Patient", "p-secret", "FHIR/Secret", `{"gender":"secret-gender"}`}) + s.seed(t, fixture) + engine := s.engine(t, recorder) + narrowedBindings := recipe.RuntimeBindings{Project: "P1", DatasetGeneration: "gen-a", AuthResourcePaths: []string{"FHIR/Patient"}} + narrowed, err := engine.CompileResolvedBundle(s.ctx, exec06csPatientBundle(), narrowedBindings) + if err != nil { + t.Fatalf("CompileResolvedBundle narrowed: %s", s.scrub(err)) + } + result, err := exec06csScan(t, s, engine, narrowed, dataframeexecution.CategoryScanRequest{Output: "patients", Column: "gender", MaxValues: 8}) + if err != nil { + t.Fatalf("narrowed scan: %s", s.scrub(err)) + } + if !reflect.DeepEqual(result.Values, exec06csWantValues()) { + t.Fatalf("narrowed values = %#v, want the seven in-scope groups", result.Values) + } + call := recorder.calls[recorder.counts()-1] + argTexts := make([]string, 0, len(call.args)) + for _, arg := range call.args { + argTexts = append(argTexts, fmt.Sprintf("%s", arg.Value)) + } + joined := strings.Join(argTexts, "\n") + if !strings.Contains(joined, "FHIR/Patient") { + t.Fatalf("authorized path FHIR/Patient missing from the scoped statement arguments:\n%s\nstatement: %s", joined, call.sql) + } + if strings.Contains(joined, "FHIR/Secret") { + t.Fatalf("unauthorized path FHIR/Secret appears in the scoped statement arguments:\n%s", joined) + } + widenedBindings := recipe.RuntimeBindings{Project: "P1", DatasetGeneration: "gen-a", AuthResourcePaths: []string{"FHIR/Patient", "FHIR/Secret"}} + widened, err := engine.CompileResolvedBundle(s.ctx, exec06csPatientBundle(), widenedBindings) + if err != nil { + t.Fatalf("CompileResolvedBundle widened: %s", s.scrub(err)) + } + wide, err := exec06csScan(t, s, engine, widened, dataframeexecution.CategoryScanRequest{Output: "patients", Column: "gender", MaxValues: 8}) + if err != nil { + t.Fatalf("widened scan: %s", s.scrub(err)) + } + found := false + for _, value := range wide.Values { + if text, ok := value.Value.(string); ok && text == "secret-gender" { + found = true + } + } + if !found { + t.Fatalf("widened scan = %#v, want secret-gender once the path is authorized", wide.Values) + } +} + +func TestPostgresCategoryStageScanLivePG18(t *testing.T) { + s := newExec06csService(t) + recorder := &exec06csRecorder{s: s} + s.migrate(t) + s.seed(t, []exec04pvFixtureRow{ + {"P1", "gen-a", "Observation", "o-final-1", "FHIR/Observation", `{"status":"final","valueInteger":1}`}, + {"P1", "gen-a", "Observation", "o-final-2", "FHIR/Observation", `{"status":"final","valueInteger":2}`}, + {"P1", "gen-a", "Observation", "o-amended", "FHIR/Observation", `{"status":"amended","valueInteger":3}`}, + {"P2", "gen-a", "Observation", "o-decoy", "FHIR/Observation", `{"status":"decoy","valueInteger":4}`}, + }) + engine := s.engine(t, recorder) + bindings := recipe.RuntimeBindings{Project: "P1", DatasetGeneration: "gen-a", AuthResourcePaths: []string{"FHIR/Observation"}} + resolved, err := engine.CompileResolvedBundle(s.ctx, exec06ConstructionBundle(), bindings) + if err != nil { + t.Fatalf("CompileResolvedBundle (construction): %s", s.scrub(err)) + } + result, err := engine.ScanCategories(s.ctx, resolved, dataframeexecution.CategoryScanRequest{ + Output: "trace", StageID: recipe.ConstructionSourceProjectionID, + ColumnID: "status_id", ValueColumnID: "amount_id", MaxValues: 8, + }) + if err != nil { + t.Fatalf("stage scan: %s", s.scrub(err)) + } + want := []dataframeexecution.CategoryValue{{Present: true, Value: "amended"}, {Present: true, Value: "final"}} + if !reflect.DeepEqual(result.Values, want) { + t.Fatalf("stage values = %#v, want %#v", result.Values, want) + } + if !result.Complete || result.Overflow { + t.Fatalf("stage status = Complete:%v Overflow:%v", result.Complete, result.Overflow) + } + if result.Proof.Version != 2 || result.Proof.StageID != recipe.ConstructionSourceProjectionID || + result.Proof.ColumnID != "status_id" || result.Proof.ValueColumnID != "amount_id" { + t.Fatalf("stage proof not bound: %#v", result.Proof) + } + if recorder.counts() != 1 { + t.Fatalf("port calls = %d, want one", recorder.counts()) + } + if recorder.calls[0].tag != command.CompiledBackendPostgreSQL { + t.Fatalf("tag = %q, want POSTGRESQL", recorder.calls[0].tag) + } +} + +func TestPostgresCategoryRefusalsZeroTouchLivePG18(t *testing.T) { + s := newExec06csService(t) + recorder := &exec06csRecorder{s: s} + s.migrate(t) + s.seed(t, []exec04pvFixtureRow{ + {"P1", "gen-a", "Observation", "o-final", "FHIR/Observation", `{"status":"final","valueInteger":1}`}, + }) + engine := s.engine(t, recorder) + bindings := recipe.RuntimeBindings{Project: "P1", DatasetGeneration: "gen-a", AuthResourcePaths: []string{"FHIR/Observation"}} + resolved, err := engine.CompileResolvedBundle(s.ctx, exec06ConstructionBundle(), bindings) + if err != nil { + t.Fatalf("CompileResolvedBundle (construction): %s", s.scrub(err)) + } + cases := []struct { + name string + request dataframeexecution.CategoryScanRequest + check func(*testing.T, error) + }{ + {"composed stage keeps its typed capability gap", + dataframeexecution.CategoryScanRequest{Output: "trace", StageID: "derive_total", ColumnID: "status_id", ValueColumnID: "amount_id", MaxValues: 8}, + func(t *testing.T, err error) { + if !errors.Is(err, pgrender.ErrUnsupportedPostgresPlan) { + t.Fatalf("composed-stage refusal = %v, want errors.Is(ErrUnsupportedPostgresPlan)", err) + } + }}, + {"unknown stage refuses at the compiler entry", + dataframeexecution.CategoryScanRequest{Output: "trace", StageID: "nope_stage", ColumnID: "status_id", ValueColumnID: "amount_id", MaxValues: 8}, + func(t *testing.T, err error) { + code, ok := compiler.CategoryScanRefusalCodeOf(err) + if !ok || code != compiler.CategoryScanStageUnknown { + t.Fatalf("unknown-stage refusal = %v, want typed CATEGORY_STAGE_UNKNOWN", err) + } + }}, + {"unknown stage column refuses at the compiler entry", + dataframeexecution.CategoryScanRequest{Output: "trace", StageID: recipe.ConstructionSourceProjectionID, ColumnID: "nope_id", ValueColumnID: "amount_id", MaxValues: 8}, + func(t *testing.T, err error) { + code, ok := compiler.CategoryScanRefusalCodeOf(err) + if !ok || code != compiler.CategoryScanColumnUnknown { + t.Fatalf("unknown-column refusal = %v, want typed CATEGORY_COLUMN_UNKNOWN", err) + } + }}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + before := recorder.counts() + result, err := engine.ScanCategories(s.ctx, resolved, test.request) + if err == nil { + t.Fatalf("refusal expected, got result %#v", result) + } + test.check(t, err) + if result.Complete || result.Overflow || result.ConclusiveMissing || len(result.Values) != 0 { + t.Fatalf("refusal softened into a result page: %#v", result) + } + if recorder.counts() != before { + t.Fatalf("refusal executed %d statements, want zero before the port", recorder.counts()-before) + } + }) + } + result, err := engine.ScanCategories(s.ctx, resolved, dataframeexecution.CategoryScanRequest{ + Output: "trace", StageID: recipe.ConstructionSourceProjectionID, + ColumnID: "status_id", ValueColumnID: "amount_id", MaxValues: 8, + }) + if err != nil { + t.Fatalf("ALIVE control stage scan: %s", s.scrub(err)) + } + if !result.Complete || len(result.Values) != 1 || result.Values[0] != (dataframeexecution.CategoryValue{Present: true, Value: "final"}) { + t.Fatalf("ALIVE control result = %#v", result) + } +} + +func TestZZPostgresCategoryZeroLeftoverFixtures(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 category scan acceptance tests") + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + scrub, _ := exec04pvScrubberFor(raw) + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for audit: %s", scrub(err)) + } + defer admin.Close() + owned := exec06csRegistrySnapshot() + for db := range owned { + var count int + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Fatalf("audit %s: %s", db, scrub(err)) + } + if count != 0 { + t.Errorf("owned fixture database %s survived", db) + } + } + baseline := map[string]bool{} + for _, name := range exec06csBaselineNames { + baseline[name] = true + } + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec06\\_cs\\_%' ESCAPE '\\'") + if err != nil { + t.Fatalf("audit sweep: %s", scrub(err)) + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("audit scan: %s", scrub(err)) + } + if !baseline[name] { + t.Errorf("unregistered survivor under owned prefix: %s", name) + } + } + if err := rows.Err(); err != nil { + t.Fatalf("audit rows: %s", scrub(err)) + } + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatalf("audit timed out") + } +} diff --git a/internal/dataframe/execution/engine_postgres_cell_trace_pg18_test.go b/internal/dataframe/execution/engine_postgres_cell_trace_pg18_test.go new file mode 100644 index 000000000..3f5f1aafe --- /dev/null +++ b/internal/dataframe/execution/engine_postgres_cell_trace_pg18_test.go @@ -0,0 +1,1431 @@ +package execution_test + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/authscope" + postgres "github.com/calypr/loom/internal/backends/postgres" + storepg "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" + "github.com/calypr/loom/internal/explorer" + "github.com/calypr/loom/internal/explorer/capability" + "github.com/calypr/loom/internal/explorer/lifecycle" + "github.com/calypr/loom/internal/projectid" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + exec09ctPrefix = "verify_exec09_ct_" + exec09ctNameFmt = exec09ctPrefix + "%d_%s" + + exec09ctProject = "project-a" + exec09ctGeneration = "gen-a" + exec09ctAuthPath = "FHIR/Patient" + exec09ctExplorer = "patients" +) + +var exec09ctNamePattern = regexp.MustCompile(`^verify_exec09_ct_[0-9]+_[0-9a-f]+$`) + +var ( + exec09ctOwnedMu sync.Mutex + exec09ctOwnedByName = map[string]string{} +) + +func exec09ctRegister(db, schema string) { + exec09ctOwnedMu.Lock() + defer exec09ctOwnedMu.Unlock() + exec09ctOwnedByName[db] = schema +} + +func exec09ctRegistrySnapshot() map[string]string { + exec09ctOwnedMu.Lock() + defer exec09ctOwnedMu.Unlock() + out := make(map[string]string, len(exec09ctOwnedByName)) + for db, schema := range exec09ctOwnedByName { + out[db] = schema + } + return out +} + +var ( + exec09ctBaselineOnce sync.Once + exec09ctBaselineNames []string + exec09ctBaselineErr error +) + +func exec09ctCaptureBaseline(raw string) { + exec09ctBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec09ctBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + rows, err := admin.Query(ctx, `SELECT datname FROM pg_database WHERE datname LIKE 'verify\_exec09\_ct\_%' ESCAPE '\' UNION ALL SELECT nspname FROM pg_namespace WHERE nspname LIKE 'verify\_exec09\_ct\_%' ESCAPE '\'`) + if err != nil { + exec09ctBaselineErr = fmt.Errorf("read baseline: %w", err) + return + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + exec09ctBaselineErr = fmt.Errorf("scan baseline: %w", err) + return + } + exec09ctBaselineNames = append(exec09ctBaselineNames, name) + } + exec09ctBaselineErr = rows.Err() + }) +} + +func exec09ctRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +type exec09ctService struct { + ctx context.Context + iso *pgxpool.Pool + schema string + target compiler.Target + scrub func(any) string + fail func(step string, err error) + adapter dataframeexecution.SourceRows +} + +func newExec09ctService(t *testing.T) *exec09ctService { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 cell-trace acceptance tests") + } + scrub, addSecret := exec04pvScrubberFor(raw) + exec09ctCaptureBaseline(raw) + if exec09ctBaselineErr != nil { + t.Fatalf("capture pg_database baseline: %s", scrub(exec09ctBaselineErr)) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, "SELECT current_setting('server_version_num')::int8, current_setting('server_version')").Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if err := storepg.CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required PostgreSQL 18 backend: %v", err) + } + + dbName := fmt.Sprintf(exec09ctNameFmt, time.Now().UnixNano(), exec09ctRandomHex(t)) + if !exec09ctNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + var created int64 + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", dbName).Scan(&created); err != nil { + fail("verify isolated database creation", err) + } + if created != 1 { + t.Fatalf("isolated database %q did not verifiably come into existence", dbName) + } + exec09ctRegister(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + addSecret(dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(exec09ctNameFmt, time.Now().UnixNano(), exec09ctRandomHex(t)) + if !exec09ctNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + exec09ctRegister(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + client, err := storepg.OpenPool(ctx, storepg.ClientConfig{DSN: dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 2 * time.Second}) + if err != nil { + fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + defer client.Close() + if _, err := storepg.MigrateSchema(ctx, client, schema); err != nil { + fail("migrate owned schema", err) + } + + binding, err := ir.NewPostgresStorageBinding(schema) + if err != nil { + fail("bind PostgreSQL storage placement", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + fail("select PostgreSQL compile target", err) + } + adapter, err := postgres.NewSourceRows(iso) + if err != nil { + fail("create PostgreSQL row adapter", err) + } + return &exec09ctService{ctx: ctx, iso: iso, schema: schema, target: target, scrub: scrub, fail: fail, adapter: adapter} +} + +func exec09ctPatientsBundle(authoredIdentity bool) recipe.Bundle { + output := recipe.Output{ + Name: exec09ctExplorer, RootResourceType: "Patient", RowGrain: "patient", + RootColumnNaming: recipe.RootColumnNamingExact, + Fields: []recipe.Field{ + {Name: "patient_id", Expr: recipe.Expression{Select: "root.id"}}, + {Name: "gender", Expr: recipe.Expression{Select: "root.gender"}}, + }, + } + if authoredIdentity { + output.Identity = &recipe.Identity{Name: "row", Expr: recipe.Expression{Select: "root.id"}} + } + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "exec09_cell_trace", + TranslationVersion: "exec09", + Outputs: []recipe.Output{output}, + } +} + +func exec09ctArmABundle(all bool) recipe.Bundle { + mode := recipe.ValueModeFirst + if all { + mode = recipe.ValueModeAll + } + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "exec09_arm_a", + TranslationVersion: "exec09", + Outputs: []recipe.Output{{ + Name: exec09ctExplorer, RootResourceType: "Patient", RowGrain: "patient", + RootColumnNaming: recipe.RootColumnNamingExact, TraversalColumnNaming: recipe.TraversalColumnNamingExact, + Fields: []recipe.Field{{Name: "patient_id", Expr: recipe.Expression{Select: "root.id"}}}, + Traversals: []recipe.Traversal{{ + Name: "subject_Patient", ToResourceType: "Condition", Alias: "condition", MatchMode: recipe.MatchOptional, + Fields: []recipe.Field{{Name: "condition_id", Expr: recipe.Expression{Select: "id"}, ValueMode: mode}}, + }}, + }}, + } +} + +func exec09ctBindings(project, generation string) recipe.RuntimeBindings { + return recipe.RuntimeBindings{ + Project: project, DatasetGeneration: generation, + AuthResourcePaths: []string{exec09ctAuthPath}, + } +} + +type exec09ctSeedRow struct { + project, generation, logicalID, authPath string + gender string +} + +func (s *exec09ctService) seed(t *testing.T, rows []exec09ctSeedRow) { + t.Helper() + repo, err := storepg.NewResourceRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + for _, row := range rows { + payload := `{"id":` + quoteExec09(row.logicalID) + `}` + if row.gender != "" { + payload = strings.TrimSuffix(payload, "}") + `,"gender":` + row.gender + `}` + } + if _, err := repo.InsertResource(s.ctx, source.Resource{ + Project: row.project, + DatasetGeneration: row.generation, + ResourceType: "Patient", + LogicalID: row.logicalID, + AuthResourcePath: row.authPath, + Payload: json.RawMessage(payload), + }); err != nil { + s.fail("insert fixture patient "+row.logicalID, err) + } + } +} + +func quoteExec09(value string) string { + encoded, err := json.Marshal(value) + if err != nil { + panic(err) + } + return string(encoded) +} + +func (s *exec09ctService) seedEdges(t *testing.T, patient string, conditions []string) { + t.Helper() + resources, err := storepg.NewResourceRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + edges, err := storepg.NewEdgeRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("edge repository: %v", err) + } + for index, condition := range conditions { + if _, err := resources.InsertResource(s.ctx, source.Resource{ + Project: exec09ctProject, DatasetGeneration: exec09ctGeneration, + ResourceType: "Condition", LogicalID: condition, AuthResourcePath: exec09ctAuthPath, + Payload: json.RawMessage(`{"id":` + quoteExec09(condition) + `}`), + }); err != nil { + s.fail("insert fixture condition "+condition, err) + } + if _, err := edges.InsertEdge(s.ctx, storepg.EdgeOccurrenceRow{ + Project: exec09ctProject, DatasetGeneration: exec09ctGeneration, AuthResourcePath: exec09ctAuthPath, + Label: "subject_Patient", SourceResourceType: "Condition", SourceLogicalID: condition, + TargetResourceType: "Patient", TargetLogicalID: patient, SourcePath: "subject", SourceOrdinal: int64(index), + }); err != nil { + s.fail("insert fixture edge "+condition+"->"+patient, err) + } + } +} + +type exec09ctCall struct { + sql string + args []command.PostgreSQLParameter + declared []string + tag command.CompiledBackend +} + +type exec09ctRecorder struct { + adapter dataframeexecution.SourceRows + mu sync.Mutex + calls []exec09ctCall + rows []map[string]any + capture bool +} + +func (r *exec09ctRecorder) port(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + sqlText, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + r.mu.Lock() + r.calls = append(r.calls, exec09ctCall{sql: sqlText, args: args, declared: append([]string(nil), request.DeclaredColumns...), tag: request.Command.Backend()}) + r.mu.Unlock() + if !r.capture { + return r.adapter(ctx, request, visit) + } + return r.adapter(ctx, request, func(row map[string]any) error { + captured := make(map[string]any, len(row)) + for key, value := range row { + captured[key] = value + } + r.mu.Lock() + r.rows = append(r.rows, captured) + r.mu.Unlock() + return visit(row) + }) +} + +func (r *exec09ctRecorder) count() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.calls) +} + +func (r *exec09ctRecorder) last() exec09ctCall { + r.mu.Lock() + defer r.mu.Unlock() + return r.calls[len(r.calls)-1] +} + +func (r *exec09ctRecorder) capturedRows() []map[string]any { + r.mu.Lock() + defer r.mu.Unlock() + out := make([]map[string]any, len(r.rows)) + copy(out, r.rows) + return out +} + +func wantCellTraceContract(t *testing.T, declared []string, identityColumn string) { + t.Helper() + want := []string{ + ir.PhysicalCellTraceValueField, ir.PhysicalCellTraceContributionsField, identityColumn, + ir.PhysicalCellTraceStatusField, ir.PhysicalCellTraceHasMoreField, ir.PhysicalCellTraceOmissionField, + } + if strings.Join(declared, ",") != strings.Join(want, ",") { + t.Fatalf("declared contract = %v, want %v", declared, want) + } +} + +func (s *exec09ctService) engine(t *testing.T, recorder *exec09ctRecorder) *dataframeexecution.Engine { + t.Helper() + config := exec04pvForbidArango() + config.CompileTarget = s.target + config.PreviewSourceRows = nil + config.ScopeDigest = func(recipe.RuntimeBindings) string { return "exec09-scope" } + config.SourceRows = recorder.port + engine, err := dataframeexecution.New(config) + if err != nil { + t.Fatalf("engine: %v", s.scrub(err)) + } + return engine +} + +func (s *exec09ctService) resolve(t *testing.T, engine *dataframeexecution.Engine, project, generation string, bundle recipe.Bundle) dataframeexecution.Resolved { + t.Helper() + resolved, err := engine.CompileResolvedBundle(s.ctx, bundle, exec09ctBindings(project, generation)) + if err != nil { + t.Fatalf("CompileResolvedBundle(%s/%s): %s", project, generation, s.scrub(err)) + } + return resolved +} + +func (s *exec09ctService) previewIdentity(t *testing.T, engine *dataframeexecution.Engine, resolved dataframeexecution.Resolved, wantID string) string { + t.Helper() + var identity string + matched := 0 + _, err := engine.PreviewOutput(s.ctx, resolved, dataframeexecution.PreviewRequest{Output: exec09ctExplorer, Limit: 50, IncludeRowIdentity: true}, func(row map[string]any) error { + if published, _ := row["patient_id"].(string); published == wantID { + matched++ + cell, ok := row["__loom_row_id"].(string) + if !ok || cell == "" { + t.Fatalf("preview row did not publish its stable identity: %#v", row) + } + identity = cell + } + return nil + }) + if err != nil { + t.Fatalf("preview: %s", s.scrub(err)) + } + if matched != 1 { + t.Fatalf("preview published %d rows for patient_id %q, want exactly one", matched, wantID) + } + return identity +} + +func exec09ctIdentityDigestOf(t *testing.T, cell any) string { + t.Helper() + var parts []any + switch typed := cell.(type) { + case []any: + parts = typed + case json.RawMessage: + if err := json.Unmarshal(typed, &parts); err != nil { + t.Fatalf("published identity parts %q do not decode: %v", string(typed), err) + } + case []byte: + if err := json.Unmarshal(typed, &parts); err != nil { + t.Fatalf("published identity parts %q do not decode: %v", string(typed), err) + } + default: + t.Fatalf("published identity parts have an unexpected shape %T", cell) + } + encoded, err := json.Marshal(parts) + if err != nil { + t.Fatalf("encode identity parts: %v", err) + } + digest := sha256.Sum256(encoded) + return hex.EncodeToString(digest[:]) +} + +func exec09ctRowIdentity(t *testing.T, rows []map[string]any, wantID string) string { + t.Helper() + for _, row := range rows { + cell, ok := row[ir.PhysicalCellTraceIdentityPartsField] + if !ok { + t.Fatalf("captured trace row carries no identity parts column: %#v", row) + } + var parts []string + switch typed := cell.(type) { + case []any: + for _, part := range typed { + if text, ok := part.(string); ok { + parts = append(parts, text) + } + } + case json.RawMessage: + _ = json.Unmarshal(typed, &parts) + case []byte: + _ = json.Unmarshal(typed, &parts) + } + if len(parts) > 0 && parts[len(parts)-1] == wantID { + return exec09ctIdentityDigestOf(t, cell) + } + } + t.Fatalf("no captured trace row carries identity parts ending in %q", wantID) + return "" +} + +func exec09ctCellTraceTriples(result dataframeexecution.CellTraceResult) []contributorTriple { + out := make([]contributorTriple, 0, len(result.Contributions)) + for _, contributor := range result.Contributions { + out = append(out, contributorTriple{contributor.ResourceType, contributor.ResourceID, fmt.Sprint(contributor.Value)}) + } + return out +} + +func TestPostgresCellTracePreviewRoundTripServesEvidenceThroughThePort(t *testing.T) { + s := newExec09ctService(t) + s.seed(t, []exec09ctSeedRow{ + {exec09ctProject, exec09ctGeneration, "p-1", exec09ctAuthPath, `"male"`}, + {exec09ctProject, exec09ctGeneration, "p-2", exec09ctAuthPath, `"female"`}, + }) + recorder := &exec09ctRecorder{adapter: s.adapter} + engine := s.engine(t, recorder) + resolved := s.resolve(t, engine, exec09ctProject, exec09ctGeneration, exec09ctPatientsBundle(true)) + + previewGender := map[string]string{} + identityByLogical := map[string]string{} + _, err := engine.PreviewOutput(s.ctx, resolved, dataframeexecution.PreviewRequest{Output: exec09ctExplorer, Limit: 50, IncludeRowIdentity: true}, func(row map[string]any) error { + id, _ := row["patient_id"].(string) + gender, _ := row["gender"].(string) + cell, _ := row["__loom_row_id"].(string) + if id == "" || cell == "" { + t.Fatalf("preview row lacks a published id or identity: %#v", row) + } + previewGender[id] = gender + identityByLogical[id] = cell + return nil + }) + if err != nil || len(previewGender) != 2 { + t.Fatalf("preview must publish both scoped patients: %v (%s)", previewGender, s.scrub(err)) + } + + for _, wantID := range []string{"p-1", "p-2"} { + before := recorder.count() + result, err := engine.CellTrace(s.ctx, resolved, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: identityByLogical[wantID], Column: "gender", Offset: 0, Limit: 25, + }) + if err != nil { + t.Fatalf("CellTrace %s: %s", wantID, s.scrub(err)) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("trace for %s ran %d statements, want exactly one", wantID, got) + } + call := recorder.last() + if call.tag != command.CompiledBackendPostgreSQL { + t.Fatalf("trace command tag = %q", call.tag) + } + identityColumn := ir.PhysicalCellTraceIdentityPartsField + if _, hasExplicit := map[string]bool{}[call.declared[2]]; !hasExplicit { + switch call.declared[2] { + case ir.PhysicalCellTraceIdentityPartsField, ir.PhysicalCellTraceExplicitIdentityField: + identityColumn = call.declared[2] + default: + t.Fatalf("third contract column = %q, want an identity spelling", call.declared[2]) + } + } + wantCellTraceContract(t, call.declared, identityColumn) + if len(call.args) == 0 { + t.Fatal("every scope and page fact must travel as an ordered parameter") + } + for _, leaked := range []string{exec09ctProject, exec09ctGeneration, exec09ctAuthPath, "male", "female", identityByLogical[wantID]} { + if strings.Contains(call.sql, leaked) { + t.Fatalf("value %q leaked into statement text: %s", leaked, call.sql) + } + } + boundScope := 0 + for _, arg := range call.args { + if text, ok := arg.Value.(string); ok { + switch text { + case exec09ctProject, exec09ctGeneration: + boundScope++ + } + } + } + if boundScope != 2 { + t.Fatalf("project and generation must arrive as ordered typed parameters: %d of 2 among %d args", boundScope, len(call.args)) + } + for _, arg := range call.args { + if text, ok := arg.Value.(string); ok && text == identityByLogical[wantID] { + t.Fatalf("the requested identity must match scan-side, not arrive as a bound parameter: %s", text) + } + } + if result.Status != dataframeexecution.CellTraceValue || result.Value != previewGender[wantID] { + t.Fatalf("traced value %#v/%s drifted from the published preview value %#v", result.Value, result.Status, previewGender[wantID]) + } + if !result.Complete || result.HasMore || result.NextOffset != 0 || result.OmissionCode != "" { + t.Fatalf("page summary = %#v, want a complete offset-free page", result) + } + want := []contributorTriple{{"Patient", wantID, previewGender[wantID]}} + if got := exec09ctCellTraceTriples(result); len(got) != 1 || got[0] != want[0] { + t.Fatalf("%s contributors = %v, want the traced row itself %v", wantID, got, want) + } + again, err := engine.CellTrace(s.ctx, resolved, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: identityByLogical[wantID], Column: "gender", Offset: 0, Limit: 25, + }) + if err != nil || again.Status != dataframeexecution.CellTraceValue || again.Value != previewGender[wantID] { + t.Fatalf("page drifted between executions: %#v (%s)", again, s.scrub(err)) + } + } +} + +func TestPostgresCellTraceNullValueStatesStayDistinctFromAValue(t *testing.T) { + s := newExec09ctService(t) + s.seed(t, []exec09ctSeedRow{ + {exec09ctProject, exec09ctGeneration, "p-1", exec09ctAuthPath, `"male"`}, + {exec09ctProject, exec09ctGeneration, "p-2", exec09ctAuthPath, "null"}, + {exec09ctProject, exec09ctGeneration, "p-3", exec09ctAuthPath, ""}, + }) + recorder := &exec09ctRecorder{adapter: s.adapter} + engine := s.engine(t, recorder) + resolved := s.resolve(t, engine, exec09ctProject, exec09ctGeneration, exec09ctPatientsBundle(true)) + + for _, wantID := range []string{"p-1", "p-2", "p-3"} { + identity := s.previewIdentity(t, engine, resolved, wantID) + before := recorder.count() + result, err := engine.CellTrace(s.ctx, resolved, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: identity, Column: "gender", Offset: 0, Limit: 25, + }) + if err != nil { + t.Fatalf("CellTrace %s: %s", wantID, s.scrub(err)) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("%s ran %d statements, want exactly one", wantID, got) + } + switch wantID { + case "p-1": + if result.Status != dataframeexecution.CellTraceValue || result.Value != "male" || len(result.Contributions) != 1 { + t.Fatalf("value state = %#v, want VALUE \"male\" with the self contributor", result) + } + case "p-2", "p-3": + if result.Status != dataframeexecution.CellTraceNoMatch { + t.Fatalf("%s status = %s (%#v), want NO_MATCH — the registered arm drops the null candidate before the status reads", wantID, result.Status, result.Value) + } + if result.Value != nil || len(result.Contributions) != 0 || result.HasMore { + t.Fatalf("%s must publish a nil value and an empty page: %#v", wantID, result) + } + } + } +} + +func TestPostgresCellTraceOffsetGateSkipsTheSoleCandidateWithoutChangingStatus(t *testing.T) { + s := newExec09ctService(t) + s.seed(t, []exec09ctSeedRow{{exec09ctProject, exec09ctGeneration, "p-1", exec09ctAuthPath, `"male"`}}) + recorder := &exec09ctRecorder{adapter: s.adapter} + engine := s.engine(t, recorder) + resolved := s.resolve(t, engine, exec09ctProject, exec09ctGeneration, exec09ctPatientsBundle(true)) + identity := s.previewIdentity(t, engine, resolved, "p-1") + + for _, offset := range []int{1, 2} { + before := recorder.count() + result, err := engine.CellTrace(s.ctx, resolved, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: identity, Column: "gender", Offset: offset, Limit: 25, + }) + if err != nil { + t.Fatalf("CellTrace offset %d: %s", offset, s.scrub(err)) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("offset %d ran %d statements, want exactly one", offset, got) + } + if result.Status != dataframeexecution.CellTraceValue || result.Value != "male" { + t.Fatalf("offset %d: status/value must stay offset-free: %#v", offset, result) + } + if len(result.Contributions) != 0 || result.HasMore || !result.Complete { + t.Fatalf("offset %d must publish the empty page: %#v", offset, result) + } + } + before := recorder.count() + control, err := engine.CellTrace(s.ctx, resolved, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: identity, Column: "gender", Offset: 0, Limit: 25, + }) + if err != nil { + t.Fatalf("offset-0 control: %s", s.scrub(err)) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("control ran %d statements, want exactly one", got) + } + if want := []contributorTriple{{"Patient", "p-1", "male"}}; !equalExec09Triples(exec09ctCellTraceTriples(control), want) { + t.Fatalf("offset-0 control must publish the self contributor (the gate is not always-empty): %v", exec09ctCellTraceTriples(control)) + } +} + +func equalExec09Triples(got, want []contributorTriple) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true +} + +func TestPostgresCellTraceAuthoredIdentityResolvesThroughTheExplicitIdentityColumn(t *testing.T) { + s := newExec09ctService(t) + s.seed(t, []exec09ctSeedRow{ + {exec09ctProject, exec09ctGeneration, "p-1", exec09ctAuthPath, `"male"`}, + {exec09ctProject, exec09ctGeneration, "p-2", exec09ctAuthPath, `"female"`}, + }) + recorder := &exec09ctRecorder{adapter: s.adapter} + engine := s.engine(t, recorder) + resolved := s.resolve(t, engine, exec09ctProject, exec09ctGeneration, exec09ctPatientsBundle(true)) + identity := s.previewIdentity(t, engine, resolved, "p-1") + + before := recorder.count() + result, err := engine.CellTrace(s.ctx, resolved, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: identity, Column: "gender", Offset: 0, Limit: 25, + }) + if err != nil { + t.Fatalf("CellTrace on the authored-Identity output: %s", s.scrub(err)) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("authored-identity trace ran %d statements, want exactly one", got) + } + call := recorder.last() + wantCellTraceContract(t, call.declared, ir.PhysicalCellTraceExplicitIdentityField) + if result.Status != dataframeexecution.CellTraceValue || result.Value != "male" { + t.Fatalf("authored-identity evidence = %#v, want VALUE \"male\"", result) + } + if want := []contributorTriple{{"Patient", "p-1", "male"}}; !equalExec09Triples(exec09ctCellTraceTriples(result), want) { + t.Fatalf("authored-identity contributors = %v, want %v", exec09ctCellTraceTriples(result), want) + } + other := s.previewIdentity(t, engine, resolved, "p-2") + female, err := engine.CellTrace(s.ctx, resolved, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: other, Column: "gender", Offset: 0, Limit: 25, + }) + if err != nil || female.Status != dataframeexecution.CellTraceValue || female.Value != "female" { + t.Fatalf("authored-identity control = %#v (%s)", female, s.scrub(err)) + } +} + +func TestPostgresCellTraceForgedIdentityIsTypedNotFoundAndRealIdentityResolves(t *testing.T) { + s := newExec09ctService(t) + s.seed(t, []exec09ctSeedRow{{exec09ctProject, exec09ctGeneration, "p-1", exec09ctAuthPath, `"male"`}}) + recorder := &exec09ctRecorder{adapter: s.adapter} + engine := s.engine(t, recorder) + resolved := s.resolve(t, engine, exec09ctProject, exec09ctGeneration, exec09ctPatientsBundle(true)) + + forgedDigest := sha256.Sum256([]byte(`["nobody"]`)) + forged := hex.EncodeToString(forgedDigest[:]) + before := recorder.count() + _, err := engine.CellTrace(s.ctx, resolved, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: forged, Column: "gender", Offset: 0, Limit: 25, + }) + if !errors.Is(err, dataframeexecution.ErrCellTraceRowNotFound) { + t.Fatalf("forged identity = %v, want the typed not-found", s.scrub(err)) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("forged-identity trace ran %d statements, want exactly one", got) + } + identity := s.previewIdentity(t, engine, resolved, "p-1") + result, err := engine.CellTrace(s.ctx, resolved, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: identity, Column: "gender", Offset: 0, Limit: 25, + }) + if err != nil || result.Status != dataframeexecution.CellTraceValue || result.Value != "male" { + t.Fatalf("real identity failed on the same fixture: %#v (%s)", result, s.scrub(err)) + } +} + +func TestPostgresCellTraceWitnessCapIsIncompleteNotAbsence(t *testing.T) { + s := newExec09ctService(t) + s.seed(t, []exec09ctSeedRow{ + {exec09ctProject, exec09ctGeneration, "p-1", exec09ctAuthPath, `"male"`}, + {exec09ctProject, exec09ctGeneration, "p-2", exec09ctAuthPath, `"female"`}, + }) + probe := &exec09ctRecorder{adapter: s.adapter, capture: true} + engine := s.engine(t, probe) + resolved := s.resolve(t, engine, exec09ctProject, exec09ctGeneration, exec09ctPatientsBundle(false)) + + forgedDigest := sha256.Sum256([]byte(`["nobody"]`)) + if _, err := engine.CellTrace(s.ctx, resolved, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: hex.EncodeToString(forgedDigest[:]), Column: "gender", Offset: 0, Limit: 25, + }); !errors.Is(err, dataframeexecution.ErrCellTraceRowNotFound) { + t.Fatalf("uncapped probe = %v, want the typed not-found", s.scrub(err)) + } + rows := probe.capturedRows() + if len(rows) != 2 { + t.Fatalf("probe captured %d rows, want the two scoped patients", len(rows)) + } + first := exec09ctRowIdentity(t, rows, "p-1") + second := exec09ctRowIdentity(t, rows, "p-2") + target, firstID := second, first + if digest := exec09ctIdentityDigestOf(t, rows[0][ir.PhysicalCellTraceIdentityPartsField]); digest == second { + target = first + firstID = second + } + if digest := exec09ctIdentityDigestOf(t, rows[0][ir.PhysicalCellTraceIdentityPartsField]); digest == target { + t.Fatalf("aiming witness failed: the first visited row is the target") + } + beforeCapped := probe.count() + capped, err := engine.CellTrace(s.ctx, resolved, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: target, Column: "gender", Offset: 0, Limit: 25, MaxWitnessRows: 1, + }) + if err != nil { + t.Fatalf("capped trace: %s", s.scrub(err)) + } + if capped.Status != dataframeexecution.CellTraceIncomplete || capped.OmissionCode != "TRACE_WITNESS_INCOMPLETE" || capped.Complete { + t.Fatalf("capped trace = %#v, want INCOMPLETE/TRACE_WITNESS_INCOMPLETE, never absence", capped) + } + if got := probe.count() - beforeCapped; got != 1 { + t.Fatalf("capped trace ran %d statements, want exactly one", got) + } + uncapped, err := engine.CellTrace(s.ctx, resolved, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: target, Column: "gender", Offset: 0, Limit: 25, + }) + if err != nil || uncapped.Status != dataframeexecution.CellTraceValue { + t.Fatalf("uncapped control = %#v (%s), want the row resolved", uncapped, s.scrub(err)) + } + firstUncapped, err := engine.CellTrace(s.ctx, resolved, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: firstID, Column: "gender", Offset: 0, Limit: 25, + }) + if err != nil || firstUncapped.Status != dataframeexecution.CellTraceValue { + t.Fatalf("first-visited control = %#v (%s), want the row resolved", firstUncapped, s.scrub(err)) + } +} + +func TestPostgresCellTraceForeignWitnessesAreExcluded(t *testing.T) { + s := newExec09ctService(t) + s.seed(t, []exec09ctSeedRow{ + {exec09ctProject, exec09ctGeneration, "p-1", exec09ctAuthPath, `"female"`}, + {exec09ctProject, exec09ctGeneration, "p-2", exec09ctAuthPath, `"female"`}, + {"project-b", exec09ctGeneration, "p-foreign-project", exec09ctAuthPath, `"female"`}, + {exec09ctProject, "gen-b", "p-foreign-generation", exec09ctAuthPath, `"female"`}, + {exec09ctProject, exec09ctGeneration, "p-foreign-auth", "FHIR/Restricted", `"female"`}, + }) + recorder := &exec09ctRecorder{adapter: s.adapter} + engine := s.engine(t, recorder) + resolved := s.resolve(t, engine, exec09ctProject, exec09ctGeneration, exec09ctPatientsBundle(true)) + identity := s.previewIdentity(t, engine, resolved, "p-1") + + before := recorder.count() + result, err := engine.CellTrace(s.ctx, resolved, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: identity, Column: "gender", Offset: 0, Limit: 25, + }) + if err != nil { + t.Fatalf("CellTrace: %s", s.scrub(err)) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("trace ran %d statements, want exactly one", got) + } + if result.Status != dataframeexecution.CellTraceValue || result.Value != "female" { + t.Fatalf("in-scope trace = %#v", result) + } + if want := []contributorTriple{{"Patient", "p-1", "female"}}; !equalExec09Triples(exec09ctCellTraceTriples(result), want) { + t.Fatalf("foreign witnesses leaked: contributors = %v, want %v", exec09ctCellTraceTriples(result), want) + } + if result.HasMore { + t.Fatalf("foreign witnesses inflated hasMore: %#v", result) + } + + foreignDigest := sha256.Sum256([]byte(`["p-foreign-project"]`)) + if _, err := engine.CellTrace(s.ctx, resolved, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: hex.EncodeToString(foreignDigest[:]), Column: "gender", Offset: 0, Limit: 25, + }); !errors.Is(err, dataframeexecution.ErrCellTraceRowNotFound) { + t.Fatalf("foreign identity = %v, want the typed not-found", s.scrub(err)) + } + + seen := 0 + _, err = engine.PreviewOutput(s.ctx, resolved, dataframeexecution.PreviewRequest{Output: exec09ctExplorer, Limit: 50}, func(row map[string]any) error { + seen++ + id, _ := row["patient_id"].(string) + if id != "p-1" && id != "p-2" { + t.Fatalf("out-of-scope witness row reached the preview: %#v", row) + } + return nil + }) + if err != nil || seen != 2 { + t.Fatalf("in-scope preview affected by foreign witnesses: rows=%d err=%s", seen, s.scrub(err)) + } +} + +func exec09ctLowerArmA(t *testing.T, s *exec09ctService, all bool) lower.CompiledRecipeOutput { + t.Helper() + bundle := exec09ctArmABundle(all) + if err := bundle.Validate(); err != nil { + t.Fatalf("bundle validate: %v", err) + } + plan, err := semantic.BuildRecipePlan(bundle, exec09ctBindings(exec09ctProject, exec09ctGeneration)) + if err != nil { + t.Fatalf("BuildRecipePlan: %v", err) + } + resolved, err := semantic.ResolveRecipePlan(plan, exec09ctProject, exec09ctGeneration) + if err != nil { + t.Fatalf("ResolveRecipePlan: %v", err) + } + compiled, err := lower.CompileResolvedRecipePlan(resolved, ir.DefaultPhysicalOptimizationPolicy(). + WithRule(ir.PhysicalOptimizationRuleEndpointTraversal, false)) + if err != nil { + t.Fatalf("lower arm A: %s", s.scrub(err)) + } + for _, candidate := range compiled.Outputs { + if candidate.Name == exec09ctExplorer { + return candidate + } + } + t.Fatalf("lowered outputs do not carry %q", exec09ctExplorer) + return lower.CompiledRecipeOutput{} +} + +func TestPostgresCellTraceArmAReducedSetPagesThroughThePublicSeam(t *testing.T) { + s := newExec09ctService(t) + s.seed(t, []exec09ctSeedRow{ + {exec09ctProject, exec09ctGeneration, "p-1", exec09ctAuthPath, ""}, + {exec09ctProject, exec09ctGeneration, "p-none", exec09ctAuthPath, ""}, + }) + s.seedEdges(t, "p-1", []string{"c-1", "c-2", "c-3"}) + + recorder := &exec09ctRecorder{adapter: s.adapter, capture: true} + engine := s.engine(t, recorder) + policy := ir.DefaultPhysicalOptimizationPolicy().WithRule(ir.PhysicalOptimizationRuleEndpointTraversal, false) + output := exec09ctLowerArmA(t, s, true) + + compiled, err := compiler.CompileCellTraceOutput(s.target, output, "condition_id", 0, 2, policy) + if err != nil { + t.Fatalf("CompileCellTraceOutput (ALL mode): %s", s.scrub(err)) + } + if compiled.ValueColumn != ir.PhysicalCellTraceValueField || compiled.StatusColumn != ir.PhysicalCellTraceStatusField || + compiled.ContributionsColumn != ir.PhysicalCellTraceContributionsField || compiled.HasMoreColumn != ir.PhysicalCellTraceHasMoreField || + compiled.OmissionColumn != ir.PhysicalCellTraceOmissionField { + t.Fatalf("arm A compiled contract = %#v", compiled) + } + + forgedDigest := sha256.Sum256([]byte(`["nobody"]`)) + if _, err := engine.CellTraceCompiled(s.ctx, compiled, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: hex.EncodeToString(forgedDigest[:]), Column: "condition_id", Offset: 0, Limit: 2, + }); !errors.Is(err, dataframeexecution.ErrCellTraceRowNotFound) { + t.Fatalf("arm A probe = %v, want the typed not-found", s.scrub(err)) + } + rows := recorder.capturedRows() + if len(rows) != 2 { + t.Fatalf("arm A probe captured %d rows, want the two scoped patients", len(rows)) + } + identity := exec09ctRowIdentity(t, rows, "p-1") + noneIdentity := exec09ctRowIdentity(t, rows, "p-none") + + before := recorder.count() + page1, err := engine.CellTraceCompiled(s.ctx, compiled, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: identity, Column: "condition_id", Offset: 0, Limit: 2, + }) + if err != nil { + t.Fatalf("arm A page 1: %s", s.scrub(err)) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("arm A page 1 ran %d statements, want exactly one", got) + } + call := recorder.last() + if call.tag != command.CompiledBackendPostgreSQL { + t.Fatalf("arm A command tag = %q", call.tag) + } + wantCellTraceContract(t, call.declared, ir.PhysicalCellTraceIdentityPartsField) + if page1.Status != dataframeexecution.CellTraceValue { + t.Fatalf("arm A status = %s (%#v), want VALUE", page1.Status, page1) + } + reduced, ok := page1.Value.([]any) + if !ok || len(reduced) != 3 { + t.Fatalf("ALL-mode value column = %#v, want the published three-item reduced array", page1.Value) + } + if want := []contributorTriple{{"Condition", "c-1", "c-1"}, {"Condition", "c-2", "c-2"}}; !equalExec09Triples(exec09ctCellTraceTriples(page1), want) { + t.Fatalf("arm A page 1 = %v, want %v", exec09ctCellTraceTriples(page1), want) + } + if !page1.HasMore || page1.NextOffset != 2 { + t.Fatalf("arm A page 1 summary = %#v, want HasMore with NextOffset 2", page1) + } + + compiled2, err := compiler.CompileCellTraceOutput(s.target, output, "condition_id", 2, 2, policy) + if err != nil { + t.Fatalf("CompileCellTraceOutput offset 2: %s", s.scrub(err)) + } + before = recorder.count() + page2, err := engine.CellTraceCompiled(s.ctx, compiled2, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: identity, Column: "condition_id", Offset: 2, Limit: 2, + }) + if err != nil { + t.Fatalf("arm A page 2: %s", s.scrub(err)) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("arm A page 2 ran %d statements, want exactly one", got) + } + if want := []contributorTriple{{"Condition", "c-3", "c-3"}}; !equalExec09Triples(exec09ctCellTraceTriples(page2), want) { + t.Fatalf("arm A page 2 = %v, want %v", exec09ctCellTraceTriples(page2), want) + } + if page2.HasMore || page2.NextOffset != 0 { + t.Fatalf("arm A page 2 summary = %#v, want the terminal page", page2) + } + + before = recorder.count() + nonePage, err := engine.CellTraceCompiled(s.ctx, compiled, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: noneIdentity, Column: "condition_id", Offset: 0, Limit: 2, + }) + if err != nil { + t.Fatalf("arm A no-contributor row: %s", s.scrub(err)) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("arm A zero-contributor row ran %d statements, want exactly one", got) + } + if nonePage.Status != dataframeexecution.CellTraceNoMatch || len(nonePage.Contributions) != 0 || nonePage.HasMore { + t.Fatalf("arm A zero-contributor row = %#v, want NO_MATCH with an empty page", nonePage) + } +} + +func TestPostgresCellTraceArmALossyFirstReadsAmbiguous(t *testing.T) { + s := newExec09ctService(t) + s.seed(t, []exec09ctSeedRow{ + {exec09ctProject, exec09ctGeneration, "p-1", exec09ctAuthPath, ""}, + {exec09ctProject, exec09ctGeneration, "p-solo", exec09ctAuthPath, ""}, + }) + s.seedEdges(t, "p-1", []string{"c-1", "c-2", "c-3"}) + s.seedEdges(t, "p-solo", []string{"c-9"}) + + recorder := &exec09ctRecorder{adapter: s.adapter, capture: true} + engine := s.engine(t, recorder) + policy := ir.DefaultPhysicalOptimizationPolicy().WithRule(ir.PhysicalOptimizationRuleEndpointTraversal, false) + output := exec09ctLowerArmA(t, s, false) + compiled, err := compiler.CompileCellTraceOutput(s.target, output, "condition_id", 0, 25, policy) + if err != nil { + t.Fatalf("CompileCellTraceOutput (FIRST mode): %s", s.scrub(err)) + } + + forgedDigest := sha256.Sum256([]byte(`["nobody"]`)) + if _, err := engine.CellTraceCompiled(s.ctx, compiled, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: hex.EncodeToString(forgedDigest[:]), Column: "condition_id", Offset: 0, Limit: 25, + }); !errors.Is(err, dataframeexecution.ErrCellTraceRowNotFound) { + t.Fatalf("FIRST probe = %v, want the typed not-found", s.scrub(err)) + } + rows := recorder.capturedRows() + ambiguousID := exec09ctRowIdentity(t, rows, "p-1") + soloID := exec09ctRowIdentity(t, rows, "p-solo") + + before := recorder.count() + ambiguous, err := engine.CellTraceCompiled(s.ctx, compiled, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: ambiguousID, Column: "condition_id", Offset: 0, Limit: 25, + }) + if err != nil { + t.Fatalf("FIRST ambiguous page: %s", s.scrub(err)) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("FIRST ambiguous page ran %d statements, want exactly one", got) + } + if ambiguous.Status != dataframeexecution.CellTraceAmbiguous { + t.Fatalf("FIRST page over three candidates = %s (%#v), want AMBIGUOUS — the compiler cannot order the ties", ambiguous.Status, ambiguous) + } + if len(ambiguous.Contributions) == 0 || len(ambiguous.Contributions) > 25 { + t.Fatalf("AMBIGUOUS page must still serve its bounded contributors: %#v", ambiguous) + } + + solo, err := engine.CellTraceCompiled(s.ctx, compiled, dataframeexecution.CellTraceRequest{ + Output: exec09ctExplorer, RowID: soloID, Column: "condition_id", Offset: 0, Limit: 25, + }) + if err != nil || solo.Status != dataframeexecution.CellTraceValue { + t.Fatalf("solo control = %#v (%s), want VALUE", solo, s.scrub(err)) + } +} + +type exec09ctExplorerStore struct { + explorer.Store +} + +type exec09ctAuthorized struct { + mu sync.Mutex + snapshot capability.Snapshot + scope authscope.ReadScope + hookCalls int +} + +func (a *exec09ctAuthorized) set(snapshot capability.Snapshot, scope authscope.ReadScope) { + a.mu.Lock() + defer a.mu.Unlock() + a.snapshot = snapshot + a.scope = scope +} + +func (a *exec09ctAuthorized) authorized() (capability.Snapshot, authscope.ReadScope) { + a.mu.Lock() + defer a.mu.Unlock() + return a.snapshot, a.scope +} + +func (a *exec09ctAuthorized) calls() int { + a.mu.Lock() + defer a.mu.Unlock() + return a.hookCalls +} + +func exec09ctScopeDigest(scope authscope.ReadScope) string { + paths := append([]string(nil), scope.AuthResourcePaths...) + sort.Strings(paths) + sum := sha256.Sum256([]byte(string(scope.Mode) + "\x00" + strings.Join(paths, "\x00"))) + return hex.EncodeToString(sum[:]) +} + +func exec09ctSnapshot(project, generation, token string, scope authscope.ReadScope) capability.Snapshot { + return capability.Snapshot{ + Identity: capability.SnapshotIdentity{ + Project: project, Generation: generation, + AuthorizationScopeDigest: exec09ctScopeDigest(scope), + SchemaDigest: "exec09-schema", ShapeDigest: "exec09-shape", + }, + Status: capability.StatusReady, Complete: true, Token: token, + } +} + +func exec09ctReceipt(t *testing.T, snapshot capability.Snapshot) explorer.CompilationReceipt { + t.Helper() + bundle := exec09ctPatientsBundle(true) + contract := json.RawMessage(`{"outputs":[{"outputId":"patients","columns":[{"column":"patient_id","label":"Patient","logicalType":"string","filterable":true,"chartable":false},{"column":"gender","label":"Gender","logicalType":"string","filterable":true,"chartable":true}]}]}`) + emitted := []explorer.EmittedColumn{ + {OutputID: exec09ctExplorer, EmissionID: "em_patient_id", PublicColumn: "patient_id", Label: "Patient", LogicalType: "string", Filterable: true}, + {OutputID: exec09ctExplorer, EmissionID: "em_gender", PublicColumn: "gender", Label: "Gender", LogicalType: "string", Filterable: true, Chartable: true}, + } + bundleDigest, err := bundle.Digest() + if err != nil { + t.Fatalf("bundle digest: %v", err) + } + receipt := explorer.CompilationReceipt{ + ReceiptFormatVersion: explorer.CurrentReceiptFormatVersion, + CompilerContractVersion: explorer.CurrentCompilerContractVersion, + Project: snapshot.Identity.Project, + ExplorerID: exec09ctExplorer, + IntentDigest: "exec09-intent", + SnapshotToken: snapshot.Token, + AuthorizationScopeDigest: snapshot.Identity.AuthorizationScopeDigest, + CapabilitySchemaDigest: snapshot.Identity.SchemaDigest, + ShapeDigest: snapshot.Identity.ShapeDigest, + SourceGeneration: snapshot.Identity.Generation, + RecipeDigest: bundleDigest, + ResolvedRecipeDigest: bundleDigest, + ResolvedSchemaDigest: "exec09-resolved-schema", + CompiledConfig: json.RawMessage(`{}`), + PublicOutputContract: contract, + Bundle: bundle, + EmittedColumns: emitted, + OutputColumnProvenance: map[string]map[string]string{exec09ctExplorer: {"patient_id": "EXPLICIT", "gender": "EXPLICIT"}}, + } + receipt.OutputContractDigest, err = explorer.CompilationArtifactDigest(contract) + if err != nil { + t.Fatalf("contract digest: %v", err) + } + receipt.CompilationKey, err = explorer.CompilationKey(receipt) + if err != nil { + t.Fatalf("compilation key: %v", err) + } + receipt.ID, err = explorer.ReceiptID(receipt) + if err != nil { + t.Fatalf("receipt id: %v", err) + } + return receipt +} + +func exec09ctLifecycleService(t *testing.T, authorized *exec09ctAuthorized, engine *dataframeexecution.Engine, receipts ...explorer.CompilationReceipt) *lifecycle.Service { + t.Helper() + byID := map[string]explorer.CompilationReceipt{} + for _, receipt := range receipts { + byID[receipt.ID] = receipt + } + domain, err := explorer.NewService(&exec09ctExplorerStore{}) + if err != nil { + t.Fatalf("explorer service: %v", err) + } + config := lifecycle.Config{ + Capability: lifecycle.CapabilityResolver{ + ForExecution: func(context.Context, string, string) (lifecycle.AuthorizedCapability, error) { + snapshot, scope := authorized.authorized() + return lifecycle.AuthorizedCapability{Snapshot: snapshot, Scope: scope}, nil + }, + }, + ReceiptLookup: func(_ context.Context, _, _, receiptID string) (*explorer.CompilationReceipt, error) { + got, ok := byID[receiptID] + if !ok { + return nil, fmt.Errorf("receipt %q is not in this test's store", receiptID) + } + return &got, nil + }, + } + config.CellTrace = func(ctx context.Context, got *explorer.CompilationReceipt, bindings recipe.RuntimeBindings, request dataframeexecution.CellTraceRequest) (dataframeexecution.CellTraceResult, error) { + authorized.mu.Lock() + authorized.hookCalls++ + authorized.mu.Unlock() + resolved, err := engine.CompileResolvedBundle(ctx, got.Bundle, bindings) + if err != nil { + return dataframeexecution.CellTraceResult{}, err + } + return engine.CellTrace(ctx, resolved, request) + } + service, err := lifecycle.New(domain, config) + if err != nil { + t.Fatalf("lifecycle service: %v", err) + } + return service +} + +func exec09ctLifecycleRequest(receipt explorer.CompilationReceipt, rowID string, offset, limit int) lifecycle.CellTraceRequest { + return lifecycle.CellTraceRequest{ + Project: receipt.Project, ExplorerID: receipt.ExplorerID, ReceiptID: receipt.ID, + OutputID: exec09ctExplorer, RowID: rowID, Column: "gender", Offset: offset, Limit: limit, + } +} + +func TestPostgresCellTraceStaleReceiptClassification(t *testing.T) { + scope := authscope.ReadScope{Mode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{exec09ctAuthPath}} + fresh := exec09ctSnapshot(exec09ctProject, exec09ctGeneration, "token-1", scope) + receipt := exec09ctReceipt(t, fresh) + authorized := &exec09ctAuthorized{snapshot: fresh, scope: scope} + var calls []exec09ctCall + var mu sync.Mutex + + fakeAdapter := func(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + sqlText, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + mu.Lock() + calls = append(calls, exec09ctCall{sql: sqlText, args: args, declared: append([]string(nil), request.DeclaredColumns...), tag: request.Command.Backend()}) + mu.Unlock() + return nil + } + binding, err := ir.NewPostgresStorageBinding("loom") + if err != nil { + t.Fatalf("binding: %v", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("target: %v", err) + } + config := exec04pvForbidArango() + config.CompileTarget = target + config.PreviewSourceRows = nil + config.ScopeDigest = func(recipe.RuntimeBindings) string { return "exec09-scope" } + config.SourceRows = fakeAdapter + engine, err := dataframeexecution.New(config) + if err != nil { + t.Fatalf("engine: %v", err) + } + service := exec09ctLifecycleService(t, authorized, engine, receipt) + + statementCount := func() int { + mu.Lock() + defer mu.Unlock() + return len(calls) + } + + hooksBefore := authorized.calls() + statementsBefore := statementCount() + _, err = service.CellTrace(context.Background(), exec09ctLifecycleRequest(receipt, "offline-identity", 0, 25)) + notFound := errors.Is(err, dataframeexecution.ErrCellTraceRowNotFound) + var lifecycleErr *lifecycle.Error + previewMissing := errors.As(err, &lifecycleErr) && lifecycleErr.Class == lifecycle.ClassNotFound + if err != nil && !notFound && !previewMissing { + t.Fatalf("fresh control failed at the hook: %v", err) + } + if authorized.calls() != hooksBefore+1 { + t.Fatalf("fresh control hook witness: %d invocations, want 1", authorized.calls()-hooksBefore) + } + if statementCount()-statementsBefore < 1 { + t.Fatalf("fresh control executed %d statements, want at least one", statementCount()-statementsBefore) + } + + for _, test := range []struct { + name string + mutate func(snapshot capability.Snapshot, scope authscope.ReadScope) (capability.Snapshot, authscope.ReadScope) + }{ + {"generation moved on", func(snapshot capability.Snapshot, scope authscope.ReadScope) (capability.Snapshot, authscope.ReadScope) { + snapshot.Identity.Generation = "gen-b" + return snapshot, scope + }}, + {"authorization scope digest changed", func(snapshot capability.Snapshot, scope authscope.ReadScope) (capability.Snapshot, authscope.ReadScope) { + scope = authscope.ReadScope{Mode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{"FHIR/Other"}} + snapshot.Identity.AuthorizationScopeDigest = exec09ctScopeDigest(scope) + return snapshot, scope + }}, + {"snapshot token rotated", func(snapshot capability.Snapshot, scope authscope.ReadScope) (capability.Snapshot, authscope.ReadScope) { + snapshot.Token = "token-2" + return snapshot, scope + }}, + } { + t.Run(test.name, func(t *testing.T) { + authorized.set(test.mutate(fresh, scope)) + hooksBefore := authorized.calls() + statementsBefore := statementCount() + _, err := service.CellTrace(context.Background(), exec09ctLifecycleRequest(receipt, "offline-identity", 0, 25)) + var lifecycleErr *lifecycle.Error + if !errors.As(err, &lifecycleErr) || lifecycleErr.Class != lifecycle.ClassConflict || lifecycleErr.Code != "RECEIPT_STALE" { + t.Fatalf("stale receipt error = %v, want ClassConflict/RECEIPT_STALE", err) + } + if authorized.calls() != hooksBefore { + t.Fatalf("stale receipt reached the cell-trace hook: %d extra invocations", authorized.calls()-hooksBefore) + } + if executed := statementCount() - statementsBefore; executed != 0 { + t.Fatalf("stale receipt executed %d statements, want the statement-count-0 witness", executed) + } + }) + } + + authorized.set(fresh, scope) + hooksBefore = authorized.calls() + if _, err := service.CellTrace(context.Background(), exec09ctLifecycleRequest(receipt, "offline-identity", 0, 25)); err != nil && !errors.Is(err, dataframeexecution.ErrCellTraceRowNotFound) { + var lifecycleErr *lifecycle.Error + if !errors.As(err, &lifecycleErr) || lifecycleErr.Class != lifecycle.ClassNotFound { + t.Fatalf("restored fresh receipt failed: %v", err) + } + } + if authorized.calls() != hooksBefore+1 { + t.Fatalf("restored fresh receipt did not reach the hook: %d invocations", authorized.calls()-hooksBefore) + } +} + +func TestPostgresCellTraceStaleReceiptRejectsBeforeExecution(t *testing.T) { + s := newExec09ctService(t) + seedProject := projectid.Legacy(exec09ctProject) + scope := authscope.ReadScope{Mode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{exec09ctAuthPath}} + fresh := exec09ctSnapshot(exec09ctProject, exec09ctGeneration, "token-1", scope) + receipt := exec09ctReceipt(t, fresh) + + s.seed(t, []exec09ctSeedRow{ + {seedProject, exec09ctGeneration, "p-1", exec09ctAuthPath, `"male"`}, + {seedProject, "gen-b", "p-new", exec09ctAuthPath, `"male"`}, + }) + + recorder := &exec09ctRecorder{adapter: s.adapter} + engine := s.engine(t, recorder) + authorized := &exec09ctAuthorized{snapshot: fresh, scope: scope} + current := exec09ctSnapshot(exec09ctProject, "gen-b", "token-2", scope) + currentReceipt := exec09ctReceipt(t, current) + service := exec09ctLifecycleService(t, authorized, engine, receipt, currentReceipt) + + identity := s.previewIdentity(t, engine, s.resolve(t, engine, seedProject, exec09ctGeneration, exec09ctPatientsBundle(true)), "p-1") + result, err := service.CellTrace(s.ctx, exec09ctLifecycleRequest(receipt, identity, 0, 25)) + if err != nil || result.Trace.Status != dataframeexecution.CellTraceValue || result.Trace.Value != "male" { + t.Fatalf("fresh receipt failed on live state: %#v %s", result, s.scrub(err)) + } + + stale := fresh + stale.Identity.Generation = "gen-b" + authorized.set(stale, scope) + hooksBefore := authorized.calls() + statementsBefore := recorder.count() + _, err = service.CellTrace(s.ctx, exec09ctLifecycleRequest(receipt, identity, 0, 25)) + var lifecycleErr *lifecycle.Error + if !errors.As(err, &lifecycleErr) || lifecycleErr.Class != lifecycle.ClassConflict || lifecycleErr.Code != "RECEIPT_STALE" { + t.Fatalf("stale receipt error = %v, want ClassConflict/RECEIPT_STALE", s.scrub(err)) + } + if authorized.calls() != hooksBefore { + t.Fatalf("stale receipt reached the engine hook: %d extra invocations", authorized.calls()-hooksBefore) + } + if got := recorder.count() - statementsBefore; got != 0 { + t.Fatalf("stale receipt executed %d statements, want the statement-count-0 witness at the production adapter", got) + } + + authorized.set(current, scope) + newIdentity := s.previewIdentity(t, engine, s.resolve(t, engine, seedProject, "gen-b", exec09ctPatientsBundle(true)), "p-new") + recovered, err := service.CellTrace(s.ctx, exec09ctLifecycleRequest(currentReceipt, newIdentity, 0, 25)) + if err != nil { + t.Fatalf("current receipt failed: %s", s.scrub(err)) + } + if recovered.Trace.Status != dataframeexecution.CellTraceValue || recovered.Trace.Value != "male" { + t.Fatalf("current receipt trace = %#v, want the gen-b cell", recovered.Trace) + } +} + +func TestZZPostgresCellTraceZeroLeftoverFixtures(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return + } + exec09ctCaptureBaseline(raw) + if exec09ctBaselineErr != nil { + t.Fatalf("baseline: %v", exec09ctBaselineErr) + } + baseline := map[string]bool{} + for _, name := range exec09ctBaselineNames { + baseline[name] = true + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for audit: %v", err) + } + defer admin.Close() + var survivors []string + rows, err := admin.Query(ctx, `SELECT datname FROM pg_database WHERE datname LIKE 'verify\_exec09\_ct\_%' ESCAPE '\'`) + if err != nil { + t.Fatalf("audit databases: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("scan audit: %v", err) + } + if !baseline[name] { + survivors = append(survivors, name) + } + } + rows.Close() + rows, err = admin.Query(ctx, `SELECT nspname FROM pg_namespace WHERE nspname LIKE 'verify\_exec09\_ct\_%' ESCAPE '\'`) + if err != nil { + t.Fatalf("audit schemas: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("scan audit: %v", err) + } + if !baseline[name] { + survivors = append(survivors, name) + } + } + rows.Close() + if len(survivors) != 0 { + sort.Strings(survivors) + t.Fatalf("verify_exec09_ct_* fixtures survived the run: %v (owned registry: %v)", survivors, exec09ctRegistrySnapshot()) + } +} diff --git a/internal/dataframe/execution/engine_postgres_exclusions_pg18_test.go b/internal/dataframe/execution/engine_postgres_exclusions_pg18_test.go new file mode 100644 index 000000000..2b3f90519 --- /dev/null +++ b/internal/dataframe/execution/engine_postgres_exclusions_pg18_test.go @@ -0,0 +1,930 @@ +package execution_test + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "sync" + "testing" + "time" + + postgres "github.com/calypr/loom/internal/backends/postgres" + storepg "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + "github.com/calypr/loom/internal/dataframe/recipe" + exec "github.com/calypr/loom/internal/dataframe/recipe/exec" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + exec12exPrefix = "verify_exec12_ex_" + exec12exNameFmt = exec12exPrefix + "%d_%s" +) + +var exec12exNamePattern = regexp.MustCompile(`^verify_exec12_ex_[0-9]+_[0-9a-f]+$`) + +var ( + exec12exOwnedMu sync.Mutex + exec12exOwnedByName = map[string]string{} +) + +func exec12exRegister(db, schema string) { + exec12exOwnedMu.Lock() + defer exec12exOwnedMu.Unlock() + exec12exOwnedByName[db] = schema +} + +func exec12exRegistrySnapshot() map[string]string { + exec12exOwnedMu.Lock() + defer exec12exOwnedMu.Unlock() + out := make(map[string]string, len(exec12exOwnedByName)) + for db, schema := range exec12exOwnedByName { + out[db] = schema + } + return out +} + +var ( + exec12exBaselineOnce sync.Once + exec12exBaselineNames []string + exec12exBaselineErr error +) + +func exec12exCaptureBaseline(raw string) { + exec12exBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec12exBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec12\\_ex\\_%' ESCAPE '\\'") + if err != nil { + exec12exBaselineErr = fmt.Errorf("read baseline: %w", err) + return + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + exec12exBaselineErr = fmt.Errorf("scan baseline: %w", err) + return + } + exec12exBaselineNames = append(exec12exBaselineNames, name) + } + }) +} + +func exec12exRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +func exec12exScrubberFor(raw string) (func(any) string, func(string)) { + secrets := []string{raw} + if u, err := url.Parse(raw); err == nil && u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + return scrub, func(s string) { secrets = append(secrets, s) } +} + +type exec12exService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + schema string + dsn string + target compiler.Target + scrub func(any) string + fail func(step string, err error) + executor dataframeexecution.SourceRows +} + +func exec12exRequireURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 table-shape exclusion acceptance tests") + } + return raw +} + +func exec12exNewService(t *testing.T) *exec12exService { + t.Helper() + raw := exec12exRequireURL(t) + scrub, addSecret := exec12exScrubberFor(raw) + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + exec12exCaptureBaseline(raw) + if exec12exBaselineErr != nil { + fail("baseline census", exec12exBaselineErr) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatalf("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, "SELECT current_setting('server_version_num')::int8, current_setting('server_version')").Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if err := storepg.CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required PostgreSQL 18 backend: %v", err) + } + + dbName := fmt.Sprintf(exec12exNameFmt, time.Now().UnixNano(), exec12exRandomHex(t)) + if !exec12exNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + var created int64 + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", dbName).Scan(&created); err != nil { + fail("verify isolated database creation", err) + } + if created != 1 { + t.Fatalf("isolated database %q did not verifiably come into existence", dbName) + } + exec12exRegister(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + addSecret(dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(exec12exNameFmt, time.Now().UnixNano(), exec12exRandomHex(t)) + if !exec12exNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + exec12exRegister(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + executor, err := postgres.NewSourceRows(iso) + if err != nil { + fail("create PostgreSQL row executor", err) + } + binding, err := ir.NewPostgresStorageBinding(schema) + if err != nil { + fail("bind PostgreSQL storage placement", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + fail("select PostgreSQL compile target", err) + } + return &exec12exService{ctx: ctx, admin: admin, iso: iso, schema: schema, dsn: dsn, target: target, scrub: scrub, fail: fail, executor: executor} +} + +func (s *exec12exService) migrate(t *testing.T) { + t.Helper() + client, err := storepg.OpenPool(s.ctx, storepg.ClientConfig{DSN: s.dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 2 * time.Second}) + if err != nil { + s.fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + defer client.Close() + if _, err := storepg.MigrateSchema(s.ctx, client, s.schema); err != nil { + s.fail("migrate owned schema", err) + } +} + +type exec12exFixtureRow struct { + project, generation, resourceType, logicalID, authPath, payload string +} + +func (s *exec12exService) seed(t *testing.T, rows []exec12exFixtureRow) { + t.Helper() + repo, err := storepg.NewResourceRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + for _, row := range rows { + if _, err := repo.InsertResource(s.ctx, source.Resource{ + Project: row.project, + DatasetGeneration: row.generation, + ResourceType: row.resourceType, + LogicalID: row.logicalID, + AuthResourcePath: row.authPath, + Payload: json.RawMessage(row.payload), + }); err != nil { + s.fail("insert fixture resource "+row.logicalID, err) + } + } +} + +type exec12exCall struct { + sql string + args []command.PostgreSQLParameter + declared []string + tag command.CompiledBackend +} + +type exec12exRecorder struct { + s *exec12exService + calls []exec12exCall +} + +func (r *exec12exRecorder) port(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + sqlText, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + r.calls = append(r.calls, exec12exCall{sql: sqlText, args: args, declared: append([]string(nil), request.DeclaredColumns...), tag: request.Command.Backend()}) + return r.s.executor(ctx, request, visit) +} + +func (r *exec12exRecorder) counts() int { return len(r.calls) } + +type exec12exInvalidRegistry struct{} + +func (exec12exInvalidRegistry) LoadRecipe(context.Context, string) (exec.Entry, error) { + return exec.Entry{}, errors.New("the exclusion battery compiles authored bundles only") +} + +func (exec12exInvalidRegistry) LoadRecipeVersion(context.Context, string, string) (exec.Entry, error) { + return exec.Entry{}, errors.New("the exclusion battery compiles authored bundles only") +} + +func exec12exForbidArango() dataframeexecution.Config { + return dataframeexecution.Config{ + CompileTarget: compiler.ArangoTarget(), + Registry: exec12exInvalidRegistry{}, + SourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, _ dataframeexecution.RowVisitor) error { + _, _, dispatchErr := request.Command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return errors.New("AQL QueryRows must never run for a PostgreSQL-served evidence read") + }, + PreviewSourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, _ dataframeexecution.RowVisitor) error { + _, _, dispatchErr := request.Command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return errors.New("AQL PreviewQueryRows must never run for a PostgreSQL-served evidence read") + }, + PreparePreviewIndex: func(context.Context, compiler.PreviewCoveringIndexSpec) error { + return errors.New("preparePreviewIndex must never run for a PostgreSQL-served evidence read") + }, + PreviewCoveringIndexes: func(context.Context, string) ([]dataframeexecution.PreviewCollectionIndex, error) { + return nil, errors.New("previewCoveringIndexes must never run for a PostgreSQL-served evidence read") + }, + PreviewCollectionRevision: func(context.Context, string) (string, error) { + return "", errors.New("previewCollectionRevision must never run for a PostgreSQL-served evidence read") + }, + PreviewCollectionScans: func(context.Context, command.CompiledCommand, string) (dataframeexecution.PreviewCollectionScanEvidence, error) { + return dataframeexecution.PreviewCollectionScanEvidence{}, errors.New("previewCollectionScans must never run for a PostgreSQL-served evidence read") + }, + PreviewCollectionCount: func(context.Context, string) (int64, error) { + return 0, errors.New("previewCollectionCount must never run for a PostgreSQL-served evidence read") + }, + ClickHouseQueryRows: func(context.Context, string, []string, func(map[string]any) error, ...any) error { + return errors.New("ClickHouse callbacks must never run for a PostgreSQL-served evidence read") + }, + } +} + +func (s *exec12exService) engine(t *testing.T, port dataframeexecution.SourceRows) *dataframeexecution.Engine { + t.Helper() + config := exec12exForbidArango() + config.CompileTarget = s.target + config.ScopeDigest = func(recipe.RuntimeBindings) string { return "exec12-ex-scope" } + config.SourceRows = port + engine, err := dataframeexecution.New(config) + if err != nil { + t.Fatalf("engine: %s", s.scrub(err)) + } + return engine +} + +const ( + exec12exProject = "P1" + exec12exGeneration = "gen-a" + exec12exAuthPath = "FHIR/Observation" + exec12exListed = "alpha" +) + +func exec12exReshape(policy recipe.PivotUnlistedCategoryPolicy) *recipe.TableReshape { + listed := exec12exListed + return &recipe.TableReshape{Kind: recipe.TableReshapeGroupedPivot, GroupedPivot: &recipe.GroupedPivot{ + ConstructionID: "exec12-pivot", GroupKeys: []string{"obs_group"}, + CategoryColumn: "obs_category", ValueColumn: "obs_value", + Categories: []recipe.GroupedPivotCategory{{Key: recipe.TableScalar{Kind: recipe.TableScalarString, String: &listed}, Output: "alpha_cell", Label: "Alpha"}}, + DuplicatePolicy: recipe.PivotDuplicateSum, MissingCellPolicy: recipe.PivotMissingCellNull, + UnlistedCategoryPolicy: policy, + }} +} + +func exec12exBundle(policy recipe.PivotUnlistedCategoryPolicy, withExtraFilter bool) recipe.Bundle { + output := recipe.Output{ + Name: "observations", RootResourceType: "Observation", RowGrain: "observation", + Fields: []recipe.Field{ + {Name: "obs_group", ColumnID: "obs_group", Expr: recipe.Expression{Select: "root.status"}}, + {Name: "obs_category", ColumnID: "obs_category", Expr: recipe.Expression{Select: "root.code.text"}}, + {Name: "obs_value", ColumnID: "obs_value", Expr: recipe.Expression{Select: "root.valueQuantity.value"}}, + }, + TableReshape: exec12exReshape(policy), + } + if withExtraFilter { + value := "provisional" + output.Filters = append(output.Filters, recipe.Filter{ + Select: "root.status", Operator: recipe.FilterEquals, + Values: []recipe.FilterValue{{Kind: recipe.FilterString, String: &value}}, + }) + } + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: "exec12_exclusions", TranslationVersion: "exec12", + Outputs: []recipe.Output{output}, + } +} + +func exec12exBindings(project, generation string, authPaths ...string) recipe.RuntimeBindings { + return recipe.RuntimeBindings{ + Project: project, DatasetGeneration: generation, + AuthResourcePaths: append([]string(nil), authPaths...), + } +} + +func exec12exDoc(logicalID, group, category string) string { + if category == "__missing__" { + return fmt.Sprintf(`{"resourceType":"Observation","id":%q,"status":%q}`, logicalID, group) + } + if category == "__null__" { + return fmt.Sprintf(`{"resourceType":"Observation","id":%q,"status":%q,"code":{"text":null}}`, logicalID, group) + } + return fmt.Sprintf(`{"resourceType":"Observation","id":%q,"status":%q,"code":{"text":%q}}`, logicalID, group, category) +} + +func exec12exDocNoIdentity(group, category string) string { + return fmt.Sprintf(`{"status":%q,"code":{"text":%q}}`, group, category) +} + +func exec12exAcceptanceFixture() []exec12exFixtureRow { + return []exec12exFixtureRow{ + {exec12exProject, exec12exGeneration, "Observation", "obs-ada", exec12exAuthPath, exec12exDoc("obs-ada", "g1", "beta")}, + {exec12exProject, exec12exGeneration, "Observation", "obs-alan", exec12exAuthPath, exec12exDoc("obs-alan", "g2", "gamma")}, + {exec12exProject, exec12exGeneration, "Observation", "obs-listed", exec12exAuthPath, exec12exDoc("obs-listed", "g1", exec12exListed)}, + {exec12exProject, exec12exGeneration, "Observation", "obs-missing", exec12exAuthPath, exec12exDoc("obs-missing", "g3", "__missing__")}, + {"P2", "gen-c", "Observation", "obs-intruder-1", exec12exAuthPath, exec12exDoc("obs-intruder-1", "g1", "beta")}, + {"P2", "gen-c", "Observation", "obs-intruder-2", exec12exAuthPath, exec12exDocNoIdentity("g2", "gamma")}, + } +} + +func exec12exCanonical(value any) string { + encoded, err := json.Marshal(value) + if err != nil { + return fmt.Sprintf("unencodable:%v", err) + } + return string(encoded) +} + +func exec12exRowsDigest(exclusions []dataframeexecution.TableShapeExclusion) []string { + out := make([]string, 0, len(exclusions)) + for _, exclusion := range exclusions { + identity := "" + if exclusion.SourceIdentity != nil { + identity = exclusion.SourceIdentity.ResourceType + "/" + exclusion.SourceIdentity.ResourceID + } + out = append(out, fmt.Sprintf("%s|%s|present=%t|value=%s|type=%s|reason=%s|omission=%s", + identity, exclusion.OutputRowID, exclusion.Category.Present, exec12exCanonical(exclusion.Category.Value), + exclusion.CategoryType, exclusion.Reason, exclusion.OmissionCode)) + } + return out +} + +func TestPostgresExclusionsExactEvidenceLivePG18(t *testing.T) { + s := exec12exNewService(t) + s.migrate(t) + s.seed(t, exec12exAcceptanceFixture()) + recorder := &exec12exRecorder{s: s} + engine := s.engine(t, recorder.port) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec12exBundle(recipe.PivotUnlistedCategoryExcludeWithEvidence, false), exec12exBindings(exec12exProject, exec12exGeneration, exec12exAuthPath)) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + result, err := engine.TableShapeExclusions(s.ctx, resolved, dataframeexecution.TableShapeExclusionRequest{Output: "observations", Limit: 10}) + if err != nil { + t.Fatalf("TableShapeExclusions: %s", s.scrub(err)) + } + if recorder.counts() != 1 { + t.Fatalf("expected exactly one statement, got %d", recorder.counts()) + } + call := recorder.calls[0] + if call.tag != command.CompiledBackendPostgreSQL { + t.Fatalf("statement tag = %q", call.tag) + } + wantContract := []string{ + ir.PhysicalTableShapeExclusionResourceTypeField, + ir.PhysicalTableShapeExclusionResourceIDField, + ir.PhysicalTableShapeExclusionIdentityStatusField, + ir.PhysicalTableShapeExclusionCategoryValueField, + ir.PhysicalTableShapeExclusionCategoryPresentField, + ir.PhysicalTableShapeExclusionCategoryTypeField, + ir.PhysicalTableShapeExclusionOutputRowIDField, + ir.PhysicalTableShapeExclusionReasonField, + ir.PhysicalTableShapeExclusionOmissionField, + } + if !reflect.DeepEqual(call.declared, wantContract) { + t.Fatalf("declared contract = %v, want %v", call.declared, wantContract) + } + if len(call.args) < 2 { + t.Fatalf("statement carries only %d ordered arguments", len(call.args)) + } + if offset, ok := call.args[len(call.args)-2].Value.(int64); !ok || offset != 0 { + t.Fatalf("offset parameter = %#v", call.args[len(call.args)-2]) + } + if fetch, ok := call.args[len(call.args)-1].Value.(int64); !ok || fetch != 11 { + t.Fatalf("probe-limit parameter must be limit+1, got %#v", call.args[len(call.args)-1]) + } + if result.Status != dataframeexecution.TableShapeExclusionComplete || !result.Complete || result.HasMore || result.NextOffset != 0 { + t.Fatalf("result metadata = %#v", result) + } + want := []string{ + "Observation/obs-ada|row-|present=true|value=\"beta\"|type=STRING|reason=UNLISTED_CATEGORY|omission=", + "Observation/obs-alan|row-|present=true|value=\"gamma\"|type=STRING|reason=UNLISTED_CATEGORY|omission=", + "Observation/obs-missing|row-|present=false|value=null|type=STRING|reason=UNLISTED_CATEGORY|omission=", + } + got := exec12exRowsDigest(result.Exclusions) + for i := range want { + if i >= len(got) { + t.Fatalf("missing evidence row %d: want %q, got %v", i, want[i], got) + } + gotFields := strings.Split(got[i], "|") + wantFields := strings.Split(want[i], "|") + for j := range wantFields { + if j == 1 { + continue + } + if gotFields[j] != wantFields[j] { + t.Fatalf("evidence row %d field %d = %q, want %q (row: %q)", i, j, gotFields[j], wantFields[j], got[i]) + } + } + if gotFields[1] == "" || gotFields[1] == "row-" { + t.Fatalf("row identity must be the pivot-computed identity, got %q", got[i]) + } + } + if len(got) != len(want) { + t.Fatalf("evidence count = %d, want %d: %v", len(got), len(want), got) + } +} + +func TestPostgresExclusionsBoundedPagingLivePG18(t *testing.T) { + s := exec12exNewService(t) + s.migrate(t) + rows := exec12exAcceptanceFixture() + for i, category := range []string{"delta", "epsilon", "zeta", "eta", "theta"} { + logicalID := fmt.Sprintf("obs-page-%d", i) + rows = append(rows, exec12exFixtureRow{exec12exProject, exec12exGeneration, "Observation", logicalID, exec12exAuthPath, exec12exDoc(logicalID, "gm", category)}) + } + s.seed(t, rows) + recorder := &exec12exRecorder{s: s} + engine := s.engine(t, recorder.port) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec12exBundle(recipe.PivotUnlistedCategoryExcludeWithEvidence, false), exec12exBindings(exec12exProject, exec12exGeneration, exec12exAuthPath)) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + const total = 8 + seen := map[string]bool{} + offset := 0 + pages := 0 + for { + before := recorder.counts() + result, err := engine.TableShapeExclusions(s.ctx, resolved, dataframeexecution.TableShapeExclusionRequest{Output: "observations", Offset: offset, Limit: 3}) + if err != nil { + t.Fatalf("page at %d: %s", offset, s.scrub(err)) + } + if recorder.counts() != before+1 { + t.Fatalf("page at %d executed %d statements, want exactly 1", offset, recorder.counts()-before) + } + if len(result.Exclusions) > 3 { + t.Fatalf("page at %d delivered %d rows, over the limit", offset, len(result.Exclusions)) + } + for _, exclusion := range result.Exclusions { + key := exclusion.OutputRowID + "|" + exec12exCanonical(exclusion.Category.Value) + if seen[key] { + t.Fatalf("pages overlap at offset %d on %s", offset, key) + } + seen[key] = true + } + pages++ + if !result.HasMore { + if result.Status != dataframeexecution.TableShapeExclusionComplete || !result.Complete { + t.Fatalf("last page must be COMPLETE, got %#v", result) + } + break + } + if result.NextOffset <= offset { + t.Fatalf("NextOffset %d did not advance from %d", result.NextOffset, offset) + } + offset = result.NextOffset + if pages > total { + t.Fatalf("paging did not converge: %d pages", pages) + } + } + if len(seen) != total { + t.Fatalf("paged %d distinct exclusions, want %d", len(seen), total) + } +} + +func TestPostgresExclusionsIdentityOmissionLivePG18(t *testing.T) { + s := exec12exNewService(t) + s.migrate(t) + rows := []exec12exFixtureRow{ + {exec12exProject, exec12exGeneration, "Observation", "obs-anon", exec12exAuthPath, exec12exDocNoIdentity("ga", "beta")}, + {exec12exProject, exec12exGeneration, "Observation", "obs-exact", exec12exAuthPath, exec12exDoc("obs-exact", "gb", "gamma")}, + } + s.seed(t, rows) + recorder := &exec12exRecorder{s: s} + engine := s.engine(t, recorder.port) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec12exBundle(recipe.PivotUnlistedCategoryExcludeWithEvidence, false), exec12exBindings(exec12exProject, exec12exGeneration, exec12exAuthPath)) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + result, err := engine.TableShapeExclusions(s.ctx, resolved, dataframeexecution.TableShapeExclusionRequest{Output: "observations", Limit: 10}) + if err != nil { + t.Fatalf("TableShapeExclusions: %s", s.scrub(err)) + } + if result.Status != dataframeexecution.TableShapeExclusionIncomplete || result.Complete || len(result.Exclusions) != 2 { + t.Fatalf("identity omission must report INCOMPLETE, never complete: %#v", result) + } + var anon, exact *dataframeexecution.TableShapeExclusion + for i := range result.Exclusions { + switch result.Exclusions[i].Category.Value.(type) { + case json.RawMessage: + if exec12exCanonical(result.Exclusions[i].Category.Value) == `"beta"` { + anon = &result.Exclusions[i] + } else { + exact = &result.Exclusions[i] + } + } + } + if anon == nil || exact == nil { + t.Fatalf("fixture rows not distinguished: %#v", result.Exclusions) + } + if anon.SourceIdentity != nil || anon.OmissionCode != "TABLE_SHAPE_SOURCE_IDENTITY_UNAVAILABLE" { + t.Fatalf("missing identity must be an explicit omission, never synthesized: %#v", anon) + } + if exact.SourceIdentity == nil || exact.SourceIdentity.ResourceID != "obs-exact" || exact.OmissionCode != "" { + t.Fatalf("exact row must carry the decoded identity and SQL-NULL omission (parsed as absent): %#v", exact) + } +} + +func TestPostgresExclusionsCategoryThreeStateLivePG18(t *testing.T) { + s := exec12exNewService(t) + s.migrate(t) + rows := []exec12exFixtureRow{ + {exec12exProject, exec12exGeneration, "Observation", "obs-m", exec12exAuthPath, exec12exDoc("obs-m", "g3", "__missing__")}, + {exec12exProject, exec12exGeneration, "Observation", "obs-n", exec12exAuthPath, exec12exDoc("obs-n", "g3", "__null__")}, + {exec12exProject, exec12exGeneration, "Observation", "obs-e", exec12exAuthPath, exec12exDoc("obs-e", "g3", "")}, + } + s.seed(t, rows) + recorder := &exec12exRecorder{s: s} + engine := s.engine(t, recorder.port) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec12exBundle(recipe.PivotUnlistedCategoryExcludeWithEvidence, false), exec12exBindings(exec12exProject, exec12exGeneration, exec12exAuthPath)) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + result, err := engine.TableShapeExclusions(s.ctx, resolved, dataframeexecution.TableShapeExclusionRequest{Output: "observations", Limit: 10}) + if err != nil { + t.Fatalf("TableShapeExclusions: %s", s.scrub(err)) + } + if result.Status != dataframeexecution.TableShapeExclusionComplete || len(result.Exclusions) != 3 { + t.Fatalf("result = %#v", result) + } + states := map[string]string{} + for _, exclusion := range result.Exclusions { + id := exclusion.SourceIdentity.ResourceID + states[id] = fmt.Sprintf("present=%t value=%s", exclusion.Category.Present, exec12exCanonical(exclusion.Category.Value)) + } + if states["obs-m"] != "present=false value=null" { + t.Fatalf("missing member must read present=false value=SQL NULL(nil): %q", states["obs-m"]) + } + if states["obs-n"] != `present=true value=null` { + t.Fatalf("explicit JSON null must read present=true with the raw JSON null carried, never flattened to nil: %q", states["obs-n"]) + } + for _, exclusion := range result.Exclusions { + if exclusion.SourceIdentity.ResourceID == "obs-m" && exclusion.Category.Value != nil { + t.Fatalf("SQL NULL missing-category value must stay nil, got %#v", exclusion.Category.Value) + } + if exclusion.SourceIdentity.ResourceID == "obs-n" { + raw, ok := exclusion.Category.Value.(json.RawMessage) + if !ok || string(raw) != "null" { + t.Fatalf("JSON-null category must survive as raw JSON, got %#v", exclusion.Category.Value) + } + } + } + if states["obs-e"] != `present=true value=""` { + t.Fatalf("empty-string category must stay a present empty string: %q", states["obs-e"]) + } +} + +func TestPostgresExclusionsCancellationLivePG18(t *testing.T) { + s := exec12exNewService(t) + s.migrate(t) + rows := exec12exAcceptanceFixture() + for i, category := range []string{"delta", "epsilon", "zeta"} { + logicalID := fmt.Sprintf("obs-cancel-%d", i) + rows = append(rows, exec12exFixtureRow{exec12exProject, exec12exGeneration, "Observation", logicalID, exec12exAuthPath, exec12exDoc(logicalID, "gm", category)}) + } + s.seed(t, rows) + recorder := &exec12exRecorder{s: s} + ctx, cancel := context.WithCancel(s.ctx) + defer cancel() + cancelling := func(inner dataframeexecution.SourceRows) dataframeexecution.SourceRows { + return func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + seen := 0 + return inner(ctx, request, func(row map[string]any) error { + err := visit(row) + seen++ + if err == nil && seen == 1 { + cancel() + } + return err + }) + } + }(recorder.port) + engine := s.engine(t, cancelling) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec12exBundle(recipe.PivotUnlistedCategoryExcludeWithEvidence, false), exec12exBindings(exec12exProject, exec12exGeneration, exec12exAuthPath)) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + result, err := engine.TableShapeExclusions(ctx, resolved, dataframeexecution.TableShapeExclusionRequest{Output: "observations", Offset: 0, Limit: 10}) + if err != nil { + t.Fatalf("cancelled read must report INCOMPLETE, not fail: %s", s.scrub(err)) + } + if result.Status != dataframeexecution.TableShapeExclusionIncomplete || result.Complete || result.NextOffset != 0 { + t.Fatalf("cancellation must report INCOMPLETE at the requested offset, never complete or complete-empty-as-complete: %#v", result) + } + if recorder.counts() == 0 { + t.Fatal("cancellation leg must actually execute SQL") + } +} + +func TestPostgresExclusionsUnsupportedShapeLivePG18(t *testing.T) { + s := exec12exNewService(t) + s.migrate(t) + s.seed(t, exec12exAcceptanceFixture()) + bindings := exec12exBindings(exec12exProject, exec12exGeneration, exec12exAuthPath) + + recorder := &exec12exRecorder{s: s} + engine := s.engine(t, recorder.port) + filtered, err := engine.CompileResolvedBundle(s.ctx, exec12exBundle(recipe.PivotUnlistedCategoryExcludeWithEvidence, true), bindings) + if err != nil { + t.Fatalf("filtered bundle compile: %s", s.scrub(err)) + } + result, err := engine.TableShapeExclusions(s.ctx, filtered, dataframeexecution.TableShapeExclusionRequest{Output: "observations", Limit: 10}) + if err != nil { + t.Fatalf("outside-slice read must report, not fail: %s", s.scrub(err)) + } + if result.Status != dataframeexecution.TableShapeExclusionUnsupported || result.Complete || len(result.Exclusions) != 0 { + t.Fatalf("outside-slice result = %#v", result) + } + if recorder.counts() != 0 { + t.Fatalf("UNSUPPORTED must execute zero statements, got %d", recorder.counts()) + } + registered, err := engine.CompileResolvedBundle(s.ctx, exec12exBundle(recipe.PivotUnlistedCategoryExcludeWithEvidence, false), bindings) + if err != nil { + t.Fatalf("registered bundle compile: %s", s.scrub(err)) + } + control, err := engine.TableShapeExclusions(s.ctx, registered, dataframeexecution.TableShapeExclusionRequest{Output: "observations", Limit: 10}) + if err != nil { + t.Fatalf("registered control: %s", s.scrub(err)) + } + if control.Status != dataframeexecution.TableShapeExclusionComplete || len(control.Exclusions) == 0 || recorder.counts() != 1 { + t.Fatalf("registered control = %#v after %d statements", control, recorder.counts()) + } +} + +func TestZZPostgresExclusionsZeroLeftoverFixtures(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 table-shape exclusion acceptance tests") + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + scrub, _ := exec12exScrubberFor(raw) + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for audit: %s", scrub(err)) + } + defer admin.Close() + owned := exec12exRegistrySnapshot() + for db := range owned { + var count int + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Fatalf("audit %s: %s", db, scrub(err)) + } + if count != 0 { + t.Errorf("owned fixture database %s survived", db) + } + } + baseline := map[string]bool{} + for _, name := range exec12exBaselineNames { + baseline[name] = true + } + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec12\\_ex\\_%' ESCAPE '\\'") + if err != nil { + t.Fatalf("audit sweep: %s", scrub(err)) + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("audit scan: %s", scrub(err)) + } + if !baseline[name] { + t.Errorf("unregistered survivor under owned prefix: %s", name) + } + } + if err := rows.Err(); err != nil { + t.Fatalf("audit rows: %s", scrub(err)) + } +} + +func TestPostgresExclusionsAuthScopeLivePG18(t *testing.T) { + s := exec12exNewService(t) + s.migrate(t) + s.seed(t, exec12exAcceptanceFixture()) + + recorder := &exec12exRecorder{s: s} + engine := s.engine(t, recorder.port) + authorized, err := engine.CompileResolvedBundle(s.ctx, exec12exBundle(recipe.PivotUnlistedCategoryExcludeWithEvidence, false), exec12exBindings(exec12exProject, exec12exGeneration, exec12exAuthPath)) + if err != nil { + t.Fatalf("authorized compile: %s", s.scrub(err)) + } + result, err := engine.TableShapeExclusions(s.ctx, authorized, dataframeexecution.TableShapeExclusionRequest{Output: "observations", Limit: 10}) + if err != nil { + t.Fatalf("authorized read: %s", s.scrub(err)) + } + if result.Status != dataframeexecution.TableShapeExclusionComplete || len(result.Exclusions) != 3 { + t.Fatalf("authorized read = %#v", result) + } + for _, exclusion := range result.Exclusions { + if exclusion.SourceIdentity != nil && exclusion.SourceIdentity.ResourceID == "obs-intruder-1" { + t.Fatalf("P2/gen-c intruder leaked into the P1/gen-a evidence read: %#v", exclusion) + } + } + if recorder.counts() != 1 { + t.Fatalf("authorized read statement count = %d", recorder.counts()) + } + + unauthorized, err := engine.CompileResolvedBundle(s.ctx, exec12exBundle(recipe.PivotUnlistedCategoryExcludeWithEvidence, false), exec12exBindings(exec12exProject, exec12exGeneration, "FHIR/Encounter")) + if err != nil { + t.Fatalf("unauthorized compile: %s", s.scrub(err)) + } + recorder = &exec12exRecorder{s: s} + engine = s.engine(t, recorder.port) + denied, err := engine.TableShapeExclusions(s.ctx, unauthorized, dataframeexecution.TableShapeExclusionRequest{Output: "observations", Limit: 10}) + if err != nil { + t.Fatalf("unauthorized read: %s", s.scrub(err)) + } + if denied.Status != dataframeexecution.TableShapeExclusionComplete || len(denied.Exclusions) != 0 { + t.Fatalf("unauthorized read must yield zero evidence, got %#v", denied) + } + if recorder.counts() != 1 { + t.Fatalf("the negative leg must execute real SQL, got %d statements", recorder.counts()) + } + foundAuthArg := false + for _, arg := range recorder.calls[0].args { + if raw, ok := arg.Value.(json.RawMessage); ok && strings.Contains(string(raw), "FHIR/Encounter") { + foundAuthArg = true + } + } + if !foundAuthArg { + t.Fatalf("the unauthorized path set must cross SQL as an ordered parameter: %#v", recorder.calls[0].args) + } +} + +func TestPostgresExclusionsNoPolicyStatusDistinctLivePG18(t *testing.T) { + s := exec12exNewService(t) + s.migrate(t) + s.seed(t, exec12exAcceptanceFixture()) + bindings := exec12exBindings(exec12exProject, exec12exGeneration, exec12exAuthPath) + + recorder := &exec12exRecorder{s: s} + engine := s.engine(t, recorder.port) + + noPolicy, err := engine.CompileResolvedBundle(s.ctx, exec12exBundle(recipe.PivotUnlistedCategoryError, false), bindings) + if err != nil { + t.Fatalf("no-policy compile: %s", s.scrub(err)) + } + result, err := engine.TableShapeExclusions(s.ctx, noPolicy, dataframeexecution.TableShapeExclusionRequest{Output: "observations", Limit: 10}) + if err != nil { + t.Fatalf("no-policy read: %s", s.scrub(err)) + } + if result.Status != dataframeexecution.TableShapeExclusionNoExclusionPolicy || result.Complete || len(result.Exclusions) != 0 { + t.Fatalf("no-policy status = %#v", result) + } + if recorder.counts() != 0 { + t.Fatalf("no-policy must execute zero statements, got %d", recorder.counts()) + } + + evidence, err := engine.CompileResolvedBundle(s.ctx, exec12exBundle(recipe.PivotUnlistedCategoryExcludeWithEvidence, false), bindings) + if err != nil { + t.Fatalf("evidence compile: %s", s.scrub(err)) + } + withEvidence, err := engine.TableShapeExclusions(s.ctx, evidence, dataframeexecution.TableShapeExclusionRequest{Output: "observations", Limit: 10}) + if err != nil { + t.Fatalf("evidence read: %s", s.scrub(err)) + } + if withEvidence.Status != dataframeexecution.TableShapeExclusionComplete || len(withEvidence.Exclusions) == 0 { + t.Fatalf("evidence control = %#v", withEvidence) + } + if withEvidence.Status == result.Status || result.Status == dataframeexecution.TableShapeExclusionComplete { + t.Fatalf("NO_EXCLUSION_POLICY must be distinct from COMPLETE-with-evidence") + } + filtered, err := engine.CompileResolvedBundle(s.ctx, exec12exBundle(recipe.PivotUnlistedCategoryExcludeWithEvidence, true), bindings) + if err != nil { + t.Fatalf("filtered compile: %s", s.scrub(err)) + } + unsupported, err := engine.TableShapeExclusions(s.ctx, filtered, dataframeexecution.TableShapeExclusionRequest{Output: "observations", Limit: 10}) + if err != nil { + t.Fatalf("unsupported read: %s", s.scrub(err)) + } + if unsupported.Status != dataframeexecution.TableShapeExclusionUnsupported || unsupported.Status == result.Status { + t.Fatalf("NO_EXCLUSION_POLICY must be distinct from UNSUPPORTED: %#v", unsupported) + } + if recorder.counts() != 1 { + t.Fatalf("only the COMPLETE-with-evidence witness may run statements (the NO_POLICY and UNSUPPORTED witnesses run zero), got %d", recorder.counts()) + } + + twinConfig := exec12exForbidArango() + twinConfig.SourceRows = func(context.Context, dataframeexecution.RowsRequest, dataframeexecution.RowVisitor) error { + return errors.New("the no-policy twin must never execute a query") + } + twinConfig.ScopeDigest = func(recipe.RuntimeBindings) string { return "exec12-ex-twin-scope" } + twin, err := dataframeexecution.New(twinConfig) + if err != nil { + t.Fatalf("twin engine: %s", s.scrub(err)) + } + twinResolved, err := twin.CompileResolvedBundle(s.ctx, exec12exBundle(recipe.PivotUnlistedCategoryError, false), bindings) + if err != nil { + t.Fatalf("twin compile: %s", s.scrub(err)) + } + twinResult, err := twin.TableShapeExclusions(s.ctx, twinResolved, dataframeexecution.TableShapeExclusionRequest{Output: "observations", Limit: 10}) + if err != nil { + t.Fatalf("twin read: %s", s.scrub(err)) + } + if twinResult.Status != dataframeexecution.TableShapeExclusionNoExclusionPolicy || twinResult.Status != result.Status { + t.Fatalf("AQL twin status %q drifted from the PostgreSQL arm %q", twinResult.Status, result.Status) + } +} diff --git a/internal/dataframe/execution/engine_postgres_postquery_pg18_test.go b/internal/dataframe/execution/engine_postgres_postquery_pg18_test.go new file mode 100644 index 000000000..0e32013b6 --- /dev/null +++ b/internal/dataframe/execution/engine_postgres_postquery_pg18_test.go @@ -0,0 +1,902 @@ +package execution_test + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "sync" + "testing" + "time" + + postgres "github.com/calypr/loom/internal/backends/postgres" + storepg "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + postgresrender "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + "github.com/calypr/loom/internal/dataframe/recipe" + exec "github.com/calypr/loom/internal/dataframe/recipe/exec" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + exec03pqPrefix = "verify_exec03v1_" + exec03pqNameFmt = exec03pqPrefix + "%d_%s" +) + +var exec03pqNamePattern = regexp.MustCompile(`^verify_exec03v1_[0-9]+_[0-9a-f]+$`) + +var ( + exec03pqOwnedMu sync.Mutex + exec03pqOwnedByName = map[string]string{} +) + +func exec03pqRegister(db, schema string) { + exec03pqOwnedMu.Lock() + defer exec03pqOwnedMu.Unlock() + exec03pqOwnedByName[db] = schema +} + +func exec03pqRegistrySnapshot() map[string]string { + exec03pqOwnedMu.Lock() + defer exec03pqOwnedMu.Unlock() + out := make(map[string]string, len(exec03pqOwnedByName)) + for db, schema := range exec03pqOwnedByName { + out[db] = schema + } + return out +} + +var ( + exec03pqBaselineOnce sync.Once + exec03pqBaselineNames []string + exec03pqBaselineErr error +) + +func exec03pqCaptureBaseline(raw string) { + exec03pqBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec03pqBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec03v1\\_%' ESCAPE '\\'") + if err != nil { + exec03pqBaselineErr = fmt.Errorf("read baseline: %w", err) + return + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + exec03pqBaselineErr = fmt.Errorf("scan baseline: %w", err) + return + } + exec03pqBaselineNames = append(exec03pqBaselineNames, name) + } + }) +} + +func exec03pqRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +func exec03pqScrubberFor(raw string) (func(any) string, func(string)) { + secrets := []string{raw} + if u, err := url.Parse(raw); err == nil && u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + return scrub, func(s string) { secrets = append(secrets, s) } +} + +func TestExec03pqScrubberHidesCredentialForms(t *testing.T) { + raw := "postgres://verify_user:S3cr3t-Pw@db.internal:5432/loomsvc?sslmode=require" + scrub, addSecret := exec03pqScrubberFor(raw) + dsn := "postgres://verify_user:S3cr3t-Pw@db.internal:5432/verify_exec03v1_it" + addSecret(dsn) + probe := fmt.Errorf("cannot parse `%s`: pq: auth failed for %s via %s", raw, "verify_user:S3cr3t-Pw", dsn) + got := scrub(probe) + for _, secret := range []string{raw, "verify_user:S3cr3t-Pw", "S3cr3t-Pw", dsn} { + if strings.Contains(got, secret) { + t.Fatalf("scrubbed text still contains credential form %q: %s", secret, got) + } + } + if !strings.Contains(got, "") { + t.Fatalf("scrubbed text lost the redaction marker: %s", got) + } +} + +type exec03pqService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + schema string + dsn string + target compiler.Target + scrub func(any) string + fail func(step string, err error) + executor dataframeexecution.SourceRows +} + +func newExec03pqService(t *testing.T) *exec03pqService { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 post-query acceptance tests") + } + scrub, addSecret := exec03pqScrubberFor(raw) + exec03pqCaptureBaseline(raw) + if exec03pqBaselineErr != nil { + t.Fatalf("capture pg_database baseline: %s", scrub(exec03pqBaselineErr)) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, "SELECT current_setting('server_version_num')::int8, current_setting('server_version')").Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if err := storepg.CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required PostgreSQL 18 backend: %v", err) + } + + dbName := fmt.Sprintf(exec03pqNameFmt, time.Now().UnixNano(), exec03pqRandomHex(t)) + if !exec03pqNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + var created int64 + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", dbName).Scan(&created); err != nil { + fail("verify isolated database creation", err) + } + if created != 1 { + t.Fatalf("isolated database %q did not verifiably come into existence", dbName) + } + exec03pqRegister(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + addSecret(dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(exec03pqNameFmt, time.Now().UnixNano(), exec03pqRandomHex(t)) + if !exec03pqNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + exec03pqRegister(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + executor, err := postgres.NewSourceRows(iso) + if err != nil { + fail("create PostgreSQL row executor", err) + } + binding, err := ir.NewPostgresStorageBinding(schema) + if err != nil { + fail("bind PostgreSQL storage placement", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + fail("select PostgreSQL compile target", err) + } + return &exec03pqService{ctx: ctx, admin: admin, iso: iso, schema: schema, dsn: dsn, target: target, scrub: scrub, fail: fail, executor: executor} +} + +func (s *exec03pqService) migrate(t *testing.T) { + t.Helper() + client, err := storepg.OpenPool(s.ctx, storepg.ClientConfig{DSN: s.dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 2 * time.Second}) + if err != nil { + s.fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + defer client.Close() + if _, err := storepg.MigrateSchema(s.ctx, client, s.schema); err != nil { + s.fail("migrate owned schema", err) + } +} + +type exec03pqFixtureRow struct { + project, generation, resourceType, logicalID, authPath, payload string +} + +const ( + exec03pqDNSNamespace = "6ba7b810-9dad-11d1-80b4-00c04fd430c8" + + exec03pqUUID5Wire = `{"__loom_exact_uuid_operation":"uuid5","__loom_exact_uuid_args":["6ba7b810-9dad-11d1-80b4-00c04fd430c8","www.widgets.com"]}` + exec03pqUUID3Wire = `{"__loom_exact_uuid_operation":"uuid3","__loom_exact_uuid_args":["6ba7b810-9dad-11d1-80b4-00c04fd430c8","www.widgets.com"]}` + exec03pqUUID5Literal = "21f7f8de-8051-5b89-8680-0195ef798b6a" + exec03pqUUID3Literal = "3d813cbb-47fb-32ba-91df-831e1593ac29" + + exec03pqTextNSWire = `{"__loom_exact_uuid_operation":"uuid5","__loom_exact_uuid_args":["www.widgets.com","widgets"]}` + exec03pqTextNSLiteral = "c0f58def-a15e-5bfd-928f-ec80a4aa1516" + + exec03pqConcatMarkerWire = `{"__loom_exact_uuid_operation":"uuid5","__loom_exact_uuid_args":["6ba7b810-9dad-11d1-80b4-00c04fd430c8","www.","widgets",".com"]}` + + exec03pqNilArgWire = `{"__loom_exact_uuid_operation":"uuid5","__loom_exact_uuid_args":["6ba7b810-9dad-11d1-80b4-00c04fd430c8",null]}` + + exec03pqNestedCanonicalWire = `{"__loom_postquery_call":"canonical_json","__loom_postquery_args":[` + exec03pqUUID5Wire + `]}` + exec03pqNestedCanonicalWant = `"` + exec03pqUUID5Literal + `"` + exec03pqNestedConcatWire = `{"__loom_postquery_call":"concat","__loom_postquery_args":["x",` + exec03pqUUID3Wire + `,null,42]}` + exec03pqNestedConcatWant = "x" + exec03pqUUID3Literal + "42" + + exec03pqMalformedMarkerWire = `{"__loom_exact_uuid_operation":"uuid5","__loom_exact_uuid_args":"nope"}` +) + +func exec03pqAcceptanceFixture() []exec03pqFixtureRow { + return []exec03pqFixtureRow{ + {"P1", "gen-a", "Patient", "marker-ada", "FHIR/Patient", `{"gender":` + exec03pqUUID5Wire + `,"birthDate":"1815-12-10","meta":{"tag":[` + exec03pqUUID3Wire + `,null,1.5]}}`}, + {"P1", "gen-a", "Patient", "marker-alan", "FHIR/Patient", `{"gender":"female","birthDate":"1912-06-23","meta":{"source":"plain"}}`}, + {"P1", "gen-a", "Patient", "marker-nested", "FHIR/Patient", `{"gender":` + exec03pqNestedCanonicalWire + `,"birthDate":"1906-12-09","meta":{"k":` + exec03pqNestedConcatWire + `}}`}, + + {"P2", "gen-b", "Patient", "bad-a", "FHIR/Patient", `{"gender":"alpha","birthDate":"2000-01-01","meta":{"ok":true}}`}, + {"P2", "gen-b", "Patient", "bad-b", "FHIR/Patient", `{"gender":` + exec03pqMalformedMarkerWire + `,"birthDate":"2000-01-02","meta":{"ok":true}}`}, + {"P2", "gen-b", "Patient", "bad-c", "FHIR/Patient", `{"gender":"gamma","birthDate":"2000-01-03","meta":{"ok":true}}`}, + + {"P3", "gen-c", "Patient", "c-01-null", "FHIR/Patient", `{"gender":null,"birthDate":"1900-01-01","meta":{"a":null}}`}, + {"P3", "gen-c", "Patient", "c-02-textns", "FHIR/Patient", `{"gender":` + exec03pqTextNSWire + `,"birthDate":"1900-01-02","meta":{"ok":true}}`}, + {"P3", "gen-c", "Patient", "c-03-concatns", "FHIR/Patient", `{"gender":` + exec03pqConcatMarkerWire + `,"birthDate":"1900-01-03","meta":{"ok":true}}`}, + {"P3", "gen-c", "Patient", "c-04-nilarg", "FHIR/Patient", `{"gender":` + exec03pqNilArgWire + `,"birthDate":"1900-01-04","meta":{"ok":true}}`}, + + {"P4", "gen-s", "Patient", "s-1-uuid3", "FHIR/Patient", `{"id":"s-1","gender":` + exec03pqUUID3Wire + `,"birthDate":"1950-01-01"}`}, + {"P4", "gen-s", "Patient", "s-2-sibling", "FHIR/Patient", `{"id":"s-2","gender":"male","birthDate":"1950-01-02"}`}, + + {"P1", "gen-x", "Patient", "intruder-generation", "FHIR/Patient", `{"gender":"intruder","birthDate":"1990-01-01","meta":{"ok":true}}`}, + {"P-other", "gen-a", "Patient", "intruder-project", "FHIR/Patient", `{"gender":"intruder","birthDate":"1991-02-02","meta":{"ok":true}}`}, + } +} + +func (s *exec03pqService) seed(t *testing.T, rows []exec03pqFixtureRow) { + t.Helper() + repo, err := storepg.NewResourceRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + for _, row := range rows { + if _, err := repo.InsertResource(s.ctx, source.Resource{ + Project: row.project, + DatasetGeneration: row.generation, + ResourceType: row.resourceType, + LogicalID: row.logicalID, + AuthResourcePath: row.authPath, + Payload: json.RawMessage(row.payload), + }); err != nil { + s.fail("insert fixture resource "+row.logicalID, err) + } + } +} + +func (s *exec03pqService) countResources(t *testing.T) int64 { + t.Helper() + var count int64 + if err := s.iso.QueryRow(s.ctx, `SELECT count(*) FROM "`+s.schema+`"."fhir_resource"`).Scan(&count); err != nil { + s.fail("count fixture rows", err) + } + return count +} + +type exec03pqCall struct { + sql string + args []command.PostgreSQLParameter + declared []string + tag command.CompiledBackend +} + +type exec03pqRecorder struct { + s *exec03pqService + calls []exec03pqCall +} + +func (r *exec03pqRecorder) port(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + sqlText, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + r.calls = append(r.calls, exec03pqCall{sql: sqlText, args: args, declared: append([]string(nil), request.DeclaredColumns...), tag: request.Command.Backend()}) + return r.s.executor(ctx, request, visit) +} + +func (r *exec03pqRecorder) counts() int { return len(r.calls) } + +type exec03pqInvalidRegistry struct{} + +func (exec03pqInvalidRegistry) LoadRecipe(context.Context, string) (exec.Entry, error) { + return exec.Entry{}, errors.New("the post-query battery compiles authored bundles only") +} + +func (exec03pqInvalidRegistry) LoadRecipeVersion(context.Context, string, string) (exec.Entry, error) { + return exec.Entry{}, errors.New("the post-query battery compiles authored bundles only") +} + +func exec03pqForbidArango() dataframeexecution.Config { + return dataframeexecution.Config{ + CompileTarget: compiler.ArangoTarget(), + Registry: exec03pqInvalidRegistry{}, + SourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, _ dataframeexecution.RowVisitor) error { + _, _, dispatchErr := request.Command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return errors.New("AQL QueryRows must never run for a PostgreSQL-served plan") + }, + PreviewSourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, _ dataframeexecution.RowVisitor) error { + _, _, dispatchErr := request.Command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return errors.New("AQL PreviewQueryRows must never run for a PostgreSQL-served plan") + }, + PreparePreviewIndex: func(context.Context, compiler.PreviewCoveringIndexSpec) error { + return errors.New("preparePreviewIndex must never run for a PostgreSQL-served plan") + }, + PreviewCoveringIndexes: func(context.Context, string) ([]dataframeexecution.PreviewCollectionIndex, error) { + return nil, errors.New("previewCoveringIndexes must never run for a PostgreSQL-served plan") + }, + PreviewCollectionRevision: func(context.Context, string) (string, error) { + return "", errors.New("previewCollectionRevision must never run for a PostgreSQL-served plan") + }, + PreviewCollectionScans: func(context.Context, command.CompiledCommand, string) (dataframeexecution.PreviewCollectionScanEvidence, error) { + return dataframeexecution.PreviewCollectionScanEvidence{}, errors.New("previewCollectionScans must never run for a PostgreSQL-served plan") + }, + PreviewCollectionCount: func(context.Context, string) (int64, error) { + return 0, errors.New("previewCollectionCount must never run for a PostgreSQL-served plan") + }, + ClickHouseQueryRows: func(context.Context, string, []string, func(map[string]any) error, ...any) error { + return errors.New("ClickHouse callbacks must never run for a PostgreSQL-served plan") + }, + } +} + +func (s *exec03pqService) engine(t *testing.T, recorder *exec03pqRecorder, target compiler.Target) *dataframeexecution.Engine { + t.Helper() + config := exec03pqForbidArango() + config.CompileTarget = target + config.PreviewSourceRows = nil + config.ScopeDigest = func(recipe.RuntimeBindings) string { return "exec03-pq-scope" } + config.SourceRows = recorder.port + engine, err := dataframeexecution.New(config) + if err != nil { + t.Fatalf("engine: %v", s.scrub(err)) + } + return engine +} + +func exec03pqMarkerBundle() recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "exec03_postquery", + TranslationVersion: "exec03", + Outputs: []recipe.Output{{ + Name: "patients", RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{ + {Name: "gender", ColumnID: "gender", Expr: recipe.Expression{Select: "root.gender"}}, + {Name: "birthDate", ColumnID: "birthDate", Expr: recipe.Expression{Select: "root.birthDate"}}, + {Name: "meta", ColumnID: "meta", Expr: recipe.Expression{Select: "root.meta"}}, + }, + }}, + } +} + +func exec03pqStreamBundle() recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "exec03_postquery_stream", + TranslationVersion: "exec03", + Outputs: []recipe.Output{{ + Name: "patients", RootResourceType: "Patient", RowGrain: "patient", + Identity: &recipe.Identity{Name: "rowId", Expr: recipe.Expression{Select: "root.id"}}, + Fields: []recipe.Field{ + {Name: "gender", ColumnID: "gender", Expr: recipe.Expression{Select: "root.gender"}}, + {Name: "birthDate", ColumnID: "birthDate", Expr: recipe.Expression{Select: "root.birthDate"}}, + }, + }}, + } +} + +func exec03pqBindings(project, generation string) recipe.RuntimeBindings { + return recipe.RuntimeBindings{ + Project: project, DatasetGeneration: generation, + AuthResourcePaths: []string{"FHIR/Patient"}, + } +} + +func exec03pqCanonicalRows(rows []map[string]any) []string { + out := make([]string, 0, len(rows)) + for _, row := range rows { + encoded, err := json.Marshal(row) + if err != nil { + out = append(out, "unencodable:"+err.Error()) + continue + } + out = append(out, string(encoded)) + } + return out +} + +func chainText(err error) string { + parts := []string{} + for err != nil { + parts = append(parts, err.Error()) + un, ok := err.(interface{ Unwrap() error }) + if !ok { + break + } + err = un.Unwrap() + } + return strings.Join(parts, "; ") +} + +func exec03pqEqualStrings(got, want []string) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true +} + +func (s *exec03pqService) runPreview(t *testing.T, project, generation string, limit int) ([]map[string]any, dataframeexecution.PreviewSummary, *exec03pqRecorder, error) { + t.Helper() + recorder := &exec03pqRecorder{s: s} + engine := s.engine(t, recorder, s.target) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec03pqMarkerBundle(), exec03pqBindings(project, generation)) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + var rows []map[string]any + summary, err := engine.PreviewOutput(s.ctx, resolved, dataframeexecution.PreviewRequest{Output: "patients", Limit: limit}, func(row map[string]any) error { + rows = append(rows, row) + return nil + }) + return rows, summary, recorder, err +} + +func TestPostgresPostQueryPreviewComputesNamedUUID5(t *testing.T) { + s := newExec03pqService(t) + s.migrate(t) + s.seed(t, exec03pqAcceptanceFixture()) + rows, summary, recorder, err := s.runPreview(t, "P1", "gen-a", 5) + if err != nil { + t.Fatalf("preview: %s", s.scrub(err)) + } + if summary.RowCount != 3 || !summary.Complete || summary.Truncated { + t.Fatalf("summary %+v, want three complete rows", summary) + } + if got, ok := rows[0]["gender"].(string); !ok || got != exec03pqUUID5Literal { + t.Fatalf("preview gender = %#v (Go %T), want the UUID5 literal %s", rows[0]["gender"], rows[0]["gender"], exec03pqUUID5Literal) + } + encodedMeta, err := json.Marshal(rows[0]["meta"]) + if err != nil { + t.Fatalf("encode meta: %v", err) + } + if want := `{"tag":["` + exec03pqUUID3Literal + `",null,1.5]}`; string(encodedMeta) != want { + t.Fatalf("preview meta = %s, want %s", encodedMeta, want) + } + if rows[1]["gender"] != "female" || rows[1]["birthDate"] != "1912-06-23" { + t.Fatalf("ALIVE sibling row = %v, want the untouched stored values", rows[1]) + } + if recorder.counts() != 1 { + t.Fatalf("port calls = %d, want one", recorder.counts()) + } + if recorder.calls[0].tag != command.CompiledBackendPostgreSQL { + t.Fatalf("statement tag = %q, want PostgreSQL", recorder.calls[0].tag) + } + if strings.Contains(recorder.calls[0].sql, "@") { + t.Fatalf("statement carries AQL bind syntax: %s", recorder.calls[0].sql) + } +} + +func TestPostgresPostQueryStreamComputesNamedUUID3(t *testing.T) { + s := newExec03pqService(t) + s.migrate(t) + s.seed(t, exec03pqAcceptanceFixture()) + recorder := &exec03pqRecorder{s: s} + engine := s.engine(t, recorder, s.target) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec03pqStreamBundle(), exec03pqBindings("P4", "gen-s")) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + outputs, err := engine.Run(s.ctx, resolved) + if err != nil { + t.Fatalf("Run: %s", s.scrub(err)) + } + rows := outputs["patients"] + if len(rows) != 2 { + t.Fatalf("stream rows = %d, want both P4/gen-s rows: %v", len(rows), rows) + } + if got, ok := rows[0]["gender"].(string); !ok || got != exec03pqUUID3Literal { + t.Fatalf("stream gender = %#v (Go %T), want the UUID3 literal %s", rows[0]["gender"], rows[0]["gender"], exec03pqUUID3Literal) + } + if rows[1]["gender"] != "male" { + t.Fatalf("ALIVE sibling stream row = %v, want \"male\"", rows[1]) + } + if id, ok := rows[0]["__loom_row_id"].(string); !ok || id == "" { + t.Fatalf("stream row lost the production stable row identity: %#v", rows[0]["__loom_row_id"]) + } + if recorder.counts() != 1 { + t.Fatalf("port calls = %d, want one", recorder.counts()) + } +} + +func TestPostgresPostQueryNamedUUIDSemanticsParity(t *testing.T) { + s := newExec03pqService(t) + s.migrate(t) + s.seed(t, exec03pqAcceptanceFixture()) + rows, _, _, err := s.runPreview(t, "P3", "gen-c", 5) + if err != nil { + t.Fatalf("preview: %s", s.scrub(err)) + } + if len(rows) != 4 { + t.Fatalf("rows = %d, want the four P3/gen-c rows", len(rows)) + } + if value, present := rows[0]["gender"]; !present || value != nil { + t.Fatalf("JSON null column public value = %#v (present %v), want untyped nil", value, present) + } + if rows[1]["gender"] != exec03pqTextNSLiteral { + t.Fatalf("textual namespace = %#v, want %s", rows[1]["gender"], exec03pqTextNSLiteral) + } + if rows[2]["gender"] != exec03pqUUID5Literal { + t.Fatalf("multi-name concat = %#v, want %s", rows[2]["gender"], exec03pqUUID5Literal) + } + if value, present := rows[3]["gender"]; !present || value != nil { + t.Fatalf("nil-argument propagation = %#v (present %v), want untyped nil", value, present) + } + for i, want := range []string{"1900-01-01", "1900-01-02", "1900-01-03", "1900-01-04"} { + if rows[i]["birthDate"] != want { + t.Fatalf("row %d birthDate = %#v, want %s", i, rows[i]["birthDate"], want) + } + } +} + +func TestPostgresPostQueryNestedMarkersMaterialize(t *testing.T) { + s := newExec03pqService(t) + s.migrate(t) + s.seed(t, exec03pqAcceptanceFixture()) + rows, _, _, err := s.runPreview(t, "P1", "gen-a", 5) + if err != nil { + t.Fatalf("preview: %s", s.scrub(err)) + } + nested := rows[2] + if nested["gender"] != exec03pqNestedCanonicalWant { + t.Fatalf("nested canonical_json = %#v, want %s", nested["gender"], exec03pqNestedCanonicalWant) + } + encodedMeta, err := json.Marshal(nested["meta"]) + if err != nil { + t.Fatalf("encode meta: %v", err) + } + if want := `{"k":"` + exec03pqNestedConcatWant + `"}`; string(encodedMeta) != want { + t.Fatalf("nested meta = %s, want %s", encodedMeta, want) + } + plainMeta, ok := rows[1]["meta"].(map[string]any) + if !ok || plainMeta["source"] != "plain" { + t.Fatalf("plain meta row = %v", rows[1]["meta"]) + } +} + +func TestPostgresPostQueryMalformedMarkerFailsBeforeRow(t *testing.T) { + s := newExec03pqService(t) + s.migrate(t) + s.seed(t, exec03pqAcceptanceFixture()) + rows, summary, _, err := s.runPreview(t, "P2", "gen-b", 5) + if err == nil { + t.Fatalf("malformed marker preview returned nil error with rows %v", rows) + } + if len(rows) != 1 || rows[0]["gender"] != "alpha" { + t.Fatalf("visited rows before the malformed row = %v, want exactly [bad-a]", rows) + } + if summary.Complete || summary.Truncated { + t.Fatalf("failed preview summary Complete=%v Truncated=%v, want both false", summary.Complete, summary.Truncated) + } + if summary.RowCount != 1 { + t.Fatalf("failed preview RowCount = %d, want the one row published before the failure", summary.RowCount) + } + good, goodSummary, _, err := s.runPreview(t, "P1", "gen-a", 5) + if err != nil { + t.Fatalf("ALIVE rerun: %s", s.scrub(err)) + } + if !goodSummary.Complete || len(good) != 3 { + t.Fatalf("ALIVE rerun summary = %+v", goodSummary) + } + if got := good[0]["gender"]; got != exec03pqUUID5Literal { + t.Fatalf("ALIVE rerun first row gender = %#v, want %s", got, exec03pqUUID5Literal) + } +} + +func TestPostgresPostQueryThreeStateValues(t *testing.T) { + s := newExec03pqService(t) + s.migrate(t) + s.seed(t, exec03pqAcceptanceFixture()) + recorder := &exec03pqRecorder{s: s} + engine := s.engine(t, recorder, s.target) + bundle := exec03pqMarkerBundle() + resolved, err := engine.CompileResolvedBundle(s.ctx, bundle, exec03pqBindings("P3", "gen-c")) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + var rows []map[string]any + if _, err := engine.PreviewOutput(s.ctx, resolved, dataframeexecution.PreviewRequest{Output: "patients", Limit: 5}, func(row map[string]any) error { + rows = append(rows, row) + return nil + }); err != nil { + t.Fatalf("preview: %s", s.scrub(err)) + } + nullRow := rows[0] + if value, present := nullRow["gender"]; !present || value != nil { + t.Fatalf("JSON null column public value = %#v (present %v), want untyped nil", value, present) + } + meta, ok := nullRow["meta"].(map[string]any) + if !ok { + t.Fatalf("meta public shape = %#v, want map[string]any", nullRow["meta"]) + } + value, present := meta["a"] + if !present || value != nil { + t.Fatalf("meta null member = %#v (present %v), want untyped nil", value, present) + } + sqlNullScope := []exec03pqFixtureRow{{"P5", "gen-n", "Patient", "n-1", "FHIR/Patient", `{"gender":"ok"}`}} + s.seed(t, sqlNullScope) + nullRows, _, _, err := s.runPreview(t, "P5", "gen-n", 5) + if err != nil { + t.Fatalf("sql-null preview: %s", s.scrub(err)) + } + if len(nullRows) != 1 { + t.Fatalf("sql-null preview rows = %v", nullRows) + } + if value, present := nullRows[0]["birthDate"]; !present || value != nil { + t.Fatalf("SQL NULL column public value = %#v (present %v), want untyped nil", value, present) + } + if nullRows[0]["gender"] != "ok" { + t.Fatalf("sql-null scope gender = %#v, want \"ok\"", nullRows[0]["gender"]) + } +} + +func TestPostgresPostQueryCrossTargetJSONNullConvergence(t *testing.T) { + const ( + genderWire = `null` + tagsWire = `["a",null]` + ) + s := newExec03pqService(t) + s.migrate(t) + fixture := []exec03pqFixtureRow{ + {"P6", "gen-x", "Patient", "x-1", "FHIR/Patient", `{"gender":` + genderWire + `,"birthDate":"1777-01-01","meta":{"v":` + tagsWire + `}}`}, + {"P6", "gen-x", "Patient", "x-2-alive", "FHIR/Patient", `{"gender":"female","birthDate":"1777-01-02","meta":{"a":null}}`}, + } + s.seed(t, fixture) + rows, summary, _, err := s.runPreview(t, "P6", "gen-x", 5) + if err != nil { + t.Fatalf("preview: %s", s.scrub(err)) + } + if !summary.Complete || len(rows) != 2 { + t.Fatalf("summary %+v rows %v, want two complete rows", summary, rows) + } + if value, present := rows[0]["gender"]; !present || value != nil { + t.Fatalf("PostgreSQL leg public value for JSON null = %#v (present %v), want untyped nil", value, present) + } + meta, ok := rows[0]["meta"].(map[string]any) + if !ok { + t.Fatalf("PostgreSQL leg meta = %#v, want map[string]any", rows[0]["meta"]) + } + if items, ok := meta["v"].([]any); !ok || len(items) != 2 || items[0] != "a" || items[1] != nil { + t.Fatalf("PostgreSQL leg meta.v public shape = %#v, want []any{\"a\", nil}", meta["v"]) + } + if _, isString := rows[0]["birthDate"].(string); !isString { + t.Fatalf("PostgreSQL leg birthDate has Go type %T, want string", rows[0]["birthDate"]) + } + if rows[1]["gender"] != "female" { + t.Fatalf("ALIVE control gender = %#v, want \"female\"", rows[1]["gender"]) + } +} + +func TestPostgresPostQueryDynamicColumnsRefuseAndMarkersNeedNoDynamicSurface(t *testing.T) { + bundle := recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "exec03_dynamic", + TranslationVersion: "exec03", + Outputs: []recipe.Output{{ + Name: "patients", RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{ + {Name: "gender", ColumnID: "gender", Expr: recipe.Expression{Select: "root.gender"}}, + }, + DynamicColumns: []recipe.DynamicColumn{{ + Name: "identifier", Source: recipe.Expression{Select: "root.identifier[].value"}, + Columns: []string{"urn:hl7"}, + }}, + }}, + } + s := newExec03pqService(t) + s.migrate(t) + recorder := &exec03pqRecorder{s: s} + engine := s.engine(t, recorder, s.target) + resolved, err := engine.CompileResolvedBundle(s.ctx, bundle, exec03pqBindings("P1", "gen-a")) + if err != nil { + t.Fatalf("the metadata half of dynamic lowering must still compile: %s", s.scrub(err)) + } + _, err = engine.PreviewOutput(s.ctx, resolved, dataframeexecution.PreviewRequest{Output: "patients", Limit: 5}, func(map[string]any) error { return nil }) + if err == nil { + t.Fatal("dynamic-columns plan rendered for the PostgreSQL target; the named gap disappeared without a plan update") + } + if !errors.Is(err, postgresrender.ErrUnsupportedPostgresPlan) { + t.Fatalf("dynamic-columns refusal %v is not the typed ErrUnsupportedPostgresPlan capability gap", err) + } + if !strings.Contains(chainText(err), "EXPRESSION_LET") { + t.Fatalf("dynamic-columns refusal %v does not name the EXPRESSION_LET capability gap", err) + } + if recorder.counts() != 0 { + t.Fatalf("the refused plan touched the server %d time(s)", recorder.counts()) + } + aqlRecorder := &exec03pqRecorder{s: s} + aqlEngine := s.engine(t, aqlRecorder, compiler.ArangoTarget()) + if _, err := aqlEngine.CompileResolvedBundle(s.ctx, bundle, exec03pqBindings("P1", "gen-a")); err != nil { + t.Fatalf("ALIVE control: the same bundle refused on the Arango target: %s", s.scrub(err)) + } +} + +func TestPostgresPostQueryMutationControls(t *testing.T) { + s := newExec03pqService(t) + s.migrate(t) + s.seed(t, exec03pqAcceptanceFixture()) + + t.Run("witness flip is detected", func(t *testing.T) { + rows, _, _, err := s.runPreview(t, "P1", "gen-a", 5) + if err != nil { + t.Fatalf("preview: %s", s.scrub(err)) + } + got := exec03pqCanonicalRows(rows) + flipped := exec03pqCanonicalRows([]map[string]any{{ + "gender": strings.ToUpper(exec03pqUUID5Literal), "birthDate": "1815-12-10", + "meta": map[string]any{"tag": []any{exec03pqUUID3Literal, nil, 1.5}}, + }}) + if len(got) == 0 || got[0] == flipped[0] { + t.Fatal("mutation control failed: leg accepted a case-flipped UUID witness") + } + }) + + t.Run("fixture flip breaks the goldens", func(t *testing.T) { + s2 := newExec03pqService(t) + s2.migrate(t) + mutated := exec03pqAcceptanceFixture() + for index := range mutated { + if mutated[index].logicalID == "marker-ada" { + mutated[index].payload = strings.Replace(mutated[index].payload, exec03pqUUID5Wire, `"changed"`, 1) + } + } + s2.seed(t, mutated) + recorder := &exec03pqRecorder{s: s2} + engine := s2.engine(t, recorder, s2.target) + resolved, err := engine.CompileResolvedBundle(s2.ctx, exec03pqMarkerBundle(), exec03pqBindings("P1", "gen-a")) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s2.scrub(err)) + } + var rows []map[string]any + if _, err := engine.PreviewOutput(s2.ctx, resolved, dataframeexecution.PreviewRequest{Output: "patients", Limit: 5}, func(row map[string]any) error { + rows = append(rows, row) + return nil + }); err != nil { + t.Fatalf("mutated preview: %s", s2.scrub(err)) + } + if rows[0]["gender"] == exec03pqUUID5Literal { + t.Fatalf("mutation control failed: leg accepted the UUID witness after the payload flip") + } + if rows[0]["gender"] != "changed" { + t.Fatalf("mutated preview gender = %#v, want the mutated stored text", rows[0]["gender"]) + } + }) +} + +func TestZZPostgresPostQueryZeroLeftoverFixtures(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 post-query acceptance tests") + } + scrub, _ := exec03pqScrubberFor(raw) + exec03pqCaptureBaseline(raw) + if exec03pqBaselineErr != nil { + t.Fatalf("capture pg_database baseline for audit: %s", scrub(exec03pqBaselineErr)) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for audit: %s", scrub(err)) + } + defer admin.Close() + owned := exec03pqRegistrySnapshot() + for db := range owned { + var count int + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Fatalf("audit %s: %s", db, scrub(err)) + } + if count != 0 { + t.Errorf("owned fixture database %s survived", db) + } + } + baseline := map[string]bool{} + for _, name := range exec03pqBaselineNames { + baseline[name] = true + } + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec03v1\\_%' ESCAPE '\\'") + if err != nil { + t.Fatalf("audit sweep: %s", scrub(err)) + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("audit scan: %s", scrub(err)) + } + if !baseline[name] { + t.Errorf("unregistered survivor under owned prefix: %s", name) + } + } + if err := rows.Err(); err != nil { + t.Fatalf("audit rows: %s", scrub(err)) + } + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatalf("audit timed out") + } +} diff --git a/internal/dataframe/execution/engine_postgres_preview_pg18_test.go b/internal/dataframe/execution/engine_postgres_preview_pg18_test.go new file mode 100644 index 000000000..e33829625 --- /dev/null +++ b/internal/dataframe/execution/engine_postgres_preview_pg18_test.go @@ -0,0 +1,820 @@ +package execution_test + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "sort" + "strings" + "sync" + "testing" + "time" + + postgres "github.com/calypr/loom/internal/backends/postgres" + storepg "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + "github.com/calypr/loom/internal/dataframe/recipe" + exec "github.com/calypr/loom/internal/dataframe/recipe/exec" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + exec04pvPrefix = "verify_exec04_pv_" + exec04pvNameFmt = exec04pvPrefix + "%d_%s" +) + +var exec04pvNamePattern = regexp.MustCompile(`^verify_exec04_pv_[0-9]+_[0-9a-f]+$`) + +var ( + exec04pvOwnedMu sync.Mutex + exec04pvOwnedByName = map[string]string{} +) + +func exec04pvRegister(db, schema string) { + exec04pvOwnedMu.Lock() + defer exec04pvOwnedMu.Unlock() + exec04pvOwnedByName[db] = schema +} + +func exec04pvRegistrySnapshot() map[string]string { + exec04pvOwnedMu.Lock() + defer exec04pvOwnedMu.Unlock() + out := make(map[string]string, len(exec04pvOwnedByName)) + for db, schema := range exec04pvOwnedByName { + out[db] = schema + } + return out +} + +var ( + exec04pvBaselineOnce sync.Once + exec04pvBaselineNames []string + exec04pvBaselineErr error +) + +func exec04pvCaptureBaseline(raw string) { + exec04pvBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec04pvBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec04\\_pv\\_%' ESCAPE '\\'") + if err != nil { + exec04pvBaselineErr = fmt.Errorf("read baseline: %w", err) + return + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + exec04pvBaselineErr = fmt.Errorf("scan baseline: %w", err) + return + } + exec04pvBaselineNames = append(exec04pvBaselineNames, name) + } + }) +} + +func exec04pvRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +type exec04pvService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + schema string + dsn string + target compiler.Target + scrub func(any) string + fail func(step string, err error) + executor dataframeexecution.SourceRows +} + +func exec04pvScrubberFor(raw string) (func(any) string, func(string)) { + secrets := []string{raw} + if u, err := url.Parse(raw); err == nil && u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + return scrub, func(s string) { secrets = append(secrets, s) } +} + +func TestExec04pvScrubberHidesCredentialForms(t *testing.T) { + raw := "postgres://verify_user:S3cr3t-Pw@db.internal:5432/loomsvc?sslmode=require" + scrub, addSecret := exec04pvScrubberFor(raw) + dsn := "postgres://verify_user:S3cr3t-Pw@db.internal:5432/verify_exec04_pg_it" + addSecret(dsn) + probe := fmt.Errorf("cannot parse `%s`: pq: auth failed for %s via %s", raw, "verify_user:S3cr3t-Pw", dsn) + got := scrub(probe) + for _, secret := range []string{raw, "verify_user:S3cr3t-Pw", "S3cr3t-Pw", dsn} { + if strings.Contains(got, secret) { + t.Fatalf("scrubbed text still contains credential form %q: %s", secret, got) + } + } + if !strings.Contains(got, "") { + t.Fatalf("scrubbed text lost the redaction marker: %s", got) + } +} + +func newExec04pvService(t *testing.T) *exec04pvService { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 preview acceptance tests") + } + scrub, addSecret := exec04pvScrubberFor(raw) + exec04pvCaptureBaseline(raw) + if exec04pvBaselineErr != nil { + t.Fatalf("capture pg_database baseline: %s", scrub(exec04pvBaselineErr)) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, "SELECT current_setting('server_version_num')::int8, current_setting('server_version')").Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if err := storepg.CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required PostgreSQL 18 backend: %v", err) + } + + dbName := fmt.Sprintf(exec04pvNameFmt, time.Now().UnixNano(), exec04pvRandomHex(t)) + if !exec04pvNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + var created int64 + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", dbName).Scan(&created); err != nil { + fail("verify isolated database creation", err) + } + if created != 1 { + t.Fatalf("isolated database %q did not verifiably come into existence", dbName) + } + exec04pvRegister(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + addSecret(dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(exec04pvNameFmt, time.Now().UnixNano(), exec04pvRandomHex(t)) + if !exec04pvNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + exec04pvRegister(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + executor, err := postgres.NewSourceRows(iso) + if err != nil { + fail("create PostgreSQL row executor", err) + } + binding, err := ir.NewPostgresStorageBinding(schema) + if err != nil { + fail("bind PostgreSQL storage placement", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + fail("select PostgreSQL compile target", err) + } + return &exec04pvService{ctx: ctx, admin: admin, iso: iso, schema: schema, dsn: dsn, target: target, scrub: scrub, fail: fail, executor: executor} +} + +func (s *exec04pvService) migrate(t *testing.T) { + t.Helper() + client, err := storepg.OpenPool(s.ctx, storepg.ClientConfig{DSN: s.dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 2 * time.Second}) + if err != nil { + s.fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + defer client.Close() + if _, err := storepg.MigrateSchema(s.ctx, client, s.schema); err != nil { + s.fail("migrate owned schema", err) + } +} + +type exec04pvFixtureRow struct { + project, generation, resourceType, logicalID, authPath, payload string +} + +func exec04pvAcceptanceFixture() []exec04pvFixtureRow { + return []exec04pvFixtureRow{ + {"P1", "gen-a", "Patient", "patient-ada", "FHIR/Patient", `{"gender":"female","birthDate":"1815-12-10"}`}, + {"P1", "gen-a", "Patient", "patient-alan", "FHIR/Patient", `{"gender":"male","birthDate":"1912-06-23"}`}, + {"P1", "gen-a", "Patient", "patient-grace", "FHIR/Patient", `{"gender":"female","birthDate":"1906-12-09"}`}, + {"P2", "gen-c", "Patient", "patient-two-a", "FHIR/Patient", `{"gender":"male","birthDate":"2000-01-01"}`}, + {"P2", "gen-c", "Patient", "patient-two-b", "FHIR/Patient", `{"gender":"female","birthDate":"2001-02-02"}`}, + {"P1", "gen-b", "Patient", "patient-intruder-generation", "FHIR/Patient", `{"gender":"other","birthDate":"1990-01-01"}`}, + {"intruder-project", "gen-a", "Patient", "patient-intruder-project", "FHIR/Patient", `{"gender":"unknown","birthDate":"1991-02-02"}`}, + } +} + +func (s *exec04pvService) seed(t *testing.T, rows []exec04pvFixtureRow) { + t.Helper() + repo, err := storepg.NewResourceRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + for _, row := range rows { + if _, err := repo.InsertResource(s.ctx, source.Resource{ + Project: row.project, + DatasetGeneration: row.generation, + ResourceType: row.resourceType, + LogicalID: row.logicalID, + AuthResourcePath: row.authPath, + Payload: json.RawMessage(row.payload), + }); err != nil { + s.fail("insert fixture resource "+row.logicalID, err) + } + } +} + +func (s *exec04pvService) countResources(t *testing.T) int64 { + t.Helper() + var count int64 + if err := s.iso.QueryRow(s.ctx, `SELECT count(*) FROM "`+s.schema+`"."fhir_resource"`).Scan(&count); err != nil { + s.fail("count fixture rows", err) + } + return count +} + +type exec04pvCall struct { + sql string + args []command.PostgreSQLParameter + declared []string + tag command.CompiledBackend +} + +type exec04pvRecorder struct { + s *exec04pvService + calls []exec04pvCall +} + +func (r *exec04pvRecorder) port(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + sqlText, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + r.calls = append(r.calls, exec04pvCall{sql: sqlText, args: args, declared: append([]string(nil), request.DeclaredColumns...), tag: request.Command.Backend()}) + return r.s.executor(ctx, request, visit) +} + +func (r *exec04pvRecorder) counts() int { return len(r.calls) } + +type exec04pvInvalidRegistry struct{} + +func (exec04pvInvalidRegistry) LoadRecipe(context.Context, string) (exec.Entry, error) { + return exec.Entry{}, errors.New("the preview battery compiles authored bundles only") +} + +func (exec04pvInvalidRegistry) LoadRecipeVersion(context.Context, string, string) (exec.Entry, error) { + return exec.Entry{}, errors.New("the preview battery compiles authored bundles only") +} + +func exec04pvForbidArango() dataframeexecution.Config { + return dataframeexecution.Config{ + CompileTarget: compiler.ArangoTarget(), + Registry: exec04pvInvalidRegistry{}, + SourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, _ dataframeexecution.RowVisitor) error { + cmd := request.Command + _, _, dispatchErr := cmd.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return errors.New("AQL QueryRows must never run for a PostgreSQL-served plan") + }, + PreviewSourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, _ dataframeexecution.RowVisitor) error { + cmd := request.Command + _, _, dispatchErr := cmd.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return errors.New("AQL PreviewQueryRows must never run for a PostgreSQL-served plan") + }, + PreparePreviewIndex: func(context.Context, compiler.PreviewCoveringIndexSpec) error { + return errors.New("preparePreviewIndex must never run for a PostgreSQL-served plan") + }, + PreviewCoveringIndexes: func(context.Context, string) ([]dataframeexecution.PreviewCollectionIndex, error) { + return nil, errors.New("previewCoveringIndexes must never run for a PostgreSQL-served plan") + }, + PreviewCollectionRevision: func(context.Context, string) (string, error) { + return "", errors.New("previewCollectionRevision must never run for a PostgreSQL-served plan") + }, + PreviewCollectionScans: func(context.Context, command.CompiledCommand, string) (dataframeexecution.PreviewCollectionScanEvidence, error) { + return dataframeexecution.PreviewCollectionScanEvidence{}, errors.New("previewCollectionScans must never run for a PostgreSQL-served plan") + }, + PreviewCollectionCount: func(context.Context, string) (int64, error) { + return 0, errors.New("previewCollectionCount must never run for a PostgreSQL-served plan") + }, + ClickHouseQueryRows: func(context.Context, string, []string, func(map[string]any) error, ...any) error { + return errors.New("ClickHouse callbacks must never run for a PostgreSQL-served plan") + }, + } +} + +func (s *exec04pvService) engine(t *testing.T, recorder *exec04pvRecorder, rootPageRows int) *dataframeexecution.Engine { + t.Helper() + config := exec04pvForbidArango() + config.CompileTarget = s.target + config.PreviewSourceRows = nil + config.ScopeDigest = func(recipe.RuntimeBindings) string { return "exec04-pv-scope" } + config.RootPageRows = rootPageRows + config.SourceRows = recorder.port + engine, err := dataframeexecution.New(config) + if err != nil { + t.Fatalf("engine: %v", s.scrub(err)) + } + return engine +} + +func exec04pvBundle() recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "exec04_preview", + TranslationVersion: "exec04", + Outputs: []recipe.Output{{ + Name: "patients", RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{ + {Name: "gender", ColumnID: "gender", Expr: recipe.Expression{Select: "root.gender"}}, + {Name: "birthDate", ColumnID: "birthDate", Expr: recipe.Expression{Select: "root.birthDate"}}, + }, + }}, + } +} + +func exec04pvBindings(project, generation string) recipe.RuntimeBindings { + return recipe.RuntimeBindings{ + Project: project, DatasetGeneration: generation, + AuthResourcePaths: []string{"FHIR/Patient"}, + } +} + +func exec04pvCanonicalRows(rows []map[string]any) []string { + out := make([]string, 0, len(rows)) + for _, row := range rows { + encoded, err := json.Marshal(row) + if err != nil { + out = append(out, "unencodable:"+err.Error()) + continue + } + out = append(out, string(encoded)) + } + return out +} + +func exec04pvRowKeys(rows []map[string]any) []string { + seen := map[string]bool{} + for _, row := range rows { + keys := make([]string, 0, len(row)) + for key := range row { + keys = append(keys, key) + } + sort.Strings(keys) + seen[strings.Join(keys, ",")] = true + } + out := make([]string, 0, len(seen)) + for key := range seen { + out = append(out, key) + } + sort.Strings(out) + return out +} + +func equalStrings(got, want []string) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true +} + +const ( + exec04pvGoldAda = `{"birthDate":"1815-12-10","gender":"female"}` + exec04pvGoldAlan = `{"birthDate":"1912-06-23","gender":"male"}` + exec04pvGoldGrace = `{"birthDate":"1906-12-09","gender":"female"}` + exec04pvGoldTwoA = `{"birthDate":"2000-01-01","gender":"male"}` + exec04pvGoldTwoB = `{"birthDate":"2001-02-02","gender":"female"}` +) + +func (s *exec04pvService) runPreview(t *testing.T, rootPageRows int, project, generation string, limit int, mutateVisitor func(*[]map[string]any) func(map[string]any) error) ([]map[string]any, dataframeexecution.PreviewSummary, *exec04pvRecorder, error) { + t.Helper() + s.migrate(t) + s.seed(t, exec04pvAcceptanceFixture()) + recorder := &exec04pvRecorder{s: s} + engine := s.engine(t, recorder, rootPageRows) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec04pvBundle(), exec04pvBindings(project, generation)) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + var rows []map[string]any + visit := func(row map[string]any) error { rows = append(rows, row); return nil } + if mutateVisitor != nil { + visit = mutateVisitor(&rows) + } + summary, err := engine.PreviewOutput(s.ctx, resolved, dataframeexecution.PreviewRequest{Output: "patients", Limit: limit}, visit) + return rows, summary, recorder, err +} + +func TestPostgresPreviewTruncatedAtLimit(t *testing.T) { + s := newExec04pvService(t) + rows, summary, recorder, err := s.runPreview(t, 0, "P1", "gen-a", 2, nil) + if err != nil { + t.Fatalf("preview: %s", s.scrub(err)) + } + if got := exec04pvCanonicalRows(rows); !equalStrings(got, []string{exec04pvGoldAda, exec04pvGoldAlan}) { + t.Fatalf("bounded preview rows = %v, want exactly the first two by the compiled window", got) + } + if summary.Complete || !summary.Truncated || summary.RowCount != 2 { + t.Fatalf("summary Complete=%v Truncated=%v RowCount=%d, want Complete=false Truncated=true RowCount=2", summary.Complete, summary.Truncated, summary.RowCount) + } + if recorder.counts() != 1 { + t.Fatalf("port calls = %d, want exactly one bounded statement", recorder.counts()) + } + if recorder.calls[0].tag != command.CompiledBackendPostgreSQL { + t.Fatalf("statement tag = %q, want PostgreSQL", recorder.calls[0].tag) + } +} + +func TestPostgresPreviewNaturalExhaustion(t *testing.T) { + s := newExec04pvService(t) + rows, summary, recorder, err := s.runPreview(t, 0, "P2", "gen-c", 3, nil) + if err != nil { + t.Fatalf("preview: %s", s.scrub(err)) + } + if got := exec04pvCanonicalRows(rows); !equalStrings(got, []string{exec04pvGoldTwoA, exec04pvGoldTwoB}) { + t.Fatalf("preview rows = %v, want both P2/gen-c rows", got) + } + if !summary.Complete || summary.Truncated || summary.RowCount != 2 { + t.Fatalf("summary Complete=%v Truncated=%v RowCount=%d, want Complete=true Truncated=false RowCount=2", summary.Complete, summary.Truncated, summary.RowCount) + } + if recorder.counts() != 1 { + t.Fatalf("port calls = %d, want one", recorder.counts()) + } +} + +func TestPostgresPreviewEmptyScopeNoLeak(t *testing.T) { + s := newExec04pvService(t) + rows, summary, recorder, err := s.runPreview(t, 0, "P-none", "gen-a", 5, nil) + if err != nil { + t.Fatalf("preview: %s", s.scrub(err)) + } + if len(rows) != 0 || summary.RowCount != 0 { + t.Fatalf("empty scope returned %d rows (summary %d), want zero", len(rows), summary.RowCount) + } + if !summary.Complete || summary.Truncated { + t.Fatalf("summary Complete=%v Truncated=%v, want Complete=true Truncated=false", summary.Complete, summary.Truncated) + } + if total := s.countResources(t); total != 7 { + t.Fatalf("fixture witness: fhir_resource holds %d rows, want 7 (the empty scope must be scope-empty, not database-empty)", total) + } + if recorder.counts() != 1 { + t.Fatalf("port calls = %d, want one", recorder.counts()) + } + call := recorder.calls[0] + if len(call.args) < 5 { + t.Fatalf("statement carries %d arguments, want at least the five scope/window arguments: %s", len(call.args), call.sql) + } + if call.args[0].Type != command.PostgreSQLParameterText || call.args[0].Value != "P-none" { + t.Fatalf("project argument = %+v, want text \"P-none\"", call.args[0]) + } + if call.args[1].Type != command.PostgreSQLParameterText || call.args[1].Value != "gen-a" { + t.Fatalf("generation argument = %+v, want text \"gen-a\"", call.args[1]) + } + if call.args[2].Type != command.PostgreSQLParameterJSONB { + t.Fatalf("auth argument type = %q, want jsonb", call.args[2].Type) + } + var auth [][]string + if err := json.Unmarshal([]byte(fmt.Sprintf("%s", call.args[2].Value)), &auth); err != nil { + var flat []string + if err2 := json.Unmarshal([]byte(fmt.Sprintf("%s", call.args[2].Value)), &flat); err2 != nil { + t.Fatalf("auth argument is not decodable jsonb (%v / %v): %s", err, err2, call.args[2].Value) + } + auth = [][]string{flat} + } + foundPath := false + for _, group := range auth { + for _, path := range group { + if path == "FHIR/Patient" { + foundPath = true + } + } + } + if !foundPath { + t.Fatalf("auth argument %s does not carry the exact scope path FHIR/Patient", call.args[2].Value) + } + if call.args[3].Type != command.PostgreSQLParameterText || call.args[3].Value != "Patient" { + t.Fatalf("root type argument = %+v, want text \"Patient\"", call.args[3]) + } + if call.args[4].Type != command.PostgreSQLParameterBigint || call.args[4].Value != int64(5) { + t.Fatalf("window argument = %+v, want bigint 5", call.args[4]) + } +} + +func TestPostgresPreviewCancellationNeverComplete(t *testing.T) { + s := newExec04pvService(t) + ctx, cancel := context.WithCancel(s.ctx) + defer cancel() + s.migrate(t) + s.seed(t, exec04pvAcceptanceFixture()) + recorder := &exec04pvRecorder{s: s} + engine := s.engine(t, recorder, 0) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec04pvBundle(), exec04pvBindings("P1", "gen-a")) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + var rows []map[string]any + summary, err := engine.PreviewOutput(ctx, resolved, dataframeexecution.PreviewRequest{Output: "patients", Limit: 5}, func(row map[string]any) error { + rows = append(rows, row) + cancel() + return nil + }) + if err == nil { + t.Fatalf("cancelled preview returned nil error with %d rows, summary %+v", len(rows), summary) + } + if summary.Complete || summary.Truncated { + t.Fatalf("cancelled summary Complete=%v Truncated=%v, want both false (no false complete result)", summary.Complete, summary.Truncated) + } + if len(rows) == 0 { + t.Fatalf("cancellation must fire after at least one delivered row for this to be a mid-stream control, got 0") + } + var rerun []map[string]any + var rerunSummary dataframeexecution.PreviewSummary + var rerunErr error + for attempt := 1; attempt <= 6; attempt++ { + rerun = nil + rerunSummary, rerunErr = engine.PreviewOutput(s.ctx, resolved, dataframeexecution.PreviewRequest{Output: "patients", Limit: 2}, func(row map[string]any) error { + rerun = append(rerun, row) + return nil + }) + if rerunErr == nil { + break + } + } + if rerunErr != nil { + t.Fatalf("alive-control rerun never converged: %s", s.scrub(rerunErr)) + } + if got := exec04pvCanonicalRows(rerun); !equalStrings(got, []string{exec04pvGoldAda, exec04pvGoldAlan}) { + t.Fatalf("alive-control rerun rows = %v, want the exact first two", got) + } + if rerunSummary.Complete || !rerunSummary.Truncated { + t.Fatalf("alive-control rerun summary Complete=%v Truncated=%v, want the bounded result", rerunSummary.Complete, rerunSummary.Truncated) + } +} + +func TestPostgresPreviewDeadlineNeverComplete(t *testing.T) { + s := newExec04pvService(t) + s.migrate(t) + s.seed(t, exec04pvAcceptanceFixture()) + recorder := &exec04pvRecorder{s: s} + engine := s.engine(t, recorder, 0) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec04pvBundle(), exec04pvBindings("P1", "gen-a")) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + ctx, cancel := context.WithTimeout(s.ctx, 200*time.Millisecond) + defer cancel() + summary, err := engine.PreviewOutput(ctx, resolved, dataframeexecution.PreviewRequest{Output: "patients", Limit: 5}, func(map[string]any) error { + time.Sleep(300 * time.Millisecond) + return nil + }) + if err == nil { + t.Fatalf("deadline preview returned nil error, summary %+v", summary) + } + if summary.Complete || summary.Truncated { + t.Fatalf("deadline summary Complete=%v Truncated=%v, want both false (never a false complete result)", summary.Complete, summary.Truncated) + } + var rerun []map[string]any + var rerunErr error + for attempt := 1; attempt <= 6; attempt++ { + rerun = nil + _, rerunErr = engine.PreviewOutput(s.ctx, resolved, dataframeexecution.PreviewRequest{Output: "patients", Limit: 5}, func(row map[string]any) error { + rerun = append(rerun, row) + return nil + }) + if rerunErr == nil { + break + } + } + if rerunErr != nil { + t.Fatalf("alive-control rerun never converged: %s", s.scrub(rerunErr)) + } + if got := exec04pvCanonicalRows(rerun); !equalStrings(got, []string{exec04pvGoldAda, exec04pvGoldAlan, exec04pvGoldGrace}) { + t.Fatalf("alive-control rerun rows = %v, want all three P1/gen-a rows", got) + } +} + +func TestPostgresPreviewPublicColumnsCompiledOrder(t *testing.T) { + s := newExec04pvService(t) + rows, summary, recorder, err := s.runPreview(t, 0, "P1", "gen-a", 5, nil) + if err != nil { + t.Fatalf("preview: %s", s.scrub(err)) + } + if !equalStrings(summary.Columns, []string{"gender", "birthDate"}) { + t.Fatalf("summary columns = %v, want the compiled public order [gender birthDate]", summary.Columns) + } + if got := exec04pvRowKeys(rows); !equalStrings(got, []string{"birthDate,gender"}) { + t.Fatalf("public row key sets = %v, want exactly {gender, birthDate} (internal projections must not cross)", got) + } + if got := exec04pvCanonicalRows(rows); !equalStrings(got, []string{exec04pvGoldAda, exec04pvGoldAlan, exec04pvGoldGrace}) { + t.Fatalf("rows = %v, want the three literal witnesses in compiled window order", got) + } + if recorder.counts() != 1 { + t.Fatalf("port calls = %d, want one", recorder.counts()) + } + if strings.Contains(recorder.calls[0].sql, "@") { + t.Fatalf("statement carries AQL bind syntax: %s", recorder.calls[0].sql) + } +} + +func TestPostgresPreviewRootPagePreviewPlanNeverCarries(t *testing.T) { + s := newExec04pvService(t) + rows, summary, recorder, err := s.runPreview(t, 25, "P1", "gen-a", 5, nil) + if err != nil { + t.Fatalf("preview with RootPageRows armed: %s", s.scrub(err)) + } + if got := exec04pvCanonicalRows(rows); !equalStrings(got, []string{exec04pvGoldAda, exec04pvGoldAlan, exec04pvGoldGrace}) { + t.Fatalf("rows = %v, want all three P1/gen-a rows", got) + } + if !summary.Complete || summary.Truncated { + t.Fatalf("summary Complete=%v Truncated=%v, want natural exhaustion", summary.Complete, summary.Truncated) + } + if recorder.counts() != 1 { + t.Fatalf("port calls = %d, want exactly one — no root-key page sequence may attach to a PostgreSQL preview", recorder.counts()) + } + for _, call := range recorder.calls { + if !strings.Contains(call.sql, "LIMIT") { + t.Fatalf("the single preview statement must be the bounded window statement: %s", call.sql) + } + } +} + +func TestPostgresPreviewMutationControls(t *testing.T) { + s := newExec04pvService(t) + + t.Run("witness flip is detected", func(t *testing.T) { + rows, _, _, err := s.runPreview(t, 0, "P1", "gen-a", 2, nil) + if err != nil { + t.Fatalf("preview: %s", s.scrub(err)) + } + got := exec04pvCanonicalRows(rows) + flipped := []string{exec04pvGoldAda, `{"birthDate":"1912-06-23","gender":"female"}`} + if equalStrings(got, flipped) { + t.Fatalf("mutation control failed: leg accepted a flipped gender witness") + } + shifted := []string{exec04pvGoldAlan, exec04pvGoldGrace} + if equalStrings(got, shifted) { + t.Fatalf("mutation control failed: leg accepted a window-shifted row pair") + } + }) + + t.Run("fixture flip breaks the goldens", func(t *testing.T) { + s2 := newExec04pvService(t) + s2.migrate(t) + mutated := exec04pvAcceptanceFixture() + for index := range mutated { + if mutated[index].logicalID == "patient-ada" { + mutated[index].payload = `{"gender":"other","birthDate":"1815-12-10"}` + } + } + s2.seed(t, mutated) + recorder := &exec04pvRecorder{s: s2} + engine := s2.engine(t, recorder, 0) + resolved, err := engine.CompileResolvedBundle(s2.ctx, exec04pvBundle(), exec04pvBindings("P1", "gen-a")) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s2.scrub(err)) + } + var rows []map[string]any + if _, err := engine.PreviewOutput(s2.ctx, resolved, dataframeexecution.PreviewRequest{Output: "patients", Limit: 2}, func(row map[string]any) error { + rows = append(rows, row) + return nil + }); err != nil { + t.Fatalf("mutated preview: %s", s2.scrub(err)) + } + got := exec04pvCanonicalRows(rows) + if equalStrings(got, []string{exec04pvGoldAda, exec04pvGoldAlan}) { + t.Fatalf("mutation control failed: leg accepted the unmutated goldens after an input flip (rows %v)", got) + } + if !equalStrings(got, []string{`{"birthDate":"1815-12-10","gender":"other"}`, exec04pvGoldAlan}) { + t.Fatalf("mutated preview rows = %v, want the mutated first row and the untouched second", got) + } + }) +} + +func TestZZPostgresPreviewZeroLeftoverFixtures(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 preview acceptance tests") + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + scrub, _ := exec04pvScrubberFor(raw) + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for audit: %s", scrub(err)) + } + defer admin.Close() + owned := exec04pvRegistrySnapshot() + for db := range owned { + var count int + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Fatalf("audit %s: %s", db, scrub(err)) + } + if count != 0 { + t.Errorf("owned fixture database %s survived", db) + } + } + baseline := map[string]bool{} + for _, name := range exec04pvBaselineNames { + baseline[name] = true + } + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec04\\_pv\\_%' ESCAPE '\\'") + if err != nil { + t.Fatalf("audit sweep: %s", scrub(err)) + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("audit scan: %s", scrub(err)) + } + if !baseline[name] { + t.Errorf("unregistered survivor under owned prefix: %s", name) + } + } + if err := rows.Err(); err != nil { + t.Fatalf("audit rows: %s", scrub(err)) + } + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatalf("audit timed out") + } +} diff --git a/internal/dataframe/execution/engine_postgres_test.go b/internal/dataframe/execution/engine_postgres_test.go new file mode 100644 index 000000000..d8cb040d7 --- /dev/null +++ b/internal/dataframe/execution/engine_postgres_test.go @@ -0,0 +1,1061 @@ +package execution + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/execution/chartifact" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/calypr/loom/internal/store/clickhouse" +) + +type dispatchSpies struct { + mu sync.Mutex + aqlQueries []string + aqlPreviewQueries []string + pgFull []postgresCallRecord + pgPreview []postgresCallRecord + pgRequests int + clickHouseQueries int + arangoControls []string +} + +func (s *dispatchSpies) recordControl(name string) { + s.mu.Lock() + defer s.mu.Unlock() + s.arangoControls = append(s.arangoControls, name) +} + +func (s *dispatchSpies) controlSnapshot() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.arangoControls...) +} + +type postgresCallRecord struct { + SQL string + Args []command.PostgreSQLParameter + Columns []string +} + +func (s *dispatchSpies) arangoEngine() Config { + record := s.recordControl + return Config{ + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "exec02-test-scope" }, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + query, _, dispatchErr := request.Command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + s.mu.Lock() + s.aqlQueries = append(s.aqlQueries, query) + s.mu.Unlock() + return nil + }, + PreviewSourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + query, _, dispatchErr := request.Command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + s.mu.Lock() + s.aqlPreviewQueries = append(s.aqlPreviewQueries, query) + s.mu.Unlock() + return nil + }, + PreparePreviewIndex: func(context.Context, compiler.PreviewCoveringIndexSpec) error { + record("preparePreviewIndex") + return nil + }, + PreviewCoveringIndexes: func(context.Context, string) ([]PreviewCollectionIndex, error) { + record("previewCoveringIndexes") + return nil, nil + }, + PreviewCollectionRevision: func(context.Context, string) (string, error) { + record("previewCollectionRevision") + return "rev-1", nil + }, + PreviewCollectionScans: func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + record("previewCollectionScans") + return PreviewCollectionScanEvidence{}, nil + }, + PreviewCollectionCount: func(context.Context, string) (int64, error) { + record("previewCollectionCount") + return 1, nil + }, + ClickHouseQueryRows: func(context.Context, string, []string, func(map[string]any) error, ...any) error { + s.mu.Lock() + s.clickHouseQueries++ + s.mu.Unlock() + return nil + }, + } +} + +func exec02PostgreSQLTarget(t *testing.T) compiler.Target { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("loom") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("PostgreSQLTarget: %v", err) + } + return target +} + +func exec02TestResolved() Resolved { + output := lower.CompiledRecipeOutput{ + Name: "resources", + RootResourceType: "Patient", + Columns: []string{"id", "name"}, + OutputSchema: []lower.CompiledOutputColumn{ + {ID: "id", Name: "id", Label: "id", Kind: "string", Cardinality: "one"}, + {ID: "name", Name: "name", Label: "name", Kind: "string", Cardinality: "one"}, + }, + RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, + Plan: exec02TestPlan(exec02TestProjections(false)), + } + return Resolved{ + Bundle: recipe.Bundle{Name: "pg_dispatch"}, + Semantic: semantic.ResolvedRecipePlan{ + SemanticPlan: semantic.RecipePlan{Bindings: recipe.RuntimeBindings{ + Project: "P1", DatasetGeneration: "G1", OutputNames: []string{"resources"}, + }}, + }, + Compiled: lower.CompiledRecipe{Outputs: []lower.CompiledRecipeOutput{output}}, + StoredRecipeDigest: "pg-dispatch-digest", + } +} + +func exec02TestProjections(withSourceID bool) []ir.PhysicalProjection { + projections := []ir.PhysicalProjection{ + {Name: "id", Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}}, + {Name: "name", Expression: &ir.PhysicalExpression{ + Kind: ir.PhysicalExtractExpression, + Cardinality: ir.PhysicalScalarCardinality, + NullBehavior: ir.PhysicalPreserveNull, + Extract: &ir.PhysicalExtract{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + ResourceType: "Patient", + Selector: spec.Selector{Steps: []spec.SelectorStep{{Field: "gender"}}}, + ExecutionMode: ir.PhysicalSelectorDirectScalar, + }, + }, Presence: &ir.PhysicalProjectionPresence{ + Source: ir.PhysicalValue{Variable: "root", Path: []string{"payload"}}, + Paths: [][]string{{"gender"}}, + }}, + } + if withSourceID { + } + return projections +} + +func exec02TestPlan(projections []ir.PhysicalProjection) ir.PhysicalPlan { + return ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: map[string]any{ + "root_collection": "Patient", + "project": "P1", + "dataset_generation": "G1", + "auth_resource_paths": []string{"FHIR/Patient"}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + }, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"project"}}, Right: &ir.PhysicalValue{BindKey: "project"}}}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"dataset_generation"}}, Right: &ir.PhysicalValue{BindKey: "dataset_generation"}}}}, + {Kind: ir.PhysicalDerivedLetOp, DerivedLet: &ir.PhysicalDerivedLet{Variable: "root_auth_allowed", Operator: "AUTH_RESOURCE_PATH_ALLOWED", Inputs: []ir.PhysicalValue{ + {Variable: "root", Path: []string{"auth_resource_path"}}, + {BindKey: "auth_resource_paths"}, + {BindKey: "auth_resource_paths_unrestricted"}, + }}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root_auth_allowed"}, Right: &ir.PhysicalValue{BindKey: "scope_allowed"}}}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}}, + }, + } +} + +func exec02TestParameters() []command.PostgreSQLParameter { + return []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "P1"}, + {Type: command.PostgreSQLParameterText, Value: "G1"}, + } +} + +func exec02PostgreSQLCommand(t *testing.T) command.CompiledCommand { + t.Helper() + resolved := exec02TestResolved() + query, err := compiler.CompileRecipeOutput(exec02PostgreSQLTarget(t), + resolved.Compiled.Outputs[0], + resolved.Semantic.SemanticPlan.Bindings, + 0, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("reference CompileRecipeOutput: %v", err) + } + if query.Command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("reference command backend = %q, want PostgreSQL", query.Command.Backend()) + } + return query.Command +} + +func exec02pgPort(spies *dispatchSpies, target *[]postgresCallRecord, rows []map[string]any) SourceRows { + return func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + sqlText, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + spies.mu.Lock() + *target = append(*target, postgresCallRecord{SQL: sqlText, Args: args, Columns: append([]string(nil), request.DeclaredColumns...)}) + spies.pgRequests++ + spies.mu.Unlock() + for _, row := range rows { + if err := ctx.Err(); err != nil { + return err + } + if err := visit(row); err != nil { + return err + } + } + return nil + } +} + +func exec02pgEngine(t *testing.T, spies *dispatchSpies, rows []map[string]any) (*Engine, Config) { + t.Helper() + config := spies.arangoEngine() + config.CompileTarget = exec02PostgreSQLTarget(t) + config.SourceRows = exec02pgPort(spies, &spies.pgFull, rows) + config.PreviewSourceRows = exec02pgPort(spies, &spies.pgPreview, rows) + return config.newEngineForTest(t), config +} + +func (c Config) newEngineForTest(t *testing.T) *Engine { + t.Helper() + engine, err := New(c) + if err != nil { + t.Fatalf("New: %v", err) + } + return engine +} + +func exec02Rows() []map[string]any { + return []map[string]any{ + {"id": "patient-one", "name": "Ada"}, + {"id": "patient-two", "name": "Grace"}, + } +} + +func exec02AssertUntouchedArango(t *testing.T, spies *dispatchSpies) { + t.Helper() + if len(spies.aqlQueries) != 0 || len(spies.aqlPreviewQueries) != 0 || len(spies.controlSnapshot()) != 0 || spies.clickHouseQueries != 0 { + t.Fatalf("PostgreSQL plan touched Arango/ClickHouse callbacks: aql=%v preview=%v controls=%v clickhouse=%d", + spies.aqlQueries, spies.aqlPreviewQueries, spies.controlSnapshot(), spies.clickHouseQueries) + } +} + +func TestPostgreSQLPlanExecutesOnlyThroughPostgreSQLAdapter(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, exec02Rows()) + resolved := exec02TestResolved() + + streams, err := engine.Streams(context.Background(), resolved) + if err != nil { + t.Fatalf("Streams: %v", err) + } + if len(streams) != 1 { + t.Fatalf("streams = %d, want 1", len(streams)) + } + var got []map[string]any + result, err := streams[0].Stream(context.Background(), func(row map[string]any) error { + got = append(got, row) + return nil + }) + if err != nil { + t.Fatalf("Stream: %v", err) + } + if result.RowCount != 2 || len(got) != 2 { + t.Fatalf("rows: result=%d visited=%d, want 2/2", result.RowCount, len(got)) + } + if got[0]["id"] != "patient-one" || got[0]["name"] != "Ada" || got[1]["id"] != "patient-two" || got[1]["name"] != "Grace" { + t.Fatalf("rows drifted from golden literals: %v", got) + } + if _, present := got[0]["__loom_row_id"]; !present { + t.Fatalf("stable row identity missing from PostgreSQL stream row: %v", got[0]) + } + if len(spies.pgFull) != 1 { + t.Fatalf("PostgreSQL port calls = %d, want 1", len(spies.pgFull)) + } + call := spies.pgFull[0] + reference := exec02PostgreSQLCommand(t) + wantSQL, wantArgs, err := reference.PostgreSQLParameters() + if err != nil { + t.Fatalf("reference parameters: %v", err) + } + if call.SQL != wantSQL { + t.Fatalf("statement SQL drifted from the compiler reference render:\n got %q\nwant %q", call.SQL, wantSQL) + } + if !reflect.DeepEqual(call.Args, wantArgs) { + t.Fatalf("ordered arguments drifted:\n got %v\nwant %v", call.Args, wantArgs) + } + if !reflect.DeepEqual(call.Columns, []string{"id", "name"}) { + t.Fatalf("declared columns = %v, want [id name]", call.Columns) + } + if len(spies.pgPreview) != 0 { + t.Fatalf("full stream touched the PostgreSQL preview override: %d calls", len(spies.pgPreview)) + } + exec02AssertUntouchedArango(t, spies) + + previewSummary, err := engine.PreviewOutput(context.Background(), resolved, PreviewRequest{Output: "resources", Limit: 5}, func(map[string]any) error { return nil }) + if err != nil { + t.Fatalf("PreviewOutput: %v", err) + } + if !previewSummary.Complete || previewSummary.Truncated { + t.Fatalf("short PostgreSQL preview must report Complete=true Truncated=false, got %+v", previewSummary) + } + if len(spies.pgPreview) != 1 || len(spies.pgFull) != 1 { + t.Fatalf("preview must use the preview override only: full=%d preview=%d", len(spies.pgFull), len(spies.pgPreview)) + } + exec02AssertUntouchedArango(t, spies) +} + +func TestPostgreSQLPreviewLimitReportsTruncationNotCompletion(t *testing.T) { + spies := &dispatchSpies{} + config := spies.arangoEngine() + config.CompileTarget = exec02PostgreSQLTarget(t) + config.SourceRows = exec02pgPort(spies, &spies.pgFull, exec02Rows()) + config.PreviewSourceRows = exec02pgPort(spies, &spies.pgPreview, exec02Rows()) + engine := config.newEngineForTest(t) + var visits int + summary, err := engine.PreviewOutput(context.Background(), exec02TestResolved(), PreviewRequest{Output: "resources", Limit: 1}, func(map[string]any) error { + visits++ + return nil + }) + if err != nil { + t.Fatalf("PreviewOutput: %v", err) + } + if visits != 1 || summary.RowCount != 1 || summary.Complete || !summary.Truncated { + t.Fatalf("bounded preview: visits=%d rowcount=%d complete=%v truncated=%v", visits, summary.RowCount, summary.Complete, summary.Truncated) + } + exec02AssertUntouchedArango(t, spies) +} + +func TestPostgreSQLPreviewCancellationStopsWithoutCompletedMarker(t *testing.T) { + spies := &dispatchSpies{} + rows := []map[string]any{ + {"id": "patient-one", "name": "Ada"}, + {"id": "patient-two", "name": "Grace"}, + {"id": "patient-three", "name": "Alan"}, + } + engine, _ := exec02pgEngine(t, spies, rows) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var visits int + summary, err := engine.PreviewOutput(ctx, exec02TestResolved(), PreviewRequest{Output: "resources", Limit: 10}, func(map[string]any) error { + visits++ + cancel() + return nil + }) + if err == nil { + t.Fatalf("canceled PostgreSQL preview returned nil error with summary %+v", summary) + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled PostgreSQL preview error must satisfy errors.Is(context.Canceled): %v", err) + } + if visits != 1 { + t.Fatalf("visitor ran %d times after cancellation, want stop at 1", visits) + } + if summary.Complete || summary.Truncated { + t.Fatalf("canceled preview must carry neither completed nor truncated markers: %+v", summary) + } + exec02AssertUntouchedArango(t, spies) +} + +func TestPostgreSQLPreviewBackendFailureNeverMarksComplete(t *testing.T) { + spies := &dispatchSpies{} + failure := errors.New("postgres rows: query: FATAL simulated") + config := spies.arangoEngine() + config.CompileTarget = exec02PostgreSQLTarget(t) + failingPort := func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + if _, _, err := request.Command.PostgreSQLParameters(); err != nil { + return err + } + if err := visit(exec02Rows()[0]); err != nil { + return err + } + return failure + } + config.SourceRows = failingPort + config.PreviewSourceRows = failingPort + engine := config.newEngineForTest(t) + summary, err := engine.PreviewOutput(context.Background(), exec02TestResolved(), PreviewRequest{Output: "resources", Limit: 10}, func(map[string]any) error { return nil }) + if err == nil || !errors.Is(err, failure) { + t.Fatalf("adapter failure must reach the caller via the error chain: %v", err) + } + if summary.Complete || summary.Truncated { + t.Fatalf("failed preview must not mark completion or truncation: %+v", summary) + } + exec02AssertUntouchedArango(t, spies) +} + +func TestUnsupportedPhysicalEngineRejectsBeforeAnyCallback(t *testing.T) { + for _, engineTag := range []ir.PhysicalEngine{"MONGO", "", " CLICKHOUSE ", "clickhouse", "postgres"} { + if engineTag == "" || engineTag == "clickhouse" || engineTag == " CLICKHOUSE " { + continue + } + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, exec02Rows()) + resolved := exec02TestResolved() + resolved.Compiled.Outputs[0].Plan.Engine = engineTag + _, _, err := engine.streamForOutput(context.Background(), resolved, "resources", 5) + if err == nil || !strings.Contains(err.Error(), "unsupported physical engine") { + t.Fatalf("unknown plan engine %q must fail closed: %v", engineTag, err) + } + if _, err := engine.Run(context.Background(), resolved); err == nil || !strings.Contains(err.Error(), "unsupported physical engine") { + t.Fatalf("Run with unknown plan engine %q must fail closed: %v", engineTag, err) + } + exec02AssertUntouchedArango(t, spies) + if spies.pgRequests != 0 { + t.Fatalf("unsupported engine %q reached the PostgreSQL row port", engineTag) + } + } +} + +func TestPostgreSQLCompileFailurePropagatesWithoutFallback(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, exec02Rows()) + resolved := exec02TestResolved() + resolved.Compiled.Outputs[0].Plan.BindVars = map[string]any{"root_collection": "Patient"} + _, _, err := engine.streamForOutput(context.Background(), resolved, "resources", 5) + if err == nil { + t.Fatal("unrenderable PostgreSQL plan must fail closed") + } + if len(spies.aqlQueries) != 0 || spies.pgRequests != 0 { + t.Fatalf("compile failure must not fall back to AQL or reach the port: aql=%v pg=%d", spies.aqlQueries, spies.pgRequests) + } + exec02AssertUntouchedArango(t, spies) +} + +func TestPostgreSQLStreamRefusesCrossTagAndZeroTagCommands(t *testing.T) { + crossTag, err := command.NewAQLCommand("FOR d IN resource RETURN d", map[string]any{"project": "P1"}) + if err != nil { + t.Fatalf("build AQL command: %v", err) + } + zeroTag := command.CompiledCommand{} + for _, test := range []struct { + name string + command command.CompiledCommand + wantIs error + wantPhrase string + }{ + {"AQL-tagged command on the PostgreSQL route", crossTag, command.ErrCompiledCommandBackendMismatch, `command is tagged "AQL" while the selected target serves "POSTGRESQL"`}, + {"zero-tag command on the PostgreSQL route", zeroTag, command.ErrCompiledCommandBackendNotRegistered, "unregistered backend tag"}, + } { + t.Run(test.name, func(t *testing.T) { + spies := &dispatchSpies{} + stream := OutputStream{ + Name: "resources", Columns: []string{"id"}, + sourceRows: func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + spies.mu.Lock() + sqlText, args, _ := request.Command.PostgreSQLParameters() + spies.pgFull = append(spies.pgFull, postgresCallRecord{SQL: sqlText, Args: args, Columns: append([]string(nil), request.DeclaredColumns...)}) + spies.pgRequests++ + spies.mu.Unlock() + for _, row := range exec02Rows() { + if err := ctx.Err(); err != nil { + return err + } + if err := visit(row); err != nil { + return err + } + } + return nil + }, + command: test.command, + expectedBackend: command.CompiledBackendPostgreSQL, + commandColumns: []string{"id"}, + } + err := stream.streamRaw(context.Background(), func(map[string]any) error { return nil }) + if !errors.Is(err, test.wantIs) { + t.Fatalf("error must satisfy errors.Is(%v): %v", test.wantIs, err) + } + if !strings.Contains(err.Error(), test.wantPhrase) { + t.Fatalf("refusal must name the routing defect %q: %v", test.wantPhrase, err) + } + if len(spies.pgFull) != 0 { + t.Fatalf("mismatched command reached the PostgreSQL port") + } + exec02AssertUntouchedArango(t, spies) + }) + } +} + +func TestPostgreSQLSelectionRefusesClickHousePayloadAndBuildsPrivateCapture(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, exec02Rows()) + combinePlan := exec02TestResolved() + combine := ir.PhysicalClickHouseCombine{Inputs: []ir.PhysicalCombineInputRef{{RevisionID: "execution-a"}}} + combinePlan.Compiled.Outputs[0].Plan.ClickHouseCombine = &combine + _, _, err := engine.streamForOutput(context.Background(), combinePlan, "resources", 5) + if err == nil || !strings.Contains(err.Error(), "cannot carry a ClickHouse combine payload") { + t.Fatalf("PostgreSQL target over a ClickHouse payload must fail closed: %v", err) + } + resolved := exec02TestResolved() + stream, _, err := engine.buildCompiledOutputStream(context.Background(), resolved, resolved.Compiled.Outputs[0], 0, map[string]bool{}, true) + if err != nil { + t.Fatalf("private capture build over a PostgreSQL-served output: %v", err) + } + if stream.expectedBackend != command.CompiledBackendPostgreSQL || stream.command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("capture stream is not a fully PostgreSQL-tagged stream: expected=%q command=%q", stream.expectedBackend, stream.command.Backend()) + } + if stream.artifactScopeMode != chartifact.ScopeModeRows || stream.scopeEvidence != nil { + t.Fatalf("capture scope fields drifted from the evidence-free fixture: mode=%q evidence=%v", stream.artifactScopeMode, stream.scopeEvidence) + } + if spies.pgRequests != 0 { + t.Fatalf("capture build reached the PostgreSQL port") + } + catalog := &testArtifactCatalog{items: map[string]chartifact.Manifest{}} + clickHouse := &testArtifactClickHouse{columns: map[string][]clickhouse.Column{}, rows: map[string][]map[string]any{}} + manager, err := chartifact.New(chartifact.Config{Catalog: catalog, ClickHouse: clickHouse}) + if err != nil { + t.Fatalf("chartifact.New: %v", err) + } + _, err = stream.MaterializeClickHouseArtifact(context.Background(), manager, "exec-1", stream.stageID, nil) + if err == nil || !strings.Contains(err.Error(), "private PostgreSQL artifact requires opaque evidence for its exact complete source plan") { + t.Fatalf("evidence-free PostgreSQL capture must fail closed naming PostgreSQL: %v", err) + } + if len(catalog.items) != 0 { + t.Fatalf("refused PostgreSQL capture created %d artifacts", len(catalog.items)) + } + if spies.pgRequests != 0 { + t.Fatalf("refused capture reached the PostgreSQL port") + } + exec02AssertUntouchedArango(t, spies) +} + +func TestClickHouseCombineUnaffectedAndNeverOfferedSourcePort(t *testing.T) { + spies := &dispatchSpies{} + config := spies.arangoEngine() + combine := ir.PhysicalClickHouseCombine{ + Kind: ir.PhysicalCombineAppend, + Inputs: []ir.PhysicalCombineInputRef{ + {TableID: "table-left", RevisionID: "execution-left", OutputID: "left"}, + {TableID: "table-right", RevisionID: "execution-right", OutputID: "right"}, + }, + Projections: []ir.PhysicalCombineProjection{ + {OutputColumnID: "value", InputIndex: 0, InputColumnID: "left-value"}, + {OutputColumnID: "value", InputIndex: 1, InputColumnID: "right-value"}, + }, + Outputs: []ir.PhysicalCombineOutputColumn{{ID: "value", Name: "value", LogicalType: "string", ClickHouseType: "String"}}, + } + resolved := exec02TestResolved() + resolved.Semantic.SemanticPlan.Bindings.Project = "project-a" + resolved.Semantic.SemanticPlan.Bindings.DatasetGeneration = "generation-a" + resolved.Compiled.Outputs[0].Plan = ir.PhysicalPlan{Version: 1, Engine: ir.PhysicalEngineClickHouse, ClickHouseCombine: &combine} + resolved.Compiled.Outputs[0].OutputSchema = nil + resolved.Compiled.Outputs[0].Columns = []string{"value"} + plan := ir.ClonePhysicalPlan(resolved.Compiled.Outputs[0].Plan) + resolved.Compiled.Outputs[0].OptimizedPlan = &plan + inputs := []ir.ResolvedClickHouseTable{ + engineResolvedClickHouseInput("table-left", "execution-left", "left", "left_table", "left-value", "left_name"), + engineResolvedClickHouseInput("table-right", "execution-right", "right", "right_table", "right-value", "right_name"), + } + config.ClickHouseQueryRows = func(_ context.Context, query string, _ []string, visit func(map[string]any) error, _ ...any) error { + spies.clickHouseQueries++ + if !strings.Contains(query, "UNION ALL") { + t.Fatalf("unexpected ClickHouse combine SQL shape: %q", query) + } + return visit(map[string]any{"__loom_row_id": "stable-row", "value": "Ada", "auth_resource_path": ""}) + } + config.ResolveClickHouseInputs = func(context.Context, ir.PhysicalClickHouseCombine, recipe.RuntimeBindings) ([]ir.ResolvedClickHouseTable, error) { + return inputs, nil + } + config.WithExecutionReadPins = func(ctx context.Context, _ []string, visit func(context.Context) error) error { + return visit(ctx) + } + engine := config.newEngineForTest(t) + rows, err := engine.Run(context.Background(), resolved) + if err != nil { + t.Fatalf("Run: %v", err) + } + if spies.clickHouseQueries != 1 { + t.Fatalf("ClickHouse callbacks = %d, want 1", spies.clickHouseQueries) + } + if spies.pgRequests != 0 || len(spies.aqlQueries) != 0 || len(spies.aqlPreviewQueries) != 0 { + t.Fatalf("ClickHouse combine leaked into a source adapter: pg=%d aql=%v preview=%v", spies.pgRequests, spies.aqlQueries, spies.aqlPreviewQueries) + } + if got := rows["resources"]; len(got) != 1 || got[0]["value"] != "Ada" { + t.Fatalf("ClickHouse combine rows drifted: %v", got) + } +} + +func TestAQLPathUnchangedWhenPostgreSQLUnused(t *testing.T) { + spies := &dispatchSpies{} + config := spies.arangoEngine() + engine := config.newEngineForTest(t) + bundle := testResolvedBundle([]string{}) + resolved, err := engine.CompileResolvedBundle(context.Background(), bundle, recipe.RuntimeBindings{Project: "P1"}) + if err != nil { + t.Fatalf("compile: %v", err) + } + if _, err := engine.Preview(context.Background(), resolved, 2); err != nil { + t.Fatalf("Preview: %v", err) + } + if len(spies.aqlPreviewQueries) == 0 || spies.pgRequests != 0 { + t.Fatalf("AQL preview must use the AQL preview port only: preview=%v pg=%d", spies.aqlPreviewQueries, spies.pgRequests) + } + streams, err := engine.Streams(context.Background(), resolved) + if err != nil { + t.Fatalf("Streams: %v", err) + } + for _, stream := range streams { + if _, err := stream.Stream(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("Stream: %v", err) + } + } + if len(spies.aqlQueries) == 0 || spies.pgRequests != 0 { + t.Fatalf("AQL stream must use the AQL port only: aql=%v pg=%d", spies.aqlQueries, spies.pgRequests) + } +} + +func TestAQLPreviewStillDrivesArangoControls(t *testing.T) { + spies := &dispatchSpies{} + config := spies.arangoEngine() + engine := config.newEngineForTest(t) + groupOutput := executionGroupPreviewOutput() + resolved, err := engine.CompileResolvedBundle(context.Background(), recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: groupOutput.Name, + TranslationVersion: "test", + Outputs: []recipe.Output{groupOutput}, + }, recipe.RuntimeBindings{Project: "P1", DatasetGeneration: "G1"}) + if err != nil { + t.Fatalf("compile: %v", err) + } + if _, err := engine.PreviewOutput(context.Background(), resolved, PreviewRequest{Output: groupOutput.Name, Limit: 1}, func(map[string]any) error { return nil }); err != nil { + t.Fatalf("PreviewOutput: %v", err) + } + if len(spies.aqlPreviewQueries) == 0 { + t.Fatal("AQL preview stopped using the AQL preview port") + } + deadline := time.Now().Add(2 * time.Second) + sawPrepare := false + for time.Now().Before(deadline) { + for _, control := range spies.controlSnapshot() { + if control == "preparePreviewIndex" { + sawPrepare = true + } + } + if sawPrepare { + break + } + time.Sleep(5 * time.Millisecond) + } + if !sawPrepare { + t.Fatalf("AQL Group preview stopped driving the Arango index preparation control: %v", spies.controlSnapshot()) + } + if spies.pgRequests != 0 { + t.Fatalf("AQL Group preview leaked into the PostgreSQL port: %d calls", spies.pgRequests) + } +} + +func TestGroupPreviewOptimizerNeverEvaluatesPostgreSQLStream(t *testing.T) { + spies := &dispatchSpies{} + reference := exec02PostgreSQLCommand(t) + evaluate := func(stream *OutputStream) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), + registry: invalidRecipeRegistry{}, + sourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + _, _, dispatchErr := request.Command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }, + } + engine.previewCoveringIndexes = func(context.Context, string) ([]PreviewCollectionIndex, error) { + spies.recordControl("previewCoveringIndexes") + return nil, nil + } + engine.previewCollectionScans = func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + spies.recordControl("previewCollectionScans") + return PreviewCollectionScanEvidence{}, nil + } + engine.previewCollectionCount = func(context.Context, string) (int64, error) { + spies.recordControl("previewCollectionCount") + return 7, nil + } + query := &compiler.CompiledQuery{ + Command: mustAQLCommand("FOR d IN Patient RETURN d", map[string]any{"project": "P1"}), + PreviewGroupScan: &compiler.PreviewGroupScanSpec{ + Collection: "Patient", Sequential: mustAQLCommand("FOR d IN Patient RETURN d", nil), + ScopeCount: mustAQLCommand("RETURN 1", nil), + }, + PreviewCoveringIndex: &compiler.PreviewCoveringIndexSpec{Collection: "Patient", Name: "loom_pivot_preview_a", Fields: []string{"project"}}, + } + return engine.chooseGroupPreviewScan(context.Background(), query, stream) + } + aqlStream := &OutputStream{ + Name: "resources", sourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + _, _, dispatchErr := request.Command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }, + } + if err := evaluate(aqlStream); err != nil { + t.Fatalf("AQL group preview scan: %v", err) + } + controlsBefore := len(spies.controlSnapshot()) + if controlsBefore == 0 { + t.Fatal("control failed: the AQL-shaped stream must reach the Arango optimizer callbacks") + } + postgresStream := &OutputStream{ + Name: "resources", + sourceRows: exec02pgPort(spies, &spies.pgFull, exec02Rows()), + command: reference, + commandColumns: []string{"id", "name"}, + } + if err := evaluate(postgresStream); err != nil { + t.Fatalf("PostgreSQL group preview scan: %v", err) + } + after := spies.controlSnapshot() + if len(after) != controlsBefore { + t.Fatalf("PostgreSQL stream invoked Arango optimizer controls: %v", after[controlsBefore:]) + } + if spies.pgRequests != 0 { + t.Fatalf("optimizer ran the PostgreSQL port: %d calls", spies.pgRequests) + } +} + +func TestPostgreSQLStreamConsumesOnlyTaggedCommand(t *testing.T) { + spies := &dispatchSpies{} + reference := exec02PostgreSQLCommand(t) + rows := exec02Rows() + aqlExpectingStream := OutputStream{ + Name: "resources", Columns: []string{"id", "name"}, + sourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + _, _, dispatchErr := request.Command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return errors.New("AQL port must never run for a PostgreSQL stream") + }, + command: reference, + commandColumns: []string{"id", "name"}, + } + if err := aqlExpectingStream.streamRaw(context.Background(), func(map[string]any) error { return nil }); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("AQL-expecting port on a PostgreSQL stream must report the envelope mismatch: %v", err) + } + + stream := OutputStream{ + Name: "resources", Columns: []string{"id", "name"}, + sourceRows: exec02pgPort(spies, &spies.pgFull, rows), + command: reference, + commandColumns: []string{"id", "name"}, + } + var visited int + if err := stream.streamRaw(context.Background(), func(map[string]any) error { visited++; return nil }); err != nil { + t.Fatalf("streamRaw: %v", err) + } + if visited != len(rows) { + t.Fatalf("visited=%d want %d", visited, len(rows)) + } + wantSQL, _, err := reference.PostgreSQLParameters() + if err != nil { + t.Fatalf("reference parameters: %v", err) + } + if spies.pgRequests != 1 || spies.pgFull[0].SQL != wantSQL { + t.Fatalf("PostgreSQL port did not receive the tagged command: %+v", spies.pgFull) + } +} + +func TestPostgreSQLStreamWithoutPortNeverFallsBackToAQL(t *testing.T) { + aqlCalls := 0 + stream := OutputStream{ + Name: "resources", Columns: []string{"id", "name"}, + sourceRows: nil, + command: exec02PostgreSQLCommand(t), + commandColumns: []string{"id", "name"}, + } + _ = aqlCalls + err := stream.streamRaw(context.Background(), func(map[string]any) error { return nil }) + if !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("missing source row port must fail closed with the named sentinel: %v", err) + } + if !strings.Contains(err.Error(), "no source row port") { + t.Fatalf("refusal must name the missing-port defect: %v", err) + } +} + +func TestPostgreSQLPlanFingerprintIsStableAndSensitivityProbed(t *testing.T) { + compile := func(t *testing.T, resolved Resolved) string { + t.Helper() + query, err := compiler.CompileRecipeOutput(exec02PostgreSQLTarget(t), + resolved.Compiled.Outputs[0], + resolved.Semantic.SemanticPlan.Bindings, + 0, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("reference compile: %v", err) + } + return query.PlanDiagnostics.Fingerprint + } + base := compile(t, exec02TestResolved()) + if base == "" || base != compile(t, exec02TestResolved()) { + t.Fatalf("fingerprint must be deterministic and non-empty: %q", base) + } + scopeChange := exec02TestResolved() + scopeChange.Compiled.Outputs[0].Plan.BindVars["project"] = "P2" + if got := compile(t, scopeChange); got == base { + t.Fatal("fingerprint did not react to a scope argument change") + } + projectionChange := exec02TestResolved() + returnReturn := func(resolved *Resolved) *ir.PhysicalReturn { + t.Helper() + for index := range resolved.Compiled.Outputs[0].Plan.Operations { + if operation := &resolved.Compiled.Outputs[0].Plan.Operations[index]; operation.Kind == ir.PhysicalReturnOp { + return operation.Return + } + } + t.Fatal("fixture plan has no RETURN") + return nil + } + returnReturn(&projectionChange).Projections = returnReturn(&projectionChange).Projections[:1] + projectionChange.Compiled.Outputs[0].Columns = []string{"id"} + projectionChange.Compiled.Outputs[0].OutputSchema = projectionChange.Compiled.Outputs[0].OutputSchema[:1] + if got := compile(t, projectionChange); got == base { + t.Fatal("fingerprint did not react to a statement shape change") + } + nullBehaviorChange := exec02TestResolved() + returnReturn(&nullBehaviorChange).Projections[1].Expression.NullBehavior = ir.PhysicalEmptyOnNull + if got := compile(t, nullBehaviorChange); got == base { + t.Fatal("fingerprint did not react to a null-behavior change") + } +} + +func TestPostgreSQLSelectionRefusesAllInternalDeclaredColumns(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, exec02Rows()) + output := lower.CompiledRecipeOutput{ + Name: "resources", + RootResourceType: "Patient", + Columns: []string{"id", "name"}, + OutputSchema: []lower.CompiledOutputColumn{ + {ID: "id", Name: "id", Label: "id", Kind: "string", Cardinality: "one", Internal: true}, + {ID: "name", Name: "name", Label: "name", Kind: "string", Cardinality: "one", Internal: true}, + }, + RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, + Plan: exec02TestPlan(exec02TestProjections(false)), + } + resolved := Resolved{ + Bundle: recipe.Bundle{Name: "pg_dispatch"}, + Semantic: semantic.ResolvedRecipePlan{ + SemanticPlan: semantic.RecipePlan{Bindings: recipe.RuntimeBindings{ + Project: "P1", DatasetGeneration: "G1", OutputNames: []string{"resources"}, + }}, + }, + Compiled: lower.CompiledRecipe{Outputs: []lower.CompiledRecipeOutput{output}}, + StoredRecipeDigest: "pg-internal-only", + } + _, err := engine.Streams(context.Background(), resolved) + if err == nil || !strings.Contains(err.Error(), "internal projection") { + t.Fatalf("all-internal declared contract must be refused, got %v", err) + } + if spies.pgRequests != 0 { + t.Fatalf("refused selection must not reach the row port: %d calls", spies.pgRequests) + } + exec02AssertUntouchedArango(t, spies) +} + +func TestPostgreSQLPreviewReportsSingleSourceIdentityWithResourceType(t *testing.T) { + spies := &dispatchSpies{} + sourceID := ir.PreviewSourceResourceIDColumn + output := lower.CompiledRecipeOutput{ + Name: "resources", + RootResourceType: "Patient", + Columns: []string{"id", "name", sourceID}, + OutputSchema: []lower.CompiledOutputColumn{ + {ID: "id", Name: "id", Label: "id", Kind: "string", Cardinality: "one"}, + {ID: "name", Name: "name", Label: "name", Kind: "string", Cardinality: "one"}, + {ID: sourceID, Name: sourceID, Label: sourceID, Kind: "string", Cardinality: "one", Internal: true}, + }, + RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, + Plan: exec02TestPlan(exec02TestProjections(true)), + } + resolved := Resolved{ + Bundle: recipe.Bundle{Name: "pg_dispatch"}, + Semantic: semantic.ResolvedRecipePlan{ + SemanticPlan: semantic.RecipePlan{Bindings: recipe.RuntimeBindings{ + Project: "P1", DatasetGeneration: "G1", OutputNames: []string{"resources"}, + IncludeSourceIdentity: true, + }}, + }, + Compiled: lower.CompiledRecipe{Outputs: []lower.CompiledRecipeOutput{output}}, + StoredRecipeDigest: "pg-source-identity", + } + rows := []map[string]any{ + {"id": "patient-one", "name": "Ada", sourceID: "Patient/p1"}, + } + engine, _ := exec02pgEngine(t, spies, rows) + var public []map[string]any + if _, err := engine.PreviewOutput(context.Background(), resolved, + PreviewRequest{Output: "resources", Limit: 5}, func(row map[string]any) error { + public = append(public, row) + return nil + }); err != nil { + t.Fatalf("PreviewOutput: %v", err) + } + if len(public) != 1 { + t.Fatalf("visited=%d want 1", len(public)) + } + source, ok := public[0][previewSourceMetadataKey].(map[string]any) + if !ok { + t.Fatalf("preview row is missing the source metadata key: %v", public[0]) + } + if source["kind"] != "SINGLE" || source["resourceType"] != "Patient" || source["id"] != "Patient/p1" { + t.Fatalf("PostgreSQL preview source identity must be SINGLE with the root resource type, got %v (RootResourceType dropped from the compiled query?)", source) + } + if _, leaked := public[0][sourceID]; leaked { + t.Fatalf("internal source-id column leaked into the public row: %v", public[0]) + } +} + +func TestPostgreSQLPreviewPostLoopCancellationNeverMarksComplete(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, []map[string]any{ + {"id": "patient-one", "name": "Ada"}, + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + summary, err := engine.PreviewOutput(ctx, exec02TestResolved(), + PreviewRequest{Output: "resources", Limit: 5}, func(map[string]any) error { + cancel() + return nil + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("post-exhaustion cancellation must report the context cause: %v", err) + } + if summary.Complete || summary.Truncated { + t.Fatalf("post-loop cancelled preview emitted a completion marker: %+v", summary) + } +} + +func TestPostgreSQLStreamCarriesNoArangoOnlyPolicy(t *testing.T) { + spies := &dispatchSpies{} + reference := exec02PostgreSQLCommand(t) + var observed RowsRequest + budgetSawPostgreSQL := false + pgStream := OutputStream{ + Name: "resources", Columns: []string{"id", "name"}, + sourceRows: func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + observed = request + if _, _, err := request.Command.PostgreSQLParameters(); err != nil { + return err + } + if _, ok := QueryMemoryBudgetFromContext(ctx); ok { + budgetSawPostgreSQL = true + } + for _, row := range exec02Rows() { + if err := visit(row); err != nil { + return err + } + } + return nil + }, + command: reference, + commandColumns: []string{"id", "name"}, + batchSize: 64, + queryMemoryLimit: 1 << 30, + } + if err := pgStream.streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("PostgreSQL streamRaw: %v", err) + } + if budgetSawPostgreSQL { + t.Fatal("query-memory budget context attached to a PostgreSQL stream") + } + if observed.BatchSize != 64 { + t.Fatalf("batch hint = %d, want 64", observed.BatchSize) + } + if _, _, err := observed.Command.PostgreSQLParameters(); err != nil { + t.Fatalf("forwarded command must be the tagged PostgreSQL statement: %v", err) + } + + budgetSawAQL := false + aqlStream := OutputStream{ + Name: "resources", Columns: []string{"id"}, + sourceRows: func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + if _, _, err := request.Command.AQLParameters(); err != nil { + return err + } + _, budgetSawAQL = QueryMemoryBudgetFromContext(ctx) + return visit(map[string]any{"id": "patient-one"}) + }, + command: mustAQLCommand("FOR d IN Patient RETURN d", map[string]any{"project": "P1"}), + commandColumns: []string{"id"}, + queryMemoryLimit: 1 << 30, + } + if err := aqlStream.streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("AQL streamRaw: %v", err) + } + if !budgetSawAQL { + t.Fatal("control failed: the AQL stream stopped attaching the query-memory budget") + } + exec02AssertUntouchedArango(t, spies) +} + +func exec02StableRowIdentity(parts ...any) string { + encoded, err := json.Marshal(parts) + if err != nil { + panic(err) + } + sum := sha256.Sum256(encoded) + return hex.EncodeToString(sum[:]) +} + +func TestPostgreSQLStreamRowIdentityAndPublicColumns(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, exec02Rows()) + streams, err := engine.Streams(context.Background(), exec02TestResolved()) + if err != nil { + t.Fatalf("Streams: %v", err) + } + want := exec02StableRowIdentity("patient-one") + var got []map[string]any + if _, err := streams[0].Stream(context.Background(), func(row map[string]any) error { + got = append(got, row) + return nil + }); err != nil { + t.Fatalf("Stream: %v", err) + } + if got[0]["__loom_row_id"] != want { + t.Fatalf("stable row identity = %v, want %s", got[0]["__loom_row_id"], want) + } + for _, row := range got { + for column := range row { + switch column { + case "id", "name", "__loom_row_id": + default: + t.Fatalf("PostgreSQL stream row carries unexpected column %q: %v", column, row) + } + } + } +} diff --git a/internal/dataframe/execution/engine_run_output_class_test.go b/internal/dataframe/execution/engine_run_output_class_test.go index ae0c36d86..d99ddf48d 100644 --- a/internal/dataframe/execution/engine_run_output_class_test.go +++ b/internal/dataframe/execution/engine_run_output_class_test.go @@ -3,6 +3,7 @@ package execution import ( "context" "errors" + "github.com/calypr/loom/internal/dataframe/compiler" "strings" "testing" @@ -29,8 +30,15 @@ func newRunTestFixture(t *testing.T) (*Engine, *runOutputRegistry) { t.Helper() registry := &runOutputRegistry{} engine, err := New(Config{ - Registry: registry, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: registry, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return errors.New("query executor must not be consulted") }, }) diff --git a/internal/dataframe/execution/engine_test.go b/internal/dataframe/execution/engine_test.go index 2097277c4..4bdc023e8 100644 --- a/internal/dataframe/execution/engine_test.go +++ b/internal/dataframe/execution/engine_test.go @@ -2,20 +2,25 @@ package execution import ( "context" + "encoding/json" "errors" "fmt" + "reflect" "strings" "testing" "time" "github.com/calypr/loom/internal/authscope" "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" + postgresrender "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" "github.com/calypr/loom/internal/dataframe/execution/chartifact" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/recipe/exec" + "github.com/calypr/loom/internal/dataframe/spec" ) func TestPlanHasExecutionWindowRecognizesStandaloneGroupRowsLimit(t *testing.T) { @@ -35,8 +40,15 @@ func TestPlanHasExecutionWindowRecognizesStandaloneGroupRowsLimit(t *testing.T) func TestCompiledArtifactScopeKeepsBoundedGroupRowsPreviewWhole(t *testing.T) { engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil }, }) @@ -61,7 +73,7 @@ func TestCompiledArtifactScopeKeepsBoundedGroupRowsPreviewWhole(t *testing.T) { if err != nil || unboundedMode != chartifact.ScopeModeWhole || unboundedEvidence == nil || unboundedIdentity == nil { t.Fatalf("complete GroupRows source scope=(%q,%v,%v), err=%v; want whole-relation capture evidence", unboundedMode, unboundedEvidence != nil, unboundedIdentity != nil, err) } - preview, err := compiler.CompileRecipeOutputWithPolicy(output, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + preview, err := compiler.CompileRecipeOutput(compiler.ArangoTarget(), output, bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile bounded GroupRows preview: %v", err) } @@ -76,8 +88,15 @@ func TestCompiledArtifactScopeKeepsBoundedGroupRowsPreviewWhole(t *testing.T) { func TestCompiledArtifactScopeAcceptsAuthorizedProjectAliasesAndRejectsScopeDrift(t *testing.T) { engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil }, }) @@ -165,8 +184,15 @@ func (r invalidRecipeRegistry) LoadRecipeVersion(ctx context.Context, name, _ st func TestMaterializeMarksResolutionFailures(t *testing.T) { engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil }, }) @@ -207,9 +233,17 @@ func TestCohortPreviewDoesNotInventSingleSourceIdentityOrPageItsInput(t *testing } { t.Run(test.name, func(t *testing.T) { engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, - ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }, RootPageRows: test.rootPageRows, }) if err != nil { @@ -233,17 +267,17 @@ func TestCohortPreviewDoesNotInventSingleSourceIdentityOrPageItsInput(t *testing if !foundCohortStage { t.Fatal("cohort output has no typed COHORT_GROUP stage") } - stream, query, err := engine.streamForOutput(resolved, "Cohort", 25) + stream, query, err := engine.streamForOutput(context.Background(), resolved, "Cohort", 25) if err != nil { t.Fatalf("compile cohort preview plan: %v", err) } if stream.page != nil { t.Fatal("cohort preview paged roots before completing the group aggregation") } - if strings.Contains(query.Query, "__loom_source_resource_id") { + if strings.Contains(commandText(t, query.Command), "__loom_source_resource_id") { t.Fatal("cohort preview assigned one scalar source identity to a multi-member group") } - if !strings.Contains(query.Query, "group_revision_id") { + if !strings.Contains(commandText(t, query.Command), "group_revision_id") { t.Fatal("cohort preview query omitted its grouped-row identity") } }) @@ -279,11 +313,11 @@ func testCohortCompositionBundle() recipe.Bundle { } } -func testEngine(queryRows QueryRows) *Engine { - engine, err := New(Config{ +func testEngine(queryRows SourceRows) *Engine { + engine, err := New(Config{CompileTarget: compiler.ArangoTarget(), Registry: invalidRecipeRegistry{}, ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, - QueryRows: queryRows, + SourceRows: queryRows, }) if err != nil { panic(err) @@ -292,7 +326,13 @@ func testEngine(queryRows QueryRows) *Engine { } func TestPreviewOutputFiltersInternalColumnsAndReturnsSafePlanSummary(t *testing.T) { - e := testEngine(func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + e := testEngine(func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{ "id": "p1", "_key": "internal-key", @@ -324,9 +364,16 @@ func TestPreviewOutputFiltersInternalColumnsAndReturnsSafePlanSummary(t *testing func TestPreviewOutputCarriesFHIRSourceWithoutVisibleRootID(t *testing.T) { sourceIDProjected := false e, err := New(Config{ - Registry: invalidRecipeRegistry{}, - ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, - QueryRows: func(_ context.Context, _ string, _ int, bindVars map[string]any, visit func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, + SourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, bindVars, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if _, isSelectedRootQuery := bindVars[compiler.RootPageKeysBind]; isSelectedRootQuery { for _, value := range bindVars { if value == ir.PreviewSourceResourceIDColumn { @@ -445,7 +492,15 @@ func TestCodedGroupPreviewRunsAgainstWholeInput(t *testing.T) { }}}, }}, } - engine, err := New(Config{Registry: invalidRecipeRegistry{}, QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, ScopeDigest: func(recipe.RuntimeBindings) string { return "coded-group" }, RootPageRows: 1}) + engine, err := New(Config{ + CompileTarget: compiler.ArangoTarget(), Registry: invalidRecipeRegistry{}, SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }, ScopeDigest: func(recipe.RuntimeBindings) string { return "coded-group" }, RootPageRows: 1}) if err != nil { t.Fatal(err) } @@ -453,17 +508,17 @@ func TestCodedGroupPreviewRunsAgainstWholeInput(t *testing.T) { if err != nil { t.Fatal(err) } - stream, query, err := engine.streamForOutput(resolved, "body_structures", 25) + stream, query, err := engine.streamForOutput(context.Background(), resolved, "body_structures", 25) if err != nil { t.Fatal(err) } if stream.page != nil { t.Fatal("CODED_GROUP must scan the complete input before counting distinct source records") } - groupAt := strings.Index(query.Query, "COLLECT __loom_construction_coded_group_system") - limitAt := strings.Index(query.Query, "LIMIT @limit") - if !strings.Contains(query.Query, "FOR root IN @@root_collection") || groupAt < 0 || limitAt <= groupAt { - t.Fatalf("full-input CODED_GROUP query has an unexpected row scope:\n%s", query.Query) + groupAt := strings.Index(commandText(t, query.Command), "COLLECT __loom_construction_coded_group_system") + limitAt := strings.Index(commandText(t, query.Command), "LIMIT @limit") + if !strings.Contains(commandText(t, query.Command), "FOR root IN @@root_collection") || groupAt < 0 || limitAt <= groupAt { + t.Fatalf("full-input CODED_GROUP query has an unexpected row scope:\n%s", commandText(t, query.Command)) } if stream.sourceIdentityMode != previewSourceComposite { t.Fatalf("CODED_GROUP source mode = %q, want COMPOSITE", stream.sourceIdentityMode) @@ -472,19 +527,31 @@ func TestCodedGroupPreviewRunsAgainstWholeInput(t *testing.T) { func TestPreviewOutputUsesPreviewExecutorWithoutChangingOrdinaryStreams(t *testing.T) { var ordinaryCalls, previewCalls int - ordinaryQuery := func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + ordinaryQuery := func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + ordinaryCalls++ return visit(map[string]any{"id": "ordinary", "_key": "ordinary-key"}) } - previewQuery := func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + previewQuery := func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + previewCalls++ return visit(map[string]any{"id": "preview"}) } - e, err := New(Config{ - Registry: invalidRecipeRegistry{}, - ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, - QueryRows: ordinaryQuery, - PreviewQueryRows: previewQuery, + e, err := New(Config{CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, + SourceRows: ordinaryQuery, + PreviewSourceRows: previewQuery, }) if err != nil { t.Fatal(err) @@ -499,7 +566,7 @@ func TestPreviewOutputUsesPreviewExecutorWithoutChangingOrdinaryStreams(t *testi if previewCalls != 1 || ordinaryCalls != 0 { t.Fatalf("after preview: preview calls=%d ordinary calls=%d", previewCalls, ordinaryCalls) } - stream, _, err := e.streamForOutput(resolved, "Patient", 1) + stream, _, err := e.streamForOutput(context.Background(), resolved, "Patient", 1) if err != nil { t.Fatal(err) } @@ -516,10 +583,24 @@ func TestPreviewOutputDefersOnlyGroupIndexPreparation(t *testing.T) { queryFinished := make(chan struct{}) prepared := make(chan bool, 1) groupEngine, err := New(Config{ - Registry: invalidRecipeRegistry{}, - ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, - PreviewQueryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }, + PreviewSourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + defer close(queryFinished) return visit(map[string]any{"status": "final", "rows": 1}) }, @@ -559,10 +640,16 @@ func TestPreviewOutputDefersOnlyGroupIndexPreparation(t *testing.T) { for _, test := range []struct { name string - rows QueryRows + rows SourceRows cancelDuringQuery bool }{ - {name: "query failure", rows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + {name: "query failure", rows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return errors.New("query failed") }}, {name: "canceled", cancelDuringQuery: true}, @@ -573,16 +660,30 @@ func TestPreviewOutputDefersOnlyGroupIndexPreparation(t *testing.T) { defer cancel() rows := test.rows if test.cancelDuringQuery { - rows = func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + rows = func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + cancel() return context.Canceled } } engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, - ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, - PreviewQueryRows: rows, + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }, + PreviewSourceRows: rows, PreparePreviewIndex: func(context.Context, compiler.PreviewCoveringIndexSpec) error { prepareCalls <- struct{}{} return nil @@ -614,14 +715,28 @@ func TestPreviewOutputDefersOnlyGroupIndexPreparation(t *testing.T) { pivotOutput := executionPivotPreviewOutput() preparedBeforeQuery := false pivotEngine, err := New(Config{ - Registry: invalidRecipeRegistry{}, - ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }, PreparePreviewIndex: func(context.Context, compiler.PreviewCoveringIndexSpec) error { preparedBeforeQuery = true return nil }, - PreviewQueryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + PreviewSourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if !preparedBeforeQuery { return errors.New("Pivot query started before its index preparation") } @@ -695,7 +810,13 @@ func executionPivotPreviewOutput() recipe.Output { } func TestPreviewOutputCanExposeStableRowIdentityForComparisonSinks(t *testing.T) { - e := testEngine(func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + e := testEngine(func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{"id": "p1", "__loom_row_id": "stable-p1"}) }) resolved, err := e.CompileResolvedBundle(context.Background(), testResolvedBundle([]string{}), recipe.RuntimeBindings{Project: "P1", IncludeRowIdentity: true}) @@ -716,7 +837,13 @@ func TestPreviewOutputCanExposeStableRowIdentityForComparisonSinks(t *testing.T) func TestPreviewOutputExecutesOnlyRequestedOutputAndRejectsUnknownBeforeQuery(t *testing.T) { queries := 0 - e := testEngine(func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + e := testEngine(func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + queries++ return visit(map[string]any{"id": "p1"}) }) @@ -747,7 +874,13 @@ func TestPreviewOutputExecutesOnlyRequestedOutputAndRejectsUnknownBeforeQuery(t func TestPreviewOutputEnforcesLimitAndCancellation(t *testing.T) { rowsSeen := 0 - e := testEngine(func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + e := testEngine(func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + for index := 0; index < 10; index++ { rowsSeen++ if err := visit(map[string]any{"id": fmt.Sprintf("p%d", index)}); err != nil { @@ -780,7 +913,13 @@ func TestPreviewOutputEnforcesLimitAndCancellation(t *testing.T) { t.Fatalf("canceled preview error = %v, want CLIENT_CANCELED", err) } - cancelEngine := testEngine(func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + cancelEngine := testEngine(func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if err := visit(map[string]any{"id": "p1"}); err != nil { return err } @@ -802,7 +941,13 @@ func TestPreviewOutputEnforcesLimitAndCancellation(t *testing.T) { } func TestPreviewOutputMarksNaturalExhaustionComplete(t *testing.T) { - e := testEngine(func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + e := testEngine(func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{"id": "p1"}) }) resolved, err := e.CompileResolvedBundle(context.Background(), testResolvedBundle([]string{}), recipe.RuntimeBindings{Project: "P1"}) @@ -822,7 +967,13 @@ func TestRootKeyPagingPreservesExpandedRowsAndAdvancesPastEmptyRoots(t *testing. type calls struct{ keys, rows int } newPagedEngine := func(observed *calls) *Engine { t.Helper() - queryRows := func(_ context.Context, _ string, _ int, binds map[string]any, visit func(map[string]any) error) error { + queryRows := func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, binds, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if afterValue, ok := binds[compiler.RootPageAfterKeyBind]; ok { observed.keys++ after, _ := afterValue.(string) @@ -860,7 +1011,7 @@ func TestRootKeyPagingPreservesExpandedRowsAndAdvancesPastEmptyRoots(t *testing. } return nil } - engine, err := New(Config{Registry: invalidRecipeRegistry{}, QueryRows: queryRows, ScopeDigest: func(recipe.RuntimeBindings) string { return "scope" }, RootPageRows: 2}) + engine, err := New(Config{CompileTarget: compiler.ArangoTarget(), Registry: invalidRecipeRegistry{}, SourceRows: queryRows, ScopeDigest: func(recipe.RuntimeBindings) string { return "scope" }, RootPageRows: 2}) if err != nil { t.Fatal(err) } @@ -917,7 +1068,13 @@ func TestRootKeyPagingPreservesExpandedRowsAndAdvancesPastEmptyRoots(t *testing. func TestPreviewRootPageRowsAreClampedToRequestedLimit(t *testing.T) { var pageSizes []int - queryRows := func(_ context.Context, _ string, _ int, binds map[string]any, visit func(map[string]any) error) error { + queryRows := func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, binds, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if _, ok := binds[compiler.RootPageAfterKeyBind]; ok { pageSize, ok := binds[compiler.RootPageSizeBind].(int) if !ok { @@ -944,8 +1101,8 @@ func TestPreviewRootPageRowsAreClampedToRequestedLimit(t *testing.T) { } return nil } - engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, QueryRows: queryRows, + engine, err := New(Config{CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, SourceRows: queryRows, ScopeDigest: func(recipe.RuntimeBindings) string { return "scope" }, RootPageRows: 250, }) if err != nil { @@ -969,7 +1126,13 @@ func TestPreviewRootPageRowsAreClampedToRequestedLimit(t *testing.T) { } func TestPreviewOutputNormalizesVisitorAndDynamicSchemaErrors(t *testing.T) { - driftEngine := testEngine(func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + driftEngine := testEngine(func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{ "id": "p1", "value": "v", @@ -987,7 +1150,13 @@ func TestPreviewOutputNormalizesVisitorAndDynamicSchemaErrors(t *testing.T) { } visitorErr := errors.New("visitor failed") - visitorEngine := testEngine(func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + visitorEngine := testEngine(func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{"id": "p1"}) }) resolved, err = visitorEngine.CompileResolvedBundle(context.Background(), testResolvedBundle([]string{}), recipe.RuntimeBindings{Project: "P1"}) @@ -1056,13 +1225,20 @@ func TestPreviewLimitsHaveStableDefaultAndMaximum(t *testing.T) { func TestCompileResolvedBundleSkipsCatalogResolverAndRetainsBundle(t *testing.T) { resolverCalls := 0 e, err := New(Config{ - Registry: invalidRecipeRegistry{}, + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, ResolveBundle: func(context.Context, recipe.Bundle, recipe.RuntimeBindings) (recipe.Bundle, error) { resolverCalls++ return recipe.Bundle{}, errors.New("catalog resolver must not be called") }, ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil }, }) @@ -1095,13 +1271,20 @@ func TestCompileResolvedBundleSkipsCatalogResolverAndRetainsBundle(t *testing.T) func TestResolveBundleUsesResolverButResolvedPreviewAndMaterializeDoNot(t *testing.T) { resolverCalls := 0 e, err := New(Config{ - Registry: invalidRecipeRegistry{}, + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, ResolveBundle: func(_ context.Context, bundle recipe.Bundle, _ recipe.RuntimeBindings) (recipe.Bundle, error) { resolverCalls++ return bundle, nil }, ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, - QueryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + SourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{"id": "p1"}) }, }) @@ -1129,7 +1312,7 @@ func TestResolveBundleUsesResolverButResolvedPreviewAndMaterializeDoNot(t *testi if err != nil { t.Fatal(err) } - if len(normalStreams) != len(directStreams) || normalStreams[0].query != directStreams[0].query { + if len(normalStreams) != len(directStreams) || !reflect.DeepEqual(normalStreams[0].command, directStreams[0].command) { t.Fatalf("resolved output differs from catalog-resolved output") } if _, err := e.PreviewResolvedBundle(context.Background(), bundle, bindings); err != nil { @@ -1145,9 +1328,16 @@ func TestResolveBundleUsesResolverButResolvedPreviewAndMaterializeDoNot(t *testi func TestCompileResolvedBundleRejectsUnresolvedDynamicDeclarations(t *testing.T) { e, err := New(Config{ - Registry: invalidRecipeRegistry{}, - ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "test-scope" }, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil }, }) @@ -1172,8 +1362,22 @@ func TestAdaptiveRelatedPreviewPagesPreserveSparseRoots(t *testing.T) { keys := []string{"a", "b", "c", "d", "e", "f", "g", "h"} var sizes []int var visited []string - stream := OutputStream{rootPageRows: 4, initialRootPageRows: 1, page: &compiler.CompiledOutputPage{}} - stream.stream = func(_ context.Context, _ string, _ int, binds map[string]any, visit func(map[string]any) error) error { + stream := OutputStream{ + rootPageRows: 4, + initialRootPageRows: 1, + command: mustAQLCommand("FOR root IN Observation RETURN root", nil), + page: &compiler.CompiledOutputPage{ + RootKeys: mustAQLCommand("ROOT-KEYS", map[string]any{compiler.RootPageAfterKeyBind: "", compiler.RootPageSizeBind: 0}), + Rows: mustAQLCommand("ROWS", map[string]any{compiler.RootPageKeysBind: []string{}}), + }, + } + stream.sourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, binds, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if after, ok := binds[compiler.RootPageAfterKeyBind].(string); ok { size := binds[compiler.RootPageSizeBind].(int) sizes = append(sizes, size) @@ -1207,3 +1411,707 @@ func TestAdaptiveRelatedPreviewPagesPreserveSparseRoots(t *testing.T) { t.Fatalf("visited=%v rows=%v page sizes=%v", visited, rows, sizes) } } + +func newExec05PageCommand(t *testing.T, sql string, args []command.PostgreSQLParameter) command.CompiledCommand { + t.Helper() + created, err := command.NewPostgreSQLCommand(sql, args) + if err != nil { + t.Fatalf("canonical page-step command: %v", err) + } + if created.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("canonical page-step command tag = %q, want PostgreSQL", created.Backend()) + } + return created +} + +const ( + exec05KeySQL = "SELECT r.logical_id AS root_key FROM loom.fhir_resource r WHERE r.project = $1 AND r.logical_id > $2 ORDER BY r.logical_id ASC LIMIT $3" + exec05RowsSQL = "SELECT r.logical_id AS root_key, e.edge_seq AS child_seq, c.logical_id AS child_id FROM loom.fhir_resource r WHERE r.logical_id IN (SELECT k FROM jsonb_array_elements_text($2) AS k) ORDER BY r.logical_id ASC, e.edge_seq ASC, c.logical_id ASC" +) + +func exec05KeyArgs(cursor string, size int64) []command.PostgreSQLParameter { + return []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "P1"}, + {Type: command.PostgreSQLParameterText, Value: cursor}, + {Type: command.PostgreSQLParameterBigint, Value: size}, + } +} + +func exec05RowsArgs() []command.PostgreSQLParameter { + return []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "P1"}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`[]`)}, + } +} + +func newExec05RootPageSteps(t *testing.T, pageSize int) *PostgreSQLRootPageSteps { + t.Helper() + return &PostgreSQLRootPageSteps{ + RootKeys: newExec05PageCommand(t, exec05KeySQL, exec05KeyArgs("", 2)), + Rows: newExec05PageCommand(t, exec05RowsSQL, exec05RowsArgs()), + RootKeyColumn: "root_key", + RowRootKeyColumn: "root_key", + CursorArgIndex: 1, + PageSizeArgIndex: 2, + PageKeysArgIndex: 1, + PageSize: pageSize, + } +} + +var exec05OfflineRoots = []string{"r1", "r2", "r3", "r4", "r5"} + +func exec05OfflineRowsFor(root string) []map[string]any { + switch root { + case "r1": + return []map[string]any{ + {"root_key": "r1", "child_seq": int64(1), "child_id": "x1"}, + {"root_key": "r1", "child_seq": int64(2), "child_id": "x2"}, + } + case "r2": + return []map[string]any{{"root_key": "r2", "child_seq": int64(3), "child_id": "x2"}} + case "r3": + return []map[string]any{{"root_key": "r3", "child_seq": int64(4), "child_id": "x3"}} + case "r4": + return nil + case "r5": + return []map[string]any{ + {"root_key": "r5", "child_seq": int64(5), "child_id": "x4"}, + {"root_key": "r5", "child_seq": int64(6), "child_id": "x5"}, + } + } + return nil +} + +type exec05Recorder struct { + calls []exec05PageCall + sabotageKeys []string + sabotageRows []map[string]any + cancelAfterKey int + visitCount int + cancel context.CancelFunc + keyPageOverride func(keyCallIndex int, served []string) []string +} + +type exec05PageCall struct { + kind string + sql string + args []command.PostgreSQLParameter + columns []string +} + +func (rec *exec05Recorder) port(ctx context.Context, request RowsRequest, visit RowVisitor) error { + sqlText, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + single := len(request.DeclaredColumns) == 1 && request.DeclaredColumns[0] == "root_key" && !strings.Contains(sqlText, "child_seq") + kind := "rows" + if single { + kind = "keys" + } + rec.calls = append(rec.calls, exec05PageCall{kind: kind, sql: sqlText, args: args, columns: append([]string(nil), request.DeclaredColumns...)}) + if kind == "keys" { + cursor, _ := args[1].Value.(string) + size, _ := args[2].Value.(int64) + served := rec.sabotageKeys + if served == nil { + for _, root := range exec05OfflineRoots { + if int64(len(served)) >= size { + break + } + if root > cursor { + served = append(served, root) + } + } + } + if rec.keyPageOverride != nil { + served = rec.keyPageOverride(countCalls(rec.calls, "keys"), served) + } + for _, key := range served { + if err := visit(map[string]any{"root_key": key}); err != nil { + return err + } + } + if rec.cancel != nil && rec.cancelAfterKey > 0 && countCalls(rec.calls, "keys") == rec.cancelAfterKey { + rec.cancel() + } + return nil + } + if ctx.Err() != nil { + return errors.New("row step ran after cancellation") + } + if rec.sabotageRows != nil { + for _, row := range rec.sabotageRows { + rec.visitCount++ + if err := visit(row); err != nil { + return err + } + } + return nil + } + var keys []string + if err := json.Unmarshal(args[1].Value.(json.RawMessage), &keys); err != nil { + return fmt.Errorf("row step keys argument: %w", err) + } + for _, key := range keys { + for _, row := range exec05OfflineRowsFor(key) { + rec.visitCount++ + if err := visit(row); err != nil { + return err + } + } + } + return nil +} + +func countCalls(calls []exec05PageCall, kind string) int { + count := 0 + for _, call := range calls { + if call.kind == kind { + count++ + } + } + return count +} + +func exec05Stream(t *testing.T, steps *PostgreSQLRootPageSteps, port SourceRows) OutputStream { + t.Helper() + return OutputStream{ + Name: "expanded", Columns: []string{"root_key", "child_seq", "child_id"}, + RowIdentity: &spec.RowIdentity{Fields: []string{"root_key", "child_seq", "child_id"}}, + command: newExec05PageCommand(t, "SELECT 1", nil), + expectedBackend: command.CompiledBackendPostgreSQL, + commandColumns: []string{"root_key", "child_seq", "child_id"}, + sourceRows: port, + batchSize: 100, + pgRootPage: steps, + } +} + +func TestPostgreSQLRootPageLoopOfflineCore(t *testing.T) { + rec := &exec05Recorder{} + steps := newExec05RootPageSteps(t, 2) + stream := exec05Stream(t, steps, rec.port) + var visited []string + if err := stream.streamRaw(context.Background(), func(row map[string]any) error { + visited = append(visited, fmt.Sprintf("%v/%v/%v", row["root_key"], row["child_seq"], row["child_id"])) + return nil + }); err != nil { + t.Fatalf("paged streamRaw: %v", err) + } + want := []string{ + "r1/1/x1", "r1/2/x2", "r2/3/x2", + "r3/4/x3", + "r5/5/x4", "r5/6/x5", + } + if fmt.Sprint(visited) != fmt.Sprint(want) { + t.Fatalf("visited=%v want=%v", visited, want) + } + if got := countCalls(rec.calls, "keys"); got != 3 { + t.Fatalf("key steps = %d, want 3 (one short final page)", got) + } + if got := countCalls(rec.calls, "rows"); got != 3 { + t.Fatalf("row steps = %d, want 3", got) + } + var cursors []string + var keyPages []string + for _, call := range rec.calls { + if call.kind == "keys" { + cursors = append(cursors, call.args[1].Value.(string)) + if call.args[2].Value.(int64) != 2 { + t.Fatalf("key-step page size = %v, want the fixed 2", call.args[2].Value) + } + if call.sql != exec05KeySQL { + t.Fatalf("key-step SQL drifted from the parent template") + } + if call.args[0].Value != "P1" || call.args[0].Type != command.PostgreSQLParameterText { + t.Fatalf("key-step scope argument was rebound: %+v", call.args[0]) + } + if call.columns[0] != "root_key" || len(call.columns) != 1 { + t.Fatalf("key-step declared contract = %v", call.columns) + } + } else { + keyPages = append(keyPages, string(call.args[1].Value.(json.RawMessage))) + if call.sql != exec05RowsSQL { + t.Fatalf("row-step SQL drifted from the parent template") + } + if call.args[0].Value != "P1" { + t.Fatalf("row-step scope argument was rebound: %+v", call.args[0]) + } + if strings.Join(call.columns, ",") != "root_key,child_seq,child_id" { + t.Fatalf("row-step declared contract = %v", call.columns) + } + } + } + if fmt.Sprint(cursors) != fmt.Sprint([]string{"", "r2", "r4"}) { + t.Fatalf("cursors=%v want [ r2 r4]", cursors) + } + wantPages := []string{`["r1","r2"]`, `["r3","r4"]`, `["r5"]`} + if fmt.Sprint(keyPages) != fmt.Sprint(wantPages) { + t.Fatalf("key pages=%v want=%v", keyPages, wantPages) + } +} + +func TestPostgreSQLRootPageTerminationOffline(t *testing.T) { + t.Run("short final page ends without another key query", func(t *testing.T) { + rec := &exec05Recorder{} + stream := exec05Stream(t, newExec05RootPageSteps(t, 2), rec.port) + if err := stream.streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatal(err) + } + if keys, rows := countCalls(rec.calls, "keys"), countCalls(rec.calls, "rows"); keys != 3 || rows != 3 { + t.Fatalf("calls keys=%d rows=%d, want 3/3 (no fourth key query after the short page)", keys, rows) + } + }) + t.Run("empty final page ends without any row call", func(t *testing.T) { + rec := &exec05Recorder{} + steps := newExec05RootPageSteps(t, 5) + stream := exec05Stream(t, steps, rec.port) + if err := stream.streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatal(err) + } + if keys, rows := countCalls(rec.calls, "keys"), countCalls(rec.calls, "rows"); keys != 2 || rows != 1 { + t.Fatalf("calls keys=%d rows=%d, want 2/1 (empty key page issues no row query)", keys, rows) + } + }) + t.Run("empty first page issues no row call at all", func(t *testing.T) { + rec := &exec05Recorder{sabotageKeys: []string{}} + stream := exec05Stream(t, newExec05RootPageSteps(t, 2), rec.port) + if err := stream.streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatal(err) + } + if keys, rows := countCalls(rec.calls, "keys"), countCalls(rec.calls, "rows"); keys != 1 || rows != 0 { + t.Fatalf("calls keys=%d rows=%d, want 1/0", keys, rows) + } + }) +} + +func TestPostgreSQLRootPageResultContractSabotageOffline(t *testing.T) { + newStream := func(t *testing.T, rec *exec05Recorder) OutputStream { + t.Helper() + return exec05Stream(t, newExec05RootPageSteps(t, 2), rec.port) + } + cases := []struct { + name string + rec *exec05Recorder + wantKeys int + wantRows int + }{ + {"duplicate root key", &exec05Recorder{sabotageKeys: []string{"r1", "r1"}}, 1, 0}, + {"non-monotone root keys", &exec05Recorder{sabotageKeys: []string{"r2", "r1"}}, 1, 0}, + {"blank root key", &exec05Recorder{sabotageKeys: []string{"r1", ""}}, -1, -1}, + {"key page larger than requested", &exec05Recorder{sabotageKeys: []string{"r1", "r2", "r3"}}, -1, -1}, + {"row outside the requested page", &exec05Recorder{sabotageRows: []map[string]any{{"root_key": "r9", "child_seq": int64(1), "child_id": "x9"}}}, -1, -1}, + {"row missing its root identity", &exec05Recorder{sabotageRows: []map[string]any{{"child_seq": int64(1), "child_id": "x1"}}}, -1, -1}, + {"row root identity not a string", &exec05Recorder{sabotageRows: []map[string]any{{"root_key": 7, "child_seq": int64(1), "child_id": "x1"}}}, -1, -1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := newStream(t, tc.rec).streamRaw(context.Background(), func(map[string]any) error { return nil }) + if !errors.Is(err, ErrPostgreSQLResultContract) { + t.Fatalf("error = %v, want ErrPostgreSQLResultContract", err) + } + if tc.wantKeys >= 0 { + if got := countCalls(tc.rec.calls, "keys"); got != tc.wantKeys { + t.Fatalf("key calls = %d, want %d: refusal must land during key collection", got, tc.wantKeys) + } + } + if tc.wantRows >= 0 { + if got := countCalls(tc.rec.calls, "rows"); got != tc.wantRows { + t.Fatalf("row calls = %d, want %d: the refusing page must never dispatch its row step", got, tc.wantRows) + } + } + }) + t.Run(tc.name+" control proceeds", func(t *testing.T) { + rec := &exec05Recorder{} + if err := newStream(t, rec).streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("paired control must proceed: %v", err) + } + if rec.visitCount != 6 { + t.Fatalf("control visited %d rows, want 6", rec.visitCount) + } + }) + } + t.Run("key page does not advance past the previous page tail", func(t *testing.T) { + rec := &exec05Recorder{keyPageOverride: func(keyCallIndex int, served []string) []string { + if keyCallIndex == 2 { + return []string{"r2", "r3"} + } + return served + }} + err := newStream(t, rec).streamRaw(context.Background(), func(map[string]any) error { return nil }) + if !errors.Is(err, ErrPostgreSQLResultContract) { + t.Fatalf("error = %v, want ErrPostgreSQLResultContract for a page re-touching the previous tail", err) + } + if got := countCalls(rec.calls, "keys"); got != 2 { + t.Fatalf("key calls = %d, want 2 (refusal lands during collection of the second page)", got) + } + if got := countCalls(rec.calls, "rows"); got != 1 { + t.Fatalf("row calls = %d, want 1: the refusing page must never reach the row step", got) + } + }) + t.Run("parent cursor ignored keeps re-serving the first page", func(t *testing.T) { + rec := &exec05Recorder{keyPageOverride: func(int, []string) []string { return []string{"r1", "r2"} }} + err := newStream(t, rec).streamRaw(context.Background(), func(map[string]any) error { return nil }) + if !errors.Is(err, ErrPostgreSQLResultContract) { + t.Fatalf("error = %v, want ErrPostgreSQLResultContract for a cursor-ignoring parent key statement", err) + } + if got := countCalls(rec.calls, "keys"); got != 2 { + t.Fatalf("key calls = %d, want 2: overlapping full pages must not continue past the refusal", got) + } + if got := countCalls(rec.calls, "rows"); got != 1 { + t.Fatalf("row calls = %d, want 1: already-emitted rows must not be visited again", got) + } + }) + t.Run("control: identity key-page override proceeds", func(t *testing.T) { + rec := &exec05Recorder{keyPageOverride: func(_ int, served []string) []string { return served }} + if err := newStream(t, rec).streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("paired control: %v", err) + } + if rec.visitCount != 6 { + t.Fatalf("control visited %d rows, want 6", rec.visitCount) + } + }) +} + +func TestPostgreSQLRootPageWithinPageStrictnessOffline(t *testing.T) { + cases := []struct { + name string + override func(keyCallIndex int, served []string) []string + wantKeys int + wantRows int + }{ + { + name: "unordered first page is refused before any row step", + override: func(keyCallIndex int, served []string) []string { + if keyCallIndex == 1 { + return []string{"r2", "r1"} + } + return served + }, + wantKeys: 1, + wantRows: 0, + }, + { + name: "unordered but advancing second page is refused before its row step", + override: func(keyCallIndex int, served []string) []string { + if keyCallIndex == 2 { + return []string{"r4", "r3"} + } + return served + }, + wantKeys: 2, + wantRows: 1, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec := &exec05Recorder{keyPageOverride: tc.override} + err := exec05Stream(t, newExec05RootPageSteps(t, 2), rec.port).streamRaw(context.Background(), func(map[string]any) error { return nil }) + if !errors.Is(err, ErrPostgreSQLResultContract) { + t.Fatalf("error = %v, want ErrPostgreSQLResultContract from the within-page strict-ordering guard", err) + } + if got := countCalls(rec.calls, "keys"); got != tc.wantKeys { + t.Fatalf("key calls = %d, want %d", got, tc.wantKeys) + } + if got := countCalls(rec.calls, "rows"); got != tc.wantRows { + t.Fatalf("row calls = %d, want %d: the refusing page must never dispatch its row step and emit duplicated or unordered rows", got, tc.wantRows) + } + }) + } + t.Run("control proceeds", func(t *testing.T) { + rec := &exec05Recorder{keyPageOverride: func(_ int, served []string) []string { return served }} + if err := exec05Stream(t, newExec05RootPageSteps(t, 2), rec.port).streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("paired control: %v", err) + } + if rec.visitCount != 6 { + t.Fatalf("control visited %d rows, want 6", rec.visitCount) + } + }) +} + +func TestPostgreSQLRootPageCompositionRefusalOffline(t *testing.T) { + paired := func(t *testing.T, mutate func(*PostgreSQLRootPageSteps)) (*exec05Recorder, error) { + t.Helper() + steps := newExec05RootPageSteps(t, 2) + mutate(steps) + rec := &exec05Recorder{} + err := exec05Stream(t, steps, rec.port).streamRaw(context.Background(), func(map[string]any) error { return nil }) + return rec, err + } + t.Run("AQL-tagged key step is refused before the port", func(t *testing.T) { + rec, err := paired(t, func(steps *PostgreSQLRootPageSteps) { + steps.RootKeys = mustAQLCommand("FOR d IN resource RETURN d._key", map[string]any{"loom_root_page_after_key": ""}) + }) + if !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("error = %v, want the compiler mismatch sentinel", err) + } + if len(rec.calls) != 0 { + t.Fatalf("refusal must touch no port call, got %d", len(rec.calls)) + } + }) + t.Run("zero-tag row step is refused before the port", func(t *testing.T) { + rec, err := paired(t, func(steps *PostgreSQLRootPageSteps) { + steps.Rows = command.CompiledCommand{} + }) + if !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("error = %v, want the compiler unregistered sentinel", err) + } + if len(rec.calls) != 0 { + t.Fatalf("refusal must touch no port call, got %d", len(rec.calls)) + } + }) + shapes := []struct { + name string + mutate func(*PostgreSQLRootPageSteps) + }{ + {"cursor index out of range", func(s *PostgreSQLRootPageSteps) { s.CursorArgIndex = 9 }}, + {"negative cursor index", func(s *PostgreSQLRootPageSteps) { s.CursorArgIndex = -1 }}, + {"cursor declared as bigint", func(s *PostgreSQLRootPageSteps) { + s.RootKeys = newExec05PageCommand(t, exec05KeySQL, []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "P1"}, + {Type: command.PostgreSQLParameterBigint, Value: 0}, + {Type: command.PostgreSQLParameterBigint, Value: 2}, + }) + }}, + {"page size declared as text", func(s *PostgreSQLRootPageSteps) { + s.RootKeys = newExec05PageCommand(t, exec05KeySQL, []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "P1"}, + {Type: command.PostgreSQLParameterText, Value: ""}, + {Type: command.PostgreSQLParameterText, Value: "2"}, + }) + }}, + {"colliding cursor and size positions", func(s *PostgreSQLRootPageSteps) { s.PageSizeArgIndex = s.CursorArgIndex }}, + {"key page declared as text", func(s *PostgreSQLRootPageSteps) { + s.Rows = newExec05PageCommand(t, exec05RowsSQL, []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "P1"}, + {Type: command.PostgreSQLParameterText, Value: "[]"}, + }) + }}, + {"non-positive page size", func(s *PostgreSQLRootPageSteps) { s.PageSize = 0 }}, + {"page size beyond the sanity bound", func(s *PostgreSQLRootPageSteps) { s.PageSize = maxPostgreSQLRootPageSize + 1 }}, + {"blank root key column", func(s *PostgreSQLRootPageSteps) { s.RootKeyColumn = " " }}, + {"blank row root key column", func(s *PostgreSQLRootPageSteps) { s.RowRootKeyColumn = "" }}, + {"row root key outside the declared contract", func(s *PostgreSQLRootPageSteps) { s.RowRootKeyColumn = "not_declared" }}, + } + for _, tc := range shapes { + t.Run(tc.name, func(t *testing.T) { + rec, err := paired(t, tc.mutate) + if !errors.Is(err, ErrPostgreSQLRootPageSteps) { + t.Fatalf("error = %v, want ErrPostgreSQLRootPageSteps", err) + } + if len(rec.calls) != 0 { + t.Fatalf("refusal must touch no port call, got %d", len(rec.calls)) + } + }) + } + t.Run("control: page size at the sanity bound is accepted", func(t *testing.T) { + steps := newExec05RootPageSteps(t, 2) + steps.PageSize = maxPostgreSQLRootPageSize + rec := &exec05Recorder{} + if err := exec05Stream(t, steps, rec.port).streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("paired control at the bound: %v", err) + } + if rec.visitCount != 6 { + t.Fatalf("control at the bound visited %d rows, want 6", rec.visitCount) + } + }) + t.Run("control: valid steps attach and run", func(t *testing.T) { + rec := &exec05Recorder{} + if err := exec05Stream(t, newExec05RootPageSteps(t, 2), rec.port).streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("paired control: %v", err) + } + if len(rec.calls) == 0 { + t.Fatal("paired control must consult the port") + } + }) +} + +func TestPostgreSQLRootPageCancellationOffline(t *testing.T) { + t.Run("cancellation between key and row steps prevents later calls", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rec := &exec05Recorder{cancelAfterKey: 1, cancel: cancel} + stream := exec05Stream(t, newExec05RootPageSteps(t, 2), rec.port) + _, err := stream.Stream(ctx, func(map[string]any) error { return nil }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context.Canceled; completion must never be reported", err) + } + if keys, rows := countCalls(rec.calls, "keys"), countCalls(rec.calls, "rows"); keys != 1 || rows != 0 { + t.Fatalf("calls keys=%d rows=%d, want 1/0 (row step must never run after cancellation)", keys, rows) + } + }) + t.Run("cancellation before a later key step stops the loop", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rec := &exec05Recorder{} + wrapped := func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + if err := rec.port(ctx, request, visit); err != nil { + return err + } + if countCalls(rec.calls, "keys") == 1 && countCalls(rec.calls, "rows") == 1 { + cancel() + } + return nil + } + stream := exec05Stream(t, newExec05RootPageSteps(t, 2), wrapped) + _, err := stream.Stream(ctx, func(map[string]any) error { return nil }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context.Canceled after the first full page", err) + } + if keys, rows := countCalls(rec.calls, "keys"), countCalls(rec.calls, "rows"); keys != 1 || rows != 1 { + t.Fatalf("calls keys=%d rows=%d, want 1/1 (no call after cancellation)", keys, rows) + } + }) + t.Run("control: uncancelled run completes", func(t *testing.T) { + rec := &exec05Recorder{} + stream := exec05Stream(t, newExec05RootPageSteps(t, 2), rec.port) + result, err := stream.Stream(context.Background(), func(map[string]any) error { return nil }) + if err != nil { + t.Fatalf("paired control: %v", err) + } + if result.RowCount != 6 { + t.Fatalf("control RowCount = %d, want 6", result.RowCount) + } + }) +} + +func TestPostgreSQLRootPageProductionNeverAttaches(t *testing.T) { + t.Run("compiler still refuses PostgreSQL root-key output paging, AQL control compiles", func(t *testing.T) { + resolved := exec02TestResolved() + _, err := compiler.CompileRecipeOutputPage(exec02PostgreSQLTarget(t), + resolved.Compiled.Outputs[0], + resolved.Semantic.SemanticPlan.Bindings, + 2, ir.DefaultPhysicalOptimizationPolicy()) + if !errors.Is(err, postgresrender.ErrUnsupportedPostgresPlan) { + t.Fatalf("PostgreSQL page compile must stay refused with the typed unsupported-plan sentinel, got %v", err) + } + page, pageErr := compiler.CompileRecipeOutputPage(compiler.ArangoTarget(), + resolved.Compiled.Outputs[0], + resolved.Semantic.SemanticPlan.Bindings, + 2, ir.DefaultPhysicalOptimizationPolicy()) + if pageErr != nil { + t.Fatalf("AQL control page compile must proceed: %v", pageErr) + } + if page.RootKeys.Backend() != command.CompiledBackendAQL || page.Rows.Backend() != command.CompiledBackendAQL { + t.Fatalf("AQL control page tags = %q/%q, want AQL/AQL", page.RootKeys.Backend(), page.Rows.Backend()) + } + }) + t.Run("engine-built PostgreSQL streams never carry page fields", func(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, exec02Rows()) + engine.rootPageRows = 50 + resolved := exec02TestResolved() + streams, err := engine.Streams(context.Background(), resolved) + if err != nil { + t.Fatalf("PG streams: %v", err) + } + if len(streams) != 1 { + t.Fatalf("streams = %d, want 1", len(streams)) + } + if streams[0].page != nil || streams[0].pgRootPage != nil || streams[0].rootPageRows != 0 { + t.Fatalf("PostgreSQL stream carries page fields: page=%v pgRootPage=%v rootPageRows=%d", + streams[0].page, streams[0].pgRootPage, streams[0].rootPageRows) + } + if _, err := streams[0].Stream(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("single-statement stream: %v", err) + } + if spies.pgRequests != 1 { + t.Fatalf("port calls = %d, want the single statement", spies.pgRequests) + } + exec02AssertUntouchedArango(t, spies) + }) + t.Run("whole-input PostgreSQL output can never reach the root-page loop", func(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, exec02Rows()) + engine.rootPageRows = 50 + resolved := exec02TestResolved() + output := resolved.Compiled.Outputs[0] + output.Name = "grouped" + groupBinds := exec02TestPlan(nil).BindVars + groupBinds["preview_limit"] = 25 + output.Plan = ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: groupBinds, + Operations: []ir.PhysicalOperation{{ + Kind: ir.PhysicalGroupRowsOp, + GroupRows: &ir.PhysicalGroupRows{LimitBindKey: "preview_limit"}, + }}, + } + resolved.Compiled.Outputs = []lower.CompiledRecipeOutput{output} + resolved.Semantic.SemanticPlan.Bindings.OutputNames = []string{"grouped"} + streams, err := engine.Streams(context.Background(), resolved) + if err != nil { + t.Logf("whole-input bypass arm: compile-refused before execution (%v)", err) + if len(spies.pgFull) != 0 { + t.Fatalf("refused whole-input plan still reached the port: %+v", spies.pgFull) + } + return + } + t.Logf("whole-input bypass arm: compiled, structurally single-statement") + if len(streams) != 1 || streams[0].page != nil || streams[0].pgRootPage != nil { + t.Fatalf("whole-input stream carries page shape: %+v", streams) + } + if _, err := streams[0].Stream(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("whole-input single statement: %v", err) + } + if spies.pgRequests != 1 { + t.Fatalf("whole-input port calls = %d, want 1", spies.pgRequests) + } + exec02AssertUntouchedArango(t, spies) + }) +} + +func TestPostgreSQLRootPageLeakedFieldsStayInert(t *testing.T) { + t.Run("PostgreSQL stream with leaked AQL page fields stays single-statement", func(t *testing.T) { + calls := 0 + stream := OutputStream{ + Name: "resources", Columns: []string{"id"}, + command: newExec05PageCommand(t, "SELECT 1", nil), + expectedBackend: command.CompiledBackendPostgreSQL, + commandColumns: []string{"id"}, + rootPageRows: 50, + initialRootPageRows: 1, + page: &compiler.CompiledOutputPage{ + RootKeys: mustAQLCommand("ROOT-KEYS", map[string]any{compiler.RootPageAfterKeyBind: "", compiler.RootPageSizeBind: 0}), + Rows: mustAQLCommand("ROWS", map[string]any{compiler.RootPageKeysBind: []string{}}), + }, + sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + calls++ + if _, _, err := request.Command.PostgreSQLParameters(); err != nil { + return err + } + return visit(map[string]any{"id": "patient-one"}) + }, + } + if err := stream.streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("leaked-field PostgreSQL stream: %v", err) + } + if calls != 1 { + t.Fatalf("port calls = %d, want the single statement", calls) + } + }) + t.Run("AQL stream with leaked pgRootPage never consults it", func(t *testing.T) { + calls := 0 + stream := OutputStream{ + Name: "resources", Columns: []string{"id"}, + command: mustAQLCommand("FOR d IN resource RETURN d", map[string]any{"project": "P1"}), + expectedBackend: command.CompiledBackendAQL, + pgRootPage: newExec05RootPageSteps(t, 2), + sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + calls++ + if _, _, err := request.Command.AQLParameters(); err != nil { + return err + } + return visit(map[string]any{"id": "patient-one"}) + }, + } + if err := stream.streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("leaked-steps AQL stream: %v", err) + } + if calls != 1 { + t.Fatalf("port calls = %d, want the single AQL statement", calls) + } + }) +} diff --git a/internal/dataframe/execution/engine_workspace_artifact_test.go b/internal/dataframe/execution/engine_workspace_artifact_test.go index f296b91cb..ea921dab1 100644 --- a/internal/dataframe/execution/engine_workspace_artifact_test.go +++ b/internal/dataframe/execution/engine_workspace_artifact_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/calypr/loom/internal/dataframe/compiler" "reflect" "strings" "testing" @@ -45,7 +46,7 @@ func TestEngineStreamsNestedWorkspaceCombinesWithOnlyTerminalLimit(t *testing.T) var pinCalls, resolveCalls int var privateQueryCalls int var queries []string - engine := &Engine{ + engine := &Engine{compileTarget: compiler.ArangoTarget(), privateClickHouseArtifacts: manager, withExecutionReadPins: func(_ context.Context, revisions []string, visit func(context.Context) error) error { if !reflect.DeepEqual(revisions, []string{"revision_a", "revision_b"}) && @@ -85,9 +86,9 @@ func TestEngineStreamsNestedWorkspaceCombinesWithOnlyTerminalLimit(t *testing.T) } if strings.Contains(query, "loom_private_") { privateQueryCalls++ - want := 3 // the nested output's writer is live while its two inputs are queried. + want := 3 if privateQueryCalls%2 == 0 { - want = 2 // the terminal query sees its two completed source captures. + want = 2 } if len(catalog.items) != want { return fmt.Errorf("private query %d sees %d live captures, want %d", privateQueryCalls, len(catalog.items), want) @@ -104,7 +105,7 @@ func TestEngineStreamsNestedWorkspaceCombinesWithOnlyTerminalLimit(t *testing.T) Semantic: semantic.ResolvedRecipePlan{SemanticPlan: semantic.RecipePlan{Bindings: bindings}}, Compiled: lower.CompiledRecipe{Outputs: []lower.CompiledRecipeOutput{terminal}, WorkspaceDependencies: []lower.CompiledRecipeOutput{inner, leafA, leafB}}, } - stream, _, err := engine.streamForOutput(resolved, "terminal", 3) + stream, _, err := engine.streamForOutput(context.Background(), resolved, "terminal", 3) if err != nil { t.Fatal(err) } @@ -137,7 +138,7 @@ func TestEngineStreamsNestedWorkspaceCombinesWithOnlyTerminalLimit(t *testing.T) t.Fatalf("nested execution leaked %d private artifact manifests", len(catalog.items)) } - stream, _, err = engine.streamForOutput(resolved, "terminal", 3) + stream, _, err = engine.streamForOutput(context.Background(), resolved, "terminal", 3) if err != nil { t.Fatal(err) } @@ -150,7 +151,7 @@ func TestEngineStreamsNestedWorkspaceCombinesWithOnlyTerminalLimit(t *testing.T) t.Fatalf("nested execution leaked %d private artifact manifests after consumer cancellation", len(catalog.items)) } - stream, _, err = engine.streamForOutput(resolved, "terminal", 3) + stream, _, err = engine.streamForOutput(context.Background(), resolved, "terminal", 3) if err != nil { t.Fatal(err) } @@ -255,8 +256,15 @@ func TestEngineCapturesCompilerGroupAndPivotWorkspaceOutputs(t *testing.T) { var failedSourceTable string cleanupFailure := errors.New("injected source artifact cleanup failure") engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, PrivateClickHouseArtifacts: manager, - QueryRows: func(_ context.Context, query string, _ int, _ map[string]any, visit func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, PrivateClickHouseArtifacts: manager, + SourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + query, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if strings.Contains(query, " LIMIT ") { return fmt.Errorf("group or Pivot dependency received a preview limit") } @@ -313,7 +321,7 @@ func TestEngineCapturesCompilerGroupAndPivotWorkspaceOutputs(t *testing.T) { } wantLiveCaptures := 2 if failSourceCleanup { - wantLiveCaptures++ // the terminal writer is already live while its rows stream. + wantLiveCaptures++ } if len(catalog.items) != wantLiveCaptures { return fmt.Errorf("terminal Combine sees %d private captures, want %d", len(catalog.items), wantLiveCaptures) @@ -322,7 +330,7 @@ func TestEngineCapturesCompilerGroupAndPivotWorkspaceOutputs(t *testing.T) { for _, manifest := range catalog.items { identity := manifest.Identity if identity.OutputID == "joined" { - continue // the terminal writer is still filling its table in this callback. + continue } if identity.Project != bindings.Project || identity.DatasetGeneration != bindings.DatasetGeneration || identity.AuthScopeMode != authscope.ReadScopeUnrestricted || identity.ScopeMode != chartifact.ScopeModeWhole || identity.ScopeEvidenceDigest == "" { return fmt.Errorf("group/Pivot artifact lost whole-scope identity: %#v", identity) @@ -349,7 +357,7 @@ func TestEngineCapturesCompilerGroupAndPivotWorkspaceOutputs(t *testing.T) { if err != nil { t.Fatalf("compile Group/Pivot workspace fixture: %v", err) } - stream, _, err := engine.streamForOutput(resolved, "joined", 2) + stream, _, err := engine.streamForOutput(context.Background(), resolved, "joined", 2) if err != nil { t.Fatalf("build Group/Pivot workspace stream: %v", err) } @@ -376,7 +384,7 @@ func TestEngineCapturesCompilerGroupAndPivotWorkspaceOutputs(t *testing.T) { t.Fatalf("Group/Pivot source calls or cleanup = %d AQL calls, %d manifests, %d tables", aqlCalls, len(catalog.items), len(ch.rows)) } - stream, _, err = engine.streamForOutput(resolved, "joined", 0) + stream, _, err = engine.streamForOutput(context.Background(), resolved, "joined", 0) if err != nil { t.Fatalf("build Group/Pivot cancellation capture: %v", err) } @@ -397,7 +405,7 @@ func TestEngineCapturesCompilerGroupAndPivotWorkspaceOutputs(t *testing.T) { t.Fatalf("terminal query cancellation leaked %d manifests and %d ClickHouse tables", len(catalog.items), len(ch.rows)) } - stream, _, err = engine.streamForOutput(resolved, "joined", 0) + stream, _, err = engine.streamForOutput(context.Background(), resolved, "joined", 0) if err != nil { t.Fatalf("build Group/Pivot terminal failure capture: %v", err) } @@ -412,7 +420,7 @@ func TestEngineCapturesCompilerGroupAndPivotWorkspaceOutputs(t *testing.T) { t.Fatalf("terminal query failure leaked %d manifests and %d ClickHouse tables", len(catalog.items), len(ch.rows)) } - stream, _, err = engine.streamForOutput(resolved, "joined", 0) + stream, _, err = engine.streamForOutput(context.Background(), resolved, "joined", 0) if err != nil { t.Fatalf("build unbounded Group/Pivot capture stream: %v", err) } @@ -471,8 +479,15 @@ func TestEngineCapturesRestrictedWorkspaceRowsWithExactAuthPaths(t *testing.T) { } var sourceCalls, combineCalls int engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, PrivateClickHouseArtifacts: manager, - QueryRows: func(_ context.Context, query string, _ int, bindVars map[string]any, visit func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, PrivateClickHouseArtifacts: manager, + SourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + query, bindVars, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if !strings.Contains(query, "auth_resource_path IN @auth_resource_paths") || strings.Contains(query, " LIMIT ") { return fmt.Errorf("restricted source query omitted its exact scope filter or received a limit: %s", query) } @@ -564,7 +579,7 @@ func TestEngineCapturesRestrictedWorkspaceRowsWithExactAuthPaths(t *testing.T) { resolved.Compiled.Outputs = []lower.CompiledRecipeOutput{joined} resolved.Compiled.WorkspaceDependencies = dependencies - stream, _, err := engine.streamForOutput(resolved, "joined", 1) + stream, _, err := engine.streamForOutput(context.Background(), resolved, "joined", 1) if err != nil { t.Fatalf("build restricted workspace Combine: %v", err) } @@ -624,8 +639,15 @@ func TestOrdinaryRestrictedPreviewDoesNotRequirePrivateCaptureProof(t *testing.T } var queryCalls int engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, - QueryRows: func(_ context.Context, _ string, _ int, bindVars map[string]any, visit func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + SourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, bindVars, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + queryCalls++ if bindVars["auth_resource_paths_unrestricted"] != false || !reflect.DeepEqual(bindVars["auth_resource_paths"], []string{"/programs/a"}) { return fmt.Errorf("ordinary preview lost its exact restricted source scope: %#v", bindVars) @@ -645,8 +667,6 @@ func TestOrdinaryRestrictedPreviewDoesNotRequirePrivateCaptureProof(t *testing.T if err != nil { t.Fatal(err) } - // Model a valid typed source plan for which the stronger private-capture - // issuer declines a whole-source proof. Public preview must stay available. resolved.Compiled.Outputs[0].ScopeEvidence = nil resolved.Compiled.Outputs[0].ScopeEvidenceIdentity = nil rows := []map[string]any{} @@ -708,8 +728,16 @@ func TestPublishedOnlyKeyJoinStreamsRepeatedOutput(t *testing.T) { published("table-right", "revision-right", "right", "right-key", "right_private", false) var pinned []string engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }, WithExecutionReadPins: func(_ context.Context, revisions []string, visit func(context.Context) error) error { pinned = append([]string(nil), revisions...) return visit(context.Background()) @@ -734,7 +762,7 @@ func TestPublishedOnlyKeyJoinStreamsRepeatedOutput(t *testing.T) { Semantic: semantic.ResolvedRecipePlan{SemanticPlan: semantic.RecipePlan{Bindings: bindings}}, Compiled: lower.CompiledRecipe{Outputs: []lower.CompiledRecipeOutput{output}}, } - stream, _, err := engine.streamForOutput(resolved, "joined", 0) + stream, _, err := engine.streamForOutput(context.Background(), resolved, "joined", 0) if err != nil { t.Fatal(err) } diff --git a/internal/dataframe/execution/execution_service.go b/internal/dataframe/execution/execution_service.go index aa561417d..7ddcf0e70 100644 --- a/internal/dataframe/execution/execution_service.go +++ b/internal/dataframe/execution/execution_service.go @@ -3,12 +3,15 @@ package execution import ( "context" "fmt" + "github.com/calypr/loom/internal/dataframe/compiler" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" "sort" "time" ) -func (s *Service) runQuery(ctx context.Context, compiled CompiledQuery) (*Result, error) { +var now = time.Now + +func (s *Service) runQuery(ctx context.Context, compiled compiler.CompiledQuery) (*Result, error) { var rows []map[string]any streamed, err := s.streamQuery(ctx, compiled, func(row map[string]any) error { rows = append(rows, row) @@ -25,12 +28,7 @@ func (s *Service) runQuery(ctx context.Context, compiled CompiledQuery) (*Result }, nil } -// RunCompiled executes an already canonical-compiled query. It is the shared -// execution seam for non-GraphQL frontends (for example an ephemeral recipe -// produced by GraphQL); compilation and scope resolution remain the caller's -// responsibility. The query still uses the same cursor, row flattening, and -// diagnostics path as the ordinary recipe-backed Run. -func (s *Service) RunCompiled(ctx context.Context, compiled CompiledQuery) (*Result, error) { +func (s *Service) RunCompiled(ctx context.Context, compiled compiler.CompiledQuery) (*Result, error) { started := time.Now() result, err := s.runQuery(ctx, compiled) if err != nil { @@ -41,11 +39,10 @@ func (s *Service) RunCompiled(ctx context.Context, compiled CompiledQuery) (*Res return result, nil } -func (s *Service) streamQuery(ctx context.Context, compiled CompiledQuery, visit func(map[string]any) error) (streamResult, error) { +func (s *Service) streamQuery(ctx context.Context, compiled compiler.CompiledQuery, visit func(map[string]any) error) (streamResult, error) { if visit == nil { return streamResult{}, fmt.Errorf("row visitor is required") } - // A request the caller already abandoned must not open a backend cursor. if err := contextError(ctx); err != nil { return streamResult{}, err } @@ -58,18 +55,16 @@ func (s *Service) streamQuery(ctx context.Context, compiled CompiledQuery, visit extraColumns := map[string]struct{}{} rowCount := 0 var rowMaterialization time.Duration - queryStarted := time.Now() - if s.queryRows == nil { + queryStarted := now() + if s.sourceRows == nil { return streamResult{}, dataframeerrors.NewError(dataframeerrors.CodeBackendUnavailable, "", dataframeerrors.WithRetryable(true)) } - err := s.queryRows(ctx, compiled.Query, 1000, compiled.BindVars, func(row map[string]any) error { - // A backend adapter can keep visiting rows after the caller cancels; - // stop before materializing or delivering any further public row. + err := s.sourceRows(ctx, RowsRequest{Command: compiled.Command, DeclaredColumns: append([]string(nil), compiled.Columns...), BatchSize: 1000}, func(row map[string]any) error { if err := contextError(ctx); err != nil { return err } - rowStarted := time.Now() - defer func() { rowMaterialization += time.Since(rowStarted) }() + rowStarted := now() + defer func() { rowMaterialization += now().Sub(rowStarted) }() flatRow := flattenPivotFields(cloneRow(row), compiled.PivotFields) for key := range flatRow { if _, ok := seenColumns[key]; ok { @@ -84,10 +79,9 @@ func (s *Service) streamQuery(ctx context.Context, compiled CompiledQuery, visit rowCount++ return nil }) - queryElapsed := time.Since(queryStarted) - arangoQuery := queryElapsed - rowMaterialization - if arangoQuery < 0 { - arangoQuery = 0 + sourceQuery := now().Sub(queryStarted) - rowMaterialization + if sourceQuery < 0 { + sourceQuery = 0 } assemblyStarted := time.Now() newColumns := make([]string, 0, len(extraColumns)) @@ -100,7 +94,8 @@ func (s *Service) streamQuery(ctx context.Context, compiled CompiledQuery, visit Columns: columns, RowCount: rowCount, Diagnostics: QueryDiagnostics{ - ArangoQuery: arangoQuery, + SourceQuery: sourceQuery, + SourceBackend: compiled.Command.Backend(), RowMaterialization: rowMaterialization, ResultAssembly: time.Since(assemblyStarted), }, @@ -108,8 +103,6 @@ func (s *Service) streamQuery(ctx context.Context, compiled CompiledQuery, visit if err != nil { return result, err } - // An adapter can report a clean end of stream after losing its context; - // completion is only a success while the caller is still waiting. if err := contextError(ctx); err != nil { return result, err } diff --git a/internal/dataframe/execution/execution_test.go b/internal/dataframe/execution/execution_test.go index d4b027e5f..ecf4ec64c 100644 --- a/internal/dataframe/execution/execution_test.go +++ b/internal/dataframe/execution/execution_test.go @@ -3,6 +3,7 @@ package execution import ( "context" "errors" + "github.com/calypr/loom/internal/dataframe/compiler" "reflect" "testing" @@ -33,7 +34,13 @@ func TestFlattenPivotFieldsCreatesStableFlattenedKeys(t *testing.T) { func TestRunQueryAppendsDynamicColumnsInStableOrder(t *testing.T) { svc := NewService(ServiceConfig{ - QueryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + SourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + for _, row := range []map[string]any{ {"_key": "first", "pivot": map[string]any{"zeta": "z", "alpha": "a"}}, {"_key": "second", "pivot": map[string]any{"beta": "b"}}, @@ -46,8 +53,8 @@ func TestRunQueryAppendsDynamicColumnsInStableOrder(t *testing.T) { }, }) - result, err := svc.runQuery(context.Background(), CompiledQuery{ - Query: "RETURN []", + result, err := svc.runQuery(context.Background(), compiler.CompiledQuery{ + Command: mustAQLCommand("RETURN []", nil), Columns: []string{"_key", "pivot"}, PivotFields: []string{"pivot"}, Limit: 2, @@ -66,7 +73,13 @@ func TestRunQueryAppendsDynamicColumnsInStableOrder(t *testing.T) { func TestStreamQueryDeliversFlattenedRowsWithoutCollectingResultRows(t *testing.T) { svc := NewService(ServiceConfig{ - QueryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + SourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + for _, row := range []map[string]any{ {"_key": "first", "pivot": map[string]any{"zeta": "z", "alpha": "a"}}, {"_key": "second", "pivot": map[string]any{"beta": "b"}}, @@ -80,8 +93,8 @@ func TestStreamQueryDeliversFlattenedRowsWithoutCollectingResultRows(t *testing. }) gotRows := []map[string]any{} - result, err := svc.streamQuery(context.Background(), CompiledQuery{ - Query: "RETURN []", + result, err := svc.streamQuery(context.Background(), compiler.CompiledQuery{ + Command: mustAQLCommand("RETURN []", nil), Columns: []string{"_key", "pivot"}, PivotFields: []string{"pivot"}, Limit: 2, @@ -110,7 +123,13 @@ func TestStreamQueryDeliversFlattenedRowsWithoutCollectingResultRows(t *testing. func TestStreamQueryReturnsPartialProgressWhenVisitorStops(t *testing.T) { svc := NewService(ServiceConfig{ - QueryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + SourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if err := visit(map[string]any{"_key": "first"}); err != nil { return err } @@ -118,7 +137,7 @@ func TestStreamQueryReturnsPartialProgressWhenVisitorStops(t *testing.T) { }, }) stop := assertError("stop") - result, err := svc.streamQuery(context.Background(), CompiledQuery{Query: "RETURN []", Columns: []string{"_key"}}, func(map[string]any) error { + result, err := svc.streamQuery(context.Background(), compiler.CompiledQuery{Command: mustAQLCommand("RETURN []", nil), Columns: []string{"_key"}}, func(map[string]any) error { return stop }) if err != stop { @@ -129,10 +148,6 @@ func TestStreamQueryReturnsPartialProgressWhenVisitorStops(t *testing.T) { } } -// compiledQueryAdapter is the test-owned backend seam for the generic -// compiled-query path. It records each invocation so a test can prove no query -// is issued for an abandoned request, and it keeps visiting rows after the -// caller cancels the way a lagging driver cursor does. type compiledQueryAdapter struct { calls int rows []map[string]any @@ -156,9 +171,9 @@ func TestRunCompiledAbandonsCanceledRequestWithoutQueryingBackend(t *testing.T) ctx, cancel := context.WithCancel(context.Background()) cancel() adapter := &compiledQueryAdapter{rows: []map[string]any{{"_key": "first"}}} - svc := NewService(ServiceConfig{QueryRows: adapter.queryRows}) + svc := NewService(ServiceConfig{SourceRows: aqlRows(adapter.queryRows)}) - result, err := svc.RunCompiled(ctx, CompiledQuery{Query: "RETURN []", Columns: []string{"_key"}}) + result, err := svc.RunCompiled(ctx, compiler.CompiledQuery{Command: mustAQLCommand("RETURN []", nil), Columns: []string{"_key"}}) if !errors.Is(err, context.Canceled) { t.Fatalf("RunCompiled() error = %v, want context.Canceled", err) } @@ -179,19 +194,15 @@ func TestStreamQueryStopsDeliveryAfterCancellationDuringCallback(t *testing.T) { defer cancel() visited := []any{} adapter := &compiledQueryAdapter{ - rows: []map[string]any{{"_key": "first"}, {"_key": "second"}}, - // Cancellation lands after the first row is delivered. A missing check - // would hand the second row to the public visitor and report success. + rows: []map[string]any{{"_key": "first"}, {"_key": "second"}}, afterCancel: cancel, } - svc := NewService(ServiceConfig{QueryRows: adapter.queryRows}) + svc := NewService(ServiceConfig{SourceRows: aqlRows(adapter.queryRows)}) - result, err := svc.streamQuery(ctx, CompiledQuery{Query: "RETURN []", Columns: []string{"_key"}}, func(row map[string]any) error { + result, err := svc.streamQuery(ctx, compiler.CompiledQuery{Command: mustAQLCommand("RETURN []", nil), Columns: []string{"_key"}}, func(row map[string]any) error { visited = append(visited, row["_key"]) return nil }) - // The boundary reports the caller's own context error, wrapped as a typed - // user error, rather than inventing a fresh sentinel. if !errors.Is(err, ctx.Err()) { t.Fatalf("streamQuery() error = %v, want the caller context error %v", err, ctx.Err()) } @@ -214,9 +225,9 @@ func TestRunCompiledRejectsNilAdapterErrorAfterCancellation(t *testing.T) { rows: []map[string]any{{"_key": "first"}}, afterCancel: cancel, } - svc := NewService(ServiceConfig{QueryRows: adapter.queryRows}) + svc := NewService(ServiceConfig{SourceRows: aqlRows(adapter.queryRows)}) - result, err := svc.RunCompiled(ctx, CompiledQuery{Query: "RETURN []", Columns: []string{"_key"}}) + result, err := svc.RunCompiled(ctx, compiler.CompiledQuery{Command: mustAQLCommand("RETURN []", nil), Columns: []string{"_key"}}) if !errors.Is(err, context.Canceled) { t.Fatalf("RunCompiled() error = %v, want context.Canceled when the adapter returns nil", err) } @@ -227,9 +238,9 @@ func TestRunCompiledRejectsNilAdapterErrorAfterCancellation(t *testing.T) { func TestRunCompiledCompletesZeroRowsWhileContextIsLive(t *testing.T) { adapter := &compiledQueryAdapter{} - svc := NewService(ServiceConfig{QueryRows: adapter.queryRows}) + svc := NewService(ServiceConfig{SourceRows: aqlRows(adapter.queryRows)}) - result, err := svc.RunCompiled(context.Background(), CompiledQuery{Query: "RETURN []", Columns: []string{"_key"}}) + result, err := svc.RunCompiled(context.Background(), compiler.CompiledQuery{Command: mustAQLCommand("RETURN []", nil), Columns: []string{"_key"}}) if err != nil { t.Fatalf("RunCompiled() error = %v, want success for a legitimate empty result", err) } @@ -245,9 +256,9 @@ func TestStreamQueryReportsExpiredDeadlineInsteadOfSuccess(t *testing.T) { expired, cancel := context.WithTimeout(context.Background(), 0) defer cancel() adapter := &compiledQueryAdapter{rows: []map[string]any{{"_key": "first"}}} - svc := NewService(ServiceConfig{QueryRows: adapter.queryRows}) + svc := NewService(ServiceConfig{SourceRows: aqlRows(adapter.queryRows)}) - if _, err := svc.streamQuery(expired, CompiledQuery{Query: "RETURN []", Columns: []string{"_key"}}, func(map[string]any) error { + if _, err := svc.streamQuery(expired, compiler.CompiledQuery{Command: mustAQLCommand("RETURN []", nil), Columns: []string{"_key"}}, func(map[string]any) error { t.Fatal("visitor must not run for an expired request") return nil }); !errors.Is(err, context.DeadlineExceeded) { @@ -263,13 +274,19 @@ func TestStreamQueryPreservesBackendErrorOverCancellation(t *testing.T) { defer cancel() backendErr := assertError("arango cursor failed") svc := NewService(ServiceConfig{ - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + cancel() return backendErr }, }) - if _, err := svc.streamQuery(ctx, CompiledQuery{Query: "RETURN []", Columns: []string{"_key"}}, func(map[string]any) error { + if _, err := svc.streamQuery(ctx, compiler.CompiledQuery{Command: mustAQLCommand("RETURN []", nil), Columns: []string{"_key"}}, func(map[string]any) error { t.Fatal("visitor must not run when the adapter delivers no row") return nil }); !errors.Is(err, backendErr) { @@ -283,12 +300,10 @@ func TestStreamQueryKeepsVisitorErrorWhenItCancelsMidStream(t *testing.T) { stop := assertError("stop") visited := []any{} adapter := &compiledQueryAdapter{rows: []map[string]any{{"_key": "first"}, {"_key": "second"}}} - svc := NewService(ServiceConfig{QueryRows: adapter.queryRows}) + svc := NewService(ServiceConfig{SourceRows: aqlRows(adapter.queryRows)}) - result, err := svc.streamQuery(ctx, CompiledQuery{Query: "RETURN []", Columns: []string{"_key"}}, func(row map[string]any) error { + result, err := svc.streamQuery(ctx, compiler.CompiledQuery{Command: mustAQLCommand("RETURN []", nil), Columns: []string{"_key"}}, func(row map[string]any) error { visited = append(visited, row["_key"]) - // The consumer abandons the stream and cancels the request in the same - // step; its own error stays authoritative over the new cancellation. cancel() return stop }) diff --git a/internal/dataframe/execution/explain.go b/internal/dataframe/execution/explain.go index 3e3a469cc..eb122b0be 100644 --- a/internal/dataframe/execution/explain.go +++ b/internal/dataframe/execution/explain.go @@ -4,20 +4,32 @@ import ( "crypto/sha256" "encoding/hex" "sort" + "strconv" "strings" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" ) -// CompiledQueryFingerprint returns a deterministic, value-safe identifier for -// a compiled plan. Bind values are intentionally excluded; rendered query -// shape, bind-key names, and output metadata participate instead. -func CompiledQueryFingerprint(compiled CompiledQuery) string { - keys := make([]string, 0, len(compiled.BindVars)) - for key := range compiled.BindVars { +func CompiledQueryFingerprint(compiled compiler.CompiledQuery) string { + var queryText string + var bindVars map[string]any + if compiled.Command.Backend() == command.CompiledBackendAQL { + text, binds, dispatchErr := compiled.Command.AQLParameters() + if dispatchErr != nil { + return "" + } + queryText, bindVars = text, binds + } + keys := make([]string, 0, len(bindVars)) + for key := range bindVars { keys = append(keys, key) } sort.Strings(keys) var builder strings.Builder - builder.WriteString(compiled.Query) + builder.WriteString(queryText) builder.WriteByte('\x00') for _, key := range keys { builder.WriteString(key) @@ -38,3 +50,57 @@ func CompiledQueryFingerprint(compiled CompiledQuery) string { sum := sha256.Sum256([]byte(builder.String())) return hex.EncodeToString(sum[:]) } + +const sourcePlanShapeDomain = "loom:source-plan-shape:v1" + +func SourcePlanFingerprint(compiled compiler.CompiledQuery) string { + if compiled.Command.Backend() == command.CompiledBackendPostgreSQL { + return postgreSQLDiagnosticShapeFingerprint(compiled) + } + return CompiledQueryFingerprint(compiled) +} + +func postgreSQLDiagnosticShapeFingerprint(compiled compiler.CompiledQuery) string { + sqlText, args, err := compiled.Command.PostgreSQLParameters() + if err != nil || strings.TrimSpace(sqlText) == "" { + return "" + } + var builder strings.Builder + writeField := func(field string) { + builder.WriteString(strconv.Itoa(len(field))) + builder.WriteByte(':') + builder.WriteString(field) + } + writeField(sourcePlanShapeDomain) + writeField(string(command.CompiledBackendPostgreSQL)) + writeField(sqlText) + writeField(strconv.Itoa(len(args))) + for _, arg := range args { + writeField(string(arg.Type)) + } + for _, column := range compiled.Columns { + writeField(column) + } + for _, column := range compiled.PublicColumns { + writeField(column) + } + for _, field := range compiled.PivotFields { + writeField(field) + } + sum := sha256.Sum256([]byte(builder.String())) + return hex.EncodeToString(sum[:]) +} + +func previewPlanFingerprint(query compiler.CompiledQuery, stream OutputStream) string { + if stream.physicalEngine == ir.PhysicalEngineClickHouse { + return query.PlanDiagnostics.Fingerprint + } + return SourcePlanFingerprint(query) +} + +func requireProjectedPlanFingerprint(output string, fingerprint string) error { + if strings.TrimSpace(fingerprint) == "" { + return dataframeerrors.NewError(dataframeerrors.CodeInternalError, "compiled command for output "+output+" carries no dispatchable payload for its diagnostic plan fingerprint", dataframeerrors.WithRetryable(false)) + } + return nil +} diff --git a/internal/dataframe/execution/explain_postgres.go b/internal/dataframe/execution/explain_postgres.go new file mode 100644 index 000000000..ce07de72f --- /dev/null +++ b/internal/dataframe/execution/explain_postgres.go @@ -0,0 +1,231 @@ +package execution + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "regexp" + "strings" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" +) + +const postgreSQLExplainPrefix = "EXPLAIN (FORMAT JSON)\n" + +const ( + postgresExplainMaxRawBytes = 4 << 20 + postgresExplainMaxNodes = 512 + postgresExplainMaxDepth = 64 +) + +var postgresExplainNodePattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9 ]{0,63}$`) + +var ( + postgresExplainSequentialNodeTypes = map[string]bool{ + "Seq Scan": true, + "Sample Scan": true, + } + postgresExplainIndexNodeTypes = map[string]bool{ + "Index Scan": true, + "Index Only Scan": true, + "Bitmap Index Scan": true, + } +) + +type PostgreSQLPlanJSONExecutor func(ctx context.Context, command command.CompiledCommand) (json.RawMessage, error) + +func BuildPostgreSQLExplainCommand(compiled compiler.CompiledQuery) (command.CompiledCommand, error) { + sqlText, args, err := compiled.Command.PostgreSQLParameters() + if err != nil { + return command.CompiledCommand{}, err + } + explainCommand, err := command.NewPostgreSQLCommand(postgreSQLExplainPrefix+sqlText, args) + if err != nil { + return command.CompiledCommand{}, fmt.Errorf("tag EXPLAIN command: %w", err) + } + return explainCommand, nil +} + +func PostgreSQLExplainPlan(ctx context.Context, executor PostgreSQLPlanJSONExecutor, compiled compiler.CompiledQuery) (PostgreSQLPlanAssessment, error) { + if executor == nil { + return PostgreSQLPlanAssessment{}, fmt.Errorf("live PostgreSQL Explain requires a plan JSON executor; PostgreSQL requests never fall back to Arango") + } + explainCommand, err := BuildPostgreSQLExplainCommand(compiled) + if err != nil { + return PostgreSQLPlanAssessment{}, err + } + raw, err := executor(ctx, explainCommand) + if err != nil { + return PostgreSQLPlanAssessment{}, postgreSQLExplainExecutorError(ctx, err) + } + assessment, err := sanitizePostgreSQLPlanJSON(raw) + if err != nil { + return PostgreSQLPlanAssessment{}, postgreSQLExplainShapeError(err) + } + return assessment, nil +} + +func postgreSQLExplainExecutorError(ctx context.Context, err error) error { + if err == nil { + return nil + } + if errors.Is(err, context.Canceled) || (ctx != nil && errors.Is(ctx.Err(), context.Canceled)) { + return dataframeerrors.Wrap(err, dataframeerrors.CodeClientCanceled, "live PostgreSQL recipe Explain canceled") + } + if _, classified := dataframeerrors.AsUserError(err); classified { + return err + } + if errors.Is(err, context.DeadlineExceeded) || (ctx != nil && errors.Is(ctx.Err(), context.DeadlineExceeded)) { + return dataframeerrors.Wrap(err, dataframeerrors.CodeBackendUnavailable, "live PostgreSQL recipe Explain timed out", dataframeerrors.WithRetryable(true)) + } + return dataframeerrors.Wrap(err, dataframeerrors.CodeBackendUnavailable, "live PostgreSQL recipe Explain failed", dataframeerrors.WithRetryable(true)) +} + +func postgreSQLExplainShapeError(err error) error { + return dataframeerrors.Wrap(err, dataframeerrors.CodeBackendUnavailable, "PostgreSQL EXPLAIN returned an unexpected plan shape", + dataframeerrors.WithRetryable(false), + dataframeerrors.WithDetails(map[string]any{"backend": "postgresql", "stage": "plan-parse"}), + ) +} + +type postgresExplainRawNode struct { + NodeType *string `json:"Node Type"` + StartupCost *float64 `json:"Startup Cost"` + TotalCost *float64 `json:"Total Cost"` + PlanRows *float64 `json:"Plan Rows"` + PlanWidth *float64 `json:"Plan Width"` + Plans []postgresExplainRawNode `json:"Plans"` +} + +func sanitizePostgreSQLPlanJSON(raw json.RawMessage) (PostgreSQLPlanAssessment, error) { + if len(raw) > postgresExplainMaxRawBytes { + return PostgreSQLPlanAssessment{}, fmt.Errorf("plan JSON exceeds %d bytes", postgresExplainMaxRawBytes) + } + trimmed := strings.TrimSpace(string(raw)) + if trimmed == "" { + return PostgreSQLPlanAssessment{}, errors.New("plan JSON is empty") + } + var entries []struct { + Plan *postgresExplainRawNode `json:"Plan"` + } + decoder := json.NewDecoder(strings.NewReader(trimmed)) + decoder.UseNumber() + if err := decoder.Decode(&entries); err != nil { + return PostgreSQLPlanAssessment{}, fmt.Errorf("plan JSON is not a PostgreSQL EXPLAIN array: %v", err) + } + if err := decoder.Decode(&json.RawMessage{}); !errors.Is(err, io.EOF) { + return PostgreSQLPlanAssessment{}, errors.New("plan JSON carries trailing documents") + } + if len(entries) != 1 { + return PostgreSQLPlanAssessment{}, fmt.Errorf("plan JSON must hold exactly one statement plan, got %d", len(entries)) + } + if entries[0].Plan == nil { + return PostgreSQLPlanAssessment{}, errors.New("plan JSON entry has no Plan object") + } + builder := &postgresExplainTreeBuilder{} + root, err := builder.build(*entries[0].Plan, 0, -1) + if err != nil { + return PostgreSQLPlanAssessment{}, err + } + assessment := PostgreSQLPlanAssessment{ + Backend: string(command.CompiledBackendPostgreSQL), + Nodes: builder.nodes, + FullScans: builder.fullScans, + IndexScans: builder.indexScans, + RootEstimate: root.Estimate, + } + return assessment, nil +} + +type postgresExplainTreeBuilder struct { + nodes []PostgreSQLPlanNode + fullScans []PostgreSQLScanEvidence + indexScans []PostgreSQLScanEvidence +} + +func (b *postgresExplainTreeBuilder) build(raw postgresExplainRawNode, depth, parentOrdinal int) (PostgreSQLPlanNode, error) { + if depth > postgresExplainMaxDepth { + return PostgreSQLPlanNode{}, fmt.Errorf("plan tree exceeds %d levels of nesting", postgresExplainMaxDepth) + } + if len(b.nodes) >= postgresExplainMaxNodes { + return PostgreSQLPlanNode{}, fmt.Errorf("plan tree exceeds %d nodes", postgresExplainMaxNodes) + } + if raw.NodeType == nil { + return PostgreSQLPlanNode{}, errors.New("plan node has no Node Type") + } + nodeType := *raw.NodeType + if !postgresExplainNodePattern.MatchString(nodeType) { + return PostgreSQLPlanNode{}, errors.New("plan node reports a Node Type outside the PostgreSQL operator vocabulary") + } + estimate, err := rawEstimate(raw) + if err != nil { + return PostgreSQLPlanNode{}, fmt.Errorf("plan node %q: %v", nodeType, err) + } + ordinal := len(b.nodes) + node := PostgreSQLPlanNode{Ordinal: ordinal, ParentOrdinal: parentOrdinal, Depth: depth, NodeType: nodeType, Estimate: estimate} + b.nodes = append(b.nodes, node) + switch { + case postgresExplainSequentialNodeTypes[nodeType]: + b.fullScans = append(b.fullScans, PostgreSQLScanEvidence{Ordinal: ordinal, NodeType: nodeType, ScanKind: PostgreSQLScanSequential}) + case postgresExplainIndexNodeTypes[nodeType]: + b.indexScans = append(b.indexScans, PostgreSQLScanEvidence{Ordinal: ordinal, NodeType: nodeType, ScanKind: PostgreSQLScanIndex}) + } + for _, child := range raw.Plans { + if _, err := b.build(child, depth+1, ordinal); err != nil { + return PostgreSQLPlanNode{}, err + } + } + return node, nil +} + +func rawEstimate(raw postgresExplainRawNode) (PostgreSQLPlanEstimate, error) { + startup, err := requiredEstimate(raw.StartupCost, "Startup Cost") + if err != nil { + return PostgreSQLPlanEstimate{}, err + } + total, err := requiredEstimate(raw.TotalCost, "Total Cost") + if err != nil { + return PostgreSQLPlanEstimate{}, err + } + rows, err := requiredEstimate(raw.PlanRows, "Plan Rows") + if err != nil { + return PostgreSQLPlanEstimate{}, err + } + width, err := requiredEstimate(raw.PlanWidth, "Plan Width") + if err != nil { + return PostgreSQLPlanEstimate{}, err + } + if math.IsNaN(startup) || math.IsInf(startup, 0) || math.IsNaN(total) || math.IsInf(total, 0) || + math.IsNaN(rows) || math.IsInf(rows, 0) || math.IsNaN(width) || math.IsInf(width, 0) { + return PostgreSQLPlanEstimate{}, errors.New("plan estimates must be finite") + } + if startup < 0 || total < 0 || rows < 0 || width < 0 { + return PostgreSQLPlanEstimate{}, errors.New("plan estimates must be non-negative") + } + if width != math.Trunc(width) { + return PostgreSQLPlanEstimate{}, errors.New("Plan Width must be a whole number") + } + return PostgreSQLPlanEstimate{StartupCost: startup, TotalCost: total, PlanRows: rows, PlanWidth: int(width)}, nil +} + +func requiredEstimate(value *float64, field string) (float64, error) { + if value == nil { + return 0, fmt.Errorf("plan node is missing %s", field) + } + return *value, nil +} + +func (c Control) explainCompiledForOutput(ctx context.Context, resolved Resolved, output lower.CompiledRecipeOutput, limit int) (compiler.CompiledQuery, error) { + if output.Plan.Engine != "" && output.Plan.Engine != ir.PhysicalEngineAQL && output.Plan.Engine != ir.PhysicalEngineClickHouse { + return compiler.CompiledQuery{}, fmt.Errorf("output %q has unsupported physical engine %q", output.Name, output.Plan.Engine) + } + bindings := resolved.Semantic.SemanticPlan.Bindings.Clone() + return compiler.CompileRecipeOutput(c.Engine.compileTarget, output, bindings, limit, ir.DefaultPhysicalOptimizationPolicy()) +} diff --git a/internal/dataframe/execution/explain_test.go b/internal/dataframe/execution/explain_test.go index 1cba85068..76cbffa15 100644 --- a/internal/dataframe/execution/explain_test.go +++ b/internal/dataframe/execution/explain_test.go @@ -1,16 +1,90 @@ package execution -import "testing" +import ( + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" +) + +func TestCompiledQueryFingerprintByteGolden(t *testing.T) { + const ( + aqlText = "FOR d IN @@collection FILTER d.p == @p SORT d.id LIMIT @__loom_page_size RETURN d" + wantAQL = "47d2ce3477240c9eead395870c755b5bfbc99eae8e6d6d96480e2c6909657954" + wantEmpty = "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d" + ) + aqlCommand, err := command.NewAQLCommand(aqlText, map[string]any{ + "@collection": "patients", + "p": "abc", + "__loom_page_size": 25, + "__loom_page_after": "", + "__loom_page_keys": []string{}, + }) + if err != nil { + t.Fatalf("build AQL command: %v", err) + } + plan := ir.PhysicalPlan{Version: 1, Engine: ir.PhysicalEngineAQL} + got := CompiledQueryFingerprint(compiler.CompiledQuery{PhysicalPlan: plan, Command: aqlCommand}) + if got != wantAQL { + t.Fatalf("AQL fingerprint drifted from the byte golden: got %s want %s", got, wantAQL) + } + flipped, err := command.NewAQLCommand(aqlText[:len(aqlText)-1]+"e", map[string]any{ + "@collection": "patients", + "p": "abc", + "__loom_page_size": 25, + "__loom_page_after": "", + "__loom_page_keys": []string{}, + }) + if err != nil { + t.Fatalf("build flipped command: %v", err) + } + if breakOne := CompiledQueryFingerprint(compiler.CompiledQuery{PhysicalPlan: plan, Command: flipped}); breakOne == wantAQL { + t.Fatalf("single-byte query flip did not change the digest: %s", breakOne) + } + pgCommand, err := command.NewPostgreSQLCommand("SELECT id FROM t WHERE p = $1", + []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterText, Value: "abc"}}) + if err != nil { + t.Fatalf("build PostgreSQL command: %v", err) + } + if got := CompiledQueryFingerprint(compiler.CompiledQuery{PhysicalPlan: plan, Command: pgCommand}); got != wantEmpty { + t.Fatalf("PostgreSQL-tagged command must contribute the pinned empty AQL shape: got %s want %s", got, wantEmpty) + } + if got := CompiledQueryFingerprint(compiler.CompiledQuery{PhysicalPlan: plan}); got != wantEmpty { + t.Fatalf("untagged command must contribute the pinned empty AQL shape: got %s want %s", got, wantEmpty) + } +} func TestCompiledQueryFingerprintExcludesBindValues(t *testing.T) { - left := CompiledQuery{Query: "FOR root IN Patient FILTER root.id == @id RETURN root", BindVars: map[string]any{"id": "one"}, Columns: []string{"id"}} + query := "FOR root IN Patient FILTER root.id == @id RETURN root" + left := compiler.CompiledQuery{Command: mustAQLCommand(query, map[string]any{"id": "one"}), Columns: []string{"id"}} right := left - right.BindVars = map[string]any{"id": "two"} + right.Command = mustAQLCommand(query, map[string]any{"id": "two"}) if got, want := CompiledQueryFingerprint(left), CompiledQueryFingerprint(right); got != want { t.Fatalf("fingerprint includes bind value: left=%q right=%q", got, want) } - right.BindVars["other"] = "value" + right.Command = mustAQLCommand(query, map[string]any{"id": "two", "other": "value"}) if CompiledQueryFingerprint(left) == CompiledQueryFingerprint(right) { t.Fatal("fingerprint did not include bind-key shape") } } + +func TestCompiledQueryFingerprintKeepsPostgreSQLShapeIndependentOfText(t *testing.T) { + first, err := command.NewPostgreSQLCommand("SELECT 1 AS id", []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterBigint, Value: int64(1)}}) + if err != nil { + t.Fatal(err) + } + second, err := command.NewPostgreSQLCommand("SELECT 2 AS id WHERE x = $1", nil) + if err != nil { + t.Fatal(err) + } + left := compiler.CompiledQuery{Command: first, Columns: []string{"id"}} + right := compiler.CompiledQuery{Command: second, Columns: []string{"id"}} + if got, want := CompiledQueryFingerprint(left), CompiledQueryFingerprint(right); got != want { + t.Fatalf("PostgreSQL plan fingerprint must keep the pre-boundary empty AQL shape: left=%q right=%q", got, want) + } + right.Columns = []string{"id", "name"} + if CompiledQueryFingerprint(left) == CompiledQueryFingerprint(right) { + t.Fatal("fingerprint did not include output column shape for a PostgreSQL plan") + } +} diff --git a/internal/dataframe/execution/group_preview_cache.go b/internal/dataframe/execution/group_preview_cache.go index 651639351..f31f60983 100644 --- a/internal/dataframe/execution/group_preview_cache.go +++ b/internal/dataframe/execution/group_preview_cache.go @@ -4,6 +4,8 @@ import ( "container/list" "encoding/json" "sync" + + "github.com/calypr/loom/internal/dataframe/compiler/command" ) const ( @@ -14,9 +16,6 @@ const ( groupPreviewCacheRowOverhead = 24 ) -// groupPreviewRowsKey identifies raw rows for one exact preview query and -// source collection version. It deliberately retains the full query and -// canonical bind JSON so a cache hit cannot depend on a digest collision. type groupPreviewRowsKey struct { query string bindVars string @@ -27,8 +26,12 @@ type groupPreviewRowsKey struct { decision bool } -func newGroupPreviewRowsKey(query string, bindVars map[string]any, collection, revision string, limit int) (groupPreviewRowsKey, int, bool) { - if query == "" || collection == "" || revision == "" || limit <= 0 { +func newGroupPreviewRowsKey(command command.CompiledCommand, collection, revision string, limit int) (groupPreviewRowsKey, int, bool) { + if collection == "" || revision == "" || limit <= 0 { + return groupPreviewRowsKey{}, 0, false + } + query, bindVars, dispatchErr := command.AQLParameters() + if dispatchErr != nil || query == "" { return groupPreviewRowsKey{}, 0, false } canonicalBinds, err := json.Marshal(bindVars) @@ -46,8 +49,12 @@ func newGroupPreviewRowsKey(query string, bindVars map[string]any, collection, r return key, weight, weight <= groupPreviewCacheMaxEntryBytes } -func newGroupPreviewProjectionDecisionKey(query string, bindVars map[string]any, collection, revision, indexIdentity string) (groupPreviewRowsKey, int, bool) { - if query == "" || collection == "" || revision == "" || indexIdentity == "" { +func newGroupPreviewProjectionDecisionKey(command command.CompiledCommand, collection, revision, indexIdentity string) (groupPreviewRowsKey, int, bool) { + if collection == "" || revision == "" || indexIdentity == "" { + return groupPreviewRowsKey{}, 0, false + } + query, bindVars, dispatchErr := command.AQLParameters() + if dispatchErr != nil || query == "" { return groupPreviewRowsKey{}, 0, false } canonicalBinds, err := json.Marshal(bindVars) @@ -84,9 +91,6 @@ type groupPreviewRowsEntry struct { weight int } -// groupPreviewRowsCache is a zero-value-usable LRU. Entries contain immutable -// JSON-encoded raw query rows; consumers decode fresh maps before applying the -// normal post-query checks and visitor. type groupPreviewRowsCache struct { mu sync.Mutex entries map[groupPreviewRowsKey]*list.Element diff --git a/internal/dataframe/execution/group_preview_cache_test.go b/internal/dataframe/execution/group_preview_cache_test.go index 4bb1565db..4f46404ae 100644 --- a/internal/dataframe/execution/group_preview_cache_test.go +++ b/internal/dataframe/execution/group_preview_cache_test.go @@ -4,26 +4,33 @@ import ( "context" "errors" "fmt" + "github.com/calypr/loom/internal/dataframe/compiler" "reflect" "strings" "testing" "github.com/calypr/loom/internal/authscope" "github.com/calypr/loom/internal/dataframe/recipe" - "github.com/calypr/loom/internal/store/arango" ) -func newGroupPreviewCacheTestEngine(t *testing.T, rows QueryRows, revision func(context.Context, string) (string, error)) (*Engine, Resolved) { +func newGroupPreviewCacheTestEngine(t *testing.T, rows SourceRows, revision func(context.Context, string) (string, error)) (*Engine, Resolved) { t.Helper() engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, ScopeDigest: func(recipe.RuntimeBindings) string { return "cache-test-scope" }, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil }, - PreviewQueryRows: rows, + PreviewSourceRows: rows, PreviewCollectionRevision: revision, }) if err != nil { @@ -47,8 +54,14 @@ func compileGroupPreviewCacheTestOutput(t *testing.T, engine *Engine, bindings r return resolved } -func groupPreviewCacheRows() QueryRows { - return func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { +func groupPreviewCacheRows() SourceRows { + return func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + for _, row := range []map[string]any{ {"status": "final", "rows": float64(2)}, {"status": "preliminary", "rows": float64(1)}, @@ -64,9 +77,16 @@ func groupPreviewCacheRows() QueryRows { func TestGroupPreviewRowsCacheReusesRawRowsAndSeparatesRevisionBindingsAndLimit(t *testing.T) { queryCalls := 0 currentRevision := "revision-1" - engine, resolved := newGroupPreviewCacheTestEngine(t, func(ctx context.Context, query string, batch int, bindVars map[string]any, visit func(map[string]any) error) error { + engine, resolved := newGroupPreviewCacheTestEngine(t, func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + batch := request.BatchSize + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + queryCalls++ - return groupPreviewCacheRows()(ctx, query, batch, bindVars, visit) + return groupPreviewCacheRows()(ctx, RowsRequest{Command: command, BatchSize: batch}, visit) }, func(context.Context, string) (string, error) { return currentRevision, nil }) @@ -115,9 +135,16 @@ func TestGroupPreviewRowsCacheOnlyStoresSuccessfulStableSourceReads(t *testing.T queryCalls := 0 revisions := []string{"before", "after", "after", "after"} revisionCall := 0 - engine, resolved := newGroupPreviewCacheTestEngine(t, func(ctx context.Context, query string, batch int, bindVars map[string]any, visit func(map[string]any) error) error { + engine, resolved := newGroupPreviewCacheTestEngine(t, func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + batch := request.BatchSize + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + queryCalls++ - return groupPreviewCacheRows()(ctx, query, batch, bindVars, visit) + return groupPreviewCacheRows()(ctx, RowsRequest{Command: command, BatchSize: batch}, visit) }, func(context.Context, string) (string, error) { value := revisions[revisionCall] revisionCall++ @@ -136,9 +163,16 @@ func TestGroupPreviewRowsCacheOnlyStoresSuccessfulStableSourceReads(t *testing.T t.Run("revision read failure bypasses lookup and insertion", func(t *testing.T) { queryCalls := 0 revisionCalls := 0 - engine, resolved := newGroupPreviewCacheTestEngine(t, func(ctx context.Context, query string, batch int, bindVars map[string]any, visit func(map[string]any) error) error { + engine, resolved := newGroupPreviewCacheTestEngine(t, func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + batch := request.BatchSize + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + queryCalls++ - return groupPreviewCacheRows()(ctx, query, batch, bindVars, visit) + return groupPreviewCacheRows()(ctx, RowsRequest{Command: command, BatchSize: batch}, visit) }, func(context.Context, string) (string, error) { revisionCalls++ if revisionCalls == 2 { @@ -166,12 +200,19 @@ func TestGroupPreviewRowsCacheOnlyStoresSuccessfulStableSourceReads(t *testing.T } { t.Run(test.name, func(t *testing.T) { queryCalls := 0 - engine, resolved := newGroupPreviewCacheTestEngine(t, func(ctx context.Context, query string, batch int, bindVars map[string]any, visit func(map[string]any) error) error { + engine, resolved := newGroupPreviewCacheTestEngine(t, func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + batch := request.BatchSize + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + queryCalls++ if test.queryError && queryCalls == 1 { return errors.New("query failed") } - return groupPreviewCacheRows()(ctx, query, batch, bindVars, visit) + return groupPreviewCacheRows()(ctx, RowsRequest{Command: command, BatchSize: batch}, visit) }, func(context.Context, string) (string, error) { return "revision-1", nil }) visitor := func(map[string]any) error { if test.visitError && queryCalls == 1 { @@ -196,9 +237,16 @@ func TestGroupPreviewRowsCacheOnlyStoresSuccessfulStableSourceReads(t *testing.T ctx, cancel := context.WithCancel(context.Background()) defer cancel() revisionCalls := 0 - engine, resolved := newGroupPreviewCacheTestEngine(t, func(ctx context.Context, query string, batch int, bindVars map[string]any, visit func(map[string]any) error) error { + engine, resolved := newGroupPreviewCacheTestEngine(t, func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + batch := request.BatchSize + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + queryCalls++ - return groupPreviewCacheRows()(ctx, query, batch, bindVars, visit) + return groupPreviewCacheRows()(ctx, RowsRequest{Command: command, BatchSize: batch}, visit) }, func(context.Context, string) (string, error) { revisionCalls++ if revisionCalls == 2 { @@ -221,7 +269,13 @@ func TestGroupPreviewRowsCacheOnlyStoresSuccessfulStableSourceReads(t *testing.T func TestGroupPreviewRowsCacheReplaysDefensiveRawRowsAndOnlyEligiblePlans(t *testing.T) { queryCalls := 0 revisionCalls := 0 - engine, resolved := newGroupPreviewCacheTestEngine(t, func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine, resolved := newGroupPreviewCacheTestEngine(t, func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + queryCalls++ return visit(map[string]any{ "status": map[string]any{"labels": []any{"original"}}, @@ -295,7 +349,7 @@ func TestGroupPreviewRowsCacheEnforcesEntryAndTotalByteBounds(t *testing.T) { } func TestGroupPreviewRowsCacheKeyUsesExactQueryBindingsCollectionRevisionAndLimit(t *testing.T) { - base, _, ok := newGroupPreviewRowsKey("RETURN @value", map[string]any{"value": "x"}, "Observation", "rev-1", 25) + base, _, ok := newGroupPreviewRowsKey(mustAQLCommand("RETURN @value", map[string]any{"value": "x"}), "Observation", "rev-1", 25) if !ok { t.Fatal("base key was rejected") } @@ -315,7 +369,7 @@ func TestGroupPreviewRowsCacheKeyUsesExactQueryBindingsCollectionRevisionAndLimi func TestGroupPreviewProjectionDecisionCacheSeparatesExactScopeRevisionAndIndex(t *testing.T) { base, _, ok := newGroupPreviewProjectionDecisionKey( - "RETURN @project, @dataset_generation", map[string]any{"project": "P1", "dataset_generation": "G1"}, + mustAQLCommand("RETURN @project, @dataset_generation", map[string]any{"project": "P1", "dataset_generation": "G1"}), "Observation", "revision-1", "index:loom_pivot_preview_a", ) if !ok { @@ -346,7 +400,7 @@ func TestGroupPreviewProjectionDecisionCacheSeparatesExactScopeRevisionAndIndex( } func TestGroupProjectionIndexIdentityIncludesCompleteSelectedDefinition(t *testing.T) { - base := arango.PreviewCoveringIndex{ + base := PreviewCollectionIndex{ Collection: "Observation", Name: "loom_pivot_preview_selected", Fields: []string{"project", "dataset_generation", "auth_resource_path", "payload.code.coding"}, StoredValues: []string{"payload.status"}, @@ -356,7 +410,7 @@ func TestGroupProjectionIndexIdentityIncludesCompleteSelectedDefinition(t *testi fieldChanged.Fields[3] = "payload.component.code.coding" storedValueChanged := base storedValueChanged.StoredValues = []string{"payload.display"} - for _, variant := range []arango.PreviewCoveringIndex{fieldChanged, storedValueChanged} { + for _, variant := range []PreviewCollectionIndex{fieldChanged, storedValueChanged} { if groupProjectionIndexIdentity(base) == groupProjectionIndexIdentity(variant) { t.Fatalf("selected index identity ignored its full definition: base=%+v variant=%+v", base, variant) } diff --git a/internal/dataframe/execution/group_preview_scan.go b/internal/dataframe/execution/group_preview_scan.go index dfc6a480f..69dba0ae4 100644 --- a/internal/dataframe/execution/group_preview_scan.go +++ b/internal/dataframe/execution/group_preview_scan.go @@ -11,14 +11,12 @@ import ( "time" "github.com/calypr/loom/internal/dataframe/compiler" - "github.com/calypr/loom/internal/store/arango" + "github.com/calypr/loom/internal/dataframe/compiler/command" ) const ( - groupScanAssessmentTimeout = 500 * time.Millisecond - groupScanFullCollectionCutoff = 0.90 - // Projection assessment and execution use a fixed per-query cap; admission - // leaves most of that cap available to other query state. + groupScanAssessmentTimeout = 500 * time.Millisecond + groupScanFullCollectionCutoff = 0.90 groupPreviewQueryMemoryLimitBytes int64 = 256 << 20 groupPreviewProjectionMemoryShare = 4 groupPreviewProjectionSafetyFactor = 8 @@ -27,7 +25,10 @@ const ( ) func (e *Engine) chooseGroupPreviewScan(ctx context.Context, query *compiler.CompiledQuery, stream *OutputStream) error { - if query == nil || stream == nil || stream.stream == nil || + if stream != nil && stream.command.Backend() == command.CompiledBackendPostgreSQL { + return contextError(ctx) + } + if query == nil || stream == nil || stream.sourceRows == nil || (query.PreviewGroupScan == nil && query.PreviewGroupProjection == nil) { return contextError(ctx) } @@ -37,15 +38,14 @@ func (e *Engine) chooseGroupPreviewScan(ctx context.Context, query *compiler.Com assessmentCtx, cancel := context.WithTimeout(ctx, groupScanAssessmentTimeout) defer cancel() - ordinaryQuery := query.Query - ordinaryBinds := cloneBindVars(query.BindVars) + ordinaryCommand := query.Command disableIndex := false - var selectedProjectionIndex *arango.PreviewCoveringIndex + var selectedProjectionIndex *PreviewCollectionIndex var assessedScopeCount int64 assessedScopeCountKnown := false assessedScopeCountAttempted := false coveringIndexSelected := false - if query.PreviewGroupScan != nil && e.previewExplainQuery != nil && query.PreviewCoveringIndex != nil && e.previewCoveringIndexes != nil { + if query.PreviewGroupScan != nil && e.previewCollectionScans != nil && query.PreviewCoveringIndex != nil && e.previewCoveringIndexes != nil { indexes, indexErr := e.previewCoveringIndexes(assessmentCtx, query.PreviewGroupScan.Collection) if err := contextError(ctx); err != nil { return err @@ -55,15 +55,15 @@ func (e *Engine) chooseGroupPreviewScan(ctx context.Context, query *compiler.Com if assessmentCtx.Err() != nil { return contextError(ctx) } - indexedQuery, indexedBinds, renderErr := compiler.RenderPreviewPhysicalPlanWithRootIndexHint(query.PhysicalPlan, index.Name) + indexedCommand, renderErr := compiler.RenderPreviewPhysicalPlanWithRootIndexHint(query.PhysicalPlan, index.Name) if renderErr != nil { continue } - plan, explainErr := e.previewExplainQuery(assessmentCtx, indexedQuery, indexedBinds) + scans, scansErr := e.previewCollectionScans(assessmentCtx, indexedCommand, query.PreviewGroupScan.Collection) if assessmentCtx.Err() != nil { return contextError(ctx) } - if explainErr != nil || !hasCoveringRootIndexNamed(plan, query.PreviewGroupScan.Collection, index.Name) { + if scansErr != nil || !hasCoveringRootIndexNamed(scans, query.PreviewGroupScan.Collection, index.Name) { continue } selected := *query.PreviewCoveringIndex @@ -78,28 +78,28 @@ func (e *Engine) chooseGroupPreviewScan(ctx context.Context, query *compiler.Com selectedDefinition.Fields = append([]string(nil), index.Fields...) selectedDefinition.StoredValues = append([]string(nil), index.StoredValues...) selectedProjectionIndex = &selectedDefinition - query.Query = indexedQuery - query.BindVars = indexedBinds - stream.query = indexedQuery - stream.bindVars = indexedBinds + query.Command = indexedCommand + stream.command = indexedCommand coveringIndexSelected = true break } } } - if !coveringIndexSelected && query.PreviewGroupScan != nil && e.previewExplainQuery != nil && assessmentCtx.Err() == nil { - plan, err := e.previewExplainQuery(assessmentCtx, query.Query, query.BindVars) - if err == nil && assessmentCtx.Err() == nil && hasNonCoveringRootIndex(plan, query.PreviewGroupScan.Collection) { + if !coveringIndexSelected && query.PreviewGroupScan != nil && e.previewCollectionScans != nil && assessmentCtx.Err() == nil { + if _, _, dispatchErr := query.Command.AQLParameters(); dispatchErr != nil { + return dispatchErr + } + scans, err := e.previewCollectionScans(assessmentCtx, query.Command, query.PreviewGroupScan.Collection) + if err == nil && assessmentCtx.Err() == nil && hasNonCoveringRootIndex(scans, query.PreviewGroupScan.Collection) { if e.previewCollectionCount == nil && query.PreviewGroupProjection == nil { - // Keep the canonical indexed query when the broad-scan decision lacks exact evidence. } else { projectionScopeCount := scopeCountMatchesProjection(query) assessedScopeCountAttempted = projectionScopeCount countCtx := assessmentCtx if projectionScopeCount { - countCtx = arango.WithQueryMemoryLimit(assessmentCtx, groupPreviewQueryMemoryLimitBytes) + countCtx = WithQueryMemoryBudget(assessmentCtx, groupPreviewQueryMemoryLimitBytes) } - scanScopeCount, scanScopeCountKnown := readGroupPreviewCount(countCtx, stream, query.PreviewGroupScan.ScopeCountQuery, query.PreviewGroupScan.ScopeCountBindVars) + scanScopeCount, scanScopeCountKnown := readGroupPreviewCount(countCtx, stream, query.PreviewGroupScan.ScopeCount) if projectionScopeCount && scanScopeCountKnown { assessedScopeCount = scanScopeCount assessedScopeCountKnown = true @@ -108,10 +108,8 @@ func (e *Engine) chooseGroupPreviewScan(ctx context.Context, query *compiler.Com totalCount, countErr := e.previewCollectionCount(assessmentCtx, query.PreviewGroupScan.Collection) if countErr == nil && assessmentCtx.Err() == nil && totalCount > 0 && scanScopeCount >= 0 && scanScopeCount <= totalCount && float64(scanScopeCount)/float64(totalCount) >= groupScanFullCollectionCutoff { - query.Query = query.PreviewGroupScan.SequentialQuery - query.BindVars = query.PreviewGroupScan.SequentialBindVars - stream.query = query.Query - stream.bindVars = query.BindVars + query.Command = query.PreviewGroupScan.Sequential + stream.command = query.Command disableIndex = true } } @@ -123,7 +121,7 @@ func (e *Engine) chooseGroupPreviewScan(ctx context.Context, query *compiler.Com } if query.PreviewGroupProjection != nil && !disableIndex { if selectedIndex, selected := e.selectGroupProjectionIndex(assessmentCtx, query, selectedProjectionIndex); selected { - if err := e.chooseGroupProjectionFactorization(assessmentCtx, query, stream, ordinaryQuery, ordinaryBinds, selectedIndex, + if err := e.chooseGroupProjectionFactorization(assessmentCtx, query, stream, ordinaryCommand, selectedIndex, assessedScopeCount, assessedScopeCountKnown, assessedScopeCountAttempted); err != nil { return err } @@ -132,13 +130,13 @@ func (e *Engine) chooseGroupPreviewScan(ctx context.Context, query *compiler.Com return contextError(ctx) } -func readGroupPreviewCount(ctx context.Context, stream *OutputStream, query string, bindVars map[string]any) (int64, bool) { - if stream == nil || stream.stream == nil || query == "" { +func readGroupPreviewCount(ctx context.Context, stream *OutputStream, scopeCount command.CompiledCommand) (int64, bool) { + if stream == nil || stream.sourceRows == nil || scopeCount.Backend() != command.CompiledBackendAQL { return 0, false } var count int64 seen := false - err := stream.stream(ctx, query, 1, bindVars, func(row map[string]any) error { + err := stream.sourceRows(ctx, RowsRequest{Command: scopeCount, BatchSize: 1}, func(row map[string]any) error { if seen { return errors.New("root scope count returned more than one row") } @@ -157,50 +155,50 @@ func scopeCountMatchesProjection(query *compiler.CompiledQuery) bool { if query == nil || query.PreviewGroupScan == nil || query.PreviewGroupProjection == nil { return false } - return query.PreviewGroupScan.ScopeCountQuery == query.PreviewGroupProjection.ScopeCountQuery && - reflect.DeepEqual(query.PreviewGroupScan.ScopeCountBindVars, query.PreviewGroupProjection.ScopeCountBindVars) + return reflect.DeepEqual(query.PreviewGroupScan.ScopeCount, query.PreviewGroupProjection.ScopeCount) } -func (e *Engine) selectGroupProjectionIndex(ctx context.Context, query *compiler.CompiledQuery, preferred *arango.PreviewCoveringIndex) (arango.PreviewCoveringIndex, bool) { +func (e *Engine) selectGroupProjectionIndex(ctx context.Context, query *compiler.CompiledQuery, preferred *PreviewCollectionIndex) (PreviewCollectionIndex, bool) { if query == nil || query.PreviewGroupProjection == nil || query.PreviewGroupProjection.ReadOnlyIndex == nil || - query.PreviewGroupProjection.Collection == "" || query.PreviewGroupProjection.ScopeCountQuery == "" || - e.previewExplainQuery == nil || ctx.Err() != nil { - return arango.PreviewCoveringIndex{}, false + query.PreviewGroupProjection.Collection == "" || + query.PreviewGroupProjection.ScopeCount.Backend() != command.CompiledBackendAQL || + e.previewCollectionScans == nil || ctx.Err() != nil { + return PreviewCollectionIndex{}, false } spec := query.PreviewGroupProjection - var indexes []arango.PreviewCoveringIndex + var indexes []PreviewCollectionIndex if preferred != nil { - indexes = []arango.PreviewCoveringIndex{*preferred} + indexes = []PreviewCollectionIndex{*preferred} } else { if e.previewCoveringIndexes == nil { - return arango.PreviewCoveringIndex{}, false + return PreviewCollectionIndex{}, false } var err error indexes, err = e.previewCoveringIndexes(ctx, spec.Collection) if err != nil || ctx.Err() != nil { - return arango.PreviewCoveringIndex{}, false + return PreviewCollectionIndex{}, false } } for _, index := range compatibleGroupProjectionIndexes(spec.Collection, spec.ReadOnlyIndex, indexes) { if ctx.Err() != nil { - return arango.PreviewCoveringIndex{}, false + return PreviewCollectionIndex{}, false } - factoredQuery, factoredBinds, renderErr := compiler.RenderPreviewGroupProjectionFactorization(query.PhysicalPlan, index.Name, false) - if renderErr != nil || factoredQuery == "" { + factoredCommand, renderErr := compiler.RenderPreviewGroupProjectionFactorization(query.PhysicalPlan, index.Name, false) + if renderErr != nil { continue } - plan, explainErr := e.previewExplainQuery(ctx, factoredQuery, factoredBinds) - if explainErr != nil || ctx.Err() != nil || !hasCoveringRootIndexNamed(plan, spec.Collection, index.Name) { + scans, scansErr := e.previewCollectionScans(ctx, factoredCommand, spec.Collection) + if scansErr != nil || ctx.Err() != nil || !hasCoveringRootIndexNamed(scans, spec.Collection, index.Name) { continue } index.Fields = append([]string(nil), index.Fields...) index.StoredValues = append([]string(nil), index.StoredValues...) return index, true } - return arango.PreviewCoveringIndex{}, false + return PreviewCollectionIndex{}, false } -func compatibleGroupProjectionIndexes(collection string, required *compiler.PreviewGroupProjectionIndexSpec, indexes []arango.PreviewCoveringIndex) []arango.PreviewCoveringIndex { +func compatibleGroupProjectionIndexes(collection string, required *compiler.PreviewGroupProjectionIndexSpec, indexes []PreviewCollectionIndex) []PreviewCollectionIndex { if required == nil || len(required.RequiredFields) == 0 || required.ProjectionPath == "" { return nil } @@ -211,7 +209,7 @@ func compatibleGroupProjectionIndexes(collection string, required *compiler.Prev } requiredFields[field] = struct{}{} } - compatible := make([]arango.PreviewCoveringIndex, 0, len(indexes)) + compatible := make([]PreviewCollectionIndex, 0, len(indexes)) for _, index := range indexes { if index.Collection != collection || !strings.HasPrefix(index.Name, "loom_pivot_preview_") || len(index.Fields) == 0 { continue @@ -271,45 +269,30 @@ func compatibleGroupProjectionIndexes(collection string, required *compiler.Prev return compatible } -func hasCoveringRootIndexNamed(result arango.ExplainResult, collection, indexName string) bool { - plan := singleExplainPlan(result) - if plan == nil || collection == "" || indexName == "" { +func hasCoveringRootIndexNamed(evidence PreviewCollectionScanEvidence, collection, indexName string) bool { + if collection == "" || indexName == "" { return false } - rootIndexes := 0 - for _, node := range plan.Nodes { - if node.Collection != collection { - continue - } - switch node.Type { - case "EnumerateCollectionNode": - return false - case "IndexNode": - rootIndexes++ - if node.IndexCoversProjections == nil || !*node.IndexCoversProjections || - node.IndexCoversOutProjections == nil || !*node.IndexCoversOutProjections { - return false - } - found := false - for _, index := range node.Indexes { - if index.Name == indexName && (index.Collection == "" || index.Collection == collection) { - found = true - break - } - } - if !found { - return false - } + if evidence.FullCollectionEnumeration || len(evidence.IndexNodes) != 1 { + return false + } + node := evidence.IndexNodes[0] + if node.Coverage != PreviewIndexCoverageCovering { + return false + } + for _, name := range node.IndexNames { + if name == indexName { + return true } } - return rootIndexes == 1 + return false } func (e *Engine) chooseGroupProjectionFactorization(ctx context.Context, query *compiler.CompiledQuery, stream *OutputStream, - ordinaryQuery string, ordinaryBinds map[string]any, selectedIndex arango.PreviewCoveringIndex, + ordinaryCommand command.CompiledCommand, selectedIndex PreviewCollectionIndex, assessedScopeCount int64, assessedScopeCountKnown, assessedScopeCountAttempted bool) error { spec := query.PreviewGroupProjection - if spec == nil || spec.Collection == "" || spec.ScopeCountQuery == "" || e.previewCollectionRevision == nil { + if spec == nil || spec.Collection == "" || spec.ScopeCount.Backend() != command.CompiledBackendAQL || e.previewCollectionRevision == nil { return nil } if ctx.Err() != nil { @@ -324,7 +307,7 @@ func (e *Engine) chooseGroupProjectionFactorization(ctx context.Context, query * } indexName := selectedIndex.Name indexIdentity := groupProjectionIndexIdentity(selectedIndex) - decisionKey, _, keyOK := newGroupPreviewProjectionDecisionKey(ordinaryQuery, ordinaryBinds, spec.Collection, revision, indexIdentity) + decisionKey, _, keyOK := newGroupPreviewProjectionDecisionKey(ordinaryCommand, spec.Collection, revision, indexIdentity) if !keyOK { return nil } @@ -337,20 +320,20 @@ func (e *Engine) chooseGroupProjectionFactorization(ctx context.Context, query * } if !assessedScopeCountKnown && !assessedScopeCountAttempted { - countCtx := arango.WithQueryMemoryLimit(ctx, groupPreviewQueryMemoryLimitBytes) - assessedScopeCount, assessedScopeCountKnown = readGroupPreviewCount(countCtx, stream, spec.ScopeCountQuery, spec.ScopeCountBindVars) + countCtx := WithQueryMemoryBudget(ctx, groupPreviewQueryMemoryLimitBytes) + assessedScopeCount, assessedScopeCountKnown = readGroupPreviewCount(countCtx, stream, spec.ScopeCount) } if !assessedScopeCountKnown || assessedScopeCount <= 0 { return nil } - assessmentQuery, assessmentBinds, renderErr := compiler.RenderPreviewGroupProjectionAssessment(query.PhysicalPlan, indexName, false) - if renderErr != nil || assessmentQuery == "" { + assessmentCommand, renderErr := compiler.RenderPreviewGroupProjectionAssessment(query.PhysicalPlan, indexName, false) + if renderErr != nil || assessmentCommand.Backend() != command.CompiledBackendAQL { return nil } - assessmentCtx := arango.WithQueryMemoryLimit(ctx, groupPreviewQueryMemoryLimitBytes) + assessmentCtx := WithQueryMemoryBudget(ctx, groupPreviewQueryMemoryLimitBytes) var sample groupProjectionSample seen := false - err := stream.stream(assessmentCtx, assessmentQuery, 1, assessmentBinds, func(row map[string]any) error { + err := stream.sourceRows(assessmentCtx, RowsRequest{Command: assessmentCommand, BatchSize: 1}, func(row map[string]any) error { if seen { return errors.New("Group projection sample returned more than one row") } @@ -455,7 +438,7 @@ func estimateGroupProjectionDictionaryBytes(sample groupProjectionSample, scoped return base * entryBytes, true } -func groupProjectionIndexIdentity(selected arango.PreviewCoveringIndex) string { +func groupProjectionIndexIdentity(selected PreviewCollectionIndex) string { encoded, err := json.Marshal(selected) if err == nil { return "index:" + string(encoded) @@ -464,19 +447,17 @@ func groupProjectionIndexIdentity(selected arango.PreviewCoveringIndex) string { } func applyGroupProjectionFactorization(query *compiler.CompiledQuery, stream *OutputStream, indexName string) bool { - factoredQuery, factoredBinds, err := compiler.RenderPreviewGroupProjectionFactorization(query.PhysicalPlan, indexName, false) - if err != nil || factoredQuery == "" { + factoredCommand, err := compiler.RenderPreviewGroupProjectionFactorization(query.PhysicalPlan, indexName, false) + if err != nil || factoredCommand.Backend() != command.CompiledBackendAQL { return false } - query.Query = factoredQuery - query.BindVars = factoredBinds - stream.query = factoredQuery - stream.bindVars = factoredBinds + query.Command = factoredCommand + stream.command = factoredCommand stream.queryMemoryLimit = groupPreviewQueryMemoryLimitBytes return true } -func compatibleGroupPreviewIndexes(required *compiler.PreviewCoveringIndexSpec, indexes []arango.PreviewCoveringIndex) []arango.PreviewCoveringIndex { +func compatibleGroupPreviewIndexes(required *compiler.PreviewCoveringIndexSpec, indexes []PreviewCollectionIndex) []PreviewCollectionIndex { if required == nil || len(required.Fields) == 0 { return nil } @@ -487,7 +468,7 @@ func compatibleGroupPreviewIndexes(required *compiler.PreviewCoveringIndexSpec, } requiredFields[field] = struct{}{} } - compatible := make([]arango.PreviewCoveringIndex, 0, len(indexes)) + compatible := make([]PreviewCollectionIndex, 0, len(indexes)) for _, index := range indexes { if index.Collection != required.Collection || !strings.HasPrefix(index.Name, "loom_pivot_preview_") || len(index.Fields) < len(requiredFields) { @@ -523,39 +504,11 @@ func compatibleGroupPreviewIndexes(required *compiler.PreviewCoveringIndexSpec, return compatible } -func hasNonCoveringRootIndex(result arango.ExplainResult, collection string) bool { - plan := singleExplainPlan(result) - if plan == nil { +func hasNonCoveringRootIndex(evidence PreviewCollectionScanEvidence, collection string) bool { + if evidence.FullCollectionEnumeration || len(evidence.IndexNodes) != 1 { return false } - rootIndexes := 0 - for _, node := range plan.Nodes { - if node.Collection != collection { - continue - } - switch node.Type { - case "EnumerateCollectionNode": - return false - case "IndexNode": - rootIndexes++ - if node.IndexCoversProjections == nil || *node.IndexCoversProjections || - node.IndexCoversOutProjections == nil || *node.IndexCoversOutProjections { - return false - } - } - } - return rootIndexes == 1 -} - -func singleExplainPlan(result arango.ExplainResult) *arango.ExplainPlan { - switch { - case result.Plan != nil && len(result.Plans) == 0: - return result.Plan - case result.Plan == nil && len(result.Plans) == 1: - return &result.Plans[0] - default: - return nil - } + return evidence.IndexNodes[0].Coverage == PreviewIndexCoverageNotCovering } func previewCountValue(value any) (int64, bool) { diff --git a/internal/dataframe/execution/group_preview_scan_test.go b/internal/dataframe/execution/group_preview_scan_test.go index 3b04ce10d..865fe2ae0 100644 --- a/internal/dataframe/execution/group_preview_scan_test.go +++ b/internal/dataframe/execution/group_preview_scan_test.go @@ -10,9 +10,11 @@ import ( "github.com/arangodb/go-driver/v2/arangodb/shared" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" "github.com/calypr/loom/internal/dataframe/recipe" - "github.com/calypr/loom/internal/store/arango" ) func TestChooseGroupPreviewScanReusesSmallestCompatibleCoveringIndex(t *testing.T) { @@ -24,25 +26,32 @@ func TestChooseGroupPreviewScanReusesSmallestCompatibleCoveringIndex(t *testing. if required == nil { t.Fatal("fixture query has no compiler-owned covering index requirements") } - originalQuery := query.Query - originalBinds := query.BindVars + originalQuery := commandText(t, query.Command) + originalBinds := commandBinds(t, query.Command) largeFields := append(append([]string(nil), required.Fields...), "payload.extra", "resourceType") smallFields := []string{"project", "dataset_generation", "resourceType"} smallFields = append(smallFields, required.Fields[2:]...) - indexes := []arango.PreviewCoveringIndex{ + indexes := []PreviewCollectionIndex{ {Collection: required.Collection, Name: "loom_pivot_preview_existing_larger", Fields: largeFields}, {Collection: required.Collection, Name: "loom_pivot_preview_existing_smaller", Fields: smallFields}, } indexCalls, explainCalls := 0, 0 - engine.previewCoveringIndexes = func(_ context.Context, collection string) ([]arango.PreviewCoveringIndex, error) { + engine.previewCoveringIndexes = func(_ context.Context, collection string) ([]PreviewCollectionIndex, error) { indexCalls++ if collection != required.Collection { t.Fatalf("index collection = %q, want %q", collection, required.Collection) } return indexes, nil } - engine.previewExplainQuery = func(_ context.Context, queryText string, bindVars map[string]any) (arango.ExplainResult, error) { + engine.previewCollectionScans = func(_ context.Context, command command.CompiledCommand, collection string) (PreviewCollectionScanEvidence, error) { explainCalls++ + if collection != required.Collection { + t.Fatalf("scan collection = %q, want %q", collection, required.Collection) + } + queryText, bindVars, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return PreviewCollectionScanEvidence{}, dispatchErr + } if strings.Contains(queryText, "loom_pivot_preview_existing_larger") { t.Fatalf("larger compatible index was explained before the smaller one:\n%s", queryText) } @@ -54,13 +63,19 @@ func TestChooseGroupPreviewScanReusesSmallestCompatibleCoveringIndex(t *testing. !reflect.DeepEqual(bindVars["auth_resource_paths"], originalBinds["auth_resource_paths"]) { t.Fatalf("candidate query changed restricted authorization binds: got=%v want=%v", bindVars, originalBinds) } - return coveringGroupExplainWithIndex(required.Collection, indexes[1].Name), nil + return coveringGroupScans(indexes[1].Name), nil } engine.previewCollectionCount = func(context.Context, string) (int64, error) { t.Fatal("covering candidate should skip the sequential-scan assessment") return 0, nil } - stream.stream = func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + stream.sourceRows = func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + t.Fatal("covering candidate should skip the scoped source count") return nil } @@ -70,35 +85,51 @@ func TestChooseGroupPreviewScanReusesSmallestCompatibleCoveringIndex(t *testing. if indexCalls != 1 || explainCalls != 1 { t.Fatalf("metadata/explain calls = %d/%d, want one each", indexCalls, explainCalls) } - if query.Query == originalQuery || query.Query != stream.query || query.PreviewCoveringIndex.Name != indexes[1].Name || - !reflect.DeepEqual(query.PreviewCoveringIndex.Fields, smallFields) || !reflect.DeepEqual(query.BindVars, originalBinds) || - !reflect.DeepEqual(query.BindVars, stream.bindVars) { - t.Fatalf("selected compatible query was not applied consistently: query=%q stream=%q spec=%+v", query.Query, stream.query, query.PreviewCoveringIndex) + selectedText, selectedBinds := aqlView(t, query.Command) + streamText, streamBinds := aqlView(t, stream.command) + if selectedText == originalQuery || selectedText != streamText || query.PreviewCoveringIndex.Name != indexes[1].Name || + !reflect.DeepEqual(query.PreviewCoveringIndex.Fields, smallFields) || !reflect.DeepEqual(selectedBinds, originalBinds) || + !reflect.DeepEqual(selectedBinds, streamBinds) { + t.Fatalf("selected compatible query was not applied consistently: query=%q stream=%q spec=%+v", selectedText, streamText, query.PreviewCoveringIndex) } } func TestChooseGroupPreviewScanKeepsSequentialFallbackWhenNoCompatibleIndexExists(t *testing.T) { engine, query, stream := compiledGroupPreviewScanFixture(t, recipe.RuntimeBindings{Project: "P1", DatasetGeneration: "G1"}) - originalQuery := query.Query + originalQuery := commandText(t, query.Command) required := query.PreviewCoveringIndex - engine.previewCoveringIndexes = func(context.Context, string) ([]arango.PreviewCoveringIndex, error) { - return []arango.PreviewCoveringIndex{{ + engine.previewCoveringIndexes = func(context.Context, string) ([]PreviewCollectionIndex, error) { + return []PreviewCollectionIndex{{ Collection: required.Collection, Name: "loom_pivot_preview_missing_auth", Fields: []string{"project", "dataset_generation", "payload.status"}, }}, nil } explainCalls, scopeCountCalls := 0, 0 - engine.previewExplainQuery = func(_ context.Context, queryText string, _ map[string]any) (arango.ExplainResult, error) { + engine.previewCollectionScans = func(_ context.Context, command command.CompiledCommand, collection string) (PreviewCollectionScanEvidence, error) { explainCalls++ + if collection != required.Collection { + t.Fatalf("scan collection = %q, want %q", collection, required.Collection) + } + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return PreviewCollectionScanEvidence{}, dispatchErr + } if queryText != originalQuery { t.Fatalf("incompatible index changed the canonical query:\n%s", queryText) } - return groupExplainWithCoverage(false, false, required.Collection), nil + return notCoveringGroupScans(), nil } engine.previewCollectionCount = func(context.Context, string) (int64, error) { return 1000, nil } - stream.stream = func(_ context.Context, queryText string, batchSize int, _ map[string]any, visit func(map[string]any) error) error { + stream.sourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + batchSize := request.BatchSize + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + scopeCountCalls++ - if queryText != query.PreviewGroupScan.ScopeCountQuery || batchSize != 1 { + if !reflect.DeepEqual(command, query.PreviewGroupScan.ScopeCount) || batchSize != 1 { t.Fatalf("scope-count query = %q batch=%d", queryText, batchSize) } return visit(map[string]any{"count": float64(950)}) @@ -106,46 +137,70 @@ func TestChooseGroupPreviewScanKeepsSequentialFallbackWhenNoCompatibleIndexExist if err := engine.chooseGroupPreviewScan(context.Background(), &query, &stream); err != nil { t.Fatal(err) } - if query.Query != query.PreviewGroupScan.SequentialQuery || explainCalls != 1 || scopeCountCalls != 1 { - t.Fatalf("sequential fallback state: query=%q explains=%d scope counts=%d", query.Query, explainCalls, scopeCountCalls) + if !reflect.DeepEqual(query.Command, query.PreviewGroupScan.Sequential) || explainCalls != 1 || scopeCountCalls != 1 { + t.Fatalf("sequential fallback state: query=%q explains=%d scope counts=%d", commandText(t, query.Command), explainCalls, scopeCountCalls) } } func TestChooseGroupPreviewScanRequiresExplainToProveCoveringIndex(t *testing.T) { for _, test := range []struct { - name string - plan arango.ExplainResult + name string + candidate PreviewCollectionScanEvidence }{ - {name: "projection uncovered", plan: groupExplainWithCoverage(false, true)}, - {name: "output uncovered", plan: groupExplainWithCoverage(true, false)}, - {name: "coverage unknown", plan: arango.ExplainResult{Plan: &arango.ExplainPlan{Nodes: []arango.ExplainNode{{Type: "IndexNode", Collection: "Observation"}}}}}, + {name: "projection uncovered", candidate: unknownCoverageGroupScans()}, + {name: "output uncovered", candidate: unknownCoverageGroupScans()}, + {name: "coverage unknown", candidate: unknownCoverageGroupScans()}, + {name: "covering wrong index", candidate: coveringGroupScans("loom_pivot_preview_something_else")}, + {name: "covering twice", candidate: PreviewCollectionScanEvidence{IndexNodes: []PreviewIndexNodeEvidence{ + {IndexNames: []string{"loom_pivot_preview_unproven"}, Coverage: PreviewIndexCoverageCovering}, + {IndexNames: []string{"loom_pivot_preview_unproven"}, Coverage: PreviewIndexCoverageCovering}, + }}}, + {name: "enumeration with covering", candidate: PreviewCollectionScanEvidence{ + FullCollectionEnumeration: true, + IndexNodes: []PreviewIndexNodeEvidence{ + {IndexNames: []string{"loom_pivot_preview_unproven"}, Coverage: PreviewIndexCoverageCovering}, + }, + }}, } { t.Run(test.name, func(t *testing.T) { engine, query, stream := compiledGroupPreviewScanFixture(t, recipe.RuntimeBindings{Project: "P1", DatasetGeneration: "G1"}) - originalQuery := query.Query - candidate := arango.PreviewCoveringIndex{ + originalQuery := commandText(t, query.Command) + candidate := PreviewCollectionIndex{ Collection: query.PreviewCoveringIndex.Collection, Name: "loom_pivot_preview_unproven", Fields: append([]string(nil), query.PreviewCoveringIndex.Fields...), } - engine.previewCoveringIndexes = func(context.Context, string) ([]arango.PreviewCoveringIndex, error) { - return []arango.PreviewCoveringIndex{candidate}, nil + engine.previewCoveringIndexes = func(context.Context, string) ([]PreviewCollectionIndex, error) { + return []PreviewCollectionIndex{candidate}, nil } explainCalls := 0 - engine.previewExplainQuery = func(_ context.Context, queryText string, _ map[string]any) (arango.ExplainResult, error) { + engine.previewCollectionScans = func(_ context.Context, command command.CompiledCommand, collection string) (PreviewCollectionScanEvidence, error) { explainCalls++ + if collection != query.PreviewGroupScan.Collection && collection != query.PreviewCoveringIndex.Collection { + t.Fatalf("scan collection = %q", collection) + } + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return PreviewCollectionScanEvidence{}, dispatchErr + } if queryText == originalQuery { - return groupExplainWithCoverage(false, false, query.PreviewGroupScan.Collection), nil + return notCoveringGroupScans(), nil } - return test.plan, nil + return test.candidate, nil } engine.previewCollectionCount = func(context.Context, string) (int64, error) { return 1000, nil } - stream.stream = func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + stream.sourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{"count": float64(500)}) } if err := engine.chooseGroupPreviewScan(context.Background(), &query, &stream); err != nil { t.Fatal(err) } - if query.Query != originalQuery || stream.query != originalQuery || explainCalls != 2 { - t.Fatalf("unproven candidate changed canonical query: query=%q stream=%q explains=%d", query.Query, stream.query, explainCalls) + if !reflect.DeepEqual(query.Command, stream.command) || commandText(t, query.Command) != originalQuery || commandText(t, stream.command) != originalQuery || explainCalls != 2 { + t.Fatalf("unproven candidate changed canonical query: query=%q stream=%q explains=%d", commandText(t, query.Command), commandText(t, stream.command), explainCalls) } }) } @@ -154,31 +209,42 @@ func TestChooseGroupPreviewScanRequiresExplainToProveCoveringIndex(t *testing.T) func TestChooseGroupPreviewScanDoesNotReplaceSequentialChoiceWithProjectionIndex(t *testing.T) { engine, _, query, stream := semanticGroupProjectionExecutionFixture(t, nil) query.PreviewGroupScan = &compiler.PreviewGroupScanSpec{ - Collection: query.PreviewGroupProjection.Collection, - SequentialQuery: "sequential-query", - SequentialBindVars: map[string]any{"scan": "sequential"}, - ScopeCountQuery: query.PreviewGroupProjection.ScopeCountQuery, - ScopeCountBindVars: query.PreviewGroupProjection.ScopeCountBindVars, + Collection: query.PreviewGroupProjection.Collection, + Sequential: mustAQLCommand("sequential-query", map[string]any{"scan": "sequential"}), + ScopeCount: query.PreviewGroupProjection.ScopeCount, } - ordinaryQuery := query.Query + ordinaryQuery := commandText(t, query.Command) indexCalls := 0 - engine.previewCoveringIndexes = func(context.Context, string) ([]arango.PreviewCoveringIndex, error) { + engine.previewCoveringIndexes = func(context.Context, string) ([]PreviewCollectionIndex, error) { indexCalls++ - return []arango.PreviewCoveringIndex{{ + return []PreviewCollectionIndex{{ Collection: query.PreviewGroupProjection.Collection, Name: "loom_pivot_preview_projection", Fields: []string{"project", "dataset_generation", "auth_resource_path", "payload.code.coding"}, }}, nil } - engine.previewExplainQuery = func(_ context.Context, queryText string, _ map[string]any) (arango.ExplainResult, error) { + engine.previewCollectionScans = func(_ context.Context, command command.CompiledCommand, collection string) (PreviewCollectionScanEvidence, error) { + if collection != query.PreviewGroupProjection.Collection { + t.Fatalf("scan collection = %q, want %q", collection, query.PreviewGroupProjection.Collection) + } + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return PreviewCollectionScanEvidence{}, dispatchErr + } if queryText != ordinaryQuery { t.Fatalf("initial Group scan explain=%q, want original plan", queryText) } - return groupExplainWithCoverage(false, false, query.PreviewGroupProjection.Collection), nil + return notCoveringGroupScans(), nil } engine.previewCollectionCount = func(context.Context, string) (int64, error) { return 1000, nil } - stream.stream = func(_ context.Context, queryText string, _ int, _ map[string]any, visit func(map[string]any) error) error { - if queryText != query.PreviewGroupProjection.ScopeCountQuery { + stream.sourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + + if !reflect.DeepEqual(command, query.PreviewGroupProjection.ScopeCount) { t.Fatalf("unexpected query %q", queryText) } return visit(map[string]any{"count": int64(950)}) @@ -190,8 +256,8 @@ func TestChooseGroupPreviewScanDoesNotReplaceSequentialChoiceWithProjectionIndex if err := engine.chooseGroupPreviewScan(context.Background(), &query, &stream); err != nil { t.Fatal(err) } - if indexCalls != 0 || query.Query != "sequential-query" || query.Query == ordinaryQuery { - t.Fatalf("sequential plan was overridden: index lookups=%d query=%q", indexCalls, query.Query) + if indexCalls != 0 || commandText(t, query.Command) != "sequential-query" || commandText(t, query.Command) == ordinaryQuery { + t.Fatalf("sequential plan was overridden: index lookups=%d query=%q", indexCalls, commandText(t, query.Command)) } } @@ -202,7 +268,7 @@ func TestCompatibleGroupProjectionIndexesRequiresOwnedScopeAndProjectionCoverage } baseFields := []string{"project", "dataset_generation", "auth_resource_path"} withProjection := append(append([]string(nil), baseFields...), required.ProjectionPath) - indexes := []arango.PreviewCoveringIndex{ + indexes := []PreviewCollectionIndex{ {Collection: "Observation", Name: "other_preview_owned", Fields: withProjection}, {Collection: "Observation", Name: "loom_pivot_preview_missing_scope", Fields: []string{"project", "dataset_generation", required.ProjectionPath}}, {Collection: "Patient", Name: "loom_pivot_preview_wrong_collection", Fields: withProjection}, @@ -218,33 +284,55 @@ func TestCompatibleGroupProjectionIndexesRequiresOwnedScopeAndProjectionCoverage func TestPreviewOutputReusesStoredValuesGroupIndexWithoutPrewarmingIt(t *testing.T) { queryCalls, prepareCalls := 0, 0 - index := arango.PreviewCoveringIndex{ + index := PreviewCollectionIndex{ Collection: "Observation", Name: "loom_pivot_preview_existing_stored_values", Fields: []string{"project", "dataset_generation", "auth_resource_path", "_key", "payload.id", "payload.status"}, StoredValues: []string{"payload.valueQuantity"}, } engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, - ScopeDigest: func(recipe.RuntimeBindings) string { return "group-preview-existing-index" }, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, - PreviewQueryRows: func(ctx context.Context, query string, batch int, binds map[string]any, visit func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "group-preview-existing-index" }, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }, + PreviewSourceRows: func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + batch := request.BatchSize + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + queryCalls++ - return groupPreviewCacheRows()(ctx, query, batch, binds, visit) + return groupPreviewCacheRows()(ctx, RowsRequest{Command: command, BatchSize: batch}, visit) }, PreparePreviewIndex: func(context.Context, compiler.PreviewCoveringIndexSpec) error { prepareCalls++ return errors.New("reused indexes must not be prepared") }, - PreviewCoveringIndexes: func(context.Context, string) ([]arango.PreviewCoveringIndex, error) { - return []arango.PreviewCoveringIndex{index}, nil + PreviewCoveringIndexes: func(context.Context, string) ([]PreviewCollectionIndex, error) { + return []PreviewCollectionIndex{index}, nil }, PreviewCollectionRevision: func(context.Context, string) (string, error) { return "revision-1", nil }, - PreviewExplainQuery: func(_ context.Context, queryText string, _ map[string]any) (arango.ExplainResult, error) { + PreviewCollectionScans: func(_ context.Context, command command.CompiledCommand, collection string) (PreviewCollectionScanEvidence, error) { + if collection != index.Collection { + return PreviewCollectionScanEvidence{}, nil + } + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return PreviewCollectionScanEvidence{}, dispatchErr + } if !strings.Contains(queryText, index.Name) { - return arango.ExplainResult{}, errors.New("candidate index hint was not rendered") + return PreviewCollectionScanEvidence{}, errors.New("candidate index hint was not rendered") } - return coveringGroupExplainWithIndex(index.Collection, index.Name), nil + return coveringGroupScans(index.Name), nil }, }) if err != nil { @@ -267,56 +355,71 @@ func TestPreviewOutputReusesStoredValuesGroupIndexWithoutPrewarmingIt(t *testing func compiledGroupPreviewScanFixture(t *testing.T, bindings recipe.RuntimeBindings) (*Engine, compiler.CompiledQuery, OutputStream) { t.Helper() engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, - ScopeDigest: func(recipe.RuntimeBindings) string { return "group-preview-scan-test" }, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, - PreviewQueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "group-preview-scan-test" }, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }, + PreviewSourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }, }) if err != nil { t.Fatal(err) } resolved := compileGroupPreviewCacheTestOutput(t, engine, bindings) - stream, query, err := engine.streamForOutput(resolved, "group_preview", 25) + stream, query, err := engine.streamForOutput(context.Background(), resolved, "group_preview", 25) if err != nil { t.Fatal(err) } return engine, query, stream } -func coveringGroupExplain(collection string) arango.ExplainResult { - return groupExplainWithCoverage(true, true, collection) +func coveringGroupScans(indexName string) PreviewCollectionScanEvidence { + return PreviewCollectionScanEvidence{IndexNodes: []PreviewIndexNodeEvidence{ + {IndexNames: []string{indexName}, Coverage: PreviewIndexCoverageCovering}, + }} } -func coveringGroupExplainWithIndex(collection, indexName string) arango.ExplainResult { - result := groupExplainWithCoverage(true, true, collection) - result.Plan.Nodes[0].Indexes = arango.ExplainIndexes{{Name: indexName, Collection: collection}} - return result +func notCoveringGroupScans() PreviewCollectionScanEvidence { + return PreviewCollectionScanEvidence{IndexNodes: []PreviewIndexNodeEvidence{ + {Coverage: PreviewIndexCoverageNotCovering}, + }} } -func groupExplainWithCoverage(indexCoversProjections, indexCoversOutProjections bool, collection ...string) arango.ExplainResult { - name := "Observation" - if len(collection) != 0 { - name = collection[0] - } - return arango.ExplainResult{Plan: &arango.ExplainPlan{Nodes: []arango.ExplainNode{{ - Type: "IndexNode", Collection: name, - IndexCoversProjections: &indexCoversProjections, IndexCoversOutProjections: &indexCoversOutProjections, - }}}} +func unknownCoverageGroupScans() PreviewCollectionScanEvidence { + return PreviewCollectionScanEvidence{IndexNodes: []PreviewIndexNodeEvidence{ + {Coverage: PreviewIndexCoverageUnknown}, + }} } func TestChooseGroupPreviewScanUsesSequentialOnlyForBroadNonCoveringIndex(t *testing.T) { - covering := false - outputCovering := false explainCalls, scopeCountCalls, collectionCountCalls := 0, 0, 0 engine, query, stream := groupPreviewScanFixture() - engine.previewExplainQuery = func(_ context.Context, queryText string, _ map[string]any) (arango.ExplainResult, error) { + engine.previewCollectionScans = func(_ context.Context, command command.CompiledCommand, collection string) (PreviewCollectionScanEvidence, error) { explainCalls++ + if collection != "Specimen" { + t.Fatalf("scan collection = %q, want Specimen", collection) + } + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return PreviewCollectionScanEvidence{}, dispatchErr + } if queryText != "normal-query" { t.Fatalf("explain query = %q, want original query", queryText) } - return arango.ExplainResult{Plan: &arango.ExplainPlan{Nodes: []arango.ExplainNode{{ - Type: "IndexNode", Collection: "Specimen", IndexCoversProjections: &covering, IndexCoversOutProjections: &outputCovering, - }}}}, nil + return notCoveringGroupScans(), nil } engine.previewCollectionCount = func(_ context.Context, collection string) (int64, error) { collectionCountCalls++ @@ -325,7 +428,14 @@ func TestChooseGroupPreviewScanUsesSequentialOnlyForBroadNonCoveringIndex(t *tes } return 1000, nil } - stream.stream = func(_ context.Context, queryText string, batchSize int, _ map[string]any, visit func(map[string]any) error) error { + stream.sourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + batchSize := request.BatchSize + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + scopeCountCalls++ if queryText != "scope-count-query" || batchSize != 1 { t.Fatalf("scope query = %q batch size = %d", queryText, batchSize) @@ -333,19 +443,21 @@ func TestChooseGroupPreviewScanUsesSequentialOnlyForBroadNonCoveringIndex(t *tes return visit(map[string]any{"count": float64(950)}) } - before, _, ok := newGroupPreviewRowsKey(query.Query, query.BindVars, "Specimen", "revision-1", 25) + before, _, ok := newGroupPreviewRowsKey(query.Command, "Specimen", "revision-1", 25) if !ok { t.Fatal("original preview cache key was not created") } if err := engine.chooseGroupPreviewScan(context.Background(), &query, &stream); err != nil { t.Fatal(err) } - after, _, ok := newGroupPreviewRowsKey(query.Query, query.BindVars, "Specimen", "revision-1", 25) + after, _, ok := newGroupPreviewRowsKey(query.Command, "Specimen", "revision-1", 25) if !ok || before == after { t.Fatal("selected AQL must produce a distinct cache key") } - if query.Query != "sequential-query" || stream.query != query.Query || query.BindVars["scan"] != "sequential" || stream.bindVars["scan"] != "sequential" { - t.Fatalf("sequential query was not applied consistently: query=%q stream=%q binds=%v streamBinds=%v", query.Query, stream.query, query.BindVars, stream.bindVars) + streamText, streamBinds := aqlView(t, stream.command) + if commandText(t, query.Command) != "sequential-query" || !reflect.DeepEqual(stream.command, query.Command) || + commandBinds(t, query.Command)["scan"] != "sequential" || streamBinds["scan"] != "sequential" { + t.Fatalf("sequential query was not applied consistently: query=%q stream=%q binds=%v streamBinds=%v", commandText(t, query.Command), streamText, commandBinds(t, query.Command), streamBinds) } if explainCalls != 1 || scopeCountCalls != 1 || collectionCountCalls != 1 { t.Fatalf("calls: explain=%d scoped=%d collection=%d", explainCalls, scopeCountCalls, collectionCountCalls) @@ -367,53 +479,48 @@ func TestChooseGroupPreviewScanFallsBackWhenNotProvenBroadAndNoncovering(t *test { name: "covering index", setup: func(engine *Engine) { - covering := true - engine.previewExplainQuery = func(context.Context, string, map[string]any) (arango.ExplainResult, error) { - outputCovering := false - return arango.ExplainResult{Plan: &arango.ExplainPlan{Nodes: []arango.ExplainNode{{Type: "IndexNode", Collection: "Specimen", IndexCoversProjections: &covering, IndexCoversOutProjections: &outputCovering}}}}, nil + engine.previewCollectionScans = func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + return unknownCoverageGroupScans(), nil } }, }, { name: "covering output projections", setup: func(engine *Engine) { - outputCovering := true - engine.previewExplainQuery = func(context.Context, string, map[string]any) (arango.ExplainResult, error) { - inputCovering := false - return arango.ExplainResult{Plan: &arango.ExplainPlan{Nodes: []arango.ExplainNode{{Type: "IndexNode", Collection: "Specimen", IndexCoversProjections: &inputCovering, IndexCoversOutProjections: &outputCovering}}}}, nil + engine.previewCollectionScans = func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + return unknownCoverageGroupScans(), nil } }, }, { name: "unknown coverage", setup: func(engine *Engine) { - engine.previewExplainQuery = func(context.Context, string, map[string]any) (arango.ExplainResult, error) { - return arango.ExplainResult{Plan: &arango.ExplainPlan{Nodes: []arango.ExplainNode{{Type: "IndexNode", Collection: "Specimen"}}}}, nil + engine.previewCollectionScans = func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + return unknownCoverageGroupScans(), nil } }, }, { name: "unknown output coverage", setup: func(engine *Engine) { - engine.previewExplainQuery = func(context.Context, string, map[string]any) (arango.ExplainResult, error) { - inputCovering := false - return arango.ExplainResult{Plan: &arango.ExplainPlan{Nodes: []arango.ExplainNode{{Type: "IndexNode", Collection: "Specimen", IndexCoversProjections: &inputCovering}}}}, nil + engine.previewCollectionScans = func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + return unknownCoverageGroupScans(), nil } }, }, { name: "unknown plan", setup: func(engine *Engine) { - engine.previewExplainQuery = func(context.Context, string, map[string]any) (arango.ExplainResult, error) { - return arango.ExplainResult{}, nil + engine.previewCollectionScans = func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + return PreviewCollectionScanEvidence{}, nil } }, }, { name: "explain error", setup: func(engine *Engine) { - engine.previewExplainQuery = func(context.Context, string, map[string]any) (arango.ExplainResult, error) { - return arango.ExplainResult{}, errors.New("unavailable") + engine.previewCollectionScans = func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + return PreviewCollectionScanEvidence{}, errors.New("unavailable") } }, }, @@ -423,6 +530,28 @@ func TestChooseGroupPreviewScanFallsBackWhenNotProvenBroadAndNoncovering(t *test engine.previewCollectionCount = func(context.Context, string) (int64, error) { return 0, errors.New("unavailable") } }, }, + { + name: "enumeration with not covering", + setup: func(engine *Engine) { + engine.previewCollectionScans = func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + return PreviewCollectionScanEvidence{ + FullCollectionEnumeration: true, + IndexNodes: []PreviewIndexNodeEvidence{{Coverage: PreviewIndexCoverageNotCovering}}, + }, nil + } + }, + }, + { + name: "multiple not covering nodes", + setup: func(engine *Engine) { + engine.previewCollectionScans = func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + return PreviewCollectionScanEvidence{IndexNodes: []PreviewIndexNodeEvidence{ + {Coverage: PreviewIndexCoverageNotCovering}, + {Coverage: PreviewIndexCoverageNotCovering}, + }}, nil + } + }, + }, { name: "empty collection", setup: func(engine *Engine) { @@ -439,9 +568,15 @@ func TestChooseGroupPreviewScanFallsBackWhenNotProvenBroadAndNoncovering(t *test for _, test := range tests { t.Run(test.name, func(t *testing.T) { engine, query, stream := groupPreviewScanFixture() - engine.previewExplainQuery = nonCoveringGroupExplain + engine.previewCollectionScans = nonCoveringGroupScansFn engine.previewCollectionCount = func(context.Context, string) (int64, error) { return 1000, nil } - stream.stream = func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + stream.sourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + value := any(float64(950)) if test.name == "selective source" { value = float64(899) @@ -457,7 +592,7 @@ func TestChooseGroupPreviewScanFallsBackWhenNotProvenBroadAndNoncovering(t *test if err := engine.chooseGroupPreviewScan(context.Background(), &query, &stream); err != nil { t.Fatal(err) } - if got := query.Query == "sequential-query"; got != test.wantSequential { + if got := commandText(t, query.Command) == "sequential-query"; got != test.wantSequential { t.Fatalf("sequential selected=%t, want %t", got, test.wantSequential) } }) @@ -470,35 +605,41 @@ func TestChooseGroupPreviewScanPropagatesRequestCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() engine.previewCollectionCount = func(context.Context, string) (int64, error) { return 1000, nil } - engine.previewExplainQuery = func(context.Context, string, map[string]any) (arango.ExplainResult, error) { + engine.previewCollectionScans = func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { cancel() - return arango.ExplainResult{}, context.Canceled + return PreviewCollectionScanEvidence{}, context.Canceled } if err := engine.chooseGroupPreviewScan(ctx, &query, &stream); !errors.Is(err, context.Canceled) { t.Fatalf("error = %v, want canceled request", err) } - if query.Query != "normal-query" { - t.Fatalf("canceled preview switched to %q", query.Query) + if commandText(t, query.Command) != "normal-query" { + t.Fatalf("canceled preview switched to %q", commandText(t, query.Command)) } }) t.Run("during scoped count", func(t *testing.T) { engine, query, stream := groupPreviewScanFixture() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - engine.previewExplainQuery = nonCoveringGroupExplain + engine.previewCollectionScans = nonCoveringGroupScansFn engine.previewCollectionCount = func(context.Context, string) (int64, error) { t.Fatal("collection count called after scoped-count cancellation") return 0, nil } - stream.stream = func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + stream.sourceRows = func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + cancel() return nil } if err := engine.chooseGroupPreviewScan(ctx, &query, &stream); !errors.Is(err, context.Canceled) { t.Fatalf("error = %v, want canceled request", err) } - if query.Query != "normal-query" { - t.Fatalf("canceled preview switched to %q", query.Query) + if commandText(t, query.Command) != "normal-query" { + t.Fatalf("canceled preview switched to %q", commandText(t, query.Command)) } }) } @@ -507,46 +648,55 @@ func TestChooseGroupPreviewScanKeepsNoneligiblePlansOnCanonicalQuery(t *testing. engine, query, stream := groupPreviewScanFixture() query.PreviewGroupScan = nil called := false - engine.previewExplainQuery = func(context.Context, string, map[string]any) (arango.ExplainResult, error) { + engine.previewCollectionScans = func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { called = true - return arango.ExplainResult{}, nil + return PreviewCollectionScanEvidence{}, nil } if err := engine.chooseGroupPreviewScan(context.Background(), &query, &stream); err != nil { t.Fatal(err) } - if called || query.Query != "normal-query" { - t.Fatalf("noneligible query performed assessment or changed query: called=%t query=%q", called, query.Query) + if called || commandText(t, query.Command) != "normal-query" { + t.Fatalf("noneligible query performed assessment or changed query: called=%t query=%q", called, commandText(t, query.Command)) } } func groupPreviewScanFixture() (*Engine, compiler.CompiledQuery, OutputStream) { - return &Engine{}, compiler.CompiledQuery{ - Query: "normal-query", BindVars: map[string]any{"scan": "normal"}, + return &Engine{compileTarget: compiler.ArangoTarget()}, compiler.CompiledQuery{ + Command: mustAQLCommand("normal-query", map[string]any{"scan": "normal"}), PreviewGroupScan: &compiler.PreviewGroupScanSpec{ - Collection: "Specimen", SequentialQuery: "sequential-query", - SequentialBindVars: map[string]any{"scan": "sequential"}, - ScopeCountQuery: "scope-count-query", ScopeCountBindVars: map[string]any{"scope": true}, + Collection: "Specimen", Sequential: mustAQLCommand("sequential-query", map[string]any{"scan": "sequential"}), + ScopeCount: mustAQLCommand("scope-count-query", map[string]any{"scope": true}), }, - }, OutputStream{query: "normal-query", bindVars: map[string]any{"scan": "normal"}, stream: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + }, OutputStream{command: mustAQLCommand("normal-query", map[string]any{"scan": "normal"}), sourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil }} } -func nonCoveringGroupExplain(context.Context, string, map[string]any) (arango.ExplainResult, error) { - covering := false - outputCovering := false - return arango.ExplainResult{Plan: &arango.ExplainPlan{Nodes: []arango.ExplainNode{{ - Type: "IndexNode", Collection: "Specimen", IndexCoversProjections: &covering, IndexCoversOutProjections: &outputCovering, - }}}}, nil +func nonCoveringGroupScansFn(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + return notCoveringGroupScans(), nil } -func semanticGroupProjectionExecutionFixture(t *testing.T, previewRows QueryRows) (*Engine, Resolved, compiler.CompiledQuery, OutputStream) { +func semanticGroupProjectionExecutionFixture(t *testing.T, previewRows SourceRows) (*Engine, Resolved, compiler.CompiledQuery, OutputStream) { t.Helper() engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, - ScopeDigest: func(recipe.RuntimeBindings) string { return "group-projection-runtime-test" }, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, - PreviewQueryRows: previewRows, + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "group-projection-runtime-test" }, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return nil + }, + PreviewSourceRows: previewRows, }) if err != nil { t.Fatal(err) @@ -561,12 +711,12 @@ func semanticGroupProjectionExecutionFixture(t *testing.T, previewRows QueryRows if err != nil { t.Fatal(err) } - stream, query, err := engine.streamForOutput(resolved, "semantic_group_projection", 600) + stream, query, err := engine.streamForOutput(context.Background(), resolved, "semantic_group_projection", 600) if err != nil { t.Fatal(err) } if query.PreviewGroupProjection == nil { - _, _, eligibilityErr := compiler.RenderPreviewGroupProjectionAssessment(query.PhysicalPlan, "", false) + _, eligibilityErr := compiler.RenderPreviewGroupProjectionAssessment(query.PhysicalPlan, "", false) stageCount, ordinaryGroupKeys := 0, -1 sequenceWindow := false if sequence := query.PhysicalPlan.StageSequence; sequence != nil { @@ -615,7 +765,7 @@ func configureSemanticGroupProjectionIndex( t *testing.T, engine *Engine, query compiler.CompiledQuery, -) (arango.PreviewCoveringIndex, string, map[string]any, string, map[string]any) { +) (PreviewCollectionIndex, command.CompiledCommand, command.CompiledCommand) { t.Helper() if query.PreviewGroupProjection == nil || query.PreviewGroupProjection.ReadOnlyIndex == nil { t.Fatalf("semantic Group has no read-only projection index descriptor: %+v", query.PreviewGroupProjection) @@ -623,61 +773,70 @@ func configureSemanticGroupProjectionIndex( descriptor := query.PreviewGroupProjection.ReadOnlyIndex indexFields := append([]string(nil), descriptor.RequiredFields...) indexFields = append(indexFields, descriptor.ProjectionPath) - index := arango.PreviewCoveringIndex{ + index := PreviewCollectionIndex{ Collection: query.PreviewGroupProjection.Collection, Name: "loom_pivot_preview_runtime_existing", Fields: indexFields, } - assessmentQuery, assessmentBinds, err := compiler.RenderPreviewGroupProjectionAssessment(query.PhysicalPlan, index.Name, false) + assessmentCommand, err := compiler.RenderPreviewGroupProjectionAssessment(query.PhysicalPlan, index.Name, false) if err != nil { t.Fatal(err) } - factoredQuery, factoredBinds, err := compiler.RenderPreviewGroupProjectionFactorization(query.PhysicalPlan, index.Name, false) + factoredCommand, err := compiler.RenderPreviewGroupProjectionFactorization(query.PhysicalPlan, index.Name, false) if err != nil { t.Fatal(err) } - engine.previewCoveringIndexes = func(_ context.Context, collection string) ([]arango.PreviewCoveringIndex, error) { + engine.previewCoveringIndexes = func(_ context.Context, collection string) ([]PreviewCollectionIndex, error) { if collection != index.Collection { t.Fatalf("index collection=%q, want %q", collection, index.Collection) } - return []arango.PreviewCoveringIndex{index}, nil + return []PreviewCollectionIndex{index}, nil } - engine.previewExplainQuery = func(_ context.Context, queryText string, bindVars map[string]any) (arango.ExplainResult, error) { - if queryText != factoredQuery || !reflect.DeepEqual(bindVars, factoredBinds) { - return arango.ExplainResult{}, fmt.Errorf("candidate explain did not use exact factored query and binds") + engine.previewCollectionScans = func(_ context.Context, command command.CompiledCommand, collection string) (PreviewCollectionScanEvidence, error) { + if collection != index.Collection { + return PreviewCollectionScanEvidence{}, nil + } + factoredText, factoredBinds, dispatchErr := factoredCommand.AQLParameters() + queryText, bindVars, commandDispatchErr := command.AQLParameters() + if commandDispatchErr != nil { + return PreviewCollectionScanEvidence{}, commandDispatchErr } - return arango.ExplainResult{Plan: &arango.ExplainPlan{Nodes: []arango.ExplainNode{{ - Type: "IndexNode", Collection: index.Collection, - Indexes: arango.ExplainIndexes{{Name: index.Name, Collection: index.Collection}}, - IndexCoversProjections: boolPointer(true), - IndexCoversOutProjections: boolPointer(true), - }}}}, nil + if dispatchErr != nil || queryText != factoredText || !reflect.DeepEqual(bindVars, factoredBinds) { + return PreviewCollectionScanEvidence{}, fmt.Errorf("candidate explain did not use exact factored query and binds") + } + return coveringGroupScans(index.Name), nil } - return index, assessmentQuery, assessmentBinds, factoredQuery, factoredBinds + return index, assessmentCommand, factoredCommand } func TestChooseGroupProjectionFactorizationAdmitsLowDiversityAndCachesDecision(t *testing.T) { engine, resolved, query, _ := semanticGroupProjectionExecutionFixture(t, nil) - index, assessmentQuery, assessmentBinds, factoredQuery, factoredBinds := configureSemanticGroupProjectionIndex(t, engine, query) + index, assessmentCommand, factoredCommand := configureSemanticGroupProjectionIndex(t, engine, query) engine.previewCollectionRevision = func(context.Context, string) (string, error) { return "revision-1", nil } - decisionKey, _, keyOK := newGroupPreviewProjectionDecisionKey(query.Query, query.BindVars, + decisionKey, _, keyOK := newGroupPreviewProjectionDecisionKey(query.Command, query.PreviewGroupProjection.Collection, "revision-1", groupProjectionIndexIdentity(index)) if !keyOK { t.Fatal("could not construct exact projection decision key") } countCalls, sampleCalls, finalCalls := 0, 0, 0 - engine.previewQueryRows = func(_ context.Context, queryText string, _ int, binds map[string]any, visit func(map[string]any) error) error { + engine.previewSourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + queryText, binds, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + switch queryText { - case query.PreviewGroupProjection.ScopeCountQuery: + case commandText(t, query.PreviewGroupProjection.ScopeCount): countCalls++ return visit(map[string]any{"count": int64(10000)}) - case assessmentQuery: + case commandText(t, assessmentCommand): sampleCalls++ - if !reflect.DeepEqual(binds, assessmentBinds) { - t.Fatalf("sample binds=%v, want %v", binds, assessmentBinds) + if !reflect.DeepEqual(binds, commandBinds(t, assessmentCommand)) { + t.Fatalf("sample binds=%v, want %v", binds, commandBinds(t, assessmentCommand)) } return visit(map[string]any{"sampleRows": int64(512), "distinctProjections": int64(2), "serializedCharacters": int64(40)}) - case factoredQuery: + case commandText(t, factoredCommand): if finalCalls == 0 { if _, found := engine.groupPreviewRowsCache.getDecision(decisionKey); found { t.Fatal("positive strategy was cached before the final query completed") @@ -707,21 +866,27 @@ func TestChooseGroupProjectionFactorizationAdmitsLowDiversityAndCachesDecision(t } } } - if factoredBinds == nil || index.Name == "" { - t.Fatalf("missing selected projection index or rendered binds: index=%+v binds=%v", index, factoredBinds) + if factoredCommand.Backend() != command.CompiledBackendAQL || index.Name == "" { + t.Fatalf("missing selected projection index or rendered binds: index=%+v", index) } } func TestChooseGroupProjectionFactorizationContinuesAfterCoveringIndexSelection(t *testing.T) { engine, _, query, stream := semanticGroupProjectionExecutionFixture(t, nil) - index, assessmentQuery, _, factoredQuery, _ := configureSemanticGroupProjectionIndex(t, engine, query) + index, assessmentCommand, factoredCommand := configureSemanticGroupProjectionIndex(t, engine, query) engine.previewCollectionRevision = func(context.Context, string) (string, error) { return "revision-covered", nil } sampleCalls := 0 - stream.stream = func(_ context.Context, queryText string, _ int, _ map[string]any, visit func(map[string]any) error) error { + stream.sourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + switch queryText { - case query.PreviewGroupProjection.ScopeCountQuery: + case commandText(t, query.PreviewGroupProjection.ScopeCount): return visit(map[string]any{"count": int64(1000)}) - case assessmentQuery: + case commandText(t, assessmentCommand): sampleCalls++ return visit(map[string]any{"sampleRows": int64(512), "distinctProjections": int64(2), "serializedCharacters": int64(40)}) default: @@ -732,8 +897,8 @@ func TestChooseGroupProjectionFactorizationContinuesAfterCoveringIndexSelection( if err := engine.chooseGroupPreviewScan(context.Background(), &query, &stream); err != nil { t.Fatal(err) } - if query.PreviewCoveringIndex != nil || query.Query != factoredQuery || sampleCalls != 1 || index.Name == "" { - t.Fatalf("read-only covering-index path index=%+v factored=%t sample calls=%d selected=%+v", query.PreviewCoveringIndex, query.Query == factoredQuery, sampleCalls, index) + if query.PreviewCoveringIndex != nil || !reflect.DeepEqual(query.Command, factoredCommand) || sampleCalls != 1 || index.Name == "" { + t.Fatalf("read-only covering-index path index=%+v factored=%t sample calls=%d selected=%+v", query.PreviewCoveringIndex, reflect.DeepEqual(query.Command, factoredCommand), sampleCalls, index) } } @@ -747,14 +912,20 @@ func TestChooseGroupProjectionFactorizationRejectsHighEntropyAndAllEmptySamples( } { t.Run(test.name, func(t *testing.T) { engine, _, query, stream := semanticGroupProjectionExecutionFixture(t, nil) - _, assessmentQuery, _, _, _ := configureSemanticGroupProjectionIndex(t, engine, query) + _, assessmentCommand, _ := configureSemanticGroupProjectionIndex(t, engine, query) engine.previewCollectionRevision = func(context.Context, string) (string, error) { return "revision-1", nil } - ordinaryQuery := query.Query - stream.stream = func(_ context.Context, queryText string, _ int, _ map[string]any, visit func(map[string]any) error) error { + ordinaryQuery := commandText(t, query.Command) + stream.sourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + switch queryText { - case query.PreviewGroupProjection.ScopeCountQuery: + case commandText(t, query.PreviewGroupProjection.ScopeCount): return visit(map[string]any{"count": int64(1000)}) - case assessmentQuery: + case commandText(t, assessmentCommand): return visit(test.row) default: t.Fatalf("unexpected query %q", queryText) @@ -764,8 +935,8 @@ func TestChooseGroupProjectionFactorizationRejectsHighEntropyAndAllEmptySamples( if err := engine.chooseGroupPreviewScan(context.Background(), &query, &stream); err != nil { t.Fatal(err) } - if query.Query != ordinaryQuery || stream.queryMemoryLimit != 0 { - t.Fatalf("ineligible sample changed plan or enabled cap: query changed=%t cap=%d", query.Query != ordinaryQuery, stream.queryMemoryLimit) + if commandText(t, query.Command) != ordinaryQuery || stream.queryMemoryLimit != 0 { + t.Fatalf("ineligible sample changed plan or enabled cap: query changed=%t cap=%d", commandText(t, query.Command) != ordinaryQuery, stream.queryMemoryLimit) } }) } @@ -773,16 +944,19 @@ func TestChooseGroupProjectionFactorizationRejectsHighEntropyAndAllEmptySamples( func TestChooseGroupProjectionFactorizationKeepsPrefixEstimateAdvisory(t *testing.T) { engine, resolved, query, stream := semanticGroupProjectionExecutionFixture(t, nil) - _, assessmentQuery, _, factoredQuery, _ := configureSemanticGroupProjectionIndex(t, engine, query) + _, assessmentCommand, factoredCommand := configureSemanticGroupProjectionIndex(t, engine, query) engine.previewCollectionRevision = func(context.Context, string) (string, error) { return "revision-unique-tail", nil } - stream.stream = func(_ context.Context, queryText string, _ int, _ map[string]any, visit func(map[string]any) error) error { + stream.sourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + switch queryText { - case query.PreviewGroupProjection.ScopeCountQuery: + case commandText(t, query.PreviewGroupProjection.ScopeCount): return visit(map[string]any{"count": int64(1024)}) - case assessmentQuery: - // The first 512 rows are low diversity. A later tail can contain a - // unique Coding projection for every row; the sample is cost advice, - // never the source of the factored query's output values. + case commandText(t, assessmentCommand): return visit(map[string]any{"sampleRows": int64(512), "distinctProjections": int64(2), "serializedCharacters": int64(40)}) default: t.Fatalf("unexpected query %q", queryText) @@ -792,20 +966,26 @@ func TestChooseGroupProjectionFactorizationKeepsPrefixEstimateAdvisory(t *testin if err := engine.chooseGroupPreviewScan(context.Background(), &query, &stream); err != nil { t.Fatal(err) } - if query.Query != factoredQuery { - t.Fatalf("sample metadata was used as output; query is not the complete factored plan:\n%s", query.Query) + if !reflect.DeepEqual(query.Command, factoredCommand) { + t.Fatalf("sample metadata was used as output; query is not the complete factored plan:\n%s", commandText(t, query.Command)) } if estimate, ok := estimateGroupProjectionDictionaryBytes(groupProjectionSample{sampleRows: 512, distinctProjections: 2, serializedCharacters: 40}, 1024); !ok || estimate == 0 { t.Fatalf("advisory estimate=%d valid=%t", estimate, ok) } engine.previewCollectionRevision = func(context.Context, string) (string, error) { return "revision-unique-tail", nil } - engine.previewQueryRows = func(_ context.Context, queryText string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine.previewSourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + switch queryText { - case query.PreviewGroupProjection.ScopeCountQuery: + case commandText(t, query.PreviewGroupProjection.ScopeCount): return visit(map[string]any{"count": int64(1024)}) - case assessmentQuery: + case commandText(t, assessmentCommand): return visit(map[string]any{"sampleRows": int64(512), "distinctProjections": int64(2), "serializedCharacters": int64(40)}) - case factoredQuery: + case commandText(t, factoredCommand): for index := range 600 { if err := visit(map[string]any{"category": fmt.Sprintf("tail-%03d", index), "rows": int64(1)}); err != nil { return err @@ -839,28 +1019,40 @@ func TestEstimateGroupProjectionDictionaryBytesRejectsOverflowingPopulationEstim func TestChooseGroupProjectionFactorizationFallsBackOnUnknownEvidenceAndDeadline(t *testing.T) { t.Run("unknown revision", func(t *testing.T) { engine, _, query, stream := semanticGroupProjectionExecutionFixture(t, nil) - _, _, _, _, _ = configureSemanticGroupProjectionIndex(t, engine, query) + _, _, _ = configureSemanticGroupProjectionIndex(t, engine, query) engine.previewCollectionRevision = func(context.Context, string) (string, error) { return "", errors.New("revision unavailable") } - ordinaryQuery := query.Query - stream.stream = func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + ordinaryQuery := commandText(t, query.Command) + stream.sourceRows = func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + t.Fatal("sample ran without source revision evidence") return nil } if err := engine.chooseGroupPreviewScan(context.Background(), &query, &stream); err != nil { t.Fatal(err) } - if query.Query != ordinaryQuery { + if commandText(t, query.Command) != ordinaryQuery { t.Fatal("unknown source revision changed the ordinary plan") } }) t.Run("unknown scoped count", func(t *testing.T) { engine, _, query, stream := semanticGroupProjectionExecutionFixture(t, nil) - _, _, _, _, _ = configureSemanticGroupProjectionIndex(t, engine, query) + _, _, _ = configureSemanticGroupProjectionIndex(t, engine, query) engine.previewCollectionRevision = func(context.Context, string) (string, error) { return "revision-1", nil } - ordinaryQuery := query.Query - stream.stream = func(_ context.Context, queryText string, _ int, _ map[string]any, _ func(map[string]any) error) error { - if queryText != query.PreviewGroupProjection.ScopeCountQuery { + ordinaryQuery := commandText(t, query.Command) + stream.sourceRows = func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + + if !reflect.DeepEqual(command, query.PreviewGroupProjection.ScopeCount) { t.Fatalf("query=%q, want exact scope count", queryText) } return nil @@ -868,7 +1060,7 @@ func TestChooseGroupProjectionFactorizationFallsBackOnUnknownEvidenceAndDeadline if err := engine.chooseGroupPreviewScan(context.Background(), &query, &stream); err != nil { t.Fatal(err) } - if query.Query != ordinaryQuery { + if commandText(t, query.Command) != ordinaryQuery { t.Fatal("missing scoped count evidence changed the ordinary plan") } }) @@ -876,40 +1068,51 @@ func TestChooseGroupProjectionFactorizationFallsBackOnUnknownEvidenceAndDeadline t.Run("unknown scan plan", func(t *testing.T) { engine, _, query, stream := semanticGroupProjectionExecutionFixture(t, nil) query.PreviewGroupScan = &compiler.PreviewGroupScanSpec{ - Collection: query.PreviewGroupProjection.Collection, - ScopeCountQuery: query.PreviewGroupProjection.ScopeCountQuery, - ScopeCountBindVars: query.PreviewGroupProjection.ScopeCountBindVars, + Collection: query.PreviewGroupProjection.Collection, + ScopeCount: query.PreviewGroupProjection.ScopeCount, } - engine.previewExplainQuery = func(context.Context, string, map[string]any) (arango.ExplainResult, error) { - return arango.ExplainResult{}, nil + engine.previewCollectionScans = func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + return PreviewCollectionScanEvidence{}, nil } engine.previewCollectionRevision = func(context.Context, string) (string, error) { t.Fatal("source revision read without a resolved scan plan") return "", nil } - ordinaryQuery := query.Query - stream.stream = func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + ordinaryQuery := commandText(t, query.Command) + stream.sourceRows = func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + t.Fatal("scope count or sample ran without a resolved scan plan") return nil } if err := engine.chooseGroupPreviewScan(context.Background(), &query, &stream); err != nil { t.Fatal(err) } - if query.Query != ordinaryQuery { + if commandText(t, query.Command) != ordinaryQuery { t.Fatal("unknown scan plan changed the ordinary query") } }) t.Run("assessment deadline", func(t *testing.T) { engine, _, query, stream := semanticGroupProjectionExecutionFixture(t, nil) - _, assessmentQuery, _, _, _ := configureSemanticGroupProjectionIndex(t, engine, query) + _, assessmentCommand, _ := configureSemanticGroupProjectionIndex(t, engine, query) engine.previewCollectionRevision = func(context.Context, string) (string, error) { return "revision-deadline", nil } - ordinaryQuery := query.Query - stream.stream = func(ctx context.Context, queryText string, _ int, _ map[string]any, visit func(map[string]any) error) error { - if queryText == query.PreviewGroupProjection.ScopeCountQuery { + ordinaryQuery := commandText(t, query.Command) + stream.sourceRows = func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + + if reflect.DeepEqual(command, query.PreviewGroupProjection.ScopeCount) { return visit(map[string]any{"count": int64(1000)}) } - if queryText != assessmentQuery { + if queryText != commandText(t, assessmentCommand) { t.Fatalf("unexpected query %q", queryText) } <-ctx.Done() @@ -918,7 +1121,7 @@ func TestChooseGroupProjectionFactorizationFallsBackOnUnknownEvidenceAndDeadline if err := engine.chooseGroupPreviewScan(context.Background(), &query, &stream); err != nil { t.Fatal(err) } - if query.Query != ordinaryQuery { + if commandText(t, query.Command) != ordinaryQuery { t.Fatal("assessment timeout changed the ordinary plan") } }) @@ -926,22 +1129,29 @@ func TestChooseGroupProjectionFactorizationFallsBackOnUnknownEvidenceAndDeadline func TestPreviewOutputDoesNotRetryOrdinaryQueryAfterFactoredQueryFailure(t *testing.T) { engine, resolved, query, _ := semanticGroupProjectionExecutionFixture(t, nil) - _, assessmentQuery, _, factoredQuery, _ := configureSemanticGroupProjectionIndex(t, engine, query) + _, assessmentCommand, factoredCommand := configureSemanticGroupProjectionIndex(t, engine, query) engine.previewCollectionRevision = func(context.Context, string) (string, error) { return "revision-error", nil } wantErr := shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, ErrorMessage: "AQL: query would use more memory than allowed"} countCalls, sampleCalls, factoredCalls, ordinaryCalls := 0, 0, 0, 0 - engine.previewQueryRows = func(_ context.Context, queryText string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine.previewSourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + queryText, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + switch queryText { - case query.PreviewGroupProjection.ScopeCountQuery: + case commandText(t, query.PreviewGroupProjection.ScopeCount): countCalls++ return visit(map[string]any{"count": int64(1000)}) - case assessmentQuery: + case commandText(t, assessmentCommand): sampleCalls++ return visit(map[string]any{"sampleRows": int64(512), "distinctProjections": int64(2), "serializedCharacters": int64(40)}) - case factoredQuery: + case commandText(t, factoredCommand): factoredCalls++ - return fmt.Errorf("arango query: %w", wantErr) - case query.Query: + translated, _ := arangofactory.TranslateQueryError(fmt.Errorf("arango query: %w", wantErr)) + return translated + case commandText(t, query.Command): ordinaryCalls++ return visit(map[string]any{"category": "ordinary", "rows": int64(1)}) default: @@ -949,8 +1159,12 @@ func TestPreviewOutputDoesNotRetryOrdinaryQueryAfterFactoredQueryFailure(t *test } } _, err := engine.PreviewOutput(context.Background(), resolved, PreviewRequest{Output: "semantic_group_projection", Limit: 600}, func(map[string]any) error { return nil }) - if !arango.IsQueryResourceLimitExceeded(err) { - t.Fatalf("preview error=%v, want original factored-query error", err) + userErr, isUser := dataframeerrors.AsUserError(err) + if !isUser || userErr.Code() != string(dataframeerrors.CodeQueryMemoryLimitExceeded) { + t.Fatalf("preview error=%#v, want classified QUERY_MEMORY_LIMIT_EXCEEDED factored-query error", err) + } + if !errors.Is(err, wantErr) { + t.Fatalf("driver cause lost through the classified preview failure: %v", err) } if countCalls != 1 || sampleCalls != 1 || factoredCalls != 1 || ordinaryCalls != 0 { t.Fatalf("first preview count/sample/factored/ordinary=%d/%d/%d/%d, want 1/1/1/0", countCalls, sampleCalls, factoredCalls, ordinaryCalls) diff --git a/internal/dataframe/execution/identity_test.go b/internal/dataframe/execution/identity_test.go index 3819d2056..09e51f036 100644 --- a/internal/dataframe/execution/identity_test.go +++ b/internal/dataframe/execution/identity_test.go @@ -45,8 +45,14 @@ func TestOutputStreamStripsCompilerOnlyColumnsAfterIdentity(t *testing.T) { Name: "DocumentReference", Columns: []string{"id"}, RowIdentity: &spec.RowIdentity{Fields: []string{"_key"}}, - bindVars: map[string]any{"project": "P1"}, - stream: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + command: mustAQLCommand("FOR root IN DocumentReference RETURN root", map[string]any{"project": "P1"}), + sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{ "_key": "internal-key", "__loom_population_members": []any{"file-a", "file-b"}, diff --git a/internal/dataframe/execution/named_uuid.go b/internal/dataframe/execution/named_uuid.go index f033b9dbb..0cdba3489 100644 --- a/internal/dataframe/execution/named_uuid.go +++ b/internal/dataframe/execution/named_uuid.go @@ -1,5 +1,3 @@ -// Package execution contains the exact named-UUID contract shared by -// compiler-owned post-query execution and legacy differential tests. package execution import ( @@ -9,8 +7,6 @@ import ( "github.com/google/uuid" ) -// computeNamedUUID applies the legacy uuid3/uuid5 contract. A nil argument propagates -// null so optional recipe expressions do not mint an ID from the text "". func computeNamedUUID(operation string, args []any) (any, error) { if len(args) < 2 { return nil, fmt.Errorf("%s requires a namespace and at least one name", operation) diff --git a/internal/dataframe/execution/named_uuid_test.go b/internal/dataframe/execution/named_uuid_test.go index d1d4eead3..c76c0108f 100644 --- a/internal/dataframe/execution/named_uuid_test.go +++ b/internal/dataframe/execution/named_uuid_test.go @@ -1,6 +1,7 @@ package execution import ( + "encoding/json" "testing" "github.com/google/uuid" @@ -28,3 +29,82 @@ func TestComputeNamedUUID(t *testing.T) { t.Fatal("expected unsupported operation error") } } + +const ( + exec03AcceptedUUID5Literal = "21f7f8de-8051-5b89-8680-0195ef798b6a" + exec03AcceptedUUID3Literal = "3d813cbb-47fb-32ba-91df-831e1593ac29" + exec03DNSNamespaceString = "6ba7b810-9dad-11d1-80b4-00c04fd430c8" +) + +func TestComputeNamedUUIDPinsAcceptedPublicValues(t *testing.T) { + got, err := computeNamedUUID("uuid5", []any{exec03DNSNamespaceString, "www.widgets.com"}) + if err != nil || got != exec03AcceptedUUID5Literal { + t.Fatalf("uuid5 = %#v (err %v), want %s", got, err, exec03AcceptedUUID5Literal) + } + got, err = computeNamedUUID("uuid3", []any{exec03DNSNamespaceString, "www.widgets.com"}) + if err != nil || got != exec03AcceptedUUID3Literal { + t.Fatalf("uuid3 = %#v (err %v), want %s", got, err, exec03AcceptedUUID3Literal) + } +} + +func TestComputeNamedUUIDTextualNamespaceNormalization(t *testing.T) { + parsed, err := computeNamedUUID("uuid5", []any{exec03DNSNamespaceString, "www.widgets.com"}) + if err != nil || parsed != exec03AcceptedUUID5Literal { + t.Fatalf("parsed namespace = %#v (err %v), want %s", parsed, err, exec03AcceptedUUID5Literal) + } + upper, err := computeNamedUUID("uuid5", []any{"6BA7B810-9DAD-11D1-80B4-00C04FD430C8", "www.widgets.com"}) + if err != nil || upper != exec03AcceptedUUID5Literal { + t.Fatalf("uppercase namespace = %#v (err %v), want %s", upper, err, exec03AcceptedUUID5Literal) + } + derived, err := computeNamedUUID("uuid5", []any{"www.widgets.com", "www.widgets.com"}) + if err != nil || derived != "fcdbc73e-6d1c-5393-a6d5-79f7dbbe9862" { + t.Fatalf("textual namespace = %#v (err %v), want fcdbc73e-6d1c-5393-a6d5-79f7dbbe9862", derived, err) + } + other, err := computeNamedUUID("uuid5", []any{"other", "www.widgets.com"}) + if err != nil || other == exec03AcceptedUUID5Literal { + t.Fatalf("textual namespace %q collided with the DNS-derived witness: %#v (err %v)", "other", other, err) + } +} + +func TestComputeNamedUUIDMultipleNameConcatenationIsOrdered(t *testing.T) { + joined, err := computeNamedUUID("uuid5", []any{exec03DNSNamespaceString, "www.", "widgets", ".com"}) + if err != nil || joined != exec03AcceptedUUID5Literal { + t.Fatalf("multi-name concat = %#v (err %v), want %s", joined, err, exec03AcceptedUUID5Literal) + } + ab, err := computeNamedUUID("uuid5", []any{"www.widgets.com", "a", "b"}) + if err != nil || ab != "46d2b645-361a-5bb3-9f79-6aefcc93bf51" { + t.Fatalf("ab = %#v (err %v)", ab, err) + } + ba, err := computeNamedUUID("uuid5", []any{"www.widgets.com", "b", "a"}) + if err != nil || ba != "45ebaedb-38ed-57b9-bb5c-8bb96e736cd8" { + t.Fatalf("ba = %#v (err %v)", ba, err) + } + if ab == ba { + t.Fatal("argument order did not affect the named UUID") + } + mixed, err := computeNamedUUID("uuid5", []any{"www.widgets.com", "count=", 1}) + if err != nil { + t.Fatalf("mixed-name: %v", err) + } + if mixed != "8712a807-232e-566a-887b-cdca14b3a40e" { + t.Fatalf("mixed-name = %#v, want 8712a807-232e-566a-887b-cdca14b3a40e", mixed) + } + textual, err := computeNamedUUID("uuid5", []any{"www.widgets.com", "count=1"}) + if err != nil || mixed != textual { + t.Fatalf("mixed %#v != textual %#v (err %v)", mixed, textual, err) + } +} + +func TestComputeNamedUUIDNilPropagatesThroughPostgreSQLShape(t *testing.T) { + for _, args := range [][]any{{exec03DNSNamespaceString, nil}, {nil, "www.widgets.com"}} { + got, err := computeNamedUUID("uuid5", args) + if err != nil || got != nil { + t.Fatalf("nil propagation for %#v: got %#v (err %v)", args, got, err) + } + } + marker := `{"__loom_exact_uuid_operation":"uuid5","__loom_exact_uuid_args":["6ba7b810-9dad-11d1-80b4-00c04fd430c8",null]}` + got, err := materializePostQueryValue(json.RawMessage(marker)) + if err != nil || got != nil { + t.Fatalf("JSONB null argument materialized to %#v (err %v), want untyped nil", got, err) + } +} diff --git a/internal/dataframe/execution/population_group_preview_test.go b/internal/dataframe/execution/population_group_preview_test.go index ef4199988..d43fb3e55 100644 --- a/internal/dataframe/execution/population_group_preview_test.go +++ b/internal/dataframe/execution/population_group_preview_test.go @@ -2,6 +2,7 @@ package execution import ( "context" + "github.com/calypr/loom/internal/dataframe/compiler" "strings" "testing" @@ -52,8 +53,15 @@ func TestPreviewOutputMaterializesPopulationSourceForKeylessGroupContributors(t var executedQuery string var executedBindVars map[string]any engine, err := New(Config{ - Registry: invalidRecipeRegistry{}, - QueryRows: func(_ context.Context, query string, _ int, bindVars map[string]any, visit func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + SourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + query, bindVars, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + executedQuery = query executedBindVars = bindVars return visit(map[string]any{"row_count": int64(1)}) diff --git a/internal/dataframe/execution/population_mapping.go b/internal/dataframe/execution/population_mapping.go index b5f945471..3eb8aeae3 100644 --- a/internal/dataframe/execution/population_mapping.go +++ b/internal/dataframe/execution/population_mapping.go @@ -14,11 +14,11 @@ import ( "strings" "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" + postgresrender "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" ) -// PopulationMappingStatus is deliberately closed. Exact counts are only -// meaningful for COMPLETE results. type PopulationMappingStatus string const ( @@ -26,24 +26,16 @@ const ( PopulationMappingIncomplete PopulationMappingStatus = "INCOMPLETE" ) -// PopulationMemberReader is the smallest reader needed by the mapping -// operation. Lifecycle owns paging and authorization; execution only needs -// the immutable member IDs to compare with witness rows. type PopulationMemberReader interface { VisitMembers(context.Context, func(string) error) error } -// PopulationMemberReaderFunc adapts a lifecycle-owned member stream without -// exposing its persistence details to execution. type PopulationMemberReaderFunc func(context.Context, func(string) error) error func (f PopulationMemberReaderFunc) VisitMembers(ctx context.Context, visit func(string) error) error { return f(ctx, visit) } -// PopulationMappingRequest bounds one request-scoped mapping computation. -// AfterMemberID is an opaque lifecycle cursor position; it is never sent to -// the compiler or backend. type PopulationMappingRequest struct { Output string AfterMemberID string @@ -52,15 +44,11 @@ type PopulationMappingRequest struct { MaxWitnessRows int64 } -// PopulationMappingDiagnostic contains only safe, transport-neutral -// diagnostics. It deliberately excludes query text and storage details. type PopulationMappingDiagnostic struct { Code string Message string } -// PopulationMappingResult is the concrete execution result. Lifecycle turns -// its bounded IDs into authorized ResourceRefs and adds the receipt binding. type PopulationMappingResult struct { Status PopulationMappingStatus SelectedCount int64 @@ -74,6 +62,8 @@ type PopulationMappingResult struct { var errPopulationMappingLimit = errors.New("population mapping resource limit exceeded") +var ErrPopulationMappingPostgresUnsupported = errors.New("population mapping unsupported on the selected PostgreSQL composition") + const ( defaultPopulationMappingUnmapped = 100 maxPopulationMappingUnmapped = 1000 @@ -81,31 +71,37 @@ const ( defaultPopulationMappingWitness = 1_000_000 ) -// PopulationMapping compiles the dedicated witness terminal from the same -// resolved output used by ordinary dataframe execution, then streams its -// rows through the existing query executor. It never exposes compiler fields -// or adds provenance to ordinary dataframe rows. func (e *Engine) PopulationMapping(ctx context.Context, resolved Resolved, request PopulationMappingRequest, reader PopulationMemberReader) (PopulationMappingResult, error) { if strings.TrimSpace(request.Output) == "" { return PopulationMappingResult{}, fmt.Errorf("population mapping output is required") } - compiled, err := compilePopulationMappingOutput(resolved, request.Output) + compiled, err := compilePopulationMappingOutput(e.compileTarget, resolved, request.Output) if err != nil { + if errors.Is(err, postgresrender.ErrUnsupportedPostgresPlan) { + return PopulationMappingResult{}, fmt.Errorf("%w: %v", ErrPopulationMappingPostgresUnsupported, err) + } return PopulationMappingResult{}, err } return e.populationMappingCompiled(ctx, compiled, request, reader) } -// PopulationMappingCompiled is the execution seam for callers that already -// hold the dedicated compiler result. The resolved-output method above is the -// normal lifecycle entry point; this method keeps tests and specialized -// execution adapters from rebuilding compiler state. func (e *Engine) PopulationMappingCompiled(ctx context.Context, compiled compiler.CompiledPopulationMappingQuery, request PopulationMappingRequest, reader PopulationMemberReader) (PopulationMappingResult, error) { return e.populationMappingCompiled(ctx, compiled, request, reader) } func (e *Engine) populationMappingCompiled(ctx context.Context, compiled compiler.CompiledPopulationMappingQuery, request PopulationMappingRequest, reader PopulationMemberReader) (PopulationMappingResult, error) { - if e == nil || e.queryRows == nil { + isPostgreSQL := false + switch compiled.Command.Backend() { + case command.CompiledBackendAQL: + case command.CompiledBackendPostgreSQL: + isPostgreSQL = true + default: + return PopulationMappingResult{}, fmt.Errorf("population mapping command has unsupported backend %q", string(compiled.Command.Backend())) + } + if e == nil || e.sourceRows == nil { + if isPostgreSQL { + return PopulationMappingResult{}, fmt.Errorf("%w: no row port serves the selected PostgreSQL composition", ErrPopulationMappingPostgresUnsupported) + } return PopulationMappingResult{}, fmt.Errorf("population mapping query executor is required") } if reader == nil { @@ -154,11 +150,22 @@ func (e *Engine) populationMappingCompiled(ctx context.Context, compiled compile if compiled.MemberColumn == "" || (compiled.IdentityPartsColumn == "" && compiled.ExplicitIdentityColumn == "") { return PopulationMappingResult{}, fmt.Errorf("population mapping query is missing typed witness columns") } + stream := RowsRequest{Command: compiled.Command, BatchSize: e.batchSize} + if isPostgreSQL { + if compiled.IdentityPartsColumn != "" && compiled.ExplicitIdentityColumn != "" { + return PopulationMappingResult{}, fmt.Errorf("population mapping PostgreSQL witness requires exactly one identity column") + } + identityColumn := compiled.IdentityPartsColumn + if identityColumn == "" { + identityColumn = compiled.ExplicitIdentityColumn + } + stream.DeclaredColumns = []string{compiled.MemberColumn, identityColumn} + } mapped := make(map[string]struct{}) emitted := make(map[string]struct{}) witnessRows := int64(0) - err := e.queryRows(ctx, compiled.Query, e.batchSize, compiled.BindVars, func(row map[string]any) error { + err := e.sourceRows(ctx, stream, func(row map[string]any) error { if err := ctx.Err(); err != nil { return err } @@ -186,6 +193,9 @@ func (e *Engine) populationMappingCompiled(ctx context.Context, compiled compile if populationMappingIncomplete(err) { return incompletePopulationMappingResult(), nil } + if isPostgreSQL && errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + return PopulationMappingResult{}, fmt.Errorf("%w: PostgreSQL row adapter is not registered for the mapping command", ErrPopulationMappingPostgresUnsupported) + } return PopulationMappingResult{}, fmt.Errorf("execute population mapping: %w", err) } if err := ctx.Err(); err != nil { @@ -221,12 +231,12 @@ func (e *Engine) populationMappingCompiled(ctx context.Context, compiled compile }, nil } -func compilePopulationMappingOutput(resolved Resolved, name string) (compiler.CompiledPopulationMappingQuery, error) { +func compilePopulationMappingOutput(target compiler.Target, resolved Resolved, name string) (compiler.CompiledPopulationMappingQuery, error) { for _, output := range resolved.Compiled.Outputs { if output.Name != name { continue } - return compiler.CompilePopulationMappingOutputWithPolicy(output, resolved.Semantic.SemanticPlan.Bindings, ir.DefaultPhysicalOptimizationPolicy()) + return compiler.CompilePopulationMappingOutput(target, output, resolved.Semantic.SemanticPlan.Bindings, ir.DefaultPhysicalOptimizationPolicy()) } return compiler.CompiledPopulationMappingQuery{}, fmt.Errorf("population mapping output %q was not found", name) } @@ -235,8 +245,11 @@ func populationWitnessIdentity(query compiler.CompiledPopulationMappingQuery, ro if query.ExplicitIdentityColumn != "" { return populationExplicitIdentity(row[query.ExplicitIdentityColumn]) } - value, ok := row[query.IdentityPartsColumn].([]any) - if !ok || len(value) == 0 || query.RowIdentity == nil || len(value) != len(query.RowIdentity.Fields) { + value, err := populationIdentityParts(row[query.IdentityPartsColumn]) + if err != nil { + return "", err + } + if len(value) == 0 || query.RowIdentity == nil || len(value) != len(query.RowIdentity.Fields) { return "", fmt.Errorf("population mapping witness has an invalid row identity shape") } for _, part := range value { @@ -252,6 +265,28 @@ func populationWitnessIdentity(query compiler.CompiledPopulationMappingQuery, ro return "default:" + hex.EncodeToString(digest[:]), nil } +func populationIdentityParts(value any) ([]any, error) { + switch typed := value.(type) { + case []any: + return typed, nil + case json.RawMessage: + if len(typed) == 0 || !json.Valid(typed) { + return nil, fmt.Errorf("population mapping witness has an invalid row identity shape") + } + var decoded any + if err := json.Unmarshal(typed, &decoded); err != nil { + return nil, fmt.Errorf("population mapping witness has an invalid row identity shape: %w", err) + } + items, ok := decoded.([]any) + if !ok { + return nil, fmt.Errorf("population mapping witness has an invalid row identity shape") + } + return items, nil + default: + return nil, fmt.Errorf("population mapping witness has an invalid row identity shape") + } +} + func populationExplicitIdentity(value any) (string, error) { switch value := value.(type) { case string: diff --git a/internal/dataframe/execution/population_mapping_postgres_pg18_test.go b/internal/dataframe/execution/population_mapping_postgres_pg18_test.go new file mode 100644 index 000000000..fdf549300 --- /dev/null +++ b/internal/dataframe/execution/population_mapping_postgres_pg18_test.go @@ -0,0 +1,962 @@ +package execution_test + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/authscope" + postgres "github.com/calypr/loom/internal/backends/postgres" + storepg "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/explorer" + "github.com/calypr/loom/internal/explorer/capability" + "github.com/calypr/loom/internal/explorer/lifecycle" + "github.com/calypr/loom/internal/projectid" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + exec07Prefix = "verify_exec07_cs_" + exec07NameFmt = exec07Prefix + "%d_%s" + + exec07Project = "project-a" + exec07Generation = "gen-a" + exec07AuthPath = "FHIR/Observation" + exec07Explorer = "observations" + exec07Selection = "sel-exec07" + exec07MembersColl = "loom_explorer_selection_members" +) + +var exec07NamePattern = regexp.MustCompile(`^verify_exec07_cs_[0-9]+_[0-9a-f]+$`) + +var ( + exec07OwnedMu sync.Mutex + exec07OwnedByName = map[string]string{} +) + +func exec07Register(db, schema string) { + exec07OwnedMu.Lock() + defer exec07OwnedMu.Unlock() + exec07OwnedByName[db] = schema +} + +func exec07RegistrySnapshot() map[string]string { + exec07OwnedMu.Lock() + defer exec07OwnedMu.Unlock() + out := make(map[string]string, len(exec07OwnedByName)) + for db, schema := range exec07OwnedByName { + out[db] = schema + } + return out +} + +var ( + exec07BaselineOnce sync.Once + exec07BaselineNames []string + exec07BaselineErr error +) + +func exec07CaptureBaseline(raw string) { + exec07BaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec07BaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + rows, err := admin.Query(ctx, `SELECT datname FROM pg_database WHERE datname LIKE 'verify\_exec07\_cs\_%' ESCAPE '\' UNION ALL SELECT nspname FROM pg_namespace WHERE nspname LIKE 'verify\_exec07\_cs\_%' ESCAPE '\'`) + if err != nil { + exec07BaselineErr = fmt.Errorf("read baseline: %w", err) + return + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + exec07BaselineErr = fmt.Errorf("scan baseline: %w", err) + return + } + exec07BaselineNames = append(exec07BaselineNames, name) + } + exec07BaselineErr = rows.Err() + }) +} + +func exec07RandomHex(t *testing.T) string { + t.Helper() + buf := make([]byte, 6) + if _, err := rand.Read(buf); err != nil { + t.Fatalf("read name entropy: %v", err) + } + return hex.EncodeToString(buf) +} + +type exec07Service struct { + ctx context.Context + iso *pgxpool.Pool + schema string + target compiler.Target + scrub func(any) string + addSec func(string) + fail func(string, error) + adapter dataframeexecution.SourceRows + store *storepg.ExplorerStore + scopeDigest string +} + +func newExec07Service(t *testing.T) *exec07Service { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 population-mapping acceptance tests") + } + scrub, addSecret := exec04pvScrubberFor(raw) + exec07CaptureBaseline(raw) + if exec07BaselineErr != nil { + t.Fatalf("capture pg_database baseline: %s", scrub(exec07BaselineErr)) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, "SELECT current_setting('server_version_num')::int8, current_setting('server_version')").Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if err := storepg.CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required PostgreSQL 18 backend: %v", err) + } + + dbName := fmt.Sprintf(exec07NameFmt, time.Now().UnixNano(), exec07RandomHex(t)) + if !exec07NamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + var created int64 + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", dbName).Scan(&created); err != nil { + fail("verify isolated database creation", err) + } + if created != 1 { + t.Fatalf("isolated database %q did not verifiably come into existence", dbName) + } + exec07Register(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + addSecret(dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(exec07NameFmt, time.Now().UnixNano(), exec07RandomHex(t)) + if !exec07NamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + exec07Register(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + client, err := storepg.OpenPool(ctx, storepg.ClientConfig{DSN: dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 2 * time.Second}) + if err != nil { + fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + t.Cleanup(client.Close) + if _, err := storepg.MigrateSchema(ctx, client, schema); err != nil { + fail("migrate owned schema", err) + } + migrations, err := storepg.OwnerMigrations() + if err != nil { + fail("load owner migrations", err) + } + conn, err := iso.Acquire(ctx) + if err != nil { + fail("acquire DDL connection", err) + } + if _, err := conn.Exec(ctx, `SET search_path TO "`+schema+`"`); err != nil { + conn.Release() + fail("pin search_path for membership DDL", err) + } + applied := 0 + for _, migration := range migrations { + if migration.Version == 1 { + continue + } + if _, err := conn.Exec(ctx, migration.SQL); err != nil { + conn.Release() + fail(fmt.Sprintf("apply explorer DDL %d_%s", migration.Version, migration.Name), err) + } + applied++ + } + conn.Release() + if applied != len(migrations)-1 { + t.Fatalf("applied %d explorer migrations, want %d", applied, len(migrations)-1) + } + + pgStore, err := storepg.NewExplorerStore(client, schema) + if err != nil { + fail("open explorer store", err) + } + + binding, err := ir.NewPostgresStorageBinding(schema) + if err != nil { + fail("bind PostgreSQL storage placement", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + fail("select PostgreSQL compile target", err) + } + adapter, err := postgres.NewSourceRows(iso) + if err != nil { + fail("create PostgreSQL row adapter", err) + } + return &exec07Service{ctx: ctx, iso: iso, schema: schema, target: target, scrub: scrub, addSec: addSecret, fail: fail, adapter: adapter, store: pgStore, scopeDigest: "exec07-scope"} +} + +func (s *exec07Service) seedObservation(t *testing.T, project, generation, logicalID, status string) { + t.Helper() + repo, err := storepg.NewResourceRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + encoded, err := json.Marshal(map[string]any{"id": logicalID, "status": status}) + if err != nil { + t.Fatalf("encode payload for %s: %v", logicalID, err) + } + if _, err := repo.InsertResource(s.ctx, source.Resource{ + Project: project, + DatasetGeneration: generation, + ResourceType: "Observation", + LogicalID: logicalID, + AuthResourcePath: exec07AuthPath, + Payload: json.RawMessage(encoded), + }); err != nil { + s.fail("insert fixture resource "+logicalID, err) + } +} + +func (s *exec07Service) seedSelection(t *testing.T, memberIDs []string, scopeDigest string) explorer.SelectionRevision { + t.Helper() + project := projectid.Canonical(exec07Project) + writer := "exec07-writer" + now := time.Now().UTC() + header, err := s.store.BeginSelection(s.ctx, explorer.SelectionRevision{ + ID: exec07Selection, Project: project, Generation: exec07Generation, ResourceType: "Observation", + Rule: explorer.SelectionRule{Kind: explorer.SelectionRuleExplicit}, + Source: explorer.SelectionSource{Kind: explorer.SelectionSourceExplicit}, + ScopeDigest: scopeDigest, RuleDigest: "exec07-rule", CreatedAt: now, + }, writer) + if err != nil { + s.fail("begin selection", err) + } + members := make([]explorer.SelectionMember, 0, len(memberIDs)) + for _, memberID := range memberIDs { + members = append(members, explorer.SelectionMember{Ref: explorer.ResourceRef{ + Project: project, Generation: exec07Generation, ResourceType: "Observation", ID: memberID}}) + } + if _, err := s.store.AppendSelectionMembers(s.ctx, header.ID, writer, members); err != nil { + s.fail("append selection members", err) + } + digest, count, bytes, err := s.store.DigestSelectionMembers(s.ctx, project, header.ID) + if err != nil { + s.fail("digest selection", err) + } + completed, err := s.store.CompleteSelection(s.ctx, header.ID, writer, digest, count, bytes, time.Now().UTC()) + if err != nil { + s.fail("complete selection", err) + } + if completed.MembershipDigest != digest || completed.MemberCount != count { + t.Fatalf("completed header facts drifted: %v", completed) + } + return *completed +} + +func exec07Population(header explorer.SelectionRevision) *recipe.PopulationConstraint { + return &recipe.PopulationConstraint{ + SelectionRevisionID: header.ID, MembershipDigest: header.MembershipDigest, + MemberCount: header.MemberCount, ResourceType: header.ResourceType, + } +} + +func exec07FlatBundle(name string, header explorer.SelectionRevision) recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: name, + TranslationVersion: "exec07", + Outputs: []recipe.Output{{ + Name: exec07Explorer, RootResourceType: "Observation", RowGrain: "observation", + Fields: []recipe.Field{{Name: "status", ColumnID: "status_id", Expr: recipe.Expression{Select: "root.status"}}}, + Population: exec07Population(header), + }}, + } +} + +func exec07GroupBundle(name string, header explorer.SelectionRevision) recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: name, + TranslationVersion: "exec07", + Outputs: []recipe.Output{{ + Name: exec07Explorer, RootResourceType: "Observation", RowGrain: "observation", + Fields: []recipe.Field{{Name: "status", ColumnID: "status_id", Expr: recipe.Expression{Select: "root.status"}}}, + Construction: &recipe.Construction{ + Version: 1, + SourceColumns: []recipe.StageColumn{{ID: "status_id", Name: "status"}}, + Steps: []recipe.ConstructionStep{{ + ID: "group_status", + Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionGroupOp, Group: &recipe.ConstructionGroup{ + ConstructionID: "exec07_group", + MissingKeyPolicy: recipe.ConstructionGroupMissingKeyGroup, + Keys: []recipe.ConstructionGroupKey{{InputColumnID: "status_id", OutputColumnID: "grouped_status_id"}}, + Aggregates: []recipe.ConstructionGroupAggregate{{Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}}, + }}, + Outputs: []recipe.StageColumn{{ID: "grouped_status_id", Name: "status"}, {ID: "rows_id", Name: "rows", Type: "integer"}}, + }}, + }, + Population: exec07Population(header), + }}, + } +} + +func exec07Bindings(project, generation string) recipe.RuntimeBindings { + return recipe.RuntimeBindings{ + Project: project, DatasetGeneration: generation, + SelectionProject: projectid.Canonical(exec07Project), + SelectionMembersCollection: exec07MembersColl, + AuthResourcePaths: []string{exec07AuthPath}, + } +} + +type exec07Call struct { + sql string + args []command.PostgreSQLParameter + declared []string + tag command.CompiledBackend +} + +type exec07Recorder struct { + adapter dataframeexecution.SourceRows + mu sync.Mutex + calls []exec07Call + visits int + afterVisit func(visits int) + failNext error +} + +func (r *exec07Recorder) port(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + sqlText, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + r.mu.Lock() + fail := r.failNext + r.failNext = nil + r.calls = append(r.calls, exec07Call{sql: sqlText, args: args, declared: append([]string(nil), request.DeclaredColumns...), tag: request.Command.Backend()}) + r.mu.Unlock() + if fail != nil { + return fail + } + return r.adapter(ctx, request, func(row map[string]any) error { + r.mu.Lock() + r.visits++ + seen := r.visits + hook := r.afterVisit + r.mu.Unlock() + if err := visit(row); err != nil { + return err + } + if hook != nil { + hook(seen) + } + return nil + }) +} + +func (r *exec07Recorder) count() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.calls) +} + +func (r *exec07Recorder) last() exec07Call { + r.mu.Lock() + defer r.mu.Unlock() + return r.calls[len(r.calls)-1] +} + +func (s *exec07Service) engine(t *testing.T, recorder dataframeexecution.SourceRows) *dataframeexecution.Engine { + t.Helper() + config := exec04pvForbidArango() + config.CompileTarget = s.target + config.PreviewSourceRows = nil + config.ScopeDigest = func(recipe.RuntimeBindings) string { return s.scopeDigest } + config.SourceRows = recorder + engine, err := dataframeexecution.New(config) + if err != nil { + t.Fatalf("engine: %v", s.scrub(err)) + } + return engine +} + +func (s *exec07Service) resolve(t *testing.T, engine *dataframeexecution.Engine, bundle recipe.Bundle, generation string) dataframeexecution.Resolved { + t.Helper() + resolved, err := engine.CompileResolvedBundle(s.ctx, bundle, exec07Bindings(projectid.Legacy(exec07Project), generation)) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + return resolved +} + +type exec07ExplorerStore struct { + explorer.Store + pg *storepg.ExplorerStore +} + +func (s exec07ExplorerStore) GetSelection(ctx context.Context, project, id string) (*explorer.SelectionRevision, error) { + return s.pg.GetSelection(ctx, project, id) +} + +func (s exec07ExplorerStore) VisitSelectionMembers(ctx context.Context, project, id, after string, limit int, visit func(explorer.SelectionMember) error) (string, error) { + return s.pg.VisitSelectionMembers(ctx, project, id, after, limit, visit) +} + +type exec07Authorized struct { + mu sync.Mutex + snapshot capability.Snapshot + scope authscope.ReadScope + hookCalls int +} + +func (a *exec07Authorized) set(snapshot capability.Snapshot, scope authscope.ReadScope) { + a.mu.Lock() + defer a.mu.Unlock() + a.snapshot = snapshot + a.scope = scope +} + +func (a *exec07Authorized) authorized() (capability.Snapshot, authscope.ReadScope) { + a.mu.Lock() + defer a.mu.Unlock() + return a.snapshot, a.scope +} + +func (a *exec07Authorized) calls() int { + a.mu.Lock() + defer a.mu.Unlock() + return a.hookCalls +} + +func exec07ScopeDigest(scope authscope.ReadScope) string { + paths := append([]string(nil), scope.AuthResourcePaths...) + sort.Strings(paths) + sum := sha256.Sum256([]byte(string(scope.Mode) + "\x00" + strings.Join(paths, "\x00"))) + return hex.EncodeToString(sum[:]) +} + +func exec07Snapshot(project, generation, token string, scope authscope.ReadScope) capability.Snapshot { + return capability.Snapshot{ + Identity: capability.SnapshotIdentity{ + Project: project, Generation: generation, + AuthorizationScopeDigest: exec07ScopeDigest(scope), + SchemaDigest: "exec07-schema", ShapeDigest: "exec07-shape", + }, + Status: capability.StatusReady, Complete: true, Token: token, + } +} + +func exec07Receipt(t *testing.T, snapshot capability.Snapshot, bundle recipe.Bundle, columns []explorer.EmittedColumn, contract json.RawMessage) explorer.CompilationReceipt { + t.Helper() + receipt := explorer.CompilationReceipt{ + ReceiptFormatVersion: explorer.CurrentReceiptFormatVersion, + CompilerContractVersion: explorer.CurrentCompilerContractVersion, + Project: snapshot.Identity.Project, + ExplorerID: exec07Explorer, + IntentDigest: "exec07-intent", + SnapshotToken: snapshot.Token, + AuthorizationScopeDigest: snapshot.Identity.AuthorizationScopeDigest, + CapabilitySchemaDigest: snapshot.Identity.SchemaDigest, + ShapeDigest: snapshot.Identity.ShapeDigest, + SourceGeneration: snapshot.Identity.Generation, + CompiledConfig: json.RawMessage(`{}`), + PublicOutputContract: contract, + Bundle: bundle, + EmittedColumns: columns, + } + var err error + receipt.RecipeDigest, err = bundle.Digest() + if err != nil { + t.Fatalf("bundle digest: %v", err) + } + receipt.ResolvedRecipeDigest = receipt.RecipeDigest + receipt.ResolvedSchemaDigest = "exec07-resolved-schema" + provenance := map[string]string{} + for _, column := range columns { + provenance[column.PublicColumn] = "EXPLICIT" + } + receipt.OutputColumnProvenance = map[string]map[string]string{exec07Explorer: provenance} + receipt.OutputContractDigest, err = explorer.CompilationArtifactDigest(contract) + if err != nil { + t.Fatalf("contract digest: %v", err) + } + receipt.CompilationKey, err = explorer.CompilationKey(receipt) + if err != nil { + t.Fatalf("compilation key: %v", err) + } + receipt.ID, err = explorer.ReceiptID(receipt) + if err != nil { + t.Fatalf("receipt id: %v", err) + } + return receipt +} + +func exec07StatusColumns() []explorer.EmittedColumn { + return []explorer.EmittedColumn{ + {OutputID: exec07Explorer, EmissionID: "em_status", PublicColumn: "status", Label: "Status", LogicalType: "string", Filterable: true, Chartable: true}, + } +} + +func exec07StatusContract() json.RawMessage { + return json.RawMessage(`{"outputs":[{"outputId":"observations","columns":[{"column":"status","label":"Status","logicalType":"string","filterable":true,"chartable":true}]}]}`) +} + +func exec07GroupColumns() []explorer.EmittedColumn { + return []explorer.EmittedColumn{ + {OutputID: exec07Explorer, EmissionID: "em_status", PublicColumn: "status", Label: "Status", LogicalType: "string", Filterable: true, Chartable: true}, + {OutputID: exec07Explorer, EmissionID: "em_rows", PublicColumn: "rows", Label: "Rows", LogicalType: "integer", Filterable: true}, + } +} + +func exec07GroupContract() json.RawMessage { + return json.RawMessage(`{"outputs":[{"outputId":"observations","columns":[{"column":"status","label":"Status","logicalType":"string","filterable":true,"chartable":true},{"column":"rows","label":"Rows","logicalType":"integer","filterable":true,"chartable":false}]}]}`) +} + +func exec07LifecycleService(t *testing.T, store *storepg.ExplorerStore, authorized *exec07Authorized, engine *dataframeexecution.Engine, receipts ...explorer.CompilationReceipt) *lifecycle.Service { + t.Helper() + byID := map[string]explorer.CompilationReceipt{} + for _, receipt := range receipts { + byID[receipt.ID] = receipt + } + domain, err := explorer.NewService(&exec07ExplorerStore{pg: store}) + if err != nil { + t.Fatalf("explorer service: %v", err) + } + config := lifecycle.Config{ + SelectionMembersCollection: exec07MembersColl, + Capability: lifecycle.CapabilityResolver{ + ForExecution: func(context.Context, string, string) (lifecycle.AuthorizedCapability, error) { + snapshot, scope := authorized.authorized() + return lifecycle.AuthorizedCapability{Snapshot: snapshot, Scope: scope}, nil + }, + }, + ReceiptLookup: func(_ context.Context, _, _, receiptID string) (*explorer.CompilationReceipt, error) { + got, ok := byID[receiptID] + if !ok { + return nil, fmt.Errorf("receipt %q is not in this test's store", receiptID) + } + return &got, nil + }, + } + codec, err := lifecycle.NewHMACPopulationMappingCursorCodec("exec07-cursor-secret") + if err != nil { + t.Fatalf("cursor codec: %v", err) + } + config.PopulationMappingCursorCodec = codec + config.PopulationMapping = func(ctx context.Context, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings, output string, memberIDs []string, after string, limit int) (dataframeexecution.PopulationMappingResult, error) { + authorized.mu.Lock() + authorized.hookCalls++ + authorized.mu.Unlock() + resolved, err := engine.CompileResolvedBundle(ctx, receipt.Bundle, bindings) + if err != nil { + return dataframeexecution.PopulationMappingResult{}, err + } + reader := dataframeexecution.PopulationMemberReaderFunc(func(_ context.Context, visit func(string) error) error { + for _, id := range memberIDs { + if err := visit(id); err != nil { + return err + } + } + return nil + }) + return engine.PopulationMapping(ctx, resolved, dataframeexecution.PopulationMappingRequest{Output: output, AfterMemberID: after, MaxUnmapped: limit}, reader) + } + service, err := lifecycle.New(domain, config) + if err != nil { + t.Fatalf("lifecycle service: %v", err) + } + return service +} + +func exec07LifecycleRequest(receipt explorer.CompilationReceipt, limit int) lifecycle.PopulationMappingRequest { + return lifecycle.PopulationMappingRequest{ + Project: receipt.Project, ExplorerID: receipt.ExplorerID, ReceiptID: receipt.ID, + OutputID: exec07Explorer, Limit: limit, + } +} + +func TestPostgresPopulationMappingLiveFlatExactMembership(t *testing.T) { + s := newExec07Service(t) + seedProject := projectid.Legacy(exec07Project) + s.seedObservation(t, seedProject, exec07Generation, "obs-1", "final") + s.seedObservation(t, seedProject, exec07Generation, "obs-2", "preliminary") + header := s.seedSelection(t, []string{"obs-1", "obs-2", "obs-3"}, "exec07-scope") + + recorder := &exec07Recorder{adapter: s.adapter} + engine := s.engine(t, recorder.port) + resolved := s.resolve(t, engine, exec07FlatBundle("exec07-flat", header), exec07Generation) + result, err := engine.PopulationMapping(s.ctx, resolved, dataframeexecution.PopulationMappingRequest{Output: exec07Explorer, MaxUnmapped: 10}, + dataframeexecution.PopulationMemberReaderFunc(func(_ context.Context, visit func(string) error) error { + for _, id := range []string{"obs-1", "obs-2", "obs-3"} { + if err := visit(id); err != nil { + return err + } + } + return nil + })) + if err != nil { + t.Fatalf("live flat mapping: %s", s.scrub(err)) + } + if result.Status != dataframeexecution.PopulationMappingComplete || result.SelectedCount != 3 || result.MappedCount != 2 || result.UnmappedCount != 1 || result.EmittedRows != 2 { + t.Fatalf("live flat mapping result = %#v", result) + } + if len(result.UnmappedMemberIDs) != 1 || result.UnmappedMemberIDs[0] != "obs-3" { + t.Fatalf("live unmapped member IDs = %#v, want the phantom member", result.UnmappedMemberIDs) + } + if recorder.count() != 1 { + t.Fatalf("statements executed = %d, want the single witness terminal", recorder.count()) + } + call := recorder.last() + if call.tag != command.CompiledBackendPostgreSQL { + t.Fatalf("dispatched tag = %q", call.tag) + } + if want := []string{ir.PhysicalPopulationMappingMemberField, ir.PhysicalPopulationMappingIdentityPartsField}; len(call.declared) != 2 || call.declared[0] != want[0] || call.declared[1] != want[1] { + t.Fatalf("declared witness contract at the port = %#v", call.declared) + } + if !strings.Contains(call.sql, `"`+s.schema+`"."selection_members"`) || strings.Contains(call.sql, "@") { + t.Fatalf("witness statement is not the registered PostgreSQL terminal:\n%s", call.sql) + } + if len(call.args) == 0 { + t.Fatal("witness statement is not parameterized") + } + for _, arg := range call.args { + if arg.Type == command.PostgreSQLParameterText && arg.Value == exec07Selection { + return + } + } + t.Fatalf("selection revision identity is not a bound text parameter: %#v", call.args) +} + +func TestPostgresPopulationMappingLiveGroupDedupe(t *testing.T) { + s := newExec07Service(t) + seedProject := projectid.Legacy(exec07Project) + s.seedObservation(t, seedProject, exec07Generation, "obs-1", "final") + s.seedObservation(t, seedProject, exec07Generation, "obs-2", "final") + header := s.seedSelection(t, []string{"obs-1", "obs-2", "obs-3"}, "exec07-scope") + + recorder := &exec07Recorder{adapter: s.adapter} + engine := s.engine(t, recorder.port) + resolved := s.resolve(t, engine, exec07GroupBundle("exec07-group", header), exec07Generation) + result, err := engine.PopulationMapping(s.ctx, resolved, dataframeexecution.PopulationMappingRequest{Output: exec07Explorer, MaxUnmapped: 10}, + dataframeexecution.PopulationMemberReaderFunc(func(_ context.Context, visit func(string) error) error { + for _, id := range []string{"obs-1", "obs-2", "obs-3"} { + if err := visit(id); err != nil { + return err + } + } + return nil + })) + if err != nil { + t.Fatalf("live group mapping: %s", s.scrub(err)) + } + if result.Status != dataframeexecution.PopulationMappingComplete || result.SelectedCount != 3 || result.MappedCount != 2 || result.UnmappedCount != 1 { + t.Fatalf("live group mapping result = %#v", result) + } + if result.EmittedRows != 1 { + t.Fatalf("group witness rows did not collapse onto the single group row identity: %#v", result) + } + if recorder.count() != 1 { + t.Fatalf("statements executed = %d", recorder.count()) + } +} + +func TestPostgresPopulationMappingLiveCapAndCancellationAreIncomplete(t *testing.T) { + s := newExec07Service(t) + seedProject := projectid.Legacy(exec07Project) + s.seedObservation(t, seedProject, exec07Generation, "obs-1", "final") + s.seedObservation(t, seedProject, exec07Generation, "obs-2", "preliminary") + header := s.seedSelection(t, []string{"obs-1", "obs-2", "obs-3"}, "exec07-scope") + reader := dataframeexecution.PopulationMemberReaderFunc(func(_ context.Context, visit func(string) error) error { + for _, id := range []string{"obs-1", "obs-2", "obs-3"} { + if err := visit(id); err != nil { + return err + } + } + return nil + }) + + recorder := &exec07Recorder{adapter: s.adapter} + engine := s.engine(t, recorder.port) + resolved := s.resolve(t, engine, exec07FlatBundle("exec07-cap", header), exec07Generation) + result, err := engine.PopulationMapping(s.ctx, resolved, dataframeexecution.PopulationMappingRequest{Output: exec07Explorer, MaxWitnessRows: 1}, reader) + if err != nil { + t.Fatalf("cap mapping: %s", s.scrub(err)) + } + if result.Status != dataframeexecution.PopulationMappingIncomplete || result.SelectedCount != 0 || result.MappedCount != 0 || result.UnmappedCount != 0 || result.EmittedRows != 0 { + t.Fatalf("cap produced a misleading partial result: %#v", result) + } + if recorder.count() != 1 { + t.Fatalf("cap leg executed %d statements, want the single witness terminal", recorder.count()) + } + + ctx, cancel := context.WithCancel(s.ctx) + t.Cleanup(cancel) + recorder = &exec07Recorder{adapter: s.adapter} + recorder.afterVisit = func(visits int) { + if visits == 1 { + cancel() + } + } + engine = s.engine(t, recorder.port) + result, err = engine.PopulationMapping(ctx, resolved, dataframeexecution.PopulationMappingRequest{Output: exec07Explorer, MaxUnmapped: 10}, reader) + if err != nil { + t.Fatalf("cancellation mapping: %s", s.scrub(err)) + } + if result.Status != dataframeexecution.PopulationMappingIncomplete || result.SelectedCount != 0 || result.MappedCount != 0 || result.EmittedRows != 0 { + t.Fatalf("cancellation produced a misleading partial result: %#v", result) + } + if recorder.visits == 0 { + t.Fatal("cancellation leg never streamed a witness row") + } +} + +func TestPostgresPopulationMappingLivePortFailureIsHardError(t *testing.T) { + s := newExec07Service(t) + seedProject := projectid.Legacy(exec07Project) + s.seedObservation(t, seedProject, exec07Generation, "obs-1", "final") + header := s.seedSelection(t, []string{"obs-1"}, "exec07-scope") + reader := dataframeexecution.PopulationMemberReaderFunc(func(_ context.Context, visit func(string) error) error { + return visit("obs-1") + }) + + recorder := &exec07Recorder{adapter: s.adapter, failNext: errors.New("connection reset by peer")} + engine := s.engine(t, recorder.port) + resolved := s.resolve(t, engine, exec07FlatBundle("exec07-fail", header), exec07Generation) + result, err := engine.PopulationMapping(s.ctx, resolved, dataframeexecution.PopulationMappingRequest{Output: exec07Explorer}, reader) + if err == nil || result.Status != "" { + t.Fatalf("generic port failure was not a hard error: result=%#v err=%v", result, s.scrub(err)) + } + + recorder = &exec07Recorder{adapter: s.adapter} + recorder.failNext = fmt.Errorf("postgres rows: %w", command.ErrCompiledCommandBackendNotRegistered) + engine = s.engine(t, recorder.port) + result, err = engine.PopulationMapping(s.ctx, resolved, dataframeexecution.PopulationMappingRequest{Output: exec07Explorer}, reader) + if !errors.Is(err, dataframeexecution.ErrPopulationMappingPostgresUnsupported) { + t.Fatalf("unregistered adapter error = %v, want the typed capability refusal", s.scrub(err)) + } + if result.Status != "" || result.SelectedCount != 0 || result.MappedCount != 0 { + t.Fatalf("capability refusal exposed counts: %#v", result) + } + recorder.mu.Lock() + visited := recorder.visits + recorder.mu.Unlock() + if visited != 0 { + t.Fatalf("capability refusal visited %d rows, want zero", visited) + } + + recorder = &exec07Recorder{adapter: s.adapter} + recorder.failNext = fmt.Errorf("dispatch: %w for %q", command.ErrCompiledCommandBackendMismatch, "POSTGRESQL") + engine = s.engine(t, recorder.port) + _, err = engine.PopulationMapping(s.ctx, resolved, dataframeexecution.PopulationMappingRequest{Output: exec07Explorer}, reader) + if err == nil || !errors.Is(err, command.ErrCompiledCommandBackendMismatch) || errors.Is(err, dataframeexecution.ErrPopulationMappingPostgresUnsupported) { + t.Fatalf("routing mismatch error = %v", s.scrub(err)) + } +} + +func TestPostgresPopulationMappingLiveStaleReceiptRejectsBeforeExecution(t *testing.T) { + s := newExec07Service(t) + seedProject := projectid.Legacy(exec07Project) + s.seedObservation(t, seedProject, exec07Generation, "obs-1", "final") + s.seedObservation(t, seedProject, exec07Generation, "obs-2", "final") + scope := authscope.ReadScope{Mode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{exec07AuthPath}} + fresh := exec07Snapshot(exec07Project, exec07Generation, "token-1", scope) + s.scopeDigest = fresh.Identity.AuthorizationScopeDigest + header := s.seedSelection(t, []string{"obs-1", "obs-2", "obs-3"}, fresh.Identity.AuthorizationScopeDigest) + + recorder := &exec07Recorder{adapter: s.adapter} + engine := s.engine(t, recorder.port) + bundle := exec07FlatBundle("exec07-receipt", header) + receipt := exec07Receipt(t, fresh, bundle, exec07StatusColumns(), exec07StatusContract()) + authorized := &exec07Authorized{snapshot: fresh, scope: scope} + service := exec07LifecycleService(t, s.store, authorized, engine, receipt) + + report, err := service.PopulationMapping(s.ctx, exec07LifecycleRequest(receipt, 10)) + if err != nil { + t.Fatalf("fresh receipt: %s", s.scrub(err)) + } + if report.Report.Status != dataframeexecution.PopulationMappingComplete || report.Report.Counts == nil || + report.Report.Counts.Selected != 3 || report.Report.Counts.Mapped != 2 || report.Report.Counts.Unmapped != 1 || report.Report.Counts.EmittedRows != 2 { + t.Fatalf("fresh receipt report = %#v", report.Report) + } + if len(report.Report.Unmapped) != 1 || report.Report.Unmapped[0].ID != "obs-3" { + t.Fatalf("fresh receipt unmapped refs = %#v", report.Report.Unmapped) + } + if authorized.calls() != 1 || recorder.count() < 1 { + t.Fatalf("fresh control witnesses: hooks=%d statements=%d", authorized.calls(), recorder.count()) + } + + stale := fresh + stale.Identity.Generation = "gen-b" + authorized.set(stale, scope) + hooksBefore := authorized.calls() + statementsBefore := recorder.count() + _, err = service.PopulationMapping(s.ctx, exec07LifecycleRequest(receipt, 10)) + var lifecycleErr *lifecycle.Error + if !errors.As(err, &lifecycleErr) || lifecycleErr.Class != lifecycle.ClassConflict || lifecycleErr.Code != "RECEIPT_STALE" { + t.Fatalf("stale receipt error = %v, want ClassConflict/RECEIPT_STALE", s.scrub(err)) + } + if authorized.calls() != hooksBefore { + t.Fatalf("stale receipt reached the population-mapping hook: %d extra invocations", authorized.calls()-hooksBefore) + } + if got := recorder.count() - statementsBefore; got != 0 { + t.Fatalf("stale receipt executed %d statements, want the statement-count-0 witness at the production adapter", got) + } + + authorized.set(fresh, scope) + if _, err := service.PopulationMapping(s.ctx, exec07LifecycleRequest(receipt, 10)); err != nil { + t.Fatalf("restored fresh receipt: %s", s.scrub(err)) + } + if authorized.calls() != hooksBefore+1 { + t.Fatalf("restored receipt did not reach the hook: calls=%d", authorized.calls()) + } +} + +func TestPostgresPopulationMappingLiveNamedGapRefusesAtZeroStatements(t *testing.T) { + s := newExec07Service(t) + header := s.seedSelection(t, []string{"obs-1"}, "exec07-scope") + recorder := &exec07Recorder{adapter: s.adapter} + engine := s.engine(t, recorder.port) + bundle := exec07FlatBundle("exec07-gap", header) + bundle.Outputs[0].Expand = &recipe.Expansion{From: recipe.Expression{Select: "root.extension[]"}, As: "item"} + bundle.Outputs[0].RowGrain = "expanded" + bundle.Outputs[0].Identity = &recipe.Identity{Name: "row", Expr: recipe.Expression{Select: "root.id"}} + resolved := s.resolve(t, engine, bundle, exec07Generation) + _, err := engine.PopulationMapping(s.ctx, resolved, dataframeexecution.PopulationMappingRequest{Output: exec07Explorer}, + dataframeexecution.PopulationMemberReaderFunc(func(_ context.Context, visit func(string) error) error { + return visit("obs-1") + })) + if !errors.Is(err, dataframeexecution.ErrPopulationMappingPostgresUnsupported) { + t.Fatalf("named gap error = %v, want the typed capability refusal", s.scrub(err)) + } + if recorder.count() != 0 { + t.Fatalf("named-gap refusal executed %d statements, want the statement-count-0 witness", recorder.count()) + } +} + +func TestZZPostgresPopulationMappingZeroLeftoverFixtures(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return + } + exec07CaptureBaseline(raw) + if exec07BaselineErr != nil { + t.Fatalf("baseline: %v", exec07BaselineErr) + } + baseline := map[string]bool{} + for _, name := range exec07BaselineNames { + baseline[name] = true + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for audit: %v", err) + } + defer admin.Close() + var survivors []string + rows, err := admin.Query(ctx, `SELECT datname FROM pg_database WHERE datname LIKE 'verify\_exec07\_cs\_%' ESCAPE '\'`) + if err != nil { + t.Fatalf("audit databases: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("scan audit: %v", err) + } + if !baseline[name] { + survivors = append(survivors, name) + } + } + rows.Close() + rows, err = admin.Query(ctx, `SELECT nspname FROM pg_namespace WHERE nspname LIKE 'verify\_exec07\_cs\_%' ESCAPE '\'`) + if err != nil { + t.Fatalf("audit schemas: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("scan audit: %v", err) + } + if !baseline[name] { + survivors = append(survivors, name) + } + } + rows.Close() + if len(survivors) != 0 { + sort.Strings(survivors) + t.Fatalf("verify_exec07_cs_* fixtures survived the run: %v (owned registry: %v)", survivors, exec07RegistrySnapshot()) + } +} diff --git a/internal/dataframe/execution/population_mapping_test.go b/internal/dataframe/execution/population_mapping_test.go index 5d010258b..dca849e37 100644 --- a/internal/dataframe/execution/population_mapping_test.go +++ b/internal/dataframe/execution/population_mapping_test.go @@ -3,16 +3,28 @@ package execution import ( "context" "encoding/json" + "errors" + "fmt" + "strings" "testing" "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/spec" ) func TestPopulationMappingCompiledDeduplicatesMembersAndRows(t *testing.T) { - engine := &Engine{ + engine := &Engine{compileTarget: compiler.ArangoTarget(), batchSize: 1000, - queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + for _, row := range []map[string]any{ {"member": "file-001", "identity": []any{"specimen-001"}}, {"member": "file-002", "identity": []any{"specimen-001"}}, @@ -33,7 +45,7 @@ func TestPopulationMappingCompiledDeduplicatesMembersAndRows(t *testing.T) { return nil }) result, err := engine.PopulationMappingCompiled(context.Background(), compiler.CompiledPopulationMappingQuery{ - Query: "RETURN witnesses", + Command: mustAQLCommand("RETURN witnesses", nil), MemberColumn: "member", IdentityPartsColumn: "identity", RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, }, PopulationMappingRequest{MaxUnmapped: 10}, reader) @@ -49,9 +61,15 @@ func TestPopulationMappingCompiledDeduplicatesMembersAndRows(t *testing.T) { } func TestPopulationMappingCompiledReturnsIncompleteWithoutCountsOnLimit(t *testing.T) { - engine := &Engine{ + engine := &Engine{compileTarget: compiler.ArangoTarget(), batchSize: 1000, - queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{"member": "file-001", "identity": []any{"specimen-001"}}) }, } @@ -59,7 +77,7 @@ func TestPopulationMappingCompiledReturnsIncompleteWithoutCountsOnLimit(t *testi return visit("file-001") }) result, err := engine.PopulationMappingCompiled(context.Background(), compiler.CompiledPopulationMappingQuery{ - Query: "RETURN witnesses", MemberColumn: "member", IdentityPartsColumn: "identity", + Command: mustAQLCommand("RETURN witnesses", nil), MemberColumn: "member", IdentityPartsColumn: "identity", RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, }, PopulationMappingRequest{MaxWitnessRows: 0}, reader) if err != nil { @@ -71,7 +89,7 @@ func TestPopulationMappingCompiledReturnsIncompleteWithoutCountsOnLimit(t *testi ctx, cancel := context.WithCancel(context.Background()) cancel() incomplete, err := engine.PopulationMappingCompiled(ctx, compiler.CompiledPopulationMappingQuery{ - Query: "RETURN witnesses", MemberColumn: "member", IdentityPartsColumn: "identity", + Command: mustAQLCommand("RETURN witnesses", nil), MemberColumn: "member", IdentityPartsColumn: "identity", RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, }, PopulationMappingRequest{}, reader) if err != nil { @@ -126,9 +144,15 @@ func TestPopulationWitnessIdentityRejectsNullAndStructuredExplicitValues(t *test } func TestPopulationMappingCompiledDeduplicatesCanonicalExplicitNumericIdentities(t *testing.T) { - engine := &Engine{ + engine := &Engine{compileTarget: compiler.ArangoTarget(), batchSize: 1000, - queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + for _, row := range []map[string]any{ {"member": "file-001", "identity": float64(0.5)}, {"member": "file-001", "identity": json.Number("0.50")}, @@ -142,7 +166,7 @@ func TestPopulationMappingCompiledDeduplicatesCanonicalExplicitNumericIdentities }, } result, err := engine.PopulationMappingCompiled(context.Background(), compiler.CompiledPopulationMappingQuery{ - Query: "RETURN witnesses", MemberColumn: "member", ExplicitIdentityColumn: "identity", + Command: mustAQLCommand("RETURN witnesses", nil), MemberColumn: "member", ExplicitIdentityColumn: "identity", }, PopulationMappingRequest{MaxUnmapped: 10}, PopulationMemberReaderFunc(func(_ context.Context, visit func(string) error) error { return visit("file-001") })) @@ -153,3 +177,473 @@ func TestPopulationMappingCompiledDeduplicatesCanonicalExplicitNumericIdentities t.Fatalf("population mapping result = %#v", result) } } + +func exec07PostgreSQLTarget(t *testing.T) compiler.Target { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("loom") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("PostgreSQLTarget: %v", err) + } + return target +} + +func exec07MappingCommand(t *testing.T) command.CompiledCommand { + t.Helper() + compiled, err := command.NewPostgreSQLCommand( + `SELECT w0.member AS "c0", jsonb_build_array(s."id") AS "c1" FROM "loom"."specimen" AS s CROSS JOIN LATERAL jsonb_array_elements_text('[]'::jsonb) AS w0(member) ORDER BY w0.member COLLATE "C" ASC`, + []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterText, Value: "sha256:members"}}, + ) + if err != nil { + t.Fatalf("NewPostgreSQLCommand: %v", err) + } + return compiled +} + +type exec07Port struct { + calls int + requests []RowsRequest + rows []map[string]any + fail error + beforeVisit func(index int) + visitStarted bool +} + +func (p *exec07Port) sourceRows(ctx context.Context, request RowsRequest, visit RowVisitor) error { + if _, _, err := request.Command.PostgreSQLParameters(); err != nil { + return err + } + p.calls++ + p.requests = append(p.requests, request) + for index, row := range p.rows { + if p.beforeVisit != nil { + p.beforeVisit(index) + } + p.visitStarted = true + if err := ctx.Err(); err != nil { + return err + } + if err := visit(row); err != nil { + return err + } + } + if p.fail != nil { + return p.fail + } + return nil +} + +func exec07Members(ids ...string) PopulationMemberReader { + return PopulationMemberReaderFunc(func(_ context.Context, visit func(string) error) error { + for _, id := range ids { + if err := visit(id); err != nil { + return err + } + } + return nil + }) +} + +func exec07Query(t *testing.T, engine *Engine, request PopulationMappingRequest, reader PopulationMemberReader) (PopulationMappingResult, error) { + t.Helper() + return engine.PopulationMappingCompiled(context.Background(), compiler.CompiledPopulationMappingQuery{ + Command: exec07MappingCommand(t), MemberColumn: "c0", IdentityPartsColumn: "c1", + RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, + }, request, reader) +} + +func TestPostgresPopulationMappingExactMembershipAndDedupe(t *testing.T) { + port := &exec07Port{rows: []map[string]any{ + {"c0": "member-1", "c1": []any{"row-a"}}, + {"c0": "member-1", "c1": []any{"row-a"}}, + {"c0": "member-1", "c1": []any{"row-b"}}, + {"c0": "member-2", "c1": []any{"row-c"}}, + }} + engine := &Engine{compileTarget: exec07PostgreSQLTarget(t), batchSize: 1000, sourceRows: port.sourceRows} + result, err := exec07Query(t, engine, PopulationMappingRequest{MaxUnmapped: 10}, exec07Members("member-1", "member-2", "member-3")) + if err != nil { + t.Fatal(err) + } + if result.Status != PopulationMappingComplete || result.SelectedCount != 3 || result.MappedCount != 2 || result.UnmappedCount != 1 || result.EmittedRows != 3 { + t.Fatalf("population mapping result = %#v", result) + } + if len(result.UnmappedMemberIDs) != 1 || result.UnmappedMemberIDs[0] != "member-3" { + t.Fatalf("unmapped member IDs = %#v", result.UnmappedMemberIDs) + } + if port.calls != 1 || len(port.requests) != 1 { + t.Fatalf("port calls = %d, requests = %d", port.calls, len(port.requests)) + } + if got := port.requests[0].DeclaredColumns; len(got) != 2 || got[0] != "c0" || got[1] != "c1" { + t.Fatalf("declared columns = %#v, want [c0 c1] in renderer output order", got) + } + if port.requests[0].BatchSize != 1000 { + t.Fatalf("batch hint = %d", port.requests[0].BatchSize) + } +} + +func TestPostgresPopulationMappingRawMessagePartsMatchNativeIdentity(t *testing.T) { + port := &exec07Port{rows: []map[string]any{ + {"c0": "member-1", "c1": []any{"obs-a"}}, + {"c0": "member-2", "c1": json.RawMessage(`["obs-a"]`)}, + }} + engine := &Engine{compileTarget: exec07PostgreSQLTarget(t), batchSize: 1000, sourceRows: port.sourceRows} + result, err := exec07Query(t, engine, PopulationMappingRequest{MaxUnmapped: 10}, exec07Members("member-1", "member-2")) + if err != nil { + t.Fatal(err) + } + if result.Status != PopulationMappingComplete || result.SelectedCount != 2 || result.MappedCount != 2 || result.UnmappedCount != 0 || result.EmittedRows != 1 { + t.Fatalf("cross-shape identity equality broken: %#v, want two mapped members collapsing to one emitted identity", result) + } + for _, port := range []*exec07Port{ + {rows: []map[string]any{ + {"c0": "member-1", "c1": json.RawMessage(`["obs-a"]`)}, + {"c0": "member-2", "c1": json.RawMessage(`["obs-b"]`)}, + }}, + {rows: []map[string]any{ + {"c0": "member-1", "c1": []any{"obs-a"}}, + {"c0": "member-2", "c1": []any{"obs-b"}}, + }}, + } { + engine := &Engine{compileTarget: exec07PostgreSQLTarget(t), batchSize: 1000, sourceRows: port.sourceRows} + result, err := exec07Query(t, engine, PopulationMappingRequest{}, exec07Members("member-1", "member-2", "member-3")) + if err != nil { + t.Fatal(err) + } + if result.Status != PopulationMappingComplete || result.SelectedCount != 3 || result.MappedCount != 2 || result.UnmappedCount != 1 || result.EmittedRows != 2 || + len(result.UnmappedMemberIDs) != 1 || result.UnmappedMemberIDs[0] != "member-3" { + t.Fatalf("adapter-shape and native-shape runs must agree exactly; got %#v", result) + } + } +} + +func TestPostgresPopulationMappingRawMessageViolationsFailClosed(t *testing.T) { + for _, violation := range []struct { + name string + row map[string]any + }{ + {"empty bytes", map[string]any{"c0": "member-1", "c1": json.RawMessage("")}}, + {"malformed json", map[string]any{"c0": "member-1", "c1": json.RawMessage(`["obs-a"`)}}, + {"json null scalar", map[string]any{"c0": "member-1", "c1": json.RawMessage("null")}}, + {"json object", map[string]any{"c0": "member-1", "c1": json.RawMessage(`{"project":"P1"}`)}}, + {"json scalar", map[string]any{"c0": "member-1", "c1": json.RawMessage(`"obs-a"`)}}, + {"missing key (sql null)", map[string]any{"c0": "member-1"}}, + {"explicit nil", map[string]any{"c0": "member-1", "c1": nil}}, + } { + port := &exec07Port{rows: []map[string]any{ + {"c0": "member-1", "c1": []any{"obs-a"}}, + violation.row, + {"c0": "member-2", "c1": []any{"obs-b"}}, + }} + engine := &Engine{compileTarget: exec07PostgreSQLTarget(t), batchSize: 1000, sourceRows: port.sourceRows} + result, err := exec07Query(t, engine, PopulationMappingRequest{}, exec07Members("member-1", "member-2")) + if err == nil || !strings.Contains(err.Error(), "invalid row identity shape") { + t.Fatalf("%s: violation error = %v, want the typed invalid-shape refusal", violation.name, err) + } + if want := "population mapping witness has an invalid row identity shape"; !strings.HasSuffix(err.Error(), want) { + t.Fatalf("%s: violation error = %v, want it to end in the decoder sentinel %q", violation.name, err, want) + } + if errors.Is(err, ErrPopulationMappingPostgresUnsupported) || errors.Is(err, errPopulationMappingLimit) { + t.Fatalf("%s: violation misclassified as a capability gap or cap: %v", violation.name, err) + } + if result.Status != "" || result.SelectedCount != 0 || result.MappedCount != 0 || result.UnmappedCount != 0 || result.EmittedRows != 0 || + result.UnmappedMemberIDs != nil || result.HasMoreUnmapped { + t.Fatalf("%s: violation exposed a partial mapping: %#v", violation.name, result) + } + if !port.visitStarted || port.calls != 1 { + t.Fatalf("%s: refusal must run through the one port call after the witness stream started: calls=%d", violation.name, port.calls) + } + } +} + +func TestPostgresPopulationMappingForeignWitnessRejected(t *testing.T) { + port := &exec07Port{rows: []map[string]any{{"c0": "member-9", "c1": []any{"row-a"}}}} + engine := &Engine{compileTarget: exec07PostgreSQLTarget(t), batchSize: 1000, sourceRows: port.sourceRows} + _, err := exec07Query(t, engine, PopulationMappingRequest{}, exec07Members("member-1")) + if err == nil || !strings.Contains(err.Error(), "not selected") { + t.Fatalf("foreign witness error = %v", err) + } + if errors.Is(err, errPopulationMappingLimit) { + t.Fatalf("foreign witness misread as a cap: %v", err) + } +} + +func TestPostgresPopulationMappingCapIsIncompleteNotPartial(t *testing.T) { + port := &exec07Port{rows: []map[string]any{ + {"c0": "member-1", "c1": []any{"row-a"}}, + {"c0": "member-2", "c1": []any{"row-b"}}, + }} + engine := &Engine{compileTarget: exec07PostgreSQLTarget(t), batchSize: 1000, sourceRows: port.sourceRows} + result, err := exec07Query(t, engine, PopulationMappingRequest{MaxWitnessRows: 1}, exec07Members("member-1", "member-2")) + if err != nil { + t.Fatal(err) + } + if result.Status != PopulationMappingIncomplete || result.SelectedCount != 0 || result.MappedCount != 0 || result.UnmappedCount != 0 || result.EmittedRows != 0 { + t.Fatalf("cap produced a misleading partial result: %#v", result) + } + if !port.visitStarted { + t.Fatal("cap test never streamed a witness row") + } +} + +func TestPostgresPopulationMappingCancellationIsIncompleteAtRequest(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + port := &exec07Port{ + rows: []map[string]any{ + {"c0": "member-1", "c1": []any{"row-a"}}, + {"c0": "member-2", "c1": []any{"row-b"}}, + }, + } + port.beforeVisit = func(index int) { + if index == 1 { + cancel() + } + } + engine := &Engine{compileTarget: exec07PostgreSQLTarget(t), batchSize: 1000, sourceRows: port.sourceRows} + result, err := engine.PopulationMappingCompiled(ctx, compiler.CompiledPopulationMappingQuery{ + Command: exec07MappingCommand(t), MemberColumn: "c0", IdentityPartsColumn: "c1", + RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, + }, PopulationMappingRequest{}, exec07Members("member-1", "member-2")) + if err != nil { + t.Fatal(err) + } + if result.Status != PopulationMappingIncomplete || result.SelectedCount != 0 || result.MappedCount != 0 || result.EmittedRows != 0 { + t.Fatalf("cancellation produced a misleading partial result: %#v", result) + } + if !port.visitStarted { + t.Fatal("cancellation test never streamed a witness row") + } +} + +func TestPostgresPopulationMappingCapabilityRefusalBeforePortCall(t *testing.T) { + var portCalls int + engine := &Engine{ + compileTarget: exec07PostgreSQLTarget(t), batchSize: 1000, + sourceRows: func(context.Context, RowsRequest, RowVisitor) error { + portCalls++ + return nil + }, + } + compiled := compiler.CompiledPopulationMappingQuery{ + Command: exec07MappingCommand(t), MemberColumn: "c0", IdentityPartsColumn: "c1", + RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, + } + noPort := &Engine{compileTarget: exec07PostgreSQLTarget(t), batchSize: 1000} + result, err := noPort.PopulationMappingCompiled(context.Background(), compiled, PopulationMappingRequest{}, exec07Members("member-1")) + if !errors.Is(err, ErrPopulationMappingPostgresUnsupported) { + t.Fatalf("nil-port error = %v, want ErrPopulationMappingPostgresUnsupported", err) + } + if result.Status != "" || result.SelectedCount != 0 || result.MappedCount != 0 { + t.Fatalf("capability refusal exposed counts: %#v", result) + } + unregistered := &exec07Port{fail: fmt.Errorf("postgres rows: %w", command.ErrCompiledCommandBackendNotRegistered)} + engine.sourceRows = unregistered.sourceRows + result, err = engine.PopulationMappingCompiled(context.Background(), compiled, PopulationMappingRequest{}, exec07Members("member-1")) + if !errors.Is(err, ErrPopulationMappingPostgresUnsupported) || errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("unregistered-adapter error = %v", err) + } + if result.Status != "" || result.SelectedCount != 0 || result.MappedCount != 0 { + t.Fatalf("unregistered-adapter refusal exposed counts: %#v", result) + } + if unregistered.visitStarted { + t.Fatal("unregistered adapter visited witness rows") + } + if portCalls != 0 { + t.Fatalf("refusal path reached a port call: %d", portCalls) + } +} + +func TestPostgresPopulationMappingRoutingMismatchIsHardError(t *testing.T) { + port := &exec07Port{fail: fmt.Errorf("dispatch: %w for %q", command.ErrCompiledCommandBackendMismatch, "POSTGRESQL")} + engine := &Engine{compileTarget: exec07PostgreSQLTarget(t), batchSize: 1000, sourceRows: port.sourceRows} + result, err := exec07Query(t, engine, PopulationMappingRequest{}, exec07Members("member-1")) + if err == nil || !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("routing mismatch error = %v", err) + } + if errors.Is(err, ErrPopulationMappingPostgresUnsupported) { + t.Fatalf("routing mismatch misread as a capability gap: %v", err) + } + if result.Status != "" || result.SelectedCount != 0 || result.MappedCount != 0 { + t.Fatalf("routing mismatch exposed counts: %#v", result) + } +} + +func TestPostgresPopulationMappingGenericPortFailureIsHardError(t *testing.T) { + port := &exec07Port{rows: []map[string]any{{"c0": "member-1", "c1": []any{"row-a"}}}, fail: errors.New("connection reset")} + engine := &Engine{compileTarget: exec07PostgreSQLTarget(t), batchSize: 1000, sourceRows: port.sourceRows} + result, err := exec07Query(t, engine, PopulationMappingRequest{}, exec07Members("member-1")) + if err == nil || errors.Is(err, ErrPopulationMappingPostgresUnsupported) || result.Status == PopulationMappingIncomplete || result.Status == PopulationMappingComplete { + t.Fatalf("generic port failure was not a hard error: result=%#v err=%v", result, err) + } +} + +func TestPostgresPopulationMappingContractRefusesBeforeSQL(t *testing.T) { + var calls int + ping := func(context.Context, RowsRequest, RowVisitor) error { + calls++ + return nil + } + base := compiler.CompiledPopulationMappingQuery{MemberColumn: "c0", IdentityPartsColumn: "c1", RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}} + for _, mutate := range []func(q *compiler.CompiledPopulationMappingQuery){ + func(q *compiler.CompiledPopulationMappingQuery) { q.MemberColumn = "" }, + func(q *compiler.CompiledPopulationMappingQuery) { q.IdentityPartsColumn = "" }, + func(q *compiler.CompiledPopulationMappingQuery) { q.ExplicitIdentityColumn = "c2" }, + } { + compiled := base + compiled.Command = exec07MappingCommand(t) + mutate(&compiled) + engine := &Engine{compileTarget: exec07PostgreSQLTarget(t), batchSize: 1000, sourceRows: ping} + if _, err := engine.PopulationMappingCompiled(context.Background(), compiled, PopulationMappingRequest{}, exec07Members("member-1")); err == nil { + t.Fatalf("broken witness contract accepted: %#v", compiled) + } + } + if calls != 0 { + t.Fatalf("contract refusal reached the port: %d calls", calls) + } +} + +func TestPostgresPopulationMappingUnknownBackendIsHardError(t *testing.T) { + engine := &Engine{compileTarget: exec07PostgreSQLTarget(t), batchSize: 1000, sourceRows: func(context.Context, RowsRequest, RowVisitor) error { return nil }} + _, err := engine.PopulationMappingCompiled(context.Background(), compiler.CompiledPopulationMappingQuery{ + MemberColumn: "c0", IdentityPartsColumn: "c1", RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, + }, PopulationMappingRequest{}, exec07Members("member-1")) + if err == nil || !strings.Contains(err.Error(), "unsupported backend") { + t.Fatalf("untagged command error = %v", err) + } +} + +func exec07RouteBundle(name string) recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: name, + TranslationVersion: "exec07", + Outputs: []recipe.Output{{ + Name: "specimens", RootResourceType: "Specimen", RowGrain: "specimen", + Fields: []recipe.Field{{Name: "id", ColumnID: "specimen_id", Expr: recipe.Expression{Select: "root.id"}}}, + Population: &recipe.PopulationConstraint{ + SelectionRevisionID: "sel-exec07", MembershipDigest: "sha256:exec07-members", MemberCount: 3, + ResourceType: "DocumentReference", + Route: []recipe.PopulationRouteStep{{ResourceType: "DocumentReference", Relationship: "subject_Specimen"}}, + }, + }}, + } +} + +func exec07FlatBundle(name string) recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: name, + TranslationVersion: "exec07", + Outputs: []recipe.Output{{ + Name: "observations", RootResourceType: "Observation", RowGrain: "observation", + Fields: []recipe.Field{{Name: "status", ColumnID: "status_id", Expr: recipe.Expression{Select: "root.status"}}}, + Population: &recipe.PopulationConstraint{ + SelectionRevisionID: "sel-exec07", MembershipDigest: "sha256:exec07-members", MemberCount: 3, + ResourceType: "Observation", + }, + }}, + } +} + +func exec07Bindings(project, generation string) recipe.RuntimeBindings { + return recipe.RuntimeBindings{ + Project: project, DatasetGeneration: generation, + SelectionProject: "project-a", + SelectionMembersCollection: "loom_explorer_selection_members", + AuthResourcePaths: []string{"FHIR/Observation", "FHIR/Specimen", "FHIR/DocumentReference"}, + } +} + +func exec07Engine(t *testing.T, port SourceRows) *Engine { + t.Helper() + config := (&dispatchSpies{}).arangoEngine() + config.CompileTarget = exec07PostgreSQLTarget(t) + config.SourceRows = port + return config.newEngineForTest(t) +} + +func TestPostgresPopulationMappingEngineDispatchesFlatWitnessTerminal(t *testing.T) { + var portCalls int + var requests []RowsRequest + engine := exec07Engine(t, func(_ context.Context, request RowsRequest, visit RowVisitor) error { + if _, _, err := request.Command.PostgreSQLParameters(); err != nil { + return err + } + portCalls++ + requests = append(requests, request) + for _, row := range []map[string]any{ + {ir.PhysicalPopulationMappingMemberField: "obs-1", ir.PhysicalPopulationMappingIdentityPartsField: []any{"project-a", "obs-1"}}, + {ir.PhysicalPopulationMappingMemberField: "obs-2", ir.PhysicalPopulationMappingIdentityPartsField: []any{"project-a", "obs-2"}}, + } { + if err := visit(row); err != nil { + return err + } + } + return nil + }) + resolved, err := engine.CompileResolvedBundle(context.Background(), exec07FlatBundle("exec07-flat"), exec07Bindings("project-a", "gen-a")) + if err != nil { + t.Fatalf("CompileResolvedBundle: %v", err) + } + result, err := engine.PopulationMapping(context.Background(), resolved, PopulationMappingRequest{Output: "observations", MaxUnmapped: 10}, exec07Members("obs-1", "obs-2", "obs-3")) + if err != nil { + t.Fatal(err) + } + if result.Status != PopulationMappingComplete || result.SelectedCount != 3 || result.MappedCount != 2 || result.UnmappedCount != 1 || result.EmittedRows != 2 { + t.Fatalf("flat mapping result = %#v", result) + } + if len(result.UnmappedMemberIDs) != 1 || result.UnmappedMemberIDs[0] != "obs-3" { + t.Fatalf("unmapped member IDs = %#v", result.UnmappedMemberIDs) + } + if portCalls != 1 || len(requests) != 1 { + t.Fatalf("port calls = %d", portCalls) + } + if got := requests[0].DeclaredColumns; len(got) != 2 || + got[0] != ir.PhysicalPopulationMappingMemberField || + got[1] != ir.PhysicalPopulationMappingIdentityPartsField { + t.Fatalf("engine-declared witness contract = %#v, want the compiler-owned member and identity-parts fields", got) + } +} + +func TestPostgresPopulationMappingNamedCapabilityGapRefusesAtZeroStatements(t *testing.T) { + expansion := exec07FlatBundle("exec07-expand") + expansion.Outputs[0].Expand = &recipe.Expansion{From: recipe.Expression{Select: "root.extension[]"}, As: "item"} + expansion.Outputs[0].RowGrain = "expanded" + expansion.Outputs[0].Identity = &recipe.Identity{Name: "row", Expr: recipe.Expression{Select: "root.id"}} + for _, test := range []struct { + name string + bundle recipe.Bundle + output string + }{ + {"expansion on the mapping root", expansion, "observations"}, + {"endpoint-lookup route", exec07RouteBundle("exec07-route"), "specimens"}, + } { + t.Run(test.name, func(t *testing.T) { + var portCalls int + engine := exec07Engine(t, func(context.Context, RowsRequest, RowVisitor) error { + portCalls++ + return nil + }) + resolved, err := engine.CompileResolvedBundle(context.Background(), test.bundle, exec07Bindings("project-a", "gen-a")) + if err != nil { + t.Fatalf("CompileResolvedBundle: %v", err) + } + result, err := engine.PopulationMapping(context.Background(), resolved, PopulationMappingRequest{Output: test.output, MaxUnmapped: 10}, exec07Members("obs-1", "obs-2", "obs-3")) + if !errors.Is(err, ErrPopulationMappingPostgresUnsupported) { + t.Fatalf("error = %v, want ErrPopulationMappingPostgresUnsupported", err) + } + if result.Status != "" || result.SelectedCount != 0 || result.MappedCount != 0 { + t.Fatalf("capability refusal exposed counts: %#v", result) + } + if !strings.Contains(err.Error(), "unsupported physical plan") { + t.Fatalf("refusal lost the renderer's typed gap evidence: %v", err) + } + if portCalls != 0 { + t.Fatalf("refusal executed %d statements, want the statement-count-0 witness", portCalls) + } + }) + } +} diff --git a/internal/dataframe/execution/postgres_explain_parser_internal_test.go b/internal/dataframe/execution/postgres_explain_parser_internal_test.go new file mode 100644 index 000000000..fa09ad5e2 --- /dev/null +++ b/internal/dataframe/execution/postgres_explain_parser_internal_test.go @@ -0,0 +1,128 @@ +package execution + +import ( + "encoding/json" + "strings" + "testing" +) + +const ( + parserSentinelTable = "pat_SENTINEL-TBL-9c14" + parserSentinelColumn = "note_SENTINEL-COL-7a51" + parserSentinelBind = "SENTINEL-BIND-13ad" +) + +func parserClip(value string) string { + if len(value) > 200 { + return value[:200] + "…" + } + return value +} + +func assertParserSentinelFree(t *testing.T, surface string, value string) { + t.Helper() + for _, sentinel := range []string{parserSentinelTable, parserSentinelColumn, parserSentinelBind} { + if strings.Contains(value, sentinel) { + t.Fatalf("%s leaks hostile sentinel %q", surface, sentinel) + } + } +} + +func TestPostgresExplainSanitizerClassifiesSequentialScan(t *testing.T) { + raw := json.RawMessage(`[{"Plan":{"Node Type":"Limit","Startup Cost":0,"Total Cost":1.5,"Plan Rows":1,"Plan Width":8,"Plans":[{"Node Type":"Seq Scan","Relation Name":"` + parserSentinelTable + `","Filter":"(` + parserSentinelColumn + ` = '` + parserSentinelBind + `')","Startup Cost":0,"Total Cost":1.4,"Plan Rows":1,"Plan Width":8}]}}]`) + assessment, err := sanitizePostgreSQLPlanJSON(raw) + if err != nil { + t.Fatalf("sanitize: %v", err) + } + if len(assessment.FullScans) != 1 || assessment.FullScans[0] != (PostgreSQLScanEvidence{Ordinal: 1, NodeType: "Seq Scan", ScanKind: PostgreSQLScanSequential}) { + t.Fatalf("sequential classification: %+v", assessment.FullScans) + } + if len(assessment.IndexScans) != 0 { + t.Fatalf("unexpected index evidence: %+v", assessment.IndexScans) + } + benign, err := sanitizePostgreSQLPlanJSON(json.RawMessage(`[{"Plan":{"Node Type":"Result","Startup Cost":0,"Total Cost":0.01,"Plan Rows":1,"Plan Width":4}}]`)) + if err != nil || len(benign.FullScans) != 0 || len(benign.IndexScans) != 0 { + t.Fatalf("zero-scan control: %+v err=%v", benign, err) + } + rendered, _ := json.Marshal(assessment) + assertParserSentinelFree(t, "sequential assessment JSON", string(rendered)) +} + +func TestPostgresExplainSanitizerRejectsMalformedPlans(t *testing.T) { + nestedChain := func(levels int) string { + var builder strings.Builder + for i := 0; i < levels; i++ { + builder.WriteString(`{"Node Type":"Nested Loop","Startup Cost":0,"Total Cost":1,"Plan Rows":1,"Plan Width":4,"Plans":[`) + } + builder.WriteString(`{"Node Type":"Result","Startup Cost":0,"Total Cost":0,"Plan Rows":1,"Plan Width":4}`) + builder.WriteString(strings.Repeat(`]}`, levels)) + return builder.String() + } + deepNode := nestedChain(postgresExplainMaxDepth + 2) + wideNodes := func(count int) string { + var builder strings.Builder + builder.WriteString(`{"Node Type":"Append","Startup Cost":0,"Total Cost":1,"Plan Rows":1,"Plan Width":4,"Plans":[`) + for i := 0; i < count; i++ { + if i > 0 { + builder.WriteString(",") + } + builder.WriteString(`{"Node Type":"Result","Startup Cost":0,"Total Cost":0,"Plan Rows":1,"Plan Width":4}`) + } + builder.WriteString(`]}`) + return builder.String() + } + tests := []struct { + name string + raw string + }{ + {name: "empty", raw: ``}, + {name: "not json", raw: `EXPLAIN`}, + {name: "object not array", raw: `{"Plan":{"Node Type":"Result","Startup Cost":0,"Total Cost":0,"Plan Rows":1,"Plan Width":4}}`}, + {name: "empty array", raw: `[]`}, + {name: "two statements", raw: `[{"Plan":{"Node Type":"Result","Startup Cost":0,"Total Cost":0,"Plan Rows":1,"Plan Width":4}},{"Plan":{"Node Type":"Result","Startup Cost":0,"Total Cost":0,"Plan Rows":1,"Plan Width":4}}]`}, + {name: "no plan object", raw: `[{"NotPlan":{}}]`}, + {name: "missing node type", raw: `[{"Plan":{"Startup Cost":0,"Total Cost":0,"Plan Rows":1,"Plan Width":4}}]`}, + {name: "missing estimate", raw: `[{"Plan":{"Node Type":"Result","Total Cost":0,"Plan Rows":1,"Plan Width":4}}]`}, + {name: "estimate as string", raw: `[{"Plan":{"Node Type":"Result","Startup Cost":"0","Total Cost":0,"Plan Rows":1,"Plan Width":4}}]`}, + {name: "fractional plan width", raw: `[{"Plan":{"Node Type":"Result","Startup Cost":0,"Total Cost":0,"Plan Rows":1,"Plan Width":4.5}}]`}, + {name: "negative estimate", raw: `[{"Plan":{"Node Type":"Result","Startup Cost":-1,"Total Cost":0,"Plan Rows":1,"Plan Width":4}}]`}, + {name: "node type with quotes", raw: `[{"Plan":{"Node Type":"Seq Scan' OR 1=1 --","Startup Cost":0,"Total Cost":0,"Plan Rows":1,"Plan Width":4}}]`}, + {name: "node type with parens", raw: `[{"Plan":{"Node Type":"Index Scan (secret_idx)","Startup Cost":0,"Total Cost":0,"Plan Rows":1,"Plan Width":4}}]`}, + {name: "node type with dollar sign", raw: `[{"Plan":{"Node Type":"Result $1","Startup Cost":0,"Total Cost":0,"Plan Rows":1,"Plan Width":4}}]`}, + {name: "node type empty", raw: `[{"Plan":{"Node Type":"","Startup Cost":0,"Total Cost":0,"Plan Rows":1,"Plan Width":4}}]`}, + {name: "node type leading digit", raw: `[{"Plan":{"Node Type":"9Result","Startup Cost":0,"Total Cost":0,"Plan Rows":1,"Plan Width":4}}]`}, + {name: "trailing document", raw: `[{"Plan":{"Node Type":"Result","Startup Cost":0,"Total Cost":0,"Plan Rows":1,"Plan Width":4}}] []`}, + {name: "depth overflow", raw: `[{"Plan":` + deepNode + `}]`}, + {name: "node overflow", raw: `[{"Plan":` + wideNodes(postgresExplainMaxNodes+1) + `}]`}, + {name: "nan literal", raw: `[{"Plan":{"Node Type":"Result","Startup Cost":NaN,"Total Cost":0,"Plan Rows":1,"Plan Width":4}}]`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := sanitizePostgreSQLPlanJSON(json.RawMessage(test.raw)); err == nil { + t.Fatalf("malformed plan accepted: %s", parserClip(test.raw)) + } + }) + } + if _, err := sanitizePostgreSQLPlanJSON(json.RawMessage(`[{"Plan":{"Node Type":"Result","Startup Cost":0,"Total Cost":0,"Plan Rows":1,"Plan Width":4}}]`)); err != nil { + t.Fatalf("benign minimal plan refused: %v", err) + } + if _, err := sanitizePostgreSQLPlanJSON(json.RawMessage(strings.Repeat(" ", postgresExplainMaxRawBytes+1))); err == nil { + t.Fatal("oversized plan JSON accepted") + } + if _, err := sanitizePostgreSQLPlanJSON(json.RawMessage(`[` + strings.Repeat(" ", postgresExplainMaxRawBytes-16) + `]`)); err == nil { + _ = err + } +} + +func TestPostgresExplainPlanNodeTypeVocabularyAcceptsRealOperators(t *testing.T) { + for _, nodeType := range []string{"Seq Scan", "Index Scan", "Index Only Scan", "Bitmap Index Scan", "Bitmap Heap Scan", "Hash Join", "Nested Loop", "Merge Join", "WindowAgg", "LockRows", "Tid Range Scan", "Shared Scan", "Limit", "Sort", "Aggregate", "Group Aggregate", "Gather Merge", "Memoize", "Sample Scan"} { + raw := json.RawMessage(`[{"Plan":{"Node Type":"` + nodeType + `","Startup Cost":0,"Total Cost":0,"Plan Rows":1,"Plan Width":4}}]`) + assessment, err := sanitizePostgreSQLPlanJSON(raw) + if err != nil { + t.Fatalf("real operator %q refused: %v", nodeType, err) + } + if assessment.Nodes[0].NodeType != nodeType { + t.Fatalf("operator %q mangled to %q", nodeType, assessment.Nodes[0].NodeType) + } + } +} diff --git a/internal/dataframe/execution/postgres_target_fixture_test.go b/internal/dataframe/execution/postgres_target_fixture_test.go new file mode 100644 index 000000000..2f2c46d66 --- /dev/null +++ b/internal/dataframe/execution/postgres_target_fixture_test.go @@ -0,0 +1,84 @@ +package execution + +import ( + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" + "github.com/calypr/loom/internal/dataframe/spec" +) + +const verifyArch01PGSchema = "ab11_engine" + +func verifyArch01PostgreSQLTarget(t *testing.T) compiler.Target { + t.Helper() + binding, err := ir.NewPostgresStorageBinding(verifyArch01PGSchema) + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("PostgreSQLTarget: %v", err) + } + return target +} + +func verifyArch01PostgreSQLResolved(outputName string, columns []string) Resolved { + binds := map[string]any{ + "root_collection": "Patient", + "project": "P1", + "dataset_generation": "G1", + "auth_resource_paths": []string{"FHIR/Patient"}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + } + projections := make([]ir.PhysicalProjection, 0, len(columns)) + for _, column := range columns { + projections = append(projections, ir.PhysicalProjection{ + Name: column, + Value: ir.PhysicalValue{Variable: "root", Path: []string{"logical_id"}}, + }) + } + plan := ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: binds, + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"project"}}, Right: &ir.PhysicalValue{BindKey: "project"}}}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"dataset_generation"}}, Right: &ir.PhysicalValue{BindKey: "dataset_generation"}}}}, + {Kind: ir.PhysicalDerivedLetOp, DerivedLet: &ir.PhysicalDerivedLet{Variable: "root_auth_allowed", Operator: "AUTH_RESOURCE_PATH_ALLOWED", Inputs: []ir.PhysicalValue{ + {Variable: "root", Path: []string{"auth_resource_path"}}, + {BindKey: "auth_resource_paths"}, + {BindKey: "auth_resource_paths_unrestricted"}, + }}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root_auth_allowed"}, Right: &ir.PhysicalValue{BindKey: "scope_allowed"}}}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}}, + }, + } + schema := make([]lower.CompiledOutputColumn, 0, len(columns)) + for _, column := range columns { + schema = append(schema, lower.CompiledOutputColumn{ID: column, Name: column, Label: column, Kind: "string", Cardinality: "one"}) + } + output := lower.CompiledRecipeOutput{ + Name: outputName, + RootResourceType: "Patient", + Columns: append([]string(nil), columns...), + OutputSchema: schema, + RowIdentity: &spec.RowIdentity{Fields: append([]string(nil), columns...)}, + Plan: plan, + } + return Resolved{ + Bundle: recipe.Bundle{Name: "verify_arch01"}, + Semantic: semantic.ResolvedRecipePlan{ + SemanticPlan: semantic.RecipePlan{Bindings: recipe.RuntimeBindings{ + Project: "P1", DatasetGeneration: "G1", OutputNames: []string{outputName}, + }}, + }, + Compiled: lower.CompiledRecipe{Outputs: []lower.CompiledRecipeOutput{output}}, + StoredRecipeDigest: "verify-arch01", + } +} diff --git a/internal/dataframe/execution/postquery.go b/internal/dataframe/execution/postquery.go index 00b05e91d..81f641759 100644 --- a/internal/dataframe/execution/postquery.go +++ b/internal/dataframe/execution/postquery.go @@ -17,9 +17,6 @@ const ( postQueryTargetKey = "__loom_postquery_target" ) -// DynamicDriftError means a runtime key was not part of the dynamic-column -// schema frozen from the field catalog. It carries schema metadata only, so -// transport adapters can report the mismatch without exposing row values. type DynamicDriftError struct { DynamicName string Key string @@ -53,9 +50,6 @@ func validateDynamicDrift(row map[string]any, checks map[string]map[string]Dynam if len(checks) == 0 { return nil } - // Frozen column types are compiled guarantees, so every present value is - // checked whether or not the runtime key metadata enumerates its key: the - // metadata may cover only another family. for dynamicName, columns := range checks { for _, column := range columns { if actual, exists := row[column.ColumnName]; exists && !dynamicColumnValueMatches(actual, column) { @@ -65,8 +59,6 @@ func validateDynamicDrift(row map[string]any, checks map[string]map[string]Dynam } observed, ok := row["__loom_dynamic_runtime_keys"].(map[string]any) if !ok { - // Construction stages may drop the metadata only for families that - // explicitly allow unknown keys. for dynamicName, columns := range checks { for _, column := range columns { if !column.AllowUnknownKeys { @@ -233,6 +225,15 @@ func materializePostQueryValue(value any) (any, error) { result[index] = resolved } return result, nil + case json.RawMessage: + if len(typed) == 0 || !json.Valid(typed) { + return nil, fmt.Errorf("post-query JSONB value is not valid JSON") + } + var decoded any + if err := json.Unmarshal(typed, &decoded); err != nil { + return nil, fmt.Errorf("post-query JSONB value cannot be decoded: %w", err) + } + return materializePostQueryValue(decoded) default: return value, nil } diff --git a/internal/dataframe/execution/postquery_test.go b/internal/dataframe/execution/postquery_test.go index b017fd7a4..bda93a3b3 100644 --- a/internal/dataframe/execution/postquery_test.go +++ b/internal/dataframe/execution/postquery_test.go @@ -2,7 +2,10 @@ package execution import ( "context" + "encoding/json" "errors" + "reflect" + "strings" "testing" "github.com/calypr/loom/internal/dataframe/compiler/lower" @@ -60,8 +63,6 @@ func TestMaterializePostQueryValidatesEveryPresentColumnWhenRuntimeKeysCoverAnot "tags": {ColumnName: "identifier_tags", ValueType: "string", Many: true}, }, } - // Runtime key metadata enumerates only family A; family B columns are still - // present in the row and must satisfy their frozen types. newRow := func(dynamicValues map[string]any) map[string]any { row := map[string]any{ "category_stage": "stage-a", @@ -88,7 +89,6 @@ func TestMaterializePostQueryValidatesEveryPresentColumnWhenRuntimeKeysCoverAnot if _, err := materializePostQueryRowWithChecks(valid, checks); err != nil { t.Fatalf("valid row across both families rejected: %v", err) } - // An unselected family B column stays valid because only present values bind. if _, err := materializePostQueryRowWithChecks(newRow(map[string]any{"identifier_count": 7}), checks); err != nil { t.Fatalf("valid row with a missing unselected column rejected: %v", err) } @@ -112,11 +112,18 @@ func TestOutputStreamRejectsMalformedRowFromUnenumeratedDynamicFamily(t *testing return OutputStream{ Name: "Condition", Columns: []string{"condition", "category_stage", "identifier_count"}, + command: mustAQLCommand("FOR root IN Condition RETURN root", nil), DynamicChecks: map[string]map[string]DynamicColumnCheck{ "category_keys": {"stage": {ColumnName: "category_stage", ValueType: "string"}}, "identifier_keys": {"count": {ColumnName: "identifier_count", ValueType: "integer"}}, }, - stream: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if err := visit(rows[index]); err != nil { return err } @@ -253,3 +260,363 @@ func TestMaterializePostQueryStripsDynamicMetadata(t *testing.T) { t.Fatal("dynamic metadata leaked into logical row") } } + +const ( + exec03DNSNamespace = "6ba7b810-9dad-11d1-80b4-00c04fd430c8" + exec03UUID5Wire = `{"__loom_exact_uuid_operation":"uuid5","__loom_exact_uuid_args":["6ba7b810-9dad-11d1-80b4-00c04fd430c8","www.widgets.com"]}` + exec03UUID3Wire = `{"__loom_exact_uuid_operation":"uuid3","__loom_exact_uuid_args":["6ba7b810-9dad-11d1-80b4-00c04fd430c8","www.widgets.com"]}` + exec03UUID5CallWire = `{"__loom_postquery_call":"uuid5","__loom_postquery_args":["6ba7b810-9dad-11d1-80b4-00c04fd430c8","www.widgets.com"]}` + exec03UUID5Literal = "21f7f8de-8051-5b89-8680-0195ef798b6a" + exec03UUID3Literal = "3d813cbb-47fb-32ba-91df-831e1593ac29" +) + +func TestMaterializePostQueryJSONBMarkerComputesNamedUUID(t *testing.T) { + for _, tc := range []struct { + name string + wire string + want string + }{ + {"exact uuid5 marker", exec03UUID5Wire, exec03UUID5Literal}, + {"exact uuid3 marker", exec03UUID3Wire, exec03UUID3Literal}, + {"post-query call marker", exec03UUID5CallWire, exec03UUID5Literal}, + {"cast call keeps target", `{"__loom_postquery_call":"cast","__loom_postquery_target":"string","__loom_postquery_args":[42]}`, "42"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := materializePostQueryValue(json.RawMessage(tc.wire)) + if err != nil { + t.Fatalf("materialize: %v", err) + } + if got != tc.want { + t.Fatalf("materialized %#v, want %#v", got, tc.want) + } + if _, ok := got.(string); !ok { + t.Fatalf("materialized value has Go type %T, want string", got) + } + }) + } + if _, err := materializePostQueryValue(json.RawMessage(`{"__loom_exact_uuid_operation":"uuid5","__loom_exact_uuid_args":"nope"}`)); err == nil || !strings.Contains(err.Error(), "malformed") { + t.Fatalf("non-array exact-UUID args accepted: %v", err) + } + if _, err := materializePostQueryValue(json.RawMessage(`{"__loom_postquery_call":"concat","__loom_postquery_args":{}}`)); err == nil || !strings.Contains(err.Error(), "malformed") { + t.Fatalf("non-array post-query args accepted: %v", err) + } +} + +func TestMaterializePostQueryPGShapeMatchesDriverDecodedShape(t *testing.T) { + pgRow := map[string]any{ + "marker": json.RawMessage(exec03UUID5Wire), + "nested": json.RawMessage(`{"inner":` + exec03UUID3Wire + `}`), + "list": json.RawMessage(`[` + exec03UUID5CallWire + `,null,1.5]`), + "flag": json.RawMessage(`true`), + "name": json.RawMessage(`"w"`), + "count": json.RawMessage(`2`), + } + driverWire := `{"marker":` + exec03UUID5Wire + `,"nested":{"inner":` + exec03UUID3Wire + `},"list":[` + exec03UUID5CallWire + `,null,1.5],"flag":true,"name":"w","count":2}` + var driverRow map[string]any + if err := json.Unmarshal([]byte(driverWire), &driverRow); err != nil { + t.Fatalf("driver-shape decode: %v", err) + } + pgResult, err := materializePostQueryRowWithChecks(pgRow, nil) + if err != nil { + t.Fatalf("materialize PG shape: %v", err) + } + driverResult, err := materializePostQueryRowWithChecks(driverRow, nil) + if err != nil { + t.Fatalf("materialize driver shape: %v", err) + } + if !reflect.DeepEqual(pgResult, driverResult) { + t.Fatalf("PG shape %v != driver shape %v", pgResult, driverResult) + } + want := map[string]any{ + "marker": exec03UUID5Literal, + "nested": map[string]any{"inner": exec03UUID3Literal}, + "list": []any{exec03UUID5Literal, nil, 1.5}, + "flag": true, + "name": "w", + "count": float64(2), + } + if !reflect.DeepEqual(pgResult, want) { + t.Fatalf("materialized PG row = %v, want %v", pgResult, want) + } + if reflect.DeepEqual(map[string]any(pgRow), driverRow) { + t.Fatal("discriminating control broken: unmaterialized RawMessage row equals the decoded driver row") + } +} + +func TestMaterializePostQueryCrossTargetJSONNullEquivalence(t *testing.T) { + const ( + genderWire = `null` + codeWire = `"C50.9"` + countWire = `7` + tagsWire = `["a",null]` + flagWire = `true` + ) + wantPublic := map[string]any{ + "gender": nil, + "code": "C50.9", + "count": float64(7), + "tags": []any{"a", nil}, + "flag": true, + } + type leg struct { + name string + row map[string]any + } + postgresLeg := map[string]any{ + "gender": json.RawMessage(genderWire), + "code": json.RawMessage(codeWire), + "count": json.RawMessage(countWire), + "tags": json.RawMessage(tagsWire), + "flag": json.RawMessage(flagWire), + } + arangoLeg := map[string]any{} + for column, wire := range map[string]string{"gender": genderWire, "code": codeWire, "count": countWire, "tags": tagsWire, "flag": flagWire} { + var value any + if err := json.Unmarshal([]byte(wire), &value); err != nil { + t.Fatalf("arango-shape decode %s: %v", column, err) + } + arangoLeg[column] = value + } + var clickHouseValues []any + if err := json.Unmarshal([]byte(`[`+genderWire+","+codeWire+","+countWire+","+tagsWire+","+flagWire+`]`), &clickHouseValues); err != nil { + t.Fatalf("clickhouse-shape decode: %v", err) + } + clickHouseLeg := map[string]any{} + for i, column := range []string{"gender", "code", "count", "tags", "flag"} { + clickHouseLeg[column] = clickHouseValues[i] + } + checks := map[string]map[string]DynamicColumnCheck{ + "code": {"C50.9": {ColumnName: "code", ValueType: "string", AllowUnknownKeys: true}}, + } + legs := []leg{{"postgres", postgresLeg}, {"arangodb", arangoLeg}, {"clickhouse", clickHouseLeg}} + results := map[string]map[string]any{} + for _, entry := range legs { + row, err := materializePostQueryRowWithChecks(entry.row, checks) + if err != nil { + t.Fatalf("%s leg: %v", entry.name, err) + } + results[entry.name] = row + if !reflect.DeepEqual(row, wantPublic) { + t.Fatalf("%s leg public row = %v, want %v", entry.name, row, wantPublic) + } + if value, ok := row["gender"]; !ok || value != nil { + t.Fatalf("%s leg: JSON-null column public value is %#v (Go %T), want untyped nil", entry.name, value, value) + } + if _, ok := row["code"].(string); !ok { + t.Fatalf("%s leg: code column has Go type %T, want string", entry.name, row["code"]) + } + if _, ok := row["count"].(float64); !ok { + t.Fatalf("%s leg: count column has Go type %T, want float64", entry.name, row["count"]) + } + } + for _, pair := range [][2]string{{"postgres", "arangodb"}, {"postgres", "clickhouse"}, {"arangodb", "clickhouse"}} { + if !reflect.DeepEqual(results[pair[0]], results[pair[1]]) { + t.Fatalf("cross-target convergence broken: %s %v != %s %v", pair[0], results[pair[0]], pair[1], results[pair[1]]) + } + } + aliveWires := map[string]string{"gender": `"female"`, "code": `"C50.9"`, "count": countWire, "tags": tagsWire, "flag": flagWire} + for _, entry := range legs { + row := map[string]any{} + switch entry.name { + case "postgres": + for column, wire := range aliveWires { + row[column] = json.RawMessage(wire) + } + case "arangodb": + for column, wire := range aliveWires { + var value any + if err := json.Unmarshal([]byte(wire), &value); err != nil { + t.Fatalf("alive arango decode: %v", err) + } + row[column] = value + } + case "clickhouse": + var values []any + if err := json.Unmarshal([]byte(`["female","C50.9",7,["a",null],true]`), &values); err != nil { + t.Fatalf("alive clickhouse decode: %v", err) + } + for i, column := range []string{"gender", "code", "count", "tags", "flag"} { + row[column] = values[i] + } + } + materialized, err := materializePostQueryRowWithChecks(row, checks) + if err != nil { + t.Fatalf("%s alive control: %v", entry.name, err) + } + if materialized["gender"] != "female" { + t.Fatalf("%s alive control: gender = %#v, want \"female\"", entry.name, materialized["gender"]) + } + } + if pre := postgresLeg["gender"]; pre == nil { + t.Fatal("discriminating control broken: adapter-shape gender column is already nil before materialization, so the nil convergence below is vacuous") + } else if _, isRaw := pre.(json.RawMessage); !isRaw { + t.Fatalf("adapter-shape gender column is %T, want json.RawMessage", pre) + } +} + +func TestMaterializePostQueryJSONBNullsScalarsAndBytes(t *testing.T) { + got, err := materializePostQueryValue(json.RawMessage(`null`)) + if err != nil || got != nil { + t.Fatalf("JSON null scalar materialized to %#v (err %v), want untyped nil", got, err) + } + for _, tc := range []struct { + wire string + want any + }{ + {`"C50"`, "C50"}, + {`7`, float64(7)}, + {`true`, true}, + } { + got, err := materializePostQueryValue(json.RawMessage(tc.wire)) + if err != nil || !reflect.DeepEqual(got, tc.want) { + t.Fatalf("scalar %s materialized to %#v (err %v), want %#v", tc.wire, got, err, tc.want) + } + } + opaque := []byte(`"C50"`) + got, err = materializePostQueryValue(opaque) + if err != nil { + t.Fatalf("bytea passthrough: %v", err) + } + if bytes, ok := got.([]byte); !ok || string(bytes) != `"C50"` { + t.Fatalf("bytea value materialized to %#v (Go %T), want unchanged []byte", got, got) + } + got, err = materializePostQueryValue(json.RawMessage(nil)) + if err == nil || got != nil { + t.Fatalf("typed-nil RawMessage materialized to %#v (err %v), want refusal", got, err) + } +} + +func TestMaterializePostQueryJSONBMalformedFailsClosed(t *testing.T) { + for _, wire := range []string{"", " ", "{", "[1,", `"unterminated`, `{"a":1}}`, `nope`, `{"a":}`, `1x`, `{"__loom_exact_uuid_operation":`} { + got, err := materializePostQueryValue(json.RawMessage(wire)) + if err == nil { + t.Fatalf("malformed JSONB %q accepted with value %#v", wire, got) + } + if got != nil { + t.Fatalf("malformed JSONB %q refused but leaked value %#v", wire, got) + } + if _, err := materializePostQueryRowWithChecks(map[string]any{"col": json.RawMessage(wire)}, nil); err == nil { + t.Fatalf("row with malformed JSONB column %q published", wire) + } + } + if _, err := materializePostQueryValue(json.RawMessage(`{"a":1}`)); err != nil { + t.Fatalf("alive control rejected: %v", err) + } +} + +func TestMaterializePostQueryJSONBDeepNestingBounds(t *testing.T) { + deep := func(levels int) string { + return strings.Repeat("[", levels) + "1" + strings.Repeat("]", levels) + } + for _, levels := range []int{200, 10000} { + got, err := materializePostQueryValue(json.RawMessage(deep(levels))) + if err != nil { + t.Fatalf("%d-level nesting refused: %v", levels, err) + } + value := got + for depth := 0; depth < levels; depth++ { + items, ok := value.([]any) + if !ok || len(items) != 1 { + t.Fatalf("%d-level nesting truncated at depth %d: %#v", levels, depth, value) + } + value = items[0] + } + if value != float64(1) { + t.Fatalf("%d-level nesting leaf = %#v, want 1.0", levels, value) + } + } + for _, levels := range []int{10001, 20000} { + got, err := materializePostQueryValue(json.RawMessage(deep(levels))) + if err == nil { + t.Fatalf("%d-level nesting accepted without refusal", levels) + } + if got != nil { + t.Fatalf("%d-level nesting refused but leaked a value", levels) + } + } + wire := `{"a":{"b":[{"col":` + exec03UUID5Wire + `}]}}` + got, err := materializePostQueryValue(json.RawMessage(wire)) + if err != nil { + t.Fatalf("nested marker: %v", err) + } + outer, ok := got.(map[string]any) + if !ok { + t.Fatalf("nested marker top level = %#v", got) + } + levelA, ok := outer["a"].(map[string]any) + if !ok { + t.Fatalf("nested marker level a = %#v", outer["a"]) + } + items, ok := levelA["b"].([]any) + if !ok || len(items) != 1 { + t.Fatalf("nested marker level b = %#v", levelA["b"]) + } + item, ok := items[0].(map[string]any) + if !ok || item["col"] != exec03UUID5Literal { + t.Fatalf("nested marker value = %#v, want %s", items[0], exec03UUID5Literal) + } + got, err = materializePostQueryValue([]any{json.RawMessage(exec03UUID5Wire), "x"}) + if err != nil { + t.Fatalf("RawMessage inside array: %v", err) + } + listItems, isList := got.([]any) + if !isList || len(listItems) != 2 || listItems[0] != exec03UUID5Literal || listItems[1] != "x" { + t.Fatalf("RawMessage inside array materialized to %#v", got) + } +} + +func TestMaterializePostQueryJSONBDynamicMetadataAndChecks(t *testing.T) { + checks := map[string]map[string]DynamicColumnCheck{ + "code": {"a": {ColumnName: "code_a", ValueType: "string"}}, + "identifier_keys": { + "count": {ColumnName: "identifier_count", ValueType: "integer"}, + "tags": {ColumnName: "identifier_tags", ValueType: "string", Many: true}, + }, + } + metadataWire := `{"code":["a"],"identifier_keys":["count","tags"]}` + row, err := materializePostQueryRowWithChecks(map[string]any{ + "code_a": json.RawMessage(`"value"`), + "identifier_count": json.RawMessage(`7`), + "identifier_tags": json.RawMessage(`["a","b"]`), + "__loom_dynamic_runtime_keys": json.RawMessage(metadataWire), + }, checks) + if err != nil { + t.Fatalf("conforming PG-shaped dynamic row refused: %v", err) + } + if row["code_a"] != "value" || row["identifier_count"] != float64(7) || !reflect.DeepEqual(row["identifier_tags"], []any{"a", "b"}) { + t.Fatalf("public PG-shaped row = %v", row) + } + if _, ok := row["__loom_dynamic_runtime_keys"]; ok { + t.Fatal("JSONB dynamic metadata leaked into the public row") + } + if _, err := materializePostQueryRowWithChecks(map[string]any{ + "code_a": json.RawMessage(`"value"`), + "__loom_dynamic_runtime_keys": json.RawMessage(`{"code":["a","unexpected"]}`), + }, checks); err == nil { + t.Fatal("JSONB key drift accepted") + } else { + var drift *DynamicDriftError + if !errors.As(err, &drift) || drift.Key != "unexpected" { + t.Fatalf("JSONB key drift error = %v, want DynamicDriftError for unexpected", err) + } + } + for name, row := range map[string]map[string]any{ + "string column with number": {"code_a": json.RawMessage(`42`), "__loom_dynamic_runtime_keys": json.RawMessage(metadataWire)}, + "integer column with fraction": {"identifier_count": json.RawMessage(`7.5`), "__loom_dynamic_runtime_keys": json.RawMessage(metadataWire)}, + "array column with item drift": {"identifier_tags": json.RawMessage(`["a",42]`), "__loom_dynamic_runtime_keys": json.RawMessage(metadataWire)}, + "array column from bare string": {"identifier_tags": json.RawMessage(`"a"`), "__loom_dynamic_runtime_keys": json.RawMessage(metadataWire)}, + } { + if _, err := materializePostQueryRowWithChecks(row, checks); err == nil { + t.Fatalf("%s accepted", name) + } + } + if _, err := materializePostQueryRowWithChecks(map[string]any{ + "code_a": json.RawMessage(`"value"`), + "__loom_dynamic_runtime_keys": json.RawMessage(`{"code":"a"}`), + }, checks); err == nil { + t.Fatal("malformed JSONB metadata accepted as absent") + } + if _, err := materializePostQueryRowWithChecks(map[string]any{"code_a": json.RawMessage(`"value"`)}, checks); err == nil { + t.Fatal("missing metadata accepted for a strict family") + } +} diff --git a/internal/dataframe/execution/preview_capabilities.go b/internal/dataframe/execution/preview_capabilities.go new file mode 100644 index 000000000..060de22de --- /dev/null +++ b/internal/dataframe/execution/preview_capabilities.go @@ -0,0 +1,34 @@ +package execution + +import ( + "context" + + "github.com/calypr/loom/internal/dataframe/compiler/command" +) + +type PreviewCollectionIndex struct { + Collection string + Name string + Fields []string + StoredValues []string +} + +type PreviewIndexCoverage uint8 + +const ( + PreviewIndexCoverageUnknown PreviewIndexCoverage = iota + PreviewIndexCoverageCovering + PreviewIndexCoverageNotCovering +) + +type PreviewIndexNodeEvidence struct { + IndexNames []string + Coverage PreviewIndexCoverage +} + +type PreviewCollectionScanEvidence struct { + FullCollectionEnumeration bool + IndexNodes []PreviewIndexNodeEvidence +} + +type PreviewCollectionScans func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) diff --git a/internal/dataframe/execution/preview_capability_boundary_test.go b/internal/dataframe/execution/preview_capability_boundary_test.go new file mode 100644 index 000000000..6735a4163 --- /dev/null +++ b/internal/dataframe/execution/preview_capability_boundary_test.go @@ -0,0 +1,501 @@ +package execution + +import ( + "context" + "errors" + "github.com/calypr/loom/internal/dataframe/compiler" + "reflect" + "strings" + "testing" + + "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/recipe" + exec "github.com/calypr/loom/internal/dataframe/recipe/exec" +) + +var forbiddenPreviewPkgPrefixes = []string{ + "github.com/calypr/loom/internal/backends/arango/internal/store", + "github.com/jackc/pgx", +} + +func isForbiddenPreviewPkg(pkgPath string) bool { + for _, prefix := range forbiddenPreviewPkgPrefixes { + if pkgPath == prefix || strings.HasPrefix(pkgPath, prefix+"/") { + return true + } + } + return false +} + +func previewContractVendorOffenders(ty reflect.Type, seen map[reflect.Type]bool, depth int) []string { + if ty == nil || depth > 6 { + return nil + } + if seen[ty] { + return nil + } + seen[ty] = true + if pkg := ty.PkgPath(); pkg != "" && isForbiddenPreviewPkg(pkg) { + return []string{ty.String()} + } + switch ty.Kind() { + case reflect.Pointer, reflect.Slice, reflect.Array, reflect.Chan: + return previewContractVendorOffenders(ty.Elem(), seen, depth+1) + case reflect.Map: + offenders := previewContractVendorOffenders(ty.Key(), seen, depth+1) + return append(offenders, previewContractVendorOffenders(ty.Elem(), seen, depth+1)...) + case reflect.Func: + var offenders []string + for i := 0; i < ty.NumIn(); i++ { + offenders = append(offenders, previewContractVendorOffenders(ty.In(i), seen, depth+1)...) + } + for i := 0; i < ty.NumOut(); i++ { + offenders = append(offenders, previewContractVendorOffenders(ty.Out(i), seen, depth+1)...) + } + return offenders + case reflect.Struct: + var offenders []string + for i := 0; i < ty.NumField(); i++ { + offenders = append(offenders, previewContractVendorOffenders(ty.Field(i).Type, seen, depth+1)...) + } + return offenders + } + return nil +} + +type previewCapabilityBoundaryFixture struct { + engine *Engine + resolved Resolved + commands []capturedAQLCommand + rows []map[string]any + summary PreviewSummary + err error +} + +type capturedAQLCommand struct { + Text string + Binds map[string]any +} + +type previewCapabilityBoundaryOptions struct { + scans PreviewCollectionScans + coveringIndexes func(context.Context, string) ([]PreviewCollectionIndex, error) + rowsFailure error + visitorFailure error + restrictedAuth bool + limit int +} + +func runPreviewCapabilityBoundary(t *testing.T, options previewCapabilityBoundaryOptions) *previewCapabilityBoundaryFixture { + t.Helper() + fixture := &previewCapabilityBoundaryFixture{} + limit := options.limit + if limit == 0 { + limit = 1 + } + engine, err := New(Config{ + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "preview-capability-boundary" }, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + return dispatchErr + }, + PreviewSourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + text, binds, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + fixture.commands = append(fixture.commands, capturedAQLCommand{Text: text, Binds: binds}) + if options.rowsFailure != nil { + return options.rowsFailure + } + for _, row := range []map[string]any{ + {"status": "final", "rows": float64(2)}, + {"status": "preliminary", "rows": float64(1)}, + } { + visitor := visit + if options.visitorFailure != nil { + visitor = func(map[string]any) error { return options.visitorFailure } + } + if err := visitor(row); err != nil { + return err + } + } + return nil + }, + PreviewCollectionScans: options.scans, + PreviewCoveringIndexes: options.coveringIndexes, + PreviewCollectionRevision: func(context.Context, string) (string, error) { return "boundary-revision", nil }, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + fixture.engine = engine + bindings := recipe.RuntimeBindings{Project: "P1", DatasetGeneration: "G1"} + if options.restrictedAuth { + bindings.AuthScopeMode = authscope.ReadScopeRestricted + bindings.AuthResourcePaths = []string{"Observation/allowed"} + } + fixture.resolved = compileGroupPreviewCacheTestOutput(t, engine, bindings) + visitorCalls := 0 + fixture.summary, fixture.err = engine.PreviewOutput(context.Background(), fixture.resolved, + PreviewRequest{Output: "group_preview", Limit: limit}, func(row map[string]any) error { + visitorCalls++ + fixture.rows = append(fixture.rows, row) + return nil + }) + return fixture +} + +func comparablePreviewSummary(summary PreviewSummary) PreviewSummary { + summary.LoweringDuration = 0 + summary.SourceQueryDuration = 0 + return summary +} + +func TestPreviewCapabilityBoundaryContract(t *testing.T) { + t.Run("preview contracts carry no vendor plan or index types", func(t *testing.T) { + seen := map[reflect.Type]bool{} + if offenders := previewContractVendorOffenders(reflect.TypeOf(Config{CompileTarget: compiler.ArangoTarget()}), seen, 0); len(offenders) != 0 { + t.Fatalf("execution Config references vendor preview types: %v", offenders) + } + if offenders := previewContractVendorOffenders(reflect.TypeOf(Engine{}), seen, 0); len(offenders) != 0 { + t.Fatalf("execution Engine references vendor preview types: %v", offenders) + } + for _, contract := range []reflect.Type{ + reflect.TypeOf(PreviewCollectionIndex{}), + reflect.TypeOf(PreviewCollectionScanEvidence{}), + reflect.TypeOf(PreviewIndexNodeEvidence{}), + reflect.TypeOf(PreviewCollectionScans(nil)), + } { + if offenders := previewContractVendorOffenders(contract, map[reflect.Type]bool{}, 0); len(offenders) != 0 { + t.Fatalf("neutral preview contract %v references vendor types: %v", contract, offenders) + } + } + legacy := struct { + PreviewExplainQuery func(context.Context, string, map[string]any) (arangofactory.ExplainResult, error) + PreviewCoveringIndexes func(context.Context, string) ([]arangofactory.PreviewCoveringIndex, error) + }{} + offenders := previewContractVendorOffenders(reflect.TypeOf(legacy), map[reflect.Type]bool{}, 0) + if len(offenders) != 2 { + t.Fatalf("walker missed the superseded vendor shapes: offenders=%v, want two", offenders) + } + }) + + rowsOnly := runPreviewCapabilityBoundary(t, previewCapabilityBoundaryOptions{restrictedAuth: true}) + if rowsOnly.err != nil { + t.Fatalf("rows-only bounded preview: %v", rowsOnly.err) + } + if len(rowsOnly.commands) != 1 { + t.Fatalf("rows-only adapter invocations = %d, want exactly one canonical query", len(rowsOnly.commands)) + } + canonicalText := rowsOnly.commands[0].Text + if !strings.Contains(canonicalText, "COLLECT") { + t.Fatalf("rows-only fixture did not run a Group preview command:\n%s", canonicalText) + } + if strings.Contains(canonicalText, "loom_pivot_preview_boundary_covering") { + t.Fatalf("rows-only preview switched to the boundary candidate without evidence:\n%s", canonicalText) + } + if len(rowsOnly.rows) != 1 || rowsOnly.rows[0]["status"] != "final" { + t.Fatalf("rows-only bounded rows = %+v, want one bounded row", rowsOnly.rows) + } + if rowsOnly.summary.RowCount != 1 || !rowsOnly.summary.Truncated || rowsOnly.summary.Complete { + t.Fatalf("rows-only summary = %+v, want truncated bounded preview", rowsOnly.summary) + } + if rowsOnly.commands[0].Binds["auth_resource_paths"] == nil { + t.Fatalf("rows-only restricted preview dropped the authorization binds: %v", rowsOnly.commands[0].Binds) + } + + t.Run("fresh matching evidence enables the existing optimization", func(t *testing.T) { + index := PreviewCollectionIndex{ + Collection: "Observation", + Name: "loom_pivot_preview_boundary_covering", + Fields: []string{"project", "dataset_generation", "auth_resource_path", "_key", "payload.id", "payload.status"}, + StoredValues: []string{"payload.valueQuantity"}, + } + scanCalls := 0 + fixture := runPreviewCapabilityBoundary(t, previewCapabilityBoundaryOptions{ + restrictedAuth: true, + coveringIndexes: func(_ context.Context, collection string) ([]PreviewCollectionIndex, error) { + if collection != "Observation" { + t.Fatalf("index collection = %q, want Observation", collection) + } + return []PreviewCollectionIndex{index}, nil + }, + scans: func(_ context.Context, command command.CompiledCommand, collection string) (PreviewCollectionScanEvidence, error) { + scanCalls++ + if collection != "Observation" { + t.Fatalf("scan collection = %q, want Observation", collection) + } + text, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return PreviewCollectionScanEvidence{}, dispatchErr + } + if !strings.Contains(text, `indexHint: "`+index.Name+`"`) { + return PreviewCollectionScanEvidence{}, nil + } + return PreviewCollectionScanEvidence{IndexNodes: []PreviewIndexNodeEvidence{ + {IndexNames: []string{index.Name}, Coverage: PreviewIndexCoverageCovering}, + }}, nil + }, + }) + if fixture.err != nil { + t.Fatalf("evidence-armed bounded preview: %v", fixture.err) + } + if scanCalls == 0 { + t.Fatal("the plan-inspection capability was never consulted") + } + if len(fixture.commands) != 1 || !strings.Contains(fixture.commands[0].Text, `indexHint: "`+index.Name+`"`) { + t.Fatalf("fresh covering evidence did not reach the executed command: %+v", fixture.commands) + } + if len(fixture.rows) != 1 || fixture.rows[0]["status"] != "final" { + t.Fatalf("evidence-armed bounded rows = %+v, want the same bounded row", fixture.rows) + } + if !reflect.DeepEqual(fixture.commands[0].Binds, rowsOnly.commands[0].Binds) { + t.Fatalf("optimized command changed the authorization binds: %v vs %v", fixture.commands[0].Binds, rowsOnly.commands[0].Binds) + } + optimizedSummary, rowsOnlySummary := fixture.summary, rowsOnly.summary + optimizedFingerprint, rowsOnlyFingerprint := optimizedSummary.PlanFingerprint, rowsOnlySummary.PlanFingerprint + optimizedSummary.PlanFingerprint, rowsOnlySummary.PlanFingerprint = "", "" + if !reflect.DeepEqual(comparablePreviewSummary(optimizedSummary), comparablePreviewSummary(rowsOnlySummary)) { + t.Fatalf("optimized preview changed the bounded summary: %+v vs %+v", fixture.summary, rowsOnly.summary) + } + if optimizedFingerprint == "" || rowsOnlyFingerprint == "" { + t.Fatalf("preview summaries must carry a diagnostic plan fingerprint: optimized=%q rowsOnly=%q", optimizedFingerprint, rowsOnlyFingerprint) + } + if optimizedFingerprint == rowsOnlyFingerprint { + t.Fatal("the diagnostic plan fingerprint ignored the indexHint-rendered command change") + } + }) + + t.Run("absent or failed evidence takes the canonical fallback", func(t *testing.T) { + for _, test := range []struct { + name string + scans PreviewCollectionScans + coveringIndexes func(context.Context, string) ([]PreviewCollectionIndex, error) + }{ + { + name: "scan inspection fails", + scans: func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + return PreviewCollectionScanEvidence{}, errors.New("inspection unavailable") + }, + coveringIndexes: func(context.Context, string) ([]PreviewCollectionIndex, error) { + return []PreviewCollectionIndex{{ + Collection: "Observation", Name: "loom_pivot_preview_boundary_covering", + Fields: []string{"project", "dataset_generation", "auth_resource_path", "_key", "payload.id", "payload.status"}, + StoredValues: []string{"payload.valueQuantity"}, + }}, nil + }, + }, + { + name: "empty evidence certifies nothing", + scans: func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + return PreviewCollectionScanEvidence{}, nil + }, + coveringIndexes: func(context.Context, string) ([]PreviewCollectionIndex, error) { + return []PreviewCollectionIndex{{ + Collection: "Observation", Name: "loom_pivot_preview_boundary_covering", + Fields: []string{"project", "dataset_generation", "auth_resource_path", "_key", "payload.id", "payload.status"}, + StoredValues: []string{"payload.valueQuantity"}, + }}, nil + }, + }, + { + name: "index listing fails", + scans: func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + return PreviewCollectionScanEvidence{}, nil + }, + coveringIndexes: func(context.Context, string) ([]PreviewCollectionIndex, error) { + return nil, errors.New("listing unavailable") + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + fixture := runPreviewCapabilityBoundary(t, previewCapabilityBoundaryOptions{ + restrictedAuth: true, + scans: test.scans, + coveringIndexes: test.coveringIndexes, + }) + if fixture.err != nil { + t.Fatalf("advisory evidence failure must not fail the preview: %v", fixture.err) + } + if len(fixture.commands) != 1 || fixture.commands[0].Text != canonicalText { + t.Fatalf("fallback did not run the canonical command: %+v", fixture.commands) + } + if !reflect.DeepEqual(fixture.commands[0].Binds, rowsOnly.commands[0].Binds) { + t.Fatalf("fallback changed the authorization binds: %v vs %v", fixture.commands[0].Binds, rowsOnly.commands[0].Binds) + } + if len(fixture.rows) != 1 || fixture.rows[0]["status"] != "final" { + t.Fatalf("fallback bounded rows = %+v, want the same bounded row", fixture.rows) + } + if !reflect.DeepEqual(comparablePreviewSummary(fixture.summary), comparablePreviewSummary(rowsOnly.summary)) { + t.Fatalf("fallback changed the bounded summary: %+v vs %+v", fixture.summary, rowsOnly.summary) + } + }) + } + }) + + t.Run("execution errors remain errors", func(t *testing.T) { + queryFailure := errors.New("boundary query failed") + fixture := runPreviewCapabilityBoundary(t, previewCapabilityBoundaryOptions{ + restrictedAuth: true, + rowsFailure: queryFailure, + }) + if !errors.Is(fixture.err, queryFailure) { + t.Fatalf("preview error = %v, want the execution error to surface", fixture.err) + } + if len(fixture.commands) != 1 { + t.Fatalf("failed execution retried %d times; fallback must not re-run queries", len(fixture.commands)) + } + visitorFailure := errors.New("boundary visitor refused") + visitorFixture := runPreviewCapabilityBoundary(t, previewCapabilityBoundaryOptions{ + restrictedAuth: true, + visitorFailure: visitorFailure, + }) + if !errors.Is(visitorFixture.err, visitorFailure) { + t.Fatalf("visitor error = %v, want the visitor error to surface", visitorFixture.err) + } + }) + + t.Run("compile only explain works without a database", func(t *testing.T) { + aqlCalls, pgCalls := 0, 0 + engine, err := New(Config{ + CompileTarget: compiler.ArangoTarget(), + Registry: previewBoundaryExplainRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "preview-boundary" }, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + if _, _, err := request.Command.AQLParameters(); err != nil { + return err + } + aqlCalls++ + return nil + }, + PreviewSourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + if _, _, err := request.Command.AQLParameters(); err != nil { + return err + } + aqlCalls++ + return nil + }, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + bindings := recipe.RuntimeBindings{Project: "P1", DatasetGeneration: "G1", PreviewLimit: 25, OutputNames: []string{"resources"}} + control := Control{Engine: engine} + result, err := control.ExplainPhysical(context.Background(), "preview-boundary", bindings, false) + if err != nil { + t.Fatalf("compile-only ExplainPhysical: %v", err) + } + if aqlCalls != 0 || pgCalls != 0 { + t.Fatalf("compile-only Explain touched adapters: aql=%d pg=%d", aqlCalls, pgCalls) + } + for _, output := range result.Outputs { + if output.Live != nil || output.LivePostgreSQL != nil { + t.Fatalf("compile-only Explain carried live assessments: %+v", output) + } + } + liveOnly := Control{Engine: engine, ExplainPostgreSQLConnection: func(context.Context, compiler.CompiledQuery) (PostgreSQLPlanAssessment, error) { + pgCalls++ + return PostgreSQLPlanAssessment{}, nil + }} + if _, err := liveOnly.ExplainPhysical(context.Background(), "preview-boundary", bindings, true); err == nil || + !strings.Contains(err.Error(), "Arango") { + t.Fatalf("AQL live Explain without an Arango connection: err=%v, want refusal", err) + } + if pgCalls != 0 { + t.Fatal("the PostgreSQL Explain callback ran for an AQL-tagged command") + } + }) +} + +func TestChooseGroupPreviewScanHintLoopAbortsWhenContextExpiresDuringCapabilityCall(t *testing.T) { + engine, query, stream := compiledGroupPreviewScanFixture(t, recipe.RuntimeBindings{Project: "P1", DatasetGeneration: "G1"}) + original := commandText(t, query.Command) + candidate := PreviewCollectionIndex{ + Collection: query.PreviewCoveringIndex.Collection, Name: "loom_pivot_preview_verdict_hint", + Fields: append([]string(nil), query.PreviewCoveringIndex.Fields...), + } + engine.previewCoveringIndexes = func(context.Context, string) ([]PreviewCollectionIndex, error) { + return []PreviewCollectionIndex{candidate}, nil + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + scanCalls := 0 + engine.previewCollectionScans = func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + scanCalls++ + cancel() + return PreviewCollectionScanEvidence{IndexNodes: []PreviewIndexNodeEvidence{ + {IndexNames: []string{candidate.Name}, Coverage: PreviewIndexCoverageCovering}, + }}, nil + } + err := engine.chooseGroupPreviewScan(ctx, &query, &stream) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want the canceled-budget path", err) + } + if scanCalls != 1 { + t.Fatalf("capability consultations = %d, want exactly one (the aborted candidate)", scanCalls) + } + if got := commandText(t, query.Command); got != original || strings.Contains(got, `indexHint: "`+candidate.Name+`"`) { + t.Fatalf("a canceled inspection still flipped the decision to the hinted command: %q", got) + } + if got := commandText(t, stream.command); got != original { + t.Fatalf("stream command drifted after the canceled inspection: %q", got) + } +} + +func TestChooseGroupPreviewScanOrdinaryPathSkippedWhenAssessmentBudgetExpired(t *testing.T) { + engine, query, stream := groupPreviewScanFixture() + engine.previewCollectionCount = func(context.Context, string) (int64, error) { return 1000, nil } + scopeCountExecutions := 0 + stream.sourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + scopeCountExecutions++ + return visit(map[string]any{"count": float64(950)}) + } + scanCalls := 0 + engine.previewCollectionScans = func(ctx context.Context, _ command.CompiledCommand, _ string) (PreviewCollectionScanEvidence, error) { + scanCalls++ + <-ctx.Done() + return notCoveringGroupScans(), nil + } + if err := engine.chooseGroupPreviewScan(context.Background(), &query, &stream); err != nil { + t.Fatalf("advisory budget expiry failed the preview: %v", err) + } + if scanCalls != 1 { + t.Fatalf("capability consultations = %d, want the ordinary-path check to have run", scanCalls) + } + if scopeCountExecutions != 0 { + t.Fatalf("expired budget still executed the scoped count query %d times; the decision must be skipped", scopeCountExecutions) + } + if got := commandText(t, query.Command); got != "normal-query" { + t.Fatalf("expired budget switched the command to %q", got) + } +} + +type previewBoundaryExplainRegistry struct{} + +func (previewBoundaryExplainRegistry) LoadRecipe(context.Context, string) (exec.Entry, error) { + return exec.Entry{Bundle: recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "preview-boundary", + TranslationVersion: "test", + Outputs: []recipe.Output{{ + Name: "resources", RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{{Name: "id", Expr: recipe.Expression{Select: "root.id"}}}, + }}, + }}, nil +} + +func (r previewBoundaryExplainRegistry) LoadRecipeVersion(ctx context.Context, name, _ string) (exec.Entry, error) { + return r.LoadRecipe(ctx, name) +} diff --git a/internal/dataframe/execution/query_budget.go b/internal/dataframe/execution/query_budget.go new file mode 100644 index 000000000..e2e759299 --- /dev/null +++ b/internal/dataframe/execution/query_budget.go @@ -0,0 +1,14 @@ +package execution + +import "context" + +type queryMemoryBudgetContextKey struct{} + +func WithQueryMemoryBudget(ctx context.Context, bytes int64) context.Context { + return context.WithValue(ctx, queryMemoryBudgetContextKey{}, bytes) +} + +func QueryMemoryBudgetFromContext(ctx context.Context) (int64, bool) { + bytes, ok := ctx.Value(queryMemoryBudgetContextKey{}).(int64) + return bytes, ok +} diff --git a/internal/dataframe/execution/recipe_paging_postgres_adapter_test.go b/internal/dataframe/execution/recipe_paging_postgres_adapter_test.go new file mode 100644 index 000000000..a6d354932 --- /dev/null +++ b/internal/dataframe/execution/recipe_paging_postgres_adapter_test.go @@ -0,0 +1,13 @@ +package execution_test + +import ( + postgres "github.com/calypr/loom/internal/backends/postgres" + "github.com/calypr/loom/internal/dataframe/execution" + "github.com/jackc/pgx/v5/pgxpool" +) + +func init() { + execution.Exec05InstallRowsAdapterFactory(func(iso *pgxpool.Pool) (execution.SourceRows, error) { + return postgres.NewSourceRows(iso) + }) +} diff --git a/internal/dataframe/execution/recipe_paging_postgres_test.go b/internal/dataframe/execution/recipe_paging_postgres_test.go new file mode 100644 index 000000000..348a34694 --- /dev/null +++ b/internal/dataframe/execution/recipe_paging_postgres_test.go @@ -0,0 +1,1148 @@ +package execution + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "sync" + "testing" + "time" + + storepg "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" + "github.com/calypr/loom/internal/dataframe/spec" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + exec05Prefix = "verify_exec05_" + exec05NameFmt = exec05Prefix + "%d_%s" + exec05Project = "P1" + exec05Generation = "gen-a" + exec05AuthPath = "FHIR/Patient" + exec05EdgeLabel = "observation" + exec05ChildType = "Observation" + exec05RootType = "Patient" + exec05RootKeyCol = "root_key" + exec05ChildSeqCol = "child_seq" + exec05ChildIDCol = "child_id" +) + +var exec05NamePattern = regexp.MustCompile(`^verify_exec05_[0-9]+_[0-9a-f]+$`) + +var ( + exec05OwnedMu sync.Mutex + exec05OwnedByName = map[string]string{} +) + +func exec05Register(db, schema string) { + exec05OwnedMu.Lock() + defer exec05OwnedMu.Unlock() + exec05OwnedByName[db] = schema +} + +func exec05RegistrySnapshot() map[string]string { + exec05OwnedMu.Lock() + defer exec05OwnedMu.Unlock() + out := make(map[string]string, len(exec05OwnedByName)) + for db, schema := range exec05OwnedByName { + out[db] = schema + } + return out +} + +var ( + exec05BaselineOnce sync.Once + exec05BaselineNames []string + exec05BaselineErr error +) + +func exec05CaptureBaseline(raw string) { + exec05BaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec05BaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec05\\_%' ESCAPE '\\'") + if err != nil { + exec05BaselineErr = fmt.Errorf("read baseline: %w", err) + return + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + exec05BaselineErr = fmt.Errorf("scan baseline: %w", err) + return + } + exec05BaselineNames = append(exec05BaselineNames, name) + } + }) +} + +type exec05LiveService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + schema string + dsn string + target compiler.Target + executor SourceRows + scrub func(any) string + fail func(step string, err error) +} + +var exec05NewRowsSource func(iso *pgxpool.Pool) (SourceRows, error) + +func Exec05InstallRowsAdapterFactory(f func(iso *pgxpool.Pool) (SourceRows, error)) { + exec05NewRowsSource = f +} + +func exec05URL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 root-page acceptance tests") + } + return raw +} + +func exec05RandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +func newExec05LiveService(t *testing.T) *exec05LiveService { + t.Helper() + raw := exec05URL(t) + exec05CaptureBaseline(raw) + if exec05BaselineErr != nil { + t.Fatalf("capture pg_database baseline: %v", exec05BaselineErr) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, + "SELECT current_setting('server_version_num')::int8, current_setting('server_version')"). + Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if err := storepg.CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required PostgreSQL 18 backend: %v", err) + } + + dbName := fmt.Sprintf(exec05NameFmt, time.Now().UnixNano(), exec05RandomHex(t)) + if !exec05NamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + var created int64 + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", dbName).Scan(&created); err != nil { + fail("verify isolated database creation", err) + } + if created != 1 { + t.Fatalf("isolated database %q did not verifiably come into existence", dbName) + } + exec05Register(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(exec05NameFmt, time.Now().UnixNano(), exec05RandomHex(t)) + if !exec05NamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + exec05Register(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + if exec05NewRowsSource == nil { + t.Fatal("the live battery requires the production row adapter factory (external install)") + } + executor, err := exec05NewRowsSource(iso) + if err != nil { + fail("create PostgreSQL row executor", err) + } + binding, err := ir.NewPostgresStorageBinding(schema) + if err != nil { + fail("bind PostgreSQL storage placement", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + fail("select PostgreSQL compile target", err) + } + return &exec05LiveService{ctx: ctx, admin: admin, iso: iso, schema: schema, dsn: dsn, target: target, executor: executor, scrub: scrub, fail: fail} +} + +func (s *exec05LiveService) qualifiedTable(name string) string { + return `"` + s.schema + `"."` + name + `"` +} + +func (s *exec05LiveService) migrate(t *testing.T) { + t.Helper() + client, err := storepg.OpenPool(s.ctx, storepg.ClientConfig{ + DSN: s.dsn, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 2 * time.Second, + }) + if err != nil { + s.fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + defer client.Close() + if _, err := storepg.MigrateSchema(s.ctx, client, s.schema); err != nil { + s.fail("migrate owned schema", err) + } +} + +func (s *exec05LiveService) insertResource(t *testing.T, resourceType, logicalID, project, generation, auth, payload string) { + t.Helper() + repo, err := storepg.NewResourceRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + if _, err := repo.InsertResource(s.ctx, source.Resource{ + Project: project, + DatasetGeneration: generation, + ResourceType: resourceType, + LogicalID: logicalID, + AuthResourcePath: auth, + Payload: json.RawMessage(payload), + }); err != nil { + s.fail("insert fixture resource", err) + } +} + +func (s *exec05LiveService) insertEdge(t *testing.T, sourceID, targetID string, ordinal int64) { + t.Helper() + repo, err := storepg.NewEdgeRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("edge repository: %v", err) + } + if _, err := repo.InsertEdge(s.ctx, storepg.EdgeOccurrenceRow{ + Project: exec05Project, + DatasetGeneration: exec05Generation, + AuthResourcePath: exec05AuthPath, + Label: exec05EdgeLabel, + SourceResourceType: exec05RootType, + SourceLogicalID: sourceID, + TargetResourceType: exec05ChildType, + TargetLogicalID: targetID, + SourcePath: "entry", + SourceOrdinal: ordinal, + }); err != nil { + s.fail("insert fixture edge", err) + } +} + +func (s *exec05LiveService) seedAcceptanceFixture(t *testing.T) { + t.Helper() + s.migrate(t) + payload := `{"name":"fixture"}` + for _, id := range []string{"p0001", "p0002", "p0003", "p0004", "p0005"} { + s.insertResource(t, exec05RootType, id, exec05Project, exec05Generation, exec05AuthPath, payload) + } + for _, id := range []string{"c0001", "c0002", "c0003", "c0004", "c0005"} { + s.insertResource(t, exec05ChildType, id, exec05Project, exec05Generation, exec05AuthPath, payload) + } + s.insertResource(t, exec05RootType, "p0900", "P2", exec05Generation, exec05AuthPath, payload) + s.insertResource(t, exec05RootType, "p0901", exec05Project, "gen-b", exec05AuthPath, payload) + edges := []struct { + source, target string + ordinal int64 + }{ + {"p0001", "c0001", 0}, + {"p0001", "c0002", 1}, + {"p0002", "c0002", 0}, + {"p0003", "c0003", 0}, + {"p0005", "c0004", 0}, + {"p0005", "c0005", 1}, + } + for _, edge := range edges { + s.insertEdge(t, edge.source, edge.target, edge.ordinal) + } +} + +func (s *exec05LiveService) seedPlannerFixture(t *testing.T) { + t.Helper() + s.migrate(t) + payload := `{"name":"planner"}` + for index := 1; index <= 200; index++ { + s.insertResource(t, exec05RootType, fmt.Sprintf("p%04d", index), exec05Project, exec05Generation, exec05AuthPath, payload) + s.insertResource(t, exec05ChildType, fmt.Sprintf("c%04d", index), exec05Project, exec05Generation, exec05AuthPath, payload) + } + for index := 1; index <= 200; index++ { + sourceID := fmt.Sprintf("p%04d", index) + for slot, offset := range []int{0, 1, 7} { + targetID := fmt.Sprintf("c%04d", (index-1+offset)%200+1) + s.insertEdge(t, sourceID, targetID, int64(slot)) + } + } + if _, err := s.iso.Exec(s.ctx, "ANALYZE "+s.qualifiedTable("fhir_resource")); err != nil { + s.fail("analyze resource placement", err) + } + if _, err := s.iso.Exec(s.ctx, "ANALYZE "+s.qualifiedTable("fhir_edge_occurrence")); err != nil { + s.fail("analyze edge placement", err) + } +} + +func (s *exec05LiveService) keySQLTemplate() string { + return "SELECT r.logical_id AS root_key FROM " + s.qualifiedTable("fhir_resource") + + " r WHERE r.project = $1 AND r.dataset_generation = $2 AND r.resource_type = $3" + + " AND r.auth_resource_path = $4 AND r.logical_id > $5" + + " ORDER BY r.logical_id ASC LIMIT $6" +} + +func (s *exec05LiveService) rowsSQLTemplate() string { + return "SELECT r.logical_id AS root_key, e.edge_seq AS child_seq, c.logical_id AS child_id" + + " FROM " + s.qualifiedTable("fhir_resource") + " r" + + " JOIN " + s.qualifiedTable("fhir_edge_occurrence") + " e" + + " ON e.project = r.project AND e.dataset_generation = r.dataset_generation" + + " AND e.source_resource_type = r.resource_type AND e.source_logical_id = r.logical_id" + + " AND e.auth_resource_path = $4 AND e.label = $7" + + " JOIN " + s.qualifiedTable("fhir_resource") + " c" + + " ON c.project = e.project AND c.dataset_generation = e.dataset_generation" + + " AND c.resource_type = $6 AND c.logical_id = e.target_logical_id" + + " AND c.auth_resource_path = $4" + + " WHERE r.project = $1 AND r.dataset_generation = $2 AND r.resource_type = $3" + + " AND r.auth_resource_path = $4" + + " AND r.logical_id IN (SELECT k FROM jsonb_array_elements_text($5) AS k)" + + " ORDER BY r.logical_id ASC, e.edge_seq ASC, c.logical_id ASC" +} + +func (s *exec05LiveService) keyArgs(cursor string, size int64) []command.PostgreSQLParameter { + return []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: exec05Project}, + {Type: command.PostgreSQLParameterText, Value: exec05Generation}, + {Type: command.PostgreSQLParameterText, Value: exec05RootType}, + {Type: command.PostgreSQLParameterText, Value: exec05AuthPath}, + {Type: command.PostgreSQLParameterText, Value: cursor}, + {Type: command.PostgreSQLParameterBigint, Value: size}, + } +} + +func (s *exec05LiveService) rowsArgs() []command.PostgreSQLParameter { + return []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: exec05Project}, + {Type: command.PostgreSQLParameterText, Value: exec05Generation}, + {Type: command.PostgreSQLParameterText, Value: exec05RootType}, + {Type: command.PostgreSQLParameterText, Value: exec05AuthPath}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`[]`)}, + {Type: command.PostgreSQLParameterText, Value: exec05ChildType}, + {Type: command.PostgreSQLParameterText, Value: exec05EdgeLabel}, + } +} + +func (s *exec05LiveService) newExec05LiveSteps(t *testing.T, pageSize int) *PostgreSQLRootPageSteps { + t.Helper() + return &PostgreSQLRootPageSteps{ + RootKeys: newExec05PageCommand(t, s.keySQLTemplate(), s.keyArgs("", int64(pageSize))), + Rows: newExec05PageCommand(t, s.rowsSQLTemplate(), s.rowsArgs()), + RootKeyColumn: exec05RootKeyCol, + RowRootKeyColumn: exec05RootKeyCol, + CursorArgIndex: 4, + PageSizeArgIndex: 5, + PageKeysArgIndex: 4, + PageSize: pageSize, + } +} + +type exec05LiveCall struct { + kind string + sql string + args []command.PostgreSQLParameter + columns []string +} + +type exec05LivePort struct { + s *exec05LiveService + calls []exec05LiveCall + visitHook func(kind string, row map[string]any, index int) (map[string]any, error) + extraRows func(kind string, callIndex int) []map[string]any + afterCall func(p *exec05LivePort, kind string, callIndex int) +} + +func (p *exec05LivePort) port(ctx context.Context, request RowsRequest, visit RowVisitor) error { + sqlText, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + kind := "rows" + if len(request.DeclaredColumns) == 1 && request.DeclaredColumns[0] == exec05RootKeyCol && !strings.Contains(sqlText, exec05ChildSeqCol) { + kind = "keys" + } + p.calls = append(p.calls, exec05LiveCall{kind: kind, sql: sqlText, args: args, columns: append([]string(nil), request.DeclaredColumns...)}) + callIndex := len(p.calls) + inner := func(row map[string]any) error { + if p.visitHook != nil { + replacement, err := p.visitHook(kind, row, len(p.calls)) + if err != nil { + return err + } + if replacement == nil { + return nil + } + row = replacement + } + return visit(row) + } + if err := p.s.executor(ctx, request, inner); err != nil { + return err + } + if p.extraRows != nil { + for _, extra := range p.extraRows(kind, callIndex) { + if err := visit(extra); err != nil { + return err + } + } + } + if p.afterCall != nil { + p.afterCall(p, kind, callIndex) + } + return nil +} + +func exec05RecordedKeyPages(p *exec05LivePort) []string { + var pages []string + for _, call := range p.calls { + if call.kind == "rows" { + pages = append(pages, string(call.args[4].Value.(json.RawMessage))) + } + } + return pages +} + +func (p *exec05LivePort) counts() (keys, rows int) { + for _, call := range p.calls { + if call.kind == "keys" { + keys++ + } else { + rows++ + } + } + return keys, rows +} + +func exec05LiveStream(t *testing.T, s *exec05LiveService, steps *PostgreSQLRootPageSteps, port SourceRows) OutputStream { + t.Helper() + return OutputStream{ + Name: "expanded", Columns: []string{exec05RootKeyCol, exec05ChildSeqCol, exec05ChildIDCol}, + RowIdentity: &spec.RowIdentity{Fields: []string{exec05RootKeyCol, exec05ChildSeqCol, exec05ChildIDCol}}, + command: newExec05PageCommand(t, "SELECT 1", nil), + expectedBackend: command.CompiledBackendPostgreSQL, + commandColumns: []string{exec05RootKeyCol, exec05ChildSeqCol, exec05ChildIDCol}, + sourceRows: port, + batchSize: 100, + pgRootPage: steps, + } +} + +func exec05RowTuple(row map[string]any) string { + return fmt.Sprintf("%v/%v/%v", row[exec05RootKeyCol], row[exec05ChildSeqCol], row[exec05ChildIDCol]) +} + +func TestPostgresRootPagingMultisetAcrossPages(t *testing.T) { + s := newExec05LiveService(t) + s.seedAcceptanceFixture(t) + + adminRows := exec05AdminExpansion(t, s) + want := []string{"p0001/1/c0001", "p0001/2/c0002", "p0002/3/c0002", "p0003/4/c0003", "p0005/5/c0004", "p0005/6/c0005"} + if fmt.Sprint(adminRows) != fmt.Sprint(want) { + t.Fatalf("server-side expansion = %v, want the literal golden %v", adminRows, want) + } + + engine := s.engine(t, func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + return s.executor(ctx, request, visit) + }) + baseline, err := engine.Streams(s.ctx, exec05LiveResolved()) + if err != nil { + s.fail("compile baseline root stream", err) + } + baselineRoots := exec05CollectBaseline(t, s, baseline) + if fmt.Sprint(baselineRoots) != fmt.Sprint([]string{"p0001", "p0002", "p0003", "p0004", "p0005"}) { + t.Fatalf("compiled baseline roots = %v, want the five acceptance roots once each", baselineRoots) + } + + port := &exec05LivePort{s: s} + var visited []string + var pageBoundaries []int + port.afterCall = func(p *exec05LivePort, kind string, callIndex int) { + if kind == "rows" { + pageBoundaries = append(pageBoundaries, len(visited)) + } + } + stream := exec05LiveStream(t, s, s.newExec05LiveSteps(t, 2), port.port) + if err := stream.streamRaw(s.ctx, func(row map[string]any) error { + visited = append(visited, exec05RowTuple(row)) + return nil + }); err != nil { + s.fail("paged streamRaw", err) + } + if fmt.Sprint(visited) != fmt.Sprint(want) { + t.Fatalf("paged visitation = %v, want the exact multiset once each across pages: %v", visited, want) + } + keyPages := exec05RecordedKeyPages(port) + if len(pageBoundaries) != len(keyPages) { + t.Fatalf("page boundaries %v do not line up with %d row steps", pageBoundaries, len(keyPages)) + } + previousRoots := map[string]int{} + start := 0 + for pageIndex, end := range pageBoundaries { + var pageKeys []string + if err := json.Unmarshal([]byte(keyPages[pageIndex]), &pageKeys); err != nil { + t.Fatalf("decode requested page %d: %v", pageIndex, err) + } + requested := make(map[string]bool, len(pageKeys)) + for _, key := range pageKeys { + requested[key] = true + } + for _, tuple := range visited[start:end] { + root := strings.SplitN(tuple, "/", 2)[0] + if !requested[root] { + t.Fatalf("page %d row %v belongs to root %q outside the requested page %s", pageIndex, tuple, root, keyPages[pageIndex]) + } + if earlier, seen := previousRoots[root]; seen && earlier != pageIndex { + t.Fatalf("root %q produced rows on two pages (%d and %d): rows would duplicate across pages", root, earlier, pageIndex) + } + previousRoots[root] = pageIndex + } + start = end + } + keys, rows := port.counts() + if keys != 3 || rows != 3 { + t.Fatalf("port calls keys=%d rows=%d, want 3/3", keys, rows) + } + var cursors []string + for _, call := range port.calls { + if call.kind == "keys" { + cursors = append(cursors, call.args[4].Value.(string)) + if call.args[5].Value.(int64) != 2 { + t.Fatalf("key-step page size = %v, want the fixed 2", call.args[5].Value) + } + } + } + if fmt.Sprint(cursors) != fmt.Sprint([]string{"", "p0002", "p0004"}) { + t.Fatalf("cursors = %v, want [ p0002 p0004]", cursors) + } + wantPages := []string{`["p0001","p0002"]`, `["p0003","p0004"]`, `["p0005"]`} + if fmt.Sprint(keyPages) != fmt.Sprint(wantPages) { + t.Fatalf("key pages = %v, want %v", keyPages, wantPages) + } + for _, tuple := range visited { + if strings.HasPrefix(tuple, "p09") { + t.Fatalf("scope intruder surfaced in paged output: %v", tuple) + } + } +} + +func exec05AdminExpansion(t *testing.T, s *exec05LiveService) []string { + t.Helper() + query := "SELECT r.logical_id, e.edge_seq, c.logical_id FROM " + s.qualifiedTable("fhir_resource") + " r" + + " JOIN " + s.qualifiedTable("fhir_edge_occurrence") + " e" + + " ON e.project = r.project AND e.dataset_generation = r.dataset_generation" + + " AND e.source_resource_type = r.resource_type AND e.source_logical_id = r.logical_id" + + " AND e.auth_resource_path = $4 AND e.label = $7" + + " JOIN " + s.qualifiedTable("fhir_resource") + " c" + + " ON c.project = e.project AND c.dataset_generation = e.dataset_generation" + + " AND c.resource_type = $6 AND c.logical_id = e.target_logical_id" + + " AND c.auth_resource_path = $4" + + " WHERE r.project = $1 AND r.dataset_generation = $2 AND r.resource_type = $3 AND r.auth_resource_path = $4" + + " AND $5::jsonb IS NULL" + + " ORDER BY r.logical_id ASC, e.edge_seq ASC, c.logical_id ASC" + rows, err := s.iso.Query(s.ctx, query, exec05Project, exec05Generation, exec05RootType, exec05AuthPath, nil, exec05ChildType, exec05EdgeLabel) + if err != nil { + s.fail("admin expansion query", err) + } + defer rows.Close() + var out []string + for rows.Next() { + var root, child string + var seq int64 + if err := rows.Scan(&root, &seq, &child); err != nil { + s.fail("scan admin expansion", err) + } + out = append(out, fmt.Sprintf("%s/%d/%s", root, seq, child)) + } + if err := rows.Err(); err != nil { + s.fail("admin expansion rows error", err) + } + return out +} + +func TestPostgresRootPagingTerminatesWithoutExtraQuery(t *testing.T) { + t.Run("short final page stops without another key query", func(t *testing.T) { + s := newExec05LiveService(t) + s.seedAcceptanceFixture(t) + port := &exec05LivePort{s: s} + stream := exec05LiveStream(t, s, s.newExec05LiveSteps(t, 2), port.port) + var visited []string + if err := stream.streamRaw(s.ctx, func(row map[string]any) error { + visited = append(visited, exec05RowTuple(row)) + return nil + }); err != nil { + s.fail("short-page arm", err) + } + keys, rows := port.counts() + if keys != 3 || rows != 3 { + t.Fatalf("port calls keys=%d rows=%d, want 3/3 — the short final page must not trigger a fourth key query", keys, rows) + } + if len(visited) != 6 { + t.Fatalf("visited %d rows, want 6", len(visited)) + } + }) + t.Run("empty final page issues no row query", func(t *testing.T) { + s := newExec05LiveService(t) + s.seedAcceptanceFixture(t) + port := &exec05LivePort{s: s} + stream := exec05LiveStream(t, s, s.newExec05LiveSteps(t, 5), port.port) + var visited []string + if err := stream.streamRaw(s.ctx, func(row map[string]any) error { + visited = append(visited, exec05RowTuple(row)) + return nil + }); err != nil { + s.fail("empty-page arm", err) + } + keys, rows := port.counts() + if keys != 2 || rows != 1 { + t.Fatalf("port calls keys=%d rows=%d, want 2/1 — the empty key page must not trigger a row query", keys, rows) + } + if len(visited) != 6 { + t.Fatalf("visited %d rows, want 6", len(visited)) + } + }) +} + +func TestPostgresRootPagingCancellation(t *testing.T) { + t.Run("cancellation between key and row steps prevents the row step", func(t *testing.T) { + s := newExec05LiveService(t) + s.seedAcceptanceFixture(t) + ctx, cancel := context.WithCancel(s.ctx) + defer cancel() + port := &exec05LivePort{s: s} + port.afterCall = func(p *exec05LivePort, kind string, callIndex int) { + if kind == "keys" && callIndex == 1 { + cancel() + } + } + stream := exec05LiveStream(t, s, s.newExec05LiveSteps(t, 2), port.port) + _, err := stream.Stream(ctx, func(map[string]any) error { return nil }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context.Canceled — completion must never be reported", err) + } + keys, rows := port.counts() + if keys != 1 || rows != 0 { + t.Fatalf("port calls keys=%d rows=%d, want 1/0 — the row step must never be dispatched after cancellation", keys, rows) + } + }) + t.Run("control: uncancelled run completes on the same fixture", func(t *testing.T) { + s := newExec05LiveService(t) + s.seedAcceptanceFixture(t) + port := &exec05LivePort{s: s} + stream := exec05LiveStream(t, s, s.newExec05LiveSteps(t, 2), port.port) + result, err := stream.Stream(s.ctx, func(map[string]any) error { return nil }) + if err != nil { + t.Fatalf("paired control: %v", err) + } + if result.RowCount != 6 { + t.Fatalf("control RowCount = %d, want 6", result.RowCount) + } + }) +} + +func TestPostgresRootPagingResultContractSabotage(t *testing.T) { + t.Run("duplicate root key fails the typed result contract", func(t *testing.T) { + s := newExec05LiveService(t) + s.seedAcceptanceFixture(t) + port := &exec05LivePort{s: s} + port.extraRows = func(kind string, callIndex int) []map[string]any { + if kind == "keys" && callIndex == 1 { + return []map[string]any{{exec05RootKeyCol: "p0001"}} + } + return nil + } + stream := exec05LiveStream(t, s, s.newExec05LiveSteps(t, 2), port.port) + err := stream.streamRaw(s.ctx, func(map[string]any) error { return nil }) + if !errors.Is(err, ErrPostgreSQLResultContract) { + t.Fatalf("error = %v, want ErrPostgreSQLResultContract for a duplicated root key", err) + } + }) + t.Run("row outside the requested page fails the typed result contract", func(t *testing.T) { + s := newExec05LiveService(t) + s.seedAcceptanceFixture(t) + port := &exec05LivePort{s: s} + port.extraRows = func(kind string, callIndex int) []map[string]any { + if kind == "rows" && callIndex > 1 { + return []map[string]any{{exec05RootKeyCol: "p0900", exec05ChildSeqCol: int64(999), exec05ChildIDCol: "c0900"}} + } + return nil + } + stream := exec05LiveStream(t, s, s.newExec05LiveSteps(t, 2), port.port) + err := stream.streamRaw(s.ctx, func(map[string]any) error { return nil }) + if !errors.Is(err, ErrPostgreSQLResultContract) { + t.Fatalf("error = %v, want ErrPostgreSQLResultContract for a row outside the requested page", err) + } + }) + t.Run("key page not advancing past the previous tail is refused", func(t *testing.T) { + s := newExec05LiveService(t) + s.seedAcceptanceFixture(t) + port := &exec05LivePort{s: s} + port.visitHook = func(kind string, row map[string]any, callIndex int) (map[string]any, error) { + if kind == "keys" && callIndex == 3 && row[exec05RootKeyCol] == "p0003" { + return map[string]any{exec05RootKeyCol: "p0002"}, nil + } + return row, nil + } + stream := exec05LiveStream(t, s, s.newExec05LiveSteps(t, 2), port.port) + err := stream.streamRaw(s.ctx, func(map[string]any) error { return nil }) + if !errors.Is(err, ErrPostgreSQLResultContract) { + t.Fatalf("error = %v, want ErrPostgreSQLResultContract for a page that does not advance past the previous tail", err) + } + keys, rows := port.counts() + if keys != 2 || rows != 1 { + t.Fatalf("port calls keys=%d rows=%d, want 2/1: the refusing page must never reach the row step", keys, rows) + } + }) + t.Run("control: unsabotaged run proceeds on the same fixture", func(t *testing.T) { + s := newExec05LiveService(t) + s.seedAcceptanceFixture(t) + port := &exec05LivePort{s: s} + port.visitHook = func(kind string, row map[string]any, callIndex int) (map[string]any, error) { + return row, nil + } + stream := exec05LiveStream(t, s, s.newExec05LiveSteps(t, 2), port.port) + var visited []string + if err := stream.streamRaw(s.ctx, func(row map[string]any) error { + visited = append(visited, exec05RowTuple(row)) + return nil + }); err != nil { + t.Fatalf("paired control: %v", err) + } + if len(visited) != 6 { + t.Fatalf("control visited %d rows, want 6", len(visited)) + } + }) +} + +type exec05ExplainNode struct { + NodeType string `json:"Node Type"` + RelationName string `json:"Relation Name"` + IndexName string `json:"Index Name"` + Plans []exec05ExplainNode `json:"Plans"` +} + +func (n exec05ExplainNode) walk(visit func(exec05ExplainNode)) { + visit(n) + for _, child := range n.Plans { + child.walk(visit) + } +} + +func exec05BoundArgs(args []command.PostgreSQLParameter) []any { + bound := make([]any, 0, len(args)) + for _, arg := range args { + bound = append(bound, arg.Value) + } + return bound +} + +func exec05ExplainAnalyzedRuns(t *testing.T, s *exec05LiveService, sql string, args []command.PostgreSQLParameter) int64 { + t.Helper() + rows, err := s.iso.Query(s.ctx, "EXPLAIN (ANALYZE, FORMAT JSON) "+sql, exec05BoundArgs(args)...) + if err != nil { + s.fail("EXPLAIN ANALYZE "+sql, err) + } + defer rows.Close() + var encoded []byte + for rows.Next() { + if err := rows.Scan(&encoded); err != nil { + s.fail("scan EXPLAIN ANALYZE", err) + } + } + if err := rows.Err(); err != nil { + s.fail("EXPLAIN ANALYZE rows error", err) + } + var explained []struct { + Plan struct { + ActualRows float64 `json:"Actual Rows"` + } `json:"Plan"` + } + if err := json.Unmarshal(encoded, &explained); err != nil || len(explained) != 1 { + t.Fatalf("decode EXPLAIN ANALYZE output: %v (%s)", err, s.scrub(string(encoded))) + } + return int64(explained[0].Plan.ActualRows) +} + +func exec05Explain(t *testing.T, s *exec05LiveService, sql string, args []command.PostgreSQLParameter) exec05ExplainNode { + t.Helper() + rows, err := s.iso.Query(s.ctx, "EXPLAIN (FORMAT JSON) "+sql, exec05BoundArgs(args)...) + if err != nil { + s.fail("EXPLAIN "+sql, err) + } + defer rows.Close() + var encoded []byte + for rows.Next() { + if err := rows.Scan(&encoded); err != nil { + s.fail("scan EXPLAIN", err) + } + } + if err := rows.Err(); err != nil { + s.fail("EXPLAIN rows error", err) + } + var plans []struct { + Plan exec05ExplainNode `json:"Plan"` + } + if err := json.Unmarshal(encoded, &plans); err != nil { + t.Fatalf("decode EXPLAIN output: %v", err) + } + if len(plans) != 1 { + t.Fatalf("EXPLAIN returned %d plans, want 1", len(plans)) + } + return plans[0].Plan +} + +func TestPostgresRootPagingExplainBoundedness(t *testing.T) { + s := newExec05LiveService(t) + s.seedPlannerFixture(t) + + keyPlan := exec05Explain(t, s, s.keySQLTemplate(), s.keyArgs("p0100", 2)) + sawLimit, sawIndexScan, sawSeqScanResource := false, false, false + keyPlan.walk(func(node exec05ExplainNode) { + switch node.NodeType { + case "Limit": + sawLimit = true + case "Index Scan", "Index Only Scan", "Bitmap Index Scan": + if node.RelationName == "fhir_resource" || node.RelationName == "" { + sawIndexScan = true + } + case "Seq Scan": + if node.RelationName == "fhir_resource" { + sawSeqScanResource = true + } + } + }) + if !sawLimit { + t.Fatalf("key-step plan carries no Limit node: %v", keyPlan) + } + if sawSeqScanResource { + t.Fatalf("key-step plan sequentially scans fhir_resource instead of using the keyset index: %v", keyPlan) + } + if !sawIndexScan { + t.Fatalf("key-step plan shows no index access on fhir_resource: %v", keyPlan) + } + if keyRows := exec05ExplainAnalyzedRuns(t, s, s.keySQLTemplate(), s.keyArgs("p0100", 2)); keyRows > 2 { + t.Fatalf("key step returned %d rows under LIMIT 2: the page is not server-side bounded", keyRows) + } + + rowsArgs := s.rowsArgs() + rowsArgs[4] = command.PostgreSQLParameter{Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`["p0100","p0101"]`)} + if rowRows := exec05ExplainAnalyzedRuns(t, s, s.rowsSQLTemplate(), rowsArgs); rowRows != 6 { + t.Fatalf("row step touched %d expansion rows for a two-key page, want exactly 6 (full scope would be 600)", rowRows) + } + + port := &exec05LivePort{s: s} + stream := exec05LiveStream(t, s, s.newExec05LiveSteps(t, 100), port.port) + visited := 0 + if err := stream.streamRaw(s.ctx, func(map[string]any) error { + visited++ + return nil + }); err != nil { + s.fail("planner-fixture paged walk", err) + } + if visited != 600 { + t.Fatalf("visited %d expansion rows, want 600 (200 roots x 3 edges)", visited) + } + keys, rows := port.counts() + if keys != 3 || rows != 2 { + t.Fatalf("port calls keys=%d rows=%d, want 3/2 (two full pages + empty final page)", keys, rows) + } +} + +func TestPostgresRootPagingWholeInputBypass(t *testing.T) { + t.Run("whole-input PostgreSQL output cannot reach the root-page loop", func(t *testing.T) { + s := newExec05LiveService(t) + s.seedAcceptanceFixture(t) + calls := 0 + engine := s.engine(t, func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + calls++ + return s.executor(ctx, request, visit) + }) + engine.rootPageRows = 50 + resolved := exec05LiveResolved() + output := resolved.Compiled.Outputs[0] + output.Name = "grouped" + binds := exec05LiveBindVars() + binds["preview_limit"] = 25 + output.Plan = ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: binds, + Operations: []ir.PhysicalOperation{{ + Kind: ir.PhysicalGroupRowsOp, + GroupRows: &ir.PhysicalGroupRows{LimitBindKey: "preview_limit"}, + }}, + } + resolved.Compiled.Outputs = []lower.CompiledRecipeOutput{output} + resolved.Semantic.SemanticPlan.Bindings.OutputNames = []string{"grouped"} + streams, err := engine.Streams(s.ctx, resolved) + if err != nil { + t.Logf("whole-input bypass arm: compile-refused before execution (%s)", s.scrub(err)) + if calls != 0 { + t.Fatalf("refused whole-input plan reached the port %d times", calls) + } + return + } + t.Logf("whole-input bypass arm: compiled, structurally single-statement") + if len(streams) != 1 || streams[0].page != nil || streams[0].pgRootPage != nil || streams[0].rootPageRows != 0 { + t.Fatalf("whole-input stream carries page shape: %+v", streams[0]) + } + if _, err := streams[0].Stream(s.ctx, func(map[string]any) error { return nil }); err != nil { + s.fail("whole-input single statement", err) + } + if calls != 1 { + t.Fatalf("whole-input port calls = %d, want exactly one statement", calls) + } + }) + t.Run("control: non-group PostgreSQL stream never carries page fields", func(t *testing.T) { + s := newExec05LiveService(t) + s.seedAcceptanceFixture(t) + calls := 0 + engine := s.engine(t, func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + calls++ + return s.executor(ctx, request, visit) + }) + engine.rootPageRows = 50 + streams, err := engine.Streams(s.ctx, exec05LiveResolved()) + if err != nil { + s.fail("control streams", err) + } + if len(streams) != 1 || streams[0].page != nil || streams[0].pgRootPage != nil || streams[0].rootPageRows != 0 { + t.Fatalf("engine-built PG stream carries page shape: %+v", streams[0]) + } + result, err := streams[0].Stream(s.ctx, func(map[string]any) error { return nil }) + if err != nil { + s.fail("control single statement", err) + } + if calls != 1 || result.RowCount != 5 { + t.Fatalf("control port calls=%d RowCount=%d, want 1/5", calls, result.RowCount) + } + }) +} + +func (s *exec05LiveService) engine(t *testing.T, sourceRows SourceRows) *Engine { + t.Helper() + config := Config{ + Registry: invalidRecipeRegistry{}, + CompileTarget: s.target, + SourceRows: func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + if _, _, err := request.Command.PostgreSQLParameters(); err != nil { + return err + } + return sourceRows(ctx, request, visit) + }, + } + engine, err := New(config) + if err != nil { + t.Fatalf("New engine: %v", err) + } + return engine +} + +func exec05LiveBindVars() map[string]any { + return map[string]any{ + "root_collection": exec05RootType, + "project": exec05Project, + "dataset_generation": exec05Generation, + "auth_resource_paths": []string{exec05AuthPath}, + "auth_resource_paths_unrestricted": false, + "scope_allowed": true, + } +} + +func exec05LiveResolved() Resolved { + columns := []string{"project", "dataset_generation", "resource_type", "logical_id", "auth_resource_path"} + schema := make([]lower.CompiledOutputColumn, 0, len(columns)) + projections := make([]ir.PhysicalProjection, 0, len(columns)) + for _, column := range columns { + schema = append(schema, lower.CompiledOutputColumn{ID: column, Name: column, Label: column, Kind: "string", Cardinality: "one"}) + projections = append(projections, ir.PhysicalProjection{ + Name: column, + Value: ir.PhysicalValue{Variable: "root", Path: []string{column}}, + }) + } + output := lower.CompiledRecipeOutput{ + Name: "roots", + RootResourceType: exec05RootType, + Columns: append([]string(nil), columns...), + OutputSchema: schema, + RowIdentity: &spec.RowIdentity{Fields: []string{"logical_id"}}, + Plan: ir.PhysicalPlan{ + Version: 1, + Engine: ir.PhysicalEngineAQL, + BindVars: exec05LiveBindVars(), + Operations: []ir.PhysicalOperation{ + {Kind: ir.PhysicalRootScanOp, RootScan: &ir.PhysicalRootScan{Variable: "root", CollectionBindKey: "root_collection"}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"project"}}, Right: &ir.PhysicalValue{BindKey: "project"}}}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root", Path: []string{"dataset_generation"}}, Right: &ir.PhysicalValue{BindKey: "dataset_generation"}}}}, + {Kind: ir.PhysicalDerivedLetOp, DerivedLet: &ir.PhysicalDerivedLet{Variable: "root_auth_allowed", Operator: "AUTH_RESOURCE_PATH_ALLOWED", Inputs: []ir.PhysicalValue{ + {Variable: "root", Path: []string{"auth_resource_path"}}, + {BindKey: "auth_resource_paths"}, + {BindKey: "auth_resource_paths_unrestricted"}, + }}}, + {Kind: ir.PhysicalFilterOp, Filter: &ir.PhysicalFilter{Predicate: ir.PhysicalPredicate{Operator: "EQUALS", Left: ir.PhysicalValue{Variable: "root_auth_allowed"}, Right: &ir.PhysicalValue{BindKey: "scope_allowed"}}}}, + {Kind: ir.PhysicalReturnOp, Return: &ir.PhysicalReturn{Projections: projections}}, + }, + }, + } + return Resolved{ + Bundle: recipe.Bundle{Name: "exec05_roots"}, + Semantic: semantic.ResolvedRecipePlan{ + SemanticPlan: semantic.RecipePlan{Bindings: recipe.RuntimeBindings{ + Project: exec05Project, DatasetGeneration: exec05Generation, OutputNames: []string{"roots"}, + }}, + }, + Compiled: lower.CompiledRecipe{Outputs: []lower.CompiledRecipeOutput{output}}, + StoredRecipeDigest: "exec05-roots", + } +} + +func exec05CollectBaseline(t *testing.T, s *exec05LiveService, streams []OutputStream) []string { + t.Helper() + if len(streams) != 1 { + t.Fatalf("baseline streams = %d, want 1", len(streams)) + } + var roots []string + if _, err := streams[0].Stream(s.ctx, func(row map[string]any) error { + roots = append(roots, fmt.Sprint(row["logical_id"])) + return nil + }); err != nil { + s.fail("baseline stream", err) + } + return roots +} + +func TestZZPostgresRootPagingOwnershipAudit(t *testing.T) { + raw := exec05URL(t) + exec05CaptureBaseline(raw) + if exec05BaselineErr != nil { + t.Fatalf("capture pg_database baseline: %v", exec05BaselineErr) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for audit: %v", err) + } + defer admin.Close() + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec05\\_%' ESCAPE '\\'") + if err != nil { + t.Fatalf("audit query: %v", err) + } + defer rows.Close() + var survivors []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("audit scan: %v", err) + } + survivors = append(survivors, name) + } + if err := rows.Err(); err != nil { + t.Fatalf("audit rows error: %v", err) + } + baseline := map[string]bool{} + for _, name := range exec05BaselineNames { + baseline[name] = true + } + owned := exec05RegistrySnapshot() + for _, name := range survivors { + if !baseline[name] { + t.Errorf("unregistered survivor under verify_exec05_*: %s", name) + } + } + for name := range owned { + for _, survivor := range survivors { + if survivor == name { + t.Errorf("owned database survived cleanup: %s (schema %q)", name, owned[name]) + } + } + } +} diff --git a/internal/dataframe/execution/recipe_preview_paging_postgres_internal_test.go b/internal/dataframe/execution/recipe_preview_paging_postgres_internal_test.go new file mode 100644 index 000000000..299d6c276 --- /dev/null +++ b/internal/dataframe/execution/recipe_preview_paging_postgres_internal_test.go @@ -0,0 +1,789 @@ +package execution + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "sync" + "testing" + "time" + + storepg "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + postgresrender "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" + "github.com/calypr/loom/internal/dataframe/recipe" + exec "github.com/calypr/loom/internal/dataframe/recipe/exec" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + exec04pgPrefix = "verify_exec04_pg_" + exec04pgNameFmt = exec04pgPrefix + "%d_%s" +) + +var exec04pgNamePattern = regexp.MustCompile(`^verify_exec04_pg_[0-9]+_[0-9a-f]+$`) + +var ( + exec04pgOwnedMu sync.Mutex + exec04pgOwnedByName = map[string]string{} +) + +func exec04pgRegister(db, schema string) { + exec04pgOwnedMu.Lock() + defer exec04pgOwnedMu.Unlock() + exec04pgOwnedByName[db] = schema +} + +func exec04pgRegistrySnapshot() map[string]string { + exec04pgOwnedMu.Lock() + defer exec04pgOwnedMu.Unlock() + out := make(map[string]string, len(exec04pgOwnedByName)) + for db, schema := range exec04pgOwnedByName { + out[db] = schema + } + return out +} + +var ( + exec04pgBaselineOnce sync.Once + exec04pgBaselineNames []string + exec04pgBaselineErr error +) + +func exec04pgCaptureBaseline(raw string) { + exec04pgBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec04pgBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec04\\_pg\\_%' ESCAPE '\\'") + if err != nil { + exec04pgBaselineErr = fmt.Errorf("read baseline: %w", err) + return + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + exec04pgBaselineErr = fmt.Errorf("scan baseline: %w", err) + return + } + exec04pgBaselineNames = append(exec04pgBaselineNames, name) + } + }) +} + +func exec04pgURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 preview root-page acceptance tests") + } + return raw +} + +func exec04pgRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +type exec04pgRegistry struct{} + +func (exec04pgRegistry) LoadRecipe(context.Context, string) (exec.Entry, error) { + return exec.Entry{}, errors.New("the preview battery compiles authored bundles only") +} + +func (exec04pgRegistry) LoadRecipeVersion(context.Context, string, string) (exec.Entry, error) { + return exec.Entry{}, errors.New("the preview battery compiles authored bundles only") +} + +type exec04pgService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + schema string + dsn string + target compiler.Target + scrub func(any) string + fail func(step string, err error) + executor SourceRows +} + +func exec04pgScrubberFor(raw string) (func(any) string, func(string)) { + secrets := []string{raw} + if u, err := url.Parse(raw); err == nil && u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + return scrub, func(s string) { secrets = append(secrets, s) } +} + +func TestExec04pgScrubberHidesCredentialForms(t *testing.T) { + raw := "postgres://verify_user:S3cr3t-Pw@db.internal:5432/loomsvc?sslmode=require" + scrub, addSecret := exec04pgScrubberFor(raw) + dsn := "postgres://verify_user:S3cr3t-Pw@db.internal:5432/verify_exec04_pg_it" + addSecret(dsn) + probe := fmt.Errorf("cannot parse `%s`: pq: auth failed for %s via %s", raw, "verify_user:S3cr3t-Pw", dsn) + got := scrub(probe) + for _, secret := range []string{raw, "verify_user:S3cr3t-Pw", "S3cr3t-Pw", dsn} { + if strings.Contains(got, secret) { + t.Fatalf("scrubbed text still contains credential form %q: %s", secret, got) + } + } + if !strings.Contains(got, "") { + t.Fatalf("scrubbed text lost the redaction marker: %s", got) + } +} + +func newExec04pgService(t *testing.T) *exec04pgService { + t.Helper() + raw := exec04pgURL(t) + scrub, addSecret := exec04pgScrubberFor(raw) + exec04pgCaptureBaseline(raw) + if exec04pgBaselineErr != nil { + t.Fatalf("capture pg_database baseline: %s", scrub(exec04pgBaselineErr)) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, "SELECT current_setting('server_version_num')::int8, current_setting('server_version')").Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if err := storepg.CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required PostgreSQL 18 backend: %v", err) + } + + dbName := fmt.Sprintf(exec04pgNameFmt, time.Now().UnixNano(), exec04pgRandomHex(t)) + if !exec04pgNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + var created int64 + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", dbName).Scan(&created); err != nil { + fail("verify isolated database creation", err) + } + if created != 1 { + t.Fatalf("isolated database %q did not verifiably come into existence", dbName) + } + exec04pgRegister(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + addSecret(dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(exec04pgNameFmt, time.Now().UnixNano(), exec04pgRandomHex(t)) + if !exec04pgNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + exec04pgRegister(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + if exec05NewRowsSource == nil { + t.Fatal("the live battery requires the production row adapter factory (external install)") + } + executor, err := exec05NewRowsSource(iso) + if err != nil { + fail("create PostgreSQL row executor", err) + } + binding, err := ir.NewPostgresStorageBinding(schema) + if err != nil { + fail("bind PostgreSQL storage placement", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + fail("select PostgreSQL compile target", err) + } + return &exec04pgService{ctx: ctx, admin: admin, iso: iso, schema: schema, dsn: dsn, target: target, scrub: scrub, fail: fail, executor: executor} +} + +func (s *exec04pgService) qualifiedTable(name string) string { + return `"` + s.schema + `"."` + name + `"` +} + +func (s *exec04pgService) migrate(t *testing.T) { + t.Helper() + client, err := storepg.OpenPool(s.ctx, storepg.ClientConfig{DSN: s.dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 2 * time.Second}) + if err != nil { + s.fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + defer client.Close() + if _, err := storepg.MigrateSchema(s.ctx, client, s.schema); err != nil { + s.fail("migrate owned schema", err) + } +} + +func (s *exec04pgService) insertResource(t *testing.T, logicalID, project, generation, payload string) { + t.Helper() + repo, err := storepg.NewResourceRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + if _, err := repo.InsertResource(s.ctx, source.Resource{ + Project: project, + DatasetGeneration: generation, + ResourceType: "Patient", + LogicalID: logicalID, + AuthResourcePath: "FHIR/Patient", + Payload: json.RawMessage(payload), + }); err != nil { + s.fail("insert fixture resource "+logicalID, err) + } +} + +func (s *exec04pgService) seedRoots(t *testing.T) { + t.Helper() + s.migrate(t) + roots := []struct{ id, gender, birth string }{ + {"p0001", "female", "2001-01-01"}, + {"p0002", "male", "2002-02-02"}, + {"p0003", "female", "2003-03-03"}, + {"p0004", "male", "2004-04-04"}, + {"p0005", "female", "2005-05-05"}, + } + for _, root := range roots { + s.insertResource(t, root.id, "P1", "gen-a", fmt.Sprintf(`{"gender":%q,"birthDate":%q}`, root.gender, root.birth)) + } + s.insertResource(t, "p0900", "P2", "gen-a", `{"gender":"other","birthDate":"1990-01-01"}`) + s.insertResource(t, "p0901", "P1", "gen-b", `{"gender":"unknown","birthDate":"1991-02-02"}`) +} + +func exec04pgBundle() recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "exec04_preview", + TranslationVersion: "exec04", + Outputs: []recipe.Output{{ + Name: "patients", RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{ + {Name: "gender", ColumnID: "gender", Expr: recipe.Expression{Select: "root.gender"}}, + {Name: "birthDate", ColumnID: "birthDate", Expr: recipe.Expression{Select: "root.birthDate"}}, + }, + }}, + } +} + +func exec04pgBindings(project, generation string) recipe.RuntimeBindings { + return recipe.RuntimeBindings{ + Project: project, DatasetGeneration: generation, + AuthResourcePaths: []string{"FHIR/Patient"}, + } +} + +func exec04pgCanonicalRow(row map[string]any) string { + encoded, err := json.Marshal(row) + if err != nil { + return "unencodable:" + err.Error() + } + return string(encoded) +} + +func TestPostgresPreviewStreamNeverAttachesRootPages(t *testing.T) { + binding, err := ir.NewPostgresStorageBinding("verify_exec04_pg_it") + if err != nil { + t.Fatalf("bind offline PostgreSQL placement: %v", err) + } + pgTarget, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("offline PostgreSQL target: %v", err) + } + + t.Run("PostgreSQL preview stream never attaches either page route", func(t *testing.T) { + engine, err := New(Config{ + CompileTarget: pgTarget, + Registry: exec04pgRegistry{}, + RootPageRows: 25, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + return errors.New("no statement may run in the attach-inertness control") + }, + }) + if err != nil { + t.Fatalf("engine: %v", err) + } + resolved, err := engine.CompileResolvedBundle(context.Background(), exec04pgBundle(), exec04pgBindings("P1", "gen-a")) + if err != nil { + t.Fatalf("CompileResolvedBundle: %v", err) + } + stream, _, err := engine.streamForOutput(context.Background(), resolved, "patients", 25) + if err != nil { + t.Fatalf("streamForOutput: %v", err) + } + if stream.page != nil || stream.pgRootPage != nil || stream.rootPageRows != 0 { + t.Fatalf("PostgreSQL preview stream carries page fields: page=%v pgRootPage=%v rootPageRows=%d", + stream.page, stream.pgRootPage, stream.rootPageRows) + } + declared := false + for _, column := range stream.commandColumns { + if column == "_key" { + declared = true + } + } + if !declared { + t.Fatalf("compiled command columns %v do not carry the root identity column", stream.commandColumns) + } + }) + + t.Run("alive control: the AQL engine does page", func(t *testing.T) { + var requests []map[string]any + engine, err := New(Config{ + CompileTarget: compiler.ArangoTarget(), + Registry: exec04pgRegistry{}, + RootPageRows: 1, + SourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + _, binds, dispatchErr := request.Command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + requests = append(requests, binds) + if after, isKeyQuery := binds[compiler.RootPageAfterKeyBind]; isKeyQuery { + if cursor, _ := after.(string); cursor == "" { + return visit(map[string]any{"_key": "root-1"}) + } + return nil + } + return visit(map[string]any{"gender": "female", "birthDate": "2001-01-01"}) + }, + }) + if err != nil { + t.Fatalf("engine: %v", err) + } + resolved, err := engine.CompileResolvedBundle(context.Background(), exec04pgBundle(), exec04pgBindings("P1", "gen-a")) + if err != nil { + t.Fatalf("CompileResolvedBundle: %v", err) + } + stream, _, err := engine.streamForOutput(context.Background(), resolved, "patients", 25) + if err != nil { + t.Fatalf("streamForOutput: %v", err) + } + if stream.page == nil { + t.Fatalf("alive control failed: the AQL engine did not page with RootPageRows armed") + } + if err := stream.streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("AQL paged streamRaw: %v", err) + } + sawKeyQuery := false + for _, binds := range requests { + if _, ok := binds[compiler.RootPageAfterKeyBind]; ok { + sawKeyQuery = true + } + } + if !sawKeyQuery { + t.Fatalf("alive control failed: no root-key statement was dispatched through the AQL page loop") + } + }) +} + +func TestPostgresPreviewRootPageProducerRefusalProbe(t *testing.T) { + binding, err := ir.NewPostgresStorageBinding("verify_exec04_pg_it") + if err != nil { + t.Fatalf("bind offline PostgreSQL placement: %v", err) + } + pgTarget, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("offline PostgreSQL target: %v", err) + } + engine, err := New(Config{ + CompileTarget: pgTarget, + Registry: exec04pgRegistry{}, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + return errors.New("no statement may run in the producer-refusal probe") + }, + }) + if err != nil { + t.Fatalf("engine: %v", err) + } + resolved, err := engine.CompileResolvedBundle(context.Background(), exec04pgBundle(), exec04pgBindings("P1", "gen-a")) + if err != nil { + t.Fatalf("CompileResolvedBundle: %v", err) + } + if len(resolved.Compiled.Outputs) != 1 { + t.Fatalf("compiled outputs = %d, want 1", len(resolved.Compiled.Outputs)) + } + output := resolved.Compiled.Outputs[0] + _, err = compiler.CompileRecipeOutputPage(pgTarget, output, exec04pgBindings("P1", "gen-a"), 2, ir.DefaultPhysicalOptimizationPolicy()) + if !errors.Is(err, postgresrender.ErrUnsupportedPostgresPlan) { + t.Fatalf("PostgreSQL page producer error = %v, want typed ErrUnsupportedPostgresPlan (pg-exec-04-GAP-1 stays refused)", err) + } + if !strings.Contains(err.Error(), "root-key output paging is unsupported on the PostgreSQL compile target") { + t.Fatalf("PostgreSQL page producer refusal lost its anchor message: %v", err) + } +} + +type exec04pgCall struct { + kind string + sql string + args []command.PostgreSQLParameter + bound []string +} + +type exec04pgPort struct { + s *exec04pgService + calls []exec04pgCall + afterCall func(p *exec04pgPort, kind string, callIndex int) +} + +func (p *exec04pgPort) port(ctx context.Context, request RowsRequest, visit RowVisitor) error { + sqlText, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + kind := "rows" + if len(request.DeclaredColumns) == 1 && request.DeclaredColumns[0] == "root_key" { + kind = "keys" + } + p.calls = append(p.calls, exec04pgCall{kind: kind, sql: sqlText, args: args, bound: append([]string(nil), request.DeclaredColumns...)}) + callIndex := len(p.calls) + if err := p.s.executor(ctx, request, visit); err != nil { + return err + } + if p.afterCall != nil { + p.afterCall(p, kind, callIndex) + } + return nil +} + +func (p *exec04pgPort) counts() (keys, rows int) { + for _, call := range p.calls { + if call.kind == "keys" { + keys++ + } else { + rows++ + } + } + return keys, rows +} + +func (p *exec04pgPort) recordedKeyPages() []string { + var pages []string + for _, call := range p.calls { + if call.kind == "rows" { + pages = append(pages, string(call.args[4].Value.(json.RawMessage))) + } + } + return pages +} + +func (s *exec04pgService) keySQLTemplate() string { + return "SELECT r.logical_id AS root_key FROM " + s.qualifiedTable("fhir_resource") + + " r WHERE r.project = $1 AND r.dataset_generation = $2 AND r.resource_type = $3" + + " AND r.auth_resource_path = $4 AND r.logical_id > $5" + + " ORDER BY r.logical_id ASC LIMIT $6" +} + +func (s *exec04pgService) rowsSQLTemplate() string { + return `SELECT r.logical_id AS c0, r.payload -> 'gender' AS c1, r.payload -> 'birthDate' AS c2` + + " FROM " + s.qualifiedTable("fhir_resource") + " r" + + " WHERE r.project = $1 AND r.dataset_generation = $2 AND r.resource_type = $3" + + " AND r.auth_resource_path = $4" + + " AND r.logical_id IN (SELECT k FROM jsonb_array_elements_text($5) AS k)" + + " ORDER BY r.logical_id ASC, r.resource_seq ASC" +} + +func (s *exec04pgService) keyArgs(cursor string, size int64) []command.PostgreSQLParameter { + return []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "P1"}, + {Type: command.PostgreSQLParameterText, Value: "gen-a"}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + {Type: command.PostgreSQLParameterText, Value: "FHIR/Patient"}, + {Type: command.PostgreSQLParameterText, Value: cursor}, + {Type: command.PostgreSQLParameterBigint, Value: size}, + } +} + +func (s *exec04pgService) rowsArgs() []command.PostgreSQLParameter { + return []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: "P1"}, + {Type: command.PostgreSQLParameterText, Value: "gen-a"}, + {Type: command.PostgreSQLParameterText, Value: "Patient"}, + {Type: command.PostgreSQLParameterText, Value: "FHIR/Patient"}, + {Type: command.PostgreSQLParameterJSONB, Value: json.RawMessage(`[]`)}, + } +} + +func (s *exec04pgService) newExec04pgSteps(t *testing.T, pageSize int) *PostgreSQLRootPageSteps { + t.Helper() + return &PostgreSQLRootPageSteps{ + RootKeys: newExec05PageCommand(t, s.keySQLTemplate(), s.keyArgs("", int64(pageSize))), + Rows: newExec05PageCommand(t, s.rowsSQLTemplate(), s.rowsArgs()), + RootKeyColumn: "root_key", + RowRootKeyColumn: "_key", + CursorArgIndex: 4, + PageSizeArgIndex: 5, + PageKeysArgIndex: 4, + PageSize: pageSize, + } +} + +func TestPostgresPreviewRootPageLoopProductionRowContract(t *testing.T) { + s := newExec04pgService(t) + s.seedRoots(t) + + engine, err := New(Config{ + CompileTarget: s.target, + Registry: exec04pgRegistry{}, + SourceRows: func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + if _, _, dispatchErr := request.Command.PostgreSQLParameters(); dispatchErr != nil { + return dispatchErr + } + return s.executor(ctx, request, visit) + }, + }) + if err != nil { + t.Fatalf("engine: %v", err) + } + resolved, err := engine.CompileResolvedBundle(s.ctx, exec04pgBundle(), exec04pgBindings("P1", "gen-a")) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + stream, query, err := engine.streamForOutput(s.ctx, resolved, "patients", 0) + if err != nil { + t.Fatalf("streamForOutput: %s", s.scrub(err)) + } + if fmt.Sprint(stream.commandColumns) != fmt.Sprint(query.Columns) || len(query.Columns) == 0 { + t.Fatalf("stream columns %v do not equal the compiled command columns %v", stream.commandColumns, query.Columns) + } + if stream.command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("compiled command tag = %q, want PostgreSQL", stream.command.Backend()) + } + + want := []string{ + `{"_key":"p0001","birthDate":"2001-01-01","gender":"female"}`, + `{"_key":"p0002","birthDate":"2002-02-02","gender":"male"}`, + `{"_key":"p0003","birthDate":"2003-03-03","gender":"female"}`, + `{"_key":"p0004","birthDate":"2004-04-04","gender":"male"}`, + `{"_key":"p0005","birthDate":"2005-05-05","gender":"female"}`, + } + + t.Run("stable rows at page boundaries, no duplicate identities", func(t *testing.T) { + port := &exec04pgPort{s: s} + var visited []string + var pageBoundaries []int + port.afterCall = func(_ *exec04pgPort, kind string, _ int) { + if kind == "rows" { + pageBoundaries = append(pageBoundaries, len(visited)) + } + } + stream.pgRootPage = s.newExec04pgSteps(t, 2) + stream.sourceRows = port.port + if err := stream.streamRaw(s.ctx, func(row map[string]any) error { + visited = append(visited, exec04pgCanonicalRow(row)) + return nil + }); err != nil { + s.fail("paged streamRaw", err) + } + if fmt.Sprint(visited) != fmt.Sprint(want) { + t.Fatalf("paged visitation = %v, want the compiler-defined stable order across pages", visited) + } + keys, rows := port.counts() + if keys != 3 || rows != 3 { + t.Fatalf("port calls keys=%d rows=%d, want 3/3 (short final page ends without a fourth key query)", keys, rows) + } + var cursors []string + for _, call := range port.calls { + if call.kind == "keys" { + cursors = append(cursors, call.args[4].Value.(string)) + if call.args[5].Value.(int64) != 2 { + t.Fatalf("key-step page size = %v, want the fixed 2", call.args[5].Value) + } + } + } + if fmt.Sprint(cursors) != fmt.Sprint([]string{"", "p0002", "p0004"}) { + t.Fatalf("literal cursor progression = %v, want [ p0002 p0004]", cursors) + } + keyPages := port.recordedKeyPages() + if fmt.Sprint(keyPages) != fmt.Sprint([]string{`["p0001","p0002"]`, `["p0003","p0004"]`, `["p0005"]`}) { + t.Fatalf("literal jsonb key pages = %v", keyPages) + } + seen := map[string]int{} + start := 0 + for pageIndex, end := range pageBoundaries { + var pageKeys []string + if err := json.Unmarshal([]byte(keyPages[pageIndex]), &pageKeys); err != nil { + t.Fatalf("decode page %d: %v", pageIndex, err) + } + requested := map[string]bool{} + for _, key := range pageKeys { + requested[key] = true + } + for _, encoded := range visited[start:end] { + var row struct { + Key string `json:"_key"` + } + if err := json.Unmarshal([]byte(encoded), &row); err != nil { + t.Fatalf("decode visited row: %v", err) + } + if !requested[row.Key] { + t.Fatalf("page %d row %v belongs to root %q outside the requested key page %s", pageIndex, encoded, row.Key, keyPages[pageIndex]) + } + if earlier, duplicate := seen[row.Key]; duplicate { + t.Fatalf("root identity %q appears on pages %d and %d: duplicate identities across pages", row.Key, earlier, pageIndex) + } + seen[row.Key] = pageIndex + } + start = end + } + for _, encoded := range visited { + if strings.Contains(encoded, "p09") { + t.Fatalf("scope intruder surfaced in paged output: %s", encoded) + } + } + }) + + t.Run("cancellation between key and row step never dispatches the row step", func(t *testing.T) { + ctx, cancel := context.WithCancel(s.ctx) + defer cancel() + port := &exec04pgPort{s: s} + port.afterCall = func(p *exec04pgPort, kind string, callIndex int) { + if kind == "keys" && callIndex == 1 { + cancel() + } + } + stream.pgRootPage = s.newExec04pgSteps(t, 2) + stream.sourceRows = port.port + err := stream.streamRaw(ctx, func(map[string]any) error { return nil }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("interrupted loop error = %v, want context cancellation", err) + } + keys, rows := port.counts() + if keys != 1 || rows != 0 { + t.Fatalf("port calls keys=%d rows=%d, want 1/0 — the row step must never be dispatched after cancellation", keys, rows) + } + }) + + t.Run("composition sabotage refuses with the sentinel, alive control proceeds", func(t *testing.T) { + sabotaged := s.newExec04pgSteps(t, 2) + sabotaged.RowRootKeyColumn = "not_declared" + port := &exec04pgPort{s: s} + stream.pgRootPage = sabotaged + stream.sourceRows = port.port + err := stream.streamRaw(s.ctx, func(map[string]any) error { return nil }) + if !errors.Is(err, ErrPostgreSQLRootPageSteps) { + t.Fatalf("sabotaged steps error = %v, want ErrPostgreSQLRootPageSteps", err) + } + if len(port.calls) != 0 { + t.Fatalf("sabotaged steps dispatched %d statements before refusing", len(port.calls)) + } + livePort := &exec04pgPort{s: s} + stream.pgRootPage = s.newExec04pgSteps(t, 2) + stream.sourceRows = livePort.port + visited := 0 + if err := stream.streamRaw(s.ctx, func(map[string]any) error { visited++; return nil }); err != nil { + s.fail("alive-control streamRaw", err) + } + if visited != len(want) { + t.Fatalf("alive control visited %d rows, want %d", visited, len(want)) + } + }) +} + +func TestZZPostgresPreviewRootPageZeroLeftoverFixtures(t *testing.T) { + if strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 preview root-page acceptance tests") + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + raw := exec04pgURL(t) + scrub, _ := exec04pgScrubberFor(raw) + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for audit: %s", scrub(err)) + } + defer admin.Close() + for db := range exec04pgRegistrySnapshot() { + var count int + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Fatalf("audit %s: %s", db, scrub(err)) + } + if count != 0 { + t.Errorf("owned fixture database %s survived", db) + } + } + baseline := map[string]bool{} + for _, name := range exec04pgBaselineNames { + baseline[name] = true + } + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec04\\_pg\\_%' ESCAPE '\\'") + if err != nil { + t.Fatalf("audit sweep: %s", scrub(err)) + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("audit scan: %s", scrub(err)) + } + if !baseline[name] { + t.Errorf("unregistered survivor under owned prefix: %s", name) + } + } + if err := rows.Err(); err != nil { + t.Fatalf("audit rows: %s", scrub(err)) + } +} diff --git a/internal/dataframe/execution/related_preview_paging.go b/internal/dataframe/execution/related_preview_paging.go index 355b7d5de..2e44ea08b 100644 --- a/internal/dataframe/execution/related_preview_paging.go +++ b/internal/dataframe/execution/related_preview_paging.go @@ -2,10 +2,6 @@ package execution import "github.com/calypr/loom/internal/dataframe/compiler/ir" -// Related expansions embed the previous row identity as their first identity -// component. With root-key source identity, sorting each root page preserves -// the same preview prefix as a larger page. Start small to avoid expanding -// later roots after an early root has already filled the preview. func relatedExpandRootPageOrder(sequence *ir.PhysicalStageSequence) bool { if sequence == nil || sequence.SourceRowIdentity != "_key" || len(sequence.Stages) == 0 { return false diff --git a/internal/dataframe/execution/row_lineage.go b/internal/dataframe/execution/row_lineage.go index a4de32b04..5fb2e9806 100644 --- a/internal/dataframe/execution/row_lineage.go +++ b/internal/dataframe/execution/row_lineage.go @@ -2,6 +2,7 @@ package execution import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -38,7 +39,7 @@ func (e *Engine) RowLineage(ctx context.Context, resolved Resolved, request RowL if request.Offset < 0 || request.Limit < 0 || request.Limit > compiler.MaxRowLineageContributors { return RowLineageResult{}, fmt.Errorf("row lineage page is outside the supported range") } - compiled, err := compileRowLineageOutput(resolved, request) + compiled, err := e.compileRowLineageOutput(resolved, request) if err != nil { return RowLineageResult{}, err } @@ -46,15 +47,23 @@ func (e *Engine) RowLineage(ctx context.Context, resolved Resolved, request RowL } func (e *Engine) rowLineageCompiled(ctx context.Context, compiled compiler.CompiledRowLineageQuery) (RowLineageResult, error) { - if e == nil || e.queryRows == nil { + if e == nil || e.sourceRows == nil { return RowLineageResult{}, fmt.Errorf("row lineage query executor is required") } if compiled.FoundColumn == "" || compiled.ContributorsColumn == "" || compiled.HasMoreColumn == "" || compiled.Limit < 1 || compiled.Limit > compiler.MaxRowLineageContributors { return RowLineageResult{}, fmt.Errorf("row lineage query is missing its bounded result contract") } + declaredColumns := []string{compiled.FoundColumn, compiled.ContributorsColumn, compiled.HasMoreColumn} + for i := range declaredColumns { + for j := i + 1; j < len(declaredColumns); j++ { + if declaredColumns[i] == declaredColumns[j] { + return RowLineageResult{}, fmt.Errorf("row lineage query is missing its bounded result contract") + } + } + } var result RowLineageResult found := false - if err := e.queryRows(ctx, compiled.Query, e.batchSize, compiled.BindVars, func(row map[string]any) error { + if err := e.sourceRows(ctx, RowsRequest{Command: compiled.Command, DeclaredColumns: declaredColumns, BatchSize: e.batchSize}, func(row map[string]any) error { if err := ctx.Err(); err != nil { return err } @@ -66,9 +75,9 @@ func (e *Engine) rowLineageCompiled(ctx context.Context, compiled compiler.Compi return nil } found = true - contributors, ok := row[compiled.ContributorsColumn].([]any) - if !ok { - return fmt.Errorf("row lineage query returned an invalid contributor page") + contributors, err := rowLineageContributors(row[compiled.ContributorsColumn]) + if err != nil { + return err } if len(contributors) > compiled.Limit { return fmt.Errorf("row lineage query exceeded the requested contributor page size") @@ -109,6 +118,28 @@ func (e *Engine) rowLineageCompiled(ctx context.Context, compiled compiler.Compi return result, nil } +func rowLineageContributors(value any) ([]any, error) { + switch typed := value.(type) { + case []any: + return typed, nil + case json.RawMessage: + if len(typed) == 0 || !json.Valid(typed) { + return nil, fmt.Errorf("row lineage query returned an invalid contributor page") + } + var decoded any + if err := json.Unmarshal(typed, &decoded); err != nil { + return nil, fmt.Errorf("row lineage query returned an invalid contributor page: %w", err) + } + items, ok := decoded.([]any) + if !ok { + return nil, fmt.Errorf("row lineage query returned an invalid contributor page") + } + return items, nil + default: + return nil, fmt.Errorf("row lineage query returned an invalid contributor page") + } +} + func rowLineageString(value any) (string, bool) { text, ok := value.(string) if !ok { @@ -118,10 +149,10 @@ func rowLineageString(value any) (string, bool) { return text, text != "" } -func compileRowLineageOutput(resolved Resolved, request RowLineageRequest) (compiler.CompiledRowLineageQuery, error) { +func (e *Engine) compileRowLineageOutput(resolved Resolved, request RowLineageRequest) (compiler.CompiledRowLineageQuery, error) { for _, output := range resolved.Compiled.Outputs { if output.Name == request.Output { - return compiler.CompileRowLineageOutput(output, request.RowID, request.Offset, request.Limit, ir.DefaultPhysicalOptimizationPolicy()) + return compiler.CompileRowLineageOutput(e.compileTarget, output, request.RowID, request.Offset, request.Limit, ir.DefaultPhysicalOptimizationPolicy()) } } return compiler.CompiledRowLineageQuery{}, fmt.Errorf("row lineage output %q was not found", request.Output) diff --git a/internal/dataframe/execution/row_lineage_postgres_pg18_test.go b/internal/dataframe/execution/row_lineage_postgres_pg18_test.go new file mode 100644 index 000000000..826152b13 --- /dev/null +++ b/internal/dataframe/execution/row_lineage_postgres_pg18_test.go @@ -0,0 +1,1016 @@ +package execution_test + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/calypr/loom/internal/authscope" + postgres "github.com/calypr/loom/internal/backends/postgres" + storepg "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/explorer" + "github.com/calypr/loom/internal/explorer/capability" + "github.com/calypr/loom/internal/explorer/lifecycle" + "github.com/calypr/loom/internal/projectid" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + exec08Prefix = "verify_exec08_rl_" + exec08NameFmt = exec08Prefix + "%d_%s" + + exec08Project = "project-a" + exec08Generation = "gen-a" + exec08AuthPath = "FHIR/Observation" + exec08Explorer = "observations" +) + +const exec08GroupRowID = `[["construction","exec08_group"],["operation","GROUP"],["grouped_status_id","final"]]` + +var exec08NamePattern = regexp.MustCompile(`^verify_exec08_rl_[0-9]+_[0-9a-f]+$`) + +var ( + exec08OwnedMu sync.Mutex + exec08OwnedByName = map[string]string{} +) + +func exec08Register(db, schema string) { + exec08OwnedMu.Lock() + defer exec08OwnedMu.Unlock() + exec08OwnedByName[db] = schema +} + +func exec08RegistrySnapshot() map[string]string { + exec08OwnedMu.Lock() + defer exec08OwnedMu.Unlock() + out := make(map[string]string, len(exec08OwnedByName)) + for db, schema := range exec08OwnedByName { + out[db] = schema + } + return out +} + +var ( + exec08BaselineOnce sync.Once + exec08BaselineNames []string + exec08BaselineErr error +) + +func exec08CaptureBaseline(raw string) { + exec08BaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec08BaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + rows, err := admin.Query(ctx, `SELECT datname FROM pg_database WHERE datname LIKE 'verify\_exec08\_rl\_%' ESCAPE '\' UNION ALL SELECT nspname FROM pg_namespace WHERE nspname LIKE 'verify\_exec08\_rl\_%' ESCAPE '\'`) + if err != nil { + exec08BaselineErr = fmt.Errorf("read baseline: %w", err) + return + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + exec08BaselineErr = fmt.Errorf("scan baseline: %w", err) + return + } + exec08BaselineNames = append(exec08BaselineNames, name) + } + exec08BaselineErr = rows.Err() + }) +} + +func exec08RandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +type exec08Service struct { + ctx context.Context + iso *pgxpool.Pool + schema string + target compiler.Target + scrub func(any) string + fail func(step string, err error) + adapter dataframeexecution.SourceRows +} + +func newExec08Service(t *testing.T) *exec08Service { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 row-lineage acceptance tests") + } + scrub, addSecret := exec04pvScrubberFor(raw) + exec08CaptureBaseline(raw) + if exec08BaselineErr != nil { + t.Fatalf("capture pg_database baseline: %s", scrub(exec08BaselineErr)) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, "SELECT current_setting('server_version_num')::int8, current_setting('server_version')").Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if err := storepg.CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required PostgreSQL 18 backend: %v", err) + } + + dbName := fmt.Sprintf(exec08NameFmt, time.Now().UnixNano(), exec08RandomHex(t)) + if !exec08NamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + var created int64 + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", dbName).Scan(&created); err != nil { + fail("verify isolated database creation", err) + } + if created != 1 { + t.Fatalf("isolated database %q did not verifiably come into existence", dbName) + } + exec08Register(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + addSecret(dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(exec08NameFmt, time.Now().UnixNano(), exec08RandomHex(t)) + if !exec08NamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + exec08Register(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + client, err := storepg.OpenPool(ctx, storepg.ClientConfig{DSN: dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 2 * time.Second}) + if err != nil { + fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + defer client.Close() + if _, err := storepg.MigrateSchema(ctx, client, schema); err != nil { + fail("migrate owned schema", err) + } + + binding, err := ir.NewPostgresStorageBinding(schema) + if err != nil { + fail("bind PostgreSQL storage placement", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + fail("select PostgreSQL compile target", err) + } + adapter, err := postgres.NewSourceRows(iso) + if err != nil { + fail("create PostgreSQL row adapter", err) + } + return &exec08Service{ctx: ctx, iso: iso, schema: schema, target: target, scrub: scrub, fail: fail, adapter: adapter} +} + +func exec08Bundle() recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "exec08_row_lineage", + TranslationVersion: "exec08", + Outputs: []recipe.Output{{ + Name: exec08Explorer, RootResourceType: "Observation", RowGrain: "observation", + Fields: []recipe.Field{{Name: "status", ColumnID: "status_id", Expr: recipe.Expression{Select: "root.status"}}}, + Construction: &recipe.Construction{ + Version: 1, + SourceColumns: []recipe.StageColumn{{ID: "status_id", Name: "status"}}, + Steps: []recipe.ConstructionStep{{ + ID: "group_status", + Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionGroupOp, Group: &recipe.ConstructionGroup{ + ConstructionID: "exec08_group", + MissingKeyPolicy: recipe.ConstructionGroupMissingKeyGroup, + Keys: []recipe.ConstructionGroupKey{{InputColumnID: "status_id", OutputColumnID: "grouped_status_id"}}, + Aggregates: []recipe.ConstructionGroupAggregate{{Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}}, + }}, + Outputs: []recipe.StageColumn{{ID: "grouped_status_id", Name: "status"}, {ID: "rows_id", Name: "rows", Type: "integer"}}, + }}, + }, + }}, + } +} + +func exec08Bindings(project, generation string) recipe.RuntimeBindings { + return recipe.RuntimeBindings{ + Project: project, DatasetGeneration: generation, + AuthResourcePaths: []string{exec08AuthPath}, + } +} + +type exec08SeedRow struct { + project, generation, logicalID, authPath, status string +} + +func (s *exec08Service) seed(t *testing.T, rows []exec08SeedRow) { + t.Helper() + repo, err := storepg.NewResourceRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + for _, row := range rows { + encoded, err := json.Marshal(map[string]any{"id": row.logicalID, "status": row.status}) + if err != nil { + t.Fatalf("encode payload for %s: %v", row.logicalID, err) + } + if _, err := repo.InsertResource(s.ctx, source.Resource{ + Project: row.project, + DatasetGeneration: row.generation, + ResourceType: "Observation", + LogicalID: row.logicalID, + AuthResourcePath: row.authPath, + Payload: json.RawMessage(encoded), + }); err != nil { + s.fail("insert fixture resource "+row.logicalID, err) + } + } +} + +func (s *exec08Service) seedStoredNullID(t *testing.T, project, generation, logicalID, authPath, status string) { + t.Helper() + repo, err := storepg.NewResourceRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + if _, err := repo.InsertResource(s.ctx, source.Resource{ + Project: project, DatasetGeneration: generation, ResourceType: "Observation", + LogicalID: logicalID, AuthResourcePath: authPath, + Payload: json.RawMessage(`{"id":null,"status":"` + status + `"}`), + }); err != nil { + s.fail("insert null-id fixture resource "+logicalID, err) + } +} + +type exec08Call struct { + sql string + args []command.PostgreSQLParameter + declared []string + tag command.CompiledBackend +} + +type exec08Recorder struct { + adapter dataframeexecution.SourceRows + mu sync.Mutex + calls []exec08Call +} + +func (r *exec08Recorder) port(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + sqlText, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + r.mu.Lock() + r.calls = append(r.calls, exec08Call{sql: sqlText, args: args, declared: append([]string(nil), request.DeclaredColumns...), tag: request.Command.Backend()}) + r.mu.Unlock() + return r.adapter(ctx, request, visit) +} + +func (r *exec08Recorder) count() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.calls) +} + +func (r *exec08Recorder) last() exec08Call { + r.mu.Lock() + defer r.mu.Unlock() + return r.calls[len(r.calls)-1] +} + +func (s *exec08Service) engine(t *testing.T, recorder *exec08Recorder) *dataframeexecution.Engine { + t.Helper() + config := exec04pvForbidArango() + config.CompileTarget = s.target + config.PreviewSourceRows = nil + config.ScopeDigest = func(recipe.RuntimeBindings) string { return "exec08-scope" } + config.SourceRows = recorder.port + engine, err := dataframeexecution.New(config) + if err != nil { + t.Fatalf("engine: %v", s.scrub(err)) + } + return engine +} + +func (s *exec08Service) previewIdentity(t *testing.T, engine *dataframeexecution.Engine, resolved dataframeexecution.Resolved, wantStatus string) string { + t.Helper() + var identity string + matched := 0 + _, err := engine.PreviewOutput(s.ctx, resolved, dataframeexecution.PreviewRequest{Output: exec08Explorer, Limit: 50, IncludeRowIdentity: true}, func(row map[string]any) error { + if row["status"] == wantStatus { + matched++ + cell, ok := row["__loom_row_id"].(string) + if !ok || cell == "" { + t.Fatalf("preview row did not publish its stable identity: %#v", row) + } + identity = cell + } + return nil + }) + if err != nil { + t.Fatalf("preview: %s", s.scrub(err)) + } + if matched != 1 { + t.Fatalf("preview published %d rows for status %q, want exactly one", matched, wantStatus) + } + return identity +} + +type contributorTriple struct { + resourceType string + resourceID string + occurrence string +} + +func exec08TriplesFrom(contributors []dataframeexecution.RowLineageContributor) []contributorTriple { + out := make([]contributorTriple, 0, len(contributors)) + for _, contributor := range contributors { + out = append(out, contributorTriple{contributor.ResourceType, contributor.ResourceID, contributor.OccurrenceKey}) + } + return out +} + +func exec08Triples(result dataframeexecution.RowLineageResult) []contributorTriple { + return exec08TriplesFrom(result.Contributors) +} + +func exec08WantTriples(ids ...string) []contributorTriple { + out := make([]contributorTriple, 0, len(ids)) + for _, id := range ids { + out = append(out, contributorTriple{"Observation", id, id}) + } + return out +} + +func exec08TriplesEqual(got, want []contributorTriple) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true +} + +func TestPostgresRowLineageKnownRowExplainsContributors(t *testing.T) { + s := newExec08Service(t) + s.seed(t, []exec08SeedRow{ + {exec08Project, exec08Generation, "obs-a", exec08AuthPath, "final"}, + {exec08Project, exec08Generation, "obs-b", exec08AuthPath, "final"}, + {exec08Project, exec08Generation, "obs-c", exec08AuthPath, "amended"}, + }) + recorder := &exec08Recorder{adapter: s.adapter} + engine := s.engine(t, recorder) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec08Bundle(), exec08Bindings(exec08Project, exec08Generation)) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + identity := s.previewIdentity(t, engine, resolved, "final") + before := recorder.count() + result, err := engine.RowLineage(s.ctx, resolved, dataframeexecution.RowLineageRequest{Output: exec08Explorer, RowID: identity, Offset: 0, Limit: 25}) + if err != nil { + t.Fatalf("RowLineage: %s", s.scrub(err)) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("lineage ran %d statements, want exactly one", got) + } + call := recorder.last() + if call.tag != command.CompiledBackendPostgreSQL { + t.Fatalf("lineage command tag = %q", call.tag) + } + if want := []string{"found", "contributors", "hasMore"}; strings.Join(call.declared, ",") != strings.Join(want, ",") { + t.Fatalf("declared column contract = %v, want %v", call.declared, want) + } + if want := exec08WantTriples("obs-a", "obs-b"); !exec08TriplesEqual(exec08Triples(result), want) { + t.Fatalf("contributors = %v, want %v", exec08Triples(result), want) + } + if result.HasMore || result.NextOffset != 0 { + t.Fatalf("paging summary = %#v, want complete page", result) + } + again, err := engine.RowLineage(s.ctx, resolved, dataframeexecution.RowLineageRequest{Output: exec08Explorer, RowID: identity, Offset: 0, Limit: 25}) + if err != nil || !exec08TriplesEqual(exec08Triples(again), exec08WantTriples("obs-a", "obs-b")) { + t.Fatalf("page drifted between executions: %#v (%s)", again, s.scrub(err)) + } + for _, leaked := range []string{exec08Project, exec08Generation, exec08AuthPath, "final", "obs-a", identity} { + if strings.Contains(call.sql, leaked) { + t.Fatalf("value %q leaked into statement text: %s", leaked, call.sql) + } + } + boundIdentity := false + for _, arg := range call.args { + if text, ok := arg.Value.(string); ok && text == identity { + boundIdentity = true + } + } + if !boundIdentity { + t.Fatalf("the requested row identity never arrived as a bound parameter") + } +} + +func TestPostgresRowLineageUnknownRowReturnsTypedNotFound(t *testing.T) { + s := newExec08Service(t) + s.seed(t, []exec08SeedRow{ + {exec08Project, exec08Generation, "obs-a", exec08AuthPath, "final"}, + }) + recorder := &exec08Recorder{adapter: s.adapter} + engine := s.engine(t, recorder) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec08Bundle(), exec08Bindings(exec08Project, exec08Generation)) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + unknown := `[["construction","exec08_group"],["operation","GROUP"],["grouped_status_id","cancelled"]]` + before := recorder.count() + result, err := engine.RowLineage(s.ctx, resolved, dataframeexecution.RowLineageRequest{Output: exec08Explorer, RowID: unknown, Offset: 0, Limit: 25}) + if !errors.Is(err, dataframeexecution.ErrRowLineageRowNotFound) { + t.Fatalf("unknown row = %#v, %v, want the typed not-found", result, err) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("unknown-row lineage ran %d statements, want exactly one", got) + } + if len(result.Contributors) != 0 || result.HasMore { + t.Fatalf("unknown-row result leaked a page: %#v", result) + } +} + +func TestPostgresRowLineagePagingAdvancesWithoutDuplicatesAndStopsAtTerminalPage(t *testing.T) { + s := newExec08Service(t) + s.seed(t, []exec08SeedRow{ + {exec08Project, exec08Generation, "obs-a", exec08AuthPath, "final"}, + {exec08Project, exec08Generation, "obs-b", exec08AuthPath, "final"}, + {exec08Project, exec08Generation, "obs-c", exec08AuthPath, "final"}, + {exec08Project, exec08Generation, "obs-d", exec08AuthPath, "final"}, + {exec08Project, exec08Generation, "obs-e", exec08AuthPath, "final"}, + }) + recorder := &exec08Recorder{adapter: s.adapter} + engine := s.engine(t, recorder) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec08Bundle(), exec08Bindings(exec08Project, exec08Generation)) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + identity := s.previewIdentity(t, engine, resolved, "final") + + var pages [][]contributorTriple + var hasMore []bool + seen := map[contributorTriple]int{} + offset := 0 + for { + before := recorder.count() + result, err := engine.RowLineage(s.ctx, resolved, dataframeexecution.RowLineageRequest{Output: exec08Explorer, RowID: identity, Offset: offset, Limit: 2}) + if err != nil { + t.Fatalf("RowLineage offset %d: %s", offset, s.scrub(err)) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("offset %d ran %d statements, want exactly one", offset, got) + } + triples := exec08Triples(result) + for _, triple := range triples { + if prior, dup := seen[triple]; dup { + t.Fatalf("contributor %v repeated on page offset %d (first at offset %d)", triple, offset, prior) + } + seen[triple] = offset + } + pages = append(pages, triples) + hasMore = append(hasMore, result.HasMore) + if !result.HasMore { + if result.NextOffset != 0 { + t.Fatalf("terminal page set NextOffset %d, want 0", result.NextOffset) + } + break + } + if result.NextOffset != offset+len(triples) { + t.Fatalf("offset %d: NextOffset = %d, want %d", offset, result.NextOffset, offset+len(triples)) + } + if len(triples) == 0 { + t.Fatalf("offset %d: HasMore=true with an empty page — the loop cannot advance", offset) + } + offset = result.NextOffset + if len(pages) > 10 { + t.Fatalf("paging did not converge to the terminal page") + } + } + if len(pages) != 3 { + t.Fatalf("pages served = %d, want three", len(pages)) + } + if len(pages[0]) != 2 || len(pages[1]) != 2 || len(pages[2]) != 1 { + t.Fatalf("page sizes = %d,%d,%d, want 2,2,1", len(pages[0]), len(pages[1]), len(pages[2])) + } + if !hasMore[0] || !hasMore[1] || hasMore[2] { + t.Fatalf("hasMore progression = %v, want true,true,false", hasMore) + } + var union []contributorTriple + for _, page := range pages { + union = append(union, page...) + } + if want := exec08WantTriples("obs-a", "obs-b", "obs-c", "obs-d", "obs-e"); !exec08TriplesEqual(union, want) { + t.Fatalf("page union = %v, want %v", union, want) + } +} + +func TestPostgresRowLineageForeignWitnessesAreExcluded(t *testing.T) { + s := newExec08Service(t) + s.seed(t, []exec08SeedRow{ + {exec08Project, exec08Generation, "obs-a", exec08AuthPath, "final"}, + {exec08Project, exec08Generation, "obs-b", exec08AuthPath, "final"}, + {"project-b", exec08Generation, "obs-foreign-project", exec08AuthPath, "final"}, + {exec08Project, "gen-b", "obs-foreign-generation", exec08AuthPath, "final"}, + {exec08Project, exec08Generation, "obs-foreign-auth", "FHIR/Restricted", "final"}, + }) + recorder := &exec08Recorder{adapter: s.adapter} + engine := s.engine(t, recorder) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec08Bundle(), exec08Bindings(exec08Project, exec08Generation)) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + identity := s.previewIdentity(t, engine, resolved, "final") + before := recorder.count() + result, err := engine.RowLineage(s.ctx, resolved, dataframeexecution.RowLineageRequest{Output: exec08Explorer, RowID: identity, Offset: 0, Limit: 25}) + if err != nil { + t.Fatalf("RowLineage: %s", s.scrub(err)) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("lineage ran %d statements, want exactly one", got) + } + if want := exec08WantTriples("obs-a", "obs-b"); !exec08TriplesEqual(exec08Triples(result), want) { + t.Fatalf("foreign witnesses leaked: contributors = %v, want %v", exec08Triples(result), want) + } + if result.HasMore { + t.Fatalf("foreign witnesses inflated hasMore: %#v", result) + } + groups := 0 + rows := 0 + _, err = engine.PreviewOutput(s.ctx, resolved, dataframeexecution.PreviewRequest{Output: exec08Explorer, Limit: 50}, func(row map[string]any) error { + if row["status"] != "final" { + t.Fatalf("out-of-scope witness row reached the preview: %#v", row) + } + groups++ + if count, ok := row["rows"].(int64); ok { + rows = int(count) + } + return nil + }) + if err != nil || groups != 1 || rows != 2 { + t.Fatalf("in-scope preview affected by foreign witnesses: groups=%d rows=%d err=%s", groups, rows, s.scrub(err)) + } +} + +func TestPostgresRowLineageMalformedContributorIdentityIsContractError(t *testing.T) { + s := newExec08Service(t) + s.seed(t, []exec08SeedRow{ + {exec08Project, exec08Generation, "obs-clean", exec08AuthPath, "final-clean"}, + }) + s.seedStoredNullID(t, exec08Project, exec08Generation, "obs-null-id", exec08AuthPath, "final-null") + recorder := &exec08Recorder{adapter: s.adapter} + engine := s.engine(t, recorder) + resolved, err := engine.CompileResolvedBundle(s.ctx, exec08Bundle(), exec08Bindings(exec08Project, exec08Generation)) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + nullIdentity := s.previewIdentity(t, engine, resolved, "final-null") + before := recorder.count() + _, err = engine.RowLineage(s.ctx, resolved, dataframeexecution.RowLineageRequest{Output: exec08Explorer, RowID: nullIdentity, Offset: 0, Limit: 25}) + if err == nil || !strings.Contains(err.Error(), "without a FHIR resource identity") { + t.Fatalf("null-id contributor accepted: %v, want the contract error", s.scrub(err)) + } + if got := recorder.count() - before; got != 1 { + t.Fatalf("malformed page ran %d statements, want exactly one (refused at decode, not before execution)", got) + } + cleanIdentity := s.previewIdentity(t, engine, resolved, "final-clean") + result, err := engine.RowLineage(s.ctx, resolved, dataframeexecution.RowLineageRequest{Output: exec08Explorer, RowID: cleanIdentity, Offset: 0, Limit: 25}) + if err != nil || !exec08TriplesEqual(exec08Triples(result), exec08WantTriples("obs-clean")) { + t.Fatalf("clean control failed: %#v %s", result, s.scrub(err)) + } +} + +type exec08ExplorerStore struct { + explorer.Store +} + +type exec08Authorized struct { + mu sync.Mutex + snapshot capability.Snapshot + scope authscope.ReadScope + hookCalls int +} + +func (a *exec08Authorized) set(snapshot capability.Snapshot, scope authscope.ReadScope) { + a.mu.Lock() + defer a.mu.Unlock() + a.snapshot = snapshot + a.scope = scope +} + +func (a *exec08Authorized) authorized() (capability.Snapshot, authscope.ReadScope) { + a.mu.Lock() + defer a.mu.Unlock() + return a.snapshot, a.scope +} + +func (a *exec08Authorized) calls() int { + a.mu.Lock() + defer a.mu.Unlock() + return a.hookCalls +} + +func exec08ScopeDigest(scope authscope.ReadScope) string { + paths := append([]string(nil), scope.AuthResourcePaths...) + sort.Strings(paths) + sum := sha256.Sum256([]byte(string(scope.Mode) + "\x00" + strings.Join(paths, "\x00"))) + return hex.EncodeToString(sum[:]) +} + +func exec08Snapshot(project, generation, token string, scope authscope.ReadScope) capability.Snapshot { + return capability.Snapshot{ + Identity: capability.SnapshotIdentity{ + Project: project, Generation: generation, + AuthorizationScopeDigest: exec08ScopeDigest(scope), + SchemaDigest: "exec08-schema", ShapeDigest: "exec08-shape", + }, + Status: capability.StatusReady, Complete: true, Token: token, + } +} + +func exec08Receipt(t *testing.T, snapshot capability.Snapshot) explorer.CompilationReceipt { + t.Helper() + bundle := exec08Bundle() + contract := json.RawMessage(`{"outputs":[{"outputId":"observations","columns":[{"column":"status","label":"Status","logicalType":"string","filterable":true,"chartable":true},{"column":"rows","label":"Rows","logicalType":"integer","filterable":true,"chartable":false}]}]}`) + emitted := []explorer.EmittedColumn{ + {OutputID: exec08Explorer, EmissionID: "em_status", PublicColumn: "status", Label: "Status", LogicalType: "string", Filterable: true, Chartable: true}, + {OutputID: exec08Explorer, EmissionID: "em_rows", PublicColumn: "rows", Label: "Rows", LogicalType: "integer", Filterable: true}, + } + bundleDigest, err := bundle.Digest() + if err != nil { + t.Fatalf("bundle digest: %v", err) + } + receipt := explorer.CompilationReceipt{ + ReceiptFormatVersion: explorer.CurrentReceiptFormatVersion, + CompilerContractVersion: explorer.CurrentCompilerContractVersion, + Project: snapshot.Identity.Project, + ExplorerID: exec08Explorer, + IntentDigest: "exec08-intent", + SnapshotToken: snapshot.Token, + AuthorizationScopeDigest: snapshot.Identity.AuthorizationScopeDigest, + CapabilitySchemaDigest: snapshot.Identity.SchemaDigest, + ShapeDigest: snapshot.Identity.ShapeDigest, + SourceGeneration: snapshot.Identity.Generation, + RecipeDigest: bundleDigest, + ResolvedRecipeDigest: bundleDigest, + ResolvedSchemaDigest: "exec08-resolved-schema", + CompiledConfig: json.RawMessage(`{}`), + PublicOutputContract: contract, + Bundle: bundle, + EmittedColumns: emitted, + OutputColumnProvenance: map[string]map[string]string{exec08Explorer: {"status": "EXPLICIT", "rows": "EXPLICIT"}}, + } + receipt.OutputContractDigest, err = explorer.CompilationArtifactDigest(contract) + if err != nil { + t.Fatalf("contract digest: %v", err) + } + receipt.CompilationKey, err = explorer.CompilationKey(receipt) + if err != nil { + t.Fatalf("compilation key: %v", err) + } + receipt.ID, err = explorer.ReceiptID(receipt) + if err != nil { + t.Fatalf("receipt id: %v", err) + } + return receipt +} + +func exec08LifecycleService(t *testing.T, authorized *exec08Authorized, engine *dataframeexecution.Engine, receipts ...explorer.CompilationReceipt) *lifecycle.Service { + t.Helper() + byID := map[string]explorer.CompilationReceipt{} + for _, receipt := range receipts { + byID[receipt.ID] = receipt + } + domain, err := explorer.NewService(&exec08ExplorerStore{}) + if err != nil { + t.Fatalf("explorer service: %v", err) + } + config := lifecycle.Config{ + Capability: lifecycle.CapabilityResolver{ + ForExecution: func(context.Context, string, string) (lifecycle.AuthorizedCapability, error) { + snapshot, scope := authorized.authorized() + return lifecycle.AuthorizedCapability{Snapshot: snapshot, Scope: scope}, nil + }, + }, + ReceiptLookup: func(_ context.Context, _, _, receiptID string) (*explorer.CompilationReceipt, error) { + got, ok := byID[receiptID] + if !ok { + return nil, fmt.Errorf("receipt %q is not in this test's store", receiptID) + } + return &got, nil + }, + } + config.RowLineage = func(ctx context.Context, got *explorer.CompilationReceipt, bindings recipe.RuntimeBindings, request dataframeexecution.RowLineageRequest) (dataframeexecution.RowLineageResult, error) { + authorized.mu.Lock() + authorized.hookCalls++ + authorized.mu.Unlock() + resolved, err := engine.CompileResolvedBundle(ctx, got.Bundle, bindings) + if err != nil { + return dataframeexecution.RowLineageResult{}, err + } + return engine.RowLineage(ctx, resolved, request) + } + service, err := lifecycle.New(domain, config) + if err != nil { + t.Fatalf("lifecycle service: %v", err) + } + return service +} + +func exec08LifecycleRequest(receipt explorer.CompilationReceipt, rowID string, offset, limit int) lifecycle.RowLineageRequest { + return lifecycle.RowLineageRequest{ + Project: receipt.Project, ExplorerID: receipt.ExplorerID, ReceiptID: receipt.ID, + OutputID: exec08Explorer, RowID: rowID, Offset: offset, Limit: limit, + } +} + +func exec08OfflineEngine(t *testing.T, calls *[]exec08Call, mu *sync.Mutex) *dataframeexecution.Engine { + t.Helper() + fakeAdapter := func(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + sqlText, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + mu.Lock() + *calls = append(*calls, exec08Call{sql: sqlText, args: args, declared: append([]string(nil), request.DeclaredColumns...), tag: request.Command.Backend()}) + mu.Unlock() + return visit(map[string]any{"found": true, "contributors": json.RawMessage(`[{"resourceType":"Observation","resourceId":"obs-a","occurrenceKey":"obs-a"}]`), "hasMore": false}) + } + binding, err := ir.NewPostgresStorageBinding("loom") + if err != nil { + t.Fatalf("binding: %v", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("target: %v", err) + } + config := exec04pvForbidArango() + config.CompileTarget = target + config.PreviewSourceRows = nil + config.ScopeDigest = func(recipe.RuntimeBindings) string { return "exec08-scope" } + config.SourceRows = fakeAdapter + engine, err := dataframeexecution.New(config) + if err != nil { + t.Fatalf("engine: %v", err) + } + return engine +} + +func TestPostgresRowLineageStaleReceiptClassification(t *testing.T) { + scope := authscope.ReadScope{Mode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{exec08AuthPath}} + fresh := exec08Snapshot(exec08Project, exec08Generation, "token-1", scope) + receipt := exec08Receipt(t, fresh) + authorized := &exec08Authorized{snapshot: fresh, scope: scope} + var calls []exec08Call + var mu sync.Mutex + engine := exec08OfflineEngine(t, &calls, &mu) + service := exec08LifecycleService(t, authorized, engine, receipt) + + result, err := service.RowLineage(context.Background(), exec08LifecycleRequest(receipt, exec08GroupRowID, 0, 25)) + if err != nil || len(result.Contributors) != 1 || result.Contributors[0].ResourceID != "obs-a" { + t.Fatalf("fresh receipt failed: %#v %v", result, err) + } + if authorized.calls() != 1 { + t.Fatalf("fresh control hook witness: %d invocations, want 1", authorized.calls()) + } + mu.Lock() + freshStatements := len(calls) + mu.Unlock() + if freshStatements < 1 { + t.Fatalf("fresh control executed %d statements, want at least one", freshStatements) + } + + for _, test := range []struct { + name string + mutate func(snapshot capability.Snapshot, scope authscope.ReadScope) (capability.Snapshot, authscope.ReadScope) + }{ + {"generation moved on", func(snapshot capability.Snapshot, scope authscope.ReadScope) (capability.Snapshot, authscope.ReadScope) { + snapshot.Identity.Generation = "gen-b" + return snapshot, scope + }}, + {"authorization scope digest changed", func(snapshot capability.Snapshot, scope authscope.ReadScope) (capability.Snapshot, authscope.ReadScope) { + scope = authscope.ReadScope{Mode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{"FHIR/Other"}} + snapshot.Identity.AuthorizationScopeDigest = exec08ScopeDigest(scope) + return snapshot, scope + }}, + {"snapshot token rotated", func(snapshot capability.Snapshot, scope authscope.ReadScope) (capability.Snapshot, authscope.ReadScope) { + snapshot.Token = "token-2" + return snapshot, scope + }}, + } { + t.Run(test.name, func(t *testing.T) { + authorized.set(test.mutate(fresh, scope)) + hooksBefore := authorized.calls() + mu.Lock() + statementsBefore := len(calls) + mu.Unlock() + _, err := service.RowLineage(context.Background(), exec08LifecycleRequest(receipt, exec08GroupRowID, 0, 25)) + var lifecycleErr *lifecycle.Error + if !errors.As(err, &lifecycleErr) || lifecycleErr.Class != lifecycle.ClassConflict || lifecycleErr.Code != "RECEIPT_STALE" { + t.Fatalf("stale receipt error = %v, want ClassConflict/RECEIPT_STALE", err) + } + if authorized.calls() != hooksBefore { + t.Fatalf("stale receipt reached the row-lineage hook: %d extra invocations", authorized.calls()-hooksBefore) + } + mu.Lock() + executed := len(calls) - statementsBefore + mu.Unlock() + if executed != 0 { + t.Fatalf("stale receipt executed %d statements, want the statement-count-0 witness", executed) + } + }) + } + + authorized.set(fresh, scope) + if _, err := service.RowLineage(context.Background(), exec08LifecycleRequest(receipt, exec08GroupRowID, 0, 25)); err != nil { + t.Fatalf("restored fresh receipt failed: %v", err) + } +} + +func TestPostgresRowLineageStaleReceiptRejectsBeforeExecution(t *testing.T) { + s := newExec08Service(t) + seedProject := projectid.Legacy(exec08Project) + scope := authscope.ReadScope{Mode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{exec08AuthPath}} + fresh := exec08Snapshot(exec08Project, exec08Generation, "token-1", scope) + receipt := exec08Receipt(t, fresh) + + s.seed(t, []exec08SeedRow{ + {seedProject, exec08Generation, "obs-a", exec08AuthPath, "final"}, + {seedProject, exec08Generation, "obs-b", exec08AuthPath, "final"}, + {seedProject, "gen-b", "obs-new", exec08AuthPath, "final"}, + }) + + recorder := &exec08Recorder{adapter: s.adapter} + engine := s.engine(t, recorder) + authorized := &exec08Authorized{snapshot: fresh, scope: scope} + current := exec08Snapshot(exec08Project, "gen-b", "token-2", scope) + currentReceipt := exec08Receipt(t, current) + service := exec08LifecycleService(t, authorized, engine, receipt, currentReceipt) + + identity := s.previewIdentity(t, engine, mustExec08Resolve(t, s, engine, seedProject, exec08Generation), "final") + result, err := service.RowLineage(s.ctx, exec08LifecycleRequest(receipt, identity, 0, 25)) + if err != nil || !exec08TriplesEqual(exec08TriplesFrom(result.Contributors), exec08WantTriples("obs-a", "obs-b")) { + t.Fatalf("fresh receipt failed on live state: %#v %s", result, s.scrub(err)) + } + + stale := fresh + stale.Identity.Generation = "gen-b" + authorized.set(stale, scope) + hooksBefore := authorized.calls() + statementsBefore := recorder.count() + _, err = service.RowLineage(s.ctx, exec08LifecycleRequest(receipt, identity, 0, 25)) + var lifecycleErr *lifecycle.Error + if !errors.As(err, &lifecycleErr) || lifecycleErr.Class != lifecycle.ClassConflict || lifecycleErr.Code != "RECEIPT_STALE" { + t.Fatalf("stale receipt error = %v, want ClassConflict/RECEIPT_STALE", s.scrub(err)) + } + if authorized.calls() != hooksBefore { + t.Fatalf("stale receipt reached the engine hook: %d extra invocations", authorized.calls()-hooksBefore) + } + if got := recorder.count() - statementsBefore; got != 0 { + t.Fatalf("stale receipt executed %d statements, want the statement-count-0 witness at the production adapter", got) + } + + authorized.set(current, scope) + newIdentity := s.previewIdentity(t, engine, mustExec08Resolve(t, s, engine, seedProject, "gen-b"), "final") + freshResult, err := service.RowLineage(s.ctx, exec08LifecycleRequest(currentReceipt, newIdentity, 0, 25)) + if err != nil { + t.Fatalf("current receipt failed: %s", s.scrub(err)) + } + if !exec08TriplesEqual(exec08TriplesFrom(freshResult.Contributors), exec08WantTriples("obs-new")) { + t.Fatalf("current receipt contributors = %v, want the gen-b row", exec08TriplesFrom(freshResult.Contributors)) + } +} + +func mustExec08Resolve(t *testing.T, s *exec08Service, engine *dataframeexecution.Engine, project, generation string) dataframeexecution.Resolved { + t.Helper() + resolved, err := engine.CompileResolvedBundle(s.ctx, exec08Bundle(), exec08Bindings(project, generation)) + if err != nil { + t.Fatalf("CompileResolvedBundle(%s/%s): %s", project, generation, s.scrub(err)) + } + return resolved +} + +func TestZZPostgresRowLineageZeroLeftoverFixtures(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + return + } + exec08CaptureBaseline(raw) + if exec08BaselineErr != nil { + t.Fatalf("baseline: %v", exec08BaselineErr) + } + baseline := map[string]bool{} + for _, name := range exec08BaselineNames { + baseline[name] = true + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for audit: %v", err) + } + defer admin.Close() + var survivors []string + rows, err := admin.Query(ctx, `SELECT datname FROM pg_database WHERE datname LIKE 'verify\_exec08\_rl\_%' ESCAPE '\'`) + if err != nil { + t.Fatalf("audit databases: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("scan audit: %v", err) + } + if !baseline[name] { + survivors = append(survivors, name) + } + } + rows.Close() + rows, err = admin.Query(ctx, `SELECT nspname FROM pg_namespace WHERE nspname LIKE 'verify\_exec08\_rl\_%' ESCAPE '\'`) + if err != nil { + t.Fatalf("audit schemas: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("scan audit: %v", err) + } + if !baseline[name] { + survivors = append(survivors, name) + } + } + rows.Close() + if len(survivors) != 0 { + sort.Strings(survivors) + t.Fatalf("verify_exec08_rl_* fixtures survived the run: %v (owned registry: %v)", survivors, exec08RegistrySnapshot()) + } +} diff --git a/internal/dataframe/execution/row_lineage_test.go b/internal/dataframe/execution/row_lineage_test.go index 5475a9ca2..aa7876e18 100644 --- a/internal/dataframe/execution/row_lineage_test.go +++ b/internal/dataframe/execution/row_lineage_test.go @@ -2,14 +2,26 @@ package execution import ( "context" + "encoding/json" "errors" + "reflect" + "strings" "testing" "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/recipe" ) func TestRowLineagePageIsBoundedAndKeepsOccurrenceIdentity(t *testing.T) { - engine := &Engine{batchSize: 32, queryRows: func(_ context.Context, query string, batchSize int, binds map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), batchSize: 32, sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + batchSize := request.BatchSize + query, binds, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if batchSize != 32 || query != "row-lineage-query" || binds["row_id"] != "row-7" { t.Fatalf("unexpected query invocation: batch=%d query=%q binds=%#v", batchSize, query, binds) } @@ -22,8 +34,8 @@ func TestRowLineagePageIsBoundedAndKeepsOccurrenceIdentity(t *testing.T) { }) }} result, err := engine.rowLineageCompiled(context.Background(), compiler.CompiledRowLineageQuery{ - Query: "row-lineage-query", BindVars: map[string]any{"row_id": "row-7"}, - Offset: 24, Limit: 1, FoundColumn: "found", ContributorsColumn: "contributors", HasMoreColumn: "hasMore", + Command: mustAQLCommand("row-lineage-query", map[string]any{"row_id": "row-7"}), + Offset: 24, Limit: 1, FoundColumn: "found", ContributorsColumn: "contributors", HasMoreColumn: "hasMore", }) if err != nil { t.Fatal(err) @@ -34,20 +46,32 @@ func TestRowLineagePageIsBoundedAndKeepsOccurrenceIdentity(t *testing.T) { } func TestRowLineageDistinguishesMissingKeyedGroupFromEmptyKeylessGroup(t *testing.T) { - engine := &Engine{queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{"found": false, "contributors": []any{}, "hasMore": false}) }} _, err := engine.rowLineageCompiled(context.Background(), compiler.CompiledRowLineageQuery{ - Query: "keyed-group-query", Limit: 25, FoundColumn: "found", ContributorsColumn: "contributors", HasMoreColumn: "hasMore", + Command: mustAQLCommand("keyed-group-query", nil), Limit: 25, FoundColumn: "found", ContributorsColumn: "contributors", HasMoreColumn: "hasMore", }) if !errors.Is(err, ErrRowLineageRowNotFound) { t.Fatalf("keyed zero-match Group error = %v, want row not found", err) } - engine.queryRows = func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine.sourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{"found": true, "contributors": []any{}, "hasMore": false}) } result, err := engine.rowLineageCompiled(context.Background(), compiler.CompiledRowLineageQuery{ - Query: "keyless-empty-group-query", Limit: 25, FoundColumn: "found", ContributorsColumn: "contributors", HasMoreColumn: "hasMore", + Command: mustAQLCommand("keyless-empty-group-query", nil), Limit: 25, FoundColumn: "found", ContributorsColumn: "contributors", HasMoreColumn: "hasMore", }) if err != nil { t.Fatal(err) @@ -58,16 +82,323 @@ func TestRowLineageDistinguishesMissingKeyedGroupFromEmptyKeylessGroup(t *testin } func TestRowLineageRejectsQueryResultsLargerThanRequestedPage(t *testing.T) { - engine := &Engine{queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{"found": true, "contributors": []any{ map[string]any{"resourceType": "Observation", "resourceId": "one", "occurrenceKey": "one"}, map[string]any{"resourceType": "Observation", "resourceId": "two", "occurrenceKey": "two"}, }, "hasMore": false}) }} _, err := engine.rowLineageCompiled(context.Background(), compiler.CompiledRowLineageQuery{ - Query: "row-lineage-query", Limit: 1, FoundColumn: "found", ContributorsColumn: "contributors", HasMoreColumn: "hasMore", + Command: mustAQLCommand("row-lineage-query", nil), Limit: 1, FoundColumn: "found", ContributorsColumn: "contributors", HasMoreColumn: "hasMore", }) if err == nil { t.Fatal("row lineage accepted more contributors than the requested page size") } } + +func exec08GroupBundle() recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "exec08_row_lineage", + TranslationVersion: "exec08", + Outputs: []recipe.Output{{ + Name: "observations", RootResourceType: "Observation", RowGrain: "observation", + Fields: []recipe.Field{{Name: "status", ColumnID: "status_id", Expr: recipe.Expression{Select: "root.status"}}}, + Construction: &recipe.Construction{ + Version: 1, + SourceColumns: []recipe.StageColumn{{ID: "status_id", Name: "status"}}, + Steps: []recipe.ConstructionStep{{ + ID: "group_status", + Inputs: []recipe.ConstructionInputRef{{Kind: recipe.ConstructionSourceProjectionInput}}, + Operation: recipe.ConstructionOperation{Kind: recipe.ConstructionGroupOp, Group: &recipe.ConstructionGroup{ + ConstructionID: "exec08_group", + MissingKeyPolicy: recipe.ConstructionGroupMissingKeyGroup, + Keys: []recipe.ConstructionGroupKey{{InputColumnID: "status_id", OutputColumnID: "grouped_status_id"}}, + Aggregates: []recipe.ConstructionGroupAggregate{{Operation: recipe.ConstructionGroupCountRows, OutputColumnID: "rows_id"}}, + }}, + Outputs: []recipe.StageColumn{{ID: "grouped_status_id", Name: "status"}, {ID: "rows_id", Name: "rows", Type: "integer"}}, + }}, + }, + }}, + } +} + +type exec08PortCall struct { + tag command.CompiledBackend + args []command.PostgreSQLParameter + declared []string + batch int +} + +func exec08pgPort(calls *[]exec08PortCall, batches *[]int, serve func(callIndex int) []map[string]any) func(context.Context, RowsRequest, RowVisitor) error { + return func(_ context.Context, request RowsRequest, visit RowVisitor) error { + _, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + *calls = append(*calls, exec08PortCall{tag: request.Command.Backend(), args: args, declared: append([]string(nil), request.DeclaredColumns...), batch: request.BatchSize}) + for _, row := range serve(len(*calls) - 1) { + if err := visit(row); err != nil { + return err + } + } + return nil + } +} + +func exec08aqlPort(calls *int) func(context.Context, RowsRequest, RowVisitor) error { + return func(_ context.Context, request RowsRequest, _ RowVisitor) error { + if _, _, err := request.Command.AQLParameters(); err != nil { + return err + } + *calls++ + return nil + } +} + +func exec08taggedLineageCommand(t *testing.T) command.CompiledCommand { + t.Helper() + compiled, err := command.NewPostgreSQLCommand( + `SELECT true AS "found", '[]'::jsonb AS "contributors", false AS "hasMore" WHERE $1 = 'row-7'`, + []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterText, Value: "row-7"}}, + ) + if err != nil { + t.Fatalf("tag lineage command: %v", err) + } + return compiled +} + +func exec08compiledQuery(commandValue command.CompiledCommand, offset, limit int) compiler.CompiledRowLineageQuery { + return compiler.CompiledRowLineageQuery{ + Command: commandValue, Offset: offset, Limit: limit, + FoundColumn: "found", ContributorsColumn: "contributors", HasMoreColumn: "hasMore", + } +} + +const ( + exec08PageAB = `[{"resourceType":"Observation","resourceId":"obs-a","occurrenceKey":"obs-a"},{"resourceType":"Observation","resourceId":"obs-b","occurrenceKey":"obs-b"}]` + exec08PageOneA = `[{"resourceType":"Observation","resourceId":"obs-a","occurrenceKey":"obs-a"}]` +) + +func TestRowLineagePostgreSQLPageRunsThroughTheRowPort(t *testing.T) { + spies := &dispatchSpies{} + var calls []exec08PortCall + config := spies.arangoEngine() + config.CompileTarget = exec02PostgreSQLTarget(t) + config.PreviewSourceRows = func(context.Context, RowsRequest, RowVisitor) error { + return errors.New("row lineage must never run on the preview lane") + } + config.SourceRows = exec08pgPort(&calls, nil, func(int) []map[string]any { + return []map[string]any{{ + "found": true, + "contributors": json.RawMessage(exec08PageAB), + "hasMore": true, + }} + }) + engine, err := New(config) + if err != nil { + t.Fatalf("engine: %v", err) + } + rowID := `[["construction","exec08_group"],["operation","GROUP"],["grouped_status_id","final"]]` + resolved, err := engine.CompileResolvedBundle(context.Background(), exec08GroupBundle(), recipe.RuntimeBindings{ + Project: "P1", DatasetGeneration: "G1", OutputNames: []string{"observations"}, AuthResourcePaths: []string{"FHIR/Observation"}, + }) + if err != nil { + t.Fatalf("CompileResolvedBundle: %v", err) + } + result, err := engine.RowLineage(context.Background(), resolved, RowLineageRequest{Output: "observations", RowID: rowID, Offset: 24, Limit: 2}) + if err != nil { + t.Fatalf("RowLineage: %v", err) + } + if len(calls) != 1 { + t.Fatalf("port invocations: %d, want exactly one", len(calls)) + } + call := calls[0] + if call.tag != command.CompiledBackendPostgreSQL { + t.Fatalf("port received %q-tagged command", call.tag) + } + if want := []string{"found", "contributors", "hasMore"}; !reflect.DeepEqual(call.declared, want) { + t.Fatalf("declared column contract = %v, want %v", call.declared, want) + } + if call.batch <= 0 { + t.Fatalf("batch hint must be carried on the request: %d", call.batch) + } + identityParameters := 0 + for _, arg := range call.args { + if arg.Type == command.PostgreSQLParameterText { + if text, ok := arg.Value.(string); ok && text == rowID { + identityParameters++ + } + } + } + if identityParameters != 1 { + t.Fatalf("row identity bound as text parameter %d times, want exactly once", identityParameters) + } + if len(result.Contributors) != 2 || + result.Contributors[0] != (RowLineageContributor{ResourceType: "Observation", ResourceID: "obs-a", OccurrenceKey: "obs-a"}) || + result.Contributors[1] != (RowLineageContributor{ResourceType: "Observation", ResourceID: "obs-b", OccurrenceKey: "obs-b"}) { + t.Fatalf("typed contributor page = %#v", result) + } + if !result.HasMore || result.NextOffset != 26 { + t.Fatalf("paging summary = %#v, want HasMore=true NextOffset=26", result) + } + if len(spies.aqlQueries) != 0 || len(spies.aqlPreviewQueries) != 0 || len(spies.controlSnapshot()) != 0 || spies.clickHouseQueries != 0 { + t.Fatalf("PostgreSQL lineage touched Arango/ClickHouse controls: aql=%v preview=%v controls=%v clickhouse=%d", spies.aqlQueries, spies.aqlPreviewQueries, spies.controlSnapshot(), spies.clickHouseQueries) + } +} + +func TestRowLineagePostgreSQLContributorPageFailsClosed(t *testing.T) { + for _, test := range []struct { + name string + contributors any + want string + }{ + {"malformed json", json.RawMessage(`[{"resourceType":"Observation",`), "returned an invalid contributor page"}, + {"empty bytes", json.RawMessage(``), "returned an invalid contributor page"}, + {"json null", json.RawMessage(`null`), "returned an invalid contributor page"}, + {"json object", json.RawMessage(`{"a":1}`), "returned an invalid contributor page"}, + {"json scalar", json.RawMessage(`3`), "returned an invalid contributor page"}, + {"sql null", nil, "returned an invalid contributor page"}, + {"loose string page", "[]", "returned an invalid contributor page"}, + {"element not an object", json.RawMessage(`["obs-a"]`), "returned an invalid contributor"}, + {"blank resource id", json.RawMessage(`[{"resourceType":"Observation","resourceId":"","occurrenceKey":"obs-a"}]`), "without a FHIR resource identity"}, + {"missing occurrence key", json.RawMessage(`[{"resourceType":"Observation","resourceId":"obs-a"}]`), "without a FHIR resource identity"}, + {"non-string resource type", json.RawMessage(`[{"resourceType":7,"resourceId":"obs-a","occurrenceKey":"obs-a"}]`), "without a FHIR resource identity"}, + {"page exceeds limit", json.RawMessage(exec08PageAB), "exceeded the requested contributor page size"}, + } { + t.Run(test.name, func(t *testing.T) { + calls := 0 + engine := &Engine{batchSize: 32, sourceRows: exec08pgPort(&[]exec08PortCall{}, nil, func(int) []map[string]any { + calls++ + return []map[string]any{{"found": true, "contributors": test.contributors, "hasMore": false}} + })} + result, err := engine.rowLineageCompiled(context.Background(), exec08compiledQuery(exec08taggedLineageCommand(t), 0, 1)) + if err == nil || !strings.HasSuffix(err.Error(), test.want) { + t.Fatalf("poisoned page accepted: result=%#v err=%v, want suffix %q", result, err, test.want) + } + if result.Contributors != nil || result.HasMore || result.NextOffset != 0 { + t.Fatalf("poisoned page leaked a partial result: %#v", result) + } + if calls != 1 { + t.Fatalf("port invocations = %d, want one", calls) + } + }) + } +} + +func TestRowLineagePostgreSQLFoundSemanticsOverJSONB(t *testing.T) { + run := func(row map[string]any) (RowLineageResult, error) { + engine := &Engine{batchSize: 32, sourceRows: exec08pgPort(&[]exec08PortCall{}, nil, func(int) []map[string]any { + return []map[string]any{row} + })} + return engine.rowLineageCompiled(context.Background(), exec08compiledQuery(exec08taggedLineageCommand(t), 0, 25)) + } + if _, err := run(map[string]any{"found": false, "contributors": json.RawMessage(`[]`), "hasMore": false}); !errors.Is(err, ErrRowLineageRowNotFound) { + t.Fatalf("keyed group miss over jsonb = %v, want the typed not-found", err) + } + if _, err := run(map[string]any{"found": false, "contributors": json.RawMessage(`nonsense`), "hasMore": false}); !errors.Is(err, ErrRowLineageRowNotFound) { + t.Fatalf("found=false with unreadable page = %v, want the typed not-found", err) + } + result, err := run(map[string]any{"found": true, "contributors": json.RawMessage(`[]`), "hasMore": false}) + if err != nil { + t.Fatalf("empty keyless group page: %v", err) + } + if result.Contributors == nil || len(result.Contributors) != 0 || result.HasMore { + t.Fatalf("empty keyless group page over jsonb = %#v, want complete empty page", result) + } + if _, err := run(map[string]any{"found": nil, "contributors": json.RawMessage(`[]`), "hasMore": false}); err == nil || !strings.Contains(err.Error(), "invalid found flag") { + t.Fatalf("SQL NULL found flag = %v, want the typed flag error", err) + } + if _, err := run(map[string]any{"found": true, "contributors": json.RawMessage(`[]`), "hasMore": nil}); err == nil || !strings.Contains(err.Error(), "invalid hasMore flag") { + t.Fatalf("SQL NULL hasMore flag = %v, want the typed flag error", err) + } +} + +func TestRowLineageCrossTagCommandsFailBeforeThePort(t *testing.T) { + aqlCalls := 0 + aqlEngine := &Engine{batchSize: 32, sourceRows: exec08aqlPort(&aqlCalls)} + if _, err := aqlEngine.rowLineageCompiled(context.Background(), exec08compiledQuery(exec08taggedLineageCommand(t), 0, 25)); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("PostgreSQL command on the AQL port = %v, want the typed mismatch", err) + } + if aqlCalls != 0 { + t.Fatalf("cross-tag command reached the AQL port: %d calls", aqlCalls) + } + pgCalls := 0 + aqlCommand := mustAQLCommand("row-lineage-query", map[string]any{"row_id": "row-7"}) + pgEngine := &Engine{batchSize: 32, sourceRows: exec08pgPort(&[]exec08PortCall{}, nil, func(int) []map[string]any { + pgCalls++ + return nil + })} + if _, err := pgEngine.rowLineageCompiled(context.Background(), exec08compiledQuery(aqlCommand, 0, 25)); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("AQL command on the PostgreSQL port = %v, want the typed mismatch", err) + } + if pgCalls != 0 { + t.Fatalf("cross-tag command reached the PostgreSQL port: %d calls", pgCalls) + } + nativeEngine := &Engine{batchSize: 32, sourceRows: exec08aqlPort(&aqlCalls)} + nativeEngine.sourceRows = func(_ context.Context, request RowsRequest, visit RowVisitor) error { + if _, _, err := request.Command.AQLParameters(); err != nil { + return err + } + aqlCalls++ + return visit(map[string]any{"found": true, "contributors": []any{ + map[string]any{"resourceType": "Observation", "resourceId": "obs-a", "occurrenceKey": "obs-a"}, + }, "hasMore": false}) + } + result, err := nativeEngine.rowLineageCompiled(context.Background(), exec08compiledQuery(aqlCommand, 0, 25)) + if err != nil || len(result.Contributors) != 1 || result.Contributors[0].ResourceID != "obs-a" { + t.Fatalf("native []any arm regressed: result=%#v err=%v", result, err) + } +} + +func TestRowLineageAdmissionRefusalsRunZeroStatements(t *testing.T) { + spies := &dispatchSpies{} + var calls []exec08PortCall + config := spies.arangoEngine() + config.CompileTarget = exec02PostgreSQLTarget(t) + config.PreviewSourceRows = nil + config.SourceRows = exec08pgPort(&calls, nil, func(int) []map[string]any { return nil }) + engine, err := New(config) + if err != nil { + t.Fatalf("engine: %v", err) + } + resolved, err := engine.CompileResolvedBundle(context.Background(), exec08GroupBundle(), recipe.RuntimeBindings{ + Project: "P1", DatasetGeneration: "G1", OutputNames: []string{"observations"}, AuthResourcePaths: []string{"FHIR/Observation"}, + }) + if err != nil { + t.Fatalf("CompileResolvedBundle: %v", err) + } + if _, err := engine.RowLineage(context.Background(), resolved, RowLineageRequest{Output: "missing-output", RowID: "row-7"}); err == nil || !strings.Contains(err.Error(), "was not found") { + t.Fatalf("unknown output = %v, want refusal", err) + } + if _, err := engine.RowLineage(context.Background(), resolved, RowLineageRequest{Output: "observations", RowID: "row-7", Offset: -1}); err == nil || !strings.Contains(err.Error(), "outside the supported range") { + t.Fatalf("negative offset = %v, want refusal", err) + } + if _, err := engine.RowLineage(context.Background(), resolved, RowLineageRequest{Output: "observations", RowID: "row-7", Limit: compiler.MaxRowLineageContributors + 1}); err == nil || !strings.Contains(err.Error(), "outside the supported range") { + t.Fatalf("over-cap limit = %v, want refusal", err) + } + if _, err := engine.RowLineage(context.Background(), resolved, RowLineageRequest{Output: "observations", RowID: " "}); err == nil || !strings.Contains(err.Error(), "required") { + t.Fatalf("blank row id = %v, want refusal", err) + } + if _, err := engine.rowLineageCompiled(context.Background(), compiler.CompiledRowLineageQuery{ + Command: exec08taggedLineageCommand(t), Limit: 25, + FoundColumn: "found", ContributorsColumn: "found", HasMoreColumn: "hasMore", + }); err == nil || !strings.Contains(err.Error(), "missing its bounded result contract") { + t.Fatalf("duplicated contract name = %v, want refusal", err) + } + if _, err := engine.rowLineageCompiled(context.Background(), compiler.CompiledRowLineageQuery{ + Command: exec08taggedLineageCommand(t), Limit: 25, + FoundColumn: "found", ContributorsColumn: "contributors", HasMoreColumn: "", + }); err == nil || !strings.Contains(err.Error(), "missing its bounded result contract") { + t.Fatalf("blank contract name = %v, want refusal", err) + } + if len(calls) != 0 { + t.Fatalf("admission refusals reached the port: %d statements", len(calls)) + } +} diff --git a/internal/dataframe/execution/selected_source_contract_test.go b/internal/dataframe/execution/selected_source_contract_test.go new file mode 100644 index 000000000..8f0922c34 --- /dev/null +++ b/internal/dataframe/execution/selected_source_contract_test.go @@ -0,0 +1,573 @@ +package execution + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" + "github.com/calypr/loom/internal/dataframe/recipe" +) + +func TestSelectedSourceCompilerAndRowsContract(t *testing.T) { + t.Run("StreamRunsExactlyTheSelectedTargetCompiledCommand", func(t *testing.T) { + spies := &dispatchSpies{} + resolved := exec02TestResolved() + reference, err := compiler.CompileRecipeOutput(exec02PostgreSQLTarget(t), + resolved.Compiled.Outputs[0], + resolved.Semantic.SemanticPlan.Bindings, + 0, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("reference compile: %v", err) + } + var observed RowsRequest + calls := 0 + config := spies.arangoEngine() + config.CompileTarget = exec02PostgreSQLTarget(t) + config.SourceRows = func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + calls++ + observed = request + for _, row := range exec02Rows() { + if err := visit(row); err != nil { + return err + } + } + return nil + } + config.PreviewSourceRows = nil + engine := config.newEngineForTest(t) + streams, err := engine.Streams(context.Background(), resolved) + if err != nil { + t.Fatalf("Streams: %v", err) + } + if len(streams) != 1 { + t.Fatalf("streams: %d", len(streams)) + } + if _, err := streams[0].Stream(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("Stream: %v", err) + } + if calls != 1 { + t.Fatalf("port invocations: %d, want exactly one for the parent-tagged command", calls) + } + if observed.Command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("port received %q-tagged command", observed.Command.Backend()) + } + gotSQL, gotArgs, err := observed.Command.PostgreSQLParameters() + if err != nil { + t.Fatalf("observed command: %v", err) + } + wantSQL, wantArgs, err := reference.Command.PostgreSQLParameters() + if err != nil { + t.Fatalf("reference command: %v", err) + } + if gotSQL != wantSQL || len(gotArgs) != len(wantArgs) { + t.Fatalf("stream ran a command other than the selected target compiled") + } + if streams[0].planFingerprint != reference.PlanDiagnostics.Fingerprint { + t.Fatalf("stream fingerprint %q diverged from the reference compile %q", streams[0].planFingerprint, reference.PlanDiagnostics.Fingerprint) + } + if len(observed.DeclaredColumns) != len(reference.Columns) { + t.Fatalf("declared columns %v want %v", observed.DeclaredColumns, reference.Columns) + } + for i := range reference.Columns { + if observed.DeclaredColumns[i] != reference.Columns[i] { + t.Fatalf("declared column %d: %q want %q", i, observed.DeclaredColumns[i], reference.Columns[i]) + } + } + if observed.BatchSize <= 0 { + t.Fatalf("batch hint must be carried on the request: %d", observed.BatchSize) + } + exec02AssertUntouchedArango(t, spies) + }) + + t.Run("PreviewCoveringIndexIsArangoOnly", func(t *testing.T) { + resolved := exec02TestResolved() + pgQuery, err := compiler.CompileRecipeOutput(exec02PostgreSQLTarget(t), + resolved.Compiled.Outputs[0], resolved.Semantic.SemanticPlan.Bindings, 5, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("postgresql compile: %v", err) + } + if pgQuery.PreviewCoveringIndex != nil { + t.Fatalf("PostgreSQL command carries a preview covering index: %+v", pgQuery.PreviewCoveringIndex) + } + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, exec02Rows()) + if _, err := engine.PreviewOutput(context.Background(), resolved, + PreviewRequest{Output: "resources", Limit: 5}, func(map[string]any) error { return nil }); err != nil { + t.Fatalf("preview: %v", err) + } + for _, control := range spies.controlSnapshot() { + if strings.Contains(strings.ToLower(control), "index") { + t.Fatalf("PostgreSQL preview consulted the Arango index control %q", control) + } + } + exec02AssertUntouchedArango(t, spies) + }) + + t.Run("PreviewGroupScanAndProjectionAreArangoOnly", func(t *testing.T) { + resolved := exec02TestResolved() + pgQuery, err := compiler.CompileRecipeOutput(exec02PostgreSQLTarget(t), + resolved.Compiled.Outputs[0], resolved.Semantic.SemanticPlan.Bindings, 5, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("postgresql compile: %v", err) + } + if pgQuery.PreviewGroupScan != nil { + t.Fatalf("PostgreSQL command carries a group-scan spec: %+v", pgQuery.PreviewGroupScan) + } + if pgQuery.PreviewGroupProjection != nil { + t.Fatalf("PostgreSQL command carries a group-projection spec: %+v", pgQuery.PreviewGroupProjection) + } + spies := &dispatchSpies{} + calls := 0 + stream := OutputStream{ + Name: "resources", Columns: []string{"id", "name"}, + command: mustExec11PostgreSQLCommand(t), + expectedBackend: command.CompiledBackendPostgreSQL, + commandColumns: []string{"id", "name"}, + projectionDecisionKey: &groupPreviewRowsKey{}, + sourceRows: func(context.Context, RowsRequest, RowVisitor) error { + calls++ + return nil + }, + } + if err := stream.streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("leaked-decision stream: %v", err) + } + if calls != 1 { + t.Fatalf("port invocations: %d, want one", calls) + } + exec02AssertUntouchedArango(t, spies) + }) + + t.Run("RootPageBindsAreArangoOnly", func(t *testing.T) { + pgCalls := 0 + pgStream := OutputStream{ + Name: "resources", Columns: []string{"id", "name"}, + command: mustExec11PostgreSQLCommand(t), + expectedBackend: command.CompiledBackendPostgreSQL, + commandColumns: []string{"id", "name"}, + rootPageRows: 50, + sourceRows: func(context.Context, RowsRequest, RowVisitor) error { + pgCalls++ + return nil + }, + } + if err := pgStream.streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("leaked-page PostgreSQL stream: %v", err) + } + if pgCalls != 1 { + t.Fatalf("PostgreSQL leaked page ran %d port calls, want the single statement", pgCalls) + } + aqlCommand, err := command.NewAQLCommand("FOR d IN resource RETURN d", map[string]any{"project": "P1"}) + if err != nil { + t.Fatalf("aql command: %v", err) + } + keyCommand, err := command.NewAQLCommand("FOR d IN resource RETURN d._key", map[string]any{ + "project": "P1", + compiler.RootPageAfterKeyBind: "", + compiler.RootPageSizeBind: 50, + }) + if err != nil { + t.Fatalf("aql key command: %v", err) + } + rowsCommand, err := command.NewAQLCommand("FOR d IN resource FILTER d._key IN @loom_root_page_keys RETURN d", map[string]any{ + "project": "P1", + compiler.RootPageKeysBind: []string{}, + }) + if err != nil { + t.Fatalf("aql rows command: %v", err) + } + page := &compiler.CompiledOutputPage{RootKeys: keyCommand, Rows: rowsCommand} + aqlCalls := 0 + expectKeys, keysServed := true, false + aqlStream := OutputStream{ + Name: "resources", Columns: []string{"id"}, + command: aqlCommand, + expectedBackend: command.CompiledBackendAQL, + rootPageRows: 1, + page: page, + sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + aqlCalls++ + if aqlCalls > 8 { + return errors.New("root page loop did not terminate") + } + if expectKeys { + expectKeys = false + if keysServed { + return nil + } + keysServed = true + return visit(map[string]any{"_key": "k1"}) + } + expectKeys = true + return visit(map[string]any{"id": "patient-one"}) + }, + } + if err := aqlStream.streamRaw(context.Background(), func(map[string]any) error { return nil }); err != nil { + t.Fatalf("AQL control page loop: %v", err) + } + if aqlCalls < 2 { + t.Fatalf("AQL control must exercise the root-key page loop through the port: calls=%d", aqlCalls) + } + }) + + t.Run("CellTracePostgreSQLRunArmServesEvidenceThroughTheSharedPort", func(t *testing.T) { + resolved := exec02TestResolved() + compiled, err := compiler.CompileCellTraceOutput(exec02PostgreSQLTarget(t), + resolved.Compiled.Outputs[0], "name", 0, 2, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("registered arm B cell trace must compile on the PostgreSQL target: %v", err) + } + if _, args, portErr := compiled.Command.PostgreSQLParameters(); portErr != nil || len(args) == 0 { + t.Fatalf("compiled cell trace command must be a parameterized PostgreSQL command: %v / %d args", portErr, len(args)) + } + if compiled.ValueColumn != ir.PhysicalCellTraceValueField || compiled.StatusColumn != ir.PhysicalCellTraceStatusField || + compiled.ContributionsColumn != ir.PhysicalCellTraceContributionsField || + compiled.IdentityPartsColumn != ir.PhysicalCellTraceIdentityPartsField || + compiled.HasMoreColumn != ir.PhysicalCellTraceHasMoreField || compiled.OmissionColumn != ir.PhysicalCellTraceOmissionField { + t.Fatalf("compiled trace must carry the six registered contract columns: %#v", compiled) + } + + parts := []any{"patient-one"} + encoded, _ := json.Marshal(parts) + digest := sha256.Sum256(encoded) + rowID := hex.EncodeToString(digest[:]) + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, []map[string]any{ + {ir.PhysicalCellTraceIdentityPartsField: []any{"patient-two"}, ir.PhysicalCellTraceValueField: "Grace", + ir.PhysicalCellTraceStatusField: "VALUE", ir.PhysicalCellTraceHasMoreField: false, + ir.PhysicalCellTraceOmissionField: "", ir.PhysicalCellTraceContributionsField: []any{}}, + {ir.PhysicalCellTraceIdentityPartsField: parts, ir.PhysicalCellTraceValueField: "Ada", + ir.PhysicalCellTraceStatusField: "VALUE", ir.PhysicalCellTraceHasMoreField: false, + ir.PhysicalCellTraceOmissionField: "", ir.PhysicalCellTraceContributionsField: []any{ + map[string]any{"resourceType": "Patient", "resourceId": "patient-one", "operation": "EXTRACT", "value": "female"}, + }}, + }) + result, err := engine.CellTrace(context.Background(), resolved, + CellTraceRequest{Output: "resources", RowID: rowID, Column: "name", Offset: 0, Limit: 1}) + if err != nil { + t.Fatalf("CellTrace on the registered arm B shape: %v", err) + } + if spies.pgRequests != 1 || len(spies.pgFull) != 1 || len(spies.pgPreview) != 0 { + t.Fatalf("cell trace must run exactly once on the full source row port: full=%d preview=%d", len(spies.pgFull), len(spies.pgPreview)) + } + call := spies.pgFull[0] + for _, want := range []string{`AS "__loom_trace_value"`, `AS "__loom_trace_identity_parts"`, `AS "__loom_trace_contributions"`} { + if !strings.Contains(call.SQL, want) { + t.Fatalf("port call must carry the compiled trace plan (%s):\n%s", want, call.SQL) + } + } + if len(call.Args) == 0 { + t.Fatal("every scope and page fact must travel as an ordered parameter") + } + if !result.Complete || result.Status != CellTraceValue || result.Value != "Ada" || + result.HasMore || result.NextOffset != 0 || len(result.Contributions) != 1 || + result.Contributions[0].ResourceID != "patient-one" || result.Contributions[0].Operation != "EXTRACT" { + t.Fatalf("the first visited row must be the identity-matched published row: %#v", result) + } + exec02AssertUntouchedArango(t, spies) + }) + + t.Run("CellTracePostgreSQLRefusesComposedConstructionByNameWithZeroPortCalls", func(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, exec02Rows()) + resolved, err := engine.CompileResolvedBundle(context.Background(), exec08GroupBundle(), recipe.RuntimeBindings{ + Project: "P1", DatasetGeneration: "G1", OutputNames: []string{"observations"}, AuthResourcePaths: []string{"FHIR/Observation"}, + }) + if err != nil { + t.Fatalf("CompileResolvedBundle: %v", err) + } + _, err = engine.CellTrace(context.Background(), resolved, + CellTraceRequest{Output: "observations", RowID: "x", Column: "status", Limit: 25}) + if err == nil || !strings.Contains(err.Error(), "composed construction cell trace is unsupported on the PostgreSQL compile target") { + t.Fatalf("unregistered composed-construction trace must refuse by name: %v", err) + } + if !errors.Is(err, postgres.ErrUnsupportedPostgresPlan) { + t.Fatalf("cell trace refusal must carry the typed unsupported-plan error: %v", err) + } + if spies.pgRequests != 0 { + t.Fatalf("refused cell trace reached the row port") + } + exec02AssertUntouchedArango(t, spies) + }) + + t.Run("CellTracePostgreSQLRefusesEndpointLookupTraversalByNameWithZeroPortCalls", func(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, exec02Rows()) + resolved, err := engine.CompileResolvedBundle(context.Background(), recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "pgsql24_endpoint_control", + TranslationVersion: "pg-sql-24", + Outputs: []recipe.Output{{ + Name: "patients", RootResourceType: "Patient", RowGrain: "patient", + RootColumnNaming: recipe.RootColumnNamingExact, TraversalColumnNaming: recipe.TraversalColumnNamingExact, + Fields: []recipe.Field{{Name: "patient_id", ColumnID: "patient_id", Expr: recipe.Expression{Select: "root.id"}}}, + Traversals: []recipe.Traversal{{ + Name: "subject_Patient", ToResourceType: "Condition", Alias: "condition", + Fields: []recipe.Field{{Name: "condition_id", ColumnID: "condition_id", Expr: recipe.Expression{Select: "id"}, ValueMode: recipe.ValueModeFirst}}, + }}, + }}, + }, recipe.RuntimeBindings{ + Project: "P1", DatasetGeneration: "G1", OutputNames: []string{"patients"}, AuthResourcePaths: []string{"FHIR/Patient", "FHIR/Condition"}, + }) + if err != nil { + t.Fatalf("CompileResolvedBundle: %v", err) + } + _, err = engine.CellTrace(context.Background(), resolved, + CellTraceRequest{Output: "patients", RowID: "x", Column: "condition_id", Limit: 25}) + if err == nil || !strings.Contains(err.Error(), "endpoint-lookup") { + t.Fatalf("the default-policy traversal trace must refuse with the spine's typed endpoint-lookup voice: %v", err) + } + if !errors.Is(err, postgres.ErrUnsupportedPostgresPlan) { + t.Fatalf("cell trace refusal must carry the typed unsupported-plan error: %v", err) + } + if spies.pgRequests != 0 { + t.Fatalf("refused cell trace reached the row port") + } + exec02AssertUntouchedArango(t, spies) + }) + + t.Run("RowLineageSurfaceRefusesPostgreSQLTargetByName", func(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, exec02Rows()) + _, err := engine.RowLineage(context.Background(), exec02TestResolved(), + RowLineageRequest{Output: "resources", RowID: "patient-one"}) + if err == nil || !strings.Contains(err.Error(), "row lineage compilation is unsupported on the PostgreSQL compile target") { + t.Fatalf("row lineage must name its refusal under the PostgreSQL target: %v", err) + } + if !errors.Is(err, postgres.ErrUnsupportedPostgresPlan) { + t.Fatalf("row lineage refusal must carry the typed unsupported-plan error: %v", err) + } + if spies.pgRequests != 0 { + t.Fatalf("refused row lineage reached the row port") + } + exec02AssertUntouchedArango(t, spies) + }) + + t.Run("RowLineagePostgreSQLRunArmServesBoundedColumnContractThroughTheSharedPort", func(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, []map[string]any{{ + "found": true, "contributors": json.RawMessage(exec08PageOneA), "hasMore": false, + }}) + resolved, err := engine.CompileResolvedBundle(context.Background(), exec08GroupBundle(), recipe.RuntimeBindings{ + Project: "P1", DatasetGeneration: "G1", OutputNames: []string{"observations"}, AuthResourcePaths: []string{"FHIR/Observation"}, + }) + if err != nil { + t.Fatalf("CompileResolvedBundle: %v", err) + } + result, err := engine.RowLineage(context.Background(), resolved, RowLineageRequest{ + Output: "observations", + RowID: `[["construction","exec08_group"],["operation","GROUP"],["grouped_status_id","final"]]`, + Offset: 0, Limit: 25, + }) + if err != nil { + t.Fatalf("RowLineage: %v", err) + } + if spies.pgRequests != 1 || len(spies.pgFull) != 1 { + t.Fatalf("PostgreSQL lineage port calls = %d (records %d), want exactly one", spies.pgRequests, len(spies.pgFull)) + } + if want := "found,contributors,hasMore"; strings.Join(spies.pgFull[0].Columns, ",") != want { + t.Fatalf("declared column contract = %v, want %s", spies.pgFull[0].Columns, want) + } + if len(result.Contributors) != 1 || result.Contributors[0].ResourceID != "obs-a" || result.HasMore { + t.Fatalf("typed contributor page = %#v", result) + } + exec02AssertUntouchedArango(t, spies) + }) + + t.Run("PopulationMappingPostgreSQLRunArmServesWitnessContractThroughTheSharedPort", func(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, []map[string]any{ + {ir.PhysicalPopulationMappingMemberField: "obs-1", ir.PhysicalPopulationMappingIdentityPartsField: []any{"P1", "obs-1"}}, + {ir.PhysicalPopulationMappingMemberField: "obs-2", ir.PhysicalPopulationMappingIdentityPartsField: []any{"P1", "obs-2"}}, + }) + resolved, err := engine.CompileResolvedBundle(context.Background(), exec07FlatBundle("exec07-contract"), recipe.RuntimeBindings{ + Project: "P1", DatasetGeneration: "G1", + SelectionProject: "P1", + SelectionMembersCollection: "loom_explorer_selection_members", + AuthResourcePaths: []string{"FHIR/Observation"}, + }) + if err != nil { + t.Fatalf("CompileResolvedBundle: %v", err) + } + result, err := engine.PopulationMapping(context.Background(), resolved, + PopulationMappingRequest{Output: "observations", MaxUnmapped: 10}, + exec07Members("obs-1", "obs-2", "obs-3")) + if err != nil { + t.Fatalf("PopulationMapping: %v", err) + } + if spies.pgRequests != 1 || len(spies.pgFull) != 1 { + t.Fatalf("PostgreSQL mapping port calls = %d (records %d), want exactly one", spies.pgRequests, len(spies.pgFull)) + } + if want := ir.PhysicalPopulationMappingMemberField + "," + ir.PhysicalPopulationMappingIdentityPartsField; strings.Join(spies.pgFull[0].Columns, ",") != want { + t.Fatalf("declared witness contract = %v, want %s", spies.pgFull[0].Columns, want) + } + if result.Status != PopulationMappingComplete || result.SelectedCount != 3 || result.MappedCount != 2 || result.UnmappedCount != 1 || result.EmittedRows != 2 { + t.Fatalf("typed mapping result = %#v", result) + } + if len(result.UnmappedMemberIDs) != 1 || result.UnmappedMemberIDs[0] != "obs-3" { + t.Fatalf("unmapped member IDs = %#v", result.UnmappedMemberIDs) + } + exec02AssertUntouchedArango(t, spies) + }) + + t.Run("MisTaggedPostgreSQLCommandThroughAQLFakeFailsClosed", func(t *testing.T) { + spies := &dispatchSpies{} + stream := OutputStream{ + Name: "resources", Columns: []string{"id", "name"}, + command: mustExec11PostgreSQLCommand(t), + expectedBackend: command.CompiledBackendAQL, + commandColumns: []string{"id", "name"}, + sourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + spies.mu.Lock() + spies.pgRequests++ + spies.mu.Unlock() + return nil + }, + } + err := stream.streamRaw(context.Background(), func(map[string]any) error { return nil }) + if !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("mis-tagged command must fail with the typed mismatch: %v", err) + } + if spies.pgRequests != 0 { + t.Fatalf("mis-tagged command reached the port; zero-call probe failed") + } + }) + + t.Run("ZeroTaggedCommandFailsClosedBeforePort", func(t *testing.T) { + calls := 0 + stream := OutputStream{ + Name: "resources", Columns: []string{"id"}, + command: command.CompiledCommand{}, + commandColumns: []string{"id"}, + sourceRows: func(context.Context, RowsRequest, RowVisitor) error { + calls++ + return nil + }, + } + err := stream.streamRaw(context.Background(), func(map[string]any) error { return nil }) + if !errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + t.Fatalf("zero tag must fail with the unregistered-backend sentinel: %v", err) + } + if calls != 0 { + t.Fatalf("zero-tagged command reached the port; zero-call probe failed") + } + }) + + t.Run("EmptyDeclaredColumnsRefusedBeforePort", func(t *testing.T) { + calls := 0 + stream := OutputStream{ + Name: "resources", Columns: []string{"id", "name"}, + command: mustExec11PostgreSQLCommand(t), + expectedBackend: command.CompiledBackendPostgreSQL, + sourceRows: func(context.Context, RowsRequest, RowVisitor) error { + calls++ + return nil + }, + } + err := stream.streamRaw(context.Background(), func(map[string]any) error { return nil }) + if err == nil || !strings.Contains(err.Error(), "non-empty declared result column contract") { + t.Fatalf("PostgreSQL stream without a declared contract must fail closed: %v", err) + } + if calls != 0 { + t.Fatalf("empty declared contract reached the port; zero-call probe failed") + } + }) + + t.Run("PreviewRunsOnPreviewLaneWhenSet", func(t *testing.T) { + spies := &dispatchSpies{} + engine, _ := exec02pgEngine(t, spies, exec02Rows()) + if _, err := engine.PreviewOutput(context.Background(), exec02TestResolved(), + PreviewRequest{Output: "resources", Limit: 5}, func(map[string]any) error { return nil }); err != nil { + t.Fatalf("preview: %v", err) + } + if len(spies.pgPreview) != 1 || len(spies.pgFull) != 0 { + t.Fatalf("preview lane routing: full=%d preview=%d, want preview-only", len(spies.pgFull), len(spies.pgPreview)) + } + }) + + t.Run("PreviewFallsBackToSourceRowsWhenPreviewLaneUnset", func(t *testing.T) { + spies := &dispatchSpies{} + config := spies.arangoEngine() + config.CompileTarget = exec02PostgreSQLTarget(t) + config.SourceRows = exec02pgPort(spies, &spies.pgFull, exec02Rows()) + config.PreviewSourceRows = nil + engine := config.newEngineForTest(t) + if _, err := engine.PreviewOutput(context.Background(), exec02TestResolved(), + PreviewRequest{Output: "resources", Limit: 5}, func(map[string]any) error { return nil }); err != nil { + t.Fatalf("preview without a preview lane: %v", err) + } + if len(spies.pgFull) != 1 || len(spies.pgPreview) != 0 { + t.Fatalf("preview fallback routing: full=%d preview=%d, want SourceRows only", len(spies.pgFull), len(spies.pgPreview)) + } + }) + + t.Run("PostgreSQLCommandNeverReachesTheArangoPreviewLane", func(t *testing.T) { + spies := &dispatchSpies{} + config := spies.arangoEngine() + config.CompileTarget = exec02PostgreSQLTarget(t) + config.SourceRows = exec02pgPort(spies, &spies.pgFull, exec02Rows()) + engine := config.newEngineForTest(t) + _, err := engine.PreviewOutput(context.Background(), exec02TestResolved(), + PreviewRequest{Output: "resources", Limit: 5}, func(map[string]any) error { return nil }) + if !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("PostgreSQL command on the AQL preview lane must fail with the typed mismatch: %v", err) + } + if len(spies.aqlPreviewQueries) != 0 || len(spies.aqlQueries) != 0 { + t.Fatalf("PostgreSQL command touched the AQL adapters: preview=%v full=%v", spies.aqlPreviewQueries, spies.aqlQueries) + } + }) + t.Run("PrivateCaptureRunsExactlyTheSelectedTargetCompiledCommand", func(t *testing.T) { + fixture := newExec11Capture(t, exec11PortRows("Ada")) + captureBindings := exec11RestrictedBindings() + captureBindings.IncludeAuthResourcePath = true + reference, err := compiler.CompileRecipeOutput(exec02PostgreSQLTarget(t), + fixture.resolved.Compiled.Outputs[0], captureBindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("reference capture compile: %v", err) + } + artifact, err := fixture.stream.MaterializeClickHouseArtifact(context.Background(), fixture.manager, "exec11-contract", fixture.stream.stageID, fixture.columns) + if err != nil { + t.Fatalf("materialize PostgreSQL capture: %v", err) + } + defer func() { + if err := artifact.Release(context.Background()); err != nil { + t.Fatalf("release: %v", err) + } + }() + if len(fixture.spies.pgFull) != 1 { + t.Fatalf("capture port calls = %d, want 1", len(fixture.spies.pgFull)) + } + wantSQL, wantArgs, err := reference.Command.PostgreSQLParameters() + if err != nil { + t.Fatalf("reference command: %v", err) + } + if fixture.spies.pgFull[0].SQL != wantSQL || len(fixture.spies.pgFull[0].Args) != len(wantArgs) { + t.Fatalf("capture ran a command other than the selected target compiled") + } + if strings.Join(fixture.spies.pgFull[0].Columns, ",") != strings.Join(reference.Columns, ",") { + t.Fatalf("capture port declared columns %v, want the compiled contract %v", fixture.spies.pgFull[0].Columns, reference.Columns) + } + if !strings.Contains(wantSQL, "auth_resource_path") { + t.Fatalf("restricted capture reference must read the typed authorization column: %q", wantSQL) + } + }) +} + +func mustExec11PostgreSQLCommand(t *testing.T) command.CompiledCommand { + t.Helper() + resolved := exec02TestResolved() + query, err := compiler.CompileRecipeOutput(exec02PostgreSQLTarget(t), + resolved.Compiled.Outputs[0], resolved.Semantic.SemanticPlan.Bindings, 0, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("reference compile: %v", err) + } + return query.Command +} diff --git a/internal/dataframe/execution/service.go b/internal/dataframe/execution/service.go index d8d7c9cff..31aaf2f6b 100644 --- a/internal/dataframe/execution/service.go +++ b/internal/dataframe/execution/service.go @@ -1,17 +1,13 @@ package execution -import ( - "context" -) - type ServiceConfig struct { - QueryRows func(context.Context, string, int, map[string]any, func(map[string]any) error) error + SourceRows SourceRows } type Service struct { - queryRows func(context.Context, string, int, map[string]any, func(map[string]any) error) error + sourceRows SourceRows } func NewService(cfg ServiceConfig) *Service { - return &Service{queryRows: cfg.QueryRows} + return &Service{sourceRows: cfg.SourceRows} } diff --git a/internal/dataframe/execution/table_shape_exclusions.go b/internal/dataframe/execution/table_shape_exclusions.go index adb419ad6..af3e30f39 100644 --- a/internal/dataframe/execution/table_shape_exclusions.go +++ b/internal/dataframe/execution/table_shape_exclusions.go @@ -2,12 +2,15 @@ package execution import ( "context" + "encoding/json" "errors" "fmt" "strings" "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" + postgresrender "github.com/calypr/loom/internal/dataframe/compiler/render/postgres" ) type TableShapeExclusionStatus string @@ -55,7 +58,7 @@ func (e *Engine) TableShapeExclusions(ctx context.Context, resolved Resolved, re if output.Name != request.Output { continue } - compiled, err := compiler.CompileTableShapeExclusionsWithPolicy(output, request.Offset, request.Limit, ir.DefaultPhysicalOptimizationPolicy()) + compiled, err := compiler.CompileTableShapeExclusions(e.compileTarget, output, request.Offset, request.Limit, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { var refusal *compiler.TableShapeExclusionRefusal if errors.As(err, &refusal) { @@ -66,6 +69,9 @@ func (e *Engine) TableShapeExclusions(ctx context.Context, resolved Resolved, re return TableShapeExclusionResult{Status: TableShapeExclusionNoExclusionPolicy, Exclusions: []TableShapeExclusion{}}, nil } } + if errors.Is(err, postgresrender.ErrUnsupportedPostgresPlan) { + return TableShapeExclusionResult{Status: TableShapeExclusionUnsupported, Exclusions: []TableShapeExclusion{}}, nil + } return TableShapeExclusionResult{}, err } return e.TableShapeExclusionsCompiled(ctx, compiled) @@ -73,18 +79,42 @@ func (e *Engine) TableShapeExclusions(ctx context.Context, resolved Resolved, re return TableShapeExclusionResult{}, fmt.Errorf("table-shape exclusion output %q was not found", request.Output) } -// TableShapeExclusionsCompiled is the narrow execution seam for callers and -// tests holding the compiler-produced canonical diagnostic query. func (e *Engine) TableShapeExclusionsCompiled(ctx context.Context, compiled compiler.CompiledTableShapeExclusionQuery) (TableShapeExclusionResult, error) { - if e == nil || e.queryRows == nil { + isPostgreSQL := false + switch compiled.Command.Backend() { + case command.CompiledBackendAQL: + case command.CompiledBackendPostgreSQL: + isPostgreSQL = true + } + if e == nil || e.sourceRows == nil { + if isPostgreSQL { + return TableShapeExclusionResult{Status: TableShapeExclusionUnsupported, Exclusions: []TableShapeExclusion{}}, nil + } return TableShapeExclusionResult{}, fmt.Errorf("table-shape exclusion query executor is required") } - if compiled.Query == "" || compiled.Limit <= 0 || compiled.Offset < 0 || compiled.ResourceTypeField == "" || compiled.ResourceIDField == "" || compiled.CategoryPresentField == "" || compiled.CategoryValueField == "" || compiled.CategoryTypeField == "" || compiled.OutputRowIDField == "" || compiled.ReasonField == "" || compiled.IdentityStatusField == "" || compiled.OmissionField == "" { + if !isPostgreSQL && compiled.Command.Backend() != command.CompiledBackendAQL || + compiled.Limit <= 0 || compiled.Offset < 0 || compiled.ResourceTypeField == "" || compiled.ResourceIDField == "" || compiled.CategoryPresentField == "" || compiled.CategoryValueField == "" || compiled.CategoryTypeField == "" || compiled.OutputRowIDField == "" || compiled.ReasonField == "" || compiled.IdentityStatusField == "" || compiled.OmissionField == "" { return TableShapeExclusionResult{}, fmt.Errorf("table-shape exclusion query is missing typed result metadata") } + parse := parseTableShapeExclusion + request := RowsRequest{Command: compiled.Command, BatchSize: e.batchSize} + if isPostgreSQL { + parse = parsePostgreSQLTableShapeExclusion + request.DeclaredColumns = []string{ + compiled.ResourceTypeField, + compiled.ResourceIDField, + compiled.IdentityStatusField, + compiled.CategoryValueField, + compiled.CategoryPresentField, + compiled.CategoryTypeField, + compiled.OutputRowIDField, + compiled.ReasonField, + compiled.OmissionField, + } + } rows := make([]TableShapeExclusion, 0, compiled.Limit+1) tooMany := false - err := e.queryRows(ctx, compiled.Query, e.batchSize, compiled.BindVars, func(row map[string]any) error { + err := e.sourceRows(ctx, request, func(row map[string]any) error { if err := ctx.Err(); err != nil { return err } @@ -92,7 +122,7 @@ func (e *Engine) TableShapeExclusionsCompiled(ctx context.Context, compiled comp tooMany = true return errTableShapeExclusionOverflow } - parsed, parseErr := parseTableShapeExclusion(compiled, row) + parsed, parseErr := parse(compiled, row) if parseErr != nil { return parseErr } @@ -106,6 +136,9 @@ func (e *Engine) TableShapeExclusionsCompiled(ctx context.Context, compiled comp if ctx.Err() != nil { return TableShapeExclusionResult{Status: TableShapeExclusionIncomplete, Exclusions: []TableShapeExclusion{}, Complete: false, NextOffset: compiled.Offset}, nil } + if isPostgreSQL && errors.Is(err, command.ErrCompiledCommandBackendNotRegistered) { + return TableShapeExclusionResult{Status: TableShapeExclusionUnsupported, Exclusions: []TableShapeExclusion{}}, nil + } return TableShapeExclusionResult{}, fmt.Errorf("execute table-shape exclusions: %w", err) } if err := ctx.Err(); err != nil { @@ -136,51 +169,151 @@ func (e *Engine) TableShapeExclusionsCompiled(ctx context.Context, compiled comp var errTableShapeExclusionOverflow = errors.New("table-shape exclusion page overflow") -func parseTableShapeExclusion(compiled compiler.CompiledTableShapeExclusionQuery, row map[string]any) (TableShapeExclusion, error) { - categoryPresent, ok := row[compiled.CategoryPresentField].(bool) - if !ok { - return TableShapeExclusion{}, fmt.Errorf("table-shape exclusion has invalid category presence") - } - categoryValue, exists := row[compiled.CategoryValueField] - if !exists { - return TableShapeExclusion{}, fmt.Errorf("table-shape exclusion is missing category value") - } - categoryType, ok := row[compiled.CategoryTypeField].(string) - if !ok || categoryType == "" { +type tableShapeExclusionEvidence struct { + present bool + categoryValue any + categoryType string + categoryTypeOK bool + outputRowID string + outputRowIDOK bool + reason string + reasonOK bool + identityStatus string + identityStatusOK bool + sourceIdentityOK bool + resourceType string + resourceID string + omission string + omissionOK bool +} + +func tableShapeExclusionFromEvidence(evidence tableShapeExclusionEvidence) (TableShapeExclusion, error) { + if !evidence.categoryTypeOK || evidence.categoryType == "" { return TableShapeExclusion{}, fmt.Errorf("table-shape exclusion has invalid category type") } - rowID, ok := row[compiled.OutputRowIDField].(string) - if !ok || rowID == "" { + if !evidence.outputRowIDOK || evidence.outputRowID == "" { return TableShapeExclusion{}, fmt.Errorf("table-shape exclusion has invalid output row identity") } - reason, ok := row[compiled.ReasonField].(string) - if !ok || reason != ir.PhysicalTableShapeExclusionReasonUnlistedCategory { + if !evidence.reasonOK || evidence.reason != ir.PhysicalTableShapeExclusionReasonUnlistedCategory { return TableShapeExclusion{}, fmt.Errorf("table-shape exclusion has invalid reason") } - identityStatus, ok := row[compiled.IdentityStatusField].(string) - if !ok { + if !evidence.identityStatusOK { return TableShapeExclusion{}, fmt.Errorf("table-shape exclusion has invalid source identity status") } result := TableShapeExclusion{ - Category: CategoryValue{Present: categoryPresent, Value: categoryValue}, - CategoryType: categoryType, OutputRowID: rowID, Reason: reason, + Category: CategoryValue{Present: evidence.present, Value: evidence.categoryValue}, + CategoryType: evidence.categoryType, OutputRowID: evidence.outputRowID, Reason: evidence.reason, } - switch identityStatus { + switch evidence.identityStatus { case ir.PhysicalTableShapeExclusionIdentityExact: - resourceType, typeOK := row[compiled.ResourceTypeField].(string) - resourceID, idOK := row[compiled.ResourceIDField].(string) - if !typeOK || resourceType == "" || !idOK || resourceID == "" { + if !evidence.sourceIdentityOK || evidence.resourceType == "" || evidence.resourceID == "" { return TableShapeExclusion{}, fmt.Errorf("exact table-shape source identity is incomplete") } - result.SourceIdentity = &TableShapeSourceIdentity{ResourceType: resourceType, ResourceID: resourceID} + result.SourceIdentity = &TableShapeSourceIdentity{ResourceType: evidence.resourceType, ResourceID: evidence.resourceID} case ir.PhysicalTableShapeExclusionIdentityUnavailable: - omission, omissionOK := row[compiled.OmissionField].(string) - if !omissionOK || omission != ir.PhysicalTableShapeExclusionSourceIdentityUnavailable { + if !evidence.omissionOK || evidence.omission != ir.PhysicalTableShapeExclusionSourceIdentityUnavailable { return TableShapeExclusion{}, fmt.Errorf("table-shape source identity omission is invalid") } - result.OmissionCode = omission + result.OmissionCode = evidence.omission default: - return TableShapeExclusion{}, fmt.Errorf("table-shape exclusion has unknown source identity status %q", identityStatus) + return TableShapeExclusion{}, fmt.Errorf("table-shape exclusion has unknown source identity status %q", evidence.identityStatus) } return result, nil } + +func parseTableShapeExclusion(compiled compiler.CompiledTableShapeExclusionQuery, row map[string]any) (TableShapeExclusion, error) { + categoryPresent, ok := row[compiled.CategoryPresentField].(bool) + if !ok { + return TableShapeExclusion{}, fmt.Errorf("table-shape exclusion has invalid category presence") + } + categoryValue, exists := row[compiled.CategoryValueField] + if !exists { + return TableShapeExclusion{}, fmt.Errorf("table-shape exclusion is missing category value") + } + categoryType, categoryTypeOK := row[compiled.CategoryTypeField].(string) + outputRowID, outputRowIDOK := row[compiled.OutputRowIDField].(string) + reason, reasonOK := row[compiled.ReasonField].(string) + identityStatus, identityStatusOK := row[compiled.IdentityStatusField].(string) + resourceType, resourceTypeOK := row[compiled.ResourceTypeField].(string) + resourceID, resourceIDOK := row[compiled.ResourceIDField].(string) + omission, omissionOK := row[compiled.OmissionField].(string) + return tableShapeExclusionFromEvidence(tableShapeExclusionEvidence{ + present: categoryPresent, + categoryValue: categoryValue, + categoryType: categoryType, + categoryTypeOK: categoryTypeOK, + outputRowID: outputRowID, + outputRowIDOK: outputRowIDOK, + reason: reason, + reasonOK: reasonOK, + identityStatus: identityStatus, + identityStatusOK: identityStatusOK, + sourceIdentityOK: resourceTypeOK && resourceIDOK, + resourceType: resourceType, + resourceID: resourceID, + omission: omission, + omissionOK: omissionOK, + }) +} + +func parsePostgreSQLTableShapeExclusion(compiled compiler.CompiledTableShapeExclusionQuery, row map[string]any) (TableShapeExclusion, error) { + present, ok := decodePostgreSQLPresence(row[compiled.CategoryPresentField]) + if !ok { + return TableShapeExclusion{}, fmt.Errorf("table-shape exclusion has invalid category presence") + } + categoryValue, exists := row[compiled.CategoryValueField] + if !exists { + return TableShapeExclusion{}, fmt.Errorf("table-shape exclusion is missing category value") + } + categoryType, categoryTypeOK := row[compiled.CategoryTypeField].(string) + outputRowID, outputRowIDOK := row[compiled.OutputRowIDField].(string) + reason, reasonOK := row[compiled.ReasonField].(string) + identityStatus, identityStatusOK := row[compiled.IdentityStatusField].(string) + resourceType, resourceTypeOK := decodePostgreSQLEvidenceText(row[compiled.ResourceTypeField]) + resourceID, resourceIDOK := decodePostgreSQLEvidenceText(row[compiled.ResourceIDField]) + omission, omissionOK := row[compiled.OmissionField].(string) + return tableShapeExclusionFromEvidence(tableShapeExclusionEvidence{ + present: present, + categoryValue: categoryValue, + categoryType: categoryType, + categoryTypeOK: categoryTypeOK, + outputRowID: outputRowID, + outputRowIDOK: outputRowIDOK, + reason: reason, + reasonOK: reasonOK, + identityStatus: identityStatus, + identityStatusOK: identityStatusOK, + sourceIdentityOK: resourceTypeOK && resourceIDOK, + resourceType: resourceType, + resourceID: resourceID, + omission: omission, + omissionOK: omissionOK, + }) +} + +func decodePostgreSQLPresence(value any) (bool, bool) { + raw, ok := value.(json.RawMessage) + if !ok { + return false, false + } + switch string(raw) { + case "true": + return true, true + case "false": + return false, true + default: + return false, false + } +} + +func decodePostgreSQLEvidenceText(value any) (string, bool) { + raw, ok := value.(json.RawMessage) + if !ok || len(raw) == 0 { + return "", false + } + var text string + if err := json.Unmarshal(raw, &text); err != nil || text == "" { + return "", false + } + return text, true +} diff --git a/internal/dataframe/execution/table_shape_exclusions_test.go b/internal/dataframe/execution/table_shape_exclusions_test.go index 31cff20aa..030bc7041 100644 --- a/internal/dataframe/execution/table_shape_exclusions_test.go +++ b/internal/dataframe/execution/table_shape_exclusions_test.go @@ -2,16 +2,21 @@ package execution import ( "context" + "encoding/json" + "errors" + "fmt" "reflect" "testing" "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/recipe" ) func compiledTableShapeExclusionPage(offset, limit int) compiler.CompiledTableShapeExclusionQuery { return compiler.CompiledTableShapeExclusionQuery{ - Query: "exclusions", BindVars: map[string]any{"offset": offset, "limit": limit}, Offset: offset, Limit: limit, + Command: mustAQLCommand("exclusions", map[string]any{"offset": offset, "limit": limit}), Offset: offset, Limit: limit, ResourceTypeField: "resourceType", ResourceIDField: "resourceId", IdentityStatusField: "identityStatus", CategoryValueField: "categoryValue", CategoryPresentField: "categoryPresent", CategoryTypeField: "categoryType", OutputRowIDField: "outputRowId", ReasonField: "reason", OmissionField: "omission", @@ -26,7 +31,13 @@ func TestTableShapeExclusionsCompiledPagesExactTypedRows(t *testing.T) { tableShapeExclusionRow("Observation", "null", true, nil, "STRING", "row-null"), tableShapeExclusionRow("Observation", "missing", false, nil, "STRING", "row-missing"), } - engine := &Engine{queryRows: func(_ context.Context, query string, _ int, binds map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + query, binds, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if query != "exclusions" || binds["offset"] != 0 || binds["limit"] != 10 { t.Fatalf("unexpected query invocation %q %#v", query, binds) } @@ -65,7 +76,13 @@ func TestTableShapeExclusionsCompiledPagesAndMarksIdentityOmissionIncomplete(t * tableShapeExclusionRow("", "", false, "not-a-real-id", "STRING", "row-three"), } makeEngine := func() *Engine { - return &Engine{queryRows: func(_ context.Context, query string, _ int, binds map[string]any, visit func(map[string]any) error) error { + return &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + query, binds, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if query != "exclusions" { t.Fatalf("query = %q", query) } @@ -98,7 +115,13 @@ func TestTableShapeExclusionsCompiledPagesAndMarksIdentityOmissionIncomplete(t * } func TestTableShapeExclusionsCompiledRejectsMalformedRowsWithoutPartialSuccess(t *testing.T) { - engine := &Engine{queryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + _ = visit(tableShapeExclusionRow("Observation", "good", true, "a", "STRING", "row-a")) return visit(map[string]any{"categoryPresent": "not-bool"}) }} @@ -108,8 +131,96 @@ func TestTableShapeExclusionsCompiledRejectsMalformedRowsWithoutPartialSuccess(t } } +func TestTableShapeExclusionsCompiledParserViolationsFailLoud(t *testing.T) { + base := func() map[string]any { + return tableShapeExclusionRow("Observation", "good", true, "a", "STRING", "row-a") + } + for _, test := range []struct { + name string + mutate func(row map[string]any) + }{ + {"presence as string", func(row map[string]any) { + row["categoryPresent"] = "true" + }}, + {"presence as SQL NULL", func(row map[string]any) { + row["categoryPresent"] = nil + }}, + {"presence absent", func(row map[string]any) { + delete(row, "categoryPresent") + }}, + {"category value absent", func(row map[string]any) { + delete(row, "categoryValue") + }}, + {"category type as raw JSON", func(row map[string]any) { + row["categoryType"] = exec12JSON(`"STRING"`) + }}, + {"category type empty", func(row map[string]any) { + row["categoryType"] = "" + }}, + {"row identity as number", func(row map[string]any) { + row["outputRowId"] = 7 + }}, + {"row identity empty", func(row map[string]any) { + row["outputRowId"] = "" + }}, + {"reason drifted", func(row map[string]any) { + row["reason"] = "SOMETHING_ELSE" + }}, + {"identity status as raw JSON", func(row map[string]any) { + row["identityStatus"] = exec12JSON(`"EXACT"`) + }}, + {"identity status unknown", func(row map[string]any) { + row["identityStatus"] = "SOMETHING" + }}, + {"exact identity type not a string", func(row map[string]any) { + row["resourceType"] = 5 + }}, + {"exact identity type empty string", func(row map[string]any) { + row["resourceType"] = "" + }}, + {"exact identity type SQL NULL", func(row map[string]any) { + row["resourceType"] = nil + }}, + {"exact identity id empty string", func(row map[string]any) { + row["resourceId"] = "" + }}, + {"exact identity id absent", func(row map[string]any) { + delete(row, "resourceId") + }}, + {"unavailable omission missing", func(row map[string]any) { + row["identityStatus"] = ir.PhysicalTableShapeExclusionIdentityUnavailable + row["omission"] = nil + }}, + {"unavailable omission drifted", func(row map[string]any) { + row["identityStatus"] = ir.PhysicalTableShapeExclusionIdentityUnavailable + row["omission"] = "WRONG" + }}, + } { + t.Run(test.name, func(t *testing.T) { + row := base() + test.mutate(row) + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(_ context.Context, _ RowsRequest, visit RowVisitor) error { + return visit(row) + }} + result, err := engine.TableShapeExclusionsCompiled(context.Background(), compiledTableShapeExclusionPage(0, 10)) + if err == nil { + t.Fatalf("violating row must fail loud, got %#v", result) + } + if result.Exclusions != nil || result.Complete || result.Status == TableShapeExclusionComplete || result.Status == TableShapeExclusionIncomplete { + t.Fatalf("no partial success on parse violation: %#v", result) + } + }) + } +} + func TestTableShapeExclusionsReportsUnsupportedAndNoPolicy(t *testing.T) { - engine := testEngine(func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + engine := testEngine(func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + _, _, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + t.Fatal("refused table-shape diagnostics must not execute a query") return nil }) @@ -174,3 +285,471 @@ func tableShapeExclusionRow(resourceType, resourceID string, present bool, value "outputRowId": rowID, "reason": "UNLISTED_CATEGORY", "omission": omission, } } + +func exec12PostgreSQLTarget(t *testing.T) compiler.Target { + t.Helper() + binding, err := ir.NewPostgresStorageBinding("loom") + if err != nil { + t.Fatalf("NewPostgresStorageBinding: %v", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + t.Fatalf("PostgreSQLTarget: %v", err) + } + return target +} + +func exec12PGCommand(offset, fetchLimit int) command.CompiledCommand { + compiledCommand, err := command.NewPostgreSQLCommand( + `WITH px_src AS (SELECT 1) SELECT 1 FROM px_src AS p ORDER BY 1 OFFSET $1 LIMIT $2`, + []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterBigint, Value: int64(offset)}, + {Type: command.PostgreSQLParameterBigint, Value: int64(fetchLimit)}, + }) + if err != nil { + panic(err) + } + return compiledCommand +} + +func exec12PGCompiled(offset, limit int) compiler.CompiledTableShapeExclusionQuery { + return compiler.CompiledTableShapeExclusionQuery{ + Command: exec12PGCommand(offset, limit+1), Offset: offset, Limit: limit, + ResourceTypeField: ir.PhysicalTableShapeExclusionResourceTypeField, + ResourceIDField: ir.PhysicalTableShapeExclusionResourceIDField, + IdentityStatusField: ir.PhysicalTableShapeExclusionIdentityStatusField, + CategoryValueField: ir.PhysicalTableShapeExclusionCategoryValueField, + CategoryPresentField: ir.PhysicalTableShapeExclusionCategoryPresentField, + CategoryTypeField: ir.PhysicalTableShapeExclusionCategoryTypeField, + OutputRowIDField: ir.PhysicalTableShapeExclusionOutputRowIDField, + ReasonField: ir.PhysicalTableShapeExclusionReasonField, + OmissionField: ir.PhysicalTableShapeExclusionOmissionField, + } +} + +func exec12PGRow(resourceType, resourceID any, identityStatus string, present any, value any, categoryType, rowID string) map[string]any { + omission := any(nil) + if identityStatus == ir.PhysicalTableShapeExclusionIdentityUnavailable { + resourceType, resourceID = nil, nil + omission = ir.PhysicalTableShapeExclusionSourceIdentityUnavailable + } + return map[string]any{ + ir.PhysicalTableShapeExclusionResourceTypeField: resourceType, + ir.PhysicalTableShapeExclusionResourceIDField: resourceID, + ir.PhysicalTableShapeExclusionIdentityStatusField: identityStatus, + ir.PhysicalTableShapeExclusionCategoryValueField: value, + ir.PhysicalTableShapeExclusionCategoryPresentField: present, + ir.PhysicalTableShapeExclusionCategoryTypeField: categoryType, + ir.PhysicalTableShapeExclusionOutputRowIDField: rowID, + ir.PhysicalTableShapeExclusionReasonField: ir.PhysicalTableShapeExclusionReasonUnlistedCategory, + ir.PhysicalTableShapeExclusionOmissionField: omission, + } +} + +func exec12JSON(value string) json.RawMessage { return json.RawMessage(value) } + +func exec12EngineForTarget(t *testing.T, target compiler.Target, port SourceRows) *Engine { + t.Helper() + engine, err := New(Config{ + CompileTarget: target, + Registry: invalidRecipeRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "exec12-offline-scope" }, + SourceRows: port, + }) + if err != nil { + t.Fatalf("engine: %v", err) + } + return engine +} + +func TestPostgreSQLTableShapeExclusionsDecodesCanonicalEvidenceRows(t *testing.T) { + rows := []map[string]any{ + exec12PGRow(exec12JSON(`"Patient"`), exec12JSON(`"patient-ada"`), ir.PhysicalTableShapeExclusionIdentityExact, exec12JSON(`true`), exec12JSON(`"zeta"`), "STRING", "row-1"), + exec12PGRow(exec12JSON(`"Patient"`), exec12JSON(`"patient-alan"`), ir.PhysicalTableShapeExclusionIdentityExact, exec12JSON(`true`), exec12JSON(`null`), "STRING", "row-2"), + exec12PGRow(exec12JSON(`"Patient"`), exec12JSON(`"patient-grace"`), ir.PhysicalTableShapeExclusionIdentityExact, exec12JSON(`false`), nil, "STRING", "row-3"), + exec12PGRow(nil, nil, ir.PhysicalTableShapeExclusionIdentityUnavailable, exec12JSON(`true`), exec12JSON(`"omega"`), "STRING", "row-4"), + } + var gotRequest RowsRequest + engine := &Engine{compileTarget: exec12PostgreSQLTarget(t), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + gotRequest = request + if request.Command.Backend() != command.CompiledBackendPostgreSQL { + t.Fatalf("port received %q-tagged command", request.Command.Backend()) + } + if _, _, err := request.Command.AQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("the AQL view must refuse the evidence command: %v", err) + } + for _, row := range rows { + if err := visit(row); err != nil { + return err + } + } + return nil + }} + result, err := engine.TableShapeExclusionsCompiled(context.Background(), exec12PGCompiled(0, 10)) + if err != nil { + t.Fatal(err) + } + wantContract := []string{ + ir.PhysicalTableShapeExclusionResourceTypeField, + ir.PhysicalTableShapeExclusionResourceIDField, + ir.PhysicalTableShapeExclusionIdentityStatusField, + ir.PhysicalTableShapeExclusionCategoryValueField, + ir.PhysicalTableShapeExclusionCategoryPresentField, + ir.PhysicalTableShapeExclusionCategoryTypeField, + ir.PhysicalTableShapeExclusionOutputRowIDField, + ir.PhysicalTableShapeExclusionReasonField, + ir.PhysicalTableShapeExclusionOmissionField, + } + if !reflect.DeepEqual(gotRequest.DeclaredColumns, wantContract) { + t.Fatalf("declared contract = %v, want %v", gotRequest.DeclaredColumns, wantContract) + } + if result.Status != TableShapeExclusionIncomplete || result.Complete || result.HasMore || result.NextOffset != 0 || len(result.Exclusions) != 4 { + t.Fatalf("result = %#v", result) + } + first := result.Exclusions[0] + if first.SourceIdentity == nil || first.SourceIdentity.ResourceType != "Patient" || first.SourceIdentity.ResourceID != "patient-ada" { + t.Fatalf("exact identity = %#v", first) + } + if first.Category.Present != true || !reflect.DeepEqual(first.Category.Value, exec12JSON(`"zeta"`)) { + t.Fatalf("string category = %#v", first.Category) + } + jsonNull := result.Exclusions[1].Category + if !jsonNull.Present || !reflect.DeepEqual(jsonNull.Value, exec12JSON(`null`)) { + t.Fatalf("explicit JSON-null category must survive as raw JSON, never flattened: %#v", jsonNull) + } + missing := result.Exclusions[2].Category + if missing.Present || missing.Value != nil { + t.Fatalf("missing category must carry SQL NULL (nil), distinct from JSON null: %#v", missing) + } + omitted := result.Exclusions[3] + if omitted.SourceIdentity != nil || omitted.OmissionCode != ir.PhysicalTableShapeExclusionSourceIdentityUnavailable { + t.Fatalf("identity omission = %#v", omitted) + } + if omitted.Reason != ir.PhysicalTableShapeExclusionReasonUnlistedCategory || omitted.CategoryType != "STRING" { + t.Fatalf("frozen labels = %#v", omitted) + } +} + +func TestPostgreSQLTableShapeExclusionsProbePagesAndTruncates(t *testing.T) { + all := []map[string]any{ + exec12PGRow(exec12JSON(`"Patient"`), exec12JSON(`"p-a"`), ir.PhysicalTableShapeExclusionIdentityExact, exec12JSON(`true`), exec12JSON(`"a"`), "STRING", "row-a"), + exec12PGRow(exec12JSON(`"Patient"`), exec12JSON(`"p-b"`), ir.PhysicalTableShapeExclusionIdentityExact, exec12JSON(`true`), exec12JSON(`"b"`), "STRING", "row-b"), + exec12PGRow(exec12JSON(`"Patient"`), exec12JSON(`"p-c"`), ir.PhysicalTableShapeExclusionIdentityExact, exec12JSON(`true`), exec12JSON(`"c"`), "STRING", "row-c"), + } + engine := &Engine{compileTarget: exec12PostgreSQLTarget(t), sourceRows: func(_ context.Context, request RowsRequest, visit RowVisitor) error { + _, args, err := request.Command.PostgreSQLParameters() + if err != nil { + return err + } + offset := int(args[0].Value.(int64)) + for _, row := range all[offset:] { + if err := visit(row); err != nil { + return err + } + } + return nil + }} + first, err := engine.TableShapeExclusionsCompiled(context.Background(), exec12PGCompiled(0, 2)) + if err != nil { + t.Fatal(err) + } + if first.Status != TableShapeExclusionIncomplete || first.Complete || !first.HasMore || first.NextOffset != 2 || len(first.Exclusions) != 2 { + t.Fatalf("bounded page = %#v", first) + } + second, err := engine.TableShapeExclusionsCompiled(context.Background(), exec12PGCompiled(2, 2)) + if err != nil { + t.Fatal(err) + } + if second.Status != TableShapeExclusionComplete || !second.Complete || second.HasMore || len(second.Exclusions) != 1 { + t.Fatalf("tail page = %#v", second) + } +} + +func TestPostgreSQLTableShapeExclusionsParserViolationsFailLoud(t *testing.T) { + base := func() map[string]any { + return exec12PGRow(exec12JSON(`"Patient"`), exec12JSON(`"p-a"`), ir.PhysicalTableShapeExclusionIdentityExact, exec12JSON(`true`), exec12JSON(`"a"`), "STRING", "row-a") + } + for _, test := range []struct { + name string + mutate func(row map[string]any) + }{ + {"presence as Go bool", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionCategoryPresentField] = true + }}, + {"presence as quoted string", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionCategoryPresentField] = exec12JSON(`"true"`) + }}, + {"presence as JSON null", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionCategoryPresentField] = exec12JSON(`null`) + }}, + {"presence as SQL NULL", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionCategoryPresentField] = nil + }}, + {"presence absent", func(row map[string]any) { + delete(row, ir.PhysicalTableShapeExclusionCategoryPresentField) + }}, + {"category value absent", func(row map[string]any) { + delete(row, ir.PhysicalTableShapeExclusionCategoryValueField) + }}, + {"category type as raw JSON", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionCategoryTypeField] = exec12JSON(`"STRING"`) + }}, + {"category type empty", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionCategoryTypeField] = "" + }}, + {"row identity empty", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionOutputRowIDField] = "" + }}, + {"reason drifted", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionReasonField] = "SOMETHING_ELSE" + }}, + {"identity status as raw JSON", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionIdentityStatusField] = exec12JSON(`"EXACT"`) + }}, + {"identity status unknown", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionIdentityStatusField] = "SOMETHING" + }}, + {"exact identity type not a string", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionResourceTypeField] = exec12JSON(`5`) + }}, + {"exact identity type empty string", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionResourceTypeField] = exec12JSON(`""`) + }}, + {"exact identity type SQL NULL", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionResourceTypeField] = nil + }}, + {"exact identity id JSON null", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionResourceIDField] = exec12JSON(`null`) + }}, + {"unavailable omission missing", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionIdentityStatusField] = ir.PhysicalTableShapeExclusionIdentityUnavailable + row[ir.PhysicalTableShapeExclusionOmissionField] = nil + }}, + {"unavailable omission drifted", func(row map[string]any) { + row[ir.PhysicalTableShapeExclusionIdentityStatusField] = ir.PhysicalTableShapeExclusionIdentityUnavailable + row[ir.PhysicalTableShapeExclusionOmissionField] = "WRONG" + }}, + } { + t.Run(test.name, func(t *testing.T) { + row := base() + test.mutate(row) + engine := &Engine{compileTarget: exec12PostgreSQLTarget(t), sourceRows: func(_ context.Context, _ RowsRequest, visit RowVisitor) error { + return visit(row) + }} + result, err := engine.TableShapeExclusionsCompiled(context.Background(), exec12PGCompiled(0, 10)) + if err == nil { + t.Fatalf("violating row must fail loud, got %#v", result) + } + if result.Exclusions != nil || result.Complete || result.Status == TableShapeExclusionComplete || result.Status == TableShapeExclusionIncomplete { + t.Fatalf("no partial success on parse violation: %#v", result) + } + }) + } +} + +func TestPostgreSQLTableShapeExclusionsPortErrorsReportHonestStatuses(t *testing.T) { + t.Run("backend-not-registered is the capability gap UNSUPPORTED", func(t *testing.T) { + engine := &Engine{compileTarget: exec12PostgreSQLTarget(t), sourceRows: func(context.Context, RowsRequest, RowVisitor) error { + return fmt.Errorf("%w: no postgres adapter installed", command.ErrCompiledCommandBackendNotRegistered) + }} + result, err := engine.TableShapeExclusionsCompiled(context.Background(), exec12PGCompiled(0, 10)) + if err != nil { + t.Fatalf("capability gap must report, not fail: %v", err) + } + if result.Status != TableShapeExclusionUnsupported || result.Complete || len(result.Exclusions) != 0 { + t.Fatalf("result = %#v", result) + } + }) + t.Run("backend-mismatch stays a fail-loud routing defect", func(t *testing.T) { + engine := &Engine{compileTarget: exec12PostgreSQLTarget(t), sourceRows: func(context.Context, RowsRequest, RowVisitor) error { + return fmt.Errorf("%w: sql routed to AQL", command.ErrCompiledCommandBackendMismatch) + }} + if _, err := engine.TableShapeExclusionsCompiled(context.Background(), exec12PGCompiled(0, 10)); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("routing defect must stay a hard error, got %v", err) + } + }) + t.Run("generic port failure never reports complete evidence", func(t *testing.T) { + engine := &Engine{compileTarget: exec12PostgreSQLTarget(t), sourceRows: func(context.Context, RowsRequest, RowVisitor) error { + return errors.New("server went away") + }} + result, err := engine.TableShapeExclusionsCompiled(context.Background(), exec12PGCompiled(0, 10)) + if err == nil || result.Status == TableShapeExclusionComplete { + t.Fatalf("execution failure must fail loud, got err=%v result=%#v", err, result) + } + }) + t.Run("cancellation reports INCOMPLETE at the requested offset", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + engine := &Engine{compileTarget: exec12PostgreSQLTarget(t), sourceRows: func(_ context.Context, _ RowsRequest, visit RowVisitor) error { + if err := visit(exec12PGRow(exec12JSON(`"Patient"`), exec12JSON(`"p-a"`), ir.PhysicalTableShapeExclusionIdentityExact, exec12JSON(`true`), exec12JSON(`"a"`), "STRING", "row-a")); err != nil { + return err + } + cancel() + return ctx.Err() + }} + result, err := engine.TableShapeExclusionsCompiled(ctx, exec12PGCompiled(5, 10)) + if err != nil { + t.Fatal(err) + } + if result.Status != TableShapeExclusionIncomplete || result.Complete || len(result.Exclusions) != 0 || result.NextOffset != 5 { + t.Fatalf("cancellation result = %#v", result) + } + }) + t.Run("AQL arm keeps its hard-error law unchanged", func(t *testing.T) { + engine := &Engine{compileTarget: compiler.ArangoTarget(), sourceRows: func(context.Context, RowsRequest, RowVisitor) error { + return fmt.Errorf("%w: arango down", command.ErrCompiledCommandBackendNotRegistered) + }} + if _, err := engine.TableShapeExclusionsCompiled(context.Background(), compiledTableShapeExclusionPage(0, 10)); err == nil { + t.Fatal("AQL arm must not map port errors to UNSUPPORTED") + } + }) + t.Run("nil port is UNSUPPORTED for PostgreSQL and unchanged for AQL", func(t *testing.T) { + pg := &Engine{compileTarget: exec12PostgreSQLTarget(t)} + result, err := pg.TableShapeExclusionsCompiled(context.Background(), exec12PGCompiled(0, 10)) + if err != nil || result.Status != TableShapeExclusionUnsupported || result.Complete { + t.Fatalf("PG nil port = %#v, %v", result, err) + } + aql := &Engine{compileTarget: compiler.ArangoTarget()} + if _, err := aql.TableShapeExclusionsCompiled(context.Background(), compiledTableShapeExclusionPage(0, 10)); err == nil || err.Error() != "table-shape exclusion query executor is required" { + t.Fatalf("AQL nil port error drifted: %v", err) + } + }) + t.Run("unregistered or zero tag keeps the typed metadata error", func(t *testing.T) { + portCalls := 0 + engine := &Engine{compileTarget: exec12PostgreSQLTarget(t), sourceRows: func(context.Context, RowsRequest, RowVisitor) error { + portCalls++ + return nil + }} + if _, err := engine.TableShapeExclusionsCompiled(context.Background(), exec12PGCompiled(0, 10)); err != nil { + t.Fatal(err) + } + portCalls = 0 + broken := exec12PGCompiled(0, 10) + broken.ReasonField = "" + if _, err := engine.TableShapeExclusionsCompiled(context.Background(), broken); err == nil || err.Error() != "table-shape exclusion query is missing typed result metadata" { + t.Fatalf("missing-metadata error drifted: %v", err) + } + zero := exec12PGCompiled(0, 10) + zero.Command = command.CompiledCommand{} + if _, err := engine.TableShapeExclusionsCompiled(context.Background(), zero); err == nil || err.Error() != "table-shape exclusion query is missing typed result metadata" { + t.Fatalf("zero-tag command must be refused with the typed metadata error: %v", err) + } + if portCalls != 0 { + t.Fatalf("refused commands must never reach the port: %d calls", portCalls) + } + }) +} + +func TestTableShapeExclusionsPostgreSQLTargetEntryRefusalsStayDistinct(t *testing.T) { + evidenceReshape := func() *recipe.TableReshape { + return &recipe.TableReshape{Kind: recipe.TableReshapeGroupedPivot, GroupedPivot: &recipe.GroupedPivot{ + ConstructionID: "pivot", GroupKeys: []string{"group"}, CategoryColumn: "category", ValueColumn: "value", + Categories: []recipe.GroupedPivotCategory{{Key: recipe.TableScalar{Kind: recipe.TableScalarString, String: stringPtr("known")}, Output: "known", Label: "Known"}}, + DuplicatePolicy: recipe.PivotDuplicateSum, MissingCellPolicy: recipe.PivotMissingCellNull, + UnlistedCategoryPolicy: recipe.PivotUnlistedCategoryExcludeWithEvidence, + }} + } + noPolicyReshape := func() *recipe.TableReshape { + reshape := evidenceReshape() + reshape.GroupedPivot.UnlistedCategoryPolicy = recipe.PivotUnlistedCategoryError + return reshape + } + filteredBundle := func() recipe.Bundle { + bundle := exclusionStatusBundle(evidenceReshape()) + status := "final" + bundle.Outputs[0].Filters = append(bundle.Outputs[0].Filters, recipe.Filter{ + Select: "root.status", Operator: recipe.FilterEquals, + Values: []recipe.FilterValue{{Kind: recipe.FilterString, String: &status}}, + }) + return bundle + } + bindings := recipe.RuntimeBindings{Project: "project", DatasetGeneration: "generation"} + + statements := 0 + pgEngine := exec12EngineForTarget(t, exec12PostgreSQLTarget(t), func(_ context.Context, request RowsRequest, visit RowVisitor) error { + statements++ + if len(request.DeclaredColumns) != 9 { + t.Fatalf("registered evidence read must carry the nine-field contract, got %v", request.DeclaredColumns) + } + return visit(exec12PGRow(exec12JSON(`"Observation"`), exec12JSON(`"obs-1"`), ir.PhysicalTableShapeExclusionIdentityExact, exec12JSON(`true`), exec12JSON(`"zeta"`), "STRING", "row-1")) + }) + + t.Run("no-exclusion policy keeps its own status with zero statements", func(t *testing.T) { + statements = 0 + resolved, err := pgEngine.CompileResolvedBundle(context.Background(), exclusionStatusBundle(noPolicyReshape()), bindings) + if err != nil { + t.Fatal(err) + } + result, err := pgEngine.TableShapeExclusions(context.Background(), resolved, TableShapeExclusionRequest{Output: "Observations", Limit: 10}) + if err != nil { + t.Fatal(err) + } + if result.Status != TableShapeExclusionNoExclusionPolicy || result.Complete || len(result.Exclusions) != 0 || statements != 0 { + t.Fatalf("no-policy result = %#v after %d statements", result, statements) + } + }) + t.Run("outside-slice shape reports UNSUPPORTED with zero statements", func(t *testing.T) { + statements = 0 + resolved, err := pgEngine.CompileResolvedBundle(context.Background(), filteredBundle(), bindings) + if err != nil { + t.Fatal(err) + } + result, err := pgEngine.TableShapeExclusions(context.Background(), resolved, TableShapeExclusionRequest{Output: "Observations", Limit: 10}) + if err != nil { + t.Fatal(err) + } + if result.Status != TableShapeExclusionUnsupported || result.Complete || len(result.Exclusions) != 0 || statements != 0 { + t.Fatalf("outside-slice result = %#v after %d statements", result, statements) + } + }) + t.Run("registered shape executes real evidence: UNSUPPORTED is shape-derived, not blanket", func(t *testing.T) { + statements = 0 + resolved, err := pgEngine.CompileResolvedBundle(context.Background(), exclusionStatusBundle(evidenceReshape()), bindings) + if err != nil { + t.Fatal(err) + } + result, err := pgEngine.TableShapeExclusions(context.Background(), resolved, TableShapeExclusionRequest{Output: "Observations", Limit: 10}) + if err != nil { + t.Fatal(err) + } + if result.Status != TableShapeExclusionComplete || !result.Complete || len(result.Exclusions) != 1 || statements != 1 { + t.Fatalf("registered control = %#v after %d statements", result, statements) + } + }) + t.Run("AQL twin keeps all three current statuses (no reroute)", func(t *testing.T) { + aqlStatements := 0 + aqlEngine := exec12EngineForTarget(t, compiler.ArangoTarget(), func(_ context.Context, request RowsRequest, visit RowVisitor) error { + aqlStatements++ + if len(request.DeclaredColumns) != 0 { + t.Fatalf("the AQL arm stays contract-free, got %v", request.DeclaredColumns) + } + return visit(map[string]any{ + ir.PhysicalTableShapeExclusionResourceTypeField: "Observation", + ir.PhysicalTableShapeExclusionResourceIDField: "obs-1", + ir.PhysicalTableShapeExclusionIdentityStatusField: ir.PhysicalTableShapeExclusionIdentityExact, + ir.PhysicalTableShapeExclusionCategoryValueField: "zeta", + ir.PhysicalTableShapeExclusionCategoryPresentField: true, + ir.PhysicalTableShapeExclusionCategoryTypeField: "STRING", + ir.PhysicalTableShapeExclusionOutputRowIDField: "row-1", + ir.PhysicalTableShapeExclusionReasonField: ir.PhysicalTableShapeExclusionReasonUnlistedCategory, + ir.PhysicalTableShapeExclusionOmissionField: any(nil), + }) + }) + resolved, err := aqlEngine.CompileResolvedBundle(context.Background(), exclusionStatusBundle(noPolicyReshape()), bindings) + if err != nil { + t.Fatal(err) + } + if result, err := aqlEngine.TableShapeExclusions(context.Background(), resolved, TableShapeExclusionRequest{Output: "Observations", Limit: 10}); err != nil || result.Status != TableShapeExclusionNoExclusionPolicy { + t.Fatalf("AQL no-policy twin drifted: %#v %v", result, err) + } + filteredResolved, err := aqlEngine.CompileResolvedBundle(context.Background(), filteredBundle(), bindings) + if err != nil { + t.Fatal(err) + } + aqlStatements = 0 + result, err := aqlEngine.TableShapeExclusions(context.Background(), filteredResolved, TableShapeExclusionRequest{Output: "Observations", Limit: 10}) + if err != nil || result.Status != TableShapeExclusionComplete || len(result.Exclusions) != 1 || aqlStatements != 1 { + t.Fatalf("AQL outside-slice twin drifted: %#v %v after %d statements", result, err, aqlStatements) + } + }) +} diff --git a/internal/dataframe/execution/types.go b/internal/dataframe/execution/types.go index 550c118fa..dfc154b4d 100644 --- a/internal/dataframe/execution/types.go +++ b/internal/dataframe/execution/types.go @@ -3,12 +3,11 @@ package execution import ( "time" - "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/spec" ) -type CompiledQuery = compiler.CompiledQuery type RowIdentity = spec.RowIdentity type CompilerPlanDiagnostics = ir.CompilerPlanDiagnostics @@ -19,23 +18,18 @@ type Result struct { Diagnostics QueryDiagnostics } -// QueryDiagnostics separates the cost of turning a dataframe request into -// rows. ArangoQuery is cursor time excluding Loom's per-row processing; -// RowMaterialization is the time spent flattening and delivering rows. type QueryDiagnostics struct { InputResolution time.Duration RequestPreparation time.Duration Compilation time.Duration - ArangoQuery time.Duration + SourceQuery time.Duration + SourceBackend command.CompiledBackend RowMaterialization time.Duration ResultAssembly time.Duration Total time.Duration Plan ir.CompilerPlanDiagnostics } -// streamResult is the internal result of generic compiled-query iteration. -// The public recipe-facing StreamResult remains in engine.go and carries the -// output name and publication-safe columns. type streamResult struct { Columns []string RowCount int diff --git a/internal/dataframe/execution/vendor_import_guard_test.go b/internal/dataframe/execution/vendor_import_guard_test.go new file mode 100644 index 000000000..ba5f7480a --- /dev/null +++ b/internal/dataframe/execution/vendor_import_guard_test.go @@ -0,0 +1,87 @@ +package execution + +import ( + "context" + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" + "testing" + + arangofactory "github.com/calypr/loom/internal/backends/arango" +) + +var forbiddenVendorImports = []string{ + "github.com/calypr/loom/internal/backends/arango/internal/store", + "github.com/arangodb/go-driver", + "github.com/jackc/pgx", +} + +func vendorImportOffenders(dir string, includeTests bool) []string { + entries, err := os.ReadDir(dir) + if err != nil { + return []string{"read " + dir + ": " + err.Error()} + } + var offenders []string + fset := token.NewFileSet() + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".go") { + continue + } + if !includeTests && strings.HasSuffix(name, "_test.go") { + continue + } + file, parseErr := parser.ParseFile(fset, filepath.Join(dir, name), nil, parser.ImportsOnly) + if parseErr != nil { + offenders = append(offenders, name+": "+parseErr.Error()) + continue + } + for _, imported := range file.Imports { + path := strings.Trim(imported.Path.Value, `"`) + for _, forbidden := range forbiddenVendorImports { + if path == forbidden || strings.HasPrefix(path, forbidden+"/") { + offenders = append(offenders, name+" imports "+path) + } + } + } + } + return offenders +} + +func TestProductionExecutionFilesNeverImportDatabaseVendors(t *testing.T) { + if offenders := vendorImportOffenders(".", false); len(offenders) != 0 { + t.Fatalf("execution production files import database vendors: %v", offenders) + } + + t.Run("non_vacuity_scanner_fires_on_a_crafted_import", func(t *testing.T) { + dir := t.TempDir() + crafted := "package execution\n\nimport \"github.com/calypr/loom/internal/backends/arango/internal/store\"\n\nvar _ = arango.Client{}\n" + if err := os.WriteFile(filepath.Join(dir, "arango_import_witness.go"), []byte(crafted), 0o600); err != nil { + t.Fatal(err) + } + if offenders := vendorImportOffenders(dir, false); len(offenders) == 0 { + t.Fatal("guard did not fire on a re-added ArangoDB import; it would not die when vendors return") + } + }) + + t.Run("non_vacuity_scanner_fires_on_a_real_vendor_package", func(t *testing.T) { + if offenders := vendorImportOffenders(filepath.Join("..", "..", "backends", "arango", "internal", "store"), false); len(offenders) == 0 { + t.Fatal("scanner missed the known vendor imports in the ArangoDB adapter") + } + }) +} + +func TestQueryMemoryBudgetIsExecutionOwnedNeutralValue(t *testing.T) { + if _, ok := QueryMemoryBudgetFromContext(context.Background()); ok { + t.Fatal("unset context reported a budget") + } + ctx := WithQueryMemoryBudget(context.Background(), 4096) + if budget, ok := QueryMemoryBudgetFromContext(ctx); !ok || budget != 4096 { + t.Fatalf("budget = %d/%t, want 4096/true", budget, ok) + } + if _, ok := arangofactory.QueryMemoryLimitFromContext(ctx); ok { + t.Fatal("execution attached the ArangoDB memory-limit mechanism itself") + } +} diff --git a/internal/dataframe/execution/verify_arch01_pins_test.go b/internal/dataframe/execution/verify_arch01_pins_test.go new file mode 100644 index 000000000..c08ef0696 --- /dev/null +++ b/internal/dataframe/execution/verify_arch01_pins_test.go @@ -0,0 +1,251 @@ +package execution + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/semantic" + "github.com/calypr/loom/internal/dataframe/spec" +) + +type verifyArch01Trace struct { + mu sync.Mutex + aqlFull []string + aqlPreview []string + pgFull []string + pgPreview []string + controls []string + chCalls int +} + +func (v *verifyArch01Trace) add(list *[]string, item string) { + v.mu.Lock() + defer v.mu.Unlock() + *list = append(*list, item) +} + +func (v *verifyArch01Trace) snapshot() (aqlFull, aqlPreview, pgFull, pgPreview, controls []string, ch int) { + v.mu.Lock() + defer v.mu.Unlock() + return append([]string(nil), v.aqlFull...), append([]string(nil), v.aqlPreview...), + append([]string(nil), v.pgFull...), append([]string(nil), v.pgPreview...), + append([]string(nil), v.controls...), v.chCalls +} + +func (v *verifyArch01Trace) quietAQL(t *testing.T, stage string) { + t.Helper() + aqlFull, aqlPreview, _, _, controls, ch := v.snapshot() + if len(aqlFull) != 0 || len(aqlPreview) != 0 || len(controls) != 0 || ch != 0 { + t.Fatalf("%s: PostgreSQL plan touched Arango/ClickHouse callbacks: aql=%v preview=%v controls=%v ch=%d", + stage, aqlFull, aqlPreview, controls, ch) + } +} + +func verifyArch01Config(v *verifyArch01Trace) Config { + return Config{ + CompileTarget: compiler.ArangoTarget(), + Registry: invalidRecipeRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "verify-arch01-scope" }, + SourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + text, _, err := command.AQLParameters() + if err != nil { + return err + } + v.add(&v.aqlFull, text) + return nil + }, + PreviewSourceRows: func(_ context.Context, request RowsRequest, _ RowVisitor) error { + command := request.Command + text, _, err := command.AQLParameters() + if err != nil { + return err + } + v.add(&v.aqlPreview, text) + return nil + }, + PreparePreviewIndex: func(context.Context, compiler.PreviewCoveringIndexSpec) error { + v.add(&v.controls, "preparePreviewIndex") + return nil + }, + PreviewCoveringIndexes: func(context.Context, string) ([]PreviewCollectionIndex, error) { + v.add(&v.controls, "previewCoveringIndexes") + return nil, nil + }, + PreviewCollectionRevision: func(context.Context, string) (string, error) { + v.add(&v.controls, "previewCollectionRevision") + return "rev-arch01", nil + }, + PreviewCollectionScans: func(context.Context, command.CompiledCommand, string) (PreviewCollectionScanEvidence, error) { + v.add(&v.controls, "previewCollectionScans") + return PreviewCollectionScanEvidence{}, nil + }, + PreviewCollectionCount: func(context.Context, string) (int64, error) { + v.add(&v.controls, "previewCollectionCount") + return 3, nil + }, + ClickHouseQueryRows: func(context.Context, string, []string, func(map[string]any) error, ...any) error { + v.mu.Lock() + v.chCalls++ + v.mu.Unlock() + return nil + }, + } +} + +func verifyArch01Resolved(columns []string, schema []lower.CompiledOutputColumn) Resolved { + output := lower.CompiledRecipeOutput{ + Name: "resources", + RootResourceType: "Patient", + Columns: append([]string(nil), columns...), + OutputSchema: schema, + RowIdentity: &spec.RowIdentity{Fields: []string{"id"}}, + Plan: ir.PhysicalPlan{Version: 1, Engine: ir.PhysicalEngineAQL}, + } + return Resolved{ + Bundle: recipe.Bundle{Name: "verify_arch01"}, + Semantic: semantic.ResolvedRecipePlan{ + SemanticPlan: semantic.RecipePlan{Bindings: recipe.RuntimeBindings{ + Project: "P1", DatasetGeneration: "G1", OutputNames: []string{"resources"}, + }}, + }, + Compiled: lower.CompiledRecipe{Outputs: []lower.CompiledRecipeOutput{output}}, + StoredRecipeDigest: "verify-arch01", + } +} + +func verifyArch01Schema(names ...string) []lower.CompiledOutputColumn { + schema := make([]lower.CompiledOutputColumn, 0, len(names)) + for _, name := range names { + schema = append(schema, lower.CompiledOutputColumn{ID: name, Name: name, Label: name, Kind: "string", Cardinality: "one"}) + } + return schema +} + +func verifyArch01Command(t *testing.T) command.CompiledCommand { + t.Helper() + compiledCommand, err := command.NewPostgreSQLCommand( + `SELECT id FROM loom.resource WHERE project = $1`, + []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterText, Value: "P1"}}) + if err != nil { + t.Fatalf("forge command: %v", err) + } + return compiledCommand +} + +func verifyArch01PostgreSQLPort(v *verifyArch01Trace, target *[]string, rows []map[string]any) SourceRows { + return func(ctx context.Context, request RowsRequest, visit RowVisitor) error { + sqlText, _, err := request.Command.PostgreSQLParameters() + if err != nil { + v.add(&v.aqlFull, "misrouted:"+err.Error()) + return err + } + v.add(target, sqlText) + for _, row := range rows { + if err := ctx.Err(); err != nil { + return err + } + if err := visit(row); err != nil { + return err + } + } + return nil + } +} + +func verifyArch01Rows() []map[string]any { + return []map[string]any{{"id": "one"}, {"id": "two"}} +} + +func TestVerifyArch01ResolverPanicFailsClosedNoCallbacks(t *testing.T) { + v := &verifyArch01Trace{} + config := verifyArch01Config(v) + config.CompileTarget = verifyArch01PostgreSQLTarget(t) + config.SourceRows = func(context.Context, RowsRequest, RowVisitor) error { + panic("resolver exploded") + } + config.PreviewSourceRows = nil + engine, err := New(config) + if err != nil { + t.Fatalf("New: %v", err) + } + defer func() { + recovered := recover() + if recovered == nil { + t.Fatalf("port panic was swallowed; engine must not silently continue") + } + if msg := fmt.Sprint(recovered); !strings.Contains(msg, "resolver exploded") { + t.Fatalf("panic identity lost: %v", recovered) + } + aqlFull, aqlPreview, pgFull, _, controls, ch := v.snapshot() + if len(aqlFull) != 0 || len(aqlPreview) != 0 || len(pgFull) != 0 || len(controls) != 0 || ch != 0 { + t.Fatalf("port panic touched adapters: aql=%v preview=%v pg=%d controls=%v ch=%d", + aqlFull, aqlPreview, len(pgFull), controls, ch) + } + }() + _, err = engine.Run(context.Background(), verifyArch01PostgreSQLResolved("resources", []string{"id"})) + t.Fatalf("panic must propagate; Run returned err=%v", err) +} + +func TestVerifyArch01VisitorErrorClearsMarkers(t *testing.T) { + v := &verifyArch01Trace{} + config := verifyArch01Config(v) + config.CompileTarget = verifyArch01PostgreSQLTarget(t) + config.SourceRows = verifyArch01PostgreSQLPort(v, &v.pgFull, verifyArch01Rows()) + config.PreviewSourceRows = nil + engine, err := New(config) + if err != nil { + t.Fatalf("New: %v", err) + } + visitorFailure := errors.New("verify_arch01 visitor refused") + summary, err := engine.PreviewOutput(context.Background(), verifyArch01PostgreSQLResolved("resources", []string{"id"}), + PreviewRequest{Output: "resources", Limit: 10}, func(map[string]any) error { return visitorFailure }) + if !errors.Is(err, visitorFailure) { + t.Fatalf("visitor failure must surface: %v", err) + } + if summary.Complete || summary.Truncated { + t.Fatalf("visitor-interrupted preview must carry neither marker: %+v", summary) + } + v.quietAQL(t, "visitor error") +} + +func TestVerifyArch01ContextErrorAfterFinalRowClearsMarkers(t *testing.T) { + v := &verifyArch01Trace{} + config := verifyArch01Config(v) + config.CompileTarget = verifyArch01PostgreSQLTarget(t) + config.SourceRows = verifyArch01PostgreSQLPort(v, &v.pgFull, verifyArch01Rows()) + config.PreviewSourceRows = nil + engine, err := New(config) + if err != nil { + t.Fatalf("New: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rows := 0 + summary, err := engine.PreviewOutput(ctx, verifyArch01PostgreSQLResolved("resources", []string{"id"}), + PreviewRequest{Output: "resources", Limit: 10}, func(map[string]any) error { + rows++ + if rows == 2 { + cancel() + } + return nil + }) + if err == nil { + t.Fatalf("canceled preview must report the context error, not nil: %+v", summary) + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("must satisfy errors.Is(context.Canceled): %v", err) + } + if summary.Complete || summary.Truncated { + t.Fatalf("post-stream cancellation must clear both markers: %+v", summary) + } +} diff --git a/internal/dataframe/expression/expression_checking.go b/internal/dataframe/expression/expression_checking.go index e465b236e..7537b9c85 100644 --- a/internal/dataframe/expression/expression_checking.go +++ b/internal/dataframe/expression/expression_checking.go @@ -1,6 +1,3 @@ -// Package expression contains the typed, backend-neutral expression AST used -// by dataframe recipes. It intentionally has no knowledge of AQL, SQL, or -// output names: a physical compiler consumes the checked tree later. package expression import ( @@ -413,5 +410,3 @@ func mergeBranchTypes(a, b Type) (Type, error) { a.Cardinality = mergeCardinality(a.Cardinality, b.Cardinality) return a, nil } - -// Constructors keep callers from manufacturing malformed tagged nodes. diff --git a/internal/dataframe/expression/expression_types.go b/internal/dataframe/expression/expression_types.go index a7e6a3e7b..d07633e69 100644 --- a/internal/dataframe/expression/expression_types.go +++ b/internal/dataframe/expression/expression_types.go @@ -1,6 +1,3 @@ -// Package expression contains the typed, backend-neutral expression AST used -// by dataframe recipes. It intentionally has no knowledge of AQL, SQL, or -// output names: a physical compiler consumes the checked tree later. package expression import ( @@ -9,9 +6,6 @@ import ( "strings" ) -// ValueKind is the logical type of an expression value. Cardinality is kept -// separately in Type so a repeated string and a scalar string remain visibly -// different to a compiler. type ValueKind string const ( @@ -24,14 +18,9 @@ const ( KindDateTime ValueKind = "date_time" KindCode ValueKind = "code" KindUUID ValueKind = "uuid" - // KindObject represents a structured value whose fields are resolved by a - // lexical recipe scope. It is intentionally backend-neutral; physical - // lowerers decide whether the object is a document, object expression, or - // relationship item. - KindObject ValueKind = "object" + KindObject ValueKind = "object" ) -// Cardinality describes the number of values produced for one row context. type Cardinality string const ( @@ -47,9 +36,6 @@ func (c Cardinality) Valid() bool { func (c Cardinality) AllowsMany() bool { return c == Many } func (c Cardinality) Optional() bool { return c == OptionalOne || c == Many } -// Type is a logical expression type. Element is reserved for object-like -// types in later recipe versions; scalar arrays are represented by the same -// Kind with Cardinality=Many so functions can reason about them uniformly. type Type struct { Kind ValueKind `json:"kind"` Cardinality Cardinality `json:"cardinality"` @@ -78,8 +64,6 @@ func (t Type) String() string { func scalar(kind ValueKind) Type { return Type{Kind: kind, Cardinality: RequiredOne} } -// NullBehavior makes null handling part of semantic meaning rather than an -// incidental renderer choice. type NullBehavior string const ( @@ -92,28 +76,19 @@ func (n NullBehavior) Valid() bool { return n == "" || n == NullPropagate || n == NullEmpty || n == NullError } -// NodeKind identifies a node in the expression tree. type NodeKind string const ( - SelectorNode NodeKind = "selector" - LiteralNode NodeKind = "literal" - CallNode NodeKind = "call" - // DocumentRefNode denotes the complete document in a lexical resource - // context. It intentionally carries no selector or backend query text; - // physical lowerers decide how to materialize the document envelope. + SelectorNode NodeKind = "selector" + LiteralNode NodeKind = "literal" + CallNode NodeKind = "call" DocumentRefNode NodeKind = "document_ref" ) -// DocumentRef identifies the complete document in a named lexical context. -// Context defaults to root and is validated against the surrounding semantic -// scope by the recipe planner. type DocumentRef struct { Context string `json:"context,omitempty"` } -// SelectorRef identifies a value in a named row context. The path is a -// logical selector, not a query-language fragment. type SelectorRef struct { Context string `json:"context,omitempty"` Path string `json:"path"` @@ -142,23 +117,17 @@ func (s SelectorRef) Validate() error { return nil } -// Literal is a typed constant. Value is intentionally opaque to this package -// after type validation, which keeps physical encodings out of the AST. type Literal struct { Type Type `json:"type"` Value any `json:"value,omitempty"` } -// Call is a named generic function. Target is only used by cast and is kept -// as a type rather than accepting a backend or language type name. type Call struct { Name string `json:"name"` Args []Expression `json:"args,omitempty"` Target *Type `json:"target,omitempty"` } -// Expression is a lowering-neutral AST node. Type is an optional annotation; -// Check recomputes it and rejects a conflicting annotation. type Expression struct { Kind NodeKind `json:"kind"` Type Type `json:"type,omitempty"` @@ -169,12 +138,8 @@ type Expression struct { Document *DocumentRef `json:"document,omitempty"` } -// SelectorResolver supplies schema-aware types without coupling this package -// to a particular generated FHIR schema or storage backend. type SelectorResolver func(SelectorRef) (Type, error) -// TypeContext is the environment used to type-check selectors. Selectors map -// keys may be either SelectorRef.String() or the path alone. type TypeContext struct { Selectors map[string]Type Resolve SelectorResolver @@ -182,19 +147,15 @@ type TypeContext struct { MaxNodes int } -// CheckedExpression is an expression plus its inferred semantic result type. type CheckedExpression struct { Expression Expression Type Type } -// Check validates an expression tree and returns its inferred type. func Select(ref SelectorRef) Expression { return Expression{Kind: SelectorNode, Selector: &ref, NullBehavior: NullPropagate} } -// Document returns an expression referring to the complete document in -// context. An empty context is normalized to root by semantic planning. func Document(context string) Expression { return Expression{Kind: DocumentRefNode, Document: &DocumentRef{Context: strings.TrimSpace(context)}, NullBehavior: NullPropagate} } diff --git a/internal/dataframe/expression/recipe_adapter.go b/internal/dataframe/expression/recipe_adapter.go index 5dfcd450a..b5b9c8ff4 100644 --- a/internal/dataframe/expression/recipe_adapter.go +++ b/internal/dataframe/expression/recipe_adapter.go @@ -9,9 +9,6 @@ import ( "github.com/calypr/loom/internal/dataframe/recipe" ) -// FromRecipe converts the persisted wire expression into the typed AST. Type -// checking remains a separate operation because selector types come from the -// active generated schema/catalog. func FromRecipe(input recipe.Expression) (Expression, error) { if input.Document != nil { context := strings.TrimSpace(input.Document.Context) @@ -60,11 +57,6 @@ func FromRecipe(input recipe.Expression) (Expression, error) { return Expression{Kind: CallNode, Call: &call, NullBehavior: NullPropagate}, nil } -// FromRecipeInContexts is the scope-aware variant used by semantic recipe -// lowering. The wire format keeps selectors as strings; this helper turns a -// leading lexical alias into SelectorRef.Context while rejecting aliases that -// are not visible at the current recipe node. An unqualified selector is -// resolved by the caller's root scope. func FromRecipeInContexts(input recipe.Expression, contexts map[string]struct{}) (Expression, error) { expr, err := FromRecipe(input) if err != nil { diff --git a/internal/dataframe/lineage/source_child.go b/internal/dataframe/lineage/source_child.go index 6ae94d445..ad75482b1 100644 --- a/internal/dataframe/lineage/source_child.go +++ b/internal/dataframe/lineage/source_child.go @@ -1,5 +1,3 @@ -// Package lineage defines stable identities and provenance for columns that a -// single authored source slot expands into at compile time. package lineage import ( @@ -21,16 +19,11 @@ const ( RepeatedCountChild SourceChildKind = "REPEATED_COUNT" ) -// Coordinate identifies one concrete item at a repeated boundary. Width is -// intentionally absent: discovering more items must not change old IDs. type Coordinate struct { BoundaryPath string `json:"boundaryPath"` Index int `json:"index"` } -// SourceChild records the source slot and structural evidence that produced -// one public child column. ParentColumnIDs is sorted and may contain several -// owners when indexed projections share a repeated-count emission. type SourceChild struct { Kind SourceChildKind `json:"kind"` ParentColumnIDs []string `json:"parentColumnIds"` @@ -40,9 +33,6 @@ type SourceChild struct { Coordinates []Coordinate `json:"coordinates,omitempty"` } -// StableSourceChildID returns the deterministic public identity for a source -// child. Names, labels, and observed widths are presentation or catalog facts -// and are deliberately excluded. func StableSourceChildID(child SourceChild) (string, SourceChild, error) { child = cloneSourceChild(child) if child.Kind != IndexedValueChild && child.Kind != RepeatedCountChild { @@ -99,8 +89,6 @@ func StableSourceChildID(child SourceChild) (string, SourceChild, error) { return childID, child, nil } -// ParseSourceChildID verifies the canonical wire form and returns the -// structural lineage encoded in a generated child ID. func ParseSourceChildID(value string) (SourceChild, bool) { if !strings.HasPrefix(value, sourceChildPrefix) { return SourceChild{}, false @@ -131,14 +119,10 @@ func ParseSourceChildID(value string) (SourceChild, bool) { return child, true } -// IsSourceChildID reports whether value uses the reserved generated-child ID -// namespace, including malformed values that ParseSourceChildID rejects. func IsSourceChildID(value string) bool { return strings.HasPrefix(value, sourceChildPrefix) } -// CloneSourceChild returns a deep copy suitable for retaining in compiler -// schemas that are copied between construction stages. func CloneSourceChild(child *SourceChild) *SourceChild { if child == nil { return nil diff --git a/internal/dataframe/publication/bundle.go b/internal/dataframe/publication/bundle.go index 9651769ea..bb9a10711 100644 --- a/internal/dataframe/publication/bundle.go +++ b/internal/dataframe/publication/bundle.go @@ -16,9 +16,6 @@ import ( type DataframeSelector = dataset.DataframeSelector -// BundleState is the durable lifecycle of a multi-output publication. The -// physical ClickHouse tables are never reader-visible until the logical -// pointer and PUBLISHED execution are committed atomically. type BundleState string const ( @@ -28,7 +25,6 @@ const ( BundlePublished BundleState = "PUBLISHED" BundleFailed BundleState = "FAILED" - // Compatibility-only stored values. New workflows never write these. BundlePending BundleState = "PENDING" BundlePreflight BundleState = "PREFLIGHT" BundleLoading BundleState = "LOADING" @@ -65,8 +61,6 @@ type BundleIdentity struct { AuthResourcePaths []string `json:"authResourcePaths,omitempty"` } -// Canonical returns an identity with its set-valued authorization paths in a -// stable order without changing the persisted key shape. func (i BundleIdentity) Canonical() BundleIdentity { if i.AuthResourcePaths == nil { return i @@ -83,20 +77,6 @@ func (i BundleIdentity) Canonical() BundleIdentity { return i } -// PointerName is the visibility namespace for a published logical dataset. -// Project and generation are part of the key so two tenants can publish the -// same recipe/output name without racing a shared pointer. -// -// Repair-5 (F-B21): components are length-prefixed, mirroring the discipline -// the catalog already uses for identity digests (internal/catalog/helpers.go -// catalogIdentityDigest) and Key() below. The previous bare "\x00"-join let -// (Name="a\x00b", TranslationVersion="") and (Name="a", TranslationVersion= -// "b") fold to one visibility pointer while their Keys stayed distinct, so -// publishing the second identity silently repointed the first dataset's -// readers. The encoding is injective without needing input validation: no -// component content — including embedded NULs or digits/colons — can shift a -// component boundary, because each component is written as len:content. -// Blank TranslationVersion/OutputName remain "absent" exactly as shipped. func (i BundleIdentity) PointerName() string { parts := []string{i.Project, i.DatasetGeneration, i.Name} if strings.TrimSpace(i.TranslationVersion) != "" { @@ -146,9 +126,6 @@ type BundleOutputRecord struct { SourceRow *SourceRowMetadata `json:"sourceRow,omitempty"` } -// SourceRowMetadata is persisted only when output rows retain a proven typed -// source-resource identity. It prevents selection from guessing an ID from a -// display label or an opaque row number. type SourceRowMetadata struct { ResourceType string `json:"resourceType"` IDColumn string `json:"idColumn"` @@ -227,8 +204,6 @@ type BundlePointer struct { UpdatedAt time.Time `json:"updatedAt"` } -// BundleCatalog is the durable metadata/pointer boundary. Implementations -// make pointer updates and lease acquisition atomic in their backing store. type BundleCatalog interface { SaveExecution(context.Context, BundleExecution, string) error GetExecution(context.Context, string) (BundleExecution, error) @@ -242,22 +217,16 @@ type BundleCatalog interface { ReleaseBundleLease(context.Context, string, string) error } -// BundleExecutionPageFunc consumes one bounded reconciliation page. type BundleExecutionPageFunc func([]BundleExecution) error -// PagedBundleCatalog provides bounded, stable execution scans for repair jobs. -// Implementations must call visit sequentially and stop when it returns an error. type PagedBundleCatalog interface { VisitExecutionPages(context.Context, BundleState, time.Time, int, BundleExecutionPageFunc) error } -// ExactExecutionCatalog is consumed by project release verification. It never -// falls back to latest-by-output or a name-only recipe. type ExactExecutionCatalog interface { FindExecutionBySelector(context.Context, string, string, DataframeSelector) (BundleExecution, BundleOutputRecord, error) } -// PhaseError preserves operational context without making clients parse text. type PhaseError struct { Phase string Output string @@ -289,18 +258,11 @@ var ErrBundleNotFound = fmt.Errorf("bundle execution not found") var ErrBundlePointerConflict = fmt.Errorf("bundle pointer compare-and-swap conflict") var ErrBundleLeaseLost = fmt.Errorf("bundle lease ownership was lost") -// ErrExecutionReadPinLost means a reader no longer owns the retention pin -// that protects an exact published execution. Readers must cancel their scan -// rather than continue against a table that cleanup may remove. var ErrExecutionReadPinLost = fmt.Errorf("published execution read pin was lost") var ErrExecutionReadPinActive = fmt.Errorf("published execution has active readers") var ErrSelectionSourceNotAddressable = fmt.Errorf("published output is not addressable to source resources") var ErrSelectionSourceIdentityChanged = fmt.Errorf("published selection source identity changed") -// ExecutionReadPinCatalog is additive to BundleCatalog so existing catalog -// fakes and integrations can migrate without weakening the publication -// lifecycle. Implementations atomically arbitrate pin acquisition against -// cleanup claims for the same execution. type ExecutionReadPinCatalog interface { AcquireExecutionReadPin(context.Context, string, string, time.Time) (bool, error) RenewExecutionReadPin(context.Context, string, string, time.Time) (bool, error) @@ -310,9 +272,6 @@ type ExecutionReadPinCatalog interface { ReleaseExecutionCleanup(context.Context, string, string) error } -// SourceRowMetadataWriter lets the publication owner persist a proven -// resource-address mapping before commit. It is optional to preserve the -// existing publication transaction contract for unrelated outputs. type SourceRowMetadataWriter interface { SetSourceRowMetadata(context.Context, string, SourceRowMetadata) error } diff --git a/internal/dataframe/publication/bundle_test.go b/internal/dataframe/publication/bundle_test.go index 2ead31bca..3459e4a61 100644 --- a/internal/dataframe/publication/bundle_test.go +++ b/internal/dataframe/publication/bundle_test.go @@ -32,11 +32,6 @@ func TestFinalSchemaDigestPreservesColumnOrder(t *testing.T) { } } -// Repair-5 (F-B21): the length-prefix PointerName encoding must stay -// injective for exactly the pairs the archived CONFIRMED hijack used, plus -// delimiter-spoof and multibyte neighbours; the shipped blank-component -// exclusion semantics (whitespace-only == absent) are behaviour, so the -// encoding change must not alter them. func TestBundleIdentityPointerNameIsInjective(t *testing.T) { base := BundleIdentity{Project: "proj", DatasetGeneration: "gen", Name: "a", OutputName: "out"} pairA := base diff --git a/internal/dataframe/publication/clickhouse/abort_ackloss_e2e_test.go b/internal/dataframe/publication/clickhouse/abort_ackloss_e2e_test.go index db6d517e7..e163cce7d 100644 --- a/internal/dataframe/publication/clickhouse/abort_ackloss_e2e_test.go +++ b/internal/dataframe/publication/clickhouse/abort_ackloss_e2e_test.go @@ -10,50 +10,22 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" pubpublication "github.com/calypr/loom/internal/dataframe/publication" - pubarango "github.com/calypr/loom/internal/dataframe/publication/arango" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/calypr/loom/internal/store/clickhouse" "github.com/google/uuid" ) -// Candidate finding F-B2: clickHouseBundleTx.Abort (bundle_store.go:861-897) -// reaches claimExecutionCleanup → ClaimExecutionCleanup, whose real AQL -// (bundle_registry.go:411-427) filters only expired pins and the cleanup slot -// — no pointer and no published-state recheck — and then unconditionally -// DropTables every output. Hypothesis: a runner whose PublishExecution AQL -// succeeded server-side but whose ACK was lost calls Abort, deleting the -// physical tables of the bundle the live pointer now advertises. -// -// The wrapper injects ONLY a lost publish ACK: the real Registry AQL runs to -// completion against the real database (so the server has committed both the -// PUBLISHED row and the pointer), and the wrapper discards the client-side -// result and returns a network-style error, mirroring a connection failure -// after server commit. ClaimExecutionCleanup is the real AQL; no pins exist -// (not this test's variable). ClickHouse effects come from the established -// tocFakeClickHouse recorder. -// -// Opt-in like reconcile_toctou_e2e_test.go; the test owns a UUID database and -// deletes only that database: -// -// docker exec loom-dev-qwen-current-loom-api-1 sh -c \ -// 'cd /workspace && LOOM_TEST_ARANGO_URL=http://arangodb:8529 go test \ -// -count=1 -run "TestAbort.*AgainstArango" -v ./internal/dataframe/publication/clickhouse/' - type fbxCatalog struct { - *pubarango.Registry + *arangofactory.PublishedRegistry publishArmed bool publishRan bool publishLanded bool observedID string } -// PublishExecution runs the real AQL verbatim. Once armed, a server-committed -// publish has its successful result discarded and is replaced by a network -// error, so the production caller believes the ACK was lost after commit. -// Genuine server-side rejections are passed through untouched. func (c *fbxCatalog) PublishExecution(ctx context.Context, name, expected string, execution pubpublication.BundleExecution) error { - err := c.Registry.PublishExecution(ctx, name, expected, execution) + err := c.PublishedRegistry.PublishExecution(ctx, name, expected, execution) if c.publishArmed && !c.publishRan { c.publishRan = true if err == nil { @@ -64,13 +36,11 @@ func (c *fbxCatalog) PublishExecution(ctx context.Context, name, expected string return err } -// SaveExecution delegates verbatim; it only records the execution UUID that -// beginBundle generated so the test can re-read the row by key afterwards. func (c *fbxCatalog) SaveExecution(ctx context.Context, execution pubpublication.BundleExecution, fenceOwner string) error { if c.observedID == "" && execution.OwnerID != "" { c.observedID = execution.ID } - return c.Registry.SaveExecution(ctx, execution, fenceOwner) + return c.PublishedRegistry.SaveExecution(ctx, execution, fenceOwner) } func fbxCleanupDatabase(t *testing.T, endpoint, database string) { @@ -93,7 +63,7 @@ func fbxCleanupDatabase(t *testing.T, endpoint, database string) { } } -func fbxSetup(t *testing.T) (*pubarango.Registry, context.Context, string) { +func fbxSetup(t *testing.T) (*arangofactory.PublishedRegistry, context.Context, string) { t.Helper() endpoint := strings.TrimSpace(os.Getenv("LOOM_TEST_ARANGO_URL")) if endpoint == "" { @@ -103,18 +73,19 @@ func fbxSetup(t *testing.T) (*pubarango.Registry, context.Context, string) { t.Cleanup(cancel) database := "loom_verify_fbx_" + strings.ReplaceAll(uuid.NewString(), "-", "") t.Logf("owned arango database: %s", database) - client, err := storearango.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatalf("open arango: %v", err) } + client := arangoBackend.Client t.Cleanup(func() { _ = client.Close(context.Background()) fbxCleanupDatabase(t, endpoint, database) }) - if err := client.Bootstrap(ctx, pubarango.BootstrapSpec()); err != nil { + if err := client.Bootstrap(ctx, arangofactory.PublishedBootstrapSpec()); err != nil { t.Fatalf("bootstrap: %v", err) } - registry, err := pubarango.New(client) + registry, err := arangofactory.NewPublishedRegistry(client) if err != nil { t.Fatalf("registry: %v", err) } @@ -149,9 +120,6 @@ func fbxRows() []map[string]any { return []map[string]any{{"value": int64(1)}, {"value": int64(2)}} } -// fbxClickHouse reuses the tocFakeClickHouse effect semantics and additionally -// records the names handed to CreateTable/DropTable so the test can report the -// created versus dropped physical-table set against real registry state. type fbxClickHouse struct { *tocFakeClickHouse created []string @@ -178,17 +146,11 @@ func fbxDropped(set []string, table string) bool { return false } -// TestAbortDropsServerPublishedBundleAgainstArango drives the real -// Begin/publish/Abort flow with ONLY the publish ACK lost after a real -// server-side commit. It fails t.Fatalf("CONFIRMED ...") iff, after Abort -// returns, the real registry still advertises the execution through the -// pointer (row PUBLISHED) while its physical tables were deleted. If -// production refuse gates hold, the test passes and logs the refusing code. func TestAbortDropsServerPublishedBundleAgainstArango(t *testing.T) { registry, ctx, suffix := fbxSetup(t) identity := fbxBundleIdentity(suffix) pointerName := identity.PointerName() - wrapper := &fbxCatalog{Registry: registry, publishArmed: true} + wrapper := &fbxCatalog{PublishedRegistry: registry, publishArmed: true} ch := &fbxClickHouse{tocFakeClickHouse: &tocFakeClickHouse{}} store, err := NewBundleStore(ch, wrapper) if err != nil { @@ -246,14 +208,10 @@ func TestAbortDropsServerPublishedBundleAgainstArango(t *testing.T) { t.Logf("PREVENTED on unmodified production code: with the publish AQL committed server-side and only the client ACK lost, Commit's confirmCommit fallback (bundle_store.go:813-822, 841-858) recovered the commit (Commit treated it as successful), which marked the tx closed so Abort (bundle_store.go:862) refused, and the real ClaimExecutionCleanup path was never reached. commitErr=%v abortErr=%v", commitErr, abortErr) } -// TestAbortDropsUnpublishedExecutionAgainstArango is the control: the same -// real Begin/Abort flow with no publish issued at all. The abort drop must -// proceed (no pointer exists, the row ends FAILED) — pinning that any harm in -// the finding test comes from the lost-ACK-after-success, not the harness. func TestAbortDropsUnpublishedExecutionAgainstArango(t *testing.T) { registry, ctx, suffix := fbxSetup(t) identity := fbxBundleIdentity(suffix) - wrapper := &fbxCatalog{Registry: registry} + wrapper := &fbxCatalog{PublishedRegistry: registry} ch := &fbxClickHouse{tocFakeClickHouse: &tocFakeClickHouse{}} store, err := NewBundleStore(ch, wrapper) if err != nil { @@ -272,9 +230,6 @@ func TestAbortDropsUnpublishedExecutionAgainstArango(t *testing.T) { } cause := errors.New("runner aborts before any publish was issued") abortErr := tx.Abort(ctx, cause) - // Abort echoes the runner cause back through the cleanup join - // (bundle_store.go:891-896, failPhase :951). A declined cleanup claim - // would surface as ErrExecutionReadPinActive instead. if !errors.Is(abortErr, cause) || errors.Is(abortErr, pubpublication.ErrExecutionReadPinActive) { t.Fatalf("legitimate Abort of an unpublished execution must drop and echo the runner cause, got: %v", abortErr) } diff --git a/internal/dataframe/publication/clickhouse/bundle_store.go b/internal/dataframe/publication/clickhouse/bundle_store.go index a1d9fc350..36c04c412 100644 --- a/internal/dataframe/publication/clickhouse/bundle_store.go +++ b/internal/dataframe/publication/clickhouse/bundle_store.go @@ -26,9 +26,6 @@ var ( ErrBundleCheckpointUncertain = errors.New("bundle checkpoint outcome is uncertain") ) -// ClickHouseBundleStore publishes staged tables by advancing a durable logical -// pointer in publication.BundleCatalog. ClickHouse itself has no cross-table transaction; -// the pointer is therefore the visibility boundary. type ClickHouseBundleStore struct { clickHouse BundleClickHouseStore catalog publication.BundleCatalog @@ -41,15 +38,6 @@ type ClickHouseBundleStore struct { activeReleases ActiveReleaseReader } -// ActiveReleaseReader is the narrow read surface Reconcile consults before -// dropping an execution's physical tables (Repair-1b, F-B12 drop-path). The -// reconcile sweep's pointer guard is a one-shot read taken before the cleanup -// claim; a release Created+Activated afterwards (or a legacy/non-lease-guarded -// writer publishing behind the guard) still ends with DropTable + FAILED -// demotion while the ACTIVE release binds the execution, which hard-fails the -// whole CurrentProjectDatasets response (published/current.go:206-208). A -// live binding therefore vetoes the drop, and any binding-read failure is -// treated fail-closed as a veto. type ActiveReleaseReader interface { ReadActiveRelease(context.Context, string) (dataset.ActiveRelease, error) } @@ -82,9 +70,6 @@ func NewBundleStore(client BundleClickHouseStore, catalog publication.BundleCata }, nil } -// SetActiveReleaseReader wires the drop-path binding veto used by Reconcile -// (see ActiveReleaseReader). Without a reader the reconciler keeps its -// historical pointer-only guard; production wiring supplies it (server.go). func (s *ClickHouseBundleStore) SetActiveReleaseReader(reader ActiveReleaseReader) { s.activeReleases = reader } @@ -92,7 +77,6 @@ func (s *ClickHouseBundleStore) SetActiveReleaseReader(reader ActiveReleaseReade var _ publication.Target = (*ClickHouseBundleStore)(nil) var _ publication.Transaction = (*clickHouseBundleTx)(nil) -// SupportsObjectValues reports the native JSON support of this target. func (s *ClickHouseBundleStore) SupportsObjectValues() bool { return true } func (s *ClickHouseBundleStore) Begin(ctx context.Context, identity publication.PublicationIdentity, schemas []publication.OutputSchema) (publication.Transaction, error) { @@ -531,14 +515,10 @@ func (t *clickHouseBundleTx) CreateOutput(ctx context.Context, name string, colu if t.columns == nil { t.columns = make(map[string][]clickhouse.Column) } - // Keep the caller-facing schema separate from the physical row identity - // column. InsertRows adds generated identities when it receives this view. t.columns[name] = logicalColumns return t.save(ctx) } -// SetOutputMetadata persists semantic schema alongside the physical table -// definition. func (t *clickHouseBundleTx) SetOutputMetadata(ctx context.Context, name string, columns []publication.LogicalColumn) error { if t.idempotent { return nil @@ -579,8 +559,6 @@ func (t *clickHouseBundleTx) SetOutputMetadata(ctx context.Context, name string, return t.save(ctx) } -// SetSourceRowMetadata records the proven typed source identity before the -// publication is committed. Selection refuses outputs without this mapping. func (t *clickHouseBundleTx) SetSourceRowMetadata(ctx context.Context, name string, metadata publication.SourceRowMetadata) error { if t.idempotent { return nil @@ -615,8 +593,6 @@ func (t *clickHouseBundleTx) SetSourceRowMetadata(ctx context.Context, name stri return t.save(ctx) } -// FinalizeSchema removes discovered columns that were never populated and -// persists the retained logical/physical schema before verification. func (t *clickHouseBundleTx) FinalizeSchema(ctx context.Context, schemas []publication.OutputSchema) error { if t.idempotent { return nil @@ -857,9 +833,6 @@ func (t *clickHouseBundleTx) publishedOutputs() []publication.PublishedOutput { return result } -// confirmCommit distinguishes a lost success response from a transaction that -// definitely did not publish. Inconsistent or unreadable metadata remains -// uncertain so callers never delete tables that a live pointer may reference. func (t *clickHouseBundleTx) confirmCommit(ctx context.Context) (committed, conclusive bool, err error) { execution, executionErr := t.store.catalog.GetExecution(ctx, t.execution.ID) pointer, pointerErr := t.store.catalog.GetPointer(ctx, t.execution.PointerName()) @@ -896,9 +869,6 @@ func (t *clickHouseBundleTx) Abort(ctx context.Context, cause error) error { return errors.Join(claimErr, t.stopLease()) } defer func() { _ = releaseCleanup() }() - // Same live veto as reconcileExecution (Repair-1b / F-B1 abort-drop arm): - // never DropTable a bundle the visibility pointer currently advertises or - // the ACTIVE release currently binds; read failures veto fail-closed. if pointerNow, pointerNowErr := t.store.catalog.GetPointer(cleanupCtx, t.execution.PointerName()); pointerNowErr != nil { if !errors.Is(pointerNowErr, publication.ErrBundleNotFound) { t.closed = true @@ -1042,9 +1012,6 @@ func toColumns(columns []publication.LogicalColumn) ([]clickhouse.Column, error) case "code": columnType = "String" case "object": - // ClickHouse's native JSON type drops explicit null object paths. - // Store logical objects as JSON text so publication preserves the - // distinction between an absent key and a key whose value is null. columnType = "String" logicalType = "object" } @@ -1074,10 +1041,6 @@ func allOutputsQueryable(outputs []publication.BundleOutputRecord) bool { return true } -// Reconcile removes abandoned staging tables and repairs persisted publication -// state left behind by an interrupted commit. -// It is safe to call repeatedly during startup; published pointers and queued -// commands that have not started are never touched. func (s *ClickHouseBundleStore) Reconcile(ctx context.Context, olderThan time.Time) error { reconcilerID := "reconciler-" + uuid.NewString() process := func(executions []publication.BundleExecution) error { @@ -1089,12 +1052,6 @@ func (s *ClickHouseBundleStore) Reconcile(ctx context.Context, olderThan time.Ti return nil } states := []publication.BundleState{publication.BundleRunning, publication.BundleValidating, publication.BundleFailed} - // Repair-4 (F-B19): recurring PUBLISHED-superseded janitor arm. Enumerated - // ONLY when an ActiveReleaseReader is wired — a janitor that cannot re-check - // bindings is the m-po-janitor-blind shape (PointerOrphanLifecycle.tla - // NoBoundDeleted violated): it recreates the F-B12 bound-data loss. The - // destructive step still runs the full post-claim pointer+binding re-checks - // in reconcileExecution (m-po-fixed validated RecheckBeforeReclaim shape). if s.activeReleases != nil { states = append(states, publication.BundlePublished) } @@ -1113,12 +1070,6 @@ func (s *ClickHouseBundleStore) Reconcile(ctx context.Context, olderThan time.Ti func (s *ClickHouseBundleStore) reconcileExecution(ctx context.Context, execution publication.BundleExecution, reconcilerID string) error { superseded := execution.State.Canonical() == publication.BundlePublished if superseded { - // Repair-4 (F-B19) pre-claim janitor filter: a PUBLISHED execution that - // its pointer still names, or that the ACTIVE release binds, is live — - // skip it without claiming (no lease/_rev churn on healthy published - // rows). This filter is an optimization only: the post-claim pointer and - // binding re-checks below remain the destruction gate. Read failures - // other than "no pointer"/"no active release" veto fail-closed. if s.activeReleases == nil { return nil } @@ -1165,16 +1116,6 @@ func (s *ClickHouseBundleStore) reconcileExecution(ctx context.Context, executio return errors.Join(claimErr, s.catalog.ReleaseBundleLease(cleanupCtx, execution.Key, reconcilerID)) } defer func() { _ = releaseCleanup() }() - // Repair-1b (F-B12 drop-path): the pointer guard above and the cleanup - // claim are both historical facts by now. Re-read the pointer AND the - // active-release bindings immediately before destruction: a pointer that - // now names this execution, or any binding of the ACTIVE release to it, - // vetoes the drop. Any read failure other than "no pointer" / - // "no active release" vetoes fail-closed. Skipping only defers the sweep - // for a genuinely abandoned execution; dropping a bound one destroys - // advertised data and aborts CurrentProjectDatasets - // (published/current.go:206-208, validated by ReconcileDangleTrust.tla - // m-dangle-atomic-recheck). if pointerNow, pointerNowErr := s.catalog.GetPointer(cleanupCtx, execution.PointerName()); pointerNowErr != nil { if !errors.Is(pointerNowErr, publication.ErrBundleNotFound) { return errors.Join(pointerNowErr, s.catalog.ReleaseBundleLease(cleanupCtx, execution.Key, reconcilerID)) diff --git a/internal/dataframe/publication/clickhouse/bundle_store_test.go b/internal/dataframe/publication/clickhouse/bundle_store_test.go index f896b8ab2..95c9ee959 100644 --- a/internal/dataframe/publication/clickhouse/bundle_store_test.go +++ b/internal/dataframe/publication/clickhouse/bundle_store_test.go @@ -368,7 +368,6 @@ func TestReconcileRetiresSupersededPublishedExecutions(t *testing.T) { } }) } - // The stale (non-PUBLISHED) arm keeps its shipped marker verbatim. t.Run("stale arm marker unchanged", func(t *testing.T) { catalog := &reconcileRecheckCatalog{ execution: publication.BundleExecution{ diff --git a/internal/dataframe/publication/clickhouse/export_pin_crash_window_e2e_test.go b/internal/dataframe/publication/clickhouse/export_pin_crash_window_e2e_test.go index caf7f9cdc..827b2e72b 100644 --- a/internal/dataframe/publication/clickhouse/export_pin_crash_window_e2e_test.go +++ b/internal/dataframe/publication/clickhouse/export_pin_crash_window_e2e_test.go @@ -14,93 +14,18 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" pubpublication "github.com/calypr/loom/internal/dataframe/publication" - pubarango "github.com/calypr/loom/internal/dataframe/publication/arango" pubpublished "github.com/calypr/loom/internal/dataframe/published" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/calypr/loom/internal/store/clickhouse" "github.com/google/uuid" ) -// Candidate finding F-B3 (crash-window REALISM upgrade): a process crash inside -// StreamExactExport's pre-first-tick window leaves the ARTIFACT-supplied 24h -// read pin on an execution, and startup Reconcile then abandons the whole sweep -// (worker B verified the end-state model; this file drives the live half). -// -// Source-verified wiring (production code unmodified): -// - internal/explorer/lifecycle/artifact.go:104 identity.ExpiresAt = -// now+artifactTTL(=24h @ :21); :152 the exact-export request carries -// PinExpiresAt = identity.ExpiresAt. -// - internal/dataframe/published/export.go:120-126 accepts ANY future -// PinExpiresAt verbatim, :135 acquires the durable pin with it in the -// REAL catalog AQL (bundle_registry.go:343-376), and only the renewal -// ticker (:156/:166-168, defaultReadPinInterval=30s at selection.go:15, -// defaultReadPinTTL=2min at :14) first pulls the expiry down toward -// now+2min — at the FIRST 30s tick. A crash before that tick leaves a -// 24h live pin. -// - Nothing else can rescue the pin: rg over non-test sources shows the -// ONLY RenewExecutionReadPin callers are export.go:168 (this export's own -// in-goroutine ticker, dead with the process) and read_pins.go:90 (its own -// ticker whose owner "reader-" is generated per call and never -// persisted — it can never target another readerId). bundle_store.go:266 -// renews the CLEANUP slot lease, not reader pins. There is no pin sweeper. -// -// Why the parked export goroutine IS the crash end-state. A Go process cannot -// skip its own defers without dying: export.go:142-146 releases through -// context.WithoutCancel, so even the cancellation path releases the pin -// (published/export_test.go:262-281 proves that), and no product seam lets a -// test die mid-call. Fidelity achieved here (level 2 — live entry point, no -// production seam altered): the REAL exported entry point -// published.Reader.StreamExactExport (export.go:82) is driven against the real -// Arango registry through the REAL export.go:135 pins.AcquireExecutionReadPin -// call with the artifact-passed caller TTL (now+24h); the export is then -// parked IN FLIGHT at its first ClickHouse visit with a renewal interval far -// past the default first tick, so (a) the ticker never fires — export.go:150-176 -// is the only renewer and it never runs — and (b) the deferred release never -// executes because StreamExactExport has not returned. That is the exact -// externally observable state of a process that died inside the first window, -// confirmed independently by a direct AQL document read on -// pubarango.ExecutionReadPinsCollection keyed by executionStateKey -// (bundle_registry.go:468-471 re-implemented locally, no catalog getter -// exists). Process teardown at test end un Parks the goroutine; by then the -// finding evidence is already recorded. -// -// Distinction from worker B (pinblock_reconcile_e2e_test.go): worker B called -// registry.AcquireExecutionReadPin directly (end-state MODEL of the pin). This -// file adds the realism step — the pin's 24h horizon is produced by actually -// executing the production exact-export entry point's own acquire call with the -// artifact-supplied expiry, with the renewal half abandoned in flight. The -// blast-radius assertions (abort with ErrExecutionReadPinActive, stranded -// sibling, control reclaim) reproduce worker B's output shape. The control -// TestExportCrashWithoutCrashRenewedPinExpiresFast must PASS. -// -// FINDING expectation: the crash test fails ON PURPOSE (t.Errorf CONFIRMED) -// exactly like worker B's finding test. The control -// TestExportCrashWithoutCrashRenewedPinExpiresFast MUST PASS: one real ticker -// tick pulls the caller's 24h down to ~defaultReadPinTTL, the graceful stream -// end releases the pin, and the identical production Reconcile then reclaims -// both stale executions. -// -// Opt-in, same idiom as pinblock_reconcile_e2e_test.go. Databases are UUID -// named loom_verify_ecw_, deleted only via t.Cleanup: -// -// docker exec loom-dev-qwen-current-loom-api-1 sh -c \ -// 'cd /workspace && LOOM_TEST_ARANGO_URL=http://arangodb:8529 go test \ -// -count=1 -run TestExportCrash -v -timeout 300s ./internal/dataframe/publication/clickhouse/' - const ( - // artifactExportHorizon mirrors artifact.go: identity.ExpiresAt = - // now+defaultArtifactTTL(24h) passed through as PinExpiresAt (:152). - artifactExportHorizon = 24 * time.Hour - // production first-tick bound (selection.go:15). The crash test's renewal - // interval sits far beyond it so the first tick cannot fire before the - // crash verdict is recorded. + artifactExportHorizon = 24 * time.Hour productionFirstTickBound = 30 * time.Second ) -// ecwPinCall records one live pin operation the production entry point issued -// through the wrapper. The wrapper decides nothing: every method delegates -// verbatim to the real Registry AQL. type ecwPinCall struct { executionID string owner string @@ -108,7 +33,7 @@ type ecwPinCall struct { } type ecwCatalog struct { - *pubarango.Registry + *arangofactory.PublishedRegistry mu sync.Mutex acquires []ecwPinCall renews []ecwPinCall @@ -119,14 +44,14 @@ func (c *ecwCatalog) AcquireExecutionReadPin(ctx context.Context, executionID, o c.mu.Lock() c.acquires = append(c.acquires, ecwPinCall{executionID: executionID, owner: owner, expires: expires}) c.mu.Unlock() - return c.Registry.AcquireExecutionReadPin(ctx, executionID, owner, expires) + return c.PublishedRegistry.AcquireExecutionReadPin(ctx, executionID, owner, expires) } func (c *ecwCatalog) RenewExecutionReadPin(ctx context.Context, executionID, owner string, expires time.Time) (bool, error) { c.mu.Lock() c.renews = append(c.renews, ecwPinCall{executionID: executionID, owner: owner, expires: expires}) c.mu.Unlock() - renewed, err := c.Registry.RenewExecutionReadPin(ctx, executionID, owner, expires) + renewed, err := c.PublishedRegistry.RenewExecutionReadPin(ctx, executionID, owner, expires) return renewed, err } @@ -134,7 +59,7 @@ func (c *ecwCatalog) ReleaseExecutionReadPin(ctx context.Context, executionID, o c.mu.Lock() c.releases = append(c.releases, ecwPinCall{executionID: executionID, owner: owner}) c.mu.Unlock() - return c.Registry.ReleaseExecutionReadPin(ctx, executionID, owner) + return c.PublishedRegistry.ReleaseExecutionReadPin(ctx, executionID, owner) } func (c *ecwCatalog) snapshotAcquires() []ecwPinCall { @@ -155,9 +80,6 @@ func (c *ecwCatalog) snapshotReleases() []ecwPinCall { return append([]ecwPinCall(nil), c.releases...) } -// ecwQueryer is the published.Reader's ClickHouse seam. It records the stream -// SQL and parks at the first scan visit so StreamExactExport stays in flight -// (the crash window); releasing hold models graceful completion. type ecwQueryer struct { started chan struct{} hold chan struct{} @@ -183,7 +105,7 @@ func (q *ecwQueryer) QueryRowsArgsVisit(ctx context.Context, query string, _ []s if err := ctx.Err(); err != nil { return err } - _ = visit // zero-row scan: production Stream succeeds on an empty row set + _ = visit return nil } @@ -193,9 +115,6 @@ func (q *ecwQueryer) snapshotQueries() []string { return append([]string(nil), q.queries...) } -// ecwClickHouse records physical drops so the blast radius (which stale -// executions the reconciler actually reclaimed) is observable, matching the -// pbxClickHouse semantics including dead-context aborts. type ecwClickHouse struct { mu sync.Mutex dropped []string @@ -255,7 +174,7 @@ func ecwCleanupDatabase(t *testing.T, endpoint, database string) { } } -func ecwSetup(t *testing.T) (*storearango.Client, *pubarango.Registry, context.Context, string) { +func ecwSetup(t *testing.T) (*arangofactory.Client, *arangofactory.PublishedRegistry, context.Context, string) { t.Helper() endpoint := strings.TrimSpace(os.Getenv("LOOM_TEST_ARANGO_URL")) if endpoint == "" { @@ -265,28 +184,25 @@ func ecwSetup(t *testing.T) (*storearango.Client, *pubarango.Registry, context.C t.Cleanup(cancel) database := "loom_verify_ecw_" + strings.ReplaceAll(uuid.NewString(), "-", "") t.Logf("owned arango database: %s", database) - client, err := storearango.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatalf("open arango: %v", err) } + client := arangoBackend.Client t.Cleanup(func() { _ = client.Close(context.Background()) ecwCleanupDatabase(t, endpoint, database) }) - if err := client.Bootstrap(ctx, pubarango.BootstrapSpec()); err != nil { + if err := client.Bootstrap(ctx, arangofactory.PublishedBootstrapSpec()); err != nil { t.Fatalf("bootstrap: %v", err) } - registry, err := pubarango.New(client) + registry, err := arangofactory.NewPublishedRegistry(client) if err != nil { t.Fatalf("registry: %v", err) } return client, registry, ctx, database[len("loom_verify_ecw_"):] } -// ecwExecution builds an execution under a distinct identity. published=true -// makes it export-materializable (selection.go:62-87 requires Successful state, -// queryable output, receipt+schema); published=false seeds the stale FAILED -// sibling form Reconcile visits (bundle_store.go:1043). func ecwExecution(tag, suffix, physicalTable string, updatedAt time.Time, published bool) pubpublication.BundleExecution { identity := pubpublication.BundleIdentity{ Name: "ecw-" + tag, TranslationVersion: "1", OutputName: "main", @@ -313,9 +229,7 @@ func ecwExecution(tag, suffix, physicalTable string, updatedAt time.Time, publis } } -// ecwUpsertWithLease stores an execution row through the real leased -// SaveExecution AQL, then frees the key (fb1Seed/pbxSeed idiom). -func ecwUpsertWithLease(t *testing.T, registry *pubarango.Registry, ctx context.Context, exec pubpublication.BundleExecution, leaseOwner string) { +func ecwUpsertWithLease(t *testing.T, registry *arangofactory.PublishedRegistry, ctx context.Context, exec pubpublication.BundleExecution, leaseOwner string) { t.Helper() acquired, err := registry.AcquireBundleLease(ctx, exec.Key, leaseOwner, time.Now().UTC().Add(2*time.Minute)) if err != nil || !acquired { @@ -330,15 +244,10 @@ func ecwUpsertWithLease(t *testing.T, registry *pubarango.Registry, ctx context. } } -// ecwRowQuerier is the narrow QueryRows surface of *storearango.Client used to -// read the pins collection directly. There is no catalog GetExecutionReadPins -// API, so the pin truth is the AQL document itself. type ecwRowQuerier interface { - QueryRows(ctx context.Context, query string, batchSize int, bindVars map[string]interface{}, visit storearango.RowVisitor) error + QueryRows(ctx context.Context, query string, batchSize int, bindVars map[string]interface{}, visit arangofactory.RowVisitor) error } -// ecwStateKey re-implements bundle_registry.go:468-471 (unexported there): -// the pins collection document key for one execution. func ecwStateKey(executionID string) string { sum := sha256.Sum256([]byte("state\x00" + executionID)) return "state_" + hex.EncodeToString(sum[:]) @@ -352,11 +261,6 @@ type ecwPinRow struct { Expiries []string `json:"expiries"` } -// ecwReadPins performs the direct AQL document read of the real -// ExecutionReadPinsCollection entry: active counts pins with expiresAt >= now -// (same comparison the production ClaimExecutionCleanup AQL applies at -// bundle_registry.go:416); far counts pins still live at horizon, used to -// distinguish the caller's 24h horizon from the ticker's 2min horizon. func ecwReadPins(t *testing.T, querier ecwRowQuerier, executionID string, now, horizon time.Time) ecwPinRow { t.Helper() var row ecwPinRow @@ -366,7 +270,7 @@ LET active = LENGTH(FOR pin IN pins FILTER pin.expiresAt >= @now RETURN pin) LET far = LENGTH(FOR pin IN pins FILTER pin.expiresAt >= @horizon RETURN pin) RETURN {found: state != null, active: active, far: far, owners: (FOR pin IN pins RETURN pin.ownerId), expiries: (FOR pin IN pins RETURN TO_STRING(pin.expiresAt))}`, 1, map[string]interface{}{ - "@collection": pubarango.ExecutionReadPinsCollection, + "@collection": arangofactory.ExecutionReadPinsCollection, "stateKey": ecwStateKey(executionID), "now": now, "horizon": horizon, @@ -378,7 +282,7 @@ RETURN {found: state != null, active: active, far: far, owners: (FOR pin IN pins return json.Unmarshal(data, &row) }) if err != nil { - t.Fatalf("direct AQL read of %s for %s: %v", pubarango.ExecutionReadPinsCollection, executionID, err) + t.Fatalf("direct AQL read of %s for %s: %v", arangofactory.ExecutionReadPinsCollection, executionID, err) } return row } @@ -392,22 +296,12 @@ func ecwHasOwner(owners []string, owner string) bool { return false } -// ecwStartExactExport drives the REAL production entry point -// published.Reader.StreamExactExport (export.go:82) through a recording -// catalog wrapper that delegates every catalog call verbatim to the real -// Registry, and parks the export in flight at its first ClickHouse scan visit. -// renewalInterval configures the caller-visible PinRenewalInterval; pass a -// value >> productionFirstTickBound to model a crash inside the default -// pre-first-tick window, or a millisecond-scale value to force real ticks. -func ecwStartExactExport(t *testing.T, registry *pubarango.Registry, ctx context.Context, exec pubpublication.BundleExecution, readerID string, horizon, renewalInterval time.Duration) (*ecwCatalog, *ecwQueryer, context.CancelFunc, <-chan error, time.Time) { +func ecwStartExactExport(t *testing.T, registry *arangofactory.PublishedRegistry, ctx context.Context, exec pubpublication.BundleExecution, readerID string, horizon, renewalInterval time.Duration) (*ecwCatalog, *ecwQueryer, context.CancelFunc, <-chan error, time.Time) { t.Helper() - catalog := &ecwCatalog{Registry: registry} + catalog := &ecwCatalog{PublishedRegistry: registry} queryer := &ecwQueryer{started: make(chan struct{}), hold: make(chan struct{})} reader := &pubpublished.Reader{Catalog: catalog, ClickHouse: queryer} pinExpiresAt := time.Now().UTC().Add(horizon) - // Identity fields copied verbatim from the seeded identity so - // validateExactExportIdentity (export.go:207-221) matches the real - // materialization the registry returns. request := pubpublished.ExactExportRequest{ ExecutionID: exec.ID, OutputID: "main", @@ -438,8 +332,6 @@ func ecwStartExactExport(t *testing.T, registry *pubarango.Registry, ctx context return catalog, queryer, cancel, done, pinExpiresAt } -// ecwFindAcquire locates the acquisition the production entry point issued for -// this execution/reader, or fails. func ecwFindAcquire(t *testing.T, catalog *ecwCatalog, executionID, readerID string) ecwPinCall { t.Helper() for _, call := range catalog.snapshotAcquires() { @@ -451,22 +343,6 @@ func ecwFindAcquire(t *testing.T, catalog *ecwCatalog, executionID, readerID str return ecwPinCall{} } -// TestExportCrashInsideFirstTickLeavesCrashWindowPinAgainstArango is the F-B3 -// crash-window finding against a real Arango instance. The 24h pin is produced -// by REALLY running the exported exact-export entry point's acquire half -// (export.go:135 via the real Registry AQL) with the artifact-supplied -// caller TTL, then abandoning the export in flight before its first renewal -// tick (nothing else renews: only callers of RenewExecutionReadPin outside -// tests are export.go:168 and read_pins.go:90, both in-goroutine tickers of -// the owning process/call; bundle_store.go:266 renews the cleanup slot, not -// pins — verified by rg, quoted in the file header). Direct AQL then proves -// the orphaned pin persists at the caller's 24h horizon. Production Reconcile -// over the now-stale pinned execution reproduces worker B's blast radius: -// errors.Is ErrExecutionReadPinActive, the pinned execution stranded, and the -// unrelated pin-free sibling stranded by first-error-wins (bundle_store.go -// :1033-1052/:1076-1079). This test therefore FAILS ON PURPOSE like worker B's -// finding test; TestExportCrashWithoutCrashRenewedPinExpiresFast is the -// must-PASS control proving the stranding is caused by the crash-window pin. func TestExportCrashInsideFirstTickLeavesCrashWindowPinAgainstArango(t *testing.T) { client, registry, ctx, suffix := ecwSetup(t) @@ -477,34 +353,22 @@ func TestExportCrashInsideFirstTickLeavesCrashWindowPinAgainstArango(t *testing. ecwUpsertWithLease(t, registry, ctx, x, "ecw-seed") const readerID = "artifact-export-crash-window" - // Renewal interval far beyond the 30s production first tick: the crash - // lands before the ticker's first case (export.go:165) can ever run. catalog, queryer, cancelExport, _, pinExpiresAt := ecwStartExactExport(t, registry, ctx, x, readerID, artifactExportHorizon, 10*time.Minute) - defer cancelExport() // models test-process teardown AFTER the crash verdict, not the crash itself + defer cancelExport() - // Sleep far beyond any realistic first-tick arrival at 10min interval - // relative to the 250ms below: if a ticker could fire spuriously it - // would already have, so renewCount==0 measures the genuine window. time.Sleep(250 * time.Millisecond) - // Realism evidence 1: the REAL entry point's own acquire call executed - // with the caller TTL the artifact would pass (PinExpiresAt is honored - // verbatim by export.go:120-126). acquire := ecwFindAcquire(t, catalog, x.ID, readerID) if acquire.expires.Before(pinExpiresAt.Add(-time.Minute)) || acquire.expires.After(pinExpiresAt.Add(time.Minute)) { t.Fatalf("acquire expiry %v not the artifact-passed caller TTL %v (export.go:120-126 must pass it through)", acquire.expires, pinExpiresAt) } - // Realism evidence 2: direct AQL document read proves the crash-window - // pin is on the REAL pins collection under the caller's 24h horizon. early := ecwReadPins(t, client, x.ID, time.Now().UTC(), time.Now().UTC().Add(23*time.Hour)) if !early.Found || early.Active < 1 || early.Far < 1 || !ecwHasOwner(early.Owners, readerID) { - t.Fatalf("crash-window pin not observed on %s for %s/%s: %+v", pubarango.ExecutionReadPinsCollection, x.ID, readerID, early) + t.Fatalf("crash-window pin not observed on %s for %s/%s: %+v", arangofactory.ExecutionReadPinsCollection, x.ID, readerID, early) } t.Logf("crash-window pin observed by direct AQL: active=%d far(>=23h)=%d owners=%v expiries=%v", early.Active, early.Far, early.Owners, early.Expiries) - // Realism evidence 3: no renewal ever ticked during the window and no - // other component targets this readerId (header rg evidence). if renews := catalog.snapshotRenews(); len(renews) != 0 { t.Fatalf("renewal ticker fired inside the simulated crash window (%v): not a pre-first-tick crash", renews) } @@ -515,10 +379,6 @@ func TestExportCrashInsideFirstTickLeavesCrashWindowPinAgainstArango(t *testing. t.Fatalf("precondition: parked stream never scanned the exported physical table %q: %v", xTable, queries) } - // The crashed export's execution subsequently goes stale/FAILED (a later - // run under the same identity failed), making it a Reconcile candidate - // while the orphaned pin is still live; Y is an unrelated pin-free stale - // sibling, exactly worker B's shape. staleX := x staleX.State = pubpublication.BundleFailed staleX.UpdatedAt = base.Add(-70 * time.Minute) @@ -554,7 +414,7 @@ func TestExportCrashInsideFirstTickLeavesCrashWindowPinAgainstArango(t *testing. } if pinErr && !xReclaimed && !yReclaimed { t.Errorf("CONFIRMED with live crash realism against real Arango (F-B3): the REAL published.Reader.StreamExactExport entry point issued its own AcquireExecutionReadPin (export.go:135, recording wrapper delegating to real Registry AQL bundle_registry.go:343-376) with the artifact-supplied caller TTL %v (artifact.go:21/:104/:152 identity.ExpiresAt=now+24h); the export was abandoned in flight before its FIRST 30s renewal tick (export.go:156 ticker never fired: zero RenewExecutionReadPin calls observed; the only non-test renewers export.go:168/read_pins.go:90 are in-goroutine tickers of the owning process, bundle_store.go:266 renews the cleanup slot) and the deferred release (export.go:142-146) never ran, leaving a live 24h pin (direct AQL on %s: active=%d far=%d owners=%v). Startup Reconcile then aborted on %q with errors.Is ErrExecutionReadPinActive and the unrelated, pin-free sibling %q was STRANDED (bundle_store.go:1033-1052 first-error-wins, :1076-1079 propagation; server.go:556-558 publicationReady=false for the process lifetime). reconcileErr=%v; X table %s reclaimed=%v; Y table %s reclaimed=%v state=%q error-field=%q; dropped=%v. Contrast control TestExportCrashWithoutCrashRenewedPinExpiresFast: after ONE real ticker tick pulled the same caller 24h down to ~defaultReadPinTTL and the stream completed+released, the identical production Reconcile reclaimed BOTH executions.", - pinExpiresAt.UTC().Format(time.RFC3339), pubarango.ExecutionReadPinsCollection, after.Active, after.Far, after.Owners, x.ID, y.ID, reconcileErr, xTable, xReclaimed, yTable, yReclaimed, yRow.State, yRow.Error, dropped) + pinExpiresAt.UTC().Format(time.RFC3339), arangofactory.ExecutionReadPinsCollection, after.Active, after.Far, after.Owners, x.ID, y.ID, reconcileErr, xTable, xReclaimed, yTable, yReclaimed, yRow.State, yRow.Error, dropped) } else if pinErr && !xReclaimed && yReclaimed { t.Errorf("UNEXPECTED: pin abort did NOT strand sibling Y (dropped=%v): a compensating skip-and-continue path exists — re-examine F-B3 blast radius", dropped) } @@ -569,13 +429,6 @@ func TestExportCrashInsideFirstTickLeavesCrashWindowPinAgainstArango(t *testing. } } -// TestExportCrashWithoutCrashRenewedPinExpiresFast is the must-PASS control: -// the same real entry-point flow, but the export survives long enough for the -// production renewal ticker (export.go:156/:166-168) to tick — observed in the -// wrapper AND in the direct AQL read as the caller's 24h horizon being pulled -// down to ~defaultReadPinTTL (selection.go:14). The stream then completes and -// the deferred release (export.go:142-146) removes the pin; the identical -// production Reconcile reclaims both stale executions with nil error. func TestExportCrashWithoutCrashRenewedPinExpiresFast(t *testing.T) { client, registry, ctx, suffix := ecwSetup(t) @@ -586,8 +439,6 @@ func TestExportCrashWithoutCrashRenewedPinExpiresFast(t *testing.T) { ecwUpsertWithLease(t, registry, ctx, x, "ecw-seed") const readerID = "artifact-export-renewed" - // Millisecond renewal interval forces the production ticker to take its - // real export.go:166-168 path: next = now.Add(defaultReadPinTTL). catalog, queryer, cancelExport, exportDone, pinExpiresAt := ecwStartExactExport(t, registry, ctx, x, readerID, artifactExportHorizon, 10*time.Millisecond) defer cancelExport() @@ -596,8 +447,6 @@ func TestExportCrashWithoutCrashRenewedPinExpiresFast(t *testing.T) { t.Fatalf("control precondition: caller TTL %v not honored at acquire: %v", pinExpiresAt, acquire.expires) } - // The ticker must pull the 24h caller horizon down below the 23h bound - // (i.e. to ~defaultReadPinTTL) while the stream is still in flight. deadline := time.Now().Add(30 * time.Second) var pulled ecwPinRow for { @@ -627,7 +476,6 @@ func TestExportCrashWithoutCrashRenewedPinExpiresFast(t *testing.T) { t.Logf("renewal realism observed: %d ticker renewals, latest expiresAt=%v (caller TTL was %v); AQL pin now active=%d far(>=23h)=%d owners=%v expiries=%v", len(renews), lastRenew.expires.UTC().Format(time.RFC3339), pinExpiresAt.UTC().Format(time.RFC3339), pulled.Active, pulled.Far, pulled.Owners, pulled.Expiries) - // Graceful stream end: the deferred release at export.go:142-146 runs. close(queryer.hold) select { case err := <-exportDone: @@ -653,8 +501,6 @@ func TestExportCrashWithoutCrashRenewedPinExpiresFast(t *testing.T) { time.Sleep(25 * time.Millisecond) } - // Same stale transition as the crash test, then production Reconcile: - // with the renewed-then-released pin gone, the sweep completes normally. staleX := x staleX.State = pubpublication.BundleFailed staleX.UpdatedAt = base.Add(-70 * time.Minute) diff --git a/internal/dataframe/publication/clickhouse/orphaned_published_growth_e2e_test.go b/internal/dataframe/publication/clickhouse/orphaned_published_growth_e2e_test.go index 68696720f..9ac04fa0c 100644 --- a/internal/dataframe/publication/clickhouse/orphaned_published_growth_e2e_test.go +++ b/internal/dataframe/publication/clickhouse/orphaned_published_growth_e2e_test.go @@ -1,30 +1,5 @@ package clickhouse -// F-B19 durable regression control (Repair-4 shipped). A recipe edit mints a -// new bundle Key (the Key folds RecipeDigest/SchemaDigest/ReceiptID — -// internal/dataframe/publication/bundle.go:99-115) under the SAME PointerName -// (bundle.go:88-97). Pre-repair the superseded PUBLISHED execution was -// immortal (sweep visited only {RUNNING, VALIDATING, FAILED}), producing one -// permanently retained bundle+table per recipe edit and a monotonically -// widening name-latest mis-bind surface (F-B6 amplifier) — archived -// FAILED-ON-PURPOSE witnesses: parent runs e00a7032/e3712285, Round 15. -// -// Post-REPAIR natural-PASS (this artifact's shipped expectation): the -// binding-aware recurring janitor arm enumerates PUBLISHED (only while an -// ActiveReleaseReader is wired, the server.go production shape) and RETIRES a -// superseded orphan: FAILED with the "superseded execution reconciled" -// marker and its ClickHouse table dropped, while the pointer-named live -// execution and its table stay untouched. The REGRESSION direction keeps the -// archived CONFIRMED red text verbatim in structure: an immortal PUBLISHED -// orphan surviving a full sweep fails the test again. -// -// Safety sibling (TestPointerOrphanedPublishedJanitorRespectsActiveRelease- -// BindingsAgainstArango): a binding of the ACTIVE release to the orphan -// vetoes retirement (no claim, no drop); removing the binding and re-sweeping -// converges — the m-po-fixed shape on real stores. Formal companion: -// PointerOrphanLifecycle.tla m-po-bug arm (permanent-orphan temporal -// violation) + m-po-fixed arm (binding-aware janitor converges clean). - import ( "context" "net/http" @@ -35,10 +10,9 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" pubpublication "github.com/calypr/loom/internal/dataframe/publication" - pubarango "github.com/calypr/loom/internal/dataframe/publication/arango" "github.com/calypr/loom/internal/dataset" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/calypr/loom/internal/store/clickhouse" "github.com/google/uuid" ) @@ -47,8 +21,8 @@ const orphDatabasePrefix = "loom_verify_orph_" type orphHarness struct { ctx context.Context - client *storearango.Client - registry *pubarango.Registry + client *arangofactory.Client + registry *arangofactory.PublishedRegistry ch *clickhouse.Client database string suffix string @@ -73,20 +47,21 @@ func orphOpen(t *testing.T) *orphHarness { database := orphDatabasePrefix + strings.ReplaceAll(uuid.NewString(), "-", "") ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) t.Cleanup(cancel) - client, err := storearango.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatalf("open %s: %v", database, err) } + client := arangoBackend.Client t.Logf("ORPH_DATABASE=%s (via production store.Open)", database) t.Cleanup(func() { _ = client.Close(context.Background()) orphDropDatabase(t, endpoint, database) }) - registry, err := pubarango.New(client) + registry, err := arangofactory.NewPublishedRegistry(client) if err != nil { t.Fatalf("new registry: %v", err) } - if err := client.Bootstrap(ctx, pubarango.BootstrapSpec()); err != nil { + if err := client.Bootstrap(ctx, arangofactory.PublishedBootstrapSpec()); err != nil { t.Fatalf("bootstrap bundle collections: %v", err) } ch, err := clickhouse.New(clickhouse.Options{ @@ -111,11 +86,12 @@ func orphDropDatabase(t *testing.T, endpoint, database string) { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - client, err := storearango.Open(ctx, endpoint, "_system") + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: "_system"}) if err != nil { t.Logf("ORPH_CLEANUP_WARN: open _system for drop: %v", err) return } + client := arangoBackend.Client defer func() { _ = client.Close(context.Background()) }() request, err := http.NewRequestWithContext(ctx, http.MethodDelete, strings.TrimRight(endpoint, "/")+"/_api/database/"+url.PathEscape(database), nil) @@ -156,11 +132,6 @@ type orphPublication struct { rows int } -// orphNewPublication builds a publication whose PointerName is fixed by -// (project, generation, recipe, version, output) while recipeDigest/ -// schemaDigest/receiptId vary per edit — the shipped two identities then map -// to the same pointer but different Keys exactly as a real recipe edit does -// (bundle.go:88-115). func orphNewPublication(recipeDigest, schemaDigest, receiptID string) orphPublication { identity := pubpublication.PublicationIdentity{ Name: "orph-recipe", TranslationVersion: "v1", OutputName: "orph_out", @@ -186,9 +157,6 @@ func orphNewPublication(recipeDigest, schemaDigest, receiptID string) orphPublic return orphPublication{identity: identity, outputs: outputs, rows: 3} } -// orphBundleIdentity mirrors the production identity->bundle mapping used by -// the publication runner (same field set as publish_partial_pointer's -// factory) so Key/PointerName derive from shipped code. func orphBundleIdentity(id pubpublication.PublicationIdentity) pubpublication.BundleIdentity { return pubpublication.BundleIdentity{ Name: id.Name, TranslationVersion: id.TranslationVersion, OutputName: id.OutputName, @@ -243,9 +211,6 @@ func orphProductionPublish(ctx context.Context, store *ClickHouseBundleStore, p return pubpublication.Publish(ctx, store, p.identity, p.outputs, pubpublication.Limits{}) } -// orphReleaseReader mirrors the production wiring (server.go calls -// bundleStore.SetActiveReleaseReader(lifecycleStore)); the harness controls -// what the janitor sees as the ACTIVE release. type orphReleaseReader struct { active dataset.ActiveRelease err error @@ -257,9 +222,6 @@ func (r *orphReleaseReader) ReadActiveRelease(context.Context, string) (dataset. return r.active, r.err } -// orphPublishRecipeEdit drives two production publications that share one -// PointerName: edit 1 then the recipe edit (new Key). Returns the observed -// executions, tables and identity strings for assertions. func orphPublishRecipeEdit(t *testing.T, h *orphHarness, store *ClickHouseBundleStore) (firstExec, secondExec pubpublication.BundleExecution, firstTable, secondTable, firstKey, secondKey, firstPointer string) { t.Helper() first := orphNewPublication("sha256:orph-recipe-v1", "sha256:orph-schema-v1", "orph-receipt-1") @@ -316,10 +278,6 @@ func orphPublishRecipeEdit(t *testing.T, h *orphHarness, store *ClickHouseBundle } func TestPointerOrphanedPublishedNeverReclaimedAgainstArango(t *testing.T) { - // Drift gates (quote-first). The legacy three-state declaration must - // survive (stale arm); the Repair-4 janitor arm, pre-claim live filter and - // superseded marker must be PRESENT — removal re-opens F-B19 and the - // REGRESSION direction below must then fire. sweepSrc := orphReadFile(t, "internal/dataframe/publication/clickhouse/bundle_store.go") if !strings.Contains(sweepSrc, "states := []publication.BundleState{publication.BundleRunning, publication.BundleValidating, publication.BundleFailed}") { t.Fatalf("drift gate: stale-arm sweep state list changed — reassess the {RUNNING,VALIDATING,FAILED} baseline before ruling") @@ -333,7 +291,7 @@ func TestPointerOrphanedPublishedNeverReclaimedAgainstArango(t *testing.T) { t.Fatalf("REGRESSION drift: Repair-4 janitor text %q no longer present in bundle_store.go — F-B19 growth returns; the REGRESSION arm below must fire against product behavior too", needle) } } - registrySrc := orphReadFile(t, "internal/dataframe/publication/arango/bundle_registry.go") + registrySrc := orphReadFile(t, "internal/backends/arango/internal/publication/bundle_registry.go") if !strings.Contains(registrySrc, "SORT doc.publishedAt DESC") { t.Fatalf("drift gate: name-latest selector shape changed; reassess F-B19 name-latest eligibility claim") } @@ -341,12 +299,12 @@ func TestPointerOrphanedPublishedNeverReclaimedAgainstArango(t *testing.T) { h := orphOpen(t) ctx := h.ctx - store, err := NewBundleStore(h.ch, struct{ *pubarango.Registry }{h.registry}) + store, err := NewBundleStore(h.ch, struct { + *arangofactory.PublishedRegistry + }{h.registry}) if err != nil { t.Fatalf("NewBundleStore: %v", err) } - // Production wiring mirror: server.go wires the lifecycle store; here no - // release is active, so the janitor must retire the superseded orphan. store.SetActiveReleaseReader(&orphReleaseReader{err: dataset.ErrNoActiveRelease}) _, secondExec, firstTable, secondTable, firstKey, secondKey, _ := orphPublishRecipeEdit(t, h, store) @@ -378,8 +336,6 @@ func TestPointerOrphanedPublishedNeverReclaimedAgainstArango(t *testing.T) { t.Fatalf("unexpected: orphan marker %q lacks the superseded janitor marker", orphanExec.Error) } - // The live (pointer-named) execution and its table must be untouched by - // the janitor arm, and name-latest must resolve to it. live, err := h.registry.FindExecutionByKey(ctx, secondKey) if err != nil || live.State.Canonical() != pubpublication.BundlePublished { t.Fatalf("janitor arm harmed the live execution %s: state=%+v err=%v", secondKey, live, err) @@ -397,14 +353,12 @@ func TestPointerOrphanedPublishedNeverReclaimedAgainstArango(t *testing.T) { t.Logf("VERDICT PREVENTED (Repair-4): pointer-orphaned PUBLISHED execution %q retired by the binding-aware janitor arm (FAILED, marker %q, table dropped) while the pointer-named live execution %s (rows=%d) and its table survived; name-latest resolves to the live execution.", firstKey, orphanExec.Error, secondKey, selOutput2.RowCount) } -// TestPointerOrphanedPublishedJanitorRespectsActiveReleaseBindingsAgainstArango -// is the e2e safety sibling: an ACTIVE-release binding on the orphan vetoes -// retirement; after removing the binding, the next sweep converges — the -// m-po-fixed shape on real stores. func TestPointerOrphanedPublishedJanitorRespectsActiveReleaseBindingsAgainstArango(t *testing.T) { h := orphOpen(t) ctx := h.ctx - store, err := NewBundleStore(h.ch, struct{ *pubarango.Registry }{h.registry}) + store, err := NewBundleStore(h.ch, struct { + *arangofactory.PublishedRegistry + }{h.registry}) if err != nil { t.Fatalf("NewBundleStore: %v", err) } @@ -413,8 +367,6 @@ func TestPointerOrphanedPublishedJanitorRespectsActiveReleaseBindingsAgainstAran firstExec, _, firstTable, _, firstKey, _, _ := orphPublishRecipeEdit(t, h, store) - // Bind the ACTIVE release to the orphan: the janitor must not claim or - // drop it (m-po-janitor-blind would). reader.active = dataset.ActiveRelease{ Release: dataset.ProjectRelease{ Project: "orph-project", @@ -435,7 +387,6 @@ func TestPointerOrphanedPublishedJanitorRespectsActiveReleaseBindingsAgainstAran } t.Logf("ORPH_OBSERVED bound sweep: orphan %s untouched (binding veto, pre-claim filter), reader calls=%d", firstKey, reader.calls) - // Remove the binding: the next sweep must converge (recurring janitor). reader.active = dataset.ActiveRelease{} reader.err = dataset.ErrNoActiveRelease if err := store.Reconcile(ctx, time.Now().Add(24*time.Hour)); err != nil { diff --git a/internal/dataframe/publication/clickhouse/pinblock_reconcile_e2e_test.go b/internal/dataframe/publication/clickhouse/pinblock_reconcile_e2e_test.go index b2a901b96..888ce13a8 100644 --- a/internal/dataframe/publication/clickhouse/pinblock_reconcile_e2e_test.go +++ b/internal/dataframe/publication/clickhouse/pinblock_reconcile_e2e_test.go @@ -11,58 +11,12 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" pubpublication "github.com/calypr/loom/internal/dataframe/publication" - pubarango "github.com/calypr/loom/internal/dataframe/publication/arango" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/calypr/loom/internal/store/clickhouse" "github.com/google/uuid" ) -// Candidate finding H2-a: a stalled export read-pin makes Reconcile abandon -// every other stale execution and downgrade publication readiness. -// -// Source-verified wiring (unmodified): -// - internal/explorer/lifecycle/artifact.go:21 defaultArtifactTTL = 24h; -// :104 identity.ExpiresAt = now + artifactTTL(=24h); :152 the exact-export -// request carries PinExpiresAt = identity.ExpiresAt. -// - internal/dataframe/published/export.go:120-126 accepts ANY future -// PinExpiresAt, so a fresh artifact export pins the execution at now+24h; -// the renewal ticker (:156/:166 with defaultReadPinInterval=30s, -// defaultReadPinTTL=2min) is what first pulls it back to now+2min — but -// only at the FIRST 30s tick. A process crash inside that first window -// leaves a 24h live pin on the execution. -// -// Hypothesis: startup Reconcile (internal/server/server.go:556-558, a single -// one-shot call: err -> recordDegradation + publicationReady=false) reaches -// claimExecutionCleanup (bundle_store.go:301-313 -> ErrExecutionReadPinActive -// when the real ClaimExecutionCleanup AQL bundle_registry.go:411-427 sees an -// unexpired pin) for the pinned execution; reconcileExecution propagates the -// error (bundle_store.go:1076-1079) and Reconcile's `process` loop aborts the -// ENTIRE run on the first error (bundle_store.go:1033-1052, first error wins), -// so an UNRELATED stale execution is never reclaimed and the whole deployment -// loses publication readiness. Author intent hints otherwise (export.go:57 -// "shorter operation budget"); the doc on Reconcile (:1029-1032) advertises -// "safe to call repeatedly during startup", implying a partial sweep must not -// be able to veto the rest. -// -// This file drives the REAL Registry against a real Arango database and the -// production Reconcile/reconcileExecution/claimExecutionCleanup paths verbatim -// — the only fake is a ClickHouse effect recorder (like tocFakeClickHouse / -// fbxClickHouse) that records which physical tables were dropped. No wrapper -// injects any decision: the pin is created through the production catalog API. -// -// Opt-in, same idiom as reconcile_toctou_e2e_test.go / abort_ackloss_e2e_test.go. -// The tests own UUID databases named loom_verify_pbx_ and delete only -// those. Run against the owned stack from inside the api container: -// -// docker exec loom-dev-qwen-current-loom-api-1 sh -c \ -// 'cd /workspace && LOOM_TEST_ARANGO_URL=http://arangodb:8529 go test \ -// -count=1 -run "TestStalePin" -v ./internal/dataframe/publication/clickhouse/' - -// pbxClickHouse records the physical tables handed to DropTable so the tests -// can observe which stale executions the reconciler actually reclaimed. Every -// other driver method is a no-op; a dead context aborts the drop the way the -// real driver would (matching tocFakeClickHouse.DropTable). type pbxClickHouse struct { mu sync.Mutex dropped []string @@ -122,7 +76,7 @@ func pbxCleanupDatabase(t *testing.T, endpoint, database string) { } } -func pbxSetup(t *testing.T) (*pubarango.Registry, context.Context, string) { +func pbxSetup(t *testing.T) (*arangofactory.PublishedRegistry, context.Context, string) { t.Helper() endpoint := strings.TrimSpace(os.Getenv("LOOM_TEST_ARANGO_URL")) if endpoint == "" { @@ -132,27 +86,25 @@ func pbxSetup(t *testing.T) (*pubarango.Registry, context.Context, string) { t.Cleanup(cancel) database := "loom_verify_pbx_" + strings.ReplaceAll(uuid.NewString(), "-", "") t.Logf("owned arango database: %s", database) - client, err := storearango.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatalf("open arango: %v", err) } + client := arangoBackend.Client t.Cleanup(func() { _ = client.Close(context.Background()) pbxCleanupDatabase(t, endpoint, database) }) - if err := client.Bootstrap(ctx, pubarango.BootstrapSpec()); err != nil { + if err := client.Bootstrap(ctx, arangofactory.PublishedBootstrapSpec()); err != nil { t.Fatalf("bootstrap: %v", err) } - registry, err := pubarango.New(client) + registry, err := arangofactory.NewPublishedRegistry(client) if err != nil { t.Fatalf("registry: %v", err) } return registry, ctx, database[len("loom_verify_pbx_"):] } -// pbxExecution builds a stale FAILED execution under a distinct identity, so X -// and Y get distinct identity keys (separate bundle leases) and distinct -// pointer names while sharing the single FAILED page visited by Reconcile. func pbxExecution(tag, suffix, physicalTable string, updatedAt time.Time) pubpublication.BundleExecution { identity := pubpublication.BundleIdentity{ Name: "pbx-" + tag, TranslationVersion: "1", OutputName: "main", @@ -169,11 +121,7 @@ func pbxExecution(tag, suffix, physicalTable string, updatedAt time.Time) pubpub } } -// pbxSeed stores the stale row under a seed-held lease then frees the key — -// the identical registry state a stale candidate presents to a reconciler -// (same idiom as fb1Seed). No pointer is ever created, so both executions are -// un-advertised and the cleanup slot is free. -func pbxSeed(t *testing.T, registry *pubarango.Registry, ctx context.Context, exec pubpublication.BundleExecution) { +func pbxSeed(t *testing.T, registry *arangofactory.PublishedRegistry, ctx context.Context, exec pubpublication.BundleExecution) { t.Helper() acquired, err := registry.AcquireBundleLease(ctx, exec.Key, "seed", time.Now().UTC().Add(2*time.Minute)) if err != nil || !acquired { @@ -187,22 +135,6 @@ func pbxSeed(t *testing.T, registry *pubarango.Registry, ctx context.Context, ex } } -// TestStalePinBlocksEntireReconcileAgainstArango is the FINDING. Two stale -// FAILED executions X (older) and Y, neither pointer-advertised, each with an -// output table. X alone carries a never-released read pin through the real -// catalog API (crash simulation). Production Reconcile is run once. -// -// Expected-buggy outcome (this test therefore fails ON PURPOSE): -// -// (a) Reconcile returns an error for which errors.Is(err, ErrExecutionReadPinActive) -// (bundle_store.go:1076-1079 via claimExecutionCleanup :301-313 / -// bundle_registry.go:411-427); -// (b) X is not reclaimed — its DropTable is never issued (expected, legitimate); -// (c) DAMAGE: Y — an unrelated, pin-free, equally-stale execution — is ALSO -// not reclaimed, because process aborted on X's error (bundle_store.go:1033-1052). -// The control TestStalePinAbsentReclaimsAllAgainstArango proves Y's table -// WOULD have been dropped with no pin present, so (c) is caused by the pin, -// not harness ordering. func TestStalePinBlocksEntireReconcileAgainstArango(t *testing.T) { registry, ctx, suffix := pbxSetup(t) @@ -216,15 +148,10 @@ func TestStalePinBlocksEntireReconcileAgainstArango(t *testing.T) { pbxSeed(t, registry, ctx, x) pbxSeed(t, registry, ctx, y) - // The crash-simulated export pin: acquired on X through the REAL catalog - // API with a 24h expiry, then never released. This is exactly the pin a - // process crash leaves behind inside export.go's first 30s pre-renewal - // window (PinExpiresAt = artifact identity.ExpiresAt = now+24h). acquired, err := registry.AcquireExecutionReadPin(ctx, x.ID, "reader-crash-sim", base.Add(24*time.Hour)) if err != nil || !acquired { t.Fatalf("precondition: real AcquireExecutionReadPin(X) failed: acquired=%v err=%v", acquired, err) } - // Deliberately no ReleaseExecutionReadPin — this is the crash. ch := &pbxClickHouse{} store, err := NewBundleStore(ch, registry) @@ -246,25 +173,19 @@ func TestStalePinBlocksEntireReconcileAgainstArango(t *testing.T) { xReclaimed := pbxDropped(dropped, xTable) yReclaimed := pbxDropped(dropped, yTable) - // (a) the pin must be the reason the run errored. if !pinErr { t.Errorf("FINDING NOT REPRODUCED (abort cause): expected errors.Is(err, ErrExecutionReadPinActive); reconcile returned: %v (dropped=%v)", reconcileErr, dropped) } - // (a)+(c) the collateral: an unrelated, pin-free stale sibling is stranded. if pinErr && !xReclaimed && !yReclaimed { t.Errorf("CONFIRMED against real Arango (H2-a): one crashed export's never-released read pin on %q aborted the ENTIRE Reconcile run (bundle_store.go:1033-1052 first-error-wins; :1076-1079 propagates claimExecutionCleanup's ErrExecutionReadPinActive). Unrelated, pin-free, equally-stale execution %q was STRANDED: DropTable never issued for it. reconcileErr=%v; X(id=%s table=%s) reclaimed=%v state=%q reconcile-error-field=%q; Y(id=%s table=%s) reclaimed=%v state=%q reconcile-error-field=%q; full dropped set=%v. Contrast control TestStalePinAbsentReclaimsAllAgainstArango: with no pin BOTH %s and %s are dropped by the very same production Reconcile. Startup impact: internal/server/server.go:556-558 turns this one error into recordDegradation + publicationReady=false (bundleTarget stays nil at :560-562, consumed at :823) with NO retry/background re-run of bundle-store Reconcile anywhere in server startup.", x.ID, y.ID, reconcileErr, x.ID, xTable, xReclaimed, xRow.State, xRow.Error, y.ID, yTable, yReclaimed, yRow.State, yRow.Error, dropped, xTable, yTable) } else if pinErr && !xReclaimed && yReclaimed { - // A compensating skip-and-continue path would land here. Do not let it - // pass silently: this would refute the hypothesis and must be visible. t.Errorf("UNEXPECTED: pin abort did NOT strand sibling Y (dropped=%v); a compensating path exists in Reconcile — re-examine hypothesis H2-a", dropped) } if xReclaimed { t.Errorf("unexpected: pinned execution X was reclaimed despite the live pin (dropped=%v)", dropped) } - // Y never reached reconcileExecution, so its row is untouched: still FAILED - // and never stamped with Reconcile's cleanup marker. if yRow.State != pubpublication.BundleFailed { t.Errorf("unexpected: strayed sibling Y changed state to %q; expected untouched %q", yRow.State, pubpublication.BundleFailed) } @@ -273,11 +194,6 @@ func TestStalePinBlocksEntireReconcileAgainstArango(t *testing.T) { } } -// TestStalePinAbsentReclaimsAllAgainstArango is the CONTROL and must PASS: the -// same two stale executions, identical seeding, NO pin. Production Reconcile -// returns nil and BOTH tables are dropped, pinning that the stranding in the -// finding test is caused by the pin rather than harness ordering, paging, or -// execution order. func TestStalePinAbsentReclaimsAllAgainstArango(t *testing.T) { registry, ctx, suffix := pbxSetup(t) @@ -320,11 +236,6 @@ func TestStalePinAbsentReclaimsAllAgainstArango(t *testing.T) { t.Logf("CONTROL OK: with no pins, the identical production Reconcile reclaimed BOTH stale executions (dropped=%v); each row marked %q. This is the exact sibling that the pin-aborted run in the finding test stranded.", dropped, "stale execution reconciled") } -// TestLivePinOnOnlyStaleExecutionStillAbortsRunAgainstArango is the -// single-target expected-behavior probe and is EXPECTED TO PASS: only the -// pinned X is stale, so the run abort on ErrExecutionReadPinActive causes no -// collateral. This isolates the collateral in the finding test: it arises only -// when a sibling is present, not from the abort per se. func TestLivePinOnOnlyStaleExecutionStillAbortsRunAgainstArango(t *testing.T) { registry, ctx, suffix := pbxSetup(t) diff --git a/internal/dataframe/publication/clickhouse/pointer_name_nul_ambiguity_e2e_test.go b/internal/dataframe/publication/clickhouse/pointer_name_nul_ambiguity_e2e_test.go index 34a17b9ac..b19b600c9 100644 --- a/internal/dataframe/publication/clickhouse/pointer_name_nul_ambiguity_e2e_test.go +++ b/internal/dataframe/publication/clickhouse/pointer_name_nul_ambiguity_e2e_test.go @@ -1,49 +1,5 @@ package clickhouse -// Candidate finding F-B21 e2e: BundleIdentity.PointerName (internal/dataframe/ -// publication/bundle.go:88-97) is a bare "\x00"-join of -// (Project, DatasetGeneration, Name[, TranslationVersion][, OutputName]) while -// Key() (bundle.go:99-115) JSON-marshals every field distinctly. Two -// logically distinct identities -// -// A: Name = "a\x00b", TranslationVersion = "" -// B: Name = "a", TranslationVersion = "b" -// -// under one project/generation/share an OutputName fold to the SAME -// PointerName ("proj\0gen\0a\0b\0out") while their Keys stay distinct. No -// control-character validation guards publication identity: the shipped -// control-char gate internal/dataset/validation.go:40-44 -// (validateOpaqueIdentifier) is called ONLY from internal/dataset/ref.go:26,29 -// (dataset project/generation refs) and internal/dataset/schema.go:51 -// (generatedResourceTypes). The catalog layer already length-prefixes for -// exactly this attack (internal/catalog/helpers.go:50-62 comment). -// -// This test drives real Arango + real ClickHouse through production entries -// (publication.Publish -> ClickHouseBundleStore -> arango.Registry, and the -// pointer-gated published.Reader). Repair-5 (length-prefix PointerName) -// converted it to a natural PASS: the STEP-2 identity comparison now reports -// VERDICT PREVENTED before any store is touched. The full STEP 3-5 hijack -// choreography with its archived FAILED-ON-PURPOSE F-B21 CONFIRMED message -// stays wired as the REGRESSION direction: if the PointerName encoding ever -// folds the two identities again, the test fails with the original red text. -// -// Fixture necessity (honest): publication.ValidateOutputs requires a non-blank -// output name and the physical table suffix is the output name -// (bundle_store.go:908-910), so BOTH identities share one identical non-blank -// OutputName ("pnn_out_shared"). OutputName is therefore deliberately kept -// OUT of the colliding fields: only Name/TranslationVersion vary, and the -// collision needs only those two because PointerName appends them -// conditionally and joins with an unescaped "\x00". -// -// Opt-in (SKIP-clean without LOOM_TEST_ARANGO_URL + LOOM_CLICKHOUSE_URL, plus -// the LOOM_CLICKHOUSE_* vars the sibling harnesses use): -// -// docker exec loom-dev-qwen-current-loom-api-1 sh -c \ -// 'cd /workspace && LOOM_TEST_ARANGO_URL=http://arangodb:8529 \ -// LOOM_CLICKHOUSE_URL=http://clickhouse:8123 LOOM_CLICKHOUSE_DATABASE=loom_dev \ -// LOOM_CLICKHOUSE_USERNAME=loom_dev LOOM_CLICKHOUSE_PASSWORD=loom_dev_password \ -// go test -count=1 -run TestPointerNameNulAmbiguity -v ./internal/dataframe/publication/clickhouse/' - import ( "context" "fmt" @@ -59,20 +15,15 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/publication" - pubarango "github.com/calypr/loom/internal/dataframe/publication/arango" "github.com/calypr/loom/internal/dataframe/published" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/calypr/loom/internal/store/clickhouse" "github.com/google/uuid" ) const pnnDatabasePrefix = "loom_verify_pnn_" -// pnnChOwnedPrefix owns this suite's ClickHouse canary marker. Production -// staging tables are named loom_bundle__ -// (bundle_store.go:908-910); the shared output name embeds "pnn", so every -// artifact this test creates in ClickHouse matches LIKE '%pnn%'. const pnnChOwnedPrefix = "ch_pnn" const ( @@ -82,29 +33,19 @@ const ( pnnSchemaDigest = "sha256:pnn-schema-shared" pnnScopeDigest = "sha256:pnn-scope-shared" pnnEngineVersion = "pnn-engine" - pnnNameA = "a\x00b" // embedded NUL: the ambiguous component - pnnNameB = "a" // with TranslationVersion "b" => same PointerName + pnnNameA = "a\x00b" + pnnNameB = "a" pnnTranslationB = "b" pnnStreamBaseA = 101 pnnStreamBaseB = 201 pnnStreamRowCount = 3 ) -// --------------------------------------------------------------------------- -// STEP 0 drift gate: quote the live production sources the claim rests on and -// fail on drift. Also negative-scans the whole publication package tree for -// any control-char validator (its presence would PREVENT the finding). -// --------------------------------------------------------------------------- - var pnnRequiredQuotations = []struct { file string needles []string }{ { - // Repair-5 re-pin: the pointer encoding must stay length-prefixed. - // Removal of these anchors re-opens F-B21 and the STEP-2 identity - // comparison below then falls through to the archived CONFIRMED - // hijack assertions (the REGRESSION direction). file: "internal/dataframe/publication/bundle.go", needles: []string{ `parts := []string{i.Project, i.DatasetGeneration, i.Name}`, @@ -182,7 +123,6 @@ func pnnQuoteVerifiedSource(t *testing.T) (quotations string, controlCharHits [] } } } - // Quote the exact control-char block (validation.go:40-44 by assignment). controlNeedles := []string{ `for _, r := range value {`, `if unicode.IsControl(r) {`, @@ -196,8 +136,6 @@ func pnnQuoteVerifiedSource(t *testing.T) (quotations string, controlCharHits [] } } } - // Negative scan: no control-char/identifier validator anywhere on the - // publication path (the shipped gate lives in internal/dataset only). err := filepath.Walk(filepath.Join(root, "internal/dataframe/publication"), func(path string, info os.FileInfo, walkErr error) error { if walkErr != nil { return walkErr @@ -235,15 +173,10 @@ func mustReadPnnFile(t *testing.T, root, rel string) []byte { return data } -// --------------------------------------------------------------------------- -// owned-database + owned-ClickHouse harness (SKIP-clean opt-in), mirroring -// the sibling pppOpen pattern with pnn-owned identifiers only. -// --------------------------------------------------------------------------- - type pnnHarness struct { ctx context.Context - client *storearango.Client - registry *pubarango.Registry + client *arangofactory.Client + registry *arangofactory.PublishedRegistry ch *clickhouse.Client database string suffix string @@ -277,22 +210,21 @@ func pnnOpen(t *testing.T) *pnnHarness { } ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) t.Cleanup(cancel) - client, err := storearango.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatalf("open %s: %v", database, err) } + client := arangoBackend.Client t.Logf("PNN_DATABASE=%s", database) - // Registered FIRST so it runs LAST: ClickHouse artifacts must be dropped - // before the Arango database disappears (LIFO cleanup order). t.Cleanup(func() { _ = client.Close(context.Background()) pnnDropDatabase(t, endpoint, database) }) - registry, err := pubarango.New(client) + registry, err := arangofactory.NewPublishedRegistry(client) if err != nil { t.Fatalf("new registry: %v", err) } - if err := client.Bootstrap(ctx, pubarango.BootstrapSpec()); err != nil { + if err := client.Bootstrap(ctx, arangofactory.PublishedBootstrapSpec()); err != nil { t.Fatalf("bootstrap bundle collections: %v", err) } ch, err := clickhouse.New(clickhouse.Options{ @@ -316,9 +248,6 @@ func pnnOpen(t *testing.T) *pnnHarness { database: database, suffix: database[len(pnnDatabasePrefix):], } - // Registered second: drops every tracked ClickHouse table by exact name - // and asserts the '%pnn%' residue gate empty before LIFO proceeds to the - // Arango database removal. t.Cleanup(func() { harness.dropAndVerifyClickHouse(t) }) @@ -395,13 +324,6 @@ func pnnDropDatabase(t *testing.T, endpoint, database string) { t.Logf("PNN_DATABASE_REMOVED=%s", database) } -// --------------------------------------------------------------------------- -// identities: the ONLY difference between A and B is the ambiguous field pair -// (Name, TranslationVersion); every other identity component is identical, so -// a Key() difference can only come from the per-field JSON marshalling and a -// PointerName equality can only come from the bare-join ambiguity. -// --------------------------------------------------------------------------- - type pnnPublication struct { identity publication.PublicationIdentity key string @@ -445,15 +367,10 @@ func pnnNewPublication(name, translationVersion, generation string, streamBase i return pnnPublication{identity: identity, key: bundle.Key(), pointer: bundle.PointerName(), outputs: outputs} } -// pnnProductionPublish is the same production entry recipe_publication.go uses. func pnnProductionPublish(ctx context.Context, store *ClickHouseBundleStore, p pnnPublication) (publication.Result, error) { return publication.Publish(ctx, store, p.identity, p.outputs, publication.Limits{}) } -// --------------------------------------------------------------------------- -// direct fact readers -// --------------------------------------------------------------------------- - func pnnCount(value any) (int64, bool) { switch typed := value.(type) { case uint64: @@ -506,9 +423,6 @@ func pnnTableValues(t *testing.T, ch *clickhouse.Client, ctx context.Context, ta return values } -// pnnIsValidationRejection reports whether a publish error is the fingerprint -// of a control-character/identifier validator blocking the NUL name (a -// PREVENTED verdict), as opposed to an unrelated infrastructure failure. func pnnIsValidationRejection(err error) bool { if err == nil { return false @@ -537,14 +451,7 @@ func pnnVerdictPrevented(t *testing.T, stage string, err error, hits []string) { t.Logf("VERDICT PREVENTED (stage %s):\n%s", stage, witness.String()) } -// --------------------------------------------------------------------------- -// the reproduction -// --------------------------------------------------------------------------- - func TestPointerNameNulAmbiguityHijacksVisibilityPointerAgainstArango(t *testing.T) { - // ========================================================================= - // STEP 0 — drift gate. - // ========================================================================= quotations, controlCharHits := pnnQuoteVerifiedSource(t) t.Logf("OBSERVED STEP 0 quoted verified source (anchors located live; drift fails the test):\n%s", quotations) if len(controlCharHits) > 0 { @@ -561,11 +468,6 @@ func TestPointerNameNulAmbiguityHijacksVisibilityPointerAgainstArango(t *testing t.Fatalf("NewBundleStore: %v", err) } - // ========================================================================= - // STEP 1/2 — build the colliding identities. OutputName is identical on - // both (fixture necessity: the shared non-blank output keeps outputs OUT - // of the collision; only Name/TranslationVersion vary). - // ========================================================================= generation := "pnn-gen-" + h.suffix pubA := pnnNewPublication(pnnNameA, "", generation, pnnStreamBaseA) pubB := pnnNewPublication(pnnNameB, pnnTranslationB, generation, pnnStreamBaseB) @@ -578,15 +480,11 @@ func TestPointerNameNulAmbiguityHijacksVisibilityPointerAgainstArango(t *testing t.Logf("VERDICT PREVENTED: production BundleIdentity.PointerName() keeps the two identities distinct (A=%q B=%q); the bare-join ambiguity is gone", pubA.pointer, pubB.pointer) return } - // Mechanism evidence: equality produced by the PRODUCTION PointerName(). t.Logf("OBSERVED STEP 2 mechanism (production BundleIdentity.PointerName, bundle.go:88-97): nameA=%q tvA=%q outA=%q | nameB=%q tvB=%q outB=%q -> PointerName A==B==%q while Key A=%s != Key B=%s", pubA.identity.Name, pubA.identity.TranslationVersion, pubA.identity.OutputName, pubB.identity.Name, pubB.identity.TranslationVersion, pubB.identity.OutputName, pubA.pointer, pubA.key, pubB.key) - // ========================================================================= - // STEP 3 — publication A through production publication.Publish. - // ========================================================================= resultA, errA := pnnProductionPublish(ctx, store, pubA) if errA != nil { if pnnIsValidationRejection(errA) { @@ -620,9 +518,6 @@ func TestPointerNameNulAmbiguityHijacksVisibilityPointerAgainstArango(t *testing } t.Logf("OBSERVED STEP 3 publish A: execA=%q state=%q pointer names execA; tableA=%q present rows=%v", execA.ID, execA.State, tableA, valuesA) - // ========================================================================= - // STEP 4 — publication B: same pointer name, logically distinct identity. - // ========================================================================= resultB, errB := pnnProductionPublish(ctx, store, pubB) if errB != nil { if pnnIsValidationRejection(errB) { @@ -644,7 +539,6 @@ func TestPointerNameNulAmbiguityHijacksVisibilityPointerAgainstArango(t *testing t.Fatalf("B precondition: state %q, want PUBLISHED", execB.State) } - // The hijack: the SAME pointer name now names execB. pointerAfterB, err := h.registry.GetPointer(ctx, pubA.pointer) if err != nil { t.Fatalf("Step 4 GetPointer(%q) after publishing B: %v", pubA.pointer, err) @@ -660,12 +554,6 @@ func TestPointerNameNulAmbiguityHijacksVisibilityPointerAgainstArango(t *testing t.Logf("OBSERVED STEP 4 publish B: execB=%q; GetPointer() executionId=%q flipped-to-B=%v; execA still PUBLISHED=%v (state=%q) but no longer pointer-advertised=%v; tableB=%q rows=%v", execB.ID, pointerAfterB.ExecutionID, flipped, aStillPublished, execANow.State, aNoLongerAdvertised, tableB, valuesB) - // ========================================================================= - // STEP 5 — reader cross-check on the production pointer-gated path. - // published.Reader.CurrentProjectDatasets gates every serving decision on - // GetPointer(execution.PointerName()).ExecutionID == execution.ID - // (internal/dataframe/published/current.go:158-164). - // ========================================================================= reader := &published.Reader{ Catalog: h.registry, Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), @@ -681,7 +569,6 @@ func TestPointerNameNulAmbiguityHijacksVisibilityPointerAgainstArango(t *testing onlyB := len(datasets) == 1 && datasets[0].PhysicalTable == tableB t.Logf("OBSERVED STEP 5a pointer-gated project reader exposes exactly %v (only-B=%v); PUBLISHED execA (%d rows in %s) is unservable", served, onlyB, len(valuesA), tableA) - // Selector resolution of A's logical name via the production reader. selectorA := publication.DataframeSelector{Recipe: pnnNameA, TranslationVersion: "", Output: pnnSharedOutput} selectorB := publication.DataframeSelector{Recipe: pnnNameB, TranslationVersion: pnnTranslationB, Output: pnnSharedOutput} matA, errAQuery := reader.CurrentProjectDataset(ctx, pnnProject, selectorA) @@ -691,14 +578,10 @@ func TestPointerNameNulAmbiguityHijacksVisibilityPointerAgainstArango(t *testing if errBQuery != nil || matB.PhysicalTable != tableB { t.Fatalf("Step 5b broken: reader must resolve the B selector to tableB (table=%q err=%v)", matB.PhysicalTable, errBQuery) } - // Registry evidence for the name-latest selector (no pointer filter). selExecB, selOutB, selErr := h.registry.FindExecutionBySelector(ctx, pnnProject, generation, selectorB) _, _, selErrA := h.registry.FindExecutionBySelector(ctx, pnnProject, generation, selectorA) t.Logf("OBSERVED STEP 5c name-latest selector: B-selector->exec %q physical %q err=%v; A-selector rejected by selector.Validate (empty TV) err=%v", selExecB.ID, selOutB.PhysicalTable, selErr, selErrA) - // The direct harm chain: resolving the shared pointer NAME (the visibility - // name the first dataset was advertised under) via production GetPointer - // yields B's execution and B's physical table with B's rows. hijackedExec, err := h.registry.GetExecution(ctx, pointerAfterB.ExecutionID) if err != nil { t.Fatalf("Step 5 resolution-chain GetExecution(%q): %v", pointerAfterB.ExecutionID, err) @@ -715,9 +598,6 @@ func TestPointerNameNulAmbiguityHijacksVisibilityPointerAgainstArango(t *testing t.Logf("OBSERVED STEP 5d resolution chain: GetPointer().executionId=%q -> physical table %q rows=%v (B's rows=%v served where A's rows=%v were advertised) chainHarm=%v", pointerAfterB.ExecutionID, hijackedTable, hijackedValues, valuesB, valuesA, hijackChainHarm) - // ========================================================================= - // VERDICT - // ========================================================================= if flipped && aStillPublished && aNoLongerAdvertised && onlyB && hijackChainHarm { t.Fatalf("FAILED-ON-PURPOSE F-B21 CONFIRMED: production BundleIdentity.PointerName() (bundle.go:88-97) folded Name=%q/TV=%q and Name=%q/TV=%q to the SAME visibility pointer %q (Keys %s/%s distinct via Key() at :99-115). The shipped publication path carries NO control-char validation on identity components (negative scan clean; the shipped gate internal/dataset/validation.go:40-44 wraps only dataset refs at internal/dataset/ref.go:26,29), so publishing the second identity silently repointed the first dataset's readers: GetPointer flipped execA %q -> execB %q, execA remains state=%q PUBLISHED but the pointer-gated production reader (published/current.go:158-164) serves ONLY %q (B's rows %v where A's rows %v were advertised). Intra-project namespace hijack; contrast the length-prefix defense the catalog already uses (internal/catalog/helpers.go:50-62).", pnnNameA, "", pnnNameB, pnnTranslationB, pubA.pointer, pubA.key[:12], pubB.key[:12], execA.ID, execB.ID, execANow.State, tableB, hijackedValues, valuesA) diff --git a/internal/dataframe/publication/clickhouse/publish_partial_pointer_e2e_test.go b/internal/dataframe/publication/clickhouse/publish_partial_pointer_e2e_test.go index 820b7dd4c..604b040cf 100644 --- a/internal/dataframe/publication/clickhouse/publish_partial_pointer_e2e_test.go +++ b/internal/dataframe/publication/clickhouse/publish_partial_pointer_e2e_test.go @@ -17,129 +17,37 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" pubpublication "github.com/calypr/loom/internal/dataframe/publication" - pubarango "github.com/calypr/loom/internal/dataframe/publication/arango" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/calypr/loom/internal/store/clickhouse" "github.com/google/uuid" ) -// Candidate finding F-B16 (registered Round 11, FINDINGS). REPAIRED in Repair-1 -// (Round 26): PublishExecution now commits both legs inside ONE Arango -// transaction (bundle_registry.go WithTransaction over -// executions+pointers+leases), so the PRODUCT can no longer produce the -// two-answers split; durable post-fix proof lives in -// internal/dataframe/publication/arango/publish_atomicity_e2e_test.go (arms -// A/B: split observable only when the legs are separate requests; repaired -// publish rolls back wholly). Historical pre-fix FAILED-ON-PURPOSE evidence is -// preserved in FINDINGS.md Round 11 and the archived parent runs (artifact -// md5 4ef560d6 at dc9016bc). THIS ARTIFACT NOW SERVES AS A LABELED FAULT -// CONTROL for the same harm class applied to fabricated or legacy -// (pre-repair) rows: it fabricates the split by bypassing the repaired -// production PublishExecution (the wrapper runs the pre-repair legs itself), -// and asserts the consequences an operator would see for such a row — -// selector serves the orphan, the bulk-load gate refuses it, and the -// idempotent replay does not repair the pointer (documented residual property; -// the counterfactual sweep/replay fixes in PublishPartialPointer.tla remain -// unimplemented on purpose). If those products ever land, this artifact's -// VERDICT fails loudly so the fault-control semantics are re-derived, not -// silently weakened. -// -// Verified source facts (re-read and quoted at runtime by -// pppQuoteVerifiedSource, which fails this test if any anchor drifts): -// -// - Registry.PublishExecution (internal/dataframe/publication/arango/ -// bundle_registry.go) contains the execution UPDATE -// (`UPDATE execution WITH @execution IN @@executions`) and the -// visibility pointer UPSERT in the SAME statement, now executed inside -// one WithTransaction (Repair-1, Round 26); at dc9016bc the same -// statement ran non-transactionally and a server-side abort between the -// legs left the execution PUBLISHED while the pointer was never written -// -- the fault this artifact injects by re-running those legs manually. -// - The commit-fallback closes the tx uncertain on inconsistent metadata -// (bundle_store.go:818-821 `t.closed = true // Prevent the runner from -// deleting possibly published tables.`; confirmCommit :858 -// "publication metadata is inconsistent"). -// - An idempotent replay never repairs the pointer: beginBundle treats a -// published/ready execution as idempotent (:137-140) and Commit returns -// ExistingPublishedOutputs without touching the catalog (:762-764); -// publication.Publish returns early on tx.Idempotent() (runner.go:45-46). -// - Reconcile EXCLUDES PUBLISHED: -// `states := []publication.BundleState{publication.BundleRunning, -// publication.BundleValidating, publication.BundleFailed}` -// (bundle_store.go:1043). -// - The bulk-load activation gate refuses non-pointer executions -// (internal/api/bulk/load/service.go:199-204 "is no longer active", -// and the GetPointer error branch :200-202). -// - The name-latest selector has no pointer filter -// (bundle_registry.go:174-182 `SORT doc.publishedAt DESC ... LIMIT 1` -// never reads the pointers collection). -// -// Experiment (real Arango + real ClickHouse, production entry points): -// -// CONTROL publish exec CONTROL fully through production -// publication.Publish (the entry internal/server/recipe_publication.go:143 -// uses) against the real ClickHouse client and real Registry. -// FAULT-INJECTION a catalog wrapper that passes every BundleCatalog call -// through to the real Registry EXCEPT PublishExecution, which — -// POST-R26 — can no longer split on its own, so the wrapper -// fabricates the pre-fix split itself: it applies ONLY the exec -// UPDATE leg (same LET/FILTER prelude and the verbatim -// `UPDATE execution WITH @execution IN @@executions` via -// client.QueryRows with the same binds) and then returns a -// write-conflict-style error BEFORE the pointer UPSERT leg. This -// models a fabricated or legacy (pre-repair) row, not product -// behavior. -// REPLAY the SAME production publish entry is called again with the same -// identity; the documented residual property asserts an idempotent -// success that leaves the pointer unwritten (counterfactual repair -// unimplemented; if a product ever ships replay-then-heal, this -// assertion fails so the fault control is re-derived, not weakened). -// PROBES FindExecutionBySelector (name-latest) and a verbatim replication -// of the bulk-load gate service.go:199-204 against real GetPointer. -// -// Opt-in (SKIP-clean without LOOM_TEST_ARANGO_URL + LOOM_CLICKHOUSE_URL): -// -// docker exec loom-dev-qwen-current-loom-api-1 sh -c \ -// 'cd /workspace && LOOM_TEST_ARANGO_URL=http://arangodb:8529 \ -// LOOM_CLICKHOUSE_URL=http://clickhouse:8123 LOOM_CLICKHOUSE_DATABASE=loom_dev \ -// LOOM_CLICKHOUSE_USERNAME=loom_dev LOOM_CLICKHOUSE_PASSWORD=loom_dev_password \ -// go test -count=1 -run TestPublishPartialPointer -v ./internal/dataframe/publication/clickhouse/' - const pppDatabasePrefix = "loom_verify_ppp_" -// pppChOwnedPrefix owns this suite's ClickHouse residue marker. The canary -// table exercises the same DropTable path the cleanup uses for the -// production-created loom_bundle_* staging tables (whose names are -// loom_bundle__, bundle_store.go:908-910, so they are -// tracked by exact name rather than by prefix). const pppChOwnedPrefix = "ch_ppp" -// --------------------------------------------------------------------------- -// catalog wrapper: passes every call through except the faulted publish -// --------------------------------------------------------------------------- - type pppPublishCall struct { PointerName string Expected string ExecutionID string - Delegated bool // true = ran the real Registry.PublishExecution verbatim - LegOneApplied bool // exec UPDATE leg applied (faulted call only) - LegTwoExecuted bool // pointer UPSERT leg ever executed on this call + Delegated bool + LegOneApplied bool + LegTwoExecuted bool InjectedError string } type pppCatalog struct { - *pubarango.Registry - client *storearango.Client + *arangofactory.PublishedRegistry + client *arangofactory.Client mu sync.Mutex faultArmed bool calls []pppPublishCall } -func newPPPCatalog(registry *pubarango.Registry, client *storearango.Client) *pppCatalog { - return &pppCatalog{Registry: registry, client: client} +func newPPPCatalog(registry *arangofactory.PublishedRegistry, client *arangofactory.Client) *pppCatalog { + return &pppCatalog{PublishedRegistry: registry, client: client} } func (c *pppCatalog) armFault() { @@ -154,16 +62,6 @@ func (c *pppCatalog) snapshotCalls() []pppPublishCall { return append([]pppPublishCall(nil), c.calls...) } -// pppLegOneAQL is the PRE-FIX (dc9016bc) production PublishExecution AQL -// deconstructed into its two verified legs; this string keeps ONLY leg one: -// the LET/FILTER prelude (:284-291 @ dc9016bc) and the execution UPDATE -// (:292). The pointer UPSERT (:293-296) is deliberately absent on the -// faulted call -- fault-injection only: the repaired production publish runs -// both legs inside one WithTransaction and cannot be split this way without -// bypassing it. @name and @updatedAt are bound verbatim like production -// (:298-301) and merely parked in LET bindings because Arango rejects bind -// parameters no query text declares and only the skipped pointer-UPSERT leg -// references them. const pppLegOneAQL = `LET pointer = DOCUMENT(@@pointers, @pointerKey) LET execution = DOCUMENT(@@executions, @executionKey) LET lease = DOCUMENT(@@leases, @leaseKey) @@ -177,19 +75,11 @@ FILTER execution != null UPDATE execution WITH @execution IN @@executions RETURN {updated: true}` -// pppPointerDocumentKey mirrors pubarango pointerDocumentKey (bundle_registry.go: -// 473-476): hex(sha256(pointerName)). Reproduced because the production helper -// is unexported and this harness must hand the faulted leg the same @pointerKey -// bind the real query would use. func pppPointerDocumentKey(name string) string { sum := sha256.Sum256([]byte(name)) return hex.EncodeToString(sum[:]) } -// PublishExecution delegates verbatim while disarmed. When armed it executes -// ONLY the exec UPDATE leg of the real query (same binds) and returns a -// write-conflict-style error BEFORE the pointer UPSERT leg, simulating the -// server-side partial application of the one non-transactional AQL. func (c *pppCatalog) PublishExecution(ctx context.Context, name, expected string, execution pubpublication.BundleExecution) error { data, err := json.Marshal(execution) if err != nil { @@ -207,9 +97,9 @@ func (c *pppCatalog) PublishExecution(ctx context.Context, name, expected string if !armed { call.Delegated = true - err = c.Registry.PublishExecution(ctx, name, expected, execution) + err = c.PublishedRegistry.PublishExecution(ctx, name, expected, execution) if err == nil { - call.LegTwoExecuted = true // the real single-AQL publish covers both legs + call.LegTwoExecuted = true } c.mu.Lock() c.calls = append(c.calls, call) @@ -217,12 +107,11 @@ func (c *pppCatalog) PublishExecution(ctx context.Context, name, expected string return err } - // Leg ONE only: the exec UPDATE, same binds as bundle_registry.go:298-301. updated := false err = c.client.QueryRows(ctx, pppLegOneAQL, 32, map[string]interface{}{ - "@pointers": pubarango.BundlePointersCollection, - "@executions": pubarango.BundleExecutionsCollection, - "@leases": pubarango.BundleLeasesCollection, + "@pointers": arangofactory.BundlePointersCollection, + "@executions": arangofactory.BundleExecutionsCollection, + "@leases": arangofactory.BundleLeasesCollection, "pointerKey": pppPointerDocumentKey(name), "executionKey": execution.ID, "leaseKey": execution.Key, @@ -246,10 +135,6 @@ func (c *pppCatalog) PublishExecution(ctx context.Context, name, expected string if !updated { return errors.New("ppp fault precondition: the exec UPDATE leg was filtered out; execution/lease/pointer guards did not hold") } - // Leg TWO (pointer UPSERT, bundle_registry.go:293-296) is deliberately - // NOT executed. Return a write-conflict-style error, which is NOT - // publication.ErrBundlePointerConflict, so Commit takes the confirmCommit - // fallback at bundle_store.go:813-822 rather than the conflict branch. injected := errors.New("write conflict: document revision conflict while applying visibility pointer UPSERT (injected partial-application fault)") c.mu.Lock() c.calls[len(c.calls)-1].InjectedError = injected.Error() @@ -257,14 +142,10 @@ func (c *pppCatalog) PublishExecution(ctx context.Context, name, expected string return injected } -// --------------------------------------------------------------------------- -// owned-database + owned-ClickHouse harness (SKIP-clean opt-in) -// --------------------------------------------------------------------------- - type pppHarness struct { ctx context.Context - client *storearango.Client - registry *pubarango.Registry + client *arangofactory.Client + registry *arangofactory.PublishedRegistry catalog *pppCatalog ch *clickhouse.Client database string @@ -299,22 +180,21 @@ func pppOpen(t *testing.T) *pppHarness { } ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) t.Cleanup(cancel) - client, err := storearango.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatalf("open %s: %v", database, err) } + client := arangoBackend.Client t.Logf("PPP_DATABASE=%s", database) - // Registered FIRST so it runs LAST: ClickHouse artifacts must be dropped - // before the Arango database disappearance (LIFO cleanup order). t.Cleanup(func() { _ = client.Close(context.Background()) pppDropDatabase(t, endpoint, database) }) - registry, err := pubarango.New(client) + registry, err := arangofactory.NewPublishedRegistry(client) if err != nil { t.Fatalf("new registry: %v", err) } - if err := client.Bootstrap(ctx, pubarango.BootstrapSpec()); err != nil { + if err := client.Bootstrap(ctx, arangofactory.PublishedBootstrapSpec()); err != nil { t.Fatalf("bootstrap bundle collections: %v", err) } ch, err := clickhouse.New(clickhouse.Options{ @@ -339,14 +219,9 @@ func pppOpen(t *testing.T) *pppHarness { database: database, suffix: database[len(pppDatabasePrefix):], } - // Registered second: drops every tracked ClickHouse table (production - // staging tables by exact name plus the owned-prefix canary) and asserts - // residue emptiness before LIFO continues to the Arango database removal. t.Cleanup(func() { harness.dropAndVerifyClickHouse(t) }) - // Owned-prefix canary: proves the cleanup's DropTable path works and the - // system.tables residue gate on the owned prefix is live. harness.canaryTable = pppChOwnedPrefix + "_canary_" + harness.suffix if err := ch.CreateTable(ctx, harness.canaryTable, []clickhouse.Column{{Name: "__loom_row_id", Type: "UInt64"}}); err != nil { t.Fatalf("create canary table %s: %v", harness.canaryTable, err) @@ -420,18 +295,13 @@ func pppDropDatabase(t *testing.T, endpoint, database string) { t.Logf("PPP_DATABASE_REMOVED=%s", database) } -// --------------------------------------------------------------------------- -// fixtures -// --------------------------------------------------------------------------- - type pppPublication struct { identity pubpublication.PublicationIdentity key string pointer string outputs []pubpublication.OutputStream - tableName string // set from the execution record once published - // row values carried by every output. - rows int + tableName string + rows int } func pppNewPublication(label, project string, h *pppHarness) pppPublication { @@ -468,22 +338,14 @@ func (p pppPublication) selector() pubpublication.DataframeSelector { return pubpublication.DataframeSelector{Recipe: p.identity.Name, TranslationVersion: p.identity.TranslationVersion, Output: p.identity.OutputName} } -// --------------------------------------------------------------------------- -// production-path publish entry (same call recipe_publication.go:143 uses) -// --------------------------------------------------------------------------- - func pppProductionPublish(ctx context.Context, store *ClickHouseBundleStore, p pppPublication) (pubpublication.Result, error) { return pubpublication.Publish(ctx, store, p.identity, p.outputs, pubpublication.Limits{}) } -// --------------------------------------------------------------------------- -// direct fact readers -// --------------------------------------------------------------------------- - -func pppRawExecutionRow(t *testing.T, client *storearango.Client, ctx context.Context, execID string) (state, errField string) { +func pppRawExecutionRow(t *testing.T, client *arangofactory.Client, ctx context.Context, execID string) (state, errField string) { t.Helper() err := client.QueryRows(ctx, `RETURN DOCUMENT(@@collection, @key)`, 32, map[string]interface{}{ - "@collection": pubarango.BundleExecutionsCollection, "key": execID, + "@collection": arangofactory.BundleExecutionsCollection, "key": execID, }, func(row map[string]any) error { data, err := json.Marshal(row) if err != nil { @@ -556,50 +418,40 @@ func pppTableRows(t *testing.T, ch *clickhouse.Client, ctx context.Context, tabl return count } -// --------------------------------------------------------------------------- -// verbatim replication of the bulk-load activation gate -// (internal/api/bulk/load/service.go activateGeneration :187-205, the part -// that consults GetExecution, the publication predicates, and GetPointer). -// --------------------------------------------------------------------------- - type pppReleaseStore interface { GetExecution(context.Context, string) (pubpublication.BundleExecution, error) GetPointer(context.Context, string) (pubpublication.BundlePointer, error) } func pppBulkGateCheck(ctx context.Context, store pppReleaseStore, project, generation, executionID string) (string, error) { - execution, err := store.GetExecution(ctx, executionID) // service.go:187 + execution, err := store.GetExecution(ctx, executionID) if err != nil { return "service.go:188-190", fmt.Errorf("load dataframe release %q: %w", executionID, err) } - if execution.Project != project || execution.DatasetGeneration != generation || !execution.State.Successful() || len(execution.Outputs) == 0 { // service.go:191 + if execution.Project != project || execution.DatasetGeneration != generation || !execution.State.Successful() || len(execution.Outputs) == 0 { return "service.go:191-193", fmt.Errorf("dataframe release %q is not a published bundle for %s/%s", executionID, project, generation) } - for _, output := range execution.Outputs { // service.go:194-197 + for _, output := range execution.Outputs { if !output.State.Successful() { return "service.go:195-197", fmt.Errorf("dataframe release %q output %q is not published", executionID, output.Name) } } - pointer, err := store.GetPointer(ctx, execution.PointerName()) // service.go:199 - if err != nil { // service.go:200-202 + pointer, err := store.GetPointer(ctx, execution.PointerName()) + if err != nil { return "service.go:200-202", fmt.Errorf("resolve dataframe release pointer: %w", err) } - if pointer.ExecutionID != execution.ID { // service.go:203-204 + if pointer.ExecutionID != execution.ID { return "service.go:203-204", fmt.Errorf("dataframe release %q is no longer active", executionID) } return "service.go:205 (gate passed)", nil } -// --------------------------------------------------------------------------- -// drift-proof source citations: the claim dies if production moves the lines -// --------------------------------------------------------------------------- - var pppRequiredQuotations = []struct { file string needles []string }{ { - file: "internal/dataframe/publication/arango/bundle_registry.go", + file: "internal/backends/arango/internal/publication/bundle_registry.go", needles: []string{ "UPDATE execution WITH @execution IN @@executions", "UPSERT {_key: @pointerKey}", @@ -675,10 +527,6 @@ func pppQuoteVerifiedSource(t *testing.T) string { return builder.String() } -// --------------------------------------------------------------------------- -// the reproduction -// --------------------------------------------------------------------------- - func TestPublishPartialPointerNeverAdvertisedAgainstArango(t *testing.T) { quotations := pppQuoteVerifiedSource(t) t.Logf("OBSERVED STEP 0 quoted verified source (anchor lines located live; drift fails the test):\n%s", quotations) @@ -690,10 +538,6 @@ func TestPublishPartialPointerNeverAdvertisedAgainstArango(t *testing.T) { t.Fatalf("NewBundleStore: %v", err) } - // ========================================================================= - // STEP 1 — CONTROL: production publication.Publish publishes exec CONTROL - // fully through the real ClickHouse client and real Registry. - // ========================================================================= t.Logf("PRE-STATED EXPECT (control): publication.Publish leaves the pointer naming exec CONTROL, state PUBLISHED, and the physical ClickHouse table present") control := pppNewPublication("control", "ppp-control-project", h) controlResult, controlErr := pppProductionPublish(ctx, store, control) @@ -726,10 +570,6 @@ func TestPublishPartialPointerNeverAdvertisedAgainstArango(t *testing.T) { } t.Logf("OBSERVED STEP 1 CONTROL: production publication.Publish published exec %q state=%q pointer->%q physical table %q present with %d rows via real ClickHouse", controlID, controlExec.State, controlPointer.ExecutionID, controlTable, pppTableRows(t, h.ch, ctx, controlTable)) - // ========================================================================= - // STEP 2 — FAULT: the wrapper applies ONLY the exec UPDATE leg and fails - // before the pointer UPSERT leg; a real Commit runs through it. - // ========================================================================= t.Logf("PRE-STATED EXPECT (fault-injection control): Commit returns ErrBundleCommitUncertain (errors.Is) and the tx is closed; the execution remains queryable PUBLISHED (FindExecutionByKey/GetExecution) while GetPointer -> ErrBundleNotFound") faulty := pppNewPublication("fault", "ppp-fault-project", h) h.catalog.armFault() @@ -767,9 +607,6 @@ func TestPublishPartialPointerNeverAdvertisedAgainstArango(t *testing.T) { if !errors.Is(pointerErr, pubpublication.ErrBundleNotFound) { t.Fatalf("fault precondition: pointer under a partial application must be missing; GetPointer returned pointer=%+v err=%v", faultPointer, pointerErr) } - // Behavioral fingerprint of t.closed=true at bundle_store.go:819: the - // runner's Abort path (publication.Publish fail -> tx.Abort) refused to - // delete the staging table and never demoted the row. if !pppTablePresent(t, h.ch, ctx, faultTable) { t.Fatalf("fault precondition: the possibly-published table %q was deleted despite the uncertain-commit closed flag (bundle_store.go:819 comment)", faultTable) } @@ -780,9 +617,6 @@ func TestPublishPartialPointerNeverAdvertisedAgainstArango(t *testing.T) { t.Logf("OBSERVED STEP 2 FAULT: Commit path returned errors.Is(err, ErrBundleCommitUncertain)=true errChain=[%s]; exec UPDATE leg applied=%v pointer UPSERT leg executed=%v; exec %q queryable PUBLISHED via GetExecution and FindExecutionByKey (raw state=%q); GetPointer(%q) -> ErrBundleNotFound; possibly-published table %q preserved (%d rows) -> t.closed=true at bundle_store.go:819 held", errChain(faultErr), calls[1].LegOneApplied, calls[1].LegTwoExecuted, faultID, rawState, faulty.pointer, faultTable, pppTableRows(t, h.ch, ctx, faultTable)) - // ========================================================================= - // STEP 3 — REPAIR ATTEMPT: same production publish entry, same identity. - // ========================================================================= t.Logf("PRE-STATED EXPECT (fault-injection residual property): the identical retry succeeds idempotently (Result from ExistingPublishedOutputs) yet the pointer STILL does not name the fabricated split row; the counterfactual replay-heal (PublishPartialPointer.tla RepairOnIdempotent) is unimplemented, so non-repair is the CURRENT correct behavior for fabricated/legacy rows; the repair-then-verify step lives in publish_atomicity_e2e_test.go") repairResult, repairErr := pppProductionPublish(ctx, store, faulty) if repairErr != nil { @@ -800,9 +634,6 @@ func TestPublishPartialPointerNeverAdvertisedAgainstArango(t *testing.T) { t.Logf("OBSERVED STEP 3 REPAIR: identical retry published.Result={Outputs:%+v} err=%v; PublishExecution invocations still %d (idempotent path never publishes); GetPointer after retry -> pointer=%+v err=%v; repaired=%v", repairResult.Outputs, repairErr, len(calls), pointerAfterRetry, pointerAfterErr, repaired) - // ========================================================================= - // STEP 4 — CONSEQUENCE probes on the orphan (real production functions). - // ========================================================================= selectorExec, selectorOutput, selectorErr := h.registry.FindExecutionBySelector(ctx, faulty.identity.Project, faulty.identity.DatasetGeneration, faulty.selector()) t.Logf("OBSERVED STEP 4a name-latest selector (bundle_registry.go:174-182 has no pointer filter): FindExecutionBySelector=%q state=%q output=%q physicalTable=%q err=%v (matches orphan %v) while GetPointer for the same namespace errors=%v (repaired=%v)", selectorExec.ID, selectorExec.State, selectorOutput.Name, selectorOutput.PhysicalTable, selectorErr, selectorExec.ID == faultID, pointerAfterErr, repaired) @@ -820,9 +651,6 @@ func TestPublishPartialPointerNeverAdvertisedAgainstArango(t *testing.T) { t.Fatalf("STEP 4b broken: the bulk-load gate accepted an execution the pointer never named; branch=%q", orphanBranch) } - // ========================================================================= - // VERDICT - // ========================================================================= if repaired { t.Fatalf("VERDICT CHANGE REQUIRED: the identical production replay repaired the pointer for a fabricated split row — the counterfactual replay-heal (PublishPartialPointer.tla RepairOnIdempotent/SweepPublished) has shipped; re-derive this fault control against the new product promise instead of deleting it") } @@ -836,8 +664,6 @@ func TestPublishPartialPointerNeverAdvertisedAgainstArango(t *testing.T) { t.Logf("FAULT-CONTROL QUOTE RECORD: split legs + replay call log (pointer %q unwritten by replay; executionId=%q):\n%s", faulty.pointer, pointerAfterRetry.ExecutionID, quoting.String()) } -// helpers -------------------------------------------------------------------------------- - func mustGetExecution(t *testing.T, h *pppHarness, ctx context.Context, idOrKey, via string) pubpublication.BundleExecution { t.Helper() var ( diff --git a/internal/dataframe/publication/clickhouse/reconcile_toctou_e2e_test.go b/internal/dataframe/publication/clickhouse/reconcile_toctou_e2e_test.go index ad4144e8f..d0a4618b8 100644 --- a/internal/dataframe/publication/clickhouse/reconcile_toctou_e2e_test.go +++ b/internal/dataframe/publication/clickhouse/reconcile_toctou_e2e_test.go @@ -10,40 +10,17 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" pubpublication "github.com/calypr/loom/internal/dataframe/publication" - pubarango "github.com/calypr/loom/internal/dataframe/publication/arango" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) -// TestReconcileDropsJustPublishedBundleAgainstArango replays finding F-B1 -// end-to-end against a real Arango instance: every guard decision — lease -// takeover (AcquireBundleLease :315), the publish filter set (:287-291), the -// cleanup claim (:414-422), and the fenced save (:44-47) — is made by -// production AQL on the real database. The ClickHouse side stays the -// deterministic effect recorder from reconcile_toctou_test.go: the bug is -// which deletions the reconciler is ALLOWED to reach, decided entirely -// Arango-side, and ClickHouse honors any context it is handed. The wrapper -// only relocates when the racing runner publishes; it decides nothing. -// -// Opt-in like execution_read_pin_integration_test.go; the test owns a UUID -// database and deletes only that database. Run against the owned stack from -// inside the pinned api container, whose network resolves the arangodb -// service: -// -// docker exec -e LOOM_TEST_ARANGO_URL=http://arangodb:8529 \ -// loom-dev-qwen-current-loom-api-1 sh -c \ -// "cd /workspace && go test -count=1 -run TestReconcileDropsJustPublishedBundleAgainstArango -v ./internal/dataframe/publication/clickhouse/" - type fb1Catalog struct { - *pubarango.Registry + *arangofactory.PublishedRegistry race func() raceRan bool } -// ClaimExecutionCleanup injects the racing publish AFTER reconcileExecution's -// GetPointer (:1068) and before the production claim AQL runs — the modelled -// freeze interleaving. Everything else delegates verbatim. func (c *fb1Catalog) ClaimExecutionCleanup(ctx context.Context, executionID, owner string) (bool, error) { if !c.raceRan { c.raceRan = true @@ -51,10 +28,10 @@ func (c *fb1Catalog) ClaimExecutionCleanup(ctx context.Context, executionID, own c.race() } } - return c.Registry.ClaimExecutionCleanup(ctx, executionID, owner) + return c.PublishedRegistry.ClaimExecutionCleanup(ctx, executionID, owner) } -func fb1Setup(t *testing.T) (*pubarango.Registry, publicationSetup) { +func fb1Setup(t *testing.T) (*arangofactory.PublishedRegistry, publicationSetup) { t.Helper() endpoint := strings.TrimSpace(os.Getenv("LOOM_TEST_ARANGO_URL")) if endpoint == "" { @@ -63,10 +40,11 @@ func fb1Setup(t *testing.T) (*pubarango.Registry, publicationSetup) { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) t.Cleanup(cancel) database := "loom_verify_fb1_" + strings.ReplaceAll(uuid.NewString(), "-", "") - client, err := storearango.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatalf("open arango: %v", err) } + client := arangoBackend.Client t.Cleanup(func() { cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second) defer cleanupCancel() @@ -85,10 +63,10 @@ func fb1Setup(t *testing.T) (*pubarango.Registry, publicationSetup) { t.Errorf("drop database %s: status %d", database, response.StatusCode) } }) - if err := client.Bootstrap(ctx, pubarango.BootstrapSpec()); err != nil { + if err := client.Bootstrap(ctx, arangofactory.PublishedBootstrapSpec()); err != nil { t.Fatalf("bootstrap: %v", err) } - registry, err := pubarango.New(client) + registry, err := arangofactory.NewPublishedRegistry(client) if err != nil { t.Fatalf("registry: %v", err) } @@ -117,10 +95,7 @@ func fb1Execution(suffix string) pubpublication.BundleExecution { } } -// fb1Seed stores the stale execution row under a seed-held lease, then frees -// the key: the identical registry state a stalled reconciler faces when it -// arrives to a candidate whose previous holder's lease has lapsed. -func fb1Seed(t *testing.T, registry *pubarango.Registry, ctx context.Context, exec pubpublication.BundleExecution) { +func fb1Seed(t *testing.T, registry *arangofactory.PublishedRegistry, ctx context.Context, exec pubpublication.BundleExecution) { t.Helper() acquired, err := registry.AcquireBundleLease(ctx, exec.Key, "seed", time.Now().UTC().Add(2*time.Minute)) if err != nil || !acquired { @@ -141,7 +116,7 @@ func TestReconcileDropsJustPublishedBundleAgainstArango(t *testing.T) { pointerName := exec.PointerName() runner := "runner-" + uuid.NewString() - wrapper := &fb1Catalog{Registry: registry} + wrapper := &fb1Catalog{PublishedRegistry: registry} wrapper.race = func() { ctx := context.Background() acquired, err := registry.AcquireBundleLease(ctx, exec.Key, runner, time.Now().UTC().Add(2*time.Minute)) @@ -165,8 +140,6 @@ func TestReconcileDropsJustPublishedBundleAgainstArango(t *testing.T) { if err != nil { t.Fatalf("NewBundleStore: %v", err) } - // Stages the DB state a >=2min stall inside reconcileExecution (guard read through claim) and - // the lease-release boundary produces: the lease doc is already expired when the region arms. store.leaseTTL = -time.Second reconcileErr := store.Reconcile(setup.ctx, time.Now().UTC().Add(-time.Minute)) @@ -184,15 +157,6 @@ func TestReconcileDropsJustPublishedBundleAgainstArango(t *testing.T) { if ch.table == "" { t.Errorf("REGRESSION (F-B1, real Arango): reconcileExecution dropped the physical tables of a bundle the pointer advertises (pointer names %q, row PUBLISHED, tables deleted); the fenced SaveExecution error chain was then: %v (errors.Is ErrBundleLeaseLost=%v) — this is the archived CONFIRMED red returning", pointer.ExecutionID, reconcileErr, errors.Is(reconcileErr, pubpublication.ErrBundleLeaseLost)) } else { - // Converted Round 31 (Repair-1b): with the post-claim live pointer - // re-check shipped, the identical pre-fix race (expired lease + - // publish landing after the pre-claim guard) now ends in a VETO — - // reconcile reaches the candidate, claims cleanup, re-reads the - // pointer, finds it names the execution, releases the lease, and - // DropTable is never called. This PASS branch is the anti-regression - // assertion: the table must survive. (The binding veto is proved by - // the rrd/rdl harnesses, which wire the production reader; this one - // exercises the pointer arm and keeps a nil reader.) if reconcileErr != nil { t.Fatalf("prevented drop must reconcile cleanly, got %v", reconcileErr) } @@ -204,10 +168,6 @@ func TestReconcileDropsJustPublishedBundleAgainstArango(t *testing.T) { }) } -// TestReconcileSkipsPointerNamedExecutionAgainstArango is the real-DB control: -// when the pointer already names the execution before GetPointer, the pointer-skip -// skip must prevent deletion. Proves the wrapper injects no artificial -// permission: the same guard that protects the earlier publish is real. func TestReconcileSkipsPointerNamedExecutionAgainstArango(t *testing.T) { registry, setup := fb1Setup(t) exec := fb1Execution(setup.suffix) @@ -216,7 +176,7 @@ func TestReconcileSkipsPointerNamedExecutionAgainstArango(t *testing.T) { t.Fatalf("pre-publish pointer: %v", err) } ch := &tocFakeClickHouse{table: exec.Outputs[0].PhysicalTable} - store, err := NewBundleStore(ch, &fb1Catalog{Registry: registry}) + store, err := NewBundleStore(ch, &fb1Catalog{PublishedRegistry: registry}) if err != nil { t.Fatalf("NewBundleStore: %v", err) } diff --git a/internal/dataframe/publication/clickhouse/reconcile_toctou_test.go b/internal/dataframe/publication/clickhouse/reconcile_toctou_test.go index b4807e037..5499011bc 100644 --- a/internal/dataframe/publication/clickhouse/reconcile_toctou_test.go +++ b/internal/dataframe/publication/clickhouse/reconcile_toctou_test.go @@ -11,21 +11,6 @@ import ( "github.com/calypr/loom/internal/store/clickhouse" ) -// Adversarial verification for reconcileExecution (bundle_store.go:1056-1102). -// Hypothesis: the claim (:1076) and DropTable loop (:1084) never revalidate -// the bundle-key lease or the pointer, and the cleanup-lease renewal tick -// (cleanupLeaseRenewal = 30s, bundle_store.go:238) can never fire inside the -// 10s boundedBundleCleanupContext region. If the reconciler's lease dies -// before the region arms (stall between :1058 and :1066 counts against the -// 2-min lease but not the 10-s context), a runner retrying a retryable FAILED -// execution (:1092) can publish between the fresh GetPointer (:1068) and the -// drop, and the reconciler deletes the just-published tables. The fenced -// SaveExecution (:1097) then fails AFTER the deletion. -// -// leaseTTL = -1s stages the identical DB state a >=leaseTTL stall produces -// (lease doc expired at the moment the reconciler enters its region) without -// modifying production code. - type tocFakeCatalog struct { mu sync.Mutex leases map[string]tocLease @@ -71,7 +56,6 @@ func (f *tocFakeCatalog) state(execID string) *tocExecState { return s } -// AcquireBundleLease mirrors bundle_registry.go:315. func (f *tocFakeCatalog) AcquireBundleLease(_ context.Context, key, owner string, expires time.Time) (bool, error) { f.mu.Lock() defer f.mu.Unlock() @@ -123,7 +107,6 @@ func (f *tocFakeCatalog) CompareAndSwapPointer(_ context.Context, name, expected return nil } -// PublishExecution mirrors the single-AQL guard set bundle_registry.go:287-291. func (f *tocFakeCatalog) PublishExecution(_ context.Context, name, expected string, execution publication.BundleExecution) error { f.mu.Lock() defer f.mu.Unlock() @@ -144,7 +127,6 @@ func (f *tocFakeCatalog) PublishExecution(_ context.Context, name, expected stri return nil } -// SaveExecution mirrors the fenced save bundle_registry.go:26-47. func (f *tocFakeCatalog) SaveExecution(_ context.Context, execution publication.BundleExecution, fenceOwner string) error { f.mu.Lock() defer f.mu.Unlock() @@ -182,7 +164,6 @@ func (f *tocFakeCatalog) ListExecutions(_ context.Context, _ publication.BundleS return nil, nil } -// VisitExecutionPages feeds the stale page for the seeded state only. func (f *tocFakeCatalog) VisitExecutionPages(_ context.Context, state publication.BundleState, _ time.Time, _ int, visit publication.BundleExecutionPageFunc) error { f.mu.Lock() page := make([]publication.BundleExecution, 0, 1) @@ -237,11 +218,6 @@ func (f *tocFakeCatalog) ReleaseExecutionReadPin(_ context.Context, execID, owne return nil } -// ClaimExecutionCleanup mirrors bundle_registry.go:414-422 exactly: no lease, -// state, or pointer check; pins must all be expired; slot free/expired/self. -// The injected race commits before the claim is evaluated, modelling Arango -// ordering where the publish lands after GetPointer (:1068) but before the -// claim query executes. func (f *tocFakeCatalog) ClaimExecutionCleanup(_ context.Context, execID, owner string) (bool, error) { if f.race != nil && !f.raceRan { f.raceRan = true @@ -296,7 +272,6 @@ type tocFakeClickHouse struct { drop func(ctx context.Context) error } -// DropTable mirrors driver context honoring: a dead ctx aborts the drop. func (c *tocFakeClickHouse) DropTable(ctx context.Context, table string) error { if err := ctx.Err(); err != nil { return err @@ -350,16 +325,6 @@ func tocStore(t *testing.T, ch *tocFakeClickHouse, cat *tocFakeCatalog, leaseTTL return s } -// TestReconcileDropsJustPublishedBundleStagesLeaseLoss is the converted F-B1 -// TOCTOU reproduction: lease dead (stall-equivalent) + racing publish that -// lands after the pre-claim GetPointer and before the claim. Pre-Repair-1b -// the reconciler's one-shot guard let the drop through onto that live -// pointer (archived CONFIRMED-BUG red; guardless copy reproduces it verbatim: -// "deleted the physical tables of a bundle the pointer advertises as live"). -// With the post-claim live pointer re-check the drop is denied, so the test -// now PASSES naturally and the former CONFIRMED branch is retained as the -// regression direction: if the table ever disappears under a live pointer, -// this fails again. func TestReconcileDropsJustPublishedBundleStagesLeaseLoss(t *testing.T) { cat := newTOCCatalog() ch := &tocFakeClickHouse{table: tocTable} @@ -408,8 +373,6 @@ func TestReconcileDropsJustPublishedBundleStagesLeaseLoss(t *testing.T) { } } -// TestReconcileSkipsPointerNamedExecution is the control for the :1073 guard: -// publish lands BEFORE GetPointer, reconciler must leave tables alone. func TestReconcileSkipsPointerNamedExecution(t *testing.T) { cat := newTOCCatalog() ch := &tocFakeClickHouse{table: tocTable} @@ -426,8 +389,6 @@ func TestReconcileSkipsPointerNamedExecution(t *testing.T) { } } -// TestReconcileDropsLongStaleUnpublishedExecution is the ordinary-path control: -// with a live lease and no pointer the reconciler must delete and fence-save. func TestReconcileDropsLongStaleUnpublishedExecution(t *testing.T) { cat := newTOCCatalog() ch := &tocFakeClickHouse{table: tocTable} diff --git a/internal/dataframe/publication/clickhouse/release_dangle_reconcile_e2e_test.go b/internal/dataframe/publication/clickhouse/release_dangle_reconcile_e2e_test.go index 9ce52e685..f75b28a00 100644 --- a/internal/dataframe/publication/clickhouse/release_dangle_reconcile_e2e_test.go +++ b/internal/dataframe/publication/clickhouse/release_dangle_reconcile_e2e_test.go @@ -16,119 +16,22 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" pubpublication "github.com/calypr/loom/internal/dataframe/publication" - pubarango "github.com/calypr/loom/internal/dataframe/publication/arango" pubpublished "github.com/calypr/loom/internal/dataframe/published" "github.com/calypr/loom/internal/dataset" - datasetstore "github.com/calypr/loom/internal/dataset/arango" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/calypr/loom/internal/store/clickhouse" "github.com/google/uuid" ) -// rdlDatabasePrefix owns this suite's databases independently of the sibling -// verification workers (ecw_/db_/nlb_/fb1_): residue is greppable as loom_verify_rdl_*. const rdlDatabasePrefix = "loom_verify_rdl_" const ( - // rdlSeedAge is how far in the past every seeded execution's updatedAt is - // parked; Reconcile's enumeration filter is `doc.updatedAt < olderThan` - // (bundle_registry.go:107-112), so anything younger is invisible to it. - rdlSeedAge = 90 * time.Minute - // rdlReconcileOlderThan is the `olderThan` passed to Reconcile. Seed docs - // are rdlSeedAge(90m) old, strictly older than this 60m bound. + rdlSeedAge = 90 * time.Minute rdlReconcileOlderThan = 60 * time.Minute - rdlReconcileMarker = "stale execution reconciled" // bundle_store.go:1091 + rdlReconcileMarker = "stale execution reconciled" ) -// --------------------------------------------------------------------------- -// Candidates this file adjudicates on real Arango + real Go services: -// -// - F-B9: the dataset-release Create/Activate path never takes a reader pin -// on the execution it binds, so the ONLY thing that can shield an -// active-release-bound execution from startup Reconcile is the pointer -// visibility guard at bundle_store.go:1068-1075. (Pre-run audit, reported -// verbatim in the task's STEP 0a: AcquireExecutionReadPin has exactly two -// non-test call sites, read_pins.go:63 and export.go:135, ZERO in -// internal/dataset/**, and there is no internal/explorer/arango/read_pins.go.) -// - F-B1: ClickHouseBundleStore.Reconcile enumerates RUNNING/VALIDATING/FAILED -// executions older than the cutoff (bundle_store.go:1046-1053) and, once it -// claims the key, drops their physical tables and demotes the row to FAILED -// with marker "stale execution reconciled" (:1080-1093). -// - F-B7: a single unresolvable binding force-fails the whole -// published.Reader.CurrentProjectDatasets response (current.go:206-208, -// `return nil, err`), hiding the independently-valid siblings — the -// all-or-nothing discovery-list behaviour the sibling -// dangling_binding_e2e_test.go already source-verifies. -// -// The composite HYPOTHESIS (F-B9 x F-B1 x F-B7): because the release path pins -// nothing (F-B9), a startup Reconcile that races a publish with a stale -// RUNNING enumeration snapshot treats an ACTIVE-release-bound execution as a -// FAILED-candidate (F-B1) and drops/demotes it, and one dangling binding then -// force-fails the current dataset list wholesale (F-B7). -// -// THE LINE THAT BREAKS THE CHAIN: reconcileExecution re-reads the visibility -// pointer AFTER claiming the key, at bundle_store.go:1068, and: -// -// pointer, pointerErr := s.catalog.GetPointer(cleanupCtx, execution.PointerName()) -// ... -// if pointerErr == nil && pointer.ExecutionID == execution.ID { -// return s.catalog.ReleaseBundleLease(cleanupCtx, execution.Key, reconcilerID) -// } -// -// If the pointer names the execution, cleanup is aborted (lease released, no -// DropTable, no SaveExecution demotion). The current-release READ path -// (releaseMaterialization, current.go:94-111) never consults that pointer, so -// the guard is the SOLE defense — and F-B9 tells us a pin is not there. -// -// This test therefore runs ONE controlled experiment with the pointer as the -// single manipulated variable, against real Arango AQL for every guard -// decision and deterministic recorders for the ClickHouse effect: -// -// - PRIMARY (the task's prescribed scenario): the active-release-bound -// execution is created via the real publish flow so its pointer names it -// and its outputs are queryable, then bound to an ACTIVE release via the -// production dataset.ReleaseService Create+Activate over the real -// datasetstore.Store. Inject the stale RUNNING enumeration snapshot, free -// the lease, and run the production Reconcile. Predicted: the pointer guard -// at :1073 skips cleanup -> NOT dangled -> current list stays healthy => -// PREVENTED. (Verdict gate; FAILED-ON-PURPOSE only if it actually dangles.) -// -// - LIVE-MACHINERY CONTROL: a second RUNNING, lease-free, PIN-FREE execution -// that is neither pointer-named nor release-bound sits in the same Reconcile -// sweep. It MUST be reclaimed (table dropped, row FAILED + marker), proving -// enumeration + claim + DropTable + fenced save are genuinely live. Without -// it, the PRIMARY's survival could be dismissed as a no-op reconciler. -// -// - ABSENT-GUARD CONTROL: a THIRD execution is bound to an ACTIVE release -// exactly like the PRIMARY but its pointer was repointed to a shadow -// execution (the F-B6 "release binds an execution the pointer no longer -// names" shape, which F-B7's pointer-ignoring read happily serves). In the -// same sweep the guard's precondition is FALSE, so reconcileDrops it: table -// dropped, row FAILED, and CurrentProjectDatasets for that project fails -// wholesale. This is the F-B9xF-B1xF-B7 chain materialising ONLY when the -// guard's pointer condition is removed — it labels the guard as -// load-bearing, WITHOUT feeding the PRIMARY verdict. -// -// Modeled production interleaving (documented per the task): Reconcile -// enumerates at call time, so a test cannot freeze a mid-Run snapshot without a -// production seam (forbidden here). We instead use the honest equivalent the -// task authorises: park the execution at RUNNING with EXPIRED/FREE lease and -// run Reconcile(olderThan) so it treats the ACTIVE-release-bound execution as a -// FAILED-candidate cleanup, exactly the end-state of the "enumerate-before- -// publish" startup race where the publisher completes PublishExecution+pointer -// (and the release activates) between the reconciler's enumeration pass and its -// per-execution cleanup. Because reconcileExecution reads the pointer at :1068 -// AFTER claiming — not from a frozen snapshot — the guard is evaluated against -// the CURRENT (post-publish, pointer-named) pointer in both the frozen-snapshot -// and live-race interleavings; that is what PREVENTS the chain. See TEST RESULT -// logs for the concrete observation. -// --------------------------------------------------------------------------- - -// rdlClickHouse is the reconciler's deterministic effect recorder. Every drop -// is recorded and a dead context aborts, mirroring BundleClickHouseStore -// (bundle_store.go:53-61) so the observation "did reconcile drop table X" is -// trustworthy. type rdlClickHouse struct { mu sync.Mutex dropped map[string]int @@ -175,27 +78,23 @@ func (c *rdlClickHouse) didDrop(table string) bool { return c.dropped[table] > 0 } -// rdlCatalog wraps the real Registry so the reconciler's guard decisions run on -// REAL Arango AQL while we observe that reconcileExecution really reached the -// candidate (lease claim) and really re-read the pointer (:1068). The wrapper -// decides nothing; every method delegates verbatim to the embedded Registry. type rdlCatalog struct { - *pubarango.Registry + *arangofactory.PublishedRegistry mu sync.Mutex - leaseClaims map[string]string // execution key -> reconciler owner observed - pointerReads map[string]int // pointer name -> times reconcileExecution read it + leaseClaims map[string]string + pointerReads map[string]int } -func newRDLCatalog(registry *pubarango.Registry) *rdlCatalog { +func newRDLCatalog(registry *arangofactory.PublishedRegistry) *rdlCatalog { return &rdlCatalog{ - Registry: registry, - leaseClaims: map[string]string{}, - pointerReads: map[string]int{}, + PublishedRegistry: registry, + leaseClaims: map[string]string{}, + pointerReads: map[string]int{}, } } func (c *rdlCatalog) AcquireBundleLease(ctx context.Context, key, owner string, expires time.Time) (bool, error) { - claimed, err := c.Registry.AcquireBundleLease(ctx, key, owner, expires) + claimed, err := c.PublishedRegistry.AcquireBundleLease(ctx, key, owner, expires) if claimed { c.mu.Lock() c.leaseClaims[key] = owner @@ -208,7 +107,7 @@ func (c *rdlCatalog) GetPointer(ctx context.Context, name string) (pubpublicatio c.mu.Lock() c.pointerReads[name]++ c.mu.Unlock() - return c.Registry.GetPointer(ctx, name) + return c.PublishedRegistry.GetPointer(ctx, name) } func (c *rdlCatalog) claimedOwner(key string) (string, bool) { @@ -224,11 +123,6 @@ func (c *rdlCatalog) pointerReadCount(name string) int { return c.pointerReads[name] } -// rdlVerifier is the publication verifier for the production release path. It -// echoes back a matching-selector verification for ONE selector, so -// dataset.ReleaseService.Create binds the execution we chose — the same shape -// the F-B6 chain artifact (name_latest_binding_e2e_test.go) used, but with a -// controlled verifier instead of a live FindExecutionBySelector. type rdlVerifier struct { want dataset.DataframeSelector execution string @@ -250,11 +144,7 @@ func (v rdlVerifier) VerifyPublication(_ context.Context, _, generation string, }, nil } -// --------------------------------------------------------------------------- -// owned-database harness (LOOM_TEST_ARANGO_URL opt-in, SKIP-clean) -// --------------------------------------------------------------------------- - -func rdlOpen(t *testing.T, budget time.Duration) (context.Context, *storearango.Client, *pubarango.Registry, *datasetstore.Store, string) { +func rdlOpen(t *testing.T, budget time.Duration) (context.Context, *arangofactory.Client, *arangofactory.PublishedRegistry, *arangofactory.DatasetStore, string) { t.Helper() endpoint := strings.TrimSpace(os.Getenv("LOOM_TEST_ARANGO_URL")) if endpoint == "" { @@ -266,10 +156,11 @@ func rdlOpen(t *testing.T, budget time.Duration) (context.Context, *storearango. } ctx, cancel := context.WithTimeout(context.Background(), budget) t.Cleanup(cancel) - client, err := storearango.Open(ctx, endpoint, name) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: name}) if err != nil { t.Fatalf("open %s: %v", name, err) } + client := arangoBackend.Client t.Logf("RDL_DATABASE=%s", name) t.Cleanup(func() { _ = client.Close(context.Background()) @@ -287,34 +178,29 @@ func rdlOpen(t *testing.T, budget time.Duration) (context.Context, *storearango. return } defer resp.Body.Close() - // 403 = already-gone race tolerated, matching the sibling workers. if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusForbidden { t.Errorf("drop database %s: status %d", name, resp.StatusCode) return } t.Logf("RDL_DATABASE_REMOVED=%s", name) }) - if err := client.Bootstrap(ctx, pubarango.BootstrapSpec()); err != nil { + if err := client.Bootstrap(ctx, arangofactory.PublishedBootstrapSpec()); err != nil { t.Fatalf("bootstrap bundle collections: %v", err) } - if err := client.Bootstrap(ctx, datasetstore.BootstrapSpec()); err != nil { + if err := client.Bootstrap(ctx, arangofactory.DatasetBootstrapSpec()); err != nil { t.Fatalf("bootstrap dataset lifecycle collections: %v", err) } - registry, err := pubarango.New(client) + registry, err := arangofactory.NewPublishedRegistry(client) if err != nil { t.Fatalf("new registry: %v", err) } - lifecycle, err := datasetstore.New(client) + lifecycle, err := arangofactory.NewDatasetStore(client) if err != nil { t.Fatalf("new dataset lifecycle store: %v", err) } return ctx, client, registry, lifecycle, name[len(rdlDatabasePrefix):] } -// --------------------------------------------------------------------------- -// seeding through the production registry paths -// --------------------------------------------------------------------------- - type rdlExecSpec struct { id string key string @@ -329,11 +215,6 @@ func rdlSelector(recipe, output string) dataset.DataframeSelector { return dataset.DataframeSelector{Recipe: recipe, TranslationVersion: "v1", Output: output} } -// rdlBuildExecution assembles a RUNNING execution whose single output is fully -// queryable (State PUBLISHED + VerifiedAt + PhysicalTable) so both the release -// read and the reconciler see a valid materialization target. The output -// Selector is set explicitly to the binding selector so -// releaseMaterialization's outputSelector comparison (current.go:103) is exact. func rdlBuildExecution(spec rdlExecSpec, owner string, age time.Time) pubpublication.BundleExecution { stamp := age verified := age.Add(-time.Minute) @@ -361,10 +242,7 @@ func rdlBuildExecution(spec rdlExecSpec, owner string, age time.Time) pubpublica } } -// rdlSeedRunning stores a RUNNING execution under a live seed lease then frees -// the key, so a later reconciler can claim it (the stalled-publisher end state -// reconcileExecution expects). -func rdlSeedRunning(t *testing.T, registry *pubarango.Registry, ctx context.Context, spec rdlExecSpec, owner string) pubpublication.BundleExecution { +func rdlSeedRunning(t *testing.T, registry *arangofactory.PublishedRegistry, ctx context.Context, spec rdlExecSpec, owner string) pubpublication.BundleExecution { t.Helper() if acquired, err := registry.AcquireBundleLease(ctx, spec.key, owner, time.Now().UTC().Add(5*time.Minute)); err != nil || !acquired { t.Fatalf("rdl seed acquire lease %s: acquired=%v err=%v", spec.key, acquired, err) @@ -379,13 +257,7 @@ func rdlSeedRunning(t *testing.T, registry *pubarango.Registry, ctx context.Cont return exec } -// rdlSeedPublished completes the PUBLICISH flow through the production entry -// points: AcquireBundleLease (owner), SaveExecution (RUNNING doc exists so -// PublishExecution's `execution != null` guard holds), then PublishExecution -// which atomically flips the doc to PUBLISHED and creates the visibility -// pointer naming exec in one AQL (bundle_registry.go:274-310), then frees the -// lease so a reconcile can claim the key. Returns the published execution. -func rdlSeedPublished(t *testing.T, registry *pubarango.Registry, ctx context.Context, spec rdlExecSpec, owner string) pubpublication.BundleExecution { +func rdlSeedPublished(t *testing.T, registry *arangofactory.PublishedRegistry, ctx context.Context, spec rdlExecSpec, owner string) pubpublication.BundleExecution { t.Helper() publishedAt := time.Now().UTC().Add(-rdlSeedAge) if acquired, err := registry.AcquireBundleLease(ctx, spec.key, owner, time.Now().UTC().Add(5*time.Minute)); err != nil || !acquired { @@ -411,20 +283,14 @@ func rdlSeedPublished(t *testing.T, registry *pubarango.Registry, ctx context.Co return pub } -// rdlFlipToRunning direct-patches ONLY the top-level `state` and `updatedAt` -// of the execution document (the "stale enumeration snapshot" the reconciler -// is tricked into seeing), leaving outputs, key, identity fields, the pointer -// (separate collection) and any reconcile-written marker untouched. Same -// direct-AQL UPDATE technique the sibling export_pin_crash_window / -// dangling_binding harnesses use to force an otherwise-unreachable state. -func rdlFlipToRunning(t *testing.T, client *storearango.Client, ctx context.Context, execID string) { +func rdlFlipToRunning(t *testing.T, client *arangofactory.Client, ctx context.Context, execID string) { t.Helper() patched := false err := client.QueryRows(ctx, `LET doc = DOCUMENT(@@collection, @key) FILTER doc != null UPDATE doc WITH {state: "RUNNING", updatedAt: @updatedAt} IN @@collection RETURN {patched: true}`, 32, map[string]interface{}{ - "@collection": pubarango.BundleExecutionsCollection, + "@collection": arangofactory.BundleExecutionsCollection, "key": execID, "updatedAt": time.Now().UTC().Add(-rdlSeedAge), }, func(row map[string]any) error { @@ -436,20 +302,14 @@ RETURN {patched: true}`, 32, map[string]interface{}{ } } -// rdlRestorePublished undoes ONLY this harness's own simulated RUNNING flag, -// returning the document to the PUBLISHED materialization it genuinely was -// (reconcile never wrote it, verified by the empty Error marker). This is the -// honest way to re-check the current-release READ result: with the injected -// stale-enumeration flag removed, the release read's health reflects exactly -// what reconcile did (which, under the guard, is nothing). -func rdlRestorePublished(t *testing.T, client *storearango.Client, ctx context.Context, execID string) { +func rdlRestorePublished(t *testing.T, client *arangofactory.Client, ctx context.Context, execID string) { t.Helper() patched := false err := client.QueryRows(ctx, `LET doc = DOCUMENT(@@collection, @key) FILTER doc != null UPDATE doc WITH {state: "PUBLISHED", updatedAt: @updatedAt} IN @@collection RETURN {patched: true}`, 32, map[string]interface{}{ - "@collection": pubarango.BundleExecutionsCollection, + "@collection": arangofactory.BundleExecutionsCollection, "key": execID, "updatedAt": time.Now().UTC().Add(-rdlSeedAge), }, func(row map[string]any) error { @@ -461,24 +321,7 @@ RETURN {patched: true}`, 32, map[string]interface{}{ } } -// --------------------------------------------------------------------------- -// production dataset release Create + Activate over the real datasetstore.Store -// --------------------------------------------------------------------------- - -// rdlCreateAndActivateRelease runs the PRODUCTION release path so the binding -// is written the way the product writes it: persist a STAGED manifest through -// datasetstore.CreateManifest/TransitionManifest, then dataset.ReleaseService -// with Manifests=Releases=lifecycle and a controlled verifier, and finally its -// Activate (Create + the real CompareAndSwapActivateRelease AQL). -// -// Premise STEP 0b is demonstrated here at runtime, not just quoted: the CAS -// Activate writes the release VERBATIM. release_store.go's activateReleaseAQL -// (:152-176) reads only @@lifecycle_collection documents and patches just the -// active-release + active-generation pointers; the release document itself was -// persisted byte-for-byte from Create's publicationBinding by SaveRelease via -// lifecycleDocument (release_store.go:105-115). It never opens the executions -// collection and never looks the execution up. -func rdlCreateAndActivateRelease(t *testing.T, ctx context.Context, lifecycle *datasetstore.Store, project, generation, execID string, selector dataset.DataframeSelector) dataset.ActiveRelease { +func rdlCreateAndActivateRelease(t *testing.T, ctx context.Context, lifecycle *arangofactory.DatasetStore, project, generation, execID string, selector dataset.DataframeSelector) dataset.ActiveRelease { t.Helper() ref, err := dataset.NewRef(project, generation) if err != nil { @@ -518,8 +361,6 @@ func rdlCreateAndActivateRelease(t *testing.T, ctx context.Context, lifecycle *d if err != nil { t.Fatalf("rdl production ReleaseService.Activate (CompareAndSwapActivateRelease): %v", err) } - // Independent round-trip through the production reader: the CAS wrote the - // binding verbatim and activated the pointer. roundTrip, err := lifecycle.ReadActiveRelease(ctx, project) if err != nil { t.Fatalf("rdl ReadActiveRelease after activate: %v", err) @@ -539,19 +380,12 @@ func rdlCreateAndActivateRelease(t *testing.T, ctx context.Context, lifecycle *d return roundTrip } -// --------------------------------------------------------------------------- -// direct AQL fact readers -// --------------------------------------------------------------------------- - -// rdlExecutionRow reads an execution document verbatim (raw, no GetExecution -// canonicalization) so the reconcile-written marker is observed exactly as the -// reconciler wrote it. -func rdlExecutionRow(t *testing.T, client *storearango.Client, ctx context.Context, execID string) (string, string) { +func rdlExecutionRow(t *testing.T, client *arangofactory.Client, ctx context.Context, execID string) (string, string) { t.Helper() state := "" errField := "" err := client.QueryRows(ctx, `RETURN DOCUMENT(@@collection, @key)`, 32, map[string]interface{}{ - "@collection": pubarango.BundleExecutionsCollection, "key": execID, + "@collection": arangofactory.BundleExecutionsCollection, "key": execID, }, func(row map[string]any) error { data, err := json.Marshal(row) if err != nil { @@ -573,11 +407,7 @@ func rdlExecutionRow(t *testing.T, client *storearango.Client, ctx context.Conte return state, errField } -// rdlActivePins reads the ExecutionReadPinsCollection document for an execution -// directly, returning the count of live pins (expiresAt >= now). This is the -// live probe for the F-B9 premise: an active-release-bound execution carries NO -// reader pin, so a pin can never be the reason it survives Reconcile. -func rdlActivePins(t *testing.T, client *storearango.Client, ctx context.Context, execID string) int { +func rdlActivePins(t *testing.T, client *arangofactory.Client, ctx context.Context, execID string) int { t.Helper() sum := sha256.Sum256([]byte("state\x00" + execID)) stateKey := "state_" + hex.EncodeToString(sum[:]) @@ -585,7 +415,7 @@ func rdlActivePins(t *testing.T, client *storearango.Client, ctx context.Context err := client.QueryRows(ctx, `LET state = DOCUMENT(@@collection, @stateKey) LET pins = state == null ? [] : NOT_NULL(state.pins, []) RETURN {active: LENGTH(FOR pin IN pins FILTER pin.expiresAt >= @now RETURN pin), found: state != null}`, 32, map[string]interface{}{ - "@collection": pubarango.ExecutionReadPinsCollection, "stateKey": stateKey, "now": time.Now().UTC(), + "@collection": arangofactory.ExecutionReadPinsCollection, "stateKey": stateKey, "now": time.Now().UTC(), }, func(row map[string]any) error { if a, ok := row["active"].(float64); ok { active = int(a) @@ -601,11 +431,6 @@ RETURN {active: LENGTH(FOR pin IN pins FILTER pin.expiresAt >= @now RETURN pin), return active } -// --------------------------------------------------------------------------- -// guard-source citation: fails the test if the :1068-1075 branch drifts, so the -// PREVENTED verdict can never survive a silent refactor of the guard. -// --------------------------------------------------------------------------- - func rdlRepoRoot(t *testing.T) string { t.Helper() dir, err := os.Getwd() @@ -630,11 +455,6 @@ var rdlGuardRequiredLines = []string{ "return s.catalog.ReleaseBundleLease(cleanupCtx, execution.Key, reconcilerID)", } -// rdlQuoteGuardBranch locates the reconcileExecution pointer-visibility guard in -// the real source, asserts its shape is intact, prints the quoted :1068-1075 -// block, and proves the brief's "skipping bundle with pointer" log line does NOT -// exist (the skip is silent — the guard returns after releasing the lease), so -// the skip is proven behaviorally, never by a log. func rdlQuoteGuardBranch(t *testing.T) string { t.Helper() raw, err := os.ReadFile(filepath.Join(rdlRepoRoot(t), "internal/dataframe/publication/clickhouse/bundle_store.go")) @@ -685,40 +505,6 @@ func rdlErrChain(err error) string { return strings.Join(parts, " <- ") } -// --------------------------------------------------------------------------- -// the composite reproduction -// --------------------------------------------------------------------------- - -// TestReconcileDanglesActiveReleaseBindingAgainstArango is the composite -// F-B9 x F-B1 x F-B7 adjudication. Pre-stated expectations (declared before it -// ever runs): -// -// E1 healthy baseline: CurrentProjectDatasets(active project) == 1 -// materialization {ID=:out-a, PhysicalTable=, State=READY}, -// err == nil. Pins on the active exec == 0 (F-B9 live). -// E2 guard skip (PRIMARY): after the stale-RUNNING flip + free lease + -// production Reconcile, reconcile enumerates+claims the candidate -// (rdlCatalog recorded the Reconcile's AcquireBundleLease and its -// GetPointer of the pointer that names the exec) and skips cleanup: -// active physical table NOT dropped, active row Error != reconcile marker. -// The current-release read, with ONLY the test's own RUNNING flag undone, -// is healthy again (reconcile did not degrade the binding). => PREVENTED. -// E3 live machinery: the unpointered/unbound stale RUNNING control in the SAME -// sweep IS reclaimed (table dropped, row FAILED + marker), so E2's survival -// is attributable to the guard, not to an inert reconciler. -// E4 binding-veto arm: the ACTIVE-release-bound execution whose pointer was -// repointed to a shadow must SURVIVE the same sweep: the post-claim live -// re-read finds no naming pointer but the ACTIVE release binds it, so the -// binding veto (Repair-1b) denies the drop. Pre-Fix this arm dropped the -// table and CurrentProjectDatasets(shadow project) failed wholesale -// (nil + error) — the archived F-B12 red (FAILED-ON-PURPOSE). If the table -// ever disappears under an active binding again, this arm fails as the -// regression direction. The verdict gates on E1-E3 alongside this arm. -// -// Verdict: CONFIRMED iff the PRIMARY active-release-bound execution ends -// dangling (its table dropped OR reconcile wrote the FAILED marker) AND its -// current-list call would fail wholesale; then t.Fatalf FAILED-ON-PURPOSE. -// PREVENTED (the expected outcome): quote :1068-1075, log the guard skip, PASS. func TestReconcileDanglesActiveReleaseBindingAgainstArango(t *testing.T) { quote := rdlQuoteGuardBranch(t) ctx, client, registry, lifecycle, suffix := rdlOpen(t, 150*time.Second) @@ -735,7 +521,6 @@ func TestReconcileDanglesActiveReleaseBindingAgainstArango(t *testing.T) { selectorActive := rdlSelector("rdl-recipe-a", "out-a") selectorShadow := rdlSelector("rdl-recipe-b", "out-b") - // ---- PRIMARY: publish + activate + baseline ------------------------------- pubActive := rdlSeedPublished(t, registry, ctx, specActive, "rdl-pub-active") if got := rdlActivePins(t, client, ctx, pubActive.ID); got != 0 { t.Fatalf("E1/F-B9 precondition: active-release-bound exec %s unexpectedly carries %d live pins at baseline", pubActive.ID, got) @@ -752,10 +537,8 @@ func TestReconcileDanglesActiveReleaseBindingAgainstArango(t *testing.T) { } t.Logf("E1 OBSERVED healthy baseline (pre-fault): CurrentProjectDatasets=%s err=%v; F-B9 live pins on active=%d", rdlMaterializationDigest(list), err, rdlActivePins(t, client, ctx, pubActive.ID)) - // ---- LIVE-MACHINERY CONTROL: stale, not pointer-named, not release-bound --- stale := rdlSeedRunning(t, registry, ctx, specStale, "rdl-seed-stale") - // ---- ABSENT-GUARD CONTROL: publish the shadow, then repoint its pointer away shadow := rdlSeedPublished(t, registry, ctx, specShadow, "rdl-pub-shadow") savedShadow, err := registry.GetExecution(ctx, shadow.ID) if err != nil { @@ -772,34 +555,23 @@ func TestReconcileDanglesActiveReleaseBindingAgainstArango(t *testing.T) { t.Fatalf("ABSENT-GUARD CONTROL baseline broken: un-named active-release-bound exec not listed before fault: list=%s err=%v", rdlMaterializationDigest(shadowBaseline), shadowBaselineErr) } - // ---- force the faults: every candidate looks like a stale RUNNING doc ------- rdlFlipToRunning(t, client, ctx, pubActive.ID) rdlFlipToRunning(t, client, ctx, shadow.ID) - // stale was seeded RUNNING already and its lease was freed on seed. - // ---- one production Reconcile sweep covers all three candidates ---------------- catalog := newRDLCatalog(registry) ch := newRDLClickHouse() store, err := NewBundleStore(ch, catalog) if err != nil { t.Fatalf("NewBundleStore: %v", err) } - // Production wiring mirror (server.go): the reconciler re-checks the - // ACTIVE release bindings immediately before any drop (Repair-1b). The - // PREVENTED verdict gate then holds instead of FAILED-ON-PURPOSE. store.SetActiveReleaseReader(lifecycle) olderThan := time.Now().UTC().Add(-rdlReconcileOlderThan) reconcileErr := store.Reconcile(ctx, olderThan) if reconcileErr != nil { - // The scenario must reconcile cleanly: the guard returns nil for the - // candidate, and the two cleanups fenced-save nil. A non-nil return is a - // real observation, not a pass/fail of the composite. t.Logf("NOTE: production Reconcile returned %v (expected nil for all three candidates)", reconcileErr) } dropped := ch.snapshotDropped() - // read every candidate's post-reconcile doc verbatim (raw: the reconcile - // "stale execution reconciled" marker is only observable un-canonicalized). activeState, activeErr := rdlExecutionRow(t, client, ctx, pubActive.ID) staleState, staleErr := rdlExecutionRow(t, client, ctx, stale.ID) shadowState, shadowErr := rdlExecutionRow(t, client, ctx, shadow.ID) @@ -808,10 +580,7 @@ func TestReconcileDanglesActiveReleaseBindingAgainstArango(t *testing.T) { primaryDemotedByReconcile := activeErr == rdlReconcileMarker || (pubpublication.BundleState(activeState).Canonical() == pubpublication.BundleFailed) - // ---- (1) verdict gate: the PRIMARY composite reproducing means FAILED ------- if primaryTableDropped || primaryDemotedByReconcile { - // Before declaring CONFIRMED, confirm the current-list actually broke, - // i.e. F-B7 is triggered by this dangling. hardList, hardErr := reader.CurrentProjectDatasets(ctx, projectActive) listBrokeHard := hardErr != nil && len(hardList) == 0 if listBrokeHard { @@ -824,7 +593,6 @@ func TestReconcileDanglesActiveReleaseBindingAgainstArango(t *testing.T) { } } - // ---- (2) PREVENTED: guard must have held the PRIMARY ------------------------ if primaryTableDropped { t.Fatalf("REGRESSION: production Reconcile dropped the physical table of a pointer-named, ACTIVE-release-bound execution (dropped=%v)", dropped) } @@ -840,8 +608,6 @@ func TestReconcileDanglesActiveReleaseBindingAgainstArango(t *testing.T) { if pins := rdlActivePins(t, client, ctx, pubActive.ID); pins != 0 { t.Fatalf("F-B9 evidence violated: the active-release-bound exec unexpectedly carries pins post-Reconcile: count=%d", pins) } - // Undo the test's own simulated RUNNING flag; reconcile wrote nothing, so the - // current-release read's health now reflects the reconciler's true non-effect. rdlRestorePublished(t, client, ctx, pubActive.ID) healedList, healedErr := reader.CurrentProjectDatasets(ctx, projectActive) if healedErr != nil || len(healedList) != 1 || healedList[0].ID != pubActive.ID+":out-a" || healedList[0].PhysicalTable != specActive.table { @@ -854,13 +620,11 @@ func TestReconcileDanglesActiveReleaseBindingAgainstArango(t *testing.T) { t.Logf("E2 OBSERVED PREVENTED: reconcile enumerated+claimed (owner=%s) and re-read the pointer %q naming %q %d time(s), and its production guard branch skipped cleanup; active=%q post state=%q reconcileErrorField=%q table=%q dropped=%v; CurrentProjectDatasets post-fault =1=%s err=%v; single=%q err=%v", mustOwner(t, catalog, specActive.key), pubActive.PointerName(), pubActive.ID, catalog.pointerReadCount(pubActive.PointerName()), pubActive.ID, activeState, activeErr, specActive.table, ch.didDrop(specActive.table), rdlMaterializationDigest(healedList), healedErr, singleAfter.ID, singleErr) - // ---- (3) live-machinery control --------------------------------------------- if !ch.didDrop(specStale.table) || staleErr != rdlReconcileMarker || pubpublication.BundleState(staleState).Canonical() != pubpublication.BundleFailed { t.Fatalf("LIVE-MACHINERY control broken: reconcile did not reclaim the unpointered/unbound stale control (staleState=%q staleErr=%q dropped=%v) — the sweep was not genuinely live, so the PREVENTED verdict above would be vacuous", staleState, staleErr, dropped) } t.Logf("E3 OBSERVED live machinery: the unpointered stale RUNNING control %s was reclaimed (state=%q marker=%q tableDropped=%v)", stale.ID, staleState, staleErr, ch.didDrop(specStale.table)) - // ---- (4) binding-veto arm: unpointered ACTIVE-bound exec must survive ---- if ch.didDrop(specShadow.table) || shadowErr == rdlReconcileMarker || pubpublication.BundleState(shadowState).Canonical() != pubpublication.BundleRunning { t.Errorf("REGRESSION (F-B12): the unpointered ACTIVE-release-bound execution %s was reclaimed by Reconcile (state=%q marker=%q tableDropped=%v) — the post-claim active-release binding veto must deny this drop; the ACTIVE release still binds it (archived FAILED-ON-PURPOSE red: this drop destroyed advertised data and failed CurrentProjectDatasets wholesale)", shadow.ID, shadowState, shadowErr, ch.didDrop(specShadow.table)) } diff --git a/internal/dataframe/publication/clickhouse/unpinned_page_read_e2e_test.go b/internal/dataframe/publication/clickhouse/unpinned_page_read_e2e_test.go index afd4969ac..3bb2647b5 100644 --- a/internal/dataframe/publication/clickhouse/unpinned_page_read_e2e_test.go +++ b/internal/dataframe/publication/clickhouse/unpinned_page_read_e2e_test.go @@ -1,95 +1,5 @@ package clickhouse -// F-B18 (adversarial adjudication of the "unpinned GraphQL page read" hypothesis). -// -// This file owns EXACTLY one verification artifact. It never mutates product -// code and never removes a production guard; the honest verdict is asserted -// against unmodified production source read live from disk plus a real Arango + -// real ClickHouse execution driven only through production entry points. -// -// --------------------------------------------------------------------------- -// SOURCE-VERIFIED FACTS (quoted verbatim from the compiled repo at run time by -// the quote gate below; the gate FAILS the test if any anchor drifts, so the -// claim can never survive a silent refactor). -// -// 1. The GraphQL page-read surface resolves the current PUBLISHED execution and -// immediately queries its ClickHouse table with NO execution read pin: -// - internal/api/graphql/graph/dataframe/rows.go:62 resolves via -// `value, access, err := s.currentProjectDataset(...)` and rows.go:67 -// forwards to the reader with `page, err := s.reader.Page(ctx, value, ...)`. -// - internal/dataframe/published/page.go:36 `func (r *Reader) Page(...)` -// builds, at :72, `query := fmt.Sprintf("SELECT %s FROM `%s`", selectColumns, -// materialization.PhysicalTable)` and issues it, at :114, with -// `rows, err := r.ClickHouse.QueryRowsArgs(ctx, query, readColumns, args...)` -// — a bare FROM on the resolved table with no pin acquisition anywhere -// in the resolve->query path. -// Verified by the quote gate's forgotten-symbol scan (uprQuotes entries with -// forgotten: AcquireExecutionReadPin / WithExecutionReadPins) that -// rows.go, page.go and current.go each contain ZERO occurrences of them. -// 2. Resolution is pointer-gated, so a page read can ONLY ever target a -// pointer-named PUBLISHED output: internal/dataframe/published/current.go:117 -// `pointer, err := r.Catalog.GetPointer(ctx, execution.PointerName())` and -// current.go:121 `if pointer.ExecutionID != execution.ID {`. -// 3. The two, and only two, non-test execution read pins are on the EXPORT / -// exact-stream path, not the page-read path: -// - internal/dataframe/published/read_pins.go:63 -// `owned, err := pins.AcquireExecutionReadPin(ctx, id, owner, ...)` -// (inside withExecutionReadPins, which resolves revisions INSIDE visit so -// cleanup cannot drop a table between metadata lookup and pin acquire), and -// - internal/dataframe/published/export.go:135 -// `acquired, err := pins.AcquireExecutionReadPin(ctx, result.ExecutionID, -// result.ReaderID, expires)` (StreamExactExport). -// These are the pinned contrast. (The brief cited internal/dataframe/pinned/ -// executor.go; that package does not exist at this commit — the pins live as -// Reader methods in published/read_pins.go and published/export.go. Reported -// as a correction, not an assumption.) -// 4. The production reconcile sweep cannot reach the page read's target: -// - Reconcile enumerates only RUNNING/VALIDATING/FAILED, EXCLUDING PUBLISHED -// (bundle_store.go:1043). -// - reconcileExecution re-reads the visibility pointer AFTER claiming the -// key and returns after releasing the lease when it still names the -// execution (bundle_store.go:1068-1075); the DropTable is reachable only -// when that guard is blind (`stale execution reconciled` at :1090). -// -// EXPERIMENT (real Arango + real ClickHouse + real Reconcile): -// -// Baseline (PREVENTED expected; FAILED-ON-PURPOSE if a real drop hits the -// page read's resolved-but-not-yet-queried execution with no fault): -// - publish E2 through production publication.Publish against the real -// ClickHouse client (real table, 3 rows, pointer names it, 0 read pins); -// - simulate a live page-read session: resolve via the same production -// current.go pointer check (reader.CurrentProjectDataset), then query via -// reader.Page; -// - run the production Reconcile sweep CONCURRENTLY in the same process with -// a hard synchronization point so the sweep runs BETWEEN resolve and query. -// Companion controls in the same sweep make the result non-vacuous: -// - KEEP: a stale (old updatedAt) RUNNING doc whose pointer still names it -// -> the :1073 guard must skip it (table retained, no marker); -// - DROP: a stale RUNNING doc never named by any pointer -> must be dropped -// + demoted with the reconcile marker (sweep is live). -// Verdict PREVENTED, preventing lines: bundle_store.go:1043 (excludes -// PUBLISHED) and bundle_store.go:1068-1075 (guard, guarded by :1073). -// -// Absent-guard CONTROL ONLY (never feeds the verdict; the F-B12 / release_dangle -// pointer-blindness shape, reproduced at the ClickHouse boundary with a -// page-read victim added): a PUBLISHED pointer-named output's table must be -// reachable ONLY after both baseline protections are removed by fixture data — -// the row flipped RUNNING (raw AQL) AND the pointer repointed away -// (production CompareAndSwapPointer, i.e. the pointer row replaced), so the -// guard's precondition is false while the page read is mid resolve->query. -// If the unpinned page read's query then errors on the dropped table, that -// records the amplifier as a FAULT CONTROL — it does not establish a baseline -// drop and does not make the test fail. -// -// Opt-in (SKIP-clean without the env): -// -// docker exec loom-dev-qwen-current-loom-api-1 sh -c \ -// 'cd /workspace && LOOM_TEST_ARANGO_URL=http://arangodb:8529 \ -// LOOM_CLICKHOUSE_URL=http://clickhouse:8123 LOOM_CLICKHOUSE_DATABASE=loom_dev \ -// LOOM_CLICKHOUSE_USERNAME=loom_dev LOOM_CLICKHOUSE_PASSWORD=loom_dev_password \ -// go test -count=1 -timeout 150s -v -run TestUnpinnedPageRead \ -// ./internal/dataframe/publication/clickhouse/' - import ( "context" "encoding/json" @@ -105,31 +15,21 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" pubpublication "github.com/calypr/loom/internal/dataframe/publication" - pubarango "github.com/calypr/loom/internal/dataframe/publication/arango" dataframepublished "github.com/calypr/loom/internal/dataframe/published" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/calypr/loom/internal/store/clickhouse" "github.com/google/uuid" ) -// uprDatabasePrefix owns this suite's databases; residue is greppable as -// loom_verify_upr_*. uprOwnedTag is embedded in every ClickHouse table this -// file creates (owned canaries, hand-seeded reconciler targets, and the -// production-published execution's output name) so every table name contains -// "upr" and is individually deletable + residue-greppable. const ( uprDatabasePrefix = "loom_verify_upr_" uprOwnedTag = "upr" uprReconcileAge = 90 * time.Minute uprOlderThan = 60 * time.Minute - uprReconcileMark = "stale execution reconciled" // bundle_store.go:1090 + uprReconcileMark = "stale execution reconciled" ) -// --------------------------------------------------------------------------- -// observed bookkeeping -// --------------------------------------------------------------------------- - type uprLog struct { mu sync.Mutex lines []string @@ -150,24 +50,20 @@ func (l *uprLog) emit(t *testing.T) { } } -// uprCatalog observes lease claims / pointer reads / read-pin acquisitions so the -// sweep's decisions (not just its effects) are recorded. It embeds the real -// *Registry and overrides only observers — every decision is still made by real -// Arango AQL in the embedded registry; this wrapper never gates or decides. type uprCatalog struct { - *pubarango.Registry + *arangofactory.PublishedRegistry mu sync.Mutex claims map[string]string pointerReads map[string]int pinsAcquired int } -func newUprCatalog(registry *pubarango.Registry) *uprCatalog { - return &uprCatalog{Registry: registry, claims: map[string]string{}, pointerReads: map[string]int{}} +func newUprCatalog(registry *arangofactory.PublishedRegistry) *uprCatalog { + return &uprCatalog{PublishedRegistry: registry, claims: map[string]string{}, pointerReads: map[string]int{}} } func (c *uprCatalog) AcquireBundleLease(ctx context.Context, key, owner string, expires time.Time) (bool, error) { - claimed, err := c.Registry.AcquireBundleLease(ctx, key, owner, expires) + claimed, err := c.PublishedRegistry.AcquireBundleLease(ctx, key, owner, expires) if claimed { c.mu.Lock() c.claims[key] = owner @@ -180,14 +76,14 @@ func (c *uprCatalog) GetPointer(ctx context.Context, name string) (pubpublicatio c.mu.Lock() c.pointerReads[name]++ c.mu.Unlock() - return c.Registry.GetPointer(ctx, name) + return c.PublishedRegistry.GetPointer(ctx, name) } func (c *uprCatalog) AcquireExecutionReadPin(ctx context.Context, executionID, owner string, expires time.Time) (bool, error) { c.mu.Lock() c.pinsAcquired++ c.mu.Unlock() - return c.Registry.AcquireExecutionReadPin(ctx, executionID, owner, expires) + return c.PublishedRegistry.AcquireExecutionReadPin(ctx, executionID, owner, expires) } func (c *uprCatalog) claimOwner(key string) (string, bool) { @@ -209,15 +105,11 @@ func (c *uprCatalog) pinAcquireCount() int { return c.pinsAcquired } -// --------------------------------------------------------------------------- -// harness (owned database + owned ClickHouse tables; SKIP-clean opt-in) -// --------------------------------------------------------------------------- - type uprHarness struct { ctx context.Context endpoint string - client *storearango.Client - registry *pubarango.Registry + client *arangofactory.Client + registry *arangofactory.PublishedRegistry ch *clickhouse.Client database string suffix string @@ -251,22 +143,21 @@ func uprOpen(t *testing.T) *uprHarness { } ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) t.Cleanup(cancel) - client, err := storearango.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatalf("open %s: %v", database, err) } + client := arangoBackend.Client t.Logf("UPR_DATABASE=%s", database) - // Registered FIRST so it runs LAST: ClickHouse artifacts must be dropped and - // asserted empty before the Arango database disappears (LIFO cleanup order). t.Cleanup(func() { _ = client.Close(context.Background()) uprDropDatabase(t, endpoint, database) }) - registry, err := pubarango.New(client) + registry, err := arangofactory.NewPublishedRegistry(client) if err != nil { t.Fatalf("new registry: %v", err) } - if err := client.Bootstrap(ctx, pubarango.BootstrapSpec()); err != nil { + if err := client.Bootstrap(ctx, arangofactory.PublishedBootstrapSpec()); err != nil { t.Fatalf("bootstrap bundle collections: %v", err) } ch, err := clickhouse.New(clickhouse.Options{ @@ -296,8 +187,6 @@ func uprOpen(t *testing.T) *uprHarness { return harness } -// uprOwnedPrefix returns a globally-unique ClickHouse table prefix that contains -// the uprOwnedTag marker, e.g. "loom_bundle_upr5f3a..._". func (h *uprHarness) uprOwnedPrefix(label string) string { return "loom_bundle_" + uprOwnedTag + h.suffix + "_" + label } @@ -331,7 +220,6 @@ func (h *uprHarness) seedPublishedOutputRecord(name, table, project, generation, } } -// tablePresent reports whether a ClickHouse table exists via system.tables. func (h *uprHarness) tablePresent(ctx context.Context, table string) bool { rows, err := h.ch.QueryRowsArgs(ctx, "SELECT count() AS c FROM system.tables WHERE database = currentDatabase() AND name = ?", []string{"c"}, table) if err != nil || len(rows) != 1 { @@ -353,8 +241,6 @@ func (h *uprHarness) drainTablesAndClose(t *testing.T) { t.Errorf("UPR CH drop %s: %v", table, err) } } - // Residue gate scoped to THIS file's unique prefix so it cannot collide with - // the sibling workers sharing loom_dev. tagged := "%" + uprOwnedTag + h.suffix + "%" names, err := h.ch.QueryRowsArgs(ctx, fmt.Sprintf("SELECT name FROM system.tables WHERE database = currentDatabase() AND name LIKE '%s' ORDER BY name", tagged), []string{"name"}) if err != nil { @@ -397,10 +283,6 @@ func uprDropDatabase(t *testing.T, endpoint, database string) { t.Logf("UPR_DATABASE_REMOVED=%s", database) } -// --------------------------------------------------------------------------- -// direct-AQL fact readers / state fixture helpers -// --------------------------------------------------------------------------- - func uprCount(value any) (int64, bool) { switch typed := value.(type) { case uint64: @@ -419,11 +301,9 @@ func uprCount(value any) (int64, bool) { return 0, false } -// uprRawExecutionRow reads the doc verbatim (no GetExecution canonicalization) to -// observe the raw reconcile-written `state` and `error` marker. func (h *uprHarness) rawExecutionRow(ctx context.Context, execID string) (state, errField string) { err := h.client.QueryRows(ctx, `RETURN DOCUMENT(@@collection, @key)`, 32, map[string]interface{}{ - "@collection": pubarango.BundleExecutionsCollection, "key": execID, + "@collection": arangofactory.BundleExecutionsCollection, "key": execID, }, func(row map[string]any) error { data, err := json.Marshal(row) if err != nil { @@ -445,12 +325,6 @@ func (h *uprHarness) rawExecutionRow(ctx context.Context, execID string) (state, return state, errField } -// flipToRunningStale parks an execution at RUNNING with an old updatedAt, the -// end state of a stalled publisher that reconcile is meant to reclaim. It is the -// identical direct-data technique release_dangle's rdlFlipToRunning / -// dangling_binding / export_pin_crash_window harnesses use (a data-plane fixture -// for an otherwise-unreachable state; it does not touch product code or any -// guard). func (h *uprHarness) flipToRunningStale(t *testing.T, ctx context.Context, execID string) { t.Helper() patched := false @@ -458,7 +332,7 @@ func (h *uprHarness) flipToRunningStale(t *testing.T, ctx context.Context, execI FILTER doc != null UPDATE doc WITH {state: "RUNNING", updatedAt: @updatedAt} IN @@collection RETURN {patched: true}`, 32, map[string]interface{}{ - "@collection": pubarango.BundleExecutionsCollection, + "@collection": arangofactory.BundleExecutionsCollection, "key": execID, "updatedAt": time.Now().UTC().Add(-uprReconcileAge), }, func(row map[string]any) error { @@ -470,10 +344,6 @@ RETURN {patched: true}`, 32, map[string]interface{}{ } } -// --------------------------------------------------------------------------- -// source-quote gate (drift-proof) -// --------------------------------------------------------------------------- - func uprRepoRoot(t *testing.T) string { t.Helper() dir, err := os.Getwd() @@ -495,7 +365,7 @@ func uprRepoRoot(t *testing.T) string { type uprQuote struct { file string needles []string - forgotten []string // substrings that must NOT appear (pin never added to this path) + forgotten []string anchorDesc string } @@ -508,7 +378,6 @@ var uprQuotes = []uprQuote{ "page, err := s.reader.Page(ctx, value, dfmaterialization.PageRequest", "s.logReadFailure(ctx, \"clickhouse_rows\"", }, - // No read pin may ever be acquired on the page-read surface. forgotten: []string{"AcquireExecutionReadPin", "WithExecutionReadPins"}, }, { @@ -591,10 +460,6 @@ func uprQuoteAndVerifySource(t *testing.T) string { return builder.String() } -// --------------------------------------------------------------------------- -// page-read session driven through the same primitives rows.go uses -// --------------------------------------------------------------------------- - type uprReader interface { CurrentProjectDataset(ctx context.Context, project string, selector dataframepublished.DataframeSelector) (dataframepublished.Materialization, error) CurrentProjectDatasets(ctx context.Context, project string) ([]dataframepublished.Materialization, error) @@ -627,15 +492,6 @@ func uprMaterializationDigest(value dataframepublished.Materialization) string { return fmt.Sprintf("{ID=%q Name=%q PhysicalTable=%q State=%q}", value.ID, value.Name, value.PhysicalTable, value.State) } -// --------------------------------------------------------------------------- -// 1) BASELINE: unpinned GraphQL page read vs concurrent production reconcile. -// Pre-stated expectation: the page read is PREVENTED from being a victim, -// because reconcile excludes PUBLISHED (:1043) and skips pointer-named rows -// (:1068-1075), and the page read can only resolve pointer-named PUBLISHED -// rows (current.go:117/121). FAILED-ON-PURPOSE only if a real drop hits the -// page read's resolved-but-not-yet-queried execution with no fault. -// --------------------------------------------------------------------------- - func TestUnpinnedPageReadBaselineAgainstArango(t *testing.T) { quote := uprQuoteAndVerifySource(t) t.Logf("OBSERVED STEP 0 quoted verified source (anchors located live; drift fails the test):\n%s", quote) @@ -643,8 +499,6 @@ func TestUnpinnedPageReadBaselineAgainstArango(t *testing.T) { h := uprOpen(t) ctx := h.ctx - // Two stores: one wraps the catalog for observation (the sweep is driven - // through it), one is plain (the page-read resolution store mirrors rows.go). observeCatalog := newUprCatalog(h.registry) sweepStore, err := NewBundleStore(h.ch, observeCatalog) if err != nil { @@ -655,13 +509,6 @@ func TestUnpinnedPageReadBaselineAgainstArango(t *testing.T) { t.Fatalf("NewBundleStore(publish): %v", err) } - // ========================================================================= - // STEP 1 — publish E2 (the page-read victim) through production.Publish. - // Real ClickHouse client creates a real table with 3 rows; PublishExecution - // names the pointer; the bundle lease is then released (frees the key), which - // is the ONLY extra step vs the production commit path and merely drops a - // completed publisher's lease. 0 execution read pins are present anywhere. - // ========================================================================= e2Project := "upr.e2.project" e2Recipe := "upr.e2.recipe" e2Output := uprOwnedTag + h.suffix + "_out" @@ -706,8 +553,6 @@ func TestUnpinnedPageReadBaselineAgainstArango(t *testing.T) { } e2Key := e2Bundle.Key() e2Pointer := e2Bundle.PointerName() - // Free the completed publisher's lease so the sweep COULD claim the key if it - // were ever eligible (proves survival is the guard, not a held lease). _ = h.registry.ReleaseBundleLease(ctx, e2Key, "upr-e2-seed") _ = h.registry.ReleaseBundleLease(ctx, e2Key, "owner") @@ -731,8 +576,6 @@ func TestUnpinnedPageReadBaselineAgainstArango(t *testing.T) { } _ = h.registry.ReleaseBundleLease(ctx, e2Key, "upr-precheck") - // Resolution must use the production current.go pointer check; assert - // pointer-named, state PUBLISHED, queryable, matching selector. selector := dataframepublished.DataframeSelector{Recipe: e2Recipe, TranslationVersion: "v1", Output: e2Output} reader := uprNewReader(h) baselineMat, err := reader.CurrentProjectDataset(ctx, e2Project, selector) @@ -746,11 +589,6 @@ func TestUnpinnedPageReadBaselineAgainstArango(t *testing.T) { h.log.addf("STEP 1 E2: production.Publish created PUBLISHED exec %q (table %s, 3 real ClickHouse rows), pointer %q names it, key lease is FREE, reader.CurrentProjectDataset resolves it via the current.go:117/121 pointer check and Reader.Page returns 3 rows, all with 0 execution read pins — pinsAcquired=%d", e2ID, e2Table, e2Pointer, observeCatalog.pinAcquireCount()) - // ========================================================================= - // STEP 2 — companion controls so the sweep is demonstrably LIVE: - // KEEP: stale RUNNING doc whose pointer still names it -> guard skips it; - // DROP: stale RUNNING doc never pointer-named -> reclaimed. - // ========================================================================= keepPrefix := h.uprOwnedPrefix("keep") keepTable := keepPrefix + "t" if err := h.seedTable(ctx, keepTable, 3); err != nil { @@ -779,13 +617,6 @@ func TestUnpinnedPageReadBaselineAgainstArango(t *testing.T) { h.log.addf("STEP 2 controls: KEEP=%s (state RUNNING stale via fixture, pointer names it), DROP=%s (stale RUNNING, never pointer-named); both are stale-enumeration snapshots reconcile enumerates", keepExec.ID, dropExec.ID) - // ========================================================================= - // STEP 3 — CONCURRENT page read vs production Reconcile, with a hard - // synchronization point so the sweep runs strictly BETWEEN the page read's - // resolve (reader.CurrentProjectDataset) and its query (reader.Page). - // (Controlled interleaving — labeled scope limit: this is a controlled - // interleaving, not a real scheduler race.) - // ========================================================================= type pageOutcome struct { phase string table string @@ -806,7 +637,6 @@ func TestUnpinnedPageReadBaselineAgainstArango(t *testing.T) { resolvedMat, resolveErr = reader.CurrentProjectDataset(ctx, e2Project, selector) close(resolveDone) if resolveErr != nil { - // Never block the sweep if resolution itself failed. close(sweepDone) race.phase = "resolve" race.err = resolveErr @@ -828,9 +658,6 @@ func TestUnpinnedPageReadBaselineAgainstArango(t *testing.T) { }() wg.Wait() - // ========================================================================= - // STEP 4 — classify. - // ========================================================================= e2StillPresent := h.tablePresent(ctx, e2Table) keepStillPresent := h.tablePresent(ctx, keepTable) dropDropped := !h.tablePresent(ctx, dropTable) @@ -849,7 +676,6 @@ func TestUnpinnedPageReadBaselineAgainstArango(t *testing.T) { race.table, race.phase, race.rows, uprErrChain(race.err), observeCatalog.pinAcquireCount()) h.log.emit(t) - // Non-vacuity: the sweep must be genuinely live (DROP reclaimed and marked). if !dropDropped { t.Fatalf("INCONCLUSIVE: production Reconcile did not drop the never-pointer-named stale control %q (state=%q marker=%q) — the sweep may have been a no-op, which would make any E2 survival vacuous", dropExec.ID, dropState, dropErrField) } @@ -860,8 +686,6 @@ func TestUnpinnedPageReadBaselineAgainstArango(t *testing.T) { t.Fatalf("INCONCLUSIVE: reconcile never claimed the stale control under a reconciler ID (owner=%q claimed=%v) — the sweep may not have reached cleanup", dropOwner, dropClaimed) } - // KEEP must not be dropped; the guard skips it. Attributed live because the - // concurrent DROP in the SAME sweep was reclaimed. if !keepStillPresent { t.Fatalf("REGRESSION: the pointer guard did not hold — a stale RUNNING execution still named by its pointer (%s) was dropped by reconcile", keepExec.ID) } @@ -875,9 +699,6 @@ func TestUnpinnedPageReadBaselineAgainstArango(t *testing.T) { t.Fatalf("guard-skip not proven: reconcile never reached GetPointer on the KEEP pointer %q (reads=%d)", keepExec.PointerName(), reads) } - // PRIMARY page-read verdict gate: FAILED-ON-PURPOSE only if a real drop hit - // the resolved-but-not-yet-queried page read with no product change / guard - // removal. if !e2StillPresent || race.err != nil { _, hardErr := reader.Page(ctx, resolvedMat, uprPageRequest()) _, e2Marker := h.rawExecutionRow(ctx, e2ID) @@ -887,7 +708,6 @@ func TestUnpinnedPageReadBaselineAgainstArango(t *testing.T) { if race.phase != "query" || race.rows != 3 { t.Fatalf("baseline page read did not complete the resolve->query cycle: phase=%s rows=%d err=%s", race.phase, race.rows, uprErrChain(race.err)) } - // The sweep must never even have reached the PUBLISHED page-read target. if e2Claimed { t.Fatalf("attribution check: reconcile claimed the PUBLISHED E2 key (owner=%q) despite the :1043 exclusion — investigate why PUBLISHED was enumerated", e2Owner) } @@ -900,17 +720,6 @@ func TestUnpinnedPageReadBaselineAgainstArango(t *testing.T) { dropExec.ID, e2ID, e2StillPresent, race.rows, keepExec.ID, preventing) } -// --------------------------------------------------------------------------- -// 2) ABSENT-GUARD CONTROL ONLY (never feeds the verdict): reproduce the F-B12 / -// release_dangle pointer-blindness shape at the ClickHouse boundary with an -// unpinned page-read victim added. The drop must be reachable ONLY after BOTH -// baseline protections are removed by fixture data (row flipped RUNNING via -// raw AQL AND the pointer row replaced via production CompareAndSwapPointer), -// which makes the :1073 guard's precondition false while the page read is mid -// resolve->query. If the unpinned query then errors on the dropped table, that -// records the amplifier as a fault control; it does NOT fail the test. -// --------------------------------------------------------------------------- - func TestUnpinnedPageReadAbsentGuardFaultControlAgainstArango(t *testing.T) { quote := uprQuoteAndVerifySource(t) t.Logf("OBSERVED STEP 0 quoted verified source (anchors located live; drift fails the test):\n%s", quote) @@ -928,7 +737,6 @@ func TestUnpinnedPageReadAbsentGuardFaultControlAgainstArango(t *testing.T) { t.Fatalf("NewBundleStore(publish): %v", err) } - // Publish the victim through production so its table is a real product table. proj := "upr.fault.project" recipe := "upr.fault.recipe" output := uprOwnedTag + h.suffix + "_fault_out" @@ -967,7 +775,6 @@ func TestUnpinnedPageReadAbsentGuardFaultControlAgainstArango(t *testing.T) { } key := bundle.Key() pointerName := bundle.PointerName() - // Free the completed publisher's lease so a (blind) sweep could claim it. _ = h.registry.ReleaseBundleLease(ctx, key, "upr-fault-seed") _ = h.registry.ReleaseBundleLease(ctx, key, "owner") @@ -984,8 +791,6 @@ func TestUnpinnedPageReadAbsentGuardFaultControlAgainstArango(t *testing.T) { h.log.addf("CONTROL setup: PUBLISHED pointer-named victim %q (table %s) resolves and pages 3 rows via current.go pointer check, 0 read pins (pinsAcquired=%d)", victimID, table, observeCatalog.pinAcquireCount()) - // Resolve -> [sweep] -> query, with the absent-guard fault injected by the - // sweep side between resolve and query. resolveDone := make(chan struct{}) sweepDone := make(chan struct{}) var ( @@ -1017,9 +822,6 @@ func TestUnpinnedPageReadAbsentGuardFaultControlAgainstArango(t *testing.T) { go func() { defer wg.Done() <-resolveDone - // ABSENT-GUARD FAULT (fixture data only; no product-code change): - // (a) remove the PUBLISHED-state protection, (b) replace the pointer row - // (F-B12 / release_dangle) so the :1073 guard's precondition is false. h.flipToRunningStale(t, ctx, victimID) if csErr := h.registry.CompareAndSwapPointer(ctx, pointerName, victimID, uprOwnedTag+"-shadow-"+uuid.NewString()[:8]); csErr != nil { h.log.addf("CONTROL repoint pointer %q away from victim failed: %v", pointerName, uprErrChain(csErr)) @@ -1035,9 +837,6 @@ func TestUnpinnedPageReadAbsentGuardFaultControlAgainstArango(t *testing.T) { h.log.addf("CONTROL post-sweep: victim table %s present=%v (raw state=%q marker=%q), key claimed=%v by=%q; sweep err=%v; page read phase=%s rows=%d queryErr=%s; pinsAcquired=%d", table, stillPresent, state, errField, claimed, owner, uprErrChain(sweepErr), racePhase, raceRows, uprErrChain(queryErr), observeCatalog.pinAcquireCount()) - // Preconditions of the fault being truly "absent guard" (not the baseline - // protections standing): sweep must have claimed the key and re-read (and - // then ignored) the pointer because it no longer named the victim. if !claimed || !strings.HasPrefix(owner, "reconciler-") { t.Errorf("fault-control precondition unmet: sweep never claimed the victim key under a reconciler ID (owner=%q claimed=%v)", owner, claimed) } @@ -1057,10 +856,6 @@ func TestUnpinnedPageReadAbsentGuardFaultControlAgainstArango(t *testing.T) { victimID, table, !stillPresent, uprErrChain(queryErr)) } -// --------------------------------------------------------------------------- -// shared seeded-execution helpers (RUNNING docs the sweep claims) -// --------------------------------------------------------------------------- - type uprExecSpec struct { id string project string @@ -1100,10 +895,6 @@ func (h *uprHarness) buildRunningExec(spec uprExecSpec, owner, table string) pub } } -// seedRunningWithPointer: RUNNING doc (old updatedAt) under a live seed lease, -// pointer created naming it via production CompareAndSwapPointer, then lease -// freed. State stays RUNNING so reconcile enumerates it; the LIVE pointer must -// make the :1073 guard skip it. func (h *uprHarness) seedRunningWithPointer(t *testing.T, ctx context.Context, spec uprExecSpec, table string) pubpublication.BundleExecution { t.Helper() owner := "upr-seed-" + uuid.NewString()[:8] @@ -1123,8 +914,6 @@ func (h *uprHarness) seedRunningWithPointer(t *testing.T, ctx context.Context, s return exec } -// seedRunningNoPointer: RUNNING doc (old updatedAt) never named by any pointer, -// lease freed. Reconcile must reclaim it (live-machinery control). func (h *uprHarness) seedRunningNoPointer(t *testing.T, ctx context.Context, spec uprExecSpec, table string) pubpublication.BundleExecution { t.Helper() owner := "upr-seed-" + uuid.NewString()[:8] @@ -1141,12 +930,6 @@ func (h *uprHarness) seedRunningNoPointer(t *testing.T, ctx context.Context, spe return exec } -// --------------------------------------------------------------------------- -// misc helpers -// --------------------------------------------------------------------------- - -// mustE2ID resolves the execution ID for a bundle Key via production -// FindExecutionByKey (the store generates an opaque ID we cannot predict). func mustE2ID(t *testing.T, h *uprHarness, key string) string { t.Helper() exec, err := h.registry.FindExecutionByKey(h.ctx, key) diff --git a/internal/dataframe/publication/flat_columns.go b/internal/dataframe/publication/flat_columns.go index 5fa0b0579..5114bfdd4 100644 --- a/internal/dataframe/publication/flat_columns.go +++ b/internal/dataframe/publication/flat_columns.go @@ -6,9 +6,6 @@ import ( "unicode" ) -// FlatColumnName is the stable column contract for Explorer-facing dataframe -// indexes. Root fields carry the root resource/table prefix; traversal fields -// already carry their alias path and remain unchanged. func FlatColumnName(resourceType, name string) string { if name == "" || name == "auth_resource_path" || name == "project_id" || name == "_key" || strings.HasPrefix(name, "__loom_") || strings.Contains(name, "__") { return name @@ -31,8 +28,6 @@ func flatResourcePrefix(resourceType string) string { return builder.String() } -// QualifyFlatRow applies the same naming contract as FlatColumnName and -// rejects source fields that would collapse onto one published column. func QualifyFlatRow(resourceType string, row map[string]any) (map[string]any, error) { if row == nil { return nil, nil diff --git a/internal/dataframe/publication/pointer_name_encoding_test.go b/internal/dataframe/publication/pointer_name_encoding_test.go index c1ecc92eb..ff266b481 100644 --- a/internal/dataframe/publication/pointer_name_encoding_test.go +++ b/internal/dataframe/publication/pointer_name_encoding_test.go @@ -5,10 +5,6 @@ import ( "testing" ) -// historicalBareJoin replays the pre-repair PointerName EXACTLY as shipped: -// conditional optional components joined with a bare "\x00". It pins the -// fault premise (the archived CONFIRMED pairs fold HERE) while production -// PointerName must separate them (Repair-5). func historicalBareJoin(i BundleIdentity) string { parts := []string{i.Project, i.DatasetGeneration, i.Name} if strings.TrimSpace(i.TranslationVersion) != "" { @@ -40,9 +36,6 @@ func TestPointerNameLengthPrefixDefeatsNulFold(t *testing.T) { t.Logf("FB21 control: historical bare join folded both to %q; production PointerName now separates %q vs %q", historicalBareJoin(nulA), nulA.PointerName(), foldB.PointerName()) } -// TestPointerNameIsInjectiveOverDistinctComponents drives the archived -// fold-class pairs plus delimiter-spoof neighbours: every DISTINCT -// (component, value) assignment must yield a distinct pointer name. func TestPointerNameIsInjectiveOverDistinctComponents(t *testing.T) { seen := map[string]string{} entries := []struct { @@ -67,9 +60,6 @@ func TestPointerNameIsInjectiveOverDistinctComponents(t *testing.T) { } } -// TestPointerNameOptionalComponentSemanticsAreShipPreserved pins the shipped -// exclusion behaviour (whitespace-only == absent) that the Repair-5 encoder -// must not alter. func TestPointerNameOptionalComponentSemanticsAreShipPreserved(t *testing.T) { base := BundleIdentity{Project: "proj", DatasetGeneration: "gen", Name: "orders"} withTV := func(tv, out string) BundleIdentity { @@ -87,19 +77,6 @@ func TestPointerNameOptionalComponentSemanticsAreShipPreserved(t *testing.T) { if withTV("v1 ", "main").PointerName() == withTV("v1", "main").PointerName() { t.Fatalf("trailing-space TranslationVersion was collapsed; the shipped encoding keeps the raw value inside a live component") } - // Documented boundary equality (NOT a fold-class defect): blank-valued - // optional components are EXCLUDED, so (TV="x", Out blank) and - // (TV blank, Out="x") occupy one 4-component namespace, matching the - // pre-repair bare join. That boundary cannot steal a pointer: a document - // publish REQUIRES a non-blank TranslationVersion - // (internal/dataframe/recipe/document_validation.go:39), so the blank-TV / - // Output-only twin of a blank-Output identity is never publishable; a - // blank-Output identity only arises from a multi-output publication — - // incrementalPublicationOutput returns "" unless there is exactly one - // binding and one stream - // (internal/server/recipe_publication.go:214-218) — and its 4 components - // never collide with a 5-component single-output name; Key() still - // distinguishes the executions themselves. left := withTV("x", "") right := withTV("", "x") if left.PointerName() != right.PointerName() || historicalBareJoin(left) != historicalBareJoin(right) { diff --git a/internal/dataframe/publication/quality.go b/internal/dataframe/publication/quality.go index 0eed0723d..6e21235fb 100644 --- a/internal/dataframe/publication/quality.go +++ b/internal/dataframe/publication/quality.go @@ -34,9 +34,6 @@ const ( QualityFailed QualityVerdict = "FAILED" ) -// QualityPolicy bounds the state retained while the publication stream is -// checked. An empty Version disables reporting for compatibility callers; -// Explorer publication always supplies an explicit versioned policy. type QualityPolicy struct { Version string MaxRows int64 @@ -63,10 +60,6 @@ type ColumnQuality struct { const relatedSourceAllMatchesBasis = "ALL_MATCHES_NO_FILTER_OR_WINDOW" -// RelatedSourcePopulation records exact row and list-entry counts for one -// compiler-identified RELATED_SOURCE column. Null entries may also represent -// an absent source field because the published list does not retain that -// distinction. type RelatedSourcePopulation struct { Basis string `json:"basis"` OutputRows int64 `json:"outputRows"` @@ -78,17 +71,11 @@ type RelatedSourcePopulation struct { UnknownRows int64 `json:"unknownRows"` } -// QualityLimits records the exact bounds under which the full-population -// report was produced. A complete report is meaningful only together with -// the limits that could have stopped its scan. type QualityLimits struct { MaxRows int64 `json:"maxRows"` MaxDistinctKeys int64 `json:"maxDistinctKeys"` } -// QualityIssues counts data-dependent failures enforced by the same stream -// that materializes the candidate publication. Successful reports therefore -// contain explicit zeroes instead of leaving these checks implicit. type QualityIssues struct { Ambiguous int64 `json:"ambiguous"` InvalidType int64 `json:"invalidType"` @@ -101,9 +88,6 @@ type KeyIntegrity struct { Duplicate int64 `json:"duplicate"` } -// QualityReport is immutable evidence about one fully consumed output stream. -// Its identity is derived only after the receipt already exists, so it can -// never participate in receipt identity. type QualityReport struct { ID string `json:"id"` ReceiptID string `json:"receiptId"` @@ -132,8 +116,6 @@ type QualityFailedError struct{ Reports []QualityReport } func (e *QualityFailedError) Error() string { return ErrQualityFailed.Error() } func (e *QualityFailedError) Unwrap() error { return ErrQualityFailed } -// CloneQualityReports prevents callers from mutating evidence held by a -// durable publication execution. func CloneQualityReports(reports []QualityReport) []QualityReport { cloned := make([]QualityReport, len(reports)) for index, report := range reports { @@ -150,8 +132,6 @@ func CloneQualityReports(reports []QualityReport) []QualityReport { return cloned } -// ValidateQualityReports enforces the activation-grade relationship between -// evidence and the durable publication it describes. func ValidateQualityReports(identity BundleIdentity, outputs []BundleOutputRecord, reports []QualityReport) error { if err := ValidateQualityReportBindings(identity, outputs, reports); err != nil { return err @@ -170,10 +150,6 @@ func ValidateQualityReports(identity BundleIdentity, outputs []BundleOutputRecor return nil } -// ValidateQualityReportBindings permits partial or failed evidence to be -// retained on a failed candidate execution while still proving that every -// report belongs to that exact publication attempt. Activation uses the -// stricter ValidateQualityReports contract above. func ValidateQualityReportBindings(identity BundleIdentity, outputs []BundleOutputRecord, reports []QualityReport) error { known := make(map[string]struct{}, len(outputs)) for _, output := range outputs { @@ -244,9 +220,6 @@ func newQualityAccumulator(identity PublicationIdentity, output OutputStream, po return accumulator } -// fail finalizes evidence for a stream that did not reach natural exhaustion. -// Known semantic failures are complete negative findings; cancellation and -// infrastructure failures are incomplete because absence was not proven. func (a *qualityAccumulator) fail(cause error) (QualityReport, error) { if a == nil { return QualityReport{}, nil diff --git a/internal/dataframe/publication/row_smuggling_prevention_test.go b/internal/dataframe/publication/row_smuggling_prevention_test.go index a62165e4d..2c5ae6ce4 100644 --- a/internal/dataframe/publication/row_smuggling_prevention_test.go +++ b/internal/dataframe/publication/row_smuggling_prevention_test.go @@ -7,36 +7,9 @@ import ( "testing" ) -// F-B23 prevention evidence: can an author forge the row-level security -// metadata columns (auth_resource_path / project_id) through publication so a -// restricted reader sees rows it should not (or provenance is spoofed)? -// Adversary: author-controlled data (pivot ColumnExpr column names, dynamic -// unknown keys) plus authored schema. Verdict encoded here: PREVENTED by -// three independent lines, each pinned below. -// -// L1 internal/dataframe/execution/engine.go publicStreamRow copies ONLY the -// compiled s.Columns plus the two compiler-retained keys -// {__loom_row_id, auth_resource_path} — a data-born row key outside the -// compiled column set is dropped upstream of this package, so smuggled -// keys never reach the publication runner. Pinned by source-quote gate -// and by the documented stream-flow (engine.Stream -> visit -> -// QualifyFlatRow -> injector). Data-born key semantics cannot be -// behavior-tested from this package (publicStreamRow is package -// `execution`); the source pin is the enforced contract. -// L2 injectPublicationMetadata refuses a DECLARED auth_resource_path column -// (reserved-by-Loom) and strips author-declared project_id from schema. -// L3 row["project_id"] is unconditionally overwritten with identity.Project -// ("never trust a source row value"). -// -// The injector's "set auth_resource_path only if absent" branch is NOT a -// hole: a surviving value can only be the compiler's multi-path scope -// projection because L1 drops every other key. If the L1 fragments below -// change, this artifact must be re-adjudicated. - func TestFB23PublicationRowMetadataSmugglingPrevented(t *testing.T) { identity := PublicationIdentity{Project: "prj-auth", AuthResourcePaths: []string{"p1", "p2"}} - // L2a: declaring the reserved security column is refused. reserved := OutputStream{ Name: "patients", Columns: []LogicalColumn{{Name: "auth_resource_path", Kind: "string"}}, @@ -50,7 +23,6 @@ func TestFB23PublicationRowMetadataSmugglingPrevented(t *testing.T) { t.Logf("FB23 L2a | reserved-column refusal | %v", err) } - // L2b + L3: declared and row-forged project_id are both neutralized. sneaky := OutputStream{ Name: "patients", Columns: []LogicalColumn{{Name: "project_id", Kind: "string"}, {Name: "age", Kind: "integer"}}, @@ -92,7 +64,6 @@ func TestFB23PublicationRowMetadataSmugglingPrevented(t *testing.T) { t.Logf("FB23 note | auth_resource_path stays compiler-owned: absent rows get empty for 0/2+ paths; row-supplied values are intentionally preserved because L1 filters row keys upstream of this package") } - // L1 pin: the engine-side declared-column filter must remain quote-identical. src, err := os.ReadFile("../execution/engine.go") if err != nil { t.Fatalf("FB23 L1 pin unreadable: %v", err) diff --git a/internal/dataframe/publication/runner.go b/internal/dataframe/publication/runner.go index 0f211c09d..c7afc05a2 100644 --- a/internal/dataframe/publication/runner.go +++ b/internal/dataframe/publication/runner.go @@ -13,8 +13,6 @@ import ( dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" ) -// Publish consumes each stream once and keeps at most one bounded batch in -// memory. A target is not committed until every stream has passed validation. func Publish(ctx context.Context, target Target, identity PublicationIdentity, outputs []OutputStream, limits Limits) (Result, error) { if target == nil { return Result{}, fmt.Errorf("publication target is required") @@ -232,7 +230,6 @@ func injectPublicationMetadata(identity PublicationIdentity, outputs []OutputStr row["auth_resource_path"] = "" } } - // project_id is Loom-owned metadata; never trust a source row value. row["project_id"] = identity.Project return visit(row) }) diff --git a/internal/dataframe/publication/types.go b/internal/dataframe/publication/types.go index 44d31d9c3..5be258fbf 100644 --- a/internal/dataframe/publication/types.go +++ b/internal/dataframe/publication/types.go @@ -1,7 +1,3 @@ -// Package publication owns the backend-neutral publication contract for -// resolved dataframe streams. Targets stage rows and decide how a publication -// becomes visible; this package owns validation, bounded batching, and the -// lifecycle around that target. package publication import ( @@ -18,18 +14,9 @@ const ( ColumnDiscovered ColumnProvenance = "DISCOVERED" ) -// LogicalColumn is the backend-independent schema emitted by the compiler. -// Kind is one of string, code, uuid, date, date-time, integer, decimal, -// boolean, or object. Object values are rejected by the generic MVP runner -// unless a target explicitly opts into a serialization policy. type LogicalColumn struct { - // ID is the stable compiler identity used by constructed-table inputs. It - // is persisted with the exact published revision, never inferred from a - // mutable display name. - ID string - Name string - // SemanticPath is the stable FHIR/provenance identity. It is persisted - // alongside the physical schema but never used to name ClickHouse columns. + ID string + Name string SemanticPath string Kind string Repeated bool @@ -45,8 +32,6 @@ type OutputSchema struct { SourceRow *SourceRowMetadata } -// OutputStream is consumed exactly once. The callback must invoke visit for -// each row and must stop when visit returns an error. type OutputStream struct { Name string Columns []LogicalColumn @@ -57,19 +42,16 @@ type OutputStream struct { type PublicationIdentity struct { Name string TranslationVersion string - // OutputName scopes single-output publications. The field is empty for - // legacy/full-bundle publications and is used by incremental Explorer - // loads so each output can retain its own visibility pointer. - OutputName string - Project string - DatasetGeneration string - RecipeDigest string - SchemaDigest string - ReceiptID string - ScopeDigest string - EngineVersion string - AuthScopeMode string - AuthResourcePaths []string + OutputName string + Project string + DatasetGeneration string + RecipeDigest string + SchemaDigest string + ReceiptID string + ScopeDigest string + EngineVersion string + AuthScopeMode string + AuthResourcePaths []string } type PublishedOutput struct { @@ -96,8 +78,6 @@ type Transaction interface { ExistingQualityReports() []QualityReport } -// FinalSchemaDigest computes the versioned digest for the schema actually -// staged. Physical names and discovery provenance are deliberately excluded. func FinalSchemaDigest(identity PublicationIdentity, schemas []OutputSchema) string { type contract struct { ID string `json:"id,omitempty"` diff --git a/internal/dataframe/published/aggregate_plan.go b/internal/dataframe/published/aggregate_plan.go index 8241b72a6..e16e2cb51 100644 --- a/internal/dataframe/published/aggregate_plan.go +++ b/internal/dataframe/published/aggregate_plan.go @@ -108,9 +108,6 @@ func buildAggregatePlan(dataset Materialization, req AggregateBatchRequest) aggr plan.statements = append(plan.statements, aggregateStatementPlan{kind: kind, jobs: jobs, filters: jobs[0].filters}) } } - // Terms buckets need a missing count. Fold those expressions into the - // compatible scalar scan (or create one scalar scan for the filter group) - // instead of issuing a second query per terms specification. for _, job := range plan.jobs { if job.job.ResponseMode != AggregateResponseTerms { continue diff --git a/internal/dataframe/published/aggregate_sql.go b/internal/dataframe/published/aggregate_sql.go index f458e266a..056b26735 100644 --- a/internal/dataframe/published/aggregate_sql.go +++ b/internal/dataframe/published/aggregate_sql.go @@ -6,11 +6,6 @@ import ( "strings" ) -// Aggregate statements can contain a large number of grouping sets. Keep the -// query from competing with the rest of the Loom process for all pod memory: -// ClickHouse spills aggregation/sort state before the container limit is hit, -// and a small thread cap avoids one state arena per source worker. These are -// statement-local settings and do not change the server-wide profile. const aggregateExternalMemoryBytes = 256 * 1024 * 1024 const aggregateMaxThreads = 2 @@ -95,14 +90,6 @@ func buildGroupingSetSQL(dataset Materialization, req AggregateBatchRequest, sta return buildTermsGroupingSetSQL(dataset, req, statement) } -// buildLegacyGroupingSetSQL keeps the aggregate state narrow even when an -// operation contains hundreds of one-column facets. ClickHouse expands a -// large GROUPING SETS clause into one aggregation state per set; on the -// production 156-facet shape that alone can exceed the server's memory cap. -// We fan out the logical facet slot/key pairs once with arrayJoin and perform -// one physical GROUPING SETS aggregation over the narrow pair. The source -// union is still built once per statement, and every logical job remains -// represented by its slot in the grouped key. func buildLegacyGroupingSetSQL(dataset Materialization, req AggregateBatchRequest, statement aggregateStatementPlan) (aggregateSQL, error) { union, args, sourceColumns, err := aggregateSource(dataset, req, statement) if err != nil { @@ -242,10 +229,6 @@ func buildTermsGroupingSetSQL(dataset Materialization, req AggregateBatchRequest if len(where) > 0 { query += " WHERE " + strings.Join(where, " AND ") } - // ClickHouse 25.7's analyzer rejects GROUPING() with - // group_by_use_nulls=1 (even when every argument is a grouping key). - // Keep the setting explicit at 0 and use the grouping masks to distinguish - // rolled-up defaults from real NULL values. query += " GROUP BY GROUPING SETS (" + strings.Join(sets, ", ") + ") " + aggregateSettings() + ") AS __loom_grouped" limitParts := make([]string, 0, len(statement.jobs)*2+1) for i, planned := range statement.jobs { @@ -294,12 +277,6 @@ func groupingCondition(all, groups, masks []string) string { return strings.Join(conditions, " AND ") } -// groupingMaskExpression deliberately calls GROUPING with one column at a -// time. ClickHouse's analyzer has a compatibility bug for the variadic form -// (GROUPING(a, b, ...)) when GROUPING SETS contains sparse sets: it can reject -// a valid key as not being part of GROUP BY. Combining the scalar GROUPING -// results into the same bit mask is semantically identical and works across -// the ClickHouse versions supported by Loom. func groupingMaskExpression(columns []string) string { if len(columns) == 0 { return "toUInt64(0)" diff --git a/internal/dataframe/published/artifact.go b/internal/dataframe/published/artifact.go index 89dbc6bd5..25f945a4f 100644 --- a/internal/dataframe/published/artifact.go +++ b/internal/dataframe/published/artifact.go @@ -35,15 +35,10 @@ const ( ) var ( - // ErrArtifactRowLimit means the row stream exceeded the caller's bound. - ErrArtifactRowLimit = errors.New("artifact row limit exceeded") - // ErrArtifactByteLimit means the archive exceeded the caller's byte bound. + ErrArtifactRowLimit = errors.New("artifact row limit exceeded") ErrArtifactByteLimit = errors.New("artifact byte limit exceeded") ) -// ArtifactIdentity is the immutable publication identity copied into the -// artifact manifest. All fields are required; a selector or current pointer -// is not a substitute for an execution identity. type ArtifactIdentity struct { Project string `json:"project"` DatasetGeneration string `json:"datasetGeneration"` @@ -55,8 +50,6 @@ type ArtifactIdentity struct { OutputContractDigest string `json:"outputContractDigest"` } -// ArtifactColumn supplies the stable exported feature order and its declared -// shape. The encoder does not infer a second schema from row values. type ArtifactColumn struct { Name string `json:"name"` OutputKey string `json:"outputKey,omitempty"` @@ -109,9 +102,6 @@ type ArtifactInterpretationIdentity struct { RuleID string `json:"ruleId"` } -// ArtifactRequest contains the data and metadata members supplied by the -// lifecycle owner. Raw metadata must be valid JSON; it is compacted into a -// deterministic representation before writing. type ArtifactRequest struct { Identity ArtifactIdentity Descriptor ArtifactDescriptor @@ -128,25 +118,16 @@ type ArtifactRequest struct { MaxBytes int64 } -// ArtifactRowVisitor consumes one row without requiring the encoder to -// retain the dataset. type ArtifactRowVisitor func(map[string]any) error -// ArtifactRowStream supplies rows to the artifact encoder. It should invoke -// the visitor once for each row and stop when the visitor returns an error. type ArtifactRowStream func(ArtifactRowVisitor) error -// ArtifactMember records an uncompressed member's digest and byte size. The -// manifest contains every member except manifest.json itself. type ArtifactMember struct { Name string `json:"name"` SHA256 string `json:"sha256"` Bytes int64 `json:"bytes"` } -// ArtifactResult is complete only when every member and the ZIP central -// directory were written successfully. On any error WriteArtifact returns a -// zero result, even though an io.Writer may contain an unusable prefix. type ArtifactResult struct { Complete bool ArchiveSHA256 string @@ -177,10 +158,6 @@ type artifactSchema struct { ArrayEncoding string `json:"arrayEncoding"` } -// WriteArtifact writes the fixed, safe artifact members to out in a stable -// order. It uses ZIP Store entries so member bytes and the resulting archive -// digest are deterministic for the same input. The manifest is emitted last, -// after all other member checksums are final. func WriteArtifact(ctx context.Context, out io.Writer, request ArtifactRequest, stream ArtifactRowStream) (ArtifactResult, error) { if ctx == nil { return ArtifactResult{}, fmt.Errorf("artifact context is required") diff --git a/internal/dataframe/published/current.go b/internal/dataframe/published/current.go index 3bec1d9da..f91c1f91f 100644 --- a/internal/dataframe/published/current.go +++ b/internal/dataframe/published/current.go @@ -15,9 +15,6 @@ import ( var errReleaseBindingUnavailable = errors.New("active release binding is unavailable") -// CurrentProjectDataset returns the one pointer-visible, successful output for -// a project and exact selector. Project identity is explicit at this boundary; -// this reader never unions tables from different projects. func (r *Reader) CurrentProjectDataset(ctx context.Context, project string, selector DataframeSelector) (Materialization, error) { project = strings.TrimSpace(project) if r == nil || r.Catalog == nil { @@ -131,9 +128,6 @@ func (r *Reader) currentMaterialization(ctx context.Context, execution bundlepub return result, true, nil } -// CurrentProjectDatasets returns every available current output for one -// project. A broken optional release binding is omitted; a required binding or -// transient catalog failure still aborts the list. func (r *Reader) CurrentProjectDatasets(ctx context.Context, project string) ([]Materialization, error) { project = strings.TrimSpace(project) if r == nil || r.Catalog == nil { diff --git a/internal/dataframe/published/errors.go b/internal/dataframe/published/errors.go index bc5fc8774..a5fd75656 100644 --- a/internal/dataframe/published/errors.go +++ b/internal/dataframe/published/errors.go @@ -7,9 +7,6 @@ import ( dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" ) -// backendCallError classifies errors returned by ClickHouse/catalog calls. -// Typed request errors and cancellation remain intact; an untyped dependency -// failure is retryable and keeps its private cause for server-side logging. func backendCallError(err error) error { if err == nil { return nil diff --git a/internal/dataframe/published/export.go b/internal/dataframe/published/export.go index 513e81117..f466eafe3 100644 --- a/internal/dataframe/published/export.go +++ b/internal/dataframe/published/export.go @@ -29,11 +29,6 @@ type ExportRequest struct { Filename string } -// ExactExportRequest identifies the immutable published output that an -// export is allowed to read. It deliberately contains no selector fallback: -// the caller must supply the execution ID that was checked and shown to the -// user. The authorization fields are the already-resolved effective scope; -// the reader never discovers a broader scope while streaming. type ExactExportRequest struct { ExecutionID string OutputID string @@ -48,21 +43,11 @@ type ExactExportRequest struct { Unrestricted bool IncludeRowIdentity bool - // ReaderID is the durable owner of the export's execution read pin. It is - // supplied by the lifecycle operation so a retry or cancellation can be - // audited without exposing a generated UUID to the transport. - ReaderID string - // PinExpiresAt is the initial lease expiry. A zero value uses the standard - // reader lease. The adapter renews the lease while the bounded stream runs. - PinExpiresAt time.Time - // PinRenewalInterval is primarily useful to lifecycle callers with a - // shorter operation budget and deterministic tests. A zero value uses the - // standard reader renewal interval. + ReaderID string + PinExpiresAt time.Time PinRenewalInterval time.Duration } -// ExactExportResult is returned only as metadata. Rows are delivered one at a -// time to the visitor, so the result never retains the exported dataset. type ExactExportResult struct { ExecutionID string OutputID string @@ -72,13 +57,8 @@ type ExactExportResult struct { Rows int64 } -// ExactExportVisitor consumes one row from the exact published output. The -// visitor must not assume that rows are retained after it returns. type ExactExportVisitor func(map[string]any) error -// StreamExactExport resolves and streams one exact published execution while -// holding a renewable read-retention pin. It never follows a project-current -// pointer and never returns success after pin loss or a partial visitor read. func (r *Reader) StreamExactExport(ctx context.Context, request ExactExportRequest, visit ExactExportVisitor) (ExactExportResult, error) { result := ExactExportResult{ ExecutionID: strings.TrimSpace(request.ExecutionID), diff --git a/internal/dataframe/published/nullsort_page_cursor_e2e_test.go b/internal/dataframe/published/nullsort_page_cursor_e2e_test.go index b2158818c..632c30ab1 100644 --- a/internal/dataframe/published/nullsort_page_cursor_e2e_test.go +++ b/internal/dataframe/published/nullsort_page_cursor_e2e_test.go @@ -1,53 +1,5 @@ package published -// F-B24 (CONFIRMED Round 24; converted Round 30 to a naturally-PASSING -// regression control after the NULL-aware keyset repair): sorting a published -// dataset by a legitimate NULLABLE column used to make browser pagination -// silently drop every NULL-sort row (both directions) while TotalCount -// advertised them; the archived discovery reds (FB24 RED-ON-PURPOSE: "5 of 7", -// deterministic, real ClickHouse 25.3, production Reader.Page) are the -// before-fault witnesses, and the guardless pre-fix copy of this package -// re-raises them verbatim (see FINDINGS Round 30). -// -// The repair (published/page.go + read.go, cursor Version 2): -// - ORDER BY pins "`col` DIR NULLS LAST, `__loom_row_id` ASC" so the engine -// default can never disagree with the predicate; -// - pageCursorPredicate always admits `col IS NULL` rows after a non-NULL -// cursor and resumes inside the NULL bucket with -// "`col` IS NULL AND `__loom_row_id` > ?"; -// - a page ending on a NULL row now ENCODES a null-bucket cursor instead of -// failing with INVALID_CURSOR (the latent second defect: the former -// `if sortValue == nil { return Page{}, invalidCursor() }`). -// If any of those guarantees regresses, the traversal assertions below fail -// again through the original harm (unreachable rows / mid-list error), and -// the source gate below retires this artifact loudly if page.go drifts. -// -// User-visible path quoted by the source gate below: -// - internal/api/graphql/graph/dataframe/rows.go forwards the CLIENT-chosen -// sort column verbatim: `Sort: convertSort(input)` into -// `s.reader.Page(ctx, value, dfmaterialization.PageRequest{...})`. -// - internal/dataframe/published/page.go builds that cursor and its -// predicate; NULL is legitimate author data: the production writer maps -// Nullable logical columns to `Nullable()` ClickHouse columns -// (publication/clickhouse/bundle_store.go: `columnType = "Nullable(" + -// columnType + ")"`) and the schema carries Column.Nullable. -// - internal/dataframe/published/errors.go keeps invalidCursor -// (`dataframeerrors.NewError(dataframeerrors.CodeInvalidCursor, "")`) -// reserved for genuinely malformed cursors only. - -// Experiment (real ClickHouse, production Reader.Page; Unrestricted=true for -// harness simplicity — the cursor code path is independent of the auth -// predicate, which is prepended separately at page.go `auth_resource_path` -// IN ?`): a 7-row table, `middle Nullable(String)` with NULLs interleaved, -// sorted pagination through production Reader.Page: -// ASC, first=1: ClickHouse sorts NULLs first, so the FIRST page's last row -// is NULL -> entire first read errors. -// DESC, first=2: traversal g,f | d,b | then the page ending on NULL errors; -// rows past the NULL block can never be listed via sort traversal. -// Control (no NULLs on the boundary: sort by val): same traversal completes -// all 7 rows — the harness and cursor machinery work when NULLs are absent, -// so the failure is attributable to NULL sort values, not to setup. - import ( "context" "fmt" @@ -70,9 +22,6 @@ func nspColumns() []storeclickhouse.Column { } } -// middles arranged so that a DESC traversal ends a page exactly on a NULL: -// sorted DESC: g,f,d,b,a,NULL,NULL ; first=2 -> [g,f],[d,b],[a,NULL] boom. -// With ASC + NULLs-first, page size 1 dies on page 1: first row is NULL. var nspRows = []map[string]any{ {"middle": "a", "val": int64(1), "__loom_row_id": "r1"}, {"middle": "b", "val": int64(2), "__loom_row_id": "r2"}, @@ -94,7 +43,6 @@ func nspMaterialization(table string) Materialization { } func TestFB24NullSortPageEnumerationIsComplete(t *testing.T) { - // --- source gates (quote drift retires the claim loudly) --- for path, fragments := range map[string][]string{ "page.go": { "sortValueNull = sortValue == nil", @@ -137,7 +85,7 @@ func TestFB24NullSortPageEnumerationIsComplete(t *testing.T) { if err != nil { t.Fatalf("clickhouse client: %v", err) } - t.Cleanup(func() { _ = client.Close() }) // registered before the drop => runs after it (LIFO) + t.Cleanup(func() { _ = client.Close() }) if err := client.Ping(ctx); err != nil { t.Fatalf("ping: %v", err) } @@ -160,7 +108,6 @@ func TestFB24NullSortPageEnumerationIsComplete(t *testing.T) { reader := &Reader{ClickHouse: client} m := nspMaterialization(table) - // Helper: full sort traversal, mirroring what a browser does with First/After. traverse := func(sort *Sort, first int) (map[string]bool, int64, string, error) { seen, total, cursor := map[string]bool{}, int64(-1), "" for i := 0; i < 20; i++ { @@ -199,25 +146,18 @@ func TestFB24NullSortPageEnumerationIsComplete(t *testing.T) { return gone } - // Control A: unsorted (row-id) traversal shows ALL rows, proving the NULL - // rows exist in the table and are absent ONLY from sorted pagination. seenPlain, _, _, errPlain := traverse(nil, 2) if errPlain != nil || len(seenPlain) != 7 { t.Fatalf("F-B24 control A lost — unsorted traversal must enumerate 7 rows, got seen=%d err=%v", len(seenPlain), errPlain) } t.Logf("FB24 CONTROL-A | unsorted traversal shows every row | rows=%d (including the 2 NULL-middle rows)", len(seenPlain)) - // Control B: sort by non-null column completes the full enumeration. seenB, _, _, err := traverse(&Sort{Column: "val"}, 2) if err != nil || len(seenB) != 7 { t.Fatalf("F-B24 control B lost — non-null sort traversal must enumerate 7 rows, got seen=%d err=%v", len(seenB), err) } t.Logf("FB24 CONTROL-B | nullable-free sort traversal completes | rows=%d", len(seenB)) - // The converted checks: sorted traversal by the nullable column must reach - // all 7 unique rows in BOTH directions with no error — including both - // NULL-sort rows — and TotalCount must agree. These branches carry the - // archived discovery reds as their regression direction. seenAsc1, totalAsc1, cursorAsc1, errAsc := traverse(&Sort{Column: "middle"}, 1) seenDesc2, totalDesc2, cursorDesc2, errDesc := traverse(&Sort{Column: "middle", Desc: true}, 2) @@ -242,9 +182,6 @@ func TestFB24NullSortPageEnumerationIsComplete(t *testing.T) { t.Logf("FB24 PASS | DESC first=2 reaches all 7 unique rows incl. both NULL-sort rows") } - // Ordered-completeness witness: NULLS LAST is pinned, so a fine-grained - // ASC walk must observe val 3 and 5 (the NULL-middle rows) exactly last, - // in row-id order, after every non-NULL middle. seenAsc3, _, _, errAsc3 := traverse(&Sort{Column: "middle"}, 3) if errAsc3 != nil || len(seenAsc3) != 7 || !seenAsc3["3"] || !seenAsc3["5"] { t.Fatalf("F-B24 ordering witness broken: ASC first=3 must include both NULL-sort rows vals {3,5}, got seen=%d err=%v", len(seenAsc3), errAsc3) diff --git a/internal/dataframe/published/page.go b/internal/dataframe/published/page.go index e1424d36a..29a0fd50d 100644 --- a/internal/dataframe/published/page.go +++ b/internal/dataframe/published/page.go @@ -8,9 +8,6 @@ import ( "strings" ) -// PageRequest describes a bounded keyset page over one published table. -// AuthResourcePaths is the already-resolved effective read scope. An -// unrestricted request deliberately omits the auth_resource_path predicate. type PageRequest struct { Columns []string Filters []Filter @@ -21,7 +18,6 @@ type PageRequest struct { Unrestricted bool } -// StreamRequest describes a bounded-memory scan over one published table. type StreamRequest struct { Columns []string Filters []Filter @@ -31,8 +27,6 @@ type StreamRequest struct { IncludeRowIdentity bool } -// Page reads one active/queryable materialization. It never constructs a -// cross-project UNION and therefore carries only one table's cursor state. func (r *Reader) Page(ctx context.Context, materialization Materialization, req PageRequest) (Page, error) { if r == nil || r.ClickHouse == nil { return Page{}, backendUnavailable() @@ -106,9 +100,6 @@ func (r *Reader) Page(ctx context.Context, materialization Materialization, req if req.Sort.Desc { direction = "DESC" } - // F-B24: NULLS LAST is pinned explicitly in BOTH directions so the - // engine default can never disagree with pageCursorPredicate's - // NULL-bucket ordering, and ordered pages stay total. query += fmt.Sprintf(" ORDER BY `%s` %s NULLS LAST, `__loom_row_id` ASC", req.Sort.Column, direction) } else { query += " ORDER BY `__loom_row_id` ASC" @@ -146,9 +137,6 @@ func (r *Reader) Page(ctx context.Context, materialization Materialization, req sortValueNull := false if req.Sort != nil { sortValue = last[req.Sort.Column] - // F-B24: a NULL last row parks the cursor inside the NULL bucket - // instead of failing the whole page (formerly invalidCursor, the - // latent second defect of the same root cause). sortValueNull = sortValue == nil } if binding == "" { @@ -177,7 +165,6 @@ func (r *Reader) Page(ctx context.Context, materialization Materialization, req return Page{Materialization: materialization, Columns: columns, Rows: rows, RowIDs: rowIDs, TotalCount: total, HasNext: hasNext, NextCursor: next}, nil } -// Stream scans one published table without retaining all rows in memory. func (r *Reader) Stream(ctx context.Context, materialization Materialization, req StreamRequest, visit func(map[string]any) error) error { if r == nil || r.ClickHouse == nil { return backendUnavailable() @@ -219,9 +206,6 @@ func (r *Reader) Stream(ctx context.Context, materialization Materialization, re if req.Sort.Desc { direction = "DESC" } - // F-B24: NULLS LAST is pinned explicitly in BOTH directions so the - // engine default can never disagree with pageCursorPredicate's - // NULL-bucket ordering, and ordered pages stay total. query += fmt.Sprintf(" ORDER BY `%s` %s NULLS LAST, `__loom_row_id` ASC", req.Sort.Column, direction) } else { query += " ORDER BY `__loom_row_id` ASC" @@ -245,9 +229,6 @@ func (r *Reader) Stream(ctx context.Context, materialization Materialization, re return nil } -// decodeJSONTextObjectColumns decodes only logical objects stored in the new -// JSON-text physical representation. Legacy JSON columns still arrive from -// ClickHouse as maps and deliberately bypass this path. func decodeJSONTextObjectColumns(row map[string]any, columns []Column) error { for _, column := range columns { if strings.ToLower(strings.TrimSpace(column.LogicalType)) != "object" { @@ -399,9 +380,6 @@ func pageCursorPredicate(cursor *pageCursor, sortBy *Sort) (string, []any, error return "`__loom_row_id` > ?", []any{cursor.RowID}, nil } if cursor.SortValueNull { - // Cursor parked in the NULL bucket: everything after it is NULL rows - // ordered by row identity, in either direction (NULLs sort last and - // the tiebreak is always row-id ASC). return fmt.Sprintf("(`%s` IS NULL AND `__loom_row_id` > ?)", sortBy.Column), []any{cursor.RowID}, nil } if cursor.SortValue == nil { @@ -411,10 +389,6 @@ func pageCursorPredicate(cursor *pageCursor, sortBy *Sort) (string, []any, error if sortBy.Desc { op = "<" } - // F-B24: NULL rows sit after every non-NULL row in this order (NULLS - // LAST, pinned in the ORDER BY), so they always satisfy "after a - // non-NULL cursor" and must be included; without this arm they were - // unreachable in both directions while TotalCount advertised them. return fmt.Sprintf("(`%s` IS NULL OR `%s` %s ? OR (`%s` = ? AND `__loom_row_id` > ?))", sortBy.Column, sortBy.Column, op, sortBy.Column), []any{cursor.SortValue, cursor.SortValue, cursor.RowID}, nil } diff --git a/internal/dataframe/published/read.go b/internal/dataframe/published/read.go index 376cfa3b8..f649aeabf 100644 --- a/internal/dataframe/published/read.go +++ b/internal/dataframe/published/read.go @@ -145,15 +145,11 @@ func emptyFilterCollection(value any) bool { } type pageCursor struct { - Version int `json:"version,omitempty"` - Binding string `json:"binding,omitempty"` - RowID string `json:"rowId"` - SortValue any `json:"sortValue,omitempty"` - // SortValueNull marks a cursor parked inside the NULL bucket of a sorted - // page (F-B24): SortValue is then meaningless and ordering continues by - // row identity. Version 2 cursors carry this flag; Version 1 could never - // encode a NULL sort value. - SortValueNull bool `json:"sortValueNull,omitempty"` + Version int `json:"version,omitempty"` + Binding string `json:"binding,omitempty"` + RowID string `json:"rowId"` + SortValue any `json:"sortValue,omitempty"` + SortValueNull bool `json:"sortValueNull,omitempty"` } func encodeCursor(rowID string, sortValue any) string { diff --git a/internal/dataframe/published/read_pins.go b/internal/dataframe/published/read_pins.go index 111410ee2..7023d615f 100644 --- a/internal/dataframe/published/read_pins.go +++ b/internal/dataframe/published/read_pins.go @@ -12,10 +12,6 @@ import ( "github.com/google/uuid" ) -// WithExecutionReadPins holds renewable retention pins for every exact -// published execution while visit resolves its outputs and streams rows. -// Callers must resolve the supplied revisions inside visit so cleanup cannot -// remove a table between metadata lookup and pin acquisition. func (r *Reader) WithExecutionReadPins(ctx context.Context, executionIDs []string, visit func(context.Context) error) error { return r.withExecutionReadPins(ctx, executionIDs, defaultReadPinTTL, defaultReadPinInterval, visit) } diff --git a/internal/dataframe/published/selection.go b/internal/dataframe/published/selection.go index 8cb74b6fe..ef4b201cf 100644 --- a/internal/dataframe/published/selection.go +++ b/internal/dataframe/published/selection.go @@ -15,8 +15,6 @@ const ( defaultReadPinInterval = 30 * time.Second ) -// SelectionSourceAdapter adapts the exact published reader to Explorer -// lifecycle source resolution without exposing current-pointer lookup. type SelectionSourceAdapter struct { Reader *Reader } @@ -43,9 +41,6 @@ func (a SelectionSourceAdapter) StreamSelectionSource(ctx context.Context, mater return a.Reader.StreamExactExecution(ctx, materialization.Revision, materialization.Selector.Output, req, visit) } -// ExactExecutionMaterialization resolves one stored execution/output without -// consulting a moving project-current pointer. It is the only resolver a -// selection or export may use after persisting source identity. func (r *Reader) ExactExecutionMaterialization(ctx context.Context, executionID, outputID string) (Materialization, error) { if r == nil || r.Catalog == nil { return Materialization{}, fmt.Errorf("bundle catalog dependency is required") @@ -87,9 +82,6 @@ func (r *Reader) ExactExecutionMaterialization(ctx context.Context, executionID, return Materialization{}, publication.ErrBundleNotFound } -// SourceResourceRef extracts a typed source identity from one output row. It -// deliberately refuses labels, synthetic row numbers, and missing identity -// metadata. func SourceResourceRef(materialization Materialization, row map[string]any) (string, string, error) { if materialization.SourceRow == nil || !materialization.SourceRow.Valid() { return "", "", publication.ErrSelectionSourceNotAddressable @@ -113,9 +105,6 @@ func SourceResourceRef(materialization Materialization, row map[string]any) (str return materialization.SourceRow.ResourceType, id, nil } -// StreamExactExecution acquires a renewable reader pin before scanning an -// exact output. Pin loss cancels the ClickHouse query and is surfaced as a -// failure; callers must never return a partial membership as complete. func (r *Reader) StreamExactExecution(ctx context.Context, executionID, outputID string, req StreamRequest, visit func(map[string]any) error) error { return r.WithExecutionReadPins(ctx, []string{executionID}, func(scanCtx context.Context) error { materialization, err := r.ExactExecutionMaterialization(scanCtx, executionID, outputID) @@ -126,9 +115,6 @@ func (r *Reader) StreamExactExecution(ctx context.Context, executionID, outputID }) } -// ValidateTypedFilters is the boundary check used by selection creation. It -// accepts the existing data-reader filter representation but requires a -// bounded, supported operator/value shape before the reader builds SQL. func ValidateTypedFilters(filters []Filter) error { for _, filter := range filters { if strings.TrimSpace(filter.Column) == "" || strings.TrimSpace(filter.Op) == "" { diff --git a/internal/dataframe/recipe/canonical.go b/internal/dataframe/recipe/canonical.go index 1add6a7fd..fa928218f 100644 --- a/internal/dataframe/recipe/canonical.go +++ b/internal/dataframe/recipe/canonical.go @@ -1,6 +1,3 @@ -// Package recipe defines the persistence-neutral recipe document used by the -// dataframe compiler. A recipe describes semantic row shaping only; it never -// carries database collection, table, AQL, or SQL details. package recipe import ( @@ -9,8 +6,6 @@ import ( "encoding/json" ) -// CanonicalJSONWithWorkspaceOutputs is limited to compiler fragments whose -// sibling IDs are validated by the enclosing authoring workspace. func (b Bundle) CanonicalJSONWithWorkspaceOutputs(allowed []string) ([]byte, error) { if err := b.ValidateWithWorkspaceOutputs(allowed); err != nil { return nil, err @@ -18,8 +13,6 @@ func (b Bundle) CanonicalJSONWithWorkspaceOutputs(allowed []string) ([]byte, err return json.Marshal(b) } -// DigestWithWorkspaceOutputs includes exact sibling IDs while validating the -// fragment against its enclosing workspace context. func (b Bundle) DigestWithWorkspaceOutputs(allowed []string) (string, error) { canonical, err := b.CanonicalJSONWithWorkspaceOutputs(allowed) if err != nil { @@ -36,9 +29,6 @@ func (b Bundle) CanonicalJSON() ([]byte, error) { return json.Marshal(b) } -// CanonicalBundle expands fragment macros and round-trips the validated -// document so every registration and compilation boundary receives the same -// immutable, standalone recipe shape. func (b Bundle) CanonicalBundle() (Bundle, error) { if b.Fragments != nil { expanded, err := b.ExpandFragments() @@ -58,7 +48,6 @@ func (b Bundle) CanonicalBundle() (Bundle, error) { return normalized, nil } -// Digest returns the SHA-256 digest of CanonicalJSON encoded as lowercase hex. func (b Bundle) Digest() (string, error) { canonical, err := b.CanonicalJSON() if err != nil { @@ -68,7 +57,6 @@ func (b Bundle) Digest() (string, error) { return hex.EncodeToString(sum[:]), nil } -// Explain returns the validated semantic shape and canonical digest. func (b Bundle) Explain() (Explanation, error) { digest, err := b.Digest() if err != nil { diff --git a/internal/dataframe/recipe/construction.go b/internal/dataframe/recipe/construction.go index ded69fd5c..ad2bf96bb 100644 --- a/internal/dataframe/recipe/construction.go +++ b/internal/dataframe/recipe/construction.go @@ -12,29 +12,18 @@ import ( "github.com/calypr/loom/internal/dataframe/spec" ) -// ConstructionSourceProjectionID names the implicit source projection stage -// that precedes every authored construction step. const ConstructionSourceProjectionID = "source_projection" -// ConstructionCohortGroupStageID identifies the compiler-owned stage inserted -// at the persisted explicit-group boundary. Authored steps may reference it -// only as the exact effective predecessor when a cohort is present. const ConstructionCohortGroupStageID = "group_rows" const maxConstructionSteps = 128 type Construction struct { - Version int `json:"version"` - // SourceColumns is populated by the resolved compiler frontend. It binds - // persisted projection-slot IDs to the exact public names emitted by the - // source compiler; authoring documents keep their own single source of truth. + Version int `json:"version"` SourceColumns []StageColumn `json:"sourceColumns,omitempty"` Steps []ConstructionStep `json:"steps"` } -// ConstructionStep applies one typed operation to the preceding stage and -// declares that stage's complete output schema. Column IDs are stable across -// display-name edits; Names are physical output names. type ConstructionStep struct { ID string `json:"id"` Inputs []ConstructionInputRef `json:"inputs"` @@ -43,8 +32,6 @@ type ConstructionStep struct { RowValues []ConstructionRowValue `json:"rowValues,omitempty"` } -// ConstructionRowValue collects distinct source values from rows that -// contribute to a grouped or pivoted output row. type ConstructionRowValue struct { InputColumnID string `json:"inputColumnId,omitempty"` FieldSource *ConstructionGroupFieldSource `json:"fieldSource,omitempty"` @@ -68,9 +55,6 @@ const ( ConstructionWorkspaceOutputInput ConstructionInputKind = "WORKSPACE_OUTPUT" ) -// ConstructionInputRef is a closed reference to an input row set. Workspace -// output references name a sibling in the same immutable bundle; table -// revision references identify an exact immutable publication. type ConstructionInputRef struct { Kind ConstructionInputKind `json:"kind"` StepID string `json:"stepId,omitempty"` @@ -79,18 +63,12 @@ type ConstructionInputRef struct { OutputID string `json:"outputId,omitempty"` } -// StageColumn is the logical schema contract for one stage output. Type is an -// optional authoring hint (or "INFER"); the compiler derives the authoritative -// type from source and operation semantics. type StageColumn struct { - ID string `json:"id"` - Name string `json:"name"` - Label string `json:"label,omitempty"` - Type string `json:"type,omitempty"` - Nullable bool `json:"nullable,omitempty"` - // SourceChild keeps the stable structural provenance for a compiler- - // generated child of an INDEXED source slot. It is populated only on the - // resolved compiler contract, never persisted as a second authored schema. + ID string `json:"id"` + Name string `json:"name"` + Label string `json:"label,omitempty"` + Type string `json:"type,omitempty"` + Nullable bool `json:"nullable,omitempty"` SourceChild *lineage.SourceChild `json:"sourceChild,omitempty"` } @@ -112,8 +90,6 @@ const ( ConstructionRelatedFieldOp ConstructionOperationKind = "RELATED_FIELD" ) -// ConstructionOperation is a closed tagged union. Its operands refer to -// stable stage column IDs, never mutable display or physical names. type ConstructionOperation struct { Kind ConstructionOperationKind `json:"kind"` Pivot *ConstructionPivot `json:"pivot,omitempty"` @@ -178,17 +154,12 @@ const ( RelatedEligibilityCountAtLeast = "COUNT_AT_LEAST" ) -// ConstructionRelatedField projects one scalar field from the exact terminal -// resource retained by a preceding RELATED_EXPAND stage. type ConstructionRelatedField struct { ChoiceID string `json:"choiceId"` Source ConstructionRelatedFieldSource `json:"source"` OutputColumnID string `json:"outputColumnId"` } -// ConstructionRelatedPredicate narrows the selected source field. CandidateID -// is repeated deliberately so a recipe cannot silently apply a predicate to a -// different source field. type ConstructionRelatedPredicate struct { CandidateID string `json:"candidateId"` Quantifier ArrayQuantifier `json:"quantifier,omitempty"` @@ -201,8 +172,6 @@ type ConstructionRelatedRepeatedBoundary struct { MaxItems int `json:"maxItems"` } -// ConstructionRelatedFieldSource identifies an exact scalar leaf. Cardinality -// is many when the path crosses one or more repeated boundaries. type ConstructionRelatedFieldSource struct { CandidateID string `json:"candidateId"` NodeID string `json:"nodeId"` @@ -275,8 +244,6 @@ type ConstructionUnpivotInput struct { Key TableScalar `json:"key"` } -// ConstructionGroup reduces the preceding row set by zero or more typed -// columns. An empty GroupKeys slice summarizes the whole table. type ConstructionGroup struct { ConstructionID string `json:"constructionId"` MissingKeyPolicy ConstructionGroupMissingKeyPolicy `json:"missingKeyPolicy"` @@ -284,8 +251,6 @@ type ConstructionGroup struct { Aggregates []ConstructionGroupAggregate `json:"aggregates,omitempty"` } -// ConstructionCodedGroup groups source rows by a tuple read from the same -// repeated FHIR Coding object. type ConstructionCodedGroup struct { ConstructionID string `json:"constructionId"` Source ConstructionCodedGroupSource `json:"source"` @@ -306,8 +271,6 @@ type ConstructionCodedGroupSource struct { Route []ConstructionRelatedRouteStep `json:"route"` } -// ConstructionCodedPivot evaluates selected codes against values owned by the -// same repeated FHIR item, then emits one row per direct root resource. type ConstructionCodedPivot struct { ConstructionID string `json:"constructionId"` Source ConstructionCodedPivotSource `json:"source"` @@ -385,17 +348,12 @@ func (group *ConstructionGroup) UnmarshalJSON(raw []byte) error { } type ConstructionGroupKey struct { - InputColumnID string `json:"inputColumnId,omitempty"` - OutputColumnID string `json:"outputColumnId"` - FieldSource *ConstructionGroupFieldSource `json:"fieldSource,omitempty"` - // SemanticCategory is the server-resolved coded field family. It is - // mutually exclusive with InputColumnID so legacy scalar keys keep their - // original wire shape. + InputColumnID string `json:"inputColumnId,omitempty"` + OutputColumnID string `json:"outputColumnId"` + FieldSource *ConstructionGroupFieldSource `json:"fieldSource,omitempty"` SemanticCategory *ConstructionGroupSemanticCategory `json:"semanticCategory,omitempty"` } -// ConstructionGroupFieldSource binds one source variant to one exact compiler -// source context. It is private input to Group, not a public stage column. type ConstructionGroupFieldSource struct { Kind string `json:"kind"` ChoiceID string `json:"choiceId,omitempty"` @@ -404,9 +362,6 @@ type ConstructionGroupFieldSource struct { SourceShapeID string `json:"sourceShapeId,omitempty"` Input *ConstructionGroupFieldSourceInput `json:"input,omitempty"` - // These flattened fields are accepted only for durable local bindings - // written before the source sum was introduced. Validation normalizes them - // to Input.Kind=CURRENT_STAGE and rejects them alongside Input. SourceRole string `json:"sourceRole,omitempty"` OccurrenceID string `json:"occurrenceId,omitempty"` OwnerResourceType string `json:"ownerResourceType,omitempty"` @@ -426,8 +381,6 @@ func (source *ConstructionGroupFieldSource) UnmarshalJSON(raw []byte) error { return nil } -// ConstructionGroupFieldSourceInput is a tagged sum. Exactly one payload is -// present and it must match Kind. type ConstructionGroupFieldSourceInput struct { Kind string `json:"kind"` CurrentStage *ConstructionGroupCurrentStageFieldSource `json:"currentStage,omitempty"` @@ -471,9 +424,6 @@ type ConstructionGroupCurrentStageFieldSource struct { } type ConstructionGroupRelatedOccurrenceFieldSource struct { - // RouteID is compiler/capability metadata. It is intentionally omitted from - // the durable recipe because the route facts and sourceShapeId already bind - // the exact traversal. RouteID string `json:"-"` AnchorOccurrenceID string `json:"anchorOccurrenceId"` AnchorNodeID string `json:"anchorNodeId"` @@ -517,8 +467,6 @@ const ( ConstructionGroupCodingEmptyPolicyError ConstructionGroupCodingEmptyPolicy = "ERROR" ) -// ConstructionGroupSemanticCategory pins one exact observed coding family. -// Version is empty only for the versionless family; it is never a wildcard. type ConstructionGroupSemanticCategory struct { Kind string `json:"kind"` BindingID string `json:"bindingId,omitempty"` @@ -549,9 +497,6 @@ const ( ConstructionGroupMean ConstructionGroupAggregateOp = "MEAN" ) -// ConstructionExpand replaces one array-valued stage column with one row per -// item. OrdinalColumnID is optional; row identity always includes the zero- -// based item ordinal, including when the ordinal is not exposed publicly. type ConstructionExpand struct { ConstructionID string `json:"constructionId"` InputColumnID string `json:"inputColumnId"` @@ -564,8 +509,6 @@ func (construction Construction) Validate(sourceFields []Field) error { return construction.validate(sourceFields, nil) } -// ValidateWithGroupRows validates construction against the effective stage -// sequence formed by inserting the pinned cohort after GroupRows.AfterStepID. func (construction Construction) ValidateWithGroupRows(sourceFields []Field, groupRows *GroupRows) error { if groupRows == nil { return construction.Validate(sourceFields) @@ -585,9 +528,6 @@ func (construction Construction) validate(sourceFields []Field, groupRows *Group } sourceColumns := append([]StageColumn(nil), construction.SourceColumns...) if len(sourceColumns) == 0 { - // Recipe-only callers can validate simple field projections without a - // resolved source schema. Explorer compilation supplies SourceColumns so - // aggregate, lookup, and other projection variants retain slot identity. sourceColumns = make([]StageColumn, 0, len(sourceFields)) for _, field := range sourceFields { if strings.TrimSpace(field.ColumnID) == "" { @@ -598,9 +538,6 @@ func (construction Construction) validate(sourceFields []Field, groupRows *Group } allowEmptyCodedPivotSource := len(sourceColumns) == 0 && len(sourceFields) == 0 && len(construction.Steps) > 0 && construction.Steps[0].Operation.Kind == ConstructionCodedPivotOp - // A new rooted output can intentionally start with no user-selected - // columns. A direct root RELATED_EXPAND can be its first operation because - // the compiler supplies and verifies the hidden source identity. allowEmptyRootRelatedExpandSource := len(construction.Steps) > 0 && construction.Steps[0].Operation.Kind == ConstructionRelatedExpandOp && construction.Steps[0].Operation.RelatedExpand != nil && construction.Steps[0].Operation.RelatedExpand.AnchorColumnID == "_key" @@ -922,10 +859,6 @@ func validateSourceChildLineage(columns []StageColumn) error { return nil } -// TerminalCombineStep reports whether this construction is the standalone -// terminal Combine form. It deliberately excludes source and intermediate -// stages; those require a materialized engine boundary before ClickHouse can -// consume their rows. func (construction Construction) TerminalCombineStep() (ConstructionStep, bool) { if len(construction.Steps) != 1 || construction.Steps[0].Operation.Kind != ConstructionCombineOp { return ConstructionStep{}, false @@ -1628,9 +1561,6 @@ func validConstructionLogicalType(value string) bool { } } -// IsValidConstructionLogicalType reports whether a construction stage can -// declare this logical type. Explorer capability choices use the same rule so -// they cannot advertise a type the persisted recipe validator will reject. func IsValidConstructionLogicalType(value string) bool { return validConstructionLogicalType(value) } diff --git a/internal/dataframe/recipe/construction_combine.go b/internal/dataframe/recipe/construction_combine.go index bcfbdd2bd..076d587fe 100644 --- a/internal/dataframe/recipe/construction_combine.go +++ b/internal/dataframe/recipe/construction_combine.go @@ -5,8 +5,6 @@ import ( "strings" ) -// ConstructionCombineKind names one bounded operation over immutable table -// revisions. Every input is resolved by exact table, revision, and output ID. type ConstructionCombineKind string const ( @@ -22,10 +20,6 @@ const ( ConstructionCombineLeftJoin ConstructionCombineJoinType = "LEFT" ) -// ConstructionCombineRightMatchPolicy makes duplicate-key behavior explicit. -// PRESERVE_ALL retains every right match, including its expected row -// multiplication. The compiler does not deduplicate or select an arbitrary -// right row. type ConstructionCombineRightMatchPolicy string const ConstructionCombinePreserveAllMatches ConstructionCombineRightMatchPolicy = "PRESERVE_ALL" @@ -37,10 +31,6 @@ const ( ConstructionCombineExcludeMatches ConstructionCombineMembershipMode = "EXCLUDE" ) -// ConstructionCombine describes one relational operation over the ordered -// TABLE_REVISION inputs on its construction step. Output IDs are authored -// identities; physical names and logical types are resolved from each exact -// published revision before lowering. type ConstructionCombine struct { Kind ConstructionCombineKind `json:"kind"` Keys []ConstructionCombineKey `json:"keys,omitempty"` @@ -50,25 +40,17 @@ type ConstructionCombine struct { MembershipMode ConstructionCombineMembershipMode `json:"membershipMode,omitempty"` } -// ConstructionCombineKey pairs a left and right stable column ID. Key order -// defines tuple order when a compound key is used. type ConstructionCombineKey struct { LeftColumnID string `json:"leftColumnId"` RightColumnID string `json:"rightColumnId"` } -// ConstructionCombineProjection maps a stable output column ID to a stable -// input column ID. APPEND uses one mapping per input/output pair so sources -// with different column IDs can share one output schema. type ConstructionCombineProjection struct { OutputColumnID string `json:"outputColumnId"` InputIndex int `json:"inputIndex"` InputColumnID string `json:"inputColumnId"` } -// Validate checks operation structure against ordered inputs and the complete -// declared output schema. Column compatibility is checked after exact input -// revisions have been resolved by the compiler. func (combine ConstructionCombine) Validate(inputCount int, outputs []StageColumn) error { if inputCount < 2 { return fmt.Errorf("combine requires at least two immutable table inputs") diff --git a/internal/dataframe/recipe/derived.go b/internal/dataframe/recipe/derived.go index 57f8b2e28..2c8c3738d 100644 --- a/internal/dataframe/recipe/derived.go +++ b/internal/dataframe/recipe/derived.go @@ -1,7 +1,5 @@ package recipe -// DerivedColumn is one named arithmetic projection over values already -// present in an output row. It contains no FHIR selectors or backend syntax. type DerivedColumn struct { ConstructionID string `json:"constructionId"` Name string `json:"name"` @@ -36,8 +34,6 @@ const ( DivisionByZeroError DivisionByZeroPolicy = "ERROR" ) -// DerivedOperand is either a reference to a named output column or a typed -// numeric literal. Validate enforces the tagged union at the recipe boundary. type DerivedOperand struct { Kind DerivedOperandKind `json:"kind"` Column string `json:"column,omitempty"` diff --git a/internal/dataframe/recipe/document_types.go b/internal/dataframe/recipe/document_types.go index 736e7a549..5de317868 100644 --- a/internal/dataframe/recipe/document_types.go +++ b/internal/dataframe/recipe/document_types.go @@ -1,6 +1,3 @@ -// Package recipe defines the persistence-neutral recipe document used by the -// dataframe compiler. A recipe describes semantic row shaping only; it never -// carries database collection, table, AQL, or SQL details. package recipe import ( @@ -14,7 +11,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// CurrentSchemaVersion is the first stable recipe document schema. const CurrentSchemaVersion = 1 const ( @@ -23,7 +19,6 @@ const ( maxLiteralArray = 256 ) -// Bundle is an immutable, versioned collection of dataframe outputs. type Bundle struct { RecipeSchemaVersion int `json:"recipeSchemaVersion"` Name string `json:"name"` @@ -32,8 +27,6 @@ type Bundle struct { Outputs []Output `json:"outputs"` } -// Output describes one row-shaped result. Names are semantic names, not -// storage identifiers. type Output struct { Name string `json:"name"` RootResourceType string `json:"rootResourceType"` @@ -57,21 +50,16 @@ type Output struct { CatalogProjections []CatalogProjection `json:"catalogProjections,omitempty"` ColumnTransformations []ColumnTransformation `json:"columnTransformations,omitempty"` DerivedColumns []DerivedColumn `json:"derivedColumns,omitempty"` - // Construction is an ordered typed sequence over the source projection. - // When present, legacy output-level shape operations are not used. - Construction *Construction `json:"construction,omitempty"` - Population *PopulationConstraint `json:"population,omitempty"` - CollisionPolicy string `json:"collisionPolicy,omitempty"` + Construction *Construction `json:"construction,omitempty"` + Population *PopulationConstraint `json:"population,omitempty"` + CollisionPolicy string `json:"collisionPolicy,omitempty"` } -// ColumnTransformation binds one closed value transformation to its stable -// public output column name. It is applied only after normal source lowering. type ColumnTransformation struct { Column string `json:"column"` Transformation columntransform.ValueTransformation `json:"transformation"` } -// GroupRows pins an output to an immutable explicit-group revision. type GroupRows struct { RevisionID string `json:"revisionId"` UnassignedMemberPolicy string `json:"unassignedMemberPolicy"` @@ -79,16 +67,11 @@ type GroupRows struct { RowValues []GroupRowValuePolicy `json:"rowValues,omitempty"` } -// GroupRowValuePolicy binds a selected root field projection to its reduction -// across the exact member records of one explicit group. type GroupRowValuePolicy struct { ColumnID string `json:"columnId"` Policy ConstructionRowValuePolicy `json:"policy"` } -// PopulationConstraint is a storage-neutral row-root membership constraint. -// Selection identity is resolved before compilation; the physical collection -// used to read its members remains a runtime binding. type PopulationConstraint struct { SelectionRevisionID string `json:"selectionRevisionId"` MembershipDigest string `json:"membershipDigest"` @@ -121,12 +104,6 @@ func (n RootColumnNaming) Normalized() RootColumnNaming { return n } -// TraversalColumnNaming controls how traversal aliases contribute to public -// output column names. PATH is the backwards-compatible recipe behavior and -// includes every ancestor alias. ALIAS treats each traversal alias as a -// globally scoped output namespace. EXACT emits the authored field name -// without coupling it to the traversal position; callers must provide globally -// unique field names. type TraversalColumnNaming string const ( @@ -146,11 +123,8 @@ func (n TraversalColumnNaming) Normalized() TraversalColumnNaming { return n } -// Field projects one named semantic value into an output row. type Field struct { - Name string `json:"name"` - // ColumnID is the stable authoring identity used by construction stages. - // Legacy recipes may omit it and retain name-based behavior. + Name string `json:"name"` ColumnID string `json:"columnId,omitempty"` Label string `json:"label,omitempty"` FieldRef string `json:"fieldRef,omitempty"` @@ -160,9 +134,6 @@ type Field struct { Discovered bool `json:"-"` } -// ValueMode controls how a checked selector contributes values to one row. -// The empty value is the backwards-compatible AUTO default and is omitted from -// persisted JSON so existing recipe bytes and digests remain stable. type ValueMode string const ( @@ -183,8 +154,6 @@ func (m ValueMode) Normalized() ValueMode { return m } -// Filter is a typed, storage-independent predicate. Select is the executable -// selector; FieldRef is optional frontend/catalog provenance. type Filter struct { Select string `json:"select"` FieldRef string `json:"fieldRef,omitempty"` @@ -258,8 +227,6 @@ type CodeValue struct { Version string `json:"version,omitempty"` } -// FilterValue is a strict tagged union. Pointer scalar members preserve false, -// zero, and empty string values during JSON decoding. type FilterValue struct { Kind FilterValueKind `json:"kind"` String *string `json:"string,omitempty"` @@ -271,40 +238,26 @@ type FilterValue struct { DateTime *string `json:"dateTime,omitempty"` } -// Pivot describes a bounded, schema-validated column/value mapping. type Pivot struct { - Name string `json:"name"` - FieldRef string `json:"fieldRef,omitempty"` - ColumnExpr Expression `json:"columnExpr"` - ValueExpr Expression `json:"valueExpr"` - ValueFallbacks []Expression `json:"valueFallbacks,omitempty"` - ItemSource Expression `json:"itemSource,omitempty"` - ItemResourceType string `json:"itemResourceType,omitempty"` - Columns []string `json:"columns"` - // ColumnAliases binds each discovered/raw pivot key to its authored public - // output name. Correlated pivots use this explicit mapping so a public - // column may remain stable when the selected terminology code changes. - ColumnAliases map[string]string `json:"columnAliases,omitempty"` - // ProjectionMode is the lookup's declared reduction policy. Empty is the - // legacy FIRST default; correlated pivots preserve ALL, DISTINCT, VALUE, - // and FIRST through physical lowering. - ProjectionMode string `json:"projectionMode,omitempty"` - Discovery *PivotDiscovery `json:"discovery,omitempty"` - // Correlation is the closed system+code/value binding for terminology - // pivots. Nil preserves the legacy selector pivot representation. - Correlation *fhirschema.CorrelatedBinding `json:"correlation,omitempty"` - CorrelationSystem string `json:"correlationSystem,omitempty"` - CorrelationCode string `json:"correlationCode,omitempty"` - CorrelationVersion string `json:"correlationVersion,omitempty"` - // ExtensionCorrelation is the closed ancestor-aware extension binding. It - // is mutually exclusive with Correlation and legacy selector expressions. - ExtensionCorrelation *fhirschema.ExtensionBinding `json:"extensionCorrelation,omitempty"` - Discovered bool `json:"-"` -} - -// OwnerRecordProjection preserves each repeated owner selected by a checked -// code/value binding as one ordered object in a list-valued output column. -// It does not change the output row grain. + Name string `json:"name"` + FieldRef string `json:"fieldRef,omitempty"` + ColumnExpr Expression `json:"columnExpr"` + ValueExpr Expression `json:"valueExpr"` + ValueFallbacks []Expression `json:"valueFallbacks,omitempty"` + ItemSource Expression `json:"itemSource,omitempty"` + ItemResourceType string `json:"itemResourceType,omitempty"` + Columns []string `json:"columns"` + ColumnAliases map[string]string `json:"columnAliases,omitempty"` + ProjectionMode string `json:"projectionMode,omitempty"` + Discovery *PivotDiscovery `json:"discovery,omitempty"` + Correlation *fhirschema.CorrelatedBinding `json:"correlation,omitempty"` + CorrelationSystem string `json:"correlationSystem,omitempty"` + CorrelationCode string `json:"correlationCode,omitempty"` + CorrelationVersion string `json:"correlationVersion,omitempty"` + ExtensionCorrelation *fhirschema.ExtensionBinding `json:"extensionCorrelation,omitempty"` + Discovered bool `json:"-"` +} + type OwnerRecordProjection struct { Name string `json:"name"` FieldRef string `json:"fieldRef,omitempty"` @@ -319,8 +272,6 @@ const ( PivotProjectionDistinct = "DISTINCT" ) -// ValidPivotProjectionMode is intentionally closed. Lookup sources use the -// uppercase authoring spelling; empty retains the historical FIRST behavior. func ValidPivotProjectionMode(mode string) bool { switch strings.ToUpper(strings.TrimSpace(mode)) { case "", PivotProjectionValue, PivotProjectionFirst, PivotProjectionAll, PivotProjectionDistinct: @@ -338,11 +289,6 @@ func NormalizedPivotProjectionMode(mode string) string { return mode } -// MarshalJSON permits catalog-backed pivots to omit selectors in their stored -// declaration. The schema resolver fills those selectors from the scoped -// catalog metadata before semantic compilation; ordinary static pivots still -// serialize both required expressions and therefore retain the strict AST -// contract. func (p Pivot) MarshalJSON() ([]byte, error) { type pivotJSON struct { Name string `json:"name"` @@ -382,9 +328,6 @@ func (e Expression) zero() bool { return e.Select == "" && e.Call == "" && e.Literal == nil && e.Document == nil && len(e.Args) == 0 } -// PivotDiscovery requests a bounded column set from the scoped field catalog. -// The schema resolver replaces this declaration with concrete Columns before -// semantic typing and physical lowering. type PivotDiscovery struct { Family string `json:"family,omitempty"` Path string `json:"path,omitempty"` @@ -435,8 +378,6 @@ const ( TemporalTieResourceKey TemporalTiePolicy = "RESOURCE_KEY" ) -// ContributorWindow selects related resources relative to a root-row anchor. -// Offsets are seconds relative to the anchor and form an explicit interval. type ContributorWindow struct { Timestamp Expression `json:"timestamp"` Anchor Expression `json:"anchor"` @@ -447,7 +388,6 @@ type ContributorWindow struct { Precision TemporalPrecision `json:"precision"` } -// TemporalOrdering chooses one eligible contributor for FIRST_ORDERED. type TemporalOrdering struct { Timestamp Expression `json:"timestamp"` Direction TemporalDirection `json:"direction"` @@ -468,10 +408,6 @@ type Aggregate struct { UnitNormalization *UnitNormalizationPolicy `json:"unitNormalization,omitempty"` } -// UnitNormalizationPolicy binds exact source unit selectors to an approved, -// versioned conversion set. It belongs to the recipe because selectors are -// recipe semantics; the conversion coefficients remain in the compiler-owned -// backend-neutral unit domain. type UnitNormalizationPolicy struct { SystemPath string `json:"systemPath"` CodePath string `json:"codePath"` @@ -510,8 +446,6 @@ type RepresentativeSlice struct { Fields []Field `json:"fields"` } -// TraversalMatchMode is deliberately closed. Empty remains the legacy -// optional default and is normalized only at semantic lowering time. type TraversalMatchMode string const ( @@ -532,8 +466,6 @@ func (m TraversalMatchMode) Normalized() TraversalMatchMode { } } -// Traversal describes a relationship traversal without naming a physical -// graph collection or edge table. type Traversal struct { Name string `json:"name"` OccurrenceID string `json:"occurrenceId,omitempty"` @@ -553,7 +485,6 @@ type Traversal struct { CatalogProjections []CatalogProjection `json:"catalogProjections,omitempty"` } -// Expansion turns a repeated expression into one row per element. type Expansion struct { OwnerOccurrenceID string `json:"ownerOccurrenceId,omitempty"` From Expression `json:"from"` @@ -581,7 +512,6 @@ func (p ExpansionEmptyPolicy) Normalized() ExpansionEmptyPolicy { return p } -// Identity derives a deterministic row identity. type Identity struct { Name string `json:"name"` Expr Expression `json:"expr,omitempty"` @@ -599,39 +529,22 @@ func (i Identity) MarshalJSON() ([]byte, error) { return json.Marshal(identityWire(i)) } -// ExpansionIdentity declares that the compiler derives row identity from the -// selected expansion occurrence and its owner-relative ordinal. type ExpansionIdentity struct{} -// DynamicColumn discovers a bounded set of key/value columns. The compiler -// freezes discovered keys before materialization. type DynamicColumn struct { - Name string `json:"name"` - ValueMode ValueMode `json:"valueMode,omitempty"` - // ColumnPrefix controls the public prefix of frozen dynamic columns. When - // omitted, Name remains the prefix for backwards compatibility. An explicit - // empty string permits a dynamic family such as URL-keyed extensions to - // publish its normalized keys directly (for example source_path). - ColumnPrefix *string `json:"columnPrefix,omitempty"` - Source Expression `json:"source"` - Key *Expression `json:"key,omitempty"` - Value *Expression `json:"value,omitempty"` - Columns []string `json:"columns,omitempty"` - MaxColumns int `json:"maxColumns,omitempty"` - // ColumnTypes carries resolver-observed logical types for frozen keys. It - // is optional for legacy dynamicColumns and is populated for typed - // extensionColumns. - ColumnTypes map[string]string `json:"columnTypes,omitempty"` - // ColumnSourceKeys decouples a public frozen column name from the runtime - // keyed-map lookup key. Extension columns use this for parent__child names - // while matching the leaf Extension.url segment inside its scoped source. + Name string `json:"name"` + ValueMode ValueMode `json:"valueMode,omitempty"` + ColumnPrefix *string `json:"columnPrefix,omitempty"` + Source Expression `json:"source"` + Key *Expression `json:"key,omitempty"` + Value *Expression `json:"value,omitempty"` + Columns []string `json:"columns,omitempty"` + MaxColumns int `json:"maxColumns,omitempty"` + ColumnTypes map[string]string `json:"columnTypes,omitempty"` ColumnSourceKeys map[string]string `json:"columnSourceKeys,omitempty"` Discovered bool `json:"-"` } -// ExtensionColumn is a bounded, URL-keyed projection of FHIR Extension -// values. Unlike dynamicColumns, its resolver understands nested Extension -// arrays and freezes both the URL mapping and the value representation. type ExtensionColumn struct { Name string `json:"name"` Source Expression `json:"source"` @@ -641,10 +554,6 @@ type ExtensionColumn struct { Discovered bool `json:"-"` } -// ExtensionColumnMapping is immutable schema-discovery output. Name is the -// normalized public key, URL is the raw Extension.url, and ValuePath is the -// relative item selector for a single primitive value[x]. Empty ValuePath -// means the extension object is represented by canonical JSON. type ExtensionColumnMapping struct { Name string `json:"name"` URL string `json:"url"` @@ -653,9 +562,6 @@ type ExtensionColumnMapping struct { ValueType string `json:"valueType"` } -// CatalogProjection describes a bounded family of populated FHIR paths. It -// is storage-neutral; the scoped schema resolver turns it into concrete typed -// recipe fields before semantic compilation. type CatalogProjection struct { Name string `json:"name"` IncludePaths []string `json:"includePaths,omitempty"` @@ -673,51 +579,23 @@ const ( ColumnNamingPathSuffix ColumnNaming = "PATH_SUFFIX" ) -// RuntimeBindings are request-scoped and deliberately not part of a stored -// recipe digest. type RuntimeBindings struct { - Project string - // RecipeDigest selects an immutable project-scoped revision when supplied. - // Empty preserves the server-owned name lookup used by legacy callers. - RecipeDigest string - DatasetGeneration string - AuthResourcePaths []string - AuthScopeMode authscope.ReadScopeMode - // SelectionProject is the canonical project identity stored with immutable - // selection members. Project may use the legacy FHIR-storage spelling. - SelectionProject string - // SelectionMembersCollection is a runtime-only collection binding used by - // population constraints. It is never serialized into a recipe digest. + Project string + RecipeDigest string + DatasetGeneration string + AuthResourcePaths []string + AuthScopeMode authscope.ReadScopeMode + SelectionProject string SelectionMembersCollection string PreviewLimit int - // IncludeAuthResourcePath is set only for ClickHouse publication streams. - // It keeps the reserved row-level authorization field out of ordinary - // dataframe previews while ensuring published rows carry their source path. - IncludeAuthResourcePath bool - // OutputNames limits execution to named outputs for preview requests. An - // empty list preserves the bundle's all-output behavior and is not part of - // recipe or schema identity. - OutputNames []string - // IncludeRowIdentity is an internal, lifecycle-scoped preview option. It - // exposes the already-validated stable row identity to a comparison sink; - // ordinary previews keep compiler-owned identity columns hidden. - IncludeRowIdentity bool - // IncludeSourceIdentity is set only for an authorized Builder preview. It - // privately carries the root FHIR id to the preview sink without adding a - // public output column or changing the published schema. - IncludeSourceIdentity bool -} - -// Clone returns request-scoped bindings with independent authorization paths. -// Runtime bindings are never serialized into a recipe or included in its -// digest. + IncludeAuthResourcePath bool + OutputNames []string + IncludeRowIdentity bool + IncludeSourceIdentity bool +} + func (b RuntimeBindings) Clone() RuntimeBindings { b.AuthResourcePaths = append([]string(nil), b.AuthResourcePaths...) b.OutputNames = append([]string(nil), b.OutputNames...) return b } - -// ExpandFragments resolves the optional declarative fragment library and -// returns a standalone bundle suitable for semantic compilation. The stored -// library is omitted from the returned document because the expanded recipe -// is the immutable compiler input and its digest is what runtime plans carry. diff --git a/internal/dataframe/recipe/document_validation.go b/internal/dataframe/recipe/document_validation.go index 28846569d..61618ff2a 100644 --- a/internal/dataframe/recipe/document_validation.go +++ b/internal/dataframe/recipe/document_validation.go @@ -1,6 +1,3 @@ -// Package recipe defines the persistence-neutral recipe document used by the -// dataframe compiler. A recipe describes semantic row shaping only; it never -// carries database collection, table, AQL, or SQL details. package recipe import ( @@ -15,10 +12,6 @@ func (b Bundle) Validate() error { return b.validate(nil) } -// ValidateWithWorkspaceOutputs validates a one-output compiler fragment while -// allowing WORKSPACE_OUTPUT references to sibling outputs supplied by the -// enclosing authoring workspace. The completed workspace bundle must still -// pass Validate, which resolves every edge and rejects cycles. func (b Bundle) ValidateWithWorkspaceOutputs(allowed []string) error { external := make(map[string]struct{}, len(allowed)) for _, outputID := range allowed { @@ -287,11 +280,6 @@ func validateTraversals(items []Traversal, path string, depth int) error { if err := validateOccurrenceID(t.OccurrenceID, p+".occurrenceId"); err != nil { return err } - // The relationship label is not an output namespace: two routes may - // legitimately use the same edge label while targeting different FHIR - // resources (for example Patient -> Condition and Patient -> Specimen). - // Alias is the lexical identity when supplied; otherwise the edge name - // remains the backwards-compatible uniqueness key. traversalKey := t.Alias if traversalKey == "" { traversalKey = t.Name @@ -466,5 +454,3 @@ func validateLiteral(raw json.RawMessage, path string) error { return validationError("invalid_literal", path, "literal must be a scalar or bounded scalar array") } } - -// CanonicalJSON returns stable compact JSON for a validated document. diff --git a/internal/dataframe/recipe/exec/persistent.go b/internal/dataframe/recipe/exec/persistent.go index 452d98838..e4a6c5f59 100644 --- a/internal/dataframe/recipe/exec/persistent.go +++ b/internal/dataframe/recipe/exec/persistent.go @@ -8,15 +8,11 @@ import ( "github.com/calypr/loom/internal/dataframe/recipe" ) -// Reader is the exact recipe lookup contract used by execution. Name-only -// lookup intentionally means "current", while exact version lookup is always -// available without a runtime capability assertion. type Reader interface { LoadRecipe(context.Context, string) (Entry, error) LoadRecipeVersion(context.Context, string, string) (Entry, error) } -// Store is the persistence boundary for immutable versioned registration. type Store interface { Reader SaveRecipe(context.Context, Entry) error @@ -32,14 +28,10 @@ type Entry struct { Digest string } -// PersistentRegistry owns immutable/idempotent recipe registration. It never -// evaluates recipe data. type PersistentRegistry struct { Store Store } -// RegisterVersion retains recipe history by (name, translationVersion). An -// exact version is replaceable only until its first durable execution exists. func (r PersistentRegistry) RegisterVersion(ctx context.Context, bundle recipe.Bundle) (Entry, error) { entry, err := canonicalEntry(bundle) if err != nil { diff --git a/internal/dataframe/recipe/fragments.go b/internal/dataframe/recipe/fragments.go index 13125733d..8dc25d697 100644 --- a/internal/dataframe/recipe/fragments.go +++ b/internal/dataframe/recipe/fragments.go @@ -9,9 +9,6 @@ import ( "strings" ) -// Fragment is a versioned, expression-only recipe macro. Parameters are -// substituted hygienically in selector placeholders `$name` or `${name}`. -// Fragments contain no storage or output-specific behavior. type Fragment struct { Name string `json:"name"` Version string `json:"version"` @@ -63,8 +60,6 @@ func (l FragmentLibrary) Digest() (string, error) { return hex.EncodeToString(sum[:]), nil } -// ExpandExpression resolves fragment calls and returns a normal expression -// tree that contains only built-in recipe operators. func (l FragmentLibrary) ExpandExpression(input Expression) (Expression, error) { if err := l.Validate(); err != nil { return Expression{}, err @@ -72,9 +67,6 @@ func (l FragmentLibrary) ExpandExpression(input Expression) (Expression, error) return l.expand(input, nil, 0) } -// ExpandBundle returns a validated bundle with every fragment invocation -// replaced by built-in recipe expressions. The returned canonical document is -// therefore safe to digest and persist as the resolved translation source. func (l FragmentLibrary) ExpandBundle(bundle Bundle) (Bundle, error) { if err := bundle.Validate(); err != nil { return Bundle{}, err @@ -371,7 +363,6 @@ func substitute(input Expression, bindings map[string]Expression) Expression { return input } -// SortedNames makes explanation and registration output deterministic. func (l FragmentLibrary) SortedNames() []string { names := make([]string, 0, len(l.Fragments)) for name := range l.Fragments { diff --git a/internal/dataframe/recipe/fragments_test.go b/internal/dataframe/recipe/fragments_test.go index 7938ed716..2bc5e64bd 100644 --- a/internal/dataframe/recipe/fragments_test.go +++ b/internal/dataframe/recipe/fragments_test.go @@ -29,12 +29,6 @@ func TestFragmentCycleRejected(t *testing.T) { } } -// TestFragmentExpansionFailuresAreClassifiedValidationError pins that every -// fragment-expansion user fault reaches the typed *ValidationError channel. -// Before this repair these two legs escaped it as raw fmt.Errorf and the -// register mutation wire-returned 500 INTERNAL_ERROR (parent RED log -// verification/publication-bundle/evidence/recipe-revision/RED-fragment-expansion.log), -// while the two control legs below already wire-returned 400 INVALID_REQUEST. func TestFragmentExpansionFailuresAreClassifiedValidationError(t *testing.T) { classified := func(err error) bool { var validation *ValidationError diff --git a/internal/dataframe/recipe/json_encoding.go b/internal/dataframe/recipe/json_encoding.go index ebd1f389c..2d9a5e434 100644 --- a/internal/dataframe/recipe/json_encoding.go +++ b/internal/dataframe/recipe/json_encoding.go @@ -1,6 +1,3 @@ -// Package recipe defines the persistence-neutral recipe document used by the -// dataframe compiler. A recipe describes semantic row shaping only; it never -// carries database collection, table, AQL, or SQL details. package recipe import ( @@ -23,7 +20,6 @@ func (b Bundle) ExpandFragments() (Bundle, error) { return expanded, nil } -// Explanation is a storage-independent summary of a validated recipe. type Explanation struct { RecipeSchemaVersion int `json:"recipeSchemaVersion"` Name string `json:"name"` @@ -32,7 +28,6 @@ type Explanation struct { Outputs []OutputExplanation `json:"outputs"` } -// OutputExplanation intentionally omits physical implementation details. type OutputExplanation struct { Name string `json:"name"` RootResourceType string `json:"rootResourceType"` @@ -45,14 +40,10 @@ type OutputExplanation struct { CatalogProjections []string `json:"catalogProjections,omitempty"` } -// DocumentRef identifies the complete document in a lexical recipe context. -// Context defaults to root when omitted. type DocumentRef struct { Context string `json:"context,omitempty"` } -// Expression is the generic recipe expression input AST. Exactly one of -// Select, Literal, Call, or Document is set; Call nodes may contain Args. type Expression struct { Select string Literal json.RawMessage @@ -183,12 +174,6 @@ func (e *Expression) UnmarshalJSON(data []byte) error { return nil } -// UnmarshalJSON accepts the small amount of metadata emitted by the Explorer -// Builder while keeping catalog-only fields out of the executable recipe. -// Label and ColumnID are durable recipe metadata; logical type and selection -// details are resolved by the compiler from the expression and schema. -// Keep this compatibility surface explicit so unrelated recipe typos remain -// strict parse errors. func (f *Field) UnmarshalJSON(data []byte) error { type fieldJSON struct { Name string `json:"name"` @@ -199,8 +184,6 @@ func (f *Field) UnmarshalJSON(data []byte) error { Fallbacks []Expression `json:"fallbacks,omitempty"` ValueMode ValueMode `json:"valueMode,omitempty"` - // Explorer Builder catalog/presentation metadata. These fields are - // intentionally not copied into Field. LogicalType string `json:"logicalType,omitempty"` Repeated bool `json:"repeated,omitempty"` Family string `json:"family,omitempty"` @@ -230,7 +213,6 @@ func (f *Field) UnmarshalJSON(data []byte) error { return nil } -// Parse strictly decodes and validates a recipe document. func Parse(data []byte) (Bundle, error) { if err := detectDuplicateKeys(data); err != nil { return Bundle{}, err @@ -253,6 +235,3 @@ func Parse(data []byte) (Bundle, error) { } return b, nil } - -// Validate checks semantic document invariants and resource-independent -// expression safety. diff --git a/internal/dataframe/recipe/publication_identity_nul_http_test.go b/internal/dataframe/recipe/publication_identity_nul_http_test.go index 2c70a8c16..3a2d77cce 100644 --- a/internal/dataframe/recipe/publication_identity_nul_http_test.go +++ b/internal/dataframe/recipe/publication_identity_nul_http_test.go @@ -8,28 +8,6 @@ import ( "github.com/calypr/loom/internal/dataframe/publication" ) -// F-B21-http durable regression control (Repair-5 shipped). An -// HTTP-authoring document (transport = encoding/json on the production -// Bundle type, the shape the Explorer compiler receives) CAN still carry NUL -// bytes through the recipe validation gate (document_validation.go:36-41 -// still checks only non-emptiness for $.name and $.translationVersion while -// output/column names pass validateRecipeName's ^[A-Za-z_][A-Za-z0-9_]*$ -// regex) — the boundary this artifact originally watched stayed open at the -// VALIDATOR. Repair-5 closed the collision at the root instead: production -// BundleIdentity.PointerName() is length-prefix-encoded, so the two -// HTTP-decodable distinct documents below get DISTINCT visibility pointers. -// Natural PASS is now the repaired expectation; the REGRESSION direction -// keeps the archived CONFIRMED reachability message if the pointer encoding -// ever re-opens the fold. The validator-side observation remains a t.Errorf: -// if control characters stop surviving JSON decode, or a control-char gate -// lands in Validate(), flip/re-point this artifact deliberately. -// -// Scope honesty: no live HTTP server is driven here (GraphQL/OpenAPI handlers -// decode JSON via encoding/json, so transport fidelity is asserted at the -// decoder the handler itself uses). Live Arango and ClickHouse acceptance of -// NUL-bearing names is parent-verified by -// internal/dataframe/publication/clickhouse/pointer_name_nul_ambiguity_e2e_test.go. - func decodeDocument(t *testing.T, wire string) Bundle { t.Helper() var b Bundle @@ -52,9 +30,6 @@ func identityOf(b Bundle, outputName string) publication.BundleIdentity { } func TestPublicationPointerNULFromHTTPDecodedDocuments(t *testing.T) { - // Two DISTINCT author-facing documents, both minimal-valid except that a - // pointer component embeds NUL. Wire form uses the \u0000 escape exactly as - // an HTTP client would send it. docA := `{"recipeSchemaVersion":` + itoa(CurrentSchemaVersion) + `,"name":"p\u0000q","translationVersion":"r","outputs":[{"name":"patients","rootResourceType":"Patient","rowGrain":"patient"}]}` docB := `{"recipeSchemaVersion":` + itoa(CurrentSchemaVersion) + `,"name":"p","translationVersion":"q\u0000r","outputs":[{"name":"patients","rootResourceType":"Patient","rowGrain":"patient"}]}` @@ -66,12 +41,6 @@ func TestPublicationPointerNULFromHTTPDecodedDocuments(t *testing.T) { } t.Logf("FB21-http GATE | nul-survives-json-decode | decoded $.name=%q preserves the U+0000 the HTTP body carried", a.Name) - // Defense-in-depth OBSERVATION (not a failing assertion): Repair-5 closed - // the F-B21 harm at the root (injective PointerName encoding), so - // Validate() accepting control characters in $.name/$.translationVersion - // no longer enables a namespace hijack. This line documents that the - // validator-side gate is still open; if a control-char gate lands in - // Validate(), flip this note to an expectation of rejection. errA := a.Validate() errB := b.Validate() if errA == nil && errB == nil { @@ -80,9 +49,6 @@ func TestPublicationPointerNULFromHTTPDecodedDocuments(t *testing.T) { t.Logf("FB21-http note: Validate rejected A=%v B=%v — validator-side gate now covers pointer components", errA, errB) } - // Production wiring (server/recipe_publication.go publishResolvedRecipe): - // identity Name/TranslationVersion come straight from the bundle; OutputName - // is the (regex-validated, shared) output name. ia := identityOf(a, "patients") ib := identityOf(b, "patients") diff --git a/internal/dataframe/recipe/revisions.go b/internal/dataframe/recipe/revisions.go index 608e15a7a..43eb5cf89 100644 --- a/internal/dataframe/recipe/revisions.go +++ b/internal/dataframe/recipe/revisions.go @@ -6,9 +6,6 @@ import ( "time" ) -// RecipeRevision is an immutable, project-scoped recipe document. The digest -// is the canonical Bundle digest and is therefore safe to use as an optimistic -// concurrency and publication address. type RecipeRevision struct { Project string `json:"project"` Name string `json:"name"` diff --git a/internal/dataframe/recipe/schema/discovery.go b/internal/dataframe/recipe/schema/discovery.go index 648a9ef5f..69a95eb59 100644 --- a/internal/dataframe/recipe/schema/discovery.go +++ b/internal/dataframe/recipe/schema/discovery.go @@ -1,6 +1,3 @@ -// Package schema resolves catalog-backed recipe declarations into a -// concrete, typed recipe before semantic compilation. It deliberately knows -// nothing about AQL, Arango collections, or output-specific behavior. package schema import ( @@ -25,10 +22,6 @@ func resolveProjectionSets(ctx context.Context, scope Scope, discovery Discovery return nil, err } selected := filterCandidates(candidates, set) - // Catalog projections are optional shape discovery. A resource type can - // be absent from a valid dataset (or have no populated fields matching - // the requested kind); retain the static recipe fields and resolve this - // projection set to zero columns rather than rejecting the whole bundle. if len(selected) == 0 { continue } @@ -84,9 +77,6 @@ func resolvePivots(ctx context.Context, scope Scope, discovery Discovery, resour } if spec, ok := fhirschema.DefaultPivotSpec(resourceType, candidate.Path, observedValuePath); ok { pivotSpec, hasPivotSpec = spec, true - // Persisted catalog metadata is an immutable observation of - // the loaded shape. Prefer it when present, while retaining the - // generated schema default for older catalog documents. if candidate.PivotColumnSelect != "" { pivotSpec.ColumnSelector = fhirschema.FieldSelectorSpecFromPath(candidate.PivotColumnSelect) } @@ -234,20 +224,10 @@ func resolveDynamicColumns(ctx context.Context, scope Scope, discovery Discovery if wasUnresolved { dynamic.Discovered = true } - // A keyed family is optional: a valid FHIR dataset may have no values - // for an extension/identifier family at all. Keep the declaration with - // an empty frozen column set so resolution remains schema-stable and the - // compiler emits no columns for that family instead of rejecting the - // entire dataframe. } return nil } -// dynamicDiscoveryKey derives the catalog path that supplies raw dynamic keys -// and the deterministic transforms the physical key expression applies. The -// catalog stores only observed scalar paths, so a transformed key (for -// example last_segment(item.url)) must be frozen from the transformed catalog -// values; freezing the original URL would make physical map lookups miss. func dynamicDiscoveryKey(dynamic recipe.DynamicColumn, alias string) (string, string, []string, error) { source := strings.TrimPrefix(strings.TrimSpace(dynamic.Source.Select), alias+".") if source == "" { @@ -385,11 +365,6 @@ func projectionName(set recipe.CatalogProjection, fieldPath string) string { return strings.Trim(b.String(), "_") } -// uniqueProjectionName preserves the established PATH column spelling when it -// is unambiguous. FHIR catalogs may contain both scalar and repeated variants -// of a path (for example author.reference and author[].reference), which -// normalize to the same SQL-safe name. Keep both catalog-backed fields rather -// than rejecting an otherwise valid, dataset-independent recipe. func uniqueProjectionName(set recipe.CatalogProjection, fieldPath string, used map[string]struct{}) string { base := projectionName(set, fieldPath) if _, exists := used[base]; !exists { diff --git a/internal/dataframe/recipe/schema/extension_columns.go b/internal/dataframe/recipe/schema/extension_columns.go index c39181bb4..9ad2137e5 100644 --- a/internal/dataframe/recipe/schema/extension_columns.go +++ b/internal/dataframe/recipe/schema/extension_columns.go @@ -10,10 +10,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// resolveExtensionColumns freezes URL-to-value mappings from the same scoped -// catalog snapshot used by the other recipe discovery features. Catalog paths -// below nested Extension arrays are deliberately retained; the semantic layer -// lowers each frozen mapping as a bounded dynamic family. func resolveExtensionColumns(ctx context.Context, scope Scope, discovery Discovery, resourceType, alias string, items []recipe.ExtensionColumn) error { for index := range items { item := &items[index] @@ -50,9 +46,6 @@ func resolveExtensionColumns(ctx context.Context, scope Scope, discovery Discove byURL[identity] = append(byURL[identity], extensionValueCandidate{url: observation.URL, path: observation.SourcePath, valuePath: observation.ValuePath, valueType: observation.ValueType, urlPath: urlPath, fields: candidates}) } if len(candidate.ExtensionValues) > 0 { - // Correlated observations supersede the legacy flattened - // distinct-value fallback for this URL field. Mixing both would - // incorrectly assign every observed value[x] kind to every URL. continue } if !strings.HasPrefix(candidate.Path, selector+".") || !strings.HasSuffix(candidate.Path, ".url") { @@ -149,8 +142,6 @@ func extensionValueMapping(resourceType string, candidates []extensionValueCandi } } if len(values) != 1 { - // Complex and mixed value[x] representations are preserved as a JSON - // string by the semantic/physical extension mapper. return "", "string", sourcePath } for _, value := range values { diff --git a/internal/dataframe/recipe/schema/resolve.go b/internal/dataframe/recipe/schema/resolve.go index 7c74d123a..569d6d003 100644 --- a/internal/dataframe/recipe/schema/resolve.go +++ b/internal/dataframe/recipe/schema/resolve.go @@ -1,6 +1,3 @@ -// Package schema resolves catalog-backed recipe declarations into a -// concrete, typed recipe before semantic compilation. It deliberately knows -// nothing about AQL, Arango collections, or output-specific behavior. package schema import ( @@ -23,14 +20,11 @@ type Scope struct { } type FieldCandidate struct { - ResourceType string - Path string - Kind string - DistinctValues []string - DistinctTruncated bool - // ExtensionValues is optional profile-level correlation metadata. When - // present it preserves the URL/value[x] pairing that a flattened field - // catalog cannot represent with independent distinct-value lists. + ResourceType string + Path string + Kind string + DistinctValues []string + DistinctTruncated bool ExtensionValues []ExtensionValueObservation PivotCandidate bool PivotFamily string @@ -50,8 +44,6 @@ type ExtensionValueObservation struct { URLPath []string } -// Discovery is the only backend-facing seam needed by recipe resolution. The -// implementation in server adapts Loom's existing field catalog reader. type Discovery interface { Fields(context.Context, Scope, string) ([]FieldCandidate, error) } @@ -66,9 +58,6 @@ type Resolved struct { SourceSnapshot SourceSnapshot } -// SourceSnapshot records the exact logical dataset namespace used during -// discovery. An empty generation is the legacy-null namespace, not an -// unscoped request. type SourceSnapshot struct { Project string DatasetGeneration string @@ -77,11 +66,6 @@ type SourceSnapshot struct { AuthResourcePaths []string } -// memoizedDiscovery keeps one immutable catalog snapshot per resource type for -// the lifetime of a resolution. A recipe can have several projection sets, -// pivots, and keyed maps on the same node; issuing the catalog query once per -// declaration needlessly adds latency and can produce an internally -// inconsistent schema if the catalog changes between reads. type memoizedDiscovery struct { delegate Discovery byType map[string][]FieldCandidate diff --git a/internal/dataframe/recipe/strict_json.go b/internal/dataframe/recipe/strict_json.go index 80bcd1e19..52435d78e 100644 --- a/internal/dataframe/recipe/strict_json.go +++ b/internal/dataframe/recipe/strict_json.go @@ -8,9 +8,6 @@ import ( "strings" ) -// detectDuplicateKeys walks the JSON token stream before typed decoding. The -// standard library's struct decoder accepts duplicate object keys, which -// would make recipe digests dependent on decoder behavior. func detectDuplicateKeys(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) dec.UseNumber() diff --git a/internal/dataframe/recipe/table_reshape.go b/internal/dataframe/recipe/table_reshape.go index 98d6aa7f1..df48c936d 100644 --- a/internal/dataframe/recipe/table_reshape.go +++ b/internal/dataframe/recipe/table_reshape.go @@ -9,9 +9,6 @@ import ( "strings" ) -// TableReshape is a document-level row operation over the finalized authored -// output columns. It is distinct from Pivot, which projects correlated values -// from one FHIR row. type TableReshape struct { Kind TableReshapeKind `json:"kind"` GroupedPivot *GroupedPivot `json:"groupedPivot,omitempty"` diff --git a/internal/dataframe/recipe/validation_rich.go b/internal/dataframe/recipe/validation_rich.go index 98a6b28b9..738aa6506 100644 --- a/internal/dataframe/recipe/validation_rich.go +++ b/internal/dataframe/recipe/validation_rich.go @@ -227,9 +227,6 @@ func (p Pivot) validateAt(path string, budget *int) error { return validationError("invalid_correlation", path+".correlation", "selected version requires a versionPath") } if p.ItemResourceType == "" && p.ColumnExpr.zero() && p.ValueExpr.zero() && p.ItemSource.zero() && len(p.ValueFallbacks) == 0 { - // Correlated pivots are a closed binding variant. Their selectors are - // validated from Correlation and are deliberately not duplicated as - // writable legacy expressions. return nil } return validationError("ambiguous_correlation", path, "correlated pivot must not carry legacy selector expressions") @@ -246,8 +243,6 @@ func (p Pivot) validateAt(path string, budget *int) error { } return nil } - // A catalog-backed pivot may omit selectors: the scoped resolver fills them - // from the catalog's validated pivot metadata before semantic compilation. if p.Discovery != nil && p.ColumnExpr.Select == "" && p.ValueExpr.Select == "" && p.ColumnExpr.Call == "" && p.ValueExpr.Call == "" { return nil } @@ -306,9 +301,6 @@ func (p CatalogProjection) validateAt(path string) error { return nil } -// Validate checks a standalone catalog projection before it is attached to an -// output or traversal. Bundle validation uses the path-aware variant so -// callers still receive the precise location of malformed declarations. func (p CatalogProjection) Validate() error { return p.validateAt("catalogProjection") } diff --git a/internal/dataframe/recipe/workspace_output_dependencies.go b/internal/dataframe/recipe/workspace_output_dependencies.go index 91d15b088..5f2378145 100644 --- a/internal/dataframe/recipe/workspace_output_dependencies.go +++ b/internal/dataframe/recipe/workspace_output_dependencies.go @@ -2,9 +2,6 @@ package recipe import "fmt" -// OutputDependencyOrder returns bundle output indexes in stable dependency -// order. The normal Validate path is closed over this bundle and therefore -// rejects missing sibling IDs rather than consulting a mutable publication. func (b Bundle) OutputDependencyOrder() ([]int, error) { return b.outputDependencyOrder(nil) } diff --git a/internal/dataframe/semantic/doc.go b/internal/dataframe/semantic/doc.go index f9e877982..e65446f3a 100644 --- a/internal/dataframe/semantic/doc.go +++ b/internal/dataframe/semantic/doc.go @@ -1,3 +1 @@ -// Package semantic converts a validated dataframe request into a logical FHIR -// dataframe plan without choosing storage routes or emitting AQL. package semantic diff --git a/internal/dataframe/semantic/dynamic_schema.go b/internal/dataframe/semantic/dynamic_schema.go index e8e359bea..eeff0c064 100644 --- a/internal/dataframe/semantic/dynamic_schema.go +++ b/internal/dataframe/semantic/dynamic_schema.go @@ -1,5 +1,3 @@ -// Package plan contains immutable, backend-neutral plans produced after -// recipe validation and scoped schema discovery. package semantic import ( @@ -36,9 +34,6 @@ type FrozenSchema struct { Digest string `json:"digest"` } -// Freeze converts discovery candidates into a deterministic bounded schema. -// It does not perform discovery itself; callers provide candidates obtained -// through the same scoped physical plan used for execution. func Freeze(spec DynamicSpec, candidates []Candidate) (FrozenSchema, error) { if strings.TrimSpace(spec.Name) == "" { return FrozenSchema{}, fmt.Errorf("dynamic column name is required") diff --git a/internal/dataframe/semantic/execution_context.go b/internal/dataframe/semantic/execution_context.go index e00219d7b..43abf8eb5 100644 --- a/internal/dataframe/semantic/execution_context.go +++ b/internal/dataframe/semantic/execution_context.go @@ -2,10 +2,6 @@ package semantic import "github.com/calypr/loom/internal/authscope" -// ExecutionContext carries request-scoped provenance into physical lowering. -// It is deliberately separate from OutputPlan so runtime authorization and -// dataset selection can never become part of a persisted recipe or semantic -// output digest. type ExecutionContext struct { Project string DatasetGeneration string diff --git a/internal/dataframe/semantic/recipe_filter.go b/internal/dataframe/semantic/recipe_filter.go index a8f8e3db4..8e7d4b42e 100644 --- a/internal/dataframe/semantic/recipe_filter.go +++ b/internal/dataframe/semantic/recipe_filter.go @@ -9,18 +9,10 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// LowerRecipeFilters converts the closed recipe filter wire type into the -// canonical typed-filter representation consumed by the semantic and physical -// compilers. Selectors are relative to resourceType unless they are prefixed -// with root. func LowerRecipeFilters(resourceType string, filters []recipe.Filter) ([]spec.TypedFilter, error) { return lowerRecipeFiltersForAlias(resourceType, "root", filters) } -// LowerRecipeFiltersForAlias is the traversal-local variant of -// LowerRecipeFilters. The alias prefix is accepted for GraphQL/catalog -// round-trips, but is removed before schema resolution; the canonical typed -// filter always carries a resource-relative selector. func LowerRecipeFiltersForAlias(resourceType, alias string, filters []recipe.Filter) ([]spec.TypedFilter, error) { return lowerRecipeFiltersForAlias(resourceType, alias, filters) } @@ -65,9 +57,6 @@ func lowerRecipeFilterForAlias(resourceType, alias string, input recipe.Filter) if err != nil { return spec.TypedFilter{}, fmt.Errorf("filter selector %q: %w", canonical, err) } - // Terminal metadata includes repetition inherited from a repeated parent; - // the cardinality helper also proves that every repeated path was explicitly - // iterated. Keep the two facts in agreement rather than trusting the wire. if repeated != metadata.Repeated { return spec.TypedFilter{}, fmt.Errorf("filter selector %q has inconsistent generated cardinality", canonical) } @@ -124,10 +113,6 @@ func recipeFilterKind(primitive fhirschema.PrimitiveKind, canonical string) (spe case fhirschema.PrimitiveDateTime: return spec.FilterDateTime, nil case fhirschema.PrimitiveString: - // Generated metadata does not carry terminology identity separately; - // code is safely recognized only at a terminal `.code` path. The shared - // validator rejects system/display values until paired Coding lowering - // exists in canonical physical IR. if canonical == "code" || strings.HasSuffix(canonical, ".code") { return spec.FilterCode, nil } diff --git a/internal/dataframe/semantic/recipe_plan_build.go b/internal/dataframe/semantic/recipe_plan_build.go index 19edc9de2..f224619a5 100644 --- a/internal/dataframe/semantic/recipe_plan_build.go +++ b/internal/dataframe/semantic/recipe_plan_build.go @@ -1,10 +1,5 @@ package semantic -// This file is the single semantic boundary for persisted recipes and the -// existing GraphQL dataframe request. It deliberately stops before physical -// lowering: no collection, AQL, SQL, or backend implementation detail belongs -// in these types. - import ( "fmt" "strings" @@ -45,10 +40,6 @@ func BuildRecipePlan(bundle recipe.Bundle, bindings recipe.RuntimeBindings) (Rec type scopeBinding struct { ResourceType string Prefix string - // ExpandedItem means this alias denotes one element of a repeated - // collection. The prefix retains the array path for schema resolution, - // but that collection's cardinality must not leak into expressions scoped - // to the item. ExpandedItem bool } @@ -122,10 +113,6 @@ func (s scopeFrame) resolve(ref expression.SelectorRef) (expression.Type, error) return expression.Type{}, err } canonicalPath := selector.CanonicalPath() - // An unqualified dotted selector may be either a nested root field or an - // alias-qualified selector. If the generated schema cannot resolve it as a - // root path, report the lexical alias error before the lower-level schema - // diagnostic so recipe authors get an actionable scope failure. if ref.Context == "" { if parts := strings.SplitN(path, ".", 2); len(parts) == 2 { if _, visible := s.aliases[parts[0]]; !visible { @@ -148,14 +135,9 @@ func (s scopeFrame) resolve(ref expression.SelectorRef) (expression.Type, error) } kind := primitiveKind(metadata.Primitive) if kind == expression.KindObject { - // A repeated object is a valid expansion source even though it has no - // terminal scalar metadata. repeated = repeated || metadata.Repeated } if binding.ExpandedItem { - // The expansion itself is the row-shaping operation. Any [] markers in - // the item-relative selector still represent a many-valued expression, - // while the repeated prefix used to reach the item does not. repeated = strings.Contains(path, "[]") } cardinality := expression.OptionalOne diff --git a/internal/dataframe/semantic/recipe_plan_output.go b/internal/dataframe/semantic/recipe_plan_output.go index f6d6f4345..1ac0f9065 100644 --- a/internal/dataframe/semantic/recipe_plan_output.go +++ b/internal/dataframe/semantic/recipe_plan_output.go @@ -1,10 +1,5 @@ package semantic -// This file is the single semantic boundary for persisted recipes and the -// existing GraphQL dataframe request. It deliberately stops before physical -// lowering: no collection, AQL, SQL, or backend implementation detail belongs -// in these types. - import ( "fmt" "strings" @@ -16,10 +11,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// structuralUserRejection classifies a rejected caller-supplied structural -// identifier (grain, resource type, alias, group pairing). Internal-invariant -// checks over already-derived structures deliberately keep raw errors so a -// genuine compiler fault still surfaces as INTERNAL_ERROR instead of a 400. func structuralUserRejection(message string) error { return dataframeerrors.NewError(dataframeerrors.CodeInvalidRequest, message, dataframeerrors.WithRetryable(false)) } @@ -36,10 +27,6 @@ func buildRecipeOutput(output recipe.Output, bindings recipe.RuntimeBindings) (O return OutputPlan{}, structuralUserRejection("group rows require the groups row grain") } if err := spec.ValidateRootGrain(output.RootResourceType, grain); err != nil { - // Persisted recipes may introduce a product-specific grain when they - // also declare the row-shaping operation and an explicit identity. The - // GraphQL request contract remains strict and continues to use - // ValidateRootGrain above. if (output.Expand == nil || output.Identity == nil) && output.GroupRows == nil || !validCustomGrain(string(grain)) { return OutputPlan{}, err } @@ -244,10 +231,6 @@ func expansionItemScope(ownerScope scopeFrame, expansion SemanticRowExpansion) ( path = binding.Prefix + "." + path } if semantics, ok := fhirschema.ResolveFieldSemantics(binding.ResourceType, path); ok && semantics.Kind == fhirschema.FieldKindArray && semantics.Reference != "" { - // Once UNNEST has selected an object item, resolve item-relative - // selectors against that item's generated FHIR definition. Keeping the - // owner resource and array prefix here makes e.g. item.code resolve as - // Observation.code instead of Coding.code. return ownerScope.child(expansion.ItemBinding, scopeBinding{ResourceType: semantics.Reference, ExpandedItem: true}) } prefix := binding.Prefix @@ -489,9 +472,6 @@ func buildRecipeExtensionMaps(items []recipe.ExtensionColumn, scope scopeFrame, return nil, err } for index := range maps { - // Each extension mapping projects one URL from an Extension array. - // Other URLs in the same array are valid siblings and may have their - // own independently discovered mappings. maps[index].AllowUnknownKeys = true } result = append(result, maps...) @@ -505,8 +485,6 @@ func buildRecipeDynamicMaps(items []recipe.DynamicColumn, scope scopeFrame, path for index, dynamic := range items { columns := []string(nil) if dynamic.Columns != nil { - // Preserve non-nil empty slices: the resolver uses that distinction - // to mark an optional family as resolved with zero discovered keys. columns = append([]string{}, dynamic.Columns...) } item := SemanticDynamicMap{Name: dynamic.Name, ValueMode: dynamic.ValueMode, ScopeAlias: scopeAlias, ResourceType: resourceType, Columns: columns, MaxColumns: dynamic.MaxColumns, ColumnTypes: dynamic.ColumnTypes, ColumnSourceKeys: dynamic.ColumnSourceKeys, AllowUnknownKeys: len(dynamic.ColumnSourceKeys) > 0, Discovered: dynamic.Discovered} diff --git a/internal/dataframe/semantic/recipe_plan_types.go b/internal/dataframe/semantic/recipe_plan_types.go index 59ea5eccf..3a9d42e8b 100644 --- a/internal/dataframe/semantic/recipe_plan_types.go +++ b/internal/dataframe/semantic/recipe_plan_types.go @@ -1,10 +1,5 @@ package semantic -// This file is the single semantic boundary for persisted recipes and the -// existing GraphQL dataframe request. It deliberately stops before physical -// lowering: no collection, AQL, SQL, or backend implementation detail belongs -// in these types. - import ( "fmt" "regexp" @@ -17,9 +12,6 @@ import ( var semanticBindingNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) -// RecipePlan is an immutable, checked semantic representation of a recipe -// bundle. Runtime bindings are request-scoped and are intentionally excluded -// from the persisted recipe digest. type RecipePlan struct { Version int RecipeDigest string @@ -36,7 +28,6 @@ type OutputPlan struct { RootColumnNaming recipe.RootColumnNaming TraversalColumnNaming recipe.TraversalColumnNaming Identity *SemanticExpression - // RowExpansion is the sole semantic row-producing operation for an output. RowExpansion *SemanticRowExpansion GroupRows *SemanticGroupRows TableReshape *SemanticTableReshape @@ -50,7 +41,6 @@ type OutputPlan struct { Population *SemanticPopulation } -// SemanticGroupRows pins the output row set to one immutable explicit-group revision. type SemanticGroupRows struct { RevisionID string UnassignedMemberPolicy string @@ -58,9 +48,6 @@ type SemanticGroupRows struct { RowValues []recipe.GroupRowValuePolicy } -// SemanticTableReshape is the document-level row operation over finalized -// output columns. GroupedPivot is deliberately separate from SemanticPivot, -// which remains a correlated FHIR-column projection. type SemanticTableReshape struct { Kind recipe.TableReshapeKind GroupedPivot *SemanticGroupedPivot @@ -113,9 +100,6 @@ type SemanticPopulationRouteStep struct { StorageDirection string } -// SemanticExpression keeps the checked typed AST together with the logical -// source location and lexical context that produced it. SourcePath is a -// recipe JSON path for diagnostics, not a physical query fragment. type SemanticExpression struct { Expression expression.Expression Type expression.Type @@ -123,8 +107,6 @@ type SemanticExpression struct { Context string } -// ExpansionEmptyPolicy defines what an existing owner contributes when its -// selected collection is null or empty. type ExpansionEmptyPolicy string const ( @@ -139,9 +121,6 @@ type SemanticOccurrence struct { ResourceType string } -// SemanticRowExpansion is the single checked cardinality boundary for an -// output. Its owner is an exact authored route occurrence, not merely a -// resource type or selector prefix. type SemanticRowExpansion struct { Owner SemanticOccurrence Source SemanticExpression @@ -193,16 +172,11 @@ type SemanticDynamicMap struct { Columns []string ColumnTypes map[string]string ColumnSourceKeys map[string]string - // AllowUnknownKeys is used by schema-aware projections that intentionally - // select one key from a shared runtime map. Other keys in that map are - // siblings, not schema drift; frozen matching keys still receive type checks. AllowUnknownKeys bool MaxColumns int Discovered bool } -// RecipePlanExplanation is stable diagnostic output and contains only logical -// types and source paths. It never exposes a backend query or storage name. type RecipePlanExplanation struct { Version int RecipeDigest string @@ -236,7 +210,6 @@ type ExpansionExplanation struct { EmptyPolicy ExpansionEmptyPolicy } -// Explain returns a backend-neutral summary suitable for API diagnostics. func (p RecipePlan) Explain() RecipePlanExplanation { out := RecipePlanExplanation{Version: p.Version, RecipeDigest: p.RecipeDigest, TranslationVersion: p.TranslationVersion, Outputs: make([]OutputPlanExplanation, 0, len(p.Outputs))} for _, output := range p.Outputs { @@ -262,7 +235,3 @@ func (p RecipePlan) Explain() RecipePlanExplanation { func explainExpression(e SemanticExpression) ExpressionExplanation { return ExpressionExplanation{SourcePath: e.SourcePath, Context: e.Context, Type: e.Type, Kind: e.Expression.Kind} } - -// BuildRecipePlan lowers and type-checks every expression in a stored bundle. -// The recipe remains data; no output name or resource-specific branch is used -// here. Type resolution is lexical and schema-backed. diff --git a/internal/dataframe/semantic/recipe_plan_user_fault_class_test.go b/internal/dataframe/semantic/recipe_plan_user_fault_class_test.go index 47281f61c..06ec02a63 100644 --- a/internal/dataframe/semantic/recipe_plan_user_fault_class_test.go +++ b/internal/dataframe/semantic/recipe_plan_user_fault_class_test.go @@ -10,15 +10,6 @@ import ( "github.com/calypr/loom/internal/dataframe/recipe" ) -// Structural user-input rejections of a registered recipe bundle must reach -// the GraphQL class layer as typed, non-retryable INVALID_REQUEST errors. -// These payloads pass every Register-path validation (CanonicalBundle + -// Digest); before the classification repair they escaped as raw fmt.Errorf -// values, matched no recipeGraphQLError case, and surfaced as INTERNAL_ERROR -// 500 with retry bait (evidence/recipe-control-classification). Exact wire -// text, including the outputs[i]/traversals[i] context prefixes, is pinned so -// clients keep the messages they already parse. - func buildPlanForClass(t *testing.T, raw string) error { t.Helper() var bundle recipe.Bundle @@ -85,9 +76,6 @@ func TestBuildRecipePlanGroupsGrainWithoutGroupRowsIsUserFault(t *testing.T) { assertStructuralUserFault(t, err, `groups row grain requires a pinned group revision`) } -// Contrast: sibling duplicate aliases are caught EARLY and typed by the -// register gate (recipe.ValidationError), not by the semantic layer; the -// repair must not have moved that class. func TestBuildRecipePlanDuplicateAliasStillCaughtAtRegisterGate(t *testing.T) { var bundle recipe.Bundle if err := json.Unmarshal([]byte(`{"recipeSchemaVersion":1,"name":"p","translationVersion":"p","outputs":[{"name":"rows","rootResourceType":"Patient","rowGrain":"patient","traversals":[{"name":"a","toResourceType":"Observation","alias":"e"},{"name":"b","toResourceType":"Observation","alias":"e"}]}]}`), &bundle); err != nil { @@ -100,7 +88,6 @@ func TestBuildRecipePlanDuplicateAliasStillCaughtAtRegisterGate(t *testing.T) { } } -// Structure-preservation legs: valid payloads must still build unchanged. func TestBuildRecipePlanValidStructuralPayloadsStillBuild(t *testing.T) { for _, raw := range []string{ `{"recipeSchemaVersion":1,"name":"p","translationVersion":"p","outputs":[{"name":"rows","rootResourceType":"Patient","rowGrain":"patient","traversals":[{"name":"obs","toResourceType":"Observation","alias":"o"}]}]}`, @@ -120,9 +107,6 @@ func TestBuildRecipePlanValidStructuralPayloadsStillBuild(t *testing.T) { } } -// Message-stability guard: the typed errors must not silently acquire the -// wrapper prefix twice or lose it, and the raw-escape classes must remain -// distinguishable from typed user errors. func TestStructuralUserFaultMessagesAreComposedOnce(t *testing.T) { err := buildPlanForClass(t, `{"recipeSchemaVersion":1,"name":"p","translationVersion":"p","outputs":[{"name":"rows","rootResourceType":"Patient","rowGrain":"Patient"}]}`) if strings.Count(err.Error(), "outputs[0]") != 1 { @@ -130,10 +114,6 @@ func TestStructuralUserFaultMessagesAreComposedOnce(t *testing.T) { } } -// Boundary canary: internal-invariant rejections of derived structures are -// deliberately NOT user errors. A corrupted semantic graph must still reach -// the class layer untyped so real compiler faults keep INTERNAL_ERROR -// semantics instead of masquerading as 400s. func TestSemanticGraphInvariantRejectionsStayUntyped(t *testing.T) { if err := ValidateSemanticGraph(SemanticNode{ResourceType: "NotAThing", Alias: "root"}); err == nil { t.Fatal("expected invariant rejection for a bogus graph root") @@ -142,11 +122,6 @@ func TestSemanticGraphInvariantRejectionsStayUntyped(t *testing.T) { } } -// Reachable sites that stay untested without row expansion: with expand set, -// the occurrence index (recipe_plan_output.go) walks the traversals before -// the build side does, and expansionItemScope passes child()'s shadow -// rejection through un-rewrapped. All three must stay typed user faults. - func TestBuildRecipePlanGroupRowsWithoutGroupsGrainIsUserFault(t *testing.T) { err := buildPlanForClass(t, `{"recipeSchemaVersion":1,"name":"p","translationVersion":"p","outputs":[{"name":"rows","rootResourceType":"Patient","rowGrain":"patient","groupRows":{"revisionId":"gen-group-1","unassignedMemberPolicy":"EXCLUDE"}}]}`) assertStructuralUserFault(t, err, `group rows require the groups row grain`) diff --git a/internal/dataframe/semantic/recipe_projection.go b/internal/dataframe/semantic/recipe_projection.go index 9eafb3fd0..63722c389 100644 --- a/internal/dataframe/semantic/recipe_projection.go +++ b/internal/dataframe/semantic/recipe_projection.go @@ -1,11 +1,5 @@ package semantic -// This file owns the recipe-to-semantic projection boundary. Recipe fields -// are deliberately normalized here, before physical lowering, so selector -// fallbacks continue to use the same typed extraction contract as recipes -// fields. In particular, this code must not choose a renderer or construct -// backend expressions. - import ( "fmt" "strings" @@ -15,16 +9,6 @@ import ( "github.com/calypr/loom/internal/dataframe/spec" ) -// buildRecipeProjection checks one recipe field in the supplied lexical scope -// and converts it to the canonical semantic field consumed by generic -// lowering. -// The function is intentionally kept separate from BuildRecipePlan so the -// root and traversal builders can use exactly the same fallback/value-mode -// rules, including slice fields. -// -// Schema-version-one fallbacks are selector-only. Their checked expression -// forms remain attached to the field; selector metadata is derived only by -// the physical lowerer when it needs schema-specialized extraction. func normalizeRecipeProjection(field recipe.Field, scope scopeFrame, path string) (SemanticField, error) { if strings.TrimSpace(field.Name) == "" { return SemanticField{}, fmt.Errorf("%s.name is required", path) @@ -46,9 +30,6 @@ func normalizeRecipeProjection(field recipe.Field, scope scopeFrame, path string fallbacks = append(fallbacks, fallback) } - // A selector primary plus selector fallbacks is the optimized physical - // representation. Keep only the checked expressions here; selectors are - // derived from them by ResolveSemanticField at the physical boundary. if primary.Expression.Selector != nil { cardinality := spec.CardinalityOptionalOne if primary.Type.Cardinality == expression.Many { @@ -80,19 +61,11 @@ func normalizeRecipeProjection(field recipe.Field, scope scopeFrame, path string } selection, err := ResolveSemanticField(resourceTypeForScope(scope, primary.Expression), selectorContext(primary.Expression), 0, fieldSemantic) if err != nil { - // ResolveSemanticField needs the resource type owning the selector. A - // malformed/unknown context is already rejected by scope.expression; - // retain its useful diagnostic here rather than silently weakening the - // cardinality contract. return SemanticField{}, fmt.Errorf("%s: %w", path, err) } projected := primary switch selection.Projection { case spec.ProjectionFirst: - // FIRST over a repeated selector can produce no value, but it can - // never produce the source array. Carry that effective cardinality - // into the publication schema while retaining the selector AST for - // optimized PhysicalExtract lowering. projected.Type.Cardinality = expression.OptionalOne case spec.ProjectionArray, spec.ProjectionDistinctArray: projected.Type.Cardinality = expression.Many @@ -126,16 +99,10 @@ func semanticCardinality(input expression.Type) spec.Cardinality { return spec.CardinalityOptionalOne } -// recipeProjectionField returns the canonical selector-bearing SemanticField -// for a projection. For non-selector expressions it returns a field whose -// Expr is the checked expression; physical expression lowering owns that -// case, while selector lowering remains schema-backed. func recipeProjectionField(field recipe.Field, scope scopeFrame, path string) (SemanticField, error) { return normalizeRecipeProjection(field, scope, path) } -// recipeSelector converts a checked expression selector to the schema selector -// used by the existing semantic and physical extraction contracts. func recipeSelector(input expression.Expression) (spec.Selector, error) { if input.Selector == nil { return spec.Selector{}, fmt.Errorf("expression is not a selector") @@ -174,10 +141,6 @@ func validateRecipeProjectionTypes(primary expression.Type, fallbacks []Semantic return nil } -// applyRecipeValueMode applies an explicit value mode to a non-selector -// expression using the typed expression AST. Selector expressions intentionally -// bypass this function so their optimized PhysicalExtract representation keeps -// fallback and selector specialization metadata. func applyRecipeValueMode(scope scopeFrame, input SemanticExpression, mode recipe.ValueMode) (SemanticExpression, error) { normalized := mode.Normalized() if normalized == recipe.ValueModeAuto && input.Type.Cardinality != expression.Many { diff --git a/internal/dataframe/semantic/recipe_rich_shaping.go b/internal/dataframe/semantic/recipe_rich_shaping.go index 6f8593404..127ffc8c4 100644 --- a/internal/dataframe/semantic/recipe_rich_shaping.go +++ b/internal/dataframe/semantic/recipe_rich_shaping.go @@ -1,9 +1,5 @@ package semantic -// This file owns the recipe rich-shaping boundary. Pivots, aggregates, and -// representative slices are converted into canonical semantic operations. Nothing here chooses a traversal strategy or emits -// a backend expression; those decisions remain in the common compiler. - import ( "fmt" "regexp" @@ -18,10 +14,6 @@ import ( var recipeRichNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) -// lowerRecipePivots converts bounded recipe pivots into canonical semantic -// pivots. Both expressions must be selectors owned by the containing node; -// schema metadata, rather than a client-supplied family, proves that the -// selector pair is supported. func lowerRecipePivots(resourceType, alias string, scope scopeFrame, pivots []recipe.Pivot) ([]SemanticPivot, error) { if !fhirschema.HasResource(resourceType) { return nil, structuralUserRejection(fmt.Sprintf("pivot resource type %q is not represented by the active generated FHIR schema", resourceType)) @@ -144,9 +136,6 @@ func lowerRecipePivots(resourceType, alias string, scope scopeFrame, pivots []re } itemResourceType := pivotSpec.ItemResourceType if pivotSpec.ItemSourcePath == "" { - // A resource-level pivot has no nested item scope; leave the item - // type empty so the physical renderer uses its singleton-document - // lowering rather than requiring an item-source selector. itemResourceType = "" } else if itemResourceType == "" { itemResourceType = resourceType @@ -162,10 +151,6 @@ func lowerRecipePivots(resourceType, alias string, scope scopeFrame, pivots []re } } if pivotSpec.ItemSourcePath != "" { - // The schema owns the relative selector conversion. Reparse the - // normalized selectors so component pivots are evaluated against one - // generated backbone item rather than independently against the - // containing Observation. column, err = spec.ParseSelector(fhirschema.SelectorExpression(pivotSpec.ColumnSelector)) if err != nil { return nil, fmt.Errorf("%s column selector: %w", path, err) @@ -296,11 +281,6 @@ func correlatedValueKind(logicalType string) expression.ValueKind { } } -// pivotValueKind derives the flat output type from the value selectors rather -// than from the pivot map that temporarily carries them. FHIR choice values -// can mix primitive kinds (for example Observation.valueQuantity.value and -// Observation.valueString); those values share one discovered column family, -// so normalize that heterogeneous family to strings at the physical boundary. func pivotValueKind(resourceType string, primary spec.Selector, fallbacks []spec.Selector) (expression.ValueKind, bool, error) { selectors := append([]spec.Selector{primary}, fallbacks...) kinds := make([]expression.ValueKind, 0, len(selectors)) @@ -369,10 +349,6 @@ func selectorPath(input string) string { return selectorPathFromExpression(input) } -// lowerRecipeAggregates converts recipe aggregate declarations into the -// canonical aggregate representation. Version-one aggregate expressions are -// selector-only because SemanticAggregate intentionally stores selector -// metadata and the existing physical aggregate lowerer consumes that shape. func lowerRecipeAggregates(resourceType, alias string, scope scopeFrame, aggregates []recipe.Aggregate) ([]SemanticAggregate, error) { if !fhirschema.HasResource(resourceType) { return nil, structuralUserRejection(fmt.Sprintf("aggregate resource type %q is not represented by the active generated FHIR schema", resourceType)) @@ -578,11 +554,6 @@ func selectorIterates(selector spec.Selector) bool { return false } -// lowerRecipeSlices converts representative slices into canonical bounded -// slices. Slice fields use the ordinary recipe projection checker, but the -// canonical SemanticSlice representation intentionally remains selector-based -// so the existing physical slice lowerer can retain typed extraction, -// fallback, and value-mode behavior. func lowerRecipeSlices(resourceType, alias string, scope scopeFrame, slices []recipe.RepresentativeSlice) ([]SemanticSlice, error) { if !fhirschema.HasResource(resourceType) { return nil, structuralUserRejection(fmt.Sprintf("slice resource type %q is not represented by the active generated FHIR schema", resourceType)) @@ -643,10 +614,6 @@ func lowerRecipeSlices(resourceType, alias string, scope scopeFrame, slices []re return out, nil } -// lowerRecipeRichShaping is the coordinated entrypoint used by the recipe -// node builder. Keeping the three conversions together makes it easy for the -// caller to attach them to exactly one semantic node while preserving the -// independent validation and tests for each operation family. func lowerRecipeRichShaping(resourceType, alias string, scope scopeFrame, pivots []recipe.Pivot, aggregates []recipe.Aggregate, slices []recipe.RepresentativeSlice) ([]SemanticPivot, []SemanticAggregate, []SemanticSlice, error) { pivotPlan, err := lowerRecipePivots(resourceType, alias, scope, pivots) if err != nil { @@ -684,10 +651,6 @@ func recipeNodeSelector(resourceType, alias string, scope scopeFrame, input reci return selector, nil } -// lowerRecipePredicate maps the only predicate shape currently representable -// by SemanticAggregate and SemanticSlice: selector existence or string/code -// equality. Richer boolean predicates remain ordinary node filters until the -// canonical physical predicate IR grows an equivalent typed representation. func lowerRecipePredicate(resourceType, alias string, scope scopeFrame, input *recipe.Filter, path string) (*spec.Selector, string, spec.FilterValueKind, error) { if input == nil { return nil, "", "", nil @@ -733,9 +696,6 @@ func lowerRecipePredicate(resourceType, alias string, scope scopeFrame, input *r } } -// lowerRecipeTypedPredicate is the lossless semantic boundary for a rich -// shaping predicate. Keeping the complete TypedFilter here avoids silently -// dropping a quantifier or value kind when later compiler stages consume it. func lowerRecipeTypedPredicate(resourceType, alias string, scope scopeFrame, input *recipe.Filter, path string) (*spec.TypedFilter, error) { if input == nil { return nil, nil diff --git a/internal/dataframe/semantic/recipe_traversal.go b/internal/dataframe/semantic/recipe_traversal.go index e6d42fec1..0ddd5c23b 100644 --- a/internal/dataframe/semantic/recipe_traversal.go +++ b/internal/dataframe/semantic/recipe_traversal.go @@ -8,12 +8,6 @@ import ( "github.com/calypr/loom/internal/dataframe/spec" ) -// qualifyTraversalLocals makes the traversal node's local selectors explicit -// before expression checking. Recipe selectors are intentionally concise -// ("status", "type[].coding[].display") while the semantic AST needs a -// lexical alias to distinguish them from parent/root references. Existing -// aliases remain untouched so a recipe can deliberately read a visible parent -// binding. func qualifyTraversalLocals(input recipe.Traversal, alias string, visible map[string]scopeBinding) recipe.Traversal { copy := input copy.Fields = append([]recipe.Field(nil), input.Fields...) @@ -113,10 +107,6 @@ func qualifyRecipeSelector(input, alias string, visible map[string]scopeBinding) return alias + "." + strings.TrimPrefix(strings.TrimSpace(path), ".") + suffix } -// NormalizeRecipeMatchMode applies the persisted recipe default and returns -// the canonical semantic relationship mode. Empty and case-insensitive values -// remain OPTIONAL for version-1 recipe compatibility; unknown values fail -// before traversal lowering. func NormalizeRecipeMatchMode(mode recipe.TraversalMatchMode) (spec.TraversalMatchMode, error) { if !mode.Valid() { return "", fmt.Errorf("unsupported traversal match mode %q", mode) @@ -130,8 +120,3 @@ func NormalizeRecipeMatchMode(mode recipe.TraversalMatchMode) (spec.TraversalMat return "", fmt.Errorf("unsupported traversal match mode %q", mode) } } - -// LowerRecipeTraversalFilters performs the closed semantic portion of one -// traversal. The caller attaches the returned filters to the child -// SemanticNode, preserving the optional/required boundary for the common -// physical lowering path. diff --git a/internal/dataframe/semantic/resolved_plan.go b/internal/dataframe/semantic/resolved_plan.go index 83035d9f9..d364be562 100644 --- a/internal/dataframe/semantic/resolved_plan.go +++ b/internal/dataframe/semantic/resolved_plan.go @@ -9,18 +9,12 @@ import ( "strings" ) -// ResolvedColumn is a discovered output column whose name and logical value -// type have been frozen before execution. It is intentionally independent of -// ClickHouse or any other storage type. type ResolvedColumn struct { Output string DynamicName string Column Column } -// ResolvedRecipePlan is the only recipe representation accepted by a -// production execution/materialization adapter. Stored recipe data and -// request-scoped discovery are never mutated in place. type ResolvedRecipePlan struct { SemanticPlan RecipePlan ResolvedColumns map[string][]ResolvedColumn @@ -29,17 +23,10 @@ type ResolvedRecipePlan struct { SourceGeneration string } -// ResolveRecipePlan freezes all dynamic schemas and records the scope and -// source generation used to obtain them. A plan with no dynamic maps still -// receives a deterministic schema digest. Dynamic schemas must already have -// been resolved by the catalog-backed recipe resolver before this boundary. func ResolveRecipePlan(plan RecipePlan, scopeDigest, sourceGeneration string) (ResolvedRecipePlan, error) { if strings.TrimSpace(sourceGeneration) == "" { sourceGeneration = plan.Bindings.DatasetGeneration } - // An empty generation is the intentional legacy-null dataset namespace. It - // remains a precise scope when the catalog resolver used the same empty - // generation during discovery; physical lowering renders the nil bind. if strings.TrimSpace(scopeDigest) == "" { for _, output := range plan.Outputs { if outputHasDynamicMaps(output.Root) || len(output.DynamicMaps) > 0 { @@ -62,9 +49,6 @@ func ResolveRecipePlan(plan RecipePlan, scopeDigest, sourceGeneration string) (R return fmt.Errorf("output %q dynamic map %q is unresolved; resolve it through the catalog-backed recipe resolver first", output.Name, dynamic.Name) } if len(dynamic.Columns) == 0 { - // An empty catalog result is a valid optional family. The - // resolver has already frozen it to zero columns, so there is - // no schema or physical lookup to attach to this node. continue } candidates := make([]Candidate, 0, len(dynamic.Columns)) @@ -102,7 +86,6 @@ func ResolveRecipePlan(plan RecipePlan, scopeDigest, sourceGeneration string) (R return ResolvedRecipePlan{}, err } } - // Marshal map keys deterministically by normalizing through sorted keys. keys := make([]string, 0, len(resolved.ResolvedColumns)) for key := range resolved.ResolvedColumns { keys = append(keys, key) diff --git a/internal/dataframe/semantic/selection_semantics.go b/internal/dataframe/semantic/selection_semantics.go index adfc11fcc..fe36b4b2b 100644 --- a/internal/dataframe/semantic/selection_semantics.go +++ b/internal/dataframe/semantic/selection_semantics.go @@ -9,10 +9,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// PrimarySelector derives the schema selector used by the physical compiler -// from the checked primary expression. Semantic plans keep expressions as -// their single source of truth so diagnostics, type information, and -// lowering cannot observe different selector values. func (field SemanticField) PrimarySelector() (spec.Selector, error) { if field.Expr.Expression.Selector == nil { return spec.Selector{}, fmt.Errorf("field %q expression is not a selector", field.Name) @@ -24,10 +20,6 @@ func (field SemanticField) PrimarySelector() (spec.Selector, error) { return selector, nil } -// FallbackSelectors derives the ordered selector fallback chain from checked -// semantic expressions. Fallbacks are selector-only in the current recipe -// contract; rejecting anything else here keeps that constraint at the -// semantic/physical boundary instead of relying on a parallel selector list. func (field SemanticField) FallbackSelectors() ([]spec.Selector, error) { selectors := make([]spec.Selector, 0, len(field.Fallbacks)) for index, fallback := range field.Fallbacks { @@ -43,8 +35,6 @@ func (field SemanticField) FallbackSelectors() ([]spec.Selector, error) { return selectors, nil } -// SelectionSemanticSpec is the compiler-ready meaning of one field selection. -// It deliberately contains selectors and schema facts, not rendered AQL. type SelectionSemanticSpec struct { Alias string NodeAlias string @@ -58,7 +48,6 @@ type SelectionSemanticSpec struct { RepeatedPaths []string } -// ResolveSemanticField resolves a single field and all fallback selectors. func ResolveSemanticField(resourceType, nodeAlias string, index int, field SemanticField) (SelectionSemanticSpec, error) { if !fhirschema.DefinitionExists(resourceType) { return SelectionSemanticSpec{}, fmt.Errorf("field %q: FHIR definition %q is not in the active schema", field.Name, resourceType) @@ -91,9 +80,6 @@ func ResolveSemanticField(resourceType, nodeAlias string, index int, field Seman projection := field.Projection legacyAuto := false if projection == "" { - // Direct semantic graph callers may omit Projection. Preserve the - // historical AUTO behavior for that transport-neutral input while - // recipe plans record the resolved projection explicitly. projection, legacyAuto, err = projectionForValueMode("", cardinality) if err != nil { return SelectionSemanticSpec{}, fmt.Errorf("field %q: %w", field.Name, err) @@ -121,8 +107,6 @@ func projectionForValueMode(valueMode string, cardinality spec.Cardinality) (spe switch strings.ToUpper(strings.TrimSpace(valueMode)) { case "", "AUTO": if cardinality.AllowsMany() { - // Legacy AUTO selected FIRST for an array-bearing selector. Preserve - // that behavior as an explicit semantic decision, not compiler magic. return spec.ProjectionFirst, true, nil } return spec.ProjectionScalar, true, nil diff --git a/internal/dataframe/semantic/semantic_nodes.go b/internal/dataframe/semantic/semantic_nodes.go index fc4358e40..e123af3ac 100644 --- a/internal/dataframe/semantic/semantic_nodes.go +++ b/internal/dataframe/semantic/semantic_nodes.go @@ -7,9 +7,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// SemanticNode is the canonical backend-independent graph node used by every -// dataframe frontend. Runtime request provenance belongs in ExecutionContext, -// while output-specific shaping belongs in OutputPlan. type SemanticNode struct { OccurrenceID string Alias string @@ -28,14 +25,10 @@ type SemanticNode struct { } type SemanticField struct { - Name string - ColumnID string - Label string - FieldRef string - // Expr and Fallbacks are the checked semantic expressions that produced - // this field. Selectors are intentionally derived from these expressions - // at the physical boundary; keeping both representations here allowed the - // recipe planner and lowerer to silently diverge. + Name string + ColumnID string + Label string + FieldRef string Expr SemanticExpression Fallbacks []SemanticExpression Projection spec.ProjectionMode @@ -43,29 +36,24 @@ type SemanticField struct { } type SemanticPivot struct { - Name string - FieldRef string - ColumnSelector spec.Selector - ValueSelector spec.Selector - ValueFallbacks []spec.Selector - ValueKind expression.ValueKind - StringifyValue bool - ItemSource spec.Selector - ItemResourceType string - Columns []string - ColumnAliases map[string]string - ProjectionMode string - Family string - Discovered bool - // Correlation is the closed system+code/value binding used when a pivot - // must preserve one Coding and one owning repeated item. Nil retains the - // legacy validated pivot path. - Correlation *fhirschema.CorrelatedBinding - CorrelationSystem string - CorrelationCode string - CorrelationVersion string - // ExtensionCorrelation carries the checked ancestor-aware extension - // binding. It shares the physical correlation IR with terminology pivots. + Name string + FieldRef string + ColumnSelector spec.Selector + ValueSelector spec.Selector + ValueFallbacks []spec.Selector + ValueKind expression.ValueKind + StringifyValue bool + ItemSource spec.Selector + ItemResourceType string + Columns []string + ColumnAliases map[string]string + ProjectionMode string + Family string + Discovered bool + Correlation *fhirschema.CorrelatedBinding + CorrelationSystem string + CorrelationCode string + CorrelationVersion string ExtensionCorrelation *fhirschema.ExtensionBinding } @@ -77,20 +65,15 @@ type SemanticOwnerRecords struct { } type SemanticAggregate struct { - Name string - OutputName string - Operation string - FieldRef string - // Predicate is the complete canonical typed contributor predicate. It is - // retained until physical lowering so operator, quantifier, kind, and value - // cannot diverge across compiler stages. - Predicate *spec.TypedFilter - Selector *spec.Selector - ValueMode string - RequiredValues []string - ValueKind expression.ValueKind - // UnitNormalization is compiler-resolved and contains the pinned registry - // rules. Authoring/recipe references do not cross this boundary. + Name string + OutputName string + Operation string + FieldRef string + Predicate *spec.TypedFilter + Selector *spec.Selector + ValueMode string + RequiredValues []string + ValueKind expression.ValueKind UnitNormalization *unit.UnitNormalization UnitSystemSelector *spec.Selector UnitCodeSelector *spec.Selector diff --git a/internal/dataframe/semantic/semantic_validation.go b/internal/dataframe/semantic/semantic_validation.go index ad8eedf5c..bf11ffd67 100644 --- a/internal/dataframe/semantic/semantic_validation.go +++ b/internal/dataframe/semantic/semantic_validation.go @@ -7,9 +7,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// ValidateSemanticGraph checks the schema and structural safety of an already -// constructed semantic graph. It performs no observed-data or authorization -// checks and does not mutate the graph. func ValidateSemanticGraph(root SemanticNode) error { if strings.TrimSpace(root.ResourceType) == "" { return fmt.Errorf("semantic graph root resource type is required") @@ -66,10 +63,6 @@ func (s *semanticValidationState) validateChildren(parent SemanticNode, path []s } edge := fmt.Sprintf("%s -[%s]-> %s", parent.ResourceType, child.EdgeLabel, child.ResourceType) - // Semantic plans are finite values: every child is explicitly present in - // the request and therefore has a bounded traversal route. Repeating an - // edge (including a self-reference) is valid and is left to the explicit - // physical cost policy rather than an authoring-time hop or cycle cap. if err := s.validateChildren(child, append(path, edge)); err != nil { return err } diff --git a/internal/dataframe/spec/doc.go b/internal/dataframe/spec/doc.go index d6547920e..f09cd57a1 100644 --- a/internal/dataframe/spec/doc.go +++ b/internal/dataframe/spec/doc.go @@ -1,4 +1 @@ -// Package spec defines the backend-independent dataframe request language. -// FHIR schema metadata validates selectors and typed filters here; AQL and -// Arango execution do not belong in this package. package spec diff --git a/internal/dataframe/spec/filter.go b/internal/dataframe/spec/filter.go index b800a1ffe..8c4ccb2e5 100644 --- a/internal/dataframe/spec/filter.go +++ b/internal/dataframe/spec/filter.go @@ -10,9 +10,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// FilterOperator is the closed set of operations accepted by the typed filter -// AST. Compilers must map these values to bound expressions rather than treat -// them as query fragments. type FilterOperator string const ( @@ -48,8 +45,6 @@ const ( QuantifierNone ArrayQuantifier = "NONE" ) -// CodeValue preserves the terminology identity instead of filtering only on a -// human-readable display string. type CodeValue struct { System string `json:"system,omitempty"` Code string `json:"code"` @@ -57,8 +52,6 @@ type CodeValue struct { Version string `json:"version,omitempty"` } -// FilterValue is a tagged value. Exactly the member selected by Kind must be -// populated. Pointer scalar fields preserve false, zero, and empty values. type FilterValue struct { Kind FilterValueKind `json:"kind"` String *string `json:"string,omitempty"` @@ -70,13 +63,8 @@ type FilterValue struct { DateTime *string `json:"dateTime,omitempty"` } -// TypedFilter identifies a field through a stable reference. It deliberately -// contains no AQL or selector expression supplied by the user. type TypedFilter struct { - FieldRef string `json:"fieldRef"` - // Selector is a resolved, canonical FHIR selector. Product callers should - // normally supply FieldRef and receive this selector from the semantic - // registry; the compiler requires it before physical lowering. + FieldRef string `json:"fieldRef"` Selector string `json:"selector,omitempty"` FieldKind FilterValueKind `json:"fieldKind"` Repeated bool `json:"repeated,omitempty"` @@ -247,10 +235,6 @@ func (v FilterValue) validate(requireCode bool) error { return nil } -// Ordered comparison has point-in-time semantics, unlike exact FHIR temporal -// matching which can legitimately use partial precision (for example, 2024). -// Require full values before emitting DATE_TIMESTAMP so offsets are normalized -// in AQL rather than compared lexically. func validateOrderedTemporalValue(operator FilterOperator, value FilterValue) error { switch operator { case FilterGreaterThan, FilterGreaterEq, FilterLessThan, FilterLessEq: diff --git a/internal/dataframe/spec/filter_semantics.go b/internal/dataframe/spec/filter_semantics.go index ba30b23a5..63b351078 100644 --- a/internal/dataframe/spec/filter_semantics.go +++ b/internal/dataframe/spec/filter_semantics.go @@ -7,11 +7,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// ValidateTypedFilterForResource proves that a resolved filter selector has a -// compatible shape in the active generated FHIR schema. Runtime catalog -// validation remains responsible for whether that field is populated in the -// selected project; this function prevents a request from using a value type -// that cannot be represented by the generated resource definition. func ValidateTypedFilterForResource(resourceType string, filter TypedFilter) error { if err := filter.Validate(); err != nil { return err @@ -68,9 +63,6 @@ func filterKindMatchesGeneratedPrimitive(kind FilterValueKind, primitive fhirsch if kind == FilterString { return true } - // A code filter is safe only when the resolved generated selector ends - // in a code primitive. This remains deliberately conservative: fields - // such as Patient.gender are strings, not terminology codes. return kind == FilterCode && (canonicalPath == "code" || strings.HasSuffix(canonicalPath, ".code")) case fhirschema.PrimitiveBoolean: return kind == FilterBoolean diff --git a/internal/dataframe/spec/grain.go b/internal/dataframe/spec/grain.go index 98cd68c96..d0b97fa33 100644 --- a/internal/dataframe/spec/grain.go +++ b/internal/dataframe/spec/grain.go @@ -8,13 +8,9 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// RowGrain identifies the resource represented by one output row. type RowGrain string const ( - // RowGrainResource is the safe fallback for a generated FHIR resource that - // does not yet have a product-specific "one row per" label. It still has a - // stable root resource identity and is therefore safe to compile. RowGrainResource RowGrain = "resource" RowGrainPatient RowGrain = "patient" RowGrainSpecimen RowGrain = "specimen" @@ -22,12 +18,8 @@ const ( RowGrainDiagnosis RowGrain = "diagnosis" RowGrainObservation RowGrain = "observation" RowGrainStudyEnrollment RowGrain = "study_enrollment" - // RowGrainExpanded is a synthetic grain whose identity is supplied by a - // recipe expansion expression rather than by the root resource key. - RowGrainExpanded RowGrain = "expanded" - // RowGrainGroups is a synthetic grain whose identity is an immutable - // explicit group in a pinned group revision. - RowGrainGroups RowGrain = "groups" + RowGrainExpanded RowGrain = "expanded" + RowGrainGroups RowGrain = "groups" ) func (g RowGrain) Validate() error { @@ -42,10 +34,6 @@ func (g RowGrain) Validate() error { } } -// InferRowGrain returns the product grain naturally represented by a generated -// FHIR root type. Valid schema resource types without a named product grain use -// RowGrainResource so every valid semantic request still carries a stable row -// identity. func InferRowGrain(resourceType string) (RowGrain, bool) { canonical, ok := fhirschema.CanonicalResourceType(resourceType) if !ok { @@ -69,9 +57,6 @@ func InferRowGrain(resourceType string) (RowGrain, bool) { } } -// RootResourceForGrain returns the only root resource type currently capable -// of representing a named product grain without row expansion. The generic -// resource grain deliberately accepts any generated resource root. func RootResourceForGrain(grain RowGrain) (string, bool) { switch grain { case RowGrainResource: @@ -95,10 +80,6 @@ func RootResourceForGrain(grain RowGrain) (string, bool) { } } -// ValidateRootGrain prevents an API caller from asking for a named row grain -// while compiling rows rooted at a different resource. Cross-grain output will -// require an explicit future EXPLODE/root-rewrite operation; accepting it now -// would silently lie about the output identity. func ValidateRootGrain(resourceType string, grain RowGrain) error { if err := grain.Validate(); err != nil { return err @@ -117,9 +98,6 @@ func ValidateRootGrain(resourceType string, grain RowGrain) error { return nil } -// DefaultRowIdentity uses the immutable project/key pair emitted by the graph -// loader. The grain remains explicit so downstream code cannot silently use a -// related resource as the output identity. func DefaultRowIdentity(grain RowGrain) (RowIdentity, bool) { if err := grain.Validate(); err != nil { return RowIdentity{}, false @@ -130,7 +108,6 @@ func DefaultRowIdentity(grain RowGrain) (RowIdentity, bool) { return RowIdentity{Grain: grain, Fields: []string{"project", "_key"}}, true } -// ProjectionMode defines how values at a selected path contribute to a row. type ProjectionMode string const ( @@ -156,13 +133,8 @@ func (m ProjectionMode) Validate() error { } } -// ExpandsRows reports whether the projection deliberately changes row -// multiplicity instead of reducing values into the current row grain. func (m ProjectionMode) ExpandsRows() bool { return m == ProjectionExplode } -// Cardinality describes the formal multiplicity of a relationship or field. -// Observed data may refine UnknownObservedMany, but must not weaken a declared -// required, optional, or repeated relationship. type Cardinality string const ( @@ -190,8 +162,6 @@ func (c Cardinality) AllowsMany() bool { func (c Cardinality) IsRequired() bool { return c == CardinalityRequiredOne } -// ValidateProjection rejects implicit row expansion. Repeated values must be -// reduced explicitly or deliberately exploded into rows. func ValidateProjection(cardinality Cardinality, mode ProjectionMode) error { if err := cardinality.Validate(); err != nil { return err @@ -205,16 +175,11 @@ func ValidateProjection(cardinality Cardinality, mode ProjectionMode) error { return nil } -// RowIdentity declares the stable fields that uniquely identify a row at a -// particular grain. Fields are ordered so callers can construct deterministic -// composite keys. type RowIdentity struct { Grain RowGrain Fields []string } -// Clone returns an independent row-identity value for transport between -// compiler stages. func (i *RowIdentity) Clone() *RowIdentity { if i == nil { return nil diff --git a/internal/dataframe/spec/grain_classification_test.go b/internal/dataframe/spec/grain_classification_test.go index 229203a1b..6df3b4df2 100644 --- a/internal/dataframe/spec/grain_classification_test.go +++ b/internal/dataframe/spec/grain_classification_test.go @@ -6,10 +6,6 @@ import ( dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" ) -// The explorer row-source boundary (explorer/lifecycle source_row.go) calls -// these validators without a prior schema gate, so their rejections are -// directly user-facing and must arrive typed and non-retryable. - func assertTypedUserFault(t *testing.T, err error, wantMessage string) { t.Helper() if err == nil { diff --git a/internal/dataframe/spec/relationship_match.go b/internal/dataframe/spec/relationship_match.go index 35815be9c..87f84355c 100644 --- a/internal/dataframe/spec/relationship_match.go +++ b/internal/dataframe/spec/relationship_match.go @@ -4,13 +4,6 @@ import ( "fmt" ) -// TraversalMatchMode controls whether a relationship contributes values to a -// dataframe row only, or must exist for that root row to be included at all. -// -// OPTIONAL is the legacy behavior: a missing child yields an empty child -// projection but does not remove the root row. REQUIRED is intentionally -// opt-in and lowers to a root-scoped semi-join. The empty value is OPTIONAL so -// callers retain the same schema-backed relationship behavior. type TraversalMatchMode string const ( diff --git a/internal/dataframe/spec/selector_validation.go b/internal/dataframe/spec/selector_validation.go index 13c8de012..67a41adb0 100644 --- a/internal/dataframe/spec/selector_validation.go +++ b/internal/dataframe/spec/selector_validation.go @@ -7,9 +7,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// selectorCardinality is schema validation shared by typed filters and -// semantic selection normalization. It reports repeated paths without making -// a physical projection or AQL decision. func selectorCardinality(resourceType string, selector Selector) (bool, []string, error) { if len(selector.Steps) == 0 { return false, nil, fmt.Errorf("selector is required") @@ -39,8 +36,6 @@ func selectorCardinality(resourceType string, selector Selector) (bool, []string return len(repeatedPaths) > 0, repeatedPaths, nil } -// SelectorCardinality exposes schema-only cardinality to the semantic layer. -// It deliberately does not expose any physical lowering decision. func SelectorCardinality(resourceType string, selector Selector) (bool, []string, error) { return selectorCardinality(resourceType, selector) } diff --git a/internal/dataframe/spec/selectors.go b/internal/dataframe/spec/selectors.go index edc0523d8..901b84278 100644 --- a/internal/dataframe/spec/selectors.go +++ b/internal/dataframe/spec/selectors.go @@ -15,9 +15,6 @@ func ParseSelector(input string) (Selector, error) { return fhirschema.ParseSelector(input) } -// ParseDirectScalarSelector accepts only a non-repeating property path. It -// rejects array traversal, indexing, and filters so field choices can be -// lowered as one exact payload lookup with no implicit collection behavior. func ParseDirectScalarSelector(input string) ([]string, error) { if strings.TrimSpace(input) == "" || strings.TrimSpace(input) != input { return nil, fmt.Errorf("selector must be exact and non-empty") diff --git a/internal/dataframe/unit/arithmetic.go b/internal/dataframe/unit/arithmetic.go index 84e220819..ce969aee6 100644 --- a/internal/dataframe/unit/arithmetic.go +++ b/internal/dataframe/unit/arithmetic.go @@ -2,7 +2,6 @@ package unit import "fmt" -// ArithmeticOperation is one of the supported numeric operators. type ArithmeticOperation uint8 const ( @@ -13,7 +12,6 @@ const ( ArithmeticDivide ) -// String returns the stable authoring spelling of the operation. func (operation ArithmeticOperation) String() string { switch operation { case ArithmeticAdd: @@ -37,14 +35,11 @@ const ( literalArithmeticOperand ) -// ArithmeticOperand distinguishes a column, which may carry a normalized -// unit identity, from a numeric literal, which is always unitless. type ArithmeticOperand struct { kind arithmeticOperandKind unit *UnitIdentity } -// ColumnArithmeticOperand constructs a column operand with its normalized unit, if any. func ColumnArithmeticOperand(normalizedUnit *UnitIdentity) ArithmeticOperand { if normalizedUnit == nil { return ArithmeticOperand{kind: columnArithmeticOperand} @@ -53,12 +48,10 @@ func ColumnArithmeticOperand(normalizedUnit *UnitIdentity) ArithmeticOperand { return ArithmeticOperand{kind: columnArithmeticOperand, unit: &identity} } -// LiteralArithmeticOperand constructs a unitless numeric-literal operand. func LiteralArithmeticOperand() ArithmeticOperand { return ArithmeticOperand{kind: literalArithmeticOperand} } -// ArithmeticUnitErrorReason identifies why an arithmetic result unit is unsupported. type ArithmeticUnitErrorReason string const ( @@ -69,7 +62,6 @@ const ( ArithmeticUnitUnrepresentable ArithmeticUnitErrorReason = "UNREPRESENTABLE_RESULT" ) -// ArithmeticUnitError reports unit incompatibility for a derived operation. type ArithmeticUnitError struct { Operation ArithmeticOperation Reason ArithmeticUnitErrorReason @@ -79,9 +71,6 @@ func (e *ArithmeticUnitError) Error() string { return fmt.Sprintf("arithmetic operation %s unit resolution failed: %s", e.Operation.String(), e.Reason) } -// ResolveArithmeticUnit checks the supported dimensional rules and returns -// the normalized identity of the result. A nil identity means the result is -// unitless. func ResolveArithmeticUnit(operation ArithmeticOperation, left, right ArithmeticOperand) (*UnitIdentity, error) { if !validArithmeticOperation(operation) { return nil, &ArithmeticUnitError{Operation: operation, Reason: ArithmeticUnitInvalidOperation} diff --git a/internal/dataframe/unit/unit.go b/internal/dataframe/unit/unit.go index f21dee3d5..8bbe00e0a 100644 --- a/internal/dataframe/unit/unit.go +++ b/internal/dataframe/unit/unit.go @@ -13,9 +13,6 @@ var ( ErrUnitDimensionIncompatible = errors.New("UNIT_DIMENSION_INCOMPATIBLE") ) -// UnitIdentity is the stable identity of a source or target unit. Display -// labels are intentionally absent. FHIR Quantity.unit is a human label and -// must never select a conversion rule. type UnitIdentity struct { System string `json:"system"` Code string `json:"code"` @@ -40,9 +37,6 @@ func (u UnitIdentity) Equal(other UnitIdentity) bool { return a.System == b.System && a.Code == b.Code } -// UnitDimension is a dimensional-analysis vector. The named exponents cover -// the dimensions supported by the initial approved conversion library. A -// conversion is valid only when every exponent matches the target. type UnitDimension struct { Length int `json:"length,omitempty"` Mass int `json:"mass,omitempty"` @@ -65,9 +59,6 @@ const ( UnitConversionAffine UnitConversionKind = "AFFINE" ) -// UnitRuleReference is the only unit rule shape accepted from an authoring -// document. The compiler resolves it against the immutable registry below; -// callers cannot supply executable coefficients or dimensions. type UnitRuleReference struct { ID string `json:"id"` Version string `json:"version"` @@ -80,17 +71,12 @@ type ApprovedUnitPolicy struct { Rules []UnitRuleReference } -// UnitNormalization is the compiler-owned, resolved policy carried into the -// backend-neutral semantic and physical plans. type UnitNormalization struct { Target UnitIdentity Dimension UnitDimension Rules []UnitConversionRule } -// UnitConversionRule is an immutable, pinned interpretation rule. Scale and -// offset are applied as output = input*scale + offset. Identity and linear -// rules require zero offset; affine rules may carry a non-zero offset. type UnitConversionRule struct { ID string `json:"id"` Version string `json:"version"` @@ -102,9 +88,6 @@ type UnitConversionRule struct { Offset float64 `json:"offset"` } -// ValidateUnitRuleReference validates the client-supplied part of a policy. -// Coefficients and dimensions are resolved from the compiler-owned registry; -// callers cannot make an arbitrary rule executable by choosing its numbers. func ValidateUnitRuleReference(rule UnitRuleReference) error { if strings.TrimSpace(rule.ID) == "" || strings.TrimSpace(rule.Version) == "" { return fmt.Errorf("unit conversion rule id and version are required") @@ -181,9 +164,6 @@ func (policy ApprovedUnitPolicy) SupportsSources(sources []UnitIdentity) bool { return true } -// ResolveApprovedUnitRules replaces rule references with immutable registry -// entries. Identity is the only target-relative rule: it derives source and -// target from the policy target and still pins a named registry version. func ResolveApprovedUnitRules(references []UnitRuleReference, target UnitIdentity) (UnitDimension, []UnitConversionRule, error) { if !target.Valid() { return UnitDimension{}, nil, fmt.Errorf("%w: target system and code are required", ErrUnitIdentityUnknown) diff --git a/internal/dataset/arango/collections.go b/internal/dataset/arango/collections.go deleted file mode 100644 index 71b0d6cbc..000000000 --- a/internal/dataset/arango/collections.go +++ /dev/null @@ -1,31 +0,0 @@ -package arango - -import arangostore "github.com/calypr/loom/internal/store/arango" - -const ( - // LifecycleCollection stores two internal document shapes: immutable - // generation manifests and one active-generation pointer per project. They - // intentionally share a collection so activation can use one AQL UPDATE - // operation to change both records atomically. - LifecycleCollection = "loom_dataset_lifecycle" - - manifestRecordType = "manifest" - activeRecordType = "active_generation" - snapshotRecordType = "snapshot_generation" - releaseRecordType = "project_release" - activeReleaseRecordType = "active_project_release" -) - -// CollectionSpecs returns a fresh bootstrap specification for persistent -// dataset lifecycle metadata. It never requests truncation: a FHIR reload -// must not erase manifest history or an active-generation selection. -func CollectionSpecs() []arangostore.CollectionSpec { - return []arangostore.CollectionSpec{{Name: LifecycleCollection}} -} - -// BootstrapSpec returns the Arango bootstrap work needed by this adapter. -// Callers own when to bootstrap it; this package never wires the collection -// into ingest's truncate-oriented bootstrap path. -func BootstrapSpec() arangostore.BootstrapSpec { - return arangostore.BootstrapSpec{Collections: CollectionSpecs()} -} diff --git a/internal/dataset/arango/doc.go b/internal/dataset/arango/doc.go deleted file mode 100644 index b7819781d..000000000 --- a/internal/dataset/arango/doc.go +++ /dev/null @@ -1,13 +0,0 @@ -// Package arango persists the immutable dataset generation lifecycle in -// ArangoDB. -// -// The adapter deliberately owns only manifests and the selected active -// generation. It does not load FHIR resources, build a catalog, resolve -// authorization, execute dataframes, or expose a public API. In particular it -// never stores raw authorization paths, tokens, subjects, or claims: -// authorization scope is a per-read concern, not persistent generation -// metadata. -// -// Manifest and active-pointer records share one collection so activation is a -// single revision-checked pointer update. -package arango diff --git a/internal/dataset/dataframe_selector.go b/internal/dataset/dataframe_selector.go index 2e7807434..81b03996c 100644 --- a/internal/dataset/dataframe_selector.go +++ b/internal/dataset/dataframe_selector.go @@ -5,8 +5,6 @@ import ( "strings" ) -// DataframeSelector is the exact, versioned identity of one recipe output. -// It is shared by publication, release activation, GraphQL, and ETL. type DataframeSelector struct { Recipe string `json:"recipe"` TranslationVersion string `json:"translationVersion"` @@ -24,13 +22,10 @@ func (s DataframeSelector) Validate() error { return nil } -// Valid reports whether the selector contains three non-blank components. -// It is retained as a convenience for callers that only need a boolean. func (s DataframeSelector) Valid() bool { return s.Validate() == nil } -// Key is a collision-safe stable key for maps and persisted documents. func (s DataframeSelector) Key() string { return fmt.Sprintf("%d:%s%d:%s%d:%s", len(s.Recipe), s.Recipe, len(s.TranslationVersion), s.TranslationVersion, len(s.Output), s.Output) } diff --git a/internal/dataset/doc.go b/internal/dataset/doc.go index fbf73fba5..8ff540fd4 100644 --- a/internal/dataset/doc.go +++ b/internal/dataset/doc.go @@ -1,3 +1 @@ -// Package publication defines the manifest and active-pointer contract that -// makes a completed ingest visible to readers. package dataset diff --git a/internal/dataset/manifest.go b/internal/dataset/manifest.go index cf1b4755d..3afac2390 100644 --- a/internal/dataset/manifest.go +++ b/internal/dataset/manifest.go @@ -7,11 +7,8 @@ type State string const ( StateLoading State = "LOADING" StateStaged State = "STAGED" - // StateReady is retained for compatibility with manifests written before - // staged project releases became the visibility boundary. New workflows - // must write StateStaged. - StateReady State = "READY" - StateFailed State = "FAILED" + StateReady State = "READY" + StateFailed State = "FAILED" ) type Manifest struct { @@ -52,8 +49,6 @@ func (m Manifest) Transition(next State) (Manifest, error) { return m, nil } -// IsStaged reports whether graph loading completed. READY is accepted only as -// a legacy spelling; new transitions use STAGED. func (m Manifest) IsStaged() bool { return m.Validate() == nil && (m.State == StateStaged || m.State == StateReady) } diff --git a/internal/dataset/manifest_port.go b/internal/dataset/manifest_port.go new file mode 100644 index 000000000..80a146977 --- /dev/null +++ b/internal/dataset/manifest_port.go @@ -0,0 +1,11 @@ +package dataset + +import "context" + +type ManifestLifecycle interface { + CreateManifest(ctx context.Context, manifest Manifest) (Manifest, error) + ReadManifest(ctx context.Context, ref Ref) (Manifest, error) + TransitionManifest(ctx context.Context, manifest Manifest, next State) (Manifest, error) + ResolveActiveManifest(ctx context.Context, project string) (Manifest, error) + ActivateGeneration(ctx context.Context, candidate Manifest, expected *Ref) error +} diff --git a/internal/dataset/manifest_test.go b/internal/dataset/manifest_test.go index 4e7fa0485..793496436 100644 --- a/internal/dataset/manifest_test.go +++ b/internal/dataset/manifest_test.go @@ -41,12 +41,6 @@ func TestManifestTransitions(t *testing.T) { } } -// TestManifestTerminalStatesRejectEveryOutgoingTransition closes the gap -// finding 2026-11-04k (adversarial review, repair-27): TestManifestTransitions -// only probes FAILED/READY -> LOADING. A widening of Transition's source guard -// that admitted FAILED -> STAGED had no coverage anywhere in the repository, -// and FAILED -> STAGED -> Activate is a second route to activating a failed -// generation (activateAQL accepts any persisted STAGED document). func TestManifestTerminalStatesRejectEveryOutgoingTransition(t *testing.T) { for _, terminal := range []State{StateFailed, StateReady} { source, err := testManifest(t).Transition(terminal) diff --git a/internal/dataset/ref.go b/internal/dataset/ref.go index 57c2dbe23..1ef190626 100644 --- a/internal/dataset/ref.go +++ b/internal/dataset/ref.go @@ -5,16 +5,12 @@ import ( "strings" ) -// Ref names one immutable generation in one project. type Ref struct { Project string `json:"project"` Generation string `json:"generation"` } func NewRef(project, generation string) (Ref, error) { - // Clone transport-provided strings before persisting them. Fiber/fasthttp - // path parameter strings may otherwise alias a request buffer that is reused - // after the handler returns. ref := Ref{Project: strings.Clone(project), Generation: strings.Clone(generation)} if err := ref.Validate(); err != nil { return Ref{}, err diff --git a/internal/dataset/release.go b/internal/dataset/release.go index b7419425c..8467bda80 100644 --- a/internal/dataset/release.go +++ b/internal/dataset/release.go @@ -47,9 +47,6 @@ type ContractVerification struct { ErrorCode string `json:"errorCode,omitempty"` } -// ProjectRelease is immutable. Active visibility is represented only by the -// release pointer revision stored alongside it. Inactive candidates remain -// retained for audit; this package does not delete release history. type ProjectRelease struct { ID string `json:"id"` Project string `json:"project"` @@ -82,11 +79,6 @@ func (e *ReleaseRequirementsError) Error() string { } func (e *ReleaseRequirementsError) Unwrap() error { return ErrReleaseRequirementsUnmet } -// ErrPublicationRecordMissing marks a deterministic absence of the durable -// publication record for a selector. The publication store maps its not-found -// sentinel to this error so release activation can separate permanent contract -// misses (typed requirements failures, non-retryable) from transient -// verification-store faults (plain errors, retryable by clients). var ErrPublicationRecordMissing = errors.New("publication record is missing for selector") type PublicationVerifier interface { @@ -98,15 +90,11 @@ type ManifestReader interface { } type ReleaseRepository interface { - // SaveRelease retains the immutable candidate even when a later activation - // compare-and-swap loses a revision race. SaveRelease(context.Context, ProjectRelease) (ProjectRelease, error) ReadActiveRelease(context.Context, string) (ActiveRelease, error) CompareAndSwapActivateRelease(context.Context, ProjectRelease, int64) (ActiveRelease, error) } -// ActiveReleaseResolver exposes the immutable publication bindings selected by -// the active project release without granting readers mutation access. type ActiveReleaseResolver interface { ReadActiveRelease(context.Context, string) (ActiveRelease, error) } @@ -119,10 +107,6 @@ type ReleaseService struct { Now func() time.Time } -// ValidateGeneration verifies the immutable graph-generation prerequisite used -// by release creation. Callers that perform expensive work before activating a -// release can use this as a preflight and avoid producing orphaned dataframe -// publications for a generation that cannot be activated. func (s ReleaseService) ValidateGeneration(ctx context.Context, project, generation string) error { if s.Manifests == nil { return fmt.Errorf("manifest reader is required") @@ -167,7 +151,7 @@ func (s ReleaseService) Create(ctx context.Context, request ActivationRequest) ( if previousErr != nil && !errors.Is(previousErr, ErrNoActiveRelease) { return ProjectRelease{}, previousErr } - _ = previous // kept as fail-fast validation of the active pointer; the ID no longer folds the revision (F-B15). + _ = previous for _, selector := range s.Required { if err := selector.Validate(); err != nil { return ProjectRelease{}, fmt.Errorf("invalid required selector: %w", err) @@ -188,9 +172,6 @@ func (s ReleaseService) Create(ctx context.Context, request ActivationRequest) ( contract := contractVerification(selector, verification, verifyErr) verifications = append(verifications, contract) if verifyErr != nil && !errors.Is(verifyErr, ErrPublicationRecordMissing) { - // A verification-store fault is not a contract verdict: keep it a - // plain error so callers classify it retryable/unavailable instead - // of telling the client to fix data that is already correct. return ProjectRelease{}, fmt.Errorf("publication verification store fault for required selector %s: %w", selector.Key(), verifyErr) } if verifyErr != nil || verification.Selector != selector || verification.State != "PUBLISHED" || !verification.Queryable || verification.Generation != ref.Generation { @@ -235,11 +216,6 @@ func (s ReleaseService) Create(ctx context.Context, request ActivationRequest) ( Project: ref.Project, GitCommit: ref.Generation, Generation: ref.Generation, Publications: publications, RequiredVerifications: verifications, CreatedAt: now, } - // F-B15 repair: the ID is CONTENT-ADDRESSED (revision excluded). The - // activation CAS's alreadyActive guard needs the SAME releaseId to make an - // identical re-activation a no-op; folding the live revision into the ID - // made every authoring click mint a new immutable document. Create stays - // deterministic for byte-identical requests regardless of revision churn. release.ID = releaseID(release) return s.Releases.SaveRelease(ctx, release) } @@ -286,101 +262,14 @@ func uniqueSelectors(values []DataframeSelector) []DataframeSelector { return result } -// releaseIDDomain namespaces every release identity so a later framing change -// can never collide with a digest an older binary derived from a narrower -// field set (the pre-widening encode hashed only Project, Generation and each -// publication's Selector.Key() plus ExecutionID). const releaseIDDomain = "loom-release-id-v2|" -// releaseID fingerprints the content-addressed identity of a ProjectRelease. -// An identical re-submission reproduces the same id regardless of revision -// churn (F-B15), and a DISTINCT immutable snapshot always produces a distinct -// id. -// -// FIELD SCOPE (why every persisted immutable category must feed the digest). -// The stores key the immutable document on this id -// (internal/dataset/arango/release_store.go: releaseDocument["_key"] = -// releaseDocumentKey(release.ID)) and refuse to overwrite an existing key whose -// content differs, so the digest must separate exactly the fields the document -// persists. A framing over Project/Generation/(Selector.Key(), ExecutionID) -// only — the pre-widening shape — folded two genuinely different candidates: -// reclassifying the same selector under the same execution (Required true -> -// false, or a Stale carry-forward) kept every hashed field, so SaveRelease -// derived the EXISTING key and rejected the new candidate with "release already -// exists with different content" and the policy change could never be saved or -// activated. The digest now covers Project, GitCommit, Generation, every -// ReleasePublication field (Selector, ExecutionID, Generation, Required, Stale, -// VerifiedAt) and every ContractVerification field (Selector, ExecutionID, -// Generation, State, Queryable, VerifiedAt, ErrorCode). Only the derived ID and -// the creation timestamp stay excluded. -// -// Exclusions and canonicalization: -// - CreatedAt is the first-creation timestamp and is excluded: both stores -// keep the original document untouched on a re-save, so it cannot drift, -// and folding it would make an identical snapshot retry mint a new identity. -// Per-execution VerifiedAt values are NOT excluded — they are immutable -// evidence carried inside the document. -// - Publications and RequiredVerifications are hashed as SETS in canonical -// Selector.Key() order (the order Create itself emits, release.go:229), so -// equivalent submissions in different array order derive one id. -// - Timestamps canonicalize to their UTC instant, so the same instant in -// another zone (or carrying a monotonic reading) hashes alike while a -// genuinely different instant does not. -// - A nil and an empty RequiredVerifications slice are one service form. -// -// Injectivity (hardening, defense-in-depth). Every field is written -// length-prefixed, exactly like Selector.Key() (dataframe_selector.go:34) and -// BundleIdentity.PointerName() (F-B21, -// internal/dataframe/publication/bundle.go:100-115): a scalar becomes -// len:content, Selector.Key() is emitted verbatim because it already -// self-delimits, each repeated section is count-prefixed, and a bool is one -// fixed byte. No field content — NUL, ':', digits, or otherwise — can shift a -// boundary, so the encoder is injective by construction and cannot be broken -// even if an upstream validator is weakened. The archived fold classes (a NUL -// blurred across a bare join, an un-delimited ExecutionID impersonating the -// next publication group) are replayed against the pinned historical framing in -// release_id_framing_test.go, which asserts they still collapse there and are -// separated here. -// -// OUTPUT-STABILITY ANALYSIS (derived from the store code). Changing this -// function changes every digest, and releaseId IS persisted and re-compared -// across binary versions: internal/dataset/arango/release_store.go keys the -// document on it ("releaseDocument[\"_key\"] = releaseDocumentKey(release.ID)" -// at L17/L73), stores the id field, and patches it onto the pointers -// ("releaseKey: @release_key, releaseId: @release_id" at L176-177). It is -// re-compared against a freshly derived id in activateReleaseAQL: -// -// L162 LET alreadyActive = releasePointer != null AND releasePointer.releaseId == @release_id -// L165 FILTER ... storedRelease.id == @release_id ... -// -// Yet the only genuine CROSS-VERSION re-derivation is L162's alreadyActive -// idempotency guard, and readActiveReleaseAQL never re-derives the id — it -// follows the stored key ("LET release = DOCUMENT(@@lifecycle_collection, -// pointer.releaseKey)" at L151) — so every reader treats releaseId as an -// opaque immutable content label. Therefore a digest change cannot yield a -// wrong serve, orphan active content, or produce an incorrect verdict: at the -// upgrade boundary the sole effect is that an already-active release's first -// content-identical re-activation misses the L162 no-op and mints one -// duplicate, content-identical immutable document plus a single revision bump -// under the new id, after which alreadyActive hits again (SaveRelease keeps -// the old document untouched via its same-version `compatible` guard at -// L139-141). That bounded, self-converging, security-neutral churn is the -// whole cross-version cost, and no correctness or security guarantee depends -// on historical digests staying bit-stable, so — matching the F-B21 precedent -// which likewise made a persisted content-derived identity injective by -// construction — we adopt the widened length-prefixed encode. func releaseID(release ProjectRelease) string { digest := sha256.New() _, _ = digest.Write(releaseIDFraming(release)) return "release_" + hex.EncodeToString(digest.Sum(nil)) } -// releaseIDFraming renders the exact byte string hashed by releaseID so the -// injectivity controls can decode and round-trip it without duplicating the -// framing in the test. Length-prefixing every field and count-prefixing every -// repeated section makes the string uniquely decodable: read len digits, a ':', -// then len bytes, and repeat — Selector.Key() consumes three such prefixed -// sub-fields, so it needs no separator of its own. func releaseIDFraming(release ProjectRelease) []byte { var frame strings.Builder frame.WriteString(releaseIDDomain) @@ -411,11 +300,6 @@ func releaseIDFraming(release ProjectRelease) []byte { return []byte(frame.String()) } -// canonicalPublications returns the publication SET in canonical order, -// mirroring the ordering ReleaseService.Create applies, so equivalent -// submissions derive one digest. The tie-break on the whole binding keeps the -// order total when two bindings share a Selector.Key() (Create's map makes -// that unreachable, but a total order removes any reliance on sort stability). func canonicalPublications(publications []ReleasePublication) []ReleasePublication { out := append(make([]ReleasePublication, 0, len(publications)), publications...) sort.Slice(out, func(i, j int) bool { @@ -428,8 +312,6 @@ func canonicalPublications(publications []ReleasePublication) []ReleasePublicati return out } -// canonicalVerifications returns the verification SET in canonical order; nil -// and empty are one service form. func canonicalVerifications(verifications []ContractVerification) []ContractVerification { if len(verifications) == 0 { return nil @@ -466,9 +348,6 @@ func contractVerificationSignature(verification ContractVerification) string { return signature.String() } -// canonicalReleaseTime renders one instant in exactly one way: the UTC -// RFC3339Nano form, with the zero instant as the empty string. Location and -// monotonic readings are transport detail, not immutable content. func canonicalReleaseTime(value time.Time) string { if value.IsZero() { return "" diff --git a/internal/dataset/release_id_framing_test.go b/internal/dataset/release_id_framing_test.go index 9afa13d73..76a9f3cc9 100644 --- a/internal/dataset/release_id_framing_test.go +++ b/internal/dataset/release_id_framing_test.go @@ -12,31 +12,6 @@ import ( "time" ) -// Controls for the releaseID content-addressed framing (release.go). Two kinds -// live here: -// -// 1. The R2 field-scope premise: the pre-widening encode hashed only Project, -// Generation and each publication's (Selector.Key(), ExecutionID), so a -// required->optional reclassification of the SAME selector under the SAME -// execution produced one digest for two genuinely different immutable -// snapshots. historicalReleaseIDFrame pins that collapse; production -// releaseID must separate the pair. -// 2. Injectivity hardening (defense-in-depth, NOT a defect repair): the fold -// classes below are proven-unreachable through the public surface today. -// Project and Generation reject control bytes at ref.go:26-30 -// (validateOpaqueIdentifier), selector components are confined to the -// injective DataframeSelector.Key() encoding (dataframe_selector.go:34), -// and ExecutionID arrives from the publication store as an opaque id a -// caller of ReleaseService.Create cannot forge on this path. The controls -// replay the archived fold classes against the historical framing (they -// still fold there) and prove the shipped framing is injective by -// construction, so the property does not depend on those upstream -// validators staying in place. - -// historicalReleaseIDFrame replays the pre-widening framing EXACTLY as shipped: -// project and generation joined by a bare "\x00", then one "\x00"-joined -// (Selector.Key(), ExecutionID) group per publication. It is the fault premise: -// the pairs below must COLLAPSE here while production releaseID separates them. func historicalReleaseIDFrame(release ProjectRelease) string { var b strings.Builder b.WriteString(release.Project) @@ -51,8 +26,6 @@ func historicalReleaseIDFrame(release ProjectRelease) string { return b.String() } -// legacyReleaseID is the identity the pre-widening binary derived from the same -// content, so the upgrade-boundary controls below can seed an OLD stored id. func legacyReleaseID(release ProjectRelease) string { digest := sha256.Sum256([]byte(historicalReleaseIDFrame(release))) return "release_" + hex.EncodeToString(digest[:]) @@ -82,14 +55,8 @@ var ( ridLater = ridInstant.Add(time.Second) ) -// ridTimePool is the timestamp space the probes draw from: the zero instant, one -// UTC instant, the SAME instant expressed in a fixed non-UTC zone (canonical -// equivalence) and a genuinely later instant. var ridTimePool = []time.Time{{}, ridInstant, ridOtherZone, ridLater} -// ridTimeLabel renders a timestamp the way the framing canonicalizes it, without -// calling production code, so the decode-inverse comparison stays independent of -// the encoder under test. func ridTimeLabel(value time.Time) string { if value.IsZero() { return "" @@ -97,10 +64,6 @@ func ridTimeLabel(value time.Time) string { return value.UTC().Format(time.RFC3339Nano) } -// ridContentLabel is an unambiguous, order-insensitive canonicalization of every -// field releaseID hashes. Two releases sharing a digest but differing in label -// is the injectivity failure these controls hunt; %q keeps the label itself -// separator-safe. func ridContentLabel(release ProjectRelease) string { pubLabels := make([]string, 0, len(release.Publications)) for _, publication := range release.Publications { @@ -131,8 +94,6 @@ func ridVerificationLabel(verification ContractVerification) string { boolDigit(verification.Queryable), ridTimeLabel(verification.VerifiedAt), verification.ErrorCode) } -// ridClone deep-copies the repeated sections so a mutation probe cannot write -// through the shared backing array of the release it was derived from. func ridClone(release ProjectRelease) ProjectRelease { clone := release if release.Publications != nil { @@ -151,8 +112,6 @@ func boolDigit(value bool) int { return 0 } -// decodedRelease carries a release reconstructed from the exact bytes -// releaseIDFraming emits, keeping timestamps in their canonical string form. type decodedRelease struct { project string generation string @@ -197,11 +156,6 @@ func (d decodedRelease) label() string { strings.Join(pubLabels, ","), strings.Join(verLabels, ",")) } -// decodeReleaseIDFrame reconstructs the hashed content from the exact bytes -// releaseIDFraming emits. Its existence is the constructive injectivity proof: a -// total left-inverse means two distinct contents can never share a framing -// string. readLen rejects a spoofed length prefix (non-digit or leading-zero -// digits) instead of silently accepting it. func decodeReleaseIDFrame(frame []byte) (decodedRelease, bool) { rest := string(frame) if !strings.HasPrefix(rest, releaseIDDomain) { @@ -310,14 +264,7 @@ func readLen(s string) (string, string, bool) { return s[colon+1 : colon+1+n], s[colon+1+n:], true } -// TestReleaseIDHistoricalFramingFoldWitnessesPinnedByFix replays the archived -// fold classes plus the R2 field-scope fold. Every pair COLLAPSEs under -// historicalReleaseIDFrame (the premise) and SEPARATES under releaseID + -// releaseIDFraming (the repair). func TestReleaseIDHistoricalFramingFoldWitnessesPinnedByFix(t *testing.T) { - // Class A: a NUL smuggled across the Project/Generation bare-join edge. - // (Public Create rejects control bytes in project/generation, so this is - // the defense-in-depth / upstream-weakening surface, kept honest.) classAA := ridRelease("p\x00q", "g") classAB := ridRelease("p", "q\x00g") if historicalReleaseIDFrame(classAA) != historicalReleaseIDFrame(classAB) { @@ -327,9 +274,6 @@ func TestReleaseIDHistoricalFramingFoldWitnessesPinnedByFix(t *testing.T) { t.Fatalf("REGRESSION: length-prefixed framing folds the Project/Generation NUL pair (frame %q)", string(releaseIDFraming(classAA))) } - // Class B: the release-specific class — an un-delimited ExecutionID lets one - // publication's bytes impersonate the next (Key(), ExecutionID) group - // (forgeable on this layer, the closest thing to a live surface). leftover := "a\x00" + ridPub("r", "t", "o", "").Selector.Key() + "\x00" classBA := ridRelease("p", "g", ridPub("R", "T", "O", leftover)) classBB := ridRelease("p", "g", ridPub("R", "T", "O", "a"), ridPub("r", "t", "o", "")) @@ -346,11 +290,6 @@ func TestReleaseIDHistoricalFramingFoldWitnessesPinnedByFix(t *testing.T) { t.Fatalf("decode round-trip failed for class B: ok=%v got=%s", ok, got.label()) } - // Class C: the R2 production fold — a required->optional reclassification of - // the same selector under the same execution. Every historically hashed field - // is unchanged, so the pre-widening digest could not tell the two immutable - // candidates apart and SaveRelease rejected the new one with "release already - // exists with different content". selector := ridSelector("core", "v1", "Patient") classCA := ridRelease("p", "g", ReleasePublication{Selector: selector, ExecutionID: "execution-a", Generation: "g", Required: true}) classCB := ridRelease("p", "g", ReleasePublication{Selector: selector, ExecutionID: "execution-a", Generation: "g", Required: false}) @@ -361,8 +300,6 @@ func TestReleaseIDHistoricalFramingFoldWitnessesPinnedByFix(t *testing.T) { t.Fatalf("REGRESSION: releaseID folds the required/optional policy pair that SaveRelease compares as different content") } - // Class D: the same narrow-fold shape on the verification side — a contract - // verdict that changed state while selector/execution stayed put. classDA := ridProbeRelease("p", "g", nil, []ContractVerification{{Selector: selector, ExecutionID: "execution-a", Generation: "g", State: "PUBLISHED", Queryable: true}}) classDB := ridProbeRelease("p", "g", nil, []ContractVerification{{Selector: selector, ExecutionID: "execution-a", Generation: "g", State: "STALE", Queryable: true}}) if releaseID(classDA) == releaseID(classDB) { @@ -371,9 +308,6 @@ func TestReleaseIDHistoricalFramingFoldWitnessesPinnedByFix(t *testing.T) { t.Logf("VERDICT=REPAIRED archived fold classes plus the R2 required/optional fold collapse under the historical framing and are separated by releaseID") } -// TestReleaseIDIncludesEveryImmutableField asserts acceptance criterion: every -// persisted immutable field CATEGORY moves the id, so two snapshots the store -// compares as different content can never share an immutable document key. func TestReleaseIDIncludesEveryImmutableField(t *testing.T) { const project, generation = "project-a", "commit-a" base := func() ProjectRelease { @@ -456,9 +390,6 @@ func assertDistinctID(t *testing.T, left, right ProjectRelease, field string) { } } -// TestReleaseIDIgnoresCreatedAt asserts that only the creation timestamp differs -// yet the identity is unchanged, while a per-binding VerifiedAt change (immutable -// evidence inside the document) still moves it. func TestReleaseIDIgnoresCreatedAt(t *testing.T) { base := ridProbeRelease("project-a", "commit-a", []ReleasePublication{{Selector: ridSelector("core", "v1", "Patient"), ExecutionID: "execution-a", Generation: "commit-a", Required: true, VerifiedAt: ridInstant}}, @@ -476,8 +407,6 @@ func TestReleaseIDIgnoresCreatedAt(t *testing.T) { if first.CreatedAt.Equal(second.CreatedAt) || first.CreatedAt.Equal(third.CreatedAt) { t.Fatalf("witness precondition broken: the CreatedAt probes must be different instants") } - // The excluded timestamp is the release-level one only: VerifiedAt is persisted - // immutable evidence and must still distinguish. moved := ridClone(first) moved.Publications[0].VerifiedAt = ridLater if releaseID(moved) == releaseID(first) { @@ -491,9 +420,6 @@ func TestReleaseIDIgnoresCreatedAt(t *testing.T) { t.Logf("VERDICT=PASS CreatedAt alone (three distinct instants) keeps one id; per-binding VerifiedAt still distinguishes") } -// TestReleaseIDCanonicalEquivalence asserts the canonical forms: equivalent -// submission order, equivalent instants in other representations, and the -// nil/empty service form all keep the service identity deterministic. func TestReleaseIDCanonicalEquivalence(t *testing.T) { first := ReleasePublication{Selector: ridSelector("core", "v1", "Patient"), ExecutionID: "execution-a", Generation: "commit-a", Required: true, VerifiedAt: ridInstant} second := ReleasePublication{Selector: ridSelector("core", "v1", "Observation"), ExecutionID: "execution-b", Generation: "commit-a", Required: false, Stale: true, VerifiedAt: ridLater} @@ -506,8 +432,6 @@ func TestReleaseIDCanonicalEquivalence(t *testing.T) { t.Fatalf("REGRESSION: submission order leaked into releaseID: %s vs %s", releaseID(ordered), releaseID(reversed)) } - // The same instants expressed in another zone, and stripped of a monotonic - // reading, are the same immutable content. elsewhere := ridProbeRelease("project-a", "commit-a", []ReleasePublication{{Selector: first.Selector, ExecutionID: first.ExecutionID, Generation: first.Generation, Required: true, VerifiedAt: ridOtherZone}}, []ContractVerification{{Selector: firstVer.Selector, ExecutionID: firstVer.ExecutionID, Generation: firstVer.Generation, State: "PUBLISHED", Queryable: true, VerifiedAt: ridOtherZone}}, @@ -535,14 +459,12 @@ func TestReleaseIDCanonicalEquivalence(t *testing.T) { if releaseID(monotonic) != releaseID(stripped) { t.Fatalf("REGRESSION: a monotonic reading leaked into releaseID: %s vs %s", releaseID(monotonic), releaseID(stripped)) } - // A genuinely different instant still separates. later := ridClone(utcForm) later.Publications[0].VerifiedAt = ridLater if releaseID(later) == releaseID(utcForm) { t.Fatalf("REGRESSION: a different VerifiedAt instant failed to move releaseID") } - // nil and empty repeated sections are one service form. withEmpties := ridProbeRelease("project-a", "commit-a", []ReleasePublication{first}, nil) withEmpties.RequiredVerifications = []ContractVerification{} if releaseID(withEmpties) != releaseID(ridProbeRelease("project-a", "commit-a", []ReleasePublication{first}, nil)) { @@ -554,20 +476,8 @@ func TestReleaseIDCanonicalEquivalence(t *testing.T) { t.Logf("VERDICT=PASS order, zone, monotonic and nil/empty canonicalization keep releaseID deterministic; distinct instants still separate") } -// TestReleaseIDFramingIsInjectiveSmallAlphabet EXHAUSTIVELY enumerates -// Project/Generation against 0/1-publication releases over the breach alphabet -// {a, ':', NUL, digit}, then exhaustively enumerates 2-publication releases -// (fixed Project/Generation) over that publication space, and 2-publication -// releases over a Project/Generation cross with a capped publication set. Every -// distinct content must map to a distinct framing string, a distinct releaseID, -// and round-trip through the decoder. func TestReleaseIDFramingIsInjectiveSmallAlphabet(t *testing.T) { components := ridBuildComponents([]string{"a", ":", "\x00", "1"}, 1) - // pgSet drives Project/Generation, all written by writeLengthPrefixed. The - // multibyte tokens (CJK/emoji/accented — LEGAL under validateOpaqueIdentifier, - // which only requires valid-UTF8 + no-control, ref.go:26-30 / validation.go:31-44) - // kill the length-vs-rune-count mutant class: a RuneCountInString encoder - // mis-sizes these prefixes and the decode-inverse check below rejects it. pgSet := ridBuildComponents([]string{"a", ":", "\x00", "0", "あ", "🎯", "é"}, 2) type probePub struct { @@ -575,9 +485,6 @@ func TestReleaseIDFramingIsInjectiveSmallAlphabet(t *testing.T) { required, stale bool at time.Time } - // The breach alphabet stays on the four historically framed fields; the fields - // added by the widening are derived from the enumeration index so they sweep - // their whole value domain across the space without multiplying it. var pubSpace []probePub for index, r := range components { for _, tv := range components { @@ -629,8 +536,6 @@ func TestReleaseIDFramingIsInjectiveSmallAlphabet(t *testing.T) { checked++ } - // P/G cross over 0/1 publication releases: exercises the Project/Generation - // boundary and single-publication framing at every delimiter value. for _, project := range pgSet { for _, generation := range pgSet { add(project, generation, nil) @@ -639,13 +544,11 @@ func TestReleaseIDFramingIsInjectiveSmallAlphabet(t *testing.T) { } } } - // Exhaustive 2-publication fold space at a representative Project/Generation. for _, one := range pubSpace { for _, two := range pubSpace { add("p\x00q", "g", []probePub{one, two}) } } - // 2-publication releases that also vary Project/Generation over a small set. pairSet := pubSpace[:min(len(pubSpace), 20)] for _, project := range []string{"p", "\x00", "a:b"} { for _, generation := range []string{"g", "\x00", ":"} { @@ -659,8 +562,6 @@ func TestReleaseIDFramingIsInjectiveSmallAlphabet(t *testing.T) { t.Logf("VERDICT=HARDENED | exhaustive small-alphabet injectivity: %d distinct-content releases, zero framing/digest folds, all decoded", checked) } -// ridBuildComponents returns all strings formed from the alphabet with length -// 0..maxLen, deterministic ordering. func ridBuildComponents(alphabet []string, maxLen int) []string { out := []string{""} level := []string{""} @@ -677,14 +578,6 @@ func ridBuildComponents(alphabet []string, maxLen int) []string { return out } -// TestReleaseIDFramingInjectiveKeySpace enumerates selector triples over a -// delimiter-heavy alphabet down to depth 3, embeds each in a fully populated -// release (publication AND contract verification), and decodes the framing back. -// Because decodeReleaseIDFrame is a total left-inverse, decode(encode(x))==x over -// the ENTIRE enumerated domain proves the framing is injective on it without -// materialising a 64M-entry map. The alphabet depth is bounded to protect the -// gate budget while still driving NUL, ':', and digit bytes through the -// self-delimiting Key(). func TestReleaseIDFramingInjectiveKeySpace(t *testing.T) { components := ridBuildComponents([]string{"a", "1", ":", "\x00", "-"}, 3) count := 0 @@ -707,9 +600,6 @@ func TestReleaseIDFramingInjectiveKeySpace(t *testing.T) { t.Logf("VERDICT=HARDENED | exhaustive Key-space probe: %d selector triples over a delimiter-heavy alphabet (incl NUL/':'), decode-inverse injected", count) } -// TestReleaseIDFramingInjectiveVerificationSpace sweeps the contract-verification -// section (added by the widening) through per-field probes, the nil/empty -// equivalence, and order equivalence, each checked against the decoder. func TestReleaseIDFramingInjectiveVerificationSpace(t *testing.T) { base := func() ProjectRelease { return ridProbeRelease("proj", "gen", []ReleasePublication{ridPub("core", "v1", "Patient", "exec-a")}, []ContractVerification{ @@ -750,19 +640,8 @@ func TestReleaseIDFramingInjectiveVerificationSpace(t *testing.T) { t.Logf("VERDICT=HARDENED | verification-section sweep: 6 field probes, nil/empty and order equivalence, all decoded") } -// TestReleaseIDFramingInjectiveRandomProbe is the bounded random probe over the -// full v2 field set: randomized delimiter-laden releases must stay injective. The -// decode-inverse check is the correctness proof; the collision map is a cheap -// direct fold-hunt kept for the report count. Sample size is capped to protect -// the gate budget. func TestReleaseIDFramingInjectiveRandomProbe(t *testing.T) { const samples = 300_000 - // Multibyte tokens (CJK/emoji/accented) plus digit-colon traps: the former - // exercise the writeLengthPrefixed length-vs-rune-count class on Project, - // Generation, GitCommit AND the opaque ExecutionID; the latter ('2:ab', - // '4:a\x00b', the '00:x' leading-zero decoy) stress the readLen - // ':'/canonical-digit parse at every field position. All multibyte bytes are - // legal under validateOpaqueIdentifier (valid-UTF8 + no-control only). pool := []string{"", "a", "b", ":", "\x00", "1", "2", "a\x00b", "\x00:", "::", "0", "-", "あ", "🎯", "é", "Ω", "2:ab", "4:a\x00b", "00:x"} randomString := func(r *rand.Rand) string { diff --git a/internal/dataset/release_idempotency_test.go b/internal/dataset/release_idempotency_test.go index 52acc83fa..063228dde 100644 --- a/internal/dataset/release_idempotency_test.go +++ b/internal/dataset/release_idempotency_test.go @@ -8,20 +8,6 @@ import ( "time" ) -// Controls for the retry half of the release-identity contract: identical -// snapshots converge instead of minting identities, and a pre-widening stored -// id converges after AT MOST one new-id transition. -// -// Both drive the production ReleaseService.Create/Activate path against the -// package release fixture. The fixture's activation guard has the same shape as -// the production Arango CAS alreadyActive branch -// (internal/dataset/arango/release_store.go: "releasePointer.releaseId == -// @release_id" makes a re-submission that already owns the pointer a no-op), -// modelled there as "expectedRevision+1 == current revision AND same id". So -// the replay asserted here re-sends the successful expected revision. The real -// store converges for a live revision as well, and that is exercised against -// Arango in internal/dataset/arango/release_policy_change_e2e_test.go. - func storedReleaseCandidates(store *MemoryLifecycleStore, project string) map[string]ProjectRelease { store.mu.RLock() defer store.mu.RUnlock() @@ -47,10 +33,6 @@ func r2FixtureService(store *MemoryLifecycleStore, now time.Time, required ...Da return ReleaseService{Manifests: store, Releases: store, Verifier: nil, Required: required, Now: func() time.Time { return now }} } -// TestIdenticalReleaseContentRemainsIdempotent asserts that re-submitting one -// immutable snapshot neither bumps the active revision nor creates a repeated -// identity: content addressing makes the retry land on the same document key, -// and the already-active guard makes it a no-op. func TestIdenticalReleaseContentRemainsIdempotent(t *testing.T) { ctx := context.Background() store := NewMemoryLifecycleStore() @@ -68,8 +50,6 @@ func TestIdenticalReleaseContentRemainsIdempotent(t *testing.T) { t.Fatalf("first activation = %#v, %v", first, err) } - // Replays of the same immutable snapshot keep the whole active record, so - // neither the revision nor the identity moved. for attempt := 2; attempt <= 4; attempt++ { repeat, err := service.Activate(ctx, request) if err != nil { @@ -80,8 +60,6 @@ func TestIdenticalReleaseContentRemainsIdempotent(t *testing.T) { } } - // The authoring side is content-addressed too: repeated Create calls - // reproduce one id and never mint another immutable candidate. for attempt := 1; attempt <= 3; attempt++ { created, err := service.Create(ctx, ActivationRequest{Project: request.Project, Generation: request.Generation}) if err != nil { @@ -106,10 +84,6 @@ func TestIdenticalReleaseContentRemainsIdempotent(t *testing.T) { t.Logf("VERDICT=PASS 4 identical activations and 3 identical saves converge: revision 1, id %s, exactly one immutable candidate", first.Release.ID) } -// TestLegacyReleaseIDReactivationConvergesOnce pins the upgrade boundary: an -// immutable candidate persisted and activated by the pre-widening binary keeps -// its old id, and the widened framing transitions it to the new id exactly -// once. Every later retry converges on that id. func TestLegacyReleaseIDReactivationConvergesOnce(t *testing.T) { ctx := context.Background() now := time.Date(2026, 9, 4, 10, 0, 0, 0, time.UTC) @@ -118,8 +92,6 @@ func TestLegacyReleaseIDReactivationConvergesOnce(t *testing.T) { selector.Key(): published(selector, "execution-a", "commit-a", now), }, errors: map[string]error{}} - // Derive the content the current binary produces for this snapshot, so the - // seeded legacy document is the SAME content under the old digest. probe := NewMemoryLifecycleStore() stageManifest(t, probe, "project-a", "commit-a") probeService := r2FixtureService(probe, now, selector) @@ -137,8 +109,6 @@ func TestLegacyReleaseIDReactivationConvergesOnce(t *testing.T) { t.Fatalf("witness precondition broken: the historical framing must derive a different id for this content") } - // Seed the upgrade boundary: the old binary saved AND activated this - // content under its narrow digest. store := NewMemoryLifecycleStore() stageManifest(t, store, "project-a", "commit-a") service := r2FixtureService(store, now, selector) @@ -154,8 +124,6 @@ func TestLegacyReleaseIDReactivationConvergesOnce(t *testing.T) { t.Fatalf("legacy seeding = %#v, %v (want revision 1, id %s)", active, err, legacy.ID) } - // First reactivation under the widened framing: exactly one new-id - // transition, one revision bump, and the id is the current derived id. reactivation := ActivationRequest{Project: "project-a", Generation: "commit-a", ExpectedRevision: 1} transitioned, err := service.Activate(ctx, reactivation) if err != nil { @@ -168,8 +136,6 @@ func TestLegacyReleaseIDReactivationConvergesOnce(t *testing.T) { t.Fatalf("REGRESSION: first reactivation must converge on %s at revision 2, got %#v", derived.ID, transitioned) } - // Every later retry of the same request converges: no second new-id - // transition and no further revision churn. for attempt := 3; attempt <= 5; attempt++ { repeat, err := service.Activate(ctx, reactivation) if err != nil { diff --git a/internal/dataset/release_requirements_e2e_test.go b/internal/dataset/release_requirements_e2e_test.go index 3610488ab..3d218cee3 100644 --- a/internal/dataset/release_requirements_e2e_test.go +++ b/internal/dataset/release_requirements_e2e_test.go @@ -1,67 +1,5 @@ package dataset_test -// VERIFICATION-ONLY artifact for the F-B14 "requirements-miss -> HTTP 503 with -// no explicit retryability" end-to-end-boundary record. It never mutates -// product code. It creates ONE fully owned ArangoDB database -// (loom_verify_r14_) through the production store.Open path and drops it -// with an authenticated HTTP DELETE in t.Cleanup, logging removal, then proves -// residue is gone with an authenticated database list. -// -// PACKAGE CHOICE (documented deviation, forced by the Go compiler): the -// assignment asked for `package dataset`, but the real `datasetarango` store it -// requires lives in internal/dataset/arango, which IMPORTS internal/dataset -// (`go list -deps ./internal/dataset/arango` lists -// github.com/calypr/loom/internal/dataset). A same-package test file -// (`package dataset`) that imports a package importing `dataset` is an import -// cycle that Go rejects ("import cycle not allowed in test"). The external test -// package `dataset_test` in this same directory is the only way to hold the -// REAL datasetarango store as Releases. Every type under test is the production -// dataset.* type; nothing is faked except the PublicationVerifier seam (which -// the server itself injects its own concrete implementation of in production). -// -// CLAIM UNDER TEST (CONFIRMED-by-reading in verification/publication-bundle/ -// FINDINGS.md F-B14; this run gives it a RUNTIME upgrade or refutation): -// -// dataset.ReleaseService.Activate -> dataset.ReleaseService.Create -// (internal/dataset/release.go) verifies each required DataframeSelector -// through the production PublicationVerifier seam (release.go:181-190). A -// required selector whose verification is not PUBLISHED+Queryable collapses -// into `&ReleaseRequirementsError{Verifications: failed}` with -// contract.ErrorCode="RELEASE_REQUIREMENTS_UNMET" (release.go:185-193), and -// Activate returns it verbatim from Create without ever reaching the release -// CAS (release.go:232-236). Because Create returns before SaveRelease and -// before CompareAndSwapActivateRelease, NO release document, active pointer, -// or revision state changes. -// -// This typed error is the surface the authoring publish route turns into an -// HTTP 503 with no retryability: internal/explorer/lifecycle/publish.go:87-89 -// wraps it as `unavailable(...)` and the reliability contract -// (docs/loom-reliability-contract.md "Retryability is explicit; clients do not -// infer it from an HTTP status or message.") says clients must not infer retry -// from the status, yet the authoring contract carries no `retryable` field -// (F-B14). The second leg proves the caller-side gap is real by showing the -// error is also `errors.Is`-detectable against the ErrReleaseRequirementsUnmet -// sentinel (ReleaseRequirementsError.Unwrap, release.go:82) — the classification -// the HTTP boundary then discards. -// -// PRE-STATED EXPECTATIONS (declared BEFORE the run; the verdict is computed -// ONLY from OBSERVED values): -// -// - Expected-if-confirmed: Activate with the unsatisfiable required selector -// returns an error satisfying errors.As(err, **dataset.ReleaseRequirementsError), -// whose Verifications[0].ErrorCode == "RELEASE_REQUIREMENTS_UNMET", whose -// Verifications[0] carries the required selector with State "PENDING" and -// Queryable==false, and for which NO pointer/revision/lifecycle-document -// state changed (before/after snapshot equal). errors.Is(err, -// dataset.ErrReleaseRequirementsUnmet) is ALSO true. => -// t.Fatalf("FAILED-ON-PURPOSE CONFIRMED: deterministic requirements miss -// leaves only classification choice to the caller ..."). -// -// - Expected-if-prevented: Activate SUCCEEDS, or returns a transient-flavoured -// error that is NOT the typed requirements error (so the 503-boundary record -// would rest on no real requirements-error surface). => t.Log PREVENTED with -// the exact preventing source lines quoted. - import ( "context" "encoding/json" @@ -76,9 +14,8 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" dataset "github.com/calypr/loom/internal/dataset" - datasetarango "github.com/calypr/loom/internal/dataset/arango" - arangostore "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -90,9 +27,6 @@ type r14Citation struct { want string } -// r14Citations pin the load-bearing production lines so the verdict cannot -// silently point at drifted source. TrimSpace-insensitive line match. These are -// the exact hops F-B14's 503 boundary rests on. var r14Citations = []r14Citation{ {"internal/dataset/release.go", 82, `func (e *ReleaseRequirementsError) Unwrap() error { return ErrReleaseRequirementsUnmet }`}, {"internal/dataset/release.go", 185, `if verifyErr != nil || verification.Selector != selector || verification.State != "PUBLISHED" || !verification.Queryable || verification.Generation != ref.Generation {`}, @@ -105,9 +39,6 @@ var r14Citations = []r14Citation{ {"internal/explorer/lifecycle/publish.go", 89, `return PublishResult{}, unavailable("activation", "MATERIALIZATION_ACTIVATION_FAILED", "dataset release preparation failed; the prior Explorer revision was retained", err)`}, } -// r14ContractProse is the reliability-contract sentence the F-B14 record invokes -// (matched by content, not line number, so prose reflow cannot produce a false -// refutation): retryability must be explicit, never inferred from an HTTP status. const r14ContractProse = "clients do not infer it from an HTTP status or message" func r14RepoRoot(t *testing.T) string { @@ -128,8 +59,6 @@ func r14RepoRoot(t *testing.T) string { } } -// r14CheckCitations verifies the code lines verbatim and the contract prose by -// content, and returns a quote map for the PREVENTED branch. func r14CheckCitations(t *testing.T) map[string]string { t.Helper() root := r14RepoRoot(t) @@ -148,9 +77,6 @@ func r14CheckCitations(t *testing.T) map[string]string { if c.line < 1 || c.line > len(lines) { t.Fatalf("citation drift: %s has %d lines, cited :%d == %q", c.file, len(lines), c.line, c.want) } - // Drift-proof: the contract is the EXACT source text, not the pin - // (repairs shift lines constantly; require the statement verbatim - // somewhere and only note pin drift). found := false for _, line := range lines { if strings.TrimSpace(line) == strings.TrimSpace(c.want) { @@ -163,13 +89,6 @@ func r14CheckCitations(t *testing.T) map[string]string { } quoted[fmt.Sprintf("%s:%d", c.file, c.line)] = c.want } - // The reliability contract must still forbid inferring retry from a status. - // Compare against whitespace-normalized text so a line reflow in the prose - // cannot produce a false refutation (the sentence wraps mid-clause in the - // file, docs/loom-reliability-contract.md:94-95). The container test image - // does not ship docs/, so a missing file is an environmental absence, NOT a - // refutation: we log it (the sentence is verified out-of-band on the host) - // and only fail when the file is present but no longer states the rule. raw, err := os.ReadFile(filepath.Join(root, "docs/loom-reliability-contract.md")) if err != nil { if os.IsNotExist(err) { @@ -186,14 +105,6 @@ func r14CheckCitations(t *testing.T) map[string]string { return quoted } -// r14FakeVerifier is declared in THIS test file and IS the production -// PublicationVerifier seam (dataset.PublicationVerifier). It answers exactly one -// required selector with a PENDING, non-queryable verification. In production -// the server injects its own concrete PublicationVerifier (see -// verification/publication-bundle brief: internal/server/release_verifier.go); -// here the fake stands in for an unsatisfied publication so the deterministic -// requirements-miss branch (release.go:185-193) fires. It records the request so -// the run can prove the verification loop was actually reached. type r14FakeVerifier struct { calls int sought dataset.DataframeSelector @@ -202,10 +113,6 @@ type r14FakeVerifier struct { func (v *r14FakeVerifier) VerifyPublication(_ context.Context, _, generation string, selector dataset.DataframeSelector) (dataset.PublicationVerification, error) { v.calls++ v.sought = selector - // Echo the identity fields faithfully so the only reason verification fails - // is the not-yet-queryable PENDING state (the realistic unsatisfied shape), - // NOT a malformed selector. State != "PUBLISHED" && !Queryable is what - // release.go:185 rejects. return dataset.PublicationVerification{ Selector: selector, ExecutionID: "r14-exec-pending", @@ -217,9 +124,7 @@ func (v *r14FakeVerifier) VerifyPublication(_ context.Context, _, generation str var _ dataset.PublicationVerifier = (*r14FakeVerifier)(nil) -// r14Open is the env-gated owned-database harness (SKIP-clean when -// LOOM_TEST_ARANGO_URL is unset), opening only the dataset lifecycle collection. -func r14Open(t *testing.T, budget time.Duration) (context.Context, *arangostore.Client, *datasetarango.Store) { +func r14Open(t *testing.T, budget time.Duration) (context.Context, *arangofactory.Client, *arangofactory.DatasetStore) { t.Helper() endpoint := os.Getenv("LOOM_TEST_ARANGO_URL") if endpoint == "" { @@ -231,14 +136,13 @@ func r14Open(t *testing.T, budget time.Duration) (context.Context, *arangostore. } ctx, cancel := context.WithTimeout(context.Background(), budget) t.Cleanup(cancel) - client, err := arangostore.Open(ctx, endpoint, name) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: name}) if err != nil { t.Fatalf("open %s: %v", name, err) } + client := arangoBackend.Client t.Logf("R14_DATABASE=%s endpoint=%s", name, endpoint) - // Registered FIRST so it runs LAST (Go cleanups are LIFO): after the drop - // below it does the authenticated residue proof on this owned database. t.Cleanup(func() { dbs, listErr := r14AuthenticatedDatabaseList(endpoint) if listErr != nil { @@ -283,10 +187,10 @@ func r14Open(t *testing.T, budget time.Duration) (context.Context, *arangostore. t.Logf("R14_DATABASE_REMOVED=%s", name) }) - if err := client.Bootstrap(ctx, datasetarango.BootstrapSpec()); err != nil { + if err := client.Bootstrap(ctx, arangofactory.DatasetBootstrapSpec()); err != nil { t.Fatalf("bootstrap dataset lifecycle collections: %v", err) } - lifecycle, err := datasetarango.New(client) + lifecycle, err := arangofactory.NewDatasetStore(client) if err != nil { t.Fatalf("new dataset lifecycle store: %v", err) } @@ -322,11 +226,7 @@ func r14AuthenticatedDatabaseList(endpoint string) ([]string, error) { return payload.Result, nil } -// r14StageManifest creates a LOADING manifest and transitions it to STAGED, the -// exact pre-state dataset.ReleaseService.Create's validateGeneration requires -// (release.go:154 -> validateGeneration -> manifest.IsStaged). The manifest docs -// live only in the owned database. -func r14StageManifest(t *testing.T, ctx context.Context, lifecycle *datasetarango.Store, project, generation string) error { +func r14StageManifest(t *testing.T, ctx context.Context, lifecycle *arangofactory.DatasetStore, project, generation string) error { t.Helper() ref, err := dataset.NewRef(project, generation) if err != nil { @@ -356,18 +256,14 @@ func r14StageManifest(t *testing.T, ctx context.Context, lifecycle *datasetarang return nil } -// r14Snapshot captures every lifecycle document (count), the project_release -// document ids, the two pointer documents' revision-bearing fields (including -// _rev), and the production ReadActiveRelease result. Two snapshots are equal -// iff Activate did not change any pointer/revision/release state. type r14Snapshot struct { totalDocs int releaseDocCount int releaseIDs []string - releasePointer string // active_project_release canonical (recordType/_rev/revision/releaseId/releaseKey) - genPointer string // active_generation canonical (recordType/_rev/manifestKey/releaseId/releaseRevision) - canonical string // sorted per-doc "_key|recordType|_rev|revision|releaseId|releaseKey|manifestKey" join - activeErr string // ReadActiveRelease error text ("" if ok) + releasePointer string + genPointer string + canonical string + activeErr string activeRevision int64 activeID string } @@ -392,7 +288,7 @@ func r14CanonicalDoc(d map[string]any) string { r14Any(d, "revision"), r14Any(d, "releaseId"), r14Any(d, "releaseKey"), r14Any(d, "manifestKey")) } -func r14TakeSnapshot(t *testing.T, ctx context.Context, client *arangostore.Client, lifecycle *datasetarango.Store, project string) r14Snapshot { +func r14TakeSnapshot(t *testing.T, ctx context.Context, client *arangofactory.Client, lifecycle *arangofactory.DatasetStore, project string) r14Snapshot { t.Helper() s := r14Snapshot{} active, err := lifecycle.ReadActiveRelease(ctx, project) @@ -405,7 +301,7 @@ func r14TakeSnapshot(t *testing.T, ctx context.Context, client *arangostore.Clie } var docs []map[string]any - err = client.QueryRows(ctx, `FOR d IN @@c RETURN d`, 500, map[string]any{"@c": datasetarango.LifecycleCollection}, func(row map[string]any) error { + err = client.QueryRows(ctx, `FOR d IN @@c RETURN d`, 500, map[string]any{"@c": arangofactory.DatasetLifecycleCollection}, func(row map[string]any) error { docs = append(docs, row) return nil }) @@ -413,7 +309,7 @@ func r14TakeSnapshot(t *testing.T, ctx context.Context, client *arangostore.Clie t.Fatalf("dump lifecycle collection: %v", err) } s.totalDocs = len(docs) - activeGenKey := datasetarango.ActiveDocumentKey(project) + activeGenKey := arangofactory.DatasetActiveDocumentKey(project) canon := make([]string, 0, len(docs)) for _, d := range docs { canon = append(canon, r14CanonicalDoc(d)) @@ -432,9 +328,7 @@ func r14TakeSnapshot(t *testing.T, ctx context.Context, client *arangostore.Clie r14Str(d, "_key"), rt, r14Str(d, "_rev"), r14Any(d, "manifestKey"), r14Any(d, "releaseId"), r14Any(d, "releaseRevision")) } } - // Fallback detection if the recordType strings ever rename. if s.releasePointer == "" { - // active_project_release is the only doc carrying releaseKey. for _, d := range docs { if _, ok := d["releaseKey"]; ok { s.releasePointer = r14CanonicalDoc(d) @@ -476,9 +370,6 @@ func r14QuoteAll(quoted map[string]string, keys []string) string { return strings.Join(lines, "\n") } -// r14VerificationsDigest renders the ContractVerification slice embedded in the -// *ReleaseRequirementsError, which is runtime evidence that the 503-boundary -// record rests on a real, populated error surface (not an empty sentinel). func r14VerificationsDigest(vs []dataset.ContractVerification) string { if len(vs) == 0 { return "" @@ -499,11 +390,6 @@ func r14ErrorChain(err error) string { return strings.Join(chain, "\n <- ") } -// TestReleaseRequirementsBoundaryCitations pins the production lines and the -// reliability-contract prose the F-B14 503-boundary record rests on. It needs -// no ArangoDB and runs whenever the suite runs (including SKIP-clean host runs -// where docs/ is present), so the citation guard is exercised for real even -// when the owned-database test above is gated out. func TestReleaseRequirementsBoundaryCitations(t *testing.T) { quoted := r14CheckCitations(t) t.Logf("R14_CITATIONS_PINNED=%d production lines verified verbatim (release.go reject/typed-error/Unwrap/Activate->Create->CAS hops + publish.go 87/89 unavailable wrap)", len(r14Citations)) @@ -512,10 +398,6 @@ func TestReleaseRequirementsBoundaryCitations(t *testing.T) { } } -// TestReleaseRequirementsUnmetActivateAgainstArango drives the REAL -// dataset.ReleaseService (struct in release.go: Releases/Verifier/Required/Now) -// against a REAL datasetarango store, with a fake Verifier returning PENDING / -// !Queryable for one required selector, and adjudicates the F-B14 boundary. func TestReleaseRequirementsUnmetActivateAgainstArango(t *testing.T) { ctx, client, lifecycle := r14Open(t, 180*time.Second) quoted := r14CheckCitations(t) @@ -523,11 +405,6 @@ func TestReleaseRequirementsUnmetActivateAgainstArango(t *testing.T) { const project = "r14-project" const generation = "r14-gen-unmet" - // PRE-STATED EXPECT: a fully-STAGED manifest but ONE required selector that - // the (fake) production verifier reports PENDING / !Queryable. Create must - // reject at release.go:185-193 and Activate must return the typed - // *ReleaseRequirementsError, with NO pointer/revision/release-doc change and - // errors.Is(err, ErrReleaseRequirementsUnmet) true. t.Log("PRE-STATED EXPECT: Activate(unsatisfiable required selector) => *ReleaseRequirementsError with Verifications[0].ErrorCode=RELEASE_REQUIREMENTS_UNMET, pointer/revision unchanged, errors.Is(err, ErrReleaseRequirementsUnmet)=true => FAILED-ON-PURPOSE CONFIRMED.") if err := r14StageManifest(t, ctx, lifecycle, project, generation); err != nil { @@ -556,7 +433,6 @@ func TestReleaseRequirementsUnmetActivateAgainstArango(t *testing.T) { after := r14TakeSnapshot(t, ctx, client, lifecycle, project) after.log("after-activate") - // Snapshot equality == no pointer/revision/release-doc state change. stateUnchanged := before.canonical == after.canonical && before.totalDocs == after.totalDocs && before.releaseDocCount == after.releaseDocCount && @@ -568,7 +444,6 @@ func TestReleaseRequirementsUnmetActivateAgainstArango(t *testing.T) { verificationLoopReached := verifier.calls > 0 - // Classify the error surface. var reqErr *dataset.ReleaseRequirementsError asTyped := errors.As(err, &reqErr) isSentinel := errors.Is(err, dataset.ErrReleaseRequirementsUnmet) @@ -581,7 +456,6 @@ func TestReleaseRequirementsUnmetActivateAgainstArango(t *testing.T) { } } - // OBSERVED runtime evidence lines (the point of the run). fmt.Printf("R14_OBSERVED activate_returned_err=%v\n", err) if err != nil { fmt.Printf("R14_OBSERVED error_%%v=%v\n", err) @@ -594,12 +468,8 @@ func TestReleaseRequirementsUnmetActivateAgainstArango(t *testing.T) { fmt.Printf("R14_OBSERVED state_unchanged(before==after)=%v releasedocs=%d->%d totalDocs=%d->%d activeErr=%q->%q\n", stateUnchanged, before.releaseDocCount, after.releaseDocCount, before.totalDocs, after.totalDocs, before.activeErr, after.activeErr) - // Adjudicate from OBSERVED values only. switch { case err == nil: - // Activation should never succeed for an unsatisfiable required selector. - // If it did, the 503-boundary record would rest on no real requirements- - // error surface at this layer. t.Log("PREVENTED: dataset.ReleaseService.Activate SUCCEEDED for an unsatisfiable required selector; no requirements error was produced, so the F-B14 503 surface has no backing error at this layer.") t.Logf("PREVENTING/source lines that WOULD reject but did not fire, quoted verbatim from source:\n%s", r14QuoteAll(quoted, []string{ "internal/dataset/release.go:185", "internal/dataset/release.go:186", @@ -607,8 +477,6 @@ func TestReleaseRequirementsUnmetActivateAgainstArango(t *testing.T) { })) t.Errorf("PREVENTED-but-unexpected: Activate returned no error (active{Revision=%d ID=%s}); required verification loop reached=%v verifier_calls=%d — the requirements-miss surface did not manifest; investigate whether Required/Verifier wiring is inert (F-B14 config-gate: nil Required means the branch never executes).", active.Revision, active.Release.ID, verificationLoopReached, verifier.calls) case asTyped: - // First leg: the typed requirements error arrived. Second leg (below) - // confirms the caller-side gap via the sentinel. fmt.Printf("R14_OBSERVED typed_error_confirmed errorCode=%q verifications=%d state_unchanged=%v sentinel_is=%v loop_reached=%v\n", errorCode, len(reqErr.Verifications), stateUnchanged, isSentinel, verificationLoopReached) if errorCode != "RELEASE_REQUIREMENTS_UNMET" { @@ -618,23 +486,16 @@ func TestReleaseRequirementsUnmetActivateAgainstArango(t *testing.T) { t.Fatalf("INCONCLUSIVE: typed requirements error was returned BUT lifecycle state changed (before==after=false), which contradicts Create returning before SaveRelease/CAS. before:\n%s\nafter:\n%s", before.canonical, after.canonical) } if !isSentinel { - // The whole caller-side-gap claim (F-B14) hinges on the error also - // being Is-detectable against the sentinel the HTTP layer discards. t.Fatalf("REFUTED caller-side leg: *ReleaseRequirementsError did NOT satisfy errors.Is(err, ErrReleaseRequirementsUnmet); ReleaseRequirementsError.Unwrap (release.go:82) is not linking the sentinel, so the classification the 503 boundary discards is not actually reachable. %s", r14Quote(quoted, "internal/dataset/release.go:82")) } if !verificationLoopReached { t.Fatalf("INCONCLUSIVE: typed requirements error returned but the required-selector verification loop was never entered (verifier_calls=0); the miss did not flow through the production PublicationVerifier seam at release.go:181-190.") } - // Both legs confirmed against real Arango. The typed error IS the - // 503-boundary surface and the sentinel classification IS reachable. t.Fatalf("FAILED-ON-PURPOSE CONFIRMED: deterministic requirements miss leaves only classification choice to the caller — dataset.ReleaseService.Activate against the REAL datasetarango store returned *dataset.ReleaseRequirementsError (errors.As, Verifications[0].ErrorCode=%q, %d verification(s)) built from the PENDING/!Queryable required selector {%q,%q,%q}, returned VERBATIM by Activate from Create (release.go:232 -> release.go:192-193) WITHOUT reaching the release CAS (release.go:236), and with NO pointer/revision/release-document state change (before==after: releasedocs %d->%d, totalDocs %d->%d, releasePointer=%s, genPointer=%s). SECOND LEG: the same error satisfies errors.Is(err, dataset.ErrReleaseRequirementsUnmet)=true via ReleaseRequirementsError.Unwrap (release.go:82), yet the authoring publish route wraps it as an unavailable/HTTP-503 diagnostic with NO explicit retryability (internal/explorer/lifecycle/publish.go:87-88), violating docs/loom-reliability-contract.md (%q). The runtime error surface that F-B14's 503-boundary record rests on is REAL; the classification (retryability) that distinguishes a permanent miss from a transient outage is DISCARDED at the HTTP boundary.", errorCode, len(reqErr.Verifications), required[0].Recipe, required[0].TranslationVersion, required[0].Output, before.releaseDocCount, after.releaseDocCount, before.totalDocs, after.totalDocs, after.releasePointer, after.genPointer, r14ContractProse) default: - // A non-requirements error (transient-flavoured or a store/precondition - // failure). This would mean the requirements-miss did not surface as the - // typed error this layer promises. t.Log("PREVENTED: dataset.ReleaseService.Activate returned a NON-requirements error (no *ReleaseRequirementsError surface); the F-B14 typed-requirements-error surface did not manifest and cannot back the 503-boundary record here.") t.Logf("PREVENTING/source lines that WOULD have produced the typed error, quoted verbatim from source:\n%s", r14QuoteAll(quoted, []string{ "internal/dataset/release.go:185", "internal/dataset/release.go:186", diff --git a/internal/dataset/release_verifier_fault_classification_test.go b/internal/dataset/release_verifier_fault_classification_test.go index f2089a603..0f17de259 100644 --- a/internal/dataset/release_verifier_fault_classification_test.go +++ b/internal/dataset/release_verifier_fault_classification_test.go @@ -1,11 +1,5 @@ package dataset -// Durable controls (F-B14 revision, round 41): a genuine publication-verification -// store fault (transport failure) must never fold into *ReleaseRequirementsError, -// so clients keep retrying faults, while a deterministic record miss stays a -// non-retryable contract verdict. The store maps its not-found sentinel to -// ErrPublicationRecordMissing; every other verifier error is a fault. - import ( "context" "errors" diff --git a/internal/explorer/artifact.go b/internal/explorer/artifact.go index 6b6a94ecb..7baf05d56 100644 --- a/internal/explorer/artifact.go +++ b/internal/explorer/artifact.go @@ -21,8 +21,6 @@ const ( ArtifactFailed ArtifactState = "FAILED" ) -// ArtifactRecord is the durable server-side identity of one exact training -// artifact. The archive is downloadable only when State is COMPLETE. type ArtifactRecord struct { ID string `json:"id"` Project string `json:"project"` @@ -82,8 +80,6 @@ type ArtifactCompletion struct { CompletedAt time.Time } -// ArtifactStage is an uncommitted archive. Abort must be idempotent and Commit -// must make the completed file and metadata visible atomically to readers. type ArtifactStage interface { io.WriteCloser Commit(context.Context, ArtifactCompletion) (*ArtifactRecord, error) diff --git a/internal/explorer/artifactfs/reconcile_test.go b/internal/explorer/artifactfs/reconcile_test.go index a32c55737..bc145f12a 100644 --- a/internal/explorer/artifactfs/reconcile_test.go +++ b/internal/explorer/artifactfs/reconcile_test.go @@ -14,15 +14,6 @@ import ( "github.com/calypr/loom/internal/explorer" ) -// Durable coverage for the filesystem artifact expiry promise: read paths -// (Get/Open) already refused expired records, but nothing bounded on-disk -// retention, so every export's archive+metadata pair survived its advertised -// TTL forever on the artifact directory (default /tmp/loom-artifacts, cleared -// only where deployment clears it). Reconcile is the store enforcing the same -// ExpiresAt it validates in Begin/Open; the server runs it on a ticker beside -// the ClickHouse artifact reconciler. Discovery pass 2026-11-04m; archived -// pre-fix growth witnesses under verification/publication-bundle/evidence/. - func reconcileClock(base time.Time) *time.Time { clock := base return &clock @@ -192,11 +183,6 @@ func TestReconcileLeavesUnprovableFilesAlone(t *testing.T) { } } -// Durable regressions for the confirmed panel findings (repair-28 review): -// R1 executed repros proved fail-open on the under-lock re-read, a -// non-convergent removal order (record deleted while archive removal failed), -// and Begin permanently wedged by crash-residue staging without a record. - func TestReconcileRetainsSetWhenRecordBecomesUnreadableUnderLock(t *testing.T) { root := t.TempDir() store, err := New(root) @@ -358,14 +344,6 @@ func TestBeginReclaimsCrashResidueStagingWithoutRecord(t *testing.T) { _ = reader.Close() } -// Panel finding (repair-28 R3/M7 + parent rerun): two independent boundaries -// keep foreign files in the artifact directory out of the sweep, each pinned -// by its own planting: (1) the filename queue gate (safeID on the derived id), -// killed by dropping that condition while the self-named foreign pair stays -// otherwise valid; (2) read()'s record-ID-equals-filename rule, killed by -// deleting that mismatch check while the aliased pair stays valid. Both -// pairings carry genuinely decodable expired records, so an implementation -// lacking either boundary deletes user data. func TestReconcileNeverRemovesFilesWhoseRecordDisownsTheirName(t *testing.T) { root := t.TempDir() store, err := New(root) @@ -427,10 +405,6 @@ func TestReconcileNeverRemovesFilesWhoseRecordDisownsTheirName(t *testing.T) { } } -// Panel finding (repair-28 R3/M9): the per-set cancellation check inside the -// removal loop was unobservable — existing canceled-context coverage only -// exercised the entry check after everything was already deleted. The -// deterministic hook below forces the mid-sweep branch. func TestReconcileStopsMidSweepOnCancellation(t *testing.T) { root := t.TempDir() store, err := New(root) diff --git a/internal/explorer/artifactfs/store.go b/internal/explorer/artifactfs/store.go index 60911b0a2..055dce7eb 100644 --- a/internal/explorer/artifactfs/store.go +++ b/internal/explorer/artifactfs/store.go @@ -19,11 +19,9 @@ import ( var ErrArtifactUnavailable = errors.New("artifact is not complete or has expired") type Store struct { - root string - now func() time.Time - mu sync.Mutex - // ctxCheck mirrors context cancellation for one Reconcile pass; tests - // inject a deterministic countdown there. Nil means ctx.Err() directly. + root string + now func() time.Time + mu sync.Mutex ctxCheck func(context.Context) error } @@ -68,12 +66,6 @@ func (s *Store) Begin(ctx context.Context, record explorer.ArtifactRecord) (expl stagingPath := s.path(record.ID, ".staging") file, err := os.OpenFile(stagingPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) if errors.Is(err, os.ErrExist) { - // Under this same mutex, in this process, a live prepare always - // carries a readable record before Begin returns, so a staging file - // with no readable record is crash debris from an earlier attempt at - // this id. Reclaim it once and retry; anything else keeps the - // original failure. Cross-process sharing of the root is unsupported - // (see Reconcile). if _, readErr := s.read(record.ID); errors.Is(readErr, os.ErrNotExist) { if removeErr := os.Remove(stagingPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { return nil, nil, fmt.Errorf("reclaim stale artifact staging: %w", removeErr) @@ -135,25 +127,6 @@ func (s *Store) canceled(ctx context.Context) error { return ctx.Err() } -// Reconcile enforces the expiry promise the store already applies at read -// time: sets whose readable record has expired leave the store, so retention -// of record-proven artifacts is bounded by the product's TTL. Sets without a -// readable record are never removed — identity (and therefore harmlessness) -// cannot be proven, and unclaimable residue surfaces to operators for a -// deliberate purge instead of a silent guess. Every mutation path holds the -// same mutex, so the under-lock re-read excludes any race with an active -// Begin on a recycled id, and any read failure at re-read time leaves the set -// for a later pass. The record (.json) is removed only after archive and -// staging are gone, keeping the queueing proof alive for the next sweep. -// limit bounds one pass, matching the sibling ClickHouse artifact reconciler. -// A set is claimed for removal only when its record re-read under the mutex -// has expired; s.read already refuses metadata whose ID disowns the filename -// (identity mismatch), and queueing additionally requires the filename id to -// pass safeID, so neither a disowned record nor a foreign-named file can be -// removed. The -// mutex is per process: this store assumes it owns its directory (a single -// API process on local disk, as deployed); sharing a root across processes -// is unsupported for the same reason Begin's O_EXCL staging contract is. func (s *Store) Reconcile(ctx context.Context, now time.Time, limit int) (int, error) { if err := ctx.Err(); err != nil { return 0, err diff --git a/internal/explorer/authoringv2/canonical.go b/internal/explorer/authoringv2/canonical.go index 3f88217c0..278eb0c5f 100644 --- a/internal/explorer/authoringv2/canonical.go +++ b/internal/explorer/authoringv2/canonical.go @@ -51,15 +51,6 @@ func (w Workspace) CanonicalJSON() ([]byte, error) { return json.Marshal(n) } -// NormalizePresentationOrders gives every table column one unambiguous, -// contiguous presentation position. Authored order is the primary key and the -// stable public column identity breaks ties. The normalized order also becomes -// the recipe projection order, so presentation and execution cannot disagree. -// -// Duplicate presentation positions are valid mutable Builder input. Freezing -// them here keeps equivalent requests from depending on frontend collection or -// map iteration order and makes the normalized workspace safe to persist and -// return to the Builder. func (w Workspace) NormalizePresentationOrders() Workspace { n := w n.Documents = make([]Document, len(w.Documents)) @@ -196,10 +187,6 @@ func (w Workspace) Digest() (string, error) { return "sha256:" + hex.EncodeToString(sum[:]), nil } -// LegacyNilConstructionStepsDigest returns the digest produced by the prior -// canonical writer when an empty construction's nil Steps slice was encoded -// as null. It is used only to recognize and migrate persisted drafts created -// before empty construction sequences were normalized to arrays. func (w Workspace) LegacyNilConstructionStepsDigest() (string, error) { legacyStepLists := 0 for _, document := range w.Documents { @@ -355,10 +342,6 @@ func DecodeWorkspace(raw []byte) (Workspace, error) { } return out, nil } else { - // Legacy source wires are accepted only by persisted-draft decoding. - // New mutation requests use the current ColumnSource decoder and never - // enter this compatibility path. Workspace.Validate below still rejects - // future semantics versions after a successful legacy decode. migrated, migrationErr := decodePersistedLegacyWorkspace(raw) if migrationErr != nil { return out, err @@ -469,9 +452,6 @@ func decodePersistedSource(raw json.RawMessage, semanticsVersion int) (ColumnSou if err := strictDecode(raw, &legacy); err == nil { return decodeLegacyFlatSource(legacy) } - // The first v3 draft wire used nested source payloads and a nested where. - // Keep that wire private to persisted-draft migration; AggregateSource's - // current decoder deliberately rejects where on writable requests. var nested struct { Kind string `json:"kind"` Field *FieldSource `json:"field,omitempty"` diff --git a/internal/explorer/authoringv2/commands.go b/internal/explorer/authoringv2/commands.go index 22a5f766d..2a7c9ffca 100644 --- a/internal/explorer/authoringv2/commands.go +++ b/internal/explorer/authoringv2/commands.go @@ -62,10 +62,6 @@ const ( InitialPresentationChart = "CHART" ) -// ApplyCommandsRequest is the browser's mutation envelope. CommandID is an -// idempotency token retained only for the last successfully applied command, -// never a durable authoring identity; retrying an older command is a draft -// conflict after a later command advances the draft. type ApplyCommandsRequest struct { CommandID string `json:"commandId"` SemanticsVersion int `json:"semanticsVersion"` @@ -76,8 +72,6 @@ type ApplyCommandsRequest struct { draftSourceContext *DraftSourceContext } -// DraftSourceContext is attached by lifecycle after resolving the request's -// authorized snapshot. It is never accepted from the browser. type DraftSourceContext struct { SnapshotToken string SourceGeneration string @@ -160,8 +154,6 @@ type Command struct { resolvedDraftRevision *Workspace } -// ConstructionOutputPresentation updates user-facing metadata for a stable -// construction output. Name and Type remain compiler-owned identities. type ConstructionOutputPresentation struct { StepID string `json:"stepId"` ColumnID string `json:"columnId"` @@ -186,8 +178,6 @@ func (p ConstructionOutputPresentation) Validate() error { return nil } -// ResolveDraftRevision binds a restore command to a workspace resolved by the -// lifecycle service from the owner's exact previous-draft pointer. func (c *Command) ResolveDraftRevision(workspace Workspace) error { if c == nil || c.Type != CommandRestoreDraftRevision || strings.TrimSpace(c.DraftRevisionID) == "" || c.DraftRevisionID != strings.TrimSpace(c.DraftRevisionID) { return fmt.Errorf("RESTORE_DRAFT_REVISION requires an exact server-resolved draft revision") @@ -200,8 +190,6 @@ func (c *Command) ResolveDraftRevision(workspace Workspace) error { return nil } -// ColumnTransformationChange replaces or removes the one typed value -// transformation attached to an authored column. type ColumnTransformationChange struct { Kind string `json:"kind"` Transformation *columntransform.ValueTransformation `json:"transformation,omitempty"` @@ -224,8 +212,6 @@ func (c ColumnTransformationChange) Validate() error { } } -// ConstructionChoiceSelection carries only the server-issued source identity -// and the form the author chose from its current supported options. type ConstructionChoiceSelection struct { RowValuePolicy ConstructionRowValuePolicy `json:"rowValuePolicy,omitempty"` ChoiceID string `json:"choiceId"` @@ -271,8 +257,6 @@ func (s ConstructionChoiceSelection) validate() error { } } -// ResolvedConstructionChoice is populated by lifecycle after it re-authorizes -// and reconstructs the choice against the current compiler snapshot. type ResolvedConstructionChoice struct { CandidateID string Source ColumnSource @@ -281,8 +265,6 @@ type ResolvedConstructionChoice struct { FrameID string } -// SemanticSelection is client intent plus an internal-only resolution filled -// by lifecycle after authorization and context checks. type SemanticSelection struct { ConceptID string `json:"conceptId"` BindingID string `json:"bindingId"` @@ -302,10 +284,6 @@ func (s *SemanticSelection) UnmarshalJSON(raw []byte) error { return nil } -// ApplyInterpretationCandidate is the closed, receipt-backed payload for an -// interpretation repair. The receipt ID and revision ID are both required: -// the former proves the preview that was shown to the user, while the latter -// prevents replaying that preview with a different immutable revision. type ApplyInterpretationCandidate struct { CandidateReceiptID string `json:"candidateReceiptId"` RevisionID string `json:"revisionId"` @@ -403,9 +381,6 @@ func rejectUnknownProposalCommandFields(raw []byte, commandType string) error { return nil } -// ResolveRowDefinitionProposal attaches the lifecycle-validated row definition -// to an apply command. The resolved value is deliberately not part of the -// browser command or its serialized payload. func (c *Command) ResolveRowDefinitionProposal(rows RowDefinition) error { if c == nil || c.Type != CommandApplyRowDefinitionProposal { return fmt.Errorf("row definition can only be resolved for APPLY_ROW_DEFINITION_PROPOSAL") @@ -425,8 +400,6 @@ func (c *Command) ResolveRowDefinitionProposal(rows RowDefinition) error { return nil } -// ResolveTableShapeProposal attaches the lifecycle-validated table shape to an -// apply command. A nil shape is a resolved removal, not an unresolved command. func (c *Command) ResolveTableShapeProposal(shape *TableShape) error { if c == nil || c.Type != CommandApplyTableShapeProposal { return fmt.Errorf("table shape can only be resolved for APPLY_TABLE_SHAPE_PROPOSAL") @@ -444,9 +417,6 @@ func (c *Command) ResolveTableShapeProposal(shape *TableShape) error { return nil } -// ResolveConstructionProposal attaches the lifecycle-validated candidate -// document's construction and row intent to an apply command. Both values are -// server-resolved state and never appear in the browser command payload. func (c *Command) ResolveConstructionProposal(candidate *Document) error { if c == nil || c.Type != CommandApplyConstructionProposal { return fmt.Errorf("construction can only be resolved for APPLY_CONSTRUCTION_PROPOSAL") @@ -475,9 +445,6 @@ func (c *Command) ResolveConstructionProposal(candidate *Document) error { return nil } -// ResolvePopulationMemberProposal attaches the server-revalidated candidate -// population to a closed proposal apply command. The browser can submit only -// outputId and proposalId; selection and route data are never caller supplied. func (c *Command) ResolvePopulationMemberProposal(selectionRevisionID string, route []PopulationRouteStep) error { if c == nil || c.Type != CommandApplyPopulationMemberProposal { return fmt.Errorf("population can only be resolved for APPLY_POPULATION_MEMBER_PROPOSAL") @@ -835,8 +802,6 @@ func (c Command) validate() error { return nil } -// ApplyCommands is a pure reducer over canonical authoring state. It clones -// the input and either returns one fully valid workspace or no mutation. func ApplyCommands(workspace Workspace, catalog CatalogSnapshot, commandID string, commands []Command) (Workspace, []CommandResult, error) { working, err := cloneWorkspace(workspace) if err != nil { @@ -1379,8 +1344,6 @@ func applyCommand(workspace *Workspace, catalog CatalogSnapshot, commandID strin Kind: FeatureInterpretationPinned, Pinned: &PinnedInterpretation{RevisionID: revisionID}, } - // A pinned human interpretation owns the feature meaning. Any inline - // contributor is stale for this exact candidate and must not survive. current.Contributor = nil return CommandResult{Type: CommandResultTableChanged, OutputID: command.OutputID, Column: current.Column}, nil } @@ -1594,10 +1557,6 @@ func setPopulationRoute(document *Document, selectionRevisionID string, route [] document.Population = &Population{SelectionRevisionID: strings.TrimSpace(selectionRevisionID), Route: steps} } -// editableSource adds the explicit related-resource policy to new child -// selections. A child column is rendered through a first matching resource -// even when its projection mode is VALUE or INDEXED, so publication must not -// silently present that reduction as lossless. func editableSource(occurrenceID string, source ColumnSource) ColumnSource { source = source.Normalized() mode := strings.ToUpper(strings.TrimSpace(source.ProjectionMode())) @@ -1673,9 +1632,6 @@ func cloneWorkspace(value Workspace) (Workspace, error) { if err := json.Unmarshal(raw, &result); err != nil { return Workspace{}, err } - // SourceWhere is a private persisted-draft compatibility payload and is - // intentionally omitted from current JSON. Preserve it for transactional - // legacy migration and frozen in-memory callers while keeping it unwritable. for documentIndex := range value.Documents { if documentIndex >= len(result.Documents) { break @@ -1756,10 +1712,6 @@ func validateWorkspaceContributors(workspace Workspace, catalog CatalogSnapshot) return nil } -// ValidateContributorForCatalog proves one authored contributor against the -// pinned occurrence and catalog. It is shared by command application and the -// compilation boundary so a stale/foreign candidate cannot become a recipe -// selector. func ValidateContributorForCatalog(document Document, catalog CatalogSnapshot, occurrenceID string, source ColumnSource, predicate ContributorPredicate) error { if source.Kind != SourceAggregate || source.Aggregate == nil { return fmt.Errorf("contributor is only supported for aggregate sources") @@ -1835,8 +1787,6 @@ func validateEditableSource(document Document, catalog CatalogSnapshot, occurren if _, err := fhirschema.ValidateCorrelatedBinding(occurrence.ResourceType, *source.Lookup.Binding); err != nil { return fmt.Errorf("source binding: %w", err) } - // Correlated lookups carry their structural source in Binding. They do - // not have a legacy field path to resolve against a catalog candidate. return nil } if source.OwnerRecords != nil && source.Kind == SourceOwnerRecords { @@ -2160,7 +2110,6 @@ func applyInitialPresentation(column *Column, presentation string, tableOrder in func nextTableOrder(document Document) int { result := 0 if document.Rows.Kind == RowDefinitionGroups && document.Rows.Groups != nil && document.Rows.Groups.Source.Kind == GroupSourceExplicit { - // Cohort label, ordinal and members precede the contributed fields. result = 3 + len(document.Rows.Groups.RowValues) } for _, column := range document.Columns { diff --git a/internal/explorer/authoringv2/construction.go b/internal/explorer/authoringv2/construction.go index 989266100..df38c88dd 100644 --- a/internal/explorer/authoringv2/construction.go +++ b/internal/explorer/authoringv2/construction.go @@ -37,19 +37,12 @@ const ( ConstructionOperationRelatedField ConstructionOperationKind = "RELATED_FIELD" ) -// Construction stores the ordered, durable operations applied after the -// document's source projection. A zero-step construction is the identity plan -// over that source projection. A terminal Combine is a standalone plan over -// exact published table revisions. Version identifies this operation -// contract, independently of the workspace's older V2 semantics version. type Construction struct { Version int `json:"version"` Steps []ConstructionStep `json:"steps"` SourceProjections []ConstructionSourceProjection `json:"sourceProjections,omitempty"` } -// ConstructionSourceProjection is a compiler-authorized source field needed -// by a construction operation but excluded from the document's public columns. type ConstructionSourceProjection struct { ColumnID string `json:"columnId"` OwnerStepID string `json:"ownerStepId,omitempty"` @@ -60,9 +53,6 @@ type ConstructionSourceProjection struct { Label string `json:"label"` } -// MarshalJSON keeps an empty construction as an explicit empty sequence at -// persistence and API boundaries. A nil slice can arrive from older drafts -// that encoded "steps": null, but clients consume this field as an array. func (c Construction) MarshalJSON() ([]byte, error) { type wire Construction if c.Steps == nil { @@ -71,10 +61,6 @@ func (c Construction) MarshalJSON() ([]byte, error) { return json.Marshal(wire(c)) } -// ConstructionStep is one saved analytical operation. Its Inputs identify -// the exact stage or immutable table revision it consumes. Outputs declare -// the complete resulting stage schema; IDs remain stable when names or labels -// change. type ConstructionStep struct { ID string `json:"id"` OwnerStepID string `json:"ownerStepId,omitempty"` @@ -91,8 +77,6 @@ const ( ConstructionRowValueOne ConstructionRowValuePolicy = "ONE" ) -// ConstructionRowValue populates a shaped row from the same records that -// contributed to its grouping or pivot. It does not alter row membership. type ConstructionRowValue struct { InputColumnID string `json:"inputColumnId,omitempty"` FieldSource *ConstructionGroupFieldSource `json:"fieldSource,omitempty"` @@ -122,10 +106,6 @@ func (value *ConstructionRowValue) UnmarshalJSON(raw []byte) error { return nil } -// ConstructionInputRef is a closed source-stage, prior-stage, sibling-output, -// or immutable table-revision reference. A table input always names a -// concrete revision; floating references to a table's current head are not -// supported. type ConstructionInputRef struct { Kind ConstructionInputKind `json:"kind"` StepID string `json:"stepId,omitempty"` @@ -171,9 +151,6 @@ func (r ConstructionInputRef) Validate() error { return nil } -// StageColumn describes one column at a stage boundary. Type is advisory for -// source-derived operations, while terminal Combine requires a concrete scalar -// type and uses Nullable to declare left-join output behavior. type StageColumn struct { ID string `json:"id"` Name string `json:"name"` @@ -200,10 +177,6 @@ type ConstructionOperation struct { RelatedField *ConstructionRelatedField `json:"relatedField,omitempty"` } -// ConstructionRelatedSource adds one compiler-authorized field from related -// resources to rows at the exact input stage. Route and field identity are -// persisted alongside the choice token so saved constructions remain -// self-contained after the capability snapshot expires. type ConstructionRelatedSource struct { AnchorColumnID string `json:"anchorColumnId"` ChoiceID string `json:"choiceId"` @@ -216,8 +189,6 @@ type ConstructionRelatedSource struct { OutputColumnID string `json:"outputColumnId"` } -// ConstructionRelatedExpand emits one row per distinct resource reached by -// the exact, server-authorized route. type ConstructionRelatedExpand struct { AnchorColumnID string `json:"anchorColumnId"` ChoiceID string `json:"choiceId"` @@ -231,9 +202,6 @@ type ConstructionRelatedExpand struct { RelatedRecordColumnID string `json:"relatedRecordColumnId"` } -// ConstructionRelatedEligibility filters rows by the number of distinct -// terminal resources reached through an exact authorized route. It preserves -// the input schema and row identity. type ConstructionRelatedEligibility struct { AnchorColumnID string `json:"anchorColumnId"` ChoiceID string `json:"choiceId"` @@ -257,8 +225,6 @@ const ( ConstructionRelatedEligibilityCountAtLeast = "COUNT_AT_LEAST" ) -// ConstructionRelatedField adds one scalar field from the exact terminal -// resource retained by a preceding RELATED_EXPAND stage. type ConstructionRelatedField struct { ChoiceID string `json:"choiceId"` Source ConstructionRelatedFieldSource `json:"source"` @@ -293,9 +259,6 @@ func (o *ConstructionOperation) UnmarshalJSON(raw []byte) error { return nil } -// ConstructionCombine describes one standalone operation over exact published -// table revisions. InputColumnIDs refer to the schemas resolved for those -// revisions; OutputColumnIDs remain stable in the authored output schema. type ConstructionCombine struct { Kind ConstructionCombineKind `json:"kind"` Keys []ConstructionCombineKey `json:"keys,omitempty"` @@ -381,9 +344,6 @@ type ConstructionPivotCategory struct { OutputColumnID string `json:"outputColumnId"` } -// ConstructionCodedPivot pivots a compiler-authorized coded value family -// directly from root resources. Choice IDs authorize a proposal and are -// cleared before the durable source and category facts are persisted. type ConstructionCodedPivot struct { ConstructionID string `json:"constructionId"` SourceChoiceID string `json:"sourceChoiceId,omitempty"` @@ -567,8 +527,6 @@ type ConstructionUnpivotInput struct { Key TableScalar `json:"key"` } -// ConstructionGroup replaces a stage's rows with one row per distinct key -// tuple, or one table summary row when Keys is empty. type ConstructionGroup struct { ConstructionID string `json:"constructionId"` MissingKeyPolicy ConstructionGroupMissingKeyPolicy `json:"missingKeyPolicy"` @@ -576,9 +534,6 @@ type ConstructionGroup struct { Aggregates []ConstructionGroupAggregate `json:"aggregates,omitempty"` } -// ConstructionCodedGroup groups source rows by one correlated tuple from a -// repeated Coding path. The source facts are durable compiler inputs; ChoiceID -// is proposal authority only and is not required to recompile a saved step. type ConstructionCodedGroup struct { ConstructionID string `json:"constructionId"` ChoiceID string `json:"choiceId,omitempty"` @@ -678,10 +633,6 @@ func (key *ConstructionGroupKey) UnmarshalJSON(raw []byte) error { return nil } -// ConstructionGroupFieldSource is a signed Group-key choice or its durable -// binding. A binding pins exactly one input provenance variant. Flat fields -// remain accepted for saved CURRENT_STAGE bindings and are preserved when -// those saved values cross the authoring boundary again. type ConstructionGroupFieldSource struct { Kind string `json:"kind"` ChoiceID string `json:"choiceId,omitempty"` @@ -690,8 +641,6 @@ type ConstructionGroupFieldSource struct { SourceShapeID string `json:"sourceShapeId,omitempty"` Input *ConstructionGroupFieldSourceInput `json:"input,omitempty"` - // Legacy flat CURRENT_STAGE fields. They are populated only when decoding - // saved recipes that predate the input provenance sum. SourceRole string `json:"-"` OccurrenceID string `json:"-"` ResourceType string `json:"-"` @@ -1020,9 +969,6 @@ func (source ConstructionGroupFieldSource) legacyCurrentStage() *ConstructionGro } } -// ConstructionGroupSemanticCategory is either an ephemeral CHOICE supplied -// by current stage capabilities or the durable server-resolved BINDING. -// Version is an exact string; the empty string means versionless, not any. type ConstructionGroupSemanticCategory struct { Kind string `json:"kind"` ChoiceID string `json:"choiceId,omitempty"` @@ -1052,8 +998,6 @@ const ( ConstructionGroupMean ConstructionGroupAggregateOp = "MEAN" ) -// ConstructionExpand replaces one public list column with its items and, if -// requested, the zero-based position of each item. type ConstructionExpand struct { ConstructionID string `json:"constructionId"` InputColumnID string `json:"inputColumnId"` @@ -1063,14 +1007,6 @@ type ConstructionExpand struct { ExpandedField *ConstructionExpandedFieldSource `json:"expandedField,omitempty"` } -// ConstructionExpandedFieldSource records the server-derived root array field -// facts that authorize an EXPAND step to consume an array column materialized -// through SourceProjections. The lane sets it only after resolving the signed -// row choice against the current capability snapshot; Apply re-authorization -// re-derives it and refuses drift, and the compiler independently re-verifies -// the path against the generated FHIR schema before treating the input column -// as array-valued. Client-authored values beyond a structurally consistent -// echo of the accepted proposal are never trusted. type ConstructionExpandedFieldSource struct { Path string `json:"path"` Cardinality string `json:"cardinality"` @@ -1078,17 +1014,8 @@ type ConstructionExpandedFieldSource struct { LogicalType string `json:"logicalType"` } -// ConstructionExpandedFieldCardinalityMany mirrors the lowercase cardinality -// spelling of expression.Many ("many") used by every compiled payload gate -// (precedent: related source cardinality checks compare the same strings in -// the compilation package, not authoringv2). const ConstructionExpandedFieldCardinalityMany = "many" -// ValidateConstructionExpandedFieldSource shape-checks a signed expanded -// field payload. A nil payload is a valid absence (plain selected-column -// EXPAND). Shape checks here are structural only; the compile path re-proves -// the array facts against the generated FHIR schema and never trusts this -// payload for cardinality. func ValidateConstructionExpandedFieldSource(field *ConstructionExpandedFieldSource) error { if field == nil { return nil diff --git a/internal/explorer/authoringv2/construction_change.go b/internal/explorer/authoringv2/construction_change.go index 2f9e81764..8f57ab331 100644 --- a/internal/explorer/authoringv2/construction_change.go +++ b/internal/explorer/authoringv2/construction_change.go @@ -11,8 +11,6 @@ type ConstructionDependencyIssue struct { ColumnID string `json:"columnId"` } -// ConstructionImpact describes the stages that must be recalculated for a -// proposed edit or removal and any stable column references that need repair. type ConstructionImpact struct { ChangedStepID string `json:"changedStepId,omitempty"` RemovedStepIDs []string `json:"removedStepIds,omitempty"` @@ -24,10 +22,6 @@ func (i ConstructionImpact) HasMissingInputs() bool { return len(i.MissingInputs) != 0 } -// AnalyzeStepEdit returns an isolated candidate document with the selected -// stable step ID replaced. Later stage schemas are recalculated from the -// candidate. Missing column references are reported instead of cascading -// deletion or invalidating the accepted document. func (d Document) AnalyzeStepEdit(replacement ConstructionStep) (Document, ConstructionImpact, error) { if d.Construction == nil { return d, ConstructionImpact{}, fmt.Errorf("document has no staged construction") @@ -56,11 +50,6 @@ func (d Document) AnalyzeStepEdit(replacement ConstructionStep) (Document, Const return candidate, impact, nil } -// AnalyzeConstructionCandidate accepts one complete proposal and verifies its -// identity-level changes against the saved sequence. It supports replacing one -// existing step, appending one new step, and removing only explicitly named -// steps. Surviving steps retain their order; downstream edits remain available -// for repairing dependencies in the submitted candidate. func (d Document) AnalyzeConstructionCandidate(candidateConstruction Construction, changedStepID string, removeStepIDs []string) (Document, ConstructionImpact, error) { base := d if d.Construction == nil { @@ -172,8 +161,6 @@ func (d Document) AnalyzeConstructionCandidate(candidateConstruction Constructio if oldStep.OwnerStepID == changedStepID && oldStep.Operation.Kind == ConstructionOperationCodedPivot && candidateStep.OwnerStepID == changedStepID && candidateStep.Operation.Kind == ConstructionOperationCodedPivot && !reflect.DeepEqual(oldStep, candidateStep) && oldIndex < firstChanged { - // A direct-source coded-value input is part of its owning GROUP proposal. - // Recalculate from this helper when the owner edits its selected codes. firstChanged = oldIndex } if oldStep.ID == changedStepID { @@ -317,10 +304,6 @@ func appendedRelatedSourceSuffixStart(existing, candidate []ConstructionStep, st return start, true } -// ProposeStepRemoval removes only the selected step and the downstream steps -// explicitly named in removeStepIDs. It retains unrelated later steps, rewires -// their stage input to the nearest surviving stage, and reports missing column -// references for repair before application. func (d Document) ProposeStepRemoval(stepID string, removeStepIDs []string) (Document, ConstructionImpact, error) { if d.Construction == nil { return d, ConstructionImpact{}, fmt.Errorf("document has no staged construction") @@ -817,7 +800,6 @@ func rebuildStageColumns(step ConstructionStep, input []StageColumn, authoredSou if column.Type == "" || column.Type == "INFER" { column.Type = step.Operation.RelatedField.Source.LogicalType } - // The terminal record can be absent after PRESERVE_PARENT expansion. column.Nullable = true outputs = append(outputs, column) default: @@ -943,8 +925,6 @@ func cloneTablePresentation(presentation *TablePresentation) *TablePresentation return &cloned } -// stagedSourceColumnID assigns the immutable identity when a source column is -// authored, before the table has a construction or a published revision. func stagedSourceColumnID(outputID, commandID string, commandIndex int, commandType string, identity ...any) string { values := make([]any, 0, len(identity)+5) values = append(values, "staged-source-column/v1", outputID, commandID, commandIndex, commandType) diff --git a/internal/explorer/authoringv2/construction_choice.go b/internal/explorer/authoringv2/construction_choice.go index 3cbc6a638..b320e8312 100644 --- a/internal/explorer/authoringv2/construction_choice.go +++ b/internal/explorer/authoringv2/construction_choice.go @@ -93,11 +93,6 @@ func validateResolvedConstructionChoiceSource(document Document, catalog Catalog return validate(source) } - // Route-level ALL is compiler-supported for scalar fields even though the - // terminal field candidate itself advertises only scalar VALUE. The choice - // has already been re-authorized against its exact route and form by the - // lifecycle; validate its underlying field identity using that advertised - // scalar mode while preserving ALL in the saved source. valueSource := source field := *source.Field field.ProjectionMode = "VALUE" diff --git a/internal/explorer/authoringv2/construction_migration.go b/internal/explorer/authoringv2/construction_migration.go index 79f4c02c9..bebf14128 100644 --- a/internal/explorer/authoringv2/construction_migration.go +++ b/internal/explorer/authoringv2/construction_migration.go @@ -6,10 +6,6 @@ import ( "fmt" ) -// UpgradeDocumentToConstruction converts a legacy document to the staged -// operation contract. It is intended for the first staged edit. The input is -// never mutated, and TableShape is cleared only after every legacy operation -// has been represented and the staged document validates. func UpgradeDocumentToConstruction(document Document) (Document, error) { upgraded := document if document.Columns == nil { diff --git a/internal/explorer/authoringv2/construction_projection_replay_test.go b/internal/explorer/authoringv2/construction_projection_replay_test.go index 4ca8ca9ec..f4c6fabf2 100644 --- a/internal/explorer/authoringv2/construction_projection_replay_test.go +++ b/internal/explorer/authoringv2/construction_projection_replay_test.go @@ -1,18 +1,5 @@ package authoringv2 -// Projection owner-branch replay family: closes the FORMAL-62-U1 replay-lane -// gap (verification/publication-bundle/FINDINGS.md Round 61 §1): the -// validatePivotSourceProjection dispatch arms and the carry-prefix loop were -// unreachable to the model/replay. The ConstructionProjectionOwnership model -// (cf0ca558) covers decision semantics with eight violating-by-design fault -// cfgs (m-prj-f-*); this family pins the Go branch outcomes and their exact -// reject strings. Boundary is disclosed per branch: accept cases and every -// rejection reachable through the public Construction.Validate run full-suite; -// nil-payload and sorting-owner isolations collide with earlier structural -// validators inside full-suite and additionally pin the exact branch string -// through the direct validator entry, mirroring how the model's branchAccepts -// maps to construction_validation.go:338-397. - import ( "encoding/json" "fmt" @@ -105,8 +92,6 @@ func projectionPivotStepWithValue(id, inputStepID, valueColumnID string, outputs } func projectionCaseConstruction(branch string) (*Construction, []Column, error) { - // root_filter is always index 0; extraOutputs are the non-public columns a - // carrying prefix step must publish for the projection to reach its owner. rootExtras := []string{} addRoot := func(ids ...string) { rootExtras = append(rootExtras, ids...) } finish := func(extraSteps ...ConstructionStep) (*Construction, []Column, error) { @@ -203,9 +188,6 @@ func projectionCaseProjection(branch string) ConstructionSourceProjection { return projectionFacts(column, owner) } -// projectionBranchExpectation is the single Go-side source of truth for the -// observable outcome of every branch; the Node lane and contract assert the -// same id/accept/substring triples through the batch channel. func projectionBranchExpectation(branch string) (accepted bool, contains string) { switch branch { case "pivot-accept", "group-accept", "filter-accept", "derive-accept", "unpivot-accept": diff --git a/internal/explorer/authoringv2/construction_row_values.go b/internal/explorer/authoringv2/construction_row_values.go index 26f7bd5a3..5b1a770c1 100644 --- a/internal/explorer/authoringv2/construction_row_values.go +++ b/internal/explorer/authoringv2/construction_row_values.go @@ -15,8 +15,6 @@ func constructionShapesRows(kind ConstructionOperationKind) bool { } } -// CanPopulateConstructionRows proves that a source column can reach every -// shaping boundary and continue through the remaining construction. func CanPopulateConstructionRows(construction *Construction) bool { if construction == nil { return false diff --git a/internal/explorer/authoringv2/construction_row_values_test.go b/internal/explorer/authoringv2/construction_row_values_test.go index e7c864f0b..96b636682 100644 --- a/internal/explorer/authoringv2/construction_row_values_test.go +++ b/internal/explorer/authoringv2/construction_row_values_test.go @@ -158,7 +158,6 @@ func TestNormalizeConstructionOutputsUsesAnchoredCohortSchema(t *testing.T) { { ID: "filter_cohorts", Inputs: []ConstructionInputRef{{Kind: ConstructionInputStepOutput, StepID: "group_rows"}}, Operation: ConstructionOperation{Kind: ConstructionOperationFilter, Filter: &ConstructionFilter{ColumnID: "group_label", Operator: ConstructionFilterExists}}, - // This is the UI's visible cohort schema: group_id is an internal identity. Outputs: []StageColumn{ {ID: "group_label", Name: "group_label", Label: "Group label", Type: "string"}, {ID: "group_ordinal", Name: "group_ordinal", Label: "Group ordinal", Type: "integer"}, diff --git a/internal/explorer/authoringv2/construction_validation.go b/internal/explorer/authoringv2/construction_validation.go index 0083e64ec..407fbd046 100644 --- a/internal/explorer/authoringv2/construction_validation.go +++ b/internal/explorer/authoringv2/construction_validation.go @@ -63,10 +63,6 @@ func (c *Construction) Validate(sourceColumns []Column) error { return validateConstructionStepOwnership(c) } -// ValidateWithExplicitGroups checks authored operations against the effective -// sequence formed by inserting the compiler-owned cohort stage at its saved -// boundary. References remain immediate: only the first operation after the -// cohort may consume the virtual group_rows stage. func (c *Construction) ValidateWithExplicitGroups(sourceColumns []Column, groups *GroupedRows) error { if groups == nil { return c.Validate(sourceColumns) @@ -398,8 +394,6 @@ func validatePivotSourceProjection(construction *Construction, projection Constr return nil } -// ConstructionCanCarrySourceProjection reports whether an operation preserves -// a root scalar source field in its output schema. func ConstructionCanCarrySourceProjection(kind ConstructionOperationKind) bool { switch kind { case ConstructionOperationDerive, ConstructionOperationFilter, ConstructionOperationExpand, diff --git a/internal/explorer/authoringv2/framing.go b/internal/explorer/authoringv2/framing.go index 033e8f41b..95cea205e 100644 --- a/internal/explorer/authoringv2/framing.go +++ b/internal/explorer/authoringv2/framing.go @@ -38,9 +38,6 @@ func validFrameForm(form capability.ConstructionChoiceForm) bool { } } -// FrameDefinition selects one metadata-backed correlated code/value family -// and an exact route. It is source guidance until matching categories are -// added; it does not change the row grain or compile an evaluator of its own. type FrameDefinition struct { ID string `json:"id"` Title string `json:"title"` @@ -93,7 +90,6 @@ func (f FrameDefinition) Validate() error { f.ZeroPolicy == FrameZeroNull && f.ManyPolicy == FrameManyFirst && f.Form == capability.ConstructionChoiceFirst || f.ZeroPolicy == FrameZeroEmptyList && f.ManyPolicy == FrameManyAll && f.Form == capability.ConstructionChoiceAll || f.ZeroPolicy == FrameZeroEmptyList && f.ManyPolicy == FrameManyDistinct && f.Form == capability.ConstructionChoiceDistinct { - // The selected form and its zero/many behavior are a closed policy pair. } else { return fmt.Errorf("frame zero/many policy does not match form %q", f.Form) } @@ -213,7 +209,6 @@ func cloneFrameRoute(route []capability.ConstructionRouteStep) []capability.Cons return cloned } -// ResolveFrameSource attaches a frame choice already authorized by lifecycle. func (c *Command) ResolveFrameSource(frame FrameDefinition) error { if c == nil || c.Type != CommandSetFrameSource && c.Type != CommandReplaceFrameSource { return fmt.Errorf("a frame source can only be resolved for SET_FRAME_SOURCE or REPLACE_FRAME_SOURCE") diff --git a/internal/explorer/authoringv2/migration.go b/internal/explorer/authoringv2/migration.go index b7efa0488..5587ae26c 100644 --- a/internal/explorer/authoringv2/migration.go +++ b/internal/explorer/authoringv2/migration.go @@ -7,9 +7,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// MigrateAggregateTemporalPolicy advances v8 workspaces to the split -// contributor-window and ordering contract. Persisted source conversion runs -// in DecodeWorkspace before this version boundary is applied. func MigrateAggregateTemporalPolicy(workspace Workspace) Workspace { if workspace.SemanticsVersion == aggregateTemporalPolicySemanticsVersion-1 { workspace.SemanticsVersion = aggregateTemporalPolicySemanticsVersion @@ -17,10 +14,6 @@ func MigrateAggregateTemporalPolicy(workspace Workspace) Workspace { return workspace } -// MigrateLegacyContributors converts the private v3 aggregate where wire into -// the v4 catalog-bound contributor intent. It is intentionally error-returning -// and transactional: callers must have the pinned catalog before interpreting -// a path, and an ambiguous draft is returned unchanged with an error. func MigrateLegacyContributors(workspace Workspace, catalog CatalogSnapshot) (Workspace, error) { if !workspaceHasLegacyContributors(workspace) { return workspace, nil @@ -82,9 +75,6 @@ func MigrateLegacyContributors(workspace Workspace, catalog CatalogSnapshot) (Wo return migrated, nil } -// PrepareWorkspaceForCompilation returns the canonical workspace representation -// used to bind compilation receipts. It works on a detached copy so callers can -// retain the exact persisted draft and its compare-and-swap digest. func PrepareWorkspaceForCompilation(workspace Workspace, catalog CatalogSnapshot) (Workspace, error) { prepared, err := cloneWorkspace(workspace) if err != nil { @@ -149,15 +139,10 @@ func contributorKindForCandidate(resourceType string, candidate CatalogCandidate return "" } -// MigrateLosslessDefaults upgrades mutable pre-v4 workspaces. Repeated field -// repetition can migrate to INDEXED when the catalog proves it. Related -// resource FIRST remains an explicit, unacknowledged lossy policy. Published -// receipts are immutable and never pass through this migration. func MigrateLosslessDefaults(workspace Workspace, catalog CatalogSnapshot) Workspace { if workspace.SemanticsVersion >= CurrentSemanticsVersion && !workspaceHasLegacyContributors(workspace) { return workspace } - // Install the v6 root-record meaning before the current version is assigned. workspace = migrateMissingRowsBeforeCurrent(workspace) legacyContributors := workspaceHasLegacyContributors(workspace) workspace.Documents = append([]Document(nil), workspace.Documents...) @@ -176,9 +161,6 @@ func MigrateLosslessDefaults(workspace Workspace, catalog CatalogSnapshot) Works if column.Source.Kind != SourceField || column.Source.Field == nil { continue } - // Columns are copied above, but their nested source payloads are - // pointers. Clone the payload before adding migration metadata so - // callers retain an immutable pre-migration workspace value. column.Source = column.Source.Normalized() mode := strings.ToUpper(strings.TrimSpace(column.Source.Field.ProjectionMode)) if mode == "" { @@ -197,8 +179,6 @@ func MigrateLosslessDefaults(workspace Workspace, catalog CatalogSnapshot) Works if mode != "FIRST" { continue } - // Semantics v2 made FIRST explicit. Only pre-v2 persisted drafts - // may reinterpret a repeated default FIRST as INDEXED. if workspace.SemanticsVersion >= 2 { continue } @@ -217,9 +197,6 @@ func MigrateLosslessDefaults(workspace Workspace, catalog CatalogSnapshot) Works } } } - // A legacy aggregate predicate cannot be interpreted without the pinned - // catalog. MigrateLegacyContributors is the required boundary for it; do - // not silently claim v4 while retaining the private v3 meaning. if !legacyContributors { workspace.SemanticsVersion = CurrentSemanticsVersion } diff --git a/internal/explorer/authoringv2/row_change.go b/internal/explorer/authoringv2/row_change.go index ebc4e9911..69bb3fcc0 100644 --- a/internal/explorer/authoringv2/row_change.go +++ b/internal/explorer/authoringv2/row_change.go @@ -37,8 +37,6 @@ type rowChangeCandidateRoute struct { AnchorDepth int } -// RowChangeRequest identifies a candidate row root and any deliberate choices -// needed to make the route rebase unambiguous. type RowChangeRequest struct { Mode string `json:"mode,omitempty"` OutputID string `json:"outputId"` @@ -47,9 +45,6 @@ type RowChangeRequest struct { RouteRebase []RouteRebaseChoice `json:"routeRebase,omitempty"` } -// RouteRebaseChoice selects the catalog edge used when an existing route edge -// must be inverted. OccurrenceID names the original parent that becomes a -// child after rerooting. type RouteRebaseChoice struct { OccurrenceID string `json:"occurrenceId"` EdgeID string `json:"edgeId"` @@ -105,9 +100,6 @@ type RowChangeAssessment struct { Unresolved []RowChangeUnresolvedReference `json:"unresolved"` } -// AssessRowChange is read-only. It promotes an authored descendant occurrence -// or derives a preserving catalog route to an eligible root absent from the -// authored route, provided each relationship can be inverted. func AssessRowChange(workspace Workspace, catalog CatalogSnapshot, request RowChangeRequest) (RowChangeAssessment, error) { if request.Mode == RowChangeModeReplace { return AssessRowReplacement(workspace, catalog, request) @@ -272,8 +264,6 @@ func AssessRowChange(workspace Workspace, catalog CatalogSnapshot, request RowCh return assessment, nil } -// AssessRowReplacement creates a destructive, same-output replacement proposal -// independently of preserve-route reachability. func AssessRowReplacement(workspace Workspace, catalog CatalogSnapshot, request RowChangeRequest) (RowChangeAssessment, error) { index := documentIndex(&workspace, request.OutputID) node, found := catalogNode(catalog, request.RootNodeID) @@ -321,8 +311,6 @@ func AssessRowReplacement(workspace Workspace, catalog CatalogSnapshot, request Mode: RowChangeModeReplace, PreservedFeatureKeys: []string{}, RemovedFeatureKeys: removed, WorkspaceOutputConsumers: consumers, SharedFilterBindings: bindings, Unresolved: []RowChangeUnresolvedReference{}}, nil } -// ApplyRowReplacement re-derives the proposal against the current workspace and -// exact authorized catalog snapshot before returning an immutable candidate. func ApplyRowReplacement(workspace Workspace, catalog CatalogSnapshot, proposal RowChangeProposal) (Workspace, error) { if proposal.Mode != RowChangeModeReplace { return Workspace{}, fmt.Errorf("row replacement proposal has unsupported mode %q", proposal.Mode) @@ -789,10 +777,6 @@ func rebaseRelatedSourceRoute( if related.SourceOccurrenceID != related.Source.NodeID { return nil, fmt.Errorf("saved source occurrence %q does not match its source node %q", related.SourceOccurrenceID, related.Source.NodeID) } - // Keep the original source anchor: reach the prior row root through the - // inverse selected path, then replay the saved related-source route from - // that root. A shared prefix does not make the source equivalent when the - // new root is itself related to the old root through another record. prefix, err := reverseConstructionRoutePath(path, pathNodeIDs, inverseChoices, catalog, 0) if err != nil { return nil, err @@ -854,11 +838,6 @@ func rebasePopulationRoute(prefix, existing []PopulationRouteStep, newRootNodeID return combined } - // Only remove a newly added rebase prefix against the old route at their - // seam. Exact catalog endpoints, relationship labels, and storage directions - // prove that each removed pair returns to the same node; resource-type - // equality alone is not sufficient because catalogs can contain distinct - // nodes with the same FHIR resource type. prefixEnd, existingStart := len(prefix), 0 for prefixEnd > 0 && existingStart < len(existing) && populationRouteStepsAreExactInverses(prefix[prefixEnd-1], existing[existingStart], catalog) { prefixEnd-- @@ -1078,8 +1057,6 @@ func deriveRowChangeCandidateRoutes(document Document, catalog CatalogSnapshot, } } if maxPrefix > 0 { - // A catalog path is attached to the deepest already-authored prefix, - // keeping authored occurrences and their match modes in place. for _, anchor := range anchors { if routeNodeIDs[anchor] == targetNodeID { continue diff --git a/internal/explorer/authoringv2/row_change_test.go b/internal/explorer/authoringv2/row_change_test.go index 195e81ebf..71b735882 100644 --- a/internal/explorer/authoringv2/row_change_test.go +++ b/internal/explorer/authoringv2/row_change_test.go @@ -638,7 +638,6 @@ func TestAssessRowChangePrefersTheAuthoredInverseAmongDifferentRelationships(t * t.Fatalf("rebased route=%#v err=%v, want Observation → Patient on the authored subject relationship", rebased.Route, err) } - // An explicit user choice still takes priority over the inferred inverse. assessment, err = AssessRowChange(workspace, catalog, RowChangeRequest{ OutputID: "patients", RootNodeID: "observation", RootOccurrenceID: "observation", RouteRebase: []RouteRebaseChoice{{OccurrenceID: RootOccurrenceID, EdgeID: "observation-patient-focus"}}, diff --git a/internal/explorer/authoringv2/row_definition.go b/internal/explorer/authoringv2/row_definition.go index 0fe1ed0fb..a5505a143 100644 --- a/internal/explorer/authoringv2/row_definition.go +++ b/internal/explorer/authoringv2/row_definition.go @@ -16,8 +16,6 @@ const ( RowDefinitionExpanded RowDefinitionKind = "EXPANDED" ) -// RowDefinition is a closed persisted union. Exactly one payload must match -// Kind; future composition semantics require a separate explicit variant. type RowDefinition struct { Kind RowDefinitionKind `json:"kind"` Records *RecordRows `json:"records,omitempty"` @@ -33,8 +31,6 @@ type GroupedRows struct { RowValues []ExplicitGroupRowValue `json:"rowValues,omitempty"` } -// ExplicitGroupRowValue retains a selected root FHIR field on each named -// group row using values from that group's authorized members. type ExplicitGroupRowValue struct { ColumnID string `json:"columnId"` Policy ConstructionRowValuePolicy `json:"policy"` @@ -47,8 +43,6 @@ const ( GroupSourceExplicit GroupSourceKind = "EXPLICIT" ) -// GroupSource is a closed union between a field key and an immutable group -// revision reference. type GroupSource struct { Kind GroupSourceKind `json:"kind"` Field *FieldGroupSource `json:"field,omitempty"` @@ -61,8 +55,6 @@ type FieldGroupSource struct { MissingKeyPolicy MissingKeyPolicy `json:"missingKeyPolicy"` } -// ExplicitGroupSource references a revision whose declared groups remain -// present even when empty. Its policy applies to root members in no group. type ExplicitGroupSource struct { RevisionID string `json:"revisionId"` UnassignedMemberPolicy UnassignedMemberPolicy `json:"unassignedMemberPolicy"` @@ -249,7 +241,6 @@ func requireTrimmedIdentity(value, name string) error { return nil } -// RecordsRowDefinition returns the default row definition for one row per root record. func RecordsRowDefinition() RowDefinition { return RowDefinition{Kind: RowDefinitionRecords, Records: &RecordRows{}} } @@ -270,9 +261,6 @@ func missingRowDefinition(rows RowDefinition) bool { return rows.Kind == "" && rows.Records == nil && rows.Groups == nil && rows.Expanded == nil } -// migrateMissingRowsBeforeCurrent installs the historical root-record behavior only -// across the semantics-version boundary. It copies the document slice before -// editing and deliberately leaves the version bump to its migration owner. func migrateMissingRowsBeforeCurrent(workspace Workspace) Workspace { if workspace.SemanticsVersion >= explicitRowsSemanticsVersion { return workspace diff --git a/internal/explorer/authoringv2/semantic_command.go b/internal/explorer/authoringv2/semantic_command.go index f8bdd23c3..00033b594 100644 --- a/internal/explorer/authoringv2/semantic_command.go +++ b/internal/explorer/authoringv2/semantic_command.go @@ -28,9 +28,6 @@ func (r SemanticSelectionReadiness) Addable() bool { return r.Status == SemanticReadinessReady || r.Status == SemanticReadinessReadyWithWarning } -// SemanticSelectionPlan is the shared result used by inventory browsing and -// command application. Source is absent when the observation cannot yet be -// represented as one complete compiler input. type SemanticSelectionPlan struct { Source *ColumnSource LogicalType string @@ -161,8 +158,6 @@ func semanticCategoryDefaultTitle(resourceType, sourcePath, system string) strin return strings.TrimSpace(resourceType + " " + field) } -// ResolveSemanticSelectionPlan validates the observed FHIR structure and -// produces the one source form used by both browsing and command application. func ResolveSemanticSelectionPlan(observation catalog.SemanticObservation) SemanticSelectionPlan { if observation.SchemaVersion != catalog.SemanticObservationSchemaVersion || observation.RuleVersion != strconv.Itoa(catalog.SemanticObservationRuleVersion) { return semanticUnsupported("SEMANTIC_OBSERVATION_VERSION_UNSUPPORTED", "This catalog observation uses an unsupported schema or rule version.") diff --git a/internal/explorer/authoringv2/semantic_types.go b/internal/explorer/authoringv2/semantic_types.go index f581ec9fa..ddb7a076a 100644 --- a/internal/explorer/authoringv2/semantic_types.go +++ b/internal/explorer/authoringv2/semantic_types.go @@ -20,9 +20,6 @@ type ExplorerMetadata struct { Description string `json:"description,omitempty"` } -// RouteNode is one semantic occurrence in a document route tree. Relationship -// is empty only for the root; every child names the relationship from its -// parent. Resource identities are stable FHIR resource types, not catalog IDs. type RouteNode struct { OccurrenceID string `json:"occurrenceId"` ResourceType string `json:"resourceType"` @@ -32,8 +29,6 @@ type RouteNode struct { Children []RouteNode `json:"children,omitempty"` } -// RouteMatchMode distinguishes row-defining relationships from feature-only -// relationships. Empty is the legacy OPTIONAL default. type RouteMatchMode string const ( @@ -60,17 +55,11 @@ type ColumnSource struct { OwnerRecords *OwnerRecordsSource `json:"ownerRecords,omitempty"` } -// OwnerRecordsSource preserves one repeated FHIR owner as one structured -// record in a list-valued cell. Binding and Key are server-resolved semantic -// identity; clients never supply arbitrary selectors for this source. type OwnerRecordsSource struct { Binding fhirschema.CorrelatedBinding `json:"binding"` Key fhirschema.CorrelatedKey `json:"key"` } -// UnitNormalizationPolicy is intentionally a nontechnical preset reference. -// The compiler owns target identity, dimensions, conversion rules, and FHIR -// Quantity sibling selectors. type UnitNormalizationPolicy struct { PolicyID string `json:"policyId"` Version string `json:"version"` @@ -117,9 +106,6 @@ func (p UnitNormalizationPolicy) Validate() error { return unit.ValidateUnitPolicyReference(p.PolicyID, p.Version) } -// FieldSource is the closed payload for a direct field projection. The -// related-selection annotation is intentionally nested in the field variant. -// It cannot accidentally be attached to a project, lookup, or aggregate. type FieldSource struct { Path string `json:"path"` ProjectionMode string `json:"projectionMode,omitempty"` @@ -130,9 +116,6 @@ type AggregateSource struct { Operation string `json:"operation"` Path string `json:"path,omitempty"` UnitNormalization *UnitNormalizationPolicy `json:"unitNormalization,omitempty"` - // Where is retained only for decoding immutable/pre-v3 in-memory recipes. - // It is deliberately omitted from the current authoring JSON contract; - // writable contributor intent lives on Column.Contributor instead. Where *SourceWhere `json:"-"` RequiredValues []string `json:"requiredValues,omitempty"` ContributorWindow *ContributorWindowSource `json:"contributorWindow,omitempty"` @@ -160,10 +143,6 @@ type SourceWhere struct { Equals string `json:"equals"` } -// UnmarshalJSON keeps the current aggregate source shape closed. Persisted -// legacy sources are decoded by decodePersistedSource, which is the only -// compatibility boundary allowed to interpret wherePath/whereEquals or a -// nested where object. func (s *AggregateSource) UnmarshalJSON(raw []byte) error { type wire struct { Operation string `json:"operation"` @@ -186,23 +165,13 @@ func (s *AggregateSource) UnmarshalJSON(raw []byte) error { } type LookupSource struct { - Match string `json:"match,omitempty"` - Path string `json:"path,omitempty"` - ProjectionMode string `json:"projectionMode,omitempty"` - // Identifier is the checked namespace/value alternative for - // identifierBySystem. It is mutually exclusive with legacy Match/Path and - // the coding or extension binding alternatives. - Identifier *fhirschema.IdentifierBinding `json:"identifier,omitempty"` - // Binding is the validated correlated FHIR shape used by codedValue. - // It is a closed alternative to legacy Match/Path, never a precedence rule - // between two writable lookup meanings. - Binding *fhirschema.CorrelatedBinding `json:"binding,omitempty"` - // Key is required with Binding and carries the selected system/code - // identity. - Key *fhirschema.CorrelatedKey `json:"key,omitempty"` - // Extension is the ancestor-aware closed alternative for extensionByUrl. - // It cannot coexist with legacy Match/Path or the terminology Binding/Key. - Extension *fhirschema.ExtensionBinding `json:"extension,omitempty"` + Match string `json:"match,omitempty"` + Path string `json:"path,omitempty"` + ProjectionMode string `json:"projectionMode,omitempty"` + Identifier *fhirschema.IdentifierBinding `json:"identifier,omitempty"` + Binding *fhirschema.CorrelatedBinding `json:"binding,omitempty"` + Key *fhirschema.CorrelatedKey `json:"key,omitempty"` + Extension *fhirschema.ExtensionBinding `json:"extension,omitempty"` } type RelatedSelection struct { @@ -210,8 +179,6 @@ type RelatedSelection struct { Acknowledged bool `json:"acknowledged"` } -// UnmarshalJSON keeps the writable source contract closed. Legacy flat source -// payloads are decoded only by DecodeWorkspace's persisted-draft migration. func (s *ColumnSource) UnmarshalJSON(raw []byte) error { var wire struct { Kind string `json:"kind"` @@ -397,8 +364,6 @@ const ( ) type Column struct { - // ColumnID is the stable identity used by staged construction steps. Column - // remains the public physical name and Label remains presentation text. ColumnID string `json:"columnId,omitempty"` FrameID string `json:"frameId,omitempty"` Column string `json:"column"` @@ -408,17 +373,12 @@ type Column struct { Source ColumnSource `json:"source"` ValueTransformation *columntransform.ValueTransformation `json:"valueTransformation,omitempty"` Contributor *ContributorPredicate `json:"contributor,omitempty"` - // Interpretation is nil for the existing inline meaning. A non-nil value - // is a closed authoring reference: PINNED names one immutable revision and - // never follows a mutable library head. - Interpretation *FeatureInterpretation `json:"interpretation,omitempty"` - Table *TablePresentation `json:"table,omitempty"` - Filter *FilterPresentation `json:"filter,omitempty"` - Chart *ChartPresentation `json:"chart,omitempty"` + Interpretation *FeatureInterpretation `json:"interpretation,omitempty"` + Table *TablePresentation `json:"table,omitempty"` + Filter *FilterPresentation `json:"filter,omitempty"` + Chart *ChartPresentation `json:"chart,omitempty"` } -// FeatureInterpretationKind is deliberately closed so a pinned column cannot -// smuggle a second interpretation language into the workspace. type FeatureInterpretationKind string const ( @@ -446,9 +406,6 @@ func (i FeatureInterpretation) Validate() error { return nil } -// ContributorPredicate is catalog intent scoped to one aggregate feature. -// CandidateID is resolved against the occurrence's pinned catalog at compile -// time; callers never provide a selector or AQL expression. type ContributorPredicate struct { CandidateID string `json:"candidateId"` Operator ContributorOperator `json:"operator"` diff --git a/internal/explorer/authoringv2/table_shape.go b/internal/explorer/authoringv2/table_shape.go index 0955c1307..af6cd1c05 100644 --- a/internal/explorer/authoringv2/table_shape.go +++ b/internal/explorer/authoringv2/table_shape.go @@ -49,8 +49,6 @@ type ColumnOutput struct { Label string `json:"label"` } -// ValidateTableShapeOutput applies the authoring document's exact output-name -// and label contract to one server-resolved construction descriptor. func ValidateTableShapeOutput(output ColumnOutput) error { if !physicalColumnPattern.MatchString(output.Column) { return fmt.Errorf("output column %q is not a valid public column name", output.Column) diff --git a/internal/explorer/authoringv2/transformations.go b/internal/explorer/authoringv2/transformations.go index a2a365bb5..bda4bf29d 100644 --- a/internal/explorer/authoringv2/transformations.go +++ b/internal/explorer/authoringv2/transformations.go @@ -33,8 +33,6 @@ type aggregateTransformNode struct { rowRootEligible bool } -// AggregateTransformationCapabilitiesForCatalog derives choices from the -// resolved catalog candidates and generated FHIR schema. func AggregateTransformationCapabilitiesForCatalog(catalog CatalogSnapshot, candidateID string) AggregateTransformationCapabilities { nodes := make([]aggregateTransformNode, 0, len(catalog.Nodes)) for _, node := range catalog.Nodes { @@ -61,8 +59,6 @@ func AggregateTransformationCapabilitiesForCatalog(catalog CatalogSnapshot, cand return aggregateTransformationCapabilities(nodes, candidates, candidateID) } -// AggregateTransformationCapabilitiesForCapability derives choices from the -// capability snapshot used at compilation time. func AggregateTransformationCapabilitiesForCapability(snapshot capability.Snapshot, candidateID string) AggregateTransformationCapabilities { nodes := make([]aggregateTransformNode, 0, len(snapshot.Nodes)) for _, node := range snapshot.Nodes { @@ -142,12 +138,10 @@ func aggregateTransformationCapabilities(nodes []aggregateTransformNode, candida return result } -// SupportsTimestamp reports whether the exact scalar timestamp is advertised. func (c TemporalReductionCapabilities) SupportsTimestamp(resourceType, fieldPath string) bool { return temporalFieldAvailable(c.TimestampFields, resourceType, fieldPath) } -// SupportsAnchor reports whether the exact root-row anchor is advertised. func (c TemporalReductionCapabilities) SupportsAnchor(resourceType, fieldPath string) bool { return temporalFieldAvailable(c.AnchorFields, resourceType, fieldPath) } diff --git a/internal/explorer/authoringv2/types.go b/internal/explorer/authoringv2/types.go index 000611878..9f51ac4ef 100644 --- a/internal/explorer/authoringv2/types.go +++ b/internal/explorer/authoringv2/types.go @@ -1,6 +1,3 @@ -// Package authoringv2 contains the hard-cut Explorer Builder authoring -// protocol. It is deliberately separate from the V1 HTTP and persistence -// contracts: this package describes user intent and catalog facts only. package authoringv2 import ( @@ -24,8 +21,6 @@ const ( tableShapeSemanticsVersion = 9 ) -// Document is the complete durable Builder intent. Route occurrences form a -// semantic resource tree rooted at RootResourceType. type Document struct { APIVersion string `json:"-"` Kind string `json:"kind"` @@ -37,17 +32,11 @@ type Document struct { Frames []FrameDefinition `json:"frames,omitempty"` Columns []Column `json:"columns"` TableShape *TableShape `json:"tableShape,omitempty"` - // Construction is the canonical post-source operation sequence. When it is - // present, Route/Rows/Columns describe only the source projection and - // TableShape must be nil. A nil value preserves the existing V2 document - // semantics and compiler path. - Construction *Construction `json:"construction,omitempty"` - FixedFilters []FixedFilter `json:"fixedFilters,omitempty"` - Actions []Action `json:"actions,omitempty"` + Construction *Construction `json:"construction,omitempty"` + FixedFilters []FixedFilter `json:"fixedFilters,omitempty"` + Actions []Action `json:"actions,omitempty"` } -// Population constrains an output's row roots to resources reachable from a -// completed immutable selection. type Population struct { SelectionRevisionID string `json:"selectionRevisionId"` Route []PopulationRouteStep `json:"route"` @@ -81,14 +70,10 @@ type Output struct { RowLabel string `json:"rowLabel,omitempty"` } -// Workspace is the atomic authoring unit. Documents are independent table -// intents; tabs provide their ordered, visible runtime presentation. type Workspace struct { - APIVersion string `json:"apiVersion"` - Kind string `json:"kind"` - SemanticsVersion int `json:"semanticsVersion,omitempty"` - // MigrationDecisions records automatic interpretation of ambiguous legacy - // intent so the choice remains visible and part of the workspace digest. + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + SemanticsVersion int `json:"semanticsVersion,omitempty"` MigrationDecisions []string `json:"migrationDecisions,omitempty"` Explorer ExplorerMetadata `json:"explorer"` Documents []Document `json:"documents"` @@ -121,8 +106,6 @@ type ChartPresentation struct { Order *int `json:"order,omitempty"` } -// CatalogSnapshot is an immutable, authorization-scoped projection. The -// token identifies the exact snapshot used to validate a document. type CatalogSnapshot struct { APIVersion string `json:"-"` Kind string `json:"-"` @@ -194,8 +177,6 @@ type CatalogCandidate struct { ConceptCandidates []ConceptCandidate `json:"conceptCandidates,omitempty"` } -// ConceptCandidate is an observed structural terminology/value candidate. -// It is evidence for authoring, not a claim of clinical equivalence. type ConceptCandidate struct { SourceResourceType string `json:"sourceResourceType"` SourcePath string `json:"sourcePath,omitempty"` @@ -226,8 +207,6 @@ type RepeatedBoundary struct { MaxItems int `json:"maxItems"` } -// RoutePolicy has no default hop ceiling: nil MaxHops means every finite -// route is valid. Repeated edges and self-loops are explicit capabilities. type RoutePolicy struct { MaxHops *int `json:"maxSteps"` Unbounded bool `json:"-"` @@ -235,7 +214,6 @@ type RoutePolicy struct { AllowSelfLoops bool `json:"allowSelfLoops"` } -// BuilderState joins one workspace to the one catalog snapshot that proves it. type BuilderState struct { APIVersion string `json:"apiVersion"` Kind string `json:"kind"` diff --git a/internal/explorer/authoringv2/validation.go b/internal/explorer/authoringv2/validation.go index b00f4a761..8ce2ff2a9 100644 --- a/internal/explorer/authoringv2/validation.go +++ b/internal/explorer/authoringv2/validation.go @@ -112,10 +112,6 @@ func (w Workspace) Validate() error { return nil } -// ValidateForPublication applies constraints that are intentionally too strict -// for mutable Builder state. Changing a table root or route temporarily clears -// its selections; that intermediate workspace remains compilable, but a visible -// table must have a visible source column or final construction output to publish. func (w Workspace) ValidateForPublication() error { visibleOutputs := make(map[string]bool, len(w.Tabs)) for _, tab := range w.Tabs { @@ -356,7 +352,3 @@ func (s BuilderState) Validate() error { func emptyID(value string) bool { return strings.TrimSpace(value) == "" || value != strings.TrimSpace(value) } - -// CanonicalJSON and Digest provide content-addressed, deterministic wire -// representations. Catalog collections and unordered selections are sorted; -// routeSteps retain their authored path order. diff --git a/internal/explorer/authoringv2/workspace_output_dependencies.go b/internal/explorer/authoringv2/workspace_output_dependencies.go index c2f88ca2d..37690ee0a 100644 --- a/internal/explorer/authoringv2/workspace_output_dependencies.go +++ b/internal/explorer/authoringv2/workspace_output_dependencies.go @@ -2,9 +2,6 @@ package authoringv2 import "fmt" -// OutputDependencyOrder returns document indexes in deterministic dependency -// order. References are to stable output IDs in this exact workspace; it does -// not consult publications or mutable table heads. func (w Workspace) OutputDependencyOrder() ([]int, error) { indexes := make(map[string]int, len(w.Documents)) for index, document := range w.Documents { diff --git a/internal/explorer/canonical_json.go b/internal/explorer/canonical_json.go index 3636f0b23..ec1e2e8da 100644 --- a/internal/explorer/canonical_json.go +++ b/internal/explorer/canonical_json.go @@ -7,10 +7,6 @@ import ( "io" ) -// canonicalJSONBytes returns compact JSON with deterministic object-key -// ordering. It normalizes raw JSON fields before they participate in a -// content address, so persistence stores that materialize objects and then -// reconstruct them cannot change the digest merely by changing key order. func canonicalJSONBytes(raw []byte) ([]byte, error) { decoder := json.NewDecoder(bytes.NewReader(raw)) decoder.UseNumber() diff --git a/internal/explorer/capability/builder.go b/internal/explorer/capability/builder.go index 77ea3116e..039bfa12a 100644 --- a/internal/explorer/capability/builder.go +++ b/internal/explorer/capability/builder.go @@ -45,9 +45,6 @@ type FieldObservation struct { ConceptCandidates []ConceptCandidate } -// Evidence is the complete catalog evidence set used to build a capability -// snapshot. Availability is explicit because an empty result is valid evidence -// for some generations, while an unavailable source must fail closed. type Evidence struct { Resources []ResourceObservation Relationships []RelationshipObservation @@ -86,9 +83,6 @@ type CandidateProof struct { Reason string } -// CompilerCallbacks are the compiler's proof boundary. A Builder advertises an -// operation only after the corresponding callback succeeds. In particular, a -// field is not trusted merely because the observation source reported it. type CompilerCallbacks struct { Node func(context.Context, Node) (NodeProof, error) Edge func(context.Context, Edge) (EdgeProof, error) @@ -96,10 +90,7 @@ type CompilerCallbacks struct { } type Builder struct { - Identity SnapshotIdentity - // EntityIdentity is the pre-availability semantic identity used only to - // mint stable node, edge, and candidate IDs. Cache-only metadata belongs - // in Identity so it can invalidate snapshots without rewriting authored IDs. + Identity SnapshotIdentity EntityIdentity SnapshotIdentity Policy Policy Complete bool @@ -115,8 +106,6 @@ func NewBuilder(identity SnapshotIdentity, evidence Evidence, compiler CompilerC return Builder{Identity: identity, Evidence: evidence, Compiler: compiler, Complete: true, Policy: Policy{Route: RoutePolicy{AllowsRepeatedEdges: true, AllowsSelfLoops: true}, Projection: ProjectionPolicy{SuggestionLimit: DefaultSuggestionLimit}}} } -// Build is pure: evidence is copied before it is sorted or used, and no -// caller-owned evidence or compiler state is modified. func (b Builder) Build(ctx context.Context) (Snapshot, error) { identity := b.Identity identity.Project = strings.TrimSpace(identity.Project) @@ -125,15 +114,11 @@ func (b Builder) Build(ctx context.Context) (Snapshot, error) { return b.failed(identity, "INVALID_SNAPSHOT_IDENTITY", "project and generation are required"), fmt.Errorf("project and generation are required") } if b.Complete == false && !b.Truncated { - // Explicitly incomplete discovery is a BUILDING snapshot, not an empty - // READY graph. The caller may publish it after a later complete build. } diags := []Diagnostic{} if !b.Evidence.ResourcesAvailable || !b.Evidence.RelationshipsAvailable || !b.Evidence.FieldsAvailable || b.Compiler.Node == nil || b.Compiler.Edge == nil || b.Compiler.Candidate == nil { return b.failedWith(identity, diags, "MISSING_CAPABILITY_DEPENDENCY", "resource, relationship, field, and compiler callbacks are required"), fmt.Errorf("capability dependencies are incomplete") } - // Copy before sorting or truncating so callers retain ownership of their - // evidence and nested suggestion slices. resources := append([]ResourceObservation(nil), b.Evidence.Resources...) relationships := append([]RelationshipObservation(nil), b.Evidence.Relationships...) fields := append([]FieldObservation(nil), b.Evidence.Fields...) @@ -168,8 +153,6 @@ func (b Builder) Build(ctx context.Context) (Snapshot, error) { if policy.Route.MaxHops < 0 { return b.failedWith(identity, diags, "INVALID_ROUTE_POLICY", "route MaxHops cannot be negative"), fmt.Errorf("route MaxHops cannot be negative") } - // Zero means unlimited. Defaults explicitly advertise the two traversal - // forms that the policy permits rather than relying on omitted JSON fields. if !policy.Route.AllowsRepeatedEdges && !policy.Route.AllowsSelfLoops { policy.Route.AllowsRepeatedEdges, policy.Route.AllowsSelfLoops = true, true } @@ -185,9 +168,6 @@ func (b Builder) Build(ctx context.Context) (Snapshot, error) { } idSeed := identityDigest(entityIdentity) - // A resource type is a concrete generated FHIR type, not a collection or - // arbitrary path. Deduplicate only exact resource observations; their - // populatedness remains an annotation of the single capability. resourceMap := map[string]ResourceObservation{} for _, raw := range resources { t := strings.TrimSpace(raw.ResourceType) @@ -245,8 +225,6 @@ func (b Builder) Build(ctx context.Context) (Snapshot, error) { nodeByType[t] = n } - // Sort relationships by their complete identity before assigning ordinals; - // this makes repeated edges deterministic and preserves self-loops. sort.SliceStable(relationships, func(i, j int) bool { return relationshipSortKey(relationships[i]) < relationshipSortKey(relationships[j]) }) @@ -282,8 +260,6 @@ func (b Builder) Build(ctx context.Context) (Snapshot, error) { edges = append(edges, e) } - // Candidate IDs are local to this snapshot and occurrence-stable even when - // evidence repeats the same path. Paths are validated before any probe. sort.SliceStable(fields, func(i, j int) bool { return fieldSortKey(fields[i]) < fieldSortKey(fields[j]) }) candidates := []Candidate{} auditCandidates := []Candidate{} diff --git a/internal/explorer/capability/builder_test.go b/internal/explorer/capability/builder_test.go index ad623d327..54a39b11c 100644 --- a/internal/explorer/capability/builder_test.go +++ b/internal/explorer/capability/builder_test.go @@ -202,7 +202,6 @@ func TestIDsAreBoundToSnapshotIdentity(t *testing.T) { func TestBuilderRejectsMalformedEvidenceAndCompilerFailures(t *testing.T) { obs := Evidence{Resources: []ResourceObservation{{ResourceType: "Patient"}}, Relationships: []RelationshipObservation{{SourceResourceType: "Patient", TargetResourceType: "Bad/Type", Label: "x"}}, Fields: []FieldObservation{{ResourceType: "Patient", Path: "name..given"}}} - // Build with a rejecting node leaves no usable root and retains audit data. b := testBuilder(obs) got, buildErr := b.Build(context.Background()) if buildErr != nil { diff --git a/internal/explorer/capability/construction.go b/internal/explorer/capability/construction.go index 13f29285f..9eb397629 100644 --- a/internal/explorer/capability/construction.go +++ b/internal/explorer/capability/construction.go @@ -76,13 +76,10 @@ const ( ConstructionChoiceSourceGroupField ConstructionChoiceSourceKind = "GROUP_FIELD" ) -// ConstructionChoiceSource is closed to the variants defined by this package. type ConstructionChoiceSource interface { constructionChoiceSource() } -// FieldChoiceSource identifies one exact, compiler-proved capability field. -// Labels stay outside the identity so presentation changes do not select data. type FieldChoiceSource struct { Kind ConstructionChoiceSourceKind `json:"kind"` CandidateID string `json:"candidateId"` @@ -95,9 +92,6 @@ type FieldChoiceSource struct { func (FieldChoiceSource) constructionChoiceSource() {} -// GroupFieldChoiceSource is an ephemeral, compiler-context-bound Group field -// capability. The choice token contains one input variant; flat fields are -// accepted only to decode pre-sum CURRENT_STAGE tokens. type GroupFieldChoiceSource struct { Kind ConstructionChoiceSourceKind `json:"kind"` TargetGroupStepID string `json:"targetGroupStepId,omitempty"` @@ -113,8 +107,6 @@ type GroupFieldChoiceSource struct { Shape string `json:"shape"` Reference bool `json:"reference"` - // Flat CURRENT_STAGE capability fields retained for existing builders and - // old choice-token decoding. New tokens carry these fields under Input. SourceRole string `json:"-"` OwnerOccurrenceID string `json:"-"` OwnerResourceType string `json:"-"` @@ -414,8 +406,6 @@ func (source *GroupFieldChoiceSource) normalizeAndValidateInput() error { func (GroupFieldChoiceSource) constructionChoiceSource() {} -// SemanticBindingChoiceSource preserves the observed FHIR owner and terminology -// identity. Display text is deliberately excluded from computational identity. type SemanticBindingChoiceSource struct { Kind ConstructionChoiceSourceKind `json:"kind"` GroupOutputID string `json:"groupOutputId,omitempty"` @@ -464,9 +454,6 @@ func validateSemanticCategoryChoiceSource(source SemanticBindingChoiceSource) er return nil } -// SemanticFrameFamily is the metadata identity of one correlated code/value -// binding, independent of any one observed code. It is persisted with a frame -// so later category choices can be checked against the same binding. type SemanticFrameFamily struct { BindingID string `json:"bindingId"` ResourceType string `json:"resourceType"` @@ -484,9 +471,6 @@ type SemanticFrameFamily struct { SchemaVersion int `json:"schemaVersion"` } -// SemanticFrameChoiceSource is a server-issued frame-family choice. The -// anchor concept proves the family against the current inventory; its selected -// code is intentionally absent so the frame represents the whole family. type SemanticFrameChoiceSource struct { Kind ConstructionChoiceSourceKind `json:"kind"` AnchorConceptID string `json:"anchorConceptId"` @@ -500,8 +484,6 @@ type SemanticFrameChoiceSource struct { func (SemanticFrameChoiceSource) constructionChoiceSource() {} -// RelatedResourceChoiceSource identifies one exact terminal resource node. -// It lets lifecycle authorize a route without requiring a field selection. type RelatedResourceChoiceSource struct { Kind ConstructionChoiceSourceKind `json:"kind"` StageID string `json:"stageId"` @@ -515,8 +497,6 @@ type RelatedResourceChoiceSource struct { func (RelatedResourceChoiceSource) constructionChoiceSource() {} -// RelatedFieldChoiceSource binds one scalar field to a specific stage that -// carries an exact terminal resource identity. type RelatedFieldChoiceSource struct { Kind ConstructionChoiceSourceKind `json:"kind"` StageID string `json:"stageId"` @@ -530,9 +510,6 @@ type RelatedFieldChoiceSource struct { func (RelatedFieldChoiceSource) constructionChoiceSource() {} -// CodedGroupChoiceSource binds a root Coding[] path to one output and one -// exact construction input stage. The persisted construction stores the path -// facts separately so it remains compilable after this snapshot expires. type CodedGroupChoiceSource struct { Kind ConstructionChoiceSourceKind `json:"kind"` OutputID string `json:"outputId"` @@ -583,10 +560,6 @@ type ConstructionChoiceFact struct { Value string `json:"value"` } -// ConstructionRouteStep pins one directed traversal in a construction choice. -// Capability edge identity, endpoint identity, generated relationship label, -// and physical storage direction are all retained so apply cannot substitute -// a different route that happens to reach the same resource type. type ConstructionRouteStep struct { EdgeID string `json:"edgeId"` FromNodeID string `json:"fromNodeId"` @@ -598,8 +571,6 @@ type ConstructionRouteStep struct { MatchMode string `json:"matchMode"` } -// ConstructionChoiceIdentity is the decoded, pinned identity used to -// re-resolve and reconstruct a server-issued choice during apply. type ConstructionChoiceIdentity struct { Version string Kind ConstructionChoiceSourceKind @@ -637,20 +608,14 @@ type constructionChoiceToken struct { Route []ConstructionRouteStep `json:"route"` } -// NewFieldConstructionChoice exposes only projection modes already proven by -// the compiler. Indexed is a Builder annotation, not a compiler projection. func NewFieldConstructionChoice(snapshotToken string, candidate Candidate) (ConstructionChoice, error) { return NewFieldConstructionChoiceForRoute(snapshotToken, nil, candidate) } -// IsSupportedConstructionSourceType reports whether the recipe compiler -// accepts a declared logical type for a source projection. func IsSupportedConstructionSourceType(logicalType string) bool { return recipe.IsValidConstructionLogicalType(logicalType) } -// ValidateConstructionSourceType rejects values that the recipe compiler -// cannot emit and, when available, points to the nearest typed child field. func ValidateConstructionSourceType(candidate Candidate, candidates []Candidate) error { if IsSupportedConstructionSourceType(candidate.LogicalType) { return nil @@ -678,9 +643,6 @@ func ValidateConstructionSourceType(candidate Candidate, candidates []Candidate) return fmt.Errorf("%s", message) } -// NewFieldConstructionChoiceForRoute exposes only projection modes already -// proven by the compiler for this exact route. A nil or empty route represents -// the zero-hop row-root choice. func NewFieldConstructionChoiceForRoute(snapshotToken string, route []ConstructionRouteStep, candidate Candidate) (ConstructionChoice, error) { if strings.TrimSpace(snapshotToken) == "" || strings.TrimSpace(candidate.ID) == "" || strings.TrimSpace(candidate.NodeID) == "" || strings.TrimSpace(candidate.ResourceType) == "" || strings.TrimSpace(candidate.FieldPath) == "" { return ConstructionChoice{}, fmt.Errorf("snapshot token and exact candidate identity are required") @@ -711,8 +673,6 @@ func NewFieldConstructionChoiceForRoute(snapshotToken string, route []Constructi return choice, nil } -// NewGroupFieldConstructionChoice creates an ephemeral Group-key authority -// from compiler-issued source context and one authorized scalar candidate. func NewGroupFieldConstructionChoice(snapshotToken string, route []ConstructionRouteStep, source GroupFieldChoiceSource, candidate Candidate) (ConstructionChoice, error) { source.Kind = ConstructionChoiceSourceGroupField if strings.TrimSpace(snapshotToken) == "" { @@ -758,9 +718,6 @@ func NewGroupFieldConstructionChoice(snapshotToken string, route []ConstructionR return choice, nil } -// NewGroupCodingIdentityChoice exposes a compiler-proved current Coding item -// as its normalized terminology identity without inventing an inventory -// family binding. func NewGroupCodingIdentityChoice(snapshotToken string, route []ConstructionRouteStep, source GroupFieldChoiceSource) (ConstructionChoice, error) { source.Kind = ConstructionChoiceSourceGroupField source.ValueKind = "CODING_IDENTITY" @@ -1029,8 +986,6 @@ func containsProjectionMode(modes []ProjectionMode, want ProjectionMode) bool { return false } -// NewConstructionRelatedResourceRouteChoiceFromAnchor pins a route to one -// exact compiler-proven row anchor as well as its terminal resource. func NewConstructionRelatedResourceRouteChoiceFromAnchor( snapshotToken, stageID, anchorColumnID, anchorKind, anchorNodeID, anchorResource, targetNodeID, targetResource string, route []ConstructionRouteStep, @@ -1073,8 +1028,6 @@ func NewConstructionRelatedResourceRouteChoiceFromAnchor( }, nil } -// NewConstructionRelatedFieldChoice issues a stage-bound choice for one -// compiler catalog scalar field on the active exact terminal resource. func NewConstructionRelatedFieldChoice(snapshotToken, stageID string, candidate Candidate) (ConstructionRelatedFieldChoice, error) { if strings.TrimSpace(snapshotToken) == "" || strings.TrimSpace(stageID) == "" || strings.TrimSpace(candidate.ID) == "" || strings.TrimSpace(candidate.NodeID) == "" || @@ -1106,8 +1059,6 @@ func NewConstructionRelatedFieldChoice(snapshotToken, stageID string, candidate return ConstructionRelatedFieldChoice{ChoiceID: choiceID, Source: source, Label: label}, nil } -// NewConstructionCodedGroupChoice issues a choice for one generated root -// Coding[] path on one exact source-projection stage. func NewConstructionCodedGroupChoice( snapshot Snapshot, outputID, stageID string, @@ -1152,15 +1103,10 @@ func NewConstructionCodedGroupChoice( }, nil } -// NewSemanticConstructionChoice binds an inventory observation to the exact -// compiler candidate for its value path. Snapshot, semantic context, and build -// identities pin the choice without adding separate mutable context fields. func NewSemanticConstructionChoice(snapshotToken, semanticContextToken, buildID string, source SemanticBindingChoiceSource, candidate Candidate, ownerRecordsProved ...bool) (ConstructionChoice, error) { return NewSemanticConstructionChoiceForRoute(snapshotToken, semanticContextToken, buildID, nil, source, candidate, ownerRecordsProved...) } -// NewSemanticConstructionChoiceForRoute pins an observed semantic binding to -// the exact compiler-proved route and terminal candidate. func NewSemanticConstructionChoiceForRoute(snapshotToken, semanticContextToken, buildID string, route []ConstructionRouteStep, source SemanticBindingChoiceSource, candidate Candidate, ownerRecordsProved ...bool) (ConstructionChoice, error) { if (source.GroupOutputID == "") != (source.GroupStageID == "") || (source.GroupOutputID != "" && (strings.TrimSpace(source.GroupOutputID) != source.GroupOutputID || strings.TrimSpace(source.GroupStageID) != source.GroupStageID)) { @@ -1228,8 +1174,6 @@ func NewSemanticConstructionChoiceForRoute(snapshotToken, semanticContextToken, return choice, nil } -// NewSemanticFrameConstructionChoice issues an exact family-and-route token -// for a correlated code/value source, without binding it to one category code. func NewSemanticFrameConstructionChoice( snapshotToken, semanticContextToken, buildID string, source SemanticFrameChoiceSource, @@ -1326,8 +1270,6 @@ func ownerRecordsSourceSupported(source SemanticBindingChoiceSource, candidate C return false } -// DecodeConstructionChoiceID validates the token envelope and returns the -// typed pinned identity. The caller must still re-authorize and re-resolve it. func DecodeConstructionChoiceID(choiceID string) (ConstructionChoiceIdentity, error) { if len(choiceID) == 0 || len(choiceID) > ConstructionChoiceIDMaxLength { return ConstructionChoiceIdentity{}, fmt.Errorf("construction choice id has an invalid length") diff --git a/internal/explorer/capability/construction_routes.go b/internal/explorer/capability/construction_routes.go index 32e9cde0c..b359fac58 100644 --- a/internal/explorer/capability/construction_routes.go +++ b/internal/explorer/capability/construction_routes.go @@ -19,20 +19,11 @@ const ( constructionRouteMaxResults = 4096 ) -// ConstructionRouteSearch is an exact server-side route query. TargetNodeID -// and SourceKey are derived from the selected, authorized source at the -// lifecycle boundary; callers must not use this planner as a path authority. type ConstructionRouteSearch struct { - Snapshot Snapshot - RootResource string - // RootNodeID pins traversal to an exact retained record type when the - // route starts from a compiler-proven related-record identity. Exactly one - // of RootResource and RootNodeID must be supplied. - RootNodeID string - TargetNodeID string - // TargetNodeIDs supports route searches whose terminal is one of several - // equivalent capability nodes (for example a population's resource type). - // It is mutually exclusive with TargetNodeID. + Snapshot Snapshot + RootResource string + RootNodeID string + TargetNodeID string TargetNodeIDs []string SourceKey string Cursor string @@ -55,10 +46,6 @@ type constructionRouteCursor struct { Offset int `json:"offset"` } -// PlanConstructionRoutes enumerates directed, policy-allowed routes in -// shortest-first, stable edge-identity order. The five-hop default is an -// explicit search budget when policy is unbounded; reaching that horizon is -// reported as truncated. func PlanConstructionRoutes(request ConstructionRouteSearch) (ConstructionRoutePage, error) { page := ConstructionRoutePage{Routes: [][]ConstructionRouteStep{}} snapshot := request.Snapshot @@ -146,7 +133,6 @@ func PlanConstructionRoutes(request ConstructionRouteSearch) (ConstructionRouteP nodeID string path []ConstructionRouteStep } - // Stable-sorted adjacency and FIFO expansion keep hop layers in edge-key order. states := []routeState{{nodeID: root.ID}} expansions := 0 searchTruncated := false @@ -201,8 +187,6 @@ func PlanConstructionRoutes(request ConstructionRouteSearch) (ConstructionRouteP break } if expansionCapped { - // Count target routes already reached by the final allowed expansion. - // The pending states are still in hop and edge-identity order. for _, pending := range states[stateIndex+1:] { if targetSet[pending.nodeID] && !recordRoute(pending.path) { break diff --git a/internal/explorer/capability/domain.go b/internal/explorer/capability/domain.go index 8276735eb..16c74b522 100644 --- a/internal/explorer/capability/domain.go +++ b/internal/explorer/capability/domain.go @@ -1,7 +1,3 @@ -// Package capability contains the immutable, compiler-backed contract exposed -// by Explorer's traversal Builder. It deliberately has no dependency on the -// database, generated schema, or HTTP layers: adapters provide observations and -// compiler proofs through the small interfaces in builder.go. package capability import ( @@ -60,39 +56,29 @@ const ( ChartAvg ChartAggregation = "AVG" ) -// SnapshotIdentity contains every external identity that can change the -// meaning of a capability document. Versions are intentionally independent: -// changing a renderer/compiler artifact must invalidate old Builder state even -// when the observed graph is unchanged. type SnapshotIdentity struct { Project string `json:"project"` Generation string `json:"generation"` AuthorizationScopeDigest string `json:"authorizationScopeDigest"` - // CacheVersion invalidates current-identity lookups when the retained - // snapshot payload gains additive metadata. Exact retained tokens remain - // resolvable under their original identity. - CacheVersion string `json:"cacheVersion,omitempty"` - SchemaDigest string `json:"schemaDigest"` - ResourceInventoryDigest string `json:"resourceInventoryDigest"` - RelationshipDigest string `json:"relationshipDigest"` - FieldDigest string `json:"fieldDigest"` - ShapeDigest string `json:"shapeDigest,omitempty"` - ProtocolVersion string `json:"protocolVersion"` - CompilerVersion string `json:"compilerVersion"` - TraversalPolicyVersion string `json:"traversalPolicyVersion"` - ProjectionPolicyVersion string `json:"projectionPolicyVersion"` + CacheVersion string `json:"cacheVersion,omitempty"` + SchemaDigest string `json:"schemaDigest"` + ResourceInventoryDigest string `json:"resourceInventoryDigest"` + RelationshipDigest string `json:"relationshipDigest"` + FieldDigest string `json:"fieldDigest"` + ShapeDigest string `json:"shapeDigest,omitempty"` + ProtocolVersion string `json:"protocolVersion"` + CompilerVersion string `json:"compilerVersion"` + TraversalPolicyVersion string `json:"traversalPolicyVersion"` + ProjectionPolicyVersion string `json:"projectionPolicyVersion"` } type RoutePolicy struct { Version string `json:"version"` - MaxHops int `json:"maxHops"` // zero means unlimited + MaxHops int `json:"maxHops"` AllowsRepeatedEdges bool `json:"allowsRepeatedEdges"` AllowsSelfLoops bool `json:"allowsSelfLoops"` } -// Allows accepts every finite route (including zero-hop routes, repeated -// edges, and self-loops) unless the caller explicitly configured MaxHops. -// There is deliberately no implicit four-hop limit. func (p RoutePolicy) Allows(route []string) bool { if p.MaxHops > 0 && len(route) > p.MaxHops { return false @@ -210,9 +196,6 @@ type Candidate struct { BlockedReason string `json:"blockedReason,omitempty"` } -// IsRepeatedCardinality is the single wire-cardinality interpretation used by -// Builder catalog adapters. Capability snapshots use dataframe cardinalities, -// while older persisted snapshots may contain upper-case values. func IsRepeatedCardinality(value string) bool { switch strings.ToLower(strings.TrimSpace(value)) { case "many", "unknown_observed_many": @@ -222,9 +205,6 @@ func IsRepeatedCardinality(value string) bool { } } -// ConceptCandidate carries observed FHIR identity into Builder without -// claiming terminology equivalence. It intentionally retains unresolved and -// mixed-choice statuses so the authoring layer can present them explicitly. type ConceptCandidate struct { SourceResourceType string `json:"sourceResourceType"` SourcePath string `json:"sourcePath,omitempty"` @@ -270,10 +250,6 @@ type Diagnostic struct { Retryable bool `json:"retryable,omitempty"` } -// Snapshot is immutable by convention: constructors and accessors copy all -// maps/slices, while Token is derived from a private canonical payload. The -// exported fields keep the package ergonomic for JSON adapters; use Clone (or -// the accessors) when handing a snapshot to code that may mutate values. type Snapshot struct { Identity SnapshotIdentity `json:"identity"` Policy Policy `json:"policy"` @@ -336,7 +312,6 @@ func (s Snapshot) Candidate(id string) (Candidate, bool) { return Candidate{}, false } -// CanonicalPayload returns a fresh copy of the exact payload hashed into Token. func (s Snapshot) CanonicalPayload() []byte { c := cloneSnapshot(s) c.Token = "" diff --git a/internal/explorer/capability/population_route_choice.go b/internal/explorer/capability/population_route_choice.go index f5888361a..f4083d79c 100644 --- a/internal/explorer/capability/population_route_choice.go +++ b/internal/explorer/capability/population_route_choice.go @@ -12,9 +12,6 @@ import ( const PopulationRouteChoiceIDMaxLength = 16384 -// PopulationRouteChoiceIdentity pins an exact route to one current table and -// immutable resource selection. It is identity, not authorization: apply -// must still load the selection and re-authorize every route edge. type PopulationRouteChoiceIdentity struct { Version int SnapshotToken string @@ -31,7 +28,6 @@ type populationRouteChoiceToken struct { Route []ConstructionRouteStep `json:"route"` } -// NewPopulationRouteChoiceID creates a stable opaque identifier for a route. func NewPopulationRouteChoiceID(identity PopulationRouteChoiceIdentity) (string, error) { if identity.Version == 0 { identity.Version = 1 @@ -55,8 +51,6 @@ func NewPopulationRouteChoiceID(identity PopulationRouteChoiceIdentity) (string, return "pr1." + base64.RawURLEncoding.EncodeToString(payload) + "." + hex.EncodeToString(digest[:]), nil } -// DecodePopulationRouteChoiceID checks the token envelope and exact route. -// The caller must still re-authorize the snapshot, selection, and every edge. func DecodePopulationRouteChoiceID(value string) (PopulationRouteChoiceIdentity, error) { if len(value) == 0 || len(value) > PopulationRouteChoiceIDMaxLength { return PopulationRouteChoiceIdentity{}, fmt.Errorf("population route choice id has an invalid length") diff --git a/internal/explorer/capability/repository.go b/internal/explorer/capability/repository.go index f2bcb5f08..962b40ae8 100644 --- a/internal/explorer/capability/repository.go +++ b/internal/explorer/capability/repository.go @@ -14,9 +14,6 @@ var ( ErrTokenCollision = errors.New("capability snapshot token collision") ) -// Repository persists complete snapshots. Put is content-addressed and never -// changes an existing document. GetByIdentity returns a READY snapshot when -// one exists; otherwise it returns the best available diagnostic snapshot. type Repository interface { GetByToken(context.Context, string) (*Snapshot, error) GetByIdentity(context.Context, SnapshotIdentity) (*Snapshot, error) @@ -37,8 +34,6 @@ func validateSnapshot(snapshot Snapshot) error { return nil } -// ValidateSnapshot applies repository invariants without requiring a caller -// to select a particular persistence adapter. func ValidateSnapshot(snapshot Snapshot) error { return validateSnapshot(snapshot) } func snapshotToken(snapshot Snapshot) string { diff --git a/internal/explorer/capability/row_choice.go b/internal/explorer/capability/row_choice.go index 1be08b5bf..086d6a6bc 100644 --- a/internal/explorer/capability/row_choice.go +++ b/internal/explorer/capability/row_choice.go @@ -32,7 +32,6 @@ const ( RowChoiceArray RowChoiceShape = "ARRAY" ) -// RowChoiceFacts are detached values adapted from generated schema metadata. type RowChoiceFacts struct { ResourceType string CanonicalPath string @@ -44,7 +43,6 @@ type RowChoiceFacts struct { Description string } -// RowChoiceOccurrence identifies one exact occurrence in an authorized route. type RowChoiceOccurrence struct { OccurrenceID string NodeID string @@ -97,7 +95,6 @@ type rowChoiceToken struct { type RowChoicePathResolver func(resourceType, canonicalPath string) (RowChoiceFacts, error) -// NewRowChoice binds schema facts to exactly one authorized route occurrence. func NewRowChoice(snapshot Snapshot, occurrences []RowChoiceOccurrence, occurrenceID string, kind RowChoiceKind, facts RowChoiceFacts) (RowChoice, error) { if err := snapshot.ValidateToken(snapshot.Token); err != nil { return RowChoice{}, err @@ -180,7 +177,6 @@ func rowChoiceValueType(facts RowChoiceFacts) string { } } -// ResolveRowChoiceID rechecks a choice against the current schema and route. func ResolveRowChoiceID(snapshot Snapshot, occurrences []RowChoiceOccurrence, choiceID string, resolve RowChoicePathResolver) (RowChoice, error) { identity, err := DecodeRowChoiceID(choiceID) if err != nil { diff --git a/internal/explorer/capability/value_operations.go b/internal/explorer/capability/value_operations.go index 52839173e..2d60417b0 100644 --- a/internal/explorer/capability/value_operations.go +++ b/internal/explorer/capability/value_operations.go @@ -65,9 +65,6 @@ var aggregateOperations = []AggregateOperation{ AggregateFirstOrdered, } -// DeriveAggregateOperationCapabilities gives every operation a typed, -// row-context-specific support decision. Compiler probes are intersected with -// this domain decision by the server before the choice is advertised. func DeriveAggregateOperationCapabilities(input AggregateInput, rows AggregateRowContext) []AggregateOperationCapability { choices := make([]AggregateOperationCapability, 0, len(aggregateOperations)) for _, operation := range aggregateOperations { diff --git a/internal/explorer/compilation/acceptance_observation_aql_diagnostic_test.go b/internal/explorer/compilation/acceptance_observation_aql_diagnostic_test.go index 968bf20d0..c3e53d37a 100644 --- a/internal/explorer/compilation/acceptance_observation_aql_diagnostic_test.go +++ b/internal/explorer/compilation/acceptance_observation_aql_diagnostic_test.go @@ -16,6 +16,7 @@ import ( driver "github.com/arangodb/go-driver/v2/arangodb" "github.com/arangodb/go-driver/v2/connection" "github.com/arangodb/go-driver/v2/utils" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/compiler" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" @@ -24,7 +25,6 @@ import ( "github.com/calypr/loom/internal/dataframe/semantic" "github.com/calypr/loom/internal/explorer/authoringv2" "github.com/calypr/loom/internal/explorer/capability" - arangostore "github.com/calypr/loom/internal/store/arango" ) func TestAcceptanceObservationContributorAQLAndBinds(t *testing.T) { @@ -191,10 +191,11 @@ func executeAcceptanceObservationAQL( collections = append(collections, collection) } - store, err := arangostore.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatalf("open configured ArangoDB through repository query helper: %v", err) } + store := arangoBackend.Client project, generation := "acceptance", "tcga-brca-locked" vertices := map[string][]json.RawMessage{ patientCollection: {acceptanceRawDocument(t, map[string]any{ @@ -242,8 +243,6 @@ func executeAcceptanceObservationAQL( edges := []json.RawMessage{ acceptanceRawDocument(t, acceptanceEdge("patient_condition", conditionCollection+"/c1", patientCollection+"/p1", "Condition", "Patient", "subject_Patient", project, generation)), acceptanceRawDocument(t, acceptanceEdge("patient_specimen", specimenCollection+"/s1", patientCollection+"/p1", "Specimen", "Patient", "subject_Patient", project, generation)), - // This deliberately inconsistent stored edge models the case scalar traversal rejects: - // its endpoint is a Specimen, but its discriminator claims Condition. acceptanceRawDocument(t, acceptanceEdge("patient_specimen_wrong_edge_type", specimenCollection+"/s_wrong_edge_type", patientCollection+"/p1", "Condition", "Patient", "subject_Patient", project, generation)), acceptanceRawDocument(t, acceptanceEdge("patient_age", observationCollection+"/o_age", patientCollection+"/p1", "Observation", "Patient", "subject_Patient", project, generation)), acceptanceRawDocument(t, acceptanceEdge("patient_method", observationCollection+"/o_method", patientCollection+"/p1", "Observation", "Patient", "subject_Patient", project, generation)), @@ -304,37 +303,35 @@ func executeAcceptanceObservationAQL( } } - // Compare the direct renderer output with the canonical execution compiler - // and the selected-root page template used by production preview streams. rawRows := execute("direct-render", query, cloneBinds(bindVars)) expectValues("direct-render", rawRows[0]) previewBindings := bindings.Clone() previewBindings.PreviewLimit = 25 - full, err := compiler.CompileRecipeOutputWithPolicy(output, previewBindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + full, err := compiler.CompileRecipeOutput(compiler.ArangoTarget(), output, previewBindings, 25, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile production full-preview query: %v", err) } - fullBindLog, err := json.Marshal(full.BindVars) + fullBindLog, err := json.Marshal(testAQLBinds(t, full.Command)) if err != nil { t.Fatal(err) } - t.Logf("production full-preview AQL:\n%s\nBINDVARS: %s", full.Query, fullBindLog) - fullRows := execute("production-full", full.Query, cloneBinds(full.BindVars)) + t.Logf("production full-preview AQL:\n%s\nBINDVARS: %s", testAQLText(t, full.Command), fullBindLog) + fullRows := execute("production-full", testAQLText(t, full.Command), cloneBinds(testAQLBinds(t, full.Command))) expectValues("production-full", fullRows[0]) - page, err := compiler.CompileRecipeOutputPageWithPolicy(output, previewBindings, 1, ir.DefaultPhysicalOptimizationPolicy()) + page, err := compiler.CompileRecipeOutputPage(compiler.ArangoTarget(), output, previewBindings, 1, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile production selected-root page: %v", err) } - pageRowsBindLog, err := json.Marshal(page.RowsBindVars) + pageRowsBindLog, err := json.Marshal(testAQLBinds(t, page.Rows)) if err != nil { t.Fatal(err) } - t.Logf("production selected-root page rows AQL:\n%s\nBINDVARS: %s", page.RowsQuery, pageRowsBindLog) - keyBinds := cloneBinds(page.RootKeysBindVars) + t.Logf("production selected-root page rows AQL:\n%s\nBINDVARS: %s", testAQLText(t, page.Rows), pageRowsBindLog) + keyBinds := cloneBinds(testAQLBinds(t, page.RootKeys)) keyBinds[compiler.RootPageAfterKeyBind] = "" keyBinds = cloneBinds(keyBinds) rootKeys := make([]string, 0, 1) - if err := store.QueryRows(ctx, page.RootKeysQuery, 10, keyBinds, func(row map[string]any) error { + if err := store.QueryRows(ctx, testAQLText(t, page.RootKeys), 10, keyBinds, func(row map[string]any) error { key, ok := row["_key"].(string) if !ok || key == "" { return fmt.Errorf("root key page returned invalid _key %#v", row["_key"]) @@ -342,14 +339,14 @@ func executeAcceptanceObservationAQL( rootKeys = append(rootKeys, key) return nil }); err != nil { - t.Fatalf("execute production root-key page: %v\n%s", err, page.RootKeysQuery) + t.Fatalf("execute production root-key page: %v\n%s", err, testAQLText(t, page.RootKeys)) } if len(rootKeys) != 1 || rootKeys[0] != "p1" { t.Fatalf("production root-key page = %#v, want [p1]", rootKeys) } - pageBinds := cloneBinds(page.RowsBindVars) + pageBinds := cloneBinds(testAQLBinds(t, page.Rows)) pageBinds[compiler.RootPageKeysBind] = rootKeys - pagedRows := execute("production-paged", page.RowsQuery, pageBinds) + pagedRows := execute("production-paged", testAQLText(t, page.Rows), pageBinds) expectValues("production-paged", pagedRows[0]) for _, column := range []string{"age_at_diagnosis_days", "diagnostic_method", "days_to_death", "earliest_collection_day", "latest_collection_day"} { if rawRows[0][column] != fullRows[0][column] || rawRows[0][column] != pagedRows[0][column] { diff --git a/internal/explorer/compilation/command_helpers_test.go b/internal/explorer/compilation/command_helpers_test.go new file mode 100644 index 000000000..75e168afc --- /dev/null +++ b/internal/explorer/compilation/command_helpers_test.go @@ -0,0 +1,37 @@ +package compilation + +import ( + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" +) + +func testAQLView(t *testing.T, command command.CompiledCommand) (string, map[string]any) { + t.Helper() + query, bindVars, err := command.AQLParameters() + if err != nil { + t.Fatalf("derive AQL parameters from %s command: %v", command.Backend(), err) + } + return query, bindVars +} + +func testAQLCommand(t *testing.T, query string, bindVars map[string]any) command.CompiledCommand { + t.Helper() + compiledCommand, err := command.NewAQLCommand(query, bindVars) + if err != nil { + t.Fatalf("build AQL command for test: %v", err) + } + return compiledCommand +} + +func testAQLText(t *testing.T, command command.CompiledCommand) string { + t.Helper() + query, _ := testAQLView(t, command) + return query +} + +func testAQLBinds(t *testing.T, command command.CompiledCommand) map[string]any { + t.Helper() + _, bindVars := testAQLView(t, command) + return bindVars +} diff --git a/internal/explorer/compilation/compile.go b/internal/explorer/compilation/compile.go index a0f6b9f0a..567c7d110 100644 --- a/internal/explorer/compilation/compile.go +++ b/internal/explorer/compilation/compile.go @@ -1,6 +1,3 @@ -// Package compilation translates semantic Builder intent into the persistence- -// neutral recipe understood by the dataframe compiler. It deliberately has no -// HTTP, store, or server dependencies: a capability snapshot is its authority. package compilation import ( @@ -22,8 +19,6 @@ import ( const TranslationVersion = "authoring-v2-native-10" -// Error is a structured translation failure. Stage, Code, Path, and Details -// remain transport-neutral so adapters never parse error strings. type Error struct { Stage string Code string @@ -54,8 +49,6 @@ func fail(stage, code, path, message string, details map[string]any, cause error return &Error{Stage: stage, Code: code, Path: path, Message: message, Details: details, Cause: cause} } -// PresentationConfig is compiler-owned presentation data. The Builder may -// provide labels, visibility, and order, but never executable selectors. type PresentationConfig struct { OutputID string `json:"outputId"` Title string `json:"title"` @@ -77,7 +70,6 @@ type PresentationColumn struct { ChartOrder int `json:"chartOrder,omitempty"` } -// Result is one deterministic semantic document compilation. type Result struct { Bundle recipe.Bundle RecipeDigest string @@ -87,8 +79,6 @@ type Result struct { OutputContract explorer.PublicOutputContract } -// WorkspaceResult combines independently compiled table documents into one -// recipe and one public Explorer configuration. type WorkspaceResult struct { Bundle recipe.Bundle RecipeDigest string @@ -100,9 +90,6 @@ type WorkspaceResult struct { Workspace authoringv2.Workspace } -// ResolvedPopulation is the immutable, checked external input for one output. -// It is kept separate from the authoring workspace so changing a selection -// revision can never mutate draft intent in place. type ResolvedPopulation struct { OutputID string `json:"outputId"` SelectionRevisionID string `json:"selectionRevisionId"` @@ -112,9 +99,6 @@ type ResolvedPopulation struct { Route []authoringv2.PopulationRouteStep `json:"route,omitempty"` } -// ResolvedInterpretation is an alias of the receipt-safe explorer domain -// value. Keeping the canonical value in explorer avoids a package cycle while -// preserving a single shape for lifecycle, compilation, and receipts. type ResolvedInterpretation = explorer.ResolvedInterpretation func CompileWorkspace(ctx context.Context, project, explorerID string, workspace authoringv2.Workspace, snapshot capability.Snapshot, resolvedInputs ResolvedInputs) (WorkspaceResult, error) { @@ -140,10 +124,6 @@ func CompileWorkspace(ctx context.Context, project, explorerID string, workspace if err != nil { return WorkspaceResult{}, fail("intent", "RESOLVED_INPUTS_DIGEST_FAILED", "$.workspace", "resolved input identity could not be calculated", nil, err) } - // Keep the authored workspace (including exact PINNED references) in the - // result and receipt. Effective definitions are applied only to this clone; - // the source suggestion and inline anchor can therefore never win over a - // resolved human definition during compilation. effectiveWorkspace := applyResolvedInterpretations(workspace, resolvedInputs) if err := (authoringv2.BuilderState{APIVersion: authoringv2.APIVersion, Kind: authoringv2.StateKind, Workspace: &effectiveWorkspace, Catalog: wire}).Validate(); err != nil { return WorkspaceResult{}, fail("intent", "INVALID_RESOLVED_INTERPRETATION", "$.resolvedInputs.interpretations", err.Error(), nil, err) @@ -275,10 +255,6 @@ func validateResolvedPopulation(document authoringv2.Document, resolved Resolved return nil } -// populationRouteDirectionFromSnapshot proves a compile-only direction -// enrichment against the current authorized capability, following the exact -// saved route to the step being enriched. It deliberately does not rewrite -// authoring intent or accept a direction that conflicts with the catalog. func populationRouteDirectionFromSnapshot(document authoringv2.Document, route []authoringv2.PopulationRouteStep, targetIndex int, snapshot capability.Snapshot) (string, bool) { if targetIndex < 0 || targetIndex >= len(route) { return "", false @@ -500,19 +476,12 @@ func applyResolvedInterpretations(workspace authoringv2.Workspace, inputs Resolv value := input.Definition.Contributor.Normalized() column.Contributor = &value } - // The effective clone is inline by construction. The authored result - // retains the PINNED reference; keeping it here would make validation - // mistake the resolved human definition for an inline override. column.Interpretation = nil } } return effective } -// ResolvedInputs is the explicit compile-only seam for resolved external -// values. B01 has no external values, so callers pass the zero value. The -// normalized workspace meaning is supplied separately to the digest function; -// it is not duplicated inside this seam. type ResolvedInputs struct { Populations []ResolvedPopulation `json:"populations,omitempty"` Interpretations []ResolvedInterpretation `json:"interpretations,omitempty"` @@ -538,8 +507,6 @@ func recipePopulation(input ResolvedPopulation) *recipe.PopulationConstraint { return &recipe.PopulationConstraint{SelectionRevisionID: input.SelectionRevisionID, MembershipDigest: input.MembershipDigest, MemberCount: input.MemberCount, ResourceType: input.ResourceType, Route: route} } -// ResolvedInputsDigest returns a stable digest for normalized authoring -// meaning plus the capability identity used to resolve it. func ResolvedInputsDigest(workspace authoringv2.Workspace, snapshot capability.Snapshot, resolvedInputs ResolvedInputs) (string, error) { resolvedInputs = resolvedInputs.Canonical() normalized, err := workspace.CanonicalJSON() @@ -593,8 +560,6 @@ func (r ResolvedInputs) Canonical() ResolvedInputs { return copy } -// Compile translates one semantic V2 document against the exact capability -// snapshot. It performs no discovery, profiling, or recipe execution. func Compile(ctx context.Context, project, explorerID string, document authoringv2.Document, snapshot capability.Snapshot) (Result, error) { return compileWithWorkspaceOutputs(ctx, project, explorerID, document, snapshot, nil) } diff --git a/internal/explorer/compilation/construction.go b/internal/explorer/compilation/construction.go index 0fb6b5fec..e7bb47d5b 100644 --- a/internal/explorer/compilation/construction.go +++ b/internal/explorer/compilation/construction.go @@ -11,9 +11,6 @@ import ( "github.com/calypr/loom/internal/explorer/capability" ) -// recipeConstruction maps durable construction intent into the recipe's -// resolved compiler contract. Source names come from the resolved public -// emissions, while IDs and labels come from authored projection slots. func recipeConstruction(authored *authoringv2.Construction, columns []authoringv2.Column, emitted []explorer.EmittedColumn) (*recipe.Construction, error) { if authored == nil { return nil, nil @@ -672,8 +669,6 @@ func recipeConstructionGroupFieldSource( } input := field.Input if input == nil { - // Flat saved bindings are accepted at the authoring boundary and - // normalized to the compiler's explicit input sum. input = &authoringv2.ConstructionGroupFieldSourceInput{Kind: "CURRENT_STAGE", CurrentStage: &authoringv2.ConstructionGroupFieldCurrentStage{ SourceRole: field.SourceRole, OccurrenceID: field.OccurrenceID, OwnerResourceType: field.OwnerResourceType, ResourceType: field.ResourceType, diff --git a/internal/explorer/compilation/interpretation.go b/internal/explorer/compilation/interpretation.go index 3c0dd7dae..a55719370 100644 --- a/internal/explorer/compilation/interpretation.go +++ b/internal/explorer/compilation/interpretation.go @@ -19,16 +19,11 @@ const ( InterpretationCandidateUnsupported InterpretationCandidateResolutionState = "UNSUPPORTED" ) -// InterpretationCandidateMatch is the semantic payload available only from -// a READY resolution. Candidate IDs are copied out of the exact snapshot. type InterpretationCandidateMatch struct { CapabilityCandidateIDs []string StructuralCandidate explorer.InterpretationStructuralCandidate } -// InterpretationCandidateResolution is a closed result. Implementations are -// private so callers can inspect a READY match or an unavailable reason, but -// cannot construct a state with the wrong payload. type InterpretationCandidateResolution interface { State() InterpretationCandidateResolutionState Reason() string @@ -83,9 +78,6 @@ func (unsupportedInterpretationCandidate) Match() (InterpretationCandidateMatch, } func (unsupportedInterpretationCandidate) interpretationCandidateResolution() {} -// ResolveWorkspaceInterpretations is the pure B06 resolver. The caller owns -// loading exact revisions; this function only interprets the retained -// capability snapshot and revision map, so it has no moving-head lookup. func ResolveWorkspaceInterpretations(project string, workspace authoringv2.Workspace, snapshot capability.Snapshot, revisions map[explorer.InterpretationRevisionID]explorer.InterpretationRevision) (ResolvedInputs, error) { project = projectid.Canonical(project) inputs := ResolvedInputs{} @@ -140,9 +132,6 @@ func ResolveWorkspaceInterpretations(project string, workspace authoringv2.Works return inputs.Canonical(), nil } -// ResolveInterpretationCandidate matches one configured source against an -// exact capability snapshot. Invalid routes are request-level errors; valid -// routes with unresolved sources return a closed per-column state. func ResolveInterpretationCandidate(document authoringv2.Document, column authoringv2.Column, snapshot capability.Snapshot) (InterpretationCandidateResolution, error) { occurrences, _, err := resolveSemanticRoute(document, snapshot) if err != nil { @@ -197,8 +186,6 @@ func ResolveInterpretationCandidate(document authoringv2.Document, column author }}, nil } -// structuralCandidate adapts the closed resolution for compiler validation. -// All source and concept matching remains in ResolveInterpretationCandidate. func structuralCandidate(document authoringv2.Document, column authoringv2.Column, snapshot capability.Snapshot) (explorer.InterpretationStructuralCandidate, error) { resolution, err := ResolveInterpretationCandidate(document, column, snapshot) if err != nil { diff --git a/internal/explorer/compilation/row_expansion.go b/internal/explorer/compilation/row_expansion.go index 90b843005..0da4c6f69 100644 --- a/internal/explorer/compilation/row_expansion.go +++ b/internal/explorer/compilation/row_expansion.go @@ -10,9 +10,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// expandedItemFieldSuffix returns the selector suffix when a direct field -// belongs to the selected expanded item. The persisted FieldRef remains the -// owner-relative path for provenance; only the executable selector is rebased. func expandedItemFieldSuffix(expanded *authoringv2.ExpandedRows, occurrenceID, fieldPath string) (string, bool) { if expanded == nil || expanded.OccurrenceID != occurrenceID { return "", false diff --git a/internal/explorer/compilation/semantic_compile.go b/internal/explorer/compilation/semantic_compile.go index 545a2a6c0..2d9775ec2 100644 --- a/internal/explorer/compilation/semantic_compile.go +++ b/internal/explorer/compilation/semantic_compile.go @@ -241,8 +241,6 @@ func compileSemanticDocument(ctx context.Context, project, explorerID string, do choiceArm = choiceArmForPath(ownerRecords.Binding.ValuePath) case authoringv2.SourceExtensionByURL: if column.Source.Lookup == nil || column.Source.Lookup.Extension == nil { - // Legacy extension lookups remain readable for immutable recipes; - // writable authoring commands reject this shape before compilation. dynamic, dynamicErr := semanticFixedLookup(column, occurrence.graph.ResourceType, alias, leaf, logicalType) if dynamicErr != nil { return Result{}, fail("lower", "INVALID_TYPED_SOURCE", fmt.Sprintf("$.columns[%d].source", index), dynamicErr.Error(), nil, dynamicErr) @@ -515,11 +513,6 @@ func compileSemanticDocument(ctx context.Context, project, explorerID string, do return Result{}, fail("capability", "STALE_CONSTRUCTION_SOURCE_FIELD", fmt.Sprintf("$.construction.sourceProjections[%d]", index), "source projection no longer matches an authorized field on the root resource", map[string]any{"fieldPath": path}, nil) } if expanded != nil { - // Signed EXPAND inputs are backed by the direct-root repeated - // array itself: many cardinality, the ARRAY projection the - // builder mints for repeated fields, and only self-describing - // boundaries. The scalar VALUE backing rule below rejects this - // shape by design, so the lanes keep distinct checks. if !capability.IsRepeatedCardinality(candidate.Cardinality) || !containsProjectionMode(candidate.ProjectionModes, capability.ProjectionArray) || !semanticSelfRepeated(candidate, path) { @@ -539,11 +532,6 @@ func compileSemanticDocument(ctx context.Context, project, explorerID string, do } repeated := capability.IsRepeatedCardinality(candidate.Cardinality) if expanded != nil { - // Signed EXPAND input: the owning step's server-derived payload - // authorizes a direct-root many-cardinality array. The schema - // facts, not the payload, decide the compiled cardinality: the - // semantic selector resolver derives Many from the generated - // schema when this root Field projects the array path. if !repeated || facts.Shape != fhirschema.RowPathArray || facts.Cardinality != fhirschema.RowCardinalityMany || expanded.Path != path || expanded.Cardinality != authoringv2.ConstructionExpandedFieldCardinalityMany || expanded.LogicalType != projection.LogicalType || expanded.FHIRType != projection.FHIRType { @@ -561,10 +549,6 @@ func compileSemanticDocument(ctx context.Context, project, explorerID string, do Expr: recipe.Expression{Select: semanticAlias(authoringv2.RootOccurrenceID) + "." + path}, } if expanded != nil { - // The item column carries one array element per row; the - // compiled column must stay the many-valued array itself. An - // auto/FIRST value mode would collapse it to optional_one and - // the physical EXPAND gate would refuse its own input column. laneField.ValueMode = recipe.ValueModeAll } nodes[authoringv2.RootOccurrenceID].fields = append(nodes[authoringv2.RootOccurrenceID].fields, laneField) @@ -818,10 +802,6 @@ func semanticAlias(occurrenceID string) string { if occurrenceID == authoringv2.RootOccurrenceID { return "root" } - // Semantic Builder occurrence IDs are globally unique output namespaces. - // Keep the complete identifier so ALIAS traversal naming reproduces the - // authored public-column prefix even when an older client encoded route - // ancestry into the occurrence ID itself. return safeName(occurrenceID) } @@ -1069,9 +1049,6 @@ func semanticCodedValuePivot(column authoringv2.Column, leaf string) (recipe.Piv columns := []string{lookup.Key.Code} columnAliases := map[string]string{lookup.Key.Code: leaf} if lookup.Binding.SelectionScope == fhirschema.CorrelatedSelectionScopeSystemVersion { - // SYSTEM_VERSION selects a typed category family with no exact code. Use - // the authored output leaf as its one pivot alias so lowering never - // creates an empty output-column name or a synthetic code predicate. pivotName = "correlated_category_" + shortHash(column.Column+"\x00"+lookup.Key.System+"\x00"+lookup.Key.Version+"\x00"+lookup.Binding.KeyPath) columns = []string{leaf} columnAliases = map[string]string{leaf: leaf} @@ -1102,14 +1079,6 @@ func coalesceString(paths ...string) recipe.Expression { return recipe.Expression{Call: "coalesce_string", Args: args} } -// constructionExpandedFieldOwner returns the server-derived expanded-field -// facts that authorize this projection as a signed EXPAND input, or nil when -// the projection is an ordinary scalar source. Authorization requires ALL of: -// the projection names an existing EXPAND step's input column, that step -// carries an ExpandedField payload for exactly this path, the payload is -// shape-valid, and the projection is owned by the step. The caller still -// re-proves the array facts against the generated schema; the payload never -// supplies cardinality directly. func constructionExpandedFieldOwner(construction *authoringv2.Construction, projection authoringv2.ConstructionSourceProjection, path string) *authoringv2.ConstructionExpandedFieldSource { if projection.OwnerStepID == "" { return nil diff --git a/internal/explorer/compilation/semantic_compile_expand_source_test.go b/internal/explorer/compilation/semantic_compile_expand_source_test.go index 8060c1499..9d4ff536b 100644 --- a/internal/explorer/compilation/semantic_compile_expand_source_test.go +++ b/internal/explorer/compilation/semantic_compile_expand_source_test.go @@ -1,32 +1,5 @@ package compilation -// ROW-SOURCE-08 EXPAND compile-side contract tests for -// internal/explorer/compilation/semantic_compile.go:506-549: a signed -// EXPAND-owned SourceProjection must compile as a direct-root many-cardinality -// non-reference array, and every stale/forged variant must be refused by name. -// -// BUG-002 — expected failures below. The authoring validator -// (authoringv2/construction_validation.go validatePivotSourceProjection, -// default arm ~L376) has no ConstructionOperationExpand owner case, so an -// EXPAND-owned source projection — the exact state expand_source.go signs -// (L403-415, 468-471) and this file's verifier consumes — is rejected by -// Compile's document validation as -// -// INVALID_DOCUMENT at $: construction: source projection must be consumed by -// its owner PIVOT step -// -// before any compile verdict is reached. The tests therefore state the -// contract each input must produce once the owner arm exists, and currently -// fail with the INVALID_DOCUMENT wall. Do not "fix" them by deleting the -// EXPAND ownership: the lane and the verifier are the shipped contract -// (semantic_compile.go:521-536 is parent-owned code reached only through this -// path). See also the lane's own comment (lifecycle/expand_source.go:472-478) -// acknowledging the compiled Many-cardinality mapping of the root array column -// as an unfinished parent follow-up (BUG-004): Test...CompilesWithManyCardinality -// pins that the finished pipeline must surface the root array column as -// compiled Cardinality "many", which is what the physical EXPAND gate -// (lower/recipe_construction_reshape.go:469-478) consumes. - import ( "context" "errors" @@ -41,10 +14,6 @@ import ( "github.com/calypr/loom/internal/explorer/capability" ) -// expandCompileSnapshot is a Procedure snapshot whose root carries the real -// generated primitive root array instantiatesUri[] (canonical, ARRAY/MANY, -// FHIRType string, non-reference) plus the scalar identifier.use used by the -// payload-forgery case. func expandCompileSnapshot(uriLogicalType string) capability.Snapshot { base := fixtureSnapshot() return capability.NewSnapshot(base.Identity, base.Policy, capability.StatusReady, true, false, @@ -57,11 +26,6 @@ func expandCompileSnapshot(uriLogicalType string) capability.Snapshot { }, nil) } -// expandCompileDocument assembles a document already carrying the durable -// server-signed state: the EXPAND step with its server-derived payload and the -// expand-owned SourceProjection. A legal client candidate normally never -// authors this state; it is what the lane persists — which makes the -// BUG-002 rejection decisive. func expandCompileDocument(projection authoringv2.ConstructionSourceProjection, payload *authoringv2.ConstructionExpandedFieldSource, steps []authoringv2.ConstructionStep) authoringv2.Document { return authoringv2.Document{ Rows: authoringv2.RecordsRowDefinition(), Kind: authoringv2.Kind, @@ -100,10 +64,6 @@ func expandCompileStep(payload *authoringv2.ConstructionExpandedFieldSource) aut EmptyPolicy: authoringv2.ConstructionExpandEmptyExclude, ExpandedField: payload, }}, - // The authoring contract requires the full step schema: every input - // column except the expanded array (here the root identifier) plus the - // item column. A one-output fixture would die at document validation - // before reaching the semantic verifier under test. Outputs: []authoringv2.StageColumn{ {ID: "procedure-id", Name: "procedure_id", Label: "Procedure ID", Type: "string"}, {ID: "uri_item", Name: "uri_item", Label: "Uri item", Type: "string"}, @@ -140,10 +100,6 @@ func expandCompileError(err error) string { return "" } -// BUG-002 — expected failure: the durable signed expand state must compile and -// lower, the root stage must present the array column, the compiled field must -// read the array path, and the root column the physical EXPAND gate consumes -// must carry compiled Cardinality "many" (BUG-004's pinned guarantee). func TestCompileConstructionExpandSignedArrayProjectionCompilesWithManyCardinality(t *testing.T) { lowered := expandCompileLower(t, expandCompileDocument(expandCompileSignedProjection(), expandCompileSignedPayload(), @@ -178,12 +134,6 @@ func TestCompileConstructionExpandSignedArrayProjectionCompilesWithManyCardinali } } -// A source projection naming no owning step is refused fail-closed at the -// earliest boundary: validateLegacyGroupSourceProjection only legalizes -// unowned projections behind the exact direct-source GROUP shape, so the -// expand-styled unowned state never reaches the semantic verifier. The -// owner-specific semantic refusal (a projection owned by another legal step) -// stays covered at its own layer by TestCompileConstructionExpandWrongOwner.... func TestCompileConstructionExpandProjectionWithoutOwnerIsRefused(t *testing.T) { projection := expandCompileSignedProjection() projection.OwnerStepID = "" @@ -198,12 +148,6 @@ func TestCompileConstructionExpandProjectionWithoutOwnerIsRefused(t *testing.T) } } -// Two forged claims on an EXPAND-owned projection must be refused at the -// layer that first proves the lie. (1) A real scalar root field -// (Procedure.status) claimed as the signed many array: its snapshot backing -// is an honest scalar VALUE candidate, so the signed-array candidate gate — -// which requires the repeated array backing the EXPAND step claims — refuses -// it by name before generated-schema lookup. func TestCompileConstructionExpandPayloadClaimingScalarAsManyIsRefused(t *testing.T) { projection := expandCompileSignedProjection() projection.FieldPath = "status" @@ -224,10 +168,6 @@ func TestCompileConstructionExpandPayloadClaimingScalarAsManyIsRefused(t *testin } } -// (2) The same claim backed by a forged SNAPSHOT (status declared many with -// the ARRAY projection): the candidate gate is satisfied, and the generated -// schema — the authoritative fact source — proves status is not an ARRAY of -// MANY, which is the expanded-field verifier's named refusal. func TestCompileConstructionExpandSnapshotForgedArrayBackingIsRefused(t *testing.T) { projection := expandCompileSignedProjection() projection.FieldPath = "status" @@ -262,8 +202,6 @@ func expandCompileSnapshotWithStatus(uriLogicalType string) capability.Snapshot return base } -// BUG-002 — expected failure: array items the compiler cannot treat as scalar -// (object kind) must be refused by name (semantic_compile.go:532-535). func TestCompileConstructionExpandObjectKindItemsAreRefused(t *testing.T) { projection := expandCompileSignedProjection() projection.LogicalType = "object" @@ -277,15 +215,6 @@ func TestCompileConstructionExpandObjectKindItemsAreRefused(t *testing.T) { } } -// Control proving the refusals above are EXPAND-owner-specific and not a -// broken harness: a wrong-owner claim — an array projection "owned" by the -// DERIVE step that consumes it as a COLUMN operand, with a separate expand -// step in the construction — must be handled by the SCALAR branch, because -// constructionExpandedFieldOwner refuses to credit a payload whose step is -// not the projection's owner. The scalar lane's VALUE-backing candidate gate -// is therefore the first honest defense (an array candidate is not a scalar -// VALUE field); reaching it at all proves the payload was NOT credited, since -// a credited owner would take the signed-array branch instead. func TestCompileConstructionExpandStyledWrongOwnerProjectionRefused(t *testing.T) { projection := authoringv2.ConstructionSourceProjection{ ColumnID: "uri_column", OwnerStepID: "derive_uri_copy", OccurrenceID: authoringv2.RootOccurrenceID, @@ -294,7 +223,6 @@ func TestCompileConstructionExpandStyledWrongOwnerProjectionRefused(t *testing.T derive := authoringv2.ConstructionStep{ ID: "derive_uri_copy", Inputs: []authoringv2.ConstructionInputRef{{Kind: authoringv2.ConstructionInputSourceProjection}}, - // (the derive carries the projection column through its stage outputs) Operation: authoringv2.ConstructionOperation{Kind: authoringv2.ConstructionOperationDerive, Derive: &authoringv2.ConstructionDerive{ ConstructionID: "derive_uri_copy", OutputColumnID: "uri_copy", Operation: authoringv2.ConstructionDerivedAdd, MissingInputPolicy: authoringv2.ConstructionMissingInputPropagateNull, diff --git a/internal/explorer/compilation/source_row.go b/internal/explorer/compilation/source_row.go index 6c1f59ba4..7f488b5ac 100644 --- a/internal/explorer/compilation/source_row.go +++ b/internal/explorer/compilation/source_row.go @@ -9,19 +9,11 @@ import ( "github.com/calypr/loom/internal/explorer/authoringv2" ) -// SourceRowDescriptor identifies the physical output column that preserves a -// root resource identity. It is intentionally transport-neutral so lifecycle -// and publication can build their source-row descriptor without importing the -// authoring compiler or guessing from labels and internal row IDs. type SourceRowDescriptor struct { ResourceType string PhysicalColumn string } -// ResolveAddressableSourceRow proves that one validated workspace output can -// map rows back to source resources. The proof is deliberately narrow: -// resource-grain output, root occurrence, direct id field, VALUE projection, -// and one emitted physical column with the same source provenance. func ResolveAddressableSourceRow(workspace authoringv2.Workspace, contract explorer.PublicOutputContract, emitted []explorer.EmittedColumn) (SourceRowDescriptor, error) { if err := workspace.Validate(); err != nil { return SourceRowDescriptor{}, fmt.Errorf("authoring workspace: %w", err) diff --git a/internal/explorer/compilation_receipt.go b/internal/explorer/compilation_receipt.go index 3229a719c..4f3e53aaa 100644 --- a/internal/explorer/compilation_receipt.go +++ b/internal/explorer/compilation_receipt.go @@ -1,10 +1,5 @@ package explorer -// This file contains the immutable boundary between Explorer authoring and -// execution. A receipt is a durable, server-owned description of one exact -// compilation. It deliberately contains a resolved semantic recipe, not -// physical plans, rendered AQL, bind variables, or storage identifiers. - import ( "crypto/sha256" "encoding/hex" @@ -20,13 +15,9 @@ import ( ) const ( - // CompilationReceiptFormatVersion changes when the persisted receipt shape - // or its execution invariants change incompatibly. - CompilationReceiptFormatVersion = 6 - previousCompilationReceiptFormatVersion = 5 - legacyCompilationReceiptFormatVersion = 3 - // CompilationReceiptCompilerContractVersion changes when compilation - // semantics change in a way that can alter a resolved receipt. + CompilationReceiptFormatVersion = 6 + previousCompilationReceiptFormatVersion = 5 + legacyCompilationReceiptFormatVersion = 3 CompilationReceiptCompilerContractVersion = "loom.explorer.compiler/v18" legacyCompilationReceiptV17CompilerContractVersion = "loom.explorer.compiler/v17" legacyCompilationReceiptV16CompilerContractVersion = "loom.explorer.compiler/v16" @@ -37,30 +28,18 @@ const ( legacyCompilationReceiptOlderContractVersion = "loom.explorer.compiler/v11" legacyCompilationReceiptV10ContractVersion = "loom.explorer.compiler/v10" - // Short aliases make the current contract convenient for repositories and - // callers that do not need to distinguish the receipt prefix. CurrentReceiptFormatVersion = CompilationReceiptFormatVersion CurrentCompilerContractVersion = CompilationReceiptCompilerContractVersion CurrentCompilationContractVersion = CompilationReceiptCompilerContractVersion ) -// ErrReceiptRecompileRequired is returned for an old receipt that has no -// resolved recipe artifact. Such a receipt cannot be safely upgraded by an -// execution request because doing so would reinterpret authoring intent. var ErrReceiptRecompileRequired = errors.New("RECEIPT_RECOMPILE_REQUIRED") -// ReceiptContractSupportedForExecution distinguishes read compatibility from -// execution compatibility. The immediately previous v4/v14 artifact remains -// executable because it contains a complete frozen recipe; older historical -// contracts remain readable for migration/inspection but must be recompiled -// before native execution. func ReceiptContractSupportedForExecution(r CompilationReceipt) bool { return (r.ReceiptFormatVersion == CompilationReceiptFormatVersion && r.CompilerContractVersion == CompilationReceiptCompilerContractVersion) || (r.ReceiptFormatVersion == previousCompilationReceiptFormatVersion && r.CompilerContractVersion == previousCompilationReceiptCompilerContractVersion) } -// CompilationArtifactDigest returns the content identity used for canonical -// JSON artifacts embedded in a receipt, such as the public output contract. func CompilationArtifactDigest(raw json.RawMessage) (string, error) { canonical, err := canonicalRaw(raw) if err != nil { @@ -73,57 +52,44 @@ func CompilationArtifactDigest(raw json.RawMessage) (string, error) { return "sha256:" + hex.EncodeToString(sum[:]), nil } -// CompilationReceipt is immutable once persisted and content-addressed by -// ReceiptID. NormalizedBundle is authoring intent retained for export and -// compatibility; Bundle is the resolved recipe used by execution. Neither -// field contains a physical IR or rendered query. type CompilationReceipt struct { - ConstructionProposal *ConstructionProposalBinding `json:"constructionProposal,omitempty"` - PopulationMemberRemovalProposal *PopulationMemberRemovalProposalBinding `json:"populationMemberRemovalProposal,omitempty"` - ID string `json:"id"` - ReceiptFormatVersion int `json:"receiptFormatVersion"` - CompilerContractVersion string `json:"compilerContractVersion"` - Project string `json:"project"` - ExplorerID string `json:"explorerId"` - IntentDigest string `json:"intentDigest"` - ResolvedInputsDigest string `json:"resolvedInputsDigest,omitempty"` - ResolvedInterpretations []ResolvedInterpretation `json:"resolvedInterpretations,omitempty"` - SnapshotToken string `json:"snapshotToken"` - AuthorizationScopeDigest string `json:"authorizationScopeDigest,omitempty"` - CapabilitySchemaDigest string `json:"capabilitySchemaDigest,omitempty"` - ShapeDigest string `json:"shapeDigest,omitempty"` - SourceGeneration string `json:"sourceGeneration"` - CompilationKey string `json:"compilationKey,omitempty"` - RecipeDigest string `json:"recipeDigest"` - ResolvedRecipeDigest string `json:"resolvedRecipeDigest,omitempty"` - ResolvedSchemaDigest string `json:"resolvedSchemaDigest,omitempty"` - OutputContractDigest string `json:"outputContractDigest,omitempty"` - NormalizedBundle json.RawMessage `json:"normalizedBundle"` - Bundle recipe.Bundle `json:"compiledRecipe"` - CompiledConfig json.RawMessage `json:"compiledConfig,omitempty"` - PublicOutputContract json.RawMessage `json:"publicOutputContract,omitempty"` - IdentityMappings []IdentityMapping `json:"identityMappings"` - EmittedColumns []EmittedColumn `json:"emittedColumns"` - OutputFingerprints map[string]string `json:"outputFingerprints,omitempty"` - ConstructionStages map[string][]ReceiptConstructionStage `json:"constructionStages,omitempty"` - // OutputColumnProvenance is the durable publication behavior for every - // compiler output column. Recipe Discovered flags are compiler-local and - // intentionally do not cross the authoring recipe JSON boundary. - OutputColumnProvenance map[string]map[string]string `json:"outputColumnProvenance,omitempty"` - // CompiledOutputSchemas preserves the lowerer's exact finalized schema for - // every output in this workspace receipt. It is used only as server-owned - // source metadata when another draft output consumes a WORKSPACE_OUTPUT. - CompiledOutputSchemas map[string][]ReceiptCompiledOutputColumn `json:"compiledOutputSchemas,omitempty"` - Warnings []CompilationWarning `json:"warnings,omitempty"` - RowDefinitionProposal *RowDefinitionProposalBinding `json:"rowDefinitionProposal,omitempty"` - TableShapeProposal *TableShapeProposalBinding `json:"tableShapeProposal,omitempty"` - RequestID string `json:"requestId,omitempty"` - CreatedAt time.Time `json:"createdAt"` + ConstructionProposal *ConstructionProposalBinding `json:"constructionProposal,omitempty"` + PopulationMemberRemovalProposal *PopulationMemberRemovalProposalBinding `json:"populationMemberRemovalProposal,omitempty"` + ID string `json:"id"` + ReceiptFormatVersion int `json:"receiptFormatVersion"` + CompilerContractVersion string `json:"compilerContractVersion"` + Project string `json:"project"` + ExplorerID string `json:"explorerId"` + IntentDigest string `json:"intentDigest"` + ResolvedInputsDigest string `json:"resolvedInputsDigest,omitempty"` + ResolvedInterpretations []ResolvedInterpretation `json:"resolvedInterpretations,omitempty"` + SnapshotToken string `json:"snapshotToken"` + AuthorizationScopeDigest string `json:"authorizationScopeDigest,omitempty"` + CapabilitySchemaDigest string `json:"capabilitySchemaDigest,omitempty"` + ShapeDigest string `json:"shapeDigest,omitempty"` + SourceGeneration string `json:"sourceGeneration"` + CompilationKey string `json:"compilationKey,omitempty"` + RecipeDigest string `json:"recipeDigest"` + ResolvedRecipeDigest string `json:"resolvedRecipeDigest,omitempty"` + ResolvedSchemaDigest string `json:"resolvedSchemaDigest,omitempty"` + OutputContractDigest string `json:"outputContractDigest,omitempty"` + NormalizedBundle json.RawMessage `json:"normalizedBundle"` + Bundle recipe.Bundle `json:"compiledRecipe"` + CompiledConfig json.RawMessage `json:"compiledConfig,omitempty"` + PublicOutputContract json.RawMessage `json:"publicOutputContract,omitempty"` + IdentityMappings []IdentityMapping `json:"identityMappings"` + EmittedColumns []EmittedColumn `json:"emittedColumns"` + OutputFingerprints map[string]string `json:"outputFingerprints,omitempty"` + ConstructionStages map[string][]ReceiptConstructionStage `json:"constructionStages,omitempty"` + OutputColumnProvenance map[string]map[string]string `json:"outputColumnProvenance,omitempty"` + CompiledOutputSchemas map[string][]ReceiptCompiledOutputColumn `json:"compiledOutputSchemas,omitempty"` + Warnings []CompilationWarning `json:"warnings,omitempty"` + RowDefinitionProposal *RowDefinitionProposalBinding `json:"rowDefinitionProposal,omitempty"` + TableShapeProposal *TableShapeProposalBinding `json:"tableShapeProposal,omitempty"` + RequestID string `json:"requestId,omitempty"` + CreatedAt time.Time `json:"createdAt"` } -// ReceiptCompiledOutputColumn is the minimal stable schema projection needed -// to describe a sibling output. IDs and logical facts come from lower's final -// OutputSchema; no physical storage names are exposed to authoring clients. type ReceiptCompiledOutputColumn struct { ID string `json:"id,omitempty"` Name string `json:"name"` @@ -135,9 +101,6 @@ type ReceiptCompiledOutputColumn struct { Identity bool `json:"identity,omitempty"` } -// RowDefinitionProposalBinding freezes the exact draft and output from which a -// candidate row-definition workspace was compiled. It is optional because -// ordinary compilation receipts are not row-definition proposals. type RowDefinitionProposalBinding struct { DraftVersion int64 `json:"draftVersion"` DraftDigest string `json:"draftDigest"` @@ -170,9 +133,6 @@ func (b RowDefinitionProposalBinding) Validate(intentDigest, snapshotToken strin return nil } -// TableShapeProposalBinding freezes the exact draft and output from which a -// candidate table-shape workspace was compiled. It is optional because -// ordinary compilation receipts are not table-shape proposals. type TableShapeProposalBinding struct { DraftVersion int64 `json:"draftVersion"` DraftDigest string `json:"draftDigest"` @@ -205,9 +165,6 @@ func (b TableShapeProposalBinding) Validate(intentDigest, snapshotToken string) return nil } -// PopulationMemberRemovalProposalBinding freezes one exact immutable source -// selection change and the compiled workspace that previews it. It is absent -// from ordinary compilation receipts. type PopulationMemberRemovalProposalBinding struct { DraftVersion int64 `json:"draftVersion"` DraftDigest string `json:"draftDigest"` @@ -262,9 +219,6 @@ func (b PopulationMemberRemovalProposalBinding) Validate(intentDigest, snapshotT return nil } -// ConstructionProposalBinding freezes the saved base and exact typed candidate -// for an operation proposal. CandidateWorkspaceDigest covers all stage inputs, -// including immutable table revisions. type ConstructionProposalBinding struct { DraftVersion int64 `json:"draftVersion"` DraftDigest string `json:"draftDigest"` @@ -322,8 +276,6 @@ func (b ConstructionProposalBinding) Validate(intentDigest, snapshotToken string return nil } -// ReceiptConstructionStage preserves the exact inferred schema and supported -// operation set for a stable stage in a compiled output. type ReceiptConstructionStage struct { ID string `json:"id"` InputStageID string `json:"inputStageId"` @@ -338,9 +290,6 @@ type ReceiptConstructionStage struct { GroupFieldSourceContext *ReceiptConstructionGroupFieldSourceContext `json:"groupFieldSourceContext,omitempty"` } -// ReceiptConstructionGroupFieldSourceContext is copied from the lowerer's -// compiled stage descriptor so lifecycle can reauthorize Group sources -// against the exact row expansion shape at proposal and apply time. type ReceiptConstructionGroupFieldSourceContext struct { SourceStageID string `json:"sourceStageId"` SourceShapeID string `json:"sourceShapeId"` @@ -383,9 +332,6 @@ type ReceiptConstructionGroupFieldOwner struct { StepIndex int `json:"stepIndex"` } -// ReceiptConstructionCodedGroupChoice is attached only to a current -// construction-capabilities response. Its choice ID is snapshot-bound; the -// persisted operation stores the exact source path facts separately. type ReceiptConstructionCodedGroupChoice struct { ChoiceID string `json:"choiceId"` OccurrenceID string `json:"occurrenceId"` @@ -424,11 +370,10 @@ type ReceiptConstructionActiveRelatedRecord struct { } type ReceiptConstructionStageColumn struct { - ID string `json:"id"` - Name string `json:"name"` - Label string `json:"label"` - Type string `json:"type,omitempty"` - // Cardinality is empty only on receipts created before stage cardinality was recorded. + ID string `json:"id"` + Name string `json:"name"` + Label string `json:"label"` + Type string `json:"type,omitempty"` Cardinality expression.Cardinality `json:"cardinality,omitempty"` } @@ -753,7 +698,7 @@ func validateReceiptCompiledOutputSchemas( bundle recipe.Bundle, ) error { if len(schemas) == 0 { - return nil // Older receipts remain readable; new compiles always populate this field. + return nil } if len(schemas) != len(bundle.Outputs) { return fmt.Errorf("receipt compiled output schema set changed") @@ -798,9 +743,6 @@ type IdentityMapping struct { EmissionIDs []string `json:"emissionIds"` } -// CompilationWarning is the deterministic, request-independent diagnostic -// subset that can be frozen in a receipt. Request IDs and timestamps must not -// be included here because they would make equivalent receipts differ. type CompilationWarning struct { Severity string `json:"severity,omitempty"` Code string `json:"code"` @@ -810,9 +752,6 @@ type CompilationWarning struct { Details map[string]any `json:"details,omitempty"` } -// CompilationKey returns the idempotency identity for one authoring request. -// It covers semantic inputs and compiler contracts, but not resolved output; -// this permits a repository lookup before doing the expensive compilation. func CompilationKey(r CompilationReceipt) (string, error) { identity := struct { ReceiptFormatVersion int `json:"receiptFormatVersion"` @@ -866,8 +805,6 @@ func CompilationKey(r CompilationReceipt) (string, error) { return digestIdentity("compile_", identity) } -// ReceiptID returns the content identity for the complete immutable artifact. -// ID, RequestID, and CreatedAt are intentionally excluded. func ReceiptID(r CompilationReceipt) (string, error) { key, err := CompilationKey(r) if err != nil { @@ -935,7 +872,6 @@ func canonicalRaw(raw json.RawMessage) ([]byte, error) { return canonicalJSONBytes(raw) } -// Validate checks that a receipt is a supported, executable artifact. func (r CompilationReceipt) Validate() error { if r.RowDefinitionProposal != nil { if err := r.RowDefinitionProposal.Validate(r.IntentDigest, r.SnapshotToken); err != nil { @@ -1132,7 +1068,6 @@ func validateResolvedInterpretations(project string, values []ResolvedInterpreta return nil } -// ValidateID verifies the stored ID against the receipt's content identity. func (r CompilationReceipt) ValidateID() error { if strings.TrimSpace(r.ID) == "" { return fmt.Errorf("receipt id is required") diff --git a/internal/explorer/explicit_group_repository.go b/internal/explorer/explicit_group_repository.go index 36ae8b3c9..0238fd278 100644 --- a/internal/explorer/explicit_group_repository.go +++ b/internal/explorer/explicit_group_repository.go @@ -10,9 +10,6 @@ type ExplicitGroupMembershipCursor struct { ResourceID string } -// ExplicitGroupRepository keeps explicit group revisions separate from the -// general Explorer lifecycle store. Definition rows preserve empty groups; -// membership rows preserve overlaps between groups. type ExplicitGroupRepository interface { BeginExplicitGroupRevision(context.Context, ExplicitGroupRevision, SelectionRevision, string) (*ExplicitGroupRevision, error) PutExplicitGroupDefinitions(context.Context, ExplicitGroupRevisionID, string, []ExplicitGroupDefinition) error diff --git a/internal/explorer/explicit_groups.go b/internal/explorer/explicit_groups.go index 16b7b8c08..bc355674d 100644 --- a/internal/explorer/explicit_groups.go +++ b/internal/explorer/explicit_groups.go @@ -32,9 +32,6 @@ const ( ExplicitGroupRevisionComplete ExplicitGroupRevisionState = "COMPLETE" ) -// ExplicitGroupRevisionIDFor makes the idempotency key the primary identity -// reservation. Concurrent retries for one project and key contend on the -// same Arango document key. func ExplicitGroupRevisionIDFor(project, idempotencyKey string) ExplicitGroupRevisionID { project = strings.TrimSpace(project) idempotencyKey = strings.TrimSpace(idempotencyKey) @@ -48,8 +45,6 @@ func ExplicitGroupRevisionIDFor(project, idempotencyKey string) ExplicitGroupRev return ExplicitGroupRevisionID("grouprev_" + hex.EncodeToString(hash.Sum(nil))) } -// ExplicitGroupDefinition preserves each declared group independently of its -// members, so a group with no members still has a durable identity and order. type ExplicitGroupDefinition struct { ID ExplicitGroupID `json:"id"` Label string `json:"label"` @@ -62,8 +57,6 @@ func (g ExplicitGroupDefinition) Canonical() ExplicitGroupDefinition { return g } -// ExplicitGroupMembership binds one group to one exact resource identity. -// A resource may appear in more than one relation under different group IDs. type ExplicitGroupMembership struct { GroupID ExplicitGroupID `json:"groupId"` Ref ResourceRef `json:"ref"` @@ -135,9 +128,6 @@ func (r ExplicitGroupRevision) Validate() error { return nil } -// ValidateSource prevents a group definition created against one source -// selection from being attached to a different project, generation, type, or -// authorization scope later. func (r ExplicitGroupRevision) ValidateSource(selection SelectionRevision) error { r = r.Canonical() if err := r.Validate(); err != nil { @@ -164,8 +154,6 @@ func (r ExplicitGroupRevision) ValidateSourceSelection(selection SelectionRevisi return nil } -// CanonicalExplicitGroupDefinitions returns presentation order and rejects -// ambiguous IDs or ordinals before a revision is staged. func CanonicalExplicitGroupDefinitions(groups []ExplicitGroupDefinition) ([]ExplicitGroupDefinition, error) { canonical := append([]ExplicitGroupDefinition(nil), groups...) ids := make(map[ExplicitGroupID]struct{}, len(canonical)) @@ -267,8 +255,6 @@ func StartExplicitGroupMembershipDigest(w interface{ Write([]byte) (int, error) writeDigestField(w, "loom-explicit-group-membership-v1") } -// WriteExplicitGroupMembershipFrame writes the stable relation identity used -// by both in-memory canonicalization and paged persistence finalization. func WriteExplicitGroupMembershipFrame(w interface{ Write([]byte) (int, error) }, membership ExplicitGroupMembership) { membership = membership.Canonical() writeDigestField(w, string(membership.GroupID)) diff --git a/internal/explorer/identity.go b/internal/explorer/identity.go index 38ea34d21..3827b6c2a 100644 --- a/internal/explorer/identity.go +++ b/internal/explorer/identity.go @@ -4,8 +4,6 @@ import ( "strings" ) -// StableExplorerID turns a user-facing name into the durable URL identity -// used by the V2 lifecycle. Collisions remain store-level conflicts. func StableExplorerID(name string) string { var b strings.Builder lastDash := false diff --git a/internal/explorer/interpretation.go b/internal/explorer/interpretation.go index 893aebc85..8e943bb77 100644 --- a/internal/explorer/interpretation.go +++ b/internal/explorer/interpretation.go @@ -15,9 +15,6 @@ import ( "github.com/calypr/loom/internal/projectid" ) -// Interpretation IDs are deliberately opaque to callers. They are still -// constrained to Arango-safe, URL-safe values so a malformed ID cannot become -// a collection key or an ambiguous share URL. type InterpretationLibraryID string type InterpretationRevisionID string type InterpretationRuleID string @@ -38,8 +35,6 @@ var ( ErrInterpretationIdentityMismatch = errors.New("interpretation revision identity mismatch") ) -// InterpretationLibrary is the mutable authoring head for one project and -// library. Revisions never mutate; only this pointer advances. type InterpretationLibrary struct { ID InterpretationLibraryID `json:"id"` Project string `json:"project"` @@ -72,8 +67,6 @@ type InterpretationApplicability struct { SchemaDigests []string `json:"schemaDigests,omitempty"` } -// InterpretationStructuralMatch is a closed conjunction over observed -// catalog identity. Empty dimensions are wildcards. type InterpretationStructuralMatch struct { ResourceType string `json:"resourceType,omitempty"` SourceProfile string `json:"sourceProfile,omitempty"` @@ -86,8 +79,6 @@ type InterpretationStructuralMatch struct { Cardinality string `json:"cardinality,omitempty"` } -// The candidate is intentionally a separate type: a rule can never smuggle -// an arbitrary map or query expression into the matcher. type InterpretationStructuralCandidate struct { ResourceType string SourceProfile string @@ -113,10 +104,6 @@ type InterpretationFeatureDefinition struct { Contributor *authoringv2.ContributorPredicate `json:"contributor,omitempty"` } -// ResolvedInterpretation is the immutable, occurrence-scoped result of -// resolving one pinned feature against one exact capability snapshot. It is -// kept in the explorer domain so receipts can freeze it without creating an -// import cycle with the compilation package. type ResolvedInterpretation struct { OutputID string `json:"outputId"` Column string `json:"column"` @@ -173,9 +160,6 @@ func canonicalInterpretationProject(raw string) (string, error) { return project, nil } -// CanonicalInterpretationProject is the persistence-boundary project parser. -// It accepts the repository's legacy project spelling but always returns the -// canonical slash-separated identity. func CanonicalInterpretationProject(raw string) (string, error) { return canonicalInterpretationProject(raw) } @@ -407,8 +391,6 @@ func (r InterpretationRevision) canonicalized() (InterpretationRevision, error) func canonicalInterpretationDefinition(def InterpretationFeatureDefinition, resourceType string) (InterpretationFeatureDefinition, error) { def.Source = def.Source.Normalized() - // Document.Validate is the existing closed authoring validator. A - // synthetic document keeps B06 from inventing a second source language. document := authoringv2.Document{ Kind: authoringv2.Kind, Output: authoringv2.Output{ID: "interpretation", Title: "Interpretation"}, @@ -463,9 +445,6 @@ func validateStaticAmbiguity(rules []InterpretationRule) error { return nil } -// CanonicalContent returns executable interpretation content. Project, -// lineage, authorship, explanation, and timestamps are omitted; stable rule -// IDs remain because they are part of selected-rule provenance. func (r InterpretationRevision) CanonicalContent() ([]byte, error) { n, err := r.canonicalized() if err != nil { @@ -535,9 +514,6 @@ func (r InterpretationRevision) ComputeRevisionID() (InterpretationRevisionID, e return InterpretationRevisionID("interpretation_" + hex.EncodeToString(sum[:])), nil } -// PrepareInterpretationRevision canonicalizes an incoming revision and -// recomputes both immutable identities. Persistence adapters must call this -// instead of trusting client-supplied IDs or digests. func PrepareInterpretationRevision(r InterpretationRevision) (InterpretationRevision, error) { n, err := r.canonicalized() if err != nil { diff --git a/internal/explorer/lifecycle/artifact.go b/internal/explorer/lifecycle/artifact.go index 555ec382a..b839e1e53 100644 --- a/internal/explorer/lifecycle/artifact.go +++ b/internal/explorer/lifecycle/artifact.go @@ -24,10 +24,6 @@ const ( defaultArtifactMaxBytes = int64(2 << 30) ) -// ArtifactRequest identifies the exact published revision and output that -// should be copied into a durable, downloadable artifact. The capability -// token is intentionally not transport-visible: lifecycle obtains the -// immutable receipt token server-side and reauthorizes the current caller. type ArtifactRequest struct { Project string ExplorerID string @@ -36,25 +32,16 @@ type ArtifactRequest struct { IdempotencyKey string } -// ArtifactResult returns the durable record. A PREPARING or FAILED record can -// be returned for an idempotent retry; callers must only offer a download for -// a COMPLETE record. type ArtifactResult struct { Record *explorer.ArtifactRecord } -// ArtifactOpenRequest identifies an artifact. Lifecycle reauthorizes the -// current caller against the receipt token retained in the server-side -// artifact record; capability internals never cross this request boundary. type ArtifactOpenRequest struct { Project string ExplorerID string ArtifactID string } -// PrepareArtifact authorizes and streams one exact execution/output into an -// ArtifactStore. The stage is committed only after WriteArtifact has closed -// every member and the exact reader has completed without pin loss. func (s *Service) PrepareArtifact(ctx context.Context, request ArtifactRequest) (ArtifactResult, error) { if s.config.ArtifactStore == nil || s.config.PublishedReader == nil { return ArtifactResult{}, unavailable("artifact", "ARTIFACT_UNAVAILABLE", "artifact export is not configured", nil) @@ -215,9 +202,6 @@ func (s *Service) PrepareArtifact(ctx context.Context, request ArtifactRequest) return ArtifactResult{Record: completed}, nil } -// OpenArtifact performs all identity and current-capability checks before -// returning the archive. The store itself is still required to enforce the -// COMPLETE/expiry check at the final open operation. func (s *Service) OpenArtifact(ctx context.Context, request ArtifactOpenRequest) (io.ReadCloser, *explorer.ArtifactRecord, error) { if s.config.ArtifactStore == nil { return nil, nil, unavailable("artifact", "ARTIFACT_UNAVAILABLE", "artifact export is not configured", nil) diff --git a/internal/explorer/lifecycle/artifact_test.go b/internal/explorer/lifecycle/artifact_test.go index 41612d45c..d084286d2 100644 --- a/internal/explorer/lifecycle/artifact_test.go +++ b/internal/explorer/lifecycle/artifact_test.go @@ -485,7 +485,6 @@ func TestPrepareArtifactAbortsAfterExactReaderFailure(t *testing.T) { t.Fatalf("err=%v, want pin-loss code", err) } request := ArtifactRequest{Project: receipt.Project, ExplorerID: receipt.ExplorerID, RevisionID: "revision-a", OutputID: "patients", IdempotencyKey: "download-failure"} - // The failed record remains durable for deterministic retry behavior. identity := explorer.ArtifactRecord{Project: receipt.Project, ExplorerID: receipt.ExplorerID, RevisionID: request.RevisionID, OutputID: request.OutputID, ReceiptID: receipt.ID, ExecutionID: "execution-a", DatasetGeneration: receipt.SourceGeneration, SchemaDigest: "schema-a", AuthorizationScopeDigest: receipt.AuthorizationScopeDigest, IdempotencyKey: request.IdempotencyKey} id, _ := explorer.ArtifactID(identity) record, getErr := artifacts.Get(context.Background(), id) @@ -513,8 +512,6 @@ func TestOpenArtifactReauthorizesCurrentToken(t *testing.T) { if err != nil { t.Fatal(err) } - // Build a complete record directly to isolate the download authorization - // gate from preparation. record := explorer.ArtifactRecord{Project: receipt.Project, ExplorerID: receipt.ExplorerID, RevisionID: "revision-a", OutputID: "patients", ReceiptID: receipt.ID, ExecutionID: "execution-a", DatasetGeneration: receipt.SourceGeneration, SchemaDigest: "schema-a", AuthorizationScopeDigest: receipt.AuthorizationScopeDigest, SnapshotToken: snapshot.Token, IdempotencyKey: "open", State: explorer.ArtifactComplete, ExpiresAt: time.Now().Add(time.Hour), CreatedAt: time.Now()} record.ID, _ = explorer.ArtifactID(record) artifacts.records[record.ID] = record diff --git a/internal/explorer/lifecycle/authoring.go b/internal/explorer/lifecycle/authoring.go index e9ab4132e..d9635dfee 100644 --- a/internal/explorer/lifecycle/authoring.go +++ b/internal/explorer/lifecycle/authoring.go @@ -24,10 +24,6 @@ func (s *Service) ApplyCommands(ctx context.Context, project, explorerID string, return response.Commands, nil } -// ApplyCommandsWithConstructionPreview returns the exact Apply-time preview -// alongside a successful construction Apply. The sink factory is transport -// owned and capture failures are best-effort; they never change Apply's -// existing validation or save behavior. func (s *Service) ApplyCommandsWithConstructionPreview(ctx context.Context, project, explorerID string, request authoringv2.ApplyCommandsRequest, actor string, sinkFactory PreviewSinkFactory) (*ApplyCommandsWithConstructionPreviewResponse, error) { if len(request.Commands) != 1 || request.Commands[0].Type != authoringv2.CommandApplyConstructionProposal { return nil, malformed("commands", "APPLY_CONSTRUCTION_PROPOSAL must be the only command in its atomic request", nil) @@ -542,9 +538,6 @@ func (s *Service) Reconcile(ctx context.Context, request ReconcileRequest) (*exp return result.Receipt, nil } -// ReconcileWithWorkspace compiles either the complete saved workspace or one -// output and its current-draft dependency closure, while returning the full -// CAS-validated workspace for Builder clients. func (s *Service) ReconcileWithWorkspace(ctx context.Context, request ReconcileRequest) (ReconcileResult, error) { if strings.TrimSpace(request.SnapshotToken) == "" || request.DraftVersion < 0 { return ReconcileResult{}, malformed("reconcile", "snapshotToken, draftVersion, and draftDigest are required", nil) diff --git a/internal/explorer/lifecycle/builder_full_ui_replay_test.go b/internal/explorer/lifecycle/builder_full_ui_replay_test.go index 79956603a..b890f37d7 100644 --- a/internal/explorer/lifecycle/builder_full_ui_replay_test.go +++ b/internal/explorer/lifecycle/builder_full_ui_replay_test.go @@ -27,9 +27,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// This replay consumes the state-space inventory and the exact UI observation -// capture as separate inputs. It intentionally keys both by the stable -// (caseId, entryId, actionIndex) tuple, never by a lossy affordance label. func TestBuilderFullUIRequestReplay(t *testing.T) { phase := os.Getenv("BUILDER_FULL_UI_PHASE") if phase == "" { @@ -609,9 +606,6 @@ func mustJSONRawString(t *testing.T, body []byte) json.RawMessage { } func TestBuilderFullUIGroupFilterDeriveAggregateEditReplaysLifecycle(t *testing.T) { - // This backend-only diagnostic constructs its own candidate. It proves the - // real lifecycle/compiler path, but is not frontend payload evidence; the - // artifact replay below consumes the mounted editor's raw request unchanged. fixture, workspace, _, err := builderFullUIGroupFilterDeriveFixture() if err != nil { t.Fatal(err) @@ -1737,9 +1731,6 @@ func newBuilderFullUIHarness(fixture builderFullUICapabilityFixture, workspace a config.CompileReceipt = func(ctx context.Context, request CompileReceiptRequest) (*explorer.CompilationReceipt, error) { receipt, compileErr := compileTraceReceipt(ctx, request, fixture.Snapshot, fixture.Catalog) if compileErr == nil { - // compileTraceReceipt runs the real workspace compiler/lowering path; - // this test adapter mirrors the production receipt wrapper's exact - // proposal identity binding, which is not a compiler-derived field. receipt.RowDefinitionProposal = cloneRowDefinitionProposalBinding(request.RowDefinitionProposal) if request.RowDefinitionProposal != nil { receipt.CompilationKey, compileErr = explorer.CompilationKey(*receipt) @@ -2047,7 +2038,7 @@ func bindBuilderFullUIInventorySeeds(rows []builderFullUIInventoryRow, manifest row := &rows[index] entry, exists := manifest[row.EntryID] if !exists { - continue // A missing manifest entry is recorded as a row-local GAP during prepare. + continue } if entry.ID != row.EntryID { return fmt.Errorf("manifest lookup key %q resolved entry id %q", row.EntryID, entry.ID) diff --git a/internal/explorer/lifecycle/cell_trace.go b/internal/explorer/lifecycle/cell_trace.go index 964cbab5a..b1e47d1c2 100644 --- a/internal/explorer/lifecycle/cell_trace.go +++ b/internal/explorer/lifecycle/cell_trace.go @@ -12,8 +12,6 @@ import ( "github.com/calypr/loom/internal/projectid" ) -// CellTrace returns receipt-bound evidence for one published cell. Lifecycle -// validates every client-supplied coordinate before execution sees it. func (s *Service) CellTrace(ctx context.Context, request CellTraceRequest) (CellTraceResult, error) { if strings.TrimSpace(request.ReceiptID) == "" || strings.TrimSpace(request.OutputID) == "" || strings.TrimSpace(request.RowID) == "" || strings.TrimSpace(request.Column) == "" { return CellTraceResult{}, malformed("cellTrace", "receiptId, outputId, rowId, and column are required", nil) diff --git a/internal/explorer/lifecycle/construction_category_discovery.go b/internal/explorer/lifecycle/construction_category_discovery.go index b3715972a..6297a5fc4 100644 --- a/internal/explorer/lifecycle/construction_category_discovery.go +++ b/internal/explorer/lifecycle/construction_category_discovery.go @@ -166,8 +166,6 @@ func (s *Service) compileConstructionCategoryCandidate( if pivotIndex < 0 || candidate.Steps[pivotIndex].Operation.Pivot == nil { return constructionBase{}, "", unprocessable("construction-category-discovery", "PIVOT_SOURCE_REQUIRES_PIVOT", "the candidate Pivot was not preserved", nil) } - // Keep the Pivot as the owner of its hidden input columns, but stop its - // downstream chain and replace its categories with one compiler-valid probe. candidate.Steps = append([]authoringv2.ConstructionStep(nil), candidate.Steps[:pivotIndex+1]...) owners := make(map[string]bool, len(candidate.Steps)) for _, step := range candidate.Steps { @@ -232,9 +230,6 @@ func (s *Service) compileConstructionCategoryCandidate( ID: probeID, Name: "__category_discovery_probe", Label: "Category discovery probe", Type: value.Type, }) - // Category discovery compiles a temporary staged candidate just like a - // proposal. Materialize legacy public columns first so they have the stable - // ColumnIDs required by a staged construction. document, err := authoringv2.UpgradeDocumentToConstruction(base.document) if err != nil { return constructionBase{}, "", unprocessable("construction-category-discovery", "INVALID_CONSTRUCTION_CANDIDATE", "the output cannot be upgraded for temporary category discovery", err) @@ -517,14 +512,11 @@ func classifyConstructionCategoryScanError(ctx context.Context, err error) error dataframeerrors.CodePreviewResponseTooLarge, dataframeerrors.CodePlanTooExpensive, dataframeerrors.CodeUnauthenticated: - // The HTTP adapter maps these status-specific codes directly. return err } if dataframeerrors.IsRetryableCode(dataframeerrors.ErrorCode(code)) { return failureDetails(ClassUnavailable, stage, code, message, details, cause) } - // Keep other typed dataframe errors intact so the transport adapter can - // preserve their established code and status mapping. return err } return internal(stage, "CATEGORY_DISCOVERY_FAILED", "category discovery failed unexpectedly while reading the catalog; inspect the server log for the request ID", cause) diff --git a/internal/explorer/lifecycle/construction_category_presence_test.go b/internal/explorer/lifecycle/construction_category_presence_test.go index b640ce367..8c38139f8 100644 --- a/internal/explorer/lifecycle/construction_category_presence_test.go +++ b/internal/explorer/lifecycle/construction_category_presence_test.go @@ -106,7 +106,7 @@ func TestDiscoverConstructionCategoriesCompilesPresenceForFilteredOrdinaryProbe( if err != nil { t.Fatalf("lower lifecycle receipt recipe: %v", err) } - query, err := compiler.CompileCategoryScanStageWithPolicy(compiled.Outputs[0], scan.StageID, scan.ColumnID, scan.ValueColumnID, scan.MaxValues, ir.DefaultPhysicalOptimizationPolicy()) + query, err := compiler.CompileCategoryScanStage(compiler.ArangoTarget(), compiled.Outputs[0], scan.StageID, scan.ColumnID, scan.ValueColumnID, scan.MaxValues, ir.DefaultPhysicalOptimizationPolicy()) if err != nil { t.Fatalf("compile lifecycle category stage: %v", err) } @@ -119,7 +119,12 @@ func TestDiscoverConstructionCategoriesCompilesPresenceForFilteredOrdinaryProbe( {query.PresentColumn: true, query.ValueColumn: nil}, {query.PresentColumn: false, query.ValueColumn: nil}, } - queryRows := func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + queryRows := func(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + for _, row := range rows { if err := visit(row); err != nil { return err @@ -127,8 +132,8 @@ func TestDiscoverConstructionCategoriesCompilesPresenceForFilteredOrdinaryProbe( } return nil } - engine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: categoryDiscoveryEmptyRecipeReader{}, QueryRows: queryRows, PreviewQueryRows: queryRows, + engine, err := dataframeexecution.New(dataframeexecution.Config{CompileTarget: compiler.ArangoTarget(), + Registry: categoryDiscoveryEmptyRecipeReader{}, SourceRows: queryRows, PreviewSourceRows: queryRows, }) if err != nil { t.Fatalf("make lifecycle category engine: %v", err) diff --git a/internal/explorer/lifecycle/construction_choice_proposal.go b/internal/explorer/lifecycle/construction_choice_proposal.go index e455f32ea..5f002f304 100644 --- a/internal/explorer/lifecycle/construction_choice_proposal.go +++ b/internal/explorer/lifecycle/construction_choice_proposal.go @@ -14,9 +14,6 @@ import ( const maxConstructionChoiceProposalChoices = 100 -// ConstructionChoiceProposalSelection mirrors one APPLY_CONSTRUCTION_CHOICE -// command. Title is presentation input and remains optional, just like the -// command itself. type ConstructionChoiceProposalSelection struct { RowValuePolicy authoringv2.ConstructionRowValuePolicy `json:"rowValuePolicy,omitempty"` ChoiceID string `json:"choiceId"` @@ -68,9 +65,6 @@ func (r ConstructionChoiceProposalRequest) Validate() error { return nil } -// ConstructionChoiceProposalResponse contains the exact candidate identities -// and receipt the transport must preview. PreviewReceiptID is deliberately -// transport-internal; clients receive the rendered preview instead. type ConstructionChoiceProposalResponse struct { CommandID string `json:"commandId"` SnapshotToken string `json:"snapshotToken"` @@ -86,9 +80,6 @@ type ConstructionChoiceProposalResponse struct { PreviewReceiptID string `json:"-"` } -// ProposeConstructionChoice compiles a candidate produced by the same -// server-side choice resolver and pure reducer used by APPLY_CONSTRUCTION_CHOICE. -// It does not save the candidate workspace or advance the draft version. func (s *Service) ProposeConstructionChoice(ctx context.Context, request ConstructionChoiceProposalRequest) (ConstructionChoiceProposalResponse, error) { if err := request.Validate(); err != nil { return ConstructionChoiceProposalResponse{}, malformed("construction-choice-proposal", err.Error(), err) diff --git a/internal/explorer/lifecycle/construction_choice_test.go b/internal/explorer/lifecycle/construction_choice_test.go index 6eb593571..f72d1b0d2 100644 --- a/internal/explorer/lifecycle/construction_choice_test.go +++ b/internal/explorer/lifecycle/construction_choice_test.go @@ -185,9 +185,6 @@ func TestResolveSemanticConstructionChoiceUsesFallbackCandidateOptionsAcrossOpti } } - // A scalar semantic value on a non-empty route must advertise the same - // route-specific reductions as a field choice for the same compiler - // candidate. The framed Builder flow reissues this choice during apply. scalarEntry := entry scalarEntry.Observation.Value.FallbackSelectors = nil scalarEntry.Observation.Value.FallbackOnBlankString = false diff --git a/internal/explorer/lifecycle/construction_coded_pivot.go b/internal/explorer/lifecycle/construction_coded_pivot.go index da5e66b24..bb2514dc6 100644 --- a/internal/explorer/lifecycle/construction_coded_pivot.go +++ b/internal/explorer/lifecycle/construction_coded_pivot.go @@ -68,9 +68,6 @@ func (s *Service) resolveConstructionCodedPivotChoices( if !codedPivotSourceMatchesPinnedPrior(durableSource, prior.Source, family.Entry.Observation) { return candidate, nil, conflict("construction-proposal", "STALE_CONSTRUCTION_CHOICE", "the selected family differs from the saved coded Pivot source", nil, nil) } - // Reauthorization used the current inventory and current authorization - // route above. Keep the prior persisted identity so this edit does not - // silently add newly introduced fallback semantics to the saved step. durableSource = prior.Source } priorKeys := make(map[string]bool, len(prior.Categories)) diff --git a/internal/explorer/lifecycle/construction_pivot_source.go b/internal/explorer/lifecycle/construction_pivot_source.go index aad65c3d8..c4bf14eb3 100644 --- a/internal/explorer/lifecycle/construction_pivot_source.go +++ b/internal/explorer/lifecycle/construction_pivot_source.go @@ -147,13 +147,6 @@ func (s *Service) constructionPivotSourceCapability( return result, nil } -// rowSourceSkipClass categorizes one root field a signed source lane skipped -// during its enumeration pass. When nothing is advertised after the full -// assembly, the lane-level reasonCode names the class if, and only if, every -// recorded skip shares one class; mixed or unmappable skips keep the existing -// generic NO_ELIGIBLE_POPULATED_* verdict (ROW-SOURCE-03 clause: "precise -// reasons for unsupported version, type, route, or stage" plus "distinguish -// empty data ... inventory errors"). type rowSourceSkipClass int const ( @@ -164,8 +157,6 @@ const ( rowSourceSkipInventory ) -// rowSourceEnumeration carries the gate facts each lane already holds at the -// moment it silently continues, so an empty verdict can name its real cause. type rowSourceEnumeration struct { rootReachable bool plannerConfigured bool @@ -179,8 +170,6 @@ func (e *rowSourceEnumeration) skip(class rowSourceSkipClass) { e.skips[class]++ } -// rowSourceLaneReasons is the per-lane wording bundle; the code strings are -// derived from the stable lane stem so the four signed lanes stay parallel. type rowSourceLaneReasons struct { codeStem string inventoryReason string @@ -193,15 +182,6 @@ type rowSourceLaneReasons struct { genericReason string } -// rowSourceInventoryFailure names the payload-distinguishable inventory-error -// verdict when a lane's row-choice enumeration aborts because the planner -// itself failed — the S-B split of the "distinguish ... inventory errors" -// clause, so an unavailable inventory is never confused with the healthy -// empty-data verdicts (NO_OBSERVED_VALUES_*/NO_ELIGIBLE_POPULATED_*). The -// advertised text is static: planner error strings can carry internal detail -// and must not cross into the payload (sanitizer boundary at the API edge). -// Fail-CLOSED is preserved — callers still receive a non-nil typed error that -// wraps the real cause for server-side logs and consumers. func rowSourceInventoryFailure(reasons rowSourceLaneReasons) (string, string) { return reasons.codeStem + "_SOURCE_INVENTORY_UNAVAILABLE", reasons.inventoryReason } @@ -228,11 +208,6 @@ func (e rowSourceEnumeration) decision(reasons rowSourceLaneReasons) (string, st return reasons.genericCode, reasons.genericReason } -// onlySkipClass reports the single reason class when every recorded skip -// shares it. The inventory class (the planner advertised a choice whose -// snapshot candidate is missing or duplicated) is intentionally not mapped to -// a new code: per-lane inventory-error state is the separate S-B slice, and -// mixed causes stay on the honest generic fallback. func (e rowSourceEnumeration) onlySkipClass() (rowSourceSkipClass, bool) { if len(e.skips) != 1 { return 0, false @@ -246,10 +221,6 @@ func (e rowSourceEnumeration) onlySkipClass() (rowSourceSkipClass, bool) { return 0, false } -// rowSourceCandidateSkipClass mirrors the fused gates of -// constructionGroupSourceCandidate for diagnosis only: the lane still decides -// its advertised set with the fused helper, so this never changes which -// fields are offered — only which reason an empty lane reports. func rowSourceCandidateSkipClass(snapshot capability.Snapshot, choice capability.RowChoice) rowSourceSkipClass { var found *capability.Candidate for index := range snapshot.Candidates { @@ -278,11 +249,6 @@ func rowSourceCandidateSkipClass(snapshot capability.Snapshot, choice capability } } -// rowSourceChoiceIsStaleVersion reports a signed choice issued for a previous -// dataset snapshot or schema digest. Undecodable ids are not flagged: the -// advertised set must stay bit-identical to pre-S-A behavior for every other -// input, and malformed ids remain refused at propose time by the -// INVALID_{LANE}_SOURCE_CHOICE boundary. func rowSourceChoiceIsStaleVersion(choice capability.RowChoice, snapshot capability.Snapshot) bool { identity, err := capability.DecodeRowChoiceID(choice.ChoiceID) if err != nil { @@ -453,24 +419,6 @@ func constructionFHIRPathLabel(resourceType, path string) string { return resourceType + "." + path } -// constructionFieldDisplayLabel is the single construction-lane label builder -// shared by every signed source lane (Filter, Derive, Unpivot, Pivot). It -// returns the first non-blank TrimSpace'd display candidate — the authorized -// snapshot candidate's schema-Title label first, then the row choice's -// display label at advertise sites — and only then falls back to the legacy -// 'ResourceType.path' form produced by constructionFHIRPathLabel. Display -// candidates must come only from the authorized snapshot or the planner's -// generated FHIR schema truth, never from client input; a candidate carrying -// any Unicode control character (NUL, ESC, other C0/C1) is refused outright -// and falls through to the next candidate — advertised labels flow verbatim -// to the wire and into durable claims, so the refusal mirrors the reason-text -// sanitizer family at the API edge (reviewer witness: a Title with NUL/ESC -// bytes must never ride the label). The result is always trimmed and non-blank so the receipt contract's pre-trimmed label gates -// hold. Every one of the twelve lane label sites must go through this -// builder: the durable claim and the compiled artifact copy its exact bytes, -// so an out-of-band label writer would break the -// explorer_output_reconcile.go compiled-slot equality that anti-drift -// reconcile depends on. func constructionFieldDisplayLabel(resourceType, path string, displayCandidates ...string) string { for _, candidate := range displayCandidates { label := strings.TrimSpace(candidate) @@ -490,9 +438,6 @@ type resolvedConstructionPivotSource struct { label string } -// constructionCandidateWithPivotSources resolves the signed source choices, -// moves them into the accepted Pivot's private input chain, and returns the -// generated RELATED_FIELD steps that have already passed authorization. func (s *Service) constructionCandidateWithPivotSources( ctx context.Context, base constructionBase, @@ -551,8 +496,6 @@ func (s *Service) constructionCandidateWithPivotSources( return resolved[i].selection.ColumnID < resolved[j].selection.ColumnID }) - // Editing a saved Pivot starts from the first accepted input before its old - // owned RELATED_FIELD chain. The chain is rebuilt from signed choices. oldOwned := make(map[string]bool) for _, step := range base.construction.Steps { if step.OwnerStepID == pivotID && step.Operation.Kind == authoringv2.ConstructionOperationRelatedField { @@ -591,8 +534,6 @@ func (s *Service) constructionCandidateWithPivotSources( } candidate.SourceProjections = projections - // Source projections enter at the source stage. Preserve them through each - // row-preserving prefix stage that leads to this Pivot. for index := 0; index < pivotIndex; index++ { step := &steps[index] if !authoringv2.ConstructionCanCarrySourceProjection(step.Operation.Kind) { @@ -792,15 +733,6 @@ func preserveConstructionOwnedInputs(base authoringv2.Construction, candidate au return candidate } -// constructionOwnerInputsClearedFromCandidate reports whether the submitted -// candidate intentionally removed every durable input the accepted base keeps -// for ownerID. Under the shipped selection contract (nil never re-attaches, -// [] clears), the candidate is a full replacement of the construction, so -// dropping all of an owner's server-owned inputs in the proposal window is a -// clear: no lane may re-import them from the base. A partial chain (the -// client carried the helper but not its projection, or the reverse) is a -// client omission of server-owned state, which preservation must still -// repair. func constructionOwnerInputsClearedFromCandidate(base authoringv2.Construction, candidate authoringv2.Construction, ownerID string) bool { baseInputs := 0 for _, step := range base.Steps { @@ -829,29 +761,6 @@ func constructionOwnerInputsClearedFromCandidate(base authoringv2.Construction, return true } -// constructionRootFieldAlreadyProjected reports whether materializing this -// signed slot column would collide, in the compiler's per-output duplicate -// semantic-path pass (lower/recipe_schema.go:205-224 suffixes BOTH columns -// with #), with another public column that carries the same compiled -// semantic path. The receipt contract guard compares strings.TrimSpace of the -// compiled column's SemanticPath against the authored projection FieldPath -// verbatim (explorer_output_reconcile.go:497-512), so it fires exactly when a -// second public column shares the slot's verbatim path string. Carriers are -// therefore precisely the columns whose compiled SemanticPath is an authored -// bare field-path string on the root occurrence: user-authored SourceField -// columns (compilation/semantic_compile.go:220 emits FieldRef verbatim, and -// lower.recipeSemanticPath keeps a non-empty fieldRef verbatim) and every -// merged construction source projection — base-persisted, candidate-carried, -// or request-resolved (compilation/semantic_compile.go:522-529 emits -// FieldRef=projection.FieldPath verbatim into the same root node). A -// root.-prefixed authored path is verbatim-distinct and never collides; a -// projection under the slot's own ColumnID is the slot's re-bind entry, not a -// second carrier. Columns whose compiled name is internal (_key/__loom_*) are -// excluded together with the duplicate counter that skips internal columns. -// Only projections owned by a Derive, Filter, or Unpivot step are slot -// carriers: those are the entries the receipt contract guard actually compares at -// :1022/:1156, and shipped lifecycle contracts establish that a slot sharing -// its field with a Pivot-owned source input is legitimate state. func constructionRootFieldAlreadyProjected(document authoringv2.Document, steps []authoringv2.ConstructionStep, projections []authoringv2.ConstructionSourceProjection, slotColumnID, fieldPath string) bool { slot := strings.TrimSpace(fieldPath) if slot == "" { diff --git a/internal/explorer/lifecycle/construction_pivot_source_test.go b/internal/explorer/lifecycle/construction_pivot_source_test.go index 10f03fb18..018b860ca 100644 --- a/internal/explorer/lifecycle/construction_pivot_source_test.go +++ b/internal/explorer/lifecycle/construction_pivot_source_test.go @@ -129,11 +129,6 @@ func TestConstructionCandidateWithPivotSourcesPreservesPrefixForValueLabelEdit(t } } -// ROW-SOURCE-03 S-C: the single display-label builder returns the first -// non-blank TrimSpace'd server-truth display candidate and only then falls -// back to the legacy 'ResourceType.path'. Snapshot candidates must never be -// client input; the pinned empty-Title shape mirrors builder.go:296, where -// the capability snapshot collapses a missing Title to the raw field path. func TestConstructionFieldDisplayLabelChainAndFallbacks(t *testing.T) { rows := []struct { name string @@ -237,8 +232,6 @@ func TestPivotSourcePresentationFailureIsPrefixScopeRejection(t *testing.T) { } } -// --- ROW-SOURCE-03 S-A: precise per-skipped-class lane reason codes ------- - func TestConstructionPivotSourceCapabilityReportsPreciseSkipClassReasonCodes(t *testing.T) { index, err := fhirschema.GeneratedIndex() if err != nil { @@ -369,10 +362,6 @@ func TestConstructionPivotSourceCapabilityReportsPreciseSkipClassReasonCodes(t * } } -// TestConstructionFieldDisplayLabelRefusesControlBytes locks the reviewer -// witness: a display candidate carrying NUL/ESC/other Unicode control -// characters must never ride the advertised/durable label; the builder falls -// through to the next candidate and ultimately to the legacy Type.path form. func TestConstructionFieldDisplayLabelRefusesControlBytes(t *testing.T) { cases := []struct{ name, candidate, fallback, want string }{ {"nul-esc-in-title-falls-to-next", "Active\x00flag\x1b[31mred", "Clean Title", "Clean Title"}, diff --git a/internal/explorer/lifecycle/construction_proposal.go b/internal/explorer/lifecycle/construction_proposal.go index 10d7fe0a9..b0a9701bc 100644 --- a/internal/explorer/lifecycle/construction_proposal.go +++ b/internal/explorer/lifecycle/construction_proposal.go @@ -79,18 +79,11 @@ type ConstructionGroupFieldChoice struct { MatchedOutputKeyID string `json:"matchedOutputKeyId,omitempty"` } -// ConstructionGroupRelatedOccurrenceChoice is display/filter metadata for a -// server-issued related occurrence CHOICE. RouteID is an opaque equality key -// for one Group editor and carries no proposal authority. type ConstructionGroupRelatedOccurrenceChoice struct { RouteID string `json:"routeId"` RouteLabel string `json:"routeLabel,omitempty"` } -// ConstructionSourceProjectionAvailability indicates whether root scalar -// source fields can be carried into this exact stage. It does not certify an -// individual field; the existing construction-choice proposal validates and -// compiles each selected field before it can be applied. type ConstructionSourceProjectionAvailability struct { Supported bool `json:"supported"` StageID string `json:"stageId"` @@ -98,9 +91,6 @@ type ConstructionSourceProjectionAvailability struct { Reason string `json:"reason,omitempty"` } -// ConstructionWorkspaceInput is a current sibling output eligible to be -// consumed by a same-workspace Combine. Its columns come from the finalized -// lower.OutputSchema frozen into the exact workspace compilation receipt. type ConstructionWorkspaceInput struct { OutputID string `json:"outputId"` Title string `json:"title"` @@ -341,9 +331,6 @@ func (s *Service) GetConstructionCapabilities(ctx context.Context, request Const if err != nil { return ConstructionCapabilitiesResponse{}, err } - // Choice IDs are ephemeral response authority, not receipt state. Copy the - // stage slice before decorating the selected descriptor so a later request - // still validates the original signed compilation receipt. stages := append([]explorer.ReceiptConstructionStage(nil), base.stages...) if authoringv2.CanPopulateConstructionRows(&base.construction) && len(stages) > 0 { last := len(stages) - 1 @@ -483,11 +470,6 @@ func constructionWorkspaceInputs(base constructionBase, targetOutputID string) ( return inputs, nil } -// constructionCapabilitiesWorkspace keeps the selected output, all of its -// inputs, and independent sibling outputs in the capability receipt. Current -// draft consumers of the selected output are omitted because their authored -// projections may be stale after a source replacement; the saved workspace is -// retained separately for CAS validation and the full-workspace Publish path. func constructionCapabilitiesWorkspace(workspace authoringv2.Workspace, targetOutputID string) (authoringv2.Workspace, error) { if constructionDocumentIndex(workspace, targetOutputID) < 0 { return authoringv2.Workspace{}, fmt.Errorf("selected output %q is missing or duplicated", targetOutputID) @@ -534,8 +516,6 @@ func constructionCapabilitiesWorkspace(workspace authoringv2.Workspace, targetOu return projected, nil } -// workspaceOutputConsumerClosure includes the selected output and each -// transitive current-draft output that consumes it. func workspaceOutputConsumerClosure(workspace authoringv2.Workspace, targetOutputID string) (map[string]struct{}, error) { if constructionDocumentIndex(workspace, targetOutputID) < 0 { return nil, fmt.Errorf("selected output %q is missing or duplicated", targetOutputID) @@ -1286,9 +1266,6 @@ func reauthorizeConstructionRelatedEligibility( step authoringv2.ConstructionStep, related authoringv2.ConstructionRelatedEligibility, ) error { - // RELATED_ELIGIBILITY shares RELATED_EXPAND's signed route and contributor - // choices. Reuse that exact-stage/anchor reauthorization path; its row-grain - // policy fields are irrelevant because this operation emits no expanded row. return reauthorizeConstructionRelatedExpand(ctx, base, rootResourceType, step, authoringv2.ConstructionRelatedExpand{ AnchorColumnID: related.AnchorColumnID, ChoiceID: related.ChoiceID, TargetNodeID: related.TargetNodeID, TargetResourceType: related.TargetResourceType, diff --git a/internal/explorer/lifecycle/construction_proposal_trace_test.go b/internal/explorer/lifecycle/construction_proposal_trace_test.go index c67654385..3a5a252ff 100644 --- a/internal/explorer/lifecycle/construction_proposal_trace_test.go +++ b/internal/explorer/lifecycle/construction_proposal_trace_test.go @@ -315,9 +315,6 @@ func compileTraceReceipt(ctx context.Context, request CompileReceiptRequest, sna } func traceReceiptStages(compiled lower.CompiledRecipe) map[string][]explorer.ReceiptConstructionStage { - // Mirrors production receiptConstructionStages (explorer_receipt_contract.go:605): outputs - // with no compiled stages are absent, and an all-stage-less compilation returns nil, not an - // empty map (nil-vs-non-nil diverges under reflect.DeepEqual against harness-built receipts). var stages map[string][]explorer.ReceiptConstructionStage for _, output := range compiled.Outputs { if len(output.Stages) == 0 { diff --git a/internal/explorer/lifecycle/construction_routes.go b/internal/explorer/lifecycle/construction_routes.go index 6c25646ea..bc981272f 100644 --- a/internal/explorer/lifecycle/construction_routes.go +++ b/internal/explorer/lifecycle/construction_routes.go @@ -182,8 +182,6 @@ type relatedFieldChoiceCursor struct { Offset int `json:"offset"` } -// SearchRelatedFieldChoices returns compiler-proved scalar fields on the -// active exact terminal resource at one current output stage. func (s *Service) SearchRelatedFieldChoices(ctx context.Context, request RelatedFieldChoiceSearchRequest) (RelatedFieldChoiceSearchResponse, error) { result := RelatedFieldChoiceSearchResponse{ SnapshotToken: request.SnapshotToken, OutputID: request.OutputID, StageID: request.StageID, @@ -302,9 +300,6 @@ func (s *Service) SearchRelatedFieldChoices(ctx context.Context, request Related return result, nil } -// SearchRelatedExpandContributorChoices lists compiler-proved scalar leaves, -// including explicitly bounded repeated leaves, on the route's terminal resource. -// The route token and catalog suggestions are bound to the current draft, snapshot, and stage. func (s *Service) SearchRelatedExpandContributorChoices(ctx context.Context, request RelatedExpandContributorChoiceSearchRequest) (RelatedExpandContributorChoiceSearchResponse, error) { result := RelatedExpandContributorChoiceSearchResponse{ SnapshotToken: request.SnapshotToken, OutputID: request.OutputID, StageID: request.StageID, @@ -595,8 +590,6 @@ type ResolvedPopulationRouteChoice struct { SelectionID string } -// SearchRelatedExpandChoices returns server-issued exact routes from the -// selected compiler stage's retained root resource anchor. func (s *Service) SearchRelatedExpandChoices(ctx context.Context, request RelatedExpandChoiceSearchRequest) (RelatedExpandChoiceSearchResponse, error) { result := RelatedExpandChoiceSearchResponse{ SnapshotToken: request.SnapshotToken, OutputID: request.OutputID, StageID: request.StageID, @@ -1015,9 +1008,6 @@ func routePathToOccurrence(root authoringv2.RouteNode, occurrenceID string) ([]a return nil, false } -// SearchConstructionChoices resolves one exact field or semantic identity for -// the current table output, enumerates its distinct directed routes, and only -// returns choices that compile with that full route. func (s *Service) SearchConstructionChoices(ctx context.Context, request ConstructionChoiceSearchRequest) (ConstructionChoiceSearchResponse, error) { result := ConstructionChoiceSearchResponse{SnapshotToken: request.SnapshotToken, OutputID: request.OutputID, Choices: []capability.ConstructionChoice{}} if strings.TrimSpace(request.Project) == "" || strings.TrimSpace(request.ExplorerID) == "" || strings.TrimSpace(request.SnapshotToken) == "" || strings.TrimSpace(request.OutputID) == "" || len(request.Cursor) > 4096 { diff --git a/internal/explorer/lifecycle/construction_semantic_group.go b/internal/explorer/lifecycle/construction_semantic_group.go index bc28b6fdd..db957577e 100644 --- a/internal/explorer/lifecycle/construction_semantic_group.go +++ b/internal/explorer/lifecycle/construction_semantic_group.go @@ -22,8 +22,6 @@ const ( constructionSemanticGroupChoicePageLimit = 5 ) -// ConstructionSemanticGroupChoice is an ephemeral, exact family choice for -// the selected Group stage. The durable operation stores a resolved binding. type ConstructionSemanticGroupChoice struct { ChoiceID string `json:"choiceId"` BindingID string `json:"bindingId"` diff --git a/internal/explorer/lifecycle/derive_source.go b/internal/explorer/lifecycle/derive_source.go index 2ff83b847..3f77f09e7 100644 --- a/internal/explorer/lifecycle/derive_source.go +++ b/internal/explorer/lifecycle/derive_source.go @@ -12,10 +12,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// ConstructionDeriveSourceCapability advertises the authorized populated root -// scalar fields a DERIVE step can consume through signed source choices, -// beside the existing Pivot, Group, and Filter source inputs. DERIVE operands -// feed arithmetic, so only integer and decimal scalar fields qualify. type ConstructionDeriveSourceCapability struct { Supported bool `json:"supported"` StageID string `json:"stageId"` @@ -24,12 +20,6 @@ type ConstructionDeriveSourceCapability struct { Choices []ConstructionDeriveSourceChoice `json:"choices"` } -// ConstructionDeriveSourceChoice is pinned to exactly six wire fields. The -// UI capabilities schema is strict, so pivot-style extras (occurrenceId, -// fhirType, valueType, isIdentifier, isReference) must never be added. -// ColumnID is the signed materialized column id from durable server-derived -// state, mirroring pivot's choice.columnId, so the UI can restore a saved -// source-backed DERIVE operand; it is "" while no live materialization exists. type ConstructionDeriveSourceChoice struct { ChoiceID string `json:"choiceId"` FieldPath string `json:"fieldPath"` @@ -39,18 +29,11 @@ type ConstructionDeriveSourceChoice struct { ColumnID string `json:"columnId"` } -// ConstructionDeriveSourceSelection binds one server-signed root scalar field -// choice to the exact columnId the edited DERIVE step will consume as its -// left or right arithmetic operand. type ConstructionDeriveSourceSelection struct { ChoiceID string `json:"choiceId"` ColumnID string `json:"columnId"` } -// maxDeriveSourceSelections bounds one proposal to the two arithmetic operand -// slots a single DERIVE step owns. This is the mirror of the pivot category -// cap discipline: the request channel can never grow past what one step can -// consume, and a duplicate columnId between the two slots is refused. const maxDeriveSourceSelections = 2 func validateConstructionDeriveSourceSelections(selections []ConstructionDeriveSourceSelection) error { @@ -74,11 +57,6 @@ func validateConstructionDeriveSourceSelections(selections []ConstructionDeriveS return nil } -// deriveSourceLogicalTypeIsNumeric reports whether a root scalar field can -// feed DERIVE arithmetic. The recipe compiler refuses any derived operand -// whose compiled kind is not integer or decimal, so discovery and admission -// here are pinned to exactly the kinds the compiler accepts (expression. -// ValueKind(mapping) rejects "number" and every non-numeric logical type). func deriveSourceLogicalTypeIsNumeric(logicalType string) bool { switch strings.ToLower(strings.TrimSpace(logicalType)) { case "integer", "decimal": @@ -88,10 +66,6 @@ func deriveSourceLogicalTypeIsNumeric(logicalType string) bool { } } -// deriveOperandColumnIDs returns the column ids the two arithmetic operands -// of a DERIVE operation consume. This is the documented divergence from the -// filter source channel, which binds one selection set to the single -// Filter.ColumnID; DERIVE consumes a list of up to two operand columns. func deriveOperandColumnIDs(derive *authoringv2.ConstructionDerive) []string { if derive == nil { return nil @@ -239,12 +213,6 @@ func (s *Service) resolveConstructionDeriveSource( }, nil } -// constructionCandidateWithDeriveSources resolves the signed field choices, -// derives the durable server-owned projection facts for the edited DERIVE -// step, and carries those slots through the row-preserving prefix. It mirrors -// the Filter source contract; the documented divergence is that DERIVE owns -// two arithmetic operand slots, so every selection columnId must be one of -// the step's COLUMN operand ids instead of a single filter column. func (s *Service) constructionCandidateWithDeriveSources( ctx context.Context, base constructionBase, @@ -428,13 +396,6 @@ func clearConstructionDeriveOwnedSources(base, candidate authoringv2.Constructio return candidate, nil } -// constructionOwnedDeriveSourceColumnID returns the durable server-derived -// materialized column for a root numeric field at one exact stage, mirroring -// constructionOwnedFilterSourceColumnID. A DERIVE owns the slot only while an -// owner DERIVE step consuming this exact stage still lists the projection -// column as one of its two arithmetic operand ids. A vanished, re-pointed, or -// foreign-owned materialization returns "", so the UI never renders a restore -// into a column the server can no longer sign. func constructionOwnedDeriveSourceColumnID(construction authoringv2.Construction, stageID, path string) string { for _, projection := range construction.SourceProjections { if projection.OwnerStepID == "" || canonicalRowPath(projection.FieldPath) != canonicalRowPath(path) { @@ -455,12 +416,6 @@ func constructionOwnedDeriveSourceColumnID(construction authoringv2.Construction return "" } -// ConstructionDeriveSourceFacts derives the server-owned projection facts for -// one root numeric field from the authorized snapshot plus generated FHIR -// metadata. Apply reauthorization and receipt reconcile share this single -// derivation so a persisted derive-owned claim can never drift from the facts -// the server would have signed it from. Column and owner identities stay -// claim-sourced, exactly as the Apply-side DeepEquals expects them. func ConstructionDeriveSourceFacts(snapshot capability.Snapshot, rootResourceType, fieldPath string) (authoringv2.ConstructionSourceProjection, bool) { path := canonicalRowPath(fieldPath) if strings.TrimSpace(rootResourceType) == "" || path == "" { @@ -518,10 +473,6 @@ func isDeriveOwnedSourceProjection(construction authoringv2.Construction, projec containsConstructionColumnIDLocal(deriveOperandColumnIDs(step.Operation.Derive), projection.ColumnID) } -// reauthorizeConstructionDeriveSources re-derives the server-owned facts for -// every derive-owned projection from the live authorized snapshot so an Apply -// cannot stage a derived column whose operand field stopped qualifying after -// the proposal. func (s *Service) reauthorizeConstructionDeriveSources(ctx context.Context, base constructionBase, outputID string, construction authoringv2.Construction) error { if s.config.RowChoicePlanner == nil { return unavailable("commands", "ROW_CHOICE_UNAVAILABLE", "authorized derive source choice planning is not configured", nil) diff --git a/internal/explorer/lifecycle/derive_source_test.go b/internal/explorer/lifecycle/derive_source_test.go index c1a068c04..72d0a92a1 100644 --- a/internal/explorer/lifecycle/derive_source_test.go +++ b/internal/explorer/lifecycle/derive_source_test.go @@ -100,8 +100,6 @@ func deriveSourceBase(snapshot capability.Snapshot, construction authoringv2.Con } } -// deriveSourceAddStep adds a literal to one column operand, so the signed -// projection can only ever occupy the left COLUMN operand slot. func deriveSourceAddStep(stepID, leftID string, outputs []authoringv2.StageColumn) authoringv2.ConstructionStep { return authoringv2.ConstructionStep{ ID: stepID, @@ -178,9 +176,6 @@ func deriveSourceProposalRequest(changedStepID string, sources []ConstructionDer } } -// deriveSourcePatientHarness reuses the proven filter harness wiring (real -// compiler lowering + schema row-choice planner around the patients fixture) -// and only swaps the consumed operation semantics. type deriveSourcePatientHarness struct { *filterSourceHarness } @@ -244,8 +239,6 @@ func deriveSourcePatientDocument(construction authoringv2.Construction) authorin } } -// (a) Discovery exposes the populated unselected root INTEGER scalar with a -// signed choice id and never advertises non-numeric fields. func TestConstructionDeriveSourceCapabilityExposesPopulatedUnselectedNumericRootScalar(t *testing.T) { harness := newDeriveSourceHarness(t) result, err := harness.service.constructionDeriveSourceCapability(context.Background(), @@ -295,7 +288,6 @@ func TestConstructionDeriveSourceCapabilityExposesPopulatedUnselectedNumericRoot } } -// The non-numeric gate is enforced on admission, not only display. func TestConstructionDeriveSourceRejectsNonNumericFieldOnDiscoveryAndAdmission(t *testing.T) { snapshot := deriveSourceSnapshot("derive-source-nonnumeric") service := deriveSourceService(t) @@ -310,8 +302,6 @@ func TestConstructionDeriveSourceRejectsNonNumericFieldOnDiscoveryAndAdmission(t if result.ReasonCode != "NO_ELIGIBLE_POPULATED_DERIVE_SOURCE_FIELDS" || len(result.Choices) != 0 { t.Fatalf("unsupported capability = %#v, want the derive-source reason without choices", result) } - // The planner still issues a signed token for the boolean scalar field, so - // the proposal boundary must refuse it rather than trust the picker. nonnumeric := deriveSourceRowChoice(t, snapshot, deriveSourceRootType, deriveSourceField) if _, err := capability.DecodeRowChoiceID(nonnumeric.ChoiceID); err != nil { t.Fatalf("crafted boolean candidate produced no signed token: %v", err) @@ -328,7 +318,6 @@ func TestConstructionDeriveSourceRejectsNonNumericFieldOnDiscoveryAndAdmission(t } } -// The numeric gate is pure logic and must admit exactly integer and decimal. func TestDeriveSourceLogicalTypeIsNumericAdmitsOnlyArithmeticKinds(t *testing.T) { for _, want := range []string{"integer", "decimal", " INTEGER", "Decimal "} { if !deriveSourceLogicalTypeIsNumeric(want) { @@ -342,7 +331,6 @@ func TestDeriveSourceLogicalTypeIsNumericAdmitsOnlyArithmeticKinds(t *testing.T) } } -// (c) A token from another snapshot, or a tampered token, is refused. func TestConstructionDeriveSourceRefusesForgedAndCrossSnapshotChoices(t *testing.T) { snapshot := deriveSourceSnapshot("derive-source-current") other := deriveSourceSnapshot("derive-source-other") @@ -376,8 +364,6 @@ func TestConstructionDeriveSourceRefusesForgedAndCrossSnapshotChoices(t *testing } } -// (b) A valid selection durably owns exactly one server-derived projection and -// survives the real authoring validator, row-preserving. func TestConstructionCandidateWithDeriveSourcesMaterializesAndRowPreserves(t *testing.T) { harness := newDeriveSourceHarness(t) snapshot := harness.snapshot @@ -429,8 +415,6 @@ func TestConstructionCandidateWithDeriveSourcesMaterializesAndRowPreserves(t *te if len(analyzed.Construction.SourceProjections) != 1 || analyzed.Construction.SourceProjections[0].OwnerStepID != deriveSourceStep { t.Fatalf("analysis dropped or re-owned the derive projection: %#v", analyzed.Construction.SourceProjections) } - // DERIVE is row-preserving: the signed slot plus the derived column are the - // only additions over the selected identity. wantIDs := []string{"patient-id", deriveSourceBirth, "derived_total"} gotIDs := make([]string, 0, len(analyzed.Construction.Steps[0].Outputs)) for _, output := range analyzed.Construction.Steps[0].Outputs { @@ -441,7 +425,6 @@ func TestConstructionCandidateWithDeriveSourcesMaterializesAndRowPreserves(t *te } } -// The row-preserving prefix allow-list is enforced, not trust-me coded. func TestConstructionDeriveSourceCannotCrossRowChangingPrefix(t *testing.T) { harness := newDeriveSourceHarness(t) group := filterSourceGroupStep() @@ -475,7 +458,6 @@ func TestConstructionDeriveSourceCannotCrossRowChangingPrefix(t *testing.T) { } } -// A projection the derive step does not consume as an operand is refused. func TestConstructionDeriveSourceProjectionMustMatchItsOperand(t *testing.T) { document := deriveSourcePatientDocument(authoringv2.Construction{Version: authoringv2.ConstructionVersion}) @@ -509,8 +491,6 @@ func TestConstructionDeriveSourceProjectionMustMatchItsOperand(t *testing.T) { } } -// (e) Dropping the selections strips their projections, and a derive left -// pointing at a removed source operand fails instead of silently keeping it. func TestConstructionDeriveSourceRemovalStripsProjectionAndRefusesDanglingDerive(t *testing.T) { harness := newDeriveSourceHarness(t) snapshot := harness.snapshot @@ -556,7 +536,6 @@ func TestConstructionDeriveSourceRemovalStripsProjectionAndRefusesDanglingDerive } } -// (f) Selected-column derive behavior is unchanged by derive source support. func TestConstructionDeriveSourceLeavesSelectedColumnBehaviorUnchanged(t *testing.T) { harness := newDeriveSourceHarness(t) snapshot := harness.snapshot @@ -566,7 +545,6 @@ func TestConstructionDeriveSourceLeavesSelectedColumnBehaviorUnchanged(t *testin }} document := deriveSourcePatientDocument(construction) if _, _, err := document.AnalyzeConstructionCandidate(construction, deriveSourceStep, nil); err == nil { - // string operand is invalid at compile, so only require proposal neutrality below. } unchanged, err := service.constructionCandidateWithDeriveSources(context.Background(), deriveSourcePatientBase(snapshot, construction), deriveSourcePatientProposalRequest(deriveSourceStep, nil), construction) @@ -585,8 +563,6 @@ func TestConstructionDeriveSourceLeavesSelectedColumnBehaviorUnchanged(t *testin } } -// Duplicate or blank selection facts are refused at the request boundary, and -// more than two selections exceeds the two operand slots. func TestConstructionDeriveSourceSelectionsRequireExactDeduplicatedFacts(t *testing.T) { duplicate := deriveSourceProposalRequest(deriveSourceStep, []ConstructionDeriveSourceSelection{ {ChoiceID: "rc1.a.b", ColumnID: "c1"}, {ChoiceID: "rc1.a.b", ColumnID: "c2"}, @@ -617,9 +593,6 @@ func TestConstructionDeriveSourceSelectionsRequireExactDeduplicatedFacts(t *test } } -// (g) The advertised choice object is EXACTLY the six pinned wire fields. Any -// pivot-style extra field makes the strict UI schema reject the whole -// capabilities response, so the shape is pinned structurally and on the wire. func TestConstructionDeriveSourceChoiceWireShapeIsExactlySixFields(t *testing.T) { wantFields := map[string]bool{"choiceId": true, "fieldPath": true, "label": true, "logicalType": true, "isPopulated": true, "columnId": true} structFields := make(map[string]bool, reflect.TypeOf(ConstructionDeriveSourceChoice{}).NumField()) @@ -655,9 +628,6 @@ func TestConstructionDeriveSourceChoiceWireShapeIsExactlySixFields(t *testing.T) } } -// The advertised columnId is the signed materialized column the UI uses to -// restore a saved source-backed derive operand. It must come only from durable -// server-derived state that is still live for exactly this stage. func TestConstructionDeriveSourceCapabilityAdvertisesDurableColumnIdOnlyWhileSigned(t *testing.T) { harness := newDeriveSourceHarness(t) snapshot := harness.snapshot @@ -718,8 +688,6 @@ func TestConstructionDeriveSourceCapabilityAdvertisesDurableColumnIdOnlyWhileSig } } -// (a, wiring) GetConstructionCapabilities assembles deriveSourceInput beside -// pivotSourceInput and filterSourceInput on the owned-style fixture. func TestGetConstructionCapabilitiesAssemblesDeriveSourceInputBesidePivotSourceInput(t *testing.T) { harness := newDeriveSourceHarness(t) capabilities, err := harness.service.GetConstructionCapabilities(context.Background(), ConstructionCapabilitiesRequest{ @@ -770,8 +738,6 @@ func TestGetConstructionCapabilitiesAssemblesDeriveSourceInputBesidePivotSourceI } } -// (b) Through ProposeConstruction with the real compiler, the accepted -// candidate derives from the unselected field's compiled slot. func TestProposeConstructionDeriveSourceDerivesFromUnselectedFieldWithRealCompiler(t *testing.T) { harness := newDeriveSourceHarness(t) harness.saveDraftDerive(t, authoringv2.Construction{Version: authoringv2.ConstructionVersion, Steps: []authoringv2.ConstructionStep{ @@ -845,9 +811,6 @@ func TestProposeConstructionDeriveSourceDerivesFromUnselectedFieldWithRealCompil } } -// A derive step may consume two COLUMN operands (left projection + right -// projection or column); the compiled multiplicity is unchanged because the -// expression stays row-wise. func TestProposeConstructionDeriveSourceCompilesChainedColumnOperands(t *testing.T) { harness := newDeriveSourceHarness(t) firstDerived := authoringv2.StageColumn{ID: "chained_base", Name: "chained_base", Label: "Chained base", Type: "integer"} @@ -920,7 +883,6 @@ func TestProposeConstructionDeriveSourceCompilesChainedColumnOperands(t *testing } } -// (6) An accepted proposal whose field stops qualifying is refused at Apply. func TestApplyConstructionProposalRefusesStaleDeriveSourceBinding(t *testing.T) { for _, stale := range []bool{false, true} { name := "current" @@ -997,9 +959,6 @@ func TestApplyConstructionProposalRefusesStaleDeriveSourceBinding(t *testing.T) } } -// A restore of a saved source-backed derive whose field stopped presenting -// data must fail closed at proposal time like a forged token: the durable -// columnId alone never vouches for the vanished field. func TestProposeConstructionDeriveSourceRefusesVanishedFieldRestore(t *testing.T) { harness := newDeriveSourceHarness(t) durable := authoringv2.Construction{ @@ -1075,8 +1034,6 @@ func (h *deriveSourcePatientHarness) candidateDeriveConstruction() authoringv2.C return candidate } -// --- ROW-SOURCE-03 S-A: precise per-skipped-class lane reason codes ------- - func TestConstructionDeriveSourceCapabilityReportsPreciseSkipClassReasonCodes(t *testing.T) { index, err := fhirschema.GeneratedIndex() if err != nil { diff --git a/internal/explorer/lifecycle/draft_restore.go b/internal/explorer/lifecycle/draft_restore.go index 40a18f920..742f97db1 100644 --- a/internal/explorer/lifecycle/draft_restore.go +++ b/internal/explorer/lifecycle/draft_restore.go @@ -65,8 +65,6 @@ func (s *Service) prepareDraftRevisionRestore( return nil, conflict("commands", "DRAFT_REVISION_INVALID", "the prior draft revision failed its digest check", nil, digestErr) } } else { - // A first draft mutation can snapshot an empty draft config. Restore the - // effective prior workspace from the still-active immutable revision. if owner.ActiveRevisionID != "" { active, activeErr := s.store.ActiveRevision(ctx, project, explorerID) if activeErr != nil { diff --git a/internal/explorer/lifecycle/errors.go b/internal/explorer/lifecycle/errors.go index 8e7b2a8f5..2fa2ec338 100644 --- a/internal/explorer/lifecycle/errors.go +++ b/internal/explorer/lifecycle/errors.go @@ -6,8 +6,6 @@ import ( "github.com/calypr/loom/internal/explorer" ) -// ErrorClass lets transports map application failures to their own response -// contracts without making HTTP status codes part of the application layer. type ErrorClass string const ( @@ -20,9 +18,6 @@ const ( ClassInternal ErrorClass = "internal" ) -// Error is a typed application failure. Details are safe, structured -// diagnostics; Cause remains available to server-side logging and is never -// serialized by lifecycle. type Error struct { Class ErrorClass Stage string diff --git a/internal/explorer/lifecycle/expand_source.go b/internal/explorer/lifecycle/expand_source.go index 9c5b46e14..834f4572d 100644 --- a/internal/explorer/lifecycle/expand_source.go +++ b/internal/explorer/lifecycle/expand_source.go @@ -13,15 +13,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// ConstructionExpandSourceCapability advertises the authorized populated direct -// root primitive ARRAY fields an EXPAND step can flatten through signed source -// choices, beside the existing Pivot, Group, Filter, Derive, and Unpivot source -// inputs. EXPAND flattens one array-valued column into its scalar items, so the -// compiler's own array input gate (lowerConstructionExpand requires a public, -// Many-cardinality column whose items are a table scalar kind) qualifies only -// the direct root primitive arrays whose generated-schema element type the -// recipe construction logical-type whitelist accepts as a scalar — objects, -// Coding, nested arrays, repeated-boundary scopes, and references are refused. type ConstructionExpandSourceCapability struct { Supported bool `json:"supported"` StageID string `json:"stageId"` @@ -30,15 +21,6 @@ type ConstructionExpandSourceCapability struct { Choices []ConstructionExpandSourceChoice `json:"choices"` } -// ConstructionExpandSourceChoice is pinned to exactly six wire fields. The -// UI capabilities schema is strict, so pivot-style extras (occurrenceId, -// fhirType, valueType, isIdentifier, isReference) must never be added. -// LogicalType is the ELEMENT logical type of the array (for example "string" -// for a HumanName family array), never a container label, because the existing -// EXPAND renderer only accepts scalar items. ColumnID is the signed -// materialized column id from durable server-derived state, mirroring pivot's -// choice.columnId, so the UI can restore a saved source-backed EXPAND array -// input; it is "" while no live materialization exists. type ConstructionExpandSourceChoice struct { ChoiceID string `json:"choiceId"` FieldPath string `json:"fieldPath"` @@ -48,20 +30,11 @@ type ConstructionExpandSourceChoice struct { ColumnID string `json:"columnId"` } -// ConstructionExpandSourceSelection binds one server-signed root primitive -// array field choice to the exact columnId that is the edited EXPAND step's -// InputColumnID. The channel admits exactly one selection per proposal. type ConstructionExpandSourceSelection struct { ChoiceID string `json:"choiceId"` ColumnID string `json:"columnId"` } -// validateConstructionExpandSourceSelections is the request-boundary guard. -// There is no invented per-request cap: the honest bound is structural, and -// the candidate prep refuses any selection whose columnId is not the edited -// step's single declared array input, so the request channel can never -// outgrow what one expand consumes. A duplicate choiceId or columnId is -// refused outright. func validateConstructionExpandSourceSelections(selections []ConstructionExpandSourceSelection) error { seenChoices := make(map[string]bool, len(selections)) seenColumns := make(map[string]bool, len(selections)) @@ -80,18 +53,6 @@ func validateConstructionExpandSourceSelections(selections []ConstructionExpandS return nil } -// expandSourceElementLogicalTypeIsAdmissible is the expand lane's own admission -// of an array element as a flattenable scalar item. It delegates the primitive -// vocabulary to the persisted recipe validator's exported whitelist, -// recipe.IsValidConstructionLogicalType, so this channel can never advertise an -// element type the construction recipe would later reject. The whitelist itself -// still accepts "object" (the recipe carries object columns for other lanes), -// so the lane layers the contract's object refusal on top of the single -// delegated decision; the generated-schema gate re-checks the element through -// FHIRType (including Coding) at resolve time. If the recipe whitelist is ever -// un-exported again, this is the local canonical mirror point; the -// whitelist-drift guard test (parent-owned) must keep this gate and the recipe -// validator in lockstep. func expandSourceElementLogicalTypeIsAdmissible(logicalType string) bool { kind := strings.ToLower(strings.TrimSpace(logicalType)) if kind == "" || kind == "object" { @@ -100,10 +61,6 @@ func expandSourceElementLogicalTypeIsAdmissible(logicalType string) bool { return recipe.IsValidConstructionLogicalType(kind) } -// expandSourceElementFHIRTypeIsAdmissible refuses the object-bearing element -// FHIR types the generated index reports for non-primitive array items: the -// anonymous "object" shape and Coding, whose code/display payloads the EXPAND -// renderer cannot flatten to one scalar item column. func expandSourceElementFHIRTypeIsAdmissible(fhirType string) bool { switch strings.TrimSpace(fhirType) { case "", "object", "Coding": @@ -112,17 +69,8 @@ func expandSourceElementFHIRTypeIsAdmissible(fhirType string) bool { return true } -// expandSourceCardinality is the single wire cardinality the lane signs into -// the server-populated ConstructionExpand.ExpandedField payload. It matches -// authoringv2.ConstructionExpandedFieldCardinalityMany, which the compiler mapping turns -// into the expression.Many the physical EXPAND gate requires -// (lower/recipe_construction_reshape.go:473). const expandSourceCardinality = authoringv2.ConstructionExpandedFieldCardinalityMany -// expandSourceSignedField derives the server-owned operation payload from one -// resolved lane projection. The client never authors it: constructionCandidate- -// WithExpandSources overwrites whatever the candidate carried, and Apply-time -// reauthorization re-derives it from snapshot facts and refuses mismatch. func expandSourceSignedField(projection authoringv2.ConstructionSourceProjection) *authoringv2.ConstructionExpandedFieldSource { return &authoringv2.ConstructionExpandedFieldSource{ Path: projection.FieldPath, @@ -132,15 +80,6 @@ func expandSourceSignedField(projection authoringv2.ConstructionSourceProjection } } -// expandSourceBackingCandidate is the EXPAND-lane counterpart of the scalar -// row-source backing check. A signed array row choice is backed by a snapshot -// candidate for a direct-root repeated array: many cardinality, the ARRAY -// projection the capability builder mints for repeated fields, and boundaries -// that only ever describe the field itself (a boundary at another path marks -// nested repetition, which is not a direct-root array the EXPAND step can -// flatten). The scalar lane's constructionGroupSourceCandidate rejects exactly -// this shape by design — an array cannot be carried as a scalar VALUE — so the -// lanes must not share one backing rule. func expandSourceBackingCandidate(snapshot capability.Snapshot, nodeID, resourceType, path string) (capability.Candidate, bool) { candidate, found := uniqueRowSourceCandidate(snapshot, nodeID, resourceType, path) if !found || !candidate.Populated || !capability.IsRepeatedCardinality(candidate.Cardinality) || @@ -160,9 +99,6 @@ func expandSourceIsDirectRootRepeated(candidate capability.Candidate, path strin return true } -// expandSourceBackingSkipClass names the honest classification for a refusal -// to back an array row choice, mirroring expandSourceBackingCandidate clause -// by clause so the lane's reason codes stay as precise as the scalar lane's. func expandSourceBackingSkipClass(snapshot capability.Snapshot, choice capability.RowChoice) rowSourceSkipClass { candidate, found := uniqueRowSourceCandidate(snapshot, choice.NodeID, choice.ResourceType, choice.Path) switch { @@ -179,8 +115,6 @@ func expandSourceBackingSkipClass(snapshot capability.Snapshot, choice capabilit } } -// expandSourceStepPayloadIsConsistent reports whether the owner step carries a -// shape-valid signed payload pointing at exactly this canonical field path. func expandSourceStepPayloadIsConsistent(step authoringv2.ConstructionStep, fieldPath string) bool { if step.Operation.Expand == nil || step.Operation.Expand.ExpandedField == nil { return false @@ -192,11 +126,6 @@ func expandSourceStepPayloadIsConsistent(step authoringv2.ConstructionStep, fiel return canonicalRowPath(source.Path) == canonicalRowPath(fieldPath) } -// expandInputColumnIDs returns the column ids an EXPAND operation consumes. -// This is the documented divergence from the Unpivot source channel, which -// melts an arbitrary declared value set: EXPAND flattens exactly one declared -// array input, and the array column itself leaves the expand's own output -// schema while its prefix carries it. func expandInputColumnIDs(expand *authoringv2.ConstructionExpand) []string { if expand == nil || strings.TrimSpace(expand.InputColumnID) == "" { return nil @@ -243,13 +172,6 @@ func (s *Service) constructionExpandSourceCapability( enumeration.skip(rowSourceSkipVersion) continue } - // The planner mints an array-shaped root field only as the - // ExpandedScope class (applicableRowChoiceKinds): FieldGroupKey is - // reserved for scalar grouping keys. capability.RowChoice carries - // no Shape field, but the minting gate (capability.NewRowChoice via - // validateRowChoiceFacts) pins ARRAY<->MANY onto ExpandedScope as - // one class, so this filter stays alongside resolve's re-pin of - // identity.Shape == RowChoiceArray against the decoded signed id. if choice.Kind != capability.RowChoiceExpandedScope || choice.OccurrenceID != authoringv2.RootOccurrenceID || choice.Reference || choice.Cardinality != capability.RowChoiceMany || canonicalRowPath(choice.Path) == "resourceType" { enumeration.skip(rowSourceSkipRoute) @@ -331,10 +253,6 @@ func (s *Service) resolveConstructionExpandSource( if err != nil { return authoringv2.ConstructionSourceProjection{}, unavailable("construction-proposal", "FHIR_SCHEMA_UNAVAILABLE", "FHIR field metadata is unavailable", err) } - // The generated-schema gate mirrors the array shape rules the semantic row - // expander enforces (compilation/row_expansion.go): a canonical, generated - // ARRAY of MANY non-reference values whose element FHIRType still matches - // the signed identity. facts, err := schemaIndex.ResolveRowPath(fhirschema.DefinitionName(rootNode.ResourceType), resolved.ScopePath) if err != nil || facts.CanonicalPath != resolved.ScopePath || facts.Shape != fhirschema.RowPathArray || facts.Cardinality != fhirschema.RowCardinalityMany || facts.Reference || facts.FHIRType != identity.FHIRType { @@ -357,19 +275,6 @@ func (s *Service) resolveConstructionExpandSource( }, nil } -// constructionCandidateWithExpandSources resolves the signed field choice, -// derives the durable server-owned projection facts for the edited EXPAND -// step's single array input, and carries that column through the -// row-preserving prefix that feeds the flatten. It mirrors the Unpivot source -// contract; the documented divergences are (1) EXPAND consumes exactly one -// input column — the step's declared InputColumnID — instead of melting an -// arbitrary declared set, (2) the array column is carried by the steps -// strictly before the expand, whose own outputs leave it to be consumed -// exactly as the existing lowerConstructionExpand receives a selected-column -// expand input from an earlier stage (the EXPAND step's own declarations keep -// only its client-declared item/ordinal/preserved outputs; it must not -// re-declare its input), and (3) there is no joint melt-class check because one -// array contributes one scalar item type. func (s *Service) constructionCandidateWithExpandSources( ctx context.Context, base constructionBase, @@ -381,11 +286,6 @@ func (s *Service) constructionCandidateWithExpandSources( candidate.Steps[changedIndex].Operation.Kind == authoringv2.ConstructionOperationExpand candidate = preserveConstructionOwnedInputs(base.construction, candidate, "") if !applyExpandSources { - // A nil channel must not strand a signed Expand step: the editing - // client rebuilds the operation from form state and can never carry - // the server-signed payload back. Restore it from the durable - // base-owned projection so save/reopen round-trips compile, and - // strip payloads nothing owns server-side. return repairSignedExpandFields(base.construction, candidate, nil), nil } expand := candidate.Steps[changedIndex].Operation.Expand @@ -397,9 +297,6 @@ func (s *Service) constructionCandidateWithExpandSources( if err != nil { return authoringv2.Construction{}, unprocessable("construction-proposal", "EXPAND_SOURCE_STAGE_UNAVAILABLE", err.Error(), err) } - // Removal strips the signed payload as well: the cleared step no - // longer has a candidate-side owned projection, so the same repair - // pass that restores intact payloads strips this one's. return repairSignedExpandFields(base.construction, cleared, nil), nil } acceptedStageID := constructionStepInputStageID(candidate.Steps[changedIndex]) @@ -495,36 +392,14 @@ func (s *Service) constructionCandidateWithExpandSources( if droppedOwned[output.ID] && !reowned[output.ID] { continue } - // The EXPAND step's own declarations keep only the client-declared - // item/ordinal/preserved outputs: lowerConstructionExpand refuses any - // declaration equal to expand.InputColumnID - // (lower/recipe_construction_reshape.go:530), and the input stage — not - // this step — must present the public Many array column - // (lower/recipe_construction_reshape.go:469-478). expandOutputs = append(expandOutputs, output) } steps[changedIndex].Outputs = expandOutputs - // Sign the operation payload (RelatedExpand ContributorSource precedent): - // authoringv2.StageColumn carries no Cardinality, so the compiled Many - // cardinality of the materialized array input is assigned from this - // server-derived shape in the authoring-to-recipe mapping (owned by the - // parent at internal/explorer/compilation/construction.go). Clone the - // operation payload before mutation: the step copy above still aliases the - // caller's ConstructionExpand pointer. signed := expandSourceSignedField(resolved[0]) expanded := *steps[changedIndex].Operation.Expand expanded.ExpandedField = signed steps[changedIndex].Operation.Expand = &expanded - // First-step note: with no prefix, the array column reaches the input stage - // through construction.SourceProjections, which is how the root - // SOURCE_PROJECTION stage's public columns are produced - // (internal/explorer/compilation/construction.go:33-43 plus the semantic - // source verifier at internal/explorer/compilation/semantic_compile.go:507-533). - // The semantic source verifier carries the EXPAND-owned array accept - // branch (facts-proved RowPathArray + Many + non-reference), and the - // semantic selector resolver derives the compiled cardinality from the - // generated schema; the lane never hacks cardinality through stage text. candidate.Steps, candidate.SourceProjections = steps, projections candidate = repairSignedExpandFields(base.construction, candidate, resolved) @@ -534,16 +409,6 @@ func (s *Service) constructionCandidateWithExpandSources( return candidate, nil } -// repairSignedExpandFields reconciles every EXPAND step's server-signed -// payload against server-derived ownership. A projection counts as trusted -// ownership ONLY when it equals (byte-for-byte) a projection this wave's -// resolve minted from the live snapshot (minted) or a durable base-owned -// projection the lane's own resolve wrote and Apply re-proved. Client-authored -// candidate projections never establish ownership: without a matching minted -// or durable entry the payload is stripped, and the authoring unowned-source -// wall refuses the dangling step. Restoring an owned payload always re-derives -// it from the trusted projection, never from client payload text. Apply-time -// reauthorization re-proves everything against the live snapshot regardless. func repairSignedExpandFields(base authoringv2.Construction, candidate authoringv2.Construction, minted []authoringv2.ConstructionSourceProjection) authoringv2.Construction { baseProjections := make(map[string]authoringv2.ConstructionSourceProjection, len(base.SourceProjections)) for _, projection := range base.SourceProjections { @@ -572,9 +437,6 @@ func repairSignedExpandFields(base authoringv2.Construction, candidate authoring continue } inputID := step.Operation.Expand.InputColumnID - // Candidate entries are trusted only when byte-equal to this wave's - // mint; everything else falls back to the durable base, then to - // not-owned (stripped), never to client text. projection, owned := candidateProjections[inputID] minted, hasMinted := mintedProjections[inputID] if !owned || projection.OwnerStepID != step.ID || !hasMinted || !reflect.DeepEqual(projection, minted) { @@ -600,10 +462,6 @@ func repairSignedExpandFields(base authoringv2.Construction, candidate authoring expanded.ExpandedField = expandSourceSignedField(projection) step.Operation.Expand = &expanded } - // Drop expand-owned projections the client authored outright: any - // projection claiming an EXPAND step as owner that the server did not - // mint this wave or durably own in the base is client text and must not - // reach the receipt document. if len(candidate.SourceProjections) > 0 { filtered := make([]authoringv2.ConstructionSourceProjection, 0, len(candidate.SourceProjections)) for _, projection := range candidate.SourceProjections { @@ -620,11 +478,6 @@ func repairSignedExpandFields(base authoringv2.Construction, candidate authoring return candidate } -// validateExpandSourceColumns is the final candidate-prep gate: the selection's -// columnId must be the step's single Expand InputColumnID, and the (now -// server-signed) ExpandedField must be internally consistent and name exactly -// the resolved canonical array field with the Many wire cardinality, so a -// client-forged operation payload cannot survive this function. func validateExpandSourceColumns(steps []authoringv2.ConstructionStep, expandID string, sources []ConstructionExpandSourceSelection, resolved []authoringv2.ConstructionSourceProjection) error { index := constructionStepIndex(steps, expandID) if index < 0 || steps[index].Operation.Expand == nil { @@ -705,13 +558,6 @@ func clearConstructionExpandOwnedSources(base, candidate authoringv2.Constructio return candidate, nil } -// constructionOwnedExpandSourceColumnID returns the durable server-derived -// materialized column for a root array field at one exact stage, mirroring -// constructionOwnedUnpivotSourceColumnID. An EXPAND owns the slot only while an -// owner EXPAND step whose input stage is this exact step's input stage still -// declares the projection column as its single Expand InputColumnID. A -// vanished, re-pointed, or foreign-owned materialization returns "", so the UI -// never renders a restore into a column the server can no longer sign. func constructionOwnedExpandSourceColumnID(construction authoringv2.Construction, stageID, path string) string { for _, projection := range construction.SourceProjections { if projection.OwnerStepID == "" || canonicalRowPath(projection.FieldPath) != canonicalRowPath(path) { @@ -727,10 +573,6 @@ func constructionOwnedExpandSourceColumnID(construction authoringv2.Construction constructionStepInputStageID(step) != stageID { continue } - // The server only signs restoration of a live column while the owner - // step still carries its coherent signed payload for exactly this root - // field; a vanished, stripped, or forged payload returns "" like any - // other un-signable materialization. if !expandSourceStepPayloadIsConsistent(step, projection.FieldPath) { continue } @@ -739,12 +581,6 @@ func constructionOwnedExpandSourceColumnID(construction authoringv2.Construction return "" } -// ConstructionExpandSourceFacts derives the server-owned projection facts for -// one root primitive array field from the authorized snapshot plus generated -// FHIR metadata. Apply reauthorization and receipt reconcile share this single -// derivation so a persisted expand-owned claim can never drift from the facts -// the server would have signed it from. Column and owner identities stay -// claim-sourced, exactly as the Apply-side DeepEquals expects them. func ConstructionExpandSourceFacts(snapshot capability.Snapshot, rootResourceType, fieldPath string) (authoringv2.ConstructionSourceProjection, bool) { path := canonicalRowPath(fieldPath) if strings.TrimSpace(rootResourceType) == "" || path == "" { @@ -806,9 +642,6 @@ func isExpandOwnedSourceProjection(construction authoringv2.Construction, projec containsConstructionColumnIDLocal(expandInputColumnIDs(step.Operation.Expand), projection.ColumnID) } -// reauthorizeConstructionExpandSources re-derives the server-owned facts for -// every expand-owned projection from the live authorized snapshot so an Apply -// cannot stage a flatten whose array stopped qualifying after the proposal. func (s *Service) reauthorizeConstructionExpandSources(ctx context.Context, base constructionBase, outputID string, construction authoringv2.Construction) error { if s.config.RowChoicePlanner == nil { return unavailable("commands", "ROW_CHOICE_UNAVAILABLE", "authorized expand source choice planning is not configured", nil) @@ -830,9 +663,6 @@ func (s *Service) reauthorizeConstructionExpandSources(ctx context.Context, base if step.Operation.Expand == nil || !containsConstructionColumnIDLocal(expandInputColumnIDs(step.Operation.Expand), projection.ColumnID) { return fmt.Errorf("expand source projection %q is not the input column of its owner Expand step", projection.ColumnID) } - // Fail-closed on the operation payload itself: a lane-owned input whose - // step lost (or forged) its server-signed ExpandedField never reaches - // the claim-vs-redo DeepEqual below. signed := step.Operation.Expand.ExpandedField if signed == nil { return fmt.Errorf("expand source projection %q has no server-consistent Expand payload: missing signed array source", projection.ColumnID) diff --git a/internal/explorer/lifecycle/expand_source_test.go b/internal/explorer/lifecycle/expand_source_test.go index 1340d7c54..8c512f4ab 100644 --- a/internal/explorer/lifecycle/expand_source_test.go +++ b/internal/explorer/lifecycle/expand_source_test.go @@ -1,40 +1,5 @@ package lifecycle -// ROW-SOURCE-08 EXPAND source lane tests. Clones the Unpivot source lane test -// structure for the EXPAND lane: capability advertisement (exact six-field -// wire shape, skip-class reason codes, inventory fail-closed), request-boundary -// validation, candidate-prep refusals that fire before choice resolution, and -// the Apply-time reauthorization contract. -// -// Provenance: this file first shipped as a red deliverable whose acceptance -// tests pinned four product bugs (BUG-001..004). The three lane defects were -// repaired against these tests: -// -// BUG-001 (repaired) — the resolve/capability gates demanded a FieldGroupKey -// identity with Cardinality==Many && Shape==Array, but the mint gate -// (capability/row_choice.go validateRowChoiceFacts, re-run inside -// DecodeRowChoiceID) pins FieldGroupKey~SCALAR/ONE and ExpandedScope~ARRAY/ -// MANY as disjoint classes, and the planner mints arrays only as -// ExpandedScope (applicableRowChoiceKinds). The lane now enumerates, resolves, -// and reauthorizes the honest ExpandedScope class, so an advertised array is -// exactly an acceptable selection. -// -// BUG-002 (repaired) — authoringv2 validatePivotSourceProjection had no -// ConstructionOperationExpand owner case, so the EXPAND-owned -// SourceProjections entry the lane signs was rejected as INVALID_DOCUMENT on -// every validation path. The owner case now requires the projection to be the -// owner Expand step's declared array input. -// -// BUG-003 (repaired) — a lane-owned input with a stripped ExpandedField used -// to reach a nil-pointer dereference in Apply-time reauthorization; both -// payload consumers now refuse nil fail-closed with the contract error. -// -// BUG-004 (tracked) — the compiled root SOURCE_PROJECTION stage does not yet -// map the Many-cardinality array column the physical EXPAND gate consumes -// (lower/recipe_construction_reshape.go:469-478). The compile-package -// positive test pins the finished contract; root-stage mapping completion is -// tracked there. - import ( "context" "encoding/json" @@ -57,22 +22,12 @@ const ( expandTestOutput = "patients" ) -// expandTestPlanner hands back exactly the ExpandedScope array rows the real -// planner mints; the struct fields (not the tokens) are what the lane -// enumeration reads (expand_source.go). type expandTestPlanner struct{ choices []capability.RowChoice } func (p expandTestPlanner) ListRowChoices(context.Context, capability.Snapshot, authoringv2.Document) ([]capability.RowChoice, error) { return p.choices, nil } -// expandTestMintArrayChoice mints the only token shape that a legal ARRAY/MANY -// row can carry: capability.NewRowChoice accepts RowChoiceExpandedScope with -// ARRAY/MANY facts (capability/row_choice.go:123-126) and refuses -// FieldGroupKey for anything but SCALAR/ONE. The planner itself mints arrays -// only as ExpandedScope (applicableRowChoiceKinds), so this is the honest -// input for the resolve-path tests: exactly what the server can sign for an -// array root field. func expandTestMintArrayChoice(t *testing.T, snapshot capability.Snapshot, nodeID, resourceType, path, fhirType string) capability.RowChoice { t.Helper() occurrence := capability.RowChoiceOccurrence{OccurrenceID: authoringv2.RootOccurrenceID, NodeID: nodeID, ResourceType: resourceType} @@ -84,19 +39,12 @@ func expandTestMintArrayChoice(t *testing.T, snapshot capability.Snapshot, nodeI return choice } -// expandTestAdvertisedRow sets the display label on a minted ExpandedScope -// array row for the planner double. The lane enumerates exactly the -// ExpandedScope + Many + non-reference class (expand_source.go), matching -// what the real SchemaRowChoiceResolver mints for array root fields. func expandTestAdvertisedRow(choice capability.RowChoice, label string) capability.RowChoice { choice.Label = label return choice } func expandTestPatientArrayCandidate(id, path, logicalType, label string, populated bool) capability.Candidate { - // The capability builder mints repeated fields as many-cardinality with - // the ARRAY family of projection modes and a self-describing boundary; - // the lane's backing check requires exactly that honest shape. candidate := rowSourceLaneCandidate("node-patient", "Patient", id, path, logicalType, "many", populated) candidate.Label = label candidate.ProjectionModes = []capability.ProjectionMode{capability.ProjectionArray, capability.ProjectionFirst, capability.ProjectionDistinctArray} @@ -206,8 +154,6 @@ func expandTestPatientBase(snapshot capability.Snapshot, construction authoringv } } -// --- G1: stale stage reference --------------------------------------------- - func TestConstructionExpandSourceCapabilityReportsStaleStageReference(t *testing.T) { harness := newFilterSourceHarness(t) base := expandTestPatientBase(harness.snapshot, authoringv2.Construction{Version: authoringv2.ConstructionVersion}) @@ -220,8 +166,6 @@ func TestConstructionExpandSourceCapabilityReportsStaleStageReference(t *testing } } -// --- G2: advertisement and the pinned six-field wire shape ------------------ - func TestConstructionExpandSourceCapabilityAdvertisesPopulatedRootPrimitiveArray(t *testing.T) { harness := newFilterSourceHarness(t) snapshot := harness.snapshot @@ -242,8 +186,6 @@ func TestConstructionExpandSourceCapabilityAdvertisesPopulatedRootPrimitiveArray choice.LogicalType != "string" || !choice.IsPopulated || choice.ColumnID != "" { t.Fatalf("advertised choice = %#v, want candidate-sourced label/logical with no durable column", choice) } - // The UI capabilities schema is strict: exactly six wire fields, in the - // pinned spelling (expand_source.go:42-49), with no pivot-style extras. encoded, err := json.Marshal(choice) if err != nil { t.Fatal(err) @@ -254,8 +196,6 @@ func TestConstructionExpandSourceCapabilityAdvertisesPopulatedRootPrimitiveArray } } -// --- G3: durable columnId restore only while the owner still signs ---------- - func TestConstructionExpandSourceCapabilityAdvertisesDurableColumnIdOnlyWhileSigned(t *testing.T) { harness := newFilterSourceHarness(t) snapshot := harness.snapshot @@ -305,8 +245,6 @@ func TestConstructionExpandSourceCapabilityAdvertisesDurableColumnIdOnlyWhileSig } } -// --- G4: precise per-skipped-class lane reason codes ------------------------ - func TestConstructionExpandSourceCapabilityReportsPreciseSkipClassReasonCodes(t *testing.T) { const rootStage = recipe.ConstructionSourceProjectionID empty := authoringv2.Construction{Version: authoringv2.ConstructionVersion} @@ -346,7 +284,7 @@ func TestConstructionExpandSourceCapabilityReportsPreciseSkipClassReasonCodes(t service: nil, construction: empty, stageID: rootStage, choices: func(t *testing.T, s capability.Snapshot) []capability.RowChoice { row := photoRow(t, s) - row.FHIRType = "Coding" // expand_source.go:214 refuses object/Coding element FHIR types + row.FHIRType = "Coding" return []capability.RowChoice{row} }, wantCode: "NO_TYPE_ELIGIBLE_EXPAND_SOURCE_FIELDS", @@ -359,7 +297,7 @@ func TestConstructionExpandSourceCapabilityReportsPreciseSkipClassReasonCodes(t service: nil, construction: empty, stageID: rootStage, choices: func(t *testing.T, s capability.Snapshot) []capability.RowChoice { row := photoRow(t, s) - row.Cardinality = capability.RowChoiceOne // fails the Cardinality==MANY membership test at L205 + row.Cardinality = capability.RowChoiceOne return []capability.RowChoice{row} }, wantCode: "NO_ROUTE_ELIGIBLE_EXPAND_SOURCE_FIELDS", @@ -430,8 +368,6 @@ func TestConstructionExpandSourceCapabilityReportsPreciseSkipClassReasonCodes(t harness := newFilterSourceHarness(t) snapshot := harness.snapshot snapshot.Candidates = append(snapshot.Candidates, row.candidates...) - // Drop the fixture's scalars so they cannot skew the candidate - // verdicts; the lane decision sees only the array rows under test. snapshot.Candidates = row.candidates service := row.service if service == nil { @@ -468,8 +404,6 @@ func TestConstructionExpandSourceCapabilityReportsPreciseSkipClassReasonCodes(t } } -// --- G5: inventory errors stay distinguishable from empty data -------------- - func TestConstructionExpandSourceCapabilityNamesInventoryFailureDistinctly(t *testing.T) { harness := newFilterSourceHarness(t) snapshot := harness.snapshot @@ -485,8 +419,6 @@ func TestConstructionExpandSourceCapabilityNamesInventoryFailureDistinctly(t *te } } -// --- G6: request-boundary validation ---------------------------------------- - func TestConstructionExpandSourceSelectionsRejectMalformedInputAtRequestBoundary(t *testing.T) { base := func() ConstructionProposalRequest { return ConstructionProposalRequest{ @@ -521,8 +453,6 @@ func TestConstructionExpandSourceSelectionsRejectMalformedInputAtRequestBoundary } } -// --- G7-G10: candidate-prep refusals that fire before resolve --------------- - func TestConstructionExpandSourceCandidatePrepIgnoresChannelWithoutExpandChange(t *testing.T) { harness := newFilterSourceHarness(t) service := harness.service @@ -540,7 +470,6 @@ func TestConstructionExpandSourceCandidatePrepIgnoresChannelWithoutExpandChange( if !reflect.DeepEqual(unchanged, candidate) { t.Fatalf("non-expand candidate changed: %#v", unchanged) } - // A nil channel is likewise a no-op on an expand step. step := expandTestExpandStep(expandTestInput, expandTestCarrier) withExpand := authoringv2.Construction{Version: authoringv2.ConstructionVersion, Steps: []authoringv2.ConstructionStep{carrier, step}} untouched, err := service.constructionCandidateWithExpandSources(context.Background(), base, ConstructionProposalRequest{ @@ -557,7 +486,7 @@ func TestConstructionExpandSourceCandidatePrepIgnoresChannelWithoutExpandChange( func TestConstructionExpandSourceCandidatePrepRequiresExpandPayload(t *testing.T) { harness := newFilterSourceHarness(t) step := expandTestExpandStep(expandTestInput, expandTestCarrier) - step.Operation.Expand = nil // kind says Expand, payload absent + step.Operation.Expand = nil candidate := authoringv2.Construction{Version: authoringv2.ConstructionVersion, Steps: []authoringv2.ConstructionStep{step}} _, err := harness.service.constructionCandidateWithExpandSources(context.Background(), expandTestPatientBase(harness.snapshot, authoringv2.Construction{Version: authoringv2.ConstructionVersion}), @@ -629,7 +558,6 @@ func TestConstructionExpandSourceCandidatePrepRefusesColumnAndStageMisbinds(t *t t.Fatalf("foreign columnId error = %v, want EXPAND_SOURCE_COLUMN_MISMATCH", err) } - // No resolvable input stage: the expand step declares no inputs. orphan := step orphan.Inputs = nil orphanCandidate := authoringv2.Construction{Version: authoringv2.ConstructionVersion, Steps: []authoringv2.ConstructionStep{orphan}} @@ -639,7 +567,6 @@ func TestConstructionExpandSourceCandidatePrepRefusesColumnAndStageMisbinds(t *t t.Fatalf("unresolvable input stage error = %v, want EXPAND_SOURCE_STAGE_UNAVAILABLE", err) } - // Input stage bound to a stage absent from the accepted receipt. staleStageRequest := twoSelections() staleStageRequest.ExpandSources = []ConstructionExpandSourceSelection{{ChoiceID: "row-choice-a", ColumnID: expandTestInput}} carrierless := authoringv2.Construction{Version: authoringv2.ConstructionVersion, Steps: []authoringv2.ConstructionStep{expandTestExpandStep(expandTestInput, expandTestCarrier)}} @@ -650,8 +577,6 @@ func TestConstructionExpandSourceCandidatePrepRefusesColumnAndStageMisbinds(t *t } } -// --- G11-G12: forged or cross-snapshot tokens are refused ------------------- - func TestProposeConstructionExpandSourceRejectsForgedChoiceID(t *testing.T) { harness := newFilterSourceHarness(t) carrier := deriveSourceLiteralOnlyStep(expandTestCarrier, append(deriveSourcePatientIdentityColumns(), authoringv2.StageColumn{ID: "derived_total", Name: "derived_total", Label: "Derived total", Type: "integer"})) @@ -672,13 +597,6 @@ func TestProposeConstructionExpandSourceRejectsForgedChoiceID(t *testing.T) { } } -// Adversarial review finding (coordinator-reproduced): the candidate-side -// projection used to establish ownership, so a client editing the EXPAND step -// with a nil expandSources channel could author BOTH the expand-owned -// sourceProjection and a self-consistent signed payload and keep them through -// prep, prune, and document validation — a signed lane with no choiceId ever -// presented. Ownership of an expand input is now server-derived only: this -// shape must come out payload-stripped and projection-dropped. func TestProposeConstructionExpandSourceRefusesClientAuthoredLaneFacts(t *testing.T) { snapshot := expandTestProcedureSnapshot("tok") base := expandTestProcedureBase(snapshot, authoringv2.Construction{Version: authoringv2.ConstructionVersion}) @@ -696,14 +614,12 @@ func TestProposeConstructionExpandSourceRefusesClientAuthoredLaneFacts(t *testin Steps: []authoringv2.ConstructionStep{step}, SourceProjections: []authoringv2.ConstructionSourceProjection{clientProjection}, } - // nil planner: a genuine lane resolve would refuse outright, so anything - // surviving prep cannot have passed the mint. out, err := expandTestService(nil).constructionCandidateWithExpandSources(context.Background(), base, ConstructionProposalRequest{ Project: filterSourceProject, ExplorerID: "explorer-a", OutputID: "procedures", ChangedStepID: expandTestStep, CandidateConstruction: candidate, }, candidate) if err != nil { - return // a refusal is also a sound fail-closed outcome + return } if out.Steps[0].Operation.Expand.ExpandedField != nil { t.Fatalf("client-authored signed payload survived prep with no choiceId and no planner: %+v", *out.Steps[0].Operation.Expand.ExpandedField) @@ -720,8 +636,6 @@ func TestProposeConstructionExpandSourceRejectsChoiceFromAnotherSnapshot(t *test carrier := deriveSourceLiteralOnlyStep(expandTestCarrier, append(deriveSourcePatientIdentityColumns(), authoringv2.StageColumn{ID: "derived_total", Name: "derived_total", Label: "Derived total", Type: "integer"})) carrier.Operation.Derive.OutputColumnID = "derived_total" harness.saveDraft(t, authoringv2.Construction{Version: authoringv2.ConstructionVersion, Steps: []authoringv2.ConstructionStep{carrier}}) - // A genuinely signed ARRAY/MANY token, but issued for a different dataset - // generation than the proposal's snapshot. issued := rowSourceSnapshotWithGeneration(harness.snapshot, harness.snapshot.Identity.Generation+"-issued") stale := expandTestMintArrayChoice(t, issued, "node-patient", "Patient", "photo[]", "string") candidate := authoringv2.Construction{Version: authoringv2.ConstructionVersion, Steps: []authoringv2.ConstructionStep{ @@ -739,12 +653,8 @@ func TestProposeConstructionExpandSourceRejectsChoiceFromAnotherSnapshot(t *test } } -// --- G13: server-side fact derivation --------------------------------------- - func TestConstructionExpandSourceFactsDerivesServerSignedArrayProjection(t *testing.T) { snapshot := expandTestProcedureSnapshot("expand-facts-procedure") - // Procedure.instantiatesUri[] is a generated primitive ROOT ARRAY: - // canonical instantiatesUri[], ARRAY/MANY, FHIRType string, non-reference. facts, ok := ConstructionExpandSourceFacts(snapshot, "Procedure", "instantiatesUri[]") if !ok { t.Fatalf("facts for a populated generated primitive array were refused") @@ -753,8 +663,6 @@ func TestConstructionExpandSourceFactsDerivesServerSignedArrayProjection(t *test facts.FHIRType != "string" || facts.LogicalType != "string" || facts.Label != "Instantiates uri" { t.Fatalf("derived facts = %#v, want server-derived string-array facts with the candidate label", facts) } - // The column and owner identities are claim-sourced (left empty here), - // exactly as Apply-time DeepEquals expects. if facts.ColumnID != "" || facts.OwnerStepID != "" { t.Fatalf("facts leaked claim identities: %#v", facts) } @@ -799,8 +707,6 @@ func TestConstructionExpandSourceFactsDerivesServerSignedArrayProjection(t *test } } -// --- G14-G16: Apply-time reauthorization ------------------------------------ - func expandTestLiveProcedureConstruction(payload *authoringv2.ConstructionExpandedFieldSource) (authoringv2.Construction, capability.Snapshot) { snapshot := expandTestProcedureSnapshot("expand-reauth-procedure") step := expandTestExpandStep(expandTestInput, "") @@ -908,23 +814,6 @@ func TestReauthorizeConstructionExpandSourcesRefusesWithoutPlanner(t *testing.T) } } -// BUG-003 — expected failure: a lane-owned expand input whose step lost its -// server-signed payload must be refused fail-closed, never panic. -// expand_source.go:697-700 admits the nil payload because -// authoringv2.ValidateConstructionExpandedFieldSource explicitly treats nil as -// valid (construction.go:1093-1095 — nil is legal for client candidates), and -// line 721 then dereferences signed.Path unconditionally: -// -// panic: runtime error: invalid memory address or nil pointer dereference -// expand_source.go:721 inside reauthorizeConstructionExpandSources -// -// Every durable expand-owned projection therefore carries a crash path into -// ApplyCommands the moment its payload is stripped — exactly the state the -// comment at expand_source.go:693-696 promises to refuse ("a lane-owned input -// whose step lost (or forged) its server-signed ExpandedField never reaches -// the claim-vs-redo DeepEqual below"). The recover block below keeps the -// rest of the suite alive; the test fails today because the guard fires as a -// crash rather than as the promised refusal. func TestReauthorizeConstructionExpandSourcesKeepsFailClosedWhenPayloadStripped(t *testing.T) { construction, snapshot := expandTestLiveProcedureConstruction(expandTestSignedPayload("instantiatesUri[]", "string", "string")) construction.Steps[0].Operation.Expand.ExpandedField = nil @@ -946,24 +835,9 @@ func TestReauthorizeConstructionExpandSourcesKeepsFailClosedWhenPayloadStripped( } } -// --- Server-authoritative acceptance ---------------------------------------- -// -// These tests drive ProposeConstruction/ApplyCommands on a Procedure-rooted -// table. The generated FHIR schema carries no admissible primitive root array -// on the Patient fixture (photo[] is Attachment, meta.tag[]/security[] are -// Coding, meta.profile[] is a reference), so Procedure.instantiatesUri[] — -// the only canonical generated ARRAY/MANY non-reference primitive the lane -// can honestly sign — is the acceptance carrier, mirroring the compile -// package's EXPAND fixtures. - -// newExpandProcedureHarness reuses the live-compiler proposal harness around -// a Procedure-rooted saved table with the real planner/resolver on the -// generated schema. func newExpandProcedureHarness(t *testing.T) *filterSourceHarness { t.Helper() h := newFilterSourceHarness(t) - // Reuse the harness's accepted authorization scope so the procedure - // snapshot stays compatible with the configured read-scope check. scopeDigest := h.snapshot.Identity.AuthorizationScopeDigest shapeDigest := h.snapshot.Identity.ShapeDigest h.snapshot = expandTestProcedureSnapshot("expand-procedure-acceptance") @@ -1026,9 +900,6 @@ func expandTestProcedureCarrier() authoringv2.ConstructionStep { return carrier } -// expandTestProcedureExpandStep declares the full step schema the authoring -// contract requires: every carried input column except the expanded array -// itself, plus the item column. func expandTestProcedureExpandStep(previousStep string, carried []authoringv2.StageColumn) authoringv2.ConstructionStep { step := expandTestExpandStep(expandTestInput, previousStep) outputs := make([]authoringv2.StageColumn, 0, len(carried)+1) @@ -1043,14 +914,11 @@ func expandTestProcedureExpandStep(previousStep string, carried []authoringv2.St return step } -// Selecting the advertised array must sign the durable projection and -// overwrite any client-forged payload. func TestProposeConstructionExpandSourceSignsServerOwnedPayload(t *testing.T) { harness := newExpandProcedureHarness(t) carrier := expandTestProcedureCarrier() harness.saveDraft(t, authoringv2.Construction{Version: authoringv2.ConstructionVersion, Steps: []authoringv2.ConstructionStep{carrier}}) step := expandTestProcedureExpandStep(expandTestCarrier, carrier.Outputs) - // A client-forged payload must be overwritten by the server derivation. step.Operation.Expand.ExpandedField = expandTestSignedPayload("status", "string", "code") candidate := authoringv2.Construction{Version: authoringv2.ConstructionVersion, Steps: []authoringv2.ConstructionStep{carrier, step}} choice := expandTestMintArrayChoice(t, harness.snapshot, "procedure", "Procedure", "instantiatesUri[]", "string") @@ -1079,9 +947,6 @@ func TestProposeConstructionExpandSourceSignsServerOwnedPayload(t *testing.T) { } } -// The row-preserving prefix must gain the array column so the expand input -// stage can present it, and the expand step must not re-declare its own -// input (the lowerer refuses that). func TestProposeConstructionExpandSourceCarriesArrayThroughCarryingPrefix(t *testing.T) { harness := newExpandProcedureHarness(t) carrier := expandTestProcedureCarrier() @@ -1112,9 +977,6 @@ func TestProposeConstructionExpandSourceCarriesArrayThroughCarryingPrefix(t *tes t.Fatalf("carrier prefix outputs must include the carried array column %#v", carrierOutputs) } -// A row-changing (UNPIVOT) step earlier in the prefix must refuse to -// transport the array into the expand input -// (ConstructionCanCarrySourceProjection excludes UNPIVOT). func TestProposeConstructionExpandSourceRefusesCarryingThroughRowChangingPrefix(t *testing.T) { harness := newExpandProcedureHarness(t) melt := unpivotMeltStep("melt-prefix", []string{"procedure-id"}, unpivotMeltOutputs()) @@ -1136,8 +998,6 @@ func TestProposeConstructionExpandSourceRefusesCarryingThroughRowChangingPrefix( } } -// BUG-001 — expected failure: selecting a Coding-element array must be refused -// by name as an unsupported expand source field, not as an invalid choice. func TestProposeConstructionExpandSourceRefusesCodingElementArray(t *testing.T) { harness := newFilterSourceHarness(t) snapshot := harness.snapshot @@ -1149,9 +1009,6 @@ func TestProposeConstructionExpandSourceRefusesCodingElementArray(t *testing.T) candidate := authoringv2.Construction{Version: authoringv2.ConstructionVersion, Steps: []authoringv2.ConstructionStep{ carrier, expandTestExpandStep(expandTestInput, expandTestCarrier), }} - // meta.tag[] is a generated ARRAY/MANY non-reference element of FHIRType - // Coding: the lane's generated-schema gate must refuse it (its mint is - // legal — ARRAY/MANY — so reaching any other code hides the defect). choice := expandTestMintArrayChoice(t, harness.snapshot, "node-patient", "Patient", "meta.tag[]", "Coding") _, err := harness.service.ProposeConstruction(context.Background(), ConstructionProposalRequest{ Project: harness.store.created.Project, ExplorerID: harness.store.created.ExplorerID, SnapshotToken: harness.snapshot.Token, @@ -1165,12 +1022,6 @@ func TestProposeConstructionExpandSourceRefusesCodingElementArray(t *testing.T) } } -// --- Durable expand-owned state through Apply ------------------------------- - -// The accepted proposal's durable binding must be rechecked at Apply: when -// the signed array stops presenting observed values, Apply refuses with -// STALE_EXPAND_SOURCE_BINDING and leaves the draft untouched; while current, -// Apply saves the exact durable state the proposal carried. func TestApplyConstructionProposalRefusesStaleExpandSourceBinding(t *testing.T) { for _, stale := range []bool{false, true} { name := "current" @@ -1197,8 +1048,6 @@ func TestApplyConstructionProposalRefusesStaleExpandSourceBinding(t *testing.T) } version, digest := harness.store.created.DraftVersion, harness.store.created.DraftDigest if stale { - // The array stops presenting observed values after the - // proposal was accepted, exactly like the unpivot precedent. for index := range harness.snapshot.Candidates { if harness.snapshot.Candidates[index].FieldPath == "instantiatesUri[]" { harness.snapshot.Candidates[index].Populated = false diff --git a/internal/explorer/lifecycle/explicit_groups.go b/internal/explorer/lifecycle/explicit_groups.go index 15dd60116..4f58903c5 100644 --- a/internal/explorer/lifecycle/explicit_groups.go +++ b/internal/explorer/lifecycle/explicit_groups.go @@ -18,8 +18,6 @@ const ( explicitGroupMaxRelations = 100_000 ) -// ExplicitGroupInput is intentionally generic: members are opaque keys -// returned by the source selection API, not serialized resource references. type ExplicitGroupInput struct { ID string Label string @@ -52,9 +50,6 @@ type ExplicitGroupCreateResult struct { Groups []ExplicitGroupSummary } -// CreateExplicitGroupRevision creates one complete immutable relation set -// bound to the exact authorized selection. Opaque member keys are resolved -// against that selection before any staging write begins. func (s *Service) CreateExplicitGroupRevision(ctx context.Context, request ExplicitGroupCreateRequest) (result ExplicitGroupCreateResult, err error) { if s == nil || s.config.ExplicitGroupRepository == nil || s.config.Capability.ForCompilation == nil { return result, unavailable("explicit-groups", "AUTHORING_UNAVAILABLE", "explicit group authoring is not configured", nil) diff --git a/internal/explorer/lifecycle/filter_source.go b/internal/explorer/lifecycle/filter_source.go index 02e7c443c..404149f69 100644 --- a/internal/explorer/lifecycle/filter_source.go +++ b/internal/explorer/lifecycle/filter_source.go @@ -12,9 +12,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// ConstructionFilterSourceCapability advertises the authorized populated root -// scalar fields a FILTER step can consume through a signed source choice, -// beside the existing Pivot and Group source inputs. type ConstructionFilterSourceCapability struct { Supported bool `json:"supported"` StageID string `json:"stageId"` @@ -23,12 +20,6 @@ type ConstructionFilterSourceCapability struct { Choices []ConstructionFilterSourceChoice `json:"choices"` } -// ConstructionFilterSourceChoice is pinned to exactly six wire fields. The -// UI capabilities schema is strict, so pivot-style extras (occurrenceId, -// fhirType, valueType, isIdentifier, isReference) must never be added. -// ColumnID is the signed materialized column id from durable server-derived -// state, mirroring pivot's choice.columnId, so the UI can restore a saved -// source-backed filter; it is "" while no live materialization exists. type ConstructionFilterSourceChoice struct { ChoiceID string `json:"choiceId"` FieldPath string `json:"fieldPath"` @@ -38,8 +29,6 @@ type ConstructionFilterSourceChoice struct { ColumnID string `json:"columnId"` } -// ConstructionFilterSourceSelection binds one server-signed root scalar field -// choice to the exact columnId the edited FILTER step will consume. type ConstructionFilterSourceSelection struct { ChoiceID string `json:"choiceId"` ColumnID string `json:"columnId"` @@ -189,11 +178,6 @@ func (s *Service) resolveConstructionFilterSource( }, nil } -// constructionCandidateWithFilterSources resolves the signed field choices, -// derives the durable server-owned projection facts for the edited FILTER -// step, and carries those slots through the row-preserving prefix. It mirrors -// the Pivot source contract: the selection set is authoritative only when the -// changed step is the FILTER that consumes it. func (s *Service) constructionCandidateWithFilterSources( ctx context.Context, base constructionBase, @@ -361,14 +345,6 @@ func clearConstructionFilterOwnedSources(base, candidate authoringv2.Constructio return candidate, nil } -// constructionOwnedFilterSourceColumnID returns the durable server-derived -// materialized column for a root field at one exact stage, mirroring -// constructionOwnedRootProjectionColumnID. It only trusts state that is -// pinned to the current CAS draft and snapshot-bound receipt (enforced by -// loadConstructionBase): the projection must be owned by a FILTER step that -// consumes this exact stage and still filters on this exact column. A -// vanished, re-pointed, or foreign-owned materialization returns "", so the -// UI never renders a restore into a column the server can no longer sign. func constructionOwnedFilterSourceColumnID(construction authoringv2.Construction, stageID, path string) string { for _, projection := range construction.SourceProjections { if projection.OwnerStepID == "" || canonicalRowPath(projection.FieldPath) != canonicalRowPath(path) { @@ -388,12 +364,6 @@ func constructionOwnedFilterSourceColumnID(construction authoringv2.Construction return "" } -// ConstructionFilterSourceFacts derives the server-owned projection facts -// for one root field from the authorized snapshot plus generated FHIR -// metadata. Apply reauthorization and receipt reconcile share this single -// derivation so a persisted filter-owned claim can never drift from the -// facts the server would have signed it from. Column and owner identities -// stay claim-sourced, exactly as the Apply-side DeepEquals expects them. func ConstructionFilterSourceFacts(snapshot capability.Snapshot, rootResourceType, fieldPath string) (authoringv2.ConstructionSourceProjection, bool) { path := canonicalRowPath(fieldPath) if strings.TrimSpace(rootResourceType) == "" || path == "" { @@ -450,9 +420,6 @@ func isFilterOwnedSourceProjection(construction authoringv2.Construction, projec step.Operation.Filter.ColumnID == projection.ColumnID } -// reauthorizeConstructionFilterSources re-derives the server-owned facts for -// every filter-owned projection from the live authorized snapshot so an Apply -// cannot stage a filter whose field stopped qualifying after the proposal. func (s *Service) reauthorizeConstructionFilterSources(ctx context.Context, base constructionBase, outputID string, construction authoringv2.Construction) error { if s.config.RowChoicePlanner == nil { return unavailable("commands", "ROW_CHOICE_UNAVAILABLE", "authorized filter source choice planning is not configured", nil) diff --git a/internal/explorer/lifecycle/filter_source_pivot_clear_test.go b/internal/explorer/lifecycle/filter_source_pivot_clear_test.go index 37ea7a161..267c28df0 100644 --- a/internal/explorer/lifecycle/filter_source_pivot_clear_test.go +++ b/internal/explorer/lifecycle/filter_source_pivot_clear_test.go @@ -13,18 +13,6 @@ import ( "github.com/calypr/loom/internal/explorer/capability" ) -// ROW-SOURCE-05 durable regressions. -// -// ProposeConstruction composes the candidate through the Pivot lane first and -// the Filter lane second (construction_proposal.go). Both lanes preserved -// durable Pivot-owned inputs out of the accepted base, and both set branches -// bulk-copied every base projection except the edited owner's. The 01 wire -// contract ("nil never re-attaches; [] clears") makes the submitted candidate -// a full replacement construction: an owner whose complete base-owned chain -// is gone from the candidate is intentionally cleared, and no lane may -// re-import it from the base (constructionOwnerInputsClearedFromCandidate). -// A partial chain remains a client omission that preservation must repair. - const ( rs05PivotID = "pivot-one" rs05OtherPivotID = "pivot-two" @@ -112,14 +100,6 @@ func rs05CloneConstruction(c authoringv2.Construction) authoringv2.Construction return out } -// --------------------------------------------------------------------------- -// Lane-level: the Pivot clear must survive the Filter preparation even when -// the accepted base still holds the inputs (the not-yet-applied clear window). -// --------------------------------------------------------------------------- - -// Reproducer-equivalent from the ROW-SOURCE-05 contract: an edited Pivot that -// submits PivotSources: [] clears its owned private projections; the -// subsequent Filter preparation on that result must leave them cleared. func TestPivotSourceClearSurvivesFilterPreparation(t *testing.T) { snapshot := filterSourceSnapshot("pivot-clear-survives-filter") service := filterSourceService(t) @@ -157,9 +137,6 @@ func TestPivotSourceClearSurvivesFilterPreparation(t *testing.T) { } } -// The same sequence with a replacement selection: the pivot preparation swaps -// the owned projection, and the filter preparation must neither drop the -// replacement nor restore the replaced-away projection. func TestPivotSourceReplaceSurvivesFilterPreparation(t *testing.T) { snapshot := filterSourceSnapshot("pivot-replace-survives-filter") service := filterSourceService(t) @@ -203,7 +180,6 @@ func TestPivotSourceReplaceSurvivesFilterPreparation(t *testing.T) { } } -// A Pivot clear with no Filter preparation at all is unaffected. func TestPivotSourceClearWithoutFilterPreparationUnaffected(t *testing.T) { snapshot := filterSourceSnapshot("pivot-clear-alone") service := filterSourceService(t) @@ -233,11 +209,6 @@ func TestPivotSourceClearWithoutFilterPreparationUnaffected(t *testing.T) { } } -// --------------------------------------------------------------------------- -// Ports of the operator-accepted reproduction seeds -// (/tmp/opencode/xop-repro/internal/explorer/lifecycle/repro_filter_clears_pivot_test.go). -// --------------------------------------------------------------------------- - func rs05OtherHelperStep() authoringv2.ConstructionStep { return authoringv2.ConstructionStep{ ID: rs05Helper2ID, OwnerStepID: rs05OtherPivotID, @@ -255,7 +226,6 @@ func rs05OtherHelperStep() authoringv2.ConstructionStep { } } -// rs05HelperStep is the helper-form pivot chain used by the xop seeds. func rs05HelperStep() authoringv2.ConstructionStep { return authoringv2.ConstructionStep{ ID: rs05HelperID, OwnerStepID: rs05PivotID, @@ -349,10 +319,6 @@ func rs05AssertionPivotCleared(t *testing.T, name string, prepared authoringv2.C } } -// TestXopPortFilterPreparationRestoresClearedPivotInputs: after the Pivot -// source inputs were cleared (no helper, no projection), composing a FILTER -// candidate must not re-attach them from the durable base, both for a plain -// filter edit and for a filter preparation with a signed filter source. func TestXopPortFilterPreparationRestoresClearedPivotInputs(t *testing.T) { snapshot := filterSourceSnapshot("xop-port-filter-restores-pivot") service := filterSourceService(t) @@ -381,9 +347,6 @@ func TestXopPortFilterPreparationRestoresClearedPivotInputs(t *testing.T) { } } -// TestXopPortPivotLaneReaddsDuringFilterChange: the pivot lane, when NOT the -// applied operation (any non-Pivot edit runs it first with skipOwnerID=""), -// must already refuse to re-add the cleared pivot inputs. func TestXopPortPivotLaneReaddsDuringFilterChange(t *testing.T) { snapshot := filterSourceSnapshot("xop-port-pivot-lane-readd") service := filterSourceService(t) @@ -400,9 +363,6 @@ func TestXopPortPivotLaneReaddsDuringFilterChange(t *testing.T) { rs05AssertionPivotCleared(t, "pivot lane during filter change", prepared) } -// TestXopPortPerOwnerIsolationOnlyEditedOwnerPreserves: the durable -// re-attach must never resurrect the owner the candidate intentionally -// cleared, while foreign owners the candidate keeps must survive untouched. func TestXopPortPerOwnerIsolationOnlyEditedOwnerPreserves(t *testing.T) { snapshot := filterSourceSnapshot("xop-port-owner-isolation") service := filterSourceService(t) @@ -427,9 +387,6 @@ func TestXopPortPerOwnerIsolationOnlyEditedOwnerPreserves(t *testing.T) { rs05AssertionPivotCleared(t, "per-owner isolation", prepared) } -// TestXopPortControlDurableAlreadyClearedStaysCleared pins the trigger: -// restore only fires from the accepted base; once the durable is cleared, -// nothing is re-added and the lane's own signed materialization still works. func TestXopPortControlDurableAlreadyClearedStaysCleared(t *testing.T) { snapshot := filterSourceSnapshot("xop-port-cleared-durable") service := filterSourceService(t) @@ -450,9 +407,6 @@ func TestXopPortControlDurableAlreadyClearedStaysCleared(t *testing.T) { } } -// TestXopPortClearPivotPathEmptySliceStrips: the 01 contract holds on the -// pivot lane — an explicit empty (non-nil) PivotSources strips the helper -// and projection while a foreign filter-owned projection survives (scope). func TestXopPortClearPivotPathEmptySliceStrips(t *testing.T) { snapshot := filterSourceSnapshot("xop-port-clear-empty") service := filterSourceService(t) @@ -476,11 +430,6 @@ func TestXopPortClearPivotPathEmptySliceStrips(t *testing.T) { } } -// TestXopPortPivotSetBranchDoesNotReImportClearedFilterProjection pins the -// pivot lane SET branch (construction_pivot_source.go) foreign-owner bulk -// copy: a candidate that cleared the FILTER's durable input while the Pivot -// SET re-materializes its own must not have the filter-owned projection -// re-imported from the base. func TestXopPortPivotSetBranchDoesNotReImportClearedFilterProjection(t *testing.T) { snapshot := filterSourceSnapshot("xop-port-pivot-set-filter-clear") service := filterSourceService(t) @@ -519,10 +468,6 @@ func TestXopPortPivotSetBranchDoesNotReImportClearedFilterProjection(t *testing. } } -// TestXopPortFilterSetBranchDoesNotReImportClearedPivotProjection pins the -// filter lane SET branch (filter_source.go) foreign-owner bulk copy: a -// candidate that cleared another Pivot's durable input while the Filter SET -// materializes its own must not have the pivot-owned projection re-imported. func TestXopPortFilterSetBranchDoesNotReImportClearedPivotProjection(t *testing.T) { snapshot := filterSourceSnapshot("xop-port-filter-set-pivot-clear") service := filterSourceService(t) @@ -555,11 +500,6 @@ func TestXopPortFilterSetBranchDoesNotReImportClearedPivotProjection(t *testing. } } -// The mirror hazard direction: a candidate whose Filter-owned projection is -// already gone and whose second Pivot carried only part of its chain. Pivot -// preparation must still repair the partial client omission yet leave the -// fully cleared Filter input cleared, and the following Filter preparation -// must keep it that way. func TestFilterSourceClearNotRestoredByPivotPreparation(t *testing.T) { snapshot := filterSourceSnapshot("filter-clear-mirror") service := filterSourceService(t) @@ -610,16 +550,11 @@ func TestFilterSourceClearNotRestoredByPivotPreparation(t *testing.T) { } } -// Contract preservation: ordinary Filter source creation and restoration -// still materialize exactly the signed durable projection through the -// complete preparation sequence, alongside the other Pivot's carried chain. func TestFilterSourceCreateAndRestoreSurvivePivotPreparation(t *testing.T) { snapshot := filterSourceSnapshot("filter-create-and-restore-sequence") service := filterSourceService(t) selection := filterSourceSelection(t, snapshot, filterSourceField, filterSourceColumn) - // Create: the candidate carries the other Pivot's chain (full-replacement - // contract) and adds a new source-backed Filter ahead of it. baseConstruction := authoringv2.Construction{ Version: authoringv2.ConstructionVersion, Steps: []authoringv2.ConstructionStep{ @@ -642,8 +577,6 @@ func TestFilterSourceCreateAndRestoreSurvivePivotPreparation(t *testing.T) { t.Fatalf("creating a filter source dropped the carried pivot-two chain: %#v", created.SourceProjections) } - // Restore: reselecting a saved source-backed Filter keeps its one durable - // projection through both preparations. saved := rs05CloneConstruction(baseConstruction) restored := rs05RunBothPreparations(t, service, filterSourceBase(snapshot, saved), filterSourceProposalRequest(rs05FilterID, []ConstructionFilterSourceSelection{selection}), saved) @@ -677,12 +610,6 @@ func rs05RunBothPreparations(t *testing.T, service *Service, base constructionBa return prepared } -// --------------------------------------------------------------------------- -// FULL ProposeConstruction regressions (both ChangedStepID directions) over -// the pending-clear window: the accepted base still holds the inputs while -// the submitted full-replacement candidate has cleared them. -// --------------------------------------------------------------------------- - func rs05ProposeService(t *testing.T) (*Service, *fakeStore, capability.Snapshot) { t.Helper() service, store, snapshot := constructionProposalService(t) @@ -830,11 +757,6 @@ func rs05ProposeRequest(store *explorer.Explorer, snapshot capability.Snapshot, } } -// Direction 1 (ChangedStepID = FILTER): the pending user clear of the Pivot -// source inputs must survive the complete ProposeConstruction sequence; -// pre-fix the pivot lane (then the filter lane) re-added the inputs from the -// base, which the authoring validation rejected as a projection the filter -// no longer carries — the proposal could not even be previewed. func TestProposeConstructionFilterEditDoesNotRestoreClearedPivotSourceInput(t *testing.T) { service, store, snapshot := rs05ProposeService(t) durable := authoringv2.Construction{ @@ -874,9 +796,6 @@ func TestProposeConstructionFilterEditDoesNotRestoreClearedPivotSourceInput(t *t } } -// Direction 1 with a signed Filter source selection (the ROW-SOURCE-05 -// contract's own repro sequence): the filter lane's own materialization must -// still happen while the cleared Pivot input stays cleared. func TestProposeConstructionFilterSourceSetDoesNotRestoreClearedPivotSourceInput(t *testing.T) { service, store, snapshot := rs05ProposeService(t) durable := authoringv2.Construction{ @@ -912,9 +831,6 @@ func TestProposeConstructionFilterSourceSetDoesNotRestoreClearedPivotSourceInput } } -// Direction 2 (ChangedStepID = PIVOT): the owning lane's [] clear strips the -// Pivot inputs while a foreign filter-owned durable input must survive the -// same proposal (no over-clear) — the per-operator ownership acceptance. func TestProposeConstructionPivotClearKeepsForeignFilterSourceInput(t *testing.T) { service, store, snapshot := rs05ProposeService(t) durable := authoringv2.Construction{ @@ -953,9 +869,6 @@ func TestProposeConstructionPivotClearKeepsForeignFilterSourceInput(t *testing.T } } -// Control pin for the trigger: the restore only fires while the accepted -// base still holds the inputs. With the durable itself cleared, a filter -// edit stays clean and pre-views normally. func TestProposeConstructionFilterEditAgainstAlreadyClearedDurable(t *testing.T) { service, store, snapshot := rs05ProposeService(t) clearedDurable := authoringv2.Construction{ @@ -990,14 +903,6 @@ func rs05BaseWithHelperStage(base constructionBase) constructionBase { return base } -// --------------------------------------------------------------------------- -// b92548c7 third-lane coverage (b9-review Finding 1, mutE-proven): -// constructionCandidateWithDeriveSources shares preserveConstructionOwnedInputs -// with the two original lanes; the shared-helper repair must make the Derive -// lane inherit the candidate-cleared semantics, and the :383/:244 foreign-owner -// copies must honor a cleared DERIVE-owned durable input symmetrically. -// --------------------------------------------------------------------------- - const ( rs05DeriveID = "derive-total" rs05DeriveColumn = "derive-private-input" @@ -1038,11 +943,6 @@ func rs05DeriveRequest(changedStepID string) ConstructionProposalRequest { } } -// TestXopPortDerivePreparationDoesNotRestoreClearedPivotInput is the durable -// port of the b9 reviewer's mutE-proved Finding 1: with the filter lane fixed, -// the DERIVE preparation re-imported the intentionally cleared Pivot chain -// from the base (derive_source.go:230). The shared-helper predicate must close -// this third lane without editing the frozen derive_source.go. func TestXopPortDerivePreparationDoesNotRestoreClearedPivotInput(t *testing.T) { snapshot := filterSourceSnapshot("xop-port-derive-restores-pivot") service := filterSourceService(t) @@ -1063,10 +963,6 @@ func TestXopPortDerivePreparationDoesNotRestoreClearedPivotInput(t *testing.T) { } } -// TestXopPortPivotSetBranchDoesNotReImportClearedDeriveProjection is the -// mirror through the Pivot SET-branch foreign copy (:383): with the Pivot SET -// materializing its own signed selection, a fully client-cleared DERIVE-owned -// durable input must not be re-imported from the base. func TestXopPortPivotSetBranchDoesNotReImportClearedDeriveProjection(t *testing.T) { snapshot := filterSourceSnapshot("xop-port-pivot-set-derive-clear") service := filterSourceService(t) @@ -1110,8 +1006,6 @@ func TestXopPortPivotSetBranchDoesNotReImportClearedDeriveProjection(t *testing. } } -// TestXopPortFilterSetBranchDoesNotReImportClearedDeriveProjection is the -// mirror through the Filter SET-branch foreign copy (:244). func TestXopPortFilterSetBranchDoesNotReImportClearedDeriveProjection(t *testing.T) { snapshot := filterSourceSnapshot("xop-port-filter-set-derive-clear") service := filterSourceService(t) @@ -1167,10 +1061,6 @@ func rs05ProposeDeriveStep(literal int64) authoringv2.ConstructionStep { } } -// TestProposeConstructionDeriveEditDoesNotRestoreClearedPivotSourceInput runs -// the complete preparation sequence (pivot -> filter -> derive) with -// ChangedStepID = DERIVE and a client-cleared Pivot chain: the proposal must -// stay previewable with the cleared input still cleared. func TestProposeConstructionDeriveEditDoesNotRestoreClearedPivotSourceInput(t *testing.T) { service, store, snapshot := rs05ProposeService(t) durable := authoringv2.Construction{ @@ -1215,21 +1105,11 @@ func TestProposeConstructionDeriveEditDoesNotRestoreClearedPivotSourceInput(t *t } } -// --------------------------------------------------------------------------- -// Derive / Unpivot SET-branch foreign-owner bulk copy. The Pivot and Filter SET -// branches gate the base bulk copy on constructionOwnerInputsClearedFromCandidate -// (construction_pivot_source.go:582, filter_source.go:266-267); the Derive and -// Unpivot SET branches compose the same projection set (derive_source.go:322-325, -// unpivot_source.go:392-395) and must honour the identical set rule. -// --------------------------------------------------------------------------- - const ( rs05SetFilterID = "patient-filter-active" rs05SetFilterCol = "patient_filter_active" ) -// rs05PatientSetFilterStep is a foreign (FILTER-owned) accepted step carrying a -// server-owned signed projection, used as the owner another lane must not touch. func rs05PatientSetFilterStep(prevStepID string) authoringv2.ConstructionStep { return authoringv2.ConstructionStep{ ID: rs05SetFilterID, @@ -1252,17 +1132,11 @@ func rs05PatientSetFilterProjection() authoringv2.ConstructionSourceProjection { } } -// rs05PatientDerivedColumns is the SET-candidate derive step output set (signed -// operand slot plus the derived column) used by both SET-branch durables. func rs05PatientSetDerivedOutputs() []authoringv2.StageColumn { return append(deriveSourcePatientIdentityColumns(), deriveSourcePatientProjectionColumn(), authoringv2.StageColumn{ID: "derived_total", Name: "derived_total", Label: "Derived total", Type: "integer"}) } -// TestXopPortDeriveSetBranchDoesNotReImportClearedFilterProjection pins the -// Derive SET-branch foreign copy: a candidate that legitimately cleared a -// foreign FILTER-owned durable input while the Derive SET materializes its own -// signed operand must not have that input re-imported from the accepted base. func TestXopPortDeriveSetBranchDoesNotReImportClearedFilterProjection(t *testing.T) { harness := newDeriveSourceHarness(t) snapshot, service := harness.snapshot, harness.service @@ -1298,8 +1172,6 @@ func TestXopPortDeriveSetBranchDoesNotReImportClearedFilterProjection(t *testing } } -// TestXopPortUnpivotSetBranchDoesNotReImportClearedFilterProjection is the -// mirror through the Unpivot SET-branch foreign copy. func TestXopPortUnpivotSetBranchDoesNotReImportClearedFilterProjection(t *testing.T) { harness := newUnpivotSourceHarness(t) snapshot, service := harness.snapshot, harness.service @@ -1349,9 +1221,6 @@ func TestXopPortUnpivotSetBranchDoesNotReImportClearedFilterProjection(t *testin } } -// The second half of the same promise: a PARTIAL chain is still a client -// omission that preservation must repair, and a foreign owner the candidate -// keeps must survive untouched. These stay green across the gate fix. func TestXopPortDeriveSetBranchStillRepairsPartialPivotChain(t *testing.T) { harness := newDeriveSourceHarness(t) snapshot, service := harness.snapshot, harness.service diff --git a/internal/explorer/lifecycle/filter_source_test.go b/internal/explorer/lifecycle/filter_source_test.go index 3167acf40..391331c47 100644 --- a/internal/explorer/lifecycle/filter_source_test.go +++ b/internal/explorer/lifecycle/filter_source_test.go @@ -51,8 +51,6 @@ func filterSourceSnapshot(token string) capability.Snapshot { ) } -// filterSourceDocument keeps only the row identity selected, so every other -// eligible root scalar must reach a filter through a signed source choice. func filterSourceDocument() authoringv2.Document { return authoringv2.Document{ Kind: authoringv2.Kind, @@ -143,8 +141,6 @@ func filterSourceGroupStep() authoringv2.ConstructionStep { } } -// filterSourceRowChoice returns the signed choice the planner issues for one -// exact root field so a test can hold a legitimate, current token. func filterSourceRowChoice(t *testing.T, snapshot capability.Snapshot, path string) capability.RowChoice { t.Helper() choices, err := filterSourceResolver(t).ListRowChoices(context.Background(), snapshot, filterSourceDocument()) @@ -173,8 +169,6 @@ func filterSourceProposalRequest(changedStepID string, sources []ConstructionFil } } -// (a) Discovery exposes the populated unselected root scalar with a signed -// choice id and never advertises a schema-only field. func TestConstructionFilterSourceCapabilityExposesPopulatedUnselectedRootScalar(t *testing.T) { snapshot := filterSourceSnapshot("filter-source-discovery") result, err := filterSourceService(t).constructionFilterSourceCapability(context.Background(), @@ -224,10 +218,6 @@ func TestConstructionFilterSourceCapabilityExposesPopulatedUnselectedRootScalar( } } -// ROW-SOURCE-03 S-C: the single display-label builder must surface the exact -// schema Title on discovery and write the identical string into the durable -// claim, so the advertised label and the server-derived projection for one -// field are the same bytes (the compiled artifact copies those bytes). func TestConstructionFilterSourceTitleLabelIsSharedByDiscoveryAndDurableClaim(t *testing.T) { snapshot := filterSourceSnapshot("filter-source-title-parity") service := filterSourceService(t) @@ -270,10 +260,6 @@ func TestConstructionFilterSourceTitleLabelIsSharedByDiscoveryAndDurableClaim(t } } -// Candidate-label fallback shape (rs03-label-owner.md pinned decision): the -// capability snapshot builder collapses an empty schema Title to the bare -// field path (builder.go:296 first(TrimSpace(f.Label), path)), so the lane -// label for such a field is the raw path — not the legacy 'Type.path'. func TestConstructionFilterSourceEmptyTitleCandidateLabelIsRawPath(t *testing.T) { snapshot := filterSourceSnapshot("filter-source-empty-title") for index := range snapshot.Candidates { @@ -313,9 +299,6 @@ func TestConstructionFilterSourceEmptyTitleCandidateLabelIsRawPath(t *testing.T) } } -// labeledRowSourceChoicePlanner rewrites the planner's display label after -// signing, so a test can pin the builder chain when the snapshot candidate -// carries no label at all. Signing stays with the real resolver. type labeledRowSourceChoicePlanner struct { inner *SchemaRowChoiceResolver label string @@ -332,10 +315,6 @@ func (p labeledRowSourceChoicePlanner) ListRowChoices(ctx context.Context, snaps return choices, nil } -// With no snapshot candidate label, advertise falls through the builder -// chain to the row choice display label, and to the exact legacy -// 'ResourceType.path' when that is blank too; the durable claim never sees -// the choice label and lands on 'ResourceType.path'. func TestConstructionFilterSourceBlankCandidateLabelFallsThroughBuilderChain(t *testing.T) { snapshot := filterSourceSnapshot("filter-source-blank-label") for index := range snapshot.Candidates { @@ -377,8 +356,6 @@ func TestConstructionFilterSourceBlankCandidateLabelFallsThroughBuilderChain(t * if projection.Label != "BodyStructure.active" { t.Fatalf("durable label = %q, want the exact legacy 'Type.path' fallback", projection.Label) } - // ConstructionFilterSourceFacts is the reconcile-side twin of the durable - // write; it must produce the identical string for the identical claim. facts, ok := ConstructionFilterSourceFacts(snapshot, filterSourceRootType, filterSourceField) if !ok || facts.Label != projection.Label { t.Fatalf("facts label = %q (ok %v), want the durable label %q", facts.Label, ok, projection.Label) @@ -407,7 +384,6 @@ func TestConstructionFilterSourceCapabilityRefusesStaleStageAndRowChangingPrefix } } -// (d) The observed/populated gate is enforced on admission, not only display. func TestConstructionFilterSourceRejectsUnpopulatedFieldOnDiscoveryAndAdmission(t *testing.T) { snapshot := filterSourceSnapshot("filter-source-unpopulated") service := filterSourceService(t) @@ -421,8 +397,6 @@ func TestConstructionFilterSourceRejectsUnpopulatedFieldOnDiscoveryAndAdmission( t.Fatalf("unpopulated field offered: %#v", choice) } } - // The planner still issues a signed token for the schema-only field, so the - // proposal boundary must refuse it rather than trust the picker. unpopulated := filterSourceRowChoice(t, snapshot, "description") if _, err := capability.DecodeRowChoiceID(unpopulated.ChoiceID); err != nil { t.Fatalf("crafted schema-only candidate produced no signed token: %v", err) @@ -439,7 +413,6 @@ func TestConstructionFilterSourceRejectsUnpopulatedFieldOnDiscoveryAndAdmission( } } -// (c) A token from another snapshot, or a tampered token, is refused. func TestConstructionFilterSourceRefusesForgedAndCrossSnapshotChoices(t *testing.T) { snapshot := filterSourceSnapshot("filter-source-current") other := filterSourceSnapshot("filter-source-other") @@ -473,8 +446,6 @@ func TestConstructionFilterSourceRefusesForgedAndCrossSnapshotChoices(t *testing } } -// (b) A valid selection compiles the filter and durably owns exactly one -// server-derived projection for the unselected field. func TestConstructionCandidateWithFilterSourcesMaterializesOwnedProjection(t *testing.T) { snapshot := filterSourceSnapshot("filter-source-apply") service := filterSourceService(t) @@ -522,9 +493,6 @@ func TestConstructionCandidateWithFilterSourcesMaterializesOwnedProjection(t *te if len(analyzed.Construction.SourceProjections) != 1 || analyzed.Construction.SourceProjections[0].OwnerStepID != "filter-active" { t.Fatalf("analysis dropped or re-owned the filter projection: %#v", analyzed.Construction.SourceProjections) } - // Independently derived expectation: the surviving rows still expose just the - // selected identity, and the filtered-on field is carried as an input slot, - // so the output schema still equals the input schema. wantIDs := []string{filterSourceIdentity, filterSourceColumn} gotIDs := make([]string, 0, len(analyzed.Construction.Steps[0].Outputs)) for _, output := range analyzed.Construction.Steps[0].Outputs { @@ -552,8 +520,6 @@ func TestConstructionFilterSourceRequiresMatchingColumnAndIgnoresNonFilterSteps( t.Fatalf("mismatched column error = %v, want FILTER_SOURCE_COLUMN_MISMATCH", err) } - // Mirroring Pivot: filterSources that do not belong to the edited FILTER - // step leave the construction untouched. unedited, err := service.constructionCandidateWithFilterSources(context.Background(), base, filterSourceProposalRequest("missing-step", []ConstructionFilterSourceSelection{filterSourceSelection(t, snapshot, filterSourceField, filterSourceColumn)}), construction) if err != nil { @@ -564,7 +530,6 @@ func TestConstructionFilterSourceRequiresMatchingColumnAndIgnoresNonFilterSteps( } } -// The row-preserving prefix allow-list is enforced, not trust-me coded. func TestConstructionFilterSourceCannotCrossRowChangingPrefix(t *testing.T) { snapshot := filterSourceSnapshot("filter-source-prefix") service := filterSourceService(t) @@ -596,8 +561,6 @@ func TestConstructionFilterSourceCannotCrossRowChangingPrefix(t *testing.T) { } } -// A projection the filter does not own is refused, and so is one whose owner -// filter names a different column. func TestConstructionFilterSourceProjectionMustMatchItsFilterColumn(t *testing.T) { document := filterSourceDocument() @@ -634,8 +597,6 @@ func TestConstructionFilterSourceProjectionMustMatchItsFilterColumn(t *testing.T } } -// (e) Dropping the selection strips its projection, and a filter left pointing -// at the removed source fails instead of silently keeping it. func TestConstructionFilterSourceRemovalStripsProjectionAndRefusesDanglingFilter(t *testing.T) { snapshot := filterSourceSnapshot("filter-source-removal") service := filterSourceService(t) @@ -685,7 +646,6 @@ func TestConstructionFilterSourceRemovalStripsProjectionAndRefusesDanglingFilter } } -// (f) Selected-column filter behavior is unchanged by filter source support. func TestConstructionFilterSourceLeavesSelectedColumnFilterBehaviorUnchanged(t *testing.T) { snapshot := filterSourceSnapshot("filter-source-regression") service := filterSourceService(t) @@ -707,7 +667,6 @@ func TestConstructionFilterSourceLeavesSelectedColumnFilterBehaviorUnchanged(t * } } -// Duplicate or blank selection facts are refused at the request boundary. func TestConstructionFilterSourceSelectionsRequireExactDeduplicatedFacts(t *testing.T) { duplicate := filterSourceProposalRequest("filter-active", []ConstructionFilterSourceSelection{ {ChoiceID: "rc1.a.b", ColumnID: "c1"}, {ChoiceID: "rc1.a.b", ColumnID: "c2"}, @@ -732,10 +691,6 @@ func TestConstructionFilterSourceSelectionsRequireExactDeduplicatedFacts(t *test } } -// (g) The advertised choice object is EXACTLY the six pinned wire fields. Any -// pivot-style extra field (occurrenceId/fhirType/valueType/isIdentifier/ -// isReference) makes the strict UI schema reject the whole capabilities -// response, so the shape is pinned structurally and on the wire. func TestConstructionFilterSourceChoiceWireShapeIsExactlySixFields(t *testing.T) { wantFields := map[string]bool{"choiceId": true, "fieldPath": true, "label": true, "logicalType": true, "isPopulated": true, "columnId": true} structFields := make(map[string]bool, reflect.TypeOf(ConstructionFilterSourceChoice{}).NumField()) @@ -771,10 +726,6 @@ func TestConstructionFilterSourceChoiceWireShapeIsExactlySixFields(t *testing.T) } } -// The advertised columnId is the signed materialized column the UI uses to -// restore a saved source-backed filter. It must come only from durable -// server-derived state that is still live for exactly this stage: a vanished, -// re-pointed, or foreign-owned materialization is advertised as absent. func TestConstructionFilterSourceCapabilityAdvertisesDurableColumnIdOnlyWhileSigned(t *testing.T) { snapshot := filterSourceSnapshot("filter-source-column-binding") service := filterSourceService(t) @@ -846,8 +797,6 @@ func TestConstructionFilterSourceCapabilityAdvertisesDurableColumnIdOnlyWhileSig } } -// (a, wiring) GetConstructionCapabilities assembles filterSourceInput beside -// pivotSourceInput on the owned-style fixture. func TestGetConstructionCapabilitiesAssemblesFilterSourceInputBesidePivotSourceInput(t *testing.T) { harness := newFilterSourceHarness(t) capabilities, err := harness.service.GetConstructionCapabilities(context.Background(), ConstructionCapabilitiesRequest{ @@ -895,8 +844,6 @@ func TestGetConstructionCapabilitiesAssemblesFilterSourceInputBesidePivotSourceI } } -// (b) Through ProposeConstruction with the real compiler, the accepted -// candidate filters on the unselected field's compiled slot. func TestProposeConstructionFilterSourceFiltersUnselectedFieldWithRealCompiler(t *testing.T) { harness := newFilterSourceHarness(t) harness.saveDraft(t, authoringv2.Construction{Version: authoringv2.ConstructionVersion, Steps: []authoringv2.ConstructionStep{ @@ -966,7 +913,6 @@ func TestProposeConstructionFilterSourceFiltersUnselectedFieldWithRealCompiler(t } } -// (6) An accepted proposal whose field stops qualifying is refused at Apply. func TestApplyConstructionProposalRefusesStaleFilterSourceBinding(t *testing.T) { for _, stale := range []bool{false, true} { name := "current" @@ -1043,9 +989,6 @@ func TestApplyConstructionProposalRefusesStaleFilterSourceBinding(t *testing.T) } } -// A restore of a saved source-backed filter whose field stopped presenting -// data must fail closed at proposal time like a forged token: the durable -// columnId alone never vouches for the vanished field. func TestProposeConstructionFilterSourceRefusesVanishedFieldRestore(t *testing.T) { harness := newFilterSourceHarness(t) durable := authoringv2.Construction{ @@ -1120,8 +1063,6 @@ func filterSourcePatientFilterStep(columnID string) authoringv2.ConstructionStep } } -// filterSourceHarness wires the real workspace compiler/lowering path and the -// real schema-backed row-choice planner around the patients fixture. type filterSourceHarness struct { service *Service store *fakeStore @@ -1214,8 +1155,6 @@ func newFilterSourceHarness(t *testing.T) *filterSourceHarness { return harness } -// saveDraft stores an owned-style construction whose only selected column is -// the row identity. func (h *filterSourceHarness) saveDraft(t *testing.T, construction authoringv2.Construction) { t.Helper() workspace, err := authoringv2.DecodeWorkspace(h.store.created.DraftConfig) @@ -1277,8 +1216,6 @@ func (h *filterSourceHarness) choice(t *testing.T, path string) capability.RowCh return capability.RowChoice{} } -// unobserveBirthChange makes the selected field stop presenting observed data -// while keeping the same snapshot identity, mimicking data going empty. func (h *filterSourceHarness) unobserveBirthCount() { for index := range h.snapshot.Candidates { if h.snapshot.Candidates[index].FieldPath == "multipleBirthInteger" { @@ -1287,12 +1224,6 @@ func (h *filterSourceHarness) unobserveBirthCount() { } } -// --- ROW-SOURCE-03 S-A: precise per-skipped-class lane reason codes ------- - -// staleRowSourceChoicePlanner signs every enumerated choice against a -// different snapshot, mimicking a planner hand-off race between two dataset -// versions. The real SchemaRowChoiceResolver always pins the passed snapshot, -// so this only exists to exercise the version skip class. type staleRowSourceChoicePlanner struct { inner *SchemaRowChoiceResolver issued capability.Snapshot @@ -1454,11 +1385,6 @@ func TestConstructionFilterSourceCapabilityReportsPreciseSkipClassReasonCodes(t } } -// --- ROW-SOURCE-03 S-B: inventory-error abort verdicts ---------------------- - -// failingRowSourcePlanner aborts enumeration the way an unreachable row-choice -// inventory does; it is the injection point for the S-B verdict that must stay -// distinguishable from the healthy empty-data codes. type failingRowSourcePlanner struct{ err error } func (p failingRowSourcePlanner) ListRowChoices(context.Context, capability.Snapshot, authoringv2.Document) ([]capability.RowChoice, error) { diff --git a/internal/explorer/lifecycle/frame_source.go b/internal/explorer/lifecycle/frame_source.go index bcd4be48b..5dd303f04 100644 --- a/internal/explorer/lifecycle/frame_source.go +++ b/internal/explorer/lifecycle/frame_source.go @@ -492,9 +492,6 @@ func frameSourceFamilies(snapshot capability.Snapshot, entries []catalog.Semanti for _, entry := range entries { observation := entry.Observation if observation.RuleHint != catalog.SemanticRuleHintCodedValueV1 { - // Frames represent an existing paired code/value binding. A standalone - // Coding's value lives on the matched Coding item, so it cannot be - // represented by the owner-scoped frame family contract. continue } plan := authoringv2.ResolveSemanticSelectionPlan(observation) diff --git a/internal/explorer/lifecycle/fs1_reverse_order_collision_test.go b/internal/explorer/lifecycle/fs1_reverse_order_collision_test.go index eb4a5080c..2534951e4 100644 --- a/internal/explorer/lifecycle/fs1_reverse_order_collision_test.go +++ b/internal/explorer/lifecycle/fs1_reverse_order_collision_test.go @@ -1,29 +1,5 @@ package lifecycle -// fs1_reverse_order_collision_test.go — BS-62-6 durable locks for the F-DS1 -// signed-slot collision boundary under arrival-order reversal: -// -// - The shared collision predicate (constructionRootFieldAlreadyProjected, -// construction_pivot_source.go:658) compares SETS of signed-slot vs -// committed field identities and never the step/projection order: a -// carrier-owned collision is refused with its named *_FIELD_ALREADY_ -// PROJECTED code in both forward and reverse arrival order. -// - The pivot-first ("reverse") direction disclosed by the 62f627fb commit -// body ("reverse-order pivot collision surfaces as -// FILTER_SOURCE_CANNOT_CROSS_STAGE (residual disclosed, not silently -// fixed)") is refused at the lane boundary for every signed lane — -// never a silent accept and never the catch-all -// CONSTRUCTION_RECEIPT_CONTRACT_INVALID 500. -// - The inverse arrangement (signed step inserted before the committed -// pivot) cannot be submitted at all: the step-order preservation guard -// refuses it as INVALID_CONSTRUCTION_CANDIDATE. -// - Pivot-owner exclusion from the carrier set is a shipped contract (the -// predicate comment: "a slot sharing its field with a Pivot-owned source -// input is legitimate state"), so the predicate itself is pinned false -// for pivot-owned fields in BOTH directions; refusal is pinned to the -// boundary/guard layers above. -// - Legit non-collision arrivals (distinct fields) keep flowing through. - import ( "context" "errors" @@ -51,7 +27,6 @@ func fs1r6Projection(columnID, ownerStepID, fieldPath, logicalType string) autho } } -// P1 — predicate order invariance table at the root function. func TestFS1SignedSlotCollisionPredicateIsOrderInvariant(t *testing.T) { doc := filterSourceDocument() step := func(id string, kind authoringv2.ConstructionOperationKind) authoringv2.ConstructionStep { @@ -94,8 +69,6 @@ func TestFS1SignedSlotCollisionPredicateIsOrderInvariant(t *testing.T) { } } -// P2/P6 — the disclosed reverse-direction (pivot-first) arrival is refused at -// the lane boundary of both signed lanes with the named cross-stage codes. func TestFS1ReverseOrderSignedSlotRefusedAtPivotBoundary(t *testing.T) { buildBase := func(t *testing.T, token string) (*Service, constructionBase, capability.Snapshot) { t.Helper() @@ -171,8 +144,6 @@ func TestFS1ReverseOrderSignedSlotRefusedAtPivotBoundary(t *testing.T) { }) } -// P4 — the named collision code fires from the predicate in BOTH arrival -// directions when the committed carrier is a signed-lane projection. func TestFS1ForwardAndReverseSignedCarrierCollisionRefusedNamed(t *testing.T) { for _, row := range []struct { order string @@ -213,8 +184,6 @@ func TestFS1ForwardAndReverseSignedCarrierCollisionRefusedNamed(t *testing.T) { } } -// P5 — legit non-collision arrivals keep flowing through the lane in either -// order. func TestFS1LegitNonCollisionReverseOrderStillAccepted(t *testing.T) { snapshot := filterSourceSnapshot("fs1r6-legit") service := filterSourceService(t) @@ -239,8 +208,6 @@ func TestFS1LegitNonCollisionReverseOrderStillAccepted(t *testing.T) { } } -// --- full ProposeConstruction locks (real compiler, Patients harness) --- - func fs1r6PivotPatientConstruction() (authoringv2.Document, authoringv2.Construction) { construction := authoringv2.Construction{Version: authoringv2.ConstructionVersion, Steps: []authoringv2.ConstructionStep{ { @@ -324,10 +291,6 @@ func fs1r6SignedSlotRequest() ConstructionFilterSourceSelection { return ConstructionFilterSourceSelection{ColumnID: "fs1r6_slot_b"} } -// The inverse arrangement never reaches the collision predicate at all: -// inserting a signed step before the committed pivot is refused by the step -// order preservation guard, so no proposal can carry a signed slot in -// reverse order of the committed field. func TestFS1FullProposeRefusesSignedStepInsertedBeforeCommittedPivot(t *testing.T) { harness := newFilterSourceHarness(t) _, durable := fs1r6PivotPatientConstruction() @@ -347,9 +310,6 @@ func TestFS1FullProposeRefusesSignedStepInsertedBeforeCommittedPivot(t *testing. } } -// The recorded reverse direction (pivot committed first, signed slot added -// after) is refused end to end with the named boundary code — never -// PREVIEW_PENDING with two same-path projections, never the catch-all 500. func TestFS1FullProposeReverseDirectionRefusedAtBoundary(t *testing.T) { harness := newFilterSourceHarness(t) _, durable := fs1r6PivotPatientConstruction() diff --git a/internal/explorer/lifecycle/interpretation.go b/internal/explorer/lifecycle/interpretation.go index eae783cda..5f99145d1 100644 --- a/internal/explorer/lifecycle/interpretation.go +++ b/internal/explorer/lifecycle/interpretation.go @@ -12,9 +12,6 @@ import ( "github.com/calypr/loom/internal/projectid" ) -// resolveWorkspaceInterpretations loads each distinct exact revision once and -// then delegates all matching and rule selection to the pure compiler -// resolver. No library head is consulted. func (s *Service) resolveWorkspaceInterpretations(ctx context.Context, project string, workspace authoringv2.Workspace, snapshot capability.Snapshot) (explorercompilation.ResolvedInputs, error) { ids := make(map[explorer.InterpretationRevisionID]struct{}) for _, document := range workspace.Documents { diff --git a/internal/explorer/lifecycle/interpretation_apply.go b/internal/explorer/lifecycle/interpretation_apply.go index aaf9fa77c..cfae753d4 100644 --- a/internal/explorer/lifecycle/interpretation_apply.go +++ b/internal/explorer/lifecycle/interpretation_apply.go @@ -10,10 +10,6 @@ import ( "github.com/calypr/loom/internal/projectid" ) -// validateInterpretationCandidateCommands is the proof boundary for the one -// receipt-backed mutation. The reducer has already produced the proposed -// workspace, so a successful return means SaveDraft is the only remaining -// state change in ApplyWorkspaceCommandsChecked. func (s *Service) validateInterpretationCandidateCommands(ctx context.Context, project, explorerID string, request authoringv2.ApplyCommandsRequest, proposed authoringv2.Workspace, snapshot capability.Snapshot) error { candidateCount := 0 for _, command := range request.Commands { diff --git a/internal/explorer/lifecycle/interpretation_library.go b/internal/explorer/lifecycle/interpretation_library.go index e5effe37b..a4a3aa6b9 100644 --- a/internal/explorer/lifecycle/interpretation_library.go +++ b/internal/explorer/lifecycle/interpretation_library.go @@ -17,9 +17,6 @@ func (s *Service) interpretationRepository() (explorer.InterpretationRepository, return s.config.InterpretationRepository, nil } -// ListInterpretationLibraries returns project-scoped libraries with their exact -// current head revision expanded. The head is a browsing affordance only; no -// compiler or workspace resolver consumes it as a mutable reference. func (s *Service) ListInterpretationLibraries(ctx context.Context, rawProject string) (ListInterpretationLibrariesResult, error) { repository, err := s.interpretationRepository() if err != nil { @@ -80,10 +77,6 @@ func (s *Service) GetInterpretationRevision(ctx context.Context, request GetInte return *revision, nil } -// CreateInterpretationRevision prepares closed typed content and advances the -// repository head only if its expected parent still matches. A missing parent -// request explicitly means a root revision; callers revising an existing -// library must send its exact current head as the parent. func (s *Service) CreateInterpretationRevision(ctx context.Context, request CreateInterpretationRevisionRequest) (explorer.InterpretationRevision, error) { repository, err := s.interpretationRepository() if err != nil { diff --git a/internal/explorer/lifecycle/interpretation_preview.go b/internal/explorer/lifecycle/interpretation_preview.go index 66e7a67a5..dd698f6c1 100644 --- a/internal/explorer/lifecycle/interpretation_preview.go +++ b/internal/explorer/lifecycle/interpretation_preview.go @@ -14,10 +14,6 @@ import ( "github.com/calypr/loom/internal/projectid" ) -// PreviewInterpretationCandidate compiles the current draft and an immutable -// candidate workspace, then executes both receipts with one authorization -// binding. It deliberately has no persistence side effect other than the -// immutable compilation receipts returned by the configured compiler. func (s *Service) PreviewInterpretationCandidate(ctx context.Context, request PreviewInterpretationCandidateRequest) (PreviewInterpretationCandidateResult, error) { if err := validateInterpretationCandidatePreviewRequest(request); err != nil { return PreviewInterpretationCandidateResult{}, malformed("interpretation-preview", err.Error(), err) @@ -177,9 +173,6 @@ func (s *Service) previewInterpretationReceipt(ctx context.Context, receipt *exp } copy := make(map[string]any, len(row)) for key, item := range row { - // Preview rows are already transport-safe values. Keep their native - // numeric representation instead of a JSON round trip that could - // turn int64/uint64 values into float64. copy[key] = item } rows.Rows[rowID] = copy @@ -197,8 +190,6 @@ func authoredInterpretationEmissions(receipt *explorer.CompilationReceipt, outpu } emissions := make([]explorer.EmittedColumn, 0) for _, emission := range receipt.EmittedColumns { - // Preview rows expose public column names. An internal emission ID is - // not a row key and must never leak into the comparison sample. if emission.OutputID != outputID || emission.PublicColumn == "" || !containsString(emission.AuthoredColumns, column) { continue } diff --git a/internal/explorer/lifecycle/policy.go b/internal/explorer/lifecycle/policy.go index 505183260..6c362bac7 100644 --- a/internal/explorer/lifecycle/policy.go +++ b/internal/explorer/lifecycle/policy.go @@ -190,7 +190,6 @@ func validateAuthorizedReadScope(scope authscope.ReadScope, expectedDigest strin return fmt.Errorf("unrestricted scope must not carry resource paths") } case authscope.ReadScopeRestricted: - // Empty restricted scopes intentionally remain deny-all. default: return fmt.Errorf("authorization scope mode is missing or invalid") } diff --git a/internal/explorer/lifecycle/population.go b/internal/explorer/lifecycle/population.go index 24c5a0389..983643952 100644 --- a/internal/explorer/lifecycle/population.go +++ b/internal/explorer/lifecycle/population.go @@ -11,9 +11,6 @@ import ( "github.com/calypr/loom/internal/projectid" ) -// resolveWorkspacePopulations verifies every durable population against the -// exact capability snapshot and returns the bounded identity needed by the -// receipt compiler. Selection member rows are intentionally not loaded here. func (s *Service) resolveWorkspacePopulations(ctx context.Context, project string, workspace authoringv2.Workspace, snapshot capability.Snapshot, scopeDigest string) (explorercompilation.ResolvedInputs, error) { project = projectid.Canonical(project) inputs := explorercompilation.ResolvedInputs{} diff --git a/internal/explorer/lifecycle/population_cursor.go b/internal/explorer/lifecycle/population_cursor.go index 8dd4d0df2..7fe0d9c41 100644 --- a/internal/explorer/lifecycle/population_cursor.go +++ b/internal/explorer/lifecycle/population_cursor.go @@ -9,9 +9,6 @@ import ( "strings" ) -// PopulationMappingCursor is the complete, non-sensitive binding carried by -// a population report page cursor. It is signed as one payload so callers -// cannot move a cursor between reports or change the member position. type PopulationMappingCursor struct { Version int `json:"version"` ReceiptID string `json:"receiptId"` @@ -26,16 +23,11 @@ type PopulationMappingCursor struct { MemberKey string `json:"memberKey"` } -// PopulationMappingCursorCodec is injected by the deployment boundary. A -// stable configured key, rather than process state, keeps cursors valid across -// replicas and restarts. type PopulationMappingCursorCodec interface { Encode(PopulationMappingCursor) (string, error) Decode(string) (PopulationMappingCursor, error) } -// HMACPopulationMappingCursorCodec signs cursors with a server-configured -// secret. The wire format is base64url(payload).base64url(signature). type HMACPopulationMappingCursorCodec struct { key []byte } diff --git a/internal/explorer/lifecycle/population_member_proposal.go b/internal/explorer/lifecycle/population_member_proposal.go index 2b6ee56e1..961c675de 100644 --- a/internal/explorer/lifecycle/population_member_proposal.go +++ b/internal/explorer/lifecycle/population_member_proposal.go @@ -12,9 +12,6 @@ import ( "github.com/google/uuid" ) -// PopulationMemberRemovalProposalRequest identifies one exact member in the -// currently attached immutable selection. The caller never supplies candidate -// membership or a population route. type PopulationMemberRemovalProposalRequest struct { Project string ExplorerID string @@ -52,8 +49,6 @@ func (r PopulationMemberRemovalProposalRequest) Validate() error { return nil } -// PopulationMemberRemovalProposalResponse exposes immutable headers for -// paginated inspection; member arrays are never downloaded by this operation. type PopulationMemberRemovalProposalResponse struct { ProposalID string `json:"proposalId,omitempty"` OutputID string `json:"outputId"` @@ -68,11 +63,6 @@ type PopulationMemberRemovalProposalResponse struct { PreviewStatus string `json:"previewStatus"` } -// ProposePopulationMemberRemoval creates an unattached immutable candidate -// selection by excluding one exact member from the current attached selection, -// reissues the same authorized population route for that candidate, and stores -// a purpose-bound compilation receipt. Apply is the only operation that saves -// the changed population on the draft. func (s *Service) ProposePopulationMemberRemoval(ctx context.Context, request PopulationMemberRemovalProposalRequest) (PopulationMemberRemovalProposalResponse, error) { if err := request.Validate(); err != nil { return PopulationMemberRemovalProposalResponse{}, malformed("population-member-proposal", err.Error(), err) diff --git a/internal/explorer/lifecycle/population_report.go b/internal/explorer/lifecycle/population_report.go index eff436d96..5f0f3d2d8 100644 --- a/internal/explorer/lifecycle/population_report.go +++ b/internal/explorer/lifecycle/population_report.go @@ -17,9 +17,6 @@ const ( MaxPopulationMappingLimit = 1000 ) -// PopulationMapping performs one synchronous, receipt-bound coverage check. -// It reruns the request-scoped mapping query for later pages; no report state -// is persisted on a receipt, selection, or workspace. func (s *Service) PopulationMapping(ctx context.Context, request PopulationMappingRequest) (PopulationMappingResult, error) { if strings.TrimSpace(request.ReceiptID) == "" || strings.TrimSpace(request.OutputID) == "" { return PopulationMappingResult{}, malformed("populationMapping", "receiptId and outputId are required", nil) diff --git a/internal/explorer/lifecycle/population_report_test.go b/internal/explorer/lifecycle/population_report_test.go index 30f858f83..28c27941b 100644 --- a/internal/explorer/lifecycle/population_report_test.go +++ b/internal/explorer/lifecycle/population_report_test.go @@ -14,9 +14,6 @@ import ( "github.com/calypr/loom/internal/explorer/capability" ) -// Keep the vertical contract test close to the lifecycle boundary. The -// receipt and membership are intentionally assembled from public domain -// values; the mapping callback stands in for the configured execution engine. func TestPopulationMappingReportLiteralResult(t *testing.T) { snapshot := readySnapshot("project-a", "generation-a", "token", unrestrictedScope()) receipt := populationTestReceipt(snapshot) diff --git a/internal/explorer/lifecycle/publish_release_requirements_classification_test.go b/internal/explorer/lifecycle/publish_release_requirements_classification_test.go index 2ed0775ef..64d79a786 100644 --- a/internal/explorer/lifecycle/publish_release_requirements_classification_test.go +++ b/internal/explorer/lifecycle/publish_release_requirements_classification_test.go @@ -13,12 +13,6 @@ import ( "github.com/calypr/loom/internal/explorer/capability" ) -// F-B14 durable control. A deterministic release-requirements miss (permanent -// until data/config changes) must be classified separately from a genuine -// transient preparation failure, with explicit non-retryability, per -// docs/loom-reliability-contract.md: clients do not infer retryability from -// an HTTP status. - func publishIntoReleasePreparation(t *testing.T, prepareErr error) (*Error, *fakeStore, error) { t.Helper() snapshot := readySnapshot("project-a", "generation-a", "token", authscope.ReadScope{Mode: authscope.ReadScopeUnrestricted}) @@ -83,11 +77,6 @@ func TestPublishGenuinePreparationFailureRemainsUnavailable(t *testing.T) { } } -// RepositoryPublish sibling (F-B14b): the same activation error must be split -// the same way — typed contract miss keeps the 409 with an explicit -// non-retryable diagnostic, while a store fault stays retryable and becomes -// unavailable instead of a misleading conflict. - func repositoryPublishIntoActivation(t *testing.T, activateErr error) (*Error, *fakeStore, error) { t.Helper() visible := true diff --git a/internal/explorer/lifecycle/repository_publish.go b/internal/explorer/lifecycle/repository_publish.go index 59a6e2f17..ee16553b0 100644 --- a/internal/explorer/lifecycle/repository_publish.go +++ b/internal/explorer/lifecycle/repository_publish.go @@ -78,17 +78,11 @@ func (s *Service) PublishRepository(ctx context.Context, request RepositoryPubli } if err := s.config.ActivateRelease(ctx, projectid.Legacy(request.Project), request.Generation, selectorsForBundle(receipt.Bundle)); err != nil { if errors.Is(err, dataset.ErrReleaseActivationConflict) { - // Deterministic lost compare-and-swap race, the same contract the - // authoring publish path types as PUBLICATION_ACTIVATION_CONFLICT: - // fixed conflict text, never the wrapped transport text, and no - // retry hint for a race the client cannot win by hammering. _, _ = s.store.FailRevision(ctx, revision.ID, []explorer.Diagnostic{{Severity: "ERROR", Code: "PUBLICATION_ACTIVATION_CONFLICT", Message: err.Error(), Retryable: false}}) return RepositoryPublishResult{}, conflict("repository_publish", "PUBLICATION_ACTIVATION_CONFLICT", "another publication won activation; the prior Explorer revision was retained", nil, err) } var requirementsErr *dataset.ReleaseRequirementsError if errors.As(err, &requirementsErr) { - // Deterministic release-contract miss: a 409 with an explicit - // non-retryable diagnostic, not a fault the client should retry. _, _ = s.store.FailRevision(ctx, revision.ID, []explorer.Diagnostic{{Severity: "ERROR", Code: "RELEASE_ACTIVATION_FAILED", Message: err.Error(), Retryable: false}}) return RepositoryPublishResult{}, conflict("repository_publish", "RELEASE_ACTIVATION_FAILED", fmt.Sprintf("activate published ExplorerConfigV2: %v", err), map[string]any{"retryable": false}, err) } @@ -96,13 +90,6 @@ func (s *Service) PublishRepository(ctx context.Context, request RepositoryPubli return RepositoryPublishResult{}, unavailable("repository_publish", "RELEASE_ACTIVATION_FAILED", fmt.Sprintf("activate published ExplorerConfigV2: %v", err), err) } if err := s.store.ActivateRepositoryGeneration(ctx, request.Project, request.Generation, revision.ID); err != nil { - // The visibility switch follows the sibling CAS contract above: the - // typed lost-race sentinel keeps a deterministic fixed-text conflict - // with no retry bait, and any other failure is a transport fault that - // must never surface as a conflict or echo wrapped store text. The - // sentinel marks losing the composite activation guard (read miss at - // activation.go, and its zero-replaced/zero-updated guards, all of - // which mean a concurrent Explorer update won the switch). if errors.Is(err, explorer.ErrDraftConflict) { _, _ = s.store.FailRevision(ctx, revision.ID, []explorer.Diagnostic{{Severity: "ERROR", Code: "GENERATION_ACTIVATION_FAILED", Message: err.Error(), Retryable: false}}) return RepositoryPublishResult{}, conflict("repository_publish", "GENERATION_ACTIVATION_FAILED", "generation activation lost a concurrent Explorer update; the prior Explorer revision was retained", nil, err) diff --git a/internal/explorer/lifecycle/repository_publish_activation_conflict_test.go b/internal/explorer/lifecycle/repository_publish_activation_conflict_test.go index cb2f98c56..79def3d89 100644 --- a/internal/explorer/lifecycle/repository_publish_activation_conflict_test.go +++ b/internal/explorer/lifecycle/repository_publish_activation_conflict_test.go @@ -10,12 +10,6 @@ import ( "github.com/calypr/loom/internal/explorer" ) -// The authoring publish path types a lost release compare-and-swap as a -// PUBLICATION_ACTIVATION_CONFLICT (publish.go, pinned in service_test). The -// repository publish path used to fall through to retryable 503s whose public -// message echoed the wrapped transport text. These tests pin the mirrored -// contract on the repository path. - func TestRepositoryPublishActivationConflictMirrorsAuthoringContract(t *testing.T) { wrapped := fmt.Errorf("activate release: arango preconditions emptied: %w", dataset.ErrReleaseActivationConflict) lifecycleErr, store, err := repositoryPublishIntoActivation(t, wrapped) @@ -52,9 +46,6 @@ func TestRepositoryPublishActivationConflictUnwrappedSentinelClassifiesTheSame(t } } -// Control: an infrastructure activation fault (no conflict sentinel) must -// stay the retryable unavailable path with the retryable diagnostic, so the -// new branch does not swallow real faults into a conflict. func TestRepositoryPublishActivationInfrastructureFaultStaysRetryableUnavailable(t *testing.T) { lifecycleErr, store, _ := repositoryPublishIntoActivation(t, errors.New("arango: connection reset by peer")) if lifecycleErr.Class != ClassUnavailable || lifecycleErr.Code != "RELEASE_ACTIVATION_FAILED" { diff --git a/internal/explorer/lifecycle/repository_publish_generation_activation_classification_test.go b/internal/explorer/lifecycle/repository_publish_generation_activation_classification_test.go index 23e178b68..ebe5a8dd3 100644 --- a/internal/explorer/lifecycle/repository_publish_generation_activation_classification_test.go +++ b/internal/explorer/lifecycle/repository_publish_generation_activation_classification_test.go @@ -14,19 +14,6 @@ import ( "github.com/calypr/loom/internal/explorer/capability" ) -// The second compare-and-swap of the repository publish (store.ActivateRepositoryGeneration) -// must follow the same classification contract pinned for its sibling leg above it -// (ActivateRelease, pinned in publish_release_requirements_classification_test.go and -// repository_publish_activation_conflict_test.go): deterministic lost races get a fixed -// text 409 without retry bait, and transport faults get a retryable unavailable — never a -// 409 whose message echoes the raw store text ("activate ExplorerConfigV2: dial tcp …"). -// The store wraps every generation-activation failure in this one site, so the raw echo -// was reachable only through here and the promise derives from the sibling contract plus -// the error-classification commits (fault -> redacted retryable, conflict -> deterministic -// fixed text). The sentinel marks losing the composite activation guard: the guard-read -// miss (activation.go `if !found`) plus the zero-replaced/zero-updated guards in the same -// transaction — every site means a concurrent Explorer update won the switch. - func repositoryPublishIntoGenerationActivation(t *testing.T, generationErr error) (*Error, *fakeStore, error) { t.Helper() visible := true diff --git a/internal/explorer/lifecycle/repository_publish_release_cas_conflict_test.go b/internal/explorer/lifecycle/repository_publish_release_cas_conflict_test.go index a25eda1a7..ede06d0c7 100644 --- a/internal/explorer/lifecycle/repository_publish_release_cas_conflict_test.go +++ b/internal/explorer/lifecycle/repository_publish_release_cas_conflict_test.go @@ -14,15 +14,6 @@ import ( "github.com/calypr/loom/internal/explorer/capability" ) -// Companion control for the LOOM_DEV_REPOSITORY_ACTIVATION_CONFLICT_ONCE dev -// fault (internal/server repository_activation_conflict_dev_test.go). The dev -// fault feeds a stale expected revision into the dataset release CAS, so the -// store returns dataset.ErrReleaseActivationConflict from s.config.ActivateRelease. -// This test pins what PublishRepository must therefore leave behind: the prior -// Explorer revision stays ACTIVE and keeps the active pointer, while the -// losing candidate revision stays retained and is fail-marked with the -// repair-34 non-retryable PUBLICATION_ACTIVATION_CONFLICT diagnostic. - func TestRepositoryPublishReleaseCASConflictKeepsPriorPointerAndFailsLosingRevision(t *testing.T) { visible := true snapshot := readySnapshot("project-a", "generation-a", "token", authscope.ReadScope{Mode: authscope.ReadScopeUnrestricted}) @@ -65,9 +56,6 @@ func TestRepositoryPublishReleaseCASConflictKeepsPriorPointerAndFailsLosingRevis t.Fatalf("baseline owner active pointer = %q, want %q", store.repositoryOwner.ActiveRevisionID, first.Revision.ID) } - // A lost release compare-and-swap on the next repository publish: the - // dev fault's faithful signature is the store's own error sentinel from - // the ActivateRelease leg, never a synthetic wrapper error. activateErr = fmt.Errorf("activate release: arango preconditions emptied: a stale expected revision never wins the CAS: %w", dataset.ErrReleaseActivationConflict) _, err = publish("commit-conflict") var lifecycleErr *Error diff --git a/internal/explorer/lifecycle/row_change.go b/internal/explorer/lifecycle/row_change.go index 75143cb38..ccbc75711 100644 --- a/internal/explorer/lifecycle/row_change.go +++ b/internal/explorer/lifecycle/row_change.go @@ -9,9 +9,6 @@ import ( "github.com/calypr/loom/internal/explorer/authoringv2" ) -// AssessRowChange resolves capability evidence against one exact persisted -// draft and compiles a ready candidate without mutating the draft. Applying the -// candidate remains on the existing CAS command path. func (s *Service) AssessRowChange(ctx context.Context, request AssessRowChangeRequest) (AssessRowChangeResult, error) { if strings.TrimSpace(request.SnapshotToken) == "" || request.DraftVersion < 1 || strings.TrimSpace(request.DraftDigest) == "" || strings.TrimSpace(request.OutputID) == "" || strings.TrimSpace(request.RootNodeID) == "" { return AssessRowChangeResult{}, malformed("row-change", "snapshotToken, draftVersion, draftDigest, outputId, and rootNodeId are required", nil) @@ -92,10 +89,6 @@ func (s *Service) AssessRowChange(ctx context.Context, request AssessRowChangeRe return result, nil } -// rowChangeReceiptWorkspace keeps the receipt scoped to the output being -// replaced. Current-draft WORKSPACE_OUTPUT consumers remain unchanged in the -// persisted candidate and may need separate repair after the source schema -// changes. func rowChangeReceiptWorkspace(workspace authoringv2.Workspace, outputID string) (authoringv2.Workspace, error) { raw, err := workspace.CanonicalJSON() if err != nil { diff --git a/internal/explorer/lifecycle/row_proposal.go b/internal/explorer/lifecycle/row_proposal.go index e420a074f..67d93e09b 100644 --- a/internal/explorer/lifecycle/row_proposal.go +++ b/internal/explorer/lifecycle/row_proposal.go @@ -26,9 +26,6 @@ const ( RowDefinitionSelectionExpanded RowDefinitionSelectionKind = "EXPANDED" ) -// RowDefinitionSelection is the closed browser-intent union. It contains only -// opaque identities and user-selected policies; schema paths are resolved by -// RowChoiceResolver after the snapshot and output have been checked. type RowDefinitionSelection struct { Kind RowDefinitionSelectionKind `json:"kind"` FieldGroup *FieldGroupSelection `json:"fieldGroup,omitempty"` @@ -201,8 +198,6 @@ const ( RowChoiceExpanded RowChoiceKind = "EXPANDED" ) -// RowChoiceResolveRequest is the server-side identity used to verify a choice -// against one output and one immutable capability snapshot. type RowChoiceResolveRequest struct { Project string ExplorerID string @@ -253,8 +248,6 @@ type ExplicitGroupRevisionChoice struct { UnassignedMemberPolicies []authoringv2.UnassignedMemberPolicy `json:"unassignedMemberPolicies"` } -// ExplicitGroupRevisionProof contains the immutable identities a group -// adapter validates before a proposal can use a revision. type ExplicitGroupRevisionProof struct { RevisionID string Project string @@ -274,9 +267,6 @@ type ExplicitGroupRevisionResolver interface { ValidateCompilationReceipt(context.Context, ExplicitGroupRevisionProof, *explorer.CompilationReceipt) error } -// ProposeRowDefinition compiles a complete candidate workspace without saving -// the draft. The candidate receipt ID is the proposal ID; no mutable proposal -// store is involved. func (s *Service) ProposeRowDefinition(ctx context.Context, request RowDefinitionProposalRequest) (RowDefinitionProposal, error) { if err := request.Validate(); err != nil { return RowDefinitionProposal{}, malformed("row-definition-proposal", err.Error(), err) @@ -415,8 +405,6 @@ func (s *Service) resolveRowDefinitionSelection(ctx context.Context, request Row }}, } if existing := document.Rows.Groups; existing != nil && document.Rows.Kind == authoringv2.RowDefinitionGroups && existing.Source.Kind == authoringv2.GroupSourceExplicit { - // A policy or revision edit changes the pinned row source, not the - // stage boundary or the member fields already chosen for cohort rows. groups.AfterStepID = existing.AfterStepID groups.RowValues = append([]authoringv2.ExplicitGroupRowValue(nil), existing.RowValues...) } else if document.Construction != nil && len(document.Construction.Steps) > 0 { diff --git a/internal/explorer/lifecycle/schema_field_candidate.go b/internal/explorer/lifecycle/schema_field_candidate.go index dd78d3883..76630f846 100644 --- a/internal/explorer/lifecycle/schema_field_candidate.go +++ b/internal/explorer/lifecycle/schema_field_candidate.go @@ -9,8 +9,6 @@ import ( "github.com/calypr/loom/internal/explorer/capability" ) -// Generated identities bind a schema path to the authorized snapshot, rather -// than claiming that the path was observed in its population. func schemaFieldCandidateID(snapshotToken, nodeID, path string) string { identity, _ := json.Marshal([3]string{snapshotToken, nodeID, path}) return "schema_field." + base64.RawURLEncoding.EncodeToString(identity) diff --git a/internal/explorer/lifecycle/schema_fields.go b/internal/explorer/lifecycle/schema_fields.go index c38832f5a..841e71132 100644 --- a/internal/explorer/lifecycle/schema_fields.go +++ b/internal/explorer/lifecycle/schema_fields.go @@ -25,8 +25,6 @@ const ( schemaFieldMaxCursor = 4096 ) -// SearchSchemaFieldsRequest lists generated primitive paths for one populated -// node in the current authorized capability snapshot. type SearchSchemaFieldsRequest struct { Project string ExplorerID string @@ -37,8 +35,6 @@ type SearchSchemaFieldsRequest struct { Limit int } -// SchemaFieldOption carries generated-schema metadata and a proved default -// projection choice, without observed counts, examples, or suggestions. type SchemaFieldOption struct { Origin string `json:"origin"` NodeID string `json:"nodeId"` @@ -50,9 +46,6 @@ type SchemaFieldOption struct { ConstructionChoice *capability.ConstructionChoice `json:"constructionChoice,omitempty"` } -// SchemaFieldSearchResponse pages generated paths separately from the -// evidence-backed capability snapshot. SchemaDigest is copied from the active -// schema identity already pinned into that authorized snapshot. type SchemaFieldSearchResponse struct { SnapshotToken string `json:"snapshotToken"` SchemaDigest string `json:"schemaDigest"` @@ -80,10 +73,6 @@ type schemaFieldCursor struct { LastPath string `json:"lastPath"` } -// SearchSchemaFields returns a stable depth-first, then lexicographic page of generated -// primitive paths. The path list is derived from generated FHIR schema -// metadata, then bound to the exact snapshot, populated node, generation, -// schema digest, authorization scope, query, and page size. func (s *Service) SearchSchemaFields(ctx context.Context, request SearchSchemaFieldsRequest) (SchemaFieldSearchResponse, error) { result := SchemaFieldSearchResponse{ SnapshotToken: request.SnapshotToken, diff --git a/internal/explorer/lifecycle/selection.go b/internal/explorer/lifecycle/selection.go index 7ce355c19..9c1c617c5 100644 --- a/internal/explorer/lifecycle/selection.go +++ b/internal/explorer/lifecycle/selection.go @@ -43,9 +43,6 @@ type selectionCreateRequest struct { BaseSelectionID string } -// SelectionSourceIntent is the closed authoring input. It contains no -// execution, receipt, schema, or addressability metadata; lifecycle resolves -// those identities from the authorized snapshot and exact publication source. type SelectionSourceIntent struct { Kind string Resources []explorer.ResourceRef @@ -126,8 +123,6 @@ type SelectionReadResult struct { NextCursor string } -// SelectionPublishedReader is the exact-output adapter required for -// all-matching selections. *published.Reader satisfies this interface. type SelectionPublishedReader interface { ExactExecutionMaterialization(context.Context, string, string) (published.Materialization, error) StreamExactExecution(context.Context, string, string, published.StreamRequest, func(map[string]any) error) error @@ -152,8 +147,6 @@ func (s *Service) createSelectionResolved(ctx context.Context, req selectionCrea return s.createSelection(ctx, req, nil) } -// CreateSelection authenticates and resolves the closed source intent before -// any header or membership count can be disclosed. func (s *Service) CreateSelection(ctx context.Context, req SelectionIntentCreateRequest) (result SelectionCreateResult, err error) { result, err = s.createSelectionIntent(ctx, req) if err != nil { @@ -168,9 +161,6 @@ func (s *Service) createSelectionIntent(ctx context.Context, req SelectionIntent } resolver := s.config.SelectionSourceResolver snapshotToken := strings.TrimSpace(req.SnapshotToken) - // Creation is bound to the active immutable dataset generation. Execution - // authorization intentionally supports retained generations, so using it - // here would allow a stale snapshot to create a new selection. authorized, err := s.config.Capability.ForCompilation(ctx, req.Project, snapshotToken) if err != nil { return SelectionCreateResult{}, err @@ -523,8 +513,6 @@ func (s *Service) appendPublished(ctx context.Context, header explorer.Selection }); err != nil { return err } - // The reader callback runs synchronously, so flush a final bounded batch - // after the stream completes. return appendBatch() } @@ -584,8 +572,6 @@ func (s *Service) readSelectionResolved(ctx context.Context, req selectionReadRe return SelectionReadResult{Header: header, Members: members, NextCursor: nextCursor}, nil } -// ReadSelection derives the effective current scope before loading the -// immutable header, so a stale or narrowed scope cannot disclose counts. func (s *Service) ReadSelection(ctx context.Context, req SelectionReadIntentRequest) (result SelectionReadResult, err error) { result, err = s.readSelectionIntent(ctx, req) if err != nil { @@ -707,9 +693,6 @@ func refKey(ref explorer.ResourceRef) string { } func selectionID(project, generation, idempotency string) string { - // The idempotency key owns the durable selection identity. A retry with the - // same key must collide with the same header even if a caller changes its - // rule; BeginSelection then rejects that changed immutable intent. sum := sha256.Sum256([]byte(strings.TrimSpace(project) + "\x00" + strings.TrimSpace(generation) + "\x00" + strings.TrimSpace(idempotency))) return "selection_" + hex.EncodeToString(sum[:]) } diff --git a/internal/explorer/lifecycle/semantic_catalog.go b/internal/explorer/lifecycle/semantic_catalog.go index a8297e0c0..404a06b65 100644 --- a/internal/explorer/lifecycle/semantic_catalog.go +++ b/internal/explorer/lifecycle/semantic_catalog.go @@ -15,7 +15,6 @@ import ( "github.com/calypr/loom/internal/authscope" "github.com/calypr/loom/internal/catalog" - catalogarango "github.com/calypr/loom/internal/catalog/arango" compilerprobe "github.com/calypr/loom/internal/dataframe/compiler/capability" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/dataframe/spec" @@ -47,8 +46,6 @@ const ( fieldPresentationReplacementLimit = 100 ) -// FieldPresentationRequest scopes one bounded batch of raw candidates to an -// exact current output, draft, source node, and saved route occurrence. type FieldPresentationRequest struct { ExpectedDraftVersion int64 ExpectedDraftDigest string @@ -354,9 +351,6 @@ func (s *Service) BrowseSemanticInventory(ctx context.Context, req BrowseSemanti req.ResourceType = frame.Source.ResourceType bindingFilter = frame.Source.BindingID if isPriorPrimaryOnlySemanticFrameFamily(frame.Source) { - // The current inventory uses semantic-binding/v2 IDs. Query the - // authorized resource page without the obsolete v1 filter, then require - // an exact reconstructed v1 identity in semanticEntryForFrameFamily. bindingFilter = "" } result.FrameID, result.FrameSource = frame.ID, frame @@ -638,10 +632,10 @@ func (s *Service) semanticInventoryPresentationForOutput( if build.BuildID == "" || build.SourceKind == "" { return choices, fieldPresentationUnavailable(FieldPresentationCoverageUnavailable), nil } - coveragePaths := make([]catalogarango.SemanticCoverageCandidatePath, 0, len(current.candidates)) + coveragePaths := make([]catalog.SemanticCoverageCandidatePath, 0, len(current.candidates)) coveragePathKeys := make(map[string]struct{}, len(current.candidates)) for _, candidate := range current.candidates { - path := catalogarango.SemanticCoverageCandidatePath{ResourceType: candidate.ResourceType, CanonicalPath: candidate.FieldPath} + path := catalog.SemanticCoverageCandidatePath{ResourceType: candidate.ResourceType, CanonicalPath: candidate.FieldPath} key := semanticCoverageCandidatePathKey(path) if _, exists := coveragePathKeys[key]; exists { continue @@ -650,7 +644,7 @@ func (s *Service) semanticInventoryPresentationForOutput( coveragePaths = append(coveragePaths, path) } unrestricted := authorized.Scope.Mode == authscope.ReadScopeUnrestricted - lookup, err := s.config.SemanticCoverage(ctx, catalogarango.SemanticCoverageLookupOptions{ + lookup, err := s.config.SemanticCoverage(ctx, catalog.SemanticCoverageLookupOptions{ Project: projectid.Legacy(snapshot.Identity.Project), DatasetGeneration: snapshot.Identity.Generation, BuildID: build.BuildID, SchemaIdentity: snapshot.Identity.SchemaDigest, SourceKind: build.SourceKind, AuthResourcePaths: append([]string(nil), authorized.Scope.AuthResourcePaths...), @@ -662,16 +656,16 @@ func (s *Service) semanticInventoryPresentationForOutput( } return nil, nil, internal("catalog", "FIELD_PRESENTATION_LOOKUP_FAILED", "semantic field coverage could not be read", err) } - if lookup.State != catalogarango.SemanticCoverageBuildComplete && lookup.State != catalogarango.SemanticCoverageBuildPartial { + if lookup.State != catalog.SemanticCoverageBuildComplete && lookup.State != catalog.SemanticCoverageBuildPartial { reason := FieldPresentationCoverageUnavailable - if lookup.State == catalogarango.SemanticCoverageBuildBuilding || lookup.State == catalogarango.SemanticCoverageBuildFailed { + if lookup.State == catalog.SemanticCoverageBuildBuilding || lookup.State == catalog.SemanticCoverageBuildFailed { reason = FieldPresentationIncompleteCoverage } return choices, fieldPresentationUnavailable(reason), nil } - coverageByPath := make(map[string]catalogarango.SemanticCoverageCandidateResult, len(lookup.Candidates)) + coverageByPath := make(map[string]catalog.SemanticCoverageCandidateResult, len(lookup.Candidates)) for _, candidate := range lookup.Candidates { - key := semanticCoverageCandidatePathKey(catalogarango.SemanticCoverageCandidatePath{ + key := semanticCoverageCandidatePathKey(catalog.SemanticCoverageCandidatePath{ ResourceType: candidate.ResourceType, CanonicalPath: candidate.CanonicalPath, }) if _, duplicate := coverageByPath[key]; duplicate { @@ -716,25 +710,25 @@ func (s *Service) semanticInventoryPresentationForOutput( decisions = append(decisions, FieldPresentationKeepRaw{Kind: FieldPresentationDecisionKeepRaw, CandidateID: candidate.ID, Reason: FieldPresentationCandidateNotAddable}) continue } - coverage := coverageByPath[semanticCoverageCandidatePathKey(catalogarango.SemanticCoverageCandidatePath{ + coverage := coverageByPath[semanticCoverageCandidatePathKey(catalog.SemanticCoverageCandidatePath{ ResourceType: candidate.ResourceType, CanonicalPath: candidate.FieldPath, })] - if coverage.ResourceState != catalogarango.SemanticCoverageBuildComplete { + if coverage.ResourceState != catalog.SemanticCoverageBuildComplete { decisions = append(decisions, FieldPresentationKeepRaw{ Kind: FieldPresentationDecisionKeepRaw, CandidateID: candidate.ID, Reason: FieldPresentationIncompleteCoverage, }) continue } switch coverage.Status { - case catalogarango.SemanticCoverageCandidateKeepRaw: + case catalog.SemanticCoverageCandidateKeepRaw: reason, ok := fieldPresentationCoverageReason(string(coverage.Reason)) if !ok { return choices, fieldPresentationUnavailable(FieldPresentationIncompleteCoverage), nil } decisions = append(decisions, FieldPresentationKeepRaw{Kind: FieldPresentationDecisionKeepRaw, CandidateID: candidate.ID, Reason: reason}) - case catalogarango.SemanticCoverageCandidateUnavailable: + case catalog.SemanticCoverageCandidateUnavailable: decisions = append(decisions, FieldPresentationKeepRaw{Kind: FieldPresentationDecisionKeepRaw, CandidateID: candidate.ID, Reason: FieldPresentationIncompleteCoverage}) - case catalogarango.SemanticCoverageCandidateGroupable: + case catalog.SemanticCoverageCandidateGroupable: if strings.TrimSpace(coverage.CoverageID) == "" || len(coverage.References) == 0 { return choices, fieldPresentationUnavailable(FieldPresentationIncompleteCoverage), nil } @@ -801,7 +795,7 @@ func (s *Service) semanticInventoryPresentationForOutput( identities = append(identities, reference.identity) } if available { - coverage := coverageByPath[semanticCoverageCandidatePathKey(catalogarango.SemanticCoverageCandidatePath{ + coverage := coverageByPath[semanticCoverageCandidatePathKey(catalog.SemanticCoverageCandidatePath{ ResourceType: candidate.ResourceType, CanonicalPath: candidate.FieldPath, })] decisions[decisionIndex] = FieldPresentationGroupUnderSemantic{ @@ -915,7 +909,7 @@ func fieldPresentationUnavailable(reason FieldPresentationReason) fieldPresentat return FieldPresentationUnavailable{Kind: FieldPresentationResultUnavailable, Reason: reason} } -func semanticCoverageCandidatePathKey(path catalogarango.SemanticCoverageCandidatePath) string { +func semanticCoverageCandidatePathKey(path catalog.SemanticCoverageCandidatePath) string { return path.ResourceType + "\x00" + path.CanonicalPath } @@ -1292,10 +1286,6 @@ type semanticCandidateRouteProof struct { ok bool } -// semanticInventoryChoicesForOutput keeps table applicability separate from -// global semantic inventory. Routes are planned once per distinct candidate -// node; repeated entries reuse the exact candidate+route compiler proof, while -// each semantic binding still gets its own choice ID. func semanticInventoryChoicesForOutput( ctx context.Context, authorized AuthorizedCapability, @@ -1438,10 +1428,6 @@ func semanticInventoryChoicesForOutput( return choices, nil } -// semanticInventoryReachableNodes uses the authorized snapshot graph only as -// a negative prefilter. Every retained route is still reauthorized and -// compiler-probed below; disconnected resource nodes never trigger a full -// alternate-route search. func semanticInventoryReachableNodes(snapshot capability.Snapshot, rootResourceType string) map[string]bool { adjacency := make(map[string][]string) for _, edge := range snapshot.Edges { @@ -1488,9 +1474,6 @@ func semanticInventoryConstructionChoice(snapshot capability.Snapshot, contextTo func semanticInventoryConstructionChoiceForRoute(snapshot capability.Snapshot, contextToken, buildID string, route []capability.ConstructionRouteStep, entry catalog.SemanticInventoryEntry, candidate capability.Candidate, ownerRecordsProved bool) (capability.ConstructionChoice, error) { observation := entry.Observation if observation.RuleHint == catalog.SemanticRuleHintCodedConceptV1 { - // Owner-record output is defined over the enclosing semantic owner. A - // standalone Coding choice must stay attached to the exact matched - // Coding, so expose only the compiler-proved scalar/list choices. ownerRecordsProved = false } logicalType := observation.LogicalType @@ -1682,11 +1665,6 @@ func semanticFrameFamilyMatchesPinnedPrior(current, pinned capability.SemanticFr current.LogicalType == pinned.LogicalType && reflect.DeepEqual(current.ChoiceArms, pinned.ChoiceArms) } -// semanticEntryForFrameFamily projects a current inventory observation back -// to the exact value binding pinned by a saved frame. The only supported -// transition is the immediately preceding semantic build, whose new behavior -// adds fallback selectors. A legacy frame without fallbacks keeps its primary -// selector and cannot silently acquire the new fallback behavior. func semanticEntryForFrameFamily( entry catalog.SemanticInventoryEntry, family capability.SemanticFrameFamily, @@ -1856,9 +1834,6 @@ func semanticCandidateWithFallback(snapshot capability.Snapshot, observation cat return combined, true } -// Coded-value observations keep key and value selectors relative to the object -// that owns their correlation. Capability candidates address the scalar from -// the resource root, so the owning scope is restored only at this boundary. func semanticObservationValuePath(observation catalog.SemanticObservation) string { valuePath := canonicalInventoryPath(observation.Value.Selector) ownerPath := canonicalInventoryPath(observation.OwningScope) diff --git a/internal/explorer/lifecycle/semantic_field_presentation_test.go b/internal/explorer/lifecycle/semantic_field_presentation_test.go index a80daba45..6c108e075 100644 --- a/internal/explorer/lifecycle/semantic_field_presentation_test.go +++ b/internal/explorer/lifecycle/semantic_field_presentation_test.go @@ -10,7 +10,6 @@ import ( "github.com/calypr/loom/internal/authscope" "github.com/calypr/loom/internal/catalog" - catalogarango "github.com/calypr/loom/internal/catalog/arango" "github.com/calypr/loom/internal/explorer/authoringv2" "github.com/calypr/loom/internal/explorer/capability" ) @@ -230,7 +229,7 @@ func TestFieldPresentationReadsStayInsideCurrentAuthorizationScope(t *testing.T) Entries: []catalog.SemanticInventoryEntry{fixture.entry}, }, nil } - fixture.service.config.SemanticCoverage = func(_ context.Context, options catalogarango.SemanticCoverageLookupOptions) (catalogarango.SemanticCoverageLookupResult, error) { + fixture.service.config.SemanticCoverage = func(_ context.Context, options catalog.SemanticCoverageLookupOptions) (catalog.SemanticCoverageLookupResult, error) { fixture.coverageReads++ if options.AuthResourcePathsUnrestricted || !reflect.DeepEqual(options.AuthResourcePaths, scope.AuthResourcePaths) { t.Fatalf("coverage lookup escaped the current authorization path: %#v", options) @@ -276,8 +275,8 @@ func TestFieldPresentationDoesNotCollapseCompilerOrCoverageReadErrors(t *testing t.Run("coverage read failure", func(t *testing.T) { fixture := newFieldPresentationTestFixture(t, false) - fixture.service.config.SemanticCoverage = func(context.Context, catalogarango.SemanticCoverageLookupOptions) (catalogarango.SemanticCoverageLookupResult, error) { - return catalogarango.SemanticCoverageLookupResult{}, errors.New("injected coverage read failure") + fixture.service.config.SemanticCoverage = func(context.Context, catalog.SemanticCoverageLookupOptions) (catalog.SemanticCoverageLookupResult, error) { + return catalog.SemanticCoverageLookupResult{}, errors.New("injected coverage read failure") } _, err := fixture.service.BrowseSemanticInventory(context.Background(), fixture.request()) assertLifecycleError(t, err, ClassInternal, "FIELD_PRESENTATION_LOOKUP_FAILED") @@ -329,7 +328,7 @@ func TestFieldPresentationKeepsRawWhenCoverageProofIsMissingPartialOrStale(t *te fixture.service.config.SemanticCoverage = nil }}, {name: "build still building", mutate: func(fixture *fieldPresentationTestFixture) { - fixture.coverageResult.State = catalogarango.SemanticCoverageBuildBuilding + fixture.coverageResult.State = catalog.SemanticCoverageBuildBuilding }}, {name: "requested facet missing", mutate: func(fixture *fieldPresentationTestFixture) { fixture.coverageResult.Candidates = nil @@ -397,7 +396,7 @@ func TestFieldPresentationKeepsRawWhenReplacementIsUnavailableOrNotInCodedView(t func TestFieldPresentationKeepsRawForClosedCoverageDisposition(t *testing.T) { fixture := newFieldPresentationTestFixture(t, false) - fixture.coverageResult.Candidates[0].Status = catalogarango.SemanticCoverageCandidateKeepRaw + fixture.coverageResult.Candidates[0].Status = catalog.SemanticCoverageCandidateKeepRaw fixture.coverageResult.Candidates[0].Reason = catalog.SemanticCoverageReasonMissingSystem fixture.coverageResult.Candidates[0].References = nil response, err := fixture.service.BrowseSemanticInventory(context.Background(), fixture.request()) @@ -464,30 +463,30 @@ func TestFieldPresentationGroupsExactRelatedOccurrenceAndAppliesCurrentChoice(t func TestFieldPresentationGroupsRawCodingFacetUnderExistingCategoryFamily(t *testing.T) { assertFieldPresentationCodedFamilyProof( - t, "Observation", "code", catalogarango.SemanticCoverageBuildComplete, - catalogarango.SemanticCoverageBuildComplete, catalogarango.SemanticCoverageCandidateGroupable, true, + t, "Observation", "code", catalog.SemanticCoverageBuildComplete, + catalog.SemanticCoverageBuildComplete, catalog.SemanticCoverageCandidateGroupable, true, ) } func TestFieldPresentationGroupsSpecimenCandidateDuringPartialAudit(t *testing.T) { assertFieldPresentationCodedFamilyProof( - t, "Specimen", "type", catalogarango.SemanticCoverageBuildPartial, - catalogarango.SemanticCoverageBuildComplete, catalogarango.SemanticCoverageCandidateGroupable, true, + t, "Specimen", "type", catalog.SemanticCoverageBuildPartial, + catalog.SemanticCoverageBuildComplete, catalog.SemanticCoverageCandidateGroupable, true, ) } func TestFieldPresentationKeepsTaskCandidateRawWithoutResourceCertificate(t *testing.T) { assertFieldPresentationCodedFamilyProof( - t, "Task", "code", catalogarango.SemanticCoverageBuildPartial, - catalogarango.SemanticCoverageBuildMissing, catalogarango.SemanticCoverageCandidateUnavailable, false, + t, "Task", "code", catalog.SemanticCoverageBuildPartial, + catalog.SemanticCoverageBuildMissing, catalog.SemanticCoverageCandidateUnavailable, false, ) } func assertFieldPresentationCodedFamilyProof( t *testing.T, resourceType, ownerField string, - globalState, resourceState catalogarango.SemanticCoverageBuildState, - candidateStatus catalogarango.SemanticCoverageCandidateStatus, + globalState, resourceState catalog.SemanticCoverageBuildState, + candidateStatus catalog.SemanticCoverageCandidateStatus, wantGroup bool, ) { t.Helper() @@ -554,11 +553,11 @@ func assertFieldPresentationCodedFamilyProof( {ID: displayCandidate.ID, NodeID: displayCandidate.NodeID, FieldPath: displayCandidate.FieldPath, Label: displayCandidate.Label, LogicalType: displayCandidate.LogicalType, Cardinality: displayCandidate.Cardinality, Repeated: true, ProjectionModes: []string{"FIRST", "ARRAY"}, DefaultProjectionMode: "FIRST", ConstructionChoice: &displayChoice}, }, } - coverage := catalogarango.SemanticCoverageCandidateResult{ + coverage := catalog.SemanticCoverageCandidateResult{ ResourceType: codeCandidate.ResourceType, CanonicalPath: codeCandidate.FieldPath, CoverageID: "coding-code-coverage", ResourceState: resourceState, Status: candidateStatus, Reason: catalog.SemanticCoverageReasonIncompleteSource, } - if candidateStatus == catalogarango.SemanticCoverageCandidateGroupable { + if candidateStatus == catalog.SemanticCoverageCandidateGroupable { coverage.References = []catalog.SemanticCoverageReference{{ Kind: catalog.SemanticCoverageFamilyReference, BindingID: family.BindingID, Family: family.SemanticInventoryFamilyReference, CandidatePath: codeCandidate.FieldPath, ResourceType: codeCandidate.ResourceType, @@ -574,11 +573,11 @@ func assertFieldPresentationCodedFamilyProof( SemanticInventory: func(context.Context, catalog.SemanticInventoryPageOptions) (catalog.SemanticInventoryPage, error) { return catalog.SemanticInventoryPage{State: catalog.SemanticInventoryComplete, Build: build, Families: []catalog.SemanticInventoryFamily{family}}, nil }, - SemanticCoverage: func(_ context.Context, options catalogarango.SemanticCoverageLookupOptions) (catalogarango.SemanticCoverageLookupResult, error) { - if len(options.Candidates) != 1 || options.Candidates[0] != (catalogarango.SemanticCoverageCandidatePath{ResourceType: codeCandidate.ResourceType, CanonicalPath: codeCandidate.FieldPath}) { + SemanticCoverage: func(_ context.Context, options catalog.SemanticCoverageLookupOptions) (catalog.SemanticCoverageLookupResult, error) { + if len(options.Candidates) != 1 || options.Candidates[0] != (catalog.SemanticCoverageCandidatePath{ResourceType: codeCandidate.ResourceType, CanonicalPath: codeCandidate.FieldPath}) { t.Fatalf("coverage lookup did not use the exact Coding leaf facet: %#v", options) } - return catalogarango.SemanticCoverageLookupResult{State: globalState, Candidates: []catalogarango.SemanticCoverageCandidateResult{coverage}}, nil + return catalog.SemanticCoverageLookupResult{State: globalState, Candidates: []catalog.SemanticCoverageCandidateResult{coverage}}, nil }, ResolveSemanticInventorySelections: func(_ context.Context, options catalog.SemanticInventoryResolveOptions) (catalog.SemanticInventoryResolveResult, error) { if len(options.FamilyReferences) != 1 || options.FamilyReferences[0] != family.SemanticInventoryFamilyReference || len(options.References) != 0 { @@ -712,7 +711,7 @@ type fieldPresentationTestFixture struct { buildID string occurrenceID string rootResourceType string - coverageResult catalogarango.SemanticCoverageLookupResult + coverageResult catalog.SemanticCoverageLookupResult resolvedInventory catalog.SemanticInventoryResolveResult authorizationReads int inventoryReads int @@ -800,9 +799,9 @@ func newFieldPresentationTestFixture(t *testing.T, related bool) *fieldPresentat document.Route = authoringv2.RouteNode{OccurrenceID: authoringv2.RootOccurrenceID, ResourceType: "Observation", Children: []authoringv2.RouteNode{}} } persistTestWorkspace(t, store, workspace) - coverage := catalogarango.SemanticCoverageCandidateResult{ + coverage := catalog.SemanticCoverageCandidateResult{ ResourceType: candidate.ResourceType, CanonicalPath: candidate.FieldPath, CoverageID: "coverage-id", - ResourceState: catalogarango.SemanticCoverageBuildComplete, Status: catalogarango.SemanticCoverageCandidateGroupable, + ResourceState: catalog.SemanticCoverageBuildComplete, Status: catalog.SemanticCoverageCandidateGroupable, References: []catalog.SemanticCoverageReference{{ Kind: catalog.SemanticCoverageExactReference, ConceptID: entry.ConceptID, BindingID: entry.BindingID, CandidatePath: candidate.FieldPath, ResourceType: candidate.ResourceType, @@ -811,8 +810,8 @@ func newFieldPresentationTestFixture(t *testing.T, related bool) *fieldPresentat fixture := &fieldPresentationTestFixture{ store: store, snapshot: snapshot, candidate: candidate, entry: entry, buildID: buildID, occurrenceID: occurrenceID, rootResourceType: rootResourceType, - coverageResult: catalogarango.SemanticCoverageLookupResult{ - State: catalogarango.SemanticCoverageBuildComplete, Candidates: []catalogarango.SemanticCoverageCandidateResult{coverage}, + coverageResult: catalog.SemanticCoverageLookupResult{ + State: catalog.SemanticCoverageBuildComplete, Candidates: []catalog.SemanticCoverageCandidateResult{coverage}, }, resolvedInventory: semanticAuthoringInventory(snapshot, []catalog.SemanticInventoryEntry{entry}), } @@ -832,11 +831,11 @@ func newFieldPresentationTestFixture(t *testing.T, related bool) *fieldPresentat } return catalog.SemanticInventoryPage{State: catalog.SemanticInventoryComplete, Build: build, Entries: []catalog.SemanticInventoryEntry{entry}}, nil }, - SemanticCoverage: func(_ context.Context, options catalogarango.SemanticCoverageLookupOptions) (catalogarango.SemanticCoverageLookupResult, error) { + SemanticCoverage: func(_ context.Context, options catalog.SemanticCoverageLookupOptions) (catalog.SemanticCoverageLookupResult, error) { fixture.coverageReads++ if options.Project != project || options.DatasetGeneration != generation || options.BuildID != buildID || options.SourceKind != catalog.SemanticInventorySourceFile || !options.AuthResourcePathsUnrestricted || - len(options.Candidates) != 1 || options.Candidates[0] != (catalogarango.SemanticCoverageCandidatePath{ResourceType: candidate.ResourceType, CanonicalPath: candidate.FieldPath}) { + len(options.Candidates) != 1 || options.Candidates[0] != (catalog.SemanticCoverageCandidatePath{ResourceType: candidate.ResourceType, CanonicalPath: candidate.FieldPath}) { t.Fatalf("coverage lookup lost exact bounded scope: %#v", options) } return fixture.coverageResult, nil diff --git a/internal/explorer/lifecycle/service.go b/internal/explorer/lifecycle/service.go index fe4247861..121e460dc 100644 --- a/internal/explorer/lifecycle/service.go +++ b/internal/explorer/lifecycle/service.go @@ -7,8 +7,6 @@ import ( "github.com/calypr/loom/internal/explorer" ) -// Service is the Explorer application layer. It coordinates persistence and -// deployment adapters while leaving all wire representation to transports. type Service struct { store *explorer.Service config Config diff --git a/internal/explorer/lifecycle/table_shape_capabilities.go b/internal/explorer/lifecycle/table_shape_capabilities.go index 82b660764..7223db3e9 100644 --- a/internal/explorer/lifecycle/table_shape_capabilities.go +++ b/internal/explorer/lifecycle/table_shape_capabilities.go @@ -250,10 +250,6 @@ func (s *Service) compileTableShapeBase(ctx context.Context, request TableShapeC if !ok { return tableShapeBase{}, unprocessable("table-shape-capabilities", "OUTPUT_NOT_FOUND", "outputId is not in the compiler output contract", nil) } - // A saved reshape is a derived view of a compiler-owned base relation. Build - // editor choices from that relation, not from the reshaped final schema, - // while retaining the actual draft/document digests as the capability's - // binding. This is what makes reload able to restore removed base columns. baseWorkspace := workspace baseWorkspace.Documents = append([]authoringv2.Document(nil), workspace.Documents...) for index := range baseWorkspace.Documents { diff --git a/internal/explorer/lifecycle/table_shape_proposal.go b/internal/explorer/lifecycle/table_shape_proposal.go index b2a24b619..239c3263f 100644 --- a/internal/explorer/lifecycle/table_shape_proposal.go +++ b/internal/explorer/lifecycle/table_shape_proposal.go @@ -267,8 +267,6 @@ type tableShapeProposalTimings struct { receiptEvidence time.Duration } -// ServerTiming returns the proposal work breakdown for the HTTP response. -// These measurements are deliberately excluded from the proposal JSON body. func (p TableShapeProposal) ServerTiming() string { metrics := []struct { name string diff --git a/internal/explorer/lifecycle/table_shape_proposal_test.go b/internal/explorer/lifecycle/table_shape_proposal_test.go index f015b2197..7d165cd9d 100644 --- a/internal/explorer/lifecycle/table_shape_proposal_test.go +++ b/internal/explorer/lifecycle/table_shape_proposal_test.go @@ -1228,10 +1228,6 @@ func TestProposeTableShapeUnpivotReproducesInternalError(t *testing.T) { t.Run(variant.name, func(t *testing.T) { service, store, snapshot, workspace := tableShapeProposalService(t) if variant.realCompile { - // Rewrite the fixture draft onto real Patient element paths so the - // real workspace compiler can resolve selectors. The two unpivot - // inputs keep their authored names; code+date both melt to the - // same canonical table scalar class as the capability menu admits. reproColumns := []authoringv2.Column{ {Column: "patient_id", Label: "Patient ID", LogicalType: "string", OccurrenceID: authoringv2.RootOccurrenceID, Source: authoringv2.ColumnSource{Kind: authoringv2.SourceField, Field: &authoringv2.FieldSource{Path: "id", ProjectionMode: "VALUE"}}}, @@ -1276,18 +1272,6 @@ func TestProposeTableShapeUnpivotReproducesInternalError(t *testing.T) { request.ReshapeResolutionID = unpivot.ResolutionID request.DerivedResolutionIDs = nil if variant.realCompile { - // Mirror the production s.config.CompileReceipt wiring via - // compileTraceReceipt: the real workspace compiler/lowering path plus - // the CompiledOutputSchemas population that store-time arango - // validateReceipt runs. The harness wraps the validate error with a - // "validate trace compiler receipt:" prefix, so it does not reproduce - // production's exact *errors.errorString 500 causal type; the shared - // invariant is the validateReceiptCompiledOutputSchemas check itself. - // Production independently enforces the same check at apply time via - // verifyProposalReceipt -> receipt.Validate (proposal_receipt.go:19), - // so the adapter's red does not rest on harness-internal validation. - // Snapshot capability nodes/candidates are derived from the table's - // own source columns, matching the proven real-compile harness. compileSnapshot := snapshot rootType := "Patient" if len(workspace.Documents) > 0 && workspace.Documents[0].RootResourceType != "" { diff --git a/internal/explorer/lifecycle/types.go b/internal/explorer/lifecycle/types.go index 7af5c5686..f1ed2a462 100644 --- a/internal/explorer/lifecycle/types.go +++ b/internal/explorer/lifecycle/types.go @@ -1,7 +1,3 @@ -// Package lifecycle contains the transport-neutral Explorer application -// workflows. It owns orchestration and policy while transport packages remain -// responsible for request decoding, authorization context extraction, and -// response encoding. package lifecycle import ( @@ -10,7 +6,6 @@ import ( "github.com/calypr/loom/internal/authscope" "github.com/calypr/loom/internal/catalog" - catalogarango "github.com/calypr/loom/internal/catalog/arango" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/publication" dataframepublished "github.com/calypr/loom/internal/dataframe/published" @@ -23,9 +18,6 @@ import ( "github.com/calypr/loom/internal/explorer/tableshapecap" ) -// CapabilityResolver resolves the current or a retained immutable capability -// snapshot. The callbacks deliberately return domain values and do not expose -// HTTP, Fiber, GraphQL, or generated API types. type CapabilityResolver struct { Current func(context.Context, string, string, string) (capability.Snapshot, error) Token func(context.Context, string, string) (capability.Snapshot, error) @@ -35,8 +27,6 @@ type CapabilityResolver struct { ValidateReadScope func(authscope.ReadScope, string) error } -// AuthorizedCapability is the exact snapshot and authorization scope frozen -// into a compilation receipt or used for receipt-backed execution. type AuthorizedCapability struct { Snapshot capability.Snapshot Scope authscope.ReadScope @@ -66,9 +56,6 @@ type ReceiptReader func(context.Context, string, string, string) (*explorer.Comp type ReceiptPreviewer func(context.Context, *explorer.CompilationReceipt, recipe.RuntimeBindings, func(map[string]any) error) (dataframeexecution.PreviewSummary, error) type ReceiptStreamValidator func(context.Context, *explorer.CompilationReceipt, recipe.RuntimeBindings) error -// ConstructionSourceStageCompiler describes a compiler-resolved source -// projection for capability discovery. Its result is metadata only; callers -// must not treat an empty public schema as an executable output. type ConstructionSourceStageCompiler func(context.Context, ConstructionSourceStageRequest) (explorer.ReceiptConstructionStage, error) type ConstructionSourceStageRequest struct { @@ -80,42 +67,23 @@ type ConstructionSourceStageRequest struct { SelectionMembersCollection string } -// CategoryScanner executes an exact compiler-owned category scan for one -// already-validated receipt without exposing query details to lifecycle. type CategoryScanner func(context.Context, *explorer.CompilationReceipt, recipe.RuntimeBindings, dataframeexecution.CategoryScanRequest) (dataframeexecution.CategoryScanResult, error) -// ArtifactPublishedReader is the exact-output surface lifecycle needs when -// materializing a durable artifact. Keeping this contract narrow makes it -// impossible for artifact orchestration to accidentally follow a moving -// project-current pointer or bypass the published reader's read pin. type ArtifactPublishedReader interface { ExactExecutionMaterialization(context.Context, string, string) (dataframepublished.Materialization, error) StreamExactExport(context.Context, dataframepublished.ExactExportRequest, dataframepublished.ExactExportVisitor) (dataframepublished.ExactExportResult, error) } -// PopulationMappingExecutor is the narrow lifecycle-to-execution adapter. -// Lifecycle supplies the validated receipt, scope bindings, and immutable -// selected IDs; execution owns compiler witness interpretation. type PopulationMappingExecutor func(context.Context, *explorer.CompilationReceipt, recipe.RuntimeBindings, string, []string, string, int) (dataframeexecution.PopulationMappingResult, error) -// CellTraceExecutor explains one published cell from the immutable receipt -// and authorization bindings validated by lifecycle. type CellTraceExecutor func(context.Context, *explorer.CompilationReceipt, recipe.RuntimeBindings, dataframeexecution.CellTraceRequest) (dataframeexecution.CellTraceResult, error) -// RowLineageExecutor pages source records for one row in the immutable receipt output. type RowLineageExecutor func(context.Context, *explorer.CompilationReceipt, recipe.RuntimeBindings, dataframeexecution.RowLineageRequest) (dataframeexecution.RowLineageResult, error) -// TableShapeExclusionExecutor enumerates exact exclusions from one validated -// candidate receipt under its existing authorization bindings. type TableShapeExclusionExecutor func(context.Context, *explorer.CompilationReceipt, recipe.RuntimeBindings, dataframeexecution.TableShapeExclusionRequest) (dataframeexecution.TableShapeExclusionResult, error) -// SelectionReferenceValidator resolves explicit references against the -// authorized active generation. It must verify both resource existence and -// auth_resource_path before lifecycle persists any member or exclusion. type SelectionReferenceValidator func(context.Context, string, string, authscope.ReadScope, []explorer.ResourceRef) error -// Execution is the small logical publication result needed by Explorer. It -// intentionally avoids the GraphQL resolver's execution type. type Execution struct { ID string Name string @@ -141,28 +109,19 @@ type GenerationValidator func(context.Context, string, string) error type ReleaseActivator func(context.Context, string, string, []dataset.DataframeSelector) error type ReleasePreparer func(context.Context, string, string, []dataset.DataframeSelector) (dataset.ProjectRelease, int64, error) -// Config contains deployment adapters. Lifecycle policy calls these narrow -// callbacks, but never imports the transport packages that construct them. type Config struct { - // SelectionMembersCollection is a deployment-owned runtime binding. It is - // never copied into authoring intent, recipes, or receipt identity. SelectionMembersCollection string Capability CapabilityResolver SemanticInventory func(context.Context, catalog.SemanticInventoryPageOptions) (catalog.SemanticInventoryPage, error) - SemanticCoverage func(context.Context, catalogarango.SemanticCoverageLookupOptions) (catalogarango.SemanticCoverageLookupResult, error) + SemanticCoverage func(context.Context, catalog.SemanticCoverageLookupOptions) (catalog.SemanticCoverageLookupResult, error) ResolveSemanticInventorySelections func(context.Context, catalog.SemanticInventoryResolveOptions) (catalog.SemanticInventoryResolveResult, error) - // InterpretationRepository resolves exact immutable revision IDs. It is - // intentionally narrow so lifecycle cannot accidentally depend on heads or - // unrelated Explorer persistence methods. - InterpretationRepository explorer.InterpretationRepository - // SelectionSourceResolver resolves an exact immutable published revision - // after Capability.ForExecution has established project and scope. - SelectionSourceResolver SelectionSourceResolver - SelectionReferenceValidator SelectionReferenceValidator - RowChoiceResolver RowChoiceResolver - RowChoicePlanner RowChoicePlanner - ExplicitGroupResolver ExplicitGroupRevisionResolver - ExplicitGroupRepository explorer.ExplicitGroupRepository + InterpretationRepository explorer.InterpretationRepository + SelectionSourceResolver SelectionSourceResolver + SelectionReferenceValidator SelectionReferenceValidator + RowChoiceResolver RowChoiceResolver + RowChoicePlanner RowChoicePlanner + ExplicitGroupResolver ExplicitGroupRevisionResolver + ExplicitGroupRepository explorer.ExplicitGroupRepository CompileReceipt ReceiptCompiler ConstructionSourceStage ConstructionSourceStageCompiler @@ -179,11 +138,9 @@ type Config struct { ReceiptLookup ReceiptReader ArtifactStore explorer.ArtifactStore PublishedReader ArtifactPublishedReader - // Artifact limits are deployment policy, never caller-controlled request - // fields. Zero uses lifecycle's conservative defaults. - ArtifactTTL time.Duration - ArtifactMaxRows int64 - ArtifactMaxBytes int64 + ArtifactTTL time.Duration + ArtifactMaxRows int64 + ArtifactMaxBytes int64 ValidateReleaseGeneration GenerationValidator ActivateRelease ReleaseActivator @@ -272,14 +229,11 @@ type AssessRowChangeResult struct { } type PreviewRequest struct { - Project string - ExplorerID string - ReceiptID string - OutputID string - Limit int - // SinkFactory is supplied by the transport adapter. It receives the - // validated receipt and emitted columns and returns a row sink for native - // streaming previews. Lifecycle never sees the response encoder itself. + Project string + ExplorerID string + ReceiptID string + OutputID string + Limit int SinkFactory func(*explorer.CompilationReceipt, []explorer.EmittedColumn) (func(map[string]any) error, error) } @@ -289,29 +243,18 @@ type PreviewResult struct { Summary dataframeexecution.PreviewSummary } -// PreviewSinkFactory adapts native preview rows to a transport-owned sink. -// Lifecycle owns preview authorization and ordering; transports own encoding. type PreviewSinkFactory func(*explorer.CompilationReceipt, []explorer.EmittedColumn) (func(map[string]any) error, error) -// AcceptedConstruction is the exact Apply-time preview that validated the -// candidate construction. It is returned only after the draft CAS succeeds. type AcceptedConstruction struct { Preview PreviewResult PreviewLimit int } -// ApplyCommandsWithConstructionPreviewResponse keeps the established Apply -// result alongside an optional request-local preview for construction Apply. -// Accepted is absent for replay or when the sink factory reports capture failure. type ApplyCommandsWithConstructionPreviewResponse struct { Commands *authoringv2.ApplyCommandsResponse Accepted *AcceptedConstruction } -// PreviewInterpretationCandidateRequest identifies an immutable candidate -// revision and the exact draft against which it was proposed. The draft CAS -// is checked before either receipt is compiled, but this operation never -// saves the draft. type PreviewInterpretationCandidateRequest struct { Project string ExplorerID string @@ -340,10 +283,6 @@ const ( CandidatePreviewUnresolved CandidatePreviewState = "UNRESOLVED" ) -// InterpretationCandidatePreviewSample contains one stable output row. The -// Before and After maps are keyed by the physical public column names present -// in the preview, allowing one authored column to retain every indexed or -// repeated physical emission. type InterpretationCandidatePreviewSample struct { RowID string `json:"rowId"` Before map[string]any `json:"before"` @@ -351,8 +290,6 @@ type InterpretationCandidatePreviewSample struct { State CandidatePreviewState `json:"state"` } -// CandidatePreviewCounts describe only the bounded sample, never an -// inferred whole-population count. type CandidatePreviewCounts struct { Compared int `json:"compared"` Changed int `json:"changed"` @@ -371,10 +308,6 @@ type PreviewInterpretationCandidateResult struct { Counts CandidatePreviewCounts `json:"counts"` } -// InterpretationLibraryView expands the mutable library head into the exact -// immutable revision that a client may inspect or propose. The view never -// makes a workspace follow this head; pinned workspaces retain their exact -// revision ID. type InterpretationLibraryView struct { Library explorer.InterpretationLibrary Head *explorer.InterpretationRevision @@ -390,9 +323,6 @@ type GetInterpretationRevisionRequest struct { RevisionID string } -// CreateInterpretationRevisionRequest contains only typed authoring values. -// Author is supplied by the authenticated transport subject, never by the -// browser payload. type CreateInterpretationRevisionRequest struct { Project string LibraryID string @@ -497,9 +427,6 @@ type CellTraceResult struct { Trace dataframeexecution.CellTraceResult `json:"trace"` } -// CellTraceFeature is the receipt-owned descriptor for both explanation and -// repair navigation. AuthoredColumn is the stable Builder key; Column is the -// published physical key and may differ for expanded projections. type CellTraceFeature struct { OutputID string `json:"outputId"` Column string `json:"column"` diff --git a/internal/explorer/lifecycle/unpivot_source.go b/internal/explorer/lifecycle/unpivot_source.go index 2bfecc606..fc3643cf9 100644 --- a/internal/explorer/lifecycle/unpivot_source.go +++ b/internal/explorer/lifecycle/unpivot_source.go @@ -14,15 +14,6 @@ import ( fhirschema "github.com/calypr/loom/internal/fhir/schema" ) -// ConstructionUnpivotSourceCapability advertises the authorized populated root -// scalar fields an UNPIVOT step can melt through signed source choices, beside -// the existing Pivot, Group, Filter, and Derive source inputs. UNPIVOT melts -// value columns into one key/value pair, so only logical kinds the physical -// table compiler accepts as scalar melt inputs qualify — the kinds -// lower.TableReshapeScalarKind maps to a table scalar class, minus the date -// family: the compiler melts date and date_time into the STRING class, but -// that parity is not consent for this lane, which refuses date and date_time -// outright. type ConstructionUnpivotSourceCapability struct { Supported bool `json:"supported"` StageID string `json:"stageId"` @@ -31,13 +22,6 @@ type ConstructionUnpivotSourceCapability struct { Choices []ConstructionUnpivotSourceChoice `json:"choices"` } -// ConstructionUnpivotSourceChoice is pinned to exactly six wire fields. The -// UI capabilities schema is strict, so pivot-style extras (occurrenceId, -// fhirType, valueType, isIdentifier, isReference) must never be added. -// ColumnID is the signed materialized column id from durable server-derived -// state, mirroring pivot's choice.columnId, so the UI can restore a saved -// source-backed UNPIVOT melt input; it is "" while no live materialization -// exists. type ConstructionUnpivotSourceChoice struct { ChoiceID string `json:"choiceId"` FieldPath string `json:"fieldPath"` @@ -47,19 +31,11 @@ type ConstructionUnpivotSourceChoice struct { ColumnID string `json:"columnId"` } -// ConstructionUnpivotSourceSelection binds one server-signed root scalar field -// choice to the exact columnId the edited UNPIVOT step melts as one of its -// value inputs. type ConstructionUnpivotSourceSelection struct { ChoiceID string `json:"choiceId"` ColumnID string `json:"columnId"` } -// validateConstructionUnpivotSourceSelections is the request-boundary guard. -// There is no invented per-request cap: the honest bound is structural, and -// the candidate prep refuses any selection whose columnId is not a declared -// melt input of the edited step, so the request channel can never outgrow what -// one melt consumes. A duplicate choiceId or columnId is refused outright. func validateConstructionUnpivotSourceSelections(selections []ConstructionUnpivotSourceSelection) error { seenChoices := make(map[string]bool, len(selections)) seenColumns := make(map[string]bool, len(selections)) @@ -78,29 +54,11 @@ func validateConstructionUnpivotSourceSelections(selections []ConstructionUnpivo return nil } -// unpivotSourceLogicalTypeIsTableScalar reports whether a root scalar field can -// be melted by UNPIVOT as far as the physical compiler is concerned. The -// decision is delegated verbatim to the compiler's own melt gate, -// lower.TableReshapeScalarKind, which maps the STRING family (string, date, -// date_time, code, uuid) plus integer, decimal, and boolean to physical scalar -// classes and refuses everything else (number, object, and the misspelled -// datetime/date-time aliases). The lane's own admission is -// unpivotSourceLogicalTypeIsMeltEligible, which layers the date-family refusal -// on top of this single delegated decision so no local whitelist can drift -// from the compiler again. func unpivotSourceLogicalTypeIsTableScalar(logicalType string) bool { _, ok := lower.TableReshapeScalarKind(logicalType) return ok } -// unpivotSourceLogicalTypeIsMeltEligible is the unpivot lane's own admission -// of a root scalar field as a signed melt value column. The compiler maps the -// date family to the STRING class, but compiler parity is not consent for this -// channel: a signed date/date_time melt silently loses date semantics into a -// string value column, so the lane refuses it. The canonicalization matches -// the compiler's (trim + lowercase) so no alias can smuggle the date family -// past the exclusion; the delegation still rejects every kind the compiler -// itself refuses. func unpivotSourceLogicalTypeIsMeltEligible(logicalType string) bool { switch strings.ToLower(strings.TrimSpace(logicalType)) { case "date", "date_time": @@ -109,18 +67,6 @@ func unpivotSourceLogicalTypeIsMeltEligible(logicalType string) bool { return unpivotSourceLogicalTypeIsTableScalar(logicalType) } -// unpivotMeltKindsSatisfiable reports whether the melt's value columns can -// share one physical value type. The unpivot lane first refuses the date -// family outright (compiler parity is not consent, see -// unpivotSourceLogicalTypeIsMeltEligible). Every remaining kind is mapped -// through the compiler's authoritative melt gate, -// lower.TableReshapeScalarKind; a melt is unsatisfiable if any input is one -// the compiler refuses as a value column, never silently dropped. The -// remaining cross-class rule is taken verbatim from the compiler's -// compatibility predicate -// (ir.tableScalarKindsCompatible: output == input || (output == "DECIMAL" && -// input == "INTEGER")), which the lowerer applies to every unpivot input -// against the single chosen value-column type. func unpivotMeltKindsSatisfiable(kinds []string) bool { classes := make([]string, 0, len(kinds)) for _, kind := range kinds { @@ -151,11 +97,6 @@ func unpivotMeltKindsSatisfiable(kinds []string) bool { return false } -// unpivotInputColumnIDs returns the column ids an UNPIVOT operation melts into -// the key/value pair. This is the documented divergence from the Derive source -// channel, which binds slots to the two arithmetic operand ids; UNPIVOT melts -// an arbitrary declared set of value inputs and the melt columns themselves -// leave the row schema. func unpivotInputColumnIDs(unpivot *authoringv2.ConstructionUnpivot) []string { if unpivot == nil { return nil @@ -303,16 +244,6 @@ func (s *Service) resolveConstructionUnpivotSource( }, nil } -// constructionCandidateWithUnpivotSources resolves the signed field choices, -// derives the durable server-owned projection facts for the edited UNPIVOT -// step's melt inputs, and carries those slots through the row-preserving -// prefix that feeds the melt. It mirrors the Derive source contract; the -// documented divergences are (1) UNPIVOT melts an arbitrary declared input -// set instead of two operand slots, (2) the melted slots are consumed and -// leave the melt's own output schema, so they are carried only by the steps -// strictly before the melt, and (3) every melt value column must share one -// physical value type, so the joint kind set is checked against the -// compiler's own compatibility rule before the candidate is accepted. func (s *Service) constructionCandidateWithUnpivotSources( ctx context.Context, base constructionBase, @@ -519,13 +450,6 @@ func clearConstructionUnpivotOwnedSources(base, candidate authoringv2.Constructi return candidate, nil } -// constructionOwnedUnpivotSourceColumnID returns the durable server-derived -// materialized column for a root scalar field at one exact stage, mirroring -// constructionOwnedDeriveSourceColumnID. An UNPIVOT owns the slot only while -// an owner UNPIVOT step whose input stage is this exact stage still melts the -// projection column as one of its declared value inputs. A vanished, -// re-pointed, or foreign-owned materialization returns "", so the UI never -// renders a restore into a column the server can no longer sign. func constructionOwnedUnpivotSourceColumnID(construction authoringv2.Construction, stageID, path string) string { for _, projection := range construction.SourceProjections { if projection.OwnerStepID == "" || canonicalRowPath(projection.FieldPath) != canonicalRowPath(path) { @@ -546,12 +470,6 @@ func constructionOwnedUnpivotSourceColumnID(construction authoringv2.Constructio return "" } -// ConstructionUnpivotSourceFacts derives the server-owned projection facts for -// one root scalar field from the authorized snapshot plus generated FHIR -// metadata. Apply reauthorization and receipt reconcile share this single -// derivation so a persisted unpivot-owned claim can never drift from the facts -// the server would have signed it from. Column and owner identities stay -// claim-sourced, exactly as the Apply-side DeepEquals expects them. func ConstructionUnpivotSourceFacts(snapshot capability.Snapshot, rootResourceType, fieldPath string) (authoringv2.ConstructionSourceProjection, bool) { path := canonicalRowPath(fieldPath) if strings.TrimSpace(rootResourceType) == "" || path == "" { @@ -609,9 +527,6 @@ func isUnpivotOwnedSourceProjection(construction authoringv2.Construction, proje containsConstructionColumnIDLocal(unpivotInputColumnIDs(step.Operation.Unpivot), projection.ColumnID) } -// reauthorizeConstructionUnpivotSources re-derives the server-owned facts for -// every unpivot-owned projection from the live authorized snapshot so an Apply -// cannot stage a melt whose field stopped qualifying after the proposal. func (s *Service) reauthorizeConstructionUnpivotSources(ctx context.Context, base constructionBase, outputID string, construction authoringv2.Construction) error { if s.config.RowChoicePlanner == nil { return unavailable("commands", "ROW_CHOICE_UNAVAILABLE", "authorized unpivot source choice planning is not configured", nil) diff --git a/internal/explorer/lifecycle/unpivot_source_test.go b/internal/explorer/lifecycle/unpivot_source_test.go index 230d99efb..64ba80415 100644 --- a/internal/explorer/lifecycle/unpivot_source_test.go +++ b/internal/explorer/lifecycle/unpivot_source_test.go @@ -54,9 +54,6 @@ func unpivotMeltOutputs(extra ...authoringv2.StageColumn) []authoringv2.StageCol return outputs } -// unpivotSourceBSSnapshot reuses the BodyStructure fixture where `active` -// (boolean) and `id` (string) are populated scalars, `description` is an -// unpopulated string, and `patient` is a reference. func unpivotSourceBSSnapshot(token string) capability.Snapshot { return deriveSourceSnapshot(token) } @@ -89,8 +86,6 @@ func unpivotBSSelections(t *testing.T, snapshot capability.Snapshot, path, colum return ConstructionUnpivotSourceSelection{ChoiceID: deriveSourceRowChoice(t, snapshot, deriveSourceRootType, path).ChoiceID, ColumnID: columnID} } -// (a) Discovery offers populated root scalars of the four compiler melt kinds -// and never the unpopulated or reference fields. func TestConstructionUnpivotSourceCapabilityExposesPopulatedUnselectedRootScalars(t *testing.T) { snapshot := unpivotSourceBSSnapshot("unpivot-source-bs") service := deriveSourceService(t) @@ -132,7 +127,6 @@ func TestConstructionUnpivotSourceCapabilityExposesPopulatedUnselectedRootScalar } } -// Discovery reports the honest reason when no populated melt kind exists. func TestConstructionUnpivotSourceCapabilityReportsNoEligibleFields(t *testing.T) { snapshot := unpivotSourceEmptySnapshot("unpivot-source-empty") service := deriveSourceService(t) @@ -146,10 +140,6 @@ func TestConstructionUnpivotSourceCapabilityReportsNoEligibleFields(t *testing.T } } -// The kind gate is pure logic and must admit exactly the compiler melt class: -// lower.TableReshapeScalarKind maps STRING/DATE/DATE_TIME/CODE/UUID to the -// STRING physical class, so date, date_time, code, and uuid root scalars are -// legitimate melt value columns (finding F-1). func TestUnpivotSourceLogicalTypeIsTableScalarAdmitsOnlyCompilerKinds(t *testing.T) { for _, want := range []string{"string", "integer", "decimal", "boolean", "date", "date_time", "code", "uuid", " STRING", "Decimal ", "BOOLEAN", " Date "} { if !unpivotSourceLogicalTypeIsTableScalar(want) { @@ -163,10 +153,6 @@ func TestUnpivotSourceLogicalTypeIsTableScalarAdmitsOnlyCompilerKinds(t *testing } } -// The lane's own admission = compiler delegation minus the date family -// (operator decision #4). The trim+lowercase canonicalization matches the -// compiler's, so no alias reaches an eligible verdict, and every kind the -// compiler refuses stays refused. func TestUnpivotSourceMeltEligibilityExcludesDateFamily(t *testing.T) { for _, want := range []string{"string", "integer", "decimal", "boolean", "code", "uuid", " STRING", "Decimal ", "BOOLEAN"} { if !unpivotSourceLogicalTypeIsMeltEligible(want) { @@ -180,9 +166,6 @@ func TestUnpivotSourceMeltEligibilityExcludesDateFamily(t *testing.T) { } } -// Permanent whitelist-drift guard: the signed channel's melt gate is the -// compiler decision itself. This set-equality table fails if anyone adds a -// local kind set back to either side. func TestUnpivotSourceWhitelistIsTheCompilerDecision(t *testing.T) { corpus := map[string]struct { admitted bool @@ -215,13 +198,6 @@ func TestUnpivotSourceWhitelistIsTheCompilerDecision(t *testing.T) { } } -// The joint satisfiability rule is the compiler's own compatibility predicate -// applied to the physical class each logical kind lowers to. An unsatisfiable -// melt must be refused by class, never by silently dropping kinds the channel -// has not yet classified. On top of that, the unpivot lane refuses the date -// family outright (operator decision #4): the compiler maps date and -// date_time to the STRING class, but compiler parity is not consent for this -// channel, so any melt carrying a date-family kind is unsatisfiable. func TestUnpivotMeltKindsSatisfiableFollowsCompilerRule(t *testing.T) { for _, ok := range [][]string{nil, {}, {"integer"}, {"decimal"}, {"integer", "decimal"}, {"decimal", "integer"}, {" string "}, {"boolean"}, {"BOOLEAN", "boolean"}, {"code", "uuid", "string"}, {"Code", " STRING"}} { if !unpivotMeltKindsSatisfiable(ok) { @@ -235,7 +211,6 @@ func TestUnpivotMeltKindsSatisfiableFollowsCompilerRule(t *testing.T) { } } -// (c) A token from another snapshot, or a tampered token, is refused. func TestConstructionUnpivotSourceRefusesForgedAndCrossSnapshotChoices(t *testing.T) { snapshot := unpivotSourceBSSnapshot("unpivot-source-current") other := unpivotSourceBSSnapshot("unpivot-source-other") @@ -269,9 +244,6 @@ func TestConstructionUnpivotSourceRefusesForgedAndCrossSnapshotChoices(t *testin } } -// The melt-kind gate is enforced on admission, not only display: the planner -// still issues a signed token for a scalar date field, so the resolution -// boundary must refuse it. func unpivotNumberCandidate() capability.Candidate { return capability.Candidate{ ID: "candidate-deceased-number", NodeID: "node-patient", ResourceType: "Patient", FieldPath: "deceasedBoolean", @@ -282,9 +254,6 @@ func unpivotNumberCandidate() capability.Candidate { } } -// A declared number aggregate is a construction source type but the physical -// table compiler refuses it as a melt value column, so admission (and not a -// later compile failure) must refuse it by name. func TestConstructionUnpivotSourceRejectsNonScalarKindOnAdmission(t *testing.T) { harness := newUnpivotSourceHarness(t) snapshot := harness.snapshot @@ -305,11 +274,6 @@ func TestConstructionUnpivotSourceRejectsNonScalarKindOnAdmission(t *testing.T) } } -// Operator decision #4: the unpivot melt lane refuses the date family even -// though the compiler maps it to the STRING class. The planner still issues -// a signed token for a scalar date/date_time field, so admission must refuse -// it by name rather than fabricate an accepted melt the compiler would then -// build with silently lossy date-to-string semantics. func TestConstructionUnpivotSourceRefusesDateFamilyOnAdmission(t *testing.T) { for _, row := range []struct { id, fieldPath, label, logicalType string @@ -340,9 +304,6 @@ func TestConstructionUnpivotSourceRefusesDateFamilyOnAdmission(t *testing.T) { } } -// Operator decision #4: the discovery list must hide the date family, not -// advertise a field that admission then refuses. Permitted melt kinds stay -// advertised — the integer root scalar is the regression lock on this side. func TestConstructionUnpivotSourceCapabilityExcludesDateMeltKind(t *testing.T) { harness := newUnpivotSourceHarness(t) snapshot := harness.snapshot @@ -380,8 +341,6 @@ func TestConstructionUnpivotSourceCapabilityExcludesDateMeltKind(t *testing.T) { } } -// (b) A valid selection durably owns exactly one server-derived projection, -// the melt's carrier steps list the slot, and the melt's own outputs do not. func TestConstructionCandidateWithUnpivotSourcesMaterializesAndCarriesPrefix(t *testing.T) { harness := newUnpivotSourceHarness(t) snapshot := harness.snapshot @@ -452,7 +411,6 @@ func TestConstructionCandidateWithUnpivotSourcesMaterializesAndCarriesPrefix(t * } } -// The row-preserving prefix allow-list is enforced, not trust-me coded. func TestConstructionUnpivotSourceCannotCrossRowChangingPrefix(t *testing.T) { harness := newUnpivotSourceHarness(t) group := filterSourceGroupStep() @@ -484,9 +442,6 @@ func TestConstructionUnpivotSourceCannotCrossRowChangingPrefix(t *testing.T) { } } -// A selection the melt does not declare as a value input is refused, and the -// joint scalar-kind rule refuses a melt outside the compiler's compatibility -// predicate. func TestConstructionUnpivotSourceColumnMismatchAndIncompatibleKinds(t *testing.T) { snapshot := unpivotSourceBSSnapshot("unpivot-source-kinds") service := deriveSourceService(t) @@ -535,8 +490,6 @@ func TestConstructionUnpivotSourceColumnMismatchAndIncompatibleKinds(t *testing. } } -// A signed slot whose field is already an authored column of the table is -// refused with the shared refuse-not-dedupe decision. func TestConstructionUnpivotSourceRefusesSecondCarrierOfSameField(t *testing.T) { snapshot := unpivotSourceBSSnapshot("unpivot-source-collision") service := deriveSourceService(t) @@ -574,7 +527,6 @@ func TestConstructionUnpivotSourceRefusesSecondCarrierOfSameField(t *testing.T) } } -// Dropping the selections strips the projections from the carriers. func TestConstructionUnpivotSourceRemovalStripsProjection(t *testing.T) { harness := newUnpivotSourceHarness(t) snapshot := harness.snapshot @@ -619,7 +571,6 @@ func TestConstructionUnpivotSourceRemovalStripsProjection(t *testing.T) { } } -// An omitted unpivotSources channel is a no-op on the candidate. func TestConstructionUnpivotSourceLeavesSelectedColumnBehaviorUnchanged(t *testing.T) { harness := newUnpivotSourceHarness(t) construction := authoringv2.Construction{Version: authoringv2.ConstructionVersion, Steps: []authoringv2.ConstructionStep{ @@ -639,7 +590,6 @@ func TestConstructionUnpivotSourceLeavesSelectedColumnBehaviorUnchanged(t *testi } } -// Duplicate or blank selection facts are refused at the request boundary. func TestConstructionUnpivotSourceSelectionsRequireExactDeduplicatedFacts(t *testing.T) { duplicate := unpivotPatientProposalRequest(unpivotSourceStep, []ConstructionUnpivotSourceSelection{ {ChoiceID: "rc1.a.b", ColumnID: "c1"}, {ChoiceID: "rc1.a.b", ColumnID: "c2"}, @@ -664,7 +614,6 @@ func TestConstructionUnpivotSourceSelectionsRequireExactDeduplicatedFacts(t *tes } } -// (g) The advertised choice object is EXACTLY the six pinned wire fields. func TestConstructionUnpivotSourceChoiceWireShapeIsExactlySixFields(t *testing.T) { wantFields := map[string]bool{"choiceId": true, "fieldPath": true, "label": true, "logicalType": true, "isPopulated": true, "columnId": true} structFields := make(map[string]bool, reflect.TypeOf(ConstructionUnpivotSourceChoice{}).NumField()) @@ -695,8 +644,6 @@ func TestConstructionUnpivotSourceChoiceWireShapeIsExactlySixFields(t *testing.T } } -// The advertised columnId only survives while the owner melt still declares -// this exact slot as a value input at this exact stage. func TestConstructionUnpivotSourceCapabilityAdvertisesDurableColumnIdOnlyWhileSigned(t *testing.T) { harness := newUnpivotSourceHarness(t) snapshot := harness.snapshot @@ -757,8 +704,6 @@ func TestConstructionUnpivotSourceCapabilityAdvertisesDurableColumnIdOnlyWhileSi } } -// (a, wiring) GetConstructionCapabilities assembles unpivotSourceInput beside -// the pivot, filter, and derive source inputs. func TestGetConstructionCapabilitiesAssemblesUnpivotSourceInputBesideDeriveSourceInput(t *testing.T) { harness := newUnpivotSourceHarness(t) capabilities, err := harness.service.GetConstructionCapabilities(context.Background(), ConstructionCapabilitiesRequest{ @@ -800,8 +745,6 @@ func TestGetConstructionCapabilitiesAssemblesUnpivotSourceInputBesideDeriveSourc } } -// (b) Through ProposeConstruction with the real compiler, the accepted -// candidate melts the unselected field's compiled slot. func TestProposeConstructionUnpivotSourceMeltsUnselectedFieldWithRealCompiler(t *testing.T) { harness := newUnpivotSourceHarness(t) carriers := append(deriveSourcePatientIdentityColumns(), authoringv2.StageColumn{ID: "derived_total", Name: "derived_total", Label: "Derived total", Type: "integer"}) @@ -878,9 +821,6 @@ func TestProposeConstructionUnpivotSourceMeltsUnselectedFieldWithRealCompiler(t } } -// unpivotSaveDraftPublicNote saves the draft with one extra public authored -// field column (Patient.gender, string) so a signed STRING-class melt input -// has a compiler-compatible public partner without touching shared fixtures. func unpivotSaveDraftPublicNote(t *testing.T, harness *unpivotSourceHarness, construction authoringv2.Construction) { t.Helper() workspace, err := authoringv2.DecodeWorkspace(harness.store.created.DraftConfig) @@ -916,11 +856,6 @@ func unpivotSaveDraftPublicNote(t *testing.T, harness *unpivotSourceHarness, con harness.store.created.DraftConfig, harness.store.created.DraftDigest = draft, digest } -// Operator decision #4 accept-side: the durable proposal path (real compiler -// wiring) must refuse a signed date/date_time melt with the named code, not -// preview it. Before the refusal shipped this reached PREVIEW_PENDING at 200 -// (authority /tmp/opencode/jrev/probe-date-melt-dp4.log; finding F-1's -// accept-side lock was inverted by the operator ruling). func TestProposeConstructionUnpivotSourceRefusesDateFamilyWithRealCompiler(t *testing.T) { for _, row := range []struct { id, fieldPath, label, logicalType string @@ -986,8 +921,6 @@ func TestProposeConstructionUnpivotSourceRefusesDateFamilyWithRealCompiler(t *te } } -// Operator decision #4 regression lock, same harness shape as the refusal -// above: the permitted STRING class still melts through the real compiler. func TestProposeConstructionUnpivotSourceStillMeltsSignedStringWithRealCompiler(t *testing.T) { harness := newUnpivotSourceHarness(t) snapshot := harness.snapshot @@ -1053,10 +986,6 @@ func TestProposeConstructionUnpivotSourceStillMeltsSignedStringWithRealCompiler( } } -// Operator decision #4 joint gate: an integer+date melt must be refused by -// name. The per-selection date admission refusal fires before the class -// compatibility rule for the signed date field, so the honest code is the -// date-family refusal, not the generic UNPIVOT_SOURCE_TYPES_INCOMPATIBLE. func TestProposeConstructionUnpivotSourceRefusesIntegerWithDateMelt(t *testing.T) { harness := newUnpivotSourceHarness(t) snapshot := harness.snapshot @@ -1098,7 +1027,6 @@ func TestProposeConstructionUnpivotSourceRefusesIntegerWithDateMelt(t *testing.T } } -// (6) An accepted proposal whose field stops qualifying is refused at Apply. func TestApplyConstructionProposalRefusesStaleUnpivotSourceBinding(t *testing.T) { for _, stale := range []bool{false, true} { name := "current" @@ -1197,8 +1125,6 @@ func decodeConstruction(construction authoringv2.Construction) (authoringv2.Cons return clone, nil } -// --- ROW-SOURCE-03 S-A: precise per-skipped-class lane reason codes ------- - func TestConstructionUnpivotSourceCapabilityReportsPreciseSkipClassReasonCodes(t *testing.T) { index, err := fhirschema.GeneratedIndex() if err != nil { @@ -1409,17 +1335,6 @@ func TestConstructionUnpivotSourceCapabilityNamesInventoryFailureDistinctly(t *t } } -// The unpivot lane's advertised offer label is produced by the shared lane -// label builder (constructionFieldDisplayLabel, construction_pivot_source.go): -// the authorized candidate's schema-Title label wins over the planner -// row-choice display, a blank candidate falls through to the choice display, -// and a candidate carrying a Unicode control byte is refused (skipped) so the -// builder never returns control bytes for this lane. The builder's own -// refusal is unit-pinned in construction_pivot_source_test.go and the -// expand lane's candidate-wins ordering in expand_source_test.go; this is -// the only check that the unpivot offer call site (unpivot_source.go) -// routes candidate and choice displays through the builder, and the only -// per-call-site pin of the blank and control-byte fall-through ordering. func TestConstructionUnpivotSourceAdvertisedLabelsFollowBuilderPrecedence(t *testing.T) { baseCase := func(t *testing.T) (capability.Snapshot, string) { t.Helper() diff --git a/internal/explorer/output_contract.go b/internal/explorer/output_contract.go index ca609121a..bfb3780f5 100644 --- a/internal/explorer/output_contract.go +++ b/internal/explorer/output_contract.go @@ -14,14 +14,10 @@ import ( "github.com/calypr/loom/internal/explorer/capability" ) -// PublicOutputContracts is the current, transport-neutral public output -// contract persisted inside a compilation receipt. Outputs retain workspace -// order; executable selectors and physical names are deliberately absent. type PublicOutputContracts struct { Outputs []PublicOutputContract `json:"outputs"` } -// PublicOutputContract describes one compiled workspace output. type PublicOutputContract struct { OutputID string `json:"outputId"` RootResourceType string `json:"rootResourceType,omitempty"` @@ -57,8 +53,6 @@ type PublicOutputColumn struct { Chartable bool `json:"chartable"` UnitNormalization *PublicUnitNormalization `json:"unitNormalization,omitempty"` - // Compiler identities remain available to internal legacy tests only. They - // are deliberately absent from the V2 public contract. EmissionID string `json:"-"` PublicColumn string `json:"-"` CandidateID string `json:"-"` @@ -76,10 +70,6 @@ type PublicUnitNormalization struct { Rules []PublicUnitRuleIdentity `json:"rules"` } -// DecodePublicOutputContracts strictly decodes the current contract shape. -// Null columns are accepted because an empty V2 selection is a valid mutable -// editor state and the current compiler serializes its nil slice as null. -// Missing fields, unknown fields, duplicate keys, and trailing JSON are not accepted. func DecodePublicOutputContracts(raw json.RawMessage) (PublicOutputContracts, error) { if len(bytes.TrimSpace(raw)) == 0 || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { return PublicOutputContracts{}, invalidOutputContract("contract is required") @@ -158,8 +148,6 @@ func (c PublicOutputContracts) Output(outputID string) (PublicOutputContract, bo return PublicOutputContract{}, false } -// ValidateAgainst proves that the ordered contract set exactly matches the -// recipe outputs and each output's ordered emitted columns. func (c PublicOutputContracts) ValidateAgainst(bundle recipe.Bundle, emitted []EmittedColumn) error { if len(c.Outputs) != len(bundle.Outputs) { return invalidOutputContract("output count %d does not match recipe output count %d", len(c.Outputs), len(bundle.Outputs)) @@ -186,10 +174,6 @@ func (c PublicOutputContracts) ValidateAgainst(bundle recipe.Bundle, emitted []E return nil } -// ValidateAgainst proves that the typed public contract is exactly the -// compiler-owned emitted-column sequence for the receipt's bundle. It checks -// output identity, order, IDs, and every public metadata bit. A mismatch is a -// receipt integrity failure and therefore requires explicit recompilation. func (c PublicOutputContract) ValidateAgainst(bundle recipe.Bundle, emitted []EmittedColumn) error { if strings.TrimSpace(c.OutputID) == "" { return invalidOutputContract("outputId is required") diff --git a/internal/explorer/selection.go b/internal/explorer/selection.go index f134d5883..fdc3c9b23 100644 --- a/internal/explorer/selection.go +++ b/internal/explorer/selection.go @@ -27,9 +27,6 @@ var ( ErrResourceRefScopeMismatch = errors.New("resource reference scope mismatch") ) -// ResourceRef is the only identity accepted by a selection. A display label, -// table row number, or serialized query value is deliberately not a resource -// reference. type ResourceRef struct { Project string `json:"project"` Generation string `json:"generation"` @@ -66,13 +63,8 @@ func (r ResourceRef) key() string { return r.Project + "\x00" + r.Generation + "\x00" + r.ResourceType + "\x00" + r.ID } -// SelectionFilter is the existing published-reader filter contract. Keeping -// one predicate vocabulary prevents selection from persisting AQL/GraphQL -// fragments or inventing a second operator representation. type SelectionFilter = published.Filter -// SelectionRule describes intent, not the result membership. Reapplying a -// rule always creates a new immutable revision. type SelectionRule struct { Kind string `json:"kind"` Filters []SelectionFilter `json:"filters,omitempty"` @@ -262,11 +254,6 @@ func SelectionRuleDigest(rule SelectionRule, source SelectionSource, exclusions return SelectionRuleDigestWithMembers(rule, source, exclusions, nil) } -// SelectionRuleDigestWithMembers binds explicit request membership into the -// immutable intent digest. It is intentionally separate from the final -// MembershipDigest: exclusions can make two requests produce the same final -// rows while their requested inputs remain different and must conflict under -// one idempotency key. func SelectionRuleDigestWithMembers(rule SelectionRule, source SelectionSource, exclusions []ResourceRef, members []ResourceRef) (string, error) { rule = rule.Canonical() source = source.Canonical() @@ -301,9 +288,6 @@ func resourceRefsToMembers(refs []ResourceRef) []SelectionMember { return members } -// MembershipDigest computes an immutable digest over canonical, -// deduplicated typed references. Storage adapters use the same framing while -// streaming indexed member records. func MembershipDigest(members []SelectionMember) string { refs := make([]ResourceRef, 0, len(members)) seen := make(map[string]struct{}, len(members)) @@ -323,8 +307,6 @@ func MembershipDigest(members []SelectionMember) string { return hex.EncodeToString(hash.Sum(nil)) } -// WriteMembershipFrame writes an unambiguous length-prefixed identity frame. -// It is shared by lifecycle and persistence finalizers. func WriteMembershipFrame(w interface{ Write([]byte) (int, error) }, ref ResourceRef) { ref = ref.Canonical() for _, value := range []string{ref.Project, ref.Generation, ref.ResourceType, ref.ID} { diff --git a/internal/explorer/service.go b/internal/explorer/service.go index 30c50f815..def22864c 100644 --- a/internal/explorer/service.go +++ b/internal/explorer/service.go @@ -17,8 +17,6 @@ import ( "github.com/calypr/loom/internal/projectid" ) -// Service is the persistence-neutral lifecycle coordinator. Compiler and -// publication adapters are intentionally injected by server wiring. type Service struct { store Store now func() time.Time @@ -37,10 +35,6 @@ func (s *Service) Get(ctx context.Context, project, id string) (*Explorer, error return s.store.Get(ctx, projectid.Legacy(project), id) } -// MigrateLegacyConstructionSteps normalizes drafts written when a nil empty -// construction sequence serialized as null. It verifies that the stored -// digest matches that exact legacy representation, then records the canonical -// array representation through the normal draft compare-and-swap path. func (s *Service) MigrateLegacyConstructionSteps(ctx context.Context, project, id string, expectedVersion int64, expectedDigest string) (*Explorer, error) { owner, err := s.Get(ctx, project, id) if err != nil { @@ -72,11 +66,6 @@ func (s *Service) MigrateLegacyConstructionSteps(ctx context.Context, project, i return s.store.SaveDraft(ctx, *owner, expectedVersion, expectedDigest) } -// LoadExplorerState returns the one canonical response for the public -// Explorer GET route. It loads the identity and immutable active revision -// together, then derives the renderer-facing projection from that revision. -// The HTTP layer should not reconstruct this response from individual legacy -// adapters. func (s *Service) LoadExplorerState(ctx context.Context, project, id string) (ExplorerStateV1, error) { project = projectid.Canonical(project) id = strings.TrimSpace(id) @@ -158,9 +147,6 @@ func explorerURL(project, id string) string { return "/api/v1/projects/" + url.PathEscape(project) + "/explorers/" + url.PathEscape(id) } -// createEmptyInteractive creates only the Explorer identity. Unpublished -// Explorers have no persisted authoring content and hydrate as an empty valid -// Builder model. func (s *Service) createEmptyInteractive(ctx context.Context, project, id, title, actor string) (*Explorer, error) { if id == "" || id == "default" { return nil, fmt.Errorf("invalid interactive Explorer identity") @@ -168,9 +154,6 @@ func (s *Service) createEmptyInteractive(ctx context.Context, project, id, title return s.store.Create(ctx, Explorer{Project: projectid.Legacy(project), ExplorerID: id, Title: title, ManagementMode: ManagementInteractive, UpdatedBy: actor, UpdatedAt: s.now()}) } -// CreateInteractiveFrom creates a new Explorer identity and, when requested, -// seeds it from a server-owned source workspace without asking the browser to -// reconstruct or re-identify that workspace. func (s *Service) CreateInteractiveFrom(ctx context.Context, project, id, title, sourceExplorerID, actor string) (*Explorer, error) { if strings.TrimSpace(sourceExplorerID) == "" { return s.createEmptyInteractive(ctx, project, id, title, actor) @@ -209,18 +192,10 @@ func (s *Service) CreateInteractiveFrom(ctx context.Context, project, id, title, return s.store.Create(ctx, Explorer{Project: projectid.Legacy(project), ExplorerID: id, Title: title, ManagementMode: ManagementInteractive, DraftConfig: canonical, DraftVersion: 1, DraftDigest: digest, UpdatedBy: actor, UpdatedAt: s.now()}) } -// ApplyWorkspaceCommands is the authoritative Builder mutation boundary. It -// resolves backend-owned identities, applies a command batch atomically, and -// persists both the new draft and the bounded last-command replay record in -// one compare-and-swap. Older command IDs are rejected by draft CAS. func (s *Service) ApplyWorkspaceCommands(ctx context.Context, project, id string, catalog authoringv2.CatalogSnapshot, request authoringv2.ApplyCommandsRequest, actor string) (*authoringv2.ApplyCommandsResponse, error) { return s.applyWorkspaceCommands(ctx, project, id, catalog, request, actor, nil, nil) } -// ApplyWorkspaceCommandsChecked is the mutation boundary for workflows that -// need external immutable references resolved and validated before the draft -// CAS. The preparer runs against the single loaded workspace before reduction; -// the checker runs against the fully reduced workspace before SaveDraft. func (s *Service) ApplyWorkspaceCommandsChecked(ctx context.Context, project, id string, catalog authoringv2.CatalogSnapshot, request authoringv2.ApplyCommandsRequest, actor string, prepare func(context.Context, authoringv2.Workspace, []authoringv2.Command) ([]authoringv2.Command, error), checker func(authoringv2.Workspace) error) (*authoringv2.ApplyCommandsResponse, error) { return s.applyWorkspaceCommands(ctx, project, id, catalog, request, actor, prepare, checker) } @@ -351,9 +326,6 @@ func (s *Service) FailRevision(ctx context.Context, id string, diagnostics []Dia return s.store.FailRevision(ctx, id, diagnostics) } -// CompilationReceiptForExplorer performs the tenant-scoped lookup used by -// execution and publication. The ID-only method above remains for legacy -// callers during the receipt migration. func (s *Service) CompilationReceiptForExplorer(ctx context.Context, project, explorerID, id string) (*CompilationReceipt, error) { return s.store.GetCompilationReceiptForExplorer(ctx, projectid.Canonical(project), explorerID, id) } @@ -394,9 +366,6 @@ func (s *Service) GetRevision(ctx context.Context, revisionID string) (*Revision return s.store.GetRevision(ctx, strings.TrimSpace(revisionID)) } -// GetDraftRevision performs the tenant and Explorer scoped lookup required by -// the Builder's server-resolved restore command. Stores without draft history -// support cannot authorize a browser-supplied revision ID. func (s *Service) GetDraftRevision(ctx context.Context, project, explorerID, revisionID string) (*DraftRevision, error) { reader, ok := s.store.(DraftRevisionReader) if !ok { @@ -409,8 +378,6 @@ func (s *Service) VisitSelectionMembers(ctx context.Context, project, selectionI return s.store.VisitSelectionMembers(ctx, projectid.Canonical(project), selectionID, after, pageSize, visit) } -// PublishAuthoring atomically stores the receipt and immutable revision, then -// switches both the dataset release and Explorer active pointers. func (s *Service) PublishAuthoring(ctx context.Context, receipt CompilationReceipt, revision Revision, release dataset.ProjectRelease, expectedReleaseRevision int64) (*Revision, error) { if receipt.ID == "" || revision.ID == "" || revision.CompilationReceiptID != receipt.ID { return nil, fmt.Errorf("authoring publication identity is incomplete") @@ -420,11 +387,6 @@ func (s *Service) PublishAuthoring(ctx context.Context, receipt CompilationRecei return s.store.PublishAuthoring(ctx, receipt, revision, release, expectedReleaseRevision) } -// UpsertRepositoryV2 records the repository-owned default and its immutable -// receipt-backed ready revision after dataframe publication has succeeded. -// Activation is intentionally a separate call so callers can compose it with -// the dataset release switch where the durable adapter supports that atomic -// transaction. func (s *Service) UpsertRepositoryV2(ctx context.Context, receipt CompilationReceipt, sourceCommit, actor string, materializations []Materialization, dataset DatasetMetadata, qualityReports []publication.QualityReport, publicationMetadata PublicationMetadata) (*Explorer, *Revision, error) { if err := receipt.Validate(); err != nil { return nil, nil, fmt.Errorf("invalid repository compilation receipt: %w", err) @@ -468,10 +430,6 @@ func (s *Service) ActivateRepositoryGeneration(ctx context.Context, project, gen return s.store.ActivateRepositoryGeneration(ctx, projectid.Legacy(project), generation, revisionID) } -// RepositoryRevisionID is stable for retrying the same checked-out repository -// content and immutable compilation receipt against the same Loom generation. -// Including the receipt identity prevents a compiler-contract upgrade from -// colliding with a revision produced under older lowering semantics. func RepositoryRevisionID(project, sourceCommit, definitionDigest, sourceGeneration string, receiptIdentity ...string) string { artifact := "" if len(receiptIdentity) > 0 { diff --git a/internal/explorer/state_v1.go b/internal/explorer/state_v1.go index 3557dcc98..56fe2cf17 100644 --- a/internal/explorer/state_v1.go +++ b/internal/explorer/state_v1.go @@ -35,10 +35,7 @@ type ExplorerStateV1 struct { Draft ExplorerStateV1Draft `json:"draft"` Active ExplorerStateV1Active `json:"active"` Generated ExplorerStateV1Generated `json:"generated"` - // Runtime is explicit so an Explorer with no executable query is still a - // valid response. Clients can inspect generated.publication.state or the - // null runtime instead of treating a missing field as a malformed payload. - Runtime *ExplorerRuntimeV1 `json:"runtime"` + Runtime *ExplorerRuntimeV1 `json:"runtime"` ActiveURL string `json:"activeUrl"` UpdatedBy string `json:"updatedBy,omitempty"` diff --git a/internal/explorer/store.go b/internal/explorer/store.go index 332d4636c..eb3c4b1a9 100644 --- a/internal/explorer/store.go +++ b/internal/explorer/store.go @@ -15,21 +15,13 @@ var ( ErrImmutableRevision = errors.New("explorer revision content is immutable") ErrStaleCatalog = errors.New("stale Explorer authoring catalog") ErrIncompleteCatalog = errors.New("Explorer catalog discovery is incomplete") - // ErrCorruptReceipt means that a persisted receipt does not match its - // content-addressed identity, or that an immutable key was reused for a - // different receipt. Callers must fail closed rather than execute it. - ErrCorruptReceipt = errors.New("corrupt compilation receipt") + ErrCorruptReceipt = errors.New("corrupt compilation receipt") ) -// Store deliberately exposes only lifecycle mutations for revisions. An -// adapter must never replace definition, recipe, digests, source generation, -// or frozen materialization mappings after InsertRevision succeeds. type Store interface { List(context.Context, string) ([]Explorer, error) Get(context.Context, string, string) (*Explorer, error) Create(context.Context, Explorer) (*Explorer, error) - // SaveDraft is an optimistic compare-and-swap. Adapters must reject a - // mismatched expected version/digest and increment the persisted version. SaveDraft(context.Context, Explorer, int64, ...string) (*Explorer, error) InsertCompilationReceipt(context.Context, CompilationReceipt) (*CompilationReceipt, error) GetCompilationReceiptForExplorer(context.Context, string, string, string) (*CompilationReceipt, error) @@ -48,16 +40,10 @@ type Store interface { VisitSelectionMembers(context.Context, string, string, string, int, func(SelectionMember) error) (string, error) } -// DraftRevisionReader is the narrow history lookup used by the server-side -// restore command. It is optional on older Store implementations; accepting -// a revision ID never grants access to a draft outside its project/Explorer. type DraftRevisionReader interface { GetDraftRevision(context.Context, string, string, string) (*DraftRevision, error) } -// InterpretationRepository is intentionally narrower than Store. Lifecycle -// code that resolves an exact interpretation revision should not depend on -// unrelated Explorer draft, receipt, or publication methods. type InterpretationRepository interface { ListInterpretationLibraries(context.Context, string) ([]InterpretationLibrary, error) GetInterpretationRevision(context.Context, string, InterpretationRevisionID) (*InterpretationRevision, error) diff --git a/internal/explorer/tableshapecap/identity.go b/internal/explorer/tableshapecap/identity.go index 419df0018..6acdcaacd 100644 --- a/internal/explorer/tableshapecap/identity.go +++ b/internal/explorer/tableshapecap/identity.go @@ -108,9 +108,6 @@ func NewCategoryScanReceipt(binding Binding, parentCatalogID, categoryColumnChoi return receipt, nil } -// NewPivotDerivedOperandChoiceID creates an opaque choice scoped to the exact -// post-pivot output mapping. The pivot's own ID is intentionally excluded to -// avoid a content-addressing cycle; every other semantic pivot field is bound. func NewPivotDerivedOperandChoiceID(parentCatalogID string, pivot PivotResolution, output NamedOutput, fact TypeFact) (string, error) { pivot.DerivedOperands = nil payload, err := canonical(struct { @@ -940,9 +937,6 @@ func categoryValuesDigest(values []Scalar) (string, error) { return digest(payload), nil } -// CategoryValuesDigest computes the canonical ordered-membership digest used -// in compiler scan proofs. The caller supplies the authoritative scan facts; -// this helper only proves the frozen values match that proof. func CategoryValuesDigest(values []Scalar) (string, error) { for _, value := range values { if err := value.Validate(); err != nil { diff --git a/internal/explorer/tableshapecap/repository.go b/internal/explorer/tableshapecap/repository.go index 1134dbe21..eec28e2c3 100644 --- a/internal/explorer/tableshapecap/repository.go +++ b/internal/explorer/tableshapecap/repository.go @@ -2,11 +2,6 @@ package tableshapecap import "context" -// Repository persists immutable, tenant- and parent-scoped artifacts. Put is -// create-once: an existing ID is returned only when its canonical content is -// identical. GetCatalog verifies a complete binding; GetCatalogForLookup -// scopes retrieval to the current request identity known before loading the -// persisted base receipt referenced by that binding. type Repository interface { PutCatalog(context.Context, CatalogReceipt) (CatalogReceipt, error) GetCatalog(context.Context, Binding, string) (CatalogReceipt, error) diff --git a/internal/explorer/tableshapecap/types.go b/internal/explorer/tableshapecap/types.go index d422d7740..9d0c132c9 100644 --- a/internal/explorer/tableshapecap/types.go +++ b/internal/explorer/tableshapecap/types.go @@ -1,6 +1,3 @@ -// Package tableshapecap contains immutable, compiler-neutral capability -// artifacts used to author table-shape proposals. It deliberately does not -// depend on transport, FHIR schema, or persisted authoring-model packages. package tableshapecap import ( @@ -36,8 +33,6 @@ type Binding struct { CompilerSchemaDigest string `json:"compilerSchemaDigest"` } -// CatalogLookup is the request-bound subset available before loading the -// persisted base compilation receipt referenced by a catalog binding. type CatalogLookup struct { Project string `json:"project"` ExplorerID string `json:"explorerId"` @@ -200,8 +195,6 @@ func (a RoleAvailability) Validate() error { return nil } -// Scalar is a strict tagged union. Pointer payloads preserve empty string, -// zero, and false as values distinct from NULL and MISSING. type Scalar struct { Kind ScalarKind `json:"kind"` String *string `json:"string,omitempty"` @@ -287,9 +280,6 @@ type OperandKind string const OperandColumn OperandKind = "COLUMN" -// OperandRef is the immutable server payload behind a catalog's opaque base -// operand choice. Derived receipts reference its choice ID, never a client -// supplied column key. type OperandRef struct { Kind OperandKind `json:"kind"` ColumnKey string `json:"columnKey"` @@ -310,9 +300,6 @@ const ( ResolvedOperandLiteral ResolvedOperandKind = "LITERAL" ) -// ResolvedOperand is an expression input frozen into a derived receipt. Base -// columns remain opaque catalog choices, while prior derived outputs remain -// explicit references to earlier child receipts. type ResolvedOperand struct { Kind ResolvedOperandKind `json:"kind"` ChoiceID string `json:"choiceId,omitempty"` @@ -511,8 +498,6 @@ type CategoryProof struct { QueryProof string `json:"queryProof"` } -// DiscoveredCategory is a typed value made selectable by one complete, -// compiler-owned scan receipt. Its ID is opaque and scoped to that receipt. type DiscoveredCategory struct { ChoiceID string `json:"choiceId"` Value Scalar `json:"value"` @@ -567,8 +552,6 @@ type PivotResolution struct { UnlistedPolicyChoiceID string `json:"unlistedPolicyChoiceId"` } -// PivotDerivedOperand describes a compiler-typed output that remains after -// the pivot and can feed a subsequent derived construction. type PivotDerivedOperand struct { ChoiceID string `json:"choiceId"` OutputColumn string `json:"outputColumn"` @@ -586,8 +569,6 @@ type UnpivotResolution struct { NullPolicyChoiceID string `json:"nullPolicyChoiceId"` } -// ResolvedUnpivotInput freezes the server-owned emitted key for a selected -// source column. The browser selects only ChoiceID and cannot author Key. type ResolvedUnpivotInput struct { ChoiceID string `json:"choiceId"` Key Scalar `json:"key"` diff --git a/internal/explorer/types.go b/internal/explorer/types.go index a1b4db968..35080d8ba 100644 --- a/internal/explorer/types.go +++ b/internal/explorer/types.go @@ -36,10 +36,6 @@ type Diagnostic struct { RequestID string `json:"requestId,omitempty"` } -// DatasetMetadata is the live, server-derived shape of a published recipe. -// It is deliberately separate from ConfigV2: physical columns and readiness -// are properties of a materialized generation, not presentation authored by a -// repository or Builder user. type DatasetMetadata struct { Generation string `json:"generation,omitempty"` SchemaDigest string `json:"schemaDigest,omitempty"` @@ -64,12 +60,10 @@ type PublicationMetadata struct { } type Explorer struct { - Project string `json:"project"` - ExplorerID string `json:"explorerId"` - Title string `json:"title"` - ManagementMode ManagementMode `json:"managementMode"` - // DraftConfig is the lossless, canonical ExplorerConfigV2 packet used by - // the Builder. + Project string `json:"project"` + ExplorerID string `json:"explorerId"` + Title string `json:"title"` + ManagementMode ManagementMode `json:"managementMode"` DraftConfig json.RawMessage `json:"draftConfig,omitempty"` DraftVersion int64 `json:"draftVersion"` DraftDigest string `json:"draftDigest,omitempty"` @@ -85,10 +79,6 @@ type Explorer struct { UpdatedAt time.Time `json:"updatedAt"` } -// DraftRevision is an immutable snapshot of one accepted Explorer draft -// state. It keeps the exact serialized workspace and the source context used -// to accept it so restoration never reconstructs construction semantics from -// a command log. type DraftRevision struct { ID string `json:"id"` Project string `json:"project"` @@ -114,10 +104,9 @@ type Materialization struct { } type Revision struct { - ID string `json:"id"` - Project string `json:"project"` - ExplorerID string `json:"explorerId"` - // Config is the exact immutable ExplorerConfigV2 packet for this revision. + ID string `json:"id"` + Project string `json:"project"` + ExplorerID string `json:"explorerId"` Config json.RawMessage `json:"config,omitempty"` AuthoringBundle json.RawMessage `json:"authoringBundle,omitempty"` IntentDigest string `json:"intentDigest,omitempty"` diff --git a/internal/explorer/viewer_projection.go b/internal/explorer/viewer_projection.go index fa342f020..d260c80e7 100644 --- a/internal/explorer/viewer_projection.go +++ b/internal/explorer/viewer_projection.go @@ -45,12 +45,6 @@ func viewerProjectionIntegrity(component string, cause error) error { return &ViewerProjectionIntegrityError{Component: component, Cause: cause} } -// buildViewerProjection is the single translation from immutable Explorer -// publication state to the renderer-facing contract. It intentionally does -// not require an authored ConfigView: a valid recipe is enough to produce a -// default table that a client can extend with presentation choices. -// BuildViewerProjection is the pure immutable-revision to Viewer wire -// projection. It performs no persistence lookup and has no service state. func BuildViewerProjection(revision *Revision) (*ExplorerRuntimeV1, error) { if revision == nil { return nil, nil @@ -71,8 +65,6 @@ func BuildViewerProjection(revision *Revision) (*ExplorerRuntimeV1, error) { } } if len(bundle.Outputs) == 0 { - // A revision without a recipe is not an executable query. The caller - // still returns a valid ExplorerState response with runtime: null. return nil, nil } @@ -118,10 +110,6 @@ func BuildViewerProjection(revision *Revision) (*ExplorerRuntimeV1, error) { emittedOrder[column.OutputID] = append(emittedOrder[column.OutputID], column.PublicColumn) } emitted[column.OutputID][column.PublicColumn] = column - // Emissions are a compatibility fallback for outputs whose publication - // metadata has no columns at all. Once DatasetMetadata or Materialization - // metadata supplies any column for an output, it is authoritative and - // historical emissions must not manufacture additional runtime columns. if !authoritativeColumns[column.OutputID] { if physicalColumns[column.OutputID] == nil { physicalColumns[column.OutputID] = map[string]publication.PhysicalColumn{} @@ -135,8 +123,6 @@ func BuildViewerProjection(revision *Revision) (*ExplorerRuntimeV1, error) { } } } - // Labels are frozen by the public output contract. They are indexed by the - // authored physical column, never reconstructed from compiler identities. contractColumns := map[string]map[string]PublicOutputColumn{} contractOrder := map[string][]string{} if len(revision.PublicOutputContract) > 0 { @@ -194,9 +180,6 @@ func BuildViewerProjection(revision *Revision) (*ExplorerRuntimeV1, error) { return ExplorerRuntimeColumnV1{}, false } emission := emitted[view.Output][name] - // Runtime is an intersection of compiler emissions and authoritative - // materialized columns. Physical schema metadata is never promoted into - // a user-visible column by itself. if emission.PublicColumn == "" || strings.TrimSpace(emission.EmissionID) == "" { return ExplorerRuntimeColumnV1{}, false } @@ -262,10 +245,6 @@ func BuildViewerProjection(revision *Revision) (*ExplorerRuntimeV1, error) { } _, hasPublicContract := contractColumns[view.Output] if hasPublicContract { - // Compilation owns the public schema and its order. Legacy Viewer - // configuration may still supply presentation choices for matching - // columns, but it cannot hide or reorder columns created by a later - // table-shape compilation. for index, name := range ordered { column, ok := ensureColumn(name) if !ok { diff --git a/internal/fhir/schema/compiler_semantics.go b/internal/fhir/schema/compiler_semantics.go index 92fef1d50..2cbd1505e 100644 --- a/internal/fhir/schema/compiler_semantics.go +++ b/internal/fhir/schema/compiler_semantics.go @@ -6,9 +6,6 @@ import ( "strings" ) -// FieldKind is the compiler-facing shape of a FHIR field. It deliberately -// hides the generated schema representation so callers do not depend on -// generator-private types. type FieldKind string const ( @@ -18,19 +15,12 @@ const ( FieldKindArray FieldKind = "array" ) -// FieldSemantics describes the terminal field at a canonical FHIR path. -// ElementKind and Reference are populated for arrays and objects when the -// generated metadata provides that information. type FieldSemantics struct { Kind FieldKind ElementKind FieldKind Reference string } -// TraversalDirection is the FHIR reference direction declared by generated -// graph metadata. It is not automatically the physical AQL direction: the -// stored fhir_edge layout and catalog traversal contract decide that lowering -// concern. type TraversalDirection string const ( @@ -39,8 +29,6 @@ const ( TraversalAny TraversalDirection = "ANY" ) -// TraversalMultiplicity describes whether one source may resolve to one or -// multiple targets. Unknown generated values are rejected rather than guessed. type TraversalMultiplicity string const ( @@ -48,7 +36,6 @@ const ( TraversalMany TraversalMultiplicity = "MANY" ) -// CompilerTraversal is an immutable compiler-facing traversal description. type CompilerTraversal struct { FromType string EdgeLabel string @@ -57,18 +44,12 @@ type CompilerTraversal struct { Multiplicity TraversalMultiplicity } -// ResourceTypes returns a sorted copy of the generated FHIR resource types. func ResourceTypes() []string { out := append([]string(nil), generatedResourceTypes...) sort.Strings(out) return out } -// ConcreteResourceType resolves a graph resource type without requiring -// callers at discovery boundaries to already know the generated schema's -// casing. Only concrete FHIR roots are accepted. Backbone definitions, -// abstract Resource, and caller-supplied custom types are intentionally not -// resources even when they are present in generatedDefinitions. func ConcreteResourceType(resourceType string) (string, bool) { resourceType = strings.TrimSpace(resourceType) if resourceType == "" { @@ -86,23 +67,15 @@ func ConcreteResourceType(resourceType string) (string, bool) { return "", false } -// CanonicalResourceType is retained as a compatibility name for callers that -// need a canonical concrete resource type. func CanonicalResourceType(resourceType string) (string, bool) { return ConcreteResourceType(resourceType) } -// HasResource reports whether resourceType is a generated concrete FHIR -// resource root. func HasResource(resourceType string) bool { _, ok := ConcreteResourceType(resourceType) return ok } -// DefinitionExists reports whether name is represented by the generated FHIR -// schema. Unlike HasResource, it also accepts backbone and choice definitions -// that are valid selector sources but are not graph collections. This is a -// constant-time predicate and does not materialize or clone flattened fields. func DefinitionExists(name string) bool { name = strings.TrimSpace(name) if name == "" { @@ -112,8 +85,6 @@ func DefinitionExists(name string) bool { return ok } -// ResolveFieldSemantics resolves a canonical path using generated schema -// metadata and returns only stable, compiler-safe values. func ResolveFieldSemantics(resourceType, canonicalPath string) (FieldSemantics, bool) { resolved, ok := ResolvePath(resourceType, canonicalPath) if !ok { @@ -131,9 +102,6 @@ func ResolveFieldSemantics(resourceType, canonicalPath string) (FieldSemantics, return semantics, true } -// ResolveCompilerTraversal looks up and normalizes generated traversal -// metadata. Conflicting or unknown generated values return an error so the -// compiler cannot silently choose unsafe graph semantics. func ResolveCompilerTraversal(fromType, edgeLabel, toType string) (CompilerTraversal, bool, error) { spec, ok := LookupTraversal(fromType, edgeLabel, toType) if !ok { diff --git a/internal/fhir/schema/correlated.go b/internal/fhir/schema/correlated.go index 6fbea13c2..b2a3e1ca2 100644 --- a/internal/fhir/schema/correlated.go +++ b/internal/fhir/schema/correlated.go @@ -19,50 +19,28 @@ const ( CorrelatedValueScopeCoding CorrelatedValueScope = "CODING" ) -// CorrelatedBinding is the backend-neutral description of one terminology -// key/value relationship. OwnerPath names the repeated FHIR item that owns the -// value; KeyPath names a repeated Coding collection or singular Coding within -// that item. System and Code are relative to one Coding item, so they cannot be -// accidentally paired from different array members. type CorrelatedBinding struct { - OwnerPath string `json:"ownerPath,omitempty"` - KeyPath string `json:"keyPath"` - VersionPath string `json:"versionPath,omitempty"` - // Empty and EXACT match a selected code. SYSTEM_VERSION matches every - // code within one exact system and version, including an empty version. - SelectionScope CorrelatedSelectionScope `json:"selectionScope,omitempty"` - // ValueScope selects the object that owns ValuePath and ValueFallback. - // Empty and OWNER preserve the legacy owner-relative binding; CODING uses - // the exact Coding item matched by KeyPath. - ValueScope CorrelatedValueScope `json:"valueScope,omitempty"` - SystemPath string `json:"systemPath"` - CodePath string `json:"codePath"` - ValuePath string `json:"valuePath"` - ValueFallback []string `json:"valueFallback,omitempty"` - FallbackOnBlankString bool `json:"fallbackOnBlankString,omitempty"` - ChoiceArms []string `json:"choiceArms,omitempty"` - LogicalType string `json:"logicalType"` - UnitPath string `json:"unitPath,omitempty"` + OwnerPath string `json:"ownerPath,omitempty"` + KeyPath string `json:"keyPath"` + VersionPath string `json:"versionPath,omitempty"` + SelectionScope CorrelatedSelectionScope `json:"selectionScope,omitempty"` + ValueScope CorrelatedValueScope `json:"valueScope,omitempty"` + SystemPath string `json:"systemPath"` + CodePath string `json:"codePath"` + ValuePath string `json:"valuePath"` + ValueFallback []string `json:"valueFallback,omitempty"` + FallbackOnBlankString bool `json:"fallbackOnBlankString,omitempty"` + ChoiceArms []string `json:"choiceArms,omitempty"` + LogicalType string `json:"logicalType"` + UnitPath string `json:"unitPath,omitempty"` } -// CorrelatedKey is the selected terminology identity for one correlated -// projection. It is deliberately separate from CorrelatedBinding, which only -// describes where the identity and value are structurally found. type CorrelatedKey struct { System string `json:"system"` Code string `json:"code"` Version string `json:"version,omitempty"` } -// ExtensionBinding is the closed authoring shape for an ancestor-aware FHIR -// extension lookup. OwnerPath names the full repeated path ending at the -// terminal extension (for example extension[].extension[]); URLPath contains -// one selected URL for each extension boundary. ValuePath and its optional -// fallbacks are relative to the terminal Extension object. -// -// URLPath is deliberately a literal sequence rather than a predicate or a -// flattened URL string. This keeps each ancestor identity in the same lexical -// traversal scope as the value it owns. type ExtensionBinding struct { OwnerPath string `json:"ownerPath,omitempty"` URLPath []string `json:"urlPath"` @@ -74,9 +52,6 @@ type ExtensionBinding struct { UnitPath string `json:"unitPath,omitempty"` } -// CorrelatedBindingSpec is the checked form consumed by semantic and physical -// compilers. Key selectors are relative to the owning item unless OwnerSelector -// is empty. Value selectors use ValueScope to select the owner or matched Coding. type CorrelatedBindingSpec struct { ResourceType string OwnerSelector Selector @@ -98,10 +73,6 @@ type CorrelatedBindingSpec struct { UnitSelector *Selector } -// ExtensionBindingSpec is the checked extension form consumed by semantic and -// physical lowering. OwnerSelector addresses the repeated item immediately -// before the first extension boundary; URLSelectors are each relative to the -// current Extension item and therefore preserve nested parent identity. type ExtensionBindingSpec struct { ResourceType string OwnerSelector Selector @@ -116,10 +87,6 @@ type ExtensionBindingSpec struct { UnitSelector *Selector } -// ValidateCorrelatedBinding validates one binding against generated FHIR -// metadata and returns selectors suitable for lowering. It intentionally -// rejects an independently rooted system/code path and any key outside the -// declared owner scope. func ValidateCorrelatedBinding(resourceType string, binding CorrelatedBinding) (CorrelatedBindingSpec, error) { if !HasResource(resourceType) { return CorrelatedBindingSpec{}, fmt.Errorf("resource type %q is not represented by generated FHIR schema", resourceType) @@ -270,9 +237,6 @@ type checkedBindingValue struct { UnitSelector *Selector } -// validateBindingValue is shared by terminology and extension bindings. The -// generated schema, not a client-declared label, owns primitive compatibility -// and valid value[x] arms. func validateBindingValue(valueResource, valuePath string, valueFallback []string, fallbackOnBlankString bool, requestedChoiceArms []string, requestedLogicalType, unitPath string) (checkedBindingValue, error) { valuePath = CanonicalizePath(valuePath) if valuePath == "" { @@ -376,10 +340,6 @@ func validateBindingValue(valueResource, valuePath string, valueFallback []strin return checkedBindingValue{ValueSelector: value, ValueFallbacks: fallbacks, FallbackOnBlankString: fallbackOnBlankString, ChoiceArms: choiceArms, LogicalType: logicalType, ValuePrimitive: valueMetadata.Primitive, UnitSelector: unit}, nil } -// inferQuantityUnitSelector retains the display unit owned by the same FHIR -// Quantity as a selected numeric value. The generated datatype reference is -// the authority; a coincidental sibling named "unit" on another structure is -// not treated as a measurement unit. func inferQuantityUnitSelector(valueResource, valuePath string) (Selector, bool) { valuePath = CanonicalizePath(valuePath) separator := strings.LastIndex(valuePath, ".") @@ -400,11 +360,6 @@ func inferQuantityUnitSelector(valueResource, valuePath string) (Selector, bool) return unit, err == nil } -// ValidateExtensionBinding validates the ancestor chain and terminal value -// against generated FHIR metadata. Every path segment before the first -// extension boundary is scoped normally; once an extension boundary begins, -// only nested extension[] segments are accepted. This makes URLPath depth and -// lexical traversal unambiguous. func ValidateExtensionBinding(resourceType string, binding ExtensionBinding) (ExtensionBindingSpec, error) { if !HasResource(resourceType) { return ExtensionBindingSpec{}, fmt.Errorf("resource type %q is not represented by generated FHIR schema", resourceType) diff --git a/internal/fhir/schema/generated_metadata.go b/internal/fhir/schema/generated_metadata.go index 3df13dea2..df47883b2 100644 --- a/internal/fhir/schema/generated_metadata.go +++ b/internal/fhir/schema/generated_metadata.go @@ -2,8 +2,6 @@ package schema import generatedschema "github.com/calypr/loom/generated/fhirschema" -// These aliases keep generated FHIR metadata private to this package while -// allowing the schema facade to expose stable, hand-written semantics. type TraversalSpec = generatedschema.Traversal type generatedProperty = generatedschema.Property diff --git a/internal/fhir/schema/identifier.go b/internal/fhir/schema/identifier.go index 509de7134..50b190fb3 100644 --- a/internal/fhir/schema/identifier.go +++ b/internal/fhir/schema/identifier.go @@ -5,8 +5,6 @@ import ( "strings" ) -// IdentifierBinding is the closed authoring shape for selecting one -// namespace/value pair from the same repeated FHIR Identifier item. type IdentifierBinding struct { OwnerPath string `json:"ownerPath"` SystemPath string `json:"systemPath"` @@ -15,8 +13,6 @@ type IdentifierBinding struct { LogicalType string `json:"logicalType"` } -// IdentifierBindingSpec contains selectors checked against generated FHIR -// metadata and ready for dynamic-column lowering. type IdentifierBindingSpec struct { ResourceType string OwnerSelector Selector @@ -28,9 +24,6 @@ type IdentifierBindingSpec struct { ValuePrimitive PrimitiveKind } -// ValidateIdentifierBinding validates a namespace/value pair against the -// generated schema. The pair is deliberately restricted to Identifier.system -// and Identifier.value, which FHIR defines as belonging to the same item. func ValidateIdentifierBinding(resourceType string, binding IdentifierBinding) (IdentifierBindingSpec, error) { if !HasResource(resourceType) { return IdentifierBindingSpec{}, fmt.Errorf("resource type %q is not represented by generated FHIR schema", resourceType) diff --git a/internal/fhir/schema/index.go b/internal/fhir/schema/index.go index 46bcfe441..6e191b6b0 100644 --- a/internal/fhir/schema/index.go +++ b/internal/fhir/schema/index.go @@ -22,29 +22,23 @@ const ( JSONTypeArray JSONType = "array" ) -// Index is an immutable view of FHIR definition metadata. type Index struct { definitions map[DefinitionName]Definition } -// DefinitionView reads a definition's elements without exposing its slices. -// Its zero value represents an empty definition. type DefinitionView struct { requiredElements []string elements []Element } -// ElementListView reads a list of elements without exposing its backing slice. type ElementListView struct { elements []Element } -// ElementView reads one element without exposing its nested slices. type ElementView struct { element *Element } -// Definition contains one generated or synthetic FHIR definition. type Definition struct { Name DefinitionName Title string @@ -53,7 +47,6 @@ type Definition struct { Elements []Element } -// Element contains schema metadata for one property. type Element struct { Name string JSONType JSONType @@ -77,7 +70,6 @@ type Element struct { Description string } -// NewIndex copies definitions into an immutable lookup index. func NewIndex(definitions []Definition) (*Index, error) { index := &Index{definitions: make(map[DefinitionName]Definition, len(definitions))} for _, definition := range definitions { @@ -100,8 +92,6 @@ func NewIndex(definitions []Definition) (*Index, error) { return index, nil } -// Definition returns a copy of one definition. Changes to the returned slices -// do not alter the index. func (i *Index) Definition(name DefinitionName) (Definition, bool) { if i == nil { return Definition{}, false @@ -113,8 +103,6 @@ func (i *Index) Definition(name DefinitionName) (Definition, bool) { return cloneDefinition(definition), true } -// ReadDefinition returns a read-only view for traversal. Use Definition when -// callers need an independently mutable copy of all metadata. func (i *Index) ReadDefinition(name DefinitionName) (DefinitionView, bool) { if i == nil { return DefinitionView{}, false @@ -129,28 +117,22 @@ func (i *Index) ReadDefinition(name DefinitionName) (DefinitionView, bool) { }, true } -// RequiredElements returns the names of members required by this definition. -// The returned slice never aliases the immutable index. func (v DefinitionView) RequiredElements() []string { return cloneStrings(v.requiredElements) } -// Elements returns a read-only view of the definition's elements. func (v DefinitionView) Elements() ElementListView { return ElementListView{elements: v.elements} } -// Len returns the number of elements in the list. func (v ElementListView) Len() int { return len(v.elements) } -// At returns a read-only view of the indexed element. func (v ElementListView) At(index int) (ElementView, bool) { return elementAt(v.elements, index) } -// Name returns the element name. func (v ElementView) Name() string { if v.element == nil { return "" @@ -158,7 +140,6 @@ func (v ElementView) Name() string { return v.element.Name } -// JSONType returns the element's JSON type. func (v ElementView) JSONType() JSONType { if v.element == nil { return "" @@ -166,7 +147,6 @@ func (v ElementView) JSONType() JSONType { return v.element.JSONType } -// ReferencedType returns the element's referenced definition, if any. func (v ElementView) ReferencedType() DefinitionName { if v.element == nil { return "" @@ -174,7 +154,6 @@ func (v ElementView) ReferencedType() DefinitionName { return v.element.ReferencedType } -// ItemJSONType returns the JSON type of an array item. func (v ElementView) ItemJSONType() JSONType { if v.element == nil { return "" @@ -182,7 +161,6 @@ func (v ElementView) ItemJSONType() JSONType { return v.element.ItemJSONType } -// ArrayElementType returns the array item's referenced definition, if any. func (v ElementView) ArrayElementType() DefinitionName { if v.element == nil { return "" @@ -190,7 +168,6 @@ func (v ElementView) ArrayElementType() DefinitionName { return v.element.ArrayElementType } -// Elements returns a read-only view of inline object elements. func (v ElementView) Elements() ElementListView { if v.element == nil { return ElementListView{} @@ -198,7 +175,6 @@ func (v ElementView) Elements() ElementListView { return ElementListView{elements: v.element.Elements} } -// ArrayElements returns a read-only view of inline array-item elements. func (v ElementView) ArrayElements() ElementListView { if v.element == nil { return ElementListView{} @@ -206,7 +182,6 @@ func (v ElementView) ArrayElements() ElementListView { return ElementListView{elements: v.element.ArrayElements} } -// Snapshot returns an independently mutable copy of the element metadata. func (v ElementView) Snapshot() Element { if v.element == nil { return Element{} @@ -214,8 +189,6 @@ func (v ElementView) Snapshot() Element { return cloneElement(*v.element) } -// Definitions returns all definitions in name order. Each definition owns -// its slices and can be changed without altering the index. func (i *Index) Definitions() []Definition { if i == nil || len(i.definitions) == 0 { return []Definition{} @@ -239,7 +212,6 @@ var ( generatedIndexErr error ) -// GeneratedIndex returns the immutable index built from generated FHIR metadata. func GeneratedIndex() (*Index, error) { generatedIndexOnce.Do(func() { generatedIndex, generatedIndexErr = newGeneratedIndex() diff --git a/internal/fhir/schema/quantity.go b/internal/fhir/schema/quantity.go index 295da1149..7c6cbfa3c 100644 --- a/internal/fhir/schema/quantity.go +++ b/internal/fhir/schema/quantity.go @@ -5,8 +5,6 @@ import ( "strings" ) -// QuantityIdentityPaths validates a scalar FHIR Quantity value and returns its -// sibling system and code paths. func QuantityIdentityPaths(resourceType, valuePath string) (string, string, error) { valuePath = CanonicalizePath(valuePath) if !strings.HasSuffix(valuePath, ".value") { diff --git a/internal/fhir/schema/row_resolution.go b/internal/fhir/schema/row_resolution.go index 466d44f43..42875fd2e 100644 --- a/internal/fhir/schema/row_resolution.go +++ b/internal/fhir/schema/row_resolution.go @@ -21,7 +21,6 @@ const ( RowPathUnknown RowPathShape = "UNKNOWN" ) -// RowPathFacts contains detached structural facts used to mint row choices. type RowPathFacts struct { ResourceType DefinitionName CanonicalPath string @@ -33,7 +32,6 @@ type RowPathFacts struct { Description string } -// ResolveRowPath resolves one canonical path against immutable schema metadata. func (i *Index) ResolveRowPath(resourceType DefinitionName, path string) (RowPathFacts, error) { segments, err := parseRowPath(path) if err != nil { @@ -67,9 +65,6 @@ func (i *Index) ResolveRowPath(resourceType DefinitionName, path string) (RowPat if index == len(segments)-1 { break } - // Reference payload members such as reference and display are fields on - // the same JSON object. Only an explicit construction route follows the - // referenced resource itself. switch element.JSONType { case JSONTypeArray: current, err = i.elementChildren(element, true) diff --git a/internal/fhir/schema/structure.go b/internal/fhir/schema/structure.go index b032a1f12..669b36ce2 100644 --- a/internal/fhir/schema/structure.go +++ b/internal/fhir/schema/structure.go @@ -5,10 +5,6 @@ import ( "sort" ) -// DefinitionStructure is a bounded, detached snapshot of one definition's -// declared JSON shape. Named references remain typed expansion edges instead -// of being recursively expanded, so recursive FHIR definitions cannot make -// the structure infinite. type DefinitionStructure struct { Root DefinitionName Title string @@ -17,9 +13,6 @@ type DefinitionStructure struct { Members []StructureMember } -// StructureMember retains the complete generated metadata for one member. -// Members contains inline structure only. ReferencedType identifies the named -// definition a consumer may expand separately through Index.Structure. type StructureMember struct { CanonicalPath string Element Element @@ -28,7 +21,6 @@ type StructureMember struct { Members []StructureMember } -// Structure returns a finite copy of one definition's declared structure. func (i *Index) Structure(root DefinitionName) (DefinitionStructure, bool) { if i == nil { return DefinitionStructure{}, false @@ -77,9 +69,6 @@ func structureMembers(elements []Element, prefix string) []StructureMember { return members } -// RepeatedCodingPaths returns generated root-relative paths that end at a -// repeated Coding element. Named definitions are followed once per branch so -// recursive FHIR types cannot produce an unbounded list of paths. func (i *Index) RepeatedCodingPaths(root DefinitionName) ([]string, error) { if i == nil { return nil, fmt.Errorf("schema index is unavailable") @@ -149,8 +138,6 @@ func (i *Index) RepeatedCodingPaths(root DefinitionName) ([]string, error) { return result, nil } -// HasRepeatedCodingPath reports whether a reachable element is a repeated Coding. -// It visits named definitions once without enumerating root-relative paths. func (i *Index) HasRepeatedCodingPath(root DefinitionName) (bool, error) { if i == nil { return false, fmt.Errorf("schema index is unavailable") diff --git a/internal/fhir/schema/terminal_metadata.go b/internal/fhir/schema/terminal_metadata.go index 488ae9b0c..f760fa8b7 100644 --- a/internal/fhir/schema/terminal_metadata.go +++ b/internal/fhir/schema/terminal_metadata.go @@ -2,10 +2,6 @@ package schema import "strings" -// PrimitiveKind is the primitive shape reliably represented by generated -// schema metadata. Date and date-time are exposed only when the source graph -// schema declares the corresponding JSON Schema format; code semantics remain -// distinct from arbitrary generated strings. type PrimitiveKind string const ( @@ -18,26 +14,17 @@ const ( PrimitiveDateTime PrimitiveKind = "date_time" ) -// TerminalScalarMetadata describes the terminal property of a canonical FHIR -// path without exposing generator-private property types. Primitive is unknown -// when the terminal is an object or an array of objects. type TerminalScalarMetadata struct { Primitive PrimitiveKind Repeated bool } -// ResolveTerminalScalarMetadata resolves compiler-facing primitive and -// repetition facts from the active generated definition. func ResolveTerminalScalarMetadata(resourceType, canonicalPath string) (TerminalScalarMetadata, bool) { canonicalPath = CanonicalizePath(canonicalPath) resolved, ok := ResolvePath(resourceType, canonicalPath) if !ok { return TerminalScalarMetadata{}, false } - // A scalar below any repeated parent can yield more than one value per - // resource. Looking at only the terminal generated property would classify - // component[].valueInteger as scalar, which would let the compiler apply an - // incorrect scalar filter or projection. repeated := strings.Contains(canonicalPath, "[]") property := resolved.Property if property.Kind == "array" { diff --git a/internal/fhir/schema/terminal_metadata_test.go b/internal/fhir/schema/terminal_metadata_test.go index 2fe7c2469..d484e0209 100644 --- a/internal/fhir/schema/terminal_metadata_test.go +++ b/internal/fhir/schema/terminal_metadata_test.go @@ -79,9 +79,6 @@ func TestResolveTerminalScalarMetadataRejectsUnknownPath(t *testing.T) { if got, ok := ResolveTerminalScalarMetadata("NotAResource", "id"); ok { t.Fatalf("unexpected metadata for unknown resource: %#v", got) } - // The active generated Observation definition does not advertise - // valueDecimal, so the semantic API must not infer it from another FHIR - // version or an external model. if got, ok := ResolveTerminalScalarMetadata("Observation", "valueDecimal"); ok { t.Fatalf("unexpected metadata for unadvertised Observation.valueDecimal: %#v", got) } diff --git a/internal/fhir/schema/traversals.go b/internal/fhir/schema/traversals.go index f6821bde1..fa7e2ed78 100644 --- a/internal/fhir/schema/traversals.go +++ b/internal/fhir/schema/traversals.go @@ -1,6 +1,5 @@ package schema -// LookupTraversal returns a defensive copy of the generated traversal record. func LookupTraversal(fromType, edgeLabel, toType string) (TraversalSpec, bool) { spec, ok := generatedTraversals[traversalKey(fromType, edgeLabel, toType)] if !ok { diff --git a/internal/fhir/semantic/member_walk.go b/internal/fhir/semantic/member_walk.go index c9747f2af..0dc8ef066 100644 --- a/internal/fhir/semantic/member_walk.go +++ b/internal/fhir/semantic/member_walk.go @@ -9,7 +9,6 @@ import ( "github.com/calypr/loom/internal/fhir/schema" ) -// WalkDiagnosticCode is a closed classification of payload/schema drift. type WalkDiagnosticCode string const ( @@ -25,7 +24,6 @@ const ( WalkMembersWithResourceTypesMaxDepth = 128 ) -// WalkLimitKind classifies a hard stop in the resource-aware member walker. type WalkLimitKind string const ( @@ -34,7 +32,6 @@ const ( WalkLimitDepth WalkLimitKind = "DEPTH" ) -// WalkLimitError reports a hard limit reached by WalkMembersWithResourceTypes. type WalkLimitError struct { Kind WalkLimitKind Limit int @@ -44,7 +41,6 @@ func (e *WalkLimitError) Error() string { return fmt.Sprintf("walk FHIR members: %s limit %d reached", e.Kind, e.Limit) } -// WalkDiagnostic identifies one structural mismatch at a concrete owner. type WalkDiagnostic struct { Code WalkDiagnosticCode CanonicalPath string @@ -53,16 +49,12 @@ type WalkDiagnostic struct { ChoiceGroup string } -// ResourceBoundary records one concrete abstract Resource reference resolved -// to an allowed concrete resource type. type ResourceBoundary struct { CanonicalPath string OwnerPath string ResourceType schema.ResourceType } -// MemberFact is one present member or repeated item resolved through the -// generated schema. RawJSON is an owned copy of the exact source value. type MemberFact struct { RootDefinition schema.DefinitionName DeclaringType schema.DefinitionName @@ -79,9 +71,6 @@ type MemberFact struct { Null bool } -// WalkMembers visits every present schema member and repeated item while -// returning structural diagnostics. Named references are resolved only for -// concrete values in the payload, so recursive definitions remain finite. func WalkMembers( index *schema.Index, root schema.DefinitionName, @@ -91,9 +80,6 @@ func WalkMembers( return walkMembers(index, root, payload, visit, memberWalkOptions{}) } -// WalkMembersWithResourceTypes walks present members and resolves schema- -// declared abstract Resource references only when their exact resourceType is -// permitted by allowed and represented in index. func WalkMembersWithResourceTypes( index *schema.Index, root schema.DefinitionName, @@ -538,8 +524,6 @@ func unknownMemberNames(object map[string]json.RawMessage, declared map[string]s return unknown } - // Keep only the lexicographically first remaining diagnostic names plus - // one overflow witness, so a wide unknown object cannot grow scratch memory. capacity := state.options.limits.maxDiagnostics - diagnosticCount + 1 if capacity < 1 { capacity = 1 diff --git a/internal/fhir/semantic/walk.go b/internal/fhir/semantic/walk.go index 1d512839b..a19d4f5e6 100644 --- a/internal/fhir/semantic/walk.go +++ b/internal/fhir/semantic/walk.go @@ -9,8 +9,6 @@ import ( "github.com/calypr/loom/internal/fhir/schema" ) -// Occurrence describes one object reached through a generated type reference. -// RawJSON contains a copy of the exact JSON value for that object. type Occurrence struct { RootDefinition schema.DefinitionName ReferencedType schema.DefinitionName @@ -21,15 +19,12 @@ type Occurrence struct { RawJSON json.RawMessage } -// RepeatedBoundary records one concrete array item along an occurrence path. type RepeatedBoundary struct { CanonicalPath string OwnerPath string Index int } -// Walk reports referenced datatype objects found under root. Paths use [] for -// repeated boundaries and indexes in OwnerPath to retain each concrete owner. func Walk(index *schema.Index, root schema.DefinitionName, payload []byte, visit func(Occurrence) error) error { if index == nil { return fmt.Errorf("walk FHIR definition %q: schema index is nil", root) diff --git a/internal/ingest/backend.go b/internal/ingest/backend.go index 484509a96..64b4239d7 100644 --- a/internal/ingest/backend.go +++ b/internal/ingest/backend.go @@ -4,26 +4,21 @@ import ( "context" "encoding/json" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/catalog" - publicationarango "github.com/calypr/loom/internal/dataset/arango" - arangostore "github.com/calypr/loom/internal/store/arango" ) const EdgeCollection = "fhir_edge" -func openBackend(ctx context.Context, opts arangostore.ConnectionOptions) (*arangostore.Client, error) { - return arangostore.Open(ctx, opts.URL, opts.Database) +type ConnectionOptions = arangofactory.Config + +func openBackend(ctx context.Context, opts ConnectionOptions) (*arangofactory.Runtime, error) { + return arangofactory.Open(ctx, opts) } -func bootstrapSpecWithReporter(resourceTypes []string, truncate bool, reporter EventSink) arangostore.BootstrapSpec { - collections := make([]arangostore.CollectionSpec, 0, len(resourceTypes)+4) +func bootstrapSpecWithReporter(resourceTypes []string, truncate bool, reporter EventSink) arangofactory.BootstrapSpec { + collections := make([]arangofactory.CollectionSpec, 0, len(resourceTypes)+4) for _, name := range resourceTypes { - // Every generated FHIR resource is a possible dataframe root. The - // compiler applies project and optional authorization scope before a - // stable _key sort/limit, so these are not Patient-specific indexes. - // Without the _key suffix Arango can choose the primary index only to - // satisfy SORT root._key, then scan an entire resource collection before - // it finds a project's first preview rows. indexes := [][]string{ {"project"}, {"id"}, @@ -31,22 +26,19 @@ func bootstrapSpecWithReporter(resourceTypes []string, truncate bool, reporter E {"project", "_key"}, {"project", "auth_resource_path"}, {"project", "auth_resource_path", "_key"}, - // Dataset generations use immutable, hashed physical keys. These - // indexes support the compiler's mandatory project/generation scope - // before stable key or FHIR-id lookup. {"project", "dataset_generation", "_key"}, {"project", "dataset_generation", "auth_resource_path", "_key"}, {"project", "dataset_generation", "id"}, {"project", "dataset_generation", "auth_resource_path", "id"}, } - collections = append(collections, arangostore.CollectionSpec{ + collections = append(collections, arangofactory.CollectionSpec{ Name: name, Truncate: truncate, Indexes: indexes, }) } collections = append(collections, - arangostore.CollectionSpec{ + arangofactory.CollectionSpec{ Name: EdgeCollection, Edge: true, Truncate: truncate, @@ -58,19 +50,11 @@ func bootstrapSpecWithReporter(resourceTypes []string, truncate bool, reporter E {"project", "dataset_generation", "to_type", "label"}, {"project", "dataset_generation", "auth_resource_path", "from_type", "label"}, {"project", "dataset_generation", "auth_resource_path", "to_type", "label"}, - // These are vertex-centric persistent indexes. A dataframe - // traversal always starts at one known root/parent endpoint and - // applies exact project, generation, label, and target-type - // predicates. The default Arango edge index can locate the - // endpoint but must inspect every adjacent edge; these compound - // indexes let the optimizer restrict the adjacency list before - // materializing candidate edges. Keep the endpoint first: `_to` - // is INBOUND and `_from` is OUTBOUND. {"_to", "project", "dataset_generation", "label", "from_type"}, {"_from", "project", "dataset_generation", "label", "to_type"}, }, }, - arangostore.CollectionSpec{ + arangofactory.CollectionSpec{ Name: "fhir_field_catalog", Truncate: truncate, Indexes: [][]string{ @@ -87,7 +71,7 @@ func bootstrapSpecWithReporter(resourceTypes []string, truncate bool, reporter E {"project", "dataset_generation", "auth_resource_path", "resource_type", "pivot_candidate"}, }, }, - arangostore.CollectionSpec{ + arangofactory.CollectionSpec{ Name: catalog.RelationshipCatalogCollection, Truncate: truncate, Indexes: [][]string{ @@ -97,7 +81,7 @@ func bootstrapSpecWithReporter(resourceTypes []string, truncate bool, reporter E {"project", "dataset_generation", "auth_resource_path", "from_type"}, }, }, - arangostore.CollectionSpec{ + arangofactory.CollectionSpec{ Name: catalog.SemanticInventoryCollection, Indexes: [][]string{ {"project", "dataset_generation", "build_id"}, @@ -107,20 +91,20 @@ func bootstrapSpecWithReporter(resourceTypes []string, truncate bool, reporter E {"project", "dataset_generation", "build_id", "source_id"}, }, }, - arangostore.CollectionSpec{ + arangofactory.CollectionSpec{ Name: catalog.SemanticInventoryEntryCollection, Indexes: [][]string{ {"project", "dataset_generation", "build_id", "source_kind", "binding_id", "concept_id", "auth_resource_path"}, {"project", "dataset_generation", "build_id", "source_kind", "resource_type", "binding_id", "concept_id", "auth_resource_path"}, }, }, - arangostore.CollectionSpec{ + arangofactory.CollectionSpec{ Name: catalog.SemanticInventoryBuildCollection, Indexes: [][]string{ {"project", "dataset_generation"}, }, }, - arangostore.CollectionSpec{ + arangofactory.CollectionSpec{ Name: catalog.ResourceInventoryCollection, Indexes: [][]string{ {"project", "dataset_generation", "record_type", "resource_type", "auth_resource_path"}, @@ -128,7 +112,7 @@ func bootstrapSpecWithReporter(resourceTypes []string, truncate bool, reporter E }, }, ) - return arangostore.BootstrapSpec{ + return arangofactory.BootstrapSpec{ Collections: collections, Reporter: func(event string, fields map[string]any) { emitEvent(reporter, event, fields) @@ -136,17 +120,15 @@ func bootstrapSpecWithReporter(resourceTypes []string, truncate bool, reporter E } } -// lifecycleBootstrapSpecWithReporter keeps generation manifests outside the -// truncation-oriented FHIR bootstrap. -func lifecycleBootstrapSpecWithReporter(reporter EventSink) arangostore.BootstrapSpec { - return arangostore.BootstrapSpec{ - Collections: publicationarango.CollectionSpecs(), +func lifecycleBootstrapSpecWithReporter(reporter EventSink) arangofactory.BootstrapSpec { + return arangofactory.BootstrapSpec{ + Collections: arangofactory.DatasetCollectionSpecs(), Reporter: func(event string, fields map[string]any) { emitEvent(reporter, event, fields) }, } } -func insertRawDocuments(ctx context.Context, backend *arangostore.Client, collection string, docs []json.RawMessage, overwrite bool, writeAPI string) error { +func insertRawDocuments(ctx context.Context, backend *arangofactory.Client, collection string, docs []json.RawMessage, overwrite bool, writeAPI string) error { return backend.InsertBatchRaw(ctx, collection, docs, overwrite, writeAPI) } diff --git a/internal/ingest/backend_test.go b/internal/ingest/backend_test.go index 641232d77..d5660f5d6 100644 --- a/internal/ingest/backend_test.go +++ b/internal/ingest/backend_test.go @@ -4,9 +4,8 @@ import ( "reflect" "testing" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/catalog" - catalogarango "github.com/calypr/loom/internal/catalog/arango" - arangostore "github.com/calypr/loom/internal/store/arango" ) func TestBootstrapSpecAddsRootPreviewIndexesForEveryFHIRCollection(t *testing.T) { @@ -116,7 +115,7 @@ func TestBootstrapSpecAddsGenerationScopedIndexesWithoutTraversalSpeculation(t * func TestBootstrapSpecDoesNotCreateOptionalSemanticCoverageCollection(t *testing.T) { spec := bootstrapSpecWithReporter([]string{"Patient"}, true, nil) - if _, found := bootstrapCollection(spec, catalogarango.SemanticCoverageCollection); found { + if _, found := bootstrapCollection(spec, arangofactory.CatalogSemanticCoverageCollection); found { t.Fatal("normal generation bootstrap created the optional per-source semantic coverage collection") } } @@ -131,13 +130,13 @@ func TestLifecycleBootstrapNeverTruncatesMetadata(t *testing.T) { } } -func bootstrapCollection(spec arangostore.BootstrapSpec, name string) (arangostore.CollectionSpec, bool) { +func bootstrapCollection(spec arangofactory.BootstrapSpec, name string) (arangofactory.CollectionSpec, bool) { for _, collection := range spec.Collections { if collection.Name == name { return collection, true } } - return arangostore.CollectionSpec{}, false + return arangofactory.CollectionSpec{}, false } func containsIndex(indexes [][]string, want []string) bool { diff --git a/internal/ingest/bench_test.go b/internal/ingest/bench_test.go index 494c32b59..05ae6b81b 100644 --- a/internal/ingest/bench_test.go +++ b/internal/ingest/bench_test.go @@ -37,7 +37,6 @@ func BenchmarkValidateAndExtract(b *testing.B) { class := schema.GetClass(resourceType) - // 1. Generic Baseline b.Run(resourceType+"/Generic_Baseline", func(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { @@ -52,7 +51,6 @@ func BenchmarkValidateAndExtract(b *testing.B) { } }) - // 2. Generated Validate-only b.Run(resourceType+"/Generated_ValidateOnly", func(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { @@ -82,7 +80,6 @@ func BenchmarkValidateAndExtract(b *testing.B) { } }) - // 3. Generated Validate+Extract b.Run(resourceType+"/Generated_ValidateAndExtract", func(b *testing.B) { b.ResetTimer() stageTiming := make(map[string]float64) @@ -150,7 +147,6 @@ func BenchmarkVertexSerialization(b *testing.B) { } line := lines[0] - // 1. Prepare map payload var mapPayload map[string]any if err := sonic.ConfigFastest.Unmarshal(line, &mapPayload); err != nil { b.Fatalf("failed to decode: %v", err) @@ -163,7 +159,6 @@ func BenchmarkVertexSerialization(b *testing.B) { Payload: mapPayload, } - // 2. Prepare struct payload var structPayload fhir.Condition if err := sonic.ConfigFastest.Unmarshal(line, &structPayload); err != nil { b.Fatalf("failed to decode: %v", err) @@ -176,7 +171,6 @@ func BenchmarkVertexSerialization(b *testing.B) { Payload: &structPayload, } - // 3. Prepare RawMessage payload rawDoc := VertexDocument{ Key: "sample-key", ID: "sample-id", diff --git a/internal/ingest/doc.go b/internal/ingest/doc.go index 57e0f02f2..fbd708203 100644 --- a/internal/ingest/doc.go +++ b/internal/ingest/doc.go @@ -1,3 +1 @@ -// Package ingest implements the FHIR NDJSON ingest runtime, including file -// discovery, row building, Arango bootstrap, and load progress reporting. package ingest diff --git a/internal/ingest/documents.go b/internal/ingest/documents.go index 8bd6ed14d..06f8338c1 100644 --- a/internal/ingest/documents.go +++ b/internal/ingest/documents.go @@ -1,7 +1,3 @@ -// The upstream jsonschemagraph module exposes these document contracts only -// from an unpublished package path. Keeping the adapter here makes the Loom -// server build reproducibly from the public module graph while preserving the -// existing vertex/edge wire shape. package ingest import ( @@ -135,3 +131,15 @@ func targetTypeFromLabel(label string) (string, error) { func collectionID(resourceType, id string) string { return SanitizeKey(resourceType) + "/" + SanitizeKey(id) } + +func SplitCollectionID(value string) (resourceType, logicalID string, ok bool) { + resourceType, logicalID, found := strings.Cut(strings.TrimSpace(value), "/") + if !found { + return "", "", false + } + if strings.TrimSpace(resourceType) == "" || strings.TrimSpace(logicalID) == "" || + strings.Contains(logicalID, "/") { + return "", "", false + } + return resourceType, logicalID, true +} diff --git a/internal/ingest/documents_test.go b/internal/ingest/documents_test.go index b0a330cec..5cad635f6 100644 --- a/internal/ingest/documents_test.go +++ b/internal/ingest/documents_test.go @@ -33,3 +33,35 @@ func TestVertexFromFHIRWithExtraRejectsNonConcreteResourceTypes(t *testing.T) { t.Fatal("backbone definition was accepted as a graph vertex") } } + +func TestSplitCollectionIDInvertsCollectionEndpoints(t *testing.T) { + edge, err := EdgeFromGrip("project", " patient ", &model.Edge{Id: "edge-1", From: "patient-1", To: "specimen-1", Label: "subject_Specimen"}) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + value string + wantType, wantID string + }{ + {edge.From, "Patient", "patient-1"}, + {edge.To, "Specimen", "specimen-1"}, + {"Observation/obs.7@x", "Observation", "obs.7@x"}, + } { + gotType, gotID, ok := SplitCollectionID(tc.value) + if !ok || gotType != tc.wantType || gotID != tc.wantID { + t.Errorf("SplitCollectionID(%q) = (%q, %q, %t), want (%q, %q, true)", + tc.value, gotType, gotID, ok, tc.wantType, tc.wantID) + } + } + + for _, value := range []string{"", " ", "Patient", "/patient-1", "Patient/", "Patient//x", "Patient/a/b", " /a", "a/ "} { + if gotType, gotID, ok := SplitCollectionID(value); ok { + t.Errorf("SplitCollectionID(%q) = (%q, %q, true), want refusal", value, gotType, gotID) + } + } + + remade := collectionID("Patient", "patient-1") + if gotType, gotID, ok := SplitCollectionID(remade); !ok || gotType != "Patient" || gotID != "patient-1" { + t.Errorf("SplitCollectionID(collectionID(...)) = (%q, %q, %t), want (Patient, patient-1, true)", gotType, gotID, ok) + } +} diff --git a/internal/ingest/file_pipeline_test.go b/internal/ingest/file_pipeline_test.go index 28fdf3f6f..99eed419f 100644 --- a/internal/ingest/file_pipeline_test.go +++ b/internal/ingest/file_pipeline_test.go @@ -13,7 +13,7 @@ import ( "time" "github.com/bmeg/jsonschemagraph/graph" - arangostore "github.com/calypr/loom/internal/store/arango" + arangofactory "github.com/calypr/loom/internal/backends/arango" ) func TestLoadFileSupportsGenerationAndCancellation(t *testing.T) { @@ -29,7 +29,7 @@ func TestLoadFileSupportsGenerationAndCancellation(t *testing.T) { result, err := loadFile( context.Background(), opts, nil, schema, file, "generation-1", false, time.Now(), 0, 0, - func(context.Context, *arangostore.Client, string, []json.RawMessage, bool, string) error { + func(context.Context, *arangofactory.Client, string, []json.RawMessage, bool, string) error { return nil }, ) @@ -42,7 +42,7 @@ func TestLoadFileSupportsGenerationAndCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err = loadFile(ctx, opts, nil, schema, file, "generation-1", false, time.Now(), 0, 0, func(context.Context, *arangostore.Client, string, []json.RawMessage, bool, string) error { + _, err = loadFile(ctx, opts, nil, schema, file, "generation-1", false, time.Now(), 0, 0, func(context.Context, *arangofactory.Client, string, []json.RawMessage, bool, string) error { return nil }) if !errors.Is(err, context.Canceled) { @@ -71,7 +71,7 @@ func TestLoadFilePreservesGenerationWrites(t *testing.T) { var documents []json.RawMessage var collections []string var overwrites []bool - insert := func(_ context.Context, _ *arangostore.Client, collection string, docs []json.RawMessage, overwrite bool, _ string) error { + insert := func(_ context.Context, _ *arangofactory.Client, collection string, docs []json.RawMessage, overwrite bool, _ string) error { mu.Lock() defer mu.Unlock() collections = append(collections, collection) @@ -109,7 +109,9 @@ func TestLoadFileReportsPhysicalLinesAndBoundedErrors(t *testing.T) { t.Fatal(err) } opts := normalizeLoadOptions(LoadOptions{Project: "project"}) - insert := func(context.Context, *arangostore.Client, string, []json.RawMessage, bool, string) error { return nil } + insert := func(context.Context, *arangofactory.Client, string, []json.RawMessage, bool, string) error { + return nil + } t.Run("fail fast physical line", func(t *testing.T) { file := filepath.Join(t.TempDir(), "Patient.ndjson") @@ -165,7 +167,7 @@ func TestLoadFileBatchesAndPropagatesWriterFailure(t *testing.T) { } opts := normalizeLoadOptions(LoadOptions{Project: "project", BatchSize: 1, WriterCount: 2}) var inserted int64 - result, err := loadFile(context.Background(), opts, nil, schema, file, "generation-1", false, time.Now(), 0, 0, func(_ context.Context, _ *arangostore.Client, _ string, docs []json.RawMessage, _ bool, _ string) error { + result, err := loadFile(context.Background(), opts, nil, schema, file, "generation-1", false, time.Now(), 0, 0, func(_ context.Context, _ *arangofactory.Client, _ string, docs []json.RawMessage, _ bool, _ string) error { atomic.AddInt64(&inserted, int64(len(docs))) return nil }) @@ -179,7 +181,7 @@ func TestLoadFileBatchesAndPropagatesWriterFailure(t *testing.T) { writeErr := errors.New("writer failed") ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - _, err = loadFile(ctx, opts, nil, schema, file, "generation-1", false, time.Now(), 0, 0, func(context.Context, *arangostore.Client, string, []json.RawMessage, bool, string) error { + _, err = loadFile(ctx, opts, nil, schema, file, "generation-1", false, time.Now(), 0, 0, func(context.Context, *arangofactory.Client, string, []json.RawMessage, bool, string) error { return writeErr }) if !errors.Is(err, writeErr) { diff --git a/internal/ingest/generation_failed_cleanup_e2e_test.go b/internal/ingest/generation_failed_cleanup_e2e_test.go index 168967b90..f0ec432eb 100644 --- a/internal/ingest/generation_failed_cleanup_e2e_test.go +++ b/internal/ingest/generation_failed_cleanup_e2e_test.go @@ -1,52 +1,5 @@ package ingest -// DURABLE guarantee pin for the pre-STAGED failure leg of loadGeneration -// (generation_orchestration.go:179-190 deferred cleanup + :289-295 -// GenerationLoadIncompleteError). Recorded as coverage candidate (c) in -// FINDINGS.md pass 2026-11-04i: no prior durable test drives a real mid-load -// failure against ArangoDB and observes the store-visible consequences. -// -// PINNED CONTRACT (pre-stated before execution): -// 1. A load that passes preflight then accrues row errors returns -// GenerationLoadIncompleteError and the persisted manifest is FAILED — -// never left LOADING (evidence: store-visible ReadManifest readback; -// guard-level CAS-vs-blind-UPDATE equivalence is pinned by -// internal/dataset/arango/generation_activation_cas_e2e_test.go, not -// by this file). -// 2. The failed generation is never activatable: Store.Activate returns -// publication.ErrGenerationNotReady. -// 3. The failed generation is never active: ResolveActiveManifest returns -// publication.ErrNoActiveGeneration AND the lifecycle active document's -// manifestKey (read raw, so the resolver's own STAGED-only filter cannot -// mask a dangling pointer to the FAILED manifest) is absent. -// 4. It cannot be silently recreated: a second Load for the same dataset -// reference is rejected sequentially at the single-query -// publicationarango.CreateManifest claim with -// publication.ErrManifestAlreadyExists. (Concurrent-claim behavior is -// not pinned by this test.) -// 5. The paired semantic inventory build is marked failed -// (catalog.SemanticInventoryBuildCollection doc reaches state "failed"). -// 6. Positive control: a clean load yields a STAGED manifest reachable via -// ResolveActiveManifest; persisted row count is checked against the acked -// count (3/3); row-content equality is not asserted. -// -// FAULT MODEL (measured, not asserted-by-hope; see FINDINGS repair-27 review): -// M1 neutralize the deferred FAILED transition (generation_orchestration.go -// cleanup func returns nil) -> the observing RED witness is PIN 1. -// PINS 2-3 stay green under M1 (their guards reject every non-staged state -// identically, LOADING included); PIN 4 is state-agnostic (existence-based -// claim). PINS 2-4 are contract invariants, not sensors of the transition. -// M2 neutralize FailSemanticInventoryBuild -> the observing RED witness is -// PIN 5/5b (state stays 'running', diagnostics never set). -// M3 set the project's active.manifestKey to the failed manifest before the -// incomplete-error return -> the observing RED witness is PIN 3's raw -// active-document read (the resolver's STAGED-only join filter alone would -// stay green; measured via worker fault run). -// Each deferred cleanup leg therefore has exactly one dedicated sensor. -// -// Gating and owned-database lifecycle follow the package precedent -// keycollide_generated_e2e_test.go (skip-clean without LOOM_TEST_ARANGO_URL). - import ( "context" "encoding/json" @@ -60,10 +13,9 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" catalog "github.com/calypr/loom/internal/catalog" publication "github.com/calypr/loom/internal/dataset" - publicationarango "github.com/calypr/loom/internal/dataset/arango" - arangostore "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -95,9 +47,6 @@ func TestGenerationLoadFailureMarksManifestFailedAndBlocksActivation(t *testing. for i := 0; i < 25; i++ { rows = append(rows, failLegPatientRow(i)) } - // Malformed line placed beyond the 10-row preflight sample window - // (preflight.go:13 defaultPreflightSampleRows) so the claim, bootstrap - // and inventory begin all execute before the failure lands. rows = append(rows, `{oops`) for i := 25; i < 30; i++ { rows = append(rows, failLegPatientRow(i)) @@ -111,7 +60,7 @@ func TestGenerationLoadFailureMarksManifestFailedAndBlocksActivation(t *testing. t.Fatalf("ref: %v", err) } _, loadErr := Load(ctx, LoadOptions{ - ConnectionOptions: arangostore.ConnectionOptions{URL: endpoint, Database: database}, + ConnectionOptions: arangofactory.ConnectionOptions{URL: endpoint, Database: database}, Schema: repoPath(t, "schemas", "graph-fhir.json"), MetaDir: metaDir, Project: project, @@ -128,12 +77,13 @@ func TestGenerationLoadFailureMarksManifestFailedAndBlocksActivation(t *testing. } t.Logf("FAILLEG fault load | %v", loadErr) - client, err := arangostore.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatalf("open: %v", err) } + client := arangoBackend.Client defer func() { _ = client.Close(context.Background()) }() - store, err := publicationarango.New(client) + store, err := arangofactory.NewDatasetStore(client) if err != nil { t.Fatalf("lifecycle store: %v", err) } @@ -152,9 +102,6 @@ func TestGenerationLoadFailureMarksManifestFailedAndBlocksActivation(t *testing. if active, err := store.ResolveActiveManifest(ctx, project); !errors.Is(err, publication.ErrNoActiveGeneration) { t.Errorf("PIN 3 | failed generation reached active pointer: active=%+v err=%v", active, err) } - // Raw read, independent of readActiveAQL's manifestKey!=null and - // STAGED/READY join filters: a dangling pointer pinned to the FAILED - // manifest must not hide behind the resolver's own guard. activeKey, pointerFound, err := failLegReadActivePointer(ctx, client, project) if err != nil { t.Fatalf("PIN 3 raw active read: %v", err) @@ -164,17 +111,14 @@ func TestGenerationLoadFailureMarksManifestFailedAndBlocksActivation(t *testing. } if _, err := Load(ctx, LoadOptions{ - ConnectionOptions: arangostore.ConnectionOptions{URL: endpoint, Database: database}, + ConnectionOptions: arangofactory.ConnectionOptions{URL: endpoint, Database: database}, Schema: repoPath(t, "schemas", "graph-fhir.json"), MetaDir: metaDir, Project: project, Dataset: &ref, BatchSize: 10, ProgressEvery: 100, - }); !errors.Is(err, publicationarango.ErrManifestAlreadyExists) { - // FAILED blocks silent re-creation through the atomic claim: the - // second Load must die at CreateManifest, never reaching the - // loader loop again. + }); !errors.Is(err, arangofactory.ErrManifestAlreadyExists) { t.Errorf("PIN 4 | re-load of failed generation returned %v, want ErrManifestAlreadyExists", err) } @@ -213,7 +157,7 @@ func TestGenerationLoadFailureMarksManifestFailedAndBlocksActivation(t *testing. t.Fatalf("ref: %v", err) } summary, err := Load(ctx, LoadOptions{ - ConnectionOptions: arangostore.ConnectionOptions{URL: endpoint, Database: database}, + ConnectionOptions: arangofactory.ConnectionOptions{URL: endpoint, Database: database}, Schema: repoPath(t, "schemas", "graph-fhir.json"), MetaDir: metaDir, Project: project, @@ -224,12 +168,13 @@ func TestGenerationLoadFailureMarksManifestFailedAndBlocksActivation(t *testing. if err != nil { t.Fatalf("positive control load failed: %v", err) } - client, err := arangostore.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatalf("open: %v", err) } + client := arangoBackend.Client defer func() { _ = client.Close(context.Background()) }() - store, err := publicationarango.New(client) + store, err := arangofactory.NewDatasetStore(client) if err != nil { t.Fatalf("lifecycle store: %v", err) } @@ -252,8 +197,8 @@ func TestGenerationLoadFailureMarksManifestFailedAndBlocksActivation(t *testing. t.Cleanup(func() { failLegAssertNoResidue(t, endpoint) }) } -func failLegReadActivePointer(ctx context.Context, client *arangostore.Client, project string) (string, bool, error) { - exists, err := client.CollectionExists(ctx, publicationarango.LifecycleCollection) +func failLegReadActivePointer(ctx context.Context, client *arangofactory.Client, project string) (string, bool, error) { + exists, err := client.CollectionExists(ctx, arangofactory.DatasetLifecycleCollection) if err != nil { return "", false, err } @@ -263,8 +208,8 @@ func failLegReadActivePointer(ctx context.Context, client *arangostore.Client, p var manifestKey string var found bool err = client.QueryRows(ctx, `FOR a IN @@lifecycle FILTER a._key == @active_key AND a.recordType == "active_generation" RETURN {manifestKey: a.manifestKey}`, 1, map[string]any{ - "@lifecycle": publicationarango.LifecycleCollection, - "active_key": publicationarango.ActiveDocumentKey(project), + "@lifecycle": arangofactory.DatasetLifecycleCollection, + "active_key": arangofactory.DatasetActiveDocumentKey(project), }, func(row map[string]any) error { value, _ := row["manifestKey"].(string) manifestKey, found = value, true @@ -273,7 +218,7 @@ func failLegReadActivePointer(ctx context.Context, client *arangostore.Client, p return manifestKey, found, err } -func failLegReadBuildState(ctx context.Context, client *arangostore.Client, collection, project, generation string) (string, string, int, bool, error) { +func failLegReadBuildState(ctx context.Context, client *arangofactory.Client, collection, project, generation string) (string, string, int, bool, error) { exists, err := client.CollectionExists(ctx, collection) if err != nil { return "", "", 0, false, err diff --git a/internal/ingest/generation_identity.go b/internal/ingest/generation_identity.go index a8e3d1d18..b6a582eca 100644 --- a/internal/ingest/generation_identity.go +++ b/internal/ingest/generation_identity.go @@ -8,25 +8,10 @@ import ( "strings" ) -// generationIdentityField is deliberately distinct from the resource payload -// and from the loader's error-generation counters. It identifies the immutable -// Loom dataset generation that owns a persisted graph document. const generationIdentityField = "dataset_generation" -// logicalKeyField preserves the pre-generation Arango key for operational -// diagnostics and future stable external-ID policy. It is not used for graph -// traversal: _key, _from, and _to are all generation-qualified below. const logicalKeyField = "logical_key" -// generationRowBuilder is the one identity boundary shared by the generated -// and generic FHIR row builders. Keeping it here avoids a second FHIR model or -// a generator-wide change: both builders first produce the established Loom -// vertex/edge representation, then this wrapper qualifies its physical graph -// identity for one immutable dataset generation. -// -// It is intentionally not selected by Load until the dataset lifecycle owns -// manifest creation and active-generation reads. Writing qualified identities -// without generation-qualified readers would be an incomplete migration. type generationRowBuilder struct { delegate RowBuilder project string @@ -64,12 +49,6 @@ func (b *generationRowBuilder) Build(resourceType string, line []byte, stageSeco return namespaced, "", nil } -// namespaceRowBuildResult rewrites one generated or generic result into a -// generation-isolated graph identity. The original logical FHIR ID remains in -// the existing top-level id field; logical_key captures the original Arango -// key. Hashing is necessary because keys are global to a collection, while -// both resource IDs and generated edge IDs are otherwise reused by later -// project loads and dataset generations. func namespaceRowBuildResult(result rowBuildResult, project, datasetGeneration, resourceType string) (rowBuildResult, error) { project = strings.TrimSpace(project) datasetGeneration = strings.TrimSpace(datasetGeneration) diff --git a/internal/ingest/generation_load.go b/internal/ingest/generation_load.go new file mode 100644 index 000000000..63c804852 --- /dev/null +++ b/internal/ingest/generation_load.go @@ -0,0 +1,984 @@ +package ingest + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + "time" + + "github.com/bmeg/jsonschemagraph/graph" + postgres "github.com/calypr/loom/internal/backends/postgres" + "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/catalog" + publication "github.com/calypr/loom/internal/dataset" +) + +const ( + pgResourceBatchPass = "r" + pgEdgeBatchPass = "e" + pgBatchIDPrefix = "ing03:" + pgApplicationNameBase = "loom-ing03" + pgSemanticFlushMinimum = 2000 +) + +type PGSourceOptions struct { + DSN string + Schema string + ManifestSchema string + MaxConns int32 + AcquireTimeout time.Duration + ApplicationName string + SemanticFlushThreshold int + Catalog GenerationCatalogWriters +} + +var ( + ErrGenerationAlreadyStaged = errors.New("dataset generation is already staged") + ErrGenerationLoadAlreadyFailed = errors.New("dataset generation load already failed") + ErrGenerationManifestIdentityConflict = errors.New("dataset generation manifest identity conflict") + ErrGenerationManifestOutcomeUnknown = errors.New("dataset generation manifest commit outcome is unknown") + ErrPGGenerationDatasetRequired = errors.New("postgres generation load requires an explicit dataset generation") + ErrGenerationLoadRequiresCatalogWriters = errors.New("postgres generation load requires all four catalog evidence writers") +) + +type GenerationFieldProfileWriter interface { + WriteFieldProfile(ctx context.Context, scope control.FieldProfileScope, docs []catalog.FieldCatalogDocument, batchSize int) error +} + +type GenerationInventoryRebuilder interface { + RebuildResourceInventory(ctx context.Context, opts catalog.ResourceInventoryRebuildOptions) (catalog.ResourceInventoryRebuildSummary, error) +} + +type GenerationRelationshipDiscoverer interface { + DiscoverRelationshipObservations(ctx context.Context, opts catalog.RelationshipObservationOptions) (catalog.RelationshipObservationResult, error) +} + +type GenerationSemanticContributor interface { + WriteSemanticInventoryContributions(ctx context.Context, contributions []catalog.SemanticInventoryContribution, batchSize int) error +} + +type GenerationCatalogWriters struct { + FieldProfiles GenerationFieldProfileWriter + Inventory GenerationInventoryRebuilder + Relationships GenerationRelationshipDiscoverer + Contributions GenerationSemanticContributor +} + +func (w GenerationCatalogWriters) Complete() bool { + return w.FieldProfiles != nil && w.Inventory != nil && + w.Relationships != nil && w.Contributions != nil +} + +type pgGenerationLoad struct { + opts LoadOptions + pg PGSourceOptions + plan *generationLoadPlan + ref publication.Ref + + runtime *postgres.Runtime + lifecycle publication.ManifestLifecycle + resourceW *PostgresResourceWriter + edgeW *PostgresEdgeWriter + profiles GenerationFieldProfileWriter + inventory GenerationInventoryRebuilder + relations GenerationRelationshipDiscoverer + contribs GenerationSemanticContributor + application string + + manifest publication.Manifest + ownsManifest bool + staged bool + + pendingCatalogs map[generationCatalogKey]*catalog.Profiler + semanticBuffer []catalog.SemanticInventoryContribution + semanticFlush int + + resourceWindow int + edgeWindow int + profileBatch int + inventoryBatch int +} + +func LoadPostgresGeneration(ctx context.Context, opts LoadOptions, pg PGSourceOptions) (summary LoadSummary, err error) { + start := time.Now() + opts = normalizeLoadOptions(opts) + if !pg.Catalog.Complete() { + return LoadSummary{}, fmt.Errorf("%w: field profiles, inventory, relationships, and semantic contributions must all be wired", ErrGenerationLoadRequiresCatalogWriters) + } + + if info, statErr := os.Stat(opts.MetaDir); statErr != nil || !info.IsDir() { + return LoadSummary{}, fmt.Errorf("%w: %s", ErrGenerationLoadRequiresDirectory, opts.MetaDir) + } + files, err := DiscoverNDJSON(opts.MetaDir) + if err != nil { + return LoadSummary{}, err + } + if len(files) == 0 { + return LoadSummary{}, fmt.Errorf("%w: %s", ErrGenerationLoadRequiresFiles, opts.MetaDir) + } + summary = LoadSummary{ + Files: len(files), + Resources: map[string]int{}, + BatchCounts: map[string]int{}, + StageSeconds: map[string]float64{}, + } + + schema, err := graph.Load(opts.Schema) + if err != nil { + return summary, err + } + schemaIdentity, err := loadSchemaSnapshot(opts.Schema) + if err != nil { + return summary, err + } + summary.SchemaIdentity = &schemaIdentity + preflightSampleRows := opts.PreflightSampleRows + if preflightSampleRows <= 0 { + preflightSampleRows = defaultPreflightSampleRows + } + emitEvent(opts.EventSink, "go_preflight_start", map[string]any{ + "files": len(files), + "sampleRows": preflightSampleRows, + "schemaSha256": schemaIdentity.SchemaSHA256, + "generatedRootCount": len(schemaIdentity.GeneratedResourceTypes), + "backend": "postgres", + }) + preflightStart := time.Now() + preflight, err := PreflightFiles(files, schema, preflightSampleRows) + summary.Preflight = preflight + summary.StageSeconds["preflight"] = time.Since(preflightStart).Seconds() + emitEvent(opts.EventSink, "go_preflight_complete", map[string]any{ + "files": len(files), + "resources": len(preflight.Resources), + "issues": len(preflight.Issues), + "seconds": summary.StageSeconds["preflight"], + }) + if err != nil { + return summary, err + } + + plan, err := newGenerationLoadPlan(opts, files, schemaIdentity) + if err != nil { + return summary, err + } + if plan == nil { + return summary, fmt.Errorf("dataset generation plan is required") + } + if strings.TrimSpace(plan.Dataset.Generation) == "" { + return summary, ErrPGGenerationDatasetRequired + } + datasetRef := plan.Dataset + summary.Dataset = &datasetRef + if err := ctx.Err(); err != nil { + return summary, safePointCancel(err) + } + + resourceTypes := make([]string, 0, len(preflight.Resources)) + for _, resource := range preflight.Resources { + resourceTypes = append(resourceTypes, resource.ResourceType) + } + + load := &pgGenerationLoad{opts: opts, pg: pg, plan: plan, ref: datasetRef} + if load.application = strings.TrimSpace(pg.ApplicationName); load.application == "" { + load.application = fmt.Sprintf("%s-%d-%s", pgApplicationNameBase, time.Now().UnixNano(), pgRandomHexToken()) + } + + emitEvent(opts.EventSink, "go_pg_connect_start", map[string]any{ + "backend": "postgres", + "schema": pg.Schema, + }) + connectStart := time.Now() + if err := load.open(); err != nil { + return summary, err + } + defer func() { + err = generationCleanupError(ctx, generationCleanupTimeout, err, "close postgres generation backend", + func(context.Context) error { return load.close() }) + }() + summary.StageSeconds["pg_connect"] = time.Since(connectStart).Seconds() + emitEvent(opts.EventSink, "go_pg_connect_complete", map[string]any{ + "backend": "postgres", + "schema": pg.Schema, + "seconds": summary.StageSeconds["pg_connect"], + "app_name": load.application, + "resources": len(resourceTypes), + }) + + emitEvent(opts.EventSink, "go_pg_bootstrap_start", map[string]any{ + "schema": pg.Schema, + "truncate": false, + }) + bootstrapStart := time.Now() + if _, err := load.runtime.MigrateSchema(ctx); err != nil { + return summary, err + } + if _, err := load.runtime.MigrateDatasetReleaseSchema(ctx); err != nil { + return summary, err + } + summary.StageSeconds["pg_bootstrap"] = time.Since(bootstrapStart).Seconds() + emitEvent(opts.EventSink, "go_pg_bootstrap_complete", map[string]any{ + "schema": pg.Schema, + "seconds": summary.StageSeconds["pg_bootstrap"], + }) + + if err := load.claimManifest(ctx); err != nil { + return summary, err + } + defer func() { + if err == nil || load.staged || !load.ownsManifest { + return + } + if pgCommitOutcomeUnknown(err) { + return + } + err = generationCleanupError(ctx, generationCleanupTimeout, err, + fmt.Sprintf("mark dataset generation %s/%s failed", plan.Dataset.Project, plan.Dataset.Generation), + func(cleanupCtx context.Context) error { + _, transitionErr := load.lifecycle.TransitionManifest(cleanupCtx, load.manifest, publication.StateFailed) + return transitionErr + }) + }() + + var rowErrorCandidates [][]RowErrorSample + defer func() { + summary.RowErrors = selectGenerationRowErrorSamples(rowErrorCandidates) + }() + + if err := load.writeResourcePass(ctx, schema, files, &summary, &rowErrorCandidates); err != nil { + return summary, load.surfaceStoreError(err, "resource pass") + } + if summary.ValidationErrors != 0 || summary.GenerationErrors != 0 || summary.EdgeErrors != 0 { + return summary, &GenerationLoadIncompleteError{ + Dataset: plan.Dataset, + ValidationErrors: summary.ValidationErrors, + GenerationErrors: summary.GenerationErrors, + EdgeErrors: summary.EdgeErrors, + } + } + if err := ctx.Err(); err != nil { + return summary, safePointCancel(err) + } + + if err := load.writeEdgePass(ctx, schema, files, &summary); err != nil { + return summary, load.surfaceStoreError(err, "edge pass") + } + if err := ctx.Err(); err != nil { + return summary, safePointCancel(err) + } + + if err := load.writeCatalogEvidence(ctx, schemaIdentity, &summary); err != nil { + return summary, load.surfaceStoreError(err, "catalog evidence") + } + if err := ctx.Err(); err != nil { + return summary, safePointCancel(err) + } + + stagedManifest, transitionErr := load.lifecycle.TransitionManifest(ctx, load.manifest, publication.StateStaged) + if transitionErr != nil { + if pgCommitOutcomeUnknown(transitionErr) { + return summary, fmt.Errorf("%w: stage dataset generation %s/%s: %w", + ErrGenerationManifestOutcomeUnknown, plan.Dataset.Project, plan.Dataset.Generation, transitionErr) + } + if errors.Is(transitionErr, control.ErrManifestTransitionConflict) { + if stored, readErr := load.lifecycle.ReadManifest(ctx, load.ref); readErr == nil { + if stored.IsStaged() { + return summary, fmt.Errorf("%w: %s/%s is already STAGED (a concurrent loader of the same identity won the transition); read back the committed load instead of re-running it", + ErrGenerationAlreadyStaged, plan.Dataset.Project, plan.Dataset.Generation) + } + if stored.State == publication.StateFailed { + return summary, fmt.Errorf("%w: %s/%s; a new dataset generation is required", + ErrGenerationLoadAlreadyFailed, plan.Dataset.Project, plan.Dataset.Generation) + } + } + } + return summary, transitionErr + } + load.manifest = stagedManifest + load.staged = true + + emitEvent(opts.EventSink, "go_load_complete", map[string]any{ + "backend": "postgres", + "files": summary.Files, + "vertices_inserted": summary.VerticesInserted, + "edges_inserted": summary.EdgesInserted, + "validation_errors": summary.ValidationErrors, + "generation_errors": summary.GenerationErrors, + "edge_errors": summary.EdgeErrors, + "dataset_generation": plan.Dataset.Generation, + "seconds": SecondsSince(start), + }) + return summary, nil +} + +func ActivatePostgresGeneration(ctx context.Context, ref publication.Ref, pg PGSourceOptions) error { + if err := ref.Validate(); err != nil { + return err + } + if strings.TrimSpace(pg.ApplicationName) == "" { + pg.ApplicationName = pgApplicationNameBase + "-activate" + } + client, err := openPGGenerationClient(pg, nil) + if err != nil { + return err + } + defer client.Close() + store := client.Manifests + if store == nil { + return fmt.Errorf("postgres runtime exposes no dataset manifest lifecycle port; the DatasetReleaseSeries schema must be assigned in the factory config") + } + manifest, err := store.ReadManifest(ctx, ref) + if err != nil { + return err + } + if !manifest.IsStaged() { + if err := manifest.Validate(); err != nil { + return err + } + return fmt.Errorf("activate dataset generation %s/%s: %w: state is %s", + ref.Project, ref.Generation, publication.ErrGenerationNotReady, manifest.State) + } + var expected *publication.Ref + active, resolveErr := store.ResolveActiveManifest(ctx, ref.Project) + switch { + case resolveErr == nil: + if active.IsStaged() { + expected = &active.Dataset + } + case errors.Is(resolveErr, publication.ErrNoActiveGeneration): + expected = nil + default: + expected = nil + } + if activationErr := store.ActivateGeneration(ctx, manifest, expected); activationErr != nil { + if errors.Is(activationErr, control.ErrActivationConflict) || + errors.Is(activationErr, publication.ErrGenerationNotReady) || + errors.Is(activationErr, publication.ErrInvalidActiveGeneration) { + return activationErr + } + if pgCommitOutcomeUnknown(activationErr) { + return &ActivationOutcomeError{Dataset: ref, Err: activationErr} + } + return activationErr + } + return nil +} + +func openPGGenerationClient(pg PGSourceOptions, eventSink EventSink) (*postgres.Runtime, error) { + maxConns := pg.MaxConns + if maxConns == 0 { + maxConns = 4 + } + acquireTimeout := pg.AcquireTimeout + if acquireTimeout <= 0 { + acquireTimeout = 30 * time.Second + } + dsn := strings.TrimSpace(pg.DSN) + application := strings.TrimSpace(pg.ApplicationName) + if application != "" { + var tagged bool + dsn, tagged = joinApplicationName(dsn, application) + if !tagged { + emitEvent(eventSink, "go_pg_application_name_skipped", map[string]any{ + "application": application, + "reason": "DSN is not URL-form; keyword/value DSNs keep an untagged connection rather than a guessed syntax", + }) + } + } + return postgres.Open(context.Background(), PGLoadConfig(pg, dsn, control.SetupNone)) +} + +func PGLoadConfig(pg PGSourceOptions, taggedDSN string, setup control.SetupPolicy) control.Config { + maxConns := pg.MaxConns + if maxConns == 0 { + maxConns = 4 + } + acquireTimeout := pg.AcquireTimeout + if acquireTimeout <= 0 { + acquireTimeout = 30 * time.Second + } + schemas := make(control.SeriesSchemas, 2) + schemas[control.SeriesSource] = pg.Schema + schemas[control.SeriesDatasetRelease] = pg.ManifestSchema + return control.Config{ + DSN: taggedDSN, + MaxConns: maxConns, + MinConns: 0, + AcquireTimeout: acquireTimeout, + Schemas: schemas, + Setup: setup, + } +} + +func (l *pgGenerationLoad) open() error { + runtime, err := openPGGenerationClient(l.pg, l.opts.EventSink) + if err != nil { + return err + } + l.runtime = runtime + if err := l.bind(); err != nil { + l.close() + return err + } + return nil +} + +func (l *pgGenerationLoad) close() error { + if l.runtime != nil { + _ = l.runtime.Close() + l.runtime = nil + } + return nil +} + +func (l *pgGenerationLoad) bind() error { + schema := l.pg.Schema + var err error + if l.runtime.Manifests == nil { + return fmt.Errorf("postgres runtime exposes no dataset manifest lifecycle port; the DatasetReleaseSeries schema must be assigned in the factory config") + } + l.lifecycle = l.runtime.Manifests + if l.resourceW, err = NewPostgresResourceWriter(l.runtime.ConnSource(), schema); err != nil { + return err + } + if l.edgeW, err = NewPostgresEdgeWriter(l.runtime.ConnSource(), schema); err != nil { + return err + } + l.profiles = l.pg.Catalog.FieldProfiles + l.inventory = l.pg.Catalog.Inventory + l.relations = l.pg.Catalog.Relationships + l.contribs = l.pg.Catalog.Contributions + l.semanticFlush = l.pg.SemanticFlushThreshold + if l.semanticFlush <= 0 { + l.semanticFlush = 2 * l.opts.BatchSize + if l.semanticFlush < pgSemanticFlushMinimum { + l.semanticFlush = pgSemanticFlushMinimum + } + } + l.resourceWindow = min(l.opts.BatchSize, control.MaxBatchResources) + l.edgeWindow = min(l.opts.BatchSize, control.MaxBatchEdges) + l.profileBatch = min(l.opts.BatchSize, control.MaxFieldProfileWriteBatch) + l.inventoryBatch = min(l.opts.BatchSize, control.MaxInventoryWriteBatch) + return nil +} + +func (l *pgGenerationLoad) claimManifest(ctx context.Context) error { + created, err := l.lifecycle.CreateManifest(ctx, l.plan.Manifest) + if err == nil { + l.manifest = created + l.ownsManifest = true + emitEvent(l.opts.EventSink, "go_pg_manifest_create", map[string]any{ + "dataset_generation": l.ref.Generation, + "state": string(created.State), + }) + return nil + } + if !errors.Is(err, control.ErrManifestAlreadyExists) { + if pgCommitOutcomeUnknown(err) { + return fmt.Errorf("%w: create dataset generation manifest %s/%s: %w", + ErrGenerationManifestOutcomeUnknown, l.ref.Project, l.ref.Generation, err) + } + return err + } + stored, readErr := l.lifecycle.ReadManifest(ctx, l.ref) + if readErr != nil { + if pgCommitOutcomeUnknown(readErr) { + return fmt.Errorf("%w: read dataset generation manifest %s/%s: %w", + ErrGenerationManifestOutcomeUnknown, l.ref.Project, l.ref.Generation, readErr) + } + return readErr + } + if !reflect.DeepEqual(stored.SchemaIdentity, l.plan.Manifest.SchemaIdentity) { + return fmt.Errorf("%w: %s/%s stores a different immutable schema identity; one load cannot claim another load's manifest", + ErrGenerationManifestIdentityConflict, l.ref.Project, l.ref.Generation) + } + switch { + case stored.IsStaged(): + l.manifest = stored + return fmt.Errorf("%w: %s/%s is already STAGED; read back the committed load instead of re-running it", + ErrGenerationAlreadyStaged, l.ref.Project, l.ref.Generation) + case stored.State == publication.StateFailed: + l.manifest = stored + return fmt.Errorf("%w: %s/%s; a new dataset generation is required", + ErrGenerationLoadAlreadyFailed, l.ref.Project, l.ref.Generation) + default: + l.manifest = stored + l.ownsManifest = false + emitEvent(l.opts.EventSink, "go_pg_manifest_resume", map[string]any{ + "dataset_generation": l.ref.Generation, + "state": string(stored.State), + }) + return nil + } +} + +func (l *pgGenerationLoad) writeResourcePass( + ctx context.Context, + schema *graph.GraphSchema, + files []string, + summary *LoadSummary, + rowErrorCandidates *[][]RowErrorSample, +) error { + shapeCache := catalog.NewShapePlanCacheWithLimits(l.opts.CatalogLimits.MaxShapePlans, l.opts.CatalogLimits.MaxRetainedBytes) + catalogs := make(map[generationCatalogKey]*catalog.Profiler) + for _, file := range files { + if err := ctx.Err(); err != nil { + return safePointCancel(err) + } + fileStart := time.Now() + resourceType := ResourceTypeFromPath(file) + fileID := semanticInventorySourceFileID(l.opts.MetaDir, file) + emitEvent(l.opts.EventSink, "go_pg_load_file_start", map[string]any{ + "file": file, "resource": resourceType, "pass": "resources", + }) + + rowBuilder, err := l.newRowBuilder(schema, resourceType) + if err != nil { + return err + } + profiler := catalog.NewProfilerForGenerationWithLimits( + l.opts.Project, l.plan.Dataset.Generation, l.opts.AuthResourcePath, resourceType, + shapeCache, l.opts.CatalogLimits) + localTimings := map[string]float64{} + + scanner, closeFn, err := OpenLineScanner(file) + if err != nil { + return err + } + var windowRows []rowBuildResult + windowIndex := 0 + lineNumber := 0 + fileRows := 0 + var fileErrors fileLoadResult + scanErr := func() (loopErr error) { + defer func() { + if loopErr == nil && len(windowRows) > 0 { + loopErr = l.commitResourceWindow(ctx, fileID, windowIndex, windowRows, summary) + windowRows = nil + } + }() + for scanner.Scan() { + lineNumber++ + if err := ctx.Err(); err != nil { + return safePointCancel(err) + } + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + built, errorType, buildErr := rowBuilder.Build(resourceType, []byte(line), localTimings) + if buildErr != nil { + if l.opts.FailFast { + return fmt.Errorf("%s %s row %d: %w", filepath.Base(file), resourceType, lineNumber, buildErr) + } + *rowErrorCandidates = append(*rowErrorCandidates, []RowErrorSample{{ + File: filepath.Base(file), + Line: lineNumber, + ResourceType: resourceType, + Category: string(errorType), + Message: buildErr.Error(), + }}) + switch errorType { + case rowErrorValidation: + fileErrors.ValidationErrors++ + case rowErrorGeneration: + fileErrors.GenerationErrors++ + case rowErrorEdge: + fileErrors.EdgeErrors++ + } + continue + } + sourceID := semanticInventorySourceRecordID(fileID, lineNumber) + if err := profiler.ObservePayloadWithInventory(built.payload, localTimings, sourceID, + func(contribution catalog.SemanticInventoryContribution) { + l.semanticBuffer = append(l.semanticBuffer, contribution) + }); err != nil { + return fmt.Errorf("observe semantic inventory for %s row %d: %w", filepath.Base(file), lineNumber, err) + } + if len(l.semanticBuffer) >= l.semanticFlush { + if err := l.flushSemanticContributions(ctx, summary); err != nil { + return err + } + } + windowRows = append(windowRows, built) + fileRows++ + if lineNumber%l.resourceWindow == 0 { + if err := l.commitResourceWindow(ctx, fileID, windowIndex, windowRows, summary); err != nil { + return err + } + windowRows = nil + windowIndex++ + } + } + return scanner.Err() + }() + if err := errors.Join(scanErr, closeFn()); err != nil { + return err + } + + summary.Resources[resourceType] += fileRows + summary.ValidationErrors += fileErrors.ValidationErrors + summary.GenerationErrors += fileErrors.GenerationErrors + summary.EdgeErrors += fileErrors.EdgeErrors + for name, seconds := range localTimings { + summary.StageSeconds[name] += seconds + } + key := generationCatalogKey{ + project: l.opts.Project, + datasetGeneration: l.plan.Dataset.Generation, + authResourcePath: l.opts.AuthResourcePath, + resourceType: resourceType, + } + if merged := catalogs[key]; merged == nil { + catalogs[key] = profiler + } else if err := merged.Merge(profiler); err != nil { + return fmt.Errorf("merge catalog for dataset generation %s/%s %s: %w", + l.plan.Dataset.Project, l.plan.Dataset.Generation, resourceType, err) + } + emitEvent(l.opts.EventSink, "go_pg_load_file_complete", map[string]any{ + "file": filepath.Base(file), + "resource": resourceType, + "file_rows": fileRows, + "pass": "resources", + "seconds": time.Since(fileStart).Seconds(), + }) + } + l.pendingCatalogs = catalogs + return nil +} + +func (l *pgGenerationLoad) commitResourceWindow(ctx context.Context, fileID string, window int, rows []rowBuildResult, summary *LoadSummary) error { + if len(rows) == 0 { + return nil + } + if err := ctx.Err(); err != nil { + return safePointCancel(err) + } + batchID := pgGenerationBatchID(l.plan.Dataset.Project, l.plan.Dataset.Generation, fileID, pgResourceBatchPass, window) + result, err := l.resourceW.WriteResourceBatch(ctx, PGResourceBatchScope{ + BatchID: batchID, + Project: l.plan.Dataset.Project, + DatasetGeneration: l.plan.Dataset.Generation, + AuthResourcePath: l.opts.AuthResourcePath, + }, rows) + if err != nil { + return fmt.Errorf("commit postgres resource batch %s (%d rows): %w", batchID, len(rows), err) + } + if !result.Replay { + summary.VerticesInserted += int(result.Committed) + } + summary.BatchCounts["resource_batch"]++ + if result.Replay { + summary.BatchCounts["resource_batch_replay"]++ + } + emitEvent(l.opts.EventSink, "go_pg_resource_batch_commit", map[string]any{ + "batch": batchID, + "offered": result.Offered, + "commit": result.Committed, + "replay": result.Replay, + "digest": result.Digest, + }) + return nil +} + +func (l *pgGenerationLoad) writeEdgePass(ctx context.Context, schema *graph.GraphSchema, files []string, summary *LoadSummary) error { + for _, file := range files { + if err := ctx.Err(); err != nil { + return safePointCancel(err) + } + fileStart := time.Now() + resourceType := ResourceTypeFromPath(file) + fileID := semanticInventorySourceFileID(l.opts.MetaDir, file) + emitEvent(l.opts.EventSink, "go_pg_load_file_start", map[string]any{ + "file": file, "resource": resourceType, "pass": "edges", + }) + rowBuilder, err := l.newRowBuilder(schema, resourceType) + if err != nil { + return err + } + localTimings := map[string]float64{} + scanner, closeFn, err := OpenLineScanner(file) + if err != nil { + return err + } + var windowRows []rowBuildResult + windowIndex := 0 + lineNumber := 0 + windowEdges := 0 + scanErr := func() (loopErr error) { + defer func() { + if loopErr == nil && len(windowRows) > 0 { + loopErr = l.commitEdgeWindow(ctx, fileID, windowIndex, windowRows, summary) + windowRows = nil + } + }() + for scanner.Scan() { + lineNumber++ + if err := ctx.Err(); err != nil { + return safePointCancel(err) + } + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + built, _, buildErr := rowBuilder.Build(resourceType, []byte(line), localTimings) + if buildErr != nil { + return fmt.Errorf("%s %s row %d rebuilt with error: %w", filepath.Base(file), resourceType, lineNumber, buildErr) + } + if len(built.edges) > control.MaxBatchEdges { + return fmt.Errorf("%s %s row %d produces %d edges, exceeding the %d-per-batch edge bound; nothing was written", + filepath.Base(file), resourceType, lineNumber, len(built.edges), control.MaxBatchEdges) + } + if len(windowRows) > 0 && windowEdges+len(built.edges) > control.MaxBatchEdges { + if err := l.commitEdgeWindow(ctx, fileID, windowIndex, windowRows, summary); err != nil { + return err + } + windowRows = nil + windowEdges = 0 + windowIndex++ + } + if len(built.edges) > 0 { + windowRows = append(windowRows, built) + windowEdges += len(built.edges) + } + if lineNumber%l.edgeWindow == 0 { + if err := l.commitEdgeWindow(ctx, fileID, windowIndex, windowRows, summary); err != nil { + return err + } + windowRows = nil + windowEdges = 0 + windowIndex++ + } + } + return scanner.Err() + }() + if err := errors.Join(scanErr, closeFn()); err != nil { + return err + } + emitEvent(l.opts.EventSink, "go_pg_load_file_complete", map[string]any{ + "file": filepath.Base(file), + "resource": resourceType, + "pass": "edges", + "seconds": time.Since(fileStart).Seconds(), + }) + } + return nil +} + +func (l *pgGenerationLoad) commitEdgeWindow(ctx context.Context, fileID string, window int, rows []rowBuildResult, summary *LoadSummary) error { + if len(rows) == 0 { + return nil + } + if err := ctx.Err(); err != nil { + return safePointCancel(err) + } + batchID := pgGenerationBatchID(l.plan.Dataset.Project, l.plan.Dataset.Generation, fileID, pgEdgeBatchPass, window) + result, err := l.edgeW.WriteEdgeBatch(ctx, PGEdgeBatchScope{ + BatchID: batchID, + Project: l.plan.Dataset.Project, + DatasetGeneration: l.plan.Dataset.Generation, + AuthResourcePath: l.opts.AuthResourcePath, + }, rows) + if err != nil { + return fmt.Errorf("commit postgres edge batch %s (%d rows): %w", batchID, len(rows), err) + } + if !result.Replay { + summary.EdgesInserted += int(result.Committed) + } + summary.BatchCounts["edge_batch"]++ + if result.Replay { + summary.BatchCounts["edge_batch_replay"]++ + } + emitEvent(l.opts.EventSink, "go_pg_edge_batch_commit", map[string]any{ + "batch": batchID, + "offered": result.Offered, + "commit": result.Committed, + "replay": result.Replay, + "digest": result.Digest, + }) + return nil +} + +func (l *pgGenerationLoad) writeCatalogEvidence(ctx context.Context, schemaIdentity publication.SchemaSnapshot, summary *LoadSummary) error { + fieldStart := time.Now() + keys := sortedGenerationCatalogKeys(l.pendingCatalogs) + for _, key := range keys { + if err := ctx.Err(); err != nil { + return safePointCancel(err) + } + scope := control.FieldProfileScope{ + Project: key.project, + DatasetGeneration: key.datasetGeneration, + AuthResourcePath: key.authResourcePath, + ResourceType: key.resourceType, + } + if err := l.profiles.WriteFieldProfile(ctx, scope, l.pendingCatalogs[key].Documents(), l.profileBatch); err != nil { + return fmt.Errorf("write field profile for %s/%s %s: %w", key.project, key.datasetGeneration, key.resourceType, err) + } + summary.BatchCounts["field_profile_write"]++ + } + summary.StageSeconds["catalog_field_profiles"] = time.Since(fieldStart).Seconds() + emitEvent(l.opts.EventSink, "go_pg_catalog_field_profiles", map[string]any{"scopes": len(keys)}) + + inventoryStart := time.Now() + var inventory catalog.ResourceInventoryRebuildSummary + inventoryErr := control.WithAdvisoryLock(ctx, l.runtime.ConnSource(), + control.GenerationAdvisoryLockKey( + l.plan.Dataset.Project, l.plan.Dataset.Generation), + func() error { + var rebuildErr error + inventory, rebuildErr = l.inventory.RebuildResourceInventory(ctx, catalog.ResourceInventoryRebuildOptions{ + Project: l.plan.Dataset.Project, + DatasetGeneration: l.plan.Dataset.Generation, + ResourceTypes: schemaIdentity.GeneratedResourceTypes, + BatchSize: l.inventoryBatch, + }) + return rebuildErr + }) + if inventoryErr != nil { + return fmt.Errorf("build resource inventory for dataset generation %s/%s: %w", + l.plan.Dataset.Project, l.plan.Dataset.Generation, inventoryErr) + } + summary.StageSeconds["catalog_resource_inventory"] = time.Since(inventoryStart).Seconds() + summary.BatchCounts["resource_inventory"]++ + emitEvent(l.opts.EventSink, "go_pg_resource_inventory_complete", map[string]any{ + "dataset_generation": l.plan.Dataset.Generation, + "build_id": inventory.BuildID, + "resource_type_count": inventory.ResourceTypeCount, + "count_row_count": inventory.CountRowCount, + "document_count": inventory.DocumentCount, + }) + + relationshipStart := time.Now() + relationship, err := l.relations.DiscoverRelationshipObservations(ctx, catalog.RelationshipObservationOptions{ + Project: l.plan.Dataset.Project, + DatasetGeneration: l.plan.Dataset.Generation, + AuthResourcePaths: []string{l.opts.AuthResourcePath}, + }) + if err != nil { + return fmt.Errorf("derive relationship evidence for dataset generation %s/%s: %w", + l.plan.Dataset.Project, l.plan.Dataset.Generation, err) + } + summary.StageSeconds["catalog_relationship_evidence"] = time.Since(relationshipStart).Seconds() + summary.BatchCounts["relationship_observations"] = len(relationship.Values) + emitEvent(l.opts.EventSink, "go_pg_relationship_evidence_complete", map[string]any{ + "observations": len(relationship.Values), + "digest": relationship.Digest, + "diagnostics": len(relationship.Diagnostics), + }) + + semanticStart := time.Now() + if err := l.flushSemanticContributions(ctx, summary); err != nil { + return err + } + summary.StageSeconds["catalog_semantic_evidence"] = time.Since(semanticStart).Seconds() + return nil +} + +func (l *pgGenerationLoad) flushSemanticContributions(ctx context.Context, summary *LoadSummary) error { + if len(l.semanticBuffer) == 0 { + return nil + } + if err := ctx.Err(); err != nil { + return safePointCancel(err) + } + chunk := l.semanticBuffer + if err := l.contribs.WriteSemanticInventoryContributions(ctx, chunk, l.opts.BatchSize); err != nil { + return fmt.Errorf("write semantic inventory contributions: %w", err) + } + summary.BatchCounts["semantic_contributions"] += len(chunk) + summary.BatchCounts["semantic_chunks"]++ + emitEvent(l.opts.EventSink, "go_pg_semantic_write", map[string]any{"contributions": len(chunk)}) + l.semanticBuffer = nil + return nil +} + +func (l *pgGenerationLoad) newRowBuilder(schema *graph.GraphSchema, resourceType string) (RowBuilder, error) { + class := schema.GetClass(resourceType) + if class == nil { + return nil, fmt.Errorf("%s: class %q not found in graph schema", filepath.Base(resourceType), resourceType) + } + var rowBuilder RowBuilder + if l.opts.UseGeneric || !supportsGeneratedLoad(resourceType) { + rowBuilder = NewGenericRowBuilder(l.opts.Project, class, schema, graphExtraArgs(l.opts.Project, l.opts.AuthResourcePath)) + } else { + rowBuilder = NewGeneratedRowBuilder(l.opts.Project, l.opts.AuthResourcePath) + } + qualified, err := newGenerationRowBuilder(rowBuilder, l.opts.Project, l.plan.Dataset.Generation) + if err != nil { + return nil, err + } + return qualified, nil +} + +func pgGenerationBatchID(project, generation, fileID, pass string, window int) string { + return pgBatchIDPrefix + pass + ":" + generationHash( + "pg-ingest-03/batch/v1", project, generation, fileID, pass, strconv.Itoa(window), + ) +} + +type pgSafePointCancelError struct{ err error } + +func (e *pgSafePointCancelError) Error() string { return e.err.Error() } +func (e *pgSafePointCancelError) Unwrap() error { return e.err } + +func safePointCancel(err error) error { + if err == nil { + return nil + } + return &pgSafePointCancelError{err: err} +} + +func (l *pgGenerationLoad) surfaceStoreError(err error, stage string) error { + if err == nil || !pgCommitOutcomeUnknown(err) { + return err + } + return fmt.Errorf("%w: %s/%s %s left a commit outcome unknown: %w", + ErrGenerationManifestOutcomeUnknown, l.ref.Project, l.ref.Generation, stage, err) +} + +func pgCommitOutcomeUnknown(err error) bool { + if err == nil { + return false + } + var safeCancel *pgSafePointCancelError + if errors.As(err, &safeCancel) { + return false + } + if control.CommitOutcomeUnknown(err) { + return true + } + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return true + } + return false +} + +func joinApplicationName(dsn, application string) (string, bool) { + parsed, err := url.Parse(dsn) + if err != nil || (parsed.Scheme != "postgres" && parsed.Scheme != "postgresql") { + return dsn, false + } + query := parsed.Query() + if query.Get("application_name") != "" { + return dsn, true + } + query.Set("application_name", application) + parsed.RawQuery = query.Encode() + return parsed.String(), true +} + +func pgRandomHexToken() string { + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + return "000000000000" + } + return fmt.Sprintf("%x", b[:]) +} diff --git a/internal/ingest/generation_load_postgres_durability_test.go b/internal/ingest/generation_load_postgres_durability_test.go new file mode 100644 index 000000000..e733fec3d --- /dev/null +++ b/internal/ingest/generation_load_postgres_durability_test.go @@ -0,0 +1,1167 @@ +package ingest + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "github.com/calypr/loom/internal/backends/postgres/control" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + publication "github.com/calypr/loom/internal/dataset" +) + +func ing03AssertLedgerEquality(t *testing.T, service *ing03Service) { + t.Helper() + ledger := ing03QIdent(t, service.schema, "batch_write_ledger") + sumResources := ing03Count(t, service.iso, "SELECT COALESCE(sum(resource_count), 0) FROM "+ledger) + sumEdges := ing03Count(t, service.iso, "SELECT COALESCE(sum(edge_count), 0) FROM "+ledger) + totalIDs := ing03Count(t, service.iso, "SELECT count(*) FROM "+ledger) + distinctIDs := ing03Count(t, service.iso, "SELECT count(DISTINCT batch_id) FROM "+ledger) + storedResources := service.countTable("fhir_resource", "") + storedEdges := service.countTable("fhir_edge_occurrence", "") + if sumResources != storedResources { + t.Fatalf("ledger resource accounting %d != stored resource rows %d", sumResources, storedResources) + } + if sumEdges != storedEdges { + t.Fatalf("ledger edge accounting %d != stored edge rows %d", sumEdges, storedEdges) + } + if distinctIDs != totalIDs { + t.Fatalf("ledger batch identities not unique: %d rows, %d distinct ids", totalIDs, distinctIDs) + } +} + +func ing03WaitBothWaiting(t *testing.T, service *ing03Service, appNames []string, timeout time.Duration) []int32 { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + pids := map[int32]bool{} + for _, backend := range service.backendsByApplication(appNames...) { + if backend.WaitEventType != nil && *backend.WaitEventType == "Lock" { + pids[backend.Pid] = true + } + } + if len(pids) >= 2 { + out := make([]int32, 0, len(pids)) + for pid := range pids { + out = append(out, pid) + } + return out + } + time.Sleep(2 * time.Millisecond) + } + return nil +} + +func ing03AssertNeverActive(t *testing.T, service *ing03Service, where string) { + t.Helper() + if _, active := service.activePointerGeneration(ing03Project); active { + t.Fatalf("%s: the active generation pointer moved", where) + } +} + +func TestPostgresGenerationLoadKillInjectionAtEveryDurabilitySeam(t *testing.T) { + reference := newIngest03Service(t) + referenceDir := t.TempDir() + ing03WriteFixtures(t, referenceDir, 2, false) + referenceRef := ing03Ref(t, ing03Project, ing03Generation) + if _, err := LoadPostgresGeneration(context.Background(), + reference.loadOptions(referenceDir, ing03Project, &referenceRef), + reference.pgOptions(ing03AppBase+"-dur-ref")); err != nil { + t.Fatalf("reference load: %s", reference.scrub(err)) + } + referenceFingerprint := reference.fingerprint() + + seams := []struct { + name string + table string + manifestExpected string + rowBarrier func(t *testing.T, service *ing03Service) + rowLevel bool + }{ + {name: "manifest-create", table: "dataset_generation_manifest", manifestExpected: "absent"}, + {name: "resource-batch", table: "fhir_resource", manifestExpected: "LOADING", + rowBarrier: func(t *testing.T, service *ing03Service) { + if count := service.countTable("fhir_resource", ""); count != 0 { + t.Fatalf("resource rows = %d while the first resource batch was still parked and killed, want 0", count) + } + }}, + {name: "edge-batch", table: "fhir_edge_occurrence", manifestExpected: "LOADING"}, + {name: "field-profile", table: "fhir_field_profile", manifestExpected: "LOADING"}, + {name: "inventory", table: "fhir_resource_inventory_count", manifestExpected: "LOADING"}, + {name: "semantic-contributions", table: "fhir_semantic_inventory_contribution", manifestExpected: "LOADING"}, + {name: "staged-transition", table: "dataset_generation_manifest", manifestExpected: "LOADING", rowLevel: true}, + } + + for _, seam := range seams { + t.Run(seam.name, func(t *testing.T) { + service := newIngest03Service(t) + service.migrate() + metaDir := t.TempDir() + ing03WriteFixtures(t, metaDir, 2, false) + ref := ing03Ref(t, ing03Project, ing03Generation) + app := ing03AppBase + "-dur-" + seam.name + + manifestCreated := make(chan struct{}) + var manifestOnce sync.Once + var release func() + if !seam.rowLevel { + release = service.blockTable(seam.table) + } + + loadErrCh := make(chan error, 1) + go func() { + opts := service.loadOptions(metaDir, ing03Project, &ref) + opts.EventSink = func(event string, _ map[string]any) { + if event == "go_pg_manifest_create" { + manifestOnce.Do(func() { close(manifestCreated) }) + } + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + _, err := LoadPostgresGeneration(ctx, opts, service.pgOptions(app)) + loadErrCh <- err + }() + + if seam.rowLevel { + select { + case <-manifestCreated: + case err := <-loadErrCh: + t.Fatalf("load ended (%v) before manifest create: kill injection never armed", err) + case <-time.After(60 * time.Second): + t.Fatal("manifest create never happened: kill injection never armed") + } + release = service.blockManifestRow(ing03Project, ing03Generation) + } + + if _, parked := service.waitForBackendWaiting(app, 60*time.Second); !parked { + t.Fatalf("no backend for %s was ever witnessed waiting on the seam lock: the kill injection is vacuous", app) + } + killed := service.terminateAppBackends(app) + if killed < 1 { + t.Fatalf("pg_terminate_backend terminated %d backends for %s, want >= 1: the kill never landed", killed, app) + } + if release != nil { + release() + } + + var loadErr error + select { + case loadErr = <-loadErrCh: + case <-time.After(90 * time.Second): + t.Fatal("load did not return after the kill") + } + if loadErr == nil { + t.Fatal("killed load reported success") + } + if !errors.Is(loadErr, ErrGenerationManifestOutcomeUnknown) { + t.Fatalf("killed load error = %T %v, want %v (the killed commit's outcome is genuinely unknown)", + loadErr, loadErr, ErrGenerationManifestOutcomeUnknown) + } + + state, found := service.manifestState(ing03Project, ing03Generation) + switch seam.manifestExpected { + case "absent": + if found { + t.Fatalf("manifest exists in state %q after the manifest-create kill, want absent", state) + } + case "LOADING": + if !found || state != string(publication.StateLoading) { + t.Fatalf("manifest state = %q found=%v, want LOADING (unknown commit outcome must not mark FAILED)", state, found) + } + } + ing03AssertNeverActive(t, service, "after kill") + if seam.rowBarrier != nil { + seam.rowBarrier(t, service) + } + ing03AssertLedgerEquality(t, service) + + replayCtx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + if _, err := LoadPostgresGeneration(replayCtx, + service.loadOptions(metaDir, ing03Project, &ref), + service.pgOptions(app+"-replay")); err != nil { + t.Fatalf("replay after kill: %s", service.scrub(err)) + } + if state, found := service.manifestState(ing03Project, ing03Generation); !found || state != string(publication.StateStaged) { + t.Fatalf("replayed manifest = %q found=%v, want STAGED", state, found) + } + ing03AssertNeverActive(t, service, "after replay") + ing03AssertLedgerEquality(t, service) + if got := service.fingerprint(); got != referenceFingerprint { + t.Fatalf("replayed fingerprint = %s, want the clean-load %s", got, referenceFingerprint) + } + }) + } + + control := newIngest03Service(t) + controlDir := t.TempDir() + ing03WriteFixtures(t, controlDir, 2, false) + controlRef := ing03Ref(t, ing03Project, ing03Generation) + if _, err := LoadPostgresGeneration(context.Background(), + control.loadOptions(controlDir, ing03Project, &controlRef), + control.pgOptions(ing03AppBase+"-dur-control")); err != nil { + t.Fatalf("control load: %s", control.scrub(err)) + } + if got := control.fingerprint(); got != referenceFingerprint { + t.Fatalf("control fingerprint = %s, want the reference %s", got, referenceFingerprint) + } +} + +type ing03HelperConfig struct { + Database string `json:"database"` + Schema string `json:"schema"` + MetaDir string `json:"meta_dir"` + SchemaFile string `json:"schema_file"` + Project string `json:"project"` + Generation string `json:"generation"` + AuthPath string `json:"auth_path"` + Application string `json:"application"` + ManifestSchema string `json:"manifest_schema"` +} + +const ing03HelperRunFilter = "^TestPostgresGenerationLoadReplayHelper$" + +func TestPostgresGenerationLoadReplayHelper(t *testing.T) { + rawConfig := os.Getenv("LOOM_ING03_HELPER_CONFIG") + if rawConfig == "" { + t.Skip("child-process helper: only runs when invoked with LOOM_ING03_HELPER_CONFIG") + } + var config ing03HelperConfig + if err := json.Unmarshal([]byte(rawConfig), &config); err != nil { + fmt.Printf("EVENT helper-config-error %v\n", err) + os.Exit(4) + } + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + fmt.Println("EVENT helper-missing-url") + os.Exit(4) + } + dsn, err := ing03HelperDSN(raw, config.Database) + if err != nil { + fmt.Printf("EVENT helper-dsn-error %v\n", err) + os.Exit(4) + } + ref, err := publication.NewRef(config.Project, config.Generation) + if err != nil { + fmt.Printf("EVENT helper-ref-error %v\n", err) + os.Exit(4) + } + opts := LoadOptions{ + Schema: config.SchemaFile, + MetaDir: config.MetaDir, + Project: config.Project, + Dataset: &ref, + AuthResourcePath: config.AuthPath, + } + opts.EventSink = func(event string, _ map[string]any) { + fmt.Printf("EVENT %s\n", event) + } + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + catalogCfg, cfgErr := pgxpool.ParseConfig(dsn) + if cfgErr != nil { + fmt.Printf("EVENT helper-pool-config %v\n", cfgErr) + os.Exit(4) + } + catalogCfg.MaxConns = 4 + catalogCfg.ConnConfig.RuntimeParams["application_name"] = config.Application + catalogPool, poolErr := pgxpool.NewWithConfig(ctx, catalogCfg) + if poolErr != nil { + fmt.Printf("EVENT helper-pool-error %v\n", poolErr) + os.Exit(4) + } + defer catalogPool.Close() + catalog := ing03WireCatalogWriters(catalogPool, config.Schema, nil, func(err error) { + fmt.Printf("EVENT helper-catalog-wiring %v\n", err) + os.Exit(4) + }) + if _, err := LoadPostgresGeneration(ctx, opts, PGSourceOptions{ + DSN: dsn, Schema: config.Schema, ManifestSchema: config.ManifestSchema, + ApplicationName: config.Application, MaxConns: 4, Catalog: catalog, + }); err != nil { + switch { + case errors.Is(err, ErrGenerationAlreadyStaged): + fmt.Println("EVENT outcome already-staged") + os.Exit(2) + case errors.Is(err, ErrGenerationManifestIdentityConflict): + fmt.Println("EVENT outcome identity-conflict") + os.Exit(3) + case errors.Is(err, ErrGenerationLoadAlreadyFailed): + fmt.Println("EVENT outcome already-failed") + os.Exit(6) + } + fmt.Printf("EVENT outcome error %v\n", err) + os.Exit(4) + } + fmt.Println("EVENT outcome staged") + os.Exit(0) +} + +func ing03HelperDSN(raw, database string) (string, error) { + u, err := parsePostgresURL(raw) + if err != nil { + return "", err + } + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + database + return u.String(), nil +} + +func parsePostgresURL(raw string) (*url.URL, error) { + u, err := url.Parse(raw) + if err != nil { + return nil, fmt.Errorf("parse postgres URL: %w", err) + } + if u.Scheme != "postgres" && u.Scheme != "postgresql" { + return nil, fmt.Errorf("postgres URL scheme %q is not postgres/postgresql", u.Scheme) + } + return u, nil +} + +func ing03DatabaseFromDSN(t *testing.T, dsn string) string { + t.Helper() + u, err := parsePostgresURL(dsn) + if err != nil { + t.Fatalf("parse fixture DSN: %v", err) + } + database := strings.TrimPrefix(u.Path, "/") + if !ing03LoadNamePattern.MatchString(database) { + t.Fatalf("fixture database %q does not match the owned name pattern", database) + } + return database +} + +func runIngest03Helper(t *testing.T, service *ing03Service, config ing03HelperConfig) (int, []string, string) { + t.Helper() + encoded, err := json.Marshal(config) + if err != nil { + t.Fatalf("marshal helper config: %v", err) + } + cmd := exec.Command(os.Args[0], "-test.run="+ing03HelperRunFilter, "-test.timeout=180s", "-test.v") + cmd.Env = append(os.Environ(), "LOOM_ING03_HELPER_CONFIG="+string(encoded)) + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatalf("helper stdout: %v", err) + } + var stderr strings.Builder + cmd.Stderr = &stderr + cmd.WaitDelay = 30 * time.Second + if err := cmd.Start(); err != nil { + t.Fatalf("start helper: %v", err) + } + if cmd.Process.Pid == os.Getpid() { + t.Fatalf("helper reported the parent pid %d: it is not a separate process", cmd.Process.Pid) + } + events := []string{} + scanner := bufio.NewScanner(stdout) + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "EVENT ") { + events = append(events, strings.TrimPrefix(line, "EVENT ")) + } + } + waitErr := cmd.Wait() + code := 0 + if waitErr != nil { + var exitErr *exec.ExitError + if errors.As(waitErr, &exitErr) { + code = exitErr.ExitCode() + } else { + t.Fatalf("wait helper: %v (output=%s)", waitErr, service.scrub(stderr.String())) + } + } + return code, events, service.scrub(stderr.String()) +} + +func TestPostgresGenerationLoadSeparateProcessCrashReplayConvergence(t *testing.T) { + shared := newIngest03Service(t) + shared.migrate() + dbName := ing03DatabaseFromDSN(t, shared.dsn) + + metaDir := t.TempDir() + ing03WriteFixtures(t, metaDir, 2, false) + schemaFile := repoPath(t, "schemas", "graph-fhir.json") + + crashGeneration := "gen-crash-03" + reference := newIngest03Service(t) + referenceDir := t.TempDir() + ing03WriteFixtures(t, referenceDir, 2, false) + referenceRef := ing03Ref(t, ing03Project, crashGeneration) + if _, err := LoadPostgresGeneration(context.Background(), + reference.loadOptions(referenceDir, ing03Project, &referenceRef), + reference.pgOptions(ing03AppBase+"-crash-ref")); err != nil { + t.Fatalf("reference load: %s", reference.scrub(err)) + } + referenceFingerprint := reference.fingerprint() + + baseConfig := ing03HelperConfig{ + Database: dbName, Schema: shared.schema, ManifestSchema: shared.manifestSchema, + MetaDir: metaDir, SchemaFile: schemaFile, + Project: ing03Project, Generation: crashGeneration, AuthPath: ing03AuthPath, + } + + crashBlockRelease := shared.blockTable("fhir_edge_occurrence") + crashConfig := baseConfig + crashConfig.Application = ing03AppBase + "-crash-victim" + cmd := exec.Command(os.Args[0], "-test.run="+ing03HelperRunFilter, "-test.timeout=180s", "-test.v") + encoded, err := json.Marshal(crashConfig) + if err != nil { + crashBlockRelease() + t.Fatalf("marshal crash config: %v", err) + } + cmd.Env = append(os.Environ(), "LOOM_ING03_HELPER_CONFIG="+string(encoded)) + stdout, err := cmd.StdoutPipe() + if err != nil { + crashBlockRelease() + t.Fatalf("crash stdout: %v", err) + } + var crashStderr strings.Builder + cmd.Stderr = &crashStderr + cmd.WaitDelay = 30 * time.Second + if err := cmd.Start(); err != nil { + crashBlockRelease() + t.Fatalf("start crash helper: %v", err) + } + if cmd.Process.Pid == os.Getpid() || cmd.Process.Pid <= 0 { + crashBlockRelease() + t.Fatalf("crash helper pid %d is not a separate live process", cmd.Process.Pid) + } + batchCommits := 0 + sawManifestCreate := false + killed := false + scanner := bufio.NewScanner(stdout) + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for scanner.Scan() { + line := scanner.Text() + if !strings.HasPrefix(line, "EVENT ") { + continue + } + event := strings.TrimPrefix(line, "EVENT ") + switch event { + case "go_pg_manifest_create": + sawManifestCreate = true + case "go_pg_resource_batch_commit": + batchCommits++ + if batchCommits == 2 && !killed { + killed = true + if err := cmd.Process.Kill(); err != nil { + crashBlockRelease() + t.Fatalf("kill crash helper: %v", err) + } + } + } + } + if !sawManifestCreate || !killed { + crashBlockRelease() + t.Fatalf("crash timing was vacuous: manifestCreate=%v batchCommits=%d", sawManifestCreate, batchCommits) + } + if err := cmd.Wait(); err != nil { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() >= 0 { + crashBlockRelease() + t.Fatalf("crash helper exit = %v, want SIGKILL (-1); stderr=%s", + err, shared.scrub(crashStderr.String())) + } + } else { + crashBlockRelease() + t.Fatal("crash helper reported clean exit after SIGKILL: it never crashed mid-load") + } + crashBlockRelease() + + state, found := shared.manifestState(ing03Project, crashGeneration) + if !found || state != string(publication.StateLoading) { + t.Fatalf("post-crash manifest state = %q found=%v, want LOADING", state, found) + } + ing03AssertNeverActive(t, shared, "after process crash") + ing03AssertLedgerEquality(t, shared) + if resources := shared.countTable("fhir_resource", ""); resources != 5 { + t.Fatalf("post-crash resource rows = %d, want the 5 witnessed committed", resources) + } + if edges := shared.countTable("fhir_edge_occurrence", ""); edges != 0 { + t.Fatalf("post-crash edge rows = %d, want 0 (the parked batch never committed)", edges) + } + + replayConfig := baseConfig + replayConfig.Application = ing03AppBase + "-crash-replay" + code, events, stderr := runIngest03Helper(t, shared, replayConfig) + if code != 0 { + t.Fatalf("replay process exit = %d (events=%v stderr=%s), want 0 (converged)", code, events, stderr) + } + if state, found := shared.manifestState(ing03Project, crashGeneration); !found || state != string(publication.StateStaged) { + t.Fatalf("replayed manifest = %q found=%v, want STAGED", state, found) + } + ing03AssertNeverActive(t, shared, "after replay") + ing03AssertLedgerEquality(t, shared) + if got := shared.fingerprint(); got != referenceFingerprint { + t.Fatalf("cross-process replay fingerprint = %s, want the clean-load %s (byte-stable content)", got, referenceFingerprint) + } + + ledgerBefore := shared.countTable("batch_write_ledger", "") + repeatConfig := baseConfig + repeatConfig.Application = ing03AppBase + "-crash-repeat" + code, events, stderr = runIngest03Helper(t, shared, repeatConfig) + if code != 2 { + t.Fatalf("repeat process exit = %d (events=%v stderr=%s), want 2 (ErrGenerationAlreadyStaged)", code, events, stderr) + } + if ledgerAfter := shared.countTable("batch_write_ledger", ""); ledgerAfter != ledgerBefore { + t.Fatalf("already-staged repeat wrote %d new ledger rows, want 0", ledgerAfter-ledgerBefore) + } + if got := shared.fingerprint(); got != referenceFingerprint { + t.Fatalf("already-staged repeat changed the fingerprint to %s, want %s unchanged", got, referenceFingerprint) + } + + tamperedDir := t.TempDir() + original, err := os.ReadFile(schemaFile) + if err != nil { + t.Fatalf("read schema: %v", err) + } + tamperedFile := filepath.Join(tamperedDir, "graph-fhir-tampered.json") + if err := os.WriteFile(tamperedFile, append(append([]byte{}, original...), '\n'), 0o600); err != nil { + t.Fatalf("write tampered schema: %v", err) + } + identityConfig := baseConfig + identityConfig.Application = ing03AppBase + "-crash-identity" + identityConfig.SchemaFile = tamperedFile + code, events, stderr = runIngest03Helper(t, shared, identityConfig) + if code != 3 { + t.Fatalf("identity-conflict process exit = %d (events=%v stderr=%s), want 3 (ErrGenerationManifestIdentityConflict)", + code, events, stderr) + } + if got := shared.fingerprint(); got != referenceFingerprint { + t.Fatalf("refused identity conflict changed content to %s, want %s unchanged", got, referenceFingerprint) + } + + freshConfig := baseConfig + freshConfig.Application = ing03AppBase + "-crash-tampered-fresh" + freshConfig.Generation = "gen-crash-tampered" + code, events, stderr = runIngest03Helper(t, shared, freshConfig) + if code != 0 { + t.Fatalf("tampered-schema control exit = %d (events=%v stderr=%s), want 0: the control could not have failed above", + code, events, stderr) + } +} + +func TestPostgresGenerationLoadSameGenerationLoadRace(t *testing.T) { + var referenceFingerprint string + for round := 1; round <= ing03RaceRounds; round++ { + service := newIngest03Service(t) + metaDir := t.TempDir() + ing03WriteFixtures(t, metaDir, 2, false) + generation := fmt.Sprintf("gen-race-%02d", round) + ref := ing03Ref(t, ing03Project, generation) + apps := []string{ + ing03AppBase + fmt.Sprintf("-race-a-%02d", round), + ing03AppBase + fmt.Sprintf("-race-b-%02d", round), + } + + sampling := make(chan int, 1) + done := make(chan struct{}) + go func() { sampling <- service.maxConcurrentDistinctApps(done, apps[0], apps[1]) }() + + start := make(chan struct{}) + var wg sync.WaitGroup + results := make([]error, 2) + for i := 0; i < 2; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + _, err := LoadPostgresGeneration(ctx, + service.loadOptions(metaDir, ing03Project, &ref), + service.pgOptions(apps[i])) + results[i] = err + }(i) + } + close(start) + wg.Wait() + close(done) + overlap := <-sampling + + if overlap < 2 { + t.Fatalf("round %d overlap witness = %d distinct concurrent apps, want >= 2: the race serialized", round, overlap) + } + + for i, err := range results { + if err == nil || errors.Is(err, ErrGenerationAlreadyStaged) || + errors.Is(err, ErrGenerationLoadAlreadyFailed) || + errors.Is(err, ErrGenerationManifestOutcomeUnknown) { + continue + } + t.Fatalf("round %d racer %d error = %T %v, want success, already-staged, already-failed, or outcome-unknown", + round, i, err, err) + } + complete := 0 + for _, err := range results { + if err == nil || errors.Is(err, ErrGenerationAlreadyStaged) { + complete++ + } + } + if complete == 0 { + t.Fatalf("round %d: neither racer reported a completed or read-back-staged load: %v / %v", round, results[0], results[1]) + } + + if state, found := service.manifestState(ing03Project, generation); !found || state != string(publication.StateStaged) { + t.Fatalf("round %d manifest = %q found=%v, want STAGED", round, state, found) + } + ing03AssertNeverActive(t, service, fmt.Sprintf("race round %d", round)) + if count := service.countTable("fhir_resource", ""); count != 5 { + t.Fatalf("round %d resource rows = %d, want exactly 5 (the race duplicated or lost nothing)", round, count) + } + ing03AssertLedgerEquality(t, service) + + if round == 1 { + reference := newIngest03Service(t) + referenceDir := t.TempDir() + ing03WriteFixtures(t, referenceDir, 2, false) + referenceRef := ing03Ref(t, ing03Project, generation) + if _, err := LoadPostgresGeneration(context.Background(), + reference.loadOptions(referenceDir, ing03Project, &referenceRef), + reference.pgOptions(ing03AppBase+"-race-ref")); err != nil { + t.Fatalf("race reference load: %s", reference.scrub(err)) + } + referenceFingerprint = reference.fingerprint() + if got := service.fingerprint(); got != referenceFingerprint { + t.Fatalf("round %d raced fingerprint = %s, want the single-loader %s", round, got, referenceFingerprint) + } + } else if referenceFingerprint == "" { + t.Fatal("reference fingerprint was never captured") + } + } + + service := newIngest03Service(t) + metaDir := t.TempDir() + ing03WriteFixtures(t, metaDir, 2, false) + ref := ing03Ref(t, ing03Project, "gen-race-ctl") + apps := []string{ing03AppBase + "-race-ctl-a", ing03AppBase + "-race-ctl-b"} + sampling := make(chan int, 1) + done := make(chan struct{}) + go func() { sampling <- service.maxConcurrentDistinctApps(done, apps[0], apps[1]) }() + var mutex sync.Mutex + var wg sync.WaitGroup + results := make([]error, 2) + for i := 0; i < 2; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + mutex.Lock() + defer mutex.Unlock() + _, err := LoadPostgresGeneration(context.Background(), + service.loadOptions(metaDir, ing03Project, &ref), service.pgOptions(apps[i])) + results[i] = err + }(i) + } + wg.Wait() + close(done) + if overlap := <-sampling; overlap >= 2 { + t.Fatalf("sequential control witnessed %d concurrent apps: the overlap sampler is broken", overlap) + } + for i, err := range results { + if err != nil && !errors.Is(err, ErrGenerationAlreadyStaged) { + t.Fatalf("control racer %d error = %T %v", i, err, err) + } + } + if state, found := service.manifestState(ing03Project, "gen-race-ctl"); !found || state != string(publication.StateStaged) { + t.Fatalf("control manifest = %q found=%v, want STAGED", state, found) + } +} + +func TestPostgresGenerationLoadActivationCASRace(t *testing.T) { + for round := 1; round <= ing03RaceRounds; round++ { + service := newIngest03Service(t) + metaDir := t.TempDir() + ing03WriteFixtures(t, metaDir, 2, false) + generation := fmt.Sprintf("gen-act-race-%02d", round) + ref := ing03Ref(t, ing03Project, generation) + if _, err := LoadPostgresGeneration(context.Background(), + service.loadOptions(metaDir, ing03Project, &ref), + service.pgOptions(ing03AppBase+fmt.Sprintf("-act-race-load-%02d", round))); err != nil { + t.Fatalf("round %d load: %s", round, service.scrub(err)) + } + + apps := []string{ + ing03AppBase + fmt.Sprintf("-act-race-a-%02d", round), + ing03AppBase + fmt.Sprintf("-act-race-b-%02d", round), + } + release := service.blockTable("dataset_active_generation") + start := make(chan struct{}) + var wg sync.WaitGroup + results := make([]error, 2) + for i := 0; i < 2; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + results[i] = ActivatePostgresGeneration(ctx, ref, service.pgOptions(apps[i])) + }(i) + } + close(start) + parkedPIDs := ing03WaitBothWaiting(t, service, apps, 60*time.Second) + if len(parkedPIDs) < 2 { + release() + t.Fatalf("activation round %d: only %d activations were ever in flight on the CAS lock (%v): the race serialized", round, len(parkedPIDs), parkedPIDs) + } + release() + wg.Wait() + nils, conflicts := 0, 0 + for i, err := range results { + switch { + case err == nil: + nils++ + case errors.Is(err, control.ErrActivationConflict): + conflicts++ + default: + t.Fatalf("activation round %d racer %d error = %T %v, want success or ErrActivationConflict", + i, round, err, err) + } + } + if nils == 0 { + t.Fatalf("activation round %d: neither racer succeeded: %v / %v", round, results[0], results[1]) + } + if pointer, active := service.activePointerGeneration(ing03Project); !active || pointer != generation { + t.Fatalf("activation round %d pointer = %q active=%v, want %q", round, pointer, active, generation) + } + } + + service := newIngest03Service(t) + metaDir := t.TempDir() + ing03WriteFixtures(t, metaDir, 2, false) + ref := ing03Ref(t, ing03Project, "gen-act-ctl") + if _, err := LoadPostgresGeneration(context.Background(), + service.loadOptions(metaDir, ing03Project, &ref), + service.pgOptions(ing03AppBase+"-act-ctl-load")); err != nil { + t.Fatalf("control load: %s", service.scrub(err)) + } + if err := ActivatePostgresGeneration(context.Background(), ref, + service.pgOptions(ing03AppBase+"-act-ctl")); err != nil { + t.Fatalf("control activation: %s", service.scrub(err)) + } + if pointer, active := service.activePointerGeneration(ing03Project); !active || pointer != "gen-act-ctl" { + t.Fatalf("control pointer = %q active=%v", pointer, active) + } +} + +func TestPostgresGenerationLoadCancellationBranches(t *testing.T) { + t.Run("safe-point cancellation marks FAILED", func(t *testing.T) { + service := newIngest03Service(t) + metaDir := t.TempDir() + ing03WriteFixtures(t, metaDir, 2, false) + ref := ing03Ref(t, ing03Project, ing03Generation) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var events []string + opts := service.loadOptions(metaDir, ing03Project, &ref) + opts.EventSink = func(event string, _ map[string]any) { + events = append(events, event) + if event == "go_pg_manifest_create" { + cancel() + } + } + _, err := LoadPostgresGeneration(ctx, opts, service.pgOptions(ing03AppBase+"-cancel-safe")) + if !errors.Is(err, context.Canceled) { + t.Fatalf("safe-point cancellation = %T %v, want context.Canceled", err, err) + } + if !containsPrefix(events, "go_pg_manifest_create") { + t.Fatalf("the cancellation was never armed at the safe point: %v", events) + } + if containsPrefix(events, "go_pg_resource_batch_commit") { + t.Fatalf("safe-point cancellation still committed a resource batch: %v", events) + } + if state, found := service.manifestState(ing03Project, ing03Generation); !found || state != string(publication.StateFailed) { + t.Fatalf("safe-point manifest = %q found=%v, want FAILED (the creator owns a known-outcome cancellation)", state, found) + } + ing03AssertNeverActive(t, service, "safe-point cancellation") + if ledger := service.countTable("batch_write_ledger", ""); ledger != 0 { + t.Fatalf("safe-point cancellation wrote %d ledger rows, want 0", ledger) + } + if rows := service.countTable("fhir_resource", ""); rows != 0 { + t.Fatalf("safe-point cancellation wrote %d resource rows, want 0", rows) + } + if activateErr := ActivatePostgresGeneration(context.Background(), ref, + service.pgOptions(ing03AppBase+"-cancel-safe-act")); !errors.Is(activateErr, publication.ErrGenerationNotReady) { + t.Fatalf("activation after safe-point cancellation = %v, want ErrGenerationNotReady", activateErr) + } + if _, retryErr := LoadPostgresGeneration(context.Background(), + service.loadOptions(metaDir, ing03Project, &ref), + service.pgOptions(ing03AppBase+"-cancel-safe-retry")); !errors.Is(retryErr, ErrGenerationLoadAlreadyFailed) { + t.Fatalf("retry after safe-point cancellation = %v, want ErrGenerationLoadAlreadyFailed", retryErr) + } + }) + + t.Run("in-flight cancellation keeps LOADING and replay converges", func(t *testing.T) { + reference := newIngest03Service(t) + referenceDir := t.TempDir() + ing03WriteFixtures(t, referenceDir, 2, false) + referenceRef := ing03Ref(t, ing03Project, ing03Generation) + if _, err := LoadPostgresGeneration(context.Background(), + reference.loadOptions(referenceDir, ing03Project, &referenceRef), + reference.pgOptions(ing03AppBase+"-cancel-ref")); err != nil { + t.Fatalf("reference load: %s", reference.scrub(err)) + } + referenceFingerprint := reference.fingerprint() + + service := newIngest03Service(t) + service.migrate() + metaDir := t.TempDir() + ing03WriteFixtures(t, metaDir, 2, false) + ref := ing03Ref(t, ing03Project, ing03Generation) + release := service.blockTable("fhir_edge_occurrence") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var events []string + var eventsMu sync.Mutex + opts := service.loadOptions(metaDir, ing03Project, &ref) + opts.EventSink = func(event string, _ map[string]any) { + eventsMu.Lock() + events = append(events, event) + eventsMu.Unlock() + } + app := ing03AppBase + "-cancel-inflight" + loadErrCh := make(chan error, 1) + go func() { + _, err := LoadPostgresGeneration(ctx, opts, service.pgOptions(app)) + loadErrCh <- err + }() + + if _, parked := service.waitForBackendWaiting(app, 60*time.Second); !parked { + release() + t.Fatal("the edge statement was never witnessed parked: the cancellation would not race an in-flight statement") + } + eventsMu.Lock() + preCancel := append([]string{}, events...) + eventsMu.Unlock() + if !containsPrefix(preCancel, "go_pg_resource_batch_commit") { + release() + t.Fatalf("cancellation armed before any resource commit, not mid-statement: %v", preCancel) + } + cancel() + var loadErr error + select { + case loadErr = <-loadErrCh: + case <-time.After(90 * time.Second): + release() + t.Fatal("in-flight cancellation never returned") + } + release() + if !errors.Is(loadErr, context.Canceled) { + t.Fatalf("in-flight cancellation = %T %v, want context.Canceled", loadErr, loadErr) + } + if state, found := service.manifestState(ing03Project, ing03Generation); !found || state != string(publication.StateLoading) { + t.Fatalf("in-flight-cancelled manifest = %q found=%v, want LOADING (the cancelled statement's outcome is unknown)", state, found) + } + ing03AssertNeverActive(t, service, "in-flight cancellation") + ing03AssertLedgerEquality(t, service) + + if _, err := LoadPostgresGeneration(context.Background(), + service.loadOptions(metaDir, ing03Project, &ref), + service.pgOptions(app+"-replay")); err != nil { + t.Fatalf("replay after in-flight cancellation: %s", service.scrub(err)) + } + if state, found := service.manifestState(ing03Project, ing03Generation); !found || state != string(publication.StateStaged) { + t.Fatalf("replayed manifest = %q found=%v, want STAGED", state, found) + } + ing03AssertLedgerEquality(t, service) + if got := service.fingerprint(); got != referenceFingerprint { + t.Fatalf("replayed fingerprint = %s, want the clean-load %s", got, referenceFingerprint) + } + + control := newIngest03Service(t) + controlDir := t.TempDir() + ing03WriteFixtures(t, controlDir, 2, false) + controlRef := ing03Ref(t, ing03Project, ing03Generation) + if _, err := LoadPostgresGeneration(context.Background(), + control.loadOptions(controlDir, ing03Project, &controlRef), + control.pgOptions(ing03AppBase+"-cancel-control")); err != nil { + t.Fatalf("control load: %s", control.scrub(err)) + } + if state, found := control.manifestState(ing03Project, ing03Generation); !found || state != string(publication.StateStaged) { + t.Fatalf("control manifest = %q found=%v, want STAGED", state, found) + } + }) +} + +func ing03WriteSpillFixtures(t *testing.T, dir string, codedRows int) { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir spill fixtures: %v", err) + } + rows := make([]string, 0, codedRows) + for i := 0; i < codedRows; i++ { + document := map[string]any{ + "resourceType": "Observation", + "id": fmt.Sprintf("obs-s-%06d", i), + "status": "final", + "subject": map[string]any{"reference": "Patient/pat-0000"}, + "code": map[string]any{"coding": []any{map[string]any{ + "system": ing03LoincSystem, + "code": fmt.Sprintf("spill-%06d", i), + }}}, + } + if i%4 != 3 { + document["focus"] = []any{ + map[string]any{"reference": "Observation/obs-s-000000"}, + } + } + rows = append(rows, ing03JSONLine(t, document)) + } + if err := os.WriteFile(filepath.Join(dir, "Observation.ndjson"), + []byte(strings.Join(rows, "\n")+"\n"), 0o600); err != nil { + t.Fatalf("write spill Observation fixture: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "Patient.ndjson"), + []byte(ing03PatientLine(t, "pat-0000")+"\n"), 0o600); err != nil { + t.Fatalf("write spill Patient fixture: %v", err) + } +} + +func TestPostgresGenerationLoadSemanticSpillMidLoadKillReplayConvergence(t *testing.T) { + const spillRows = 2100 + const flushThreshold = 250 + + service := newIngest03Service(t) + service.migrate() + metaDir := t.TempDir() + ing03WriteSpillFixtures(t, metaDir, spillRows) + refRef := ing03Ref(t, ing03Project, "gen-build-spill-ref") + refKill := ing03Ref(t, ing03Project, "gen-build-spill-kill") + + refOpts := service.loadOptions(metaDir, ing03Project, &refRef) + refSpillWrites := 0 + var refEdgeOffers []int64 + refOpts.EventSink = func(event string, fields map[string]any) { + if event == "go_pg_semantic_write" { + refSpillWrites++ + } + if event == "go_pg_edge_batch_commit" { + switch offered := fields["offered"].(type) { + case int: + refEdgeOffers = append(refEdgeOffers, int64(offered)) + case int64: + refEdgeOffers = append(refEdgeOffers, offered) + case float64: + refEdgeOffers = append(refEdgeOffers, int64(offered)) + } + } + } + pgRef := service.pgOptions(ing03AppBase + "-spill-ref") + pgRef.SemanticFlushThreshold = flushThreshold + refCtx, refCancel := context.WithTimeout(context.Background(), 120*time.Second) + defer refCancel() + if _, err := LoadPostgresGeneration(refCtx, refOpts, pgRef); err != nil { + t.Fatalf("reference spill load: %s", service.scrub(err)) + } + if refSpillWrites < 2 { + t.Fatalf("reference load emitted %d semantic writes at threshold %d with %d coded rows: the mid-load spill never ran (vacuous)", + refSpillWrites, flushThreshold, spillRows) + } + if len(refEdgeOffers) == 0 { + t.Fatal("reference load emitted no edge batch commits: the boundary witness is vacuous") + } + crossed := false + for _, offered := range refEdgeOffers { + if offered > control.MaxBatchEdges { + t.Fatalf("edge batch offered %d, exceeding the store cap %d (batch events: %v)", offered, control.MaxBatchEdges, refEdgeOffers) + } + if offered >= control.MaxBatchEdges-6 && offered < control.MaxBatchEdges { + crossed = true + } + } + if !crossed { + t.Fatalf("no edge batch committed in the crossing band [%d,%d): the lookahead split never fired (offers: %v)", + control.MaxBatchEdges-6, control.MaxBatchEdges, refEdgeOffers) + } + refContribs := service.countTable("fhir_semantic_inventory_contribution", "dataset_generation = $1", refRef.Generation) + if refContribs < 2*flushThreshold { + t.Fatalf("reference persisted %d contributions, want >= %d: fixture volume never crossed the spill threshold", + refContribs, 2*flushThreshold) + } + + spillFirst := make(chan struct{}, 1) + killOpts := service.loadOptions(metaDir, ing03Project, &refKill) + killOpts.EventSink = func(event string, _ map[string]any) { + if event == "go_pg_semantic_write" { + select { + case spillFirst <- struct{}{}: + default: + } + } + } + app := ing03AppBase + "-spill-kill" + pgKill := service.pgOptions(app) + pgKill.SemanticFlushThreshold = flushThreshold + loadErrCh := make(chan error, 1) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + _, err := LoadPostgresGeneration(ctx, killOpts, pgKill) + loadErrCh <- err + }() + select { + case <-spillFirst: + case loadErr := <-loadErrCh: + t.Fatalf("killed load ended (%v) before the first spill commit: the spill kill never armed", loadErr) + case <-time.After(90 * time.Second): + t.Fatal("no spill write happened: the spill kill never armed") + } + release := service.blockTable("fhir_semantic_inventory_contribution") + defer release() + if _, parked := service.waitForBackendWaiting(app, 60*time.Second); !parked { + t.Fatalf("no backend for %s parked on the contribution lock mid-spill: the kill injection is vacuous", app) + } + if killed := service.terminateAppBackends(app); killed < 1 { + t.Fatalf("pg_terminate_backend terminated %d backends for %s, want >= 1: the spill kill never landed", killed, app) + } + release() + var loadErr error + select { + case loadErr = <-loadErrCh: + case <-time.After(120 * time.Second): + t.Fatal("killed spill load did not return") + } + if loadErr == nil { + t.Fatal("killed spill load reported success") + } + if !errors.Is(loadErr, ErrGenerationManifestOutcomeUnknown) { + t.Fatalf("killed spill load error = %T %v, want %v (the killed spill's outcome is genuinely unknown)", + loadErr, loadErr, ErrGenerationManifestOutcomeUnknown) + } + if state, found := service.manifestState(ing03Project, refKill.Generation); !found || state != string(publication.StateLoading) { + t.Fatalf("manifest state = %q found=%v, want LOADING (unknown spill outcome must not mark FAILED)", state, found) + } + ing03AssertNeverActive(t, service, "after spill kill") + ing03AssertLedgerEquality(t, service) + partial := service.countTable("fhir_semantic_inventory_contribution", "dataset_generation = $1", refKill.Generation) + if partial < flushThreshold { + t.Fatalf("partial contributions = %d, want >= %d (the first spill committed)", partial, flushThreshold) + } + if partial >= refContribs { + t.Fatalf("partial contributions = %d, want < reference %d: the killed load persisted as if complete", partial, refContribs) + } + + resumeOpts := service.loadOptions(metaDir, ing03Project, &refKill) + resumeSpills := 0 + resumeOpts.EventSink = func(event string, _ map[string]any) { + if event == "go_pg_semantic_write" { + resumeSpills++ + } + } + pgResume := service.pgOptions(ing03AppBase + "-spill-resume") + pgResume.SemanticFlushThreshold = flushThreshold + resumeCtx, resumeCancel := context.WithTimeout(context.Background(), 120*time.Second) + defer resumeCancel() + if _, err := LoadPostgresGeneration(resumeCtx, resumeOpts, pgResume); err != nil { + t.Fatalf("spill replay load: %s", service.scrub(err)) + } + if resumeSpills < 2 { + t.Fatalf("replay emitted %d spill writes, want >= 2: replay skipped the mid-load branch", resumeSpills) + } + final := service.countTable("fhir_semantic_inventory_contribution", "dataset_generation = $1", refKill.Generation) + if final != refContribs { + t.Fatalf("replayed contributions = %d, want the reference %d (convergence)", final, refContribs) + } + if state, found := service.manifestState(ing03Project, refKill.Generation); !found || state != string(publication.StateStaged) { + t.Fatalf("replayed manifest = %q found=%v, want STAGED", state, found) + } + ing03AssertLedgerEquality(t, service) +} + +func TestPostgresGenerationLoadReplaySummaryCountsOnlyNewRows(t *testing.T) { + service := newIngest03Service(t) + service.migrate() + metaDir := t.TempDir() + const killRows = 3000 + ing03WriteSpillFixtures(t, metaDir, killRows) + ref := ing03Ref(t, ing03Project, ing03Generation) + + firstEdgeCommit := make(chan struct{}, 1) + opts := service.loadOptions(metaDir, ing03Project, &ref) + opts.EventSink = func(event string, _ map[string]any) { + if event == "go_pg_edge_batch_commit" { + select { + case firstEdgeCommit <- struct{}{}: + default: + } + } + } + app := ing03AppBase + "-replay-counter" + pg := service.pgOptions(app) + loadErrCh := make(chan error, 1) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + _, err := LoadPostgresGeneration(ctx, opts, pg) + loadErrCh <- err + }() + select { + case <-firstEdgeCommit: + case loadErr := <-loadErrCh: + t.Fatalf("load ended (%v) before the first edge commit: the kill never armed", loadErr) + case <-time.After(90 * time.Second): + t.Fatal("no edge batch committed: the kill never armed") + } + release := service.blockTable("fhir_edge_occurrence") + defer release() + if _, parked := service.waitForBackendWaiting(app, 60*time.Second); !parked { + t.Fatalf("no backend for %s parked on the edge lock: the kill injection is vacuous", app) + } + if killed := service.terminateAppBackends(app); killed < 1 { + t.Fatalf("terminated %d backends, want >= 1", killed) + } + release() + var loadErr error + select { + case loadErr = <-loadErrCh: + case <-time.After(120 * time.Second): + t.Fatal("killed load did not return") + } + if !errors.Is(loadErr, ErrGenerationManifestOutcomeUnknown) { + t.Fatalf("killed load error = %T %v, want %v", loadErr, loadErr, ErrGenerationManifestOutcomeUnknown) + } + if state, found := service.manifestState(ing03Project, ing03Generation); !found || state != string(publication.StateLoading) { + t.Fatalf("manifest = %q found=%v, want LOADING", state, found) + } + edgesBefore := service.countTable("fhir_edge_occurrence", "") + if edgesBefore == 0 { + t.Fatal("no edge rows before the replay: the replay witness would be vacuous") + } + + resumeCtx, resumeCancel := context.WithTimeout(context.Background(), 120*time.Second) + defer resumeCancel() + summary, err := LoadPostgresGeneration(resumeCtx, service.loadOptions(metaDir, ing03Project, &ref), + service.pgOptions(ing03AppBase+"-replay-counter-resume")) + if err != nil { + t.Fatalf("resume after edge-window kill: %s", service.scrub(err)) + } + if summary.BatchCounts["edge_batch_replay"] == 0 { + t.Fatalf("resume replayed no edge windows (batch counts %v): the replay witness never exercised the replayed path", summary.BatchCounts) + } + edgesAfter := service.countTable("fhir_edge_occurrence", "") + if int64(summary.EdgesInserted) != edgesAfter-edgesBefore { + t.Fatalf("resumed EdgesInserted = %d, want only the %d new rows (before=%d after=%d): replayed batches inflated the counter", + summary.EdgesInserted, edgesAfter-edgesBefore, edgesBefore, edgesAfter) + } + if summary.VerticesInserted != 0 { + t.Fatalf("resumed VerticesInserted = %d, want 0 (all resource windows were already durable): replayed batches inflated the counter", summary.VerticesInserted) + } + ing03AssertLedgerEquality(t, service) +} diff --git a/internal/ingest/generation_load_postgres_test.go b/internal/ingest/generation_load_postgres_test.go new file mode 100644 index 000000000..cd6d8ac94 --- /dev/null +++ b/internal/ingest/generation_load_postgres_test.go @@ -0,0 +1,1728 @@ +package ingest + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "github.com/calypr/loom/internal/backends/postgres/control" + "net/url" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/calypr/loom/internal/catalog" + publication "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + ing03LoadPrefix = "verify_ing03_load_" + ing03LoadDBFmt = ing03LoadPrefix + "%d_%s" + ing03SchemaPfx = ing03LoadPrefix + "s_" + ing03SchemaFmt = ing03SchemaPfx + "%d_%s" + ing03ManifestSchemaPfx = ing03LoadPrefix + "sm_" + ing03ManifestSchemaFmt = ing03ManifestSchemaPfx + "%d_%s" + ing03Project = "prj-ing03-load" + ing03Generation = "gen-build-a" + ing03AuthPath = "/fhir" + ing03AppBase = "ing03-test" + ing03RaceRounds = 10 + ing03LoincSystem = "http://loinc.org" +) + +const ing03HostileID = `O'Brien"; DROP TABLE "fhir_resource"; --$1\%2` + +var ( + ing03LoadNamePattern = regexp.MustCompile(`^verify_ing03_load_[0-9]+_[0-9a-f]+$`) + ing03SchemaNamePattern = regexp.MustCompile(`^verify_ing03_load_s_[0-9]+_[0-9a-f]+$`) + ing03ManifestSchemaPattern = regexp.MustCompile(`^verify_ing03_load_sm_[0-9]+_[0-9a-f]+$`) + ing03OrdinalPattern = regexp.MustCompile(`^[0-9]+$`) + ing03PlaceholderPat = regexp.MustCompile(`^\$[0-9]+$`) +) + +func ing03TestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 generation-load integration tests") + } + return raw +} + +func ing03RandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) +} + +var ing03ControlSeq atomic.Int64 + +var ( + ing03OwnedMu sync.Mutex + ing03OwnedByName = map[string]*ing03OwnedFixture{} + ing03BaselineOnce sync.Once + ing03Baseline int64 + ing03BaselineErr error +) + +type ing03OwnedFixture struct { + database string + schema string + dropped bool +} + +func ing03RegisterFixture(db, schema string) { + ing03OwnedMu.Lock() + defer ing03OwnedMu.Unlock() + ing03OwnedByName[db] = &ing03OwnedFixture{database: db, schema: schema} +} + +func ing03MarkDropped(db string) { + ing03OwnedMu.Lock() + defer ing03OwnedMu.Unlock() + if fixture, ok := ing03OwnedByName[db]; ok { + fixture.dropped = true + } +} + +func ing03CaptureBaseline(raw string) { + ing03BaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + ing03BaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_ing03\\_load\\_%' ESCAPE '\\'"). + Scan(&count); err != nil { + ing03BaselineErr = fmt.Errorf("count baseline: %w", err) + return + } + ing03Baseline = count + }) +} + +var ing03ManifestSeriesTables = map[string]bool{ + "schema_migrations": true, + "dataset_generation_manifest": true, + "dataset_active_generation": true, + "project_release": true, + "active_project_release": true, +} + +var ing03KnownTables = map[string]bool{ + "schema_migrations": true, + "fhir_resource": true, + "dataset_generation_manifest": true, + "dataset_active_generation": true, + "fhir_edge_occurrence": true, + "fhir_resource_inventory_count": true, + "fhir_resource_inventory_build": true, + "fhir_semantic_inventory_build": true, + "fhir_field_profile": true, + "fhir_field_profile_coverage": true, + "batch_write_ledger": true, + "fhir_semantic_inventory_contribution": true, +} + +func ing03QIdent(t *testing.T, schema, table string) string { + t.Helper() + if !ing03SchemaNamePattern.MatchString(schema) && !ing03ManifestSchemaPattern.MatchString(schema) { + t.Fatalf("schema %q is not an owned verify_ing03_load_s_*/sm_* name", schema) + } + if !ing03KnownTables[table] && !ing03ManifestSeriesTables[table] { + t.Fatalf("table %q is not a known migration table", table) + } + return pgx.Identifier{schema, table}.Sanitize() +} + +func ing03OrderBy(columnCount int) string { + if columnCount < 1 { + return "" + } + ords := make([]string, 0, columnCount) + for i := 1; i <= columnCount; i++ { + ordinal := fmt.Sprintf("%d", i) + if !ing03OrdinalPattern.MatchString(ordinal) { + panic("generated ordinal violates the numeric pattern") + } + ords = append(ords, ordinal) + } + return " ORDER BY " + strings.Join(ords, ",") +} + +func ing03Placeholder(t *testing.T, n int) string { + t.Helper() + placeholder := fmt.Sprintf("$%d", n) + if !ing03PlaceholderPat.MatchString(placeholder) { + t.Fatalf("generated placeholder %q violates the $n pattern", placeholder) + } + return placeholder +} + +func ing03Exec(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) { + t.Helper() + if _, err := pool.Exec(context.Background(), sql, args...); err != nil { + t.Fatalf("exec failed: %v", err) + } +} + +func ing03Count(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) int64 { + t.Helper() + var count int64 + if err := pool.QueryRow(context.Background(), sql, args...).Scan(&count); err != nil { + t.Fatalf("count failed: %v", err) + } + return count +} + +type ing03Service struct { + t *testing.T + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + manifestSchema string + scrub func(any) string + faults *ing03CatalogFaults +} + +type ing03CatalogFaults struct { + wrapFieldProfile func(control.FieldProfileStore) control.FieldProfileStore + wrapInventory func(control.InventoryStore) control.InventoryStore + wrapRelationship func(control.RelationshipStore) control.RelationshipStore + wrapContributions func(inner GenerationSemanticContributor) GenerationSemanticContributor +} + +func ing03WireCatalogWriters(src control.ConnSource, schema string, faults *ing03CatalogFaults, fail func(error)) GenerationCatalogWriters { + profileRepo, err := control.NewFieldProfileRepository(src, schema) + if err != nil { + fail(fmt.Errorf("field profile repository: %w", err)) + } + var profileStore control.FieldProfileStore = profileRepo + if faults != nil && faults.wrapFieldProfile != nil { + profileStore = faults.wrapFieldProfile(profileRepo) + } + profiles, err := control.NewCatalogFieldProfileAdapter(profileStore) + if err != nil { + fail(fmt.Errorf("field profile store: %w", err)) + } + inventoryRepo, err := control.NewInventoryRepository(src, schema) + if err != nil { + fail(fmt.Errorf("inventory repository: %w", err)) + } + var inventoryStore control.InventoryStore = inventoryRepo + if faults != nil && faults.wrapInventory != nil { + inventoryStore = faults.wrapInventory(inventoryRepo) + } + inventory, err := control.NewCatalogInventoryAdapter(inventoryStore) + if err != nil { + fail(fmt.Errorf("inventory store: %w", err)) + } + relationshipRepo, err := control.NewCatalogRelationshipEvidenceRepository(src, schema) + if err != nil { + fail(fmt.Errorf("relationship evidence repository: %w", err)) + } + var relationshipStore control.RelationshipStore = relationshipRepo + if faults != nil && faults.wrapRelationship != nil { + relationshipStore = faults.wrapRelationship(relationshipRepo) + } + relations, err := control.NewCatalogRelationshipAdapter(relationshipStore) + if err != nil { + fail(fmt.Errorf("relationship evidence store: %w", err)) + } + contribs, err := control.NewSemanticContributionRepository(src, schema) + if err != nil { + fail(fmt.Errorf("semantic contribution repository: %w", err)) + } + var contributions GenerationSemanticContributor = contribs + if faults != nil && faults.wrapContributions != nil { + contributions = faults.wrapContributions(contribs) + } + return GenerationCatalogWriters{ + FieldProfiles: profiles, + Inventory: inventory, + Relationships: relations, + Contributions: contributions, + } +} + +func (s *ing03Service) catalogWriters() GenerationCatalogWriters { + s.t.Helper() + return ing03WireCatalogWriters(s.iso, s.schema, s.faults, func(err error) { + s.t.Fatalf("catalog wiring: %s", s.scrub(err)) + }) +} + +func newIngest03Service(t *testing.T) *ing03Service { + t.Helper() + raw := ing03TestPostgresURL(t) + ing03CaptureBaseline(raw) + if ing03BaselineErr != nil { + t.Fatalf("capture pg_database baseline: %v", ing03BaselineErr) + } + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + + ctx, cancel := context.WithTimeout(context.Background(), 900*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %s", scrub(err)) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf(ing03LoadDBFmt, time.Now().UnixNano(), ing03RandomHex(t)) + if !ing03LoadNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + t.Fatalf("create isolated database: %s", scrub(err)) + } + schema := fmt.Sprintf(ing03SchemaFmt, time.Now().UnixNano(), ing03RandomHex(t)) + if !ing03SchemaNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + ing03RegisterFixture(dbName, schema) + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(context.Background()), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + return + } + ing03MarkDropped(dbName) + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect to isolated database: %s", scrub(err)) + } + t.Cleanup(iso.Close) + + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + t.Fatalf("create owned schema: %s", scrub(err)) + } + manifestSchema := fmt.Sprintf(ing03ManifestSchemaFmt, time.Now().UnixNano(), ing03RandomHex(t)) + if !ing03ManifestSchemaPattern.MatchString(manifestSchema) { + t.Fatalf("manifest schema name %q does not match the required pattern", manifestSchema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+manifestSchema+`"`); err != nil { + t.Fatalf("create owned manifest schema: %s", scrub(err)) + } + return &ing03Service{t: t, ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, manifestSchema: manifestSchema, scrub: scrub} +} + +func (s *ing03Service) migrate() { + s.t.Helper() + client := s.openClient(2) + if _, err := control.MigrateSchema(s.ctx, client, s.schema); err != nil { + s.t.Fatalf("MigrateSchema: %s", s.scrub(err)) + } + if _, err := control.MigrateDatasetReleaseSchema(s.ctx, client, s.manifestSchema); err != nil { + s.t.Fatalf("MigrateDatasetReleaseSchema: %s", s.scrub(err)) + } +} + +func (s *ing03Service) openClient(maxConns int32) *control.Client { + s.t.Helper() + client, err := control.OpenPool(s.ctx, control.ClientConfig{ + DSN: s.dsn, + MaxConns: maxConns, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + s.t.Fatalf("open client pool: %s", s.scrub(err)) + } + s.t.Cleanup(client.Close) + return client +} + +func (s *ing03Service) namedPool(application string) *pgxpool.Pool { + s.t.Helper() + cfg, err := pgxpool.ParseConfig(s.dsn) + if err != nil { + s.t.Fatalf("parse pool config: %s", s.scrub(err)) + } + cfg.MaxConns = 4 + cfg.ConnConfig.RuntimeParams["application_name"] = application + pool, err := pgxpool.NewWithConfig(context.Background(), cfg) + if err != nil { + s.t.Fatalf("open named pool %q: %s", application, s.scrub(err)) + } + s.t.Cleanup(pool.Close) + return pool +} + +func (s *ing03Service) pgOptions(application string) PGSourceOptions { + return PGSourceOptions{ + DSN: s.dsn, + Schema: s.schema, + ManifestSchema: s.manifestSchema, + ApplicationName: application, + MaxConns: 4, + Catalog: ing03WireCatalogWriters(s.namedPool(application), s.schema, s.faults, + func(err error) { s.t.Fatalf("catalog wiring: %s", s.scrub(err)) }), + } +} + +func (s *ing03Service) loadOptions(metaDir, project string, dataset *publication.Ref) LoadOptions { + return LoadOptions{ + Schema: repoPath(s.t, "schemas", "graph-fhir.json"), + MetaDir: metaDir, + Project: project, + Dataset: dataset, + AuthResourcePath: ing03AuthPath, + } +} + +func (s *ing03Service) manifestState(project, generation string) (string, bool) { + s.t.Helper() + table := ing03QIdent(s.t, s.manifestSchema, "dataset_generation_manifest") + var state string + err := s.iso.QueryRow(context.Background(), + "SELECT state FROM "+table+" WHERE project = $1 AND generation = $2", project, generation).Scan(&state) + if errors.Is(err, pgx.ErrNoRows) { + return "", false + } + if err != nil { + s.t.Fatalf("read manifest state: %s", s.scrub(err)) + } + return state, true +} + +func (s *ing03Service) activePointerGeneration(project string) (string, bool) { + s.t.Helper() + table := ing03QIdent(s.t, s.manifestSchema, "dataset_active_generation") + var generation string + err := s.iso.QueryRow(context.Background(), + "SELECT generation FROM "+table+" WHERE project = $1", project).Scan(&generation) + if errors.Is(err, pgx.ErrNoRows) { + return "", false + } + if err != nil { + s.t.Fatalf("read active pointer: %s", s.scrub(err)) + } + return generation, true +} + +func (s *ing03Service) tableSchema(table string) string { + if ing03ManifestSeriesTables[table] && table != "schema_migrations" { + return s.manifestSchema + } + return s.schema +} + +func (s *ing03Service) countTable(table, where string, args ...any) int64 { + s.t.Helper() + fqn := ing03QIdent(s.t, s.tableSchema(table), table) + sql := "SELECT count(*) FROM " + fqn + if where != "" { + sql += " WHERE " + where + } + return ing03Count(s.t, s.iso, sql, args...) +} + +var ing03FingerprintColumns = []struct { + table string + column string +}{ + {"dataset_generation_manifest", "project, generation, state, schema_identity::text"}, + {"dataset_active_generation", "project, generation"}, + {"fhir_resource", "project, dataset_generation, resource_type, logical_id, auth_resource_path, payload::text"}, + {"fhir_edge_occurrence", "project, dataset_generation, auth_resource_path, label, source_resource_type, source_logical_id, target_resource_type, target_logical_id, source_path, source_ordinal, logical_edge_key"}, + {"batch_write_ledger", "batch_id, project, dataset_generation, digest, resource_count, edge_count"}, + {"fhir_resource_inventory_count", "count_key, build_id, project, dataset_generation, resource_type, auth_resource_path, document_count"}, + {"fhir_resource_inventory_build", "marker_key, project, dataset_generation, build_id, coverage_digest, resource_types::text, resource_type_count, count_row_count, complete"}, + {"fhir_field_profile", "profile_key, record_type, kind, doc_count, max_items, sample_count, distinct_values::text, distinct_truncated, enrichment_complete, enrichment_truncated, pivot_candidate, pivot_kind, pivot_columns::text, pivot_family, pivot_column_selector, pivot_value_selector, pivot_item_source, pivot_item_resource_type, pivot_value_selectors::text, extension_values::text, semantic_observations::text, project, dataset_generation, auth_resource_path, resource_type, path"}, + {"fhir_field_profile_coverage", "project, dataset_generation, auth_resource_path, resource_type"}, + {"fhir_semantic_inventory_build", "build_key, project, dataset_generation, build_id, state, observation_schema, inventory_schema, entry_index_version, rule_version, source_kind, source_availability, auth_resource_path, scanned_resources, checkpoint, checkpoint_collection, checkpoint_key, lease_token, lease_expires_at, diagnostics::text"}, + {"fhir_semantic_inventory_contribution", "contribution_key, project, dataset_generation, build_id, source_kind, auth_resource_path, resource_type, source_id, ordinal, concept_id, binding_id, example, payload_digest, observation::text"}, +} + +func (s *ing03Service) fingerprint() string { + s.t.Helper() + digest := sha256.New() + for _, entry := range ing03FingerprintColumns { + fqn := ing03QIdent(s.t, s.tableSchema(entry.table), entry.table) + sql := "SELECT " + entry.column + " FROM " + fqn + rows, err := s.iso.Query(context.Background(), sql) + if err != nil { + s.t.Fatalf("fingerprint query %s: %s", entry.table, s.scrub(err)) + } + var rendered []string + for rows.Next() { + values, scanErr := rows.Values() + if scanErr != nil { + rows.Close() + s.t.Fatalf("fingerprint scan %s: %s", entry.table, s.scrub(scanErr)) + } + rendered = append(rendered, fmt.Sprintf("%v", values)) + } + if err := rows.Err(); err != nil { + rows.Close() + s.t.Fatalf("fingerprint rows %s: %s", entry.table, s.scrub(err)) + } + rows.Close() + sort.Strings(rendered) + fmt.Fprintf(digest, "%s\n", entry.table) + for _, row := range rendered { + fmt.Fprintf(digest, "%s\n", row) + } + } + return hex.EncodeToString(digest.Sum(nil)) +} + +type ing03BackendWitness struct { + Pid int32 + Application string + State *string + WaitEventType *string +} + +func (s *ing03Service) backendsByApplication(appNames ...string) []ing03BackendWitness { + rows, err := s.iso.Query(context.Background(), + `SELECT pid, application_name, state, wait_event_type FROM pg_stat_activity WHERE application_name = ANY($1::text[]) AND backend_type = 'client backend' AND pid <> pg_backend_pid()`, + appNames) + if err != nil { + s.t.Fatalf("witness backends: %s", s.scrub(err)) + } + defer rows.Close() + var out []ing03BackendWitness + for rows.Next() { + var w ing03BackendWitness + if err := rows.Scan(&w.Pid, &w.Application, &w.State, &w.WaitEventType); err != nil { + s.t.Fatalf("scan backend witness: %s", s.scrub(err)) + } + out = append(out, w) + } + if err := rows.Err(); err != nil { + s.t.Fatalf("rows backend witness: %s", s.scrub(err)) + } + return out +} + +func (s *ing03Service) maxConcurrentDistinctApps(done <-chan struct{}, appNames ...string) int { + best := 0 + for { + select { + case <-done: + return best + default: + } + backends := s.backendsByApplication(appNames...) + seen := map[string]bool{} + for _, backend := range backends { + if backend.State != nil && *backend.State == "active" { + seen[backend.Application] = true + } + } + if len(seen) > best { + best = len(seen) + } + time.Sleep(2 * time.Millisecond) + } +} + +func (s *ing03Service) waitForBackend(appName string, timeout time.Duration) (ing03BackendWitness, bool) { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + backends := s.backendsByApplication(appName) + if len(backends) > 0 { + return backends[0], true + } + time.Sleep(2 * time.Millisecond) + } + return ing03BackendWitness{}, false +} + +func (s *ing03Service) waitForBackendWaiting(appName string, timeout time.Duration) (ing03BackendWitness, bool) { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + for _, backend := range s.backendsByApplication(appName) { + if backend.WaitEventType != nil && *backend.WaitEventType == "Lock" { + return backend, true + } + } + time.Sleep(2 * time.Millisecond) + } + return ing03BackendWitness{}, false +} + +func (s *ing03Service) terminateAppBackends(appName string) int { + rows, err := s.iso.Query(context.Background(), + `SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name = $1 AND backend_type = 'client backend' AND pid <> pg_backend_pid()`, + appName) + if err != nil { + s.t.Fatalf("terminate backends: %s", s.scrub(err)) + } + killed := 0 + for rows.Next() { + var ok bool + if err := rows.Scan(&ok); err != nil { + rows.Close() + s.t.Fatalf("scan terminate: %s", s.scrub(err)) + } + if ok { + killed++ + } + } + if err := rows.Err(); err != nil { + rows.Close() + s.t.Fatalf("rows terminate: %s", s.scrub(err)) + } + rows.Close() + return killed +} + +func (s *ing03Service) blockTable(table string) (release func()) { + s.t.Helper() + fqn := ing03QIdent(s.t, s.tableSchema(table), table) + conn, err := s.iso.Acquire(s.ctx) + if err != nil { + s.t.Fatalf("acquire blocker: %s", s.scrub(err)) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + tx, err := conn.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + conn.Release() + cancel() + s.t.Fatalf("begin blocker: %s", s.scrub(err)) + } + if _, err := tx.Exec(ctx, `SET LOCAL lock_timeout = '200s'`); err != nil { + _ = tx.Rollback(ctx) + conn.Release() + cancel() + s.t.Fatalf("blocker lock timeout: %s", s.scrub(err)) + } + if _, err := tx.Exec(ctx, `LOCK TABLE `+fqn+` IN ACCESS EXCLUSIVE MODE`); err != nil { + _ = tx.Rollback(ctx) + conn.Release() + cancel() + s.t.Fatalf("lock %s: %s", table, s.scrub(err)) + } + var released bool + release = func() { + if released { + return + } + released = true + _ = tx.Rollback(context.WithoutCancel(ctx)) + conn.Release() + cancel() + } + s.t.Cleanup(release) + return release +} + +func (s *ing03Service) blockManifestRow(project, generation string) (release func()) { + s.t.Helper() + fqn := ing03QIdent(s.t, s.manifestSchema, "dataset_generation_manifest") + conn, err := s.iso.Acquire(s.ctx) + if err != nil { + s.t.Fatalf("acquire manifest blocker: %s", s.scrub(err)) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + tx, err := conn.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + conn.Release() + cancel() + s.t.Fatalf("begin manifest blocker: %s", s.scrub(err)) + } + if _, err := tx.Exec(ctx, `SET LOCAL lock_timeout = '200s'`); err != nil { + _ = tx.Rollback(ctx) + conn.Release() + cancel() + s.t.Fatalf("manifest blocker lock timeout: %s", s.scrub(err)) + } + if _, err := tx.Exec(ctx, `SELECT project FROM `+fqn+` WHERE project = $1 AND generation = $2 FOR UPDATE`, + project, generation); err != nil { + _ = tx.Rollback(ctx) + conn.Release() + cancel() + s.t.Fatalf("lock manifest row: %s", s.scrub(err)) + } + var released bool + release = func() { + if released { + return + } + released = true + _ = tx.Rollback(context.WithoutCancel(ctx)) + conn.Release() + cancel() + } + s.t.Cleanup(release) + return release +} + +func ing03JSONLine(t *testing.T, document map[string]any) string { + t.Helper() + encoded, err := json.Marshal(document) + if err != nil { + t.Fatalf("marshal fixture line: %v", err) + } + return string(encoded) +} + +func ing03ObservationLine(t *testing.T, id, subject string, focusRefs []string, coded bool) string { + t.Helper() + document := map[string]any{ + "resourceType": "Observation", + "id": id, + "status": "final", + "subject": map[string]any{"reference": "Patient/" + subject}, + } + if coded { + document["code"] = map[string]any{"coding": []any{map[string]any{ + "system": ing03LoincSystem, "code": "8867-4", "display": "Heart rate"}}} + } else { + document["code"] = map[string]any{"text": ing03HostileID} + } + if len(focusRefs) > 0 { + focus := make([]any, 0, len(focusRefs)) + for _, ref := range focusRefs { + focus = append(focus, map[string]any{"reference": ref}) + } + document["focus"] = focus + } + return ing03JSONLine(t, document) +} + +func ing03PatientLine(t *testing.T, id string) string { + t.Helper() + return ing03JSONLine(t, map[string]any{"resourceType": "Patient", "id": id, "gender": "female"}) +} + +func ing03HostilePatientLine(t *testing.T) string { + t.Helper() + return ing03JSONLine(t, map[string]any{ + "resourceType": "Patient", + "id": "pat-hostile", + "gender": "female", + "name": []any{map[string]any{"text": ing03HostileID}}, + }) +} + +func ing03WriteFixtures(t *testing.T, dir string, rowsPerFile int, lateBadRow bool) { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir fixtures: %v", err) + } + observationRows := []string{ + ing03ObservationLine(t, "obs-a", "pat-0000", + []string{"Observation/obs-a", "Observation/obs-a"}, true), + ing03ObservationLine(t, "obs-b", "pat-hostile", nil, false), + } + if err := os.WriteFile(filepath.Join(dir, "Observation.ndjson"), + []byte(strings.Join(observationRows, "\n")+"\n"), 0o600); err != nil { + t.Fatalf("write Observation fixture: %v", err) + } + patientRows := make([]string, 0, rowsPerFile+1) + for i := 0; i < rowsPerFile; i++ { + patientRows = append(patientRows, ing03PatientLine(t, fmt.Sprintf("pat-%04d", i))) + } + patientRows = append(patientRows, ing03HostilePatientLine(t)) + if lateBadRow { + patientRows = append(patientRows[:11], append([]string{`{oops`}, patientRows[11:]...)...) + } + if err := os.WriteFile(filepath.Join(dir, "Patient.ndjson"), + []byte(strings.Join(patientRows, "\n")+"\n"), 0o600); err != nil { + t.Fatalf("write Patient fixture: %v", err) + } +} + +func ing03Ref(t *testing.T, project, generation string) publication.Ref { + t.Helper() + ref, err := publication.NewRef(project, generation) + if err != nil { + t.Fatalf("ref: %v", err) + } + return ref +} + +func TestPostgresGenerationLoadPreflightRunsBeforePostgresContact(t *testing.T) { + service := newIngest03Service(t) + goodDir := t.TempDir() + ing03WriteFixtures(t, goodDir, 2, false) + ref := ing03Ref(t, ing03Project, ing03Generation) + + badUnknown := t.TempDir() + writePreflightFixture(t, badUnknown, "Unknown.ndjson", `{"resourceType":"Unknown"}`+"\n") + + badDirIsFile := filepath.Join(t.TempDir(), "not-a-dir.ndjson") + if err := os.WriteFile(badDirIsFile, []byte("{}\n"), 0o600); err != nil { + t.Fatalf("write non-directory: %v", err) + } + + app := ing03AppBase + "-guard" + tests := []struct { + name string + mutate func(*LoadOptions) + wantErr error + }{ + {"dataset required", func(o *LoadOptions) { o.Dataset = nil }, ErrGenerationDatasetRequired}, + {"truncate forbidden", func(o *LoadOptions) { o.Truncate = true }, ErrGenerationLoadTruncateForbidden}, + {"project mismatch", func(o *LoadOptions) { o.Project = "prj-something-else" }, ErrGenerationDatasetProjectMismatch}, + {"requires directory", func(o *LoadOptions) { o.MetaDir = badDirIsFile }, ErrGenerationLoadRequiresDirectory}, + {"requires files", func(o *LoadOptions) { o.MetaDir = t.TempDir() }, ErrGenerationLoadRequiresFiles}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var events []string + opts := service.loadOptions(goodDir, ing03Project, &ref) + opts.EventSink = func(event string, _ map[string]any) { events = append(events, event) } + test.mutate(&opts) + _, err := LoadPostgresGeneration(context.Background(), opts, service.pgOptions(app)) + if !errors.Is(err, test.wantErr) { + t.Fatalf("error = %v, want %v", err, test.wantErr) + } + for _, event := range events { + if strings.HasPrefix(event, "go_pg_") { + t.Fatalf("guard %s opened the PG backend: %v", test.name, events) + } + } + controlEvents := ing03RunLoadCapturingEvents(t, service, goodDir, app+"-control") + if !containsPrefix(controlEvents, "go_pg_") { + t.Fatalf("control load never contacted PG; ordering witness is vacuous: %v", controlEvents) + } + }) + } + + t.Run("preflight failure precedes backend", func(t *testing.T) { + clean := newIngest03Service(t) + var events []string + opts := clean.loadOptions(badUnknown, ing03Project, &ref) + opts.EventSink = func(event string, _ map[string]any) { events = append(events, event) } + _, err := LoadPostgresGeneration(context.Background(), opts, clean.pgOptions(app)) + var preflightErr *PreflightError + if !errors.As(err, &preflightErr) { + t.Fatalf("error = %T %v, want PreflightError", err, err) + } + for _, event := range events { + if strings.HasPrefix(event, "go_pg_") { + t.Fatalf("preflight failure still reached PG: %v", events) + } + } + var missing bool + if err := clean.iso.QueryRow(context.Background(), + `SELECT to_regclass($1) IS NULL`, clean.manifestSchema+".dataset_generation_manifest").Scan(&missing); err != nil { + t.Fatalf("probe schema: %v", err) + } + if !missing { + t.Fatal("preflight failure still created the manifest table: MigrateSchema ran") + } + cleanDir := t.TempDir() + ing03WriteFixtures(t, cleanDir, 2, false) + if _, cleanErr := LoadPostgresGeneration(context.Background(), + clean.loadOptions(cleanDir, ing03Project, &ref), + clean.pgOptions(ing03AppBase+"-preflight-control")); cleanErr != nil { + t.Fatalf("preflight control load: %s", clean.scrub(cleanErr)) + } + if err := clean.iso.QueryRow(context.Background(), + `SELECT to_regclass($1) IS NOT NULL`, clean.manifestSchema+".dataset_generation_manifest").Scan(&missing); err != nil { + t.Fatalf("probe control schema: %v", err) + } + if !missing { + t.Fatal("control load did not migrate the schema: the witness is vacuous") + } + }) + + t.Run("blank dsn refused before connect", func(t *testing.T) { + _, err := LoadPostgresGeneration(context.Background(), + service.loadOptions(goodDir, ing03Project, &ref), + PGSourceOptions{DSN: "", Schema: service.schema, ManifestSchema: service.manifestSchema, ApplicationName: app, Catalog: service.catalogWriters()}) + if !errors.Is(err, control.ErrInvalidDSN) { + t.Fatalf("error = %v, want %v", err, control.ErrInvalidDSN) + } + }) + + t.Run("invalid pool config refused before connect", func(t *testing.T) { + _, err := LoadPostgresGeneration(context.Background(), + service.loadOptions(goodDir, ing03Project, &ref), + PGSourceOptions{DSN: service.dsn, Schema: service.schema, ManifestSchema: service.manifestSchema, MaxConns: -3, ApplicationName: app, Catalog: service.catalogWriters()}) + if !errors.Is(err, control.ErrInvalidPoolConfig) { + t.Fatalf("error = %v, want %v", err, control.ErrInvalidPoolConfig) + } + }) + + t.Run("invalid schema name refused by named errors", func(t *testing.T) { + _, err := LoadPostgresGeneration(context.Background(), + service.loadOptions(goodDir, ing03Project, &ref), + PGSourceOptions{DSN: service.dsn, Schema: "Not A Schema", ManifestSchema: service.manifestSchema, ApplicationName: app, Catalog: service.catalogWriters()}) + if !errors.Is(err, control.ErrInvalidSchemaName) { + t.Fatalf("error = %v, want %v", err, control.ErrInvalidSchemaName) + } + }) +} + +func ing03RunLoadCapturingEvents(t *testing.T, service *ing03Service, metaDir, app string) []string { + t.Helper() + ref := ing03Ref(t, ing03Project, fmt.Sprintf("gen-ctl-%d", ing03ControlSeq.Add(1))) + var events []string + opts := service.loadOptions(metaDir, ing03Project, &ref) + opts.EventSink = func(event string, _ map[string]any) { events = append(events, event) } + _, err := LoadPostgresGeneration(context.Background(), opts, service.pgOptions(app)) + if err != nil { + t.Fatalf("control load: %s", service.scrub(err)) + } + return events +} + +func containsPrefix(events []string, prefix string) bool { + for _, event := range events { + if strings.HasPrefix(event, prefix) { + return true + } + } + return false +} + +func TestPostgresGenerationLoadStagesMultiResourceAndNeverActivates(t *testing.T) { + for _, generic := range []bool{false, true} { + name := "generated" + if generic { + name = "generic" + } + t.Run(name, func(t *testing.T) { + service := newIngest03Service(t) + metaDir := t.TempDir() + ing03WriteFixtures(t, metaDir, 2, false) + ref := ing03Ref(t, ing03Project, ing03Generation) + opts := service.loadOptions(metaDir, ing03Project, &ref) + opts.BatchSize = 2 + opts.UseGeneric = generic + var events []string + opts.EventSink = func(event string, _ map[string]any) { events = append(events, event) } + + summary, err := LoadPostgresGeneration(context.Background(), opts, service.pgOptions(ing03AppBase+"-"+name)) + if err != nil { + t.Fatalf("load: %s (events=%v)", service.scrub(err), events) + } + if err := ing03AssertEvidenceOrder(t, events); err != nil { + t.Fatal(err) + } + if summary.VerticesInserted != 5 { + t.Fatalf("vertices = %d, want 5 committed resource rows", summary.VerticesInserted) + } + if summary.Resources["Patient"] != 3 || summary.Resources["Observation"] != 2 { + t.Fatalf("resources = %v, want Patient 3 Observation 2", summary.Resources) + } + if state, found := service.manifestState(ing03Project, ing03Generation); !found || state != string(publication.StateStaged) { + t.Fatalf("manifest state = %q found=%v, want STAGED", state, found) + } + if _, active := service.activePointerGeneration(ing03Project); active { + t.Fatal("active pointer moved: the load must never activate") + } + if count := service.countTable("fhir_resource", "logical_id = $1", "pat-hostile"); count != 1 { + t.Fatalf("hostile-name rows = %d, want exactly 1", count) + } + var payloadText string + if err := service.iso.QueryRow(context.Background(), + "SELECT payload::text FROM "+ing03QIdent(t, service.schema, "fhir_resource")+" WHERE logical_id = $1", + "pat-hostile").Scan(&payloadText); err != nil { + t.Fatalf("read hostile payload: %s", service.scrub(err)) + } + if !strings.Contains(payloadText, "DROP TABLE") { + t.Fatalf("hostile payload did not round-trip as data: %s", payloadText) + } + if got := service.countTable("fhir_resource", ""); got != 5 { + t.Fatalf("resource rows = %d, want 5 (injection changed nothing)", got) + } + for _, lane := range []struct { + schema string + want int + }{ + {service.schema, len(ing03KnownTables)}, + {service.manifestSchema, len(ing03ManifestSeriesTables)}, + } { + var relations int64 + if err := service.iso.QueryRow(context.Background(), + `SELECT count(*) FROM pg_tables WHERE schemaname = $1`, lane.schema).Scan(&relations); err != nil { + t.Fatalf("count relations: %s", service.scrub(err)) + } + if relations != int64(lane.want) { + t.Fatalf("relations in %s = %d, want exactly the %d migrated tables for that runner", lane.schema, relations, lane.want) + } + } + var obsPayloadText string + if err := service.iso.QueryRow(context.Background(), + "SELECT payload::text FROM "+ing03QIdent(t, service.schema, "fhir_resource")+" WHERE logical_id = $1", + "obs-b").Scan(&obsPayloadText); err != nil { + t.Fatalf("read obs-b payload: %s", service.scrub(err)) + } + if !strings.Contains(obsPayloadText, "DROP TABLE") { + t.Fatalf("obs-b hostile payload did not round-trip as data: %s", obsPayloadText) + } + if count := service.countTable("fhir_edge_occurrence", + "label = $1 AND target_logical_id = $2", "subject_Patient", "pat-hostile"); count != 1 { + t.Fatalf("pat-hostile forward subject edges = %d, want 1", count) + } + if edgeCount := service.countTable("fhir_edge_occurrence", ""); edgeCount != int64(summary.EdgesInserted) { + t.Fatalf("edge rows = %d, summary says %d", edgeCount, summary.EdgesInserted) + } + focus := service.countTable("fhir_edge_occurrence", + "label IN ('focus_observation', 'focus_Observation')") + if generic && focus != 4 { + t.Fatalf("generic focus occurrences = %d, want 4 (two repeated-array positions, forward+back each)", focus) + } + if !generic && focus != 2 { + t.Fatalf("generated focus occurrences = %d, want 2 (collapsed duplicate, forward+back)", focus) + } + for _, resourceType := range []string{"Patient", "Observation"} { + if count := service.countTable("fhir_field_profile_coverage", + "resource_type = $1", resourceType); count != 1 { + t.Fatalf("coverage markers for %s = %d, want 1", resourceType, count) + } + } + var documentCount int64 + if err := service.iso.QueryRow(context.Background(), + "SELECT document_count FROM "+ing03QIdent(t, service.schema, "fhir_resource_inventory_count")+ + " WHERE resource_type = $1", "Patient").Scan(&documentCount); err != nil { + t.Fatalf("read inventory: %s", service.scrub(err)) + } + if documentCount != 3 { + t.Fatalf("inventory Patient count = %d, want 3", documentCount) + } + if summary.BatchCounts["relationship_observations"] == 0 { + t.Fatal("relationship evidence empty although edges were committed") + } + if summary.BatchCounts["semantic_contributions"] == 0 { + t.Fatal("no semantic contributions although a loinc coding was staged") + } + if persisted := service.countTable("fhir_semantic_inventory_contribution", ""); persisted != + int64(summary.BatchCounts["semantic_contributions"]) { + t.Fatalf("persisted contributions = %d, summary says %d", persisted, summary.BatchCounts["semantic_contributions"]) + } + }) + } +} + +func ing03AssertEvidenceOrder(t *testing.T, events []string) error { + t.Helper() + indexOf := func(want string) int { + for i, event := range events { + if event == want { + return i + } + } + return -1 + } + preflightDone := indexOf("go_preflight_complete") + if preflightDone < 0 { + return fmt.Errorf("preflight never completed: %v", events) + } + for i, event := range events { + if i < preflightDone && strings.HasPrefix(event, "go_pg_") { + return fmt.Errorf("PG contact %q before preflight completion: %v", event, events) + } + } + firstResource, firstEdge := -1, -1 + lastResource, lastEdge := -1, -1 + for i, event := range events { + switch event { + case "go_pg_resource_batch_commit": + if firstResource < 0 { + firstResource = i + } + lastResource = i + case "go_pg_edge_batch_commit": + if firstEdge < 0 { + firstEdge = i + } + lastEdge = i + } + } + if firstResource < 0 || firstEdge < 0 { + return fmt.Errorf("missing batch commits: %v", events) + } + if lastResource > firstEdge { + return fmt.Errorf("an edge batch started before every resource batch committed: %v", events) + } + catalogFirst := -1 + for i, event := range events { + if event == "go_pg_catalog_field_profiles" || event == "go_pg_resource_inventory_complete" || event == "go_pg_relationship_evidence_complete" { + if catalogFirst < 0 { + catalogFirst = i + } + } + } + if catalogFirst >= 0 && catalogFirst < lastEdge { + return fmt.Errorf("catalog evidence interleaved with edge writes: %v", events) + } + return nil +} + +func TestPostgresGenerationLoadMalformedLateRowMarksFailedAndBlocksActivation(t *testing.T) { + service := newIngest03Service(t) + metaDir := t.TempDir() + ing03WriteFixtures(t, metaDir, 25, true) + ref := ing03Ref(t, ing03Project, ing03Generation) + opts := service.loadOptions(metaDir, ing03Project, &ref) + + _, err := LoadPostgresGeneration(context.Background(), opts, service.pgOptions(ing03AppBase+"-late")) + var incomplete *GenerationLoadIncompleteError + if !errors.As(err, &incomplete) { + t.Fatalf("error = %T %v, want GenerationLoadIncompleteError", err, err) + } + if incomplete.ValidationErrors == 0 { + t.Fatalf("incomplete error lost the validation count: %+v", incomplete) + } + if state, found := service.manifestState(ing03Project, ing03Generation); !found || state != string(publication.StateFailed) { + t.Fatalf("manifest state = %q found=%v, want FAILED", state, found) + } + if _, active := service.activePointerGeneration(ing03Project); active { + t.Fatal("late malformed row left an active pointer") + } + if activateErr := ActivatePostgresGeneration(context.Background(), ref, service.pgOptions(ing03AppBase+"-late-act")); !errors.Is(activateErr, publication.ErrGenerationNotReady) { + t.Fatalf("activate = %v, want ErrGenerationNotReady", activateErr) + } + if len(opts.MetaDir) == 0 { + t.Fatal("fixture dir vanished") + } + + cleanDir := t.TempDir() + ing03WriteFixtures(t, cleanDir, 25, false) + cleanService := newIngest03Service(t) + cleanRef := ing03Ref(t, ing03Project, ing03Generation) + if _, cleanErr := LoadPostgresGeneration(context.Background(), + cleanService.loadOptions(cleanDir, ing03Project, &cleanRef), + cleanService.pgOptions(ing03AppBase+"-late-control")); cleanErr != nil { + t.Fatalf("clean control load: %s", cleanService.scrub(cleanErr)) + } + if state, found := cleanService.manifestState(ing03Project, ing03Generation); !found || state != string(publication.StateStaged) { + t.Fatalf("clean control manifest = %q found=%v, want STAGED", state, found) + } + + ledgerBeforeRetry := service.countTable("batch_write_ledger", "") + retryEvents := []string{} + retryOpts := service.loadOptions(metaDir, ing03Project, &ref) + retryOpts.EventSink = func(event string, _ map[string]any) { retryEvents = append(retryEvents, event) } + _, retryErr := LoadPostgresGeneration(context.Background(), retryOpts, service.pgOptions(ing03AppBase+"-late-retry")) + if !errors.Is(retryErr, ErrGenerationLoadAlreadyFailed) { + t.Fatalf("retry = %v, want %v", retryErr, ErrGenerationLoadAlreadyFailed) + } + if state, _ := service.manifestState(ing03Project, ing03Generation); state != string(publication.StateFailed) { + t.Fatalf("retry moved the manifest to %q, want FAILED preserved", state) + } + if ledger := service.countTable("batch_write_ledger", ""); ledger != ledgerBeforeRetry { + t.Fatalf("retry against FAILED wrote %d new ledger rows, want 0 (before=%d after=%d)", + ledger-ledgerBeforeRetry, ledgerBeforeRetry, ledger) + } +} + +func TestPostgresGenerationLoadWriterFailuresLeaveManifestFailedAndPreserveRows(t *testing.T) { + t.Run("resource row conflict refuses overwrite", func(t *testing.T) { + service := newIngest03Service(t) + service.migrate() + client := service.openClient(2) + resourceRepo, err := control.NewResourceRepository(client, service.schema) + if err != nil { + t.Fatalf("resource repo: %v", err) + } + if _, err := resourceRepo.InsertResource(context.Background(), source.Resource{ + Project: ing03Project, DatasetGeneration: ing03Generation, ResourceType: "Patient", + LogicalID: "pat-0000", AuthResourcePath: ing03AuthPath, + Payload: json.RawMessage(`{"id":"pat-0000","resourceType":"Patient","gender":"seeded"}`), + }); err != nil { + t.Fatalf("seed conflicting row: %s", service.scrub(err)) + } + seededPayload := service.resourcePayload("Patient", "pat-0000") + + metaDir := t.TempDir() + ing03WriteFixtures(t, metaDir, 2, false) + ref := ing03Ref(t, ing03Project, ing03Generation) + _, loadErr := LoadPostgresGeneration(context.Background(), + service.loadOptions(metaDir, ing03Project, &ref), service.pgOptions(ing03AppBase+"-conflict")) + var conflict *control.BatchResourceConflictError + if !errors.As(loadErr, &conflict) { + t.Fatalf("load error = %T %v, want BatchResourceConflictError", loadErr, loadErr) + } + if state, found := service.manifestState(ing03Project, ing03Generation); !found || state != string(publication.StateFailed) { + t.Fatalf("manifest state = %q found=%v, want FAILED", state, found) + } + if got := service.resourcePayload("Patient", "pat-0000"); got != seededPayload { + t.Fatalf("retry overwrote the pre-seeded row: %q -> %q", seededPayload, got) + } + if _, active := service.activePointerGeneration(ing03Project); active { + t.Fatal("writer failure moved the active pointer") + } + }) + + t.Run("edge writer failure fails the load", func(t *testing.T) { + reference := newIngest03Service(t) + metaDir := t.TempDir() + ing03WriteFixtures(t, metaDir, 2, false) + ref := ing03Ref(t, ing03Project, ing03Generation) + if _, err := LoadPostgresGeneration(context.Background(), + reference.loadOptions(metaDir, ing03Project, &ref), + reference.pgOptions(ing03AppBase+"-edge-ref")); err != nil { + t.Fatalf("reference load: %s", reference.scrub(err)) + } + victim := reference.readOneEdge() + + service := newIngest03Service(t) + service.migrate() + client := service.openClient(2) + resourceRepo, err := control.NewResourceRepository(client, service.schema) + if err != nil { + t.Fatalf("resource repo: %v", err) + } + if _, err := resourceRepo.InsertResource(context.Background(), reference.sourceSeed(victim)); err != nil { + t.Fatalf("seed edge source %s/%s: %s", victim.SourceType, victim.SourceID, service.scrub(err)) + } + edgeRepo, err := control.NewEdgeRepository(client, service.schema) + if err != nil { + t.Fatalf("edge repo: %v", err) + } + conflicting := victim.row() + tampered := "tampered-logical-key" + conflicting.LogicalEdgeKey = &tampered + if _, err := edgeRepo.InsertEdge(context.Background(), conflicting); err != nil { + t.Fatalf("seed conflicting edge: %s", service.scrub(err)) + } + + _, loadErr := LoadPostgresGeneration(context.Background(), + service.loadOptions(metaDir, ing03Project, &ref), service.pgOptions(ing03AppBase+"-edge-fail")) + var edgeConflict *control.BatchEdgeConflictError + if !errors.As(loadErr, &edgeConflict) { + t.Fatalf("load error = %T %v, want BatchEdgeConflictError", loadErr, loadErr) + } + if state, found := service.manifestState(ing03Project, ing03Generation); !found || state != string(publication.StateFailed) { + t.Fatalf("manifest state = %q found=%v, want FAILED", state, found) + } + if _, active := service.activePointerGeneration(ing03Project); active { + t.Fatal("edge failure moved the active pointer") + } + }) + + t.Run("catalog failures mark FAILED with scoped partials", func(t *testing.T) { + for _, seam := range []string{"field-profile", "inventory", "relationship", "semantic-contributions"} { + t.Run(seam, func(t *testing.T) { + service := newIngest03Service(t) + metaDir := t.TempDir() + ing03WriteFixtures(t, metaDir, 2, false) + ref := ing03Ref(t, ing03Project, ing03Generation) + + service.installCatalogFault(t, seam) + defer func() { service.faults = nil }() + + _, loadErr := LoadPostgresGeneration(context.Background(), + service.loadOptions(metaDir, ing03Project, &ref), service.pgOptions(ing03AppBase+"-catalog-"+seam)) + if !errors.Is(loadErr, errInjectedCatalogFault) { + t.Fatalf("load error = %T %v, want injected catalog fault", loadErr, loadErr) + } + if state, found := service.manifestState(ing03Project, ing03Generation); !found || state != string(publication.StateFailed) { + t.Fatalf("manifest state = %q found=%v, want FAILED", state, found) + } + if _, active := service.activePointerGeneration(ing03Project); active { + t.Fatal("catalog failure moved the active pointer") + } + if count := service.countTable("fhir_resource", ""); count != 5 { + t.Fatalf("pre-catalog rows = %d, want the committed 5", count) + } + switch seam { + case "field-profile": + if first := service.countTable("fhir_field_profile_coverage", "resource_type = $1", "Observation"); first != 1 { + t.Fatalf("Observation coverage after injected fault = %d, want the committed 1", first) + } + if second := service.countTable("fhir_field_profile_coverage", "resource_type = $1", "Patient"); second != 0 { + t.Fatalf("Patient coverage = %d, want 0 (the failed batch wrote nothing)", second) + } + case "inventory": + if markers := service.countTable("fhir_resource_inventory_build", ""); markers != 0 { + t.Fatalf("inventory marker published despite failed scan: %d", markers) + } + case "relationship": + if markers := service.countTable("fhir_resource_inventory_build", ""); markers != 1 { + t.Fatalf("inventory marker = %d, want the published 1", markers) + } + case "semantic-contributions": + if markers := service.countTable("fhir_resource_inventory_build", ""); markers != 1 { + t.Fatalf("inventory marker = %d, want the published 1", markers) + } + if edges := service.countTable("fhir_edge_occurrence", ""); edges == 0 { + t.Fatalf("edge occurrences = %d, want the committed rows from the completed edge pass", edges) + } + if rows := service.countTable("fhir_semantic_inventory_contribution", ""); rows != 0 { + t.Fatalf("contribution rows = %d, want 0 (the failed batch wrote nothing)", rows) + } + } + }) + } + }) +} + +func TestPostgresGenerationLoadSemanticContributionsPersistWithServedStateBoundary(t *testing.T) { + service := newIngest03Service(t) + metaDir := t.TempDir() + ing03WriteFixtures(t, metaDir, 25, false) + ref := ing03Ref(t, ing03Project, ing03Generation) + + if _, err := LoadPostgresGeneration(context.Background(), + service.loadOptions(metaDir, ing03Project, &ref), service.pgOptions(ing03AppBase+"-semantics-pin")); err != nil { + t.Fatalf("load: %s", service.scrub(err)) + } + written := service.countTable("fhir_semantic_inventory_contribution", "") + if written == 0 { + t.Fatal("control load persisted no contributions: the witness is vacuous") + } + + readRepo, err := control.NewSemanticReadRepository(service.iso, service.schema) + if err != nil { + t.Fatalf("semantic read repository: %s", service.scrub(err)) + } + unrestricted := true + page, err := readRepo.PageSemanticInventory(context.Background(), catalog.SemanticInventoryPageOptions{ + Project: ing03Project, + DatasetGeneration: ing03Generation, + AuthResourcePathsUnrestricted: &unrestricted, + }) + if err != nil { + t.Fatalf("page semantic inventory: %s", service.scrub(err)) + } + if page.State != catalog.SemanticInventoryUnknown || len(page.Entries) != 0 { + t.Fatalf("page = state %q entries %d, want Unknown/empty while the file-sourced PG build lifecycle is unwired (flip this assertion when the owning slice serves the build row)", page.State, len(page.Entries)) + } +} + +func TestPostgresGenerationLoadApplicationNameTagging(t *testing.T) { + tagged, applied := joinApplicationName("postgres://u:p@h:5432/db", "app-x") + if !applied || !strings.Contains(tagged, "application_name=app-x") { + t.Fatalf("url dsn = %q applied=%v, want application_name=app-x", tagged, applied) + } + kept, applied := joinApplicationName("postgres://u:p@h:5432/db?application_name=mine", "app-x") + if !applied || !strings.Contains(kept, "application_name=mine") { + t.Fatalf("explicit tag = %q applied=%v, want the caller's mine preserved and counted as tagged", kept, applied) + } + untagged, applied := joinApplicationName("host=h dbname=d user=u", "app-x") + if applied || untagged != "host=h dbname=d user=u" { + t.Fatalf("keyword/value dsn = %q applied=%v, want unchanged and reported untagged", untagged, applied) + } + + var events []string + _, err := openPGGenerationClient(PGSourceOptions{ + DSN: "host=h dbname=d user=u", + Schema: "s", + ApplicationName: "app-x", + }, func(event string, _ map[string]any) { events = append(events, event) }) + if err == nil { + t.Fatal("keyword/value DSN opened, want a named store refusal") + } + found := false + for _, event := range events { + if event == "go_pg_application_name_skipped" { + found = true + } + } + if !found { + t.Fatalf("events = %v, want go_pg_application_name_skipped for the untaggable DSN", events) + } +} + +func TestPostgresGenerationLoadActivationIsExplicitCASPinned(t *testing.T) { + service := newIngest03Service(t) + metaDir := t.TempDir() + ing03WriteFixtures(t, metaDir, 2, false) + ref := ing03Ref(t, ing03Project, ing03Generation) + + service.migrate() + if activateErr := ActivatePostgresGeneration(context.Background(), ref, service.pgOptions(ing03AppBase+"-act-missing")); !errors.Is(activateErr, control.ErrManifestNotFound) { + t.Fatalf("activate missing = %v, want ErrManifestNotFound", activateErr) + } + + if _, err := LoadPostgresGeneration(context.Background(), + service.loadOptions(metaDir, ing03Project, &ref), service.pgOptions(ing03AppBase+"-act-load")); err != nil { + t.Fatalf("load: %s", service.scrub(err)) + } + if _, active := service.activePointerGeneration(ing03Project); active { + t.Fatal("successful load activated implicitly") + } + if activateErr := ActivatePostgresGeneration(context.Background(), ref, service.pgOptions(ing03AppBase+"-act-once")); activateErr != nil { + t.Fatalf("explicit activate: %s", service.scrub(activateErr)) + } + if generation, active := service.activePointerGeneration(ing03Project); !active || generation != ing03Generation { + t.Fatalf("pointer = %q active=%v, want the staged generation", generation, active) + } + if activateErr := ActivatePostgresGeneration(context.Background(), ref, service.pgOptions(ing03AppBase+"-act-twice")); activateErr != nil { + t.Fatalf("idempotent re-activate: %s", service.scrub(activateErr)) + } + + ref2 := ing03Ref(t, ing03Project, "gen-build-b") + metaDir2 := t.TempDir() + ing03WriteFixtures(t, metaDir2, 2, false) + if _, err := LoadPostgresGeneration(context.Background(), + service.loadOptions(metaDir2, ing03Project, &ref2), service.pgOptions(ing03AppBase+"-act-load-b")); err != nil { + t.Fatalf("load b: %s", service.scrub(err)) + } + if activateErr := ActivatePostgresGeneration(context.Background(), ref2, service.pgOptions(ing03AppBase+"-act-b")); activateErr != nil { + t.Fatalf("activate b: %s", service.scrub(activateErr)) + } + if generation, _ := service.activePointerGeneration(ing03Project); generation != "gen-build-b" { + t.Fatalf("pointer = %q, want gen-build-b", generation) + } + + _, reloadErr := LoadPostgresGeneration(context.Background(), + service.loadOptions(metaDir, ing03Project, &ref), service.pgOptions(ing03AppBase+"-act-reload")) + if !errors.Is(reloadErr, ErrGenerationAlreadyStaged) { + t.Fatalf("reload = %v, want %v", reloadErr, ErrGenerationAlreadyStaged) + } +} + +var ( + ing03SQLVerbRegexp = regexp.MustCompile(`(?i)\b(select|insert into|update\s+\w+\s+set|delete from|create table|drop table|truncate table|grant|revoke)\b`) + ing03LineCommentRegexp = regexp.MustCompile(`(?m)//.*$`) + ing03BlockCommentRe = regexp.MustCompile(`(?s)/\*.*?\*/`) +) + +func TestPostgresGenerationLoadOwnSourceHasNoSQL(t *testing.T) { + contents, err := os.ReadFile("generation_load.go") + if err != nil { + t.Fatalf("read orchestrator source: %v", err) + } + code := ing03BlockCommentRe.ReplaceAllString(ing03LineCommentRegexp.ReplaceAllString(string(contents), ""), "") + if loc := ing03SQLVerbRegexp.FindStringIndex(code); loc != nil { + start, end := max(0, loc[0]-20), min(len(code), loc[1]+20) + t.Fatalf("orchestrator source contains SQL verb %q; it must add no SQL of its own", code[start:end]) + } + if strings.Contains(code, "pgx.Identifier") { + t.Fatal("orchestrator must not build identifiers at all — no SQL belongs here") + } + _, _ = ing03Placeholder(t, 3), ing03OrderBy(2) +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func TestPostgresGenerationLoadZZZeroSurvivorAudit(t *testing.T) { + raw := ing03TestPostgresURL(t) + ing03CaptureBaseline(raw) + if ing03BaselineErr != nil { + t.Fatalf("baseline: %v", ing03BaselineErr) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer admin.Close() + var visible int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_ing03\\_load\\_%' ESCAPE '\\'"). + Scan(&visible); err != nil { + t.Fatalf("audit count: %v", err) + } + if visible != ing03Baseline { + t.Errorf("verify_ing03_load_* database count = %d, want baseline %d (zero own-prefix survivors)", + visible, ing03Baseline) + } + ing03OwnedMu.Lock() + defer ing03OwnedMu.Unlock() + for name, fixture := range ing03OwnedByName { + if !fixture.dropped { + t.Errorf("registered fixture %s was never dropped by its owner", name) + } + } +} + +var errInjectedCatalogFault = errors.New("ing03 test: injected catalog fault") + +type injectedFieldProfileStore struct { + inner control.FieldProfileStore + failAfter int + calls int +} + +func (f *injectedFieldProfileStore) UpsertFieldProfileBatch(ctx context.Context, scope control.FieldProfileScope, docs []catalog.FieldCatalogDocument, coverage control.FieldProfileCoverage, batchSize int) error { + f.calls++ + if f.calls > f.failAfter { + return fmt.Errorf("%w: field profile upsert for %s", errInjectedCatalogFault, scope.ResourceType) + } + return f.inner.UpsertFieldProfileBatch(ctx, scope, docs, coverage, batchSize) +} + +func (f *injectedFieldProfileStore) ReadFieldProfileRows(ctx context.Context, scope control.FieldProfileScope) ([]catalog.FieldCatalogDocument, error) { + return f.inner.ReadFieldProfileRows(ctx, scope) +} + +func (f *injectedFieldProfileStore) ReadFieldProfileRowsFiltered(ctx context.Context, filter control.FieldProfileFilter) ([]catalog.FieldCatalogDocument, error) { + return f.inner.ReadFieldProfileRowsFiltered(ctx, filter) +} + +func (f *injectedFieldProfileStore) ReadProfileCoverage(ctx context.Context, filter control.FieldProfileFilter) ([]control.FieldProfileCoverage, error) { + return f.inner.ReadProfileCoverage(ctx, filter) +} + +func (f *injectedFieldProfileStore) ListDistinctAuthResourcePaths(ctx context.Context, project string) ([]string, error) { + return f.inner.ListDistinctAuthResourcePaths(ctx, project) +} + +type injectedInventoryStore struct { + inner control.InventoryStore + failOnCall int + calls int +} + +func (i *injectedInventoryStore) ScanSourceCounts(ctx context.Context, project, datasetGeneration, resourceType string) ([]control.InventorySourceGroup, error) { + i.calls++ + if i.calls == i.failOnCall { + return nil, fmt.Errorf("%w: scan counts for %s", errInjectedCatalogFault, resourceType) + } + return i.inner.ScanSourceCounts(ctx, project, datasetGeneration, resourceType) +} + +func (i *injectedInventoryStore) InsertCounts(ctx context.Context, rows []catalog.ResourceInventoryCount, batchSize int) error { + return i.inner.InsertCounts(ctx, rows, batchSize) +} + +func (i *injectedInventoryStore) PublishBuild(ctx context.Context, build catalog.ResourceInventoryBuild) error { + return i.inner.PublishBuild(ctx, build) +} + +func (i *injectedInventoryStore) ReadBuild(ctx context.Context, project, datasetGeneration string) (catalog.ResourceInventoryBuild, bool, error) { + return i.inner.ReadBuild(ctx, project, datasetGeneration) +} + +func (i *injectedInventoryStore) ReadCountSums(ctx context.Context, scope control.InventoryCountScope) (map[string]int64, error) { + return i.inner.ReadCountSums(ctx, scope) +} + +type injectedRelationshipStore struct { + inner control.RelationshipStore + failNext bool +} + +func (r *injectedRelationshipStore) ScanRelationshipOccurrences(ctx context.Context, scope control.RelationshipScope) ([]control.RelationshipOccurrenceGroup, error) { + if r.failNext { + return nil, fmt.Errorf("%w: relationship scan", errInjectedCatalogFault) + } + return r.inner.ScanRelationshipOccurrences(ctx, scope) +} + +type injectedContributionWriter struct { + inner GenerationSemanticContributor + failNext bool +} + +func (w *injectedContributionWriter) WriteSemanticInventoryContributions(ctx context.Context, contributions []catalog.SemanticInventoryContribution, batchSize int) error { + if w.failNext { + return fmt.Errorf("%w: contribution write", errInjectedCatalogFault) + } + return w.inner.WriteSemanticInventoryContributions(ctx, contributions, batchSize) +} + +func (s *ing03Service) installCatalogFault(t *testing.T, seam string) { + t.Helper() + switch seam { + case "field-profile": + s.faults = &ing03CatalogFaults{ + wrapFieldProfile: func(inner control.FieldProfileStore) control.FieldProfileStore { + return &injectedFieldProfileStore{inner: inner, failAfter: 1} + }, + } + case "inventory": + s.faults = &ing03CatalogFaults{ + wrapInventory: func(inner control.InventoryStore) control.InventoryStore { + return &injectedInventoryStore{inner: inner, failOnCall: 1} + }, + } + case "relationship": + s.faults = &ing03CatalogFaults{ + wrapRelationship: func(inner control.RelationshipStore) control.RelationshipStore { + return &injectedRelationshipStore{inner: inner, failNext: true} + }, + } + case "semantic-contributions": + s.faults = &ing03CatalogFaults{ + wrapContributions: func(inner GenerationSemanticContributor) GenerationSemanticContributor { + return &injectedContributionWriter{inner: inner, failNext: true} + }, + } + default: + t.Fatalf("unknown seam %q", seam) + } +} + +func (s *ing03Service) resourcePayload(resourceType, logicalID string) string { + s.t.Helper() + var payload string + err := s.iso.QueryRow(context.Background(), + "SELECT payload::text FROM "+ing03QIdent(s.t, s.schema, "fhir_resource")+ + " WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3 AND logical_id = $4", + ing03Project, ing03Generation, resourceType, logicalID).Scan(&payload) + if err != nil { + s.t.Fatalf("read payload %s/%s: %s", resourceType, logicalID, s.scrub(err)) + } + return payload +} + +type ing03EdgeSeed struct { + Label, SourceType, SourceID, TargetType, TargetID, SourcePath string + SourceOrdinal int64 +} + +func (s *ing03Service) readOneEdge() ing03EdgeSeed { + s.t.Helper() + var seed ing03EdgeSeed + err := s.iso.QueryRow(context.Background(), + "SELECT label, source_resource_type, source_logical_id, target_resource_type, target_logical_id, source_path, source_ordinal FROM "+ + ing03QIdent(s.t, s.schema, "fhir_edge_occurrence")+ing03OrderBy(7)+" LIMIT 1"). + Scan(&seed.Label, &seed.SourceType, &seed.SourceID, &seed.TargetType, &seed.TargetID, &seed.SourcePath, &seed.SourceOrdinal) + if err != nil { + s.t.Fatalf("read seed edge: %s", s.scrub(err)) + } + return seed +} + +func (e ing03EdgeSeed) row() control.EdgeOccurrenceRow { + return control.EdgeOccurrenceRow{ + Project: ing03Project, DatasetGeneration: ing03Generation, AuthResourcePath: ing03AuthPath, + Label: e.Label, SourceResourceType: e.SourceType, SourceLogicalID: e.SourceID, + TargetResourceType: e.TargetType, TargetLogicalID: e.TargetID, + SourcePath: e.SourcePath, SourceOrdinal: e.SourceOrdinal, + } +} + +func (s *ing03Service) sourceSeed(e ing03EdgeSeed) source.Resource { + return source.Resource{ + Project: ing03Project, DatasetGeneration: ing03Generation, + ResourceType: e.SourceType, LogicalID: e.SourceID, AuthResourcePath: ing03AuthPath, + Payload: json.RawMessage(s.resourcePayload(e.SourceType, e.SourceID)), + } +} + +func TestPostgresShippedLoadConfigsValidateOffline(t *testing.T) { + shipped := PGSourceOptions{ + DSN: "postgresql://loom_test@127.0.0.1:1/loom_test?sslmode=disable", + Schema: "verify_ab06_offline_src", + ManifestSchema: "verify_ab06_offline_manifests", + } + if err := PGLoadConfig(shipped, shipped.DSN, control.SetupNone).Validate(); err != nil { + t.Fatalf("shipped composition must validate dial-free, got: %v", err) + } + if err := PGLoadConfig(shipped, shipped.DSN, control.SetupMigrate).Validate(); err != nil { + t.Fatalf("migrate-policy composition must validate: %v", err) + } + + cases := []struct { + name string + mutate func(*PGSourceOptions) + policy control.SetupPolicy + wantErr error + }{ + { + name: "manifest schema may not equal the source schema", + mutate: func(o *PGSourceOptions) { o.ManifestSchema = o.Schema }, + policy: control.SetupNone, + wantErr: control.ErrSeriesAssignment, + }, + { + name: "manifest schema must be named", + mutate: func(o *PGSourceOptions) { o.ManifestSchema = " " }, + policy: control.SetupNone, + wantErr: control.ErrSeriesAssignment, + }, + { + name: "source schema must be named", + mutate: func(o *PGSourceOptions) { o.Schema = "" }, + policy: control.SetupNone, + wantErr: control.ErrSeriesAssignment, + }, + { + name: "the removed verify policy stays refused", + mutate: func(*PGSourceOptions) {}, + policy: control.SetupPolicy("verify"), + wantErr: control.ErrInvalidConfig, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + opts := shipped + tc.mutate(&opts) + err := PGLoadConfig(opts, opts.DSN, tc.policy).Validate() + if !errors.Is(err, tc.wantErr) { + t.Fatalf("err = %v, want %v", err, tc.wantErr) + } + if err := PGLoadConfig(shipped, shipped.DSN, control.SetupNone).Validate(); err != nil { + t.Fatalf("alive control (unmutated shipped config) must validate: %v", err) + } + }) + } + + for _, file := range []struct { + path string + allowed int + }{ + {path: "generation_load.go", allowed: 1}, + {path: "postgres_writer.go", allowed: 0}, + } { + raw, err := os.ReadFile(filepath.Join(".", file.path)) + if err != nil { + t.Fatalf("read %s: %v", file.path, err) + } + if got := strings.Count(string(raw), "control.Config{"); got > file.allowed { + t.Fatalf("%s builds control.Config{ %d times, want at most %d (all shipped configs come from PGLoadConfig)", file.path, got, file.allowed) + } + } +} diff --git a/internal/ingest/generation_load_test.go b/internal/ingest/generation_load_test.go index 43b6c7036..ffb0adf72 100644 --- a/internal/ingest/generation_load_test.go +++ b/internal/ingest/generation_load_test.go @@ -13,9 +13,9 @@ import ( "time" "github.com/bmeg/jsonschemagraph/graph" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/catalog" publication "github.com/calypr/loom/internal/dataset" - arangostore "github.com/calypr/loom/internal/store/arango" ) func TestNewGenerationLoadPlanRejectsUnsafeOrIncompleteInputs(t *testing.T) { @@ -115,8 +115,8 @@ func TestGenerationLoadPreflightRunsBeforeOptionRejectionOrBackend(t *testing.T) MetaDir: dir, Project: ref.Project, Dataset: &ref, - Truncate: true, // would be rejected only after preflight succeeds. - ConnectionOptions: arangostore.ConnectionOptions{ + Truncate: true, + ConnectionOptions: arangofactory.ConnectionOptions{ URL: "http://127.0.0.1:1", Database: "generation_preflight_must_not_connect", }, @@ -150,7 +150,7 @@ func TestGenerationLoadEmptyDirectoryDoesNotOpenBackend(t *testing.T) { MetaDir: dir, Project: ref.Project, Dataset: &ref, - ConnectionOptions: arangostore.ConnectionOptions{ + ConnectionOptions: arangofactory.ConnectionOptions{ URL: "http://127.0.0.1:1", Database: "empty_generation_must_not_connect", }, @@ -231,9 +231,6 @@ func TestSelectGenerationRowErrorSamplesCoversEveryFailingFile(t *testing.T) { if got := len(samples); got != rowErrorSampleLimit { t.Fatalf("samples = %d, want the full budget of %d", got, rowErrorSampleLimit) } - // Round-robin by file order fills each smaller file completely before the - // largest file consumes the leftover budget, and the summary reports the - // picks in file-then-line order. want := []string{ "Condition.ndjson:1", "Condition.ndjson:2", "Condition.ndjson:3", "Condition.ndjson:4", "Condition.ndjson:5", @@ -271,7 +268,6 @@ func TestSelectGenerationRowErrorSamplesIsBoundedAndDeterministic(t *testing.T) if len(samples) != rowErrorSampleLimit { t.Fatalf("samples = %d, want the capped budget of %d from %d failing files", len(samples), rowErrorSampleLimit, failingFileCount) } - // More failing files than budget: the earliest files in discovery order win. for i, sample := range samples { if want := fmt.Sprintf("Patient-%02d.ndjson", i); sample.File != want { t.Fatalf("sample[%d] file = %q, want %q (selection = %v)", i, sample.File, want, generationRowErrorKeys(samples)) @@ -349,7 +345,6 @@ func TestGenerationRowErrorSamplesCoverEveryFailingStagedFile(t *testing.T) { } dir := t.TempDir() invalid := "{\n" - // Sorted discovery order: Condition, Medication, Observation, Patient. writePreflightFixture(t, dir, "Condition.ndjson", strings.Repeat(invalid, 12)) writePreflightFixture(t, dir, "Medication.ndjson", invalid+invalid+invalid) writePreflightFixture(t, dir, "Observation.ndjson", invalid+invalid) @@ -364,7 +359,9 @@ func TestGenerationRowErrorSamplesCoverEveryFailingStagedFile(t *testing.T) { } opts := normalizeLoadOptions(LoadOptions{Project: "project-a", WriterCount: 2, WorkerCount: 2}) - insert := func(context.Context, *arangostore.Client, string, []json.RawMessage, bool, string) error { return nil } + insert := func(context.Context, *arangofactory.Client, string, []json.RawMessage, bool, string) error { + return nil + } var totals fileLoadResult var candidates [][]RowErrorSample @@ -380,7 +377,6 @@ func TestGenerationRowErrorSamplesCoverEveryFailingStagedFile(t *testing.T) { candidates = append(candidates, result.RowErrors) } - // Aggregate counts are exact and independent of how samples are selected. if totals.ValidationErrors != 17 || totals.GenerationErrors != 0 || totals.EdgeErrors != 0 { t.Fatalf("totals = %+v, want 17 validation errors and no generation or edge errors", totals) } @@ -408,8 +404,6 @@ func TestGenerationRowErrorSamplesCoverEveryFailingStagedFile(t *testing.T) { if _, clean := perFile["Patient.ndjson"]; clean { t.Fatalf("clean file Patient.ndjson produced samples: %v", generationRowErrorKeys(samples)) } - // 12 + 3 + 2 invalid rows compete for ten slots: each failing file keeps its - // two earliest lines before Condition takes the remaining slots. if got, want := generationRowErrorKeys(samples), []string{ "Condition.ndjson:1", "Condition.ndjson:2", "Condition.ndjson:3", "Condition.ndjson:4", "Condition.ndjson:5", "Medication.ndjson:1", "Medication.ndjson:2", "Medication.ndjson:3", @@ -417,7 +411,6 @@ func TestGenerationRowErrorSamplesCoverEveryFailingStagedFile(t *testing.T) { }; !reflect.DeepEqual(got, want) { t.Fatalf("samples = %v, want %v", got, want) } - // Every sample reproduces a real invalid line from its own staged file. for _, sample := range samples { contents, readErr := os.ReadFile(filepath.Join(dir, sample.File)) if readErr != nil { @@ -444,7 +437,9 @@ func TestGenerationRowErrorSamplesLeaveCleanLoadWithoutSamples(t *testing.T) { t.Fatal(err) } opts := normalizeLoadOptions(LoadOptions{Project: "project-a", WriterCount: 2, WorkerCount: 2}) - insert := func(context.Context, *arangostore.Client, string, []json.RawMessage, bool, string) error { return nil } + insert := func(context.Context, *arangofactory.Client, string, []json.RawMessage, bool, string) error { + return nil + } var candidates [][]RowErrorSample var totals fileLoadResult diff --git a/internal/ingest/generation_orchestration.go b/internal/ingest/generation_orchestration.go index af7355746..c623dca8d 100644 --- a/internal/ingest/generation_orchestration.go +++ b/internal/ingest/generation_orchestration.go @@ -7,16 +7,13 @@ import ( "path/filepath" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/catalog" - catalogarango "github.com/calypr/loom/internal/catalog/arango" publication "github.com/calypr/loom/internal/dataset" - publicationarango "github.com/calypr/loom/internal/dataset/arango" "github.com/bmeg/jsonschemagraph/graph" ) -// Load selects the primary unversioned resource loader unless an immutable -// dataset reference is supplied explicitly. func Load(ctx context.Context, opts LoadOptions) (LoadSummary, error) { if opts.Dataset == nil { return loadResource(ctx, opts) @@ -34,8 +31,6 @@ func totalResourceRows(resources map[string]int) int64 { return total } -// Cleanup actions receive independent deadlines. A stalled manifest transition -// cannot consume the close deadline, so failure cleanup may use two windows. func boundedGenerationCleanup(parent context.Context, timeout time.Duration, cleanup func(context.Context) error) error { cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(parent), timeout) defer cancel() @@ -65,9 +60,6 @@ func loadGeneration(ctx context.Context, opts LoadOptions) (summary LoadSummary, StageSeconds: map[string]float64{}, } - // Keep the existing ordered evidence path: discover staged files, parse the - // graph with its authoritative loader, capture identity from the same file, - // then preflight every staged payload before contacting Arango. schema, err := graph.Load(opts.Schema) if err != nil { return summary, err @@ -105,8 +97,6 @@ func loadGeneration(ctx context.Context, opts LoadOptions) (summary LoadSummary, if err != nil { return summary, err } - // Load dispatch guarantees a plan, but retaining this check makes the - // generation-only helper safe if it is called directly in a future test. if plan == nil { return summary, fmt.Errorf("dataset generation plan is required") } @@ -127,14 +117,15 @@ func loadGeneration(ctx context.Context, opts LoadOptions) (summary LoadSummary, "database": opts.Database, }) connectStart := time.Now() - client, err := openBackend(ctx, opts.ConnectionOptions) + backend, err := openBackend(ctx, opts.ConnectionOptions) if err != nil { return summary, err } + client := backend.Client defer func() { - err = generationCleanupError(ctx, generationCleanupTimeout, err, "close generation backend", client.Close) + err = generationCleanupError(ctx, generationCleanupTimeout, err, "close generation backend", backend.Close) }() - catalogStore, err := catalogarango.New(client) + catalogStore, err := arangofactory.NewCatalogStore(client) if err != nil { return summary, err } @@ -146,8 +137,6 @@ func loadGeneration(ctx context.Context, opts LoadOptions) (summary LoadSummary, }) bootstrapStart := time.Now() - // Metadata comes first and never truncates. This order means a failed - // graph bootstrap cannot delete lifecycle history from a prior generation. emitEvent(opts.EventSink, "go_bootstrap_start", map[string]any{ "database": opts.Database, "resources": 0, @@ -167,7 +156,7 @@ func loadGeneration(ctx context.Context, opts LoadOptions) (summary LoadSummary, } summary.StageSeconds["bootstrap"] = time.Since(bootstrapStart).Seconds() - lifecycleStore, err := publicationarango.New(client) + lifecycleStore, err := arangofactory.NewDatasetStore(client) if err != nil { return summary, err } @@ -180,9 +169,6 @@ func loadGeneration(ctx context.Context, opts LoadOptions) (summary LoadSummary, if err == nil || manifestStaged { return } - // Once STAGED is persisted we deliberately leave it alone, - // because an activation error is an unknown outcome rather than proof - // that the generation failed. err = generationCleanupError(ctx, generationCleanupTimeout, err, fmt.Sprintf("mark dataset generation %s/%s failed", plan.Dataset.Project, plan.Dataset.Generation), func(cleanupCtx context.Context) error { _, err := lifecycleStore.TransitionManifest(cleanupCtx, manifest, publication.StateFailed) return err @@ -205,9 +191,6 @@ func loadGeneration(ctx context.Context, opts LoadOptions) (summary LoadSummary, catalogs := make(map[generationCatalogKey]*catalog.Profiler) relationshipCounts := make(map[catalog.RelationshipKey]int64) lastInventoryCheckpoint := "" - // Row-error candidates are kept per file and only sampled into the summary - // when loadGeneration returns, so every staged file gets a chance at the - // shared sample budget. The deferred assignment covers the early returns. var rowErrorCandidates [][]RowErrorSample defer func() { summary.RowErrors = selectGenerationRowErrorSamples(rowErrorCandidates) diff --git a/internal/ingest/generation_workers.go b/internal/ingest/generation_workers.go index 5b9ffafe0..f59271d6e 100644 --- a/internal/ingest/generation_workers.go +++ b/internal/ingest/generation_workers.go @@ -13,8 +13,8 @@ import ( "sync/atomic" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/catalog" - arangostore "github.com/calypr/loom/internal/store/arango" "github.com/bmeg/jsonschemagraph/graph" ) @@ -67,15 +67,6 @@ type fileLoadResult struct { const rowErrorSampleLimit = 10 -// selectGenerationRowErrorSamples picks the bounded row-error samples a failed -// generation load reports to operators. fileSamples holds one candidate list per -// staged file, in the sorted discovery order DiscoverNDJSON returns, and each list -// is already ordered by line. Rank zero takes the earliest sample from every failing -// file before any later file can be dropped, so a file with many bad rows cannot -// starve the samples of the files after it; later ranks round-robin the remaining -// budget in the same file order. When more files fail than the cap permits, the -// earliest files in discovery order keep the whole budget. Duplicate samples are -// ignored, so the result never exceeds the cap and keeps file-then-line order. func selectGenerationRowErrorSamples(fileSamples [][]RowErrorSample) []RowErrorSample { failing := make([][]RowErrorSample, 0, len(fileSamples)) longest := 0 @@ -124,8 +115,6 @@ func flattenRowErrorSamples(picked [][]RowErrorSample) []RowErrorSample { for _, fileSamples := range picked { samples = append(samples, fileSamples...) } - // Two staged files can share a base name in different subdirectories, so the - // group order alone does not guarantee the reported order. sort.SliceStable(samples, func(i, j int) bool { if samples[i].File != samples[j].File { return samples[i].File < samples[j].File @@ -147,15 +136,12 @@ type fileWriteTask struct { replace bool } -type documentInserter func(context.Context, *arangostore.Client, string, []json.RawMessage, bool, string) error +type documentInserter func(context.Context, *arangofactory.Client, string, []json.RawMessage, bool, string) error -// loadFile owns one scanner and closes it before returning. A blank generation -// preserves the primary resource-loader identity; overwrite controls its -// upsert contract while immutable generations always pass false. func loadFile( ctx context.Context, opts LoadOptions, - client *arangostore.Client, + client *arangofactory.Client, schema *graph.GraphSchema, file string, datasetGeneration string, diff --git a/internal/ingest/integration_test.go b/internal/ingest/integration_test.go index 9b6f47fc4..d26ffbe56 100644 --- a/internal/ingest/integration_test.go +++ b/internal/ingest/integration_test.go @@ -10,12 +10,11 @@ import ( "time" "github.com/calypr/loom/internal/catalog" - catalogarango "github.com/calypr/loom/internal/catalog/arango" - arangostore "github.com/calypr/loom/internal/store/arango" "github.com/bmeg/jsonschemagraph/graph" "github.com/bmeg/jsonschemagraph/util" "github.com/bytedance/sonic" + arangofactory "github.com/calypr/loom/internal/backends/arango" publication "github.com/calypr/loom/internal/dataset" ) @@ -67,7 +66,7 @@ func TestLoadAndQueryFixture(t *testing.T) { } database := "fhir_proto_int_" + strings.ToLower(name) + "_" + time.Now().Format("20060102150405") loadSummary, err := Load(ctx, LoadOptions{ - ConnectionOptions: arangostore.ConnectionOptions{ + ConnectionOptions: arangofactory.ConnectionOptions{ URL: "http://127.0.0.1:8529", Database: database, }, @@ -94,12 +93,13 @@ func TestLoadAndQueryFixture(t *testing.T) { } } - client, err := arangostore.Open(ctx, "http://127.0.0.1:8529", database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: "http://127.0.0.1:8529", Database: database}) if err != nil { t.Fatalf("open catalog client: %v", err) } + client := arangoBackend.Client defer client.Close(ctx) - catalogStore, err := catalogarango.New(client) + catalogStore, err := arangofactory.NewCatalogStore(client) if err != nil { t.Fatalf("create catalog store: %v", err) } diff --git a/internal/ingest/keycollide_generated_e2e_test.go b/internal/ingest/keycollide_generated_e2e_test.go index 49d53b7ab..3815ef08b 100644 --- a/internal/ingest/keycollide_generated_e2e_test.go +++ b/internal/ingest/keycollide_generated_e2e_test.go @@ -1,32 +1,5 @@ package ingest -// F-B25 (hypothesis probe): internal/ingest/documents.go SanitizeKey (:107-113) -// collapses every disallowed char to `_` via validKeyPart (:17) — non-injective. -// The vertex insert path posts with onDuplicate=replace -// (internal/store/arango/client.go:450-451,494-495 region) and the summary -// counter adds len(task.docs) SENT, not persisted -// (internal/ingest/generation_workers.go:426 region). If two DIFFERENT FHIR -// resource ids sanitize onto one `_key`, the second write silently replaces -// the first while LoadSummary still reports both as VerticesInserted — a lost -// acknowledged write. -// -// This artifact drives the REAL production Load() entry point on a fresh -// verification Arango database, per builder path (UseGeneric false and true — -// spelling matches LoadOptions field), and compares the acknowledged counter -// against the real collection count: -// acknowledged == collected -> the path prevents the collision -// (typically via id charset validation; the RowErrors/ValidationErrors -// samples logged per path classify WHICH prevention fired). -// acknowledged > collected -> CONFIRMED silent replacement on -// that path. -// -// Collision pair: "coll/1" and "coll_1" — both non-blank strings; SanitizeKey -// maps '/' -> '_', so both land on _key "coll_1" while the JSON payload keeps -// the different ids. Control id "x9" must survive next to them. -// -// Cleanup: every created database is dropped via the registered first (runs -// last, LIFO) helper; final DB list is asserted to exclude the prefix. - import ( "context" "encoding/json" @@ -39,8 +12,8 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" publication "github.com/calypr/loom/internal/dataset" - arangostore "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -81,8 +54,6 @@ func TestFB25SanitizeKeyCollisionLosesAcknowledgedWrite(t *testing.T) { database := kcollPrefix + strings.ReplaceAll(uuid.NewString(), "-", "") ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) defer cancel() - // Registered FIRST => runs LAST: drop the verification database - // after all sub-assertions, then nothing else holds it. t.Cleanup(func() { kcollDropDatabase(t, endpoint, database) }) metaDir := t.TempDir() @@ -95,7 +66,7 @@ func TestFB25SanitizeKeyCollisionLosesAcknowledgedWrite(t *testing.T) { t.Fatalf("ref: %v", err) } summary, err := Load(ctx, LoadOptions{ - ConnectionOptions: arangostore.ConnectionOptions{URL: endpoint, Database: database}, + ConnectionOptions: arangofactory.ConnectionOptions{URL: endpoint, Database: database}, Schema: repoPath(t, "schemas", "graph-fhir.json"), MetaDir: metaDir, Project: "KCOLL_TEST", @@ -114,10 +85,11 @@ func TestFB25SanitizeKeyCollisionLosesAcknowledgedWrite(t *testing.T) { t.Logf("FB25 %s | LOAD_SUMMARY acked=%d validation_errors=%d generation_errors=%d row_errors=%s", label, summary.VerticesInserted, summary.ValidationErrors, summary.GenerationErrors, samples) - client, err := arangostore.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatalf("open: %v", err) } + client := arangoBackend.Client defer func() { _ = client.Close(context.Background()) }() exists, err := client.CollectionExists(ctx, "Patient") if err != nil { @@ -140,9 +112,6 @@ func TestFB25SanitizeKeyCollisionLosesAcknowledgedWrite(t *testing.T) { case summary.VerticesInserted == 0 && (summary.ValidationErrors+summary.GenerationErrors) > 0: t.Errorf("FB25 %s unexpected total rejection (all %d rows errored): row_errors=%s — probe premise about this path's validation strength lost; classify separately", label, len(lines), samples) default: - // Partial rejection: validation caught the disallowed id on - // this path. Honest PREVENTED via validation — but only if the - // SURVIVING pair still round-trips with x9 absent collisions. if collected == int64(summary.VerticesInserted) { t.Logf("FB25 %s verdict | PREVENTED by validation on this path | acknowledged=%d persisted=%d, %d row(s) rejected before write (see row_errors above)", label, summary.VerticesInserted, collected, summary.ValidationErrors+summary.GenerationErrors) } else { @@ -151,7 +120,6 @@ func TestFB25SanitizeKeyCollisionLosesAcknowledgedWrite(t *testing.T) { } }) } - // Residue: no verification databases may survive. t.Cleanup(func() { kcollAssertNoResidue(t, endpoint) }) _ = fmt.Sprint() } @@ -164,11 +132,12 @@ func kcollDropDatabase(t *testing.T, endpoint, database string) { } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - client, err := arangostore.Open(ctx, endpoint, "_system") + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: "_system"}) if err != nil { t.Logf("FB25_CLEANUP_WARN: open _system: %v", err) return } + client := arangoBackend.Client defer func() { _ = client.Close(context.Background()) }() request, err := http.NewRequestWithContext(ctx, http.MethodDelete, strings.TrimRight(endpoint, "/")+"/_api/database/"+url.PathEscape(database), nil) diff --git a/internal/ingest/load_options.go b/internal/ingest/load_options.go index 81471e9df..bb02e1949 100644 --- a/internal/ingest/load_options.go +++ b/internal/ingest/load_options.go @@ -7,40 +7,29 @@ import ( "github.com/calypr/loom/internal/catalog" publication "github.com/calypr/loom/internal/dataset" - arangostore "github.com/calypr/loom/internal/store/arango" ) type LoadOptions struct { - arangostore.ConnectionOptions - Schema string - MetaDir string - Project string - AuthResourcePath string - BatchSize int - ProgressEvery int - WriterCount int - WorkerCount int - LineQueueSize int - WriteQueueSize int - CatalogLimits catalog.ProfileLimits - Truncate bool - FailFast bool - UseGeneric bool - WriteAPI string - EventSink EventSink - // Dataset identifies the complete immutable generation being loaded. The - // generation activates only after every graph file and catalog finalization - // succeeds. - Dataset *publication.Ref - // StageOnly leaves a successfully loaded immutable generation STAGED. It is - // selected for reads only after project-release activation succeeds. - StageOnly bool - // DeferActivation is the compatibility spelling used by the generation-load - // HTTP API. Like StageOnly, it leaves the generation STAGED and inactive. - DeferActivation bool - // PreflightSampleRows bounds the number of payloads inspected from every - // staged file before Loom opens or mutates Arango. Zero uses the safe - // default; full row validation still happens in the loader. + ConnectionOptions + Schema string + MetaDir string + Project string + AuthResourcePath string + BatchSize int + ProgressEvery int + WriterCount int + WorkerCount int + LineQueueSize int + WriteQueueSize int + CatalogLimits catalog.ProfileLimits + Truncate bool + FailFast bool + UseGeneric bool + WriteAPI string + EventSink EventSink + Dataset *publication.Ref + StageOnly bool + DeferActivation bool PreflightSampleRows int } @@ -53,31 +42,21 @@ type RowErrorSample struct { } type LoadSummary struct { - Files int `json:"files"` - VerticesInserted int `json:"vertices_inserted"` - EdgesInserted int `json:"edges_inserted"` - ValidationErrors int `json:"validation_errors"` - GenerationErrors int `json:"generation_errors"` - EdgeErrors int `json:"edge_errors"` - BatchCounts map[string]int `json:"batch_counts,omitempty"` - Resources map[string]int `json:"resources"` - StageSeconds map[string]float64 `json:"stage_seconds"` - Preflight PreflightReport `json:"preflight"` - RowErrors []RowErrorSample `json:"row_errors,omitempty"` - // SchemaIdentity is the exact configured graph-schema evidence used for - // this load. It remains nil when Loom cannot load the configured schema, so - // an early failure never looks like a successful schema observation. - SchemaIdentity *publication.SchemaSnapshot `json:"schema_identity,omitempty"` - // Dataset is the immutable target when this was a generation load. It is - // present even on a failed generation load so callers can identify the - // inactive manifest that needs operational inspection. - Dataset *publication.Ref `json:"dataset,omitempty"` + Files int `json:"files"` + VerticesInserted int `json:"vertices_inserted"` + EdgesInserted int `json:"edges_inserted"` + ValidationErrors int `json:"validation_errors"` + GenerationErrors int `json:"generation_errors"` + EdgeErrors int `json:"edge_errors"` + BatchCounts map[string]int `json:"batch_counts,omitempty"` + Resources map[string]int `json:"resources"` + StageSeconds map[string]float64 `json:"stage_seconds"` + Preflight PreflightReport `json:"preflight"` + RowErrors []RowErrorSample `json:"row_errors,omitempty"` + SchemaIdentity *publication.SchemaSnapshot `json:"schema_identity,omitempty"` + Dataset *publication.Ref `json:"dataset,omitempty"` } -// normalizeLoadOptions applies the operational defaults shared by both loader -// modes. The immutable-generation loader deliberately has a different write -// lifecycle, but both modes must agree on batching, progress, concurrency, and -// the Arango write API before any input is examined. func normalizeLoadOptions(opts LoadOptions) LoadOptions { if opts.BatchSize <= 0 { opts.BatchSize = 5000 @@ -105,26 +84,14 @@ func normalizeLoadOptions(opts LoadOptions) LoadOptions { } var ( - // ErrGenerationLoadRequiresDirectory prevents a one-file or arbitrary-file - // load from being mistaken for a complete immutable dataset snapshot. - ErrGenerationLoadRequiresDirectory = errors.New("dataset generation load requires a directory") - // ErrGenerationLoadRequiresFiles prevents an empty staged directory from - // becoming an active but meaningless generation. - ErrGenerationLoadRequiresFiles = errors.New("dataset generation load requires at least one NDJSON file") - // ErrGenerationLoadTruncateForbidden prevents a new generation from - // deleting active or historical graph data. - ErrGenerationLoadTruncateForbidden = errors.New("dataset generation load cannot truncate collections") - // ErrGenerationDatasetProjectMismatch prevents graph documents and their - // lifecycle manifest from being scoped to different projects. + ErrGenerationLoadRequiresDirectory = errors.New("dataset generation load requires a directory") + ErrGenerationLoadRequiresFiles = errors.New("dataset generation load requires at least one NDJSON file") + ErrGenerationLoadTruncateForbidden = errors.New("dataset generation load cannot truncate collections") ErrGenerationDatasetProjectMismatch = errors.New("dataset generation project does not match load project") ErrGenerationSingleResourceUnsupported = errors.New("single-resource imports cannot create a dataset generation") ErrGenerationDatasetRequired = errors.New("dataset generation is required") ) -// ActivationOutcomeError means the generation reached STAGED but Loom could -// not prove that the legacy active-generation pointer was updated. STAGED is kept for -// an operator to reconcile; it must never be downgraded to FAILED because the -// activation request may have committed before its error reached the caller. type ActivationOutcomeError struct { Dataset publication.Ref Err error @@ -144,10 +111,6 @@ func (e *ActivationOutcomeError) Unwrap() error { return e.Err } -// GenerationLoadIncompleteError means row-level validation, generation, or -// edge construction failures were observed while FailFast was disabled. The -// graph may contain partial immutable documents, but the lifecycle manifest is -// deliberately left FAILED and cannot be selected for reads. type GenerationLoadIncompleteError struct { Dataset publication.Ref ValidationErrors int @@ -174,8 +137,6 @@ type generationLoadPlan struct { Manifest publication.Manifest } -// newGenerationLoadPlan validates and snapshots all immutable information -// after input preflight and before a database connection is opened. func newGenerationLoadPlan(opts LoadOptions, files []string, schemaSnapshot publication.SchemaSnapshot) (*generationLoadPlan, error) { if opts.Dataset == nil { return nil, ErrGenerationDatasetRequired diff --git a/internal/ingest/postgres_writer.go b/internal/ingest/postgres_writer.go new file mode 100644 index 000000000..e6c82bb40 --- /dev/null +++ b/internal/ingest/postgres_writer.go @@ -0,0 +1,921 @@ +package ingest + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "sort" + "strings" + + "github.com/calypr/loom/internal/backends/postgres/control" + fhirschema "github.com/calypr/loom/internal/fhir/schema" + "github.com/calypr/loom/internal/source" +) + +var ErrInvalidPGResourceWrite = errors.New("ingest: invalid postgres resource write") + +type PGResourceWriteScopeError struct { + BatchID string + Field string + Reason string +} + +func (e *PGResourceWriteScopeError) Error() string { + return fmt.Sprintf("postgres resource write refused: batch %q has an invalid %s: %s; "+ + "nothing was written", e.BatchID, e.Field, e.Reason) +} + +func (e *PGResourceWriteScopeError) Unwrap() error { return ErrInvalidPGResourceWrite } + +type PGResourceRowIdentityError struct { + BatchID string + Row int + Column string + Reason string +} + +func (e *PGResourceRowIdentityError) Error() string { + return fmt.Sprintf("postgres resource write refused: batch %q row %d has an invalid %s: %s; "+ + "nothing was written", e.BatchID, e.Row, e.Column, e.Reason) +} + +func (e *PGResourceRowIdentityError) Unwrap() error { return ErrInvalidPGResourceWrite } + +type PGResourceBatchScope struct { + BatchID string + Project string + DatasetGeneration string + AuthResourcePath string +} + +type PGResourceWriteResult struct { + BatchID string + Digest string + Offered int + Committed int64 + SkippedIdentical int64 + Replay bool +} + +type resourceBatchCommitter interface { + WriteResourceBatch(ctx context.Context, batch control.ResourceBatch) (control.ResourceBatchResult, error) +} + +type PostgresResourceWriter struct { + committer resourceBatchCommitter +} + +func NewPostgresResourceWriter(src control.ConnSource, schema string) (*PostgresResourceWriter, error) { + writer, err := control.NewResourceWriter(src, schema) + if err != nil { + return nil, err + } + return &PostgresResourceWriter{committer: writer}, nil +} + +func (w *PostgresResourceWriter) WriteResourceBatch( + ctx context.Context, + scope PGResourceBatchScope, + rows []rowBuildResult, +) (PGResourceWriteResult, error) { + if strings.TrimSpace(scope.BatchID) == "" { + return PGResourceWriteResult{}, &PGResourceWriteScopeError{ + BatchID: scope.BatchID, Field: "batch id", Reason: "it is blank", + } + } + if len(scope.BatchID) > control.MaxBatchIDLength { + return PGResourceWriteResult{}, &PGResourceWriteScopeError{ + BatchID: scope.BatchID, Field: "batch id", + Reason: fmt.Sprintf("it is %d bytes, want at most %d", len(scope.BatchID), control.MaxBatchIDLength), + } + } + if strings.TrimSpace(scope.Project) == "" { + return PGResourceWriteResult{}, &PGResourceWriteScopeError{ + BatchID: scope.BatchID, Field: "project", Reason: "it is blank", + } + } + if len(rows) == 0 { + return PGResourceWriteResult{}, &PGResourceWriteScopeError{ + BatchID: scope.BatchID, Field: "rows", Reason: "the batch carries no rows", + } + } + if len(rows) > control.MaxBatchResources { + return PGResourceWriteResult{}, &PGResourceWriteScopeError{ + BatchID: scope.BatchID, Field: "rows", + Reason: fmt.Sprintf("the batch offers %d rows, want at most %d", + len(rows), control.MaxBatchResources), + } + } + + decoded := make([]source.Resource, 0, len(rows)) + for index, built := range rows { + record, err := decodeBuiltResourceVertex(built.vertex, scope, index) + if err != nil { + return PGResourceWriteResult{}, err + } + decoded = append(decoded, record) + } + + result, err := w.committer.WriteResourceBatch(ctx, control.ResourceBatch{ + ID: scope.BatchID, + Project: scope.Project, + DatasetGeneration: scope.DatasetGeneration, + Rows: decoded, + }) + if err != nil { + return PGResourceWriteResult{}, err + } + return PGResourceWriteResult{ + BatchID: result.BatchID, + Digest: result.Digest, + Offered: len(rows), + Committed: result.Committed, + SkippedIdentical: result.SkippedIdentical, + Replay: result.Replay, + }, nil +} + +func decodeBuiltResourceVertex( + vertex json.RawMessage, + scope PGResourceBatchScope, + index int, +) (source.Resource, error) { + refuse := func(column string, format string, args ...any) (source.Resource, error) { + return source.Resource{}, &PGResourceRowIdentityError{ + BatchID: scope.BatchID, Row: index, Column: column, + Reason: fmt.Sprintf(format, args...), + } + } + invalid := func(column string, err error) (source.Resource, error) { + return refuse(column, "%v", err) + } + + document, err := decodeTopLevelDocument(vertex) + if err != nil { + return refuse("vertex", "the built document is undecodable: %v", err) + } + + record := source.Resource{ + Project: scope.Project, + DatasetGeneration: scope.DatasetGeneration, + AuthResourcePath: scope.AuthResourcePath, + } + + project, err := requiredDocumentString(document, "project") + if err != nil { + return invalid("project", err) + } + if project != scope.Project { + return refuse("project", "vertex project %q does not match the batch project %q", project, scope.Project) + } + projectID, err := requiredDocumentString(document, "project_id") + if err != nil { + return invalid("project_id", err) + } + if projectID != scope.Project { + return refuse("project_id", "vertex project_id %q does not match the batch project %q", projectID, scope.Project) + } + + generation, stamped, err := optionalDocumentString(document, generationIdentityField) + if err != nil { + return invalid(generationIdentityField, err) + } + if scope.DatasetGeneration != "" { + if !stamped { + return refuse(generationIdentityField, + "the row carries no dataset_generation stamp but the batch commits into generation %q", + scope.DatasetGeneration) + } + if generation != scope.DatasetGeneration { + return refuse(generationIdentityField, "vertex generation %q does not match the batch generation %q", + generation, scope.DatasetGeneration) + } + } else if stamped { + return refuse(generationIdentityField, + "the row is stamped for generation %q but the batch is the legacy (unversioned) namespace", generation) + } + + resourceType, err := requiredDocumentString(document, "resourceType") + if err != nil { + return invalid("resourceType", err) + } + record.ResourceType = resourceType + + logicalID, err := requiredDocumentString(document, "id") + if err != nil { + return invalid("id", err) + } + record.LogicalID = logicalID + + authPath, authPresent, err := optionalDocumentString(document, "auth_resource_path") + if err != nil { + return invalid("auth_resource_path", err) + } + if !authPresent { + if scope.AuthResourcePath != "" { + return refuse("auth_resource_path", + "the row carries no auth_resource_path but the batch is scoped to %q", scope.AuthResourcePath) + } + } else if authPath != scope.AuthResourcePath { + return refuse("auth_resource_path", "vertex auth_resource_path %q does not match the batch auth path %q", + authPath, scope.AuthResourcePath) + } + + if rawPayload, present := document["payload"]; present { + canonical, err := canonicalPayload(rawPayload) + if err != nil { + return invalid("payload", fmt.Errorf("the built payload document is undecodable: %w", err)) + } + record.Payload = canonical + } + return record, nil +} + +func canonicalPayload(raw []byte) ([]byte, error) { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + var document any + if err := decoder.Decode(&document); err != nil { + return nil, err + } + if _, err := decoder.Token(); err != io.EOF { + return nil, fmt.Errorf("unexpected content after the JSON document: %v", err) + } + return json.Marshal(document) +} + +func optionalDocumentString(document map[string]json.RawMessage, field string) (string, bool, error) { + raw, present := document[field] + if !present { + return "", false, nil + } + var value string + if err := json.Unmarshal(raw, &value); err != nil { + return "", false, fmt.Errorf("%s must be a string", field) + } + return value, true, nil +} + +var ErrInvalidPGEdgeWrite = errors.New("ingest: invalid postgres edge write") + +type PGEdgeWriteScopeError struct { + BatchID string + Field string + Reason string +} + +func (e *PGEdgeWriteScopeError) Error() string { + return fmt.Sprintf("postgres edge write refused: batch %q has an invalid %s: %s; "+ + "nothing was written", e.BatchID, e.Field, e.Reason) +} + +func (e *PGEdgeWriteScopeError) Unwrap() error { return ErrInvalidPGEdgeWrite } + +type PGEdgeRowIdentityError struct { + BatchID string + Row int + Category rowErrorType + Column string + Reason string +} + +func (e *PGEdgeRowIdentityError) Error() string { + return fmt.Sprintf("postgres edge write refused: batch %q row %d (%s failure) has an invalid %s: %s; "+ + "nothing was written", e.BatchID, e.Row, e.Category, e.Column, e.Reason) +} + +func (e *PGEdgeRowIdentityError) Unwrap() error { return ErrInvalidPGEdgeWrite } + +type PGEdgeBatchScope struct { + BatchID string + Project string + DatasetGeneration string + AuthResourcePath string +} + +type PGEdgeWriteResult struct { + BatchID string + Digest string + Rows int + Offered int + Committed int64 + SkippedIdentical int64 + Replay bool +} + +type edgeBatchCommitter interface { + WriteEdgeBatch(ctx context.Context, batch control.EdgeBatch) (control.EdgeBatchResult, error) +} + +type PostgresEdgeWriter struct { + committer edgeBatchCommitter +} + +func NewPostgresEdgeWriter(src control.ConnSource, schema string) (*PostgresEdgeWriter, error) { + writer, err := control.NewEdgeWriter(src, schema) + if err != nil { + return nil, err + } + return &PostgresEdgeWriter{committer: writer}, nil +} + +func (w *PostgresEdgeWriter) WriteEdgeBatch( + ctx context.Context, + scope PGEdgeBatchScope, + rows []rowBuildResult, +) (PGEdgeWriteResult, error) { + if strings.TrimSpace(scope.BatchID) == "" { + return PGEdgeWriteResult{}, &PGEdgeWriteScopeError{ + BatchID: scope.BatchID, Field: "batch id", Reason: "it is blank", + } + } + if len(scope.BatchID) > control.MaxBatchIDLength { + return PGEdgeWriteResult{}, &PGEdgeWriteScopeError{ + BatchID: scope.BatchID, Field: "batch id", + Reason: fmt.Sprintf("it is %d bytes, want at most %d", len(scope.BatchID), control.MaxBatchIDLength), + } + } + if strings.TrimSpace(scope.Project) == "" { + return PGEdgeWriteResult{}, &PGEdgeWriteScopeError{ + BatchID: scope.BatchID, Field: "project", Reason: "it is blank", + } + } + if len(rows) == 0 { + return PGEdgeWriteResult{}, &PGEdgeWriteScopeError{ + BatchID: scope.BatchID, Field: "rows", Reason: "the batch carries no rows", + } + } + + offered := make([]control.EdgeOccurrenceRow, 0, len(rows)) + for index, built := range rows { + converted, err := convertBuiltEdgeOccurrences(built, scope, index) + if err != nil { + return PGEdgeWriteResult{}, err + } + offered = append(offered, converted...) + } + if len(offered) == 0 { + return PGEdgeWriteResult{}, &PGEdgeWriteScopeError{ + BatchID: scope.BatchID, Field: "rows", + Reason: "the batch's rows carry no built edge occurrences", + } + } + if len(offered) > control.MaxBatchEdges { + return PGEdgeWriteResult{}, &PGEdgeWriteScopeError{ + BatchID: scope.BatchID, Field: "rows", + Reason: fmt.Sprintf("the batch offers %d edge occurrences, want at most %d", + len(offered), control.MaxBatchEdges), + } + } + + result, err := w.committer.WriteEdgeBatch(ctx, control.EdgeBatch{ + ID: scope.BatchID, + Project: scope.Project, + DatasetGeneration: scope.DatasetGeneration, + Rows: offered, + }) + if err != nil { + return PGEdgeWriteResult{}, err + } + return PGEdgeWriteResult{ + BatchID: result.BatchID, + Digest: result.Digest, + Rows: len(rows), + Offered: len(offered), + Committed: result.Committed, + SkippedIdentical: result.SkippedIdentical, + Replay: result.Replay, + }, nil +} + +type pgEdgeDocument struct { + fromType, fromID string + toType, toID string + label string + project string + authResourcePath string + authPresent bool + datasetGeneration string + stamped bool + key string + logicalEdgeKey *string +} + +func convertBuiltEdgeOccurrences( + built rowBuildResult, + scope PGEdgeBatchScope, + index int, +) ([]control.EdgeOccurrenceRow, error) { + refuse := func(category rowErrorType, column string, format string, args ...any) ([]control.EdgeOccurrenceRow, error) { + return nil, &PGEdgeRowIdentityError{ + BatchID: scope.BatchID, Row: index, Category: category, Column: column, + Reason: fmt.Sprintf(format, args...), + } + } + + if _, err := decodeBuiltResourceVertex(built.vertex, PGResourceBatchScope{ + BatchID: scope.BatchID, + Project: scope.Project, + DatasetGeneration: scope.DatasetGeneration, + AuthResourcePath: scope.AuthResourcePath, + }, index); err != nil { + var rowErr *PGResourceRowIdentityError + if errors.As(err, &rowErr) { + category := rowErrorGeneration + return refuse(category, rowErr.Column, "%s", rowErr.Reason) + } + return refuse(rowErrorGeneration, "vertex", "%v", err) + } + + document, err := decodeTopLevelDocument(built.vertex) + if err != nil { + return refuse(rowErrorGeneration, "vertex", "the built document is undecodable: %v", err) + } + resourceType, err := requiredDocumentString(document, "resourceType") + if err != nil { + return refuse(rowErrorGeneration, "resourceType", "%v", err) + } + logicalID, err := requiredDocumentString(document, "id") + if err != nil { + return refuse(rowErrorGeneration, "id", "%v", err) + } + vertexKey, err := requiredDocumentString(document, "_key") + if err != nil { + return refuse(rowErrorGeneration, "_key", "%v", err) + } + if len(built.edges) == 0 { + return nil, nil + } + + var positions []pgReferencePosition + if rawPayload, present := document["payload"]; present { + canonical, err := canonicalPayload(rawPayload) + if err != nil { + return refuse(rowErrorGeneration, "payload", "the built payload document is undecodable: %v", err) + } + positions, err = scanPayloadReferencePositions(canonical) + if err != nil { + return refuse(rowErrorGeneration, "payload", "%v", err) + } + } + + ownEndpoint := SanitizeKey(resourceType) + "/" + vertexKey + + groupOrder := make([]pgEdgeIdentity, 0, len(built.edges)) + groups := make(map[pgEdgeIdentity][]*pgEdgeDocument, len(built.edges)) + for edgeIndex, rawEdge := range built.edges { + doc, err := decodeBuiltEdgeDocument(rawEdge, scope, index, edgeIndex) + if err != nil { + var rowErr *PGEdgeRowIdentityError + if errors.As(err, &rowErr) { + return nil, rowErr + } + return refuse(rowErrorEdge, fmt.Sprintf("edge %d", edgeIndex), "%v", err) + } + identity, err := classifyEdgeOwnership(doc, ownEndpoint, index, edgeIndex, scope) + if err != nil { + return nil, err + } + if _, seen := groups[identity]; !seen { + groupOrder = append(groupOrder, identity) + } + groups[identity] = append(groups[identity], doc) + } + + out := make([]control.EdgeOccurrenceRow, 0, len(built.edges)) + for _, identity := range groupOrder { + members := groups[identity] + peerType, peerKey := identity.toType, identity.toID + ownIsSource := !identity.directionBackref + direction := "forward" + if !ownIsSource { + peerType, peerKey = identity.fromType, identity.fromID + direction = "back-reference" + } + qualify := sanitizeKeyPart + if scope.DatasetGeneration != "" { + project, generation := scope.Project, scope.DatasetGeneration + qualify = func(logicalID string) string { + return generationDocumentKey(project, generation, peerType, SanitizeKey(logicalID)) + } + } + candidates := selectReferencePositions(positions, peerType, peerKey, identity.label, qualify, identity.directionBackref) + if len(candidates) == 0 { + column := fmt.Sprintf("edge _to (first of group %s)", identity.key) + if !ownIsSource { + column = fmt.Sprintf("edge _from (first of group %s)", identity.key) + } + return refuse(rowErrorEdge, column, + "the %s edge document %s/%s -%s-> %s/%s matches no reference position in the vertex payload", + direction, identity.fromType, identity.fromID, identity.label, identity.toType, identity.toID) + } + if len(members) > len(candidates) { + return refuse(rowErrorEdge, fmt.Sprintf("edge _key (group %s)", identity.key), + "the row offers %d identical %s edge documents for %s/%s -%s-> %s/%s but the payload holds only %d matching reference positions", + len(members), direction, identity.fromType, identity.fromID, identity.label, + identity.toType, identity.toID, len(candidates)) + } + ordinals := make(map[string]int64, len(candidates)) + rowType := SanitizeKey(resourceType) + for i := range members { + position := candidates[i] + ordinal := ordinals[position.path] + ordinals[position.path] = ordinal + 1 + sourceType, sourceID, targetType, targetID := identity.fromType, identity.fromID, identity.toType, identity.toID + if ownIsSource { + sourceID = logicalID + targetID = position.logicalID + if position.refType != "" { + canonical, ok := fhirschema.ConcreteResourceType(position.refType) + if !ok { + return refuse(rowErrorEdge, fmt.Sprintf("edge _to (first of group %s)", identity.key), + "the matched payload reference %q at %q does not resolve to a concrete FHIR resource type", + position.refType, position.path) + } + if canonical != identity.toType { + return refuse(rowErrorEdge, fmt.Sprintf("edge _to (first of group %s)", identity.key), + "the forward edge document declares target type %q, but the matched payload reference %s/%s at %q carries type %q", + identity.toType, canonical, position.logicalID, position.path, position.refType) + } + } + } else { + sourceID = position.logicalID + targetID = logicalID + if position.refType != "" { + canonical, ok := fhirschema.ConcreteResourceType(position.refType) + if !ok { + return refuse(rowErrorEdge, fmt.Sprintf("edge _from (first of group %s)", identity.key), + "the matched payload reference %q at %q does not resolve to a concrete FHIR resource type", + position.refType, position.path) + } + if canonical != identity.fromType { + if identity.fromType == rowType { + sourceType = canonical + } else { + return refuse(rowErrorEdge, fmt.Sprintf("edge _from (first of group %s)", identity.key), + "the back-reference edge document declares source type %q, but the matched payload reference %s/%s at %q carries type %q and the declared type is not the row's own type %q", + identity.fromType, canonical, position.logicalID, position.path, position.refType, rowType) + } + } else { + sourceType = canonical + } + } + } + out = append(out, control.EdgeOccurrenceRow{ + Project: scope.Project, + DatasetGeneration: scope.DatasetGeneration, + AuthResourcePath: scope.AuthResourcePath, + Label: identity.label, + SourceResourceType: sourceType, + SourceLogicalID: sourceID, + TargetResourceType: targetType, + TargetLogicalID: targetID, + SourcePath: position.path, + SourceOrdinal: ordinal, + LogicalEdgeKey: members[i].logicalEdgeKey, + }) + } + } + return out, nil +} + +func sanitizeKeyPart(logicalID string) string { + return SanitizeKey(logicalID) +} + +func decodeBuiltEdgeDocument( + raw json.RawMessage, + scope PGEdgeBatchScope, + rowIndex int, + edgeIndex int, +) (*pgEdgeDocument, error) { + column := func(name string) string { return fmt.Sprintf("edge %d %s", edgeIndex, name) } + refuse := func(column string, format string, args ...any) (*pgEdgeDocument, error) { + return nil, &PGEdgeRowIdentityError{ + BatchID: scope.BatchID, Row: rowIndex, Category: rowErrorEdge, Column: column, + Reason: fmt.Sprintf(format, args...), + } + } + + document, err := decodeTopLevelDocument(raw) + if err != nil { + return refuse(column("document"), "the built edge document is undecodable: %v", err) + } + if _, advertised := document["edge_seq"]; advertised { + return refuse(column("edge_seq"), + "the storage identity is server-assigned; an edge document must not advertise one") + } + + doc := &pgEdgeDocument{} + if doc.project, err = requiredDocumentString(document, "project"); err != nil { + return refuse(column("project"), "%v", err) + } + if doc.project != scope.Project { + return refuse(column("project"), "edge project %q does not match the batch project %q", doc.project, scope.Project) + } + if doc.label, err = requiredDocumentString(document, "label"); err != nil { + return refuse(column("label"), "%v", err) + } + if doc.key, err = requiredDocumentString(document, "_key"); err != nil { + return refuse(column("_key"), "%v", err) + } + if logicalKey, present := document[logicalKeyField]; present { + var value string + if err := json.Unmarshal(logicalKey, &value); err != nil { + return refuse(column(logicalKeyField), "%s must be a string", logicalKeyField) + } + doc.logicalEdgeKey = &value + } else { + key := doc.key + doc.logicalEdgeKey = &key + } + from, err := requiredDocumentString(document, "_from") + if err != nil { + return refuse(column("_from"), "%v", err) + } + to, err := requiredDocumentString(document, "_to") + if err != nil { + return refuse(column("_to"), "%v", err) + } + declaredFromType, err := requiredDocumentString(document, "from_type") + if err != nil { + return refuse(column("from_type"), "%v", err) + } + declaredToType, err := requiredDocumentString(document, "to_type") + if err != nil { + return refuse(column("to_type"), "%v", err) + } + fromType, fromID, parseErr := parseEdgeEndpoint(column("from_type"), declaredFromType, from) + if parseErr != nil { + parseErr.BatchID = scope.BatchID + parseErr.Row = rowIndex + return nil, parseErr + } + toType, toID, parseErr := parseEdgeEndpoint(column("to_type"), declaredToType, to) + if parseErr != nil { + parseErr.BatchID = scope.BatchID + parseErr.Row = rowIndex + return nil, parseErr + } + doc.fromType, doc.fromID = fromType, fromID + doc.toType, doc.toID = toType, toID + + if doc.authResourcePath, doc.authPresent, err = optionalDocumentString(document, "auth_resource_path"); err != nil { + return refuse(column("auth_resource_path"), "%v", err) + } + if !doc.authPresent { + if scope.AuthResourcePath != "" { + return refuse(column("auth_resource_path"), + "the edge carries no auth_resource_path but the batch is scoped to %q", scope.AuthResourcePath) + } + } else if doc.authResourcePath != scope.AuthResourcePath { + return refuse(column("auth_resource_path"), "edge auth_resource_path %q does not match the batch auth path %q", + doc.authResourcePath, scope.AuthResourcePath) + } + + if doc.datasetGeneration, doc.stamped, err = optionalDocumentString(document, generationIdentityField); err != nil { + return refuse(column(generationIdentityField), "%v", err) + } + if scope.DatasetGeneration != "" { + if !doc.stamped { + return refuse(column(generationIdentityField), + "the edge carries no dataset_generation stamp but the batch commits into generation %q", + scope.DatasetGeneration) + } + if doc.datasetGeneration != scope.DatasetGeneration { + return refuse(column(generationIdentityField), + "edge generation %q does not match the batch generation %q", doc.datasetGeneration, scope.DatasetGeneration) + } + } else if doc.stamped { + return refuse(column(generationIdentityField), + "the edge is stamped for generation %q but the batch is the legacy (unversioned) namespace", doc.datasetGeneration) + } + return doc, nil +} + +func parseEdgeEndpoint(column, declaredType, endpoint string) (string, string, *PGEdgeRowIdentityError) { + resourceType, logicalID, ok := SplitCollectionID(endpoint) + if !ok { + return "", "", &PGEdgeRowIdentityError{ + Category: rowErrorEdge, Column: column, + Reason: fmt.Sprintf("endpoint %q is not a well-formed Type/id collection identity", endpoint), + } + } + if resourceType != declaredType { + return "", "", &PGEdgeRowIdentityError{ + Category: rowErrorEdge, Column: column, + Reason: fmt.Sprintf("endpoint %q has collection type %q but the document declares %q", + endpoint, resourceType, declaredType), + } + } + return resourceType, logicalID, nil +} + +type pgEdgeIdentity struct { + fromType string + fromID string + label string + toType string + toID string + directionBackref bool + key string +} + +func classifyEdgeOwnership( + doc *pgEdgeDocument, + ownEndpoint string, + rowIndex int, + edgeIndex int, + scope PGEdgeBatchScope, +) (pgEdgeIdentity, error) { + identity := pgEdgeIdentity{ + fromType: doc.fromType, + fromID: doc.fromID, + label: doc.label, + toType: doc.toType, + toID: doc.toID, + key: doc.key, + } + fromEndpoint := SanitizeKey(doc.fromType) + "/" + doc.fromID + toEndpoint := SanitizeKey(doc.toType) + "/" + doc.toID + switch ownEndpoint { + case fromEndpoint: + return identity, nil + case toEndpoint: + identity.directionBackref = true + return identity, nil + default: + return pgEdgeIdentity{}, &PGEdgeRowIdentityError{ + BatchID: scope.BatchID, Row: rowIndex, Category: rowErrorEdge, + Column: fmt.Sprintf("edge %d _from", edgeIndex), + Reason: fmt.Sprintf("neither _from %q nor _to %q is the owning row's endpoint %q; "+ + "an edge document must be owned by (or back-reference) its row", + fromEndpoint, toEndpoint, ownEndpoint), + } + } +} + +type pgReferencePosition struct { + path string + refType string + logicalID string +} + +func scanPayloadReferencePositions(canonical []byte) ([]pgReferencePosition, error) { + trimmed := strings.TrimSpace(string(canonical)) + if trimmed == "null" || trimmed == "" { + return nil, nil + } + var object map[string]json.RawMessage + if err := json.Unmarshal([]byte(trimmed), &object); err != nil { + return nil, fmt.Errorf("the built payload document must be a JSON object: %w", err) + } + if object == nil { + return nil, fmt.Errorf("the built payload document must be a JSON object") + } + var out []pgReferencePosition + collectReferencePositions(object, "", &out) + return out, nil +} + +func collectReferencePositions(object map[string]json.RawMessage, prefix string, out *[]pgReferencePosition) { + keys := make([]string, 0, len(object)) + for key := range object { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + walkReferenceValue(object[key], joinPath(prefix, key), out) + } +} + +func walkReferenceValue(raw json.RawMessage, path string, out *[]pgReferencePosition) { + trimmed := strings.TrimSpace(string(raw)) + switch { + case strings.HasPrefix(trimmed, "{"): + var object map[string]json.RawMessage + if err := json.Unmarshal(raw, &object); err != nil || object == nil { + return + } + collectReferencePositions(object, path, out) + case strings.HasPrefix(trimmed, "["): + var items []json.RawMessage + if err := json.Unmarshal(raw, &items); err != nil { + return + } + for _, item := range items { + walkReferenceValue(item, path, out) + } + case path == "reference" || strings.HasSuffix(path, ".reference"): + var value string + if err := json.Unmarshal(raw, &value); err != nil { + return + } + if refType, logicalID, ok := parseFHIRReferenceString(value); ok { + *out = append(*out, pgReferencePosition{ + path: strings.TrimSuffix(strings.TrimSuffix(path, "reference"), "."), + refType: refType, + logicalID: logicalID, + }) + } + } +} + +func parseFHIRReferenceString(value string) (refType, logicalID string, ok bool) { + first := strings.IndexByte(value, '/') + if first <= 0 || first == len(value)-1 { + return "", value, true + } + refType = value[:first] + logicalID = value[first+1:] + if second := strings.IndexByte(logicalID, '/'); second >= 0 { + logicalID = logicalID[:second] + } + if logicalID == "" { + return "", "", false + } + return refType, logicalID, true +} + +func joinPath(prefix, key string) string { + if prefix == "" { + return key + } + return prefix + "." + key +} + +func selectReferencePositions( + positions []pgReferencePosition, + peerType, peerKey, label string, + qualify func(logicalID string) string, + backref bool, +) []pgReferencePosition { + var idMatches []pgReferencePosition + for _, position := range positions { + if qualify(position.logicalID) == peerKey { + idMatches = append(idMatches, position) + } + } + if len(idMatches) == 0 { + return nil + } + pick := func(predicate func(pgReferencePosition) bool) ([]pgReferencePosition, bool) { + var kept []pgReferencePosition + for _, position := range idMatches { + if predicate(position) { + kept = append(kept, position) + } + } + return kept, len(kept) > 0 + } + if backref { + if kept, ok := pick(func(p pgReferencePosition) bool { return labelNamesPathToken(label, p.path) }); ok { + idMatches = kept + } + if kept, ok := pick(func(p pgReferencePosition) bool { return positionTypeMatchesPeer(p.refType, peerType) }); ok { + idMatches = kept + } else if kept, ok := pick(func(p pgReferencePosition) bool { return p.refType == "" }); ok { + idMatches = kept + } + return idMatches + } + if kept, ok := pick(func(p pgReferencePosition) bool { return positionTypeMatchesPeer(p.refType, peerType) }); ok { + idMatches = kept + } else if kept, ok := pick(func(p pgReferencePosition) bool { return p.refType == "" }); ok { + idMatches = kept + } + if kept, ok := pick(func(p pgReferencePosition) bool { return labelNamesPathToken(label, p.path) }); ok { + idMatches = kept + } + return idMatches +} + +func positionTypeMatchesPeer(positionType, peerType string) bool { + if positionType == peerType { + return true + } + if positionType == "" || peerType == "" { + return false + } + canonical, ok := fhirschema.ConcreteResourceType(positionType) + return ok && canonical == peerType +} + +func labelNamesPathToken(label, path string) bool { + for _, pathSegment := range strings.Split(path, ".") { + if pathSegment == "" { + continue + } + for _, labelToken := range strings.FieldsFunc(label, func(r rune) bool { + return r == '_' || r == '.' + }) { + if labelToken == pathSegment { + return true + } + } + } + return false +} diff --git a/internal/ingest/postgres_writer_test.go b/internal/ingest/postgres_writer_test.go new file mode 100644 index 000000000..7df07e068 --- /dev/null +++ b/internal/ingest/postgres_writer_test.go @@ -0,0 +1,2997 @@ +package ingest + +import ( + "context" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "github.com/calypr/loom/internal/backends/postgres/control" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/bmeg/jsonschemagraph/graph" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + ing01IngestPrefix = "verify_ing01_ing_" + ing01IngestFmt = ing01IngestPrefix + "%d_%s" + + ing01IngestProject = "prj-ing01-ing" + ing01IngestGen = "gen-build-a" + ing01IngestAuth = "/fhir" +) + +var ing01IngestNamePattern = regexp.MustCompile(`^verify_ing01_ing_[0-9]+_[0-9a-f]+$`) + +func ing01TestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 integration tests") + } + return raw +} + +var ing01IngestOwnedMu sync.Mutex +var ing01IngestOwnedByName = map[string]string{} + +func ing01IngestRegister(db, schema string) { + ing01IngestOwnedMu.Lock() + defer ing01IngestOwnedMu.Unlock() + ing01IngestOwnedByName[db] = schema +} + +var ( + ing01IngestBaselineOnce sync.Once + ing01IngestBaselineCount int64 + ing01IngestBaselineErr error +) + +func ing01IngestCaptureBaseline(raw string) { + ing01IngestBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + ing01IngestBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_ing01\\_ing\\_%' ESCAPE '\\'"). + Scan(&count); err != nil { + ing01IngestBaselineErr = fmt.Errorf("count baseline: %w", err) + return + } + ing01IngestBaselineCount = count + }) +} + +func ing01RandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) +} + +func str(s string) *string { return &s } + +type ing01IngestService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func newIngest01IngestService(t *testing.T) *ing01IngestService { + t.Helper() + raw := ing01TestPostgresURL(t) + ing01IngestCaptureBaseline(raw) + if ing01IngestBaselineErr != nil { + t.Fatalf("capture pg_database baseline: %v", ing01IngestBaselineErr) + } + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf(ing01IngestFmt, time.Now().UnixNano(), ing01RandomHex(t)) + if !ing01IngestNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(ing01IngestFmt, time.Now().UnixNano(), ing01RandomHex(t)) + if !ing01IngestNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + ing01IngestRegister(dbName, schema) + return &ing01IngestService{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} +} + +func (s *ing01IngestService) migrate(t *testing.T) { + t.Helper() + client := s.openClient(t, 4) + if _, err := control.MigrateSchema(s.ctx, client, s.schema); err != nil { + s.fail("MigrateSchema", err) + } +} + +func (s *ing01IngestService) openClient(t *testing.T, maxConns int32) *control.Client { + t.Helper() + client, err := control.OpenPool(s.ctx, control.ClientConfig{ + DSN: s.dsn, + MaxConns: maxConns, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *ing01IngestService) openAppClient(t *testing.T, app string, maxConns int32) *control.Client { + t.Helper() + u, err := url.Parse(s.dsn) + if err != nil { + s.fail("parse isolated dsn for app client", err) + } + q := u.Query() + q.Set("application_name", app) + u.RawQuery = q.Encode() + client, err := control.OpenPool(s.ctx, control.ClientConfig{ + DSN: u.String(), + MaxConns: maxConns, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + s.fail("open app client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *ing01IngestService) newWriterOn(t *testing.T, src control.ConnSource) *PostgresResourceWriter { + t.Helper() + writer, err := NewPostgresResourceWriter(src, s.schema) + if err != nil { + s.fail("new postgres resource writer", err) + } + return writer +} + +func (s *ing01IngestService) newWriter(t *testing.T) *PostgresResourceWriter { + t.Helper() + return s.newWriterOn(t, s.openClient(t, 4)) +} + +func (s *ing01IngestService) qualified(table string) string { + return pgx.Identifier{s.schema, table}.Sanitize() +} + +func (s *ing01IngestService) mustInt64(t *testing.T, sql string, args ...any) int64 { + t.Helper() + var v int64 + if err := s.iso.QueryRow(s.ctx, sql, args...).Scan(&v); err != nil { + s.fail("witness "+sql, err) + } + return v +} + +func (s *ing01IngestService) countIn(t *testing.T, project, generation string) int64 { + t.Helper() + return s.mustInt64(t, "SELECT count(*) FROM "+s.qualified("fhir_resource")+ + " WHERE project = $1 AND dataset_generation = $2", project, generation) +} + +func (s *ing01IngestService) requireResourceRow(t *testing.T, project, generation, rtype, lid, authPath string, wantPayloadText *string) { + t.Helper() + var gotAuth string + var isNull bool + var payload *string + err := s.iso.QueryRow(s.ctx, + "SELECT auth_resource_path, payload IS NULL, payload::text FROM "+s.qualified("fhir_resource")+ + " WHERE project = $1 AND dataset_generation = $2 AND resource_type = $3 AND logical_id = $4", + project, generation, rtype, lid).Scan(&gotAuth, &isNull, &payload) + if err != nil { + s.fail("witness resource row "+rtype+"/"+lid, err) + } + if gotAuth != authPath { + t.Fatalf("resource %s/%s auth path = %q, want %q", rtype, lid, gotAuth, authPath) + } + if wantPayloadText == nil { + if !isNull { + t.Fatalf("resource %s/%s payload = %v, want SQL NULL", rtype, lid, *payload) + } + return + } + if isNull { + t.Fatalf("resource %s/%s payload is SQL NULL, want document %s", rtype, lid, *wantPayloadText) + } + if *payload != *wantPayloadText { + t.Fatalf("resource %s/%s payload text = %s, want %s", rtype, lid, *payload, *wantPayloadText) + } +} + +func (s *ing01IngestService) witnessLedger(t *testing.T, batchID string) (digest string, resources int64, found bool) { + t.Helper() + err := s.iso.QueryRow(s.ctx, + "SELECT digest, resource_count FROM "+s.qualified("batch_write_ledger")+" WHERE batch_id = $1", batchID). + Scan(&digest, &resources) + if errors.Is(err, pgx.ErrNoRows) { + return "", 0, false + } + if err != nil { + s.fail("witness ledger row", err) + } + return digest, resources, true +} + +func (s *ing01IngestService) scopeFingerprint(t *testing.T, project, generation string) string { + t.Helper() + var fp string + sql := "SELECT coalesce(md5(string_agg(entry, E'\n' ORDER BY resource_type, logical_id, auth_resource_path)),'EMPTY') FROM (" + + "SELECT resource_type, logical_id, auth_resource_path," + + " resource_type || '|' || logical_id || '|' || auth_resource_path || '|' ||" + + " coalesce(payload::text, '') AS entry" + + " FROM " + s.qualified("fhir_resource") + " WHERE project = $1 AND dataset_generation = $2) x" + if err := s.iso.QueryRow(s.ctx, sql, project, generation).Scan(&fp); err != nil { + s.fail("scope fingerprint", err) + } + return fp +} + +func (s *ing01IngestService) poolBackendPID(t *testing.T, client *control.Client) int64 { + t.Helper() + conn, err := client.Acquire(s.ctx) + if err != nil { + s.fail("acquire for backend pid", err) + } + defer conn.Release() + var pid int64 + if err := conn.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + s.fail("witness pg_backend_pid", err) + } + return pid +} + +func (s *ing01IngestService) waitBackendWaiting(t *testing.T, app string) int64 { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for { + var pid int64 + var waitType, waitEvent *string + err := s.iso.QueryRow(s.ctx, + "SELECT pid, wait_event_type, wait_event FROM pg_stat_activity"+ + " WHERE application_name = $1 AND state = 'active'", app). + Scan(&pid, &waitType, &waitEvent) + if err == nil && waitType != nil && *waitType == "Lock" { + return pid + } + if time.Now().After(deadline) { + s.fail("waiting for backend "+app+" to block on a lock", + fmt.Errorf("backend never entered a lock wait")) + } + time.Sleep(10 * time.Millisecond) + } +} + +const ( + ing01ResearchSubjectLine = `{"resourceType":"ResearchSubject","id":"research-subject-1","status":"active","study":{"reference":"ResearchStudy/study-1"},"subject":{"reference":"Patient/patient-1"},"project_id":"source-value","unknown_payload_field":{"nested":["kept",42]}}` + ing01SubstanceLine = `{"resourceType":"SubstanceDefinition","id":"substance-1","name":[{"name":"Gemcitabine"}]}` + + ing01ResearchSubjectPayloadText = `{"id": "research-subject-1", "study": {"reference": "ResearchStudy/study-1"}, "status": "active", "subject": {"reference": "Patient/patient-1"}, "project_id": "prj-ing01-ing", "resourceType": "ResearchSubject", "unknown_payload_field": {"nested": ["kept", 42]}}` + ing01SubstancePayloadText = `{"id": "substance-1", "name": [{"name": "Gemcitabine"}], "project_id": "prj-ing01-ing", "resourceType": "SubstanceDefinition", "auth_resource_path": "/fhir"}` + + ing01BuiltBatchID = "ing01-ing/built/0" + ing01BuiltBatchDigest = "7fe13c3d615ec75a169f03491708478a30756bb68aa8d7c02bbd3dc0cbc040e2" + ing01HandBatchID = "ing01-ing/hand/0" + ing01HandBatchDigest = "9bf323bc4bcba52cc891cacce1007c3bebdcabf3c182711d6c4a50fc3de4903f" +) + +func ing01LoadGraphSchema(t *testing.T) *graph.GraphSchema { + t.Helper() + schema, err := graph.Load(repoPath(t, "schemas", "graph-fhir.json")) + if err != nil { + t.Fatalf("load graph schema: %v", err) + } + return schema +} + +func buildGeneratedRow(t *testing.T, project, generation, authPath, resourceType, line string) rowBuildResult { + t.Helper() + var builder RowBuilder = NewGeneratedRowBuilder(project, authPath) + if strings.TrimSpace(generation) != "" { + var err error + builder, err = newGenerationRowBuilder(builder, project, generation) + if err != nil { + t.Fatalf("newGenerationRowBuilder: %v", err) + } + } + built, errorType, err := builder.Build(resourceType, []byte(line), map[string]float64{}) + if err != nil { + t.Fatalf("build %s row: kind=%s err=%v", resourceType, errorType, err) + } + return built +} + +func buildGenericRow(t *testing.T, project, generation, authPath, resourceType, line string) rowBuildResult { + t.Helper() + schema := ing01LoadGraphSchema(t) + class := schema.GetClass(resourceType) + if class == nil { + t.Fatalf("class %q not found", resourceType) + } + var builder RowBuilder = NewGenericRowBuilder(project, class, schema, graphExtraArgs(project, authPath)) + if strings.TrimSpace(generation) != "" { + var err error + builder, err = newGenerationRowBuilder(builder, project, generation) + if err != nil { + t.Fatalf("newGenerationRowBuilder: %v", err) + } + } + built, errorType, err := builder.Build(resourceType, []byte(line), map[string]float64{}) + if err != nil { + t.Fatalf("build %s row: kind=%s err=%v", resourceType, errorType, err) + } + return built +} + +func handBuiltVertex(project, generation, resourceType, logicalID, authPath, payload string, extraFields map[string]string) rowBuildResult { + var b strings.Builder + b.WriteString(`{"_key":"`) + b.WriteString(SanitizeKey(logicalID)) + b.WriteString(`","id":"`) + b.WriteString(logicalID) + b.WriteString(`","project":"`) + b.WriteString(project) + b.WriteString(`","project_id":"`) + b.WriteString(project) + b.WriteString(`","resourceType":"`) + b.WriteString(resourceType) + b.WriteString(`"`) + if authPath != "" { + b.WriteString(`,"auth_resource_path":"`) + b.WriteString(authPath) + b.WriteString(`"`) + } + if generation != "" { + b.WriteString(`,"dataset_generation":"`) + b.WriteString(generation) + b.WriteString(`"`) + } + for k, v := range extraFields { + b.WriteString(`,"`) + b.WriteString(k) + b.WriteString(`":`) + b.WriteString(v) + } + if payload != "" { + b.WriteString(`,"payload":`) + b.WriteString(payload) + } + b.WriteString(`}`) + return rowBuildResult{vertex: json.RawMessage(b.String())} +} + +func replaceField(t *testing.T, built rowBuildResult, field string, rawValue json.RawMessage) rowBuildResult { + t.Helper() + document, err := decodeTopLevelDocument(built.vertex) + if err != nil { + t.Fatalf("decode vertex for defect injection: %v", err) + } + document[field] = rawValue + encoded, err := json.Marshal(document) + if err != nil { + t.Fatalf("re-encode vertex: %v", err) + } + built.vertex = encoded + return built +} + +func dropField(t *testing.T, built rowBuildResult, field string) rowBuildResult { + t.Helper() + document, err := decodeTopLevelDocument(built.vertex) + if err != nil { + t.Fatalf("decode vertex for defect injection: %v", err) + } + delete(document, field) + encoded, err := json.Marshal(document) + if err != nil { + t.Fatalf("re-encode vertex: %v", err) + } + built.vertex = encoded + return built +} + +type ing01Recorder struct { + mu sync.Mutex + calls int + batches []control.ResourceBatch +} + +func (r *ing01Recorder) WriteResourceBatch(_ context.Context, batch control.ResourceBatch) (control.ResourceBatchResult, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.calls++ + r.batches = append(r.batches, batch) + return control.ResourceBatchResult{ + BatchID: batch.ID, + Digest: "recorder-digest", + Committed: int64(len(batch.Rows)), + }, nil +} + +func (r *ing01Recorder) callCount() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.calls +} + +func (r *ing01Recorder) lastBatch(t *testing.T) control.ResourceBatch { + t.Helper() + r.mu.Lock() + defer r.mu.Unlock() + if r.calls == 0 { + t.Fatal("recorder has no recorded batch") + } + return r.batches[r.calls-1] +} + +func TestPostgresResourceWriterWritesBuiltRowsWithExactTypedColumnsAndLedger(t *testing.T) { + s := newIngest01IngestService(t) + s.migrate(t) + writer := s.newWriter(t) + + rows := []rowBuildResult{ + buildGeneratedRow(t, ing01IngestProject, ing01IngestGen, ing01IngestAuth, "ResearchSubject", ing01ResearchSubjectLine), + buildGenericRow(t, ing01IngestProject, ing01IngestGen, ing01IngestAuth, "SubstanceDefinition", ing01SubstanceLine), + handBuiltVertex(ing01IngestProject, ing01IngestGen, "Observation", "hand-1", ing01IngestAuth, + `{"resourceType":"Observation","id":"hand-1","effective":"2026"}`, + map[string]string{"resource_seq": "999", "logical_key": `"hand-1"`}), + } + scope := PGResourceBatchScope{ + BatchID: ing01BuiltBatchID, Project: ing01IngestProject, + DatasetGeneration: ing01IngestGen, AuthResourcePath: ing01IngestAuth, + } + res, err := writer.WriteResourceBatch(s.ctx, scope, rows) + if err != nil { + s.fail("WriteResourceBatch", err) + } + if res.BatchID != ing01BuiltBatchID || res.Offered != 3 || res.Committed != 3 || + res.SkippedIdentical != 0 || res.Replay { + t.Fatalf("result = %+v, want a fresh 3-row commit", res) + } + if res.Digest != ing01BuiltBatchDigest { + t.Fatalf("digest = %s, want the pinned golden %s", res.Digest, ing01BuiltBatchDigest) + } + + s.requireResourceRow(t, ing01IngestProject, ing01IngestGen, "ResearchSubject", "research-subject-1", + ing01IngestAuth, str(ing01ResearchSubjectPayloadText)) + s.requireResourceRow(t, ing01IngestProject, ing01IngestGen, "SubstanceDefinition", "substance-1", + ing01IngestAuth, str(ing01SubstancePayloadText)) + s.requireResourceRow(t, ing01IngestProject, ing01IngestGen, "Observation", "hand-1", + ing01IngestAuth, str(`{"id": "hand-1", "effective": "2026", "resourceType": "Observation"}`)) + if n := s.countIn(t, ing01IngestProject, ing01IngestGen); n != 3 { + t.Fatalf("rows = %d, want 3", n) + } + + var advertised bool + if err := s.iso.QueryRow(s.ctx, + "SELECT coalesce(bool_or(resource_seq = 999), false) FROM "+s.qualified("fhir_resource")+ + " WHERE project = $1 AND dataset_generation = $2", ing01IngestProject, ing01IngestGen). + Scan(&advertised); err != nil { + s.fail("witness server-assigned seqs", err) + } + if advertised { + t.Fatal("a caller-advertised resource_seq was stored; the storage identity must be server-assigned") + } + distinct := s.mustInt64(t, "SELECT count(DISTINCT resource_seq) FROM "+s.qualified("fhir_resource")+ + " WHERE project = $1 AND dataset_generation = $2", ing01IngestProject, ing01IngestGen) + if distinct != 3 { + t.Fatalf("distinct resource_seq = %d, want 3", distinct) + } + + digest, resources, found := s.witnessLedger(t, ing01BuiltBatchID) + if !found || digest != ing01BuiltBatchDigest || resources != 3 { + t.Fatalf("marker = (%s, %d, found %v), want the golden digest and 3 committed rows", + digest, resources, found) + } +} + +func TestPostgresResourceWriterBatchContentMatchesLiteralCommittedBatch(t *testing.T) { + recorder := &ing01Recorder{} + writer := &PostgresResourceWriter{committer: recorder} + + rows := []rowBuildResult{ + handBuiltVertex(ing01IngestProject, ing01IngestGen, "Observation", "cap-1", ing01IngestAuth, + `{"resourceType":"Observation","id":"cap-1","status":"final"}`, nil), + handBuiltVertex(ing01IngestProject, ing01IngestGen, "Observation", "cap-2", ing01IngestAuth, + `null`, nil), + handBuiltVertex(ing01IngestProject, ing01IngestGen, "Observation", "cap-3", ing01IngestAuth, + "", nil), + } + scope := PGResourceBatchScope{ + BatchID: "ing01-ing/capture/0", Project: ing01IngestProject, + DatasetGeneration: ing01IngestGen, AuthResourcePath: ing01IngestAuth, + } + if _, err := writer.WriteResourceBatch(context.Background(), scope, rows); err != nil { + t.Fatalf("WriteResourceBatch: %v", err) + } + if got := recorder.callCount(); got != 1 { + t.Fatalf("committer calls = %d, want 1", got) + } + want := control.ResourceBatch{ + ID: "ing01-ing/capture/0", + Project: ing01IngestProject, + DatasetGeneration: ing01IngestGen, + Rows: []source.Resource{ + {Project: ing01IngestProject, DatasetGeneration: ing01IngestGen, ResourceType: "Observation", + LogicalID: "cap-1", AuthResourcePath: ing01IngestAuth, + Payload: json.RawMessage(`{"id":"cap-1","resourceType":"Observation","status":"final"}`)}, + {Project: ing01IngestProject, DatasetGeneration: ing01IngestGen, ResourceType: "Observation", + LogicalID: "cap-2", AuthResourcePath: ing01IngestAuth, Payload: json.RawMessage(`null`)}, + {Project: ing01IngestProject, DatasetGeneration: ing01IngestGen, ResourceType: "Observation", + LogicalID: "cap-3", AuthResourcePath: ing01IngestAuth, Payload: nil}, + }, + } + if got := recorder.lastBatch(t); !reflect.DeepEqual(got, want) { + t.Fatalf("committed batch drifted from the literal:\n got %+v\nwant %+v", got, want) + } +} + +func TestPostgresResourceWriterCanonicalPayloadEncoding(t *testing.T) { + for _, tc := range []struct{ name, in, out string }{ + {"sorted keys, compact, numbers verbatim, duplicates last-wins", + `{ "b" : 1 , "a" : [ 2.50 , 1e3 , {"z":null,"y":[]} ] , "a2":true , "b":2 }`, + `{"a":[2.50,1e3,{"y":[],"z":null}],"a2":true,"b":2}`}, + {"null document stays null", " null ", `null`}, + {"empty object", `{}`, `{}`}, + {"unicode escapes round trip", `{"k":"\u00e9"}`, `{"k":"é"}`}, + } { + got, err := canonicalPayload([]byte(tc.in)) + if err != nil { + t.Fatalf("canonicalPayload(%s): %v", tc.in, err) + } + if string(got) != tc.out { + t.Fatalf("canonicalPayload(%s) = %s, want %s", tc.in, got, tc.out) + } + } + for _, malformed := range []string{`{`, `{"a":1}{"b":2}`, `{"a":}`, ``, `[1,2`, `nul`, `12abc`} { + if _, err := canonicalPayload([]byte(malformed)); err == nil { + t.Fatalf("canonicalPayload(%q) accepted malformed JSON", malformed) + } + } +} + +func TestPostgresResourceWriterLegacyNamespaceAndPayloadNullFidelity(t *testing.T) { + s := newIngest01IngestService(t) + s.migrate(t) + writer := s.newWriter(t) + + rows := []rowBuildResult{ + handBuiltVertex(ing01IngestProject, "", "Patient", "legacy-1", "", + `{"resourceType":"Patient","id":"legacy-1"}`, nil), + handBuiltVertex(ing01IngestProject, "", "Patient", "legacy-null", "", `null`, nil), + handBuiltVertex(ing01IngestProject, "", "Patient", "legacy-absent", "", "", nil), + } + scope := PGResourceBatchScope{BatchID: ing01HandBatchID, Project: ing01IngestProject} + res, err := writer.WriteResourceBatch(s.ctx, scope, rows) + if err != nil { + s.fail("legacy WriteResourceBatch", err) + } + if res.Committed != 3 || res.Replay || res.Digest != ing01HandBatchDigest { + t.Fatalf("legacy result = %+v, want a fresh 3-row commit under the pinned digest %s", + res, ing01HandBatchDigest) + } + s.requireResourceRow(t, ing01IngestProject, "", "Patient", "legacy-1", "", str(`{"id": "legacy-1", "resourceType": "Patient"}`)) + s.requireResourceRow(t, ing01IngestProject, "", "Patient", "legacy-null", "", str("null")) + s.requireResourceRow(t, ing01IngestProject, "", "Patient", "legacy-absent", "", nil) + + nulls := s.mustInt64(t, "SELECT count(*) FROM "+s.qualified("fhir_resource")+ + " WHERE project = $1 AND dataset_generation = '' AND payload IS NULL", ing01IngestProject) + if nulls != 1 { + t.Fatalf("SQL NULL payloads = %d, want exactly 1 (absent key)", nulls) + } + jsonNulls := s.mustInt64(t, "SELECT count(*) FROM "+s.qualified("fhir_resource")+ + " WHERE project = $1 AND dataset_generation = '' AND payload = 'null'::jsonb", ing01IngestProject) + if jsonNulls != 1 { + t.Fatalf("JSON null documents = %d, want exactly 1 (explicit null)", jsonNulls) + } + digest, resources, found := s.witnessLedger(t, ing01HandBatchID) + if !found || digest != ing01HandBatchDigest || resources != 3 { + t.Fatalf("legacy marker = (%s, %d, found %v), want the pinned digest and 3 rows", digest, resources, found) + } +} + +func TestPostgresResourceWriterReplayIdenticalBatchIsLedgerNoOp(t *testing.T) { + s := newIngest01IngestService(t) + s.migrate(t) + writer := s.newWriter(t) + + rows := []rowBuildResult{ + buildGeneratedRow(t, ing01IngestProject, ing01IngestGen, ing01IngestAuth, "ResearchSubject", ing01ResearchSubjectLine), + handBuiltVertex(ing01IngestProject, ing01IngestGen, "Observation", "replay-1", ing01IngestAuth, + `{"resourceType":"Observation","id":"replay-1"}`, nil), + } + scope := PGResourceBatchScope{ + BatchID: "ing01-ing/replay/0", Project: ing01IngestProject, + DatasetGeneration: ing01IngestGen, AuthResourcePath: ing01IngestAuth, + } + first, err := writer.WriteResourceBatch(s.ctx, scope, rows) + if err != nil { + s.fail("first WriteResourceBatch", err) + } + before := s.scopeFingerprint(t, ing01IngestProject, ing01IngestGen) + + for attempt := 0; attempt < 3; attempt++ { + replay, err := writer.WriteResourceBatch(s.ctx, scope, rows) + if err != nil { + s.fail("replay", err) + } + if !replay.Replay || replay.BatchID != first.BatchID || replay.Digest != first.Digest || + replay.Offered != first.Offered || replay.Committed != first.Committed || + replay.SkippedIdentical != first.SkippedIdentical { + t.Fatalf("replay attempt %d = %+v, want a byte-identical replay of %+v", attempt, replay, first) + } + } + if n := s.countIn(t, ing01IngestProject, ing01IngestGen); n != 2 { + t.Fatalf("rows after replays = %d, want 2 — replay must write nothing", n) + } + if fp := s.scopeFingerprint(t, ing01IngestProject, ing01IngestGen); fp != before { + t.Fatalf("fingerprint drifted across replays: %s != %s", fp, before) + } + if _, _, found := s.witnessLedger(t, scope.BatchID); !found { + t.Fatal("ledger marker missing after replays") + } +} + +func TestPostgresResourceWriterDuplicateNaturalIdentityDifferentPayloadRefused(t *testing.T) { + s := newIngest01IngestService(t) + s.migrate(t) + writer := s.newWriter(t) + + scope := PGResourceBatchScope{ + BatchID: "ing01-ing/dup/first", Project: ing01IngestProject, + DatasetGeneration: ing01IngestGen, AuthResourcePath: ing01IngestAuth, + } + rows := []rowBuildResult{ + buildGeneratedRow(t, ing01IngestProject, ing01IngestGen, ing01IngestAuth, "ResearchSubject", ing01ResearchSubjectLine), + } + if _, err := writer.WriteResourceBatch(s.ctx, scope, rows); err != nil { + s.fail("first commit", err) + } + stored := s.scopeFingerprint(t, ing01IngestProject, ing01IngestGen) + + changedLine := strings.Replace(ing01ResearchSubjectLine, `"status":"active"`, `"status":"withdrawn"`, 1) + second := []rowBuildResult{ + buildGeneratedRow(t, ing01IngestProject, ing01IngestGen, ing01IngestAuth, "ResearchSubject", changedLine), + handBuiltVertex(ing01IngestProject, ing01IngestGen, "Observation", "dup-extra-1", ing01IngestAuth, + `{"resourceType":"Observation","id":"dup-extra-1"}`, nil), + } + secondScope := scope + secondScope.BatchID = "ing01-ing/dup/second" + _, err := writer.WriteResourceBatch(s.ctx, secondScope, second) + var conflict *control.BatchResourceConflictError + if !errors.As(err, &conflict) { + s.fail("duplicate payload batch", fmt.Errorf("err = %v, want *control.BatchResourceConflictError", err)) + } + if !errors.Is(err, control.ErrBatchRowConflict) { + t.Fatalf("err does not match ErrBatchRowConflict: %v", err) + } + if conflict.InBatch || conflict.ResourceType != "ResearchSubject" || conflict.LogicalID != "research-subject-1" || + conflict.Project != ing01IngestProject || conflict.DatasetGeneration != ing01IngestGen { + t.Fatalf("conflict = %+v, want stored-row ResearchSubject/research-subject-1 under %s/%s", + conflict, ing01IngestProject, ing01IngestGen) + } + if fp := s.scopeFingerprint(t, ing01IngestProject, ing01IngestGen); fp != stored { + t.Fatalf("stored scope drifted after refused batch: %s != %s", fp, stored) + } + if n := s.mustInt64(t, "SELECT count(*) FROM "+s.qualified("fhir_resource")+ + " WHERE logical_id = 'dup-extra-1'"); n != 0 { + t.Fatalf("refused batch leaked its fresh row (%d), want whole-batch rollback", n) + } + if _, _, found := s.witnessLedger(t, secondScope.BatchID); found { + t.Fatal("refused batch left a ledger marker") + } + + thirdScope := scope + thirdScope.BatchID = "ing01-ing/dup/control" + res, err := writer.WriteResourceBatch(s.ctx, thirdScope, rows) + if err != nil { + s.fail("control re-offer", err) + } + if res.Committed != 0 || res.SkippedIdentical != 1 || res.Replay { + t.Fatalf("control = %+v, want 0 committed and 1 identical row skipped", res) + } +} + +func ing01GoodVertex() rowBuildResult { + return handBuiltVertex(ing01IngestProject, ing01IngestGen, "Observation", "refuse-ok", ing01IngestAuth, + `{"resourceType":"Observation","id":"refuse-ok"}`, nil) +} + +type ing01Refusal struct { + name string + scope PGResourceBatchScope + rows []rowBuildResult + scopeErr bool + field string + row int + column string + contains string +} + +func ing01RefusalCases(t *testing.T, goodScope PGResourceBatchScope, goodRows []rowBuildResult) []ing01Refusal { + overCapacity := make([]rowBuildResult, 0, control.MaxBatchResources+1) + for i := 0; i <= control.MaxBatchResources; i++ { + overCapacity = append(overCapacity, handBuiltVertex( + ing01IngestProject, ing01IngestGen, "Observation", fmt.Sprintf("over-%d", i), ing01IngestAuth, + `{"resourceType":"Observation","id":"over"}`, nil)) + } + return []ing01Refusal{ + {name: "blank batch id", + scope: PGResourceBatchScope{BatchID: " ", Project: ing01IngestProject, DatasetGeneration: ing01IngestGen, AuthResourcePath: ing01IngestAuth}, + rows: goodRows, scopeErr: true, field: "batch id", contains: "blank"}, + {name: "over-long batch id", + scope: PGResourceBatchScope{BatchID: strings.Repeat("x", control.MaxBatchIDLength+1), Project: ing01IngestProject, DatasetGeneration: ing01IngestGen, AuthResourcePath: ing01IngestAuth}, + rows: goodRows, scopeErr: true, field: "batch id", contains: "at most 255"}, + {name: "blank project", + scope: PGResourceBatchScope{BatchID: "ing01-ing/refuse/project", Project: " ", DatasetGeneration: ing01IngestGen, AuthResourcePath: ing01IngestAuth}, + rows: goodRows, scopeErr: true, field: "project", contains: "blank"}, + {name: "empty batch", + scope: goodScope, rows: nil, scopeErr: true, field: "rows", contains: "no rows"}, + {name: "over capacity", + scope: PGResourceBatchScope{BatchID: "ing01-ing/refuse/over", Project: ing01IngestProject, DatasetGeneration: ing01IngestGen, AuthResourcePath: ing01IngestAuth}, + rows: overCapacity, scopeErr: true, field: "rows", contains: "at most 1000"}, + {name: "vertex project mismatch", scope: goodScope, + rows: []rowBuildResult{replaceFieldDirect(t, ing01GoodVertex(), "project", json.RawMessage(`"wrong-project"`))}, + row: 0, column: "project", contains: "does not match the batch project"}, + {name: "vertex project_id missing", scope: goodScope, + rows: []rowBuildResult{dropFieldDirect(t, ing01GoodVertex(), "project_id")}, + row: 0, column: "project_id", contains: "missing project_id"}, + {name: "vertex project_id mismatch", scope: goodScope, + rows: []rowBuildResult{replaceFieldDirect(t, ing01GoodVertex(), "project_id", json.RawMessage(`"other-project"`))}, + row: 0, column: "project_id", contains: "does not match the batch project"}, + {name: "missing generation stamp", scope: goodScope, + rows: []rowBuildResult{dropFieldDirect(t, ing01GoodVertex(), "dataset_generation")}, + row: 0, column: "dataset_generation", contains: "no dataset_generation stamp"}, + {name: "generation stamp mismatch", scope: goodScope, + rows: []rowBuildResult{replaceFieldDirect(t, ing01GoodVertex(), "dataset_generation", json.RawMessage(`"gen-other"`))}, + row: 0, column: "dataset_generation", contains: "does not match the batch generation"}, + {name: "stamped row under legacy batch", + scope: PGResourceBatchScope{BatchID: "ing01-ing/refuse/legacy", Project: ing01IngestProject}, + rows: []rowBuildResult{ing01GoodVertex()}, + row: 0, column: "dataset_generation", contains: "legacy (unversioned) namespace"}, + {name: "blank resource type", scope: goodScope, + rows: []rowBuildResult{replaceFieldDirect(t, ing01GoodVertex(), "resourceType", json.RawMessage(`" "`))}, + row: 0, column: "resourceType", contains: "resourceType must be a non-empty string"}, + {name: "blank logical id", scope: goodScope, + rows: []rowBuildResult{replaceFieldDirect(t, ing01GoodVertex(), "id", json.RawMessage(`""`))}, + row: 0, column: "id", contains: "id must be a non-empty string"}, + {name: "auth path mismatch", scope: goodScope, + rows: []rowBuildResult{replaceFieldDirect(t, ing01GoodVertex(), "auth_resource_path", json.RawMessage(`"/other"`))}, + row: 0, column: "auth_resource_path", contains: "does not match the batch auth path"}, + {name: "missing auth key under scoped batch", scope: goodScope, + rows: []rowBuildResult{dropFieldDirect(t, ing01GoodVertex(), "auth_resource_path")}, + row: 0, column: "auth_resource_path", contains: "no auth_resource_path"}, + {name: "undecodable vertex", scope: goodScope, + rows: []rowBuildResult{{vertex: json.RawMessage(`{"broken":`)}}, + row: 0, column: "vertex", contains: "undecodable"}, + } +} + +func TestPostgresResourceWriterRejectsInvalidScopeAndIdentityBeforeAnyRow(t *testing.T) { + recorder := &ing01Recorder{} + recWriter := &PostgresResourceWriter{committer: recorder} + + goodScope := PGResourceBatchScope{ + BatchID: "ing01-ing/refuse", Project: ing01IngestProject, + DatasetGeneration: ing01IngestGen, AuthResourcePath: ing01IngestAuth, + } + goodRows := []rowBuildResult{ing01GoodVertex()} + + for _, tc := range ing01RefusalCases(t, goodScope, goodRows) { + _, err := recWriter.WriteResourceBatch(context.Background(), tc.scope, tc.rows) + if !errors.Is(err, ErrInvalidPGResourceWrite) { + t.Fatalf("%s: err = %v, want ErrInvalidPGResourceWrite", tc.name, err) + } + if tc.scopeErr { + var scopeErr *PGResourceWriteScopeError + if !errors.As(err, &scopeErr) { + t.Fatalf("%s: err = %v, want *PGResourceWriteScopeError", tc.name, err) + } + if scopeErr.Field != tc.field || !strings.Contains(scopeErr.Reason, tc.contains) { + t.Fatalf("%s: scope error = %+v, want field %q naming %q", tc.name, scopeErr, tc.field, tc.contains) + } + } else { + var rowErr *PGResourceRowIdentityError + if !errors.As(err, &rowErr) { + t.Fatalf("%s: err = %v, want *PGResourceRowIdentityError", tc.name, err) + } + if rowErr.Row != tc.row || rowErr.Column != tc.column || + !strings.Contains(rowErr.Reason, tc.contains) { + t.Fatalf("%s: row error = %+v, want row %d column %q naming %q", + tc.name, rowErr, tc.row, tc.column, tc.contains) + } + } + if _, err := recWriter.WriteResourceBatch(context.Background(), goodScope, goodRows); err != nil { + t.Fatalf("%s: control batch refused: %v", tc.name, err) + } + } + + cases := ing01RefusalCases(t, goodScope, goodRows) + if got := recorder.callCount(); got != len(cases) { + t.Fatalf("committer calls = %d, want %d (controls only — every refusal must stop before the store)", + got, len(cases)) + } + + s := newIngest01IngestService(t) + s.migrate(t) + live := s.newWriter(t) + liveCases := []struct { + name string + rows []rowBuildResult + batch string + }{ + {"project mismatch live", []rowBuildResult{replaceFieldDirect(t, ing01GoodVertex(), "project", json.RawMessage(`"wrong-project"`))}, "live-project-mismatch"}, + {"missing stamp live", []rowBuildResult{dropFieldDirect(t, ing01GoodVertex(), "dataset_generation")}, "live-missing-stamp"}, + {"undecodable live", []rowBuildResult{{vertex: json.RawMessage(`[1,2,3]`)}}, "live-undecodable"}, + } + for _, lc := range liveCases { + scope := goodScope + scope.BatchID = "ing01-ing/refuse/" + lc.batch + _, err := live.WriteResourceBatch(s.ctx, scope, lc.rows) + if !errors.Is(err, ErrInvalidPGResourceWrite) { + t.Fatalf("%s: live err = %v, want ErrInvalidPGResourceWrite", lc.name, err) + } + if n := s.countIn(t, ing01IngestProject, ing01IngestGen); n != 0 { + t.Fatalf("%s: live scope holds %d rows, want 0 — refusal must precede any write", lc.name, n) + } + if _, _, found := s.witnessLedger(t, scope.BatchID); found { + t.Fatalf("%s: refused batch left a ledger marker", lc.name) + } + } + + scope := goodScope + scope.BatchID = "ing01-ing/refuse/live-control" + res, err := live.WriteResourceBatch(s.ctx, scope, []rowBuildResult{ + handBuiltVertex(ing01IngestProject, ing01IngestGen, "Observation", "refuse-ok", ing01IngestAuth, + `{"resourceType":"Observation","id":"refuse-ok"}`, nil)}) + if err != nil { + s.fail("live control commit", err) + } + if res.Committed != 1 || res.Replay { + t.Fatalf("live control = %+v, want a fresh single-row commit", res) + } + if n := s.countIn(t, ing01IngestProject, ing01IngestGen); n != 1 { + t.Fatalf("rows after live control = %d, want 1", n) + } +} + +func replaceFieldDirect(t *testing.T, built rowBuildResult, field string, rawValue json.RawMessage) rowBuildResult { + t.Helper() + return replaceField(t, built, field, rawValue) +} + +func dropFieldDirect(t *testing.T, built rowBuildResult, field string) rowBuildResult { + t.Helper() + return dropField(t, built, field) +} + +func TestPostgresResourceWriterConcurrentIdenticalBatchCommitsOnceTwoConnections(t *testing.T) { + const rounds = 12 + s := newIngest01IngestService(t) + s.migrate(t) + + clientA := s.openAppClient(t, "ing01-ing-race-a", 1) + clientB := s.openAppClient(t, "ing01-ing-race-b", 1) + pidA := s.poolBackendPID(t, clientA) + pidB := s.poolBackendPID(t, clientB) + if pidA == 0 || pidB == 0 || pidA == pidB { + t.Fatalf("race pools share backend %d/%d, want distinct real connections", pidA, pidB) + } + writerA := s.newWriterOn(t, clientA) + writerB := s.newWriterOn(t, clientB) + + for round := 0; round < rounds; round++ { + generation := fmt.Sprintf("gen-race-%d", round) + rows := []rowBuildResult{ + buildGeneratedRow(t, ing01IngestProject, generation, ing01IngestAuth, "ResearchSubject", ing01ResearchSubjectLine), + handBuiltVertex(ing01IngestProject, generation, "Observation", fmt.Sprintf("race-obs-%d", round), ing01IngestAuth, + `{"resourceType":"Observation"}`, nil), + } + scope := PGResourceBatchScope{ + BatchID: fmt.Sprintf("ing01-ing/race/%d", round), Project: ing01IngestProject, + DatasetGeneration: generation, AuthResourcePath: ing01IngestAuth, + } + + start := make(chan struct{}) + type outcome struct { + res PGResourceWriteResult + err error + } + ra := make(chan outcome, 1) + rb := make(chan outcome, 1) + go func() { <-start; res, err := writerA.WriteResourceBatch(s.ctx, scope, rows); ra <- outcome{res, err} }() + go func() { <-start; res, err := writerB.WriteResourceBatch(s.ctx, scope, rows); rb <- outcome{res, err} }() + close(start) + o1, o2 := <-ra, <-rb + if o1.err != nil { + s.fail(fmt.Sprintf("round %d connection A", round), o1.err) + } + if o2.err != nil { + s.fail(fmt.Sprintf("round %d connection B", round), o2.err) + } + + if o1.res.BatchID != o2.res.BatchID || o1.res.Digest != o2.res.Digest || + o1.res.Offered != o2.res.Offered || o1.res.Committed != o2.res.Committed || + o1.res.SkippedIdentical != o2.res.SkippedIdentical { + t.Fatalf("round %d results diverged: %+v vs %+v", round, o1.res, o2.res) + } + if !o1.res.Replay && !o2.res.Replay || o1.res.Replay && o2.res.Replay { + t.Fatalf("round %d replays = (%v, %v), want exactly one fresh commit", round, o1.res.Replay, o2.res.Replay) + } + if o1.res.Committed != 2 || o1.res.Offered != 2 || o1.res.SkippedIdentical != 0 || o1.res.Digest == "" { + t.Fatalf("round %d shared accounting = %+v, want 2 committed under a real digest", round, o1.res) + } + + if n := s.countIn(t, ing01IngestProject, generation); n != 2 { + t.Fatalf("round %d rows = %d, want exactly 2 (committed once)", round, n) + } + digest, resources, found := s.witnessLedger(t, scope.BatchID) + if !found || digest != o1.res.Digest || resources != 2 { + t.Fatalf("round %d marker = (%s, %d, found %v), want the shared digest with 2 rows", + round, digest, resources, found) + } + } +} + +func TestPostgresResourceWriterReportedCountsChangeOnlyAtCommitTwoConnections(t *testing.T) { + s := newIngest01IngestService(t) + s.migrate(t) + + writerClient := s.openAppClient(t, "ing01-ing-commit-w", 1) + writer := s.newWriterOn(t, writerClient) + writerPID := s.poolBackendPID(t, writerClient) + + observer, err := pgxpool.New(s.ctx, s.dsn) + if err != nil { + s.fail("open observer pool", err) + } + t.Cleanup(observer.Close) + obsConn, err := observer.Acquire(s.ctx) + if err != nil { + s.fail("acquire observer connection", err) + } + defer obsConn.Release() + var observerPID int64 + if err := obsConn.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&observerPID); err != nil { + s.fail("witness observer pid", err) + } + if observerPID == writerPID { + t.Fatalf("observer and writer share backend %d, want two distinct real PG connections", observerPID) + } + + const rounds = 12 + for round := 0; round < rounds; round++ { + generation := fmt.Sprintf("gen-visibility-%d", round) + rows := []rowBuildResult{ + handBuiltVertex(ing01IngestProject, generation, "Observation", fmt.Sprintf("vis-%d", round), ing01IngestAuth, + `{"resourceType":"Observation"}`, nil), + } + scope := PGResourceBatchScope{ + BatchID: fmt.Sprintf("ing01-ing/visibility/%d", round), Project: ing01IngestProject, + DatasetGeneration: generation, AuthResourcePath: ing01IngestAuth, + } + + if _, err := obsConn.Exec(s.ctx, "BEGIN"); err != nil { + s.fail("observer begin", err) + } + if _, err := obsConn.Exec(s.ctx, "LOCK TABLE "+s.qualified("fhir_resource")+" IN ACCESS EXCLUSIVE MODE"); err != nil { + s.fail("observer lock", err) + } + + type outcome struct { + res PGResourceWriteResult + err error + } + done := make(chan outcome, 1) + go func() { + res, err := writer.WriteResourceBatch(context.Background(), scope, rows) + done <- outcome{res, err} + }() + + blockedPID := s.waitBackendWaiting(t, "ing01-ing-commit-w") + if blockedPID != writerPID || blockedPID == observerPID { + t.Fatalf("round %d blocked backend pid = %d, want the writer backend %d (observer is %d)", + round, blockedPID, writerPID, observerPID) + } + + var midRows int64 + if err := obsConn.QueryRow(s.ctx, "SELECT count(*) FROM "+s.qualified("fhir_resource")+ + " WHERE project = $1 AND dataset_generation = $2", ing01IngestProject, generation). + Scan(&midRows); err != nil { + s.fail("mid-transaction row witness", err) + } + if midRows != 0 { + t.Fatalf("round %d rows visible mid-transaction = %d, want 0 — counts may only change at commit", round, midRows) + } + if _, _, found := s.witnessLedger(t, scope.BatchID); found { + t.Fatalf("round %d ledger marker visible mid-transaction, want none", round) + } + + if _, err := obsConn.Exec(s.ctx, "COMMIT"); err != nil { + s.fail("observer commit", err) + } + var o outcome + select { + case o = <-done: + case <-time.After(60 * time.Second): + s.fail("waiting for batch commit", fmt.Errorf("round %d batch never completed after the lock was released", round)) + } + if o.err != nil { + s.fail(fmt.Sprintf("round %d WriteResourceBatch", round), o.err) + } + if o.res.Committed != 1 || o.res.Replay { + t.Fatalf("round %d result = %+v, want a fresh single-row commit", round, o.res) + } + if n := s.countIn(t, ing01IngestProject, generation); n != int64(o.res.Committed) { + t.Fatalf("round %d rows after commit = %d, want the reported %d", round, n, o.res.Committed) + } + if _, resources, found := s.witnessLedger(t, scope.BatchID); !found || resources != 1 { + t.Fatalf("round %d marker missing or miscounted after commit (found %v, resources %d)", round, found, resources) + } + } +} + +func TestZZZeroLeftoverIngestPostgresResourceWriterFixtures(t *testing.T) { + raw := ing01TestPostgresURL(t) + ing01IngestOwnedMu.Lock() + owned := make(map[string]string, len(ing01IngestOwnedByName)) + for db, schema := range ing01IngestOwnedByName { + owned[db] = schema + } + ing01IngestOwnedMu.Unlock() + if len(owned) == 0 { + t.Skip("this process created no verify_ing01_ing_* fixtures") + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service for audit: %v", err) + } + defer admin.Close() + + for db, schema := range owned { + if !strings.HasPrefix(db, ing01IngestPrefix) { + t.Fatalf("registry holds foreign database %q", db) + } + var count int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Errorf("audit owned database %s: %v", db, err) + continue + } + if count != 0 { + t.Errorf("leftover owned database %s after all cleanups", db) + if schema != "" { + u, err := url.Parse(raw) + if err == nil { + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + db + if probe, err := pgxpool.New(ctx, u.String()); err == nil { + var schemaCount int + if err := probe.QueryRow(ctx, "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1", schema).Scan(&schemaCount); err != nil { + t.Errorf("audit owned schema %s in %s: %v", schema, db, err) + } else if schemaCount != 0 { + t.Errorf("leftover owned schema %s in leftover database %s", schema, db) + } + probe.Close() + } + } + } + } + } + + var visible int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_ing01\\_ing\\_%' ESCAPE '\\'"). + Scan(&visible); err != nil { + t.Fatalf("audit verify_ing01_ing_* database count: %v", err) + } + if visible != ing01IngestBaselineCount { + t.Errorf("verify_ing01_ing_* databases = %d, want baseline %d", visible, ing01IngestBaselineCount) + } +} + +const ( + ing02IngestPrefix = "verify_ing02_ing_" + ing02IngestFmt = ing02IngestPrefix + "%d_%s" + + ing02IngestProject = "prj-ing02-ing" + ing02IngestGen = "gen-build-a" + ing02IngestAuth = "/fhir" + + ing02GenericDupFocusLine = `{"resourceType":"Observation","id":"obs-generic","status":"final","code":{"text":"dup focus"},"focus":[{"reference":"Observation/focus-1"},{"reference":"Observation/focus-1"}]}` + ing02GeneratedDupFocusLine = `{"resourceType":"Observation","id":"obs-generated","status":"final","code":{"text":"dup focus"},"focus":[{"reference":"Observation/focus-1"},{"reference":"Observation/focus-1"}],"subject":{"reference":"Patient/patient-1"}}` + ing02FocusTargetLine = `{"resourceType":"Observation","id":"focus-1","status":"final","code":{"text":"target"}}` + ing02PatientLine = `{"resourceType":"Patient","id":"patient-1","gender":"male"}` + + ing02GenericCrossTypeLine = `{"resourceType":"Observation","id":"obs-cross","status":"final","code":{"text":"cross"},"focus":[{"reference":"Observation/focus-1"}],"subject":{"reference":"Patient/patient-1"}}` + + ing02GeneratedCaseRefLine = `{"resourceType":"Observation","id":"obs-case","status":"final","code":{"text":"case"},"subject":{"reference":"patient/patient-1"}}` + + ing02KeyCaseSubjectFwd = "b530ab0f-b761-56aa-ae92-e3fd0298c2c4" + ing02KeyCaseSubjectBack = "3b338272-bc4e-5ca6-9ea9-e75862e04da8" +) + +var ing02IngestNamePattern = regexp.MustCompile(`^verify_ing02_ing_[0-9]+_[0-9a-f]+$`) + +var ( + ing02IngestOwnedMu sync.Mutex + ing02IngestOwnedByName = map[string]string{} +) + +func ing02IngestRegister(db, schema string) { + ing02IngestOwnedMu.Lock() + defer ing02IngestOwnedMu.Unlock() + ing02IngestOwnedByName[db] = schema +} + +var ( + ing02IngestBaselineOnce sync.Once + ing02IngestBaselineCount int64 + ing02IngestBaselineErr error +) + +func ing02IngestCaptureBaseline(raw string) { + ing02IngestBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + ing02IngestBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_ing02\\_ing\\_%' ESCAPE '\\'"). + Scan(&count); err != nil { + ing02IngestBaselineErr = fmt.Errorf("count baseline: %w", err) + return + } + ing02IngestBaselineCount = count + }) +} + +type ing02IngestService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + dsn string + schema string + scrub func(any) string + fail func(step string, err error) +} + +func newIngest02IngestService(t *testing.T) *ing02IngestService { + t.Helper() + raw := ing01TestPostgresURL(t) + ing02IngestCaptureBaseline(raw) + if ing02IngestBaselineErr != nil { + t.Fatalf("capture pg_database baseline: %v", ing02IngestBaselineErr) + } + + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + dbName := fmt.Sprintf(ing02IngestFmt, time.Now().UnixNano(), ing01RandomHex(t)) + if !ing02IngestNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(ing02IngestFmt, time.Now().UnixNano(), ing01RandomHex(t)) + if !ing02IngestNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + ing02IngestRegister(dbName, schema) + return &ing02IngestService{ctx: ctx, admin: admin, iso: iso, dsn: dsn, schema: schema, scrub: scrub, fail: fail} +} + +func (s *ing02IngestService) migrate(t *testing.T) { + t.Helper() + client := s.openClient(t, 4) + if _, err := control.MigrateSchema(s.ctx, client, s.schema); err != nil { + s.fail("MigrateSchema", err) + } +} + +func (s *ing02IngestService) openClient(t *testing.T, maxConns int32) *control.Client { + t.Helper() + client, err := control.OpenPool(s.ctx, control.ClientConfig{ + DSN: s.dsn, + MaxConns: maxConns, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + s.fail("open client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *ing02IngestService) openAppClient(t *testing.T, app string, maxConns int32) *control.Client { + t.Helper() + u, err := url.Parse(s.dsn) + if err != nil { + s.fail("parse isolated dsn for app client", err) + } + q := u.Query() + q.Set("application_name", app) + u.RawQuery = q.Encode() + client, err := control.OpenPool(s.ctx, control.ClientConfig{ + DSN: u.String(), + MaxConns: maxConns, + MinConns: 0, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + s.fail("open app client pool", err) + } + t.Cleanup(client.Close) + return client +} + +func (s *ing02IngestService) newEdgeWriterOn(t *testing.T, src control.ConnSource) *PostgresEdgeWriter { + t.Helper() + writer, err := NewPostgresEdgeWriter(src, s.schema) + if err != nil { + s.fail("new postgres edge writer", err) + } + return writer +} + +func (s *ing02IngestService) newEdgeWriter(t *testing.T) *PostgresEdgeWriter { + t.Helper() + return s.newEdgeWriterOn(t, s.openClient(t, 4)) +} + +func (s *ing02IngestService) newResourceWriter(t *testing.T) *PostgresResourceWriter { + t.Helper() + writer, err := NewPostgresResourceWriter(s.openClient(t, 4), s.schema) + if err != nil { + s.fail("new postgres resource writer", err) + } + return writer +} + +func (s *ing02IngestService) qualified(table string) string { + return pgx.Identifier{s.schema, table}.Sanitize() +} + +func (s *ing02IngestService) mustInt64(t *testing.T, sql string, args ...any) int64 { + t.Helper() + var v int64 + if err := s.iso.QueryRow(s.ctx, sql, args...).Scan(&v); err != nil { + s.fail("witness "+sql, err) + } + return v +} + +func (s *ing02IngestService) countEdges(t *testing.T, project, generation string) int64 { + t.Helper() + return s.mustInt64(t, "SELECT count(*) FROM "+s.qualified("fhir_edge_occurrence")+ + " WHERE project = $1 AND dataset_generation = $2", project, generation) +} + +func (s *ing02IngestService) requireEdgeRow(t *testing.T, project, generation, authPath, label, + srcType, srcID, tgtType, tgtID, sourcePath string, sourceOrdinal int64, wantKey *string, +) { + t.Helper() + var gotAuth string + var keyPresent bool + var key *string + err := s.iso.QueryRow(s.ctx, + "SELECT auth_resource_path, logical_edge_key IS NOT NULL, logical_edge_key FROM "+s.qualified("fhir_edge_occurrence")+ + " WHERE project = $1 AND dataset_generation = $2 AND source_resource_type = $3"+ + " AND source_logical_id = $4 AND label = $5 AND target_resource_type = $6"+ + " AND target_logical_id = $7 AND source_path = $8 AND source_ordinal = $9", + project, generation, srcType, srcID, label, tgtType, tgtID, sourcePath, sourceOrdinal). + Scan(&gotAuth, &keyPresent, &key) + if err != nil { + s.fail(fmt.Sprintf("witness occurrence %s/%s -%s-> %s/%s at %q#%d", + srcType, srcID, label, tgtType, tgtID, sourcePath, sourceOrdinal), err) + } + if gotAuth != authPath { + t.Fatalf("occurrence %s/%s -%s-> %s/%s auth = %q, want %q", srcType, srcID, label, tgtType, tgtID, gotAuth, authPath) + } + if (wantKey != nil) != keyPresent { + t.Fatalf("occurrence %s/%s -%s-> %s/%s key presence = %v, want %v", + srcType, srcID, label, tgtType, tgtID, keyPresent, wantKey != nil) + } + if wantKey != nil && *key != *wantKey { + t.Fatalf("occurrence %s/%s -%s-> %s/%s key = %q, want %q", + srcType, srcID, label, tgtType, tgtID, *key, *wantKey) + } +} + +func (s *ing02IngestService) witnessEdgeLedger(t *testing.T, batchID string) (digest string, edges int64, found bool) { + t.Helper() + err := s.iso.QueryRow(s.ctx, + "SELECT digest, edge_count FROM "+s.qualified("batch_write_ledger")+" WHERE batch_id = $1", batchID). + Scan(&digest, &edges) + if errors.Is(err, pgx.ErrNoRows) { + return "", 0, false + } + if err != nil { + s.fail("witness ledger row", err) + } + return digest, edges, true +} + +func (s *ing02IngestService) edgeFingerprint(t *testing.T, project, generation string) string { + t.Helper() + var fp string + sql := "SELECT coalesce(md5(string_agg(entry, E'\n' ORDER BY edge_seq)),'EMPTY') FROM (" + + "SELECT edge_seq, source_resource_type || '|' || source_logical_id || '|' || label || '|' ||" + + " target_resource_type || '|' || target_logical_id || '|' || source_path || '|' ||" + + " source_ordinal::text || '|' || auth_resource_path || '|' || coalesce(logical_edge_key, '') AS entry" + + " FROM " + s.qualified("fhir_edge_occurrence") + " WHERE project = $1 AND dataset_generation = $2) x" + if err := s.iso.QueryRow(s.ctx, sql, project, generation).Scan(&fp); err != nil { + s.fail("edge fingerprint", err) + } + return fp +} + +func (s *ing02IngestService) poolBackendPID(t *testing.T, client *control.Client) int64 { + t.Helper() + conn, err := client.Acquire(s.ctx) + if err != nil { + s.fail("acquire for backend pid", err) + } + defer conn.Release() + var pid int64 + if err := conn.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&pid); err != nil { + s.fail("witness pg_backend_pid", err) + } + return pid +} + +func (s *ing02IngestService) waitBackendWaiting(t *testing.T, app string) int64 { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for { + var pid int64 + var waitType, waitEvent *string + err := s.iso.QueryRow(s.ctx, + "SELECT pid, wait_event_type, wait_event FROM pg_stat_activity"+ + " WHERE application_name = $1 AND state = 'active'", app). + Scan(&pid, &waitType, &waitEvent) + if err == nil && waitType != nil && *waitType == "Lock" { + return pid + } + if time.Now().After(deadline) { + s.fail("waiting for backend "+app+" to block on a lock", + fmt.Errorf("backend never entered a lock wait")) + } + time.Sleep(10 * time.Millisecond) + } +} + +func ing02DupFocusRows(t *testing.T, authPath string) []rowBuildResult { + t.Helper() + return []rowBuildResult{ + buildGenericRow(t, ing02IngestProject, ing02IngestGen, authPath, "Observation", ing02GenericDupFocusLine), + buildGeneratedRow(t, ing02IngestProject, ing02IngestGen, authPath, "Observation", ing02GeneratedDupFocusLine), + buildGeneratedRow(t, ing02IngestProject, ing02IngestGen, authPath, "Patient", ing02PatientLine), + buildGeneratedRow(t, ing02IngestProject, ing02IngestGen, authPath, "Observation", ing02FocusTargetLine), + } +} + +func handBuiltEdgeDocument(from, to, label, fromType, toType, project, generation, auth string) json.RawMessage { + var b strings.Builder + b.WriteString(`{"_key":"`) + b.WriteString(SanitizeKey(from) + "_" + SanitizeKey(to) + "_" + label) + b.WriteString(`","_from":"`) + b.WriteString(from) + b.WriteString(`","_to":"`) + b.WriteString(to) + b.WriteString(`","label":"`) + b.WriteString(label) + b.WriteString(`","project":"`) + b.WriteString(project) + b.WriteString(`","from_type":"`) + b.WriteString(fromType) + b.WriteString(`","to_type":"`) + b.WriteString(toType) + b.WriteString(`"`) + if generation != "" { + b.WriteString(`,"dataset_generation":"`) + b.WriteString(generation) + b.WriteString(`"`) + } + if auth != "" { + b.WriteString(`,"auth_resource_path":"`) + b.WriteString(auth) + b.WriteString(`"`) + } + b.WriteString(`}`) + return json.RawMessage(b.String()) +} + +func replaceEdges(t *testing.T, built rowBuildResult, edges ...json.RawMessage) rowBuildResult { + t.Helper() + built.edges = edges + return built +} + +func ing02LegacyRefRow(t *testing.T) rowBuildResult { + t.Helper() + built := handBuiltVertex(ing02IngestProject, "", "Observation", "ref-1", "", + `{"resourceType":"Observation","id":"ref-1","status":"final",`+ + `"focus":[{"reference":"Observation/focus-1"},{"reference":"Observation/focus-1"}],`+ + `"subject":{"reference":"Patient/patient-1"}}`, nil) + fwd := handBuiltEdgeDocument("Observation/ref-1", "Observation/focus-1", "focus_Observation", "Observation", "Observation", ing02IngestProject, "", "") + rev := handBuiltEdgeDocument("Observation/focus-1", "Observation/ref-1", "focus_observation", "Observation", "Observation", ing02IngestProject, "", "") + subj := handBuiltEdgeDocument("Observation/ref-1", "Patient/patient-1", "subject_Patient", "Observation", "Patient", ing02IngestProject, "", "") + subjRev := handBuiltEdgeDocument("Patient/patient-1", "Observation/ref-1", "subject_observation", "Patient", "Observation", ing02IngestProject, "", "") + return replaceEdges(t, built, fwd, rev, subj, subjRev) +} + +type ing02Recorder struct { + mu sync.Mutex + calls int + batches []control.EdgeBatch +} + +func (r *ing02Recorder) WriteEdgeBatch(_ context.Context, batch control.EdgeBatch) (control.EdgeBatchResult, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.calls++ + r.batches = append(r.batches, batch) + return control.EdgeBatchResult{ + BatchID: batch.ID, + Digest: "recorder-digest", + Committed: int64(len(batch.Rows)), + }, nil +} + +func (r *ing02Recorder) callCount() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.calls +} + +func (r *ing02Recorder) lastBatch(t *testing.T) control.EdgeBatch { + t.Helper() + r.mu.Lock() + defer r.mu.Unlock() + if r.calls == 0 { + t.Fatal("recorder has no recorded batch") + } + return r.batches[r.calls-1] +} + +const ( + ing02DupBatchID = "ing02-ing/duplicate/0" + ing02DupBatchDigest = "f57c41f7895b2bcc2deef22b140d1d2a17884503bff90040ffea55fe5233955d" + + ing02KeyGenericFocusFwd = "4e618179-60ea-5808-b9cd-4a3ec93e0ef2" + ing02KeyGenericFocusRev = "df2a2588-3ed1-5a05-853d-657b040b6d95" + ing02KeyGenFocusFwd = "07e79b4d-7f23-5a35-a6f0-bf4d75251c3a" + ing02KeyGenFocusRev = "52c9d7b4-48ab-5de4-8678-28b43c0fdf53" + ing02KeyGenSubjectFwd = "fa3062be-d9ac-5e3b-8613-42b02cfae2e8" + ing02KeyGenSubjectRev = "e9a7804a-a8d3-5d34-a32d-789c9a8946fa" +) + +func ing02ExpectedOccurrences(authPath string) []control.EdgeOccurrenceRow { + mk := func(label, srcType, srcID, tgtType, tgtID, path string, ord int64, key string) control.EdgeOccurrenceRow { + k := key + return control.EdgeOccurrenceRow{ + Project: ing02IngestProject, DatasetGeneration: ing02IngestGen, AuthResourcePath: authPath, + Label: label, SourceResourceType: srcType, SourceLogicalID: srcID, + TargetResourceType: tgtType, TargetLogicalID: tgtID, + SourcePath: path, SourceOrdinal: ord, LogicalEdgeKey: &k, + } + } + return []control.EdgeOccurrenceRow{ + mk("focus_Observation", "Observation", "obs-generic", "Observation", "focus-1", "focus", 0, ing02KeyGenericFocusFwd), + mk("focus_Observation", "Observation", "obs-generic", "Observation", "focus-1", "focus", 1, ing02KeyGenericFocusFwd), + mk("focus_observation", "Observation", "focus-1", "Observation", "obs-generic", "focus", 0, ing02KeyGenericFocusRev), + mk("focus_observation", "Observation", "focus-1", "Observation", "obs-generic", "focus", 1, ing02KeyGenericFocusRev), + mk("focus_Observation", "Observation", "obs-generated", "Observation", "focus-1", "focus", 0, ing02KeyGenFocusFwd), + mk("focus_observation", "Observation", "focus-1", "Observation", "obs-generated", "focus", 0, ing02KeyGenFocusRev), + mk("subject_Patient", "Observation", "obs-generated", "Patient", "patient-1", "subject", 0, ing02KeyGenSubjectFwd), + mk("subject_observation", "Patient", "patient-1", "Observation", "obs-generated", "subject", 0, ing02KeyGenSubjectRev), + } +} + +func TestPostgresEdgeWriterPersistsOccurrencesEndToEndWithExactRows(t *testing.T) { + s := newIngest02IngestService(t) + s.migrate(t) + rows := ing02DupFocusRows(t, ing02IngestAuth) + + scope := PGResourceBatchScope{ + BatchID: "ing02-ing/resources/0", Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + } + if _, err := s.newResourceWriter(t).WriteResourceBatch(s.ctx, scope, rows); err != nil { + s.fail("WriteResourceBatch", err) + } + + edgeWriter := s.newEdgeWriter(t) + edgeScope := PGEdgeBatchScope{ + BatchID: ing02DupBatchID, Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + } + res, err := edgeWriter.WriteEdgeBatch(s.ctx, edgeScope, rows) + if err != nil { + s.fail("WriteEdgeBatch", err) + } + if res.BatchID != ing02DupBatchID || res.Rows != 4 || res.Offered != 8 || + res.Committed != 8 || res.SkippedIdentical != 0 || res.Replay { + t.Fatalf("result = %+v, want a fresh 8-occurrence commit over 4 rows", res) + } + if res.Digest != ing02DupBatchDigest { + t.Fatalf("digest = %s, want the golden %s", res.Digest, ing02DupBatchDigest) + } + + for _, want := range ing02ExpectedOccurrences(ing02IngestAuth) { + s.requireEdgeRow(t, want.Project, want.DatasetGeneration, want.AuthResourcePath, want.Label, + want.SourceResourceType, want.SourceLogicalID, want.TargetResourceType, want.TargetLogicalID, + want.SourcePath, want.SourceOrdinal, want.LogicalEdgeKey) + } + if n := s.countEdges(t, ing02IngestProject, ing02IngestGen); n != 8 { + t.Fatalf("edge rows = %d, want 8", n) + } + distinct := s.mustInt64(t, "SELECT count(DISTINCT edge_seq) FROM "+s.qualified("fhir_edge_occurrence")+ + " WHERE project = $1 AND dataset_generation = $2", ing02IngestProject, ing02IngestGen) + if distinct != 8 { + t.Fatalf("distinct edge_seq = %d, want 8 — occurrences must be independently addressable", distinct) + } + digest, edges, found := s.witnessEdgeLedger(t, ing02DupBatchID) + if !found || digest != ing02DupBatchDigest || edges != 8 { + t.Fatalf("marker = (%s, %d, found %v), want the golden digest with 8 rows", digest, edges, found) + } + + mutatedLine := strings.Replace(ing02GeneratedDupFocusLine, + `{"reference":"Observation/focus-1"}],"subject"`, + `{"reference":"Observation/focus-other"}],"subject"`, 1) + mutated := ing02DupFocusRows(t, ing02IngestAuth) + mutated[1] = buildGeneratedRow(t, ing02IngestProject, ing02IngestGen, ing02IngestAuth, "Observation", mutatedLine) + _, err = edgeWriter.WriteEdgeBatch(s.ctx, edgeScope, mutated) + var conflict *control.BatchIdentityConflictError + if !errors.As(err, &conflict) || !errors.Is(err, control.ErrBatchIdentityConflict) { + t.Fatalf("mutated replay under the golden identity = %v, want *BatchIdentityConflictError", err) + } + if conflict.LedgerDigest != ing02DupBatchDigest || conflict.ProvidedDigest == ing02DupBatchDigest { + t.Fatalf("conflict digests = (%s, %s), want the golden ledger digest against a changed digest", + conflict.LedgerDigest, conflict.ProvidedDigest) + } + if n := s.countEdges(t, ing02IngestProject, ing02IngestGen); n != 8 { + t.Fatalf("edge rows after refused mutated replay = %d, want the original 8", n) + } +} + +func TestPostgresEdgeWriterCommitmentToLiteralConvertedBatch(t *testing.T) { + recorder := &ing02Recorder{} + writer := &PostgresEdgeWriter{committer: recorder} + rows := ing02DupFocusRows(t, ing02IngestAuth) + scope := PGEdgeBatchScope{ + BatchID: "ing02-ing/capture/0", Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + } + if _, err := writer.WriteEdgeBatch(context.Background(), scope, rows); err != nil { + t.Fatalf("WriteEdgeBatch: %v", err) + } + if got := recorder.callCount(); got != 1 { + t.Fatalf("committer calls = %d, want 1", got) + } + got := recorder.lastBatch(t) + if got.ID != scope.BatchID || got.Project != ing02IngestProject || got.DatasetGeneration != ing02IngestGen { + t.Fatalf("batch scope drifted: %+v", got) + } + want := ing02ExpectedOccurrences(ing02IngestAuth) + if !reflect.DeepEqual(got.Rows, want) { + t.Fatalf("converted batch drifted from the literal:\n got %+v\nwant %+v", got.Rows, want) + } +} + +func TestPostgresEdgeWriterTargetResolutionIsScopeExact(t *testing.T) { + s := newIngest02IngestService(t) + s.migrate(t) + rows := ing02DupFocusRows(t, ing02IngestAuth) + + scopeRes := func(tag, generation string, sub []rowBuildResult) { + t.Helper() + _, err := s.newResourceWriter(t).WriteResourceBatch(s.ctx, PGResourceBatchScope{ + BatchID: "ing02-ing/res/" + tag, Project: ing02IngestProject, + DatasetGeneration: generation, AuthResourcePath: ing02IngestAuth, + }, sub) + if err != nil { + s.fail("resources "+tag, err) + } + } + scopeEdges := func(tag, generation string, sub []rowBuildResult) { + t.Helper() + _, err := s.newEdgeWriter(t).WriteEdgeBatch(s.ctx, PGEdgeBatchScope{ + BatchID: "ing02-ing/edges/" + tag, Project: ing02IngestProject, + DatasetGeneration: generation, AuthResourcePath: ing02IngestAuth, + }, sub) + if err != nil { + s.fail("edges "+tag, err) + } + } + + scopeRes("full", ing02IngestGen, rows) + scopeEdges("full", ing02IngestGen, rows) + + decoyVertex := handBuiltVertex(ing02IngestProject, "gen-decoy", "Observation", "obs-decoy", ing02IngestAuth, + `{"resourceType":"Observation","id":"obs-decoy","status":"final",`+ + `"focus":[{"reference":"Observation/focus-1"}],`+ + `"subject":{"reference":"Patient/patient-1"}}`, nil) + qualifiedFocus := "Observation/" + generationDocumentKey(ing02IngestProject, "gen-decoy", "Observation", SanitizeKey("focus-1")) + qualifiedPatient := "Patient/" + generationDocumentKey(ing02IngestProject, "gen-decoy", "Patient", SanitizeKey("patient-1")) + decoyEdgeFocus := handBuiltEdgeDocument("Observation/obs-decoy", qualifiedFocus, + "focus_Observation", "Observation", "Observation", ing02IngestProject, "gen-decoy", ing02IngestAuth) + decoyEdgeSubject := handBuiltEdgeDocument("Observation/obs-decoy", qualifiedPatient, + "subject_Patient", "Observation", "Patient", ing02IngestProject, "gen-decoy", ing02IngestAuth) + decoyRow := replaceEdges(t, decoyVertex, decoyEdgeFocus, decoyEdgeSubject) + scopeRes("decoy", "gen-decoy", []rowBuildResult{decoyVertex}) + scopeEdges("decoy", "gen-decoy", []rowBuildResult{decoyRow}) + + repo, err := control.NewEdgeRepository(s.openClient(t, 2), s.schema) + if err != nil { + s.fail("new edge repository", err) + } + unrestricted := true + ownScope := control.EdgeScope{ + Project: ing02IngestProject, DatasetGeneration: ing02IngestGen, + AuthResourcePathsUnrestricted: &unrestricted, + } + decoyScope := control.EdgeScope{ + Project: ing02IngestProject, DatasetGeneration: "gen-decoy", + AuthResourcePathsUnrestricted: &unrestricted, + } + + resolvedTargets := map[string]bool{} + for _, source := range []struct{ rtype, lid string }{ + {"Observation", "obs-generic"}, {"Observation", "obs-generated"}, + } { + edges, _, err := repo.ReadEdgesFromSource(s.ctx, ownScope, source.rtype, source.lid, 0, 50) + if err != nil { + s.fail("traverse "+source.lid, err) + } + for _, edge := range edges { + resolvedTargets[edge.TargetResourceType+"/"+edge.TargetLogicalID] = edge.TargetResolved + } + } + if got := resolvedTargets["Observation/focus-1"]; !got { + t.Fatal("same-generation stored target focus-1 reported unresolved") + } + if got := resolvedTargets["Patient/patient-1"]; !got { + t.Fatal("same-generation stored target patient-1 reported unresolved") + } + + decoyEdges, _, err := repo.ReadEdgesFromSource(s.ctx, decoyScope, "Observation", "obs-decoy", 0, 50) + if err != nil { + s.fail("decoy traverse", err) + } + if len(decoyEdges) != 2 { + t.Fatalf("decoy occurrences = %d, want 2 persisted typed endpoints", len(decoyEdges)) + } + for _, edge := range decoyEdges { + if edge.TargetResourceType+"/"+edge.TargetLogicalID != "Observation/focus-1" && + edge.TargetResourceType+"/"+edge.TargetLogicalID != "Patient/patient-1" { + t.Fatalf("unexpected decoy target %s/%s", edge.TargetResourceType, edge.TargetLogicalID) + } + if edge.TargetResolved { + t.Fatalf("decoy target %s/%s resolved from another generation — resolution must stay inside the edge's own scope", + edge.TargetResourceType, edge.TargetLogicalID) + } + } + + if n := s.countEdges(t, ing02IngestProject, ing02IngestGen); n != 8 { + t.Fatalf("original occurrences after decoys = %d, want 8", n) + } +} + +func TestPostgresEdgeWriterMissingSourceBatchPersistsNothing(t *testing.T) { + s := newIngest02IngestService(t) + s.migrate(t) + rows := ing02DupFocusRows(t, ing02IngestAuth) + + edgeScope := PGEdgeBatchScope{ + BatchID: "ing02-ing/missing-source", Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + } + + res, err := s.newEdgeWriter(t).WriteEdgeBatch(s.ctx, edgeScope, rows) + var missing *control.EdgeSourceMissingError + if !errors.As(err, &missing) || !errors.Is(err, control.ErrEdgeSourceMissing) { + t.Fatalf("unseeded WriteEdgeBatch = %v, want *EdgeSourceMissingError", err) + } + if res.Committed != 0 || res.SkippedIdentical != 0 || res.Replay { + t.Fatalf("refused batch reported %+v, want zero counts", res) + } + if n := s.countEdges(t, ing02IngestProject, ing02IngestGen); n != 0 { + t.Fatalf("edge rows after refused batch = %d, want 0", n) + } + if _, _, found := s.witnessEdgeLedger(t, edgeScope.BatchID); found { + t.Fatal("refused batch wrote a ledger marker") + } + resources := s.mustInt64(t, "SELECT count(*) FROM "+s.qualified("fhir_resource")) + if resources != 0 { + t.Fatalf("resource rows after refused edge batch = %d, want 0 — edge writes never touch resources", resources) + } + + scopeRes := PGResourceBatchScope{ + BatchID: "ing02-ing/missing-source-res", Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + } + if _, err := s.newResourceWriter(t).WriteResourceBatch(s.ctx, scopeRes, rows); err != nil { + s.fail("control resources", err) + } + res, err = s.newEdgeWriter(t).WriteEdgeBatch(s.ctx, edgeScope, rows) + if err != nil { + s.fail("control edge batch", err) + } + if res.Committed != 8 || res.Offered != 8 { + t.Fatalf("control result = %+v, want 8 committed", res) + } + if n := s.countEdges(t, ing02IngestProject, ing02IngestGen); n != 8 { + t.Fatalf("edge rows after control = %d, want 8", n) + } +} + +func TestPostgresEdgeWriterPartialSourceLossRollsBackWholeBatch(t *testing.T) { + s := newIngest02IngestService(t) + s.migrate(t) + rows := ing02DupFocusRows(t, ing02IngestAuth) + + partial := rows[0] + focusVertex := handBuiltVertex(ing02IngestProject, ing02IngestGen, "Observation", "focus-1", ing02IngestAuth, + `{"resourceType":"Observation","id":"focus-1","status":"final","code":{"text":"target"}}`, nil) + seedRes := PGResourceBatchScope{ + BatchID: "ing02-ing/rollback-res", Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + } + if _, err := s.newResourceWriter(t).WriteResourceBatch(s.ctx, seedRes, + []rowBuildResult{partial, focusVertex}); err != nil { + s.fail("seed resources", err) + } + + res, err := s.newEdgeWriter(t).WriteEdgeBatch(s.ctx, PGEdgeBatchScope{ + BatchID: "ing02-ing/rollback", Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + }, rows) + var missing *control.EdgeSourceMissingError + if !errors.As(err, &missing) { + t.Fatalf("mixed batch = %v, want *EdgeSourceMissingError", err) + } + if res.Committed != 0 { + t.Fatalf("mixed batch reported %+v, want zero committed", res) + } + if n := s.countEdges(t, ing02IngestProject, ing02IngestGen); n != 0 { + t.Fatalf("edge rows after partial-source refusal = %d, want 0 (whole-batch rollback)", n) + } + if _, _, found := s.witnessEdgeLedger(t, "ing02-ing/rollback"); found { + t.Fatal("rolled-back batch left a ledger marker") + } +} + +func editEdgeDocument(t *testing.T, built rowBuildResult, index int, mutate func(map[string]json.RawMessage)) rowBuildResult { + t.Helper() + if index < 0 || index >= len(built.edges) { + t.Fatalf("edge index %d out of range (%d edges)", index, len(built.edges)) + } + document, err := decodeTopLevelDocument(built.edges[index]) + if err != nil { + t.Fatalf("decode edge for defect injection: %v", err) + } + mutate(document) + encoded, err := json.Marshal(document) + if err != nil { + t.Fatalf("re-encode edge: %v", err) + } + built.edges[index] = encoded + return built +} + +func edgeIndexOf(t *testing.T, built rowBuildResult, label string) int { + t.Helper() + for i, raw := range built.edges { + document, err := decodeTopLevelDocument(raw) + if err != nil { + t.Fatalf("decode edge %d: %v", i, err) + } + var got string + if err := json.Unmarshal(document["label"], &got); err != nil { + t.Fatalf("label of edge %d: %v", i, err) + } + if got == label { + return i + } + } + t.Fatalf("no edge document with label %q", label) + return -1 +} + +func ing02LegacyScope(batchID string) PGEdgeBatchScope { + return PGEdgeBatchScope{BatchID: batchID, Project: ing02IngestProject} +} + +func ing02GenScope(batchID string) PGEdgeBatchScope { + return PGEdgeBatchScope{ + BatchID: batchID, Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + } +} + +func ing02GenDupRow(t *testing.T) rowBuildResult { + t.Helper() + return buildGeneratedRow(t, ing02IngestProject, ing02IngestGen, ing02IngestAuth, "Observation", ing02GeneratedDupFocusLine) +} + +func bigHandBuiltRow(t *testing.T, n int) rowBuildResult { + t.Helper() + refs := strings.TrimSuffix(strings.Repeat(`{"reference":"Observation/focus-1"},`, n), ",") + payload := `{"resourceType":"Observation","id":"ref-big","status":"final","focus":[` + refs + `]}` + built := handBuiltVertex(ing02IngestProject, "", "Observation", "ref-big", "", payload, nil) + doc := handBuiltEdgeDocument("Observation/ref-big", "Observation/focus-1", + "focus_Observation", "Observation", "Observation", ing02IngestProject, "", "") + edges := make([]json.RawMessage, n) + for i := range edges { + edges[i] = doc + } + return replaceEdges(t, built, edges...) +} + +type edgeRefusalCase struct { + name string + rows []rowBuildResult + scope PGEdgeBatchScope + controlScope PGEdgeBatchScope + controlRows []rowBuildResult + scopeErr bool + wantField string + wantCategory rowErrorType + wantColumn string + wantReason string +} + +func TestPostgresEdgeWriterRefusalMatrix(t *testing.T) { + recorder := &ing02Recorder{} + writer := &PostgresEdgeWriter{committer: recorder} + + legacy := func() rowBuildResult { return ing02LegacyRefRow(t) } + genDup := ing02GenDupRow + + cases := []edgeRefusalCase{ + { + name: "blank batch id", rows: []rowBuildResult{legacy()}, + scope: ing02LegacyScope(""), controlRows: []rowBuildResult{legacy()}, + scopeErr: true, wantField: "batch id", + }, + { + name: "overlong batch id", rows: []rowBuildResult{legacy()}, + scope: ing02LegacyScope(strings.Repeat("x", control.MaxBatchIDLength+1)), + controlScope: ing02LegacyScope("ing02-ing/matrix/longid/control"), + controlRows: []rowBuildResult{legacy()}, + scopeErr: true, wantField: "batch id", wantReason: "want at most", + }, + { + name: "blank project", rows: []rowBuildResult{legacy()}, + scope: PGEdgeBatchScope{BatchID: "ing02-ing/matrix/project"}, + controlScope: ing02LegacyScope("ing02-ing/matrix/project/control"), + controlRows: []rowBuildResult{legacy()}, + scopeErr: true, wantField: "project", + }, + { + name: "no rows", rows: nil, + scope: ing02LegacyScope("ing02-ing/matrix/norows"), + controlRows: []rowBuildResult{legacy()}, + scopeErr: true, wantField: "rows", wantReason: "no rows", + }, + { + name: "rows without edges", rows: []rowBuildResult{ + handBuiltVertex(ing02IngestProject, "", "Observation", "edgeless", "", + `{"resourceType":"Observation","id":"edgeless","status":"final"}`, nil), + }, + scope: ing02LegacyScope("ing02-ing/matrix/noedges"), + controlRows: []rowBuildResult{legacy()}, + scopeErr: true, wantField: "rows", wantReason: "no built edge occurrences", + }, + { + name: "vertex project mismatch", + rows: []rowBuildResult{replaceField(t, legacy(), "project", json.RawMessage(`"prj-other"`))}, + scope: ing02LegacyScope("ing02-ing/matrix/vtx-project"), + controlRows: []rowBuildResult{legacy()}, + wantCategory: rowErrorGeneration, wantColumn: "project", wantReason: "does not match", + }, + { + name: "vertex missing generation stamp", + rows: []rowBuildResult{dropField(t, genDup(t), generationIdentityField)}, + scope: ing02GenScope("ing02-ing/matrix/vtx-stamp"), + controlRows: []rowBuildResult{genDup(t)}, + wantCategory: rowErrorGeneration, wantColumn: generationIdentityField, + wantReason: "dataset_generation", + }, + { + name: "edge generation mismatch", + rows: []rowBuildResult{editEdgeDocument(t, genDup(t), 0, func(m map[string]json.RawMessage) { + m[generationIdentityField] = json.RawMessage(`"gen-wrong"`) + })}, + scope: ing02GenScope("ing02-ing/matrix/edge-stamp"), + controlRows: []rowBuildResult{genDup(t)}, + wantCategory: rowErrorEdge, wantColumn: fmt.Sprintf("edge %d %s", 0, generationIdentityField), + wantReason: "does not match", + }, + { + name: "edge missing generation stamp under versioned batch", + rows: []rowBuildResult{editEdgeDocument(t, genDup(t), 0, func(m map[string]json.RawMessage) { + delete(m, generationIdentityField) + })}, + scope: ing02GenScope("ing02-ing/matrix/edge-nostamp"), + controlRows: []rowBuildResult{genDup(t)}, + wantCategory: rowErrorEdge, wantColumn: fmt.Sprintf("edge %d %s", 0, generationIdentityField), + wantReason: "carries no dataset_generation stamp", + }, + { + name: "stamped edge under legacy batch", + rows: []rowBuildResult{replaceEdges(t, + handBuiltVertex(ing02IngestProject, "", "Observation", "ref-1", "", + `{"resourceType":"Observation","id":"ref-1","status":"final","focus":[{"reference":"Observation/focus-1"}]}`, nil), + handBuiltEdgeDocument("Observation/ref-1", "Observation/focus-1", "focus_Observation", + "Observation", "Observation", ing02IngestProject, ing02IngestGen, ""))}, + scope: ing02LegacyScope("ing02-ing/matrix/edge-stamped-legacy"), + controlRows: []rowBuildResult{replaceEdges(t, + handBuiltVertex(ing02IngestProject, "", "Observation", "ref-1", "", + `{"resourceType":"Observation","id":"ref-1","status":"final","focus":[{"reference":"Observation/focus-1"}]}`, nil), + handBuiltEdgeDocument("Observation/ref-1", "Observation/focus-1", "focus_Observation", + "Observation", "Observation", ing02IngestProject, "", ""))}, + wantCategory: rowErrorEdge, wantColumn: "edge 0 " + generationIdentityField, + wantReason: "legacy", + }, + { + name: "edge project mismatch", + rows: []rowBuildResult{editEdgeDocument(t, legacy(), 0, func(m map[string]json.RawMessage) { + m["project"] = json.RawMessage(`"prj-other"`) + })}, + scope: ing02LegacyScope("ing02-ing/matrix/edge-project"), + controlRows: []rowBuildResult{legacy()}, + wantCategory: rowErrorEdge, wantColumn: "edge 0 project", wantReason: "does not match the batch project", + }, + { + name: "edge auth mismatch", + rows: func() []rowBuildResult { + built := genDup(t) + i := edgeIndexOf(t, built, "subject_Patient") + return []rowBuildResult{editEdgeDocument(t, built, i, func(m map[string]json.RawMessage) { + m["auth_resource_path"] = json.RawMessage(`"/other"`) + })} + }(), + scope: ing02GenScope("ing02-ing/matrix/edge-auth"), + controlRows: []rowBuildResult{genDup(t)}, + wantCategory: rowErrorEdge, + wantColumn: fmt.Sprintf("edge %d auth_resource_path", edgeIndexOf(t, genDup(t), "subject_Patient")), + wantReason: "does not match the batch auth path", + }, + { + name: "edge missing auth under scoped batch", + rows: func() []rowBuildResult { + built := genDup(t) + i := edgeIndexOf(t, built, "subject_Patient") + return []rowBuildResult{editEdgeDocument(t, built, i, func(m map[string]json.RawMessage) { + delete(m, "auth_resource_path") + })} + }(), + scope: ing02GenScope("ing02-ing/matrix/edge-noauth"), + controlRows: []rowBuildResult{genDup(t)}, + wantCategory: rowErrorEdge, + wantColumn: fmt.Sprintf("edge %d auth_resource_path", edgeIndexOf(t, genDup(t), "subject_Patient")), + wantReason: "carries no auth_resource_path", + }, + { + name: "edge advertises edge_seq", + rows: []rowBuildResult{editEdgeDocument(t, legacy(), 0, func(m map[string]json.RawMessage) { + m["edge_seq"] = json.RawMessage(`7`) + })}, + scope: ing02LegacyScope("ing02-ing/matrix/edge-seq"), + controlRows: []rowBuildResult{legacy()}, + wantCategory: rowErrorEdge, wantColumn: "edge 0 edge_seq", wantReason: "server-assigned", + }, + { + name: "edge _to malformed", + rows: []rowBuildResult{editEdgeDocument(t, legacy(), 0, func(m map[string]json.RawMessage) { + m["_to"] = json.RawMessage(`"badendpoint"`) + })}, + scope: ing02LegacyScope("ing02-ing/matrix/edge-malformed"), + controlRows: []rowBuildResult{legacy()}, + wantCategory: rowErrorEdge, wantColumn: "edge 0 to_type", wantReason: "not a well-formed", + }, + { + name: "edge to_type disagrees with endpoint", + rows: []rowBuildResult{editEdgeDocument(t, legacy(), 0, func(m map[string]json.RawMessage) { + m["_to"] = json.RawMessage(`"Patient/focus-1"`) + })}, + scope: ing02LegacyScope("ing02-ing/matrix/edge-type"), + controlRows: []rowBuildResult{legacy()}, + wantCategory: rowErrorEdge, wantColumn: "edge 0 to_type", wantReason: "has collection type", + }, + { + name: "edge owned by neither endpoint", + rows: []rowBuildResult{editEdgeDocument(t, editEdgeDocument(t, legacy(), 0, func(m map[string]json.RawMessage) { + m["_from"] = json.RawMessage(`"Observation/alien-1"`) + }), 0, func(m map[string]json.RawMessage) { + m["_to"] = json.RawMessage(`"Observation/alien-2"`) + })}, + scope: ing02LegacyScope("ing02-ing/matrix/edge-unowned"), + controlRows: []rowBuildResult{legacy()}, + wantCategory: rowErrorEdge, wantColumn: "edge 0 _from", wantReason: "is the owning row's endpoint", + }, + { + name: "edge not backed by any payload reference", + rows: []rowBuildResult{editEdgeDocument(t, legacy(), 0, func(m map[string]json.RawMessage) { + m["_to"] = json.RawMessage(`"Observation/ghost-1"`) + m["_key"] = json.RawMessage(`"Observation_ref-1_Observation_ghost-1_focus_Observation"`) + })}, + scope: ing02LegacyScope("ing02-ing/matrix/edge-unbacked"), + controlRows: []rowBuildResult{legacy()}, + wantCategory: rowErrorEdge, wantColumn: "matches-no-position", wantReason: "matches no reference position", + }, + { + name: "edge multiplicity exceeds payload positions", + rows: func() []rowBuildResult { + built := legacy() + i := edgeIndexOf(t, built, "subject_Patient") + dup := make([]json.RawMessage, 0, len(built.edges)+1) + dup = append(dup, built.edges...) + dup = append(dup, built.edges[i]) + return []rowBuildResult{replaceEdges(t, built, dup...)} + }(), + scope: ing02LegacyScope("ing02-ing/matrix/edge-multiplicity"), + controlRows: []rowBuildResult{legacy()}, + wantCategory: rowErrorEdge, wantColumn: "matches-no-position-group", + wantReason: "only 1 matching reference positions", + }, + { + name: "back-reference source type contradicts payload and is not the row type", + rows: func() []rowBuildResult { + built := legacy() + i := edgeIndexOf(t, built, "subject_observation") + return []rowBuildResult{editEdgeDocument(t, built, i, func(m map[string]json.RawMessage) { + m["_from"] = json.RawMessage(`"Practitioner/patient-1"`) + m["from_type"] = json.RawMessage(`"Practitioner"`) + })} + }(), + scope: ing02LegacyScope("ing02-ing/matrix/backref-contradiction"), + controlRows: []rowBuildResult{legacy()}, + wantCategory: rowErrorEdge, wantColumn: "matches-no-position-contradiction", + wantReason: "declares source type", + }, + { + name: "forward target type contradicts payload", + rows: []rowBuildResult{editEdgeDocument(t, legacy(), 0, func(m map[string]json.RawMessage) { + m["_to"] = json.RawMessage(`"Patient/focus-1"`) + m["to_type"] = json.RawMessage(`"Patient"`) + })}, + scope: ing02LegacyScope("ing02-ing/matrix/fwd-contradiction"), + controlRows: []rowBuildResult{legacy()}, + wantCategory: rowErrorEdge, wantColumn: "matches-no-position-contradiction", + wantReason: "declares target type", + }, + { + name: "edge capacity exceeded", + rows: []rowBuildResult{bigHandBuiltRow(t, control.MaxBatchEdges+1)}, + scope: ing02LegacyScope("ing02-ing/matrix/edge-capacity"), + controlRows: []rowBuildResult{bigHandBuiltRow(t, control.MaxBatchEdges)}, + scopeErr: true, wantField: "rows", wantReason: "want at most", + }, + } + + calls := 0 + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + before := recorder.callCount() + _, err := writer.WriteEdgeBatch(context.Background(), tc.scope, tc.rows) + if err == nil { + t.Fatalf("defect was accepted: %+v", tc) + } + if recorder.callCount() != before { + t.Fatal("refused batch reached the store surface") + } + if tc.scopeErr { + var scopeErr *PGEdgeWriteScopeError + if !errors.As(err, &scopeErr) || !errors.Is(err, ErrInvalidPGEdgeWrite) { + t.Fatalf("err = %v, want *PGEdgeWriteScopeError", err) + } + if scopeErr.BatchID != tc.scope.BatchID { + t.Fatalf("scope error batch id = %q, want %q", scopeErr.BatchID, tc.scope.BatchID) + } + if scopeErr.Field != tc.wantField { + t.Fatalf("scope error field = %q, want %q", scopeErr.Field, tc.wantField) + } + } else { + var rowErr *PGEdgeRowIdentityError + if !errors.As(err, &rowErr) || !errors.Is(err, ErrInvalidPGEdgeWrite) { + t.Fatalf("err = %v, want *PGEdgeRowIdentityError", err) + } + if rowErr.Category != tc.wantCategory { + t.Fatalf("category = %q, want %q (err: %s)", rowErr.Category, tc.wantCategory, err) + } + if !strings.HasPrefix(tc.wantColumn, "matches-no-position") { + if rowErr.Column != tc.wantColumn { + t.Fatalf("column = %q, want %q", rowErr.Column, tc.wantColumn) + } + } + } + if tc.wantReason != "" && !strings.Contains(err.Error(), tc.wantReason) { + t.Fatalf("err = %v, want reason containing %q", err, tc.wantReason) + } + + cscope := tc.controlScope + if cscope.Project == "" { + cscope = tc.scope + if cscope.BatchID == "" { + cscope.BatchID = "ing02-ing/matrix/control/" + ing01RandomHex(t) + } else { + cscope.BatchID += "/control" + } + } + res, err := writer.WriteEdgeBatch(context.Background(), cscope, tc.controlRows) + if err != nil { + t.Fatalf("control rejected at the same gate: %v", err) + } + if res.Replay || res.Committed == 0 { + t.Fatalf("control result = %+v, want a live offer to the recorder", res) + } + calls++ + }) + } + if got := recorder.callCount(); got != calls { + t.Fatalf("store-surface calls = %d, want %d (one per control only)", got, calls) + } +} + +func TestPostgresEdgeWriterLiveRefusalsPersistNothing(t *testing.T) { + s := newIngest02IngestService(t) + s.migrate(t) + writer := s.newEdgeWriter(t) + rows := ing02DupFocusRows(t, ing02IngestAuth) + + defects := []struct { + name string + rows func(t *testing.T) []rowBuildResult + wantCategory rowErrorType + }{ + { + name: "vertex project mismatch", + rows: func(t *testing.T) []rowBuildResult { + bad := ing02DupFocusRows(t, ing02IngestAuth) + bad[0] = replaceField(t, bad[0], "project", json.RawMessage(`"prj-other"`)) + return bad + }, + wantCategory: rowErrorGeneration, + }, + { + name: "edge advertises edge_seq", + rows: func(t *testing.T) []rowBuildResult { + bad := ing02DupFocusRows(t, ing02IngestAuth) + bad[1] = editEdgeDocument(t, bad[1], 0, func(m map[string]json.RawMessage) { + m["edge_seq"] = json.RawMessage(`3`) + }) + return bad + }, + wantCategory: rowErrorEdge, + }, + { + name: "edge owned by neither endpoint", + rows: func(t *testing.T) []rowBuildResult { + bad := ing02DupFocusRows(t, ing02IngestAuth) + bad[1] = editEdgeDocument(t, bad[1], 0, func(m map[string]json.RawMessage) { + m["_from"] = json.RawMessage(`"Observation/alien-9"`) + m["_to"] = json.RawMessage(`"Observation/alien-8"`) + m["from_type"] = json.RawMessage(`"Observation"`) + m["to_type"] = json.RawMessage(`"Observation"`) + }) + return bad + }, + wantCategory: rowErrorEdge, + }, + } + + for i, d := range defects { + scope := PGEdgeBatchScope{ + BatchID: fmt.Sprintf("ing02-ing/live-refuse/%d", i), Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + } + _, err := writer.WriteEdgeBatch(s.ctx, scope, d.rows(t)) + var rowErr *PGEdgeRowIdentityError + if !errors.As(err, &rowErr) || rowErr.Category != d.wantCategory { + t.Fatalf("%s: err = %v, want *PGEdgeRowIdentityError category %q", d.name, err, d.wantCategory) + } + if n := s.countEdges(t, ing02IngestProject, ing02IngestGen); n != 0 { + t.Fatalf("%s: edge rows = %d after refused batch, want 0", d.name, n) + } + if _, _, found := s.witnessEdgeLedger(t, scope.BatchID); found { + t.Fatalf("%s: refused batch left a ledger marker", d.name) + } + if n := s.mustInt64(t, "SELECT count(*) FROM "+s.qualified("fhir_resource")); n != 0 { + t.Fatalf("%s: resource rows = %d, want 0", d.name, n) + } + } + + if _, err := s.newResourceWriter(t).WriteResourceBatch(s.ctx, PGResourceBatchScope{ + BatchID: "ing02-ing/live-refuse-res", Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + }, rows); err != nil { + s.fail("live-refuse control resources", err) + } + res, err := writer.WriteEdgeBatch(s.ctx, PGEdgeBatchScope{ + BatchID: "ing02-ing/live-refuse-control", Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + }, rows) + if err != nil || res.Committed != 8 || res.Replay { + t.Fatalf("control = (%+v, %v), want a fresh 8-row commit", res, err) + } +} + +func ing02RowsForGen(t *testing.T, generation, authPath string) []rowBuildResult { + t.Helper() + return []rowBuildResult{ + buildGenericRow(t, ing02IngestProject, generation, authPath, "Observation", ing02GenericDupFocusLine), + buildGeneratedRow(t, ing02IngestProject, generation, authPath, "Observation", ing02GeneratedDupFocusLine), + buildGeneratedRow(t, ing02IngestProject, generation, authPath, "Patient", ing02PatientLine), + buildGeneratedRow(t, ing02IngestProject, generation, authPath, "Observation", ing02FocusTargetLine), + } +} + +func ing02SeedResources(t *testing.T, s *ing02IngestService, rows []rowBuildResult, generation, tag string) { + t.Helper() + _, err := s.newResourceWriter(t).WriteResourceBatch(s.ctx, PGResourceBatchScope{ + BatchID: "ing02-ing/seed/" + tag, Project: ing02IngestProject, + DatasetGeneration: generation, AuthResourcePath: ing02IngestAuth, + }, rows) + if err != nil { + s.fail("seed resources "+tag, err) + } +} + +func TestPostgresEdgeWriterReplayIsByteIdenticalNoOp(t *testing.T) { + s := newIngest02IngestService(t) + s.migrate(t) + rows := ing02DupFocusRows(t, ing02IngestAuth) + ing02SeedResources(t, s, rows, ing02IngestGen, "replay") + + writer := s.newEdgeWriter(t) + scope := PGEdgeBatchScope{ + BatchID: "ing02-ing/replay", Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + } + first, err := writer.WriteEdgeBatch(s.ctx, scope, rows) + if err != nil || first.Committed != 8 || first.Replay { + t.Fatalf("first = (%+v, %v), want fresh 8-row commit", first, err) + } + firstFP := s.edgeFingerprint(t, ing02IngestProject, ing02IngestGen) + + for attempt := 0; attempt < 3; attempt++ { + replay, err := writer.WriteEdgeBatch(s.ctx, scope, rows) + if err != nil { + s.fail(fmt.Sprintf("replay %d", attempt), err) + } + if !replay.Replay || replay.Digest != first.Digest || replay.Committed != first.Committed || + replay.SkippedIdentical != first.SkippedIdentical || replay.BatchID != first.BatchID { + t.Fatalf("replay %d = %+v, want the first result with the Replay flag set", attempt, replay) + } + } + if n := s.countEdges(t, ing02IngestProject, ing02IngestGen); n != 8 { + t.Fatalf("edge rows after replays = %d, want 8", n) + } + if fp := s.edgeFingerprint(t, ing02IngestProject, ing02IngestGen); fp != firstFP { + t.Fatal("replays disturbed stored rows — fingerprint drifted") + } + if _, edges, found := s.witnessEdgeLedger(t, scope.BatchID); !found || edges != 8 { + t.Fatalf("marker after replays = (found %v, edges %d), want 8", found, edges) + } + + fresh, err := writer.WriteEdgeBatch(s.ctx, PGEdgeBatchScope{ + BatchID: "ing02-ing/replay/reoffer", Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + }, rows) + if err != nil { + s.fail("identical re-offer", err) + } + if fresh.Replay || fresh.Committed != 0 || fresh.SkippedIdentical != 8 { + t.Fatalf("identical re-offer = %+v, want 0 committed / 8 skipped", fresh) + } + if _, edges, found := s.witnessEdgeLedger(t, "ing02-ing/replay/reoffer"); !found || edges != 0 { + t.Fatalf("re-offer marker = (found %v, edges %d), want a 0-edge marker", found, edges) + } +} + +func TestPostgresEdgeWriterConcurrentIdenticalEdgeBatchCommitsOnce(t *testing.T) { + s := newIngest02IngestService(t) + s.migrate(t) + clientA := s.openAppClient(t, "ing02-ing-race-a", 1) + clientB := s.openAppClient(t, "ing02-ing-race-b", 1) + pidA := s.poolBackendPID(t, clientA) + pidB := s.poolBackendPID(t, clientB) + if pidA == pidB { + t.Fatalf("race clients share backend %d, want two distinct real PG connections", pidA) + } + writerA, writerB := s.newEdgeWriterOn(t, clientA), s.newEdgeWriterOn(t, clientB) + + type outcome struct { + res PGEdgeWriteResult + err error + } + const rounds = 12 + for round := 0; round < rounds; round++ { + generation := fmt.Sprintf("gen-race-%d", round) + rows := ing02RowsForGen(t, generation, ing02IngestAuth) + ing02SeedResources(t, s, rows, generation, fmt.Sprintf("race-%d", round)) + scope := PGEdgeBatchScope{ + BatchID: fmt.Sprintf("ing02-ing/race/%d", round), Project: ing02IngestProject, + DatasetGeneration: generation, AuthResourcePath: ing02IngestAuth, + } + + start := make(chan struct{}) + race := func(w *PostgresEdgeWriter) <-chan outcome { + done := make(chan outcome, 1) + go func() { + <-start + res, err := w.WriteEdgeBatch(context.Background(), scope, rows) + done <- outcome{res, err} + }() + return done + } + chA, chB := race(writerA), race(writerB) + close(start) + + var oA, oB outcome + select { + case oA = <-chA: + case <-time.After(60 * time.Second): + s.fail(fmt.Sprintf("round %d: writer A", round), fmt.Errorf("never completed")) + } + select { + case oB = <-chB: + case <-time.After(60 * time.Second): + s.fail(fmt.Sprintf("round %d: writer B", round), fmt.Errorf("never completed")) + } + if oA.err != nil || oB.err != nil { + s.fail(fmt.Sprintf("round %d concurrent commits", round), + fmt.Errorf("A: %v; B: %v", oA.err, oB.err)) + } + if oA.res.Replay == oB.res.Replay { + t.Fatalf("round %d replays = (%v, %v), want exactly one fresh commit", + round, oA.res.Replay, oB.res.Replay) + } + if oA.res.Digest != oB.res.Digest || oA.res.Digest == "" { + t.Fatalf("round %d digests = (%s, %s), want one shared digest", round, oA.res.Digest, oB.res.Digest) + } + if n := s.countEdges(t, ing02IngestProject, generation); n != 8 { + t.Fatalf("round %d converged rows = %d, want 8", round, n) + } + if _, edges, found := s.witnessEdgeLedger(t, scope.BatchID); !found || edges != 8 { + t.Fatalf("round %d marker = (found %v, edges %d), want 8", round, found, edges) + } + } +} + +func TestPostgresEdgeWriterCountsChangeOnlyAtCommit(t *testing.T) { + s := newIngest02IngestService(t) + s.migrate(t) + writerClient := s.openAppClient(t, "ing02-ing-commit-w", 1) + writerPID := s.poolBackendPID(t, writerClient) + writer := s.newEdgeWriterOn(t, writerClient) + + obsConn, err := s.iso.Acquire(s.ctx) + if err != nil { + s.fail("acquire observer connection", err) + } + defer obsConn.Release() + var observerPID int64 + if err := obsConn.QueryRow(s.ctx, "SELECT pg_backend_pid()").Scan(&observerPID); err != nil { + s.fail("witness observer pid", err) + } + if observerPID == writerPID { + t.Fatalf("observer and writer share backend %d, want two distinct real PG connections", observerPID) + } + + type outcome struct { + res PGEdgeWriteResult + err error + } + const rounds = 12 + for round := 0; round < rounds; round++ { + generation := fmt.Sprintf("gen-visibility-%d", round) + rows := ing02RowsForGen(t, generation, ing02IngestAuth) + ing02SeedResources(t, s, rows, generation, fmt.Sprintf("visibility-%d", round)) + scope := PGEdgeBatchScope{ + BatchID: fmt.Sprintf("ing02-ing/visibility/%d", round), Project: ing02IngestProject, + DatasetGeneration: generation, AuthResourcePath: ing02IngestAuth, + } + + if _, err := obsConn.Exec(s.ctx, "BEGIN"); err != nil { + s.fail("observer begin", err) + } + if _, err := obsConn.Exec(s.ctx, + "LOCK TABLE "+s.qualified("fhir_edge_occurrence")+" IN ACCESS EXCLUSIVE MODE"); err != nil { + s.fail("observer lock", err) + } + + done := make(chan outcome, 1) + go func() { + res, err := writer.WriteEdgeBatch(context.Background(), scope, rows) + done <- outcome{res, err} + }() + + blockedPID := s.waitBackendWaiting(t, "ing02-ing-commit-w") + if blockedPID != writerPID || blockedPID == observerPID { + t.Fatalf("round %d blocked backend pid = %d, want the writer backend %d (observer is %d)", + round, blockedPID, writerPID, observerPID) + } + var midRows int64 + if err := obsConn.QueryRow(s.ctx, "SELECT count(*) FROM "+s.qualified("fhir_edge_occurrence")+ + " WHERE project = $1 AND dataset_generation = $2", ing02IngestProject, generation). + Scan(&midRows); err != nil { + s.fail("mid-transaction row witness", err) + } + if midRows != 0 { + t.Fatalf("round %d rows visible mid-transaction = %d, want 0 — counts may only change at commit", round, midRows) + } + var midMarkers int64 + if err := obsConn.QueryRow(s.ctx, "SELECT count(*) FROM "+s.qualified("batch_write_ledger")+ + " WHERE batch_id = $1", scope.BatchID).Scan(&midMarkers); err != nil { + s.fail("mid-transaction marker witness", err) + } + if midMarkers != 0 { + t.Fatalf("round %d ledger marker visible mid-transaction, want none", round) + } + + if _, err := obsConn.Exec(s.ctx, "COMMIT"); err != nil { + s.fail("observer commit", err) + } + var o outcome + select { + case o = <-done: + case <-time.After(60 * time.Second): + s.fail(fmt.Sprintf("round %d: waiting for batch commit", round), + fmt.Errorf("batch never completed after the lock released")) + } + if o.err != nil { + s.fail(fmt.Sprintf("round %d WriteEdgeBatch", round), o.err) + } + if o.res.Committed != 8 || o.res.Replay { + t.Fatalf("round %d result = %+v, want fresh 8-row commit", round, o.res) + } + if n := s.countEdges(t, ing02IngestProject, generation); n != o.res.Committed { + t.Fatalf("round %d rows after commit = %d, want the reported %d", round, n, o.res.Committed) + } + } +} + +func TestPostgresEdgeWriterOrdinalAssignmentIsReproducibleAcrossSerializations(t *testing.T) { + recorder := &ing02Recorder{} + writer := &PostgresEdgeWriter{committer: recorder} + + edge := handBuiltEdgeDocument("Observation/ref-d", "Observation/focus-1", + "focus_Observation", "Observation", "Observation", ing02IngestProject, "", "") + rev := handBuiltEdgeDocument("Observation/focus-1", "Observation/ref-d", + "focus_observation", "Observation", "Observation", ing02IngestProject, "", "") + + payloadA := `{"focus":[{"reference":"Observation/focus-1"},{"reference":"Observation/focus-1"}],` + + `"id":"ref-d","resourceType":"Observation","status":"final"}` + payloadB := `{"status":"final","id":"ref-d","resourceType":"Observation",` + + `"focus":[{"reference":"Observation/focus-1"},{"reference":"Observation/focus-1"}]}` + if payloadA == payloadB { + t.Fatal("fixture degenerated: serializations are identical") + } + + rowA := replaceEdges(t, handBuiltVertex(ing02IngestProject, "", "Observation", "ref-d", "", payloadA, nil), edge, edge, rev, rev) + rowB := replaceEdges(t, handBuiltVertex(ing02IngestProject, "", "Observation", "ref-d", "", payloadB, nil), edge, edge, rev, rev) + + if _, err := writer.WriteEdgeBatch(context.Background(), + ing02LegacyScope("ing02-ing/repro/a"), []rowBuildResult{rowA}); err != nil { + t.Fatalf("conversion A: %v", err) + } + if _, err := writer.WriteEdgeBatch(context.Background(), + ing02LegacyScope("ing02-ing/repro/b"), []rowBuildResult{rowB}); err != nil { + t.Fatalf("conversion B: %v", err) + } + batchA, batchB := recorder.batches[0], recorder.batches[1] + if !reflect.DeepEqual(batchA.Rows, batchB.Rows) { + t.Fatalf("converted rows drifted between serializations:\n A %+v\n B %+v", batchA.Rows, batchB.Rows) + } + if len(batchA.Rows) != 4 { + t.Fatalf("converted %d rows, want 2 per direction for the duplicate pair", len(batchA.Rows)) + } + ordinals := map[string][]int64{} + for _, row := range batchA.Rows { + key := row.Label + "|" + row.SourcePath + ordinals[key] = append(ordinals[key], row.SourceOrdinal) + } + for key, got := range ordinals { + if len(got) != 2 || got[0] != 0 || got[1] != 1 { + t.Fatalf("%s ordinals = %v, want [0 1] — duplicates are independently addressable", key, got) + } + } + + s := newIngest02IngestService(t) + s.migrate(t) + live := s.newEdgeWriter(t) + for i, row := range []rowBuildResult{rowA, rowB} { + generation := fmt.Sprintf("gen-repro-%d", i) + qualifiedPeer := "Observation/" + generationDocumentKey(ing02IngestProject, generation, "Observation", SanitizeKey("focus-1")) + qualifiedRow := replaceEdges(t, + handBuiltVertex(ing02IngestProject, generation, "Observation", "ref-d", "", + []string{payloadA, payloadB}[i], nil), + handBuiltEdgeDocument("Observation/ref-d", qualifiedPeer, "focus_Observation", + "Observation", "Observation", ing02IngestProject, generation, ""), + handBuiltEdgeDocument("Observation/ref-d", qualifiedPeer, "focus_Observation", + "Observation", "Observation", ing02IngestProject, generation, ""), + handBuiltEdgeDocument(qualifiedPeer, "Observation/ref-d", "focus_observation", + "Observation", "Observation", ing02IngestProject, generation, ""), + handBuiltEdgeDocument(qualifiedPeer, "Observation/ref-d", "focus_observation", + "Observation", "Observation", ing02IngestProject, generation, "")) + _ = row + _, err := s.newResourceWriter(t).WriteResourceBatch(s.ctx, PGResourceBatchScope{ + BatchID: fmt.Sprintf("ing02-ing/repro-res/%d", i), Project: ing02IngestProject, + DatasetGeneration: generation, + }, []rowBuildResult{qualifiedRow, handBuiltVertex(ing02IngestProject, generation, + "Observation", "focus-1", "", + `{"resourceType":"Observation","id":"focus-1","status":"final"}`, nil)}) + if err != nil { + s.fail(fmt.Sprintf("repro resource commit %d", i), err) + } + if _, err := live.WriteEdgeBatch(s.ctx, PGEdgeBatchScope{ + BatchID: fmt.Sprintf("ing02-ing/repro-edge/%d", i), Project: ing02IngestProject, + DatasetGeneration: generation, + }, []rowBuildResult{qualifiedRow}); err != nil { + s.fail(fmt.Sprintf("repro edge commit %d", i), err) + } + } + keyless := func(generation string) string { + var fp string + sql := "SELECT coalesce(md5(string_agg(entry, E'\n' ORDER BY edge_seq)),'EMPTY') FROM (" + + "SELECT edge_seq, source_resource_type || '|' || source_logical_id || '|' || label || '|' ||" + + " target_resource_type || '|' || target_logical_id || '|' || source_path || '|' ||" + + " source_ordinal::text || '|' || auth_resource_path AS entry" + + " FROM " + s.qualified("fhir_edge_occurrence") + " WHERE project = $1 AND dataset_generation = $2) x" + if err := s.iso.QueryRow(s.ctx, sql, ing02IngestProject, generation).Scan(&fp); err != nil { + s.fail("keyless fingerprint", err) + } + return fp + } + fpA, fpB := keyless("gen-repro-0"), keyless("gen-repro-1") + if fpA != fpB || fpA == "EMPTY" { + t.Fatalf("stored rows drifted across serializations: %s vs %s", fpA, fpB) + } +} + +const ( + ing02KeyCrossFocusFwd = "35af41b5-d5e6-5541-a691-9612d0c6c0ed" + ing02KeyCrossFocusBack = "5190907f-9a27-58cd-b917-c0931230e5cb" + ing02KeyCrossSubjectFwd = "df75cc57-662e-5be7-a651-2a4ebe5b7973" + ing02KeyCrossSubjectBk = "838c4a72-1e7c-5f70-a781-321ef7656b5d" +) + +func TestPostgresEdgeWriterGenericCrossTypeBackReferenceStoresPayloadSource(t *testing.T) { + s := newIngest02IngestService(t) + s.migrate(t) + crossRow := buildGenericRow(t, ing02IngestProject, ing02IngestGen, ing02IngestAuth, "Observation", ing02GenericCrossTypeLine) + + mislabelWitnessed := false + for _, raw := range crossRow.edges { + document, err := decodeTopLevelDocument(raw) + if err != nil { + s.fail("decode cross edge", err) + } + var label, from, fromType string + if err := json.Unmarshal(document["label"], &label); err != nil { + s.fail("cross edge label", err) + } + if label != "subject_observation" { + continue + } + if err := json.Unmarshal(document["_from"], &from); err != nil { + s.fail("cross edge _from", err) + } + if err := json.Unmarshal(document["from_type"], &fromType); err != nil { + s.fail("cross edge from_type", err) + } + if fromType != "Observation" || !strings.HasPrefix(from, "Observation/") { + t.Fatalf("fixture drifted: generic back-reference declares %q %q, want the EdgeFromGrip row-type stamp", from, fromType) + } + mislabelWitnessed = true + } + if !mislabelWitnessed { + t.Fatal("generic builder emitted no subject_observation back-reference") + } + + rows := []rowBuildResult{ + crossRow, + buildGeneratedRow(t, ing02IngestProject, ing02IngestGen, ing02IngestAuth, "Patient", ing02PatientLine), + buildGeneratedRow(t, ing02IngestProject, ing02IngestGen, ing02IngestAuth, "Observation", ing02FocusTargetLine), + } + if _, err := s.newResourceWriter(t).WriteResourceBatch(s.ctx, PGResourceBatchScope{ + BatchID: "ing02-ing/cross-res", Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + }, rows); err != nil { + s.fail("cross resources", err) + } + res, err := s.newEdgeWriter(t).WriteEdgeBatch(s.ctx, PGEdgeBatchScope{ + BatchID: "ing02-ing/cross-edges", Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + }, rows[:1]) + if err != nil { + s.fail("cross edges", err) + } + if res.Committed != 4 || res.Replay { + t.Fatalf("cross result = %+v, want 4 committed", res) + } + k := func(v string) *string { return &v } + s.requireEdgeRow(t, ing02IngestProject, ing02IngestGen, ing02IngestAuth, "focus_Observation", + "Observation", "obs-cross", "Observation", "focus-1", "focus", 0, k(ing02KeyCrossFocusFwd)) + s.requireEdgeRow(t, ing02IngestProject, ing02IngestGen, ing02IngestAuth, "focus_observation", + "Observation", "focus-1", "Observation", "obs-cross", "focus", 0, k(ing02KeyCrossFocusBack)) + s.requireEdgeRow(t, ing02IngestProject, ing02IngestGen, ing02IngestAuth, "subject_Patient", + "Observation", "obs-cross", "Patient", "patient-1", "subject", 0, k(ing02KeyCrossSubjectFwd)) + s.requireEdgeRow(t, ing02IngestProject, ing02IngestGen, ing02IngestAuth, "subject_observation", + "Patient", "patient-1", "Observation", "obs-cross", "subject", 0, k(ing02KeyCrossSubjectBk)) + if n := s.countEdges(t, ing02IngestProject, ing02IngestGen); n != 4 { + t.Fatalf("cross edge rows = %d, want 4", n) + } +} + +func TestPostgresEdgeWriterBackReferenceSelectionPrefersLabelOverMistypedDecoy(t *testing.T) { + payload := `{"resourceType":"Observation","id":"ref-e","status":"final",` + + `"focus":[{"reference":"Observation/patient-1"}],` + + `"subject":{"reference":"Patient/patient-1"}}` + built := handBuiltVertex(ing02IngestProject, "", "Observation", "ref-e", "", payload, nil) + back := handBuiltEdgeDocument("Observation/patient-1", "Observation/ref-e", + "subject_observation", "Observation", "Observation", ing02IngestProject, "", "") + row := replaceEdges(t, built, back) + + recorder := &ing02Recorder{} + writer := &PostgresEdgeWriter{committer: recorder} + if _, err := writer.WriteEdgeBatch(context.Background(), + ing02LegacyScope("ing02-ing/decoy/capture"), []rowBuildResult{row}); err != nil { + t.Fatalf("conversion: %v", err) + } + got := recorder.lastBatch(t).Rows + if len(got) != 1 { + t.Fatalf("converted %d rows, want 1", len(got)) + } + r := got[0] + if r.SourceResourceType != "Patient" || r.SourceLogicalID != "patient-1" || + r.TargetResourceType != "Observation" || r.TargetLogicalID != "ref-e" || + r.SourcePath != "subject" || r.SourceOrdinal != 0 { + t.Fatalf("decoy position won selection: %+v, want Patient/patient-1 at subject#0", r) + } + + s := newIngest02IngestService(t) + s.migrate(t) + ownVertex := handBuiltVertex(ing02IngestProject, "", "Observation", "ref-e", "", payload, nil) + patientVertex := handBuiltVertex(ing02IngestProject, "", "Patient", "patient-1", "", + `{"resourceType":"Patient","id":"patient-1","gender":"male"}`, nil) + if _, err := s.newResourceWriter(t).WriteResourceBatch(s.ctx, PGResourceBatchScope{ + BatchID: "ing02-ing/decoy-res", Project: ing02IngestProject, + }, []rowBuildResult{ownVertex, patientVertex}); err != nil { + s.fail("decoy resources", err) + } + res, err := s.newEdgeWriter(t).WriteEdgeBatch(s.ctx, ing02LegacyScope("ing02-ing/decoy-edges"), []rowBuildResult{row}) + if err != nil { + s.fail("decoy edges", err) + } + if res.Committed != 1 || res.Replay { + t.Fatalf("decoy result = %+v, want 1 committed", res) + } + key := SanitizeKey("Observation/patient-1") + "_" + SanitizeKey("Observation/ref-e") + "_subject_observation" + s.requireEdgeRow(t, ing02IngestProject, "", "", "subject_observation", + "Patient", "patient-1", "Observation", "ref-e", "subject", 0, &key) +} + +func TestPostgresEdgeWriterCaseVariantReferencePrefixStoresCanonicalTypes(t *testing.T) { + caseRow := buildGeneratedRow(t, ing02IngestProject, ing02IngestGen, ing02IngestAuth, "Observation", ing02GeneratedCaseRefLine) + + recorder := &ing02Recorder{} + writer := &PostgresEdgeWriter{committer: recorder} + if _, err := writer.WriteEdgeBatch(context.Background(), ing02GenScope("ing02-ing/case/capture"), + []rowBuildResult{caseRow}); err != nil { + t.Fatalf("lower-case prefix refused: %v", err) + } + rows := recorder.lastBatch(t).Rows + if len(rows) != 2 { + t.Fatalf("converted %d rows, want 2", len(rows)) + } + for _, r := range rows { + if r.SourceResourceType != "Patient" && r.TargetResourceType != "Patient" { + t.Fatalf("no Patient endpoint on %+v", r) + } + if r.SourceResourceType == "patient" || r.TargetResourceType == "patient" { + t.Fatalf("caller casing persisted: %+v", r) + } + } + + s := newIngest02IngestService(t) + s.migrate(t) + rowsAll := []rowBuildResult{ + caseRow, + buildGeneratedRow(t, ing02IngestProject, ing02IngestGen, ing02IngestAuth, "Patient", ing02PatientLine), + } + if _, err := s.newResourceWriter(t).WriteResourceBatch(s.ctx, PGResourceBatchScope{ + BatchID: "ing02-ing/case-res", Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + }, rowsAll); err != nil { + s.fail("case resources", err) + } + res, err := s.newEdgeWriter(t).WriteEdgeBatch(s.ctx, PGEdgeBatchScope{ + BatchID: "ing02-ing/case-edges", Project: ing02IngestProject, + DatasetGeneration: ing02IngestGen, AuthResourcePath: ing02IngestAuth, + }, rowsAll[:1]) + if err != nil { + s.fail("case edges", err) + } + if res.Committed != 2 || res.Replay { + t.Fatalf("case result = %+v, want 2 committed", res) + } + k := func(v string) *string { return &v } + s.requireEdgeRow(t, ing02IngestProject, ing02IngestGen, ing02IngestAuth, "subject_Patient", + "Observation", "obs-case", "Patient", "patient-1", "subject", 0, k(ing02KeyCaseSubjectFwd)) + s.requireEdgeRow(t, ing02IngestProject, ing02IngestGen, ing02IngestAuth, "subject_observation", + "Patient", "patient-1", "Observation", "obs-case", "subject", 0, k(ing02KeyCaseSubjectBack)) + + payload := `{"resourceType":"Observation","id":"ref-c","status":"final",` + + `"subject":{"reference":"patient/patient-1"}}` + built := handBuiltVertex(ing02IngestProject, "", "Observation", "ref-c", "", payload, nil) + back := handBuiltEdgeDocument("Observation/patient-1", "Observation/ref-c", + "subject_observation", "Observation", "Observation", ing02IngestProject, "", "") + recorder2 := &ing02Recorder{} + writer2 := &PostgresEdgeWriter{committer: recorder2} + if _, err := writer2.WriteEdgeBatch(context.Background(), + ing02LegacyScope("ing02-ing/case/recovery"), + []rowBuildResult{replaceEdges(t, built, back)}); err != nil { + t.Fatalf("recovery conversion: %v", err) + } + got := recorder2.lastBatch(t).Rows + if len(got) != 1 || got[0].SourceResourceType != "Patient" || got[0].SourceLogicalID != "patient-1" || + got[0].SourcePath != "subject" { + t.Fatalf("recovery stored %+v, want canonical Patient/patient-1 at subject", got) + } + + badPayload := `{"resourceType":"Observation","id":"ref-u","status":"final",` + + `"subject":{"reference":"notatype/patient-1"}}` + badBuilt := handBuiltVertex(ing02IngestProject, "", "Observation", "ref-u", "", badPayload, nil) + badBack := handBuiltEdgeDocument("Observation/patient-1", "Observation/ref-u", + "subject_observation", "Observation", "Observation", ing02IngestProject, "", "") + _, err = writer2.WriteEdgeBatch(context.Background(), + ing02LegacyScope("ing02-ing/case/unresolvable"), + []rowBuildResult{replaceEdges(t, badBuilt, badBack)}) + var rowErr *PGEdgeRowIdentityError + if !errors.As(err, &rowErr) || !errors.Is(err, ErrInvalidPGEdgeWrite) || + !strings.Contains(err.Error(), "does not resolve to a concrete FHIR resource type") { + t.Fatalf("unresolvable prefix = %v, want typed rowErrorEdge refusal", err) + } + if rowErr.Category != rowErrorEdge { + t.Fatalf("unresolvable refusal category = %q, want %q", rowErr.Category, rowErrorEdge) + } +} + +func TestZZZeroLeftoverIngestPostgresEdgeFixtures(t *testing.T) { + raw := ing01TestPostgresURL(t) + ing02IngestOwnedMu.Lock() + owned := make(map[string]string, len(ing02IngestOwnedByName)) + for db, schema := range ing02IngestOwnedByName { + owned[db] = schema + } + ing02IngestOwnedMu.Unlock() + if len(owned) == 0 { + t.Skip("this process created no verify_ing02_ing_* fixtures") + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service for audit: %v", err) + } + defer admin.Close() + + for db, schema := range owned { + if !strings.HasPrefix(db, ing02IngestPrefix) { + t.Fatalf("registry holds foreign database %q", db) + } + var count int + if err := admin.QueryRow(ctx, "SELECT COUNT(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Errorf("audit owned database %s: %v", db, err) + continue + } + if count != 0 { + t.Errorf("leftover owned database %s after all cleanups", db) + if schema != "" { + u, err := url.Parse(raw) + if err == nil { + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + db + if probe, err := pgxpool.New(ctx, u.String()); err == nil { + var schemaCount int + if err := probe.QueryRow(ctx, "SELECT COUNT(*) FROM pg_namespace WHERE nspname = $1", schema).Scan(&schemaCount); err != nil { + t.Errorf("audit owned schema %s in %s: %v", schema, db, err) + } else if schemaCount != 0 { + t.Errorf("leftover owned schema %s in leftover database %s", schema, db) + } + probe.Close() + } + } + } + } + } + + var visible int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname LIKE 'verify\\_ing02\\_ing\\_%' ESCAPE '\\'"). + Scan(&visible); err != nil { + t.Fatalf("audit verify_ing02_ing_* database count: %v", err) + } + if visible != ing02IngestBaselineCount { + t.Errorf("verify_ing02_ing_* databases = %d, want baseline %d", visible, ing02IngestBaselineCount) + } +} diff --git a/internal/ingest/preflight.go b/internal/ingest/preflight.go index e5fc80588..80211b77c 100644 --- a/internal/ingest/preflight.go +++ b/internal/ingest/preflight.go @@ -12,10 +12,6 @@ import ( const defaultPreflightSampleRows = 10 -// IngestionMode describes the loader selected for a resource type after -// checking the active graph schema. Generated is a fast path; Generic remains -// the supported schema-backed path for resources outside that optimized -// switch. type IngestionMode string const ( @@ -24,9 +20,6 @@ const ( IngestionModeUnsupported IngestionMode = "unsupported" ) -// PreflightResource is a deterministic per-resource report. Resource types -// come from the staged file names because that is the load contract; sampled -// payload resourceType values are checked below before any database mutation. type PreflightResource struct { ResourceType string `json:"resourceType"` Files []string `json:"files"` @@ -36,9 +29,6 @@ type PreflightResource struct { Mode IngestionMode `json:"mode"` } -// PreflightIssue is intentionally structured so an HTTP or CLI caller can -// present every actionable issue rather than leaving a partially loaded -// project behind after the first invalid file. type PreflightIssue struct { Code string `json:"code"` File string `json:"file"` @@ -56,8 +46,6 @@ func (r PreflightReport) Valid() bool { return len(r.Issues) == 0 } -// PreflightError preserves the complete report for callers while retaining a -// compact Error implementation for the existing synchronous Load API. type PreflightError struct { Report PreflightReport } @@ -70,10 +58,6 @@ func (e *PreflightError) Error() string { return fmt.Sprintf("ingestion preflight failed: %s", issue.Message) } -// PreflightFiles validates the staged filename-to-resource contract and -// chooses the generated or generic loader for every active graph-schema class. -// It samples a bounded number of records per file; full validation remains in -// the streaming loader so large imports are not read twice. func PreflightFiles(files []string, schema *graph.GraphSchema, sampleRows int) (PreflightReport, error) { if schema == nil { return PreflightReport{}, fmt.Errorf("graph schema is required") diff --git a/internal/ingest/preflight_test.go b/internal/ingest/preflight_test.go index 2be2a10fc..ef8773f92 100644 --- a/internal/ingest/preflight_test.go +++ b/internal/ingest/preflight_test.go @@ -8,8 +8,6 @@ import ( "reflect" "testing" - arangostore "github.com/calypr/loom/internal/store/arango" - "github.com/bmeg/jsonschemagraph/graph" ) @@ -104,9 +102,7 @@ func TestLoadReturnsPreflightReportBeforeOpeningArango(t *testing.T) { summary, err := Load(context.Background(), LoadOptions{ Schema: schemaPath, MetaDir: dir, - // A connection to this endpoint would fail. Receiving a structured - // preflight error instead proves validation happens before backend open. - ConnectionOptions: arangostore.ConnectionOptions{ + ConnectionOptions: ConnectionOptions{ URL: "http://127.0.0.1:1", Database: "preflight_must_not_connect", }, diff --git a/internal/ingest/related_medication_fixture_test.go b/internal/ingest/related_medication_fixture_test.go index 7bebf22d9..8b5e0bc5d 100644 --- a/internal/ingest/related_medication_fixture_test.go +++ b/internal/ingest/related_medication_fixture_test.go @@ -141,7 +141,7 @@ func readMedicationFixture(t *testing.T) (medicationFixtureScope, map[string]med file.Close() t.Fatal(err) } - edge.Generation = scope.Generation // Dataset loaders attach this scope to extracted edges. + edge.Generation = scope.Generation edges = append(edges, edge) } } diff --git a/internal/ingest/resource_load.go b/internal/ingest/resource_load.go index d9323cdcb..73fb6a216 100644 --- a/internal/ingest/resource_load.go +++ b/internal/ingest/resource_load.go @@ -7,15 +7,10 @@ import ( "time" "github.com/bmeg/jsonschemagraph/graph" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/catalog" - catalogarango "github.com/calypr/loom/internal/catalog/arango" ) -// LoadIncompleteError means row-level validation, generation, or edge -// construction failures were observed while the mutable resource load ran with -// FailFast disabled. The collections may hold partial documents, so the load -// must never report success. It mirrors GenerationLoadIncompleteError for the -// unversioned project/resourceType path that has no dataset generation. type LoadIncompleteError struct { ValidationErrors int GenerationErrors int @@ -34,9 +29,6 @@ func (e *LoadIncompleteError) Error() string { ) } -// loadIncompleteError returns the typed incomplete-load error when any -// row-level failure counter is nonzero, mirroring the generation loader so a -// partial mutable load cannot exit 0. It returns nil for a clean summary. func loadIncompleteError(summary LoadSummary) error { if summary.ValidationErrors != 0 || summary.GenerationErrors != 0 || summary.EdgeErrors != 0 { return &LoadIncompleteError{ @@ -48,9 +40,6 @@ func loadIncompleteError(summary LoadSummary) error { return nil } -// loadResource preserves the unversioned, overwrite/upsert contract used by -// the project/resourceType endpoint. It deliberately does not create or -// activate a dataset generation. func loadResource(ctx context.Context, opts LoadOptions) (LoadSummary, error) { opts = normalizeLoadOptions(opts) start := time.Now() @@ -102,12 +91,13 @@ func loadResource(ctx context.Context, opts LoadOptions) (LoadSummary, error) { "database": opts.Database, }) connectStart := time.Now() - client, err := openBackend(ctx, opts.ConnectionOptions) + backend, err := openBackend(ctx, opts.ConnectionOptions) if err != nil { return summary, err } - defer func() { _ = client.Close(context.WithoutCancel(ctx)) }() - catalogStore, err := catalogarango.New(client) + client := backend.Client + defer func() { _ = backend.Close(context.WithoutCancel(ctx)) }() + catalogStore, err := arangofactory.NewCatalogStore(client) if err != nil { return summary, err } diff --git a/internal/ingest/resource_load_test.go b/internal/ingest/resource_load_test.go index 249135001..756c60008 100644 --- a/internal/ingest/resource_load_test.go +++ b/internal/ingest/resource_load_test.go @@ -7,11 +7,6 @@ import ( "testing" ) -// TestLoadIncompleteErrorRejectsPartialResourceLoad pins the mutable resource -// path's honest-error guarantee (docs/loom-reliability-contract.md:76). The -// accumulation loops in loadResource leave the row-level counters set when -// FailFast is disabled, so a load that inserted partial rows must surface a -// typed error rather than the nil it returned before the incomplete-check existed. func TestLoadIncompleteErrorRejectsPartialResourceLoad(t *testing.T) { if err := loadIncompleteError(LoadSummary{}); err != nil { t.Fatalf("loadIncompleteError(clean) = %v, want nil", err) diff --git a/internal/ingest/row_builder.go b/internal/ingest/row_builder.go index 8de5a2c92..5420c5b1a 100644 --- a/internal/ingest/row_builder.go +++ b/internal/ingest/row_builder.go @@ -110,9 +110,6 @@ func NewGenericRowBuilder(project string, class *jsonschema.Schema, schema *grap } func (b *GenericRowBuilder) Build(resourceType string, line []byte, stageSeconds map[string]float64) (rowBuildResult, rowErrorType, error) { - // jsonschemagraph currently deletes its optional namespace entry from the - // extra-arguments map. Generic builders are shared by parallel ingest - // workers, so each row must give that dependency an isolated mutable map. extraArgs := maps.Clone(b.extraArgs) var payload map[string]any diff --git a/internal/ingest/semantic_inventory_backfill.go b/internal/ingest/semantic_inventory_backfill.go index 63418a08c..0266ab33f 100644 --- a/internal/ingest/semantic_inventory_backfill.go +++ b/internal/ingest/semantic_inventory_backfill.go @@ -12,7 +12,6 @@ import ( "time" "github.com/calypr/loom/internal/catalog" - catalogarango "github.com/calypr/loom/internal/catalog/arango" publication "github.com/calypr/loom/internal/dataset" fhirschema "github.com/calypr/loom/internal/fhir/schema" ) @@ -63,7 +62,7 @@ type SemanticInventoryBackfillReport struct { type semanticInventoryBackfillBackend interface { PrepareSemanticInventoryBackfill(context.Context) error - ReadRetainedSemanticInventoryPage(context.Context, string, string, string, string, int) (catalogarango.RetainedSemanticInventoryPage, error) + ReadRetainedSemanticInventoryPage(context.Context, string, string, string, string, int) (catalog.RetainedSemanticInventoryPage, error) ClaimSemanticInventoryBackfill(context.Context, catalog.SemanticInventoryBuild, string, time.Duration) (catalog.SemanticInventoryBuild, bool, error) AdvanceSemanticInventoryBackfill(context.Context, catalog.SemanticInventoryBuild, string, catalog.SemanticInventoryCheckpoint, catalog.SemanticInventoryCheckpoint, int64, time.Duration) (catalog.SemanticInventoryBuild, error) CompleteSemanticInventoryBackfill(context.Context, catalog.SemanticInventoryBuild, string, catalog.SemanticInventoryCheckpoint, int64) (catalog.SemanticInventoryBuild, error) @@ -72,9 +71,6 @@ type semanticInventoryBackfillBackend interface { EnsureSemanticInventoryEntries(context.Context, catalog.SemanticInventoryBuild) (catalog.SemanticInventoryBuild, error) } -// BackfillSemanticInventory reconstructs the inventory from retained immutable -// generation vertices. Source rows are read in bounded pages; contribution -// batches are replaced by deterministic keys before the typed checkpoint moves. func BackfillSemanticInventory(ctx context.Context, backend semanticInventoryBackfillBackend, manifest publication.Manifest, opts SemanticInventoryBackfillOptions) (SemanticInventoryBackfillReport, error) { started := time.Now() if backend == nil { @@ -292,7 +288,7 @@ func validateSemanticInventoryBackfillRequest(manifest publication.Manifest, opt return nil } -func retainedSemanticPayload(source catalogarango.RetainedSemanticInventoryRow, resourceType string) (map[string]any, error) { +func retainedSemanticPayload(source catalog.RetainedSemanticInventoryRow, resourceType string) (map[string]any, error) { if source.Key == "" || len(source.Payload) == 0 || string(source.Payload) == "null" { return nil, fmt.Errorf("retained %s/%s source is missing payload", resourceType, source.Key) } diff --git a/internal/ingest/semantic_inventory_backfill_integration_test.go b/internal/ingest/semantic_inventory_backfill_integration_test.go index 5c390d3ae..ae52acc3e 100644 --- a/internal/ingest/semantic_inventory_backfill_integration_test.go +++ b/internal/ingest/semantic_inventory_backfill_integration_test.go @@ -9,10 +9,9 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/catalog" - catalogarango "github.com/calypr/loom/internal/catalog/arango" publication "github.com/calypr/loom/internal/dataset" - arangostore "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -27,17 +26,18 @@ func TestRetainedSemanticInventoryBackfillArangoIntegration(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - client, err := arangostore.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client defer client.Close(context.Background()) project := "c01-backfill-" + strings.ReplaceAll(uuid.NewString(), "-", "") generation := strings.ReplaceAll(uuid.NewString(), "-", "") - if err := client.Bootstrap(ctx, arangostore.BootstrapSpec{Collections: []arangostore.CollectionSpec{{Name: "Observation"}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Observation"}}}); err != nil { t.Fatal(err) } - store, err := catalogarango.New(client) + store, err := arangofactory.NewCatalogStore(client) if err != nil { t.Fatal(err) } @@ -72,7 +72,7 @@ func TestRetainedSemanticInventoryBackfillArangoIntegration(t *testing.T) { if _, claimed, err := store.ClaimSemanticInventoryBackfill(ctx, build, "integration-owner", time.Minute); err != nil || !claimed { t.Fatalf("initial explicit claim claimed=%v err=%v", claimed, err) } - if _, claimed, err := store.ClaimSemanticInventoryBackfill(ctx, build, "integration-contender", time.Minute); err != catalogarango.ErrSemanticInventoryClaimed || claimed { + if _, claimed, err := store.ClaimSemanticInventoryBackfill(ctx, build, "integration-contender", time.Minute); err != arangofactory.ErrSemanticInventoryClaimed || claimed { t.Fatalf("concurrent explicit claim claimed=%v err=%v, want claim rejection", claimed, err) } if err := store.FailSemanticInventoryBackfill(ctx, build, "integration-owner", "release test claim"); err != nil { @@ -133,13 +133,13 @@ func backfillIntegrationPayload(scope string, distinct int) map[string]any { return map[string]any{"resourceType": "Observation", "id": scope, "component": components} } -func readBackfillIntegrationSources(t *testing.T, ctx context.Context, store *catalogarango.Store, project, generation string) map[string]catalogarango.RetainedSemanticInventoryRow { +func readBackfillIntegrationSources(t *testing.T, ctx context.Context, store *arangofactory.CatalogStore, project, generation string) map[string]catalog.RetainedSemanticInventoryRow { t.Helper() page, err := store.ReadRetainedSemanticInventoryPage(ctx, project, generation, "Observation", "", 10) if err != nil { t.Fatal(err) } - rows := make(map[string]catalogarango.RetainedSemanticInventoryRow, len(page.Rows)) + rows := make(map[string]catalog.RetainedSemanticInventoryRow, len(page.Rows)) for _, row := range page.Rows { rows[row.Key] = row } @@ -164,7 +164,7 @@ func integrationBackfillManifest(t *testing.T, project, generation string) publi return manifest } -func assertRetainedInventoryPageCodes(t *testing.T, ctx context.Context, store *catalogarango.Store, project, generation string, paths []string, wantCount int, prefix string, sharedPopulation int64) { +func assertRetainedInventoryPageCodes(t *testing.T, ctx context.Context, store *arangofactory.CatalogStore, project, generation string, paths []string, wantCount int, prefix string, sharedPopulation int64) { t.Helper() codes, err := collectRetainedInventoryPageCodes(ctx, store, project, generation, paths) if err != nil { @@ -195,7 +195,7 @@ func assertRetainedInventoryPageCodes(t *testing.T, ctx context.Context, store * } } -func collectRetainedInventoryPageCodes(ctx context.Context, store *catalogarango.Store, project, generation string, paths []string) (map[string]catalog.SemanticObservation, error) { +func collectRetainedInventoryPageCodes(ctx context.Context, store *arangofactory.CatalogStore, project, generation string, paths []string) (map[string]catalog.SemanticObservation, error) { options := catalog.SemanticInventoryPageOptions{ Project: project, DatasetGeneration: generation, AuthResourcePaths: paths, AuthResourcePathsUnrestricted: catalog.ExplicitAuthResourcePathsUnrestricted(false), diff --git a/internal/ingest/semantic_inventory_backfill_test.go b/internal/ingest/semantic_inventory_backfill_test.go index 19c5d89e5..2e16cd8c9 100644 --- a/internal/ingest/semantic_inventory_backfill_test.go +++ b/internal/ingest/semantic_inventory_backfill_test.go @@ -11,20 +11,20 @@ import ( "testing" "time" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/catalog" - catalogarango "github.com/calypr/loom/internal/catalog/arango" publication "github.com/calypr/loom/internal/dataset" ) func TestBackfillSemanticInventoryResumesPartialPageWithoutDuplicates(t *testing.T) { backend := newBackfillFake() - backend.rows["Observation"] = []catalogarango.RetainedSemanticInventoryRow{ + backend.rows["Observation"] = []catalog.RetainedSemanticInventoryRow{ backfillSourceRow(t, "a", "scope-a", backfillObservation("shared", "v1", false)), backfillSourceRow(t, "b", "scope-b", backfillObservation("shared", "v1", false)), backfillSourceRow(t, "c", "scope-a", backfillObservation("third", "v1", true)), } original := cloneBackfillRows(backend.rows["Observation"]) - backend.failWriteAt = 4 // First event for source c persists, its next event is interrupted. + backend.failWriteAt = 4 options := SemanticInventoryBackfillOptions{Project: "project", DatasetGeneration: "generation", PageSize: 2, BatchSize: 1} manifest := backfillManifest(t, publication.StateStaged) @@ -81,7 +81,7 @@ func TestBackfillSemanticInventoryRejectsConcurrentClaim(t *testing.T) { _, err := BackfillSemanticInventory(context.Background(), backend, backfillManifest(t, publication.StateStaged), SemanticInventoryBackfillOptions{ Project: "project", DatasetGeneration: "generation", }) - if !errors.Is(err, catalogarango.ErrSemanticInventoryClaimed) { + if !errors.Is(err, arangofactory.ErrSemanticInventoryClaimed) { t.Fatalf("concurrent claim error = %v, want ErrSemanticInventoryClaimed", err) } if backend.readCalls != 0 || len(backend.contributions) != 0 { @@ -91,7 +91,7 @@ func TestBackfillSemanticInventoryRejectsConcurrentClaim(t *testing.T) { func TestBackfillSemanticInventoryMalformedPayloadFailsAndPreservesSource(t *testing.T) { backend := newBackfillFake() - backend.rows["Observation"] = []catalogarango.RetainedSemanticInventoryRow{{Key: "bad", AuthResourcePath: strptr("scope-a"), Payload: json.RawMessage(`[]`)}} + backend.rows["Observation"] = []catalog.RetainedSemanticInventoryRow{{Key: "bad", AuthResourcePath: strptr("scope-a"), Payload: json.RawMessage(`[]`)}} original := cloneBackfillRows(backend.rows["Observation"]) _, err := BackfillSemanticInventory(context.Background(), backend, backfillManifest(t, publication.StateReady), SemanticInventoryBackfillOptions{ Project: "project", DatasetGeneration: "generation", @@ -135,7 +135,7 @@ func TestBackfillSemanticInventoryRequiresMatchingImmutableManifest(t *testing.T func TestBackfillSemanticInventoryMarksMissingUnprofiledRootUnproven(t *testing.T) { backend := newBackfillFake() - backend.rows["Observation"] = []catalogarango.RetainedSemanticInventoryRow{backfillSourceRow(t, "a", "", backfillObservation("one", "v1", false))} + backend.rows["Observation"] = []catalog.RetainedSemanticInventoryRow{backfillSourceRow(t, "a", "", backfillObservation("one", "v1", false))} backend.missingCollections["Patient"] = true manifest := backfillManifestWithRoots(t, publication.StateStaged, []string{"Observation", "Patient"}) report, err := BackfillSemanticInventory(context.Background(), backend, manifest, SemanticInventoryBackfillOptions{ @@ -154,7 +154,7 @@ func TestBackfillSemanticInventoryMarksMissingUnprofiledRootUnproven(t *testing. func TestBackfillSemanticInventoryCancellationReleasesClaimForImmediateResume(t *testing.T) { backend := newBackfillFake() - backend.rows["Observation"] = []catalogarango.RetainedSemanticInventoryRow{ + backend.rows["Observation"] = []catalog.RetainedSemanticInventoryRow{ backfillSourceRow(t, "a", "", backfillObservation("one", "v1", false)), backfillSourceRow(t, "b", "", backfillObservation("two", "v1", false)), } @@ -214,13 +214,13 @@ func backfillObservation(code, version string, withComponent bool) map[string]an return payload } -func backfillSourceRow(t *testing.T, key, authPath string, payload map[string]any) catalogarango.RetainedSemanticInventoryRow { +func backfillSourceRow(t *testing.T, key, authPath string, payload map[string]any) catalog.RetainedSemanticInventoryRow { t.Helper() encoded, err := json.Marshal(payload) if err != nil { t.Fatal(err) } - return catalogarango.RetainedSemanticInventoryRow{Key: key, AuthResourcePath: strptr(authPath), Payload: encoded} + return catalog.RetainedSemanticInventoryRow{Key: key, AuthResourcePath: strptr(authPath), Payload: encoded} } func backfillManifest(t *testing.T, state publication.State) publication.Manifest { @@ -245,8 +245,8 @@ func backfillManifestWithRoots(t *testing.T, state publication.State, resourceTy return manifest } -func cloneBackfillRows(rows []catalogarango.RetainedSemanticInventoryRow) []catalogarango.RetainedSemanticInventoryRow { - cloned := make([]catalogarango.RetainedSemanticInventoryRow, len(rows)) +func cloneBackfillRows(rows []catalog.RetainedSemanticInventoryRow) []catalog.RetainedSemanticInventoryRow { + cloned := make([]catalog.RetainedSemanticInventoryRow, len(rows)) for index, row := range rows { cloned[index] = row cloned[index].Payload = append(json.RawMessage(nil), row.Payload...) @@ -260,7 +260,7 @@ func cloneBackfillRows(rows []catalogarango.RetainedSemanticInventoryRow) []cata func strptr(value string) *string { return &value } type backfillFake struct { - rows map[string][]catalogarango.RetainedSemanticInventoryRow + rows map[string][]catalog.RetainedSemanticInventoryRow contributions map[string]catalog.SemanticInventoryContribution build catalog.SemanticInventoryBuild claimedByOther bool @@ -275,7 +275,7 @@ type backfillFake struct { func newBackfillFake() *backfillFake { return &backfillFake{ - rows: map[string][]catalogarango.RetainedSemanticInventoryRow{}, + rows: map[string][]catalog.RetainedSemanticInventoryRow{}, contributions: map[string]catalog.SemanticInventoryContribution{}, missingCollections: map[string]bool{}, } @@ -286,20 +286,20 @@ func (f *backfillFake) PrepareSemanticInventoryBackfill(context.Context) error { return nil } -func (f *backfillFake) ReadRetainedSemanticInventoryPage(ctx context.Context, project, generation, collection, afterKey string, limit int) (catalogarango.RetainedSemanticInventoryPage, error) { +func (f *backfillFake) ReadRetainedSemanticInventoryPage(ctx context.Context, project, generation, collection, afterKey string, limit int) (catalog.RetainedSemanticInventoryPage, error) { if err := ctx.Err(); err != nil { - return catalogarango.RetainedSemanticInventoryPage{}, err + return catalog.RetainedSemanticInventoryPage{}, err } f.readCalls++ if project != "project" || generation != "generation" { - return catalogarango.RetainedSemanticInventoryPage{}, fmt.Errorf("unexpected source scope %s/%s", project, generation) + return catalog.RetainedSemanticInventoryPage{}, fmt.Errorf("unexpected source scope %s/%s", project, generation) } - rows := append([]catalogarango.RetainedSemanticInventoryRow(nil), f.rows[collection]...) + rows := append([]catalog.RetainedSemanticInventoryRow(nil), f.rows[collection]...) if f.missingCollections[collection] { - return catalogarango.RetainedSemanticInventoryPage{Rows: []catalogarango.RetainedSemanticInventoryRow{}}, nil + return catalog.RetainedSemanticInventoryPage{Rows: []catalog.RetainedSemanticInventoryRow{}}, nil } sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key }) - page := catalogarango.RetainedSemanticInventoryPage{Rows: make([]catalogarango.RetainedSemanticInventoryRow, 0, limit), SourceExists: true} + page := catalog.RetainedSemanticInventoryPage{Rows: make([]catalog.RetainedSemanticInventoryRow, 0, limit), SourceExists: true} for _, row := range rows { if row.Key <= afterKey { continue @@ -315,13 +315,13 @@ func (f *backfillFake) ReadRetainedSemanticInventoryPage(ctx context.Context, pr func (f *backfillFake) ClaimSemanticInventoryBackfill(_ context.Context, build catalog.SemanticInventoryBuild, token string, lease time.Duration) (catalog.SemanticInventoryBuild, bool, error) { if f.claimedByOther { - return catalog.SemanticInventoryBuild{}, false, catalogarango.ErrSemanticInventoryClaimed + return catalog.SemanticInventoryBuild{}, false, arangofactory.ErrSemanticInventoryClaimed } if f.build.State == catalog.SemanticInventoryComplete { return f.build, false, nil } if f.build.LeaseToken != "" && f.build.LeaseToken != token { - return catalog.SemanticInventoryBuild{}, false, catalogarango.ErrSemanticInventoryClaimed + return catalog.SemanticInventoryBuild{}, false, arangofactory.ErrSemanticInventoryClaimed } if f.build.Key == "" { f.build = build @@ -340,7 +340,7 @@ func (f *backfillFake) ClaimSemanticInventoryBackfill(_ context.Context, build c func (f *backfillFake) AdvanceSemanticInventoryBackfill(_ context.Context, build catalog.SemanticInventoryBuild, token string, expected, next catalog.SemanticInventoryCheckpoint, scanned int64, lease time.Duration) (catalog.SemanticInventoryBuild, error) { f.advanceCalls++ if f.build.LeaseToken != token || f.build.SourceCheckpoint != expected { - return catalog.SemanticInventoryBuild{}, catalogarango.ErrSemanticInventoryClaimLost + return catalog.SemanticInventoryBuild{}, arangofactory.ErrSemanticInventoryClaimLost } f.build.SourceCheckpoint = next f.build.ScannedResources = scanned @@ -352,7 +352,7 @@ func (f *backfillFake) AdvanceSemanticInventoryBackfill(_ context.Context, build func (f *backfillFake) CompleteSemanticInventoryBackfill(_ context.Context, build catalog.SemanticInventoryBuild, token string, expected catalog.SemanticInventoryCheckpoint, scanned int64) (catalog.SemanticInventoryBuild, error) { f.completeCalls++ if f.build.LeaseToken != token || f.build.SourceCheckpoint != expected { - return catalog.SemanticInventoryBuild{}, catalogarango.ErrSemanticInventoryClaimLost + return catalog.SemanticInventoryBuild{}, arangofactory.ErrSemanticInventoryClaimLost } f.build.State = catalog.SemanticInventoryComplete f.build.ScannedResources = scanned @@ -368,7 +368,7 @@ func (f *backfillFake) FailSemanticInventoryBackfill(ctx context.Context, _ cata return err } if f.build.LeaseToken != token { - return catalogarango.ErrSemanticInventoryClaimLost + return arangofactory.ErrSemanticInventoryClaimLost } f.build.State = catalog.SemanticInventoryFailed f.build.LeaseToken = "" diff --git a/internal/ingest/semantic_inventory_test.go b/internal/ingest/semantic_inventory_test.go index 024920000..a02c777e5 100644 --- a/internal/ingest/semantic_inventory_test.go +++ b/internal/ingest/semantic_inventory_test.go @@ -9,9 +9,8 @@ import ( "time" "github.com/bmeg/jsonschemagraph/graph" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/catalog" - catalogarango "github.com/calypr/loom/internal/catalog/arango" - arangostore "github.com/calypr/loom/internal/store/arango" ) func TestLoadFilePersistsSemanticInventoryFromSharedEmitter(t *testing.T) { @@ -35,8 +34,8 @@ func TestLoadFilePersistsSemanticInventoryFromSharedEmitter(t *testing.T) { var inventoryDocs []json.RawMessage var inventoryOverwrite []bool result, err := loadFile(context.Background(), opts, nil, schema, file, "generation-1", false, time.Now(), 0, 0, - func(_ context.Context, _ *arangostore.Client, collection string, docs []json.RawMessage, overwrite bool, _ string) error { - if collection == catalogarango.SemanticCoverageCollection { + func(_ context.Context, _ *arangofactory.Client, collection string, docs []json.RawMessage, overwrite bool, _ string) error { + if collection == arangofactory.CatalogSemanticCoverageCollection { t.Fatalf("normal generation load wrote optional semantic coverage collection %q", collection) } if collection == catalog.SemanticInventoryCollection { diff --git a/internal/ingest/source_schema.go b/internal/ingest/source_schema.go index a02fd9d49..1a019c3e5 100644 --- a/internal/ingest/source_schema.go +++ b/internal/ingest/source_schema.go @@ -17,14 +17,10 @@ import ( ) var ( - // ErrGraphSchemaPathRequired reports a missing configured graph-schema path. ErrGraphSchemaPathRequired = errors.New("graph schema path is required") - // ErrMalformedGraphSchema reports JSON that cannot safely provide schema - // identity metadata. - ErrMalformedGraphSchema = errors.New("malformed graph schema") + ErrMalformedGraphSchema = errors.New("malformed graph schema") ) -// loadSchemaSnapshot records exact source bytes and generated FHIR roots. func loadSchemaSnapshot(path string) (publication.SchemaSnapshot, error) { if strings.TrimSpace(path) == "" { return publication.SchemaSnapshot{}, ErrGraphSchemaPathRequired diff --git a/internal/projectid/projectid.go b/internal/projectid/projectid.go index 7a4df3788..8f7ed2b96 100644 --- a/internal/projectid/projectid.go +++ b/internal/projectid/projectid.go @@ -1,7 +1,3 @@ -// Package projectid defines the public project identity used by Loom's -// Explorer APIs. Existing generations and Explorer records may still be -// keyed by the historical program-project form, so callers can derive that -// storage alias without putting the legacy format back on the wire. package projectid import ( @@ -9,9 +5,6 @@ import ( "strings" ) -// Canonical returns the GitHub-style program/project identity. Legacy IDs are -// accepted at compatibility boundaries and split at their first hyphen so a -// project name may continue to contain additional hyphens. func Canonical(raw string) string { raw = strings.TrimSpace(raw) if raw == "" { @@ -30,9 +23,6 @@ func Canonical(raw string) string { } if index := strings.IndexByte(raw, '-'); index > 0 && index < len(raw)-1 { program := raw[:index] - // Keep opaque single-component test/legacy IDs such as project-a - // untouched. Real program/project IDs use the established program - // namespace (for example HTAN_INT or EllrottLab) and are converted. if program != strings.ToLower(program) || strings.Contains(program, "_") { return program + "/" + raw[index+1:] } @@ -40,8 +30,6 @@ func Canonical(raw string) string { return raw } -// Legacy returns the historical program-project storage identity for a -// canonical ID. IDs that cannot be split are returned unchanged. func Legacy(raw string) string { canonical := Canonical(raw) if index := strings.IndexByte(canonical, '/'); index > 0 && index < len(canonical)-1 { diff --git a/internal/repositoryseed/preflight.go b/internal/repositoryseed/preflight.go index ff3987931..477d4a636 100644 --- a/internal/repositoryseed/preflight.go +++ b/internal/repositoryseed/preflight.go @@ -1,5 +1,3 @@ -// Package repositoryseed validates a repository checkout before the local -// launcher is allowed to mutate Docker or database state. package repositoryseed import ( diff --git a/internal/server/append_workspace_rows_integration_test.go b/internal/server/append_workspace_rows_integration_test.go index e4534557a..df757e947 100644 --- a/internal/server/append_workspace_rows_integration_test.go +++ b/internal/server/append_workspace_rows_integration_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/calypr/loom/internal/dataframe/compiler" "net/http" "os" "reflect" @@ -14,11 +15,10 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" execution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/execution/chartifact" - chartifactarango "github.com/calypr/loom/internal/dataframe/execution/chartifact/arango" "github.com/calypr/loom/internal/dataframe/recipe" - storearango "github.com/calypr/loom/internal/store/arango" storeclickhouse "github.com/calypr/loom/internal/store/clickhouse" "github.com/google/uuid" ) @@ -128,10 +128,11 @@ func TestCurrentDraftAppendRowsAgainstArangoClickHouse(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() database := "loom_append_rows_" + strings.ReplaceAll(uuid.NewString(), "-", "") - graph, err := storearango.Open(ctx, arangoURL, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: arangoURL, Database: database}) if err != nil { t.Fatal(err) } + graph := arangoBackend.Client t.Cleanup(func() { graph.Close(context.Background()) request, _ := http.NewRequest(http.MethodDelete, strings.TrimRight(arangoURL, "/")+"/_api/database/"+database, nil) @@ -145,7 +146,7 @@ func TestCurrentDraftAppendRowsAgainstArangoClickHouse(t *testing.T) { t.Errorf("private database cleanup HTTP%d", response.StatusCode) } }) - for _, spec := range []storearango.BootstrapSpec{chartifactarango.BootstrapSpec(), {Collections: []storearango.CollectionSpec{{Name: "Patient"}}}} { + for _, spec := range []arangofactory.BootstrapSpec{arangofactory.ArtifactBootstrapSpec(), {Collections: []arangofactory.CollectionSpec{{Name: "Patient"}}}} { if err = graph.Bootstrap(ctx, spec); err != nil { t.Fatal(err) } @@ -167,7 +168,7 @@ func TestCurrentDraftAppendRowsAgainstArangoClickHouse(t *testing.T) { t.Error(err) } }) - catalog, err := chartifactarango.NewCatalog(graph) + catalog, err := arangofactory.NewArtifactCatalog(graph) if err != nil { t.Fatal(err) } @@ -175,8 +176,15 @@ func TestCurrentDraftAppendRowsAgainstArangoClickHouse(t *testing.T) { if err != nil { t.Fatal(err) } - engine, err := execution.New(execution.Config{Registry: compilerTestRegistry{}, QueryRows: func(ctx context.Context, query string, batch int, vars map[string]any, visit func(map[string]any) error) error { - return graph.QueryRows(ctx, query, batch, vars, storearango.RowVisitor(visit)) + engine, err := execution.New(execution.Config{CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: func(ctx context.Context, request execution.RowsRequest, visit execution.RowVisitor) error { + command := request.Command + batch := request.BatchSize + query, vars, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + + return graph.QueryRows(ctx, query, batch, vars, arangofactory.RowVisitor(visit)) }, ClickHouseQueryRows: ch.QueryRowsArgsVisit, PrivateClickHouseArtifacts: manager}) if err != nil { t.Fatal(err) @@ -236,7 +244,7 @@ func TestCurrentDraftAppendRowsAgainstArangoClickHouse(t *testing.T) { sortAppendCurrentDraftRows(actual) sortAppendCurrentDraftRows(step.expected) var manifests int - if e = graph.QueryRows(ctx, "FOR d IN @@c RETURN d", 10, map[string]any{"@c": chartifactarango.PrivateArtifactsCollection}, func(map[string]any) error { manifests++; return nil }); e != nil || manifests != 0 { + if e = graph.QueryRows(ctx, "FOR d IN @@c RETURN d", 10, map[string]any{"@c": arangofactory.PrivateArtifactsCollection}, func(map[string]any) error { manifests++; return nil }); e != nil || manifests != 0 { t.Fatalf("capture cleanup: manifests=%d err=%v", manifests, e) } if !reflect.DeepEqual(actual, step.expected) { diff --git a/internal/server/artifact_dev.go b/internal/server/artifact_dev.go index d3cc91254..efc533775 100644 --- a/internal/server/artifact_dev.go +++ b/internal/server/artifact_dev.go @@ -8,9 +8,6 @@ import ( "github.com/calypr/loom/internal/explorer/lifecycle" ) -// artifactDelayReader makes export/publication races deterministic in the -// isolated no-auth development stack. Production configuration cannot enable -// it because the delay is populated only by applyDevFaultOverrides. type artifactDelayReader struct { next lifecycle.ArtifactPublishedReader delay time.Duration diff --git a/internal/server/artifact_reconciler_wiring_test.go b/internal/server/artifact_reconciler_wiring_test.go index e1c7ea29b..6073cac5d 100644 --- a/internal/server/artifact_reconciler_wiring_test.go +++ b/internal/server/artifact_reconciler_wiring_test.go @@ -59,9 +59,6 @@ func TestStartArtifactFSReconcilerSweepsImmediatelyAndStopsOnCancel(t *testing.T } } -// Drives runReconcilerLoop synchronously (manual ticks, no goroutine) so the -// logger buffer is only ever touched by this goroutine: the fs sweep must be -// labeled distinctly from the ClickHouse private artifact sweep. func TestFilesystemReconcilerLoopUsesFilesystemArtifactLabel(t *testing.T) { buffer := &bytes.Buffer{} sweeps := 0 diff --git a/internal/server/auth_wiring.go b/internal/server/auth_wiring.go index 618499697..0a2ec62bc 100644 --- a/internal/server/auth_wiring.go +++ b/internal/server/auth_wiring.go @@ -14,6 +14,7 @@ type authComponents struct { authenticator authscope.Authenticator authorizer authscope.Authorizer scopeResolver *authscope.ScopeResolver + scopePolicy routeScopePolicy } func wireAuth(cfg Config, noAuth bool, listExisting func(context.Context, catalog.AuthResourcePathOptions) ([]string, error)) (authComponents, error) { @@ -21,6 +22,7 @@ func wireAuth(cfg Config, noAuth bool, listExisting func(context.Context, catalo return authComponents{ authenticator: authscope.StaticAuthenticator{Principal: authscope.Principal{Subject: "anonymous"}}, authorizer: authscope.AllowAllAuthorizer{}, + scopePolicy: unrestrictedRouteScope(), }, nil } switch { @@ -28,6 +30,7 @@ func wireAuth(cfg Config, noAuth bool, listExisting func(context.Context, catalo return authComponents{ authenticator: authscope.BasicAuthenticator{Username: cfg.Auth.Basic.Username, Password: cfg.Auth.Basic.Password}, authorizer: authscope.AllowAllAuthorizer{}, + scopePolicy: unrestrictedRouteScope(), }, nil case strings.EqualFold(cfg.Auth.Mode, "calypr"): client := &http.Client{Timeout: cfg.Auth.Calypr.RequestTimeout} @@ -44,6 +47,7 @@ func wireAuth(cfg Config, noAuth bool, listExisting func(context.Context, catalo authenticator: authscope.CalyprAuthenticator{}, authorizer: authscope.ScopeAuthorizer{Resolver: resolver}, scopeResolver: resolver, + scopePolicy: scopedRouteScope(resolver), }, nil default: return authComponents{}, fmt.Errorf("unsupported auth mode %q", cfg.Auth.Mode) diff --git a/internal/server/backend_architecture_conformance_test.go b/internal/server/backend_architecture_conformance_test.go new file mode 100644 index 000000000..6eb7fa465 --- /dev/null +++ b/internal/server/backend_architecture_conformance_test.go @@ -0,0 +1,2527 @@ +package server + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "regexp" + "runtime" + "slices" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/gofiber/fiber/v3" + "github.com/jackc/pgx/v5/pgxpool" + + loomapi "github.com/calypr/loom/generated/loomapi" + "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" + backendspostgres "github.com/calypr/loom/internal/backends/postgres" + "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler/command" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + publication "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/explorer" + "github.com/calypr/loom/internal/explorer/authoringv2" + "github.com/calypr/loom/internal/ingest" +) + +const ( + ab10Project1 = "ab10-p1" + ab10Project2 = "ab10-p2" + ab10GenA = "ab10-gen-a" + ab10GenB = "ab10-gen-b" + ab10GenC = "ab10-gen-c" + + ab10PathMain = "/programs/prog-a/projects/ab10-p1/clinical" + ab10PathForeign = "/programs/prog-a/projects/ab10-p2/clinical" + ab10PathUngrant = "/programs/prog-b/projects/ab10-p3/clinical" + + ab10SubjectGranted = "ab10-granted" + ab10SubjectUnrelated = "ab10-unrelated" + ab10SubjectBroad = "ab10-broad" +) + +type ab10Literal = map[string]map[string]string + +func ab10PatientLines(t *testing.T) []string { + t.Helper() + return ab10JSONLines(t, + map[string]any{"resourceType": "Patient", "id": "ab10-patient-ada", "gender": "female", "birthDate": "1992-04-11"}, + map[string]any{"resourceType": "Patient", "id": "ab10-patient-nogender", "birthDate": "1988-01-02"}, + map[string]any{"resourceType": "Patient", "id": "ab10-patient-nullfield", "gender": nil, "birthDate": "2000-12-31"}, + ) +} + +func ab10ObservationLines(t *testing.T) []string { + t.Helper() + return ab10JSONLines(t, + map[string]any{"resourceType": "Observation", "id": "ab10-obs-1", "status": "final", "code": map[string]any{"coding": []map[string]any{{"system": "http://loinc.org", "code": "13457-7", "display": "Cholesterol in LDL [Mass/volume] in Serum or Plasma"}}}, "subject": map[string]any{"reference": "Patient/ab10-patient-ada"}}, + map[string]any{"resourceType": "Observation", "id": "ab10-obs-2", "status": "final", "code": map[string]any{"coding": []map[string]any{{"system": "http://loinc.org", "code": "13457-7", "display": "Cholesterol in LDL [Mass/volume] in Serum or Plasma"}}}, "subject": map[string]any{"reference": "Patient/ab10-patient-ada"}}, + ) +} + +func ab10ForeignPatientLines(t *testing.T) []string { + t.Helper() + return ab10JSONLines(t, + map[string]any{"resourceType": "Patient", "id": "ab10-patient-foreign", "gender": "male", "birthDate": "1975-07-07"}, + ) +} + +func ab10StagedPatientLines(t *testing.T) []string { + t.Helper() + return ab10JSONLines(t, + map[string]any{"resourceType": "Patient", "id": "ab10-patient-staged", "gender": "other", "birthDate": "1999-09-09"}, + ) +} + +func ab10JSONLines(t *testing.T, docs ...map[string]any) []string { + t.Helper() + lines := make([]string, 0, len(docs)) + for _, doc := range docs { + encoded, err := json.Marshal(doc) + if err != nil { + t.Fatalf("marshal shared-graph document: %v", err) + } + lines = append(lines, string(encoded)) + } + return lines +} + +func ab10GenFiles(t *testing.T) map[string][]string { + t.Helper() + return map[string][]string{ + "Patient.ndjson": ab10PatientLines(t), + "Observation.ndjson": ab10ObservationLines(t), + } +} + +const ab10PatientQueryBody = `{"query":"mutation($input: FhirDataframeInput!, $limit: Int) { runFhirDataframe(input: $input, limit: $limit) { columns rowCount rows } }","variables":{"input":{"project":"` + ab10Project1 + `","rootResourceType":"Patient","rootFields":[{"name":"id","selector":{"valuePath":"id"}},{"name":"gender","selector":{"valuePath":"gender"}},{"name":"birthDate","selector":{"valuePath":"birthDate"}}]},"limit":10}}` + +const ab10ForeignQueryBody = `{"query":"mutation($input: FhirDataframeInput!, $limit: Int) { runFhirDataframe(input: $input, limit: $limit) { columns rowCount rows } }","variables":{"input":{"project":"` + ab10Project2 + `","rootResourceType":"Patient","rootFields":[{"name":"id","selector":{"valuePath":"id"}},{"name":"gender","selector":{"valuePath":"gender"}},{"name":"birthDate","selector":{"valuePath":"birthDate"}}]},"limit":10}}` + +const ab10ObservationQueryBody = `{"query":"mutation($input: FhirDataframeInput!, $limit: Int) { runFhirDataframe(input: $input, limit: $limit) { columns rowCount rows } }","variables":{"input":{"project":"` + ab10Project1 + `","rootResourceType":"Observation","rootFields":[{"name":"id","selector":{"valuePath":"id"}},{"name":"status","selector":{"valuePath":"status"}},{"name":"subject","selector":{"valuePath":"subject.reference"}}]},"limit":10}}` + +func ab10PagedQueryBody(limit int) string { + return `{"query":"mutation($input: FhirDataframeInput!, $limit: Int) { runFhirDataframe(input: $input, limit: $limit) { columns rowCount rows } }","variables":{"input":{"project":"` + ab10Project1 + `","rootResourceType":"Patient","rootFields":[{"name":"id","selector":{"valuePath":"id"}},{"name":"gender","selector":{"valuePath":"gender"}},{"name":"birthDate","selector":{"valuePath":"birthDate"}}]},"limit":` + strconv.Itoa(limit) + `}}` +} + +type ab10Row map[string]json.RawMessage + +type ab10OracleTester interface { + Helper() + Fatalf(format string, args ...any) +} + +func ab10DecodeRows(t ab10OracleTester, what, body string) ab10RowBy { + t.Helper() + var envelope struct { + Errors []struct { + Message string `json:"message"` + } `json:"errors"` + Data struct { + RunFhirDataframe struct { + Columns []string `json:"columns"` + RowCount int `json:"rowCount"` + Rows []ab10Row `json:"rows"` + } `json:"runFhirDataframe"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(body), &envelope); err != nil { + t.Fatalf("decode %s envelope: %v (body %s)", what, err, body) + } + if len(envelope.Errors) != 0 { + t.Fatalf("%s returned GraphQL errors: %s", what, body) + } + by := ab10RowBy{Columns: envelope.Data.RunFhirDataframe.Columns, RowCount: envelope.Data.RunFhirDataframe.RowCount} + for _, row := range envelope.Data.RunFhirDataframe.Rows { + key := ab10RawText(t, what, row, "_key") + by.Rows = append(by.Rows, ab10DecodedRow{Key: key, ID: ab10RawText(t, what, row, "id"), Values: row}) + } + sort.Slice(by.Rows, func(i, j int) bool { + if by.Rows[i].ID != by.Rows[j].ID { + return by.Rows[i].ID < by.Rows[j].ID + } + return by.Rows[i].Key < by.Rows[j].Key + }) + return by +} + +type ab10DecodedRow struct { + Key string + ID string + Values ab10Row +} + +type ab10RowBy struct { + Columns []string + RowCount int + Rows []ab10DecodedRow +} + +func ab10RawText(t ab10OracleTester, what string, row ab10Row, column string) string { + t.Helper() + raw, ok := row[column] + if !ok { + t.Fatalf("%s row lost column %q: %#v", what, column, row) + } + var text string + if err := json.Unmarshal(raw, &text); err != nil { + t.Fatalf("%s column %q is not a JSON string (%s): %#v", what, column, err, row) + } + return text +} + +func ab10ScalarLiteral(t ab10OracleTester, what string, row ab10Row, column, want string) { + t.Helper() + raw, present := row[column] + if !present { + t.Fatalf("%s: column %q absent, want literal %q", what, column, want) + } + var text string + if err := json.Unmarshal(raw, &text); err != nil { + t.Fatalf("%s: column %q = %s, want literal %q", what, column, raw, want) + } + if text != want { + t.Fatalf("%s: column %q = %q, want literal %q", what, column, text, want) + } +} + +var ( + ab10PGForbiddenTells = []string{"arango", "aql", "for d in ", "@auth_resource_paths", "fhir_field_catalog"} + ab10ArangoForbiddenSQL = []string{"SELECT ", "INSERT INTO", "pg_stat", "postgres"} +) + +func ab10AssertNoCrossDialectTell(t *testing.T, arm, what, body string) { + t.Helper() + tells := ab10PGForbiddenTells + if arm == "arangodb" { + tells = ab10ArangoForbiddenSQL + } + lower := strings.ToLower(body) + for _, tell := range tells { + if strings.Contains(lower, strings.ToLower(tell)) { + t.Fatalf("%s arm: %s carries a foreign-dialect tell %q: %s", arm, what, tell, body) + } + } +} + +var ( + ab10OwnedMu sync.Mutex + ab10Owned = map[string]ab10Mint{} + ab10StrayMarks []string + ab10ForeignSeen []string +) + +type ab10Mint struct { + backend string + leg string + dropped bool + seq int +} + +var ab10MintInventory = map[string]int{ + "TestBackendArchitectureConformance/postgresql": 1, + "TestBackendArchitectureConformance/postgresql/upgradeFromPinnedPreRefactorReopensSafely": 1, + "TestBackendArchitectureConformance/arangodb": 1, + "TestBackendArchitectureConformance/crossBackendCandidateIdentityParity": 2, +} + +var ab10MintSeq int + +func ab10Register(name, backend, leg string) { + ab10OwnedMu.Lock() + defer ab10OwnedMu.Unlock() + ab10MintSeq++ + ab10Owned[name] = ab10Mint{backend: backend, leg: leg, seq: ab10MintSeq} +} + +func ab10AuditLegMints(t *testing.T, arm string, want int) { + t.Helper() + ab10OwnedMu.Lock() + start := ab10MintSeq + ab10OwnedMu.Unlock() + t.Cleanup(func() { + ab10OwnedMu.Lock() + defer ab10OwnedMu.Unlock() + got := 0 + var names []string + for name, mint := range ab10Owned { + if mint.backend == arm && mint.leg == t.Name() && mint.seq > start { + got++ + names = append(names, name+" (leg "+mint.leg+")") + } + } + sort.Strings(names) + if got != want { + t.Errorf("%s arm census: leg %s minted %d resources, inventory expects %d: %v", arm, t.Name(), got, want, names) + } + }) +} + +func ab10MarkDropped(name string) { + ab10OwnedMu.Lock() + defer ab10OwnedMu.Unlock() + rec, ok := ab10Owned[name] + if !ok { + ab10StrayMarks = append(ab10StrayMarks, name) + return + } + rec.dropped = true + ab10Owned[name] = rec +} + +func ab10AuditArm(t *testing.T, arm string) { + t.Helper() + if !t.Failed() { + ab10AuditRegistry(t, arm) + } + ab10AuditSurvivors(t, arm) +} + +func ab10AuditRegistry(t *testing.T, arm string) { + t.Helper() + ab10OwnedMu.Lock() + defer ab10OwnedMu.Unlock() + prefix := "verify_ab10pg_" + if arm == "arangodb" { + prefix = "loom_test_ab10_arango_" + } + knownLegs := map[string]bool{} + for leg := range ab10MintInventory { + if strings.HasPrefix(leg, "TestBackendArchitectureConformance/"+arm) || strings.HasPrefix(leg, "TestBackendArchitectureConformance/crossBackend") { + knownLegs[leg] = true + } + } + gotLegs := map[string]int{} + for name, mint := range ab10Owned { + if mint.backend != arm { + continue + } + if !strings.HasPrefix(name, prefix) { + ab10ForeignSeen = append(ab10ForeignSeen, name) + t.Errorf("%s arm census: owned mint %q violates the owned prefix %q", arm, name, prefix) + continue + } + gotLegs[mint.leg]++ + if !mint.dropped { + t.Errorf("%s arm census: mint %q (leg %s) was never dropped", arm, name, mint.leg) + } + } + for leg := range gotLegs { + if !knownLegs[leg] { + t.Errorf("%s arm census: unexpected minting leg %q (update ab10MintInventory deliberately, with review)", arm, leg) + } + } + if len(ab10StrayMarks) != 0 { + t.Errorf("%s arm census: stray drop marks %v", arm, ab10StrayMarks) + } +} + +func ab10AuditSurvivors(t *testing.T, arm string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if arm == "postgresql" { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Errorf("postgresql arm census: LOOM_TEST_POSTGRES_URL unset at arm end") + return + } + pool, err := pgxpool.New(ctx, raw) + if err != nil { + t.Errorf("postgresql arm census: connect for survivor count: %v", err) + return + } + defer pool.Close() + var count int64 + if err := pool.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname ~ '^verify_ab10pg_[0-9]+_[0-9a-f]{12}$'").Scan(&count); err != nil { + t.Errorf("postgresql arm census: live survivor count: %v", err) + return + } + if count != 0 { + t.Errorf("postgresql arm census: %d verify_ab10pg_* databases survived the arm", count) + } + return + } + endpoint := strings.TrimSpace(os.Getenv("LOOM_TEST_ARANGO_URL")) + if endpoint == "" { + t.Errorf("arangodb arm census: LOOM_TEST_ARANGO_URL unset at arm end") + return + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, + strings.TrimRight(endpoint, "/")+"/_api/database", nil) + if err != nil { + t.Errorf("arangodb arm census: build database list request: %v", err) + return + } + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Errorf("arangodb arm census: list databases: %v", err) + return + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Errorf("arangodb arm census: list databases HTTP %d", response.StatusCode) + return + } + var listing struct { + Result []string `json:"result"` + } + if err := json.NewDecoder(response.Body).Decode(&listing); err != nil { + t.Errorf("arangodb arm census: decode database list: %v", err) + return + } + for _, name := range listing.Result { + if strings.HasPrefix(name, "loom_test_ab10_arango_") { + t.Errorf("arangodb arm census: database %q survived the arm", name) + } + } +} + +func TestBackendArchitectureConformance(t *testing.T) { + t.Run("postgresql", testBackendArchitectureConformancePostgres) + t.Run("arangodb", testBackendArchitectureConformanceArango) + t.Run("crossBackendCandidateIdentityParity", testBackendArchitectureConformanceCrossBackendParity) +} + +func ab10Logger() *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) +} + +func ab10RandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) +} + +func ab10RepoPath(t *testing.T, elems ...string) string { + t.Helper() + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("locate test file for repository root") + } + dir := filepath.Dir(thisFile) + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return filepath.Join(append([]string{dir}, elems...)...) + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("no go.mod found above the test file") + } + dir = parent + } +} + +var ( + ab10PGDBPattern = regexp.MustCompile(`^verify_ab10pg_[0-9]+_[0-9a-f]{12}$`) + ab10PGSchemaPattern = regexp.MustCompile(`^verify_ab10pg_[0-9]+_[0-9a-f]{12}_(src|exp|pub|rel|rcp|exe)$`) + ab10PGSeriesTokens = []struct { + series control.Series + token string + }{ + {control.SeriesSource, "src"}, + {control.SeriesExplorerControl, "exp"}, + {control.SeriesPublicationControl, "pub"}, + {control.SeriesDatasetRelease, "rel"}, + {control.SeriesRecipeChains, "rcp"}, + {control.SeriesRecipeExecRegistries, "exe"}, + } + ab10PGBaselineMu sync.Mutex + ab10PGBaseline int64 + ab10PGBaselineSet bool + ab10PGBaselineErr error +) + +type ab10PGFixture struct { + t *testing.T + dsn string + schemas map[control.Series]string + scrub func(any) string + admin *pgxpool.Pool + iso *pgxpool.Pool + database string +} + +func newAB10PGFixture(t *testing.T) *ab10PGFixture { + t.Helper() + raw := ab10PGURL(t) + ab10PGCaptureBaseline(t, raw) + ab10PGBaselineMu.Lock() + baselineErr, baseline := ab10PGBaselineErr, ab10PGBaseline + ab10PGBaselineMu.Unlock() + if baselineErr != nil { + t.Fatalf("capture pg_database baseline: %v", baselineErr) + } + if baseline != 0 { + t.Fatalf("zero-survivor precondition: %d verify_ab10pg_* databases existed before this arm", baseline) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + secrets := []string{raw} + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + + ctx, cancel := context.WithTimeout(context.Background(), 540*time.Second) + t.Cleanup(cancel) + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %s", scrub(err)) + } + t.Cleanup(admin.Close) + + database := fmt.Sprintf("verify_ab10pg_%d_%s", time.Now().UnixNano(), ab10RandomHex(t)) + if !ab10PGDBPattern.MatchString(database) { + t.Fatalf("minted database %q violates the owned-name pattern", database) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+database+`"`); err != nil { + t.Fatalf("create isolated database: %s", scrub(err)) + } + ab10Register(database, "postgresql", t.Name()) + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(context.Background()), + `DROP DATABASE IF EXISTS "`+database+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + return + } + ab10MarkDropped(database) + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + } + u.Path = "/" + database + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect to isolated database: %s", scrub(err)) + } + t.Cleanup(iso.Close) + schemas := map[control.Series]string{} + for _, placement := range ab10PGSeriesTokens { + schema := fmt.Sprintf("verify_ab10pg_%d_%s_%s", time.Now().UnixNano(), ab10RandomHex(t), placement.token) + if !ab10PGSchemaPattern.MatchString(schema) { + t.Fatalf("minted schema %q violates the owned-name pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + t.Fatalf("create owned schema for %s: %s", placement.series, scrub(err)) + } + schemas[placement.series] = schema + } + return &ab10PGFixture{t: t, dsn: dsn, schemas: schemas, scrub: scrub, admin: admin, iso: iso, database: database} +} + +func ab10PGURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL conformance arm (blocked acceptance: adjudicated by the CI conformance lane)") + } + return raw +} + +func ab10PGCaptureBaseline(t *testing.T, raw string) { + t.Helper() + ab10PGBaselineMu.Lock() + defer ab10PGBaselineMu.Unlock() + if ab10PGBaselineSet { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + ab10PGBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname ~ '^verify_ab10pg_[0-9]+_[0-9a-f]{12}$'"). + Scan(&count); err != nil { + ab10PGBaselineErr = fmt.Errorf("count baseline: %w", err) + return + } + ab10PGBaseline = count + ab10PGBaselineSet = true +} + +func (f *ab10PGFixture) openConfig(setup control.SetupPolicy) control.Config { + return control.Config{ + DSN: f.dsn, + MaxConns: 8, + MinConns: 1, + AcquireTimeout: 30 * time.Second, + Schemas: control.SeriesSchemas(f.schemas), + Setup: setup, + } +} + +func (f *ab10PGFixture) migrate(t *testing.T) { + t.Helper() + rt, err := backendspostgres.Open(context.Background(), f.openConfig(control.SetupMigrate)) + if err != nil { + t.Fatalf("migrate six-series fixture: %s", f.scrub(err)) + } + if err := rt.Close(); err != nil { + t.Fatalf("close migrated fixture runtime: %s", f.scrub(err)) + } +} + +func (f *ab10PGFixture) catalogRuntime(t *testing.T) *backendspostgres.Runtime { + t.Helper() + rt, err := backendspostgres.Open(context.Background(), f.openConfig(control.SetupNone)) + if err != nil { + t.Fatalf("open catalog fixture runtime: %s", f.scrub(err)) + } + if rt.FieldProfiles == nil || rt.Inventory == nil || rt.Relationships == nil || rt.Contributions == nil { + _ = rt.Close() + t.Fatal("fixture runtime assembled without catalog evidence ports") + } + return rt +} + +func (f *ab10PGFixture) controlRuntime(t *testing.T) *backendspostgres.Runtime { + t.Helper() + rt, err := backendspostgres.Open(context.Background(), f.openConfig(control.SetupNone)) + if err != nil { + t.Fatalf("open control runtime: %s", f.scrub(err)) + } + if rt.Releases == nil || rt.Manifests == nil || rt.Explorer == nil || rt.SourceRows == nil { + _ = rt.Close() + t.Fatal("control runtime assembled without required ports") + } + return rt +} + +func (f *ab10PGFixture) pgOptions(application string) ingest.PGSourceOptions { + return ingest.PGSourceOptions{ + DSN: f.dsn, + Schema: f.schemas[control.SeriesSource], + ManifestSchema: f.schemas[control.SeriesDatasetRelease], + ApplicationName: application, + } +} + +func (f *ab10PGFixture) load(t *testing.T, project, generation, authPath string, files map[string][]string) { + t.Helper() + dir := t.TempDir() + for name, lines := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil { + t.Fatalf("write staging fixture: %v", err) + } + } + catalogRT := f.catalogRuntime(t) + defer func() { _ = catalogRT.Close() }() + ref, err := publication.NewRef(project, generation) + if err != nil { + t.Fatalf("build dataset ref: %v", err) + } + opts := ingest.LoadOptions{ + Schema: ab10RepoPath(t, "schemas", "graph-fhir.json"), + MetaDir: dir, + Project: project, + Dataset: &ref, + AuthResourcePath: authscope.NormalizeAuthResourcePath(authPath), + StageOnly: true, + FailFast: true, + } + pg := f.pgOptions("verify-ab10pg") + pg.Catalog = ingest.GenerationCatalogWriters{ + FieldProfiles: catalogRT.FieldProfiles, + Inventory: catalogRT.Inventory, + Relationships: catalogRT.Relationships, + Contributions: catalogRT.Contributions, + } + if _, err := ingest.LoadPostgresGeneration(context.Background(), opts, pg); err != nil { + t.Fatalf("load %s/%s: %s", project, generation, f.scrub(err)) + } +} + +func (f *ab10PGFixture) activate(t *testing.T, project, generation string) { + t.Helper() + ref, err := publication.NewRef(project, generation) + if err != nil { + t.Fatalf("build dataset ref: %v", err) + } + if err := ingest.ActivatePostgresGeneration(context.Background(), ref, f.pgOptions("verify-ab10pg-activate")); err != nil { + t.Fatalf("activate %s/%s: %s", project, generation, f.scrub(err)) + } +} + +func (f *ab10PGFixture) bootConfig(t *testing.T, listen string) Config { + t.Helper() + cfg := ab10PGBootConfigBase(t, f, listen) + cfg.Server.AllowUnauthenticated = true + return cfg +} + +func (f *ab10PGFixture) bootConfigCalypr(t *testing.T, listen, trustedIssuer string) Config { + t.Helper() + cfg := ab10PGBootConfigBase(t, f, listen) + cfg.Auth.Mode = "calypr" + cfg.Auth.Calypr.TrustedIssuer = trustedIssuer + cfg.Auth.Calypr.CacheTTL = time.Minute + cfg.Auth.Calypr.RequestTimeout = 10 * time.Second + return cfg +} + +func ab10PGBootConfigBase(t *testing.T, f *ab10PGFixture, listen string) Config { + t.Helper() + u, err := url.Parse(f.dsn) + if err != nil { + t.Fatalf("parse fixture dsn: %v", err) + } + host, portText, err := net.SplitHostPort(u.Host) + if err != nil { + t.Fatalf("split fixture dsn host: %v", err) + } + port, err := strconv.Atoi(portText) + if err != nil { + t.Fatalf("fixture dsn port %q: %v", portText, err) + } + password := "" + if pw, ok := u.User.Password(); ok { + password = pw + } + sslMode := u.Query().Get("sslmode") + if sslMode == "" { + sslMode = "disable" + } + cfg := Config{} + cfg.Server.Backend = sourceBackendPostgres + cfg.Server.Listen = listen + cfg.Server.Schema = ab10RepoPath(t, "schemas", "graph-fhir.json") + cfg.Server.ArtifactDirectory = t.TempDir() + cfg.Server.PopulationMappingCursorSecret = "ab10-pg-cursor-secret-0123456789ab" + schemas := map[string]string{} + for _, placement := range ab10PGSeriesTokens { + schemas[string(placement.series)] = f.schemas[placement.series] + } + cfg.Server.PostgreSQL = PostgreSQLConfig{ + Host: host, Port: port, Database: strings.TrimPrefix(u.Path, "/"), + User: u.User.Username(), Password: password, SSLMode: sslMode, + MaxOpenConns: 8, MaxIdleConns: 2, + Schemas: schemas, + } + return cfg +} + +func (f *ab10PGFixture) probeCount(t *testing.T, query string, args ...any) int64 { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + var count int64 + if err := f.iso.QueryRow(ctx, query, args...).Scan(&count); err != nil { + t.Fatalf("census probe: %s", f.scrub(err)) + } + return count +} + +func testBackendArchitectureConformancePostgres(t *testing.T) { + ab10PGURL(t) + t.Cleanup(func() { ab10AuditArm(t, "postgresql") }) + ab10AuditLegMints(t, "postgresql", 1) + fixture := newAB10PGFixture(t) + fixture.migrate(t) + fixture.load(t, ab10Project1, ab10GenA, ab10PathMain, ab10GenFiles(t)) + fixture.load(t, ab10Project1, ab10GenB, ab10PathMain, map[string][]string{"Patient.ndjson": ab10StagedPatientLines(t)}) + fixture.load(t, ab10Project2, ab10GenA, ab10PathForeign, map[string][]string{"Patient.ndjson": ab10ForeignPatientLines(t)}) + fixture.activate(t, ab10Project1, ab10GenA) + fixture.activate(t, ab10Project2, ab10GenA) + + fence := ab10StartFenceStub(t) + + ctx, cancel := context.WithTimeout(context.Background(), 420*time.Second) + defer cancel() + server, closeRuntime, err := composePostgreSQLFullRuntime(ctx, fixture.bootConfig(t, "127.0.0.1:0"), ab10Logger()) + if err != nil { + t.Fatalf("production compose (postgresql arm): %s", fixture.scrub(err)) + } + defer func() { + if err := closeRuntime(); err != nil { + t.Errorf("close postgresql arm runtime: %s", fixture.scrub(err)) + } + }() + + arm := &ab10Arm{ + name: "postgresql", + app: server.App(), + fixture: fixture, + fence: fence, + scrub: fixture.scrub, + } + + ab10CellScalarQuery(t, arm) + ab10CellRepeatedOccurrence(t, arm) + ab10CellNullAndMissing(t, arm) + ab10CellAuthFamily(t, arm) + ab10CellUnrestrictedSeesEverything(t, arm) + ab10CellStablePaging(t, arm) + ab10CellDraftCASOneWinner(t, arm) + ab10CellPublicationControlFence(t, arm) + ab10CellCancellationNoFallback(t, arm) + ab10CellBackendUnavailableNoFallback(t, arm) + ab10CellGenerationIsolation(t, arm) + ab10CellUpgradeReopen(t, arm) + + ab10AuditPG(t, fixture) +} + +func ab10AuditPG(t *testing.T, fixture *ab10PGFixture) { + t.Helper() + var survivors []string + rows, err := fixture.admin.Query(context.Background(), + `SELECT datname FROM pg_database WHERE datname LIKE 'verify\_ab10pg\_%' ESCAPE '\'`) + if err != nil { + t.Fatalf("pg_database survivor audit: %s", fixture.scrub(err)) + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("scan survivor row: %s", fixture.scrub(err)) + } + survivors = append(survivors, name) + } + if err := rows.Err(); err != nil { + t.Fatalf("survivor rows: %s", fixture.scrub(err)) + } + var expected []string + ab10OwnedMu.Lock() + for name, mint := range ab10Owned { + if mint.backend == "postgresql" && !mint.dropped { + expected = append(expected, name) + } + } + ab10OwnedMu.Unlock() + sort.Strings(survivors) + sort.Strings(expected) + if len(survivors) != len(expected) { + t.Errorf("zero-survivor audit: live %v, registered set %v", survivors, expected) + } else { + for i := range survivors { + if survivors[i] != expected[i] { + t.Errorf("zero-survivor audit: live %v, registered set %v", survivors, expected) + break + } + } + } +} + +func ab10ArangoEndpoint(t *testing.T) string { + t.Helper() + endpoint := strings.TrimSpace(os.Getenv("LOOM_TEST_ARANGO_URL")) + if endpoint == "" { + t.Skip("set LOOM_TEST_ARANGO_URL to run the ArangoDB conformance arm (blocked acceptance: adjudicated by the go-arango and CI conformance lanes)") + } + return endpoint +} + +func ab10ArangoOwnDatabase(t *testing.T, endpoint, leg string) string { + t.Helper() + database := "loom_test_ab10_arango_" + ab10RandomHex(t) + ab10RandomHex(t) + request, err := http.NewRequestWithContext(context.Background(), http.MethodPost, + strings.TrimRight(endpoint, "/")+"/_api/database", + strings.NewReader(`{"name":"`+database+`"}`)) + if err != nil { + t.Fatalf("build create-database request: %v", err) + } + request.Header.Set("Content-Type", "application/json") + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatalf("mint owned database: %v", err) + } + defer response.Body.Close() + var minted struct { + Result bool `json:"result"` + } + body, readErr := io.ReadAll(io.LimitReader(response.Body, 1<<16)) + if readErr != nil { + t.Fatalf("read create-database response: %v", readErr) + } + if !(response.StatusCode >= 200 && response.StatusCode < 300) || + json.Unmarshal(body, &minted) != nil || !minted.Result { + t.Fatalf("mint owned database %q: HTTP %d body %s (refusing a pre-existing name)", database, response.StatusCode, body) + } + ab10Register(database, "arangodb", leg) + t.Cleanup(func() { + cleanup, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(cleanup, http.MethodDelete, + strings.TrimRight(endpoint, "/")+"/_api/database/"+url.PathEscape(database), nil) + if err != nil { + t.Errorf("build drop request: %v", err) + return + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Errorf("drop owned database %s: %v", database, err) + return + } + defer resp.Body.Close() + if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNotFound) { + t.Errorf("drop owned database %s: HTTP %d", database, resp.StatusCode) + return + } + ab10MarkDropped(database) + }) + return database +} + +func testBackendArchitectureConformanceArango(t *testing.T) { + endpoint := ab10ArangoEndpoint(t) + t.Cleanup(func() { ab10AuditArm(t, "arangodb") }) + ab10AuditLegMints(t, "arangodb", 1) + database := ab10ArangoOwnDatabase(t, endpoint, t.Name()) + + ctx, cancel := context.WithTimeout(context.Background(), 420*time.Second) + defer cancel() + backend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) + if err != nil { + t.Fatalf("open owned arango database through the factory: %v", err) + } + defer func() { _ = backend.Close(context.Background()) }() + + refA, err := publication.NewRef(ab10Project1, ab10GenA) + if err != nil { + t.Fatalf("build gen-a dataset ref: %v", err) + } + refB, err := publication.NewRef(ab10Project1, ab10GenB) + if err != nil { + t.Fatalf("build gen-b dataset ref: %v", err) + } + refForeign, err := publication.NewRef(ab10Project2, ab10GenA) + if err != nil { + t.Fatalf("build foreign dataset ref: %v", err) + } + stampMain := authscope.NormalizeAuthResourcePath(ab10PathMain) + stampForeign := authscope.NormalizeAuthResourcePath(ab10PathForeign) + if _, err := ingest.Load(ctx, ingest.LoadOptions{ + ConnectionOptions: ingest.ConnectionOptions{URL: endpoint, Database: database}, + Schema: ab10RepoPath(t, "schemas", "graph-fhir.json"), + MetaDir: ab10WriteMetaDir(t, ab10GenFiles(t)), + Project: ab10Project1, + Dataset: &refA, + AuthResourcePath: stampMain, + }); err != nil { + t.Fatalf("arango ingest load+activate %s/%s: %v", ab10Project1, ab10GenA, err) + } + if _, err := ingest.Load(ctx, ingest.LoadOptions{ + ConnectionOptions: ingest.ConnectionOptions{URL: endpoint, Database: database}, + Schema: ab10RepoPath(t, "schemas", "graph-fhir.json"), + MetaDir: ab10WriteMetaDir(t, map[string][]string{"Patient.ndjson": ab10StagedPatientLines(t)}), + Project: ab10Project1, + Dataset: &refB, + AuthResourcePath: stampMain, + StageOnly: true, + }); err != nil { + t.Fatalf("arango ingest staged %s/%s: %v", ab10Project1, ab10GenB, err) + } + if _, err := ingest.Load(ctx, ingest.LoadOptions{ + ConnectionOptions: ingest.ConnectionOptions{URL: endpoint, Database: database}, + Schema: ab10RepoPath(t, "schemas", "graph-fhir.json"), + MetaDir: ab10WriteMetaDir(t, map[string][]string{"Patient.ndjson": ab10ForeignPatientLines(t)}), + Project: ab10Project2, + Dataset: &refForeign, + AuthResourcePath: stampForeign, + }); err != nil { + t.Fatalf("arango ingest load+activate %s/%s: %v", ab10Project2, ab10GenA, err) + } + + fence := ab10StartFenceStub(t) + + arm := &ab10Arm{ + name: "arangodb", + fence: fence, + endpoint: endpoint, + database: database, + backend: backend, + probe: &ab10ArangoProbe{endpoint: endpoint, database: database}, + scrub: func(v any) string { return fmt.Sprint(v) }, + } + arm.app = ab10BootArangoApp(t, ctx, arm, true) + + ab10CellScalarQuery(t, arm) + ab10CellRepeatedOccurrence(t, arm) + ab10CellNullAndMissing(t, arm) + ab10CellAuthFamily(t, arm) + ab10CellUnrestrictedSeesEverything(t, arm) + ab10CellStablePaging(t, arm) + ab10CellDraftCASOneWinner(t, arm) + ab10CellPublicationControlFence(t, arm) + ab10CellCancellationNoFallback(t, arm) + ab10CellBackendUnavailableNoFallback(t, arm) + ab10CellGenerationIsolation(t, arm) + ab10CellUpgradeReopen(t, arm) +} + +const ab10ParityExplorer = "ab10-parity-probe" + +func ab10ParityPatientLines(t *testing.T) []string { + t.Helper() + return ab10JSONLines(t, + map[string]any{"resourceType": "Patient", "id": "ab10-patient-parity", "gender": "female", "birthDate": "1990-02-24"}, + ) +} + +func ab10ParityObservationLines(t *testing.T) []string { + t.Helper() + subject := map[string]any{"reference": "Patient/ab10-patient-parity"} + code := func(codeValue, display string) map[string]any { + return map[string]any{"coding": []map[string]any{{"system": "https://bench.example/codes", "code": codeValue, "display": display}}} + } + return ab10JSONLines(t, + map[string]any{"resourceType": "Observation", "id": "ab10-parity-obs-string", "status": "final", "code": code("PARITY-STRING", "Parity String"), "subject": subject, "valueString": "self-reported:no"}, + map[string]any{"resourceType": "Observation", "id": "ab10-parity-obs-concept", "status": "final", "code": code("PARITY-CONCEPT", "Parity Concept"), "subject": subject, "valueCodeableConcept": map[string]any{"coding": []map[string]any{{"system": "https://bench.example/codes", "code": "PARITY-STAGE-B", "display": "Stage B"}}}}, + map[string]any{"resourceType": "Observation", "id": "ab10-parity-obs-concept-text", "status": "final", "code": code("PARITY-TEXT", "Parity Text"), "subject": subject, "valueCodeableConcept": map[string]any{"text": "STAGE-A"}}, + map[string]any{"resourceType": "Observation", "id": "ab10-parity-obs-quantity", "status": "final", "code": code("PARITY-QUANTITY", "Parity Quantity"), "subject": subject, "valueQuantity": map[string]any{"value": 42, "unit": "a", "system": "http://unitsofmeasure.org", "code": "a"}}, + map[string]any{"resourceType": "Observation", "id": "ab10-parity-obs-mixed", "status": "final", "code": map[string]any{"text": "Parity Mixed Text", "coding": []map[string]any{{"system": "https://bench.example/codes", "code": "PARITY-MIXED", "display": "Parity Mixed"}}}, "subject": subject, "valueQuantity": map[string]any{"value": 7}, "valueString": "mixed-arm"}, + map[string]any{"resourceType": "Observation", "id": "ab10-parity-obs-bare", "status": "final", "code": code("PARITY-BARE", "Parity Bare"), "subject": subject}, + ) +} + +func ab10ParityCatalogIdentity(t *testing.T, arm *ab10Arm, ctx context.Context) (loomapi.Catalog, string) { + t.Helper() + ab10SeedDraft(t, arm, ctx, ab10ParityExplorer, nil) + response, state := ab10Builder(t, arm, ab10ParityExplorer) + if response.StatusCode != http.StatusOK { + t.Fatalf("%s arm: parity builder status %d body %s", arm.name, response.StatusCode, response.Body) + } + if state.Catalog.SnapshotToken == "" { + t.Fatalf("%s arm: parity builder catalog carries no snapshot identity: %s", arm.name, response.Body) + } + return state.Catalog, arm.scrub(response.Body) +} + +func ab10ParitySortedIDs[T any](items []T, id func(T) string) []string { + ids := make([]string, 0, len(items)) + for _, item := range items { + ids = append(ids, id(item)) + } + sort.Strings(ids) + return ids +} + +func testBackendArchitectureConformanceCrossBackendParity(t *testing.T) { + endpoint := strings.TrimSpace(os.Getenv("LOOM_TEST_ARANGO_URL")) + if strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) == "" || endpoint == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL and LOOM_TEST_ARANGO_URL to run the cross-backend candidate identity pin (blocked acceptance: adjudicated by the CI conformance lane)") + } + t.Cleanup(func() { + ab10AuditArm(t, "postgresql") + ab10AuditArm(t, "arangodb") + }) + ab10AuditLegMints(t, "postgresql", 1) + ab10AuditLegMints(t, "arangodb", 1) + + files := map[string][]string{ + "Patient.ndjson": ab10ParityPatientLines(t), + "Observation.ndjson": ab10ParityObservationLines(t), + } + + fixture := newAB10PGFixture(t) + fixture.migrate(t) + fixture.load(t, ab10Project1, ab10GenA, ab10PathMain, files) + fixture.activate(t, ab10Project1, ab10GenA) + + database := ab10ArangoOwnDatabase(t, endpoint, t.Name()) + ctx, cancel := context.WithTimeout(context.Background(), 420*time.Second) + defer cancel() + backend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) + if err != nil { + t.Fatalf("open owned arango database through the factory: %v", err) + } + defer func() { _ = backend.Close(context.Background()) }() + ref, err := publication.NewRef(ab10Project1, ab10GenA) + if err != nil { + t.Fatalf("build parity dataset ref: %v", err) + } + if _, err := ingest.Load(ctx, ingest.LoadOptions{ + ConnectionOptions: ingest.ConnectionOptions{URL: endpoint, Database: database}, + Schema: ab10RepoPath(t, "schemas", "graph-fhir.json"), + MetaDir: ab10WriteMetaDir(t, files), + Project: ab10Project1, + Dataset: &ref, + AuthResourcePath: authscope.NormalizeAuthResourcePath(ab10PathMain), + }); err != nil { + t.Fatalf("arango parity ingest load+activate %s/%s: %v", ab10Project1, ab10GenA, err) + } + + pgServer, closePG, err := composePostgreSQLFullRuntime(ctx, fixture.bootConfig(t, "127.0.0.1:0"), ab10Logger()) + if err != nil { + t.Fatalf("production compose (parity postgresql arm): %s", fixture.scrub(err)) + } + defer func() { + if err := closePG(); err != nil { + t.Errorf("close parity postgresql arm runtime: %s", fixture.scrub(err)) + } + }() + pgArm := &ab10Arm{name: "parity-postgresql", app: pgServer.App(), fixture: fixture, scrub: fixture.scrub} + + arangoArm := &ab10Arm{ + name: "parity-arangodb", + endpoint: endpoint, + database: database, + backend: backend, + probe: &ab10ArangoProbe{endpoint: endpoint, database: database}, + scrub: func(v any) string { return fmt.Sprint(v) }, + } + arangoArm.app = ab10BootArangoApp(t, ctx, arangoArm, true) + + pgCatalog, pgBody := ab10ParityCatalogIdentity(t, pgArm, ctx) + arangoCatalog, arangoBody := ab10ParityCatalogIdentity(t, arangoArm, ctx) + + if len(pgCatalog.Candidates) == 0 || len(arangoCatalog.Candidates) == 0 { + t.Fatalf("parity pin is vacuous: postgresql carries %d candidates (%s), arangodb carries %d candidates (%s)", len(pgCatalog.Candidates), pgBody, len(arangoCatalog.Candidates), arangoBody) + } + if len(pgCatalog.Nodes) == 0 || len(arangoCatalog.Nodes) == 0 { + t.Fatalf("parity pin is vacuous: postgresql carries %d nodes, arangodb carries %d nodes", len(pgCatalog.Nodes), len(arangoCatalog.Nodes)) + } + observationCandidates := func(catalog loomapi.Catalog) int { + observationNodeIDs := map[string]struct{}{} + for _, node := range catalog.Nodes { + if node.ResourceType == "Observation" { + observationNodeIDs[node.NodeId] = struct{}{} + } + } + count := 0 + for _, candidate := range catalog.Candidates { + if _, ok := observationNodeIDs[candidate.NodeId]; ok { + count++ + } + } + return count + } + if observationCandidates(pgCatalog) == 0 || observationCandidates(arangoCatalog) == 0 { + t.Fatalf("parity pin is vacuous: postgresql carries %d Observation candidates, arangodb carries %d Observation candidates", observationCandidates(pgCatalog), observationCandidates(arangoCatalog)) + } + if pgCatalog.SnapshotToken != arangoCatalog.SnapshotToken { + t.Fatalf("cross-backend snapshot identity diverged for the identical authored dataset: postgresql token %q, arangodb token %q", pgCatalog.SnapshotToken, arangoCatalog.SnapshotToken) + } + candidates := func(catalog loomapi.Catalog) []string { + return ab10ParitySortedIDs(catalog.Candidates, func(candidate loomapi.CatalogCandidate) string { return candidate.CandidateId }) + } + nodes := func(catalog loomapi.Catalog) []string { + return ab10ParitySortedIDs(catalog.Nodes, func(node loomapi.CatalogNode) string { return node.NodeId }) + } + edges := func(catalog loomapi.Catalog) []string { + return ab10ParitySortedIDs(catalog.Edges, func(edge loomapi.CatalogEdge) string { return edge.EdgeId }) + } + pgCandidates, arangoCandidates := candidates(pgCatalog), candidates(arangoCatalog) + if !slices.Equal(pgCandidates, arangoCandidates) { + t.Fatalf("cross-backend candidate-ID multiset diverged for the identical authored dataset: postgresql %d ids %v, arangodb %d ids %v", len(pgCandidates), pgCandidates, len(arangoCandidates), arangoCandidates) + } + pgNodes, arangoNodes := nodes(pgCatalog), nodes(arangoCatalog) + if !slices.Equal(pgNodes, arangoNodes) { + t.Fatalf("cross-backend node-ID multiset diverged for the identical authored dataset: postgresql %d ids %v, arangodb %d ids %v", len(pgNodes), pgNodes, len(arangoNodes), arangoNodes) + } + pgEdges, arangoEdges := edges(pgCatalog), edges(arangoCatalog) + if !slices.Equal(pgEdges, arangoEdges) { + t.Fatalf("cross-backend edge-ID multiset diverged for the identical authored dataset: postgresql %d ids %v, arangodb %d ids %v", len(pgEdges), pgEdges, len(arangoEdges), arangoEdges) + } +} + +func ab10WriteMetaDir(t *testing.T, files map[string][]string) string { + t.Helper() + dir := t.TempDir() + for name, lines := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil { + t.Fatalf("write staging fixture: %v", err) + } + } + return dir +} + +type ab10ArangoProbe struct { + endpoint string + database string +} + +func (p *ab10ArangoProbe) countDocuments(t *testing.T, collection, project, generation string) int { + t.Helper() + query := `FOR d IN ` + collection + ` FILTER d.project == @project AND d.dataset_generation == @generation RETURN 1` + bind := map[string]any{"project": project, "generation": generation} + body := fmt.Sprintf(`{"query":%q,"bindVars":%s}`, query, mustAB10JSON(t, bind)) + request, err := http.NewRequestWithContext(context.Background(), http.MethodPost, + strings.TrimRight(p.endpoint, "/")+"/_db/"+url.PathEscape(p.database)+"/_api/cursor", + strings.NewReader(body)) + if err != nil { + t.Fatalf("build cursor request: %v", err) + } + request.Header.Set("Content-Type", "application/json") + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatalf("census cursor: %v", err) + } + defer response.Body.Close() + if !(response.StatusCode == http.StatusOK || response.StatusCode == http.StatusCreated) { + payload, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) + t.Fatalf("census cursor HTTP %d: %s", response.StatusCode, payload) + } + var payload struct { + Result []int `json:"result"` + } + if err := json.NewDecoder(response.Body).Decode(&payload); err != nil { + t.Fatalf("decode cursor result: %v", err) + } + return len(payload.Result) +} + +func (p *ab10ArangoProbe) payloadHasAttribute(t *testing.T, collection, project, generation, logicalKey, attribute string) bool { + t.Helper() + query := `LET d = FIRST(FOR doc IN @@collection FILTER doc.project == @project AND doc.dataset_generation == @generation AND doc.logical_key == @logicalKey RETURN doc) RETURN {found: d != null, hasAttribute: d != null && d.payload != null && HAS(d.payload, @attribute)}` + body := fmt.Sprintf(`{"query":%q,"bindVars":%s}`, query, mustAB10JSON(t, map[string]any{ + "@collection": collection, "project": project, "generation": generation, + "logicalKey": logicalKey, "attribute": attribute, + })) + request, err := http.NewRequestWithContext(context.Background(), http.MethodPost, + strings.TrimRight(p.endpoint, "/")+"/_db/"+url.PathEscape(p.database)+"/_api/cursor", + strings.NewReader(body)) + if err != nil { + t.Fatalf("build payload attribute request: %v", err) + } + request.Header.Set("Content-Type", "application/json") + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatalf("payload attribute cursor: %v", err) + } + defer response.Body.Close() + if !(response.StatusCode == http.StatusOK || response.StatusCode == http.StatusCreated) { + payload, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) + t.Fatalf("payload attribute cursor HTTP %d: %s", response.StatusCode, payload) + } + var payload struct { + Result []struct { + Found bool `json:"found"` + HasAttribute bool `json:"hasAttribute"` + } `json:"result"` + } + if err := json.NewDecoder(response.Body).Decode(&payload); err != nil { + t.Fatalf("decode payload attribute result: %v", err) + } + if len(payload.Result) != 1 { + t.Fatalf("payload attribute probe returned %d rows, want 1", len(payload.Result)) + } + verdict := payload.Result[0] + if !verdict.Found { + t.Fatalf("payload attribute probe: no stored %s document for project %s generation %s logical key %s — refusing to read a missing row as an absent attribute", + collection, project, generation, logicalKey) + } + return verdict.HasAttribute +} + +func mustAB10JSON(t *testing.T, value any) string { + t.Helper() + encoded, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal probe payload: %v", err) + } + return string(encoded) +} + +type ab10Arm struct { + name string + app *fiber.App + fixture *ab10PGFixture + endpoint string + database string + backend *arangofactory.Runtime + probe *ab10ArangoProbe + fence *ab10Fence + scrub func(any) string +} + +func ab10BootArangoApp(t *testing.T, ctx context.Context, arm *ab10Arm, unauthenticated bool) *fiber.App { + t.Helper() + cfg := Config{} + cfg.Server.URL = arm.endpoint + cfg.Server.Database = arm.database + cfg.Server.Listen = "127.0.0.1:0" + cfg.Server.Schema = ab10RepoPath(t, "schemas", "graph-fhir.json") + cfg.Server.ArtifactDirectory = t.TempDir() + cfg.Server.PopulationMappingCursorSecret = "ab10-arango-cursor-secret-0123" + if unauthenticated { + cfg.Server.AllowUnauthenticated = true + } else { + cfg.Auth.Mode = "calypr" + cfg.Auth.Calypr.TrustedIssuer = arm.fence.issuer + cfg.Auth.Calypr.CacheTTL = time.Minute + cfg.Auth.Calypr.RequestTimeout = 10 * time.Second + } + server, closeRuntime, err := composeArangoRuntime(ctx, cfg, ab10Logger()) + if err != nil { + t.Fatalf("production compose (arangodb arm, auth=%v): %s", unauthenticated, arm.scrub(err)) + } + t.Cleanup(func() { + if err := closeRuntime(); err != nil { + t.Errorf("close arangodb arm runtime (auth=%v): %s", unauthenticated, arm.scrub(err)) + } + }) + return server.App() +} + +type ab10Fence struct { + issuer string + tokens map[string]string + mu sync.Mutex + access map[string]map[string][]map[string]string +} + +func ab10StartFenceStub(t *testing.T) *ab10Fence { + t.Helper() + fence := &ab10Fence{tokens: map[string]string{}, access: map[string]map[string][]map[string]string{}} + stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/user" { + w.WriteHeader(http.StatusNotFound) + return + } + subject, ok := fence.subjectFor(r.Header.Get("Authorization")) + if !ok { + w.WriteHeader(http.StatusUnauthorized) + return + } + fence.mu.Lock() + grants := fence.access[subject] + fence.mu.Unlock() + if len(grants) == 0 { + grants = map[string][]map[string]string{"/programs/decoy/projects/decoy/none": {{"method": "read", "service": "*"}}} + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"authz": grants}) + })) + t.Cleanup(stub.Close) + fence.issuer = stub.URL + return fence +} + +func (f *ab10Fence) subjectFor(authorization string) (string, bool) { + f.mu.Lock() + defer f.mu.Unlock() + for subject, header := range f.tokens { + if header == authorization { + return subject, true + } + } + return "", false +} + +func (f *ab10Fence) grant(subject string, paths ...string) { + f.mu.Lock() + defer f.mu.Unlock() + grants := map[string][]map[string]string{} + for _, path := range paths { + grants[path] = []map[string]string{{"method": "read", "service": "*"}} + } + f.access[subject] = grants +} + +func (f *ab10Fence) token(t *testing.T, subject string) string { + t.Helper() + f.mu.Lock() + defer f.mu.Unlock() + if header, ok := f.tokens[subject]; ok { + return header + } + header := "Bearer " + ab10UnsignedJWT(t, map[string]any{"iss": f.issuer, "sub": subject}) + f.tokens[subject] = header + return header +} + +func ab10UnsignedJWT(t *testing.T, claims map[string]any) string { + t.Helper() + encode := func(value any) string { + raw, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal JWT segment: %v", err) + } + return base64.RawURLEncoding.EncodeToString(raw) + } + return encode(map[string]any{"alg": "none", "typ": "JWT"}) + "." + encode(claims) + ".ab10" +} + +func ab10Graph(t *testing.T, arm *ab10Arm, body string) string { + t.Helper() + return ab10GraphAuth(t, arm, body, nil) +} + +func ab10GraphAuth(t *testing.T, arm *ab10Arm, body string, headers map[string]string) string { + t.Helper() + response := requestJSONWithHeaders(t, arm.app, http.MethodPost, "/graphql/graph", body, headers) + if response.StatusCode != http.StatusOK { + t.Fatalf("%s arm: graphql transport status %d: %s", arm.name, response.StatusCode, response.Body) + } + ab10AssertNoCrossDialectTell(t, arm.name, "graphql response", response.Body) + return response.Body +} + +type ab10WantRow struct { + Key string + Values map[string]*string +} + +func ab10CheckRowsLiteral(t ab10OracleTester, arm *ab10Arm, what string, body string, want []ab10WantRow) { + t.Helper() + got := ab10DecodeRows(t, what, body) + if len(got.Rows) != len(want) { + t.Fatalf("%s arm: %s served %d rows, want exactly %d (body %s)", arm.name, what, len(got.Rows), len(want), body) + } + for i, expected := range want { + row := got.Rows[i] + if row.ID != expected.Key { + t.Fatalf("%s arm: %s row %d logical id = %q (_key %q), want literal %q", arm.name, what, i, row.ID, row.Key, expected.Key) + } + for column, value := range expected.Values { + if value == nil { + continue + } + ab10ScalarLiteral(t, arm.name+" arm: "+what, row.Values, column, *value) + } + } +} + +func ab10CheckNoRows(t ab10OracleTester, arm *ab10Arm, what string, body string) { + t.Helper() + got := ab10DecodeRows(t, what, body) + if got.RowCount != 0 || len(got.Rows) != 0 { + t.Fatalf("%s arm: %s must serve zero rows, got %d (body %s)", arm.name, what, got.RowCount, body) + } +} + +func ab10Str(value string) *string { return &value } + +func ab10CellScalarQuery(t *testing.T, arm *ab10Arm) { + t.Run("scalarQueryServesLiteralScalars", func(t *testing.T) { + body := ab10Graph(t, arm, ab10PatientQueryBody) + ab10CheckRowsLiteral(t, arm, "scalar patient query", body, []ab10WantRow{ + {Key: "ab10-patient-ada", Values: map[string]*string{"gender": ab10Str("female"), "birthDate": ab10Str("1992-04-11")}}, + {Key: "ab10-patient-nogender", Values: map[string]*string{"birthDate": ab10Str("1988-01-02")}}, + {Key: "ab10-patient-nullfield", Values: map[string]*string{"birthDate": ab10Str("2000-12-31")}}, + }) + decoded := ab10DecodeRows(t, "scalar patient query", body) + for _, required := range []string{"_key", "gender", "birthDate"} { + found := false + for _, column := range decoded.Columns { + if column == required { + found = true + } + } + if !found { + t.Fatalf("%s arm: scalar query lost column %q, got %v", arm.name, required, decoded.Columns) + } + } + if t.Failed() { + return + } + mutated := []ab10WantRow{{Key: "ab10-patient-ada", Values: map[string]*string{"gender": ab10Str("male")}}} + if red := ab10OracleGoesRed(func(shadow ab10OracleTester) { + ab10CheckRowsLiteral(shadow, arm, "scalar patient query", body, mutated) + }); !red { + t.Fatalf("%s arm: literal oracle accepted a mutated expected value — oracle is vacuous", arm.name) + } + }) +} + +func ab10OracleGoesRed(assert func(t ab10OracleTester)) bool { + shadow := &testing.T{} + done := make(chan struct{}) + go func() { + defer close(done) + assert(shadow) + }() + <-done + return shadow.Failed() +} + +func ab10CellRepeatedOccurrence(t *testing.T, arm *ab10Arm) { + t.Run("repeatedRelatedOccurrenceStaysDistinct", func(t *testing.T) { + body := ab10Graph(t, arm, ab10ObservationQueryBody) + ab10CheckRowsLiteral(t, arm, "observation query", body, []ab10WantRow{ + {Key: "ab10-obs-1", Values: map[string]*string{"status": ab10Str("final"), "subject": ab10Str("Patient/ab10-patient-ada")}}, + {Key: "ab10-obs-2", Values: map[string]*string{"status": ab10Str("final"), "subject": ab10Str("Patient/ab10-patient-ada")}}, + }) + ab10ProbeOccurrenceCount(t, arm, "Observation", ab10Project1, ab10GenA, 2) + }) +} + +func ab10ProbeOccurrenceCount(t *testing.T, arm *ab10Arm, resourceType, project, generation string, want int) { + t.Helper() + if arm.probe != nil { + if got := arm.probe.countDocuments(t, resourceType, project, generation); got != want { + t.Fatalf("%s arm census: %s/%s/%s has %d rows, want %d", arm.name, resourceType, project, generation, got, want) + } + return + } + var count int64 + if arm.name == "postgresql" { + count = arm.fixture.probeCount(t, + `SELECT count(*) FROM "`+arm.fixture.schemas[control.SeriesSource]+`".fhir_resource + WHERE project=$1 AND dataset_generation=$2 AND resource_type=$3`, + project, generation, resourceType) + } + if int(count) != want { + t.Fatalf("%s arm census: %s/%s/%s has %d rows, want %d", arm.name, resourceType, project, generation, count, want) + } +} + +func ab10ProbeStoredHasExplicitAttribute(t *testing.T, arm *ab10Arm, resourceType, project, generation, logicalID, attribute string) bool { + t.Helper() + if arm.probe != nil { + return arm.probe.payloadHasAttribute(t, resourceType, project, generation, logicalID, attribute) + } + stored := arm.fixture.probeCount(t, + `SELECT count(*) FROM "`+arm.fixture.schemas[control.SeriesSource]+`".fhir_resource + WHERE project=$1 AND dataset_generation=$2 AND resource_type=$3 AND logical_id=$4`, + project, generation, resourceType, logicalID) + if stored != 1 { + t.Fatalf("%s arm census probe: stored %s rows for project %s generation %s logical id %s, want exactly 1 — refusing to read a missing row as an absent attribute", + arm.name, resourceType, project, generation, logicalID) + } + count := arm.fixture.probeCount(t, + `SELECT count(*) FROM "`+arm.fixture.schemas[control.SeriesSource]+`".fhir_resource + WHERE project=$1 AND dataset_generation=$2 AND resource_type=$3 AND logical_id=$4 AND payload ? $5`, + project, generation, resourceType, logicalID, attribute) + if count > 1 { + t.Fatalf("%s arm census probe: %d attribute-carrying rows for one logical id, want 0 or 1", arm.name, count) + } + return count == 1 +} + +func ab10CellNullAndMissing(t *testing.T, arm *ab10Arm) { + t.Run("nullAndMissingSemanticsPreserved", func(t *testing.T) { + body := ab10Graph(t, arm, ab10PatientQueryBody) + decoded := ab10DecodeRows(t, "null/missing query", body) + byKey := map[string]ab10DecodedRow{} + for _, row := range decoded.Rows { + byKey[row.ID] = row + } + ada, ok := byKey["ab10-patient-ada"] + if !ok { + t.Fatalf("%s arm: explicit-value row missing", arm.name) + } + ab10ScalarLiteral(t, arm.name+" arm: explicit gender", ada.Values, "gender", "female") + nullfield, ok := byKey["ab10-patient-nullfield"] + if !ok { + t.Fatalf("%s arm: explicit-null row missing", arm.name) + } + nogender, ok := byKey["ab10-patient-nogender"] + if !ok { + t.Fatalf("%s arm: missing-field row missing", arm.name) + } + nullRaw, nullPresent := nullfield.Values["gender"] + missingRaw, missingPresent := nogender.Values["gender"] + if nullPresent != missingPresent || string(nullRaw) != "null" || string(missingRaw) != "null" { + t.Fatalf("%s arm: explicit-null and missing-source genders did not both serve the wire literal null (%s present=%v vs %s present=%v)", + arm.name, nullRaw, nullPresent, missingRaw, missingPresent) + } + if !ab10ProbeStoredHasExplicitAttribute(t, arm, "Patient", ab10Project1, ab10GenA, "ab10-patient-nullfield", "gender") { + t.Fatalf("%s arm: explicit-null gender was not preserved in the stored payload", arm.name) + } + if ab10ProbeStoredHasExplicitAttribute(t, arm, "Patient", ab10Project1, ab10GenA, "ab10-patient-nogender", "gender") { + t.Fatalf("%s arm: missing-source gender gained a stored attribute it never had", arm.name) + } + ab10ScalarLiteral(t, arm.name+" arm: nullfield birthDate", nullfield.Values, "birthDate", "2000-12-31") + ab10ScalarLiteral(t, arm.name+" arm: nogender birthDate", nogender.Values, "birthDate", "1988-01-02") + }) +} + +func ab10CellAuthFamily(t *testing.T, arm *ab10Arm) { + calyprApp := arm.app + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + if arm.name == "postgresql" { + server, closeRuntime, err := composePostgreSQLFullRuntime(ctx, arm.fixture.bootConfigCalypr(t, "127.0.0.1:0", arm.fence.issuer), ab10Logger()) + if err != nil { + t.Fatalf("production compose (postgresql arm, calypr): %s", arm.scrub(err)) + } + t.Cleanup(func() { + if err := closeRuntime(); err != nil { + t.Errorf("close calypr-lane runtime: %s", arm.scrub(err)) + } + }) + calyprApp = server.App() + } else { + calyprApp = ab10BootArangoApp(t, ctx, arm, false) + } + arm.fence.grant(ab10SubjectGranted, ab10PathMain) + arm.fence.grant(ab10SubjectUnrelated) + granted := map[string]string{"Authorization": arm.fence.token(t, ab10SubjectGranted)} + unrelated := map[string]string{"Authorization": arm.fence.token(t, ab10SubjectUnrelated)} + + t.Run("explicitScopeServesAuthorizedAndExcludesUnauthorized", func(t *testing.T) { + inner := *arm + inner.app = calyprApp + body := ab10GraphAuth(t, &inner, ab10PatientQueryBody, granted) + ab10CheckRowsLiteral(t, &inner, "authorized p1 query", body, []ab10WantRow{ + {Key: "ab10-patient-ada", Values: map[string]*string{"gender": ab10Str("female"), "birthDate": ab10Str("1992-04-11")}}, + {Key: "ab10-patient-nogender", Values: map[string]*string{"birthDate": ab10Str("1988-01-02")}}, + {Key: "ab10-patient-nullfield", Values: map[string]*string{"birthDate": ab10Str("2000-12-31")}}, + }) + foreign := ab10GraphAuth(t, &inner, ab10ForeignQueryBody, granted) + ab10CheckNoRows(t, &inner, "foreign p2 query for granted subject", foreign) + ab10ProbeOccurrenceCount(t, arm, "Patient", ab10Project2, ab10GenA, 1) + }) + + t.Run("restrictedEmptyScopeServesNothing", func(t *testing.T) { + inner := *arm + inner.app = calyprApp + body := ab10GraphAuth(t, &inner, ab10PatientQueryBody, unrelated) + ab10CheckNoRows(t, &inner, "p1 query for unrelated subject", body) + foreign := ab10GraphAuth(t, &inner, ab10ForeignQueryBody, unrelated) + ab10CheckNoRows(t, &inner, "p2 query for unrelated subject", foreign) + check := ab10Graph(t, arm, ab10PatientQueryBody) + if len(ab10DecodeRows(t, "unrestricted control", check).Rows) != 3 { + t.Fatalf("%s arm: unrestricted control must serve the three literal rows", arm.name) + } + }) +} + +func ab10CellUnrestrictedSeesEverything(t *testing.T, arm *ab10Arm) { + t.Run("unrestrictedSeesEverything", func(t *testing.T) { + body := ab10Graph(t, arm, ab10PatientQueryBody) + ab10CheckRowsLiteral(t, arm, "unrestricted p1", body, []ab10WantRow{ + {Key: "ab10-patient-ada", Values: map[string]*string{"gender": ab10Str("female")}}, + {Key: "ab10-patient-nogender", Values: map[string]*string{}}, + {Key: "ab10-patient-nullfield", Values: map[string]*string{}}, + }) + foreign := ab10Graph(t, arm, ab10ForeignQueryBody) + ab10CheckRowsLiteral(t, arm, "unrestricted p2", foreign, []ab10WantRow{ + {Key: "ab10-patient-foreign", Values: map[string]*string{"gender": ab10Str("male"), "birthDate": ab10Str("1975-07-07")}}, + }) + }) +} + +func ab10CellStablePaging(t *testing.T, arm *ab10Arm) { + t.Run("stablePagingIsDeterministic", func(t *testing.T) { + firstBody := ab10Graph(t, arm, ab10PagedQueryBody(1)) + first := ab10DecodeRows(t, "page one", firstBody) + second := ab10DecodeRows(t, "page two", ab10Graph(t, arm, ab10PagedQueryBody(2))) + if len(first.Rows) != 1 || len(second.Rows) != 2 { + t.Fatalf("%s arm: limit paging served %d then %d rows, want 1 then 2", arm.name, len(first.Rows), len(second.Rows)) + } + repeatBody := ab10Graph(t, arm, ab10PagedQueryBody(1)) + if firstBody != repeatBody { + t.Fatalf("%s arm: paging is nondeterministic: %s then %s", arm.name, firstBody, repeatBody) + } + seen := map[string]bool{} + for _, row := range second.Rows { + if seen[row.ID] { + t.Fatalf("%s arm: duplicate logical id %q across paging", arm.name, row.ID) + } + seen[row.ID] = true + } + if !seen[first.Rows[0].ID] { + t.Fatalf("%s arm: page of 2 lost the page-of-1 row %q, got %v", arm.name, first.Rows[0].ID, second.Rows) + } + full := ab10DecodeRows(t, "full page", ab10Graph(t, arm, ab10PagedQueryBody(10))) + if len(full.Rows) != 3 { + t.Fatalf("%s arm: terminal page served %d rows, want the full 3", arm.name, len(full.Rows)) + } + for _, want := range []string{"ab10-patient-ada", "ab10-patient-nogender", "ab10-patient-nullfield"} { + found := false + for _, row := range full.Rows { + if row.ID == want { + found = true + } + } + if !found { + t.Fatalf("%s arm: terminal page lost logical id %q", arm.name, want) + } + } + }) +} + +func ab10CellGenerationIsolation(t *testing.T, arm *ab10Arm) { + t.Run("generationIsolationServesActiveOnly", func(t *testing.T) { + body := ab10Graph(t, arm, ab10PatientQueryBody) + for _, row := range ab10DecodeRows(t, "isolation query", body).Rows { + if row.ID == "ab10-patient-staged" { + t.Fatalf("%s arm: staged generation row %q surfaced while gen-a active", arm.name, row.Key) + } + } + ab10ProbeOccurrenceCount(t, arm, "Patient", ab10Project1, ab10GenB, 1) + }) + + t.Run("releaseIntakeActivatesThroughPublicRoute", func(t *testing.T) { + if state := ab10GenerationStatus(t, arm, ab10Project1, ab10GenA); state != "STAGED" { + t.Fatalf("%s arm: active gen-a manifest state = %q, want literal STAGED", arm.name, state) + } + if state := ab10GenerationStatus(t, arm, ab10Project1, ab10GenB); state != "STAGED" { + t.Fatalf("%s arm: staged gen-b state = %q, want literal STAGED", arm.name, state) + } + upload, boundary := pgCompMultipart(t, + map[string]string{"auth_resource_path": ab10PathMain}, + map[string]string{"Patient.ndjson": strings.Join(ab10StagedPatientLines(t), "\n") + "\n"}) + request := httptest.NewRequest(http.MethodPost, + "/api/v1/datasets/"+ab10Project1+"/generations/"+ab10GenC, bytes.NewReader(upload)) + request.Header.Set("Content-Type", "multipart/form-data; boundary="+boundary) + activation, err := arm.app.Test(request, fiber.TestConfig{Timeout: 60 * time.Second, FailOnTimeout: true}) + if err != nil { + t.Fatalf("%s arm: public-route load+activate request: %v", arm.name, err) + } + raw, _ := io.ReadAll(activation.Body) + activation.Body.Close() + if activation.StatusCode < 200 || activation.StatusCode >= 300 { + t.Fatalf("%s arm: public-route load+activate: status %d body %s", arm.name, activation.StatusCode, arm.scrub(string(raw))) + } + if !strings.Contains(string(raw), "\"activated\":true") { + t.Fatalf("%s arm: public-route load did not report activated=true: %s", arm.name, arm.scrub(string(raw))) + } + if state := ab10GenerationStatus(t, arm, ab10Project1, ab10GenC); state != "STAGED" { + t.Fatalf("%s arm: post-activation gen-c manifest state = %q, want literal STAGED", arm.name, state) + } + body := ab10Graph(t, arm, ab10PatientQueryBody) + ab10CheckRowsLiteral(t, arm, "activated gen-c query", body, []ab10WantRow{ + {Key: "ab10-patient-staged", Values: map[string]*string{"gender": ab10Str("other"), "birthDate": ab10Str("1999-09-09")}}, + }) + for _, row := range ab10DecodeRows(t, "post-activation query", body).Rows { + if row.ID == "ab10-patient-ada" { + t.Fatalf("%s arm: deactivated generation rows still served", arm.name) + } + } + ab10ProbeOccurrenceCount(t, arm, "Patient", ab10Project1, ab10GenA, 3) + }) +} + +func ab10GenerationStatus(t *testing.T, arm *ab10Arm, project, generation string) string { + t.Helper() + response := requestJSON(t, arm.app, http.MethodGet, + "/api/v1/datasets/"+project+"/generations/"+generation, "") + if response.StatusCode != http.StatusOK { + t.Fatalf("%s arm: generation status read %s/%s: status %d body %s", arm.name, project, generation, response.StatusCode, response.Body) + } + var payload struct { + State string `json:"state"` + } + if err := json.Unmarshal([]byte(response.Body), &payload); err != nil { + t.Fatalf("%s arm: decode generation status: %v (body %s)", arm.name, err, response.Body) + } + return payload.State +} + +const ( + ab10ExplorerID = "ab10-draft-cas" + ab10TokenProbeExplorer = "ab10-token-probe" +) + +func ab10DraftWorkspace() authoringv2.Workspace { + return authoringv2.Workspace{ + APIVersion: authoringv2.APIVersion, Kind: authoringv2.WorkspaceKind, + Explorer: authoringv2.ExplorerMetadata{Title: "Patients"}, + Documents: []authoringv2.Document{{Kind: authoringv2.Kind, Rows: authoringv2.RecordsRowDefinition(), Output: authoringv2.Output{ID: "patients", Title: "Patients", RowLabel: "Patients"}, RootResourceType: "Patient", Route: authoringv2.RouteNode{OccurrenceID: authoringv2.RootOccurrenceID, ResourceType: "Patient"}, Columns: []authoringv2.Column{{ColumnID: "id", Column: "c_patient", Label: "Patient ID", LogicalType: "string", OccurrenceID: authoringv2.RootOccurrenceID, Source: authoringv2.ColumnSource{Kind: authoringv2.SourceField, Field: &authoringv2.FieldSource{Path: "id", ProjectionMode: "VALUE"}}}}}}, + Tabs: []authoringv2.Tab{{ID: "patients-tab", Title: "Patients", OutputID: "patients", Order: 0, Visible: true}}, + } +} + +func ab10CellDraftCASOneWinner(t *testing.T, arm *ab10Arm) { + t.Run("explorerDraftCASOneWinnerExactPreimage", func(t *testing.T) { + ctx := context.Background() + workspace := ab10DraftWorkspace() + raw, err := workspace.CanonicalJSON() + if err != nil { + t.Fatalf("canonical draft bytes: %v", err) + } + digest, err := workspace.Digest() + if err != nil { + t.Fatalf("draft digest: %v", err) + } + if digest != ab10DraftDigest { + t.Fatalf("%s arm: authored workspace digest = %q, want pinned literal %q", arm.name, digest, ab10DraftDigest) + } + if sha := fmt.Sprintf("%x", sha256.Sum256(raw)); sha != ab10DraftBytesSHA { + t.Fatalf("%s arm: canonical draft bytes sha256 = %s, want pinned literal %s", arm.name, sha, ab10DraftBytesSHA) + } + + ab10SeedDraft(t, arm, ctx, ab10TokenProbeExplorer, nil) + token, scopeDigest, generation := ab10BuilderSnapshot(t, arm, ab10TokenProbeExplorer) + ab10SeedDraft(t, arm, ctx, ab10ExplorerID, &ab10DraftSeed{ + Raw: raw, Digest: digest, Token: token, ScopeDigest: scopeDigest, Generation: generation, + }) + + response, state := ab10Builder(t, arm, ab10ExplorerID) + if response.StatusCode != http.StatusOK { + t.Fatalf("%s arm: builder preimage status %d body %s", arm.name, response.StatusCode, response.Body) + } + if state.DraftVersion != 1 || state.DraftDigest != ab10DraftDigest { + t.Fatalf("%s arm: preimage draft identity = (%d,%q), want literal (1,%q)", arm.name, state.DraftVersion, state.DraftDigest, ab10DraftDigest) + } + + type outcome struct { + status int + body string + err string + } + results := make([]outcome, 2) + labels := []string{"ab10-winner-label", "ab10-loser-label"} + var start sync.WaitGroup + start.Add(1) + var done sync.WaitGroup + for i := 0; i < 2; i++ { + done.Add(1) + go func(i int) { + defer done.Done() + start.Wait() + body := ab10UpdateColumnBody(token, fmt.Sprintf("ab10-cmd-%d", i), 1, digest, labels[i]) + response, err := ab10AttemptJSON(arm.app, http.MethodPost, + "/api/v1/projects/"+ab10Project1+"/explorers/"+ab10ExplorerID+"/authoring/v2/commands", body) + if err != nil { + results[i] = outcome{err: err.Error()} + return + } + results[i] = outcome{status: response.StatusCode, body: response.Body} + }(i) + } + start.Done() + done.Wait() + for i, result := range results { + if result.err != "" { + t.Fatalf("%s arm: CAS writer %d transport failure: %s", arm.name, i, result.err) + } + } + + var winners, losers []outcome + for _, result := range results { + switch { + case result.status == http.StatusOK: + winners = append(winners, result) + case result.status == http.StatusConflict && strings.Contains(result.body, "DRAFT_CONFLICT"): + losers = append(losers, result) + default: + t.Fatalf("%s arm: draft CAS returned status %d body %s — want one 200 and one 409 DRAFT_CONFLICT", arm.name, result.status, result.body) + } + } + if len(winners) != 1 || len(losers) != 1 { + t.Fatalf("%s arm: draft CAS produced %d winners and %d losers, want exactly 1 and 1", arm.name, len(winners), len(losers)) + } + var ack struct { + DraftVersion int64 `json:"draftVersion"` + DraftDigest string `json:"draftDigest"` + CommandId string `json:"commandId"` + } + if err := json.Unmarshal([]byte(winners[0].body), &ack); err != nil { + t.Fatalf("%s arm: decode winner ack: %v (body %s)", arm.name, err, winners[0].body) + } + if ack.DraftVersion != 2 || ack.DraftDigest == ab10DraftDigest || ack.DraftDigest == "" { + t.Fatalf("%s arm: winner ack = (%d,%q), want draftVersion literal 2 and a new digest", arm.name, ack.DraftVersion, ack.DraftDigest) + } + + after, afterState := ab10Builder(t, arm, ab10ExplorerID) + if after.StatusCode != http.StatusOK { + t.Fatalf("%s arm: builder post-CAS status %d body %s", arm.name, after.StatusCode, after.Body) + } + if afterState.DraftVersion != 2 || afterState.DraftDigest != ack.DraftDigest { + t.Fatalf("%s arm: persisted head = (%d,%q), must match winner ack (%d,%q)", arm.name, afterState.DraftVersion, afterState.DraftDigest, ack.DraftVersion, ack.DraftDigest) + } + loserLabel := labels[0] + if strings.Contains(winners[0].body, labels[0]) { + loserLabel = labels[1] + } + if strings.Contains(after.Body, loserLabel) { + t.Fatalf("%s arm: refused save left ghost bytes %q in the persisted draft", arm.name, loserLabel) + } + winnerLabel := labels[1] + if loserLabel == labels[1] { + winnerLabel = labels[0] + } + if !strings.Contains(after.Body, winnerLabel) { + t.Fatalf("%s arm: winner label %q absent from the persisted draft", arm.name, winnerLabel) + } + }) +} + +const ( + ab10DraftDigest = "sha256:8219b9afd37fc338b0ccfbcf148c210594b68981b8cf8593bdc588230b669b56" + ab10DraftBytesSHA = "8219b9afd37fc338b0ccfbcf148c210594b68981b8cf8593bdc588230b669b56" +) + +func ab10UpdateColumnBody(token, commandID string, expectedVersion int64, expectedDigest, label string) string { + column := map[string]any{ + "columnID": "id", "column": "c_patient", "label": label, "logicalType": "string", + "occurrenceID": authoringv2.RootOccurrenceID, + "source": map[string]any{"kind": "field", "field": map[string]any{"path": "id", "projectionMode": "VALUE"}}, + } + encoded, _ := json.Marshal(column) + return fmt.Sprintf(`{"commandId":%q,"semanticsVersion":%d,"snapshotToken":%q,"expectedDraftVersion":%d,"expectedDraftDigest":%q,"commands":[{"type":"UPDATE_COLUMN","outputId":"patients","column":"c_patient","columnValue":%s}]}`, + commandID, authoringv2.CurrentSemanticsVersion, token, expectedVersion, expectedDigest, encoded) +} + +type ab10DraftSeed struct { + Raw []byte + Digest string + Token string + ScopeDigest string + Generation string +} + +func ab10SeedDraft(t *testing.T, arm *ab10Arm, ctx context.Context, explorerID string, seed *ab10DraftSeed) { + t.Helper() + value := explorer.Explorer{ + Project: ab10Project1, ExplorerID: explorerID, Title: "Patients", + ManagementMode: explorer.ManagementInteractive, + } + if seed != nil { + value.DraftConfig = seed.Raw + value.DraftVersion = 1 + value.DraftDigest = seed.Digest + value.DraftSnapshotToken = seed.Token + value.DraftSourceGeneration = seed.Generation + value.DraftAuthorizationScopeDigest = seed.ScopeDigest + } + var err error + if arm.backend != nil { + _, err = arm.backend.Explorer.Create(ctx, value) + } else { + runtime := arm.fixture.catalogRuntime(t) + defer func() { _ = runtime.Close() }() + _, err = runtime.Explorer.Create(ctx, value) + } + if err != nil { + t.Fatalf("%s arm: seed draft %s through the production store port: %s", arm.name, explorerID, arm.scrub(err)) + } +} + +func ab10Builder(t *testing.T, arm *ab10Arm, explorerID string) (testHTTPResponse, loomapi.BuilderState) { + t.Helper() + response := requestJSON(t, arm.app, http.MethodGet, + "/api/v1/projects/"+ab10Project1+"/explorers/"+explorerID+"/authoring/v2/builder", "") + var state loomapi.BuilderState + if response.StatusCode == http.StatusOK { + if err := json.Unmarshal([]byte(response.Body), &state); err != nil { + t.Fatalf("%s arm: decode builder state: %v (body %s)", arm.name, err, response.Body) + } + } + return response, state +} + +func ab10BuilderSnapshot(t *testing.T, arm *ab10Arm, explorerID string) (token, scopeDigest, generation string) { + t.Helper() + response, state := ab10Builder(t, arm, explorerID) + if response.StatusCode != http.StatusOK { + t.Fatalf("%s arm: builder snapshot status %d body %s", arm.name, response.StatusCode, response.Body) + } + if state.Catalog.SnapshotToken == "" || state.Catalog.Generation == "" { + t.Fatalf("%s arm: builder catalog carries no snapshot identity: %s", arm.name, response.Body) + } + return state.Catalog.SnapshotToken, state.Catalog.AuthorizationScopeDigest, state.Catalog.Generation +} + +func ab10CellPublicationControlFence(t *testing.T, arm *ab10Arm) { + t.Run("publicationControlFenceRejectsStaleOwner", func(t *testing.T) { + ctx := context.Background() + release1 := publication.ProjectRelease{ + ID: "ab10-release-1", Project: ab10Project2, GitCommit: "ab10-commit-1", Generation: ab10GenA, + } + release2 := publication.ProjectRelease{ + ID: "ab10-release-2", Project: ab10Project2, GitCommit: "ab10-commit-2", Generation: ab10GenA, + } + type releaseStore interface { + SaveRelease(context.Context, publication.ProjectRelease) (publication.ProjectRelease, error) + CompareAndSwapActivateRelease(context.Context, publication.ProjectRelease, int64) (publication.ActiveRelease, error) + ReadActiveRelease(context.Context, string) (publication.ActiveRelease, error) + } + var store releaseStore + if arm.backend != nil { + store = arm.backend.Dataset + } else { + store = arm.fixture.controlRuntime(t).Releases + } + save := func(release publication.ProjectRelease) { + if _, err := store.SaveRelease(ctx, release); err != nil { + t.Fatalf("%s arm: save %s: %s", arm.name, release.ID, arm.scrub(err)) + } + } + cas := func(release publication.ProjectRelease, expected int64) (publication.ActiveRelease, error) { + return store.CompareAndSwapActivateRelease(ctx, release, expected) + } + casErr := func(release publication.ProjectRelease, expected int64) error { + _, err := cas(release, expected) + return err + } + readActive := func() (publication.ActiveRelease, error) { + return store.ReadActiveRelease(ctx, ab10Project2) + } + save(release1) + save(release2) + + if err := casErr(release1, 1<<40); err == nil { + t.Fatalf("%s arm: activation CAS accepted a stale expected revision with no active pointer — the fence is not enforced", arm.name) + } + if err := casErr(release1, 0); err != nil { + t.Fatalf("%s arm: first activation (expected revision 0) refused: %s", arm.name, arm.scrub(err)) + } + active, err := readActive() + if err != nil || active.Revision != 1 || active.Release.ID != release1.ID { + t.Fatalf("%s arm: first activation did not persist the pointer: active=%+v err=%v, want literal (release-1, revision 1)", arm.name, active, arm.scrub(err)) + } + if err := casErr(release1, 1<<40); err != nil { + t.Fatalf("%s arm: re-activating the active release refused instead of no-op: %s", arm.name, arm.scrub(err)) + } + if err := casErr(release2, 1<<40); err == nil { + t.Fatalf("%s arm: foreign release activated on a stale expected revision — the fence is not enforced", arm.name) + } + type casVerdict struct { + active publication.ActiveRelease + err error + } + race := func(expected int64, targets ...publication.ProjectRelease) []casVerdict { + verdicts := make([]casVerdict, len(targets)) + var wg sync.WaitGroup + for i := range targets { + wg.Add(1) + go func(i int) { + defer wg.Done() + verdicts[i].active, verdicts[i].err = cas(targets[i], expected) + }(i) + } + wg.Wait() + return verdicts + } + + for i, verdict := range race(1, release2, release2, release2, release2) { + if verdict.err != nil { + t.Fatalf("%s arm: concurrent driver %d refused the already-converged end state: %s", arm.name, i, arm.scrub(verdict.err)) + } + if verdict.active.Revision != 2 || verdict.active.Release.ID != release2.ID { + t.Fatalf("%s arm: concurrent driver %d reported (%s, revision %d), want the single converged (release-2, revision 2)", arm.name, i, verdict.active.Release.ID, verdict.active.Revision) + } + } + if active, err := readActive(); err != nil || active.Revision != 2 || active.Release.ID != release2.ID { + t.Fatalf("%s arm: takeover did not persist (release-2, revision 2): active=%+v err=%v", arm.name, active, arm.scrub(err)) + } + + release3 := publication.ProjectRelease{ + ID: "ab10-release-3", Project: ab10Project2, GitCommit: "ab10-commit-3", Generation: ab10GenA, + } + release4 := publication.ProjectRelease{ + ID: "ab10-release-4", Project: ab10Project2, GitCommit: "ab10-commit-4", Generation: ab10GenA, + } + save(release3) + save(release4) + + wins, losers := 0, 0 + for i, verdict := range race(2, release3, release4) { + switch { + case verdict.err == nil: + wins++ + if verdict.active.Revision != 3 || verdict.active.Release.ID != release3.ID && verdict.active.Release.ID != release4.ID { + t.Fatalf("%s arm: racing driver %d committed (%s, revision %d), want revision 3 on one of the two candidates", arm.name, i, verdict.active.Release.ID, verdict.active.Revision) + } + case errors.Is(verdict.err, publication.ErrReleaseActivationConflict): + losers++ + default: + t.Fatalf("%s arm: racing driver %d returned an untied failure: %s", arm.name, i, arm.scrub(verdict.err)) + } + } + if wins != 1 || losers != 1 { + t.Fatalf("%s arm: two distinct candidates on one expected revision: %d winners, %d conflicts, want exactly 1 and 1", arm.name, wins, losers) + } + taken, err := readActive() + if err != nil || taken.Revision != 3 { + t.Fatalf("%s arm: the two-way race did not land exactly one write: active=%+v err=%v, want revision 3", arm.name, taken, arm.scrub(err)) + } + + release5 := publication.ProjectRelease{ + ID: "ab10-release-5", Project: ab10Project2, GitCommit: "ab10-commit-5", Generation: ab10GenA, + } + release6 := publication.ProjectRelease{ + ID: "ab10-release-6", Project: ab10Project2, GitCommit: "ab10-commit-6", Generation: ab10GenA, + } + release7 := publication.ProjectRelease{ + ID: "ab10-release-7", Project: ab10Project2, GitCommit: "ab10-commit-7", Generation: ab10GenA, + } + save(release5) + save(release6) + save(release7) + wins, conflicts := 0, 0 + verdicts := race(3, release5, release6, release7) + for i, verdict := range verdicts { + switch { + case verdict.err == nil: + wins++ + if verdict.active.Revision != 4 { + t.Fatalf("%s arm: three-way winner %d committed revision %d, want 4 (one write past revision 3)", arm.name, i, verdict.active.Revision) + } + case errors.Is(verdict.err, publication.ErrReleaseActivationConflict): + conflicts++ + default: + t.Fatalf("%s arm: three-way driver %d returned an untied failure: %s", arm.name, i, arm.scrub(verdict.err)) + } + } + if wins != 1 || conflicts != 2 { + t.Fatalf("%s arm: three distinct candidates on one expected revision: %d winners, %d conflicts, want exactly 1 and 2", arm.name, wins, conflicts) + } + final, ferr := readActive() + if ferr != nil || final.Revision != 4 { + t.Fatalf("%s arm: three-way race did not land exactly one write: active=%+v err=%v, want revision 4", arm.name, final, arm.scrub(ferr)) + } + for i, verdict := range verdicts { + if verdict.err != nil { + continue + } + if verdict.active.Release.ID != final.Release.ID || verdict.active.Revision != final.Revision { + t.Fatalf("%s arm: winner %d reported (%s, revision %d), want the pointer's end state (%s, revision %d)", arm.name, i, verdict.active.Release.ID, verdict.active.Revision, final.Release.ID, final.Revision) + } + } + + if err := casErr(release2, 2); err == nil { + t.Fatalf("%s arm: superseded expected revision completed after takeover", arm.name) + } + + if arm.backend != nil { + registry := arm.backend.PublishedRegistry + key, owner, stale := "ab10-lease-key", "ab10-owner-a", "ab10-owner-b" + acquired, err := registry.AcquireBundleLease(ctx, key, owner, time.Now().Add(time.Minute)) + if err != nil || !acquired { + t.Fatalf("%s arm: lease acquire for owner A: acquired=%v err=%v", arm.name, acquired, err) + } + if acquired, err := registry.AcquireBundleLease(ctx, key, stale, time.Now().Add(time.Minute)); err != nil { + t.Fatalf("%s arm: foreign lease probe failed at the driver: %v", arm.name, err) + } else if acquired { + t.Fatalf("%s arm: foreign owner acquired a held lease — fence is not enforced", arm.name) + } + if renewed, err := registry.RenewBundleLease(ctx, key, stale, time.Now().Add(time.Minute)); err != nil || renewed { + t.Fatalf("%s arm: stale owner renewed the lease (renewed=%v err=%v) — false completion allowed", arm.name, renewed, err) + } + if err := registry.ReleaseBundleLease(ctx, key, stale); err != nil { + t.Fatalf("%s arm: foreign release errored instead of the documented no-op: %v", arm.name, err) + } + if renewed, err := registry.RenewBundleLease(ctx, key, owner, time.Now().Add(time.Minute)); err != nil || !renewed { + t.Fatalf("%s arm: current owner renewal refused: renewed=%v err=%v", arm.name, renewed, err) + } + if err := registry.ReleaseBundleLease(ctx, key, owner); err != nil { + t.Fatalf("%s arm: owner release refused: %v", arm.name, err) + } + if _, err := registry.AcquireBundleLease(ctx, key+"-expired", owner, time.Now().Add(-time.Second)); err != nil { + t.Fatalf("%s arm: acquiring an already-expired lease: %v", arm.name, err) + } + if acquired, err := registry.AcquireBundleLease(ctx, key+"-expired", stale, time.Now().Add(time.Minute)); err != nil || !acquired { + t.Fatalf("%s arm: expired-lease takeover refused for B: acquired=%v err=%v", arm.name, acquired, err) + } + _ = registry.ReleaseBundleLease(ctx, key+"-expired", stale) + } else { + runtime := arm.fixture.controlRuntime(t) + defer func() { _ = runtime.Close() }() + genA := mustAB10Ref(t, ab10Project1, ab10GenA) + genB := mustAB10Ref(t, ab10Project1, ab10GenB) + activeGen := func() string { + t.Helper() + resolved, err := runtime.Manifests.ResolveActiveManifest(ctx, ab10Project1) + if err != nil { + t.Fatalf("%s arm: resolve active manifest: %s", arm.name, arm.scrub(err)) + } + return resolved.Dataset.Generation + } + if got := activeGen(); got != ab10GenA { + t.Fatalf("%s arm: served generation before the fence check is %q, want %q", arm.name, got, ab10GenA) + } + stagedB, err := runtime.Manifests.ReadManifest(ctx, genB) + if err != nil { + t.Fatalf("%s arm: read staged manifest: %s", arm.name, arm.scrub(err)) + } + ghost := mustAB10Ref(t, ab10Project1, ab10GenC) + if err := runtime.Manifests.ActivateGeneration(ctx, stagedB, &ghost); err == nil { + t.Fatalf("%s arm: activation accepted a wrong expected active pointer — fence is not enforced", arm.name) + } else if errors.Is(err, publication.ErrGenerationNotReady) || !strings.Contains(err.Error(), "expected active generation") { + t.Fatalf("%s arm: wrong-pointer refusal was not the activation conflict: %s", arm.name, arm.scrub(err)) + } + if got := activeGen(); got != ab10GenA { + t.Fatalf("%s arm: refused activation moved the pointer to %q, want %q", arm.name, got, ab10GenA) + } + if err := runtime.Manifests.ActivateGeneration(ctx, stagedB, &genA); err != nil { + t.Fatalf("%s arm: activation with the matching expected pointer refused: %s", arm.name, arm.scrub(err)) + } + if got := activeGen(); got != ab10GenB { + t.Fatalf("%s arm: pointer did not move: served generation %q, want %q", arm.name, got, ab10GenB) + } + if err := runtime.Manifests.ActivateGeneration(ctx, stagedB, &ghost); err != nil { + t.Fatalf("%s arm: re-activating the active generation refused instead of completing idempotently: %s", arm.name, arm.scrub(err)) + } + stagedA, err := runtime.Manifests.ReadManifest(ctx, genA) + if err != nil { + t.Fatalf("%s arm: read manifest to restore: %s", arm.name, arm.scrub(err)) + } + if err := runtime.Manifests.ActivateGeneration(ctx, stagedA, &genB); err != nil { + t.Fatalf("%s arm: restoring the served generation refused: %s", arm.name, arm.scrub(err)) + } + if got := activeGen(); got != ab10GenA { + t.Fatalf("%s arm: served generation after restore is %q, want %q", arm.name, got, ab10GenA) + } + } + }) +} + +func mustAB10Ref(t *testing.T, project, generation string) publication.Ref { + t.Helper() + ref, err := publication.NewRef(project, generation) + if err != nil { + t.Fatalf("build ref: %v", err) + } + return ref +} + +func ab10ArmRowsDispatch(t *testing.T, arm *ab10Arm, ctx context.Context, visit func(map[string]any) error) error { + t.Helper() + if arm.backend != nil { + return selectedAQLAdapter{exec: arm.backend.Client}.serviceRows(ctx, dataframeexecution.RowsRequest{ + Command: ab10MustAQL(t, "FOR d IN Patient SORT d._key RETURN d"), + }, visit) + } + runtime := arm.fixture.catalogRuntime(t) + defer func() { _ = runtime.Close() }() + return runtime.SourceRows(ctx, dataframeexecution.RowsRequest{ + DeclaredColumns: []string{"logical_id"}, + Command: ab10MustPG(t, fmt.Sprintf( + `SELECT logical_id FROM "%s".fhir_resource WHERE project=$1 AND dataset_generation=$2 AND resource_type='Patient' ORDER BY resource_seq`, + arm.fixture.schemas[control.SeriesSource]), + command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: ab10Project1}, + command.PostgreSQLParameter{Type: command.PostgreSQLParameterText, Value: ab10GenA}), + }, visit) +} + +func ab10AssertNoForeignFallbackTell(t *testing.T, arm *ab10Arm, stage string, err error) { + t.Helper() + tell := []string{"arango", "aql"} + if arm.name != "postgresql" { + tell = []string{"postgres", "sql"} + } + low := strings.ToLower(err.Error()) + for _, term := range tell { + if strings.Contains(low, term) { + t.Fatalf("%s arm: %s reports a foreign-backend failure (fallback tell %q): %v", arm.name, stage, term, err) + } + } +} + +func ab10CellCancellationNoFallback(t *testing.T, arm *ab10Arm) { + t.Run("cancellationTypedFailureNoFallback", func(t *testing.T) { + total := 0 + censusCtx, censusCancel := context.WithCancel(context.Background()) + defer censusCancel() + if err := ab10ArmRowsDispatch(t, arm, censusCtx, func(map[string]any) error { + total++ + return nil + }); err != nil { + t.Fatalf("%s arm: row census through the production port failed: %s", arm.name, arm.scrub(err)) + } + if total < 2 { + t.Fatalf("%s arm: cancellation cell saw %d rows, want at least 2 — vacuous stop-early witness", arm.name, total) + } + + cancelledCtx, cancel := context.WithCancel(context.Background()) + cancel() + visitsOnCancelled := 0 + err := ab10ArmRowsDispatch(t, arm, cancelledCtx, func(map[string]any) error { + visitsOnCancelled++ + return nil + }) + if err == nil { + t.Fatalf("%s arm: cancelled row execution completed without a typed failure — the caller cannot observe the abort", arm.name) + } + if !errors.Is(err, context.Canceled) && !strings.Contains(strings.ToLower(err.Error()), "canceled") { + t.Fatalf("%s arm: cancellation returned an untied failure: %v", arm.name, err) + } + ab10AssertNoForeignFallbackTell(t, arm, "cancelled dispatch", err) + if visitsOnCancelled != 0 { + t.Fatalf("%s arm: cancelled execution visited %d rows, want 0", arm.name, visitsOnCancelled) + } + + abort := errors.New("ab10 client aborted") + visited := 0 + streamCtx, streamCancel := context.WithCancel(context.Background()) + defer streamCancel() + err = ab10ArmRowsDispatch(t, arm, streamCtx, func(map[string]any) error { + visited++ + if visited == 1 { + return abort + } + return nil + }) + if !errors.Is(err, abort) { + t.Fatalf("%s arm: visitor abort did not keep its identity: %v, want the caller's error", arm.name, err) + } + ab10AssertNoForeignFallbackTell(t, arm, "visitor abort", err) + if visited != 1 { + t.Fatalf("%s arm: visitor abort did not stop iteration: visited %d, want 1 of %d", arm.name, visited, total) + } + }) +} + +func ab10MustAQL(t *testing.T, query string) command.CompiledCommand { + t.Helper() + compiled, err := command.NewAQLCommand(query, nil) + if err != nil { + t.Fatalf("compile AQL witness command: %v", err) + } + return compiled +} + +func ab10MustPG(t *testing.T, sql string, args ...command.PostgreSQLParameter) command.CompiledCommand { + t.Helper() + compiled, err := command.NewPostgreSQLCommand(sql, args) + if err != nil { + t.Fatalf("compile PostgreSQL witness command: %v", err) + } + return compiled +} + +func ab10CellBackendUnavailableNoFallback(t *testing.T, arm *ab10Arm) { + t.Run("backendUnavailableAndWrongTagNoFallback", func(t *testing.T) { + t.Run("unavailableEndpointRefused", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + if arm.name == "postgresql" { + original := openArangoBackend + arangoOpens := 0 + openArangoBackend = func(ctx context.Context, cfg arangofactory.Config) (*arangofactory.Runtime, error) { + arangoOpens++ + return original(ctx, cfg) + } + t.Cleanup(func() { openArangoBackend = original }) + dead := arm.fixture.bootConfig(t, "127.0.0.1:0") + dead.Server.PostgreSQL.Port = 1 + _, closeRuntime, err := composePostgreSQLFullRuntime(ctx, dead, ab10Logger()) + if err == nil { + _ = closeRuntime() + t.Fatalf("%s arm: composing against an unavailable PostgreSQL endpoint succeeded — no typed failure", arm.name) + } + if arangoOpens != 0 { + t.Fatalf("%s arm: unavailable PostgreSQL lane opened the ArangoDB backend %d times — cross-backend fallback", arm.name, arangoOpens) + } + return + } + original := openPostgresBackend + pgOpens := 0 + openPostgresBackend = func(ctx context.Context, cfg control.Config) (*backendspostgres.Runtime, error) { + pgOpens++ + return original(ctx, cfg) + } + t.Cleanup(func() { openPostgresBackend = original }) + cfg := Config{} + cfg.Server.URL = "http://127.0.0.1:1" + cfg.Server.Database = arm.database + cfg.Server.Listen = "127.0.0.1:0" + cfg.Server.AllowUnauthenticated = true + cfg.Server.Schema = ab10RepoPath(t, "schemas", "graph-fhir.json") + cfg.Server.ArtifactDirectory = t.TempDir() + cfg.Server.PopulationMappingCursorSecret = "ab10-arango-cursor-secret-0123" + server, closeRuntime, err := composeArangoRuntime(ctx, cfg, ab10Logger()) + if err == nil { + _ = closeRuntime() + t.Fatalf("%s arm: composing against an unavailable ArangoDB endpoint succeeded — no typed failure", arm.name) + } + _ = server + if pgOpens != 0 { + t.Fatalf("%s arm: unavailable ArangoDB lane opened the PostgreSQL backend %d times — cross-backend fallback", arm.name, pgOpens) + } + }) + + t.Run("wrongTagCommandRefusedPreDial", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + pgCommand := ab10MustPG(t, "SELECT 1") + if _, _, err := pgCommand.AQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("%s arm: AQL accessor on a PostgreSQL-tagged command: %v, want ErrCompiledCommandBackendMismatch", arm.name, err) + } + aqlCommand := ab10MustAQL(t, "FOR d IN Patient RETURN d") + if _, _, err := aqlCommand.PostgreSQLParameters(); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("%s arm: PostgreSQL accessor on an AQL-tagged command: %v, want ErrCompiledCommandBackendMismatch", arm.name, err) + } + var err error + var before int64 + if arm.backend != nil { + before = ab10ServerStatementWitness(t, arm) + err = selectedAQLAdapter{exec: arm.backend.Client}.serviceRows(ctx, + dataframeexecution.RowsRequest{Command: ab10MustPG(t, "SELECT 1")}, func(map[string]any) error { return nil }) + } else { + runtime := arm.fixture.catalogRuntime(t) + defer func() { _ = runtime.Close() }() + before = ab10ServerStatementWitness(t, arm) + err = runtime.SourceRows(ctx, dataframeexecution.RowsRequest{Command: ab10MustAQL(t, "FOR d IN Patient RETURN d")}, func(map[string]any) error { return nil }) + } + if err == nil || !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("%s arm: cross-tag dispatch through the row adapter: %v, want typed ErrCompiledCommandBackendMismatch before any dial", arm.name, err) + } + ab10AssertNoServerStatements(t, arm, before) + }) + }) +} + +func ab10ServerStatementWitness(t *testing.T, arm *ab10Arm) int64 { + t.Helper() + if arm.name == "postgresql" { + return arm.fixture.probeCount(t, + `SELECT COALESCE(xact_commit, 0) FROM pg_stat_database WHERE datname = current_database()`) + } + return 0 +} + +func ab10AssertNoServerStatements(t *testing.T, arm *ab10Arm, before int64) { + t.Helper() + if arm.name != "postgresql" { + return + } + after := ab10ServerStatementWitness(t, arm) + if after-before > 1 { + t.Fatalf("%s arm: cross-tag dispatch reached the server: xact_commit grew by %d, want 1 (the before-witness read alone)", arm.name, after-before) + } +} + +func ab10AttemptJSON(app *fiber.App, method, path, body string) (testHTTPResponse, error) { + request := httptest.NewRequest(method, path, strings.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + response, err := app.Test(request) + if err != nil { + return testHTTPResponse{}, err + } + defer response.Body.Close() + raw, err := io.ReadAll(response.Body) + if err != nil { + return testHTTPResponse{}, err + } + return testHTTPResponse{StatusCode: response.StatusCode, Body: string(raw)}, nil +} + +func ab10CellUpgradeReopen(t *testing.T, arm *ab10Arm) { + t.Run("upgradeFromPinnedPreRefactorReopensSafely", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 420*time.Second) + defer cancel() + if arm.name == "postgresql" { + ab10CellUpgradeReopenPG(t, arm, ctx) + return + } + first := ab10BootArangoApp(t, ctx, arm, true) + body := ab10Graph(t, &ab10Arm{name: arm.name, app: first, scrub: arm.scrub}, ab10PatientQueryBody) + firstState := ab10DecodeRows(t, "reopen first boot", body) + second := ab10BootArangoApp(t, ctx, arm, true) + bodyAgain := ab10Graph(t, &ab10Arm{name: arm.name, app: second, scrub: arm.scrub}, ab10PatientQueryBody) + secondState := ab10DecodeRows(t, "reopen second boot", bodyAgain) + if len(firstState.Rows) != len(secondState.Rows) { + t.Fatalf("%s arm: repeat boot changed the row set: %d vs %d", arm.name, len(firstState.Rows), len(secondState.Rows)) + } + for i := range firstState.Rows { + if firstState.Rows[i].Key != secondState.Rows[i].Key { + t.Fatalf("%s arm: repeat boot reordered rows at %d", arm.name, i) + } + for column, raw := range firstState.Rows[i].Values { + if other, ok := secondState.Rows[i].Values[column]; !ok || string(raw) != string(other) { + t.Fatalf("%s arm: repeat boot changed cell %s/%s", arm.name, firstState.Rows[i].Key, column) + } + } + } + }) +} + +func ab10CellUpgradeReopenPG(t *testing.T, arm *ab10Arm, ctx context.Context) { + t.Helper() + ab10AuditLegMints(t, "postgresql", 1) + fixture := newAB10PGFixture(t) + fixture.migrate(t) + fixture.load(t, ab10Project1, ab10GenA, ab10PathMain, ab10GenFiles(t)) + fixture.activate(t, ab10Project1, ab10GenA) + source := fixture.schemas[control.SeriesSource] + rowsDigest := ab10PGRowsDigest(t, fixture, source) + ledgerDigest := ab10PGLedgerDigest(t, fixture, source) + + server, closeRuntime, err := composePostgreSQLFullRuntime(ctx, fixture.bootConfig(t, "127.0.0.1:0"), ab10Logger()) + if err != nil { + t.Fatalf("postgresql arm: reopen migrated database through production compose: %s", fixture.scrub(err)) + } + app := server.App() + body := ab10Graph(t, &ab10Arm{name: arm.name, app: app, scrub: arm.scrub}, ab10PatientQueryBody) + ab10CheckRowsLiteral(t, &ab10Arm{name: arm.name, app: app, scrub: arm.scrub}, "reopened query", body, []ab10WantRow{ + {Key: "ab10-patient-ada", Values: map[string]*string{"gender": ab10Str("female"), "birthDate": ab10Str("1992-04-11")}}, + {Key: "ab10-patient-nogender", Values: map[string]*string{"birthDate": ab10Str("1988-01-02")}}, + {Key: "ab10-patient-nullfield", Values: map[string]*string{"birthDate": ab10Str("2000-12-31")}}, + }) + if got := ab10PGRowsDigest(t, fixture, source); got != rowsDigest { + t.Fatalf("postgresql arm: row checksum changed across reopen: %s vs %s", got, rowsDigest) + } + if got := ab10PGLedgerDigest(t, fixture, source); got != ledgerDigest { + t.Fatalf("postgresql arm: migration history changed across reopen: %s vs %s", got, ledgerDigest) + } + if err := closeRuntime(); err != nil { + t.Fatalf("close first reopen: %s", fixture.scrub(err)) + } + second, closeSecond, err := composePostgreSQLFullRuntime(ctx, fixture.bootConfig(t, "127.0.0.1:0"), ab10Logger()) + if err != nil { + t.Fatalf("postgresql arm: second reopen: %s", fixture.scrub(err)) + } + defer func() { + if err := closeSecond(); err != nil { + t.Errorf("close second reopen: %s", fixture.scrub(err)) + } + }() + bodyAgain := ab10Graph(t, &ab10Arm{name: arm.name, app: second.App(), scrub: arm.scrub}, ab10PatientQueryBody) + if ab10DecodeRows(t, "second reopen", bodyAgain).RowCount != 3 { + t.Fatalf("postgresql arm: second reopen served %d rows, want literal 3", ab10DecodeRows(t, "second reopen", bodyAgain).RowCount) + } +} + +func ab10PGRowsDigest(t *testing.T, fixture *ab10PGFixture, source string) string { + t.Helper() + var digest *string + err := fixture.iso.QueryRow(context.Background(), + `SELECT md5(string_agg(project || '|' || dataset_generation || '|' || resource_type || '|' || logical_id || '|' || payload::text, E'\n' ORDER BY resource_seq)) + FROM "`+source+`".fhir_resource`).Scan(&digest) + if err != nil { + t.Fatalf("postgresql arm: rows digest: %s", fixture.scrub(err)) + } + if digest == nil { + return "" + } + return *digest +} + +func ab10PGLedgerDigest(t *testing.T, fixture *ab10PGFixture, source string) string { + t.Helper() + var digest *string + err := fixture.iso.QueryRow(context.Background(), + `SELECT md5(string_agg(version::text || ':' || checksum, E'\n' ORDER BY version)) + FROM "`+source+`".schema_migrations`).Scan(&digest) + if err != nil { + t.Fatalf("postgresql arm: ledger digest: %s", fixture.scrub(err)) + } + if digest == nil { + t.Fatalf("postgresql arm: migration ledger is empty after migrate") + } + return *digest +} diff --git a/internal/server/backend_composition_contract_test.go b/internal/server/backend_composition_contract_test.go new file mode 100644 index 000000000..0d7116392 --- /dev/null +++ b/internal/server/backend_composition_contract_test.go @@ -0,0 +1,272 @@ +package server + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + httpapi "github.com/calypr/loom/internal/api/http" + authscope "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" + backendspostgres "github.com/calypr/loom/internal/backends/postgres" + "github.com/calypr/loom/internal/backends/postgres/control" + clickhousestore "github.com/calypr/loom/internal/store/clickhouse" +) + +var errCompositionSpyFault = errors.New("composition spy: open refused") + +type compositionLedger struct { + arango []arangofactory.Config + pg []control.Config + ch []clickhousestore.Options +} + +func installCompositionSpies(t *testing.T, passthrough bool) *compositionLedger { + t.Helper() + ledger := &compositionLedger{} + origArango, origPG, origCH := openArangoBackend, openPostgresBackend, newClickHouseClient + t.Cleanup(func() { + openArangoBackend, openPostgresBackend, newClickHouseClient = origArango, origPG, origCH + }) + openArangoBackend = func(ctx context.Context, cfg arangofactory.Config) (*arangofactory.Runtime, error) { + ledger.arango = append(ledger.arango, cfg) + if passthrough { + return origArango(ctx, cfg) + } + return nil, errCompositionSpyFault + } + openPostgresBackend = func(ctx context.Context, cfg control.Config) (*backendspostgres.Runtime, error) { + ledger.pg = append(ledger.pg, cfg) + if passthrough { + return origPG(ctx, cfg) + } + return nil, errCompositionSpyFault + } + newClickHouseClient = func(opts clickhousestore.Options) (*clickhousestore.Client, error) { + ledger.ch = append(ledger.ch, opts) + if passthrough { + return origCH(opts) + } + return nil, errCompositionSpyFault + } + return ledger +} + +func compositionTestLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})) +} + +const ( + spyPGPassword = "spy-pg-password-9f3c" + spyPGUser = "spy_pg_user" +) + +func spyLadderConfig() Config { + cfg := Config{} + cfg.Server.Backend = sourceBackendPostgres + cfg.Server.Listen = "127.0.0.1:0" + cfg.Server.PostgreSQL = PostgreSQLConfig{ + Host: "127.0.0.1", Port: 5432, Database: "spydb", + User: spyPGUser, Password: spyPGPassword, SSLMode: "disable", + MaxOpenConns: 4, MaxIdleConns: 1, + Schemas: map[string]string{ + "source": "spy_src", + "explorer_control": "spy_exp", + "publication_control": "spy_pub", + "dataset_release": "spy_rel", + "recipe_chains": "spy_rcp", + "recipe_exec_registries": "spy_exe", + }, + } + return cfg +} + +func spyFirstQueryConfig() Config { + cfg := spyLadderConfig() + cfg.Server.PostgreSQL.Schemas = map[string]string{ + "source": "spy_src", + "dataset_release": "spy_rel", + } + return cfg +} + +func assertLedger(t *testing.T, what string, ledger *compositionLedger, arango, pg, ch int) { + t.Helper() + if len(ledger.arango) != arango || len(ledger.pg) != pg || len(ledger.ch) != ch { + t.Fatalf("%s: open ledger arango=%d postgres=%d clickhouse=%d, want %d/%d/%d", + what, len(ledger.arango), len(ledger.pg), len(ledger.ch), arango, pg, ch) + } +} + +func TestBackendCompositionContract(t *testing.T) { + t.Run("arangoDefault_opensOnlyArangoOnce", func(t *testing.T) { + ledger := installCompositionSpies(t, false) + cfg := Config{} + cfg.Server.URL = "http://127.0.0.1:8529" + cfg.Server.Database = "spy_arango_db" + _, _, err := composeArangoRuntime(context.Background(), cfg, compositionTestLogger()) + if !errors.Is(err, errCompositionSpyFault) { + t.Fatalf("compose must stop at the arango open with the spy fault, got %v", err) + } + assertLedger(t, "arango default", ledger, 1, 0, 0) + if ledger.arango[0].URL != cfg.Server.URL || ledger.arango[0].Database != cfg.Server.Database { + t.Fatalf("arango open carried foreign config: %+v", ledger.arango[0]) + } + }) + + t.Run("postgresLadder_opensOnlyPostgres", func(t *testing.T) { + ledger := installCompositionSpies(t, false) + cfg := spyLadderConfig() + _, _, err := composePostgreSQLFullRuntime(context.Background(), cfg, compositionTestLogger()) + if !errors.Is(err, errCompositionSpyFault) { + t.Fatalf("compose must stop at the postgres open with the spy fault, got %v", err) + } + assertLedger(t, "postgres ladder", ledger, 0, 1, 0) + opened := ledger.pg[0] + if len(opened.Schemas) != 6 { + t.Fatalf("full runtime must open the complete six-series ladder, got %d series", len(opened.Schemas)) + } + for _, series := range []control.Series{ + control.SeriesSource, control.SeriesExplorerControl, + control.SeriesPublicationControl, control.SeriesDatasetRelease, + control.SeriesRecipeChains, control.SeriesRecipeExecRegistries, + } { + if opened.Schemas[series] == "" { + t.Fatalf("series %s unassigned in the ladder open", series) + } + } + if opened.Setup != control.SetupNone { + t.Fatalf("production compose must never ask the factory to mutate schema state, got Setup=%v", opened.Setup) + } + if strings.Contains(err.Error(), spyPGPassword) { + t.Fatal("open refusal echoed the PostgreSQL password") + } + }) + + t.Run("twoSeries_firstQueryArm_onlyTwoSeries", func(t *testing.T) { + ledger := installCompositionSpies(t, false) + _, _, err := composePostgreSQLFirstQueryRoute(context.Background(), spyFirstQueryConfig(), compositionTestLogger()) + if !errors.Is(err, errCompositionSpyFault) { + t.Fatalf("first-query compose must stop at the postgres open with the spy fault, got %v", err) + } + assertLedger(t, "first-query pair", ledger, 0, 1, 0) + opened := ledger.pg[0] + if len(opened.Schemas) != 2 { + t.Fatalf("first-query route must open exactly two series, got %d: %v", len(opened.Schemas), opened.Schemas) + } + if opened.Schemas[control.SeriesSource] != "spy_src" || + opened.Schemas[control.SeriesDatasetRelease] != "spy_rel" { + t.Fatalf("first-query route opened foreign placements: %v", opened.Schemas) + } + if opened.Setup != control.SetupNone { + t.Fatalf("first-query compose must use SetupNone, got %v", opened.Setup) + } + }) + + t.Run("invalidShape_sentinelPreDial_zeroOpens", func(t *testing.T) { + shapes := map[string]func(cfg *PostgreSQLConfig){ + "no-schemas": func(pg *PostgreSQLConfig) { pg.Schemas = nil }, + "one-schema": func(pg *PostgreSQLConfig) { pg.Schemas = map[string]string{"source": "spy_src"} }, + "three-schemas": func(pg *PostgreSQLConfig) { + pg.Schemas = map[string]string{"source": "spy_src", "dataset_release": "spy_rel", "recipe_chains": "spy_rcp"} + }, + "duplicated-names": func(pg *PostgreSQLConfig) { pg.Schemas["recipe_chains"] = "spy_src" }, + "blank-series": func(pg *PostgreSQLConfig) { pg.Schemas["recipe_exec_registries"] = " " }, + "foreign-series": func(pg *PostgreSQLConfig) { pg.Schemas["made_up_series"] = "spy_extra" }, + } + for name, mutate := range shapes { + ledger := installCompositionSpies(t, false) + cfg := spyLadderConfig() + mutate(&cfg.Server.PostgreSQL) + err := RunWithConfig(context.Background(), cfg) + if !errors.Is(err, ErrPostgreSQLRuntimeNotWired) { + t.Fatalf("%s: want the byte-stable startup refusal through dispatch, got %v", name, err) + } + if err.Error() != ErrPostgreSQLRuntimeNotWired.Error() { + t.Fatalf("%s: refusal text drifted from the pinned sentinel: %q", name, err.Error()) + } + assertLedger(t, name, ledger, 0, 0, 0) + } + }) + + t.Run("openFault_neverFallsBackToArango", func(t *testing.T) { + ledger := installCompositionSpies(t, false) + err := RunWithConfig(context.Background(), spyLadderConfig()) + if !errors.Is(err, errCompositionSpyFault) { + t.Fatalf("the postgres open fault must propagate, got %v", err) + } + assertLedger(t, "open fault", ledger, 0, 1, 0) + }) + + t.Run("clickHouseDisabled_noClickHouseConstruction", func(t *testing.T) { + ledger := installCompositionSpies(t, false) + cfg := spyLadderConfig() + cfg.Server.ClickHouse.Enabled = false + if _, _, err := composePostgreSQLFullRuntime(context.Background(), cfg, compositionTestLogger()); !errors.Is(err, errCompositionSpyFault) { + t.Fatalf("full compose must stop at the postgres open, got %v", err) + } + cfg2 := spyFirstQueryConfig() + cfg2.Server.ClickHouse.Enabled = false + if _, _, err := composePostgreSQLFirstQueryRoute(context.Background(), cfg2, compositionTestLogger()); !errors.Is(err, errCompositionSpyFault) { + t.Fatalf("first-query compose must stop at the postgres open, got %v", err) + } + assertLedger(t, "clickhouse disabled", ledger, 0, 2, 0) + }) + + t.Run("liveness_doesNoDatabaseWork", func(t *testing.T) { + ledger := installCompositionSpies(t, false) + observed := 0 + server, err := httpapi.NewHTTPServer(httpapi.HTTPConfig{ + Authorizer: authscope.AllowAllAuthorizer{}, + CoreReadyCheck: func(context.Context) error { observed++; return nil }, + }) + if err != nil { + t.Fatalf("build probe server: %v", err) + } + body, status := server.Liveness(context.Background()) + if status != http.StatusOK || body["status"] != "live" { + t.Fatalf("liveness status=%d body=%v", status, body) + } + if observed != 0 { + t.Fatalf("liveness consulted %d dependency checks; must be DB-free", observed) + } + assertLedger(t, "liveness", ledger, 0, 0, 0) + }) + + t.Run("liveProductionCompose_ledgerHolds", func(t *testing.T) { + fixture := newPgCompFixture(t) + ledger := installCompositionSpies(t, true) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + server, closeRuntime, err := composePostgreSQLFullRuntime(ctx, pgCompBootConfig(t, fixture, "127.0.0.1:0"), compositionTestLogger()) + if err != nil { + t.Fatalf("production compose: %s", fixture.scrub(err)) + } + assertLedger(t, "live full compose", ledger, 0, 1, 0) + if ledger.pg[0].Setup != control.SetupNone || len(ledger.pg[0].Schemas) != 6 { + t.Fatalf("live compose opened the wrong shape: Setup=%v series=%d", ledger.pg[0].Setup, len(ledger.pg[0].Schemas)) + } + response, err := server.App().Test(httptest.NewRequest(http.MethodGet, "/livez", nil)) + if err != nil { + t.Fatalf("livez request: %s", fixture.scrub(err)) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + t.Fatalf("livez status=%d", response.StatusCode) + } + assertLedger(t, "live liveness", ledger, 0, 1, 0) + if err := closeRuntime(); err != nil { + t.Errorf("production closer: %s", fixture.scrub(err)) + } + assertLedger(t, "live close", ledger, 0, 1, 0) + if err := closeRuntime(); err != nil { + t.Errorf("second production closer call: %s", fixture.scrub(err)) + } + assertLedger(t, "second live close", ledger, 0, 1, 0) + }) +} diff --git a/internal/server/backend_error_boundary_contract_test.go b/internal/server/backend_error_boundary_contract_test.go new file mode 100644 index 000000000..2b8803d99 --- /dev/null +++ b/internal/server/backend_error_boundary_contract_test.go @@ -0,0 +1,290 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/calypr/loom/internal/backends/postgres/control" + "net/http" + "strings" + "testing" + + "github.com/arangodb/go-driver/v2/arangodb/shared" + httpapi "github.com/calypr/loom/internal/api/http" + arangofactory "github.com/calypr/loom/internal/backends/arango" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +func driveAQLAdapter(t *testing.T, adapterFailure error) error { + t.Helper() + adapter := configuredAQLSourceRows(nil, "contract", func(context.Context, dataframeexecution.RowsRequest, dataframeexecution.RowVisitor) error { + return adapterFailure + }) + err := adapter(context.Background(), dataframeexecution.RowsRequest{Command: testAQLCommand(t, "RETURN 1", nil), BatchSize: 10}, func(map[string]any) error { return nil }) + if err == nil { + t.Fatal("adapter failure was swallowed by the composition") + } + return err +} + +func arangoAdapterFailure(driverErr error) error { + classified, ok := arangofactory.TranslateQueryError(fmt.Errorf("arango query: %w", driverErr)) + if !ok { + panic("fixture error was not classified by the adapter") + } + return classified +} + +func bodyJSON(t *testing.T, mapped httpapi.MappedError) string { + t.Helper() + encoded, err := json.Marshal(mapped.Body) + if err != nil { + t.Fatal(err) + } + return string(encoded) +} + +func TestBackendErrorBoundaryContract(t *testing.T) { + t.Run("arangodb_resource_limits_keep_existing_public_semantics", func(t *testing.T) { + cases := []struct { + name string + driverErr shared.ArangoError + wantCode string + wantStatus int + resource string + }{ + { + name: "memory_limit", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, ErrorMessage: "AQL: query would use more memory than allowed"}, + wantCode: "QUERY_MEMORY_LIMIT_EXCEEDED", + wantStatus: http.StatusServiceUnavailable, + resource: "query_memory", + }, + { + name: "resource_limit", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, ErrorMessage: "resource limit exceeded"}, + wantCode: "QUERY_RESOURCE_LIMIT_EXCEEDED", + wantStatus: http.StatusServiceUnavailable, + resource: "query", + }, + { + name: "backend_out_of_memory", + driverErr: shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrOutOfMemory, ErrorMessage: "out of memory"}, + wantCode: "QUERY_BACKEND_OUT_OF_MEMORY", + wantStatus: http.StatusServiceUnavailable, + resource: "memory", + }, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + driven := driveAQLAdapter(t, arangoAdapterFailure(testCase.driverErr)) + userErr, ok := dataframeerrors.AsUserError(driven) + if !ok || userErr.Code() != testCase.wantCode { + t.Fatalf("composition lost the classification: %#v", driven) + } + mapped := httpapi.MapDataframeError(driven, "req-1") + if mapped.Status != testCase.wantStatus { + t.Fatalf("status = %d, want %d", mapped.Status, testCase.wantStatus) + } + if mapped.Body.Error.Code != testCase.wantCode { + t.Fatalf("code = %q, want %q", mapped.Body.Error.Code, testCase.wantCode) + } + if mapped.Body.Error.Retryable { + t.Fatal("resource-limit failures must not advertise a blanket retry") + } + if got := mapped.Body.Error.Details["backend"]; got != "arangodb" { + t.Fatalf("backend detail = %v", got) + } + if got := mapped.Body.Error.Details["resource"]; got != testCase.resource { + t.Fatalf("resource detail = %v, want %q", got, testCase.resource) + } + if !errors.Is(driven, testCase.driverErr) { + t.Fatal("vendor cause identity lost through composition") + } + }) + } + }) + + t.Run("arangodb_assertions_keep_existing_public_semantics", func(t *testing.T) { + for _, code := range []dataframeerrors.ErrorCode{ + dataframeerrors.CodeTablePivotCellCardinality, + dataframeerrors.CodeTablePivotUnlistedCategory, + dataframeerrors.CodeConstructionExpansionEmpty, + dataframeerrors.CodeExplicitGroupUnassignedMember, + dataframeerrors.CodeRelationshipCardinalityViolation, + } { + t.Run(string(code), func(t *testing.T) { + driverErr := shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: " + string(code) + " (while executing)"} + driven := driveAQLAdapter(t, arangoAdapterFailure(driverErr)) + mapped := httpapi.MapDataframeError(driven, "req-1") + if mapped.Body.Error.Code != string(code) { + t.Fatalf("code = %q, want %q", mapped.Body.Error.Code, code) + } + if mapped.Status != http.StatusUnprocessableEntity { + t.Fatalf("status = %d, want 422", mapped.Status) + } + if mapped.Body.Error.Retryable { + t.Fatal("assertion failures must not be retryable") + } + if body := bodyJSON(t, mapped); strings.Contains(body, "AQL") { + t.Fatalf("public body leaked AQL text: %s", body) + } + }) + } + }) + + t.Run("postgres_failures_produce_equivalent_public_semantics", func(t *testing.T) { + pgErr := &pgconn.PgError{ + Code: "57014", SeverityUnlocalized: "ERROR", + Message: `canceling statement due to statement_timeout; while binding {"secret":"hunter2","bind":"SELECT SECRET_COLUMN"}`, + } + translated, ok := control.TranslateError(fmt.Errorf("postgres rows: query: %w", pgErr)) + if !ok { + t.Fatal("boundary did not classify the server cancel") + } + userErr, isUser := dataframeerrors.AsUserError(translated) + if !isUser || userErr.Code() != string(dataframeerrors.CodeQueryResourceLimitExceeded) { + t.Fatalf("PG cancel did not reach the shared resource-limit semantics: %#v", translated) + } + if userErr.Retryable() { + t.Fatal("server cancel must not advertise a blanket retry") + } + mapped := httpapi.MapDataframeError(translated, "req-2") + if mapped.Status != http.StatusServiceUnavailable || mapped.Body.Error.Code != "QUERY_RESOURCE_LIMIT_EXCEEDED" { + t.Fatalf("mapped = %d %q, want 503 QUERY_RESOURCE_LIMIT_EXCEEDED", mapped.Status, mapped.Body.Error.Code) + } + if got := mapped.Body.Error.Details["backend"]; got != "postgres" { + t.Fatalf("backend detail = %v, want postgres", got) + } + body := bodyJSON(t, mapped) + for _, forbidden := range []string{"hunter2", "SECRET_COLUMN", "SELECT", "statement_timeout", "canceling"} { + if strings.Contains(strings.ToLower(body), strings.ToLower(forbidden)) { + t.Fatalf("public payload leaked %q: %s", forbidden, body) + } + } + }) + + t.Run("cancellation_and_deadline_identity_survive_wrapping", func(t *testing.T) { + canceled := fmt.Errorf("arango stream: %w", context.Canceled) + if err := preserveConfiguredQueryError(canceled); !errors.Is(err, context.Canceled) { + t.Fatalf("preview route lost cancellation identity: %v", err) + } + mapped := httpapi.MapDataframeError(canceled, "req-3") + if mapped.Body.Error.Code != "CLIENT_CANCELED" || mapped.Status != 499 { + t.Fatalf("canceled mapped = %d %q", mapped.Status, mapped.Body.Error.Code) + } + deadline := fmt.Errorf("arango stream: %w", context.DeadlineExceeded) + if err := preserveConfiguredQueryError(deadline); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("composition rewrote the deadline failure: %v", err) + } + mapped = httpapi.MapDataframeError(deadline, "req-4") + if mapped.Body.Error.Code != "BACKEND_UNAVAILABLE" || !mapped.Body.Error.Retryable { + t.Fatalf("deadline mapped = %d %q retryable=%t", mapped.Status, mapped.Body.Error.Code, mapped.Body.Error.Retryable) + } + poolWait := fmt.Errorf("acquire: %w: %w", control.ErrConnect, context.Canceled) + translated, ok := control.TranslateError(poolWait) + if !ok { + t.Fatalf("pool cancellation not classified by the boundary: %v", poolWait) + } + if !errors.Is(translated, context.Canceled) { + t.Fatalf("PG translation lost the cancellation sentinel: %v", translated) + } + if userErr, isUser := dataframeerrors.AsUserError(translated); isUser && userErr.Retryable() { + t.Fatal("caller cancellation must never be marked retryable") + } + }) + + t.Run("unknown_unique_violation_is_not_a_cas_conflict", func(t *testing.T) { + pgErr := &pgconn.PgError{ + Code: "23505", SeverityUnlocalized: "ERROR", + Message: `duplicate key value violates unique constraint "uq_explorer_draft_revision_internal"`, + ConstraintName: "uq_explorer_draft_revision_internal", + Detail: `Key (draft_id)=(42) already exists.`, + } + translated, ok := control.TranslateError(fmt.Errorf("postgres rows: query: %w", pgErr)) + if !ok { + t.Fatal("boundary did not classify the unique violation") + } + if errors.Is(translated, control.ErrUniqueConflict) { + t.Fatal("a bare unique violation was labeled as a proven domain conflict") + } + if control.RetryableTransactionConflict(translated) { + t.Fatal("unique violation must not invite transaction retries") + } + mapped := httpapi.MapDataframeError(translated, "req-5") + if mapped.Body.Error.Code != "INTERNAL_ERROR" || mapped.Status != http.StatusInternalServerError { + t.Fatalf("mapped = %d %q, want 500 INTERNAL_ERROR", mapped.Status, mapped.Body.Error.Code) + } + if mapped.Body.Error.Retryable { + t.Fatal("unproven unique violation must not be retryable") + } + body := bodyJSON(t, mapped) + for _, forbidden := range []string{"uq_explorer_draft_revision_internal", "draft_id", "duplicate key", "violates unique constraint", "already exists"} { + if strings.Contains(strings.ToLower(body), strings.ToLower(forbidden)) { + t.Fatalf("public payload leaked constraint material %q: %s", forbidden, body) + } + } + }) + + t.Run("ambiguous_commit_stays_distinguishable_from_proven_rollback", func(t *testing.T) { + proven, ok := control.TranslateCommitError(fmt.Errorf("commit: %w", pgx.ErrTxCommitRollback)) + if !ok { + t.Fatal("commit rollback not classified") + } + if errors.Is(proven, control.ErrCommitOutcomeUnknown) { + t.Fatal("a proven rollback must not be labeled ambiguous") + } + ambiguousPG := &pgconn.PgError{Code: "57P01", SeverityUnlocalized: "FATAL", Message: "terminating connection due to administrator command"} + ambiguous, ok := control.TranslateCommitError(fmt.Errorf("commit: %w", ambiguousPG)) + if !ok { + t.Fatal("mid-commit shutdown not classified") + } + if !errors.Is(ambiguous, control.ErrCommitOutcomeUnknown) { + t.Fatal("session death mid-commit must be labeled outcome-unknown") + } + var translatedErr *control.TranslatedError + if !errors.As(ambiguous, &translatedErr) || !translatedErr.AmbiguousCommit { + t.Fatalf("ambiguous flag missing: %#v", ambiguous) + } + mappedAmbiguous := httpapi.MapDataframeError(ambiguous, "req-6") + mappedProven := httpapi.MapDataframeError(proven, "req-7") + if mappedAmbiguous.Body.Error.Retryable || mappedProven.Body.Error.Retryable { + t.Fatal("commit outcomes must never advertise a blanket retry") + } + for _, mapped := range []httpapi.MappedError{mappedAmbiguous, mappedProven} { + body := bodyJSON(t, mapped) + for _, forbidden := range []string{"terminating connection due to administrator command", "commit unexpectedly resulted in rollback"} { + if strings.Contains(strings.ToLower(body), strings.ToLower(forbidden)) { + t.Fatalf("public payload leaked %q: %s", forbidden, body) + } + } + } + }) + + t.Run("retryable_transaction_states_do_not_cause_blanket_write_retries", func(t *testing.T) { + serialization := &pgconn.PgError{Code: "40001", SeverityUnlocalized: "ERROR", Message: "could not serialize access due to concurrent update"} + translated, ok := control.TranslateError(fmt.Errorf("postgres rows: query: %w", serialization)) + if !ok { + t.Fatal("serialization failure not classified") + } + if !control.RetryableTransactionConflict(translated) { + t.Fatal("operation policy must still recognize the serialization conflict") + } + userErr, isUser := dataframeerrors.AsUserError(translated) + if !isUser || userErr.Retryable() { + t.Fatalf("boundary marked the serialization failure blanket-retryable: %#v", translated) + } + mapped := httpapi.MapDataframeError(translated, "req-8") + if mapped.Body.Error.Retryable { + t.Fatal("transport must not advertise retry for a serialization conflict") + } + body := bodyJSON(t, mapped) + if strings.Contains(strings.ToLower(body), "could not serialize access") { + t.Fatalf("public payload leaked the server message: %s", body) + } + }) +} diff --git a/internal/server/backend_neutral_diagnostics_live_test.go b/internal/server/backend_neutral_diagnostics_live_test.go new file mode 100644 index 000000000..e2e4f81d9 --- /dev/null +++ b/internal/server/backend_neutral_diagnostics_live_test.go @@ -0,0 +1,227 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "regexp" + "strings" + "testing" + "time" + + "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + "github.com/calypr/loom/internal/dataframe/recipe" + exec "github.com/calypr/loom/internal/dataframe/recipe/exec" + "github.com/google/uuid" +) + +type arch12UnusedRegistry struct{} + +func (arch12UnusedRegistry) LoadRecipe(context.Context, string) (exec.Entry, error) { + return exec.Entry{}, fmt.Errorf("live diagnostics witness compiles bundles directly; the registry is never addressed") +} + +func (arch12UnusedRegistry) LoadRecipeVersion(context.Context, string, string) (exec.Entry, error) { + return exec.Entry{}, fmt.Errorf("live diagnostics witness compiles bundles directly; the registry is never addressed") +} + +func TestBackendNeutralDiagnosticsAgainstArango(t *testing.T) { + url, database := os.Getenv("LOOM_TEST_ARANGO_URL"), os.Getenv("LOOM_TEST_ARANGO_DATABASE") + if url == "" || database == "" { + t.Skip("LOOM_TEST_ARANGO_URL and LOOM_TEST_ARANGO_DATABASE must point to the supplied Docker Arango service") + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + t.Cleanup(cancel) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: url, Database: database}) + if err != nil { + t.Fatal(err) + } + client := arangoBackend.Client + t.Cleanup(func() { _ = client.Close(context.Background()) }) + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Patient"}}}); err != nil { + t.Fatal(err) + } + project, generation := "loom_arch12_diag_"+uuid.NewString(), "arch12-generation" + owned := []string{} + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cleanupCancel() + for _, key := range owned { + if err := client.ExecuteAQL(cleanupCtx, "REMOVE @key IN Patient", map[string]any{"key": key}); err != nil { + t.Errorf("remove owned arch12 fixture %s: %v", key, err) + } + } + }) + for _, fixture := range []struct{ key, id, gender string }{ + {key: "ada", id: "patient-ada", gender: "female"}, + {key: "alan", id: "patient-alan", gender: "male"}, + } { + key := project + "_" + fixture.key + owned = append(owned, key) + if err := client.ExecuteAQL(ctx, "INSERT {\"_key\": @key, id: @id, project: @project, project_id: @project, dataset_generation: @generation, resourceType: \"Patient\", gender: @gender, payload: {id: @id, gender: @gender}} IN Patient", map[string]any{ + "key": key, "id": fixture.id, "project": project, "generation": generation, "gender": fixture.gender, + }); err != nil { + t.Fatalf("insert arch12 patient fixture: %v", err) + } + } + rowsPort := func(queryCtx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + query, bindVars, dispatchErr := request.Command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return client.QueryRows(queryCtx, query, request.BatchSize, bindVars, arangofactory.RowVisitor(visit)) + } + engine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: compiler.ArangoTarget(), + Registry: arch12UnusedRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "arch12-scope" }, + SourceRows: rowsPort, + }) + if err != nil { + t.Fatal(err) + } + bundle := recipe.Bundle{RecipeSchemaVersion: recipe.CurrentSchemaVersion, Name: "arch12 diagnostics", TranslationVersion: "test", Outputs: []recipe.Output{{ + Name: "Rows", RootResourceType: "Patient", RowGrain: "resource", + Fields: []recipe.Field{ + {Name: "ids", ColumnID: "ids", Expr: recipe.Expression{Select: "root.id"}}, + {Name: "gender", ColumnID: "gender", Expr: recipe.Expression{Select: "root.gender"}}, + }, + }}} + bindings := recipe.RuntimeBindings{ + Project: project, DatasetGeneration: generation, + AuthScopeMode: authscope.ReadScopeUnrestricted, PreviewLimit: 25, OutputNames: []string{"Rows"}, + } + resolved, err := engine.CompileResolvedBundle(ctx, bundle, bindings) + if err != nil { + t.Fatalf("compile arch12 bundle: %v", err) + } + compiled, err := compiler.CompileRecipeOutput(compiler.ArangoTarget(), resolved.Compiled.Outputs[0], resolved.Semantic.SemanticPlan.Bindings, 25, ir.DefaultPhysicalOptimizationPolicy()) + if err != nil { + t.Fatalf("compile arch12 command: %v", err) + } + if compiled.Command.Backend() != command.CompiledBackendAQL { + t.Fatalf("fixture command tag = %q, want aql", compiled.Command.Backend()) + } + service := dataframeexecution.NewService(dataframeexecution.ServiceConfig{SourceRows: rowsPort}) + result, err := service.RunCompiled(ctx, compiled) + if err != nil { + t.Fatalf("RunCompiled against Arango: %v", err) + } + if result.RowCount != 2 { + t.Fatalf("live row count = %d, want 2", result.RowCount) + } + if result.Diagnostics.SourceQuery <= 0 { + t.Fatalf("real Arango cursor must produce a positive source interval, got %v", result.Diagnostics.SourceQuery) + } + if result.Diagnostics.SourceBackend != command.CompiledBackendAQL { + t.Fatalf("source backend tag = %q, want aql", result.Diagnostics.SourceBackend) + } + if result.Diagnostics.RowMaterialization < 0 { + t.Fatalf("row materialization must be non-negative, got %v", result.Diagnostics.RowMaterialization) + } + summary, err := engine.PreviewOutput(ctx, resolved, dataframeexecution.PreviewRequest{Output: "Rows", Limit: 25}, func(map[string]any) error { return nil }) + if err != nil { + t.Fatalf("PreviewOutput against Arango: %v", err) + } + if summary.SourceQueryDuration <= 0 { + t.Fatalf("real Arango preview cursor must produce a positive source interval, got %v", summary.SourceQueryDuration) + } + hex64 := regexp.MustCompile(`^[0-9a-f]{64}$`) + if !hex64.MatchString(summary.PlanFingerprint) { + t.Fatalf("preview plan fingerprint is not a 64-hex shape digest: %q", summary.PlanFingerprint) + } + explanation, err := (dataframeexecution.Control{Engine: engine}).ExplainResolved(ctx, resolved, false) + if err != nil { + t.Fatalf("ExplainResolved against Arango: %v", err) + } + if len(explanation.Outputs) != 1 || explanation.Outputs[0].PlanFingerprint != summary.PlanFingerprint { + t.Fatalf("Explain and preview diverged against the live service: explain=%+v preview=%q", explanation.Outputs, summary.PlanFingerprint) + } + encoded, err := json.Marshal(struct { + Summary dataframeexecution.PreviewSummary `json:"summary"` + Diagnostics dataframeexecution.QueryDiagnostics `json:"diagnostics"` + }{Summary: summary, Diagnostics: result.Diagnostics}) + if err != nil { + t.Fatal(err) + } + for _, tell := range []string{"FOR d IN ", "LIMIT @limit", "@auth_resource_paths", "patient-ada", project} { + if strings.Contains(string(encoded), tell) { + t.Fatalf("public diagnostics carry the tell %q: %s", tell, encoded) + } + } +} + +func TestBackendNeutralDiagnosticsLivePostgreSQL(t *testing.T) { + fixture := newQRouteFixture(t) + fixture.load(t, qRouteProject, qRouteGenA, map[string][]string{ + "Patient.ndjson": qRoutePatientLines(t, "patient-ada", "patient-alan"), + }) + fixture.activate(t, qRouteProject, qRouteGenA) + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + server, closeRuntime, err := composePostgreSQLFirstQueryRoute(ctx, qRouteBootConfig(t, fixture, "127.0.0.1:0"), logger) + if err != nil { + t.Fatalf("compose production route: %s", fixture.scrub(err)) + } + defer func() { + if err := closeRuntime(); err != nil { + t.Errorf("close production route runtime: %s", fixture.scrub(err)) + } + }() + body := `{"query":"mutation($input: FhirDataframeInput!, $limit: Int) { runFhirDataframe(input: $input, limit: $limit) { rowCount diagnostics { sourceQueryMs arangoQueryMs compilationMs totalMs } } }","variables":{"input":{"project":"` + qRouteProject + `","rootResourceType":"Patient","rootFields":[{"name":"gender","selector":{"valuePath":"gender"}}]},"limit":10}}` + response := qRouteGraphQL(t, server.App(), body) + if response.StatusCode != http.StatusOK { + t.Fatalf("runFhirDataframe HTTP %d: %s", response.StatusCode, fixture.scrub(response.Body)) + } + var envelope struct { + Errors []struct { + Message string `json:"message"` + } `json:"errors"` + Data struct { + RunFhirDataframe struct { + RowCount int `json:"rowCount"` + Diagnostics *struct { + SourceQueryMs float64 `json:"sourceQueryMs"` + ArangoQueryMs float64 `json:"arangoQueryMs"` + CompilationMs float64 `json:"compilationMs"` + TotalMs float64 `json:"totalMs"` + } `json:"diagnostics"` + } `json:"runFhirDataframe"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(response.Body), &envelope); err != nil { + t.Fatalf("decode handler envelope: %v (body %s)", err, fixture.scrub(response.Body)) + } + if len(envelope.Errors) != 0 { + t.Fatalf("handler returned GraphQL errors: %s", fixture.scrub(response.Body)) + } + run := envelope.Data.RunFhirDataframe + if run.RowCount != 2 { + t.Fatalf("rowCount = %d, want 2: %s", run.RowCount, fixture.scrub(response.Body)) + } + if run.Diagnostics == nil { + t.Fatal("diagnostics must be projected on the PostgreSQL lane") + } + if run.Diagnostics.SourceQueryMs <= 0 { + t.Fatalf("real PostgreSQL cursor must produce a positive sourceQueryMs, got %v", run.Diagnostics.SourceQueryMs) + } + if run.Diagnostics.ArangoQueryMs != 0 { + t.Fatalf("arangoQueryMs must be exactly zero on the PostgreSQL lane, got %v", run.Diagnostics.ArangoQueryMs) + } + for _, tell := range []string{"SELECT ", "FROM loom.", "$1", "$2", "patient-ada", qRouteProject} { + if strings.Contains(response.Body, tell) { + t.Fatalf("PostgreSQL diagnostics response carries the SQL/value tell %q: %s", tell, fixture.scrub(response.Body)) + } + } +} diff --git a/internal/server/case008_row_definition_preview_route_test.go b/internal/server/case008_row_definition_preview_route_test.go index 06ac4012c..9cb8f389b 100644 --- a/internal/server/case008_row_definition_preview_route_test.go +++ b/internal/server/case008_row_definition_preview_route_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" @@ -93,8 +94,14 @@ func TestCase008ExplicitGroupRowDefinitionPreviewCompilesCandidateReceipt(t *tes var basePreviewColumns []string var candidatePreviewColumns []string engine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, ScopeDigest: recipeScopeDigest, - QueryRows: func(_ context.Context, _ string, _ int, bindVars map[string]any, visit func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, ScopeDigest: recipeScopeDigest, + SourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, bindVars, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + queryCalls++ if _, grouped := bindVars["group_rows_revision_id"]; grouped { return visit(map[string]any{ diff --git a/internal/server/command_helpers_test.go b/internal/server/command_helpers_test.go new file mode 100644 index 000000000..47ec8879a --- /dev/null +++ b/internal/server/command_helpers_test.go @@ -0,0 +1,37 @@ +package server + +import ( + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" +) + +func testAQLView(t *testing.T, command command.CompiledCommand) (string, map[string]any) { + t.Helper() + query, bindVars, err := command.AQLParameters() + if err != nil { + t.Fatalf("derive AQL parameters from %s command: %v", command.Backend(), err) + } + return query, bindVars +} + +func testAQLCommand(t *testing.T, query string, bindVars map[string]any) command.CompiledCommand { + t.Helper() + compiledCommand, err := command.NewAQLCommand(query, bindVars) + if err != nil { + t.Fatalf("build AQL command for test: %v", err) + } + return compiledCommand +} + +func testAQLText(t *testing.T, command command.CompiledCommand) string { + t.Helper() + query, _ := testAQLView(t, command) + return query +} + +func testAQLBinds(t *testing.T, command command.CompiledCommand) map[string]any { + t.Helper() + _, bindVars := testAQLView(t, command) + return bindVars +} diff --git a/internal/server/compose.go b/internal/server/compose.go new file mode 100644 index 000000000..2e0557627 --- /dev/null +++ b/internal/server/compose.go @@ -0,0 +1,747 @@ +package server + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "sync/atomic" + "time" + + loadapi "github.com/calypr/loom/internal/api/bulk/load" + queryapi "github.com/calypr/loom/internal/api/graphql/graph/query" + graphresolver "github.com/calypr/loom/internal/api/graphql/graph/resolver" + httpapi "github.com/calypr/loom/internal/api/http" + "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" + backendspostgres "github.com/calypr/loom/internal/backends/postgres" + "github.com/calypr/loom/internal/catalog" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/lower" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + "github.com/calypr/loom/internal/dataframe/execution/chartifact" + publication "github.com/calypr/loom/internal/dataframe/publication" + publicationclickhouse "github.com/calypr/loom/internal/dataframe/publication/clickhouse" + "github.com/calypr/loom/internal/dataframe/published" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/dataframe/recipe/exec" + publicationcontract "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/explorer" + "github.com/calypr/loom/internal/explorer/artifactfs" + "github.com/calypr/loom/internal/explorer/capability" + "github.com/calypr/loom/internal/explorer/lifecycle" + "github.com/calypr/loom/internal/explorer/tableshapecap" + fhirschema "github.com/calypr/loom/internal/fhir/schema" + "github.com/calypr/loom/internal/ingest" + clickhousestore "github.com/calypr/loom/internal/store/clickhouse" +) + +var ( + openArangoBackend = arangofactory.Open + openPostgresBackend = backendspostgres.Open + newClickHouseClient = clickhousestore.New +) + +func serveRuntime(ctx context.Context, server *httpapi.HTTPServer, listen string, closeRuntime func() error, closeLabel string, logger *slog.Logger, startupAttrs ...any) error { + defer func() { + if closeRuntime == nil { + return + } + if closeErr := closeRuntime(); closeErr != nil { + logger.Error(closeLabel, "error", closeErr) + } + }() + errCh := make(chan error, 1) + go func() { + logger.Info("starting HTTP server", startupAttrs...) + errCh <- server.App().Listen(listen) + }() + + select { + case err := <-errCh: + if err != nil { + return fmt.Errorf("server stopped: %w", err) + } + case <-ctx.Done(): + logger.Info("shutting down HTTP server", "reason", ctx.Err()) + shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) + defer cancel() + if err := server.App().ShutdownWithContext(shutdownCtx); err != nil { + return fmt.Errorf("shutdown failed: %w", err) + } + } + return nil +} + +type publicationPlaneInput struct { + InitialDegradation error + InitialPublicationReady bool + ClickHouse clickhousestore.Options + AcquireArtifactCatalog func(context.Context, *clickhousestore.Client) (chartifact.Catalog, error) + BundleCatalog publication.BundleCatalog + ActiveManifestResolver publicationcontract.ActiveResolver + ActiveReleaseResolver publicationcontract.ActiveReleaseResolver + Scopes *authscope.ScopeResolver +} + +type publicationPlane struct { + ClickHouse *clickhousestore.Client + PrivateArtifacts *chartifact.Manager + MaterializationReader *published.Reader + ClickHouseQueryRows dataframeexecution.ClickHouseQueryRows + ResolveClickHouseInputs dataframeexecution.ResolveClickHouseInputs + WithExecutionReadPins dataframeexecution.WithExecutionReadPins + BundleTarget publication.Target + Degradation error + PublicationReady bool +} + +type publicationSweeper func(ctx context.Context, clickhouse *clickhousestore.Client, bundleStore *publicationclickhouse.ClickHouseBundleStore, cutoff time.Time) (stage string, cause error) + +func newPublicationPlane(ctx context.Context, serverConfig Config, logger *slog.Logger, in publicationPlaneInput) (plane *publicationPlane, closeClickHouse func() error, err error) { + plane = &publicationPlane{Degradation: in.InitialDegradation, PublicationReady: in.InitialPublicationReady} + plane.ClickHouse, err = newClickHouseClient(in.ClickHouse) + if err != nil { + return plane, nil, fmt.Errorf("create ClickHouse client: %w", err) + } + closeClickHouse = plane.ClickHouse.Close + if err := plane.ClickHouse.EnsureDatabase(ctx); err != nil { + plane.Degradation = recordDegradation(logger, plane.Degradation, "ClickHouse database", err) + plane.PublicationReady = false + } + artifactCatalog, err := in.AcquireArtifactCatalog(ctx, plane.ClickHouse) + if err != nil { + return plane, closeClickHouse, err + } + plane.PrivateArtifacts, err = chartifact.New(chartifact.Config{Catalog: artifactCatalog, ClickHouse: plane.ClickHouse, + BatchRows: serverConfig.Server.RecipeBatchRows, + BatchBytes: serverConfig.Server.RecipeBatchBytes, + }) + if err != nil { + return plane, closeClickHouse, fmt.Errorf("create private ClickHouse artifact manager: %w", err) + } + plane.MaterializationReader = &published.Reader{ClickHouse: plane.ClickHouse, Catalog: in.BundleCatalog, Logger: logger, MaxPage: 1000, ActiveManifestResolver: in.ActiveManifestResolver, ActiveReleaseResolver: in.ActiveReleaseResolver} + plane.ClickHouseQueryRows = plane.ClickHouse.QueryRowsArgsVisit + resolver := clickHouseCombineInputResolver{reader: plane.MaterializationReader, scopes: in.Scopes} + plane.ResolveClickHouseInputs = resolver.resolve + plane.WithExecutionReadPins = plane.MaterializationReader.WithExecutionReadPins + return plane, closeClickHouse, nil +} + +func (p *publicationPlane) reconcileBundles(ctx context.Context, logger *slog.Logger, ready bool, degradation error, bundleCatalog publication.BundleCatalog, activeReleaseReader publicationclickhouse.ActiveReleaseReader, sweep publicationSweeper) error { + if !ready { + return nil + } + bundleStore, err := publicationclickhouse.NewBundleStore(p.ClickHouse, bundleCatalog) + if err != nil { + return fmt.Errorf("create dataframe bundle store: %w", err) + } + bundleStore.SetActiveReleaseReader(activeReleaseReader) + if stage, cause := sweep(ctx, p.ClickHouse, bundleStore, time.Now().UTC().Add(-2*time.Minute)); cause != nil { + degradation = recordDegradation(logger, degradation, stage, cause) + p.PublicationReady = false + } + p.Degradation = degradation + if p.PublicationReady { + p.BundleTarget = bundleStore + } + return nil +} + +type releaseHooks struct { + Activate func(context.Context, string, string, []publicationcontract.DataframeSelector) error + Prepare func(context.Context, string, string, []publicationcontract.DataframeSelector) (publicationcontract.ProjectRelease, int64, error) + ValidateGeneration func(context.Context, string, string) error +} + +func newReleaseHooks(serverConfig Config, manifests publicationcontract.ManifestReader, releases publicationcontract.ReleaseRepository, executions publication.ExactExecutionCatalog) releaseHooks { + verificationStore := publicationVerificationStore{executions: executions} + releaseService := &publicationcontract.ReleaseService{Manifests: manifests, Releases: releases, Verifier: verificationStore, Required: serverConfig.Server.RequiredDataframeSelectors} + var activationConflictOnce atomic.Bool + activationConflictOnce.Store(serverConfig.Server.DevActivationConflictOnce) + var repositoryActivationConflictOnce atomic.Bool + repositoryActivationConflictOnce.Store(serverConfig.Server.DevRepositoryActivationConflictOnce) + activateExplorerRelease := newExplorerReleaseActivator(releaseService, &repositoryActivationConflictOnce) + prepareExplorerRelease := func(ctx context.Context, project, generation string, selectors []publicationcontract.DataframeSelector) (publicationcontract.ProjectRelease, int64, error) { + expectedRevision := int64(0) + active, err := releaseService.Active(ctx, project) + if err == nil { + expectedRevision = active.Revision + } else if !errors.Is(err, publicationcontract.ErrNoActiveRelease) { + return publicationcontract.ProjectRelease{}, 0, err + } + release, err := releaseService.Create(ctx, publicationcontract.ActivationRequest{ + Project: project, Generation: generation, GitCommit: generation, + OptionalSelectors: selectors, + }) + if err == nil && activationConflictOnce.CompareAndSwap(true, false) { + expectedRevision++ + } + return release, expectedRevision, err + } + validateExplorerReleaseGeneration := func(ctx context.Context, project, generation string) error { + return releaseService.ValidateGeneration(ctx, project, generation) + } + return releaseHooks{Activate: activateExplorerRelease, Prepare: prepareExplorerRelease, ValidateGeneration: validateExplorerReleaseGeneration} +} + +type explicitGroupRevisionSource interface { + explorer.ExplicitGroupRepository + GetSelection(context.Context, string, string) (*explorer.SelectionRevision, error) +} + +type lifecycleLanePorts struct { + SemanticInventory func(context.Context, catalog.SemanticInventoryPageOptions) (catalog.SemanticInventoryPage, error) + ResolveSemanticInventorySelections func(context.Context, catalog.SemanticInventoryResolveOptions) (catalog.SemanticInventoryResolveResult, error) + SelectionMembersCollection string + InterpretationRepository explorer.InterpretationRepository + TableShapeCapabilities tableshapecap.Repository + ExplicitGroups explicitGroupRevisionSource + SelectionReferenceValidator lifecycle.SelectionReferenceValidator + ReceiptExecutions receiptExecutionFinder +} + +type lifecycleCompositionInput struct { + ServerConfig Config + Logger *slog.Logger + RecipeEngine *dataframeexecution.Engine + CapabilityResolver *explorerCapabilityResolver + ExplorerService *explorer.Service + ArtifactStore *artifactfs.Store + MaterializationReader *published.Reader + ScopeResolver *authscope.ScopeResolver + BundleTarget publication.Target + Degradation error + Hooks releaseHooks + Ports lifecycleLanePorts +} + +func buildLifecycleConfig(in lifecycleCompositionInput) (lifecycle.Config, error) { + serverConfig := in.ServerConfig + logger := in.Logger + recipeEngine := in.RecipeEngine + capabilityResolver := in.CapabilityResolver + explorerService := in.ExplorerService + materializationReader := in.MaterializationReader + scopeResolver := in.ScopeResolver + artifactStore := in.ArtifactStore + var lifecycleConfig lifecycle.Config + compileReceipt := func(ctx context.Context, request lifecycle.CompileReceiptRequest) (*explorer.CompilationReceipt, error) { + var resolveCombineSchemas combineInputSchemaResolver + if materializationReader != nil { + resolveCombineSchemas = func(ctx context.Context, output lower.CompiledRecipeOutput, bindings recipe.RuntimeBindings) (resolvedCombineInputSchema, error) { + return exactPublishedCombineInputSchemas(ctx, output, bindings, materializationReader, materializationReader.WithExecutionReadPins, scopeResolver) + } + } + return compileExplorerReceipt(ctx, request, capabilityResolver, recipeEngine, explorerService, logger, resolveCombineSchemas) + } + persistPublishedWorkspace, err := localWorkspaceWriter(serverConfig.Server.LocalWorkspaceWriteback, serverConfig.Server.LocalWorkspaceProject) + if err != nil { + return lifecycle.Config{}, fmt.Errorf("configure local workspace writeback: %w", err) + } + populationMappingCursorCodec, err := lifecycle.NewHMACPopulationMappingCursorCodec(serverConfig.Server.PopulationMappingCursorSecret) + if err != nil { + return lifecycle.Config{}, fmt.Errorf("configure population mapping cursor signing: %w", err) + } + var artifactPublishedReader lifecycle.ArtifactPublishedReader + if materializationReader != nil { + artifactPublishedReader = materializationReader + if serverConfig.Server.DevArtifactRowDelay > 0 { + artifactPublishedReader = artifactDelayReader{next: artifactPublishedReader, delay: serverConfig.Server.DevArtifactRowDelay} + } + } + schemaIndex, err := fhirschema.GeneratedIndex() + if err != nil { + return lifecycle.Config{}, fmt.Errorf("load generated schema index for row choices: %w", err) + } + rowChoiceResolver, err := lifecycle.NewSchemaRowChoiceResolver(schemaIndex) + if err != nil { + return lifecycle.Config{}, fmt.Errorf("configure row-choice schema resolver: %w", err) + } + explicitGroupResolver, err := lifecycle.NewRepositoryExplicitGroupRevisionResolver(in.Ports.ExplicitGroups) + if err != nil { + return lifecycle.Config{}, fmt.Errorf("configure explicit group revision resolver: %w", err) + } + lifecycleConfig = lifecycle.Config{ + SemanticInventory: in.Ports.SemanticInventory, + ResolveSemanticInventorySelections: in.Ports.ResolveSemanticInventorySelections, + SelectionMembersCollection: in.Ports.SelectionMembersCollection, + InterpretationRepository: in.Ports.InterpretationRepository, + PopulationMappingCursorCodec: populationMappingCursorCodec, + SelectionSourceResolver: published.SelectionSourceAdapter{Reader: materializationReader}, + SelectionReferenceValidator: in.Ports.SelectionReferenceValidator, + RowChoiceResolver: rowChoiceResolver, + RowChoicePlanner: rowChoiceResolver, + ExplicitGroupResolver: explicitGroupResolver, + ExplicitGroupRepository: in.Ports.ExplicitGroups, + TableShapeCapabilities: in.Ports.TableShapeCapabilities, + ScanCategories: configuredCategoryScanner(logger, recipeEngine), + CompileReceipt: compileReceipt, + ConstructionSourceStage: func(ctx context.Context, request lifecycle.ConstructionSourceStageRequest) (explorer.ReceiptConstructionStage, error) { + return compileConstructionSourceStage(ctx, request, recipeEngine) + }, + Capability: lifecycle.CapabilityResolver{ + Current: func(ctx context.Context, project, _ string, generation string) (capability.Snapshot, error) { + return capabilityResolver.Resolve(ctx, project, generation) + }, + Token: capabilityResolver.ResolveToken, + ForCompilation: capabilityResolver.ResolveForCompilation, + ForExecution: capabilityResolver.ResolveForExecution, + Catalog: authoringV2Catalog, + }, + ReceiptLookup: func(ctx context.Context, project, explorerID, receiptID string) (*explorer.CompilationReceipt, error) { + return explorerService.CompilationReceiptForExplorer(ctx, project, explorerID, receiptID) + }, + ArtifactStore: artifactStore, + PublishedReader: artifactPublishedReader, + ArtifactTTL: serverConfig.Server.ArtifactTTL, + ArtifactMaxRows: serverConfig.Server.ArtifactMaxRows, + ArtifactMaxBytes: serverConfig.Server.ArtifactMaxBytes, + PreviewReceipt: func(ctx context.Context, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings, visit func(map[string]any) error) (dataframeexecution.PreviewSummary, error) { + if receipt == nil { + return dataframeexecution.PreviewSummary{}, fmt.Errorf("compilation receipt is required") + } + resolved, err := compileValidatedReceiptResolution(ctx, recipeEngine, receipt, bindings) + if err != nil { + logger.Error("Explorer receipt preview resolution failed", "receipt_id", receipt.ID, "error", err) + return dataframeexecution.PreviewSummary{}, classifyReceiptPreviewResolutionError(receipt.ID, err) + } + output := "" + if len(bindings.OutputNames) > 0 { + output = bindings.OutputNames[0] + } + summary, previewErr := recipeEngine.PreviewOutput(ctx, resolved, dataframeexecution.PreviewRequest{Output: output, Limit: bindings.PreviewLimit, IncludeRowIdentity: bindings.IncludeRowIdentity}, visit) + if previewErr != nil { + logReceiptPreviewContractFailure(logger, requestIDFromContext(ctx), receipt.ID, output, previewErr) + } + return summary, previewErr + }, + ValidateReceiptStream: func(ctx context.Context, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings) error { + err := validateReceiptFullStream(ctx, recipeEngine, receipt, bindings) + if err != nil { + logger.Error("Explorer row-value policy full-stream validation failed", "receipt_id", receipt.ID, "error", err) + } + return err + }, + PopulationMapping: func(ctx context.Context, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings, output string, memberIDs []string, after string, limit int) (dataframeexecution.PopulationMappingResult, error) { + if receipt == nil { + return dataframeexecution.PopulationMappingResult{}, fmt.Errorf("compilation receipt is required") + } + resolved, err := compileValidatedReceiptResolution(ctx, recipeEngine, receipt, bindings) + if err != nil { + logger.Error("Explorer receipt population mapping resolution failed", "receipt_id", receipt.ID, "error", err) + return dataframeexecution.PopulationMappingResult{}, classifyReceiptPreviewResolutionError(receipt.ID, err) + } + reader := dataframeexecution.PopulationMemberReaderFunc(func(_ context.Context, visit func(string) error) error { + for _, id := range memberIDs { + if err := visit(id); err != nil { + return err + } + } + return nil + }) + return recipeEngine.PopulationMapping(ctx, resolved, dataframeexecution.PopulationMappingRequest{Output: output, AfterMemberID: after, MaxUnmapped: limit}, reader) + }, + CellTrace: func(ctx context.Context, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings, request dataframeexecution.CellTraceRequest) (dataframeexecution.CellTraceResult, error) { + if receipt == nil { + return dataframeexecution.CellTraceResult{}, fmt.Errorf("compilation receipt is required") + } + resolved, err := compileValidatedReceiptResolution(ctx, recipeEngine, receipt, bindings) + if err != nil { + logger.Error("Explorer receipt cell trace resolution failed", "receipt_id", receipt.ID, "error", err) + return dataframeexecution.CellTraceResult{}, classifyReceiptPreviewResolutionError(receipt.ID, err) + } + return recipeEngine.CellTrace(ctx, resolved, request) + }, + RowLineage: func(ctx context.Context, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings, request dataframeexecution.RowLineageRequest) (dataframeexecution.RowLineageResult, error) { + if receipt == nil { + return dataframeexecution.RowLineageResult{}, fmt.Errorf("compilation receipt is required") + } + resolved, err := compileValidatedReceiptResolution(ctx, recipeEngine, receipt, bindings) + if err != nil { + logger.Error("Explorer receipt row lineage resolution failed", "receipt_id", receipt.ID, "error", err) + return dataframeexecution.RowLineageResult{}, classifyReceiptPreviewResolutionError(receipt.ID, err) + } + return recipeEngine.RowLineage(ctx, resolved, request) + }, + TableShapeExclusions: func(ctx context.Context, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings, request dataframeexecution.TableShapeExclusionRequest) (dataframeexecution.TableShapeExclusionResult, error) { + if receipt == nil { + return dataframeexecution.TableShapeExclusionResult{}, fmt.Errorf("compilation receipt is required") + } + resolved, err := compileValidatedReceiptResolution(ctx, recipeEngine, receipt, bindings) + if err != nil { + logger.Error("Explorer receipt table-shape exclusion resolution failed", "receipt_id", receipt.ID, "error", err) + return dataframeexecution.TableShapeExclusionResult{}, classifyReceiptPreviewResolutionError(receipt.ID, err) + } + return recipeEngine.TableShapeExclusions(ctx, resolved, request) + }, + MaterializeReceipt: explorerReceiptMaterializer( + recipeEngine, in.BundleTarget, in.Ports.ReceiptExecutions, in.Degradation, logger, + serverConfig.Server.RecipeBatchRows, serverConfig.Server.RecipeBatchBytes, + serverConfig.Server.RecipeQualityMaxRows, serverConfig.Server.RecipeQualityMaxDistinctKeys, + ), + ValidateReleaseGeneration: in.Hooks.ValidateGeneration, + ActivateRelease: in.Hooks.Activate, + PrepareRelease: in.Hooks.Prepare, + PersistPublishedWorkspace: persistPublishedWorkspace, + } + return lifecycleConfig, nil +} + +func wireExplorerRoutes(server *httpapi.HTTPServer, generationService *loadapi.Service, authorizer authscope.Authorizer, graphResolver *graphresolver.Resolver, scopeResolver *authscope.ScopeResolver, explorerService *explorer.Service, lifecycleConfig lifecycle.Config, inputs constructionInputsCatalog, deps routeDependencies) error { + explorerHandlers := newExplorerHTTPHandlers(authorizer, func(ctx context.Context, principal *authscope.Principal, project string) error { + if scopeResolver == nil { + return nil + } + return scopeResolver.AuthorizeReadProject(ctx, principal, project) + }, explorerService, lifecycleConfig) + explorerHandlers.constructionInputs = inputs + if err := registerRoutes(server, generationService, authorizer, graphResolver, explorerHandlers, deps); err != nil { + return fmt.Errorf("register HTTP routes: %w", err) + } + return nil +} + +func composeArangoRuntime(ctx context.Context, serverConfig Config, logger *slog.Logger) (server *httpapi.HTTPServer, closeRuntime func() error, err error) { + var cleanups []func() error + defer func() { + if err == nil { + return + } + _ = runRuntimeCloseSteps(cleanups) + }() + arangoBackend, err := openArangoBackend(ctx, arangofactory.Config{ + URL: serverConfig.Server.URL, + Database: serverConfig.Server.Database, + }) + if err != nil { + return nil, nil, err + } + cleanups = append(cleanups, func() error { + closeCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) + defer cancel() + return arangoBackend.Close(closeCtx) + }) + lifecycleClient := arangoBackend.Client + var degradation error + if err := lifecycleClient.Bootstrap(ctx, arangofactory.RecipeBootstrapSpec()); err != nil { + degradation = recordDegradation(logger, degradation, "bootstrap recipe registry", err) + } + if err := lifecycleClient.Bootstrap(ctx, arangofactory.RevisionBootstrapSpec()); err != nil { + degradation = recordDegradation(logger, degradation, "bootstrap recipe revision registry", err) + } + recipeRegistry := arangoBackend.RecipeRegistry + if serverConfig.Server.ClickHouse.Enabled { + data, err := os.ReadFile(serverConfig.Server.Dataframer.Recipe) + if err != nil { + return nil, nil, fmt.Errorf("read dataframer recipe %q: %w", serverConfig.Server.Dataframer.Recipe, err) + } + defaultBundle, err := recipe.Parse(data) + if err != nil { + return nil, nil, fmt.Errorf("parse dataframer recipe %q: %w", serverConfig.Server.Dataframer.Recipe, err) + } + if _, err := (exec.PersistentRegistry{Store: recipeRegistry}).RegisterVersion(ctx, defaultBundle); err != nil { + degradation = recordDegradation(logger, degradation, "register default dataframe recipe", err) + } + } + lifecycleStore := arangoBackend.Dataset + activeManifestResolver := publicationcontract.ActiveResolver(lifecycleStore) + + discoveryCache := catalog.NewCache() + catalogStore := arangoBackend.Catalog + discoverFields := discoveryCache.DiscoverFields(catalogStore.DiscoverFields) + discoverReferences := discoveryCache.DiscoverReferences(catalogStore.DiscoverReferences) + + auth, err := wireAuth(serverConfig, serverConfig.Server.AllowUnauthenticated || serverConfig.Auth.AllowUnauthenticated, catalogStore.DiscoverExistingAuthResourcePaths) + if err != nil { + return nil, nil, fmt.Errorf("configure authentication: %w", err) + } + authenticator, authorizer, scopeResolver := auth.authenticator, auth.authorizer, auth.scopeResolver + + aqlRows := selectedAQLAdapter{exec: lifecycleClient} + dataframes := dataframeexecution.NewService(dataframeexecution.ServiceConfig{SourceRows: aqlRows.serviceRows}) + sourceCompileTarget := compiler.ArangoTarget() + + publishedRegistry := arangoBackend.PublishedRegistry + publicationReady := true + if err := lifecycleClient.Bootstrap(ctx, arangofactory.PublishedBootstrapSpec()); err != nil { + degradation = recordDegradation(logger, degradation, "bootstrap dataframe registry", err) + publicationReady = false + } + var clickhouse *clickhousestore.Client + var materializationReader *published.Reader + var privateClickHouseArtifacts *chartifact.Manager + var plane *publicationPlane + if serverConfig.Server.ClickHouse.Enabled { + var closeClickHouse func() error + var planeErr error + plane, closeClickHouse, planeErr = newPublicationPlane(ctx, serverConfig, logger, publicationPlaneInput{ + InitialDegradation: degradation, + InitialPublicationReady: publicationReady, + ClickHouse: clickhousestore.Options{URL: serverConfig.Server.ClickHouse.URL, Database: serverConfig.Server.ClickHouse.Database, Username: serverConfig.Server.ClickHouse.Username, Password: serverConfig.Server.ClickHouse.Password}, + AcquireArtifactCatalog: func(ctx context.Context, clickhouse *clickhousestore.Client) (chartifact.Catalog, error) { + if err := lifecycleClient.Bootstrap(ctx, arangofactory.ArtifactBootstrapSpec()); err != nil { + return nil, fmt.Errorf("bootstrap private dataframe artifact catalog: %w", err) + } + privateArtifactCatalog, err := arangofactory.NewArtifactCatalog(lifecycleClient) + if err != nil { + return nil, fmt.Errorf("create private dataframe artifact catalog: %w", err) + } + return privateArtifactCatalog, nil + }, + BundleCatalog: publishedRegistry, + ActiveManifestResolver: activeManifestResolver, + ActiveReleaseResolver: lifecycleStore, + Scopes: scopeResolver, + }) + if closeClickHouse != nil { + cleanups = append(cleanups, closeClickHouse) + } + if planeErr != nil { + return nil, nil, planeErr + } + clickhouse = plane.ClickHouse + privateClickHouseArtifacts = plane.PrivateArtifacts + materializationReader = plane.MaterializationReader + degradation = plane.Degradation + publicationReady = plane.PublicationReady + } + recipeRevisions := arangoBackend.RecipeRevisions + var clickHouseQueryRows dataframeexecution.ClickHouseQueryRows + var resolveClickHouseInputs dataframeexecution.ResolveClickHouseInputs + var withExecutionReadPins dataframeexecution.WithExecutionReadPins + if plane != nil { + clickHouseQueryRows = plane.ClickHouseQueryRows + resolveClickHouseInputs = plane.ResolveClickHouseInputs + withExecutionReadPins = plane.WithExecutionReadPins + } + recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: sourceCompileTarget, + Registry: recipeRegistry, + Revisions: recipeRevisions, + ResolveBundle: recipeSchemaResolver(catalogStore.DiscoverFields, discoveryCache), + PreparePreviewIndex: loggedPreviewIndexPreparation(logger, func(ctx context.Context, spec compiler.PreviewCoveringIndexSpec) error { + var err error + if len(spec.StoredValues) != 0 { + if spec.Supersedes != nil { + err = lifecycleClient.EnsurePreviewCoveringIndexWithStoredValuesReplacing( + ctx, spec.Collection, spec.Name, spec.Fields, spec.StoredValues, + spec.Supersedes.Name, spec.Supersedes.Fields, spec.Supersedes.StoredValues, + ) + } else { + err = lifecycleClient.EnsurePreviewCoveringIndexWithStoredValues(ctx, spec.Collection, spec.Name, spec.Fields, spec.StoredValues) + } + } else if spec.Supersedes != nil { + err = lifecycleClient.EnsurePreviewCoveringIndexReplacing(ctx, spec.Collection, spec.Name, spec.Fields, spec.Supersedes.Name, spec.Supersedes.Fields) + } else { + err = lifecycleClient.EnsurePreviewCoveringIndex(ctx, spec.Collection, spec.Name, spec.Fields) + } + return err + }), + PreviewCoveringIndexes: func(ctx context.Context, collection string) ([]dataframeexecution.PreviewCollectionIndex, error) { + indexes, err := lifecycleClient.ListPreviewCoveringIndexes(ctx, collection) + if err != nil { + return nil, err + } + return previewCollectionIndexesFromStore(indexes), nil + }, + PreviewCollectionRevision: lifecycleClient.CollectionRevision, + PreviewCollectionScans: func(ctx context.Context, command command.CompiledCommand, collection string) (dataframeexecution.PreviewCollectionScanEvidence, error) { + query, bindVars, dispatchErr := command.AQLParameters() + if dispatchErr != nil { + return dataframeexecution.PreviewCollectionScanEvidence{}, dispatchErr + } + result, err := lifecycleClient.Explain(ctx, arangofactory.ExplainRequest{Query: query, BindVars: bindVars}) + if err != nil { + return dataframeexecution.PreviewCollectionScanEvidence{}, err + } + return PreviewCollectionScanEvidenceFromExplain(result, collection), nil + }, + PreviewCollectionCount: lifecycleClient.CollectionCount, + ClickHouseQueryRows: clickHouseQueryRows, + PreviewSourceRows: withConfiguredAQLRows(logger, "preview_query_rows", aqlRows.dispatchPreviewRows), + ResolveClickHouseInputs: resolveClickHouseInputs, + WithExecutionReadPins: withExecutionReadPins, + PrivateClickHouseArtifacts: privateClickHouseArtifacts, + SourceRows: withConfiguredAQLRows(logger, "query_rows", aqlRows.dispatchRows), + ScopeDigest: recipeScopeDigest, + RootPageRows: serverConfig.Server.RecipeQueryPageRows, + }) + if err != nil { + return nil, nil, fmt.Errorf("create dataframe recipe engine: %w", err) + } + var bundleTarget publication.Target + if plane != nil { + if err := plane.reconcileBundles(ctx, logger, publicationReady, degradation, publishedRegistry, lifecycleStore, + func(ctx context.Context, clickhouse *clickhousestore.Client, bundleStore *publicationclickhouse.ClickHouseBundleStore, cutoff time.Time) (string, error) { + if err := bundleStore.Reconcile(ctx, cutoff); err != nil { + return "dataframe publication reconciliation", err + } + return "", nil + }); err != nil { + return nil, nil, err + } + publicationReady = plane.PublicationReady + degradation = plane.Degradation + bundleTarget = plane.BundleTarget + } + hooks := newReleaseHooks(serverConfig, lifecycleStore, lifecycleStore, publishedRegistry) + explorerStore := arangoBackend.Explorer + if err := explorerStore.MigrateLegacyRepositoryConfigs(ctx); err != nil { + return nil, nil, fmt.Errorf("migrate legacy Explorer repository configs: %w", err) + } + capabilitySnapshots := arangoBackend.ExplorerCapabilitySnapshots + capabilityResolver, err := newExplorerCapabilityResolver(catalogStore, scopeResolver, activeManifestResolver, capabilitySnapshots) + if err != nil { + return nil, nil, fmt.Errorf("create Explorer capability resolver: %w", err) + } + explorerService, err := explorer.NewService(explorerStore) + if err != nil { + return nil, nil, fmt.Errorf("create Explorer service: %w", err) + } + artifactStore, err := artifactfs.New(serverConfig.Server.ArtifactDirectory) + if err != nil { + return nil, nil, fmt.Errorf("create Explorer artifact store: %w", err) + } + resolver := graphresolver.NewResolver(graphresolver.ResolverConfig{ + DataframeQuery: queryapi.Config{ + CompileTarget: sourceCompileTarget, + DiscoverReferences: discoverReferences, + DiscoverFields: discoverFields, + Dataframes: dataframes, + ScopeResolver: scopeResolver, + ActiveManifestResolver: activeManifestResolver, + Explain: func(ctx context.Context, compiled compiler.CompiledQuery) error { + _, err := explainCompiledQuery(ctx, lifecycleClient, compiled) + return err + }, + }, + MaterializationReader: materializationReader, + Logger: logger, + RecipeControl: dataframeexecution.Control{Engine: recipeEngine, ExplainConnection: func(ctx context.Context, compiled compiler.CompiledQuery) (dataframeexecution.ExplainAssessment, error) { + return explainCompiledQuery(ctx, lifecycleClient, compiled) + }, ExplainPostgreSQLConnection: func(ctx context.Context, compiled compiler.CompiledQuery) (dataframeexecution.PostgreSQLPlanAssessment, error) { + return explainPostgreSQLCompiledQuery(ctx, nil, compiled) + }}, + RecipeAuthorizer: recipeAuthorization{resolver: scopeResolver, selectionMembersCollection: arangofactory.SelectionMembersCollection}, + RecipeRevisions: recipeRevisions, + RecipeExecutions: graphresolver.NewAuthorizedRecipeExecutionReader(publishedRegistry, scopeResolver), + }) + ingestRunner := loadapi.IngestRunner{BaseOptions: ingest.LoadOptions{ + ConnectionOptions: arangofactory.Config{ + URL: serverConfig.Server.URL, + Database: serverConfig.Server.Database, + }, + Schema: serverConfig.Server.Schema, + }} + generationService, err := loadapi.NewService(loadapi.ServiceConfig{ + LoadGeneration: ingestRunner.RunGeneration, + GenerationActivator: lifecycleStore, + DataframeReleases: publishedRegistry, + Logger: logger, + OnSuccess: func(project string) { + discoveryCache.InvalidateProject(project) + if scopeResolver != nil { + scopeResolver.InvalidateProject(project) + } + }, + }) + if err != nil { + return nil, nil, fmt.Errorf("create generation load service: %w", err) + } + tableShapeCapabilities, err := arangofactory.NewTableShapeCapabilityRepository(lifecycleClient) + if err != nil { + return nil, nil, fmt.Errorf("configure table-shape capability repository: %w", err) + } + lifecycleConfig, err := buildLifecycleConfig(lifecycleCompositionInput{ + ServerConfig: serverConfig, + Logger: logger, + RecipeEngine: recipeEngine, + CapabilityResolver: capabilityResolver, + ExplorerService: explorerService, + ArtifactStore: artifactStore, + MaterializationReader: materializationReader, + ScopeResolver: scopeResolver, + BundleTarget: bundleTarget, + Degradation: degradation, + Hooks: hooks, + Ports: lifecycleLanePorts{ + SemanticInventory: catalogStore.PageSemanticInventory, + ResolveSemanticInventorySelections: catalogStore.ResolveSemanticInventorySelections, + SelectionMembersCollection: arangofactory.SelectionMembersCollection, + InterpretationRepository: explorerStore, + TableShapeCapabilities: tableShapeCapabilities, + ExplicitGroups: explorerStore, + SelectionReferenceValidator: explorerStore.ValidateSelectionReferences, + ReceiptExecutions: publishedRegistry, + }, + }) + if err != nil { + return nil, nil, err + } + server, err = httpapi.NewHTTPServer(httpapi.HTTPConfig{Authenticator: authenticator, Authorizer: authorizer, Logger: logger, + CoreReadyCheck: func(ctx context.Context) error { + return lifecycleClient.QueryRows(ctx, "RETURN {ready: true}", 1, nil, func(map[string]any) error { return nil }) + }, + ClickHouseReadyCheck: func(ctx context.Context) error { + if degradation != nil { + return degradation + } + if clickhouse == nil { + return nil + } + return clickhouse.Ping(ctx) + }, ClickHouseEnabled: serverConfig.Server.ClickHouse.Enabled}) + if err != nil { + return nil, nil, fmt.Errorf("create HTTP server: %w", err) + } + if err := wireExplorerRoutes(server, generationService, authorizer, resolver, scopeResolver, explorerService, lifecycleConfig, constructionInputsCatalog{ + reader: materializationReader, catalog: publishedRegistry, capabilities: lifecycleConfig.Capability, + scopes: scopeResolver, explorers: explorerService, revisions: explorerService, + }, routeDependencies{releases: publishedRegistry, scopes: auth.scopePolicy}); err != nil { + return nil, nil, err + } + if privateClickHouseArtifacts != nil { + reconcileCtx, cancelReconciler := context.WithCancel(ctx) + reconcilerDone := startPrivateArtifactReconciler(reconcileCtx, privateClickHouseArtifacts, logger) + cleanups = append(cleanups, func() error { + cancelReconciler() + timer := time.NewTimer(cleanupTimeout) + defer timer.Stop() + select { + case <-reconcilerDone: + case <-timer.C: + logger.Error("private ClickHouse artifact reconciler did not stop before shutdown deadline") + } + return nil + }) + } + if artifactStore != nil { + artifactReconcileCtx, cancelArtifactReconciler := context.WithCancel(ctx) + artifactReconcilerDone := startArtifactFSReconciler(artifactReconcileCtx, artifactStore, logger) + cleanups = append(cleanups, func() error { + cancelArtifactReconciler() + timer := time.NewTimer(cleanupTimeout) + defer timer.Stop() + select { + case <-artifactReconcilerDone: + case <-timer.C: + logger.Error("filesystem artifact reconciler did not stop before shutdown deadline") + } + return nil + }) + } + closeRuntime = func() error { return runRuntimeCloseSteps(cleanups) } + return server, closeRuntime, nil +} + +func runArangoRuntime(ctx context.Context, serverConfig Config, logger *slog.Logger) error { + server, closeRuntime, err := composeArangoRuntime(ctx, serverConfig, logger) + if err != nil { + return err + } + return serveRuntime(ctx, server, serverConfig.Server.Listen, closeRuntime, "close Arango server runtime", logger, + "listen", serverConfig.Server.Listen, "database", serverConfig.Server.Database, "no_auth", serverConfig.Server.AllowUnauthenticated || serverConfig.Auth.AllowUnauthenticated) +} diff --git a/internal/server/config_test.go b/internal/server/config_test.go index e831aa0d7..5b1bdd8d2 100644 --- a/internal/server/config_test.go +++ b/internal/server/config_test.go @@ -1,7 +1,9 @@ package server import ( + "context" "flag" + "fmt" "os" "path/filepath" "strings" @@ -263,8 +265,6 @@ func TestRequiredDataframeSelectorsFromEnvironment(t *testing.T) { t.Fatalf("environment config = %#v", cfg.Server) } if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "dataframer.recipe") { - // Environment parsing succeeded; the unrelated default ClickHouse recipe - // requirement remains enforced by Validate. t.Fatalf("validation error = %v", err) } } @@ -354,3 +354,368 @@ func TestCalyprAuthNoAuthBypassDoesNotRequireTrustedIssuer(t *testing.T) { }) } } + +func postgresReadyConfig() Config { + cfg := DefaultConfig() + cfg.Server.Backend = sourceBackendPostgres + cfg.Server.ClickHouse.Enabled = false + cfg.Auth.AllowUnauthenticated = true + cfg.Server.PostgreSQL.Host = "db.internal" + cfg.Server.PostgreSQL.Database = "loom_pg" + cfg.Server.PostgreSQL.User = "loom" + return cfg +} + +func writeServerConfigFile(t *testing.T, contents string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func TestSourceBackendDefaultsToArangoForLegacyConfigs(t *testing.T) { + cfg := DefaultConfig() + if cfg.Server.Backend != sourceBackendArango { + t.Fatalf("default source backend = %q, want %q", cfg.Server.Backend, sourceBackendArango) + } + if cfg.Server.URL != defaultServerURL || cfg.Server.Database != defaultServerDatabase { + t.Fatalf("legacy Arango defaults changed: url=%q database=%q", cfg.Server.URL, cfg.Server.Database) + } + pg := cfg.Server.PostgreSQL + if pg.Port != defaultPostgresPort || pg.SSLMode != defaultPostgresSSLMode || pg.MaxOpenConns != defaultPostgresMaxOpenConns || pg.MaxIdleConns != defaultPostgresMaxIdleConns || pg.ConnMaxLifetime != defaultPostgresConnMaxLifetime { + t.Fatalf("postgres connection defaults = %#v", pg) + } + + loaded, err := LoadConfig("") + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + if loaded.Server.Backend != sourceBackendArango { + t.Fatalf("environment-loaded backend = %q", loaded.Server.Backend) + } + + path := writeServerConfigFile(t, "server:\n url: http://arango:8529\n database: fhir_proto\n clickhouse:\n enabled: false\n allow_unauthenticated: true\n") + legacy, err := LoadConfig(path) + if err != nil { + t.Fatalf("legacy YAML LoadConfig() error = %v", err) + } + if legacy.Server.Backend != sourceBackendArango || legacy.Server.URL != "http://arango:8529" || legacy.Server.Database != "fhir_proto" { + t.Fatalf("legacy YAML resolution = %#v", legacy.Server) + } + if err := legacy.Validate(); err != nil { + t.Fatalf("legacy YAML Validate() error = %v", err) + } + + options, err := parseServerOptions([]string{"--listen", ":19091", "--no-auth", "--dataframer-recipe", "recipe.json"}, flag.ContinueOnError) + if err != nil { + t.Fatalf("legacy CLI parseServerOptions() error = %v", err) + } + if options.Server.Backend != sourceBackendArango || options.Server.URL != defaultServerURL || options.Server.Database != defaultServerDatabase { + t.Fatalf("legacy CLI resolution = %#v", options.Server) + } +} + +func TestLoadConfigExplicitArangoBackendParsesUnchanged(t *testing.T) { + path := writeServerConfigFile(t, "server:\n backend: arango\n url: http://arango:8529\n database: custom_graph\n clickhouse:\n enabled: false\n allow_unauthenticated: true\n") + cfg, err := LoadConfig(path) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + if err := cfg.Validate(); err != nil { + t.Fatalf("explicit arango Validate() error = %v", err) + } + if cfg.Server.Backend != sourceBackendArango || cfg.Server.URL != "http://arango:8529" || cfg.Server.Database != "custom_graph" { + t.Fatalf("explicit arango resolution = %#v", cfg.Server) + } + if cfg.Server.PostgreSQL.Port != defaultPostgresPort || cfg.Server.PostgreSQL.SSLMode != defaultPostgresSSLMode { + t.Fatalf("explicit arango changed postgres defaults: %#v", cfg.Server.PostgreSQL) + } +} + +func TestLoadConfigPostgresBackendParsesConnectionSettings(t *testing.T) { + path := writeServerConfigFile(t, "server:\n backend: postgres\n clickhouse:\n enabled: false\n allow_unauthenticated: true\n postgresql:\n host: db.internal\n port: 5433\n database: loom_pg\n user: loom\n password: yaml-pg-secret-canary\n sslmode: verify-full\n max_open_conns: 20\n max_idle_conns: 8\n conn_max_lifetime: 15m\n") + cfg, err := LoadConfig(path) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + if err := cfg.Validate(); err != nil { + t.Fatalf("postgres Validate() error = %v", err) + } + pg := cfg.Server.PostgreSQL + if pg.Host != "db.internal" || pg.Port != 5433 || pg.Database != "loom_pg" || pg.User != "loom" || pg.Password != "yaml-pg-secret-canary" || pg.SSLMode != "verify-full" || pg.MaxOpenConns != 20 || pg.MaxIdleConns != 8 || pg.ConnMaxLifetime != 15*time.Minute { + t.Fatalf("postgres settings = %#v", pg) + } + if cfg.Server.Backend != sourceBackendPostgres { + t.Fatalf("backend = %q", cfg.Server.Backend) + } + if cfg.Server.URL != defaultServerURL || cfg.Server.Database != defaultServerDatabase { + t.Fatalf("postgres selection changed legacy Arango fields: url=%q database=%q", cfg.Server.URL, cfg.Server.Database) + } +} + +func TestLoadConfigRejectsUnknownPostgresField(t *testing.T) { + path := writeServerConfigFile(t, "server:\n backend: postgres\n postgresql:\n host: db.internal\n ssl_mode: require\n") + if _, err := LoadConfig(path); err == nil { + t.Fatal("unknown postgres field unexpectedly accepted") + } +} + +func TestValidateRejectsUnknownSourceBackendBeforeStartup(t *testing.T) { + cfg := DefaultConfig() + cfg.Server.Backend = "cockroachdb" + cfg.Auth.AllowUnauthenticated = true + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "server.backend must be arango or postgres") { + t.Fatalf("unknown backend Validate() error = %v", err) + } + + if _, err := parseServerOptions([]string{"--no-auth", "--dataframer-recipe", "recipe.json", "--backend", "graphdb"}, flag.ContinueOnError); err == nil || !strings.Contains(err.Error(), "invalid server config: server.backend must be arango or postgres") { + t.Fatalf("unknown backend CLI error = %v", err) + } + + path := writeServerConfigFile(t, "server:\n backend: mongodb\n clickhouse:\n enabled: false\n allow_unauthenticated: true\n") + loaded, err := LoadConfig(path) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + if err := loaded.Validate(); err == nil || !strings.Contains(err.Error(), "server.backend must be arango or postgres") { + t.Fatalf("unknown YAML backend Validate() error = %v", err) + } +} + +func TestValidateRequiresCompletePostgresConnectionSettings(t *testing.T) { + for _, test := range []struct { + name string + mutate func(*Config) + want string + }{ + {name: "missing host", mutate: func(c *Config) { c.Server.PostgreSQL.Host = "" }, want: "server.postgresql.host"}, + {name: "missing database", mutate: func(c *Config) { c.Server.PostgreSQL.Database = "" }, want: "server.postgresql.database"}, + {name: "missing user", mutate: func(c *Config) { c.Server.PostgreSQL.User = "" }, want: "server.postgresql.user"}, + {name: "zero port", mutate: func(c *Config) { c.Server.PostgreSQL.Port = 0 }, want: "server.postgresql.port"}, + {name: "port out of range", mutate: func(c *Config) { c.Server.PostgreSQL.Port = 70000 }, want: "server.postgresql.port"}, + {name: "unknown sslmode", mutate: func(c *Config) { c.Server.PostgreSQL.SSLMode = "needed" }, want: "server.postgresql.sslmode"}, + {name: "blank sslmode", mutate: func(c *Config) { c.Server.PostgreSQL.SSLMode = "" }, want: "server.postgresql.sslmode"}, + {name: "zero max open conns", mutate: func(c *Config) { c.Server.PostgreSQL.MaxOpenConns = 0 }, want: "server.postgresql.max_open_conns"}, + {name: "negative max open conns", mutate: func(c *Config) { c.Server.PostgreSQL.MaxOpenConns = -3 }, want: "server.postgresql.max_open_conns"}, + {name: "idle above open", mutate: func(c *Config) { c.Server.PostgreSQL.MaxOpenConns = 4; c.Server.PostgreSQL.MaxIdleConns = 5 }, want: "server.postgresql.max_idle_conns"}, + {name: "negative idle", mutate: func(c *Config) { c.Server.PostgreSQL.MaxIdleConns = -1 }, want: "server.postgresql.max_idle_conns"}, + {name: "negative lifetime", mutate: func(c *Config) { c.Server.PostgreSQL.ConnMaxLifetime = -time.Second }, want: "server.postgresql.conn_max_lifetime"}, + } { + t.Run(test.name, func(t *testing.T) { + cfg := postgresReadyConfig() + test.mutate(&cfg) + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("Validate() error = %v, want mention of %q", err, test.want) + } + }) + } +} + +func TestValidateAcceptsPostgresWithoutPasswordAndWithZeroLifetime(t *testing.T) { + cfg := postgresReadyConfig() + cfg.Server.PostgreSQL.Password = "" + cfg.Server.PostgreSQL.ConnMaxLifetime = 0 + if err := cfg.Validate(); err != nil { + t.Fatalf("passwordless postgres Validate() error = %v", err) + } +} + +func TestArangoBackendIgnoresPostgresContract(t *testing.T) { + cfg := DefaultConfig() + cfg.Auth.AllowUnauthenticated = true + cfg.Server.ClickHouse.Enabled = false + cfg.Server.PostgreSQL.Port = 0 + cfg.Server.PostgreSQL.SSLMode = "nonsense" + cfg.Server.PostgreSQL.MaxIdleConns = 99 + cfg.Server.PostgreSQL.MaxOpenConns = -2 + if err := cfg.Validate(); err != nil { + t.Fatalf("arango backend must not validate postgres contract: %v", err) + } +} + +func TestPostgresBackendRejectsConflictingArangoSettings(t *testing.T) { + t.Run("customized yaml url", func(t *testing.T) { + cfg := postgresReadyConfig() + cfg.Server.URL = "http://arango:8529" + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "server.url") { + t.Fatalf("customized url with postgres error = %v", err) + } + }) + t.Run("customized yaml database", func(t *testing.T) { + cfg := postgresReadyConfig() + cfg.Server.Database = "legacy_graph" + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "server.database") { + t.Fatalf("customized database with postgres error = %v", err) + } + }) + base := []string{"--no-auth", "--dataframer-recipe", "recipe.json", "--backend", "postgres", "--pg-host", "db.internal", "--pg-database", "loom_pg", "--pg-user", "loom"} + for _, test := range []struct { + name string + args []string + want string + }{ + {name: "explicit url", args: []string{"--url", "http://arango:8529"}, want: "--url"}, + {name: "explicit url default value", args: []string{"--url", defaultServerURL}, want: "--url"}, + {name: "explicit database", args: []string{"--database", "other_db"}, want: "--database"}, + {name: "explicit database default value", args: []string{"--database", defaultServerDatabase}, want: "--database"}, + } { + t.Run(test.name, func(t *testing.T) { + args := append(append([]string{}, base...), test.args...) + if _, err := parseServerOptions(args, flag.ContinueOnError); err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("postgres + %s error = %v, want mention of %q", test.name, err, test.want) + } + }) + } + t.Run("config path explicit url", func(t *testing.T) { + path := writeServerConfigFile(t, "server:\n backend: postgres\n clickhouse:\n enabled: false\n allow_unauthenticated: true\n postgresql:\n host: db.internal\n database: loom_pg\n user: loom\n") + if _, err := parseServerOptions([]string{"--config", path, "--url", "http://arango:8529"}, flag.ContinueOnError); err == nil || !strings.Contains(err.Error(), "--url") { + t.Fatalf("--config postgres + explicit --url error = %v", err) + } + }) + t.Run("config path explicit backend cannot silently select arango", func(t *testing.T) { + path := writeServerConfigFile(t, "server:\n clickhouse:\n enabled: false\n allow_unauthenticated: true\n") + _, err := parseServerOptions([]string{"--config", path, "--backend", "postgres"}, flag.ContinueOnError) + if err == nil || !strings.Contains(err.Error(), "--backend") || !strings.Contains(err.Error(), "--config file") { + t.Fatalf("--config without backend + explicit --backend postgres error = %v", err) + } + }) + t.Run("config path owns backend even when redundant", func(t *testing.T) { + path := writeServerConfigFile(t, "server:\n backend: arango\n clickhouse:\n enabled: false\n allow_unauthenticated: true\n") + if _, err := parseServerOptions([]string{"--config", path, "--backend", "arango"}, flag.ContinueOnError); err == nil || !strings.Contains(err.Error(), "--backend") { + t.Fatalf("--config + redundant explicit --backend error = %v", err) + } + }) + t.Run("config path explicit pg flag cannot silently keep yaml sslmode", func(t *testing.T) { + path := writeServerConfigFile(t, "server:\n backend: postgres\n clickhouse:\n enabled: false\n allow_unauthenticated: true\n postgresql:\n host: db.internal\n database: loom_pg\n user: loom\n") + _, err := parseServerOptions([]string{"--config", path, "--pg-sslmode", "verify-full"}, flag.ContinueOnError) + if err == nil || !strings.Contains(err.Error(), "--pg-sslmode") || !strings.Contains(err.Error(), "--config file") { + t.Fatalf("--config postgres + explicit --pg-sslmode error = %v", err) + } + }) + t.Run("config path explicit pg flag rejected for arango too", func(t *testing.T) { + path := writeServerConfigFile(t, "server:\n backend: arango\n clickhouse:\n enabled: false\n allow_unauthenticated: true\n") + if _, err := parseServerOptions([]string{"--config", path, "--pg-host", "db.internal", "--pg-password", "canary-secret"}, flag.ContinueOnError); err == nil || !strings.Contains(err.Error(), "--pg-host") || strings.Contains(err.Error(), "canary-secret") { + t.Fatalf("--config arango + explicit --pg-host error = %v", err) + } + }) + t.Run("arango backend keeps customized url", func(t *testing.T) { + options, err := parseServerOptions([]string{"--no-auth", "--dataframer-recipe", "recipe.json", "--url", "http://arango:8529"}, flag.ContinueOnError) + if err != nil { + t.Fatalf("arango backend customized url error = %v", err) + } + if options.Server.URL != "http://arango:8529" { + t.Fatalf("customized url = %q", options.Server.URL) + } + }) +} + +func TestParseServerOptionsPostgresBackendFromFlags(t *testing.T) { + options, err := parseServerOptions([]string{ + "--no-auth", "--dataframer-recipe", "recipe.json", + "--backend", "Postgres", + "--pg-host", "db.internal", "--pg-port", "5433", "--pg-database", "loom_pg", "--pg-user", "loom", + "--pg-password", "cli-secret", "--pg-sslmode", "require", + "--pg-max-open-conns", "25", "--pg-max-idle-conns", "10", "--pg-conn-max-lifetime", "45m", + }, flag.ContinueOnError) + if err != nil { + t.Fatalf("postgres CLI parseServerOptions() error = %v", err) + } + if options.Server.Backend != sourceBackendPostgres { + t.Fatalf("backend = %q", options.Server.Backend) + } + pg := options.Server.PostgreSQL + if pg.Host != "db.internal" || pg.Port != 5433 || pg.Database != "loom_pg" || pg.User != "loom" || pg.Password != "cli-secret" || pg.SSLMode != "require" || pg.MaxOpenConns != 25 || pg.MaxIdleConns != 10 || pg.ConnMaxLifetime != 45*time.Minute { + t.Fatalf("postgres CLI settings = %#v", pg) + } + if options.Server.URL != defaultServerURL || options.Server.Database != defaultServerDatabase { + t.Fatalf("postgres selection changed legacy Arango fields: url=%q database=%q", options.Server.URL, options.Server.Database) + } +} + +func TestRunRefusesPostgresBackendBeforeOpeningStores(t *testing.T) { + cfg := postgresReadyConfig() + if err := run(context.Background(), cfg); err == nil || !strings.Contains(err.Error(), "postgres") || !strings.Contains(err.Error(), "not wired") { + t.Fatalf("run() with postgres backend error = %v", err) + } + + cfg.Server.Backend = "graphdb" + if err := run(context.Background(), cfg); err == nil || !strings.Contains(err.Error(), "server.backend must be arango or postgres") { + t.Fatalf("run() with unknown backend error = %v", err) + } +} + +func TestPostgresCredentialsNeverAppearInConfigRenderingOrErrors(t *testing.T) { + const credentialCanary = "Sup3r-Secret-Pa55w0rd" + pgPassword := "postgres://loom:" + credentialCanary + "@db.internal" + + renderedConfig := func() Config { + cfg := postgresReadyConfig() + cfg.Server.PostgreSQL.Password = pgPassword + cfg.Server.ClickHouse.Enabled = true + cfg.Server.ClickHouse.Password = "clickhouse-" + credentialCanary + cfg.Auth.Basic.Username = "loom" + cfg.Auth.Basic.Password = "basic-" + credentialCanary + cfg.Server.PopulationMappingCursorSecret = "cursor-" + credentialCanary + cfg.Server.URL = "http://root:url" + credentialCanary + "@127.0.0.1:8529" + return cfg + } + + renders := map[string]func(Config) string{ + "%v": func(c Config) string { return fmt.Sprintf("%v", c) }, + "%s": func(c Config) string { return fmt.Sprintf("%s", c) }, + "%+v": func(c Config) string { return fmt.Sprintf("%+v", c) }, + "Sprint": func(c Config) string { return fmt.Sprint(c) }, + "errorf": func(c Config) string { return fmt.Errorf("startup config: %w", fmt.Errorf("%v", c)).Error() }, + } + for verb, render := range renders { + out := render(renderedConfig()) + if strings.Contains(out, credentialCanary) { + t.Fatalf("%s rendering leaked credential: %s", verb, out) + } + if strings.Contains(out, "postgres://") || strings.Contains(out, "root:") { + t.Fatalf("%s rendering leaked DSN or URL credentials: %s", verb, out) + } + if !strings.Contains(out, redactedConfigSecret) { + t.Fatalf("%s rendering lost redaction marker: %s", verb, out) + } + } + + for _, mutate := range []func(*Config){ + func(c *Config) { c.Server.PostgreSQL.Host = "" }, + func(c *Config) { c.Server.PostgreSQL.Database = "" }, + func(c *Config) { c.Server.PostgreSQL.User = "" }, + func(c *Config) { c.Server.PostgreSQL.Port = 0 }, + func(c *Config) { c.Server.PostgreSQL.SSLMode = "needed" }, + func(c *Config) { c.Server.PostgreSQL.MaxOpenConns = 0 }, + func(c *Config) { c.Server.PostgreSQL.MaxIdleConns = 99 }, + func(c *Config) { c.Server.PostgreSQL.ConnMaxLifetime = -time.Second }, + func(c *Config) { c.Server.URL = "http://arango:8529" }, + func(c *Config) { c.Server.Database = "legacy_graph" }, + } { + cfg := postgresReadyConfig() + cfg.Server.PostgreSQL.Password = pgPassword + mutate(&cfg) + err := cfg.Validate() + if err == nil { + t.Fatalf("expected validation failure for a postgres contract mutation") + } + if strings.Contains(err.Error(), credentialCanary) || strings.Contains(err.Error(), "postgres://") { + t.Fatalf("validation error leaked credential: %v", err) + } + } + + if _, err := parseServerOptions([]string{"--no-auth", "--dataframer-recipe", "recipe.json", "--backend", "postgres", "--pg-host", "db.internal", "--pg-database", "loom_pg", "--pg-password", pgPassword}, flag.ContinueOnError); err == nil || !strings.Contains(err.Error(), "server.postgresql.user") { + t.Fatalf("incomplete postgres CLI error = %v", err) + } else if strings.Contains(err.Error(), credentialCanary) || strings.Contains(err.Error(), "postgres://") { + t.Fatalf("CLI validation error leaked credential: %v", err) + } + + path := writeServerConfigFile(t, "server:\n backend: postgres\n postgresql:\n host: db.internal\n database: loom_pg\n user: loom\n password: "+pgPassword+"\n typo_field: true\n") + if _, err := LoadConfig(path); err == nil { + t.Fatal("unknown postgres field with password unexpectedly accepted") + } else if strings.Contains(err.Error(), credentialCanary) || strings.Contains(err.Error(), "postgres://") { + t.Fatalf("YAML decode error leaked credential: %v", err) + } +} diff --git a/internal/server/construction_inputs_catalog.go b/internal/server/construction_inputs_catalog.go index 6d4485a2c..0e15fd3c8 100644 --- a/internal/server/construction_inputs_catalog.go +++ b/internal/server/construction_inputs_catalog.go @@ -164,14 +164,51 @@ func (c constructionInputsCatalog) list(ctx context.Context, project, explorerID return response, nil } + candidates, err := c.collectConstructionInputCandidates(ctx, project, snapshot.Identity.Generation, query, scope) + if err != nil { + return constructionInputsResponse{}, err + } + pageCandidates := make([]constructionInputCandidate, 0, limit) + for _, candidate := range candidates { + if cursor != nil && !constructionInputAfterCursor(candidate.entry, *cursor) { + continue + } + pageCandidates = append(pageCandidates, candidate) + if len(pageCandidates) > limit { + break + } + } + hasMore := len(pageCandidates) > limit + if hasMore { + pageCandidates = pageCandidates[:limit] + } + if len(pageCandidates) == 0 { + return response, nil + } + page, err := c.resolveConstructionInputPage(ctx, pageCandidates, project, snapshot.Identity.Generation, scope) + if err != nil { + return constructionInputsResponse{}, err + } + if hasMore { + last := page[len(page)-1] + response.NextCursor, err = encodeConstructionInputsCursor(query, request.SnapshotToken, request.ExpectedDraftVersion, request.ExpectedDraftDigest, last) + if err != nil { + return constructionInputsResponse{}, fmt.Errorf("encode construction input catalog cursor: %w", err) + } + } + response.Entries = page + return response, nil +} + +func (c constructionInputsCatalog) collectConstructionInputCandidates(ctx context.Context, project, generation, query string, scope authscope.ReadScope) ([]constructionInputCandidate, error) { executions, err := c.catalog.ListExecutions(ctx, publication.BundlePublished, time.Now().UTC().Add(time.Second)) if err != nil { - return constructionInputsResponse{}, constructionInputsUnavailable(err) + return nil, constructionInputsUnavailable(err) } references := make([]constructionInputReference, 0) for _, execution := range executions { execution = execution.CanonicalizeLegacy() - if !execution.State.Successful() || projectid.Canonical(execution.Project) != project || execution.DatasetGeneration != snapshot.Identity.Generation || strings.TrimSpace(execution.ID) == "" { + if !execution.State.Successful() || projectid.Canonical(execution.Project) != project || execution.DatasetGeneration != generation || strings.TrimSpace(execution.ID) == "" { continue } for _, output := range execution.Outputs { @@ -189,7 +226,7 @@ func (c constructionInputsCatalog) list(ctx context.Context, project, explorerID } } if len(references) == 0 { - return response, nil + return []constructionInputCandidate{}, nil } candidates := make([]constructionInputCandidate, 0, len(references)) @@ -197,7 +234,7 @@ func (c constructionInputsCatalog) list(ctx context.Context, project, explorerID publicationMetadataLoaded := make(map[string]bool) for _, reference := range references { materialization := constructionInputMaterialization(reference) - if err := validateConstructionInputMaterialization(materialization, reference, project, snapshot.Identity.Generation); err != nil { + if err := validateConstructionInputMaterialization(materialization, reference, project, generation); err != nil { continue } persistedScope, err := persistedMaterializationScope(materialization) @@ -216,7 +253,7 @@ func (c constructionInputsCatalog) list(ctx context.Context, project, explorerID if errors.Is(err, authscope.ErrForbidden) || errors.Is(err, authscope.ErrUnauthenticated) { continue } - return constructionInputsResponse{}, constructionInputsUnavailable(err) + return nil, constructionInputsUnavailable(err) } if effectiveScope.Mode == authscope.ReadScopeRestricted && len(effectiveScope.AuthResourcePaths) == 0 || !sameClickHouseReadScope(effectiveScope, scope) { continue @@ -228,14 +265,14 @@ func (c constructionInputsCatalog) list(ctx context.Context, project, explorerID if !publicationMetadataLoaded[metadataKey] { metadata, err := c.constructionInputPublicationMetadata(ctx, reference.execution, project) if err != nil { - return constructionInputsResponse{}, constructionInputsUnavailable(err) + return nil, constructionInputsUnavailable(err) } publicationMetadata[metadataKey] = metadata publicationMetadataLoaded[metadataKey] = true } presentation, err := constructionInputPresentationFor(materialization, publicationMetadata[metadataKey]) if err != nil { - return constructionInputsResponse{}, constructionInputsUnavailable(err) + return nil, constructionInputsUnavailable(err) } columns := constructionInputColumns(materialization.Columns, presentation.columnLabels) if len(columns) == 0 { @@ -261,23 +298,10 @@ func (c constructionInputsCatalog) list(ctx context.Context, project, explorerID } return leftEntry.RevisionID < rightEntry.RevisionID }) - pageCandidates := make([]constructionInputCandidate, 0, limit) - for _, candidate := range candidates { - if cursor != nil && !constructionInputAfterCursor(candidate.entry, *cursor) { - continue - } - pageCandidates = append(pageCandidates, candidate) - if len(pageCandidates) > limit { - break - } - } - hasMore := len(pageCandidates) > limit - if hasMore { - pageCandidates = pageCandidates[:limit] - } - if len(pageCandidates) == 0 { - return response, nil - } + return candidates, nil +} + +func (c constructionInputsCatalog) resolveConstructionInputPage(ctx context.Context, pageCandidates []constructionInputCandidate, project, generation string, scope authscope.ReadScope) ([]constructionInputEntry, error) { pagePins := make([]string, 0, len(pageCandidates)) seenPagePins := make(map[string]bool, len(pageCandidates)) for _, candidate := range pageCandidates { @@ -288,14 +312,14 @@ func (c constructionInputsCatalog) list(ctx context.Context, project, explorerID } } page := make([]constructionInputEntry, 0, len(pageCandidates)) - err = c.reader.WithExecutionReadPins(ctx, pagePins, func(scanCtx context.Context) error { + err := c.reader.WithExecutionReadPins(ctx, pagePins, func(scanCtx context.Context) error { for _, candidate := range pageCandidates { reference := candidate.reference materialization, err := c.reader.ExactExecutionMaterialization(scanCtx, reference.execution.ID, reference.output.Name) if err != nil { return fmt.Errorf("resolve exact published construction input %q/%q: %w", reference.execution.ID, reference.output.Name, err) } - if err := validateConstructionInputMaterialization(materialization, reference, project, snapshot.Identity.Generation); err != nil { + if err := validateConstructionInputMaterialization(materialization, reference, project, generation); err != nil { return fmt.Errorf("exact published construction input identity changed: %w", err) } persistedScope, err := persistedMaterializationScope(materialization) @@ -341,19 +365,11 @@ func (c constructionInputsCatalog) list(ctx context.Context, project, explorerID if err != nil { var authoringErr *explorer.AuthoringError if errors.As(err, &authoringErr) { - return constructionInputsResponse{}, err + return nil, err } - return constructionInputsResponse{}, constructionInputsUnavailable(err) + return nil, constructionInputsUnavailable(err) } - if hasMore { - last := page[len(page)-1] - response.NextCursor, err = encodeConstructionInputsCursor(query, request.SnapshotToken, request.ExpectedDraftVersion, request.ExpectedDraftDigest, last) - if err != nil { - return constructionInputsResponse{}, fmt.Errorf("encode construction input catalog cursor: %w", err) - } - } - response.Entries = page - return response, nil + return page, nil } func (c constructionInputsCatalog) authorizedSnapshot(ctx context.Context, project, explorerID string, request constructionInputsRequest) (*explorer.Explorer, capability.Snapshot, authscope.ReadScope, error) { diff --git a/internal/server/construction_proposal_zero_column_test.go b/internal/server/construction_proposal_zero_column_test.go index 0ef73f001..44a0edfb1 100644 --- a/internal/server/construction_proposal_zero_column_test.go +++ b/internal/server/construction_proposal_zero_column_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" @@ -115,8 +116,16 @@ func TestConstructionProposalFromZeroColumnPopulationAppliesRelatedExpansion(t * t.Fatal(err) } engine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, ScopeDigest: recipeScopeDigest, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, ScopeDigest: recipeScopeDigest, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, RootPageRows: 100, }) if err != nil { diff --git a/internal/server/core_authoring_protocol_arango_conformance_test.go b/internal/server/core_authoring_protocol_arango_conformance_test.go index 1ac029402..a67f85185 100644 --- a/internal/server/core_authoring_protocol_arango_conformance_test.go +++ b/internal/server/core_authoring_protocol_arango_conformance_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/calypr/loom/internal/dataframe/compiler" "net/http" "net/url" "os" @@ -14,27 +15,17 @@ import ( loomapi "github.com/calypr/loom/generated/loomapi" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" - explorerarango "github.com/calypr/loom/internal/explorer/arango" "github.com/calypr/loom/internal/explorer/authoringv2" "github.com/calypr/loom/internal/explorer/capability" "github.com/calypr/loom/internal/explorer/lifecycle" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/gofiber/fiber/v3" "github.com/google/uuid" ) -// TestCoreAuthoringProtocolAgainstArango is the joined authoring-protocol witness -// against the real Arango Explorer repository (plan C10/C12/C16/C17): every claim is -// read back from persisted Arango bytes through the repository (or a second -// independent client), never from a service echo or the in-memory fixture. It drives -// the same production generated strict adapter, handlers, lifecycle service and -// compile chain as newCoreProtocolFixture, and owns assertions the map-store -// witnesses cannot make: restart persistence through a fresh repository instance, -// compare-and-swap rejection against real Arango transaction state, and -// exactly-one-document integrity through a raw AQL count. func TestCoreAuthoringProtocolAgainstArango(t *testing.T) { endpoint := strings.TrimSpace(os.Getenv("LOOM_TEST_ARANGO_URL")) if endpoint == "" { @@ -44,10 +35,11 @@ func TestCoreAuthoringProtocolAgainstArango(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := storearango.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client t.Cleanup(func() { _ = client.Close(context.Background()) cleanupCtx, stop := context.WithTimeout(context.Background(), 5*time.Second) @@ -67,17 +59,14 @@ func TestCoreAuthoringProtocolAgainstArango(t *testing.T) { t.Errorf("remove owned disposable database status=%d", response.StatusCode) } }) - if err := client.Bootstrap(ctx, explorerarango.BootstrapSpec()); err != nil { + if err := client.Bootstrap(ctx, arangofactory.ExplorerBootstrapSpec()); err != nil { t.Fatal(err) } - store, err := explorerarango.New(client) + store, err := arangofactory.NewExplorerStore(client) if err != nil { t.Fatal(err) } - // Seed through the same store with the exact identity values used by the - // map-store fixture (core_authoring_protocol_test.go:126-149) so both witnesses - // describe the same protocol; the DraftConfig raw bytes are identical. snapshot := testAuthoringV2CapabilitySnapshot() workspace := authoringv2.Workspace{ APIVersion: authoringv2.APIVersion, Kind: authoringv2.WorkspaceKind, @@ -105,8 +94,15 @@ func TestCoreAuthoringProtocolAgainstArango(t *testing.T) { if err != nil { t.Fatal(err) } - queryRows := func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil } - engine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: queryRows, PreviewQueryRows: queryRows}) + queryRows := func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + } + engine, err := dataframeexecution.New(dataframeexecution.Config{CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: queryRows, PreviewSourceRows: queryRows}) if err != nil { t.Fatal(err) } @@ -143,9 +139,6 @@ func TestCoreAuthoringProtocolAgainstArango(t *testing.T) { registerGeneratedExplorerTestRoutes(app, authscope.AllowAllAuthorizer{}, func(context.Context, *authscope.Principal, string) error { return nil }, domain, serviceConfig) fixture := &coreProtocolFixture{t: t, app: app, snapshot: snapshot, workspace: workspace, initialDigest: digest, basePath: "/api/v1/projects/project-a/explorers/core-protocol/authoring/v2"} - // (17) Ack identity is what the Arango document actually carries. The map-store - // witness sees the same in-memory object the handler manipulated; this reads the - // committed document back through the Arango repository. saved, ack := fixture.saveLabel("core-arango-save-1", "Patient Identifier", 1, digest) if saved.StatusCode != http.StatusOK { t.Fatalf("commands status=%d body=%s", saved.StatusCode, saved.Body) @@ -174,7 +167,6 @@ func TestCoreAuthoringProtocolAgainstArango(t *testing.T) { t.Fatalf("persisted head points to previousDraftRevisionId=%q, acknowledged=%q", persisted.PreviousDraftRevisionID, *ack.PreviousDraftRevisionId) } - // The persisted bytes themselves round-trip to the exact post-command workspace. expectedPost := workspace expectedPost.SemanticsVersion = authoringv2.CurrentSemanticsVersion expectedPost.Documents[0].Columns[0].Label = "Patient Identifier" @@ -194,8 +186,6 @@ func TestCoreAuthoringProtocolAgainstArango(t *testing.T) { t.Fatalf("persisted Arango draft config differs from the expected post-command workspace\nexpected=%s\npersisted=%s", expectedRaw, persistedRaw) } - // The acknowledged previous revision id resolves to the exact pre-command head - // bytes in Arango (the revision id scheme lives in drafts.go; never hardcoded). prior, err := store.GetDraftRevision(ctx, "project-a", "core-protocol", *ack.PreviousDraftRevisionId) if err != nil { t.Fatalf("read acknowledged previous revision %s from Arango: %v", *ack.PreviousDraftRevisionId, err) @@ -218,11 +208,6 @@ func TestCoreAuthoringProtocolAgainstArango(t *testing.T) { t.Fatalf("previous revision bytes differ from the pre-command head\nexpected=%s\npersisted=%s", raw, priorRaw) } - // (16) The compile receipt is persisted in Arango and references the exact - // committed head identity. CompilationReceipt carries no draftVersion/draftDigest - // header field; the strongest identity binding the type actually has is the - // workspace content IntentDigest, the content-addressed ID, and admission - // through the store's validateReceipt on read — asserted below. compiled := fixture.decodeCompile(fixture.MustReconcile(ack.DraftVersion, ack.DraftDigest, "core-arango-reconcile")) persistedReceipt, err := store.GetCompilationReceiptForExplorer(ctx, "project-a", "core-protocol", compiled.ReceiptId) if err != nil { @@ -261,9 +246,6 @@ func TestCoreAuthoringProtocolAgainstArango(t *testing.T) { t.Fatalf("persisted receipt public contract label=%q want %q", receiptLabel, "Patient Identifier") } - // (16) Receipt-bound preview re-execution keeps the accepted identity: rows are - // the fixture's controlled empty array and the single contract label is the - // saved label (same expectations as TestCoreAuthoringProtocolVerticalWitness). previewed := fixture.preview(compiled.ReceiptId) if previewed.StatusCode != http.StatusOK { t.Fatalf("preview status=%d body=%s", previewed.StatusCode, previewed.Body) @@ -283,14 +265,13 @@ func TestCoreAuthoringProtocolAgainstArango(t *testing.T) { t.Fatalf("preview contract labels=%v want [Patient Identifier]", previewLabels) } - // (12) Restart persistence: a fresh Open + fresh repository + fresh service on - // the same database must reproduce the committed truth from persisted bytes. - restartClient, err := storearango.Open(ctx, endpoint, database) + restartBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatal(err) } + restartClient := restartBackend.Client t.Cleanup(func() { _ = restartClient.Close(context.Background()) }) - restartStore, err := explorerarango.New(restartClient) + restartStore, err := arangofactory.NewExplorerStore(restartClient) if err != nil { t.Fatal(err) } @@ -331,10 +312,6 @@ func TestCoreAuthoringProtocolAgainstArango(t *testing.T) { t.Fatalf("fresh service previous revision identity drifted: got version=%d digest=%q want version=1 digest=%q", restartedPrior.DraftVersion, restartedPrior.DraftDigest, digest) } - // (10) Compare-and-swap truth on real Arango: replaying a save with the - // pre-command identity must be refused at the store's transactional CAS, and - // the committed document must be unchanged (no partial write, no phantom - // second revision). stale, staleAck := fixture.saveLabel("core-arango-stale-save", "Patient Stale Ghost", 1, digest) if stale.StatusCode != http.StatusConflict || !strings.Contains(stale.Body, "DRAFT_CONFLICT") { t.Fatalf("stale save expected 409 DRAFT_CONFLICT, got status=%d body=%s ack=%+v", stale.StatusCode, stale.Body, staleAck) @@ -361,19 +338,14 @@ func TestCoreAuthoringProtocolAgainstArango(t *testing.T) { t.Fatalf("refused stale save wrote ghost bytes into the persisted draft") } - // (10) Independent-probe integrity: raw AQL counts through the second, exec-free - // client prove exactly one explorer document and exactly one draft revision for - // this identity — the rejected save and repeated compiles created no shadows. - if count := countDocuments(ctx, t, restartClient, explorerarango.ExplorersCollection, "project-a", "core-protocol"); count != 1 { + if count := countDocuments(ctx, t, restartClient, arangofactory.ExplorersCollection, "project-a", "core-protocol"); count != 1 { t.Fatalf("loom_explorers holds %d documents for project-a/core-protocol, want exactly 1", count) } - if count := countDocuments(ctx, t, restartClient, explorerarango.DraftRevisionsCollection, "project-a", "core-protocol"); count != 1 { + if count := countDocuments(ctx, t, restartClient, arangofactory.DraftRevisionsCollection, "project-a", "core-protocol"); count != 1 { t.Fatalf("loom_explorer_draft_revisions holds %d documents for project-a/core-protocol, want exactly 1 (the captured pre-command head)", count) } } -// digestForWorkspace is the committed-head digest the server itself reports for an -// expected post-command workspace. func digestForWorkspace(t *testing.T, workspace authoringv2.Workspace) string { t.Helper() value, err := workspace.Digest() @@ -383,9 +355,7 @@ func digestForWorkspace(t *testing.T, workspace authoringv2.Workspace) string { return value } -// countDocuments runs an exec-free raw AQL count through the given client for one -// (project, explorerId) identity. -func countDocuments(ctx context.Context, t *testing.T, client *storearango.Client, collection, project, explorerID string) int64 { +func countDocuments(ctx context.Context, t *testing.T, client *arangofactory.Client, collection, project, explorerID string) int64 { t.Helper() var count int64 err := client.QueryRows(ctx, `FOR d IN @@c FILTER d.project == @project AND d.explorerId == @explorerId COLLECT WITH COUNT INTO total RETURN {total: total}`, 1, map[string]any{"@c": collection, "project": project, "explorerId": explorerID}, func(row map[string]any) error { diff --git a/internal/server/core_authoring_protocol_formal_replay_test.go b/internal/server/core_authoring_protocol_formal_replay_test.go index 15158fb30..824c7cb11 100644 --- a/internal/server/core_authoring_protocol_formal_replay_test.go +++ b/internal/server/core_authoring_protocol_formal_replay_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/calypr/loom/internal/dataframe/compiler" "net/http" "net/url" "os" @@ -16,48 +17,21 @@ import ( loomapi "github.com/calypr/loom/generated/loomapi" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" - explorerarango "github.com/calypr/loom/internal/explorer/arango" "github.com/calypr/loom/internal/explorer/capability" "github.com/calypr/loom/internal/explorer/lifecycle" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/gofiber/fiber/v3" "github.com/google/uuid" ) -// Formal replay harness for the CoreAuthoringProtocol TLA model -// (verification/builder-interactions/CoreAuthoringProtocol.tla). The adapter -// scripts/lib/formal-core-protocol-replay.mjs schedules the exact server-side -// interleavings the model explores and validates the recorded wire observations -// against the model's invariants. This test never asserts on semantic anomalies: -// it records what the production handlers did and lets the model adjudicate. -// -// Plan C19 (required-store schedules): every schedule runs against the in-memory -// map store by default, and additionally against a real ArangoDB store owned by -// the test (same container-side pattern as TestCoreAuthoringProtocolAgainstArango) -// whenever LOOM_TEST_ARANGO_URL is set. Both modes execute one shared schedule -// body; the only store-coupled operations are the two persisted-state reads -// behind coreProtocolReplayStores, which go through each store's own read path -// in both modes (never a direct map iteration shared with Arango mode). The -// map-mode witness file keeps the exact adapter-pinned shape (schemaVersion + -// results with the registered observation keys, one row per case); when the -// Arango mode is active the witness additionally records the store mode per -// schedule because a single output file then carries both modes' rows. - const ( - coreProtocolReplayInputEnv = "LOOM_FORMAL_CORE_PROTOCOL_INPUT" - coreProtocolReplayOutputEnv = "LOOM_FORMAL_CORE_PROTOCOL_OUTPUT" - // The concrete-prefix lane (plan C14) runs the SAME production-boundary - // schedules with a concrete value-identity prefix A -> B -> A, where the - // model's abstract head channels merge the two A histories; the concrete - // wire record is what justifies that merge to scripts/lib/formal-core- - // protocol-replay.mjs instead of a prose justification. - coreProtocolConcreteInputEnv = "LOOM_FORMAL_CORE_PROTOCOL_CONCRETE_INPUT" - coreProtocolConcreteOutputEnv = "LOOM_FORMAL_CORE_PROTOCOL_CONCRETE_OUTPUT" - // coreProtocolReplayArangoURLEnv is the same owned-stack endpoint contract - // TestCoreAuthoringProtocolAgainstArango uses; absent means map-store mode. + coreProtocolReplayInputEnv = "LOOM_FORMAL_CORE_PROTOCOL_INPUT" + coreProtocolReplayOutputEnv = "LOOM_FORMAL_CORE_PROTOCOL_OUTPUT" + coreProtocolConcreteInputEnv = "LOOM_FORMAL_CORE_PROTOCOL_CONCRETE_INPUT" + coreProtocolConcreteOutputEnv = "LOOM_FORMAL_CORE_PROTOCOL_CONCRETE_OUTPUT" coreProtocolReplayArangoURLEnv = "LOOM_TEST_ARANGO_URL" coreProtocolReplayOwningStep = "Replay all generated schedules and population attachment cases" @@ -100,13 +74,6 @@ type coreProtocolReplayObservation struct { HeadLabel string `json:"headLabel"` Error string `json:"error"` - // Events is the ordered replay log: one entry per real call boundary of - // this case, recorded at the awaited call site in production call order. - // The adapter validates this stream against the model's legal transitions; - // a skipped entry CAS or a receipt persisted without capture cannot produce - // a legal log because each event is appended where the production handler - // boundary it names actually completes (or is proven to have completed via - // the existing beforeCompile/receipt-decode seams). Events []coreProtocolReplayEvent `json:"events"` reconcileReceiptID string @@ -139,23 +106,12 @@ func (ob *coreProtocolReplayObservation) event(name string, claims map[string]an return len(ob.Events) - 1 } -// coreProtocolAbsentHeadDigest stands in for a head identity that was never -// persisted (model schedule S5: reconcile of a head the save never committed). const coreProtocolAbsentHeadDigest = "sha256:formal-replay-absent-head-2-identity" -// coreProtocolConcreteProgram selects the concrete-prefix schedule. The plan's -// C14 row requires replaying the concrete A -> B -> A prefix whose two A -// histories the model's head channels merge; the program below drives exactly -// that prefix through the production handlers and records the wire evidence the -// validator needs (identity comparisons are adjudicated validator-side, never -// asserted here, so a semantically wrong server stays a red claim instead of -// aborting the witness). type coreProtocolConcreteProgram struct { ID string `json:"id"` } -// coreProtocolConcreteAttempt records one reconcile call against the -// production handler: the identity it presented and what the handler did. type coreProtocolConcreteAttempt struct { RequestID string `json:"requestId"` DraftVersion int64 `json:"draftVersion"` @@ -167,9 +123,6 @@ type coreProtocolConcreteAttempt struct { ReceiptID string `json:"receiptId,omitempty"` } -// coreProtocolConcreteRevision captures one acknowledged draft revision read -// back through the store seam, so value-equal heads (merged by the model) can -// be proven persisted as distinct revisions. type coreProtocolConcreteRevision struct { RevisionID string `json:"revisionId"` Found bool `json:"found"` @@ -177,18 +130,6 @@ type coreProtocolConcreteRevision struct { DraftDigest string `json:"draftDigest"` } -// coreProtocolConcreteObservation is the wire record of the concrete-prefix -// schedule. The merged-value evidence is value-level, not digest-level: the -// save path re-canonicalizes through MigrateLegacyContributors, -// MigrateLosslessDefaults and NormalizePresentationOrders -// (ApplyCommands, internal/explorer/authoringv2/commands.go), so a value-equal -// revert stores a distinct digest; the value identity that the model's head -// channels merge is witnessed by the emitted receipt label of head 3 equaling -// the seed column label while the three stored digests stay pairwise distinct. -// The record also carries the three stale rejections (both-stale plus the -// two single-coordinate probes isolating each entry CAS clause), the admitted -// fresh A, the replayed request, the preview bound to the persisted receipt, -// and the two acknowledged revisions. type coreProtocolConcreteObservation struct { ID string `json:"id"` SeedVersion int `json:"seedVersion"` @@ -233,32 +174,14 @@ func (ob *coreProtocolReplayObservation) note(format string, args ...any) { ob.Error = ob.notes[len(ob.notes)-1] } -// coreProtocolReplayStores is the single parameterization seam of the shared -// schedule body: the fixture under replay plus the two persisted-state reads -// the observation records. Map mode binds them to the test store's own read -// path; Arango mode binds them to the Arango repository's read path. The -// schedule logic itself has exactly one implementation. type coreProtocolReplayStores struct { - mode string - fixture *coreProtocolFixture - // receipt reads one persisted receipt identity through the store interface - // (map: testExplorerStore.GetCompilationReceiptForExplorer; arango: the - // repository read that admits receipts through validateReceipt), mirroring - // the previous direct fixture.store.receipts lookup without a map-only - // assumption. - receipt func(context.Context, string) (*explorer.CompilationReceipt, bool) - // hasReceipts reports whether the store holds any receipt for this case's - // explorer identity, replacing the direct len(fixture.store.receipts) read. + mode string + fixture *coreProtocolFixture + receipt func(context.Context, string) (*explorer.CompilationReceipt, bool) hasReceipts func(context.Context) bool - // history reads one acknowledged draft revision back through the store's - // own read path (map: testExplorerStore.GetDraftRevision; arango: the - // repository's GetDraftRevision), used by the concrete-prefix lane to - // prove value-equal heads persisted as distinct revisions. - history func(context.Context, string) (explorer.DraftRevision, bool) + history func(context.Context, string) (explorer.DraftRevision, bool) } -// coreProtocolReplayMapStores wires the schedule body to the in-memory map -// store fixture exactly as it ran before the required-store extension. func coreProtocolReplayMapStores(t *testing.T) coreProtocolReplayStores { t.Helper() fixture := newCoreProtocolFixture(t) @@ -284,23 +207,8 @@ func coreProtocolReplayMapStores(t *testing.T) coreProtocolReplayStores { } } -// coreProtocolReplayArangoStores wires the schedule body to the real Arango -// store. Fixture wiring is the exact same mount as newCoreProtocolFixture -// (fiber + generated strict adapter routes + authscope.AllowAllAuthorizer + -// no-op dataframe execution engine + domain through explorer.NewService), and -// the seed goes through store.Create with the identical workspace/snapshot -// bytes and the identical project-a/core-protocol identity: the workspace, -// capability snapshot and initial digest are taken from the map fixture -// itself, so both modes describe exactly the same protocol identity. Isolation -// between schedules comes from the per-schedule disposable database, not from -// identity mutation: the capability snapshot pins its identity project -// (compileExplorerReceipt rejects any request whose project does not canonicalize -// to the snapshot's, so identities must stay exactly at the map fixture's). -func coreProtocolReplayArangoStores(t *testing.T, client *storearango.Client, graph *explorerarango.Store, database string) coreProtocolReplayStores { +func coreProtocolReplayArangoStores(t *testing.T, client *arangofactory.Client, graph *arangofactory.ExplorerStore, database string) coreProtocolReplayStores { t.Helper() - // The map fixture is the single source for the seed bytes; constructing it - // and discarding its app/store costs one rebuild and guarantees the seeded - // DraftConfig, snapshot token, generation and digest equal map mode's. probe := newCoreProtocolFixture(t) workspace, snapshot, digest := probe.workspace, probe.snapshot, probe.initialDigest raw, err := workspace.CanonicalJSON() @@ -321,8 +229,15 @@ func coreProtocolReplayArangoStores(t *testing.T, client *storearango.Client, gr if err != nil { t.Fatal(err) } - queryRows := func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil } - engine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: queryRows, PreviewQueryRows: queryRows}) + queryRows := func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + } + engine, err := dataframeexecution.New(dataframeexecution.Config{CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: queryRows, PreviewSourceRows: queryRows}) if err != nil { t.Fatal(err) } @@ -366,7 +281,7 @@ func coreProtocolReplayArangoStores(t *testing.T, client *storearango.Client, gr registerGeneratedExplorerTestRoutes(app, authscope.AllowAllAuthorizer{}, func(context.Context, *authscope.Principal, string) error { return nil }, domain, serviceConfig) fixture.app = app t.Logf("core protocol replay store=arango: disposable database=%s project=%s explorerId=%s (identity bytes identical to map mode; isolation comes from the per-schedule database)", database, coreProtocolReplayProject, coreProtocolReplayExplorerID) - t.Log("store=arango documented divergence: draft revision ids are sha256-derived by the Arango store (internal/explorer/arango/drafts.go via key() in store.go), not the map store's draft_revision_core-protocol_N literal; the replay body never hardcodes a revision id, so both modes drive head identity solely from the save acknowledgement and stored bytes") + t.Log("store=arango documented divergence: draft revision ids are sha256-derived by the Arango store (internal/backends/arango/internal/explorer/drafts.go via key() in store.go), not the map store's draft_revision_core-protocol_N literal; the replay body never hardcodes a revision id, so both modes drive head identity solely from the save acknowledgement and stored bytes") return coreProtocolReplayStores{ mode: coreProtocolReplayStoreNameArango, fixture: fixture, @@ -382,7 +297,7 @@ func coreProtocolReplayArangoStores(t *testing.T, client *storearango.Client, gr hasReceipts: func(context.Context) bool { readCtx, readCancel := context.WithTimeout(context.Background(), 30*time.Second) defer readCancel() - return countDocuments(readCtx, t, client, explorerarango.CompilationReceiptsCollection, coreProtocolReplayProject, coreProtocolReplayExplorerID) > 0 + return countDocuments(readCtx, t, client, arangofactory.CompilationReceiptsCollection, coreProtocolReplayProject, coreProtocolReplayExplorerID) > 0 }, history: func(_ context.Context, revisionID string) (explorer.DraftRevision, bool) { readCtx, readCancel := context.WithTimeout(context.Background(), 30*time.Second) @@ -396,9 +311,6 @@ func coreProtocolReplayArangoStores(t *testing.T, client *storearango.Client, gr } } -// countCompilationReceipts answers the Observe hasReceipts claim through the -// test store's own lock, mirroring how the Arango mode counts receipt documents -// through a raw AQL read instead of reaching into map internals. func (s *testExplorerStore) countCompilationReceipts() int { s.mu.Lock() defer s.mu.Unlock() @@ -433,7 +345,6 @@ func runCoreProtocolReplayCase(t *testing.T, replayCase coreProtocolReplayCase, commitSave := func() { expectedVersion := int64(1) if replayCase.STarget == 2 { - // A stale client believes the head is already 2; the server CAS must reject. expectedVersion = 2 } saved, ack := fixture.saveLabel("formal-save-"+replayCase.ID, "Patient Competing B", expectedVersion, fixture.initialDigest) @@ -456,10 +367,6 @@ func runCoreProtocolReplayCase(t *testing.T, replayCase coreProtocolReplayCase, if replayCase.SavePosition == "before-entry" { commitSave() } - // The reconcile click targets exactly the model-captured head identity: - // head 1 is the seeded revision, head 2 exists only once a save committed it - // (before-entry) and its digest is then only knowable from the acknowledgement. - // Reconciling an uncommitted head 2 uses a stand-in digest nobody ever stored. if replayCase.RTarget == 2 { reconcileVersion = 2 switch { @@ -469,7 +376,6 @@ func runCoreProtocolReplayCase(t *testing.T, replayCase coreProtocolReplayCase, reconcileDigest = coreProtocolAbsentHeadDigest } } - // Model schedules where reconcile targets a head no save committed (S5). if replayCase.RTarget == 2 && replayCase.SavePosition != "before-entry" { reconcileVersion, reconcileDigest = 2, coreProtocolAbsentHeadDigest } @@ -487,11 +393,6 @@ func runCoreProtocolReplayCase(t *testing.T, replayCase coreProtocolReplayCase, }() select { case <-compileEntered: - // The existing compile seam only fires after the production entry CAS - // validated the requested identity and the handler decoded (captured) - // the saved workspace (authoring.go:552-564), so entry acceptance and - // capture have really completed at this awaited boundary. The captured - // head claim is filled from the persisted receipt evidence below. entryEventIndex = observation.event(coreProtocolActionEntryAck, map[string]any{"captured": 0}) observation.event(coreProtocolActionCapture, map[string]any{}) case <-time.After(15 * time.Second): @@ -522,8 +423,6 @@ func runCoreProtocolReplayCase(t *testing.T, replayCase coreProtocolReplayCase, } else { observation.reconcileReceiptID = compiled.ReceiptId observation.ReceiptReported = true - // Persisted-receipt evidence is read through the replay store seam - // (store interface in both modes), never a map-store internals read. if receipt, ok := stores.receipt(context.Background(), compiled.ReceiptId); ok { observation.ReceiptEmitted = true for _, column := range receipt.EmittedColumns { @@ -536,11 +435,6 @@ func runCoreProtocolReplayCase(t *testing.T, replayCase coreProtocolReplayCase, } } } - // The reconcile entered and captured for real (a 200 with a receipt id only - // follows the accepted entry CAS, decode and compile chain); claim the - // captured head from the persisted receipt's own label evidence, never from - // the schedule. An unbacked capture stays claimed 0 so the validator rejects - // it instead of trusting a completed-looking response. capturedClaim := 0 if observation.ReceiptEmitted && observation.ReceiptLabel != "" { capturedClaim = coreProtocolHeadForLabel(observation.ReceiptLabel) @@ -647,27 +541,19 @@ func observeReconcileConflictBody(t *testing.T, observation *coreProtocolReplayO } } -// runCoreProtocolReplayAgainstArango owns the container-side Arango pattern of -// TestCoreAuthoringProtocolAgainstArango: endpoint from LOOM_TEST_ARANGO_URL, a -// per-test disposable database auto-created by storearango.Open, t.Cleanup that -// closes with a fresh context and then raw-DELETEs the database (t.Error on -// failure, still attempting the drop), Bootstrap + explorerarango.New, and the -// schedules S1-S8 executed through the SAME shared replay body as map mode. func runCoreProtocolReplayAgainstArango(t *testing.T, replayCases []coreProtocolReplayCase, endpoint string, observations *[]coreProtocolReplayObservation) { t.Helper() for _, replayCase := range replayCases { replayCase := replayCase t.Run(replayCase.ID, func(t *testing.T) { - // Every schedule gets its own auto-created disposable database, so a - // real Arango run replays each case against exactly the state the map - // mode's fresh per-case store holds. database := coreProtocolReplayDatabasePrefix + strings.ReplaceAll(uuid.NewString(), "-", "") ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := storearango.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client t.Cleanup(func() { _ = client.Close(context.Background()) cleanupCtx, stop := context.WithTimeout(context.Background(), 5*time.Second) @@ -687,10 +573,10 @@ func runCoreProtocolReplayAgainstArango(t *testing.T, replayCases []coreProtocol t.Errorf("remove owned disposable database status=%d", response.StatusCode) } }) - if err := client.Bootstrap(ctx, explorerarango.BootstrapSpec()); err != nil { + if err := client.Bootstrap(ctx, arangofactory.ExplorerBootstrapSpec()); err != nil { t.Fatal(err) } - graph, err := explorerarango.New(client) + graph, err := arangofactory.NewExplorerStore(client) if err != nil { t.Fatal(err) } @@ -699,11 +585,6 @@ func runCoreProtocolReplayAgainstArango(t *testing.T, replayCases []coreProtocol } } -// coreProtocolReplayWitnessRow is only used for witness files that carry more -// than one store mode: the map-only record keeps the exact adapter-pinned -// observation shape (formal-core-protocol-replay.mjs checkCoreProtocolResults -// pins the key set and one row per case), while a dual-mode record tags every -// per-schedule row with the store that produced it. type coreProtocolReplayWitnessRow struct { coreProtocolReplayObservation Store string `json:"store"` @@ -783,10 +664,6 @@ func TestCoreAuthoringProtocolModelReplay(t *testing.T) { } } -// derefRevisionID reads one optional acknowledged revision identity. Revision -// addressing is always through the identity the server itself acknowledged -// (previousDraftRevisionId), never a constructed literal, so the same wire -// evidence holds in map and Arango modes where revision ids differ. func derefRevisionID(value *string) string { if value == nil { return "" @@ -794,15 +671,6 @@ func derefRevisionID(value *string) string { return *value } -// runCoreProtocolReplayConcreteCase drives the plan-C14 value-identity prefix -// A -> B -> A (save-to-B, value-equal revert back to A) through the production -// handlers on one store seam, then attempts the merged-value schedules: a -// stale reconcile presenting the head-1 identity while the server head is the -// value-equal head 3, a fresh reconcile at the real head identity, a -// same-request replay, the preview bound to the persisted receipt, and the -// read-back of both acknowledged revisions. It records wire facts only; every -// identity comparison is adjudicated validator-side so a semantically wrong -// server produces a red claim instead of aborting the witness. func runCoreProtocolReplayConcreteCase(program coreProtocolConcreteProgram, stores coreProtocolReplayStores) coreProtocolConcreteObservation { fixture := stores.fixture observation := coreProtocolConcreteObservation{ID: program.ID} @@ -814,9 +682,6 @@ func runCoreProtocolReplayConcreteCase(program coreProtocolConcreteProgram, stor } _ = headResponse - // A -> B: the first real save, CASed on the seeded identity. The digest is - // sha256 of canonical workspace JSON (internal/explorer/authoringv2/canonical.go), - // so B carries a distinct digest. saveB, ackB := fixture.saveLabel("formal-c14-"+program.ID+"-b", coreProtocolCompetingLabel, 1, fixture.initialDigest) observation.SaveBStatus = saveB.StatusCode observation.SaveBDigest = ackB.DraftDigest @@ -824,16 +689,6 @@ func runCoreProtocolReplayConcreteCase(program coreProtocolConcreteProgram, stor observation.note("save B rejected: status=%d ack=%+v body=%s", saveB.StatusCode, ackB, saveB.Body) } - // B -> A: a fresh save of the ORIGINAL column value, so head 3 and head 1 - // carry the same authored value (label 'Patient ID'). Whether the stored - // digest of that revert equals the seed digest is a property of the save - // path's canonicalization, not of value identity: ApplyCommands migrates - // and normalizes before hashing, so the stored digests of the two - // value-equal heads legitimately differ (observed). mergeValueZone - // records the digest-level fact verbatim; the value-level identity the - // model merges is witnessed by the emitted receipt label (validLabel). - // The stale client below presents exactly the head-1 identity while the - // server head is version 3. saveA, ackA := fixture.saveLabel("formal-c14-"+program.ID+"-a", coreProtocolInitialLabel, 2, ackB.DraftDigest) observation.SaveAStatus = saveA.StatusCode observation.SaveAVersion = int(ackA.DraftVersion) @@ -843,18 +698,6 @@ func runCoreProtocolReplayConcreteCase(program coreProtocolConcreteProgram, stor } observation.MergeValueZone = saveA.StatusCode == http.StatusOK && ackA.DraftDigest == fixture.initialDigest - // Three stale identities, all attempted BEFORE any receipt exists, so the - // only production mechanism that can own the head for any of them is the - // reconcile entry CAS (authoring.go): the realistic both-stale value-equal - // probe (stored head-1 version and digest) plus the two single-coordinate - // probes that make the CAS's two clauses independently falsifiable. If a - // fault removes only the version clause the current-version/stale-digest - // probe is still rejected by digest and the stale-version/current-digest - // probe alone is admitted; removing only the digest clause isolates the - // other clause exactly. All other schedules keep their 409s under either - // single-clause fault (the replay S lane's rejected schedules present - // coreProtocolAbsentHeadDigest, and every completed identity matches on - // both coordinates), so each probe is the unique falsifier of its clause. staleProbe := func(version int64, digest, suffix string) (coreProtocolConcreteAttempt, string) { response, responseErr := fixture.reconcile(version, digest, "formal-c14-"+program.ID+"-"+suffix) if responseErr != nil { @@ -894,7 +737,6 @@ func runCoreProtocolReplayConcreteCase(program coreProtocolConcreteProgram, stor observation.Stale2Attempt, observation.Stale2Error = staleProbe(3, fixture.initialDigest, "stale2") observation.Stale3Attempt, observation.Stale3Error = staleProbe(1, ackA.DraftDigest, "stale3") - // Valid fresh-A reconcile at the real head 3 identity. valid, validErr := fixture.reconcile(3, ackA.DraftDigest, "formal-c14-"+program.ID+"-valid") if validErr != nil { valid = testHTTPResponse{StatusCode: -1, Body: validErr.Error()} @@ -930,10 +772,6 @@ func runCoreProtocolReplayConcreteCase(program coreProtocolConcreteProgram, stor observation.note("fresh-A reconcile status=%d body=%s", valid.StatusCode, valid.Body) } - // Same-request replay of the fresh-A reconcile (idempotence boundary). For - // a digest-keyed receipt identity this must not produce a second head or a - // divergent head identity; the validator compares this attempt against the - // valid one. replay, replayErr := fixture.reconcile(3, ackA.DraftDigest, "formal-c14-"+program.ID+"-valid") if replayErr != nil { replay = testHTTPResponse{StatusCode: -1, Body: replayErr.Error()} @@ -956,7 +794,6 @@ func runCoreProtocolReplayConcreteCase(program coreProtocolConcreteProgram, stor } } - // Preview through the handler path bound to the persisted valid receipt. if observation.ValidAttempt.ReceiptID != "" { previewed := fixture.preview(observation.ValidAttempt.ReceiptID) observation.PreviewStatus = previewed.StatusCode @@ -975,13 +812,6 @@ func runCoreProtocolReplayConcreteCase(program coreProtocolConcreteProgram, stor } } - // Persisted-history seam: each save acknowledgement names the revision the - // store committed for the head it replaced (store.SaveDraft persists the - // prior identity then stamps it on the owner), so the two reads below - // recover the intermediate head-1/seed and head-2/B revisions through the - // store's own reader. Distinct persisted identities for value-equal and - // replaced heads are what justify the model merging the two A histories - // into one head channel without erasing the B history. for _, revisionID := range []string{derefRevisionID(ackB.PreviousDraftRevisionId), derefRevisionID(ackA.PreviousDraftRevisionId)} { if revisionID == "" { observation.note("an acknowledgement omitted previousDraftRevisionId") @@ -1002,19 +832,11 @@ func runCoreProtocolReplayConcreteCase(program coreProtocolConcreteProgram, stor return observation } -// coreProtocolConcreteWitnessRow tags one concrete-prefix observation with the -// store mode that produced it, mirroring coreProtocolReplayWitnessRow: a -// map-only run keeps the plain observation shape, a dual-mode run records the -// store per row in one output file. type coreProtocolConcreteWitnessRow struct { coreProtocolConcreteObservation Store string `json:"store"` } -// runCoreProtocolConcreteAgainstArango mirrors the owned-disposable-database -// pattern of runCoreProtocolReplayAgainstArango for the concrete-prefix lane: -// each program gets its own auto-created database so the seeded A -> B -> A -// prefix runs against exactly the fresh state the map mode provides. func runCoreProtocolConcreteAgainstArango(t *testing.T, programs []coreProtocolConcreteProgram, endpoint string, observations *[]coreProtocolConcreteObservation) { t.Helper() for _, program := range programs { @@ -1023,10 +845,11 @@ func runCoreProtocolConcreteAgainstArango(t *testing.T, programs []coreProtocolC database := coreProtocolReplayDatabasePrefix + strings.ReplaceAll(uuid.NewString(), "-", "") ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := storearango.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client t.Cleanup(func() { _ = client.Close(context.Background()) cleanupCtx, stop := context.WithTimeout(context.Background(), 5*time.Second) @@ -1046,10 +869,10 @@ func runCoreProtocolConcreteAgainstArango(t *testing.T, programs []coreProtocolC t.Errorf("remove owned disposable database status=%d", response.StatusCode) } }) - if err := client.Bootstrap(ctx, explorerarango.BootstrapSpec()); err != nil { + if err := client.Bootstrap(ctx, arangofactory.ExplorerBootstrapSpec()); err != nil { t.Fatal(err) } - graph, err := explorerarango.New(client) + graph, err := arangofactory.NewExplorerStore(client) if err != nil { t.Fatal(err) } @@ -1058,11 +881,6 @@ func runCoreProtocolConcreteAgainstArango(t *testing.T, programs []coreProtocolC } } -// TestCoreAuthoringProtocolConcretePrefixReplay is the C14 concrete-prefix -// lane: the adapter hands it the concrete programs and validates the recorded -// wire record, in addition to the abstract model schedules the S-lane entry -// records. It is skipped when neither concrete env pair is set (supporting -// unit runs keep their original shape) and fails when only half the pair is. func TestCoreAuthoringProtocolConcretePrefixReplay(t *testing.T) { inputPath := os.Getenv(coreProtocolConcreteInputEnv) outputPath := os.Getenv(coreProtocolConcreteOutputEnv) diff --git a/internal/server/core_authoring_protocol_receipts_test.go b/internal/server/core_authoring_protocol_receipts_test.go index 04001ac18..fe9de7ef7 100644 --- a/internal/server/core_authoring_protocol_receipts_test.go +++ b/internal/server/core_authoring_protocol_receipts_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/calypr/loom/internal/dataframe/compiler" "net/http" "strings" "testing" @@ -26,16 +27,6 @@ const ( c1011OutputID = "patients" ) -// coreProtocolReceiptsFixture is the fixture for the C10 receipt-binding matrix and -// the C11 historical/revoked witness (docs/verification/CORE_PROTOCOL_VERIFICATION_PLAN.md:379-380). -// It mirrors newCoreProtocolFixture's production wiring (generated strict adapter, -// real authoring handlers, lifecycle service, compile chain and persisted receipt -// store) with one deliberate difference: CapabilityResolver.ForExecution consults the -// repository-backed capability store through capability.Repository.GetByToken — the -// production execution-admission check — so revocation is caused by removing the -// retained snapshot from that store, not by a stubbed authorization result. Every -// request goes through the same public routes; the only test-side action besides -// those requests is persisted-state seeding through the exposed store handle. type coreProtocolReceiptsFixture struct { t *testing.T app *fiber.App @@ -84,8 +75,15 @@ func newCoreProtocolReceiptsFixture(t *testing.T) *coreProtocolReceiptsFixture { if err != nil { t.Fatal(err) } - queryRows := func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil } - engine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: queryRows, PreviewQueryRows: queryRows}) + queryRows := func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + } + engine, err := dataframeexecution.New(dataframeexecution.Config{CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: queryRows, PreviewSourceRows: queryRows}) if err != nil { t.Fatal(err) } @@ -132,9 +130,6 @@ func newCoreProtocolReceiptsFixture(t *testing.T) *coreProtocolReceiptsFixture { return fixture } -// saveLabel posts one public command through the production route at the fixture's -// tracked head and advances that head from the acknowledgement, exactly as the -// authoring protocol requires. func (f *coreProtocolReceiptsFixture) saveLabel(commandID, label string) (testHTTPResponse, loomapi.ApplyCommandsResponse) { f.t.Helper() edited := f.workspace @@ -157,9 +152,6 @@ func (f *coreProtocolReceiptsFixture) saveLabel(commandID, label string) (testHT return response, ack } -// compileAtHead reconciles the tracked head through the public route and returns the -// accepted compile payload plus the persisted receipt. Call it after saveLabel or, -// for idempotence checks, repeatedly at one unchanged head. func (f *coreProtocolReceiptsFixture) compileAtHead(requestID string) (loomapi.CompileResponse, explorer.CompilationReceipt) { f.t.Helper() body := fmt.Sprintf(`{"snapshotToken":%q,"draftVersion":%d,"draftDigest":%q}`, f.snapshot.Token, f.headVersion, f.headDigest) @@ -181,8 +173,6 @@ func (f *coreProtocolReceiptsFixture) compileAtHead(requestID string) (loomapi.C return compiled, receipt } -// compileFresh advances the head one revision with a distinct column label, then -// compiles and persists a receipt so each boundary case starts from a genuine receipt. func (f *coreProtocolReceiptsFixture) compileFresh(commandID, label string) (loomapi.CompileResponse, explorer.CompilationReceipt) { f.t.Helper() saved, ack := f.saveLabel(commandID, label) @@ -206,8 +196,6 @@ func (f *coreProtocolReceiptsFixture) previewOutput(receiptID, outputID string) return coreProtocolRequest(f.t, f.app, http.MethodPost, f.basePath+"/preview", body, "") } -// receiptEmittedLabel reads the persisted frozen contract, not the decoded response, -// so label assertions are independent of the public serializer. func (f *coreProtocolReceiptsFixture) receiptEmittedLabel(t *testing.T, receiptID, publicColumn string) string { t.Helper() receipt, ok := f.store.receipts[receiptID] @@ -223,8 +211,6 @@ func (f *coreProtocolReceiptsFixture) receiptEmittedLabel(t *testing.T, receiptI return "" } -// insertReceipt persists one receipt through the same store seam the compile chain -// uses and returns the stored value so later assertions read what the boundary reads. func (f *coreProtocolReceiptsFixture) insertReceipt(receipt explorer.CompilationReceipt) explorer.CompilationReceipt { f.t.Helper() stored, err := f.store.InsertCompilationReceipt(context.Background(), receipt) @@ -234,10 +220,6 @@ func (f *coreProtocolReceiptsFixture) insertReceipt(receipt explorer.Compilation return *stored } -// deleteAll unretains every snapshot through the same capability store fields the -// repository uses, so the retained-capability lookup that ForExecution consults stops -// resolving. This is the store mutation that stands in for losing a snapshot's -// authorization/retention, not a mocked authorization decision. func (s *testCapabilityStore) deleteAll() { s.byToken = make(map[string]capability.Snapshot) s.byIdentity = make(map[capability.SnapshotIdentity]string) @@ -277,15 +259,6 @@ func previewResponseColumns(t *testing.T, response testHTTPResponse) []string { return labels } -// TestCoreAuthoringProtocolReceiptBindingBoundaries implements plan row C10 -// (docs/verification/CORE_PROTOCOL_VERIFICATION_PLAN.md:379) for the server side. -// Each subtest returns a receipt whose response identity is correct except for ONE -// binding and asserts that the boundary the server actually runs rejects it as a -// typed failure: the scoped store lookup, the receipt route identity/integrity check, -// the execution-admission boundary or the output-contract check, as appropriate. The -// baseline and idempotence positives prove the same request succeeds when every -// binding matches and that equivalent identities stay accepted, so each rejection is -// attributable to the deliberately mismatched binding alone. func TestCoreAuthoringProtocolReceiptBindingBoundaries(t *testing.T) { baseline := newCoreProtocolReceiptsFixture(t) _, receipt := baseline.compileFresh("c10-baseline-save", "Baseline Patient") @@ -319,10 +292,6 @@ func TestCoreAuthoringProtocolReceiptBindingBoundaries(t *testing.T) { } }) - // Foreign project and foreign explorer are seeded as fully self-consistent - // receipts (compilation key and content id recomputed from the foreign scope), so - // the record is valid on its own terms and the only reason the boundary refuses it - // is the binding that was changed. GetCompilationReceipt proves presence. t.Run("foreign-project", func(t *testing.T) { fixture := newCoreProtocolReceiptsFixture(t) _, genuine := fixture.compileFresh("c10-project-save", "Project Binding") @@ -387,11 +356,6 @@ func TestCoreAuthoringProtocolReceiptBindingBoundaries(t *testing.T) { _, genuine := fixture.compileFresh("c10-"+binding.name+"-save", "Authorized Binding") receipt := genuine binding.tamper(&receipt, fixture.snapshot) - // Recompute key and id so the receipt stays internally consistent: the - // compile-integrity gate must pass and only the retained-capability - // comparison can reject it, otherwise the case would prove the wrong - // boundary. The genuine companion still previews while the same record - // with one changed capability binding does not. recomputeReceiptIdentity(t, &receipt, true) if err := receipt.Validate(); err != nil { t.Fatalf("forged %s receipt must remain internally valid so the capability binding is the only difference: %v", binding.name, err) @@ -434,19 +398,9 @@ func TestCoreAuthoringProtocolReceiptBindingBoundaries(t *testing.T) { _, genuine := fixture.compileFresh("c10-"+binding.name+"-save", "Integrity Binding") receipt := genuine binding.tamper(&receipt) - // Prove the specific integrity check inside Validate rejects this single - // binding change. The HTTP route deliberately withholds the reason and - // publishes one uniform recompile message (lifecycle/errors.go:33 never - // serialises the cause), so the specific cause can only be asserted against - // the domain error, never against the response envelope. if err := receipt.Validate(); err == nil || !strings.Contains(err.Error(), binding.cause) { t.Fatalf("tampered %s must fail Validate with a cause containing %q: err=%v", binding.name, binding.cause, err) } - // keepKey false leaves the compilation key stale against the tampered - // source so the key mismatch is the first failing check; keepKey true - // leaves the key honest and exercises the resolved-digest/contract gate - // inside Validate. Each tampered copy carries a fresh id so it lands in - // its own row instead of colliding with the genuine receipt. recomputeReceiptIdentity(t, &receipt, binding.keepKey) if _, err := fixture.store.GetCompilationReceipt(context.Background(), genuine.ID); err != nil { t.Fatalf("genuine companion missing: %v", err) @@ -463,11 +417,6 @@ func TestCoreAuthoringProtocolReceiptBindingBoundaries(t *testing.T) { } } -// recomputeReceiptIdentity refreshes the two content-addressed fields after a -// deliberate field change. The compilation key is recomputed when keepKey is true; -// the id is always recomputed from the mutated value so the tampered record occupies -// its own row while the intended integrity or capability check remains the one that -// rejects it. func recomputeReceiptIdentity(t *testing.T, receipt *explorer.CompilationReceipt, keepKey bool) { t.Helper() key, err := explorer.CompilationKey(*receipt) @@ -484,15 +433,6 @@ func recomputeReceiptIdentity(t *testing.T, receipt *explorer.CompilationReceipt receipt.ID = id } -// TestCoreAuthoringProtocolHistoricalAndRevokedReceipts implements plan row C11 -// (docs/verification/CORE_PROTOCOL_VERIFICATION_PLAN.md:380) for the server side. -// A validated frozen receipt keeps executing under its captured contract after the -// head advances, and both receipts fail as stale once their snapshot is removed from -// the capability store that the execution boundary consulted through -// capability.Repository.GetByToken. The restore step proves that retention, and not -// receipt age or label text, is what changed: it is the causal control for the -// revocation assertion. Whether an older receipt may be shown as current UI work is -// mounted-driver ownership and is reported as a gap rather than asserted here. func TestCoreAuthoringProtocolHistoricalAndRevokedReceipts(t *testing.T) { fixture := newCoreProtocolReceiptsFixture(t) _, historical := fixture.compileFresh("c11-historical-save", "Historical Patient") @@ -504,8 +444,6 @@ func TestCoreAuthoringProtocolHistoricalAndRevokedReceipts(t *testing.T) { t.Fatalf("persisted head disagrees with compiled head: %+v", head) } - // Historical execution succeeds while its own snapshot stays authorized and - // retained, and it still reports its frozen contract rather than the current draft. if labels := previewResponseColumns(t, fixture.preview(historical.ID)); len(labels) != 1 || labels[0] != "Historical Patient" { t.Fatalf("historical preview labels=%v want [Historical Patient]", labels) } diff --git a/internal/server/core_authoring_protocol_restore_test.go b/internal/server/core_authoring_protocol_restore_test.go index 44856e018..52a02bb6c 100644 --- a/internal/server/core_authoring_protocol_restore_test.go +++ b/internal/server/core_authoring_protocol_restore_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/calypr/loom/internal/dataframe/compiler" "net/http" "strings" "sync" @@ -21,23 +22,6 @@ import ( "github.com/gofiber/fiber/v3" ) -// This file closes the SERVER half of plan row C15 -// (docs/verification/CORE_PROTOCOL_VERIFICATION_PLAN.md:384): restore a previous -// revision while a command, reconcile, recovery GET or preview is pending, then -// complete the old work. Every claim is read back from the real contracts: -// -// - restore is an ordinary command through the same compare-and-swap boundary as -// any edit (lifecycle/draft_restore.go:26-46 owner/Undo-target checks, then -// explorer/service.go:324 SaveDraft), -// - reconcile admits on the head identity read at entry and afterwards compiles -// only that captured workspace (lifecycle/authoring.go:552-568), -// - preview admits on the retained receipt and never requires head equality -// (lifecycle/preview.go:28-44), -// - hydration reads version/digest/workspace from one persisted-state snapshot -// (lifecycle/query.go:101-135). -// -// Client-side completion ordering (the mounted-driver half of C15) is out of scope. - const ( coreProtocolRestoreOriginalLabel = "Patient ID" coreProtocolRestoreSavedLabel = "Patient Head A" @@ -46,8 +30,6 @@ const ( coreProtocolRestoreFollowUpLabel = "Patient Follow-Up After Restore" ) -// coreProtocolRevisionID mirrors the server-owned history identity the persisted -// store mints on every accepted draft CAS (http_test_helpers_test.go:156). func coreProtocolRevisionID(explorerID string, draftVersion int64) string { return fmt.Sprintf("draft_revision_%s_%d", explorerID, draftVersion) } @@ -77,17 +59,12 @@ func coreProtocolDecodeAck(t *testing.T, response testHTTPResponse) authoringv2. return ack } -// restoreRevision posts the production RESTORE_DRAFT_REVISION command with the exact -// CAS identity the Builder client sends. The acknowledgement is decoded leniently so -// a rejected attempt can still be asserted through its typed error envelope. func (f *coreProtocolFixture) restoreRevision(commandID, revisionID string, expectedVersion int64, expectedDigest string) (testHTTPResponse, authoringv2.ApplyCommandsResponse) { f.t.Helper() response := coreProtocolRequest(f.t, f.app, http.MethodPost, f.basePath+"/commands", coreProtocolRestoreBody(f, commandID, revisionID, expectedVersion, expectedDigest), "builder-command-"+commandID) return response, coreProtocolDecodeAck(f.t, response) } -// restoreAttempt is restoreRevision without the t.Fatal on transport failure, for -// calls that must run suspended in a goroutine. func (f *coreProtocolFixture) restoreAttempt(commandID, revisionID string, expectedVersion int64, expectedDigest string) (testHTTPResponse, authoringv2.ApplyCommandsResponse, error) { response, err := coreProtocolDo(f.app, http.MethodPost, f.basePath+"/commands", coreProtocolRestoreBody(f, commandID, revisionID, expectedVersion, expectedDigest), "builder-command-"+commandID) if err != nil || response.StatusCode != http.StatusOK { @@ -96,7 +73,6 @@ func (f *coreProtocolFixture) restoreAttempt(commandID, revisionID string, expec return response, coreProtocolDecodeAck(f.t, response), err } -// saveLabelAttempt is fixture.saveLabel's shape for goroutine-suspended calls. func (f *coreProtocolFixture) saveLabelAttempt(commandID, label string, expectedVersion int64, expectedDigest string) (testHTTPResponse, authoringv2.ApplyCommandsResponse, error) { body, err := coreProtocolSaveLabelBody(f, commandID, label, expectedVersion, expectedDigest) if err != nil { @@ -109,8 +85,6 @@ func (f *coreProtocolFixture) saveLabelAttempt(commandID, label string, expected return response, coreProtocolDecodeAck(f.t, response), nil } -// MustRestoreRevision restores the hydrated head's server-resolved Undo target and -// returns the accepted head identity. func (f *coreProtocolFixture) MustRestoreRevision(commandID string, hydrated loomapi.BuilderState) (int64, string, string) { f.t.Helper() if hydrated.PreviousDraftRevisionId == nil || *hydrated.PreviousDraftRevisionId == "" { @@ -127,15 +101,11 @@ func (f *coreProtocolFixture) MustRestoreRevision(commandID string, hydrated loo return ack.DraftVersion, ack.DraftDigest, revisionID } -// previewPending is fixture.preview's transport without the t.Fatal on transport -// error, so a preview can be suspended in a goroutine at the preview seam. func (f *coreProtocolFixture) previewPending(receiptID string) (testHTTPResponse, error) { body := fmt.Sprintf(`{"receiptId":%q,"outputId":"patients","limit":10}`, receiptID) return coreProtocolDo(f.app, http.MethodPost, f.basePath+"/preview", body, "") } -// coreProtocolSeedRestorableHead saves one accepted edit onto the fixture's first head -// and returns that saved head A's identity plus its server-resolved Undo target. func coreProtocolSeedRestorableHead(t *testing.T, fixture *coreProtocolFixture, commandID, label string) (int64, string, string) { t.Helper() _, hydrated := fixture.getBuilder() @@ -155,11 +125,6 @@ func coreProtocolSeedRestorableHead(t *testing.T, fixture *coreProtocolFixture, return ack.DraftVersion, ack.DraftDigest, *ack.PreviousDraftRevisionId } -// coreProtocolSeedDraftRevisionRecord persists a well-formed history record through the -// exposed store handle (the same seeding surface openapi_explorer_draft_restore_test.go -// uses) with the current head's source context, so a browser-supplied draftRevisionId -// that passes every identity/adjacency check but is NOT the head's recorded Undo target -// tests the Undo-ownership guard on its own. func coreProtocolSeedDraftRevisionRecord(t *testing.T, fixture *coreProtocolFixture, revisionID string, draftVersion int64, raw []byte, digest string) { t.Helper() head, err := fixture.store.Get(context.Background(), "project-a", "core-protocol") @@ -185,8 +150,6 @@ func coreProtocolStoredHead(t *testing.T, fixture *coreProtocolFixture) explorer return *current } -// coreProtocolExpectHead proves the persisted head still is the expected revision; it -// is how each case shows the completed or rejected stale work moved nothing. func coreProtocolExpectHead(t *testing.T, fixture *coreProtocolFixture, wantVersion int64, wantDigest, stage string) explorer.Explorer { t.Helper() stored := coreProtocolStoredHead(t, fixture) @@ -205,9 +168,6 @@ func coreProtocolAwait(t *testing.T, opened <-chan struct{}, what string) { } } -// coreProtocolDecodeHydratedState decodes the public Builder hydration envelope and -// re-derives the workspace digest, which is what lets the recovery-GET cases prove the -// announced version/digest pair belongs to the state actually returned. func coreProtocolDecodeHydratedState(t *testing.T, response testHTTPResponse, wantVersion int64, wantDigest, wantUndoTarget, wantLabel, stage string) { t.Helper() if response.StatusCode != http.StatusOK { @@ -244,11 +204,6 @@ func coreProtocolDecodeHydratedState(t *testing.T, response testHTTPResponse, wa } } -// coreProtocolSaveGate records and parks the persisted draft CAS of specific accepted -// commands so two writers can be provably inside the compare-and-swap window at the -// same time. It wraps the same store SaveDraft the production save path performs -// (explorer/service.go:324 → http_test_helpers_test.go:142); it never decides the CAS -// outcome, it only observes the attempts and releases parked calls on request. type coreProtocolSaveGate struct { mu sync.Mutex entries map[string]*coreProtocolSaveGateEntry @@ -308,8 +263,6 @@ func (g *coreProtocolSaveGate) releaseAll() { } } -// record logs only the compare-and-swap calls of the watched racing commands, so the -// witness can prove which attempt the store accepted and which it rejected. func (g *coreProtocolSaveGate) record(attempt coreProtocolSaveAttempt) { g.mu.Lock() defer g.mu.Unlock() @@ -353,14 +306,6 @@ func (s *coreProtocolGatedStore) SaveDraft(ctx context.Context, value explorer.E return stored, err } -// coreProtocolPendingFixture reuses the shared core-protocol fixture (same persisted -// store and seed, same wire helpers, same beforeCompile compile seam) and mounts the -// production route stack over that store with two extra test-owned scheduling -// observations, both at boundaries the fixture contract already declares test-owned: -// entry to PreviewReceipt execution (core_authoring_protocol_test.go:121-123) and -// entry to the persisted draft CAS. The compile/preview closures are the same -// compileExplorerReceipt and compileValidatedReceiptResolution+PreviewOutput calls the -// shared fixture installs, so no production behaviour is substituted. type coreProtocolPendingFixture struct { *coreProtocolFixture gate *coreProtocolSaveGate @@ -371,8 +316,15 @@ func newCoreProtocolPendingFixture(t *testing.T) *coreProtocolPendingFixture { t.Helper() base := newCoreProtocolFixture(t) pending := &coreProtocolPendingFixture{coreProtocolFixture: base, gate: newCoreProtocolSaveGate()} - queryRows := func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil } - engine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: queryRows, PreviewQueryRows: queryRows}) + queryRows := func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + } + engine, err := dataframeexecution.New(dataframeexecution.Config{CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: queryRows, PreviewSourceRows: queryRows}) if err != nil { t.Fatal(err) } @@ -421,19 +373,11 @@ func newCoreProtocolPendingFixture(t *testing.T) *coreProtocolPendingFixture { return pending } -// TestCoreAuthoringProtocolRestoreDuringPendingWork is the C15 server-contract -// witness. Each subtest parks one kind of pending work at a real boundary, commits a -// restore of an older revision while it is parked, then completes the parked work and -// reads back both persisted and public state. func TestCoreAuthoringProtocolRestoreDuringPendingWork(t *testing.T) { if label := newCoreProtocolFixture(t).workspace.Documents[0].Columns[0].Label; label != coreProtocolRestoreOriginalLabel { t.Fatalf("fixture baseline label=%q, this witness pins it as the restore target", label) } - // 1. A reconcile of captured head A stays parked inside the shared compile seam - // while a restore commits head R. The completed reconcile may only describe - // captured A: its returned workspace and its persisted receipt keep A's identity, - // the head stays R, and a fresh attempt at the obsolete reconcile is refused. t.Run("reconcile-pending-while-restore-commits", func(t *testing.T) { fixture := newCoreProtocolFixture(t) headVersion, headDigest, undoRevision := coreProtocolSeedRestorableHead(t, fixture, "c15-reconcile-save-a", coreProtocolRestoreSavedLabel) @@ -484,7 +428,6 @@ func TestCoreAuthoringProtocolRestoreDuringPendingWork(t *testing.T) { close(release) t.Fatalf("restored head still carries the obsolete saved intent: %s", headWhilePending.DraftConfig) } - // Hydration inside the same window already describes the restored head. coreProtocolDecodeHydratedState(t, coreProtocolRequest(t, fixture.app, http.MethodGet, fixture.basePath+"/builder", "", ""), wantRestoredVersion, ackRestore.DraftDigest, coreProtocolRevisionID("core-protocol", headVersion), coreProtocolRestoreOriginalLabel, "hydration during pending reconcile and pending restore") @@ -518,19 +461,12 @@ func TestCoreAuthoringProtocolRestoreDuringPendingWork(t *testing.T) { coreProtocolBoundaryFailure(t, stale, "fresh reconcile against the replaced head", http.StatusConflict, "DRAFT_CONFLICT", "reconcile", "") }) - // 2. After the restore commits R, admission follows the CAS/ownership contracts in - // both directions: obsolete intent is refused (obsolete-identity restore and - // obsolete-identity command), the committed restore replays without re-applying, - // and the next command binds to R's identity and advances the head. Only the - // current Undo target is restorable, so a later legitimate restore still works. t.Run("next-command-binds-to-restored-head", func(t *testing.T) { fixture := newCoreProtocolFixture(t) headVersion, headDigest, undoRevision := coreProtocolSeedRestorableHead(t, fixture, "c15-command-save-a", coreProtocolRestoreSavedLabel) _, hydratedBeforeRestore := fixture.getBuilder() restoredVersion, restoredDigest, _ := fixture.MustRestoreRevision("c15-command-restore-r", hydratedBeforeRestore) - // The bounded last-command replay record makes the committed restore an - // idempotent acknowledgement; a replayed restore must never re-apply. replay, ackReplay := fixture.restoreRevision("c15-command-restore-r", undoRevision, headVersion, headDigest) if replay.StatusCode != http.StatusOK { t.Fatalf("committed restore must replay as an idempotent acknowledgement: status=%d body=%s", replay.StatusCode, replay.Body) @@ -577,11 +513,6 @@ func TestCoreAuthoringProtocolRestoreDuringPendingWork(t *testing.T) { } coreProtocolExpectHead(t, fixture, ackFollowUp.DraftVersion, ackFollowUp.DraftDigest, "rejected stale Undo") - // The Undo-ownership guard on its own: a persisted history record whose - // version is exactly head-1 (so version adjacency would pass, and whose bytes - // and digest are identical to the legitimate Undo target) must still be refused - // because it is not the head's recorded Undo target. The positive control is the - // identical content restored through the real Undo target below. foreignRaw, foreignErr := fixture.workspace.CanonicalJSON() if foreignErr != nil { t.Fatal(foreignErr) @@ -607,11 +538,6 @@ func TestCoreAuthoringProtocolRestoreDuringPendingWork(t *testing.T) { redoneVersion, redoneDigest, coreProtocolRevisionID("core-protocol", ackFollowUp.DraftVersion), coreProtocolRestoreOriginalLabel, "restore after follow-up") }) - // 3. A restore commits while a preview for a retained receipt of A is parked at - // the execution boundary. The preview keeps executing per the immutable-receipt - // contract (lifecycle/preview.go:28-44 admits on the receipt, never on head - // equality) and its displayed server data stays A's frozen contract; afterwards a - // fresh reconcile of R produces a receipt bound to R. t.Run("preview-pending-while-restore-commits", func(t *testing.T) { fixture := newCoreProtocolPendingFixture(t) base := fixture.coreProtocolFixture @@ -693,11 +619,6 @@ func TestCoreAuthoringProtocolRestoreDuringPendingWork(t *testing.T) { } }) - // 4. A restore and a competing command save race at the store CAS. Both writers are - // provably parked inside the compare-and-swap window with the same expected - // identity, exactly one CAS is accepted, the loser's rejection comes from the CAS - // itself (recorded by the store as a rejected compare-and-swap for the pre-race - // identity), and head plus history revision stay coherent for either winner. t.Run("restore-races-command-save", func(t *testing.T) { for _, winner := range []string{"restore", "save"} { winner := winner @@ -783,13 +704,6 @@ func TestCoreAuthoringProtocolRestoreDuringPendingWork(t *testing.T) { if attempts[0].commandID != winnerCommandID || !attempts[0].acceptedByStoreCAS { t.Fatalf("first recorded CAS is not the accepted winner: %+v", attempts[0]) } - // Common origin: CAS semantics accept a writer only against the head it - // read, so two accepted writers would be legitimate only if the second - // were a different origin riding the first's result. That would be a - // chained/sequential dispatch masquerading as a race, so both recorded - // CAS attempts must compare-and-swap the single pre-race head identity - // (C15, docs/verification/CORE_PROTOCOL_VERIFICATION_PLAN.md:384: restore - // and subsequent admission follow the actual CAS/ownership contract). if attempts[0].expectedVersion != headVersion || attempts[0].expectedDigest != headDigest { t.Fatalf("winner did not race from the common pre-race head: %+v want version=%d digest=%s", attempts[0], headVersion, headDigest) } @@ -844,11 +758,6 @@ func TestCoreAuthoringProtocolRestoreDuringPendingWork(t *testing.T) { } }) - // 5. Recovery GET (Builder hydration) while the captured-A reconcile is still - // parked returns a coherent envelope, checked before the restore commits and again - // after, with the announced digest re-derived from the returned workspace so a - // mixed version/digest pair cannot pass. Completing the parked reconcile afterwards - // leaves head R coherent. t.Run("recovery-get-during-pending-reconcile", func(t *testing.T) { fixture := newCoreProtocolFixture(t) headVersion, headDigest, undoRevision := coreProtocolSeedRestorableHead(t, fixture, "c15-recovery-save-a", coreProtocolRestoreSavedLabel) diff --git a/internal/server/core_authoring_protocol_rowschema_test.go b/internal/server/core_authoring_protocol_rowschema_test.go index b63771a47..a7b8cbf5d 100644 --- a/internal/server/core_authoring_protocol_rowschema_test.go +++ b/internal/server/core_authoring_protocol_rowschema_test.go @@ -1,58 +1,10 @@ package server -// C19 — independent row/schema semantics (docs/verification/CORE_PROTOCOL_VERIFICATION_PLAN.md:69 -// family row "Independent row semantics | C19, M4 | Reuse literal fixtures and real -// Arango/ClickHouse execution for types, values, nulls, duplicates and cardinality after -// edit/restore", and the scenario-floor row at :388 "Run the same authored definitions -// through compile/receipt/preview and independently evaluate expected schema and rows. -// Include a saved edit and restore. | Exact public identities, types, cardinality, values, -// nulls and duplicates match the specified fixture. Expectations are not computed by the -// compiler implementation under test."). -// -// This file owns the row/schema SEMANTICS half of C19. The protocol-identity half of the -// same scenario is covered elsewhere in the family (identity/competing save in -// core_authoring_protocol_test.go, restore admission in core_authoring_protocol_restore_test.go, -// receipt bindings in core_authoring_protocol_receipts_test.go, required-store schedules in -// core_authoring_protocol_formal_replay_test.go:35-45), so nothing here re-asserts those. -// -// Independence rule: every expectation below is a hand-written literal in this file — -// the contract labels, the public column identities, the logical types, the row values, -// the expected row counts and the expected sampled flags. No expectation is read back from -// the compiled receipt, the compiled recipe, the engine plan or any other compiler output. -// The compiled output is only ever the value under test. The authored fixtures (capability -// candidates, workspace columns and the store rows) are also literals authored here, so the -// oracle never consults the implementation under test. -// -// Server-enforced versus observed-only, recorded honestly: -// - Enforced and asserted: preview identity/limit admission -// (internal/explorer/lifecycle/preview.go:13-21), the public-column projection that -// publishes only compiled public columns and only keys the store row actually carries -// (internal/dataframe/execution/engine.go:1316-1329), the row-visitor count that becomes -// the wire rowCount (internal/dataframe/execution/engine.go:1101-1122,1148 and -// internal/server/explorer_preview_response.go:233-286,305-311), truncation/sampled -// (internal/dataframe/execution/engine.go:1153-1156 and -// internal/server/openapi_explorer_authoring_direct.go:539) and the empty-array-not-null -// row shape (internal/server/explorer_preview_response.go:227-229,292). -// - Observed-only, recorded with t.Log and NOT asserted as a promise: rowCount has no -// independent second source (the wire field and the delivered rows come from the same -// visitor counter, so "rowCount vs actual rows" can only be checked as self-agreement); -// nothing de-duplicates rows or row identities; no negative/zero-limit validation exists -// at the route boundary even though openapi/openapi.yaml:6465 declares minimum 1 -// (explicit 0 is rewritten to -1 at internal/server/openapi_explorer_authoring_direct.go:504-510 -// and only the engine rejects it at internal/dataframe/execution/engine.go:1256-1261). -// - Compiler-influenced expectations: the four logical types and shapes asserted here are -// the logical types of the capability candidates THIS file authors, which the compiler is -// required to echo verbatim (internal/explorer/compilation/semantic_compile.go:137-142 -// rejects any mismatch with CAPABILITY_LOGICAL_TYPE_MISMATCH, and -// internal/explorer/compilation/semantic_compile.go:483 copies the candidate type into the -// emission). The shape/nullable literals are the fixed values that same emission records -// carry for a root-occurrence scalar field projection (ibid. :400, :483). They are echo -// guarantees of authored input, not values computed by the compiler, so they stay literal. - import ( "context" "encoding/json" "fmt" + "github.com/calypr/loom/internal/dataframe/compiler" "net/http" "os" "reflect" @@ -63,14 +15,13 @@ import ( loomapi "github.com/calypr/loom/generated/loomapi" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" - explorerarango "github.com/calypr/loom/internal/explorer/arango" "github.com/calypr/loom/internal/explorer/authoringv2" "github.com/calypr/loom/internal/explorer/capability" "github.com/calypr/loom/internal/explorer/lifecycle" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/gofiber/fiber/v3" "github.com/google/uuid" ) @@ -80,18 +31,11 @@ const ( c19ExplorerID = "core-protocol-c19" c19OutputID = "patients" - // Public column identities are the authored Column values below. The compiled contract - // must publish exactly these names (internal/explorer/compilation/semantic_compile.go:483 - // uses the authored Column as PublicColumn). Order is the canonicalized order the server - // commits at reconcile: presentation class, then presentation order, then public column - // name ascending (internal/explorer/authoringv2/canonical.go:95-107, applied by - // lifecycle/authoring.go:391), so the expected sequence below is alphabetical. c19ColumnPatient = "c_patient" c19ColumnActive = "c_active" c19ColumnBirthOrder = "c_birth_order" c19ColumnBirthDate = "c_birth_date" - // Contract labels as authored; the saved-edit cases change exactly one of them. c19LabelPatient = "Patient ID" c19LabelActive = "Patient active" c19LabelBirthOrder = "Birth order" @@ -101,8 +45,6 @@ const ( c19RowIdentityPrefix = "c19-row-" ) -// c19ContractWant is one hand-written expected preview schema. Every field is supplied as a -// literal at the call site; the type only documents that these are the four parallel lists. type c19ContractWant struct { columns []string labels []string @@ -111,10 +53,6 @@ type c19ContractWant struct { nullable []bool } -// c19StoredHead reads the persisted draft head for THIS fixture's project/explorer ids. -// The shared coreProtocolStoredHead helper at core_authoring_protocol_restore_test.go:179 -// hardcodes the "core-protocol" id, so this file reads its own id through the same store -// double instead of reusing that helper. func c19StoredHead(t *testing.T, fixture *coreProtocolFixture) explorer.Explorer { t.Helper() current, err := fixture.store.Get(context.Background(), c19Project, c19ExplorerID) @@ -124,7 +62,6 @@ func c19StoredHead(t *testing.T, fixture *coreProtocolFixture) explorer.Explorer return *current } -// c19ExpectHead proves the persisted head still is the expected revision after a command. func c19ExpectHead(t *testing.T, fixture *coreProtocolFixture, wantVersion int64, wantDigest, stage string) explorer.Explorer { t.Helper() stored := c19StoredHead(t, fixture) @@ -134,12 +71,6 @@ func c19ExpectHead(t *testing.T, fixture *coreProtocolFixture, wantVersion int64 return stored } -// c19ExpectRestoreHistory asserts the traced history semantics of a restore commit against -// whichever store the fixture runs on: the new head's pointer must be that store's snapshot -// of the pre-restore head (savedVersion/savedDigest and the edited label), while the undo -// target the command consumed must be its snapshot of the head the rows/schema were restored -// from (baselineVersion/baselineDigest and the original label). Revision ids differ per store -// (http_test_helpers_test.go:152 vs arango/drafts.go:139-141), so only content is asserted. func c19ExpectRestoreHistory(t *testing.T, storeName, pointer, undoTarget string, savedVersion int64, savedDigest, savedLabel string, baselineVersion int64, baselineDigest, baselineLabel string, @@ -180,8 +111,6 @@ func c19ExpectRestoreHistory(t *testing.T, storeName, pointer, undoTarget string t.Logf("C19 (:69/:388) %s: restore head pointer names the pre-restore snapshot (version %d, the redo slot) while the consumed undo target names the restored-from revision (version %d); pointer and undo target are distinct store-minted ids, matching internal/explorer/lifecycle/draft_restore.go:33/47 and the store write path", storeName, savedVersion, baselineVersion) } -// c19AuthoredLabel reads one authored label back out of a persisted workspace by public -// column name, so the read does not depend on the canonicalized column order. func c19AuthoredLabel(workspace authoringv2.Workspace, column string) string { for _, document := range workspace.Documents { for _, candidate := range document.Columns { @@ -193,10 +122,6 @@ func c19AuthoredLabel(workspace authoringv2.Workspace, column string) string { return "" } -// c19WantBaselineContract is the authored contract, unchanged by any command: the four -// scalar columns and the logical types this file's capability snapshot advertises, listed in -// the canonical publish order (alphabetical by public column name, per the comment on the -// column constants above). func c19WantBaselineContract() c19ContractWant { return c19ContractWant{ columns: []string{c19ColumnActive, c19ColumnBirthDate, c19ColumnBirthOrder, c19ColumnPatient}, @@ -207,22 +132,12 @@ func c19WantBaselineContract() c19ContractWant { } } -// c19WantEditedContract is the baseline with exactly one label replaced by the saved edit. func c19WantEditedContract() c19ContractWant { want := c19WantBaselineContract() want.labels = []string{c19LabelActive, c19LabelBirthDate, c19LabelBirthOrder, c19LabelPatientEdit} return want } -// c19CapabilitySnapshot is a test-owned literal snapshot with one scalar candidate per -// authored column. Each candidate's logical type is the FHIR terminal primitive kind of its -// field path on Patient, so no expectation depends on the compiler inventing a type: -// Patient.active is a boolean and Patient.multipleBirthInteger an integer and -// Patient.birthDate a date-formatted string in the generated definition -// (generated/fhirschema/generated.go:10946-10950 and the Patient block from :10874), which -// internal/fhir/schema/terminal_metadata.go:31-53 classifies as boolean/integer/date. -// A floating/decimal column is NOT exercised: the generated Patient schema has no decimal -// primitive terminal, so no honest fixture exists for it — recorded, not faked. func c19CapabilitySnapshot() capability.Snapshot { return capability.NewSnapshot( capability.SnapshotIdentity{ @@ -251,10 +166,6 @@ func c19CapabilitySnapshot() capability.Snapshot { ) } -// c19Workspace is the authored contract under test: one records-row output over Patient with -// the four scalar columns. The authored order here is not the publish order: reconcile -// canonicalizes the column list (internal/explorer/authoringv2/canonical.go:95-107), and the -// expectations below state that canonical order explicitly. func c19Workspace(t *testing.T, snapshot capability.Snapshot) authoringv2.Workspace { t.Helper() raw := c19AuthoredWorkspace() @@ -265,19 +176,6 @@ func c19Workspace(t *testing.T, snapshot capability.Snapshot) authoringv2.Worksp return prepared } -// c19AuthoredWorkspace is the literal authored contract. It is deliberately NOT what the -// fixture seeds: a first head created outside the command surface must already be in the -// compilation-prepared form that every command save produces (authoring v2 PrepareWorkspace- -// ForCompilation = MigrateLegacyContributors + MigrateLosslessDefaults + -// NormalizePresentationOrders, internal/explorer/authoringv2/migration.go:88-99, applied as a -// pair in internal/server/construction_proposal_zero_column_test.go:71-75). Seeding raw bytes -// instead leaves SemanticsVersion unset, so the compile prepare fills defaults the stored -// bytes never got and the receipt intent (digest of the prepared form, -// internal/server/explorer_receipt_contract.go:142) legitimately diverges from the stored CAS -// digests — observed in a capture at -// /tmp/opencode/loom-core-protocol/evidence/c19/run-08-arango-localizer.log (intent -// sha256:5a83... vs CAS sha256:af5a..., response identical to the persisted Arango doc, so -// NOT a store-parity finding). func c19AuthoredWorkspace() authoringv2.Workspace { scalar := func(path string) *authoringv2.FieldSource { return &authoringv2.FieldSource{Path: path, ProjectionMode: "VALUE"} @@ -301,10 +199,6 @@ func c19AuthoredWorkspace() authoringv2.Workspace { } } -// c19RowStore is the controlled execution boundary that stands in for the dataset backend. -// It hands the engine exactly the literal rows a test sets, in order, and remembers the -// batch size each call asked for. It never reads anything from the compiled plan, so it -// cannot turn a compiler output into its own oracle. type c19RowStore struct { mu sync.Mutex rows []map[string]any @@ -326,17 +220,16 @@ func (r *c19RowStore) observations() (int, []int) { return r.calls, append([]int(nil), r.batches...) } -// previewRows is the engine's PreviewQueryRows seam. Each row is deep-copied before the -// visitor sees it, because the engine writes the stable row identity into the map it is -// given (internal/dataframe/execution/engine.go:1754-1783); the authored fixture must stay -// unchanged so one row set can drive several previews. A visitor error (the engine signals -// its limit that way, internal/dataframe/execution/engine.go:1098-1101) stops the walk -// immediately, like a real backend cursor would. -func (r *c19RowStore) previewRows() dataframeexecution.QueryRows { - return func(_ context.Context, _ string, batch int, _ map[string]any, visit func(map[string]any) error) error { +func (r *c19RowStore) previewRows() dataframeexecution.SourceRows { + return func(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + r.mu.Lock() r.calls++ - r.batches = append(r.batches, batch) + r.batches = append(r.batches, request.BatchSize) pending := append([]map[string]any(nil), r.rows...) r.mu.Unlock() for _, row := range pending { @@ -356,11 +249,6 @@ func cloneC19Row(row map[string]any) map[string]any { return clone } -// c19Row is a store row with a stable identity already carried, which the execution boundary -// explicitly accepts as a store-supplied identity -// (internal/dataframe/execution/engine.go:1758-1760). Row-identity GENERATION is not a C19 -// claim; identity preservation through the public boundary is asserted where it matters -// (duplicate rows keep one identical identity). func c19Row(identity string, values map[string]any) map[string]any { row := map[string]any{"__loom_row_id": c19RowIdentityPrefix + identity} for key, value := range values { @@ -369,12 +257,6 @@ func c19Row(identity string, values map[string]any) map[string]any { return row } -// c19NewFixture wires the production generated strict adapter, real authoring handlers, -// lifecycle service, compile chain and persisted receipt store exactly as -// newCoreProtocolFixture does (core_authoring_protocol_test.go:124-196), with the one -// deliberate difference this row needs: the preview execution seam returns this test's -// literal rows instead of the empty set. Only the row/preview data source differs; every -// request still goes through the public routes. func c19NewFixture(t *testing.T) (*coreProtocolFixture, *c19RowStore) { t.Helper() snapshot := c19CapabilitySnapshot() @@ -401,10 +283,15 @@ func c19NewFixture(t *testing.T) (*coreProtocolFixture, *c19RowStore) { t.Fatal(err) } rows := &c19RowStore{} - // Compile-time row reads stay empty: C19 claims nothing about compile-time execution, - // and a non-empty read here would be indistinguishable from a preview read. - emptyRows := func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil } - engine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: emptyRows, PreviewQueryRows: rows.previewRows()}) + emptyRows := func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + } + engine, err := dataframeexecution.New(dataframeexecution.Config{CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: emptyRows, PreviewSourceRows: rows.previewRows()}) if err != nil { t.Fatal(err) } @@ -444,8 +331,6 @@ func c19NewFixture(t *testing.T) (*coreProtocolFixture, *c19RowStore) { return fixture, rows } -// c19Preview posts the public preview request with an explicit limit, or with the limit -// omitted when limit is zero, so the documented default is reachable. func c19Preview(t *testing.T, fixture *coreProtocolFixture, receiptID string, limit int) testHTTPResponse { t.Helper() body := fmt.Sprintf(`{"receiptId":%q,"outputId":%q}`, receiptID, c19OutputID) @@ -467,8 +352,6 @@ func c19DecodePreview(t *testing.T, response testHTTPResponse) loomapi.PreviewRe return preview } -// c19ExpectContract compares the published preview schema against a hand-written expected -// schema, field by field, including column order. func c19ExpectContract(t *testing.T, preview loomapi.PreviewResponse, want c19ContractWant) { t.Helper() if len(preview.Columns) != len(want.columns) { @@ -491,9 +374,6 @@ func c19ExpectContract(t *testing.T, preview loomapi.PreviewResponse, want c19Co } } -// c19ExpectRows compares the published rows against hand-written literal rows. Values are -// compared as the public JSON contract delivers them (JSON numbers decode to float64), and -// DeepEqual on the whole map makes a MISSING key distinguishable from an explicit null. func c19ExpectRows(t *testing.T, preview loomapi.PreviewResponse, body string, want []map[string]any, wantRowCount int, wantSampled bool) { t.Helper() if preview.RowCount != wantRowCount { @@ -518,8 +398,6 @@ func c19ExpectRows(t *testing.T, preview loomapi.PreviewResponse, body string, w } } -// c19TypedRow is the single fully-populated typed row: one string, one boolean, one integer -// and one date-shaped string, each a literal written here. func c19TypedRow() map[string]any { return c19Row("typed", map[string]any{ c19ColumnPatient: "patient-alpha", @@ -539,20 +417,8 @@ func c19TypedRowExpectation() map[string]any { } } -// TestCoreAuthoringProtocolRowSchemaSemantics is the C19 row/schema semantics witness -// (docs/verification/CORE_PROTOCOL_VERIFICATION_PLAN.md:69, :388) on the saved-draft path -// and the restore path, with the persisted-store variant opt-in through -// LOOM_TEST_ARANGO_URL exactly like TestCoreAuthoringProtocolAgainstArango -// (core_authoring_protocol_arango_conformance_test.go:38-42). func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { - // ------------------------------------------------------------------ - // Saved-draft path: one accepted edit (UPDATE_COLUMN on the output's - // column contract) then reconcile + receipt-bound preview. - // ------------------------------------------------------------------ - - // C19 "types ... values": the published schema carries the authored logical types in - // authored order and the delivered row carries each value with its own JSON type. t.Run("saved-edit-contract-and-typed-values", func(t *testing.T) { fixture, rows := c19NewFixture(t) _, hydrated := fixture.getBuilder() @@ -578,7 +444,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { c19ExpectContract(t, preview, c19WantEditedContract()) c19ExpectRows(t, preview, response.Body, []map[string]any{c19TypedRowExpectation()}, 1, false) - // The saved head really carries the edit, and the receipt describes that head. stored := c19StoredHead(t, fixture) if !strings.Contains(string(stored.DraftConfig), c19LabelPatientEdit) { t.Fatalf("persisted draft lacks the saved label: %s", stored.DraftConfig) @@ -588,8 +453,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { t.Fatalf("receipt intentDigest=%q does not name the saved head digest=%q", emitted.IntentDigest, ack.DraftDigest) } - // Wire-level type check: the integer stays a JSON number and the boolean stays a - // JSON boolean rather than becoming a string. for _, want := range []string{`"c_patient":"patient-alpha"`, `"c_active":true`, `"c_birth_order":2`, `"c_birth_date":"2001-02-03"`} { if !strings.Contains(response.Body, want) { t.Fatalf("preview body is missing the typed encoding %s: %s", want, response.Body) @@ -600,8 +463,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { } }) - // C19 "nulls": a key the store row omits, a key the store row sets to null, and a key - // set to the empty string are three distinguishable public outcomes. t.Run("saved-edit-null-handling", func(t *testing.T) { fixture, rows := c19NewFixture(t) _, hydrated := fixture.getBuilder() @@ -612,20 +473,17 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { compiled := fixture.decodeCompile(fixture.MustReconcile(ack.DraftVersion, ack.DraftDigest, "c19-nulls-reconcile")) rows.set([]map[string]any{ - // (a) birthDate key ABSENT from the store row; active explicitly null. c19Row("missing", map[string]any{ c19ColumnPatient: "patient-beta", c19ColumnActive: nil, c19ColumnBirthOrder: 1, }), - // (b) birthDate explicitly null. c19Row("explicit-null", map[string]any{ c19ColumnPatient: "patient-gamma", c19ColumnActive: true, c19ColumnBirthOrder: 3, c19ColumnBirthDate: nil, }), - // (c) birthDate present but the empty string, which is not null. c19Row("empty-string", map[string]any{ c19ColumnPatient: "patient-delta", c19ColumnActive: false, @@ -659,10 +517,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { }, }, 3, false) - // Missing-vs-null is a wire-visible distinction, asserted on the bytes too: a missing - // key publishes no member at all (internal/dataframe/execution/engine.go:1316-1321 - // copies only keys the store row carries), an explicit null stays an explicit JSON - // null, and an empty string stays an empty JSON string. if _, absent := preview.Rows[0][c19ColumnBirthDate]; absent { t.Fatalf("a row whose store value was absent published a %s member anyway: %s", c19ColumnBirthDate, response.Body) } @@ -676,17 +530,8 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { t.Fatalf("an empty-string date was not preserved as an empty string: %s", response.Body) } - // Observed-only, recorded not asserted: the schema still says nullable - // (internal/explorer/compilation/semantic_compile.go:483 records Nullable on the - // emission), but nothing in the preview path validates a published value against the - // published logical type. A row whose c_active value is a string would still be - // published; C19 therefore claims type FIDELITY for the values supplied, not type - // ENFORCEMENT against the contract, and this run supplies type-correct values. t.Log("C19 (:69/:388) observed-only: the server publishes store values without validating them against the published logicalType/nullable; fidelity is asserted, enforcement is not claimed") - // A row the store hands over with no public keys at all still becomes an object (the - // encoder only guarantees an object per visited row, - // internal/server/explorer_preview_response.go:237-239), never a null element. rows.set([]map[string]any{{"__loom_row_id": c19RowIdentityPrefix + "identity-only"}}) identityOnly := c19Preview(t, fixture, compiled.ReceiptId, 10) identityOnlyPreview := c19DecodePreview(t, identityOnly) @@ -698,8 +543,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { } }) - // C19 "duplicates": two identical store rows are published twice, byte-identical, and the - // server neither de-duplicates them nor assigns them distinct identities. t.Run("saved-edit-duplicate-rows", func(t *testing.T) { fixture, rows := c19NewFixture(t) _, hydrated := fixture.getBuilder() @@ -721,8 +564,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { t.Log("C19 (:69/:388) observed-only: duplicate rows and duplicate row identities are preserved as delivered; the preview path has no de-duplication step to assert") }) - // C19 "cardinality": what the caller asks for, what the store can supply, and what the - // response claims must line up, and the claimed count must agree with the delivered rows. t.Run("saved-edit-cardinality-and-limits", func(t *testing.T) { fixture, rows := c19NewFixture(t) _, hydrated := fixture.getBuilder() @@ -738,8 +579,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { c19Row("three", map[string]any{c19ColumnPatient: "patient-three", c19ColumnActive: true, c19ColumnBirthOrder: 3, c19ColumnBirthDate: "2003-03-03"}), } - // (a) Store supplies fewer rows than the requested limit: all three arrive, nothing is - // padded, sampled stays false (the preview proved it exhausted the output). rows.set(five) response := c19Preview(t, fixture, compiled.ReceiptId, 10) preview := c19DecodePreview(t, response) @@ -749,11 +588,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { {"__loom_row_id": c19RowIdentityPrefix + "three", c19ColumnPatient: "patient-three", c19ColumnActive: true, c19ColumnBirthOrder: float64(3), c19ColumnBirthDate: "2003-03-03"}, }, 3, false) - // (b) Store supplies more rows than the requested limit: the limit is enforced - // (internal/dataframe/execution/engine.go:1098-1101,1124-1126), the first requested - // rows arrive in store order, the excess never appears, and sampled flips true - // (internal/dataframe/execution/engine.go:1153-1156 → - // internal/server/openapi_explorer_authoring_direct.go:539). response = c19Preview(t, fixture, compiled.ReceiptId, 2) preview = c19DecodePreview(t, response) c19ExpectRows(t, preview, response.Body, []map[string]any{ @@ -766,8 +600,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { calls, batches := rows.observations() t.Logf("C19 (:69/:388) cardinality: limit=2 stopped after %d preview reads with backend batch hints %v; the engine, not this fixture, enforced it", calls, batches) - // (c) An empty store result is an empty array with rowCount 0, never null and never a - // missing field (internal/server/explorer_preview_response.go:227-229,292,305-311). rows.set(nil) response = c19Preview(t, fixture, compiled.ReceiptId, 10) preview = c19DecodePreview(t, response) @@ -776,10 +608,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { t.Fatalf("an empty result must publish an empty array: %s", response.Body) } - // (d) The documented default applies when the caller omits the limit - // (internal/explorer/lifecycle/preview.go:16-18 with - // internal/dataframe/execution/engine.go:47-50): 26 available rows yield exactly the - // documented 25 and sampled=true. twentySix := make([]map[string]any, 0, 26) for index := 0; index < 26; index++ { twentySix = append(twentySix, c19Row(fmt.Sprintf("default-%02d", index), map[string]any{ @@ -802,18 +630,10 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { t.Fatalf("the row past the documented default limit was published: %s", response.Body) } - // (e) A limit above the documented maximum is refused with the typed contract failure - // the handler promises (internal/explorer/lifecycle/preview.go:19-21), and no rows are - // published. rows.set(five) over := c19Preview(t, fixture, compiled.ReceiptId, dataframeexecution.MaxPreviewLimit+1) coreProtocolBoundaryFailure(t, over, "preview above the maximum limit", http.StatusUnprocessableEntity, "INVALID_PREVIEW_LIMIT", "preview", "limit must be between 1 and 1000") - // (f) A row count the client cannot forge: rowCount is server-derived only, so there - // is no client-side surface to make it disagree with the delivered rows, and the two - // fields come from one counter (internal/server/explorer_preview_response.go:233-286 - // increments once per visited row and :305-311 emits that same counter). The check - // below is therefore self-agreement, not an independent cross-check. consistent := c19Preview(t, fixture, compiled.ReceiptId, 10) consistentPreview := c19DecodePreview(t, consistent) if consistentPreview.RowCount != len(consistentPreview.Rows) { @@ -828,14 +648,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { } t.Log("C19 (:69/:388) observed-only: rowCount is derived from the same visitor count that produces rows, so the plan's 'rowCount vs actual rows' comparison has a single source; no independent authoritative count exists to compare against and the request carries no rowCount field to tamper with") - // (g) Zero/negative limits have NO route-level validation even though the public - // contract declares minimum 1 (openapi/openapi.yaml:6465): the handler rewrites an - // explicit 0 to -1 (internal/server/openapi_explorer_authoring_direct.go:504-510), the - // lifecycle default only covers an absent value - // (internal/explorer/lifecycle/preview.go:16-18), and the rejection happens deep in the - // engine (internal/dataframe/execution/engine.go:1256-1261). Only what the traced - // handler promises is asserted — no rows, and the failure stays a typed preview-stage - // error (internal/server/explorer_preview_response.go:44-104 stamps Stage "preview"). for _, limitCase := range []struct { name string value int @@ -867,14 +679,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { } }) - // ------------------------------------------------------------------ - // Restore path: after a draft restore, the published rows and schema must - // belong to the restored head's contract, not the head that was restored - // away from. Preview admits on the receipt and never requires head - // equality (internal/explorer/lifecycle/preview.go:28-44), so the older - // edited-contract receipt stays executable and stays distinguishable. - // ------------------------------------------------------------------ - t.Run("restore-rows-and-schema-follow-the-restored-head", func(t *testing.T) { fixture, rows := c19NewFixture(t) _, hydrated := fixture.getBuilder() @@ -882,8 +686,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { t.Fatalf("baseline hydration identity mismatch: %+v", hydrated) } - // Full literal row set reused across every state: typed values, an explicit null, an - // absent key, an empty string, a duplicate, and rows beyond any requested limit. typed := c19TypedRow() duplicate := cloneC19Row(typed) nullRow := c19Row("null-date", map[string]any{c19ColumnPatient: "patient-gamma", c19ColumnActive: true, c19ColumnBirthOrder: 3, c19ColumnBirthDate: nil}) @@ -896,7 +698,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { {"__loom_row_id": c19RowIdentityPrefix + "empty-date", c19ColumnPatient: "patient-delta", c19ColumnActive: false, c19ColumnBirthOrder: float64(4), c19ColumnBirthDate: ""}, } - // Saved edit, then reconcile: edited contract over the same rows. saved, ack := fixture.saveLabel("c19-restore-edit", c19LabelPatientEdit, hydrated.DraftVersion, hydrated.DraftDigest) if saved.StatusCode != http.StatusOK { t.Fatalf("save status=%d body=%s", saved.StatusCode, saved.Body) @@ -907,7 +708,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { c19ExpectContract(t, editedPreview, c19WantEditedContract()) c19ExpectRows(t, editedPreview, editedResponse.Body, wantFullRows, 4, false) - // Restore through the public command surface (core_authoring_protocol_restore_test.go:112-128). _, headAfterEdit := fixture.getBuilder() restoredVersion, restoredDigest, undoRevision := fixture.MustRestoreRevision("c19-restore-1", headAfterEdit) if restoredVersion != ack.DraftVersion+1 { @@ -921,16 +721,6 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { t.Fatalf("the restored head lost the original label: %s", stored.DraftConfig) } - // History-pointer semantics read from the WRITE path, not from fixture bookkeeping: - // every accepted draft CAS snapshots the head as it stood BEFORE the command and - // publishes that snapshot's store-minted id as the new head's history pointer - // (map double internal/server/http_test_helpers_test.go:152-161; Arango - // internal/explorer/arango/drafts.go:52-77). service.go:324 never touches the - // pointer, and draft_restore.go:33/47 validates the consumed undo target against the - // OLD head's pointer instead of copying it forward. So the restored head's pointer is - // the snapshot of the EDITED head (the next Undo slot), distinct from the revision the - // restore read. Ids are store-spelled, so the assertion goes through each store's own - // GetDraftRevision content, never the id string. c19ExpectRestoreHistory(t, "map store", stored.PreviousDraftRevisionID, undoRevision, ack.DraftVersion, ack.DraftDigest, c19LabelPatientEdit, hydrated.DraftVersion, hydrated.DraftDigest, c19LabelPatient, @@ -938,17 +728,10 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { return fixture.store.GetDraftRevision(context.Background(), c19Project, c19ExplorerID, revisionID) }) - // Reconcile the restored head: the published schema is the ORIGINAL contract again, - // the rows are the same literal rows, and this is a distinct receipt from the edited - // compilation. restoredReceipt := fixture.decodeCompile(fixture.MustReconcile(restoredVersion, restoredDigest, "c19-restore-reconcile-restored")) if restoredReceipt.ReceiptId == edited.ReceiptId { t.Fatal("the restored head reused the edited head's receipt identity") } - // Same traced digest provenance as the Arango leg (see the block in - // c19AgainstArango): on a restored head the CAS digest republishes the restored-from - // revision digest while the receipt intents the compilation-prepare form; the map - // double stores the response verbatim, so double and Arango agree here. if restoredDigest != hydrated.DraftDigest { t.Fatalf("restored head CAS digest=%q must republish the restored-from revision digest %q", restoredDigest, hydrated.DraftDigest) } @@ -967,15 +750,11 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { c19ExpectContract(t, restoredPreview, c19WantBaselineContract()) c19ExpectRows(t, restoredPreview, restoredResponse.Body, wantFullRows, 4, false) - // Cardinality after restore: the limit is still honoured and truncation still reports - // sampled, so restore did not quietly change the row contract. limitedResponse := c19Preview(t, fixture, restoredReceipt.ReceiptId, 2) limited := c19DecodePreview(t, limitedResponse) c19ExpectRows(t, limited, limitedResponse.Body, wantFullRows[:2], 2, true) c19ExpectContract(t, limited, c19WantBaselineContract()) - // Null/duplicate semantics survive restore unchanged: one absent key, one explicit - // null, one empty string, and the duplicate pair keep their exact public shape. if _, present := restoredPreview.Rows[0][c19ColumnBirthDate]; !present { t.Fatalf("the typed row lost its date after restore: %#v", restoredPreview.Rows[0]) } @@ -986,31 +765,17 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { t.Fatalf("the duplicate pair stopped being identical after restore: %#v", restoredPreview.Rows[:2]) } - // The superseded edited receipt stays executable and keeps describing ITS OWN frozen - // contract (immutable-receipt contract, - // internal/explorer/lifecycle/preview.go:28-44 admits on the receipt without head - // equality), which is what makes "the current head's rows/schema" a meaningful claim: - // the current receipt shows the restored label, the retained receipt still shows the - // saved-edit label. supersededResponse := c19Preview(t, fixture, edited.ReceiptId, 10) superseded := c19DecodePreview(t, supersededResponse) c19ExpectContract(t, superseded, c19WantEditedContract()) c19ExpectRows(t, superseded, supersededResponse.Body, wantFullRows, 4, false) - // And a receipt published for the ORIGINAL head recompiles to the same content-addressed - // receipt the restored head uses, so "restored" means the same contract identity rather - // than a look-alike. again := fixture.decodeCompile(fixture.MustReconcile(restoredVersion, restoredDigest, "c19-restore-reconcile-again")) if again.ReceiptId != restoredReceipt.ReceiptId { t.Fatalf("recompiling the identical restored head changed the receipt identity: %s vs %s", again.ReceiptId, restoredReceipt.ReceiptId) } }) - // ------------------------------------------------------------------ - // Opt-in real-store variant, same pattern as - // TestCoreAuthoringProtocolAgainstArango. - // ------------------------------------------------------------------ - t.Run("persisted-state-against-arango", func(t *testing.T) { endpoint := strings.TrimSpace(os.Getenv("LOOM_TEST_ARANGO_URL")) if endpoint == "" { @@ -1020,27 +785,17 @@ func TestCoreAuthoringProtocolRowSchemaSemantics(t *testing.T) { }) } -// c19AgainstArango runs the same literal row/schema expectations while the draft head, the -// draft history and the compile receipts live in a real owned Arango database, and reads the -// persisted bytes back through a second repository instance so the claim does not rest on the -// service's in-process objects. -// -// Honest bound: this leg verifies the AUTHORING-side persistence (head bytes after the saved -// edit and after the restore, receipt identity) behind the same rows/schema. The row VALUES -// still come from the controlled preview execution seam, because no dataset ingest runs here; -// claims about dataset-resident values/types belong to the real-store rows work that the -// family notes at core_authoring_protocol_formal_replay_test.go:35-45 and the plan M4 row -// assign to owned Arango/ClickHouse fixtures, and are NOT claimed by this test. func c19AgainstArango(t *testing.T, endpoint string) { t.Helper() database := "loom_core_rowschema_c19_" + strings.ReplaceAll(uuid.NewString(), "-", "") ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - client, err := storearango.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client t.Cleanup(func() { _ = client.Close(context.Background()) cleanupCtx, stop := context.WithTimeout(context.Background(), 5*time.Second) @@ -1060,10 +815,10 @@ func c19AgainstArango(t *testing.T, endpoint string) { t.Errorf("remove owned C19 database status=%d", response.StatusCode) } }) - if err := client.Bootstrap(ctx, explorerarango.BootstrapSpec()); err != nil { + if err := client.Bootstrap(ctx, arangofactory.ExplorerBootstrapSpec()); err != nil { t.Fatal(err) } - store, err := explorerarango.New(client) + store, err := arangofactory.NewExplorerStore(client) if err != nil { t.Fatal(err) } @@ -1091,8 +846,15 @@ func c19AgainstArango(t *testing.T, endpoint string) { t.Fatal(err) } rows := &c19RowStore{} - emptyRows := func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil } - engine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: emptyRows, PreviewQueryRows: rows.previewRows()}) + emptyRows := func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + } + engine, err := dataframeexecution.New(dataframeexecution.Config{CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: emptyRows, PreviewSourceRows: rows.previewRows()}) if err != nil { t.Fatal(err) } @@ -1144,7 +906,6 @@ func c19AgainstArango(t *testing.T, endpoint string) { t.Fatalf("commands status=%d body=%s", saved.StatusCode, saved.Body) } - // The edited head is the committed Arango document, byte for byte. persisted, err := store.Get(ctx, c19Project, c19ExplorerID) if err != nil { t.Fatalf("read persisted head from Arango: %v", err) @@ -1176,20 +937,6 @@ func c19AgainstArango(t *testing.T, endpoint string) { _, headAfterEdit := fixture.getBuilder() restoredVersion, restoredDigest, undoRevision := fixture.MustRestoreRevision("c19-arango-restore", headAfterEdit) restoredReceipt := fixture.decodeCompile(fixture.MustReconcile(restoredVersion, restoredDigest, "c19-arango-reconcile-restored")) - // Digest provenance on a RESTORED head, traced to the write and compile paths: - // - the CAS head digest is the restored revision's own digest, because the restore - // republishes those stored bytes (draft_restore.go:66-68 admits the revision only when - // its re-derived digest matches; service.go:296-301/324 stores them), so it equals the - // baseline head digest this fixture started from, NOT a freshly derived value; - // - the receipt intent is the digest of the compilation-prepare form, - // PrepareWorkspaceForCompilation = migrations followed by NormalizePresentationOrders - // (internal/explorer/authoringv2/migration.go:97, canonical.go:95-107), digested at - // internal/server/explorer_receipt_contract.go:142. Because this fixture seeds its - // first head through that same preparation (see c19Workspace), preparing an already - // prepared head is a no-op, so intent and CAS name the same value here and both digests - // are pinned. Seeding raw bytes instead splits them (the migration fills defaults the - // stored bytes never received) — that harness artifact, not a store difference, was the - // original failure. if restoredDigest != hydrated.DraftDigest { t.Fatalf("restored head CAS digest=%q must republish the restored-from revision digest %q", restoredDigest, hydrated.DraftDigest) } @@ -1201,14 +948,13 @@ func c19AgainstArango(t *testing.T, endpoint string) { c19ExpectContract(t, restoredPreview, c19WantBaselineContract()) c19ExpectRows(t, restoredPreview, restoredResponse.Body, wantRows, 3, false) - // Restart-equivalent read: a fresh repository instance reproduces the restored head from - // persisted bytes, so the restored rows/schema rest on committed state. - restartClient, err := storearango.Open(ctx, endpoint, database) + restartBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatal(err) } + restartClient := restartBackend.Client t.Cleanup(func() { _ = restartClient.Close(context.Background()) }) - restartStore, err := explorerarango.New(restartClient) + restartStore, err := arangofactory.NewExplorerStore(restartClient) if err != nil { t.Fatal(err) } @@ -1226,12 +972,6 @@ func c19AgainstArango(t *testing.T, endpoint string) { if got := c19AuthoredLabel(restartedWorkspace, c19ColumnPatient); got != c19LabelPatient { t.Fatalf("persisted Arango head after restore carries label=%q want %q", got, c19LabelPatient) } - // Same traced semantics as the map-store leg: the pointer names the store's snapshot of - // the head as it stood before the restore (the edited head at ack.DraftVersion), and the - // consumed undo target names the revision whose bytes were restored. Arango mints those - // ids as "draft_revision_" + sha256-keyed parts (drafts.go:139-141 with store/arango - // store.go:27-34), a different spelling from the map double, so BOTH sides are read back - // through a second repository instance and asserted on content. c19ExpectRestoreHistory(t, "arango restart", restarted.PreviousDraftRevisionID, undoRevision, ack.DraftVersion, ack.DraftDigest, c19LabelPatientEdit, hydrated.DraftVersion, hydrated.DraftDigest, c19LabelPatient, @@ -1256,10 +996,6 @@ func c19AgainstArango(t *testing.T, endpoint string) { if err != nil { t.Fatalf("fresh repository lost the restored receipt: %v", err) } - // Store fidelity, asserted the same way on both repositories: what Arango persisted is the - // reconcile response byte for byte — intent and CAS stay distinct. (This is why the - // restored-head mismatch is expectation drift, not a store-parity finding: the double and - // Arango returned identical bytes for the same request.) if persistedRestored.IntentDigest != restoredReceipt.IntentDigest || restartedReceipt.IntentDigest != restoredReceipt.IntentDigest { t.Fatalf("persisted restored receipt intent live=%q restart=%q response=%q", persistedRestored.IntentDigest, restartedReceipt.IntentDigest, restoredReceipt.IntentDigest) } diff --git a/internal/server/core_authoring_protocol_scoped_compile_test.go b/internal/server/core_authoring_protocol_scoped_compile_test.go index 2255e0b79..828cc101f 100644 --- a/internal/server/core_authoring_protocol_scoped_compile_test.go +++ b/internal/server/core_authoring_protocol_scoped_compile_test.go @@ -1,38 +1,10 @@ package server -// TestCoreAuthoringProtocolOutputScopedCompilation is the SERVER half of plan -// row C06 (docs/verification/CORE_PROTOCOL_VERIFICATION_PLAN.md:375). -// -// Derivation note (required by the plan's "independent literal output and -// dependency membership and schemas" requirement): every expected member, -// public column id, label, and logical type asserted below was hand-written -// from the workspace construction inputs in scopedCompileTestWorkspace at the -// top of this file. No expectation is read back from a compiler result first; -// if the compiler disagrees with the by-hand contract the test fails. -// -// Honest accommodation of the compiler's actual dependency mechanism (report, -// plus in-file accommodation): the workspace-output dependency the production -// code enforces is a construction step input with -// Kind == authoringv2.ConstructionInputWorkspaceOutput. It is consumed only by -// the terminal Combine form (recipe.Construction.validateTerminalCombine), -// which mandates at least two distinct immutable inputs -// ("steps[0].inputs must contain at least two exact inputs"), and the second -// alternative input kind (TABLE_REVISION) needs the published-input schema -// resolver that this fixture deliberately leaves nil (compileExplorerReceipt -// errors with "exact published input schema resolver is required"). A -// encounters output therefore cannot depend on patients alone, and it cannot -// take conditions as its second input without conditions joining encounters' -// required closure (destroying the "unrelated output" half of C06). The -// workspace adds one extra patient-grain sibling (visits) so encounters' -// terminal APPEND combine is compiler-legal while conditions stays in NO -// dependency relationship with encounters: reconciling encounters compiles -// patients + visits + encounters and never conditions; reconciling conditions -// compiles conditions alone. - import ( "context" "encoding/json" "fmt" + "github.com/calypr/loom/internal/dataframe/compiler" "net/http" "reflect" "sort" @@ -50,9 +22,6 @@ import ( "github.com/gofiber/fiber/v3" ) -// Hand-written workspace identities. These literals are the single hand -// source of truth: scopedCompileTestWorkspace builds the persisted draft from -// them and the assertions below compare compiler receipts against them. const ( scopedCompileProject = "project-a" scopedCompileExplorer = "core-protocol" @@ -90,14 +59,6 @@ const ( scopedUnknownOutputID = "core-protocol-ghost-output" ) -// scopedCompileTestWorkspace builds the persisted draft inputs: -// - patients : root Patient, one authored field column (candidate Patient.id). -// - visits : independent patient-grain sibling used only as the second -// compiler-mandated input of encounters' terminal APPEND combine. -// - encounters: derived output whose single terminal combine step declares -// WORKSPACE_OUTPUT inputs on patients and visits. -// - conditions: UNRELATED independent root; nothing references it and it -// references nothing. func scopedCompileTestWorkspace() authoringv2.Workspace { fieldColumn := func(columnID, column, label string) authoringv2.Column { return authoringv2.Column{ @@ -154,13 +115,6 @@ func scopedCompileTestWorkspace() authoringv2.Workspace { } } -// scopedLiteralColumn is one hand-written public contract expectation for one -// output: identity, label, and type as declared in the workspace above. -// Cardinality and nullability are additionally fixed for the combine output, -// whose declared stage column (Type string, Nullable true) is the by-hand -// source; the field-backed columns' nullability comes from the shared test -// capability snapshot and is intentionally not restated here as authored -// intent. type scopedLiteralColumn struct { public string label string @@ -179,9 +133,6 @@ var scopedLiteralColumns = map[string][]scopedLiteralColumn{ scopedOutputEncounters: {{public: scopedEncounterColumn, label: scopedEncounterLabel, logicalType: scopedLogicalTypeString, nullable: scopedBool(true), cardinality: scopedString(scopedCombineEmissionCard)}}, } -// Hand-written closure membership, sorted for order-independent comparison. -// scopedCompileTestWorkspace declares documents [patients, encounters, visits, -// conditions] with encounters' terminal combine depending on patients+visits. var ( scopedClosureEncounters = []string{scopedOutputEncounters, scopedOutputPatients, scopedOutputVisits} scopedClosureConditions = []string{scopedOutputConditions} @@ -222,8 +173,15 @@ func newScopedCompileFixture(t *testing.T) *scopedCompileFixture { if err != nil { t.Fatal(err) } - queryRows := func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil } - engine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: queryRows, PreviewQueryRows: queryRows}) + queryRows := func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + } + engine, err := dataframeexecution.New(dataframeexecution.Config{CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: queryRows, PreviewSourceRows: queryRows}) if err != nil { t.Fatal(err) } @@ -273,9 +231,6 @@ func (f *scopedCompileFixture) reconcile(outputID, requestID string) testHTTPRes return response } -// scopedCompileDecodeReceipt decodes a successful reconcile through the strict -// response contract and returns the receipt identity, the receipt-adjacent -// outputs list, the returned builder workspace, and the persisted receipt. func (f *scopedCompileFixture) scopedCompileDecodeReceipt(t *testing.T, response testHTTPResponse, stage string) (loomapi.CompileResponse, authoringv2.Workspace, *explorer.CompilationReceipt) { t.Helper() if response.StatusCode != http.StatusOK { @@ -301,9 +256,6 @@ func (f *scopedCompileFixture) scopedCompileDecodeReceipt(t *testing.T, response return compiled, decoded.Builder, &persisted } -// scopedExpectCompleteWorkspace asserts the reconcile response returned the -// coherent complete validated workspace: every document and tab of the saved -// head with the exact persisted head digest. func scopedExpectCompleteWorkspace(t *testing.T, stage string, builder authoringv2.Workspace, initialDigest string) { t.Helper() if responseDigest, err := builder.Digest(); err != nil || responseDigest != initialDigest { @@ -340,8 +292,6 @@ func scopedExpectStrings(t *testing.T, stage, component string, got, want []stri } } -// scopedExpectReceipt asserts membership and schemas of one persisted receipt -// against the hand-written literals for the requested closure. func scopedExpectReceipt(t *testing.T, stage string, receipt *explorer.CompilationReceipt, wantOutputs, absentTokens []string) { t.Helper() bundleNames := make([]string, 0, len(receipt.Bundle.Outputs)) @@ -477,8 +427,6 @@ func scopedExpectReceipt(t *testing.T, stage string, receipt *explorer.Compilati } } -// scopedExpectCombineRecipe asserts the encounters recipe entry against the -// hand-written dependency declaration above. func scopedExpectCombineRecipe(t *testing.T, stage string, output recipe.Output) { t.Helper() if output.Construction == nil || len(output.Construction.Steps) != 1 { @@ -507,20 +455,9 @@ func scopedExpectCombineRecipe(t *testing.T, stage string, output recipe.Output) } } -// TestCoreAuthoringProtocolOutputScopedCompilation reconciles the dependent -// output, the unrelated output, and the complete workspace through the -// production strict reconcile route (fixture pattern of -// core_authoring_protocol_test.go), asserting per case that the persisted -// receipt contains exactly the requested closure's recipe/emission/contract -// members while the response keeps returning the full validated workspace. func TestCoreAuthoringProtocolOutputScopedCompilation(t *testing.T) { fixture := newScopedCompileFixture(t) - // Case 1: output-scoped reconcile of the dependent output compiles its - // required closure only. patients-side artifacts the contract includes - // (patients+visits recipe outputs, emissions, fingerprints, compiled - // schemas and contracts) accompany the encounters entry; no conditions - // receipt/recipe entry may exist anywhere in the receipt. first := fixture.reconcile(scopedOutputEncounters, "c06-scoped-encounters") compiledEncounters, builderEncounters, receiptEncounters := fixture.scopedCompileDecodeReceipt(t, first, "output-scoped encounters reconcile") scopedExpectReceipt(t, "output-scoped encounters reconcile", receiptEncounters, scopedClosureEncounters, []string{scopedOutputConditions}) @@ -534,10 +471,6 @@ func TestCoreAuthoringProtocolOutputScopedCompilation(t *testing.T) { t.Fatalf("output-scoped encounters reconcile persisted %d receipts, want exactly 1", len(fixture.store.receipts)) } - // Case 3 (run before the full reconcile so the unrelated output is proven - // independent of the encounters dependency chain): reconciling conditions - // compiles conditions alone, excluding every encounters dependency-chain - // artifact (patients, visits, and the encounters step itself). second := fixture.reconcile(scopedOutputConditions, "c06-scoped-conditions") compiledConditions, builderConditions, receiptConditions := fixture.scopedCompileDecodeReceipt(t, second, "output-scoped conditions reconcile") scopedExpectReceipt(t, "output-scoped conditions reconcile", receiptConditions, scopedClosureConditions, []string{scopedOutputEncounters, scopedOutputPatients, scopedOutputVisits}) @@ -552,9 +485,6 @@ func TestCoreAuthoringProtocolOutputScopedCompilation(t *testing.T) { t.Fatalf("output-scoped conditions reconcile persisted %d receipts, want exactly 2", len(fixture.store.receipts)) } - // Case 2: the complete-workspace reconcile covers all outputs, each with - // its literal contract, and the response workspace is the coherent full - // validated head. full := fixture.reconcile("", "c06-full-workspace") compiledFull, builderFull, receiptFull := fixture.scopedCompileDecodeReceipt(t, full, "complete workspace reconcile") scopedExpectReceipt(t, "complete workspace reconcile", receiptFull, scopedClosureFull, nil) @@ -562,23 +492,16 @@ func TestCoreAuthoringProtocolOutputScopedCompilation(t *testing.T) { if compiledFull.ReceiptId == compiledEncounters.ReceiptId || compiledFull.ReceiptId == compiledConditions.ReceiptId { t.Fatal("the complete workspace reconcile reused a scoped receipt identity") } - // The complete workspace compile is a different intent than either scoped - // compile (IntentDigest is the compiled-identity of the whole workspace, - // not the raw CAS draft digest: lifecycle compiles the presentation- - // normalized workspace). if receiptFull.IntentDigest == receiptEncounters.IntentDigest || receiptFull.IntentDigest == receiptConditions.IntentDigest { t.Fatal("the complete workspace reconcile reused a scoped compile intent") } if len(fixture.store.receipts) != 3 { t.Fatalf("the three successful reconciles persisted %d receipts, want exactly 3", len(fixture.store.receipts)) } - // A scoped compile is never the full compile: intents differ even where - // the requested closure equals the whole workspace minus nothing. if receiptFull.RecipeDigest == receiptEncounters.RecipeDigest { t.Fatal("the complete workspace recipe digest matches the scoped encounters recipe digest") } - // Case 4: an unknown outputId is a typed rejection with nothing persisted. receiptsBefore := len(fixture.store.receipts) unknown := fixture.reconcile(scopedUnknownOutputID, "c06-scoped-unknown") coreProtocolBoundaryFailure(t, unknown, "output-scoped reconcile of unknown outputId", http.StatusUnprocessableEntity, "INVALID_OUTPUT_SCOPE", "reconcile", scopedUnknownOutputID) diff --git a/internal/server/core_authoring_protocol_test.go b/internal/server/core_authoring_protocol_test.go index ef827348d..1c27976b4 100644 --- a/internal/server/core_authoring_protocol_test.go +++ b/internal/server/core_authoring_protocol_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/calypr/loom/internal/dataframe/compiler" "io" "net/http" "net/http/httptest" @@ -116,11 +117,6 @@ type coreProtocolFixture struct { beforeCompile func() } -// newCoreProtocolFixture mounts the production generated strict adapter with the -// real authoring handlers, lifecycle service, compile chain and persisted receipt -// store. CompileReceipt/PreviewReceipt are the repository's established test-owned -// service seams (mirroring the injected production implementations); the compile -// callback stays available for scheduling interleavings at that existing boundary. func newCoreProtocolFixture(t *testing.T) *coreProtocolFixture { t.Helper() snapshot := testAuthoringV2CapabilitySnapshot() @@ -151,8 +147,15 @@ func newCoreProtocolFixture(t *testing.T) *coreProtocolFixture { if err != nil { t.Fatal(err) } - queryRows := func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil } - engine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: queryRows, PreviewQueryRows: queryRows}) + queryRows := func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + } + engine, err := dataframeexecution.New(dataframeexecution.Config{CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: queryRows, PreviewSourceRows: queryRows}) if err != nil { t.Fatal(err) } @@ -273,16 +276,9 @@ func (f *coreProtocolFixture) receiptEmittedLabel(t *testing.T, receiptID, publi return "" } -// TestCoreAuthoringProtocolVerticalWitness is the first joined production witness -// (plan step 4 / M2): public Builder GET hydration, real POST commands save with -// CAS identity, public reconcile compile with receipt validation and persistence, -// and receipt-bound preview. Requests and identities are the ones the production -// UI client actually sends. Rows come from the controlled execution boundary; -// semantic row correctness is a separate milestone. func TestCoreAuthoringProtocolVerticalWitness(t *testing.T) { fixture := newCoreProtocolFixture(t) - // 1. Initial hydration through the production read handler. first, hydrated := fixture.getBuilder() if first.StatusCode != http.StatusOK { t.Fatalf("builder status=%d body=%s", first.StatusCode, first.Body) @@ -291,7 +287,6 @@ func TestCoreAuthoringProtocolVerticalWitness(t *testing.T) { t.Fatalf("builder identity mismatch: %+v", hydrated) } - // 2. Real save with the identities the production UI client sends. saved, ack := fixture.saveLabel("core-witness-save-1", "Patient Identifier", hydrated.DraftVersion, hydrated.DraftDigest) if saved.StatusCode != http.StatusOK { t.Fatalf("commands status=%d body=%s", saved.StatusCode, saved.Body) @@ -319,13 +314,11 @@ func TestCoreAuthoringProtocolVerticalWitness(t *testing.T) { t.Fatalf("history revision did not capture prior head: %+v err=%v", prior, err) } - // 3. Public reload agrees with the acknowledged revision (later coherent read). second, reloaded := fixture.getBuilder() if second.StatusCode != http.StatusOK || reloaded.DraftVersion != 2 || reloaded.DraftDigest != ack.DraftDigest || reloaded.PreviousDraftRevisionId == nil || *reloaded.PreviousDraftRevisionId != wantRevision { t.Fatalf("reload disagrees with acknowledgement: %+v", reloaded) } - // 4. Reconciling the obsolete pre-save head is rejected at the CAS boundary. stale, err := fixture.reconcile(1, fixture.initialDigest, "builder-1-1") if err != nil { t.Fatal(err) @@ -334,7 +327,6 @@ func TestCoreAuthoringProtocolVerticalWitness(t *testing.T) { t.Fatalf("stale reconcile status=%d body=%s", stale.StatusCode, stale.Body) } - // 5. Reconcile the saved head: real compile, validation and receipt persistence. response := fixture.MustReconcile(2, ack.DraftDigest, "builder-1-2") compiled := fixture.decodeCompile(response) coreProtocolExpectLabels(t, response.Body, "builder", "patients", "Patient Identifier") @@ -349,7 +341,6 @@ func TestCoreAuthoringProtocolVerticalWitness(t *testing.T) { t.Fatalf("persisted receipt contract label=%q want Patient Identifier", label) } - // 6. Receipt-bound preview keeps the accepted compilation identity. previewed := fixture.preview(compiled.ReceiptId) if previewed.StatusCode != http.StatusOK { t.Fatalf("preview status=%d body=%s", previewed.StatusCode, previewed.Body) @@ -372,21 +363,12 @@ func TestCoreAuthoringProtocolVerticalWitness(t *testing.T) { t.Fatalf("preview contract labels=%v", previewLabels) } - // 7. A substituted receipt label cannot publish rows as success. foreign := coreProtocolRequest(t, fixture.app, http.MethodPost, fixture.basePath+"/preview", `{"receiptId":"core-witness-foreign-receipt","outputId":"patients","limit":10}`, "") if foreign.StatusCode == http.StatusOK || strings.Contains(foreign.Body, "Patient Identifier") { t.Fatalf("foreign receipt label unexpectedly admitted: status=%d body=%s", foreign.StatusCode, foreign.Body) } } -// TestCoreAuthoringProtocolCompetingSaveDuringReconcile exercises one competing -// save/refetch schedule: reconcile of saved head A is paused inside the compile -// service seam, a concurrent commands request saves head B, then A's compile -// completes. Acceptance: A's receipt and returned workspace describe the captured -// CAS-validated A (not the later head), the public head is B, A's retained receipt -// still executes (immutable-receipt contract), and B independently compiles and -// previews. UI admission of A against B's current ownership is a separate mounted -// driver boundary; this witness owns the server-side contract observations. func TestCoreAuthoringProtocolCompetingSaveDuringReconcile(t *testing.T) { fixture := newCoreProtocolFixture(t) first, hydrated := fixture.getBuilder() @@ -419,7 +401,6 @@ func TestCoreAuthoringProtocolCompetingSaveDuringReconcile(t *testing.T) { t.Fatal("compile seam was not reached") } - // Save B while A's compile is suspended after CAS validation. savedB, ackB := fixture.saveLabel("core-competing-save-b", "Patient Competing B", 1, fixture.initialDigest) if savedB.StatusCode != http.StatusOK { close(release) @@ -437,7 +418,6 @@ func TestCoreAuthoringProtocolCompetingSaveDuringReconcile(t *testing.T) { } compiled := fixture.decodeCompile(response) - // The receipt and the returned workspace describe captured A, never B. coreProtocolExpectLabels(t, response.Body, "builder", "patients", "Patient ID") if label := fixture.receiptEmittedLabel(t, compiled.ReceiptId, "c_patient"); label != "Patient ID" { t.Fatalf("receipt from A compile described %q, not the captured saved head", label) @@ -446,13 +426,11 @@ func TestCoreAuthoringProtocolCompetingSaveDuringReconcile(t *testing.T) { t.Fatalf("reconcile response leaked the later head into the validated workspace: %s", response.Body) } - // The authoritative head advanced to B; the later coherent read shows it. second, reloaded := fixture.getBuilder() if second.StatusCode != http.StatusOK || reloaded.DraftVersion != 2 || reloaded.DraftDigest != ackB.DraftDigest { t.Fatalf("head did not advance to the concurrent save: %+v", reloaded) } - // A retained authorized historical receipt executes despite the newer head. previewA := fixture.preview(compiled.ReceiptId) if previewA.StatusCode != http.StatusOK { t.Fatalf("historical receipt execution rejected: status=%d body=%s", previewA.StatusCode, previewA.Body) @@ -465,7 +443,6 @@ func TestCoreAuthoringProtocolCompetingSaveDuringReconcile(t *testing.T) { t.Fatalf("historical preview identity mismatch: %+v", previewResponse) } - // The current head B compiles, previews and returns its own contract. responseB := fixture.MustReconcile(2, ackB.DraftDigest, "builder-1-2") compiledB := fixture.decodeCompile(responseB) if compiledB.ReceiptId == compiled.ReceiptId { @@ -480,16 +457,11 @@ func TestCoreAuthoringProtocolCompetingSaveDuringReconcile(t *testing.T) { t.Fatalf("current preview B rejected or unbound: status=%d body=%s", previewB.StatusCode, previewB.Body) } - // A's older receipt remains distinguishable by its frozen contract after B exists. if labelA := fixture.receiptEmittedLabel(t, compiled.ReceiptId, "c_patient"); labelA != "Patient ID" { t.Fatalf("frozen receipt A was mutated by the later head: %q", labelA) } } -// coreProtocolSeedStoredDraft replaces the persisted draft head through the same -// CAS store write the production command path performs (testExplorerStore.SaveDraft, -// http_test_helpers_test.go:142) so malformed or poisoned persisted state is seeded -// without substituting any production service seam. It returns the accepted head. func coreProtocolSeedStoredDraft(t *testing.T, fixture *coreProtocolFixture, raw []byte, digest string) (int64, string) { t.Helper() current, err := fixture.store.Get(context.Background(), "project-a", "core-protocol") @@ -506,13 +478,6 @@ func coreProtocolSeedStoredDraft(t *testing.T, fixture *coreProtocolFixture, raw return stored.DraftVersion, stored.DraftDigest } -// coreProtocolBoundaryFailure asserts the production strict route reported the typed -// boundary failure (status, error code, error stage) and published no successful -// compile/preview payload. Every authored error reaches this shape through -// authoringErrorForOpenAPI (openapi_explorer_authoring_direct.go:738), which recovers the -// lifecycle class/code/stage even when the preview transport wrapped it -// (previewRouteError unwraps via explorer.AuthoringError.Unwrap, -// explorer/authoring_errors.go:26). func coreProtocolBoundaryFailure(t *testing.T, response testHTTPResponse, stage string, wantStatus int, wantCode, wantErrorStage, wantMessageContains string) { t.Helper() if response.StatusCode != wantStatus { @@ -545,22 +510,7 @@ func coreProtocolBoundaryFailure(t *testing.T, response testHTTPResponse, stage } } -// TestCoreAuthoringProtocolErrorBoundaries is the C18 error family -// (docs/verification/CORE_PROTOCOL_VERIFICATION_PLAN.md:335): malformed/incomplete -// decoded data, missing output/receipt, compile failure, unsupported receipt contract -// and invalid preview response must each be reported by the correct boundary as the -// intended typed failure, without publishing a partial successful result, while the -// relevant valid companion keeps working and retained prior state stays distinguishable -// from newly accepted state. Requests go only through the production generated strict -// adapter, handlers, lifecycle service, compile chain and persisted receipt store; the -// only test-side action is persisted-state seeding through the exposed store handle. func TestCoreAuthoringProtocolErrorBoundaries(t *testing.T) { - // Malformed/incomplete decoded data at the reconcile boundary. - // ReconcileWithWorkspace decodes the persisted draft after CAS - // (authoring.go:559-561) and reports conflict AUTHORING_STATE_MISSING, so a draft - // that fails workspace decoding is a 409 typed failure, never a partial compile. - // The same undecodable head is refused by hydration as DRAFT_STATE_INVALID - // (query.go:127-130) rather than a partially hydrated Builder state. t.Run("malformed-stored-draft", func(t *testing.T) { fixture := newCoreProtocolFixture(t) if _, hydrated := fixture.getBuilder(); hydrated.DraftVersion != 1 || hydrated.DraftDigest != fixture.initialDigest { @@ -568,7 +518,6 @@ func TestCoreAuthoringProtocolErrorBoundaries(t *testing.T) { } receiptsBefore := len(fixture.store.receipts) - // (a) Truncated JSON: the persisted bytes cannot be decoded at all. truncated := []byte(`{"apiVersion":"explorer.calypr.org/authoringv2"`) if _, err := authoringv2.DecodeWorkspace(truncated); err == nil { t.Fatal("truncated seed unexpectedly decodes as a workspace") @@ -582,11 +531,6 @@ func TestCoreAuthoringProtocolErrorBoundaries(t *testing.T) { hydrated, _ := fixture.getBuilder() coreProtocolBoundaryFailure(t, hydrated, "builder hydration of truncated stored draft", http.StatusConflict, "DRAFT_STATE_INVALID", "builder", "") - // (b) Complete JSON that violates the persisted-workspace contract: the same - // canonical bytes with a blank explorer.title, which the persisted decoder rejects - // through authoringv2.Workspace.Validate ("explorer.title is required") - // (canonical.go:343-351). The seeded bytes are re-checked against the very decoder - // the handler uses, so this case cannot silently degrade into valid input. raw, err := fixture.workspace.CanonicalJSON() if err != nil { t.Fatal(err) @@ -617,7 +561,6 @@ func TestCoreAuthoringProtocolErrorBoundaries(t *testing.T) { t.Fatalf("rejected malformed drafts persisted receipts: %d new", len(fixture.store.receipts)-receiptsBefore) } - // Valid companion: the same route accepts and compiles restored valid state. restoredVersion, restoredDigest := coreProtocolSeedStoredDraft(t, fixture, raw, fixture.initialDigest) saved, ack := fixture.saveLabel("core-errors-malformed-companion", "Patient Identifier", restoredVersion, restoredDigest) if saved.StatusCode != http.StatusOK || ack.DraftVersion != restoredVersion+1 { @@ -632,11 +575,6 @@ func TestCoreAuthoringProtocolErrorBoundaries(t *testing.T) { } }) - // Missing receipt at the preview boundary. Preview looks the receipt up through the - // configured lookup (preview.go:28) which is domain.CompilationReceiptForExplorer for - // this fixture; a never-issued id yields explorer.ErrNotFound, wrapped as - // ClassNotFound COMPILE_RECEIPT_NOT_FOUND (lifecycle/errors.go:89-94) and mapped to 404 - // by lifecycleErrorStatus (openapi_explorer_authoring_direct.go:810). t.Run("missing-receipt-preview", func(t *testing.T) { fixture := newCoreProtocolFixture(t) compiled := fixture.decodeCompile(fixture.MustReconcile(1, fixture.initialDigest, "errors-missing-receipt-baseline")) @@ -657,25 +595,12 @@ func TestCoreAuthoringProtocolErrorBoundaries(t *testing.T) { t.Fatalf("missing receipt preview moved the saved head") } - // Valid companion: the real receipt still previews. valid := fixture.preview(compiled.ReceiptId) if valid.StatusCode != http.StatusOK || !strings.Contains(valid.Body, `"rows":[]`) { t.Fatalf("valid companion preview rejected: status=%d body=%s", valid.StatusCode, valid.Body) } }) - // Compile failure at the reconcile/compile boundary, driven by input the real engine - // rejects rather than by a substituted seam: the saved column source addresses a field - // for which the resolved capability snapshot has no candidate, so the document fails - // with the compiler's typed intent STALE_FIELD (semantic_compile.go:124-127), which - // CompileWorkspace reports as compile DOCUMENT_COMPILE_FAILED carrying that cause and - // JSON path (compile.go:184); lifecycle then maps *explorercompilation.Error to - // ClassUnprocessable with compilationErrorCode (authoring.go:410-415) → 422 - // (openapi_explorer_authoring_direct.go:814). The fixture hook surface (beforeCompile, - // core_authoring_protocol_test.go:173-178) only observes the existing compile seam and - // cannot inject a compile error; the repository's existing tests drive it purely for - // scheduling interleavings (core_authoring_protocol_formal_replay_test.go:87-94), so it - // is deliberately not reused or widened here. t.Run("compile-failure", func(t *testing.T) { fixture := newCoreProtocolFixture(t) retained := fixture.decodeCompile(fixture.MustReconcile(1, fixture.initialDigest, "errors-compile-retained")) @@ -698,7 +623,6 @@ func TestCoreAuthoringProtocolErrorBoundaries(t *testing.T) { if err != nil { t.Fatal(err) } - // The rejection must come from the compiler, not from an undecodable draft. if _, err := authoringv2.DecodeWorkspace(variantRaw); err != nil { t.Fatalf("compiler-rejection variant must stay a decodable draft: %v", err) } @@ -714,7 +638,6 @@ func TestCoreAuthoringProtocolErrorBoundaries(t *testing.T) { } coreProtocolBoundaryFailure(t, response, "reconcile of compiler-rejected saved draft", http.StatusUnprocessableEntity, "DOCUMENT_COMPILE_FAILED", "compile", "STALE_FIELD") - // No receipt published for the rejected attempt, retained state is untouched. if len(fixture.store.receipts) != receiptsBefore { t.Fatalf("failed compile persisted %d new receipts", len(fixture.store.receipts)-receiptsBefore) } @@ -728,17 +651,10 @@ func TestCoreAuthoringProtocolErrorBoundaries(t *testing.T) { if retainedAfter.StatusCode != http.StatusOK || !strings.Contains(retainedAfter.Body, `"rows":[]`) { t.Fatalf("retained receipt no longer previews after a failed compile: status=%d body=%s", retainedAfter.StatusCode, retainedAfter.Body) } - // Retained vs newly accepted: the head carries the rejected draft identity while - // the only persisted receipt still describes the earlier accepted compilation. if _, head := fixture.getBuilder(); head.DraftVersion != rejectedVersion || head.DraftDigest != rejectedDigest { t.Fatalf("rejected draft did not remain the described head: %+v", head) } - // Valid companion. Restoring the identical draft first shows the receipt store's - // content-addressed identity (the same compilation resolves to the same persisted - // receipt), so the newly accepted state is demonstrated with a real saved edit: a - // distinct receipt is published while the retained earlier receipt keeps its frozen - // contract and keeps previewing. restoredVersion, restoredDigest := coreProtocolSeedStoredDraft(t, fixture, baseline, fixture.initialDigest) restoredCompile := fixture.decodeCompile(fixture.MustReconcile(restoredVersion, restoredDigest, "errors-compile-companion-restore")) if restoredCompile.ReceiptId != retained.ReceiptId { @@ -769,21 +685,6 @@ func TestCoreAuthoringProtocolErrorBoundaries(t *testing.T) { } }) - // Unsupported receipt contract at the preview boundary. The fixture wires - // ReceiptLookup straight to domain.CompilationReceiptForExplorer - // (core_authoring_protocol_test.go:179-181) and exposes no lookup hook, so this - // variant is driven only by persisted receipt state through the production lookup - // chain (service.go:357-359 → store.GetCompilationReceiptForExplorer) and the - // contract branches of validateReceiptRoute (policy.go:125-142): a receipt whose - // compiler contract is not executable is conflict RECEIPT_RECOMPILE_REQUIRED, and so - // is an incomplete receipt identity. Both are 409 via lifecycleErrorStatus. - // Boundaries recorded honestly instead of faked: a receipt with a foreign project or - // Explorer is rejected before any contract branch, at the lookup, as - // COMPILE_RECEIPT_NOT_FOUND (http_test_helpers_test.go:200-206) — that rejection is - // T1's coverage and is not re-asserted here; and the public-output-contract decode - // branch of validateReceiptOutputContract (policy.go:206-225) collapses into the same - // typed UNKNOWN_AUTHORING_OUTPUT as recipe membership, asserted in - // invalid-preview-response. t.Run("unsupported-receipt-contract", func(t *testing.T) { fixture := newCoreProtocolFixture(t) compiled := fixture.decodeCompile(fixture.MustReconcile(1, fixture.initialDigest, "errors-contract-baseline")) @@ -796,9 +697,6 @@ func TestCoreAuthoringProtocolErrorBoundaries(t *testing.T) { } receiptsBefore := len(fixture.store.receipts) - // (a) Historical compiler contract: readable, but not executable natively. - // Constants mirror the unexported legacy identities in - // internal/explorer/compilation_receipt.go:27,36. unsupported := genuine unsupported.ID = "core-protocol-unsupported-contract" unsupported.ReceiptFormatVersion = 3 @@ -812,7 +710,6 @@ func TestCoreAuthoringProtocolErrorBoundaries(t *testing.T) { t.Fatalf("unsupported-contract receipt executed: %s", response.Body) } - // (b) Incomplete receipt identity (no recipe digest / no compiled outputs). incomplete := genuine incomplete.ID = "core-protocol-incomplete-contract" incomplete.RecipeDigest = "" @@ -826,21 +723,12 @@ func TestCoreAuthoringProtocolErrorBoundaries(t *testing.T) { t.Fatalf("preview failures changed persisted receipts: before=%d now=%d", receiptsBefore, len(fixture.store.receipts)) } - // Valid companion: the genuine current-contract receipt is unaffected. valid := fixture.preview(compiled.ReceiptId) if valid.StatusCode != http.StatusOK || !strings.Contains(valid.Body, `"rows":[]`) || !strings.Contains(valid.Body, "Patient ID") { t.Fatalf("genuine receipt preview rejected: status=%d body=%s", valid.StatusCode, valid.Body) } }) - // Invalid preview request/response at the preview boundary. Missing or blank - // identities are the malformed class of Preview itself, which trims them away - // (preview.go:13-15 → MALFORMED_REQUEST → 400), and an output the receipt never - // emitted is unprocessable UNKNOWN_AUTHORING_OUTPUT (preview.go:42-44). Neither can - // publish state. Note a nil body (openapi_explorer_authoring_direct.go:501-503) is - // unreachable through the generated strict adapter, which always binds a JSON object - // into the request struct (api.gen.go:20945-20949), so MALFORMED_AUTHORING_REQUEST is - // not exercisable from this route and is recorded as a boundary, not faked. t.Run("invalid-preview-response", func(t *testing.T) { fixture := newCoreProtocolFixture(t) compiled := fixture.decodeCompile(fixture.MustReconcile(1, fixture.initialDigest, "errors-preview-baseline")) @@ -864,8 +752,6 @@ func TestCoreAuthoringProtocolErrorBoundaries(t *testing.T) { } } - // An output the receipt never emitted, whether absent from the recipe or outside - // its public contract, is one typed rejection at the same boundary. response := coreProtocolRequest(t, fixture.app, http.MethodPost, fixture.basePath+"/preview", fmt.Sprintf(`{"receiptId":%q,"outputId":"never-emitted-output","limit":10}`, compiled.ReceiptId), "") coreProtocolBoundaryFailure(t, response, "preview of output absent from the receipt", http.StatusUnprocessableEntity, "UNKNOWN_AUTHORING_OUTPUT", "preview", "") @@ -876,7 +762,6 @@ func TestCoreAuthoringProtocolErrorBoundaries(t *testing.T) { t.Fatalf("invalid preview requests moved the saved head") } - // Valid companion: the well-formed receipt-bound request still succeeds. valid := fixture.preview(compiled.ReceiptId) var preview loomapi.PreviewResponse if err := json.Unmarshal([]byte(valid.Body), &preview); err != nil { diff --git a/internal/server/dataframe_query_error_test.go b/internal/server/dataframe_query_error_test.go index 8c167443c..c052d3826 100644 --- a/internal/server/dataframe_query_error_test.go +++ b/internal/server/dataframe_query_error_test.go @@ -1,188 +1,199 @@ package server import ( + "context" "errors" + "fmt" "net/http" "testing" "github.com/arangodb/go-driver/v2/arangodb/shared" + arangofactory "github.com/calypr/loom/internal/backends/arango" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/explorer" ) -func TestClassifyDataframeQueryErrorPreservesArangoMemoryLimit(t *testing.T) { - driverErr := shared.ArangoError{ - HasError: true, - Code: 500, - ErrorNum: shared.ErrResourceLimit, - ErrorMessage: "AQL: query would use more memory than allowed", - } - - err := classifyDataframeQueryError(driverErr) - userErr, ok := dataframeerrors.AsUserError(err) +func classifyThroughAdapter(t *testing.T, driverErr error) error { + t.Helper() + classified, ok := arangofactory.TranslateQueryError(fmt.Errorf("arango query: %w", driverErr)) if !ok { - t.Fatalf("classifyDataframeQueryError() = %v, want dataframe user error", err) - } - if userErr.Code() != string(dataframeerrors.CodeQueryMemoryLimitExceeded) { - t.Fatalf("code = %q, want %q", userErr.Code(), dataframeerrors.CodeQueryMemoryLimitExceeded) - } - if got := userErr.Details()["backend"]; got != "arangodb" { - t.Fatalf("backend = %v, want arangodb", got) - } - if !errors.Is(err, driverErr) { - t.Fatal("classified error did not preserve the driver cause") + t.Fatalf("adapter did not classify %v", driverErr) } + return classified } -func TestClassifyDataframeQueryErrorPreservesArangoOutOfMemory(t *testing.T) { +func TestPreserveConfiguredQueryErrorRoutesAdapterClassifiedAssertion(t *testing.T) { driverErr := shared.ArangoError{ - HasError: true, - Code: 500, - ErrorNum: shared.ErrOutOfMemory, - ErrorMessage: "out of memory", - } - - err := classifyDataframeQueryError(driverErr) - userErr, ok := dataframeerrors.AsUserError(err) - if !ok || userErr.Code() != string(dataframeerrors.CodeQueryBackendOutOfMemory) { - t.Fatalf("classifyDataframeQueryError() = %#v, want %s", userErr, dataframeerrors.CodeQueryBackendOutOfMemory) - } -} - -func TestClassifyDataframeQueryErrorPreservesOtherArangoResourceLimit(t *testing.T) { - driverErr := shared.ArangoError{ - HasError: true, - Code: 500, - ErrorNum: shared.ErrResourceLimit, - ErrorMessage: "resource limit exceeded", - } - - err := classifyDataframeQueryError(driverErr) - userErr, ok := dataframeerrors.AsUserError(err) - if !ok || userErr.Code() != string(dataframeerrors.CodeQueryResourceLimitExceeded) { - t.Fatalf("classifyDataframeQueryError() = %#v, want %s", userErr, dataframeerrors.CodeQueryResourceLimitExceeded) + HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, + ErrorMessage: "AQL: CONSTRUCTION_EXPANSION_EMPTY: related expansion construction related_1 has no related records for row row-17 (while executing)", } -} - -func TestClassifyDataframeQueryErrorLeavesOtherFailuresUntouched(t *testing.T) { - want := errors.New("visitor failed") - if got := classifyDataframeQueryError(want); !errors.Is(got, want) || got != want { - t.Fatalf("classifyDataframeQueryError() = %v, want original error", got) + err := preserveConfiguredQueryError(classifyThroughAdapter(t, driverErr)) + var routeErr *explorer.AuthoringError + if !errors.As(err, &routeErr) || routeErr.Status != http.StatusUnprocessableEntity || + routeErr.Diagnostic.Code != string(dataframeerrors.CodeConstructionExpansionEmpty) { + t.Fatalf("preview route error = %#v, want non-retryable construction expansion diagnostic with HTTP 422", routeErr) } -} - -func TestClassifyDataframeQueryErrorPreservesRelationshipCardinalityViolation(t *testing.T) { - driverErr := shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: RELATIONSHIP_CARDINALITY_VIOLATION (while executing)"} - userErr, ok := dataframeerrors.AsUserError(classifyDataframeQueryError(driverErr)) - if !ok || userErr.Code() != string(dataframeerrors.CodeRelationshipCardinalityViolation) || userErr.Retryable() { - t.Fatalf("classified error=%#v", userErr) + if !errors.Is(err, driverErr) { + t.Fatal("adapter-classified error lost the driver cause") } } -func TestClassifyDataframeQueryErrorPreservesUnlistedPivotCategory(t *testing.T) { - driverErr := shared.ArangoError{ - HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, - ErrorMessage: "AQL: TABLE_PIVOT_UNLISTED_CATEGORY (while executing)", - } - err := classifyDataframeQueryError(driverErr) +func TestPreserveConfiguredQueryErrorRoutesGroupedValueMultiplicity(t *testing.T) { + cause := shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: CONSTRUCTION_ROW_VALUE_MULTIPLE_VALUES (while executing)"} + err := preserveConfiguredQueryError(classifyThroughAdapter(t, cause)) userErr, ok := dataframeerrors.AsUserError(err) - if !ok || userErr.Code() != "TABLE_PIVOT_UNLISTED_CATEGORY" || userErr.Retryable() { - t.Fatalf("classified error=%#v, want non-retryable TABLE_PIVOT_UNLISTED_CATEGORY", userErr) + if !ok || userErr.Code() != string(dataframeerrors.CodeConstructionRowValueMultipleValues) || userErr.Retryable() { + t.Fatalf("expected actionable grouped-value error, got %v", err) } - if !errors.Is(err, driverErr) { - t.Fatal("classified error did not preserve the Arango cause") + var routeErr *explorer.AuthoringError + if !errors.As(err, &routeErr) || routeErr.Status != 422 { + t.Fatalf("expected 422 grouped-value error, got %v", routeErr) + } + if !errors.Is(err, cause) { + t.Fatal("driver cause lost") } } -func TestClassifyDataframeQueryErrorPreservesPivotCellCardinality(t *testing.T) { +func TestPreserveConfiguredQueryErrorRoutesPivotCellCardinality(t *testing.T) { driverErr := shared.ArangoError{ HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: TABLE_PIVOT_CELL_CARDINALITY (while executing)", } - err := classifyDataframeQueryError(driverErr) + err := preserveConfiguredQueryError(classifyThroughAdapter(t, driverErr)) userErr, ok := dataframeerrors.AsUserError(err) if !ok || userErr.Code() != string(dataframeerrors.CodeTablePivotCellCardinality) || userErr.Retryable() { - t.Fatalf("classified error=%#v, want non-retryable %s", userErr, dataframeerrors.CodeTablePivotCellCardinality) - } - if !errors.Is(err, driverErr) { - t.Fatal("classified error did not preserve the Arango cause") + t.Fatalf("classified error=%#v, want non-retryable %s", err, dataframeerrors.CodeTablePivotCellCardinality) } } -func TestClassifyDataframeQueryErrorPreservesEmptyConstructionExpansion(t *testing.T) { +func TestPreserveConfiguredQueryErrorPreservesUnlistedPivotCategory(t *testing.T) { driverErr := shared.ArangoError{ HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, - ErrorMessage: "AQL: CONSTRUCTION_EXPANSION_EMPTY: related expansion construction related_1 has no related records for row row-17 (while executing)", + ErrorMessage: "AQL: TABLE_PIVOT_UNLISTED_CATEGORY (while executing)", } - err := classifyDataframeQueryError(driverErr) + err := preserveConfiguredQueryError(classifyThroughAdapter(t, driverErr)) userErr, ok := dataframeerrors.AsUserError(err) - if !ok || userErr.Code() != string(dataframeerrors.CodeConstructionExpansionEmpty) || userErr.Retryable() { - t.Fatalf("classified error=%#v, want non-retryable %s", userErr, dataframeerrors.CodeConstructionExpansionEmpty) + if !ok || userErr.Code() != "TABLE_PIVOT_UNLISTED_CATEGORY" || userErr.Retryable() { + t.Fatalf("classified error=%#v, want non-retryable TABLE_PIVOT_UNLISTED_CATEGORY", err) } if !errors.Is(err, driverErr) { t.Fatal("classified error did not preserve the Arango cause") } - var routeErr *explorer.AuthoringError - if !errors.As(previewRouteError(err), &routeErr) || routeErr.Status != http.StatusUnprocessableEntity || - routeErr.Diagnostic.Code != string(dataframeerrors.CodeConstructionExpansionEmpty) { - t.Fatalf("preview route error = %#v, want non-retryable construction expansion diagnostic with HTTP 422", routeErr) +} + +func TestPreserveConfiguredQueryErrorRoutesTemporalAssertions(t *testing.T) { + for _, code := range []dataframeerrors.ErrorCode{ + dataframeerrors.CodeTemporalAnchorInvalid, + dataframeerrors.CodeTemporalPrecisionUnsupported, + dataframeerrors.CodeTemporalTieAmbiguous, + dataframeerrors.CodeUnitIdentityUnknown, + dataframeerrors.CodeUnitDimensionIncompatible, + dataframeerrors.CodeExplicitGroupUnassignedMember, + dataframeerrors.CodeRelationshipCardinalityViolation, + } { + t.Run(string(code), func(t *testing.T) { + driverErr := shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: " + string(code) + " (while executing)"} + userErr, ok := dataframeerrors.AsUserError(preserveConfiguredQueryError(classifyThroughAdapter(t, driverErr))) + if !ok || userErr.Code() != string(code) || userErr.Retryable() { + t.Fatalf("classified error=%#v", userErr) + } + }) } } -func TestClassifyDataframeQueryErrorPreservesUnassignedExplicitGroup(t *testing.T) { - driverErr := shared.ArangoError{ - HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, - ErrorMessage: "AQL: EXPLICIT_GROUP_UNASSIGNED_MEMBER (while executing)", +func TestPreserveConfiguredQueryErrorPreservesResourceLimitClassifications(t *testing.T) { + memory := shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, ErrorMessage: "AQL: query would use more memory than allowed"} + userErr, ok := dataframeerrors.AsUserError(preserveConfiguredQueryError(classifyThroughAdapter(t, memory))) + if !ok || userErr.Code() != string(dataframeerrors.CodeQueryMemoryLimitExceeded) { + t.Fatalf("memory-limit classification lost through composition: %#v", userErr) } - err := classifyDataframeQueryError(driverErr) - userErr, ok := dataframeerrors.AsUserError(err) - if !ok || userErr.Code() != string(dataframeerrors.CodeExplicitGroupUnassignedMember) || userErr.Retryable() { - t.Fatalf("classified error=%#v, want non-retryable %s", userErr, dataframeerrors.CodeExplicitGroupUnassignedMember) + if got := userErr.Details()["backend"]; got != "arangodb" { + t.Fatalf("backend = %v, want arangodb", got) } - if !errors.Is(err, driverErr) { - t.Fatal("classified error did not preserve the Arango cause") + if got := userErr.Details()["resource"]; got != "query_memory" { + t.Fatalf("resource = %v, want query_memory", got) + } + outOfMemory := shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrOutOfMemory, ErrorMessage: "out of memory"} + userErr, ok = dataframeerrors.AsUserError(preserveConfiguredQueryError(classifyThroughAdapter(t, outOfMemory))) + if !ok || userErr.Code() != string(dataframeerrors.CodeQueryBackendOutOfMemory) { + t.Fatalf("out-of-memory classification lost: %#v", userErr) + } + otherResource := shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, ErrorMessage: "resource limit exceeded"} + userErr, ok = dataframeerrors.AsUserError(preserveConfiguredQueryError(classifyThroughAdapter(t, otherResource))) + if !ok || userErr.Code() != string(dataframeerrors.CodeQueryResourceLimitExceeded) { + t.Fatalf("resource-limit classification lost: %#v", userErr) } } -func TestClassifyDataframeQueryErrorDoesNotInferEmptyExpansionFromMessage(t *testing.T) { - driverErr := shared.ArangoError{ +func TestPreserveConfiguredQueryErrorLeavesOtherFailuresUntouched(t *testing.T) { + want := errors.New("visitor failed") + if got := preserveConfiguredQueryError(want); !errors.Is(got, want) || got != want { + t.Fatalf("preserveConfiguredQueryError() = %v, want original error", got) + } + unrecognized := shared.ArangoError{ HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: row expansion construction expand_tags has no items for row row-17 (while executing)", } - if got := classifyDataframeQueryError(driverErr); got != driverErr { - t.Fatalf("classifyDataframeQueryError() = %v, want original unrecognized assertion", got) + wrapped := fmt.Errorf("arango query: %w", unrecognized) + if got := preserveConfiguredQueryError(wrapped); got != wrapped { + t.Fatalf("preserveConfiguredQueryError() = %v, want original unrecognized assertion", got) } } -func TestClassifyDataframeQueryErrorPreservesTemporalAssertions(t *testing.T) { - tests := []dataframeerrors.ErrorCode{ - dataframeerrors.CodeTemporalAnchorInvalid, - dataframeerrors.CodeTemporalPrecisionUnsupported, - dataframeerrors.CodeTemporalTieAmbiguous, +func TestPreserveConfiguredQueryErrorMapsAuthorizationCodes(t *testing.T) { + unauthenticated := dataframeerrors.NewError(dataframeerrors.CodeUnauthenticated, "") + var routeErr *explorer.AuthoringError + if err := preserveConfiguredQueryError(unauthenticated); !errors.As(err, &routeErr) || routeErr.Status != http.StatusUnauthorized { + t.Fatalf("unauthenticated routing = %#v, want 401 AuthoringError", err) } - for _, code := range tests { - t.Run(string(code), func(t *testing.T) { - driverErr := shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: " + string(code) + " (while executing)"} - userErr, ok := dataframeerrors.AsUserError(classifyDataframeQueryError(driverErr)) - if !ok || userErr.Code() != string(code) || userErr.Retryable() { - t.Fatalf("classified error=%#v", userErr) - } - }) + forbidden := dataframeerrors.NewError(dataframeerrors.CodeForbidden, "") + if err := preserveConfiguredQueryError(forbidden); !errors.As(err, &routeErr) || routeErr.Status != http.StatusForbidden { + t.Fatalf("forbidden routing = %#v, want 403 AuthoringError", err) } } -func TestClassifyGroupedColumnMultiplicity(t *testing.T) { - cause := shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrQueryUserAssert, ErrorMessage: "AQL: CONSTRUCTION_ROW_VALUE_MULTIPLE_VALUES (while executing)"} - err := classifyDataframeQueryError(cause) - userErr, ok := dataframeerrors.AsUserError(err) - if !ok || userErr.Code() != string(dataframeerrors.CodeConstructionRowValueMultipleValues) || userErr.Retryable() { - t.Fatalf("expected actionable grouped-value error, got %v", err) +func TestConfiguredAQLQueryRowsWiresExecutionBudgetIntoArangoAdapter(t *testing.T) { + var captured context.Context + adapter := configuredAQLSourceRows(nil, "test_phase", func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + captured = ctx + return nil + }) + compiledCommand := testAQLCommand(t, "RETURN 1", nil) + + ctx := dataframeexecution.WithQueryMemoryBudget(context.Background(), 900000) + if err := adapter(ctx, dataframeexecution.RowsRequest{Command: compiledCommand, BatchSize: 10}, func(map[string]any) error { return nil }); err != nil { + t.Fatal(err) } - var routeErr *explorer.AuthoringError - if !errors.As(previewRouteError(err), &routeErr) || routeErr.Status != 422 { - t.Fatalf("expected 422 grouped-value error, got %v", routeErr) + if limit, ok := arangofactory.QueryMemoryLimitFromContext(captured); !ok || limit != 900000 { + t.Fatalf("adapter memory limit = %d/%t, want the execution budget 900000", limit, ok) } - if !errors.Is(err, cause) { - t.Fatal("driver cause lost") + + if err := adapter(context.Background(), dataframeexecution.RowsRequest{Command: compiledCommand, BatchSize: 10}, func(map[string]any) error { return nil }); err != nil { + t.Fatal(err) } + if limit, ok := arangofactory.QueryMemoryLimitFromContext(captured); ok { + t.Fatalf("adapter memory limit attached without an execution budget: %d", limit) + } +} + +func TestDiagnosticErrorCauseGoldenArangoRenderings(t *testing.T) { + t.Run("value_with_message", func(t *testing.T) { + err := fmt.Errorf("arango query: %w", shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, ErrorMessage: "boom"}) + if got := diagnosticErrorCause(err); got != "ArangoDB error number 32 (HTTP 500): boom" { + t.Fatalf("cause = %q", got) + } + }) + t.Run("pointer_without_message", func(t *testing.T) { + err := fmt.Errorf("prepare index: %w", &shared.ArangoError{HasError: true, Code: 400, ErrorNum: 1023}) + if got := diagnosticErrorCause(err); got != "ArangoDB error number 1023 (HTTP 400)" { + t.Fatalf("cause = %q", got) + } + }) + t.Run("typed_cause_wins_over_vendor_rendering", func(t *testing.T) { + classified := classifyThroughAdapter(t, shared.ArangoError{HasError: true, Code: 500, ErrorNum: shared.ErrResourceLimit, ErrorMessage: "AQL: query would use more memory than allowed"}) + if got := diagnosticErrorCause(classified); got != dataframeerrors.PublicMessage(classified) { + t.Fatalf("classified cause = %q, want the public message", got) + } + }) } diff --git a/internal/server/derived_policy_rows_integration_test.go b/internal/server/derived_policy_rows_integration_test.go index acf0bbfd6..2d24bd665 100644 --- a/internal/server/derived_policy_rows_integration_test.go +++ b/internal/server/derived_policy_rows_integration_test.go @@ -14,10 +14,11 @@ import ( "github.com/arangodb/go-driver/v2/arangodb/shared" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" + "github.com/calypr/loom/internal/dataframe/compiler" "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/expression" "github.com/calypr/loom/internal/dataframe/recipe" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -140,10 +141,11 @@ func TestDerivedArithmeticPoliciesAgainstArango(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() database := "loom_derived_policy_" + strings.ReplaceAll(uuid.NewString(), "-", "") - graph, err := storearango.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatal(err) } + graph := arangoBackend.Client t.Cleanup(func() { graph.Close(context.Background()) request, _ := http.NewRequest(http.MethodDelete, strings.TrimRight(endpoint, "/")+"/_api/database/"+database, nil) @@ -157,12 +159,20 @@ func TestDerivedArithmeticPoliciesAgainstArango(t *testing.T) { t.Errorf("owned derived database cleanup HTTP%d", response.StatusCode) } }) - if err = graph.Bootstrap(ctx, storearango.BootstrapSpec{Collections: []storearango.CollectionSpec{{Name: "Observation"}}}); err != nil { + if err = graph.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Observation"}}}); err != nil { t.Fatal(err) } - engine, err := execution.New(execution.Config{Registry: compilerTestRegistry{}, QueryRows: func(ctx context.Context, query string, batch int, binds map[string]any, visit func(map[string]any) error) error { - return graph.QueryRows(ctx, query, batch, binds, storearango.RowVisitor(visit)) - }}) + engine, err := execution.New(execution.Config{ + CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: func(ctx context.Context, request execution.RowsRequest, visit execution.RowVisitor) error { + command := request.Command + batch := request.BatchSize + query, binds, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + + return graph.QueryRows(ctx, query, batch, binds, arangofactory.RowVisitor(visit)) + }}) if err != nil { t.Fatal(err) } diff --git a/internal/server/explain_wiring.go b/internal/server/explain_wiring.go index 0c7a4926a..dd3d0d7ef 100644 --- a/internal/server/explain_wiring.go +++ b/internal/server/explain_wiring.go @@ -2,15 +2,120 @@ package server import ( "context" + "errors" + arangofactory "github.com/calypr/loom/internal/backends/arango" + "github.com/calypr/loom/internal/dataframe/compiler" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" - arangostore "github.com/calypr/loom/internal/store/arango" ) -func explainCompiledQuery(ctx context.Context, client *arangostore.Client, compiled dataframeexecution.CompiledQuery) (dataframeexecution.ExplainAssessment, error) { - result, err := client.Explain(ctx, arangostore.ExplainRequest{Query: compiled.Query, BindVars: compiled.BindVars}) +func explainCompiledQuery(ctx context.Context, client *arangofactory.Client, compiled compiler.CompiledQuery) (dataframeexecution.ExplainAssessment, error) { + query, bindVars, dispatchErr := compiled.Command.AQLParameters() + if dispatchErr != nil { + return dataframeexecution.ExplainAssessment{}, dispatchErr + } + result, err := client.Explain(ctx, arangofactory.ExplainRequest{Query: query, BindVars: bindVars}) if err != nil { return dataframeexecution.ExplainAssessment{}, err } - return dataframeexecution.AssessmentFromArango(arangostore.AssessExplainResult(result)), nil + return arangoAssessment(arangofactory.AssessExplainResult(result)), nil +} + +func arangoAssessment(input arangofactory.ExplainAssessment) dataframeexecution.ExplainAssessment { + out := dataframeexecution.ExplainAssessment{ + Plans: make([]dataframeexecution.ExplainPlanEstimate, 0, len(input.Plans)), + FullCollectionScans: make([]dataframeexecution.ExplainCollectionScan, 0, len(input.FullCollectionScans)), + Indexes: make([]dataframeexecution.ExplainIndexSummary, 0, len(input.Indexes)), + Warnings: make([]dataframeexecution.ExplainWarning, 0, len(input.Warnings)), + AppliedOptimizerRules: append([]string(nil), input.AppliedOptimizerRules...), + } + for _, plan := range input.Plans { + out.Plans = append(out.Plans, dataframeexecution.ExplainPlanEstimate{Plan: plan.Plan, EstimatedCost: plan.EstimatedCost, EstimatedNrItems: plan.EstimatedNrItems}) + } + for _, scan := range input.FullCollectionScans { + out.FullCollectionScans = append(out.FullCollectionScans, dataframeexecution.ExplainCollectionScan{Plan: scan.Plan, NodeID: scan.NodeID, Collection: scan.Collection}) + } + for _, index := range input.Indexes { + copyIndex := dataframeexecution.ExplainIndexSummary{Collection: index.Collection, ID: index.ID, Name: index.Name, Type: index.Type, Fields: append([]string(nil), index.Fields...), Uses: make([]dataframeexecution.ExplainIndexLocation, 0, len(index.Uses))} + for _, use := range index.Uses { + copyIndex.Uses = append(copyIndex.Uses, dataframeexecution.ExplainIndexLocation{Plan: use.Plan, NodeID: use.NodeID}) + } + out.Indexes = append(out.Indexes, copyIndex) + } + for _, warning := range input.Warnings { + out.Warnings = append(out.Warnings, dataframeexecution.ExplainWarning{Code: warning.Code, Message: warning.Message}) + } + return out +} + +func PreviewCollectionScanEvidenceFromExplain(result arangofactory.ExplainResult, collection string) dataframeexecution.PreviewCollectionScanEvidence { + var plan *arangofactory.ExplainPlan + switch { + case result.Plan != nil && len(result.Plans) == 0: + plan = result.Plan + case result.Plan == nil && len(result.Plans) == 1: + plan = &result.Plans[0] + default: + return dataframeexecution.PreviewCollectionScanEvidence{} + } + evidence := dataframeexecution.PreviewCollectionScanEvidence{} + for _, node := range plan.Nodes { + if node.Collection != collection { + continue + } + switch node.Type { + case "EnumerateCollectionNode": + evidence.FullCollectionEnumeration = true + case "IndexNode": + evidence.IndexNodes = append(evidence.IndexNodes, previewIndexNodeEvidence(node, collection)) + } + } + return evidence +} + +func previewIndexNodeEvidence(node arangofactory.ExplainNode, collection string) dataframeexecution.PreviewIndexNodeEvidence { + evidence := dataframeexecution.PreviewIndexNodeEvidence{Coverage: dataframeexecution.PreviewIndexCoverageUnknown} + if node.IndexCoversProjections != nil && node.IndexCoversOutProjections != nil { + switch { + case *node.IndexCoversProjections && *node.IndexCoversOutProjections: + evidence.Coverage = dataframeexecution.PreviewIndexCoverageCovering + case !*node.IndexCoversProjections && !*node.IndexCoversOutProjections: + evidence.Coverage = dataframeexecution.PreviewIndexCoverageNotCovering + } + } + for _, index := range node.Indexes { + if index.Collection == "" || index.Collection == collection { + evidence.IndexNames = append(evidence.IndexNames, index.Name) + } + } + return evidence +} + +func previewCollectionIndexesFromStore(indexes []arangofactory.PreviewCoveringIndex) []dataframeexecution.PreviewCollectionIndex { + if indexes == nil { + return nil + } + out := make([]dataframeexecution.PreviewCollectionIndex, 0, len(indexes)) + for _, index := range indexes { + out = append(out, dataframeexecution.PreviewCollectionIndex{ + Collection: index.Collection, + Name: index.Name, + Fields: index.Fields, + StoredValues: index.StoredValues, + }) + } + return out +} + +func explainPostgreSQLCompiledQuery(ctx context.Context, executor dataframeexecution.PostgreSQLPlanJSONExecutor, compiled compiler.CompiledQuery) (dataframeexecution.PostgreSQLPlanAssessment, error) { + if executor == nil { + return dataframeexecution.PostgreSQLPlanAssessment{}, dataframeerrors.Wrap( + errors.New("live PostgreSQL recipe Explain requires a PostgreSQL connection; the PostgreSQL source runtime is not wired"), + dataframeerrors.CodeBackendUnavailable, "live PostgreSQL recipe Explain is unavailable", + dataframeerrors.WithRetryable(false), + dataframeerrors.WithDetails(map[string]any{"backend": "postgresql", "stage": "wiring"}), + ) + } + return dataframeexecution.PostgreSQLExplainPlan(ctx, executor, compiled) } diff --git a/internal/server/explain_wiring_pg18_test.go b/internal/server/explain_wiring_pg18_test.go new file mode 100644 index 000000000..b77675a9b --- /dev/null +++ b/internal/server/explain_wiring_pg18_test.go @@ -0,0 +1,515 @@ +package server + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "sync" + "testing" + "time" + + postgres "github.com/calypr/loom/internal/backends/postgres" + "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + exec10wPrefix = "verify_exec10w_" + exec10wNameFmt = exec10wPrefix + "%d_%s" +) + +var exec10wNamePattern = regexp.MustCompile(`^verify_exec10w_[0-9]+_[0-9a-f]+$`) + +const ( + exec10wTable = "pat_SENTINEL-TBL-9c14" + exec10wIndex = "pat_resource_seq_SENTINEL-IDX-4e02" + exec10wColumn = "note_SENTINEL-COL-7a51" + exec10wLiteral = "SENTINEL-LIT-bf23" + exec10wBind = "SENTINEL-BIND-13ad" +) + +func exec10wSentinels() []string { + return []string{exec10wTable, exec10wIndex, exec10wColumn, exec10wLiteral, exec10wBind} +} + +var ( + exec10wOwnedMu sync.Mutex + exec10wOwnedDBs = map[string]string{} + exec10wOwnedRole = map[string]bool{} +) + +func exec10wRegisterDB(db, schema string) { + exec10wOwnedMu.Lock() + defer exec10wOwnedMu.Unlock() + exec10wOwnedDBs[db] = schema +} + +func exec10wRegisterRole(role string) { + exec10wOwnedMu.Lock() + defer exec10wOwnedMu.Unlock() + exec10wOwnedRole[role] = true +} + +func exec10wSnapshot() (map[string]string, map[string]bool) { + exec10wOwnedMu.Lock() + defer exec10wOwnedMu.Unlock() + dbs := make(map[string]string, len(exec10wOwnedDBs)) + for k, v := range exec10wOwnedDBs { + dbs[k] = v + } + roles := make(map[string]bool, len(exec10wOwnedRole)) + for k, v := range exec10wOwnedRole { + roles[k] = v + } + return dbs, roles +} + +var ( + exec10wBaselineOnce sync.Once + exec10wBaselineDBs []string + exec10wBaselineRoles []string + exec10wBaselineErr error +) + +func exec10wCaptureBaseline(raw string) { + exec10wBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec10wBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec10w\\_%' ESCAPE '\\'") + if err != nil { + exec10wBaselineErr = fmt.Errorf("read database baseline: %w", err) + return + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + exec10wBaselineErr = fmt.Errorf("scan database baseline: %w", err) + return + } + exec10wBaselineDBs = append(exec10wBaselineDBs, name) + } + rows.Close() + roleRows, err := admin.Query(ctx, "SELECT rolname FROM pg_roles WHERE rolname LIKE 'verify\\_exec10w\\_%' ESCAPE '\\'") + if err != nil { + exec10wBaselineErr = fmt.Errorf("read role baseline: %w", err) + return + } + for roleRows.Next() { + var name string + if err := roleRows.Scan(&name); err != nil { + roleRows.Close() + exec10wBaselineErr = fmt.Errorf("scan role baseline: %w", err) + return + } + exec10wBaselineRoles = append(exec10wBaselineRoles, name) + } + roleRows.Close() + }) +} + +func exec10wURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 Explain wiring integration tests") + } + return raw +} + +func exec10wRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +type exec10wService struct { + ctx context.Context + iso *pgxpool.Pool + client *control.Client + database string + dsn string + schema string + secrets []string + fail func(step string, err error) +} + +func newExec10wService(t *testing.T) *exec10wService { + t.Helper() + raw := exec10wURL(t) + exec10wCaptureBaseline(raw) + if exec10wBaselineErr != nil { + t.Fatalf("capture pg_database/pg_roles baseline: %v", exec10wBaselineErr) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + secrets := []string{raw} + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + fail := func(step string, err error) { + text := fmt.Sprintf("%v", err) + for _, secret := range secrets { + if secret != "" { + text = strings.ReplaceAll(text, secret, "") + } + } + t.Fatalf("%s: %s", step, text) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + lockConn, err := admin.Acquire(ctx) + if err != nil { + fail("acquire serialization connection", err) + } + if _, err := lockConn.Exec(ctx, `SELECT pg_advisory_lock(hashtext('loom/pg-exec-10/live-mutation')::bigint)`); err != nil { + lockConn.Release() + fail("take PG mutation serialization lock", err) + } + t.Cleanup(func() { + _, _ = lockConn.Exec(context.WithoutCancel(ctx), `SELECT pg_advisory_unlock(hashtext('loom/pg-exec-10/live-mutation')::bigint)`) + lockConn.Release() + }) + + dbName := fmt.Sprintf(exec10wNameFmt, time.Now().UnixNano(), exec10wRandomHex(t)) + if !exec10wNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + var created int64 + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", dbName).Scan(&created); err != nil { + fail("verify isolated database creation", err) + } + if created != 1 { + t.Fatalf("isolated database %q did not verifiably come into existence", dbName) + } + exec10wRegisterDB(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), + `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", err) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + secrets = append(secrets, dsn, dbName) + + client, err := control.OpenPool(ctx, control.ClientConfig{DSN: dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 10 * time.Second}) + if err != nil { + fail("open readiness-checked PostgreSQL 18 pool", err) + } + t.Cleanup(client.Close) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect fixture writer", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(exec10wNameFmt, time.Now().UnixNano(), exec10wRandomHex(t)) + if !exec10wNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + exec10wRegisterDB(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), + `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", err) + } + }) + return &exec10wService{ctx: ctx, iso: iso, client: client, database: dbName, dsn: dsn, schema: schema, secrets: append(secrets, schema), fail: fail} +} + +func (s *exec10wService) exec(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := s.iso.Exec(s.ctx, sql, args...); err != nil { + s.fail("fixture exec", err) + } +} + +func (s *exec10wService) seedFixture(t *testing.T) string { + t.Helper() + table := `"` + s.schema + `"."` + exec10wTable + `"` + s.exec(t, `CREATE TABLE `+table+` (resource_seq bigint, "`+exec10wColumn+`" text)`) + s.exec(t, `CREATE INDEX "`+exec10wIndex+`" ON `+table+` (resource_seq)`) + s.exec(t, `INSERT INTO `+table+` SELECT i::bigint, 'note-' || i::text FROM generate_series(1, 500) AS i`) + s.exec(t, `ANALYZE `+table) + return table +} + +func (s *exec10wService) assertSentinelFree(t *testing.T, surface, value string) { + t.Helper() + for _, sentinel := range exec10wSentinels() { + if strings.Contains(value, sentinel) { + t.Fatalf("%s leaks hostile sentinel %q", surface, sentinel) + } + } + for _, secret := range s.secrets { + if secret != "" && strings.Contains(value, secret) { + t.Fatalf("%s leaks connection/schema secret material", surface) + } + } +} + +func TestPostgresExplainWiringNilExecutorFailsClosed(t *testing.T) { + compiledCommand, err := command.NewPostgreSQLCommand(`SELECT id FROM t WHERE x = $1`, + []command.PostgreSQLParameter{{Type: command.PostgreSQLParameterText, Value: "v"}}) + if err != nil { + t.Fatal(err) + } + _, err = explainPostgreSQLCompiledQuery(context.Background(), nil, compiler.CompiledQuery{Command: compiledCommand}) + userErr, ok := dataframeerrors.AsUserError(err) + if !ok { + t.Fatalf("unwired PostgreSQL Explain must fail with a typed error: %v", err) + } + if userErr.Code() != string(dataframeerrors.CodeBackendUnavailable) { + t.Fatalf("code: %s", userErr.Code()) + } + if userErr.Retryable() { + t.Fatal("an unwired runtime is not retryable until the runtime exists") + } + details, _ := json.Marshal(userErr.Details()) + if !strings.Contains(string(details), "postgresql") { + t.Fatalf("details must name the backend: %s", details) + } + if strings.Contains(err.Error(), "arangodb") || strings.Contains(strings.ToLower(err.Error()), "fallback") && !strings.Contains(err.Error(), "never") { + t.Fatalf("failure must not mention or offer an Arango fallback: %v", err) + } +} + +func TestPostgresExplainWiringCrossTagRefusals(t *testing.T) { + pgCommand, err := command.NewPostgreSQLCommand(`SELECT 1`, nil) + if err != nil { + t.Fatal(err) + } + if _, err := explainCompiledQuery(context.Background(), nil, compiler.CompiledQuery{Command: pgCommand}); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("PG-tagged command into the Arango surface: %v", err) + } + aqlCommand, err := command.NewAQLCommand("FOR d IN c RETURN d", nil) + if err != nil { + t.Fatal(err) + } + querierCalled := false + runner, err := postgres.NewPlanJSONExecutor(exec10wCountingQuerier{called: &querierCalled}) + if err != nil { + t.Fatal(err) + } + if _, err := explainPostgreSQLCompiledQuery(context.Background(), runner, compiler.CompiledQuery{Command: aqlCommand}); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("AQL-tagged command into the PostgreSQL surface: %v", err) + } + if querierCalled { + t.Fatal("cross-tag command reached the querier") + } +} + +type exec10wCountingQuerier struct{ called *bool } + +func (q exec10wCountingQuerier) Query(context.Context, string, ...any) (pgx.Rows, error) { + *q.called = true + return nil, errors.New("unexpected querier call") +} + +func TestPostgresExplainWiringLivePG18SanitizedPlan(t *testing.T) { + s := newExec10wService(t) + table := s.seedFixture(t) + conn, err := s.client.Acquire(s.ctx) + if err != nil { + s.fail("acquire pool connection", err) + } + defer conn.Release() + runner, err := postgres.NewPlanJSONExecutor(conn) + if err != nil { + s.fail("postgres.NewPlanJSONExecutor", err) + } + sqlText := fmt.Sprintf(`SELECT "%s" FROM %s WHERE "%s" = $1 AND "%s"::text <> '%s' ORDER BY resource_seq`, + exec10wColumn, table, exec10wColumn, exec10wColumn, exec10wLiteral) + compiledCommand, err := command.NewPostgreSQLCommand(sqlText, []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterText, Value: exec10wBind}, + }) + if err != nil { + t.Fatal(err) + } + assessment, err := explainPostgreSQLCompiledQuery(s.ctx, runner, compiler.CompiledQuery{Command: compiledCommand}) + if err != nil { + s.fail("live explain through server wiring", err) + } + if assessment.Backend != string(command.CompiledBackendPostgreSQL) { + t.Fatalf("backend tag: %q", assessment.Backend) + } + if len(assessment.Nodes) == 0 || len(assessment.FullScans) == 0 { + t.Fatalf("expected a sequential-scan plan: %+v", assessment) + } + rendered, err := json.Marshal(assessment) + if err != nil { + t.Fatal(err) + } + s.assertSentinelFree(t, "wired live assessment", string(rendered)) + indexedSQL := fmt.Sprintf(`SELECT resource_seq FROM %s WHERE resource_seq = $1 ORDER BY resource_seq`, table) + indexedCommand, err := command.NewPostgreSQLCommand(indexedSQL, []command.PostgreSQLParameter{ + {Type: command.PostgreSQLParameterBigint, Value: int64(42)}, + }) + if err != nil { + t.Fatal(err) + } + indexed, err := explainPostgreSQLCompiledQuery(s.ctx, runner, compiler.CompiledQuery{Command: indexedCommand}) + if err != nil { + s.fail("live indexed explain through server wiring", err) + } + if len(indexed.IndexScans) == 0 { + t.Fatalf("indexed fixture query must classify index scans: %+v", indexed) + } + renderedIndexed, _ := json.Marshal(indexed) + s.assertSentinelFree(t, "wired live indexed assessment", string(renderedIndexed)) + + role := fmt.Sprintf(exec10wPrefix+"r_%d_%s", time.Now().UnixNano(), exec10wRandomHex(t)) + if !regexp.MustCompile(`^verify_exec10w_r_[0-9]+_[0-9a-f]+$`).MatchString(role) { + t.Fatalf("role name %q outside the owned pattern", role) + } + if _, err := s.iso.Exec(s.ctx, `CREATE ROLE "`+role+`" NOSUPERUSER NOLOGIN`); err != nil { + s.fail("create role", err) + } + exec10wRegisterRole(role) + t.Cleanup(func() { + if _, err := s.iso.Exec(context.WithoutCancel(s.ctx), `REVOKE ALL ON `+table+` FROM "`+role+`"`); err != nil { + t.Errorf("revoke table from role %s: %v", role, err) + } + if _, err := s.iso.Exec(context.WithoutCancel(s.ctx), `DROP ROLE IF EXISTS "`+role+`"`); err != nil { + t.Errorf("drop role %s: %v", role, err) + } + }) + plain, err := pgx.Connect(s.ctx, s.dsn) + if err != nil { + s.fail("connect dedicated permission connection", err) + } + defer plain.Close(s.ctx) + if _, err := plain.Exec(s.ctx, `SET ROLE "`+role+`"`); err != nil { + s.fail("set role", err) + } + planExecutor, err := postgres.NewPlanJSONExecutor(plain) + if err != nil { + s.fail("build production plan executor", err) + } + _, err = explainPostgreSQLCompiledQuery(s.ctx, planExecutor, compiler.CompiledQuery{Command: indexedCommand}) + userErr, ok := dataframeerrors.AsUserError(err) + if !ok || userErr.Code() != string(dataframeerrors.CodeInternalError) { + t.Fatalf("permission-denied EXPLAIN must carry the boundary internal classification: %v", err) + } + var translated *control.TranslatedError + if !errors.As(err, &translated) || translated.Condition != control.ConditionServerRejected || translated.SQLState != "42501" { + t.Fatalf("permission-denied EXPLAIN boundary classification: %v", err) + } + s.assertSentinelFree(t, "permission error surface", err.Error()) +} + +func pgValues(args []command.PostgreSQLParameter) []any { + values := make([]any, 0, len(args)) + for _, arg := range args { + values = append(values, arg.Value) + } + return values +} + +func TestZZZeroLeftoverExec10WiringPostgresFixtures(t *testing.T) { + raw := exec10wURL(t) + exec10wCaptureBaseline(raw) + if exec10wBaselineErr != nil { + t.Fatalf("capture baseline: %v", exec10wBaselineErr) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer admin.Close() + dbs, roles := exec10wSnapshot() + for db := range dbs { + var exists bool + if err := admin.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_database WHERE datname = $1)`, db).Scan(&exists); err != nil { + t.Fatalf("check %s: %v", db, err) + } + if exists { + t.Fatalf("created database %s was not dropped", db) + } + } + for role := range roles { + var exists bool + if err := admin.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_roles WHERE rolname = $1)`, role).Scan(&exists); err != nil { + t.Fatalf("check role %s: %v", role, err) + } + if exists { + t.Fatalf("created role %s was not dropped", role) + } + } + count := func(query string, baseline []string, kind string) { + rows, err := admin.Query(ctx, query) + if err != nil { + t.Fatalf("%s prefix scan: %v", kind, err) + } + var survivors []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + t.Fatalf("%s prefix read: %v", kind, err) + } + survivors = append(survivors, name) + } + rows.Close() + if len(survivors) != len(baseline) { + t.Fatalf("%s verify_exec10w_* survivors beyond baseline: got %v, baseline %v", kind, survivors, baseline) + } + known := map[string]bool{} + for _, name := range baseline { + known[name] = true + } + for _, name := range survivors { + if !known[name] { + t.Fatalf("unregistered verify_exec10w_* %s survivor (fatal): %s", kind, name) + } + } + } + count("SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec10w\\_%' ESCAPE '\\'", exec10wBaselineDBs, "database") + count("SELECT rolname FROM pg_roles WHERE rolname LIKE 'verify\\_exec10w\\_%' ESCAPE '\\'", exec10wBaselineRoles, "role") +} diff --git a/internal/server/explain_wiring_preview_test.go b/internal/server/explain_wiring_preview_test.go new file mode 100644 index 000000000..a4f9800cc --- /dev/null +++ b/internal/server/explain_wiring_preview_test.go @@ -0,0 +1,213 @@ +package server + +import ( + "reflect" + "testing" + + arangofactory "github.com/calypr/loom/internal/backends/arango" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" +) + +func explainBool(value bool) *bool { return &value } + +func TestPreviewCollectionScanEvidenceFromExplainSinglePlanRule(t *testing.T) { + node := arangofactory.ExplainNode{Type: "IndexNode", Collection: "Observation", + IndexCoversProjections: explainBool(true), IndexCoversOutProjections: explainBool(true), + Indexes: arangofactory.ExplainIndexes{{Name: "loom_pivot_preview_a", Collection: "Observation"}}} + want := dataframeexecution.PreviewCollectionScanEvidence{IndexNodes: []dataframeexecution.PreviewIndexNodeEvidence{ + {IndexNames: []string{"loom_pivot_preview_a"}, Coverage: dataframeexecution.PreviewIndexCoverageCovering}, + }} + if got := PreviewCollectionScanEvidenceFromExplain(arangofactory.ExplainResult{Plan: &arangofactory.ExplainPlan{Nodes: []arangofactory.ExplainNode{node}}}, "Observation"); !reflect.DeepEqual(got, want) { + t.Fatalf("lone Plan evidence = %+v, want %+v", got, want) + } + if got := PreviewCollectionScanEvidenceFromExplain(arangofactory.ExplainResult{Plans: []arangofactory.ExplainPlan{{Nodes: []arangofactory.ExplainNode{node}}}}, "Observation"); !reflect.DeepEqual(got, want) { + t.Fatalf("lone alternative Plans evidence = %+v, want %+v", got, want) + } + for _, result := range []arangofactory.ExplainResult{ + {}, + {Plan: &arangofactory.ExplainPlan{Nodes: []arangofactory.ExplainNode{node}}, Plans: []arangofactory.ExplainPlan{{Nodes: []arangofactory.ExplainNode{node}}}}, + {Plans: []arangofactory.ExplainPlan{{Nodes: []arangofactory.ExplainNode{node}}, {Nodes: []arangofactory.ExplainNode{node}}}}, + } { + if got := PreviewCollectionScanEvidenceFromExplain(result, "Observation"); !reflect.DeepEqual(got, dataframeexecution.PreviewCollectionScanEvidence{}) { + t.Fatalf("ambiguous plan shape produced evidence: %+v from %+v", got, result) + } + } +} + +func TestPreviewCollectionScanEvidenceFromExplainFiltersToQueriedCollection(t *testing.T) { + result := arangofactory.ExplainResult{Plan: &arangofactory.ExplainPlan{Nodes: []arangofactory.ExplainNode{ + {Type: "EnumerateCollectionNode", Collection: "Patient"}, + {Type: "IndexNode", Collection: "Patient", IndexCoversProjections: explainBool(false), IndexCoversOutProjections: explainBool(false)}, + {Type: "IndexNode", Collection: "Observation", IndexCoversProjections: explainBool(false), IndexCoversOutProjections: explainBool(false)}, + }}} + got := PreviewCollectionScanEvidenceFromExplain(result, "Observation") + if got.FullCollectionEnumeration { + t.Fatal("an enumeration on another collection blocked the queried collection") + } + if len(got.IndexNodes) != 1 || got.IndexNodes[0].Coverage != dataframeexecution.PreviewIndexCoverageNotCovering { + t.Fatalf("index nodes from another collection leaked in: %+v", got.IndexNodes) + } + blocked := PreviewCollectionScanEvidenceFromExplain(arangofactory.ExplainResult{Plan: &arangofactory.ExplainPlan{Nodes: []arangofactory.ExplainNode{ + {Type: "IndexNode", Collection: "Observation", IndexCoversProjections: explainBool(true), IndexCoversOutProjections: explainBool(true)}, + {Type: "EnumerateCollectionNode", Collection: "Observation"}, + }}}, "Observation") + if !blocked.FullCollectionEnumeration { + t.Fatal("an enumeration on the queried collection must block every optimization") + } + ordered := PreviewCollectionScanEvidenceFromExplain(arangofactory.ExplainResult{Plan: &arangofactory.ExplainPlan{Nodes: []arangofactory.ExplainNode{ + {Type: "FilterNode", Collection: "Observation"}, + {Type: "IndexNode", Collection: "Observation"}, + {Type: "CalculationNode", Collection: "Observation"}, + {Type: "IndexNode", Collection: "Observation"}, + }}}, "Observation") + if len(ordered.IndexNodes) != 2 { + t.Fatalf("scan-meaningless nodes leaked or order changed: %+v", ordered.IndexNodes) + } +} + +func TestPreviewCollectionScanEvidenceFromExplainCoverageTriState(t *testing.T) { + cases := []struct { + name string + node arangofactory.ExplainNode + want dataframeexecution.PreviewIndexCoverage + }{ + {"both true", arangofactory.ExplainNode{IndexCoversProjections: explainBool(true), IndexCoversOutProjections: explainBool(true)}, dataframeexecution.PreviewIndexCoverageCovering}, + {"both false", arangofactory.ExplainNode{IndexCoversProjections: explainBool(false), IndexCoversOutProjections: explainBool(false)}, dataframeexecution.PreviewIndexCoverageNotCovering}, + {"mixed true false", arangofactory.ExplainNode{IndexCoversProjections: explainBool(true), IndexCoversOutProjections: explainBool(false)}, dataframeexecution.PreviewIndexCoverageUnknown}, + {"mixed false true", arangofactory.ExplainNode{IndexCoversProjections: explainBool(false), IndexCoversOutProjections: explainBool(true)}, dataframeexecution.PreviewIndexCoverageUnknown}, + {"both missing", arangofactory.ExplainNode{}, dataframeexecution.PreviewIndexCoverageUnknown}, + {"one missing", arangofactory.ExplainNode{IndexCoversProjections: explainBool(false)}, dataframeexecution.PreviewIndexCoverageUnknown}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + node := test.node + node.Type = "IndexNode" + node.Collection = "Observation" + got := PreviewCollectionScanEvidenceFromExplain(arangofactory.ExplainResult{Plan: &arangofactory.ExplainPlan{Nodes: []arangofactory.ExplainNode{node}}}, "Observation") + if len(got.IndexNodes) != 1 || got.IndexNodes[0].Coverage != test.want { + t.Fatalf("coverage = %+v, want %v", got.IndexNodes, test.want) + } + }) + } +} + +func TestPreviewCollectionScanEvidenceFromExplainFiltersIndexNamesToCollection(t *testing.T) { + result := arangofactory.ExplainResult{Plan: &arangofactory.ExplainPlan{Nodes: []arangofactory.ExplainNode{{ + Type: "IndexNode", Collection: "Observation", + IndexCoversProjections: explainBool(true), IndexCoversOutProjections: explainBool(true), + Indexes: arangofactory.ExplainIndexes{ + {Name: "loom_pivot_preview_named", Collection: "Observation"}, + {Name: "loom_pivot_preview_unowned"}, + {Name: "loom_pivot_preview_foreign", Collection: "Patient"}, + }, + }}}} + got := PreviewCollectionScanEvidenceFromExplain(result, "Observation") + if len(got.IndexNodes) != 1 { + t.Fatalf("index nodes = %+v", got.IndexNodes) + } + if want := []string{"loom_pivot_preview_named", "loom_pivot_preview_unowned"}; !reflect.DeepEqual(got.IndexNodes[0].IndexNames, want) { + t.Fatalf("index names = %v, want %v", got.IndexNodes[0].IndexNames, want) + } +} + +func TestArangoAssessmentCopiesStoreAssessmentFieldForField(t *testing.T) { + input := arangofactory.ExplainAssessment{ + Plans: []arangofactory.ExplainPlanEstimate{ + {Plan: 0, EstimatedCost: 12.5, EstimatedNrItems: 3}, + {Plan: 1, EstimatedCost: 100.25, EstimatedNrItems: 10}, + }, + FullCollectionScans: []arangofactory.ExplainCollectionScan{{Plan: 1, NodeID: 4, Collection: "Patient"}}, + Indexes: []arangofactory.ExplainIndexSummary{{ + Collection: "Patient", ID: "Patient/idx1", Name: "idx1", Type: "skiplist", + Fields: []string{"id"}, Uses: []arangofactory.ExplainIndexLocation{{Plan: 0, NodeID: 2}}, + }}, + Warnings: []arangofactory.ExplainWarning{{Code: 1004, Message: "full collection scan"}}, + AppliedOptimizerRules: []string{"move-calculations-up"}, + } + got := arangoAssessment(input) + want := dataframeexecution.ExplainAssessment{ + Plans: []dataframeexecution.ExplainPlanEstimate{ + {Plan: 0, EstimatedCost: 12.5, EstimatedNrItems: 3}, + {Plan: 1, EstimatedCost: 100.25, EstimatedNrItems: 10}, + }, + FullCollectionScans: []dataframeexecution.ExplainCollectionScan{{Plan: 1, NodeID: 4, Collection: "Patient"}}, + Indexes: []dataframeexecution.ExplainIndexSummary{{ + Collection: "Patient", ID: "Patient/idx1", Name: "idx1", Type: "skiplist", + Fields: []string{"id"}, Uses: []dataframeexecution.ExplainIndexLocation{{Plan: 0, NodeID: 2}}, + }}, + Warnings: []dataframeexecution.ExplainWarning{{Code: 1004, Message: "full collection scan"}}, + AppliedOptimizerRules: []string{"move-calculations-up"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("assessment translation drift: got %+v want %+v", got, want) + } + input.Plans[0].EstimatedCost = 999 + input.Indexes[0].Fields[0] = "mutated" + input.Indexes[0].Uses[0].NodeID = 77 + if got.Plans[0].EstimatedCost != 12.5 || got.Indexes[0].Fields[0] != "id" || got.Indexes[0].Uses[0].NodeID != 2 { + t.Fatalf("assessment shared mutable state with the store: %+v", got) + } + empty := arangoAssessment(arangofactory.ExplainAssessment{}) + if empty.Plans == nil || empty.FullCollectionScans == nil || empty.Indexes == nil || empty.Warnings == nil { + t.Fatalf("empty assessment lost the non-nil list shape: %+v", empty) + } + if empty.AppliedOptimizerRules != nil { + t.Fatalf("empty rules must stay nil, matching the removed conversion: %v", empty.AppliedOptimizerRules) + } +} + +func TestPreviewCollectionIndexesFromStoreConvertsFieldForField(t *testing.T) { + if got := previewCollectionIndexesFromStore(nil); got != nil { + t.Fatalf("nil listing converted to %+v, want nil", got) + } + indexes := []arangofactory.PreviewCoveringIndex{{ + Collection: "Observation", Name: "loom_pivot_preview_a", + Fields: []string{"project", "dataset_generation"}, StoredValues: []string{"payload.status"}, + }} + got := previewCollectionIndexesFromStore(indexes) + want := []dataframeexecution.PreviewCollectionIndex{{ + Collection: "Observation", Name: "loom_pivot_preview_a", + Fields: []string{"project", "dataset_generation"}, StoredValues: []string{"payload.status"}, + }} + if !reflect.DeepEqual(got, want) { + t.Fatalf("listing conversion = %+v, want %+v", got, want) + } +} + +func TestPreviewCollectionScanEvidenceFromExplainUnknownNodeKindNeverImpliesEnumeration(t *testing.T) { + for _, nodeType := range []string{ + "FilterNode", "CalculationNode", "SortNode", "LimitNode", + "MaterializedDocumentSourceNode", "RemoteExecutionNode", + "EnumerationNode", "IndexNodeAlias", "FetchNode", "WindowNode", + "unknown-kind", "", + } { + got := PreviewCollectionScanEvidenceFromExplain(arangofactory.ExplainResult{Plan: &arangofactory.ExplainPlan{ + Nodes: []arangofactory.ExplainNode{{Type: nodeType, Collection: "Observation"}}, + }}, "Observation") + if got.FullCollectionEnumeration { + t.Fatalf("node kind %q implicitly certified a full collection enumeration: %+v", nodeType, got) + } + if len(got.IndexNodes) != 0 { + t.Fatalf("node kind %q produced index-node evidence: %+v", nodeType, got.IndexNodes) + } + } + mixed := PreviewCollectionScanEvidenceFromExplain(arangofactory.ExplainResult{Plan: &arangofactory.ExplainPlan{ + Nodes: []arangofactory.ExplainNode{ + {Type: "FilterNode", Collection: "Observation"}, + {Type: "ForeignScanNode", Collection: "Observation"}, + {Type: "IndexNode", Collection: "Observation", IndexCoversProjections: explainBool(true), IndexCoversOutProjections: explainBool(true)}, + }, + }}, "Observation") + if mixed.FullCollectionEnumeration || len(mixed.IndexNodes) != 1 || mixed.IndexNodes[0].Coverage != dataframeexecution.PreviewIndexCoverageCovering { + t.Fatalf("unknown kinds rode into the evidence alongside a covering index node: %+v", mixed) + } + enumerated := PreviewCollectionScanEvidenceFromExplain(arangofactory.ExplainResult{Plan: &arangofactory.ExplainPlan{ + Nodes: []arangofactory.ExplainNode{ + {Type: "WindowNode", Collection: "Observation"}, + {Type: "EnumerateCollectionNode", Collection: "Observation"}, + }, + }}, "Observation") + if !enumerated.FullCollectionEnumeration { + t.Fatalf("the only certified enumeration source stopped setting the fact: %+v", enumerated) + } +} diff --git a/internal/server/explorer_capability.go b/internal/server/explorer_capability.go index 160955d07..9a37272ae 100644 --- a/internal/server/explorer_capability.go +++ b/internal/server/explorer_capability.go @@ -39,10 +39,6 @@ func explorerScopeDigest(scope authscope.ReadScope) string { return hex.EncodeToString(sum[:]) } -// explorerCapabilityResolver is the one construction boundary for Builder -// capabilities. Catalog observations are evidence only; every public item is -// intersected with the real dataframe compiler before the immutable snapshot -// is persisted or returned. type explorerCapabilityResolver struct { evidence catalog.CapabilityEvidenceReader scopes *authscope.ScopeResolver @@ -62,9 +58,6 @@ func (r *explorerCapabilityResolver) Resolve(ctx context.Context, project, reque project = projectid.Canonical(project) storageProject := projectid.Legacy(project) principal, _ := authscope.PrincipalFromContext(ctx) - // Project membership is independent of whether generation-level Fence - // scopes are configured. A nil ScopeResolver means unrestricted dataframe - // reads, not that a principal's explicit project allowlist is ignored. if err := authscope.AuthorizeProject(principal, project, false); err != nil { return capability.Snapshot{}, err } @@ -158,9 +151,6 @@ func (r *explorerCapabilityResolver) Resolve(ctx context.Context, project, reque return value.(capability.Snapshot).Clone(), nil } -// ResolveToken loads an exact retained snapshot. It never substitutes the -// current active generation, and it re-authorizes the caller against the -// snapshot's generation and exact authorization-scope digest. func (r *explorerCapabilityResolver) ResolveToken(ctx context.Context, project, token string) (capability.Snapshot, error) { authorized, err := r.ResolveForExecution(ctx, project, token) if err != nil { @@ -169,11 +159,6 @@ func (r *explorerCapabilityResolver) ResolveToken(ctx context.Context, project, return authorized.Snapshot.Clone(), nil } -// ResolveForExecution loads an exact retained snapshot and its effective -// generation-scoped read authorization. Retention is intentional here: -// preview and other execution paths may use an inactive generation while its -// immutable capability remains available. The caller's current effective -// scope must still match the scope digest captured by the snapshot. func (r *explorerCapabilityResolver) ResolveForExecution(ctx context.Context, project, token string) (lifecycle.AuthorizedCapability, error) { project = projectid.Canonical(project) token = strings.TrimSpace(token) @@ -201,10 +186,6 @@ func (r *explorerCapabilityResolver) ResolveForExecution(ctx context.Context, pr return lifecycle.AuthorizedCapability{Snapshot: snapshot.Clone(), Scope: scope.Clone()}, nil } -// ResolveForCompilation loads an exact token for a new compile. Unlike -// execution, compilation is only valid against the currently active immutable -// dataset generation; a retained inactive token is rejected even when its -// scope is otherwise still authorized. func (r *explorerCapabilityResolver) ResolveForCompilation(ctx context.Context, project, token string) (lifecycle.AuthorizedCapability, error) { project = projectid.Canonical(project) authorized, err := r.ResolveForExecution(ctx, project, token) @@ -255,11 +236,6 @@ func requireCompleteCapabilityEvidence(value catalog.CapabilityEvidence) error { for _, item := range value.ResourceInventory.Values { documents += item.DocumentCount } - // A staged generation proves the catalog write phase completed, and every - // non-empty FHIR resource necessarily profiles at least its identity fields. - // Therefore a populated inventory with zero enrichment rows is a missing - // artifact, not a legitimate empty catalog (notably for older generations - // created before capability evidence existed). if documents > 0 && len(value.FieldEnrichment.Values) == 0 { return fmt.Errorf("%w: field enrichment artifact is empty for a populated generation", capability.ErrSnapshotUnavailable) } @@ -327,11 +303,6 @@ func capabilityEvidenceFromCatalog(value catalog.CapabilityEvidence) capability. observed.Label, observed.StorageDirection, observed.BuilderDirection = item.Label, item.StorageDirection, builderDirection observed.ObservedEdgeCount += item.EdgeCount observedRelationships[observedKey] = observed - // A stored FHIR reference is navigable in both directions. Catalog - // evidence retains the physical OUTBOUND orientation, while the - // dataframe compiler proves whether either generated traversal is - // legal. Publish both observations here and let ProbeEdge remove any - // direction the generated schema cannot lower. for _, route := range []struct { from, to, direction string }{ @@ -443,9 +414,6 @@ func capabilityEvidenceFromCatalog(value catalog.CapabilityEvidence) capability. } metadata, scalar := fhirschema.ResolveTerminalScalarMetadata(resourceType, targetPath) if !scalar || metadata.Primitive == fhirschema.PrimitiveUnknown { - // A semantic observation with no generated scalar value path is - // retained only in the source catalog. Never turn an object or - // unresolved structural path into a projectable capability. continue } maxPopulation := int64(0) @@ -481,9 +449,6 @@ func capabilityEvidenceFromCatalog(value catalog.CapabilityEvidence) capability. for _, aggregate := range fields { key := aggregate.observation.ResourceType + "\x00" + aggregate.observation.Path if _, semanticOwner := semanticOwnerFields[key]; semanticOwner { - // The repeated object is an evidence owner, not a scalar Builder - // candidate. Its concepts are attached to the checked value path; - // its maxItems remains available to boundary derivation above. continue } for suggestion := range aggregate.values { @@ -508,11 +473,6 @@ func capabilityEvidenceFromCatalog(value catalog.CapabilityEvidence) capability. return out } -// semanticConceptValuePath returns the generated scalar field that carries a -// concept's value. Concept evidence is often discovered on an object owner -// such as Observation.component[]; that owner is not itself a projectable -// capability candidate. The owning scope and value selector preserve the -// original pairing without advertising the object as scalar. func semanticConceptValuePath(observation catalog.SemanticObservation) string { selector := strings.Trim(strings.TrimSpace(observation.Value.Selector), ".") selector = strings.TrimPrefix(selector, "root.") @@ -578,10 +538,6 @@ func (c explorerCapabilityCompiler) ProbeCandidate(ctx context.Context, candidat if len(proof.ChartAggregations) > 0 { proof.SupportedOperations = append(proof.SupportedOperations, capability.OperationChart, capability.OperationAggregate) } - // Catalog candidates are route-independent. Advertise operations that the - // field can support when placed on a related occurrence; command validation - // derives the same contract again from the selected occurrence and rejects - // related-only operations at the row root. input := capability.AggregateInput{ LogicalType: string(result.Candidate.Primitive), Cardinality: string(result.Candidate.Cardinality), @@ -839,10 +795,6 @@ func authoringCatalogAvailability(value *capability.Availability) *authoringv2.C } func stringProjectionModes(values []capability.ProjectionMode) []string { - // ALL and DISTINCT are stable authoring-v2 spellings. Legacy workspaces - // keep decoding ALL as array projection; DISTINCT is only needed when the - // capability snapshot can prove distinct-array semantics, so no persisted - // workspace version migration is needed here. out := make([]string, len(values)) for i, value := range values { switch value { diff --git a/internal/server/explorer_capability_test.go b/internal/server/explorer_capability_test.go index 6a4f404e1..1e79b3066 100644 --- a/internal/server/explorer_capability_test.go +++ b/internal/server/explorer_capability_test.go @@ -588,8 +588,6 @@ func TestExplorerCapabilityResolverRejectsChangedScopeAndPreservesRestrictedEmpt if authorized.Scope.Mode != emptyScope.Mode { t.Fatalf("restricted-empty mode = %q, want %q", authorized.Scope.Mode, emptyScope.Mode) } - // The returned scope is a defensive copy, not the resolver's retained - // authorization result. authorized.Scope.AuthResourcePaths = append(authorized.Scope.AuthResourcePaths, "mutated") again, err := scopedResolver.ResolveForExecution(ctx, "project-a", empty.Token) if err != nil || len(again.Scope.AuthResourcePaths) != 0 || again.Scope.Unrestricted() { @@ -740,9 +738,6 @@ func TestExplorerCapabilityCacheSeparatesFieldCoverageReadiness(t *testing.T) { if len(known.Edges) == 0 || !sameCapabilityEntityIDs(known, legacy) { t.Fatalf("availability readiness changed compiler graph sizes: known=%#v legacy=%#v", known, legacy) } - // Before availability metadata was added, the entity ID seed used the - // semantic evidence identity: no cache version and a digest of field values - // alone. Rebuild that exact identity to catch stable-but-incompatible IDs. preAvailabilityIdentity := known.Identity preAvailabilityIdentity.CacheVersion = "" preAvailabilityIdentity.FieldDigest, err = catalog.FieldEnrichmentDigest(fields) diff --git a/internal/server/explorer_coded_value_capability_test.go b/internal/server/explorer_coded_value_capability_test.go index c37c78efc..2fba1587d 100644 --- a/internal/server/explorer_coded_value_capability_test.go +++ b/internal/server/explorer_coded_value_capability_test.go @@ -88,8 +88,6 @@ func TestCapabilityEvidenceSynthesizesCodeableConceptTextFieldFromProfiler(t *te t.Fatalf("synthesized scalar concept = %#v, want the profiler identity and text value selector with both examples", concept) } - // The bridge exposes the checked text scalar while catalog consumers retain - // the original blank-aware fallback on their semantic observation. var retained *catalog.SemanticObservation for _, field := range fieldValues { for index := range field.SemanticObservations { diff --git a/internal/server/explorer_config_route.go b/internal/server/explorer_config_route.go index 70c0f48b9..56b562ccb 100644 --- a/internal/server/explorer_config_route.go +++ b/internal/server/explorer_config_route.go @@ -10,9 +10,6 @@ import ( type explorerConfigReadAuthorizer func(context.Context, *authscope.Principal, string) error -// explorerHTTPHandlers holds the dependencies used by the generated OpenAPI -// Explorer adapter. Fiber routing is registered once, from the generated -// server, rather than through a parallel handwritten handler tree. type explorerHTTPHandlers struct { authorizer authscope.Authorizer authorizeRead explorerConfigReadAuthorizer diff --git a/internal/server/explorer_diagnostic_logging_test.go b/internal/server/explorer_diagnostic_logging_test.go index d1c85fc8a..17b50ce4c 100644 --- a/internal/server/explorer_diagnostic_logging_test.go +++ b/internal/server/explorer_diagnostic_logging_test.go @@ -36,9 +36,14 @@ func TestConfiguredAQLQueryRowsLogsSafeRequestCorrelatedRecords(t *testing.T) { query := "FOR d IN patients FILTER d.id == @secret RETURN d" bindVars := map[string]any{"secret": "private-bind-value"} failure := errors.New("arangodb unavailable: connection refused; query=RETURN @secret auth_token=private-auth-value") - gotErr := configuredAQLQueryRows(logger, phase, func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + gotErr := configuredAQLSourceRows(logger, phase, func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return failure - })(ctx, query, 16, bindVars, func(map[string]any) error { return nil }) + })(ctx, dataframeexecution.RowsRequest{Command: testAQLCommand(t, query, bindVars), BatchSize: 16}, func(map[string]any) error { return nil }) if gotErr != failure { t.Fatalf("callback error = %v, want original error", gotErr) } @@ -252,7 +257,12 @@ func TestConfiguredCategoryScannerUsesPivotTimeoutAndPreservesBackendErrors(t *t var queryDeadline time.Time var queryRuntime time.Duration - queryRows := func(ctx context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + queryRows := func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + var ok bool queryDeadline, ok = ctx.Deadline() if !ok { @@ -261,8 +271,8 @@ func TestConfiguredCategoryScannerUsesPivotTimeoutAndPreservesBackendErrors(t *t queryRuntime = previewQueryMaxRuntime(ctx) return visit(map[string]any{"present": true, "value": "patient-1"}) } - scanEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, QueryRows: queryRows, PreviewQueryRows: queryRows, + scanEngine, err := dataframeexecution.New(dataframeexecution.Config{CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, SourceRows: queryRows, PreviewSourceRows: queryRows, }) if err != nil { t.Fatalf("create category scan engine: %v", err) @@ -291,8 +301,14 @@ func TestConfiguredCategoryScannerUsesPivotTimeoutAndPreservesBackendErrors(t *t backendFailure := errors.New("synthetic category backend failure") failingEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return backendFailure }, }) @@ -392,8 +408,14 @@ func diagnosticCompileFixture(t *testing.T, compileRole string) (lifecycle.Compi t.Fatal(err) } engine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return visit(map[string]any{"present": true, "value": "patient-1"}) }, }) diff --git a/internal/server/explorer_output_reconcile.go b/internal/server/explorer_output_reconcile.go index 5e29b0765..c5ea0162c 100644 --- a/internal/server/explorer_output_reconcile.go +++ b/internal/server/explorer_output_reconcile.go @@ -74,8 +74,6 @@ func groupFieldSourceOutputIdentityForReconcile( identity.SourcePath = related.CodingSelector } case binding.Input == nil: - // Saved flat bindings are retained for direct-stage recipes written before - // the tagged source input was added. identity.CandidateID = binding.CandidateID identity.OccurrenceID = binding.OccurrenceID identity.SourceResourceType = binding.ResourceType @@ -401,11 +399,6 @@ func reconcileFinalOutputMetadataWithCombineInputs( return reconciled, nil } -// orderReconciledOutputColumns keeps physical execution order in the resolved -// compiler schema while making the durable receipt's public columns follow the -// authored table presentation order. The presentation records the original -// physical position as a deterministic tie breaker for legacy duplicate -// orders and synthesized outputs. func orderReconciledOutputColumns(contract *explorer.PublicOutputContract, presentation []explorercompilation.PresentationColumn, emitted []explorer.EmittedColumn) { tableOrder := make(map[string]int, len(presentation)) physicalOrder := make(map[string]int, len(presentation)) @@ -489,11 +482,6 @@ func explicitGroupEmissionID(revisionID, column string) string { return "explicit-group:" + revisionID + ":" + column } -// compiledFilterSourceSlotMatches ties a durable filter-owned projection to -// the compiled source-stage column the accepted plan materialized: same ID, -// derived public name, claimed FHIR path, logical kind, and label. Compiled -// slots mirror the accepted claim, so a durable claim that drifted from the -// compiled artifact of the plan that produced it fails here. func compiledFilterSourceSlotMatches(compiledOutput lower.CompiledRecipeOutput, projection authoringv2.ConstructionSourceProjection) bool { if len(compiledOutput.Stages) == 0 || compiledOutput.Stages[0].ID != recipe.ConstructionSourceProjectionID { return false @@ -511,22 +499,10 @@ func compiledFilterSourceSlotMatches(compiledOutput lower.CompiledRecipeOutput, return false } -// compiledDeriveSourceSlotMatches ties a durable derive-owned projection to -// the compiled source-stage column the accepted plan materialized: same ID, -// derived public name, claimed FHIR path, logical kind, and label. Compiled -// slots mirror the accepted claim, so a durable claim that drifted from the -// compiled artifact of the plan that produced it fails here. A signed source -// projection compiles through the same source-stage slot contract as a filter -// source, so both channels share one predicate. func compiledDeriveSourceSlotMatches(compiledOutput lower.CompiledRecipeOutput, projection authoringv2.ConstructionSourceProjection) bool { return compiledFilterSourceSlotMatches(compiledOutput, projection) } -// compiledUnpivotSourceSlotMatches ties a durable unpivot-owned projection to -// the compiled source-stage column the accepted plan materialized: same ID, -// derived public name, claimed FHIR path, logical kind, and label. A signed -// melt input compiles through the same source-stage slot contract as a filter -// or derive source, so the channel shares the one slot predicate. func compiledUnpivotSourceSlotMatches(compiledOutput lower.CompiledRecipeOutput, projection authoringv2.ConstructionSourceProjection) bool { return compiledFilterSourceSlotMatches(compiledOutput, projection) } @@ -665,6 +641,20 @@ func authoredConstructionOutputs(document authoringv2.Document, authored map[str return authoredConstructionOutputsWithCombineInputs(document, authored, lower.CompiledRecipeOutput{}, nil, capability.Snapshot{}) } +type constructionStepContext struct { + document authoringv2.Document + step authoringv2.ConstructionStep + stepIndex int + prior map[string]string + authored map[string]authoredOutputColumn + sourceOnlyIDs map[string]struct{} + compiledOutput lower.CompiledRecipeOutput + resolvedInputs map[int]ir.ResolvedClickHouseTable + rowSourceSnapshot capability.Snapshot + nextAuthored map[string]authoredOutputColumn + addOutput func(outputColumnID, constructionID string, inputIDs []string, quality constructedOutputQuality) error +} + func authoredConstructionOutputsWithCombineInputs( document authoringv2.Document, authored map[string]authoredOutputColumn, @@ -794,618 +784,749 @@ func authoredConstructionOutputsWithCombineInputs( } return nil } + stepContext := constructionStepContext{ + document: document, + step: step, + stepIndex: stepIndex, + prior: prior, + authored: authored, + sourceOnlyIDs: sourceOnlyIDs, + compiledOutput: compiledOutput, + resolvedInputs: resolvedInputs, + rowSourceSnapshot: rowSourceSnapshot, + nextAuthored: nextAuthored, + addOutput: addOutput, + } + var stepErr error switch step.Operation.Kind { case authoringv2.ConstructionOperationGroup: - if step.Operation.Group == nil { - return fmt.Errorf("group step %q has no operation payload", step.ID) - } - group := step.Operation.Group - quality := constructedOutputQuality{ - StructuralSuitability: "requires-review", - LossReasons: []string{"TABLE_SHAPE_GROUP_CHANGES_ROW_GRAIN"}, - } - for _, key := range group.Keys { - if key.SemanticCategory != nil { - category := key.SemanticCategory - if category.Kind != "BINDING" || strings.TrimSpace(category.BindingID) == "" || - strings.TrimSpace(category.FamilyID) == "" || strings.TrimSpace(category.ResourceType) == "" || - strings.TrimSpace(category.CodingPath) == "" { - return fmt.Errorf("step %q semantic Group output %q has no resolved binding identity", step.ID, key.OutputColumnID) - } - var compiledKey *lower.CompiledOutputColumn - for stageIndex := range compiledOutput.Stages { - stage := &compiledOutput.Stages[stageIndex] - if stage.ID != step.ID || stage.Operation != string(step.Operation.Kind) { - continue - } - for columnIndex := range stage.Columns { - column := &stage.Columns[columnIndex] - if column.ID == key.OutputColumnID { - if compiledKey != nil { - return fmt.Errorf("step %q semantic Group output %q has duplicate compiled schema identities", step.ID, key.OutputColumnID) - } - compiledKey = column - } - } - } - wantSemanticPath := "construction_semantic_group:" + category.BindingID + ":" + category.FamilyID - if compiledKey == nil || compiledKey.SemanticPath != wantSemanticPath { - return fmt.Errorf("step %q semantic Group output %q does not match its compiled binding metadata", step.ID, key.OutputColumnID) - } - if err := addOutput(key.OutputColumnID, group.ConstructionID, nil, quality); err != nil { - return err - } - metadata := nextAuthored[compiledKey.Name] - metadata.SourceResourceType = category.ResourceType - metadata.SourcePath = category.CodingPath - nextAuthored[compiledKey.Name] = metadata - continue - } - if key.FieldSource != nil { - binding := key.FieldSource - identity, ok := groupFieldSourceOutputIdentityForReconcile(binding) - if !ok { - return fmt.Errorf("step %q Group output %q has no resolved field-source binding identity", step.ID, key.OutputColumnID) - } - if err := addOutput(key.OutputColumnID, group.ConstructionID, nil, quality); err != nil { - return err - } - for _, output := range step.Outputs { - if output.ID != key.OutputColumnID { - continue - } - metadata := nextAuthored[output.Name] - metadata.CandidateID = identity.CandidateID - metadata.OccurrenceID = identity.OccurrenceID - metadata.SourceResourceType = identity.SourceResourceType - metadata.SourcePath = identity.SourcePath - nextAuthored[output.Name] = metadata - break - } - continue - } - if err := addOutput(key.OutputColumnID, group.ConstructionID, []string{key.InputColumnID}, quality); err != nil { - return err - } - inputName := prior[key.InputColumnID] - for _, output := range step.Outputs { - if output.ID == key.OutputColumnID && output.Name == inputName { - metadata := nextAuthored[output.Name] - metadata.PreservesSourceIdentity = true - nextAuthored[output.Name] = metadata - break - } - } - } - for _, aggregate := range group.Aggregates { - inputs := []string(nil) - if aggregate.Operation != authoringv2.ConstructionGroupCountRows { - inputs = []string{aggregate.InputColumnID} - } - if err := addOutput(aggregate.OutputColumnID, group.ConstructionID, inputs, quality); err != nil { - return err - } - } - for _, rowValue := range step.RowValues { - rowQuality := quality - if rowValue.Policy == authoringv2.ConstructionRowValueAll { - rowQuality.Shape = "array" - } - inputs := []string(nil) - var fieldSourceIdentity groupFieldSourceOutputIdentity - if rowValue.FieldSource != nil { - binding := rowValue.FieldSource - var ok bool - fieldSourceIdentity, ok = groupFieldSourceOutputIdentityForReconcile(binding) - if !ok || binding.ValueKind != "SCALAR" { - return fmt.Errorf("step %q Group row-value output %q has no resolved scalar field-source binding identity", step.ID, rowValue.OutputColumnID) - } - } else { - inputs = []string{rowValue.InputColumnID} - } - if err := addOutput(rowValue.OutputColumnID, group.ConstructionID, inputs, rowQuality); err != nil { - return err - } - if rowValue.FieldSource == nil { - continue - } - for _, output := range step.Outputs { - if output.ID != rowValue.OutputColumnID { - continue - } - metadata := nextAuthored[output.Name] - metadata.CandidateID = fieldSourceIdentity.CandidateID - metadata.OccurrenceID = fieldSourceIdentity.OccurrenceID - metadata.SourceResourceType = fieldSourceIdentity.SourceResourceType - metadata.SourcePath = fieldSourceIdentity.SourcePath - nextAuthored[output.Name] = metadata - break - } - } + nextAuthored, stepErr = applyGroupConstructionStep(&stepContext) case authoringv2.ConstructionOperationCodedGroup: - if step.Operation.CodedGroup == nil { - return fmt.Errorf("coded group step %q has no operation payload", step.ID) - } - group := step.Operation.CodedGroup - quality := constructedOutputQuality{ - Lossless: false, StructuralSuitability: "requires-review", - LossReasons: []string{"CODED_GROUP_REDUCES_TO_DISTINCT_SOURCE_RECORDS"}, - } - inputIDs := make([]string, 0, len(document.Columns)) - for _, column := range document.Columns { - if column.OccurrenceID == group.Source.OccurrenceID { - inputIDs = append(inputIDs, column.ColumnID) - } - } - if len(inputIDs) == 0 { - return fmt.Errorf("coded group step %q has no source columns on occurrence %q for lineage", step.ID, group.Source.OccurrenceID) - } - for _, output := range []struct { - id string - path string - }{ - {id: group.SystemOutputColumnID, path: group.Source.CodingPath + ".system"}, - {id: group.VersionOutputColumnID, path: group.Source.CodingPath + ".version"}, - {id: group.CodeOutputColumnID, path: group.Source.CodingPath + ".code"}, - {id: group.DistinctSourceCountOutputColumnID, path: group.Source.CodingPath}, - } { - if err := addOutput(output.id, group.ConstructionID, inputIDs, quality); err != nil { - return err - } - for _, declared := range step.Outputs { - if declared.ID == output.id { - metadata := nextAuthored[declared.Name] - metadata.OccurrenceID = group.Source.OccurrenceID - metadata.SourceResourceType = group.Source.ResourceType - metadata.SourcePath = output.path - nextAuthored[declared.Name] = metadata - break - } - } - } + nextAuthored, stepErr = applyCodedGroupConstructionStep(&stepContext) case authoringv2.ConstructionOperationExpand: - if step.Operation.Expand == nil { - return fmt.Errorf("expand step %q has no operation payload", step.ID) - } - expand := step.Operation.Expand - quality := constructedOutputQuality{ - StructuralSuitability: "requires-review", - LossReasons: []string{"TABLE_SHAPE_EXPAND_CHANGES_ROW_GRAIN"}, - } - if err := addOutput(expand.OutputColumnID, expand.ConstructionID, []string{expand.InputColumnID}, quality); err != nil { - return err - } - if expand.OrdinalColumnID != "" { - if err := addOutput(expand.OrdinalColumnID, expand.ConstructionID, []string{expand.InputColumnID}, quality); err != nil { - return err - } - } + nextAuthored, stepErr = applyExpandConstructionStep(&stepContext) case authoringv2.ConstructionOperationDerive: - if step.Operation.Derive == nil { - return fmt.Errorf("derive step %q has no operation payload", step.ID) - } - derive := step.Operation.Derive - inputs := make([]string, 0, 2) - for _, operand := range []authoringv2.ConstructionOperand{derive.Left, derive.Right} { - switch operand.Kind { - case authoringv2.ConstructionColumnOperand: - inputs = append(inputs, operand.ColumnID) - case authoringv2.ConstructionLiteralOperand: - default: - return fmt.Errorf("derive step %q has unsupported operand kind %q", step.ID, operand.Kind) - } - } - quality, err := derivedOutputQuality(string(derive.Operation)) - if err != nil { - return fmt.Errorf("derive step %q: %w", step.ID, err) - } - if err := addOutput(derive.OutputColumnID, derive.ConstructionID, inputs, quality); err != nil { - return err - } - for _, projection := range document.Construction.SourceProjections { - if projection.OwnerStepID != step.ID { - continue - } - consumed := false - for _, operand := range []authoringv2.ConstructionOperand{derive.Left, derive.Right} { - if operand.Kind == authoringv2.ConstructionColumnOperand && operand.ColumnID == projection.ColumnID { - consumed = true - } - } - if !consumed { - return fmt.Errorf("derive step %q carries source projection column %q it does not consume", step.ID, projection.ColumnID) - } - var declared *authoringv2.StageColumn - for outputIndex := range step.Outputs { - if step.Outputs[outputIndex].ID == projection.ColumnID { - declared = &step.Outputs[outputIndex] - break - } - } - if declared == nil || !requiredConstructionID(declared.ID) || strings.TrimSpace(declared.Name) == "" || strings.TrimSpace(declared.Label) == "" { - return fmt.Errorf("derive step %q source projection output %q is missing a declared stable ID, name, or label", step.ID, projection.ColumnID) - } - if prior[declared.ID] != declared.Name { - return fmt.Errorf("derive step %q source projection output %q does not carry the compiled source column identity", step.ID, declared.ID) - } - if !compiledDeriveSourceSlotMatches(compiledOutput, projection) { - return fmt.Errorf("derive step %q source projection output %q does not carry the compiled source column identity", step.ID, declared.ID) - } - if strings.TrimSpace(rowSourceSnapshot.Token) == "" { - return fmt.Errorf("derive step %q source projection output %q cannot be claimed without an authorized capability snapshot", step.ID, declared.ID) - } - derived, ok := lifecycle.ConstructionDeriveSourceFacts(rowSourceSnapshot, document.RootResourceType, projection.FieldPath) - expected := authoringv2.ConstructionSourceProjection{ - ColumnID: projection.ColumnID, OwnerStepID: projection.OwnerStepID, - OccurrenceID: derived.OccurrenceID, FieldPath: derived.FieldPath, - FHIRType: derived.FHIRType, LogicalType: derived.LogicalType, Label: derived.Label, - } - if !ok || expected != projection { - return fmt.Errorf("derive step %q source projection output %q no longer matches the authorized derive source derivation for field %q", step.ID, declared.ID, projection.FieldPath) - } - if _, duplicate := nextAuthored[declared.Name]; duplicate { - return fmt.Errorf("construction output %q is duplicated for document %q", declared.Name, document.Output.ID) - } - nextAuthored[declared.Name] = authoredOutputColumn{ - ConstructionID: declared.ID, Label: declared.Label, TypedStageOutput: true, - Quality: constructedOutputQuality{ - Lossless: false, StructuralSuitability: "requires-review", - LossReasons: []string{"CONSTRUCTION_DERIVE_SOURCE_ARITHMETIC_FROM_UNSELECTED_FIELD"}, - }, - } - } + nextAuthored, stepErr = applyDeriveConstructionStep(&stepContext) case authoringv2.ConstructionOperationPivot: - if step.Operation.Pivot == nil { - return fmt.Errorf("pivot step %q has no operation payload", step.ID) - } - pivot := step.Operation.Pivot - dependencies := append([]string(nil), pivot.GroupKeyIDs...) - dependencies = append(dependencies, pivot.CategoryColumnID, pivot.ValueColumnID) - quality, err := pivotOutputQuality(&authoringv2.PivotConstruction{ - DuplicatePolicy: string(pivot.DuplicatePolicy), MissingCellPolicy: string(pivot.MissingCellPolicy), - UnlistedCategoryPolicy: string(pivot.UnlistedCategoryPolicy), - }) - if err != nil { - return fmt.Errorf("pivot step %q: %w", step.ID, err) + nextAuthored, stepErr = applyPivotConstructionStep(&stepContext) + case authoringv2.ConstructionOperationCodedPivot: + nextAuthored, stepErr = applyCodedPivotConstructionStep(&stepContext) + case authoringv2.ConstructionOperationUnpivot: + nextAuthored, stepErr = applyUnpivotConstructionStep(&stepContext) + case authoringv2.ConstructionOperationFilter: + nextAuthored, stepErr = applyFilterConstructionStep(&stepContext) + case authoringv2.ConstructionOperationCombine: + nextAuthored, stepErr = applyCombineConstructionStep(&stepContext) + case authoringv2.ConstructionOperationRelatedSource: + nextAuthored, stepErr = applyRelatedSourceConstructionStep(&stepContext) + case authoringv2.ConstructionOperationRelatedExpand: + nextAuthored, stepErr = applyRelatedExpandConstructionStep(&stepContext) + case authoringv2.ConstructionOperationRelatedEligibility: + nextAuthored, stepErr = applyRelatedEligibilityConstructionStep(&stepContext) + case authoringv2.ConstructionOperationRelatedField: + nextAuthored, stepErr = applyRelatedFieldConstructionStep(&stepContext) + default: + return fmt.Errorf("step %q has unsupported operation kind %q", step.ID, step.Operation.Kind) + } + if stepErr != nil { + return stepErr + } + next := make(map[string]string, len(step.Outputs)) + for _, output := range step.Outputs { + if !requiredConstructionID(output.ID) || strings.TrimSpace(output.Name) == "" { + return fmt.Errorf("step %q output schema contains a missing stable ID or public name", step.ID) } - for _, keyID := range pivot.GroupKeyIDs { - _, privateSource := sourceOnlyIDs[keyID] - priorMetadata, constructedInput := authored[prior[keyID]] - if !privateSource && !constructedInput { + if _, duplicate := next[output.ID]; duplicate { + return fmt.Errorf("step %q output schema duplicates column ID %q", step.ID, output.ID) + } + next[output.ID] = output.Name + } + clear(authored) + for name, metadata := range nextAuthored { + authored[name] = metadata + } + prior = next + } + return nil +} + +func applyGroupConstructionStep(stepContext *constructionStepContext) (map[string]authoredOutputColumn, error) { + step := stepContext.step + prior := stepContext.prior + compiledOutput := stepContext.compiledOutput + nextAuthored := stepContext.nextAuthored + addOutput := stepContext.addOutput + if step.Operation.Group == nil { + return nil, fmt.Errorf("group step %q has no operation payload", step.ID) + } + group := step.Operation.Group + quality := constructedOutputQuality{ + StructuralSuitability: "requires-review", + LossReasons: []string{"TABLE_SHAPE_GROUP_CHANGES_ROW_GRAIN"}, + } + for _, key := range group.Keys { + if key.SemanticCategory != nil { + category := key.SemanticCategory + if category.Kind != "BINDING" || strings.TrimSpace(category.BindingID) == "" || + strings.TrimSpace(category.FamilyID) == "" || strings.TrimSpace(category.ResourceType) == "" || + strings.TrimSpace(category.CodingPath) == "" { + return nil, fmt.Errorf("step %q semantic Group output %q has no resolved binding identity", step.ID, key.OutputColumnID) + } + var compiledKey *lower.CompiledOutputColumn + for stageIndex := range compiledOutput.Stages { + stage := &compiledOutput.Stages[stageIndex] + if stage.ID != step.ID || stage.Operation != string(step.Operation.Kind) { continue } - for _, output := range step.Outputs { - if output.ID == keyID { - if privateSource { - priorMetadata = authoredOutputColumn{ - ConstructionID: pivot.ConstructionID, TypedStageOutput: true, Quality: quality, - } + for columnIndex := range stage.Columns { + column := &stage.Columns[columnIndex] + if column.ID == key.OutputColumnID { + if compiledKey != nil { + return nil, fmt.Errorf("step %q semantic Group output %q has duplicate compiled schema identities", step.ID, key.OutputColumnID) } - priorMetadata.Label = output.Label - nextAuthored[output.Name] = priorMetadata - break + compiledKey = column } } } - for _, category := range pivot.Categories { - if err := addOutput(category.OutputColumnID, pivot.ConstructionID, dependencies, quality); err != nil { - return err - } - } - case authoringv2.ConstructionOperationCodedPivot: - if step.Operation.CodedPivot == nil || step.Operation.CodedPivot.Source == nil { - return fmt.Errorf("coded pivot step %q has no source", step.ID) - } - pivot := step.Operation.CodedPivot - quality, err := pivotOutputQuality(&authoringv2.PivotConstruction{ - DuplicatePolicy: string(pivot.DuplicatePolicy), MissingCellPolicy: string(pivot.MissingCellPolicy), - UnlistedCategoryPolicy: "EXCLUDE_WITH_EVIDENCE", - }) - if err != nil { - return fmt.Errorf("coded pivot step %q: %w", step.ID, err) - } - quality.LossReasons = []string{"CODED_PIVOT_SELECTED_CATEGORIES_ONLY"} - if pivot.DuplicatePolicy != authoringv2.ConstructionPivotDuplicateError { - quality.LossReasons = append(quality.LossReasons, "TABLE_SHAPE_PIVOT_DUPLICATES_AGGREGATED") - } - for _, category := range pivot.Categories { - if err := addOutput(category.OutputColumnID, pivot.ConstructionID, nil, quality); err != nil { - return err - } - for _, output := range step.Outputs { - if output.ID == category.OutputColumnID { - metadata := nextAuthored[output.Name] - metadata.NodeID = pivot.Source.NodeID - metadata.CandidateID = pivot.Source.CandidateID - metadata.SourceResourceType = pivot.Source.Family.ResourceType - metadata.SourcePath = pivot.Source.Family.ValuePath - nextAuthored[output.Name] = metadata - break - } - } + wantSemanticPath := "construction_semantic_group:" + category.BindingID + ":" + category.FamilyID + if compiledKey == nil || compiledKey.SemanticPath != wantSemanticPath { + return nil, fmt.Errorf("step %q semantic Group output %q does not match its compiled binding metadata", step.ID, key.OutputColumnID) } - case authoringv2.ConstructionOperationUnpivot: - if step.Operation.Unpivot == nil { - return fmt.Errorf("unpivot step %q has no operation payload", step.ID) + if err := addOutput(key.OutputColumnID, group.ConstructionID, nil, quality); err != nil { + return nil, err } - unpivot := step.Operation.Unpivot - dependencies := make([]string, 0, len(unpivot.Inputs)) - for _, input := range unpivot.Inputs { - dependencies = append(dependencies, input.ColumnID) + metadata := nextAuthored[compiledKey.Name] + metadata.SourceResourceType = category.ResourceType + metadata.SourcePath = category.CodingPath + nextAuthored[compiledKey.Name] = metadata + continue + } + if key.FieldSource != nil { + binding := key.FieldSource + identity, ok := groupFieldSourceOutputIdentityForReconcile(binding) + if !ok { + return nil, fmt.Errorf("step %q Group output %q has no resolved field-source binding identity", step.ID, key.OutputColumnID) } - quality, err := unpivotOutputQuality(&authoringv2.UnpivotConstruction{NullRowPolicy: string(unpivot.NullRowPolicy)}) - if err != nil { - return fmt.Errorf("unpivot step %q: %w", step.ID, err) + if err := addOutput(key.OutputColumnID, group.ConstructionID, nil, quality); err != nil { + return nil, err } - for _, projection := range document.Construction.SourceProjections { - if projection.OwnerStepID != step.ID { + for _, output := range step.Outputs { + if output.ID != key.OutputColumnID { continue } - audited := make([]string, 0, len(quality.LossReasons)+1) - audited = append(audited, quality.LossReasons...) - audited = append(audited, "CONSTRUCTION_UNPIVOT_SOURCE_ROWS_UNPIVOTED_FROM_UNSELECTED_FIELD") - quality.LossReasons = audited + metadata := nextAuthored[output.Name] + metadata.CandidateID = identity.CandidateID + metadata.OccurrenceID = identity.OccurrenceID + metadata.SourceResourceType = identity.SourceResourceType + metadata.SourcePath = identity.SourcePath + nextAuthored[output.Name] = metadata break } - for _, columnID := range []string{unpivot.KeyOutputColumnID, unpivot.ValueOutputColumnID} { - if err := addOutput(columnID, unpivot.ConstructionID, dependencies, quality); err != nil { - return err - } + continue + } + if err := addOutput(key.OutputColumnID, group.ConstructionID, []string{key.InputColumnID}, quality); err != nil { + return nil, err + } + inputName := prior[key.InputColumnID] + for _, output := range step.Outputs { + if output.ID == key.OutputColumnID && output.Name == inputName { + metadata := nextAuthored[output.Name] + metadata.PreservesSourceIdentity = true + nextAuthored[output.Name] = metadata + break } - for _, projection := range document.Construction.SourceProjections { - if projection.OwnerStepID != step.ID { - continue - } - consumed := false - for _, input := range unpivot.Inputs { - if input.ColumnID == projection.ColumnID { - consumed = true - } - } - if !consumed { - return fmt.Errorf("unpivot step %q carries source projection column %q it does not melt as an input", step.ID, projection.ColumnID) - } - var declared *authoringv2.StageColumn - for carrierIndex := 0; carrierIndex < stepIndex; carrierIndex++ { - for outputIndex := range document.Construction.Steps[carrierIndex].Outputs { - if document.Construction.Steps[carrierIndex].Outputs[outputIndex].ID == projection.ColumnID { - declared = &document.Construction.Steps[carrierIndex].Outputs[outputIndex] - break - } - } - if declared != nil { - break - } - } - if declared == nil && stepIndex == 0 && compiledUnpivotSourceSlotMatches(compiledOutput, projection) { - // Melt-first signed input: the compiler legitimately seeds the - // melt-owned projection into the SOURCE_PROJECTION stage, so no - // authored step can carry a column the melt itself consumes. Guard - // the compiler-materialized source slot instead of mislabelling it - // receipt corruption; genuine corruption still leaves declared nil. - for _, column := range compiledOutput.Stages[0].Columns { - if column.ID == projection.ColumnID && !column.Internal && !column.Identity { - declared = &authoringv2.StageColumn{ID: column.ID, Name: column.Name, Label: column.Label} - break - } - } - } - if declared == nil || !requiredConstructionID(declared.ID) || strings.TrimSpace(declared.Name) == "" || strings.TrimSpace(declared.Label) == "" { - return fmt.Errorf("unpivot step %q source projection output %q is missing a declared stable ID, name, or label", step.ID, projection.ColumnID) - } - if prior[declared.ID] != declared.Name { - return fmt.Errorf("unpivot step %q source projection output %q does not carry the compiled source column identity", step.ID, declared.ID) - } - if !compiledUnpivotSourceSlotMatches(compiledOutput, projection) { - return fmt.Errorf("unpivot step %q source projection output %q does not carry the compiled source column identity", step.ID, declared.ID) - } - if strings.TrimSpace(rowSourceSnapshot.Token) == "" { - return fmt.Errorf("unpivot step %q source projection output %q cannot be claimed without an authorized capability snapshot", step.ID, declared.ID) - } - derived, ok := lifecycle.ConstructionUnpivotSourceFacts(rowSourceSnapshot, document.RootResourceType, projection.FieldPath) - expected := authoringv2.ConstructionSourceProjection{ - ColumnID: projection.ColumnID, OwnerStepID: projection.OwnerStepID, - OccurrenceID: derived.OccurrenceID, FieldPath: derived.FieldPath, - FHIRType: derived.FHIRType, LogicalType: derived.LogicalType, Label: derived.Label, - } - if !ok || expected != projection { - return fmt.Errorf("unpivot step %q source projection output %q no longer matches the authorized unpivot source derivation for field %q", step.ID, declared.ID, projection.FieldPath) - } - // Unlike Filter and Derive sources, an UNPIVOT melt input is - // consumed and never surfaces in the final table schema, so - // this channel validates identity and authorization but does - // not claim the slot as an authored table-shape output. + } + } + for _, aggregate := range group.Aggregates { + inputs := []string(nil) + if aggregate.Operation != authoringv2.ConstructionGroupCountRows { + inputs = []string{aggregate.InputColumnID} + } + if err := addOutput(aggregate.OutputColumnID, group.ConstructionID, inputs, quality); err != nil { + return nil, err + } + } + for _, rowValue := range step.RowValues { + rowQuality := quality + if rowValue.Policy == authoringv2.ConstructionRowValueAll { + rowQuality.Shape = "array" + } + inputs := []string(nil) + var fieldSourceIdentity groupFieldSourceOutputIdentity + if rowValue.FieldSource != nil { + binding := rowValue.FieldSource + var ok bool + fieldSourceIdentity, ok = groupFieldSourceOutputIdentityForReconcile(binding) + if !ok || binding.ValueKind != "SCALAR" { + return nil, fmt.Errorf("step %q Group row-value output %q has no resolved scalar field-source binding identity", step.ID, rowValue.OutputColumnID) } - case authoringv2.ConstructionOperationFilter: - if step.Operation.Filter == nil { - return fmt.Errorf("filter step %q has no operation payload", step.ID) + } else { + inputs = []string{rowValue.InputColumnID} + } + if err := addOutput(rowValue.OutputColumnID, group.ConstructionID, inputs, rowQuality); err != nil { + return nil, err + } + if rowValue.FieldSource == nil { + continue + } + for _, output := range step.Outputs { + if output.ID != rowValue.OutputColumnID { + continue } - for _, projection := range document.Construction.SourceProjections { - if projection.OwnerStepID != step.ID { - continue - } - if projection.ColumnID != step.Operation.Filter.ColumnID { - return fmt.Errorf("filter step %q carries source projection column %q it does not consume", step.ID, projection.ColumnID) - } - var declared *authoringv2.StageColumn - for outputIndex := range step.Outputs { - if step.Outputs[outputIndex].ID == projection.ColumnID { - declared = &step.Outputs[outputIndex] - break - } - } - if declared == nil || !requiredConstructionID(declared.ID) || strings.TrimSpace(declared.Name) == "" || strings.TrimSpace(declared.Label) == "" { - return fmt.Errorf("filter step %q source projection output %q is missing a declared stable ID, name, or label", step.ID, projection.ColumnID) - } - if prior[declared.ID] != declared.Name { - return fmt.Errorf("filter step %q source projection output %q does not carry the compiled source column identity", step.ID, declared.ID) - } - if !compiledFilterSourceSlotMatches(compiledOutput, projection) { - return fmt.Errorf("filter step %q source projection output %q does not carry the compiled source column identity", step.ID, declared.ID) - } - if strings.TrimSpace(rowSourceSnapshot.Token) == "" { - return fmt.Errorf("filter step %q source projection output %q cannot be claimed without an authorized capability snapshot", step.ID, declared.ID) - } - derived, ok := lifecycle.ConstructionFilterSourceFacts(rowSourceSnapshot, document.RootResourceType, projection.FieldPath) - expected := authoringv2.ConstructionSourceProjection{ - ColumnID: projection.ColumnID, OwnerStepID: projection.OwnerStepID, - OccurrenceID: derived.OccurrenceID, FieldPath: derived.FieldPath, - FHIRType: derived.FHIRType, LogicalType: derived.LogicalType, Label: derived.Label, - } - if !ok || expected != projection { - return fmt.Errorf("filter step %q source projection output %q no longer matches the authorized filter source derivation for field %q", step.ID, declared.ID, projection.FieldPath) - } - if _, duplicate := nextAuthored[declared.Name]; duplicate { - return fmt.Errorf("construction output %q is duplicated for document %q", declared.Name, document.Output.ID) - } - nextAuthored[declared.Name] = authoredOutputColumn{ - ConstructionID: declared.ID, Label: declared.Label, TypedStageOutput: true, - Quality: constructedOutputQuality{ - Lossless: false, StructuralSuitability: "requires-review", - LossReasons: []string{"CONSTRUCTION_FILTER_SOURCE_ROWS_FILTERED"}, - }, - } + metadata := nextAuthored[output.Name] + metadata.CandidateID = fieldSourceIdentity.CandidateID + metadata.OccurrenceID = fieldSourceIdentity.OccurrenceID + metadata.SourceResourceType = fieldSourceIdentity.SourceResourceType + metadata.SourcePath = fieldSourceIdentity.SourcePath + nextAuthored[output.Name] = metadata + break + } + } + return nextAuthored, nil +} + +func applyCodedGroupConstructionStep(stepContext *constructionStepContext) (map[string]authoredOutputColumn, error) { + document := stepContext.document + step := stepContext.step + nextAuthored := stepContext.nextAuthored + addOutput := stepContext.addOutput + if step.Operation.CodedGroup == nil { + return nil, fmt.Errorf("coded group step %q has no operation payload", step.ID) + } + group := step.Operation.CodedGroup + quality := constructedOutputQuality{ + Lossless: false, StructuralSuitability: "requires-review", + LossReasons: []string{"CODED_GROUP_REDUCES_TO_DISTINCT_SOURCE_RECORDS"}, + } + inputIDs := make([]string, 0, len(document.Columns)) + for _, column := range document.Columns { + if column.OccurrenceID == group.Source.OccurrenceID { + inputIDs = append(inputIDs, column.ColumnID) + } + } + if len(inputIDs) == 0 { + return nil, fmt.Errorf("coded group step %q has no source columns on occurrence %q for lineage", step.ID, group.Source.OccurrenceID) + } + for _, output := range []struct { + id string + path string + }{ + {id: group.SystemOutputColumnID, path: group.Source.CodingPath + ".system"}, + {id: group.VersionOutputColumnID, path: group.Source.CodingPath + ".version"}, + {id: group.CodeOutputColumnID, path: group.Source.CodingPath + ".code"}, + {id: group.DistinctSourceCountOutputColumnID, path: group.Source.CodingPath}, + } { + if err := addOutput(output.id, group.ConstructionID, inputIDs, quality); err != nil { + return nil, err + } + for _, declared := range step.Outputs { + if declared.ID == output.id { + metadata := nextAuthored[declared.Name] + metadata.OccurrenceID = group.Source.OccurrenceID + metadata.SourceResourceType = group.Source.ResourceType + metadata.SourcePath = output.path + nextAuthored[declared.Name] = metadata + break } - case authoringv2.ConstructionOperationCombine: - if step.Operation.Combine == nil { - return fmt.Errorf("combine step %q has no operation payload", step.ID) + } + } + return nextAuthored, nil +} + +func applyExpandConstructionStep(stepContext *constructionStepContext) (map[string]authoredOutputColumn, error) { + step := stepContext.step + nextAuthored := stepContext.nextAuthored + addOutput := stepContext.addOutput + if step.Operation.Expand == nil { + return nil, fmt.Errorf("expand step %q has no operation payload", step.ID) + } + expand := step.Operation.Expand + quality := constructedOutputQuality{ + StructuralSuitability: "requires-review", + LossReasons: []string{"TABLE_SHAPE_EXPAND_CHANGES_ROW_GRAIN"}, + } + if err := addOutput(expand.OutputColumnID, expand.ConstructionID, []string{expand.InputColumnID}, quality); err != nil { + return nil, err + } + if expand.OrdinalColumnID != "" { + if err := addOutput(expand.OrdinalColumnID, expand.ConstructionID, []string{expand.InputColumnID}, quality); err != nil { + return nil, err + } + } + return nextAuthored, nil +} + +func applyDeriveConstructionStep(stepContext *constructionStepContext) (map[string]authoredOutputColumn, error) { + document := stepContext.document + step := stepContext.step + prior := stepContext.prior + compiledOutput := stepContext.compiledOutput + rowSourceSnapshot := stepContext.rowSourceSnapshot + nextAuthored := stepContext.nextAuthored + addOutput := stepContext.addOutput + if step.Operation.Derive == nil { + return nil, fmt.Errorf("derive step %q has no operation payload", step.ID) + } + derive := step.Operation.Derive + inputs := make([]string, 0, 2) + for _, operand := range []authoringv2.ConstructionOperand{derive.Left, derive.Right} { + switch operand.Kind { + case authoringv2.ConstructionColumnOperand: + inputs = append(inputs, operand.ColumnID) + case authoringv2.ConstructionLiteralOperand: + default: + return nil, fmt.Errorf("derive step %q has unsupported operand kind %q", step.ID, operand.Kind) + } + } + quality, err := derivedOutputQuality(string(derive.Operation)) + if err != nil { + return nil, fmt.Errorf("derive step %q: %w", step.ID, err) + } + if err := addOutput(derive.OutputColumnID, derive.ConstructionID, inputs, quality); err != nil { + return nil, err + } + for _, projection := range document.Construction.SourceProjections { + if projection.OwnerStepID != step.ID { + continue + } + consumed := false + for _, operand := range []authoringv2.ConstructionOperand{derive.Left, derive.Right} { + if operand.Kind == authoringv2.ConstructionColumnOperand && operand.ColumnID == projection.ColumnID { + consumed = true } - quality := constructedOutputQuality{ - Lossless: false, StructuralSuitability: "requires-review", - LossReasons: []string{"CONSTRUCTION_COMBINE_CHANGES_ROW_SOURCE"}, + } + if !consumed { + return nil, fmt.Errorf("derive step %q carries source projection column %q it does not consume", step.ID, projection.ColumnID) + } + var declared *authoringv2.StageColumn + for outputIndex := range step.Outputs { + if step.Outputs[outputIndex].ID == projection.ColumnID { + declared = &step.Outputs[outputIndex] + break } - for _, output := range step.Outputs { - if !requiredConstructionID(output.ID) || strings.TrimSpace(output.Name) == "" || strings.TrimSpace(output.Label) == "" { - return fmt.Errorf("combine step %q output is missing a stable ID, name, or label", step.ID) - } - if _, duplicate := nextAuthored[output.Name]; duplicate { - return fmt.Errorf("construction output %q is duplicated for document %q", output.Name, document.Output.ID) - } - inputNames := make([]string, 0) - for _, projection := range step.Operation.Combine.Projections { - if projection.OutputColumnID != output.ID { - continue - } - name, err := combineProjectionInputName(compiledOutput, resolvedInputs, projection.InputIndex, projection.InputColumnID) - if err != nil { - return fmt.Errorf("combine step %q output %q: %w", step.ID, output.Name, err) + } + if declared == nil || !requiredConstructionID(declared.ID) || strings.TrimSpace(declared.Name) == "" || strings.TrimSpace(declared.Label) == "" { + return nil, fmt.Errorf("derive step %q source projection output %q is missing a declared stable ID, name, or label", step.ID, projection.ColumnID) + } + if prior[declared.ID] != declared.Name { + return nil, fmt.Errorf("derive step %q source projection output %q does not carry the compiled source column identity", step.ID, declared.ID) + } + if !compiledDeriveSourceSlotMatches(compiledOutput, projection) { + return nil, fmt.Errorf("derive step %q source projection output %q does not carry the compiled source column identity", step.ID, declared.ID) + } + if strings.TrimSpace(rowSourceSnapshot.Token) == "" { + return nil, fmt.Errorf("derive step %q source projection output %q cannot be claimed without an authorized capability snapshot", step.ID, declared.ID) + } + derived, ok := lifecycle.ConstructionDeriveSourceFacts(rowSourceSnapshot, document.RootResourceType, projection.FieldPath) + expected := authoringv2.ConstructionSourceProjection{ + ColumnID: projection.ColumnID, OwnerStepID: projection.OwnerStepID, + OccurrenceID: derived.OccurrenceID, FieldPath: derived.FieldPath, + FHIRType: derived.FHIRType, LogicalType: derived.LogicalType, Label: derived.Label, + } + if !ok || expected != projection { + return nil, fmt.Errorf("derive step %q source projection output %q no longer matches the authorized derive source derivation for field %q", step.ID, declared.ID, projection.FieldPath) + } + if _, duplicate := nextAuthored[declared.Name]; duplicate { + return nil, fmt.Errorf("construction output %q is duplicated for document %q", declared.Name, document.Output.ID) + } + nextAuthored[declared.Name] = authoredOutputColumn{ + ConstructionID: declared.ID, Label: declared.Label, TypedStageOutput: true, + Quality: constructedOutputQuality{ + Lossless: false, StructuralSuitability: "requires-review", + LossReasons: []string{"CONSTRUCTION_DERIVE_SOURCE_ARITHMETIC_FROM_UNSELECTED_FIELD"}, + }, + } + } + return nextAuthored, nil +} + +func applyPivotConstructionStep(stepContext *constructionStepContext) (map[string]authoredOutputColumn, error) { + step := stepContext.step + prior := stepContext.prior + authored := stepContext.authored + sourceOnlyIDs := stepContext.sourceOnlyIDs + nextAuthored := stepContext.nextAuthored + addOutput := stepContext.addOutput + if step.Operation.Pivot == nil { + return nil, fmt.Errorf("pivot step %q has no operation payload", step.ID) + } + pivot := step.Operation.Pivot + dependencies := append([]string(nil), pivot.GroupKeyIDs...) + dependencies = append(dependencies, pivot.CategoryColumnID, pivot.ValueColumnID) + quality, err := pivotOutputQuality(&authoringv2.PivotConstruction{ + DuplicatePolicy: string(pivot.DuplicatePolicy), MissingCellPolicy: string(pivot.MissingCellPolicy), + UnlistedCategoryPolicy: string(pivot.UnlistedCategoryPolicy), + }) + if err != nil { + return nil, fmt.Errorf("pivot step %q: %w", step.ID, err) + } + for _, keyID := range pivot.GroupKeyIDs { + _, privateSource := sourceOnlyIDs[keyID] + priorMetadata, constructedInput := authored[prior[keyID]] + if !privateSource && !constructedInput { + continue + } + for _, output := range step.Outputs { + if output.ID == keyID { + if privateSource { + priorMetadata = authoredOutputColumn{ + ConstructionID: pivot.ConstructionID, TypedStageOutput: true, Quality: quality, } - inputNames = append(inputNames, name) } - uniqueNames, err := uniqueColumnsInOrder(inputNames) - if err != nil { - return fmt.Errorf("combine step %q output %q: %w", step.ID, output.Name, err) - } - if len(uniqueNames) == 0 { - return fmt.Errorf("combine step %q output %q has no compiler-resolved input columns", step.ID, output.Name) - } - nextAuthored[output.Name] = authoredOutputColumn{ - ConstructionID: step.ID, Label: output.Label, InputColumns: uniqueNames, - ExternalInputLineage: true, Quality: quality, - } - } - case authoringv2.ConstructionOperationRelatedSource: - if step.Operation.RelatedSource == nil { - return fmt.Errorf("related-source step %q has no operation payload", step.ID) - } - related := step.Operation.RelatedSource - if (related.Form != capability.ConstructionChoiceAll && related.Form != capability.ConstructionChoiceCount && related.Form != capability.ConstructionChoicePresence) || related.ContributorRule.Policy != authoringv2.ConstructionRelatedAllMatches { - return fmt.Errorf("related-source step %q has unsupported output form or contributor policy", step.ID) - } - quality := constructedOutputQuality{ - Lossless: false, Shape: "array", StructuralSuitability: "requires-review", - LossReasons: []string{"RELATED_SOURCE_AUTHORIZED_MATCHES_ONLY"}, - } - if related.Form != capability.ConstructionChoiceAll || related.RowValuePolicy == authoringv2.ConstructionRowValueOne { - quality.Shape = "" + priorMetadata.Label = output.Label + nextAuthored[output.Name] = priorMetadata + break } - if related.Form != capability.ConstructionChoiceAll { - quality.LossReasons = []string{"RELATED_SOURCE_DISTINCT_RECORD_REDUCTION"} + } + } + for _, category := range pivot.Categories { + if err := addOutput(category.OutputColumnID, pivot.ConstructionID, dependencies, quality); err != nil { + return nil, err + } + } + return nextAuthored, nil +} + +func applyCodedPivotConstructionStep(stepContext *constructionStepContext) (map[string]authoredOutputColumn, error) { + step := stepContext.step + nextAuthored := stepContext.nextAuthored + addOutput := stepContext.addOutput + if step.Operation.CodedPivot == nil || step.Operation.CodedPivot.Source == nil { + return nil, fmt.Errorf("coded pivot step %q has no source", step.ID) + } + pivot := step.Operation.CodedPivot + quality, err := pivotOutputQuality(&authoringv2.PivotConstruction{ + DuplicatePolicy: string(pivot.DuplicatePolicy), MissingCellPolicy: string(pivot.MissingCellPolicy), + UnlistedCategoryPolicy: "EXCLUDE_WITH_EVIDENCE", + }) + if err != nil { + return nil, fmt.Errorf("coded pivot step %q: %w", step.ID, err) + } + quality.LossReasons = []string{"CODED_PIVOT_SELECTED_CATEGORIES_ONLY"} + if pivot.DuplicatePolicy != authoringv2.ConstructionPivotDuplicateError { + quality.LossReasons = append(quality.LossReasons, "TABLE_SHAPE_PIVOT_DUPLICATES_AGGREGATED") + } + for _, category := range pivot.Categories { + if err := addOutput(category.OutputColumnID, pivot.ConstructionID, nil, quality); err != nil { + return nil, err + } + for _, output := range step.Outputs { + if output.ID == category.OutputColumnID { + metadata := nextAuthored[output.Name] + metadata.NodeID = pivot.Source.NodeID + metadata.CandidateID = pivot.Source.CandidateID + metadata.SourceResourceType = pivot.Source.Family.ResourceType + metadata.SourcePath = pivot.Source.Family.ValuePath + nextAuthored[output.Name] = metadata + break } - if err := addOutput(related.OutputColumnID, step.ID, nil, quality); err != nil { - return err + } + } + return nextAuthored, nil +} + +func applyUnpivotConstructionStep(stepContext *constructionStepContext) (map[string]authoredOutputColumn, error) { + document := stepContext.document + step := stepContext.step + stepIndex := stepContext.stepIndex + prior := stepContext.prior + compiledOutput := stepContext.compiledOutput + rowSourceSnapshot := stepContext.rowSourceSnapshot + nextAuthored := stepContext.nextAuthored + addOutput := stepContext.addOutput + if step.Operation.Unpivot == nil { + return nil, fmt.Errorf("unpivot step %q has no operation payload", step.ID) + } + unpivot := step.Operation.Unpivot + dependencies := make([]string, 0, len(unpivot.Inputs)) + for _, input := range unpivot.Inputs { + dependencies = append(dependencies, input.ColumnID) + } + quality, err := unpivotOutputQuality(&authoringv2.UnpivotConstruction{NullRowPolicy: string(unpivot.NullRowPolicy)}) + if err != nil { + return nil, fmt.Errorf("unpivot step %q: %w", step.ID, err) + } + for _, projection := range document.Construction.SourceProjections { + if projection.OwnerStepID != step.ID { + continue + } + audited := make([]string, 0, len(quality.LossReasons)+1) + audited = append(audited, quality.LossReasons...) + audited = append(audited, "CONSTRUCTION_UNPIVOT_SOURCE_ROWS_UNPIVOTED_FROM_UNSELECTED_FIELD") + quality.LossReasons = audited + break + } + for _, columnID := range []string{unpivot.KeyOutputColumnID, unpivot.ValueOutputColumnID} { + if err := addOutput(columnID, unpivot.ConstructionID, dependencies, quality); err != nil { + return nil, err + } + } + for _, projection := range document.Construction.SourceProjections { + if projection.OwnerStepID != step.ID { + continue + } + consumed := false + for _, input := range unpivot.Inputs { + if input.ColumnID == projection.ColumnID { + consumed = true } - for _, output := range step.Outputs { - if output.ID == related.OutputColumnID { - metadata := nextAuthored[output.Name] - metadata.NodeID = related.Source.NodeID - metadata.CandidateID = related.Source.CandidateID - metadata.OccurrenceID = related.SourceOccurrenceID - metadata.SourceResourceType = related.Source.ResourceType - metadata.SourcePath = related.Source.Path - nextAuthored[output.Name] = metadata + } + if !consumed { + return nil, fmt.Errorf("unpivot step %q carries source projection column %q it does not melt as an input", step.ID, projection.ColumnID) + } + var declared *authoringv2.StageColumn + for carrierIndex := 0; carrierIndex < stepIndex; carrierIndex++ { + for outputIndex := range document.Construction.Steps[carrierIndex].Outputs { + if document.Construction.Steps[carrierIndex].Outputs[outputIndex].ID == projection.ColumnID { + declared = &document.Construction.Steps[carrierIndex].Outputs[outputIndex] break } } - case authoringv2.ConstructionOperationRelatedExpand: - if step.Operation.RelatedExpand == nil { - return fmt.Errorf("related-expansion step %q has no operation payload", step.ID) - } - related := step.Operation.RelatedExpand - quality := constructedOutputQuality{ - Lossless: false, StructuralSuitability: "requires-review", - LossReasons: []string{"RELATED_EXPANSION_CHANGES_ROW_GRAIN"}, - } - if err := addOutput(related.RelatedRecordColumnID, step.ID, nil, quality); err != nil { - return err + if declared != nil { + break } - for _, output := range step.Outputs { - if output.ID == related.RelatedRecordColumnID { - metadata := nextAuthored[output.Name] - metadata.NodeID = related.TargetNodeID - metadata.SourceResourceType = related.TargetResourceType - metadata.SourcePath = "id" - nextAuthored[output.Name] = metadata + } + if declared == nil && stepIndex == 0 && compiledUnpivotSourceSlotMatches(compiledOutput, projection) { + for _, column := range compiledOutput.Stages[0].Columns { + if column.ID == projection.ColumnID && !column.Internal && !column.Identity { + declared = &authoringv2.StageColumn{ID: column.ID, Name: column.Name, Label: column.Label} break } } - case authoringv2.ConstructionOperationRelatedEligibility: - if step.Operation.RelatedEligibility == nil { - return fmt.Errorf("related-eligibility step %q has no operation payload", step.ID) - } - case authoringv2.ConstructionOperationRelatedField: - if step.Operation.RelatedField == nil { - return fmt.Errorf("related-field step %q has no operation payload", step.ID) + } + if declared == nil || !requiredConstructionID(declared.ID) || strings.TrimSpace(declared.Name) == "" || strings.TrimSpace(declared.Label) == "" { + return nil, fmt.Errorf("unpivot step %q source projection output %q is missing a declared stable ID, name, or label", step.ID, projection.ColumnID) + } + if prior[declared.ID] != declared.Name { + return nil, fmt.Errorf("unpivot step %q source projection output %q does not carry the compiled source column identity", step.ID, declared.ID) + } + if !compiledUnpivotSourceSlotMatches(compiledOutput, projection) { + return nil, fmt.Errorf("unpivot step %q source projection output %q does not carry the compiled source column identity", step.ID, declared.ID) + } + if strings.TrimSpace(rowSourceSnapshot.Token) == "" { + return nil, fmt.Errorf("unpivot step %q source projection output %q cannot be claimed without an authorized capability snapshot", step.ID, declared.ID) + } + derived, ok := lifecycle.ConstructionUnpivotSourceFacts(rowSourceSnapshot, document.RootResourceType, projection.FieldPath) + expected := authoringv2.ConstructionSourceProjection{ + ColumnID: projection.ColumnID, OwnerStepID: projection.OwnerStepID, + OccurrenceID: derived.OccurrenceID, FieldPath: derived.FieldPath, + FHIRType: derived.FHIRType, LogicalType: derived.LogicalType, Label: derived.Label, + } + if !ok || expected != projection { + return nil, fmt.Errorf("unpivot step %q source projection output %q no longer matches the authorized unpivot source derivation for field %q", step.ID, declared.ID, projection.FieldPath) + } + } + return nextAuthored, nil +} + +func applyFilterConstructionStep(stepContext *constructionStepContext) (map[string]authoredOutputColumn, error) { + document := stepContext.document + step := stepContext.step + prior := stepContext.prior + compiledOutput := stepContext.compiledOutput + rowSourceSnapshot := stepContext.rowSourceSnapshot + nextAuthored := stepContext.nextAuthored + if step.Operation.Filter == nil { + return nil, fmt.Errorf("filter step %q has no operation payload", step.ID) + } + for _, projection := range document.Construction.SourceProjections { + if projection.OwnerStepID != step.ID { + continue + } + if projection.ColumnID != step.Operation.Filter.ColumnID { + return nil, fmt.Errorf("filter step %q carries source projection column %q it does not consume", step.ID, projection.ColumnID) + } + var declared *authoringv2.StageColumn + for outputIndex := range step.Outputs { + if step.Outputs[outputIndex].ID == projection.ColumnID { + declared = &step.Outputs[outputIndex] + break } - related := step.Operation.RelatedField - quality := constructedOutputQuality{ + } + if declared == nil || !requiredConstructionID(declared.ID) || strings.TrimSpace(declared.Name) == "" || strings.TrimSpace(declared.Label) == "" { + return nil, fmt.Errorf("filter step %q source projection output %q is missing a declared stable ID, name, or label", step.ID, projection.ColumnID) + } + if prior[declared.ID] != declared.Name { + return nil, fmt.Errorf("filter step %q source projection output %q does not carry the compiled source column identity", step.ID, declared.ID) + } + if !compiledFilterSourceSlotMatches(compiledOutput, projection) { + return nil, fmt.Errorf("filter step %q source projection output %q does not carry the compiled source column identity", step.ID, declared.ID) + } + if strings.TrimSpace(rowSourceSnapshot.Token) == "" { + return nil, fmt.Errorf("filter step %q source projection output %q cannot be claimed without an authorized capability snapshot", step.ID, declared.ID) + } + derived, ok := lifecycle.ConstructionFilterSourceFacts(rowSourceSnapshot, document.RootResourceType, projection.FieldPath) + expected := authoringv2.ConstructionSourceProjection{ + ColumnID: projection.ColumnID, OwnerStepID: projection.OwnerStepID, + OccurrenceID: derived.OccurrenceID, FieldPath: derived.FieldPath, + FHIRType: derived.FHIRType, LogicalType: derived.LogicalType, Label: derived.Label, + } + if !ok || expected != projection { + return nil, fmt.Errorf("filter step %q source projection output %q no longer matches the authorized filter source derivation for field %q", step.ID, declared.ID, projection.FieldPath) + } + if _, duplicate := nextAuthored[declared.Name]; duplicate { + return nil, fmt.Errorf("construction output %q is duplicated for document %q", declared.Name, document.Output.ID) + } + nextAuthored[declared.Name] = authoredOutputColumn{ + ConstructionID: declared.ID, Label: declared.Label, TypedStageOutput: true, + Quality: constructedOutputQuality{ Lossless: false, StructuralSuitability: "requires-review", - LossReasons: []string{"RELATED_FIELD_AUTHORIZED_EXACT_RECORD"}, - } - if err := addOutput(related.OutputColumnID, step.ID, nil, quality); err != nil { - return err - } - for _, output := range step.Outputs { - if output.ID == related.OutputColumnID { - metadata := nextAuthored[output.Name] - metadata.NodeID = related.Source.NodeID - metadata.CandidateID = related.Source.CandidateID - metadata.SourceResourceType = related.Source.ResourceType - metadata.SourcePath = related.Source.Path - nextAuthored[output.Name] = metadata - break - } - } - default: - return fmt.Errorf("step %q has unsupported operation kind %q", step.ID, step.Operation.Kind) + LossReasons: []string{"CONSTRUCTION_FILTER_SOURCE_ROWS_FILTERED"}, + }, } - next := make(map[string]string, len(step.Outputs)) - for _, output := range step.Outputs { - if !requiredConstructionID(output.ID) || strings.TrimSpace(output.Name) == "" { - return fmt.Errorf("step %q output schema contains a missing stable ID or public name", step.ID) + } + return nextAuthored, nil +} + +func applyCombineConstructionStep(stepContext *constructionStepContext) (map[string]authoredOutputColumn, error) { + document := stepContext.document + step := stepContext.step + compiledOutput := stepContext.compiledOutput + resolvedInputs := stepContext.resolvedInputs + nextAuthored := stepContext.nextAuthored + if step.Operation.Combine == nil { + return nil, fmt.Errorf("combine step %q has no operation payload", step.ID) + } + quality := constructedOutputQuality{ + Lossless: false, StructuralSuitability: "requires-review", + LossReasons: []string{"CONSTRUCTION_COMBINE_CHANGES_ROW_SOURCE"}, + } + for _, output := range step.Outputs { + if !requiredConstructionID(output.ID) || strings.TrimSpace(output.Name) == "" || strings.TrimSpace(output.Label) == "" { + return nil, fmt.Errorf("combine step %q output is missing a stable ID, name, or label", step.ID) + } + if _, duplicate := nextAuthored[output.Name]; duplicate { + return nil, fmt.Errorf("construction output %q is duplicated for document %q", output.Name, document.Output.ID) + } + inputNames := make([]string, 0) + for _, projection := range step.Operation.Combine.Projections { + if projection.OutputColumnID != output.ID { + continue } - if _, duplicate := next[output.ID]; duplicate { - return fmt.Errorf("step %q output schema duplicates column ID %q", step.ID, output.ID) + name, err := combineProjectionInputName(compiledOutput, resolvedInputs, projection.InputIndex, projection.InputColumnID) + if err != nil { + return nil, fmt.Errorf("combine step %q output %q: %w", step.ID, output.Name, err) } - next[output.ID] = output.Name + inputNames = append(inputNames, name) } - clear(authored) - for name, metadata := range nextAuthored { - authored[name] = metadata + uniqueNames, err := uniqueColumnsInOrder(inputNames) + if err != nil { + return nil, fmt.Errorf("combine step %q output %q: %w", step.ID, output.Name, err) + } + if len(uniqueNames) == 0 { + return nil, fmt.Errorf("combine step %q output %q has no compiler-resolved input columns", step.ID, output.Name) + } + nextAuthored[output.Name] = authoredOutputColumn{ + ConstructionID: step.ID, Label: output.Label, InputColumns: uniqueNames, + ExternalInputLineage: true, Quality: quality, } - prior = next } - return nil + return nextAuthored, nil +} + +func applyRelatedSourceConstructionStep(stepContext *constructionStepContext) (map[string]authoredOutputColumn, error) { + step := stepContext.step + nextAuthored := stepContext.nextAuthored + addOutput := stepContext.addOutput + if step.Operation.RelatedSource == nil { + return nil, fmt.Errorf("related-source step %q has no operation payload", step.ID) + } + related := step.Operation.RelatedSource + if (related.Form != capability.ConstructionChoiceAll && related.Form != capability.ConstructionChoiceCount && related.Form != capability.ConstructionChoicePresence) || related.ContributorRule.Policy != authoringv2.ConstructionRelatedAllMatches { + return nil, fmt.Errorf("related-source step %q has unsupported output form or contributor policy", step.ID) + } + quality := constructedOutputQuality{ + Lossless: false, Shape: "array", StructuralSuitability: "requires-review", + LossReasons: []string{"RELATED_SOURCE_AUTHORIZED_MATCHES_ONLY"}, + } + if related.Form != capability.ConstructionChoiceAll || related.RowValuePolicy == authoringv2.ConstructionRowValueOne { + quality.Shape = "" + } + if related.Form != capability.ConstructionChoiceAll { + quality.LossReasons = []string{"RELATED_SOURCE_DISTINCT_RECORD_REDUCTION"} + } + if err := addOutput(related.OutputColumnID, step.ID, nil, quality); err != nil { + return nil, err + } + for _, output := range step.Outputs { + if output.ID == related.OutputColumnID { + metadata := nextAuthored[output.Name] + metadata.NodeID = related.Source.NodeID + metadata.CandidateID = related.Source.CandidateID + metadata.OccurrenceID = related.SourceOccurrenceID + metadata.SourceResourceType = related.Source.ResourceType + metadata.SourcePath = related.Source.Path + nextAuthored[output.Name] = metadata + break + } + } + return nextAuthored, nil +} + +func applyRelatedExpandConstructionStep(stepContext *constructionStepContext) (map[string]authoredOutputColumn, error) { + step := stepContext.step + nextAuthored := stepContext.nextAuthored + addOutput := stepContext.addOutput + if step.Operation.RelatedExpand == nil { + return nil, fmt.Errorf("related-expansion step %q has no operation payload", step.ID) + } + related := step.Operation.RelatedExpand + quality := constructedOutputQuality{ + Lossless: false, StructuralSuitability: "requires-review", + LossReasons: []string{"RELATED_EXPANSION_CHANGES_ROW_GRAIN"}, + } + if err := addOutput(related.RelatedRecordColumnID, step.ID, nil, quality); err != nil { + return nil, err + } + for _, output := range step.Outputs { + if output.ID == related.RelatedRecordColumnID { + metadata := nextAuthored[output.Name] + metadata.NodeID = related.TargetNodeID + metadata.SourceResourceType = related.TargetResourceType + metadata.SourcePath = "id" + nextAuthored[output.Name] = metadata + break + } + } + return nextAuthored, nil +} + +func applyRelatedEligibilityConstructionStep(stepContext *constructionStepContext) (map[string]authoredOutputColumn, error) { + step := stepContext.step + nextAuthored := stepContext.nextAuthored + if step.Operation.RelatedEligibility == nil { + return nil, fmt.Errorf("related-eligibility step %q has no operation payload", step.ID) + } + return nextAuthored, nil +} + +func applyRelatedFieldConstructionStep(stepContext *constructionStepContext) (map[string]authoredOutputColumn, error) { + step := stepContext.step + nextAuthored := stepContext.nextAuthored + addOutput := stepContext.addOutput + if step.Operation.RelatedField == nil { + return nil, fmt.Errorf("related-field step %q has no operation payload", step.ID) + } + related := step.Operation.RelatedField + quality := constructedOutputQuality{ + Lossless: false, StructuralSuitability: "requires-review", + LossReasons: []string{"RELATED_FIELD_AUTHORIZED_EXACT_RECORD"}, + } + if err := addOutput(related.OutputColumnID, step.ID, nil, quality); err != nil { + return nil, err + } + for _, output := range step.Outputs { + if output.ID == related.OutputColumnID { + metadata := nextAuthored[output.Name] + metadata.NodeID = related.Source.NodeID + metadata.CandidateID = related.Source.CandidateID + metadata.SourceResourceType = related.Source.ResourceType + metadata.SourcePath = related.Source.Path + nextAuthored[output.Name] = metadata + break + } + } + return nextAuthored, nil } func combineProjectionInputName( diff --git a/internal/server/explorer_output_reconcile_unpivot_source_test.go b/internal/server/explorer_output_reconcile_unpivot_source_test.go index e4f1fdeed..d060a59ac 100644 --- a/internal/server/explorer_output_reconcile_unpivot_source_test.go +++ b/internal/server/explorer_output_reconcile_unpivot_source_test.go @@ -138,10 +138,6 @@ func TestUnpivotSourceProjectionReconcileRefusesDriftedClaims(t *testing.T) { } }) - // Fact-branch exclusivity (finding F-2): the compiled-slot arm mirrors the - // claim's Label, so keeping the compiled column in sync leaves ONLY the - // durable-claim versus re-derived-facts comparison to catch a drifted - // Label. Mutation-proven: removing the facts DeepEquals must kill this. t.Run("claim label drifted while the compiled slot mirrors it", func(t *testing.T) { document, compiled, snapshot, _ := unpivotSourceReconcileFixture(t) document.Construction.SourceProjections[0].Label = "TAMPERED-LABEL" @@ -152,8 +148,6 @@ func TestUnpivotSourceProjectionReconcileRefusesDriftedClaims(t *testing.T) { } }) - // The compiled-slot arm checks neither FHIRType nor OccurrenceID, so this - // kind of drift is reachable only through the facts comparison. t.Run("claim fhir type drifted from the rederived schema facts", func(t *testing.T) { document, compiled, snapshot, _ := unpivotSourceReconcileFixture(t) document.Construction.SourceProjections[0].FHIRType = "TAMPERED-TYPE" @@ -193,14 +187,6 @@ func TestUnpivotSourceProjectionReconcileRefusesDriftedClaims(t *testing.T) { }) } -// --- F-A durables: melt-first signed reconcile keeps its audits loaded --- -// (Authority /tmp/opencode/fa-report.md §3-§4. With the melt as FIRST step the -// compiler legitimately seeds the melt-owned projection into the -// SOURCE_PROJECTION stage; no authored step can declare a column the melt -// consumes. Reconcile must accept the compiler-materialized slot while every -// downstream audit — slot equality, stable identity, authorized snapshot, -// facts equality — stays loaded and keeps refusing genuine drift.) - func unpivotMeltFirstReconcileFixture(t *testing.T) (authoringv2.Document, lower.CompiledRecipeOutput, capability.Snapshot) { t.Helper() base, _, snapshot, projection := unpivotSourceReconcileFixture(t) @@ -257,8 +243,6 @@ func TestUnpivotSourceProjectionReconcileMeltFirst(t *testing.T) { } }) - // Audit-loading mutations: each mutation of one seed path must still be - // caught by its named audit, proving the guard only guards the legit state. t.Run("compiled slot identity drift still falls to the declaration refusal", func(t *testing.T) { document, compiled, snapshot := unpivotMeltFirstReconcileFixture(t) compiled.Stages[0].Columns[2].Label = "DRIFTED-COMPILED-LABEL" diff --git a/internal/server/explorer_preview_response.go b/internal/server/explorer_preview_response.go index a8d9cd990..deabd1ce8 100644 --- a/internal/server/explorer_preview_response.go +++ b/internal/server/explorer_preview_response.go @@ -38,9 +38,6 @@ func classifyReceiptPreviewResolutionError(receiptID string, err error) error { return err } -// previewRouteError maps execution failures to the stable authoring error -// contract without requiring a Fiber context. The generated OpenAPI adapter -// uses this form directly. func previewRouteError(err error) error { if errors.Is(err, ErrPreviewResponseTooLarge) { return &explorer.AuthoringError{Status: 413, Diagnostic: explorer.AuthoringDiagnostic{Severity: "ERROR", Stage: "preview", Code: "RESPONSE_TOO_LARGE", Message: "preview response exceeds the maximum size"}, Cause: err} diff --git a/internal/server/explorer_preview_security.go b/internal/server/explorer_preview_security.go index eb1956855..f78000069 100644 --- a/internal/server/explorer_preview_security.go +++ b/internal/server/explorer_preview_security.go @@ -10,16 +10,8 @@ import ( "github.com/calypr/loom/internal/explorer" ) -// ErrReceiptExecutionContract identifies a receipt/capability mismatch that -// must be rejected before a receipt-backed query or materialization starts. -// It is deliberately transport-neutral; HTTP adapters can map it to their -// existing receipt-input conflict diagnostic. var ErrReceiptExecutionContract = errors.New("receipt execution contract mismatch") -// validateAuthorizedReadScope verifies both the digest and the explicit mode. -// In particular, restricted + zero paths is a valid deny-all result, while an -// empty mode is rejected because downstream legacy code may interpret it as -// unrestricted when the path list is empty. func validateAuthorizedReadScope(scope authscope.ReadScope, expectedDigest string) error { switch scope.Mode { case authscope.ReadScopeUnrestricted: @@ -27,7 +19,6 @@ func validateAuthorizedReadScope(scope authscope.ReadScope, expectedDigest strin return receiptExecutionContractError("unrestricted scope must not carry resource paths") } case authscope.ReadScopeRestricted: - // A restricted-empty scope is intentional and must remain restricted. default: return receiptExecutionContractError("authorization scope mode is missing or invalid") } @@ -37,10 +28,6 @@ func validateAuthorizedReadScope(scope authscope.ReadScope, expectedDigest strin return nil } -// validateReceiptEnginePublicColumns proves that the engine and receipt expose -// exactly the same public columns. Column order is deliberately not compared: -// the compiler owns execution order while the receipt's output contract owns -// presentation order. Internal identity/provenance projections are excluded. func validateReceiptEnginePublicColumns(receipt *explorer.CompilationReceipt, resolved dataframeexecution.Resolved) error { if receipt == nil { return receiptExecutionContractError("receipt is required") @@ -70,9 +57,6 @@ func validateReceiptEnginePublicColumns(receipt *explorer.CompilationReceipt, re for _, output := range resolved.Compiled.Outputs { seen[output.Name] = true if groupedOutputs[output.Name] { - // Explicit groups are terminal row sources, so their executable - // columns are not the authored root-field emissions. Bind this - // exception to the receipt's full compiler-derived schema and plan. if expected, actual := receipt.OutputFingerprints[output.Name], groupedFingerprints[output.Name]; expected == "" || actual == "" || expected != actual { return contractMismatch("output_execution", output.Name, expected, actual) } diff --git a/internal/server/explorer_preview_security_test.go b/internal/server/explorer_preview_security_test.go index 9679ee03c..358be8ee2 100644 --- a/internal/server/explorer_preview_security_test.go +++ b/internal/server/explorer_preview_security_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" "github.com/calypr/loom/internal/dataframe/compiler/lower" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" @@ -72,9 +73,15 @@ func TestValidateReceiptEnginePublicColumnsUsesExplicitGroupSchema(t *testing.T) }}, } engine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - RootPageRows: 25, - QueryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + RootPageRows: 25, + SourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return visit(map[string]any{ "group_revision_id": "grouprev_pinned", "group_id": "j03-reviewed", "group_label": "J03 reviewed", "group_ordinal": 0, "members": []any{map[string]any{"source_identity": map[string]any{"id": "dev-patient-001"}}}, diff --git a/internal/server/explorer_publication.go b/internal/server/explorer_publication.go index 9e31b3d2d..490f44481 100644 --- a/internal/server/explorer_publication.go +++ b/internal/server/explorer_publication.go @@ -7,7 +7,6 @@ import ( dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/publication" - materializationarango "github.com/calypr/loom/internal/dataframe/publication/arango" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" "github.com/calypr/loom/internal/explorer/authoringv2" @@ -15,10 +14,11 @@ import ( "github.com/calypr/loom/internal/explorer/lifecycle" ) -// explorerReceiptMaterializer keeps publication bound to the immutable -// compilation receipt. The receipt is the only artifact crossing the HTTP to -// execution boundary; its resolved recipe is lowered in memory by the execution package. -func explorerReceiptMaterializer(recipeEngine *dataframeexecution.Engine, target publication.Target, registry *materializationarango.Registry, degradation error, logger *slog.Logger, batchRows, batchBytes int, qualityMaxRows, qualityMaxDistinctKeys int64) lifecycle.ReceiptMaterializer { +type receiptExecutionFinder interface { + FindExecutionByKey(ctx context.Context, key string) (publication.BundleExecution, error) +} + +func explorerReceiptMaterializer(recipeEngine *dataframeexecution.Engine, target publication.Target, registry receiptExecutionFinder, degradation error, logger *slog.Logger, batchRows, batchBytes int, qualityMaxRows, qualityMaxDistinctKeys int64) lifecycle.ReceiptMaterializer { return func(ctx context.Context, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings) (lifecycle.Execution, error) { if receipt == nil { return lifecycle.Execution{}, fmt.Errorf("compilation receipt is required") diff --git a/internal/server/explorer_receipt_contract.go b/internal/server/explorer_receipt_contract.go index 9672a39db..f1356b0f7 100644 --- a/internal/server/explorer_receipt_contract.go +++ b/internal/server/explorer_receipt_contract.go @@ -25,10 +25,6 @@ import ( "github.com/calypr/loom/internal/projectid" ) -// compileConstructionSourceStage runs the normal semantic and physical -// compiler for a source-only capability probe. The empty document never -// becomes a receipt or public output; only the compiler-derived source schema, -// row identity, and operation reasons are returned to lifecycle discovery. func compileConstructionSourceStage(ctx context.Context, request lifecycle.ConstructionSourceStageRequest, recipeEngine *dataframeexecution.Engine) (explorer.ReceiptConstructionStage, error) { if recipeEngine == nil { return explorer.ReceiptConstructionStage{}, fmt.Errorf("recipe engine is required") @@ -314,9 +310,6 @@ func classifyReceiptRecipeError(err error) error { return err } -// persistValidatedReceipt proves that the execution engine can reproduce the -// complete immutable receipt before the receipt store sees it. A receipt that -// fails this check must never become an executable immutable artifact. func persistValidatedReceipt(ctx context.Context, recipeEngine *dataframeexecution.Engine, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings, persist func(context.Context, explorer.CompilationReceipt) (*explorer.CompilationReceipt, error)) (*explorer.CompilationReceipt, error) { if _, err := compileValidatedReceiptResolution(ctx, recipeEngine, receipt, bindings); err != nil { id := "" @@ -498,10 +491,6 @@ func firstNonEmptyWorkspaceTitle(values ...string) string { return "Explorer" } -// validateReceiptResolution proves that deterministic runtime lowering still -// describes the exact resolved semantic artifact frozen in the receipt. It -// intentionally compares no AQL or physical IR because those are -// request-scoped implementation details. func validateReceiptResolution(receipt *explorer.CompilationReceipt, resolved *dataframeexecution.Resolved) error { if receipt == nil { return fmt.Errorf("compilation receipt is required") @@ -564,10 +553,6 @@ func validateReceiptResolution(receipt *explorer.CompilationReceipt, resolved *d return nil } -// receiptCompiledOutputSchemas freezes the lowerer's finalized public and -// internal projection schema for every output. The capability response later -// filters internal columns and derives compatibility keys from these exact -// compiler values; no workspace presentation rows are used as schema input. func receiptCompiledOutputSchemas(resolved *dataframeexecution.Resolved) (map[string][]explorer.ReceiptCompiledOutputColumn, error) { if resolved == nil { return nil, fmt.Errorf("resolved compilation is required") @@ -595,9 +580,6 @@ func receiptCompiledOutputSchemas(resolved *dataframeexecution.Resolved) (map[st return schemas, nil } -// receiptConstructionStages freezes the compiler's exact stage schemas and -// capabilities into the receipt identity. The implicit source projection is -// represented with an empty operation because it is not an authored step. func receiptConstructionStages(resolved *dataframeexecution.Resolved) (map[string][]explorer.ReceiptConstructionStage, error) { if resolved == nil { return nil, fmt.Errorf("resolved compilation is required") @@ -723,10 +705,6 @@ func receiptConstructionStageFromDescriptor(descriptor lower.CompiledStageDescri return stage, nil } -// compileValidatedReceiptResolution validates the complete immutable receipt -// even when the caller intends to execute only one output. OutputNames is an -// execution selector: allowing it to narrow compilation here would compare a -// partial fingerprint and column set with the receipt's full contract. func compileValidatedReceiptResolution(ctx context.Context, recipeEngine *dataframeexecution.Engine, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings) (dataframeexecution.Resolved, error) { validationBindings := bindings validationBindings.OutputNames = nil diff --git a/internal/server/explorer_receipt_contract_test.go b/internal/server/explorer_receipt_contract_test.go index 6501eb909..1bcf7cce1 100644 --- a/internal/server/explorer_receipt_contract_test.go +++ b/internal/server/explorer_receipt_contract_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" "github.com/calypr/loom/internal/dataframe/compiler/ir" "github.com/calypr/loom/internal/dataframe/compiler/lower" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" @@ -101,8 +102,15 @@ func TestCompileExplorerReceiptReconcilesAuthoredDerivedOutput(t *testing.T) { }, }} recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -226,8 +234,15 @@ func TestCompileExplorerReceiptPersistsCompilerConstructionStages(t *testing.T) t.Fatalf("validate typed construction workspace: %v", err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -403,8 +418,15 @@ func TestCompileExplorerReceiptReconcilesCodedGroupProposalOutputsAndLineage(t * t.Fatalf("validate CODED_GROUP proposal workspace: %v", err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -630,8 +652,15 @@ func TestCompileExplorerReceiptReconcilesTypedConstructionOutputs(t *testing.T) t.Fatalf("validate typed derived construction: %v", err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -1606,8 +1635,15 @@ func TestCompileExplorerReceiptBindsRowDefinitionProposalBeforeIdentity(t *testi t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -1669,8 +1705,15 @@ func TestCompileExplorerReceiptBindsTableShapeProposalBeforeIdentity(t *testing. t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -1717,8 +1760,14 @@ func TestCompileExplorerReceiptBindsTableShapeProposalBeforeIdentity(t *testing. func TestCompileValidatedReceiptResolutionChecksAllOutputsForScopedPreview(t *testing.T) { recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil }, }) @@ -1758,8 +1807,6 @@ func TestCompileValidatedReceiptResolutionChecksAllOutputsForScopedPreview(t *te OutputFingerprints: fingerprints, OutputColumnProvenance: provenance, EmittedColumns: []explorer.EmittedColumn{ - // Presentation order is intentionally different from compiler field - // order. It must not invalidate an otherwise identical receipt. {OutputID: "patients", PublicColumn: "patient_gender"}, {OutputID: "patients", PublicColumn: "patient_id"}, {OutputID: "specimens", PublicColumn: "specimen_id"}, @@ -1780,7 +1827,14 @@ func TestCompileValidatedReceiptResolutionChecksAllOutputsForScopedPreview(t *te } func TestCompileValidatedReceiptResolutionSurvivesCompilerMetadataJSONRoundTrip(t *testing.T) { - recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }}) + recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }}) if err != nil { t.Fatal(err) } @@ -1827,8 +1881,14 @@ func TestCompileValidatedReceiptResolutionSurvivesCompilerMetadataJSONRoundTrip( func TestCompileValidatedReceiptResolutionSurvivesCohortConstructionRoundTrip(t *testing.T) { recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil }, }) @@ -1902,7 +1962,14 @@ func TestCompileValidatedReceiptResolutionSurvivesCohortConstructionRoundTrip(t } func TestPersistValidatedReceiptRejectsRuntimeMismatchBeforeStore(t *testing.T) { - recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }}) + recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }}) if err != nil { t.Fatal(err) } @@ -1943,7 +2010,14 @@ func TestPersistValidatedReceiptRejectsRuntimeMismatchBeforeStore(t *testing.T) } func TestResolvedOutputFingerprintExcludesOptimizerAndTransientProvenance(t *testing.T) { - recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }}) + recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }}) if err != nil { t.Fatal(err) } @@ -1991,8 +2065,15 @@ func TestResolvedOutputFingerprintExcludesOptimizerAndTransientProvenance(t *tes func TestPreviewSourceIdentityDoesNotChangeReceiptArtifacts(t *testing.T) { recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -2045,10 +2126,16 @@ func TestPreviewSourceIdentityDoesNotChangeReceiptArtifacts(t *testing.T) { func TestPreviewSourceIdentityDoesNotChangeUnpivotReceiptArtifacts(t *testing.T) { var renderedQuery string - recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: func(_ context.Context, query string, _ int, _ map[string]any, _ func(map[string]any) error) error { - renderedQuery = query - return nil - }}) + recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + query, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + + renderedQuery = query + return nil + }}) if err != nil { t.Fatal(err) } @@ -2474,8 +2561,15 @@ func TestCompileExplorerReceiptAllowsEmptyConstructionTargetWithSiblingOutputs(t } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -2608,8 +2702,15 @@ func TestCompileExplorerReceiptReconcilesSparsePublishedAppendWithExactSourceSch } engine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) diff --git a/internal/server/explorer_receipt_validation.go b/internal/server/explorer_receipt_validation.go index f271c1938..60758d57f 100644 --- a/internal/server/explorer_receipt_validation.go +++ b/internal/server/explorer_receipt_validation.go @@ -25,8 +25,6 @@ func validateReceiptFullStream(ctx context.Context, recipeEngine *dataframeexecu if err != nil { return classifyReceiptPreviewResolutionError(receipt.ID, err) } - // Receipt validation compiles every output. Restore the exact execution - // selector afterward so unrelated receipt outputs are never streamed. resolved.Semantic.SemanticPlan.Bindings.OutputNames = []string{output} streams, err := recipeEngine.Streams(ctx, resolved) if err != nil { diff --git a/internal/server/explorer_reconcile_scope_test.go b/internal/server/explorer_reconcile_scope_test.go index fc82e3e07..0a6c5cf7b 100644 --- a/internal/server/explorer_reconcile_scope_test.go +++ b/internal/server/explorer_reconcile_scope_test.go @@ -3,6 +3,7 @@ package server import ( "context" "encoding/json" + "github.com/calypr/loom/internal/dataframe/compiler" "net/http" "testing" @@ -79,8 +80,15 @@ func TestReconcileSelectedOutputScopesReceiptAndPreviewPastStaleConsumer(t *test if err != nil { t.Fatal(err) } - queryRows := func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil } - engine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: queryRows, PreviewQueryRows: queryRows}) + queryRows := func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + } + engine, err := dataframeexecution.New(dataframeexecution.Config{CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: queryRows, PreviewSourceRows: queryRows}) if err != nil { t.Fatal(err) } diff --git a/internal/server/fs1_signed_slot_collision_test.go b/internal/server/fs1_signed_slot_collision_test.go index ce36029fd..4b9d2444f 100644 --- a/internal/server/fs1_signed_slot_collision_test.go +++ b/internal/server/fs1_signed_slot_collision_test.go @@ -1,16 +1,5 @@ package server -// fs1_signed_slot_collision_test.go — F-DS1 durable regressions at the shipped -// server-handler boundary (fiber app + real compileExplorerReceipt + recipe -// engine, same harness as the derive/filter HTTP contract tests). -// -// Contract: a signed source slot (deriveSources/filterSources) for a root -// field the user has already projected as a public ADD_COLUMN must be refused -// with a NAMED 4xx reasonCode on both channels — never the catch-all 500 -// CONSTRUCTION_RECEIPT_CONTRACT_INVALID that the duplicate semantic path -// triggers through the compiler's deliberate # disambiguation. Legit -// neighbors (bind-to-existing-column dedupe, distinct fields) keep their 200. - import ( "bytes" "context" @@ -20,6 +9,7 @@ import ( "testing" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" @@ -83,8 +73,16 @@ func fs1New(t *testing.T) *fs1Env { t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -130,8 +128,6 @@ func fs1New(t *testing.T) *fs1Env { return &fs1Env{t: t, app: app, basePath: "/api/v1/projects/project-a/explorers/custom/authoring/v2", service: service, snapshot: snapshot, initialVersion: 1, initialDigest: digest} } -// fs1AddColumn performs the shipped picker ADD_COLUMN flow for one candidate -// and returns the persisted upgraded document plus the current draft pin. func (e *fs1Env) fs1AddColumn(commandID, candidateID, title string) (authoringv2.Document, int64, string) { e.t.Helper() stored, err := e.service.Get(context.Background(), "project-a", "custom") @@ -278,11 +274,6 @@ func fs1MarshalConstruction(t *testing.T, construction authoringv2.Construction) return string(raw) } -// TestFS1DeriveSignedSlotSameFieldRefused: user ADD_COLUMN of -// Patient.multipleBirthInteger, then the CALCULATE lane signs the SAME field -// through a fresh deriveSources slot. Pre-fix this composed into the -// catch-all 500 (F-DS1 RED). The repair contract: named 422 refusal from the -// lifecycle lane entry. func TestFS1DeriveSignedSlotSameFieldRefused(t *testing.T) { env := fs1New(t) document, version, digest := env.fs1AddColumn("fs1-derive-add", "c_patient_birth", "Birth count") @@ -301,9 +292,6 @@ func TestFS1DeriveSignedSlotSameFieldRefused(t *testing.T) { } } -// TestFS1DeriveSignedSlotDistinctFieldStaysOK: the refusal boundary must not -// over-reach — a signed slot for one field alongside a user column of a -// DIFFERENT field stays a normal previewable proposal. func TestFS1DeriveSignedSlotDistinctFieldStaysOK(t *testing.T) { env := fs1New(t) document, version, digest := env.fs1AddColumn("fs1-distinct-add", "c_patient_id", "ID") @@ -320,10 +308,6 @@ func TestFS1DeriveSignedSlotDistinctFieldStaysOK(t *testing.T) { } } -// TestFS1DeriveOperandExistingColumnStaysOK pins the probe6 case-D dedupe -// shape: the DERIVE left operand reuses the user's OWN column id for the same -// field with no signed slot binding (200 pre- and post-fix). The refusal must -// not fire for the same-column dedupe. func TestFS1DeriveOperandExistingColumnStaysOK(t *testing.T) { env := fs1New(t) document, version, digest := env.fs1AddColumn("fs1-dedupe-add", "c_patient_birth", "Birth count") @@ -345,11 +329,6 @@ func TestFS1DeriveOperandExistingColumnStaysOK(t *testing.T) { } } -// TestFS1FilterSignedSlotSameFieldRefused is the filter-lane twin of F-DS1: -// user ADD_COLUMN of Patient.active, then a signed filterSources slot for the -// SAME field on a fresh column. It shares the reconcile predicate through -// c53759fe; pre-fix it must be captured at its own boundary and refused with -// a named 422 post-fix. func TestFS1FilterSignedSlotSameFieldRefused(t *testing.T) { env := fs1New(t) document, version, digest := env.fs1AddColumn("fs1-filter-add", "c_patient_active", "Active") @@ -366,8 +345,6 @@ func TestFS1FilterSignedSlotSameFieldRefused(t *testing.T) { } } -// TestFS1FilterSignedSlotDistinctFieldStaysOK keeps the filter lane's legit -// neighborhood: user column for id + signed filter slot for active → 200. func TestFS1FilterSignedSlotDistinctFieldStaysOK(t *testing.T) { env := fs1New(t) document, version, digest := env.fs1AddColumn("fs1-filter-distinct-add", "c_patient_id", "ID") @@ -382,8 +359,6 @@ func TestFS1FilterSignedSlotDistinctFieldStaysOK(t *testing.T) { } } -// --- v2 reviewer-scenario replicas (fs1-patch-review.md F1/F2) --- - func (e *fs1Env) fs1V2Pin() (int64, string, authoringv2.Document) { e.t.Helper() stored, err := e.service.Get(context.Background(), "project-a", "custom") @@ -414,13 +389,6 @@ func (e *fs1Env) fs1V2Command(commandID, commandsJSON string, version int64, dig return requestJSON(e.t, e.app, http.MethodPost, e.basePath+"/commands", body) } -// TestFS1RootPrefixedUserColumnStaysOK replicates reviewer finding F1: the -// shipped UPDATE_COLUMN_SOURCE authors a user column whose Source.Field.Path -// is the root.-prefixed cosmetic variant of the same field. The reconcile -// guard (explorer_output_reconcile.go:507) compares strings.TrimSpace on both -// sides verbatim, so "root.multipleBirthInteger" never collides with the -// slot's "multipleBirthInteger" in the compiler's duplicate semantic-path -// pass — clean b72a1322 returns 200 and must stay 200. func TestFS1RootPrefixedUserColumnStaysOK(t *testing.T) { env := fs1New(t) document, _, _ := env.fs1AddColumn("fs1-prefix-add", "c_patient_birth", "Birth count") @@ -479,12 +447,6 @@ func fs1AppliedStep(stepID, outputColID, slot string, columnsOut []authoringv2.S } } -// TestFS1SecondSignedSlotConflictsAppliedSlot replicates reviewer finding F2: -// slot A is signed+APPLIED for multipleBirthInteger, then a second signed -// slot B for the same field arrives in a new proposal. The merged construction -// carries both projections, so the compiler suffixes both public columns and -// the reconcile guard 500s. Scanning only document.Columns misses this; the -// refusal must name the merged source-projection carriers too. func TestFS1SecondSignedSlotConflictsAppliedSlot(t *testing.T) { env := fs1New(t) document, version, digest := env.fs1AddColumn("fs1-two-add", "c_patient_id", "ID") diff --git a/internal/server/health_readiness_wiring_test.go b/internal/server/health_readiness_wiring_test.go new file mode 100644 index 000000000..75ed369c5 --- /dev/null +++ b/internal/server/health_readiness_wiring_test.go @@ -0,0 +1,180 @@ +package server + +import ( + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "strings" + "testing" + + "github.com/calypr/loom/internal/backends/postgres/control" +) + +func httpConfigLiteral(t *testing.T, file, funcName string) *ast.CompositeLit { + t.Helper() + set := token.NewFileSet() + parsed, err := parser.ParseFile(set, file, nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", file, err) + } + var found *ast.CompositeLit + for _, decl := range parsed.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Name.Name != funcName || fn.Body == nil { + continue + } + ast.Inspect(fn.Body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok || sel.Sel.Name != "NewHTTPServer" { + return true + } + if len(call.Args) != 1 { + return true + } + lit, ok := call.Args[0].(*ast.CompositeLit) + if !ok { + return true + } + tv, ok := lit.Type.(*ast.SelectorExpr) + if !ok || tv.Sel.Name != "HTTPConfig" { + return true + } + if found != nil { + t.Fatalf("%s has more than one httpapi.NewHTTPServer HTTPConfig literal; the witness must be re-read, not widened", funcName) + } + found = lit + return true + }) + } + if found == nil { + t.Fatalf("no httpapi.NewHTTPServer(HTTPConfig{...}) literal found in %s::%s", file, funcName) + } + return found +} + +func literalField(lit *ast.CompositeLit, key string) (ast.Expr, bool) { + for _, elt := range lit.Elts { + kv, ok := elt.(*ast.KeyValueExpr) + if !ok { + continue + } + if k, ok := kv.Key.(*ast.Ident); ok && k.Name == key { + return kv.Value, true + } + } + return nil, false +} + +func callsSelector(node ast.Node, recv, name string) bool { + found := false + ast.Inspect(node, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + if sel, ok := call.Fun.(*ast.SelectorExpr); ok { + if id, ok := sel.X.(*ast.Ident); ok && id.Name == recv && sel.Sel.Name == name { + found = true + } + } + return true + }) + return found +} + +func TestHealthWiringPerCompositionArm(t *testing.T) { + markReadinessLeg(t, "TestHealthWiringPerCompositionArm") + for _, arm := range []struct { + file, funcName string + coreRecv, coreMethod string + }{ + {file: "runtime.go", funcName: "composePostgreSQLFullRuntime", coreRecv: "runtime", coreMethod: "Readiness"}, + {file: "server.go", funcName: "composePostgreSQLFirstQueryRoute", coreRecv: "runtime", coreMethod: "Readiness"}, + } { + arm := arm + t.Run(arm.funcName, func(t *testing.T) { + lit := httpConfigLiteral(t, arm.file, arm.funcName) + core, ok := literalField(lit, "CoreReadyCheck") + if !ok { + t.Fatal("CoreReadyCheck missing") + } + if !callsSelector(core, arm.coreRecv, arm.coreMethod) { + t.Fatalf("CoreReadyCheck must call %s.%s (the full probe with the ledger check); a narrower hook silently regresses schema-awareness", arm.coreRecv, arm.coreMethod) + } + state, ok := literalField(lit, "CoreState") + if !ok { + t.Fatal("CoreState classifier missing on the PostgreSQL arm") + } + if ident, ok := state.(*ast.Ident); !ok || ident.Name != "postgreSQLCoreReadinessState" { + t.Fatalf("CoreState must be postgreSQLCoreReadinessState, got %#v", state) + } + }) + } + + t.Run("ArangoArm_unchanged_and_without_classifier", func(t *testing.T) { + lit := httpConfigLiteral(t, "compose.go", "composeArangoRuntime") + for _, key := range []string{"CoreReadyCheck", "ClickHouseReadyCheck", "ClickHouseEnabled"} { + if _, ok := literalField(lit, key); !ok { + t.Fatalf("Arango arm lost its published field %s", key) + } + } + if _, ok := literalField(lit, "CoreState"); ok { + t.Fatal("the Arango arm must not gain a CoreState classifier: its published bodies stay byte-verbatim") + } + core, _ := literalField(lit, "CoreReadyCheck") + if !callsSelector(core, "lifecycleClient", "QueryRows") { + t.Fatal("the Arango arm's core probe closure changed; this ticket is additive-only for other backends' arms") + } + }) +} + +func TestPostgreSQLCoreReadinessStateClassification(t *testing.T) { + markReadinessLeg(t, "TestPostgreSQLCoreReadinessStateClassification") + for _, tt := range []struct { + name string + err error + want string + }{ + {"closed", fmt.Errorf("postgres backend: %w", control.ErrRuntimeClosed), pgReadinessStateClosed}, + {"connectivity", fmt.Errorf("postgres backend: schema readiness for series %q: %w", "source", control.ErrConnect), pgReadinessStateConnectivityDown}, + {"version", fmt.Errorf("ping: %w", control.ErrUnsupportedVersion), pgReadinessStateUnsupportedVer}, + {"schema missing", fmt.Errorf("verify: %w: schema \"x\" does not exist", control.ErrInvalidSchemaName), pgReadinessStateSchemaMissing}, + {"unmigrated", fmt.Errorf("verify: %w: 7 pending migration(s), first version 1 (ledger)", control.ErrMigrationPending), pgReadinessStateSchemaUnmigrated}, + {"checksum", fmt.Errorf("verify: %w", control.ErrMigrationChecksumMismatch), pgReadinessStateSchemaMismatch}, + {"future", fmt.Errorf("verify: %w", control.ErrFutureSchemaVersion), pgReadinessStateSchemaMismatch}, + {"vanished", fmt.Errorf("verify: %w", control.ErrMigrationMissing), pgReadinessStateSchemaMismatch}, + {"out of order", fmt.Errorf("verify: %w", control.ErrMigrationOutOfOrder), pgReadinessStateSchemaMismatch}, + {"unknown", errors.New("some other fault"), pgReadinessStateUnknown}, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + if got := postgreSQLCoreReadinessState(tt.err); got != tt.want { + t.Fatalf("postgreSQLCoreReadinessState(%v) = %q, want %q", tt.err, got, tt.want) + } + }) + } + if got := postgreSQLCoreReadinessState(nil); got != "" { + t.Fatalf("a healthy check must classify to the empty token, got %q", got) + } + credErr := fmt.Errorf(`acquire: dial tcp postgres://svc:hunter2@db.internal:5432/loom: %w`, control.ErrConnect) + if token := postgreSQLCoreReadinessState(credErr); token != pgReadinessStateConnectivityDown || strings.Contains(token, "hunter2") { + t.Fatalf("classifier must report the fixed token only, got %q", token) + } + published := map[string]bool{ + pgReadinessStateClosed: true, pgReadinessStateConnectivityDown: true, + pgReadinessStateUnsupportedVer: true, pgReadinessStateSchemaMissing: true, + pgReadinessStateSchemaUnmigrated: true, pgReadinessStateSchemaMismatch: true, + pgReadinessStateUnknown: true, "": true, + } + for _, err := range []error{nil, errors.New("x"), control.ErrConnect, control.ErrMigrationPending} { + if !published[postgreSQLCoreReadinessState(err)] { + t.Fatalf("classifier produced an unpublishable token %q", postgreSQLCoreReadinessState(err)) + } + } +} diff --git a/internal/server/openapi_explorer_authoring_direct.go b/internal/server/openapi_explorer_authoring_direct.go index 0cd98eb9b..9033c4ef8 100644 --- a/internal/server/openapi_explorer_authoring_direct.go +++ b/internal/server/openapi_explorer_authoring_direct.go @@ -9,9 +9,9 @@ import ( "net/http" "time" - shared "github.com/arangodb/go-driver/v2/arangodb/shared" loomapi "github.com/calypr/loom/generated/loomapi" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/explorer" @@ -49,39 +49,11 @@ func authResourcePathFromParam(value *loomapi.AuthResourcePath) string { return string(*value) } -// Authorizer storage faults are reported with the repo's backend-unavailable -// named fault rather than 403, so a store/client failure is never mistaken for -// a permission decision. The name follows the shared taxonomy: authscope -// authorizes the boundary with ErrAuthorizationBackendUnavailable and the -// dataframe layer maps it to dataframeerrors.CodeBackendUnavailable (see -// internal/api/http/errors.go:53-55 and internal/api/authpolicy/policy.go:36-37). const ( authoringBackendUnavailableCode = "BACKEND_UNAVAILABLE" authoringBackendUnavailableMessage = "authorization backend unavailable" ) -// isAuthorizerStorageFault classifies an AuthorizeWrite error as a storage or -// transport fault (not a permission decision) by error TYPE and explicit -// numeric response fields — there is no message-substring matching. It -// extends the repo's authz taxonomy exactly as -// internal/api/authpolicy/policy.go:36-37 groups context.DeadlineExceeded, -// dataframeerrors.ErrBackendUnavailable and authscope.ErrAuthorizationBackendUnavailable -// into the retryable backend-unavailable class (mirrored at -// internal/api/http/errors.go:53-55), and recognises Arango client faults by -// the driver error TYPE (errors.As on shared.ArangoError, read for value and -// pointer form as in internal/server/server.go:237-243). A driver response -// that carries the Arango permission-decision fields (HTTP 401/403 or -// ErrorNum shared.ErrForbidden=11 — the fields the driver itself defines as -// ErrHttpUnauthorized/ErrHttpForbidden/ErrForbidden) is a store-level -// permission DENIAL, not a backend fault: it keeps the 403 arm. Query-fault -// responses (Code 400 family, bind-parameter error numbers such as -// shared.ErrQueryBindParameterUndeclared) remain storage faults. Genuine -// permission denials from the ScopeResolver wrap -// authscope.ErrForbidden/ErrUnauthenticated (scope_resolver.go) and are -// checked first, so this predicate can only ever lift a non-permission -// storage fault out of the 403 arm; it can never relabel a denial — sentinel -// or store-native — as 5xx. An unclassified error stays in the 403 arm -// (unchanged pre-repair posture) — fail closed. func isAuthorizerStorageFault(err error) bool { if err == nil { return false @@ -94,29 +66,12 @@ func isAuthorizerStorageFault(err error) bool { errors.Is(err, context.DeadlineExceeded) { return true } - var arangoErr shared.ArangoError - if errors.As(err, &arangoErr) { - return !isArangoPermissionResponse(arangoErr) - } - var arangoErrPointer *shared.ArangoError - if errors.As(err, &arangoErrPointer) && arangoErrPointer != nil { - return !isArangoPermissionResponse(*arangoErrPointer) + if arangofactory.IsArangoErrorResponse(err) { + return !arangofactory.IsArangoPermissionResponse(err) } return false } -// isArangoPermissionResponse reports whether an Arango driver response is the -// store's own permission decision (fail closed to 403) rather than a backend -// fault (retryable 5xx). Matching is on the driver's numeric response fields -// only: go-driver v2 shared/error.go defines ErrHttpUnauthorized=401, -// ErrHttpForbidden=403 and ErrForbidden=11; those are Arango saying -// "not authorized", which must never surface as retryable availability. -func isArangoPermissionResponse(arangoErr shared.ArangoError) bool { - return arangoErr.Code == shared.ErrHttpUnauthorized || - arangoErr.Code == shared.ErrHttpForbidden || - arangoErr.ErrorNum == shared.ErrForbidden -} - func (h *explorerHTTPHandlers) authoringWriteDirect(ctx context.Context, project, authResourcePath string) error { if h == nil || h.authorizer == nil { return explorerUnavailable("authorization", "AUTHORING_UNAVAILABLE", "Explorer authoring is not configured") diff --git a/internal/server/openapi_explorer_authoring_write_fault_test.go b/internal/server/openapi_explorer_authoring_write_fault_test.go index bd7999918..578850c99 100644 --- a/internal/server/openapi_explorer_authoring_write_fault_test.go +++ b/internal/server/openapi_explorer_authoring_write_fault_test.go @@ -20,11 +20,6 @@ import ( "github.com/gofiber/fiber/v3" ) -// faultInjectionAuthPathStore is the overlay stub at the catalog-store -// boundary the ScopeResolver consults for the F-B27 project binding -// (ScopeResolverConfig.ListExistingAuthResourcePaths — the production seam -// wired to catalogStore.DiscoverExistingAuthResourcePaths, which is exactly -// where the dc52ed0a dataset_generation bind fault originated). type faultInjectionAuthPathStore struct { calls int paths []string @@ -80,9 +75,6 @@ func faultInjectionRouteStack(handlers *explorerHTTPHandlers) *fiber.App { return app } -// arangoQueryFault mirrors the exact production chain of the masked bind -// fault: store/arango queryRows wraps the Arango client error with -// "arango query: %w" (client.go:345) and the catalog store propagates it raw. func arangoQueryFault() error { return fmt.Errorf("arango query: %w", shared.ArangoError{HasError: true, Code: http.StatusBadRequest, ErrorNum: 1561, ErrorMessage: "bind parameter 'dataset_generation' was not declared in the query"}) } @@ -101,13 +93,6 @@ func authoringWriteStorageFaultCases() []struct { } } -// TestAuthoringWriteDirectStorageFaultNotMaskedAsForbidden is the durable for -// the filed masking class: a storage-class fault injected at the catalog-store -// boundary reaches authoringWriteDirect through the real ScopeResolver F-B27 -// project-binding branch (scope_resolver.go:242 region) and must not be -// classified as a permission denial. The named fault is 503-class and the -// injected cause stays visible through errors.Is. Pre-fix (HEAD d3a1d12a) -// every case returned the 403 FORBIDDEN envelope. func TestAuthoringWriteDirectStorageFaultNotMaskedAsForbidden(t *testing.T) { for _, test := range authoringWriteStorageFaultCases() { t.Run(test.name, func(t *testing.T) { @@ -143,12 +128,6 @@ func TestAuthoringWriteDirectStorageFaultNotMaskedAsForbidden(t *testing.T) { } } -// TestAuthoringWriteWriteDeniedReferencesKeepForbiddenEnvelope proves the -// precision half of the repair from both directions: genuine denials from the -// real ScopeResolver (outside caller scope, and the F-B27 recorded-path -// branch with a healthy store) keep the byte-identical 403 FORBIDDEN -// envelope and are never reinterpreted as 5xx. These assertions hold before -// and after the classification change. func TestAuthoringWriteDeniedReferencesKeepForbiddenEnvelope(t *testing.T) { t.Run("outside-caller-scope", func(t *testing.T) { store := &faultInjectionAuthPathStore{paths: []string{faultInjectionCallerPath}} @@ -207,10 +186,6 @@ func decodeAuthoringFaultCode(t *testing.T, response *http.Response) string { return body.Error.Code } -// TestPublishExplorerRouteStorageFaultRendersNamed503 proves the class fix at -// the HTTP boundary for the generated authoring mutation routes: the store -// fault renders the named 5xx envelope instead of 403, while a real denial -// renders the unchanged 403. func TestPublishExplorerRouteStorageFaultRendersNamed503(t *testing.T) { store := &faultInjectionAuthPathStore{fault: arangoQueryFault()} app := faultInjectionRouteStack(faultInjectionHandlers(store)) @@ -248,11 +223,6 @@ func TestPublishExplorerRouteRealDenialStillRenders403(t *testing.T) { } } -// arangoDriverFault builds a store-seam fault that carries the go-driver -// response fields (HTTP Code + Arango ErrorNum) the way store/arango -// queryRows propagates them ("arango query: %w" at client.go:345). The -// ErrorMessage is wire content only; the classifier must decide on the -// numeric fields, never on it. func arangoDriverFault(code, errorNum int, message string) error { return fmt.Errorf("arango query: %w", shared.ArangoError{HasError: true, Code: code, ErrorNum: errorNum, ErrorMessage: message}) } @@ -263,11 +233,6 @@ type authoringWriteArangoResponseCase struct { wantStorage bool } -// authoringWriteArangoResponseCases is the C2 matrix: auth-class driver -// responses (HTTP 401/403, ErrorNum 11 ErrForbidden) must classify as a -// permission decision (403 arm), while query-fault-class responses (Code 400 -// family, bind-parameter/undeclared error numbers) stay storage faults → 5xx. -// The bind-1552 case mirrors the dc52ed0a production fault verbatim. func authoringWriteArangoResponseCases() []authoringWriteArangoResponseCase { return []authoringWriteArangoResponseCase{ {name: "http-401-response", fault: arangoDriverFault(shared.ErrHttpUnauthorized, 0, "not authorized to execute this operation"), wantStorage: false}, @@ -279,12 +244,6 @@ func authoringWriteArangoResponseCases() []authoringWriteArangoResponseCase { } } -// TestAuthoringWriteDirectArangoResponseClassification pins the C2 repair at -// the filed choke point: store-native permission responses stay in the -// byte-identical 403 FORBIDDEN arm (never BACKEND_UNAVAILABLE), query faults -// stay named 5xx, and the cause chain survives in both directions. Pre-fix -// (blanket errors.As arm, no response-field guard) the three auth-class -// cases returned 503 BACKEND_UNAVAILABLE. func TestAuthoringWriteDirectArangoResponseClassification(t *testing.T) { for _, test := range authoringWriteArangoResponseCases() { t.Run(test.name, func(t *testing.T) { @@ -317,8 +276,6 @@ func TestAuthoringWriteDirectArangoResponseClassification(t *testing.T) { } } -// TestPublishExplorerRouteArangoResponseClassification pins the same matrix -// through the generated authoring route (HTTP render + response body code). func TestPublishExplorerRouteArangoResponseClassification(t *testing.T) { for _, test := range authoringWriteArangoResponseCases() { t.Run(test.name, func(t *testing.T) { @@ -349,8 +306,6 @@ func TestPublishExplorerRouteArangoResponseClassification(t *testing.T) { } } -// TestCreateExplorerLegacyArangoResponseClassification pins the matrix at the -// legacy createExplorer AuthorizeWrite site (500-class arm). func TestCreateExplorerLegacyArangoResponseClassification(t *testing.T) { for _, test := range authoringWriteArangoResponseCases() { t.Run(test.name, func(t *testing.T) { @@ -385,9 +340,6 @@ func TestCreateExplorerLegacyArangoResponseClassification(t *testing.T) { } } -// TestPublishRepositoryLegacyArangoResponseClassification pins the matrix at -// the legacy publishRepositoryExplorerConfig AuthorizeWrite site (503-class -// arm for genuine storage faults; permission responses stay 403). func TestPublishRepositoryLegacyArangoResponseClassification(t *testing.T) { authPath := faultInjectionCallerPath request := loomapi.PublishRepositoryExplorerConfigRequestObject{ @@ -423,11 +375,6 @@ func TestPublishRepositoryLegacyArangoResponseClassification(t *testing.T) { } } -// TestAuthoringWriteDirectArangoResponseCompositePrecedence pins the C1 -// ordering after the response-field guard: a composite carrying the -// authscope denial sentinel keeps the 403 arm even when joined with an -// Arango 403 response, while a composite explicitly claiming the backend -// sentinel stays the named 5xx availability fault. Both keep their causes. func TestAuthoringWriteDirectArangoResponseCompositePrecedence(t *testing.T) { t.Run("joined-denial-beats-arango-403", func(t *testing.T) { composite := errors.Join( @@ -466,10 +413,6 @@ func TestAuthoringWriteDirectArangoResponseCompositePrecedence(t *testing.T) { }) } -// third AuthorizeWrite masking site (legacy publishRepositoryExplorerConfig). -// Its OpenAPI route supports a 503 arm, so a storage fault renders the named -// 503-class legacy envelope; a real denial keeps the byte-identical -// 403 forbidden string form. func TestPublishRepositoryLegacyAuthorizerFaultNotMaskedAsForbidden(t *testing.T) { authPath := faultInjectionCallerPath request := loomapi.PublishRepositoryExplorerConfigRequestObject{ @@ -510,10 +453,6 @@ func TestPublishRepositoryLegacyAuthorizerFaultNotMaskedAsForbidden(t *testing.T }) } -// legacy twin of the same mask on POST /explorers (openapi_explorer_direct.go -// authorizer call, whose generated route has no 503 arm, so the named fault -// is the 500-class envelope with the cause preserved). Real denials keep the -// byte-identical legacy FORBIDDEN. func TestCreateExplorerLegacyAuthorizerFaultNotMaskedAsForbidden(t *testing.T) { t.Run("storage-fault", func(t *testing.T) { store := &faultInjectionAuthPathStore{fault: arangoQueryFault()} diff --git a/internal/server/openapi_explorer_commands_notfound_test.go b/internal/server/openapi_explorer_commands_notfound_test.go index 2ec355c72..3ded0f4ea 100644 --- a/internal/server/openapi_explorer_commands_notfound_test.go +++ b/internal/server/openapi_explorer_commands_notfound_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" @@ -19,11 +20,6 @@ import ( "github.com/gofiber/fiber/v3" ) -// countingExplorerStore wraps testExplorerStore and instruments the single -// Get method the Explorer Service reaches through for owner reads. faultGetOn -// (1-based within the current apply request) returns an identity-mismatched -// owner that loadConstructionBase classifies as typed notFound -// EXPLORER_NOT_FOUND (internal/explorer/lifecycle/construction_proposal.go:1451-1453). type countingExplorerStore struct { *testExplorerStore getCalls int @@ -51,12 +47,6 @@ type groupFieldSourceProposalFixture struct { store *countingExplorerStore } -// newGroupFieldSourceProposalFixture builds, through the real routes, a saved -// GROUP construction proposal whose candidate receipt carries compiled Group -// field sources (BINDING keys). That is the precondition for -// prepareConstructionProposal's group-field-source branch -// (internal/explorer/lifecycle/construction_proposal.go:1637-1638), which is -// the only apply-path branch that reaches loadConstructionBase. func newGroupFieldSourceProposalFixture(t *testing.T) *groupFieldSourceProposalFixture { t.Helper() scope := authscope.ReadScope{Mode: authscope.ReadScopeUnrestricted} @@ -144,7 +134,15 @@ func newGroupFieldSourceProposalFixture(t *testing.T) *groupFieldSourceProposalF t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -283,19 +281,6 @@ func (f *groupFieldSourceProposalFixture) apply(t *testing.T, commandID string) )) } -// TestApplyConstructionProposalOwnerVanishReturnsTypedNotFound is a RED -// reproduction: applying an APPLY_CONSTRUCTION_PROPOSAL command whose saved -// candidate carries a compiled Group field source, while the owner read in -// loadConstructionBase (internal/explorer/lifecycle/construction_proposal.go:1447-1453) -// returns an identity-mismatched Explorer, must surface the typed lifecycle -// notFound EXPLORER_NOT_FOUND as HTTP 404 via the route's 404 arm -// (internal/server/openapi_explorer_routes.go). Before the AuthoringNotFound -// response was declared on this operation, the same fault produced -// HTTP 500 "applyExplorerBuilderCommands returned undocumented HTTP status 404". -// The control subtest proves the fixture reaches the group-field-source prepare -// path (200 with >=2 store.Get calls during apply: the applyWorkspaceCommands -// owner read plus the loadConstructionBase read); removing the 404 arm turns -// this test back red with that 500 signature. func TestApplyConstructionProposalOwnerVanishReturnsTypedNotFound(t *testing.T) { controlApplyGetCalls := 0 diff --git a/internal/server/openapi_explorer_construction_apply_preview_test.go b/internal/server/openapi_explorer_construction_apply_preview_test.go index 75c23e76c..d9d08967d 100644 --- a/internal/server/openapi_explorer_construction_apply_preview_test.go +++ b/internal/server/openapi_explorer_construction_apply_preview_test.go @@ -274,8 +274,16 @@ func applyConstructionPreviewRouteFixture(t *testing.T, failApplyCapture, emptyA t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) diff --git a/internal/server/openapi_explorer_construction_derive_source_test.go b/internal/server/openapi_explorer_construction_derive_source_test.go index b7f0d58cf..035718990 100644 --- a/internal/server/openapi_explorer_construction_derive_source_test.go +++ b/internal/server/openapi_explorer_construction_derive_source_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" @@ -136,8 +137,16 @@ func TestConstructionDeriveSourceHTTPContractPreservesCapabilitiesAndProposals(t t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -328,8 +337,6 @@ func TestConstructionDeriveSourceHTTPContractPreservesCapabilitiesAndProposals(t t.Fatalf("forged deriveSources token did not reach choice validation: %s", forgedTokenHTTP.Body) } - // The boolean active choice is not advertised as a derive source; a token - // forged from the filter capability surface must not compile as an operand. nonNumericHTTP := requestJSON(t, app, http.MethodPost, basePath+"/construction-proposals", fmt.Sprintf( `{"snapshotToken":%q,"expectedDraftVersion":1,"expectedDraftDigest":%q,"outputId":"patients","changedStepId":"derive_step","candidateConstruction":%s,"deriveSources":[{"choiceId":%q,"columnId":%q}]}`, snapshot.Token, digest, candidateJSON, choiceID+"x", projectionColumnID, diff --git a/internal/server/openapi_explorer_construction_expand_source_test.go b/internal/server/openapi_explorer_construction_expand_source_test.go index 579eb7519..c725af080 100644 --- a/internal/server/openapi_explorer_construction_expand_source_test.go +++ b/internal/server/openapi_explorer_construction_expand_source_test.go @@ -1,15 +1,5 @@ package server -// Wire-level contract for the server-signed EXPAND source lane over HTTP. -// The lane signs ConstructionExpand.ExpandedField from resolved root primitive -// array facts; the capabilities DTO once dropped that field entirely (only a -// browser journey caught it). These assertions parse the RAW HTTP body, which -// is sound precisely because the handlers marshal through the generated -// loomapi DTOs (getConstructionCapabilitiesDirect / construction-proposal -// return typed loomapi responses via directAuthoringJSON); if a refactor ever -// returned raw authoring JSON directly, the DTO-drop class would be -// structurally impossible but this comment's premise must be re-verified. - import ( "bytes" "context" @@ -20,6 +10,7 @@ import ( "testing" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" @@ -54,11 +45,6 @@ type expandSourceHTTPInputProbe struct { Choices []expandSourceHTTPChoiceProbe `json:"choices"` } -// expandSourceHTTPProcedureSnapshot is the authorized catalog for a -// Procedure-rooted table. Procedure.instantiatesUri[] is the generated -// primitive direct-root array (canonical instantiatesUri[], ARRAY/MANY, -// FHIRType string, non-reference) that the generated schema admits for the -// signed EXPAND lane; Patient admits none. func expandSourceHTTPProcedureSnapshot() capability.Snapshot { snapshot := testAuthoringV2CapabilitySnapshot() snapshot.Nodes = append(snapshot.Nodes, capability.Node{ @@ -86,12 +72,6 @@ func expandSourceHTTPProcedureSnapshot() capability.Snapshot { return snapshot } -// expandSourceHTTPProcedureWorkspace mirrors baselineExplorerWorkspaceV2 for a -// Procedure root with one saved output column; the construction prefix stays -// empty so the EXPAND step arrives as the first proposed step over HTTP. -// The signed EXPAND lane only accepts a direct-root primitive many-cardinality -// non-reference array, which the generated schema admits for Procedure via -// instantiatesUri[]. func expandSourceHTTPProcedureWorkspace() []byte { visible, order := true, 0 workspace := authoringv2.Workspace{APIVersion: authoringv2.APIVersion, Kind: authoringv2.WorkspaceKind, Explorer: authoringv2.ExplorerMetadata{Title: "Procedures"}, Documents: []authoringv2.Document{{Kind: authoringv2.Kind, Output: authoringv2.Output{ID: "procedures", Title: "Procedures", RowLabel: "Procedures"}, RootResourceType: "Procedure", Route: authoringv2.RouteNode{OccurrenceID: "base", ResourceType: "Procedure"}, Columns: []authoringv2.Column{{Column: "procedure_id", Label: "Procedure ID", LogicalType: "string", OccurrenceID: "base", Source: authoringv2.ColumnSource{Kind: authoringv2.SourceField, Field: &authoringv2.FieldSource{Path: "id", ProjectionMode: "FIRST"}}, Table: &authoringv2.TablePresentation{Visible: &visible, Order: &order}}}}}, Tabs: []authoringv2.Tab{{ID: "procedures-tab", Title: "Procedures", OutputID: "procedures", Order: 0, Visible: true}}} @@ -120,12 +100,6 @@ func expandSourceHTTPDecodedInput(t *testing.T, body string) (map[string]json.Ra return keys, &input } -// expandSourceHTTPSignedField walks the RAW wire construction JSON down to -// steps[0].operation.expand and returns both the raw expand subtree and the -// server-signed expandedField parsed as a string map. Nothing here decodes -// into generated loomapi DTOs: a DTO that silently drops expandedField must -// fail this helper, which is the exact regression the browser journey used to -// be the only witness for. func expandSourceHTTPSignedField(t *testing.T, rawConstruction []byte, contextText string) (json.RawMessage, map[string]string) { t.Helper() var construction map[string]json.RawMessage @@ -228,8 +202,16 @@ func TestConstructionExpandSourceHTTPContractSignsAndPersistsExpandedField(t *te t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -275,7 +257,6 @@ func TestConstructionExpandSourceHTTPContractSignsAndPersistsExpandedField(t *te registerGeneratedExplorerTestRoutes(app, authscope.AllowAllAuthorizer{}, func(context.Context, *authscope.Principal, string) error { return nil }, service, config) basePath := "/api/v1/projects/project-a/explorers/custom/authoring/v2" - // --- 1. capabilities advertises the signed EXPAND source lane on raw wire --- capabilitiesHTTP := requestJSON(t, app, http.MethodPost, basePath+"/construction-capabilities", fmt.Sprintf( `{"snapshotToken":%q,"expectedDraftVersion":1,"expectedDraftDigest":%q,"outputId":"procedures","stageId":"source_projection"}`, snapshot.Token, digest, @@ -340,7 +321,6 @@ func TestConstructionExpandSourceHTTPContractSignsAndPersistsExpandedField(t *te } choiceID := *choice.ChoiceID - // --- 2. propose with a forged expandedField: the server signs over it --- candidateConstruction := authoringv2.Construction{Version: authoringv2.ConstructionVersion, Steps: []authoringv2.ConstructionStep{{ ID: expandSourceHTTPStep, Inputs: []authoringv2.ConstructionInputRef{{Kind: authoringv2.ConstructionInputSourceProjection}}, @@ -425,7 +405,6 @@ func TestConstructionExpandSourceHTTPContractSignsAndPersistsExpandedField(t *te t.Fatalf("expand-source proposal mutated the stored draft: before=%#v after=%#v", storedBefore, storedUntouched) } - // --- 3. negative: garbage expandSources choiceId is refused by code --- garbageChoiceHTTP := requestJSON(t, app, http.MethodPost, basePath+"/construction-proposals", fmt.Sprintf( `{"snapshotToken":%q,"expectedDraftVersion":1,"expectedDraftDigest":%q,"outputId":"procedures","changedStepId":%q,"candidateConstruction":%s,"expandSources":[{"choiceId":%q,"columnId":%q}]}`, snapshot.Token, digest, expandSourceHTTPStep, candidateJSON, "forged-choice", expandSourceHTTPInput, @@ -437,7 +416,6 @@ func TestConstructionExpandSourceHTTPContractSignsAndPersistsExpandedField(t *te t.Fatalf("garbage expandSources choiceId did not reach choice validation: %s", garbageChoiceHTTP.Body) } - // --- 4. apply advances the draft; the persisted signed lane is durable --- applyHTTP := requestJSON(t, app, http.MethodPost, basePath+"/commands", fmt.Sprintf( `{"commandId":"expand-source-apply","semanticsVersion":%d,"snapshotToken":%q,"expectedDraftVersion":1,"expectedDraftDigest":%q,"commands":[{"type":"APPLY_CONSTRUCTION_PROPOSAL","outputId":"procedures","proposalId":%q}]}`, authoringv2.CurrentSemanticsVersion, snapshot.Token, digest, proposal.ProposalID, @@ -456,9 +434,6 @@ func TestConstructionExpandSourceHTTPContractSignsAndPersistsExpandedField(t *te t.Fatalf("expand-source apply did not advance the draft: %#v body=%s", applied, applyHTTP.Body) } - // --- 5. PARITY GUARD: after the draft advanced, capabilities must still - // carry the server-signed expandedField on the RAW wire. This is the wire - // contract the capabilities DTO once silently dropped. --- advancedCapabilitiesHTTP := requestJSON(t, app, http.MethodPost, basePath+"/construction-capabilities", fmt.Sprintf( `{"snapshotToken":%q,"expectedDraftVersion":2,"expectedDraftDigest":%q,"outputId":"procedures","stageId":"source_projection"}`, snapshot.Token, applied.DraftDigest, @@ -496,9 +471,6 @@ func TestConstructionExpandSourceHTTPContractSignsAndPersistsExpandedField(t *te t.Fatalf("durable baseConstruction sourceProjections = %#v, want the expand-owned %q projection: %s", baseProjections.SourceProjections, expandSourceHTTPInput, baseConstructionRaw) } - // The lane restores the durable materialized column into the advertised - // choice only while the owner step still carries the coherent signed - // payload — the same state the parity guard just proved is persisted. _, advancedInput := expandSourceHTTPDecodedInput(t, advancedCapabilitiesHTTP.Body) if len(advancedInput.Choices) != 1 || advancedInput.Choices[0].ColumnID == nil || *advancedInput.Choices[0].ColumnID != expandSourceHTTPInput { t.Fatalf("post-apply expandSourceInput.choices = %#v, want the owned materialized columnId %q", advancedInput.Choices, expandSourceHTTPInput) diff --git a/internal/server/openapi_explorer_construction_filter_source_test.go b/internal/server/openapi_explorer_construction_filter_source_test.go index 32c7990e2..00d014692 100644 --- a/internal/server/openapi_explorer_construction_filter_source_test.go +++ b/internal/server/openapi_explorer_construction_filter_source_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" "github.com/calypr/loom/internal/dataframe/compiler/lower" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" @@ -118,8 +119,16 @@ func TestConstructionFilterSourceHTTPContractPreservesCapabilitiesAndProposals(t t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -319,13 +328,6 @@ func rawFilterSourceChoices(t *testing.T, body string) []json.RawMessage { func filterSourceHTTPBool(value bool) *bool { return &value } -// --- Reviewer F-1: durable filter-owned projections against reconcile --- -// -// The applied-draft threat model: a projection persisted with honest owner -// and column identity (the checks that existed) but tampered field facts. -// These probes pair the durable claims with the compiled artifact of the -// honest accepted plan -- the state reconcile must refuse. - const filterSourceF1Column = "col_reconcile_integer" const filterSourceF1HonestField = "multipleBirthInteger" const filterSourceF1OwnerStep = "only_step" @@ -403,9 +405,6 @@ func filterSourceF1Workspace(t *testing.T, snapshot capability.Snapshot, project return workspace } -// filterSourceF1Compile compiles the honest accepted plan; its compiled -// source column identity is the artifact a tampered durable projection is -// measured against, mirroring the production compileExplorerReceipt pairing. func filterSourceF1Compile(t *testing.T, snapshot capability.Snapshot) (authoringv2.Workspace, dataframeexecution.Resolved) { t.Helper() workspace := filterSourceF1Workspace(t, snapshot, filterSourceF1HonestProjection()) @@ -414,8 +413,16 @@ func filterSourceF1Compile(t *testing.T, snapshot capability.Snapshot) (authorin t.Fatalf("compile honest filter-source plan: %v", err) } engine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -461,8 +468,6 @@ func TestConstructionFilterSourceReconcileRefusesLogicalTypeFlip(t *testing.T) { filterSourceF1Refuse(t, tampered, "compiled source column identity") } -// The honest durable projection must keep passing the branch with the -// authorized snapshot threaded through. func TestConstructionFilterSourceReconcileClaimsHonestProjection(t *testing.T) { snapshot := filterSourceF1Snapshot() workspace, resolved := filterSourceF1Compile(t, snapshot) @@ -475,11 +480,6 @@ func TestConstructionFilterSourceReconcileClaimsHonestProjection(t *testing.T) { } } -// Reviewer R1/P5: reconcile runs only from compileExplorerReceipt, which -// always threads Authorized.Snapshot, so an unarmed call is test-only today -// -- but the armed facts gate must never SKIP silently. An honest durable -// filter-owned claim whose field stopped qualifying (unpopulated here) must -// be refused, not published, when the caller brings a blank snapshot token. func TestConstructionFilterSourceReconcileClaimsOnlyWithAuthorizedSnapshot(t *testing.T) { armedSnapshot := filterSourceF1Snapshot() workspace, resolved := filterSourceF1Compile(t, armedSnapshot) diff --git a/internal/server/openapi_explorer_construction_proposal_test.go b/internal/server/openapi_explorer_construction_proposal_test.go index 56b73723e..eba4c5126 100644 --- a/internal/server/openapi_explorer_construction_proposal_test.go +++ b/internal/server/openapi_explorer_construction_proposal_test.go @@ -14,6 +14,7 @@ import ( loomapi "github.com/calypr/loom/generated/loomapi" "github.com/calypr/loom/internal/authscope" "github.com/calypr/loom/internal/catalog" + "github.com/calypr/loom/internal/dataframe/compiler" "github.com/calypr/loom/internal/dataframe/compiler/lower" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" @@ -166,8 +167,16 @@ func TestConstructionProposalHTTPContractPreviewsAndAppliesRemovalOnly(t *testin t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -436,8 +445,16 @@ func TestRelatedExpandChoiceHTTPReturnsExactStageBoundRoute(t *testing.T) { t.Fatal(err) } engine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -672,8 +689,16 @@ func TestConstructionProposalSemanticGroupChoiceReconcilesCompiledOutputMetadata t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -899,7 +924,15 @@ func TestConstructionProposalCurrentCodingChoiceReconcilesAndPreviewsHTTP(t *tes t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -1342,7 +1375,15 @@ func TestConstructionProposalRelatedCodingGroupReconcilesReceiptMetadataHTTP(t * t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -1822,9 +1863,17 @@ func TestRelatedSourceConstructionProposalHTTPPreviewsAllMatchesAsList(t *testin t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - ScopeDigest: recipeScopeDigest, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + ScopeDigest: recipeScopeDigest, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, RootPageRows: 100, }) if err != nil { @@ -1950,8 +1999,6 @@ func TestRelatedSourceConstructionProposalHTTPPreviewsAllMatchesAsList(t *testin t.Fatalf("related-source default ALL value = %#v, want the full matching list", got) } - // Exercise the same exact proposal through the generated OpenAPI request DTO. - // A missing wire field silently resets ONE to the default ALL array. related.RowValuePolicy = authoringv2.ConstructionRowValueOne candidateJSON, err = json.Marshal(candidateConstruction) if err != nil { @@ -2006,7 +2053,6 @@ func TestBuilderHTTPSerializesEmptyConstructionStepsAsArray(t *testing.T) { if !bytes.Contains(draft, []byte(`"steps":[]`)) { t.Fatalf("test draft did not contain canonical empty steps: %s", draft) } - // Simulate a legacy persisted draft that decoded its empty slice as null. draft = bytes.Replace(draft, []byte(`"steps":[]`), []byte(`"steps":null`), 1) legacyDigest, err := workspace.LegacyNilConstructionStepsDigest() if err != nil { @@ -2165,8 +2211,16 @@ func TestConstructionProposalCurrentCodingGroupSpecimenFocusObservationHTTP(t *t t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, ScopeDigest: recipeScopeDigest, RootPageRows: 100, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, ScopeDigest: recipeScopeDigest, RootPageRows: 100, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -2392,8 +2446,16 @@ func TestEmptyConstructionBootstrapAddFirstColumnAndProposeOperation(t *testing. t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) diff --git a/internal/server/openapi_explorer_construction_unpivot_source_test.go b/internal/server/openapi_explorer_construction_unpivot_source_test.go index a170706b2..63ff72f2f 100644 --- a/internal/server/openapi_explorer_construction_unpivot_source_test.go +++ b/internal/server/openapi_explorer_construction_unpivot_source_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" @@ -154,8 +155,16 @@ func TestConstructionUnpivotSourceHTTPContractPreservesCapabilitiesAndProposals( t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -380,10 +389,6 @@ func TestConstructionUnpivotSourceHTTPContractPreservesCapabilitiesAndProposals( `{"snapshotToken":%q,"expectedDraftVersion":1,"expectedDraftDigest":%q,"outputId":"patients","changedStepId":"unpivot_step","candidateConstruction":%s,"unpivotSources":[{"choiceId":%q,"columnId":%q}]}`, snapshot.Token, digest, func() string { - // Melt a string-valued identity column together with the signed - // integer slot: the joint kind set is outside the compiler's melt - // rule (tableScalarKindsCompatible), so the lifecycle lane must - // refuse before the candidate reaches the compiler. incompatibleMelt := authoringv2.ConstructionStep{ ID: "unpivot_step", Inputs: []authoringv2.ConstructionInputRef{{Kind: authoringv2.ConstructionInputStepOutput, StepID: "derive_step"}}, @@ -439,12 +444,6 @@ func TestConstructionUnpivotSourceHTTPContractPreservesCapabilitiesAndProposals( } } -// --- F-A durable: melt-first signed UNPIVOT at the shipped handler boundary --- -// (Journaled defect F-A, authority /tmp/opencode/fa-report.md §1-§4; durable -// port of the reviewer's /tmp/opencode/fa-repro/fa_melt_first_overlay_test.go, -// renamed to the lane family. Same harness as the shipped contract test: -// fiber + real compileExplorerReceipt + recipe engine.) - type unpivotMeltFirstEnv struct { t *testing.T app *fiber.App @@ -496,8 +495,16 @@ func newUnpivotMeltFirstEnvWithCandidates(t *testing.T, extra ...capability.Cand t.Fatal(err) } recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) @@ -567,10 +574,6 @@ func (e *unpivotMeltFirstEnv) meltFirstSignedChoice(t *testing.T, fieldPath stri return "" } -// meltFirstCompiledColumnID returns the COMPILED source_projection stage -// column id for an authored name; the melt reads stage columns, not authored -// column ids, and using an authored id short-circuits to NEEDS_REPAIR before -// compile+reconcile, masking F-A. func (e *unpivotMeltFirstEnv) meltFirstCompiledColumnID(t *testing.T, authorName string) string { t.Helper() var envelope struct { @@ -633,13 +636,6 @@ func unpivotMeltFirstConstruction(publicColumnID, stepID, signedColumnID string, }}} } -// F-A regression: the compiler legitimately seeds the melt-signed projection -// into SOURCE_PROJECTION (compilation/construction.go -> sourceColumns -> -// lower/recipe_construction.go resolveConstructionSourceSchema), so the melt -// as FIRST step is a legit compiled state with no possible authored carrier: -// reconcile must never answer the catch-all 500 -// CONSTRUCTION_RECEIPT_CONTRACT_INVALID. RED pre-fix (500), GREEN post-fix -// (honest 200 preview; reviewer cross-check /tmp/opencode/fa-repro). func TestConstructionUnpivotSourceMeltFirstSignedIsNotContractInvalid(t *testing.T) { env := newUnpivotMeltFirstEnv(t) choiceID := env.meltFirstSignedChoice(t, "gender") @@ -665,8 +661,6 @@ func TestConstructionUnpivotSourceMeltFirstSignedIsNotContractInvalid(t *testing } } -// Control, isolating F-A to the signed-melt-first gap: the same first-step -// melt without a signed binding stays a clean 200 preview pre- and post-fix. func TestConstructionUnpivotSourceMeltFirstPublicOnlyStaysGreen(t *testing.T) { env := newUnpivotMeltFirstEnv(t) const stepID = "unpivot_public_first" @@ -689,13 +683,6 @@ func TestConstructionUnpivotSourceMeltFirstPublicOnlyStaysGreen(t *testing.T) { } } -// --- dated date-family melt refusal (operator decision #4) --- -// A signed date or date_time melt through the shipped proposal handler must -// return the named 422 refusal, discovery must hide the date-family choices, -// and the permitted STRING class must stay a clean 200 preview. Pre-fix, the -// signed effectiveDateTime melt reached a 200 decorated preview on the -// accepted binary (authority /tmp/opencode/jrev/probe-date-melt-dp4.log). - func unpivotDateRejectCandidates() []capability.Candidate { return []capability.Candidate{ { @@ -715,10 +702,6 @@ func unpivotDateRejectCandidates() []capability.Candidate { } } -// The date family is hidden from discovery by the refusal, so the request -// token is minted through the RowChoicePlanner directly (same authority the -// lifecycle boundary tests use), proving discovery and admission fail closed -// together. func unpivotPlannerSignedChoice(t *testing.T, snapshot capability.Snapshot, path string) string { t.Helper() index, err := fhirschema.GeneratedIndex() diff --git a/internal/server/openapi_explorer_direct.go b/internal/server/openapi_explorer_direct.go index 9649cd777..360c6743f 100644 --- a/internal/server/openapi_explorer_direct.go +++ b/internal/server/openapi_explorer_direct.go @@ -14,9 +14,6 @@ import ( "github.com/calypr/loom/internal/projectid" ) -// explorerLegacyOperationError carries the legacy Explorer response contract. -// It is a server-side transport value; lifecycle errors are mapped into it at -// the generated OpenAPI boundary. type explorerLegacyOperationError struct { status int code string @@ -74,9 +71,6 @@ func lifecycleLegacyFailure(err error) error { return legacyOperationFailureDetails(status, value.Code, value.Message, value.Details, err) } -// legacyOperationFailureDetails keeps the lifecycle-error details (explicit -// retryability and other safe structured diagnostics) so the legacy encoder can -// emit them; the contract forbids clients inferring retryability from status. func legacyOperationFailureDetails(status int, code, message string, details map[string]any, cause error) error { return &explorerLegacyOperationError{status: status, code: code, message: message, details: details, cause: cause} } diff --git a/internal/server/openapi_explorer_draft_restore_test.go b/internal/server/openapi_explorer_draft_restore_test.go index c682130a9..2ad400d35 100644 --- a/internal/server/openapi_explorer_draft_restore_test.go +++ b/internal/server/openapi_explorer_draft_restore_test.go @@ -137,7 +137,7 @@ func TestRestoreDraftRevisionPublicAPIIsScopedAndReloadVisible(t *testing.T) { t.Fatalf("restore idempotency status=%d body=%s", replayed.StatusCode, replayed.Body) } - assertReload() // Duplicate command must not advance or overwrite the restored draft. + assertReload() staleCASBody := fmt.Sprintf(`{"commandId":"stale-restore-cas","semanticsVersion":%d,"snapshotToken":%q,"expectedDraftVersion":4,"expectedDraftDigest":%q,"commands":[{"type":"RESTORE_DRAFT_REVISION","draftRevisionId":%q}]}`, authoringv2.CurrentSemanticsVersion, snapshot.Token, currentDigest, restored.PreviousDraftRevisionID) @@ -146,7 +146,7 @@ func TestRestoreDraftRevisionPublicAPIIsScopedAndReloadVisible(t *testing.T) { t.Fatalf("stale restore CAS status=%d body=%s", staleCAS.StatusCode, staleCAS.Body) } - assertReload() // A rejected compare-and-swap must leave the saved workspace intact. + assertReload() store.mu.Lock() key := testExplorerKey(owner.Project, owner.ExplorerID) + "\x00" + restored.PreviousDraftRevisionID @@ -160,7 +160,7 @@ func TestRestoreDraftRevisionPublicAPIIsScopedAndReloadVisible(t *testing.T) { if staleScope.StatusCode != http.StatusConflict || !strings.Contains(staleScope.Body, `"code":"STALE_DRAFT_SOURCE_CONTEXT"`) { t.Fatalf("stale restore source scope status=%d body=%s", staleScope.StatusCode, staleScope.Body) } - assertReload() // A rejected source-scope check must not write draft state. + assertReload() } func containsJSONField(raw, key, value string) bool { diff --git a/internal/server/openapi_explorer_legacy_details_control_test.go b/internal/server/openapi_explorer_legacy_details_control_test.go index b9c239631..3e37bb558 100644 --- a/internal/server/openapi_explorer_legacy_details_control_test.go +++ b/internal/server/openapi_explorer_legacy_details_control_test.go @@ -1,11 +1,5 @@ package server -// Durable control (F-B28, round 43): docs/loom-reliability-contract.md states -// retryability is explicit and clients must not infer it from an HTTP status or -// message. The legacy Explorer error encoder must therefore carry -// lifecycle.Error.Details (e.g. retryable:false for a permanent release -// activation miss) into the marshalled body instead of dropping it. - import ( "encoding/json" "errors" @@ -19,8 +13,6 @@ import ( func marshalLegacyBody(t *testing.T, err error) (map[string]any, int) { t.Helper() status, legacy := explorerLegacyResponse(err) - // Route wrappers embed this value verbatim into the status-specific JSON - // response, so marshalling it reproduces the client-visible envelope. raw, merr := json.Marshal(legacy) if merr != nil { t.Fatalf("marshal legacy envelope: %v", merr) @@ -37,7 +29,6 @@ func marshalLegacyBody(t *testing.T, err error) (map[string]any, int) { } func TestLegacyFailureCarriesExplicitRetryabilityForActivationMiss(t *testing.T) { - // Same value repository_publish.go:84-88 builds for a typed activation miss. miss := &lifecycle.Error{Class: lifecycle.ClassConflict, Stage: "repository_publish", Code: "RELEASE_ACTIVATION_FAILED", Message: "activate published ExplorerConfigV2: requirements unmet", Details: map[string]any{"retryable": false}, Cause: errors.New("release requirements unmet")} body, status := marshalLegacyBody(t, lifecycleLegacyFailure(miss)) if status != http.StatusConflict { @@ -52,8 +43,6 @@ func TestLegacyFailureCarriesExplicitRetryabilityForActivationMiss(t *testing.T) } func TestLegacyFailureKeepsUnrelatedErrorsUnwidened(t *testing.T) { - // Plain (non-lifecycle) errors keep the existing structured-INTERNAL_ERROR - // envelope shape: no invented retryability keys. body, status := marshalLegacyBody(t, lifecycleLegacyFailure(errors.New("store exploded"))) if status != http.StatusInternalServerError { t.Fatalf("plain error status = %d, want 500", status) diff --git a/internal/server/openapi_explorer_population_mapping_construction_test.go b/internal/server/openapi_explorer_population_mapping_construction_test.go index 6cb45777b..f5445f8ad 100644 --- a/internal/server/openapi_explorer_population_mapping_construction_test.go +++ b/internal/server/openapi_explorer_population_mapping_construction_test.go @@ -13,6 +13,7 @@ import ( loomapi "github.com/calypr/loom/generated/loomapi" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" "github.com/calypr/loom/internal/dataframe/compiler/ir" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" @@ -95,8 +96,14 @@ func TestPopulationMappingRouteUsesReceiptBoundAuthoredFilterConstruction(t *tes } var mappingCalls, queryCalls int engine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(_ context.Context, query string, _ int, _ map[string]any, visit func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + query, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + queryCalls++ if !strings.Contains(query, "FILTER") { return fmt.Errorf("population mapping query omitted the authored filter: %s", query) diff --git a/internal/server/openapi_explorer_population_member_proposal_test.go b/internal/server/openapi_explorer_population_member_proposal_test.go index 51a4182b4..ae04b2b78 100644 --- a/internal/server/openapi_explorer_population_member_proposal_test.go +++ b/internal/server/openapi_explorer_population_member_proposal_test.go @@ -13,6 +13,7 @@ import ( loomapi "github.com/calypr/loom/generated/loomapi" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" @@ -214,8 +215,16 @@ func TestPopulationMemberRemovalProposalHTTPPreviewsAndAppliesExactScopedCAS(t * t.Fatal(err) } engine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) diff --git a/internal/server/openapi_explorer_restricted_starting_collection_integration_test.go b/internal/server/openapi_explorer_restricted_starting_collection_integration_test.go index 835f774a3..180bc6033 100644 --- a/internal/server/openapi_explorer_restricted_starting_collection_integration_test.go +++ b/internal/server/openapi_explorer_restricted_starting_collection_integration_test.go @@ -14,12 +14,10 @@ import ( loomapi "github.com/calypr/loom/generated/loomapi" "github.com/calypr/loom/internal/authscope" - catalogarango "github.com/calypr/loom/internal/catalog/arango" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/explorer" - explorerarango "github.com/calypr/loom/internal/explorer/arango" "github.com/calypr/loom/internal/explorer/capability" "github.com/calypr/loom/internal/explorer/lifecycle" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/google/uuid" ) @@ -44,11 +42,12 @@ func TestRestrictedStartingCollectionRejectsDeniedReferences(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - client, err := storearango.Open(ctx, arangoURL, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: arangoURL, Database: database}) if err != nil { t.Fatal(err) } - if err := client.Bootstrap(ctx, storearango.BootstrapSpec{Collections: []storearango.CollectionSpec{ + client := arangoBackend.Client + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{ {Name: "Patient"}, {Name: "fhir_field_catalog"}, }}); err != nil { @@ -146,11 +145,11 @@ FOR d IN Patient t.Fatalf("independent denied-path fixture=%v; want [%s]", got, deniedWitness) } - catalogStore, err := catalogarango.New(client) + catalogStore, err := arangofactory.NewCatalogStore(client) if err != nil { t.Fatal(err) } - explorerStore, err := explorerarango.New(client) + explorerStore, err := arangofactory.NewExplorerStore(client) if err != nil { t.Fatal(err) } @@ -158,9 +157,7 @@ FOR d IN Patient ListExistingAuthResourcePaths: catalogStore.DiscoverExistingAuthResourcePaths, }) principal, err := (authscope.StaticAuthenticator{Principal: authscope.Principal{ - Subject: "restricted-starting-collection-test", - // The project grant authorizes Explorer metadata; only this resource - // path is granted for the active generation. + Subject: "restricted-starting-collection-test", AuthResourcePaths: []string{project, allowedPath}, }}).Authenticate(ctx, nil) if err != nil { diff --git a/internal/server/openapi_explorer_route_roundtrip_test.go b/internal/server/openapi_explorer_route_roundtrip_test.go index b05509557..c6a56850c 100644 --- a/internal/server/openapi_explorer_route_roundtrip_test.go +++ b/internal/server/openapi_explorer_route_roundtrip_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/calypr/loom/internal/dataframe/compiler" "net/http" "reflect" "sort" @@ -13,25 +14,21 @@ import ( loomapi "github.com/calypr/loom/generated/loomapi" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" - explorerarango "github.com/calypr/loom/internal/explorer/arango" "github.com/calypr/loom/internal/explorer/authoringv2" "github.com/calypr/loom/internal/explorer/capability" "github.com/calypr/loom/internal/explorer/lifecycle" - storearango "github.com/calypr/loom/internal/store/arango" "github.com/gofiber/fiber/v3" ) -// Recovers the deleted saved-route journey's command/identity guarantees at the -// real public HTTP/service/compiler boundary. Preview execution is deliberately -// empty: this does not pretend a stubbed preview proves graph-row semantics. func TestSavedRelatedRouteProposalApplyRestoreAndFreshRead(t *testing.T) { savedRelatedRouteRoundtrip(t, nil) } -func savedRelatedRouteRoundtrip(t *testing.T, rowClient *storearango.Client) { +func savedRelatedRouteRoundtrip(t *testing.T, rowClient *arangofactory.Client) { t.Helper() snapshot := testAuthoringV2CapabilitySnapshot() for _, resource := range []string{"Observation", "Group", "Specimen"} { @@ -91,10 +88,10 @@ func savedRelatedRouteRoundtrip(t *testing.T, rowClient *storearango.Client) { } var store explorer.Store = newTestExplorerStore() if rowClient != nil { - if err := rowClient.Bootstrap(context.Background(), explorerarango.BootstrapSpec()); err != nil { + if err := rowClient.Bootstrap(context.Background(), arangofactory.ExplorerBootstrapSpec()); err != nil { t.Fatal(err) } - store, err = explorerarango.New(rowClient) + store, err = arangofactory.NewExplorerStore(rowClient) if err != nil { t.Fatal(err) } @@ -110,13 +107,20 @@ func savedRelatedRouteRoundtrip(t *testing.T, rowClient *storearango.Client) { if err != nil { t.Fatal(err) } - queryRows := func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil } + queryRows := func(context.Context, dataframeexecution.RowsRequest, dataframeexecution.RowVisitor) error { return nil } if rowClient != nil { - queryRows = func(ctx context.Context, query string, batch int, variables map[string]any, visit func(map[string]any) error) error { - return rowClient.QueryRows(ctx, query, batch, variables, storearango.RowVisitor(visit)) + queryRows = func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + batch := request.BatchSize + query, variables, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + + return rowClient.QueryRows(ctx, query, batch, variables, arangofactory.RowVisitor(visit)) } } - engine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: queryRows}) + engine, err := dataframeexecution.New(dataframeexecution.Config{CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: queryRows}) if err != nil { t.Fatal(err) } @@ -250,22 +254,21 @@ func savedRelatedRouteRoundtrip(t *testing.T, rowClient *storearango.Client) { if proposed.ProposalId == nil || proposed.Preview == nil || proposed.Preview.ReceiptId != *proposed.ProposalId { t.Fatalf("proposal missing reviewed preview: %#v", proposed) } - // Cancel is client-local: merely reviewing a proposal must not mutate the draft. canceled := load() raw, _ := canceled.Workspace.CanonicalJSON() if canceled.Version != 3 || canceled.Digest != digest || !bytes.Equal(raw, original) { t.Fatal("unapplied proposal changed saved draft") } - command := func(id string, version int64, expected string, commands string) string { + compiledCommand := func(id string, version int64, expected string, commands string) string { return fmt.Sprintf(`{"commandId":%q,"semanticsVersion":%d,"snapshotToken":%q,"expectedDraftVersion":%d,"expectedDraftDigest":%q,"commands":%s}`, id, authoringv2.CurrentSemanticsVersion, snapshot.Token, version, expected, commands) } apply := fmt.Sprintf(`[{"type":"APPLY_CONSTRUCTION_PROPOSAL","outputId":"patients","proposalId":%q}]`, *proposed.ProposalId) - rejected := requestJSON(t, app, http.MethodPost, base+"/commands", command("stale-route-apply", 2, digest, apply)) + rejected := requestJSON(t, app, http.MethodPost, base+"/commands", compiledCommand("stale-route-apply", 2, digest, apply)) if rejected.StatusCode != http.StatusConflict { t.Fatalf("stale Apply status=%d body=%s", rejected.StatusCode, rejected.Body) } assertRoute(load(), routeA) - appliedHTTP := requestJSON(t, app, http.MethodPost, base+"/commands", command("apply-route-b", 3, digest, apply)) + appliedHTTP := requestJSON(t, app, http.MethodPost, base+"/commands", compiledCommand("apply-route-b", 3, digest, apply)) if appliedHTTP.StatusCode != http.StatusOK { t.Fatalf("Apply status=%d body=%s", appliedHTTP.StatusCode, appliedHTTP.Body) } @@ -277,14 +280,14 @@ func savedRelatedRouteRoundtrip(t *testing.T, rowClient *storearango.Client) { if rowClient != nil { assertRows(reconcile(applied.Version, applied.Digest), []string{"p1/o2"}) } - replay := requestJSON(t, app, http.MethodPost, base+"/commands", command("apply-route-b", 3, digest, apply)) + replay := requestJSON(t, app, http.MethodPost, base+"/commands", compiledCommand("apply-route-b", 3, digest, apply)) if replay.StatusCode != http.StatusOK { t.Fatalf("duplicate Apply status=%d body=%s", replay.StatusCode, replay.Body) } if got := load(); got.Version != applied.Version || got.Digest != applied.Digest { t.Fatal("duplicate Apply mutated saved draft") } - restoredHTTP := requestJSON(t, app, http.MethodPost, base+"/commands", command("restore-route-a", applied.Version, applied.Digest, fmt.Sprintf(`[{"type":"RESTORE_DRAFT_REVISION","draftRevisionId":%q}]`, applied.Previous))) + restoredHTTP := requestJSON(t, app, http.MethodPost, base+"/commands", compiledCommand("restore-route-a", applied.Version, applied.Digest, fmt.Sprintf(`[{"type":"RESTORE_DRAFT_REVISION","draftRevisionId":%q}]`, applied.Previous))) if restoredHTTP.StatusCode != http.StatusOK { t.Fatalf("Restore status=%d body=%s", restoredHTTP.StatusCode, restoredHTTP.Body) } @@ -294,7 +297,7 @@ func savedRelatedRouteRoundtrip(t *testing.T, rowClient *storearango.Client) { if restored.Version != 5 || restored.Digest != digest || !bytes.Equal(restoredRaw, original) { t.Fatalf("restore did not recover exact route-A workspace/version: %#v", restored) } - lateApply := requestJSON(t, app, http.MethodPost, base+"/commands", command("late-route-b", 3, digest, apply)) + lateApply := requestJSON(t, app, http.MethodPost, base+"/commands", compiledCommand("late-route-b", 3, digest, apply)) if lateApply.StatusCode != http.StatusConflict { t.Fatalf("late Apply after restore status=%d body=%s", lateApply.StatusCode, lateApply.Body) } @@ -311,9 +314,7 @@ func savedRelatedRouteRoundtrip(t *testing.T, rowClient *storearango.Client) { t.Fatal("draft roundtrip changed published revision") } if rowClient != nil { - // A fresh repository instance reads persisted bytes rather than the service's - // response or the in-memory fixture used by the fast structural case. - fresh, err := explorerarango.New(rowClient) + fresh, err := arangofactory.NewExplorerStore(rowClient) if err != nil { t.Fatal(err) } diff --git a/internal/server/openapi_explorer_route_rows_arango_test.go b/internal/server/openapi_explorer_route_rows_arango_test.go index 5c44d1c51..63fd303b3 100644 --- a/internal/server/openapi_explorer_route_rows_arango_test.go +++ b/internal/server/openapi_explorer_route_rows_arango_test.go @@ -11,13 +11,10 @@ import ( "testing" "time" - storearango "github.com/calypr/loom/internal/store/arango" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/google/uuid" ) -// Uses real HTTP routes/compiler/executor, literal graph data, and the actual -// Arango Explorer/history repository in a fresh owned database. It proves stored -// read-back across repository instances, not a database-process restart. func TestSavedRelatedRouteRowsAgainstArango(t *testing.T) { endpoint := strings.TrimSpace(os.Getenv("LOOM_TEST_ARANGO_URL")) if endpoint == "" { @@ -26,10 +23,11 @@ func TestSavedRelatedRouteRowsAgainstArango(t *testing.T) { database := "loom_test_route_roundtrip_" + uuid.NewString() ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) defer cancel() - client, err := storearango.Open(ctx, endpoint, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: endpoint, Database: database}) if err != nil { t.Fatal(err) } + client := arangoBackend.Client t.Cleanup(func() { _ = client.Close(context.Background()) cleanupCtx, stop := context.WithTimeout(context.Background(), 5*time.Second) @@ -49,7 +47,7 @@ func TestSavedRelatedRouteRowsAgainstArango(t *testing.T) { t.Errorf("remove owned disposable database status=%d", response.StatusCode) } }) - if err := client.Bootstrap(ctx, storearango.BootstrapSpec{Collections: []storearango.CollectionSpec{{Name: "Patient"}, {Name: "Observation"}, {Name: "Group"}, {Name: "Specimen"}, {Name: "fhir_edge", Edge: true}}}); err != nil { + if err := client.Bootstrap(ctx, arangofactory.BootstrapSpec{Collections: []arangofactory.CollectionSpec{{Name: "Patient"}, {Name: "Observation"}, {Name: "Group"}, {Name: "Specimen"}, {Name: "fhir_edge", Edge: true}}}); err != nil { t.Fatal(err) } insert := func(collection string, documents []map[string]any) { diff --git a/internal/server/openapi_graphql_routes.go b/internal/server/openapi_graphql_routes.go index baa3af759..20b5776db 100644 --- a/internal/server/openapi_graphql_routes.go +++ b/internal/server/openapi_graphql_routes.go @@ -13,13 +13,6 @@ import ( "github.com/gofiber/fiber/v3" ) -// serveGraphQL invokes the native GraphQL http.Handler through a bounded -// request/response adapter. GraphQL remains a native HTTP transport; unlike -// the removed REST path this does not create a Fiber context or re-enter a -// Fiber handler. -// nilHandlerGraphQLEnvelope keeps the documented GraphQL response shape even -// on the defensive unwired path: every executed /graphql response, including -// this one, is a GraphQL envelope with top-level data/errors members. const nilHandlerGraphQLEnvelope = `{"data":null,"errors":[{"message":"GraphQL handler is not configured"}]}` func serveGraphQL(ctx context.Context, handler http.Handler, body any, method string) (int, []byte, error) { diff --git a/internal/server/openapi_graphql_status_test.go b/internal/server/openapi_graphql_status_test.go index e72d51479..78de3431c 100644 --- a/internal/server/openapi_graphql_status_test.go +++ b/internal/server/openapi_graphql_status_test.go @@ -16,16 +16,6 @@ import ( "github.com/gofiber/fiber/v3" ) -// Undeclared escalated statuses used to exit through unexpectedResponseStatus -// and surface as an opaque HTTP 500 {"error":{...}} service envelope, hiding -// the typed GraphQL extension code (round-87 live repro: a ghost selector on -// dataframeRows derived DATASET_NOT_FOUND, and the adapter collapsed it to -// INTERNAL_ERROR). In addition, $ref-component response classes generated key -// -wrapped embedded RawJSON structs, so even correctly-routed arms served -// {"GraphQLNotFoundJSONResponse":{...}} instead of a GraphQL envelope. Each -// documented GraphQL status must arrive on the wire — through the real strict -// handler and production error handler — with the correct status and an intact -// top-level data/errors envelope. func TestGraphQLRoutesServeDocumentedStatusesAsGraphQLEnvelopesOnTheWire(t *testing.T) { cases := []struct { code string diff --git a/internal/server/openapi_graphql_transport_review_test.go b/internal/server/openapi_graphql_transport_review_test.go index 4747070ea..5ebb151e8 100644 --- a/internal/server/openapi_graphql_transport_review_test.go +++ b/internal/server/openapi_graphql_transport_review_test.go @@ -95,7 +95,7 @@ func TestGraphQLTransportStatusMatrix(t *testing.T) { name string write func(w http.ResponseWriter, body string) json bool - expect int // wire status + expect int }{ {"405", func(w http.ResponseWriter, b string) { w.WriteHeader(405); _, _ = w.Write([]byte(b)) }, true, 500}, {"415", func(w http.ResponseWriter, b string) { w.WriteHeader(415); _, _ = w.Write([]byte(b)) }, true, 500}, diff --git a/internal/server/openapi_interpretation_routes_test.go b/internal/server/openapi_interpretation_routes_test.go index 9a7aec36c..3e2f671ca 100644 --- a/internal/server/openapi_interpretation_routes_test.go +++ b/internal/server/openapi_interpretation_routes_test.go @@ -27,8 +27,6 @@ func generatedInterpretationReadTestApp(readErr error) *fiber.App { } func registerGeneratedInterpretationRoutes(app *fiber.App, routes *HTTPRoutes) { - // RegisterHandlers uses the same generated strict adapter as production. A - // small helper keeps this test focused on auth/status behavior. loomapi.RegisterHandlersWithOptions(app, loomapi.NewStrictHandler(routes, nil), loomapi.FiberServerOptions{}) } diff --git a/internal/server/openapi_recipe_execution.go b/internal/server/openapi_recipe_execution.go index 2f2f29ca9..282724893 100644 --- a/internal/server/openapi_recipe_execution.go +++ b/internal/server/openapi_recipe_execution.go @@ -15,8 +15,6 @@ import ( "github.com/calypr/loom/internal/dataframe/publication" ) -// recipeExecution returns the legacy execution document consumed by ETL. A -// denied execution remains indistinguishable from a missing catalog entry. func (r *HTTPRoutes) recipeExecution(ctx context.Context, id string) (any, int) { if r.releases == nil { return map[string]any{"error": "recipe execution not found"}, http.StatusNotFound @@ -71,42 +69,8 @@ func (r *HTTPRoutes) recipeExecution(ctx context.Context, id string) (any, int) RowCount: output.RowCount, Columns: columns, } - // F-R3 twin parity (REPAIR-20b, adversarial review - // ses_e65cad76effe F-1/F-2/F-3): the output failure fields are built - // through the SAME encoding chain the GraphQL twin uses — the twin's - // own constructor dataframeapi.PersistedFailure - // (internal/api/graphql/graph/dataframe/model.go:73-84) exactly as - // executionModel (recipe_control_graphql.go:319,334) consumes it, and - // the same non-empty Phase gate it applies (:329-331). The raw text - // given to the constructor for the message slot is the reader-level - // mirror of execution.Error, gated by output health (REPAIR-20c): - // recipe_execution_reader.go mirrors execution.Error only onto - // outputs that are not themselves healthy (durable READY/PUBLISHED - // publish no mirrored text), so the lease-lost reconciliation shape - // (bundle_store.go:1207-1214 fails the execution, leaving READY - // outputs untouched) no longer stamps a phantom per-output trio — - // the ledger entry REPAIR-20b recorded as deliberately unfixed on - // the twin side is now repaired on both twins. A genuine per-output - // failure (failPhase bundle_store.go:989-995 persists its own - // FailureCode/Phase, or leaves the output FAILED) still projects. - // What that chain publishes — never a raw durable copy: - // - persisted code present -> the code's stable public message - // (errors.PublicMessage substitution) with the persisted code and - // retryable; - // - code-less failure text -> errors.SanitizePersistedFailure - // (internal/dataframe/errors/errors.go:271-277): unknown text is - // Normalize-inferred to a public message/INTERNAL_ERROR code/ - // inferred retryable and is never echoed to clients; - // whitespace-only text becomes present-but-empty "", "", false; - // - no persisted code and no raw text -> all three nil, so the - // omitempty generated keys stay absent (additive-optional - // contract, matching the twin publishing nothing when healthy). mirroredError := "" if item.State != string(publication.BundleReady) { - // recipeExecutionHTTPState renders both durable success states - // (READY and PUBLISHED) as READY; anything else (FAILED, in - // flight) keeps the execution.Error mirror, exactly as the - // GraphQL twin reader gates it. mirroredError = execution.Error } message, code, retryable := dataframeapi.PersistedFailure(mirroredError, output.FailureCode, output.FailureRetryable) @@ -126,16 +90,6 @@ func (r *HTTPRoutes) recipeExecution(ctx context.Context, id string) (any, int) State: recipeExecutionHTTPState(execution.State), Outputs: outputs, } - // F-R3 twin parity: the execution-level failure trio is built through the - // same twin constructor+sanitizer chain as the outputs above - // (PersistedFailure over execution.Error/FailureCode/FailureRetryable, - // plus the non-empty FailurePhase gate at recipe_control_graphql.go: - // 339-341), so both transports publish identical bytes for the same - // durable record (bundle_store.go failPhase:985-988, lease-lost :1207- - // 1213): public-message substitution for persisted codes, never echoing - // unknown durable text, explicit true/false errorRetryable on failure - // (docs/loom-reliability-contract.md "Retryability is explicit"), and - // total absence (nil -> omitempty) when no failure is persisted. response.Error, response.ErrorCode, response.ErrorRetryable = dataframeapi.PersistedFailure(execution.Error, execution.FailureCode, execution.FailureRetryable) if execution.FailurePhase != "" { response.Phase = &execution.FailurePhase @@ -143,26 +97,10 @@ func (r *HTTPRoutes) recipeExecution(ctx context.Context, id string) (any, int) return response, http.StatusOK } -// recipeExecutionReadScope consults the wired resolver for the auth paths an -// execution declares. The variable is the test seam for the CoversExecutionPaths -// defense-in-depth gate against resolver drift: a production resolver's -// Restricted success always echoes the requested path set back (scope_resolver.go -// ResolveReadScopeForGeneration), so the subset loop that makes Covers the -// sole disclosure gate is otherwise unreachable in e2e. The default remains -// the authentic resolver call; production behavior is unchanged. var recipeExecutionReadScope = func(ctx context.Context, scopes *authscope.ScopeResolver, principal *authscope.Principal, project, datasetGeneration string, declared []string) (authscope.ReadScope, error) { return scopes.ResolveReadScopeForGeneration(ctx, principal, project, datasetGeneration, declared) } -// recipeExecutionLookupResponse is the HTTP twin of the GraphQL reader's -// recipeExecutionLookupError (recipe_execution_reader.go:81-99). The durable -// not-found sentinel is the only signal that maps to a permanent, concealed -// not-found; every other store failure stays an explicit retryable outage so a -// transient Arango fault never collapses into a non-retryable 404. Unclassified -// faults fail safe as retryable dependency faults per -// docs/loom-reliability-contract.md, and denial-concealment per authpolicy -// keeps the not-found envelope. No new response shape is introduced: retryable -// faults reuse the same shared service error the authorization outage emits. func recipeExecutionLookupResponse(ctx context.Context, err error) (any, int) { if err == nil || errors.Is(err, publication.ErrBundleNotFound) { return map[string]any{"error": "recipe execution not found"}, http.StatusNotFound @@ -182,24 +120,11 @@ func recipeExecutionLookupResponse(ctx context.Context, err error) (any, int) { return response, status } -// recipeExecutionAuthorizationResponse keeps concealment for denials while -// exposing a retryable shared service envelope for an authorization outage. -// The generated OpenAPI adapter owns the final 503 response type; this seam -// keeps the semantic route behavior testable without hand-editing generated -// code. func recipeExecutionAuthorizationResponse(ctx context.Context, err error) (any, int) { status := recipeExecutionAuthorizationStatus(err) if status != http.StatusServiceUnavailable { return map[string]any{"error": "recipe execution not found"}, status } - // F-4: the default 503 arm must keep this function's docstring promise (and - // the GraphQL twin recipe_execution_reader.go:98 fail-safe) of a retryable - // shared service envelope for an authorization outage. A known sentinel - // outage already maps to the 503 BACKEND_UNAVAILABLE envelope and is - // untouched; an arbitrary catalog/authorization fault that would otherwise - // map to a non-retryable INTERNAL_ERROR gets the same fail-safe wrapper the - // lookup twin applies just above (mapServiceError then re-map when the - // shared mapper does not yield 503), so no new response shape is invented. mappedStatus, response := mapServiceError(err, ctx) if mappedStatus != http.StatusServiceUnavailable { _, response = mapServiceError(dataframeerrors.Wrap(err, dataframeerrors.CodeBackendUnavailable, "", dataframeerrors.WithRetryable(true)), ctx) @@ -212,16 +137,6 @@ func recipeExecutionAuthorizationStatus(err error) int { case errors.Is(err, authscope.ErrUnauthenticated): return http.StatusUnauthorized case errors.Is(err, authscope.ErrForbidden): - // F-B32: a per-generation scope denial (ResolveReadScopeForGeneration -> - // ErrForbidden for a restricted caller missing an auth_resource_path) is - // an unauthorized-to-see execution and must conceal to 404, matching the - // GraphQL twin (recipe_execution_reader.go:39-41 -> recipeExecutionLookupError:94 - // -> authpolicy Classify(ErrForbidden, OperationConcealedLookup)=CodeRecipeExecutionNotFound, - // policy.go:40-41,47-52) and this route's sibling uncovered-path branch - // (openapi_recipe_execution.go:40-42). A 403 here was a status-only - // existence oracle over the shared "recipe execution not found" body. - // Unauthenticated (401) and authorization-backend failure (503) arms are - // preserved unchanged. return http.StatusNotFound case errors.Is(err, authscope.ErrAuthorizationBackendUnavailable): return http.StatusServiceUnavailable diff --git a/internal/server/openapi_recipe_execution_test.go b/internal/server/openapi_recipe_execution_test.go index 011d659c1..bc496e188 100644 --- a/internal/server/openapi_recipe_execution_test.go +++ b/internal/server/openapi_recipe_execution_test.go @@ -46,7 +46,6 @@ func TestRecipeExecutionHTTPStatePreservesExplorerReadyContract(t *testing.T) { } func TestRecipeExecutionAuthorizationStatusDistinguishesDenialFromOutage(t *testing.T) { - // F-B32: concealed 404 — unified with GraphQL twin (reader:94 + authpolicy concealed-lookup) and CoversExecutionPaths sibling if got := recipeExecutionAuthorizationStatus(authscope.ErrForbidden); got != http.StatusNotFound { t.Fatalf("forbidden status = %d, want %d", got, http.StatusNotFound) } @@ -81,7 +80,6 @@ func TestRecipeAuthorizationAdapterUsesOperationAuthPolicy(t *testing.T) { func TestRecipeExecutionAuthorizationResponseConcealsDenialAndStructuresOutage(t *testing.T) { denial, status := recipeExecutionAuthorizationResponse(context.Background(), authscope.ErrForbidden) denialBody, ok := denial.(map[string]any) - // F-B32: concealed 404 — unified with GraphQL twin (reader:94 + authpolicy concealed-lookup) and CoversExecutionPaths sibling if status != http.StatusNotFound || !ok || denialBody["error"] != "recipe execution not found" { t.Fatalf("denial response = %#v, %d; want concealed 404", denial, status) } @@ -96,14 +94,6 @@ func TestRecipeExecutionAuthorizationResponseConcealsDenialAndStructuresOutage(t } } -// F-4 (reviewer ses_e66757becffe8uCNimGXbEeIfc): the default arm's own -// promise. recipeExecutionAuthorizationResponse's docstring keeps a "retryable -// shared service envelope" for an authorization outage, and the GraphQL twin -// (recipe_execution_reader.go:94-98 -> recipeExecutionLookupError) emits -// retryable BACKEND_UNAVAILABLE for an arbitrary catalog/authorization fault. -// A non-sentinel fault leaving the resolver seam therefore has to keep that -// envelope here too; the shipped arm instead emitted a non-retryable -// INTERNAL_ERROR under the 503. func TestRecipeExecutionAuthorizationResponseStructuresArbitraryFaultLikeTheTwin(t *testing.T) { outage, status := recipeExecutionAuthorizationResponse(context.Background(), errors.New("catalog auth path listing exploded")) if status != http.StatusServiceUnavailable { @@ -129,10 +119,6 @@ func TestRecipeExecutionAuthorizationResponseStructuresArbitraryFaultLikeTheTwin } } -// Same promise at the client-visible wire: an arbitrary (non-sentinel) fault -// surfacing from the per-generation resolver must reach the caller as the -// generated retryable 503 outage envelope, exactly like the sentinel outage -// case below. func TestGetRecipeExecutionAuthorizationArbitraryFaultIsRetryableOutage(t *testing.T) { routes := &HTTPRoutes{ releases: recipeExecutionCatalog{execution: publication.BundleExecution{ @@ -178,13 +164,6 @@ func TestGetRecipeExecutionReturnsGeneratedRetryableOutage(t *testing.T) { } } -// assertRecipeExecutionRetryableOutage proves the client-visible wire shape is -// the generated typed retryable outage envelope: the concrete response type is -// the retryable 503 discriminator, and the marshaled body carries an explicit -// retryable=true flag with the stable BACKEND_UNAVAILABLE code. Derived from -// docs/loom-reliability-contract.md ("Retryability is explicit; clients do not -// infer it from an HTTP status or message") and openapi/openapi.yaml -// getRecipeExecution's declared 503, not from the handler's internals. func assertRecipeExecutionRetryableOutage(t *testing.T, response loomapi.GetRecipeExecutionResponseObject, cause string) { t.Helper() typed, ok := response.(loomapi.GetRecipeExecution503JSONResponse) @@ -207,9 +186,6 @@ func assertRecipeExecutionRetryableOutage(t *testing.T, response loomapi.GetReci } } -// A raw Arango fault must stay an explicit retryable outage instead of folding -// into a permanent non-retryable not-found. Mirror of the GraphQL twin -// recipe_execution_reader.go:81-99 (faults -> retryable BACKEND_UNAVAILABLE). func TestGetRecipeExecutionStoreFaultIsRetryableOutage(t *testing.T) { routes := &HTTPRoutes{releases: recipeExecutionFaultCatalog{err: errors.New("arango connection reset")}} response, err := routes.GetRecipeExecution(context.Background(), loomapi.GetRecipeExecutionRequestObject{Id: "execution-a"}) @@ -219,7 +195,6 @@ func TestGetRecipeExecutionStoreFaultIsRetryableOutage(t *testing.T) { assertRecipeExecutionRetryableOutage(t, response, "raw store fault") } -// A deadline overrun is a transient fault, not evidence the execution is gone. func TestGetRecipeExecutionDeadlineExceededIsRetryableOutage(t *testing.T) { routes := &HTTPRoutes{releases: recipeExecutionFaultCatalog{err: context.DeadlineExceeded}} response, err := routes.GetRecipeExecution(context.Background(), loomapi.GetRecipeExecutionRequestObject{Id: "execution-a"}) @@ -229,8 +204,6 @@ func TestGetRecipeExecutionDeadlineExceededIsRetryableOutage(t *testing.T) { assertRecipeExecutionRetryableOutage(t, response, "deadline exceeded") } -// The only permanent-missing signal is the durable not-found sentinel; a -// wrapped sentinel keeps the existing concealed not-found body. func TestGetRecipeExecutionWrappedBundleNotFoundStaysNotFound(t *testing.T) { routes := &HTTPRoutes{releases: recipeExecutionFaultCatalog{err: fmt.Errorf("arango lookup failed: %w", publication.ErrBundleNotFound)}} response, err := routes.GetRecipeExecution(context.Background(), loomapi.GetRecipeExecutionRequestObject{Id: "execution-a"}) diff --git a/internal/server/openapi_semantic_inventory_test.go b/internal/server/openapi_semantic_inventory_test.go index 2c6d6e09c..0f7e14143 100644 --- a/internal/server/openapi_semantic_inventory_test.go +++ b/internal/server/openapi_semantic_inventory_test.go @@ -108,8 +108,6 @@ func TestBrowseSemanticInventoryThroughPublicAPI(t *testing.T) { status int }{ {`{"snapshotToken":"snapshot","rowRoot":"Observation","limit":51}`, http.StatusBadRequest}, - // A self-declared authorization scope in the body is rejected at the boundary - // (additionalProperties:false -> documented 400); scope only ever comes from the principal. {`{"snapshotToken":"snapshot","rowRoot":"Observation","query":"glucose","limit":2,"authResourcePaths":["/secret"]}`, http.StatusBadRequest}, {`{"snapshotToken":"old","rowRoot":"Observation"}`, http.StatusConflict}, } { @@ -120,19 +118,11 @@ func TestBrowseSemanticInventoryThroughPublicAPI(t *testing.T) { } denied = true response = requestJSON(t, app, http.MethodPost, path, body) - // reads==1: only the top-level browse reached the provider; the body-scoped - // injection is refused at the boundary before any read. if response.StatusCode != http.StatusForbidden || reads != 1 { t.Fatalf("denied status=%d reads=%d body=%s", response.StatusCode, reads, response.Body) } } -// TestBrowseSemanticInventoryOutputScopeReturnsTypedNotFound pins that lifecycle -// notFound classifications for output-scoped browsing reach the client as typed 404s. -// Before the 404/422 response mappings existed, the adapter collapsed them into -// 500 INTERNAL_ERROR ("browseExplorerSemanticInventory returned undocumented HTTP status 404"). -// Two branches are covered: EXPLORER_NOT_FOUND (no saved workspace) and -// OUTPUT_NOT_FOUND (saved doc-less workspace, semantic_catalog.go findSemanticOutput). func TestBrowseSemanticInventoryOutputScopeReturnsTypedNotFound(t *testing.T) { scope := authscope.ReadScope{Mode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{"/allowed"}} digest := sha256.Sum256([]byte(string(scope.Mode) + "\x00/allowed")) diff --git a/internal/server/openapi_status_coverage_test.go b/internal/server/openapi_status_coverage_test.go index 79f16f48e..cd9b8f34b 100644 --- a/internal/server/openapi_status_coverage_test.go +++ b/internal/server/openapi_status_coverage_test.go @@ -16,10 +16,6 @@ import ( loomapi "github.com/calypr/loom/generated/loomapi" ) -// TestOpenAPIAdapterStatusesMatchDeclaredResponses links the status switches -// in each strict adapter to the same embedded OpenAPI operation that generated -// the response union. Authentication failures are handled by the shared HTTP -// middleware, so a declared 401 need not be repeated in every adapter switch. func TestOpenAPIAdapterStatusesMatchDeclaredResponses(t *testing.T) { declared, err := declaredOpenAPIStatuses() if err != nil { diff --git a/internal/server/openapi_test_helpers_test.go b/internal/server/openapi_test_helpers_test.go index c73178bfd..4291d09eb 100644 --- a/internal/server/openapi_test_helpers_test.go +++ b/internal/server/openapi_test_helpers_test.go @@ -10,8 +10,6 @@ import ( "github.com/gofiber/fiber/v3" ) -// registerGeneratedExplorerTestRoutes exercises the same generated strict -// adapter used in production; tests no longer maintain a second route tree. func registerGeneratedExplorerTestRoutes(router fiber.Router, authorizer authscope.Authorizer, authorizeRead func(context.Context, *authscope.Principal, string) error, service *explorer.Service, config lifecycle.Config) { routes := &HTTPRoutes{explorer: newExplorerHTTPHandlers(authorizer, authorizeRead, service, config)} loomapi.RegisterHandlersWithOptions(router, loomapi.NewStrictHandler(routes, nil), loomapi.FiberServerOptions{}) diff --git a/internal/server/options.go b/internal/server/options.go index 0bcc373df..807913089 100644 --- a/internal/server/options.go +++ b/internal/server/options.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "os" + "regexp" "strconv" "strings" "time" @@ -22,37 +23,31 @@ type Config struct { } type ServerConfig struct { - Listen string `yaml:"listen"` - URL string `yaml:"url"` - Database string `yaml:"database"` - Schema string `yaml:"schema"` - ClickHouse ClickHouseConfig `yaml:"clickhouse"` - Dataframer DataframerConfig `yaml:"dataframer"` - RecipeBatchRows int `yaml:"recipe_batch_rows"` - RecipeBatchBytes int `yaml:"recipe_batch_bytes"` - RecipeQueryPageRows int `yaml:"recipe_query_page_rows"` - RecipeQualityMaxRows int64 `yaml:"recipe_quality_max_rows"` - RecipeQualityMaxDistinctKeys int64 `yaml:"recipe_quality_max_distinct_keys"` - ArtifactDirectory string `yaml:"artifact_directory"` - ArtifactTTL time.Duration `yaml:"artifact_ttl"` - ArtifactMaxRows int64 `yaml:"artifact_max_rows"` - ArtifactMaxBytes int64 `yaml:"artifact_max_bytes"` - PopulationMappingCursorSecret string `yaml:"population_mapping_cursor_secret"` - AllowUnauthenticated bool `yaml:"allow_unauthenticated"` - RequiredDataframeSelectors []dataset.DataframeSelector `yaml:"required_dataframe_selectors"` - LocalWorkspaceWriteback string `yaml:"local_workspace_writeback"` - LocalWorkspaceProject string `yaml:"local_workspace_project"` - // DevActivationConflictOnce is intentionally not configurable through YAML. - // It is a one-shot fault injector for the isolated, --no-auth development - // stack and is ignored unless both server and auth are unauthenticated. - DevActivationConflictOnce bool `yaml:"-"` - // DevRepositoryActivationConflictOnce is the repository-publish sibling of - // DevActivationConflictOnce. It is intentionally not configurable through - // YAML, arms the one-shot corrupted-compare fault on the repository - // activation leg, and is honored only under the same local no-auth mode - // gate as the other dev fault switches. - DevRepositoryActivationConflictOnce bool `yaml:"-"` - DevArtifactRowDelay time.Duration `yaml:"-"` + Listen string `yaml:"listen"` + Backend string `yaml:"backend"` + URL string `yaml:"url"` + Database string `yaml:"database"` + Schema string `yaml:"schema"` + PostgreSQL PostgreSQLConfig `yaml:"postgresql"` + ClickHouse ClickHouseConfig `yaml:"clickhouse"` + Dataframer DataframerConfig `yaml:"dataframer"` + RecipeBatchRows int `yaml:"recipe_batch_rows"` + RecipeBatchBytes int `yaml:"recipe_batch_bytes"` + RecipeQueryPageRows int `yaml:"recipe_query_page_rows"` + RecipeQualityMaxRows int64 `yaml:"recipe_quality_max_rows"` + RecipeQualityMaxDistinctKeys int64 `yaml:"recipe_quality_max_distinct_keys"` + ArtifactDirectory string `yaml:"artifact_directory"` + ArtifactTTL time.Duration `yaml:"artifact_ttl"` + ArtifactMaxRows int64 `yaml:"artifact_max_rows"` + ArtifactMaxBytes int64 `yaml:"artifact_max_bytes"` + PopulationMappingCursorSecret string `yaml:"population_mapping_cursor_secret"` + AllowUnauthenticated bool `yaml:"allow_unauthenticated"` + RequiredDataframeSelectors []dataset.DataframeSelector `yaml:"required_dataframe_selectors"` + LocalWorkspaceWriteback string `yaml:"local_workspace_writeback"` + LocalWorkspaceProject string `yaml:"local_workspace_project"` + DevActivationConflictOnce bool `yaml:"-"` + DevRepositoryActivationConflictOnce bool `yaml:"-"` + DevArtifactRowDelay time.Duration `yaml:"-"` } type ClickHouseConfig struct { @@ -63,6 +58,40 @@ type ClickHouseConfig struct { Password string `yaml:"password"` } +type PostgreSQLConfig struct { + Host string `yaml:"host"` + Port int `yaml:"port"` + Database string `yaml:"database"` + User string `yaml:"user"` + Password string `yaml:"password"` + SSLMode string `yaml:"sslmode"` + MaxOpenConns int `yaml:"max_open_conns"` + MaxIdleConns int `yaml:"max_idle_conns"` + ConnMaxLifetime time.Duration `yaml:"conn_max_lifetime"` + Schemas map[string]string `yaml:"schemas"` +} + +const ( + sourceBackendArango = "arango" + sourceBackendPostgres = "postgres" + + defaultServerURL = "http://127.0.0.1:8529" + defaultServerDatabase = "fhir_proto" + + defaultPostgresPort = 5432 + defaultPostgresSSLMode = "prefer" + defaultPostgresMaxOpenConns = 10 + defaultPostgresMaxIdleConns = 5 + defaultPostgresConnMaxLifetime = 30 * time.Minute +) + +var postgresSSLModes = []string{"disable", "allow", "prefer", "require", "verify-ca", "verify-full"} + +var postgresCLIFlagNames = []string{ + "pg-host", "pg-port", "pg-database", "pg-user", "pg-password", "pg-sslmode", + "pg-max-open-conns", "pg-max-idle-conns", "pg-conn-max-lifetime", +} + type DataframerConfig struct { Recipe string `yaml:"recipe"` } @@ -90,8 +119,14 @@ const defaultRecipeQueryPageRows = 250 func DefaultConfig() Config { return Config{ Server: ServerConfig{ - Listen: ":8080", URL: "http://127.0.0.1:8529", Database: "fhir_proto", - Schema: "schemas/graph-fhir.json", ClickHouse: ClickHouseConfig{Enabled: true, URL: "clickhouse://127.0.0.1:9000", Database: "loom", Username: "default"}, + Listen: ":8080", Backend: sourceBackendArango, URL: defaultServerURL, Database: defaultServerDatabase, + Schema: "schemas/graph-fhir.json", + PostgreSQL: PostgreSQLConfig{ + Port: defaultPostgresPort, SSLMode: defaultPostgresSSLMode, + MaxOpenConns: defaultPostgresMaxOpenConns, MaxIdleConns: defaultPostgresMaxIdleConns, + ConnMaxLifetime: defaultPostgresConnMaxLifetime, + }, + ClickHouse: ClickHouseConfig{Enabled: true, URL: "clickhouse://127.0.0.1:9000", Database: "loom", Username: "default"}, RecipeBatchRows: 1000, RecipeBatchBytes: 4 << 20, RecipeQueryPageRows: defaultRecipeQueryPageRows, RecipeQualityMaxRows: 1_000_000, RecipeQualityMaxDistinctKeys: 1_000_000, ArtifactDirectory: "/tmp/loom-artifacts", ArtifactTTL: 24 * time.Hour, @@ -106,9 +141,19 @@ func parseServerOptions(args []string, handling flag.ErrorHandling) (Config, err configPath string listen string noAuth bool + backend string url string database string schema string + pgHost string + pgPort int + pgDatabase string + pgUser string + pgPassword string + pgSSLMode string + pgMaxOpenConns int + pgMaxIdleConns int + pgConnMaxLifetime time.Duration clickhouseURL string clickhouseDatabase string clickhouseUsername string @@ -126,9 +171,19 @@ func parseServerOptions(args []string, handling flag.ErrorHandling) (Config, err fs.StringVar(&configPath, "config", "", "YAML server configuration file") fs.StringVar(&listen, "listen", ":8080", "HTTP listen address") fs.BoolVar(&noAuth, "no-auth", false, "disable scoped authorization for local development") - fs.StringVar(&url, "url", "http://127.0.0.1:8529", "ArangoDB URL") - fs.StringVar(&database, "database", "fhir_proto", "ArangoDB database") + fs.StringVar(&backend, "backend", sourceBackendArango, "source graph/catalog backend: arango or postgres") + fs.StringVar(&url, "url", defaultServerURL, "ArangoDB URL (ArangoDB backend only)") + fs.StringVar(&database, "database", defaultServerDatabase, "ArangoDB database (ArangoDB backend only)") fs.StringVar(&schema, "schema", "schemas/graph-fhir.json", "FHIR graph schema path for imports") + fs.StringVar(&pgHost, "pg-host", "", "PostgreSQL host (required with --backend postgres)") + fs.IntVar(&pgPort, "pg-port", defaultPostgresPort, "PostgreSQL port") + fs.StringVar(&pgDatabase, "pg-database", "", "PostgreSQL database (required with --backend postgres)") + fs.StringVar(&pgUser, "pg-user", "", "PostgreSQL user (required with --backend postgres)") + fs.StringVar(&pgPassword, "pg-password", "", "PostgreSQL password; never rendered or logged") + fs.StringVar(&pgSSLMode, "pg-sslmode", defaultPostgresSSLMode, "PostgreSQL TLS mode: disable, allow, prefer, require, verify-ca, or verify-full") + fs.IntVar(&pgMaxOpenConns, "pg-max-open-conns", defaultPostgresMaxOpenConns, "PostgreSQL maximum open pool connections") + fs.IntVar(&pgMaxIdleConns, "pg-max-idle-conns", defaultPostgresMaxIdleConns, "PostgreSQL maximum idle pool connections; must not exceed --pg-max-open-conns") + fs.DurationVar(&pgConnMaxLifetime, "pg-conn-max-lifetime", defaultPostgresConnMaxLifetime, "PostgreSQL maximum connection lifetime; zero disables expiry") fs.StringVar(&clickhouseURL, "clickhouse-url", "clickhouse://127.0.0.1:9000", "ClickHouse native URL for published dataframe reads") fs.StringVar(&clickhouseDatabase, "clickhouse-database", "loom", "ClickHouse database for published dataframe reads") fs.StringVar(&clickhouseUsername, "clickhouse-username", "default", "ClickHouse username for published dataframe reads") @@ -144,16 +199,14 @@ func parseServerOptions(args []string, handling flag.ErrorHandling) (Config, err if err := fs.Parse(args); err != nil { return Config{}, err } + explicitFlags := make(map[string]bool, fs.NFlag()) + fs.Visit(func(f *flag.Flag) { explicitFlags[f.Name] = true }) if strings.TrimSpace(configPath) != "" { cfg, err := LoadConfig(configPath) if err != nil { return Config{}, err } - // --no-auth is an explicit local-development override. Keep the - // configuration file authoritative for normal server settings, but honor - // this flag even when the server is started with --config (as the Helm - // local-development path does). if noAuth { cfg.Server.AllowUnauthenticated = true cfg.Auth.AllowUnauthenticated = true @@ -165,6 +218,9 @@ func parseServerOptions(args []string, handling flag.ErrorHandling) (Config, err cfg.Server.LocalWorkspaceWriteback = localWorkspaceWriteback cfg.Server.LocalWorkspaceProject = localWorkspaceProject } + if err := checkSourceBackendConflicts(&cfg, explicitFlags, true); err != nil { + return Config{}, fmt.Errorf("invalid server config: %w", err) + } if err := cfg.Validate(); err != nil { return Config{}, fmt.Errorf("invalid server config: %w", err) } @@ -176,9 +232,21 @@ func parseServerOptions(args []string, handling flag.ErrorHandling) (Config, err return Config{}, err } cfg.Server.Listen = listen + cfg.Server.Backend = backend cfg.Server.URL = url cfg.Server.Database = database cfg.Server.Schema = schema + cfg.Server.PostgreSQL = PostgreSQLConfig{ + Host: pgHost, + Port: pgPort, + Database: pgDatabase, + User: pgUser, + Password: pgPassword, + SSLMode: pgSSLMode, + MaxOpenConns: pgMaxOpenConns, + MaxIdleConns: pgMaxIdleConns, + ConnMaxLifetime: pgConnMaxLifetime, + } cfg.Server.ClickHouse.URL = clickhouseURL cfg.Server.ClickHouse.Database = clickhouseDatabase cfg.Server.ClickHouse.Username = clickhouseUsername @@ -198,6 +266,9 @@ func parseServerOptions(args []string, handling flag.ErrorHandling) (Config, err if err := applyDevFaultOverrides(&cfg); err != nil { return Config{}, err } + if err := checkSourceBackendConflicts(&cfg, explicitFlags, false); err != nil { + return Config{}, fmt.Errorf("invalid server config: %w", err) + } if err := cfg.Validate(); err != nil { return Config{}, fmt.Errorf("invalid server config: %w", err) } @@ -245,9 +316,6 @@ func applyEnvironment(cfg Config) (Config, error) { return cfg, nil } -// applyDevFaultOverrides is deliberately gated by the local no-auth mode. -// These switches provide deterministic fault scenarios for the isolated dev -// Compose stack without creating a production request or OpenAPI surface. func applyDevFaultOverrides(cfg *Config) error { if cfg == nil || !cfg.Server.AllowUnauthenticated || !cfg.Auth.AllowUnauthenticated { return nil @@ -297,6 +365,170 @@ func applyDevFaultOverrides(cfg *Config) error { return nil } +func normalizeSourceBackend(value string) (string, error) { + backend := strings.ToLower(strings.TrimSpace(value)) + if backend == "" { + return sourceBackendArango, nil + } + switch backend { + case sourceBackendArango, sourceBackendPostgres: + return backend, nil + default: + return "", fmt.Errorf("server.backend must be %s or %s, got %q", sourceBackendArango, sourceBackendPostgres, backend) + } +} + +func checkSourceBackendConflicts(cfg *Config, explicitFlags map[string]bool, usingConfigFile bool) error { + backend, err := normalizeSourceBackend(cfg.Server.Backend) + if err != nil { + return err + } + cfg.Server.Backend = backend + if usingConfigFile { + if explicitFlags["backend"] { + return fmt.Errorf("server.backend is owned by the --config file; remove the explicit --backend override") + } + var pgFlags []string + for _, name := range postgresCLIFlagNames { + if explicitFlags[name] { + pgFlags = append(pgFlags, "--"+name) + } + } + if len(pgFlags) > 0 { + return fmt.Errorf("server.postgresql settings are owned by the --config file; remove the explicit %s override", strings.Join(pgFlags, ", ")) + } + } + if backend != sourceBackendPostgres { + return nil + } + var conflicting []string + if explicitFlags["url"] { + conflicting = append(conflicting, "--url") + } + if explicitFlags["database"] { + conflicting = append(conflicting, "--database") + } + if len(conflicting) > 0 { + return fmt.Errorf("server.backend %s conflicts with explicitly set ArangoDB settings %s; remove them when selecting postgres", sourceBackendPostgres, strings.Join(conflicting, " and ")) + } + return nil +} + +func (p PostgreSQLConfig) validate() error { + if strings.TrimSpace(p.Host) == "" { + return errors.New("server.postgresql.host is required when server.backend is postgres") + } + if p.Port < 1 || p.Port > 65535 { + return fmt.Errorf("server.postgresql.port must be between 1 and 65535, got %d", p.Port) + } + if strings.TrimSpace(p.Database) == "" { + return errors.New("server.postgresql.database is required when server.backend is postgres") + } + if strings.TrimSpace(p.User) == "" { + return errors.New("server.postgresql.user is required when server.backend is postgres") + } + sslMode := strings.ToLower(strings.TrimSpace(p.SSLMode)) + known := false + for _, mode := range postgresSSLModes { + if sslMode == mode { + known = true + break + } + } + if !known { + return fmt.Errorf("server.postgresql.sslmode must be one of %s, got %q", strings.Join(postgresSSLModes, ", "), p.SSLMode) + } + if p.MaxOpenConns <= 0 { + return fmt.Errorf("server.postgresql.max_open_conns must be positive, got %d", p.MaxOpenConns) + } + if p.MaxIdleConns < 0 || p.MaxIdleConns > p.MaxOpenConns { + return fmt.Errorf("server.postgresql.max_idle_conns must be between 0 and server.postgresql.max_open_conns (%d), got %d", p.MaxOpenConns, p.MaxIdleConns) + } + if p.ConnMaxLifetime < 0 { + return errors.New("server.postgresql.conn_max_lifetime cannot be negative") + } + if len(p.Schemas) == 0 { + return nil + } + const ( + seriesSource = "source" + seriesExplorerControl = "explorer_control" + seriesPublicationControl = "publication_control" + seriesDatasetRelease = "dataset_release" + seriesRecipeChains = "recipe_chains" + seriesRecipeExecRegistries = "recipe_exec_registries" + ) + ladder := []string{seriesSource, seriesExplorerControl, seriesPublicationControl, seriesDatasetRelease, seriesRecipeChains, seriesRecipeExecRegistries} + _, hasSource := p.Schemas[seriesSource] + _, hasDatasetRelease := p.Schemas[seriesDatasetRelease] + firstQueryPair := len(p.Schemas) == 2 && hasSource && hasDatasetRelease + fullLadder := len(p.Schemas) == len(ladder) + for _, series := range ladder { + if _, assigned := p.Schemas[series]; !assigned && fullLadder { + fullLadder = false + break + } + } + for series := range p.Schemas { + known := false + for _, candidate := range ladder { + if series == candidate { + known = true + break + } + } + if !known { + return fmt.Errorf("server.postgresql.schemas: series %q is not consumed by any production factory; the consumable shapes are the exact first-query-route pair (source and dataset_release) or the complete six-series ladder", series) + } + } + if !firstQueryPair && !fullLadder { + return fmt.Errorf("server.postgresql.schemas: the consumable shapes are the exact first-query-route pair (source and dataset_release) or the complete six-series ladder (source, explorer_control, publication_control, dataset_release, recipe_chains, recipe_exec_registries); got %d entries", len(p.Schemas)) + } + for series, schema := range p.Schemas { + if strings.TrimSpace(schema) == "" { + return fmt.Errorf("server.postgresql.schemas: the %s entry cannot be blank", series) + } + } + trimmed := make(map[string]string, len(p.Schemas)) + for series, schema := range p.Schemas { + name := strings.TrimSpace(schema) + if other, clash := trimmed[name]; clash { + return fmt.Errorf("server.postgresql.schemas: %s and %s must name distinct schemas; their pinned migration ledgers disagree, and the shared-schema refusal stays authoritative at storepostgres.ValidateAssignment", series, other) + } + trimmed[name] = series + } + return nil +} + +const redactedConfigSecret = "[redacted]" + +func redactedConfigSecretFor(value string) string { + if strings.TrimSpace(value) == "" { + return "" + } + return redactedConfigSecret +} + +var configURLCredentials = regexp.MustCompile(`(?i)([a-z][a-z0-9+.-]*://)[^/\s:@]+:[^/\s@]+@`) + +func redactConfigURL(value string) string { + return configURLCredentials.ReplaceAllString(value, "${1}[redacted]@") +} + +func (c Config) String() string { + return fmt.Sprintf( + "server{listen=%q backend=%q url=%q database=%q schema=%q postgresql{host=%q port=%d database=%q user=%q password=%q sslmode=%q max_open_conns=%d max_idle_conns=%d conn_max_lifetime=%s} clickhouse{enabled=%t url=%q database=%q username=%q password=%q} artifact_directory=%q population_mapping_cursor_secret=%q} auth{mode=%q basic{username=%q password=%q}}", + c.Server.Listen, c.Server.Backend, redactConfigURL(c.Server.URL), c.Server.Database, c.Server.Schema, + c.Server.PostgreSQL.Host, c.Server.PostgreSQL.Port, c.Server.PostgreSQL.Database, c.Server.PostgreSQL.User, + redactedConfigSecretFor(c.Server.PostgreSQL.Password), c.Server.PostgreSQL.SSLMode, + c.Server.PostgreSQL.MaxOpenConns, c.Server.PostgreSQL.MaxIdleConns, c.Server.PostgreSQL.ConnMaxLifetime, + c.Server.ClickHouse.Enabled, redactConfigURL(c.Server.ClickHouse.URL), c.Server.ClickHouse.Database, + c.Server.ClickHouse.Username, redactedConfigSecretFor(c.Server.ClickHouse.Password), + c.Server.ArtifactDirectory, redactedConfigSecretFor(c.Server.PopulationMappingCursorSecret), + c.Auth.Mode, c.Auth.Basic.Username, redactedConfigSecretFor(c.Auth.Basic.Password), + ) +} + func (c Config) Validate() error { cfg := c if cfg.Server.RecipeQueryPageRows < 0 { @@ -308,6 +540,18 @@ func (c Config) Validate() error { if strings.TrimSpace(cfg.Server.ArtifactDirectory) == "" || cfg.Server.ArtifactTTL <= 0 || cfg.Server.ArtifactMaxRows <= 0 || cfg.Server.ArtifactMaxBytes <= 0 { return errors.New("server artifact directory, TTL, row limit, and byte limit must be configured") } + backend, err := normalizeSourceBackend(cfg.Server.Backend) + if err != nil { + return err + } + if backend == sourceBackendPostgres { + if strings.TrimSpace(cfg.Server.URL) != defaultServerURL || strings.TrimSpace(cfg.Server.Database) != defaultServerDatabase { + return errors.New("server.url and server.database are ArangoDB settings and cannot be customized while server.backend is postgres") + } + if err := cfg.Server.PostgreSQL.validate(); err != nil { + return err + } + } if strings.TrimSpace(cfg.Server.LocalWorkspaceWriteback) != "" && !(cfg.Server.AllowUnauthenticated || cfg.Auth.AllowUnauthenticated) { return errors.New("server.local_workspace_writeback requires unauthenticated local-development mode") } diff --git a/internal/server/options_schemas_test.go b/internal/server/options_schemas_test.go new file mode 100644 index 000000000..d63bd1491 --- /dev/null +++ b/internal/server/options_schemas_test.go @@ -0,0 +1,197 @@ +package server + +import ( + "context" + "strings" + "testing" + "time" +) + +func validPostgreSQLConfig() PostgreSQLConfig { + return PostgreSQLConfig{ + Host: "db.internal", + Port: 5432, + Database: "loom", + User: "loom", + Password: "ab05-Sup3r-Pg-Pa55w0rd-canary", + SSLMode: "require", + MaxOpenConns: 4, + MaxIdleConns: 1, + ConnMaxLifetime: 0, + } +} + +func TestPostgreSQLConfigSchemasConsumablePairAndFailClosed(t *testing.T) { + tests := []struct { + name string + schemas map[string]string + accept bool + wantSubstrings []string + }{ + { + name: "absent schemas stay optional (route gate then keeps the startup refusal)", + schemas: nil, + accept: true, + }, + { + name: "empty schemas map accepted", + schemas: map[string]string{}, + accept: true, + }, + { + name: "the consumable pair is accepted once the factory honors it", + schemas: map[string]string{"source": "loom_source", "dataset_release": "loom_release"}, + accept: true, + }, + { + name: "source alone rejected: neither consumable shape", + schemas: map[string]string{"source": "loom_source"}, + wantSubstrings: []string{"consumable shapes", "got 1 entries"}, + }, + { + name: "dataset_release alone rejected: neither consumable shape", + schemas: map[string]string{"dataset_release": "loom_release"}, + wantSubstrings: []string{"consumable shapes", "got 1 entries"}, + }, + { + name: "blank source rejected", + schemas: map[string]string{"source": " ", "dataset_release": "loom_release"}, + wantSubstrings: []string{"cannot be blank"}, + }, + { + name: "blank dataset_release rejected", + schemas: map[string]string{"source": "loom_source", "dataset_release": ""}, + wantSubstrings: []string{"cannot be blank"}, + }, + { + name: "source and dataset_release sharing one schema rejected", + schemas: map[string]string{"source": "shared", "dataset_release": "shared"}, + wantSubstrings: []string{"must name distinct schemas"}, + }, + { + name: "whitespace-only-differing pair gets the precise distinct-schema refusal", + schemas: map[string]string{"source": " shared ", "dataset_release": "shared"}, + wantSubstrings: []string{"must name distinct schemas"}, + }, + { + name: "lone explorer_control entry rejected: neither consumable shape", + schemas: map[string]string{"explorer_control": "loom_explorer"}, + wantSubstrings: []string{"explorer_control", "consumable shapes", "got 1 entries"}, + }, + { + name: "lone publication_control entry rejected: neither consumable shape", + schemas: map[string]string{"publication_control": "loom_publication"}, + wantSubstrings: []string{"publication_control", "consumable shapes", "got 1 entries"}, + }, + { + name: "lone recipe_chains entry rejected: neither consumable shape", + schemas: map[string]string{"recipe_chains": "loom_recipe"}, + wantSubstrings: []string{"recipe_chains", "consumable shapes", "got 1 entries"}, + }, + { + name: "lone recipe_exec_registries entry rejected: neither consumable shape", + schemas: map[string]string{"recipe_exec_registries": "loom_exec"}, + wantSubstrings: []string{"recipe_exec_registries", "consumable shapes", "got 1 entries"}, + }, + { + name: "a series beside the consumable pair that does not complete the ladder fails closed", + schemas: map[string]string{ + "source": "loom_source", "dataset_release": "loom_release", + "explorer_control": "loom_explorer", + }, + wantSubstrings: []string{"explorer_control", "consumable shapes", "got 3 entries"}, + }, + { + name: "the complete distinct six-series ladder is accepted (full runtime)", + schemas: map[string]string{ + "source": "loom_source", "explorer_control": "loom_explorer", + "publication_control": "loom_publication", "dataset_release": "loom_release", + "recipe_chains": "loom_recipe", "recipe_exec_registries": "loom_exec", + }, + accept: true, + }, + { + name: "the six-series ladder with two series sharing one schema fails closed", + schemas: map[string]string{ + "source": "loom_source", "explorer_control": "loom_explorer", + "publication_control": "loom_publication", "dataset_release": "loom_release", + "recipe_chains": "loom_exec", "recipe_exec_registries": "loom_exec", + }, + wantSubstrings: []string{"must name distinct schemas"}, + }, + { + name: "unknown series rejected", + schemas: map[string]string{ + "warehouse_ledger": "loom_anything", + }, + wantSubstrings: []string{"warehouse_ledger", "not consumed by any production factory"}, + }, + { + name: "six entries with a typo'd ladder member rejected as foreign series", + schemas: map[string]string{ + "source": "loom_source", "explorer_control": "loom_explorer", + "publication_control": "loom_publication", "dataset_release": "loom_release", + "recipe_chainz": "loom_recipe", "recipe_exec_registries": "loom_exec", + }, + wantSubstrings: []string{"recipe_chainz", "not consumed by any production factory"}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := validPostgreSQLConfig() + cfg.Schemas = tc.schemas + err := cfg.validate() + if tc.accept { + if err != nil { + t.Fatalf("validate with schemas %v = %v, want acceptance", tc.schemas, err) + } + return + } + if err == nil { + t.Fatalf("validate accepted schemas %v; an unconsumed or malformed assignment must fail closed, not pass silently", tc.schemas) + } + msg := err.Error() + if !strings.Contains(msg, "server.postgresql.schemas") { + t.Fatalf("error %q must name the server.postgresql.schemas field", msg) + } + for _, want := range tc.wantSubstrings { + if !strings.Contains(msg, want) { + t.Fatalf("error %q must contain %q", msg, want) + } + } + if cfg.Password == "" { + t.Fatal("fixture must carry a non-empty Password so the no-secret check is exercised") + } + for _, secret := range []string{cfg.Password, cfg.Host, cfg.User, cfg.Database} { + if strings.Contains(msg, secret) { + t.Fatalf("error %q must not echo config value %q", msg, secret) + } + } + }) + } +} + +func TestPostgreSQLConfigSchemasUnsafeIdentifierFailsPreDial(t *testing.T) { + cfg := Config{} + cfg.Server.Backend = sourceBackendPostgres + cfg.Server.PostgreSQL = validPostgreSQLConfig() + cfg.Server.PostgreSQL.Host = "192.0.2.1" + cfg.Server.PostgreSQL.Port = 60543 + cfg.Server.PostgreSQL.Schemas = map[string]string{"source": "Loom Source", "dataset_release": "loom_release"} + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + err := RunWithConfig(ctx, cfg) + if err == nil { + t.Fatal("unsafe schema identifier unexpectedly started the server") + } + if strings.Contains(err.Error(), cfg.Server.PostgreSQL.Password) { + t.Fatalf("refusal leaked the connection password: %v", err) + } + if err == ErrPostgreSQLRuntimeNotWired { + t.Fatalf("unsafe identifier was refused by the consumability sentinel instead of the factory's pre-dial identifier law: %v", err) + } + if !strings.Contains(err.Error(), "Loom Source") && !strings.Contains(err.Error(), "schema") { + t.Fatalf("error %q must name the unsafe schema identifier or the schema law", err) + } +} diff --git a/internal/server/patient_one_construction_choice_route_test.go b/internal/server/patient_one_construction_choice_route_test.go index a38656343..c20d1e9e4 100644 --- a/internal/server/patient_one_construction_choice_route_test.go +++ b/internal/server/patient_one_construction_choice_route_test.go @@ -13,6 +13,7 @@ import ( loomapi "github.com/calypr/loom/generated/loomapi" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" @@ -92,7 +93,6 @@ func TestPatientOneChoiceProposalRouteRefusesOneAndRepairsWithAll(t *testing.T) nil, ) - // Use the same production catalog projection that emits signed choice IDs. catalog := authoringV2Catalog(snapshot, explorerID) var choiceID string for _, candidate := range catalog.Candidates { @@ -141,11 +141,14 @@ func TestPatientOneChoiceProposalRouteRefusesOneAndRepairsWithAll(t *testing.T) candidatePublicColumn := "" previewQueryCalls := 0 recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, ScopeDigest: recipeScopeDigest, - // QueryRows stands in for Arango at the database boundary; the renderer test - // checks the generated assertion, while this route test supplies the two - // literal IDs from the retained raw fixture as the query result oracle. - QueryRows: func(_ context.Context, query string, _ int, _ map[string]any, visit func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, ScopeDigest: recipeScopeDigest, + SourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + query, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + previewQueryCalls++ if !strings.Contains(query, "root.payload.id") { return fmt.Errorf("Patient.id chooser query did not read the raw source id: %s", query) diff --git a/internal/server/population_member_receipt_test.go b/internal/server/population_member_receipt_test.go index f96e3edc6..0835ce317 100644 --- a/internal/server/population_member_receipt_test.go +++ b/internal/server/population_member_receipt_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" @@ -125,8 +126,16 @@ func TestCompileExplorerReceiptBindsPopulationRemovalWithGroupedRelatedSummary(t } readScope := scope engine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }, + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }, }) if err != nil { t.Fatal(err) diff --git a/internal/server/postgres_query_test.go b/internal/server/postgres_query_test.go new file mode 100644 index 000000000..d71e86157 --- /dev/null +++ b/internal/server/postgres_query_test.go @@ -0,0 +1,1018 @@ +package server + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "mime/multipart" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" + + backendspostgres "github.com/calypr/loom/internal/backends/postgres" + "github.com/calypr/loom/internal/backends/postgres/control" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + publication "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/ingest" + "github.com/gofiber/fiber/v3" + "github.com/jackc/pgx/v5/pgxpool" +) + +func qRoutePostgreSQLConfig() PostgreSQLConfig { + pg := validPostgreSQLConfig() + pg.Host = "192.0.2.1" + pg.Port = 60543 + pg.Database = "verify_qroute_predial_witness" + pg.User = "verify_qroute_predial_user" + pg.Password = "qroute-Pg-Pa55w0rd-canary" + return pg +} + +func TestPostgreSQLFirstRoutePreDialRefusals(t *testing.T) { + cases := []struct { + name string + schemas map[string]string + }{ + { + name: "absent schemas keep the arch-backend-06 refusal", + schemas: nil, + }, + { + name: "empty schemas map keeps the arch-backend-06 refusal", + schemas: map[string]string{}, + }, + { + name: "source alone is not a consumable assignment", + schemas: map[string]string{"source": "verify_qroute_source"}, + }, + { + name: "dataset_release alone is not a consumable assignment", + schemas: map[string]string{"dataset_release": "verify_qroute_release"}, + }, + { + name: "an unconsumed series beside the pair stays refused", + schemas: map[string]string{ + "source": "verify_qroute_source", "dataset_release": "verify_qroute_release", + "explorer_control": "verify_qroute_explorer", + }, + }, + { + name: "blank source is refused at the gate, not dialed with", + schemas: map[string]string{ + "source": " ", "dataset_release": "verify_qroute_release", + }, + }, + { + name: "the two consumable series sharing one schema are refused", + schemas: map[string]string{ + "source": "verify_qroute_shared", "dataset_release": "verify_qroute_shared", + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg := Config{} + cfg.Server.Backend = sourceBackendPostgres + cfg.Server.PostgreSQL = qRoutePostgreSQLConfig() + cfg.Server.PostgreSQL.Schemas = tc.schemas + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + started := time.Now() + err := RunWithConfig(ctx, cfg) + elapsed := time.Since(started) + + if !errors.Is(err, ErrPostgreSQLRuntimeNotWired) { + t.Fatalf("RunWithConfig(postgres, schemas=%v) err = %v, want errors.Is(err, ErrPostgreSQLRuntimeNotWired)", tc.schemas, err) + } + if got := err.Error(); got != postgreSQLRefusalMessage { + t.Fatalf("refusal message drifted: got %q, want byte-stable %q", got, postgreSQLRefusalMessage) + } + if elapsed > 1500*time.Millisecond { + t.Fatalf("refusal took %v: the consumability gate must refuse before any connection attempt", elapsed) + } + pg := cfg.Server.PostgreSQL + for _, secret := range []string{pg.Host, pg.User, pg.Password, pg.Database} { + if strings.Contains(err.Error(), secret) { + t.Fatalf("refusal error leaked connection material %q: %v", secret, err) + } + } + }) + } +} + +func TestPostgreSQLFirstQueryRouteScrubberPinsCredentials(t *testing.T) { + pg := qRoutePostgreSQLConfig() + dsn := postgresFirstQueryRouteDSN(pg) + if !strings.Contains(dsn, pg.Password) { + t.Fatalf("fixture DSN must actually carry the password for the scrub to be exercised: %q", dsn) + } + cause := errors.New("dial failed") + for _, raw := range []string{ + "open PostgreSQL source runtime: " + dsn, + "auth failed for user " + pg.User + ":" + pg.Password + " (sqlstate 28P01)", + "server rejected password " + pg.Password, + } { + got := postgresFirstQueryRouteError(fmt.Errorf("%s: %w", raw, cause), pg, dsn) + if got == nil { + t.Fatalf("scrubber dropped a non-nil error %q", raw) + } + for _, secret := range []string{dsn, pg.Password, pg.User + ":" + pg.Password} { + if strings.Contains(got.Error(), secret) { + t.Fatalf("scrubbed error %q still contains credential %q", got.Error(), secret) + } + } + if !strings.Contains(got.Error(), "") { + t.Fatalf("scrubbed error %q lost the redaction marker", got.Error()) + } + if !strings.Contains(got.Error(), "dial failed") { + t.Fatalf("scrubbed error %q lost the non-secret cause text", got.Error()) + } + var wrapped *redactedRouteError + if !errors.As(got, &wrapped) || wrapped.Unwrap() == nil { + t.Fatalf("scrubbed error %v must expose the original cause through Unwrap", got) + } + if !errors.Is(got, cause) { + t.Fatalf("scrubbed error %v lost the original cause identity", got) + } + } + + esc := qRoutePostgreSQLConfig() + esc.Password = "qroute p@ss:w0rd/s?x=&%" + escDSN := postgresFirstQueryRouteDSN(esc) + queryEscaped, pathEscaped := url.QueryEscape(esc.Password), url.PathEscape(esc.Password) + if queryEscaped == esc.Password || pathEscaped == esc.Password { + t.Fatalf("variant password must require escaping in both encodings: query=%q path=%q", queryEscaped, pathEscaped) + } + if !strings.Contains(escDSN, esc.User) { + t.Fatalf("variant DSN lost the user: %q", escDSN) + } + for _, raw := range []string{ + "auth failed for user " + esc.User + ":" + queryEscaped + " (sqlstate 28P01)", + "server rejected password " + pathEscaped, + "driver echoed parsed credential " + esc.User + ":" + pathEscaped + " while dialing " + escDSN, + } { + got := postgresFirstQueryRouteError(fmt.Errorf("%s: %w", raw, cause), esc, escDSN) + if got == nil { + t.Fatalf("scrubber dropped a non-nil error %q", raw) + } + for _, secret := range []string{escDSN, esc.Password, queryEscaped, pathEscaped} { + if secret != "" && strings.Contains(got.Error(), secret) { + t.Fatalf("scrubbed error %q still contains credential %q", got.Error(), secret) + } + } + if !strings.Contains(got.Error(), "") || !strings.Contains(got.Error(), "dial failed") { + t.Fatalf("scrubbed error %q lost the marker or the cause text", got.Error()) + } + if !errors.Is(got, cause) { + t.Fatalf("scrubbed error %v lost the original cause identity", got) + } + } + + plain := errors.New("schema loom_source does not exist") + if got := postgresFirstQueryRouteError(plain, pg, dsn); got != plain { + t.Fatalf("secret-free error was rewritten: %v", got) + } + if postgresFirstQueryRouteError(nil, pg, dsn) != nil { + t.Fatal("scrubber must map nil to nil") + } +} + +func TestPostgreSQLFirstQueryRouteWiringErrorShape(t *testing.T) { + err := postgreSQLFirstQueryRouteWiringError("Server-side generation activation", "pg-runtime-server-composition") + var typed *dataframeerrors.Error + if !errors.As(err, &typed) { + t.Fatalf("refusal %v is not a typed dataframe error", err) + } + if typed.Code() != string(dataframeerrors.CodeBackendUnavailable) { + t.Fatalf("refusal code = %q, want %q", typed.Code(), dataframeerrors.CodeBackendUnavailable) + } + if typed.Retryable() { + t.Fatal("a wiring refusal must not advertise retry: the route will not serve it on retry either") + } + details := typed.Details() + if details["backend"] != "postgresql" || details["stage"] != "wiring" { + t.Fatalf("refusal details %v must tag backend=postgresql stage=wiring", details) + } + if details["enablingTicket"] != "pg-runtime-server-composition" { + t.Fatalf("refusal details %v must name the enabling ticket", details) + } + if !strings.Contains(typed.Error(), "Server-side generation activation") || + !strings.Contains(typed.Error(), "pg-runtime-server-composition") { + t.Fatalf("refusal text %q must name both the operation and the enabling ticket", typed.Error()) + } +} + +const ( + qRouteDBFmt = "verify_qroute_%d_%s" + qRouteSrcFmt = "verify_qroute_%d_%s_src" + qRouteRelFmt = "verify_qroute_%d_%s_rel" + qRouteProject = "qroute-p1" + qRouteForeign = "qroute-p2" + qRouteGenA = "generation-a" + qRouteGenB = "generation-b" + qRouteAppTag = "verify-qroute-server" +) + +var ( + qRouteDBPattern = regexp.MustCompile(`^verify_qroute_[0-9]+_[0-9a-f]+$`) + qRouteSrcPattern = regexp.MustCompile(`^verify_qroute_[0-9]+_[0-9a-f]+_src$`) + qRouteRelPattern = regexp.MustCompile(`^verify_qroute_[0-9]+_[0-9a-f]+_rel$`) + qRouteCensusSibling = regexp.MustCompile(`^verify_qroute_q[0-9]+_[0-9a-f]+$`) + qRouteOwnedMu sync.Mutex + qRouteOwned = map[string]qRouteMint{} + qRouteStrayMarks = []string{} + qRouteBaselineMu sync.Mutex + qRouteBaseline int64 + qRouteBaselineSet bool + qRouteBaselineErr error +) + +type qRouteMint struct { + leg string + dropped bool +} + +var qRouteMintInventory = map[string]int{ + "TestPostgreSQLQueryRouteExplorerAndRowOpsExplicitlyUnavailable": 1, + "TestPostgreSQLQueryRouteServesExactScalarsThroughProductionHandler": 1, + "TestPostgreSQLFirstQueryRouteLivenessAndReadiness": 1, + "TestBackendNeutralDiagnosticsLivePostgreSQL": 1, +} + +func qRouteTestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL first query route integration tests") + } + return raw +} + +func qRouteRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) +} + +func qRouteRegister(db, leg string) { + qRouteOwnedMu.Lock() + defer qRouteOwnedMu.Unlock() + qRouteOwned[db] = qRouteMint{leg: leg} +} + +func qRouteMarkDropped(db string) { + qRouteOwnedMu.Lock() + defer qRouteOwnedMu.Unlock() + rec, ok := qRouteOwned[db] + if !ok { + qRouteStrayMarks = append(qRouteStrayMarks, db) + return + } + rec.dropped = true + qRouteOwned[db] = rec +} + +func qRouteCaptureBaseline(t *testing.T, raw string) { + t.Helper() + qRouteBaselineMu.Lock() + defer qRouteBaselineMu.Unlock() + if qRouteBaselineSet { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + qRouteBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname ~ '^verify_qroute_[0-9]+_[0-9a-f]+$'"). + Scan(&count); err != nil { + qRouteBaselineErr = fmt.Errorf("count baseline: %w", err) + return + } + qRouteBaseline = count + qRouteBaselineSet = true +} + +type qRouteFixture struct { + dsn string + srcSchema string + relSchema string + scrub func(any) string + secrets []string + admin *pgxpool.Pool + database string +} + +func newQRouteFixture(t *testing.T) *qRouteFixture { + t.Helper() + raw := qRouteTestPostgresURL(t) + qRouteCaptureBaseline(t, raw) + if qRouteBaselineErr != nil { + t.Fatalf("capture pg_database baseline: %v", qRouteBaselineErr) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + + ctx, cancel := context.WithTimeout(context.Background(), 600*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %s", scrub(err)) + } + t.Cleanup(admin.Close) + + database := fmt.Sprintf(qRouteDBFmt, time.Now().UnixNano(), qRouteRandomHex(t)) + if !qRouteDBPattern.MatchString(database) { + t.Fatalf("minted database %q violates the owned-name pattern", database) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+database+`"`); err != nil { + t.Fatalf("create isolated database: %s", scrub(err)) + } + qRouteRegister(database, t.Name()) + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(context.Background()), + `DROP DATABASE IF EXISTS "`+database+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + return + } + qRouteMarkDropped(database) + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + } + u.Path = "/" + database + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect to isolated database: %s", scrub(err)) + } + defer iso.Close() + srcSchema := fmt.Sprintf(qRouteSrcFmt, time.Now().UnixNano(), qRouteRandomHex(t)) + relSchema := fmt.Sprintf(qRouteRelFmt, time.Now().UnixNano(), qRouteRandomHex(t)) + if !qRouteSrcPattern.MatchString(srcSchema) || !qRouteRelPattern.MatchString(relSchema) { + t.Fatalf("minted schemas %q/%q violate the owned-name patterns", srcSchema, relSchema) + } + for _, schema := range []string{srcSchema, relSchema} { + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + t.Fatalf("create owned schema: %s", scrub(err)) + } + } + return &qRouteFixture{dsn: dsn, srcSchema: srcSchema, relSchema: relSchema, scrub: scrub, secrets: secrets, admin: admin, database: database} +} + +func (f *qRouteFixture) migrate(t *testing.T) *backendspostgres.Runtime { + t.Helper() + runtime, err := backendspostgres.Open(context.Background(), control.Config{ + DSN: f.dsn, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 30 * time.Second, + Schemas: control.SeriesSchemas{ + control.SeriesSource: f.srcSchema, + control.SeriesDatasetRelease: f.relSchema, + }, + Setup: control.SetupMigrate, + }) + if err != nil { + t.Fatalf("migrate fixture series: %s", f.scrub(err)) + } + if runtime.FieldProfiles == nil || runtime.Inventory == nil || runtime.Relationships == nil || runtime.Contributions == nil { + _ = runtime.Close() + t.Fatal("fixture runtime assembled without catalog evidence ports") + } + return runtime +} + +func (f *qRouteFixture) pgOptions(application string) ingest.PGSourceOptions { + return ingest.PGSourceOptions{ + DSN: f.dsn, + Schema: f.srcSchema, + ManifestSchema: f.relSchema, + ApplicationName: application, + } +} + +func (f *qRouteFixture) load(t *testing.T, project, generation string, files map[string][]string) { + t.Helper() + dir := t.TempDir() + for name, lines := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil { + t.Fatalf("write staging fixture: %v", err) + } + } + catalog := f.migrate(t) + defer func() { _ = catalog.Close() }() + ref, err := publication.NewRef(project, generation) + if err != nil { + t.Fatalf("build dataset ref: %v", err) + } + opts := ingest.LoadOptions{ + Schema: qRouteRepoPath(t, "schemas", "graph-fhir.json"), + MetaDir: dir, + Project: project, + Dataset: &ref, + AuthResourcePath: "/fhir", + StageOnly: true, + } + pg := f.pgOptions(qRouteAppTag) + pg.Catalog = ingest.GenerationCatalogWriters{ + FieldProfiles: catalog.FieldProfiles, + Inventory: catalog.Inventory, + Relationships: catalog.Relationships, + Contributions: catalog.Contributions, + } + if _, err := ingest.LoadPostgresGeneration(context.Background(), opts, pg); err != nil { + t.Fatalf("load %s/%s: %s", project, generation, f.scrub(err)) + } +} + +func (f *qRouteFixture) activate(t *testing.T, project, generation string) { + t.Helper() + ref, err := publication.NewRef(project, generation) + if err != nil { + t.Fatalf("build dataset ref: %v", err) + } + if err := ingest.ActivatePostgresGeneration(context.Background(), ref, f.pgOptions(qRouteAppTag+"-activate")); err != nil { + t.Fatalf("activate %s/%s: %s", project, generation, f.scrub(err)) + } +} + +func qRouteRepoPath(t *testing.T, elems ...string) string { + t.Helper() + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("locate test file for repository root") + } + dir := filepath.Dir(thisFile) + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return filepath.Join(append([]string{dir}, elems...)...) + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("no go.mod found above the test file") + } + dir = parent + } +} + +func qRouteBootConfig(t *testing.T, f *qRouteFixture, listen string) Config { + t.Helper() + u, err := url.Parse(f.dsn) + if err != nil { + t.Fatalf("parse fixture dsn: %v", err) + } + host, portText, err := net.SplitHostPort(u.Host) + if err != nil { + t.Fatalf("split fixture dsn host: %v", err) + } + port, err := strconv.Atoi(portText) + if err != nil { + t.Fatalf("fixture dsn port %q: %v", portText, err) + } + password := "" + if pw, ok := u.User.Password(); ok { + password = pw + } + sslMode := u.Query().Get("sslmode") + if sslMode == "" { + sslMode = "disable" + } + cfg := Config{} + cfg.Server.Backend = sourceBackendPostgres + cfg.Server.Listen = listen + cfg.Server.AllowUnauthenticated = true + cfg.Server.PostgreSQL = PostgreSQLConfig{ + Host: host, Port: port, Database: strings.TrimPrefix(u.Path, "/"), + User: u.User.Username(), Password: password, SSLMode: sslMode, + MaxOpenConns: 4, MaxIdleConns: 1, + Schemas: map[string]string{"source": f.srcSchema, "dataset_release": f.relSchema}, + } + return cfg +} + +func qRouteGraphQL(t *testing.T, app *fiber.App, body string) testHTTPResponse { + t.Helper() + return requestJSON(t, app, http.MethodPost, "/graphql/graph", body) +} + +func qRoutePatientLines(t *testing.T, ids ...string) []string { + t.Helper() + lines := make([]string, 0, len(ids)) + for _, id := range ids { + encoded, err := json.Marshal(map[string]any{ + "resourceType": "Patient", "id": id, + "gender": "female", "birthDate": "1900-01-01", + }) + if err != nil { + t.Fatalf("marshal patient fixture: %v", err) + } + lines = append(lines, string(encoded)) + } + return lines +} + +func TestPostgreSQLQueryRouteExplorerAndRowOpsExplicitlyUnavailable(t *testing.T) { + fixture := newQRouteFixture(t) + fixture.load(t, qRouteProject, qRouteGenA, map[string][]string{ + "Patient.ndjson": qRoutePatientLines(t, "patient-ada", "patient-alan"), + "Observation.ndjson": {`{"resourceType":"Observation","id":"obs-1","status":"final","code":{"coding":[{"system":"http://loinc.org","code":"8867-4","display":"Heart rate"}]},"subject":{"reference":"Patient/patient-ada"}}`}, + }) + fixture.load(t, qRouteProject, qRouteGenB, map[string][]string{"Patient.ndjson": qRoutePatientLines(t, "patient-zoe")}) + fixture.load(t, qRouteForeign, qRouteGenA, map[string][]string{"Patient.ndjson": qRoutePatientLines(t, "patient-foreign")}) + fixture.activate(t, qRouteProject, qRouteGenA) + fixture.activate(t, qRouteForeign, qRouteGenA) + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + server, closeRuntime, err := composePostgreSQLFirstQueryRoute(ctx, qRouteBootConfig(t, fixture, "127.0.0.1:0"), logger) + if err != nil { + t.Fatalf("compose production route: %s", fixture.scrub(err)) + } + defer func() { + if err := closeRuntime(); err != nil { + t.Errorf("close production route runtime: %s", fixture.scrub(err)) + } + }() + app := server.App() + + const base = "/api/v1" + + t.Run("generation_status_is_a_live_postgres_read", func(t *testing.T) { + response := requestJSON(t, app, http.MethodGet, base+"/datasets/"+qRouteProject+"/generations/"+qRouteGenA, "") + if response.StatusCode != http.StatusOK || !strings.Contains(response.Body, qRouteGenA) { + t.Fatalf("active-generation status = %d %s, want 200 naming %s", response.StatusCode, response.Body, qRouteGenA) + } + }) + + t.Run("load_route_refuses_with_the_cli_deferral", func(t *testing.T) { + body, boundary := qRouteMultipart(t, map[string]string{"Patient.ndjson": "{\"resourceType\":\"Patient\",\"id\":\"route-should-never-load\"}\n"}) + request := httptest.NewRequest(http.MethodPost, base+"/datasets/"+qRouteProject+"/generations/generation-route-attempt", bytes.NewReader(body)) + request.Header.Set("Content-Type", "multipart/form-data; boundary="+boundary) + response, err := app.Test(request, fiber.TestConfig{Timeout: 15 * time.Second, FailOnTimeout: true}) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + raw, _ := io.ReadAll(response.Body) + text := string(raw) + t.Logf("QR-D2 verbatim load-route refusal: status=%d body=%s", response.StatusCode, fixture.scrub(text)) + if response.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("load-route status = %d body=%s, want 503 typed refusal", response.StatusCode, fixture.scrub(text)) + } + for _, want := range []string{"BACKEND_UNAVAILABLE", "loom-data CLI", "pg-runtime-server-composition"} { + if !strings.Contains(text, want) { + t.Fatalf("load-route refusal %q must contain %q", fixture.scrub(text), want) + } + } + }) + + t.Run("activation_route_refuses", func(t *testing.T) { + response := requestJSON(t, app, http.MethodPost, + base+"/datasets/"+qRouteProject+"/generations/"+qRouteGenB+"/activate?dataframe_execution_id=exec-census-1", `{}`) + t.Logf("activation refusal: status=%d body=%s", response.StatusCode, fixture.scrub(response.Body)) + if response.StatusCode != http.StatusServiceUnavailable || !strings.Contains(response.Body, "BACKEND_UNAVAILABLE") || + !strings.Contains(response.Body, "pg-runtime-server-composition") { + t.Fatalf("activation refusal = %d %s, want 503 typed refusal naming the enabling ticket", response.StatusCode, fixture.scrub(response.Body)) + } + }) + + authoringRefusals := []struct { + name, method, path, body string + wantCode string + }{ + {"explorer_authoring_suggestions", http.MethodPost, base + "/projects/" + qRouteProject + "/explorers/ex-1/authoring/v2/suggestions", `{"query":"patient"}`, "AUTHORING_UNAVAILABLE"}, + {"explorer_preview", http.MethodPost, base + "/projects/" + qRouteProject + "/explorers/ex-1/authoring/v2/preview", `{}`, "AUTHORING_UNAVAILABLE"}, + {"explorer_cell_trace", http.MethodPost, base + "/projects/" + qRouteProject + "/explorers/ex-1/authoring/v2/cell-trace", `{}`, "TRACE_UNAVAILABLE"}, + {"explorer_row_lineage", http.MethodPost, base + "/projects/" + qRouteProject + "/explorers/ex-1/authoring/v2/row-lineage", `{}`, "ROW_LINEAGE_UNAVAILABLE"}, + } + for _, leg := range authoringRefusals { + leg := leg + t.Run(leg.name, func(t *testing.T) { + response := requestJSON(t, app, leg.method, leg.path, leg.body) + t.Logf("%s verbatim refusal: status=%d body=%s", leg.name, response.StatusCode, fixture.scrub(response.Body)) + if response.StatusCode >= 200 && response.StatusCode < 300 { + t.Fatalf("%s unexpectedly served data: %d %s", leg.name, response.StatusCode, fixture.scrub(response.Body)) + } + if response.StatusCode >= 500 && response.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("%s returned server fault %d: %s", leg.name, response.StatusCode, fixture.scrub(response.Body)) + } + if !strings.Contains(response.Body, leg.wantCode) { + t.Fatalf("%s refusal %q must carry %q", leg.name, fixture.scrub(response.Body), leg.wantCode) + } + if response.StatusCode == http.StatusServiceUnavailable && strings.Contains(response.Body, "INTERNAL_ERROR") { + t.Fatalf("%s 503 lost its typed code: %s", leg.name, fixture.scrub(response.Body)) + } + if lower := strings.ToLower(response.Body); strings.Contains(lower, "arango") || strings.Contains(lower, "aql") { + t.Fatalf("%s refusal leaks an Arango surface: %s", leg.name, fixture.scrub(response.Body)) + } + }) + } + + graphqlLegs := []struct { + name string + query string + wantCode string + wantRetryable bool + }{ + {"fhirGraph_traversal", `{"query":"query { fhirGraph(input: {project: \"` + qRouteProject + `\", rootResourceType: \"Patient\"}) { returnedCount } }"}`, "INVALID_REQUEST", false}, + {"recipe_validate", `{"query":"mutation { validateDataframeRecipe(input: {name: \"r\", bindings: {project: \"` + qRouteProject + `\"}}) { name } }"}`, "BACKEND_UNAVAILABLE", false}, + {"recipe_run", `{"query":"mutation { runDataframeRecipe(input: {name: \"r\", bindings: {project: \"` + qRouteProject + `\"}}) { name } }"}`, "BACKEND_UNAVAILABLE", false}, + {"recipe_preview", `{"query":"mutation { previewDataframeRecipe(input: {name: \"r\", bindings: {project: \"` + qRouteProject + `\"}}) { name } }"}`, "BACKEND_UNAVAILABLE", false}, + {"recipe_explain", `{"query":"query { explainDataframeRecipe(input: {name: \"r\", bindings: {project: \"` + qRouteProject + `\"}}) { name } }"}`, "BACKEND_UNAVAILABLE", false}, + } + for _, leg := range graphqlLegs { + leg := leg + t.Run(leg.name, func(t *testing.T) { + response := qRouteGraphQL(t, app, leg.query) + t.Logf("%s verbatim refusal: status=%d body=%s", leg.name, response.StatusCode, fixture.scrub(response.Body)) + if !strings.Contains(response.Body, leg.wantCode) { + t.Fatalf("%s refusal %q must carry %s", leg.name, fixture.scrub(response.Body), leg.wantCode) + } + wantRetryable := `"retryable":true` + if !leg.wantRetryable { + wantRetryable = `"retryable":false` + } + if !strings.Contains(response.Body, wantRetryable) { + t.Fatalf("%s refusal %q must classify retryable=%v", leg.name, fixture.scrub(response.Body), leg.wantRetryable) + } + if lower := strings.ToLower(response.Body); strings.Contains(lower, "arango") || strings.Contains(lower, "aql") { + t.Fatalf("%s refusal leaks an Arango surface: %s", leg.name, fixture.scrub(response.Body)) + } + }) + } + + t.Run("fhirDataframe_serves_via_read_lane", func(t *testing.T) { + response := qRouteGraphQL(t, app, `{"query":"mutation { runFhirDataframe(input: {project: \"`+qRouteProject+`\", rootResourceType: \"Patient\", rootFields: [{name: \"gender\", selector: {valuePath: \"gender\"}}]}) { rows rowCount } }"}`) + t.Logf("fhirDataframe serving witness: status=%d body=%s", response.StatusCode, fixture.scrub(response.Body)) + if response.StatusCode != http.StatusOK || !strings.Contains(response.Body, "\"rowCount\":2") || + strings.Contains(response.Body, "\"errors\"") { + t.Fatalf("fhirDataframe must serve through the read lane, got %d %s", response.StatusCode, fixture.scrub(response.Body)) + } + if lower := strings.ToLower(response.Body); strings.Contains(lower, "arango") || strings.Contains(lower, "aql") { + t.Fatalf("served answer leaks an Arango surface: %s", fixture.scrub(response.Body)) + } + }) +} + +func qRouteMultipart(t *testing.T, files map[string]string) ([]byte, string) { + t.Helper() + var buffer bytes.Buffer + writer := multipart.NewWriter(&buffer) + for name, body := range files { + part, err := writer.CreateFormFile("file", name) + if err != nil { + t.Fatalf("create multipart file: %v", err) + } + if _, err := part.Write([]byte(body)); err != nil { + t.Fatalf("write multipart file: %v", err) + } + } + if err := writer.Close(); err != nil { + t.Fatalf("close multipart: %v", err) + } + return buffer.Bytes(), writer.Boundary() +} + +func TestPostgreSQLQueryRouteServesExactScalarsThroughProductionHandler(t *testing.T) { + fixture := newQRouteFixture(t) + fixture.load(t, qRouteProject, qRouteGenA, map[string][]string{ + "Patient.ndjson": qRoutePatientLines(t, "patient-ada", "patient-alan"), + }) + fixture.activate(t, qRouteProject, qRouteGenA) + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + server, closeRuntime, err := composePostgreSQLFirstQueryRoute(ctx, qRouteBootConfig(t, fixture, "127.0.0.1:0"), logger) + if err != nil { + t.Fatalf("compose production route: %s", fixture.scrub(err)) + } + defer func() { + if err := closeRuntime(); err != nil { + t.Errorf("close production route runtime: %s", fixture.scrub(err)) + } + }() + app := server.App() + + body := `{"query":"mutation($input: FhirDataframeInput!, $limit: Int) { runFhirDataframe(input: $input, limit: $limit) { columns rowCount rows } }","variables":{"input":{"project":"` + qRouteProject + `","rootResourceType":"Patient","rootFields":[{"name":"gender","selector":{"valuePath":"gender"}},{"name":"birthDate","selector":{"valuePath":"birthDate"}}]},"limit":10}}` + response := qRouteGraphQL(t, app, body) + if response.StatusCode != http.StatusOK { + t.Fatalf("runFhirDataframe HTTP %d: %s", response.StatusCode, fixture.scrub(response.Body)) + } + var envelope struct { + Errors []struct { + Message string `json:"message"` + } `json:"errors"` + Data struct { + RunFhirDataframe struct { + Columns []string `json:"columns"` + RowCount int `json:"rowCount"` + Rows []map[string]any `json:"rows"` + } `json:"runFhirDataframe"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(response.Body), &envelope); err != nil { + t.Fatalf("decode handler envelope: %v (body %s)", err, fixture.scrub(response.Body)) + } + if len(envelope.Errors) != 0 { + t.Fatalf("handler returned GraphQL errors: %s", fixture.scrub(response.Body)) + } + result := envelope.Data.RunFhirDataframe + if result.RowCount != 2 || len(result.Rows) != 2 { + t.Fatalf("handler rows = %d/%d, want 2: %s", result.RowCount, len(result.Rows), fixture.scrub(response.Body)) + } + gotColumns := append([]string(nil), result.Columns...) + sort.Strings(gotColumns) + if want := []string{"_key", "birthDate", "gender"}; !reflect.DeepEqual(gotColumns, want) { + t.Fatalf("handler columns = %v, want %v", result.Columns, want) + } + sort.Slice(result.Rows, func(i, j int) bool { + return fmt.Sprint(result.Rows[i]["_key"]) < fmt.Sprint(result.Rows[j]["_key"]) + }) + for i, want := range []map[string]string{ + {"_key": "patient-ada", "birthDate": "1900-01-01", "gender": "female"}, + {"_key": "patient-alan", "birthDate": "1900-01-01", "gender": "female"}, + } { + for column, wantValue := range want { + got, present := result.Rows[i][column] + if !present { + t.Fatalf("row %d lost column %q: %#v", i, column, result.Rows[i]) + } + if gotText, ok := got.(string); !ok || gotText != wantValue { + t.Fatalf("row %d column %q = %#v, want %q", i, column, got, wantValue) + } + } + } + for _, aqlTell := range []string{"FOR d IN ", "@auth_resource_paths", "fhir_field_catalog"} { + if strings.Contains(response.Body, aqlTell) { + t.Fatalf("response body carries AQL tell %q: %s", aqlTell, fixture.scrub(response.Body)) + } + } +} + +func TestPostgreSQLFirstQueryRouteLivenessAndReadiness(t *testing.T) { + markReadinessLeg(t, "TestPostgreSQLFirstQueryRouteLivenessAndReadiness") + fixture := newQRouteFixture(t) + catalog := fixture.migrate(t) + _ = catalog.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + server, closeRuntime, err := composePostgreSQLFirstQueryRoute(ctx, qRouteBootConfig(t, fixture, "127.0.0.1:0"), logger) + if err != nil { + t.Fatalf("compose production route: %s", fixture.scrub(err)) + } + var closeOnce sync.Once + closeNow := func() { + closeOnce.Do(func() { + if err := closeRuntime(); err != nil { + t.Errorf("close production route runtime: %s", fixture.scrub(err)) + } + }) + } + defer closeNow() + app := server.App() + + live := requestJSON(t, app, http.MethodGet, "/livez", "") + if live.StatusCode != http.StatusOK || live.Body != `{"status":"live"}` { + t.Fatalf("livez on the first-query route = %d %s; want 200 {\"status\":\"live\"}", live.StatusCode, fixture.scrub(live.Body)) + } + response := requestJSON(t, app, http.MethodGet, "/readyz", "") + if response.StatusCode != http.StatusOK || + !strings.Contains(response.Body, `"status":"ready"`) || + !strings.Contains(response.Body, `"core":"ready"`) || + !strings.Contains(response.Body, `"dataframe":"disabled"`) { + t.Fatalf("migrated first-query-route readiness = %d %s; want 200 ready/ready/disabled", + response.StatusCode, fixture.scrub(response.Body)) + } + if strings.Contains(strings.ToLower(response.Body), "password") || strings.Contains(response.Body, fixture.dsn) { + t.Fatalf("readiness body leaked connection material: %s", fixture.scrub(response.Body)) + } + + closeNow() + liveAfter := requestJSON(t, app, http.MethodGet, "/livez", "") + if liveAfter.StatusCode != http.StatusOK || liveAfter.Body != `{"status":"live"}` { + t.Fatalf("livez with a closed pool = %d %s; liveness must be DB-free", liveAfter.StatusCode, fixture.scrub(liveAfter.Body)) + } + refused := requestJSON(t, app, http.MethodGet, "/readyz", "") + if refused.StatusCode != http.StatusServiceUnavailable || + !strings.Contains(refused.Body, `"core":"not_ready"`) || + !strings.Contains(refused.Body, `"core_state":"runtime_closed"`) { + t.Fatalf("readyz with a closed pool = %d %s; want 503 core=not_ready core_state=runtime_closed", + refused.StatusCode, fixture.scrub(refused.Body)) + } +} + +func TestQRouteCensusVocabularyCorrespondence(t *testing.T) { + qrHex := func() string { + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("sample suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) + } + for i := 0; i < 200; i++ { + name := fmt.Sprintf(qRouteDBFmt, time.Now().UnixNano(), qrHex()) + if !qRouteDBPattern.MatchString(name) { + t.Fatalf("mint-shaped name %q does not match the own census regex", name) + } + if qRouteCensusSibling.MatchString(name) { + t.Fatalf("own-vocabulary name %q matched the query package's census regex: parallel census race is back", name) + } + } + ownPositives := []string{"verify_qroute_1731234567890_0123456789ab"} + ownNegatives := []string{ + "verify_qroute_q_zz", + "verify_qroute_1_x", + "verify_qroute_q1731234567890_0123456789ab", + "verify_qroute_", + "verify_qroute_1731234567890_", + "verify_qroute__0123456789ab", + "verify_qroute_1731234567890_0123456789AB", + "verify_qroute_1731234567890_0123456789ab_src", + "verify_qroute_1731234567890_0123456789ab_rel", + "verify_qroute_0x1f_0123456789ab", + "xverify_qroute_1731234567890_0123456789ab", + "verify_qroute_1731234567890_0123456789abx", + } + for _, name := range ownPositives { + if !qRouteDBPattern.MatchString(name) { + t.Errorf("positive %q must match the own census regex", name) + } + if qRouteCensusSibling.MatchString(name) { + t.Errorf("positive %q must not match the sibling census regex", name) + } + } + for _, name := range ownNegatives { + if qRouteDBPattern.MatchString(name) { + t.Errorf("never-minted shape %q matched the own census regex: the census can now hide foreign churn under this file's name", name) + } + } + for i := 0; i < 200; i++ { + sibling := fmt.Sprintf("verify_qroute_q%d_%s", time.Now().UnixNano(), qrHex()) + if qRouteDBPattern.MatchString(sibling) { + t.Fatalf("sibling-vocabulary name %q matched this file's census regex", sibling) + } + } +} + +func TestZZQRouteZeroLeftoverFixtures(t *testing.T) { + raw := qRouteTestPostgresURL(t) + if !qRouteBaselineSet { + t.Skip("no live fixture was created in this run; nothing to audit") + } + if qRouteBaselineErr != nil { + t.Fatalf("baseline was never captured: %v", qRouteBaselineErr) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for audit: %s", err) + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname ~ '^verify_qroute_[0-9]+_[0-9a-f]+$'"). + Scan(&count); err != nil { + t.Fatalf("count leftovers: %s", err) + } + qRouteOwnedMu.Lock() + registered, outstanding, strays := 0, []string{}, append([]string(nil), qRouteStrayMarks...) + perLeg := map[string]int{} + unmatched := []string{} + for db, rec := range qRouteOwned { + registered++ + perLeg[rec.leg]++ + if !rec.dropped { + outstanding = append(outstanding, db) + } + if !qRouteDBPattern.MatchString(db) { + unmatched = append(unmatched, db) + } + } + qRouteOwnedMu.Unlock() + if len(outstanding) > 0 { + t.Fatalf("owned fixtures still live at audit: %v", outstanding) + } + if len(strays) > 0 { + t.Fatalf("drop marks for names this process never minted: %v", strays) + } + if len(unmatched) > 0 { + t.Fatalf("registered mints outside the census regex: %v", unmatched) + } + for leg, want := range qRouteMintInventory { + if got := perLeg[leg]; got != want { + t.Errorf("mint inventory: leg %q minted %d databases, inventoried %d", leg, got, want) + } + delete(perLeg, leg) + } + for leg, got := range perLeg { + t.Errorf("mint inventory: leg %q minted %d databases but is not inventoried", leg, got) + } + if count != qRouteBaseline { + rows, err := admin.Query(ctx, + "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_qroute\\_%' ESCAPE '\\' ORDER BY datname") + names := []string{} + if err == nil { + for rows.Next() { + var name string + if err := rows.Scan(&name); err == nil { + names = append(names, name) + } + } + rows.Close() + } + t.Fatalf("own-prefix census: live=%d baseline=%d registered=%d names=%v", count, qRouteBaseline, registered, names) + } +} + +func TestPostgreSQLFirstQueryRouteDeadEndpointFailsBounded(t *testing.T) { + qRouteTestPostgresURL(t) + cfg := Config{} + cfg.Server.Backend = sourceBackendPostgres + cfg.Server.PostgreSQL = qRoutePostgreSQLConfig() + cfg.Server.PostgreSQL.Schemas = map[string]string{ + "source": "verify_qroute_dead_src", "dataset_release": "verify_qroute_dead_rel", + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + started := time.Now() + err := runPostgreSQLFirstQueryRoute(ctx, cfg, logger) + elapsed := time.Since(started) + + if err == nil { + t.Fatal("dead-endpoint composition unexpectedly started the server") + } + if errors.Is(err, ErrPostgreSQLRuntimeNotWired) { + t.Fatalf("dead endpoint was refused by the consumability sentinel instead of the factory's dial law: %v", err) + } + if elapsed > 5*time.Minute { + t.Fatalf("dead-endpoint failure took %v: the route must converge on a typed failure, not hang-forever", elapsed) + } + pg := cfg.Server.PostgreSQL + for _, secret := range []string{pg.Password, pg.User + ":" + pg.Password, pg.Host} { + if strings.Contains(err.Error(), secret) { + t.Fatalf("dead-endpoint failure leaked credential material %q: %v", secret, err) + } + } + var wrapped *redactedRouteError + if errors.As(err, &wrapped) && wrapped.Unwrap() == nil { + t.Fatalf("scrubbed failure lost its cause: %v", err) + } + t.Logf("dead-endpoint bound witness: elapsed=%v errKind=%T", elapsed, err) +} diff --git a/internal/server/postgres_refusal_contract_test.go b/internal/server/postgres_refusal_contract_test.go new file mode 100644 index 000000000..1cc119225 --- /dev/null +++ b/internal/server/postgres_refusal_contract_test.go @@ -0,0 +1,58 @@ +package server + +import ( + "context" + "errors" + "strings" + "testing" + "time" +) + +const postgreSQLRefusalMessage = "server.backend postgres is configured, but the PostgreSQL source runtime is not wired yet; refusing to start on the ArangoDB backend" + +func TestPostgreSQLBackendStartupRefusalInvariant(t *testing.T) { + t.Run("postgres_selection_refuses_with_the_named_sentinel", func(t *testing.T) { + cfg := Config{} + cfg.Server.Backend = sourceBackendPostgres + cfg.Server.PostgreSQL.Host = "192.0.2.1" + cfg.Server.PostgreSQL.Port = 60543 + cfg.Server.PostgreSQL.Database = "loom_refusal_witness" + cfg.Server.PostgreSQL.User = "loom_refusal_witness" + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + started := time.Now() + err := RunWithConfig(ctx, cfg) + elapsed := time.Since(started) + + if !errors.Is(err, ErrPostgreSQLRuntimeNotWired) { + t.Fatalf("RunWithConfig(postgres) err = %v, want errors.Is(err, ErrPostgreSQLRuntimeNotWired)", err) + } + if got := err.Error(); got != postgreSQLRefusalMessage { + t.Fatalf("refusal message drifted: got %q, want byte-stable %q", got, postgreSQLRefusalMessage) + } + if elapsed > 1500*time.Millisecond { + t.Fatalf("refusal took %v: the gate must refuse before any connection attempt", elapsed) + } + if strings.Contains(err.Error(), "192.0.2.1") || strings.Contains(err.Error(), "loom_refusal_witness") { + t.Fatalf("refusal error leaked connection material: %v", err) + } + }) + + t.Run("alive_control_arango_selection_passes_the_gate", func(t *testing.T) { + cfg := Config{} + cfg.Server.Backend = sourceBackendArango + cfg.Server.URL = "http://127.0.0.1:1" + cfg.Server.Database = "loom_gate_alive_control" + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + err := RunWithConfig(ctx, cfg) + if err == nil { + t.Fatal("RunWithConfig(arango) unexpectedly started against a closed port") + } + if errors.Is(err, ErrPostgreSQLRuntimeNotWired) { + t.Fatal("Arango selection was refused by the PostgreSQL gate") + } + }) +} diff --git a/internal/server/preview_postgres_wiring_pg18_test.go b/internal/server/preview_postgres_wiring_pg18_test.go new file mode 100644 index 000000000..bc539e362 --- /dev/null +++ b/internal/server/preview_postgres_wiring_pg18_test.go @@ -0,0 +1,453 @@ +package server + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + postgres "github.com/calypr/loom/internal/backends/postgres" + storepg "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/command" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + "github.com/calypr/loom/internal/dataframe/recipe" + "github.com/calypr/loom/internal/source" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + exec04srvPrefix = "verify_exec04_srv_" + exec04srvNameFmt = exec04srvPrefix + "%d_%s" +) + +var exec04srvNamePattern = regexp.MustCompile(`^verify_exec04_srv_[0-9]+_[0-9a-f]+$`) + +var ( + exec04srvOwnedMu sync.Mutex + exec04srvOwnedByName = map[string]string{} +) + +func exec04srvRegister(db, schema string) { + exec04srvOwnedMu.Lock() + defer exec04srvOwnedMu.Unlock() + exec04srvOwnedByName[db] = schema +} + +func exec04srvRegistrySnapshot() map[string]string { + exec04srvOwnedMu.Lock() + defer exec04srvOwnedMu.Unlock() + out := make(map[string]string, len(exec04srvOwnedByName)) + for db, schema := range exec04srvOwnedByName { + out[db] = schema + } + return out +} + +var ( + exec04srvBaselineOnce sync.Once + exec04srvBaselineNames []string + exec04srvBaselineErr error +) + +func exec04srvCaptureBaseline(raw string) { + exec04srvBaselineOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + exec04srvBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec04\\_srv\\_%' ESCAPE '\\'") + if err != nil { + exec04srvBaselineErr = fmt.Errorf("read baseline: %w", err) + return + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + exec04srvBaselineErr = fmt.Errorf("scan baseline: %w", err) + return + } + exec04srvBaselineNames = append(exec04srvBaselineNames, name) + } + }) +} + +type exec04srvService struct { + ctx context.Context + admin *pgxpool.Pool + iso *pgxpool.Pool + schema string + dsn string + target compiler.Target + scrub func(any) string + fail func(step string, err error) + executor dataframeexecution.SourceRows +} + +func exec04srvScrubberFor(raw string) (func(any) string, func(string)) { + secrets := []string{raw} + if u, err := url.Parse(raw); err == nil && u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + return scrub, func(s string) { secrets = append(secrets, s) } +} + +func TestExec04srvScrubberHidesCredentialForms(t *testing.T) { + raw := "postgres://verify_user:S3cr3t-Pw@db.internal:5432/loomsvc?sslmode=require" + scrub, addSecret := exec04srvScrubberFor(raw) + dsn := "postgres://verify_user:S3cr3t-Pw@db.internal:5432/verify_exec04_srv_it" + addSecret(dsn) + probe := fmt.Errorf("cannot parse `%s`: pq: auth failed for %s via %s", raw, "verify_user:S3cr3t-Pw", dsn) + got := scrub(probe) + for _, secret := range []string{raw, "verify_user:S3cr3t-Pw", "S3cr3t-Pw", dsn} { + if strings.Contains(got, secret) { + t.Fatalf("scrubbed text still contains credential form %q: %s", secret, got) + } + } + if !strings.Contains(got, "") { + t.Fatalf("scrubbed text lost the redaction marker: %s", got) + } +} + +func newExec04srvService(t *testing.T) *exec04srvService { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 server-preview wiring tests") + } + scrub, addSecret := exec04srvScrubberFor(raw) + exec04srvCaptureBaseline(raw) + if exec04srvBaselineErr != nil { + t.Fatalf("capture pg_database baseline: %s", scrub(exec04srvBaselineErr)) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + fail := func(step string, err error) { + t.Fatalf("%s: %s", step, scrub(err)) + } + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + fail("connect to test service", err) + } + t.Cleanup(admin.Close) + + var versionNum int64 + var version string + if err := admin.QueryRow(ctx, "SELECT current_setting('server_version_num')::int8, current_setting('server_version')").Scan(&versionNum, &version); err != nil { + fail("read server version", err) + } + if err := storepg.CheckServerVersion(versionNum, version); err != nil { + t.Fatalf("test service is not the required PostgreSQL 18 backend: %v", err) + } + + dbName := fmt.Sprintf(exec04srvNameFmt, time.Now().UnixNano(), exec04srvRandomHex(t)) + if !exec04srvNamePattern.MatchString(dbName) { + t.Fatalf("isolated database name %q does not match the required pattern", dbName) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+dbName+`"`); err != nil { + fail("create isolated database", err) + } + var created int64 + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", dbName).Scan(&created); err != nil { + fail("verify isolated database creation", err) + } + if created != 1 { + t.Fatalf("isolated database %q did not verifiably come into existence", dbName) + } + exec04srvRegister(dbName, "") + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(ctx), `DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + } + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + u.RawQuery = q.Encode() + } + u.Path = "/" + dbName + dsn := u.String() + addSecret(dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + fail("connect to isolated database", err) + } + t.Cleanup(iso.Close) + + schema := fmt.Sprintf(exec04srvNameFmt, time.Now().UnixNano(), exec04srvRandomHex(t)) + if !exec04srvNamePattern.MatchString(schema) { + t.Fatalf("owned schema name %q does not match the required pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + fail("create owned schema", err) + } + exec04srvRegister(dbName, schema) + t.Cleanup(func() { + if _, err := iso.Exec(context.WithoutCancel(ctx), `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop owned schema: %s", scrub(err)) + } + }) + + executor, err := postgres.NewSourceRows(iso) + if err != nil { + fail("create PostgreSQL row executor", err) + } + binding, err := ir.NewPostgresStorageBinding(schema) + if err != nil { + fail("bind PostgreSQL storage placement", err) + } + target, err := compiler.PostgreSQLTarget(binding) + if err != nil { + fail("select PostgreSQL compile target", err) + } + return &exec04srvService{ctx: ctx, admin: admin, iso: iso, schema: schema, dsn: dsn, target: target, scrub: scrub, fail: fail, executor: executor} +} + +func exec04srvRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return hex.EncodeToString(b[:]) +} + +func (s *exec04srvService) migrateSeed(t *testing.T) { + t.Helper() + client, err := storepg.OpenPool(s.ctx, storepg.ClientConfig{DSN: s.dsn, MaxConns: 4, MinConns: 1, AcquireTimeout: 2 * time.Second}) + if err != nil { + s.fail("open pg-store client (requires a live PostgreSQL 18 service)", err) + } + defer client.Close() + if _, err := storepg.MigrateSchema(s.ctx, client, s.schema); err != nil { + s.fail("migrate owned schema", err) + } + repo, err := storepg.NewResourceRepository(s.iso, s.schema) + if err != nil { + t.Fatalf("resource repository: %v", err) + } + for _, row := range []struct{ logical, payload string }{ + {"patient-ada", `{"gender":"female","birthDate":"1815-12-10"}`}, + {"patient-alan", `{"gender":"male","birthDate":"1912-06-23"}`}, + } { + if _, err := repo.InsertResource(s.ctx, source.Resource{ + Project: "P1", DatasetGeneration: "gen-a", ResourceType: "Patient", + LogicalID: row.logical, AuthResourcePath: "FHIR/Patient", Payload: json.RawMessage(row.payload), + }); err != nil { + s.fail("insert fixture resource", err) + } + } +} + +func exec04srvBundle() recipe.Bundle { + return recipe.Bundle{ + RecipeSchemaVersion: recipe.CurrentSchemaVersion, + Name: "exec04_srv_preview", + TranslationVersion: "exec04", + Outputs: []recipe.Output{{ + Name: "patients", RootResourceType: "Patient", RowGrain: "patient", + Fields: []recipe.Field{ + {Name: "gender", ColumnID: "gender", Expr: recipe.Expression{Select: "root.gender"}}, + {Name: "birthDate", ColumnID: "birthDate", Expr: recipe.Expression{Select: "root.birthDate"}}, + }, + }}, + } +} + +func exec04srvBindings() recipe.RuntimeBindings { + return recipe.RuntimeBindings{ + Project: "P1", DatasetGeneration: "gen-a", + AuthResourcePaths: []string{"FHIR/Patient"}, + } +} + +func (s *exec04srvService) compile(t *testing.T, target compiler.Target) dataframeexecution.Resolved { + t.Helper() + engine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: target, + Registry: compilerTestRegistry{}, + SourceRows: func(context.Context, dataframeexecution.RowsRequest, dataframeexecution.RowVisitor) error { + return errors.New("compile leg streams nothing") + }, + }) + if err != nil { + t.Fatalf("engine: %s", s.scrub(err)) + } + resolved, err := engine.CompileResolvedBundle(s.ctx, exec04srvBundle(), exec04srvBindings()) + if err != nil { + t.Fatalf("CompileResolvedBundle: %s", s.scrub(err)) + } + return resolved +} + +func TestPostgresPreviewServerWiringFailClosesPGPlan(t *testing.T) { + s := newExec04srvService(t) + s.migrateSeed(t) + resolved := s.compile(t, s.target) + + var arangoCalls atomic.Int64 + arangoLandmine := func(context.Context, dataframeexecution.RowsRequest, dataframeexecution.RowVisitor) error { + arangoCalls.Add(1) + return errors.New("the Arango adapter must never run for a PostgreSQL-served preview") + } + engine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: s.target, + Registry: compilerTestRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "exec04-srv" }, + SourceRows: configuredAQLSourceRows(nil, "query_rows", arangoLandmine), + PreviewSourceRows: configuredAQLSourceRows(nil, "preview_query_rows", func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + arangoCalls.Add(1) + return arangoLandmine(ctx, request, visit) + }), + }) + if err != nil { + t.Fatalf("engine: %s", s.scrub(err)) + } + summary, err := engine.PreviewOutput(s.ctx, resolved, dataframeexecution.PreviewRequest{Output: "patients", Limit: 2}, func(map[string]any) error { return nil }) + if !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("server AQL wiring over a PostgreSQL preview error = %v (%s), want the typed backend mismatch refusal", err, s.scrub(err)) + } + if summary.Complete || summary.Truncated || summary.RowCount != 0 { + t.Fatalf("refused preview carried a result marker: %+v", summary) + } + if arangoCalls.Load() != 0 { + t.Fatalf("Arango callback ran %d times for a PostgreSQL-tagged preview", arangoCalls.Load()) + } + + t.Run("alive: the same compiled preview streams through the real PostgreSQL adapter", func(t *testing.T) { + adapterEngine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: s.target, + Registry: compilerTestRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "exec04-srv" }, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + if _, _, dispatchErr := request.Command.PostgreSQLParameters(); dispatchErr != nil { + return dispatchErr + } + return s.executor(ctx, request, visit) + }, + }) + if err != nil { + t.Fatalf("adapter engine: %s", s.scrub(err)) + } + var rows []map[string]any + adapterSummary, adapterErr := adapterEngine.PreviewOutput(s.ctx, resolved, dataframeexecution.PreviewRequest{Output: "patients", Limit: 3}, func(row map[string]any) error { + rows = append(rows, row) + return nil + }) + if adapterErr != nil { + t.Fatalf("alive control through the real adapter: %s", s.scrub(adapterErr)) + } + if len(rows) != 2 || !adapterSummary.Complete || adapterSummary.Truncated { + t.Fatalf("alive control rows=%d summary Complete=%v Truncated=%v, want 2 rows naturally exhausted", len(rows), adapterSummary.Complete, adapterSummary.Truncated) + } + if arangoCalls.Load() != 0 { + t.Fatalf("the Arango landmine ran on the alive adapter leg") + } + }) + + t.Run("alive: the same server wiring proceeds for an AQL preview", func(t *testing.T) { + aqlCalls := atomic.Int64{} + aqlEngine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + ScopeDigest: func(recipe.RuntimeBindings) string { return "exec04-srv" }, + SourceRows: configuredAQLSourceRows(nil, "query_rows", func(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + aqlCalls.Add(1) + if _, _, dispatchErr := request.Command.AQLParameters(); dispatchErr != nil { + return dispatchErr + } + return visit(map[string]any{"gender": "female", "birthDate": "1815-12-10"}) + }), + }) + if err != nil { + t.Fatalf("AQL engine: %s", s.scrub(err)) + } + aqlResolved := s.compile(t, compiler.ArangoTarget()) + aqlRows := 0 + if _, aqlErr := aqlEngine.PreviewOutput(s.ctx, aqlResolved, dataframeexecution.PreviewRequest{Output: "patients", Limit: 1}, func(map[string]any) error { aqlRows++; return nil }); aqlErr != nil { + t.Fatalf("alive AQL control through the same server wiring: %s", s.scrub(aqlErr)) + } + if aqlCalls.Load() == 0 || aqlRows == 0 { + t.Fatalf("alive AQL control did not proceed: calls=%d rows=%d", aqlCalls.Load(), aqlRows) + } + }) +} + +func TestZZPostgresPreviewServerZeroLeftoverFixtures(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL 18 server-preview wiring tests") + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + scrub, _ := exec04srvScrubberFor(raw) + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for audit: %s", scrub(err)) + } + defer admin.Close() + for db := range exec04srvRegistrySnapshot() { + var count int + if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_database WHERE datname = $1", db).Scan(&count); err != nil { + t.Fatalf("audit %s: %s", db, scrub(err)) + } + if count != 0 { + t.Errorf("owned fixture database %s survived", db) + } + } + baseline := map[string]bool{} + for _, name := range exec04srvBaselineNames { + baseline[name] = true + } + rows, err := admin.Query(ctx, "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_exec04\\_srv\\_%' ESCAPE '\\'") + if err != nil { + t.Fatalf("audit sweep: %s", scrub(err)) + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("audit scan: %s", scrub(err)) + } + if !baseline[name] { + t.Errorf("unregistered survivor under owned prefix: %s", name) + } + } + if err := rows.Err(); err != nil { + t.Fatalf("audit rows: %s", scrub(err)) + } +} diff --git a/internal/server/private_artifact_reconciler.go b/internal/server/private_artifact_reconciler.go index 785992927..c22ba8411 100644 --- a/internal/server/private_artifact_reconciler.go +++ b/internal/server/private_artifact_reconciler.go @@ -28,9 +28,6 @@ func startPrivateArtifactReconciler(ctx context.Context, manager privateArtifact return startTickerReconciler(ctx, sweep, logger, privateArtifactReconcileInterval, "private ClickHouse artifact") } -// startArtifactFSReconciler enforces the exported-archive expiry promise on -// the filesystem artifact store, mirroring the ClickHouse artifact sweep so -// committed archives do not outlive their advertised TTL on disk. func startArtifactFSReconciler(ctx context.Context, store artifactFSReconcileTarget, logger *slog.Logger) <-chan struct{} { if store == nil { done := make(chan struct{}) @@ -55,9 +52,6 @@ func startTickerReconciler(ctx context.Context, sweep func(context.Context) erro return done } -// runPrivateArtifactReconciler performs an immediate sweep, then one bounded -// sweep per tick. It processes ticks serially so a slow cleanup cannot overlap -// with the next pass. func runPrivateArtifactReconciler(ctx context.Context, manager privateArtifactReconcileTarget, ticks <-chan time.Time, logger *slog.Logger) { if manager == nil { return diff --git a/internal/server/publication_rows_integration_test.go b/internal/server/publication_rows_integration_test.go index 9ffb73b95..515605f58 100644 --- a/internal/server/publication_rows_integration_test.go +++ b/internal/server/publication_rows_integration_test.go @@ -12,26 +12,20 @@ import ( "time" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" + "github.com/calypr/loom/internal/dataframe/compiler" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" - publicationarango "github.com/calypr/loom/internal/dataframe/publication/arango" publicationclickhouse "github.com/calypr/loom/internal/dataframe/publication/clickhouse" "github.com/calypr/loom/internal/dataset" - datasetarango "github.com/calypr/loom/internal/dataset/arango" "github.com/calypr/loom/internal/explorer" - explorerarango "github.com/calypr/loom/internal/explorer/arango" "github.com/calypr/loom/internal/explorer/authoringv2" "github.com/calypr/loom/internal/explorer/capability" "github.com/calypr/loom/internal/explorer/lifecycle" - storearango "github.com/calypr/loom/internal/store/arango" storeclickhouse "github.com/calypr/loom/internal/store/clickhouse" "github.com/gofiber/fiber/v3" "github.com/google/uuid" ) -// A bounded assembled publication checkpoint: actual AQL/compiler/stream, -// ClickHouse materialization, registry and atomic Explorer/release activation. -// Authorization/capability discovery are fixture-owned. Fresh clients prove -// stored read-back, not database process restart or browser rendering. func TestPublicationRowsAndFreshViewerAgainstArangoClickHouse(t *testing.T) { arangoURL, clickhouseURL := os.Getenv("LOOM_TEST_ARANGO_URL"), os.Getenv("LOOM_CLICKHOUSE_URL") if arangoURL == "" || clickhouseURL == "" { @@ -40,10 +34,11 @@ func TestPublicationRowsAndFreshViewerAgainstArangoClickHouse(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) defer cancel() database := "loom_publication_rows_" + strings.ReplaceAll(uuid.NewString(), "-", "") - graph, err := storearango.Open(ctx, arangoURL, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: arangoURL, Database: database}) if err != nil { t.Fatal(err) } + graph := arangoBackend.Client t.Cleanup(func() { graph.Close(context.Background()) request, _ := http.NewRequest(http.MethodDelete, strings.TrimRight(arangoURL, "/")+"/_api/database/"+database, nil) @@ -57,7 +52,7 @@ func TestPublicationRowsAndFreshViewerAgainstArangoClickHouse(t *testing.T) { t.Errorf("owned graph cleanup: HTTP%d", response.StatusCode) } }) - for _, spec := range []storearango.BootstrapSpec{explorerarango.BootstrapSpec(), publicationarango.BootstrapSpec(), datasetarango.BootstrapSpec(), {Collections: []storearango.CollectionSpec{{Name: "Patient"}}}} { + for _, spec := range []arangofactory.BootstrapSpec{arangofactory.ExplorerBootstrapSpec(), arangofactory.PublishedBootstrapSpec(), arangofactory.DatasetBootstrapSpec(), {Collections: []arangofactory.CollectionSpec{{Name: "Patient"}}}} { if err := graph.Bootstrap(ctx, spec); err != nil { t.Fatal(err) } @@ -79,7 +74,7 @@ func TestPublicationRowsAndFreshViewerAgainstArangoClickHouse(t *testing.T) { t.Error(err) } }) - registry, err := publicationarango.New(graph) + registry, err := arangofactory.NewPublishedRegistry(graph) if err != nil { t.Fatal(err) } @@ -108,7 +103,7 @@ func TestPublicationRowsAndFreshViewerAgainstArangoClickHouse(t *testing.T) { if err != nil { t.Fatal(err) } - store, err := explorerarango.New(graph) + store, err := arangofactory.NewExplorerStore(graph) if err != nil { t.Fatal(err) } @@ -120,9 +115,17 @@ func TestPublicationRowsAndFreshViewerAgainstArangoClickHouse(t *testing.T) { if err != nil { t.Fatal(err) } - engine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: func(ctx context.Context, query string, batch int, vars map[string]any, visit func(map[string]any) error) error { - return graph.QueryRows(ctx, query, batch, vars, storearango.RowVisitor(visit)) - }}) + engine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + batch := request.BatchSize + query, vars, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + + return graph.QueryRows(ctx, query, batch, vars, arangofactory.RowVisitor(visit)) + }}) if err != nil { t.Fatal(err) } @@ -131,7 +134,7 @@ func TestPublicationRowsAndFreshViewerAgainstArangoClickHouse(t *testing.T) { if err != nil { t.Fatal(err) } - releases, err := datasetarango.New(graph) + releases, err := arangofactory.NewDatasetStore(graph) if err != nil { t.Fatal(err) } @@ -201,7 +204,6 @@ func TestPublicationRowsAndFreshViewerAgainstArangoClickHouse(t *testing.T) { if result.ReceiptID != receipt.ID || result.RevisionID == "" { t.Fatalf("publication identity: %#v", result) } - // Later draft/source edits must not change the already materialized revision. workspace.Documents[0].Columns[0].Label = "Unpublished label" laterRaw, err := workspace.CanonicalJSON() if err != nil { @@ -237,11 +239,12 @@ func TestPublicationRowsAndFreshViewerAgainstArangoClickHouse(t *testing.T) { t.Fatalf("repeat changed immutable publication: %#v", repeat) } for read := 0; read < 2; read++ { - freshGraph, err := storearango.Open(ctx, arangoURL, database) + arangoBackend, err := arangofactory.Open(ctx, arangofactory.Config{URL: arangoURL, Database: database}) if err != nil { t.Fatal(err) } - freshStore, err := explorerarango.New(freshGraph) + freshGraph := arangoBackend.Client + freshStore, err := arangofactory.NewExplorerStore(freshGraph) if err != nil { t.Fatal(err) } @@ -273,7 +276,7 @@ func TestPublicationRowsAndFreshViewerAgainstArangoClickHouse(t *testing.T) { if output.OutputID != "patients" || output.Materialization == nil || output.Selector != (dataset.DataframeSelector{Recipe: receipt.Bundle.Name, TranslationVersion: receipt.Bundle.TranslationVersion, Output: "patients"}) || len(output.Columns) != 1 || output.Columns[0].Label != "Patient ID" { t.Fatalf("Viewer output mismatch: %#v", output) } - freshRegistry, err := publicationarango.New(freshGraph) + freshRegistry, err := arangofactory.NewPublishedRegistry(freshGraph) if err != nil { t.Fatal(err) } diff --git a/internal/server/publication_viewer_agreement_test.go b/internal/server/publication_viewer_agreement_test.go index b4cc6a776..d539860ca 100644 --- a/internal/server/publication_viewer_agreement_test.go +++ b/internal/server/publication_viewer_agreement_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/publication" "github.com/calypr/loom/internal/dataframe/recipe" @@ -22,9 +23,6 @@ import ( "github.com/gofiber/fiber/v3" ) -// The repository and materializer are explicit test boundaries. Production -// compiler, lifecycle policy, strict HTTP routes and Viewer projection execute; -// this is not a database-restart, real materialization, or browser-row proof. type agreementPublicationStore struct { *testExplorerStore publishCalls int @@ -97,8 +95,6 @@ func TestPublicationHTTPReceiptRevisionAndViewerAgreeAfterFreshRead(t *testing.T if persisted.ID != result.RevisionID || persisted.CompilationReceiptID != receipt.ID || persisted.IntentDigest != receipt.IntentDigest || persisted.RecipeDigest != receipt.RecipeDigest || persisted.ResolvedSchemaDigest != receipt.ResolvedSchemaDigest || !reflect.DeepEqual(persisted.PublicOutputContract, receipt.PublicOutputContract) { t.Fatalf("persisted publication identity diverged from receipt: revision=%#v receipt=%#v", persisted, receipt) } - // Deliberately change the live draft after publication. A new GET must - // continue projecting the immutable published receipt, not this draft. unpublished, err := authoringv2.DecodeWorkspace(receipt.NormalizedBundle) if err != nil { t.Fatal(err) @@ -211,7 +207,15 @@ func publicationAgreementFixture(t *testing.T) (*fiber.App, *agreementPublicatio if err != nil { t.Fatal(err) } - engine, err := dataframeexecution.New(dataframeexecution.Config{Registry: compilerTestRegistry{}, QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { return nil }}) + engine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: compiler.ArangoTarget(), Registry: compilerTestRegistry{}, SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return nil + }}) if err != nil { t.Fatal(err) } diff --git a/internal/server/receipt_preview_diagnostic.go b/internal/server/receipt_preview_diagnostic.go index ccc6e5a99..1e6c12790 100644 --- a/internal/server/receipt_preview_diagnostic.go +++ b/internal/server/receipt_preview_diagnostic.go @@ -7,8 +7,6 @@ import ( dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" ) -// logReceiptPreviewContractFailure records only receipt-contract failures. The -// transport continues to expose its existing redacted 409 response. func logReceiptPreviewContractFailure(logger *slog.Logger, requestID, receiptID, outputID string, err error) { if logger == nil || err == nil { return diff --git a/internal/server/recipe_authorization.go b/internal/server/recipe_authorization.go index 78b607501..3cec7b8e3 100644 --- a/internal/server/recipe_authorization.go +++ b/internal/server/recipe_authorization.go @@ -9,12 +9,12 @@ import ( "github.com/calypr/loom/internal/authscope" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" "github.com/calypr/loom/internal/dataframe/recipe" - explorerarango "github.com/calypr/loom/internal/explorer/arango" "github.com/calypr/loom/internal/projectid" ) type recipeAuthorization struct { - resolver *authscope.ScopeResolver + resolver *authscope.ScopeResolver + selectionMembersCollection string } var _ graphresolver.RecipeAuthorizer = recipeAuthorization{} @@ -25,7 +25,7 @@ func (a recipeAuthorization) AuthorizeRead(ctx context.Context, bindings recipe. } bindings.Project = strings.TrimSpace(bindings.Project) bindings.SelectionProject = projectid.Canonical(bindings.Project) - bindings.SelectionMembersCollection = explorerarango.SelectionMembersCollection + bindings.SelectionMembersCollection = a.selectionMembersCollection if a.resolver == nil { bindings.AuthScopeMode = authscope.ReadScopeUnrestricted return bindings, nil @@ -49,7 +49,7 @@ func (a recipeAuthorization) AuthorizeWrite(ctx context.Context, bindings recipe } bindings.Project = strings.TrimSpace(bindings.Project) bindings.SelectionProject = projectid.Canonical(bindings.Project) - bindings.SelectionMembersCollection = explorerarango.SelectionMembersCollection + bindings.SelectionMembersCollection = a.selectionMembersCollection if a.resolver == nil { bindings.AuthScopeMode = authscope.ReadScopeUnrestricted return bindings, nil diff --git a/internal/server/recipe_authorization_test.go b/internal/server/recipe_authorization_test.go index 9664feb5c..f02c793ea 100644 --- a/internal/server/recipe_authorization_test.go +++ b/internal/server/recipe_authorization_test.go @@ -5,12 +5,12 @@ import ( "testing" "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" "github.com/calypr/loom/internal/dataframe/recipe" - explorerarango "github.com/calypr/loom/internal/explorer/arango" ) func TestRecipeAuthorizationBindsSavedSelectionToAuthorizedProject(t *testing.T) { - authorizer := recipeAuthorization{} + authorizer := recipeAuthorization{selectionMembersCollection: arangofactory.SelectionMembersCollection} for name, authorize := range map[string]func(context.Context, recipe.RuntimeBindings) (recipe.RuntimeBindings, error){ "read": authorizer.AuthorizeRead, "write": authorizer.AuthorizeWrite, @@ -23,7 +23,7 @@ func TestRecipeAuthorizationBindsSavedSelectionToAuthorizedProject(t *testing.T) if err != nil { t.Fatal(err) } - if bindings.SelectionProject != "HTAN_INT/project-a" || bindings.SelectionMembersCollection != explorerarango.SelectionMembersCollection { + if bindings.SelectionProject != "HTAN_INT/project-a" || bindings.SelectionMembersCollection != arangofactory.SelectionMembersCollection { t.Fatalf("saved selection is not bound to the authorized project and server-owned store: project=%q collection=%q", bindings.SelectionProject, bindings.SelectionMembersCollection) } if bindings.Project != "HTAN_INT-project-a" || bindings.DatasetGeneration != "generation-a" || bindings.AuthScopeMode != authscope.ReadScopeUnrestricted { @@ -31,4 +31,25 @@ func TestRecipeAuthorizationBindsSavedSelectionToAuthorizedProject(t *testing.T) } }) } + pgAuthorizer := recipeAuthorization{} + for name, authorize := range map[string]func(context.Context, recipe.RuntimeBindings) (recipe.RuntimeBindings, error){ + "read": pgAuthorizer.AuthorizeRead, + "write": pgAuthorizer.AuthorizeWrite, + } { + t.Run("postgres_policy_"+name, func(t *testing.T) { + bindings, err := authorize(context.Background(), recipe.RuntimeBindings{ + Project: "HTAN_INT-project-a", DatasetGeneration: "generation-a", + SelectionProject: "another/project", SelectionMembersCollection: "untrusted_collection", + }) + if err != nil { + t.Fatal(err) + } + if bindings.SelectionMembersCollection != "" { + t.Fatalf("the PostgreSQL policy must clear a caller-supplied selection-members collection, got %q", bindings.SelectionMembersCollection) + } + if bindings.SelectionProject != "HTAN_INT/project-a" { + t.Fatalf("saved selection is not bound to the authorized project: %q", bindings.SelectionProject) + } + }) + } } diff --git a/internal/server/recipe_blank_project_test.go b/internal/server/recipe_blank_project_test.go index 4502597d3..828e63391 100644 --- a/internal/server/recipe_blank_project_test.go +++ b/internal/server/recipe_blank_project_test.go @@ -8,6 +8,7 @@ import ( "github.com/calypr/loom/generated/graphql/graph/model" graphresolver "github.com/calypr/loom/internal/api/graphql/graph/resolver" "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/dataframe/compiler" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" @@ -15,9 +16,6 @@ import ( "github.com/calypr/loom/internal/dataframe/semantic" ) -// blankProjectRegistryFailures prove the engine-level project guard fires -// before any registry lookup, mirroring the deployed 500 ordering where a -// whitespace-only project never reached LoadRecipe. type blankProjectRegistryFailures struct{ loads int } func (r *blankProjectRegistryFailures) LoadRecipe(context.Context, string) (recipeexec.Entry, error) { @@ -34,8 +32,15 @@ func blankProjectEngine(t *testing.T) (*dataframeexecution.Engine, *blankProject t.Helper() registry := &blankProjectRegistryFailures{} engine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: registry, - QueryRows: func(context.Context, string, int, map[string]any, func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: registry, + SourceRows: func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + command := request.Command + _, _, deriveErr := command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + return errors.New("query executor consulted for blank project") }, }) @@ -66,8 +71,6 @@ func assertTypedProjectRequired(t *testing.T, err error, site string) dataframee return userErr } -// (a) Throwing-site reproduction: the deployed 500 escape path -// Control.Run/Preview/Resolve -> Engine.Resolve raw guard error. func TestRecipeControlBlankProjectGuardTypesThrowsAtEscapeSite(t *testing.T) { engine, registry := blankProjectEngine(t) control := dataframeexecution.Control{Engine: engine} @@ -98,7 +101,6 @@ func TestRecipeControlBlankProjectGuardTypesThrowsAtEscapeSite(t *testing.T) { } } -// (b) Authorizer parity: whitespace-only == empty, both non-retryable typed. func TestRecipeAuthorizationTreatsWhitespaceProjectAsEmpty(t *testing.T) { authorizer := recipeAuthorization{} for name, authorize := range map[string]func(context.Context, recipe.RuntimeBindings) (recipe.RuntimeBindings, error){ @@ -151,9 +153,6 @@ func (c *blankProjectFamilyControl) Run(context.Context, string, recipe.RuntimeB return dataframeexecution.Preview{}, c.rawFault() } -// (b) Family legs: run/preview/preflight must reject whitespace-only with the -// same typed PROJECT_REQUIRED the family already publishes for empty, before -// the control plane is consulted. func TestRecipeFamilyLegsRejectWhitespaceProjectBeforeControl(t *testing.T) { legs := []struct { name string @@ -222,9 +221,6 @@ func (s *blankProjectRevisionStore) List(context.Context, string, string) ([]rec return nil, nil } -// (b2) Revision-side family legs: register/revision(s)/explain must classify -// whitespace-only exactly like empty, at the authorizer, before the revision -// registry or control plane is consulted. func TestRecipeRevisionFamilyLegsRejectWhitespaceProjectBeforeStore(t *testing.T) { legs := []struct { name string @@ -279,8 +275,6 @@ func TestRecipeRevisionFamilyLegsRejectWhitespaceProjectBeforeStore(t *testing.T } } -// (c) Foreign faults still surface untouched: genuinely unclassifiable -// control errors keep falling through to INTERNAL_ERROR. func TestRecipeControlForeignFaultStillFallsThroughToInternal(t *testing.T) { control := &blankProjectFamilyControl{} resolver := graphresolver.NewResolver(graphresolver.ResolverConfig{ @@ -303,8 +297,6 @@ func TestRecipeControlForeignFaultStillFallsThroughToInternal(t *testing.T) { } } -// (c) Auth resolver faults keep typing BACKEND_UNAVAILABLE retryable through -// recipeAuthorizationError (unchanged semantics). func TestRecipeAuthorizationResolverFaultStaysBackendUnavailable(t *testing.T) { resolver := authscope.NewScopeResolver(authscope.ScopeResolverConfig{ ResourceAccess: fencedFaultClient{}, diff --git a/internal/server/recipe_execution_etl_failure_fields_e2e_test.go b/internal/server/recipe_execution_etl_failure_fields_e2e_test.go index 76968feec..d00236f6a 100644 --- a/internal/server/recipe_execution_etl_failure_fields_e2e_test.go +++ b/internal/server/recipe_execution_etl_failure_fields_e2e_test.go @@ -21,44 +21,6 @@ import ( dataset "github.com/calypr/loom/internal/dataset" ) -// REPAIR-20 / REPAIR-20b (F-R3). Promise under test: GET -// /api/v1/dataframe/recipe-executions/{id} (the legacy execution document -// consumed by ETL, openapi_recipe_execution.go) publishes the same explicit -// failure fields the GraphQL twin publishes from the same durable record — -// execution error/errorCode/errorRetryable/phase and per-output -// errorCode/errorRetryable/phase, with output.error MIRRORING twin semantics -// exactly (recipe_execution_reader.go gates the execution.Error mirror: it -// projects onto outputs that are not themselves healthy; durable -// READY/PUBLISHED outputs publish no mirrored trio — REPAIR-20c closed the -// lease-lost phantom-trio ledger entry on both twins). Durable writers: -// publication/clickhouse/bundle_store.go failPhase:985-993 and the lease-lost -// reconciliation at :1207-1213 persist these fields; the REST reader may not -// silently drop them. -// -// REPAIR-20b (adversarial review ses_e65cad76effe REQUEST-CHANGES, F-1/F-2/ -// F-3/F-5): parity means the twin's FULL encoding chain, not a raw copy. The -// twin encodes the failure trio through dataframeapi.PersistedFailure -// (internal/api/graphql/graph/dataframe/model.go:73-84) exactly as -// executionModel consumes it (recipe_control_graphql.go:319,334), whose -// code-less branch is errors.SanitizePersistedFailure -// (internal/dataframe/errors/errors.go:271-277): whitespace-only raw text -// becomes present-empty/"INTERNAL_ERROR"-free ("","","",false), unknown raw -// text NEVER echoes — it is Normalize-inferred to a public message/code/ -// retryable — and a persisted code substitutes the code's stable public -// message (PublicMessage). The oracle for this file is the reviewer's 64-cell -// comparison table (8 shapes x exec/output x 4 fields, verdict-level -// equality per field, ABSENT vs null vs present+type+value; compare_r20.py -// semantics) over shapes.json + the live executionModel twin wire. The twin -// side is pinned separately in -// internal/api/graphql/graph/resolver/recipe_execution_twin_wire_parity_test.go -// (real reader + executionModel == twin_golden.json), so the CONTROL this -// file composes (reader + the same PersistedFailure constructor + the -// non-empty Phase gate recipe_control_graphql.go:329-331/339-341) is anchored -// to the genuine executionModel output. -// -// The 404/503 arms are re-proven here unchanged so the additive fields cannot -// regress the concealment/outage contracts. - const fr20ConcealedErr = "recipe execution not found" const ( @@ -72,13 +34,6 @@ const ( fr20OutputName = "rows" ) -// matrix shapes (REPAIR-20b), verbatim from the adversarial review -// /tmp/r20-out/shapes.json (committed at -// testdata/r20_failure_parity/shapes.json; the twin resolver harness reads -// the identical copy). Shapes 1,2 mirror bundle_store.go failPhase:985-993, -// shape 5 mirrors the lease-lost reconciler :1207-1213; shapes 3,4,6,8 are -// the reviewer's sanctioned synthetics pinning the sanitizer branches the -// durable writers cannot currently produce. const fr20TestdataDir = "testdata/r20_failure_parity" type fr20Store struct { @@ -97,9 +52,6 @@ func (s *fr20Store) GetExecution(_ context.Context, id string) (publication.Bund return publication.BundleExecution{}, nil } -// fr20FailedExecution is a FAILED execution exactly as the only failure sink -// persists it (bundle_store.go failPhase:985-993): execution-level Error, -// FailureCode, FailureRetryable, FailurePhase, and per-output Failure* fields. func fr20FailedExecution() publication.BundleExecution { return publication.BundleExecution{ ID: fr20ExecID, @@ -126,8 +78,6 @@ func fr20FailedExecution() publication.BundleExecution { } } -// fr20SucceededExecution is a healthy execution as the durable store holds it: -// no Error, no FailureCode, no FailurePhase anywhere. func fr20SucceededExecution() publication.BundleExecution { return publication.BundleExecution{ ID: fr20SuccessID, @@ -198,8 +148,6 @@ func fr20PointerValue(t *testing.T, object map[string]any, key, want string) { } } -// fr20RequireExplicitBool pins "retryable is JSON true/false, not absent": -// the decoded value must exist and be a real JSON boolean. func fr20RequireExplicitBool(t *testing.T, scope string, object map[string]any, key string, want bool) { t.Helper() raw, ok := object[key] @@ -215,17 +163,9 @@ func fr20RequireExplicitBool(t *testing.T, scope string, object map[string]any, } } -// Subject arm 1: the FAILED execution publishes the twin-encoded failure trio -// (message/code/retryable exactly as the twin's PersistedFailure chain -// encodes them for this record — derived here, never hardcoded) plus phase. -// RED on a raw-copy handler: the wire text differs from the twin substitution -// (F-1/F-2), or the keys are absent entirely (pre REPAIR-20). func TestFR20EtlFailureFieldsPublishedForFailedExecution(t *testing.T) { store := &fr20Store{byID: map[string]publication.BundleExecution{fr20ExecID: fr20FailedExecution()}} - // CONTROL: the GraphQL twin reader over the same durable record pins the - // persisted values (reader level), and the twin's own encoding chain - // computes the expected wire values. principal := &authscope.Principal{Subject: "operator-P", Projects: []string{fr20Project}} ctx := authscope.ContextWithPrincipal(context.Background(), principal) twin, err := resolver.NewAuthorizedRecipeExecutionReader(store, nil)(ctx, fr20ExecID) @@ -243,7 +183,6 @@ func TestFR20EtlFailureFieldsPublishedForFailedExecution(t *testing.T) { t.Logf("F-R20 CONTROL | reader persisted | error=%q errorCode=%q retryable=%v phase=%q | twin wire encoding | error=%q errorCode=%q retryable=%v", twin.Error, twin.ErrorCode, twin.ErrorRetryable, twin.Phase, *expectMessage, *expectCode, *expectRetryable) - // SUBJECT: the REST/ETL twin over the real generated Fiber stack. status, body, fields := fr20HTTPGet(t, store, fr20ExecID) if status != http.StatusOK { t.Fatalf("authorized member read of own FAILED execution must be 200, got %d %s", status, body) @@ -287,11 +226,6 @@ func TestFR20EtlFailureFieldsPublishedForFailedExecution(t *testing.T) { t.Logf("F-R20 VERDICT | PASS | REST twin publishes the twin-encoded failure fields (execution + output, output.error mirrors execution.error through the shared sanitizer chain)") } -// Subject arm 2: the additive-optional contract. A SUCCESS execution gains NO -// empty-string/zero failure fields — the twin Publishes none for a record with -// no persisted failure (PersistedFailure returns all-nil when neither a raw -// error nor a code is persisted; executionModel emits Phase only when -// non-empty). func TestFR20EtlFailureFieldsAbsentForSucceededExecution(t *testing.T) { store := &fr20Store{byID: map[string]publication.BundleExecution{fr20SuccessID: fr20SucceededExecution()}} status, body, fields := fr20HTTPGet(t, store, fr20SuccessID) @@ -317,9 +251,6 @@ func TestFR20EtlFailureFieldsAbsentForSucceededExecution(t *testing.T) { } } -// Subject arm 3: the concealment and outage arms stay byte-unchanged by the -// additive fidelity (unknown id -> 404 concealed envelope; store fault -> -// retryable 503 BACKEND_UNAVAILABLE). func TestFR20EtlFailureFieldsLeave404And503ArmsUnchanged(t *testing.T) { _, body, fields := fr20HTTPGet(t, &fr20Store{byID: map[string]publication.BundleExecution{fr20ExecID: fr20FailedExecution()}}, fr20UnknownID) if !strings.Contains(body, fr20ConcealedErr) || fields["id"] != nil || fields["errorCode"] != nil { @@ -342,18 +273,12 @@ func TestFR20EtlFailureFieldsLeave404And503ArmsUnchanged(t *testing.T) { } } -// --------------------------------------------------------------------------- -// REPAIR-20b parity matrix (reviewer harness ses_e65cad76effe absorbed). -// --------------------------------------------------------------------------- - var fr20ParityFields = []string{"error", "errorCode", "errorRetryable", "phase"} type fr20Shape struct { Name string `json:"name"` } -// fr20ClassifyCell mirrors compare_r20.py cls(): ABSENT / null / -// present(type) canonical-value, with python type names str/bool. type fr20Cell struct { state string kind string @@ -408,16 +333,6 @@ func fr20VerdictRow(shapeName, level, field string, twin, rest map[string]any) f return fr20Row{shape: shapeName, level: level, field: field, twin: render(tv), rest: render(rv), verdict: verdict} } -// fr20TwinWire builds the CONTROL: the live GraphQL twin wire for one shape. -// It feeds the durable record through the production -// resolver.NewAuthorizedRecipeExecutionReader and encodes the failure trio -// with dataframeapi.PersistedFailure + the non-empty Phase gate — exactly the -// primitives executionModel (recipe_control_graphql.go:316-346) consumes and -// marshals to GraphQL JSON. The reviewer-committed executionModel golden -// (testdata/r20_failure_parity/twin_golden.json, produced by the twin-side -// harness over the real executionModel) anchors this composition against -// executionModel drift; the resolver-package harness re-checks live -// executionModel == golden every run. func fr20TwinWire(t *testing.T, execution publication.BundleExecution) map[string]any { t.Helper() principal := &authscope.Principal{Subject: "operator-P", Projects: []string{fr20Project}} @@ -492,9 +407,6 @@ func fr20ReadTestdata(t *testing.T, name string) []byte { return raw } -// fr20JSONEqual is a canonical (order-insensitive) JSON tree equality check -// used to anchor the composed CONTROL against the reviewer golden without -// pinning key order. func fr20JSONEqual(path string, left, right any) string { switch lv := left.(type) { case map[string]any: @@ -546,13 +458,6 @@ func fr20JSONEqual(path string, left, right any) string { } } -// TestFR20bEtlFailureFieldsTwinParityMatrix is the durable oracle from the -// adversarial review: 8 shapes x {execution, output[0]} x {error, errorCode, -// errorRetryable, phase} = 64 verdict-level comparisons between the live -// twin wire (CONTROL, real reader + the twin's own encoding primitives, -// golden-anchored) and the generated strict-stack REST body (OBSERVED). -// Pre-fix (raw-copy handler) this reproduces exactly the reviewer's 18 -// divergences; post root-fix it must end with 0 DIVERGE / 0 NULL_EQUIV. func TestFR20bEtlFailureFieldsTwinParityMatrix(t *testing.T) { var shapes []json.RawMessage if err := json.Unmarshal(fr20ReadTestdata(t, "shapes.json"), &shapes); err != nil { @@ -573,9 +478,6 @@ func TestFR20bEtlFailureFieldsTwinParityMatrix(t *testing.T) { t.Fatalf("shape header: %v", err) } name := header.Name - // Round-trip the same durable JSON into the real durable type the - // production catalog hands GetExecution back (reviewer harness - // discipline, shapes.json -> publication.BundleExecution). var execution publication.BundleExecution if err := json.Unmarshal(rawShape, &execution); err != nil { t.Fatalf("%s: durable round-trip: %v", name, err) @@ -602,8 +504,6 @@ func TestFR20bEtlFailureFieldsTwinParityMatrix(t *testing.T) { } observedOutput := fr20FailedOutput(t, observed) controlOutput := fr20FailedOutput(t, control) - // REPAIR-20c positive pins on every shape, both transports (RED on - // the lease-lost phantom-trio baseline, shape 5). fr20AssertHealthyOutputsPublishNoFailureTrio(t, name, "twin", execution, control) fr20AssertHealthyOutputsPublishNoFailureTrio(t, name, "rest", execution, observed) for _, field := range fr20ParityFields { @@ -630,17 +530,6 @@ func TestFR20bEtlFailureFieldsTwinParityMatrix(t *testing.T) { t.Logf("R20B VERDICT | PASS | 64/64 comparisons MATCH the twin oracle") } -// fr20AssertHealthyOutputsPublishNoFailureTrio is the REST twin of -// r20AssertHealthyOutputsPublishNoFailureTrio (resolver harness): the promise -// authority openapi_recipe_execution.go:95-97 — a durable output in a -// successful state (READY/PUBLISHED) that persisted no own failure code -// publishes no error/errorCode/errorRetryable, even when the execution page -// is FAILED. Shape 5 (lease-lost reconciler bundle_store.go:1207-1214: -// execution fields set, outputs untouched and READY) is the reaching case; -// the phantom trio was the reviewer-recorded twin-side defect (REPAIR-20c). -// Genuine per-output failures persist a own FailureCode (failPhase -// bundle_store.go:989-995, sole per-output failure writer) or carry an -// unhealthy state, and both keep projecting. func fr20AssertHealthyOutputsPublishNoFailureTrio(t *testing.T, shapeName, transport string, durable publication.BundleExecution, body map[string]any) { t.Helper() outputs, ok := body["outputs"].([]any) @@ -671,16 +560,6 @@ const ( fr20GenuinePhase = "VALIDATING" ) -// TestFR20GenuineOutputFailureProjectsDespiteHealthyExecution is the REST -// twin of the resolver package's suppression-mutant guard for REPAIR-20c. -// A genuine per-output failure stamped exactly as the sole production -// per-output failure writer persists it (failPhase bundle_store.go:989-995: -// the output's OWN state plus persisted FailureCode/FailureRetryable/ -// FailurePhase) must still project the full trio plus phase on the generated -// strict-stack body even though the execution page itself is healthy: (a) -// output FAILED while the execution is PUBLISHED with no execution-level -// error, (b) a READY output carrying a own persisted failure code (shape-6 -// direction). A "suppress all errors on ready/healthy" fix fails this test. func TestFR20GenuineOutputFailureProjectsDespiteHealthyExecution(t *testing.T) { records := []struct { name string diff --git a/internal/server/recipe_execution_malformed_persisted_scope_test.go b/internal/server/recipe_execution_malformed_persisted_scope_test.go index c4fa22124..64e660a84 100644 --- a/internal/server/recipe_execution_malformed_persisted_scope_test.go +++ b/internal/server/recipe_execution_malformed_persisted_scope_test.go @@ -14,40 +14,6 @@ import ( "github.com/calypr/loom/internal/dataframe/publication" ) -// Durable R4 control for the production REST execution reader. -// -// Reproduction. A persisted execution declares auth_resource_paths -// ["p1","foreign\x00secret"]. A project member granted only p1 was served the -// execution metadata over GET /api/v1/dataframe/recipe-executions/:id, because -// normalizeAuthResourcePathList drops the malformed non-blank entry and the -// surviving path covered the truncated declaration. -// ReadScope.CoversExecutionPaths now refuses a declaration holding any -// malformed non-blank entry, so the handler's restricted-mode Covers gate -// (openapi_recipe_execution.go) conceals it exactly like a missing id. -// -// Requests enter through the same generated Fiber stack the FB20 twin uses -// (twinHTTPStack/twinGetExecution); only the durable catalog behind -// HTTPRoutes.releases and the catalog lister behind the authentic -// authscope.ScopeResolver are substituted. The Covers gate is reached through -// the authentic resolver call (no recipeExecutionReadScope seam override): -// the resolver returns Restricted for this caller (non-nil principal with a -// held path) and echoes back the caller's own clean grant, so only the -// declaration-side refusal can deny the malformed read. -// -// Contract pinned here: -// - malformed persisted scope -> the same concealed 404 envelope as a -// genuinely missing id, byte-identical, with zero metadata disclosure and -// zero leak of the malformed bytes; -// - permitted normal execution -> still 200 with full metadata (no -// over-denial); -// - legitimate blank declarations -> unchanged pre-existing policy (blank -// padding around real paths stays absent; a blank-only declaration is not -// a resolved authorization); -// - membership/generation/authorization behavior -> non-member 404 without -// catalog consultation, resolver path denial 404, anonymous disclosure of -// legitimate executions, Fence-less bearer header 503, unauthenticated -// 401, store/catalog faults retryable BACKEND_UNAVAILABLE 503. - const ( mscHomeProject = "P" mscGeneration = "g-msc" @@ -66,9 +32,6 @@ const ( mscSecretFingerprint = "secret" ) -// mscProbe is a generation-aware catalog lister for the authentic resolver. -// It keeps the FB20 twin's twinScopeProbe untouched while recording the exact -// lookups so gate ordering stays observable. type mscProbe struct { pathsByProjectGeneration map[string][]string calls []catalog.AuthResourcePathOptions @@ -91,8 +54,6 @@ func (p *mscProbe) listExisting(_ context.Context, opts catalog.AuthResourcePath return append([]string(nil), p.pathsByProjectGeneration[mscKey(opts.Project, opts.DatasetGeneration)]...), nil } -// mscDeniedTokenClient is the Fence stand-in that rejects the bearer token, -// driving the shipped unauthenticated mapping without a live Fence. type mscDeniedTokenClient struct{} func (mscDeniedTokenClient) GetAllowedResources(context.Context, string, string, string) ([]string, error) { @@ -128,8 +89,6 @@ func mscStore() *twinExecutionStore { }} } -// mscFailingStore answers every fetch with a durable fault, the store twin of -// a transient Arango outage. type mscFailingStore struct { publication.BundleCatalog err error @@ -155,8 +114,6 @@ func mscMemberPrincipal() *authscope.Principal { } } -// mscAssertAuthorizedBody pins the full metadata disclosure for a permitted -// read so the concealment arms cannot pass by over-denial. func mscAssertAuthorizedBody(t *testing.T, fields map[string]any, id string) { t.Helper() want := map[string]string{ @@ -193,9 +150,6 @@ func mscAssertAuthorizedBody(t *testing.T, fields map[string]any, id string) { } } -// mscAssertConcealment extends twinAssertConcealment's leak checks with this -// file's own fixture constants, including the malformed declaration's secret -// substring, its NUL escape, and the row count. func mscAssertConcealment(t *testing.T, status int, body string) { t.Helper() if status != http.StatusNotFound || !strings.Contains(body, twinConcealedError) || @@ -209,13 +163,7 @@ func mscAssertConcealment(t *testing.T, status int, body string) { } } -// TestRESTRecipeExecutionConcealsMalformedPersistedScope is the required -// regression for acceptance criterion [2]. func TestRESTRecipeExecutionConcealsMalformedPersistedScope(t *testing.T) { - // Precondition: the shipped authorizer admits this member and the real - // resolver resolves this caller to a Restricted scope echoing the clean - // grant, so the Covers gate (not membership, not a resolver refusal) - // decides the malformed read. principal := mscMemberPrincipal() if err := authscope.AuthorizeProject(principal, mscHomeProject, false); err != nil { t.Fatalf("precondition: shipped authorizer must admit the project member, got %v", err) @@ -236,30 +184,21 @@ func TestRESTRecipeExecutionConcealsMalformedPersistedScope(t *testing.T) { routes := mscRoutesWith(probe, nil) app := twinHTTPStack(routes, principal) - // Arm 1 (the R4 reproduction): mixed valid-plus-malformed declaration. malformedStatus, malformedBody, _ := twinGetExecution(t, app, mscMalformedExecID) mscAssertConcealment(t, malformedStatus, malformedBody) - // Arm 2: indistinguishable from a genuinely missing id (the same generated - // legacy 404 envelope, byte for byte). missingStatus, missingBody, _ := twinGetExecution(t, app, mscMissingID) if malformedStatus != missingStatus || malformedBody != missingBody { t.Fatalf("malformed-scope execution must be indistinguishable from a missing id; got %d %q vs %d %q", malformedStatus, malformedBody, missingStatus, missingBody) } - // Arm 3: the permitted normal execution remains readable through the same - // handler, resolver and scope shape. cleanStatus, cleanBody, cleanFields := twinGetExecution(t, app, mscCleanExecID) if cleanStatus != http.StatusOK { t.Fatalf("R4 OVER-DENIAL: member granted p1 could not read the clean p1 execution: %d %s", cleanStatus, cleanBody) } mscAssertAuthorizedBody(t, cleanFields, mscCleanExecID) - // Arm 4: legitimate blank policy is the shipped one — blank padding around - // real paths stays absent (serves), and a declaration with no non-blank - // path is not a resolved authorization (restricted coverage refuses it, - // unchanged pre-repair behavior). blankStatus, blankBody, _ := twinGetExecution(t, app, mscBlankExecID) mscAssertConcealment(t, blankStatus, blankBody) paddingStatus, paddingBody, paddingFields := twinGetExecution(t, app, mscPaddingExecID) @@ -268,9 +207,6 @@ func TestRESTRecipeExecutionConcealsMalformedPersistedScope(t *testing.T) { } mscAssertAuthorizedBody(t, paddingFields, mscPaddingExecID) - // Gate ordering: every resolver-path request consulted the catalog for the - // execution's exact project+generation, and the store really fetched each - // id (no early synthetic not-found keyed on the malformed bytes). if len(probe.calls) == 0 { t.Fatal("the malformed read must travel the resolver path (catalog consulted), got zero calls") } @@ -283,13 +219,7 @@ func TestRESTRecipeExecutionConcealsMalformedPersistedScope(t *testing.T) { malformedStatus, cleanStatus, blankStatus, paddingStatus, missingStatus, malformedBody) } -// TestRESTRecipeExecutionMalformedScopePreservesAuthorizationArms pins the -// second half of acceptance criterion [2]: the pre-existing membership, -// generation, unauthenticated and backend-fault classification survive the -// fails-closed rule. func TestRESTRecipeExecutionMalformedScopePreservesAuthorizationArms(t *testing.T) { - // Non-member: refused by the membership gate before the resolver, and - // byte-identical to the malformed-scope concealment. outsider := &authscope.Principal{ Subject: "operator-foreign", Projects: []string{"FOREIGN"}, @@ -302,8 +232,6 @@ func TestRESTRecipeExecutionMalformedScopePreservesAuthorizationArms(t *testing. t.Fatalf("non-member denial must not consult the catalog; got %d calls: %v", len(probeOutsider.calls), probeOutsider.calls) } - // Restricted member missing the declared path: the authentic resolver - // refuses with ErrForbidden, which stays the unified concealed 404. missingPath := &authscope.Principal{ Subject: "operator-other", Projects: []string{mscHomeProject}, @@ -316,8 +244,6 @@ func TestRESTRecipeExecutionMalformedScopePreservesAuthorizationArms(t *testing. t.Fatal("resolver-path denial expected a catalog lookup, got zero calls") } - // Anonymous caller: the shipped unrestricted posture keeps serving - // legitimate executions. probeAnon := &mscProbe{pathsByProjectGeneration: mscKnownPaths()} anonStatus, anonBody, anonFields := twinGetExecution(t, twinHTTPStack(mscRoutesWith(probeAnon, nil), nil), mscCleanExecID) if anonStatus != http.StatusOK { @@ -325,8 +251,6 @@ func TestRESTRecipeExecutionMalformedScopePreservesAuthorizationArms(t *testing. } mscAssertAuthorizedBody(t, anonFields, mscCleanExecID) - // Bearer header without a configured Fence client: authorization-backend - // failure keeps the retryable 503 envelope, not the concealed 404. bearer := &authscope.Principal{ Subject: "operator-bearer", Projects: []string{mscHomeProject}, @@ -342,8 +266,6 @@ func TestRESTRecipeExecutionMalformedScopePreservesAuthorizationArms(t *testing. t.Fatalf("Fence-less bearer read must keep the retryable BACKEND_UNAVAILABLE envelope, got %s", bearerBody) } - // Rejected token through a wired client: the unauthenticated mapping stays - // 401 (with the concealed phrase body), not a coverage-based 404. rejected := &authscope.Principal{ Subject: "operator-rejected", Projects: []string{mscHomeProject}, @@ -356,8 +278,6 @@ func TestRESTRecipeExecutionMalformedScopePreservesAuthorizationArms(t *testing. t.Fatalf("rejected token must stay the 401 concealed-phrase envelope, got %d %s", rejectedStatus, rejectedBody) } - // Catalog lister fault: unclassified authorization-backend failure stays a - // retryable 503 even for the malformed-scope execution. probeFaulty := &mscProbe{pathsByProjectGeneration: mscKnownPaths(), fault: errors.New("arangodb connection reset")} faultyStatus, faultyBody, faultyFields := twinGetExecution(t, twinHTTPStack(mscRoutesWith(probeFaulty, nil), mscMemberPrincipal()), mscMalformedExecID) if faultyStatus != http.StatusServiceUnavailable { @@ -367,8 +287,6 @@ func TestRESTRecipeExecutionMalformedScopePreservesAuthorizationArms(t *testing. t.Fatalf("catalog fault must keep the retryable BACKEND_UNAVAILABLE envelope, got %s", faultyBody) } - // Durable store fault likewise stays 503 rather than collapsing into the - // concealed 404. storeFault := errors.New("arangodb connection reset") probeStore := &mscProbe{pathsByProjectGeneration: mscKnownPaths()} routesStore := mscRoutesWith(probeStore, nil) diff --git a/internal/server/recipe_execution_malformed_scope_header_probe_test.go b/internal/server/recipe_execution_malformed_scope_header_probe_test.go index 85a74b97b..9064c4257 100644 --- a/internal/server/recipe_execution_malformed_scope_header_probe_test.go +++ b/internal/server/recipe_execution_malformed_scope_header_probe_test.go @@ -12,24 +12,6 @@ import ( "github.com/gofiber/fiber/v3" ) -// Adversarial probe: header-indistinguishability of the malformed-scope -// concealment. -// -// The R4 repair makes the REST reader answer a persisted execution whose -// declaration carries a malformed non-blank path with the same concealed 404 a -// missing id gets. The question this probe attacks: can a caller turn that -// concealment into an oracle by varying request headers — sending an -// Authorization header, an X-Request-Id, duplicate or empty header values, or -// unusual content negotiation — so that the malformed-scope response becomes -// distinguishable from a genuinely missing execution (status, body, header -// set, or header values)? -// -// Contract pinned here: across every request-header variation, the -// malformed-scope response and the missing-id response stay byte-identical -// (status, exact body bytes, and the full header map), carry no Authorization -// or WWW-Authenticate material, and never echo a fragment of the malformed -// declaration. Backend-fault classification is likewise header-invariant. - func mscGetWithHeaders(t *testing.T, app *fiber.App, id string, headers [][2]string) *http.Response { t.Helper() request := httptest.NewRequest(http.MethodGet, twinRoutePrefix+id, nil) @@ -112,10 +94,6 @@ func TestRESTRecipeExecutionMalformedScopeIsHeaderIndistinguishable(t *testing.T } } -// TestRESTRecipeExecutionMalformedScopeFaultClassificationIsHeaderInvariant -// pins that the fault-vs-concealment split does not move when request headers -// move: an authorization-backend failure stays the retryable 503 even with a -// bearer header, and a clean malformed read stays the concealed 404. func TestRESTRecipeExecutionMalformedScopeFaultClassificationIsHeaderInvariant(t *testing.T) { bearer := &authscope.Principal{ Subject: "operator-bearer", @@ -133,18 +111,12 @@ func TestRESTRecipeExecutionMalformedScopeFaultClassificationIsHeaderInvariant(t probeFaulty := &mscProbe{pathsByProjectGeneration: mscKnownPaths(), fault: errors.New("arangodb connection reset")} appFaulty := twinHTTPStack(mscRoutesWith(probeFaulty, nil), mscMemberPrincipal()) - // The malformed-scope read reaches the resolver (the record exists), so its - // backend fault stays the retryable 503 under every request header — it - // must never collapse into the concealed 404 depending on headers. for _, variation := range mscHeaderVariations { faultyStatus, faultyBody, _ := mscReadResponse(t, mscGetWithHeaders(t, appFaulty, mscMalformedExecID, variation.headers)) if faultyStatus != http.StatusServiceUnavailable || !strings.Contains(faultyBody, "BACKEND_UNAVAILABLE") { t.Fatalf("resolver-path fault must stay the retryable 503 under headers %v, got %d %s", variation.headers, faultyStatus, faultyBody) } } - // A missing id short-circuits at the durable store before the resolver - // (pre-existing ordering), so a resolver fault is invisible to it under - // every header variation: the concealed 404 stays the concealed 404. for _, variation := range mscHeaderVariations { missingStatus, missingBody, _ := mscReadResponse(t, mscGetWithHeaders(t, appFaulty, mscMissingID, variation.headers)) if missingStatus != http.StatusNotFound || !strings.Contains(missingBody, twinConcealedError) { diff --git a/internal/server/recipe_execution_status_twin_e2e_test.go b/internal/server/recipe_execution_status_twin_e2e_test.go index 3ec289ce3..b77acbf1a 100644 --- a/internal/server/recipe_execution_status_twin_e2e_test.go +++ b/internal/server/recipe_execution_status_twin_e2e_test.go @@ -19,39 +19,6 @@ import ( "github.com/gofiber/fiber/v3" ) -// This file is the REST twin of the GraphQL authorization control in -// internal/api/graphql/graph/resolver/recipe_execution_reader_authz_e2e_test.go. -// -// History. The characterization phase recorded the shipped defect: when -// HTTPRoutes.scopes was nil the handler served foreign-project execution -// metadata to any principal with no project-membership assertion, and even -// with scopes wired, restricted mode accepted on a held auth-path string -// alone. Round 34 (Repair-6/F-B20) closed the twin: the handler now unconditionally -// calls authscope.AuthorizeProject with the execution's project before the -// scopes-conditional resolver block, and restricted mode uses the shared -// normalized path-set rule (ReadScope.CoversExecutionPaths). -// -// The archived shipped-baseline verdict survives verbatim in each case's -// FB20-TWIN REGRESSION message, so an accidental revert re-derives the -// original defect wording. A principal-less (anonymous) request keeps its -// shipped disclosure posture by design (the shipped authorizer allows a nil -// principal); this is documented, not asserted as a defect. -// -// What "real handler stack" owns here. Requests enter through the generated -// Fiber router registration (generated/loomapi/api.gen.go -// GET /api/v1/dataframe/recipe-executions/:id), the generated strict adapter -// (loomapi.NewStrictHandler + RegisterHandlersWithOptions, the same mounting -// done by internal/server/routes.go), routes.GetRecipeExecution and -// routes.recipeExecution. Only the collaborators behind HTTPRoutes.releases -// (publication.BundleCatalog) and HTTPRoutes.scopes (*authscope.ScopeResolver, -// driven through its declared ScopeResolverConfig seam, and — for case T-C's -// drifted-result arms only — through the resolver-consultation variable seam -// documented on recipeExecutionReadScope) are substituted. -// Principal injection mirrors the production middleware seam exactly -// (authscope.ContextWithPrincipal); the authenticator that turns headers into -// principals is out of scope, so nothing here claims the shipped authenticator -// produces per-project principals — it demonstrates handler behaviour once one -// is present, which is the multi-tenant deployment shape. const ( twinForeignProject = "Q" twinForeignGeneration = "gQ" @@ -80,9 +47,6 @@ const ( twinRoutePrefix = "/api/v1/dataframe/recipe-executions/" ) -// twinQuotedFacts are the exact repaired production fragments this control is -// built on. Any disappearance is drift and must fail STEP 0 rather than let a -// stale guarantee pass. var twinQuotedFacts = []struct { name string source string @@ -127,10 +91,6 @@ var twinQuotedFacts = []struct { }, } -// twinExecutionStore is the smallest fake for publication.BundleCatalog, the -// same embedding discipline as the existing recipeExecutionCatalog fake in -// openapi_recipe_execution_test.go. Unknown ids yield the zero execution the -// handler treats as missing. type twinExecutionStore struct { publication.BundleCatalog executions map[string]publication.BundleExecution @@ -145,10 +105,6 @@ func (s *twinExecutionStore) GetExecution(_ context.Context, id string) (publica return publication.BundleExecution{}, nil } -// twinScopeProbe drives the declared authscope.ScopeResolverConfig seam -// ListExistingAuthResourcePaths so the production resolver can resolve -// restricted scopes without a live Fence or catalog, keyed by the project the -// resolver actually asked about. The call log also proves gate ordering. type twinScopeProbe struct { existingByProject map[string][]string calls []catalog.AuthResourcePathOptions @@ -194,9 +150,6 @@ func twinLocalBundle() publication.BundleExecution { } } -// twinHTTPStack mounts the generated router the same way internal/server/ -// routes.go does, and injects the principal through the same fiber seam the -// production authentication middleware uses. func twinHTTPStack(routes *HTTPRoutes, principal *authscope.Principal) *fiber.App { app := fiber.New() if principal != nil { @@ -234,9 +187,6 @@ func twinGetExecution(t *testing.T, app *fiber.App, id string) (int, string, map return response.StatusCode, body, fields } -// twinRequireRepairedGuards is STEP 0. It pins the repaired fragments and -// forbids both archived pre-fix shapes: the resolver-conditional-only -// authorization block and the absence of any membership assertion. func twinRequireRepairedGuards(t *testing.T) { t.Helper() raw, err := os.ReadFile(twinHandlerFile) @@ -308,9 +258,6 @@ func twinAssertAuthorizedForeignBody(t *testing.T, fields map[string]any) { } } -// twinAssertConcealment pins the contract that a refused read is -// indistinguishable from a missing execution: 404, the missing-execution -// phrase, and no foreign project/generation/digest/row-count leak. func twinAssertConcealment(t *testing.T, status int, body string) { t.Helper() if status != http.StatusNotFound || !strings.Contains(body, twinConcealedError) || @@ -362,11 +309,6 @@ func twinWiredRoutes() (*HTTPRoutes, *twinExecutionStore, *twinScopeProbe) { return &HTTPRoutes{releases: store, scopes: resolver}, store, probe } -// Case T-A: the archived primary defect arm. HTTPRoutes.scopes is nil -// (routes.go leaves it nil whenever no *authscope.ScopeResolver is wired) and -// the request carries a principal whose Projects hold only "P". Pre-fix this -// returned 200 with full foreign metadata; the repaired handler must answer -// with the exact concealed missing-execution envelope and no metadata leak. func TestFB20RESTTwinNilScopesRefusesForeignPrincipalOverHTTP(t *testing.T) { twinRequireRepairedGuards(t) store := twinForeignStore() @@ -402,10 +344,6 @@ func TestFB20RESTTwinNilScopesRefusesForeignPrincipalOverHTTP(t *testing.T) { } } -// Case T-B1: the archived acceptance arm. A principal holds the execution's -// auth-path string and its equal-count scope resolves against the FOREIGN -// project/generation, while Projects excludes Q. Pre-fix the handler accepted -// it; now membership refuses before any resolver consultation. func TestFB20RESTTwinWiredHeldPathAloneNoLongerAdmitsNonMember(t *testing.T) { twinRequireRepairedGuards(t) routes, _, probe := twinWiredRoutes() @@ -443,9 +381,6 @@ func TestFB20RESTTwinWiredHeldPathAloneNoLongerAdmitsNonMember(t *testing.T) { "held-path non-member refused with status="+twinItoa(status)+" and zero new catalog calls beyond the precondition probe") } -// Case T-B2: uniform concealment with proven gate ordering. The missing-path -// member arrives via the resolver (catalog consulted); the non-member arrives -// via membership (catalog untouched) with the identical envelope. func TestFB20RESTTwinDeniedMemberAndNonMemberAreIndistinguishable(t *testing.T) { twinRequireRepairedGuards(t) memberPrincipal := &authscope.Principal{ @@ -460,18 +395,6 @@ func TestFB20RESTTwinDeniedMemberAndNonMemberAreIndistinguishable(t *testing.T) twinAssertAuthorizedForeignBody(t, memberFields) t.Fatalf("FB20-TWIN REGRESSION: member without the execution path read the foreign execution: %s", memberBody) } - // F-B32 unified concealment: a restricted caller whose read scope lacks one - // of the execution's auth_resource_paths (resolver returns ErrForbidden) is - // a nonexistent-or-unauthorized-to-see execution and must be 404, the exact - // twin of (a) the GraphQL reader recipe_execution_reader.go:39-41 -> - // recipeExecutionLookupError:94 -> authpolicy Classify(ErrForbidden, - // OperationConcealedLookup)=CodeRecipeExecutionNotFound (policy.go:40-41,47-52), - // (b) REST's own sibling branch openapi_recipe_execution.go:40-42 which - // already conceals the uncovered CoversExecutionPaths case to 404, and (c) - // the "recipe execution not found" body the 403 already emitted. A - // status-only 403-vs-404 split against those three sources is an existence - // oracle and is not permitted. Other authorization outcomes are unaffected: - // unauthenticated stays 401, authorization-backend failure stays 503. if memberStatus != http.StatusNotFound || !strings.Contains(memberBody, twinConcealedError) || strings.Contains(memberBody, twinForeignGeneration) || strings.Contains(memberBody, twinRecipeDigest) || strings.Contains(memberBody, "7373737") { @@ -504,41 +427,12 @@ func TestFB20RESTTwinDeniedMemberAndNonMemberAreIndistinguishable(t *testing.T) t.Fatalf("FB20-TWIN REGRESSION: the anonymous (single-operator) posture must keep working; got %d %s", anonStatus, anonBody) } twinAssertAuthorizedForeignBody(t, anonFields) - // A principal-less caller resolves unrestricted (documented dev posture), - // so the resolver short-circuits before the catalog seam by design. t.Logf("FB20-TWIN CONTROL | anonymous-unrestricted-shortcut | catalog calls=%d (0 expected for nil principal)", len(probeU.calls)) twinVerdict(t, "case-T-B2", "REPAIRED (uniform concealment; documented anonymous posture)", "member-missing path refused via resolver ("+twinItoa(memberStatus)+"), non-member refused via membership ("+twinItoa(nonMemberStatus)+ ") with byte-identical envelopes; nil-principal disclosure preserved by design; body="+nonMemberBody) } -// Case T-C (F-3, reviewer ses_e66757becffe8uCNimGXbEeIfc): the Covers subset -// loop must be demonstrable as the sole gate preventing disclosure. -// CoversExecutionPaths is the route's defense-in-depth gate against resolver -// drift, and the twins' call pattern never lets it discriminate on its own: -// ResolveReadScopeForGeneration echoes the requested declared set back on a -// Restricted success (scope_resolver.go:200-205), so with requested equal to -// declared the Covers check is always-pass. This case therefore feeds the -// handler an otherwise fully authorized read (membership passes; the scope -// consultation resolves successfully in Restricted mode) whose drifted result -// is a proper subset of what the execution declares, injected at the -// resolver-consultation seam recipeExecutionReadScope. The seam's default is -// the authentic ResolveReadScopeForGeneration call, and the production -// resolver can only DENY this read (proven by the precondition below), so the -// concealed 404 below cannot come from a resolver denial, 401, or the -// membership gate — it can only come from the Covers subset loop. Assertions -// are behavioral only; no source text is pinned beyond STEP 0. -// -// (a) scope ⊇ declared (equal set) -> 200 with the full foreign metadata. -// (b) scope ⊊ declared (p-extra gone) -> concealed 404 whose status and body -// -// are byte-identical to the missing-execution envelope. -// -// (c) Run-proven mutation control: the reviewer's Mutant C overlay (subset -// -// membership loop removed from CoversExecutionPaths, /tmp copy of -// scope_resolver.go via go test -overlay, repo authscope untouched) makes -// arm (b) serve 200 and this test fail. Evidence: /tmp/opencode/fb16/. func TestFB20RESTTwinCoversSubsetGateIsSoleDisclosureGate(t *testing.T) { twinRequireRepairedGuards(t) @@ -569,7 +463,6 @@ func TestFB20RESTTwinCoversSubsetGateIsSoleDisclosureGate(t *testing.T) { defer func() { recipeExecutionReadScope = original }() app := twinHTTPStack(routes, principal) - // Arm (a): Restricted success whose scope covers the whole declared set. recipeExecutionReadScope = func(context.Context, *authscope.ScopeResolver, *authscope.Principal, string, string, []string) (authscope.ReadScope, error) { return authscope.ReadScope{Mode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{twinCallerPath, twinCoversExtraPath}}, nil } @@ -592,9 +485,6 @@ func TestFB20RESTTwinCoversSubsetGateIsSoleDisclosureGate(t *testing.T) { t.Fatalf("case T-C(a): output[0].rowCount = %v, want 7373737", output["rowCount"]) } - // Arm (b): the only change is the proper-subset scope (p-extra dropped). - // Membership still passes and the consultation still "succeeds", so a - // refusal here can only be the Covers subset loop. recipeExecutionReadScope = func(context.Context, *authscope.ScopeResolver, *authscope.Principal, string, string, []string) (authscope.ReadScope, error) { return authscope.ReadScope{Mode: authscope.ReadScopeRestricted, AuthResourcePaths: []string{twinCallerPath}}, nil } diff --git a/internal/server/recipe_execution_twin_authz_repair_test.go b/internal/server/recipe_execution_twin_authz_repair_test.go index a8a50f274..1eddb7edf 100644 --- a/internal/server/recipe_execution_twin_authz_repair_test.go +++ b/internal/server/recipe_execution_twin_authz_repair_test.go @@ -10,13 +10,6 @@ import ( "github.com/calypr/loom/internal/dataframe/publication" ) -// TestRecipeExecutionTwinRefusesForeignProjectPrincipal is the durable F-B20 -// REST-sibling regression check. An authenticated principal whose project -// allowlist excludes the execution's project must receive the same concealed -// not-found envelope as a missing id, even when no scope resolver is wired: -// the project gate is unconditional, matching the dataframe and explorer -// capability authorizers. The anonymous (principal-less) deployment posture -// remains disclosure-by-design and the same-project member path keeps working. func TestRecipeExecutionTwinRefusesForeignProjectPrincipal(t *testing.T) { foreign := publication.BundleExecution{ ID: "execution-foreign", diff --git a/internal/server/recipe_project_keying_test.go b/internal/server/recipe_project_keying_test.go index 4b3b03a52..69a0df8a1 100644 --- a/internal/server/recipe_project_keying_test.go +++ b/internal/server/recipe_project_keying_test.go @@ -12,11 +12,6 @@ import ( "github.com/calypr/loom/internal/projectid" ) -// Padded spelling of the live RED project (verification/publication-bundle/ -// evidence/recipe-identity-keying). The keying contract is surrounding-space -// trimming only: the registry key keeps the raw trimmed spelling, so a -// boundary canonicalization (program-project rewrite) would break these -// assertions and is deliberately not the contract. const ( keyingPaddedProject = " loom_dev_verify_zzz-padkey-7k " keyingCleanProject = "loom_dev_verify_zzz-padkey-7k" @@ -28,9 +23,6 @@ const ( keyingBundlePatients = `{"recipeSchemaVersion":1,"name":"patients","translationVersion":"1","fragments":{"fragments":{}},"outputs":[]}` ) -// keyingRevisionStore mirrors a key-exact store: it keys solely by the -// project argument it receives, so padded and clean spellings become two -// revision sets unless the boundary normalizes first. type keyingRevisionStore struct { registeredProjects []string listedProjects []string @@ -83,9 +75,6 @@ func assertKeyedOnce(t *testing.T, keys []string, want string, site string) { } } -// The live RED: register under the padded spelling and query with the clean -// spelling. Both spellings must address ONE revision set, keyed clean at -// every register/Get/List consult. func TestRecipeRevisionResolversAddressOneProjectKey(t *testing.T) { store := &keyingRevisionStore{} resolver := keyingResolver(store) @@ -136,7 +125,6 @@ func TestRecipeRevisionResolversAddressOneProjectKey(t *testing.T) { keys := store.gotProjects[len(store.gotProjects)-1:] assertKeyedOnce(t, keys, keyingCleanProject, "revision padded") - // Valid clean path still works and folds onto the same key. second, err := resolver.Mutation().RegisterDataframeRecipeRevision(ctx, model.RegisterDataframeRecipeRevisionInput{ ProjectID: keyingCleanProject, Recipe: []byte(keyingBundlePatients), @@ -161,10 +149,6 @@ func TestRecipeRevisionResolversAddressOneProjectKey(t *testing.T) { } } -// The authorizer must hand every downstream consumer (scope resolution, -// SelectionProject derivation, engine, registry) one key: -// bindings.Project normalized to the trimmed spelling, while the guard keeps -// rejecting blank-after-trim with the typed PROJECT_REQUIRED error. func TestRecipeAuthorizationWritesBackTrimmedProject(t *testing.T) { authorizer := recipeAuthorization{} for _, leg := range []struct { @@ -205,10 +189,6 @@ func (l *keyingPathLister) list(_ context.Context, options catalog.AuthResourceP return l.paths, nil } -// Scope resolution receives the same spelling the registry keys off: the -// padded and clean inputs must produce an identical store consult. -// (listExistingPaths canonicalizes internally, so byte equality there is a -// contract lock, not a mutation detector; the detector is bindings.Project.) func TestRecipeAuthorizationScopeResolutionSeesNormalizedProject(t *testing.T) { principal := &authscope.Principal{Subject: "keying-tester", AuthResourcePaths: []string{"myproj/padata"}} ctx := authscope.ContextWithPrincipal(context.Background(), principal) diff --git a/internal/server/recipe_publication.go b/internal/server/recipe_publication.go index b6c2b593a..86254c226 100644 --- a/internal/server/recipe_publication.go +++ b/internal/server/recipe_publication.go @@ -23,10 +23,6 @@ func recipeScopeDigest(bindings recipe.RuntimeBindings) string { return hex.EncodeToString(hash[:]) } -// recipeOutputLogicalColumns is the one conversion point from the finalized -// compiler schema to the backend-neutral publication schema. Publication must -// not reconstruct nested names from semantic recipe nodes because those names -// are finalized by physical lowering. func recipeOutputLogicalColumns(plan dataframeexecution.Resolved, outputName string) []publication.LogicalColumn { for _, output := range plan.Compiled.Outputs { if output.Name != outputName { @@ -40,9 +36,6 @@ func recipeOutputLogicalColumns(plan dataframeexecution.Resolved, outputName str if kind == "" { kind = "string" } - // Group row identity is a structured tuple in the compiler. The - // publication row key is its canonical JSON text so ClickHouse can - // order and page by the hidden identity column. if output.RowGrain == spec.RowGrainGroups && kind == "object" { kind = "string" } diff --git a/internal/server/release_verifier.go b/internal/server/release_verifier.go index 7fce5e5bc..5e3703ed8 100644 --- a/internal/server/release_verifier.go +++ b/internal/server/release_verifier.go @@ -9,9 +9,6 @@ import ( "github.com/calypr/loom/internal/dataset" ) -// publicationVerificationStore is a narrow integration adapter over durable -// execution metadata. Publication owns the record shape; release activation -// consumes only its frozen exact-selector contract. type publicationVerificationStore struct { executions publication.ExactExecutionCatalog } @@ -23,7 +20,6 @@ func (s publicationVerificationStore) VerifyPublication(ctx context.Context, pro execution, output, err := s.executions.FindExecutionBySelector(ctx, project, generation, selector) if err != nil { if errors.Is(err, publication.ErrBundleNotFound) { - // Deterministic absence: a contract miss, not a store fault. return dataset.PublicationVerification{}, fmt.Errorf("%w: %w", dataset.ErrPublicationRecordMissing, err) } return dataset.PublicationVerification{}, err diff --git a/internal/server/release_verifier_fault_mapping_test.go b/internal/server/release_verifier_fault_mapping_test.go index b58148246..39a497de5 100644 --- a/internal/server/release_verifier_fault_mapping_test.go +++ b/internal/server/release_verifier_fault_mapping_test.go @@ -1,10 +1,5 @@ package server -// Durable control (F-B14 revision, round 41): the production verifier adapter -// must map a deterministic not-found to dataset.ErrPublicationRecordMissing so -// release activation can classify it as a non-retryable contract miss, while -// every other catalog error stays a raw store fault for retryable handling. - import ( "context" "errors" diff --git a/internal/server/repository_activation_conflict_dev_test.go b/internal/server/repository_activation_conflict_dev_test.go index a585dd01a..f31318854 100644 --- a/internal/server/repository_activation_conflict_dev_test.go +++ b/internal/server/repository_activation_conflict_dev_test.go @@ -12,13 +12,6 @@ import ( publicationcontract "github.com/calypr/loom/internal/dataset" ) -// Dev-only one-shot fault for the REPOSITORY publish activation leg, the -// parallel of LOOM_DEV_ACTIVATION_CONFLICT_ONCE (authoring/revision leg). -// These tests pin both the no-auth-only wiring of the knob and the consumption -// seam: the armed fault must make the real release CAS reject by corrupting -// exactly the expected-revision CAS input once, never by returning a synthetic -// error that hides a successful activation. - func TestDevRepositoryActivationConflictOnceIsNoAuthOnly(t *testing.T) { t.Setenv("LOOM_DEV_REPOSITORY_ACTIVATION_CONFLICT_ONCE", "true") @@ -74,14 +67,6 @@ func (r *devFaultReleaseRepository) ReadActiveRelease(_ context.Context, _ strin return *r.active, nil } -// CompareAndSwapActivateRelease mirrors the shipped store guard in -// internal/dataset/arango/release_store.go: activateReleaseAQL compares the -// pointer revision with FILTER currentRevision == @expected_revision OR -// alreadyActive, and an emptied result becomes -// dataset.ErrReleaseActivationConflict (release_store.go:99-100). The -// comparison (and the pointer mutation on success) belongs to this fake -// store, not to the fault: the fault's only visible act is the CAS input it -// feeds in. func (r *devFaultReleaseRepository) CompareAndSwapActivateRelease(_ context.Context, release publicationcontract.ProjectRelease, expectedRevision int64) (publicationcontract.ActiveRelease, error) { r.casComparisons++ current := int64(0) @@ -129,9 +114,6 @@ func newDevFaultReleaseService(repository publicationcontract.ReleaseRepository) } } -// seedActiveRelease establishes a prior active release so the CAS guard is -// the discriminator: with a pointer at revision 7, a correct expected value of -// 7 wins and a corrupted 8 must lose through the store's own comparison. func seedActiveRelease(t *testing.T, repository *devFaultReleaseRepository, project string, revision int64) publicationcontract.ProjectRelease { t.Helper() release := publicationcontract.ProjectRelease{ID: "release_prior", Project: project, GitCommit: "generation-prior", Generation: "generation-prior", CreatedAt: time.Now().UTC()} @@ -159,18 +141,13 @@ func TestDevRepositoryActivationConflictOnceTripsTheRealStoreCASOnce(t *testing. if repository.casComparisons != 1 { t.Fatalf("armed activation performed %d CAS attempts, want exactly one", repository.casComparisons) } - // The fault corrupted the CAS input: the store saw expected=8 while its - // pointer still read 7, so the store's guard rejected. if len(repository.casExpectedSeen) != 1 || repository.casExpectedSeen[0] != 8 || repository.casCurrentSeen[0] != 7 { t.Fatalf("fault did not corrupt exactly the CAS input once: expected=%v current=%v", repository.casExpectedSeen, repository.casCurrentSeen) } - // A rejected CAS moves nothing: the prior active release pointer stands. if repository.active == nil || repository.active.Release.ID != prior.ID || repository.active.Revision != 7 { t.Fatalf("lost CAS moved the prior active pointer: %#v", repository.active) } - // Consumption is one-shot: the next activation feeds the true CAS input and - // the store's own guard lets it win, converging the pointer. if err := activate(context.Background(), project, generation, nil); err != nil { t.Fatalf("post-fault activation error = %v, want success after the one-shot was consumed", err) } @@ -206,11 +183,6 @@ func TestDevRepositoryActivationConflictUnarmedNeverFaults(t *testing.T) { } } -// Reviewer finding C-F1 (adopted, root-caused): the one-shot arm was consumed -// only inside the successful-Active branch, so an armed request with no active -// pointer left the fault live to silently hijack a LATER publish (proven on a -// copy: request 1 unconsumed, request 2 corrupted from 7 to 8). Consumption -// must belong to the request that arrived, not to the branch that can corrupt. func TestDevRepositoryActivationConflictOnceCannotJumpToLaterRequest(t *testing.T) { const project, generation = "project-a", "generation-a" repository := &devFaultReleaseRepository{} diff --git a/internal/server/repository_publish_wiring_pg18_test.go b/internal/server/repository_publish_wiring_pg18_test.go new file mode 100644 index 000000000..8bd3b6617 --- /dev/null +++ b/internal/server/repository_publish_wiring_pg18_test.go @@ -0,0 +1,430 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/calypr/loom/internal/backends/postgres/control" +) + +const pgPubCommitHeader = "X-Loom-Source-Commit" + +func TestPostgresRuntimeRepositoryPublishRouteActivatesThroughFinalFlip(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL repository-publish route integration tests") + } + clickhouseURL := strings.TrimSpace(os.Getenv("LOOM_CLICKHOUSE_URL")) + if clickhouseURL == "" { + t.Skip("set LOOM_CLICKHOUSE_URL to run the PostgreSQL repository-publish route integration tests (materialization is real)") + } + fixture := newPgCompFixture(t) + fixture.migrate(t) + fixture.load(t, pgCompProject, pgCompGenA, map[string][]string{ + "Patient.ndjson": pgCompPatientLines(t, "patient-publish-01", "patient-publish-02"), + }) + fixture.activate(t, pgCompProject, pgCompGenA) + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + bootConfig := pgCompBootConfig(t, fixture, "127.0.0.1:0") + bootConfig.Server.ClickHouse = ClickHouseConfig{ + Enabled: true, + URL: clickhouseURL, + Database: os.Getenv("LOOM_CLICKHOUSE_DATABASE"), + Username: os.Getenv("LOOM_CLICKHOUSE_USERNAME"), + Password: os.Getenv("LOOM_CLICKHOUSE_PASSWORD"), + } + server, closeRuntime, err := composePostgreSQLFullRuntime(ctx, bootConfig, logger) + if err != nil { + t.Fatalf("compose ClickHouse-enabled full runtime: %s", fixture.scrub(err)) + } + defer func() { + if err := closeRuntime(); err != nil { + t.Errorf("close ClickHouse-enabled full runtime: %s", fixture.scrub(err)) + } + }() + app := server.App() + + path := "/api/v1/projects/" + pgCompProject + "/generations/" + pgCompGenA + "/explorer-config" + deploy := requestJSONWithHeaders(t, app, http.MethodPost, path, string(baselineExplorerWorkspaceV2()), + map[string]string{pgPubCommitHeader: "commit-publish-01"}) + pgCompAssertNoLaneLeak(t, fixture, "repository publish deploy", deploy.Body) + if deploy.StatusCode != http.StatusOK { + t.Fatalf("repository publish deploy through the final activation flip returned %d, want 200: %s", + deploy.StatusCode, fixture.scrub(deploy.Body)) + } + var result struct { + Project string `json:"project"` + Generation string `json:"generation"` + ExplorerID string `json:"explorerId"` + ReceiptID string `json:"receiptId"` + RevisionID string `json:"revisionId"` + ExecutionID string `json:"executionId"` + Activated *bool `json:"activated"` + } + if err := json.Unmarshal([]byte(deploy.Body), &result); err != nil { + t.Fatalf("decode deploy body: %v\n%s", err, fixture.scrub(deploy.Body)) + } + if result.Project != pgCompProject || result.Generation != pgCompGenA || result.ExplorerID != "default" { + t.Fatalf("deploy identity = %+v, want %s/%s/default", result, pgCompProject, pgCompGenA) + } + if result.ReceiptID == "" || result.RevisionID == "" || result.ExecutionID == "" { + t.Fatalf("deploy lost the publication identity: %+v", result) + } + if result.Activated == nil || !*result.Activated { + t.Fatalf("deploy did not report activated=true: %s", fixture.scrub(deploy.Body)) + } + + statusBefore, activatedAtBefore, pointerBefore := pgPubWitness(t, fixture, result.RevisionID) + if statusBefore != "ACTIVE" || activatedAtBefore.IsZero() { + t.Fatalf("witness after deploy: revision %s status=%s activated_at=%v, want ACTIVE/non-zero", + result.RevisionID, statusBefore, activatedAtBefore) + } + if pointerBefore != result.RevisionID { + t.Fatalf("witness after deploy: owner active_revision_id = %q, want the deployed revision %q", + pointerBefore, result.RevisionID) + } + + replay := requestJSONWithHeaders(t, app, http.MethodPost, path, string(baselineExplorerWorkspaceV2()), + map[string]string{pgPubCommitHeader: "commit-publish-01"}) + if replay.StatusCode != http.StatusOK { + t.Fatalf("replayed deploy (battery retry after an ambiguous response) returned %d, want 200: %s", + replay.StatusCode, fixture.scrub(replay.Body)) + } + var replayed struct { + RevisionID string `json:"revisionId"` + Activated *bool `json:"activated"` + } + if err := json.Unmarshal([]byte(replay.Body), &replayed); err != nil { + t.Fatalf("decode replay body: %v\n%s", err, fixture.scrub(replay.Body)) + } + if replayed.RevisionID != result.RevisionID || replayed.Activated == nil || !*replayed.Activated { + t.Fatalf("replay must be the same activated revision: got %+v, want revisionId %q activated=true", + replayed, result.RevisionID) + } + statusAfter, activatedAtAfter, pointerAfter := pgPubWitness(t, fixture, result.RevisionID) + if statusAfter != "ACTIVE" || pointerAfter != result.RevisionID { + t.Fatalf("replay moved activation state: status=%s pointer=%q", statusAfter, pointerAfter) + } + if !activatedAtBefore.Equal(activatedAtAfter) { + t.Fatalf("replay restamped activated_at: %v -> %v (the no-op replay law forbids it)", + activatedAtBefore, activatedAtAfter) + } + + reload := requestJSON(t, app, http.MethodGet, "/api/v1/projects/"+pgCompProject+"/explorers/default", "") + if reload.StatusCode != http.StatusOK || !strings.Contains(reload.Body, "\"Patient ID\"") { + t.Fatalf("viewer state must serve the activated revision, got %d %s", + reload.StatusCode, fixture.scrub(reload.Body)) + } +} + +func pgPubWitness(t *testing.T, fixture *pgCompFixture, revisionID string) (string, time.Time, string) { + t.Helper() + pool, err := pgxpool.New(context.Background(), fixture.dsn) + if err != nil { + t.Fatalf("connect witness pool: %s", fixture.scrub(err)) + } + defer pool.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + ctlSchema := fixture.schemas[control.SeriesExplorerControl] + if !pgCompSchemaPattern.MatchString(ctlSchema) { + t.Fatalf("witness refuses a control schema %q outside the owned namespace", ctlSchema) + } + ctl := `"` + ctlSchema + `"` + var status string + var activatedAt time.Time + if err := pool.QueryRow(ctx, + `SELECT status, activated_at FROM `+ctl+`.explorer_revisions WHERE id = $1`, revisionID). + Scan(&status, &activatedAt); err != nil { + t.Fatalf("witness revision %s: %s", revisionID, fixture.scrub(err)) + } + var pointer string + if err := pool.QueryRow(ctx, + `SELECT active_revision_id FROM `+ctl+`.explorers WHERE project = $1 AND explorer_id = 'default'`, pgCompProject). + Scan(&pointer); err != nil { + t.Fatalf("witness owner pointer for %s: %s", pgCompProject, fixture.scrub(err)) + } + return status, activatedAt, pointer +} + +func TestPostgresRuntimeRepositoryPublishRetryAfterActivationFailureRevives(t *testing.T) { + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL repository-publish route integration tests") + } + clickhouseURL := strings.TrimSpace(os.Getenv("LOOM_CLICKHOUSE_URL")) + if clickhouseURL == "" { + t.Skip("set LOOM_CLICKHOUSE_URL to run the PostgreSQL repository-publish route integration tests (materialization is real)") + } + fixture := newPgCompFixture(t) + fixture.migrate(t) + fixture.load(t, pgCompProject, pgCompGenA, map[string][]string{ + "Patient.ndjson": pgCompPatientLines(t, "patient-publish-11", "patient-publish-12"), + }) + fixture.activate(t, pgCompProject, pgCompGenA) + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + bootConfig := pgCompBootConfig(t, fixture, "127.0.0.1:0") + bootConfig.Server.ClickHouse = ClickHouseConfig{ + Enabled: true, + URL: clickhouseURL, + Database: os.Getenv("LOOM_CLICKHOUSE_DATABASE"), + Username: os.Getenv("LOOM_CLICKHOUSE_USERNAME"), + Password: os.Getenv("LOOM_CLICKHOUSE_PASSWORD"), + } + pathA := "/api/v1/projects/" + pgCompProject + "/generations/" + pgCompGenA + "/explorer-config" + workspace := string(baselineExplorerWorkspaceV2()) + headers := map[string]string{pgPubCommitHeader: "commit-publish-11"} + + bootstrapServer, closeBootstrap, err := composePostgreSQLFullRuntime(ctx, bootConfig, logger) + if err != nil { + t.Fatalf("compose bootstrap runtime: %s", fixture.scrub(err)) + } + bootstrap := requestJSONWithHeaders(t, bootstrapServer.App(), http.MethodPost, pathA, workspace, headers) + pgCompAssertNoLaneLeak(t, fixture, "bootstrap deploy", bootstrap.Body) + if bootstrap.StatusCode != http.StatusOK { + closeBootstrap() + t.Fatalf("bootstrap deploy returned %d, want 200: %s", bootstrap.StatusCode, fixture.scrub(bootstrap.Body)) + } + var bootResult struct { + RevisionID string `json:"revisionId"` + Activated *bool `json:"activated"` + } + if err := json.Unmarshal([]byte(bootstrap.Body), &bootResult); err != nil { + closeBootstrap() + t.Fatalf("decode bootstrap deploy body: %v\n%s", err, fixture.scrub(bootstrap.Body)) + } + if bootResult.RevisionID == "" || bootResult.Activated == nil || !*bootResult.Activated { + closeBootstrap() + t.Fatalf("bootstrap deploy did not activate: %s", fixture.scrub(bootstrap.Body)) + } + if err := closeBootstrap(); err != nil { + t.Fatalf("close bootstrap runtime: %s", fixture.scrub(err)) + } + + const conflictGeneration = "generation-b" + fixture.load(t, pgCompProject, conflictGeneration, map[string][]string{ + "Patient.ndjson": pgCompPatientLines(t, "patient-publish-13", "patient-publish-14"), + }) + fixture.activate(t, pgCompProject, conflictGeneration) + + bootConfig.Server.DevRepositoryActivationConflictOnce = true + server, closeRuntime, err := composePostgreSQLFullRuntime(ctx, bootConfig, logger) + if err != nil { + t.Fatalf("compose conflict-armed runtime: %s", fixture.scrub(err)) + } + defer func() { + if err := closeRuntime(); err != nil { + t.Errorf("close conflict-armed runtime: %s", fixture.scrub(err)) + } + }() + app := server.App() + pathB := "/api/v1/projects/" + pgCompProject + "/generations/" + conflictGeneration + "/explorer-config" + + first := requestJSONWithHeaders(t, app, http.MethodPost, pathB, workspace, headers) + pgCompAssertNoLaneLeak(t, fixture, "conflict-armed deploy", first.Body) + if first.StatusCode != http.StatusConflict || !strings.Contains(first.Body, "PUBLICATION_ACTIVATION_CONFLICT") { + t.Fatalf("conflict-armed deploy returned %d, want 409 PUBLICATION_ACTIVATION_CONFLICT: %s", + first.StatusCode, fixture.scrub(first.Body)) + } + revisionID := pgPubPinnedRevision(t, fixture, pgCompProject, conflictGeneration) + status, failedAt, diagnostics, pointer := pgPubPinnedWitness(t, fixture, pgCompProject, revisionID) + if status != "FAILED" || failedAt.IsZero() { + t.Fatalf("revision after the lost release CAS = %s/failed_at %v, want FAILED with failed_at set", status, failedAt) + } + if !strings.Contains(diagnostics, "PUBLICATION_ACTIVATION_CONFLICT") { + t.Fatalf("lifecycle did not mark the release-conflict evidence: diagnostics = %s", fixture.scrub(diagnostics)) + } + if pointer != bootResult.RevisionID { + t.Fatalf("conflict deploy moved the owner pointer to %q, want the bootstrap revision %q", + pointer, bootResult.RevisionID) + } + + var revivedAt time.Time + for attempt := 2; attempt <= 3; attempt++ { + retry := requestJSONWithHeaders(t, app, http.MethodPost, pathB, workspace, headers) + pgCompAssertNoLaneLeak(t, fixture, fmt.Sprintf("retry %d after activation failure", attempt), retry.Body) + if retry.StatusCode != http.StatusOK { + t.Fatalf("retry %d after an activation failure returned %d, want 200 revival: %s", + attempt, retry.StatusCode, fixture.scrub(retry.Body)) + } + var revived struct { + RevisionID string `json:"revisionId"` + Activated *bool `json:"activated"` + } + if err := json.Unmarshal([]byte(retry.Body), &revived); err != nil { + t.Fatalf("decode retry %d body: %v\n%s", attempt, err, fixture.scrub(retry.Body)) + } + if revived.RevisionID != revisionID || revived.Activated == nil || !*revived.Activated { + t.Fatalf("retry %d must activate the pinned revision: got %+v, want revisionId %q activated=true", + attempt, revived, revisionID) + } + if got := pgPubPinnedRevision(t, fixture, pgCompProject, conflictGeneration); got != revisionID { + t.Fatalf("retry %d minted a different revision %q, want the pinned identity %q", attempt, got, revisionID) + } + retryStatus, retryAt, retryDiagnostics, retryPointer := pgPubPinnedWitness(t, fixture, pgCompProject, revisionID) + if retryStatus != "ACTIVE" || retryAt.IsZero() { + t.Fatalf("retry %d left the pinned revision %s/at %v, want ACTIVE with a non-zero activated_at", + attempt, retryStatus, retryAt) + } + if retryDiagnostics != "[]" { + t.Fatalf("retry %d left revival diagnostics %s, want []", attempt, fixture.scrub(retryDiagnostics)) + } + if retryPointer != revisionID { + t.Fatalf("retry %d left the owner pointer at %q, want the revived revision %q", attempt, retryPointer, revisionID) + } + if attempt == 2 { + revivedAt = retryAt + continue + } + if !retryAt.Equal(revivedAt) { + t.Fatalf("retry %d restamped activated_at: %v -> %v (the no-op replay law forbids it)", attempt, revivedAt, retryAt) + } + } + + const siblingProject = "pgcomp-p2" + fixture.load(t, siblingProject, pgCompGenA, map[string][]string{ + "Patient.ndjson": pgCompPatientLines(t, "patient-publish-21", "patient-publish-22"), + }) + fixture.activate(t, siblingProject, pgCompGenA) + siblingPath := "/api/v1/projects/" + siblingProject + "/generations/" + pgCompGenA + "/explorer-config" + deploy := requestJSONWithHeaders(t, app, http.MethodPost, siblingPath, workspace, + map[string]string{pgPubCommitHeader: "commit-publish-21"}) + pgCompAssertNoLaneLeak(t, fixture, "sibling deploy", deploy.Body) + if deploy.StatusCode != http.StatusOK { + t.Fatalf("sibling deploy returned %d, want 200: %s", deploy.StatusCode, fixture.scrub(deploy.Body)) + } + var result struct { + RevisionID string `json:"revisionId"` + Activated *bool `json:"activated"` + } + if err := json.Unmarshal([]byte(deploy.Body), &result); err != nil { + t.Fatalf("decode sibling deploy body: %v\n%s", err, fixture.scrub(deploy.Body)) + } + if result.RevisionID == "" || result.Activated == nil || !*result.Activated { + t.Fatalf("sibling deploy did not report an activated revision: %s", fixture.scrub(deploy.Body)) + } + siblingStatus, siblingActivatedAt, _, siblingPointer := pgPubPinnedWitness(t, fixture, siblingProject, result.RevisionID) + if siblingStatus != "ACTIVE" || siblingActivatedAt.IsZero() || siblingPointer != result.RevisionID { + t.Fatalf("sibling witness after deploy: %s/activated_at %v/pointer %q, want ACTIVE/non-zero/%q", + siblingStatus, siblingActivatedAt, siblingPointer, result.RevisionID) + } + replay := requestJSONWithHeaders(t, app, http.MethodPost, siblingPath, workspace, + map[string]string{pgPubCommitHeader: "commit-publish-21"}) + if replay.StatusCode != http.StatusOK { + t.Fatalf("sibling replay returned %d, want 200: %s", replay.StatusCode, fixture.scrub(replay.Body)) + } + var replayed struct { + RevisionID string `json:"revisionId"` + Activated *bool `json:"activated"` + } + if err := json.Unmarshal([]byte(replay.Body), &replayed); err != nil { + t.Fatalf("decode sibling replay body: %v\n%s", err, fixture.scrub(replay.Body)) + } + if replayed.RevisionID != result.RevisionID || replayed.Activated == nil || !*replayed.Activated { + t.Fatalf("sibling replay must be the same activated revision: got %+v, want revisionId %q activated=true", + replayed, result.RevisionID) + } + siblingReplayStatus, siblingReplayAt, _, siblingReplayPointer := pgPubPinnedWitness(t, fixture, siblingProject, result.RevisionID) + if siblingReplayStatus != "ACTIVE" || siblingReplayPointer != result.RevisionID { + t.Fatalf("sibling replay moved activation state: %s/pointer %q", siblingReplayStatus, siblingReplayPointer) + } + if !siblingActivatedAt.Equal(siblingReplayAt) { + t.Fatalf("sibling replay restamped activated_at: %v -> %v (the no-op replay law forbids it)", + siblingActivatedAt, siblingReplayAt) + } +} + +func pgPubPinnedRevision(t *testing.T, fixture *pgCompFixture, project, generation string) string { + t.Helper() + pool, err := pgxpool.New(context.Background(), fixture.dsn) + if err != nil { + t.Fatalf("connect witness pool: %s", fixture.scrub(err)) + } + defer pool.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + ctlSchema := fixture.schemas[control.SeriesExplorerControl] + if !pgCompSchemaPattern.MatchString(ctlSchema) { + t.Fatalf("witness refuses a control schema %q outside the owned namespace", ctlSchema) + } + rows, err := pool.Query(ctx, + `SELECT id FROM "`+ctlSchema+`".explorer_revisions WHERE project = $1 AND explorer_id = 'default' AND source_generation = $2`, + project, generation) + if err != nil { + t.Fatalf("witness revisions for %s/%s: %s", project, generation, fixture.scrub(err)) + } + defer rows.Close() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatalf("scan revision id for %s/%s: %s", project, generation, fixture.scrub(err)) + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + t.Fatalf("read revision ids for %s/%s: %s", project, generation, fixture.scrub(err)) + } + if len(ids) != 1 { + t.Fatalf("witness revisions for %s/%s = %v, want exactly one pinned revision", project, generation, ids) + } + return ids[0] +} + +func pgPubPinnedWitness(t *testing.T, fixture *pgCompFixture, project, revisionID string) (string, time.Time, string, string) { + t.Helper() + pool, err := pgxpool.New(context.Background(), fixture.dsn) + if err != nil { + t.Fatalf("connect witness pool: %s", fixture.scrub(err)) + } + defer pool.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + ctlSchema := fixture.schemas[control.SeriesExplorerControl] + if !pgCompSchemaPattern.MatchString(ctlSchema) { + t.Fatalf("witness refuses a control schema %q outside the owned namespace", ctlSchema) + } + ctl := `"` + ctlSchema + `"` + var status string + var failedAt, activatedAt *time.Time + var diagnostics string + if err := pool.QueryRow(ctx, + `SELECT status, failed_at, activated_at, diagnostics::text FROM `+ctl+`.explorer_revisions WHERE id = $1`, revisionID). + Scan(&status, &failedAt, &activatedAt, &diagnostics); err != nil { + t.Fatalf("witness revision %s: %s", revisionID, fixture.scrub(err)) + } + var pointer *string + if err := pool.QueryRow(ctx, + `SELECT active_revision_id FROM `+ctl+`.explorers WHERE project = $1 AND explorer_id = 'default'`, project). + Scan(&pointer); err != nil { + t.Fatalf("witness owner pointer for %s: %s", project, fixture.scrub(err)) + } + at := time.Time{} + if status == "FAILED" && failedAt != nil { + at = *failedAt + } + if status == "ACTIVE" && activatedAt != nil { + at = *activatedAt + } + pointerText := "" + if pointer != nil { + pointerText = *pointer + } + return status, at, diagnostics, pointerText +} diff --git a/internal/server/route_composition_contract_test.go b/internal/server/route_composition_contract_test.go new file mode 100644 index 000000000..b84479c91 --- /dev/null +++ b/internal/server/route_composition_contract_test.go @@ -0,0 +1,106 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + httpapi "github.com/calypr/loom/internal/api/http" + "github.com/calypr/loom/internal/authscope" +) + +func newRouteTestHTTPServer(t *testing.T) *httpapi.HTTPServer { + t.Helper() + server, err := httpapi.NewHTTPServer(httpapi.HTTPConfig{Authorizer: authscope.AllowAllAuthorizer{}}) + if err != nil { + t.Fatal(err) + } + return server +} + +func TestRouteScopePolicyValidation(t *testing.T) { + for _, tc := range []struct { + name string + policy routeScopePolicy + wantErr string + }{ + {name: "unset_zero_policy_is_refused", policy: routeScopePolicy{}, wantErr: "missing explicit scope policy"}, + {name: "scoped_without_resolver_is_refused", policy: scopedRouteScope(nil), wantErr: "requires its scope resolver"}, + {name: "unrestricted_with_resolver_is_refused", policy: routeScopePolicy{mode: routeScopeUnrestricted, resolver: &authscope.ScopeResolver{}}, wantErr: "must not carry a scope resolver"}, + {name: "unrestricted_is_valid", policy: unrestrictedRouteScope()}, + {name: "scoped_with_resolver_is_valid", policy: scopedRouteScope(&authscope.ScopeResolver{})}, + } { + t.Run(tc.name, func(t *testing.T) { + err := tc.policy.validate() + if tc.wantErr == "" { + if err != nil { + t.Fatalf("valid policy rejected: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("expected refusal containing %q, got %v", tc.wantErr, err) + } + }) + } +} + +func TestRegisterRoutesRefusesIncompleteScopedCompositionBeforeInstall(t *testing.T) { + server := newRouteTestHTTPServer(t) + err := registerRoutes(server, nil, authscope.AllowAllAuthorizer{}, nil, nil, routeDependencies{scopes: scopedRouteScope(nil)}) + if err == nil { + t.Fatal("incomplete scoped composition must be refused") + } + if !strings.Contains(err.Error(), "requires its scope resolver") { + t.Fatalf("refusal must name the missing resolver, got %v", err) + } + if _, status := server.Liveness(nil); status != 200 { + t.Fatalf("liveness must stay dependency-free, got %d", status) + } + response, err := server.App().Test(httptest.NewRequest(http.MethodPost, "/api/v1/datasets/acme/generations/gen-1", nil)) + if err != nil { + t.Fatalf("load-route probe after refused install: %v", err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusNotFound { + t.Fatalf("a refused registerRoutes must leave no load route registered: POST load got %d, want 404", response.StatusCode) + } +} + +func TestRegisterRoutesUnsetPolicyIsRefused(t *testing.T) { + server := newRouteTestHTTPServer(t) + if err := registerRoutes(server, nil, authscope.AllowAllAuthorizer{}, nil, nil, routeDependencies{}); err == nil || !strings.Contains(err.Error(), "missing explicit scope policy") { + t.Fatalf("unset scope policy must be refused, got %v", err) + } +} + +func TestWireAuthScopePoliciesAreExplicit(t *testing.T) { + noAuth, err := wireAuth(Config{}, true, nil) + if err != nil { + t.Fatal(err) + } + if noAuth.scopePolicy.mode != routeScopeUnrestricted || noAuth.scopePolicy.resolverOrNil() != nil { + t.Fatalf("no-auth must be explicitly unrestricted, got %+v", noAuth.scopePolicy) + } + basic, err := wireAuth(Config{Auth: AuthConfig{Mode: "basic", Basic: BasicAuthConfig{Username: "u", Password: "p"}}}, false, nil) + if err != nil { + t.Fatal(err) + } + if basic.scopePolicy.mode != routeScopeUnrestricted || basic.scopePolicy.resolverOrNil() != nil { + t.Fatalf("basic must be explicitly unrestricted, got %+v", basic.scopePolicy) + } + calypr, err := wireAuth(Config{Auth: AuthConfig{Mode: "calypr", Calypr: CalyprAuthConfig{TrustedIssuer: "https://issuer.example"}}}, false, nil) + if err != nil { + t.Fatal(err) + } + if calypr.scopePolicy.mode != routeScopeScoped || calypr.scopePolicy.resolverOrNil() == nil { + t.Fatalf("calypr must carry its required resolver, got %+v", calypr.scopePolicy) + } + if calypr.scopePolicy.resolverOrNil() != calypr.scopeResolver { + t.Fatal("the scoped policy must carry the same resolver the authorizer uses") + } + if err := calypr.scopePolicy.validate(); err != nil { + t.Fatalf("calypr policy must validate: %v", err) + } +} diff --git a/internal/server/routes.go b/internal/server/routes.go index 22b3f1637..24fd3a852 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -1,6 +1,7 @@ package server import ( + "errors" "fmt" loomapi "github.com/calypr/loom/generated/loomapi" @@ -12,26 +13,73 @@ import ( publication "github.com/calypr/loom/internal/dataframe/publication" ) -func registerRoutes(server *api.HTTPServer, generationService *loadapi.Service, authorizer authscope.Authorizer, graphResolver *resolver.Resolver, explorerHandlers *explorerHTTPHandlers, optional ...any) error { +type routeScopeMode uint8 + +const ( + routeScopeUnset routeScopeMode = iota + routeScopeUnrestricted + routeScopeScoped +) + +type routeScopePolicy struct { + mode routeScopeMode + resolver *authscope.ScopeResolver +} + +func unrestrictedRouteScope() routeScopePolicy { + return routeScopePolicy{mode: routeScopeUnrestricted} +} + +func scopedRouteScope(resolver *authscope.ScopeResolver) routeScopePolicy { + return routeScopePolicy{mode: routeScopeScoped, resolver: resolver} +} + +func (p routeScopePolicy) validate() error { + switch p.mode { + case routeScopeUnrestricted: + if p.resolver != nil { + return errors.New("route composition is inconsistent: unrestricted scope policy must not carry a scope resolver") + } + return nil + case routeScopeScoped: + if p.resolver == nil { + return errors.New("route composition is incomplete: the calypr scoped policy requires its scope resolver before routes are installed") + } + return nil + default: + return fmt.Errorf("route composition is incomplete: missing explicit scope policy (got mode %d)", uint8(p.mode)) + } +} + +func (p routeScopePolicy) resolverOrNil() *authscope.ScopeResolver { + if p.mode != routeScopeScoped { + return nil + } + return p.resolver +} + +type routeDependencies struct { + releases publication.BundleCatalog + scopes routeScopePolicy +} + +func (d routeDependencies) validate() error { + return d.scopes.validate() +} + +func registerRoutes(server *api.HTTPServer, generationService *loadapi.Service, authorizer authscope.Authorizer, graphResolver *resolver.Resolver, explorerHandlers *explorerHTTPHandlers, deps routeDependencies) error { + if err := deps.validate(); err != nil { + return fmt.Errorf("validate route composition: %w", err) + } loadHandler, err := loadapi.NewHandler(loadapi.Config{Service: generationService, Authorizer: authorizer}) if err != nil { return fmt.Errorf("create generation load handler: %w", err) } - var releases publication.BundleCatalog - var scopes *authscope.ScopeResolver - for _, value := range optional { - switch typed := value.(type) { - case publication.BundleCatalog: - releases = typed - case *authscope.ScopeResolver: - scopes = typed - } - } routes := &HTTPRoutes{ server: server, load: loadHandler, - releases: releases, - scopes: scopes, + releases: deps.releases, + scopes: deps.scopes.resolverOrNil(), graphql: graphapi.RouteConfig{Handler: graphapi.NewHandler(graphResolver, server.Logger()), Playground: graphapi.NewPlaygroundHandler("/graphql/graph"), Sandbox: graphapi.NewApolloSandboxHandler("/graphql/graph")}, explorer: explorerHandlers, } diff --git a/internal/server/routes_test.go b/internal/server/routes_test.go index 014899f48..7b4fe1ab2 100644 --- a/internal/server/routes_test.go +++ b/internal/server/routes_test.go @@ -32,7 +32,7 @@ func TestRegisterRoutesExposesGenerationReleaseWorkflow(t *testing.T) { if err != nil { t.Fatal(err) } - if err := registerRoutes(server, resourceService, authscope.AllowAllAuthorizer{}, &resolver.Resolver{}, &explorerHTTPHandlers{}); err != nil { + if err := registerRoutes(server, resourceService, authscope.AllowAllAuthorizer{}, &resolver.Resolver{}, &explorerHTTPHandlers{}, routeDependencies{scopes: unrestrictedRouteScope()}); err != nil { t.Fatal(err) } diff --git a/internal/server/row_sources.go b/internal/server/row_sources.go new file mode 100644 index 000000000..3491c3feb --- /dev/null +++ b/internal/server/row_sources.go @@ -0,0 +1,96 @@ +package server + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "log/slog" + "time" + + arangofactory "github.com/calypr/loom/internal/backends/arango" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" +) + +type aqlRowExecutor interface { + QueryRows(ctx context.Context, query string, batchSize int, bindVars map[string]any, visit arangofactory.RowVisitor) error + QueryRowsWithMaxRuntime(ctx context.Context, query string, batchSize int, bindVars map[string]any, maxRuntime time.Duration, visit arangofactory.RowVisitor) error +} + +type selectedAQLAdapter struct { + exec aqlRowExecutor +} + +func (a selectedAQLAdapter) serviceRows(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + query, bindVars, dispatchErr := request.Command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + return a.exec.QueryRows(ctx, query, request.BatchSize, bindVars, arangofactory.RowVisitor(visit)) +} + +func (a selectedAQLAdapter) dispatchRows(ctx context.Context, query string, batchSize int, bindVars map[string]any, _ dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + return a.exec.QueryRows(ctx, query, batchSize, bindVars, arangofactory.RowVisitor(visit)) +} + +func (a selectedAQLAdapter) dispatchPreviewRows(ctx context.Context, query string, batchSize int, bindVars map[string]any, _ dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + return a.exec.QueryRowsWithMaxRuntime(ctx, query, batchSize, bindVars, previewQueryMaxRuntime(ctx), arangofactory.RowVisitor(visit)) +} + +type aqlRowDispatch func(ctx context.Context, query string, batchSize int, bindVars map[string]any, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error + +func configuredAQLSourceRows(logger *slog.Logger, phase string, execute dataframeexecution.SourceRows) dataframeexecution.SourceRows { + return withConfiguredAQLRows(logger, phase, func(ctx context.Context, _ string, _ int, _ map[string]any, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + return execute(ctx, request, visit) + }) +} + +func withConfiguredAQLRows(logger *slog.Logger, phase string, dispatch aqlRowDispatch) dataframeexecution.SourceRows { + return func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + query, bindVars, dispatchErr := request.Command.AQLParameters() + if dispatchErr != nil { + return dispatchErr + } + if budget, ok := dataframeexecution.QueryMemoryBudgetFromContext(ctx); ok && budget > 0 { + ctx = arangofactory.WithQueryMemoryLimit(ctx, budget) + } + started := time.Now() + digest := sha256.Sum256([]byte(query)) + queryHash := hex.EncodeToString(digest[:]) + if logger != nil { + logger.Info("dataframe AQL started", "request_id", requestIDFromContext(ctx), "phase", phase+"_start", + "query_hash", queryHash, "query_bytes", len(query), "bind_vars", len(bindVars), "cursor_batch_size", request.BatchSize) + } + err := preserveConfiguredQueryError(dispatch(ctx, query, request.BatchSize, bindVars, request, visit)) + level := slog.LevelInfo + if err != nil { + level = slog.LevelError + } + logServerDiagnostic(logger, level, "dataframe AQL execution", requestIDFromContext(ctx), phase, time.Since(started), err, + "query_hash", queryHash) + return err + } +} + +func configuredPostgreSQLSourceRows(logger *slog.Logger, phase string, execute dataframeexecution.SourceRows) dataframeexecution.SourceRows { + return func(ctx context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + sqlText, bindValues, dispatchErr := request.Command.PostgreSQLParameters() + if dispatchErr != nil { + return dispatchErr + } + started := time.Now() + digest := sha256.Sum256([]byte(sqlText)) + queryHash := hex.EncodeToString(digest[:]) + if logger != nil { + logger.Info("dataframe SQL started", "request_id", requestIDFromContext(ctx), "phase", phase+"_start", + "query_hash", queryHash, "statement_bytes", len(sqlText), "bind_values", len(bindValues)) + } + err := preserveConfiguredQueryError(execute(ctx, request, visit)) + level := slog.LevelInfo + if err != nil { + level = slog.LevelError + } + logServerDiagnostic(logger, level, "dataframe SQL execution", requestIDFromContext(ctx), phase, time.Since(started), err, + "query_hash", queryHash) + return err + } +} diff --git a/internal/server/row_tag_guard_contract_test.go b/internal/server/row_tag_guard_contract_test.go new file mode 100644 index 000000000..ad2b63abd --- /dev/null +++ b/internal/server/row_tag_guard_contract_test.go @@ -0,0 +1,46 @@ +package server + +import ( + "context" + "errors" + "testing" + + "github.com/calypr/loom/internal/dataframe/compiler/command" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" +) + +func TestRowCoreCrossTagGuardsRefusePreDispatch(t *testing.T) { + visit := func(map[string]any) error { return nil } + + pgCommand, err := command.NewPostgreSQLCommand(`SELECT 1`, nil) + if err != nil { + t.Fatal(err) + } + aqlDispatches := 0 + aqlLane := configuredAQLSourceRows(nil, "contract", func(context.Context, dataframeexecution.RowsRequest, dataframeexecution.RowVisitor) error { + aqlDispatches++ + return nil + }) + if err := aqlLane(context.Background(), dataframeexecution.RowsRequest{Command: pgCommand, BatchSize: 10}, visit); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("PostgreSQL-tagged command into the AQL row core: %v", err) + } + if aqlDispatches != 0 { + t.Fatalf("AQL core dispatched %d times for a cross-tag command; the tag guard must reject before dispatch", aqlDispatches) + } + + aqlCommand, err := command.NewAQLCommand("RETURN 1", nil) + if err != nil { + t.Fatal(err) + } + pgDispatches := 0 + pgLane := configuredPostgreSQLSourceRows(nil, "contract", func(context.Context, dataframeexecution.RowsRequest, dataframeexecution.RowVisitor) error { + pgDispatches++ + return nil + }) + if err := pgLane(context.Background(), dataframeexecution.RowsRequest{Command: aqlCommand, BatchSize: 10}, visit); !errors.Is(err, command.ErrCompiledCommandBackendMismatch) { + t.Fatalf("AQL-tagged command into the PostgreSQL row core: %v", err) + } + if pgDispatches != 0 { + t.Fatalf("PostgreSQL core dispatched %d times for a cross-tag command; the tag guard must reject before dispatch", pgDispatches) + } +} diff --git a/internal/server/row_value_policy_stream_test.go b/internal/server/row_value_policy_stream_test.go index f65bb9ed5..b62d00082 100644 --- a/internal/server/row_value_policy_stream_test.go +++ b/internal/server/row_value_policy_stream_test.go @@ -6,6 +6,7 @@ import ( "fmt" "testing" + "github.com/calypr/loom/internal/dataframe/compiler" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" "github.com/calypr/loom/internal/dataframe/recipe" "github.com/calypr/loom/internal/explorer" @@ -17,8 +18,14 @@ func TestValidateReceiptFullStreamScansPastPreviewCapAndOnlySelectedOutput(t *te rowsVisited := 0 queries := 0 recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: compilerTestRegistry{}, - QueryRows: func(_ context.Context, _ string, _ int, _ map[string]any, visit func(map[string]any) error) error { + CompileTarget: compiler.ArangoTarget(), + Registry: compilerTestRegistry{}, + SourceRows: func(_ context.Context, request dataframeexecution.RowsRequest, visit dataframeexecution.RowVisitor) error { + _, _, deriveErr := request.Command.AQLParameters() + if deriveErr != nil { + return deriveErr + } + queries++ if queries != 1 { return fmt.Errorf("unrelated receipt output was streamed") diff --git a/internal/server/runtime.go b/internal/server/runtime.go new file mode 100644 index 000000000..c698c2c12 --- /dev/null +++ b/internal/server/runtime.go @@ -0,0 +1,523 @@ +package server + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "strings" + "time" + + loadapi "github.com/calypr/loom/internal/api/bulk/load" + queryapi "github.com/calypr/loom/internal/api/graphql/graph/query" + graphresolver "github.com/calypr/loom/internal/api/graphql/graph/resolver" + httpapi "github.com/calypr/loom/internal/api/http" + "github.com/calypr/loom/internal/authscope" + "github.com/calypr/loom/internal/backends/postgres/control" + "github.com/calypr/loom/internal/catalog" + "github.com/calypr/loom/internal/dataframe/compiler" + "github.com/calypr/loom/internal/dataframe/compiler/ir" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" + "github.com/calypr/loom/internal/dataframe/execution/chartifact" + publication "github.com/calypr/loom/internal/dataframe/publication" + publicationclickhouse "github.com/calypr/loom/internal/dataframe/publication/clickhouse" + "github.com/calypr/loom/internal/dataframe/published" + publicationcontract "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/explorer" + "github.com/calypr/loom/internal/explorer/artifactfs" + "github.com/calypr/loom/internal/ingest" + clickhousestore "github.com/calypr/loom/internal/store/clickhouse" +) + +const postgreSQLRuntimeAcquireTimeout = 30 * time.Second + +func postgreSQLRuntimeUnavailable(operation string) error { + return dataframeerrors.NewError(dataframeerrors.CodeBackendUnavailable, + operation+" has no PostgreSQL adapter yet; the PostgreSQL runtime refuses it instead of serving ArangoDB data", + dataframeerrors.WithRetryable(false), + dataframeerrors.WithDetails(map[string]any{"backend": "postgresql", "stage": "adapter"})) +} + +func postgresRuntimeAssignment(pg PostgreSQLConfig) (control.SeriesSchemas, map[control.Series]string, error) { + required := []control.Series{ + control.SeriesSource, + control.SeriesExplorerControl, + control.SeriesPublicationControl, + control.SeriesDatasetRelease, + control.SeriesRecipeChains, + control.SeriesRecipeExecRegistries, + } + if len(pg.Schemas) != len(required) { + return nil, nil, ErrPostgreSQLRuntimeNotWired + } + assignment := control.SeriesSchemas{} + names := map[control.Series]string{} + seen := map[string]bool{} + for _, series := range required { + schema, assigned := pg.Schemas[string(series)] + schema = strings.TrimSpace(schema) + if !assigned || schema == "" || seen[schema] { + return nil, nil, ErrPostgreSQLRuntimeNotWired + } + seen[schema] = true + assignment[series] = schema + names[series] = schema + } + return assignment, names, nil +} + +type repositoryGenerationActivator interface { + ActivateRepositoryGeneration(ctx context.Context, project, generation, revisionID string) error +} + +type pgExplorerStore struct { + *control.ExplorerStore + publish *control.PublishStore + activate repositoryGenerationActivator +} + +var _ explorer.Store = pgExplorerStore{} + +func (s pgExplorerStore) PublishAuthoring(ctx context.Context, receipt explorer.CompilationReceipt, revision explorer.Revision, release publicationcontract.ProjectRelease, expectedReleaseRevision int64) (*explorer.Revision, error) { + if s.publish == nil { + return nil, postgreSQLRuntimeUnavailable("Explorer publication (the dataset_release assignment is missing)") + } + return s.publish.PublishAuthoring(ctx, receipt, revision, release, expectedReleaseRevision) +} + +func (s pgExplorerStore) ActivateRepositoryGeneration(ctx context.Context, project, generation, revisionID string) error { + if s.activate == nil { + return postgreSQLRuntimeUnavailable("Repository-generation activation (the dataset_release or publication_control assignment is missing)") + } + return s.activate.ActivateRepositoryGeneration(ctx, project, generation, revisionID) +} + +func (s pgExplorerStore) ValidateSelectionReferences(context.Context, string, string, authscope.ReadScope, []explorer.ResourceRef) error { + return postgreSQLRuntimeUnavailable("Explicit-selection reference validation") +} + +type pgExplorerReads struct { + *control.ExplorerStore + *control.ExplicitGroupStore +} + +var _ explorer.ExplicitGroupRepository = pgExplorerReads{} + +type pgIngestRunner struct { + BaseOptions ingest.LoadOptions + PG ingest.PGSourceOptions +} + +func (r pgIngestRunner) RunGeneration(ctx context.Context, req loadapi.GenerationLoadRequest, sink ingest.EventSink) (ingest.LoadSummary, error) { + ref, err := publicationcontract.NewRef(req.Project, req.Generation) + if err != nil { + return ingest.LoadSummary{}, err + } + opts := r.BaseOptions + opts.Project = req.Project + opts.AuthResourcePath = req.AuthResourcePath + opts.MetaDir = req.StagedDir + opts.Dataset = &ref + opts.DeferActivation = req.DeferActivation + opts.Truncate = false + opts.EventSink = sink + opts.StageOnly = false + summary, err := ingest.LoadPostgresGeneration(ctx, opts, r.PG) + if err != nil { + return summary, err + } + if !req.DeferActivation { + if err := ingest.ActivatePostgresGeneration(ctx, ref, r.PG); err != nil { + return summary, err + } + } + return summary, nil +} + +type pgGenerationActivator struct { + manifests publicationcontract.ManifestLifecycle +} + +func (a pgGenerationActivator) ReadManifest(ctx context.Context, ref publicationcontract.Ref) (publicationcontract.Manifest, error) { + if a.manifests == nil { + return publicationcontract.Manifest{}, postgreSQLRuntimeUnavailable("Generation manifest reads (the dataset_release assignment is missing)") + } + return a.manifests.ReadManifest(ctx, ref) +} + +func (a pgGenerationActivator) Activate(ctx context.Context, manifest publicationcontract.Manifest) error { + if a.manifests == nil { + return postgreSQLRuntimeUnavailable("Server-side generation activation (the dataset_release assignment is missing)") + } + ref := manifest.Dataset + if !manifest.IsStaged() { + if err := manifest.Validate(); err != nil { + return err + } + return fmt.Errorf("activate dataset generation %s/%s: %w: state is %s", ref.Project, ref.Generation, publicationcontract.ErrGenerationNotReady, manifest.State) + } + var expected *publicationcontract.Ref + active, resolveErr := a.manifests.ResolveActiveManifest(ctx, ref.Project) + switch { + case resolveErr == nil: + if active.IsStaged() { + expected = &active.Dataset + } + case errors.Is(resolveErr, publicationcontract.ErrNoActiveGeneration): + expected = nil + default: + expected = nil + } + if err := a.manifests.ActivateGeneration(ctx, manifest, expected); err != nil { + if errors.Is(err, control.ErrActivationConflict) || + errors.Is(err, publicationcontract.ErrGenerationNotReady) || + errors.Is(err, publicationcontract.ErrInvalidActiveGeneration) { + return err + } + if pgActivationOutcomeUnknown(err) { + return &ingest.ActivationOutcomeError{Dataset: ref, Err: err} + } + return err + } + return nil +} + +func pgActivationOutcomeUnknown(err error) bool { + if err == nil { + return false + } + if control.CommitOutcomeUnknown(err) { + return true + } + return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) +} + +func composePostgreSQLFullRuntime(ctx context.Context, serverConfig Config, logger *slog.Logger) (server *httpapi.HTTPServer, closeRuntime func() error, err error) { + var cleanups []func() error + defer func() { + if err == nil { + return + } + _ = runRuntimeCloseSteps(cleanups) + }() + assignment, schemas, err := postgresRuntimeAssignment(serverConfig.Server.PostgreSQL) + if err != nil { + return nil, nil, err + } + pg := serverConfig.Server.PostgreSQL + if err := pg.validate(); err != nil { + return nil, nil, err + } + dsn := postgresFirstQueryRouteDSN(pg) + runtime, err := openPostgresBackend(ctx, control.Config{ + DSN: dsn, + MaxConns: int32(pg.MaxOpenConns), + MinConns: int32(pg.MaxIdleConns), + AcquireTimeout: postgreSQLRuntimeAcquireTimeout, + Schemas: assignment, + Setup: control.SetupNone, + }) + if err != nil { + return nil, nil, postgresFirstQueryRouteError(fmt.Errorf("open PostgreSQL server runtime: %w", err), pg, dsn) + } + cleanups = append(cleanups, func() error { + return postgresFirstQueryRouteError(runtime.Close(), pg, dsn) + }) + if runtime.Explorer == nil || runtime.Publish == nil || runtime.Activate == nil || runtime.Interpretations == nil || + runtime.ExplicitGroups == nil || runtime.CapabilitySnapshots == nil || runtime.TableShapes == nil || + runtime.Bundles == nil || runtime.RecipeRevisions == nil || runtime.RecipeExecutions == nil || + runtime.Catalog == nil || runtime.SemanticReads == nil || runtime.Releases == nil || runtime.Manifests == nil { + return nil, nil, postgresFirstQueryRouteError(fmt.Errorf("%w: assembled PostgreSQL runtime is missing a required data-plane port", control.ErrBootstrap), pg, dsn) + } + + binding, err := ir.NewPostgresStorageBinding(schemas[control.SeriesSource]) + if err != nil { + return nil, nil, postgresFirstQueryRouteError(fmt.Errorf("bind PostgreSQL storage placement: %w", err), pg, dsn) + } + sourceCompileTarget, err := compiler.PostgreSQLTarget(binding) + if err != nil { + return nil, nil, postgresFirstQueryRouteError(fmt.Errorf("select PostgreSQL compile target: %w", err), pg, dsn) + } + + dataframes := dataframeexecution.NewService(dataframeexecution.ServiceConfig{ + SourceRows: configuredPostgreSQLSourceRows(logger, "query_rows", runtime.SourceRows), + }) + + discoveryCache := catalog.NewCache() + discoverFields := discoveryCache.DiscoverFields(runtime.DiscoverFields) + + auth, err := wireAuth(serverConfig, serverConfig.Server.AllowUnauthenticated || serverConfig.Auth.AllowUnauthenticated, runtime.DiscoverExistingAuthResourcePaths) + if err != nil { + return nil, nil, fmt.Errorf("configure authentication: %w", err) + } + authenticator, authorizer, scopeResolver := auth.authenticator, auth.authorizer, auth.scopeResolver + + degradation := error(nil) + publicationReady := true + var clickhouse *clickhousestore.Client + var materializationReader *published.Reader + var privateClickHouseArtifacts *chartifact.Manager + var plane *publicationPlane + if serverConfig.Server.ClickHouse.Enabled { + var closeClickHouse func() error + var planeErr error + plane, closeClickHouse, planeErr = newPublicationPlane(ctx, serverConfig, logger, publicationPlaneInput{ + InitialDegradation: degradation, + InitialPublicationReady: publicationReady, + ClickHouse: clickhousestore.Options{URL: serverConfig.Server.ClickHouse.URL, Database: serverConfig.Server.ClickHouse.Database, Username: serverConfig.Server.ClickHouse.Username, Password: serverConfig.Server.ClickHouse.Password}, + AcquireArtifactCatalog: func(context.Context, *clickhousestore.Client) (chartifact.Catalog, error) { + return runtime.Chartifacts, nil + }, + BundleCatalog: runtime.Bundles, + ActiveManifestResolver: runtime.Manifests, + ActiveReleaseResolver: runtime.Releases, + Scopes: scopeResolver, + }) + if closeClickHouse != nil { + cleanups = append(cleanups, closeClickHouse) + } + if planeErr != nil { + return nil, nil, planeErr + } + clickhouse = plane.ClickHouse + privateClickHouseArtifacts = plane.PrivateArtifacts + materializationReader = plane.MaterializationReader + degradation = plane.Degradation + publicationReady = plane.PublicationReady + } + + var clickHouseQueryRows dataframeexecution.ClickHouseQueryRows + var resolveClickHouseInputs dataframeexecution.ResolveClickHouseInputs + var withExecutionReadPins dataframeexecution.WithExecutionReadPins + if plane != nil { + clickHouseQueryRows = plane.ClickHouseQueryRows + resolveClickHouseInputs = plane.ResolveClickHouseInputs + withExecutionReadPins = plane.WithExecutionReadPins + } + + recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ + CompileTarget: sourceCompileTarget, + Registry: runtime.RecipeExecutions, + Revisions: runtime.RecipeRevisions, + ResolveBundle: recipeSchemaResolver(runtime.DiscoverFields, discoveryCache), + PreparePreviewIndex: nil, + PreviewCoveringIndexes: nil, + PreviewCollectionRevision: nil, + PreviewCollectionScans: nil, + PreviewCollectionCount: nil, + ClickHouseQueryRows: clickHouseQueryRows, + PreviewSourceRows: configuredPostgreSQLSourceRows(logger, "preview_query_rows", runtime.SourceRows), + ResolveClickHouseInputs: resolveClickHouseInputs, + WithExecutionReadPins: withExecutionReadPins, + PrivateClickHouseArtifacts: privateClickHouseArtifacts, + SourceRows: configuredPostgreSQLSourceRows(logger, "query_rows", runtime.SourceRows), + ScopeDigest: recipeScopeDigest, + RootPageRows: serverConfig.Server.RecipeQueryPageRows, + }) + if err != nil { + return nil, nil, fmt.Errorf("create dataframe recipe engine: %w", err) + } + if serverConfig.Server.ClickHouse.Enabled && !recipeEngine.CombinePortsConfigured() { + return nil, nil, fmt.Errorf("PostgreSQL runtime has ClickHouse enabled but the recipe engine is missing a private workspace Combine port") + } + + var bundleTarget publication.Target + if plane != nil { + if err := plane.reconcileBundles(ctx, logger, publicationReady, degradation, runtime.Bundles, runtime.Releases, + func(ctx context.Context, clickhouse *clickhousestore.Client, _ *publicationclickhouse.ClickHouseBundleStore, cutoff time.Time) (string, error) { + reconciler, err := runtime.NewPublicationReconciler(runtime.Releases) + if err != nil { + return "publication reconciler", err + } + if err := reconciler.SweepStale(ctx, cutoff, func(ctx context.Context, physicalTable string) error { + return clickhouse.DropTable(ctx, physicalTable) + }); err != nil { + return "dataframe publication reconciliation", err + } + return "", nil + }); err != nil { + return nil, nil, err + } + publicationReady = plane.PublicationReady + degradation = plane.Degradation + bundleTarget = plane.BundleTarget + } + + hooks := newReleaseHooks(serverConfig, runtime.Manifests, runtime.Releases, runtime.Bundles) + + explorerStore := pgExplorerStore{ExplorerStore: runtime.Explorer, publish: runtime.Publish, activate: runtime.Activate} + explorerReads := pgExplorerReads{ExplorerStore: runtime.Explorer, ExplicitGroupStore: runtime.ExplicitGroups} + capabilityResolver, err := newExplorerCapabilityResolver(runtime.Catalog, scopeResolver, runtime.Manifests, runtime.CapabilitySnapshots) + if err != nil { + return nil, nil, fmt.Errorf("create Explorer capability resolver: %w", err) + } + explorerService, err := explorer.NewService(explorerStore) + if err != nil { + return nil, nil, fmt.Errorf("create Explorer service: %w", err) + } + artifactStore, err := artifactfs.New(serverConfig.Server.ArtifactDirectory) + if err != nil { + return nil, nil, fmt.Errorf("create Explorer artifact store: %w", err) + } + + resolver := graphresolver.NewResolver(graphresolver.ResolverConfig{ + DataframeQuery: queryapi.Config{ + CompileTarget: sourceCompileTarget, + DiscoverReferences: nil, + DiscoverFields: discoverFields, + Dataframes: dataframes, + ScopeResolver: scopeResolver, + ActiveManifestResolver: runtime.Manifests, + Explain: nil, + }, + MaterializationReader: materializationReader, + Logger: logger, + RecipeControl: dataframeexecution.Control{ + Engine: recipeEngine, + ExplainPostgreSQLConnection: func(ctx context.Context, compiled compiler.CompiledQuery) (dataframeexecution.PostgreSQLPlanAssessment, error) { + return explainPostgreSQLCompiledQuery(ctx, runtime.PlanJSON, compiled) + }, + }, + RecipeAuthorizer: recipeAuthorization{resolver: scopeResolver}, + RecipeRevisions: runtime.RecipeRevisions, + RecipeExecutions: graphresolver.NewAuthorizedRecipeExecutionReader(runtime.Bundles, scopeResolver), + }) + + ingestRunner := pgIngestRunner{ + BaseOptions: ingest.LoadOptions{Schema: serverConfig.Server.Schema}, + PG: ingest.PGSourceOptions{ + DSN: dsn, + Schema: schemas[control.SeriesSource], + ManifestSchema: schemas[control.SeriesDatasetRelease], + Catalog: ingest.GenerationCatalogWriters{ + FieldProfiles: runtime.FieldProfiles, + Inventory: runtime.Inventory, + Relationships: runtime.Relationships, + Contributions: runtime.Contributions, + }, + }, + } + generationService, err := loadapi.NewService(loadapi.ServiceConfig{ + LoadGeneration: ingestRunner.RunGeneration, + GenerationActivator: pgGenerationActivator{manifests: runtime.Manifests}, + DataframeReleases: runtime.Bundles, + Logger: logger, + OnSuccess: func(project string) { + discoveryCache.InvalidateProject(project) + if scopeResolver != nil { + scopeResolver.InvalidateProject(project) + } + }, + }) + if err != nil { + return nil, nil, fmt.Errorf("create generation load service: %w", err) + } + + lifecycleConfig, err := buildLifecycleConfig(lifecycleCompositionInput{ + ServerConfig: serverConfig, + Logger: logger, + RecipeEngine: recipeEngine, + CapabilityResolver: capabilityResolver, + ExplorerService: explorerService, + ArtifactStore: artifactStore, + MaterializationReader: materializationReader, + ScopeResolver: scopeResolver, + BundleTarget: bundleTarget, + Degradation: degradation, + Hooks: hooks, + Ports: lifecycleLanePorts{ + SemanticInventory: runtime.SemanticReads.PageSemanticInventory, + ResolveSemanticInventorySelections: runtime.SemanticReads.ResolveSemanticInventorySelections, + SelectionMembersCollection: "", + InterpretationRepository: runtime.Interpretations, + TableShapeCapabilities: runtime.TableShapes, + ExplicitGroups: explorerReads, + SelectionReferenceValidator: explorerStore.ValidateSelectionReferences, + ReceiptExecutions: runtime.Bundles, + }, + }) + if err != nil { + return nil, nil, err + } + server, err = httpapi.NewHTTPServer(httpapi.HTTPConfig{Authenticator: authenticator, Authorizer: authorizer, Logger: logger, + CoreReadyCheck: func(ctx context.Context) error { + return runtime.Readiness(ctx) + }, + CoreState: postgreSQLCoreReadinessState, + ClickHouseReadyCheck: func(ctx context.Context) error { + if clickhouse == nil { + return nil + } + return clickhouse.Ping(ctx) + }, ClickHouseEnabled: serverConfig.Server.ClickHouse.Enabled}) + if err != nil { + return nil, nil, fmt.Errorf("create HTTP server: %w", err) + } + if err := wireExplorerRoutes(server, generationService, authorizer, resolver, scopeResolver, explorerService, lifecycleConfig, constructionInputsCatalog{ + reader: materializationReader, catalog: runtime.Bundles, capabilities: lifecycleConfig.Capability, + scopes: scopeResolver, explorers: explorerService, revisions: explorerService, + }, routeDependencies{releases: runtime.Bundles, scopes: auth.scopePolicy}); err != nil { + return nil, nil, err + } + if privateClickHouseArtifacts != nil { + reconcileCtx, cancelReconciler := context.WithCancel(ctx) + reconcilerDone := startPrivateArtifactReconciler(reconcileCtx, privateClickHouseArtifacts, logger) + cleanups = append(cleanups, func() error { + cancelReconciler() + timer := time.NewTimer(cleanupTimeout) + defer timer.Stop() + select { + case <-reconcilerDone: + case <-timer.C: + logger.Error("private ClickHouse artifact reconciler did not stop before shutdown deadline") + } + return nil + }) + } + if artifactStore != nil { + artifactReconcileCtx, cancelArtifactReconciler := context.WithCancel(ctx) + artifactReconcilerDone := startArtifactFSReconciler(artifactReconcileCtx, artifactStore, logger) + cleanups = append(cleanups, func() error { + cancelArtifactReconciler() + timer := time.NewTimer(cleanupTimeout) + defer timer.Stop() + select { + case <-artifactReconcilerDone: + case <-timer.C: + logger.Error("filesystem artifact reconciler did not stop before shutdown deadline") + } + return nil + }) + } + closeRuntime = func() error { return runRuntimeCloseSteps(cleanups) } + return server, closeRuntime, nil +} + +func runRuntimeCloseSteps(steps []func() error) error { + var firstErr error + for i := len(steps) - 1; i >= 0; i-- { + if cerr := steps[i](); cerr != nil && firstErr == nil { + firstErr = cerr + } + } + return firstErr +} + +func runPostgreSQLFullRuntime(ctx context.Context, serverConfig Config, logger *slog.Logger) error { + server, closeRuntime, err := composePostgreSQLFullRuntime(ctx, serverConfig, logger) + if err != nil { + return err + } + _, schemas, _ := postgresRuntimeAssignment(serverConfig.Server.PostgreSQL) + return serveRuntime(ctx, server, serverConfig.Server.Listen, closeRuntime, "close PostgreSQL server runtime", logger, + "listen", serverConfig.Server.Listen, "backend", serverConfig.Server.Backend, + "source_schema", schemas[control.SeriesSource], + "explorer_schema", schemas[control.SeriesExplorerControl], + "publication_schema", schemas[control.SeriesPublicationControl], + "dataset_release_schema", schemas[control.SeriesDatasetRelease], + "recipe_chains_schema", schemas[control.SeriesRecipeChains], + "recipe_exec_schema", schemas[control.SeriesRecipeExecRegistries], + "no_auth", serverConfig.Server.AllowUnauthenticated || serverConfig.Auth.AllowUnauthenticated) +} diff --git a/internal/server/runtime_postgres_live_test.go b/internal/server/runtime_postgres_live_test.go new file mode 100644 index 000000000..caa1085a9 --- /dev/null +++ b/internal/server/runtime_postgres_live_test.go @@ -0,0 +1,1022 @@ +package server + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/json" + "fmt" + "io" + "log/slog" + "mime/multipart" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/gofiber/fiber/v3" + "github.com/jackc/pgx/v5/pgxpool" + + backendspostgres "github.com/calypr/loom/internal/backends/postgres" + "github.com/calypr/loom/internal/backends/postgres/control" + publication "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/ingest" +) + +const ( + pgCompDBFmt = "verify_pgcomp_%d_%s" + pgCompSchemaFmt = "verify_pgcomp_%d_%s_%s" + pgCompProject = "pgcomp-p1" + pgCompGenA = "generation-a" + pgCompAppTag = "verify-pgcomp-server" + firstQueryRouteDeferral = "first query route" +) + +var pgCompSeriesTokens = []struct { + series control.Series + token string +}{ + {control.SeriesSource, "src"}, + {control.SeriesExplorerControl, "exp"}, + {control.SeriesPublicationControl, "pub"}, + {control.SeriesDatasetRelease, "rel"}, + {control.SeriesRecipeChains, "rcp"}, + {control.SeriesRecipeExecRegistries, "exe"}, +} + +var ( + pgCompDBPattern = regexp.MustCompile(`^verify_pgcomp_[0-9]+_[0-9a-f]+$`) + pgCompSchemaPattern = regexp.MustCompile(`^verify_pgcomp_[0-9]+_[0-9a-f]+_(src|exp|pub|rel|rcp|exe)$`) + pgCompOwnedMu sync.Mutex + pgCompOwned = map[string]pgCompMint{} + pgCompStrayMarks = []string{} + pgCompBaselineMu sync.Mutex + pgCompBaseline int64 + pgCompBaselineSet bool + pgCompBaselineErr error +) + +type pgCompMint struct { + leg string + dropped bool +} + +var pgCompMintInventory = map[string]int{ + "TestPostgresRuntimeFullLadderServesProductionReads": 1, + "TestPostgresRuntimeServerLoadRouteServesEndToEnd": 1, + "TestPostgresRuntimeHealthReadinessIsIndependentAndSchemaAware": 1, + "TestPostgresRuntimeReadinessReportsNeverMigratedSchema": 1, + "TestPostgresRuntimeReadinessReportsMissingSchema": 1, + "TestPostgresRuntimeVerifySchemaIsReadOnly": 1, + "TestBackendCompositionContract/liveProductionCompose_ledgerHolds": 1, +} + +var pgCompClickHouseGatedMintInventory = map[string]int{ + "TestPostgresRuntimeRepositoryPublishRouteActivatesThroughFinalFlip": 1, + "TestPostgresRuntimeRepositoryPublishRetryAfterActivationFailureRevives": 1, +} + +func pgCompClickHouseConfigured() bool { + return strings.TrimSpace(os.Getenv("LOOM_CLICKHOUSE_URL")) != "" +} + +func pgCompTestPostgresURL(t *testing.T) string { + t.Helper() + raw := strings.TrimSpace(os.Getenv("LOOM_TEST_POSTGRES_URL")) + if raw == "" { + t.Skip("set LOOM_TEST_POSTGRES_URL to run the PostgreSQL full-runtime integration tests") + } + return raw +} + +func pgCompRandomHex(t *testing.T) string { + t.Helper() + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("generate isolation suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) +} + +func pgCompRegister(db, leg string) { + pgCompOwnedMu.Lock() + defer pgCompOwnedMu.Unlock() + pgCompOwned[db] = pgCompMint{leg: leg} +} + +func pgCompMarkDropped(db string) { + pgCompOwnedMu.Lock() + defer pgCompOwnedMu.Unlock() + rec, ok := pgCompOwned[db] + if !ok { + pgCompStrayMarks = append(pgCompStrayMarks, db) + return + } + rec.dropped = true + pgCompOwned[db] = rec +} + +func pgCompCaptureBaseline(t *testing.T, raw string) { + t.Helper() + pgCompBaselineMu.Lock() + defer pgCompBaselineMu.Unlock() + if pgCompBaselineSet { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + pgCompBaselineErr = fmt.Errorf("connect for baseline: %w", err) + return + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname ~ '^verify_pgcomp_[0-9]+_[0-9a-f]+$'"). + Scan(&count); err != nil { + pgCompBaselineErr = fmt.Errorf("count baseline: %w", err) + return + } + pgCompBaseline = count + pgCompBaselineSet = true +} + +type pgCompFixture struct { + dsn string + schemas map[control.Series]string + scrub func(any) string + secrets []string + admin *pgxpool.Pool + database string +} + +func newPgCompFixture(t *testing.T) *pgCompFixture { + t.Helper() + raw := pgCompTestPostgresURL(t) + pgCompCaptureBaseline(t, raw) + if pgCompBaselineErr != nil { + t.Fatalf("capture pg_database baseline: %v", pgCompBaselineErr) + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + t.Fatal("LOOM_TEST_POSTGRES_URL must be a postgres:// or postgresql:// URL") + } + var secrets []string + secrets = append(secrets, raw) + if u.User != nil { + secrets = append(secrets, u.User.String()) + if pw, ok := u.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + scrub := func(v any) string { + s := fmt.Sprintf("%v", v) + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, "") + } + } + return s + } + + ctx, cancel := context.WithTimeout(context.Background(), 600*time.Second) + t.Cleanup(cancel) + + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect to test service: %s", scrub(err)) + } + t.Cleanup(admin.Close) + + database := fmt.Sprintf(pgCompDBFmt, time.Now().UnixNano(), pgCompRandomHex(t)) + if !pgCompDBPattern.MatchString(database) { + t.Fatalf("minted database %q violates the owned-name pattern", database) + } + if _, err := admin.Exec(ctx, `CREATE DATABASE "`+database+`"`); err != nil { + t.Fatalf("create isolated database: %s", scrub(err)) + } + pgCompRegister(database, t.Name()) + t.Cleanup(func() { + if _, err := admin.Exec(context.WithoutCancel(context.Background()), + `DROP DATABASE IF EXISTS "`+database+`" WITH (FORCE)`); err != nil { + t.Errorf("drop isolated database: %s", scrub(err)) + return + } + pgCompMarkDropped(database) + }) + + if q := u.Query(); q.Get("dbname") != "" { + q.Del("dbname") + } + u.Path = "/" + database + dsn := u.String() + secrets = append(secrets, dsn) + + iso, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect to isolated database: %s", scrub(err)) + } + defer iso.Close() + schemas := map[control.Series]string{} + for _, placement := range pgCompSeriesTokens { + schema := fmt.Sprintf(pgCompSchemaFmt, time.Now().UnixNano(), pgCompRandomHex(t), placement.token) + if !pgCompSchemaPattern.MatchString(schema) { + t.Fatalf("minted schema %q violates the owned-name pattern", schema) + } + if _, err := iso.Exec(ctx, `CREATE SCHEMA "`+schema+`"`); err != nil { + t.Fatalf("create owned schema for %s: %s", placement.series, scrub(err)) + } + schemas[placement.series] = schema + } + return &pgCompFixture{dsn: dsn, schemas: schemas, scrub: scrub, secrets: secrets, admin: admin, database: database} +} + +func (f *pgCompFixture) migrate(t *testing.T) { + t.Helper() + runtime, err := backendspostgres.Open(context.Background(), control.Config{ + DSN: f.dsn, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 30 * time.Second, + Schemas: control.SeriesSchemas(f.schemas), + Setup: control.SetupMigrate, + }) + if err != nil { + t.Fatalf("migrate six-series fixture: %s", f.scrub(err)) + } + if err := runtime.Close(); err != nil { + t.Fatalf("close migrated fixture runtime: %s", f.scrub(err)) + } +} + +func (f *pgCompFixture) catalogRuntime(t *testing.T) *backendspostgres.Runtime { + t.Helper() + runtime, err := backendspostgres.Open(context.Background(), control.Config{ + DSN: f.dsn, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 30 * time.Second, + Schemas: control.SeriesSchemas(f.schemas), + Setup: control.SetupNone, + }) + if err != nil { + t.Fatalf("open catalog fixture runtime: %s", f.scrub(err)) + } + if runtime.FieldProfiles == nil || runtime.Inventory == nil || runtime.Relationships == nil || runtime.Contributions == nil { + _ = runtime.Close() + t.Fatal("fixture runtime assembled without catalog evidence ports") + } + return runtime +} + +func (f *pgCompFixture) pgOptions(application string) ingest.PGSourceOptions { + return ingest.PGSourceOptions{ + DSN: f.dsn, + Schema: f.schemas[control.SeriesSource], + ManifestSchema: f.schemas[control.SeriesDatasetRelease], + ApplicationName: application, + } +} + +func (f *pgCompFixture) load(t *testing.T, project, generation string, files map[string][]string) { + t.Helper() + dir := t.TempDir() + for name, lines := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil { + t.Fatalf("write staging fixture: %v", err) + } + } + catalog := f.catalogRuntime(t) + defer func() { _ = catalog.Close() }() + ref, err := publication.NewRef(project, generation) + if err != nil { + t.Fatalf("build dataset ref: %v", err) + } + opts := ingest.LoadOptions{ + Schema: pgCompRepoPath(t, "schemas", "graph-fhir.json"), + MetaDir: dir, + Project: project, + Dataset: &ref, + AuthResourcePath: "/fhir", + StageOnly: true, + } + pg := f.pgOptions(pgCompAppTag) + pg.Catalog = ingest.GenerationCatalogWriters{ + FieldProfiles: catalog.FieldProfiles, + Inventory: catalog.Inventory, + Relationships: catalog.Relationships, + Contributions: catalog.Contributions, + } + if _, err := ingest.LoadPostgresGeneration(context.Background(), opts, pg); err != nil { + t.Fatalf("load %s/%s: %s", project, generation, f.scrub(err)) + } +} + +func (f *pgCompFixture) activate(t *testing.T, project, generation string) { + t.Helper() + ref, err := publication.NewRef(project, generation) + if err != nil { + t.Fatalf("build dataset ref: %v", err) + } + if err := ingest.ActivatePostgresGeneration(context.Background(), ref, f.pgOptions(pgCompAppTag+"-activate")); err != nil { + t.Fatalf("activate %s/%s: %s", project, generation, f.scrub(err)) + } +} + +func pgCompRepoPath(t *testing.T, elems ...string) string { + t.Helper() + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("locate test file for repository root") + } + dir := filepath.Dir(thisFile) + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return filepath.Join(append([]string{dir}, elems...)...) + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("no go.mod found above the test file") + } + dir = parent + } +} + +func pgCompBootConfig(t *testing.T, f *pgCompFixture, listen string) Config { + t.Helper() + u, err := url.Parse(f.dsn) + if err != nil { + t.Fatalf("parse fixture dsn: %v", err) + } + host, portText, err := net.SplitHostPort(u.Host) + if err != nil { + t.Fatalf("split fixture dsn host: %v", err) + } + port, err := strconv.Atoi(portText) + if err != nil { + t.Fatalf("fixture dsn port %q: %v", portText, err) + } + password := "" + if pw, ok := u.User.Password(); ok { + password = pw + } + sslMode := u.Query().Get("sslmode") + if sslMode == "" { + sslMode = "disable" + } + cfg := Config{} + cfg.Server.Backend = sourceBackendPostgres + cfg.Server.Listen = listen + cfg.Server.AllowUnauthenticated = true + cfg.Server.Schema = pgCompRepoPath(t, "schemas", "graph-fhir.json") + cfg.Server.ArtifactDirectory = t.TempDir() + cfg.Server.PopulationMappingCursorSecret = "pgcomp-cursor-secret-0123456789ab" + schemas := map[string]string{} + for _, placement := range pgCompSeriesTokens { + schemas[string(placement.series)] = f.schemas[placement.series] + } + cfg.Server.PostgreSQL = PostgreSQLConfig{ + Host: host, Port: port, Database: strings.TrimPrefix(u.Path, "/"), + User: u.User.Username(), Password: password, SSLMode: sslMode, + MaxOpenConns: 8, MaxIdleConns: 2, + Schemas: schemas, + } + return cfg +} + +func pgCompPatientLines(t *testing.T, ids ...string) []string { + t.Helper() + lines := make([]string, 0, len(ids)) + for _, id := range ids { + encoded, err := json.Marshal(map[string]any{ + "resourceType": "Patient", "id": id, + "gender": "female", "birthDate": "1900-01-01", + }) + if err != nil { + t.Fatalf("marshal patient fixture: %v", err) + } + lines = append(lines, string(encoded)) + } + return lines +} + +func pgCompGraphQL(t *testing.T, app *fiber.App, body string) testHTTPResponse { + t.Helper() + return requestJSON(t, app, http.MethodPost, "/graphql/graph", body) +} + +func pgCompAssertNoLaneLeak(t *testing.T, f *pgCompFixture, what, body string) { + t.Helper() + lower := strings.ToLower(body) + for _, tell := range []string{"arango", "aql", "for d in ", "@auth_resource_paths", "fhir_field_catalog"} { + if strings.Contains(lower, tell) { + t.Fatalf("%s leaks a non-PostgreSQL surface (%q): %s", what, tell, f.scrub(body)) + } + } + if strings.Contains(lower, firstQueryRouteDeferral) { + t.Fatalf("%s resurfaced the first-query-route deferral on the full runtime: %s", what, f.scrub(body)) + } +} + +func TestPostgresRuntimeFullLadderServesProductionReads(t *testing.T) { + fixture := newPgCompFixture(t) + fixture.migrate(t) + fixture.load(t, pgCompProject, pgCompGenA, map[string][]string{ + "Patient.ndjson": pgCompPatientLines(t, "patient-ada", "patient-alan"), + }) + fixture.activate(t, pgCompProject, pgCompGenA) + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + server, closeRuntime, err := composePostgreSQLFullRuntime(ctx, pgCompBootConfig(t, fixture, "127.0.0.1:0"), logger) + if err != nil { + t.Fatalf("compose full runtime: %s", fixture.scrub(err)) + } + defer func() { + if err := closeRuntime(); err != nil { + t.Errorf("close full runtime: %s", fixture.scrub(err)) + } + }() + app := server.App() + + t.Run("generation_status_is_a_live_manifest_read", func(t *testing.T) { + response := requestJSON(t, app, http.MethodGet, "/api/v1/datasets/"+pgCompProject+"/generations/"+pgCompGenA, "") + pgCompAssertNoLaneLeak(t, fixture, "generation status", response.Body) + if response.StatusCode != http.StatusOK || !strings.Contains(response.Body, pgCompGenA) { + t.Fatalf("active-generation status = %d %s, want 200 naming %s", response.StatusCode, fixture.scrub(response.Body), pgCompGenA) + } + }) + + t.Run("run_fhir_dataframe_serves_exact_scalars", func(t *testing.T) { + body := `{"query":"mutation { runFhirDataframe(input: {project: \"` + pgCompProject + `\", rootResourceType: \"Patient\", rootFields: [{name: \"gender\", selector: {valuePath: \"gender\"}}, {name: \"birthDate\", selector: {valuePath: \"birthDate\"}}]}) { columns rowCount rows } }"}` + response := pgCompGraphQL(t, app, body) + pgCompAssertNoLaneLeak(t, fixture, "runFhirDataframe", response.Body) + if response.StatusCode != http.StatusOK || !strings.Contains(response.Body, "\"rowCount\":2") || strings.Contains(response.Body, "\"errors\"") { + t.Fatalf("runFhirDataframe must serve 2 rows from PostgreSQL, got %d %s", response.StatusCode, fixture.scrub(response.Body)) + } + }) + + graphqlLegs := []struct{ name, query string }{ + {"recipe_validate", `{"query":"mutation { validateDataframeRecipe(input: {name: \"r\", bindings: {project: \"` + pgCompProject + `\"}}) { name } }"}`}, + {"recipe_explain", `{"query":"query { explainDataframeRecipe(input: {name: \"r\", bindings: {project: \"` + pgCompProject + `\"}}) { name } }"}`}, + {"fhirGraph_traversal", `{"query":"query { fhirGraph(input: {project: \"` + pgCompProject + `\", rootResourceType: \"Patient\"}) { returnedCount } }"}`}, + } + for _, leg := range graphqlLegs { + leg := leg + t.Run(leg.name, func(t *testing.T) { + response := pgCompGraphQL(t, app, leg.query) + t.Logf("%s full-runtime answer: status=%d body=%s", leg.name, response.StatusCode, fixture.scrub(response.Body)) + pgCompAssertNoLaneLeak(t, fixture, leg.name, response.Body) + if response.StatusCode >= 500 && response.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("%s returned an untyped server fault %d: %s", leg.name, response.StatusCode, fixture.scrub(response.Body)) + } + }) + } + + if clickhouseURL := strings.TrimSpace(os.Getenv("LOOM_CLICKHOUSE_URL")); clickhouseURL != "" { + t.Run("clickhouse_client_survives_compose_return", func(t *testing.T) { + chConfig := pgCompBootConfig(t, fixture, "127.0.0.1:0") + chConfig.Server.ClickHouse = ClickHouseConfig{ + Enabled: true, + URL: clickhouseURL, + Database: os.Getenv("LOOM_CLICKHOUSE_DATABASE"), + Username: os.Getenv("LOOM_CLICKHOUSE_USERNAME"), + Password: os.Getenv("LOOM_CLICKHOUSE_PASSWORD"), + } + chServer, chClose, err := composePostgreSQLFullRuntime(ctx, chConfig, logger) + if err != nil { + t.Fatalf("compose ClickHouse-enabled full runtime: %s", fixture.scrub(err)) + } + defer func() { + if err := chClose(); err != nil { + t.Errorf("close ClickHouse-enabled full runtime: %s", fixture.scrub(err)) + } + }() + body, status := chServer.Readiness(ctx) + dataframe, _ := body["dataframe"].(string) + if status != http.StatusOK || body["status"] != "ready" || dataframe != "ready" { + t.Fatalf("ClickHouse-enabled readiness after compose returned = %d %#v; want 200 ready/ready (a client closed by compose instead of the closer fails the Ping)", status, body) + } + }) + } +} + +func TestPostgresRuntimeServerLoadRouteServesEndToEnd(t *testing.T) { + fixture := newPgCompFixture(t) + fixture.migrate(t) + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + server, closeRuntime, err := composePostgreSQLFullRuntime(ctx, pgCompBootConfig(t, fixture, "127.0.0.1:0"), logger) + if err != nil { + t.Fatalf("compose full runtime: %s", fixture.scrub(err)) + } + defer func() { + if err := closeRuntime(); err != nil { + t.Errorf("close full runtime: %s", fixture.scrub(err)) + } + }() + app := server.App() + + generation := "generation-server-loaded" + body, boundary := pgCompMultipart(t, + map[string]string{"auth_resource_path": "/fhir"}, + map[string]string{ + "Patient.ndjson": "{\"resourceType\":\"Patient\",\"id\":\"patient-server-loaded\",\"gender\":\"male\",\"birthDate\":\"1988-02-02\"}\n", + }) + request := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/"+pgCompProject+"/generations/"+generation, bytes.NewReader(body)) + request.Header.Set("Content-Type", "multipart/form-data; boundary="+boundary) + response, err := app.Test(request, fiber.TestConfig{Timeout: 60 * time.Second, FailOnTimeout: true}) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + raw, _ := io.ReadAll(response.Body) + text := string(raw) + t.Logf("server-load answer: status=%d body=%s", response.StatusCode, fixture.scrub(text)) + pgCompAssertNoLaneLeak(t, fixture, "server-side load", text) + if response.StatusCode < 200 || response.StatusCode >= 300 { + t.Fatalf("server-side load must serve through the PostgreSQL ingest lane, got %d %s", response.StatusCode, fixture.scrub(text)) + } + + { + witness := fixture.catalogRuntime(t) + defer func() { _ = witness.Close() }() + active, err := witness.Manifests.ResolveActiveManifest(context.Background(), pgCompProject) + if err != nil { + t.Fatalf("resolve active pointer after server-side load: %s", fixture.scrub(err)) + } + if active.Dataset.Generation != generation { + t.Fatalf("active pointer = %s, want the server-loaded %s", active.Dataset.Generation, generation) + } + } + + status := requestJSON(t, app, http.MethodGet, "/api/v1/datasets/"+pgCompProject+"/generations/"+generation, "") + pgCompAssertNoLaneLeak(t, fixture, "loaded-generation status", status.Body) + if status.StatusCode != http.StatusOK || !strings.Contains(status.Body, generation) || + !strings.Contains(status.Body, "\"reusable\":true") { + t.Fatalf("server-loaded generation must be a persisted reusable manifest, got %d %s", status.StatusCode, fixture.scrub(status.Body)) + } + + query := `{"query":"mutation { runFhirDataframe(input: {project: \"` + pgCompProject + `\", rootResourceType: \"Patient\", rootFields: [{name: \"gender\", selector: {valuePath: \"gender\"}}]}) { rowCount rows } }"}` + served := pgCompGraphQL(t, app, query) + pgCompAssertNoLaneLeak(t, fixture, "server-loaded read", served.Body) + if served.StatusCode != http.StatusOK || !strings.Contains(served.Body, "\"rowCount\":1") || + !strings.Contains(served.Body, "patient-server-loaded") { + t.Fatalf("the read lane must serve what the server loaded, got %d %s", served.StatusCode, fixture.scrub(served.Body)) + } +} + +func TestPostgresRuntimeHealthReadinessIsIndependentAndSchemaAware(t *testing.T) { + markReadinessLeg(t, "TestPostgresRuntimeHealthReadinessIsIndependentAndSchemaAware") + fixture := newPgCompFixture(t) + fixture.migrate(t) + + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + defer cancel() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + server, closeRuntime, err := composePostgreSQLFullRuntime(ctx, pgCompBootConfig(t, fixture, "127.0.0.1:0"), logger) + if err != nil { + t.Fatalf("compose full runtime: %s", fixture.scrub(err)) + } + var closeOnce sync.Once + closeNow := func() { + closeOnce.Do(func() { + if err := closeRuntime(); err != nil { + t.Errorf("close full runtime: %s", fixture.scrub(err)) + } + }) + } + defer closeNow() + app := server.App() + + t.Run("ready_and_disabled_dataframe_on_a_migrated_backend", func(t *testing.T) { + response := requestJSON(t, app, http.MethodGet, "/readyz", "") + pgCompAssertNoLaneLeak(t, fixture, "readyz", response.Body) + if response.StatusCode != http.StatusOK || + !strings.Contains(response.Body, `"status":"ready"`) || + !strings.Contains(response.Body, `"core":"ready"`) || + !strings.Contains(response.Body, `"dataframe":"disabled"`) { + t.Fatalf("migrated full runtime readiness = %d %s; want 200 status=ready core=ready dataframe=disabled", + response.StatusCode, fixture.scrub(response.Body)) + } + live := requestJSON(t, app, http.MethodGet, "/livez", "") + if live.StatusCode != http.StatusOK || live.Body != `{"status":"live"}` { + t.Fatalf("livez on the full runtime = %d %s; want 200 {\"status\":\"live\"}", live.StatusCode, fixture.scrub(live.Body)) + } + }) + + t.Run("clickhouse_failure_never_hides_the_selected_backend", func(t *testing.T) { + chConfig := pgCompBootConfig(t, fixture, "127.0.0.1:0") + chConfig.Server.ClickHouse = ClickHouseConfig{ + Enabled: true, + URL: "http://127.0.0.1:1", + Database: "loom_health_probe", + } + chServer, chClose, err := composePostgreSQLFullRuntime(ctx, chConfig, logger) + if err != nil { + t.Fatalf("compose ClickHouse-dead full runtime: EnsureDatabase must degrade, not kill startup: %s", fixture.scrub(err)) + } + defer func() { + if err := chClose(); err != nil { + t.Errorf("close ClickHouse-dead runtime: %s", fixture.scrub(err)) + } + }() + response := requestJSON(t, chServer.App(), http.MethodGet, "/readyz", "") + pgCompAssertNoLaneLeak(t, fixture, "ClickHouse-dead readyz", response.Body) + if response.StatusCode != http.StatusServiceUnavailable || + !strings.Contains(response.Body, `"core":"ready"`) || + !strings.Contains(response.Body, `"dataframe":"backend_unavailable"`) { + t.Fatalf("ClickHouse-down readiness = %d %s; want 503 with core=ready AND dataframe=backend_unavailable", + response.StatusCode, fixture.scrub(response.Body)) + } + health := requestJSON(t, chServer.App(), http.MethodGet, "/health", "") + if health.StatusCode != http.StatusOK || + !strings.Contains(health.Body, `"status":"degraded"`) || + !strings.Contains(health.Body, `"core":"ready"`) { + t.Fatalf("ClickHouse-down /health = %d %s; want 200 degraded with core=ready", health.StatusCode, fixture.scrub(health.Body)) + } + }) + + t.Run("schema_mismatch_yields_not_ready_and_repairs_back_to_ready", func(t *testing.T) { + pool, err := pgxpool.New(ctx, fixture.dsn) + if err != nil { + t.Fatalf("connect ledger mutator: %s", fixture.scrub(err)) + } + defer pool.Close() + ledger := `"` + fixture.schemas[control.SeriesSource] + `".schema_migrations` + var version int64 + var original string + if err := pool.QueryRow(ctx, "SELECT version, checksum FROM "+ledger+" ORDER BY version LIMIT 1").Scan(&version, &original); err != nil { + t.Fatalf("read ledger row: %s", fixture.scrub(err)) + } + restore := func(when context.Context) { + if _, err := pool.Exec(when, "UPDATE "+ledger+" SET checksum = $1 WHERE version = $2", original, version); err != nil { + t.Errorf("restore ledger checksum: %s", fixture.scrub(err)) + } + } + defer restore(context.WithoutCancel(context.Background())) + if _, err := pool.Exec(ctx, "UPDATE "+ledger+" SET checksum = $1 WHERE version = $2", strings.Repeat("0", 64), version); err != nil { + t.Fatalf("corrupt ledger checksum: %s", fixture.scrub(err)) + } + response := requestJSON(t, app, http.MethodGet, "/readyz", "") + pgCompAssertNoLaneLeak(t, fixture, "schema-mismatch readyz", response.Body) + if response.StatusCode != http.StatusServiceUnavailable || + !strings.Contains(response.Body, `"core":"not_ready"`) || + !strings.Contains(response.Body, `"core_state":"schema_mismatch"`) { + t.Fatalf("edited-ledger readiness = %d %s; want 503 core=not_ready core_state=schema_mismatch", + response.StatusCode, fixture.scrub(response.Body)) + } + restore(ctx) + recovered := requestJSON(t, app, http.MethodGet, "/readyz", "") + if recovered.StatusCode != http.StatusOK || !strings.Contains(recovered.Body, `"core":"ready"`) { + t.Fatalf("repaired ledger must flip /readyz back without cache, got %d %s", + recovered.StatusCode, fixture.scrub(recovered.Body)) + } + }) + + t.Run("closed_pool_keeps_liveness_and_refuses_readiness", func(t *testing.T) { + closeNow() + live := requestJSON(t, app, http.MethodGet, "/livez", "") + if live.StatusCode != http.StatusOK || live.Body != `{"status":"live"}` { + t.Fatalf("livez with a closed pool = %d %s; liveness must be DB-free", live.StatusCode, fixture.scrub(live.Body)) + } + response := requestJSON(t, app, http.MethodGet, "/readyz", "") + if response.StatusCode != http.StatusServiceUnavailable || + !strings.Contains(response.Body, `"core":"not_ready"`) || + !strings.Contains(response.Body, `"core_state":"runtime_closed"`) { + t.Fatalf("readyz with a closed pool = %d %s; want 503 core=not_ready core_state=runtime_closed", + response.StatusCode, fixture.scrub(response.Body)) + } + }) +} + +func TestPostgresRuntimeReadinessReportsNeverMigratedSchema(t *testing.T) { + markReadinessLeg(t, "TestPostgresRuntimeReadinessReportsNeverMigratedSchema") + fixture := newPgCompFixture(t) + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + server, closeRuntime, err := composePostgreSQLFullRuntime(ctx, pgCompBootConfig(t, fixture, "127.0.0.1:0"), logger) + if err != nil { + t.Fatalf("compose over never-migrated schemas (open is version-only): %s", fixture.scrub(err)) + } + defer func() { + if err := closeRuntime(); err != nil { + t.Errorf("close never-migrated runtime: %s", fixture.scrub(err)) + } + }() + app := server.App() + + response := requestJSON(t, app, http.MethodGet, "/readyz", "") + pgCompAssertNoLaneLeak(t, fixture, "never-migrated readyz", response.Body) + if response.StatusCode != http.StatusServiceUnavailable || + !strings.Contains(response.Body, `"core":"not_ready"`) || + !strings.Contains(response.Body, `"core_state":"schema_unmigrated"`) { + t.Fatalf("never-migrated readiness = %d %s; want 503 core=not_ready core_state=schema_unmigrated", + response.StatusCode, fixture.scrub(response.Body)) + } + if strings.Contains(strings.ToLower(response.Body), "password") || strings.Contains(response.Body, fixture.dsn) { + t.Fatalf("readiness body leaked connection material: %s", fixture.scrub(response.Body)) + } + live := requestJSON(t, app, http.MethodGet, "/livez", "") + if live.StatusCode != http.StatusOK || live.Body != `{"status":"live"}` { + t.Fatalf("livez on never-migrated schemas = %d %s; liveness is dependency-free", live.StatusCode, fixture.scrub(live.Body)) + } +} + +func TestPostgresRuntimeVerifySchemaIsReadOnly(t *testing.T) { + markReadinessLeg(t, "TestPostgresRuntimeVerifySchemaIsReadOnly") + fixture := newPgCompFixture(t) + fixture.migrate(t) + + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + defer cancel() + backendRuntime, err := backendspostgres.Open(ctx, control.Config{ + DSN: fixture.dsn, + MaxConns: 4, + MinConns: 1, + AcquireTimeout: 30 * time.Second, + Schemas: control.SeriesSchemas(fixture.schemas), + Setup: control.SetupNone, + }) + if err != nil { + t.Fatalf("open setup-none control runtime for readiness witness: %s", fixture.scrub(err)) + } + defer backendRuntime.Close() + pool, err := pgxpool.New(ctx, fixture.dsn) + if err != nil { + t.Fatalf("connect fingerprint reader: %s", fixture.scrub(err)) + } + defer pool.Close() + + fingerprint := func(schema string) map[string]string { + t.Helper() + out := map[string]string{} + queries := map[string]string{ + "ledger": "SELECT COALESCE(md5(string_agg(version::text || '|' || name || '|' || checksum || '|' || applied_at::text, E'\n' ORDER BY version)), 'empty') " + + `FROM "` + schema + `".schema_migrations`, + "tables": "SELECT COALESCE(md5(string_agg(c.relkind::text || ':' || c.relname, ',' ORDER BY c.relkind, c.relname)), 'empty') " + + "FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace " + + "WHERE n.nspname = $1 AND c.relkind IN ('r','p','i','S')", + "columns": "SELECT COALESCE(md5(string_agg(t.relname || '.' || a.attname || ':' || format_type(a.atttypid, a.atttypmod) || ':' || a.attnotnull::text || ':' || a.attgenerated::text, ',' ORDER BY t.relname, a.attnum)), 'empty') " + + "FROM pg_attribute a JOIN pg_class t ON t.oid = a.attrelid JOIN pg_namespace n ON n.oid = t.relnamespace " + + "WHERE n.nspname = $1 AND a.attnum > 0 AND NOT a.attisdropped AND t.relkind IN ('r','p')", + "indexes": "SELECT COALESCE(md5(string_agg(tablename || '.' || indexname || ':' || indexdef, ',' ORDER BY tablename, indexname)), 'empty') " + + "FROM pg_indexes WHERE schemaname = $1", + } + for _, key := range []string{"ledger", "tables", "columns", "indexes"} { + var fp string + var err error + if key == "ledger" { + err = pool.QueryRow(ctx, queries[key]).Scan(&fp) + } else { + err = pool.QueryRow(ctx, queries[key], schema).Scan(&fp) + } + if err != nil { + t.Fatalf("fingerprint %s of schema %s: %s", key, schema, fixture.scrub(err)) + } + out[key] = fp + } + return out + } + + assigned := map[control.Series]string{} + for _, series := range control.AllSeries() { + if schema, ok := fixture.schemas[series]; ok { + assigned[series] = schema + } + } + if len(assigned) == 0 { + t.Fatal("fixture assigned no series; the readiness read-only witness would be vacuous") + } + pre := map[control.Series]map[string]string{} + for series, schema := range assigned { + pre[series] = fingerprint(schema) + } + if err := backendRuntime.Readiness(ctx); err != nil { + t.Fatalf("Readiness over the migrated assigned schemas must pass: %s", fixture.scrub(err)) + } + for _, series := range control.AllSeries() { + schema, ok := assigned[series] + if !ok { + continue + } + series, schema := series, schema + t.Run(string(series), func(t *testing.T) { + post := fingerprint(schema) + for key, before := range pre[series] { + if post[key] != before { + t.Fatalf("Readiness on series %q mutated %s of schema %s: fingerprint %s -> %s", series, key, schema, before, post[key]) + } + } + }) + } +} + +func TestPostgresRuntimeReadinessReportsMissingSchema(t *testing.T) { + markReadinessLeg(t, "TestPostgresRuntimeReadinessReportsMissingSchema") + fixture := newPgCompFixture(t) + fixture.migrate(t) + + cfg := pgCompBootConfig(t, fixture, "127.0.0.1:0") + cfg.Server.PostgreSQL.Schemas[string(control.SeriesRecipeExecRegistries)] = + fmt.Sprintf(pgCompSchemaFmt, time.Now().UnixNano(), pgCompRandomHex(t), "exec_missing") + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + server, closeRuntime, err := composePostgreSQLFullRuntime(ctx, cfg, logger) + if err != nil { + t.Fatalf("compose over one never-created schema placement (open is version-only): %s", fixture.scrub(err)) + } + defer func() { + if err := closeRuntime(); err != nil { + t.Errorf("close missing-schema runtime: %s", fixture.scrub(err)) + } + }() + app := server.App() + + response := requestJSON(t, app, http.MethodGet, "/readyz", "") + pgCompAssertNoLaneLeak(t, fixture, "missing-schema readyz", response.Body) + if response.StatusCode != http.StatusServiceUnavailable || + !strings.Contains(response.Body, `"core":"not_ready"`) || + !strings.Contains(response.Body, `"core_state":"schema_missing"`) { + t.Fatalf("never-created-schema readiness = %d %s; want 503 core=not_ready core_state=schema_missing (F1: this token was unreachable while the facade aliased a duplicate sentinel)", + response.StatusCode, fixture.scrub(response.Body)) + } + if strings.Contains(strings.ToLower(response.Body), "password") || strings.Contains(response.Body, fixture.dsn) { + t.Fatalf("readiness body leaked connection material: %s", fixture.scrub(response.Body)) + } + live := requestJSON(t, app, http.MethodGet, "/livez", "") + if live.StatusCode != http.StatusOK || live.Body != `{"status":"live"}` { + t.Fatalf("livez on a missing-schema placement = %d %s; liveness is dependency-free", live.StatusCode, fixture.scrub(live.Body)) + } +} + +func TestPgCompCensusVocabularyCorrespondence(t *testing.T) { + hex := func() string { + var b [6]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatalf("sample suffix: %v", err) + } + return fmt.Sprintf("%x", b[:]) + } + for i := 0; i < 200; i++ { + name := fmt.Sprintf(pgCompDBFmt, time.Now().UnixNano(), hex()) + if !pgCompDBPattern.MatchString(name) { + t.Fatalf("mint-shaped name %q does not match the own census regex", name) + } + for _, token := range pgCompSeriesTokens { + schema := fmt.Sprintf(pgCompSchemaFmt, time.Now().UnixNano(), hex(), token.token) + if !pgCompSchemaPattern.MatchString(schema) { + t.Fatalf("mint-shaped schema %q does not match the own schema regex", schema) + } + if pgCompDBPattern.MatchString(schema) { + t.Fatalf("schema name %q would double-count as a database in the census", schema) + } + } + } + positives := []string{"verify_pgcomp_1731234567890_0123456789ab"} + negatives := []string{ + "verify_pgcomp_", + "verify_pgcomp_1731234567890_", + "verify_pgcomp__0123456789ab", + "verify_pgcomp_1731234567890_0123456789AB", + "verify_pgcomp_1731234567890_0123456789ab_src", + "verify_pgcomp_1731234567890_0123456789abzz", + "verify_qroute_1731234567890_0123456789ab", + "xverify_pgcomp_1731234567890_0123456789ab", + } + for _, name := range positives { + if !pgCompDBPattern.MatchString(name) { + t.Errorf("positive %q must match the own census regex", name) + } + } + for _, name := range negatives { + if pgCompDBPattern.MatchString(name) { + t.Errorf("never-minted shape %q matched the own census regex", name) + } + } +} + +func TestZZPgCompZeroLeftoverFixtures(t *testing.T) { + raw := pgCompTestPostgresURL(t) + if !pgCompBaselineSet { + t.Skip("no live fixture was created in this run; nothing to audit") + } + if pgCompBaselineErr != nil { + t.Fatalf("baseline was never captured: %v", pgCompBaselineErr) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + admin, err := pgxpool.New(ctx, raw) + if err != nil { + t.Fatalf("connect for audit: %s", err) + } + defer admin.Close() + var count int64 + if err := admin.QueryRow(ctx, + "SELECT count(*) FROM pg_database WHERE datname ~ '^verify_pgcomp_[0-9]+_[0-9a-f]+$'"). + Scan(&count); err != nil { + t.Fatalf("count leftovers: %s", err) + } + pgCompOwnedMu.Lock() + registered, outstanding, strays := 0, []string{}, append([]string(nil), pgCompStrayMarks...) + perLeg := map[string]int{} + unmatched := []string{} + for db, rec := range pgCompOwned { + registered++ + perLeg[rec.leg]++ + if !rec.dropped { + outstanding = append(outstanding, db) + } + if !pgCompDBPattern.MatchString(db) { + unmatched = append(unmatched, db) + } + } + pgCompOwnedMu.Unlock() + if len(outstanding) > 0 { + t.Fatalf("owned fixtures still live at audit: %v", outstanding) + } + if len(strays) > 0 { + t.Fatalf("drop marks for names this process never minted: %v", strays) + } + if len(unmatched) > 0 { + t.Fatalf("registered mints outside the census regex: %v", unmatched) + } + for leg, want := range pgCompMintInventory { + if got := perLeg[leg]; got != want { + t.Errorf("mint inventory: leg %q minted %d databases, inventoried %d", leg, got, want) + } + delete(perLeg, leg) + } + for leg, want := range pgCompClickHouseGatedMintInventory { + if !pgCompClickHouseConfigured() { + if got := perLeg[leg]; got != 0 { + t.Errorf("mint inventory: ClickHouse-gated leg %q minted %d databases while LOOM_CLICKHOUSE_URL is unconfigured", leg, got) + } + delete(perLeg, leg) + continue + } + if got := perLeg[leg]; got != want { + t.Errorf("mint inventory: ClickHouse-enabled leg %q minted %d databases, inventoried %d", leg, got, want) + } + delete(perLeg, leg) + } + for leg, got := range perLeg { + t.Errorf("mint inventory: leg %q minted %d databases but is not inventoried", leg, got) + } + if count != pgCompBaseline { + rows, err := admin.Query(ctx, + "SELECT datname FROM pg_database WHERE datname LIKE 'verify\\_pgcomp\\_%' ESCAPE '\\' ORDER BY datname") + names := []string{} + if err == nil { + for rows.Next() { + var name string + if err := rows.Scan(&name); err == nil { + names = append(names, name) + } + } + rows.Close() + } + t.Fatalf("own-prefix census: live=%d baseline=%d registered=%d names=%v", count, pgCompBaseline, registered, names) + } +} + +func pgCompMultipart(t *testing.T, fields, files map[string]string) ([]byte, string) { + t.Helper() + var buffer bytes.Buffer + writer := multipart.NewWriter(&buffer) + for name, value := range fields { + if err := writer.WriteField(name, value); err != nil { + t.Fatalf("write multipart field: %v", err) + } + } + for name, fileBody := range files { + part, err := writer.CreateFormFile("file", name) + if err != nil { + t.Fatalf("create multipart file: %v", err) + } + if _, err := part.Write([]byte(fileBody)); err != nil { + t.Fatalf("write multipart file: %v", err) + } + } + if err := writer.Close(); err != nil { + t.Fatalf("close multipart: %v", err) + } + return buffer.Bytes(), writer.Boundary() +} diff --git a/internal/server/runtime_test.go b/internal/server/runtime_test.go new file mode 100644 index 000000000..11f916584 --- /dev/null +++ b/internal/server/runtime_test.go @@ -0,0 +1,495 @@ +package server + +import ( + "context" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io" + "net" + "os" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgconn" + + "github.com/calypr/loom/internal/authscope" + backendsarango "github.com/calypr/loom/internal/backends/arango" + "github.com/calypr/loom/internal/backends/postgres/control" + dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" + "github.com/calypr/loom/internal/dataframe/recipe" + publicationcontract "github.com/calypr/loom/internal/dataset" + "github.com/calypr/loom/internal/explorer" + "github.com/calypr/loom/internal/ingest" +) + +var ( + _ receiptExecutionFinder = (*backendsarango.PublishedRegistry)(nil) + _ receiptExecutionFinder = (*control.BundleCatalog)(nil) +) + +func fullLadderSchemas() map[string]string { + return map[string]string{ + "source": "loom_source", + "explorer_control": "loom_explorer", + "publication_control": "loom_publication", + "dataset_release": "loom_release", + "recipe_chains": "loom_recipe", + "recipe_exec_registries": "loom_exec", + } +} + +func TestPostgresRuntimeAssignmentAcceptsFullLadder(t *testing.T) { + assignment, names, err := postgresRuntimeAssignment(PostgreSQLConfig{Schemas: fullLadderSchemas()}) + if err != nil { + t.Fatalf("the complete six-series ladder must be consumable: %v", err) + } + if len(assignment) != 6 || len(names) != 6 { + t.Fatalf("assignment must carry all six series, got %v", assignment) + } + for series, schema := range fullLadderSchemas() { + if string(assignment[control.Series(series)]) != schema { + t.Fatalf("series %s must bind schema %s, got %v", series, schema, assignment) + } + if names[control.Series(series)] != schema { + t.Fatalf("resolved names must mirror the assignment for %s", series) + } + } +} + +func TestPostgresRuntimeAssignmentKeepsTheSentinelForEveryOtherShape(t *testing.T) { + cases := []struct { + name string + schemas map[string]string + }{ + {"absent", nil}, + {"empty", map[string]string{}}, + {"first-query pair is the route's own, not the ladder", map[string]string{"source": "s", "dataset_release": "r"}}, + {"ladder minus one", func() map[string]string { + s := fullLadderSchemas() + delete(s, "recipe_exec_registries") + return s + }()}, + {"ladder plus foreign", func() map[string]string { + s := fullLadderSchemas() + s["warehouse"] = "w" + return s + }()}, + {"duplicate schema across two series", func() map[string]string { + s := fullLadderSchemas() + s["recipe_chains"] = s["recipe_exec_registries"] + return s + }()}, + {"whitespace-colliding pair", func() map[string]string { + s := fullLadderSchemas() + s["recipe_chains"] = " loom_exec " + return s + }()}, + {"blank member", func() map[string]string { + s := fullLadderSchemas() + s["publication_control"] = " " + return s + }()}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, _, err := postgresRuntimeAssignment(PostgreSQLConfig{Schemas: tc.schemas}); !errors.Is(err, ErrPostgreSQLRuntimeNotWired) { + t.Fatalf("schemas %v must keep the byte-stable not-wired sentinel, got %v", tc.schemas, err) + } + }) + } +} + +func TestPostgresRuntimeDispatchReachesThePostgreSQLPathPreDial(t *testing.T) { + cfg := Config{} + cfg.Server.Backend = sourceBackendPostgres + cfg.Server.PostgreSQL = PostgreSQLConfig{ + Host: "192.0.2.1", Port: 60543, Database: "loom", User: "loom", + Password: "pgcomp-Canary-Pa55w0rd", SSLMode: "require", + MaxOpenConns: 2, MaxIdleConns: 1, + Schemas: map[string]string{ + "source": "Loom Source", + "explorer_control": "loom_explorer", + "publication_control": "loom_publication", + "dataset_release": "loom_release", + "recipe_chains": "loom_recipe", + "recipe_exec_registries": "loom_exec", + }, + } + ctx, cancel := context.WithTimeout(context.Background(), 10*1024*1024*1024) + defer cancel() + err := RunWithConfig(ctx, cfg) + if err == nil { + t.Fatal("ladder config unexpectedly started the server") + } + if errors.Is(err, ErrPostgreSQLRuntimeNotWired) { + t.Fatalf("a consumable ladder was refused by the not-wired sentinel: %v", err) + } + if strings.Contains(err.Error(), cfg.Server.PostgreSQL.Password) { + t.Fatalf("refusal leaked the connection password: %v", err) + } + if !strings.Contains(err.Error(), "Loom Source") && !strings.Contains(err.Error(), "schema") { + t.Fatalf("error %q must name the unsafe schema identifier or the schema law", err) + } +} + +type fakeRepositoryGenerationActivator struct { + err error + project string + generation string + revisionID string +} + +func (f *fakeRepositoryGenerationActivator) ActivateRepositoryGeneration(_ context.Context, project, generation, revisionID string) error { + f.project, f.generation, f.revisionID = project, generation, revisionID + return f.err +} + +func TestPostgresRuntimeExplorerStoreTypedRefusals(t *testing.T) { + store := pgExplorerStore{ExplorerStore: &control.ExplorerStore{}} + if err := store.ActivateRepositoryGeneration(context.Background(), "p", "g", "r"); err == nil { + t.Fatal("activation without the dataset_release/publication_control assignment must refuse") + } else if !isTypedExplicitUnavailable(err, "Repository-generation activation") { + t.Fatalf("refusal %v must be the typed explicit-unavailable posture", err) + } + fake := &fakeRepositoryGenerationActivator{} + store.activate = fake + if err := store.ActivateRepositoryGeneration(context.Background(), "p", "g", "r"); err != nil { + t.Fatalf("wired activation must ride the adapter without refusal: %v", err) + } + if fake.project != "p" || fake.generation != "g" || fake.revisionID != "r" { + t.Fatalf("activation arguments not delegated verbatim: %q/%q/%q", fake.project, fake.generation, fake.revisionID) + } + activationFault := errors.New("activation lost the race") + fake.err = activationFault + if err := store.ActivateRepositoryGeneration(context.Background(), "p", "g", "r"); !errors.Is(err, activationFault) { + t.Fatalf("adapter errors must pass through unwrapped and unrefused: %v", err) + } + if err := store.ValidateSelectionReferences(context.Background(), "p", "g", authscope.ReadScope{}, nil); err == nil { + t.Fatal("selection-reference validation must refuse without an Arango fall-through") + } else if !isTypedExplicitUnavailable(err, "Explicit-selection reference validation") { + t.Fatalf("refusal %v must be the typed explicit-unavailable posture", err) + } + if _, err := store.PublishAuthoring(context.Background(), explorer.CompilationReceipt{}, explorer.Revision{}, publicationcontract.ProjectRelease{}, 0); err == nil { + t.Fatal("publication without the dataset_release assignment must refuse") + } else if !isTypedExplicitUnavailable(err, "Explorer publication") { + t.Fatalf("refusal %v must be the typed explicit-unavailable posture", err) + } +} + +func isTypedExplicitUnavailable(err error, operationSubstring string) bool { + userErr, ok := dataframeerrors.AsUserError(err) + if !ok { + return false + } + if userErr.Code() != string(dataframeerrors.CodeBackendUnavailable) { + return false + } + if userErr.Retryable() { + return false + } + if !strings.Contains(userErr.Error(), operationSubstring) { + return false + } + backend, _ := userErr.Details()["backend"].(string) + return backend == "postgresql" +} + +func TestPostgresRuntimeAuthorizerNeverStampsTheArangoCollection(t *testing.T) { + pgRead, err := recipeAuthorization{}.AuthorizeRead(context.Background(), recipe.RuntimeBindings{Project: "p1", DatasetGeneration: "g"}) + if err != nil { + t.Fatalf("unrestricted read authorization: %v", err) + } + if pgRead.SelectionMembersCollection != "" { + t.Fatalf("the PostgreSQL authorizer must leave SelectionMembersCollection empty, got %q", pgRead.SelectionMembersCollection) + } + if pgRead.SelectionProject != "p1" { + t.Fatalf("SelectionProject must be the canonical project, got %q", pgRead.SelectionProject) + } + pgWrite, err := recipeAuthorization{}.AuthorizeWrite(context.Background(), recipe.RuntimeBindings{Project: "p1", DatasetGeneration: "g"}) + if err != nil { + t.Fatalf("unrestricted write authorization: %v", err) + } + if pgWrite.SelectionMembersCollection != "" { + t.Fatalf("the PostgreSQL write authorizer must leave SelectionMembersCollection empty, got %q", pgWrite.SelectionMembersCollection) + } + arangoRead, err := recipeAuthorization{selectionMembersCollection: backendsarango.SelectionMembersCollection}.AuthorizeRead(context.Background(), recipe.RuntimeBindings{Project: "p1", DatasetGeneration: "g"}) + if err != nil { + t.Fatalf("arango-lane read authorization: %v", err) + } + if arangoRead.SelectionMembersCollection == "" { + t.Fatal("the Arango lane must keep stamping its selection-members collection; the contrast fixture went vacuous") + } +} + +func TestPostgresRuntimeCompositionHasNoArangoImport(t *testing.T) { + source, err := os.ReadFile("runtime.go") + if err != nil { + t.Fatalf("read the composition source: %v", err) + } + text := string(source) + importBlock := text + if start := strings.Index(text, "import ("); start >= 0 { + if end := strings.Index(text[start:], "\n)"); end >= 0 { + importBlock = text[start : start+end] + } + } + forbidden := []string{ + "backends/arango", + "compiler/render/aql", + } + for _, path := range forbidden { + if strings.Contains(importBlock, path) { + t.Fatalf("the PostgreSQL composition must not import %s", path) + } + } + for _, token := range []string{"arangostore.", "AQLParameters"} { + if strings.Contains(text, token) { + t.Fatalf("the PostgreSQL composition must not mention %s (no AQL/Arango surface on the PostgreSQL lane)", token) + } + } +} + +func TestPostgresRuntimeComposeOwnsNoDeferredCleanup(t *testing.T) { + for _, arm := range []struct{ file, funcName string }{ + {file: "runtime.go", funcName: "composePostgreSQLFullRuntime"}, + {file: "compose.go", funcName: "composeArangoRuntime"}, + } { + arm := arm + t.Run(arm.funcName, func(t *testing.T) { + assertComposeOwnsNoDeferredCleanup(t, arm.file, arm.funcName) + }) + } +} + +func assertComposeOwnsNoDeferredCleanup(t *testing.T, file, funcName string) { + source, err := os.ReadFile(file) + if err != nil { + t.Fatalf("read the composition source: %v", err) + } + fileSet := token.NewFileSet() + parsed, err := parser.ParseFile(fileSet, file, source, parser.ParseComments) + if err != nil { + t.Fatalf("parse the composition source: %v", err) + } + var compose *ast.FuncDecl + for _, decl := range parsed.Decls { + if fn, ok := decl.(*ast.FuncDecl); ok && fn.Name.Name == funcName { + compose = fn + } + } + if compose == nil { + t.Fatalf("%s is missing from the composition source %s", funcName, file) + } + var bodyDefers []*ast.DeferStmt + var insideFuncLit []bool + ast.Inspect(compose.Body, func(n ast.Node) bool { + if n == nil { + insideFuncLit = insideFuncLit[:len(insideFuncLit)-1] + return false + } + nested := len(insideFuncLit) > 0 && insideFuncLit[len(insideFuncLit)-1] + _, isLiteral := n.(*ast.FuncLit) + insideFuncLit = append(insideFuncLit, nested || isLiteral) + if stmt, ok := n.(*ast.DeferStmt); ok && !nested { + bodyDefers = append(bodyDefers, stmt) + } + return true + }) + if len(bodyDefers) != 1 { + positions := make([]string, 0, len(bodyDefers)) + for _, d := range bodyDefers { + positions = append(positions, fileSet.Position(d.Pos()).String()) + } + t.Fatalf("compose must own exactly one deferred cleanup sweep (resources that outlive compose belong to the returned closer); found %d at %v", len(bodyDefers), positions) + } + sweep := bodyDefers[0] + for _, before := range compose.Body.List { + if before == sweep { + break + } + decl, isDecl := before.(*ast.DeclStmt) + if !isDecl { + t.Fatalf("nothing may run before the deferred cleanup sweep (it must precede every acquisition), found %T at %s", before, fileSet.Position(before.Pos()).String()) + } + gen, ok := decl.Decl.(*ast.GenDecl) + if !ok { + t.Fatalf("statements before the sweep must be plain declarations, found %T at %s", decl.Decl, fileSet.Position(before.Pos()).String()) + } + for _, spec := range gen.Specs { + if value, ok := spec.(*ast.ValueSpec); ok && len(value.Values) != 0 { + t.Fatalf("statements before the sweep must be bare declarations (an initialized var can acquire); found initializer at %s", fileSet.Position(spec.Pos()).String()) + } + } + } + literal, ok := sweep.Call.Fun.(*ast.FuncLit) + if !ok || len(literal.Type.Params.List) != 0 || literal.Type.Results != nil { + t.Fatalf("the deferred sweep must be a zero-argument closure, got %s", fileSet.Position(sweep.Pos()).String()) + } + if !expressionReferences(literal.Body, "cleanups") { + t.Fatal("the deferred sweep must drain the cleanups stack") + } + if !expressionReferences(literal.Body, "runRuntimeCloseSteps") { + t.Fatal("the deferred sweep must tear down through runRuntimeCloseSteps (one ordering implementation for both paths)") + } + composeText := string(source)[fileSet.Position(compose.Pos()).Offset:fileSet.Position(compose.End()).Offset] + if got := strings.Count(composeText, "cleanups = append("); got != 4 { + t.Fatalf("the full composition must push exactly four closer steps (pool, ClickHouse client, private-artifact reconciler, artifact-fs reconciler), found %d", got) + } + if !strings.Contains(composeText, "closeRuntime = func() error { return runRuntimeCloseSteps(cleanups) }") { + t.Fatal("the returned closer must be the reverse-order cleanups drain") + } +} + +func expressionReferences(node ast.Node, name string) bool { + found := false + ast.Inspect(node, func(n ast.Node) bool { + if ident, ok := n.(*ast.Ident); ok && ident.Name == name { + found = true + } + return true + }) + return found +} + +func TestPostgresRuntimeCloseStepsTearDownInReverseAndKeepFirstError(t *testing.T) { + var order []int + errAcquiredThird := errors.New("third-acquired step failed") + errAcquiredSecond := errors.New("second-acquired step failed") + steps := []func() error{ + func() error { order = append(order, 0); return nil }, + func() error { order = append(order, 1); return errAcquiredSecond }, + func() error { order = append(order, 2); return nil }, + func() error { order = append(order, 3); return errAcquiredThird }, + func() error { order = append(order, 4); return nil }, + } + if err := runRuntimeCloseSteps(steps); !errors.Is(err, errAcquiredThird) { + t.Fatalf("teardown runs newest-first, so the most recently acquired failure (step 3) must be reported, got %v", err) + } + if want := []int{4, 3, 2, 1, 0}; len(order) != len(want) { + t.Fatalf("every step must run even after a failure: wanted %v, got %v", want, order) + } else { + for i := range want { + if order[i] != want[i] { + t.Fatalf("teardown must run in reverse acquisition order: wanted %v, got %v", want, order) + } + } + } + if err := runRuntimeCloseSteps(nil); err != nil { + t.Fatalf("an empty teardown must report no failure, got %v", err) + } +} + +type activationMatrixStore struct { + activateErr error +} + +func (s activationMatrixStore) CreateManifest(context.Context, publicationcontract.Manifest) (publicationcontract.Manifest, error) { + panic("CreateManifest is outside the activation path") +} + +func (s activationMatrixStore) ReadManifest(context.Context, publicationcontract.Ref) (publicationcontract.Manifest, error) { + panic("ReadManifest is outside the activation path") +} + +func (s activationMatrixStore) TransitionManifest(context.Context, publicationcontract.Manifest, publicationcontract.State) (publicationcontract.Manifest, error) { + panic("TransitionManifest is outside the activation path") +} + +func (s activationMatrixStore) ResolveActiveManifest(context.Context, string) (publicationcontract.Manifest, error) { + return publicationcontract.Manifest{}, publicationcontract.ErrNoActiveGeneration +} + +func (s activationMatrixStore) ActivateGeneration(_ context.Context, manifest publicationcontract.Manifest, expected *publicationcontract.Ref) error { + if !manifest.IsStaged() { + return errors.New("matrix fixture: manifest must reach the CAS staged") + } + if expected != nil { + return errors.New("matrix fixture: no active generation means a nil expected pointer") + } + return s.activateErr +} + +func stagedActivationManifest(t *testing.T) publicationcontract.Manifest { + t.Helper() + ref, err := publicationcontract.NewRef("acme", "gen-1") + if err != nil { + t.Fatalf("build dataset ref: %v", err) + } + snapshot, err := publicationcontract.NewSchemaSnapshot("", "", strings.Repeat("a", 64), []string{"Patient"}) + if err != nil { + t.Fatalf("build schema snapshot: %v", err) + } + manifest, err := publicationcontract.NewManifest(ref, snapshot) + if err != nil { + t.Fatalf("build manifest: %v", err) + } + staged, err := manifest.Transition(publicationcontract.StateStaged) + if err != nil { + t.Fatalf("stage manifest: %v", err) + } + return staged +} + +type activationTimeoutError struct{} + +func (activationTimeoutError) Error() string { return "activation network timeout" } +func (activationTimeoutError) Timeout() bool { return true } +func (activationTimeoutError) Temporary() bool { return false } + +var _ net.Error = activationTimeoutError{} + +func TestPostgresRuntimeActivationOutcomeClassification(t *testing.T) { + manifest := stagedActivationManifest(t) + outcomeUnknownCases := []struct { + name string + err error + }{ + {"store commit-outcome-unknown sentinel", control.ErrCommitOutcomeUnknown}, + {"connect sentinel", fmt.Errorf("dial postgres: %w", control.ErrConnect)}, + {"batch-write sentinel", fmt.Errorf("copy rows: %w", control.ErrBatchWrite)}, + {"sqlstate connection exception", &pgconn.PgError{Code: "08006", Message: "connection failure"}}, + {"sqlstate admin shutdown", &pgconn.PgError{Code: "57P01", Message: "admin shutdown"}}, + {"sqlstate crash shutdown", &pgconn.PgError{Code: "57P02", Message: "crash shutdown"}}, + {"context cancellation", fmt.Errorf("activate cas: %w", context.Canceled)}, + {"context deadline exceeded", fmt.Errorf("activate cas: %w", context.DeadlineExceeded)}, + {"network timeout", activationTimeoutError{}}, + {"eof mid-commit", io.EOF}, + {"unexpected eof mid-commit", io.ErrUnexpectedEOF}, + } + for _, tc := range outcomeUnknownCases { + err := pgGenerationActivator{manifests: activationMatrixStore{activateErr: tc.err}}.Activate(context.Background(), manifest) + var outcome *ingest.ActivationOutcomeError + if !errors.As(err, &outcome) { + t.Fatalf("%s: connectivity-class activation failure must surface as *ingest.ActivationOutcomeError (CLI parity), got %v", tc.name, err) + } + if !errors.Is(outcome.Err, tc.err) { + t.Fatalf("%s: the underlying failure must stay wrapped for the audit, got %v", tc.name, outcome.Err) + } + if outcome.Dataset != manifest.Dataset { + t.Fatalf("%s: the outcome error must name the dataset generation, got %+v", tc.name, outcome.Dataset) + } + } + knownRefusalCases := []struct { + name string + err error + }{ + {"activation conflict", control.ErrActivationConflict}, + {"generation not ready", publicationcontract.ErrGenerationNotReady}, + {"invalid active generation", publicationcontract.ErrInvalidActiveGeneration}, + {"conflict outranks connectivity", fmt.Errorf("%w: %w", control.ErrActivationConflict, control.ErrConnect)}, + {"plain store refusal", errors.New("no such staged manifest")}, + } + for _, tc := range knownRefusalCases { + err := pgGenerationActivator{manifests: activationMatrixStore{activateErr: tc.err}}.Activate(context.Background(), manifest) + if !errors.Is(err, tc.err) { + t.Fatalf("%s: known typed refusal must surface unchanged, got %v", tc.name, err) + } + var outcome *ingest.ActivationOutcomeError + if errors.As(err, &outcome) { + t.Fatalf("%s: known typed refusal must not be laundered into an outcome-unknown audit", tc.name) + } + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 94fb4b24b..a4492194c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -2,13 +2,12 @@ package server import ( "context" - "crypto/sha256" - "encoding/hex" "errors" "flag" "fmt" "log/slog" "net/http" + "net/url" "os" "os/signal" "regexp" @@ -17,42 +16,27 @@ import ( "syscall" "time" - shared "github.com/arangodb/go-driver/v2/arangodb/shared" loadapi "github.com/calypr/loom/internal/api/bulk/load" queryapi "github.com/calypr/loom/internal/api/graphql/graph/query" graphresolver "github.com/calypr/loom/internal/api/graphql/graph/resolver" httpapi "github.com/calypr/loom/internal/api/http" - "github.com/calypr/loom/internal/authscope" + arangofactory "github.com/calypr/loom/internal/backends/arango" + "github.com/calypr/loom/internal/backends/postgres/control" "github.com/calypr/loom/internal/catalog" - catalogarango "github.com/calypr/loom/internal/catalog/arango" "github.com/calypr/loom/internal/dataframe/compiler" - "github.com/calypr/loom/internal/dataframe/compiler/lower" + "github.com/calypr/loom/internal/dataframe/compiler/ir" dataframeerrors "github.com/calypr/loom/internal/dataframe/errors" dataframeexecution "github.com/calypr/loom/internal/dataframe/execution" - "github.com/calypr/loom/internal/dataframe/execution/chartifact" - chartifactarango "github.com/calypr/loom/internal/dataframe/execution/chartifact/arango" publication "github.com/calypr/loom/internal/dataframe/publication" - bundlearango "github.com/calypr/loom/internal/dataframe/publication/arango" - publicationclickhouse "github.com/calypr/loom/internal/dataframe/publication/clickhouse" - "github.com/calypr/loom/internal/dataframe/published" "github.com/calypr/loom/internal/dataframe/recipe" - "github.com/calypr/loom/internal/dataframe/recipe/exec" - recipearango "github.com/calypr/loom/internal/dataframe/recipe/exec/arango" + "github.com/calypr/loom/internal/dataframe/semantic" publicationcontract "github.com/calypr/loom/internal/dataset" - publicationarango "github.com/calypr/loom/internal/dataset/arango" "github.com/calypr/loom/internal/explorer" - explorerarango "github.com/calypr/loom/internal/explorer/arango" - "github.com/calypr/loom/internal/explorer/artifactfs" - "github.com/calypr/loom/internal/explorer/capability" explorercompilation "github.com/calypr/loom/internal/explorer/compilation" "github.com/calypr/loom/internal/explorer/lifecycle" - fhirschema "github.com/calypr/loom/internal/fhir/schema" "github.com/calypr/loom/internal/ingest" - arangostore "github.com/calypr/loom/internal/store/arango" - clickhousestore "github.com/calypr/loom/internal/store/clickhouse" ) -// Run starts the Loom HTTP server using the process command-line flags. func Run() { options, err := parseServerOptions(os.Args[1:], flag.ContinueOnError) if err != nil { @@ -82,81 +66,7 @@ func recordDegradation(logger *slog.Logger, current error, stage string, cause e return errors.Join(current, fmt.Errorf("%s: %w", stage, cause)) } -func classifyDataframeQueryError(err error) error { - if err == nil { - return err - } - switch { - case arangostore.IsQueryUserAssertion(err, string(dataframeerrors.CodeTablePivotCellCardinality)): - return dataframeerrors.Wrap(err, dataframeerrors.CodeTablePivotCellCardinality, "") - case arangostore.IsQueryUserAssertion(err, string(dataframeerrors.CodeTablePivotUnlistedCategory)): - return dataframeerrors.Wrap(err, dataframeerrors.CodeTablePivotUnlistedCategory, "") - case arangostore.IsQueryUserAssertion(err, string(dataframeerrors.CodeConstructionRowValueMultipleValues)): - return dataframeerrors.Wrap(err, dataframeerrors.CodeConstructionRowValueMultipleValues, "") - case arangostore.IsQueryUserAssertion(err, string(dataframeerrors.CodeConstructionExpansionEmpty)): - return dataframeerrors.Wrap(err, dataframeerrors.CodeConstructionExpansionEmpty, "") - case arangostore.IsQueryUserAssertion(err, string(dataframeerrors.CodeExplicitGroupUnassignedMember)): - return dataframeerrors.Wrap(err, dataframeerrors.CodeExplicitGroupUnassignedMember, "") - case arangostore.IsQueryUserAssertion(err, string(dataframeerrors.CodeRelationshipCardinalityViolation)): - return dataframeerrors.Wrap(err, dataframeerrors.CodeRelationshipCardinalityViolation, "") - case arangostore.IsQueryUserAssertion(err, string(dataframeerrors.CodeTemporalAnchorInvalid)): - return dataframeerrors.Wrap(err, dataframeerrors.CodeTemporalAnchorInvalid, "") - case arangostore.IsQueryUserAssertion(err, string(dataframeerrors.CodeTemporalPrecisionUnsupported)): - return dataframeerrors.Wrap(err, dataframeerrors.CodeTemporalPrecisionUnsupported, "") - case arangostore.IsQueryUserAssertion(err, string(dataframeerrors.CodeTemporalTieAmbiguous)): - return dataframeerrors.Wrap(err, dataframeerrors.CodeTemporalTieAmbiguous, "") - case arangostore.IsQueryUserAssertion(err, string(dataframeerrors.CodeUnitIdentityUnknown)): - return dataframeerrors.Wrap(err, dataframeerrors.CodeUnitIdentityUnknown, "") - case arangostore.IsQueryUserAssertion(err, string(dataframeerrors.CodeUnitDimensionIncompatible)): - return dataframeerrors.Wrap(err, dataframeerrors.CodeUnitDimensionIncompatible, "") - case arangostore.IsQueryMemoryLimitExceeded(err): - return dataframeerrors.Wrap( - err, - dataframeerrors.CodeQueryMemoryLimitExceeded, - "", - dataframeerrors.WithDetails(map[string]any{"backend": "arangodb", "resource": "query_memory"}), - ) - case arangostore.IsQueryResourceLimitExceeded(err): - return dataframeerrors.Wrap( - err, - dataframeerrors.CodeQueryResourceLimitExceeded, - "", - dataframeerrors.WithDetails(map[string]any{"backend": "arangodb", "resource": "query"}), - ) - case arangostore.IsQueryOutOfMemory(err): - return dataframeerrors.Wrap( - err, - dataframeerrors.CodeQueryBackendOutOfMemory, - "", - dataframeerrors.WithDetails(map[string]any{"backend": "arangodb", "resource": "memory"}), - ) - default: - return err - } -} - -func configuredAQLQueryRows(logger *slog.Logger, phase string, execute dataframeexecution.QueryRows) dataframeexecution.QueryRows { - return func(ctx context.Context, query string, batchSize int, bindVars map[string]any, visit func(map[string]any) error) error { - started := time.Now() - digest := sha256.Sum256([]byte(query)) - queryHash := hex.EncodeToString(digest[:]) - if logger != nil { - logger.Info("dataframe AQL started", "request_id", requestIDFromContext(ctx), "phase", phase+"_start", - "query_hash", queryHash, "query_bytes", len(query), "bind_vars", len(bindVars), "cursor_batch_size", batchSize) - } - err := preserveConfiguredQueryError(execute(ctx, query, batchSize, bindVars, visit)) - level := slog.LevelInfo - if err != nil { - level = slog.LevelError - } - logServerDiagnostic(logger, level, "dataframe AQL execution", requestIDFromContext(ctx), phase, time.Since(started), err, - "query_hash", queryHash) - return err - } -} - func preserveConfiguredQueryError(err error) error { - err = classifyDataframeQueryError(err) if err == nil || errors.Is(err, context.DeadlineExceeded) { return err } @@ -234,13 +144,8 @@ func diagnosticErrorCauseValue(err error) string { case errors.Is(err, context.DeadlineExceeded): return "operation deadline exceeded" } - var arangoErr shared.ArangoError - if errors.As(err, &arangoErr) { - return arangoDiagnosticCause(arangoErr) - } - var arangoErrPointer *shared.ArangoError - if errors.As(err, &arangoErrPointer) && arangoErrPointer != nil { - return arangoDiagnosticCause(*arangoErrPointer) + if cause, ok := arangofactory.DiagnosticCause(err); ok { + return cause } return err.Error() } @@ -254,14 +159,6 @@ func capDiagnosticCause(cause string) string { return strings.ToValidUTF8(cause[:maxDiagnosticCauseBytes-3], "") + "..." } -func arangoDiagnosticCause(err shared.ArangoError) string { - cause := fmt.Sprintf("ArangoDB error number %d (HTTP %d)", err.ErrorNum, err.Code) - if err.ErrorMessage != "" { - cause += ": " + err.ErrorMessage - } - return cause -} - var ( diagnosticCredentialAssignment = regexp.MustCompile(`(?i)((?:"|')?(?:authorization|proxy-authorization|access[_ -]?token|refresh[_ -]?token|token|password|secret|api[_ -]?key|apikey|credential)(?:"|')?\s*[:=]\s*)(?:(?:bearer|basic)\s+)?(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|[^\s,;}\]]+)`) diagnosticBearerCredential = regexp.MustCompile(`(?i)\b(?:bearer|basic)\s+[A-Za-z0-9._~+/-]+=*`) @@ -346,7 +243,6 @@ func previewQueryMaxRuntime(ctx context.Context) time.Duration { func configuredCategoryScanner(logger *slog.Logger, recipeEngine *dataframeexecution.Engine) lifecycle.CategoryScanner { return func(ctx context.Context, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings, request dataframeexecution.CategoryScanRequest) (dataframeexecution.CategoryScanResult, error) { - // Category discovery is Pivot-only and uses the extended query runtime limit. scanCtx, cancel := context.WithTimeout(ctx, pivotCategoryScanTimeout) defer cancel() scanCtx = withPivotPreviewQueryBudget(scanCtx) @@ -376,51 +272,13 @@ func configuredCategoryScanner(logger *slog.Logger, recipeEngine *dataframeexecu } } -// newExplorerReleaseActivator builds the dataset release activation hook for -// the repository publish path (lifecycle repository_publish.go ActivateRelease). -// It reads the live release revision and submits it as the store's -// compare-and-swap input, so a concurrent winning publication makes the -// store's own guard reject: internal/dataset/arango/release_store.go -// activateReleaseAQL compares it with `FILTER currentRevision == -// @expected_revision OR alreadyActive`, and the emptied result becomes -// dataset.ErrReleaseActivationConflict (release_store.go:99-100), which the -// lifecycle types as the repair-34 409 repository_publish -// PUBLICATION_ACTIVATION_CONFLICT with the prior revision retained. -// -// devReleaseCASFault is the dev-only repository sibling of the -// prepareExplorerRelease fault (LOOM_DEV_ACTIVATION_CONFLICT_ONCE) below: it -// is armed only through applyDevFaultOverrides, therefore only under the -// local no-auth mode. Consumed exactly once, it corrupts the expected-revision -// CAS input after a successful Active read so the release CAS legitimately -// loses while the retained candidate release stays durable. The seam sits -// above the store on purpose: the second, Explorer-side activation leg -// (internal/explorer/arango/activation.go compositeActivationReadAQL guards -// plus activateRevisionAndOwner) never yields dataset.ErrReleaseActivationConflict -// — every guard on that leg produces only explorer.ErrDraftConflict or plain -// faults — so the sentinel this contract classifies can only come from the -// release CAS input computed here. A rival document could not be activated -// instead: activateReleaseAQL's alreadyActive guard makes same-content -// re-activation a zero-write no-op, so bumping the pointer revision would -// require minting a competing release document that no publish flow created -// (invented state) and would itself leave the dataset pointer on the rival -// while the Explorer pointer stayed on the prior revision (split-brain). -// The fault therefore corrupts one number instead: the store still decides -// the outcome, zero writes happen on the loss, and no successful activation -// is ever hidden behind a synthetic error. func newExplorerReleaseActivator(releaseService *publicationcontract.ReleaseService, devReleaseCASFault *atomic.Bool) func(context.Context, string, string, []publicationcontract.DataframeSelector) error { return func(ctx context.Context, project, generation string, selectors []publicationcontract.DataframeSelector) error { expectedRevision := int64(0) - // Consume the one-shot arm at request entry, never conditionally: an - // armed request whose Active read has no pointer to corrupt must not - // leave the fault live to silently hijack a LATER publish request. devFaultArmed := devReleaseCASFault != nil && devReleaseCASFault.CompareAndSwap(true, false) active, err := releaseService.Active(ctx, project) if err == nil { expectedRevision = active.Revision - // Feed a stale expected revision so the real - // store CAS loses exactly once. Corrupt only after a successful - // Active read: with no prior pointer the store rejects on the - // missing-pointer precondition instead of the compare-and-swap. if devFaultArmed { expectedRevision++ } @@ -435,524 +293,320 @@ func newExplorerReleaseActivator(releaseService *publicationcontract.ReleaseServ } } +var ErrPostgreSQLRuntimeNotWired = errors.New("server.backend postgres is configured, but the PostgreSQL source runtime is not wired yet; refusing to start on the ArangoDB backend") + +func RunWithConfig(ctx context.Context, serverConfig Config) error { + return run(ctx, serverConfig) +} + func run(ctx context.Context, serverConfig Config) error { logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{})) - connOpts := arangostore.ConnectionOptions{ - URL: serverConfig.Server.URL, - Database: serverConfig.Server.Database, - } - - lifecycleClient, err := arangostore.Open(ctx, connOpts.URL, connOpts.Database) + backend, err := normalizeSourceBackend(serverConfig.Server.Backend) if err != nil { - return fmt.Errorf("open dataset lifecycle store: %w", err) + return err } - defer func() { - closeCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) - defer cancel() - _ = lifecycleClient.Close(closeCtx) - }() - if err := lifecycleClient.Bootstrap(ctx, publicationarango.BootstrapSpec()); err != nil { - return fmt.Errorf("bootstrap dataset lifecycle store: %w", err) + if backend == sourceBackendPostgres { + if _, _, err := postgresRuntimeAssignment(serverConfig.Server.PostgreSQL); err == nil { + return runPostgreSQLFullRuntime(ctx, serverConfig, logger) + } + return runPostgreSQLFirstQueryRoute(ctx, serverConfig, logger) } - var degradation error - if err := lifecycleClient.Bootstrap(ctx, recipearango.BootstrapSpec()); err != nil { - degradation = recordDegradation(logger, degradation, "bootstrap recipe registry", err) + return runArangoRuntime(ctx, serverConfig, logger) +} + +const postgreSQLFirstQueryRouteAcquireTimeout = 30 * time.Second + +func postgreSQLFirstQueryRouteWiringError(operation, enablingTicket string) error { + return dataframeerrors.NewError(dataframeerrors.CodeBackendUnavailable, + operation+" is not served by the PostgreSQL first query route; it belongs to "+enablingTicket, + dataframeerrors.WithRetryable(false), + dataframeerrors.WithDetails(map[string]any{"backend": "postgresql", "stage": "wiring", "enablingTicket": enablingTicket})) +} + +func postgresFirstQueryRouteAssignment(pg PostgreSQLConfig) (control.SeriesSchemas, string, string, error) { + source, hasSource := pg.Schemas[string(control.SeriesSource)] + manifest, hasManifest := pg.Schemas[string(control.SeriesDatasetRelease)] + source, manifest = strings.TrimSpace(source), strings.TrimSpace(manifest) + if !hasSource || !hasManifest || source == "" || manifest == "" || source == manifest || len(pg.Schemas) != 2 { + return nil, "", "", ErrPostgreSQLRuntimeNotWired + } + return control.SeriesSchemas{ + control.SeriesSource: source, + control.SeriesDatasetRelease: manifest, + }, source, manifest, nil +} + +func postgresFirstQueryRouteDSN(pg PostgreSQLConfig) string { + u := &url.URL{ + Scheme: "postgres", + User: url.UserPassword(pg.User, pg.Password), + Host: fmt.Sprintf("%s:%d", pg.Host, pg.Port), + Path: "/" + pg.Database, + RawQuery: "sslmode=" + url.QueryEscape(strings.ToLower(strings.TrimSpace(pg.SSLMode))), } - if err := lifecycleClient.Bootstrap(ctx, recipearango.RevisionBootstrapSpec()); err != nil { - degradation = recordDegradation(logger, degradation, "bootstrap recipe revision registry", err) + return u.String() +} + +type redactedRouteError struct { + msg string + cause error +} + +func (e *redactedRouteError) Error() string { return e.msg } +func (e *redactedRouteError) Unwrap() error { return e.cause } + +func postgresFirstQueryRouteError(err error, pg PostgreSQLConfig, dsn string) error { + if err == nil { + return nil + } + text := err.Error() + queryEscaped := url.QueryEscape(pg.Password) + pathEscaped := url.PathEscape(pg.Password) + secrets := []string{ + dsn, + pg.Password, + pg.User + ":" + pg.Password, + queryEscaped, + pathEscaped, + pg.User + ":" + queryEscaped, + pg.User + ":" + pathEscaped, + } + for _, secret := range secrets { + if secret != "" { + text = strings.ReplaceAll(text, secret, "") + } } - if err := lifecycleClient.Bootstrap(ctx, explorerarango.BootstrapSpec()); err != nil { - return fmt.Errorf("bootstrap Explorer store: %w", err) + if text == err.Error() { + return err } - recipeRegistry, err := recipearango.New(lifecycleClient) - if err != nil { - return fmt.Errorf("create recipe registry: %w", err) + return &redactedRouteError{msg: text, cause: err} +} + +const ( + pgReadinessStateClosed = "runtime_closed" + pgReadinessStateConnectivityDown = "connectivity_down" + pgReadinessStateUnsupportedVer = "unsupported_version" + pgReadinessStateSchemaMissing = "schema_missing" + pgReadinessStateSchemaUnmigrated = "schema_unmigrated" + pgReadinessStateSchemaMismatch = "schema_mismatch" + pgReadinessStateUnknown = "core_state_unknown" +) + +func postgreSQLCoreReadinessState(err error) string { + switch { + case err == nil: + return "" + case errors.Is(err, control.ErrRuntimeClosed): + return pgReadinessStateClosed + case errors.Is(err, control.ErrConnect): + return pgReadinessStateConnectivityDown + case errors.Is(err, control.ErrUnsupportedVersion): + return pgReadinessStateUnsupportedVer + case errors.Is(err, control.ErrInvalidSchemaName): + return pgReadinessStateSchemaMissing + case errors.Is(err, control.ErrMigrationPending): + return pgReadinessStateSchemaUnmigrated + case errors.Is(err, control.ErrMigrationChecksumMismatch), + errors.Is(err, control.ErrFutureSchemaVersion), + errors.Is(err, control.ErrMigrationMissing), + errors.Is(err, control.ErrMigrationOutOfOrder): + return pgReadinessStateSchemaMismatch + default: + return pgReadinessStateUnknown } - if serverConfig.Server.ClickHouse.Enabled { - data, err := os.ReadFile(serverConfig.Server.Dataframer.Recipe) - if err != nil { - return fmt.Errorf("read dataframer recipe %q: %w", serverConfig.Server.Dataframer.Recipe, err) - } - defaultBundle, err := recipe.Parse(data) - if err != nil { - return fmt.Errorf("parse dataframer recipe %q: %w", serverConfig.Server.Dataframer.Recipe, err) - } - if _, err := (exec.PersistentRegistry{Store: recipeRegistry}).RegisterVersion(ctx, defaultBundle); err != nil { - degradation = recordDegradation(logger, degradation, "register default dataframe recipe", err) - } +} + +type postgreSQLGenerationStatusReader struct { + manifests publicationcontract.ManifestLifecycle +} + +func (r postgreSQLGenerationStatusReader) ReadManifest(ctx context.Context, ref publicationcontract.Ref) (publicationcontract.Manifest, error) { + if r.manifests == nil { + return publicationcontract.Manifest{}, postgreSQLFirstQueryRouteWiringError("Generation status reads", "pg-runtime-server-composition") } - lifecycleStore, err := publicationarango.New(lifecycleClient) - if err != nil { - return fmt.Errorf("create dataset lifecycle store: %w", err) + return r.manifests.ReadManifest(ctx, ref) +} + +func (r postgreSQLGenerationStatusReader) Activate(context.Context, publicationcontract.Manifest) error { + return postgreSQLFirstQueryRouteWiringError("Server-side generation activation", "pg-runtime-server-composition") +} + +type postgreSQLFirstQueryRecipeControl struct { + dataframeexecution.Control +} + +func (c postgreSQLFirstQueryRecipeControl) Validate(ctx context.Context, name string, bindings recipe.RuntimeBindings) (dataframeexecution.Validation, error) { + if c.Control.Engine == nil { + return dataframeexecution.Validation{}, postgreSQLFirstQueryRouteWiringError("Recipe validation", "pg-runtime-server-composition") } - activeManifestResolver := publicationcontract.ActiveResolver(lifecycleStore) + return c.Control.Validate(ctx, name, bindings) +} - discoveryCache := catalog.NewCache() - catalogStore, err := catalogarango.New(lifecycleClient) - if err != nil { - return fmt.Errorf("create catalog store: %w", err) +func (c postgreSQLFirstQueryRecipeControl) Explain(ctx context.Context, name string, bindings recipe.RuntimeBindings) (semantic.RecipePlanExplanation, error) { + if c.Control.Engine == nil { + return semantic.RecipePlanExplanation{}, postgreSQLFirstQueryRouteWiringError("Recipe explain", "pg-runtime-server-composition") } - discoverFields := discoveryCache.DiscoverFields(catalogStore.DiscoverFields) - discoverReferences := discoveryCache.DiscoverReferences(catalogStore.DiscoverReferences) + return c.Control.Explain(ctx, name, bindings) +} - auth, err := wireAuth(serverConfig, serverConfig.Server.AllowUnauthenticated || serverConfig.Auth.AllowUnauthenticated, catalogStore.DiscoverExistingAuthResourcePaths) - if err != nil { - return fmt.Errorf("configure authentication: %w", err) +func (c postgreSQLFirstQueryRecipeControl) ExplainPhysical(ctx context.Context, name string, bindings recipe.RuntimeBindings, live bool) (dataframeexecution.PhysicalExplanation, error) { + if c.Control.Engine == nil { + return dataframeexecution.PhysicalExplanation{}, postgreSQLFirstQueryRouteWiringError("Recipe physical explain", "pg-runtime-server-composition") } - authenticator, authorizer, scopeResolver := auth.authenticator, auth.authorizer, auth.scopeResolver + return c.Control.ExplainPhysical(ctx, name, bindings, live) +} - dataframes := dataframeexecution.NewService(dataframeexecution.ServiceConfig{QueryRows: func(ctx context.Context, query string, batch int, binds map[string]any, visit func(map[string]any) error) error { - return classifyDataframeQueryError(lifecycleClient.QueryRows(ctx, query, batch, binds, visit)) - }}) - // The lifecycle client already owns this Arango database. Reusing it avoids - // a second connection that can fail independently during optional startup. - publishedRegistry, err := bundlearango.New(lifecycleClient) - if err != nil { - return fmt.Errorf("create published dataframe registry: %w", err) +func (c postgreSQLFirstQueryRecipeControl) Resolve(ctx context.Context, name string, bindings recipe.RuntimeBindings) (semantic.ResolvedRecipePlan, error) { + if c.Control.Engine == nil { + return semantic.ResolvedRecipePlan{}, postgreSQLFirstQueryRouteWiringError("Recipe resolution", "pg-runtime-server-composition") } - publicationReady := true - if err := lifecycleClient.Bootstrap(ctx, bundlearango.BootstrapSpec()); err != nil { - degradation = recordDegradation(logger, degradation, "bootstrap dataframe registry", err) - publicationReady = false + return c.Control.Resolve(ctx, name, bindings) +} + +func (c postgreSQLFirstQueryRecipeControl) Preview(ctx context.Context, name string, bindings recipe.RuntimeBindings) (dataframeexecution.Preview, error) { + if c.Control.Engine == nil { + return dataframeexecution.Preview{}, postgreSQLFirstQueryRouteWiringError("Recipe preview", "pg-runtime-server-composition") } - var clickhouse *clickhousestore.Client - var materializationReader *published.Reader - var privateClickHouseArtifacts *chartifact.Manager - if serverConfig.Server.ClickHouse.Enabled { - clickhouse, err = clickhousestore.New(clickhousestore.Options{URL: serverConfig.Server.ClickHouse.URL, Database: serverConfig.Server.ClickHouse.Database, Username: serverConfig.Server.ClickHouse.Username, Password: serverConfig.Server.ClickHouse.Password}) - if err != nil { - return fmt.Errorf("create ClickHouse client: %w", err) - } - defer clickhouse.Close() - // The Arango-backed dataframe loader publishes into this database. Create - // it during server startup so a fresh ClickHouse instance does not require - // an operator to run a separate DDL/API step before materialization. - if err := clickhouse.EnsureDatabase(ctx); err != nil { - degradation = recordDegradation(logger, degradation, "ClickHouse database", err) - publicationReady = false - } - if err := lifecycleClient.Bootstrap(ctx, chartifactarango.BootstrapSpec()); err != nil { - return fmt.Errorf("bootstrap private dataframe artifact catalog: %w", err) - } - privateArtifactCatalog, err := chartifactarango.NewCatalog(lifecycleClient) - if err != nil { - return fmt.Errorf("create private dataframe artifact catalog: %w", err) - } - privateClickHouseArtifacts, err = chartifact.New(chartifact.Config{ - Catalog: privateArtifactCatalog, ClickHouse: clickhouse, - BatchRows: serverConfig.Server.RecipeBatchRows, BatchBytes: serverConfig.Server.RecipeBatchBytes, - }) - if err != nil { - return fmt.Errorf("create private ClickHouse artifact manager: %w", err) - } - materializationReader = &published.Reader{ClickHouse: clickhouse, Catalog: publishedRegistry, Logger: logger, MaxPage: 1000, ActiveManifestResolver: activeManifestResolver, ActiveReleaseResolver: lifecycleStore} + return c.Control.Preview(ctx, name, bindings) +} + +func (c postgreSQLFirstQueryRecipeControl) Run(ctx context.Context, name string, bindings recipe.RuntimeBindings) (dataframeexecution.Preview, error) { + if c.Control.Engine == nil { + return dataframeexecution.Preview{}, postgreSQLFirstQueryRouteWiringError("Recipe run", "pg-runtime-server-composition") } - recipeRevisions, err := recipearango.NewRevisionRegistry(lifecycleClient) - if err != nil { - return fmt.Errorf("create recipe revision registry: %w", err) - } - var clickHouseQueryRows dataframeexecution.ClickHouseQueryRows - if clickhouse != nil { - clickHouseQueryRows = clickhouse.QueryRowsArgsVisit - } - var resolveClickHouseInputs dataframeexecution.ResolveClickHouseInputs - var withExecutionReadPins dataframeexecution.WithExecutionReadPins - if materializationReader != nil { - resolver := clickHouseCombineInputResolver{reader: materializationReader, scopes: scopeResolver} - resolveClickHouseInputs = resolver.resolve - withExecutionReadPins = materializationReader.WithExecutionReadPins - } - recipeEngine, err := dataframeexecution.New(dataframeexecution.Config{ - Registry: recipeRegistry, - Revisions: recipeRevisions, - ResolveBundle: recipeSchemaResolver(catalogStore.DiscoverFields, discoveryCache), - PreparePreviewIndex: loggedPreviewIndexPreparation(logger, func(ctx context.Context, spec compiler.PreviewCoveringIndexSpec) error { - var err error - if len(spec.StoredValues) != 0 { - if spec.Supersedes != nil { - err = lifecycleClient.EnsurePreviewCoveringIndexWithStoredValuesReplacing( - ctx, spec.Collection, spec.Name, spec.Fields, spec.StoredValues, - spec.Supersedes.Name, spec.Supersedes.Fields, spec.Supersedes.StoredValues, - ) - } else { - err = lifecycleClient.EnsurePreviewCoveringIndexWithStoredValues(ctx, spec.Collection, spec.Name, spec.Fields, spec.StoredValues) - } - } else if spec.Supersedes != nil { - err = lifecycleClient.EnsurePreviewCoveringIndexReplacing(ctx, spec.Collection, spec.Name, spec.Fields, spec.Supersedes.Name, spec.Supersedes.Fields) - } else { - err = lifecycleClient.EnsurePreviewCoveringIndex(ctx, spec.Collection, spec.Name, spec.Fields) - } - return err - }), - PreviewCoveringIndexes: lifecycleClient.ListPreviewCoveringIndexes, - PreviewCollectionRevision: lifecycleClient.CollectionRevision, - PreviewExplainQuery: func(ctx context.Context, query string, bindVars map[string]any) (arangostore.ExplainResult, error) { - return lifecycleClient.Explain(ctx, arangostore.ExplainRequest{Query: query, BindVars: bindVars}) - }, - PreviewCollectionCount: lifecycleClient.CollectionCount, - ClickHouseQueryRows: clickHouseQueryRows, - PreviewQueryRows: configuredAQLQueryRows(logger, "preview_query_rows", func(ctx context.Context, query string, batchSize int, bindVars map[string]any, visit func(map[string]any) error) error { - return lifecycleClient.QueryRowsWithMaxRuntime(ctx, query, batchSize, bindVars, previewQueryMaxRuntime(ctx), arangostore.RowVisitor(visit)) - }), - ResolveClickHouseInputs: resolveClickHouseInputs, - WithExecutionReadPins: withExecutionReadPins, - PrivateClickHouseArtifacts: privateClickHouseArtifacts, - QueryRows: configuredAQLQueryRows(logger, "query_rows", func(ctx context.Context, query string, batchSize int, bindVars map[string]any, visit func(map[string]any) error) error { - return lifecycleClient.QueryRows(ctx, query, batchSize, bindVars, arangostore.RowVisitor(visit)) - }), - ScopeDigest: recipeScopeDigest, - RootPageRows: serverConfig.Server.RecipeQueryPageRows, - }) + return c.Control.Run(ctx, name, bindings) +} + +type postgreSQLFirstQueryDataframeReleases struct{} + +func (postgreSQLFirstQueryDataframeReleases) GetExecution(context.Context, string) (publication.BundleExecution, error) { + return publication.BundleExecution{}, postgreSQLFirstQueryRouteWiringError("Dataframe release reads", "pg-runtime-server-composition") +} + +func (postgreSQLFirstQueryDataframeReleases) GetPointer(context.Context, string) (publication.BundlePointer, error) { + return publication.BundlePointer{}, postgreSQLFirstQueryRouteWiringError("Dataframe release pointer reads", "pg-runtime-server-composition") +} + +func composePostgreSQLFirstQueryRoute(ctx context.Context, serverConfig Config, logger *slog.Logger) (server *httpapi.HTTPServer, closeRuntime func() error, err error) { + assignment, sourceSchema, _, err := postgresFirstQueryRouteAssignment(serverConfig.Server.PostgreSQL) if err != nil { - return fmt.Errorf("create dataframe recipe engine: %w", err) + return nil, nil, err } - var bundleTarget publication.Target - if serverConfig.Server.ClickHouse.Enabled && publicationReady { - bundleStore, err := publicationclickhouse.NewBundleStore(clickhouse, publishedRegistry) - if err != nil { - return fmt.Errorf("create dataframe bundle store: %w", err) - } - bundleStore.SetActiveReleaseReader(lifecycleStore) - if err := bundleStore.Reconcile(ctx, time.Now().UTC().Add(-2*time.Minute)); err != nil { - degradation = recordDegradation(logger, degradation, "dataframe publication reconciliation", err) - publicationReady = false - } - if publicationReady { - bundleTarget = bundleStore - } + pg := serverConfig.Server.PostgreSQL + if err := pg.validate(); err != nil { + return nil, nil, err } - verificationStore := publicationVerificationStore{executions: publishedRegistry} - releaseService := &publicationcontract.ReleaseService{Manifests: lifecycleStore, Releases: lifecycleStore, Verifier: verificationStore, Required: serverConfig.Server.RequiredDataframeSelectors} - var activationConflictOnce atomic.Bool - activationConflictOnce.Store(serverConfig.Server.DevActivationConflictOnce) - var repositoryActivationConflictOnce atomic.Bool - repositoryActivationConflictOnce.Store(serverConfig.Server.DevRepositoryActivationConflictOnce) - activateExplorerRelease := newExplorerReleaseActivator(releaseService, &repositoryActivationConflictOnce) - prepareExplorerRelease := func(ctx context.Context, project, generation string, selectors []publicationcontract.DataframeSelector) (publicationcontract.ProjectRelease, int64, error) { - expectedRevision := int64(0) - active, err := releaseService.Active(ctx, project) + if serverConfig.Server.ClickHouse.Enabled { + return nil, nil, errors.New("server.clickhouse.enabled cannot be combined with server.backend postgres in this slice; the PostgreSQL publication composition belongs to pg-runtime-server-composition") + } + + var cleanups []func() error + defer func() { if err == nil { - expectedRevision = active.Revision - } else if !errors.Is(err, publicationcontract.ErrNoActiveRelease) { - return publicationcontract.ProjectRelease{}, 0, err - } - release, err := releaseService.Create(ctx, publicationcontract.ActivationRequest{ - Project: project, Generation: generation, GitCommit: generation, - OptionalSelectors: selectors, - }) - // Return a deliberately stale CAS revision exactly once in the isolated - // dev fault scenario. The candidate remains retained for audit, while - // PublishAuthoring's atomic activation fails without moving the prior - // active release pointer or exposing candidate rows. - if err == nil && activationConflictOnce.CompareAndSwap(true, false) { - expectedRevision++ + return } - return release, expectedRevision, err - } - validateExplorerReleaseGeneration := func(ctx context.Context, project, generation string) error { - return releaseService.ValidateGeneration(ctx, project, generation) - } - explorerStore, err := explorerarango.New(lifecycleClient) - if err != nil { - return fmt.Errorf("create Explorer store: %w", err) - } - if err := explorerStore.MigrateLegacyRepositoryConfigs(ctx); err != nil { - return fmt.Errorf("migrate legacy Explorer repository configs: %w", err) - } - capabilitySnapshots, err := explorerarango.NewCapabilitySnapshotStore(lifecycleClient) + _ = runRuntimeCloseSteps(cleanups) + }() + dsn := postgresFirstQueryRouteDSN(pg) + runtime, err := openPostgresBackend(ctx, control.Config{ + DSN: dsn, + MaxConns: int32(pg.MaxOpenConns), + MinConns: int32(pg.MaxIdleConns), + AcquireTimeout: postgreSQLFirstQueryRouteAcquireTimeout, + Schemas: assignment, + Setup: control.SetupNone, + }) if err != nil { - return fmt.Errorf("create Explorer capability snapshot store: %w", err) + return nil, nil, postgresFirstQueryRouteError(fmt.Errorf("open PostgreSQL source runtime: %w", err), pg, dsn) } - capabilityResolver, err := newExplorerCapabilityResolver(catalogStore, scopeResolver, activeManifestResolver, capabilitySnapshots) + cleanups = append(cleanups, func() error { + return postgresFirstQueryRouteError(runtime.Close(), pg, dsn) + }) + closeRuntime = func() error { return runRuntimeCloseSteps(cleanups) } + + binding, err := ir.NewPostgresStorageBinding(sourceSchema) if err != nil { - return fmt.Errorf("create Explorer capability resolver: %w", err) + return nil, nil, postgresFirstQueryRouteError(fmt.Errorf("bind PostgreSQL storage placement: %w", err), pg, dsn) } - explorerService, err := explorer.NewService(explorerStore) + sourceCompileTarget, err := compiler.PostgreSQLTarget(binding) if err != nil { - return fmt.Errorf("create Explorer service: %w", err) + return nil, nil, postgresFirstQueryRouteError(fmt.Errorf("select PostgreSQL compile target: %w", err), pg, dsn) } - artifactStore, err := artifactfs.New(serverConfig.Server.ArtifactDirectory) + + dataframes := dataframeexecution.NewService(dataframeexecution.ServiceConfig{ + SourceRows: configuredPostgreSQLSourceRows(logger, "query_rows", runtime.SourceRows), + }) + + discoveryCache := catalog.NewCache() + discoverFields := discoveryCache.DiscoverFields(runtime.DiscoverFields) + + auth, err := wireAuth(serverConfig, serverConfig.Server.AllowUnauthenticated || serverConfig.Auth.AllowUnauthenticated, runtime.DiscoverExistingAuthResourcePaths) if err != nil { - return fmt.Errorf("create Explorer artifact store: %w", err) + return nil, nil, fmt.Errorf("configure authentication: %w", err) } + authenticator, authorizer, scopeResolver := auth.authenticator, auth.authorizer, auth.scopeResolver + resolver := graphresolver.NewResolver(graphresolver.ResolverConfig{ DataframeQuery: queryapi.Config{ - DiscoverReferences: discoverReferences, + CompileTarget: sourceCompileTarget, + DiscoverReferences: nil, DiscoverFields: discoverFields, Dataframes: dataframes, ScopeResolver: scopeResolver, - ActiveManifestResolver: activeManifestResolver, - Explain: func(ctx context.Context, compiled dataframeexecution.CompiledQuery) error { - _, err := explainCompiledQuery(ctx, lifecycleClient, compiled) - return err - }, + ActiveManifestResolver: runtime.Manifests, + Explain: nil, }, - MaterializationReader: materializationReader, + MaterializationReader: nil, Logger: logger, - RecipeControl: dataframeexecution.Control{Engine: recipeEngine, ExplainConnection: func(ctx context.Context, compiled dataframeexecution.CompiledQuery) (dataframeexecution.ExplainAssessment, error) { - return explainCompiledQuery(ctx, lifecycleClient, compiled) + RecipeControl: postgreSQLFirstQueryRecipeControl{Control: dataframeexecution.Control{ + ExplainPostgreSQLConnection: func(ctx context.Context, compiled compiler.CompiledQuery) (dataframeexecution.PostgreSQLPlanAssessment, error) { + return explainPostgreSQLCompiledQuery(ctx, runtime.PlanJSON, compiled) + }, }}, RecipeAuthorizer: recipeAuthorization{resolver: scopeResolver}, - RecipeRevisions: recipeRevisions, - RecipeExecutions: graphresolver.NewAuthorizedRecipeExecutionReader(publishedRegistry, scopeResolver), + RecipeRevisions: nil, + RecipeExecutions: nil, }) - ingestRunner := loadapi.IngestRunner{BaseOptions: ingest.LoadOptions{ - ConnectionOptions: connOpts, - Schema: serverConfig.Server.Schema, - }} + generationService, err := loadapi.NewService(loadapi.ServiceConfig{ - LoadGeneration: ingestRunner.RunGeneration, - GenerationActivator: lifecycleStore, - DataframeReleases: publishedRegistry, - Logger: logger, - OnSuccess: func(project string) { - discoveryCache.InvalidateProject(project) - if scopeResolver != nil { - scopeResolver.InvalidateProject(project) - } + LoadGeneration: func(context.Context, loadapi.GenerationLoadRequest, ingest.EventSink) (ingest.LoadSummary, error) { + return ingest.LoadSummary{}, postgreSQLFirstQueryRouteWiringError("Server-side PostgreSQL generation load", "pg-runtime-server-composition (the loom-data CLI drives the production ingest path today)") }, + GenerationActivator: postgreSQLGenerationStatusReader{manifests: runtime.Manifests}, + DataframeReleases: postgreSQLFirstQueryDataframeReleases{}, + Logger: logger, }) if err != nil { - return fmt.Errorf("create generation load service: %w", err) - } - compileReceipt := func(ctx context.Context, request lifecycle.CompileReceiptRequest) (*explorer.CompilationReceipt, error) { - var resolveCombineSchemas combineInputSchemaResolver - if materializationReader != nil { - resolveCombineSchemas = func(ctx context.Context, output lower.CompiledRecipeOutput, bindings recipe.RuntimeBindings) (resolvedCombineInputSchema, error) { - return exactPublishedCombineInputSchemas(ctx, output, bindings, materializationReader, materializationReader.WithExecutionReadPins, scopeResolver) - } - } - return compileExplorerReceipt(ctx, request, capabilityResolver, recipeEngine, explorerService, logger, resolveCombineSchemas) - } - persistPublishedWorkspace, err := localWorkspaceWriter(serverConfig.Server.LocalWorkspaceWriteback, serverConfig.Server.LocalWorkspaceProject) - if err != nil { - return fmt.Errorf("configure local workspace writeback: %w", err) + return nil, nil, fmt.Errorf("create generation load service: %w", err) } - populationMappingCursorCodec, err := lifecycle.NewHMACPopulationMappingCursorCodec(serverConfig.Server.PopulationMappingCursorSecret) - if err != nil { - return fmt.Errorf("configure population mapping cursor signing: %w", err) - } - var artifactPublishedReader lifecycle.ArtifactPublishedReader - if materializationReader != nil { - artifactPublishedReader = materializationReader - if serverConfig.Server.DevArtifactRowDelay > 0 { - artifactPublishedReader = artifactDelayReader{next: artifactPublishedReader, delay: serverConfig.Server.DevArtifactRowDelay} - } - } - schemaIndex, err := fhirschema.GeneratedIndex() - if err != nil { - return fmt.Errorf("load generated schema index for row choices: %w", err) - } - rowChoiceResolver, err := lifecycle.NewSchemaRowChoiceResolver(schemaIndex) - if err != nil { - return fmt.Errorf("configure row-choice schema resolver: %w", err) - } - explicitGroupResolver, err := lifecycle.NewRepositoryExplicitGroupRevisionResolver(explorerStore) - if err != nil { - return fmt.Errorf("configure explicit group revision resolver: %w", err) - } - tableShapeCapabilities, err := explorerarango.NewTableShapeCapabilityRepository(lifecycleClient) - if err != nil { - return fmt.Errorf("configure table-shape capability repository: %w", err) - } - lifecycleConfig := lifecycle.Config{ - SemanticInventory: catalogStore.PageSemanticInventory, - ResolveSemanticInventorySelections: catalogStore.ResolveSemanticInventorySelections, - SelectionMembersCollection: explorerarango.SelectionMembersCollection, - InterpretationRepository: explorerStore, - PopulationMappingCursorCodec: populationMappingCursorCodec, - SelectionSourceResolver: published.SelectionSourceAdapter{Reader: materializationReader}, - SelectionReferenceValidator: explorerStore.ValidateSelectionReferences, - RowChoiceResolver: rowChoiceResolver, - RowChoicePlanner: rowChoiceResolver, - ExplicitGroupResolver: explicitGroupResolver, - ExplicitGroupRepository: explorerStore, - TableShapeCapabilities: tableShapeCapabilities, - ScanCategories: configuredCategoryScanner(logger, recipeEngine), - CompileReceipt: compileReceipt, - ConstructionSourceStage: func(ctx context.Context, request lifecycle.ConstructionSourceStageRequest) (explorer.ReceiptConstructionStage, error) { - return compileConstructionSourceStage(ctx, request, recipeEngine) - }, - Capability: lifecycle.CapabilityResolver{ - Current: func(ctx context.Context, project, _ string, generation string) (capability.Snapshot, error) { - return capabilityResolver.Resolve(ctx, project, generation) - }, - Token: capabilityResolver.ResolveToken, - ForCompilation: capabilityResolver.ResolveForCompilation, - ForExecution: capabilityResolver.ResolveForExecution, - Catalog: authoringV2Catalog, - }, - ReceiptLookup: func(ctx context.Context, project, explorerID, receiptID string) (*explorer.CompilationReceipt, error) { - return explorerService.CompilationReceiptForExplorer(ctx, project, explorerID, receiptID) - }, - ArtifactStore: artifactStore, - PublishedReader: artifactPublishedReader, - ArtifactTTL: serverConfig.Server.ArtifactTTL, - ArtifactMaxRows: serverConfig.Server.ArtifactMaxRows, - ArtifactMaxBytes: serverConfig.Server.ArtifactMaxBytes, - PreviewReceipt: func(ctx context.Context, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings, visit func(map[string]any) error) (dataframeexecution.PreviewSummary, error) { - if receipt == nil { - return dataframeexecution.PreviewSummary{}, fmt.Errorf("compilation receipt is required") - } - resolved, err := compileValidatedReceiptResolution(ctx, recipeEngine, receipt, bindings) - if err != nil { - logger.Error("Explorer receipt preview resolution failed", "receipt_id", receipt.ID, "error", err) - return dataframeexecution.PreviewSummary{}, classifyReceiptPreviewResolutionError(receipt.ID, err) - } - output := "" - if len(bindings.OutputNames) > 0 { - output = bindings.OutputNames[0] - } - summary, previewErr := recipeEngine.PreviewOutput(ctx, resolved, dataframeexecution.PreviewRequest{Output: output, Limit: bindings.PreviewLimit, IncludeRowIdentity: bindings.IncludeRowIdentity}, visit) - if previewErr != nil { - logReceiptPreviewContractFailure(logger, requestIDFromContext(ctx), receipt.ID, output, previewErr) - } - return summary, previewErr - }, - ValidateReceiptStream: func(ctx context.Context, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings) error { - err := validateReceiptFullStream(ctx, recipeEngine, receipt, bindings) - if err != nil { - logger.Error("Explorer row-value policy full-stream validation failed", "receipt_id", receipt.ID, "error", err) - } - return err - }, - PopulationMapping: func(ctx context.Context, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings, output string, memberIDs []string, after string, limit int) (dataframeexecution.PopulationMappingResult, error) { - if receipt == nil { - return dataframeexecution.PopulationMappingResult{}, fmt.Errorf("compilation receipt is required") - } - resolved, err := compileValidatedReceiptResolution(ctx, recipeEngine, receipt, bindings) - if err != nil { - logger.Error("Explorer receipt population mapping resolution failed", "receipt_id", receipt.ID, "error", err) - return dataframeexecution.PopulationMappingResult{}, classifyReceiptPreviewResolutionError(receipt.ID, err) - } - reader := dataframeexecution.PopulationMemberReaderFunc(func(_ context.Context, visit func(string) error) error { - for _, id := range memberIDs { - if err := visit(id); err != nil { - return err - } - } - return nil - }) - return recipeEngine.PopulationMapping(ctx, resolved, dataframeexecution.PopulationMappingRequest{Output: output, AfterMemberID: after, MaxUnmapped: limit}, reader) - }, - CellTrace: func(ctx context.Context, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings, request dataframeexecution.CellTraceRequest) (dataframeexecution.CellTraceResult, error) { - if receipt == nil { - return dataframeexecution.CellTraceResult{}, fmt.Errorf("compilation receipt is required") - } - resolved, err := compileValidatedReceiptResolution(ctx, recipeEngine, receipt, bindings) - if err != nil { - logger.Error("Explorer receipt cell trace resolution failed", "receipt_id", receipt.ID, "error", err) - return dataframeexecution.CellTraceResult{}, classifyReceiptPreviewResolutionError(receipt.ID, err) - } - return recipeEngine.CellTrace(ctx, resolved, request) - }, - RowLineage: func(ctx context.Context, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings, request dataframeexecution.RowLineageRequest) (dataframeexecution.RowLineageResult, error) { - if receipt == nil { - return dataframeexecution.RowLineageResult{}, fmt.Errorf("compilation receipt is required") - } - resolved, err := compileValidatedReceiptResolution(ctx, recipeEngine, receipt, bindings) - if err != nil { - logger.Error("Explorer receipt row lineage resolution failed", "receipt_id", receipt.ID, "error", err) - return dataframeexecution.RowLineageResult{}, classifyReceiptPreviewResolutionError(receipt.ID, err) - } - return recipeEngine.RowLineage(ctx, resolved, request) - }, - TableShapeExclusions: func(ctx context.Context, receipt *explorer.CompilationReceipt, bindings recipe.RuntimeBindings, request dataframeexecution.TableShapeExclusionRequest) (dataframeexecution.TableShapeExclusionResult, error) { - if receipt == nil { - return dataframeexecution.TableShapeExclusionResult{}, fmt.Errorf("compilation receipt is required") - } - resolved, err := compileValidatedReceiptResolution(ctx, recipeEngine, receipt, bindings) - if err != nil { - logger.Error("Explorer receipt table-shape exclusion resolution failed", "receipt_id", receipt.ID, "error", err) - return dataframeexecution.TableShapeExclusionResult{}, classifyReceiptPreviewResolutionError(receipt.ID, err) - } - return recipeEngine.TableShapeExclusions(ctx, resolved, request) - }, - MaterializeReceipt: explorerReceiptMaterializer( - recipeEngine, bundleTarget, publishedRegistry, degradation, logger, - serverConfig.Server.RecipeBatchRows, serverConfig.Server.RecipeBatchBytes, - serverConfig.Server.RecipeQualityMaxRows, serverConfig.Server.RecipeQualityMaxDistinctKeys, - ), - ValidateReleaseGeneration: validateExplorerReleaseGeneration, - ActivateRelease: activateExplorerRelease, - PrepareRelease: prepareExplorerRelease, - PersistPublishedWorkspace: persistPublishedWorkspace, - } - server, err := httpapi.NewHTTPServer(httpapi.HTTPConfig{Authenticator: authenticator, Authorizer: authorizer, Logger: logger, + + server, err = httpapi.NewHTTPServer(httpapi.HTTPConfig{ + Authenticator: authenticator, + Authorizer: authorizer, + Logger: logger, CoreReadyCheck: func(ctx context.Context) error { - return lifecycleClient.QueryRows(ctx, "RETURN {ready: true}", 1, nil, func(map[string]any) error { return nil }) + return runtime.Readiness(ctx) }, - ClickHouseReadyCheck: func(ctx context.Context) error { - if degradation != nil { - return degradation - } - if clickhouse == nil { - return nil - } - return clickhouse.Ping(ctx) - }, ClickHouseEnabled: serverConfig.Server.ClickHouse.Enabled}) + CoreState: postgreSQLCoreReadinessState, + ClickHouseReadyCheck: nil, + ClickHouseEnabled: false, + }) if err != nil { - return fmt.Errorf("create HTTP server: %w", err) + return nil, nil, fmt.Errorf("create HTTP server: %w", err) } - explorerHandlers := newExplorerHTTPHandlers(authorizer, func(ctx context.Context, principal *authscope.Principal, project string) error { - if scopeResolver == nil { - return nil - } - return scopeResolver.AuthorizeReadProject(ctx, principal, project) - }, explorerService, lifecycleConfig) - explorerHandlers.constructionInputs = constructionInputsCatalog{ - reader: materializationReader, catalog: publishedRegistry, capabilities: lifecycleConfig.Capability, - scopes: scopeResolver, explorers: explorerService, revisions: explorerService, - } - if err := registerRoutes(server, generationService, authorizer, resolver, explorerHandlers, publishedRegistry, scopeResolver); err != nil { - return fmt.Errorf("register HTTP routes: %w", err) - } - if privateClickHouseArtifacts != nil { - reconcileCtx, cancelReconciler := context.WithCancel(ctx) - reconcilerDone := startPrivateArtifactReconciler(reconcileCtx, privateClickHouseArtifacts, logger) - defer func() { - cancelReconciler() - timer := time.NewTimer(cleanupTimeout) - defer timer.Stop() - select { - case <-reconcilerDone: - case <-timer.C: - logger.Error("private ClickHouse artifact reconciler did not stop before shutdown deadline") - } - }() - } - if artifactStore != nil { - artifactReconcileCtx, cancelArtifactReconciler := context.WithCancel(ctx) - artifactReconcilerDone := startArtifactFSReconciler(artifactReconcileCtx, artifactStore, logger) - defer func() { - cancelArtifactReconciler() - timer := time.NewTimer(cleanupTimeout) - defer timer.Stop() - select { - case <-artifactReconcilerDone: - case <-timer.C: - logger.Error("filesystem artifact reconciler did not stop before shutdown deadline") - } - }() + if err := registerRoutes(server, generationService, authorizer, resolver, nil, routeDependencies{scopes: auth.scopePolicy}); err != nil { + return nil, nil, fmt.Errorf("register HTTP routes: %w", err) } - errCh := make(chan error, 1) - go func() { - logger.Info("starting HTTP server", "listen", serverConfig.Server.Listen, "database", serverConfig.Server.Database, "no_auth", serverConfig.Server.AllowUnauthenticated || serverConfig.Auth.AllowUnauthenticated) - errCh <- server.App().Listen(serverConfig.Server.Listen) - }() + return server, closeRuntime, nil +} - select { - case err := <-errCh: - if err != nil { - return fmt.Errorf("server stopped: %w", err) - } - case <-ctx.Done(): - logger.Info("shutting down HTTP server", "reason", ctx.Err()) - shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) - defer cancel() - if err := server.App().ShutdownWithContext(shutdownCtx); err != nil { - return fmt.Errorf("shutdown failed: %w", err) - } +func runPostgreSQLFirstQueryRoute(ctx context.Context, serverConfig Config, logger *slog.Logger) error { + server, closeRuntime, err := composePostgreSQLFirstQueryRoute(ctx, serverConfig, logger) + if err != nil { + return err } - return nil + _, startSourceSchema, _, _ := postgresFirstQueryRouteAssignment(serverConfig.Server.PostgreSQL) + return serveRuntime(ctx, server, serverConfig.Server.Listen, closeRuntime, "close PostgreSQL source runtime", logger, + "listen", serverConfig.Server.Listen, "backend", serverConfig.Server.Backend, + "source_schema", startSourceSchema, "no_auth", serverConfig.Server.AllowUnauthenticated || serverConfig.Auth.AllowUnauthenticated) } diff --git a/internal/server/zz_readiness_contract_guard_test.go b/internal/server/zz_readiness_contract_guard_test.go new file mode 100644 index 000000000..6a361ef00 --- /dev/null +++ b/internal/server/zz_readiness_contract_guard_test.go @@ -0,0 +1,78 @@ +package server + +import ( + "flag" + "fmt" + "os" + "regexp" + "sync" + "testing" +) + +var readinessBattery = []string{ + "TestPostgresRuntimeHealthReadinessIsIndependentAndSchemaAware", + "TestPostgresRuntimeReadinessReportsNeverMigratedSchema", + "TestPostgresRuntimeReadinessReportsMissingSchema", + "TestPostgresRuntimeVerifySchemaIsReadOnly", + "TestPostgreSQLFirstQueryRouteLivenessAndReadiness", + "TestHealthWiringPerCompositionArm", + "TestPostgreSQLCoreReadinessStateClassification", +} + +var ( + readinessInvokedMu sync.Mutex + readinessInvoked = map[string]bool{} +) + +func markReadinessLeg(t *testing.T, name string) { + t.Helper() + readinessInvokedMu.Lock() + defer readinessInvokedMu.Unlock() + readinessInvoked[name] = true +} + +func TestMain(m *testing.M) { + rc := m.Run() + if fail := readinessContractViolation(); fail != "" { + fmt.Fprintf(os.Stderr, "readiness contract guard: %s\n", fail) + if rc == 0 { + rc = 1 + } + } + os.Exit(rc) +} + +func readinessContractViolation() string { + var pattern string + if f := flag.Lookup("test.run"); f != nil { + pattern = f.Value.String() + } + readinessInvokedMu.Lock() + defer readinessInvokedMu.Unlock() + if pattern == "" { + pattern = ".*" + } + re, err := regexp.Compile(pattern) + if err != nil { + return "" + } + matched := 0 + for _, name := range readinessBattery { + if re.MatchString(name) { + matched++ + } + } + if matched < 2 { + return "" + } + var missing []string + for _, name := range readinessBattery { + if !readinessInvoked[name] { + missing = append(missing, name) + } + } + if len(missing) > 0 { + return fmt.Sprintf("contract-scheduled -run %q invoked an incomplete battery: never invoked %v; pinned battery %v — a renamed or dropped leg must fail the contract, not pass vacuously (F2)", pattern, missing, readinessBattery) + } + return "" +} diff --git a/internal/source/doc.go b/internal/source/doc.go new file mode 100644 index 000000000..d150341cd --- /dev/null +++ b/internal/source/doc.go @@ -0,0 +1 @@ +package source diff --git a/internal/source/ownership_contract_test.go b/internal/source/ownership_contract_test.go new file mode 100644 index 000000000..4d7ad9ff1 --- /dev/null +++ b/internal/source/ownership_contract_test.go @@ -0,0 +1,378 @@ +package source + +import ( + "encoding/json" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +type namedFile struct { + name string + f *ast.File +} + +func repoRoot(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed; cannot locate the repository root") + } + dir := filepath.Dir(file) + root := filepath.Dir(filepath.Dir(dir)) + if fi, err := os.Stat(filepath.Join(root, "go.mod")); err != nil || fi.IsDir() { + t.Fatalf("resolved root %q has no go.mod: %v", root, err) + } + return root +} + +func isStdlibImport(path string) bool { + first := path + if i := strings.Index(path, "/"); i >= 0 { + first = path[:i] + } + return !strings.Contains(first, ".") +} + +func parsePackageFiles(t *testing.T, dir string, withTests bool) []namedFile { + t.Helper() + fset := token.NewFileSet() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read %s: %v", dir, err) + } + var files []namedFile + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".go") { + continue + } + if !withTests && strings.HasSuffix(name, "_test.go") { + continue + } + f, err := parser.ParseFile(fset, filepath.Join(dir, name), nil, parser.ParseComments) + if err != nil { + t.Fatalf("parse %s/%s: %v", dir, name, err) + } + files = append(files, namedFile{name: name, f: f}) + } + if len(files) == 0 { + t.Fatalf("no Go files found in %s", dir) + } + return files +} + +func importedPaths(files []namedFile) []string { + var out []string + for _, nf := range files { + for _, imp := range nf.f.Imports { + out = append(out, strings.Trim(imp.Path.Value, `"`)) + } + } + return out +} + +func selectorName(t *testing.T, expr ast.Expr) string { + t.Helper() + switch e := expr.(type) { + case *ast.Ident: + return e.Name + case *ast.SelectorExpr: + pkg, ok := e.X.(*ast.Ident) + if !ok { + t.Fatalf("selector %v has non-ident package", expr) + } + return pkg.Name + "." + e.Sel.Name + case *ast.StarExpr: + return "*" + selectorName(t, e.X) + case *ast.ArrayType: + return "[]" + selectorName(t, e.Elt) + case *ast.MapType: + return "map[" + selectorName(t, e.Key) + "]" + selectorName(t, e.Value) + default: + t.Fatalf("unsupported type expression %T", expr) + return "" + } +} + +func TestSourceResourceOwnershipContract(t *testing.T) { + root := repoRoot(t) + sourceDir := filepath.Join(root, "internal", "source") + catalogDir := filepath.Join(root, "internal", "catalog") + storeDir := filepath.Join(root, "internal", "backends", "postgres", "internal", "store") + ingestDir := filepath.Join(root, "internal", "ingest") + + t.Run("SourceLeafIsStdlibOnly", func(t *testing.T) { + files := parsePackageFiles(t, sourceDir, false) + for _, imp := range importedPaths(files) { + if !isStdlibImport(imp) { + t.Errorf("internal/source production import %q is not standard library; "+ + "the logical contract leaf must depend on no backend, driver, or catalog package", imp) + } + } + for _, imp := range []string{"github.com/calypr/loom/internal/catalog", + "github.com/calypr/loom/internal/backends/postgres/internal/store", + "github.com/jackc/pgx/v5", "github.com/arangodb/go-driver"} { + for _, got := range importedPaths(files) { + if got == imp { + t.Errorf("internal/source imports %q; forbidden for the neutral contract", imp) + } + } + } + }) + + t.Run("ResourceCarriesOnlyLogicalIdentity", func(t *testing.T) { + want := []string{"Project", "DatasetGeneration", "ResourceType", "LogicalID", "AuthResourcePath", "Payload"} + typ := reflect.TypeOf(Resource{}) + if typ.NumField() != len(want) { + t.Fatalf("Resource has %d fields, want exactly %d (%v)", typ.NumField(), len(want), want) + } + for i, name := range want { + f := typ.Field(i) + if f.Name != name { + t.Errorf("field %d is %q, want %q", i, f.Name, name) + } + if f.Name == "Payload" { + if f.Type != reflect.TypeOf(json.RawMessage(nil)) { + t.Errorf("Payload type = %v, want json.RawMessage", f.Type) + } + continue + } + if f.Type.Kind() != reflect.String { + t.Errorf("field %s kind = %v, want string", f.Name, f.Type.Kind()) + } + } + for i := 0; i < typ.NumField(); i++ { + f := typ.Field(i) + switch f.Type.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + t.Errorf("field %s has integer kind %v; the logical contract must not carry a "+ + "storage sequence or any database-assigned number", f.Name, f.Type.Kind()) + } + if strings.Contains(strings.ToLower(f.Name), "seq") || + strings.Contains(strings.ToLower(f.Name), "cursor") { + t.Errorf("field %s names a storage sequence/cursor; physical layout must stay "+ + "owned by the storage implementation", f.Name) + } + } + }) + + t.Run("ScopeTriStatesAreDistinctAndExplicit", func(t *testing.T) { + typ := reflect.TypeOf(ResourceScope{}) + flag, ok := typ.FieldByName("AuthResourcePathsUnrestricted") + if !ok { + t.Fatal("ResourceScope lost AuthResourcePathsUnrestricted") + } + if flag.Type.Kind() != reflect.Ptr || flag.Type.Elem().Kind() != reflect.Bool { + t.Fatalf("AuthResourcePathsUnrestricted type = %v, want *bool (tri-state)", flag.Type) + } + yes := true + no := false + restricted := ResourceScope{Project: "p", AuthResourcePathsUnrestricted: &no, + AuthResourcePaths: []string{"a/b"}} + restrictedEmpty := ResourceScope{Project: "p", AuthResourcePathsUnrestricted: &no} + unrestricted := ResourceScope{Project: "p", AuthResourcePathsUnrestricted: &yes} + omittedGeneration := ResourceScope{Project: "p", AuthResourcePathsUnrestricted: &no} + if reflect.DeepEqual(restrictedEmpty, unrestricted) { + t.Error("restricted-empty and explicit-unrestricted scopes compare equal; the two modes must stay distinct") + } + if reflect.DeepEqual(restricted, restrictedEmpty) || reflect.DeepEqual(restricted, unrestricted) { + t.Error("a restricted scope with authorized paths must differ from both restricted-empty and unrestricted scopes") + } + if omittedGeneration.DatasetGeneration != "" { + t.Error("omitted dataset generation must be the empty legacy namespace") + } + nilFlag := ResourceScope{Project: "p"} + if nilFlag.AuthResourcePathsUnrestricted != nil { + t.Error("zero-value scope must leave the unrestricted flag nil (rejected state)") + } + }) + + t.Run("GenericCatalogExportsNoPostgreSQLRowOrScope", func(t *testing.T) { + files := parsePackageFiles(t, catalogDir, false) + var bad []string + for _, nf := range files { + f := nf.f + for _, decl := range f.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.TYPE { + continue + } + for _, spec := range gen.Specs { + ts, ok := spec.(*ast.TypeSpec) + if !ok || !ts.Name.IsExported() { + continue + } + n := ts.Name.Name + if strings.HasPrefix(n, "PG") || strings.HasSuffix(n, "ResourceRow") || + strings.HasSuffix(n, "ResourceScope") || strings.Contains(n, "ResourceSeq") { + bad = append(bad, nf.name+":"+n) + } + } + } + } + if len(bad) > 0 { + t.Errorf("internal/catalog exports PostgreSQL storage DTO names %v; physical resource "+ + "rows/scopes belong to the storage implementation", bad) + } + forbidden := []string{"internal/store", "internal/backends", "pgx", "arangodb", + "jackc", "database/sql"} + for _, imp := range importedPaths(files) { + for _, bad := range forbidden { + if strings.Contains(imp, bad) { + t.Errorf("internal/catalog production import %q touches backend/driver territory (%q)", imp, bad) + } + } + } + }) + + t.Run("PhysicalRowAndWriteSurfacesHaveOneOwnerEach", func(t *testing.T) { + files := parsePackageFiles(t, storeDir, false) + holders := map[string]string{} + structs := map[string]*ast.StructType{} + for _, nf := range files { + for _, decl := range nf.f.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.TYPE { + continue + } + for _, spec := range gen.Specs { + ts, ok := spec.(*ast.TypeSpec) + if !ok { + continue + } + st, ok := ts.Type.(*ast.StructType) + if !ok { + continue + } + structs[ts.Name.Name] = st + for _, fld := range st.Fields.List { + for _, name := range fld.Names { + if name.Name == "ResourceSeq" { + holders[ts.Name.Name] = nf.name + } + } + } + } + } + } + if len(holders) != 1 || holders["ResourceRow"] == "" { + t.Errorf("structs carrying ResourceSeq = %v; exactly one store-owned physical "+ + "row type (ResourceRow) may hold the storage sequence", holders) + } + if rr, ok := structs["ResourceRow"]; !ok { + t.Fatal("the backend store lost its physical ResourceRow owner") + } else { + hasSeq := false + hasLogical := false + for _, fld := range rr.Fields.List { + if len(fld.Names) == 0 && selectorName(t, fld.Type) == "source.Resource" { + hasLogical = true + continue + } + if len(fld.Names) == 1 && fld.Names[0].Name == "ResourceSeq" { + hasSeq = true + continue + } + t.Errorf("ResourceRow carries an unexpected field %v; the physical row is exactly "+ + "the logical source contract plus the store-assigned sequence", fld.Names) + } + if !hasLogical { + t.Error("ResourceRow must anonymously embed source.Resource so promoted logical fields stay stable") + } + if !hasSeq { + t.Error("ResourceRow must declare the physical ResourceSeq field") + } + } + + surfaces := map[string]string{} + for _, nf := range files { + base := nf.name + for _, decl := range nf.f.Decls { + switch d := decl.(type) { + case *ast.FuncDecl: + switch d.Name.Name { + case "InsertResource": + for _, fl := range d.Type.Params.List { + for _, nm := range fl.Names { + if nm.Name == "rec" { + surfaces[base+":InsertResource.rec"] = selectorName(t, fl.Type) + } + } + } + } + case *ast.GenDecl: + if d.Tok != token.TYPE { + continue + } + for _, spec := range d.Specs { + ts, ok := spec.(*ast.TypeSpec) + if !ok { + continue + } + st, ok := ts.Type.(*ast.StructType) + if !ok { + continue + } + for _, fld := range st.Fields.List { + for _, nm := range fld.Names { + key := base + ":" + ts.Name.Name + "." + nm.Name + if key == "resource_writer.go:ResourceBatch.Rows" || + key == "batch_writer.go:Batch.Resources" { + surfaces[key] = selectorName(t, fld.Type) + } + } + } + } + } + } + } + want := map[string]string{ + "resource.go:InsertResource.rec": "source.Resource", + "resource_writer.go:ResourceBatch.Rows": "[]source.Resource", + "batch_writer.go:Batch.Resources": "[]source.Resource", + } + for key, expected := range want { + got, ok := surfaces[key] + if !ok { + t.Errorf("write surface %s not found in the store AST", key) + continue + } + if got != expected { + t.Errorf("write surface %s has type %q, want %q; callers must never be able to "+ + "offer the physical row (or its server-assigned sequence)", key, got, expected) + } + } + }) + + t.Run("IngestNeverTouchesPhysicalSequence", func(t *testing.T) { + files := parsePackageFiles(t, ingestDir, false) + var hits []string + for _, nf := range files { + ast.Inspect(nf.f, func(n ast.Node) bool { + switch e := n.(type) { + case *ast.SelectorExpr: + if e.Sel.Name == "ResourceSeq" { + hits = append(hits, nf.name) + } + case *ast.Ident: + if e.Name == "ResourceSeq" { + hits = append(hits, nf.name) + } + } + return true + }) + } + if len(hits) > 0 { + t.Errorf("internal/ingest production files reference the physical ResourceSeq in %v; "+ + "ingestion must build only the logical source contract", hits) + } + }) +} diff --git a/internal/source/resource.go b/internal/source/resource.go new file mode 100644 index 000000000..d6b8ebc99 --- /dev/null +++ b/internal/source/resource.go @@ -0,0 +1,19 @@ +package source + +import "encoding/json" + +type Resource struct { + Project string + DatasetGeneration string + ResourceType string + LogicalID string + AuthResourcePath string + Payload json.RawMessage +} + +type ResourceScope struct { + Project string + DatasetGeneration string + AuthResourcePathsUnrestricted *bool + AuthResourcePaths []string +} diff --git a/internal/store/clickhouse/client.go b/internal/store/clickhouse/client.go index 6111f2e7a..9d359754a 100644 --- a/internal/store/clickhouse/client.go +++ b/internal/store/clickhouse/client.go @@ -1,6 +1,3 @@ -// Package clickhouse is Loom's narrow typed boundary around the official -// ClickHouse Go driver. The materialization service never receives a raw SQL -// connection or a table name from the browser. package clickhouse import ( @@ -19,9 +16,6 @@ import ( ) type Options struct { - // URL accepts clickhouse://host:9000 or http://host:8123 URLs. The - // official driver supports both protocols; native TCP is the default - // recommended production transport. URL string Database string Username string @@ -70,7 +64,6 @@ func (c *Client) Close() error { return c.conn.Close() } -// Ping verifies that the configured ClickHouse connection is reachable. func (c *Client) Ping(ctx context.Context) error { if c == nil || c.conn == nil { return fmt.Errorf("ClickHouse connection is not configured") @@ -103,8 +96,6 @@ func (c *Client) EnsureDatabase(ctx context.Context) error { if err := validateIdentifier(c.opts.Database); err != nil { return err } - // CREATE DATABASE must execute through a connection whose default database - // is known to exist; the main connection intentionally targets the new DB. options, err := parseOptions(c.opts) if err != nil { return err @@ -135,7 +126,6 @@ func (c *Client) DropTable(ctx context.Context, table string) error { return c.conn.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS `%s`", table)) } -// DropColumns removes validated staging columns in one ALTER TABLE operation. func (c *Client) DropColumns(ctx context.Context, table string, columns []string) error { if err := validateIdentifier(table); err != nil { return err @@ -153,9 +143,6 @@ func (c *Client) DropColumns(ctx context.Context, table string, columns []string return c.conn.Exec(ctx, fmt.Sprintf("ALTER TABLE `%s` %s", table, strings.Join(parts, ", "))) } -// VerifyOutput confirms that a staged publication table is readable, has the -// expected physical columns, and contains exactly the rows acknowledged by -// the publisher before any visibility pointer is advanced. func (c *Client) VerifyOutput(ctx context.Context, table string, columns []Column, expectedRows int64) error { if err := validateIdentifier(table); err != nil { return err @@ -585,9 +572,6 @@ func wrappedClickHouseType(value, wrapper string) (string, bool) { return strings.TrimSpace(value[len(prefix) : len(value)-1]), true } -// QueryRowsArgs executes a SELECT with driver-bound positional arguments and -// decodes each row through ClickHouse's native driver. Query-controlled values -// must be supplied as args rather than interpolated into query text. func (c *Client) QueryRowsArgs(ctx context.Context, query string, columns []string, args ...any) ([]map[string]any, error) { result := make([]map[string]any, 0) if err := c.QueryRowsArgsVisit(ctx, query, columns, func(row map[string]any) error { @@ -599,9 +583,6 @@ func (c *Client) QueryRowsArgs(ctx context.Context, query string, columns []stri return result, nil } -// QueryRowsArgsVisit executes a SELECT and visits decoded rows as they arrive. -// The callback must not retain the map after returning unless it makes a copy. -// Query-controlled values remain driver-bound positional arguments. func (c *Client) QueryRowsArgsVisit(ctx context.Context, query string, columns []string, visit func(map[string]any) error, args ...any) error { if len(columns) == 0 { return fmt.Errorf("ClickHouse query columns are required") @@ -658,9 +639,6 @@ func parseOptions(opts Options) (*ch.Options, error) { parsed.Auth.Password = opts.Password parsed.DialTimeout = opts.Timeout parsed.ReadTimeout = opts.Timeout - // Native JSON columns are used for logical object-valued dataframe fields. - // Keep the setting on every connection so DDL and native batch inserts work - // consistently against ClickHouse 25.3 deployments. if parsed.Settings == nil { parsed.Settings = ch.Settings{} } diff --git a/internal/store/clickhouse/integration_test.go b/internal/store/clickhouse/integration_test.go index 9de9671cb..561fda6cf 100644 --- a/internal/store/clickhouse/integration_test.go +++ b/internal/store/clickhouse/integration_test.go @@ -16,10 +16,6 @@ import ( "github.com/google/uuid" ) -// Run with LOOM_CLICKHOUSE_URL and optional LOOM_CLICKHOUSE_USERNAME/ -// LOOM_CLICKHOUSE_PASSWORD to exercise the native driver against a real -// ClickHouse instance. The default unit suite remains hermetic when ClickHouse -// is not running locally. func TestClickHouseNativeRoundTrip(t *testing.T) { url := os.Getenv("LOOM_CLICKHOUSE_URL") if url == "" { @@ -216,10 +212,6 @@ func TestClickHouseNativeOWNERRecordsJSONRoundTrip(t *testing.T) { if err := decoder.Decode(&input); err != nil { t.Fatalf("decode OWNER_RECORDS fixture %d: %v", index, err) } - // The materialized OWNER_RECORDS contract distinguishes an explicit - // null-valued object path from a path that was absent in the source. - // Preserve the exact source values here, including null object fields - // and null array elements, so this roundtrip catches either being lost. want[index] = input rows[index] = map[string]any{ "__loom_row_id": fmt.Sprintf("%d", index+1), diff --git a/loom-dev-build-stamp.sh b/loom-dev-build-stamp.sh index 70575e63d..65f5d2841 100755 --- a/loom-dev-build-stamp.sh +++ b/loom-dev-build-stamp.sh @@ -2,9 +2,6 @@ set -eu source_digest() { - # The source digest includes file contents and names, so edits and deletions - # cannot be mistaken for the prior process merely because a timestamp is - # unchanged. { sha256sum /workspace/go.mod /workspace/go.sum find /workspace/cmd /workspace/internal /workspace/generated /workspace/schemas \ @@ -47,10 +44,6 @@ if [ "${1:-}" = "--check" ]; then esac done printf '%s %s %s\n' "$expected_source" "$source_digest" "$running_binary" - # Air may unlink its build artifact after exec. The running executable - # remains readable through /proc, so compare it with the digest recorded - # immediately after the successful build instead of requiring the path to - # remain present. [ "$expected_source" = "$source_digest" ] && [ "$running_binary" = "$expected_binary" ] exit fi diff --git a/openapi/oapi-codegen.yaml b/openapi/oapi-codegen.yaml index 7cc590d4d..28f33921f 100644 --- a/openapi/oapi-codegen.yaml +++ b/openapi/oapi-codegen.yaml @@ -1,4 +1,3 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/oapi-codegen/oapi-codegen/v2.8.0/configuration-schema.json package: loomapi generate: models: true diff --git a/scripts/acceptance-performance.sh b/scripts/acceptance-performance.sh index 9361e312f..3702a7e81 100755 --- a/scripts/acceptance-performance.sh +++ b/scripts/acceptance-performance.sh @@ -57,8 +57,6 @@ if ! git archive "$base_ref" | tar -x -C "$base_source"; then exit $? fi -# Validate/fetch once on the host. Each Compose project mounts this same -# content-addressed directory, so base/current variants do not refetch it. if [[ -n "${LOOM_ACCEPTANCE_GOCACHE:-}" ]]; then GOCACHE="$LOOM_ACCEPTANCE_GOCACHE" go run ./cmd/loom-acceptance --fixture-only --fixture-cache "$cache" else @@ -113,8 +111,6 @@ run_variant() { acceptance_status=$? fi - # Only generated project names reach this cleanup. The canonical loom-demo - # project is never passed to demo-down by the performance driver. if LOOM_DEMO_COMPOSE_PROJECT="$project" "$repo_root/scripts/demo-down.sh" --volumes --remove-orphans \ >"$variant_artifacts/compose-down.log" 2>&1; then active_project="" diff --git a/scripts/acceptance-real.sh b/scripts/acceptance-real.sh index e59d22204..4ab1dd5da 100755 --- a/scripts/acceptance-real.sh +++ b/scripts/acceptance-real.sh @@ -35,8 +35,6 @@ deployment_project=$canonical_project acceptance_project=$canonical_project teardown_acceptance=false if [[ "$isolated" == false ]]; then - # Rebuild the user's canonical demo from the current source without changing - # its repository data. The locked fixture runs separately below. LOOM_DEMO_SEED=false \ LOOM_DEMO_SOURCE_ROOT="$source_root" \ "$repo_root/scripts/demo-up.sh" diff --git a/scripts/audit-http-500-responses.py b/scripts/audit-http-500-responses.py index 7fbc83bbd..1b964998e 100644 --- a/scripts/audit-http-500-responses.py +++ b/scripts/audit-http-500-responses.py @@ -40,7 +40,6 @@ def mapper_for(lines: list[str], start: int, end: int) -> str: def case_returns_case_body(lines: list[str], case_index: int) -> str: - # Generated adapters keep the return on the case line or the next line. window = "\n".join(lines[case_index : case_index + 3]) match = BODY_RE.search(window) return match.group("var") if match else "" diff --git a/scripts/audit-lightpanda-coverage.mjs b/scripts/audit-lightpanda-coverage.mjs index ba357ad20..7b6338f10 100644 --- a/scripts/audit-lightpanda-coverage.mjs +++ b/scripts/audit-lightpanda-coverage.mjs @@ -10,7 +10,6 @@ const recovery = '5646633e0e5c7ff8de568a4f111478d121e7f75f'; const git = (...args) => execFileSync('git', args, { cwd: root, encoding: 'utf8', maxBuffer: 16 * 1024 * 1024 }); const paths = git('ls-tree', '-r', '--name-only', recovery, 'scripts').trim().split('\n').filter(path => path.endsWith('.spec.mjs')); -// These are coverage dispositions, not claims of equivalent replacement tests. const families = [ { id: 'small-preview', names: ['builder-preview-read-only', 'load-preview', 'builder-load'], decision: 'Consolidate into one exact-row load/reload candidate; Lightpanda timing pilot only.', diff --git a/scripts/backend-conformance-skipset-capture.json b/scripts/backend-conformance-skipset-capture.json new file mode 100644 index 000000000..b4869db5f --- /dev/null +++ b/scripts/backend-conformance-skipset-capture.json @@ -0,0 +1,9 @@ +{ + "package": "./internal/server", + "goVersion": "go version go1.26.5 linux/amd64", + "skips": [ + "TestBackendArchitectureConformance/arangodb", + "TestBackendArchitectureConformance/crossBackendCandidateIdentityParity", + "TestBackendArchitectureConformance/postgresql" + ] +} diff --git a/scripts/backend-conformance-skipset-verify.mjs b/scripts/backend-conformance-skipset-verify.mjs new file mode 100644 index 000000000..b7f934c52 --- /dev/null +++ b/scripts/backend-conformance-skipset-verify.mjs @@ -0,0 +1,40 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { backendArchitectureConformance } from './run-arango-integration.mjs'; + +const here = dirname(fileURLToPath(import.meta.url)); +const CAPTURE_PATH = join(here, 'backend-conformance-skipset-capture.json'); +const CAPTURE_SHA256 = 'e57f9462aa7014306909a444f9f150d1e77c3e56df19d42d041f14f3dc893554'; + +const raw = readFileSync(CAPTURE_PATH); +const observed = createHash('sha256').update(raw).digest('hex'); +assert.equal(observed, CAPTURE_SHA256, `capture drifted from its pinned sha256: ${observed}`); +const capture = JSON.parse(raw); +assert.match(capture.goVersion, /^go version go1\.26\.5\b/, `capture pinned an unexpected toolchain: ${capture.goVersion}`); + +const captureSet = new Set(capture.skips); +assert.ok(capture.skips.length > 0, 'capture holds no skip identities'); + +for (const name of captureSet) assert.ok(backendArchitectureConformance.has(name), `captured skip not in allowlist: ${name}`); +for (const name of backendArchitectureConformance) assert.ok(captureSet.has(name), `allowlist entry with no captured skip: ${name}`); + +const stdout = execFileSync('go', ['test', capture.package, '-count=1', '-json', '-timeout', '300s', '-run', '^TestBackendArchitectureConformance$'], { + cwd: join(here, '..'), + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + env: { ...process.env, LOOM_TEST_POSTGRES_URL: '', LOOM_TEST_ARANGO_URL: '' }, +}); +const fresh = new Set(); +for (const line of stdout.trim().split('\n').filter(Boolean)) { + const event = JSON.parse(line); + if (event.Test && event.Action === 'skip' && event.Test.startsWith('TestBackendArchitectureConformance')) fresh.add(event.Test); + assert.ok(!(event.Action === 'fail'), `offline go test reported a failure: ${event.Test ?? capture.package}`); +} +for (const name of fresh) assert.ok(captureSet.has(name), `fresh offline skip missing from the capture: ${name}`); +for (const name of captureSet) assert.ok(fresh.has(name), `captured skip no longer skips offline: ${name}`); + +console.log(`backend-conformance skipset verified: ${captureSet.size} identities, capture sha256 ${observed}`); diff --git a/scripts/browser/artifact-cache.mjs b/scripts/browser/artifact-cache.mjs index 6320acb20..b27206c97 100644 --- a/scripts/browser/artifact-cache.mjs +++ b/scripts/browser/artifact-cache.mjs @@ -15,7 +15,6 @@ async function verifyEntry(directory, key) { assert.ok(names.includes('app.js') && names.includes('app.css')); assert.equal(new Set(names).size, names.length); assert.ok(names.every(validName)); - // Read the bytes we hash, so later writes copy exactly what was verified. return Promise.all(manifest.files.map(async file => { const bytes = await readFile(join(directory, key, file.name)); assert.equal(digest(bytes), file.sha256, `Corrupt cached asset ${file.name}`); @@ -31,9 +30,6 @@ export async function restoreArtifacts({ directory, key, destination }) { verified = await verifyEntry(directory, key); } catch (error) { const status = error.code === 'ENOENT' ? 'miss' : 'invalid'; - // A key is content-addressed, so a non-verifying entry is never legitimate. - // Remove it so the rebuild's publish can commit instead of looping on - // ENOTEMPTY forever. Best effort: publish re-verifies and repairs too. if (status === 'invalid') await rm(entry, { recursive: true, force: true }).catch(() => {}); return { status, reason: error.message }; } @@ -60,9 +56,9 @@ export async function publishArtifacts({ directory, key, source }) { try { await rename(staging, entry); break; } catch (error) { if (!['EEXIST', 'ENOTEMPTY'].includes(error.code)) throw error; - try { await verifyEntry(directory, key); break; } // A concurrent publisher committed a valid entry. + try { await verifyEntry(directory, key); break; } catch { if (attempt >= 3) throw error; } - await rm(entry, { recursive: true, force: true }); // Stale or corrupt: the fresh staging replaces it. + await rm(entry, { recursive: true, force: true }); } } } finally { await rm(staging, { recursive: true, force: true }); } diff --git a/scripts/browser/artifact-cache.test.mjs b/scripts/browser/artifact-cache.test.mjs index 8a87aceb3..a58d9837c 100644 --- a/scripts/browser/artifact-cache.test.mjs +++ b/scripts/browser/artifact-cache.test.mjs @@ -45,9 +45,9 @@ test('publish replaces a stale partial entry instead of swallowing the collision await writeFile(join(source, 'app.js'), 'healed js'); await writeFile(join(source, 'app.css'), '.healed { color: blue }'); await publishArtifacts({ directory, key, source }); - await rm(join(directory, key, 'app.css')); // Asset removed after a valid publish. + await rm(join(directory, key, 'app.css')); assert.equal((await restoreArtifacts({ directory, key, destination })).status, 'miss'); - await publishArtifacts({ directory, key, source }); // Must replace the stale directory, not no-op on ENOTEMPTY. + await publishArtifacts({ directory, key, source }); assert.equal((await restoreArtifacts({ directory, key, destination })).status, 'hit', 'The next publish commits over a stale entry'); assert.equal(await readFile(join(destination, 'app.css'), 'utf8'), '.healed { color: blue }'); } finally { await rm(directory, { recursive: true, force: true }); } diff --git a/scripts/browser/benchmark-chromium.mjs b/scripts/browser/benchmark-chromium.mjs index 047e4d21a..badaace70 100644 --- a/scripts/browser/benchmark-chromium.mjs +++ b/scripts/browser/benchmark-chromium.mjs @@ -17,7 +17,7 @@ options.runs = Number(options.runs); assert.ok(Number.isInteger(options.runs) && options.runs >= 1 && options.runs <= 30, '--runs must be 1..30'); assert.ok(['cold', 'warm'].includes(options.cache), '--cache must be cold or warm'); for (const key of ['baseline', 'candidate', 'chrome', 'out']) options[key] = resolve(options[key]); -await mkdir(options.out); // Every invocation owns a new evidence directory. +await mkdir(options.out); const receipt = { status: 'running', options, host: { cores: availableParallelism(), loadStart: loadavg() }, samples: [] }; const fingerprints = new Map(); let expected; @@ -36,7 +36,6 @@ async function run(side, index) { const out = join(options.out, `${index}-${side}`); const log = await open(`${out}.log`, 'wx'); const started = performance.now(); - // The separate process group belongs exclusively to this trial, including Chrome. const child = spawn(process.execPath, ['scripts/browser/run.mjs', '--feature', 'all', '--chrome', options.chrome, '--out', out, ...(side === 'candidate' && options.cache === 'cold' ? ['--no-cache'] : [])], { cwd: options[side], detached: true, stdio: ['ignore', log.fd, log.fd] }); diff --git a/scripts/browser/browser-pool.mjs b/scripts/browser/browser-pool.mjs index f6ecfc3af..414ce569e 100644 --- a/scripts/browser/browser-pool.mjs +++ b/scripts/browser/browser-pool.mjs @@ -38,7 +38,7 @@ export async function openBrowserPool({ engine, concurrency, chrome, lightpanda, while (performance.now() < deadline) { if (spawnError) throw spawnError; assert.equal(child.exitCode, null, `Lightpanda exited: ${resource.stderr}`); - try { ready = (await fetch(`http://127.0.0.1:${port}/json/version`, { signal: AbortSignal.timeout(500) })).ok; } catch { /* Starting. */ } + try { ready = (await fetch(`http://127.0.0.1:${port}/json/version`, { signal: AbortSignal.timeout(500) })).ok; } catch { } if (ready) break; await delay(10); } @@ -71,8 +71,6 @@ export async function openBrowserPool({ engine, concurrency, chrome, lightpanda, lightpandaStopped: resources.every(({ child }) => !child?.pid || child.exitCode !== null || child.signalCode !== null), errors }; } try { - // Chromium already parallelizes renderers. Lightpanda needs independent V8 - // processes to use several cores. Keep both alive across the entire batch. const chromiumBrowser = engine === 'chromium' ? await openBrowser() : undefined; const ready = await Promise.allSettled(Array.from({ length: concurrency }, async (_, id) => { const browser = chromiumBrowser ?? await openBrowser(); diff --git a/scripts/browser/cdp.mjs b/scripts/browser/cdp.mjs index 4a33c6e95..620407ddc 100644 --- a/scripts/browser/cdp.mjs +++ b/scripts/browser/cdp.mjs @@ -1,4 +1,3 @@ -// Shared dependency-free Chrome DevTools transport. Importing this module starts nothing. export class CDPClient { constructor(socket) { this.socket = socket; diff --git a/scripts/browser/compare-engines.mjs b/scripts/browser/compare-engines.mjs index bfc62e236..614ea5fd9 100644 --- a/scripts/browser/compare-engines.mjs +++ b/scripts/browser/compare-engines.mjs @@ -146,8 +146,6 @@ async function trial(engine, index, concurrency, phase = 'confirm', existingPool sample(); profiler = setInterval(sample, 500); } if (options.mode === 'throughput') { - // Warm each reused context/cache identically. Blank startup and shader/cache - // warmup are outside steady time, but inside the complete batch duration. result.warmupMs = existingPool ? 0 : await warmPool(pool); const before = performance.now(), cpu = process.cpuUsage(), beforeWorkRequests = { ...served }; result.cases = await runJobs(jobs, pool.workers, (job, worker) => executeJob(job, worker, phase, index, concurrency)); @@ -157,8 +155,6 @@ async function trial(engine, index, concurrency, phase = 'confirm', existingPool result.driverCoreUtilization = result.driverCpuMs / result.steadyMs; result.workRequests = requestDelta(beforeWorkRequests); } else { - // Native compatibility obligations retain fresh contexts and all original - // visibility, typing and pointer checks. They are not semantic throughput. await bounded(pool.workers[0].context.close(), 'release unused worker context'); for (const job of jobs) { const browser = pool.workers[0].browser; @@ -291,8 +287,6 @@ try { } finalized.push({ engine, pool, group }); } - // Close every pool before persisting receipts. A full/read-only evidence - // volume must not strand another engine's live workers. for (const { engine, pool, group } of finalized) { for (const result of group) await writeFile(join(out, `confirm-${result.concurrency}-${result.index}-${engine}.json`), JSON.stringify(result, null, 2) + '\n'); diff --git a/scripts/browser/engine-verdict.mjs b/scripts/browser/engine-verdict.mjs index b66e2d36a..f19c98d46 100644 --- a/scripts/browser/engine-verdict.mjs +++ b/scripts/browser/engine-verdict.mjs @@ -12,8 +12,6 @@ export function chooseConcurrency(trials) { const valid = trials.filter(trial => trial.status === 'passed' && Number.isFinite(trial.steadyMs) && trial.steadyMs > 0); if (!valid.length) throw new Error('No correct tuning configuration'); const fastest = Math.min(...valid.map(trial => trial.steadyMs)); - // Prefer fewer processes when exploratory work times are within five percent. - // Separate confirmation trials validate the selected setting. return valid.filter(trial => trial.steadyMs <= fastest * 1.05).sort((a, b) => a.concurrency - b.concurrency)[0].concurrency; } diff --git a/scripts/browser/lightpanda-cases.mjs b/scripts/browser/lightpanda-cases.mjs index 4176b698a..e1dd1c45c 100644 --- a/scripts/browser/lightpanda-cases.mjs +++ b/scripts/browser/lightpanda-cases.mjs @@ -10,8 +10,6 @@ const rows = page => page.locator('[data-testid="route-preview"] [role="row"]'). .filter(Boolean)).filter(row => row.length)); async function semanticSnapshot(page, selector) { - // One browser-side observation avoids injected locator/actionability machinery - // for DOM-only contracts. Native visibility/input checks stay in `cases` below. return page.evaluate(async selector => { if (!document.querySelector(selector)) { await new Promise((resolve, reject) => { diff --git a/scripts/check-boundaries/main.go b/scripts/check-boundaries/main.go new file mode 100644 index 000000000..65151d92f --- /dev/null +++ b/scripts/check-boundaries/main.go @@ -0,0 +1,409 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/calypr/loom/scripts/packagepolicy" +) + +func main() { + selfTest := flag.Bool("self-test", false, "run negative fixture controls instead of the repository scan") + root := flag.String("root", ".", "repository root") + flag.Parse() + + if *selfTest { + if err := runSelfTest(); err != nil { + fmt.Fprintf(os.Stderr, "boundary self-test FAILED: %v\n", err) + os.Exit(1) + } + fmt.Println("boundary self-test: ok") + return + } + + res := packagepolicy.ScanAll(*root) + for _, e := range res.Errors { + fmt.Fprintln(os.Stderr, e.String()) + } + for _, f := range res.Findings { + fmt.Fprintln(os.Stderr, f.String()) + } + absences := packagepolicy.CheckAbsences(*root, packagepolicy.RepoAbsences()) + for _, e := range absences { + fmt.Fprintln(os.Stderr, e.String()) + } + for _, rule := range packagepolicy.RepoNoReExport() { + for _, e := range packagepolicy.CheckNoReExport(*root, rule) { + absences = append(absences, e) + fmt.Fprintln(os.Stderr, e.String()) + } + } + if len(res.Findings) > 0 || len(res.Errors) > 0 || len(absences) > 0 { + fmt.Fprintf(os.Stderr, "dataframe package boundary check failed: %d violation(s), %d verification failure(s), %d absence failure(s) over %d file(s)\n", + len(res.Findings), len(res.Errors), len(absences), res.Scanned) + os.Exit(1) + } + fmt.Printf("dataframe package boundaries: ok (%d rules, %d files scanned)\n", + len(packagepolicy.RepoRules()), res.Scanned) +} + +type control struct { + name string + setup func(t *fixture) + check func(t *fixture) error +} + +type fixture struct { + dir string + phase string +} + +func runSelfTest() error { + for _, c := range controls() { + dir, err := os.MkdirTemp("", "arch09-selftest-*") + if err != nil { + return err + } + f := &fixture{dir: dir} + err = runControl(c, f) + rmErr := os.RemoveAll(dir) + if err != nil { + return fmt.Errorf("control %q: %w", c.name, err) + } + if rmErr != nil { + return fmt.Errorf("control %q: cleanup: %w", c.name, rmErr) + } + } + return nil +} + +func runControl(c control, f *fixture) error { + f.phase = "setup" + c.setup(f) + f.phase = "check" + return c.check(f) +} + +func (f *fixture) write(rel, body string) string { + path := filepath.Join(f.dir, rel) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + panic(fmt.Sprintf("fixture write %s: %v", path, err)) + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + panic(fmt.Sprintf("fixture write %s: %v", path, err)) + } + return path +} + +func forbiddenImportRule() packagepolicy.Rule { + rule, ok := packagepolicy.ByLabel("compiler-production: no backend implementations or database drivers") + if !ok { + panic("authoritative compiler-production rule missing from the table") + } + return rule +} + +func neutralDomainRule() packagepolicy.Rule { + rule, ok := packagepolicy.ByLabel("neutral-domains: no backend implementations or database drivers") + if !ok { + panic("authoritative neutral-domains rule missing from the table") + } + return rule +} + +func rendererRules() (packagepolicy.Rule, packagepolicy.Rule) { + aql, ok := packagepolicy.ByLabel("render-aql: no lowerer, optimizer, runtime, backends, or drivers") + if !ok { + panic("authoritative render-aql rule missing from the table") + } + pg, ok := packagepolicy.ByLabel("render-postgres: no runtime, backends, stores, or drivers") + if !ok { + panic("authoritative render-postgres rule missing from the table") + } + return aql, pg +} + +func oldPathsRule() packagepolicy.Rule { + rule, ok := packagepolicy.ByLabel("old-concrete-paths: relocated under internal/backends by arch-backend-06/07") + if !ok { + panic("authoritative old-concrete-paths rule missing from the table") + } + return rule +} + +func querierBridgeAbsenceRule() packagepolicy.AbsenceRule { + for _, r := range packagepolicy.RepoAbsences() { + if r.Label == "removed untyped row-querier bridge (arch-backend-09)" { + return r + } + } + panic("authoritative QuerierAsAny absence rule missing from the table") +} + +func expectAbsenceError(errs []packagepolicy.ScanError, kind packagepolicy.ErrorKind, wantSub string) error { + for _, e := range errs { + if e.Kind == kind && strings.Contains(e.Reason, wantSub) { + return nil + } + } + return fmt.Errorf("expected a %s absence failure naming %q; got %v", kind, wantSub, errs) +} + +func expectAbsencesClean(errs []packagepolicy.ScanError) error { + if len(errs) != 0 { + return fmt.Errorf("expected clean absences; got %v", errs) + } + return nil +} + +func scanPath(rule packagepolicy.Rule, path string) packagepolicy.Result { + return packagepolicy.ScanTarget(path, rule.Label, rule.Owner, rule.Forbidden, rule.RejectNonStdlib, rule.Scope, true) +} + +func expectFinding(res packagepolicy.Result, ruleLabel, victimFile, victimImport string) error { + if len(res.Errors) != 0 { + return fmt.Errorf("verification failures must not stand in for detection: %v", res.Errors) + } + for _, f := range res.ImportFindings() { + if strings.Contains(f.File, victimFile) && strings.Contains(f.Import, victimImport) && f.Rule == ruleLabel { + if f.Owner == "" { + return fmt.Errorf("finding %q must carry allowed-owner guidance", f.String()) + } + if !strings.Contains(f.String(), "allowed owner") { + return fmt.Errorf("finding rendering must name the allowed owner: %s", f.String()) + } + return nil + } + } + return fmt.Errorf("guard did not name victim %s importing %s under rule %q; findings: %v", + victimFile, victimImport, ruleLabel, res.Findings) +} + +func expectClean(res packagepolicy.Result) error { + if !res.Clean() { + return fmt.Errorf("expected a clean, fully-established scan; findings %v errors %v", res.Findings, res.Errors) + } + if res.Scanned == 0 { + return fmt.Errorf("clean scan covered zero files: vacuous") + } + return nil +} + +func expectOnlyErrors(res packagepolicy.Result, kind packagepolicy.ErrorKind) error { + if res.HasImportFindings() { + return fmt.Errorf("%s must never produce import findings: %v", kind, res.Findings) + } + for _, e := range res.Errors { + if e.Kind == kind { + return nil + } + } + return fmt.Errorf("expected exactly a %s verification failure; got errors %v", kind, res.Errors) +} + +func controls() []control { + const ( + compilerImport = "github.com/calypr/loom/internal/backends/postgres/internal/store" + pgxImport = "github.com/jackc/pgx/v5/pgxpool" + runtimeImport = "github.com/calypr/loom/internal/dataframe/execution" + factoryImport = "github.com/calypr/loom/internal/backends/postgres" + physicalRows = "github.com/calypr/loom/internal/backends/postgres/internal/rows" + oldStorePG = "github.com/calypr/loom/internal/store/postgres" + compilerRoot = "github.com/calypr/loom/internal/dataframe/compiler" + ) + goFile := func(pkg, imp string) string { + if imp == "" { + return "package " + pkg + "\n" + } + return "package " + pkg + "\n\nimport \"" + imp + "\"\n" + } + + return []control{ + {"compiler-imports-backend-implementation", func(f *fixture) { + f.write("compiler/victim_backends.go", goFile("compiler", compilerImport)) + }, func(f *fixture) error { + return expectFinding(scanPath(forbiddenImportRule(), filepath.Join(f.dir, "compiler")), + forbiddenImportRule().Label, "victim_backends.go", "internal/backends/postgres") + }}, + {"driver-leak-hidden-in-postgres-renderer", func(f *fixture) { + f.write("render/aql/clean.go", goFile("aql", "")) + f.write("render/postgres/victim_pgx.go", goFile("postgres", pgxImport)) + }, func(f *fixture) error { + aql, pg := rendererRules() + if err := expectOnlyCleanOrSkip(scanPath(aql, filepath.Join(f.dir, "render/aql"))); err != nil { + return err + } + return expectFinding(scanPath(pg, filepath.Join(f.dir, "render/postgres")), + pg.Label, "victim_pgx.go", "github.com/jackc/pgx") + }}, + {"renderer-imports-runtime", func(f *fixture) { + f.write("render/aql/victim_runtime.go", goFile("aql", runtimeImport)) + f.write("render/postgres/victim_runtime.go", goFile("postgres", runtimeImport)) + f.write("render/postgres/victim_store.go", goFile("postgres", "github.com/calypr/loom/internal/store/clickhouse")) + f.write("render/postgres/victim_root.go", goFile("postgres", compilerRoot)) + }, func(f *fixture) error { + aql, pg := rendererRules() + if err := expectFinding(scanPath(aql, filepath.Join(f.dir, "render/aql")), + aql.Label, "victim_runtime.go", "internal/dataframe/execution"); err != nil { + return err + } + for _, victim := range []struct{ file, imp string }{ + {"victim_runtime.go", "internal/dataframe/execution"}, + {"victim_store.go", "internal/store/"}, + {"victim_root.go", "internal/dataframe/compiler"}, + } { + if err := expectFinding(scanPath(pg, filepath.Join(f.dir, "render/postgres")), + pg.Label, victim.file, victim.imp); err != nil { + return err + } + } + return nil + }}, + {"domain-imports-backend-factory", func(f *fixture) { + f.write("catalog/victim_factory.go", goFile("catalog", factoryImport)) + f.write("catalog/victim_physical_rows.go", goFile("catalog", physicalRows)) + }, func(f *fixture) error { + rule := neutralDomainRule() + res := scanPath(rule, filepath.Join(f.dir, "catalog")) + if err := expectFinding(res, rule.Label, "victim_factory.go", "internal/backends/postgres"); err != nil { + return err + } + return expectFinding(res, rule.Label, "victim_physical_rows.go", "internal/backends/postgres/internal/rows") + }}, + {"old-concrete-import-path-revived", func(f *fixture) { + f.write("tool/victim_oldpath.go", goFile("tool", oldStorePG)) + }, func(f *fixture) error { + rule := oldPathsRule() + return expectFinding(scanPath(rule, filepath.Join(f.dir, "tool")), + rule.Label, "victim_oldpath.go", "internal/store/postgres") + }}, + {"forbidden-import-removal-converges", func(f *fixture) { + f.write("compiler/leak.go", goFile("compiler", compilerImport)) + f.phase = "remove" + f.write("compiler/leak.go", goFile("compiler", "")) + }, func(f *fixture) error { + return expectClean(scanPath(forbiddenImportRule(), filepath.Join(f.dir, "compiler"))) + }}, + {"vacuous-nonexistent-path-fails", func(f *fixture) {}, func(f *fixture) error { + return expectOnlyErrors(scanPath(forbiddenImportRule(), filepath.Join(f.dir, "no-such-directory")), + packagepolicy.ErrMissing) + }}, + {"vacuous-empty-scan-fails", func(f *fixture) { + if err := os.MkdirAll(filepath.Join(f.dir, "empty"), 0o755); err != nil { + panic(err) + } + }, func(f *fixture) error { + return expectOnlyErrors(scanPath(forbiddenImportRule(), filepath.Join(f.dir, "empty")), + packagepolicy.ErrVacuous) + }}, + {"malformed-input-is-verification-failure", func(f *fixture) { + f.write("compiler/broken.go", "!!! not go source >>> {{{") + }, func(f *fixture) error { + return expectOnlyErrors(scanPath(forbiddenImportRule(), filepath.Join(f.dir, "compiler")), + packagepolicy.ErrParse) + }}, + {"legitimate-adapter-and-factory-domain-imports-accepted", func(f *fixture) { + f.write("adapter/rows_adapter.go", goFile("adapter", "github.com/calypr/loom/internal/catalog")) + f.write("factory/factory.go", goFile("factory", "github.com/calypr/loom/internal/dataframe/recipe")) + f.write("adapter/reads_execution_ports.go", goFile("adapter", runtimeImport)) + }, func(f *fixture) error { + return expectClean(scanPath(neutralDomainRule(), f.dir)) + }}, + {"removed-bridge-identifier-reappearance-caught", func(f *fixture) { + f.write("internal/bridge.go", "package internalx\n\n// QuerierAsAny in a comment must never trip the AST scan.\nfunc QuerierAsAny() any { return nil }\n") + f.write("cmd/tool/tool.go", "package tool\n") + }, func(f *fixture) error { + return expectAbsenceError(packagepolicy.CheckAbsences(f.dir, []packagepolicy.AbsenceRule{querierBridgeAbsenceRule()}), + packagepolicy.ErrVacuous, "QuerierAsAny") + }}, + {"removed-bridge-identifier-absent-passes", func(f *fixture) { + f.write("internal/bridge.go", "package internalx\n\n// mentions QuerierAsAny only in a comment and a string\nvar note = \"QuerierAsAny\"\n") + f.write("cmd/tool/tool.go", "package tool\n") + }, func(f *fixture) error { + return expectAbsencesClean(packagepolicy.CheckAbsences(f.dir, []packagepolicy.AbsenceRule{querierBridgeAbsenceRule()})) + }}, + {"removed-bridge-scan-malformed-is-verification-failure", func(f *fixture) { + f.write("internal/broken.go", "!!! not go source >>> {{{") + f.write("cmd/tool/tool.go", "package tool\n") + }, func(f *fixture) error { + return expectAbsenceError(packagepolicy.CheckAbsences(f.dir, []packagepolicy.AbsenceRule{querierBridgeAbsenceRule()}), + packagepolicy.ErrParse, "broken.go") + }}, + {"removed-bridge-scan-vacuous-roots-fail", func(f *fixture) { + if err := os.MkdirAll(filepath.Join(f.dir, "internal"), 0o755); err != nil { + panic(err) + } + if err := os.MkdirAll(filepath.Join(f.dir, "cmd"), 0o755); err != nil { + panic(err) + } + }, func(f *fixture) error { + return expectAbsenceError(packagepolicy.CheckAbsences(f.dir, []packagepolicy.AbsenceRule{querierBridgeAbsenceRule()}), + packagepolicy.ErrVacuous, "zero Go files") + }}, + {"facade-reexport-caught-and-removal-converges", func(f *fixture) { + f.write("control/defs.go", "package control\n\nimport \"errors\"\n\n"+ + "type Config struct{ N int }\n\n"+ + "var ErrInvalid = errors.New(\"x\")\n\n"+ + "func NewStore() *Store { return nil }\n\n"+ + "type Store struct{}\n") + f.write("postgres/aliases.go", "package postgres\n\nimport \"github.com/calypr/loom/internal/backends/postgres/control\"\n\n"+ + "type Config = control.Config\n\n"+ + "var ErrInvalid = control.ErrInvalid\n") + }, func(f *fixture) error { + rule := packagepolicy.NoReExportRule{Label: "facade", Owner: "import control directly", + Target: "postgres", OwnerPkg: "control", OwnerImport: "github.com/calypr/loom/internal/backends/postgres/control"} + if err := expectAbsenceError(packagepolicy.CheckNoReExport(f.dir, rule), packagepolicy.ErrVacuous, "Config"); err != nil { + return err + } + f.write("postgres/aliases.go", "package postgres\n\nimport \"github.com/calypr/loom/internal/backends/postgres/control\"\n\n"+ + "func Open(c control.Config) error { return nil }\n") + if errs := packagepolicy.CheckNoReExport(f.dir, rule); len(errs) != 0 { + return fmt.Errorf("composition must stay legal: %v", errs) + } + f.write("postgres/aliases.go", "package postgres\n\nimport \"github.com/calypr/loom/internal/backends/postgres/control\"\n\n"+ + "func NewStore() *control.Store { return control.NewStore() }\n") + if err := expectAbsenceError(packagepolicy.CheckNoReExport(f.dir, rule), packagepolicy.ErrVacuous, "NewStore"); err != nil { + return err + } + f.write("postgres/aliases.go", "package postgres\n\nimport \"github.com/calypr/loom/internal/backends/postgres/control\"\n\n"+ + "type Store struct{ *control.Store }\n") + return expectAbsenceError(packagepolicy.CheckNoReExport(f.dir, rule), packagepolicy.ErrVacuous, "Store") + }}, + {"control-imports-root-facade-caught", func(f *fixture) { + }, func(f *fixture) error { + rule, ok := packagepolicy.ByLabel("postgres-control: cycle-free control plane") + if !ok { + panic("authoritative control-cycle-free rule missing from the table") + } + f.write("internal/backends/postgres/control/leak.go", goFile("control", "github.com/calypr/loom/internal/backends/postgres")) + if err := expectFinding(scanPath(rule, filepath.Join(f.dir, "internal/backends/postgres/control")), + rule.Label, "leak.go", "internal/backends/postgres"); err != nil { + return err + } + f.write("internal/backends/postgres/control/leak.go", goFile("control", "github.com/calypr/loom/internal/backends/postgres/control")) + return expectClean(scanPath(rule, filepath.Join(f.dir, "internal/backends/postgres/control"))) + }}, + {"control-reaches-rows-or-execution-caught", func(f *fixture) { + }, func(f *fixture) error { + rule, ok := packagepolicy.ByLabel("postgres-control: cycle-free control plane") + if !ok { + panic("authoritative control-cycle-free rule missing from the table") + } + f.write("internal/backends/postgres/control/rows_leak.go", goFile("control", "github.com/calypr/loom/internal/backends/postgres/internal/rows")) + f.write("internal/backends/postgres/control/exec_leak.go", goFile("control", "github.com/calypr/loom/internal/dataframe/execution")) + if err := expectFinding(scanPath(rule, filepath.Join(f.dir, "internal/backends/postgres/control")), + rule.Label, "rows_leak.go", "internal/rows"); err != nil { + return err + } + return expectFinding(scanPath(rule, filepath.Join(f.dir, "internal/backends/postgres/control")), + rule.Label, "exec_leak.go", "dataframe/execution") + }}, + } +} + +func expectOnlyCleanOrSkip(res packagepolicy.Result) error { + return expectClean(res) +} diff --git a/scripts/check_dataframe_package_boundaries.sh b/scripts/check_dataframe_package_boundaries.sh index ab59f400a..c6e0bc31e 100755 --- a/scripts/check_dataframe_package_boundaries.sh +++ b/scripts/check_dataframe_package_boundaries.sh @@ -1,50 +1,64 @@ #!/usr/bin/env bash set -euo pipefail -# Keep the dataframe pipeline directional. This is intentionally a small -# source check rather than a Go test so it can run before compilation in CI. root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if ! command -v go >/dev/null 2>&1; then + echo "dataframe package boundary check requires go; refusing to pass vacuously" >&2 + exit 1 +fi + +if ! command -v rg >/dev/null 2>&1; then + echo "dataframe package boundary check requires rg; refusing to pass vacuously (install ripgrep)" >&2 + exit 1 +fi + +(cd "$root" && go run ./scripts/check-boundaries --self-test) + +(cd "$root" && go run ./scripts/check-boundaries --root "$root") + +require_dir() { + local label="$1" dir="$2" + if [[ ! -d "$dir" ]]; then + echo "dataframe package boundary check target missing: $label ($dir)" >&2 + exit 1 + fi +} + fail_if_matches() { local label="$1" local pattern="$2" shift 2 - if rg -n --glob '*.go' --glob '!**/*_test.go' "$pattern" "$@"; then + require_dir "$label" "$1" + local matches="" rc=0 entry + matches="$(rg -n --glob '*.go' --glob '!**/*_test.go' "$pattern" "$@")" || rc=$? + if (( rc > 1 )); then + echo "dataframe package boundary check failed to run: $label (rg exit $rc)" >&2 + exit 1 + fi + if [[ -z "$matches" ]]; then + return 0 + fi + local filtered="" grc=0 + filtered="$(printf '%s\n' "$matches" | grep -v -E '^[^:]+_test\.go:')" || grc=$? + if (( grc > 1 )); then + echo "dataframe package boundary check failed to run: $label (grep exit $grc)" >&2 + exit 1 + fi + matches="$filtered" + if [[ -n "$matches" ]]; then + printf '%s\n' "$matches" >&2 echo "dataframe package boundary violation: $label" >&2 exit 1 fi } -fail_if_matches "spec imports runtime/compiler implementation" \ - 'internal/dataframe/(execution|compiler|semantic|materialization|template|errors)' \ - "$root/internal/dataframe/spec" - -fail_if_matches "semantic imports physical compiler implementation" \ - 'internal/dataframe/compiler/(ir|lower|optimize|render)' \ - "$root/internal/dataframe/semantic" - -fail_if_matches "ir imports lowering, optimization, rendering, or runtime" \ - 'internal/dataframe/compiler/(lower|optimize|render)|internal/dataframe/execution|internal/store/arango' \ - "$root/internal/dataframe/compiler/ir" - -fail_if_matches "optimizer imports renderer, lowerer, runtime, or schema routes" \ - 'internal/dataframe/compiler/(lower|render)|internal/dataframe/execution|ResolveStorageRoute|fhirschema' \ +fail_if_matches "optimizer references storage-route resolution or the schema package" \ + 'ResolveStorageRoute|fhirschema' \ "$root/internal/dataframe/compiler/optimize" -fail_if_matches "renderer imports lowerer, optimizer, runtime, or Arango" \ - 'internal/dataframe/compiler/(lower|optimize)|internal/dataframe/execution|internal/store/arango|ResolveStorageRoute' \ +fail_if_matches "AQL renderer references storage-route resolution" \ + 'ResolveStorageRoute' \ "$root/internal/dataframe/compiler/render/aql" -fail_if_matches "stored recipe domain imports execution or storage layers" \ - 'internal/dataframe/(compiler|execution|materialization|publication|store)' \ - "$root"/internal/dataframe/recipe/*.go - -fail_if_matches "publication targets import dataframe execution orchestration" \ - 'internal/dataframe/execution|internal/dataframe/published|internal/dataframe/materialization' \ - "$root"/internal/dataframe/publication - -fail_if_matches "authoring intent imports orchestration or storage" \ - 'internal/(server|store|explorer/(arango|lifecycle|compilation))|internal/dataframe/(execution|materialization|publication|published)' \ - "$root/internal/explorer/authoringv2" - echo "dataframe package boundaries: ok" diff --git a/scripts/check_no_code_comments.py b/scripts/check_no_code_comments.py new file mode 100644 index 000000000..328da61f9 --- /dev/null +++ b/scripts/check_no_code_comments.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python3 +"""Repository-wide handwritten-comment gate. + +Drives scripts/lib/no_code_comments_backends.py over every tracked path, +re-enumerated from git (git ls-tree -r HEAD; git ls-files is rejected because +an empty index silently reports zero files). Fails on handwritten comments; +permits only enumerated language directives and provenance-ledger generator +output. Markdown and data assets are inventoried, never rewritten. Unsupported +formats are reported, never silently excluded. + +Usage: + check_no_code_comments.py --all-tracked [--json-out PATH] + check_no_code_comments.py PATH [PATH ...] [--json-out PATH] +""" + +import argparse +import json +import os +import re +import subprocess +import sys + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(SCRIPT_DIR, "lib")) + +import no_code_comments_backends as b + +REPO = os.path.abspath(os.path.join(SCRIPT_DIR, "..")) + +"""Provenance-backed generator output. Each entry records the generator and +its Makefile regen command; the checker additionally requires the expected +banner in the file, so a listing alone never exempts handwritten comments.""" +PROVENANCE_LEDGER = { + "generated/fhir/extract.go": {"generator": "cmd/generate", "regen": "make generate-fhir", + "banner": r"Code generated by cmd/generate/main\.go\. DO NOT EDIT\."}, + "generated/fhir/graphql.go": {"generator": "cmd/generate", "regen": "make generate-fhir", + "banner": r"Code generated by cmd/generate/main\.go\. DO NOT EDIT\."}, + "generated/fhir/helpers.go": {"generator": "cmd/generate", "regen": "make generate-fhir", + "banner": r"Code generated by cmd/generate/main\.go\. DO NOT EDIT\."}, + "generated/fhir/model.go": {"generator": "cmd/generate", "regen": "make generate-fhir", + "banner": r"Code generated by cmd/generate/main\.go\. DO NOT EDIT\."}, + "generated/fhir/resources.go": {"generator": "cmd/generate", "regen": "make generate-fhir", + "banner": r"Code generated by cmd/generate/main\.go\. DO NOT EDIT\."}, + "generated/fhir/validate.go": {"generator": "cmd/generate", "regen": "make generate-fhir", + "banner": r"Code generated by cmd/generate/main\.go\. DO NOT EDIT\."}, + "generated/fhirschema/generated.go": {"generator": "cmd/generate", "regen": "make generate-fhir", + "banner": r"Code generated by cmd/generate/main\.go\. DO NOT EDIT\."}, + "generated/graphql/graph/schema/fhir_schema.graphqls": { + "generator": "cmd/generate", "regen": "make generate-fhir", + "banner": r"Generated from schemas/graph-fhir\.json"}, + "generated/graphql/graph/executor/fhir_schema.generated.go": { + "generator": "gqlgen", "regen": "make generate-graphql", + "banner": r"Code generated by github\.com/99designs/gqlgen, DO NOT EDIT\."}, + "generated/graphql/graph/executor/prelude.generated.go": { + "generator": "gqlgen", "regen": "make generate-graphql", + "banner": r"Code generated by github\.com/99designs/gqlgen, DO NOT EDIT\."}, + "generated/graphql/graph/executor/root_.generated.go": { + "generator": "gqlgen", "regen": "make generate-graphql", + "banner": r"Code generated by github\.com/99designs/gqlgen, DO NOT EDIT\."}, + "generated/graphql/graph/executor/schema.generated.go": { + "generator": "gqlgen", "regen": "make generate-graphql", + "banner": r"Code generated by github\.com/99designs/gqlgen, DO NOT EDIT\."}, + "generated/graphql/graph/model/models.go": { + "generator": "gqlgen", "regen": "make generate-graphql", + "banner": r"Code generated by github\.com/99designs/gqlgen, DO NOT EDIT\."}, + "generated/loomapi/api.gen.go": { + "generator": "oapi-codegen", "regen": "make generate-openapi", + "banner": r"Code generated by github\.com/oapi-codegen/oapi-codegen/v2"}, +} + +"""gqlgen writes resolver stubs under internal/; implementations are copied +through on regen. Banner check gates the exemption like generated/.""" +GQLGEN_RESOLVERS = { + "internal/api/graphql/graph/resolver/fhir_schema.resolvers.go", + "internal/api/graphql/graph/resolver/schema.resolvers.go", +} +GQLGEN_RESOLVERS = {os.path.join(REPO, p) for p in GQLGEN_RESOLVERS} +PROVENANCE_LEDGER = {os.path.join(REPO, k): v for k, v in PROVENANCE_LEDGER.items()} +GQLGEN_RESOLVER_BANNER = r"Code generated by github\.com/99designs/gqlgen version" + +GOVERNANCE_DIRECTIVES = { + "internal/dataframe/compiler/render/postgres/stage_sequence.go": [ + {"pattern": re.compile(r"CTE terminal re-sort over GROUP-last per coordinator ruling"), + "tool": "TestSQL16LawCells (stage_sequence_test.go)", + "purpose": "D-16a coordinator-ruling attribution string, asserted verbatim by the law-cell test"}, + ], + "internal/dataframe/compiler/render/postgres/stage_sequence_test.go": [ + {"pattern": re.compile(r"D-16a attribution \(mandatory, coordinator ruling 2026-11-11\)"), + "tool": "TestSQL16LawCells own-file check", + "purpose": "file-level D-16a attribution copy, asserted by the law-cell test as a // D-16a attribution comment line"}, + ], +} +GOVERNANCE_DIRECTIVE_INDEX = { + os.path.join(REPO, k): v for k, v in GOVERNANCE_DIRECTIVES.items() +} + +REVIEWED_SOURCE_LEDGERS = ( + "verification/builder-interactions/reviewed-sources.json", + "verification/publication-bundle/reviewed-sources.json", +) + + +def ledger_bound_paths(): + """Paths byte-pinned by verification reviewed-source ledgers (md5/sha + verified). Their comments are evidence payloads; never rewritten.""" + bound = {} + for rel in REVIEWED_SOURCE_LEDGERS: + full = os.path.join(REPO, rel) + if not os.path.isfile(full): + continue + with open(full) as f: + data = json.load(f) + files = data.get("files") + if isinstance(files, dict): + for p in files: + bound[os.path.normpath(os.path.join(REPO, p))] = rel + return bound + + +"""Bytes here are runtime payload, not source: checksummed into the postgres +migration ledger (ErrMigrationChecksumMismatch) or consumed as testdata +fixtures. Stripping comments changes hashes = behavior change.""" +MIGRATION_TREE = "internal/backends/postgres/internal/" +AB05_FIXTURE_TREE = ("internal/backends/postgres/internal/store/testdata/" + "ab05_pre_refactor/") + + +def is_checksum_payload(rel): + return (rel.startswith(MIGRATION_TREE) and not rel.startswith(AB05_FIXTURE_TREE) + and "/migrations/" in rel and rel.endswith(".sql")) + + +def is_fixture_payload(rel): + return rel.startswith(AB05_FIXTURE_TREE) and rel.endswith(".sql") + + + +def tracked_paths(): + p = subprocess.run(["git", "-C", REPO, "ls-tree", "-r", "--name-only", "HEAD"], + capture_output=True, text=True, check=True) + return [l for l in p.stdout.splitlines() if l] + + +def read(rel): + full = os.path.join(REPO, rel) + if not os.path.isfile(full): + return None + with open(full, "rb") as f: + raw = f.read() + if b"\x00" in raw[:8192]: + return None + return raw.decode("utf-8", errors="replace") + + +GUARD_QUOTE_MIN = 25 +GUARD_PREFIXES = ("", "*/", "*)", "]]") + + +def guard_strip(text): + t = text.strip() + for p in GUARD_PREFIXES: + if t.startswith(p): + t = t[len(p):] + break + for s in GUARD_SUFFIXES: + if t.endswith(s): + t = t[:-len(s)] + return t.strip() + + +def is_test_like(rel): + base = os.path.basename(rel) + return (base.endswith("_test.go") or base.endswith(".test.mjs") + or base.endswith(".test.ts") or base.endswith(".test.tsx") + or base.endswith(".test.js") or base.endswith(".test.jsx") + or (base.startswith("test_") and base.endswith(".py"))) + + +def guard_quote_corpus(paths): + corpus = {} + for rel in paths: + if not is_test_like(rel): + continue + relkey = os.path.relpath(os.path.abspath(os.path.join(REPO, rel)), REPO) + src = read(relkey) + if src is not None: + corpus[rel] = src + return corpus + + +def guard_quoted(comment, comments, rel, corpus): + line_at = {} + for cc in comments: + if cc.get("kind") != "block": + line_at.setdefault(cc["line"], cc) + lo = hi = comment["line"] + while lo - 1 in line_at: + lo -= 1 + while hi + 1 in line_at: + hi += 1 + members = [comment] + [line_at[ln] for ln in range(lo, hi + 1)] + for m in members: + for ln in (m["text"].splitlines() or [m["text"]]): + stripped = guard_strip(ln) + if len(stripped) < GUARD_QUOTE_MIN: + continue + for trel, tsrc in corpus.items(): + if trel != rel and stripped in tsrc: + return True + return False + + +def scan(paths, json_out=None): + inventory = {"asset": [], "unsupported": [], "binary": [], + "checksum-payload": [], "fixture-payload": [], + "provenance-generated": [], "ledger-bound": [], + "unresolvable-ledger-key": []} + ledger_bound = ledger_bound_paths() + tracked = set(tracked_paths()) + for key in sorted(ledger_bound): + rel = os.path.relpath(key, REPO) + if rel not in tracked: + inventory["unresolvable-ledger-key"].append( + {"path": rel, "ledger": ledger_bound[key]}) + violations = [] + allowed = [] + corpus = None + ts_paths = [p for p in paths if b.classify(p) == "ts"] + ts_results = {} + ts_scanner_ok = True + if ts_paths: + try: + ts_results = b.detect_ts_family([os.path.join(REPO, p) for p in ts_paths], REPO) + except RuntimeError: + """Fail closed: without the TypeScript scanner, ts-family files are + inventoried unsupported (gate-red), never silently passed.""" + ts_scanner_ok = False + ts_batch = {} + for p in ts_paths: + ts_batch[p] = ts_results.get(os.path.join(REPO, p), []) + for rel in paths: + relkey = os.path.relpath(os.path.abspath(os.path.join(REPO, rel)), REPO) + backend = b.classify(rel) + if backend == "asset": + inventory["asset"].append(rel) + continue + if backend == "UNSUPPORTED": + inventory["unsupported"].append(rel) + continue + if backend == "ts" and not ts_scanner_ok: + inventory["unsupported"].append(rel) + continue + if is_checksum_payload(relkey): + inventory["checksum-payload"].append(rel) + continue + if is_fixture_payload(relkey): + inventory["fixture-payload"].append(rel) + continue + src = read(relkey) + if src is None: + inventory["binary"].append(rel) + continue + relabs = os.path.abspath(os.path.join(REPO, rel)) + provenance = {} + if relabs in PROVENANCE_LEDGER: + entry = PROVENANCE_LEDGER[relabs] + if re.search(entry["banner"], src): + provenance[relabs] = entry + inventory["provenance-generated"].append(rel) + else: + violations.append({"path": rel, "backend": backend, "line": 0, + "kind": "provenance", "text": + "listed generator file missing expected banner: " + + entry["banner"]}) + elif relabs in GQLGEN_RESOLVERS: + if re.search(GQLGEN_RESOLVER_BANNER, src): + provenance[relabs] = {"generator": "gqlgen", "regen": "make generate-graphql"} + inventory["provenance-generated"].append(rel) + else: + violations.append({"path": rel, "backend": backend, "line": 0, + "kind": "provenance", "text": + "gqlgen resolver missing generated banner"}) + governance = {} + if relabs in GOVERNANCE_DIRECTIVE_INDEX: + governance[relabs] = GOVERNANCE_DIRECTIVE_INDEX[relabs] + if relabs in ledger_bound: + inventory["ledger-bound"].append(rel) + if backend == "ts": + comments = ts_batch[rel] + else: + backend_name, comments, error = b.detect_file(rel, src) + if error: + violations.append({"path": rel, "backend": backend, "line": 0, + "kind": "parse", "text": error}) + continue + lines = src.splitlines() + for c in comments: + line_text = lines[c["line"] - 1] if 0 < c["line"] <= len(lines) else "" + category = b.allowlist_category(backend, c, line_text, relabs, + provenance, governance) + if category is None: + if corpus is None: + try: + corpus = guard_quote_corpus(paths) + except OSError: + corpus = {} + if corpus and guard_quoted(c, comments, rel, corpus): + category = "guard-quoted" + record = {"path": rel, "backend": backend, "line": c["line"], + "kind": c["kind"], "text": c["text"][:120]} + if category: + record["category"] = category + allowed.append(record) + else: + violations.append(record) + summary = { + "filesScanned": len(paths), + "violations": violations, + "allowed": allowed, + "inventory": inventory, + } + if json_out: + with open(json_out, "w") as f: + json.dump(summary, f, indent=1, sort_keys=True) + return summary + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--all-tracked", action="store_true") + ap.add_argument("--json-out") + ap.add_argument("paths", nargs="*") + args = ap.parse_args() + paths = tracked_paths() if args.all_tracked else args.paths + if not paths: + print("no paths selected", file=sys.stderr) + return 2 + summary = scan(paths, args.json_out) + for v in summary["violations"]: + print(f"VIOLATION {v['path']}:{v['line']} [{v['backend']}/{v['kind']}] {v['text']!r}") + for rel in summary["inventory"]["unsupported"]: + print(f"UNSUPPORTED {rel}") + for entry in summary["inventory"]["unresolvable-ledger-key"]: + print(f"UNRESOLVABLE-LEDGER-KEY {entry['path']} ({entry['ledger']})") + inv = summary["inventory"] + print(f"scanned={summary['filesScanned']} violations={len(summary['violations'])} " + f"allowed={len(summary['allowed'])} assets={len(inv['asset'])} " + f"checksum-payload={len(inv['checksum-payload'])} " + f"fixture-payload={len(inv['fixture-payload'])} " + f"provenance-generated={len(inv['provenance-generated'])} " + f"ledger-bound={len(inv['ledger-bound'])} " + f"unresolvable-ledger-key={len(inv['unresolvable-ledger-key'])} " + f"unsupported={len(inv['unsupported'])} binary={len(inv['binary'])}") + return 1 if summary["violations"] or summary["inventory"]["unsupported"] else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/list-node-tests.failure.test.mjs b/scripts/ci/list-node-tests.failure.test.mjs index 87871104c..854d7a56a 100644 --- a/scripts/ci/list-node-tests.failure.test.mjs +++ b/scripts/ci/list-node-tests.failure.test.mjs @@ -74,7 +74,6 @@ test('CLI fails on a missing test-owned URL fixture with its filesystem code', ( }); test('an unreadable test-owned URL fixture fails the inventory with its filesystem code', async () => { - // A symlink loop is unopenable for every user, unlike a mode only root can read. const root = project({ 'scripts/root.test.mjs': "readFile(new URL('./blocked.json', import.meta.url));", ...HEALTHY }); try { symlinkSync(path.join(root, 'scripts/blocked.json'), path.join(root, 'scripts/blocked.json')); @@ -93,8 +92,6 @@ test('an unreadable test-owned URL fixture fails the inventory with its filesyst }); test('an unreadable retained dependency fails the inventory with its filesystem code', async () => { - // A directory where a module is expected yields EISDIR for any user, unlike - // a mode that only the current user cannot read. const root = project({ 'scripts/root.test.mjs': "import './blocked.mjs';", ...HEALTHY }); try { mkdirSync(path.join(root, 'scripts/blocked.mjs'), { recursive: true }); diff --git a/scripts/ci/list-node-tests.mjs b/scripts/ci/list-node-tests.mjs index cc4e81be0..6f972f6f2 100644 --- a/scripts/ci/list-node-tests.mjs +++ b/scripts/ci/list-node-tests.mjs @@ -2,8 +2,6 @@ import { access, constants, readdir, readFile } from 'node:fs/promises'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; -// Tokenize enough to ignore quoted test fixtures/comments; this is a dependency -// inventory, not a JavaScript parser or proof about computed imports. function tokens(source) { const result = [], contexts = []; let index = 0, expressionStart = true; @@ -69,8 +67,6 @@ function tokens(source) { if (operator === '(') contexts.push(controls.has(previous)); expressionStart = operator === ')' ? Boolean(contexts.pop()) : operator === ']' || operator === '++' || operator === '--' || operator === '?.' || operator === '.' ? false - // A closing block can begin a new regex statement. Ambiguous brace - // contexts prefer retaining a test over treating regex text as imports. : operator === '}' ? true : !/^[0-9]$/.test(operator); result.push({ value: operator, string: false }); } @@ -112,7 +108,6 @@ function formatNodeTestProblems(problems) { const POLICY = 'Relative static and literal dynamic import/require closure plus literal test-owned URL fixture existence; executable Playwright package imports and the retired browser directory are explicit nonfatal exclusions, incidental strings retained. A missing or unreadable retained dependency or test-owned fixture fails the inventory. Computed imports and external package availability are not proven.'; -/** A retained Node test can no longer execute because its source closure is broken. */ export class NodeTestInventoryError extends Error { constructor(problems) { super(formatNodeTestProblems(problems)); @@ -136,14 +131,13 @@ async function scanRoots(projectRoot) { return candidates.sort(); } -/** Classify every candidate root against the shared dependency-closure policy. */ async function nodeTestClassification(projectRoot) { const graph = new Map(); const relative = file => path.relative(projectRoot, file).split(path.sep).join('/'); async function load(file) { if (graph.has(file)) return; const node = { dependencies: [], problems: [] }; - graph.set(file, node); // Register before recursion; cycles are checked per root below. + graph.set(file, node); let source; try { source = await readFile(file, 'utf8'); } catch (error) { @@ -154,7 +148,6 @@ async function nodeTestClassification(projectRoot) { if (asset) { if (!file.endsWith('.test.mjs') || !specifier.startsWith('.')) continue; const dependency = path.resolve(path.dirname(file), specifier); - // Read access, not mere existence: the retained test reads this fixture. try { await access(dependency, constants.R_OK); } catch (error) { node.problems.push({ reason: 'missing-source-fixture', file: relative(dependency), code: error.code, target: relative(dependency) }); } continue; @@ -177,9 +170,6 @@ async function nodeTestClassification(projectRoot) { continue; } await load(candidate); - // Inspect the complete retained closure: an intentional browser exclusion - // in one branch must not hide a missing dependency in another branch, - // including one reachable only through an import cycle. const queue = [[candidate]], seen = new Set(), found = []; for (let index = 0; index < queue.length; index++) { const chain = queue[index], file = chain.at(-1); @@ -202,12 +192,6 @@ async function nodeTestClassification(projectRoot) { return { included, excluded, problems }; } -/** - * List the retained Node tests for the CI shards. A missing or unreadable - * retained dependency or test-owned URL fixture rejects instead of returning a - * silently shrunken selection; explicit Playwright and retired-browser - * exclusions stay ordinary results. - */ export async function nodeTestInventory(projectRoot = process.cwd()) { const { included, excluded, problems } = await nodeTestClassification(projectRoot); if (problems.length) throw new NodeTestInventoryError(problems); @@ -224,7 +208,6 @@ if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.ar } catch (error) { if (!(error instanceof NodeTestInventoryError)) throw error; - // Never emit a shrunken selection that CI could run as if it were complete. await new Promise((resolve, reject) => process.stderr.write(`${error.message}\n`, error2 => error2 ? reject(error2) : resolve())); process.exitCode = 1; } diff --git a/scripts/ci/vet.mjs b/scripts/ci/vet.mjs index 51a425e07..b79867bba 100644 --- a/scripts/ci/vet.mjs +++ b/scripts/ci/vet.mjs @@ -4,9 +4,6 @@ import { pathToFileURL } from 'node:url'; const root = path.resolve(import.meta.dirname, '../..'); -/** Run one `go vet` invocation. Passing any analyzer-enable flag substitutes - * the default analyzer set (the disable form `-NAME=false` instead runs all - * analyzers not disabled; the gate only uses the enable form). */ export function goVet({ packagePatterns = ['./...'], analyzerArgs = [], overlayFile } = {}) { if (!Array.isArray(packagePatterns) || packagePatterns.length === 0) throw new Error('packagePatterns required'); if (packagePatterns.some(pattern => pattern.startsWith('-'))) throw new Error('packagePatterns must not start with - (vet flag escape)'); @@ -21,10 +18,6 @@ export function goVet({ packagePatterns = ['./...'], analyzerArgs = [], overlayF return { ok: result.status === 0, status: result.status, output: `${result.stdout ?? ''}${result.stderr ?? ''}` }; } -/** The CI boundary: `go test` vets only its fixed analyzer subset, so copylocks - * (and anything else outside that subset) ships green unless a full `go vet` - * gates it. Both passes must stay green; the explicit copylocks pass pins the - * analyzer even if the default set drifts. */ export function vetGate({ packagePatterns = ['./...'], overlayFile } = {}) { const passes = [ { name: 'default', ...goVet({ packagePatterns, overlayFile }) }, diff --git a/scripts/demo-up.sh b/scripts/demo-up.sh index 06fe3483d..f6474c6cd 100755 --- a/scripts/demo-up.sh +++ b/scripts/demo-up.sh @@ -35,10 +35,6 @@ fi [[ -f "$ui_build_context/apps/demo/Dockerfile" ]] || { echo "UI build context has no apps/demo/Dockerfile: $ui_build_context" >&2; exit 2; } case "$seed" in true|false) ;; *) echo "invalid LOOM_DEMO_SEED: $seed" >&2; exit 2 ;; esac -# Acceptance scripts prewarm an explicit host cache. Go's MkdirTemp creates -# content-addressed cache roots as 0700, and the bind mount masks the image's -# arango-fhir ownership. Grant the container's matching supplementary group -# read/traverse access only; keep the application process non-root. fixture_cache_dir=${LOOM_DEMO_FIXTURE_CACHE_DIR:-} if [[ -n "$fixture_cache_dir" ]]; then [[ -d "$fixture_cache_dir" ]] || { echo "fixture cache directory does not exist: $fixture_cache_dir" >&2; exit 2; } diff --git a/scripts/formal-core-protocol-manifest.mjs b/scripts/formal-core-protocol-manifest.mjs index d68bd0ba1..1f05753fc 100644 --- a/scripts/formal-core-protocol-manifest.mjs +++ b/scripts/formal-core-protocol-manifest.mjs @@ -1,8 +1,4 @@ #!/usr/bin/env node -// M6 final frozen combined audit manifest (plan F: core-protocol/conformance.json). -// Single result source of truth; every field derived from on-disk evidence. Missing -// selected artifacts are recorded as failed/not-run statuses, never silently passed. -// Usage: node scripts/formal-core-protocol-manifest.mjs [--evidence DIR] import { readFileSync, writeFileSync, existsSync, readdirSync, statSync } from 'node:fs'; import { createHash } from 'node:crypto'; import { execFileSync } from 'node:child_process'; @@ -130,7 +126,7 @@ sel('full-self-test-final-snapshot', 'node scripts/verify-builder-formal-core.mj if (v.execution?.subprocessTimeoutMs !== 600_000) return `BLOCKED: CI mirrors run with 600000ms child timeouts, record shows ${v.execution?.subprocessTimeoutMs}`; const log = exists(join(selftestDir, 'selftest.log')) ? readFileSync(join(selftestDir, 'selftest.log'), 'utf8') : ''; if (!/EXIT=0/.test(log)) return 'NOT-RUN: verification.json present but no recorded EXIT=0 in selftest.log'; - return true; // v.baselineCommit is the harness review-ledger anchor (per-file digests enforced at run start), recorded alongside source.baselineCommit below + return true; }); for (const f of faults) { if (f.preimage.files.length === 1 && !f.preimage.markerUniqueNow) throw new Error(`manifest integrity: ${f.fault} CAS preimage marker not unique in current source`); diff --git a/scripts/lib/check-go-test-events.mjs b/scripts/lib/check-go-test-events.mjs new file mode 100644 index 000000000..39649a45f --- /dev/null +++ b/scripts/lib/check-go-test-events.mjs @@ -0,0 +1,27 @@ +#!/usr/bin/env node +import { readFileSync } from 'node:fs'; +import { checkGoEvents } from '../run-postgres-integration.mjs'; + +const required = process.argv.slice(2); +if (required.length === 0) { + console.error('usage: check-go-test-events.mjs ... < go-test.json'); + process.exit(2); +} +for (const name of required) { + if (!/^Test\w+(\/\S+)?$/.test(name)) { + console.error(`invalid required test name: ${name}`); + process.exit(2); + } +} + +const stdin = readFileSync(0, 'utf8'); +try { + const { passes } = checkGoEvents(stdin, required); + console.log( + `named event gate OK: ${required.length} required events run+passed, zero skips, ` + + `${passes.length} total passing events`, + ); +} catch (error) { + console.error(`named event gate FAILED: ${error.message}`); + process.exit(1); +} diff --git a/scripts/lib/formal-api.mjs b/scripts/lib/formal-api.mjs index 817d45550..a84721658 100644 --- a/scripts/lib/formal-api.mjs +++ b/scripts/lib/formal-api.mjs @@ -11,8 +11,8 @@ const boundary = 'production HTTP routes, services and compiler; test-owned in-m export const apiChecks = [ { id: 'derived-policy', title: 'Bounded actual AQL derived arithmetic, missing input and division policies', package: './internal/server', tests: ['TestDerivedArithmeticPoliciesAgainstArango'], integration: true, requiredEnvironment: ['LOOM_TEST_ARANGO_URL'], boundary: '400 authored source/literal inputs through production recipe compiler and Engine with actual Arango; eight guard-order conformance cases; no arbitrary numerics, both-input-missing, FHIR profile, HTTP or browser claim' }, { id: 'append-rows', title: 'Actual current-workspace Group, Calculate and APPEND rows after edit and restoration', package: './internal/server', tests: ['TestCurrentDraftAppendRowsAgainstArangoClickHouse'], integration: true, requiredEnvironment: ['LOOM_TEST_ARANGO_URL', 'LOOM_CLICKHOUSE_URL'], boundary: 'production recipe compiler, current-workspace capture and Engine execution against actual Arango and ClickHouse; fixed authored recipes, exact row multisets, nullable padding and cleanup; no public save, HTTP, browser or exhaustive-input claim' }, - { id: 'release-dangle-reconcile', title: 'Active-release bindings fence startup ClickHouse reconciliation', package: './internal/dataframe/publication/arango', tests: ['TestReconcileDanglesActiveReleaseAgainstArangoClickHouse'], integration: true, requiredEnvironment: ['LOOM_TEST_ARANGO_URL', 'LOOM_CLICKHOUSE_URL'], boundary: 'real Arango registry/release state and ClickHouse tables under a controlled publish/reconcile interleaving; proves active-release-bound executions are not physically dropped or demoted; test-owned database/tables with cleanup; no database process restart or browser claim' }, - { id: 'draft-cas', title: 'Real draft write conflict classification and transactional history rollback', package: './internal/explorer/arango', tests: ['TestDraftCASConcurrentWritersAgainstArango', 'TestDraftCASAfterWinnerCommitAgainstArango', 'TestDraftCASPublicApplyConflictClassAgainstArango', 'TestDraftCASUpdateFailureRollsBackHistoryAgainstArango'], integration: true, requiredEnvironment: ['LOOM_TEST_ARANGO_URL'], boundary: 'production SaveDraft and public lifecycle Apply with actual Arango transactions in private databases; controlled read barriers and update failure; no HTTP, browser or database-process-restart claim' }, + { id: 'release-dangle-reconcile', title: 'Active-release bindings fence startup ClickHouse reconciliation', package: './internal/backends/arango/internal/publication', tests: ['TestReconcileDanglesActiveReleaseAgainstArangoClickHouse'], integration: true, requiredEnvironment: ['LOOM_TEST_ARANGO_URL', 'LOOM_CLICKHOUSE_URL'], boundary: 'real Arango registry/release state and ClickHouse tables under a controlled publish/reconcile interleaving; proves active-release-bound executions are not physically dropped or demoted; test-owned database/tables with cleanup; no database process restart or browser claim' }, + { id: 'draft-cas', title: 'Real draft write conflict classification and transactional history rollback', package: './internal/backends/arango/internal/explorer', tests: ['TestDraftCASConcurrentWritersAgainstArango', 'TestDraftCASAfterWinnerCommitAgainstArango', 'TestDraftCASPublicApplyConflictClassAgainstArango', 'TestDraftCASUpdateFailureRollsBackHistoryAgainstArango'], integration: true, requiredEnvironment: ['LOOM_TEST_ARANGO_URL'], boundary: 'production SaveDraft and public lifecycle Apply with actual Arango transactions in private databases; controlled read barriers and update failure; no HTTP, browser or database-process-restart claim' }, { id: 'publication-rows', title: 'Actual compiled rows through publication storage and fresh Viewer', package: './internal/server', tests: ['TestPublicationRowsAndFreshViewerAgainstArangoClickHouse'], integration: true, requiredEnvironment: ['LOOM_TEST_ARANGO_URL', 'LOOM_CLICKHOUSE_URL'], boundary: 'production compiler/AQL, ClickHouse materializer, Arango registry and Explorer activation, fresh database clients and public Viewer; test-owned auth/capabilities; no browser or process-restart claim' }, { id: 'route-rows', title: 'Actual Arango route rows, persisted history and fresh repository read-back', package: './internal/server', tests: ['TestSavedRelatedRouteRowsAgainstArango'], integration: true, requiredEnvironment: ['LOOM_TEST_ARANGO_URL'], boundary: 'production public HTTP/service/compiler/query execution and actual Arango repository; isolated test-owned database; no browser or database-process-restart claim' }, { id: 'route-roundtrip', title: 'Saved Related route proposal, Apply, restore and fresh-read identity', package: './internal/server', tests: ['TestSavedRelatedRouteProposalApplyRestoreAndFreshRead'], boundary }, @@ -80,7 +80,6 @@ export function apiDoctor() { } throw new Error(`Install Go ${declared.slice(1).join('.')} or set GO to its executable; no compatible Go found (${attempts.join(', ')})`); } -/** A successful package with no matching tests is not a successful API check. */ export function checkApiEvents(stdout, check) { assert.ok(Array.isArray(check.tests) && check.tests.length > 0, 'API check selects no named tests'); assert.ok(check.tests.every(name => /^Test\w+$/.test(name)), 'Invalid API test name'); diff --git a/scripts/lib/formal-api.test.mjs b/scripts/lib/formal-api.test.mjs index 53cfd4ce8..27f687e33 100644 --- a/scripts/lib/formal-api.test.mjs +++ b/scripts/lib/formal-api.test.mjs @@ -17,7 +17,6 @@ test('API checks select real named public handler tests', () => { assert.throws(() => selectApiChecks('missing'), /Unknown API feature/); for (const check of apiChecks) { const directory = new URL(`../../${check.package.replace(/^\.\//, '')}/`, import.meta.url); - // Discover declarations from the selected package, independent of registry IDs. const source = readdirSync(directory).filter(file => file.endsWith('_test.go')) .map(file => readFileSync(new URL(file, directory), 'utf8')).join('\n'); for (const name of check.tests) assert.match(source, new RegExp(`func ${name}\\(t \\*testing\\.T\\)`)); diff --git a/scripts/lib/formal-apply-replay.mjs b/scripts/lib/formal-apply-replay.mjs index ed53f8401..448da49d3 100644 --- a/scripts/lib/formal-apply-replay.mjs +++ b/scripts/lib/formal-apply-replay.mjs @@ -8,7 +8,6 @@ const productionFile = join(root, 'ui/packages/loom-ui/src/features/ExplorerBuil export async function loadApplyImplementation(source = readFileSync(productionFile, 'utf8')) { return import(`data:text/javascript;base64,${Buffer.from(stripTypeScriptTypes(source)).toString('base64')}`); } -/** Each adapter invokes the real synchronous production ownership component. */ export function replayApplyPath(path, implementation, property) { const owners = { A: { name: 'A' }, B: { name: 'B' } }; const names = new Map(Object.entries(owners).map(([name, owner]) => [owner, name])); @@ -130,7 +129,6 @@ export async function checkApplyGraph(graph, { source, witnessFile, property } = assert.deepEqual([...coverage].sort(), ['ApplyReturns', 'BeginApply', 'Cancel', 'ChangeContext', 'SetCurrentOwner']); return { states: graph.states.length, transitions: graph.steps.length, replays: paths.length, maxDepth: Math.max(...paths.map((path) => path.length)) }; } -/** Run one serializable input against the real component, including duplicate completion. */ export function replayIdentityInput(inputs, implementation, property = 'Inv_CompletionOwnsCurrentContext') { const { submittedOwner, liveOwner, submittedKey, liveKey, submittedGeneration, liveGeneration, applied } = inputs; assert.ok(['A', 'B'].includes(submittedOwner) && ['A', 'B'].includes(liveOwner), 'Invalid identity witness owner'); @@ -154,7 +152,6 @@ export function replayIdentityInput(inputs, implementation, property = 'Inv_Comp throw error; } } -/** Independent bounded reader inputs include identity dimensions collapsed by W1/W2. */ export async function checkApplyIdentityInputs(source, property = 'Inv_CompletionOwnsCurrentContext') { const implementation = await loadApplyImplementation(source); let count = 0; @@ -170,8 +167,6 @@ export async function checkApplyIdentityInputs(source, property = 'Inv_Completio } return count; } -/** Replay recorded inputs, not the model's predicted outputs. A held property is - * model drift/unreproduced, never an assertion that the whole application passed. */ export async function replayWitness(witness, { source } = {}) { assert.equal(witness.schemaVersion, 1, 'Unknown witness schema'); assert.equal(witness.model, 'ApplySettlementRace', 'Unsupported witness model'); diff --git a/scripts/lib/formal-behavior.mjs b/scripts/lib/formal-behavior.mjs index 1a82f3a73..0aca44409 100644 --- a/scripts/lib/formal-behavior.mjs +++ b/scripts/lib/formal-behavior.mjs @@ -4,7 +4,6 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from import { join, resolve } from 'node:path'; import { spawnSync } from 'node:child_process'; import { root } from './formal-tools.mjs'; -// A partition of existing suites, not a claim that every test is a formal proof. export const features = [ { id: 'ownership', model: ['BuilderLifecycle', 'ApplySettlementRace'], pattern: /^src\/features\/ExplorerBuilder\/constructionWorkspace\/useConstructionLifecycle\./ }, { id: 'core-protocol', model: [], pattern: /^src\/features\/ExplorerBuilder\/BuilderWorkspace\.coreProtocol\./ }, @@ -49,8 +48,6 @@ export function runBehavior({ feature = 'all', shard, pattern, output }) { if (pattern) args.push('--testNamePattern', pattern); const env = { ...process.env, LOOM_MUTATION_TARGET: '', LOOM_MUTATION_SOURCE: '', LOOM_MUTATION_OVERRIDES: '', LOOM_BUILDER_TRACE_OUTPUT: '', LOOM_VITEST_CACHE_DIR: join(output, 'cache') }; - // These optional capture/replay modes belong to retired external generators. - // Ambient settings must not turn a normal behavioral check into a fixture replay. for (const key of Object.keys(env)) if (key.startsWith('BUILDER_')) delete env[key]; diff --git a/scripts/lib/formal-core-protocol-replay.mjs b/scripts/lib/formal-core-protocol-replay.mjs index f09585922..9b18cb5f8 100644 --- a/scripts/lib/formal-core-protocol-replay.mjs +++ b/scripts/lib/formal-core-protocol-replay.mjs @@ -17,7 +17,6 @@ const modelDirectory = join(root, 'verification/builder-interactions'); const initialLabel = 'Patient ID'; const competingLabel = 'Patient Competing B'; -// The ten invariants of CoreAuthoringProtocol.cfg in declaration order (TypeOK excluded). export const coreProtocolProperties = ['Inv_EntryCASDecidesCompilation', 'Inv_CaptureEqualsCommittedHead', 'Inv_ReceiptPersistedForCapture', 'Inv_PreviewRequiresPersistedReceipt', 'Inv_ServerEchoesRequestedReceipt', 'Inv_NoAdmissionWithoutExecution', 'Inv_StaleAdmissionIsUnreachable', 'Inv_HeadAdvancedOnlyByCommittedSave', 'Inv_LostAckCannotAdmitAdvancedHead', 'Inv_StaleSaveDoesNotMoveHead']; const files = () => ['go.mod', 'go.sum', coreProtocolAdapter, 'internal/server/core_authoring_protocol_test.go', ...['internal/explorer/lifecycle', 'internal/explorer'].flatMap(directory => readdirSync(join(root, directory)).filter(file => file.endsWith('.go') && !file.endsWith('_test.go')).map(file => `${directory}/${file}`))].sort(); @@ -33,17 +32,8 @@ const cap = value => value === 1 ? 'cap1' : 'cap2'; const content = receipt => receipt === 'cap1' ? 1 : 2; const caps = ['cap1', 'cap2']; -// The event-log taxonomy the Go adapter records at real call boundaries. -// Every name except 'Observe' corresponds to exactly one named model action -// (or branch); 'Observe' is the documented stutter (see checkCoreProtocolEventPath). export const coreProtocolReplayEvents = ['SaveDispatch', 'SaveAck', 'REntryAccept', 'REntryReject', 'RCapture', 'RPersist', 'RComplete', 'PreviewExec', 'Observe']; -// The eight replay schedules the model explores at the production boundary: -// reconcile of head rTarget versus a competing save of head sTarget, with the -// save issued before reconcile entry, while compile is suspended, or omitted, -// optionally re-previewing after the head observably advanced. The client's -// save acknowledgement is not wire-observable and has no field here; the two -// mounted-driver vitest proofs own the acknowledgement/admission identities. export const coreProtocolCases = [ { id: 'S1', rTarget: 1, sTarget: 1, savePosition: 'before-entry', previewAfterAdvance: false }, { id: 'S2', rTarget: 1, sTarget: 1, savePosition: 'in-compile', previewAfterAdvance: false }, @@ -55,21 +45,6 @@ export const coreProtocolCases = [ { id: 'S8', rTarget: 1, sTarget: 1, savePosition: 'in-compile', previewAfterAdvance: true }, ]; -// Plan C14: the model's head channels merge the two value-equal heads of an -// A -> B -> A prefix, so the merge's justification cannot come from the -// abstract schedules themselves (head identity is all they carry). The -// concrete-prefix lane replays exactly that prefix through the production -// boundary and records raw wire facts; the named claims below, adjudicated -// here, are what the C14 row requires: the revert's value identity is -// witnessed by the emitted receipt label while the stored digests stay -// pairwise distinct (ApplyCommands canonicalizes through migration and -// normalization on every save, so a value-equal revert legitimately stores a -// new digest — the model merges values, not stored forms), a stale client -// holding that value-equal head identity cannot own the current head, a -// fresh request at the real head completes with a persisted receipt, the -// same-request replay does not move or diverge the head, the preview echoes -// the persisted receipt, and the store persisted each replaced head as a -// distinct revision. export const coreProtocolConcretePrograms = [{ id: 'C14-A-B-A' }]; export const coreProtocolConcreteProperties = ['C14_ValueMergeZoneRealized', 'C14_StaleValueEqualReconcileCannotOwnHead', 'C14_StaleDigestAtCurrentVersionCannotOwnHead', 'C14_StaleVersionWithCurrentDigestCannotOwnHead', 'C14_ValidFreshReconcileCompletes', 'C14_SameRequestReplayPreservesHeadAndReceipt', 'C14_PreviewEchoesPersistedValidReceipt', 'C14_PersistedHistoryHasDistinctHeadRevisions', 'C14_NoRecordedWireAnomalies']; @@ -121,10 +96,6 @@ function successors(state) { return next; } -// Mirrors scratchControl in scripts/verify-builder-formal-core.mjs, widened with -// the model's CONSTANT assignments so the worker can evaluate this model, and -// using a generated structural view because the shared statechart view does not -// name CoreAuthoringProtocol variables. const controlView = "export const title = 'Core authoring protocol controls';\nexport const state = (v) => Object.entries(v).map(([name, value]) => `${name}=${JSON.stringify(value)}`).join('; ');\n"; function scratchControl([property, action, before, after], reportDirectory) { @@ -184,18 +155,6 @@ export function coreProtocolFrontier() { return { states: [...states.values()], steps }; } -// --- Replay event-path validation ------------------------------------------------- -// The Go adapter records one event per real call boundary in production call -// order. Here each event must extend the replica along EXACTLY ONE named model -// action's legal successor edge (single source of truth: the successors() -// function shared with the frontier — no duplicated step logic), and its claims -// must agree with the replica variables of the state reached. 'Observe' events -// (getBuilder reads) are the documented stutter: the model's Next has no stutter -// action over vars, so an Observe is allowed only when it changes no model -// variable and its claims (headVersion, hasReceipts) match the current replica -// state exactly. Wrong order, omitted relevant transitions and substituted -// identities therefore have no legal walk and are rejected as FORMAL_CONFORMANCE -// before any terminal comparison could mask them. function eventPathError(message, replayCase, eventIndex, event, expectedAction, replica) { const available = [...new Set(successors(replica).map(edge => edge.action))]; const witness = { schemaVersion: 1, kind: 'core-protocol-event-path', model: coreProtocolModel, case: replayCase, eventIndex, event: event ?? null, expectedAction: expectedAction ?? null, availableActions: available, replicaState: replica }; @@ -258,9 +217,6 @@ export function checkCoreProtocolEventPath(replayCase, events) { `Malformed core protocol replay event ${index} in case ${replayCase.id}`); names.push(event.name); if (event.name === 'Observe') { - // Documented stutter: an observation may not change model variables and - // must match the replica state it names exactly. A headVersion the - // replica never reached has no stutter witness and is rejected. const matches = event.claims.headVersion === replica.headVersion && (!Object.hasOwn(event.claims, 'hasReceipts') || event.claims.hasReceipts === (replica.receipts.length > 0)) && Object.keys(event.claims).every(name => ['headVersion', 'hasReceipts'].includes(name)); @@ -404,9 +360,6 @@ export function checkCoreProtocolResults(record, cases, { property } = {}) { assert.deepEqual(Object.keys(record).sort(), ['results', 'schemaVersion'].sort()); assert.equal(record.schemaVersion, 1); assert.ok(Array.isArray(record.results)); - // When Arango mode is active the Go lane tags every witness row with its store - // mode (dual replay); the store-less shape is the map-only witness. Every - // store present must cover every case, and store values are closed. const dual = record.results.some(row => row && Object.hasOwn(row, 'store')); const rowKeys = dual ? [...observationKeys, 'store'] : observationKeys; const stores = [...new Set(record.results.map(row => row && row.store).filter(value => value !== undefined))]; @@ -450,11 +403,6 @@ export function checkCoreProtocolResults(record, cases, { property } = {}) { try { const expected = expectedObservation(replayCase); for (const name of observationKeys.filter(name => name !== 'id' && name !== 'events')) assert.equal(actual[name], expected[name], `FORMAL_CONFORMANCE: ${name}`); - // Replaying the recorded call boundaries against the model's legal - // transitions is the conformance gate: terminal compatibility alone - // could let an illegal production prefix land on a compatible final - // state. Terminal fields are then re-checked against the replica - // terminal state the event stream actually derived. const replica = checkCoreProtocolEventPath(replayCase, actual.events); checkCoreProtocolTerminalConformance(replayCase, actual, replica); } catch (error) { error.code = 'FORMAL_CONFORMANCE'; conformance.push(error); } @@ -475,14 +423,7 @@ export function checkCoreProtocolResults(record, cases, { property } = {}) { }; } -// --- Concrete-prefix lane (plan C14) ------------------------------------------- -// The Go lane records raw wire facts from the production handlers and never -// asserts, so every C14 guarantee is adjudicated here against the recorded -// row. A semantically wrong server therefore lands as a named red claim with -// a witness, not as an aborted or self-certifying witness. const concreteRowKeys = ['id', 'seedVersion', 'seedDigest', 'saveBStatus', 'saveBDigest', 'saveAStatus', 'saveAVersion', 'saveADigest', 'mergeValueZone', 'staleAttempt', 'stale2Attempt', 'stale3Attempt', 'validAttempt', 'replayAttempt', 'previewStatus', 'previewReceipt', 'validLabel', 'revisions', 'staleError', 'stale2Error', 'stale3Error', 'staleNotes', 'notes']; -// The Go wire record omits these when empty (omitempty), so absence is a -// legal shape for them while every other key must always be present. const concreteRowOptionalKeys = ['previewReceipt', 'validLabel', 'staleError', 'stale2Error', 'stale3Error', 'staleNotes', 'notes']; const concreteAttemptKeys = ['requestId', 'draftVersion', 'draftDigest', 'success', 'headVersion', 'headDigest', 'hasReceipt', 'receiptId']; const concreteRevisionKeys = ['draftDigest', 'draftVersion', 'found', 'revisionId']; @@ -498,9 +439,6 @@ function validateConcreteRow(record) { assert.deepEqual(Object.keys(record).sort(), ['results', 'schemaVersion'].sort()); assert.equal(record.schemaVersion, 1); assert.ok(Array.isArray(record.results)); - // Arango mode tags every concrete wire row with its store mode (dual - // replay); the store-less shape is the map-only witness. Store values are - // closed and handled per (store, program) below. const dual = record.results.some(row => row && typeof row === 'object' && Object.hasOwn(row, 'store')); const allowedKeys = dual ? [...concreteRowKeys, 'store'] : concreteRowKeys; const stores = [...new Set(record.results.map(row => row && typeof row === 'object' ? row.store : undefined).filter(value => value !== undefined))]; @@ -554,25 +492,10 @@ export function checkCoreProtocolConcreteResults(record, programs, { property } assert.ok(row, 'Missing concrete program'); const local = [], fail = name => local.push(concretePropertyError(name, program, row)); const valid = row.validAttempt, stale = row.staleAttempt, replay = row.replayAttempt; - // Merge zone (value level): the two stored A digests are distinct from B - // and from the seed's stored digest (normalizing save path), mergeValueZone - // pins the digest-level fact as false, and the emitted receipt label of - // the head-3 compile equals the seed column label (claimed under - // ValidFreshReconcileCompletes) — value identity, not stored form, is - // what the model's head channels merge. const digestsPairwiseDistinct = row.seedDigest !== '' && row.saveBDigest !== '' && row.saveADigest !== '' && row.seedDigest !== row.saveBDigest && row.saveADigest !== row.seedDigest && row.saveADigest !== row.saveBDigest; if (!(row.seedVersion === 1 && row.mergeValueZone === false && digestsPairwiseDistinct && row.saveAStatus === 200 && row.saveAVersion === 3 && row.saveBStatus === 200)) fail(coreProtocolConcreteProperties[0]); if (!(stale.success === false && stale.draftVersion === 1 && stale.draftDigest === row.seedDigest && stale.headVersion === 3 && stale.headDigest === row.saveADigest && stale.headDigest !== stale.draftDigest && stale.hasReceipt === false && row.staleError === 'DRAFT_CONFLICT')) fail(coreProtocolConcreteProperties[1]); - // Single-coordinate stale probes: with only the entry CAS version clause - // removed, [2] stays green and [3] alone is admitted; with only the - // digest clause removed, [2] alone is admitted; removing both turns all - // three red. Each claim is therefore the unique falsifier of one clause. - // The single-coordinate claims deliberately omit a hasReceipt condition - // (unlike the first, both-stale probe, which is scheduled first and can - // therefore witness receipt absence): under a single-clause fault the - // admitted probe persists a receipt before the later probe's wire read, - // and only the presented-identity comparisons are clause-deterministic. const stale2 = row.stale2Attempt, stale3 = row.stale3Attempt; if (!(stale2.success === false && stale2.draftVersion === 3 && stale2.draftDigest === row.seedDigest && stale2.headVersion === 3 && stale2.headDigest === row.saveADigest && row.stale2Error === 'DRAFT_CONFLICT')) fail(coreProtocolConcreteProperties[2]); if (!(stale3.success === false && stale3.draftVersion === 1 && stale3.draftDigest === row.saveADigest && stale3.headVersion === 3 && stale3.headDigest === row.saveADigest && row.stale3Error === 'DRAFT_CONFLICT')) fail(coreProtocolConcreteProperties[3]); @@ -665,10 +588,6 @@ export async function checkCoreProtocolGraphControls(graph) { return mutations.length + checkCoreProtocolEventControls().length; } -// Case-level controls proving the event validator bites: fabricated order, -// substituted identity, omitted transition and stutter abuse must all be -// rejected by the pure validator, and the legal stream must be accepted. -// These run over hand-built events, without touching Go. function expectEventPath(control, replayCase, events, expectation) { let error; try { checkCoreProtocolEventPath(replayCase, structuredClone(events)); } catch (caught) { error = caught; } @@ -720,8 +639,6 @@ function validateAbstractState(value) { assert.ok(['none', ...caps].includes(value.previewEcho) && ['none', ...caps].includes(value.admitted)); } -// Model-fault successors per property, mirroring coreProtocolModelControls so a -// counterexample edge is exactly one supported model fault away from healthy. function modelFaultSuccessors(state, property) { const healthy = successors(state); const edges = healthy.map(edge => ({ action: edge.action, vars: structuredClone(edge.vars) })); @@ -862,11 +779,6 @@ export async function checkCoreProtocolSourceProof(proof, graph, output) { assert.ok(known); assert.deepEqual(proof, known); if (proof.kind === 'vitest') { - // The shared vitest harness registers these proofs in the - // formal-source-proofs registry, which statically imports this module. - // A cyclic dynamic import from within this module's own evaluation would - // deadlock (Node ESM), so hand the proof to a child process whose entry - // is not part of this module graph; the harness itself is unchanged. const sourceProofModule = pathToFileURL(join(root, 'scripts/lib/formal-source-proofs.mjs')).href; const script = `const { checkSourceProof } = await import(${JSON.stringify(sourceProofModule)}); await checkSourceProof(JSON.parse(process.env.LOOM_CORE_PROTOCOL_VITEST_PROOF), undefined, process.env.LOOM_CORE_PROTOCOL_VITEST_OUTPUT);`; const result = spawnSync(process.execPath, ['--input-type=module', '-e', script], { cwd: root, encoding: 'utf8', timeout: verificationTimeoutMs, maxBuffer: 8 * 1024 * 1024, detached: true, env: { ...process.env, LOOM_CORE_PROTOCOL_VITEST_PROOF: JSON.stringify(proof), LOOM_CORE_PROTOCOL_VITEST_OUTPUT: output } }); diff --git a/scripts/lib/formal-draft-transaction.mjs b/scripts/lib/formal-draft-transaction.mjs index c45c1c9fe..e21577c19 100644 --- a/scripts/lib/formal-draft-transaction.mjs +++ b/scripts/lib/formal-draft-transaction.mjs @@ -9,14 +9,14 @@ import { apiDoctor, checkApiEvents } from './formal-api.mjs'; import { verificationTimeoutMs, goTestTimeout } from './verification-budget.mjs'; export const draftTransactionModel = 'DraftSaveTransaction'; -export const draftTransactionFile = 'internal/explorer/arango/drafts.go'; -export const draftTransactionAdapter = 'internal/explorer/arango/draft_transaction_formal_replay_test.go'; +export const draftTransactionFile = 'internal/backends/arango/internal/explorer/drafts.go'; +export const draftTransactionAdapter = 'internal/backends/arango/internal/explorer/draft_transaction_formal_replay_test.go'; export const draftTransactionProperties = ['Inv_NoPartialDraftHistory', 'Inv_ConflictIsDomainConflict', 'Inv_SuccessRequiresConfirmedCommit', 'Inv_MissingCASAborts', 'Inv_ValidSaveWorks', 'Inv_KnownFailureDoesNotPublish']; const modelDirectory = join(root, 'verification/builder-interactions'); const testName = 'TestDraftTransactionModelReplay'; const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object' ? Object.fromEntries(Object.keys(value).sort().map(name => [name, canonical(value[name])])) : value; const key = value => JSON.stringify(canonical(value)); -const files = () => ['go.mod', 'go.sum', draftTransactionAdapter, ...['internal/explorer/arango', 'internal/explorer', 'internal/store/arango'].flatMap(directory => readdirSync(join(root, directory)).filter(file => file.endsWith('.go') && !file.endsWith('_test.go')).map(file => `${directory}/${file}`))].sort(); +const files = () => ['go.mod', 'go.sum', draftTransactionAdapter, ...['internal/backends/arango/internal/explorer', 'internal/explorer', 'internal/backends/arango/internal/store'].flatMap(directory => readdirSync(join(root, directory)).filter(file => file.endsWith('.go') && !file.endsWith('_test.go')).map(file => `${directory}/${file}`))].sort(); export const draftTransactionSourceHashes = () => Object.fromEntries(files().map(file => [file, digest(join(root, file))])); const fields = ['begin', 'read', 'archive', 'update', 'commit']; const initial = () => ({ phase: 'begin', outcomes: Object.fromEntries(fields.map(name => [name, 'none'])), stagedHistory: false, stagedHead: false, history: false, head: false, transactionResult: 'none', result: 'none', backendConflict: false }); @@ -106,7 +106,7 @@ function runGo(cases,{productionSource,output}={}) { const input=join(scratch,'input.json'),out=join(scratch,'output.json');writeFileSync(input,JSON.stringify({schemaVersion:1,cases:cases.map(({id,outcomes})=>({id,outcomes}))}));const inputSha=digest(input); const args=['test','-json',`-timeout=${goTestTimeout}`,'-count=1','-run',`^${testName}$`]; if(productionSource!==undefined){const replacement=join(scratch,'drafts.go'),overlay=join(scratch,'overlay.json');writeFileSync(replacement,productionSource);writeFileSync(overlay,JSON.stringify({Replace:{[join(root,draftTransactionFile)]:replacement}}));args.push('-overlay',overlay);} - args.push('./internal/explorer/arango');const env={...process.env,GOFLAGS:'',GOWORK:'off',GOTOOLCHAIN:'local',LOOM_FORMAL_DRAFT_TRANSACTION_INPUT:input,LOOM_FORMAL_DRAFT_TRANSACTION_OUTPUT:out}; + args.push('./internal/backends/arango/internal/explorer');const env={...process.env,GOFLAGS:'',GOWORK:'off',GOTOOLCHAIN:'local',LOOM_FORMAL_DRAFT_TRANSACTION_INPUT:input,LOOM_FORMAL_DRAFT_TRANSACTION_OUTPUT:out}; if(tool.executable==='/workspace/.loom-setup/go/bin/go'){if(!env.GOMODCACHE&&existsSync('/workspace/.loom-setup/gomod'))env.GOMODCACHE='/workspace/.loom-setup/gomod';if(!env.GOCACHE&&existsSync('/workspace/loom/.gocache'))env.GOCACHE='/workspace/loom/.gocache';} const result=spawnSync(tool.executable,args,{cwd:root,env,encoding:'utf8',timeout:verificationTimeoutMs,maxBuffer:8*1024*1024,detached:true}); if(result.error&&result.pid){try{process.kill(-result.pid,'SIGKILL');}catch(error){if(error.code!=='ESRCH')throw error;}} diff --git a/scripts/lib/formal-implementation-conformance.mjs b/scripts/lib/formal-implementation-conformance.mjs index 592d01f19..ffab9226c 100644 --- a/scripts/lib/formal-implementation-conformance.mjs +++ b/scripts/lib/formal-implementation-conformance.mjs @@ -195,8 +195,6 @@ function verifyTrace(root, scratch, header, expected, checkModel, outputDirector copyFileSync(join(scratch, 'vitest.json'), join(outputDirectory, 'vitest.json')); } checkModel('ImplementationConformance', directory); - // A passing full run above pstack's graph budget still gets a clearly scoped - // reader diagram. Select actual traces covering distinct observed edge kinds. const edgeKinds = (trace) => trace.slice(1).map((o, i) => JSON.stringify([trace[i].status, o.event, o.status, o.staleEvent, o.completionObserved, o.completionAccepted])); const uncovered = new Set(traces.flatMap(edgeKinds)); const selected = []; @@ -217,7 +215,6 @@ function verifyTrace(root, scratch, header, expected, checkModel, outputDirector assert.ok(sampleResult.chart, 'Representative trace diagram exceeded the pstack chart budget'); if (outputDirectory) writeFileSync(join(outputDirectory, 'diagram-scope.json'), JSON.stringify({ selection: 'greedy coverage of observed edge kinds; not all model schedules', selected: selected.map((trace) => trace[0].name), allTraces: traces.length, selectedStates: selected.reduce((sum, trace) => sum + trace.length, 0) }, null, 2)); - // Controls for the bridge itself: a corrupted real observation must fail its exact rule. const isHook = traces[0][0].kind === 'lifecycle'; const corrupt = structuredClone(traces); const record = corrupt.flat().find((o) => isHook ? o.canApply : o.receiptId !== ''); @@ -229,7 +226,6 @@ function verifyTrace(root, scratch, header, expected, checkModel, outputDirector else record.receiptSnapshot = 'wrong-snapshot'; writeFileSync(join(directory, 'ImplementationConformance.tla'), withData(corrupt)); - // Isolate the rule: another valid failure must not mask the intended one. const invariant = isHook ? 'Inv_VisibleReadyIsReviewed' : 'Inv_RenderedReconcileReceiptIsCurrent'; writeFileSync(join(directory, 'ImplementationConformance.cfg'), `SPECIFICATION Spec\nCHECK_DEADLOCK FALSE\nCONSTANT Traces <- Observations\nINVARIANT ${invariant}\n`); checkModel('ImplementationConformance', directory, invariant); diff --git a/scripts/lib/formal-model-graph.mjs b/scripts/lib/formal-model-graph.mjs index 01471b3ec..731f86bf8 100644 --- a/scripts/lib/formal-model-graph.mjs +++ b/scripts/lib/formal-model-graph.mjs @@ -6,7 +6,6 @@ import { tmpdir } from 'node:os'; import { pathToFileURL } from 'node:url'; import { spawnSync } from 'node:child_process'; import { tools, digest, root } from './formal-tools.mjs'; -/** Dump the same checked bounded model; parse it with pstack's own IR frontend. */ export async function modelGraph(modelDirectory, output) { const { scripts, jar } = tools(); const directory = mkdtempSync(join(tmpdir(), 'loom-model-graph-')); diff --git a/scripts/lib/formal-operation-history-model.mjs b/scripts/lib/formal-operation-history-model.mjs index 77d5c086c..b18ea6b09 100644 --- a/scripts/lib/formal-operation-history-model.mjs +++ b/scripts/lib/formal-operation-history-model.mjs @@ -1,4 +1,3 @@ -// Fixed witnesses adapted from 5646633e:verification/builder-interactions/operationHistoryModel.mjs. import { createHash } from 'node:crypto'; const MAX_DOWNSTREAM_STEPS = 2; @@ -129,7 +128,6 @@ function modelView(workspace) { .filter((step) => step.kind === 'CALCULATE') .map((step) => [step.id, { columnId: step.outputColumnId, - // The bounded expression is aggregate + integer literal 1; it has no user cast. type: workspace.aggregate === 'COUNT_ROWS' ? 'integer' : 'decimal', }])); return { @@ -266,7 +264,6 @@ function assertState(node) { const rawCandidatePreservedByProposal = node.action?.kind !== 'propose' || JSON.stringify(state.candidate) === JSON.stringify(node.parent.state.candidate); const proposalSnapshotMatches = !state.proposalSnapshot || state.proposalSnapshot === reviewedSnapshot(state.candidate, state.resolvedCandidate, state.removedStepIds); - // This oracle is independent from resolveCandidate: only the Group's current key and aggregate IDs are available inputs. const groupColumns = new Set([state.candidate.aggregateOutputId]); if (state.candidate.keyOutputId) groupColumns.add(state.candidate.keyOutputId); const expectedSurvivors = state.candidate.downstream.filter(({ columnId }) => groupColumns.has(columnId)); diff --git a/scripts/lib/formal-process-pool.mjs b/scripts/lib/formal-process-pool.mjs index a5e6e90fa..d87714519 100644 --- a/scripts/lib/formal-process-pool.mjs +++ b/scripts/lib/formal-process-pool.mjs @@ -59,9 +59,6 @@ export async function runProcessPool(jobs, { concurrency = 2, timeoutMs = 30000, throw error; } } - // Some sandboxes deny negative-pid (process group) signals even for own children. - // Individual SIGKILL to each enumerated descendant is then attempted; blocked attempts - // are recorded and termination is proven below rather than assumed. if (blocked.length) for (const pid of descendantsOf(entry.child.pid)) { try { process.kill(pid, 'SIGKILL'); } catch (error) { if (error.code === 'ESRCH') continue; if (error.code === 'EPERM') { blocked.push(`pid ${pid}`); continue; } throw error; } diff --git a/scripts/lib/formal-projection-replay.mjs b/scripts/lib/formal-projection-replay.mjs index 70a48408c..e4b6d065f 100644 --- a/scripts/lib/formal-projection-replay.mjs +++ b/scripts/lib/formal-projection-replay.mjs @@ -12,8 +12,6 @@ export const projectionValidationFile = 'internal/explorer/authoringv2/construct export const projectionAdapterFile = 'internal/explorer/authoringv2/construction_projection_replay_test.go'; export const projectionTestName = 'TestProjectionOwnerBranchReplay'; -// Single Node-side mirror of the Go branch table; the lane asserts that every -// observation matches, so any drift between the two tables is a loud failure. export const projectionBranchExpectations = { 'pivot-accept': { accept: true, contains: null }, 'pivot-not-consumed': { accept: false, contains: 'source projection must be consumed by its owner PIVOT step' }, @@ -33,9 +31,6 @@ export const projectionBranchExpectations = { 'carry-missing-typed-output': { accept: false, contains: 'does not carry source projection' }, }; -// Falsification pairs per U1 arm. Each fault neutralizes exactly one guard; -// the named branch must then diverge (flipped acceptance, lost reject string, -// or a crashing validator) with the restored run green. export const projectionSourceProofs = [ { branch: 'owner-stepid-empty', arm: 'owner-required-id', edits: [['if !requiredID(projection.OwnerStepID) {', 'if false && !requiredID(projection.OwnerStepID) {']] }, { branch: 'owner-missing', arm: 'owner-exists', edits: [['if ownerIndex < 0 {', 'if false && ownerIndex < 0 {']] }, diff --git a/scripts/lib/formal-reconcile-controls.mjs b/scripts/lib/formal-reconcile-controls.mjs index a3ab9a56a..0312afec8 100644 --- a/scripts/lib/formal-reconcile-controls.mjs +++ b/scripts/lib/formal-reconcile-controls.mjs @@ -2,7 +2,6 @@ import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { stripTypeScriptTypes } from 'node:module'; import { join } from 'node:path'; -// Bounded implementation checks, not a proof of the React caller or transport. export async function checkReconcileImplementation(root) { const source = readFileSync(join(root, 'ui/packages/loom-ui/src/features/ExplorerBuilder/authoring/reconcileRecovery.ts'), 'utf8'); const load = (text) => import(`data:text/javascript;base64,${Buffer.from(stripTypeScriptTypes(text)).toString('base64')}`); diff --git a/scripts/lib/formal-reconcile-replay.mjs b/scripts/lib/formal-reconcile-replay.mjs index 7f98772bf..d12a7a127 100644 --- a/scripts/lib/formal-reconcile-replay.mjs +++ b/scripts/lib/formal-reconcile-replay.mjs @@ -51,7 +51,6 @@ export function validateAcceptanceInput(input) { function permutations(values) { return values.length < 2 ? [values] : values.flatMap((value, index) => permutations(values.filter((_, i) => i !== index)).map(rest => [value, ...rest])); } -/** This independent reader oracle defines the four observable caller branches. */ export function expectedAcceptance(input) { validateAcceptanceInput(input); if (input.currentOwner !== 'A' || input.currentFingerprint !== 'F0' || input.requestGeneration !== input.currentGeneration) @@ -110,8 +109,6 @@ export async function replayReconcileWitness(witness, sources) { return { status: 'reproduced', ...error.witness }; } } -/** Convert an upstream decoded TLC path to external inputs for production replay. - * Recompute the reader promise instead of trusting the broken model's outcome. */ export function acceptanceCounterexampleWitness(counterexample) { assert.ok(Array.isArray(counterexample.states) && counterexample.states.length === 3, 'Acceptance counterexample must cross its two synchronous actions'); const [initial, chosen, checked] = counterexample.states; @@ -129,7 +126,6 @@ export function acceptanceCounterexampleWitness(counterexample) { variant: { swapOwner: false, swapFingerprint: false, outputOrder: [...input.responseOutputs].sort() }, expected, actions: ['ChooseInputs', 'ClassifyResponse'] }; } -/** The checked two-action frontier is parsed by upstream pstack's DOT->IR frontend. */ export async function reconcileAcceptanceGraph(output) { const { scripts, jar } = tools(); const directory = mkdtempSync(join(tmpdir(), 'loom-reconcile-graph-')); diff --git a/scripts/lib/formal-recovery-replay.mjs b/scripts/lib/formal-recovery-replay.mjs index c0470d197..262a17d98 100644 --- a/scripts/lib/formal-recovery-replay.mjs +++ b/scripts/lib/formal-recovery-replay.mjs @@ -21,7 +21,6 @@ const allowedActions = new Set(['AwaitRecovery', 'ChangeExplorer', 'EditWorkspac const source = file => readFileSync(join(root, file), 'utf8'); const tsModule = text => `data:text/javascript;base64,${Buffer.from(stripTypeScriptTypes(text, { mode: 'strip' })).toString('base64')}`; -/** Extract one actual production callback; do not duplicate its async branches. */ export function extractReconcileCallback(text) { const anchor = ' const reconcileCurrent = useCallback('; assert.equal(text.split(anchor).length, 2, 'Expected one production reconcile callback'); @@ -46,8 +45,6 @@ export async function loadRecoveryImplementation({ workspaceSource = source(work const accepted = await import(`data:text/javascript;base64,${Buffer.from(acceptanceJS).toString('base64')}`); const callback = extractReconcileCallback(workspaceSource); const names = ['latestState', 'workspaceAttached', 'compileGeneration', 'activeCompile', 'projectId', 'authResourcePath', 'dispatch', 'reconcileBuilder', 'refetchBuilder', 'syncBuilderData', 'setMessage', 'diagnosticsFromError', 'window', 'intentFingerprint', 'workspaceFromState', 'reconcileResultIsCurrent', 'reconcileRecoveryAction', 'reconcileAcceptance']; - // No surrounding component code is evaluated. Missing exercised free variables - // throw rather than falling back to a test reimplementation. const factory = new Function(...names, `"use strict"; return (${callback});`); return dependencies => factory(...names.map(name => name in dependencies ? dependencies[name] : ({ ...recovery, ...authoring, ...accepted })[name])); } @@ -86,7 +83,6 @@ export function validateRecoveryState(value) { if (value.refreshKind === 'backoff') assert.equal(value.hasData, false, 'Backoff has no builder DTO'); } -/** Run an actual suspended production callback through a concrete modeled path. */ export async function replayRecoverySchedule(schedule, { workspaceSource, implementation, automatic = false, rename = false } = {}) { assert.ok(workspaceSource === undefined || typeof workspaceSource === 'string' && workspaceSource.trim(), 'Recovery source override must be nonempty'); assert.ok(implementation === undefined || typeof implementation === 'function', 'Invalid recovery implementation'); @@ -157,7 +153,6 @@ export async function replayRecoverySchedule(schedule, { workspaceSource, implem return { status: 'property-held', expected: contract, actual, returnedReceipt: value !== undefined, requests: requests.length }; } -/** The adapter rejects action/state drift before scheduling the production callback. */ export function validateRecoveryTransition(from, action, to, { brokenResume = false } = {}) { validateRecoveryState(from); validateRecoveryState(to); const next = { ...from }; diff --git a/scripts/lib/formal-source-proofs.mjs b/scripts/lib/formal-source-proofs.mjs index dcc1359c7..46f20dec4 100644 --- a/scripts/lib/formal-source-proofs.mjs +++ b/scripts/lib/formal-source-proofs.mjs @@ -76,7 +76,6 @@ const predicateProperties = { identity: 'Inv_AcceptedReceiptIsCurrent', snapshot: 'Inv_AcceptedReceiptMatchesRequest', output: 'Inv_ReceiptSelectsRequestedOutput', retry: 'Inv_RetryBounds', refresh: 'Inv_RetryBounds', }; -/** Evaluate the production predicate and a separate finite-domain oracle. */ function predicateInput(api, kind, property, inputs) { assert.ok(Object.hasOwn(predicateProperties, kind), 'Unknown predicate witness kind'); assert.equal(predicateProperties[kind], property, 'Unknown predicate witness property/kind'); diff --git a/scripts/lib/formal-statechart-worker.mjs b/scripts/lib/formal-statechart-worker.mjs index bdc9ec0af..cbc8affec 100644 --- a/scripts/lib/formal-statechart-worker.mjs +++ b/scripts/lib/formal-statechart-worker.mjs @@ -1,4 +1,3 @@ -// Integration with the pinned upstream API, not a copy of its CLI or compiler. import { copyFileSync, mkdirSync, writeFileSync } from 'node:fs'; import { basename, join } from 'node:path'; import { pathToFileURL } from 'node:url'; diff --git a/scripts/lib/formal-verdict.mjs b/scripts/lib/formal-verdict.mjs index f3c05e200..2dfe40bf4 100644 --- a/scripts/lib/formal-verdict.mjs +++ b/scripts/lib/formal-verdict.mjs @@ -1,8 +1,5 @@ import assert from 'node:assert/strict'; -/** Read the repository's explicit one-rule-per-line TLC configuration format. - * Unsupported rule syntax blocks instead of silently omitting a declaration. - */ export function configuredModelRules(config) { assert.equal(typeof config, 'string', 'Missing model configuration'); const rules = []; @@ -18,9 +15,6 @@ export function configuredModelRules(config) { return rules; } -/** Validate the pinned statechart worker's structured verdict, never stdout text. - * A named negative control may fail in Init before TLC records an initial state. - */ export function validateModelVerdict(checked, processExit, expectedInvariant, expectedRules) { assert.ok(checked && typeof checked === 'object' && !Array.isArray(checked), 'Missing structured statechart verdict'); assert.ok(expectedInvariant === undefined || typeof expectedInvariant === 'string' && expectedInvariant.trim().length > 0, 'Invalid expected invariant'); @@ -49,7 +43,6 @@ export function validateModelVerdict(checked, processExit, expectedInvariant, ex assert.equal(counterexample.property, expectedInvariant, 'Counterexample does not witness the expected invariant'); assert.ok(Array.isArray(counterexample.states) && counterexample.states.length > 0, 'Counterexample contains no decoded states'); for (const state of counterexample.states) { - // TLC can use an empty action label for a violation in the initial state. assert.ok(state && typeof state === 'object' && !Array.isArray(state) && typeof state.action === 'string', 'Malformed decoded counterexample state'); assert.ok(state.vars && typeof state.vars === 'object' && !Array.isArray(state.vars) && Object.keys(state.vars).length > 0, 'Missing decoded counterexample variables'); } diff --git a/scripts/lib/no_code_comments_backends.py b/scripts/lib/no_code_comments_backends.py new file mode 100644 index 000000000..5b175c9f6 --- /dev/null +++ b/scripts/lib/no_code_comments_backends.py @@ -0,0 +1,888 @@ +#!/usr/bin/env python3 +"""Comment detection backends for scripts/check_no_code_comments.py. + +Detection-only language lexers: each backend returns every comment token it +finds in a file, with the classification decision applied separately by the +checker. Detection reuses the machine-law-validated lexers from the lane +comment-removal lever; string, template, heredoc, block-scalar, raw-string and +nesting rules are preserved verbatim from those proven implementations. +""" + +import html.parser +import io +import json +import os +import re +import subprocess +import sys +import tokenize +import token as token_mod + +TS_FAMILY = {".js", ".mjs", ".cjs", ".mts", ".ts", ".tsx", ".jsx"} + +SPECIAL_JSON = {".gitnexusrc"} +SPECIAL_MAKE = {"Makefile", "makefile"} +SPECIAL_IGNORE = {".gitignore", ".dockerignore", ".codexignore", ".gitnexusignore"} +SHELL_KEEP = re.compile(r"^\s*(shellcheck\b|shellcase|shellcheck-disable|vim:|@shlint)") +PY_DIRECTIVE = re.compile( + r"^\s*#\s*(type:\s|noqa\b|mypy:\s|pylint:\s|fmt:\s|ruff:\s|isort:\s|pragma:|coding[:=])") +GO_DIRECTIVE = re.compile( + r"^//\s*(go:[a-z][a-z0-9]*|line\s+\d|extern\s|export\s|#cgo\b)") +JS_DIRECTIVE = re.compile( + r"^(?://|/\*)\s*(@?eslint\b|eslint-\w[\w-]*|@ts-(ignore|expect-error|nocheck|check|strict)\b" + r"|@vitest-environment\s+\S+|@__PURE__|@vite-ignore|istanbul\b" + r"|EXTRACT-(?:START|END):\w+)") +DOCKERFILE_DIRECTIVE = re.compile(r"^#\s*(syntax|escape|check)\s*=") +GOMOD_INDIRECT = re.compile(r"^//\s*indirect\s*$") +GOVERNANCE_ATTRIBUTION = re.compile( + r"D-\d+[a-z]*\s+attribution\b.*coordinator ruling \d{4}-\d{2}-\d{2}") + + +def spans_to_comments(src, spans): + comments = [] + for (a, b) in spans: + comments.append({ + "line": src.count("\n", 0, a) + 1, + "kind": "block" if "\n" in src[a:b] or src[a:a + 2] in ("/*", "(*") else "line", + "text": src[a:b], + }) + return comments + + +def go_comment_spans(src): + spans = [] + i = 0 + n = len(src) + if src.startswith("#!"): + j = src.find("\n") + spans.append((0, j if j != -1 else n)) + i = spans[-1][1] + while i < n: + c = src[i] + if c == "/" and i + 1 < n and src[i + 1] == "/": + j = src.find("\n", i) + spans.append((i, j if j != -1 else n)) + i = spans[-1][1] + continue + if c == "/" and i + 1 < n and src[i + 1] == "*": + j = src.find("*/", i + 2) + if j == -1: + raise RuntimeError("unterminated /* block comment") + spans.append((i, j + 2)) + i = j + 2 + continue + if c == '"': + i += 1 + while i < n: + if src[i] == "\\": + i += 2 + continue + if src[i] == '"': + break + i += 1 + i += 1 + continue + if c == "`": + j = src.find("`", i + 1) + if j == -1: + raise RuntimeError("unterminated raw string") + i = j + 1 + continue + if c == "'": + i += 1 + while i < n: + if src[i] == "\\": + i += 2 + continue + if src[i] == "'": + break + i += 1 + i += 1 + continue + i += 1 + return spans + + +def sh_comment_spans(src): + spans = [] + i = 0 + n = len(src) + prev = "\n" + heredoc_delim = None + heredoc_dash = False + in_heredoc = False + + def read_line(pos): + j = src.find("\n", pos) + return (j + 1 if j != -1 else n), (src[pos:j] if j != -1 else src[pos:]) + + while i < n: + if in_heredoc: + line_end, line = read_line(i) + test = line.rstrip("\r") + if heredoc_dash: + test = test.lstrip("\t") + if test == heredoc_delim: + in_heredoc = False + heredoc_delim = None + i = line_end + continue + ch = src[i] + if ch == "\n": + i += 1 + prev = "\n" + if heredoc_delim is not None: + in_heredoc = True + continue + if ch == "'": + j = src.find("'", i + 1) + if j == -1: + raise RuntimeError("unterminated single quote") + i = j + 1 + prev = "'" + continue + if ch == '"': + j = i + 1 + while j < n: + if src[j] == "\\": + j += 2 + continue + if src[j] == '"': + break + if src[j] == "`": + k = j + 1 + while k < n and src[k] != "`": + k += 2 if src[k] == "\\" else 1 + j = k + j += 1 + i = j + 1 + prev = '"' + continue + if ch == "`": + j = i + 1 + while j < n and src[j] != "`": + j += 2 if src[j] == "\\" else 1 + i = j + 1 + prev = "`" + continue + if ch == "$" and i + 1 < n and src[i + 1] == "'": + j = i + 2 + while j < n: + if src[j] == "\\": + j += 2 + continue + if src[j] == "'": + break + j += 1 + i = j + 1 + prev = "'" + continue + if ch == "#": + if prev in set(" \t\n;|&("): + j = src.find("\n", i) + if j == -1: + j = n + spans.append((i, j)) + i = j + prev = "\n" + continue + if ch == "<" and i + 1 < n and src[i + 1] == "<": + m = re.match(r"<<(-?)[ \t]*(['\"]?)([A-Za-z_][A-Za-z0-9_]*)\2", src[i:]) + if m: + heredoc_dash = m.group(1) == "-" + heredoc_delim = m.group(3) + i += m.end() + prev = heredoc_delim[-1] + continue + if ch == "\\" and i + 1 < n and src[i + 1] == "\n": + i += 2 + continue + prev = ch + i += 1 + return spans + + +def yaml_comment_spans(src): + spans = [] + lines = src.splitlines(keepends=True) + block_indent = None + block_key_re = re.compile(r"^(\s*)(?:[^#]*?:\s*[|>][+-]?\d*)\s*(#.*)?$") + in_dq = False + for ln_i, line in enumerate(lines): + stripped = line.rstrip("\n") + off = sum(len(l) for l in lines[:ln_i]) + if block_indent is not None: + if stripped.strip() == "" or (len(stripped) - len(stripped.lstrip())) >= block_indent: + continue + block_indent = None + if in_dq: + j = 0 + closed = False + while j < len(stripped): + if stripped[j] == "\\": + j += 2 + continue + if stripped[j] == '"': + closed = True + j += 1 + break + j += 1 + if not closed: + continue + in_dq = False + stripped = stripped[j:] + off += j + m = block_key_re.match(stripped) + i = 0 + in_s = in_d2 = False + found = -1 + while i < len(stripped): + c = stripped[i] + if in_s: + if c == "'": + if i + 1 < len(stripped) and stripped[i + 1] == "'": + i += 1 + else: + in_s = False + elif in_d2: + if c == "\\": + i += 1 + elif c == '"': + in_d2 = False + else: + if c == "'": + in_s = True + elif c == '"': + in_d2 = True + elif c == "#" and (i == 0 or stripped[i - 1] in " \t"): + found = i + break + i += 1 + if found >= 0: + spans.append((off + found, off + len(stripped))) + if in_d2: + in_dq = True + if m and not in_s: + key_indent = len(m.group(1)) + block_indent = key_indent + 1 + return spans + + +def toml_comment_spans(src): + spans = [] + src_lines = src.splitlines(keepends=True) + in_ml_d = in_ml_s = False + for ln_i, line in enumerate(src_lines): + stripped = line.rstrip("\n") + off = sum(len(l) for l in src_lines[:ln_i]) + if in_ml_d: + if '"""' in stripped: + in_ml_d = False + continue + if in_ml_s: + if "'''" in stripped: + in_ml_s = False + continue + i = 0 + in_s = in_d = False + while i < len(stripped): + c = stripped[i] + if in_d: + if c == "\\": + i += 1 + elif c == '"': + if stripped[i:i + 3] == '"""': + in_ml_d = True + break + in_d = False + elif in_s: + if c == "'": + if stripped[i:i + 3] == "'''": + in_ml_s = True + break + in_s = False + else: + if c == '"': + if stripped[i:i + 3] == chr(34) * 3: + in_ml_d = True + break + in_d = True + elif c == "'": + if stripped[i:i + 3] == chr(39) * 3: + in_ml_s = True + break + in_s = True + elif c == "#" and (i == 0 or stripped[i - 1] in " \t"): + spans.append((off + i, off + len(stripped))) + break + i += 1 + return spans + + +def tla_comment_spans(src): + spans = [] + i = 0 + n = len(src) + while i < n: + c = src[i] + if c == '"': + i += 1 + while i < n: + if src[i] == "\\": + i += 2 + continue + if src[i] == '"': + if i + 1 < n and src[i + 1] == '"': + i += 2 + continue + break + i += 1 + i += 1 + continue + if src.startswith("\\*", i): + j = src.find("\n", i) + spans.append((i, j if j != -1 else n)) + i = spans[-1][1] + continue + if src.startswith("(*", i): + depth = 1 + j = i + 2 + while j < n and depth: + if src[j] == '"': + j += 1 + while j < n: + if src[j] == "\\": + j += 2 + continue + if src[j] == '"': + if j + 1 < n and src[j + 1] == '"': + j += 2 + continue + break + j += 1 + j += 1 + continue + if src.startswith("(*", j): + depth += 1 + j += 2 + elif src.startswith("*)", j): + depth -= 1 + j += 2 + else: + j += 1 + if depth != 0: + raise RuntimeError("unbalanced (* block comment") + spans.append((i, j)) + i = j + continue + if c == "%" and (i == 0 or src[i - 1] == "\n"): + j = src.find("\n", i) + spans.append((i, j if j != -1 else n)) + i = spans[-1][1] + continue + i += 1 + return spans + + +def graphql_comment_spans(src): + spans = [] + i = 0 + n = len(src) + while i < n: + if src.startswith('"""', i): + j = src.find('"""', i + 3) + i = (j + 3) if j != -1 else n + continue + if src[i] == '"': + i += 1 + while i < n: + if src[i] == "\\": + i += 2 + continue + if src[i] == '"': + break + i += 1 + i += 1 + continue + if src[i] == "#": + j = src.find("\n", i) + spans.append((i, j if j != -1 else n)) + i = spans[-1][1] + continue + i += 1 + return spans + + +def sql_comment_spans(src): + spans = [] + i = 0 + n = len(src) + while i < n: + c = src[i] + if c == "'" : + i += 1 + while i < n: + if src[i] == "'": + if i + 1 < n and src[i + 1] == "'": + i += 2 + continue + break + i += 1 + i += 1 + continue + if c == '"': + i += 1 + while i < n and src[i] != '"': + i += 1 + i += 1 + continue + m = re.match(r"\$[A-Za-z_0-9]*\$", src[i:]) + if m and (i == 0 or src[i - 1] in " \t\n("): + tag = m.group(0) + j = src.find(tag, i + len(tag)) + if j != -1: + i = j + len(tag) + continue + if src.startswith("--", i): + j = src.find("\n", i) + spans.append((i, j if j != -1 else n)) + i = spans[-1][1] + continue + if src.startswith("/*", i): + depth = 1 + j = i + 2 + while j < n and depth: + if src.startswith("/*", j): + depth += 1 + j += 2 + elif src.startswith("*/", j): + depth -= 1 + j += 2 + else: + j += 1 + if depth != 0: + raise RuntimeError("unterminated nested block comment") + spans.append((i, j)) + i = j + continue + i += 1 + return spans + + +def css_comment_spans(src): + spans = [] + i = 0 + n = len(src) + while i < n: + c = src[i] + if c == '"' or c == "'": + q = c + i += 1 + while i < n and src[i] != q: + i += 2 if src[i] == "\\" else 1 + i += 1 + continue + if src.startswith("/*", i): + j = src.find("*/", i + 2) + if j == -1: + raise RuntimeError("unterminated /* block comment") + spans.append((i, j + 2)) + i = j + 2 + continue + i += 1 + return spans + + +def tmpl_comment_spans(src): + spans = [] + i = 0 + n = len(src) + while i < n: + j = src.find("{{", i) + if j == -1: + break + k = src.find("}}", j + 2) + inner_start = j + 2 + inner_end = k if k != -1 else n + body = src[inner_start:inner_end] + body = body[1:] if body.startswith("-") else body + body = body[:-1] if body.endswith("-") else body + body = body.strip() + if body.startswith("/*") and body.endswith("*/"): + spans.append((j, inner_end if k != -1 else n)) + i = inner_end + continue + i = (k + 2) if k != -1 else n + return spans + + +def py_comment_spans(src): + spans = [] + try: + toks = list(tokenize.generate_tokens(io.StringIO(src).readline)) + except (tokenize.TokenError, IndentationError, SyntaxError) as exc: + raise RuntimeError(f"python tokenize failed: {exc}") from exc + for t in toks: + if t.type == token_mod.COMMENT: + (srow, scol), (erow, ecol) = t.start, t.end + a = sum(len(l) for l in src.splitlines(keepends=True)[:srow - 1]) + scol + b = sum(len(l) for l in src.splitlines(keepends=True)[:erow - 1]) + ecol + spans.append((a, b)) + return spans + + +def prefix_line_spans(src, prefixes): + spans = [] + off = 0 + for line in src.splitlines(keepends=True): + body = line.lstrip() + indent = len(line) - len(body) + if any(body.startswith(p) for p in prefixes): + spans.append((off + indent, off + len(line.rstrip("\n")))) + off += len(line) + return spans + + +def detect_json(path, src): + json.loads(src) + return [] + + +def detect_ndjson(path, src): + for line in src.splitlines(): + if line.strip(): + json.loads(line) + return [] + + +def detect_html(path, src): + found = [] + + class P(html.parser.HTMLParser): + def handle_comment(self, data): + found.append(data) + P().feed(src) + return [{"line": src.count("\n", 0, src.find("\n```python\n# code fence\n```\n") + s = self.scan(md) + self.assertEqual([], s["violations"]) + self.assertEqual([], s["allowed"]) + self.assertEqual([md], s["inventory"]["asset"]) + + def test_unsupported_extension_is_reported_not_dropped(self): + c = self.write("mystery.conf", "server {\n listen 80;\n}\n") + s = self.scan(c) + self.assertEqual([c], s["inventory"]["unsupported"]) + + def test_nginx_comments_are_detected(self): + c = self.write("nginx.conf", "server {\n # route api upstream\n listen 80;\n" + ' location "/a#b" { proxy_pass http://x; }\n}\n') + s = self.scan(c) + self.assertEqual(1, len(s["violations"])) + self.assertIn("route api", s["violations"][0]["text"]) + + def test_checksum_migration_payload_is_inventory_not_violation(self): + rel = "internal/backends/postgres/internal/store/migrations/000001_ledger.sql" + full = os.path.join(chk.REPO, rel) + s = self.scan(full) + self.assertEqual([], s["violations"]) + self.assertEqual([full], s["inventory"]["checksum-payload"]) + self.assertIn("--", open(full).read()) + + def test_non_migration_sql_is_still_scanned(self): + p = self.write("seed.sql", "-- handwritten seeding note\nSELECT 1; -- trailing\n") + s = self.scan(p) + self.assertEqual(2, len(s["violations"])) + self.assertEqual([], s["inventory"]["checksum-payload"]) + + def test_payload_binding_is_path_evidence_not_extension(self): + fake = self.write("migrations/000001_x.sql", "-- free-standing note\nSELECT 1;\n") + s = self.scan(fake) + self.assertEqual(1, len(s["violations"])) + self.assertEqual([], s["inventory"]["checksum-payload"]) + + def test_testdata_fixture_payload_is_inventory(self): + full = os.path.join(chk.REPO, "internal/backends/postgres/internal/store/" + "testdata/ab05_pre_refactor/internal/dataset/" + "postgres/testdata/migrations-releases/" + "000008_project_release.sql") + s = self.scan(full) + self.assertEqual([], s["violations"]) + self.assertEqual([full], s["inventory"]["fixture-payload"]) + + +@unittest.skipUnless(ts_available(), "typescript module not resolvable") +class TypeScriptFamily(Harness): + def test_jsx_and_script_prose_fail(self): + p = self.write("c.tsx", "export const C = () => (
hi{/* jsx note */}
);\n" + "// tail note\n") + s = self.scan(p) + self.assertEqual(2, len(s["violations"])) + + def test_vitest_environment_directive_passes(self): + p = self.write("u.unit.test.ts", "// @vitest-environment jsdom\nexport const t = 1;\n") + s = self.scan(p) + self.assertEqual([], s["violations"]) + self.assertEqual(["js-tool-directive"], [a["category"] for a in s["allowed"]]) + + def test_eslint_and_istanbul_directives_pass(self): + p = self.write("e.ts", "/* eslint-disable-next-line no-console */\n" + "console.log(1);\n/* istanbul ignore next */\n" + "const never = () => 2;\n") + self.assertEqual([], self.scan(p)["violations"]) + + def test_urls_regexes_templates_are_not_comments(self): + p = self.write("a.mjs", 'const u = "https://x//y#z";\n' + "const re = /a\\/\\/b/;\n" + "const t = `link https://a.example//p ${1} end`;\n" + "const t2 = `${ /r\\/\\/s/ }`;\n") + self.assertEqual([], self.scan(p)["violations"]) + + def test_string_decoys_are_not_comments(self): + p = self.write("d.js", 'const s = "/* not a comment */";\n' + 'const t2 = "{/* planted string decoy */}";\n') + self.assertEqual([], self.scan(p)["violations"]) + + def test_commented_out_code_fails(self): + p = self.write("o.ts", "function f() {\n // const a = 1;\n // return a;\n}\n") + self.assertEqual(2, len(self.scan(p)["violations"])) + + +class InventoryAndCli(Harness): + def test_parse_error_is_violation_not_skip(self): + p = self.write("broken.json", '{"a": ') + s = self.scan(p) + self.assertEqual(["parse"], [v["kind"] for v in s["violations"]]) + + def test_scanner_absence_routes_ts_family_to_unsupported_fail_closed(self): + p = self.write("f.mjs", "// handwritten prose note\nconst x = 1;\n") + original = b.detect_ts_family + def unavailable(*a, **k): + raise RuntimeError("ts scanner unavailable: typescript module not found") + b.detect_ts_family = unavailable + self.addCleanup(setattr, b, "detect_ts_family", original) + s = self.scan(p) + self.assertIn(p, s["inventory"]["unsupported"]) + self.assertEqual([], s["violations"]) + self.assertEqual([], s["allowed"]) + + def test_unsupported_is_reported_and_exit_code_nonzero(self): + c = self.write("mystery.zzz", "content\n") + s = self.scan(c) + self.assertEqual([c], s["inventory"]["unsupported"]) + argv = sys.argv + sys.argv = ["check_no_code_comments.py", c] + try: + self.assertEqual(1, chk.main()) + finally: + sys.argv = argv + + def test_self_compliance_of_owned_files(self): + files = [ + os.path.join(chk.REPO, "scripts/check_no_code_comments.py"), + os.path.join(chk.REPO, "scripts/lib/no_code_comments_backends.py"), + os.path.join(chk.REPO, "scripts/tests/test_no_code_comments.py"), + os.path.join(chk.REPO, ".github/workflows/tests.yaml"), + ] + for f in files: + self.assertTrue(os.path.isfile(f), f) + s = self.scan(*files) + self.assertEqual([], [(v["path"], v["text"]) for v in s["violations"]]) + + +class GovernanceAttribution(Harness): + ATTR = ("// D-16a attribution (mandatory, coordinator ruling 2026-11-11): CTE terminal re-sort " + "over GROUP-last per coordinator ruling 2026-11-11; SQL CTE order unspecified; " + "refines registered group total order.") + + def test_attribution_comment_allowed_on_any_path(self): + g = self.write("z.go", "package z\n\n" + self.ATTR + "\nvar X = 1\n") + s = self.scan(g) + self.assertEqual([], [(v["path"], v["text"]) for v in s["violations"]]) + self.assertIn("governance-directive", {r["category"] for r in s["allowed"]}) + + def test_forged_governance_without_directive_token_fails(self): + g = self.write("y.go", "package y\n\n// per coordinator ruling 2026-11-11 this prose is exempt\nvar X = 1\n") + self.assertEqual(1, len(self.scan(g)["violations"])) + + def test_forged_governance_without_dated_ruling_fails(self): + g = self.write("x.go", "package x\n\n// D-16a attribution (mandatory): free-form exemption prose\nvar X = 1\n") + self.assertEqual(1, len(self.scan(g)["violations"])) + + def test_attribution_tokens_on_different_lines_not_allowed(self): + g = self.write("w.go", "package w\n\n/*\nD-16a attribution\ncoordinator ruling 2026-11-11\n*/\nvar X = 1\n") + self.assertEqual(1, len(self.scan(g)["violations"])) + + def test_registered_path_pattern_scoped_to_registered_file(self): + body = "// CTE terminal re-sort over GROUP-last per coordinator ruling 2026-11-11\nvar X = 1\n" + a = self.write("stage_sequence.go", "package pg\n\n" + body) + chk.GOVERNANCE_DIRECTIVE_INDEX[a] = [ + {"pattern": __import__("re").compile(r"CTE terminal re-sort over GROUP-last per coordinator ruling"), + "tool": "probe", "purpose": "probe"}] + s_ok = self.scan(a) + self.assertEqual([], s_ok["violations"]) + c = self.write("unregistered.go", "package pg\n\n" + body) + s_bad = self.scan(c) + self.assertEqual(1, len(s_bad["violations"])) + + def test_repo_stage_sequence_test_passes_gate_with_restored_attribution(self): + p = "internal/dataframe/compiler/render/postgres/stage_sequence_test.go" + if not os.path.isfile(os.path.join(chk.REPO, p)): + self.skipTest("repo file absent in scratch checkout") + s = chk.scan([p]) + self.assertEqual([], [(v["path"], v["line"], v["text"]) for v in s["violations"]]) + self.assertIn("governance-directive", {r["category"] for r in s["allowed"]}) + + +class GuardQuotedAnchors(Harness): + def test_block_with_live_test_quote_allowed(self): + src = self.write("svc.go", "package svc\n\n" + "// ComputeDigest hashes a length-prefixed sequence rather than a\n" + "// delimiter-joined string so the encoding stays injective.\n" + "func ComputeDigest() string { return \"\" }\n") + tst = self.write("svc_test.go", "package svc\n\n" + "const anchor = \"ComputeDigest hashes a length-prefixed sequence rather than a\"\n") + s = self.scan(src, tst) + self.assertEqual([], [(v["path"], v["text"]) for v in s["violations"]]) + self.assertIn("guard-quoted", {r["category"] for r in s["allowed"]}) + + def test_unquoted_comment_still_violates(self): + src = self.write("a.go", "package a\n\n// nobody quotes this prose anywhere in the suite\nvar X = 1\n") + tst = self.write("a_test.go", "package a\n\nfunc TestY(t *testing.T) {}\n") + s = self.scan(src, tst) + self.assertEqual(1, len(s["violations"])) + + def test_short_quote_below_threshold_not_honored(self): + src = self.write("b.go", "package b\n\n// short needle here\nvar X = 1\n") + tst = self.write("b_test.go", "package b\n\nconst q = \"short needle here\"\n") + s = self.scan(src, tst) + self.assertEqual(1, len(s["violations"])) + + def test_quote_outside_test_files_not_honored(self): + src = self.write("c.go", "package c\n\n// this prose lives in a non test file copy\nvar X = 1\n") + other = self.write("d.go", "package d\n\nconst echo = \"this prose lives in a non test file copy\"\n") + s = self.scan(src, other) + self.assertEqual(1, len(s["violations"])) + + def test_self_quoting_test_file_does_not_guard_itself(self): + tst = self.write("e_test.go", "package e\n\n" + "// a comment in the test file that must stay red\n" + "const q = \"a comment in the test file that must stay red\"\n") + s = self.scan(tst) + self.assertEqual(1, len(s["violations"])) + + def test_inline_trailing_comment_with_live_quote_allowed(self): + src = self.write("f.go", "package f\n\nfunc g() bool {\n\tclosed := true // Prevent the runner from deleting possibly published tables.\n\treturn closed\n}\n") + tst = self.write("f_test.go", "package f\n\n" + "const anchor = \"Prevent the runner from deleting possibly published tables.\"\n") + s = self.scan(src, tst) + self.assertEqual([], [(v["path"], v["text"]) for v in s["violations"]]) + self.assertIn("guard-quoted", {r["category"] for r in s["allowed"]}) + + def test_guard_drops_when_quote_removed_from_test(self): + src = self.write("h.go", "package h\n\n// ReconcileDangle trusts only ledger-issued witness tokens today\nvar Y = 2\n") + tst = self.write("h_test.go", "package h\n\nfunc TestW(t *testing.T) {}\n") + s = self.scan(src, tst) + self.assertEqual(1, len(s["violations"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_recompute_reviewed_ledger.py b/scripts/tests/test_recompute_reviewed_ledger.py new file mode 100644 index 000000000..4cf573212 --- /dev/null +++ b/scripts/tests/test_recompute_reviewed_ledger.py @@ -0,0 +1,290 @@ +import contextlib +import hashlib +import io +import json +import os +import subprocess +import sys +import tempfile +import unittest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +import recompute_reviewed_ledger as rc + +LEDGER = "verification/publication-bundle/reviewed-sources.json" +WIDGET = "internal/widget/widget.go" +PROBE = "verification/publication-bundle/Probe.tla" +BLOB = "data/blob.bin" + + +def sh(cwd, *argv): + p = subprocess.run(argv, cwd=cwd, capture_output=True) + if p.returncode != 0: + raise AssertionError((argv, p.stderr.decode("utf-8", "replace"))) + return p.stdout.decode().strip() + + +def md5b(data): + return hashlib.md5(data).hexdigest() + + +def sha256b(data): + return hashlib.sha256(data).hexdigest() + + +class Fixture(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.repo = os.path.join(self._tmp.name, "repo") + os.makedirs(self.repo) + sh(self.repo, "git", "init", "-q", ".") + sh(self.repo, "git", "config", "user.email", "fixture@example.invalid") + sh(self.repo, "git", "config", "user.name", "Fixture") + self._orig = (rc.REPO, dict(rc.RELOCATIONS), dict(rc.RETIREMENTS)) + rc.REPO = self.repo + rc.RELOCATIONS.clear() + rc.RETIREMENTS.clear() + self.addCleanup(self._restore) + self.c1 = b"package widget\n// v1\n" + self.c2 = b"package widget\n// v2\n" + self.probe = b"---- MODULE Probe ----\n" + self.b1 = b"alpha-blob\n" + self.b2 = b"beta-blob\n" + self.write(WIDGET, self.c1) + self.write(PROBE, self.probe) + self.write(BLOB, self.b1) + self.commit1 = self.commit("c1") + self.write(WIDGET, self.c2) + self.write(BLOB, self.b2) + self.commit2 = self.commit("c2") + + def _restore(self): + rc.REPO, reloc, retir = self._orig + rc.RELOCATIONS.clear() + rc.RELOCATIONS.update(reloc) + rc.RETIREMENTS.clear() + rc.RETIREMENTS.update(retir) + + def write(self, rel, data): + path = os.path.join(self.repo, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as f: + f.write(data) + + def commit(self, msg): + sh(self.repo, "git", "add", "-A") + sh(self.repo, "git", "commit", "-qm", msg) + return sh(self.repo, "git", "rev-parse", "HEAD") + + def put_ledger(self, doc): + with open(os.path.join(self.repo, LEDGER), "w") as f: + f.write(json.dumps(doc, indent=2, ensure_ascii=False) + "\n") + + def read_ledger(self): + with open(os.path.join(self.repo, LEDGER)) as f: + return f.read() + + def doc(self): + return json.loads(self.read_ledger()) + + def run_main(self, *argv): + buf = io.StringIO() + old = sys.argv + sys.argv = ["recompute_reviewed_ledger.py", "--ledger", LEDGER, *argv] + try: + with contextlib.redirect_stdout(buf): + code = rc.main() + finally: + sys.argv = old + return code, buf.getvalue() + + def scan_rows(self, doc): + return rc.Checker(LEDGER, "HEAD").scan(doc) + + def test_apply_repins_mismatch_with_receipt_and_canonical_form(self): + self.put_ledger({"files": {WIDGET: {"md5": "0" * 32, + "sha256": sha256b(self.c2)}}}) + code, out = self.run_main("--apply", "--ticket", "FIXTURE") + self.assertEqual(code, 0) + self.assertIn("FIX re-pinned", out) + row = self.doc()["files"][WIDGET] + self.assertEqual(row["md5"], md5b(self.c2)) + self.assertEqual(row["sha256"], sha256b(self.c2)) + blk = row["pubbundleRepin"] + self.assertEqual(blk["ticket"], "FIXTURE") + self.assertEqual(blk["fields"]["md5"], + {"pre": "0" * 32, "post": md5b(self.c2)}) + self.assertEqual(blk.get("movedAt"), self.commit2[:8]) + text = self.read_ledger() + self.assertEqual(text, + json.dumps(json.loads(text), indent=2, + ensure_ascii=False) + "\n") + + def test_shape_anomaly_and_prose_value_gate(self): + doc = {"files": {WIDGET: {"md5": sha256b(self.c2)}, + PROBE: {"sha256": "value recorded by hand, not hex"}}} + self.put_ledger(doc) + classes = [r["class"] for r in self.scan_rows(doc)] + self.assertIn("SHAPE-ANOMALY", classes) + self.assertIn("PROSE-VALUE", classes) + code, out = self.run_main() + self.assertEqual(code, 1) + self.assertIn("GATED-FAIL", out) + + def test_repoint_merge_when_both_keys_exist(self): + rc.RELOCATIONS["legacy/widget.go"] = WIDGET + doc = {"files": {"legacy/widget.go": {"role": "old lane receipt"}, + WIDGET: {"md5": md5b(self.c2)}}} + self.put_ledger(doc) + code, out = self.run_main("--apply", "--ticket", "FIXTURE") + self.assertEqual(code, 0) + files = self.doc()["files"] + self.assertNotIn("legacy/widget.go", files) + self.assertIn("repoint merge note", files[WIDGET]["role"]) + self.assertIn("legacy/widget.go", files[WIDGET]["role"]) + self.assertIn("merged repointed", out) + + def test_repoint_moves_untracked_row_with_receipt(self): + rc.RELOCATIONS["legacy/widget.go"] = WIDGET + doc = {"files": {"legacy/widget.go": {"md5": md5b(self.c1)}}} + self.put_ledger(doc) + code, _ = self.run_main("--apply", "--ticket", "FIXTURE") + self.assertEqual(code, 0) + files = self.doc()["files"] + self.assertNotIn("legacy/widget.go", files) + self.assertEqual(files[WIDGET]["pubbundleRepin"]["repointFrom"], + "legacy/widget.go") + + def test_prose_string_row_upgraded_with_note(self): + prose = "audit r9 observed md5 " + "a" * 32 + " on disk" + self.put_ledger({"files": {WIDGET: prose}}) + code, out = self.run_main("--apply", "--ticket", "FIXTURE") + self.assertEqual(code, 0) + row = self.doc()["files"][WIDGET] + self.assertEqual(row["md5"], md5b(self.c2)) + self.assertEqual(row["note"], prose) + blk = row["pubbundleRepin"] + self.assertEqual(blk["fields"]["md5"]["pre"], ["a" * 32]) + self.assertEqual(blk["fields"]["md5"]["post"], md5b(self.c2)) + self.assertIn("upgraded prose row", out) + + def test_model_hex_tla_field_gates_and_repins(self): + doc = {"model": {"probe": {"file": PROBE, "tla": "f" * 32}}} + self.put_ledger(doc) + tla_rows = [r for r in self.scan_rows(doc) if r["field"] == "tla"] + self.assertEqual([r["class"] for r in tla_rows], ["MISMATCH"]) + code, _ = self.run_main() + self.assertEqual(code, 1) + code, out = self.run_main("--apply", "--ticket", "FIXTURE") + self.assertEqual(code, 0) + sec = self.doc()["model"]["probe"] + self.assertEqual(sec["tla"], md5b(self.probe)) + self.assertIn("tla", sec["pubbundleRepin"]["fields"]) + + def test_artifact_head_gates_without_evidence(self): + doc = {"confirmed": [{"id": "X", "sourceSnapshot": self.commit2, + "artifacts": [{"path": BLOB, + "sha256": sha256b(self.b1)}]}]} + self.put_ledger(doc) + code, out = self.run_main() + self.assertEqual(code, 1) + self.assertIn("GATED-FAIL", out) + self.assertIn("MISMATCH", out) + + def test_artifact_snapshot_and_evidence_stay_report_only(self): + old_claim = sha256b(self.b1) + doc = {"confirmed": [ + {"id": "S", "sourceSnapshot": self.commit1, + "artifacts": [{"path": BLOB, "sha256": old_claim}]}, + {"id": "V", "sourceSnapshot": self.commit2, + "artifacts": [{"path": BLOB, "sha256": old_claim, + "sha256Verification": + {"method": "file-history-blob", + "commit": self.commit1, + "queriedPath": BLOB}}]}, + {"id": "H", "artifacts": + [{"path": BLOB, "sha256": sha256b(self.b2)}]}]} + self.put_ledger(doc) + code, _ = self.run_main() + self.assertEqual(code, 0) + heads = [r for r in self.scan_rows(doc) if r["field"] == "sha256@head"] + self.assertEqual({r["class"] for r in heads}, + {"SNAPSHOT-DRIFT", "MATCH"}) + + def test_apply_noop_on_converged_artifact_verifications(self): + doc = {"confirmed": [ + {"id": "S", "sourceSnapshot": self.commit1, + "artifacts": [ + {"path": BLOB, "sha256": sha256b(self.b2), + "sha256Verification": {"method": "head-blob", + "commit": self.commit2[:8], + "queriedPath": BLOB}}, + {"path": BLOB, "sha256": sha256b(self.b1), + "sha256Verification": {"method": "file-history-blob", + "commit": self.commit1, + "matchingCommits": + [self.commit1[:8]], + "queriedPath": BLOB}}]}]} + self.put_ledger(doc) + before = self.read_ledger() + code, out = self.run_main("--apply", "--ticket", "FIXTURE") + self.assertEqual(code, 0) + self.assertEqual(self.read_ledger(), before) + self.assertNotIn("verified via", out) + self.assertIn("no rewrite", out) + code, _ = self.run_main("--apply", "--ticket", "FIXTURE") + self.assertEqual(code, 0) + self.assertEqual(self.read_ledger(), before) + + def test_stale_artifact_verification_is_rewritten(self): + doc = {"confirmed": [ + {"id": "X", "sourceSnapshot": self.commit2, + "artifacts": [{"path": BLOB, "sha256": sha256b(self.b2), + "sha256Verification": + {"method": "file-history-blob", + "commit": self.commit1, + "queriedPath": BLOB}}]}]} + self.put_ledger(doc) + before = self.read_ledger() + code, out = self.run_main("--apply", "--ticket", "FIXTURE") + self.assertEqual(code, 0) + self.assertNotEqual(self.read_ledger(), before) + self.assertIn("claim verified via", out) + ver = (self.doc()["confirmed"][0]["artifacts"][0] + ["sha256Verification"]) + self.assertEqual(ver["method"], "sourceSnapshot-blob") + self.assertEqual(ver["commit"], self.commit2) + + def test_superseded_quote_reported_then_stamped_idempotent(self): + quote = ("r9 SELF-RUN: live md5 == git show " + f"{self.commit1[:8]}:{WIDGET} == on-disk") + live = (f"live md5 == git show {self.commit1[:8]}:{PROBE} == on-disk") + doc = {"files": { + WIDGET: {"md5": md5b(self.c2), "rederived": quote}, + PROBE: {"md5": md5b(self.probe), "rederived": live}}} + self.put_ledger(doc) + rows = self.scan_rows(doc) + sup = [r for r in rows if r["class"] == "SUPERSEDED-PROSE"] + self.assertEqual([r["key"] for r in sup], [WIDGET]) + code, _ = self.run_main() + self.assertEqual(code, 0) + code, out = self.run_main("--apply", "--ticket", "FIXTURE") + self.assertEqual(code, 0) + self.assertIn("stamped superseded quote", out) + stamp = (self.doc()["files"][WIDGET]["pubbundleRepin"] + ["rederivedSuperseded"]["rederived"]) + self.assertEqual(stamp["quotes"][0]["commit"], self.commit1[:8]) + self.assertEqual(stamp["pinnedClaims"]["md5"], md5b(self.c2)) + rows = self.scan_rows(self.doc()) + self.assertEqual([r for r in rows + if r["class"] == "SUPERSEDED-PROSE"], []) + code, out = self.run_main("--apply", "--ticket", "FIXTURE") + self.assertEqual(code, 0) + self.assertNotIn("stamped superseded", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_ui_ci_workflow.py b/scripts/tests/test_ui_ci_workflow.py index 11b97648e..afc9fef31 100644 --- a/scripts/tests/test_ui_ci_workflow.py +++ b/scripts/tests/test_ui_ci_workflow.py @@ -63,7 +63,6 @@ def assert_formal_coverage(self, source: str) -> None: self.assertIn("if: always()", proof_job) self.assertIn("name: formal-source-proofs-${{ matrix.shard }}", proof_job) self.assertIn("path: /tmp/loom-source-proofs-${{ matrix.shard }}", proof_job) - # This is the CLI's selectShard predicate, applied to the actual registry. assignments = [[proof["index"] for proof in self.proofs if proof["index"] % count == shard - 1] for shard in shards] self.assertTrue(self.proofs) @@ -101,7 +100,6 @@ def test_ci_runs_complete_suites_with_ten_minute_budgets(self) -> None: workflow = WORKFLOW.read_text(encoding="utf-8") formal = FORMAL_WORKFLOW.read_text(encoding="utf-8") - # Complete coverage may span disjoint formal source-proof shards. for source in (workflow, formal): jobs = re.findall(r"^ [a-z][\w-]*:\n(.*?)(?=^ [a-z][\w-]*:\n|\Z)", source.split("jobs:\n", 1)[1], re.M | re.S) self.assertTrue(jobs) diff --git a/scripts/verify-builder-formal-core.mjs b/scripts/verify-builder-formal-core.mjs index ecf200fe7..8c7917120 100644 --- a/scripts/verify-builder-formal-core.mjs +++ b/scripts/verify-builder-formal-core.mjs @@ -54,14 +54,8 @@ const emit = async (value) => new Promise((resolve, reject) => process.stdout.wr if (options.phase === 'inventory') { assert.ok(!options.feature && !options.pattern && !options.list, 'Inventory is a read-only command'); const { verificationInventory } = await import('./lib/verification-inventory.mjs'); - // A broken retained dependency closure rejects here, so the phase fails - // nonzero instead of emitting a reduced inventory. const inventory = await verificationInventory(options.projectRoot ? resolve(options.projectRoot) : root); if (options.out) { - // Resolve the output directory like every other --out path, then write a - // unique temp file in that same directory and rename it into place, so an - // interrupted or ENOSPC write can never leave a truncated document for the - // if-always upload step to publish. Same directory keeps the rename atomic. const outDirectory = resolve(options.out); mkdirSync(outDirectory, { recursive: true }); const target = join(outDirectory, 'verification-inventory.json'); diff --git a/scripts/verify-builder-traces.mjs b/scripts/verify-builder-traces.mjs index 1b22f380e..c8db79faf 100644 --- a/scripts/verify-builder-traces.mjs +++ b/scripts/verify-builder-traces.mjs @@ -2977,7 +2977,6 @@ function allowedEvents(contract) { function normalizedActionLabel(model, label, actions) { if (actions.has(label)) return label; - // BuilderPreview's DiscoveryComplete action records its outcome in lastAction. if (model === 'BuilderPreview' && /^DiscoveryComplete\(/.test(label) && actions.has('DiscoveryComplete')) { return 'DiscoveryComplete'; } @@ -5342,13 +5341,11 @@ function generateTraceModule(trace, moduleName, hasGroupLifecycle, journeyCase = if (journeyCase) { return { moduleText: generateCdaTraceModule(trace, moduleName, journeyCase), - // Cda TraceActions is exactly the JS actions sequence, indexed from zero. traceIndexOffset: 0, }; } return { moduleText: generateModule(trace, moduleName, hasGroupLifecycle), - // The generic TraceActions sequence prepends its "Init" sentinel. traceIndexOffset: 1, }; } diff --git a/scripts/verify-cda-mixed-scope-selectors.mjs b/scripts/verify-cda-mixed-scope-selectors.mjs index 912d37dc1..373cadc0c 100644 --- a/scripts/verify-cda-mixed-scope-selectors.mjs +++ b/scripts/verify-cda-mixed-scope-selectors.mjs @@ -5,7 +5,6 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { parseArgs } from 'node:util'; -// Public recipe regression. Builder does not currently author field fallbacks. const { values } = parseArgs({ options: { 'api-origin': { type: 'string', default: 'http://127.0.0.1:8188' }, project: { type: 'string', default: 'loom_dev_cda_fhir' }, @@ -57,8 +56,6 @@ try { } assert(witnesses.length, 'No bounded nested coding witness with distinct component values'); report.oracle = { project: values.project, generation: values.generation, scanned: scanned.length, witnesses }; - // Persist exact source membership rather than scanning the full CDA population - // through a post-projection recipe filter. const explorer = `cda-mixed-scope-${Date.now()}-${randomUUID().slice(0, 8)}`; const root = `/api/v1/projects/${encodeURIComponent(values.project)}/explorers`; const rest = async (path, body) => { @@ -111,8 +108,6 @@ try { const normalize = rows => rows.map(({ observation_id, coding_code, mixed_values, mixed_first, mixed_distinct }) => { assert(Array.isArray(mixed_distinct), 'DISTINCT must return an array'); assert.equal(new Set(mixed_distinct).size, mixed_distinct.length, 'DISTINCT must not repeat values'); - // Arango sorts strings with its own collation. Compare distinct membership - // independently of that ordering; ALL and FIRST retain exact order checks. return { observation_id, coding_code, mixed_values, mixed_first, mixed_distinct: [...mixed_distinct].sort() }; }).sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); for (const phase of ['initial', 'persisted-revision-repeat']) { diff --git a/scripts/verify-construction-removals.mjs b/scripts/verify-construction-removals.mjs index fb952454f..ead7825a1 100644 --- a/scripts/verify-construction-removals.mjs +++ b/scripts/verify-construction-removals.mjs @@ -36,7 +36,7 @@ try { const steps = document.construction.steps; if (values['expect-cascade-step']) assert(steps.some(step => step.id === values['expect-cascade-step']), 'Expected cascade step is absent from the saved construction'); for (const [index, step] of steps.entries()) { - if (step.ownerStepId) continue; // Owned prerequisites are edited through their visible owner. + if (step.ownerStepId) continue; for (const mode of ['single', 'suffix']) { if (mode === 'suffix' && index === steps.length - 1) continue; const removed = mode === 'single' ? [step.id] : steps.slice(index).map(item => item.id); diff --git a/scripts/verify-ui/contracts/source-projection-contracts.mjs b/scripts/verify-ui/contracts/source-projection-contracts.mjs index c725ecb16..6a1e77d09 100644 --- a/scripts/verify-ui/contracts/source-projection-contracts.mjs +++ b/scripts/verify-ui/contracts/source-projection-contracts.mjs @@ -1,9 +1,3 @@ -// Source-projection owner-branch contract: durable Node-side assertions for -// the FORMAL-62-U1 replay family (construction_projection_replay_test.go + -// scripts/lib/formal-projection-replay.mjs). Round 61 §1 recorded that the -// reject direction carried zero Go/Node assertions before this contract; it -// now shares one expectation table with the lane, so a reject string renamed, -// removed, or swapped in production fails here with the branch named. import assert from 'node:assert/strict'; import { projectionBranchExpectations } from '../../lib/formal-projection-replay.mjs'; diff --git a/scripts/verify-ui/helpers/api-build-freeze.mjs b/scripts/verify-ui/helpers/api-build-freeze.mjs index 5c39ec13b..c0e214fd8 100644 --- a/scripts/verify-ui/helpers/api-build-freeze.mjs +++ b/scripts/verify-ui/helpers/api-build-freeze.mjs @@ -86,11 +86,6 @@ async function observe(readStamp) { } } -/** - * Capture the result of a caller-provided running-API build-stamp check. - * The callback returns { status, stdout, signal?, errorCode?, failureKind?, diagnostic? } - * and stays injectable so portable tests and non-Docker deployments need no Docker tool. - */ export async function captureApiBuildFreeze(readStamp) { if (typeof readStamp !== 'function') throw new TypeError('readStamp must be a function'); const before = await observe(readStamp); @@ -112,16 +107,10 @@ export async function captureApiBuildFreeze(readStamp) { }); } -/** Return the local CDA API container name without starting or changing it. */ export function localCDAApiContainer(env = process.env) { return env.LOOM_API_CONTAINER?.trim() || defaultLocalCDAApiContainer; } -/** - * Read the existing dev build stamp from one explicitly selected container. - * This is an optional Docker adapter; callers may instead inject any check - * into captureApiBuildFreeze. - */ export function checkContainerApiBuildStamp(container, { execFileImpl = execFile, timeoutMs = 10000, diff --git a/scripts/verify-ui/helpers/basic-add-column-coverage-oracle.mjs b/scripts/verify-ui/helpers/basic-add-column-coverage-oracle.mjs index b8e2bc85b..cb7c8f0e4 100644 --- a/scripts/verify-ui/helpers/basic-add-column-coverage-oracle.mjs +++ b/scripts/verify-ui/helpers/basic-add-column-coverage-oracle.mjs @@ -52,11 +52,6 @@ const expectedPatient = Object.freeze({ const MAX_FIXTURE_BYTES = 4096; -/** - * Reads the exact three-row Add Columns fixture and returns a raw-source oracle. - * The row states retain the missing/null distinction for the oracle; populated - * counts and examples describe only values the native UI can display. - */ export function readBasicAddColumnCoverageOracle({ fixtureDir = BASIC_ADD_COLUMN_COVERAGE_FIXTURE_DIR, } = {}) { diff --git a/scripts/verify-ui/helpers/builder-combine-draft-helpers.mjs b/scripts/verify-ui/helpers/builder-combine-draft-helpers.mjs index 7320066c6..7266b3be6 100644 --- a/scripts/verify-ui/helpers/builder-combine-draft-helpers.mjs +++ b/scripts/verify-ui/helpers/builder-combine-draft-helpers.mjs @@ -53,8 +53,6 @@ const normalizeConstructionOutputNullability = (construction) => { }; }; -// Go omits false StageColumn.Nullable values on the response wire. Compare the -// complete construction after only that one default-value normalization. export const constructionCandidateWireEquivalent = (request, response) => isDeepStrictEqual( normalizeConstructionOutputNullability(request), diff --git a/scripts/verify-ui/helpers/cda-nullable-code-join-oracle.mjs b/scripts/verify-ui/helpers/cda-nullable-code-join-oracle.mjs index 23fbfa90c..1c02a0354 100644 --- a/scripts/verify-ui/helpers/cda-nullable-code-join-oracle.mjs +++ b/scripts/verify-ui/helpers/cda-nullable-code-join-oracle.mjs @@ -176,7 +176,7 @@ export const cdaNullableEmptyRemovalPreviewEvidence = ({ proposal, outputId, tar const snapshotToken = currentDraft?.catalog?.snapshotToken; const expectedRoute = `/api/v1/projects/${encodeURIComponent(project ?? '')}/explorers/${encodeURIComponent(explorer ?? '')}/authoring/v2/construction-proposals`; let expectedOrigin = null; - try { expectedOrigin = new URL(uiOrigin).origin; } catch { /* invalid scope fails below */ } + try { expectedOrigin = new URL(uiOrigin).origin; } catch { } const readyResponse = response?.previewStatus === 'READY'; const createdCatalog = targetCreateBase?.catalog; const createdTargets = (targetCreateBase?.workspace?.documents ?? []).filter(document => document?.output?.id === outputId); diff --git a/scripts/verify-ui/helpers/cda-quantity-category-oracle-runner.mjs b/scripts/verify-ui/helpers/cda-quantity-category-oracle-runner.mjs index 311fd9c75..99ed1e04e 100644 --- a/scripts/verify-ui/helpers/cda-quantity-category-oracle-runner.mjs +++ b/scripts/verify-ui/helpers/cda-quantity-category-oracle-runner.mjs @@ -6,25 +6,6 @@ import { const RESULT_MARKER = '__LOOM_CDA_QUANTITY_ORACLE_RESULT__'; -/** - * Run the independent raw-Arango quantity-category oracle for a local CDA - * browser verification. - * - * The permanent Builder verifier runs against the local Compose API started - * with `--no-auth` (see `compose.yaml`). That path has a nil ScopeResolver, - * which source resolves to `ReadScopeUnrestricted`; therefore this runner - * accepts only the corresponding explicit allow-mode binds. It deliberately - * does not treat arbitrary resource-path arrays as proof of a restricted - * caller's effective Fence scope. For a restricted/authenticated deployment, - * obtain its resolved scope through a separately verified authorization path - * before building an oracle query. - * - * @param {{project:string,dataset_generation:string,auth_resource_paths:string[],auth_resource_paths_unrestricted:boolean,scope_allowed:true}} scopeBinds - * Exact project and immutable generation from the selected CDA source, plus - * the local no-auth allow-mode scope. All fields are required. - * @param {{container?:string,database?:string,dockerCommand?:string[],timeoutMs?:number,maxRuntime?:number}} [options] - * @returns {{durationMs:number,oracle:object}} - */ export function runCDAQuantityCategoryOracle(scopeBinds, options = {}) { const { query, bindVars } = buildCDAQuantityCategoryOracle(scopeBinds); if (bindVars.auth_resource_paths_unrestricted !== true || bindVars.auth_resource_paths.length !== 0) { @@ -44,13 +25,6 @@ export function runCDAQuantityCategoryOracle(scopeBinds, options = {}) { if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) throw new Error('timeoutMs must be a positive integer'); if (!Number.isFinite(maxRuntime) || maxRuntime <= 0) throw new Error('maxRuntime must be a positive number of seconds'); - // Arangosh canonicalizes --javascript.execute paths and rejects /dev/stdin - // in the container image. Feed the program through Docker stdin into a - // uniquely named, private script file and remove it from an EXIT trap. The - // shell text is fixed; database and file path are positional arguments, and - // AQL/binds are JSON literals rather than shell text. This is only a - // temporary execution script; oracle results return over stdout and do not - // depend on any /tmp capture file. const script = `try { const rows = db._query(${JSON.stringify(query)}, ${JSON.stringify(bindVars)}, { maxRuntime: ${maxRuntime} }).toArray(); print(${JSON.stringify(RESULT_MARKER)} + JSON.stringify({ ok: true, rows })); diff --git a/scripts/verify-ui/helpers/cda-quantity-category-oracle.mjs b/scripts/verify-ui/helpers/cda-quantity-category-oracle.mjs index 0f1cac77d..f1e7c7e1f 100644 --- a/scripts/verify-ui/helpers/cda-quantity-category-oracle.mjs +++ b/scripts/verify-ui/helpers/cda-quantity-category-oracle.mjs @@ -1,5 +1,3 @@ -// Independent raw-storage oracle for the CDA Specimen -> Patient -> Observation route. -// Enumerate all category values; keep one source witness per value instead of all route rows. export function buildCDAQuantityCategoryOracle(scopeBinds) { if(!scopeBinds || typeof scopeBinds!=='object' || Array.isArray(scopeBinds)) throw new Error('Explicit CDA quantity oracle scope binds are required'); if(scopeBinds.scope_allowed!==true) throw new Error('This oracle requires allow-mode scope'); diff --git a/scripts/verify-ui/helpers/cda-row-oracle.mjs b/scripts/verify-ui/helpers/cda-row-oracle.mjs index 8e41af0e1..3fb99eedb 100644 --- a/scripts/verify-ui/helpers/cda-row-oracle.mjs +++ b/scripts/verify-ui/helpers/cda-row-oracle.mjs @@ -30,7 +30,6 @@ const completePopulationRowKey = row => JSON.stringify(row.map((value) => Array. ? [...value].sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right))) : value)); -/** Compare complete row populations independent of row order and ALL cell item order. */ export function assertCompletePopulationRowsMatchOracle(actualRows, oracleRows, { label = 'complete row population' } = {}) { assert(Array.isArray(actualRows), `${label}: browser result must be an array`); assert(Array.isArray(oracleRows), `${label}: independent oracle must be an array`); diff --git a/scripts/verify-ui/helpers/construction-capabilities-readiness.mjs b/scripts/verify-ui/helpers/construction-capabilities-readiness.mjs index 5eb532026..63cb6e415 100644 --- a/scripts/verify-ui/helpers/construction-capabilities-readiness.mjs +++ b/scripts/verify-ui/helpers/construction-capabilities-readiness.mjs @@ -57,7 +57,6 @@ const validateScope = ({ page, project, explorer, apiOrigin }) => { return { page, project, explorer, origin }; }; -/** Extract the one output's authoritative saved snapshot/draft identity from a Builder response. */ export const builderCapabilitiesIdentityFromState = (state, { outputId } = {}) => { if (!state || typeof state !== 'object' || typeof outputId !== 'string' || !outputId) { throw new TypeError('Saved Builder identity requires a response object and exact output ID.'); @@ -80,7 +79,6 @@ export const builderCapabilitiesIdentityFromState = (state, { outputId } = {}) = return identity; }; -/** Observe only the exact post-Apply capabilities request and its Playwright response/terminal events. */ export const watchConstructionCapabilitiesReadiness = (scopeInput) => { const scope = validateScope(scopeInput); const recordsByRequest = new Map(); @@ -198,7 +196,6 @@ export const watchConstructionCapabilitiesReadiness = (scopeInput) => { }; }; -/** Read the authoritative saved draft identity used to match a browser capabilities request. */ export const readBuilderCapabilitiesIdentity = async ({ apiOrigin, project, explorer, outputId, timeoutMs, fetchImpl = globalThis.fetch, } = {}) => { diff --git a/scripts/verify-ui/helpers/coverage-status.mjs b/scripts/verify-ui/helpers/coverage-status.mjs index c0c34ac1e..04af29189 100644 --- a/scripts/verify-ui/helpers/coverage-status.mjs +++ b/scripts/verify-ui/helpers/coverage-status.mjs @@ -787,9 +787,6 @@ export const currentCoverageBaseline = ({ cwd = process.cwd(), env = process.env try { currentSourceFingerprint = sourceFingerprint(cwd); } catch {} return { sourceFingerprint: currentSourceFingerprint, - // The running API identity is intentionally opt-in. Reports need the exact - // three-part digest emitted by loom-dev-build-stamp.sh --check; an absent - // value leaves build freshness unknown instead of guessing from a freeze. apiBuildIdentity: normalizeApiBuildIdentity(env.LOOM_VERIFY_UI_API_BUILD_IDENTITY), }; }; diff --git a/scripts/verify-ui/helpers/focused-check-groups.mjs b/scripts/verify-ui/helpers/focused-check-groups.mjs index 1e52ecd6d..5020f3620 100644 --- a/scripts/verify-ui/helpers/focused-check-groups.mjs +++ b/scripts/verify-ui/helpers/focused-check-groups.mjs @@ -69,10 +69,6 @@ function parseCommand(command, location) { return { runner, fileOffset: 4 }; } -/** - * Validate command-array metadata and create argv-safe process plans. - * This function performs no filesystem access and starts no processes. - */ export function planFocusedCheckGroups(groups, repoRoot) { if (!Array.isArray(groups)) fail('groups', 'expected an array'); if (typeof repoRoot !== 'string' || !path.isAbsolute(repoRoot)) { diff --git a/scripts/verify-ui/helpers/inventory.mjs b/scripts/verify-ui/helpers/inventory.mjs index 39dcc1778..336341fd8 100644 --- a/scripts/verify-ui/helpers/inventory.mjs +++ b/scripts/verify-ui/helpers/inventory.mjs @@ -162,9 +162,6 @@ const scanCss = (path) => { return records; }; -// useQuery and useMutation are the framework primitives that domain hooks wrap. -// Their call sites stay in the inventory, but registering the primitive itself -// would not identify the data dependency a scenario owns. const frameworkHooks = new Set(['useQuery', 'useMutation']); export const inventoryCoverageDrift = (records, scenarios = registry) => { diff --git a/scripts/verify-ui/helpers/native-browser-api-scope.mjs b/scripts/verify-ui/helpers/native-browser-api-scope.mjs index 1e70e39c7..52e77124e 100644 --- a/scripts/verify-ui/helpers/native-browser-api-scope.mjs +++ b/scripts/verify-ui/helpers/native-browser-api-scope.mjs @@ -2,7 +2,6 @@ import { fixtureProjectExplorerCollectionPath } from './fixture-context.mjs'; const belongsToPath = (pathname, prefix) => pathname === prefix || pathname.startsWith(`${prefix}/`); -/** Classify browser API traffic against the local UI proxy and one owned Explorer. */ export const classifyNativeBrowserApiRequest = (requestUrl, { uiOrigin, apiOrigin, @@ -32,7 +31,6 @@ export const classifyNativeBrowserApiRequest = (requestUrl, { return { kind: 'reject', reason: 'outside-project-explorer-scope', url }; }; -/** Require the proxied response to remain bound to the captured UI request URL. */ export const isSameUiProxyResponse = (requestUrl, responseUrl, options) => { let request; let response; @@ -49,7 +47,6 @@ export const isSameUiProxyResponse = (requestUrl, responseUrl, options) => { request.url.search === response.url.search; }; -/** Accept a bound response or a fully classified owner cancellation before response headers. */ export const nativeRequestsHaveOwnedTransportOutcomes = (entries, uiOrigin) => { if (!Array.isArray(entries) || entries.length === 0 || typeof uiOrigin !== 'string') return false; return entries.every((entry) => { diff --git a/scripts/verify-ui/helpers/nested-authored-expand-oracle.mjs b/scripts/verify-ui/helpers/nested-authored-expand-oracle.mjs index e2f3b1e12..2b05ec572 100644 --- a/scripts/verify-ui/helpers/nested-authored-expand-oracle.mjs +++ b/scripts/verify-ui/helpers/nested-authored-expand-oracle.mjs @@ -142,12 +142,6 @@ const hasDuplicateCoding = record => { return false; }; -/** - * Select a bounded raw Observation pair for one explicit nested-list shape. - * The default requires a component with multiple codings; the separate - * single-coding-per-component mode proves the flattened outer component list. - * The empty-parent row is selected when available and otherwise stays a gap. - */ export function selectNestedAuthoredExpandWitnesses(rows, { project, generation, diff --git a/scripts/verify-ui/helpers/nullable-related-all.mjs b/scripts/verify-ui/helpers/nullable-related-all.mjs index a8d32ace2..90f41c1dc 100644 --- a/scripts/verify-ui/helpers/nullable-related-all.mjs +++ b/scripts/verify-ui/helpers/nullable-related-all.mjs @@ -23,7 +23,5 @@ export function expectedNullableRelatedAll({ relatedCount, sourceState }) { if (sourceState !== 'missing' && sourceState !== 'explicit-null') { throw new TypeError('The gender-null ALL mode accepts only a missing or explicit-null source value.'); } - // ALL retains one member per linked owner. A missing optional scalar reads as - // null in the projection, while an empty owner population has no members. return [null]; } diff --git a/scripts/verify-ui/helpers/owned-cda-target-config.mjs b/scripts/verify-ui/helpers/owned-cda-target-config.mjs index 82fb80ed2..b59cbd200 100644 --- a/scripts/verify-ui/helpers/owned-cda-target-config.mjs +++ b/scripts/verify-ui/helpers/owned-cda-target-config.mjs @@ -88,10 +88,6 @@ function validateEnvironmentOverrides(env, expected) { } } -/** - * Load one explicitly selected machine-local CDA target config and bind it to - * the selected case's independent registry identity before Docker/browser work. - */ export async function loadOwnedCdaTargetConfig({ targetPath, repositoryRoot, diff --git a/scripts/verify-ui/helpers/owned-cda-target.mjs b/scripts/verify-ui/helpers/owned-cda-target.mjs index e6bfaaf71..b01b352f5 100644 --- a/scripts/verify-ui/helpers/owned-cda-target.mjs +++ b/scripts/verify-ui/helpers/owned-cda-target.mjs @@ -30,7 +30,6 @@ function inspectVolumes(names, docker) { return names.length ? JSON.parse(docker(['volume', 'inspect', ...names])) : []; } -/** Validate that browser/API calls and raw source oracles resolve to one isolated Compose project. */ export async function assertOwnedCdaTarget({ project, apiOrigin, diff --git a/scripts/verify-ui/helpers/pivot-keyword-fixture-oracle.mjs b/scripts/verify-ui/helpers/pivot-keyword-fixture-oracle.mjs index fbec6d6cb..538a6b5d7 100644 --- a/scripts/verify-ui/helpers/pivot-keyword-fixture-oracle.mjs +++ b/scripts/verify-ui/helpers/pivot-keyword-fixture-oracle.mjs @@ -1,16 +1,3 @@ -// Fixture-internal expectations for the keyword-named pivot journey -// (verify-explorer-pivot-keyword-names.mjs). Authority: -// testdata/devloop-fixture/Observation.ndjson ONLY — six rows, one distinct -// status group (null), categories subject.reference (two patients) plus the -// MISSING sentinel, summed over valueQuantity.value. Recomputed from the -// fixture bytes at runtime; the literals below pin the last audited values -// so silent fixture edits fail loudly (same convention as -// basic-unpivot-pivot-fixture-oracle.mjs). -// NOT live-stack ground truth: the served Arango dataset diverges (its status -// values include 'final', per the 2026-11-03 manual-pivot probe), so the -// journey asserts keyword rendering and non-zero numeric cells there, never -// the fixture absolutes below. The keyword pair is the contract this helper -// exists to pin: AQL reserved words typed verbatim through the UI. import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; diff --git a/scripts/verify-ui/helpers/playwright-observations.mjs b/scripts/verify-ui/helpers/playwright-observations.mjs index 34ae0c12f..ab9f1d3eb 100644 --- a/scripts/verify-ui/helpers/playwright-observations.mjs +++ b/scripts/verify-ui/helpers/playwright-observations.mjs @@ -1,4 +1,3 @@ -/** Wait for observable DOM state using a Playwright callback and serializable condition data. */ export function waitForCondition(page, condition, timeout = 30000) { return page.waitForFunction((current) => { const element = selector => document.querySelector(selector); diff --git a/scripts/verify-ui/helpers/playwright-preview-rows.mjs b/scripts/verify-ui/helpers/playwright-preview-rows.mjs index e60089d2d..c5e5fd2aa 100644 --- a/scripts/verify-ui/helpers/playwright-preview-rows.mjs +++ b/scripts/verify-ui/helpers/playwright-preview-rows.mjs @@ -8,7 +8,6 @@ export function previewHeaderIndex(headers, label) { return matches[0]; } -/** Read every rendered preview row while scrolling with Playwright's native wheel input. */ export async function collectPreviewRows(page, { containerSelector = '[data-testid="preview-table-scroll"]', tableSelector = '[role="table"]', diff --git a/scripts/verify-ui/helpers/policy-one-trace.mjs b/scripts/verify-ui/helpers/policy-one-trace.mjs index 7d430590c..39a78f606 100644 --- a/scripts/verify-ui/helpers/policy-one-trace.mjs +++ b/scripts/verify-ui/helpers/policy-one-trace.mjs @@ -1313,7 +1313,6 @@ function traceActionRecords(context) { return actions; } -/** Validate captured choice evidence and insert its model observations into the canonical journey trace. */ export function buildPolicyOneTrace({ expected, requests, responses, saved, dom, fixtureIDs, context }) { validateExpected(expected, fixtureIDs); record(requests, 'request captures'); diff --git a/scripts/verify-ui/helpers/policy-one-transport-observer.mjs b/scripts/verify-ui/helpers/policy-one-transport-observer.mjs index 9647260da..ef9301bf8 100644 --- a/scripts/verify-ui/helpers/policy-one-transport-observer.mjs +++ b/scripts/verify-ui/helpers/policy-one-transport-observer.mjs @@ -13,7 +13,6 @@ const waitForPromises = (promises, timeoutMs) => new Promise((resolve, reject) = ]).then(resolve, reject).finally(() => clearTimeout(timer)); }); -/** Capture Policy One's complete setup and formal authoring transport span by exact Playwright Request identity. */ export const createPolicyOneTransportObserver = (page, { authoringPrefix = AUTHORING_PREFIX } = {}) => { const captures = []; const entriesByRequest = new WeakMap(); diff --git a/scripts/verify-ui/helpers/population-member-removal-proposal.mjs b/scripts/verify-ui/helpers/population-member-removal-proposal.mjs index 922ea4be0..619820abe 100644 --- a/scripts/verify-ui/helpers/population-member-removal-proposal.mjs +++ b/scripts/verify-ui/helpers/population-member-removal-proposal.mjs @@ -8,14 +8,12 @@ const exactRef = (value) => ({ id: value?.id, }); -/** The verifier decodes every response belonging to its exact project and Explorer. */ export const shouldCapturePopulationMemberNativeResponse = (request, { project, explorer }) => { if (!request || typeof request.path !== 'string') return false; return request.scopeProject === project && request.scopeExplorer === explorer && request.path.startsWith(`/api/v1/projects/${encodeURIComponent(project)}/explorers/${encodeURIComponent(explorer)}/`); }; -/** Compare complete preview populations without assuming the backend's row order. */ export const assertExactPreviewMultiset = (actual, expected, label = 'preview rows') => { assert(Array.isArray(actual) && Array.isArray(expected), `${label} must be arrays`); const unmatched = [...expected]; @@ -28,7 +26,6 @@ export const assertExactPreviewMultiset = (actual, expected, label = 'preview ro assert.equal(unmatched.length, 0, `${label} is missing rows or multiplicity: ${String(unmatched)}`); }; -/** Assert that a native response is the exact attached-selection minus one member proposal. */ export const assertPopulationMemberRemovalProposal = (proposal, expected) => { assert(proposal && typeof proposal === 'object', 'member-removal proposal response is required'); for (const key of ['proposalId', 'baseDocumentDigest', 'candidateWorkspaceDigest']) { @@ -64,7 +61,6 @@ export const assertPopulationMemberRemovalProposal = (proposal, expected) => { return candidate; }; -/** Assert that one native authoring request applies only the current removal receipt. */ export const assertPopulationMemberRemovalApplyCommand = (requests, expected) => { assert(Array.isArray(requests), 'native requests must be an array'); assert(typeof expected.path === 'string' && expected.path.length > 0, 'expected scoped commands path is required'); diff --git a/scripts/verify-ui/helpers/proposal-preview-readiness.mjs b/scripts/verify-ui/helpers/proposal-preview-readiness.mjs index 1d95cfad8..7171384b3 100644 --- a/scripts/verify-ui/helpers/proposal-preview-readiness.mjs +++ b/scripts/verify-ui/helpers/proposal-preview-readiness.mjs @@ -2,9 +2,6 @@ const panelSelector = '[data-testid="construction-proposal-panel"][data-proposal const previewSelector = '[data-testid="construction-proposal-preview"][data-preview-status="ready"]'; const previewRowSelector = 'tbody tr[data-testid="construction-proposal-preview-row"]'; -// The proposal summary panel and preview table are siblings in the Builder DOM. -// Bind them by receipt and output identity so stale or unrelated preview tables -// cannot satisfy a readiness wait. export const proposalPreviewReadinessExpression = (expectedOutputId, expectedRowCount) => { if (typeof expectedOutputId !== 'string' || expectedOutputId.trim() === '') { throw new TypeError('proposal preview readiness requires the expected output ID'); diff --git a/scripts/verify-ui/helpers/proposal-reopen-binding.mjs b/scripts/verify-ui/helpers/proposal-reopen-binding.mjs index b41162308..eaa46ccbd 100644 --- a/scripts/verify-ui/helpers/proposal-reopen-binding.mjs +++ b/scripts/verify-ui/helpers/proposal-reopen-binding.mjs @@ -1,6 +1,5 @@ import assert from 'node:assert/strict'; -/** Verify that a post-Cancel removal candidate was newly captured and is still bound to the unchanged draft. */ export const assertReopenedProposalAfterCancel = ({ previous, cancelledAt, current, state, outputId, removeStepIds }) => { assert(Number.isInteger(previous?.requestIndex) && Number.isInteger(current?.requestIndex), 'Both proposal captures must retain their native request indices'); diff --git a/scripts/verify-ui/helpers/quantity-pivot-budget.mjs b/scripts/verify-ui/helpers/quantity-pivot-budget.mjs index 51e21f55b..a6962f024 100644 --- a/scripts/verify-ui/helpers/quantity-pivot-budget.mjs +++ b/scripts/verify-ui/helpers/quantity-pivot-budget.mjs @@ -45,7 +45,6 @@ export async function waitForPivotObservable({ page, fallbackWait, predicate, ar if (budgetMs > DEFAULT_ACTION_TO_RENDER_BUDGET_MS) { if (typeof predicate !== 'function') throw new TypeError('Browser waits require an inspection function.'); if (typeof argument === 'string') throw new TypeError('Browser wait arguments must be structured data, not source code.'); - // The scoped exception uses the same Playwright wait primitive without the shared helper's five-second cap. await page.waitForFunction(predicate, argument ?? {}, { timeout: boundedTimeoutMs }); return; } diff --git a/scripts/verify-ui/helpers/related-policy-construction.mjs b/scripts/verify-ui/helpers/related-policy-construction.mjs index bf8ba7abe..6c5e5583c 100644 --- a/scripts/verify-ui/helpers/related-policy-construction.mjs +++ b/scripts/verify-ui/helpers/related-policy-construction.mjs @@ -1,9 +1,5 @@ import assert from 'node:assert/strict'; -// Build the only construction shape a saved RELATED_EXPAND policy edit may -// produce when EXCLUDE changes to PRESERVE_PARENT: the authored policy changes, -// and its stable related-record output becomes nullable at this step and every -// downstream step that carries the same output column. export function expectedPreserveParentRelatedEdit(construction, stepIndex) { const expected = JSON.parse(JSON.stringify(construction)); assert(Number.isInteger(stepIndex) && stepIndex >= 0 && stepIndex < expected.steps.length, diff --git a/scripts/verify-ui/helpers/related-quantity-pivot-discovery-artifact.mjs b/scripts/verify-ui/helpers/related-quantity-pivot-discovery-artifact.mjs index 9e84d7c13..42e516214 100644 --- a/scripts/verify-ui/helpers/related-quantity-pivot-discovery-artifact.mjs +++ b/scripts/verify-ui/helpers/related-quantity-pivot-discovery-artifact.mjs @@ -244,7 +244,6 @@ function validateQueryEvidence(run, manifest) { } } -/** Load a trusted complete acquisition after validating its source, scope, query manifest, and full result. */ export async function loadRelatedQuantityPivotDiscoveryArtifact({ artifactPath, artifactSha256, diff --git a/scripts/verify-ui/helpers/report.mjs b/scripts/verify-ui/helpers/report.mjs index b8c63c174..0402b2599 100644 --- a/scripts/verify-ui/helpers/report.mjs +++ b/scripts/verify-ui/helpers/report.mjs @@ -40,7 +40,6 @@ export const recordUntested = (report, dimension, name, reason) => { current.evidence.push({ name, status: 'untested', reason }); }; -// This is a diagnostic label only. The Playwright observer owns native failure assertions. export const classifyNetworkRecord = (record = {}) => { if (['exception', 'console-error', 'console', 'pageerror', 'runtime'].includes(record.kind) || record.internalError) { return 'unexpected-error'; @@ -66,7 +65,6 @@ export const isActionable = (snapshot) => && snapshot?.pointerEvents !== 'none' && Boolean(snapshot?.receivesPointer); -// Report status describes recorded evidence; it is never used to decide whether a native test passed. export const finishReport = (report) => { const assertions = Array.isArray(report.assertions) ? report.assertions : []; const requiredChecks = Array.isArray(report.requiredChecks) ? report.requiredChecks : []; diff --git a/scripts/verify-ui/helpers/saved-preview-binding.mjs b/scripts/verify-ui/helpers/saved-preview-binding.mjs index c3fd38d45..6544502e5 100644 --- a/scripts/verify-ui/helpers/saved-preview-binding.mjs +++ b/scripts/verify-ui/helpers/saved-preview-binding.mjs @@ -1,6 +1,5 @@ import { isDeepStrictEqual } from 'node:util'; -/** Select a successful native Preview request for one action/reload window. */ export const selectSavedPreviewRequest = (requests, { startIndex, path, receiptId, outputId }) => { if (!Array.isArray(requests) || !Number.isInteger(startIndex) || startIndex < 0 || startIndex > requests.length) return undefined; if (![path, receiptId, outputId].every(value => typeof value === 'string' && value.length > 0)) return undefined; @@ -11,7 +10,6 @@ export const selectSavedPreviewRequest = (requests, { startIndex, path, receiptI ); }; -/** Reuse a saved Preview only when Cancel restores the same draft and visible receipt. */ export const selectCanceledSavedPreviewRequest = (requests, { startIndex, path, outputId, before, after }) => { if (!Array.isArray(requests) || !before || !after || !Number.isInteger(startIndex) || startIndex < 0 || startIndex > requests.length) return undefined; if (before.outputId !== outputId || after.outputId !== outputId) return undefined; @@ -42,7 +40,6 @@ export const selectCanceledSavedPreviewRequest = (requests, { startIndex, path, const completedSuccessfulResponse = entry => entry?.status === 200 && Number.isFinite(entry.completedAt) && !entry.failure && !entry.responseReadError && entry.response && entry.response.bodyNotRead !== true; -/** Bind a proposal preview to the exact accepted choice command and persisted reconciliation receipt. */ export const selectAcceptedChoicePreviewBinding = (requests, { startIndex, proposalPath, commandPath, reconcilePath, outputId, before, after, }) => { @@ -124,7 +121,6 @@ export const selectAcceptedChoicePreviewBinding = (requests, { return undefined; }; -/** A cached accepted choice preview survives Cancel only for the exact same current draft and visible receipt. */ export const selectCanceledAcceptedChoicePreviewBinding = (before, after) => { if (!before || !after || before.kind !== 'accepted-choice' || after.kind !== 'accepted-choice') return undefined; if (![before.receiptId, before.outputId, before.commandId, before.snapshotToken, before.draftDigest].every(value => diff --git a/scripts/verify-ui/helpers/scope-001-restricted-auth-bootstrap.mjs b/scripts/verify-ui/helpers/scope-001-restricted-auth-bootstrap.mjs index 0c0e1fd8f..5c36a1812 100644 --- a/scripts/verify-ui/helpers/scope-001-restricted-auth-bootstrap.mjs +++ b/scripts/verify-ui/helpers/scope-001-restricted-auth-bootstrap.mjs @@ -173,12 +173,6 @@ export function createScope001SyntheticBearer(issuer, subject = 'scope-001-local return `${header}.${claims}.`; } -/** - * Start the local Fence `/user` endpoint used only by the isolated auth target. - * It exposes one exact read/write grant and accepts only the matching in-memory - * bearer. The app's Calypr path consumes this grant; this helper does not test - * JWT signatures or external identity/ACL services. - */ export async function startScope001FenceStub({ bindAddress = '0.0.0.0', containerHostname = 'host.docker.internal', @@ -331,7 +325,6 @@ function scope001MutationBindVars(target) { }; } -/** Seed the isolated generation, move only the blocked target document to its sibling path, and prove both rows remain loaded. */ export async function seedAndScopeScope001Fixture({ target, bearer, fetchImpl = globalThis.fetch, execFileSyncImpl = execFileSync } = {}) { assert(target && target.project === 'loom_dev_scope001-auth' && target.generation === 'scope001-auth-v1', 'restricted-auth seeding requires the exact owned project and generation'); diff --git a/scripts/verify-ui/helpers/scope-001-restricted-auth-oracle.mjs b/scripts/verify-ui/helpers/scope-001-restricted-auth-oracle.mjs index daaa2d05e..78dd0b454 100644 --- a/scripts/verify-ui/helpers/scope-001-restricted-auth-oracle.mjs +++ b/scripts/verify-ui/helpers/scope-001-restricted-auth-oracle.mjs @@ -38,11 +38,6 @@ const assertNoAuthResourcePath = resources => { } }; -/** - * Read the literal SCOPE-001 restricted-auth FHIR fixture and derive its visible rows. - * Both Observations are loaded from NDJSON; the blocked witness is denied only by - * its post-load target-document authorization path, while its subject edge remains allowed. - */ export const readScope001RestrictedAuthOracle = fixtureDir => { const patients = parseNDJSON(join(fixtureDir, 'Patient.ndjson')); const observations = parseNDJSON(join(fixtureDir, 'Observation.ndjson')); @@ -120,7 +115,6 @@ export const assertScope001RestrictedAuthVisibleRows = (actualRows, fixtureDir) return oracle; }; -/** Verify the fixture-only post-load path split before the Builder assertion. */ const assertScope001LoadedAuthorizationShape = ({ project, generation, diff --git a/scripts/verify-ui/helpers/scope-001-restricted-auth-runtime.mjs b/scripts/verify-ui/helpers/scope-001-restricted-auth-runtime.mjs index 8b9e56b73..f29472e1f 100644 --- a/scripts/verify-ui/helpers/scope-001-restricted-auth-runtime.mjs +++ b/scripts/verify-ui/helpers/scope-001-restricted-auth-runtime.mjs @@ -5,7 +5,6 @@ const API_EXECUTABLE = 'arango-fhir-server'; const CONFIG_ARGUMENT = '--config /workspace/scope-001-restricted-auth.server.yaml'; const NO_AUTH_ARGUMENT = /(?:^|\s)--no-auth(?:\s|$)/; -/** Verify the owned auth target runs the configured Calypr server entrypoint. */ export function assertScope001RestrictedAuthRuntime({ apiContainer, dockerTop = container => execFileSync('docker', ['top', container, '-eo', 'pid,args'], { encoding: 'utf8', timeout: 10_000 }) }) { assert.match(apiContainer ?? '', /^[A-Za-z0-9_.-]+$/, 'The validated owned API container is required.'); diff --git a/scripts/verify-ui/helpers/superseded-capabilities-abort.mjs b/scripts/verify-ui/helpers/superseded-capabilities-abort.mjs index 2dc89091d..3d9bdee8a 100644 --- a/scripts/verify-ui/helpers/superseded-capabilities-abort.mjs +++ b/scripts/verify-ui/helpers/superseded-capabilities-abort.mjs @@ -11,12 +11,6 @@ const uniqueRequestMembership = (requests, request) => { requests.filter((candidate) => candidate.requestId === request.requestId).length === 1; }; -/** - * Prove that a failed capabilities request was aborted as a stale read after a - * concrete column mutation changed the draft key and a matching fresh read began. - * Returns detailed chronology evidence or undefined; it never classifies by error - * text, endpoint, or later response alone. - */ export function proveSupersededCapabilitiesAbort(requests, abortedRequest) { if (!Array.isArray(requests)) throw new TypeError('requests must be an array'); if (!capabilityPath(abortedRequest) || abortedRequest.method !== 'POST' || abortedRequest.failure !== 'net::ERR_ABORTED' || diff --git a/scripts/verify-ui/helpers/tests/basic-named-cohort-presence-oracle.test.mjs b/scripts/verify-ui/helpers/tests/basic-named-cohort-presence-oracle.test.mjs index 47170741b..669155535 100644 --- a/scripts/verify-ui/helpers/tests/basic-named-cohort-presence-oracle.test.mjs +++ b/scripts/verify-ui/helpers/tests/basic-named-cohort-presence-oracle.test.mjs @@ -57,9 +57,6 @@ test('row assertion requires exact group membership and Boolean results', () => ], { memberColumn: 'Members', presenceColumn: 'Presence', oracle, phase: 'mutated presence' }), /exact Boolean false/); }); -// The former browser workflow's call-order/source-regex checks are retired. -// These assertions cover the retained oracle and route helpers, not browser -// creation, selection requests, or fresh UI reload after API seeding. test('browser Explorer collection paths encode the canonical project identity', () => { const project = 'loom_dev_verify_example/cda-fhir-v1'; assert.equal(fixtureProjectExplorerCollectionPath(project), '/api/v1/projects/loom_dev_verify_example%252Fcda-fhir-v1/explorers'); diff --git a/scripts/verify-ui/helpers/tests/cohort-identities.test.mjs b/scripts/verify-ui/helpers/tests/cohort-identities.test.mjs index 90f527072..807ad19bc 100644 --- a/scripts/verify-ui/helpers/tests/cohort-identities.test.mjs +++ b/scripts/verify-ui/helpers/tests/cohort-identities.test.mjs @@ -136,9 +136,6 @@ test('rejects a missing member binding after the UI operation', () => { const cohortRevisionId = 'grouprev_cf467f0bfc6f7dda2f5568fd963333f9a495101424b660b19e817d02b2027c3b'; const selectionRevisionId = 'selection_30228fde63d4b7c29ae74361a9ab502366e02a62aba50c7ffee8c0f09e2fc706'; const resourceTypeColumnId = 'source_d44aab344dd675e370dd19b7'; -// Observed fields copied from domain.savedFieldDocument in the retained -// standalone-cohort-fields-domain-json-f9677b4ebb0c60e8277a6a2a7ad4d5a3a41abbe4.json attachment. -// This excerpt intentionally omits saved-document fields not needed by this helper contract. const capturedSavedFieldDocumentExcerpt = () => ({ rows: { kind: 'GROUPS', diff --git a/scripts/verify-ui/helpers/tests/created-explorer-scope.test.mjs b/scripts/verify-ui/helpers/tests/created-explorer-scope.test.mjs index f82fbce2a..5126540c3 100644 --- a/scripts/verify-ui/helpers/tests/created-explorer-scope.test.mjs +++ b/scripts/verify-ui/helpers/tests/created-explorer-scope.test.mjs @@ -21,8 +21,6 @@ test('created Explorer scope exposes canonical browser command, reconcile, and m `${expectedRoot}/authoring/v2/population-mapping`, ]); - // This covers the route helper; the retired population browser workflow's - // use of these routes remains a separate, unverified wiring guarantee. }); diff --git a/scripts/verify-ui/helpers/tests/fixture-context.test.mjs b/scripts/verify-ui/helpers/tests/fixture-context.test.mjs index 32508b67a..58e70eec2 100644 --- a/scripts/verify-ui/helpers/tests/fixture-context.test.mjs +++ b/scripts/verify-ui/helpers/tests/fixture-context.test.mjs @@ -24,8 +24,6 @@ test('isolated DATE oracle is written under QA isolation with its real project a assert.deepEqual(actualOracle.rows, report.target.qaIsolation.fixtureRows, 'The literal DATE rows must match the retained QA isolation rows.'); - // The retired journey report gate is not exercised here. This checks the - // retained producer's project/generation binding against the literal oracle. assert.equal(report.target.qaIsolation.fixtureOracle, undefined); const fixtureOracle = { diff --git a/scripts/verify-ui/helpers/tests/pivot-keyword-names.test.mjs b/scripts/verify-ui/helpers/tests/pivot-keyword-names.test.mjs index 34129ecc1..fc47e7833 100644 --- a/scripts/verify-ui/helpers/tests/pivot-keyword-names.test.mjs +++ b/scripts/verify-ui/helpers/tests/pivot-keyword-names.test.mjs @@ -41,7 +41,9 @@ test('pivot keyword-names script keeps identity, 5xx-closed-set, anti-vacuity, a 'Cancel must not change the draft digest (byte stability)', 'duplicate names were silently accepted into the document', 'the duplicate-ID class needs a second category', - 'switching intent is the real close path', + 'construction-row-intent-records', + 'pivot editor closed', + "await realClick('[data-testid=\"construction-row-intent-records\"]');\n await until(async () => !(await mounted('[data-testid=\"construction-reshape-pivot\"]')), 'pivot editor closed'", 'Pivot output name', 'construction-apply-proposal', 'construction-row-remove-', diff --git a/scripts/verify-ui/helpers/tests/pivot-keyword-oracle.test.mjs b/scripts/verify-ui/helpers/tests/pivot-keyword-oracle.test.mjs index f6855e6e4..d8e670f4b 100644 --- a/scripts/verify-ui/helpers/tests/pivot-keyword-oracle.test.mjs +++ b/scripts/verify-ui/helpers/tests/pivot-keyword-oracle.test.mjs @@ -1,5 +1,3 @@ -// Offline static checks for pivot-keyword-fixture-oracle.mjs: -// node --test scripts/verify-ui/helpers/tests/pivot-keyword-oracle.test.mjs import test from 'node:test'; import assert from 'node:assert/strict'; import { mkdtempSync, writeFileSync } from 'node:fs'; diff --git a/scripts/verify-ui/helpers/tests/policy-self-test-structured-failure.test.mjs b/scripts/verify-ui/helpers/tests/policy-self-test-structured-failure.test.mjs index 356e175c8..01bd843c3 100644 --- a/scripts/verify-ui/helpers/tests/policy-self-test-structured-failure.test.mjs +++ b/scripts/verify-ui/helpers/tests/policy-self-test-structured-failure.test.mjs @@ -6,8 +6,6 @@ import test from 'node:test'; import { validateModelVerdict, configuredModelRules } from '../../../lib/formal-verdict.mjs'; import { tools } from '../../../lib/formal-tools.mjs'; -// The retired source-sliced receipt builder and TraceProgress model are not -// current guarantees. Exercise the active statechart verdict boundary instead. const positive = () => ({ exit: 0, counterexamples: [], diff --git a/scripts/verify-ui/helpers/tests/related-quantity-pivot-discovery-parity.test.mjs b/scripts/verify-ui/helpers/tests/related-quantity-pivot-discovery-parity.test.mjs index 322ce154c..c3d641f54 100644 --- a/scripts/verify-ui/helpers/tests/related-quantity-pivot-discovery-parity.test.mjs +++ b/scripts/verify-ui/helpers/tests/related-quantity-pivot-discovery-parity.test.mjs @@ -190,11 +190,6 @@ const patientBatchB = { }], }; -// The request result is grouped by Patient and does not retain Observation IDs. -// This second Patient's identical one-row FINAL/d numeric group models the same -// Observation value (2) reached through a distinct Patient lineage; the host -// must count that Patient's route separately while preserving each Patient's -// source group until the final cross-root merge. const patientBatchC = { project: 'case018-project', generation: 'case018-generation', @@ -315,8 +310,6 @@ test('CASE-018 paged discovery weights shared routes and matches literal typed h assert.equal(Object.hasOwn(builder.bindVars, 'visible_text_keys'), false, 'Raw key discovery must not depend on preview-derived text keys'); } - // Scope is enforced independently at every collection and edge boundary in - // each query; a filter on one join endpoint cannot stand in for another. assert.match(firstPageQuery.query, /s\._key > \@after_specimen_key/); assertEntityScope(firstPageQuery.query, { alias: 's', resourceType: 'Specimen', allowedVariable: 'allowed', filterPrefix: 's\\._key > @after_specimen_key\\s+AND', @@ -573,8 +566,6 @@ test('CASE-018 streaming discovery preserves one shared Observation on distinct nextAfterSpecimenKey: null, }); - // Each duplicate raw edge pair is represented once in the bounded AQL - // response, while the same Observation remains paired with both Patients. const specimenPatientEdges = [ ['Specimen/root-a', 'Patient/patient-a'], ['Specimen/root-a', 'Patient/patient-a'], ['Specimen/root-b', 'Patient/patient-b'], ['Specimen/root-b', 'Patient/patient-b'], @@ -605,8 +596,6 @@ test('CASE-018 streaming discovery preserves one shared Observation on distinct ]); assert.deepEqual(accumulator.nextPatientBatch(2), ['Patient/patient-a', 'Patient/patient-b']); - // The query groups by (observationId, patientId); its bounded response then - // carries one aggregate group for each Patient lineage. const sharedObservationGroups = uniqueObservationPatientEdges.map(([observationId, patientId]) => { assert.equal(observationId, 'Observation/shared'); return { diff --git a/scripts/verify-ui/helpers/tests/row96-harness-lifecycle.test.mjs b/scripts/verify-ui/helpers/tests/row96-harness-lifecycle.test.mjs index f5981699a..cb9b5812f 100644 --- a/scripts/verify-ui/helpers/tests/row96-harness-lifecycle.test.mjs +++ b/scripts/verify-ui/helpers/tests/row96-harness-lifecycle.test.mjs @@ -10,12 +10,11 @@ const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..', ' const SUBJECT = join(REPO_ROOT, 'scripts/verify-ui/workflows/verify-explorer-row-source-four-state.mjs'); const MAIN_MARKER = 'const main = async () => {'; const INJECTED = `${MAIN_MARKER} if (process.env.RS96_FORCE_FAIL === '1') throw new Error('injected failure');`; -const DEEP_MARKER = " markPhase('cdp-connect');"; // throws right after connect+listeners completed and the phase was recorded +const DEEP_MARKER = " markPhase('cdp-connect');"; const DEEP_INJECTED = `${DEEP_MARKER} if (process.env.RS96_FORCE_FAIL_DEEP === '1') throw new Error('injected failure');`; const SPAWN_TIMEOUT_MS = 120000; const POST_EXIT_GRACE_MS = 1000; const STACK_ENV = { - // Constructor inputs only: the staged lifecycle run never contacts this stack. LOOM_DEV_SOURCE_ROOT: REPO_ROOT, LOOM_DEV_COMPOSE_PROJECT: 'loom-dev-stravinsky', LOOM_DEV_API_PORT: '8182', @@ -24,10 +23,7 @@ const STACK_ENV = { const listProfiles = (directory) => readdirSync(directory).filter((name) => name.startsWith('rs96-prof-')); -// Process-group survivor scan: zombies (empty cmdline, invisible to pgrep -f / cmdline -// filters) are still group members, and no unrelated path-mentioning process is touched. const groupMembers = (pgid) => { - // zombie entries awaiting reap are terminally dead; only live group members count if (!Number.isInteger(pgid)) return []; try { return readdirSync('/proc').filter((entry) => { @@ -42,10 +38,6 @@ const groupMembers = (pgid) => { const stageFailedRun = async (slug) => { const root = mkdtempSync(join(tmpdir(), 'rs96-lifecycle-')); - // Preserve the subject's repository-relative geometry so its three-level REPO_ROOT - // calculation resolves to this disposable tree. Dependencies remain symlinks into - // the real, read-only source checkout; the fixture lookup resolves through a staged - // testdata symlink instead of escaping to /tmp/testdata. const nested = join(root, 'scripts', 'verify-ui', 'workflows'); mkdirSync(nested, { recursive: true }); symlinkSync(join(REPO_ROOT, 'scripts/loom-dev.mjs'), join(root, 'scripts', 'loom-dev.mjs')); @@ -60,9 +52,6 @@ const stageFailedRun = async (slug) => { const injected = deep ? DEEP_INJECTED : INJECTED; const markerCount = source.split(marker).length - 1; assert.equal(markerCount, 1, 'forced-failure injection must land on a unique marker'); - // This test owns browser lifecycle, not the application journey. Remove only the - // staged copy's server identity/fixture setup; keep real Chrome/CDP and the - // production failure reporting, process-tree teardown, and profile deletion. let stagedSource = source; if (deep) { const cdpMarker = " const portFile = join(profile, 'DevToolsActivePort');"; @@ -88,7 +77,7 @@ const stageFailedRun = async (slug) => { child.stderr.on('data', (chunk) => { stderr = (stderr + chunk).slice(-16000); }); let timedOut = false; const exit = await new Promise((done) => { - const killer = setTimeout(() => { timedOut = true; try { child.kill('SIGKILL'); } catch { /* already gone */ } }, SPAWN_TIMEOUT_MS); + const killer = setTimeout(() => { timedOut = true; try { child.kill('SIGKILL'); } catch { } }, SPAWN_TIMEOUT_MS); child.on('exit', (code, signal) => { clearTimeout(killer); done({ code, signal }); }); }); await new Promise((done) => setTimeout(done, POST_EXIT_GRACE_MS)); @@ -97,26 +86,18 @@ const stageFailedRun = async (slug) => { const treeMarker = stdout.match(/RUN_TREE pgid=(\d+)/); const treePgid = treeMarker ? Number(treeMarker[1]) : null; const survivors = treePgid === null ? [] : groupMembers(treePgid).map((pid) => ({ pid, profilePath: profilePaths.join(',') || 'none' })); - // Measured BEFORE this harness's own cleanup: deletion is the child's responsibility. - // The bounded poll absorbs the volume's stale-metadata reads; process-group members - // (above) are the authoritative survivors — rmSync succeeds even while a live holder - // keeps the tree open, so dirGone alone is never the lifecycle proof. let dirGoneAtExit = false; for (let attempt = 0; attempt < 60 && !dirGoneAtExit; attempt += 1) { dirGoneAtExit = profilePaths.every((profilePath) => !existsSync(profilePath)); if (!dirGoneAtExit && attempt < 59) await new Promise((done) => setTimeout(done, 50)); } const journeyPath = join(out, 'journey.json'); - // Buffered at exit+grace BEFORE any cleanup: this docker volume intermittently - // ENOENTs a file that milliseconds-earlier reads see intact, so retry then keep - // the parsed value; later assertions never reopen the path. let report = null; for (let attempt = 0; attempt < 20 && report === null; attempt += 1) { try { report = JSON.parse(readFileSync(journeyPath, 'utf8')); } catch { if (attempt < 19) await new Promise((done) => setTimeout(done, 50)); } } - // kill(-pgid) can be denied (EPERM) in sandboxed kernels: signal members individually - for (const pid of groupMembers(treePgid)) { try { process.kill(pid, 'SIGKILL'); } catch { /* racing exit */ } } + for (const pid of groupMembers(treePgid)) { try { process.kill(pid, 'SIGKILL'); } catch { } } for (const profilePath of profilePaths) rmSync(profilePath, { recursive: true, force: true }); if (process.env.RS96_KEEP_STAGED !== '1') rmSync(root, { recursive: true, force: true }); return { ...exit, timedOut, stdout, stderr, out, journeyPath, report, dirGone: dirGoneAtExit, stagedRoot: root, profilePaths, survivors }; @@ -148,8 +129,6 @@ const assertNoSurvivors = async (slug) => { assert.equal(run.profilePaths.length, 1, `deep failure must still create exactly one profile; ${summarize(run)}`); assert.match(run.stdout, /RUN_TREE pgid=\d+/, `deep case must have launched the tracked chrome tree; ${summarize(run)}`); assert.deepEqual(run.survivors, [], `no process may remain in the chrome process group after child exit plus grace; ${run.survivors.map((entry) => `${entry.pid}@${entry.profilePath}`).join(' , ')}; ${summarize(run)}`); - // D-4 (reviewer): the deep case must actually transit the CDP phase or it can - // green-off on an early identity/fixture throw without ever launching a browser tree. const reachedCdp = (run.report?.phaseTimings?.phases ?? []).some((phase) => phase.phase === 'cdp-connect'); assert.equal(run.report?.lifecycleOnly, true, 'deep case must use isolated lifecycle preconditions'); assert.equal(run.report?.status, 'failed', 'deep injected failure must be recorded'); @@ -157,7 +136,6 @@ const assertNoSurvivors = async (slug) => { assert.equal(run.report?.buildIdentity, null, 'lifecycle test must not claim a serving build identity'); assert.equal(run.report?.fixture, null, 'lifecycle test must not create a serving fixture'); assert.ok(reachedCdp, `deep failed run must reach the cdp-connect phase (identity/fixture early throw made it vacuous); phases=${JSON.stringify(run.report?.phaseTimings?.phases ?? null)}; ${summarize(run)}`); - // profile teardown guarantee: the harness removes its own user-data-dir even on failure assert.ok(run.dirGone, `run profile dir must be deleted by harness teardown, persisted: ${run.profilePaths.join(' , ')}`); } finally { delete process.env.RS96_FORCE_FAIL_DEEP; diff --git a/scripts/verify-ui/helpers/tests/row96-killtree-escalation.test.mjs b/scripts/verify-ui/helpers/tests/row96-killtree-escalation.test.mjs index f809ea68d..3a31eb63c 100644 --- a/scripts/verify-ui/helpers/tests/row96-killtree-escalation.test.mjs +++ b/scripts/verify-ui/helpers/tests/row96-killtree-escalation.test.mjs @@ -5,10 +5,6 @@ import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'n import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; -// F1/F2 anti-regression (reviewer ses_e611c1be9ffe): the group SIGKILL must not hinge on -// the leader's exit, and an inherited stderr pipe may not pin the journey's event loop. -// The killTree body is EXTRACTED from the subject under test, so this suite breaks the -// moment someone weakens escalation back to the leader-exit timeout path. const REPO_ROOT = resolve(import.meta.dirname, '..', '..', '..', '..'); const SUBJECT = join(REPO_ROOT, 'scripts/verify-ui/workflows/verify-explorer-row-source-four-state.mjs'); @@ -34,7 +30,6 @@ const LEADER_SRC = [ '', ].join('\n'); -// argv: leaderPath stubbornPath infoPath stderrMode const CASE_HEAD = [ "import { spawn } from 'node:child_process';", "import { readdirSync, readFileSync } from 'node:fs';", @@ -77,7 +72,7 @@ const spawnCase = (mode) => new Promise((done) => { child.on('exit', (code, signal) => { clearTimeout(killer); let helperPid = null; - try { helperPid = Number(readFileSync(infoPath, 'utf8').match(/pid=(\d+)/)[1]); } catch { /* case died before leader wrote */ } + try { helperPid = Number(readFileSync(infoPath, 'utf8').match(/pid=(\d+)/)[1]); } catch { } done({ code, signal, stdout, helperPid }); }); }); diff --git a/scripts/verify-ui/helpers/tests/row96-row-source-four-state.test.mjs b/scripts/verify-ui/helpers/tests/row96-row-source-four-state.test.mjs index 0daf2ee4b..714825510 100644 --- a/scripts/verify-ui/helpers/tests/row96-row-source-four-state.test.mjs +++ b/scripts/verify-ui/helpers/tests/row96-row-source-four-state.test.mjs @@ -63,9 +63,6 @@ test('row96 matrix wire proof binds the applying commands echo with load-time bo assert.ok(source.includes(probe), `matrix wire-proof wiring missing: ${probe}`); } assert.ok(!source.includes('const matrixRowSets ='), 'the retired proposals-based proof must not silently return'); - // Pin the extractor's behavior inside its own definition: textual probes - // elsewhere (assert messages, listener registration) survive a body stub, - // so the proof surfaces are asserted on the brace-matched definition body. const cmdDefAt = source.indexOf('const commandRowSets ='); assert.ok(cmdDefAt >= 0, 'commandRowSets definition missing'); const cmdDefOpen = source.indexOf('{', cmdDefAt); @@ -90,9 +87,6 @@ test('row96 matrix wire proof binds the applying commands echo with load-time bo ]) { assert.ok(commandRowSetsBody.includes(probe), `commandRowSets definition lost a proof surface: ${probe}`); } - // Each leg's server-multiset proof must read only captures from after its own - // leg-start watermark (project binding blocks the cross-project class; this - // blocks the cross-leg class). assert.match(source, /const cmdBeforeExclude = matrixCmdWire\.length;[\s\S]*commandRowSets\(cmdBeforeExclude\)/, 'EXCLUDE proof is not leg-watermarked on the commands echo'); assert.match(source, /const cmdBeforePreserve = matrixCmdWire\.length;[\s\S]*commandRowSets\(cmdBeforePreserve\)/, 'PRESERVE proof is not leg-watermarked on the commands echo'); }); diff --git a/scripts/verify-ui/helpers/viewer-output-dataset-zip-oracle.mjs b/scripts/verify-ui/helpers/viewer-output-dataset-zip-oracle.mjs index c7404fb1e..ba5ba44cd 100644 --- a/scripts/verify-ui/helpers/viewer-output-dataset-zip-oracle.mjs +++ b/scripts/verify-ui/helpers/viewer-output-dataset-zip-oracle.mjs @@ -32,9 +32,6 @@ const crc32 = bytes => { return (crc ^ 0xffffffff) >>> 0; }; -// Read the stored members from the actual archive bytes. The layout follows -// the central directory, so entries written with ZIP data descriptors remain -// readable even when their local size fields are zero. const readStoredZip = archivePath => { const archive = readFileSync(archivePath); let endOffset = -1; diff --git a/scripts/verify-ui/workflows/core-protocol-browser.mjs b/scripts/verify-ui/workflows/core-protocol-browser.mjs index 6e6049f24..83713ffc1 100644 --- a/scripts/verify-ui/workflows/core-protocol-browser.mjs +++ b/scripts/verify-ui/workflows/core-protocol-browser.mjs @@ -113,7 +113,7 @@ const watchdog = setTimeout(() => { report.status = 'failed'; report.error = { message: `Whole-run budget of ${options.budgetMs}ms exceeded` }; report.timingMs = { total: nowMs(), budget: options.budgetMs }; - try { writeFileSync(join(out, 'browser.json'), JSON.stringify(report, null, 2)); } catch { /* best effort */ } + try { writeFileSync(join(out, 'browser.json'), JSON.stringify(report, null, 2)); } catch { } console.log(JSON.stringify({ ...finishSummary(), watchdog: true })); process.exit(1); }, options.budgetMs); @@ -633,8 +633,6 @@ try { passed('witness-A-receipt-re-executable-by-id-over-real-wire', { receiptId: aReconcile.receiptId, rowCount: replayA.rowCount, columns: (replayA.columns ?? []).map(column => column.label), rowsContainFixtureIDs: oracle.ids.every(id => JSON.stringify(replayA.rows).includes(id)) }); report.witnessB = { role: 'second save persists while the first compile stays wire-held', columns: [{ column: 'id', from: labelAlpha, to: labelBeta }, { column: 'name[].family', from: labelFor('name[].family'), to: labelGamma }] }; - // Arm before the commit lands: the client fires the A-prime reconcile as soon as the command wave is acknowledged, - // which can precede our out-of-band persistence poll. holdArmed = true; holdExpectedDraftVersion = Number(aHead.draftVersion) + 1; report.witnessB.aPrimePersisted = await renameColumn('id', labelAlpha, labelBeta); @@ -745,21 +743,6 @@ try { await click('[data-testid="preview-table-header-controls"] button'); await until(async () => !(await columnsOpen()), 'column settings panel closed'); } - // ---- Witness C: a genuinely late-delivered real response for an obsolete owning context ---- - // Trace (why this is the only un-aborted in-flight core-protocol request at detach): - // ui/packages/loom-ui/src/react.tsx:151-171 - mutation triggers own their AbortController and abort - // only through the returned abort(); nothing in unmount reaches them. - // ui/packages/loom-ui/src/react.tsx:104-114 - query resources abort their controller when the last - // subscriber leaves, so the automatic-preview compile (a useKeyedQuery) is transport-aborted at - // unmount through BuilderWorkspace.tsx:2999 -> :2982-2996 (execution.compileRequest?.abort()). - // ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:3310 - the Review dataset - // affordance fires the ownerless whole-workspace compile: void reconcileCurrent(). - // BuilderWorkspace.tsx:2593-2644 - reconcileCurrent stores that request in activeCompile and adopts - // it only while requestIsCurrent() (workspaceAttached + generation + intent fingerprint) holds. - // BuilderWorkspace.tsx:461-464,4269 - unmount calls the workspace host ref with null, so - // workspaceAttached flips false; the abort call sites (:758 selectExplorer, :819 new command wave, - // :2583 delete Explorer) are none of them reachable from the Viewer context switch - // (ui/apps/demo/src/main.tsx:44-52,74-75 - setState + history.replaceState, no document navigation). report.witnessC = { role: 'real response for a released-but-obsolete whole-workspace compile is admitted nowhere', affordance: { @@ -990,13 +973,6 @@ try { receiptBeforeDetach: panelJustBeforeDetach.previewReceiptId, }); - // ---- Witness D: the production Go handler already answered, only the response delivery is held - // (Fetch Response stage keeps the bytes at the fetch layer; unlike the Witness B request-stage hold, - // the compile genuinely ran — its receipt exists server-side). A second, competing real save then - // persists underneath the withheld response; the stale compile must not own the display when it lands. - // Trace: ui/packages/loom-ui/src/features/ExplorerBuilder/BuilderWorkspace.tsx:2593-2644 adopts a - // compile result only while requestIsCurrent() holds; a result whose receipt belongs to a replaced - // head is an obsolete attempt for this column context and must be discarded without re-issue. const witnessD = { role: 'server-answered compile withheld at the fetch layer while a competing save persists underneath' }; report.witnessD = witnessD; if ((await selectedTableOutputId()) !== outputId) { @@ -1035,10 +1011,6 @@ try { assert.ok(Number(eHead.draftVersion) > Number(dHead.draftVersion), 'the competing save did not advance the head past the held compile'); assert.notEqual(eHead.draftDigest, dHead.draftDigest); const fixtureRowCheck = (snapshot, label) => { - // Gender source-value oracle leg intentionally suspended here: witness D renamed the Gender column - // (assertPatientRows' gender check is label-bound), while the baseline and witness A/B/C oracle - // checks keep the full gender-value discipline under the untouched 'Gender' header. Rows identity - // + cardinality discipline (assertPatientRows with null gender map) still runs here. assert.ok(snapshot.rows && snapshot.rows.length >= 3, `${label}: preview table rendered without the two fixture rows`); return assertPatientRows(snapshot.rows.slice(1), oracle.ids, null, snapshot.rows[0]); }; @@ -1099,12 +1071,6 @@ try { }); await screenshot('witness-d-post-release.png'); - // ---- Witness E: the mirror image. An in-flight real preview response for the CURRENT attempt is - // withheld at the fetch layer, then released; unlike the stale witness D response, this one must be - // adopted behind its own receipt — proving the client discards by obsolete context, not by transport - // delay, over the joined production stack. - // Trace: BuilderWorkspace preview binding requires the delivered preview receipt to equal the bound - // compile receipt of the current head before rows render (same path Witness B asserted for B's head). const witnessE = { role: 'settled current-attempt preview withheld at the fetch layer, adopted on release' }; report.witnessE = witnessE; const familyBeforeE = await persistedState('name[].family'); @@ -1154,10 +1120,6 @@ try { }); await screenshot('witness-e.png'); - // ---- M5 real-app lifecycle legs on this same joined page (launch/doctor ran out-of-band: - // make dev-doctor -> DEV_DOCTOR_PASSED, evidence/m5/doctor-run1.log). A document reload must - // re-hydrate the display from persisted truth, and the real Delete-table / Undo controls must - // delete and restore through the production commands channel. const m5 = { reload: null, deletions: [], restore: null }; report.m5 = m5; const m5Truth = async observation => { @@ -1307,12 +1269,12 @@ try { const image = await withTimeout(client.send('Page.captureScreenshot', { format: 'png' }), 5000, 'failure screenshot'); await writeFile(join(out, 'failure.png'), Buffer.from(image.data, 'base64')); report.screenshots.push('failure.png'); - } catch { /* evidence best effort */ } + } catch { } } process.exitCode = 1; } finally { clearTimeout(watchdog); - try { client?.close(); } catch { /* already gone */ } + try { client?.close(); } catch { } if (chrome && chrome.exitCode === null && chrome.signalCode === null) { chrome.kill('SIGTERM'); await Promise.race([new Promise(resolveExit => chrome.once('exit', resolveExit)), wait(1000)]); diff --git a/scripts/verify-ui/workflows/verify-base-settings.mjs b/scripts/verify-ui/workflows/verify-base-settings.mjs index 2088b4620..e51d25da7 100644 --- a/scripts/verify-ui/workflows/verify-base-settings.mjs +++ b/scripts/verify-ui/workflows/verify-base-settings.mjs @@ -265,8 +265,6 @@ try { report.coverage.push({ state: variant.name, explorer: variant.explorer, authoredSteps: hasSteps, expandedPaths: discovery.choices.filter(c => c.kind === 'EXPANDED').map(c => c.fieldPath), explicitGroups: discovery.explicitGroups.length }); if (hasSteps) assert.equal(discovery.explicitGroups.length, 0, 'Discovery advertised a saved cohort that cannot compose with authored steps'); else assert(discovery.explicitGroups.some(g => g.revisionId === variant.group.revisionId), 'Source-only cohorts must remain usable'); - // The reported multi-resource chain is a discovery regression. Data oracles - // below use a single selected Observation, so never guess its chain counts. if (variant === related) continue; for (const item of selections(discovery, hasSteps ? undefined : variant.group).filter(item => variant !== partial || item.selection.kind === 'EXPLICIT_GROUP')) { const record = { id: `${variant.name}-${report.cases.length}`, name: item.name, selection: item.selection, status: 'running' }; diff --git a/scripts/verify-ui/workflows/verify-cda-filter-browser.mjs b/scripts/verify-ui/workflows/verify-cda-filter-browser.mjs index 774f5beb3..8dba6f41b 100644 --- a/scripts/verify-ui/workflows/verify-cda-filter-browser.mjs +++ b/scripts/verify-ui/workflows/verify-cda-filter-browser.mjs @@ -717,7 +717,6 @@ export async function filterBrowserWorkflow({ page, cda }) { lifecycle.cancel = { status: 'passed', preservedDraftDigest: baseline.draftDigest, expectedRows: [sourceRow], actualRows: cancelledRows, durationMs }; await cda.click('[data-testid="construction-action-keep-rows"]'); await cda.wait(() => Boolean(document.querySelector('[data-testid="construction-filter-editor"] select[aria-label="Condition"]:not(:disabled)')), []); - // Toggle away and back so a canceled proposal is recreated even if the editor retained its form. await cda.selectOption('[data-testid="construction-filter-editor"] select[aria-label="Condition"]', 'EQUALS'); start = Date.now(); await cda.selectOption('[data-testid="construction-filter-editor"] select[aria-label="Condition"]', 'MISSING'); diff --git a/scripts/verify-ui/workflows/verify-cda-last-table.mjs b/scripts/verify-ui/workflows/verify-cda-last-table.mjs index bdbd13b77..490a42377 100644 --- a/scripts/verify-ui/workflows/verify-cda-last-table.mjs +++ b/scripts/verify-ui/workflows/verify-cda-last-table.mjs @@ -140,7 +140,6 @@ try { assert.deepEqual(rows.map(row => row.ordinal), Array.from({ length: 25 }, (_, index) => index + 1), `${phase}: the preview must expose every ordinal from 1 through 25`); - // Restore the visible top with ordinary wheel input so each lifecycle comparison uses the same viewport. let current = scrollTop; while (current > metrics.initialScrollTop + 0.5) { const previous = current; diff --git a/scripts/verify-ui/workflows/verify-cda-legacy-collection.mjs b/scripts/verify-ui/workflows/verify-cda-legacy-collection.mjs index 5d1142a23..ab3450594 100644 --- a/scripts/verify-ui/workflows/verify-cda-legacy-collection.mjs +++ b/scripts/verify-ui/workflows/verify-cda-legacy-collection.mjs @@ -3,7 +3,6 @@ import { readFile } from 'node:fs/promises'; import { spawnSync } from 'node:child_process'; export async function legacyCollectionWorkflow({ page, cda }) { - // Replay an owned, pre-direction-field QA Explorer without changing its saved draft. const seedPath = cda.env.LOOM_LEGACY_COLLECTION_SEED_REPORT ?? process.env.LOOM_LEGACY_COLLECTION_SEED_REPORT; assert(seedPath, 'Set LOOM_LEGACY_COLLECTION_SEED_REPORT to a retained owned legacy fixture report'); const seed = JSON.parse(await readFile(seedPath, 'utf8')); diff --git a/scripts/verify-ui/workflows/verify-compound-coded-group.mjs b/scripts/verify-ui/workflows/verify-compound-coded-group.mjs index a48e8159f..6b76660bb 100644 --- a/scripts/verify-ui/workflows/verify-compound-coded-group.mjs +++ b/scripts/verify-ui/workflows/verify-compound-coded-group.mjs @@ -2,7 +2,6 @@ import assert from 'node:assert/strict'; import { execFileSync } from 'node:child_process'; import { isDeepStrictEqual } from 'node:util'; -// Native Playwright body for compound coded grouping lifecycles. export async function verifyCompoundCodedGroup({ page, cda, check, mode = 'single' }) { const origin = cda.uiOrigin; const apiOrigin = cda.apiOrigin; diff --git a/scripts/verify-ui/workflows/verify-explorer-derive-row-source.mjs b/scripts/verify-ui/workflows/verify-explorer-derive-row-source.mjs index 5d7c0e400..fad2ca993 100644 --- a/scripts/verify-ui/workflows/verify-explorer-derive-row-source.mjs +++ b/scripts/verify-ui/workflows/verify-explorer-derive-row-source.mjs @@ -1,36 +1,4 @@ #!/usr/bin/env node -// ROW-SOURCE-02 §18 DERIVE row-source journey — committed repeatable verification. -// Standalone raw-CDP browser journey against the owned loom-dev stack, following the -// scripts/verify-ui/workflows raw-CDP convention (see core-protocol-browser.mjs and the -// sibling verify-explorer-filter-row-source.mjs): a directly-runnable node script, not a -// Playwright registry entry. -// -// Legs (sibling of the filter journey; slice-1 contract from internal/explorer/lifecycle/derive_source.go): -// 1 DISCOVERY More actions -> Calculate entry enabled; editor optgroup "Dataset fields (not in output)" -// exposes the signed populated root scalar valueQuantity.value (decimal) -// 2 CONFIGURE guided ADD, left = signed option, right = literal 3 -> proposal READY -// preview keeps all 6 rows (PROPAGATE_NULL row-preservation), derived == value + 3 -// for the 3 populated rows and blank for the 3 rows without valueQuantity.value -// 3 CANCEL byte-stable pre-proposal construction -// 4 APPLY persisted DERIVE step + server-owned signed source projection (unselected field, -// ownerStepId == derive step id) + exact rows/cells -// 4 RELOAD replacement banner + capability echo deriveSourceInput columnId == persisted projection -// 4 EDIT literal 3 -> 2 => value + 2 preview, persisted; back to 3 => value + 3, persisted -// 4 REMOVE derive steps 1->0, derived column gone, no derive-owned projection left behind -// 4 RE-ADD honest re-materialization: new derive-owned columnId, capability echo equal -// API negatives 9/9 + 2 pinned contract behaviours (valid control decorated, forged sig 409, -// cross-snapshot stale 409, tampered path 409, empty/absent choiceId 400, -// three selections 400 at-most-2, duplicate 400, column mismatch 422, -// unpopulated valueInteger never offered/never accepted, non-DERIVE step 200-undecorated) -// -// Exit-code contract (kept from the sibling): every named check must run AND pass. A skipped -// (missing) check or a thrown assertion is a failure; an empty identity where rows are required -// is a failure. Any failure exits 1. -// -// Reproduce (owned stack env identifies the target, per loom-dev conventions): -// LOOM_DEV_SOURCE_ROOT= LOOM_DEV_COMPOSE_PROJECT=loom-dev-stravinsky \ -// LOOM_DEV_API_PORT=8182 LOOM_DEV_UI_PORT=3182 \ -// node scripts/verify-ui/workflows/verify-explorer-derive-row-source.mjs --out import assert from 'node:assert/strict'; import { spawn, spawnSync } from 'node:child_process'; import { randomUUID } from 'node:crypto'; @@ -103,12 +71,6 @@ const seed = async (result) => { projectionMode: idCandidate.defaultProjectionMode, initialPresentation: 'TABLE', title: 'ID', }], builder); assert.equal(response.status, 200, `seed ADD_COLUMN failed (${response.status}): ${JSON.stringify(response.value).slice(0, 400)}`); - // The stage DERIVE capability requires >=1 public scalar numeric column on the append stage - // (internal/dataframe/compiler/lower/recipe_construction.go:2242), so seed a projected numeric - // column; the signed dataset-field lane still binds its own server-owned source column. - // valueInteger (not valueQuantity.value) is deliberate: projecting the SAME field the signed - // slot claims makes the compiler emit 500 CONSTRUCTION_RECEIPT_CONTRACT_INVALID with no - // actionable diagnostic (live-pinned defect, evidence /tmp/opencode/derive-probe/probe6 runs). builder = (await call('builder')).value; response = await commands(`${runStem}-value`, [{ type: 'ADD_COLUMN', outputId, occurrenceId: 'base', candidateId: valueCandidate.candidateId, @@ -141,11 +103,6 @@ const record = (name, evidence) => { }; const save = (name, value) => writeFileSync(join(out, name), typeof value === 'string' ? value : JSON.stringify(value, null, 2)); -// Independent oracle over the FULL live dataset for the base Observation node: the pinned -// testdata/devloop-fixture ndjson plus the repo-generated J01 concept observations. The base -// Observation catalog node spans all routes (1006 rows) and the filter lane only signs -// cardinality-one scalars, so grids are validated row-by-row against this full oracle, with one -// deterministic scoped grid on valueQuantity.unit='cm' (exactly the two devloop cm Observations). const computeOracle = () => { const observations = readFileSync(join(session.fixtureDir, 'Observation.ndjson'), 'utf8') .split(/\r?\n/).filter((line) => line.trim()).map((line) => JSON.parse(line)); @@ -173,12 +130,8 @@ const computeOracle = () => { return { byId, rows, cmRows, derived: (row, offset) => (row.value === null ? null : Number((row.value + offset).toFixed(6))) }; }; -// Docker build-stamp identity via the scoped bridge's permitted docker exec channel -// (shared collector in loom-dev.mjs: label-resolved API container, /workspace stamp -// script --check plus the recorded build json), same channel as the CDA captures. const collectBuildIdentity = () => sharedCollectBuildIdentity(session, { repoRoot: REPO_ROOT }); -// ---- Raw CDP harness (flags and DevToolsActivePort path match loom-dev/core-protocol) ---- const chromePath = options.chrome ?? process.env.LOOM_E2E_CHROME ?? ['/usr/bin/chromium', '/opt/google/chrome/chrome'].find((candidate) => existsSync(candidate)); assert.ok(chromePath && existsSync(chromePath), 'No Chrome binary: pass --chrome, set LOOM_E2E_CHROME, or install /opt/google/chrome/chrome'); @@ -219,7 +172,7 @@ try { const value = await probe(); if (value) return value; if (Date.now() > deadline) { - try { save(`FAIL-${String(message).replace(/\W+/g, '_')}.html`, await evaluate('document.documentElement.outerHTML')); } catch { /* best effort */ } + try { save(`FAIL-${String(message).replace(/\W+/g, '_')}.html`, await evaluate('document.documentElement.outerHTML')); } catch { } assert.fail(`condition failed: ${message}`); } await wait(80); @@ -272,7 +225,6 @@ try { if (await mounted('[data-testid="construction-close-operation-editor"]')) await realClick('[data-testid="construction-close-operation-editor"]'); await until(() => evaluate(`!document.querySelector('[data-testid="construction-calculate-editor"]')`), 'calculate editor closed', 15000); }; - // Guided form: ADD, left = signed dataset-field option, right = numeric literal. const configureCalculateForm = async (signedOptionValue, literalText) => { await setSelect('[data-testid="construction-calculate-editor"] select[aria-label="Operation"]', 'ADD'); const guided = await mounted('[data-testid="construction-calculate-editor"] select[aria-label="Operation"]'); @@ -280,8 +232,6 @@ try { await setSelect('[data-testid="construction-calculate-editor"] select[aria-label="First value kind"]', 'column'); await until(() => mounted('[data-testid="construction-calculate-left-column"]'), 'left column select', 15000); const committed = await setSelect('[data-testid="construction-calculate-editor"] select[aria-label="First value column"]', signedOptionValue); - // Binding a signed option re-keys the slot to its server-shaped columnId and removes the - // signed choice from the options list (FormulaOperand filters boundChoiceIds). assert.ok(/^derive-input_/.test(committed), `picking signed left option must materialize a derive-input_ column slot, got ${JSON.stringify(committed)}`); const stillOffered = await evaluate(`(()=>{const s=document.querySelector('[data-testid="construction-calculate-editor"] select[aria-label="First value column"]');if(!s)return 'missing';const og=s.querySelector('optgroup[label="Dataset fields (not in output)"]');if(!og)return 'removed';return [...og.querySelectorAll('option')].some(o=>o.value===${JSON.stringify(signedOptionValue)})?'still-listed':'removed';})()`); @@ -302,14 +252,6 @@ try { const opt=[...og.querySelectorAll('option')].find(o=>o.getAttribute('title')==='valueQuantity.value' && o.getAttribute('data-testid')==='construction-calculate-left-source-option'); return opt?{value:opt.value,label:opt.textContent.trim(),title:opt.getAttribute('title')}:null; })()`); - // Server-truth doc read (bs-62-2 twin fix from verify-explorer-filter-row-source.mjs:329-338). - // It deliberately does NOT default the construction object to {}: an absent construction turned - // every `(doc.steps || []).filter(...)` census below into an empty scan that its own - // `length === 0` claim then certified as success. Callers only reach this after at least one - // construction proposal has been applied (authoringv2/commands.go:1447 sets Document.Construction - // on APPLY_CONSTRUCTION_PROPOSAL; live-pinned: a seeded doc carries no construction key yet, so - // every pre-first-proposal poll in this file keeps a tolerant read instead); swallowing its - // absence into {} after that point could only mask a broken read. const builderDoc = async () => { const builder = (await mainCtx.call('builder')).value; const doc = builder?.workspace?.documents?.[0]?.construction; @@ -404,7 +346,6 @@ try { assert.fail('no history step opened the requested editor'); }; - // Build identity BEFORE the run (pin the deployed build). const stampBefore = collectBuildIdentity(); report.buildIdentity = { before: stampBefore, after: null }; save('build-identity.before.json', stampBefore); @@ -413,14 +354,9 @@ try { `build identity must be fully recorded (runningBinary/liveSource/expectedSource/gitHead non-null): ${JSON.stringify(stampBefore)}`); requireRecordedIdentity(stampBefore, 'pre-run:'); - // Independent oracle. const oracle = computeOracle(); report.oracle = oracle; - // Self-seed three independent fixtures: A carries the raw-CDP UI journey, B is a pristine proposal - // host for the API negatives (its candidate references a single new step; a persisted DERIVE step in - // the same project would be a second step and mask candidate validation), and C supplies a valid - // signed valueQuantity.value choice whose embedded snapshot differs from B (the provably-stale 409). const mainFixture = await createFreshVerificationFixture(session, mainLabel, { populateBootstrap: false }); const negFixture = await createFreshVerificationFixture(session, negLabel, { populateBootstrap: false }); const staleFixture = await createFreshVerificationFixture(session, staleLabel, { populateBootstrap: false }); @@ -437,7 +373,6 @@ try { await Promise.all([send('Page.enable'), send('Runtime.enable')]); - // STEP 0 behavior: capabilities expose the deriveSourceInput valueQuantity.value choice. await navigate(); { const builder = (await mainCtx.call('builder')).value; @@ -452,17 +387,12 @@ try { assert.ok(['integer', 'decimal'].includes(String(numericChoice.logicalType).toLowerCase()), `step 0 signed option must be numeric kind, got ${numericChoice.logicalType}`); assert.equal(numericChoice.isPopulated, true, 'step 0 signed option must report isPopulated'); - // Population flags come from catalog-global candidate stats, so which fields carry - // isPopulated=false varies; capture them for the editor filter assertion below. unpopulatedPaths = dsi.choices.filter((choice) => choice.isPopulated !== true).map((choice) => choice.fieldPath); report.steps.push({ step: 'behave', status: caps.status, supported: dsi.supported, choicesPaths: dsi.choices.map((c) => c.fieldPath), valueQuantityColumnIdBefore: numericChoice.columnId ?? null }); record('step-0-behavior-capabilities', { status: caps.status, supported: dsi.supported, choicesPaths: dsi.choices.map((c) => c.fieldPath) }); } - // STEP 1 PRELUDE: a saved row-preserving FILTER step (status EQUALS final). The Calculate - // action is gated on construction.steps.length >= 1 (BuilderWorkspace calculateAvailable), so - // the calculate lane is only reachable after one saved operation; this filter keeps every row. { await realClick('[data-testid="construction-action-keep-rows"]'); await until(() => mounted('[data-testid="construction-filter-editor"]'), 'filter-editor mounted', 20000); @@ -475,11 +405,6 @@ try { await setInput('[data-testid="construction-filter-editor"] input[aria-label="Value"]', 'final'); await until(() => mounted('[data-testid="construction-proposal-ready"]'), 'prelude proposal ready', 30000); await realClick('[data-testid="construction-apply-proposal"]'); - // Pre-first-persistence window: construction is legitimately nil until the first - // APPLY_CONSTRUCTION_PROPOSAL (live-pinned: a seeded doc carries no construction key; - // authoringv2/commands.go:1447 sets it on proposal apply). This poll therefore keeps the - // filter lane's tolerant read shape; the post-loop presence assert below is the - // observed-transition floor — a never-persisted FILTER stays loudly red. for (let i = 0; i < 25; i += 1) { const builder = (await mainCtx.call('builder')).value; if ((builder.workspace.documents[0].construction?.steps || []).some((s) => s.operation?.kind === 'FILTER')) break; @@ -497,7 +422,6 @@ try { record('step-1-prelude-filter-step', { preludeIdentityCount: preludeIds.length }); } - // STEP 1 DISCOVERY: Calculate entry + signed dataset-field option in the editor. { const entry = await openCalculateEditor(); assert.ok(/calc/i.test(entry?.aria || ''), `Calculate action aria-label missing name: ${JSON.stringify(entry)}`); @@ -529,7 +453,6 @@ try { await closeCalculateEditor(); } - // STEP 2 CONFIGURE + oracle-exact preview (row-preserving), then preview evidence. { await openCalculateEditor(); await setSelect('[data-testid="construction-calculate-editor"] select[aria-label="First value kind"]', 'column'); @@ -544,8 +467,6 @@ try { await closeCalculateEditor(); } - // STEP 3 CANCEL byte-stability against the post-prelude baseline (form was cancelled; only the - // row-preserving prelude legitimately mutated the draft before this point). { const baseline = postPreludeBaseline; const after = await mainCtx.call('builder'); @@ -557,7 +478,6 @@ try { record('step-3-cancel-byte-stable', { sameDoc: same, draftDigest: after.value.draftDigest }); } - // STEP 4 APPLY -> persisted DERIVE step + signed unselected source projection + exact rows. { await navigate(); await openCalculateEditor(); @@ -591,8 +511,6 @@ try { const main = await evaluate(`(()=>{const t=document.querySelector('[data-testid="preview-table-content"]');if(!t)return null;const h=[...t.querySelectorAll('[role=columnheader]')].map(c=>c.textContent.trim());const rs=[...t.querySelectorAll('[role=row]')].map(r=>[...r.querySelectorAll('[role=cell]')].map(c=>c.textContent.trim()));return{h,rs};})()`); await screenshot('step4-after-apply'); const preview = { headers: main?.h || [], rows: main?.rs || [] }; - // The signed slot column header is present though the field was never selected; its data - // materialization is asserted exactly in the scoped cm grid (small deterministic page). assert.ok((main?.h || []).some((header) => /valueQuantity\.value/i.test(header)), `APPLY: signed slot column header absent: ${JSON.stringify(main?.h)}`); const seen = checkPreviewAgainstOracle(preview, 3, 'STEP 4 APPLY'); @@ -604,7 +522,6 @@ try { report.deriveStepId = step.id; } - // STEP 4 RELOAD -> replacement banner + capability echo columnId equals the persisted projection. { await navigate(); const doc = await builderDoc(); @@ -644,8 +561,6 @@ try { record('step-4-reload-materialized-echo', { echoColumnId: echo.columnId, persistedProjectionCol: projection.columnId }); } - // STEP 4 EDIT: DERIVE literal 3 -> 2 (every visible derived cell becomes value + 2) -> back to - // 3, each persisted. Row-identity and PROPAGATE_NULL blanks are re-validated on every page. { await navigate(); await wait(300); await openEditorFromHistory(() => mounted('[data-testid="construction-calculate-editor"] input[aria-label="Second value number"]')); @@ -685,17 +600,12 @@ try { record('step-4-edit-back-three', { previewRows: seenThree, persistedLiteral: 3 }); } - // STEP 4 REMOVE -> derived column gone, no derive-owned projection left behind. { if (await mounted('[data-testid="construction-close-operation-editor"]')) { await realClick('[data-testid="construction-close-operation-editor"]'); await wait(200); } await navigate(); await wait(300); const docBefore = await builderDoc(); const step = (docBefore.steps || []).find((s) => s.operation?.kind === 'DERIVE'); assert.ok(step, 'REMOVE: no DERIVE step to remove'); - // Zero-match discipline (filter-lane twin): the post-removal `length === 0` asserts below are - // only meaningful when the pre-removal census really contained the DERIVE step together with its - // derive-owned valueQuantity.value projection; observing the pre-state turns "went to zero" into - // a witnessed transition instead of an empty scan that certifies itself. assert.ok((docBefore.sourceProjections || []).some((projection) => projection.fieldPath === 'valueQuantity.value' && projection.ownerStepId), `REMOVE pre-remove census: no derive-owned valueQuantity.value projection in the persisted doc; "no leftover owned projection" must never be certified from a census that was empty to begin with: ${JSON.stringify((docBefore.sourceProjections || []).map((projection) => ({ path: projection.fieldPath, owner: projection.ownerStepId })))}`); const historyIds = await evaluate(`(()=>[...document.querySelectorAll('[data-testid^="construction-history-step-"]')].map(e=>e.getAttribute('data-testid').replace('construction-history-step-','')))()`); @@ -731,7 +641,6 @@ try { record('step-4-remove-unowned', { afterRemoveIdentity: afterIds, unowned: leftovers.length === 0 }); } - // STEP 4 RE-ADD -> honest re-materialization (new derive-owned columnId, capability echo equal). { await navigate(); await wait(200); await openCalculateEditor(); @@ -760,9 +669,6 @@ try { record('step-4-re-add-honest-rematerialization', { deriveStepId: step.id, projCol: projection.columnId, projOwner: projection.ownerStepId, capEcho: echo.columnId, previewRows: seen }); } - // STEP 5 SCOPED GRID: a second saved FILTER (valueQuantity.unit EQUALS cm) deterministically - // reduces to the two cm Observations, so the derived column must show exactly oracle value + 3 - // for both rows with matching identity — the arithmetic leg at exact-scope precision. { if (await mounted('[data-testid="construction-close-operation-editor"]')) { await realClick('[data-testid="construction-close-operation-editor"]'); await wait(200); } await navigate(); await wait(200); @@ -787,16 +693,12 @@ try { if (filters.some((s) => s.operation?.filter?.values?.[0]?.string === 'cm')) break; await wait(300); } - // Observed-transition discipline (bs-62-2 class): an exhausted poll must not fall through into - // the exact-scope asserts below, which would otherwise certify a scoped grid nobody persisted. const scopedDoc = await builderDoc(); assert.ok((scopedDoc.steps || []).some((s) => s.operation?.kind === 'FILTER' && s.operation?.filter?.values?.[0]?.string === 'cm'), `STEP 5 SCOPED-cm: the persisted doc never showed the cm FILTER step within ${25 * 300}ms; falling through an unobserved apply would leave the exact-row asserts below scanning an unscoped table`); await wait(400); const table = await evaluate(`(()=>{const t=document.querySelector('[data-testid="preview-table-content"]');if(!t)return null;const h=[...t.querySelectorAll('[role=columnheader]')].map(c=>c.textContent.trim());const rs=[...t.querySelectorAll('[role=row]')].map(r=>[...r.querySelectorAll('[role=cell]')].map(c=>c.textContent.trim()));return{h,rs};})()`); const cmTableSeen = checkPreviewAgainstOracle({ headers: table?.h || [], rows: table?.rs || [] }, 3, 'STEP 5 SCOPED-cm-table', oracle.cmRows.map((row) => row.id)); - // Exact-scope materialization of the unselected signed field: its slot column carries the - // raw valueQuantity.value data (never user-selected; only the server-owned projection knows it). const slotHeaderIndex = (table?.h || []).findIndex((header) => /valueQuantity\.value/i.test(header)); assert.ok(slotHeaderIndex >= 0, `SCOPED-cm: signed slot column absent from scoped table: ${JSON.stringify(table?.h)}`); (table?.rs || []).filter((row) => row.some((cell) => /^dev-/.test(cell))).forEach((row, rowIndex) => { @@ -813,10 +715,8 @@ try { record('row-evidence-blanks-and-numerics', { blankCells: evidence.blankCells, numericCells: evidence.numericCells }); } - // ---- API negatives against the deployed binary, on a pristine proposal host ---- await runApiNegatives(negCtx, staleCtx); - // Post-run build identity: source and running binary must be unchanged across the run. const stampAfter = collectBuildIdentity(); requireRecordedIdentity(stampAfter, 'post-run:'); report.buildIdentity.after = stampAfter; @@ -827,7 +727,6 @@ try { assert.deepEqual(exceptions, [], `uncaught page exceptions: ${JSON.stringify(exceptions)}`); - // Exit-code contract: every named check must have run and passed. const expectedChecks = [ 'step-0-behavior-capabilities', 'step-1-prelude-filter-step', 'step-1-discovery', 'step-2-configure-oracle-preview', 'step-3-cancel-byte-stable', 'step-4-apply-exact-rows', 'step-4-reload-materialized-echo', 'step-4-edit-literal-two', 'step-4-edit-back-three', @@ -841,7 +740,6 @@ try { const skipped = expectedChecks.filter((name) => !present.has(name)); assert.equal(skipped.length, 0, `skipped checks (each assertion must run): ${JSON.stringify(skipped)}`); - // Evidence floor: every leg must have produced its screenshot; silently dropped captures fail. const EXPECTED_SCREENSHOT_FLOOR = 11; assert.ok(report.screenshots.length >= EXPECTED_SCREENSHOT_FLOOR, `screenshot tally below the per-leg floor: wrote ${report.screenshots.length}, expected at least ${EXPECTED_SCREENSHOT_FLOOR}`); @@ -853,13 +751,12 @@ try { } catch (error) { report.status = 'failed'; report.error = error instanceof Error ? error.message : String(error); - try { save('journey.json', report); } catch { /* best effort */ } + try { save('journey.json', report); } catch { } console.error(`JOURNEY_FAILED ${report.error}`); process.exitCode = 1; } finally { try { if (chrome && chrome.exitCode === null && chrome.signalCode === null) { - // detached spawn => own process group; leader-only kills orphan the helper tree const groupSignal = (signal) => { try { process.kill(-chrome.pid, signal); return true; } catch { return chrome.kill(signal); } }; const exited = new Promise((done) => chrome.once('exit', done)); if (groupSignal('SIGTERM')) { @@ -867,21 +764,14 @@ try { } if (chrome.exitCode === null && chrome.signalCode === null) groupSignal('SIGKILL'); } - } catch { /* teardown best effort */ } - if (profile) { try { rmSync(profile, { recursive: true, force: true }); } catch { /* reported via absence of deletion */ } } + } catch { } + if (profile) { try { rmSync(profile, { recursive: true, force: true }); } catch { } } } function assertNonEmpty(values, label) { assert.ok(Array.isArray(values) && values.length > 0, `${label}: zero rows matched, expected at least one oracle row`); } -// Zero-match discipline for the build-stamp barrier (bs-62-2 twin fix from -// verify-explorer-filter-row-source.mjs:676-688): the stamp script compares the recorded identity -// internally but a failed or truncated `cat` of the recorded stamp leaves -// recordedBinary/recordedSource null in OUR hands, and the checkExit gate alone never notices. -// Safety: loom-dev-build-stamp.sh --check exits 0 only when expected_source == source_digest && -// running_binary == expected_binary read from the same recorded file, so on every legitimate green -// these equalities already hold and this floor cannot false-fail a real run. function requireRecordedIdentity(stamp, when) { assert.ok(stamp.recordedBinary && stamp.recordedSource, `${when} recorded build identity is unreadable or empty (recordedBinary/recordedSource must be non-null): ${JSON.stringify(stamp)}`); @@ -908,23 +798,11 @@ async function runApiNegatives(host, staleCtx) { const embeddedSnapshot = (token) => { try { return JSON.parse(Buffer.from(token.split('.')[1], 'base64url').toString()).snapshotToken ?? null; } catch { return null; } }; - // Decodes the `rc1..` row-choice token (see capability/row_choice.go:257-278). - // Without it, a token that fails to decode makes embeddedSnapshot() null, and `null !== hostToken` - // would satisfy the staleness check by accident — the zero-match twin of the vacuous-pass class - // (bs-62-2 N2 analog, mirrored from verify-explorer-filter-row-source.mjs:707-712). const decodedPayload = (token) => { try { return JSON.parse(Buffer.from(token.split('.')[1], 'base64url').toString()); } catch { return null; } }; const capabilities = async (ctx) => { const builder = (await ctx.call('builder')).value; - // Report-only (bs-62-2 class, same disposition as the filter lane): the pristine host is a - // zero-step claim over an optionally-missing census. A seeded doc legitimately carries NO - // construction object until the first construction proposal is applied (live-pinned 2026-11-02: - // seeded builder payload has no construction key; authoringv2/commands.go:1447 sets it on - // APPLY_CONSTRUCTION_PROPOSAL), so a construction-present assert could not gate this read - // without false-failing. The vacuity window is closed only by the caps.status === 200 plus the - // choice asserts below; closing it properly needs a proposal-persisted sentinel read, which is - // out of this lane's proven-safe scope. const stepCount = (builder.workspace?.documents?.[0]?.construction?.steps || []).length; assert.equal(stepCount, 0, 'the negatives host must be a pristine proposal host so candidateConstruction references only its new step'); const caps = await ctx.call('construction-capabilities', { @@ -937,17 +815,11 @@ async function runApiNegatives(host, staleCtx) { const hostCaps = await capabilities(host); const numericChoice = hostCaps.choices.find((choice) => choice.fieldPath === 'valueQuantity.value'); assert.ok(numericChoice, 'valueQuantity.value choice must exist on the negatives host'); - // The host token is the reference for every token-level negative below: it must actually decode - // and must embed the host catalog snapshot (row_choice.go sets SnapshotToken from the minting - // snapshot), otherwise a null-embedded undecodable token could satisfy staleness comparisons by - // accident (bs-62-2 N2 class; shape from the filter lane :727-729). assert.ok(decodedPayload(numericChoice.choiceId), `the host valueQuantity.value choiceId must decode as a signed row-choice token, or every token-level negative below is unverifiable: ${String(numericChoice.choiceId).slice(0, 24)}`); assert.equal(embeddedSnapshot(numericChoice.choiceId), hostCaps.builder.catalog.snapshotToken, `the host valueQuantity.value choiceId must embed the host catalog snapshot; a null/foreign snapshot would satisfy the staleness comparison by accident: ${JSON.stringify({ embedded: embeddedSnapshot(numericChoice.choiceId), host: hostCaps.builder.catalog.snapshotToken })}`); - // Provably-stale choiceId: a valid signed token whose embedded snapshot comes from another - // fixture, so it is stale relative to the proposal host (the cross-snapshot 409 negative). const staleCaps = await capabilities(staleCtx); const staleNumeric = staleCaps.choices.find((choice) => choice.fieldPath === 'valueQuantity.value'); assert.ok(staleNumeric, 'valueQuantity.value choice must exist on the cross-snapshot fixture'); @@ -1022,10 +894,6 @@ async function runApiNegatives(host, staleCtx) { await expectCase('D8-column-mismatch', body, '422', { diagnosticCode: 'DERIVE_SOURCE_COLUMN_MISMATCH' }); } { - // Non-numeric refusal, LIVE-observable (pinned 422 UNSUPPORTED_DERIVE_SOURCE_FIELD): row-choice - // tokens are signed in one domain, so a genuinely-signed filter-lane token for a non-numeric - // field (status/unit/id/effectiveDateTime) is a valid signature on the DERIVE channel. The - // numeric admission guard must refuse it. const builder = (await host.call('builder')).value; const caps = await host.call('construction-capabilities', { snapshotToken: builder.catalog.snapshotToken, expectedDraftVersion: builder.draftVersion, @@ -1038,9 +906,6 @@ async function runApiNegatives(host, staleCtx) { await expectCase('D9-nonnumeric-choice-refusal', proposalRequest([nonNumericChoice.choiceId]), '422', { diagnosticCode: 'UNSUPPORTED_DERIVE_SOURCE_FIELD' }); } { - // Contract behaviour (live-pinned): deriveSources attached to a NON-DERIVE changed step are - // ignored (200) instead of refused; the guard rail that must hold is that the signed field is - // never decorated into the ignored candidate. Any decoration here would be a source leak. const response = await host.call('construction-proposals', proposalRequest([numericChoice.choiceId], (step) => { step.operation = { kind: 'FILTER', filter: { columnId: 'derive-input_x', operator: 'EQUALS', values: [{ kind: 'STRING', string: 'x' }] } }; })); diff --git a/scripts/verify-ui/workflows/verify-explorer-filter-row-source.mjs b/scripts/verify-ui/workflows/verify-explorer-filter-row-source.mjs index 7a29b8a4a..0926998be 100644 --- a/scripts/verify-ui/workflows/verify-explorer-filter-row-source.mjs +++ b/scripts/verify-ui/workflows/verify-explorer-filter-row-source.mjs @@ -1,29 +1,4 @@ #!/usr/bin/env node -// ROW-SOURCE-01 §18 Filter row-source journey — committed repeatable verification. -// Standalone raw-CDP browser journey against the owned loom-dev stack, following the -// scripts/verify-ui/workflows raw-CDP convention (see core-protocol-browser.mjs): a -// directly-runnable node script, not a Playwright registry entry. -// -// Legs (assertions carried over verbatim from the authoritative journey-run v3 report): -// 1 DISCOVERY optgroup "Dataset fields (not in output)" exposes the unselected gender field -// 2 CONFIGURE native picker -> EQUALS female -> proposal READY -> DOM preview == oracle -// 3 CANCEL byte-stable pre-proposal construction -// 4 APPLY persisted FILTER step + materialized gender source projection + exact rows -// 4 RELOAD capability echo gender columnId == persisted source projection (materialized echo) -// 4 EDIT value -> male => 0 rows; value -> female => oracle rows; persisted -// 4 REMOVE filter steps 1->0, rows return to unfiltered, gender projection unowned -// 4 RE-ADD honest re-materialization: new filter-owned columnId, capability echo equal -// API negatives 7/7 (valid control, forged sig 409, cross-snapshot stale 409, tampered path 409, -// empty/absent choiceId 400, unpopulated field never offered/never accepted) -// -// Exit-code contract (kept from the raw harness): every named check must run AND pass. A skipped -// (missing) check or a thrown assertion is a failure; an empty identity where rows are required -// is a failure. Any failure exits 1. -// -// Reproduce (owned stack env identifies the target, per loom-dev conventions): -// LOOM_DEV_SOURCE_ROOT= LOOM_DEV_COMPOSE_PROJECT=loom-dev-stravinsky \ -// LOOM_DEV_API_PORT=8182 LOOM_DEV_UI_PORT=3182 \ -// node scripts/verify-ui/workflows/verify-explorer-filter-row-source.mjs --out import assert from 'node:assert/strict'; import { spawn, spawnSync } from 'node:child_process'; import { randomUUID } from 'node:crypto'; @@ -117,7 +92,6 @@ const record = (name, evidence) => { }; const save = (name, value) => writeFileSync(join(out, name), typeof value === 'string' ? value : JSON.stringify(value, null, 2)); -// Independent oracle computed straight from the pinned source fixture. const computeOracle = () => { const patients = readFileSync(join(session.fixtureDir, 'Patient.ndjson'), 'utf8') .split(/\r?\n/).filter((line) => line.trim()).map((line) => JSON.parse(line)); @@ -128,12 +102,8 @@ const computeOracle = () => { return { equalFemale, missingGender, all: patients.map((patient) => patient.id).sort() }; }; -// Docker build-stamp identity via the scoped bridge's permitted docker exec channel -// (shared collector in loom-dev.mjs: label-resolved API container, /workspace stamp -// script --check plus the recorded build json), same channel as the CDA captures. const collectBuildIdentity = () => sharedCollectBuildIdentity(session, { repoRoot: REPO_ROOT }); -// ---- Raw CDP harness (flags and DevToolsActivePort path match loom-dev/core-protocol) ---- const chromePath = options.chrome ?? process.env.LOOM_E2E_CHROME ?? ['/usr/bin/chromium', '/opt/google/chrome/chrome'].find((candidate) => existsSync(candidate)); assert.ok(chromePath && existsSync(chromePath), 'No Chrome binary: pass --chrome, set LOOM_E2E_CHROME, or install /opt/google/chrome/chrome'); @@ -172,7 +142,7 @@ try { const value = await probe(); if (value) return value; if (Date.now() > deadline) { - try { save(`FAIL-${String(message).replace(/\W+/g, '_')}.html`, await evaluate('document.documentElement.outerHTML')); } catch { /* best effort */ } + try { save(`FAIL-${String(message).replace(/\W+/g, '_')}.html`, await evaluate('document.documentElement.outerHTML')); } catch { } assert.fail(`condition failed: ${message}`); } await wait(80); @@ -211,8 +181,6 @@ try { await wait(300); }; - // Build identity BEFORE the run (pin the deployed build). Barrier must PASS, never certify on a - // stale or unreadable identity (mirrors the derive script's gate). const stampBefore = collectBuildIdentity(); assert.equal(stampBefore.checkExit, 0, `serving build identity must be current before the journey runs: ${JSON.stringify(stampBefore)}`); assert.ok(stampBefore.runningBinary && stampBefore.liveSource && stampBefore.expectedSource && stampBefore.gitHead, @@ -221,15 +189,9 @@ try { report.buildIdentity = { before: stampBefore, after: null }; save('build-identity.before.json', stampBefore); - // Independent oracle. const oracle = computeOracle(); report.oracle = oracle; - // Self-seed three independent fixtures: A carries the raw-CDP UI journey, B is a pristine - // zero-step proposal host for the API negatives (candidateConstruction references a single new - // step, which a project with a persisted step would reject), and C supplies a valid signed - // gender choice whose embedded snapshot differs from B (the provably-stale choiceId, matching - // the v3 cross-snapshot negative). const mainFixture = await createFreshVerificationFixture(session, mainLabel, { populateBootstrap: false }); const negFixture = await createFreshVerificationFixture(session, negLabel, { populateBootstrap: false }); const staleFixture = await createFreshVerificationFixture(session, staleLabel, { populateBootstrap: false }); @@ -275,10 +237,6 @@ try { const setBack = await setInput('[data-testid="construction-filter-editor"] input[aria-label="Value"]', value); assert.equal(setBack, value, 'value input set failed'); await until(() => mounted('[data-testid="construction-proposal-ready"]'), 'proposal ready', 30000); - // Post-selection read-back (the silent-vacuous half of this helper): every read above happens in - // the same turn as our own write, so an app-side reset or an option that never bound could satisfy - // the whole helper. Re-read what the picker actually holds once it announces READY; zero matches - // (no selected option) is a failure, not a pass. const bound = await evaluate(`(()=>{ const s=document.querySelector('[data-testid="construction-filter-editor"] select[aria-label="Column"]'); const c=document.querySelector('[data-testid="construction-filter-editor"] select[aria-label="Condition"]'); @@ -299,9 +257,6 @@ try { report.steps.push({ step: 'picker', ...picked }); await wait(400); }; - // Server-truth doc read. It deliberately does NOT default the construction object to {}: an absent - // construction turned every `(doc.steps || []).filter(...)` census below into an empty scan that - // its own `length === 0` assert then certified as success (the mut-H2 silent-vacuous channel). const builderDoc = async () => { const builder = (await mainCtx.call('builder')).value; const doc = builder?.workspace?.documents?.[0]?.construction; @@ -310,7 +265,6 @@ try { return doc; }; - // STEP 0 behavior: capabilities expose the filterSourceInput gender choice. await navigate(); { const builder = (await mainCtx.call('builder')).value; @@ -326,7 +280,6 @@ try { record('step-0-behavior-capabilities', { status: caps.status, supported: fsi.supported, choicesPaths: fsi.choices.map((c) => c.fieldPath) }); } - // STEP 1 DISCOVERY { await realClick('[data-testid="construction-action-keep-rows"]'); await until(() => mounted('[data-testid="construction-filter-editor"]'), 'filter editor', 20000); @@ -349,7 +302,6 @@ try { await until(() => evaluate(`!document.querySelector('[data-testid="construction-filter-editor"]')`), 'filter editor closed', 10000); } - // STEP 2 CONFIGURE + oracle-exact preview, then CANCEL { await openKeepRowsEditorAndPick('gender', 'female'); const domPreview = await readDomPreview(); @@ -369,7 +321,6 @@ try { record('step-2-configure-oracle-preview', { previewIdentity: domIdentity, oracle: oracle.equalFemale }); } - // STEP 3 CANCEL byte-stability { const after = await mainCtx.call('builder'); const same = JSON.stringify(after.value.workspace) === JSON.stringify(baseline.workspace) @@ -380,7 +331,6 @@ try { record('step-3-cancel-byte-stable', { sameDoc: same, draftDigest: after.value.draftDigest }); } - // STEP 4 APPLY -> persisted doc + exact rows { await navigate(); await openKeepRowsEditorAndPick('gender', 'female'); @@ -412,7 +362,6 @@ try { record('step-4-apply-exact-rows', { identity, sourceProjections: (doc.sourceProjections || []).map((p) => ({ col: p.columnId, path: p.fieldPath })) }); } - // STEP 4 RELOAD -> materialized server-echo columnId equals the persisted source projection { await navigate(); await realClick('[data-testid="construction-action-keep-rows"]'); @@ -442,8 +391,6 @@ try { record('step-4-reload-materialized-echo', { genderColumnId: genderEcho.columnId, persistedGenderCol }); } - // STEP 4 EDIT -> male => 0 rows, female => oracle rows, both persisted (server-truth assert on - // the persisted FILTER value, mirroring the derive script's persisted-literal gate). { await navigate(); await wait(300); const persistedFilterValue = async () => { @@ -511,14 +458,9 @@ try { record('step-4-edit-back-female-rows', { identity: backIds, oracle: oracle.equalFemale, persistedFilterValue: 'female' }); } - // STEP 4 REMOVE -> rows return to unfiltered, gender projection unowned { if (await mounted('[data-testid="construction-close-operation-editor"]')) { await realClick('[data-testid="construction-close-operation-editor"]'); await wait(200); } await navigate(); await wait(300); - // Zero-match discipline: both post-removal asserts below are `length === 0` claims over a filter - // predicate, so they are only meaningful when the pre-removal census really contained the FILTER - // step and its filter-owned gender projection. Observing the pre-state first turns "went to zero" - // into a witnessed transition instead of an empty scan that certifies itself. const preRemoveDoc = await builderDoc(); assert.ok((preRemoveDoc.steps || []).some((step) => step.operation?.kind === 'FILTER'), `REMOVE pre-remove census: no FILTER step in the persisted doc (${(preRemoveDoc.steps || []).length} step(s)); the post-removal zero asserts below would be vacuous: ${JSON.stringify((preRemoveDoc.steps || []).map((step) => step.operation?.kind))}`); @@ -555,7 +497,6 @@ try { record('step-4-remove-unfiltered', { afterRemoveIdentity: afterRemoveIds, unowned: leftovers.length === 0 }); } - // STEP 4 RE-ADD -> honest re-materialization (new filter-owned columnId, capability echo equal) { await navigate(); await wait(200); await openKeepRowsEditorAndPick('gender', 'female'); @@ -594,10 +535,8 @@ try { record('step-4-re-add-honest-rematerialization', { filterStepId: filterStep.id, projCol: projection.columnId, projOwner: projection.ownerStepId, capEcho: genderEcho.columnId }); } - // ---- API negatives (7/7) against the deployed binary, on a pristine zero-step host ---- await runApiNegatives(negCtx, staleCtx); - // Post-run build identity: source and running binary must be unchanged across the run. const stampAfter = collectBuildIdentity(); requireRecordedIdentity(stampAfter, 'post-run:'); report.buildIdentity.after = stampAfter; @@ -608,7 +547,6 @@ try { assert.deepEqual(exceptions, [], `uncaught page exceptions: ${JSON.stringify(exceptions)}`); - // Exit-code contract: every named check must have run and passed. const expectedChecks = [ 'step-0-behavior-capabilities', 'step-1-discovery', 'step-2-configure-oracle-preview', 'step-3-cancel-byte-stable', 'step-4-apply-exact-rows', 'step-4-reload-materialized-echo', 'step-4-edit-male-zero-rows', 'step-4-edit-back-female-rows', @@ -620,7 +558,6 @@ try { const skipped = expectedChecks.filter((name) => !present.has(name)); assert.equal(skipped.length, 0, `skipped checks (each assertion must run): ${JSON.stringify(skipped)}`); - // Evidence floor: every leg must have produced its screenshot; silently dropped captures fail. const EXPECTED_SCREENSHOT_FLOOR = 11; assert.ok(report.screenshots.length >= EXPECTED_SCREENSHOT_FLOOR, `screenshot tally below the per-leg floor: wrote ${report.screenshots.length}, expected at least ${EXPECTED_SCREENSHOT_FLOOR}`); @@ -632,13 +569,12 @@ try { } catch (error) { report.status = 'failed'; report.error = error instanceof Error ? error.message : String(error); - try { save('journey.json', report); } catch { /* best effort */ } + try { save('journey.json', report); } catch { } console.error(`JOURNEY_FAILED ${report.error}`); process.exitCode = 1; } finally { try { if (chrome && chrome.exitCode === null && chrome.signalCode === null) { - // detached spawn => own process group; leader-only kills orphan the helper tree const groupSignal = (signal) => { try { process.kill(-chrome.pid, signal); return true; } catch { return chrome.kill(signal); } }; const exited = new Promise((done) => chrome.once('exit', done)); if (groupSignal('SIGTERM')) { @@ -646,19 +582,14 @@ try { } if (chrome.exitCode === null && chrome.signalCode === null) groupSignal('SIGKILL'); } - } catch { /* teardown best effort */ } - if (profile) { try { rmSync(profile, { recursive: true, force: true }); } catch { /* reported via absence of deletion */ } } + } catch { } + if (profile) { try { rmSync(profile, { recursive: true, force: true }); } catch { } } } function assertNonEmpty(values, label) { assert.ok(Array.isArray(values) && values.length > 0, `${label}: zero rows matched, expected at least one oracle row`); } -// Zero-match discipline for the build-stamp barrier: the stamp script only compares the recorded -// identity when its own --check runs, so a failed or truncated `cat` of the recorded stamp leaves -// recordedBinary/recordedSource null and the checkExit gate alone never notices. An empty identity -// census must FAIL the run rather than certify it (same discipline as assertNonEmpty above and the -// unpivot twin's "no empty running column" gate). Every floor that existed before stays in place. function requireRecordedIdentity(stamp, when) { assert.ok(stamp.recordedBinary && stamp.recordedSource, `${when} recorded build identity is unreadable or empty (recordedBinary/recordedSource must be non-null): ${JSON.stringify(stamp)}`); @@ -685,9 +616,6 @@ async function runApiNegatives(host, staleCtx) { const embeddedSnapshot = (token) => { try { return JSON.parse(Buffer.from(token.split('.')[1], 'base64url').toString()).snapshotToken ?? null; } catch { return null; } }; - // Decodes the `rc1..` row-choice token (see capability/row_choice.go:257-278). - // Without it, a token that fails to decode makes embeddedSnapshot() null, and `null !== hostToken` - // would satisfy the staleness check by accident — the zero-match twin of the vacuous-pass class. const decodedPayload = (token) => { try { return JSON.parse(Buffer.from(token.split('.')[1], 'base64url').toString()); } catch { return null; } }; @@ -710,8 +638,6 @@ async function runApiNegatives(host, staleCtx) { `the host gender choiceId must embed the host catalog snapshot (capability/row_choice.go:131 sets it from the minting snapshot); a null snapshot would satisfy the staleness comparison by accident: ${JSON.stringify({ embedded: embeddedSnapshot(genderChoice.choiceId), host: hostCaps.builder.catalog.snapshotToken })}`); const activeInCaps = hostCaps.choices.some((choice) => choice.fieldPath === 'active'); - // Provably-stale choiceId: a valid signed token whose embedded snapshot comes from another - // fixture, so it is stale relative to the proposal host (the v3 cross-snapshot 409 negative). const staleCaps = await capabilities(staleCtx); const staleGender = staleCaps.choices.find((choice) => choice.fieldPath === 'gender'); assert.ok(staleGender, 'gender choice must exist on the cross-snapshot fixture'); diff --git a/scripts/verify-ui/workflows/verify-explorer-pivot-keyword-names.mjs b/scripts/verify-ui/workflows/verify-explorer-pivot-keyword-names.mjs index 130a50fd2..e0a342009 100644 --- a/scripts/verify-ui/workflows/verify-explorer-pivot-keyword-names.mjs +++ b/scripts/verify-ui/workflows/verify-explorer-pivot-keyword-names.mjs @@ -1,18 +1,4 @@ #!/usr/bin/env node -// Keyword-named Pivot UI journey (docs/EXPLORER_BACKLOG.md disposition 2026-11-03): -// the AQL-keyword class (category output named exactly 'null') was fixed and -// dispositioned at the compile/API level (recipe_table_reshape.go keyword gate -// + lowerRecipeGroupedPivot duplicate-ID repair, /tmp/opencode/rs08/nullpivot*.log). -// This journey closes the remaining slice: the BROWSER path — typing reserved-keyword -// names in the reshape pivot editor, applying them, seeing them in the comparison -// grid, persisting across reload, being gated on duplicates, restored on remove, -// with byte-identical state when cancelled. Raw-CDP against the owned loom-dev -// stack, following the raw-CDP convention (verify-explorer-row-source-four-state.mjs). -// Selector authority (self-derived from the served source, worker report discarded): -// RowDefinitionSettingsPanel.tsx:983/:995/:1028/:1033/:1062, -// constructionOperations/ConstructionReshapeEditor.tsx:2440/:2535/:4586/:4620/:4628/:4660/:4668/:4675/:4684/:4699/:4809/:4813; -// RowDefinitionSettingsPanel.tsx lives under components/; the reshape editor under -// constructionOperations/ (exact dirs per adversarial review PKW-R6). import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { spawn } from 'node:child_process'; @@ -52,8 +38,6 @@ const expectedChecks = [ 'pk-remove-restores', ]; -// Legitimate rejection families. Anything else on a mutating call is a defect; -// INTERNAL_ERROR/5xx is NEVER legitimate (the pivot-500 class this journey guards). const legit4xx = /^(TABLE_SHAPE_|DUPLICATE_OUTPUT_NAME|INVALID_AUTHORING_COMMAND|DRAFT_CONFLICT|TABLE_PIVOT_|INVALID_|UNRESOLVED_|NAME_)/; const staticValidate = () => { @@ -141,8 +125,8 @@ const markPhase = (name) => { const moduleSend = (method, params = {}, ms = 25000) => withDeadline(client.send(method, params), ms, method); const killTree = async () => { clearTrackedTimers(); - try { client.close(); } catch { /* already closed */ } - try { chrome.stderr?.destroy(); } catch { /* stream already released */ } + try { client.close(); } catch { } + try { chrome.stderr?.destroy(); } catch { } const groupMembers = () => { if (!Number.isInteger(chrome.pid)) return []; try { @@ -159,8 +143,8 @@ const killTree = async () => { const groupSignal = (signal) => { let groupOk = false; if (Number.isInteger(chrome.pid)) { try { process.kill(-chrome.pid, signal); groupOk = true; } catch { groupOk = false; } } - if (!groupOk) { try { chrome.kill(signal); } catch { /* leader gone */ } } - for (const pid of groupMembers()) { try { process.kill(pid, signal); } catch { /* racing exit */ } } + if (!groupOk) { try { chrome.kill(signal); } catch { } } + for (const pid of groupMembers()) { try { process.kill(pid, signal); } catch { } } return true; }; if (chrome.exitCode === null && chrome.signalCode === null) { @@ -211,9 +195,6 @@ const main = async () => { const exceptions = []; report.exceptions = exceptions; client.on('Runtime.exceptionThrown', (event) => exceptions.push(event.exceptionDetails?.exception?.description || event.exceptionDetails?.text || 'exception')); - // F-5 (review): UI-originated authoring fetches are observed too. Any browser-side - // 5xx on /authoring/v2/ is a hard failure at the tail, even when the journey's own - // assertNoInternal legs passed (app-internal fetches belong to no leg's response). const browser5xx = []; const browser4xx = []; const cmdResponses = []; @@ -248,7 +229,7 @@ const main = async () => { const value = await probe(); if (value) return value; if (Date.now() > deadline) { - try { writeFileSync(join(out, `FAIL-${String(message).replace(/\W+/g, '_')}.html`), await evaluate('document.documentElement.outerHTML')); } catch { /* best effort */ } + try { writeFileSync(join(out, `FAIL-${String(message).replace(/\W+/g, '_')}.html`), await evaluate('document.documentElement.outerHTML')); } catch { } assert.fail(`condition failed: ${message}`); } await wait(80); @@ -274,7 +255,6 @@ const main = async () => { }; const readGrid = () => evaluate(`(()=>{const sec=document.querySelector('[data-testid="construction-preview"]');const t=sec?.querySelector('[data-testid="preview-table-content"]')||sec;if(!t)return null;const headerRow=t.querySelector('[role="row"]');const headers=headerRow?[...headerRow.querySelectorAll('[role="columnheader"],[role="cell"]')].map(c=>c.textContent.trim()):[];const trs=[...t.querySelectorAll('[role="row"]')].filter(r=>!r.className.includes('construction-preview-header-row'));const rows=trs.map(tr=>[...tr.querySelectorAll('[role="cell"]')].map(td=>td.textContent.trim())).filter(cells=>cells.length>0);return{headers,rowCount:rows.length,cells:rows,previewStatus:sec?.getAttribute('data-preview-status')??null,previewReceipt:sec?.getAttribute('data-preview-receipt-id')??null};})()`); - // ---- 1. pk-seed-table: API seed identical in shape to the dispositioned keyword probe const builder0 = await api('builder'); assert.equal(builder0.status, 200, `builder fetch failed (${builder0.status})`); const nodes = (builder0.value.catalog?.nodes ?? []).filter((n) => n.rowRootEligible); @@ -305,7 +285,6 @@ const main = async () => { record('pk-seed-table', { outputId, rootNode: obsRoot.resourceType, columns: adds.map((a) => a.title) }); markPhase('seed'); - // ---- open the reshape pivot editor in the browser const datasetLoaded = async () => { if (await mounted('[data-testid="construction-workspace"]')) return true; const alert = await evaluate(`(()=>{const a=document.querySelector('[role="alert"]');return a?a.textContent:null;})()`); @@ -316,9 +295,6 @@ const main = async () => { await until(datasetLoaded, 'dataset loaded (construction-workspace mounted)', 60000); await realClick(`[data-testid="construction-table-${outputId}"]`); await wait(500); - // F-4 (review): positive base shape for the remove leg — snapshot the seeded grid - // (before any pivot mutation) so restoration is compared, not inferred from the - // absence of the keyword columns. const seededGrid = await until(async () => { const g = await readGrid(); return g && g.headers.length >= 4 && g.rowCount > 0 ? g : null; @@ -337,9 +313,6 @@ const main = async () => { await realClick('[data-testid="construction-rows-settings-trigger"]'); await until(() => mounted('[data-testid="construction-row-intent-reshape"]'), 'reshape intent re-present', 20000); await realClick('[data-testid="construction-row-intent-reshape"]'); - // Once the pivot is applied the variant trigger is gated off (pivotAlternative - // requires both coded/table pivot offers on the pending form, BuilderWorkspace.tsx: - // 3681); the authoritative reopen path is the applied-changes history Edit control. if (!(await mounted('[data-testid="construction-action-table-pivot-rows"]'))) { const editButtons = await evaluate(`(()=>{const s=document.querySelector('[data-testid="construction-row-operation-history"]');if(!s)return[];return [...s.querySelectorAll('[data-testid^="construction-row-edit-"]')].map(b=>b.getAttribute('data-testid'));})()`); assert.ok(editButtons.length > 0, 'applied pivot offers neither the variant trigger nor a history Edit control; reopen would be vacuous'); @@ -350,12 +323,6 @@ const main = async () => { await realClick('[data-testid="construction-action-table-pivot-rows"]'); await until(() => mounted('[data-testid="construction-reshape-pivot"]'), 'pivot editor remounted', 20000); }; - // Live-probe derived (manual-pivot.mjs 2026-11-03): the pivot form needs a GROUP KEY - // checked before discovery auto-fires; the category field must be subject_reference - // (the live dataset's single-value status column yields one category); discovery - // completes ~500 ms after the second select;
children are un-rendered - // until the summary is clicked open (programmatic .click() bypasses layout, so - // checkboxes are engaged via evaluate, not realClick). const configureFieldsAndCategories = async () => { await until(async () => (await evaluate(`document.querySelectorAll('input[aria-label^="Pivot group "]').length`)) >= 3, 'pivot group checkboxes rendered', 20000); const groupToggled = await evaluate(`(()=>{const els=[...document.querySelectorAll('input[aria-label^="Pivot group "]')];const t=els.find(e=>/status/i.test(e.getAttribute('aria-label'))&&!e.disabled);if(!t)return 'none';if(!t.checked)t.click();return t.getAttribute('aria-label');})()`); @@ -374,10 +341,6 @@ const main = async () => { const checkedCount = await evaluate(`(()=>{const s=document.querySelector('[data-testid="construction-reshape-pivot-categories"]');return [...(s?.querySelectorAll('input[type="checkbox"][aria-label^="Include category"],input[type="checkbox"][aria-label^="Keep saved category"]:checked')??[])].length;})()`); assert.ok(selectedCount >= 2, `discovery yielded ${selectedCount} category rows; the duplicate-ID class needs a second category`); assert.ok(checkedCount >= 2, `only ${checkedCount} categories checked after select-all`); - // The numeric value column repeats per cell; the default ERROR duplicate policy - // keeps the proposal in its error state (construction-proposal-needs-repair, - // no apply button), so choose SUM — the policy the probe path uses and the - // only one that yields non-zero pivoted numbers. if (!(await evaluate(`document.querySelector('select[aria-label="Pivot duplicate policy"]')?.value === 'SUM'`))) { await realClick('[data-testid="construction-reshape-pivot-advanced"] summary'); const setPolicy = await evaluate(`(()=>{const s=document.querySelector('select[aria-label="Pivot duplicate policy"]');if(!s)return 'missing';const setter=Object.getOwnPropertyDescriptor(window.HTMLSelectElement.prototype,'value').set;setter.call(s,'SUM');s.dispatchEvent(new Event('change',{bubbles:true}));return s.value;})()`); @@ -387,7 +350,6 @@ const main = async () => { return { categoryLabel: String(catLabel), valueLabel: String(valLabel), groupKey: String(groupToggled), discovered: selectedCount, checked: checkedCount }; }; - // ---- 2/3/4 fields + categories, then keyword names const config = await configureFieldsAndCategories(); record('pk-fields-selected', { category: config.categoryLabel, values: config.valueLabel, groupKey: config.groupKey }); assert.ok(config.checked <= 6, `more category rows than the owned dataset can supply (${config.checked}); the keyword-name assignment below would be misaligned`); @@ -403,9 +365,6 @@ const main = async () => { assert.ok(Array.isArray(got) && got.length === 2 && got.every((e) => e && e.value === names[i]), `keyword name/label inputs ${i} did not hold the typed value: ${JSON.stringify(got)}`); applied.push(got); } - // F-3 (review): re-read only AFTER React re-renders. A same-tick read cannot - // distinguish "the controlled component accepted the value" from "the DOM setter - // wrote it and the next render will revert it". await wait(250); const held = await evaluate(`(()=>{const s=document.querySelector('[data-testid="construction-reshape-pivot"]');const out=[];for(const pre of ['Pivot output name','Pivot output label']){const els=[...s.querySelectorAll('input[aria-label^="'+pre+' "]')].filter(e=>!e.disabled).slice(0,${names.length});out.push(els.map(e=>e.value));}return out;})()`); assert.ok(Array.isArray(held) && held.length === 2 @@ -416,7 +375,6 @@ const main = async () => { return applied; }; - // stable-snapshot BEFORE typing, so the cancel leg can compare bytes const beforeTyping = await api('builder'); const baseline = { draftVersion: beforeTyping.value.draftVersion, draftDigest: beforeTyping.value.draftDigest, tableShape: (beforeTyping.value.workspace.documents ?? [])[0]?.output?.tableShape ?? null }; @@ -424,9 +382,6 @@ const main = async () => { record('pk-keyword-names-applied', { names: keywordNames, inputs: appliedNames }); markPhase('names'); - // ---- 5. cancel keeps every byte - // row-intent sessions render no close header (BuilderWorkspace.tsx:3771 - // `{!rowIntentSession ?
...`); switching intent is the real close path. await realClick('[data-testid="construction-row-intent-records"]'); await until(async () => !(await mounted('[data-testid="construction-reshape-pivot"]')), 'pivot editor closed', 10000); const afterCancel = await api('builder'); @@ -436,7 +391,6 @@ const main = async () => { assert.deepEqual((afterCancel.value.workspace.documents ?? [])[0]?.output?.tableShape ?? null, baseline.tableShape, 'Cancel must not persist a tableShape'); record('pk-cancel-byte-stability', { draftVersion: baseline.draftVersion, draftDigest: baseline.draftDigest.slice(0, 24) }); - // ---- 6. server-side equivalence: same named pivot through the comparison API is AVAILABLE and carries the keyword columns const state3 = (await api('builder')).value; const caps = await api('table-shape-capabilities', { snapshotToken: state3.catalog.snapshotToken, expectedDraftVersion: state3.draftVersion, expectedDraftDigest: state3.draftDigest, outputId, @@ -457,8 +411,6 @@ const main = async () => { }); assertNoInternal(discovery, 'category-discoveries'); assert.equal(discovery.status, 200, `discovery failed (${discovery.status}): ${JSON.stringify(discovery.value).slice(0, 300)}`); - // Every discovered category must be listed (server TABLE_PIVOT_UNLISTED_CATEGORY); - // first two names stay the verbatim keywords, later ones keep the keyword stem. const discoveredCategories = (discovery.value.categories ?? []).slice(0, 8); assert.ok(discoveredCategories.length >= 2, `comparison probe needs >=2 categories, saw ${(discovery.value.categories ?? []).length}`); const probeName = (i) => (i < keywordNames.length ? keywordNames[i] : `${keywordNames[i % keywordNames.length]}${i}`); @@ -493,7 +445,6 @@ const main = async () => { record('pk-comparison-proposal-available', { changedColumns: changedColumns.slice(0, 8), categories: discoveredCategories.map((c) => c.key ?? c.choiceId), status: 'AVAILABLE' }); markPhase('server-probe'); - // ---- 7. UI apply with the keyword names await openPivotEditor(); await configureFieldsAndCategories(); await setNameInputs(keywordNames); @@ -511,9 +462,6 @@ const main = async () => { report.steps.push({ step: 'apply-click', draftBeforeApply, applyReqs, postClickPanel }); const afterApply = await api('builder'); assertNoInternal(afterApply, 'builder-after-apply'); - // Row-intent pivot persistence (ground truth, pkw7 re-derivation): the pivot step - // lands in doc.construction (operation.kind PIVOT, categories by key) and each - // typed category name lands in doc.columns[] as {id: 'pivot-column_*', name, label}. const doc0 = (afterApply.value.workspace.documents ?? [])[0] ?? {}; const collectPivotColumnNames = (document) => { const found = []; @@ -539,9 +487,6 @@ const main = async () => { assert.ok(constructionHasPivot || appliedPivotNames.length > 0, `apply bumped the draft but persisted no pivot representation: doc=${JSON.stringify(doc0?.construction ?? null).slice(0, 500)}`); record('pk-apply-success', { persistedVia: 'construction-step', persistedNames: appliedPivotNames.slice(0, 4), draftVersion: draftAfterApply }); - // ---- 8. the comparison grid renders the keyword columns and real values. - // After apply the mounted preview may still hold the pre-apply receipt; force a - // fresh fetch by re-selecting the table (and, once, by full reload) before asserting. await realClick(`[data-testid="construction-table-${outputId}"]`); await wait(700); for (let attempt = 0; attempt < 2; attempt += 1) { @@ -565,14 +510,10 @@ const main = async () => { await screenshot('step3-keyword-grid'); markPhase('grid'); - // ---- 9. persisted document carries the exact names. The UI row-intent lane persists - // the pivot as a construction step (deep search), the API table-shape channel as - // output.tableShape.pivot (see the disposition note in docs/EXPLORER_BACKLOG.md). const outputs = pivotColumnNames((afterApply.value.workspace.documents ?? [])[0] ?? {}); assert.deepEqual(outputs.slice(0, 2).sort(), [...keywordNames].sort(), `persisted pivot column names drifted from the typed keywords: ${JSON.stringify(outputs)}`); record('pk-document-persistence', { outputs, persistedColumnCount: outputs.length }); - // ---- 10. reload keeps the keyword grid await navigateToLane('post-apply reload'); await until(async () => (await mounted('[data-testid="construction-workspace"]')) && (await readGrid()), 'workspace + grid after reload', 60000); await realClick(`[data-testid="construction-table-${outputId}"]`); @@ -588,7 +529,6 @@ const main = async () => { await screenshot('step4-reload'); markPhase('reload'); - // ---- 11. duplicate keyword name is gated (UI block or typed server 4xx), never 5xx, never silent await openPivotEditor(); await configureFieldsAndCategories(); await setNameInputs([keywordNames[0], keywordNames[0]]); @@ -599,12 +539,6 @@ const main = async () => { if (!applyStateDup.present || applyStateDup.disabled) { dupOutcome = { lane: 'ui-gate', applyState: applyStateDup, statusText }; } else { - // F-2 (review): the server lane must PROVE what it claims. The duplicate names - // are confirmed in the form above (setNameInputs re-reads post-render), so the - // click is a genuine duplicate attempt. Observe the mutating response: no - // commands request ⇒ client itself blocked (UI lane, recorded distinctly); - // a response ⇒ 5xx never admissible, 4xx must carry a typed legitimate code read - // from the response body. A 2xx that persists duplicates fails below. const cmdWindowStart = cmdResponses.length; await realClick('[data-testid="construction-apply-proposal"]'); await wait(700); @@ -645,9 +579,6 @@ const main = async () => { record('pk-duplicate-negative-blocked', dupOutcome); await screenshot('step5-duplicate-gate'); - // ---- 12. valid rename re-enables apply (gate is not one-way sticky, and the - // duplicate-lane absence above was not a permanently-absent apply control). - // The proposal lifecycle is debounced/re-proposed server-side, so settle-wait. await setNameInputs([keywordNames[0], keywordNames[1]]); const sampleProposal = () => evaluate(`(()=>{const b=document.querySelector('[data-testid="construction-apply-proposal"]');const p=document.querySelector('[data-testid="construction-proposal-panel"]');return {apply:b?{present:true,disabled:b.disabled}:{present:false},panel:p?{status:p.getAttribute('data-proposal-status'),stats:[...p.querySelectorAll('[role="status"],[role="alert"]')].map(e=>e.textContent.trim().slice(0,120))}:null};})()`); const recoverState = await until(async () => { @@ -658,9 +589,6 @@ const main = async () => { assert.fail(`valid rename did not re-enable apply (positive control for the duplicate gate): ${JSON.stringify(diag)} vs dup ${JSON.stringify(applyStateDup)} (${String(error?.message).slice(0, 100)})`); }); record('pk-rename-recovers', { from: dupOutcome.lane, to: recoverState }); - // History-edit mode renders the step editor without the intent tab bar, so the - // intent-switch close exists only on the creation flow; step editing closes via - // the proposal panel Cancel (discards the pending edit without a server write). if (await mounted('[data-testid="construction-row-intent-records"]')) { await realClick('[data-testid="construction-row-intent-records"]'); } else { @@ -671,10 +599,6 @@ const main = async () => { const postCloseNames = pivotColumnNames((afterRecoverClose.value.workspace.documents ?? [])[0] ?? {}); assert.ok(!(postCloseNames.filter((n) => n === keywordNames[0]).length >= 2), `the discarded duplicate-name edit leaked into the document: ${JSON.stringify(postCloseNames)}`); - // ---- 13. remove restores the base grid; document loses the pivot. - // A full reload first: removes clicked from a panel that just finished step-editing - // silently no-op (its busy state swallows the click), so the removal runs from - // settled page state, mirroring how a user would return to the changes list. await send('Page.reload', {}); await until(() => mounted('[data-testid="construction-workspace"]'), 'workspace after pre-remove reload', 30000); await realClick(`[data-testid="construction-table-${outputId}"]`); @@ -690,9 +614,6 @@ const main = async () => { for (const selector of removeSelectors) { await realClick(`[data-testid="${selector}"]`); await wait(350); - // history Remove stages a removal proposal; the panel's apply control confirms it - // ('Apply removal'), mirroring the creation apply path. No click is needed if the - // control never appears (direct removal) — draft movement is asserted below. const removalApplyAppeared = await until(async () => evaluate(`(()=>{const b=document.querySelector('[data-testid="construction-apply-proposal"]');return b&&!b.disabled?b.textContent.trim():null;})()`), 'removal confirm control appears', 1500).catch(() => null); if (removalApplyAppeared) { await realClick('[data-testid="construction-apply-proposal"]'); @@ -712,7 +633,6 @@ const main = async () => { const stillPivotNames = pivotColumnNames((removedDoc.value.workspace.documents ?? [])[0] ?? {}); const stillPivotStep = JSON.stringify((removedDoc.value.workspace.documents ?? [])[0]?.construction ?? null).includes('"PIVOT"'); assert.ok(stillPivotNames.length === 0 && !stillPivotStep, `pivot survived remove: names=${JSON.stringify(stillPivotNames)} stepPresent=${stillPivotStep}`); - // F-4 (review): assert the FULL positive base shape, not just keyword absence. assert.ok(Array.isArray(baseGrid.headers) && baseGrid.headers.length >= 4, `post-remove grid has no base shape: ${JSON.stringify(baseGrid.headers)}`); assert.ok(baseGrid.rowCount > 0, 'post-remove grid lost all rows'); assert.deepEqual(baseGrid.headers, seededGrid.headers, 'remove did not restore the exact seeded base grid headers'); @@ -720,7 +640,6 @@ const main = async () => { await screenshot('step6-remove-restored'); markPhase('remove'); - // ---- 14. stamp identity const identityAfter = collectBuildIdentity(); report.buildIdentity.after = identityAfter; assert.equal(identityAfter.runningBinary, identityBefore.runningBinary, `serving binary moved during run: ${identityAfter.runningBinary} != ${identityBefore.runningBinary}`); @@ -729,13 +648,9 @@ const main = async () => { report.buildIdentity.stampVerified = true; markPhase('identity-after'); - // F-1 (review): uncaught browser exceptions must fail the run, per the raw-CDP - // convention this journey declares (four-state :792). A client-side error - // boundary that swallows a pivot bug must not print JOURNEY_OK. if (exceptions.length > 0) { throw new Error('uncaught browser exceptions during the journey: ' + exceptions.join(';').slice(0, 300)); } - // F-5 (review): browser-originated 5xx on authoring endpoints fail closed. if (browser5xx.length > 0) { throw new Error('browser-side 5xx on authoring endpoints: ' + JSON.stringify(browser5xx).slice(0, 400)); } diff --git a/scripts/verify-ui/workflows/verify-explorer-row-source-four-state.mjs b/scripts/verify-ui/workflows/verify-explorer-row-source-four-state.mjs index cca811fc2..b9607ad44 100644 --- a/scripts/verify-ui/workflows/verify-explorer-row-source-four-state.mjs +++ b/scripts/verify-ui/workflows/verify-explorer-row-source-four-state.mjs @@ -1,14 +1,4 @@ #!/usr/bin/env node -// RS03-03 (docs/BUILDER_VERIFICATION.tsv row 96) four-state focused-native journey: the -// healthy / per-field-refusal / inventory-fault / empty-data contract of the signed-source -// table-shape lane, raw-CDP against the owned loom-dev stack, following the -// scripts/verify-ui/workflows raw-CDP convention (see verify-explorer-filter-row-source.mjs). -// Label authority: /tmp/opencode/label-journey-coupling.md (catalog-path literals are the -// contract; construction-lane humanized text is the only flip family). Reason-class -// authority: internal/explorer/lifecycle/construction_pivot_source.go (rowSourceLaneReasons: -// codeStem + _SOURCE_INVENTORY_UNAVAILABLE ; NO_OBSERVED_VALUES_/NO_ELIGIBLE_POPULATED_/ -// NO_ROUTE_ELIGIBLE_/NO_TYPE_ELIGIBLE_/STALE_/ROOT_SOURCE_PROJECTION_UNAVAILABLE). -// Live window execution is pending; --dry-run runs the full static self-validation now. import assert from 'node:assert/strict'; import { spawn, spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; @@ -42,9 +32,6 @@ const expectedChecks = [ 'row96-expand-policy-matrix', ]; -// ROW-SOURCE-08 policy matrix (docs/ROW_OPERATION_NEXT_BATCH.md §ROW-SOURCE-08): five -// parents carrying [red, red, blue] + [green] + empty + missing + null arrays; the contract -// row counts EXCLUDE=4 / PRESERVE_PARENT=7 and the ERROR wire code are asserted in-window. const matrixParents = [ { id: 'rs08m-red', values: ['red', 'red', 'blue'] }, { id: 'rs08m-green', values: ['green'] }, @@ -54,19 +41,8 @@ const matrixParents = [ ]; const matrixExcludeMultiset = [['rs08m-red', 'red'], ['rs08m-red', 'red'], ['rs08m-red', 'blue'], ['rs08m-green', 'green']]; const matrixKeptParentIds = ['rs08m-empty', 'rs08m-missing', 'rs08m-null']; -// Wire-code authority: internal/dataframe/errors/errors.go:77 defines CodeConstructionExpansionEmpty -// = CONSTRUCTION_EXPANSION_EMPTY; internal/dataframe/compiler/render/aql/construction_expand.go:33 -// ASSERTs it against LENGTH(items)==0 under EmptyPolicy ERROR; internal/server/ -// explorer_preview_response.go:65 + server.go:96 map it to the 422 preview failure body. const matrixErrorWireCode = 'CONSTRUCTION_EXPANSION_EMPTY'; -// Honest-absence family: row-choice verdicts (lane reason authority above) plus the -// compiler capability gates emitted for an under-seeded table (dataframe/compiler/lower/ -// recipe_construction.go:2242-2289: NO_NUMERIC_COLUMN/NO_PUBLIC_COLUMNS/ -// INSUFFICIENT_SCALAR_COLUMNS/NO_COMPATIBLE_UNPIVOT_COLUMNS/NO_ARRAY_COLUMNS/ -// NO_PUBLIC_SOURCE_COLUMNS) and related-field identity refusal -// (construction_proposal.go:983/993 NO_ACTIVE_RELATED_RECORD). Inventory-unavailable -// remains disjoint — that split is the S-B guarantee under test. const emptyDataFamily = /^(NO_OBSERVED_VALUES_|NO_ELIGIBLE_POPULATED_|NO_ROUTE_ELIGIBLE_|NO_TYPE_ELIGIBLE_|STALE_|ROOT_SOURCE_PROJECTION_UNAVAILABLE$|NO_NUMERIC_COLUMN$|NO_PUBLIC_COLUMNS$|NO_PUBLIC_SOURCE_COLUMNS$|NO_COMPATIBLE_UNPIVOT_COLUMNS$|INSUFFICIENT_SCALAR_COLUMNS$|NO_ARRAY_COLUMNS$|NO_ACTIVE_RELATED_RECORD$)/; const inventoryClass = /_SOURCE_INVENTORY_UNAVAILABLE$/; @@ -111,13 +87,6 @@ const record = (name, evidence) => { report.checks.push({ name, status: 'passed', evidence }); }; -// ROW-SOURCE-08: this journey's signed-EXPAND check needs a direct-root -// primitive array carried by real data. The generated schema admits none on the -// shipped fixture resources (Patient, Observation, DiagnosticReport, -// DocumentReference, Group, ResearchStudy, Specimen), so the run seeds a -// per-run fixture copy plus three Procedures whose instantiatesUri[] arrays -// (the lane's only honest primitive root array here: 2 + 1 + 0 values) give the -// offer, refusal, and EXCLUDE-policy paths real data to work on. const rs08FixtureDir = (() => { if (options.dryRun || process.env.LOOM_DEV_FIXTURE_DIR) return process.env.LOOM_DEV_FIXTURE_DIR; const dir = mkdtempSync(join(tmpdir(), 'rs96-fixture-')); @@ -140,11 +109,6 @@ const rs08FixtureDir = (() => { })(); if (rs08FixtureDir) process.env.LOOM_DEV_FIXTURE_DIR = rs08FixtureDir; -// ROW-SOURCE-08 policy matrix: the contract parents [red, red, blue] / [green] / empty / -// missing / null must land in a row space the signed-expand-lane check's byte-stable -// four-row Procedure preview never traverses, so the matrix gets its OWN per-run fresh -// fixture project (createFreshVerificationFixture design) seeded from this second copy. -// Same runner machinery as above; nothing on the shared stack is mutated. const rs08MatrixFixtureDir = (() => { if (options.dryRun) return undefined; const dir = mkdtempSync(join(tmpdir(), 'rs96-matrix-')); @@ -168,9 +132,6 @@ const rs08MatrixFixtureDir = (() => { const session = createDevSession(process.env, REPO_ROOT); const collectBuildIdentity = () => { const identity = sharedCollectBuildIdentity(session, { repoRoot: REPO_ROOT }); - // Test-harness escape hatch used ONLY by the lifecycle contract test (staged copy has - // no gitdir by design). Never overrides a resolvable real HEAD: an ambient pin must - // not forge recorded identity or neutralise the mid-run HEAD-move guard. const pin = process.env.LOOM_DEV_HEAD_OVERRIDE; if (pin) { if (!/^[a-f0-9]{40}$/.test(pin)) throw new Error('LOOM_DEV_HEAD_OVERRIDE must be a 40-char commit hash'); @@ -195,9 +156,6 @@ const chrome = spawn(chromePath, ['--headless=new', '--no-sandbox', '--disable-d { stdio: ['ignore', 'ignore', 'pipe'], detached: true }); let chromeErr = ''; chrome.stderr.on('data', (chunk) => { chromeErr = (chromeErr + chunk).slice(-4000); }); -// F3: an async spawn failure (EACCES/EISDIR/EAGAIN) surfaces as an 'error' EVENT, not the -// :98 existsSync assert; record it and let main()'s CDP gate fail, so the tail still runs -// (journey.json written, tree killed, profile removed) instead of crashing the module. let chromeSpawnError = null; chrome.on('error', (error) => { chromeSpawnError = String(error?.message ?? error).slice(0, 300); }); chrome.unref(); @@ -227,12 +185,9 @@ const moduleSend = (method, params = {}, ms = 25000) => withDeadline(client.send // EXTRACT-START:killTree const killTree = async () => { clearTrackedTimers(); - try { client.close(); } catch { /* already closed */ } - try { chrome.stderr?.destroy(); } catch { /* stream already released */ } - // F1: escalation must not hinge on the leader's exit. A helper that ignores SIGTERM - // (or exits late) would otherwise survive while the journey prints JOURNEY_OK. + try { client.close(); } catch { } + try { chrome.stderr?.destroy(); } catch { } const groupMembers = () => { - // zombie entries awaiting reap are terminally dead; only live group members count if (!Number.isInteger(chrome.pid)) return []; try { return readdirSync('/proc').filter((entry) => { @@ -245,24 +200,20 @@ const killTree = async () => { }).map(Number); } catch { return []; } }; - // Some sandboxes deny kill(-pgid) even for own children (EPERM): try the group - // signal, then fall back to leader, then to each enumerated group member. const groupSignal = (signal) => { let groupOk = false; if (Number.isInteger(chrome.pid)) { try { process.kill(-chrome.pid, signal); groupOk = true; } catch { groupOk = false; } } let leaderOk = false; if (!groupOk) { try { leaderOk = chrome.kill(signal); } catch { leaderOk = false; } } - for (const pid of groupMembers()) { try { process.kill(pid, signal); } catch { /* racing exit */ } } + for (const pid of groupMembers()) { try { process.kill(pid, signal); } catch { } } return true; }; if (chrome.exitCode === null && chrome.signalCode === null) { - // await BEFORE sending: a fast exit can land between send and listen and strand the promise const exited = new Promise((done) => chrome.once('exit', done)); if (groupSignal('SIGTERM')) { await withDeadline(exited, 5000, 'chrome SIGTERM exit').catch(() => { groupSignal('SIGKILL'); }); } } - // bounded drain of stragglers (zombies are reaped by this parent, so members are live) for (let attempt = 0; attempt < 10 && groupMembers().length > 0; attempt += 1) { if (attempt === 5) groupSignal('SIGKILL'); await wait(100); @@ -297,9 +248,6 @@ const main = async () => { for (let i = 0; i < 150 && !existsSync(portFile); i += 1) await wait(100); assert.ok(existsSync(portFile), `chrome devtools port not advertised; spawnError=${chromeSpawnError ?? 'none'}; stderr=${chromeErr.slice(-400)}`); const [port] = readFileSync(portFile, 'utf8').trim().split('\n'); - // Bounded like every other transport wait (adversarial review G3: a chrome that - // advertises DevToolsActivePort then wedges previously hung the journey forever with - // zero artifacts; SIGSTOP'd-tree probe: R6b log). const targetResponse = await withDeadline(fetch(`http://127.0.0.1:${port}/json/new?about:blank`, { method: 'PUT' }), 20000, 'devtools /json/new'); const target = await withDeadline(targetResponse.json(), 10000, 'devtools /json/new body'); client = await withDeadline(CDPClient.connect(target.webSocketDebuggerUrl), 30000, 'CDP websocket open'); @@ -315,7 +263,7 @@ const main = async () => { const value = await probe(); if (value) return value; if (Date.now() > deadline) { - try { writeFileSync(join(out, `FAIL-${String(message).replace(/\W+/g, '_')}.html`), await evaluate('document.documentElement.outerHTML')); } catch { /* best effort */ } + try { writeFileSync(join(out, `FAIL-${String(message).replace(/\W+/g, '_')}.html`), await evaluate('document.documentElement.outerHTML')); } catch { } assert.fail(`condition failed: ${message}`); } await wait(80); @@ -335,8 +283,6 @@ const main = async () => { }; const navigateToLane = async (label) => { await send('Page.navigate', { url: laneURL() }); - // Readiness, not wall-clock: settle when chrome has mounted the workspace or the - // app has entered its degraded alert path; bounded so a hung SPA still fails loudly. await until(async () => (await mounted('[data-testid="construction-workspace"]')) || Boolean(await evaluate(`(()=>{const a=document.querySelector('[role="alert"]');return a?a.textContent:null;})()`)), `${label} settled`, 18000); }; @@ -369,10 +315,6 @@ const main = async () => { }); const created = await commands('row96-table', [{ type: 'CREATE_TABLE', title: 'RS96 four-state shape', rootNodeId: root.nodeId }], builder.value); assert.equal(created.status, 200, `CREATE_TABLE failed (${created.status}): ${JSON.stringify(created.value).slice(0, 300)}`); - // Public-column prerequisite (identical seed command to the certified filter journey, - // verify-explorer-filter-row-source.mjs:88-92): without a public output column the - // lane editor legitimately refuses with "filter requires a public output column" and - // the signed-source picker never renders its option/title set. const afterCreate = await api('builder'); const idCandidate = (afterCreate.value?.catalog?.candidates ?? []).find((candidate) => candidate.nodeId === root.nodeId && String(candidate.fieldPath || '').replace(/^root\./, '') === 'id'); @@ -389,8 +331,6 @@ const main = async () => { assert.ok(outputId, 'seeded table has no output; downstream lane asserts would be vacuous'); await navigateToLane('post-seed nav'); - // Dataset-load gate: the app renders role=alert "Couldn't load this dataset." on an - // initial fetch miss; retry the reload (bounded) before the window's asserts start. const datasetLoaded = async () => { if (await mounted('[data-testid="construction-workspace"]')) return true; const alert = await evaluate(`(()=>{const a=document.querySelector('[role="alert"]');return a?a.textContent:null;})()`); @@ -417,11 +357,6 @@ const main = async () => { report.steps.push({ step: 'seed-and-healthy', outputId, stageId }); await screenshot('step1-healthy-lane'); - // The default table view exposes no titled